simple communication

CMakeLists.txt

@@ -125,6 +125,18 @@ if(EXISTS ${FASTCGIPP_LIB})
   link_directories(${FASTCGIPP_LIB})
 endif(EXISTS ${FASTCGIPP_LIB})
 
+# ----------------------------------------------------
+# libpq
+# ----------------------------------------------------
+find_library(PQ_LIB NAMES pq REQUIRED)
+find_path(PQ_INCLUDE libpq-fe.h HINTS "/usr/include/postgresql")
+
+if(EXISTS ${PQ_LIB})
+  message(STATUS "Found libpq")
+  include_directories(${PQ_INCLUDE})
+  link_directories(${PQ_LIB})
+endif(EXISTS ${PQ_LIB})
+
 # ----------------------------------------------------
 # Concordia
 # ----------------------------------------------------

TODO.txt

@@ -0,0 +1 @@
+- implement connection pooling with  PgBouncer

concordia-server/CMakeLists.txt

@@ -4,6 +4,12 @@ target_link_libraries(hello_world fcgi fcgi++)
 add_executable(echo-cpp echo-cpp.cpp)
 target_link_libraries(echo-cpp fcgi fcgi++)
 
-add_executable(concordia_server_process concordia_server_process.cpp concordia_server.cpp)
+add_executable(concordia_server_process
-target_link_libraries(concordia_server_process fcgi fcgi++ concordia config++ log4cpp ${Boost_LIBRARIES} divsufsort utf8case)
+                  concordia_server_process.cpp
+                  concordia_server.cpp
+                  index_controller.cpp
+                  searcher_controller.cpp
+                  json_generator.cpp
+              )
+target_link_libraries(concordia_server_process fcgi fcgi++ pq concordia config++ log4cpp ${Boost_LIBRARIES} divsufsort utf8case)
 

concordia-server/concordia_server.cpp

@@ -1,21 +1,23 @@
 #include "concordia_server.hpp"
 
 #include <sstream>
-#include <concordia/example.hpp>
+#include <string>
-#include <concordia/substring_occurence.hpp>
+
-#include <boost/ptr_container/ptr_vector.hpp>
+#include "json_generator.hpp"
-#include <boost/foreach.hpp>
+
+#define OPERATION_PARAM "operation"
+#define SENTENCE_PARAM "sentence"
+
+#define ADD_SENTENCE_OP "addSentence"
+#define SIMPLE_SEARCH_OP "simpleSearch"
+#define CONCORDIA_SEARCH_OP "concordiaSearch"
 
-#include "rapidjson/document.h"
-#include "rapidjson/stringbuffer.h"
-#include "rapidjson/writer.h"
-#include "rapidjson/error/en.h"
 
 ConcordiaServer::ConcordiaServer(const std::string & configFilePath)
                                          throw(ConcordiaException) {
-    _concordia = boost::shared_ptr<Concordia> (
+    boost::shared_ptr<Concordia> concordia(new Concordia(configFilePath));
-                            new Concordia(configFilePath));
+    _indexController = boost::shared_ptr<IndexController> (new IndexController(concordia));
-
+    _searcherController = boost::shared_ptr<SearcherController> (new SearcherController(concordia));
 }
 
 ConcordiaServer::~ConcordiaServer() {
@@ -29,26 +31,6 @@ string ConcordiaServer::handleRequest(string & requestString) {
 
     try {
         outputString << "Content-type: application/json\r\n\r\n";
-        /*
-        ss << "    <h1>Adding content as example:</h1>\n";
-
-        Example example1(requestString, 0);
-        Example example2("Ala ma kota", 1);
-        Example example3("Marysia nie ma kota chyba", 2);
-        _concordia->addExample(example1);
-        _concordia->addExample(example2);
-        _concordia->addExample(example3);
-
-        _concordia->refreshSAfromRAM();
-
-        ss << "    <h1>Searching ma kota:</h1>\n";
-        std::vector<SubstringOccurence> result =
-                                 _concordia->simpleSearch("ma kota");
-        BOOST_FOREACH(SubstringOccurence occurence, result) {
-            ss << "\t\tfound match in sentence number: "
-                      << occurence.getId() << "<br/><br/>";
-        } 
-        */
         rapidjson::Document d;
         bool hasError = d.Parse(requestString.c_str()).HasParseError();
 
@@ -56,31 +38,25 @@ string ConcordiaServer::handleRequest(string & requestString) {
             stringstream errorstream;
             errorstream << "json parse error at offset: " << d.GetErrorOffset() <<
                            ", description: " <<  GetParseError_En(d.GetParseError());
-            jsonWriter.StartObject();
+            JsonGenerator::signalError(jsonWriter, errorstream.str());
-            jsonWriter.String("status");
+        } else { // json parsed
-            jsonWriter.String("error");
+            string operation = d[OPERATION_PARAM].GetString();
-            jsonWriter.String("message");
+            string sentence = d[SENTENCE_PARAM].GetString();
-            jsonWriter.String(errorstream.str().c_str());
+            if (operation == ADD_SENTENCE_OP) {
-            jsonWriter.EndObject();            
+                _indexController->addSentence(jsonWriter, sentence);
+            } else if (operation == SIMPLE_SEARCH_OP) {
+                _searcherController->simpleSearch(jsonWriter, sentence);
+            } else if (operation == CONCORDIA_SEARCH_OP) {
+                _searcherController->concordiaSearch(jsonWriter, sentence);         
             } else {
-            jsonWriter.StartObject();
+                JsonGenerator::signalError(jsonWriter, "no such operation");            
-            jsonWriter.String("status");
+            }
-            jsonWriter.String("success");
-            jsonWriter.String("passedOperation");
-            jsonWriter.String(d["operation"].GetString());
-            jsonWriter.EndObject();            
         }
              
     } catch (ConcordiaException & e) {
         stringstream errorstream;
         errorstream << "concordia error: " << e.what();
-        jsonWriter.StartObject();
+        JsonGenerator::signalError(jsonWriter, errorstream.str());        
-        jsonWriter.String("status");
-        jsonWriter.String("error");
-        jsonWriter.String("message");
-        jsonWriter.String(errorstream.str().c_str());
-        jsonWriter.EndObject();            
-        
     }
 
     outputString << outputJson.GetString();

concordia-server/concordia_server.hpp

@@ -6,6 +6,13 @@
 #include <boost/shared_ptr.hpp>
 #include <concordia/concordia.hpp>
 
+#include "rapidjson/document.h"
+#include "rapidjson/stringbuffer.h"
+#include "rapidjson/writer.h"
+#include "rapidjson/error/en.h"
+
+#include "index_controller.hpp"
+#include "searcher_controller.hpp"
 
 using namespace std;
 
@@ -24,7 +31,10 @@ public:
     string handleRequest(string & requestString);
 
 private:    
-    boost::shared_ptr<Concordia> _concordia;
+    boost::shared_ptr<IndexController> _indexController;
+
+    boost::shared_ptr<SearcherController> _searcherController;
+
 };
 
 #endif

concordia-server/db_tests.cpp

@@ -0,0 +1,96 @@
+#include <libpq-fe.h>
+
+static void
+exit_nicely(PGconn *conn)
+{
+    PQfinish(conn);
+    throw ConcordiaException("Closing connection and shutting down");
+}
+
+
+            const char *conninfo;
+            PGconn     *conn;
+            PGresult   *res;
+            int         nFields;
+            int         i,
+                        j;
+            
+            conninfo = "dbname=concordia_server user=concordia";
+
+            /* Make a connection to the database */
+            conn = PQconnectdb(conninfo);
+
+            /* Check to see that the backend connection was successfully made */
+            if (PQstatus(conn) != CONNECTION_OK)
+            {
+                fprintf(stderr, "Connection to database failed: %s",
+                        PQerrorMessage(conn));
+                exit_nicely(conn);
+            }
+
+            /*
+             * Our test case here involves using a cursor, for which we must be inside
+             * a transaction block.  We could do the whole thing with a single
+             * PQexec() of "select * from pg_database", but that's too trivial to make
+             * a good example.
+             */
+
+            /* Start a transaction block */
+            res = PQexec(conn, "BEGIN");
+            if (PQresultStatus(res) != PGRES_COMMAND_OK)
+            {
+                fprintf(stderr, "BEGIN command failed: %s", PQerrorMessage(conn));
+                PQclear(res);
+                exit_nicely(conn);
+            }
+
+            /*
+             * Should PQclear PGresult whenever it is no longer needed to avoid memory
+             * leaks
+             */
+            PQclear(res);
+
+            /*
+             * Fetch rows from pg_database, the system catalog of databases
+             */
+            res = PQexec(conn, "SELECT * FROM language");
+            if (PQresultStatus(res) != PGRES_TUPLES_OK)
+            {
+                fprintf(stderr, "SELECT failed: %s", PQerrorMessage(conn));
+                PQclear(res);
+                exit_nicely(conn);
+            }
+
+            jsonWriter.String("attributes");
+            jsonWriter.StartArray();
+            /* first, print out the attribute names */
+            nFields = PQnfields(res);
+            for (i = 0; i < nFields; i++)
+                jsonWriter.String(PQfname(res, i));
+
+            jsonWriter.EndArray();
+
+
+            jsonWriter.String("values");
+            jsonWriter.StartArray();
+            /* next, print out the rows */
+            for (i = 0; i < PQntuples(res); i++)
+            {
+                jsonWriter.StartArray();
+                for (j = 0; j < nFields; j++)
+                    jsonWriter.String(PQgetvalue(res, i, j));
+                jsonWriter.EndArray();
+            }
+
+            jsonWriter.EndArray();
+
+            PQclear(res);
+
+            /* end the transaction */
+            res = PQexec(conn, "END");
+            PQclear(res);
+
+            /* close the connection to the database and cleanup */
+            PQfinish(conn);
+            
+

concordia-server/index_controller.cpp

@@ -0,0 +1,31 @@
+#include "index_controller.hpp"
+
+#include "json_generator.hpp"
+
+IndexController::IndexController(boost::shared_ptr<Concordia> concordia)
+                                         throw(ConcordiaException):
+                                         _concordia(concordia) {
+}
+
+IndexController::~IndexController() {
+}
+
+
+void IndexController::addSentence(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter, string & sentence) {
+
+    try {
+        Example example(sentence, 0);
+        _concordia->addExample(example);
+        _concordia->refreshSAfromRAM();
+        
+        jsonWriter.StartObject();
+        jsonWriter.String("status");
+        jsonWriter.String("success");
+        jsonWriter.EndObject();
+    } catch (ConcordiaException & e) {
+        stringstream errorstream;
+        errorstream << "concordia error: " << e.what();
+        JsonGenerator::signalError(jsonWriter, errorstream.str());        
+    }
+}
+

concordia-server/index_controller.hpp

@@ -0,0 +1,31 @@
+#ifndef INDEX_CONTROLLER_HDR
+#define INDEX_CONTROLLER_HDR
+
+#include <string>
+#include <boost/shared_ptr.hpp>
+#include <concordia/concordia.hpp>
+#include <concordia/concordia_exception.hpp>
+
+#include "rapidjson/writer.h"
+
+
+using namespace std;
+
+class IndexController {
+public:
+    /*! Constructor.
+    */
+    explicit IndexController(boost::shared_ptr<Concordia> concordia)
+                                            throw(ConcordiaException);
+    /*! Destructor.
+    */
+    virtual ~IndexController();
+
+    void addSentence(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter, string & sentence);
+
+private:
+    boost::shared_ptr<Concordia> _concordia;
+
+};
+
+#endif

concordia-server/json_generator.cpp

@@ -0,0 +1,21 @@
+#include "json_generator.hpp"
+
+
+JsonGenerator::JsonGenerator() {
+}
+
+JsonGenerator::~JsonGenerator() {
+}
+
+
+void JsonGenerator::signalError(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter, string  message) {
+    jsonWriter.StartObject();
+    jsonWriter.String("status");
+    jsonWriter.String("error");
+    jsonWriter.String("message");
+    jsonWriter.String(message.c_str());
+    jsonWriter.EndObject();
+}
+    
+
+

concordia-server/json_generator.hpp

@@ -0,0 +1,26 @@
+#ifndef JSON_GENERATOR_HDR
+#define JSON_GENERATOR_HDR
+
+#include <string>
+
+#include "rapidjson/writer.h"
+
+
+using namespace std;
+
+class JsonGenerator {
+public:
+    /*! Constructor.
+    */
+    JsonGenerator();
+    /*! Destructor.
+    */
+    virtual ~JsonGenerator();
+
+    static void signalError(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter, string  message);
+
+private:
+
+};
+
+#endif

concordia-server/searcher_controller.cpp

@@ -0,0 +1,38 @@
+#include "searcher_controller.hpp"
+
+#include <concordia/substring_occurence.hpp>
+#include <vector>
+
+SearcherController::SearcherController(boost::shared_ptr<Concordia> concordia)
+                                         throw(ConcordiaException):
+                                         _concordia(concordia) {
+}
+
+SearcherController::~SearcherController() {
+}
+
+
+void SearcherController::simpleSearch(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter, string & pattern) {
+    vector<SubstringOccurence> result = _concordia->simpleSearch(pattern);
+
+    jsonWriter.StartObject();
+    jsonWriter.String("status");
+    jsonWriter.String("success");
+    jsonWriter.String("count");
+    jsonWriter.Int(result.size());
+    jsonWriter.String("firstId");
+    jsonWriter.Int(result.at(0).getId());
+    jsonWriter.String("firstOffset");
+    jsonWriter.Int(result.at(0).getOffset());
+    jsonWriter.EndObject();            
+}
+
+void SearcherController::concordiaSearch(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter, string & pattern) {
+    jsonWriter.StartObject();
+    jsonWriter.String("status");
+    jsonWriter.String("error");
+    jsonWriter.String("data");
+    jsonWriter.String("concordia searching not yet implemented");
+    jsonWriter.EndObject();            
+}
+

concordia-server/searcher_controller.hpp

@@ -0,0 +1,33 @@
+#ifndef SEARCHER_CONTROLLER_HDR
+#define SEARCHER_CONTROLLER_HDR
+
+#include <string>
+#include <boost/shared_ptr.hpp>
+#include <concordia/concordia.hpp>
+#include <concordia/concordia_exception.hpp>
+
+#include "rapidjson/writer.h"
+
+
+using namespace std;
+
+class SearcherController {
+public:
+    /*! Constructor.
+    */
+    explicit SearcherController(boost::shared_ptr<Concordia> concordia)
+                                            throw(ConcordiaException);
+    /*! Destructor.
+    */
+    virtual ~SearcherController();
+
+    void simpleSearch(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter, string & pattern);
+
+    void concordiaSearch(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter, string & pattern);
+
+private:
+    boost::shared_ptr<Concordia> _concordia;
+
+};
+
+#endif

db/concordiaDb.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+sudo -u concordia psql concordia_server

db/concordia_server.sql

@@ -0,0 +1,23 @@
+DROP TABLE IF EXISTS tm;
+CREATE TABLE tm (
+    id SERIAL PRIMARY KEY,
+    source_lang_id integer,
+    target_lang_id integer,
+    name varchar(40)
+);
+
+DROP TABLE IF EXISTS language;
+CREATE TABLE language (
+    id SERIAL PRIMARY KEY,
+    code varchar(10),
+    name varchar(30)
+);
+
+DROP TABLE IF EXISTS unit;
+CREATE TABLE unit (
+    id SERIAL PRIMARY KEY,
+    tm_id integer,
+    source_segment text,
+    target_segment text
+);
+

db/init/00_lang.sql

@@ -0,0 +1,6 @@
+INSERT INTO language(code, name) VALUES ('pl', 'Polish');
+INSERT INTO language(code, name) VALUES ('en', 'English');
+INSERT INTO language(code, name) VALUES ('de', 'German');
+INSERT INTO language(code, name) VALUES ('es', 'Spanish');
+INSERT INTO language(code, name) VALUES ('it', 'Italian');
+

db/recreateDb.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+echo "Recreating database schema..."
+sudo -u concordia psql concordia_server -f concordia_server.sql
+
+echo "Inserting initial data..."
+for initFile in `ls init/*`
+do
+    echo "Init file:" $initFile
+    sudo -u concordia psql concordia_server -f $initFile
+done
+
+echo "Concordia server database recreation complete!"

scripts/cmake_stubs/stop.sh.in

@@ -1,16 +1,14 @@
 #!/bin/sh
 
-set -e
-
 pidFile="@SCRIPTS_PATH@/concordia-server.PID"
 
 if [ -e $pidFile ]
 then
-    kill `cat $pidFile`
+    pid=`cat $pidFile`
     rm $pidFile
+    kill $pid
     echo "concordia-server stopped"
 else
     echo "no PID file found at:" $pidFile "- is concordia-server running?" >&2
-    exit 1
 fi
 

scripts/concordia-server.PID

@@ -1 +1 @@
-6201
+14823

scripts/stop.sh

@@ -1,16 +1,14 @@
 #!/bin/sh
 
-set -e
-
 pidFile="/home/rafalj/projects/concordia-server/scripts/concordia-server.PID"
 
 if [ -e $pidFile ]
 then
-    kill `cat $pidFile`
+    pid=`cat $pidFile`
     rm $pidFile
+    kill $pid
     echo "concordia-server stopped"
 else
     echo "no PID file found at:" $pidFile "- is concordia-server running?" >&2
-    exit 1
 fi
 

tests/testCurl.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
 
-curl -H "Content-Type: application/json" -X POST -d '{"operation":"zażółć gęślą jaźń"}' http://localhost
+#curl -H "Content-Type: application/json" -X POST -d '{"operation":"addSentence", "sentence":"zupełnie nowe zdanie"}' http://localhost
+curl -H "Content-Type: application/json" -X POST -d '{"operation":"simpleSearch", "sentence":"zupełnie snowe"}' http://localhost