diff --git a/CMakeLists.txt b/CMakeLists.txt index ae22979..2361184 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -125,6 +125,18 @@ if(EXISTS ${FASTCGIPP_LIB}) link_directories(${FASTCGIPP_LIB}) endif(EXISTS ${FASTCGIPP_LIB}) +# ---------------------------------------------------- +# libpq +# ---------------------------------------------------- +find_library(PQ_LIB NAMES pq REQUIRED) +find_path(PQ_INCLUDE libpq-fe.h HINTS "/usr/include/postgresql") + +if(EXISTS ${PQ_LIB}) + message(STATUS "Found libpq") + include_directories(${PQ_INCLUDE}) + link_directories(${PQ_LIB}) +endif(EXISTS ${PQ_LIB}) + # ---------------------------------------------------- # Concordia # ---------------------------------------------------- diff --git a/TODO.txt b/TODO.txt new file mode 100644 index 0000000..f82bd99 --- /dev/null +++ b/TODO.txt @@ -0,0 +1 @@ +- implement connection pooling with PgBouncer diff --git a/concordia-server/CMakeLists.txt b/concordia-server/CMakeLists.txt index 5ae2f70..6df122f 100644 --- a/concordia-server/CMakeLists.txt +++ b/concordia-server/CMakeLists.txt @@ -4,6 +4,12 @@ target_link_libraries(hello_world fcgi fcgi++) add_executable(echo-cpp echo-cpp.cpp) target_link_libraries(echo-cpp fcgi fcgi++) -add_executable(concordia_server_process concordia_server_process.cpp concordia_server.cpp) -target_link_libraries(concordia_server_process fcgi fcgi++ concordia config++ log4cpp ${Boost_LIBRARIES} divsufsort utf8case) +add_executable(concordia_server_process + concordia_server_process.cpp + concordia_server.cpp + index_controller.cpp + searcher_controller.cpp + json_generator.cpp + ) +target_link_libraries(concordia_server_process fcgi fcgi++ pq concordia config++ log4cpp ${Boost_LIBRARIES} divsufsort utf8case) diff --git a/concordia-server/concordia_server.cpp b/concordia-server/concordia_server.cpp index 4ecec33..477b104 100644 --- a/concordia-server/concordia_server.cpp +++ b/concordia-server/concordia_server.cpp @@ -1,21 +1,23 @@ #include "concordia_server.hpp" #include -#include -#include -#include -#include +#include + +#include "json_generator.hpp" + +#define OPERATION_PARAM "operation" +#define SENTENCE_PARAM "sentence" + +#define ADD_SENTENCE_OP "addSentence" +#define SIMPLE_SEARCH_OP "simpleSearch" +#define CONCORDIA_SEARCH_OP "concordiaSearch" -#include "rapidjson/document.h" -#include "rapidjson/stringbuffer.h" -#include "rapidjson/writer.h" -#include "rapidjson/error/en.h" ConcordiaServer::ConcordiaServer(const std::string & configFilePath) throw(ConcordiaException) { - _concordia = boost::shared_ptr ( - new Concordia(configFilePath)); - + boost::shared_ptr concordia(new Concordia(configFilePath)); + _indexController = boost::shared_ptr (new IndexController(concordia)); + _searcherController = boost::shared_ptr (new SearcherController(concordia)); } ConcordiaServer::~ConcordiaServer() { @@ -29,26 +31,6 @@ string ConcordiaServer::handleRequest(string & requestString) { try { outputString << "Content-type: application/json\r\n\r\n"; - /* - ss << "

Adding content as example:

\n"; - - Example example1(requestString, 0); - Example example2("Ala ma kota", 1); - Example example3("Marysia nie ma kota chyba", 2); - _concordia->addExample(example1); - _concordia->addExample(example2); - _concordia->addExample(example3); - - _concordia->refreshSAfromRAM(); - - ss << "

Searching ma kota:

\n"; - std::vector result = - _concordia->simpleSearch("ma kota"); - BOOST_FOREACH(SubstringOccurence occurence, result) { - ss << "\t\tfound match in sentence number: " - << occurence.getId() << "

"; - } - */ rapidjson::Document d; bool hasError = d.Parse(requestString.c_str()).HasParseError(); @@ -56,31 +38,25 @@ string ConcordiaServer::handleRequest(string & requestString) { stringstream errorstream; errorstream << "json parse error at offset: " << d.GetErrorOffset() << ", description: " << GetParseError_En(d.GetParseError()); - jsonWriter.StartObject(); - jsonWriter.String("status"); - jsonWriter.String("error"); - jsonWriter.String("message"); - jsonWriter.String(errorstream.str().c_str()); - jsonWriter.EndObject(); - } else { - jsonWriter.StartObject(); - jsonWriter.String("status"); - jsonWriter.String("success"); - jsonWriter.String("passedOperation"); - jsonWriter.String(d["operation"].GetString()); - jsonWriter.EndObject(); + JsonGenerator::signalError(jsonWriter, errorstream.str()); + } else { // json parsed + string operation = d[OPERATION_PARAM].GetString(); + string sentence = d[SENTENCE_PARAM].GetString(); + if (operation == ADD_SENTENCE_OP) { + _indexController->addSentence(jsonWriter, sentence); + } else if (operation == SIMPLE_SEARCH_OP) { + _searcherController->simpleSearch(jsonWriter, sentence); + } else if (operation == CONCORDIA_SEARCH_OP) { + _searcherController->concordiaSearch(jsonWriter, sentence); + } else { + JsonGenerator::signalError(jsonWriter, "no such operation"); + } } } catch (ConcordiaException & e) { stringstream errorstream; errorstream << "concordia error: " << e.what(); - jsonWriter.StartObject(); - jsonWriter.String("status"); - jsonWriter.String("error"); - jsonWriter.String("message"); - jsonWriter.String(errorstream.str().c_str()); - jsonWriter.EndObject(); - + JsonGenerator::signalError(jsonWriter, errorstream.str()); } outputString << outputJson.GetString(); diff --git a/concordia-server/concordia_server.hpp b/concordia-server/concordia_server.hpp index ffe8b06..a1efdae 100644 --- a/concordia-server/concordia_server.hpp +++ b/concordia-server/concordia_server.hpp @@ -6,6 +6,13 @@ #include #include +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" +#include "rapidjson/error/en.h" + +#include "index_controller.hpp" +#include "searcher_controller.hpp" using namespace std; @@ -23,8 +30,11 @@ public: string handleRequest(string & requestString); -private: - boost::shared_ptr _concordia; +private: + boost::shared_ptr _indexController; + + boost::shared_ptr _searcherController; + }; #endif diff --git a/concordia-server/db_tests.cpp b/concordia-server/db_tests.cpp new file mode 100644 index 0000000..da8f43e --- /dev/null +++ b/concordia-server/db_tests.cpp @@ -0,0 +1,96 @@ +#include + +static void +exit_nicely(PGconn *conn) +{ + PQfinish(conn); + throw ConcordiaException("Closing connection and shutting down"); +} + + + const char *conninfo; + PGconn *conn; + PGresult *res; + int nFields; + int i, + j; + + conninfo = "dbname=concordia_server user=concordia"; + + /* Make a connection to the database */ + conn = PQconnectdb(conninfo); + + /* Check to see that the backend connection was successfully made */ + if (PQstatus(conn) != CONNECTION_OK) + { + fprintf(stderr, "Connection to database failed: %s", + PQerrorMessage(conn)); + exit_nicely(conn); + } + + /* + * Our test case here involves using a cursor, for which we must be inside + * a transaction block. We could do the whole thing with a single + * PQexec() of "select * from pg_database", but that's too trivial to make + * a good example. + */ + + /* Start a transaction block */ + res = PQexec(conn, "BEGIN"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + fprintf(stderr, "BEGIN command failed: %s", PQerrorMessage(conn)); + PQclear(res); + exit_nicely(conn); + } + + /* + * Should PQclear PGresult whenever it is no longer needed to avoid memory + * leaks + */ + PQclear(res); + + /* + * Fetch rows from pg_database, the system catalog of databases + */ + res = PQexec(conn, "SELECT * FROM language"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + fprintf(stderr, "SELECT failed: %s", PQerrorMessage(conn)); + PQclear(res); + exit_nicely(conn); + } + + jsonWriter.String("attributes"); + jsonWriter.StartArray(); + /* first, print out the attribute names */ + nFields = PQnfields(res); + for (i = 0; i < nFields; i++) + jsonWriter.String(PQfname(res, i)); + + jsonWriter.EndArray(); + + + jsonWriter.String("values"); + jsonWriter.StartArray(); + /* next, print out the rows */ + for (i = 0; i < PQntuples(res); i++) + { + jsonWriter.StartArray(); + for (j = 0; j < nFields; j++) + jsonWriter.String(PQgetvalue(res, i, j)); + jsonWriter.EndArray(); + } + + jsonWriter.EndArray(); + + PQclear(res); + + /* end the transaction */ + res = PQexec(conn, "END"); + PQclear(res); + + /* close the connection to the database and cleanup */ + PQfinish(conn); + + diff --git a/concordia-server/index_controller.cpp b/concordia-server/index_controller.cpp new file mode 100644 index 0000000..76f6ee7 --- /dev/null +++ b/concordia-server/index_controller.cpp @@ -0,0 +1,31 @@ +#include "index_controller.hpp" + +#include "json_generator.hpp" + +IndexController::IndexController(boost::shared_ptr concordia) + throw(ConcordiaException): + _concordia(concordia) { +} + +IndexController::~IndexController() { +} + + +void IndexController::addSentence(rapidjson::Writer & jsonWriter, string & sentence) { + + try { + Example example(sentence, 0); + _concordia->addExample(example); + _concordia->refreshSAfromRAM(); + + jsonWriter.StartObject(); + jsonWriter.String("status"); + jsonWriter.String("success"); + jsonWriter.EndObject(); + } catch (ConcordiaException & e) { + stringstream errorstream; + errorstream << "concordia error: " << e.what(); + JsonGenerator::signalError(jsonWriter, errorstream.str()); + } +} + diff --git a/concordia-server/index_controller.hpp b/concordia-server/index_controller.hpp new file mode 100644 index 0000000..716450d --- /dev/null +++ b/concordia-server/index_controller.hpp @@ -0,0 +1,31 @@ +#ifndef INDEX_CONTROLLER_HDR +#define INDEX_CONTROLLER_HDR + +#include +#include +#include +#include + +#include "rapidjson/writer.h" + + +using namespace std; + +class IndexController { +public: + /*! Constructor. + */ + explicit IndexController(boost::shared_ptr concordia) + throw(ConcordiaException); + /*! Destructor. + */ + virtual ~IndexController(); + + void addSentence(rapidjson::Writer & jsonWriter, string & sentence); + +private: + boost::shared_ptr _concordia; + +}; + +#endif diff --git a/concordia-server/json_generator.cpp b/concordia-server/json_generator.cpp new file mode 100644 index 0000000..3bd7d84 --- /dev/null +++ b/concordia-server/json_generator.cpp @@ -0,0 +1,21 @@ +#include "json_generator.hpp" + + +JsonGenerator::JsonGenerator() { +} + +JsonGenerator::~JsonGenerator() { +} + + +void JsonGenerator::signalError(rapidjson::Writer & jsonWriter, string message) { + jsonWriter.StartObject(); + jsonWriter.String("status"); + jsonWriter.String("error"); + jsonWriter.String("message"); + jsonWriter.String(message.c_str()); + jsonWriter.EndObject(); +} + + + diff --git a/concordia-server/json_generator.hpp b/concordia-server/json_generator.hpp new file mode 100644 index 0000000..db5c142 --- /dev/null +++ b/concordia-server/json_generator.hpp @@ -0,0 +1,26 @@ +#ifndef JSON_GENERATOR_HDR +#define JSON_GENERATOR_HDR + +#include + +#include "rapidjson/writer.h" + + +using namespace std; + +class JsonGenerator { +public: + /*! Constructor. + */ + JsonGenerator(); + /*! Destructor. + */ + virtual ~JsonGenerator(); + + static void signalError(rapidjson::Writer & jsonWriter, string message); + +private: + +}; + +#endif diff --git a/concordia-server/searcher_controller.cpp b/concordia-server/searcher_controller.cpp new file mode 100644 index 0000000..040822f --- /dev/null +++ b/concordia-server/searcher_controller.cpp @@ -0,0 +1,38 @@ +#include "searcher_controller.hpp" + +#include +#include + +SearcherController::SearcherController(boost::shared_ptr concordia) + throw(ConcordiaException): + _concordia(concordia) { +} + +SearcherController::~SearcherController() { +} + + +void SearcherController::simpleSearch(rapidjson::Writer & jsonWriter, string & pattern) { + vector result = _concordia->simpleSearch(pattern); + + jsonWriter.StartObject(); + jsonWriter.String("status"); + jsonWriter.String("success"); + jsonWriter.String("count"); + jsonWriter.Int(result.size()); + jsonWriter.String("firstId"); + jsonWriter.Int(result.at(0).getId()); + jsonWriter.String("firstOffset"); + jsonWriter.Int(result.at(0).getOffset()); + jsonWriter.EndObject(); +} + +void SearcherController::concordiaSearch(rapidjson::Writer & jsonWriter, string & pattern) { + jsonWriter.StartObject(); + jsonWriter.String("status"); + jsonWriter.String("error"); + jsonWriter.String("data"); + jsonWriter.String("concordia searching not yet implemented"); + jsonWriter.EndObject(); +} + diff --git a/concordia-server/searcher_controller.hpp b/concordia-server/searcher_controller.hpp new file mode 100644 index 0000000..9d21c29 --- /dev/null +++ b/concordia-server/searcher_controller.hpp @@ -0,0 +1,33 @@ +#ifndef SEARCHER_CONTROLLER_HDR +#define SEARCHER_CONTROLLER_HDR + +#include +#include +#include +#include + +#include "rapidjson/writer.h" + + +using namespace std; + +class SearcherController { +public: + /*! Constructor. + */ + explicit SearcherController(boost::shared_ptr concordia) + throw(ConcordiaException); + /*! Destructor. + */ + virtual ~SearcherController(); + + void simpleSearch(rapidjson::Writer & jsonWriter, string & pattern); + + void concordiaSearch(rapidjson::Writer & jsonWriter, string & pattern); + +private: + boost::shared_ptr _concordia; + +}; + +#endif diff --git a/db/concordiaDb.sh b/db/concordiaDb.sh new file mode 100755 index 0000000..511ef18 --- /dev/null +++ b/db/concordiaDb.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +sudo -u concordia psql concordia_server diff --git a/db/concordia_server.sql b/db/concordia_server.sql new file mode 100644 index 0000000..d563cbc --- /dev/null +++ b/db/concordia_server.sql @@ -0,0 +1,23 @@ +DROP TABLE IF EXISTS tm; +CREATE TABLE tm ( + id SERIAL PRIMARY KEY, + source_lang_id integer, + target_lang_id integer, + name varchar(40) +); + +DROP TABLE IF EXISTS language; +CREATE TABLE language ( + id SERIAL PRIMARY KEY, + code varchar(10), + name varchar(30) +); + +DROP TABLE IF EXISTS unit; +CREATE TABLE unit ( + id SERIAL PRIMARY KEY, + tm_id integer, + source_segment text, + target_segment text +); + diff --git a/db/init/00_lang.sql b/db/init/00_lang.sql new file mode 100644 index 0000000..4b79655 --- /dev/null +++ b/db/init/00_lang.sql @@ -0,0 +1,6 @@ +INSERT INTO language(code, name) VALUES ('pl', 'Polish'); +INSERT INTO language(code, name) VALUES ('en', 'English'); +INSERT INTO language(code, name) VALUES ('de', 'German'); +INSERT INTO language(code, name) VALUES ('es', 'Spanish'); +INSERT INTO language(code, name) VALUES ('it', 'Italian'); + diff --git a/db/recreateDb.sh b/db/recreateDb.sh new file mode 100755 index 0000000..300959f --- /dev/null +++ b/db/recreateDb.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +echo "Recreating database schema..." +sudo -u concordia psql concordia_server -f concordia_server.sql + +echo "Inserting initial data..." +for initFile in `ls init/*` +do + echo "Init file:" $initFile + sudo -u concordia psql concordia_server -f $initFile +done + +echo "Concordia server database recreation complete!" diff --git a/scripts/cmake_stubs/stop.sh.in b/scripts/cmake_stubs/stop.sh.in index ab4fba8..4a93eff 100755 --- a/scripts/cmake_stubs/stop.sh.in +++ b/scripts/cmake_stubs/stop.sh.in @@ -1,16 +1,14 @@ #!/bin/sh -set -e - pidFile="@SCRIPTS_PATH@/concordia-server.PID" if [ -e $pidFile ] then - kill `cat $pidFile` + pid=`cat $pidFile` rm $pidFile + kill $pid echo "concordia-server stopped" else echo "no PID file found at:" $pidFile "- is concordia-server running?" >&2 - exit 1 fi diff --git a/scripts/concordia-server.PID b/scripts/concordia-server.PID index 35ac9f9..129a03b 100644 --- a/scripts/concordia-server.PID +++ b/scripts/concordia-server.PID @@ -1 +1 @@ -6201 +14823 diff --git a/scripts/stop.sh b/scripts/stop.sh index 87c6370..5cd2d94 100755 --- a/scripts/stop.sh +++ b/scripts/stop.sh @@ -1,16 +1,14 @@ #!/bin/sh -set -e - pidFile="/home/rafalj/projects/concordia-server/scripts/concordia-server.PID" if [ -e $pidFile ] then - kill `cat $pidFile` + pid=`cat $pidFile` rm $pidFile + kill $pid echo "concordia-server stopped" else echo "no PID file found at:" $pidFile "- is concordia-server running?" >&2 - exit 1 fi diff --git a/tests/testCurl.sh b/tests/testCurl.sh index 4d36e06..975c4a0 100755 --- a/tests/testCurl.sh +++ b/tests/testCurl.sh @@ -1,4 +1,5 @@ #!/bin/sh -curl -H "Content-Type: application/json" -X POST -d '{"operation":"zażółć gęślą jaźń"}' http://localhost +#curl -H "Content-Type: application/json" -X POST -d '{"operation":"addSentence", "sentence":"zupełnie nowe zdanie"}' http://localhost +curl -H "Content-Type: application/json" -X POST -d '{"operation":"simpleSearch", "sentence":"zupełnie snowe"}' http://localhost