simple communication

This commit is contained in:
rjawor 2015-07-31 14:11:13 +02:00
parent 61817e9b0c
commit 596a440835
20 changed files with 388 additions and 65 deletions

View File

@ -125,6 +125,18 @@ if(EXISTS ${FASTCGIPP_LIB})
link_directories(${FASTCGIPP_LIB}) link_directories(${FASTCGIPP_LIB})
endif(EXISTS ${FASTCGIPP_LIB}) endif(EXISTS ${FASTCGIPP_LIB})
# ----------------------------------------------------
# libpq
# ----------------------------------------------------
find_library(PQ_LIB NAMES pq REQUIRED)
find_path(PQ_INCLUDE libpq-fe.h HINTS "/usr/include/postgresql")
if(EXISTS ${PQ_LIB})
message(STATUS "Found libpq")
include_directories(${PQ_INCLUDE})
link_directories(${PQ_LIB})
endif(EXISTS ${PQ_LIB})
# ---------------------------------------------------- # ----------------------------------------------------
# Concordia # Concordia
# ---------------------------------------------------- # ----------------------------------------------------

1
TODO.txt Normal file
View File

@ -0,0 +1 @@
- implement connection pooling with PgBouncer

View File

@ -4,6 +4,12 @@ target_link_libraries(hello_world fcgi fcgi++)
add_executable(echo-cpp echo-cpp.cpp) add_executable(echo-cpp echo-cpp.cpp)
target_link_libraries(echo-cpp fcgi fcgi++) target_link_libraries(echo-cpp fcgi fcgi++)
add_executable(concordia_server_process concordia_server_process.cpp concordia_server.cpp) add_executable(concordia_server_process
target_link_libraries(concordia_server_process fcgi fcgi++ concordia config++ log4cpp ${Boost_LIBRARIES} divsufsort utf8case) concordia_server_process.cpp
concordia_server.cpp
index_controller.cpp
searcher_controller.cpp
json_generator.cpp
)
target_link_libraries(concordia_server_process fcgi fcgi++ pq concordia config++ log4cpp ${Boost_LIBRARIES} divsufsort utf8case)

View File

@ -1,21 +1,23 @@
#include "concordia_server.hpp" #include "concordia_server.hpp"
#include <sstream> #include <sstream>
#include <concordia/example.hpp> #include <string>
#include <concordia/substring_occurence.hpp>
#include <boost/ptr_container/ptr_vector.hpp> #include "json_generator.hpp"
#include <boost/foreach.hpp>
#define OPERATION_PARAM "operation"
#define SENTENCE_PARAM "sentence"
#define ADD_SENTENCE_OP "addSentence"
#define SIMPLE_SEARCH_OP "simpleSearch"
#define CONCORDIA_SEARCH_OP "concordiaSearch"
#include "rapidjson/document.h"
#include "rapidjson/stringbuffer.h"
#include "rapidjson/writer.h"
#include "rapidjson/error/en.h"
ConcordiaServer::ConcordiaServer(const std::string & configFilePath) ConcordiaServer::ConcordiaServer(const std::string & configFilePath)
throw(ConcordiaException) { throw(ConcordiaException) {
_concordia = boost::shared_ptr<Concordia> ( boost::shared_ptr<Concordia> concordia(new Concordia(configFilePath));
new Concordia(configFilePath)); _indexController = boost::shared_ptr<IndexController> (new IndexController(concordia));
_searcherController = boost::shared_ptr<SearcherController> (new SearcherController(concordia));
} }
ConcordiaServer::~ConcordiaServer() { ConcordiaServer::~ConcordiaServer() {
@ -29,26 +31,6 @@ string ConcordiaServer::handleRequest(string & requestString) {
try { try {
outputString << "Content-type: application/json\r\n\r\n"; outputString << "Content-type: application/json\r\n\r\n";
/*
ss << " <h1>Adding content as example:</h1>\n";
Example example1(requestString, 0);
Example example2("Ala ma kota", 1);
Example example3("Marysia nie ma kota chyba", 2);
_concordia->addExample(example1);
_concordia->addExample(example2);
_concordia->addExample(example3);
_concordia->refreshSAfromRAM();
ss << " <h1>Searching ma kota:</h1>\n";
std::vector<SubstringOccurence> result =
_concordia->simpleSearch("ma kota");
BOOST_FOREACH(SubstringOccurence occurence, result) {
ss << "\t\tfound match in sentence number: "
<< occurence.getId() << "<br/><br/>";
}
*/
rapidjson::Document d; rapidjson::Document d;
bool hasError = d.Parse(requestString.c_str()).HasParseError(); bool hasError = d.Parse(requestString.c_str()).HasParseError();
@ -56,31 +38,25 @@ string ConcordiaServer::handleRequest(string & requestString) {
stringstream errorstream; stringstream errorstream;
errorstream << "json parse error at offset: " << d.GetErrorOffset() << errorstream << "json parse error at offset: " << d.GetErrorOffset() <<
", description: " << GetParseError_En(d.GetParseError()); ", description: " << GetParseError_En(d.GetParseError());
jsonWriter.StartObject(); JsonGenerator::signalError(jsonWriter, errorstream.str());
jsonWriter.String("status"); } else { // json parsed
jsonWriter.String("error"); string operation = d[OPERATION_PARAM].GetString();
jsonWriter.String("message"); string sentence = d[SENTENCE_PARAM].GetString();
jsonWriter.String(errorstream.str().c_str()); if (operation == ADD_SENTENCE_OP) {
jsonWriter.EndObject(); _indexController->addSentence(jsonWriter, sentence);
} else { } else if (operation == SIMPLE_SEARCH_OP) {
jsonWriter.StartObject(); _searcherController->simpleSearch(jsonWriter, sentence);
jsonWriter.String("status"); } else if (operation == CONCORDIA_SEARCH_OP) {
jsonWriter.String("success"); _searcherController->concordiaSearch(jsonWriter, sentence);
jsonWriter.String("passedOperation"); } else {
jsonWriter.String(d["operation"].GetString()); JsonGenerator::signalError(jsonWriter, "no such operation");
jsonWriter.EndObject(); }
} }
} catch (ConcordiaException & e) { } catch (ConcordiaException & e) {
stringstream errorstream; stringstream errorstream;
errorstream << "concordia error: " << e.what(); errorstream << "concordia error: " << e.what();
jsonWriter.StartObject(); JsonGenerator::signalError(jsonWriter, errorstream.str());
jsonWriter.String("status");
jsonWriter.String("error");
jsonWriter.String("message");
jsonWriter.String(errorstream.str().c_str());
jsonWriter.EndObject();
} }
outputString << outputJson.GetString(); outputString << outputJson.GetString();

View File

@ -6,6 +6,13 @@
#include <boost/shared_ptr.hpp> #include <boost/shared_ptr.hpp>
#include <concordia/concordia.hpp> #include <concordia/concordia.hpp>
#include "rapidjson/document.h"
#include "rapidjson/stringbuffer.h"
#include "rapidjson/writer.h"
#include "rapidjson/error/en.h"
#include "index_controller.hpp"
#include "searcher_controller.hpp"
using namespace std; using namespace std;
@ -23,8 +30,11 @@ public:
string handleRequest(string & requestString); string handleRequest(string & requestString);
private: private:
boost::shared_ptr<Concordia> _concordia; boost::shared_ptr<IndexController> _indexController;
boost::shared_ptr<SearcherController> _searcherController;
}; };
#endif #endif

View File

@ -0,0 +1,96 @@
#include <libpq-fe.h>
static void
exit_nicely(PGconn *conn)
{
PQfinish(conn);
throw ConcordiaException("Closing connection and shutting down");
}
const char *conninfo;
PGconn *conn;
PGresult *res;
int nFields;
int i,
j;
conninfo = "dbname=concordia_server user=concordia";
/* Make a connection to the database */
conn = PQconnectdb(conninfo);
/* Check to see that the backend connection was successfully made */
if (PQstatus(conn) != CONNECTION_OK)
{
fprintf(stderr, "Connection to database failed: %s",
PQerrorMessage(conn));
exit_nicely(conn);
}
/*
* Our test case here involves using a cursor, for which we must be inside
* a transaction block. We could do the whole thing with a single
* PQexec() of "select * from pg_database", but that's too trivial to make
* a good example.
*/
/* Start a transaction block */
res = PQexec(conn, "BEGIN");
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "BEGIN command failed: %s", PQerrorMessage(conn));
PQclear(res);
exit_nicely(conn);
}
/*
* Should PQclear PGresult whenever it is no longer needed to avoid memory
* leaks
*/
PQclear(res);
/*
* Fetch rows from pg_database, the system catalog of databases
*/
res = PQexec(conn, "SELECT * FROM language");
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
fprintf(stderr, "SELECT failed: %s", PQerrorMessage(conn));
PQclear(res);
exit_nicely(conn);
}
jsonWriter.String("attributes");
jsonWriter.StartArray();
/* first, print out the attribute names */
nFields = PQnfields(res);
for (i = 0; i < nFields; i++)
jsonWriter.String(PQfname(res, i));
jsonWriter.EndArray();
jsonWriter.String("values");
jsonWriter.StartArray();
/* next, print out the rows */
for (i = 0; i < PQntuples(res); i++)
{
jsonWriter.StartArray();
for (j = 0; j < nFields; j++)
jsonWriter.String(PQgetvalue(res, i, j));
jsonWriter.EndArray();
}
jsonWriter.EndArray();
PQclear(res);
/* end the transaction */
res = PQexec(conn, "END");
PQclear(res);
/* close the connection to the database and cleanup */
PQfinish(conn);

View File

@ -0,0 +1,31 @@
#include "index_controller.hpp"
#include "json_generator.hpp"
IndexController::IndexController(boost::shared_ptr<Concordia> concordia)
throw(ConcordiaException):
_concordia(concordia) {
}
IndexController::~IndexController() {
}
void IndexController::addSentence(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter, string & sentence) {
try {
Example example(sentence, 0);
_concordia->addExample(example);
_concordia->refreshSAfromRAM();
jsonWriter.StartObject();
jsonWriter.String("status");
jsonWriter.String("success");
jsonWriter.EndObject();
} catch (ConcordiaException & e) {
stringstream errorstream;
errorstream << "concordia error: " << e.what();
JsonGenerator::signalError(jsonWriter, errorstream.str());
}
}

View File

@ -0,0 +1,31 @@
#ifndef INDEX_CONTROLLER_HDR
#define INDEX_CONTROLLER_HDR
#include <string>
#include <boost/shared_ptr.hpp>
#include <concordia/concordia.hpp>
#include <concordia/concordia_exception.hpp>
#include "rapidjson/writer.h"
using namespace std;
class IndexController {
public:
/*! Constructor.
*/
explicit IndexController(boost::shared_ptr<Concordia> concordia)
throw(ConcordiaException);
/*! Destructor.
*/
virtual ~IndexController();
void addSentence(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter, string & sentence);
private:
boost::shared_ptr<Concordia> _concordia;
};
#endif

View File

@ -0,0 +1,21 @@
#include "json_generator.hpp"
JsonGenerator::JsonGenerator() {
}
JsonGenerator::~JsonGenerator() {
}
void JsonGenerator::signalError(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter, string message) {
jsonWriter.StartObject();
jsonWriter.String("status");
jsonWriter.String("error");
jsonWriter.String("message");
jsonWriter.String(message.c_str());
jsonWriter.EndObject();
}

View File

@ -0,0 +1,26 @@
#ifndef JSON_GENERATOR_HDR
#define JSON_GENERATOR_HDR
#include <string>
#include "rapidjson/writer.h"
using namespace std;
class JsonGenerator {
public:
/*! Constructor.
*/
JsonGenerator();
/*! Destructor.
*/
virtual ~JsonGenerator();
static void signalError(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter, string message);
private:
};
#endif

View File

@ -0,0 +1,38 @@
#include "searcher_controller.hpp"
#include <concordia/substring_occurence.hpp>
#include <vector>
SearcherController::SearcherController(boost::shared_ptr<Concordia> concordia)
throw(ConcordiaException):
_concordia(concordia) {
}
SearcherController::~SearcherController() {
}
void SearcherController::simpleSearch(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter, string & pattern) {
vector<SubstringOccurence> result = _concordia->simpleSearch(pattern);
jsonWriter.StartObject();
jsonWriter.String("status");
jsonWriter.String("success");
jsonWriter.String("count");
jsonWriter.Int(result.size());
jsonWriter.String("firstId");
jsonWriter.Int(result.at(0).getId());
jsonWriter.String("firstOffset");
jsonWriter.Int(result.at(0).getOffset());
jsonWriter.EndObject();
}
void SearcherController::concordiaSearch(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter, string & pattern) {
jsonWriter.StartObject();
jsonWriter.String("status");
jsonWriter.String("error");
jsonWriter.String("data");
jsonWriter.String("concordia searching not yet implemented");
jsonWriter.EndObject();
}

View File

@ -0,0 +1,33 @@
#ifndef SEARCHER_CONTROLLER_HDR
#define SEARCHER_CONTROLLER_HDR
#include <string>
#include <boost/shared_ptr.hpp>
#include <concordia/concordia.hpp>
#include <concordia/concordia_exception.hpp>
#include "rapidjson/writer.h"
using namespace std;
class SearcherController {
public:
/*! Constructor.
*/
explicit SearcherController(boost::shared_ptr<Concordia> concordia)
throw(ConcordiaException);
/*! Destructor.
*/
virtual ~SearcherController();
void simpleSearch(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter, string & pattern);
void concordiaSearch(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter, string & pattern);
private:
boost::shared_ptr<Concordia> _concordia;
};
#endif

3
db/concordiaDb.sh Executable file
View File

@ -0,0 +1,3 @@
#!/bin/sh
sudo -u concordia psql concordia_server

23
db/concordia_server.sql Normal file
View File

@ -0,0 +1,23 @@
DROP TABLE IF EXISTS tm;
CREATE TABLE tm (
id SERIAL PRIMARY KEY,
source_lang_id integer,
target_lang_id integer,
name varchar(40)
);
DROP TABLE IF EXISTS language;
CREATE TABLE language (
id SERIAL PRIMARY KEY,
code varchar(10),
name varchar(30)
);
DROP TABLE IF EXISTS unit;
CREATE TABLE unit (
id SERIAL PRIMARY KEY,
tm_id integer,
source_segment text,
target_segment text
);

6
db/init/00_lang.sql Normal file
View File

@ -0,0 +1,6 @@
INSERT INTO language(code, name) VALUES ('pl', 'Polish');
INSERT INTO language(code, name) VALUES ('en', 'English');
INSERT INTO language(code, name) VALUES ('de', 'German');
INSERT INTO language(code, name) VALUES ('es', 'Spanish');
INSERT INTO language(code, name) VALUES ('it', 'Italian');

13
db/recreateDb.sh Executable file
View File

@ -0,0 +1,13 @@
#!/bin/sh
echo "Recreating database schema..."
sudo -u concordia psql concordia_server -f concordia_server.sql
echo "Inserting initial data..."
for initFile in `ls init/*`
do
echo "Init file:" $initFile
sudo -u concordia psql concordia_server -f $initFile
done
echo "Concordia server database recreation complete!"

View File

@ -1,16 +1,14 @@
#!/bin/sh #!/bin/sh
set -e
pidFile="@SCRIPTS_PATH@/concordia-server.PID" pidFile="@SCRIPTS_PATH@/concordia-server.PID"
if [ -e $pidFile ] if [ -e $pidFile ]
then then
kill `cat $pidFile` pid=`cat $pidFile`
rm $pidFile rm $pidFile
kill $pid
echo "concordia-server stopped" echo "concordia-server stopped"
else else
echo "no PID file found at:" $pidFile "- is concordia-server running?" >&2 echo "no PID file found at:" $pidFile "- is concordia-server running?" >&2
exit 1
fi fi

View File

@ -1 +1 @@
6201 14823

View File

@ -1,16 +1,14 @@
#!/bin/sh #!/bin/sh
set -e
pidFile="/home/rafalj/projects/concordia-server/scripts/concordia-server.PID" pidFile="/home/rafalj/projects/concordia-server/scripts/concordia-server.PID"
if [ -e $pidFile ] if [ -e $pidFile ]
then then
kill `cat $pidFile` pid=`cat $pidFile`
rm $pidFile rm $pidFile
kill $pid
echo "concordia-server stopped" echo "concordia-server stopped"
else else
echo "no PID file found at:" $pidFile "- is concordia-server running?" >&2 echo "no PID file found at:" $pidFile "- is concordia-server running?" >&2
exit 1
fi fi

View File

@ -1,4 +1,5 @@
#!/bin/sh #!/bin/sh
curl -H "Content-Type: application/json" -X POST -d '{"operation":"zażółć gęślą jaźń"}' http://localhost #curl -H "Content-Type: application/json" -X POST -d '{"operation":"addSentence", "sentence":"zupełnie nowe zdanie"}' http://localhost
curl -H "Content-Type: application/json" -X POST -d '{"operation":"simpleSearch", "sentence":"zupełnie snowe"}' http://localhost