diff --git a/concordia-server/concordia_server.cpp b/concordia-server/concordia_server.cpp index 13a2c94..b11eac9 100644 --- a/concordia-server/concordia_server.cpp +++ b/concordia-server/concordia_server.cpp @@ -312,6 +312,25 @@ std::string ConcordiaServer::handleRequest(std::string & requestString) { jsonWriter.String("newSourceId"); jsonWriter.Int(newId); jsonWriter.EndObject(); + } else if (operation == ADD_SOURCES_OP) { + + std::vector externalIds; + std::vector names; + std::vector links; + + const rapidjson::Value & sourcesArray = d[SOURCES_PARAM]; + for (rapidjson::SizeType i = 0; i < sourcesArray.Size(); i++) { + externalIds.push_back(sourcesArray[i][0].GetInt()); + names.push_back(sourcesArray[i][1].GetString()); + links.push_back(sourcesArray[i][2].GetString()); + } + + _sourceDAO.addSources(externalIds, names, links); + + jsonWriter.StartObject(); + jsonWriter.String("status"); + jsonWriter.String("success"); + jsonWriter.EndObject(); } else { JsonGenerator::signalError(jsonWriter, "no such operation: " + operation); } diff --git a/concordia-server/config.hpp.in b/concordia-server/config.hpp.in index 88bb67d..6599031 100644 --- a/concordia-server/config.hpp.in +++ b/concordia-server/config.hpp.in @@ -30,6 +30,7 @@ #define INTERVALS_PARAM "intervals" #define EXTERNAL_ID_PARAM "externalId" #define LINK_PARAM "link" +#define SOURCES_PARAM "sources" #define ADD_SENTENCE_OP "addSentence" #define ADD_SENTENCES_OP "addSentences" @@ -49,5 +50,6 @@ #define CONCORDIA_PHRASE_SEARCH_OP "concordiaPhraseSearch" #define ADD_TM_OP "addTm" #define ADD_SOURCE_OP "addSource" +#define ADD_SOURCES_OP "addSources" #define LEMMATIZER_DELIMITER "@#@" diff --git a/concordia-server/source_dao.cpp b/concordia-server/source_dao.cpp index a40fb84..4ab51ca 100644 --- a/concordia-server/source_dao.cpp +++ b/concordia-server/source_dao.cpp @@ -34,4 +34,38 @@ int SourceDAO::addSource(const int externalId, const std::string & name, const s return newId; -} \ No newline at end of file +} + +void SourceDAO::addSources(const std::vector & externalIds, + const std::vector & names, + const std::vector & links) { + + + + DBconnection connection; + connection.startTransaction(); + + for(int i=0;i params; + params.push_back(new IntParam(externalId)); + params.push_back(new StringParam(name)); + params.push_back(new StringParam(link)); + + PGresult * result = connection.execute(query, params); + connection.clearResult(result); + BOOST_FOREACH (QueryParam * param, params) { + delete param; + } + + } + + connection.endTransaction(); + + + +} diff --git a/concordia-server/source_dao.hpp b/concordia-server/source_dao.hpp index 09cd89e..c7ca509 100644 --- a/concordia-server/source_dao.hpp +++ b/concordia-server/source_dao.hpp @@ -17,6 +17,10 @@ public: int addSource(const int externalId, const std::string & name, const std::string & link); + void addSources(const std::vector & externalIds, + const std::vector & names, + const std::vector & links); + private: }; diff --git a/tests/addFastAlignedTM.py b/tests/addFastAlignedTM.py index 213a8b2..64fc9bb 100755 --- a/tests/addFastAlignedTM.py +++ b/tests/addFastAlignedTM.py @@ -24,7 +24,7 @@ def file_len(fname): def add_examples(examplesData): req = urllib2.Request(address) req.add_header('Content-Type', 'application/json') - response = json.loads(urllib2.urlopen(req, json.dumps(examplesData), 3600).read()) + response = json.loads(urllib2.urlopen(req, json.dumps(examplesData), timeout = 3600).read()) print(response) if response['status'] == 'error': raise Exception(response['message']) @@ -64,7 +64,7 @@ data = { req = urllib2.Request(address) req.add_header('Content-Type', 'application/json') -response = json.loads(urllib2.urlopen(req, json.dumps(data), 3600).read()) +response = json.loads(urllib2.urlopen(req, json.dumps(data), timeout = 3600).read()) print(response) tmId = int(response['newTmId']) print "Added new tm: %d" % tmId @@ -111,7 +111,7 @@ data = { } req = urllib2.Request(address) req.add_header('Content-Type', 'application/json') -urllib2.urlopen(req, json.dumps(data), 3600).read() +urllib2.urlopen(req, json.dumps(data), timeout = 3600).read() end = time.time() print "Index regeneration complete. The operation took %.4f s" % (end - start) diff --git a/tests/addSources.py b/tests/addSources.py index 004caad..193e727 100755 --- a/tests/addSources.py +++ b/tests/addSources.py @@ -7,6 +7,20 @@ import sys import time import host +BUFFER_SIZE = 500 + + +def addSources(sources_buffer): + data = { + 'operation': 'addSources', + 'sources':sources + } + + req = urllib2.Request(address) + req.add_header('Content-Type', 'application/json') + urllib2.urlopen(req, json.dumps(data)) + + address = 'http://'+host.concordia_host if len(host.concordia_port) > 0: address += ':'+host.concordia_port @@ -14,23 +28,20 @@ if len(host.concordia_port) > 0: with open(sys.argv[1]) as sources_file: counter = 0 + sources_buffer = [] for line in sources_file: counter += 1 - idStr, link, name = line.rstrip().split('\t') + sources_buffer.append(line.rstrip().split('\t')) + if len(sources_buffer) == BUFFER_SIZE: + addSources(sources_buffer) + sources_buffer = [] + print("Added %d sources" % counter) + + +if len(sources_buffer) > 0: + addSources(sources_buffer) + +print("Added all %d sources" % counter) - data = { - 'operation': 'addSource', - 'externalId':int(idStr), - 'name':name, - 'link':link - } - - req = urllib2.Request(address) - req.add_header('Content-Type', 'application/json') - urllib2.urlopen(req, json.dumps(data)).read() - - if counter % 1000 == 0: - print("Done %d sources" % counter) -