diff --git a/cat/publish.py b/cat/publish.py index 67b08d7..392a974 100755 --- a/cat/publish.py +++ b/cat/publish.py @@ -43,6 +43,12 @@ with open('concordia_search.php_pattern', 'r') as search_pattern_file, open(root line = re.sub('@concordia_port@', concordia_port, line) search_file.write(line) +with open('tm_info.php_pattern', 'r') as tm_info_pattern_file, open(root_dir+'/tm_info.php', 'w') as tm_info_file: + for line in tm_info_pattern_file: + line = re.sub('@concordia_host@', concordia_host, line) + line = re.sub('@concordia_port@', concordia_port, line) + tm_info_file.write(line) + versions_dir = 'versions_enabled' diff --git a/cat/tm_info.php_pattern b/cat/tm_info.php_pattern new file mode 100644 index 0000000..dccb18d --- /dev/null +++ b/cat/tm_info.php_pattern @@ -0,0 +1,25 @@ + "getTmsInfo"); + +// use key 'http' even if you send the request to https://... +$options = array( + 'http' => array( + 'header' => "Content-type: application/x-www-form-urlencoded\r\n", + 'method' => 'POST', + 'content' => json_encode($request), + ), +); +$context = stream_context_create($options); +$response = file_get_contents($url, false, $context); + + +$data = json_decode($response); + +foreach ($data->tms as $tm) { + echo $tm->id."\t".$tm->name." (".$tm->sourceLanguageCode."-> ".$tm->targetLanguageCode.")\n"; +} + + +?> diff --git a/cat/versions_available/jrc_enpl.cfg b/cat/versions_available/jrc_enpl.cfg index c1ccff0..5113e81 100644 --- a/cat/versions_available/jrc_enpl.cfg +++ b/cat/versions_available/jrc_enpl.cfg @@ -1,8 +1,8 @@ dir@#@jrc_enpl concordia_host@#@concordia.vm.wmi.amu.edu.pl concordia_port@#@8800 -tmid@#@2 -desc@#@Welcome to the interactive Concordia demo. The system finds the longest matches of the pattern sentence in its translation memory. This translation memory is 200 000 sentences taken from English-Polish corpus of European Law. Please enter an English sentence in the field below and press Enter (or use the search button). This instance of Concordia works best with law sentences, but is very likely to output some results for any English sentence. You can also use predefined samples, simply use the link "show/hide samples" and apply one of the sample sentences. After the search, click on the highlighted fragments to see their context. +tmid@#@1 +desc@#@Welcome to the interactive Concordia demo. The system finds the longest matches of the pattern sentence in its translation memory. This translation memory is over 1.5M sentences taken from English-Polish corpus of European Law (Europarl + JRC-Acquis). Please enter an English sentence in the field below and press Enter (or use the search button). This instance of Concordia works best with law sentences, but is very likely to output some results for any English sentence. You can also use predefined samples, simply use the link "show/hide samples" and apply one of the sample sentences. After the search, click on the highlighted fragments to see their context. enjoy@#@Enjoy your work with the system! prompt@#@Enter search pattern (English sentence): suggestion@#@Every ship in the European Union must have a crew of 50 or more workers. diff --git a/cat/versions_available/jrc_plen.cfg b/cat/versions_available/jrc_plen.cfg index 9396af3..724257c 100644 --- a/cat/versions_available/jrc_plen.cfg +++ b/cat/versions_available/jrc_plen.cfg @@ -1,7 +1,7 @@ dir@#@jrc_plen concordia_host@#@concordia.vm.wmi.amu.edu.pl concordia_port@#@8800 -tmid@#@1 +tmid@#@2 desc@#@Witamy w interaktywnym demo systemu Concordia. System znajduje najdłuższe fragmenty zdania wejściowego w pamięci tłumaczeń. W pamięci tej znajduje się 200 000 zdań z polsko-angielskiego korpusu ustawodawstwa Unii Europejskiej JRC-Acquis. Proszę wpisać polskie zdanie w poniższe pole i nacisnąć Enter (albo użyć przycisku "search"). Ta wersja Concordii działa najlepiej ze zdaniami prawniczymi, ale jest bardzo prawdopodobne, że znalezione zostaną choćby krótkie fragmenty dowolnego polskiego zdania. Aby zapoznać się z systemem możesz użyć wcześniej przygotowanych przykładów - po prostu kliknij link "apply" przy wybranym przykładzie. Po wyszukaniu, kliknij na wybrany podświetlony fragment, aby zobaczyć jego kontekst. enjoy@#@Życzymy udanej pracy z systemem! prompt@#@Wprowadź zdanie (po polsku): diff --git a/concordia-server/concordia_server.cpp b/concordia-server/concordia_server.cpp index a5689d7..fa539ee 100644 --- a/concordia-server/concordia_server.cpp +++ b/concordia-server/concordia_server.cpp @@ -12,6 +12,7 @@ #include "json_generator.hpp" #include "config.hpp" #include "logger.hpp" +#include "tm.hpp" #include "rapidjson/rapidjson.h" #include #include @@ -118,6 +119,30 @@ std::string ConcordiaServer::handleRequest(std::string & requestString) { } } _indexController->addAlignedLemmatizedSentences(jsonWriter, sourceSentences, targetSentences, alignmentStrings, tmId); + } else if (operation == GET_TMS_INFO_PARAM) { + std::vector tms = _tmDAO.getTms(); + + jsonWriter.StartObject(); + jsonWriter.String("status"); + jsonWriter.String("success"); + jsonWriter.String("tms"); + jsonWriter.StartArray(); + BOOST_FOREACH(Tm & tm, tms) { + jsonWriter.StartObject(); + jsonWriter.String("id"); + jsonWriter.Int(tm.getId()); + jsonWriter.String("name"); + jsonWriter.String(tm.getName().c_str()); + jsonWriter.String("sourceLanguageCode"); + jsonWriter.String(tm.getSourceLanguageCode().c_str()); + jsonWriter.String("targetLanguageCode"); + jsonWriter.String(tm.getTargetLanguageCode().c_str()); + jsonWriter.EndObject(); + } + jsonWriter.EndArray(); + jsonWriter.EndObject(); + + } else if (operation == "lemmatize") { std::string sentence = _getStringParameter(d, "sentence"); std::string languageCode = _getStringParameter(d, "languageCode"); diff --git a/concordia-server/config.hpp.in b/concordia-server/config.hpp.in index 571d18a..c287822 100644 --- a/concordia-server/config.hpp.in +++ b/concordia-server/config.hpp.in @@ -23,6 +23,7 @@ #define TARGET_LANG_PARAM "targetLangId" #define NAME_PARAM "name" #define INTERVALS_PARAM "intervals" +#define GET_TMS_INFO_PARAM "getTmsInfo" #define ADD_SENTENCE_OP "addSentence" #define ADD_SENTENCES_OP "addSentences" diff --git a/concordia-server/searcher_controller.cpp b/concordia-server/searcher_controller.cpp index dd7eb03..10343f2 100644 --- a/concordia-server/searcher_controller.cpp +++ b/concordia-server/searcher_controller.cpp @@ -115,8 +115,9 @@ void SearcherController::concordiaSearch(rapidjson::Writer::iterator it = _concordiasMap->find(tmId); if (it != _concordiasMap->end()) { - pattern = _lemmatizerFacade->lemmatizeIfNeeded(pattern, tmId); - CompleteConcordiaSearchResult result = _unitDAO.getConcordiaResult(it->second->concordiaSearch(pattern)); + std::string lemmatizedPattern = _lemmatizerFacade->lemmatizeIfNeeded(pattern, tmId); + TokenizedSentence originalPattern = it->second->tokenize(pattern, true, false); + CompleteConcordiaSearchResult result = _unitDAO.getConcordiaResult(it->second->concordiaSearch(lemmatizedPattern), originalPattern); jsonWriter.StartObject(); jsonWriter.String("status"); diff --git a/concordia-server/tm.cpp b/concordia-server/tm.cpp new file mode 100644 index 0000000..4b2158e --- /dev/null +++ b/concordia-server/tm.cpp @@ -0,0 +1,13 @@ +#include "tm.hpp" + +Tm::Tm(const int id, + const std::string & name, + const std::string & sourceLanguageCode, + const std::string & targetLanguageCode) : + _id(id),_name(name), + _sourceLanguageCode(sourceLanguageCode), + _targetLanguageCode(targetLanguageCode) { +} + +Tm::~Tm() { +} diff --git a/concordia-server/tm.hpp b/concordia-server/tm.hpp new file mode 100644 index 0000000..e895a45 --- /dev/null +++ b/concordia-server/tm.hpp @@ -0,0 +1,46 @@ +#ifndef TM_HDR +#define TM_HDR + +#include +#include + +class Tm { +public: + /*! Constructor. + */ + Tm(const int id, + const std::string & name, + const std::string & sourceLanguageCode, + const std::string & targetLanguageCode); + /*! Destructor. + */ + virtual ~Tm(); + + int getId() const { + return _id; + } + + const std::string & getName() const { + return _name; + } + + const std::string & getSourceLanguageCode() const { + return _sourceLanguageCode; + } + + const std::string & getTargetLanguageCode() const { + return _targetLanguageCode; + } + + +private: + int _id; + + std::string _name; + + std::string _sourceLanguageCode; + + std::string _targetLanguageCode; +}; + +#endif diff --git a/concordia-server/tm_dao.cpp b/concordia-server/tm_dao.cpp index 4b2e2da..711aa86 100644 --- a/concordia-server/tm_dao.cpp +++ b/concordia-server/tm_dao.cpp @@ -32,6 +32,30 @@ std::vector TmDAO::getTmIds() { return result; } +std::vector TmDAO::getTms() { + std::vector result; + DBconnection connection; + connection.startTransaction(); + std::string query = "select tm.id, tm.name, src_lang.code as src_code, trg_lang.code as trg_code from tm inner join language as src_lang on src_lang.id = tm.source_lang_id inner join language as trg_lang on trg_lang.id = tm.target_lang_id;"; + PGresult * dbResult = connection.execute(query); + for (int i=0;i #include "db_connection.hpp" +#include "tm.hpp" class TmDAO { public: @@ -23,6 +24,8 @@ public: std::vector getTmIds(); + std::vector getTms(); + std::pair getTmInfo(int tmId); private: diff --git a/concordia-server/unit_dao.cpp b/concordia-server/unit_dao.cpp index 9fbaa5b..4ab849b 100644 --- a/concordia-server/unit_dao.cpp +++ b/concordia-server/unit_dao.cpp @@ -80,6 +80,15 @@ CompleteConcordiaSearchResult UnitDAO::getConcordiaResult(boost::shared_ptr rawConcordiaResult, TokenizedSentence originalPattern) { + CompleteConcordiaSearchResult result(rawConcordiaResult->getBestOverlayScore()); + _getResultsFromFragments(result.getBestOverlay(), + rawConcordiaResult->getBestOverlay(), + originalPattern); + return result; +} + + void UnitDAO::_getResultsFromFragments( std::vector & results, const std::vector & fragments, @@ -212,7 +221,7 @@ int UnitDAO::_addAlignedUnit ( // sentence, because giza can truncate the sentence. In this case, we have to // truncate the source sentence too. - + } std::string query = "INSERT INTO unit(source_segment, target_segment, tm_id, source_tokens, target_tokens) values($1::text,$2::text,$3::integer,$4,$5) RETURNING id"; diff --git a/concordia-server/unit_dao.hpp b/concordia-server/unit_dao.hpp index 7159320..814b1aa 100644 --- a/concordia-server/unit_dao.hpp +++ b/concordia-server/unit_dao.hpp @@ -45,6 +45,8 @@ public: CompleteConcordiaSearchResult getConcordiaResult(boost::shared_ptr rawConcordiaResult); + CompleteConcordiaSearchResult getConcordiaResult(boost::shared_ptr rawConcordiaResult, TokenizedSentence originalPattern); + private: void _getResultsFromFragments(std::vector & results, const std::vector & fragments, diff --git a/tests/addLemmatizedTM.sh b/tests/addLemmatizedTM.sh index 5fd445b..b533764 100755 --- a/tests/addLemmatizedTM.sh +++ b/tests/addLemmatizedTM.sh @@ -1,7 +1,7 @@ #!/bin/sh -CORPUS_NAME="europarljrc" +CORPUS_NAME="setimes_enhr" SRC_LANG_ID=2 -TRG_LANG_ID=1 +TRG_LANG_ID=6 ./addAlignedLemmatizedTM.py $CORPUS_NAME ../mgiza-aligner/corpora/$CORPUS_NAME/src.tok $SRC_LANG_ID ../mgiza-aligner/corpora/$CORPUS_NAME/trg.tok $TRG_LANG_ID ../mgiza-aligner/corpora/$CORPUS_NAME/aligned.txt diff --git a/tests/getTmsInfo.py b/tests/getTmsInfo.py new file mode 100755 index 0000000..18fa572 --- /dev/null +++ b/tests/getTmsInfo.py @@ -0,0 +1,23 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import json +import urllib2 +import sys +import time +import host + +address = 'http://'+host.concordia_host +if len(host.concordia_port) > 0: + address += ':'+host.concordia_port + + +data = { + 'operation': 'getTmsInfo' +} + +req = urllib2.Request(address) +req.add_header('Content-Type', 'application/json') +response = json.loads(urllib2.urlopen(req, json.dumps(data)).read()) + +print response