rjawor/concordia-server
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

occurrence refactoring

Browse Source
This commit is contained in:
Rafał Jaworski 2019-01-22 14:32:25 +01:00
parent e4b4a77de3
commit 66a51192a7
17 changed files with 115 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
cat/concordia_gate.php_pattern
View File

@ -2,7 +2,14 @@
$url = 'http://@concordia_host@:@concordia_port@'; $url = 'http://@concordia_host@:@concordia_port@';
$intervalsArray = array(); $intervalsArray = array();
$data = array("operation" => $_POST["operation"],"tmId" => intval($_POST["tmId"]),"limit" => intval($_POST["limit"]),"offset" => intval($_POST["offset"]),"pattern" => $_POST["pattern"],"intervals" => $intervalsArray); $data = array("operation" => $_POST["operation"],"tmId" => intval($_POST["tmId"]),"pattern" => $_POST["pattern"],"intervals" => $intervalsArray);
if(array_key_exists("limit", $_POST)) {
$data["limit"] = intval($_POST["limit"]);
}
if(array_key_exists("offset", $_POST)) {
$data["offset"] = intval($_POST["offset"]);
}
// use key 'http' even if you send the request to https://... // use key 'http' even if you send the request to https://...
$options = array( $options = array(

2
cat/index.html_pattern
View File

@ -47,7 +47,7 @@
</div> </div>
<input type="hidden" id="current-offset" value="0" /> <input type="hidden" id="current-offset" value="0" />
<div id="occurences"> <div id="occurrences">
</div> </div>
</div> </div>
</body> </body>

12
cat/js/cat.js
View File

@ -64,8 +64,8 @@ function presentFullSearchResults(data) {
result += '<button onclick=prevPage()>previous page</button>&nbsp;'; result += '<button onclick=prevPage()>previous page</button>&nbsp;';
result += '<button onclick=nextPage()>next page</button><br>'; result += '<button onclick=nextPage()>next page</button><br>';
for (j=0;j<data['result']['occurences'].length;j++) { for (j=0;j<data['result']['occurrences'].length;j++) {
var occurence = data['result']['occurences'][j]; var occurence = data['result']['occurrences'][j];
result += '<table class="example"><tr><td>'; result += '<table class="example"><tr><td>';
// source segment // source segment
@ -91,7 +91,7 @@ function presentFullSearchResults(data) {
result += '</td></tr></table>'; result += '</td></tr></table>';
} }
$('#occurences').html(result); $('#occurrences').html(result);
} }
@ -168,7 +168,7 @@ function phraseSearchHandle(tmid, intervals) {
} }
function renderResult(data) { function renderResult(data) {
$('#occurences').html(''); $('#occurrences').html('');
var res = ''; var res = '';
var disablePhraseSearch = true; var disablePhraseSearch = true;
@ -222,8 +222,8 @@ function htmlEncode(value){
function renderFragment(fragment, number) { function renderFragment(fragment, number) {
var result = '<div style="display:none" id="fragment'+number+'" class="fragmentDetails">'; var result = '<div style="display:none" id="fragment'+number+'" class="fragmentDetails">';
for (j=0;j<fragment['occurences'].length;j++) { for (j=0;j<fragment['occurrences'].length;j++) {
var occurence = fragment['occurences'][j]; var occurence = fragment['occurrences'][j];
result += '<table class="example"><tr><td>'; result += '<table class="example"><tr><td>';
// source segment // source segment

2
cat/versions_available/stocznia_plen.cfg
View File

@ -1,7 +1,7 @@
dir@#@stocznia_plen dir@#@stocznia_plen
concordia_host@#@concordia.poleng concordia_host@#@concordia.poleng
concordia_port@#@8800 concordia_port@#@8800
tmid@#@2 tmid@#@1
desc@#@Witamy w interaktywnym demo systemu Concordia. System znajduje najdłuższe fragmenty zdania wejściowego w pamięci tłumaczeń. Proszę wpisać polskie zdanie w poniższe pole i nacisnąć Enter (albo użyć przycisku "search"). Aby zapoznać się z systemem możesz użyć wcześniej przygotowanych przykładów - po prostu kliknij link "apply" przy wybranym przykładzie. Po wyszukaniu, kliknij na wybrany podświetlony fragment, aby zobaczyć jego kontekst. desc@#@Witamy w interaktywnym demo systemu Concordia. System znajduje najdłuższe fragmenty zdania wejściowego w pamięci tłumaczeń. Proszę wpisać polskie zdanie w poniższe pole i nacisnąć Enter (albo użyć przycisku "search"). Aby zapoznać się z systemem możesz użyć wcześniej przygotowanych przykładów - po prostu kliknij link "apply" przy wybranym przykładzie. Po wyszukaniu, kliknij na wybrany podświetlony fragment, aby zobaczyć jego kontekst.
enjoy@#@Życzymy udanej pracy z systemem! enjoy@#@Życzymy udanej pracy z systemem!
prompt@#@Wprowadź zdanie (po polsku): prompt@#@Wprowadź zdanie (po polsku):

1
cat/versions_enabled/stocznia_plen.cfg Symbolic link
View File

@ -0,0 +1 @@
../versions_available/stocznia_plen.cfg

8
concordia-server/example_occurence.cpp → concordia-server/example_occurrence.cpp
View File

@ -1,6 +1,6 @@
#include "example_occurence.hpp" #include "example_occurrence.hpp"
ExampleOccurence::ExampleOccurence( ExampleOccurrence::ExampleOccurrence(
const int id, const int id,
const int matchedExampleStart, const int matchedExampleStart,
const int matchedExampleEnd, const int matchedExampleEnd,
@ -13,9 +13,9 @@ ExampleOccurence::ExampleOccurence(
_targetSegment(targetSegment) { _targetSegment(targetSegment) {
} }
ExampleOccurence::~ExampleOccurence() { ExampleOccurrence::~ExampleOccurrence() {
} }
void ExampleOccurence::addMatchedTargetFragment(const std::pair<int,int> & targetFragment) { void ExampleOccurrence::addMatchedTargetFragment(const std::pair<int,int> & targetFragment) {
_targetFragments.push_back(targetFragment); _targetFragments.push_back(targetFragment);
} }

10
concordia-server/example_occurence.hpp → concordia-server/example_occurrence.hpp
View File

@ -1,14 +1,14 @@
#ifndef EXAMPLE_OCCURENCE_HDR #ifndef EXAMPLE_OCCURRENCE_HDR
#define EXAMPLE_OCCURENCE_HDR #define EXAMPLE_OCCURRENCE_HDR
#include <string> #include <string>
#include <vector> #include <vector>
class ExampleOccurence { class ExampleOccurrence {
public: public:
/*! Constructor. /*! Constructor.
*/ */
ExampleOccurence (const int id, ExampleOccurrence (const int id,
const int matchedExampleStart, const int matchedExampleStart,
const int matchedExampleEnd, const int matchedExampleEnd,
const std::string & sourceSegment, const std::string & sourceSegment,
@ -16,7 +16,7 @@ public:
); );
/*! Destructor. /*! Destructor.
*/ */
virtual ~ExampleOccurence(); virtual ~ExampleOccurrence();
int getId() const { int getId() const {
return _id; return _id;

4
concordia-server/full_search_result.cpp
View File

@ -8,6 +8,6 @@ FullSearchResult::FullSearchResult(
FullSearchResult::~FullSearchResult() { FullSearchResult::~FullSearchResult() {
} }
void FullSearchResult::addOccurence(const ExampleOccurence & occurence) { void FullSearchResult::addOccurrence(const ExampleOccurrence & occurrence) {
_occurences.push_back(occurence); _occurrences.push_back(occurrence);
} }

10
concordia-server/full_search_result.hpp
View File

@ -1,7 +1,7 @@
#ifndef FULL_SEARCH_RESULT_HDR #ifndef FULL_SEARCH_RESULT_HDR
#define FULL_SEARCH_RESULT_HDR #define FULL_SEARCH_RESULT_HDR
#include "example_occurence.hpp" #include "example_occurrence.hpp"
#include <string> #include <string>
#include <vector> #include <vector>
@ -18,16 +18,16 @@ public:
return _totalCount; return _totalCount;
} }
std::vector<ExampleOccurence> getOccurences() const { std::vector<ExampleOccurrence> getOccurrences() const {
return _occurences; return _occurrences;
} }
void addOccurence(const ExampleOccurence & occurence); void addOccurrence(const ExampleOccurrence & occurrence);
void offsetPattern(int offset); void offsetPattern(int offset);
private: private:
std::vector<ExampleOccurence> _occurences; std::vector<ExampleOccurrence> _occurrences;
int _totalCount; int _totalCount;
}; };

90
concordia-server/json_generator.cpp
View File

@ -1,7 +1,7 @@
#include "json_generator.hpp" #include "json_generator.hpp"
#include <boost/foreach.hpp> #include <boost/foreach.hpp>
#include "example_occurence.hpp" #include "example_occurrence.hpp"
JsonGenerator::JsonGenerator() { JsonGenerator::JsonGenerator() {
} }
@ -21,42 +21,44 @@ void JsonGenerator::signalError(rapidjson::Writer<rapidjson::StringBuffer> & jso
} }
void JsonGenerator::writeSimpleSearchResult(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter, void JsonGenerator::writeSimpleSearchResult(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter,
const SimpleSearchResult & result) { const SimpleSearchResult & result, bool writeOccurrences) {
jsonWriter.StartObject(); jsonWriter.StartObject();
jsonWriter.String("matchedPatternStart"); jsonWriter.String("matchedPatternStart");
jsonWriter.Int(result.getMatchedPatternStart()); jsonWriter.Int(result.getMatchedPatternStart());
jsonWriter.String("matchedPatternEnd"); jsonWriter.String("matchedPatternEnd");
jsonWriter.Int(result.getMatchedPatternEnd()); jsonWriter.Int(result.getMatchedPatternEnd());
jsonWriter.String("occurences");
jsonWriter.StartArray();
BOOST_FOREACH(ExampleOccurence occurence, result.getOccurences()) { if (writeOccurrences) {
jsonWriter.StartObject(); jsonWriter.String("occurrences");
jsonWriter.String("id"); jsonWriter.StartArray();
jsonWriter.Int(occurence.getId());
jsonWriter.String("matchedExampleStart"); BOOST_FOREACH(ExampleOccurrence occurrence, result.getOccurrences()) {
jsonWriter.Int(occurence.getMatchedExampleStart()); jsonWriter.StartObject();
jsonWriter.String("matchedExampleEnd"); jsonWriter.String("id");
jsonWriter.Int(occurence.getMatchedExampleEnd()); jsonWriter.Int(occurrence.getId());
jsonWriter.String("sourceSegment"); jsonWriter.String("matchedExampleStart");
jsonWriter.String(occurence.getSourceSegment().c_str()); jsonWriter.Int(occurrence.getMatchedExampleStart());
jsonWriter.String("targetSegment"); jsonWriter.String("matchedExampleEnd");
jsonWriter.String(occurence.getTargetSegment().c_str()); jsonWriter.Int(occurrence.getMatchedExampleEnd());
jsonWriter.String("targetFragments"); jsonWriter.String("sourceSegment");
jsonWriter.StartArray(); // all target fragments jsonWriter.String(occurrence.getSourceSegment().c_str());
for (std::vector<std::pair<int,int> >::const_iterator it = occurence.getTargetFragments().begin(); jsonWriter.String("targetSegment");
it != occurence.getTargetFragments().end(); it++) { jsonWriter.String(occurrence.getTargetSegment().c_str());
jsonWriter.StartArray(); // single target fragment jsonWriter.String("targetFragments");
jsonWriter.Int(it->first); jsonWriter.StartArray(); // all target fragments
jsonWriter.Int(it->second); for (std::vector<std::pair<int,int> >::const_iterator it = occurrence.getTargetFragments().begin();
jsonWriter.EndArray(); // single target fragment it != occurrence.getTargetFragments().end(); it++) {
jsonWriter.StartArray(); // single target fragment
jsonWriter.Int(it->first);
jsonWriter.Int(it->second);
jsonWriter.EndArray(); // single target fragment
}
jsonWriter.EndArray(); // all target fragments
jsonWriter.EndObject(); // occurrence
} }
jsonWriter.EndArray(); // all target fragments
jsonWriter.EndObject(); // occurence jsonWriter.EndArray(); //occurrences
} }
jsonWriter.EndArray(); //occurences
jsonWriter.EndObject(); //simple search result jsonWriter.EndObject(); //simple search result
} }
@ -65,35 +67,35 @@ void JsonGenerator::writeFullSearchResult(rapidjson::Writer<rapidjson::StringBuf
jsonWriter.StartObject(); jsonWriter.StartObject();
jsonWriter.String("totalCount"); jsonWriter.String("totalCount");
jsonWriter.Int(result.getTotalCount()); jsonWriter.Int(result.getTotalCount());
jsonWriter.String("occurences"); jsonWriter.String("occurrences");
jsonWriter.StartArray(); jsonWriter.StartArray();
BOOST_FOREACH(ExampleOccurence occurence, result.getOccurences()) { BOOST_FOREACH(ExampleOccurrence occurrence, result.getOccurrences()) {
jsonWriter.StartObject(); jsonWriter.StartObject();
jsonWriter.String("id"); jsonWriter.String("id");
jsonWriter.Int(occurence.getId()); jsonWriter.Int(occurrence.getId());
jsonWriter.String("matchedExampleStart"); jsonWriter.String("matchedExampleStart");
jsonWriter.Int(occurence.getMatchedExampleStart()); jsonWriter.Int(occurrence.getMatchedExampleStart());
jsonWriter.String("matchedExampleEnd"); jsonWriter.String("matchedExampleEnd");
jsonWriter.Int(occurence.getMatchedExampleEnd()); jsonWriter.Int(occurrence.getMatchedExampleEnd());
jsonWriter.String("sourceSegment"); jsonWriter.String("sourceSegment");
jsonWriter.String(occurence.getSourceSegment().c_str()); jsonWriter.String(occurrence.getSourceSegment().c_str());
jsonWriter.String("targetSegment"); jsonWriter.String("targetSegment");
jsonWriter.String(occurence.getTargetSegment().c_str()); jsonWriter.String(occurrence.getTargetSegment().c_str());
jsonWriter.String("targetFragments"); jsonWriter.String("targetFragments");
jsonWriter.StartArray(); // all target fragments jsonWriter.StartArray(); // all target fragments
for (std::vector<std::pair<int,int> >::const_iterator it = occurence.getTargetFragments().begin(); for (std::vector<std::pair<int,int> >::const_iterator it = occurrence.getTargetFragments().begin();
it != occurence.getTargetFragments().end(); it++) { it != occurrence.getTargetFragments().end(); it++) {
jsonWriter.StartArray(); // single target fragment jsonWriter.StartArray(); // single target fragment
jsonWriter.Int(it->first); jsonWriter.Int(it->first);
jsonWriter.Int(it->second); jsonWriter.Int(it->second);
jsonWriter.EndArray(); // single target fragment jsonWriter.EndArray(); // single target fragment
} }
jsonWriter.EndArray(); // all target fragments jsonWriter.EndArray(); // all target fragments
jsonWriter.EndObject(); // occurence jsonWriter.EndObject(); // occurrence
} }
jsonWriter.EndArray(); //occurences jsonWriter.EndArray(); //occurrences
jsonWriter.EndObject(); //full search result jsonWriter.EndObject(); //full search result
} }
@ -103,13 +105,13 @@ void JsonGenerator::writeLexiconSearchResult(rapidjson::Writer<rapidjson::String
const SimpleSearchResult & result) { const SimpleSearchResult & result) {
jsonWriter.StartArray(); jsonWriter.StartArray();
BOOST_FOREACH(ExampleOccurence occurence, result.getOccurences()) { BOOST_FOREACH(ExampleOccurrence occurrence, result.getOccurrences()) {
jsonWriter.StartObject(); jsonWriter.StartObject();
jsonWriter.String("sourceSegment"); jsonWriter.String("sourceSegment");
jsonWriter.String(occurence.getSourceSegment().c_str()); jsonWriter.String(occurrence.getSourceSegment().c_str());
jsonWriter.String("targetSegment"); jsonWriter.String("targetSegment");
jsonWriter.String(occurence.getTargetSegment().c_str()); jsonWriter.String(occurrence.getTargetSegment().c_str());
jsonWriter.EndObject(); // occurence jsonWriter.EndObject(); // occurrence
} }
jsonWriter.EndArray(); //simple search result jsonWriter.EndArray(); //simple search result

2
concordia-server/json_generator.hpp
View File

@ -21,7 +21,7 @@ public:
const std::string & message); const std::string & message);
static void writeSimpleSearchResult(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter, static void writeSimpleSearchResult(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter,
const SimpleSearchResult & result); const SimpleSearchResult & result, bool writeOccurrences = true);
static void writeFullSearchResult(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter, static void writeFullSearchResult(rapidjson::Writer<rapidjson::StringBuffer> & jsonWriter,
const FullSearchResult & result); const FullSearchResult & result);

4
concordia-server/searcher_controller.cpp
View File

@ -102,7 +102,7 @@ void SearcherController::concordiaPhraseSearch(rapidjson::Writer<rapidjson::Stri
jsonWriter.String("status"); jsonWriter.String("status");
jsonWriter.String("success"); jsonWriter.String("success");
jsonWriter.String("found"); jsonWriter.String("found");
if (shortPatternResult.getOccurences().size() > 0) { if (shortPatternResult.getOccurrences().size() > 0) {
jsonWriter.Bool(true); jsonWriter.Bool(true);
@ -169,7 +169,7 @@ void SearcherController::concordiaSearch(rapidjson::Writer<rapidjson::StringBuff
jsonWriter.String("bestOverlay"); jsonWriter.String("bestOverlay");
jsonWriter.StartArray(); jsonWriter.StartArray();
BOOST_FOREACH(const SimpleSearchResult & simpleResult, result.getBestOverlay()) { BOOST_FOREACH(const SimpleSearchResult & simpleResult, result.getBestOverlay()) {
JsonGenerator::writeSimpleSearchResult(jsonWriter, simpleResult); JsonGenerator::writeSimpleSearchResult(jsonWriter, simpleResult, false);
} }
jsonWriter.EndArray(); jsonWriter.EndArray();
jsonWriter.EndObject(); jsonWriter.EndObject();

4
concordia-server/simple_search_result.cpp
View File

@ -10,8 +10,8 @@ SimpleSearchResult::SimpleSearchResult(
SimpleSearchResult::~SimpleSearchResult() { SimpleSearchResult::~SimpleSearchResult() {
} }
void SimpleSearchResult::addOccurence(const ExampleOccurence & occurence) { void SimpleSearchResult::addOccurrence(const ExampleOccurrence & occurrence) {
_occurences.push_back(occurence); _occurrences.push_back(occurrence);
} }
void SimpleSearchResult::offsetPattern(int offset) { void SimpleSearchResult::offsetPattern(int offset) {

10
concordia-server/simple_search_result.hpp
View File

@ -1,7 +1,7 @@
#ifndef SIMPLE_SEARCH_RESULT_HDR #ifndef SIMPLE_SEARCH_RESULT_HDR
#define SIMPLE_SEARCH_RESULT_HDR #define SIMPLE_SEARCH_RESULT_HDR
#include "example_occurence.hpp" #include "example_occurrence.hpp"
#include <string> #include <string>
#include <vector> #include <vector>
@ -31,16 +31,16 @@ public:
_matchedPatternEnd = newEnd; _matchedPatternEnd = newEnd;
} }
std::vector<ExampleOccurence> getOccurences() const { std::vector<ExampleOccurrence> getOccurrences() const {
return _occurences; return _occurrences;
} }
void addOccurence(const ExampleOccurence & occurence); void addOccurrence(const ExampleOccurrence & occurrence);
void offsetPattern(int offset); void offsetPattern(int offset);
private: private:
std::vector<ExampleOccurence> _occurences; std::vector<ExampleOccurrence> _occurrences;
int _matchedPatternStart; int _matchedPatternStart;

36
concordia-server/unit_dao.cpp
View File

@ -12,7 +12,7 @@
#include "int_array_param.hpp" #include "int_array_param.hpp"
#include "int_2d_array_param.hpp" #include "int_2d_array_param.hpp"
#include "logger.hpp" #include "logger.hpp"
#include "example_occurence.hpp" #include "example_occurrence.hpp"
#include <libpq-fe.h> #include <libpq-fe.h>
#include <boost/foreach.hpp> #include <boost/foreach.hpp>
@ -78,13 +78,13 @@ SimpleSearchResult UnitDAO::getSimpleSearchResult(const MatchedPatternFragment &
return _getResultFromFragment(fragment, ts, true); return _getResultFromFragment(fragment, ts, true);
} }
FullSearchResult UnitDAO::getFullSearchResult(const OccurencesList & occurencesList, const int patternLength) { FullSearchResult UnitDAO::getFullSearchResult(const OccurrencesList & occurrencesList, const int patternLength) {
FullSearchResult result(occurencesList.getTotalCount()); FullSearchResult result(occurrencesList.getTotalCount());
DBconnection connection; DBconnection connection;
connection.startTransaction(); connection.startTransaction();
BOOST_FOREACH(SubstringOccurence sOccurence, occurencesList.getOccurences()) { BOOST_FOREACH(SubstringOccurrence sOccurrence, occurrencesList.getOccurrences()) {
result.addOccurence(_getExampleOccurence(connection, sOccurence, patternLength)); result.addOccurrence(_getExampleOccurrence(connection, sOccurrence, patternLength));
} }
connection.endTransaction(); connection.endTransaction();
@ -117,7 +117,7 @@ SimpleSearchResult UnitDAO::_getResultFromFragment(
SimpleSearchResult UnitDAO::_getResultFromFragment( SimpleSearchResult UnitDAO::_getResultFromFragment(
const MatchedPatternFragment & fragment, const MatchedPatternFragment & fragment,
const TokenizedSentence & tokenizedPattern, const TokenizedSentence & tokenizedPattern,
bool getOccurences) { bool getOccurrences) {
DBconnection connection; DBconnection connection;
connection.startTransaction(); connection.startTransaction();
@ -134,9 +134,9 @@ SimpleSearchResult UnitDAO::_getResultFromFragment(
if (getOccurences) { if (getOccurrences) {
BOOST_FOREACH(SubstringOccurence sOccurence, fragment.getOccurences()) { BOOST_FOREACH(SubstringOccurrence sOccurrence, fragment.getOccurrences()) {
ssResult.addOccurence(_getExampleOccurence(connection, sOccurence, fragment.getMatchedLength())); ssResult.addOccurrence(_getExampleOccurrence(connection, sOccurrence, fragment.getMatchedLength()));
} }
} }
@ -145,14 +145,14 @@ SimpleSearchResult UnitDAO::_getResultFromFragment(
return ssResult; return ssResult;
} }
ExampleOccurence UnitDAO::_getExampleOccurence(DBconnection & connection, const SubstringOccurence sOccurence, const int matchedLength) { ExampleOccurrence UnitDAO::_getExampleOccurrence(DBconnection & connection, const SubstringOccurrence sOccurrence, const int matchedLength) {
std::string query = "SELECT id, source_segment, target_segment, source_tokens[$1::integer], source_tokens[$2::integer], target_tokens, alignments FROM unit WHERE id = $3::integer;"; std::string query = "SELECT id, source_segment, target_segment, source_tokens[$1::integer], source_tokens[$2::integer], target_tokens, alignments FROM unit WHERE id = $3::integer;";
std::vector<QueryParam*> params; std::vector<QueryParam*> params;
params.push_back(new IntParam(2*sOccurence.getOffset()+1)); params.push_back(new IntParam(2*sOccurrence.getOffset()+1));
params.push_back(new IntParam(2*(sOccurence.getOffset()+matchedLength))); params.push_back(new IntParam(2*(sOccurrence.getOffset()+matchedLength)));
params.push_back(new IntParam(sOccurence.getId())); params.push_back(new IntParam(sOccurrence.getId()));
PGresult * result = connection.execute(query, params); PGresult * result = connection.execute(query, params);
ExampleOccurence occurence(connection.getIntValue(result,0,0), // example id ExampleOccurrence occurrence(connection.getIntValue(result,0,0), // example id
connection.getIntValue(result,0,3), // matched example start connection.getIntValue(result,0,3), // matched example start
connection.getIntValue(result,0,4), // matched example end connection.getIntValue(result,0,4), // matched example end
connection.getStringValue(result,0,1), // source segment connection.getStringValue(result,0,1), // source segment
@ -169,7 +169,7 @@ ExampleOccurence UnitDAO::_getExampleOccurence(DBconnection & connection, const
std::vector<std::vector<int> > alignments = _get2DArray(alignmentsRaw); std::vector<std::vector<int> > alignments = _get2DArray(alignmentsRaw);
std::set<int> matchedTargetTokens; std::set<int> matchedTargetTokens;
for(int sourceTokenIndex = sOccurence.getOffset(); sourceTokenIndex < sOccurence.getOffset()+matchedLength; sourceTokenIndex++) { for(int sourceTokenIndex = sOccurrence.getOffset(); sourceTokenIndex < sOccurrence.getOffset()+matchedLength; sourceTokenIndex++) {
BOOST_FOREACH(int & targetTokenIndex, alignments.at(sourceTokenIndex)) { BOOST_FOREACH(int & targetTokenIndex, alignments.at(sourceTokenIndex)) {
matchedTargetTokens.insert(targetTokenIndex); matchedTargetTokens.insert(targetTokenIndex);
} }
@ -188,7 +188,7 @@ ExampleOccurence UnitDAO::_getExampleOccurence(DBconnection & connection, const
if (prevPos < targetPos - 1) { // beginning of detached fragment if (prevPos < targetPos - 1) { // beginning of detached fragment
// check if there is a fragment to end // check if there is a fragment to end
if (currStart >= 0) { if (currStart >= 0) {
occurence.addMatchedTargetFragment(std::pair<int,int>(currStart,currEnd)); occurrence.addMatchedTargetFragment(std::pair<int,int>(currStart,currEnd));
} }
currStart = targetStart; currStart = targetStart;
} }
@ -200,10 +200,10 @@ ExampleOccurence UnitDAO::_getExampleOccurence(DBconnection & connection, const
// check if there are remaining fragments // check if there are remaining fragments
if (currStart >= 0) { if (currStart >= 0) {
occurence.addMatchedTargetFragment(std::pair<int,int>(currStart,currEnd)); occurrence.addMatchedTargetFragment(std::pair<int,int>(currStart,currEnd));
} }
return occurence; return occurrence;
} }

10
concordia-server/unit_dao.hpp
View File

@ -6,9 +6,9 @@
#include <concordia/common/config.hpp> #include <concordia/common/config.hpp>
#include <concordia/tokenized_sentence.hpp> #include <concordia/tokenized_sentence.hpp>
#include <concordia/substring_occurence.hpp> #include <concordia/substring_occurrence.hpp>
#include <concordia/matched_pattern_fragment.hpp> #include <concordia/matched_pattern_fragment.hpp>
#include <concordia/occurences_list.hpp> #include <concordia/occurrences_list.hpp>
#include <concordia/concordia_search_result.hpp> #include <concordia/concordia_search_result.hpp>
#include <concordia/concordia_exception.hpp> #include <concordia/concordia_exception.hpp>
#include <boost/shared_ptr.hpp> #include <boost/shared_ptr.hpp>
@ -45,7 +45,7 @@ public:
SimpleSearchResult getSimpleSearchResult(const MatchedPatternFragment & fragment); SimpleSearchResult getSimpleSearchResult(const MatchedPatternFragment & fragment);
FullSearchResult getFullSearchResult(const OccurencesList & occurencesList, const int patternLength); FullSearchResult getFullSearchResult(const OccurrencesList & occurrencesList, const int patternLength);
CompleteConcordiaSearchResult getConcordiaResult(boost::shared_ptr<ConcordiaSearchResult> rawConcordiaResult); CompleteConcordiaSearchResult getConcordiaResult(boost::shared_ptr<ConcordiaSearchResult> rawConcordiaResult);
@ -59,11 +59,11 @@ private:
SimpleSearchResult _getResultFromFragment( SimpleSearchResult _getResultFromFragment(
const MatchedPatternFragment & fragment, const MatchedPatternFragment & fragment,
const TokenizedSentence & tokenizedPattern, const TokenizedSentence & tokenizedPattern,
bool getOccurences); bool getOccurrences);
std::vector<int> _getTokenPositions(const TokenizedSentence & ts); std::vector<int> _getTokenPositions(const TokenizedSentence & ts);
ExampleOccurence _getExampleOccurence(DBconnection & connection, const SubstringOccurence sOccurence, const int matchedLength); ExampleOccurrence _getExampleOccurrence(DBconnection & connection, const SubstringOccurrence sOccurrence, const int matchedLength);
int _addSingleSentence( int _addSingleSentence(
DBconnection & connection, DBconnection & connection,

6
tests/addLemmatizedTM.sh
View File

@ -1,7 +1,7 @@
#!/bin/sh #!/bin/sh
CORPUS_NAME="europarl_sample" CORPUS_NAME="stocznia_plen"
SRC_LANG_ID=2 SRC_LANG_ID=1
TRG_LANG_ID=1 TRG_LANG_ID=2
./addAlignedLemmatizedTM.py $CORPUS_NAME ../mgiza-aligner/corpora/$CORPUS_NAME/src_final.txt $SRC_LANG_ID ../mgiza-aligner/corpora/$CORPUS_NAME/trg_final.txt $TRG_LANG_ID ../mgiza-aligner/corpora/$CORPUS_NAME/aligned_final.txt ./addAlignedLemmatizedTM.py $CORPUS_NAME ../mgiza-aligner/corpora/$CORPUS_NAME/src_final.txt $SRC_LANG_ID ../mgiza-aligner/corpora/$CORPUS_NAME/trg_final.txt $TRG_LANG_ID ../mgiza-aligner/corpora/$CORPUS_NAME/aligned_final.txt