rjawor/concordia-library
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

simple search returns matched pattern fragments

Browse Source
This commit is contained in:
rjawor 2015-08-07 12:54:57 +02:00
parent 28704c2f43
commit a765443a01
9 changed files with 61 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
concordia-console/concordia-console.cpp
View File

@ -7,7 +7,7 @@
#include <boost/foreach.hpp>
#include "concordia/concordia.hpp"
#include "concordia/substring_occurence.hpp"
#include "concordia/matched_pattern_fragment.hpp"
#include "concordia/token_annotation.hpp"
#include "concordia/common/config.hpp"
#include "concordia/common/utils.hpp"
@ -158,7 +158,7 @@ int main(int argc, char** argv) {
std::cout << "\tSearching for pattern: \"" << pattern <<
"\"" << std::endl;
time_start = boost::posix_time::microsec_clock::local_time();
std::vector<SubstringOccurence> result =
std::vector<MatchedPatternFragment> result =
concordia.simpleSearch(pattern);
time_end = boost::posix_time::microsec_clock::local_time();
msdiff = time_end - time_start;
@ -166,9 +166,9 @@ int main(int argc, char** argv) {
<< "Search took: " <<
msdiff.total_milliseconds() << "ms." << std::endl;
if (!cli.count("silent")) {
BOOST_FOREACH(SubstringOccurence occurence, result) {
BOOST_FOREACH(MatchedPatternFragment occurence, result) {
std::cout << "\t\tfound match in sentence number: "
<< occurence.getId() << std::endl;
<< occurence.getExampleId() << std::endl;
}
}
} else if (cli.count("anubis-search")) {

5
concordia/concordia.cpp
View File

@ -49,7 +49,6 @@ boost::shared_ptr<TokenizedSentence>
_hashGenerator->generateHash(sentence);
_hashGenerator->serializeWordMap();
return result;
}
@ -156,14 +155,14 @@ void Concordia::_initializeIndex() throw(ConcordiaException) {
}
}
std::vector<SubstringOccurence> Concordia::simpleSearch(
std::vector<MatchedPatternFragment> Concordia::simpleSearch(
const std::string & pattern)
throw(ConcordiaException) {
if (_T->size() > 0) {
return _searcher->simpleSearch(_hashGenerator, _T,
_markers, _SA, pattern);
} else {
std::vector<SubstringOccurence> result;
std::vector<MatchedPatternFragment> result;
return result;
}
}

5
concordia/concordia.hpp
View File

@ -8,7 +8,7 @@
#include "concordia/common/config.hpp"
#include "concordia/example.hpp"
#include "concordia/substring_occurence.hpp"
#include "concordia/matched_pattern_fragment.hpp"
#include "concordia/concordia_config.hpp"
#include "concordia/concordia_index.hpp"
#include "concordia/index_searcher.hpp"
@ -96,7 +96,8 @@ public:
\returns vector of matched results
\throws ConcordiaException
*/
std::vector<SubstringOccurence> simpleSearch(const std::string & pattern)
std::vector<MatchedPatternFragment> simpleSearch(
const std::string & pattern)
throw(ConcordiaException);
/*! \deprecated

2
concordia/concordia_index.cpp
View File

@ -150,7 +150,7 @@ boost::shared_ptr<TokenizedSentence> ConcordiaIndex::_addSingleExample(
hashGenerator->generateHash(example.getSentence());
_addSingleTokenizedExample(hashedIndexFile, markersFile, hashGenerator,
T, markers, hashedPattern, example.getId());
return hashedPattern;
}

11
concordia/index_searcher.cpp
View File

@ -13,13 +13,13 @@ IndexSearcher::IndexSearcher() {
IndexSearcher::~IndexSearcher() {
}
std::vector<SubstringOccurence> IndexSearcher::simpleSearch(
std::vector<MatchedPatternFragment> IndexSearcher::simpleSearch(
boost::shared_ptr<HashGenerator> hashGenerator,
boost::shared_ptr<std::vector<sauchar_t> > T,
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,
boost::shared_ptr<std::vector<saidx_t> > SA,
const std::string & pattern) throw(ConcordiaException) {
std::vector<SubstringOccurence> result;
std::vector<MatchedPatternFragment> result;
int left;
std::vector<INDEX_CHARACTER_TYPE> hash =
@ -40,8 +40,11 @@ std::vector<SubstringOccurence> IndexSearcher::simpleSearch(
// removes these accidental results.
saidx_t actualResultPos = resultPos / sizeof(INDEX_CHARACTER_TYPE);
SUFFIX_MARKER_TYPE marker = markers->at(actualResultPos);
result.push_back(SubstringOccurence(marker));
result.push_back(MatchedPatternFragment(
Utils::getIdFromMarker(marker),
Utils::getOffsetFromMarker(marker),
0,
hash.size()));
}
}

4
concordia/index_searcher.hpp
View File

@ -7,7 +7,7 @@
#include <vector>
#include "concordia/common/config.hpp"
#include "concordia/substring_occurence.hpp"
#include "concordia/matched_pattern_fragment.hpp"
#include "concordia/hash_generator.hpp"
#include "concordia/concordia_exception.hpp"
#include "concordia/concordia_searcher.hpp"
@ -45,7 +45,7 @@ public:
\returns vector of occurences of the pattern in the index
\throws ConcordiaException
*/
std::vector<SubstringOccurence> simpleSearch(
std::vector<MatchedPatternFragment> simpleSearch(
boost::shared_ptr<HashGenerator> hashGenerator,
boost::shared_ptr<std::vector<sauchar_t> > T,
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,

54
concordia/t/test_concordia.cpp
View File

@ -3,6 +3,7 @@
#include "concordia/anubis_search_result.hpp"
#include "concordia/tokenized_sentence.hpp"
#include "concordia/token_annotation.hpp"
#include "concordia/matched_pattern_fragment.hpp"
#include "tests/common/test_resources_manager.hpp"
#include "concordia/common/config.hpp"
@ -64,16 +65,18 @@ BOOST_AUTO_TEST_CASE( ConcordiaSimpleSearch1 )
*/
std::vector<SubstringOccurence> searchResult1 = concordia.simpleSearch("posiada rysia");
std::vector<SubstringOccurence> searchResult2 = concordia.simpleSearch("posiada kota Ala");
std::vector<MatchedPatternFragment> searchResult1 = concordia.simpleSearch("posiada rysia");
std::vector<MatchedPatternFragment> searchResult2 = concordia.simpleSearch("posiada kota Ala");
concordia.clearIndex();
BOOST_CHECK_EQUAL(searchResult1.size(), 2);
BOOST_CHECK_EQUAL(searchResult1.at(0).getId(), 123);
BOOST_CHECK_EQUAL(searchResult1.at(0).getOffset(), 1);
BOOST_CHECK_EQUAL(searchResult1.at(1).getId(), 51);
BOOST_CHECK_EQUAL(searchResult1.at(1).getOffset(), 1);
BOOST_CHECK_EQUAL(searchResult1.at(0).getExampleId(), 123);
BOOST_CHECK_EQUAL(searchResult1.at(0).getExampleOffset(), 1);
BOOST_CHECK_EQUAL(searchResult1.at(0).getMatchedLength(), 2);
BOOST_CHECK_EQUAL(searchResult1.at(1).getExampleId(), 51);
BOOST_CHECK_EQUAL(searchResult1.at(1).getExampleOffset(), 1);
BOOST_CHECK_EQUAL(searchResult1.at(1).getMatchedLength(), 2);
// Checking pattern spanning over 2 segments
BOOST_CHECK_EQUAL(searchResult2.size(), 0);
@ -126,24 +129,29 @@ BOOST_AUTO_TEST_CASE( ConcordiaSimpleSearch2 )
*/
Concordia concordia2 = Concordia(TestResourcesManager::getTestConcordiaConfigFilePath("concordia.cfg"));
std::vector<SubstringOccurence> searchResult1 = concordia2.simpleSearch("xto xjest");
std::vector<SubstringOccurence> searchResult2 = concordia2.simpleSearch("xjest okno");
std::vector<MatchedPatternFragment> searchResult1 = concordia2.simpleSearch("xto xjest");
std::vector<MatchedPatternFragment> searchResult2 = concordia2.simpleSearch("xjest okno");
concordia2.clearIndex();
BOOST_CHECK_EQUAL(searchResult1.size(), 3);
BOOST_CHECK_EQUAL(searchResult1.at(0).getId(), 312);
BOOST_CHECK_EQUAL(searchResult1.at(0).getOffset(), 0);
BOOST_CHECK_EQUAL(searchResult1.at(1).getId(), 45);
BOOST_CHECK_EQUAL(searchResult1.at(1).getOffset(), 1);
BOOST_CHECK_EQUAL(searchResult1.at(2).getId(), 29);
BOOST_CHECK_EQUAL(searchResult1.at(2).getOffset(), 0);
BOOST_CHECK_EQUAL(searchResult1.at(0).getExampleId(), 312);
BOOST_CHECK_EQUAL(searchResult1.at(0).getExampleOffset(), 0);
BOOST_CHECK_EQUAL(searchResult1.at(0).getMatchedLength(), 2);
BOOST_CHECK_EQUAL(searchResult1.at(1).getExampleId(), 45);
BOOST_CHECK_EQUAL(searchResult1.at(1).getExampleOffset(), 1);
BOOST_CHECK_EQUAL(searchResult1.at(1).getMatchedLength(), 2);
BOOST_CHECK_EQUAL(searchResult1.at(2).getExampleId(), 29);
BOOST_CHECK_EQUAL(searchResult1.at(2).getExampleOffset(), 0);
BOOST_CHECK_EQUAL(searchResult1.at(2).getMatchedLength(), 2);
BOOST_CHECK_EQUAL(searchResult2.size(), 2);
BOOST_CHECK_EQUAL(searchResult2.at(0).getId(), 202);
BOOST_CHECK_EQUAL(searchResult2.at(0).getOffset(), 1);
BOOST_CHECK_EQUAL(searchResult2.at(1).getId(), 312);
BOOST_CHECK_EQUAL(searchResult2.at(1).getOffset(), 1);
BOOST_CHECK_EQUAL(searchResult2.at(0).getExampleId(), 202);
BOOST_CHECK_EQUAL(searchResult2.at(0).getExampleOffset(), 1);
BOOST_CHECK_EQUAL(searchResult2.at(0).getMatchedLength(), 2);
BOOST_CHECK_EQUAL(searchResult2.at(1).getExampleId(), 312);
BOOST_CHECK_EQUAL(searchResult2.at(1).getExampleOffset(), 1);
BOOST_CHECK_EQUAL(searchResult2.at(1).getMatchedLength(), 2);
}
BOOST_AUTO_TEST_CASE( ConcordiaSimpleSearch3 )
@ -155,13 +163,14 @@ BOOST_AUTO_TEST_CASE( ConcordiaSimpleSearch3 )
concordia.addAllExamples(testExamples);
Concordia concordia2 = Concordia(TestResourcesManager::getTestConcordiaConfigFilePath("concordia.cfg"));
std::vector<SubstringOccurence> searchResult1 = concordia2.simpleSearch("on w szczególności prawo do podjęcia");
std::vector<MatchedPatternFragment> searchResult1 = concordia2.simpleSearch("on w szczególności prawo do podjęcia");
concordia2.clearIndex();
BOOST_CHECK_EQUAL(searchResult1.size(), 1);
BOOST_CHECK_EQUAL(searchResult1.at(0).getId(), 312);
BOOST_CHECK_EQUAL(searchResult1.at(0).getOffset(), 2);
BOOST_CHECK_EQUAL(searchResult1.at(0).getExampleId(), 312);
BOOST_CHECK_EQUAL(searchResult1.at(0).getExampleOffset(), 2);
BOOST_CHECK_EQUAL(searchResult1.at(0).getMatchedLength(), 6);
}
BOOST_AUTO_TEST_CASE( ConcordiaAnubisSearch1 )
@ -335,6 +344,9 @@ BOOST_AUTO_TEST_CASE( Tokenize )
4,11 type: 1 value: posiada
12,16 type: 1 value: kota
*/
concordia.clearIndex();
BOOST_CHECK_EQUAL(ts->getTokens().size(), 3);
BOOST_CHECK_EQUAL(ts->getTokens().at(1).getStart(), 9);
BOOST_CHECK_EQUAL(ts->getTokens().at(1).getEnd(), 16);

10
concordia/tutorial.dox
View File

@ -41,7 +41,7 @@ This code snippet shows the basic Concordia functionality - simple substring loo
File simple_search.cpp:
\verbatim
#include <concordia/concordia.hpp>
#include <concordia/substring_occurence.hpp>
#include <concordia/matched_pattern_fragment.hpp>
#include <concordia/example.hpp>
#include "config.hpp"
@ -65,12 +65,12 @@ int main() {
// searching
cout << "Searching for pattern: has a" << endl;
vector<SubstringOccurence> result = concordia.simpleSearch("has a");
vector<MatchedPatternFragment> result = concordia.simpleSearch("has a");
// printing results
for(vector<SubstringOccurence>::iterator it = result.begin();
for(vector<MatchedPatternFragment>::iterator it = result.begin();
it != result.end(); ++it) {
cout << "Found substring in sentence: " << it->getId() << " at offset: " << it->getOffset() << endl;
cout << "Found substring in sentence: " << it->getExampleId() << " at offset: " << it->getExampleOffset() << endl;
}
// clearing index
@ -82,7 +82,7 @@ First, sentences are added to the index along with their integer IDs. The pair (
After adding the examples, index needs to be generated using the method refreshSAfromRAM. Details of this operation are covered in \ref tutorial2.
The search returns a vector of SubstringOccurence objects, which is then printed out. Each occurence represents a single match of the pattern. The pattern has to be matched within a single sentence. Information about the match consists of two integer values: ID of the sentence where the match occured and word-level, 0-based offset of the matched pattern in the sentence. The above code should return the following results:
The search returns a vector of MatchedPatternFragment objects, which is then printed out. Each matched fragment represents a single match of the pattern. The pattern has to be matched within a single sentence. Information about the match consists of two integer values: ID of the sentence where the match occured and word-level, 0-based offset of the matched pattern in the sentence. The above code should return the following results:
\verbatim
Found substring in sentence: 56 at offset: 1

8
examples/simple_search.cpp
View File

@ -1,5 +1,5 @@
#include <concordia/concordia.hpp>
#include <concordia/substring_occurence.hpp>
#include <concordia/matched_pattern_fragment.hpp>
#include <concordia/example.hpp>
#include "config.hpp"
@ -23,12 +23,12 @@ int main() {
// searching
cout << "Searching for pattern: has a" << endl;
vector<SubstringOccurence> result = concordia.simpleSearch("has a");
vector<MatchedPatternFragment> result = concordia.simpleSearch("has a");
// printing results
for(vector<SubstringOccurence>::iterator it = result.begin();
for(vector<MatchedPatternFragment>::iterator it = result.begin();
it != result.end(); ++it) {
cout << "Found substring in sentence: " << it->getId() << " at offset: " << it->getOffset() << endl;
cout << "Found substring in sentence: " << it->getExampleId() << " at offset: " << it->getExampleOffset() << endl;
}
// clearing index