424 lines
19 KiB
C++
424 lines
19 KiB
C++
#include <iostream>
|
|
#include <fstream>
|
|
#include <boost/program_options.hpp>
|
|
#include <boost/algorithm/string.hpp>
|
|
#include <boost/date_time/posix_time/posix_time.hpp>
|
|
#include <boost/shared_ptr.hpp>
|
|
#include <boost/foreach.hpp>
|
|
|
|
#include "concordia/concordia.hpp"
|
|
#include "concordia/substring_occurence.hpp"
|
|
#include "concordia/token_annotation.hpp"
|
|
#include "concordia/common/config.hpp"
|
|
#include "concordia/common/utils.hpp"
|
|
#include "build/libdivsufsort/include/divsufsort.h"
|
|
|
|
#define READ_BUFFER_LENGTH 10000
|
|
|
|
namespace po = boost::program_options;
|
|
|
|
void reportError(long lineNumber, const std::string & error) {
|
|
std::stringstream ss;
|
|
ss << "Search error in line " << lineNumber << ": " << error;
|
|
throw ConcordiaException(ss.str());
|
|
}
|
|
|
|
void checkConcordiaResults(
|
|
const std::vector<ConcordiaSearchResult> & results,
|
|
long baseLineCount) {
|
|
long lineIndex = 1;
|
|
BOOST_FOREACH(ConcordiaSearchResult result, results) {
|
|
SUFFIX_MARKER_TYPE patternSize =
|
|
result.getTokenizedPattern()->getTokens().size();
|
|
if (patternSize > 0) {
|
|
if (result.getBestOverlay().size() != 1) {
|
|
reportError(baseLineCount + lineIndex,
|
|
"best overlay has more than one fragment.");
|
|
}
|
|
if (result.getBestOverlay().at(0).getMatchedLength()
|
|
!= patternSize) {
|
|
reportError(baseLineCount + lineIndex,
|
|
"best overlay fragment has different size than pattern.");
|
|
}
|
|
if (result.getBestOverlayScore() != 1) {
|
|
reportError(baseLineCount + lineIndex,
|
|
"best overlay score is not 1.");
|
|
}
|
|
if (result.getFragments().size() == 0) {
|
|
reportError(baseLineCount + lineIndex,
|
|
"there are no matched fragments.");
|
|
}
|
|
if (result.getFragments().at(0).getMatchedLength()
|
|
!= patternSize) {
|
|
reportError(baseLineCount + lineIndex,
|
|
"the first fragment does not cover the whole pattern.");
|
|
}
|
|
}
|
|
lineIndex++;
|
|
}
|
|
}
|
|
|
|
void performSearch(Concordia & concordia,
|
|
std::vector<std::string> & buffer,
|
|
long lineCount) {
|
|
long baseLineCount = 0;
|
|
if (lineCount % READ_BUFFER_LENGTH == 0) {
|
|
baseLineCount = lineCount - READ_BUFFER_LENGTH;
|
|
} else {
|
|
baseLineCount = (lineCount / READ_BUFFER_LENGTH)
|
|
* READ_BUFFER_LENGTH;
|
|
}
|
|
std::vector<ConcordiaSearchResult> results;
|
|
boost::posix_time::ptime timeStart =
|
|
boost::posix_time::microsec_clock::local_time();
|
|
// perform searching
|
|
BOOST_FOREACH(std::string pattern, buffer) {
|
|
results.push_back(*concordia.concordiaSearch(pattern));
|
|
}
|
|
boost::posix_time::ptime timeEnd =
|
|
boost::posix_time::microsec_clock::local_time();
|
|
boost::posix_time::time_duration msdiff =
|
|
timeEnd - timeStart;
|
|
|
|
int sentencesSearched = buffer.size();
|
|
buffer.clear();
|
|
|
|
long timeElapsed = msdiff.total_milliseconds();
|
|
if (timeElapsed == 0) {
|
|
timeElapsed++;
|
|
}
|
|
double speed = static_cast<double>(
|
|
1000 * sentencesSearched / timeElapsed);
|
|
std::cout << "\tSearched a portion of " <<
|
|
sentencesSearched << " sentences in "
|
|
<< timeElapsed << "ms. The speed: " <<
|
|
speed << " sentences per second" << std::endl;
|
|
std::cout << "\tChecking this portion..." << std::endl;
|
|
checkConcordiaResults(results, baseLineCount);
|
|
std::cout << "\tno errors." << std::endl;
|
|
std::cout << "\tTotal search progress: " <<
|
|
lineCount << " sentences searched." << std::endl;
|
|
}
|
|
|
|
int main(int argc, char** argv) {
|
|
po::options_description desc("Allowed options");
|
|
|
|
desc.add_options()
|
|
("help,h", "Display this message")
|
|
("config,c", boost::program_options::value<std::string>(),
|
|
"Concordia configuration file (required)")
|
|
("simple-search,s", boost::program_options::value<std::string>(),
|
|
"Pattern to be searched in the index")
|
|
("silent,n",
|
|
"While searching, do not output search results")
|
|
("anubis-search,a", boost::program_options::value<std::string>(),
|
|
"Pattern to be searched by anubis search in the index")
|
|
("concordia-search,x", boost::program_options::value<std::string>(),
|
|
"Pattern to be searched by concordia search in the index")
|
|
("read-file,r", boost::program_options::value<std::string>(),
|
|
"File to be read and added to index")
|
|
("test,t", boost::program_options::value<std::string>(),
|
|
"Run performance and correctness tests on file");
|
|
|
|
po::variables_map cli;
|
|
po::store(po::parse_command_line(argc, argv, desc), cli);
|
|
po::notify(cli);
|
|
|
|
if (cli.count("help")) {
|
|
std::cerr << desc << std::endl;
|
|
return 1;
|
|
}
|
|
|
|
std::string configFile;
|
|
if (cli.count("config")) {
|
|
configFile = cli["config"].as<std::string>();
|
|
} else {
|
|
std::cerr << "No Concordia configuration file given. Terminating."
|
|
<< std::endl;
|
|
return 1;
|
|
}
|
|
|
|
|
|
try {
|
|
std::cout << "\tInitializing concordia..." << std::endl;
|
|
boost::posix_time::ptime time_start =
|
|
boost::posix_time::microsec_clock::local_time();
|
|
Concordia concordia(configFile);
|
|
boost::posix_time::ptime time_end =
|
|
boost::posix_time::microsec_clock::local_time();
|
|
boost::posix_time::time_duration msdiff = time_end - time_start;
|
|
std::cout << "\tInitialization (loading index from disk "
|
|
<< "and regenerating SA) took: "
|
|
<< msdiff.total_milliseconds() << "ms." << std::endl;
|
|
|
|
std::cout << "\tWelcome to Concordia. Version = "
|
|
<< concordia.getVersion() << std::endl;
|
|
if (cli.count("simple-search")) {
|
|
std::string pattern = cli["simple-search"].as<std::string>();
|
|
std::cout << "\tSearching for pattern: \"" << pattern <<
|
|
"\"" << std::endl;
|
|
time_start = boost::posix_time::microsec_clock::local_time();
|
|
std::vector<SubstringOccurence> result =
|
|
concordia.simpleSearch(pattern);
|
|
time_end = boost::posix_time::microsec_clock::local_time();
|
|
msdiff = time_end - time_start;
|
|
std::cout << "\tFound: " << result.size() << " matches. "
|
|
<< "Search took: " <<
|
|
msdiff.total_milliseconds() << "ms." << std::endl;
|
|
if (!cli.count("silent")) {
|
|
BOOST_FOREACH(SubstringOccurence occurence, result) {
|
|
std::cout << "\t\tfound match in sentence number: "
|
|
<< occurence.getId() << std::endl;
|
|
}
|
|
}
|
|
} else if (cli.count("anubis-search")) {
|
|
std::string pattern = cli["anubis-search"].as<std::string>();
|
|
std::cout << "\tAnubis searching for pattern: \"" << pattern <<
|
|
"\"" << std::endl;
|
|
time_start = boost::posix_time::microsec_clock::local_time();
|
|
std::vector<AnubisSearchResult> result =
|
|
concordia.anubisSearch(pattern);
|
|
time_end = boost::posix_time::microsec_clock::local_time();
|
|
msdiff = time_end - time_start;
|
|
std::cout << "\tFound: " << result.size() << " matches. "
|
|
<< "Search took: " <<
|
|
msdiff.total_milliseconds() << "ms." << std::endl;
|
|
if (!cli.count("silent")) {
|
|
BOOST_FOREACH(AnubisSearchResult searchResult, result) {
|
|
std::cout << "\t\tfound matching sentence number: "
|
|
<< searchResult.getExampleId()
|
|
<< ", score: " << searchResult.getScore()
|
|
<< std::endl;
|
|
}
|
|
}
|
|
} else if (cli.count("concordia-search")) {
|
|
std::string pattern = cli["concordia-search"].as<std::string>();
|
|
std::cout << "\tConcordia searching for pattern: \"" << pattern <<
|
|
"\"" << std::endl;
|
|
time_start = boost::posix_time::microsec_clock::local_time();
|
|
boost::shared_ptr<ConcordiaSearchResult> result =
|
|
concordia.concordiaSearch(pattern);
|
|
time_end = boost::posix_time::microsec_clock::local_time();
|
|
msdiff = time_end - time_start;
|
|
|
|
std::cout << "\tPattern used: " << std::endl << "\t\t";
|
|
BOOST_FOREACH(TokenAnnotation annotation,
|
|
result->getTokenizedPattern()->getTokens()) {
|
|
std::cout << annotation.getValue() << " ";
|
|
}
|
|
std::cout << std::endl;
|
|
|
|
std::cout << "\tFound: " << result->getFragments().size()
|
|
<< " fragments. " << "Search took: " <<
|
|
msdiff.total_milliseconds() << "ms." << std::endl;
|
|
if (!cli.count("silent")) {
|
|
std::cout << "\tBest overlay (score="<<
|
|
result->getBestOverlayScore()<< "): " << std::endl;
|
|
BOOST_FOREACH(MatchedPatternFragment fragment,
|
|
result->getBestOverlay()) {
|
|
std::cout << "\t\tfragment [" << fragment.getStart()
|
|
<< "," << fragment.getEnd()
|
|
<< "] (exampleId, exampleOffset,"
|
|
<< " patternOffset, length): "
|
|
<< fragment.getExampleId() << ","
|
|
<< fragment.getExampleOffset() << ","
|
|
<< fragment.getPatternOffset() << ","
|
|
<< fragment.getMatchedLength()
|
|
<< std::endl;
|
|
}
|
|
|
|
std::cout << "\tAll pattern fragments: " << std::endl;
|
|
BOOST_FOREACH(MatchedPatternFragment fragment,
|
|
result->getFragments()) {
|
|
std::cout << "\t\tfragment [" << fragment.getStart()
|
|
<< "," << fragment.getEnd()
|
|
<< "] (exampleId, exampleOffset,"
|
|
<< " patternOffset, length): "
|
|
<< fragment.getExampleId() << ","
|
|
<< fragment.getExampleOffset() << ","
|
|
<< fragment.getPatternOffset() << ","
|
|
<< fragment.getMatchedLength()
|
|
<< std::endl;
|
|
}
|
|
}
|
|
} else if (cli.count("read-file")) {
|
|
std::string filePath = cli["read-file"].as<std::string>();
|
|
std::cout << "\tReading sentences from file: " << filePath <<
|
|
std::endl;
|
|
std::ifstream text_file(filePath.c_str());
|
|
std::string line;
|
|
if (text_file.is_open()) {
|
|
std::cout << "\tFile open" << std::endl;
|
|
long lineCount = 0;
|
|
std::vector<Example> buffer;
|
|
boost::posix_time::ptime timeStart =
|
|
boost::posix_time::microsec_clock::local_time();
|
|
while (getline(text_file, line)) {
|
|
lineCount++;
|
|
buffer.push_back(Example(line, lineCount));
|
|
if (lineCount % READ_BUFFER_LENGTH == 0) {
|
|
concordia.addAllExamples(buffer);
|
|
buffer.clear();
|
|
boost::posix_time::ptime timeEnd =
|
|
boost::posix_time::microsec_clock::local_time();
|
|
boost::posix_time::time_duration msdiff =
|
|
timeEnd - timeStart;
|
|
long timeElapsed = msdiff.total_milliseconds();
|
|
if (timeElapsed == 0) {
|
|
timeElapsed++;
|
|
}
|
|
double speed = static_cast<double>(
|
|
1000 * lineCount / timeElapsed);
|
|
std::cout << "\tRead and added to index " <<
|
|
lineCount << " sentences in " << timeElapsed
|
|
<< "ms. Current speed: " << speed <<
|
|
" sentences per second" << std::endl;
|
|
}
|
|
}
|
|
if (buffer.size() > 0) {
|
|
concordia.addAllExamples(buffer);
|
|
}
|
|
text_file.close();
|
|
boost::posix_time::ptime timeTotalEnd =
|
|
boost::posix_time::microsec_clock::local_time();
|
|
boost::posix_time::time_duration totalMsdiff =
|
|
timeTotalEnd - timeStart;
|
|
long totalTimeElapsed = totalMsdiff.total_milliseconds();
|
|
if (totalTimeElapsed == 0) {
|
|
totalTimeElapsed++;
|
|
}
|
|
|
|
double totalSpeed =
|
|
static_cast<double>(1000 * lineCount / totalTimeElapsed);
|
|
std::cout << "\tReading finished. Read and added to index "
|
|
<< lineCount << " sentences in " << totalTimeElapsed <<
|
|
"ms. Overall speed: " << totalSpeed <<
|
|
" sentences per second" << std::endl;
|
|
} else {
|
|
std::cerr << "Unable to open file: "<< filePath;
|
|
return 1;
|
|
}
|
|
} else if (cli.count("test")) {
|
|
std::string filePath = cli["test"].as<std::string>();
|
|
std::cout << "\tTesting on file: " << filePath <<
|
|
std::endl;
|
|
// adding to index
|
|
std::ifstream text_file(filePath.c_str());
|
|
std::string line;
|
|
if (text_file.is_open()) {
|
|
long lineCount = 0;
|
|
std::vector<Example> buffer;
|
|
boost::posix_time::ptime timeStart =
|
|
boost::posix_time::microsec_clock::local_time();
|
|
while (getline(text_file, line)) {
|
|
lineCount++;
|
|
buffer.push_back(Example(line, lineCount));
|
|
if (lineCount % READ_BUFFER_LENGTH == 0) {
|
|
concordia.addAllExamples(buffer);
|
|
buffer.clear();
|
|
boost::posix_time::ptime timeEnd =
|
|
boost::posix_time::microsec_clock::local_time();
|
|
boost::posix_time::time_duration msdiff =
|
|
timeEnd - timeStart;
|
|
long timeElapsed = msdiff.total_milliseconds();
|
|
if (timeElapsed == 0) {
|
|
timeElapsed++;
|
|
}
|
|
double speed = static_cast<double>(
|
|
1000 * lineCount / timeElapsed);
|
|
std::cout << "\tRead and added to index " <<
|
|
lineCount << " sentences in " << timeElapsed
|
|
<< "ms. Current speed: " << speed <<
|
|
" sentences per second" << std::endl;
|
|
}
|
|
}
|
|
if (buffer.size() > 0) {
|
|
concordia.addAllExamples(buffer);
|
|
}
|
|
text_file.close();
|
|
boost::posix_time::ptime timeTotalEnd =
|
|
boost::posix_time::microsec_clock::local_time();
|
|
boost::posix_time::time_duration totalMsdiff =
|
|
timeTotalEnd - timeStart;
|
|
long totalTimeElapsed = totalMsdiff.total_milliseconds();
|
|
if (totalTimeElapsed == 0) {
|
|
totalTimeElapsed++;
|
|
}
|
|
double totalSpeed =
|
|
static_cast<double>(1000 * lineCount / totalTimeElapsed);
|
|
std::cout << "\tReading finished. Read and added to index "
|
|
<< lineCount << " sentences in " << totalTimeElapsed <<
|
|
"ms. Overall speed: " << totalSpeed <<
|
|
" sentences per second" << std::endl;
|
|
} else {
|
|
std::cerr << "Unable to open file: "<< filePath;
|
|
return 1;
|
|
}
|
|
|
|
// generating SA
|
|
std::cout << "\tGenerating SA from RAM..." << std::endl;
|
|
boost::posix_time::ptime SAStart =
|
|
boost::posix_time::microsec_clock::local_time();
|
|
concordia.refreshSAfromRAM();
|
|
boost::posix_time::ptime SAEnd =
|
|
boost::posix_time::microsec_clock::local_time();
|
|
boost::posix_time::time_duration SAdiff =
|
|
SAEnd - SAStart;
|
|
long SAtimeElapsed = SAdiff.total_milliseconds();
|
|
std::cout << "\tSA generated in " << SAtimeElapsed
|
|
<< "ms." << std::endl;
|
|
|
|
// searching
|
|
std::ifstream test_file(filePath.c_str());
|
|
if (test_file.is_open()) {
|
|
long lineCount = 0;
|
|
std::vector<std::string> buffer;
|
|
while (getline(test_file, line)) {
|
|
lineCount++;
|
|
buffer.push_back(line);
|
|
if (lineCount % READ_BUFFER_LENGTH == 0) {
|
|
performSearch(concordia, buffer, lineCount);
|
|
}
|
|
}
|
|
if (buffer.size() > 0) {
|
|
performSearch(concordia, buffer, lineCount);
|
|
}
|
|
test_file.close();
|
|
std::cout << "\tSearching finished. No errors reported."
|
|
<< std::endl;
|
|
|
|
} else {
|
|
std::cerr << "Unable to open file: "<< filePath;
|
|
return 1;
|
|
}
|
|
|
|
} else {
|
|
std::cerr << "One of the options: simple-search, anubis-search, "
|
|
<< "concordia-search or read-file must be provided."
|
|
<< "See the options specification: "
|
|
<< std::endl << desc << std::endl;
|
|
return 1;
|
|
}
|
|
|
|
std::cout << "\tConcordia operation completed without errors."
|
|
<< std::endl;
|
|
} catch(ConcordiaException & e) {
|
|
std::cerr << "ConcordiaException caught with message: "
|
|
<< std::endl
|
|
<< e.what()
|
|
<< std::endl
|
|
<< "Terminating execution."
|
|
<< std::endl;
|
|
return 1;
|
|
} catch(std::exception & e) {
|
|
std::cerr << "Unexpected exception caught with message: "
|
|
<< std::endl
|
|
<< e.what()
|
|
<< std::endl
|
|
<< "Terminating execution."
|
|
<< std::endl;
|
|
return 1;
|
|
}
|
|
return 0;
|
|
}
|