rjawor/concordia-library
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

removed using namespace std

Browse Source 
Former-commit-id: dbb5129e1f94d83eca887ada0f89d6bb45250f1e
This commit is contained in:
rjawor 2015-04-15 14:14:10 +02:00
parent a09999c130
commit 0d4bdf12de
49 changed files with 333 additions and 392 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
concordia-console/concordia-console.cpp
View File

@ -84,11 +84,11 @@ int main(int argc, char** argv) {
std::string filePath = cli["read-file"].as<std::string>();
std::cout << "\tReading sentences from file: " << filePath <<
std::endl;
ifstream text_file(filePath.c_str());
std::ifstream text_file(filePath.c_str());
std::string line;
if (text_file.is_open()) {
long lineCount = 0;
vector<Example> buffer;
std::vector<Example> buffer;
boost::posix_time::ptime timeStart =
boost::posix_time::microsec_clock::local_time();
while (getline(text_file, line)) {
@ -147,7 +147,7 @@ int main(int argc, char** argv) {
<< "Terminating execution."
<< std::endl;
return 1;
} catch(exception & e) {
} catch(std::exception & e) {
std::cerr << "Unexpected exception caught with message: "
<< std::endl
<< e.what()

2
concordia/anubis_search_result.hpp
View File

@ -8,8 +8,6 @@
*/
using namespace std;
class AnubisSearchResult {
public:
explicit AnubisSearchResult(const SUFFIX_MARKER_TYPE & exampleId,

99
concordia/anubis_searcher.cpp
View File

@ -18,8 +18,9 @@ std::vector<AnubisSearchResult> AnubisSearcher::anubisSearch(
boost::shared_ptr<std::vector<saidx_t> > SA,
const std::vector<INDEX_CHARACTER_TYPE> & pattern)
throw(ConcordiaException) {
boost::shared_ptr<TmMatchesMap> tmMatchesMap = getTmMatches(T, markers, SA, pattern);
boost::shared_ptr<TmMatchesMap> tmMatchesMap =
getTmMatches(T, markers, SA, pattern);
// get the tmMatches list sorted descending by score
std::vector<AnubisSearchResult> result;
return result;
@ -31,7 +32,6 @@ boost::shared_ptr<TmMatchesMap> AnubisSearcher::getTmMatches(
boost::shared_ptr<std::vector<saidx_t> > SA,
const std::vector<INDEX_CHARACTER_TYPE> & pattern)
throw(ConcordiaException) {
std::vector<sauchar_t> patternVector =
Utils::indexVectorToSaucharVector(pattern);
@ -45,7 +45,7 @@ boost::shared_ptr<TmMatchesMap> AnubisSearcher::getTmMatches(
int highResOffset = offset * sizeof(INDEX_CHARACTER_TYPE);
std::vector<sauchar_t> currentPattern(
patternVector.begin()+highResOffset, patternVector.end());
saidx_t patternLength = 0;
saidx_t size = SA->size();
saidx_t left = 0;
@ -64,8 +64,8 @@ boost::shared_ptr<TmMatchesMap> AnubisSearcher::getTmMatches(
saidx_t localLeft;
size = sa_search(T->data(), (saidx_t) T->size(),
(const sauchar_t *) patternArray, patternLength,
SAleft, size, &localLeft);
(const sauchar_t *) patternArray, patternLength,
SAleft, size, &localLeft);
left += localLeft;
@ -75,20 +75,24 @@ boost::shared_ptr<TmMatchesMap> AnubisSearcher::getTmMatches(
// Add to tm matches map results surrounding the main stream.
// from left
for (saidx_t i = prevLeft; i < left; i++) {
_addToMap(SA, markers, tmMatchesMap, i, pattern.size(), (patternLength / sizeof(INDEX_CHARACTER_TYPE)) -1, offset);
}
_addToMap(SA, markers, tmMatchesMap, i, pattern.size(),
(patternLength / sizeof(INDEX_CHARACTER_TYPE)) -1,
offset);
}
// from right
for (saidx_t i = left+size; i < prevLeft+prevSize; i++) {
_addToMap(SA, markers, tmMatchesMap, i, pattern.size(), (patternLength / sizeof(INDEX_CHARACTER_TYPE)) -1, offset);
}
_addToMap(SA, markers, tmMatchesMap, i, pattern.size(),
(patternLength / sizeof(INDEX_CHARACTER_TYPE)) - 1,
offset);
}
}
} while (patternLength < currentPattern.size() && size > 0);
if (size > 0) {
for (saidx_t i = left; i < left+size; i++) {
_addToMap(SA, markers, tmMatchesMap, i, pattern.size(), patternLength / sizeof(INDEX_CHARACTER_TYPE), offset);
}
_addToMap(SA, markers, tmMatchesMap, i, pattern.size(),
patternLength / sizeof(INDEX_CHARACTER_TYPE), offset);
}
}
}
@ -121,16 +125,16 @@ std::vector<SubstringOccurence> AnubisSearcher::lcpSearch(
saidx_t localLeft;
size = sa_search(T->data(), (saidx_t) T->size(),
(const sauchar_t *) patternArray, patternLength,
SAleft, size, &localLeft);
SAleft, size, &localLeft);
left += localLeft;
SAleft += localLeft;
} while (patternLength < pattern.size() && size > 0);
vector<SubstringOccurence> result;
std::vector<SubstringOccurence> result;
if (size == 0) {
// The search managed to find exactly the longest common prefixes.
length = patternLength - sizeof(INDEX_CHARACTER_TYPE);
if (length > 0) {
// Get the results of the previous search
@ -149,27 +153,28 @@ std::vector<SubstringOccurence> AnubisSearcher::lcpSearch(
}
void AnubisSearcher::_collectResults(
vector<SubstringOccurence> & result,
std::vector<SubstringOccurence> & result,
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,
boost::shared_ptr<std::vector<saidx_t> > SA,
saidx_t left, saidx_t size) {
for (saidx_t i = 0; i < size; i++) {
saidx_t resultPos = SA->at(left + i);
if (resultPos % sizeof(INDEX_CHARACTER_TYPE) == 0) {
SUFFIX_MARKER_TYPE marker = markers->at(resultPos / sizeof(INDEX_CHARACTER_TYPE));
SUFFIX_MARKER_TYPE marker =
markers->at(resultPos / sizeof(INDEX_CHARACTER_TYPE));
result.push_back(SubstringOccurence(marker));
}
}
}
void AnubisSearcher::_addToMap(boost::shared_ptr<std::vector<saidx_t> > SA,
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,
boost::shared_ptr<TmMatchesMap> tmMatchesMap,
saidx_t sa_pos,
SUFFIX_MARKER_TYPE totalPatternLength,
SUFFIX_MARKER_TYPE matchedFragmentLength,
SUFFIX_MARKER_TYPE patternOffset) {
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,
boost::shared_ptr<TmMatchesMap> tmMatchesMap,
saidx_t sa_pos,
SUFFIX_MARKER_TYPE totalPatternLength,
SUFFIX_MARKER_TYPE matchedFragmentLength,
SUFFIX_MARKER_TYPE patternOffset) {
SubstringOccurence occurence;
if (_getOccurenceFromSA(SA, markers, sa_pos, occurence)) {
_addOccurenceToMap(tmMatchesMap,
@ -178,24 +183,24 @@ void AnubisSearcher::_addToMap(boost::shared_ptr<std::vector<saidx_t> > SA,
matchedFragmentLength,
patternOffset);
}
}
bool AnubisSearcher::_getOccurenceFromSA(
boost::shared_ptr<std::vector<saidx_t> > SA,
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,
saidx_t sa_pos,
SubstringOccurence & occurence) {
saidx_t resultPos = SA->at(sa_pos);
if (resultPos % sizeof(INDEX_CHARACTER_TYPE) == 0) {
SUFFIX_MARKER_TYPE marker = markers->at(resultPos / sizeof(INDEX_CHARACTER_TYPE));
occurence.enterDataFromMarker(marker);
}
}
void AnubisSearcher::_addOccurenceToMap(boost::shared_ptr<TmMatchesMap> tmMatchesMap,
bool AnubisSearcher::_getOccurenceFromSA(
boost::shared_ptr<std::vector<saidx_t> > SA,
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,
saidx_t sa_pos,
SubstringOccurence & occurence) {
saidx_t resultPos = SA->at(sa_pos);
if (resultPos % sizeof(INDEX_CHARACTER_TYPE) == 0) {
SUFFIX_MARKER_TYPE marker =
markers->at(resultPos / sizeof(INDEX_CHARACTER_TYPE));
occurence.enterDataFromMarker(marker);
}
}
void AnubisSearcher::_addOccurenceToMap(
boost::shared_ptr<TmMatchesMap> tmMatchesMap,
SubstringOccurence & occurence,
SUFFIX_MARKER_TYPE totalPatternLength,
SUFFIX_MARKER_TYPE matchedFragmentLength,
@ -213,16 +218,12 @@ void AnubisSearcher::_addOccurenceToMap(boost::shared_ptr<TmMatchesMap> tmMatche
SUFFIX_MARKER_TYPE key = occurence.getId();
tmMatchesMap->insert(key, tmMatches);
}
// add intervals to tmMatches
tmMatches->addExampleInterval(
occurence.getOffset(),
occurence.getOffset() + matchedFragmentLength
);
occurence.getOffset(),
occurence.getOffset() + matchedFragmentLength);
tmMatches->addPatternInterval(
patternOffset,
patternOffset + matchedFragmentLength
);
patternOffset + matchedFragmentLength);
}

11
concordia/anubis_searcher.hpp
View File

@ -10,6 +10,7 @@
#include "concordia/anubis_search_result.hpp"
#include "concordia/tm_matches.hpp"
#include<vector>
#include <divsufsort.h>
/*!
@ -17,8 +18,6 @@
*/
using namespace std;
class AnubisSearcher {
public:
explicit AnubisSearcher();
@ -49,7 +48,7 @@ public:
SUFFIX_MARKER_TYPE & length) throw(ConcordiaException);
private:
void _collectResults(vector<SubstringOccurence> & result,
void _collectResults(std::vector<SubstringOccurence> & result,
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,
boost::shared_ptr<std::vector<saidx_t> > SA,
saidx_t left, saidx_t size);
@ -63,9 +62,9 @@ private:
SUFFIX_MARKER_TYPE patternOffset);
bool _getOccurenceFromSA(boost::shared_ptr<std::vector<saidx_t> > SA,
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,
saidx_t sa_pos,
SubstringOccurence & occurence);
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,
saidx_t sa_pos,
SubstringOccurence & occurence);
void _addOccurenceToMap(boost::shared_ptr<TmMatchesMap> tmMatchesMap,
SubstringOccurence & occurence,

4
concordia/common/text_utils.cpp
View File

@ -10,10 +10,10 @@ TextUtils::TextUtils() {
StringCaseConverterManager::getInstance().getUpperCaseConverter("pl");
}
string TextUtils::toLowerCase(const string & text) {
std::string TextUtils::toLowerCase(const std::string & text) {
return simpleConvert(*_lowerConverter, text);
}
string TextUtils::toUpperCase(const string & text) {
std::string TextUtils::toUpperCase(const std::string & text) {
return simpleConvert(*_upperConverter, text);
}

7
concordia/common/text_utils.hpp
View File

@ -7,9 +7,6 @@
#include "utf8case/case_converter_factory.hpp"
#include "utf8case/string_case_converter_manager.hpp"
using namespace std;
/*! Utility class for performing simple string operations.
*/
class TextUtils {
@ -26,13 +23,13 @@ public:
\param text input string
\returns lower case version of the input string.
*/
string toLowerCase(const string & text);
std::string toLowerCase(const std::string & text);
/*! A method for converting all string letters to upper case.
\param text input string
\returns upper case version of the input string.
*/
string toUpperCase(const string & text);
std::string toUpperCase(const std::string & text);
private:
explicit TextUtils(TextUtils const&); // Don't Implement

16
concordia/common/utils.cpp
View File

@ -7,35 +7,35 @@ Utils::Utils() {
Utils::~Utils() {
}
void Utils::writeIndexCharacter(ofstream & file,
void Utils::writeIndexCharacter(std::ofstream & file,
INDEX_CHARACTER_TYPE character) {
file.write(reinterpret_cast<char *>(&character), sizeof(character));
}
void Utils::writeMarker(ofstream & file,
void Utils::writeMarker(std::ofstream & file,
SUFFIX_MARKER_TYPE marker) {
file.write(reinterpret_cast<char *>(&marker), sizeof(marker));
}
INDEX_CHARACTER_TYPE Utils::readIndexCharacter(ifstream & file) {
INDEX_CHARACTER_TYPE Utils::readIndexCharacter(std::ifstream & file) {
INDEX_CHARACTER_TYPE character;
file.read(reinterpret_cast<char *>(&character), sizeof(character));
return character;
}
SUFFIX_MARKER_TYPE Utils::readMarker(ifstream & file) {
SUFFIX_MARKER_TYPE Utils::readMarker(std::ifstream & file) {
SUFFIX_MARKER_TYPE marker;
file.read(reinterpret_cast<char *>(&marker), sizeof(marker));
return marker;
}
sauchar_t * Utils::indexVectorToSaucharArray(
const vector<INDEX_CHARACTER_TYPE> & input) {
const std::vector<INDEX_CHARACTER_TYPE> & input) {
const int kArraySize = input.size()*sizeof(INDEX_CHARACTER_TYPE);
sauchar_t * patternArray =
new sauchar_t[kArraySize];
int pos = 0;
for (vector<INDEX_CHARACTER_TYPE>::const_iterator it = input.begin();
for (std::vector<INDEX_CHARACTER_TYPE>::const_iterator it = input.begin();
it != input.end(); ++it) {
_insertCharToSaucharArray(patternArray, *it, pos);
pos += sizeof(INDEX_CHARACTER_TYPE);
@ -44,9 +44,9 @@ sauchar_t * Utils::indexVectorToSaucharArray(
}
std::vector<sauchar_t> Utils::indexVectorToSaucharVector(
const vector<INDEX_CHARACTER_TYPE> & input) {
const std::vector<INDEX_CHARACTER_TYPE> & input) {
std::vector<sauchar_t> result;
for (vector<INDEX_CHARACTER_TYPE>::const_iterator it = input.begin();
for (std::vector<INDEX_CHARACTER_TYPE>::const_iterator it = input.begin();
it != input.end(); ++it) {
appendCharToSaucharVector(result, *it);
}

18
concordia/common/utils.hpp
View File

@ -11,8 +11,6 @@
#include "concordia/concordia_exception.hpp"
#include <divsufsort.h>
using namespace std;
class Utils {
public:
explicit Utils();
@ -21,21 +19,21 @@ public:
*/
virtual ~Utils();
static void writeIndexCharacter(ofstream & file,
static void writeIndexCharacter(std::ofstream & file,
INDEX_CHARACTER_TYPE character);
static void writeMarker(ofstream & file,
static void writeMarker(std::ofstream & file,
SUFFIX_MARKER_TYPE marker);
static INDEX_CHARACTER_TYPE readIndexCharacter(ifstream & file);
static INDEX_CHARACTER_TYPE readIndexCharacter(std::ifstream & file);
static SUFFIX_MARKER_TYPE readMarker(ifstream & file);
static SUFFIX_MARKER_TYPE readMarker(std::ifstream & file);
static sauchar_t * indexVectorToSaucharArray(
const vector<INDEX_CHARACTER_TYPE> & input);
const std::vector<INDEX_CHARACTER_TYPE> & input);
static std::vector<sauchar_t> indexVectorToSaucharVector(
const vector<INDEX_CHARACTER_TYPE> & input);
const std::vector<INDEX_CHARACTER_TYPE> & input);
static void appendCharToSaucharVector(
boost::shared_ptr<std::vector<sauchar_t> > vector,
@ -70,8 +68,8 @@ private:
template <typename T>
void Utils::printVector(const std::vector<T> & vector) {
for (int i = 0; i < vector.size(); i++) {
cout << static_cast<int>(vector.at(i)) << " ";
std::cout << static_cast<int>(vector.at(i)) << " ";
}
cout << endl;
std::cout << std::endl;
}
#endif

21
concordia/concordia.cpp
View File

@ -62,12 +62,13 @@ void Concordia::loadRAMIndexFromDisk() throw(ConcordiaException) {
&& boost::filesystem::exists(_config->getMarkersFilePath())) {
// reading index from file
_T->clear();
ifstream hashedIndexFile;
hashedIndexFile.open(_config->getHashedIndexFilePath().c_str(), ios::in
| ios::ate | ios::binary);
std::ifstream hashedIndexFile;
hashedIndexFile.open(
_config->getHashedIndexFilePath().c_str(), std::ios::in
| std::ios::ate | std::ios::binary);
saidx_t hiFileSize = hashedIndexFile.tellg();
if (hiFileSize > 0) {
hashedIndexFile.seekg(0, ios::beg);
hashedIndexFile.seekg(0, std::ios::beg);
while (!hashedIndexFile.eof()) {
INDEX_CHARACTER_TYPE character =
@ -82,12 +83,12 @@ void Concordia::loadRAMIndexFromDisk() throw(ConcordiaException) {
// reading markers from file
_markers->clear();
ifstream markersFile;
markersFile.open(_config->getMarkersFilePath().c_str(), ios::in
| ios::ate | ios::binary);
std::ifstream markersFile;
markersFile.open(_config->getMarkersFilePath().c_str(), std::ios::in
| std::ios::ate | std::ios::binary);
saidx_t maFileSize = markersFile.tellg();
if (maFileSize > 0) {
markersFile.seekg(0, ios::beg);
markersFile.seekg(0, std::ios::beg);
while (!markersFile.eof()) {
SUFFIX_MARKER_TYPE marker =
@ -132,7 +133,7 @@ void Concordia::_initializeIndex() throw(ConcordiaException) {
}
std::vector<SubstringOccurence> Concordia::simpleSearch(
const string & pattern)
const std::string & pattern)
throw(ConcordiaException) {
if (_T->size() > 0) {
return _searcher->simpleSearch(_hashGenerator, _T,
@ -144,7 +145,7 @@ std::vector<SubstringOccurence> Concordia::simpleSearch(
}
std::vector<AnubisSearchResult> Concordia::anubisSearch(
const string & pattern)
const std::string & pattern)
throw(ConcordiaException) {
if (_T->size() > 0) {
return _searcher->anubisSearch(_hashGenerator, _T,

8
concordia/concordia_config.cpp
View File

@ -13,13 +13,13 @@
#define NAMED_ENTITIES_PARAM "named_entities_path"
#define STOP_SYMBOLS_PARAM "stop_symbols_path"
ConcordiaConfig::ConcordiaConfig(const string & configFilePath)
ConcordiaConfig::ConcordiaConfig(const std::string & configFilePath)
throw(ConcordiaException) {
try {
_config.readFile(configFilePath.c_str());
} catch(ParseException & e) {
} catch(libconfig::ParseException & e) {
throw ConcordiaException("Error parsing config file: "+configFilePath);
} catch(FileIOException & e) {
} catch(libconfig::FileIOException & e) {
throw ConcordiaException("I/O error reading config file: "
+configFilePath);
}
@ -49,7 +49,7 @@ ConcordiaConfig::ConcordiaConfig(const string & configFilePath)
ConcordiaConfig::~ConcordiaConfig() {
}
string ConcordiaConfig::_readConfigParameterStr(const string & name)
std::string ConcordiaConfig::_readConfigParameterStr(const std::string & name)
throw(ConcordiaException) {
if (!_config.exists(name)) {
throw ConcordiaException("Config error: "+name+" setting not found");

51
concordia/concordia_config.hpp
View File

@ -7,9 +7,6 @@
#include "concordia/concordia_exception.hpp"
using namespace std;
using namespace libconfig;
/*!
Class representing the Concordia configuration.
*/
@ -20,8 +17,8 @@ public:
\param configFilePath path of the configuration file (see \ref running3 for file specification).
\throws ConcordiaException
*/
explicit ConcordiaConfig(const string & configFilePath)
throw(ConcordiaException);
explicit ConcordiaConfig(const std::string & configFilePath)
throw(ConcordiaException);
/*! Destructor.
*/
@ -30,70 +27,70 @@ public:
/*! Getter for the puddle file path parameter.
\returns file path of the puddle tagset
*/
string & getPuddleTagsetFilePath() {
std::string & getPuddleTagsetFilePath() {
return _puddleTagsetFilePath;
}
string & getWordMapFilePath() {
std::string & getWordMapFilePath() {
return _wordMapFilePath;
}
string & getHashedIndexFilePath() {
std::string & getHashedIndexFilePath() {
return _hashedIndexFilePath;
}
string & getMarkersFilePath() {
std::string & getMarkersFilePath() {
return _markersFilePath;
}
string & getSuffixArrayFilePath() {
std::string & getSuffixArrayFilePath() {
return _suffixArrayFilePath;
}
string & getHtmlTagsFilePath() {
std::string & getHtmlTagsFilePath() {
return _htmlTagsFilePath;
}
string & getSpaceSymbolsFilePath() {
std::string & getSpaceSymbolsFilePath() {
return _spaceSymbolsFilePath;
}
string & getStopWordsFilePath() {
std::string & getStopWordsFilePath() {
return _stopWordsFilePath;
}
string & getNamedEntitiesFilePath() {
std::string & getNamedEntitiesFilePath() {
return _namedEntitiesFilePath;
}
string & getStopSymbolsFilePath() {
std::string & getStopSymbolsFilePath() {
return _stopSymbolsFilePath;
}
private:
Config _config;
libconfig::Config _config;
string _puddleTagsetFilePath;
std::string _puddleTagsetFilePath;
string _wordMapFilePath;
std::string _wordMapFilePath;
string _hashedIndexFilePath;
std::string _hashedIndexFilePath;
string _markersFilePath;
std::string _markersFilePath;
string _suffixArrayFilePath;
std::string _suffixArrayFilePath;
string _htmlTagsFilePath;
std::string _htmlTagsFilePath;
string _spaceSymbolsFilePath;
std::string _spaceSymbolsFilePath;
string _stopWordsFilePath;
std::string _stopWordsFilePath;
string _namedEntitiesFilePath;
std::string _namedEntitiesFilePath;
string _stopSymbolsFilePath;
std::string _stopSymbolsFilePath;
string _readConfigParameterStr(const string & name)
std::string _readConfigParameterStr(const std::string & name)
throw(ConcordiaException);
};

2
concordia/concordia_exception.cpp
View File

@ -4,7 +4,7 @@ ConcordiaException::ConcordiaException() throw():
_message("Concordia exception") {
}
ConcordiaException::ConcordiaException(const string & message) throw():
ConcordiaException::ConcordiaException(const std::string & message) throw():
_message(message) {
}

8
concordia/concordia_exception.hpp
View File

@ -5,12 +5,10 @@
#include<string>
#include<string.h>
using namespace std;
/*!
Class representing an internal exception thrown in the Concordia library.
*/
class ConcordiaException : public exception {
class ConcordiaException : public std::exception {
public:
/*! Constructor.
*/
@ -19,7 +17,7 @@ public:
/*! Constructor with a message.
\param message message of the exception
*/
explicit ConcordiaException(const string & message) throw();
explicit ConcordiaException(const std::string & message) throw();
/*! Destructor.
*/
@ -30,7 +28,7 @@ public:
virtual const char* what() const throw();
private:
string _message;
std::string _message;
};
#endif

62
concordia/concordia_index.cpp
View File

@ -7,8 +7,8 @@
#include <iostream>
#include <climits>
ConcordiaIndex::ConcordiaIndex(const string & hashedIndexFilePath,
const string & markersFilePath)
ConcordiaIndex::ConcordiaIndex(const std::string & hashedIndexFilePath,
const std::string & markersFilePath)
throw(ConcordiaException) :
_hashedIndexFilePath(hashedIndexFilePath),
_markersFilePath(markersFilePath) {
@ -17,15 +17,15 @@ ConcordiaIndex::ConcordiaIndex(const string & hashedIndexFilePath,
ConcordiaIndex::~ConcordiaIndex() {
}
boost::shared_ptr<vector<saidx_t> > ConcordiaIndex::generateSuffixArray(
boost::shared_ptr<vector<sauchar_t> > T) {
boost::shared_ptr<std::vector<saidx_t> > ConcordiaIndex::generateSuffixArray(
boost::shared_ptr<std::vector<sauchar_t> > T) {
saidx_t * SA_array = new saidx_t[T->size()];
if (divsufsort(T->data(), SA_array, (saidx_t) T->size()) != 0) {
throw ConcordiaException("Error creating suffix array.");
}
boost::shared_ptr<vector<saidx_t> > result =
boost::shared_ptr<vector<saidx_t> >(new vector<saidx_t>);
boost::shared_ptr<std::vector<saidx_t> > result =
boost::shared_ptr<std::vector<saidx_t> >(new std::vector<saidx_t>);
for (int i = 0; i < T->size(); i++) {
result->push_back(SA_array[i]);
}
@ -36,15 +36,15 @@ boost::shared_ptr<vector<saidx_t> > ConcordiaIndex::generateSuffixArray(
void ConcordiaIndex::addExample(
boost::shared_ptr<HashGenerator> hashGenerator,
boost::shared_ptr<vector<sauchar_t> > T,
boost::shared_ptr<vector<SUFFIX_MARKER_TYPE> > markers,
boost::shared_ptr<std::vector<sauchar_t> > T,
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,
const Example & example) {
ofstream hashedIndexFile;
hashedIndexFile.open(_hashedIndexFilePath.c_str(), ios::out|
ios::app|ios::binary);
ofstream markersFile;
markersFile.open(_markersFilePath.c_str(), ios::out|
ios::app|ios::binary);
std::ofstream hashedIndexFile;
hashedIndexFile.open(_hashedIndexFilePath.c_str(), std::ios::out|
std::ios::app|std::ios::binary);
std::ofstream markersFile;
markersFile.open(_markersFilePath.c_str(), std::ios::out|
std::ios::app|std::ios::binary);
_addSingleExample(hashedIndexFile, markersFile, hashGenerator,
T, markers, example);
hashedIndexFile.close();
@ -54,15 +54,15 @@ void ConcordiaIndex::addExample(
void ConcordiaIndex::addAllExamples(
boost::shared_ptr<HashGenerator> hashGenerator,
boost::shared_ptr<vector<sauchar_t> > T,
boost::shared_ptr<vector<SUFFIX_MARKER_TYPE> > markers,
const vector<Example> & examples) {
ofstream hashedIndexFile;
hashedIndexFile.open(_hashedIndexFilePath.c_str(), ios::out|
ios::app|ios::binary);
ofstream markersFile;
markersFile.open(_markersFilePath.c_str(), ios::out|
ios::app|ios::binary);
boost::shared_ptr<std::vector<sauchar_t> > T,
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,
const std::vector<Example> & examples) {
std::ofstream hashedIndexFile;
hashedIndexFile.open(_hashedIndexFilePath.c_str(), std::ios::out|
std::ios::app|std::ios::binary);
std::ofstream markersFile;
markersFile.open(_markersFilePath.c_str(), std::ios::out|
std::ios::app|std::ios::binary);
BOOST_FOREACH(Example example, examples) {
_addSingleExample(hashedIndexFile, markersFile, hashGenerator,
@ -75,16 +75,16 @@ void ConcordiaIndex::addAllExamples(
}
void ConcordiaIndex::_addSingleExample(
ofstream & hashedIndexFile,
ofstream & markersFile,
boost::shared_ptr<HashGenerator> hashGenerator,
boost::shared_ptr<std::vector<sauchar_t> > T,
boost::shared_ptr<vector<SUFFIX_MARKER_TYPE> > markers,
const Example & example) {
vector<INDEX_CHARACTER_TYPE> hash
std::ofstream & hashedIndexFile,
std::ofstream & markersFile,
boost::shared_ptr<HashGenerator> hashGenerator,
boost::shared_ptr<std::vector<sauchar_t> > T,
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,
const Example & example) {
std::vector<INDEX_CHARACTER_TYPE> hash
= hashGenerator->generateHash(example.getSentence());
int offset = 0;
for (vector<INDEX_CHARACTER_TYPE>::iterator it = hash.begin();
for (std::vector<INDEX_CHARACTER_TYPE>::iterator it = hash.begin();
it != hash.end(); ++it) {
INDEX_CHARACTER_TYPE character = *it;
Utils::writeIndexCharacter(hashedIndexFile, character);

36
concordia/concordia_index.hpp
View File

@ -18,12 +18,10 @@
*/
using namespace std;
class ConcordiaIndex {
public:
explicit ConcordiaIndex(const string & hashedIndexFilePath,
const string & markersFilePath)
explicit ConcordiaIndex(const std::string & hashedIndexFilePath,
const std::string & markersFilePath)
throw(ConcordiaException);
/*! Destructor.
@ -32,31 +30,31 @@ public:
void addExample(
boost::shared_ptr<HashGenerator> hashGenerator,
boost::shared_ptr<vector<sauchar_t> > T,
boost::shared_ptr<vector<SUFFIX_MARKER_TYPE> > markers,
boost::shared_ptr<std::vector<sauchar_t> > T,
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,
const Example & example);
void addAllExamples(
boost::shared_ptr<HashGenerator> hashGenerator,
boost::shared_ptr<vector<sauchar_t> > T,
boost::shared_ptr<vector<SUFFIX_MARKER_TYPE> > markers,
const vector<Example> & examples);
boost::shared_ptr<std::vector<sauchar_t> > T,
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,
const std::vector<Example> & examples);
boost::shared_ptr<vector<saidx_t> > generateSuffixArray(
boost::shared_ptr<vector<sauchar_t> > T);
boost::shared_ptr<std::vector<saidx_t> > generateSuffixArray(
boost::shared_ptr<std::vector<sauchar_t> > T);
private:
// Add example to disk index and update RAM index.
void _addSingleExample(ofstream & hashedIndexFile,
ofstream & markersFile,
boost::shared_ptr<HashGenerator> hashGenerator,
boost::shared_ptr<std::vector<sauchar_t> > T,
boost::shared_ptr<vector<SUFFIX_MARKER_TYPE> > markers,
const Example & example);
void _addSingleExample(std::ofstream & hashedIndexFile,
std::ofstream & markersFile,
boost::shared_ptr<HashGenerator> hashGenerator,
boost::shared_ptr<std::vector<sauchar_t> > T,
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,
const Example & example);
string _hashedIndexFilePath;
std::string _hashedIndexFilePath;
string _markersFilePath;
std::string _markersFilePath;
};
#endif

3
concordia/example.cpp
View File

@ -1,8 +1,7 @@
#include "concordia/example.hpp"
#include <climits>
#include <iostream>
Example::Example(const string & sentence, const SUFFIX_MARKER_TYPE & id)
Example::Example(const std::string & sentence, const SUFFIX_MARKER_TYPE & id)
throw(ConcordiaException):
_sentence(sentence),
_id(id) {

11
concordia/example.hpp
View File

@ -10,18 +10,17 @@
*/
using namespace std;
class Example {
public:
explicit Example(const string & sentence, const SUFFIX_MARKER_TYPE & id)
throw(ConcordiaException);
explicit Example(const std::string & sentence,
const SUFFIX_MARKER_TYPE & id)
throw(ConcordiaException);
/*! Destructor.
*/
virtual ~Example();
string getSentence() const {
std::string getSentence() const {
return _sentence;
}
@ -30,7 +29,7 @@ public:
}
private:
string _sentence;
std::string _sentence;
SUFFIX_MARKER_TYPE _id;
};

23
concordia/hash_generator.cpp
View File

@ -15,7 +15,7 @@ HashGenerator::HashGenerator(boost::shared_ptr<ConcordiaConfig> config)
_sentenceAnonymizer(boost::shared_ptr<SentenceAnonymizer>(
new SentenceAnonymizer(config))) {
if (boost::filesystem::exists(_wordMapFilePath)) {
ifstream ifs(_wordMapFilePath.c_str(), std::ios::binary);
std::ifstream ifs(_wordMapFilePath.c_str(), std::ios::binary);
boost::archive::binary_iarchive ia(ifs);
boost::shared_ptr<WordMap> restoredWordMap(new WordMap);
ia >> *_wordMap;
@ -25,16 +25,16 @@ HashGenerator::HashGenerator(boost::shared_ptr<ConcordiaConfig> config)
HashGenerator::~HashGenerator() {
}
vector<INDEX_CHARACTER_TYPE> HashGenerator::generateHash(
const string & sentence) throw(ConcordiaException) {
vector<INDEX_CHARACTER_TYPE> result;
vector<string> tokenTexts = generateTokenVector(sentence);
std::vector<INDEX_CHARACTER_TYPE> HashGenerator::generateHash(
const std::string & sentence) throw(ConcordiaException) {
std::vector<INDEX_CHARACTER_TYPE> result;
std::vector<std::string> tokenTexts = generateTokenVector(sentence);
if (tokenTexts.size() > Utils::maxSentenceSize) {
throw ConcordiaException("Trying to add too long sentence.");
}
for (vector<string>::iterator it = tokenTexts.begin();
for (std::vector<std::string>::iterator it = tokenTexts.begin();
it != tokenTexts.end(); ++it) {
string token = *it;
std::string token = *it;
INDEX_CHARACTER_TYPE code = _wordMap->getWordCode(token);
result.push_back(code);
}
@ -42,10 +42,11 @@ vector<INDEX_CHARACTER_TYPE> HashGenerator::generateHash(
return result;
}
vector<string> HashGenerator::generateTokenVector(const string & sentence) {
string anonymizedSentence = _sentenceAnonymizer->anonymize(sentence);
std::vector<std::string> HashGenerator::generateTokenVector(
const std::string & sentence) {
std::string anonymizedSentence = _sentenceAnonymizer->anonymize(sentence);
boost::trim(anonymizedSentence);
vector<string> tokenTexts;
std::vector<std::string> tokenTexts;
boost::split(tokenTexts, anonymizedSentence, boost::is_any_of(" \t\r\n"),
boost::algorithm::token_compress_on);
return tokenTexts;
@ -53,7 +54,7 @@ vector<string> HashGenerator::generateTokenVector(const string & sentence) {
void HashGenerator::serializeWordMap() {
ofstream ofs(_wordMapFilePath.c_str(), std::ios::binary);
std::ofstream ofs(_wordMapFilePath.c_str(), std::ios::binary);
boost::archive::binary_oarchive oa(ofs);
oa << *_wordMap;
}

8
concordia/hash_generator.hpp
View File

@ -18,8 +18,6 @@
*/
using namespace std;
class HashGenerator {
public:
explicit HashGenerator(boost::shared_ptr<ConcordiaConfig> config)
@ -29,10 +27,10 @@ public:
*/
virtual ~HashGenerator();
vector<INDEX_CHARACTER_TYPE> generateHash(const string & sentence)
std::vector<INDEX_CHARACTER_TYPE> generateHash(const std::string & sentence)
throw(ConcordiaException);
vector<string> generateTokenVector(const string & sentence);
std::vector<std::string> generateTokenVector(const std::string & sentence);
void serializeWordMap();
@ -41,7 +39,7 @@ private:
boost::shared_ptr<SentenceAnonymizer> _sentenceAnonymizer;
string _wordMapFilePath;
std::string _wordMapFilePath;
};
#endif

16
concordia/index_searcher.cpp
View File

@ -12,16 +12,17 @@ IndexSearcher::IndexSearcher() {
IndexSearcher::~IndexSearcher() {
}
vector<SubstringOccurence> IndexSearcher::simpleSearch(
std::vector<SubstringOccurence> IndexSearcher::simpleSearch(
boost::shared_ptr<HashGenerator> hashGenerator,
boost::shared_ptr<std::vector<sauchar_t> > T,
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,
boost::shared_ptr<std::vector<saidx_t> > SA,
const string & pattern) throw(ConcordiaException) {
vector<SubstringOccurence> result;
const std::string & pattern) throw(ConcordiaException) {
std::vector<SubstringOccurence> result;
int left;
vector<INDEX_CHARACTER_TYPE> hash = hashGenerator->generateHash(pattern);
std::vector<INDEX_CHARACTER_TYPE> hash =
hashGenerator->generateHash(pattern);
saidx_t patternLength = hash.size()*sizeof(INDEX_CHARACTER_TYPE);
sauchar_t * patternArray = Utils::indexVectorToSaucharArray(hash);
@ -47,12 +48,13 @@ vector<SubstringOccurence> IndexSearcher::simpleSearch(
return result;
}
vector<AnubisSearchResult> IndexSearcher::anubisSearch(
std::vector<AnubisSearchResult> IndexSearcher::anubisSearch(
boost::shared_ptr<HashGenerator> hashGenerator,
boost::shared_ptr<std::vector<sauchar_t> > T,
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,
boost::shared_ptr<std::vector<saidx_t> > SA,
const string & pattern) throw(ConcordiaException) {
vector<INDEX_CHARACTER_TYPE> hash = hashGenerator->generateHash(pattern);
const std::string & pattern) throw(ConcordiaException) {
std::vector<INDEX_CHARACTER_TYPE> hash =
hashGenerator->generateHash(pattern);
return _anubisSearcher->anubisSearch(T, markers, SA, hash);
}

10
concordia/index_searcher.hpp
View File

@ -20,8 +20,6 @@
*/
using namespace std;
class IndexSearcher {
public:
explicit IndexSearcher();
@ -30,19 +28,19 @@ public:
*/
virtual ~IndexSearcher();
vector<SubstringOccurence> simpleSearch(
std::vector<SubstringOccurence> simpleSearch(
boost::shared_ptr<HashGenerator> hashGenerator,
boost::shared_ptr<std::vector<sauchar_t> > T,
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,
boost::shared_ptr<std::vector<saidx_t> > SA,
const string & pattern) throw(ConcordiaException);
const std::string & pattern) throw(ConcordiaException);
vector<AnubisSearchResult> anubisSearch(
std::vector<AnubisSearchResult> anubisSearch(
boost::shared_ptr<HashGenerator> hashGenerator,
boost::shared_ptr<std::vector<sauchar_t> > T,
boost::shared_ptr<std::vector<SUFFIX_MARKER_TYPE> > markers,
boost::shared_ptr<std::vector<saidx_t> > SA,
const string & pattern) throw(ConcordiaException);
const std::string & pattern) throw(ConcordiaException);
private:
boost::shared_ptr<AnubisSearcher> _anubisSearcher;
};

7
concordia/interval.cpp
View File

@ -1,9 +1,10 @@
#include "concordia/interval.hpp"
Interval::Interval(const SUFFIX_MARKER_TYPE start, const SUFFIX_MARKER_TYPE end):
_start(start),
_end(end) {
Interval::Interval(const SUFFIX_MARKER_TYPE start,
const SUFFIX_MARKER_TYPE end):
_start(start),
_end(end) {
}
Interval::~Interval() {

5
concordia/interval.hpp
View File

@ -8,11 +8,10 @@
*/
using namespace std;
class Interval {
public:
explicit Interval(const SUFFIX_MARKER_TYPE start, const SUFFIX_MARKER_TYPE end);
explicit Interval(const SUFFIX_MARKER_TYPE start,
const SUFFIX_MARKER_TYPE end);
/*! Destructor.
*/

7
concordia/regex_replacement.cpp
View File

@ -3,7 +3,8 @@
#include <boost/exception/all.hpp>
#include <boost/throw_exception.hpp>
RegexReplacement::RegexReplacement(string patternString, string replacement,
RegexReplacement::RegexReplacement(std::string patternString,
std::string replacement,
bool caseSensitive)
throw(ConcordiaException):
_replacement(replacement) {
@ -15,7 +16,7 @@ RegexReplacement::RegexReplacement(string patternString, string replacement,
boost::regex::icase);
}
} catch(const std::exception & e) {
stringstream ss;
std::stringstream ss;
ss << "Bad regex pattern: " << patternString <<
" Detailed info: " << e.what();
@ -31,7 +32,7 @@ RegexReplacement::RegexReplacement(string patternString, string replacement,
RegexReplacement::~RegexReplacement() {
}
string RegexReplacement::apply(const string & text) {
std::string RegexReplacement::apply(const std::string & text) {
try {
return boost::u32regex_replace(text, _pattern, _replacement,
boost::match_default | boost::format_all);

12
concordia/regex_replacement.hpp
View File

@ -14,26 +14,24 @@
*/
using namespace std;
typedef boost::error_info<struct my_tag, std::string> my_tag_error_info;
class RegexReplacement {
public:
RegexReplacement(string patternString, string replacement,
bool caseSensitive = true)
throw(ConcordiaException);
RegexReplacement(std::string patternString, std::string replacement,
bool caseSensitive = true)
throw(ConcordiaException);
/*! Destructor.
*/
virtual ~RegexReplacement();
string apply(const string & text);
std::string apply(const std::string & text);
private:
boost::u32regex _pattern;
string _replacement;
std::string _replacement;
};
#endif

34
concordia/sentence_anonymizer.cpp
View File

@ -23,8 +23,8 @@ SentenceAnonymizer::SentenceAnonymizer(
SentenceAnonymizer::~SentenceAnonymizer() {
}
string SentenceAnonymizer::anonymize(const string & sentence) {
string result = sentence;
std::string SentenceAnonymizer::anonymize(const std::string & sentence) {
std::string result = sentence;
result = _htmlTags->apply(result);
@ -41,20 +41,20 @@ string SentenceAnonymizer::anonymize(const string & sentence) {
return result;
}
void SentenceAnonymizer::_createNeRules(string & namedEntitiesPath) {
void SentenceAnonymizer::_createNeRules(std::string & namedEntitiesPath) {
if (boost::filesystem::exists(namedEntitiesPath)) {
string line;
ifstream neFile(namedEntitiesPath.c_str());
std::string line;
std::ifstream neFile(namedEntitiesPath.c_str());
if (neFile.is_open()) {
int lineCounter = 0;
while (getline(neFile, line)) {
lineCounter++;
boost::shared_ptr<vector<string> >
tokenTexts(new vector<string>());
boost::shared_ptr<std::vector<std::string> >
tokenTexts(new std::vector<std::string>());
boost::split(*tokenTexts, line, boost::is_any_of(" "),
boost::token_compress_on);
if (tokenTexts->size() != 2) {
stringstream ss;
std::stringstream ss;
ss << "Invalid line: " << lineCounter
<< " in NE file: " << namedEntitiesPath;
throw ConcordiaException(ss.str());
@ -72,11 +72,11 @@ void SentenceAnonymizer::_createNeRules(string & namedEntitiesPath) {
}
}
void SentenceAnonymizer::_createHtmlTagsRule(string & htmlTagsPath) {
string tagsExpression = "<\\/?(";
void SentenceAnonymizer::_createHtmlTagsRule(std::string & htmlTagsPath) {
std::string tagsExpression = "<\\/?(";
if (boost::filesystem::exists(htmlTagsPath)) {
string line;
ifstream tagsFile(htmlTagsPath.c_str());
std::string line;
std::ifstream tagsFile(htmlTagsPath.c_str());
if (tagsFile.is_open()) {
while (getline(tagsFile, line)) {
tagsExpression += "|";
@ -95,12 +95,12 @@ void SentenceAnonymizer::_createHtmlTagsRule(string & htmlTagsPath) {
}
boost::shared_ptr<RegexReplacement>
SentenceAnonymizer::_getMultipleReplacementRule(
string & filePath, string replacement, bool wholeWord) {
string expression = "(";
SentenceAnonymizer::_getMultipleReplacementRule(
std::string & filePath, std::string replacement, bool wholeWord) {
std::string expression = "(";
if (boost::filesystem::exists(filePath)) {
string line;
ifstream ruleFile(filePath.c_str());
std::string line;
std::ifstream ruleFile(filePath.c_str());
if (ruleFile.is_open()) {
while (getline(ruleFile, line)) {
if (wholeWord) {

16
concordia/sentence_anonymizer.hpp
View File

@ -16,8 +16,6 @@
*/
using namespace std;
class SentenceAnonymizer {
public:
explicit SentenceAnonymizer(boost::shared_ptr<ConcordiaConfig> config)
@ -27,19 +25,19 @@ public:
*/
virtual ~SentenceAnonymizer();
string anonymize(const string & sentence);
std::string anonymize(const std::string & sentence);
private:
void _createNeRules(string & namedEntitiesPath);
void _createNeRules(std::string & namedEntitiesPath);
void _createHtmlTagsRule(string & htmlTagsPath);
void _createHtmlTagsRule(std::string & htmlTagsPath);
boost::shared_ptr<RegexReplacement> _getMultipleReplacementRule(
string & filePath,
string replacement,
bool wholeWord = false);
std::string & filePath,
std::string replacement,
bool wholeWord = false);
vector<RegexReplacement> _namedEntities;
std::vector<RegexReplacement> _namedEntities;
boost::shared_ptr<RegexReplacement> _htmlTags;

5
concordia/substring_occurence.cpp
View File

@ -7,10 +7,11 @@ SubstringOccurence::SubstringOccurence() {
SubstringOccurence::SubstringOccurence(const SUFFIX_MARKER_TYPE & marker) {
_id = Utils::getIdFromMarker(marker);
_offset = Utils::getOffsetFromMarker(marker);
_exampleLength = Utils::getLengthFromMarker(marker);
_exampleLength = Utils::getLengthFromMarker(marker);
}
void SubstringOccurence::enterDataFromMarker(const SUFFIX_MARKER_TYPE & marker) {
void SubstringOccurence::enterDataFromMarker(
const SUFFIX_MARKER_TYPE & marker) {
_id = Utils::getIdFromMarker(marker);
_offset = Utils::getOffsetFromMarker(marker);
_exampleLength = Utils::getLengthFromMarker(marker);

6
concordia/substring_occurence.hpp
View File

@ -9,8 +9,6 @@
*/
using namespace std;
class SubstringOccurence {
public:
SubstringOccurence();
@ -35,7 +33,7 @@ public:
SUFFIX_MARKER_TYPE getExampleLength() const {
return _exampleLength;
}
void enterDataFromMarker(const SUFFIX_MARKER_TYPE & marker);
private:
@ -43,7 +41,7 @@ private:
SUFFIX_MARKER_TYPE _offset;
// the example
// the example
SUFFIX_MARKER_TYPE _exampleLength;
};

24
concordia/t/test_anubis_searcher.cpp
View File

@ -12,8 +12,6 @@
#include "concordia/common/logging.hpp"
#include "tests/common/test_resources_manager.hpp"
using namespace std;
BOOST_AUTO_TEST_SUITE(anubis_searcher)
BOOST_AUTO_TEST_CASE( LcpSearch1 )
@ -187,7 +185,7 @@ BOOST_AUTO_TEST_CASE( LcpSearch1 )
pattern2.push_back(2);
SUFFIX_MARKER_TYPE highResLength2;
vector<SubstringOccurence> result2 = searcher.lcpSearch(T, markers, SA, pattern2, highResLength2);
std::vector<SubstringOccurence> result2 = searcher.lcpSearch(T, markers, SA, pattern2, highResLength2);
SUFFIX_MARKER_TYPE length2 = highResLength2 / sizeof(INDEX_CHARACTER_TYPE);
/* Expecting to get one result from SA:
@ -230,7 +228,7 @@ BOOST_AUTO_TEST_CASE( LcpSearch1 )
pattern3.push_back(3);
SUFFIX_MARKER_TYPE highResLength3;
vector<SubstringOccurence> result3 = searcher.lcpSearch(T, markers, SA, pattern3, highResLength3);
std::vector<SubstringOccurence> result3 = searcher.lcpSearch(T, markers, SA, pattern3, highResLength3);
SUFFIX_MARKER_TYPE length3 = highResLength3 / sizeof(INDEX_CHARACTER_TYPE);
/* Expecting to get one result from SA:
@ -267,7 +265,7 @@ BOOST_AUTO_TEST_CASE( LcpSearch1 )
pattern4.push_back(4);
SUFFIX_MARKER_TYPE highResLength4;
vector<SubstringOccurence> result4 = searcher.lcpSearch(T, markers, SA, pattern4, highResLength4);
std::vector<SubstringOccurence> result4 = searcher.lcpSearch(T, markers, SA, pattern4, highResLength4);
SUFFIX_MARKER_TYPE length4 = highResLength4 / sizeof(INDEX_CHARACTER_TYPE);
/* Expecting to get 2 results from SA:
@ -298,7 +296,7 @@ BOOST_AUTO_TEST_CASE( LcpSearch1 )
pattern5.push_back(4);
SUFFIX_MARKER_TYPE highResLength5;
vector<SubstringOccurence> result5 = searcher.lcpSearch(T, markers, SA, pattern5, highResLength5);
std::vector<SubstringOccurence> result5 = searcher.lcpSearch(T, markers, SA, pattern5, highResLength5);
SUFFIX_MARKER_TYPE length5 = highResLength5 / sizeof(INDEX_CHARACTER_TYPE);
/* Expecting to get 0 results from SA, lcp length = 0;
@ -322,7 +320,7 @@ BOOST_AUTO_TEST_CASE( LcpSearch1 )
pattern6.push_back(0);
SUFFIX_MARKER_TYPE highResLength6;
vector<SubstringOccurence> result6 = searcher.lcpSearch(T, markers, SA, pattern5, highResLength6);
std::vector<SubstringOccurence> result6 = searcher.lcpSearch(T, markers, SA, pattern5, highResLength6);
SUFFIX_MARKER_TYPE length6 = highResLength6 / sizeof(INDEX_CHARACTER_TYPE);
/* Expecting to get 0 results from SA, lcp length = 0;
@ -393,38 +391,38 @@ BOOST_AUTO_TEST_CASE( TmMatchesTest )
// example 14
// example interval list: [(1,2)]
vector<Interval> exampleIntervals14 = tmMatches14->getExampleIntervals();
std::vector<Interval> exampleIntervals14 = tmMatches14->getExampleIntervals();
BOOST_CHECK_EQUAL(exampleIntervals14.size(), 1);
BOOST_CHECK_EQUAL(exampleIntervals14[0].getStart(), 1);
BOOST_CHECK_EQUAL(exampleIntervals14[0].getEnd(), 2);
// pattern interval list: [(1,2)]
vector<Interval> patternIntervals14 = tmMatches14->getPatternIntervals();
std::vector<Interval> patternIntervals14 = tmMatches14->getPatternIntervals();
BOOST_CHECK_EQUAL(patternIntervals14.size(), 1);
BOOST_CHECK_EQUAL(patternIntervals14[0].getStart(), 1);
BOOST_CHECK_EQUAL(patternIntervals14[0].getEnd(), 2);
// example 51
// example interval list: [(1,3)]
vector<Interval> exampleIntervals51 = tmMatches51->getExampleIntervals();
std::vector<Interval> exampleIntervals51 = tmMatches51->getExampleIntervals();
BOOST_CHECK_EQUAL(exampleIntervals51.size(), 1);
BOOST_CHECK_EQUAL(exampleIntervals51[0].getStart(), 1);
BOOST_CHECK_EQUAL(exampleIntervals51[0].getEnd(), 3);
// pattern interval list: [(1,3)]
vector<Interval> patternIntervals51 = tmMatches51->getPatternIntervals();
std::vector<Interval> patternIntervals51 = tmMatches51->getPatternIntervals();
BOOST_CHECK_EQUAL(patternIntervals51.size(), 1);
BOOST_CHECK_EQUAL(patternIntervals51[0].getStart(), 1);
BOOST_CHECK_EQUAL(patternIntervals51[0].getEnd(), 3);
// example 123
// example interval list: [(1,3), (0,1)]
vector<Interval> exampleIntervals123 = tmMatches123->getExampleIntervals();
std::vector<Interval> exampleIntervals123 = tmMatches123->getExampleIntervals();
BOOST_CHECK_EQUAL(exampleIntervals123.size(), 2);
BOOST_CHECK_EQUAL(exampleIntervals123[0].getStart(), 1);
BOOST_CHECK_EQUAL(exampleIntervals123[0].getEnd(), 3);
BOOST_CHECK_EQUAL(exampleIntervals123[1].getStart(), 0);
BOOST_CHECK_EQUAL(exampleIntervals123[1].getEnd(), 1);
// pattern interval list: [(1,3), (3,4)]
vector<Interval> patternIntervals123 = tmMatches123->getPatternIntervals();
std::vector<Interval> patternIntervals123 = tmMatches123->getPatternIntervals();
BOOST_CHECK_EQUAL(patternIntervals123.size(), 2);
BOOST_CHECK_EQUAL(patternIntervals123[0].getStart(), 1);
BOOST_CHECK_EQUAL(patternIntervals123[0].getEnd(), 3);

22
concordia/t/test_concordia.cpp
View File

@ -9,14 +9,12 @@
#include <string>
using namespace std;
BOOST_AUTO_TEST_SUITE(concordia_main)
BOOST_AUTO_TEST_CASE( ConcordiaVersion )
{
Concordia concordia = Concordia(TestResourcesManager::getTestConcordiaConfigFilePath("concordia.cfg"));
string version = concordia.getVersion();
std::string version = concordia.getVersion();
BOOST_CHECK_EQUAL( version , "0.1");
}
@ -51,8 +49,8 @@ BOOST_AUTO_TEST_CASE( ConcordiaSimpleSearch1 )
*/
vector<SubstringOccurence> searchResult1 = concordia.simpleSearch("posiada rysia");
vector<SubstringOccurence> searchResult2 = concordia.simpleSearch("posiada kota Ala");
std::vector<SubstringOccurence> searchResult1 = concordia.simpleSearch("posiada rysia");
std::vector<SubstringOccurence> searchResult2 = concordia.simpleSearch("posiada kota Ala");
boost::filesystem::remove(TestResourcesManager::getTestFilePath("temp",TEMP_WORD_MAP));
boost::filesystem::remove(TestResourcesManager::getTestFilePath("temp",TEMP_MARKERS));
@ -73,7 +71,7 @@ BOOST_AUTO_TEST_CASE( ConcordiaSimpleSearch2 )
{
// modified stop words to avoid anonymization
Concordia concordia = Concordia(TestResourcesManager::getTestConcordiaConfigFilePath("concordia.cfg"));
vector<Example> testExamples;
std::vector<Example> testExamples;
testExamples.push_back(Example("xto xjest okno",312));
testExamples.push_back(Example("czy xjest okno otwarte",202));
testExamples.push_back(Example("chyba xto xjest xtutaj",45));
@ -106,8 +104,8 @@ BOOST_AUTO_TEST_CASE( ConcordiaSimpleSearch2 )
*/
Concordia concordia2 = Concordia(TestResourcesManager::getTestConcordiaConfigFilePath("concordia.cfg"));
vector<SubstringOccurence> searchResult1 = concordia2.simpleSearch("xto xjest");
vector<SubstringOccurence> searchResult2 = concordia2.simpleSearch("xjest okno");
std::vector<SubstringOccurence> searchResult1 = concordia2.simpleSearch("xto xjest");
std::vector<SubstringOccurence> searchResult2 = concordia2.simpleSearch("xjest okno");
boost::filesystem::remove(TestResourcesManager::getTestFilePath("temp",TEMP_WORD_MAP));
boost::filesystem::remove(TestResourcesManager::getTestFilePath("temp",TEMP_MARKERS));
@ -131,13 +129,13 @@ BOOST_AUTO_TEST_CASE( ConcordiaSimpleSearch2 )
BOOST_AUTO_TEST_CASE( ConcordiaSimpleSearch3 )
{
Concordia concordia = Concordia(TestResourcesManager::getTestConcordiaConfigFilePath("concordia.cfg"));
vector<Example> testExamples;
std::vector<Example> testExamples;
testExamples.push_back(Example("2. Ma on w szczególności prawo do podjęcia zatrudnienia dostępnego na terytorium innego Państwa Członkowskiego z takim samym pierwszeństwem, z jakiego korzystają obywatele tego państwa.",312));
testExamples.push_back(Example("czy xjest żółte otwarte",202));
concordia.addAllExamples(testExamples);
Concordia concordia2 = Concordia(TestResourcesManager::getTestConcordiaConfigFilePath("concordia.cfg"));
vector<SubstringOccurence> searchResult1 = concordia2.simpleSearch("on w szczególności prawo do podjęcia");
std::vector<SubstringOccurence> searchResult1 = concordia2.simpleSearch("on w szczególności prawo do podjęcia");
boost::filesystem::remove(TestResourcesManager::getTestFilePath("temp",TEMP_WORD_MAP));
boost::filesystem::remove(TestResourcesManager::getTestFilePath("temp",TEMP_MARKERS));
@ -176,8 +174,8 @@ BOOST_AUTO_TEST_CASE( ConcordiaAnubisSearch1 )
n: 0 1 2 3 4 5 6 7 8 9 10 11
SA[n]: 0 4 1 9 5 2 10 6 8 11 3 7
vector<AnubisSearchResult> searchResult1 = concordia.anubisSearch("posiada rysia chyba");
vector<AnubisSearchResult> searchResult2 = concordia.anubisSearch("posiada kota Ala");
std::vector<AnubisSearchResult> searchResult1 = concordia.anubisSearch("posiada rysia chyba");
std::vector<AnubisSearchResult> searchResult2 = concordia.anubisSearch("posiada kota Ala");
boost::filesystem::remove(TestResourcesManager::getTestFilePath("temp",TEMP_WORD_MAP));
boost::filesystem::remove(TestResourcesManager::getTestFilePath("temp",TEMP_MARKERS));

6
concordia/t/test_concordia_config.cpp
View File

@ -7,8 +7,6 @@
#include <list>
#include <boost/algorithm/string/predicate.hpp>
using namespace std;
BOOST_AUTO_TEST_SUITE(concordia_config)
BOOST_AUTO_TEST_CASE( ConfigParameters )
@ -29,7 +27,7 @@ BOOST_AUTO_TEST_CASE( ConfigParameters )
BOOST_AUTO_TEST_CASE( NonexistentConfigTest )
{
bool exceptionThrown = false;
string message = "";
std::string message = "";
try {
ConcordiaConfig config(TestResourcesManager::getTestConcordiaConfigFilePath("foo.cfg"));
} catch (ConcordiaException & e) {
@ -44,7 +42,7 @@ BOOST_AUTO_TEST_CASE( NonexistentConfigTest )
BOOST_AUTO_TEST_CASE( InvalidConfigTest )
{
bool exceptionThrown = false;
string message = "";
std::string message = "";
try {
ConcordiaConfig config(TestResourcesManager::getTestConcordiaConfigFilePath("invalid.cfg"));
} catch (ConcordiaException & e) {

14
concordia/t/test_concordia_index.cpp
View File

@ -7,8 +7,6 @@
#include <boost/algorithm/string/predicate.hpp>
#include <boost/filesystem.hpp>
using namespace std;
BOOST_AUTO_TEST_SUITE(concordia_index)
@ -16,7 +14,7 @@ BOOST_AUTO_TEST_CASE( SuffixArrayGenerationTest )
{
ConcordiaIndex index(TestResourcesManager::getTestFilePath("temp","test_hash_index.bin"),
TestResourcesManager::getTestFilePath("temp","test_markers.bin"));
boost::shared_ptr<vector<sauchar_t> > T = boost::shared_ptr<vector<sauchar_t> >(new vector<sauchar_t>());
boost::shared_ptr<std::vector<sauchar_t> > T = boost::shared_ptr<std::vector<sauchar_t> >(new std::vector<sauchar_t>());
// Test hashed index:
// n: 0 1 2 3 4 5 6 7 8
// T[n]: 0 1 2 0 1 3 4 1 3
@ -36,7 +34,7 @@ BOOST_AUTO_TEST_CASE( SuffixArrayGenerationTest )
boost::shared_ptr<std::vector<saidx_t> > SA = index.generateSuffixArray(T);
boost::shared_ptr<vector<saidx_t> > expectedSA = boost::shared_ptr<vector<saidx_t> >(new vector<saidx_t>());
boost::shared_ptr<std::vector<saidx_t> > expectedSA = boost::shared_ptr<std::vector<saidx_t> >(new std::vector<saidx_t>());
expectedSA->push_back(0);
expectedSA->push_back(3);
expectedSA->push_back(1);
@ -53,7 +51,7 @@ BOOST_AUTO_TEST_CASE( SuffixArrayGenerationTest2 )
{
ConcordiaIndex index(TestResourcesManager::getTestFilePath("temp","test_hash_index.bin"),
TestResourcesManager::getTestFilePath("temp","test_markers.bin"));
boost::shared_ptr<vector<sauchar_t> > T = boost::shared_ptr<vector<sauchar_t> >(new vector<sauchar_t>());
boost::shared_ptr<std::vector<sauchar_t> > T = boost::shared_ptr<std::vector<sauchar_t> >(new std::vector<sauchar_t>());
//Test hashed index:
// n: 0 1 2 3 4 5 6 7 8 9 10 11
@ -77,7 +75,7 @@ BOOST_AUTO_TEST_CASE( SuffixArrayGenerationTest2 )
boost::shared_ptr<std::vector<saidx_t> > SA = index.generateSuffixArray(T);
boost::shared_ptr<vector<saidx_t> > expectedSA = boost::shared_ptr<vector<saidx_t> >(new vector<saidx_t>());
boost::shared_ptr<std::vector<saidx_t> > expectedSA = boost::shared_ptr<std::vector<saidx_t> >(new std::vector<saidx_t>());
expectedSA->push_back(0);
expectedSA->push_back(4);
expectedSA->push_back(1);
@ -97,7 +95,7 @@ BOOST_AUTO_TEST_CASE( SuffixArrayGenerationTest3 )
{
ConcordiaIndex index(TestResourcesManager::getTestFilePath("temp","test_hash_index.bin"),
TestResourcesManager::getTestFilePath("temp","test_markers.bin"));
boost::shared_ptr<vector<sauchar_t> > T = boost::shared_ptr<vector<sauchar_t> >(new vector<sauchar_t>());
boost::shared_ptr<std::vector<sauchar_t> > T = boost::shared_ptr<std::vector<sauchar_t> >(new std::vector<sauchar_t>());
//Test hashed index:
// n: 0 1 2 3 4 5
@ -115,7 +113,7 @@ BOOST_AUTO_TEST_CASE( SuffixArrayGenerationTest3 )
boost::shared_ptr<std::vector<saidx_t> > SA = index.generateSuffixArray(T);
boost::shared_ptr<vector<saidx_t> > expectedSA = boost::shared_ptr<vector<saidx_t> >(new vector<saidx_t>());
boost::shared_ptr<std::vector<saidx_t> > expectedSA = boost::shared_ptr<std::vector<saidx_t> >(new std::vector<saidx_t>());
expectedSA->push_back(0);
expectedSA->push_back(5);
expectedSA->push_back(3);

4
concordia/t/test_example.cpp
View File

@ -5,8 +5,6 @@
#include "concordia/example.hpp"
using namespace std;
BOOST_AUTO_TEST_SUITE(exampleTest)
BOOST_AUTO_TEST_CASE( ExceedingId )
@ -15,7 +13,7 @@ BOOST_AUTO_TEST_CASE( ExceedingId )
Example example1("Test", maxId);
bool exceptionThrown = false;
string message = "";
std::string message = "";
try {
Example example2("Test", maxId+1);
} catch (ConcordiaException & e) {

27
concordia/t/test_hash_generator.cpp
View File

@ -8,9 +8,6 @@
#include "concordia/hash_generator.hpp"
#include "tests/common/test_resources_manager.hpp"
using namespace std;
BOOST_AUTO_TEST_SUITE(hash_generator)
BOOST_AUTO_TEST_CASE( SimpleHashTest )
@ -23,8 +20,8 @@ BOOST_AUTO_TEST_CASE( SimpleHashTest )
HashGenerator hashGenerator = HashGenerator(config);
vector<INDEX_CHARACTER_TYPE> hash = hashGenerator.generateHash("Ala posiada kota");
vector<INDEX_CHARACTER_TYPE> expected;
std::vector<INDEX_CHARACTER_TYPE> hash = hashGenerator.generateHash("Ala posiada kota");
std::vector<INDEX_CHARACTER_TYPE> expected;
expected.push_back(0);
expected.push_back(1);
expected.push_back(2);
@ -44,17 +41,17 @@ BOOST_AUTO_TEST_CASE( TooLongHashTest )
HashGenerator hashGenerator = HashGenerator(config);
stringstream ss;
std::stringstream ss;
for (int i=0;i<65537;i++) {
ss << "xx" << i << " ";
}
string longSentence = ss.str();
std::string longSentence = ss.str();
bool exceptionThrown = false;
string message = "";
std::string message = "";
try {
boost::shared_ptr<vector<INDEX_CHARACTER_TYPE> > hash = hashGenerator.generateHash(longSentence);
boost::shared_ptr<std::vector<INDEX_CHARACTER_TYPE> > hash = hashGenerator.generateHash(longSentence);
} catch (ConcordiaException & e) {
exceptionThrown = true;
message = e.what();
@ -76,8 +73,8 @@ BOOST_AUTO_TEST_CASE( HashSerializationTest )
HashGenerator hashGenerator1 = HashGenerator(config);
vector<INDEX_CHARACTER_TYPE> hash1 = hashGenerator1.generateHash("Ala posiada kota");
vector<INDEX_CHARACTER_TYPE> expected1;
std::vector<INDEX_CHARACTER_TYPE> hash1 = hashGenerator1.generateHash("Ala posiada kota");
std::vector<INDEX_CHARACTER_TYPE> expected1;
expected1.push_back(0);
expected1.push_back(1);
expected1.push_back(2);
@ -86,8 +83,8 @@ BOOST_AUTO_TEST_CASE( HashSerializationTest )
hashGenerator1.serializeWordMap();
HashGenerator hashGenerator2 = HashGenerator(config);
vector<INDEX_CHARACTER_TYPE> hash2 = hashGenerator2.generateHash("Ala posiada psa");
vector<INDEX_CHARACTER_TYPE> expected2;
std::vector<INDEX_CHARACTER_TYPE> hash2 = hashGenerator2.generateHash("Ala posiada psa");
std::vector<INDEX_CHARACTER_TYPE> expected2;
expected2.push_back(0);
expected2.push_back(1);
expected2.push_back(3);
@ -106,8 +103,8 @@ BOOST_AUTO_TEST_CASE( TokenVectorTest )
HashGenerator hashGenerator = HashGenerator(config);
vector<string> tokenVector = hashGenerator.generateTokenVector("12.02.2014 o godzinie 17:40 doszło do kolizji na ulicy Grobla; policjanci ustalili, że kierowca zaparkował samochód.");
vector<string> expected;
std::vector<std::string> tokenVector = hashGenerator.generateTokenVector("12.02.2014 o godzinie 17:40 doszło do kolizji na ulicy Grobla; policjanci ustalili, że kierowca zaparkował samochód.");
std::vector<std::string> expected;
expected.push_back("ne_date");
expected.push_back("godzinie");
expected.push_back("ne_number");

2
concordia/t/test_interval.cpp
View File

@ -2,8 +2,6 @@
#include "concordia/interval.hpp"
#include "concordia/common/config.hpp"
using namespace std;
BOOST_AUTO_TEST_SUITE(interval)
BOOST_AUTO_TEST_CASE( IntervalIntersects1 )

2
concordia/t/test_logging.cpp
View File

@ -6,8 +6,6 @@
#define TMP_LOG_FILE "/tmp/concordia.log"
using namespace std;
BOOST_AUTO_TEST_SUITE(logging)
BOOST_AUTO_TEST_CASE( LoggingTest )

4
concordia/t/test_regex_replacement.cpp
View File

@ -5,8 +5,6 @@
#include <boost/locale.hpp>
#include <boost/algorithm/string/case_conv.hpp>
using namespace std;
BOOST_AUTO_TEST_SUITE(regex_replacement)
BOOST_AUTO_TEST_CASE( SimpleReplacement )
@ -18,7 +16,7 @@ BOOST_AUTO_TEST_CASE( SimpleReplacement )
BOOST_AUTO_TEST_CASE( BadRegex )
{
bool exceptionThrown = false;
string message = "";
std::string message = "";
try {
RegexReplacement rr("+a","b");
} catch (ConcordiaException & e) {

15
concordia/t/test_sentence_anonymizer.cpp
View File

@ -8,9 +8,6 @@
#include "concordia/sentence_anonymizer.hpp"
#include "tests/common/test_resources_manager.hpp"
using namespace std;
BOOST_AUTO_TEST_SUITE(sentence_anonymizer)
BOOST_AUTO_TEST_CASE( NETest )
@ -19,7 +16,7 @@ BOOST_AUTO_TEST_CASE( NETest )
SentenceAnonymizer anonymizer(config);
string sentence = "Date: 12.04.2012, mail: test@example.com, number: 5.34";
std::string sentence = "Date: 12.04.2012, mail: test@example.com, number: 5.34";
BOOST_CHECK_EQUAL(anonymizer.anonymize(sentence),"date ne_date mail ne_email number ne_number");
}
@ -29,7 +26,7 @@ BOOST_AUTO_TEST_CASE( HtmlTagsTest )
SentenceAnonymizer anonymizer(config);
string sentence = "<a href='http://wp.pl'>link</a> and <b>bold</b> and newline <br/>";
std::string sentence = "<a href='http://wp.pl'>link</a> and <b>bold</b> and newline <br/>";
BOOST_CHECK_EQUAL(anonymizer.anonymize(sentence),"link and bold and newline ");
}
@ -40,7 +37,7 @@ BOOST_AUTO_TEST_CASE( StopWordsTest )
SentenceAnonymizer anonymizer(config);
string sentence = "Aczkolwiek nie wiem, czy to konieczne";
std::string sentence = "Aczkolwiek nie wiem, czy to konieczne";
BOOST_CHECK_EQUAL(anonymizer.anonymize(sentence)," wiem konieczne");
}
@ -51,7 +48,7 @@ BOOST_AUTO_TEST_CASE( StopSymbolsTest )
SentenceAnonymizer anonymizer(config);
string sentence = "xxx, . xxx # xx $xx@ xx";
std::string sentence = "xxx, . xxx # xx $xx@ xx";
BOOST_CHECK_EQUAL(anonymizer.anonymize(sentence),"xxx xxx xx xx xx");
}
@ -62,7 +59,7 @@ BOOST_AUTO_TEST_CASE( SpaceSymbolsTest )
SentenceAnonymizer anonymizer(config);
string sentence = "xxx-xxx xx|xx";
std::string sentence = "xxx-xxx xx|xx";
BOOST_CHECK_EQUAL(anonymizer.anonymize(sentence),"xxx xxx xx xx");
}
@ -72,7 +69,7 @@ BOOST_AUTO_TEST_CASE( WeirdSentenceTest )
boost::shared_ptr<ConcordiaConfig> config(new ConcordiaConfig(TestResourcesManager::getTestConcordiaConfigFilePath("concordia.cfg")));
SentenceAnonymizer anonymizer(config);
string sentence = "Sony | DXC-M7PKDXC-M7PDXC-M7PHDXC-M7PK/1DXC-M7P/1DXC-M7PH/1DXC-327PKDXC-327PLDXC-327PHDXC-327APKDXC-327APLDXC-327AHDXC-537PKDXC-537PLDXC-537PHDXC-537APKDXC-537APLDXC-537APHEVW-537PKEVW-327PKDXC-637PDXC-637PKDXC-637PLDXC-637PHPVW-637PKPVW-637PLDXC-D30PFDXC-D30PKDXC-D30PLDXC-D30PHDSR-130PFDSR-130PKDSR-130PLPVW-D30PFPVW-D30PKPVW-D30PLDXC-327BPFDXC-327BPKDXC-327BPLDXC-327BPHDXC-D30WSPDXC-D35PHDXC-D35PLDXC-D35PKDXC-D35WSPLDSR-135PL | DXF-3000CEDXF-325CEDXF-501CEDXF-M3CEDXF-M7CEDXF-40CEDXF-40ACEDXF-50CEDXF-601CEDXF-40BCEDXF-50BCEDXF-701CEDXF-WSCEDXF-801CEHDVF-C30W | CCU-M3PCCU-M5PCCU-M7PCUU-M5AP | RM-M7GRM-M7E | — | CA-325PCA-325APCA-325BCA-327PCA-537PCA-511CA-512PCA-513VCT-U14 |";
std::string sentence = "Sony | DXC-M7PKDXC-M7PDXC-M7PHDXC-M7PK/1DXC-M7P/1DXC-M7PH/1DXC-327PKDXC-327PLDXC-327PHDXC-327APKDXC-327APLDXC-327AHDXC-537PKDXC-537PLDXC-537PHDXC-537APKDXC-537APLDXC-537APHEVW-537PKEVW-327PKDXC-637PDXC-637PKDXC-637PLDXC-637PHPVW-637PKPVW-637PLDXC-D30PFDXC-D30PKDXC-D30PLDXC-D30PHDSR-130PFDSR-130PKDSR-130PLPVW-D30PFPVW-D30PKPVW-D30PLDXC-327BPFDXC-327BPKDXC-327BPLDXC-327BPHDXC-D30WSPDXC-D35PHDXC-D35PLDXC-D35PKDXC-D35WSPLDSR-135PL | DXF-3000CEDXF-325CEDXF-501CEDXF-M3CEDXF-M7CEDXF-40CEDXF-40ACEDXF-50CEDXF-601CEDXF-40BCEDXF-50BCEDXF-701CEDXF-WSCEDXF-801CEHDVF-C30W | CCU-M3PCCU-M5PCCU-M7PCUU-M5AP | RM-M7GRM-M7E | — | CA-325PCA-325APCA-325BCA-327PCA-537PCA-511CA-512PCA-513VCT-U14 |";
BOOST_CHECK_EQUAL(anonymizer.anonymize(sentence),"sony dxc mne_numberpkdxc mne_numberpdxc mne_numberphdxc mne_numberpk ne_numberdxc mne_numberp ne_numberdxc mne_numberph ne_numberdxc ne_numberpkdxc ne_numberpldxc ne_numberphdxc ne_numberapkdxc ne_numberapldxc ne_numberahdxc ne_numberpkdxc ne_numberpldxc ne_numberphdxc ne_numberapkdxc ne_numberapldxc ne_numberaphevw ne_numberpkevw ne_numberpkdxc ne_numberpdxc ne_numberpkdxc ne_numberpldxc ne_numberphpvw ne_numberpkpvw ne_numberpldxc dne_numberpfdxc dne_numberpkdxc dne_numberpldxc dne_numberphdsr ne_numberpfdsr ne_numberpkdsr ne_numberplpvw dne_numberpfpvw dne_numberpkpvw dne_numberpldxc ne_numberbpfdxc ne_numberbpkdxc ne_numberbpldxc ne_numberbphdxc dne_numberwspdxc dne_numberphdxc dne_numberpldxc dne_numberpkdxc dne_numberwspldsr ne_numberpl dxf ne_numbercedxf ne_numbercedxf ne_numbercedxf mne_numbercedxf mne_numbercedxf ne_numbercedxf ne_numberacedxf ne_numbercedxf ne_numbercedxf ne_numberbcedxf ne_numberbcedxf ne_numbercedxf wscedxf ne_numbercehdvf cne_numberw ccu mne_numberpccu mne_numberpccu mne_numberpcuu mne_numberap rm mne_numbergrm mne_numbere — ca ne_numberpca ne_numberapca ne_numberbca ne_numberpca ne_numberpca ne_numberca ne_numberpca ne_numbervct une_number ");
}

6
concordia/t/test_text_utils.cpp
View File

@ -2,19 +2,17 @@
#include "concordia/common/config.hpp"
#include "concordia/common/text_utils.hpp"
using namespace std;
BOOST_AUTO_TEST_SUITE(text_utils)
BOOST_AUTO_TEST_CASE( ToLower )
{
string str = "ZAŻÓŁĆ GĘŚLĄ JAŹŃ";
std::string str = "ZAŻÓŁĆ GĘŚLĄ JAŹŃ";
BOOST_CHECK_EQUAL(TextUtils::getInstance().toLowerCase(str),"zażółć gęślą jaźń");
}
BOOST_AUTO_TEST_CASE( ToUpper )
{
string str = "zażółć gęślą jaźń";
std::string str = "zażółć gęślą jaźń";
BOOST_CHECK_EQUAL(TextUtils::getInstance().toUpperCase(str),"ZAŻÓŁĆ GĘŚLĄ JAŹŃ");
}

2
concordia/t/test_tm_matches.cpp
View File

@ -3,8 +3,6 @@
#include "concordia/tm_matches.hpp"
#include "concordia/common/config.hpp"
using namespace std;
BOOST_AUTO_TEST_SUITE(tm_matches)
BOOST_AUTO_TEST_CASE( TmMatchesSimpleScore1 )

23
concordia/t/test_utils.cpp
View File

@ -5,23 +5,20 @@
#include <boost/filesystem.hpp>
#include "divsufsort.h"
#include <string>
using namespace std;
BOOST_AUTO_TEST_SUITE(utils)
BOOST_AUTO_TEST_CASE( WriteReadSingleCharacter )
{
ofstream testFileOutput;
std::ofstream testFileOutput;
testFileOutput.open(TestResourcesManager::getTestFilePath("temp","temp_file.bin").c_str(),
ios::out|ios::binary);
std::ios::out|std::ios::binary);
INDEX_CHARACTER_TYPE testCharacter = 123456789; //in hex: 75BCD15
Utils::writeIndexCharacter(testFileOutput,testCharacter);
testFileOutput.close();
ifstream testFileInput;
testFileInput.open(TestResourcesManager::getTestFilePath("temp","temp_file.bin").c_str(),ios::in|ios::binary);
std::ifstream testFileInput;
testFileInput.open(TestResourcesManager::getTestFilePath("temp","temp_file.bin").c_str(),std::ios::in|std::ios::binary);
INDEX_CHARACTER_TYPE retrievedCharacter = Utils::readIndexCharacter(testFileInput);
BOOST_CHECK_EQUAL(retrievedCharacter, testCharacter);
testFileInput.close();
@ -31,7 +28,7 @@ BOOST_AUTO_TEST_CASE( WriteReadSingleCharacter )
BOOST_AUTO_TEST_CASE( IndexVectorToSaucharArray )
{
vector<INDEX_CHARACTER_TYPE> hash;
std::vector<INDEX_CHARACTER_TYPE> hash;
hash.push_back(123456789); // in hex: 75BCD15
// in memory: 15 cd 5b 07
// in memory DEC: 21 205 91 7
@ -41,13 +38,13 @@ BOOST_AUTO_TEST_CASE( IndexVectorToSaucharArray )
// in memory DEC: 177 104 222 58
sauchar_t * dataArray = Utils::indexVectorToSaucharArray(hash);
vector<INDEX_CHARACTER_TYPE> result;
std::vector<INDEX_CHARACTER_TYPE> result;
for (int i=0;i<8;i++) {
INDEX_CHARACTER_TYPE a = dataArray[i];
result.push_back(a);
}
vector<INDEX_CHARACTER_TYPE> expected;
std::vector<INDEX_CHARACTER_TYPE> expected;
expected.push_back(21);
expected.push_back(205);
expected.push_back(91);
@ -62,7 +59,7 @@ BOOST_AUTO_TEST_CASE( IndexVectorToSaucharArray )
BOOST_AUTO_TEST_CASE( IndexVectorToSaucharVector )
{
vector<INDEX_CHARACTER_TYPE> hash;
std::vector<INDEX_CHARACTER_TYPE> hash;
hash.push_back(123456789); // in hex: 75BCD15
// in memory: 15 cd 5b 07
// in memory DEC: 21 205 91 7
@ -70,9 +67,9 @@ BOOST_AUTO_TEST_CASE( IndexVectorToSaucharVector )
hash.push_back(987654321); // in hex: 3ADE68B1
// in memory: b1 68 de 3a
// in memory DEC: 177 104 222 58
vector<sauchar_t> result = Utils::indexVectorToSaucharVector(hash);
std::vector<sauchar_t> result = Utils::indexVectorToSaucharVector(hash);
vector<sauchar_t> expected;
std::vector<sauchar_t> expected;
expected.push_back(21);
expected.push_back(205);
expected.push_back(91);

4
concordia/t/test_word_map.cpp
View File

@ -1,10 +1,6 @@
#include "tests/unit-tests/unit_tests_globals.hpp"
#include "concordia/word_map.hpp"
#include <string>
using namespace std;
BOOST_AUTO_TEST_SUITE(word_map)
BOOST_AUTO_TEST_CASE( WordCodeTest )

4
concordia/tm_matches.cpp
View File

@ -54,7 +54,7 @@ void TmMatches::addPatternInterval(int start, int end) {
}
bool TmMatches::_alreadyIntersects(
const vector<Interval> & intervalList,
const std::vector<Interval> & intervalList,
int start, int end) {
Interval tempInterval(start, end);
BOOST_FOREACH(Interval oldInterval, intervalList) {
@ -66,7 +66,7 @@ bool TmMatches::_alreadyIntersects(
}
double TmMatches::_getLogarithmicOverlay(
const vector<Interval> & intervalList,
const std::vector<Interval> & intervalList,
SUFFIX_MARKER_TYPE sentenceSize,
double k) {
double overlayScore = 0;

16
concordia/tm_matches.hpp
View File

@ -14,12 +14,10 @@
*/
using namespace std;
class TmMatches {
public:
TmMatches();
TmMatches(const SUFFIX_MARKER_TYPE exampleId,
const SUFFIX_MARKER_TYPE exampleSize,
const SUFFIX_MARKER_TYPE patternSize);
@ -32,11 +30,11 @@ public:
return _score;
}
vector<Interval> getExampleIntervals() const {
std::vector<Interval> getExampleIntervals() const {
return _exampleMatchedRegions;
}
vector<Interval> getPatternIntervals() const {
std::vector<Interval> getPatternIntervals() const {
return _patternMatchedRegions;
}
@ -53,18 +51,18 @@ public:
void addPatternInterval(int start, int end);
private:
bool _alreadyIntersects(const vector<Interval> & intervalList,
bool _alreadyIntersects(const std::vector<Interval> & intervalList,
int start, int end);
double _getLogarithmicOverlay(const vector<Interval> & intervalList,
double _getLogarithmicOverlay(const std::vector<Interval> & intervalList,
SUFFIX_MARKER_TYPE sentenceSize,
double k);
SUFFIX_MARKER_TYPE _exampleId;
vector<Interval> _exampleMatchedRegions;
std::vector<Interval> _exampleMatchedRegions;
vector<Interval> _patternMatchedRegions;
std::vector<Interval> _patternMatchedRegions;
SUFFIX_MARKER_TYPE _patternSize;

2
concordia/word_map.cpp
View File

@ -9,7 +9,7 @@ WordMap::WordMap() throw(ConcordiaException) {
WordMap::~WordMap() {
}
INDEX_CHARACTER_TYPE WordMap::getWordCode(const string & word)
INDEX_CHARACTER_TYPE WordMap::getWordCode(const std::string & word)
throw(ConcordiaException) {
if (_map.find(word) == _map.end()) {
if (_nextFree == INDEX_CHARACTER_TYPE_MAX_VALUE) {

8
concordia/word_map.hpp
View File

@ -14,8 +14,6 @@
*/
using namespace std;
class WordMap {
public:
explicit WordMap() throw(ConcordiaException);
@ -24,8 +22,8 @@ public:
*/
virtual ~WordMap();
INDEX_CHARACTER_TYPE getWordCode(const string & word)
throw(ConcordiaException);
INDEX_CHARACTER_TYPE getWordCode(const std::string & word)
throw(ConcordiaException);
private:
friend class boost::serialization::access;
@ -37,7 +35,7 @@ private:
ar & _nextFree;
}
map<string, INDEX_CHARACTER_TYPE> _map;
std::map<std::string, INDEX_CHARACTER_TYPE> _map;
INDEX_CHARACTER_TYPE _nextFree;
};

16
tests/common/test_resources_manager.cpp
View File

@ -4,24 +4,24 @@
#define CONCORDIA_TAGSET_DIRECTORY "concordia-tagset"
#define CONCORDIA_CONFIG_DIRECTORY "concordia-config"
string TestResourcesManager::getPuddleFilePath(const string & filename) {
string result = string(TEST_RESOURCES_DIRECTORY);
std::string TestResourcesManager::getPuddleFilePath(const std::string & filename) {
std::string result = std::string(TEST_RESOURCES_DIRECTORY);
return result + "/" + PUDDLE_TEST_DIRECTORY + "/" + filename;
}
string TestResourcesManager::getTestConcordiaConfigFilePath(const string & filename) {
string result = string(TEST_RESOURCES_DIRECTORY);
std::string TestResourcesManager::getTestConcordiaConfigFilePath(const std::string & filename) {
std::string result = std::string(TEST_RESOURCES_DIRECTORY);
return result + "/" + CONCORDIA_CONFIG_DIRECTORY + "/" + filename;
}
string TestResourcesManager::getProdConcordiaConfigFilePath(const string & filename) {
string result = string(PROD_RESOURCES_DIRECTORY);
std::string TestResourcesManager::getProdConcordiaConfigFilePath(const std::string & filename) {
std::string result = std::string(PROD_RESOURCES_DIRECTORY);
return result + "/" + CONCORDIA_CONFIG_DIRECTORY + "/" + filename;
}
string TestResourcesManager::getTestFilePath(const string & module, const string & filename) {
string result = string(TEST_RESOURCES_DIRECTORY);
std::string TestResourcesManager::getTestFilePath(const std::string & module, const std::string & filename) {
std::string result = std::string(TEST_RESOURCES_DIRECTORY);
return result + "/" + module + "/" + filename;
}

10
tests/common/test_resources_manager.hpp
View File

@ -6,17 +6,15 @@
#include "concordia/common/config.hpp"
using namespace std;
class TestResourcesManager {
public:
static string getPuddleFilePath(const string & filename);
static std::string getPuddleFilePath(const std::string & filename);
static string getTestConcordiaConfigFilePath(const string & filename);
static std::string getTestConcordiaConfigFilePath(const std::string & filename);
static string getProdConcordiaConfigFilePath(const string & filename);
static std::string getProdConcordiaConfigFilePath(const std::string & filename);
static string getTestFilePath(const string & module, const string & filename);
static std::string getTestFilePath(const std::string & module, const std::string & filename);
};