90 lines
3.3 KiB
C++
90 lines
3.3 KiB
C++
#include "concordia/tokenized_sentence.hpp"
|
|
#include "concordia/common/text_utils.hpp"
|
|
|
|
#include <iostream>
|
|
#include <sstream>
|
|
#include <boost/foreach.hpp>
|
|
#include <boost/algorithm/string.hpp>
|
|
|
|
TokenizedSentence::TokenizedSentence(std::string sentence):
|
|
_sentence(sentence) {
|
|
}
|
|
|
|
TokenizedSentence::~TokenizedSentence() {
|
|
}
|
|
|
|
void TokenizedSentence::addAnnotations(
|
|
std::vector<TokenAnnotation> annotations) {
|
|
std::vector<TokenAnnotation>::iterator newAnnotation =
|
|
annotations.begin();
|
|
std::list<TokenAnnotation>::iterator existingAnnotation =
|
|
_tokenAnnotations.begin();
|
|
|
|
while (newAnnotation != annotations.end()) {
|
|
if (existingAnnotation != _tokenAnnotations.end()) {
|
|
// there are still some existing annotations, so perform checks
|
|
if (newAnnotation->intersects(*existingAnnotation)) {
|
|
// The new annotation intersects with the existing.
|
|
// We can not add it, so let us just move on to the
|
|
// next new annoation.
|
|
++newAnnotation;
|
|
} else {
|
|
// it is now important whether the new interval is before
|
|
// or after existing
|
|
if (newAnnotation->getStart() <
|
|
existingAnnotation->getStart()) {
|
|
// New interval does not intersect and is
|
|
// before existing. We add it.
|
|
_tokenAnnotations.insert(existingAnnotation,
|
|
*newAnnotation);
|
|
++newAnnotation;
|
|
} else {
|
|
// If the new interval is after existing
|
|
// we move to the next existing annoation.
|
|
++existingAnnotation;
|
|
}
|
|
}
|
|
} else {
|
|
// no more existing annotations, so just add the new annotation
|
|
_tokenAnnotations.push_back(*newAnnotation);
|
|
++newAnnotation;
|
|
}
|
|
}
|
|
}
|
|
|
|
void TokenizedSentence::toLowerCase() {
|
|
_sentence = TextUtils::getInstance().toLowerCase(_sentence);
|
|
}
|
|
|
|
void TokenizedSentence::generateHash(boost::shared_ptr<WordMap> wordMap) {
|
|
BOOST_FOREACH(TokenAnnotation annotation, _tokenAnnotations) {
|
|
if (annotation.getType() == TokenAnnotation::WORD ||
|
|
annotation.getType() == TokenAnnotation::NE) {
|
|
_codes.push_back(wordMap->getWordCode(annotation.getValue()));
|
|
_tokens.push_back(annotation);
|
|
}
|
|
}
|
|
}
|
|
|
|
void TokenizedSentence::generateTokens() {
|
|
BOOST_FOREACH(TokenAnnotation annotation, _tokenAnnotations) {
|
|
if (annotation.getType() == TokenAnnotation::WORD ||
|
|
annotation.getType() == TokenAnnotation::NE) {
|
|
_tokens.push_back(annotation);
|
|
}
|
|
}
|
|
}
|
|
|
|
std::string TokenizedSentence::getTokenizedSentence() const {
|
|
std::stringstream ss;
|
|
BOOST_FOREACH(TokenAnnotation annotation, _tokenAnnotations) {
|
|
if (annotation.getType() == TokenAnnotation::WORD ||
|
|
annotation.getType() == TokenAnnotation::NE) {
|
|
ss << annotation.getValue() << " ";
|
|
}
|
|
}
|
|
std::string result = ss.str();
|
|
boost::trim_right(result);
|
|
return result;
|
|
}
|