137 lines
3.9 KiB
C++
137 lines
3.9 KiB
C++
#ifndef TOKENIZED_SENTENCE_HDR
|
|
#define TOKENIZED_SENTENCE_HDR
|
|
|
|
#include "concordia/common/config.hpp"
|
|
#include "concordia/token_annotation.hpp"
|
|
#include "concordia/word_map.hpp"
|
|
|
|
#include <boost/shared_ptr.hpp>
|
|
#include <string>
|
|
#include <vector>
|
|
#include <list>
|
|
#include <iostream>
|
|
#include <boost/foreach.hpp>
|
|
|
|
|
|
|
|
/*!
|
|
A sentence after tokenizing operations. The class
|
|
holds the current string represenation of the sentence
|
|
along with the annotations list. The class also allows
|
|
for generating hash. After that operation the class
|
|
also holds the list of hashed codes and corresponding
|
|
tokens.
|
|
*/
|
|
|
|
class TokenizedSentence {
|
|
public:
|
|
/*!
|
|
Constructor.
|
|
|
|
*/
|
|
explicit TokenizedSentence(std::string sentence);
|
|
|
|
/*! Destructor.
|
|
*/
|
|
virtual ~TokenizedSentence();
|
|
|
|
/*! Getter for the string sentence, which is used for extracting tokens.
|
|
\returns sentence
|
|
*/
|
|
std::string getSentence() const {
|
|
return _sentence;
|
|
}
|
|
|
|
/*! Method for getting tokenized sentence in a string format (
|
|
tokens separated by single spaces.
|
|
\returns tokenized sentence
|
|
*/
|
|
std::string getTokenizedSentence() const;
|
|
|
|
/*! Getter for all annotations list. This method returns
|
|
all annotations, including those which are not considered
|
|
in the hash, i.e. stop words and html tags.
|
|
\returns annotations list
|
|
*/
|
|
std::list<TokenAnnotation> getAnnotations() const {
|
|
return _tokenAnnotations;
|
|
}
|
|
|
|
/*! Getter for codes list. This data is available after calling
|
|
the hashGenerator method.
|
|
\returns codes list
|
|
*/
|
|
std::vector<INDEX_CHARACTER_TYPE> getCodes() const {
|
|
return _codes;
|
|
}
|
|
|
|
/*! Getter for tokens list. This method returns
|
|
only those annotations considered
|
|
in the hash, i.e. words and named entities.
|
|
\returns tokens list
|
|
*/
|
|
std::vector<TokenAnnotation> getTokens() const {
|
|
return _tokens;
|
|
}
|
|
|
|
/*! Method for generating hash based on annotations.
|
|
This method takes into account annotations of type
|
|
word and named entity. These are encoded and added
|
|
to code list. Annotations corresponding to these
|
|
tokens are added to the tokens list.
|
|
\param wordMap word map to use when encoding tokens
|
|
*/
|
|
void generateHash(boost::shared_ptr<WordMap> wordMap);
|
|
|
|
/*! Method for generating tokens based on annotations.
|
|
This method takes into account annotations of type
|
|
word and named entity. Unlike in generateHash,
|
|
these are not encoded or added to code list.
|
|
Annotations corresponding to these
|
|
tokens are added to the tokens list.
|
|
*/
|
|
void generateTokens();
|
|
|
|
/*!
|
|
Transform the sentence to lower case.
|
|
*/
|
|
void toLowerCase();
|
|
|
|
/*!
|
|
Add new annotations to the existing annotations list. Assumptions:
|
|
1. existing _tokenAnnotations vector contains disjoint, sorted intervals;
|
|
2. the annotations to be added list also has the above properties.
|
|
The below algorithm will only add the annotations that do not
|
|
intersect with any of the existing ones.
|
|
|
|
\param annotations list of annotations to be added
|
|
*/
|
|
void addAnnotations(std::vector<TokenAnnotation> annotations);
|
|
|
|
friend std::ostream & operator << (std::ostream & o,
|
|
const TokenizedSentence & ts) {
|
|
int index = 0;
|
|
BOOST_FOREACH(TokenAnnotation token, ts.getAnnotations()) {
|
|
o << "[" << token.getStart() << "," << token.getEnd() << "]["
|
|
<< token.getType() << "][" << token.getValue() <<"]";
|
|
if (index < ts.getAnnotations().size() - 1) {
|
|
o << " ";
|
|
}
|
|
index++;
|
|
}
|
|
return o;
|
|
}
|
|
|
|
|
|
private:
|
|
std::string _sentence;
|
|
|
|
std::list<TokenAnnotation> _tokenAnnotations;
|
|
|
|
std::vector<INDEX_CHARACTER_TYPE> _codes;
|
|
|
|
std::vector<TokenAnnotation> _tokens;
|
|
};
|
|
|
|
#endif
|