concordia-library/concordia/sentence_tokenizer.hpp

#ifndef SENTENCE_TOKENIZER_HDR
#define SENTENCE_TOKENIZER_HDR

#include <string>
#include <vector>
#include "concordia/common/config.hpp"
#include "concordia/tokenized_sentence.hpp"
#include "concordia/regex_rule.hpp"
#include "concordia/concordia_config.hpp"
#include "concordia/concordia_exception.hpp"
#include <boost/shared_ptr.hpp>
#include <boost/filesystem.hpp>


/*!
  Class for tokenizing sentence before generating hash.
  This operation is is used to
  remove unnecessary symbols and possibly words from sentences added to index
  and search patterns. Tokenizer annotates html tags, removes stop words (if the option is enabled),
  as well as annotates named entities and special symbols. All these have to be listed in files
  (see \ref tutorial3).
*/

class SentenceTokenizer {
public:
    /*! Constructor.
      \param config config object, holding paths to necessary files
    */
    explicit SentenceTokenizer(boost::shared_ptr<ConcordiaConfig> config)
                                                 throw(ConcordiaException);

    /*! Destructor.
    */
    virtual ~SentenceTokenizer();

    /*! Tokenizes the sentence.
      \param sentence input sentence
      \returns altered version of the input sentence
    */
    boost::shared_ptr<TokenizedSentence>
                                   tokenize(const std::string & sentence);

private:
    void _createNeRules(std::string & namedEntitiesPath);

    void _createHtmlTagsRule(std::string & htmlTagsPath);

    boost::shared_ptr<RegexRule> _getMultipleRegexRule(
                                             std::string filePath,
                                             char annotationType,
                                             std::string value,
                                             bool wholeWord = false);

    std::vector<RegexRule> _namedEntities;

    boost::shared_ptr<RegexRule> _htmlTags;

    bool _stopWordsEnabled;

    boost::shared_ptr<RegexRule> _stopWords;

};

#endif