concordia-library/concordia/tokenized_sentence.hpp

#ifndef TOKENIZED_SENTENCE_HDR
#define TOKENIZED_SENTENCE_HDR

#include "concordia/common/config.hpp"
#include "concordia/token_annotation.hpp"
#include "concordia/word_map.hpp"

#include <boost/shared_ptr.hpp>
#include <string>
#include <vector>
#include <list>
#include <iostream>
#include <boost/foreach.hpp>



/*!
  A sentence after tokenizing operations. The class
  holds the current string represenation of the sentence
  along with the annotations list. The class also allows
  for generating hash. After that operation the class
  also holds the list of hashed codes and corresponding
  tokens.
*/

class TokenizedSentence {
public:
    /*!
      Constructor.

    */
    explicit TokenizedSentence(std::string sentence);

    /*! Destructor.
    */
    virtual ~TokenizedSentence();

    /*! Getter for the string sentence,
        which might have been modified during tokenization.
      \returns sentence
    */
    std::string getSentence() const {
        return _sentence;
    }

    /*! Getter for the original string sentence,
        which was used for extracting tokens.
      \returns originalSentence
    */
    std::string getOriginalSentence() const {
        return _originalSentence;
    }

    /*! Method for getting tokenized sentence in a string format (
    tokens separated by single spaces.
      \returns tokenized sentence
    */
    std::string getTokenizedSentence() const;

    /*! Getter for all annotations list. This method returns
        all annotations, including those which are not considered
        in the hash, i.e. stop words and html tags.
      \returns annotations list
    */
    std::list<TokenAnnotation> getAnnotations() const {
        return _tokenAnnotations;
    }

    /*! Getter for codes list. This data is available after calling
        the hashGenerator method.
      \returns codes list
    */
    std::vector<INDEX_CHARACTER_TYPE> getCodes() const {
        return _codes;
    }

    /*! Getter for tokens list. This method returns
        only those annotations considered
        in the hash, i.e. words and named entities.
      \returns tokens list
    */
    std::vector<TokenAnnotation> getTokens() const {
        return _tokens;
    }

    /*! Method for generating hash based on annotations.
        This method takes into account annotations of type
        word and named entity. These are encoded and added
        to code list. Annotations corresponding to these
        tokens are added to the tokens list.
      \param wordMap word map to use when encoding tokens
    */
    void generateHash(boost::shared_ptr<WordMap> wordMap);

    /*! Method for generating tokens based on annotations.
        This method takes into account annotations of type
        word and named entity. Unlike in generateHash,
        these are not encoded or added to code list.
        Annotations corresponding to these
        tokens are added to the tokens list.
    */
    void generateTokens();

    /*!
        Transform the sentence to lower case.
    */
    void toLowerCase();

    /*!
        Add new annotations to the existing annotations list. Assumptions:
        1. existing _tokenAnnotations vector contains disjoint, sorted intervals;
        2. the annotations to be added list also has the above properties.
        The below algorithm will only add the annotations that do not
        intersect with any of the existing ones.

        \param annotations list of annotations to be added
    */
    void addAnnotations(std::vector<TokenAnnotation> annotations);

    friend std::ostream & operator << (std::ostream & o,
                          const TokenizedSentence & ts) {
        int index = 0;
        BOOST_FOREACH(TokenAnnotation token, ts.getAnnotations()) {
            o << "[" << token.getStart() << "," << token.getEnd() << "]["
              << token.getType() << "][" << token.getValue() <<"]";
            if (index < ts.getAnnotations().size() - 1) {
                o << " ";
            }
            index++;
        }
        return o;
    }


private:
    std::string _sentence;

    std::string _originalSentence;

    std::list<TokenAnnotation> _tokenAnnotations;

    std::vector<INDEX_CHARACTER_TYPE> _codes;

    std::vector<TokenAnnotation> _tokens;
};

#endif