added lowercasing when tokenizing by space

2015-12-29 21:44:46 +01:00 · 2015-12-29 21:44:46 +01:00 · bbf3853d2a
commit bbf3853d2a
parent 0a8d2fdd39
1 changed files with 4 additions and 2 deletions
--- a/concordia/sentence_tokenizer.cpp
+++ b/concordia/sentence_tokenizer.cpp
@ -29,6 +29,8 @@ TokenizedSentence SentenceTokenizer::tokenize(const std::string & sentence,
    TokenizedSentence result(sentence);

    if (byWhitespace) {
+        result.toLowerCase();
+
        boost::shared_ptr<RegexRule> whitespaceRule(
                            new RegexRule("\\S+",
                                          TokenAnnotation::WORD, ""));