added lowercasing when tokenizing by space

concordia/sentence_tokenizer.cpp

@@ -28,7 +28,9 @@ TokenizedSentence SentenceTokenizer::tokenize(const std::string & sentence,
                                               bool byWhitespace) {
     TokenizedSentence result(sentence);
 
-    if(byWhitespace) {
+    if (byWhitespace) {
+        result.toLowerCase();
+
         boost::shared_ptr<RegexRule> whitespaceRule(
                             new RegexRule("\\S+",
                                           TokenAnnotation::WORD, ""));
@@ -54,7 +56,7 @@ TokenizedSentence SentenceTokenizer::tokenize(const std::string & sentence,
                             new RegexRule("\\p{L}", TokenAnnotation::WORD, ""));
         singleLetterWordsRule->apply(result);
     }
-    
+
     return result;
 }