added lowercasing when tokenizing by space

concordia/sentence_tokenizer.cpp

@@ -28,7 +28,9 @@ TokenizedSentence SentenceTokenizer::tokenize(const std::string & sentence,
                                               bool byWhitespace) {
     TokenizedSentence result(sentence);
 
-    if(byWhitespace) {
+    if (byWhitespace) {
+        result.toLowerCase();
+
         boost::shared_ptr<RegexRule> whitespaceRule(
                             new RegexRule("\\S+",
                                           TokenAnnotation::WORD, ""));