rjawor/concordia-library
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

added lowercasing when tokenizing by space

Browse Source
This commit is contained in:
rjawor 2015-12-29 21:44:46 +01:00
parent 0a8d2fdd39
commit bbf3853d2a
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
concordia/sentence_tokenizer.cpp
View File

@ -29,6 +29,8 @@ TokenizedSentence SentenceTokenizer::tokenize(const std::string & sentence,
TokenizedSentence result(sentence); TokenizedSentence result(sentence);
if (byWhitespace) { if (byWhitespace) {
result.toLowerCase();
boost::shared_ptr<RegexRule> whitespaceRule( boost::shared_ptr<RegexRule> whitespaceRule(
new RegexRule("\\S+", new RegexRule("\\S+",
TokenAnnotation::WORD, "")); TokenAnnotation::WORD, ""));