concordia-library/concordia/sentence_anonymizer.cpp

#include "concordia/sentence_anonymizer.hpp"

#include <boost/foreach.hpp>
#include <fstream>
#include <sstream>
#include <iostream>
#include <boost/algorithm/string.hpp>

SentenceAnonymizer::SentenceAnonymizer(
                        boost::shared_ptr<ConcordiaConfig> config)
                                         throw(ConcordiaException) {
    _createNeRules(config->getNamedEntitiesFilePath());
    _createHtmlTagsRule(config->getHtmlTagsFilePath());
    _stopWordsEnabled = config->isStopWordsEnabled();
    if (_stopWordsEnabled) {
        _stopWords = _getMultipleReplacementRule(
                                  config->getStopWordsFilePath(), "", true);
    }
    _stopSymbols = _getMultipleReplacementRule(
                              config->getStopSymbolsFilePath(), "");
    _spaceSymbols = _getMultipleReplacementRule(
                              config->getSpaceSymbolsFilePath(), " ");
}

SentenceAnonymizer::~SentenceAnonymizer() {
}

boost::shared_ptr<AnonymizedSentence>
              SentenceAnonymizer::anonymize(const std::string & sentence) {
    boost::shared_ptr<AnonymizedSentence> 
                    result(new AnonymizedSentence(sentence));

    _htmlTags->apply(result);

    BOOST_FOREACH(RegexRule & neRule, _namedEntities) {
        neRule.apply(result);
    }

    result->toLowerCase();

    if (_stopWordsEnabled) {
        _stopWords->apply(result);
    }
    _stopSymbols->apply(result);
    _spaceSymbols->apply(result);

    return result;
}

void SentenceAnonymizer::_createNeRules(std::string & namedEntitiesPath) {
    if (boost::filesystem::exists(namedEntitiesPath)) {
        std::string line;
        std::ifstream neFile(namedEntitiesPath.c_str());
        if (neFile.is_open()) {
            int lineCounter = 0;
            while (getline(neFile, line)) {
                lineCounter++;
                boost::shared_ptr<std::vector<std::string> >
                                 tokenTexts(new std::vector<std::string>());
                boost::split(*tokenTexts, line, boost::is_any_of(" "),
                             boost::token_compress_on);
                if (tokenTexts->size() != 2) {
                    std::stringstream ss;
                    ss << "Invalid line: " << lineCounter
                       << " in NE file: " << namedEntitiesPath;
                    throw ConcordiaException(ss.str());
                } else {
                    _namedEntities.push_back(RegexRule(
                                tokenTexts->at(0), tokenTexts->at(1)));
                }
            }
            neFile.close();
        } else {
            throw ConcordiaException("Unable to read named entities file.");
        }
    } else {
        throw ConcordiaException("No named entities file.");
    }
}

void SentenceAnonymizer::_createHtmlTagsRule(std::string & htmlTagsPath) {
    std::string tagsExpression = "<\\/?(";
    if (boost::filesystem::exists(htmlTagsPath)) {
        std::string line;
        std::ifstream tagsFile(htmlTagsPath.c_str());
        if (tagsFile.is_open()) {
            while (getline(tagsFile, line)) {
                tagsExpression += "|";
            }
           tagsFile.close();
        } else {
            throw ConcordiaException("Unable to read html tags file.");
        }
    } else {
        throw ConcordiaException("No html tags file.");
    }
    tagsExpression = tagsExpression.substr(0, tagsExpression.size()-1);
    tagsExpression += "br).*?>";
    _htmlTags = boost::shared_ptr<RegexRule>(
                        new RegexRule(tagsExpression, "", false));
}

boost::shared_ptr<RegexRule>
        SentenceAnonymizer::_getMultipleReplacementRule(
            std::string & filePath, std::string replacement, bool wholeWord) {
    std::string expression = "(";
    if (boost::filesystem::exists(filePath)) {
        std::string line;
        std::ifstream ruleFile(filePath.c_str());
        if (ruleFile.is_open()) {
            while (getline(ruleFile, line)) {
                if (wholeWord) {
                    expression += "\\b";
                }
                expression += line;
                if (wholeWord) {
                    expression += "\\b";
                }
                expression += "|";
            }
           ruleFile.close();
        } else {
            throw ConcordiaException("Unable to read file: "+filePath);
        }
    } else {
        throw ConcordiaException("No "+filePath+" file.");
    }
    expression = expression.substr(0, expression.size()-1);
    expression += ")";
    return boost::shared_ptr<RegexRule>(
                        new RegexRule(expression, replacement, false));
}