concordia-server/fast-aligner/clean_corpus.py

#!/usr/bin/python3
# -*- coding: utf-8 -*-

import sys

max_tokens = 100
max_ratio = 4.0

#./clean_corpus.py corpora/$(CORPUS_NAME)/src.txt corpora/$(CORPUS_NAME)/trg.txt corpora/$(CORPUS_NAME)/ids.txt corpora/$(CORPUS_NAME)/src.lem corpora/$(CORPUS_NAME)/trg.lem corpora/$(CORPUS_NAME)/src_clean.txt corpora/$(CORPUS_NAME)/trg_clean.txt corpora/$(CORPUS_NAME)/ids_clean.txt corpora/$(CORPUS_NAME)/falign_corpus.txt


with open(sys.argv[1]) as src, open(sys.argv[2]) as trg, open(sys.argv[3]) as ids, open(sys.argv[4]) as src_lem, open(sys.argv[5]) as trg_lem, open(sys.argv[6], 'w') as src_clean, open(sys.argv[7], 'w') as trg_clean, open(sys.argv[8], 'w') as ids_clean, open(sys.argv[9], 'w') as falign_corpus:
    for line in src:
        src_line_orig = line.strip()
        trg_line_orig = trg.readline().strip()
        id_orig = ids.readline().strip()
        src_line_lem = src_lem.readline().strip()
        trg_line_lem = trg_lem.readline().strip()
        src_token_count = len(src_line_lem.split())
        trg_token_count = len(trg_line_lem.split())
        if (src_token_count > 0 and trg_token_count > 0 and src_token_count <= max_tokens and trg_token_count <= max_tokens):
            ratio = float(src_token_count/trg_token_count) if src_token_count > trg_token_count else float(trg_token_count/src_token_count)
            if (ratio <= max_ratio):
                src_clean.write(src_line_orig+"\n")
                trg_clean.write(trg_line_orig+"\n")
                ids_clean.write(id_orig+"\n")
                falign_corpus.write("%s|||%s\n" % (src_line_lem, trg_line_lem))