rjawor/concordia-aligner
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lemmatizer

Browse Source
This commit is contained in:
rjawor 2019-06-26 09:08:00 +02:00
parent 021c483a2c
commit 8b9deb906f
2 changed files with 17 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Makefile
View File

@ -34,17 +34,20 @@ corpora/$(CORPUS_NAME)/falign_result.txt: corpora/$(CORPUS_NAME)/falign_corpus.t
corpora/$(CORPUS_NAME)/src.lem: corpora/$(CORPUS_NAME)/src.txt
./sentence_lemmatizer.py $< $(SRC_LANG) corpora/$(CORPUS_NAME)/src.norm corpora/$(CORPUS_NAME)/src.lem
corpora/$(CORPUS_NAME)/trg.lem: corpora/$(CORPUS_NAME)/trg.txt
./sentence_lemmatizer.py $< $(TRG_LANG) corpora/$(CORPUS_NAME)/trg.norm corpora/$(CORPUS_NAME)/trg.lem
corpora/$(CORPUS_NAME)/src_clean.txt corpora/$(CORPUS_NAME)/src_clean.lem corpora/$(CORPUS_NAME)/trg_clean.txt corpora/$(CORPUS_NAME)/ids_clean.txt corpora/$(CORPUS_NAME)/falign_corpus.txt: corpora/$(CORPUS_NAME)/src.txt corpora/$(CORPUS_NAME)/trg.txt corpora/$(CORPUS_NAME)/ids.txt corpora/$(CORPUS_NAME)/src.lem corpora/$(CORPUS_NAME)/trg.lem corpora/$(CORPUS_NAME)/src.dict corpora/$(CORPUS_NAME)/trg.dict corpora/$(CORPUS_NAME)/src_clean.txt corpora/$(CORPUS_NAME)/src_clean.lem corpora/$(CORPUS_NAME)/trg_clean.txt corpora/$(CORPUS_NAME)/ids_clean.txt corpora/$(CORPUS_NAME)/falign_corpus.txt: corpora/$(CORPUS_NAME)/src.txt corpora/$(CORPUS_NAME)/trg.txt corpora/$(CORPUS_NAME)/ids.txt corpora/$(CORPUS_NAME)/src.lem corpora/$(CORPUS_NAME)/trg.lem corpora/$(CORPUS_NAME)/src.dict corpora/$(CORPUS_NAME)/trg.dict
./prepare_corpus.py corpora/$(CORPUS_NAME)/src.norm corpora/$(CORPUS_NAME)/trg.norm corpora/$(CORPUS_NAME)/ids.txt corpora/$(CORPUS_NAME)/src.lem corpora/$(CORPUS_NAME)/trg.lem corpora/$(CORPUS_NAME)/src.dict corpora/$(CORPUS_NAME)/trg.dict corpora/$(CORPUS_NAME)/src_clean.txt corpora/$(CORPUS_NAME)/src_clean.lem corpora/$(CORPUS_NAME)/trg_clean.txt corpora/$(CORPUS_NAME)/ids_clean.txt corpora/$(CORPUS_NAME)/falign_corpus.txt $(SRC_LANG) $(TRG_LANG) ./prepare_corpus.py corpora/$(CORPUS_NAME)/src.norm corpora/$(CORPUS_NAME)/trg.norm corpora/$(CORPUS_NAME)/ids.txt corpora/$(CORPUS_NAME)/src.lem corpora/$(CORPUS_NAME)/trg.lem corpora/$(CORPUS_NAME)/src.dict corpora/$(CORPUS_NAME)/trg.dict corpora/$(CORPUS_NAME)/src_clean.txt corpora/$(CORPUS_NAME)/src_clean.lem corpora/$(CORPUS_NAME)/trg_clean.txt corpora/$(CORPUS_NAME)/ids_clean.txt corpora/$(CORPUS_NAME)/falign_corpus.txt $(SRC_LANG) $(TRG_LANG)
corpora/$(CORPUS_NAME)/src.lem: corpora/$(CORPUS_NAME)/src.tok
./sentence_lemmatizer.py $< $(SRC_LANG) $@
corpora/$(CORPUS_NAME)/trg.lem: corpora/$(CORPUS_NAME)/trg.tok
./sentence_lemmatizer.py $< $(TRG_LANG) $@
corpora/$(CORPUS_NAME)/trg.tok: corpora/$(CORPUS_NAME)/trg.txt corpora/$(CORPUS_NAME)/trg.tok: corpora/$(CORPUS_NAME)/trg.txt
/usr/local/bin/concordia-sentence-tokenizer -c /code/prod/resources/concordia-config/concordia.cfg < $< > $@ /usr/local/bin/concordia-sentence-tokenizer -c /code/prod/resources/concordia-config/concordia.cfg < $< > $@

26
sentence_lemmatizer.py
View File

@ -9,40 +9,32 @@ BUFFER_SIZE = 500
def lemmatize_sentences(language_code, sentences): def lemmatize_sentences(language_code, sentences):
data = { data = {
'lemmatize': True,
'language':language_code, 'language':language_code,
'sentences':sentences 'sentences':sentences
} }
response = requests.post(url = 'http://concordia-preprocessor:9001/preprocess', json = data, timeout = 120) response = requests.post(url = 'http://concordia-preprocessor:9001/lemmatize', json = data, timeout = 120)
response_json = json.loads(response.text) response_json = json.loads(response.text)
result = {'normalized':[], 'lemmatized':[]} return response_json['processed_sentences']
for processed_sentence in response_json['processed_sentences']:
result['normalized'].append(processed_sentence['normalized'])
result['lemmatized'].append(processed_sentence['tokens'])
return result
def write_result(result, norm_file, lem_file): def write_result(result, lem_file):
for s in result['normalized']: for s in result:
norm_file.write(s+'\n') lem_file.write(s['tokens']+'\n')
for s in result['lemmatized']:
lem_file.write(s+'\n')
file_name = sys.argv[1] file_name = sys.argv[1]
language_code = sys.argv[2] language_code = sys.argv[2]
norm_output_name = sys.argv[3] lem_output_name = sys.argv[3]
lem_output_name = sys.argv[4]
sentences_buffer = [] sentences_buffer = []
with open(file_name) as in_file, open(norm_output_name, 'w') as out_norm, open(lem_output_name, 'w') as out_lem: with open(file_name) as in_file, open(lem_output_name, 'w') as out_lem:
for line in in_file: for line in in_file:
sentences_buffer.append(line.rstrip()) sentences_buffer.append(line.rstrip())
if len(sentences_buffer) == BUFFER_SIZE: if len(sentences_buffer) == BUFFER_SIZE:
write_result(lemmatize_sentences(language_code,sentences_buffer), out_norm, out_lem) write_result(lemmatize_sentences(language_code,sentences_buffer), out_lem)
sentences_buffer = [] sentences_buffer = []
if len(sentences_buffer) > 0: if len(sentences_buffer) > 0:
write_result(lemmatize_sentences(language_code,sentences_buffer), out_norm, out_lem) write_result(lemmatize_sentences(language_code,sentences_buffer), out_lem)