working simple search

This commit is contained in:
rjawor 2019-08-29 23:16:48 +02:00
parent 0fe53f05fb
commit f38734c6a7
3 changed files with 48 additions and 33 deletions

View File

@ -2,4 +2,4 @@
source corpus.cfg
./add_fast_aligned_TM.py $CORPUS_NAME $CORPUS_PATH/src_clean.txt $CORPUS_PATH/src_clean.lem $SRC_LANG_ID $CORPUS_PATH/trg_clean.txt $TRG_LANG_ID $CORPUS_PATH/alignments.txt $CORPUS_PATH/ids_clean.txt
./add_fast_aligned_TM.py $CORPUS_NAME $CORPUS_PATH/src_clean.txt $CORPUS_PATH/src_clean.tok $CORPUS_PATH/src_clean.lem $SRC_LANG_ID $CORPUS_PATH/trg_clean.txt $TRG_LANG_ID $CORPUS_PATH/alignments.txt $CORPUS_PATH/ids_clean.txt

View File

@ -18,33 +18,40 @@ def file_len(fname):
pass
return i + 1
def add_examples(examplesData):
def add_examples(examples, tmId):
examplesData = {
'operation': 'addSentences',
'tmId':tmId,
'examples':examples
}
req = urllib2.Request(address)
req.add_header('Content-Type', 'application/json')
response = json.loads(urllib2.urlopen(req, json.dumps(examplesData), timeout = 3600).read())
print(response)
#print(response)
if response['status'] == 'error':
raise Exception(response['message'])
if len(sys.argv) != 9:
if len(sys.argv) != 10:
raise Exception("wrong number of arguments")
name = sys.argv[1]
sourceFile = sys.argv[2]
lemmatizedSourceFile = sys.argv[3]
sourceLangId = int(sys.argv[4])
targetFile = sys.argv[5]
targetLangId = int(sys.argv[6])
alignmentsFile = sys.argv[7]
sourceIdsFile = sys.argv[8]
tokenizedSourceFile = sys.argv[3]
lemmatizedSourceFile = sys.argv[4]
sourceLangId = int(sys.argv[5])
targetFile = sys.argv[6]
targetLangId = int(sys.argv[7])
alignmentsFile = sys.argv[8]
sourceIdsFile = sys.argv[9]
sourceFileLength = file_len(sourceFile)
tokenizedSourceFileLength = file_len(tokenizedSourceFile)
lemmatizedSourceFileLength = file_len(lemmatizedSourceFile)
targetFileLength = file_len(targetFile)
alignmentsFileLength = file_len(alignmentsFile)
sourceIdsFileLength = file_len(sourceIdsFile)
if not (sourceFileLength == lemmatizedSourceFileLength and lemmatizedSourceFileLength == targetFileLength and targetFileLength == alignmentsFileLength and alignmentsFileLength == sourceIdsFileLength):
if not (sourceFileLength == tokenizedSourceFileLength and tokenizedSourceFileLength == lemmatizedSourceFileLength and lemmatizedSourceFileLength == targetFileLength and targetFileLength == alignmentsFileLength and alignmentsFileLength == sourceIdsFileLength):
print("File lengths:")
print("source file: %d\nlemmatized source file: %d\ntarget file: %d\nalignments file: %d\nsource ids file: %d" % (sourceFileLength, lemmatizedSourceFileLength, targetFileLength, alignmentsFileLength, sourceIdsFileLength))
raise Exception("files are not of the same length!")
@ -52,59 +59,67 @@ if not (sourceFileLength == lemmatizedSourceFileLength and lemmatizedSourceFileL
totalExamples = sourceFileLength / LEAVE_OUT
data = {
'operation': 'addTm',
'operation': 'addPairedTms',
'sourceLangId':sourceLangId,
'targetLangId':targetLangId,
'name':name,
'tmLemmatized':True
'name':name
}
req = urllib2.Request(address)
req.add_header('Content-Type', 'application/json')
response = json.loads(urllib2.urlopen(req, json.dumps(data), timeout = 3600).read())
print(response)
tmId = int(response['newTmId'])
print "Added new tm: %d" % tmId
#print(response)
data = {
'operation': 'addSentences',
'tmId':tmId
}
lemmatizedTmId = int(response['lemmatizedTmId'])
nonLemmatizedTmId = int(response['nonLemmatizedTmId'])
print "Added new paired tm: lemmatized id: %d, non lemmatized id: %d" % (lemmatizedTmId, nonLemmatizedTmId)
examples = []
examples_lemmatized = []
start = time.time()
with open(sourceFile) as source_file, open(lemmatizedSourceFile) as lemmatized_source_file, open(targetFile) as target_file, open(alignmentsFile) as alignments_file, open(sourceIdsFile) as source_ids_file:
with open(sourceFile) as source_file, open(tokenizedSourceFile) as tokenized_source_file, open(lemmatizedSourceFile) as lemmatized_source_file, open(targetFile) as target_file, open(alignmentsFile) as alignments_file, open(sourceIdsFile) as source_ids_file:
addedCount = 0
for lineNumber in range(sourceFileLength):
if lineNumber % LEAVE_OUT == 0:
sourceSentence = source_file.readline().strip()
tokenizedSourceSentence = tokenized_source_file.readline().strip()
lemmatizedSourceSentence = lemmatized_source_file.readline().strip()
targetSentence = target_file.readline().strip()
alignment = json.loads(alignments_file.readline().strip())
sourceId = int(source_ids_file.readline().strip())
examples.append([sourceSentence, lemmatizedSourceSentence, targetSentence, alignment, sourceId])
examples.append([sourceSentence, tokenizedSourceSentence, targetSentence, alignment, sourceId])
examples_lemmatized.append([sourceSentence, lemmatizedSourceSentence, targetSentence, alignment, sourceId])
addedCount += 1
if len(examples) >= BUFFER_SIZE:
data['examples'] = examples
add_examples(data)
add_examples(examples, nonLemmatizedTmId)
add_examples(examples_lemmatized, lemmatizedTmId)
mark = time.time()
print "Added %d of %d lemmatized examples. Time elapsed: %.4f s, current speed: %.4f examples/second" % (addedCount, totalExamples, mark-start, addedCount/(mark-start))
print "Added %d of %d examples. Time elapsed: %.4f s, current speed: %.4f examples/second" % (addedCount, totalExamples, mark-start, addedCount/(mark-start))
examples = []
examples_lemmatized = []
if len(examples) > 0:
data['examples'] = examples
add_examples(data)
add_examples(examples, nonLemmatizedTmId)
add_examples(examples_lemmatized, lemmatizedTmId)
end = time.time()
print "Added all %d lemmatized sentences. Time elapsed: %.4f s, overall speed: %.4f sentences/second" % (addedCount, end-start, addedCount/(end-start))
print "Added all %d sentences. Time elapsed: %.4f s, overall speed: %.4f sentences/second" % (addedCount, end-start, addedCount/(end-start))
print "Generating index..."
print "Generating indexes..."
start = time.time()
data = {
'operation': 'refreshIndex',
'tmId' : tmId
'tmId' : nonLemmatizedTmId
}
req = urllib2.Request(address)
req.add_header('Content-Type', 'application/json')
urllib2.urlopen(req, json.dumps(data), timeout = 3600).read()
data = {
'operation': 'refreshIndex',
'tmId' : lemmatizedTmId
}
req = urllib2.Request(address)
req.add_header('Content-Type', 'application/json')

View File

@ -11,7 +11,7 @@ import host
data = {
'operation': 'simpleSearch',
'pattern':sys.argv[1],
'tmId':int(sys.argv[2])
'tmId':int(sys.argv[2]),
}
address = 'http://'+host.concordia_host