working simple search

concordia-aligner/add_corpus.sh

@@ -2,4 +2,4 @@
 
 source corpus.cfg
 
-./add_fast_aligned_TM.py $CORPUS_NAME $CORPUS_PATH/src_clean.txt $CORPUS_PATH/src_clean.lem $SRC_LANG_ID  $CORPUS_PATH/trg_clean.txt $TRG_LANG_ID  $CORPUS_PATH/alignments.txt  $CORPUS_PATH/ids_clean.txt
+./add_fast_aligned_TM.py $CORPUS_NAME $CORPUS_PATH/src_clean.txt $CORPUS_PATH/src_clean.tok $CORPUS_PATH/src_clean.lem $SRC_LANG_ID  $CORPUS_PATH/trg_clean.txt $TRG_LANG_ID  $CORPUS_PATH/alignments.txt  $CORPUS_PATH/ids_clean.txt

concordia-aligner/add_fast_aligned_TM.py

@@ -18,33 +18,40 @@ def file_len(fname):
             pass
     return i + 1
 
-def add_examples(examplesData):
+def add_examples(examples, tmId):
+    examplesData = {
+    'operation': 'addSentences',
+    'tmId':tmId,
+    'examples':examples
+    }
     req = urllib2.Request(address)
     req.add_header('Content-Type', 'application/json')
     response = json.loads(urllib2.urlopen(req, json.dumps(examplesData), timeout = 3600).read())
-    print(response)
+    #print(response)
     if response['status'] == 'error':
         raise Exception(response['message'])
 
-if len(sys.argv) != 9:
+if len(sys.argv) != 10:
     raise Exception("wrong number of arguments")
 
 name = sys.argv[1]
 sourceFile = sys.argv[2]
-lemmatizedSourceFile = sys.argv[3]
+tokenizedSourceFile = sys.argv[3]
-sourceLangId = int(sys.argv[4])
+lemmatizedSourceFile = sys.argv[4]
-targetFile = sys.argv[5]
+sourceLangId = int(sys.argv[5])
-targetLangId = int(sys.argv[6])
+targetFile = sys.argv[6]
-alignmentsFile = sys.argv[7]
+targetLangId = int(sys.argv[7])
-sourceIdsFile = sys.argv[8]
+alignmentsFile = sys.argv[8]
+sourceIdsFile = sys.argv[9]
 
 sourceFileLength = file_len(sourceFile)
+tokenizedSourceFileLength = file_len(tokenizedSourceFile)
 lemmatizedSourceFileLength = file_len(lemmatizedSourceFile)
 targetFileLength = file_len(targetFile)
 alignmentsFileLength = file_len(alignmentsFile)
 sourceIdsFileLength = file_len(sourceIdsFile)
 
-if not (sourceFileLength == lemmatizedSourceFileLength and lemmatizedSourceFileLength == targetFileLength and targetFileLength == alignmentsFileLength and alignmentsFileLength == sourceIdsFileLength):
+if not (sourceFileLength == tokenizedSourceFileLength and tokenizedSourceFileLength == lemmatizedSourceFileLength and lemmatizedSourceFileLength == targetFileLength and targetFileLength == alignmentsFileLength and alignmentsFileLength == sourceIdsFileLength):
     print("File lengths:")
     print("source file: %d\nlemmatized source file: %d\ntarget file: %d\nalignments file: %d\nsource ids file: %d" % (sourceFileLength, lemmatizedSourceFileLength, targetFileLength, alignmentsFileLength, sourceIdsFileLength))
     raise Exception("files are not of the same length!")
@@ -52,63 +59,71 @@ if not (sourceFileLength == lemmatizedSourceFileLength and lemmatizedSourceFileL
 totalExamples = sourceFileLength / LEAVE_OUT
 
 data = {
-    'operation': 'addTm',
+    'operation': 'addPairedTms',
     'sourceLangId':sourceLangId,
     'targetLangId':targetLangId,
-    'name':name,
+    'name':name
-    'tmLemmatized':True
 }
 
 req = urllib2.Request(address)
 req.add_header('Content-Type', 'application/json')
 response = json.loads(urllib2.urlopen(req, json.dumps(data), timeout = 3600).read())
-print(response)
+#print(response)
-tmId = int(response['newTmId'])
-print "Added new tm: %d" % tmId
 
-data = {
+lemmatizedTmId = int(response['lemmatizedTmId'])
-    'operation': 'addSentences',
+nonLemmatizedTmId = int(response['nonLemmatizedTmId'])
-    'tmId':tmId
+print "Added new paired tm: lemmatized id: %d, non lemmatized id: %d" % (lemmatizedTmId, nonLemmatizedTmId)
-}
 
 examples = []
+examples_lemmatized = []
 start = time.time()
-with open(sourceFile) as source_file, open(lemmatizedSourceFile) as lemmatized_source_file, open(targetFile) as target_file, open(alignmentsFile) as alignments_file, open(sourceIdsFile) as source_ids_file:
+with open(sourceFile) as source_file, open(tokenizedSourceFile) as tokenized_source_file, open(lemmatizedSourceFile) as lemmatized_source_file, open(targetFile) as target_file, open(alignmentsFile) as alignments_file, open(sourceIdsFile) as source_ids_file:
     addedCount = 0
     for lineNumber in range(sourceFileLength):
         if lineNumber % LEAVE_OUT == 0:
             sourceSentence = source_file.readline().strip()
+            tokenizedSourceSentence = tokenized_source_file.readline().strip()
             lemmatizedSourceSentence = lemmatized_source_file.readline().strip()
             targetSentence = target_file.readline().strip()
             alignment = json.loads(alignments_file.readline().strip())
             sourceId = int(source_ids_file.readline().strip())
 
-            examples.append([sourceSentence, lemmatizedSourceSentence, targetSentence, alignment, sourceId])
+            examples.append([sourceSentence, tokenizedSourceSentence, targetSentence, alignment, sourceId])
+            examples_lemmatized.append([sourceSentence, lemmatizedSourceSentence, targetSentence, alignment, sourceId])
             addedCount += 1
             if len(examples) >= BUFFER_SIZE:
-                data['examples'] = examples
+                add_examples(examples, nonLemmatizedTmId)
-                add_examples(data)
+                add_examples(examples_lemmatized, lemmatizedTmId)
                 mark = time.time()
-                print "Added %d of %d lemmatized examples. Time elapsed: %.4f s, current speed: %.4f examples/second" % (addedCount, totalExamples, mark-start, addedCount/(mark-start))
+                print "Added %d of %d examples. Time elapsed: %.4f s, current speed: %.4f examples/second" % (addedCount, totalExamples, mark-start, addedCount/(mark-start))
                 examples = []
+                examples_lemmatized = []
 
 
 if len(examples) > 0:
-    data['examples'] = examples
+    add_examples(examples, nonLemmatizedTmId)
-    add_examples(data)
+    add_examples(examples_lemmatized, lemmatizedTmId)
 
 end = time.time()
-print "Added all %d lemmatized sentences. Time elapsed: %.4f s, overall speed: %.4f sentences/second" % (addedCount, end-start, addedCount/(end-start))
+print "Added all %d sentences. Time elapsed: %.4f s, overall speed: %.4f sentences/second" % (addedCount, end-start, addedCount/(end-start))
 
-print "Generating index..."
+print "Generating indexes..."
 start = time.time()
 data = {
     'operation': 'refreshIndex',
-    'tmId' : tmId
+    'tmId' : nonLemmatizedTmId
+}
+req = urllib2.Request(address)
+req.add_header('Content-Type', 'application/json')
+urllib2.urlopen(req, json.dumps(data), timeout = 3600).read()
+
+data = {
+    'operation': 'refreshIndex',
+    'tmId' : lemmatizedTmId
 }
 req = urllib2.Request(address)
 req.add_header('Content-Type', 'application/json')
 urllib2.urlopen(req, json.dumps(data), timeout = 3600).read()
 
 end = time.time()
-print "Index regeneration complete. The operation took %.4f s" % (end - start)
+print "Index regeneration complete. The operation took %.4f s" % (end - start)

tools/simpleSearch.py

@@ -11,7 +11,7 @@ import host
 data = {
     'operation': 'simpleSearch',
     'pattern':sys.argv[1],
-    'tmId':int(sys.argv[2])
+    'tmId':int(sys.argv[2]),
 }
 
 address = 'http://'+host.concordia_host