From c20a8ae58a544d775c3849d48c589d14502baf70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rafa=C5=82=20Jaworski?= <rjawor@amu.edu.pl>
Date: Tue, 26 Feb 2019 14:00:10 +0100
Subject: [PATCH] leave out parameter

---
 tests/addFastAlignedTM.py | 39 +++++++++++++++++++++------------------
 tests/addFastAlignedTM.sh |  2 +-
 2 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/tests/addFastAlignedTM.py b/tests/addFastAlignedTM.py
index ca7e626..4ec1155 100755
--- a/tests/addFastAlignedTM.py
+++ b/tests/addFastAlignedTM.py
@@ -8,6 +8,7 @@ import host
 import time
 
 BUFFER_SIZE = 500
+LEAVE_OUT = 2 # this leaves out every second sentence
 
 address = 'http://'+host.concordia_host
 if len(host.concordia_port) > 0:
@@ -23,7 +24,7 @@ def file_len(fname):
 def add_examples(examplesData):
     req = urllib2.Request(address)
     req.add_header('Content-Type', 'application/json')
-    response = json.loads(urllib2.urlopen(req, json.dumps(examplesData)).read())
+    response = json.loads(urllib2.urlopen(req, json.dumps(examplesData), 3600).read())
     print(response)
     if response['status'] == 'error':
         raise Exception(response['message'])
@@ -51,7 +52,7 @@ if not (sourceFileLength == lemmatizedSourceFileLength and lemmatizedSourceFileL
     print("source file: %d\nlemmatized source file: %d\ntarget file: %d\nalignments file: %d\nsource ids file: %d" % (sourceFileLength, lemmatizedSourceFileLength, targetFileLength, alignmentsFileLength, sourceIdsFileLength))
     raise Exception("files are not of the same length!")
 
-totalExamples = file_len(sourceFile)
+totalExamples = sourceFileLength / LEAVE_OUT
 
 data = {
     'operation': 'addTm',
@@ -63,7 +64,7 @@ data = {
 
 req = urllib2.Request(address)
 req.add_header('Content-Type', 'application/json')
-response = json.loads(urllib2.urlopen(req, json.dumps(data)).read())
+response = json.loads(urllib2.urlopen(req, json.dumps(data), 3600).read())
 print(response)
 tmId = int(response['newTmId'])
 print "Added new tm: %d" % tmId
@@ -76,21 +77,23 @@ data = {
 examples = []
 start = time.time()
 with open(sourceFile) as source_file, open(lemmatizedSourceFile) as lemmatized_source_file, open(targetFile) as target_file, open(alignmentsFile) as alignments_file, open(sourceIdsFile) as source_ids_file:
+    addedCount = 0
     for lineNumber in range(totalExamples):
-        sourceSentence = source_file.readline().strip()
-        lemmatizedSourceSentence = lemmatized_source_file.readline().strip()
-        targetSentence = target_file.readline().strip()
-        alignment = json.loads(alignments_file.readline().strip())
-        sourceId = int(source_ids_file.readline().strip())
+        if lineNumber % LEAVE_OUT == 0:
+            sourceSentence = source_file.readline().strip()
+            lemmatizedSourceSentence = lemmatized_source_file.readline().strip()
+            targetSentence = target_file.readline().strip()
+            alignment = json.loads(alignments_file.readline().strip())
+            sourceId = int(source_ids_file.readline().strip())
 
-        examples.append([sourceSentence, lemmatizedSourceSentence, targetSentence, alignment, sourceId])
-
-        if len(examples) >= BUFFER_SIZE:
-            data['examples'] = examples
-            add_examples(data)
-            mark = time.time()
-            print "Added %d of %d lemmatized examples. Time elapsed: %.4f s, current speed: %.4f examples/second" % ( (lineNumber+1), totalExamples, mark-start, (lineNumber+1)/(mark-start))
-            examples = []
+            examples.append([sourceSentence, lemmatizedSourceSentence, targetSentence, alignment, sourceId])
+            addedCount += 1
+            if len(examples) >= BUFFER_SIZE:
+                data['examples'] = examples
+                add_examples(data)
+                mark = time.time()
+                print "Added %d of %d lemmatized examples. Time elapsed: %.4f s, current speed: %.4f examples/second" % (addedCount, totalExamples, mark-start, addedCount/(mark-start))
+                examples = []
 
 
 if len(examples) > 0:
@@ -98,7 +101,7 @@ if len(examples) > 0:
     add_examples(data)
 
 end = time.time()
-print "Added all %d lemmatized sentences. Time elapsed: %.4f s, overall speed: %.4f sentences/second" % ((lineNumber+1), end-start, (lineNumber+1)/(end-start))
+print "Added all %d lemmatized sentences. Time elapsed: %.4f s, overall speed: %.4f sentences/second" % (addedCount, end-start, addedCount/(end-start))
 
 print "Generating index..."
 start = time.time()
@@ -108,7 +111,7 @@ data = {
 }
 req = urllib2.Request(address)
 req.add_header('Content-Type', 'application/json')
-urllib2.urlopen(req, json.dumps(data)).read()
+urllib2.urlopen(req, json.dumps(data), 3600).read()
 
 end = time.time()
 print "Index regeneration complete. The operation took %.4f s" % (end - start)
diff --git a/tests/addFastAlignedTM.sh b/tests/addFastAlignedTM.sh
index 8b73620..5a079a6 100755
--- a/tests/addFastAlignedTM.sh
+++ b/tests/addFastAlignedTM.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-CORPUS_NAME=opensubtitles_sample
+CORPUS_NAME=opensubtitles
 CORPUS_PATH=../fast-aligner/corpora/$CORPUS_NAME
 SRC_LANG_ID=1
 TRG_LANG_ID=2