Logistic regression with bigrams

classifier.py

@@ -18,14 +18,14 @@ with open('train/expected.tsv', 'rt') as train_expected:
         Y_train.append(int(text))
 
 print("Training TFIDF...")
-vectorizer = TfidfVectorizer(decode_error="replace", stop_words="english", max_df=0.8, sublinear_tf=True)
+vectorizer = TfidfVectorizer(ngram_range=(1, 2), decode_error="replace", stop_words="english", max_df=0.3, max_features=500000)
 X_train = vectorizer.fit_transform(X_train)
 
 print("Training...")
 model = LogisticRegression()
 model.fit(X_train, Y_train)
 
-print("Predicting dev...")
+print("Predicting dev-0...")
 X_dev = []
 with open('dev-0/in.tsv', 'r', encoding="utf-8") as dev_in:
     for line in dev_in:
@@ -39,6 +39,20 @@ with open("dev-0/out.tsv", "w") as out_file:
     for pred in predictions:
         out_file.write(str(pred) + "\n")
 
+print("Predicting dev-1...")
+X_dev = []
+with open('dev-1/in.tsv', 'r', encoding="utf-8") as dev_in:
+    for line in dev_in:
+        text = line.split("\t")[0].strip()
+        X_dev.append(text)
+
+X_dev = vectorizer.transform(X_dev)
+predictions = model.predict(X_dev)
+
+with open("dev-1/out.tsv", "w") as out_file:
+    for pred in predictions:
+        out_file.write(str(pred) + "\n")
+
 print("Predicting test...")
 X_test = []
 with open('test-A/in.tsv', 'r', encoding="utf-8") as test_in: