Fix preprocessing

classifier.py

@@ -1,22 +1,10 @@
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.naive_bayes import MultinomialNB
 
-stopwords = []
-# stopwords source - https://github.com/bieli/stopwords/blob/master/polish.stopwords.txt
-with open('stopwords') as f:
-    stopwords = [line.rstrip() for line in f]
-
 classifier = MultinomialNB()
 vectorizer = TfidfVectorizer()
 
 
-def preprocess(doc):
-    doc = doc.lower().split(' ')
-    doc = list(filter(lambda word: (word not in stopwords) and (word != ''), doc))
-    doc = ' '.join(doc)
-    return doc
-
-
 def train():
     with open('train/train.tsv') as f:
         docs = [line.rstrip() for line in f]
@@ -26,7 +14,7 @@ def train():
         y_with_doc = doc.split('\t')
         y.append(y_with_doc[0])
         doc = y_with_doc[1]
-        docs_preprocessed.append(preprocess(doc))
+        docs_preprocessed.append(doc)
     y = [int(value) for value in y]
     x = vectorizer.fit_transform(docs_preprocessed)
     classifier.fit(x, y)
@@ -35,9 +23,6 @@ def train():
 def classify(path):
     with open(path + 'in.tsv') as f:
         docs = [line.rstrip() for line in f]
-    docs_preprocessed = []
-    for doc in docs:
-        docs_preprocessed.append(preprocess(doc))
     test_x = vectorizer.transform(docs)
     predictions = classifier.predict(test_x)
     with open(path + 'out.tsv', 'w') as file:
@@ -47,4 +32,4 @@ def classify(path):
 
 train()
 classify('dev-0/')
-classify('test-A/')
+classify('test-A/')

dev-0/out.tsv

@@ -418,7 +418,7 @@
 0
 1
 0
-0
+1
 0
 1
 1
@@ -1318,7 +1318,7 @@
 1
 1
 1
-0
+1
 0
 0
 0
@@ -1371,7 +1371,7 @@
 1
 1
 1
-1
+0
 1
 1
 1
@@ -3200,7 +3200,7 @@
 1
 1
 1
-0
+1
 1
 1
 0
@@ -3614,7 +3614,7 @@
 1
 1
 1
-0
+1
 0
 0
 1
@@ -4389,7 +4389,7 @@
 1
 1
 0
-0
+1
 1
 0
 1
@@ -4987,7 +4987,7 @@
 1
 1
 0
-0
+1
 1
 1
 1
@@ -5444,7 +5444,7 @@
 1
 1
 0
-0
+1
 1
 1
 0

test-A/out.tsv

@@ -204,7 +204,7 @@
 1
 0
 0
-0
+1
 0
 1
 0
@@ -1719,7 +1719,7 @@
 0
 1
 1
-0
+1
 1
 0
 1
@@ -1807,7 +1807,7 @@
 1
 0
 1
-0
+1
 1
 1
 1
@@ -1946,7 +1946,7 @@
 1
 0
 0
-0
+1
 1
 1
 0
@@ -2080,7 +2080,7 @@
 0
 1
 1
-0
+1
 0
 0
 1
@@ -2210,7 +2210,7 @@
 0
 1
 1
-1
+0
 1
 1
 1
@@ -2737,7 +2737,7 @@
 1
 0
 0
-0
+1
 0
 1
 1
@@ -3109,7 +3109,7 @@
 1
 1
 1
-0
+1
 1
 1
 0
@@ -3365,7 +3365,7 @@
 1
 0
 1
-0
+1
 1
 1
 1
@@ -4065,7 +4065,7 @@
 1
 1
 1
-1
+0
 1
 0
 1