better solution

run.py

@@ -1,30 +1,50 @@
-import pandas as pd
-from sklearn.feature_extraction.text import TfidfVectorizer
+import lzma
+
 from sklearn.naive_bayes import MultinomialNB
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics import accuracy_score
 
-with open('train/in.tsv', 'r', encoding='utf-8') as f:
-    x_train = pd.DataFrame([line.strip().split('\t') for line in f.readlines()], columns=['text', 'text_id'])
-with open('dev-0/in.tsv', 'r', encoding='utf-8') as f:
-    x_dev = pd.DataFrame([line.strip().split('\t') for line in f.readlines()], columns=['text', 'text_id'])
-with open('train/in.tsv', 'r', encoding='utf-8') as f:
-    x_test = pd.DataFrame([line.strip().split('\t') for line in f.readlines()], columns=['text', 'text_id'])
 
-y_train = pd.read_csv('train/expected.tsv', sep='\t', names=['paranormal'], encoding='utf-8')
-tfidf_vectorizer = TfidfVectorizer(max_df=0.95, max_features=500)
-x_train_vectorized = tfidf_vectorizer.fit_transform(x_train['text'].values)
+def get_data(file_name, data_type):
+    lines = []
+    if data_type == "tsv":
+        with open(file_name, encoding="utf-8") as file:
+            for line in file.readlines():
+                lines.append(int(line.replace("\n", "")))
+    else:
+        with lzma.open(f"{file_name}.{data_type}") as file:
+            for line in file.readlines():
+                lines.append(line.rstrip().decode("utf-8"))
+    return lines
 
-mnb_model = MultinomialNB().fit(x_train_vectorized, y_train.values.ravel())
 
-# Dev data
-x_dev_prepared = tfidf_vectorizer.transform(x_dev['text'].values)
-predictions = mnb_model.predict(x_dev_prepared)
-with open('dev-0/out.tsv', 'w') as f:
-    for pred in predictions:
-        f.write(f'{pred}\n')
+def bayes(train):
+    x_data = get_data(f"{train}/in.tsv", "xz")
+    Y_data = get_data(f"{train}/expected.tsv", "tsv")
 
-# Test data
-x_test_vectorized = tfidf_vectorizer.transform(x_test['text'].values)
-predictions = mnb_model.predict(x_test_vectorized)
-with open('test-A/out.tsv', 'w') as f:
-    for pred in predictions:
-        f.write(f'{pred}\n')
+    vectorizer = TfidfVectorizer(stop_words="english")
+    X_data = vectorizer.fit_transform(x_data)
+
+    clf = MultinomialNB()
+    y_pred = clf.fit(X_data, Y_data)
+
+    for predct in ["test-A", "dev-0"]:
+        Y_test = get_data(f"{predct}/in.tsv", "xz")
+        y_prediction = y_pred.predict(vectorizer.transform(Y_test))
+
+        with open(f"{predct}\out.tsv", "w", encoding="UTF-8") as file_out:
+            for single_pred in y_prediction:
+                file_out.writelines(f"{str(single_pred)}\n")
+
+
+bayes("train")
+
+# y_true = []
+# with open("dev-0/expected.tsv", encoding='utf-8') as file:
+#             for line in file.readlines():
+#                 y_true.append(line)
+# y_pred = []
+# with open("dev-0/out.tsv", encoding='utf-8') as file:
+#             for line in file.readlines():
+#                 y_pred.append(line)
+# print(accuracy_score(y_true, y_pred))