log reg

run.py

@@ -1,32 +1,91 @@
+import numpy as np
+import pandas as pd
+import torch
+import csv
 import lzma
-import nltk
-from sklearn.naive_bayes import MultinomialNB
-from sklearn.feature_extraction.text import TfidfVectorizer
+import gensim.downloader
+from nltk import word_tokenize
+
+#print('wczytanie danych')
+
+x_train = pd.read_table('train/in.tsv.xz', compression='xz', sep='\t', header=None, quoting=3)
+y_train = pd.read_table('train/expected.tsv', sep='\t', header=None, quoting=3)
+x_dev = pd.read_table('dev-0/in.tsv.xz', compression='xz', sep='\t', header=None, quoting=3)
+x_test = pd.read_table('test-A/in.tsv.xz', compression='xz', sep='\t', header=None, quoting=3)
+
+#print('inicjalizacja modelu')
+class NeuralNetworkModel(torch.nn.Module):
+    def __init__(self):
+        super(NeuralNetworkModel, self).__init__()
+        self.l01 = torch.nn.Linear(300, 300)
+        self.l02 = torch.nn.Linear(300, 1)
+
+    def forward(self, x):
+        x = self.l01(x)
+        x = torch.relu(x)
+        x = self.l02(x)
+        x = torch.sigmoid(x)
+        return x
 
 
-with lzma.open("train/in.tsv.xz", "rt", encoding="utf-8") as train_file:
-    in_train = [x.strip().lower() for x in train_file.readlines()]
+#print('przygotowanie danych')
 
-with open("train/expected.tsv", "r", encoding="utf-8") as train_file:
-    out_train = [int(x.strip()) for x in train_file.readlines()]
+x_train = x_train[0].str.lower()
+y_train = y_train[0]
+x_dev = x_dev[0].str.lower()
+x_test = x_test[0].str.lower()
 
-with lzma.open("dev-0/in.tsv.xz", "rt", encoding="utf-8") as dev_file:
-    in_dev = [x.strip().lower() for x in dev_file.readlines()]
+x_train = [word_tokenize(x) for x in x_train]
+x_dev = [word_tokenize(x) for x in x_dev]
+x_test = [word_tokenize(x) for x in x_test]
 
-with lzma.open("test-A/in.tsv.xz", "rt", encoding="utf-8") as test_file:
-    in_test = [x.strip().lower() for x in test_file.readlines()]
+word2vec = gensim.downloader.load('word2vec-google-news-300')
+x_train = [np.mean([word2vec[word] for word in content if word in word2vec] or [np.zeros(300)], axis=0) for content in x_train]
+x_dev = [np.mean([word2vec[word] for word in content if word in word2vec] or [np.zeros(300)], axis=0) for content in x_dev]
+x_test = [np.mean([word2vec[word] for word in content if word in word2vec] or [np.zeros(300)], axis=0) for content in x_test]
 
-tfidf_vectorizer=TfidfVectorizer()
-IN_train = tfidf_vectorizer.fit_transform(in_train)
-classifier = MultinomialNB()
-y_pred = classifier.fit(IN_train, out_train)
 
-y_prediction = y_pred.predict(tfidf_vectorizer.transform(in_test))
-with open("test-A/out.tsv", "w", encoding="utf-8") as test_out_file:
-    for single_pred in y_prediction:
-        test_out_file.writelines(f"{str(single_pred)}\n")
+#print('trenowanie modelu')
+model = NeuralNetworkModel()
+BATCH_SIZE = 5
+criterion = torch.nn.BCELoss()
+optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
 
-pred_dev = y_pred.predict(tfidf_vectorizer.transform(in_test))
-with open("dev-0/out.tsv", "w", encoding="utf-8") as dev_out_file:
-    for single_pred in pred_dev:
-        dev_out_file.writelines(f"{str(single_pred)}\n")
+for epoch in range(BATCH_SIZE):
+    model.train()
+    for i in range(0, y_train.shape[0], BATCH_SIZE):
+        X = x_train[i:i + BATCH_SIZE]
+        X = torch.tensor(X)
+        y = y_train[i:i + BATCH_SIZE]
+        y = torch.tensor(y.astype(np.float32).to_numpy()).reshape(-1, 1)
+        optimizer.zero_grad()
+        outputs = model(X.float())
+        loss = criterion(outputs, y)
+        loss.backward()
+        optimizer.step()
+
+#print('predykcja wynikow')
+y_dev = []
+y_test = []
+model.eval()
+
+with torch.no_grad():
+    for i in range(0, len(x_dev), BATCH_SIZE):
+        X = x_dev[i:i + BATCH_SIZE]
+        X = torch.tensor(X)
+        outputs = model(X.float())
+        prediction = (outputs > 0.5)
+        y_dev += prediction.tolist()
+
+    for i in range(0, len(x_test), BATCH_SIZE):
+        X = x_test[i:i + BATCH_SIZE]
+        X = torch.tensor(X)
+        outputs = model(X.float())
+        y = (outputs >= 0.5)
+        y_test += prediction.tolist()
+
+# print('eksportowanie do plików')
+y_dev = np.asarray(y_dev, dtype=np.int32)
+y_test = np.asarray(y_test, dtype=np.int32)
+y_dev.tofile('./dev-0/out.tsv', sep='\n')
+y_test.tofile('./test-A/out.tsv', sep='\n')

run_bayes.py

@@ -0,0 +1,31 @@
+import lzma
+import nltk
+from sklearn.naive_bayes import MultinomialNB
+from sklearn.feature_extraction.text import TfidfVectorizer
+
+with lzma.open("train/in.tsv.xz", "rt", encoding="utf-8") as train_file:
+    in_train = [x.strip().lower() for x in train_file.readlines()]
+
+with open("train/expected.tsv", "r", encoding="utf-8") as train_file:
+    out_train = [int(x.strip()) for x in train_file.readlines()]
+
+with lzma.open("dev-0/in.tsv.xz", "rt", encoding="utf-8") as dev_file:
+    in_dev = [x.strip().lower() for x in dev_file.readlines()]
+
+with lzma.open("test-A/in.tsv.xz", "rt", encoding="utf-8") as test_file:
+    in_test = [x.strip().lower() for x in test_file.readlines()]
+
+tfidf_vectorizer=TfidfVectorizer()
+IN_train = tfidf_vectorizer.fit_transform(in_train)
+classifier = MultinomialNB()
+y_pred = classifier.fit(IN_train, out_train)
+
+y_prediction = y_pred.predict(tfidf_vectorizer.transform(in_test))
+with open("test-A/out.tsv", "w", encoding="utf-8") as test_out_file:
+    for single_pred in y_prediction:
+        test_out_file.writelines(f"{str(single_pred)}\n")
+
+pred_dev = y_pred.predict(tfidf_vectorizer.transform(in_test))
+with open("dev-0/out.tsv", "w", encoding="utf-8") as dev_out_file:
+    for single_pred in pred_dev:
+        dev_out_file.writelines(f"{str(single_pred)}\n")