run.py

@@ -1,51 +0,0 @@
-import lzma
-
-from sklearn.naive_bayes import MultinomialNB
-from sklearn.feature_extraction.text import TfidfVectorizer
-# from sklearn.metrics import accuracy_score
-
-
-def get_data(file_name, data_type):
-    lines = []
-    if data_type == "tsv":
-        with open(file_name, encoding="utf-8") as file:
-            for line in file.readlines():
-                lines.append(int(line.replace("\n", "")))
-    else:
-        with lzma.open(f"{file_name}.{data_type}") as file:
-            for line in file.readlines():
-                lines.append(line.rstrip().decode("utf-8"))
-    return lines
-
-
-def bayes(train):
-    x_data = get_data(f"{train}/in.tsv", "xz")
-    Y_data = get_data(f"{train}/expected.tsv", "tsv")
-
-    vectorizer = TfidfVectorizer(stop_words="english")
-    X_data = vectorizer.fit_transform(x_data)
-
-    clf = MultinomialNB()
-    y_pred = clf.fit(X_data, Y_data)
-
-    for predct in ["test-A", "dev-0"]:
-        Y_test = get_data(f"{predct}/in.tsv", "xz")
-        y_prediction = y_pred.predict(vectorizer.transform(Y_test))
-
-        with open(f"{predct}\out.tsv", "w", encoding="UTF-8") as file_out:
-            for single_pred in y_prediction:
-                file_out.writelines(f"{str(single_pred)}\n")
-
-
-bayes("train")
-
-'''y_true = []
-with open("dev-0/expected.tsv", encoding='utf-8') as file:
-            for line in file.readlines():
-                y_true.append(line)
-y_pred = []
-with open("dev-0/out.tsv", encoding='utf-8') as file:
-            for line in file.readlines():
-                y_pred.append(line)
-print(accuracy_score(y_true, y_pred))'''
-

run_nn.py

@@ -1,116 +0,0 @@
-import lzma
-# from sklearn.metrics import accuracy_score
-import numpy as np
-import gensim
-import gensim.downloader
-import torch
-import pandas as pd
-import re
-
-BATCH_SIZE = 5
-
-
-class LogisticRegressionModel(torch.nn.Module):
-
-    def __init__(self):
-        super(LogisticRegressionModel, self).__init__()
-        self.layer_one = torch.nn.Linear(100, 500)
-        self.layer_two = torch.nn.Linear(500, 1)
-
-    def forward(self, x):
-        x = self.layer_one(x)
-        x = torch.relu(x)
-        x = self.layer_two(x)
-        x = torch.sigmoid(x)
-        return x
-
-
-def get_data(file_name, data_type):
-    lines = []
-    if data_type == "tsv":
-        with open(file_name, encoding="utf-8") as file:
-            for line in file.readlines():
-                lines.append(int(line.replace("\n", "")))
-    else:
-        with lzma.open(f"{file_name}.{data_type}") as file:
-            for line in file.readlines():
-                lines.append(line.rstrip().decode("utf-8"))
-    return lines
-
-
-def clean_data(x_data):
-    for sentance in range(len(x_data)):
-        x_data[sentance] = re.sub(r"\\n"," ", x_data[sentance])
-        x_data[sentance] = re.sub(r"\W"," ", x_data[sentance])
-    return x_data
-
-
-def train_model(word2vector):
-    x_data = clean_data(get_data(f"train/in.tsv", "xz"))
-    Y_train = pd.Series(get_data(f"train/expected.tsv", "tsv"))
-    X_train = [np.mean([word2vector[word] for word in sentance.split() if word in word2vector] or [np.zeros(100)], axis=0) for sentance in x_data]
-    lr_model = LogisticRegressionModel()
-
-    acc_score = 0
-    items_total = 0
-    lr_model.train()
-    for i in range(0, len(Y_train), BATCH_SIZE):
-        #print(i, end=", ")
-        X = X_train[i:i+BATCH_SIZE]
-        X = torch.tensor(np.array(X))
-
-        Y = Y_train[i:i+BATCH_SIZE]
-        Y = torch.tensor(Y.astype(np.float32).to_numpy()).reshape(-1,1)
-        Y_predictions = lr_model(X.float())
-
-        acc_score += torch.sum((Y_predictions > 0.5) == Y).item()
-        items_total += len(Y_train) 
-        criterion = torch.nn.BCELoss()
-        optimizer = torch.optim.SGD(lr_model.parameters(), lr = 0.1)
-
-        optimizer.zero_grad()
-        loss = criterion(Y_predictions, Y)
-        loss.backward()
-        optimizer.step()
-    #print(f"acc score: {acc_score}")
-    return lr_model
-
-
-def prediction(lr_model, word2vector, name_of_file):
-    x_data = clean_data(get_data(f"{name_of_file}/in.tsv", "xz"))
-    x_data = [np.mean([word2vector[word] for word in sentance.split() if word in word2vector] or [np.zeros(100)], axis=0) for sentance in x_data]
-             
-    y_predictions = []
-    with torch.no_grad():
-        for i in range(0, len(x_data), BATCH_SIZE):
-            x = x_data[i:i + BATCH_SIZE]
-            prediction_x = lr_model(torch.tensor(x).float())
-            y_predictions.extend((prediction_x > 0.5))
-    
-    results = []
-    for result in y_predictions:
-        results.append(result.int()[0].item())
-    
-    with open(f"{name_of_file}\out.tsv", "w", encoding="UTF-8") as file_out:
-        for single_pred in results:
-            file_out.writelines(f"{str(single_pred)}\n")
-
-
-word2vector = gensim.downloader.load("glove-wiki-gigaword-100")
-
-lr_model = train_model(word2vector)
-prediction(lr_model, word2vector, "dev-0")
-prediction(lr_model, word2vector, "test-A")
-
-
-'''y_true = []
-with open("dev-0/expected.tsv", encoding='utf-8') as file:
-            for line in file.readlines():
-                y_true.append(line)
-y_pred = []
-with open("dev-0/out.tsv", encoding='utf-8') as file:
-            for line in file.readlines():
-                y_pred.append(line)
-print(accuracy_score(y_true, y_pred))'''
-
-