117 lines
3.6 KiB
Python
117 lines
3.6 KiB
Python
import lzma
|
|
# from sklearn.metrics import accuracy_score
|
|
import numpy as np
|
|
import gensim
|
|
import gensim.downloader
|
|
import torch
|
|
import pandas as pd
|
|
import re
|
|
|
|
BATCH_SIZE = 5
|
|
|
|
|
|
class LogisticRegressionModel(torch.nn.Module):
|
|
|
|
def __init__(self):
|
|
super(LogisticRegressionModel, self).__init__()
|
|
self.layer_one = torch.nn.Linear(100, 500)
|
|
self.layer_two = torch.nn.Linear(500, 1)
|
|
|
|
def forward(self, x):
|
|
x = self.layer_one(x)
|
|
x = torch.relu(x)
|
|
x = self.layer_two(x)
|
|
x = torch.sigmoid(x)
|
|
return x
|
|
|
|
|
|
def get_data(file_name, data_type):
|
|
lines = []
|
|
if data_type == "tsv":
|
|
with open(file_name, encoding="utf-8") as file:
|
|
for line in file.readlines():
|
|
lines.append(int(line.replace("\n", "")))
|
|
else:
|
|
with lzma.open(f"{file_name}.{data_type}") as file:
|
|
for line in file.readlines():
|
|
lines.append(line.rstrip().decode("utf-8"))
|
|
return lines
|
|
|
|
|
|
def clean_data(x_data):
|
|
for sentance in range(len(x_data)):
|
|
x_data[sentance] = re.sub(r"\\n"," ", x_data[sentance])
|
|
x_data[sentance] = re.sub(r"\W"," ", x_data[sentance])
|
|
return x_data
|
|
|
|
|
|
def train_model(word2vector):
|
|
x_data = clean_data(get_data(f"train/in.tsv", "xz"))
|
|
Y_train = pd.Series(get_data(f"train/expected.tsv", "tsv"))
|
|
X_train = [np.mean([word2vector[word] for word in sentance.split() if word in word2vector] or [np.zeros(100)], axis=0) for sentance in x_data]
|
|
lr_model = LogisticRegressionModel()
|
|
|
|
acc_score = 0
|
|
items_total = 0
|
|
lr_model.train()
|
|
for i in range(0, len(Y_train), BATCH_SIZE):
|
|
#print(i, end=", ")
|
|
X = X_train[i:i+BATCH_SIZE]
|
|
X = torch.tensor(np.array(X))
|
|
|
|
Y = Y_train[i:i+BATCH_SIZE]
|
|
Y = torch.tensor(Y.astype(np.float32).to_numpy()).reshape(-1,1)
|
|
Y_predictions = lr_model(X.float())
|
|
|
|
acc_score += torch.sum((Y_predictions > 0.5) == Y).item()
|
|
items_total += len(Y_train)
|
|
criterion = torch.nn.BCELoss()
|
|
optimizer = torch.optim.SGD(lr_model.parameters(), lr = 0.1)
|
|
|
|
optimizer.zero_grad()
|
|
loss = criterion(Y_predictions, Y)
|
|
loss.backward()
|
|
optimizer.step()
|
|
#print(f"acc score: {acc_score}")
|
|
return lr_model
|
|
|
|
|
|
def prediction(lr_model, word2vector, name_of_file):
|
|
x_data = clean_data(get_data(f"{name_of_file}/in.tsv", "xz"))
|
|
x_data = [np.mean([word2vector[word] for word in sentance.split() if word in word2vector] or [np.zeros(100)], axis=0) for sentance in x_data]
|
|
|
|
y_predictions = []
|
|
with torch.no_grad():
|
|
for i in range(0, len(x_data), BATCH_SIZE):
|
|
x = x_data[i:i + BATCH_SIZE]
|
|
prediction_x = lr_model(torch.tensor(x).float())
|
|
y_predictions.extend((prediction_x > 0.5))
|
|
|
|
results = []
|
|
for result in y_predictions:
|
|
results.append(result.int()[0].item())
|
|
|
|
with open(f"{name_of_file}\out.tsv", "w", encoding="UTF-8") as file_out:
|
|
for single_pred in results:
|
|
file_out.writelines(f"{str(single_pred)}\n")
|
|
|
|
|
|
word2vector = gensim.downloader.load("glove-wiki-gigaword-100")
|
|
|
|
lr_model = train_model(word2vector)
|
|
prediction(lr_model, word2vector, "dev-0")
|
|
prediction(lr_model, word2vector, "test-A")
|
|
|
|
|
|
'''y_true = []
|
|
with open("dev-0/expected.tsv", encoding='utf-8') as file:
|
|
for line in file.readlines():
|
|
y_true.append(line)
|
|
y_pred = []
|
|
with open("dev-0/out.tsv", encoding='utf-8') as file:
|
|
for line in file.readlines():
|
|
y_pred.append(line)
|
|
print(accuracy_score(y_true, y_pred))'''
|
|
|
|
|