6.2 KiB
6.2 KiB
from sklearn.naive_bayes import GaussianNB
import pandas as pd
import torch
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import TfidfVectorizer
import gensim
from gensim.models import KeyedVectors
import gensim.downloader
import nltk
import csv
import numpy as np
import numpy as np
def tokenize_data(data):
data_tokenize = [nltk.word_tokenize(x) for x in data]
for doc in data_tokenize:
i = 0
while i < len(doc):
if doc[i].isalpha():
doc[i] = doc[i].lower()
else:
del doc[i]
i += 1
return data_tokenize
class NeuralNetwork(torch.nn.Module):
def __init__(self, input_size, hidden_size, num_classes):
super(NeuralNetwork, self).__init__()
self.l1 = torch.nn.Linear(input_size, hidden_size)
self.l2 = torch.nn.Linear(hidden_size, num_classes)
def forward(self, x):
x = self.l1(x)
x = torch.relu(x)
x = self.l2(x)
x = torch.sigmoid(x)
return x
r_in = './train/train.tsv'
r_ind_ev = './dev-0/in.tsv'
tsv_read = pd.read_table(r_in, error_bad_lines=False, quoting=csv.QUOTE_NONE, sep='\t', header=None)
tsv_read_dev = pd.read_table(r_ind_ev, error_bad_lines=False, sep='\t', quoting=csv.QUOTE_NONE, header=None)
tsv_read_test_in = pd.read_table('./test-A/in.tsv', error_bad_lines=False,quoting=csv.QUOTE_NONE, header= None)
y_train = tsv_read[0].values
X_train = tsv_read[1].values
X_dev = tsv_read_dev[0].values
X_test= tsv_read_test_in[0].values
X_train = tokenize_data(X_train)
X_dev = tokenize_data(X_dev)
X_test = tokenize_data(X_test)
b'Skipping line 25706: expected 2 fields, saw 3\nSkipping line 58881: expected 2 fields, saw 3\nSkipping line 73761: expected 2 fields, saw 3\n' b'Skipping line 1983: expected 1 fields, saw 2\nSkipping line 5199: expected 1 fields, saw 2\n'
model = KeyedVectors.load("./word2vec/word2vec_100_3_polish.bin")
X_train = [np.mean([model[word] for word in content if word in model] or [np.zeros(100)], axis=0) for content in X_train]
x_train_vectors = np.array(X_train, dtype=np.float32)
x_train_tensor = torch.tensor(x_train_vectors.astype(np.float32))
X_dev = [np.mean([model[word] for word in content if word in model] or [np.zeros(100)], axis=0) for content in X_dev]
x_dev_vectors = np.array(X_dev, dtype=np.float32)
X_test = [np.mean([model[word] for word in content if word in model] or [np.zeros(100)], axis=0) for content in X_test]
x_test_vectors = np.array(X_test, dtype=np.float32)
model = NeuralNetwork(100, 200, 1)
criterion = torch.nn.BCELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
batch_size = 12
for epoch in range(6):
loss_score = 0
acc_score = 0
items_total = 0
model.train()
for i in range(0, y_train.shape[0], batch_size):
X = x_train_vectors[i:i+batch_size]
X = torch.tensor(X.astype(np.float32))
Y = y_train[i:i+batch_size]
Y = torch.tensor(Y.astype(np.float32)).reshape(-1,1)
Y_predictions = model(X)
acc_score += torch.sum((Y_predictions > 0.5) == Y).item()
items_total += Y.shape[0]
optimizer.zero_grad()
loss = criterion(Y_predictions, Y)
loss.backward()
optimizer.step()
loss_score += loss.item() * Y.shape[0]
predictions = model(torch.tensor(x_dev_vectors.astype(np.float32)))
predictions = predictions.cpu().detach().numpy()
predictions = (predictions > 0.5)
predictions = np.asarray(predictions, dtype=np.int32)
predictions.tofile('dev-0/out.tsv', sep='\n')
predictions = model(torch.tensor(x_dev_vectors.astype(np.float32)))
predictions = predictions.cpu().detach().numpy()
predictions = (predictions > 0.5)
predictions = np.asarray(predictions, dtype=np.int32)
predictions.tofile('test-A/out.tsv', sep='\n')