From 892f21fc34eac0d2e6178c7bf823a91a6ab443f7 Mon Sep 17 00:00:00 2001 From: Maciej Sobkowiak Date: Wed, 26 May 2021 00:33:17 +0200 Subject: [PATCH] Fix dimensions to fit w2v --- main.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/main.py b/main.py index ae10119..15eef8d 100644 --- a/main.py +++ b/main.py @@ -4,12 +4,13 @@ import numpy as np import torch from gensim import downloader from nltk.tokenize import word_tokenize +import csv class NeuralNetworkModel(torch.nn.Module): def __init__(self): - dim = 100 + dim = 200 super(NeuralNetworkModel, self).__init__() self.fc1 = torch.nn.Linear(dim, 500) self.fc2 = torch.nn.Linear(500, 1) @@ -27,24 +28,28 @@ def read_data(): y_labels = (pd.read_csv('out-header.tsv', sep='\t')).columns x_train = pd.read_table('train/in.tsv', error_bad_lines=False, - header=None, quoting=3, names=x_labels) + header=None, quoting=csv.QUOTE_NONE, names=x_labels) y_train = pd.read_table('train/expected.tsv', error_bad_lines=False, - header=None, quoting=3, names=y_labels) + header=None, quoting=csv.QUOTE_NONE, names=y_labels) x_dev = pd.read_table('dev-0/in.tsv', error_bad_lines=False, - header=None, quoting=3, names=x_labels) + header=None, quoting=csv.QUOTE_NONE, names=x_labels) x_test = pd.read_table('test-A/in.tsv', error_bad_lines=False, - header=None, quoting=3, names=x_labels) + header=None, quoting=csv.QUOTE_NONE, names=x_labels) # remove some rows for faster development remove_n = 200000 drop_indices = np.random.choice(x_train.index, remove_n, replace=False) x_train = x_train.drop(drop_indices) + y_train = y_train.drop(drop_indices) return x_labels, y_labels, x_train, y_train, x_dev, x_test x_labels, y_labels, x_train, y_train, x_dev, x_test = read_data() +print(len(y_train)) +print(len(x_train)) + x_train = x_train[x_labels[0]].str.lower() x_dev = x_dev[x_labels[0]].str.lower() x_test = x_test[x_labels[0]].str.lower() @@ -57,11 +62,11 @@ x_test = [word_tokenize(x) for x in x_test] w2v = downloader.load('glove-wiki-gigaword-200') x_train = [np.mean([w2v[word] for word in doc if word in w2v] or [ - np.zeros(50)], axis=0) for doc in x_train] + np.zeros(200)], axis=0) for doc in x_train] x_dev = [np.mean([w2v[word] for word in doc if word in w2v] - or [np.zeros(50)], axis=0) for doc in x_dev] + or [np.zeros(200)], axis=0) for doc in x_dev] x_test = [np.mean([w2v[word] for word in doc if word in w2v] - or [np.zeros(50)], axis=0) for doc in x_test] + or [np.zeros(200)], axis=0) for doc in x_test] nn_model = NeuralNetworkModel() BATCH_SIZE = 5 @@ -72,15 +77,16 @@ for epoch in range(5): nn_model.train() for i in range(0, y_train.shape[0], BATCH_SIZE): X = x_train[i:i+BATCH_SIZE] - X = torch.tensor(X.astype(np.float32).todense()) + X = torch.tensor(X) Y = y_train[i:i+BATCH_SIZE] - Y = torch.tensor(Y.astype(np.float32)).reshape(-1, 1) + Y = torch.tensor(Y.astype(np.float32).to_numpy()).reshape(-1, 1) - Y_predictions = nn_model(X) + Y_predictions = nn_model(X.float()) - optimizer.zero_grad() loss = criterion(Y_predictions, Y) + optimizer.zero_grad() loss.backward() optimizer.step() + print(Y_predictions)