From d6f42e965976d551168f29cd9897fbad3ebf9318 Mon Sep 17 00:00:00 2001 From: s430705 Date: Thu, 13 May 2021 15:53:20 +0200 Subject: [PATCH] xx --- neural-network.py | 154 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 109 insertions(+), 45 deletions(-) diff --git a/neural-network.py b/neural-network.py index 7ec6e86..f9c1c25 100644 --- a/neural-network.py +++ b/neural-network.py @@ -5,8 +5,14 @@ import numpy as np import torch import torch.nn as nn from tensorflow.keras.layers.experimental.preprocessing import TextVectorization +from sklearn.feature_extraction.text import HashingVectorizer +import torch.nn.functional as F +import torch.optim as optim +import torch +from torch.optim import optimizer -os.environ['CUDA_VISIBLE_DEVICES'] = '-1' +vectorizer = HashingVectorizer(n_features=20) +# os.environ['CUDA_VISIBLE_DEVICES'] = '-1' print('debug 1') train_df = pd.read_csv('train/in.tsv', header=None, sep='\t') @@ -18,85 +24,143 @@ train_text = train_df[0].tolist() test_text = test_df[0].tolist() dev_text = test_df[0].tolist() -text_data = train_text + test_text + dev_text +text_data = train_text + +# print(train_text) vectorize_layer = TextVectorization(max_tokens=5, output_mode="int") text_data = tf.data.Dataset.from_tensor_slices(text_data) vectorize_layer.adapt(text_data.batch(64)) + + inputs = tf.keras.layers.Input(shape=(1,), dtype=tf.string, name="text") outputs = vectorize_layer(inputs) model = tf.keras.Model(inputs, outputs) -print('uwaga debug') -x_train = list(map(model.predict, train_text)) -y_train = train_expected[0] -x_test = list(map(model.predict, test_text)) +print('model loaded') +train_text = train_df[0].apply(lambda x: vectorizer.transform([x])) +test_text = test_df[0].apply(lambda x: vectorizer.transform([x])) + +x_train = train_text.tolist() +x_test = test_text.tolist() +# x_train = list(map(model.predict, train_text)) +# x_train = [model.predict([x]) for x in train_text] +y_train = train_expected[0].astype(np.float32) +# x_test = list(map(model.predict, test_text)) +# x_test = [model.predict([x]) for x in test_text] loss_function = nn.CrossEntropyLoss() x_train = pd.DataFrame(x_train) x_test = pd.DataFrame(x_test) -y_train = pd.DataFrame(y_train[0]) +y_train = pd.DataFrame(y_train) +print("End of vectorization") -# (model.predict(["Murder in the forset!"])) +# print((model.predict("Murder in the forset!"))) class FeedforwardNeuralNetModel(nn.Module): - def __init__(self, input_dim, hidden_dim, output_dim): + def __init__(self): super(FeedforwardNeuralNetModel, self).__init__() - # Linear function - self.fc1 = nn.Linear(input_dim, hidden_dim) + # Linear function 1: vocab_size --> 500 + self.fc1 = nn.Linear(FEAUTERES, 500) + # Non-linearity 1 + self.fc2 = nn.Linear(500,1) + # self.relu1 = nn.ReLU() - # Non-linearity - self.sigmoid = nn.Sigmoid() + # Linear function 2: 500 --> 500 + # self.fc2 = nn.Linear(hidden_dim, hidden_dim) + # Non-linearity 2 + # self.relu2 = nn.ReLU() - # Linear function (readout) - self.fc2 = nn.Linear(hidden_dim, output_dim) + # Linear function 3 (readout): 500 --> 3 + # self.fc3 = nn.Linear(hidden_dim, output_dim) def forward(self, x): - # Linear function # LINEAR + # Linear function 1 out = self.fc1(x) + # Non-linearity 1 + out = self.relu1(out) - # Non-linearity # NON-LINEAR - out = self.sigmoid(out) + # Non-linearity 2 + out = self.relu2(out) + + # Linear function 3 (readout) - # Linear function (readout) # LINEAR - out = self.fc2(out) return out num_epochs = 2 -for epoch in range(num_epochs): - if (epoch + 1) % 25 == 0: - print("Epoch completed: " + str(epoch + 1)) - print(f"Epoch number: {epoch}") - train_loss = 0 - for index, row in x_train.iterrows(): - print(index) +x_dict = x_train.to_dict() +y_train = y_train.to_dict() - # Forward pass to get output - probs = x_train[0][index] - # Get the target label - target = y_train[0][index] - # Calculate Loss: softmax --> cross entropy loss - loss = loss_function(probs, target) - # Accumulating the loss over time - train_loss += loss.item() +nn_model = FeedforwardNeuralNetModel() +BATCH_SIZE = 5 +criterion = torch.nn.BCELoss() +optimizer = torch.optim.SGD(nn_model.parameters(), lr = 0.1) - # Getting gradients w.r.t. parameters +for epoch in range(5): + loss_score = 0 + acc_score = 0 + items_total = 0 + nn_model.train() + for i in range(0, Y_train.shape[0], BATCH_SIZE): + X = X_train[i:i+BATCH_SIZE] + X = torch.tensor(X.astype(np.float32).todense()) + Y = Y_train[i:i+BATCH_SIZE] + Y = torch.tensor(Y.astype(np.float32)).reshape(-1,1) + Y_predictions = nn_model(X) + acc_score += torch.sum((Y_predictions > 0.5) == Y).item() + items_total += Y.shape[0] + + optimizer.zero_grad() + loss = criterion(Y_predictions, Y) loss.backward() + optimizer.step() - - train_loss = 0 + loss_score += loss.item() * Y.shape[0] -bow_ff_nn_predictions = [] -original_lables_ff_bow = [] -with torch.no_grad(): - for index, row in x_test.iterrows(): - probs = x_test[0][index] - bow_ff_nn_predictions.append(torch.argmax(probs, dim=1).cpu().numpy()[0]) +# for epoch in range(num_epochs): +# if (epoch + 1) % 25 == 0: +# print("Epoch completed: " + str(epoch + 1)) +# print(f"Epoch number: {epoch}") +# train_loss = 0 +# for index, row in x_train.iterrows(): +# # for index, row in x_train.iterrows(): +# +# +# print(row, index) +# # Forward pass to get output +# probs = x_train[0][index] +# # probs = torch.tensor(probs.astype(np.float32)) +# # Get the target label +# target = y_train[0][index] +# print(target) +# # target = np.array(target).astype(np.float32) +# print(type(target)) +# # target = .astype(np.float32).reshape(-1,1) +# # target +# # target = torch.tensor(target.astype(np.float32)).reshape(-1,1) +# +# # Calculate Loss: softmax --> cross entropy loss +# loss = loss_function(probs, target) +# # Accumulating the loss over time +# train_loss += loss.item() +# +# # Getting gradients w.r.t. parameters +# loss.backward() +# +# train_loss = 0 -print(bow_ff_nn_predictions) + +# bow_ff_nn_predictions = [] +# original_lables_ff_bow = [] +# with torch.no_grad(): +# for index, row in x_test.iterrows(): +# probs = x_test[0][index] +# bow_ff_nn_predictions.append(torch.argmax(probs, dim=1).cpu().numpy()[0]) +# +# print(bow_ff_nn_predictions)