paranormal-or-skeptic-ISI-public/pytorch_classifier.ipynb

import lzma
import torch
import numpy as np
from gensim import downloader

BATCH_SIZE = 10
EPOCHS = 10
FEATURES = 200

class NeuralNetworkModel(torch.nn.Module):

    def __init__(self):
        super(NeuralNetworkModel, self).__init__()
        self.fc1 = torch.nn.Linear(FEATURES, 1000)
        self.fc2 = torch.nn.Linear(1000, 500)
        self.fc3 = torch.nn.Linear(500, 1)

    def forward(self, x):
        x = self.fc1(x)
        x = torch.relu(x)
        x = self.fc2(x)
        x = torch.relu(x)
        x = self.fc3(x)
        x = torch.sigmoid(x)
        return x

# Read train files
with lzma.open("train/in.tsv.xz", "rt", encoding="utf-8") as train_file:
    x_train = [x.strip().lower() for x in train_file.readlines()]

with open("train/expected.tsv", "r", encoding="utf-8") as train_file:
    y_train = np.array([int(x.strip()) for x in train_file.readlines()])

word2vec = downloader.load("glove-twitter-200")

x_train_w2v = [np.mean([word2vec[word.lower()] for word in doc.split() if word.lower() in word2vec]
                   or [np.zeros(FEATURES)], axis=0) for doc in x_train]

model = NeuralNetworkModel()

criterion = torch.nn.BCELoss()
optimizer = torch.optim.ASGD(model.parameters(), lr=0.05)

for epoch in range(EPOCHS):
    print(epoch)
    loss_score = 0
    acc_score = 0
    items_total = 0
    for i in range(0, y_train.shape[0], BATCH_SIZE):
        x = x_train_w2v[i:i+BATCH_SIZE]
        x = torch.tensor(np.array(x).astype(np.float32))
        y = y_train[i:i+BATCH_SIZE]
        y = torch.tensor(y.astype(np.float32)).reshape(-1, 1)
        y_pred = model(x)
        acc_score += torch.sum((y_pred > 0.5) == y).item()
        items_total += y.shape[0]

        optimizer.zero_grad()
        loss = criterion(y_pred, y)
        loss.backward()
        optimizer.step()

        loss_score += loss.item() * y.shape[0]
    
    print((loss_score / items_total), (acc_score / items_total))

0
0.5444966091123856 0.7128072132302411
1
0.5187017436751196 0.7303153888921503
2
0.5117590330604093 0.7348944502191112
3
0.5075270808198805 0.7376916143781145
4
0.5043017516287736 0.7403230206610286
5
0.5016950109024928 0.7418977204838748
6
0.49942716640870777 0.7432134236253319
7
0.49766424133924386 0.7448606425189672
8
0.49617289846816215 0.745534033890579
9
0.49471875689137873 0.7467116054686286

# Read dev files
with lzma.open("dev-0/in.tsv.xz", "rt", encoding="utf-8") as dev_file:
    x_dev = [x.strip().lower() for x in dev_file.readlines()]

x_dev_w2v = [np.mean([word2vec[word.lower()] for word in doc.split() if word.lower() in word2vec]
                   or [np.zeros(FEATURES)], axis=0) for doc in x_train]

y_dev = []
with torch.no_grad():
    for i in range(0, len(x_dev_w2v), BATCH_SIZE):
        x = x_dev_w2v[i:i+BATCH_SIZE]
        x = torch.tensor(np.array(x).astype(np.float32))
        
        outputs = model(x
        
        y = (outputs > 0.5)
        y_dev.extend(y)