paranormal-or-skeptic-ISI-p.../main.py

223 lines
3.9 KiB
Python
Raw Normal View History

2021-06-28 20:13:35 +02:00
#!/usr/bin/env python
# coding: utf-8
2021-05-08 12:51:14 +02:00
2021-06-28 20:13:35 +02:00
# In[1]:
import csv
# In[2]:
get_ipython().system('pip install gensim')
# In[17]:
import nltk
nltk.download('punkt')
# In[9]:
get_ipython().system('pip install nltk')
# In[3]:
get_ipython().system('pip install torch')
# In[4]:
import gensim.downloader
2021-05-08 12:51:14 +02:00
import numpy as np
2021-06-28 20:13:35 +02:00
import pandas as pd
import torch
# In[5]:
import torch.nn as nn
from nltk import word_tokenize
# In[13]:
header_names = ["content", "id", "label"]
# In[23]:
class FF(nn.Module):
def __init__(self, input_dim, hidden_dim, output_dim):
super(FF, self).__init__()
self.fc1 = nn.Linear(input_dim, hidden_dim)
self.relu1 = nn.ReLU()
self.fc2 = nn.Linear(hidden_dim, hidden_dim)
self.relu2 = nn.ReLU()
self.fc3 = nn.Linear(hidden_dim, output_dim)
def forward(self, x):
out = self.fc1(x)
out = self.relu1(out)
out = self.relu2(out)
out = self.fc3(out)
return torch.sigmoid(out)
train_set_labels = pd.read_table(
"train/expected.tsv",
error_bad_lines=False,
quoting=csv.QUOTE_NONE,
header=None,
names=header_names[2:],
)
train_set_features = pd.read_table(
"train/in.tsv.xz",
error_bad_lines=False,
quoting=csv.QUOTE_NONE,
header=None,
names=header_names[:2],
)
test_set = pd.read_table(
"test-A/in.tsv.xz",
error_bad_lines=False,
header=None,
quoting=csv.QUOTE_NONE,
names=header_names[:2],
)
dev_set = pd.read_table(
"dev-0/in.tsv.xz",
error_bad_lines=False,
header=None,
quoting=csv.QUOTE_NONE,
names=header_names[:2],
)
X_train = train_set_features["content"].str.lower()
y_train = train_set_labels["label"]
X_dev = dev_set["content"].str.lower()
X_test = test_set["content"].str.lower()
X_train = [word_tokenize(content) for content in X_train]
X_dev = [word_tokenize(content) for content in X_dev]
X_test = [word_tokenize(content) for content in X_test]
word2vec = gensim.downloader.load("word2vec-google-news-300")
# In[24]:
X_train = [
np.mean(
[word2vec[word] for word in content if word in word2vec] or [np.zeros(300)],
axis=0,
)
for content in X_train
]
X_dev = [
np.mean(
[word2vec[word] for word in content if word in word2vec] or [np.zeros(300)],
axis=0,
)
for content in X_dev
]
X_test = [
np.mean(
[word2vec[word] for word in content if word in word2vec] or [np.zeros(300)],
axis=0,
)
for content in X_test
]
hidden_layer = 650
epochs = 15
batch_size = 10
# In[27]:
output_dim = 1
input_dim =300
model = FF(input_dim, hidden_layer, output_dim)
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
criterion = torch.nn.BCELoss()
# In[28]:
for epoch in range(epochs):
model.train()
for i in range(0, y_train.shape[0], batch_size):
X = X_train[i : i + batch_size]
X = torch.tensor(X)
y = y_train[i : i + batch_size]
y = torch.tensor(y.astype(np.float32).to_numpy()).reshape(-1, 1)
outputs = model(X.float())
loss = criterion(outputs, y)
optimizer.zero_grad()
loss.backward()
optimizer.step()
test_prediction = []
dev_prediction = []
model.eval()
with torch.no_grad():
for i in range(0, len(X_test), batch_size):
X = X_test[i : i + batch_size]
X = torch.tensor(X)
outputs = model(X.float())
prediction = outputs > 0.5
test_prediction += prediction.tolist()
for i in range(0, len(X_dev), batch_size):
X = X_dev[i : i + batch_size]
X = torch.tensor(X)
outputs = model(X.float())
prediction = outputs > 0.5
dev_prediction += prediction.tolist()
test_prediction = np.asarray(test_prediction, dtype=np.int32)
dev_prediction = np.asarray(dev_prediction, dtype=np.int32)
test_prediction.tofile("./test-A/out.tsv", sep="\n")
dev_prediction.tofile("./dev-0/out.tsv", sep="\n")
# In[ ]:
# In[ ]:
# In[ ]:
2021-05-08 12:51:14 +02:00