223 lines
3.9 KiB
Python
223 lines
3.9 KiB
Python
|
#!/usr/bin/env python
|
||
|
# coding: utf-8
|
||
|
|
||
|
# In[1]:
|
||
|
|
||
|
|
||
|
import csv
|
||
|
|
||
|
|
||
|
# In[2]:
|
||
|
|
||
|
|
||
|
get_ipython().system('pip install gensim')
|
||
|
|
||
|
|
||
|
# In[17]:
|
||
|
|
||
|
|
||
|
import nltk
|
||
|
nltk.download('punkt')
|
||
|
|
||
|
|
||
|
# In[9]:
|
||
|
|
||
|
|
||
|
get_ipython().system('pip install nltk')
|
||
|
|
||
|
|
||
|
# In[3]:
|
||
|
|
||
|
|
||
|
get_ipython().system('pip install torch')
|
||
|
|
||
|
|
||
|
# In[4]:
|
||
|
|
||
|
|
||
|
import gensim.downloader
|
||
|
import numpy as np
|
||
|
import pandas as pd
|
||
|
import torch
|
||
|
|
||
|
|
||
|
# In[5]:
|
||
|
|
||
|
|
||
|
import torch.nn as nn
|
||
|
from nltk import word_tokenize
|
||
|
|
||
|
|
||
|
# In[13]:
|
||
|
|
||
|
|
||
|
header_names = ["content", "id", "label"]
|
||
|
|
||
|
|
||
|
# In[23]:
|
||
|
|
||
|
|
||
|
class FF(nn.Module):
|
||
|
def __init__(self, input_dim, hidden_dim, output_dim):
|
||
|
super(FF, self).__init__()
|
||
|
self.fc1 = nn.Linear(input_dim, hidden_dim)
|
||
|
self.relu1 = nn.ReLU()
|
||
|
self.fc2 = nn.Linear(hidden_dim, hidden_dim)
|
||
|
self.relu2 = nn.ReLU()
|
||
|
self.fc3 = nn.Linear(hidden_dim, output_dim)
|
||
|
|
||
|
def forward(self, x):
|
||
|
out = self.fc1(x)
|
||
|
out = self.relu1(out)
|
||
|
out = self.relu2(out)
|
||
|
out = self.fc3(out)
|
||
|
return torch.sigmoid(out)
|
||
|
|
||
|
train_set_labels = pd.read_table(
|
||
|
"train/expected.tsv",
|
||
|
error_bad_lines=False,
|
||
|
quoting=csv.QUOTE_NONE,
|
||
|
header=None,
|
||
|
names=header_names[2:],
|
||
|
)
|
||
|
|
||
|
train_set_features = pd.read_table(
|
||
|
"train/in.tsv.xz",
|
||
|
error_bad_lines=False,
|
||
|
quoting=csv.QUOTE_NONE,
|
||
|
header=None,
|
||
|
names=header_names[:2],
|
||
|
)
|
||
|
|
||
|
|
||
|
test_set = pd.read_table(
|
||
|
"test-A/in.tsv.xz",
|
||
|
error_bad_lines=False,
|
||
|
header=None,
|
||
|
quoting=csv.QUOTE_NONE,
|
||
|
names=header_names[:2],
|
||
|
)
|
||
|
|
||
|
dev_set = pd.read_table(
|
||
|
"dev-0/in.tsv.xz",
|
||
|
error_bad_lines=False,
|
||
|
header=None,
|
||
|
quoting=csv.QUOTE_NONE,
|
||
|
names=header_names[:2],
|
||
|
)
|
||
|
X_train = train_set_features["content"].str.lower()
|
||
|
y_train = train_set_labels["label"]
|
||
|
|
||
|
X_dev = dev_set["content"].str.lower()
|
||
|
X_test = test_set["content"].str.lower()
|
||
|
X_train = [word_tokenize(content) for content in X_train]
|
||
|
X_dev = [word_tokenize(content) for content in X_dev]
|
||
|
X_test = [word_tokenize(content) for content in X_test]
|
||
|
word2vec = gensim.downloader.load("word2vec-google-news-300")
|
||
|
|
||
|
|
||
|
# In[24]:
|
||
|
|
||
|
|
||
|
X_train = [
|
||
|
np.mean(
|
||
|
[word2vec[word] for word in content if word in word2vec] or [np.zeros(300)],
|
||
|
axis=0,
|
||
|
)
|
||
|
for content in X_train
|
||
|
]
|
||
|
X_dev = [
|
||
|
np.mean(
|
||
|
[word2vec[word] for word in content if word in word2vec] or [np.zeros(300)],
|
||
|
axis=0,
|
||
|
)
|
||
|
for content in X_dev
|
||
|
]
|
||
|
|
||
|
X_test = [
|
||
|
np.mean(
|
||
|
[word2vec[word] for word in content if word in word2vec] or [np.zeros(300)],
|
||
|
axis=0,
|
||
|
)
|
||
|
for content in X_test
|
||
|
]
|
||
|
hidden_layer = 650
|
||
|
epochs = 15
|
||
|
batch_size = 10
|
||
|
|
||
|
|
||
|
# In[27]:
|
||
|
|
||
|
|
||
|
output_dim = 1
|
||
|
|
||
|
|
||
|
input_dim =300
|
||
|
model = FF(input_dim, hidden_layer, output_dim)
|
||
|
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
|
||
|
criterion = torch.nn.BCELoss()
|
||
|
|
||
|
|
||
|
# In[28]:
|
||
|
|
||
|
|
||
|
for epoch in range(epochs):
|
||
|
model.train()
|
||
|
for i in range(0, y_train.shape[0], batch_size):
|
||
|
X = X_train[i : i + batch_size]
|
||
|
X = torch.tensor(X)
|
||
|
y = y_train[i : i + batch_size]
|
||
|
y = torch.tensor(y.astype(np.float32).to_numpy()).reshape(-1, 1)
|
||
|
|
||
|
outputs = model(X.float())
|
||
|
loss = criterion(outputs, y)
|
||
|
|
||
|
optimizer.zero_grad()
|
||
|
loss.backward()
|
||
|
optimizer.step()
|
||
|
test_prediction = []
|
||
|
dev_prediction = []
|
||
|
model.eval()
|
||
|
with torch.no_grad():
|
||
|
for i in range(0, len(X_test), batch_size):
|
||
|
X = X_test[i : i + batch_size]
|
||
|
X = torch.tensor(X)
|
||
|
|
||
|
outputs = model(X.float())
|
||
|
|
||
|
prediction = outputs > 0.5
|
||
|
test_prediction += prediction.tolist()
|
||
|
|
||
|
for i in range(0, len(X_dev), batch_size):
|
||
|
X = X_dev[i : i + batch_size]
|
||
|
X = torch.tensor(X)
|
||
|
|
||
|
outputs = model(X.float())
|
||
|
|
||
|
prediction = outputs > 0.5
|
||
|
dev_prediction += prediction.tolist()
|
||
|
|
||
|
test_prediction = np.asarray(test_prediction, dtype=np.int32)
|
||
|
dev_prediction = np.asarray(dev_prediction, dtype=np.int32)
|
||
|
test_prediction.tofile("./test-A/out.tsv", sep="\n")
|
||
|
dev_prediction.tofile("./dev-0/out.tsv", sep="\n")
|
||
|
|
||
|
|
||
|
# In[ ]:
|
||
|
|
||
|
|
||
|
|
||
|
|
||
|
|
||
|
# In[ ]:
|
||
|
|
||
|
|
||
|
|
||
|
|
||
|
|
||
|
# In[ ]:
|
||
|
|
||
|
|
||
|
|
||
|
|