Compare commits

...

11 Commits

Author SHA1 Message Date
8cf99e9f55 update roberta script 2021-06-22 14:25:22 +02:00
99eb245d6c update roberta script 2021-06-22 14:23:29 +02:00
e348d16dde update roberta script 2021-06-22 14:22:05 +02:00
fb4b0d95e3 add output 2021-06-21 13:15:35 +02:00
8f410ae809 add roberta script 2021-06-21 13:03:59 +02:00
1c4f054533 add bert script 2021-06-21 13:03:04 +02:00
e77c9e41d1 add bert script 2021-06-21 13:01:15 +02:00
305ae96fda add bert script 2021-06-21 13:00:35 +02:00
d40f8bfd4b add geval 2021-05-25 17:33:41 +02:00
piotr6789
a77bae1b00 add logistic regression 2021-05-25 17:26:55 +02:00
piotr6789
69719655e6 add logistic regression 2021-05-25 17:09:13 +02:00
25 changed files with 310755 additions and 0 deletions

View File

@ -0,0 +1,6 @@
{
"cells": [],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 4
}

View File

@ -0,0 +1,6 @@
{
"cells": [],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 4
}

View File

@ -0,0 +1,6 @@
{
"cells": [],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 4
}

5272
dev-0/in.tsv Normal file

File diff suppressed because one or more lines are too long

Binary file not shown.

5272
dev-0/out.tsv Normal file

File diff suppressed because it is too large Load Diff

BIN
geval Executable file

Binary file not shown.

103
logistic-regression.ipynb Normal file
View File

@ -0,0 +1,103 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import torch\n",
"import gensim.downloader as gn\n",
"import csv\n",
"from nltk.tokenize import word_tokenize"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"STEP 3 - PREPROCESSING\n"
]
}
],
"source": [
"names = ['content', 'id', 'label']\n",
"train_data_content = pd.read_table('train/in.tsv', error_bad_lines = False, header = None, quoting = csv.QUOTE_NONE, names = ['content', 'id'])\n",
"train_data_labels = pd.read_table('train/expected.tsv', error_bad_lines = False, header = None, quoting=csv.QUOTE_NONE, names = ['label'])\n",
"dev_data = pd.read_table('dev-0/in.tsv', error_bad_lines = False, header = None, quoting = csv.QUOTE_NONE, names = ['content', 'id'])\n",
"test_data = pd.read_table('test-A/in.tsv', error_bad_lines = False, header = None, quoting = csv.QUOTE_NONE, names = ['content', 'id'])\n",
"\n",
"print('STEP 3 - PREPROCESSING')\n",
"# lowercase all content\n",
"X_train = train_data_content['content'].str.lower()\n",
"y_train = train_data_labels['label']\n",
"X_dev = dev_data['content'].str.lower()\n",
"X_test = test_data['content'].str.lower()\n",
"\n",
"# tokenize datasets\n",
"X_train = [word_tokenize(content) for content in X_train]\n",
"X_dev = [word_tokenize(content) for content in X_dev]\n",
"X_test = [word_tokenize(content) for content in X_test]"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[==================================================] 100.0% 1662.8/1662.8MB downloaded\n"
]
}
],
"source": [
"w2v = gn.load('word2vec-google-news-300')\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

123
logistic-regression.py Normal file
View File

@ -0,0 +1,123 @@
import pandas as pd
import numpy as np
import csv
import torch
from nltk.tokenize import word_tokenize
from gensim import downloader
FEATURES = ['content', 'id', 'label']
PATHS = ['train/in.tsv', 'train/expected.tsv', 'dev-0/in.tsv', 'test-A/in.tsv', './dev-0/out.tsv', './test-A/out.tsv']
PRE_TRAINED = 'word2vec-google-news-300'
class NeuralNetwork(torch.nn.Module):
def __init__(self, INPUT_DIM):
super(NeuralNetwork, self).__init__()
self.l1 = torch.nn.Linear(INPUT_DIM, 500)
self.l2 = torch.nn.Linear(500, 1)
def forward(self, x):
x = self.l1(x)
x = torch.relu(x)
x = self.l2(x)
x = torch.sigmoid(x)
return x
def get_data(FEATURES, PATHS):
x_train = pd.read_table(PATHS[0], error_bad_lines = False, header = None, quoting = csv.QUOTE_NONE, names = FEATURES[:2])
y_train = pd.read_table(PATHS[1], error_bad_lines = False, header = None, quoting = csv.QUOTE_NONE, names = FEATURES[2:])
x_dev = pd.read_table(PATHS[2], error_bad_lines = False, header = None, quoting = csv.QUOTE_NONE, names = FEATURES[:2])
x_test = pd.read_table(PATHS[3], error_bad_lines = False, header = None, quoting = csv.QUOTE_NONE, names = FEATURES[:2])
return x_train, y_train, x_dev, x_test
def preprocess(x_train, y_train, x_dev, x_test):
x_train = x_train[FEATURES[0]].str.lower()
x_dev = x_dev[FEATURES[0]].str.lower()
x_test = x_test[FEATURES[0]].str.lower()
y_train = y_train[FEATURES[2]]
return x_train, y_train, x_dev, x_test
def tokenize(x_train, x_dev, x_test):
x_train = [word_tokenize(i) for i in x_train]
x_dev = [word_tokenize(i) for i in x_dev]
x_test = [word_tokenize(i) for i in x_test]
return x_train, x_dev, x_test
def use_word2vec():
w2v = downloader.load(PRE_TRAINED)
return w2v
def document_vector(w2v, x_train, x_dev, x_test):
x_train = [np.mean([w2v[word] for word in doc if word in w2v] or [np.zeros(300)], axis = 0) for doc in x_train]
x_dev = [np.mean([w2v[word] for word in doc if word in w2v] or [np.zeros(300)], axis = 0) for doc in x_dev]
x_test = [np.mean([w2v[word] for word in doc if word in w2v] or [np.zeros(300)], axis = 0) for doc in x_test]
return x_train, x_dev, x_test
def basic_config():
INPUT_DIM = 300
BATCH_SIZE = 5
return INPUT_DIM, BATCH_SIZE
def init_model(INPUT_DIM):
nn_model = NeuralNetwork(INPUT_DIM)
criterion = torch.nn.BCELoss()
optimizer = torch.optim.SGD(nn_model.parameters(), lr = 0.1)
return nn_model, optimizer, criterion
def train(nn_model, BATCH_SIZE, criterion, optimizer, x_train, y_train):
for epoch in range(5):
nn_model.train()
for i in range(0, y_train.shape[0], BATCH_SIZE):
X = x_train[i:i+BATCH_SIZE]
X = torch.tensor(X)
y = y_train[i:i+BATCH_SIZE]
y = torch.tensor(y.astype(np.float32).to_numpy()).reshape(-1, 1)
outputs = nn_model(X.float())
loss = criterion(outputs, y)
optimizer.zero_grad()
loss.backward()
optimizer.step()
def prediction(nn_model, BATCH_SIZE, x_dev, x_test):
y_dev, y_test = [], []
nn_model.eval()
with torch.no_grad():
for i in range(0, len(x_dev), BATCH_SIZE):
X = x_dev[i:i+BATCH_SIZE]
X = torch.tensor(X)
outputs = nn_model(X.float())
prediction = (outputs > 0.5)
y_dev += prediction.tolist()
for i in range(0, len(x_test), BATCH_SIZE):
X = x_test[i:i+BATCH_SIZE]
X = torch.tensor(X)
outputs = nn_model(X.float())
prediction = (outputs > 0.5)
y_test += prediction.tolist()
return y_dev, y_test
def get_result(y_dev, y_test):
np.asarray(y_dev, dtype = np.int32).tofile(PATHS[4], sep='\n')
np.asarray(y_test, dtype = np.int32).tofile(PATHS[5], sep='\n')
def main():
x_train, y_train, x_dev, x_test = get_data(FEATURES, PATHS)
x_train, y_train, x_dev, x_test = preprocess(x_train, y_train, x_dev, x_test)
x_train, x_dev, x_test = tokenize(x_train, x_dev, x_test)
w2v = use_word2vec()
x_train, x_dev, x_test = document_vector(w2v, x_train, x_dev, x_test)
INPUT_DIM, BATCH_SIZE = basic_config()
nn_model, optimizer, criterion = init_model(INPUT_DIM)
train(nn_model, BATCH_SIZE, criterion, optimizer, x_train, y_train)
y_dev, y_test = prediction(nn_model, BATCH_SIZE, x_dev, x_test)
get_result(y_dev, y_test)
if _name_ == '_main_':
main()

Binary file not shown.

Binary file not shown.

Binary file not shown.

84
roberta.py Normal file
View File

@ -0,0 +1,84 @@
from transformers import AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer
import torch
PATHS = ['train/in.tsv', 'train/expected.tsv', 'dev-0/in.tsv', 'test-A/in.tsv', './dev-0/out.tsv', './test-A/out.tsv']
OUTPUT_PATHS = ['dev-0/out.tsv', 'test-A/out.tsv']
PRE_TRAINED = ['roberta-base']
def get_data(path):
data = []
with open(path, encoding='utf-8') as f:
data = f.readlines()
return data
def generate_output(path, trainer, X_data):
data = []
with open(path, encoding='utf-8') as f:
for result in trainer.predict(X_data):
f.write(str(result) + '\n')
class IMDbDataset(torch.utils.data.Dataset):
def __init__(self, encodings, labels):
self.encodings = encodings
self.labels = labels
def __getitem__(self, idx):
item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
item['labels'] = torch.tensor(self.labels[idx])
return item
def __len__(self):
return len(self.labels)
def prepare(data_train_X, data_train_Y):
tokenizer = AutoTokenizer.from_pretrained(PRE_TRAINED[0])
model = AutoModelForSequenceClassification.from_pretrained(PRE_TRAINED[0], num_labels=2)
device = torch.device("cpu")
model.to(device)
encoded_input = tokenizer([text[0] for text in list(zip(data_train_X, data_train_Y))], truncation=True, padding=True)
train_dataset = IMDbDataset(encoded_input , [int(text[1]) for text in list(zip(data_train_X, data_train_Y))])
return train_dataset, model
def training(train_dataset, model):
training_args = TrainingArguments(
output_dir='./results', # output directory
num_train_epochs=3, # total number of training epochs
per_device_train_batch_size=16, # batch size per device during training
per_device_eval_batch_size=64, # batch size for evaluation
warmup_steps=500, # number of warmup steps for learning rate scheduler
weight_decay=0.01, # strength of weight decay
logging_dir='./logs', # directory for storing logs
logging_steps=10,
)
trainer = Trainer(
model=model, # the instantiated Transformers model to be trained
args=training_args, # training arguments, defined above
train_dataset=train_dataset, # training dataset
)
trainer.train()
return trainer
def main():
#data
X_train = get_data(PATHS[0])
y_train = get_data(PATHS[1])
X_dev = get_data(PATHS[2])
X_test = get_data(PATHS[3])
#prepare
train_dataset, model = prepare(X_train, y_train)
#trainer
trainer = training(train_dataset, model)
#output
generate_output(OUTPUT_PATHS[0], trainer, X_dev)
generate_output(OUTPUT_PATHS[1], trainer, X_test)
if __name__ == '__main__':
main()

5152
test-A/in.tsv Normal file

File diff suppressed because one or more lines are too long

Binary file not shown.

5152
test-A/out.tsv Normal file

File diff suppressed because it is too large Load Diff

289579
train/in.tsv Normal file

File diff suppressed because one or more lines are too long

Binary file not shown.