This commit is contained in:
jakubknczny 2021-06-22 22:15:01 +02:00
parent 82f05fb088
commit 50de83af73
6 changed files with 10483 additions and 24 deletions

3
.gitignore vendored
View File

@ -9,4 +9,5 @@
mlruns mlruns
results results
logs logs
.idea .idea
bert*

45
bert.py
View File

@ -1,5 +1,5 @@
import torch import torch
from transformers.file_utils import is_tf_available, is_torch_available, is_torch_tpu_available from transformers.file_utils import is_tf_available, is_torch_available
from transformers import BertTokenizerFast, BertForSequenceClassification from transformers import BertTokenizerFast, BertForSequenceClassification
from transformers import Trainer, TrainingArguments from transformers import Trainer, TrainingArguments
import numpy as np import numpy as np
@ -46,18 +46,18 @@ def compute_metrics(pred):
def get_prediction(text): def get_prediction(text):
inputs = tokenizer(text, padding=True, truncation=True, max_length=max_length, return_tensors="pt").to("cuda") inputs = tokenizer(text, padding=True, truncation=True, max_length=max_length, return_tensors="pt").to("cuda")
outputs = model(**inputs) outputs = model(**inputs)
probs = outputs[0].softmax(1) return outputs[0].softmax(1).argmax()
return probs.argmax()
set_seed(1) set_seed(1)
SAMPLES = 2000
train_texts = \ train_texts = \
pd.read_csv('train/in.tsv.xz', compression='xz', sep='\t', header=None, error_bad_lines=False, quoting=3)[0].tolist() pd.read_csv('train/in.tsv.xz', compression='xz',
train_labels = pd.read_csv('train/expected.tsv', sep='\t', header=None, quoting=3)[0].tolist() sep='\t', header=None, error_bad_lines=False, quoting=3)[0][:SAMPLES].tolist()
train_labels = pd.read_csv('train/expected.tsv', sep='\t', header=None, quoting=3)[0][:SAMPLES].tolist()
dev_texts = pd.read_csv('dev-0/in.tsv.xz', compression='xz', sep='\t', header=None, quoting=3)[0].tolist() dev_texts = pd.read_csv('dev-0/in.tsv.xz', compression='xz', sep='\t', header=None, quoting=3)[0].tolist()
dev_labels = pd.read_csv('dev-0/expected.tsv', sep='\t', header=None, quoting=3)[0].tolist() dev_labels = pd.read_csv('dev-0/expected.tsv', sep='\t', header=None, quoting=3)[0].tolist()
# test_texts = pd.read_table('test-A/in.tsv.xz', compression='xz', sep='\t', header=None, quoting=3)
model_name = "bert-base-uncased" model_name = "bert-base-uncased"
max_length = 512 max_length = 512
@ -73,31 +73,30 @@ model = BertForSequenceClassification.from_pretrained(
model_name, num_labels=len(pd.unique(train_labels))).to("cuda") model_name, num_labels=len(pd.unique(train_labels))).to("cuda")
training_args = TrainingArguments( training_args = TrainingArguments(
output_dir='./results', # output directory output_dir='./results',
num_train_epochs=3, # total number of training epochs num_train_epochs=1,
per_device_train_batch_size=1, # batch size per device during training per_device_train_batch_size=4,
per_device_eval_batch_size=1, # batch size for evaluation per_device_eval_batch_size=4,
warmup_steps=500, # number of warmup steps for learning rate scheduler warmup_steps=500,
weight_decay=0.01, # strength of weight decay weight_decay=0.005,
logging_dir='./logs', # directory for storing logs logging_dir='./logs',
load_best_model_at_end=True, # load the best model when finished training (default metric is loss) load_best_model_at_end=True,
# but you can specify `metric_for_best_model` argument to change to accuracy or other metric logging_steps=250,
logging_steps=200, # log & save weights each logging_steps evaluation_strategy="steps",
evaluation_strategy="steps", # evaluate each `logging_steps`
) )
trainer = Trainer( trainer = Trainer(
model=model, # the instantiated Transformers model to be trained model=model,
args=training_args, # training arguments, defined above args=training_args,
train_dataset=train_dataset, # training dataset train_dataset=train_dataset,
eval_dataset=valid_dataset, # evaluation dataset eval_dataset=valid_dataset,
compute_metrics=compute_metrics, # the callback that computes metrics of interest compute_metrics=compute_metrics,
) )
trainer.train() trainer.train()
trainer.evaluate() trainer.evaluate()
model_path = "bert-base-uncased" model_path = "bert-base-uncased-2k"
model.save_pretrained(model_path) model.save_pretrained(model_path)
tokenizer.save_pretrained(model_path) tokenizer.save_pretrained(model_path)

35
bert_infer.py Normal file
View File

@ -0,0 +1,35 @@
import pandas as pd
from transformers import BertForSequenceClassification, BertTokenizerFast
model_path = "bert-base-uncased-2k"
max_length = 512
DEV = 'dev-0'
TEST = 'test-A'
model = BertForSequenceClassification.from_pretrained(model_path, num_labels=2).to("cuda")
tokenizer = BertTokenizerFast.from_pretrained(model_path)
def get_prediction(text):
inputs = tokenizer(text, padding=True, truncation=True, max_length=max_length, return_tensors="pt").to("cuda")
outputs = model(**inputs)
return outputs[0].softmax(1).argmax()
def get_predictions_for(dataset):
test = pd.read_csv(f'{dataset}/in.tsv.xz', compression='xz', sep='\t',
error_bad_lines=False, header=None, quoting=3)[0].tolist()
test_infers = []
for row in test:
test_infers.append(get_prediction(row))
with open(f'{dataset}/out.tsv', 'w') as file:
for infer in test_infers:
file.write(str(infer.item()) + '\n')
get_predictions_for(DEV)
get_predictions_for(TEST)

5272
dev-0/out.tsv Normal file

File diff suppressed because it is too large Load Diff

BIN
geval Executable file

Binary file not shown.

5152
test-A/out.tsv Normal file

File diff suppressed because it is too large Load Diff