0.77 acc
This commit is contained in:
parent
82f05fb088
commit
50de83af73
3
.gitignore
vendored
3
.gitignore
vendored
@ -9,4 +9,5 @@
|
|||||||
mlruns
|
mlruns
|
||||||
results
|
results
|
||||||
logs
|
logs
|
||||||
.idea
|
.idea
|
||||||
|
bert*
|
45
bert.py
45
bert.py
@ -1,5 +1,5 @@
|
|||||||
import torch
|
import torch
|
||||||
from transformers.file_utils import is_tf_available, is_torch_available, is_torch_tpu_available
|
from transformers.file_utils import is_tf_available, is_torch_available
|
||||||
from transformers import BertTokenizerFast, BertForSequenceClassification
|
from transformers import BertTokenizerFast, BertForSequenceClassification
|
||||||
from transformers import Trainer, TrainingArguments
|
from transformers import Trainer, TrainingArguments
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@ -46,18 +46,18 @@ def compute_metrics(pred):
|
|||||||
def get_prediction(text):
|
def get_prediction(text):
|
||||||
inputs = tokenizer(text, padding=True, truncation=True, max_length=max_length, return_tensors="pt").to("cuda")
|
inputs = tokenizer(text, padding=True, truncation=True, max_length=max_length, return_tensors="pt").to("cuda")
|
||||||
outputs = model(**inputs)
|
outputs = model(**inputs)
|
||||||
probs = outputs[0].softmax(1)
|
return outputs[0].softmax(1).argmax()
|
||||||
return probs.argmax()
|
|
||||||
|
|
||||||
|
|
||||||
set_seed(1)
|
set_seed(1)
|
||||||
|
SAMPLES = 2000
|
||||||
|
|
||||||
train_texts = \
|
train_texts = \
|
||||||
pd.read_csv('train/in.tsv.xz', compression='xz', sep='\t', header=None, error_bad_lines=False, quoting=3)[0].tolist()
|
pd.read_csv('train/in.tsv.xz', compression='xz',
|
||||||
train_labels = pd.read_csv('train/expected.tsv', sep='\t', header=None, quoting=3)[0].tolist()
|
sep='\t', header=None, error_bad_lines=False, quoting=3)[0][:SAMPLES].tolist()
|
||||||
|
train_labels = pd.read_csv('train/expected.tsv', sep='\t', header=None, quoting=3)[0][:SAMPLES].tolist()
|
||||||
dev_texts = pd.read_csv('dev-0/in.tsv.xz', compression='xz', sep='\t', header=None, quoting=3)[0].tolist()
|
dev_texts = pd.read_csv('dev-0/in.tsv.xz', compression='xz', sep='\t', header=None, quoting=3)[0].tolist()
|
||||||
dev_labels = pd.read_csv('dev-0/expected.tsv', sep='\t', header=None, quoting=3)[0].tolist()
|
dev_labels = pd.read_csv('dev-0/expected.tsv', sep='\t', header=None, quoting=3)[0].tolist()
|
||||||
# test_texts = pd.read_table('test-A/in.tsv.xz', compression='xz', sep='\t', header=None, quoting=3)
|
|
||||||
|
|
||||||
model_name = "bert-base-uncased"
|
model_name = "bert-base-uncased"
|
||||||
max_length = 512
|
max_length = 512
|
||||||
@ -73,31 +73,30 @@ model = BertForSequenceClassification.from_pretrained(
|
|||||||
model_name, num_labels=len(pd.unique(train_labels))).to("cuda")
|
model_name, num_labels=len(pd.unique(train_labels))).to("cuda")
|
||||||
|
|
||||||
training_args = TrainingArguments(
|
training_args = TrainingArguments(
|
||||||
output_dir='./results', # output directory
|
output_dir='./results',
|
||||||
num_train_epochs=3, # total number of training epochs
|
num_train_epochs=1,
|
||||||
per_device_train_batch_size=1, # batch size per device during training
|
per_device_train_batch_size=4,
|
||||||
per_device_eval_batch_size=1, # batch size for evaluation
|
per_device_eval_batch_size=4,
|
||||||
warmup_steps=500, # number of warmup steps for learning rate scheduler
|
warmup_steps=500,
|
||||||
weight_decay=0.01, # strength of weight decay
|
weight_decay=0.005,
|
||||||
logging_dir='./logs', # directory for storing logs
|
logging_dir='./logs',
|
||||||
load_best_model_at_end=True, # load the best model when finished training (default metric is loss)
|
load_best_model_at_end=True,
|
||||||
# but you can specify `metric_for_best_model` argument to change to accuracy or other metric
|
logging_steps=250,
|
||||||
logging_steps=200, # log & save weights each logging_steps
|
evaluation_strategy="steps",
|
||||||
evaluation_strategy="steps", # evaluate each `logging_steps`
|
|
||||||
)
|
)
|
||||||
|
|
||||||
trainer = Trainer(
|
trainer = Trainer(
|
||||||
model=model, # the instantiated Transformers model to be trained
|
model=model,
|
||||||
args=training_args, # training arguments, defined above
|
args=training_args,
|
||||||
train_dataset=train_dataset, # training dataset
|
train_dataset=train_dataset,
|
||||||
eval_dataset=valid_dataset, # evaluation dataset
|
eval_dataset=valid_dataset,
|
||||||
compute_metrics=compute_metrics, # the callback that computes metrics of interest
|
compute_metrics=compute_metrics,
|
||||||
)
|
)
|
||||||
|
|
||||||
trainer.train()
|
trainer.train()
|
||||||
|
|
||||||
trainer.evaluate()
|
trainer.evaluate()
|
||||||
|
|
||||||
model_path = "bert-base-uncased"
|
model_path = "bert-base-uncased-2k"
|
||||||
model.save_pretrained(model_path)
|
model.save_pretrained(model_path)
|
||||||
tokenizer.save_pretrained(model_path)
|
tokenizer.save_pretrained(model_path)
|
||||||
|
35
bert_infer.py
Normal file
35
bert_infer.py
Normal file
@ -0,0 +1,35 @@
|
|||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
from transformers import BertForSequenceClassification, BertTokenizerFast
|
||||||
|
|
||||||
|
|
||||||
|
model_path = "bert-base-uncased-2k"
|
||||||
|
max_length = 512
|
||||||
|
DEV = 'dev-0'
|
||||||
|
TEST = 'test-A'
|
||||||
|
|
||||||
|
model = BertForSequenceClassification.from_pretrained(model_path, num_labels=2).to("cuda")
|
||||||
|
tokenizer = BertTokenizerFast.from_pretrained(model_path)
|
||||||
|
|
||||||
|
|
||||||
|
def get_prediction(text):
|
||||||
|
inputs = tokenizer(text, padding=True, truncation=True, max_length=max_length, return_tensors="pt").to("cuda")
|
||||||
|
outputs = model(**inputs)
|
||||||
|
return outputs[0].softmax(1).argmax()
|
||||||
|
|
||||||
|
|
||||||
|
def get_predictions_for(dataset):
|
||||||
|
test = pd.read_csv(f'{dataset}/in.tsv.xz', compression='xz', sep='\t',
|
||||||
|
error_bad_lines=False, header=None, quoting=3)[0].tolist()
|
||||||
|
|
||||||
|
test_infers = []
|
||||||
|
for row in test:
|
||||||
|
test_infers.append(get_prediction(row))
|
||||||
|
|
||||||
|
with open(f'{dataset}/out.tsv', 'w') as file:
|
||||||
|
for infer in test_infers:
|
||||||
|
file.write(str(infer.item()) + '\n')
|
||||||
|
|
||||||
|
|
||||||
|
get_predictions_for(DEV)
|
||||||
|
get_predictions_for(TEST)
|
5272
dev-0/out.tsv
Normal file
5272
dev-0/out.tsv
Normal file
File diff suppressed because it is too large
Load Diff
5152
test-A/out.tsv
Normal file
5152
test-A/out.tsv
Normal file
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user