0.77 acc

.gitignore

@@ -10,3 +10,4 @@ mlruns
 results
 logs
 .idea
+bert*

bert.py

@@ -1,5 +1,5 @@
 import torch
-from transformers.file_utils import is_tf_available, is_torch_available, is_torch_tpu_available
+from transformers.file_utils import is_tf_available, is_torch_available
 from transformers import BertTokenizerFast, BertForSequenceClassification
 from transformers import Trainer, TrainingArguments
 import numpy as np
@@ -46,18 +46,18 @@ def compute_metrics(pred):
 def get_prediction(text):
     inputs = tokenizer(text, padding=True, truncation=True, max_length=max_length, return_tensors="pt").to("cuda")
     outputs = model(**inputs)
-    probs = outputs[0].softmax(1)
-    return probs.argmax()
+    return outputs[0].softmax(1).argmax()
 
 
 set_seed(1)
+SAMPLES = 2000
 
 train_texts = \
-    pd.read_csv('train/in.tsv.xz', compression='xz', sep='\t', header=None, error_bad_lines=False, quoting=3)[0].tolist()
-train_labels = pd.read_csv('train/expected.tsv', sep='\t', header=None, quoting=3)[0].tolist()
+    pd.read_csv('train/in.tsv.xz', compression='xz',
+                sep='\t', header=None, error_bad_lines=False, quoting=3)[0][:SAMPLES].tolist()
+train_labels = pd.read_csv('train/expected.tsv', sep='\t', header=None, quoting=3)[0][:SAMPLES].tolist()
 dev_texts = pd.read_csv('dev-0/in.tsv.xz', compression='xz', sep='\t', header=None, quoting=3)[0].tolist()
 dev_labels = pd.read_csv('dev-0/expected.tsv', sep='\t', header=None, quoting=3)[0].tolist()
-# test_texts = pd.read_table('test-A/in.tsv.xz', compression='xz', sep='\t', header=None, quoting=3)
 
 model_name = "bert-base-uncased"
 max_length = 512
@@ -73,31 +73,30 @@ model = BertForSequenceClassification.from_pretrained(
     model_name, num_labels=len(pd.unique(train_labels))).to("cuda")
 
 training_args = TrainingArguments(
-    output_dir='./results',          # output directory
-    num_train_epochs=3,              # total number of training epochs
-    per_device_train_batch_size=1,  # batch size per device during training
-    per_device_eval_batch_size=1,   # batch size for evaluation
-    warmup_steps=500,                # number of warmup steps for learning rate scheduler
-    weight_decay=0.01,               # strength of weight decay
-    logging_dir='./logs',            # directory for storing logs
-    load_best_model_at_end=True,     # load the best model when finished training (default metric is loss)
-    # but you can specify `metric_for_best_model` argument to change to accuracy or other metric
-    logging_steps=200,               # log & save weights each logging_steps
-    evaluation_strategy="steps",     # evaluate each `logging_steps`
+    output_dir='./results',
+    num_train_epochs=1,
+    per_device_train_batch_size=4,
+    per_device_eval_batch_size=4,
+    warmup_steps=500,
+    weight_decay=0.005,
+    logging_dir='./logs',
+    load_best_model_at_end=True,
+    logging_steps=250,
+    evaluation_strategy="steps",
 )
 
 trainer = Trainer(
-    model=model,                         # the instantiated Transformers model to be trained
-    args=training_args,                  # training arguments, defined above
-    train_dataset=train_dataset,         # training dataset
-    eval_dataset=valid_dataset,          # evaluation dataset
-    compute_metrics=compute_metrics,     # the callback that computes metrics of interest
+    model=model,
+    args=training_args,
+    train_dataset=train_dataset,
+    eval_dataset=valid_dataset,
+    compute_metrics=compute_metrics,
 )
 
 trainer.train()
 
 trainer.evaluate()
 
-model_path = "bert-base-uncased"
+model_path = "bert-base-uncased-2k"
 model.save_pretrained(model_path)
 tokenizer.save_pretrained(model_path)

bert_infer.py

@@ -0,0 +1,35 @@
+import pandas as pd
+
+from transformers import BertForSequenceClassification, BertTokenizerFast
+
+
+model_path = "bert-base-uncased-2k"
+max_length = 512
+DEV = 'dev-0'
+TEST = 'test-A'
+
+model = BertForSequenceClassification.from_pretrained(model_path, num_labels=2).to("cuda")
+tokenizer = BertTokenizerFast.from_pretrained(model_path)
+
+
+def get_prediction(text):
+    inputs = tokenizer(text, padding=True, truncation=True, max_length=max_length, return_tensors="pt").to("cuda")
+    outputs = model(**inputs)
+    return outputs[0].softmax(1).argmax()
+
+
+def get_predictions_for(dataset):
+    test = pd.read_csv(f'{dataset}/in.tsv.xz', compression='xz', sep='\t',
+                       error_bad_lines=False, header=None, quoting=3)[0].tolist()
+
+    test_infers = []
+    for row in test:
+        test_infers.append(get_prediction(row))
+
+    with open(f'{dataset}/out.tsv', 'w') as file:
+        for infer in test_infers:
+            file.write(str(infer.item()) + '\n')
+
+
+get_predictions_for(DEV)
+get_predictions_for(TEST)