hf challam robertabase without date (checkpoint 395000) epoch1

hf_challam_roberta_base/01_create_datasets.py

@@ -0,0 +1,13 @@
+import datetime
+
+for split in 'train', 'dev-0':
+    with open(f'../{split}/in.tsv') as f_in, open(f'../{split}/expected.tsv') as f_exp, open(f'./{split}_huggingface_format.csv', 'w') as f_hf:
+        f_hf.write('year\ttext\n')
+        for line_in,line_exp in zip(f_in,f_exp):
+            f_hf.write(line_exp.rstrip() + '\t' + line_in.split('\t')[1])
+
+for split in ('test-A',):
+    with open(f'../{split}/in.tsv') as f_in,  open(f'./{split}_huggingface_format.csv', 'w') as f_hf:
+        f_hf.write('year\ttext\n')
+        for line_in in f_in:
+            f_hf.write('0.0' + '\t' + line_in.split('\t')[1])

hf_challam_roberta_base/02_load_dataset.py

@@ -0,0 +1,53 @@
+import pickle
+from datasets import load_dataset
+from transformers import AutoTokenizer
+from tqdm import tqdm
+from sklearn.preprocessing import MinMaxScaler
+import numpy as np
+
+dataset = load_dataset('csv', sep='\t', data_files={'train': ['./train_huggingface_format.csv'], 'test': ['./dev-0_huggingface_format.csv']})
+test_dataset_A = load_dataset('csv', sep='\t', data_files='test-A_huggingface_format.csv')
+
+tokenizer = AutoTokenizer.from_pretrained('without_date/checkpoint-395000')
+
+def tokenize_function(examples):
+    t = tokenizer(examples["text"], padding="max_length", truncation=True, max_length=512)
+    return t
+
+test_tokenized_datasets_A = test_dataset_A.map(tokenize_function, batched=True)
+tokenized_datasets = dataset.map(tokenize_function, batched=True)
+
+train_dataset = tokenized_datasets["train"].shuffle(seed=42)
+eval_dataset_full = tokenized_datasets["test"]
+eval_dataset_small = tokenized_datasets["test"].select(range(2000))
+test_dataset_A = test_tokenized_datasets_A["train"]
+
+
+scalers = dict()
+scalers['year'] = MinMaxScaler().fit(np.array(train_dataset['year']).reshape(-1,1))
+
+def add_scaled(example):
+    for factor in ('year',):
+        example[factor + '_scaled'] = scalers[factor].transform(np.array(example[factor]).reshape(-1,1)).reshape(1,-1)[0].item()
+    return example
+
+train_dataset = train_dataset.map(add_scaled)
+eval_dataset_full = eval_dataset_full.map(add_scaled)
+eval_dataset_small = eval_dataset_small.map(add_scaled)
+#test_dataset_A = test_dataset_A.map(add_scaled)
+
+
+with open('train_dataset.pickle','wb') as f_p:
+    pickle.dump(train_dataset, f_p)
+
+with open('eval_dataset_small.pickle','wb') as f_p:
+    pickle.dump(eval_dataset_small, f_p)
+
+with open('eval_dataset_full.pickle','wb') as f_p:
+    pickle.dump(eval_dataset_full, f_p)
+
+with open('test_dataset_A.pickle','wb') as f_p:
+    pickle.dump(test_dataset_A, f_p)
+
+with open('scalers.pickle','wb') as f_p:
+    pickle.dump(scalers, f_p)

hf_challam_roberta_base/03_train_pytorch_regression.py

@@ -0,0 +1,98 @@
+import pickle
+from datasets import load_dataset
+from transformers import AutoTokenizer, RobertaModel, RobertaTokenizer
+from torch.utils.data import DataLoader
+from transformers import AutoModelForSequenceClassification
+from transformers import AdamW
+from transformers import get_scheduler
+import torch
+from tqdm.auto import tqdm
+
+BATCH_SIZE = 4
+
+
+with open('train_dataset.pickle','rb') as f_p:
+    train_dataset = pickle.load(f_p)
+
+with open('eval_dataset_small.pickle','rb') as f_p:
+    eval_dataset_small = pickle.load(f_p)
+
+with open('eval_dataset_full.pickle','rb') as f_p:
+    eval_dataset_full = pickle.load(f_p)
+
+
+train_dataloader = DataLoader(train_dataset, shuffle=True, batch_size=BATCH_SIZE)
+eval_dataloader = DataLoader(eval_dataset_small, batch_size=BATCH_SIZE)
+
+
+model = AutoModelForSequenceClassification.from_pretrained('without_date/checkpoint-395000', num_labels=1)
+optimizer = AdamW(model.parameters(), lr=1e-6)
+
+
+num_epochs = 1
+num_training_steps = num_epochs * len(train_dataloader)
+lr_scheduler = get_scheduler(
+    "linear",
+    optimizer=optimizer,
+    num_warmup_steps=0,
+    num_training_steps=num_training_steps
+)
+
+
+device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+model.to(device)
+
+
+progress_bar = tqdm(range(num_training_steps))
+model.train()
+
+model.train()
+model.to(device)
+
+def transform_batch(batch):
+        batch['input_ids'] = torch.stack(batch['input_ids']).permute(1,0).to(device)
+        batch['attention_mask'] = torch.stack(batch['attention_mask']).permute(1,0).to(device)
+        batch['labels'] = batch['year_scaled'].to(device).float()
+
+        batch['labels'].to(device)
+        batch['input_ids'].to(device)
+        batch['attention_mask'].to(device)
+
+        for c in set(batch.keys()) - {'input_ids', 'attention_mask', 'labels'}:
+            del batch[c]
+        return batch
+
+
+def eval():
+    model.eval()
+    eval_loss = 0.0
+    for i, batch in enumerate(eval_dataloader):
+        batch = transform_batch(batch)
+        outputs = model(**batch)
+        loss = outputs.loss
+        eval_loss += loss.item()
+    print(f'eval loss: {eval_loss /  i }')
+    model.train()
+
+
+for epoch in range(num_epochs):
+    train_loss = 0.0
+    for i, batch in enumerate(train_dataloader):
+        batch = transform_batch(batch)
+        outputs = model(**batch)
+        loss = outputs.loss
+        loss.backward()
+
+        optimizer.step()
+        lr_scheduler.step()
+        optimizer.zero_grad()
+        progress_bar.update(1)
+
+        train_loss += loss.item()
+        #import pdb; pdb.set_trace()
+        if i % 5000 == 0 and i > 1 :
+            print(f'train loss: {train_loss /  5000 }', end = '\t\t')
+            train_loss = 0.0
+            eval()
+
+model.save_pretrained('roberta_year_prediction')

hf_challam_roberta_base/04_predict.py

@@ -0,0 +1,55 @@
+import pickle
+import torch
+from transformers import AutoModelForSequenceClassification
+from torch.utils.data import DataLoader
+from tqdm.auto import tqdm
+
+with open('train_dataset.pickle','rb') as f_p:
+    train_dataset = pickle.load(f_p)
+
+with open('eval_dataset_small.pickle','rb') as f_p:
+    eval_dataset_small = pickle.load(f_p)
+
+with open('eval_dataset_full.pickle','rb') as f_p:
+    eval_dataset_full = pickle.load(f_p)
+
+with open('test_dataset_A.pickle','rb') as f_p:
+    test_dataset_A = pickle.load(f_p)
+
+
+device = 'cuda'
+model = AutoModelForSequenceClassification.from_pretrained('./roberta_year_prediction')
+model.eval()
+model.to(device)
+
+with open('scalers.pickle', 'rb') as f_scaler:
+    scalers = pickle.load(f_scaler)
+
+def predict(dataset, out_f):
+    eval_dataloader = DataLoader(dataset, batch_size=1)
+    outputs = []
+
+    progress_bar = tqdm(range(len(eval_dataloader)))
+
+    for batch in eval_dataloader:
+        batch['input_ids'] = torch.stack(batch['input_ids']).permute(1,0).to(device)
+        batch['attention_mask'] = torch.stack(batch['attention_mask']).permute(1,0).to(device)
+        batch['labels'] = batch['year_scaled'].to(device).float()
+
+        batch['labels'].to(device)
+        batch['input_ids'].to(device)
+        batch['attention_mask'].to(device)
+
+        for c in set(batch.keys()) - {'input_ids', 'attention_mask', 'labels'}:
+            del batch[c]
+        outputs.extend(model(**batch).logits.tolist())
+        progress_bar.update(1)
+        outputs_transformed = scalers['year'].inverse_transform(outputs)
+
+    with open(out_f,'w') as f_out:
+
+        for o in outputs_transformed:
+            f_out.write(str(o[0]) + '\n')
+
+predict(eval_dataset_full, '../dev-0/out.tsv')
+predict(eval_dataset_full, '../test-A/out.tsv')

hf_challam_roberta_base/04_predict_from_file.py

@@ -0,0 +1,52 @@
+import pickle
+import torch
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+from torch.utils.data import DataLoader
+from tqdm.auto import tqdm
+
+#with open('train_dataset.pickle','rb') as f_p:
+#    train_dataset = pickle.load(f_p)
+#
+#with open('eval_dataset_small.pickle','rb') as f_p:
+#    eval_dataset_small = pickle.load(f_p)
+#
+#with open('eval_dataset_full.pickle','rb') as f_p:
+#    eval_dataset_full = pickle.load(f_p)
+#
+#with open('test_dataset_A.pickle','rb') as f_p:
+#    test_dataset_A = pickle.load(f_p)
+
+with open('dev-0_huggingface_format.csv','r') as f_p:
+    eval_dataset_full = f_p.readlines()
+
+with open('test-A_huggingface_format.csv','r') as f_p:
+    test_dataset = f_p.readlines()
+
+device = 'cuda'
+model = AutoModelForSequenceClassification.from_pretrained('./roberta_year_prediction')
+tokenizer = AutoTokenizer.from_pretrained('without_date/checkpoint-395000/')
+model.eval()
+model.to(device)
+
+with open('scalers.pickle', 'rb') as f_scaler:
+    scalers = pickle.load(f_scaler)
+
+def predict(dataset, out_f):
+    outputs = []
+
+    for sample in tqdm(eval_dataset_full[1:]):
+        y, t = sample.split('\t')
+        t = t.rstrip()
+
+        t = tokenizer(t, padding="max_length", truncation=True, max_length=512, return_tensors='pt').to('cuda')
+
+        outputs.extend(model(**t).logits.tolist())
+        outputs_transformed = scalers['year'].inverse_transform(outputs)
+
+    with open(out_f,'w') as f_out:
+
+        for o in outputs_transformed:
+            f_out.write(str(o[0]) + '\n')
+
+predict(eval_dataset_full, '../dev-0/out.tsv')
+predict(test_dataset, '../test-A/out.tsv')