ireland-news-headlines/roberta_no_year/02_load_dataset.py

33 lines
1.2 KiB
Python
Raw Normal View History

2021-07-11 08:34:28 +02:00
import pickle
from datasets import load_dataset
from transformers import AutoTokenizer
from config import MODEL
dataset = load_dataset('csv', sep='\t', data_files={'train': ['../train/huggingface_format.tsv'], 'test': ['../dev-0/huggingface_format.tsv']})
test_dataset = load_dataset('csv', sep='\t', data_files ='../test-A/huggingface_format.tsv')
tokenizer = AutoTokenizer.from_pretrained(MODEL)
def tokenize_function(examples):
return tokenizer(examples["text"], padding="max_length", truncation=True)
tokenized_datasets = dataset.map(tokenize_function, batched=True)
test_tokenized_datasets = test_dataset.map(tokenize_function, batched=True)
train_dataset = tokenized_datasets["train"].shuffle(seed=42)
eval_dataset_full = tokenized_datasets["test"]
2021-09-18 09:20:14 +02:00
eval_dataset_small = tokenized_datasets["test"].select(range(2000))
2021-07-11 08:34:28 +02:00
test_dataset = test_tokenized_datasets["train"]
with open('train_dataset.pickle','wb') as f_p:
pickle.dump(train_dataset, f_p)
2021-09-18 09:20:14 +02:00
with open('eval_dataset_small.pickle','wb') as f_p:
pickle.dump(eval_dataset_small, f_p)
2021-07-11 08:34:28 +02:00
with open('eval_dataset_full.pickle','wb') as f_p:
pickle.dump(eval_dataset_full, f_p)
with open('test_dataset.pickle','wb') as f_p:
pickle.dump(test_dataset, f_p)