roberta_year_as_token_everywhere_better_finetunning

This commit is contained in:
kubapok 2021-10-19 10:46:19 +02:00
parent 443895822e
commit cede76239d
2 changed files with 46632 additions and 46632 deletions

View File

@ -5,7 +5,7 @@ from config import MODEL
from tqdm import tqdm
dataset = load_dataset('csv', sep='\t', data_files={'train': ['../train/huggingface_format_year_clear.csv'], 'test': ['../dev-0/huggingface_format_year_clear.csv']})
test_dataset = load_dataset('csv', sep='\t', data_files='../test-A/huggingface_format_year.csv')
test_dataset = load_dataset('csv', sep='\t', data_files='../test-A/huggingface_format_year_clear.csv')
tokenizer = AutoTokenizer.from_pretrained(MODEL)

File diff suppressed because it is too large Load Diff