s444415/donut
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' of https://git.wmi.amu.edu.pl/s444415/donut

Browse Source
This commit is contained in:
mkozlowskiAzimuthe 2023-03-14 15:51:05 +01:00
commit 2f1176b3c0
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
train_stream.py
View File

@ -34,6 +34,9 @@ def main(config, hug_token):
added_tokens = [] added_tokens = []
dataset = load_dataset(config.dataset_path, split="train[:80%]")
dataset = dataset.train_test_split(test_size=0.1)
train_dataset_process = DonutDatasetStream( train_dataset_process = DonutDatasetStream(
processor=processor, processor=processor,
model=model, model=model,
@ -49,7 +52,7 @@ def main(config, hug_token):
processor=processor, processor=processor,
model=model, model=model,
max_length=config.max_length, max_length=config.max_length,
split="validation", split="test",
task_start_token="<s_cord-v2>", task_start_token="<s_cord-v2>",
prompt_end_token="<s_cord-v2>", prompt_end_token="<s_cord-v2>",
added_tokens=added_tokens, added_tokens=added_tokens,