s444415/donut
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

streaming

Browse Source
This commit is contained in:
mkozlowskiAzimuthe 2023-01-25 17:59:38 +01:00
parent 679048f88a
commit cb3c3e46cb
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
utils/donut_dataset_stream.py
View File

@ -47,7 +47,7 @@ class DonutDataset(Dataset):
self.sort_json_key = sort_json_key self.sort_json_key = sort_json_key
self.added_tokens = added_tokens self.added_tokens = added_tokens
self.dataset = load_dataset(dataset_name_or_path, split=self.split, stream=True).with_format("torch") self.dataset = load_dataset(dataset_name_or_path, split=self.split, streaming=True).with_format("torch")
self.dataset_length = len(self.dataset) self.dataset_length = len(self.dataset)
self.gt_token_sequences = [] self.gt_token_sequences = []