s444476/exact_data2023
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

delete tokenizers

Browse Source
This commit is contained in:
Maciej Ścigacz 2023-06-10 13:33:31 +02:00
parent 85cac92433
commit 751e972710
4 changed files with 0 additions and 254481 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

152250
application/tokenizers/errors_tokenizer/tokenizer.json
View File

File diff suppressed because it is too large Load Diff

1
application/tokenizers/errors_tokenizer/tokenizer_config.json
View File

@ -1 +0,0 @@
{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "add_prefix_space": false, "errors": "replace", "sep_token": "</s>", "cls_token": "<s>", "pad_token": "<pad>", "mask_token": "<mask>", "trim_offsets": true, "special_tokens_map_file": null, "name_or_path": "original/polish-bart-base", "tokenizer_class": "BartTokenizer"}

102219
application/tokenizers/sentiment_tokenizer/tokenizer.json
View File

File diff suppressed because it is too large Load Diff

11
application/tokenizers/sentiment_tokenizer/tokenizer_config.json
View File

@ -1,11 +0,0 @@
{
"add_bos_token": false,
"add_prefix_space": false,
"bos_token": "<s>",
"eos_token": "</s>",
"errors": "replace",
"name_or_path": "original/polish-gpt2-small",
"special_tokens_map_file": "original/polish-gpt2-small/special_tokens_map.json",
"tokenizer_class": "GPT2Tokenizer",
"unk_token": "<unk>"
}