5 lines
463 B
Bash
5 lines
463 B
Bash
|
wget https://huggingface.co/allegro/plt5-base/raw/main/tokenizer_config.json
|
||
|
|
wget https://huggingface.co/allegro/plt5-base/raw/main/special_tokens_map.json
|
||
|
|
wget https://huggingface.co/allegro/plt5-base/raw/main/config.json
|
||
|
|
|
||
|
|
spm_train --input='/mnt/gpu_data1/zrostek/diachronia-year-prediction/regular_roberta_from_scratch/train_tok_2.csv' --model_prefix='spiece' --vocab_size=50000 --character_coverage=1.0 --model_type=unigram --train_extremely_large_corpus=true
|