24 lines
1.3 KiB
Bash
Executable File
24 lines
1.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -e
|
|
set -x
|
|
|
|
TOTAL_UPDATES=200_000 # Total number of training steps == 10 epoch (1 peoch = 20_000)
|
|
WARMUP_UPDATES=8_000 # Warmup the learning rate over this many updates
|
|
PEAK_LR=0.0001 # Peak learning rate, adjust as needed
|
|
TOKENS_PER_SAMPLE=256 # Max sequence length
|
|
MAX_POSITIONS=256 # Num. positional embeddings (usually same as above)
|
|
MAX_SENTENCES=50 # Number of sequences per batch (batch size)
|
|
UPDATE_FREQ=1 # Increase the batch size
|
|
|
|
DATA_DIR=data-bin/lm-spm-bpe
|
|
|
|
fairseq-train $DATA_DIR \
|
|
--fp16 --task masked_lm --criterion masked_lm --save-dir checkpoints/lm_roberta_small \
|
|
--arch roberta_base --sample-break-mode complete --tokens-per-sample $TOKENS_PER_SAMPLE \
|
|
--optimizer adam --adam-betas '(0.9,0.98)' --adam-eps 1e-6 --clip-norm 0.0 \
|
|
--lr-scheduler polynomial_decay --lr $PEAK_LR --warmup-updates $WARMUP_UPDATES --total-num-update $TOTAL_UPDATES \
|
|
--dropout 0.1 --attention-dropout 0.1 --encoder-layers 8 --encoder-embed-dim 512 --encoder-ffn-embed-dim 2048 --encoder-attention-heads 8 --weight-decay 0.01 \
|
|
--max-sentences $MAX_SENTENCES --max-positions $MAX_POSITIONS --update-freq $UPDATE_FREQ \
|
|
--max-epoch 10 --log-format tqdm --log-interval 1 --save-interval-updates 5000 --keep-interval-updates 5 --skip-invalid-size-inputs-valid-test
|