#!/usr/bin/env bash
set -e
set -x
TEXT=data
fairseq-preprocess \
--only-source --nwordssrc 50000 \
--trainpref $TEXT/train.txt \
--validpref $TEXT/valid.txt \
--testpref $TEXT/test.txt \
--destdir data-bin/lm-spm-bpe \
--workers 8