#!/usr/bin/env bash

set -e
set -x

fairseq-preprocess \
    --only-source \
    --trainpref "data/train.input0.spm" \
    --validpref "data/dev.input0.spm" \
    --destdir "data-bin/input0" \
    --workers 4 --srcdict roberta_base_fairseq/dict.txt

fairseq-preprocess \
    --only-source \
    --trainpref "data/train.label" \
    --validpref "data/dev.label" \
    --destdir "data-bin/label" \
    --workers 4