14 lines
616 B
Bash
14 lines
616 B
Bash
#!/bin/bash
|
|
apt update -y
|
|
apt upgrade -y
|
|
|
|
apt-get install python3 python3-pip python3-venv unzip gawk screen python-dev libhunspell-dev -y
|
|
python3 -m venv env
|
|
source ./env/bin/activate
|
|
pip install pandas spacy regex scipy wheel hunspell
|
|
python -m spacy download pl_core_news_lg
|
|
|
|
mkdir data && cd data
|
|
wget https://minio.clarin-pl.eu/ermlab/public/PoLitBert/corpus-oscar/corpus_oscar_2020-04-10_64M_lines.zip
|
|
unzip -p corpus_oscar_2020-04-10_64M_lines.zip | sed -r '/^\s*$/d' | gawk 'NF>6' > oscar_filtered.txt
|
|
split -l 1000000 --numeric-suffixes=1 --suffix-length=1 --additional-suffix=".txt" oscar_filtered.txt "" |