2022-04-24 20:48:00 +02:00
|
|
|
#!/bin/bash
|
|
|
|
|
apt update -y
|
2022-04-25 00:21:53 +02:00
|
|
|
apt upgrade -y
|
2022-04-24 20:48:00 +02:00
|
|
|
|
2022-04-25 00:24:06 +02:00
|
|
|
apt-get install python3 python3-pip python3-venv unzip gawk screen python-dev libhunspell-dev -y
|
2022-04-24 20:48:00 +02:00
|
|
|
python3 -m venv env
|
|
|
|
|
source ./env/bin/activate
|
2022-04-24 21:35:29 +02:00
|
|
|
pip install pandas spacy regex scipy hunspell
|
2022-04-24 20:48:00 +02:00
|
|
|
python -m spacy download pl_core_news_lg
|
|
|
|
|
|
|
|
|
|
mkdir data && cd data
|
|
|
|
|
wget https://minio.clarin-pl.eu/ermlab/public/PoLitBert/corpus-oscar/corpus_oscar_2020-04-10_64M_lines.zip
|
|
|
|
|
unzip -p corpus_oscar_2020-04-10_64M_lines.zip | sed -r '/^\s*$/d' | gawk 'NF>6' > oscar_filtered.txt
|
2022-04-24 21:53:16 +02:00
|
|
|
split -l 1000000 --numeric-suffixes=1 --suffix-length=1 --additional-suffix=".txt" oscar_filtered.txt ""
|
