Modelowanie-Corpus/corpus/wiki.sh

18 lines
527 B
Bash
Raw Normal View History

2023-03-21 22:26:46 +01:00
#!/bin/bash
# Download the latest Wikipedia dump file
wget https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2
# Extract the text from the dump file using native Bash commands
bzcat enwiki-latest-pages-articles.xml.bz2 | \
sed -n '/<text/,/<\/text>/p' | \
sed -e 's/<[^>]*>//g' | \
sed '/^{{/d; /^$/d; /\t{{/d; s/|/ /g; s/=/ /g; /&.*;/d' | \
sed '/^\s*$/d; /^$/d' | \
sed 's/^[[:space:]]*//' | \
tr -d "'[]{}" > \
enwiki-latest-corpus.txt
# Clean up
rm enwiki-latest-pages-articles.xml.bz2