commit 17253595a325e54495e1a07c8eb39ac48f3db310
Author: Kuba <jakeic@st.amu.edu.pl>
Date:   Wed Mar 15 23:59:24 2023 +0100

    Create corpus script

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..8567826
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+*.txt
+*.bz2
\ No newline at end of file
diff --git a/readme.md b/readme.md
new file mode 100644
index 0000000..c0e90f6
--- /dev/null
+++ b/readme.md
@@ -0,0 +1,32 @@
+# Text corbus
+based on Wikipedia EN Latest Articles -- enwiki-latest-pages-articles.xml.bz2
+created with wiki.sh script
+
+# Code
+ - include only article text in between <text> marks
+ - exclude lines without alphabet letters 
+ - clear excessive white spaces 
+ - remove special characters
+ - remove unintentional empty lines
+
+# Stats
+## No lines
+```bash
+wc -l
+
+```
+## Size
+```bash
+du -sh enwiki-latest-corpus.txt.bz2
+
+```
+## Head of file
+```bash
+bzcat enwiki-latest-corpus.txt.bz2 | head -n 5
+
+```
+## Random lines from file
+```bash
+bzcat enwiki-latest-corpus.txt.bz2 | shuf -n 5
+
+```
\ No newline at end of file
diff --git a/wiki.sh b/wiki.sh
new file mode 100755
index 0000000..ab75603
--- /dev/null
+++ b/wiki.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Download the latest Wikipedia dump file
+#wget https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2
+
+# Extract the text from the dump file using native Bash commands
+bzcat enwiki-latest-pages-articles.xml.bz2 | \
+	sed -n '/<text/,/<\/text>/p' | \
+	sed -e 's/<[^>]*>//g' | \
+	sed '/^{{/d; /^$/d; /\t{{/d; s/|/ /g; s/=/ /g; /&.*;/d' | \
+	sed '/^\s*$/d; /^$/d' | \
+	sed 's/^[[:space:]]*//' | \
+	tr -d "'[]{}" > \
+	enwiki-latest-corpus.txt
+
+# Clean up
+#rm enwiki-latest-pages-articles.xml.bz2