s416089/paranormal-or-skeptic-ISI-public-s416089
1
0
Fork 0
forked from kubapok/paranormal-or-skeptic-ISI-public
Code Issues Pull Requests Projects Releases Wiki Activity

zadanie

Browse Source
This commit is contained in:
Jedrzej Klepacki 2021-05-02 16:26:07 +02:00
parent 756ef4277a
commit 32b47206b9
9 changed files with 311242 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

323
.ipynb_checkpoints/solution-checkpoint.ipynb Normal file
View File

@ -0,0 +1,323 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"import patoolib\n",
"import os\n",
"import patoolib\n",
"from sklearn.preprocessing import LabelEncoder\n",
"from sklearn.naive_bayes import GaussianNB, MultinomialNB\n",
"from sklearn.pipeline import Pipeline\n",
"from sklearn.feature_extraction.text import TfidfVectorizer"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## TRENING"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### ROZPAKOWANIE I WCZYTANIE"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"patool: Extracting train/in.tsv.xz ...\n",
"patool: running \"\"C:\\Program Files\\Git\\mingw64\\bin\\xz.EXE\"\" -c -d -- train/in.tsv.xz > train/in.tsv\n",
"patool: with shell=True\n",
"patool: ... train/in.tsv.xz extracted to `train/'.\n"
]
}
],
"source": [
"EXPECTED_FILE = open('train/expected.tsv', 'r', encoding=\"utf-8\")\n",
"\n",
"patoolib.extract_archive(\"train/in.tsv.xz\", outdir=\"train/\")\n",
"TRAIN = open('train/in.tsv', 'r', encoding=\"utf-8\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### WRZUCENIE DO ZMIENNYCH"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"EXPECTED = []\n",
"for line in EXPECTED_FILE:\n",
" EXPECTED.append(line)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"TRAIN_DATA = []\n",
"for line in TRAIN:\n",
" TRAIN_DATA.append(line)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### ZAMKNIECIE ZMIENNYCH PLIKOW I USUNIECIE ROZPAKOWANIA"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"EXPECTED_FILE.close()\n",
"TRAIN.close()\n",
"#os.remove(\"train/in.tsv\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### MODEL TRENINGOWY"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"EXPECTED_ENCODER = LabelEncoder().fit_transform(EXPECTED)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"PIPE = Pipeline(steps=[(\"TF-IDF\",TfidfVectorizer()), (\"BAYES\", MultinomialNB())])"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"TRAIN_MODEL = PIPE.fit(TRAIN_DATA, EXPECTED_ENCODER)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## FUNKCJE"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"def BayesFit(MODEL, DOC):\n",
" PREDICTION = MODEL.predict(DOC)\n",
" return PREDICTION"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## PLIK DEV-0"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"patool: Extracting dev-0/in.tsv.xz ...\n",
"patool: running \"\"C:\\Program Files\\Git\\mingw64\\bin\\xz.EXE\"\" -c -d -- dev-0/in.tsv.xz > dev-0/in.tsv\n",
"patool: with shell=True\n",
"patool: ... dev-0/in.tsv.xz extracted to `dev-0/'.\n"
]
}
],
"source": [
"patoolib.extract_archive(\"dev-0/in.tsv.xz\", outdir=\"dev-0/\")\n",
"INFILE = open('dev-0/in.tsv', 'r', encoding=\"utf-8\")\n",
"\n",
"OUTFILE = open(\"dev-0/out.tsv\", \"w\")"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"ALL_DOC = INFILE.readlines()"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"RESULT = BayesFit(TRAIN_MODEL, ALL_DOC)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"for x in RESULT:\n",
" OUTFILE.write(str(x) + '\\n')"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"INFILE.close()\n",
"OUTFILE.close()\n",
"#os.remove(\"dev-0/in.tsv\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## PLIK TEST-A"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"patool: Extracting test-A/in.tsv.xz ...\n",
"patool: running \"\"C:\\Program Files\\Git\\mingw64\\bin\\xz.EXE\"\" -c -d -- test-A/in.tsv.xz > test-A/in.tsv\n",
"patool: with shell=True\n",
"patool: ... test-A/in.tsv.xz extracted to `test-A/'.\n"
]
}
],
"source": [
"patoolib.extract_archive(\"test-A/in.tsv.xz\", outdir=\"test-A/\")\n",
"INFILE = open('test-A/in.tsv', 'r', encoding=\"utf-8\")\n",
"\n",
"OUTFILE = open(\"test-A/out.tsv\", \"w\")"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"ALL_DOC = INFILE.readlines()"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"RESULT = BayesFit(TRAIN_MODEL, ALL_DOC)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"for x in RESULT:\n",
" OUTFILE.write(str(x) + '\\n')"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
"INFILE.close()\n",
"OUTFILE.close()\n",
"#os.remove(\"test-A/in.tsv\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

5272
dev-0/in.tsv Normal file
View File

File diff suppressed because one or more lines are too long

5272
dev-0/out.tsv Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
geval Normal file
View File

Binary file not shown.

323
solution.ipynb Normal file
View File

@ -0,0 +1,323 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"import patoolib\n",
"import os\n",
"import patoolib\n",
"from sklearn.preprocessing import LabelEncoder\n",
"from sklearn.naive_bayes import GaussianNB, MultinomialNB\n",
"from sklearn.pipeline import Pipeline\n",
"from sklearn.feature_extraction.text import TfidfVectorizer"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## TRENING"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### ROZPAKOWANIE I WCZYTANIE"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"patool: Extracting train/in.tsv.xz ...\n",
"patool: running \"\"C:\\Program Files\\Git\\mingw64\\bin\\xz.EXE\"\" -c -d -- train/in.tsv.xz > train/in.tsv\n",
"patool: with shell=True\n",
"patool: ... train/in.tsv.xz extracted to `train/'.\n"
]
}
],
"source": [
"EXPECTED_FILE = open('train/expected.tsv', 'r', encoding=\"utf-8\")\n",
"\n",
"patoolib.extract_archive(\"train/in.tsv.xz\", outdir=\"train/\")\n",
"TRAIN = open('train/in.tsv', 'r', encoding=\"utf-8\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### WRZUCENIE DO ZMIENNYCH"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"EXPECTED = []\n",
"for line in EXPECTED_FILE:\n",
" EXPECTED.append(line)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"TRAIN_DATA = []\n",
"for line in TRAIN:\n",
" TRAIN_DATA.append(line)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### ZAMKNIECIE ZMIENNYCH PLIKOW I USUNIECIE ROZPAKOWANIA"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"EXPECTED_FILE.close()\n",
"TRAIN.close()\n",
"#os.remove(\"train/in.tsv\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### MODEL TRENINGOWY"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"EXPECTED_ENCODER = LabelEncoder().fit_transform(EXPECTED)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"PIPE = Pipeline(steps=[(\"TF-IDF\",TfidfVectorizer()), (\"BAYES\", MultinomialNB())])"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"TRAIN_MODEL = PIPE.fit(TRAIN_DATA, EXPECTED_ENCODER)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## FUNKCJE"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"def BayesFit(MODEL, DOC):\n",
" PREDICTION = MODEL.predict(DOC)\n",
" return PREDICTION"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## PLIK DEV-0"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"patool: Extracting dev-0/in.tsv.xz ...\n",
"patool: running \"\"C:\\Program Files\\Git\\mingw64\\bin\\xz.EXE\"\" -c -d -- dev-0/in.tsv.xz > dev-0/in.tsv\n",
"patool: with shell=True\n",
"patool: ... dev-0/in.tsv.xz extracted to `dev-0/'.\n"
]
}
],
"source": [
"patoolib.extract_archive(\"dev-0/in.tsv.xz\", outdir=\"dev-0/\")\n",
"INFILE = open('dev-0/in.tsv', 'r', encoding=\"utf-8\")\n",
"\n",
"OUTFILE = open(\"dev-0/out.tsv\", \"w\")"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"ALL_DOC = INFILE.readlines()"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"RESULT = BayesFit(TRAIN_MODEL, ALL_DOC)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"for x in RESULT:\n",
" OUTFILE.write(str(x) + '\\n')"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"INFILE.close()\n",
"OUTFILE.close()\n",
"#os.remove(\"dev-0/in.tsv\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## PLIK TEST-A"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"patool: Extracting test-A/in.tsv.xz ...\n",
"patool: running \"\"C:\\Program Files\\Git\\mingw64\\bin\\xz.EXE\"\" -c -d -- test-A/in.tsv.xz > test-A/in.tsv\n",
"patool: with shell=True\n",
"patool: ... test-A/in.tsv.xz extracted to `test-A/'.\n"
]
}
],
"source": [
"patoolib.extract_archive(\"test-A/in.tsv.xz\", outdir=\"test-A/\")\n",
"INFILE = open('test-A/in.tsv', 'r', encoding=\"utf-8\")\n",
"\n",
"OUTFILE = open(\"test-A/out.tsv\", \"w\")"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"ALL_DOC = INFILE.readlines()"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"RESULT = BayesFit(TRAIN_MODEL, ALL_DOC)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"for x in RESULT:\n",
" OUTFILE.write(str(x) + '\\n')"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
"INFILE.close()\n",
"OUTFILE.close()\n",
"#os.remove(\"test-A/in.tsv\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

169
solution.py Normal file
View File

@ -0,0 +1,169 @@
#!/usr/bin/env python
# coding: utf-8
# In[1]:
import numpy as np
import pandas as pd
import patoolib
import os
import patoolib
from sklearn.preprocessing import LabelEncoder
from sklearn.naive_bayes import GaussianNB, MultinomialNB
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import TfidfVectorizer
# ## TRENING
# #### ROZPAKOWANIE I WCZYTANIE
# In[2]:
EXPECTED_FILE = open('train/expected.tsv', 'r', encoding="utf-8")
patoolib.extract_archive("train/in.tsv.xz", outdir="train/")
TRAIN = open('train/in.tsv', 'r', encoding="utf-8")
# #### WRZUCENIE DO ZMIENNYCH
# In[3]:
EXPECTED = []
for line in EXPECTED_FILE:
EXPECTED.append(line)
# In[4]:
TRAIN_DATA = []
for line in TRAIN:
TRAIN_DATA.append(line)
# #### ZAMKNIECIE ZMIENNYCH PLIKOW I USUNIECIE ROZPAKOWANIA
# In[5]:
EXPECTED_FILE.close()
TRAIN.close()
#os.remove("train/in.tsv")
# #### MODEL TRENINGOWY
# In[6]:
EXPECTED_ENCODER = LabelEncoder().fit_transform(EXPECTED)
# In[7]:
PIPE = Pipeline(steps=[("TF-IDF",TfidfVectorizer()), ("BAYES", MultinomialNB())])
# In[8]:
TRAIN_MODEL = PIPE.fit(TRAIN_DATA, EXPECTED_ENCODER)
# ## FUNKCJE
# In[9]:
def BayesFit(MODEL, DOC):
PREDICTION = MODEL.predict(DOC)
return PREDICTION
# ## PLIK DEV-0
# In[10]:
patoolib.extract_archive("dev-0/in.tsv.xz", outdir="dev-0/")
INFILE = open('dev-0/in.tsv', 'r', encoding="utf-8")
OUTFILE = open("dev-0/out.tsv", "w")
# In[11]:
ALL_DOC = INFILE.readlines()
# In[12]:
RESULT = BayesFit(TRAIN_MODEL, ALL_DOC)
# In[13]:
for x in RESULT:
OUTFILE.write(str(x) + '\n')
# In[14]:
INFILE.close()
OUTFILE.close()
#os.remove("dev-0/in.tsv")
# ## PLIK TEST-A
# In[15]:
patoolib.extract_archive("test-A/in.tsv.xz", outdir="test-A/")
INFILE = open('test-A/in.tsv', 'r', encoding="utf-8")
OUTFILE = open("test-A/out.tsv", "w")
# In[16]:
ALL_DOC = INFILE.readlines()
# In[17]:
RESULT = BayesFit(TRAIN_MODEL, ALL_DOC)
# In[18]:
for x in RESULT:
OUTFILE.write(str(x) + '\n')
# In[19]:
INFILE.close()
OUTFILE.close()
#os.remove("test-A/in.tsv")
# In[ ]:

5152
test-A/in.tsv Normal file
View File

File diff suppressed because one or more lines are too long

5152
test-A/out.tsv Normal file
View File

File diff suppressed because it is too large Load Diff

289579
train/in.tsv Normal file
View File

File diff suppressed because one or more lines are too long