forked from kubapok/paranormal-or-skeptic-ISI-public
zadanie
This commit is contained in:
parent
756ef4277a
commit
32b47206b9
323
.ipynb_checkpoints/solution-checkpoint.ipynb
Normal file
323
.ipynb_checkpoints/solution-checkpoint.ipynb
Normal file
@ -0,0 +1,323 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"import patoolib\n",
|
||||
"import os\n",
|
||||
"import patoolib\n",
|
||||
"from sklearn.preprocessing import LabelEncoder\n",
|
||||
"from sklearn.naive_bayes import GaussianNB, MultinomialNB\n",
|
||||
"from sklearn.pipeline import Pipeline\n",
|
||||
"from sklearn.feature_extraction.text import TfidfVectorizer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## TRENING"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### ROZPAKOWANIE I WCZYTANIE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"patool: Extracting train/in.tsv.xz ...\n",
|
||||
"patool: running \"\"C:\\Program Files\\Git\\mingw64\\bin\\xz.EXE\"\" -c -d -- train/in.tsv.xz > train/in.tsv\n",
|
||||
"patool: with shell=True\n",
|
||||
"patool: ... train/in.tsv.xz extracted to `train/'.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"EXPECTED_FILE = open('train/expected.tsv', 'r', encoding=\"utf-8\")\n",
|
||||
"\n",
|
||||
"patoolib.extract_archive(\"train/in.tsv.xz\", outdir=\"train/\")\n",
|
||||
"TRAIN = open('train/in.tsv', 'r', encoding=\"utf-8\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### WRZUCENIE DO ZMIENNYCH"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"EXPECTED = []\n",
|
||||
"for line in EXPECTED_FILE:\n",
|
||||
" EXPECTED.append(line)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"TRAIN_DATA = []\n",
|
||||
"for line in TRAIN:\n",
|
||||
" TRAIN_DATA.append(line)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### ZAMKNIECIE ZMIENNYCH PLIKOW I USUNIECIE ROZPAKOWANIA"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"EXPECTED_FILE.close()\n",
|
||||
"TRAIN.close()\n",
|
||||
"#os.remove(\"train/in.tsv\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### MODEL TRENINGOWY"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"EXPECTED_ENCODER = LabelEncoder().fit_transform(EXPECTED)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PIPE = Pipeline(steps=[(\"TF-IDF\",TfidfVectorizer()), (\"BAYES\", MultinomialNB())])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"TRAIN_MODEL = PIPE.fit(TRAIN_DATA, EXPECTED_ENCODER)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## FUNKCJE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def BayesFit(MODEL, DOC):\n",
|
||||
" PREDICTION = MODEL.predict(DOC)\n",
|
||||
" return PREDICTION"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## PLIK DEV-0"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"patool: Extracting dev-0/in.tsv.xz ...\n",
|
||||
"patool: running \"\"C:\\Program Files\\Git\\mingw64\\bin\\xz.EXE\"\" -c -d -- dev-0/in.tsv.xz > dev-0/in.tsv\n",
|
||||
"patool: with shell=True\n",
|
||||
"patool: ... dev-0/in.tsv.xz extracted to `dev-0/'.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"patoolib.extract_archive(\"dev-0/in.tsv.xz\", outdir=\"dev-0/\")\n",
|
||||
"INFILE = open('dev-0/in.tsv', 'r', encoding=\"utf-8\")\n",
|
||||
"\n",
|
||||
"OUTFILE = open(\"dev-0/out.tsv\", \"w\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"ALL_DOC = INFILE.readlines()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"RESULT = BayesFit(TRAIN_MODEL, ALL_DOC)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"for x in RESULT:\n",
|
||||
" OUTFILE.write(str(x) + '\\n')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"INFILE.close()\n",
|
||||
"OUTFILE.close()\n",
|
||||
"#os.remove(\"dev-0/in.tsv\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## PLIK TEST-A"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"patool: Extracting test-A/in.tsv.xz ...\n",
|
||||
"patool: running \"\"C:\\Program Files\\Git\\mingw64\\bin\\xz.EXE\"\" -c -d -- test-A/in.tsv.xz > test-A/in.tsv\n",
|
||||
"patool: with shell=True\n",
|
||||
"patool: ... test-A/in.tsv.xz extracted to `test-A/'.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"patoolib.extract_archive(\"test-A/in.tsv.xz\", outdir=\"test-A/\")\n",
|
||||
"INFILE = open('test-A/in.tsv', 'r', encoding=\"utf-8\")\n",
|
||||
"\n",
|
||||
"OUTFILE = open(\"test-A/out.tsv\", \"w\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"ALL_DOC = INFILE.readlines()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"RESULT = BayesFit(TRAIN_MODEL, ALL_DOC)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"for x in RESULT:\n",
|
||||
" OUTFILE.write(str(x) + '\\n')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"INFILE.close()\n",
|
||||
"OUTFILE.close()\n",
|
||||
"#os.remove(\"test-A/in.tsv\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.8.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
5272
dev-0/in.tsv
Normal file
5272
dev-0/in.tsv
Normal file
File diff suppressed because one or more lines are too long
5272
dev-0/out.tsv
Normal file
5272
dev-0/out.tsv
Normal file
File diff suppressed because it is too large
Load Diff
323
solution.ipynb
Normal file
323
solution.ipynb
Normal file
@ -0,0 +1,323 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"import patoolib\n",
|
||||
"import os\n",
|
||||
"import patoolib\n",
|
||||
"from sklearn.preprocessing import LabelEncoder\n",
|
||||
"from sklearn.naive_bayes import GaussianNB, MultinomialNB\n",
|
||||
"from sklearn.pipeline import Pipeline\n",
|
||||
"from sklearn.feature_extraction.text import TfidfVectorizer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## TRENING"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### ROZPAKOWANIE I WCZYTANIE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"patool: Extracting train/in.tsv.xz ...\n",
|
||||
"patool: running \"\"C:\\Program Files\\Git\\mingw64\\bin\\xz.EXE\"\" -c -d -- train/in.tsv.xz > train/in.tsv\n",
|
||||
"patool: with shell=True\n",
|
||||
"patool: ... train/in.tsv.xz extracted to `train/'.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"EXPECTED_FILE = open('train/expected.tsv', 'r', encoding=\"utf-8\")\n",
|
||||
"\n",
|
||||
"patoolib.extract_archive(\"train/in.tsv.xz\", outdir=\"train/\")\n",
|
||||
"TRAIN = open('train/in.tsv', 'r', encoding=\"utf-8\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### WRZUCENIE DO ZMIENNYCH"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"EXPECTED = []\n",
|
||||
"for line in EXPECTED_FILE:\n",
|
||||
" EXPECTED.append(line)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"TRAIN_DATA = []\n",
|
||||
"for line in TRAIN:\n",
|
||||
" TRAIN_DATA.append(line)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### ZAMKNIECIE ZMIENNYCH PLIKOW I USUNIECIE ROZPAKOWANIA"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"EXPECTED_FILE.close()\n",
|
||||
"TRAIN.close()\n",
|
||||
"#os.remove(\"train/in.tsv\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### MODEL TRENINGOWY"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"EXPECTED_ENCODER = LabelEncoder().fit_transform(EXPECTED)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PIPE = Pipeline(steps=[(\"TF-IDF\",TfidfVectorizer()), (\"BAYES\", MultinomialNB())])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"TRAIN_MODEL = PIPE.fit(TRAIN_DATA, EXPECTED_ENCODER)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## FUNKCJE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def BayesFit(MODEL, DOC):\n",
|
||||
" PREDICTION = MODEL.predict(DOC)\n",
|
||||
" return PREDICTION"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## PLIK DEV-0"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"patool: Extracting dev-0/in.tsv.xz ...\n",
|
||||
"patool: running \"\"C:\\Program Files\\Git\\mingw64\\bin\\xz.EXE\"\" -c -d -- dev-0/in.tsv.xz > dev-0/in.tsv\n",
|
||||
"patool: with shell=True\n",
|
||||
"patool: ... dev-0/in.tsv.xz extracted to `dev-0/'.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"patoolib.extract_archive(\"dev-0/in.tsv.xz\", outdir=\"dev-0/\")\n",
|
||||
"INFILE = open('dev-0/in.tsv', 'r', encoding=\"utf-8\")\n",
|
||||
"\n",
|
||||
"OUTFILE = open(\"dev-0/out.tsv\", \"w\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"ALL_DOC = INFILE.readlines()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"RESULT = BayesFit(TRAIN_MODEL, ALL_DOC)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"for x in RESULT:\n",
|
||||
" OUTFILE.write(str(x) + '\\n')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"INFILE.close()\n",
|
||||
"OUTFILE.close()\n",
|
||||
"#os.remove(\"dev-0/in.tsv\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## PLIK TEST-A"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"patool: Extracting test-A/in.tsv.xz ...\n",
|
||||
"patool: running \"\"C:\\Program Files\\Git\\mingw64\\bin\\xz.EXE\"\" -c -d -- test-A/in.tsv.xz > test-A/in.tsv\n",
|
||||
"patool: with shell=True\n",
|
||||
"patool: ... test-A/in.tsv.xz extracted to `test-A/'.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"patoolib.extract_archive(\"test-A/in.tsv.xz\", outdir=\"test-A/\")\n",
|
||||
"INFILE = open('test-A/in.tsv', 'r', encoding=\"utf-8\")\n",
|
||||
"\n",
|
||||
"OUTFILE = open(\"test-A/out.tsv\", \"w\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"ALL_DOC = INFILE.readlines()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"RESULT = BayesFit(TRAIN_MODEL, ALL_DOC)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"for x in RESULT:\n",
|
||||
" OUTFILE.write(str(x) + '\\n')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"INFILE.close()\n",
|
||||
"OUTFILE.close()\n",
|
||||
"#os.remove(\"test-A/in.tsv\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.8.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
169
solution.py
Normal file
169
solution.py
Normal file
@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[1]:
|
||||
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import patoolib
|
||||
import os
|
||||
import patoolib
|
||||
from sklearn.preprocessing import LabelEncoder
|
||||
from sklearn.naive_bayes import GaussianNB, MultinomialNB
|
||||
from sklearn.pipeline import Pipeline
|
||||
from sklearn.feature_extraction.text import TfidfVectorizer
|
||||
|
||||
|
||||
# ## TRENING
|
||||
|
||||
# #### ROZPAKOWANIE I WCZYTANIE
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
EXPECTED_FILE = open('train/expected.tsv', 'r', encoding="utf-8")
|
||||
|
||||
patoolib.extract_archive("train/in.tsv.xz", outdir="train/")
|
||||
TRAIN = open('train/in.tsv', 'r', encoding="utf-8")
|
||||
|
||||
|
||||
# #### WRZUCENIE DO ZMIENNYCH
|
||||
|
||||
# In[3]:
|
||||
|
||||
|
||||
EXPECTED = []
|
||||
for line in EXPECTED_FILE:
|
||||
EXPECTED.append(line)
|
||||
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
TRAIN_DATA = []
|
||||
for line in TRAIN:
|
||||
TRAIN_DATA.append(line)
|
||||
|
||||
|
||||
# #### ZAMKNIECIE ZMIENNYCH PLIKOW I USUNIECIE ROZPAKOWANIA
|
||||
|
||||
# In[5]:
|
||||
|
||||
|
||||
EXPECTED_FILE.close()
|
||||
TRAIN.close()
|
||||
#os.remove("train/in.tsv")
|
||||
|
||||
|
||||
# #### MODEL TRENINGOWY
|
||||
|
||||
# In[6]:
|
||||
|
||||
|
||||
EXPECTED_ENCODER = LabelEncoder().fit_transform(EXPECTED)
|
||||
|
||||
|
||||
# In[7]:
|
||||
|
||||
|
||||
PIPE = Pipeline(steps=[("TF-IDF",TfidfVectorizer()), ("BAYES", MultinomialNB())])
|
||||
|
||||
|
||||
# In[8]:
|
||||
|
||||
|
||||
TRAIN_MODEL = PIPE.fit(TRAIN_DATA, EXPECTED_ENCODER)
|
||||
|
||||
|
||||
# ## FUNKCJE
|
||||
|
||||
# In[9]:
|
||||
|
||||
|
||||
def BayesFit(MODEL, DOC):
|
||||
PREDICTION = MODEL.predict(DOC)
|
||||
return PREDICTION
|
||||
|
||||
|
||||
# ## PLIK DEV-0
|
||||
|
||||
# In[10]:
|
||||
|
||||
|
||||
patoolib.extract_archive("dev-0/in.tsv.xz", outdir="dev-0/")
|
||||
INFILE = open('dev-0/in.tsv', 'r', encoding="utf-8")
|
||||
|
||||
OUTFILE = open("dev-0/out.tsv", "w")
|
||||
|
||||
|
||||
# In[11]:
|
||||
|
||||
|
||||
ALL_DOC = INFILE.readlines()
|
||||
|
||||
|
||||
# In[12]:
|
||||
|
||||
|
||||
RESULT = BayesFit(TRAIN_MODEL, ALL_DOC)
|
||||
|
||||
|
||||
# In[13]:
|
||||
|
||||
|
||||
for x in RESULT:
|
||||
OUTFILE.write(str(x) + '\n')
|
||||
|
||||
|
||||
# In[14]:
|
||||
|
||||
|
||||
INFILE.close()
|
||||
OUTFILE.close()
|
||||
#os.remove("dev-0/in.tsv")
|
||||
|
||||
|
||||
# ## PLIK TEST-A
|
||||
|
||||
# In[15]:
|
||||
|
||||
|
||||
patoolib.extract_archive("test-A/in.tsv.xz", outdir="test-A/")
|
||||
INFILE = open('test-A/in.tsv', 'r', encoding="utf-8")
|
||||
|
||||
OUTFILE = open("test-A/out.tsv", "w")
|
||||
|
||||
|
||||
# In[16]:
|
||||
|
||||
|
||||
ALL_DOC = INFILE.readlines()
|
||||
|
||||
|
||||
# In[17]:
|
||||
|
||||
|
||||
RESULT = BayesFit(TRAIN_MODEL, ALL_DOC)
|
||||
|
||||
|
||||
# In[18]:
|
||||
|
||||
|
||||
for x in RESULT:
|
||||
OUTFILE.write(str(x) + '\n')
|
||||
|
||||
|
||||
# In[19]:
|
||||
|
||||
|
||||
INFILE.close()
|
||||
OUTFILE.close()
|
||||
#os.remove("test-A/in.tsv")
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
|
||||
|
||||
5152
test-A/in.tsv
Normal file
5152
test-A/in.tsv
Normal file
File diff suppressed because one or more lines are too long
5152
test-A/out.tsv
Normal file
5152
test-A/out.tsv
Normal file
File diff suppressed because it is too large
Load Diff
289579
train/in.tsv
Normal file
289579
train/in.tsv
Normal file
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue
Block a user