127 lines
3.3 KiB
Plaintext
127 lines
3.3 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"id": "5fcb7312",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from sklearn.pipeline import make_pipeline\n",
|
|
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
|
|
"from sklearn.naive_bayes import MultinomialNB\n",
|
|
"import pandas as pd\n",
|
|
"import csv\n",
|
|
"import numpy as np\n",
|
|
"from sklearn.preprocessing import LabelEncoder"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"id": "88ac1be8",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"steps = make_pipeline(TfidfVectorizer(),MultinomialNB())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 14,
|
|
"id": "4aa43416",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#training\n",
|
|
"all_train_data_in = pd.read_csv('train/in.tsv.xz', compression='xz', header=None, error_bad_lines=False, quoting=csv.QUOTE_NONE, sep='\\t', nrows=3000)\n",
|
|
"train_data_ex = pd.read_csv('train/expected.tsv', header=None, error_bad_lines=False, quoting=csv.QUOTE_NONE, sep='\\t', nrows=3000)\n",
|
|
"train_data_in = []\n",
|
|
"for value in all_train_data_in.values:\n",
|
|
" temp = \"\"\n",
|
|
" for el in value:\n",
|
|
" if(temp == \"\"):\n",
|
|
" temp = str(el)\n",
|
|
" else:\n",
|
|
" temp += '\\t' + str(el)\n",
|
|
" train_data_in.append(temp)\n",
|
|
" \n",
|
|
"nb=steps.fit(train_data_in, LabelEncoder().fit_transform(train_data_ex.values))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 17,
|
|
"id": "15c47c24",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#dev0\n",
|
|
"all_dev0_data = pd.read_csv('dev-0/in.tsv.xz', compression='xz', header=None, quoting=csv.QUOTE_NONE, sep='\\t')\n",
|
|
"dev0_data = []\n",
|
|
"for value in all_dev0_data.values:\n",
|
|
" temp = \"\"\n",
|
|
" for el in value:\n",
|
|
" if(temp == \"\"):\n",
|
|
" temp = str(el)\n",
|
|
" else:\n",
|
|
" temp += '\\t' + str(el)\n",
|
|
" dev0_data.append(temp)\n",
|
|
"\n",
|
|
"\n",
|
|
"dev0_y = nb.predict(dev0_data)\n",
|
|
"\n",
|
|
"#zapis wyników\n",
|
|
"dev0_y.tofile('dev-0/out.tsv', sep='\\n')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 16,
|
|
"id": "822b1e29",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#test-A\n",
|
|
"all_testA_data = pd.read_csv('test-A/in.tsv.xz', compression='xz', header=None, quoting=csv.QUOTE_NONE, sep='\\t')\n",
|
|
"testA_data = []\n",
|
|
"for value in all_testA_data.values:\n",
|
|
" temp = \"\"\n",
|
|
" for el in value:\n",
|
|
" if(temp == \"\"):\n",
|
|
" temp = str(el)\n",
|
|
" else:\n",
|
|
" temp += '\\t' + str(el)\n",
|
|
" testA_data.append(temp)\n",
|
|
"\n",
|
|
"\n",
|
|
"testA_y = nb.predict(testA_data)\n",
|
|
"\n",
|
|
"#zapis wyników\n",
|
|
"testA_y.tofile('test-A/out.tsv', sep='\\n')"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.8.8"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|