retroc2/.ipynb_checkpoints/run-checkpoint.ipynb
2022-05-18 01:10:51 +02:00

190 lines
4.6 KiB
Plaintext

{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "d103a6c5-a9b4-4547-9e10-f384d716972d",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import pandas as pd\n",
"import numpy as np\n",
"import sklearn\n",
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
"from sklearn.linear_model import LinearRegression\n",
"from sklearn.metrics import mean_squared_error\n",
"from sklearn.pipeline import make_pipeline"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "6a2785e6-36b0-4649-91d1-aea8fd3599c1",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"D:\\Programy\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3444: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version.\n",
"\n",
"\n",
" exec(code_obj, self.user_global_ns, self.user_ns)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"107463\n"
]
}
],
"source": [
"train = pd.read_csv('train/train.tsv', header=None, sep='\\t', error_bad_lines=False)\n",
"print(len(train))\n",
"train = train[:30000]"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "8cc00b89-1007-4c4a-8ba7-c62b57459b79",
"metadata": {},
"outputs": [],
"source": [
"x_train = train[4]\n",
"y_train = train[0]"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "0a1cce75-86a1-4f76-9416-e876e01699e3",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"Pipeline(steps=[('tfidfvectorizer', TfidfVectorizer()),\n",
" ('linearregression', LinearRegression())])"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model = make_pipeline(TfidfVectorizer(), LinearRegression())\n",
"model.fit(x_train, y_train)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "2fd18dfa-0dba-460b-a56d-21793baa7124",
"metadata": {},
"outputs": [],
"source": [
"def readFile(filename):\n",
" result = []\n",
" with open(filename, 'r', encoding=\"utf-8\") as file:\n",
" for line in file:\n",
" text = line.split(\"\\t\")[0].strip()\n",
" result.append(text)\n",
" return result"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "ce918d1f-2b8d-432c-be19-3a4966062d35",
"metadata": {},
"outputs": [],
"source": [
"x_dev0 = readFile('dev-0/in.tsv')\n",
"dev_predicted = model.predict(x_dev0)\n",
"with open('dev-0/out.tsv', 'wt') as f:\n",
" for i in dev_predicted:\n",
" f.write(str(i)+'\\n')"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "66e4c057-6a76-4d05-ad60-faa09381fdb1",
"metadata": {},
"outputs": [],
"source": [
"x_dev1 = readFile('dev-1/in.tsv')\n",
"dev_predicted = model.predict(x_dev1)\n",
"with open('dev-1/out.tsv', 'wt') as f:\n",
" for i in dev_predicted:\n",
" f.write(str(i)+'\\n')"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "3bc8418b-64f1-4163-a0ec-8e3293032341",
"metadata": {},
"outputs": [],
"source": [
"with open('test-A/in.tsv', 'r', encoding = 'utf-8') as f:\n",
" x_test = f.readlines()\n",
" \n",
"# x_test = pd.Series(x_test)\n",
"# x_test = vectorizer.transform(x_test)\n",
"\n",
"test_predicted = model.predict(x_test)\n",
"\n",
"with open('test-A/out.tsv', 'wt') as f:\n",
" for i in test_predicted:\n",
" f.write(str(i)+'\\n')"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "a18aea56-7fa1-40bd-8aa3-bbaf9d66d6b7",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"[NbConvertApp] Converting notebook run.ipynb to script\n",
"[NbConvertApp] Writing 1597 bytes to run.py\n"
]
}
],
"source": [
"!jupyter nbconvert --to script run.ipynb"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.7"
}
},
"nbformat": 4,
"nbformat_minor": 5
}