retroc2/.ipynb_checkpoints/run-checkpoint.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d103a6c5-a9b4-4547-9e10-f384d716972d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import sklearn\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.pipeline import make_pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6a2785e6-36b0-4649-91d1-aea8fd3599c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Programy\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3444: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version.\n",
      "\n",
      "\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "107463\n"
     ]
    }
   ],
   "source": [
    "train = pd.read_csv('train/train.tsv', header=None, sep='\\t', error_bad_lines=False)\n",
    "print(len(train))\n",
    "train = train.head(30000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8cc00b89-1007-4c4a-8ba7-c62b57459b79",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = train[4]\n",
    "y_train = train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dd454ce5-a06e-4fbd-a546-83fb94ad0390",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_dev_data = pd.read_csv('dev-0/in.tsv', header=None, sep='\\t')\n",
    "x_dev = x_dev_data[0]\n",
    "x_dev[19999] = \"to jest tekst testowy\"\n",
    "x_dev[20000] = \"a ten tekst jest najbardziej testowy\"\n",
    "y_dev = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0a1cce75-86a1-4f76-9416-e876e01699e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('tfidfvectorizer', TfidfVectorizer()),\n",
       "                ('linearregression', LinearRegression())])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = make_pipeline(TfidfVectorizer(), LinearRegression())\n",
    "model.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cc1270d5-29dc-4f03-82c1-dc03f3e4fa00",
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_predicted =  model.predict(x_dev)\n",
    "\n",
    "with open('dev-0/out.tsv', 'wt') as f:\n",
    "    for i in dev_predicted:\n",
    "        f.write(str(i)+'\\n')\n",
    "\n",
    "dev_out = pd.read_csv('dev-0/out.tsv', header=None, sep='\\t')\n",
    "dev_expected = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "223de995-5e91-4254-9214-4fc871c985e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4261.093474053155\n"
     ]
    }
   ],
   "source": [
    "print(mean_squared_error(dev_out, dev_expected))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3bc8418b-64f1-4163-a0ec-8e3293032341",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('test-A/in.tsv', 'r', encoding = 'utf-8') as f:\n",
    "       x_test = f.readlines()\n",
    "        \n",
    "# x_test = pd.Series(x_test)\n",
    "# x_test = vectorizer.transform(x_test)\n",
    "\n",
    "test_predicted = model.predict(x_test)\n",
    "\n",
    "with open('test-A/out.tsv', 'wt') as f:\n",
    "    for i in test_predicted:\n",
    "        f.write(str(i)+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a18aea56-7fa1-40bd-8aa3-bbaf9d66d6b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[NbConvertApp] Converting notebook run.ipynb to script\n",
      "[NbConvertApp] Writing 1607 bytes to run.py\n"
     ]
    }
   ],
   "source": [
    "!jupyter nbconvert --to script run.ipynb"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}