20k train data

.ipynb_checkpoints/run-checkpoint.ipynb

@@ -43,7 +43,7 @@
    "source": [
     "train = pd.read_csv('train/train.tsv', header=None, sep='\\t', error_bad_lines=False)\n",
     "print(len(train))\n",
-    "train = train.head(1000)"
+    "train = train.head(2000)"
    ]
   },
   {
@@ -128,7 +128,21 @@
    "execution_count": 9,
    "id": "cc1270d5-29dc-4f03-82c1-dc03f3e4fa00",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "MemoryError",
+     "evalue": "Unable to allocate 32.2 GiB for an array with shape (20000, 216394) and data type float64",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
+      "\u001b[1;32mC:\\Users\\SEBAST~1\\AppData\\Local\\Temp/ipykernel_17784/3948937349.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdev_predicted\u001b[0m \u001b[1;33m=\u001b[0m  \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx_dev\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'dev-0/out.tsv'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'wt'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdev_predicted\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m         \u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;34m'\\n'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[1;32mD:\\Programy\\anaconda3\\lib\\site-packages\\scipy\\sparse\\compressed.py\u001b[0m in \u001b[0;36mtoarray\u001b[1;34m(self, order, out)\u001b[0m\n\u001b[0;32m   1029\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mout\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0morder\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1030\u001b[0m             \u001b[0morder\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_swap\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'cf'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1031\u001b[1;33m         \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_process_toarray_args\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1032\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflags\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mc_contiguous\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflags\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mf_contiguous\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1033\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Output array must be C or F contiguous'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[1;32mD:\\Programy\\anaconda3\\lib\\site-packages\\scipy\\sparse\\base.py\u001b[0m in \u001b[0;36m_process_toarray_args\u001b[1;34m(self, order, out)\u001b[0m\n\u001b[0;32m   1200\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1201\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1202\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1203\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1204\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[1;31mMemoryError\u001b[0m: Unable to allocate 32.2 GiB for an array with shape (20000, 216394) and data type float64"
+     ]
+    }
+   ],
    "source": [
     "dev_predicted =  model.predict(x_dev.toarray())\n",
     "\n",
@@ -142,25 +156,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
    "id": "223de995-5e91-4254-9214-4fc871c985e9",
    "metadata": {},
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "3486.2683285642797\n"
-     ]
-    }
-   ],
    "source": [
     "print(mean_squared_error(dev_out, dev_expected))"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
    "id": "3bc8418b-64f1-4163-a0ec-8e3293032341",
    "metadata": {},
    "outputs": [],
@@ -180,19 +186,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
    "id": "a18aea56-7fa1-40bd-8aa3-bbaf9d66d6b7",
    "metadata": {},
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "[NbConvertApp] Converting notebook run.ipynb to script\n",
-      "[NbConvertApp] Writing 1629 bytes to run.py\n"
-     ]
-    }
-   ],
    "source": [
     "!jupyter nbconvert --to script run.ipynb"
    ]

.ipynb_checkpoints/run-rr-checkpoint.ipynb

@@ -0,0 +1,219 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "d103a6c5-a9b4-4547-9e10-f384d716972d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import sklearn\n",
+    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
+    "from sklearn.linear_model import LinearRegression\n",
+    "from sklearn.metrics import mean_squared_error"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "6a2785e6-36b0-4649-91d1-aea8fd3599c1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "D:\\Programy\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3444: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version.\n",
+      "\n",
+      "\n",
+      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "107463\n"
+     ]
+    }
+   ],
+   "source": [
+    "train = pd.read_csv('train/train.tsv', header=None, sep='\\t', error_bad_lines=False)\n",
+    "print(len(train))\n",
+    "train = train.head(1500)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "8cc00b89-1007-4c4a-8ba7-c62b57459b79",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "x_train = train[4]\n",
+    "y_train = train[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "dd454ce5-a06e-4fbd-a546-83fb94ad0390",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "x_dev_data = pd.read_csv('dev-0/in.tsv', header=None, sep='\\t')\n",
+    "x_dev = x_dev_data[0]\n",
+    "x_dev[19999] = \"to jest tekst testowy\"\n",
+    "x_dev[20000] = \"a ten tekst jest najbardziej testowy\"\n",
+    "y_dev = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "79099730-c5bd-4c5c-a0b0-788512d44226",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vectorizer = TfidfVectorizer()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "0a1cce75-86a1-4f76-9416-e876e01699e3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "x_train = vectorizer.fit_transform(x_train)\n",
+    "x_dev = vectorizer.transform(x_dev)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "ef405093-6b4c-4558-add4-40bd0ced244e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = LinearRegression()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "4354553c-6143-43c7-8845-3b2327819481",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "LinearRegression()"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model.fit(x_train.toarray(), y_train)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "cc1270d5-29dc-4f03-82c1-dc03f3e4fa00",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "MemoryError",
+     "evalue": "Unable to allocate 20.6 GiB for an array with shape (20000, 138052) and data type float64",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
+      "\u001b[1;32mC:\\Users\\SEBAST~1\\AppData\\Local\\Temp/ipykernel_7064/3948937349.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdev_predicted\u001b[0m \u001b[1;33m=\u001b[0m  \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx_dev\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'dev-0/out.tsv'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'wt'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdev_predicted\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m         \u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;34m'\\n'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[1;32mD:\\Programy\\anaconda3\\lib\\site-packages\\scipy\\sparse\\compressed.py\u001b[0m in \u001b[0;36mtoarray\u001b[1;34m(self, order, out)\u001b[0m\n\u001b[0;32m   1029\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mout\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0morder\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1030\u001b[0m             \u001b[0morder\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_swap\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'cf'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1031\u001b[1;33m         \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_process_toarray_args\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1032\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflags\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mc_contiguous\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflags\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mf_contiguous\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1033\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Output array must be C or F contiguous'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[1;32mD:\\Programy\\anaconda3\\lib\\site-packages\\scipy\\sparse\\base.py\u001b[0m in \u001b[0;36m_process_toarray_args\u001b[1;34m(self, order, out)\u001b[0m\n\u001b[0;32m   1200\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1201\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1202\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1203\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1204\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[1;31mMemoryError\u001b[0m: Unable to allocate 20.6 GiB for an array with shape (20000, 138052) and data type float64"
+     ]
+    }
+   ],
+   "source": [
+    "dev_predicted =  model.predict(x_dev.toarray())\n",
+    "\n",
+    "with open('dev-0/out.tsv', 'wt') as f:\n",
+    "    for i in dev_predicted:\n",
+    "        f.write(str(i)+'\\n')\n",
+    "\n",
+    "dev_out = pd.read_csv('dev-0/out.tsv', header=None, sep='\\t')\n",
+    "dev_expected = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "223de995-5e91-4254-9214-4fc871c985e9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(mean_squared_error(dev_out, dev_expected))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3bc8418b-64f1-4163-a0ec-8e3293032341",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open('test-A/in.tsv', 'r', encoding = 'utf-8') as f:\n",
+    "       x_test = f.readlines()\n",
+    "        \n",
+    "x_test = pd.Series(x_test)\n",
+    "x_test = vectorizer.transform(x_test)\n",
+    "\n",
+    "test_predicted = model.predict(x_test.toarray())\n",
+    "\n",
+    "with open('test-A/out.tsv', 'wt') as f:\n",
+    "    for i in test_predicted:\n",
+    "        f.write(str(i)+'\\n')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a18aea56-7fa1-40bd-8aa3-bbaf9d66d6b7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!jupyter nbconvert --to script run.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

run-rr.ipynb

@@ -0,0 +1,219 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "d103a6c5-a9b4-4547-9e10-f384d716972d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import sklearn\n",
+    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
+    "from sklearn.linear_model import LinearRegression\n",
+    "from sklearn.metrics import mean_squared_error"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "6a2785e6-36b0-4649-91d1-aea8fd3599c1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "D:\\Programy\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3444: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version.\n",
+      "\n",
+      "\n",
+      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "107463\n"
+     ]
+    }
+   ],
+   "source": [
+    "train = pd.read_csv('train/train.tsv', header=None, sep='\\t', error_bad_lines=False)\n",
+    "print(len(train))\n",
+    "train = train.head(2000)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "8cc00b89-1007-4c4a-8ba7-c62b57459b79",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "x_train = train[4]\n",
+    "y_train = train[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "dd454ce5-a06e-4fbd-a546-83fb94ad0390",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "x_dev_data = pd.read_csv('dev-0/in.tsv', header=None, sep='\\t')\n",
+    "x_dev = x_dev_data[0]\n",
+    "x_dev[19999] = \"to jest tekst testowy\"\n",
+    "x_dev[20000] = \"a ten tekst jest najbardziej testowy\"\n",
+    "y_dev = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "79099730-c5bd-4c5c-a0b0-788512d44226",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vectorizer = TfidfVectorizer()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "0a1cce75-86a1-4f76-9416-e876e01699e3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "x_train = vectorizer.fit_transform(x_train)\n",
+    "x_dev = vectorizer.transform(x_dev)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "ef405093-6b4c-4558-add4-40bd0ced244e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = LinearRegression()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "4354553c-6143-43c7-8845-3b2327819481",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "LinearRegression()"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model.fit(x_train.toarray(), y_train)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "cc1270d5-29dc-4f03-82c1-dc03f3e4fa00",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "MemoryError",
+     "evalue": "Unable to allocate 32.2 GiB for an array with shape (20000, 216394) and data type float64",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
+      "\u001b[1;32mC:\\Users\\SEBAST~1\\AppData\\Local\\Temp/ipykernel_17784/3948937349.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdev_predicted\u001b[0m \u001b[1;33m=\u001b[0m  \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx_dev\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'dev-0/out.tsv'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'wt'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdev_predicted\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m         \u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;34m'\\n'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[1;32mD:\\Programy\\anaconda3\\lib\\site-packages\\scipy\\sparse\\compressed.py\u001b[0m in \u001b[0;36mtoarray\u001b[1;34m(self, order, out)\u001b[0m\n\u001b[0;32m   1029\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mout\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0morder\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1030\u001b[0m             \u001b[0morder\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_swap\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'cf'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1031\u001b[1;33m         \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_process_toarray_args\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1032\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflags\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mc_contiguous\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflags\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mf_contiguous\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1033\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Output array must be C or F contiguous'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[1;32mD:\\Programy\\anaconda3\\lib\\site-packages\\scipy\\sparse\\base.py\u001b[0m in \u001b[0;36m_process_toarray_args\u001b[1;34m(self, order, out)\u001b[0m\n\u001b[0;32m   1200\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1201\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1202\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1203\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1204\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[1;31mMemoryError\u001b[0m: Unable to allocate 32.2 GiB for an array with shape (20000, 216394) and data type float64"
+     ]
+    }
+   ],
+   "source": [
+    "dev_predicted =  model.predict(x_dev.toarray())\n",
+    "\n",
+    "with open('dev-0/out.tsv', 'wt') as f:\n",
+    "    for i in dev_predicted:\n",
+    "        f.write(str(i)+'\\n')\n",
+    "\n",
+    "dev_out = pd.read_csv('dev-0/out.tsv', header=None, sep='\\t')\n",
+    "dev_expected = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "223de995-5e91-4254-9214-4fc871c985e9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(mean_squared_error(dev_out, dev_expected))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3bc8418b-64f1-4163-a0ec-8e3293032341",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open('test-A/in.tsv', 'r', encoding = 'utf-8') as f:\n",
+    "       x_test = f.readlines()\n",
+    "        \n",
+    "x_test = pd.Series(x_test)\n",
+    "x_test = vectorizer.transform(x_test)\n",
+    "\n",
+    "test_predicted = model.predict(x_test.toarray())\n",
+    "\n",
+    "with open('test-A/out.tsv', 'wt') as f:\n",
+    "    for i in test_predicted:\n",
+    "        f.write(str(i)+'\\n')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a18aea56-7fa1-40bd-8aa3-bbaf9d66d6b7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!jupyter nbconvert --to script run.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

run.ipynb

@@ -13,7 +13,8 @@
     "import sklearn\n",
     "from sklearn.feature_extraction.text import TfidfVectorizer\n",
     "from sklearn.linear_model import LinearRegression\n",
-    "from sklearn.metrics import mean_squared_error"
+    "from sklearn.metrics import mean_squared_error\n",
+    "from sklearn.pipeline import make_pipeline"
    ]
   },
   {
@@ -43,7 +44,7 @@
    "source": [
     "train = pd.read_csv('train/train.tsv', header=None, sep='\\t', error_bad_lines=False)\n",
     "print(len(train))\n",
-    "train = train.head(1000)"
+    "train = train.head(20000)"
    ]
   },
   {
@@ -74,63 +75,34 @@
   {
    "cell_type": "code",
    "execution_count": 5,
-   "id": "79099730-c5bd-4c5c-a0b0-788512d44226",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "vectorizer = TfidfVectorizer()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
    "id": "0a1cce75-86a1-4f76-9416-e876e01699e3",
    "metadata": {},
-   "outputs": [],
-   "source": [
-    "x_train = vectorizer.fit_transform(x_train)\n",
-    "x_dev = vectorizer.transform(x_dev)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "id": "ef405093-6b4c-4558-add4-40bd0ced244e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "model = LinearRegression()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "id": "4354553c-6143-43c7-8845-3b2327819481",
-   "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "LinearRegression()"
+       "Pipeline(steps=[('tfidfvectorizer', TfidfVectorizer()),\n",
+       "                ('linearregression', LinearRegression())])"
       ]
      },
-     "execution_count": 8,
+     "execution_count": 5,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "model.fit(x_train.toarray(), y_train)"
+    "model = make_pipeline(TfidfVectorizer(), LinearRegression())\n",
+    "model.fit(x_train, y_train)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 6,
    "id": "cc1270d5-29dc-4f03-82c1-dc03f3e4fa00",
    "metadata": {},
    "outputs": [],
    "source": [
-    "dev_predicted =  model.predict(x_dev.toarray())\n",
+    "dev_predicted =  model.predict(x_dev)\n",
     "\n",
     "with open('dev-0/out.tsv', 'wt') as f:\n",
     "    for i in dev_predicted:\n",
@@ -142,7 +114,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 7,
    "id": "223de995-5e91-4254-9214-4fc871c985e9",
    "metadata": {},
    "outputs": [
@@ -150,7 +122,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "3486.2683285642797\n"
+      "4214.6524419302405\n"
      ]
     }
    ],
@@ -160,7 +132,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
    "id": "3bc8418b-64f1-4163-a0ec-8e3293032341",
    "metadata": {},
    "outputs": [],
@@ -168,10 +140,10 @@
     "with open('test-A/in.tsv', 'r', encoding = 'utf-8') as f:\n",
     "       x_test = f.readlines()\n",
     "        \n",
-    "x_test = pd.Series(x_test)\n",
+    "# x_test = pd.Series(x_test)\n",
-    "x_test = vectorizer.transform(x_test)\n",
+    "# x_test = vectorizer.transform(x_test)\n",
     "\n",
-    "test_predicted = model.predict(x_test.toarray())\n",
+    "test_predicted = model.predict(x_test)\n",
     "\n",
     "with open('test-A/out.tsv', 'wt') as f:\n",
     "    for i in test_predicted:\n",
@@ -180,19 +152,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
    "id": "a18aea56-7fa1-40bd-8aa3-bbaf9d66d6b7",
    "metadata": {},
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "[NbConvertApp] Converting notebook run.ipynb to script\n",
-      "[NbConvertApp] Writing 1629 bytes to run.py\n"
-     ]
-    }
-   ],
    "source": [
     "!jupyter nbconvert --to script run.ipynb"
    ]

run.py

@@ -11,6 +11,7 @@ import sklearn
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.linear_model import LinearRegression
 from sklearn.metrics import mean_squared_error
+from sklearn.pipeline import make_pipeline
 
 
 # In[2]:
@@ -18,7 +19,7 @@ from sklearn.metrics import mean_squared_error
 
 train = pd.read_csv('train/train.tsv', header=None, sep='\t', error_bad_lines=False)
 print(len(train))
-train = train.head(1000)
+train = train.head(20000)
 
 
 # In[3]:
@@ -41,32 +42,14 @@ y_dev = pd.read_csv('dev-0/expected.tsv', header=None, sep='\t')
 # In[5]:
 
 
-vectorizer = TfidfVectorizer()
+model = make_pipeline(TfidfVectorizer(), LinearRegression())
+model.fit(x_train, y_train)
 
 
 # In[6]:
 
 
-x_train = vectorizer.fit_transform(x_train)
+dev_predicted =  model.predict(x_dev)
-x_dev = vectorizer.transform(x_dev)
-
-
-# In[7]:
-
-
-model = LinearRegression()
-
-
-# In[8]:
-
-
-model.fit(x_train.toarray(), y_train)
-
-
-# In[9]:
-
-
-dev_predicted =  model.predict(x_dev.toarray())
 
 with open('dev-0/out.tsv', 'wt') as f:
     for i in dev_predicted:
@@ -76,7 +59,7 @@ dev_out = pd.read_csv('dev-0/out.tsv', header=None, sep='\t')
 dev_expected = pd.read_csv('dev-0/expected.tsv', header=None, sep='\t')
 
 
-# In[10]:
+# In[7]:
 
 
 print(mean_squared_error(dev_out, dev_expected))
@@ -88,12 +71,18 @@ print(mean_squared_error(dev_out, dev_expected))
 with open('test-A/in.tsv', 'r', encoding = 'utf-8') as f:
        x_test = f.readlines()
         
-x_test = pd.Series(x_test)
+# x_test = pd.Series(x_test)
-x_test = vectorizer.transform(x_test)
+# x_test = vectorizer.transform(x_test)
 
-test_predicted = model.predict(x_test.toarray())
+test_predicted = model.predict(x_test)
 
 with open('test-A/out.tsv', 'wt') as f:
     for i in test_predicted:
         f.write(str(i)+'\n')
 
+
+# In[ ]:
+
+
+get_ipython().system('jupyter nbconvert --to script run.ipynb')
+