1k train data

.ipynb_checkpoints/run-checkpoint.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 1,
    "id": "d103a6c5-a9b4-4547-9e10-f384d716972d",
    "metadata": {},
    "outputs": [],
@@ -12,7 +12,6 @@
     "import numpy as np\n",
     "import sklearn\n",
     "from sklearn.feature_extraction.text import TfidfVectorizer\n",
-    "from sklearn.pipeline import make_pipeline\n",
     "from sklearn.linear_model import LinearRegression\n",
     "from sklearn.metrics import mean_squared_error"
    ]
@@ -32,22 +31,19 @@
       "\n",
       "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
      ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "107463\n"
+     ]
     }
    ],
    "source": [
     "train = pd.read_csv('train/train.tsv', header=None, sep='\\t', error_bad_lines=False)\n",
-    "train = train.head(500)"
+    "print(len(train))\n",
-   ]
+    "train = train.head(1000)"
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e4b5f917-bde7-4b69-a394-1ab0fe0b752a",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# with open('train/train.tsv', 'r', encoding='utf8') as file:\n",
-    "#     train = pd.read_csv(file, sep='\\t')"
    ]
   },
   {
@@ -61,16 +57,6 @@
     "y_train = train[0]"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "5faa4b35-ccf7-4656-a08a-99d1d96d8a21",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(x_train)"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": 4,
@@ -80,9 +66,9 @@
    "source": [
     "x_dev_data = pd.read_csv('dev-0/in.tsv', header=None, sep='\\t')\n",
     "x_dev = x_dev_data[0]\n",
-    "y_dev = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')\n",
+    "x_dev[19999] = \"to jest tekst testowy\"\n",
-    "# x_dev.head(1000)\n",
+    "x_dev[20000] = \"a ten tekst jest najbardziej testowy\"\n",
-    "# y_dev.head(1000)"
+    "y_dev = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')"
    ]
   },
   {
@@ -151,25 +137,12 @@
     "        f.write(str(i)+'\\n')\n",
     "\n",
     "dev_out = pd.read_csv('dev-0/out.tsv', header=None, sep='\\t')\n",
-    "# dev_out = dev_out.head(1000)\n",
+    "dev_expected = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')\n"
-    "dev_expected = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')\n",
-    "# dev_expected = dev_expected.head(1000)\n",
-    "# print(mean_squared_error(dev_out, dev_expected))\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 10,
-   "id": "7b265b2b-cac1-457c-80f9-6f6dec30045b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "dev_expected = dev_expected.head(19998)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 17,
    "id": "223de995-5e91-4254-9214-4fc871c985e9",
    "metadata": {},
    "outputs": [
@@ -177,7 +150,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "3579.7645467601897\n"
+      "3486.2683285642797\n"
      ]
     }
    ],
@@ -187,7 +160,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 11,
    "id": "3bc8418b-64f1-4163-a0ec-8e3293032341",
    "metadata": {},
    "outputs": [],
@@ -204,6 +177,25 @@
     "    for i in test_predicted:\n",
     "        f.write(str(i)+'\\n')"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "a18aea56-7fa1-40bd-8aa3-bbaf9d66d6b7",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[NbConvertApp] Converting notebook run.ipynb to script\n",
+      "[NbConvertApp] Writing 1629 bytes to run.py\n"
+     ]
+    }
+   ],
+   "source": [
+    "!jupyter nbconvert --to script run.ipynb"
+   ]
   }
  ],
  "metadata": {

run.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 1,
    "id": "d103a6c5-a9b4-4547-9e10-f384d716972d",
    "metadata": {},
    "outputs": [],
@@ -12,7 +12,6 @@
     "import numpy as np\n",
     "import sklearn\n",
     "from sklearn.feature_extraction.text import TfidfVectorizer\n",
-    "from sklearn.pipeline import make_pipeline\n",
     "from sklearn.linear_model import LinearRegression\n",
     "from sklearn.metrics import mean_squared_error"
    ]
@@ -32,22 +31,19 @@
       "\n",
       "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
      ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "107463\n"
+     ]
     }
    ],
    "source": [
     "train = pd.read_csv('train/train.tsv', header=None, sep='\\t', error_bad_lines=False)\n",
-    "train = train.head(500)"
+    "print(len(train))\n",
-   ]
+    "train = train.head(1000)"
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e4b5f917-bde7-4b69-a394-1ab0fe0b752a",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# with open('train/train.tsv', 'r', encoding='utf8') as file:\n",
-    "#     train = pd.read_csv(file, sep='\\t')"
    ]
   },
   {
@@ -61,16 +57,6 @@
     "y_train = train[0]"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "5faa4b35-ccf7-4656-a08a-99d1d96d8a21",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(x_train)"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": 4,
@@ -80,9 +66,9 @@
    "source": [
     "x_dev_data = pd.read_csv('dev-0/in.tsv', header=None, sep='\\t')\n",
     "x_dev = x_dev_data[0]\n",
-    "y_dev = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')\n",
+    "x_dev[19999] = \"to jest tekst testowy\"\n",
-    "# x_dev.head(1000)\n",
+    "x_dev[20000] = \"a ten tekst jest najbardziej testowy\"\n",
-    "# y_dev.head(1000)"
+    "y_dev = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')"
    ]
   },
   {
@@ -151,25 +137,12 @@
     "        f.write(str(i)+'\\n')\n",
     "\n",
     "dev_out = pd.read_csv('dev-0/out.tsv', header=None, sep='\\t')\n",
-    "# dev_out = dev_out.head(1000)\n",
+    "dev_expected = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')\n"
-    "dev_expected = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')\n",
-    "# dev_expected = dev_expected.head(1000)\n",
-    "# print(mean_squared_error(dev_out, dev_expected))\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 10,
-   "id": "7b265b2b-cac1-457c-80f9-6f6dec30045b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "dev_expected = dev_expected.head(19998)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 17,
    "id": "223de995-5e91-4254-9214-4fc871c985e9",
    "metadata": {},
    "outputs": [
@@ -177,7 +150,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "3579.7645467601897\n"
+      "3486.2683285642797\n"
      ]
     }
    ],
@@ -187,7 +160,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 11,
    "id": "3bc8418b-64f1-4163-a0ec-8e3293032341",
    "metadata": {},
    "outputs": [],
@@ -204,6 +177,25 @@
     "    for i in test_predicted:\n",
     "        f.write(str(i)+'\\n')"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "a18aea56-7fa1-40bd-8aa3-bbaf9d66d6b7",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[NbConvertApp] Converting notebook run.ipynb to script\n",
+      "[NbConvertApp] Writing 1629 bytes to run.py\n"
+     ]
+    }
+   ],
+   "source": [
+    "!jupyter nbconvert --to script run.ipynb"
+   ]
   }
  ],
  "metadata": {

run.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+import os
+import pandas as pd
+import numpy as np
+import sklearn
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.linear_model import LinearRegression
+from sklearn.metrics import mean_squared_error
+
+
+# In[2]:
+
+
+train = pd.read_csv('train/train.tsv', header=None, sep='\t', error_bad_lines=False)
+print(len(train))
+train = train.head(1000)
+
+
+# In[3]:
+
+
+x_train = train[4]
+y_train = train[0]
+
+
+# In[4]:
+
+
+x_dev_data = pd.read_csv('dev-0/in.tsv', header=None, sep='\t')
+x_dev = x_dev_data[0]
+x_dev[19999] = "to jest tekst testowy"
+x_dev[20000] = "a ten tekst jest najbardziej testowy"
+y_dev = pd.read_csv('dev-0/expected.tsv', header=None, sep='\t')
+
+
+# In[5]:
+
+
+vectorizer = TfidfVectorizer()
+
+
+# In[6]:
+
+
+x_train = vectorizer.fit_transform(x_train)
+x_dev = vectorizer.transform(x_dev)
+
+
+# In[7]:
+
+
+model = LinearRegression()
+
+
+# In[8]:
+
+
+model.fit(x_train.toarray(), y_train)
+
+
+# In[9]:
+
+
+dev_predicted =  model.predict(x_dev.toarray())
+
+with open('dev-0/out.tsv', 'wt') as f:
+    for i in dev_predicted:
+        f.write(str(i)+'\n')
+
+dev_out = pd.read_csv('dev-0/out.tsv', header=None, sep='\t')
+dev_expected = pd.read_csv('dev-0/expected.tsv', header=None, sep='\t')
+
+
+# In[10]:
+
+
+print(mean_squared_error(dev_out, dev_expected))
+
+
+# In[ ]:
+
+
+with open('test-A/in.tsv', 'r', encoding = 'utf-8') as f:
+       x_test = f.readlines()
+        
+x_test = pd.Series(x_test)
+x_test = vectorizer.transform(x_test)
+
+test_predicted = model.predict(x_test.toarray())
+
+with open('test-A/out.tsv', 'wt') as f:
+    for i in test_predicted:
+        f.write(str(i)+'\n')
+