run

.ipynb_checkpoints/run-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

.jovianrc

@@ -0,0 +1,3 @@
+{
+  "notebooks": {}
+}

run.ipynb

@@ -0,0 +1,129 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "f844d81d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import lzma\n",
+    "from sklearn.linear_model import LinearRegression\n",
+    "from sklearn.pipeline import make_pipeline\n",
+    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
+    "from sklearn.metrics import mean_squared_error\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "e18dcd2f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with lzma.open('train/train.tsv.xz', 'rt', encoding=\"utf-8\") as f:\n",
+    "    df = pd.read_csv(f, sep='\\t', names=['Begin', 'End', 'Title', 'Publisher', 'Text'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "23006157",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def readFile(filename):\n",
+    "    result = []\n",
+    "    with open(filename, 'r', encoding=\"utf-8\") as f:\n",
+    "        for line in f:\n",
+    "            text = line.split(\"\\t\")[0].strip()\n",
+    "            result.append(text)\n",
+    "    return result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "id": "7fc3427f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def predict(filename, predictions):\n",
+    "    with open(filename, \"w\") as f:\n",
+    "        for p in predictions:\n",
+    "            f.write(str(p) + \"\\n\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "b92a45ce",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = df[['Text', 'Begin']]\n",
+    "X_train = df['Text']\n",
+    "y_train = df['Begin']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "7c6d4186",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Pipeline(steps=[('tfidfvectorizer', TfidfVectorizer()),\n",
+       "                ('linearregression', LinearRegression())])"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model = make_pipeline(TfidfVectorizer(), LinearRegression())\n",
+    "model.fit(X_train, y_train)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "id": "7497ecb0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "filenames=[('dev-0/in.tsv',\"dev-0/out.tsv\"), ('dev-1/in.tsv', \"dev-1/out.tsv\"), ('test-A/in.tsv', 'test-A/out.tsv')]\n",
+    "for filename in filenames:\n",
+    "    f=readFile(filename[0])\n",
+    "    y_predict=model.predict(f)\n",
+    "    predict(filename[1],y_predict)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

run.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[16]:
+
+
+import lzma
+from sklearn.linear_model import LinearRegression
+from sklearn.pipeline import make_pipeline
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics import mean_squared_error
+import pandas as pd
+
+
+# In[18]:
+
+
+with lzma.open('train/train.tsv.xz', 'rt', encoding="utf-8") as f:
+    df = pd.read_csv(f, sep='\t', names=['Begin', 'End', 'Title', 'Publisher', 'Text'])
+
+
+# In[19]:
+
+
+def readFile(filename):
+    result = []
+    with open(filename, 'r', encoding="utf-8") as f:
+        for line in f:
+            text = line.split("\t")[0].strip()
+            result.append(text)
+    return result
+
+
+# In[29]:
+
+
+def predict(filename, predictions):
+    with open(filename, "w") as f:
+        for p in predictions:
+            f.write(str(p) + "\n")
+
+
+# In[22]:
+
+
+df = df[['Text', 'Begin']]
+X_train = df['Text']
+y_train = df['Begin']
+
+
+# In[23]:
+
+
+model = make_pipeline(TfidfVectorizer(), LinearRegression())
+model.fit(X_train, y_train)
+
+
+# In[30]:
+
+
+filenames=[('dev-0/in.tsv',"dev-0/out.tsv"), ('dev-1/in.tsv', "dev-1/out.tsv"), ('test-A/in.tsv', 'test-A/out.tsv')]
+for filename in filenames:
+    f=readFile(filename[0])
+    y_predict=model.predict(f)
+    predict(filename[1],y_predict)
+