diff --git a/run.ipynb b/run.ipynb new file mode 100644 index 0000000..5d9001a --- /dev/null +++ b/run.ipynb @@ -0,0 +1,159 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "greenhouse-technician", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import sklearn\n", + "import pandas as pd\n", + "from gzip import open as open_gz\n", + "from sklearn.feature_extraction.text import TfidfVectorizer\n", + "from sklearn.pipeline import make_pipeline\n", + "from sklearn.linear_model import LinearRegression\n", + "from sklearn.metrics import mean_squared_error" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "acoustic-dividend", + "metadata": {}, + "outputs": [], + "source": [ + "def predict_year(x, path_out, model):\n", + " results = model.predict(x)\n", + " with open(path_out, 'wt') as file:\n", + " for r in results:\n", + " file.write(str(r) + '\\n') " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "78c79a98-8309-4c1c-b27d-faad2ee7a2af", + "metadata": {}, + "outputs": [], + "source": [ + "def read_file(filename):\n", + " result = []\n", + " with open(filename, 'r', encoding=\"utf-8\") as file:\n", + " for line in file:\n", + " text = line.split(\"\\t\")[0].strip()\n", + " result.append(text)\n", + " return result" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "senior-harassment", + "metadata": {}, + "outputs": [], + "source": [ + "with open('train/train.tsv', 'r', encoding='utf8') as file:\n", + " train = pd.read_csv(file, sep='\\t', names=['Begin', 'End', 'Title', 'Author', 'Text'])\n", + " \n", + "train = train[0:12000]\n", + "train_x = train['Text']\n", + "#train['Date'] = (train['Date1'].astype(float) + train['Date2'].astype(float))/2\n", + "train_y = train['Begin']" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "polyphonic-coach", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Pipeline(steps=[('tfidfvectorizer', TfidfVectorizer()),\n", + " ('linearregression', LinearRegression())])" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model = make_pipeline(TfidfVectorizer(), LinearRegression())\n", + "model.fit(train_x, train_y)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "varying-wright", + "metadata": {}, + "outputs": [], + "source": [ + "x_dev_0 = read_file('dev-0/in.tsv')\n", + "predict_year(x_dev_0, 'dev-0/out.tsv', model)\n", + "x_dev_1 = read_file('dev-1/in.tsv')\n", + "predict_year(x_dev_1,'dev-1/out.tsv', model)\n", + "x_test = read_file('test-A/in.tsv')\n", + "predict_year(x_test,'test-A/out.tsv', model)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "traditional-amount", + "metadata": {}, + "outputs": [], + "source": [ + "#y_dev = pd.read_csv('dev-0/out.tsv',header = None, sep = '/t',engine = 'python')\n", + "#y_dev = y_dev[0]\n", + "#y_dev_exp = pd.read_csv('dev-0/expected.tsv',header = None, sep = '/t',engine = 'python')\n", + "#y_dev_exp = y_dev_exp[0]\n", + "#RMSE_dev = mean_squared_error(y_dev_exp, y_dev) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "close-clinton", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "official-sweet", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}