{ "cells": [ { "cell_type": "code", "execution_count": 3, "id": "57debdd5-4760-4524-9c77-409652cfb52e", "metadata": {}, "outputs": [], "source": [ "import vowpalwabbit\n", "import pandas as pd\n", "import re" ] }, { "cell_type": "code", "execution_count": 4, "id": "48357fb3-9d6f-48d0-9869-bce7a87e3ba1", "metadata": {}, "outputs": [], "source": [ "def prediction(path_in, path_out, model, categories):\n", " data = pd.read_csv(path_in, header=None, sep='\\t')\n", " data = data.drop(1, axis=1)\n", " data.columns = ['year', 'text']\n", "\n", " data['train_input'] = data.apply(lambda row: to_vowpalwabbit(row, categories), axis=1)\n", "\n", " with open(path_out, 'w', encoding='utf-8') as file:\n", " for example in data['train_input']:\n", " predicted = model.predict(example)\n", " text_predicted = dict((value, key) for key, value in categories.items()).get(predicted)\n", " file.write(str(text_predicted) + '\\n')" ] }, { "cell_type": "code", "execution_count": 5, "id": "f47c4ff9-2078-43f7-b06c-99c9fdd2022e", "metadata": {}, "outputs": [], "source": [ "def to_vowpalwabbit(row, categories):\n", " text = row['text'].replace('\\n', ' ').lower().strip()\n", " text = re.sub(\"[^a-zA-Z -']\", '', text)\n", " text = re.sub(\" +\", ' ', text)\n", " year = row['year']\n", " try:\n", " category = categories[row['category']]\n", " except KeyError:\n", " category = ''\n", "\n", " vw = f\"{category} | year:{year} text:{text}\\n\"\n", "\n", " return vw" ] }, { "cell_type": "code", "execution_count": 11, "id": "015a5ccb-8fe0-45cf-bf59-416ce9e59dad", "metadata": {}, "outputs": [], "source": [ "x_train = pd.read_csv('train/in.tsv', header=None, sep='\\t')\n", "x_train = x_train.drop(1, axis=1)\n", "x_train.columns = ['year', 'text']\n", "\n", "\n", "y_train = pd.read_csv('train/expected.tsv', header=None, sep='\\t')\n", "y_train.columns = ['category']\n", "\n", "x_train = x_train[0:800000]\n", "y_train = y_train[0:800000]\n", "\n", "data = pd.concat([x_train, y_train], axis=1)" ] }, { "cell_type": "code", "execution_count": 12, "id": "d8a2c80c-3b93-410d-98e4-ac651d0933a2", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", " | year | \n", "text | \n", "
---|---|---|
0 | \n", "2004.508197 | \n", "Sudan claims it is disarming militias | \n", "
1 | \n", "2008.442623 | \n", "Bluffer's guide to Euro 2008 | \n", "
2 | \n", "2012.587432 | \n", "Ennis tallies her highest first day total | \n", "
3 | \n", "2009.071233 | \n", "Sri Lanka continues to battle Tamil Tigers | \n", "
4 | \n", "1997.345205 | \n", "Talks today to avert new health service strike | \n", "
... | \n", "... | \n", "... | \n", "
799995 | \n", "2010.876712 | \n", "Top league stars among 135 listed online | \n", "
799996 | \n", "2000.879452 | \n", "Cabinet to consider options for animal disposal | \n", "
799997 | \n", "2004.915068 | \n", "Last orders for Bewley's this evening | \n", "
799998 | \n", "2014.797260 | \n", "Toulon; Ospreys and Toulouse win Champions Cup... | \n", "
799999 | \n", "1999.019178 | \n", "Volatile year in store for the markets | \n", "
800000 rows × 2 columns
\n", "