594 lines
15 KiB
Plaintext
594 lines
15 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 86,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import lzma\n",
|
|
"import sys\n",
|
|
"from io import StringIO\n",
|
|
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
|
|
"import pandas as pd\n",
|
|
"import numpy\n",
|
|
"\n",
|
|
"pathX = \"./train/train.tsv.xz\"\n",
|
|
"# pathX = \"./train/in.tsv\"\n",
|
|
"# pathY = \"./train/meta.tsv.xz\"\n",
|
|
"nrows = 100000"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 87,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# data = lzma.open(pathX, mode='rt', encoding='utf-8').read()\n",
|
|
"# stringIO = StringIO(data)\n",
|
|
"# df = pd.read_csv(stringIO, sep=\"\\t\", header=None)\n",
|
|
"df = pd.read_csv(pathX, sep='\\t', nrows=nrows, header=None)\n",
|
|
"# df = df.drop(df.columns, axis=1)\n",
|
|
"# topics = pd.read_csv(pathY, sep='\\t', nrows=nrows, header=None)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 88,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"100000\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"print(len(df.index))\n",
|
|
"\n",
|
|
"# print(len(topics.index))\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 66,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def mergeTexts(a, b, c):\n",
|
|
" return str(a) + \" \" + str(b) + \" \" + str(c)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 67,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def getMean(a, b):\n",
|
|
" return ((a + b)/2)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 89,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df[\"year\"] = df.apply(lambda x: getMean(x[0], x[1]), axis = 1)\n",
|
|
"df[\"text\"] = df.apply(lambda x: x[4], axis = 1)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 90,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>year</th>\n",
|
|
" <th>text</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>63552</th>\n",
|
|
" <td>2013.212329</td>\n",
|
|
" <td>dnia 10 października 2012r., znak (...)/, Zakł...</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>89500</th>\n",
|
|
" <td>2013.656164</td>\n",
|
|
" <td>postępowania, skarżąca wniosła, jak na wstępie...</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>94039</th>\n",
|
|
" <td>1925.015068</td>\n",
|
|
" <td>dzieją się ciągle awantury, nieustają podejrze...</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>62566</th>\n",
|
|
" <td>2012.348361</td>\n",
|
|
" <td>Samodzielnego Publicznego Zespołu Opieki Zdrow...</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>87553</th>\n",
|
|
" <td>1975.494521</td>\n",
|
|
" <td>doprowadzających przeładowywane produkty od st...</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" year text\n",
|
|
"63552 2013.212329 dnia 10 października 2012r., znak (...)/, Zakł...\n",
|
|
"89500 2013.656164 postępowania, skarżąca wniosła, jak na wstępie...\n",
|
|
"94039 1925.015068 dzieją się ciągle awantury, nieustają podejrze...\n",
|
|
"62566 2012.348361 Samodzielnego Publicznego Zespołu Opieki Zdrow...\n",
|
|
"87553 1975.494521 doprowadzających przeładowywane produkty od st..."
|
|
]
|
|
},
|
|
"execution_count": 90,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"df = df.drop(columns = [0,1,2,3,4], axis=1)\n",
|
|
"df.sample(5)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 91,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>text</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>85466</th>\n",
|
|
" <td>pokoJenie dOInu Burbonów, zaprzestalo we Frano...</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>416</th>\n",
|
|
" <td>non. Jakiekolwiek próby odbudowy takich instyt...</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>36354</th>\n",
|
|
" <td>000 ludzi, a który był iście tryumf.ilnym. trw...</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>95566</th>\n",
|
|
" <td>do robienia lodów. Ogrodzenia do kląbów w wiel...</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>58632</th>\n",
|
|
" <td>ftcitnciu>a4ii. Dzień już dal nogę i mrugali g...</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" text\n",
|
|
"85466 pokoJenie dOInu Burbonów, zaprzestalo we Frano...\n",
|
|
"416 non. Jakiekolwiek próby odbudowy takich instyt...\n",
|
|
"36354 000 ludzi, a który był iście tryumf.ilnym. trw...\n",
|
|
"95566 do robienia lodów. Ogrodzenia do kląbów w wiel...\n",
|
|
"58632 ftcitnciu>a4ii. Dzień już dal nogę i mrugali g..."
|
|
]
|
|
},
|
|
"execution_count": 91,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"topics = df.pop('year')\n",
|
|
"df.sample(5)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 92,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"22843 1991.500000\n",
|
|
"12830 1937.157534\n",
|
|
"63119 1919.500000\n",
|
|
"77638 2010.130137\n",
|
|
"5577 1934.768493\n",
|
|
"Name: year, dtype: float64"
|
|
]
|
|
},
|
|
"execution_count": 92,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"topics.sample(5)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 93,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"array(['00', '000', '0000', ..., 'תורהט', 'תותיירב', 'תשדוקמ'],\n",
|
|
" dtype=object)"
|
|
]
|
|
},
|
|
"execution_count": 93,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"vectorizer = TfidfVectorizer(lowercase=True, stop_words=['polish'])\n",
|
|
"X = vectorizer.fit_transform(df.to_numpy().ravel())\n",
|
|
"vectorizer.get_feature_names_out()\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 73,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# vectorizer.transform(\"Ala ma kotka\".lower().split())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 74,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# df = df.reset_index()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 94,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"tfidfVector = vectorizer.transform(df[\"text\"])\n",
|
|
"\n",
|
|
" "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 76,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# from sklearn.model_selection import train_test_split\n",
|
|
"# from sklearn.naive_bayes import GaussianNB\n",
|
|
"# \n",
|
|
"# gnb = GaussianNB()\n",
|
|
"# gnb.fit(tfidfVector.todense(), topics)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 95,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from sklearn.linear_model import LinearRegression\n",
|
|
"\n",
|
|
"reg = LinearRegression().fit(tfidfVector, topics)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 106,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"testXPath = \"./dev-0/in.tsv\"\n",
|
|
"testYPath = \"./dev-0/expected.tsv\"\n",
|
|
"\n",
|
|
"testX = pd.read_csv(testXPath, sep='\\t', nrows=19998, header=None)\n",
|
|
"\n",
|
|
"testY = pd.read_csv(testYPath, sep='\\t', nrows=19998, header=None)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 107,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>0</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>9194</th>\n",
|
|
" <td>że w moich oczach umizgasz się do niej. Rzeczy...</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3959</th>\n",
|
|
" <td>reką dopoty, dopóki nic wyiaanię !Prawiü Bye m...</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>19210</th>\n",
|
|
" <td>końcach klapy wentylacyjnej i zapobiegają odch...</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>6080</th>\n",
|
|
" <td>lat cię- owe g\\\\\\\\ allowne walki w dziennikars...</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>18845</th>\n",
|
|
" <td>elektr,yczne] ny, poszanowania ładu i po- roc1...</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" 0\n",
|
|
"9194 że w moich oczach umizgasz się do niej. Rzeczy...\n",
|
|
"3959 reką dopoty, dopóki nic wyiaanię !Prawiü Bye m...\n",
|
|
"19210 końcach klapy wentylacyjnej i zapobiegają odch...\n",
|
|
"6080 lat cię- owe g\\\\\\\\ allowne walki w dziennikars...\n",
|
|
"18845 elektr,yczne] ny, poszanowania ładu i po- roc1..."
|
|
]
|
|
},
|
|
"execution_count": 107,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"testX.sample(5)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 108,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>0</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>3849</th>\n",
|
|
" <td>1956.476776</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" 0\n",
|
|
"3849 1956.476776"
|
|
]
|
|
},
|
|
"execution_count": 108,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"testY.sample()\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 109,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"testXtfidfVector = vectorizer.transform(testX[0])\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 110,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"-0.3101240322770993"
|
|
]
|
|
},
|
|
"execution_count": 110,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"reg.score(testXtfidfVector, testY[0])\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 157,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"testXPath = \"./test-A/in.tsv\"\n",
|
|
"testYPath = \"./test-A/out.tsv\"\n",
|
|
"\n",
|
|
"# testX = pd.read_csv(testXPath, sep='\\t', nrows=nrows, header=None, skip_blank_lines=False)\n",
|
|
"# testY = pd.read_csv(testYPath, sep='\\t', nrows=nrows, header=None)\n",
|
|
"reviews = []\n",
|
|
"with open(testXPath, 'r', encoding=\"utf8\") as openfile:\n",
|
|
" for line in openfile:\n",
|
|
" reviews.append(line)\n",
|
|
" testX = pd.DataFrame(reviews)\n",
|
|
"testXtfidfVector = vectorizer.transform(testX[0])\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 158,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"14220\n",
|
|
"14220\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"print(testX.shape[0])\n",
|
|
"\n",
|
|
"print(testXtfidfVector.shape[0])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 159,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"14220\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"pred = reg.predict(testXtfidfVector)\n",
|
|
"print(len(pred))\n",
|
|
"\n",
|
|
"import csv\n",
|
|
"with open(testYPath, 'w', newline='') as f_output:\n",
|
|
" tsv_output = csv.writer(f_output, delimiter='\\n')\n",
|
|
" tsv_output.writerow(pred)"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"interpreter": {
|
|
"hash": "369f2c481f4da34e4445cda3fffd2e751bd1c4d706f27375911949ba6bb62e1c"
|
|
},
|
|
"kernelspec": {
|
|
"display_name": "Python 3.10.4 64-bit",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.10.4"
|
|
},
|
|
"orig_nbformat": 4
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|