{
"cells": [
{
"cell_type": "code",
"execution_count": 86,
"metadata": {},
"outputs": [],
"source": [
"import lzma\n",
"import sys\n",
"from io import StringIO\n",
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
"import pandas as pd\n",
"import numpy\n",
"\n",
"pathX = \"./train/train.tsv.xz\"\n",
"# pathX = \"./train/in.tsv\"\n",
"# pathY = \"./train/meta.tsv.xz\"\n",
"nrows = 100000"
]
},
{
"cell_type": "code",
"execution_count": 87,
"metadata": {},
"outputs": [],
"source": [
"# data = lzma.open(pathX, mode='rt', encoding='utf-8').read()\n",
"# stringIO = StringIO(data)\n",
"# df = pd.read_csv(stringIO, sep=\"\\t\", header=None)\n",
"df = pd.read_csv(pathX, sep='\\t', nrows=nrows, header=None)\n",
"# df = df.drop(df.columns, axis=1)\n",
"# topics = pd.read_csv(pathY, sep='\\t', nrows=nrows, header=None)"
]
},
{
"cell_type": "code",
"execution_count": 88,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"100000\n"
]
}
],
"source": [
"print(len(df.index))\n",
"\n",
"# print(len(topics.index))\n"
]
},
{
"cell_type": "code",
"execution_count": 66,
"metadata": {},
"outputs": [],
"source": [
"def mergeTexts(a, b, c):\n",
" return str(a) + \" \" + str(b) + \" \" + str(c)"
]
},
{
"cell_type": "code",
"execution_count": 67,
"metadata": {},
"outputs": [],
"source": [
"def getMean(a, b):\n",
" return ((a + b)/2)"
]
},
{
"cell_type": "code",
"execution_count": 89,
"metadata": {},
"outputs": [],
"source": [
"df[\"year\"] = df.apply(lambda x: getMean(x[0], x[1]), axis = 1)\n",
"df[\"text\"] = df.apply(lambda x: x[4], axis = 1)"
]
},
{
"cell_type": "code",
"execution_count": 90,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" year | \n",
" text | \n",
"
\n",
" \n",
" \n",
" \n",
" 63552 | \n",
" 2013.212329 | \n",
" dnia 10 października 2012r., znak (...)/, Zakł... | \n",
"
\n",
" \n",
" 89500 | \n",
" 2013.656164 | \n",
" postępowania, skarżąca wniosła, jak na wstępie... | \n",
"
\n",
" \n",
" 94039 | \n",
" 1925.015068 | \n",
" dzieją się ciągle awantury, nieustają podejrze... | \n",
"
\n",
" \n",
" 62566 | \n",
" 2012.348361 | \n",
" Samodzielnego Publicznego Zespołu Opieki Zdrow... | \n",
"
\n",
" \n",
" 87553 | \n",
" 1975.494521 | \n",
" doprowadzających przeładowywane produkty od st... | \n",
"
\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" year text\n",
"63552 2013.212329 dnia 10 października 2012r., znak (...)/, Zakł...\n",
"89500 2013.656164 postępowania, skarżąca wniosła, jak na wstępie...\n",
"94039 1925.015068 dzieją się ciągle awantury, nieustają podejrze...\n",
"62566 2012.348361 Samodzielnego Publicznego Zespołu Opieki Zdrow...\n",
"87553 1975.494521 doprowadzających przeładowywane produkty od st..."
]
},
"execution_count": 90,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df = df.drop(columns = [0,1,2,3,4], axis=1)\n",
"df.sample(5)"
]
},
{
"cell_type": "code",
"execution_count": 91,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" text | \n",
"
\n",
" \n",
" \n",
" \n",
" 85466 | \n",
" pokoJenie dOInu Burbonów, zaprzestalo we Frano... | \n",
"
\n",
" \n",
" 416 | \n",
" non. Jakiekolwiek próby odbudowy takich instyt... | \n",
"
\n",
" \n",
" 36354 | \n",
" 000 ludzi, a który był iście tryumf.ilnym. trw... | \n",
"
\n",
" \n",
" 95566 | \n",
" do robienia lodów. Ogrodzenia do kląbów w wiel... | \n",
"
\n",
" \n",
" 58632 | \n",
" ftcitnciu>a4ii. Dzień już dal nogę i mrugali g... | \n",
"
\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" text\n",
"85466 pokoJenie dOInu Burbonów, zaprzestalo we Frano...\n",
"416 non. Jakiekolwiek próby odbudowy takich instyt...\n",
"36354 000 ludzi, a który był iście tryumf.ilnym. trw...\n",
"95566 do robienia lodów. Ogrodzenia do kląbów w wiel...\n",
"58632 ftcitnciu>a4ii. Dzień już dal nogę i mrugali g..."
]
},
"execution_count": 91,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"topics = df.pop('year')\n",
"df.sample(5)"
]
},
{
"cell_type": "code",
"execution_count": 92,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"22843 1991.500000\n",
"12830 1937.157534\n",
"63119 1919.500000\n",
"77638 2010.130137\n",
"5577 1934.768493\n",
"Name: year, dtype: float64"
]
},
"execution_count": 92,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"topics.sample(5)"
]
},
{
"cell_type": "code",
"execution_count": 93,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array(['00', '000', '0000', ..., 'תורהט', 'תותיירב', 'תשדוקמ'],\n",
" dtype=object)"
]
},
"execution_count": 93,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"vectorizer = TfidfVectorizer(lowercase=True, stop_words=['polish'])\n",
"X = vectorizer.fit_transform(df.to_numpy().ravel())\n",
"vectorizer.get_feature_names_out()\n"
]
},
{
"cell_type": "code",
"execution_count": 73,
"metadata": {},
"outputs": [],
"source": [
"# vectorizer.transform(\"Ala ma kotka\".lower().split())"
]
},
{
"cell_type": "code",
"execution_count": 74,
"metadata": {},
"outputs": [],
"source": [
"# df = df.reset_index()"
]
},
{
"cell_type": "code",
"execution_count": 94,
"metadata": {},
"outputs": [],
"source": [
"tfidfVector = vectorizer.transform(df[\"text\"])\n",
"\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 76,
"metadata": {},
"outputs": [],
"source": [
"# from sklearn.model_selection import train_test_split\n",
"# from sklearn.naive_bayes import GaussianNB\n",
"# \n",
"# gnb = GaussianNB()\n",
"# gnb.fit(tfidfVector.todense(), topics)"
]
},
{
"cell_type": "code",
"execution_count": 95,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.linear_model import LinearRegression\n",
"\n",
"reg = LinearRegression().fit(tfidfVector, topics)\n"
]
},
{
"cell_type": "code",
"execution_count": 106,
"metadata": {},
"outputs": [],
"source": [
"testXPath = \"./dev-0/in.tsv\"\n",
"testYPath = \"./dev-0/expected.tsv\"\n",
"\n",
"testX = pd.read_csv(testXPath, sep='\\t', nrows=19998, header=None)\n",
"\n",
"testY = pd.read_csv(testYPath, sep='\\t', nrows=19998, header=None)\n"
]
},
{
"cell_type": "code",
"execution_count": 107,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
"
\n",
" \n",
" \n",
" \n",
" 9194 | \n",
" że w moich oczach umizgasz się do niej. Rzeczy... | \n",
"
\n",
" \n",
" 3959 | \n",
" reką dopoty, dopóki nic wyiaanię !Prawiü Bye m... | \n",
"
\n",
" \n",
" 19210 | \n",
" końcach klapy wentylacyjnej i zapobiegają odch... | \n",
"
\n",
" \n",
" 6080 | \n",
" lat cię- owe g\\\\\\\\ allowne walki w dziennikars... | \n",
"
\n",
" \n",
" 18845 | \n",
" elektr,yczne] ny, poszanowania ładu i po- roc1... | \n",
"
\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" 0\n",
"9194 że w moich oczach umizgasz się do niej. Rzeczy...\n",
"3959 reką dopoty, dopóki nic wyiaanię !Prawiü Bye m...\n",
"19210 końcach klapy wentylacyjnej i zapobiegają odch...\n",
"6080 lat cię- owe g\\\\\\\\ allowne walki w dziennikars...\n",
"18845 elektr,yczne] ny, poszanowania ładu i po- roc1..."
]
},
"execution_count": 107,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"testX.sample(5)"
]
},
{
"cell_type": "code",
"execution_count": 108,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
"
\n",
" \n",
" \n",
" \n",
" 3849 | \n",
" 1956.476776 | \n",
"
\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" 0\n",
"3849 1956.476776"
]
},
"execution_count": 108,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"testY.sample()\n"
]
},
{
"cell_type": "code",
"execution_count": 109,
"metadata": {},
"outputs": [],
"source": [
"testXtfidfVector = vectorizer.transform(testX[0])\n"
]
},
{
"cell_type": "code",
"execution_count": 110,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"-0.3101240322770993"
]
},
"execution_count": 110,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"reg.score(testXtfidfVector, testY[0])\n"
]
},
{
"cell_type": "code",
"execution_count": 157,
"metadata": {},
"outputs": [],
"source": [
"testXPath = \"./test-A/in.tsv\"\n",
"testYPath = \"./test-A/out.tsv\"\n",
"\n",
"# testX = pd.read_csv(testXPath, sep='\\t', nrows=nrows, header=None, skip_blank_lines=False)\n",
"# testY = pd.read_csv(testYPath, sep='\\t', nrows=nrows, header=None)\n",
"reviews = []\n",
"with open(testXPath, 'r', encoding=\"utf8\") as openfile:\n",
" for line in openfile:\n",
" reviews.append(line)\n",
" testX = pd.DataFrame(reviews)\n",
"testXtfidfVector = vectorizer.transform(testX[0])\n"
]
},
{
"cell_type": "code",
"execution_count": 158,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"14220\n",
"14220\n"
]
}
],
"source": [
"print(testX.shape[0])\n",
"\n",
"print(testXtfidfVector.shape[0])"
]
},
{
"cell_type": "code",
"execution_count": 159,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"14220\n"
]
}
],
"source": [
"pred = reg.predict(testXtfidfVector)\n",
"print(len(pred))\n",
"\n",
"import csv\n",
"with open(testYPath, 'w', newline='') as f_output:\n",
" tsv_output = csv.writer(f_output, delimiter='\\n')\n",
" tsv_output.writerow(pred)"
]
}
],
"metadata": {
"interpreter": {
"hash": "369f2c481f4da34e4445cda3fffd2e751bd1c4d706f27375911949ba6bb62e1c"
},
"kernelspec": {
"display_name": "Python 3.10.4 64-bit",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.4"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}