s444380
/
retroc2
forked from kubapok/retroc2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
retroc2/train.ipynb

275 lines
40 KiB
Plaintext
Raw Permalink Normal View History

Basic linear regression with tfidf
2022-05-17 20:39:17 +02:00
{
"metadata": {
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.5-final"
},
"orig_nbformat": 2,
"kernelspec": {
"name": "python3",
"display_name": "Python 3.9.5 64-bit",
"metadata": {
"interpreter": {
"hash": "ac59ebe37160ed0dfa835113d9b8498d9f09ceb179beaac4002f036b9467c963"
}
}
}
},
"nbformat": 4,
"nbformat_minor": 2,
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import lzma\n",
"import pandas as pd\n",
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
"from sklearn.linear_model import LinearRegression"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
" 0 1 \\\n",
"0 1985.493151 1985.495890 \n",
"1 1926.473973 1926.476712 \n",
"2 2013.961644 2013.964384 \n",
"3 1925.000000 1926.000000 \n",
"4 1981.000000 1982.000000 \n",
"... ... ... \n",
"107458 2013.057534 2013.060274 \n",
"107459 2013.021918 2013.024658 \n",
"107460 2013.920548 2013.923288 \n",
"107461 2013.082192 2013.084931 \n",
"107462 2013.098630 2013.101370 \n",
"\n",
" 2 3 \\\n",
"0 PRZEKRÓJ MBC \n",
"1 NADWIŚLANIN KPBC \n",
"2 SĄD APELACYJNY W ŁODZI I WYDZIAŁ CYWILNY SAOS \n",
"3 GAZETA BANKOWA eBUW \n",
"4 CIA ZBC \n",
"... ... ... \n",
"107458 SĄD REJONOWY DLA WROCŁAWIA-ŚRÓDMIEŚCIA WE WROC... SAOS \n",
"107459 WYROK W SPRAWIE KIO SAOS \n",
"107460 SĄD OKRĘGOWY W PIOTRKOWIE TRYBUNALSKIM IV WYDZ... SAOS \n",
"107461 SĄD REJONOWY W JELENIEJ GÓRZE I WYDZIAŁ CYWILNY SAOS \n",
"107462 SĄD OKRĘGOWY W ELBLĄGU I WYDZIAŁ CYWILNY SAOS \n",
"\n",
" 4 \n",
"0 nowią część kultury. U nas już nikt ich nie ch... \n",
"1 hlstorja znana w okresie piramid, jak wlaśclcl... \n",
"2 działek. Idąc dalej w swych hipotetycznych roz... \n",
"3 w Warszawie o stosunkach domowych dziatwy szko... \n",
"4 \\\\'iykład: \"Cywilizacyjna Koncepcja dziejów ¥e... \n",
"... ... \n",
"107458 M. (2) na rzecz powoda M. S. kwotę 5003,66 zł ... \n",
"107459 Zintegrowanego Systemu Informatycznego (ZSI), ... \n",
"107460 prokurator. Wyrokowi temu powołując się na prz... \n",
"107461 07 lipca 2010 r. świadczą o tym, że nie wszyst... \n",
"107462 zatem niezdolności do pracy było schorzenie sa... \n",
"\n",
"[107463 rows x 5 columns]"
],
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>0</th>\n <th>1</th>\n <th>2</th>\n <th>3</th>\n <th>4</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>0</th>\n <td>1985.493151</td>\n <td>1985.495890</td>\n <td>PRZEKRÓJ</td>\n <td>MBC</td>\n <td>nowią część kultury. U nas już nikt ich nie ch...</td>\n </tr>\n <tr>\n <th>1</th>\n <td>1926.473973</td>\n <td>1926.476712</td>\n <td>NADWIŚLANIN</td>\n <td>KPBC</td>\n <td>hlstorja znana w okresie piramid, jak wlaśclcl...</td>\n </tr>\n <tr>\n <th>2</th>\n <td>2013.961644</td>\n <td>2013.964384</td>\n <td>SĄD APELACYJNY W ŁODZI I WYDZIAŁ CYWILNY</td>\n <td>SAOS</td>\n <td>działek. Idąc dalej w swych hipotetycznych roz...</td>\n </tr>\n <tr>\n <th>3</th>\n <td>1925.000000</td>\n <td>1926.000000</td>\n <td>GAZETA BANKOWA</td>\n <td>eBUW</td>\n <td>w Warszawie o stosunkach domowych dziatwy szko...</td>\n </tr>\n <tr>\n <th>4</th>\n <td>1981.000000</td>\n <td>1982.000000</td>\n <td>CIA</td>\n <td>ZBC</td>\n <td>\\\\'iykład: \"Cywilizacyjna Koncepcja dziejów ¥e...</td>\n </tr>\n <tr>\n <th>...</th>\n <td>...</td>\n <td>...</td>\n <td>...</td>\n <td>...</td>\n <td>...</td>\n </tr>\n <tr>\n <th>107458</th>\n <td>2013.057534</td>\n <td>2013.060274</td>\n <td>SĄD REJONOWY DLA WROCŁAWIA-ŚRÓDMIEŚCIA WE WROC...</td>\n <td>SAOS</td>\n <td>M. (2) na rzecz powoda M. S. kwotę 5003,66 zł ...</td>\n </tr>\n <tr>\n <th>107459</th>\n <td>2013.021918</td>\n <td>2013.024658</td>\n <td>WYROK W SPRAWIE KIO</td>\n <td>SAOS</td>\n <td>Zintegrowanego Systemu Informatycznego (ZSI), ...</td>\n </tr>\n <tr>\n <th>107460</th>\n <td>2013.920548</td>\n <td>2013.923288</td>\n <td>SĄD OKRĘGOWY W PIOTRKOWIE TRYBUNALSKIM IV WYDZ...</td>\n <td>SAOS</td>\n <td>prokurator. Wyrokowi temu powołując się na prz...</td>\n </tr>\n <tr>\n <th>107461</th>\n <td>2013.082192</td>\n <td>2013.084931</td>\n <td>SĄD REJONOWY W JELENIEJ GÓRZE I WYDZIAŁ CYWILNY</td>\n <td>SAOS</td>\n <td>07 lipca 2010 r. świadczą o tym, że nie wszyst...</td>\n </tr>\n <tr>\n <th>107462</th>\n <td>2013.098630</td>\n <td>2013.101370</td>\n <td>SĄD OKRĘGOWY W ELBLĄGU I WYDZIAŁ CYWILNY</td>\n <td>SAOS</td>\n <td>zatem niezdolności do pracy było schorzenie sa...</td>\n </tr>\n </tbody>\n</table>\n<p>107463 rows × 5 columns</p>\n</div>"
},
"metadata": {},
"execution_count": 3
}
],
"source": [
"with open('train/train.tsv', 'r', encoding=\"utf-8\") as file:\n",
" train = pd.read_csv(file, sep=\"\\t\", header=None)\n",
"train"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"0 1985\n",
"1 1926\n",
"2 2013\n",
"3 1925\n",
"4 1981\n",
" ... \n",
"107458 2013\n",
"107459 2013\n",
"107460 2013\n",
"107461 2013\n",
"107462 2013\n",
"Length: 107463, dtype: int32"
]
},
"metadata": {},
"execution_count": 11
}
],
"source": [
"y_train = ((train[0] + train[1]) // 2).astype(int)\n",
"y_train"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"0 nowią część kultury. U nas już nikt ich nie ch...\n",
"1 hlstorja znana w okresie piramid, jak wlaśclcl...\n",
"2 działek. Idąc dalej w swych hipotetycznych roz...\n",
"3 w Warszawie o stosunkach domowych dziatwy szko...\n",
"4 \\\\'iykład: \"Cywilizacyjna Koncepcja dziejów ¥e...\n",
" ... \n",
"107458 M. (2) na rzecz powoda M. S. kwotę 5003,66 zł ...\n",
"107459 Zintegrowanego Systemu Informatycznego (ZSI), ...\n",
"107460 prokurator. Wyrokowi temu powołując się na prz...\n",
"107461 07 lipca 2010 r. świadczą o tym, że nie wszyst...\n",
"107462 zatem niezdolności do pracy było schorzenie sa...\n",
"Name: 4, Length: 107463, dtype: object"
]
},
"metadata": {},
"execution_count": 12
}
],
"source": [
"x_train = train[4]\n",
"x_train"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"<107463x4401862 sparse matrix of type '<class 'numpy.float64'>'\n",
"\twith 35660756 stored elements in Compressed Sparse Row format>"
]
},
"metadata": {},
"execution_count": 13
}
],
"source": [
"vectorizer = TfidfVectorizer()\n",
"x_train = vectorizer.fit_transform(x_train)\n",
"x_train"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"LinearRegression()"
]
},
"metadata": {},
"execution_count": 15
}
],
"source": [
"model = LinearRegression()\n",
"model.fit(x_train, y_train)"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [],
"source": [
"with open('dev-0/in.tsv', 'rt', encoding=\"utf-8\") as file:\n",
" x_dev0 = pd.read_csv(file, header=None, sep=\"\\t\")[0]"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [],
"source": [
"x_dev0 = vectorizer.transform(x_dev0)\n",
"pred_dev0 = model.predict(x_dev0)"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"19998"
]
},
"metadata": {},
"execution_count": 33
}
],
"source": [
"x_dev0.shape[0]"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {
"tags": [
"outputPrepend"
]
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"a\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\na\
]
}
],
"source": [
"with open(\"dev-0/out.tsv\", \"w\", encoding=\"utf-8\") as file:\n",
" for p in pred_dev0:\n",
" file.write(str(p) + \"\\n\")"
]
}
]
}