forked from kubapok/retroc2
1k train data
This commit is contained in:
parent
ee3f9379e0
commit
0faa3da61f
@ -2,7 +2,7 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"execution_count": 1,
|
||||
"id": "d103a6c5-a9b4-4547-9e10-f384d716972d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@ -12,7 +12,6 @@
|
||||
"import numpy as np\n",
|
||||
"import sklearn\n",
|
||||
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
|
||||
"from sklearn.pipeline import make_pipeline\n",
|
||||
"from sklearn.linear_model import LinearRegression\n",
|
||||
"from sklearn.metrics import mean_squared_error"
|
||||
]
|
||||
@ -32,22 +31,19 @@
|
||||
"\n",
|
||||
" exec(code_obj, self.user_global_ns, self.user_ns)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"107463\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"train = pd.read_csv('train/train.tsv', header=None, sep='\\t', error_bad_lines=False)\n",
|
||||
"train = train.head(500)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e4b5f917-bde7-4b69-a394-1ab0fe0b752a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# with open('train/train.tsv', 'r', encoding='utf8') as file:\n",
|
||||
"# train = pd.read_csv(file, sep='\\t')"
|
||||
"print(len(train))\n",
|
||||
"train = train.head(1000)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -61,16 +57,6 @@
|
||||
"y_train = train[0]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5faa4b35-ccf7-4656-a08a-99d1d96d8a21",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(x_train)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
@ -80,9 +66,9 @@
|
||||
"source": [
|
||||
"x_dev_data = pd.read_csv('dev-0/in.tsv', header=None, sep='\\t')\n",
|
||||
"x_dev = x_dev_data[0]\n",
|
||||
"y_dev = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')\n",
|
||||
"# x_dev.head(1000)\n",
|
||||
"# y_dev.head(1000)"
|
||||
"x_dev[19999] = \"to jest tekst testowy\"\n",
|
||||
"x_dev[20000] = \"a ten tekst jest najbardziej testowy\"\n",
|
||||
"y_dev = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -151,25 +137,12 @@
|
||||
" f.write(str(i)+'\\n')\n",
|
||||
"\n",
|
||||
"dev_out = pd.read_csv('dev-0/out.tsv', header=None, sep='\\t')\n",
|
||||
"# dev_out = dev_out.head(1000)\n",
|
||||
"dev_expected = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')\n",
|
||||
"# dev_expected = dev_expected.head(1000)\n",
|
||||
"# print(mean_squared_error(dev_out, dev_expected))\n"
|
||||
"dev_expected = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"id": "7b265b2b-cac1-457c-80f9-6f6dec30045b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"dev_expected = dev_expected.head(19998)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"execution_count": 10,
|
||||
"id": "223de995-5e91-4254-9214-4fc871c985e9",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
@ -177,7 +150,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"3579.7645467601897\n"
|
||||
"3486.2683285642797\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -187,7 +160,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"execution_count": 11,
|
||||
"id": "3bc8418b-64f1-4163-a0ec-8e3293032341",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@ -204,6 +177,25 @@
|
||||
" for i in test_predicted:\n",
|
||||
" f.write(str(i)+'\\n')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "a18aea56-7fa1-40bd-8aa3-bbaf9d66d6b7",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[NbConvertApp] Converting notebook run.ipynb to script\n",
|
||||
"[NbConvertApp] Writing 1629 bytes to run.py\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"!jupyter nbconvert --to script run.ipynb"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
39998
dev-0/out.tsv
39998
dev-0/out.tsv
File diff suppressed because it is too large
Load Diff
80
run.ipynb
80
run.ipynb
@ -2,7 +2,7 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"execution_count": 1,
|
||||
"id": "d103a6c5-a9b4-4547-9e10-f384d716972d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@ -12,7 +12,6 @@
|
||||
"import numpy as np\n",
|
||||
"import sklearn\n",
|
||||
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
|
||||
"from sklearn.pipeline import make_pipeline\n",
|
||||
"from sklearn.linear_model import LinearRegression\n",
|
||||
"from sklearn.metrics import mean_squared_error"
|
||||
]
|
||||
@ -32,22 +31,19 @@
|
||||
"\n",
|
||||
" exec(code_obj, self.user_global_ns, self.user_ns)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"107463\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"train = pd.read_csv('train/train.tsv', header=None, sep='\\t', error_bad_lines=False)\n",
|
||||
"train = train.head(500)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e4b5f917-bde7-4b69-a394-1ab0fe0b752a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# with open('train/train.tsv', 'r', encoding='utf8') as file:\n",
|
||||
"# train = pd.read_csv(file, sep='\\t')"
|
||||
"print(len(train))\n",
|
||||
"train = train.head(1000)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -61,16 +57,6 @@
|
||||
"y_train = train[0]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5faa4b35-ccf7-4656-a08a-99d1d96d8a21",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(x_train)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
@ -80,9 +66,9 @@
|
||||
"source": [
|
||||
"x_dev_data = pd.read_csv('dev-0/in.tsv', header=None, sep='\\t')\n",
|
||||
"x_dev = x_dev_data[0]\n",
|
||||
"y_dev = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')\n",
|
||||
"# x_dev.head(1000)\n",
|
||||
"# y_dev.head(1000)"
|
||||
"x_dev[19999] = \"to jest tekst testowy\"\n",
|
||||
"x_dev[20000] = \"a ten tekst jest najbardziej testowy\"\n",
|
||||
"y_dev = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -151,25 +137,12 @@
|
||||
" f.write(str(i)+'\\n')\n",
|
||||
"\n",
|
||||
"dev_out = pd.read_csv('dev-0/out.tsv', header=None, sep='\\t')\n",
|
||||
"# dev_out = dev_out.head(1000)\n",
|
||||
"dev_expected = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')\n",
|
||||
"# dev_expected = dev_expected.head(1000)\n",
|
||||
"# print(mean_squared_error(dev_out, dev_expected))\n"
|
||||
"dev_expected = pd.read_csv('dev-0/expected.tsv', header=None, sep='\\t')\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"id": "7b265b2b-cac1-457c-80f9-6f6dec30045b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"dev_expected = dev_expected.head(19998)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"execution_count": 10,
|
||||
"id": "223de995-5e91-4254-9214-4fc871c985e9",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
@ -177,7 +150,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"3579.7645467601897\n"
|
||||
"3486.2683285642797\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -187,7 +160,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"execution_count": 11,
|
||||
"id": "3bc8418b-64f1-4163-a0ec-8e3293032341",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@ -204,6 +177,25 @@
|
||||
" for i in test_predicted:\n",
|
||||
" f.write(str(i)+'\\n')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "a18aea56-7fa1-40bd-8aa3-bbaf9d66d6b7",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[NbConvertApp] Converting notebook run.ipynb to script\n",
|
||||
"[NbConvertApp] Writing 1629 bytes to run.py\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"!jupyter nbconvert --to script run.ipynb"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
99
run.py
Normal file
99
run.py
Normal file
@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[1]:
|
||||
|
||||
|
||||
import os
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import sklearn
|
||||
from sklearn.feature_extraction.text import TfidfVectorizer
|
||||
from sklearn.linear_model import LinearRegression
|
||||
from sklearn.metrics import mean_squared_error
|
||||
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
train = pd.read_csv('train/train.tsv', header=None, sep='\t', error_bad_lines=False)
|
||||
print(len(train))
|
||||
train = train.head(1000)
|
||||
|
||||
|
||||
# In[3]:
|
||||
|
||||
|
||||
x_train = train[4]
|
||||
y_train = train[0]
|
||||
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
x_dev_data = pd.read_csv('dev-0/in.tsv', header=None, sep='\t')
|
||||
x_dev = x_dev_data[0]
|
||||
x_dev[19999] = "to jest tekst testowy"
|
||||
x_dev[20000] = "a ten tekst jest najbardziej testowy"
|
||||
y_dev = pd.read_csv('dev-0/expected.tsv', header=None, sep='\t')
|
||||
|
||||
|
||||
# In[5]:
|
||||
|
||||
|
||||
vectorizer = TfidfVectorizer()
|
||||
|
||||
|
||||
# In[6]:
|
||||
|
||||
|
||||
x_train = vectorizer.fit_transform(x_train)
|
||||
x_dev = vectorizer.transform(x_dev)
|
||||
|
||||
|
||||
# In[7]:
|
||||
|
||||
|
||||
model = LinearRegression()
|
||||
|
||||
|
||||
# In[8]:
|
||||
|
||||
|
||||
model.fit(x_train.toarray(), y_train)
|
||||
|
||||
|
||||
# In[9]:
|
||||
|
||||
|
||||
dev_predicted = model.predict(x_dev.toarray())
|
||||
|
||||
with open('dev-0/out.tsv', 'wt') as f:
|
||||
for i in dev_predicted:
|
||||
f.write(str(i)+'\n')
|
||||
|
||||
dev_out = pd.read_csv('dev-0/out.tsv', header=None, sep='\t')
|
||||
dev_expected = pd.read_csv('dev-0/expected.tsv', header=None, sep='\t')
|
||||
|
||||
|
||||
# In[10]:
|
||||
|
||||
|
||||
print(mean_squared_error(dev_out, dev_expected))
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
with open('test-A/in.tsv', 'r', encoding = 'utf-8') as f:
|
||||
x_test = f.readlines()
|
||||
|
||||
x_test = pd.Series(x_test)
|
||||
x_test = vectorizer.transform(x_test)
|
||||
|
||||
test_predicted = model.predict(x_test.toarray())
|
||||
|
||||
with open('test-A/out.tsv', 'wt') as f:
|
||||
for i in test_predicted:
|
||||
f.write(str(i)+'\n')
|
||||
|
||||
28440
test-A/out.tsv
28440
test-A/out.tsv
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user