retroc2/run-checkpoint.ipynb at f130909428fcc4ecab8b97da42ec9e461d39c998

2022-05-18 00:12:24 +02:00

9.7 KiB

Raw Blame History

import os
import pandas as pd
import numpy as np
import sklearn
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error

train = pd.read_csv('train/train.tsv', header=None, sep='\t', error_bad_lines=False)
print(len(train))
train = train.head(2000)

D:\Programy\anaconda3\lib\site-packages\IPython\core\interactiveshell.py:3444: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version.


  exec(code_obj, self.user_global_ns, self.user_ns)

x_train = train[4]
y_train = train[0]

x_dev_data = pd.read_csv('dev-0/in.tsv', header=None, sep='\t')
x_dev = x_dev_data[0]
x_dev[19999] = "to jest tekst testowy"
x_dev[20000] = "a ten tekst jest najbardziej testowy"
y_dev = pd.read_csv('dev-0/expected.tsv', header=None, sep='\t')

vectorizer = TfidfVectorizer()

x_train = vectorizer.fit_transform(x_train)
x_dev = vectorizer.transform(x_dev)

model = LinearRegression()

model.fit(x_train.toarray(), y_train)

LinearRegression()

dev_predicted =  model.predict(x_dev.toarray())

with open('dev-0/out.tsv', 'wt') as f:
    for i in dev_predicted:
        f.write(str(i)+'\n')

dev_out = pd.read_csv('dev-0/out.tsv', header=None, sep='\t')
dev_expected = pd.read_csv('dev-0/expected.tsv', header=None, sep='\t')

[1;31m---------------------------------------------------------------------------[0m
[1;31mMemoryError[0m                               Traceback (most recent call last)
[1;32mC:\Users\SEBAST~1\AppData\Local\Temp/ipykernel_17784/3948937349.py[0m in [0;36m<module>[1;34m[0m
[1;32m----> 1[1;33m [0mdev_predicted[0m [1;33m=[0m  [0mmodel[0m[1;33m.[0m[0mpredict[0m[1;33m([0m[0mx_dev[0m[1;33m.[0m[0mtoarray[0m[1;33m([0m[1;33m)[0m[1;33m)[0m[1;33m[0m[1;33m[0m[0m
[0m[0;32m      2[0m [1;33m[0m[0m
[0;32m      3[0m [1;32mwith[0m [0mopen[0m[1;33m([0m[1;34m'dev-0/out.tsv'[0m[1;33m,[0m [1;34m'wt'[0m[1;33m)[0m [1;32mas[0m [0mf[0m[1;33m:[0m[1;33m[0m[1;33m[0m[0m
[0;32m      4[0m     [1;32mfor[0m [0mi[0m [1;32min[0m [0mdev_predicted[0m[1;33m:[0m[1;33m[0m[1;33m[0m[0m
[0;32m      5[0m         [0mf[0m[1;33m.[0m[0mwrite[0m[1;33m([0m[0mstr[0m[1;33m([0m[0mi[0m[1;33m)[0m[1;33m+[0m[1;34m'\n'[0m[1;33m)[0m[1;33m[0m[1;33m[0m[0m

[1;32mD:\Programy\anaconda3\lib\site-packages\scipy\sparse\compressed.py[0m in [0;36mtoarray[1;34m(self, order, out)[0m
[0;32m   1029[0m         [1;32mif[0m [0mout[0m [1;32mis[0m [1;32mNone[0m [1;32mand[0m [0morder[0m [1;32mis[0m [1;32mNone[0m[1;33m:[0m[1;33m[0m[1;33m[0m[0m
[0;32m   1030[0m             [0morder[0m [1;33m=[0m [0mself[0m[1;33m.[0m[0m_swap[0m[1;33m([0m[1;34m'cf'[0m[1;33m)[0m[1;33m[[0m[1;36m0[0m[1;33m][0m[1;33m[0m[1;33m[0m[0m
[1;32m-> 1031[1;33m         [0mout[0m [1;33m=[0m [0mself[0m[1;33m.[0m[0m_process_toarray_args[0m[1;33m([0m[0morder[0m[1;33m,[0m [0mout[0m[1;33m)[0m[1;33m[0m[1;33m[0m[0m
[0m[0;32m   1032[0m         [1;32mif[0m [1;32mnot[0m [1;33m([0m[0mout[0m[1;33m.[0m[0mflags[0m[1;33m.[0m[0mc_contiguous[0m [1;32mor[0m [0mout[0m[1;33m.[0m[0mflags[0m[1;33m.[0m[0mf_contiguous[0m[1;33m)[0m[1;33m:[0m[1;33m[0m[1;33m[0m[0m
[0;32m   1033[0m             [1;32mraise[0m [0mValueError[0m[1;33m([0m[1;34m'Output array must be C or F contiguous'[0m[1;33m)[0m[1;33m[0m[1;33m[0m[0m

[1;32mD:\Programy\anaconda3\lib\site-packages\scipy\sparse\base.py[0m in [0;36m_process_toarray_args[1;34m(self, order, out)[0m
[0;32m   1200[0m             [1;32mreturn[0m [0mout[0m[1;33m[0m[1;33m[0m[0m
[0;32m   1201[0m         [1;32melse[0m[1;33m:[0m[1;33m[0m[1;33m[0m[0m
[1;32m-> 1202[1;33m             [1;32mreturn[0m [0mnp[0m[1;33m.[0m[0mzeros[0m[1;33m([0m[0mself[0m[1;33m.[0m[0mshape[0m[1;33m,[0m [0mdtype[0m[1;33m=[0m[0mself[0m[1;33m.[0m[0mdtype[0m[1;33m,[0m [0morder[0m[1;33m=[0m[0morder[0m[1;33m)[0m[1;33m[0m[1;33m[0m[0m
[0m[0;32m   1203[0m [1;33m[0m[0m
[0;32m   1204[0m [1;33m[0m[0m

[1;31mMemoryError[0m: Unable to allocate 32.2 GiB for an array with shape (20000, 216394) and data type float64

print(mean_squared_error(dev_out, dev_expected))

with open('test-A/in.tsv', 'r', encoding = 'utf-8') as f:
       x_test = f.readlines()
        
x_test = pd.Series(x_test)
x_test = vectorizer.transform(x_test)

test_predicted = model.predict(x_test.toarray())

with open('test-A/out.tsv', 'wt') as f:
    for i in test_predicted:
        f.write(str(i)+'\n')

!jupyter nbconvert --to script run.ipynb

9.7 KiB Raw Blame History Unescape Escape

9.7 KiB

Raw Blame History