100 lines
3.9 KiB
Python
100 lines
3.9 KiB
Python
import lzma
|
|
from math import sqrt
|
|
from sklearn.feature_extraction.text import TfidfVectorizer
|
|
from sklearn.linear_model import LinearRegression
|
|
from sklearn.pipeline import make_pipeline
|
|
from sklearn.metrics import mean_squared_error
|
|
import pandas as pd
|
|
import numpy as np
|
|
from sklearn.metrics import accuracy_score
|
|
|
|
#print("lista polskich stop words")
|
|
|
|
pol_stop_words = ["aczkolwiek","albo","ależ",
|
|
"bardziej","bardzo","bowiem","byli","bynajmniej","była","było","były","będzie","będą",
|
|
"cali","cała","cały","ciebie","cokolwiek","czasami","czasem","czemu","czyli",
|
|
"daleko","dlaczego","dlatego","dobrze","dokąd","dość","dużo","dwaj","dwie","dwoje","dziś","dzisiaj",
|
|
"gdyby","gdyż","gdzie","gdziekolwiek","gdzieś",
|
|
"inna","inne","inny","innych",
|
|
"jakaś","jakby","jaki","jakichś","jakie","jakiś","jakiż","jakkolwiek","jako","jakoś","jeden","jedna","jedno","jednak","jednakże","jego","jej","jemu","jest","jestem","jeszcze","jeśli","jeżeli","już","ją",
|
|
"każdy", "kiedy","kilka","kimś","kto","ktokolwiek","ktoś","która","które","którego","której","który","których","którym","którzy","ku",
|
|
"lecz","mają","mimo","między","mnie","mogą","moim","moja","moje","może","możliwe","można","musi",
|
|
"nami","nasi","nasz","nasza","nasze","naszego","naszych","natomiast","natychmiast","nawet","nich","niego","niej","niemu","nigdy","nimi",
|
|
"obok","około","oraz","owszem",
|
|
"pana","pani","podczas","pomimo","ponad","ponieważ","powinien","powinna","powinni","powinno","poza","prawie","przecież","przed","przede","przedtem","przez","przy",
|
|
"roku","również","sama", "są","skąd","sobie","sobą","sposób","swoje",
|
|
"taka","taki","takie","także","tego","teraz","tobą","tobie","toteż","trzeba","tutaj","twoi","twoim","twoja","twoje","twym","twój","tych","tylko",
|
|
"wami","wasz","wasza","wasze","według","wiele","wielu","więc","więcej","wszyscy","wszystkich","wszystkie","wszystkim","wszystko","wtedy","właśnie",
|
|
"zapewne","zawsze","zeznowu","znów","został","żaden","żadna","żadne","żadnych","żeby"]
|
|
|
|
def rmse(Y_true, Y_predicted):
|
|
return np.sqrt(np.sum((Y_true - Y_predicted)**2)/ len(Y_true))
|
|
|
|
#print("wczytuję dane")
|
|
with lzma.open('train/train.tsv.xz', 'rt', encoding="utf-8") as f:
|
|
data = pd.read_csv(f, sep='\t',
|
|
names=['Poczatek', 'Koniec', 'Tytul', 'Wyd', 'Tresc'])
|
|
|
|
#data = data[0:5000]
|
|
|
|
#print("wyznaczam najlepszy wynik dla roku")
|
|
data['Rok'] = (data['Poczatek'].astype(float) + data['Koniec'].astype(float)) / 2
|
|
|
|
#print("model")
|
|
x = data['Tresc']
|
|
y = data['Rok']
|
|
|
|
model = make_pipeline(TfidfVectorizer(stop_words=pol_stop_words), LinearRegression())
|
|
model.fit(x, y)
|
|
|
|
#print("trenowanie modelu")
|
|
with open('dev-0/in.tsv', 'r', encoding='utf8') as f:
|
|
in_dev0 = f.readlines()
|
|
|
|
with open('dev-0/expected.tsv', 'r', encoding='utf8') as f:
|
|
expected_dev0 = f.readlines()
|
|
|
|
out_dev0 = model.predict(in_dev0)
|
|
|
|
with open('dev-0/out.tsv', 'wt') as f:
|
|
for predict in out_dev0:
|
|
f.write(str(predict)+'\n')
|
|
|
|
#print("trenowanie dev1")
|
|
|
|
with open('dev-1/in.tsv', 'r', encoding='utf8') as f:
|
|
in_dev1 = f.readlines()
|
|
|
|
with open('dev-1/expected.tsv', 'r', encoding='utf8') as f:
|
|
expected_dev1 = f.readlines()
|
|
|
|
out_dev1 = model.predict(in_dev1)
|
|
|
|
with open('dev-1/out.tsv', 'wt') as f:
|
|
for prediction in out_dev1:
|
|
f.write(str(prediction)+'\n')
|
|
|
|
#print("predykcja")
|
|
|
|
with open('test-A/in.tsv', 'r', encoding='utf8') as f:
|
|
in_test = f.readlines()
|
|
|
|
out_test = model.predict(in_test)
|
|
|
|
with open('test-A/out.tsv', 'wt') as f:
|
|
for predict in out_test:
|
|
f.write(str(predict)+'\n')
|
|
|
|
'''print("ewaluacja modelu")
|
|
|
|
y_true = []
|
|
with open("dev-0/expected.tsv", encoding='utf-8') as f:
|
|
for line in f.readlines():
|
|
y_true.append(line)
|
|
y_pred = []
|
|
with open("dev-0/out.tsv", encoding='utf-8') as f:
|
|
for line in f.readlines():
|
|
y_pred.append(line)
|
|
print(sqrt(mean_squared_error(y_true,y_pred)))'''
|
|
|