retroc2/run.py
2022-05-16 14:48:45 +02:00

100 lines
3.9 KiB
Python

import lzma
from math import sqrt
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LinearRegression
from sklearn.pipeline import make_pipeline
from sklearn.metrics import mean_squared_error
import pandas as pd
import numpy as np
from sklearn.metrics import accuracy_score
#print("lista polskich stop words")
pol_stop_words = ["aczkolwiek","albo","ależ",
"bardziej","bardzo","bowiem","byli","bynajmniej","była","było","były","będzie","będą",
"cali","cała","cały","ciebie","cokolwiek","czasami","czasem","czemu","czyli",
"daleko","dlaczego","dlatego","dobrze","dokąd","dość","dużo","dwaj","dwie","dwoje","dziś","dzisiaj",
"gdyby","gdyż","gdzie","gdziekolwiek","gdzieś",
"inna","inne","inny","innych",
"jakaś","jakby","jaki","jakichś","jakie","jakiś","jakiż","jakkolwiek","jako","jakoś","jeden","jedna","jedno","jednak","jednakże","jego","jej","jemu","jest","jestem","jeszcze","jeśli","jeżeli","już","",
"każdy", "kiedy","kilka","kimś","kto","ktokolwiek","ktoś","która","które","którego","której","który","których","którym","którzy","ku",
"lecz","mają","mimo","między","mnie","mogą","moim","moja","moje","może","możliwe","można","musi",
"nami","nasi","nasz","nasza","nasze","naszego","naszych","natomiast","natychmiast","nawet","nich","niego","niej","niemu","nigdy","nimi",
"obok","około","oraz","owszem",
"pana","pani","podczas","pomimo","ponad","ponieważ","powinien","powinna","powinni","powinno","poza","prawie","przecież","przed","przede","przedtem","przez","przy",
"roku","również","sama", "","skąd","sobie","sobą","sposób","swoje",
"taka","taki","takie","także","tego","teraz","tobą","tobie","toteż","trzeba","tutaj","twoi","twoim","twoja","twoje","twym","twój","tych","tylko",
"wami","wasz","wasza","wasze","według","wiele","wielu","więc","więcej","wszyscy","wszystkich","wszystkie","wszystkim","wszystko","wtedy","właśnie",
"zapewne","zawsze","zeznowu","znów","został","żaden","żadna","żadne","żadnych","żeby"]
def rmse(Y_true, Y_predicted):
return np.sqrt(np.sum((Y_true - Y_predicted)**2)/ len(Y_true))
#print("wczytuję dane")
with lzma.open('train/train.tsv.xz', 'rt', encoding="utf-8") as f:
data = pd.read_csv(f, sep='\t',
names=['Poczatek', 'Koniec', 'Tytul', 'Wyd', 'Tresc'])
#data = data[0:5000]
#print("wyznaczam najlepszy wynik dla roku")
data['Rok'] = (data['Poczatek'].astype(float) + data['Koniec'].astype(float)) / 2
#print("model")
x = data['Tresc']
y = data['Rok']
model = make_pipeline(TfidfVectorizer(stop_words=pol_stop_words), LinearRegression())
model.fit(x, y)
#print("trenowanie modelu")
with open('dev-0/in.tsv', 'r', encoding='utf8') as f:
in_dev0 = f.readlines()
with open('dev-0/expected.tsv', 'r', encoding='utf8') as f:
expected_dev0 = f.readlines()
out_dev0 = model.predict(in_dev0)
with open('dev-0/out.tsv', 'wt') as f:
for predict in out_dev0:
f.write(str(predict)+'\n')
#print("trenowanie dev1")
with open('dev-1/in.tsv', 'r', encoding='utf8') as f:
in_dev1 = f.readlines()
with open('dev-1/expected.tsv', 'r', encoding='utf8') as f:
expected_dev1 = f.readlines()
out_dev1 = model.predict(in_dev1)
with open('dev-1/out.tsv', 'wt') as f:
for prediction in out_dev1:
f.write(str(prediction)+'\n')
#print("predykcja")
with open('test-A/in.tsv', 'r', encoding='utf8') as f:
in_test = f.readlines()
out_test = model.predict(in_test)
with open('test-A/out.tsv', 'wt') as f:
for predict in out_test:
f.write(str(predict)+'\n')
'''print("ewaluacja modelu")
y_true = []
with open("dev-0/expected.tsv", encoding='utf-8') as f:
for line in f.readlines():
y_true.append(line)
y_pred = []
with open("dev-0/out.tsv", encoding='utf-8') as f:
for line in f.readlines():
y_pred.append(line)
print(sqrt(mean_squared_error(y_true,y_pred)))'''