forked from kubapok/retroc2
3.9 KiB
3.9 KiB
import os
import sklearn
import pandas as pd
from gzip import open as open_gz
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
def predict_year(x, path_out, model):
results = model.predict(x)
with open(path_out, 'wt') as file:
for r in results:
file.write(str(r) + '\n')
def read_file(filename):
result = []
with open(filename, 'r', encoding="utf-8") as file:
for line in file:
text = line.split("\t")[0].strip()
result.append(text)
return result
with open('train/train.tsv', 'r', encoding='utf8') as file:
train = pd.read_csv(file, sep='\t', names=['Begin', 'End', 'Title', 'Author', 'Text'])
train = train[0:12000]
train_x = train['Text']
#train['Date'] = (train['Date1'].astype(float) + train['Date2'].astype(float))/2
train_y = train['Begin']
model = make_pipeline(TfidfVectorizer(), LinearRegression())
model.fit(train_x, train_y)
Pipeline(steps=[('tfidfvectorizer', TfidfVectorizer()), ('linearregression', LinearRegression())])
x_dev_0 = read_file('dev-0/in.tsv')
predict_year(x_dev_0, 'dev-0/out.tsv', model)
x_dev_1 = read_file('dev-1/in.tsv')
predict_year(x_dev_1,'dev-1/out.tsv', model)
x_test = read_file('test-A/in.tsv')
predict_year(x_test,'test-A/out.tsv', model)
#y_dev = pd.read_csv('dev-0/out.tsv',header = None, sep = '/t',engine = 'python')
#y_dev = y_dev[0]
#y_dev_exp = pd.read_csv('dev-0/expected.tsv',header = None, sep = '/t',engine = 'python')
#y_dev_exp = y_dev_exp[0]
#RMSE_dev = mean_squared_error(y_dev_exp, y_dev)