forked from kubapok/retroc2
39 lines
1.4 KiB
Python
39 lines
1.4 KiB
Python
import string
|
||
import os
|
||
|
||
import pandas as pd
|
||
import numpy as np
|
||
from sklearn.feature_extraction.text import TfidfVectorizer
|
||
from stop_words import get_stop_words
|
||
from sklearn.linear_model import LinearRegression
|
||
from scipy.sparse import vstack
|
||
from sklearn.utils import shuffle
|
||
|
||
data_raw = pd.read_csv('retroc2/train/train.tsv', delimiter = '\t', header = None, names = ['date_from', 'date_to', 'title', 'source', 'text'])
|
||
|
||
def preprocess(item):
|
||
to_replace = '''~!@#$%^&*()_+-=[]{};\'":?/.>,<1234567890–”’'''
|
||
for r in to_replace:
|
||
item = item.replace(r, '')
|
||
return item.lower()
|
||
|
||
|
||
stop_words = get_stop_words('polish') + ['aby', 'tych', 'tym', 'tyle', 'tymi', 'też']
|
||
vectorizer = TfidfVectorizer(stop_words=stop_words, preprocessor=preprocess, max_features=30000, max_df=0.35)
|
||
tfs = vectorizer.fit_transform(data_raw.text)
|
||
data_X = vstack([tfs,tfs])
|
||
|
||
data_y = np.concatenate((data_raw.date_from, data_raw.date_to), axis = 0)
|
||
data_X, data_y = shuffle(data_X, data_y, random_state=42)
|
||
|
||
clf = LinearRegression()
|
||
|
||
clf.fit(data_X,data_y)
|
||
|
||
import csv
|
||
for dir in ['retroc2/dev-0/', 'retroc2/dev-1/', 'retroc2/test-A/']:
|
||
test_raw = pd.read_csv(dir+'in.tsv', delimiter = '\t', header = None, names = ['text'],quoting=csv.QUOTE_NONE)
|
||
vectorized = vectorizer.transform(test_raw.text)
|
||
X_test = vectorized.toarray()
|
||
y_predicted = clf.predict(X_test)
|
||
np.savetxt(dir+"out.tsv", y_predicted, delimiter="\t") |