From 078695a814957ffde843ab6b3cd00eaa8d4d5bf3 Mon Sep 17 00:00:00 2001
From: Zofia Galla <zosiagalla@gmail.com>
Date: Wed, 19 May 2021 14:08:42 +0200
Subject: [PATCH] add run.py

---
 run.py | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 run.py

diff --git a/run.py b/run.py
new file mode 100644
index 0000000..5df9af2
--- /dev/null
+++ b/run.py
@@ -0,0 +1,39 @@
+import string
+import os
+
+import pandas as pd
+import numpy as np
+from sklearn.feature_extraction.text import TfidfVectorizer
+from stop_words import get_stop_words
+from sklearn.linear_model import LinearRegression
+from scipy.sparse import vstack
+from sklearn.utils import shuffle
+
+data_raw = pd.read_csv('retroc2/train/train.tsv', delimiter = '\t', header = None, names = ['date_from', 'date_to', 'title', 'source', 'text'])
+
+def preprocess(item):
+    to_replace = '''~!@#$%^&*()_+-=[]{};\'":?/.>,<1234567890–”’'''
+    for r in to_replace:
+      item = item.replace(r, '')
+    return item.lower()
+
+
+stop_words = get_stop_words('polish') + ['aby', 'tych', 'tym', 'tyle', 'tymi', 'też']
+vectorizer = TfidfVectorizer(stop_words=stop_words, preprocessor=preprocess, max_features=30000, max_df=0.35)
+tfs = vectorizer.fit_transform(data_raw.text)
+data_X = vstack([tfs,tfs])
+
+data_y = np.concatenate((data_raw.date_from, data_raw.date_to), axis = 0)
+data_X, data_y = shuffle(data_X, data_y, random_state=42)
+
+clf = LinearRegression()
+
+clf.fit(data_X,data_y)
+
+import csv
+for dir in ['retroc2/dev-0/', 'retroc2/dev-1/', 'retroc2/test-A/']:
+  test_raw = pd.read_csv(dir+'in.tsv', delimiter = '\t', header = None, names = ['text'],quoting=csv.QUOTE_NONE)
+  vectorized = vectorizer.transform(test_raw.text)
+  X_test = vectorized.toarray()
+  y_predicted = clf.predict(X_test)
+  np.savetxt(dir+"out.tsv", y_predicted, delimiter="\t")
\ No newline at end of file