This commit is contained in:
MatOgr 2022-04-26 01:45:18 +02:00
parent 3fa5102c27
commit ff2f2b124f
4 changed files with 428578 additions and 428542 deletions

274628
dev-0/out.tsv

File diff suppressed because it is too large Load Diff

313212
dev-1/out.tsv

File diff suppressed because it is too large Load Diff

44
run.py
View File

@ -1,5 +1,41 @@
for dataset in 'dev-0', 'dev-1', 'test-A':
with open(f'{dataset}/in.tsv') as f_in, open(f'{dataset}/out.tsv','w') as f_out:
for line in f_in:
f_out.write('0.45\n')
from stop_words import get_stop_words
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
import pandas as pd
import lzma
def read_file(f_name, f_ext=""):
if f_ext == "xz":
# data = []
with lzma.open(f"{f_name}.{f_ext}") as file:
return [line.strip().decode("utf-8") for line in file.readlines()]
with open(f_name, encoding="utf-8") as file:
return [line.strip() for line in file.readlines()]
def write_file(dir_name, preds):
with open(f"{dir_name}\out.tsv", "a", encoding="utf-8") as file_out:
for val in preds:
file_out.writelines(f"{str(val)}")
x_data = read_file("train/in.tsv", "xz")
print(x_data[:5])
y_data = read_file("train/expected.tsv")
print(y_data[:5])
tfidf_vec = TfidfVectorizer(
use_idf=True, max_df=0.95, stop_words=get_stop_words("pl"))
x_vectorized = tfidf_vec.fit_transform(x_data)
model = LogisticRegression(max_iter=1000)
model.fit(x_vectorized, y_data)
# * Predictions
for data_dir in ["dev-0", "dev-1", "test-A"]:
print(f"running for the __{data_dir}__")
x_data = read_file(f"{data_dir}/in.tsv")
x_vectorized = tfidf_vec.transform(x_data)
y_predicted = model.predict(x_vectorized)
write_file(data_dir, y_predicted)

File diff suppressed because it is too large Load Diff