2021-05-12 22:10:35 +02:00

33 lines
1.1 KiB
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
data_raw = pd.read_csv('train/train.tsv', delimiter = '\t', header = None,usecols=[0, 1], names = ['y', 'X'])
import string
import os
from sklearn.feature_extraction.text import TfidfVectorizer
from stop_words import get_stop_words
def preprocess(item):
to_replace = '''~!@#$%^&*()_+-=[]{};\'":?/.>,<1234567890'''
for r in to_replace:
item = item.replace(r, '')
return item.lower()
stop_words = get_stop_words('polish') + ['aby', 'tych', 'tym', 'tyle', 'tymi', 'też']
vectorizer = TfidfVectorizer(stop_words=stop_words, preprocessor=preprocess)
tfs = vectorizer.fit_transform(data_raw.X)
from sklearn.naive_bayes import MultinomialNB
clf = MultinomialNB(),data_raw.y)
for dir in ['dev-0/', 'test-A/']:
test_raw = pd.read_csv(dir+'in.tsv', delimiter = '\t', header = None,usecols=[0], names = ['X'])
X_test = vectorizer.transform(test_raw.X)
y_predicted = clf.predict(X_test)
np.savetxt(dir+"out.tsv", y_predicted, delimiter="\t")