Compare commits
1 Commits
Author | SHA1 | Date | |
---|---|---|---|
|
1cc364e2b4 |
5452
dev-0/out.tsv
Normal file
5452
dev-0/out.tsv
Normal file
File diff suppressed because it is too large
Load Diff
33
run.py
Normal file
33
run.py
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
import pandas as pd
|
||||||
|
import numpy as np
|
||||||
|
from sklearn.feature_extraction.text import TfidfVectorizer
|
||||||
|
data_raw = pd.read_csv('train/train.tsv', delimiter = '\t', header = None,usecols=[0, 1], names = ['y', 'X'])
|
||||||
|
|
||||||
|
import string
|
||||||
|
import os
|
||||||
|
|
||||||
|
from sklearn.feature_extraction.text import TfidfVectorizer
|
||||||
|
from stop_words import get_stop_words
|
||||||
|
|
||||||
|
|
||||||
|
def preprocess(item):
|
||||||
|
to_replace = '''~!@#$%^&*()_+-=[]{};\'":?/.>,<1234567890–”’'''
|
||||||
|
for r in to_replace:
|
||||||
|
item = item.replace(r, '')
|
||||||
|
return item.lower()
|
||||||
|
|
||||||
|
|
||||||
|
stop_words = get_stop_words('polish') + ['aby', 'tych', 'tym', 'tyle', 'tymi', 'też']
|
||||||
|
vectorizer = TfidfVectorizer(stop_words=stop_words, preprocessor=preprocess)
|
||||||
|
tfs = vectorizer.fit_transform(data_raw.X)
|
||||||
|
|
||||||
|
from sklearn.naive_bayes import MultinomialNB
|
||||||
|
clf = MultinomialNB()
|
||||||
|
|
||||||
|
clf.fit(tfs,data_raw.y)
|
||||||
|
|
||||||
|
for dir in ['dev-0/', 'test-A/']:
|
||||||
|
test_raw = pd.read_csv(dir+'in.tsv', delimiter = '\t', header = None,usecols=[0], names = ['X'])
|
||||||
|
X_test = vectorizer.transform(test_raw.X)
|
||||||
|
y_predicted = clf.predict(X_test)
|
||||||
|
np.savetxt(dir+"out.tsv", y_predicted, delimiter="\t")
|
5447
test-A/out.tsv
Normal file
5447
test-A/out.tsv
Normal file
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user