91 lines
1.5 KiB
Python
91 lines
1.5 KiB
Python
|
#!/usr/bin/env python
|
||
|
|
# coding: utf-8
|
||
|
|
|
||
|
|
# In[1]:
|
||
|
|
|
||
|
|
|
||
|
|
import numpy as np
|
||
|
|
import pandas as pd
|
||
|
|
import sklearn
|
||
|
|
from sklearn.naive_bayes import GaussianNB
|
||
|
|
from sklearn.feature_extraction.text import TfidfVectorizer
|
||
|
|
from sklearn.metrics import accuracy_score
|
||
|
|
|
||
|
|
|
||
|
|
# In[2]:
|
||
|
|
|
||
|
|
|
||
|
|
train = pd.read_csv('train/train.tsv', header=None, sep='\t', error_bad_lines=False)
|
||
|
|
train = train.head(2000)
|
||
|
|
|
||
|
|
|
||
|
|
# In[3]:
|
||
|
|
|
||
|
|
|
||
|
|
x_train = train[1]
|
||
|
|
y_train = train[0]
|
||
|
|
|
||
|
|
|
||
|
|
# In[4]:
|
||
|
|
|
||
|
|
|
||
|
|
x_dev = pd.read_csv('dev-0/in.tsv', header=None, sep='\t', error_bad_lines=False)
|
||
|
|
x_dev = x_dev[0]
|
||
|
|
y_dev = pd.read_csv('dev-0/expected.tsv', header=None, sep='\t', error_bad_lines=False)
|
||
|
|
|
||
|
|
|
||
|
|
# In[5]:
|
||
|
|
|
||
|
|
|
||
|
|
vectorizer = TfidfVectorizer()
|
||
|
|
|
||
|
|
|
||
|
|
# In[6]:
|
||
|
|
|
||
|
|
|
||
|
|
x_train = vectorizer.fit_transform(x_train)
|
||
|
|
x_dev = vectorizer.transform(x_dev)
|
||
|
|
|
||
|
|
|
||
|
|
# In[7]:
|
||
|
|
|
||
|
|
|
||
|
|
gnb = GaussianNB()
|
||
|
|
|
||
|
|
|
||
|
|
# In[8]:
|
||
|
|
|
||
|
|
|
||
|
|
gnb.fit(x_train.toarray(), y_train)
|
||
|
|
|
||
|
|
|
||
|
|
# In[9]:
|
||
|
|
|
||
|
|
|
||
|
|
dev_predicted = gnb.predict(x_dev.toarray())
|
||
|
|
|
||
|
|
with open('dev-0/out.tsv', 'wt') as f:
|
||
|
|
for i in dev_predicted:
|
||
|
|
f.write(str(i)+'\n')
|
||
|
|
|
||
|
|
dev_out = pd.read_csv('dev-0/out.tsv', header=None, sep='\t')
|
||
|
|
dev_expected = pd.read_csv('dev-0/expected.tsv', header=None, sep='\t')
|
||
|
|
print(accuracy_score(dev_out, dev_expected))
|
||
|
|
|
||
|
|
|
||
|
|
# In[10]:
|
||
|
|
|
||
|
|
|
||
|
|
with open('test-A/in.tsv', 'r', encoding = 'utf-8') as f:
|
||
|
|
x_test = f.readlines()
|
||
|
|
|
||
|
|
x_test = pd.Series(x_test)
|
||
|
|
x_test = vectorizer.transform(x_test)
|
||
|
|
|
||
|
|
test_predicted = gnb.predict(x_test.toarray())
|
||
|
|
|
||
|
|
with open('test-A/out.tsv', 'wt') as f:
|
||
|
|
for i in test_predicted:
|
||
|
|
f.write(str(i)+'\n')
|
||
|
|
|