sport-text-classification-ball-ISI-public/.ipynb_checkpoints/run-checkpoint.ipynb

import numpy as np
import pandas as pd
import sklearn
from sklearn.naive_bayes import GaussianNB
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import accuracy_score

train = pd.read_csv('train/train.tsv', header=None, sep='\t', error_bad_lines=False)
train = train.head(2000)

D:\Programy\anaconda3\lib\site-packages\IPython\core\interactiveshell.py:3444: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version.


  exec(code_obj, self.user_global_ns, self.user_ns)
b'Skipping line 25706: expected 2 fields, saw 3\nSkipping line 58881: expected 2 fields, saw 3\nSkipping line 73761: expected 2 fields, saw 3\n'

x_train = train[1]
y_train = train[0]

x_dev = pd.read_csv('dev-0/in.tsv', header=None, sep='\t', error_bad_lines=False)
x_dev = x_dev[0]
y_dev = pd.read_csv('dev-0/expected.tsv', header=None, sep='\t', error_bad_lines=False)

vectorizer = TfidfVectorizer()

x_train = vectorizer.fit_transform(x_train)
x_dev = vectorizer.transform(x_dev)

gnb = GaussianNB()

gnb.fit(x_train.toarray(), y_train)

GaussianNB()

dev_predicted =  gnb.predict(x_dev.toarray())

with open('dev-0/out.tsv', 'wt') as f:
    for i in dev_predicted:
        f.write(str(i)+'\n')

dev_out = pd.read_csv('dev-0/out.tsv', header=None, sep='\t')
dev_expected = pd.read_csv('dev-0/expected.tsv', header=None, sep='\t')
print(accuracy_score(dev_out, dev_expected))

0.9418561995597946

with open('test-A/in.tsv', 'r', encoding = 'utf-8') as f:
       x_test = f.readlines()
        
x_test = pd.Series(x_test)
x_test = vectorizer.transform(x_test)

test_predicted = gnb.predict(x_test.toarray())

with open('test-A/out.tsv', 'wt') as f:
    for i in test_predicted:
        f.write(str(i)+'\n')

!jupyter nbconvert --to script run.ipynb

[NbConvertApp] Converting notebook run.ipynb to script
[NbConvertApp] Writing 1502 bytes to run.py