s444417-paranormal-or-skeptic-ISI-public/run.ipynb

import lzma
import sys
from io import StringIO
from sklearn.feature_extraction.text import TfidfVectorizer
import pandas as pd
import numpy

pathX = "./train/in.tsv.xz"
# pathX = "./train/in.tsv"
pathY = "./train/expected.tsv"
nrows = 5000

# data = lzma.open(pathX, mode='rt', encoding='utf-8').read()
# stringIO = StringIO(data)
# df = pd.read_csv(stringIO, sep="\t", header=None)
df = pd.read_csv(pathX, sep='\t', nrows=nrows, header=None)
df = df.drop(df.columns[1], axis=1)
topics = pd.read_csv(pathY, sep='\t', nrows=nrows, header=None)

print(len(df.index))

print(len(topics.index))

5000
5000

df.sample()

	0
2823	Use her own logic against her. Pharmaceutical...

vectorizer = TfidfVectorizer(lowercase=True, stop_words=['english'])
X = vectorizer.fit_transform(df.to_numpy().ravel())
# vectorizer.get_feature_names_out()

# vectorizer.transform("Ala ma kotka".lower().split())

df = df.reset_index()

tfidfVector = vectorizer.transform(df[0])

    

from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import GaussianNB

gnb = GaussianNB()
gnb.fit(tfidfVector.toarray(), topics)

c:\software\python3\lib\site-packages\sklearn\utils\validation.py:63: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  return f(*args, **kwargs)

GaussianNB()

testXPath = "./dev-0/in.tsv.xz"
testYPath = "./dev-0/expected.tsv"

testX = pd.read_csv(testXPath, sep='\t', nrows=nrows, header=None)

testY = pd.read_csv(testYPath, sep='\t', nrows=nrows, header=None)
testXtfidfVector = vectorizer.transform(testX[0])

testXPath = "./dev-0/in.tsv.xz"
testYPath = "./dev-0/out.tsv"

testX = pd.read_csv(testXPath, sep='\t', header=None)

# testY = pd.read_csv(testYPath, sep='\t', nrows=nrows, header=None)
testXtfidfVector = vectorizer.transform(testX[0])

pred = gnb.predict(testXtfidfVector.toarray())
print(pred)

import csv
with open(testYPath, 'w', newline='') as f_output:
    tsv_output = csv.writer(f_output, delimiter='\n')
    tsv_output.writerow(pred)

[0 1 1 ... 0 0 0]