s444417-paranormal-or-skeptic-ISI-public/run.ipynb

import lzma
import sys
from io import StringIO
from sklearn.feature_extraction.text import TfidfVectorizer
import pandas as pd
import numpy

pathX = "./train/in.tsv.xz"
# pathX = "./train/in.tsv"
pathY = "./train/expected.tsv"
nrows = 10000

# data = lzma.open(pathX, mode='rt', encoding='utf-8').read()
# stringIO = StringIO(data)
# df = pd.read_csv(stringIO, sep="\t", header=None)
df = pd.read_csv(pathX, sep='\t', nrows=nrows, header=None)
df = df.drop(df.columns[1], axis=1)
topics = pd.read_csv(pathY, sep='\t', nrows=nrows, header=None)

print(len(df.index))

print(len(topics.index))

10000
10000

df.sample()

	0
8910	What? It isn't a fake memo. It's a real memo. ...

vectorizer = TfidfVectorizer(lowercase=True, stop_words=['english'])
X = vectorizer.fit_transform(df.to_numpy().ravel())
vectorizer.get_feature_names_out()

array(['00', '000', '00000001', ..., 'αsynuclein', 'ಠ_ಠ', 'fibrosis'],
      dtype=object)

# vectorizer.transform("Ala ma kotka".lower().split())

df = df.reset_index()

tfidfVector = vectorizer.transform(df[0])

    

from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import GaussianNB

gnb = GaussianNB()
gnb.fit(tfidfVector.todense(), topics)

c:\Python310\lib\site-packages\sklearn\utils\validation.py:993: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
c:\Python310\lib\site-packages\sklearn\utils\validation.py:593: FutureWarning: np.matrix usage is deprecated in 1.0 and will raise a TypeError in 1.2. Please convert to a numpy array with np.asarray. For more information see: https://numpy.org/doc/stable/reference/generated/numpy.matrix.html
  warnings.warn(

GaussianNB()

testXPath = "./dev-0/in.tsv.xz"
testYPath = "./dev-0/expected.tsv"

testX = pd.read_csv(testXPath, sep='\t', nrows=nrows, header=None)

testY = pd.read_csv(testYPath, sep='\t', nrows=nrows, header=None)
testXtfidfVector = vectorizer.transform(testX[0])

testXPath = "./test-A/in.tsv.xz"
testYPath = "./test-A/expected.tsv"

testX = pd.read_csv(testXPath, sep='\t', nrows=nrows, header=None)

# testY = pd.read_csv(testYPath, sep='\t', nrows=nrows, header=None)
testXtfidfVector = vectorizer.transform(testX[0])

pred = gnb.predict(testXtfidfVector.todense())
print(pred)

import csv
with open(testYPath, 'w', newline='') as f_output:
    tsv_output = csv.writer(f_output, delimiter='\n')
    tsv_output.writerow(pred)

c:\Python310\lib\site-packages\sklearn\utils\validation.py:593: FutureWarning: np.matrix usage is deprecated in 1.0 and will raise a TypeError in 1.2. Please convert to a numpy array with np.asarray. For more information see: https://numpy.org/doc/stable/reference/generated/numpy.matrix.html
  warnings.warn(

[0 1 0 ... 0 0 1]