paranormal-or-skeptic-ISI-public-s416089/.ipynb_checkpoints/solution-checkpoint.ipynb

import numpy as np
import pandas as pd
import patoolib
import os
import patoolib
from sklearn.preprocessing import LabelEncoder
from sklearn.naive_bayes import GaussianNB, MultinomialNB
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import TfidfVectorizer

TRENING

ROZPAKOWANIE I WCZYTANIE

EXPECTED_FILE = open('train/expected.tsv', 'r', encoding="utf-8")

patoolib.extract_archive("train/in.tsv.xz", outdir="train/")
TRAIN = open('train/in.tsv', 'r', encoding="utf-8")

patool: Extracting train/in.tsv.xz ...
patool: running ""C:\Program Files\Git\mingw64\bin\xz.EXE"" -c -d -- train/in.tsv.xz > train/in.tsv
patool:     with shell=True
patool: ... train/in.tsv.xz extracted to `train/'.

WRZUCENIE DO ZMIENNYCH

EXPECTED = []
for line in EXPECTED_FILE:
    EXPECTED.append(line)

TRAIN_DATA = []
for line in TRAIN:
    TRAIN_DATA.append(line)

ZAMKNIECIE ZMIENNYCH PLIKOW I USUNIECIE ROZPAKOWANIA

EXPECTED_FILE.close()
TRAIN.close()
#os.remove("train/in.tsv")

MODEL TRENINGOWY

EXPECTED_ENCODER = LabelEncoder().fit_transform(EXPECTED)

PIPE = Pipeline(steps=[("TF-IDF",TfidfVectorizer()), ("BAYES", MultinomialNB())])

TRAIN_MODEL = PIPE.fit(TRAIN_DATA, EXPECTED_ENCODER)

FUNKCJE

def BayesFit(MODEL, DOC):
    PREDICTION =  MODEL.predict(DOC)
    return PREDICTION

PLIK DEV-0

patoolib.extract_archive("dev-0/in.tsv.xz", outdir="dev-0/")
INFILE = open('dev-0/in.tsv', 'r', encoding="utf-8")

OUTFILE = open("dev-0/out.tsv", "w")

patool: Extracting dev-0/in.tsv.xz ...
patool: running ""C:\Program Files\Git\mingw64\bin\xz.EXE"" -c -d -- dev-0/in.tsv.xz > dev-0/in.tsv
patool:     with shell=True
patool: ... dev-0/in.tsv.xz extracted to `dev-0/'.

ALL_DOC = INFILE.readlines()

RESULT = BayesFit(TRAIN_MODEL, ALL_DOC)

for x in RESULT:
    OUTFILE.write(str(x) + '\n')

INFILE.close()
OUTFILE.close()
#os.remove("dev-0/in.tsv")

PLIK TEST-A

patoolib.extract_archive("test-A/in.tsv.xz", outdir="test-A/")
INFILE = open('test-A/in.tsv', 'r', encoding="utf-8")

OUTFILE = open("test-A/out.tsv", "w")

patool: Extracting test-A/in.tsv.xz ...
patool: running ""C:\Program Files\Git\mingw64\bin\xz.EXE"" -c -d -- test-A/in.tsv.xz > test-A/in.tsv
patool:     with shell=True
patool: ... test-A/in.tsv.xz extracted to `test-A/'.

ALL_DOC = INFILE.readlines()

RESULT = BayesFit(TRAIN_MODEL, ALL_DOC)

for x in RESULT:
    OUTFILE.write(str(x) + '\n')

INFILE.close()
OUTFILE.close()
#os.remove("test-A/in.tsv")