99 lines
2.6 KiB
Python
99 lines
2.6 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""Paranormal or skeptic.ipynb
|
|
|
|
Automatically generated by Colaboratory.
|
|
|
|
Original file is located at
|
|
https://colab.research.google.com/drive/1JI_RWapDbABFZPc4NDhU-zQlZiIiXk58
|
|
"""
|
|
|
|
!git clone git://gonito.net/paranormal-or-skeptic
|
|
|
|
"""# Loading Data"""
|
|
|
|
!xzcat train/in.tsv.xz | wc -l
|
|
|
|
import matplotlib.pyplot as plt
|
|
import seaborn as sns
|
|
import pandas as pd
|
|
import numpy as np
|
|
from scipy.sparse import hstack
|
|
import csv
|
|
import datetime
|
|
|
|
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
|
|
from sklearn.pipeline import Pipeline
|
|
from sklearn.linear_model import SGDClassifier, LogisticRegression
|
|
from sklearn.svm import LinearSVC
|
|
from sklearn.naive_bayes import MultinomialNB,ComplementNB,BernoulliNB,GaussianNB
|
|
from sklearn.neural_network import MLPClassifier
|
|
|
|
def load_set(path, isTest):
|
|
dataset = pd.read_csv(path+"/in.tsv.xz", delimiter="\t",header=None,names=["text","date"],quoting=csv.QUOTE_NONE)
|
|
dataset["date"] = pd.to_datetime(dataset["date"].apply(lambda x: datetime.datetime.fromtimestamp(x).isoformat()))
|
|
if not isTest:
|
|
expected = pd.read_csv(path+"/expected.tsv",header=None,names=["class"],dtype="category")
|
|
return dataset, expected
|
|
return dataset
|
|
|
|
"""**Load all sets**"""
|
|
|
|
train_set, expected_train = load_set("train", False)
|
|
dev_set, expected_dev = load_set("dev-0", False)
|
|
test_set = load_set("test-A", True)
|
|
|
|
"""# Prepare data"""
|
|
|
|
def prepare_data(data):
|
|
data["day"] = data["date"].dt.day
|
|
data["month"] = data["date"].dt.month
|
|
data["year"] = data["date"].dt.year
|
|
return data
|
|
|
|
train_set = prepare_data(train_set)
|
|
|
|
train_set.sample(5)
|
|
|
|
"""# Train"""
|
|
|
|
vectorize = CountVectorizer(stop_words='english',ngram_range=(1,3),strip_accents='ascii')
|
|
vectorized = vectorize.fit_transform(train_set["text"])
|
|
|
|
X = vectorized
|
|
y = expected_train["class"]
|
|
|
|
bayes = LogisticRegression(max_iter=1000)
|
|
bayes.fit(X,y)
|
|
|
|
"""# Predict and evaluate"""
|
|
|
|
def predict_data(data):
|
|
prepared = prepare_data(data)
|
|
vectorized = vectorize.transform(data["text"])
|
|
predicted = bayes.predict_proba(vectorized)[:,1]
|
|
predicted[predicted < 0.05] = 0.05
|
|
predicted[predicted > 0.95] = 0.95
|
|
return predicted
|
|
|
|
dev_predicted = predict_data(dev_set)
|
|
|
|
dev_predicted
|
|
|
|
test_predicted = predict_data(test_set)
|
|
|
|
"""**Clean output for saving**"""
|
|
|
|
test_predicted = np.array([item.strip() for item in test_predicted])
|
|
dev_predicted = np.array([item.strip() for item in dev_predicted])
|
|
|
|
"""**Save to file**"""
|
|
|
|
np.savetxt('test-A/out.tsv', test_predicted, '%f')
|
|
np.savetxt('dev-0/out.tsv', dev_predicted, '%f')
|
|
|
|
"""**Check geval output**"""
|
|
|
|
!wget https://gonito.net/get/bin/geval
|
|
!chmod u+x geval
|
|
|
|
!./geval -t "dev-0" |