3.7 KiB
3.7 KiB
import pandas as pd
import numpy as np
from nltk.tokenize import word_tokenize
from nltk import pos_tag
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from sklearn.preprocessing import LabelEncoder
from collections import defaultdict
from nltk.corpus import wordnet as wn
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import model_selection, naive_bayes, svm
from sklearn.metrics import accuracy_score
from sklearn.pipeline import make_pipeline
with open("train/in.tsv", encoding='utf-8') as f:
x_train = f.readlines()
with open("train/expected.tsv", encoding='utf-8') as f:
y_train = f.readlines()
len(x_train)
289579
np.random.seed(500)
with open("svm/in.tsv", encoding='utf-8') as f:
x_dev = f.readlines()
len(y_train)
289579
y_train = LabelEncoder().fit_transform(y_train)
y_train
array([2, 0, 0, ..., 0, 0, 1], dtype=int64)
pipeline = make_pipeline(TfidfVectorizer(),svm.SVC())
model = pipeline.fit(x_train, y_train)
prediction = model.predict(x_dev)
np.savetxt("svm/outfull.tsv", prediction, fmt='%d')