43 lines
1.7 KiB
Python
43 lines
1.7 KiB
Python
import csv
|
|
import pickle
|
|
from typing import re
|
|
|
|
import numpy
|
|
import numpy as np
|
|
import pandas as pd
|
|
from sklearn.decomposition import PCA
|
|
from sklearn.feature_extraction.text import TfidfVectorizer
|
|
from sklearn.decomposition import TruncatedSVD
|
|
|
|
def predict():
|
|
input_file = open("l_regression.pkl",'rb')
|
|
l_regression = pickle.load(input_file)
|
|
input_file = open("tfidf_model.pkl",'rb')
|
|
tfidf = pickle.load(input_file,encoding='UTF-8')
|
|
|
|
dev0 = pd.read_csv("dev-0/in.tsv", delimiter="\t", header=None, names=["txt"], quoting=csv.QUOTE_NONE, error_bad_lines=False, skip_blank_lines=False)
|
|
testA = pd.read_csv("test-A/in.tsv", delimiter="\t", header=None, names=["txt"], quoting=csv.QUOTE_NONE, error_bad_lines=False, skip_blank_lines=False )
|
|
#devtxt = dev0["txt"]
|
|
#testAtxt = testA["txt"]
|
|
|
|
#print(testAtxt)
|
|
|
|
dev0_vector = tfidf.fit_transform(dev0['txt'].apply(lambda dev0_vector: np.str_(dev0_vector)))
|
|
testA_vector = tfidf.fit_transform(testA['txt'].apply(lambda testA_vector: np.str_(testA_vector)))
|
|
#dev0_vector = tfidf.fit_transform(dev0['txt'].values.astype('U'))
|
|
#testA_vector = tfidf.fit_transform(testA['txt'].values.astype('U'))
|
|
#dev0_vector = tfidf.fit_transform(dev0['txt'],y=None)
|
|
#testA_vector = tfidf.fit_transform(testA['txt'],y=None)
|
|
#print(testA_vector)
|
|
pca = TruncatedSVD(n_components=120)
|
|
|
|
dev0_pca = pca.fit_transform(dev0_vector)
|
|
testA_pca = pca.fit_transform(testA_vector)
|
|
y_dev = l_regression.predict(dev0_pca)
|
|
y_test = l_regression.predict(testA_pca)
|
|
numpy.savetxt('dev-0/out.tsv', y_dev)
|
|
numpy.savetxt('test-A/out.tsv', y_test)
|
|
#y_dev.to_csv(r'dev-0/out.csv',index=False)
|
|
#y_test.to_csv(r'test-A/out.csv',index=False)
|
|
|
|
predict() |