Logistic regression
This commit is contained in:
parent
b775a221e6
commit
a81ee51e09
54
classifier.py
Normal file
54
classifier.py
Normal file
@ -0,0 +1,54 @@
|
|||||||
|
from sklearn.feature_extraction.text import TfidfVectorizer
|
||||||
|
from sklearn.linear_model import LogisticRegression
|
||||||
|
import lzma
|
||||||
|
|
||||||
|
X_train = []
|
||||||
|
Y_train = []
|
||||||
|
|
||||||
|
print("Reading train_in...")
|
||||||
|
with lzma.open('train/in.tsv.xz', 'rt', encoding="utf-8") as train_in:
|
||||||
|
for line in train_in:
|
||||||
|
text = line.strip()
|
||||||
|
X_train.append(text)
|
||||||
|
|
||||||
|
print("Reading train_expected")
|
||||||
|
with open('train/expected.tsv', 'rt') as train_expected:
|
||||||
|
for line in train_expected:
|
||||||
|
text = line.strip()
|
||||||
|
Y_train.append(int(text))
|
||||||
|
|
||||||
|
print("Training TFIDF...")
|
||||||
|
vectorizer = TfidfVectorizer(decode_error="replace", stop_words="english", max_df=0.8, sublinear_tf=True)
|
||||||
|
X_train = vectorizer.fit_transform(X_train)
|
||||||
|
|
||||||
|
print("Training...")
|
||||||
|
model = LogisticRegression()
|
||||||
|
model.fit(X_train, Y_train)
|
||||||
|
|
||||||
|
print("Predicting dev...")
|
||||||
|
X_dev = []
|
||||||
|
with open('dev-0/in.tsv', 'r', encoding="utf-8") as dev_in:
|
||||||
|
for line in dev_in:
|
||||||
|
text = line.split("\t")[0].strip()
|
||||||
|
X_dev.append(text)
|
||||||
|
|
||||||
|
X_dev = vectorizer.transform(X_dev)
|
||||||
|
predictions = model.predict(X_dev)
|
||||||
|
|
||||||
|
with open("dev-0/out.tsv", "w") as out_file:
|
||||||
|
for pred in predictions:
|
||||||
|
out_file.write(str(pred) + "\n")
|
||||||
|
|
||||||
|
print("Predicting test...")
|
||||||
|
X_test = []
|
||||||
|
with open('test-A/in.tsv', 'r', encoding="utf-8") as test_in:
|
||||||
|
for line in test_in:
|
||||||
|
text = line.split("\t")[0].strip()
|
||||||
|
X_test.append(text)
|
||||||
|
|
||||||
|
X_test = vectorizer.transform(X_test)
|
||||||
|
predictions = model.predict(X_test)
|
||||||
|
|
||||||
|
with open("test-A/out.tsv", "w") as out_file:
|
||||||
|
for pred in predictions:
|
||||||
|
out_file.write(str(pred)+"\n")
|
137314
dev-0/out.tsv
Normal file
137314
dev-0/out.tsv
Normal file
File diff suppressed because it is too large
Load Diff
134618
test-A/out.tsv
Normal file
134618
test-A/out.tsv
Normal file
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user