retroc2/.ipynb_checkpoints/run-checkpoint.ipynb

import os
import pandas as pd
import numpy as np
import sklearn
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
from sklearn.pipeline import make_pipeline

train = pd.read_csv('train/train.tsv', header=None, sep='\t', error_bad_lines=False)
print(len(train))
train = train[:30000]

D:\Programy\anaconda3\lib\site-packages\IPython\core\interactiveshell.py:3444: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version.


  exec(code_obj, self.user_global_ns, self.user_ns)

107463

x_train = train[4]
y_train = train[0]

model = make_pipeline(TfidfVectorizer(), LinearRegression())
model.fit(x_train, y_train)

Pipeline(steps=[('tfidfvectorizer', TfidfVectorizer()),
                ('linearregression', LinearRegression())])

def readFile(filename):
    result = []
    with open(filename, 'r', encoding="utf-8") as file:
        for line in file:
            text = line.split("\t")[0].strip()
            result.append(text)
    return result

x_dev0 = readFile('dev-0/in.tsv')
dev_predicted =  model.predict(x_dev0)
with open('dev-0/out.tsv', 'wt') as f:
    for i in dev_predicted:
        f.write(str(i)+'\n')

x_dev1 = readFile('dev-1/in.tsv')
dev_predicted =  model.predict(x_dev1)
with open('dev-1/out.tsv', 'wt') as f:
    for i in dev_predicted:
        f.write(str(i)+'\n')

with open('test-A/in.tsv', 'r', encoding = 'utf-8') as f:
       x_test = f.readlines()
        
# x_test = pd.Series(x_test)
# x_test = vectorizer.transform(x_test)

test_predicted = model.predict(x_test)

with open('test-A/out.tsv', 'wt') as f:
    for i in test_predicted:
        f.write(str(i)+'\n')

!jupyter nbconvert --to script run.ipynb

[NbConvertApp] Converting notebook run.ipynb to script
[NbConvertApp] Writing 1597 bytes to run.py