maybe better

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/retroc2.iml" filepath="$PROJECT_DIR$/.idea/retroc2.iml" />
+    </modules>
+  </component>
+</project>

.idea/retroc2.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

run.py

@@ -0,0 +1,51 @@
+import lzma
+import math
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.linear_model import LinearRegression
+from sklearn.pipeline import make_pipeline
+from sklearn.metrics import mean_squared_error
+import pandas as pd
+
+X_train = []
+Y_train = []
+
+stop = 0
+
+with lzma.open('train/train.tsv.xz', 'rt', encoding="utf-8") as f:
+    data = pd.read_csv(f, sep='\t', names=['Begin', 'End', 'Text'])
+
+
+data = data[['Text', 'End']]
+data = data[0:50000]
+
+X = data['Text']
+y = data['End']
+
+model = make_pipeline(TfidfVectorizer(), LinearRegression())
+model.fit(X, y)
+
+
+def readFile(filename):
+    X_dev = []
+    with open(filename, 'r', encoding="utf-8") as dev_in:
+        for line in dev_in:
+            text = line.split("\t")[0].strip()
+            X_dev.append(text)
+    return X_dev
+
+def writePred(filename, predictions):
+    with open(filename, "w") as out_file:
+        for pred in predictions:
+            out_file.write(str(pred) + "\n")
+
+x = readFile('dev-0/in.tsv')
+pred = model.predict(x)
+writePred('dev-0/out.tsv',pred)
+
+x = readFile('dev-1/in.tsv')
+pred = model.predict(x)
+writePred('dev-1/out.tsv',pred)
+
+x = readFile('test-A/in.tsv')
+pred = model.predict(x)
+writePred('test-A/out.tsv',pred)