nn

dev-0/out.tsv

@@ -9,7 +9,7 @@
 0
 1
 0
-1
+0
 0
 1
 0
@@ -41,7 +41,7 @@
 0
 0
 0
-0
+1
 0
 0
 1
@@ -67,7 +67,7 @@
 0
 1
 1
-1
+0
 0
 0
 0
@@ -175,7 +175,7 @@
 0
 1
 1
-1
+0
 0
 0
 0
@@ -251,7 +251,7 @@
 1
 0
 0
-1
+0
 1
 0
 0
@@ -439,7 +439,7 @@
 0
 0
 0
-0
+1
 1
 0
 0
@@ -589,7 +589,7 @@
 0
 0
 1
-0
+1
 0
 0
 0
@@ -657,7 +657,7 @@
 1
 0
 0
-1
+0
 1
 0
 0
@@ -691,12 +691,12 @@
 0
 0
 1
-1
 0
 0
 0
 0
 1
+1
 0
 1
 1
@@ -761,7 +761,7 @@
 0
 0
 1
-0
+1
 1
 1
 0
@@ -782,7 +782,7 @@
 1
 0
 1
-1
+0
 0
 1
 0
@@ -790,7 +790,7 @@
 0
 1
 1
-1
+0
 1
 0
 0
@@ -874,7 +874,7 @@
 0
 0
 1
-1
+0
 1
 0
 0
@@ -965,14 +965,14 @@
 0
 0
 1
-0
+1
 1
 0
 0
 0
 0
 0
-1
+0
 0
 1
 0
@@ -981,7 +981,7 @@
 0
 1
 0
-1
+0
 0
 1
 0
@@ -1080,7 +1080,7 @@
 0
 0
 0
-1
+0
 0
 0
 1
@@ -1265,7 +1265,7 @@
 0
 1
 0
-0
+1
 0
 1
 1
@@ -1312,7 +1312,7 @@
 0
 0
 0
-0
+1
 0
 0
 0
@@ -1414,7 +1414,7 @@
 1
 0
 0
-1
+0
 0
 0
 1
@@ -1435,7 +1435,7 @@
 0
 0
 1
-0
+1
 0
 0
 1
@@ -1529,7 +1529,7 @@
 0
 1
 0
-0
+1
 1
 1
 0
@@ -1538,7 +1538,7 @@
 0
 1
 0
-1
+0
 1
 0
 1
@@ -1873,7 +1873,7 @@
 1
 1
 1
-1
+0
 0
 0
 1
@@ -1890,7 +1890,7 @@
 0
 0
 1
-0
+1
 1
 0
 0
@@ -1971,7 +1971,7 @@
 0
 0
 1
-0
+1
 1
 0
 0
@@ -2016,12 +2016,12 @@
 0
 1
 0
-1
+0
 1
 1
 1
 0
-1
+0
 0
 0
 1
@@ -2305,7 +2305,7 @@
 1
 1
 0
-1
+0
 0
 0
 0
@@ -2458,7 +2458,7 @@
 0
 0
 0
-0
+1
 0
 0
 0
@@ -2595,7 +2595,7 @@
 0
 0
 1
-0
+1
 1
 0
 0
@@ -2690,7 +2690,7 @@
 0
 1
 1
-0
+1
 0
 0
 0
@@ -2728,11 +2728,11 @@
 0
 0
 1
-1
-1
 0
+1
 0
 1
+1
 0
 0
 1
@@ -2967,7 +2967,7 @@
 0
 0
 0
-0
+1
 1
 0
 0
@@ -3009,7 +3009,7 @@
 0
 0
 1
-0
+1
 0
 1
 0
@@ -3283,7 +3283,7 @@
 0
 0
 1
-0
+1
 0
 0
 0
@@ -3391,7 +3391,7 @@
 0
 0
 0
-0
+1
 0
 0
 0
@@ -3409,7 +3409,7 @@
 0
 0
 0
-0
+1
 0
 0
 1
@@ -3564,7 +3564,7 @@
 0
 0
 0
-1
+0
 0
 0
 1
@@ -3663,7 +3663,7 @@
 0
 0
 0
-1
+0
 0
 0
 0
@@ -3811,7 +3811,7 @@
 1
 0
 0
-0
+1
 1
 0
 1
@@ -3969,7 +3969,7 @@
 0
 0
 0
-1
+0
 0
 0
 1
@@ -3985,7 +3985,7 @@
 0
 0
 0
-1
+0
 0
 0
 1
@@ -3996,7 +3996,7 @@
 1
 0
 0
-1
+0
 0
 0
 0
@@ -4037,7 +4037,7 @@
 0
 1
 0
-1
+0
 0
 1
 0
@@ -4064,7 +4064,7 @@
 1
 0
 0
-1
+0
 0
 0
 1
@@ -4110,8 +4110,8 @@
 0
 0
 1
-0
+1
-0
+1
 0
 1
 1
@@ -4168,7 +4168,7 @@
 1
 1
 0
-0
+1
 0
 0
 0
@@ -4208,7 +4208,7 @@
 0
 1
 1
-1
+0
 1
 1
 0
@@ -4265,7 +4265,7 @@
 0
 0
 0
-0
+1
 0
 1
 1
@@ -4312,7 +4312,7 @@
 0
 0
 0
-1
+0
 0
 1
 0
@@ -4374,7 +4374,7 @@
 1
 0
 0
-0
+1
 0
 0
 0
@@ -4400,7 +4400,7 @@
 1
 0
 0
-0
+1
 0
 0
 0
@@ -4469,7 +4469,7 @@
 0
 0
 0
-1
+0
 0
 1
 1
@@ -4497,7 +4497,7 @@
 0
 0
 0
-1
+0
 0
 1
 0
@@ -4532,7 +4532,7 @@
 1
 1
 1
-1
+0
 0
 0
 0
@@ -4810,7 +4810,7 @@
 1
 1
 0
-0
+1
 0
 0
 1
@@ -5043,7 +5043,7 @@
 1
 1
 0
-1
+0
 0
 0
 1
@@ -5178,7 +5178,7 @@
 0
 0
 0
-1
+0
 0
 0
 0
@@ -5236,7 +5236,7 @@
 1
 0
 1
-1
+0
 1
 1
 0

run.ipynb

@@ -16,47 +16,25 @@
     "from nltk import word_tokenize"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "cbe60d7b-850e-4838-b4ce-672f13bf2bb2",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
   {
    "cell_type": "code",
    "execution_count": 2,
-   "id": "bf211ece-e27a-4119-a1b9-9a9a610cfb46",
+   "id": "1ec57d97-a852-490e-8da4-d1e4c9676cd6",
    "metadata": {},
    "outputs": [],
    "source": [
-    "def predict_year(x, path_out, model):\n",
+    "#def read_file(filename):\n",
-    "    results = model.predict(x)\n",
+    "#    result = []\n",
-    "    with open(path_out, 'wt') as file:\n",
+    "#    with open(filename, 'r', encoding=\"utf-8\") as file:\n",
-    "        for r in results:\n",
+    "#        for line in file:\n",
-    "            file.write(str(r) + '\\n') "
+    "#            text = line.split(\"\\t\")[0].strip()\n",
+    "#            result.append(text)\n",
+    "#    return result"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 3,
-   "id": "1ec57d97-a852-490e-8da4-d1e4c9676cd6",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def read_file(filename):\n",
-    "    result = []\n",
-    "    with open(filename, 'r', encoding=\"utf-8\") as file:\n",
-    "        for line in file:\n",
-    "            text = line.split(\"\\t\")[0].strip()\n",
-    "            result.append(text)\n",
-    "    return result"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
    "id": "86fbfb79-76e7-49f5-b722-2827f93cb03f",
    "metadata": {},
    "outputs": [
@@ -163,7 +141,7 @@
        "[200000 rows x 2 columns]"
       ]
      },
-     "execution_count": 4,
+     "execution_count": 3,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -176,7 +154,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 4,
    "id": "8960c975-f756-4e36-a1ce-e9fd5fdf8fe3",
    "metadata": {},
    "outputs": [
@@ -271,7 +249,7 @@
        "[200000 rows x 1 columns]"
       ]
      },
-     "execution_count": 5,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -285,7 +263,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 5,
    "id": "6b27e6ce-e9fd-41a1-aacf-53a5fde0a7c1",
    "metadata": {},
    "outputs": [
@@ -392,7 +370,7 @@
        "[5272 rows x 2 columns]"
       ]
      },
-     "execution_count": 6,
+     "execution_count": 5,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -405,7 +383,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 6,
    "id": "99ae526d-9b7c-493f-be4f-f95b1c8f4b81",
    "metadata": {},
    "outputs": [
@@ -512,7 +490,7 @@
        "[5152 rows x 2 columns]"
       ]
      },
-     "execution_count": 7,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -525,7 +503,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 7,
    "id": "dba17668-971f-47f8-99ce-fc840b5cb74a",
    "metadata": {},
    "outputs": [],
@@ -546,7 +524,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 8,
    "id": "1a275c1d-75bc-4290-9332-56396d16a0f2",
    "metadata": {},
    "outputs": [],
@@ -558,15 +536,48 @@
     "\n",
     "x_train = [word_tokenize(x) for x in x_train]\n",
     "x_dev = [word_tokenize(x) for x in x_dev]\n",
-    "x_test = [word_tokenize(x) for x in x_test]"
+    "x_test = [word_tokenize(x) for x in x_test]\n",
+    "#x_test"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 9,
+   "id": "3125d2f2-0da9-45eb-acf1-90293c6d64a3",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "5152"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(x_test)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
    "id": "031a3670-3be7-4146-97b4-0dacd4f9ae58",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "5152"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "from gensim.test.utils import common_texts\n",
     "from gensim.models import Word2Vec\n",
@@ -574,24 +585,16 @@
     "word2vec = gensim.downloader.load('word2vec-google-news-300')\n",
     "x_train = [np.mean([word2vec[word] for word in content if word in word2vec] or [np.zeros(300)], axis=0) for content in x_train]\n",
     "x_dev = [np.mean([word2vec[word] for word in content if word in word2vec] or [np.zeros(300)], axis=0) for content in x_dev]\n",
-    "x_test = [np.mean([word2vec[word] for word in content if word in word2vec] or [np.zeros(300)], axis=0) for content in x_test]"
+    "x_test = [np.mean([word2vec[word] for word in content if word in word2vec] or [np.zeros(300)], axis=0) for content in x_test]\n",
+    "len(x_test)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 15,
    "id": "b7defd18-e281-4cf6-9941-cee560749677",
    "metadata": {},
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "C:\\Users\\korne\\AppData\\Local\\Temp\\ipykernel_22024\\3484013121.py:10: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  C:\\actions-runner\\_work\\pytorch\\pytorch\\builder\\windows\\pytorch\\torch\\csrc\\utils\\tensor_new.cpp:210.)\n",
-      "  X = torch.tensor(X)\n"
-     ]
-    }
-   ],
    "source": [
     "model = NeuralNetworkModel()\n",
     "BATCH_SIZE = 5\n",
@@ -614,10 +617,21 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 17,
    "id": "92c69ddd-fe58-477f-b2c2-06324a983bcc",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "5152"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "y_dev = []\n",
     "y_test = []\n",
@@ -635,60 +649,51 @@
     "        X = x_test[i:i + BATCH_SIZE]\n",
     "        X = torch.tensor(X)\n",
     "        outputs = model(X.float())\n",
-    "        y = (outputs >= 0.5)\n",
+    "        prediction = (outputs >= 0.5)\n",
-    "        y_test += prediction.tolist()"
+    "        y_test += prediction.tolist()\n",
+    "len(y_test)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 13,
    "id": "caff921c-d0ab-4fce-a17f-6610266b404d",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "2062"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "y_dev = np.asarray(y_dev, dtype=np.int32)\n",
-    "y_test = np.asarray(y_test, dtype=np.int32)"
+    "y_test = np.asarray(y_test, dtype=np.int32)\n",
+    "len(y_test)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": null,
    "id": "73076eb2-810f-4f85-aa3f-05ee884c413b",
    "metadata": {},
    "outputs": [],
    "source": [
-    "with open('./dev-0/out.tsv', 'wt') as file:\n",
+    "y_dev.tofile('./dev-0/out.tsv', sep='\\n')\n",
-    "    for r in y_dev:\n",
+    "y_test.tofile('./test-A/out.tsv', sep='\\n')"
-    "        file.write(str(r) + '\\n')   "
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": null,
-   "id": "ddda251c-cafa-40f8-a020-48310a9f23b6",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "with open('./test-A/out.tsv', 'wt') as file:\n",
-    "    for r in y_test:\n",
-    "        file.write(str(r) + '\\n')   "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 17,
    "id": "5730562a-0200-4c8f-8f73-992fa2b36133",
    "metadata": {},
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "[NbConvertApp] Converting notebook run.ipynb to script\n",
-      "[NbConvertApp] Writing 3816 bytes to run.py\n"
-     ]
-    }
-   ],
    "source": [
     "!jupyter nbconvert --to script run.ipynb"
    ]

run.py

@@ -13,43 +13,27 @@ import gensim.downloader
 from nltk import word_tokenize
 
 
-# In[ ]:
-
-
-
-
-
 # In[2]:
 
 
-def predict_year(x, path_out, model):
+#def read_file(filename):
-    results = model.predict(x)
+#    result = []
-    with open(path_out, 'wt') as file:
+#    with open(filename, 'r', encoding="utf-8") as file:
-        for r in results:
+#        for line in file:
-            file.write(str(r) + '\n') 
+#            text = line.split("\t")[0].strip()
+#            result.append(text)
+#    return result
 
 
 # In[3]:
 
 
-def read_file(filename):
-    result = []
-    with open(filename, 'r', encoding="utf-8") as file:
-        for line in file:
-            text = line.split("\t")[0].strip()
-            result.append(text)
-    return result
-
-
-# In[4]:
-
-
 x_train = pd.read_table('train/in.tsv', sep='\t', header=None, quoting=3)
 x_train = x_train[0:200000]
 x_train
 
 
-# In[5]:
+# In[4]:
 
 
 with open('train/expected.tsv', 'r', encoding='utf8') as file:
@@ -58,7 +42,7 @@ y_train = y_train[0:200000]
 y_train
 
 
-# In[6]:
+# In[5]:
 
 
 with open('dev-0/in.tsv', 'r', encoding='utf8') as file:
@@ -66,7 +50,7 @@ with open('dev-0/in.tsv', 'r', encoding='utf8') as file:
 x_dev
 
 
-# In[7]:
+# In[6]:
 
 
 with open('test-A/in.tsv', 'r', encoding='utf8') as file:
@@ -74,7 +58,7 @@ with open('test-A/in.tsv', 'r', encoding='utf8') as file:
 x_test
 
 
-# In[8]:
+# In[7]:
 
 
 class NeuralNetworkModel(torch.nn.Module):
@@ -91,7 +75,7 @@ class NeuralNetworkModel(torch.nn.Module):
         return x
 
 
-# In[9]:
+# In[8]:
 
 
 x_train = x_train[0].str.lower()
@@ -102,9 +86,16 @@ x_test = x_test[0].str.lower()
 x_train = [word_tokenize(x) for x in x_train]
 x_dev = [word_tokenize(x) for x in x_dev]
 x_test = [word_tokenize(x) for x in x_test]
+#x_test
 
 
-# In[11]:
+# In[9]:
+
+
+len(x_test)
+
+
+# In[10]:
 
 
 from gensim.test.utils import common_texts
@@ -114,9 +105,10 @@ word2vec = gensim.downloader.load('word2vec-google-news-300')
 x_train = [np.mean([word2vec[word] for word in content if word in word2vec] or [np.zeros(300)], axis=0) for content in x_train]
 x_dev = [np.mean([word2vec[word] for word in content if word in word2vec] or [np.zeros(300)], axis=0) for content in x_dev]
 x_test = [np.mean([word2vec[word] for word in content if word in word2vec] or [np.zeros(300)], axis=0) for content in x_test]
+len(x_test)
 
 
-# In[ ]:
+# In[15]:
 
 
 model = NeuralNetworkModel()
@@ -138,7 +130,7 @@ for epoch in range(BATCH_SIZE):
         optimizer.step()
 
 
-# In[ ]:
+# In[17]:
 
 
 y_dev = []
@@ -157,31 +149,24 @@ with torch.no_grad():
         X = x_test[i:i + BATCH_SIZE]
         X = torch.tensor(X)
         outputs = model(X.float())
-        y = (outputs >= 0.5)
+        prediction = (outputs >= 0.5)
         y_test += prediction.tolist()
+len(y_test)
 
 
-# In[ ]:
+# In[13]:
 
 
 y_dev = np.asarray(y_dev, dtype=np.int32)
 y_test = np.asarray(y_test, dtype=np.int32)
+len(y_test)
 
 
 # In[ ]:
 
 
-with open('./dev-0/out.tsv', 'wt') as file:
+y_dev.tofile('./dev-0/out.tsv', sep='\n')
-    for r in y_dev:
+y_test.tofile('./test-A/out.tsv', sep='\n')
-        file.write(str(r) + '\n')   
-
-
-# In[ ]:
-
-
-with open('./test-A/out.tsv', 'wt') as file:
-    for r in y_test:
-        file.write(str(r) + '\n')   
 
 
 # In[ ]:
@@ -189,3 +174,9 @@ with open('./test-A/out.tsv', 'wt') as file:
 
 get_ipython().system('jupyter nbconvert --to script run.ipynb')
 
+
+# In[ ]:
+
+
+
+