2022-04-23 21:35:59 +02:00
{
"cells": [
{
"cell_type": "code",
2022-06-06 11:25:46 +02:00
"execution_count": 1,
"id": "46fe9a72-d787-46ab-a9df-c6732c173a26",
2022-04-23 21:35:59 +02:00
"metadata": {},
"outputs": [],
2022-06-06 11:25:46 +02:00
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import regex as re\n",
"import csv\n",
"import torch\n",
"from torch import nn\n",
"from gensim.models import Word2Vec\n",
"from nltk.tokenize import word_tokenize"
]
2022-04-23 21:35:59 +02:00
},
{
"cell_type": "code",
"execution_count": 2,
2022-06-06 11:25:46 +02:00
"id": "6fd72918-57e4-44f2-a0e1-a71f322df5f7",
2022-04-23 21:35:59 +02:00
"metadata": {},
2022-06-06 11:25:46 +02:00
"outputs": [],
2022-04-23 21:35:59 +02:00
"source": [
2022-06-06 11:25:46 +02:00
"def clean_text(text):\n",
" text = text.lower().replace('-\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\n', '').replace('\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\n', ' ')\n",
" text = re.sub(r'\\p{P}', '', text)\n",
" text = text.replace(\"'t\", \" not\").replace(\"'s\", \" is\").replace(\"'ll\", \" will\").replace(\"'m\", \" am\").replace(\"'ve\", \" have\")\n",
"\n",
" return text"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "87dc8a73-f089-4551-b0f0-dedefd0b5a05",
"metadata": {},
"outputs": [],
"source": [
"train_data = pd.read_csv('train/in.tsv.xz', sep='\\t', on_bad_lines='skip', header=None, quoting=csv.QUOTE_NONE)\n",
"train_labels = pd.read_csv('train/expected.tsv', sep='\\t', on_bad_lines='skip', header=None, quoting=csv.QUOTE_NONE)\n",
"\n",
"train_data = train_data[[6, 7]]\n",
"train_data = pd.concat([train_data, train_labels], axis=1)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "1d9244d4-1dd7-4c02-8e22-6ea5feee9b26",
"metadata": {},
"outputs": [],
"source": [
"class TrainCorpus:\n",
" def __init__(self, data):\n",
" self.data = data\n",
" \n",
" def __iter__(self):\n",
" for _, row in self.data.iterrows():\n",
" text = str(row[6]) + str(row[0]) + str(row[7])\n",
" text = clean_text(text)\n",
" yield word_tokenize(text)"
2022-04-23 21:35:59 +02:00
]
},
{
"cell_type": "code",
"execution_count": 5,
2022-06-06 11:25:46 +02:00
"id": "5b6f017b-990f-42e2-8ef5-ce40824ace61",
2022-04-23 21:35:59 +02:00
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
2022-06-06 11:25:46 +02:00
"81477\n"
2022-04-23 21:35:59 +02:00
]
}
],
"source": [
2022-06-06 11:25:46 +02:00
"train_sentences = TrainCorpus(train_data.head(80000))\n",
"w2v_model = Word2Vec(vector_size=100, min_count=10)\n",
"w2v_model.build_vocab(corpus_iterable=train_sentences)\n",
"\n",
"key_to_index = w2v_model.wv.key_to_index\n",
"index_to_key = w2v_model.wv.index_to_key\n",
"\n",
"index_to_key.append('<unk>')\n",
"key_to_index['<unk>'] = len(index_to_key) - 1\n",
"\n",
"vocab_size = len(index_to_key)\n",
"print(vocab_size)"
2022-04-23 21:35:59 +02:00
]
},
{
"cell_type": "code",
2022-06-06 11:25:46 +02:00
"execution_count": 6,
"id": "516767c7-ce51-4483-b99f-675cfd4fe99d",
2022-04-23 21:35:59 +02:00
"metadata": {},
"outputs": [],
"source": [
2022-06-06 11:25:46 +02:00
"class TrainDataset(torch.utils.data.IterableDataset):\n",
" def __init__(self, data, index_to_key, key_to_index, reversed=False):\n",
" self.reversed = reversed\n",
" self.data = data\n",
" self.index_to_key = index_to_key\n",
" self.key_to_index = key_to_index\n",
" self.vocab_size = len(key_to_index)\n",
"\n",
" def __iter__(self):\n",
" for _, row in self.data.iterrows():\n",
" text = str(row[6]) + str(row[0]) + str(row[7])\n",
" text = clean_text(text)\n",
" tokens = word_tokenize(text)\n",
" if self.reversed:\n",
" tokens = list(reversed(tokens))\n",
" for i in range(5, len(tokens), 1):\n",
" input_context = tokens[i-5:i]\n",
" target_context = tokens[i-4:i+1]\n",
" \n",
" input_embed = [self.key_to_index[word] if word in self.key_to_index else self.key_to_index['<unk>'] for word in input_context]\n",
" target_embed = [self.key_to_index[word] if word in self.key_to_index else self.key_to_index['<unk>'] for word in target_context]\n",
" \n",
" yield np.asarray(input_embed, dtype=np.int64), np.asarray(target_embed, dtype=np.int64)"
2022-04-23 21:35:59 +02:00
]
},
{
"cell_type": "code",
2022-06-06 11:25:46 +02:00
"execution_count": 7,
"id": "678d0388-a2b2-44ca-b686-dc133a0d16e5",
2022-04-23 21:35:59 +02:00
"metadata": {},
"outputs": [],
"source": [
2022-06-06 11:25:46 +02:00
"class Model(nn.Module):\n",
" def __init__(self, embed_size, vocab_size):\n",
" super(Model, self).__init__()\n",
" self.embed_size = embed_size\n",
" self.vocab_size = vocab_size\n",
" self.lstm_size = 128\n",
" self.num_layers = 2\n",
" \n",
" self.embed = nn.Embedding(num_embeddings=vocab_size, embedding_dim=self.embed_size)\n",
" self.lstm = nn.LSTM(input_size=self.embed_size, hidden_size=self.lstm_size, num_layers=self.num_layers, dropout=0.2)\n",
" self.fc = nn.Linear(self.lstm_size, vocab_size)\n",
"\n",
" def forward(self, x, prev_state = None):\n",
" embed = self.embed(x)\n",
" output, state = self.lstm(embed, prev_state)\n",
" logits = self.fc(output)\n",
" probs = torch.softmax(logits, dim=1)\n",
" return logits, state\n",
"\n",
" def init_state(self, sequence_length):\n",
" zeros = torch.zeros(self.num_layers, sequence_length, self.gru_size).to(device)\n",
" return (zeros, zeros)"
2022-04-23 21:35:59 +02:00
]
},
{
"cell_type": "code",
2022-06-06 11:25:46 +02:00
"execution_count": 17,
"id": "487df613-15cd-450e-a8b2-7e7cd879f8f7",
2022-04-23 21:35:59 +02:00
"metadata": {},
"outputs": [],
"source": [
2022-06-06 11:25:46 +02:00
"from torch.utils.data import DataLoader\n",
"from torch.optim import Adam\n",
2022-04-23 21:35:59 +02:00
"\n",
2022-06-06 11:25:46 +02:00
"def train(dataset, model, max_epochs, batch_size):\n",
" model.train()\n",
"\n",
" dataloader = DataLoader(dataset, batch_size=batch_size)\n",
" criterion = nn.CrossEntropyLoss()\n",
" optimizer = Adam(model.parameters(), lr=0.001)\n",
"\n",
" for epoch in range(max_epochs):\n",
" for batch, (x, y) in enumerate(dataloader):\n",
" optimizer.zero_grad()\n",
" \n",
" x = x.to(device)\n",
" y = y.to(device)\n",
" \n",
" y_pred, (state_h, state_c) = model(x)\n",
" loss = criterion(y_pred.transpose(1, 2), y)\n",
"\n",
" loss.backward()\n",
" optimizer.step()\n",
" \n",
" if batch % 100 == 0:\n",
" print(f'epoch: {epoch}, update in batch {batch}/???, loss: {loss.item()}')"
2022-04-23 21:35:59 +02:00
]
},
{
"cell_type": "code",
2022-06-06 11:25:46 +02:00
"execution_count": 18,
"id": "79df35ad-cb19-4867-869c-b651455580ae",
2022-04-23 21:35:59 +02:00
"metadata": {},
"outputs": [],
"source": [
2022-06-06 11:25:46 +02:00
"torch.cuda.empty_cache()\n",
"device = 'cuda' if torch.cuda.is_available() else 'cpu'"
2022-04-23 21:35:59 +02:00
]
},
{
"cell_type": "code",
2022-06-06 11:25:46 +02:00
"execution_count": 26,
"id": "e4654fd0-6c0a-47e3-b623-73c7c04ea194",
2022-04-23 21:35:59 +02:00
"metadata": {},
"outputs": [],
"source": [
2022-06-06 11:25:46 +02:00
"train_dataset_front = TrainDataset(train_data.head(8000), index_to_key, key_to_index, False)"
2022-04-23 21:35:59 +02:00
]
},
{
"cell_type": "code",
2022-06-06 11:25:46 +02:00
"execution_count": 27,
"id": "7b5f09f5-88c4-44d4-ab8a-aa62ec8f70d5",
"metadata": {},
"outputs": [],
"source": [
"model_front = Model(100, vocab_size).to(device)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "7e886972-4994-4bbd-aca8-2ab685c7b8db",
2022-04-23 21:35:59 +02:00
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
2022-06-06 11:25:46 +02:00
"epoch: 0, update in batch 0/???, loss: 11.315739631652832\n",
"epoch: 0, update in batch 100/???, loss: 8.016324996948242\n",
"epoch: 0, update in batch 200/???, loss: 7.45602560043335\n",
"epoch: 0, update in batch 300/???, loss: 6.306332588195801\n",
"epoch: 0, update in batch 400/???, loss: 8.629552841186523\n",
"epoch: 0, update in batch 500/???, loss: 7.637443542480469\n",
"epoch: 0, update in batch 600/???, loss: 7.67318868637085\n",
"epoch: 0, update in batch 700/???, loss: 7.2209930419921875\n",
"epoch: 0, update in batch 800/???, loss: 7.739532470703125\n",
"epoch: 0, update in batch 900/???, loss: 7.219891548156738\n",
"epoch: 0, update in batch 1000/???, loss: 6.8804473876953125\n",
"epoch: 0, update in batch 1100/???, loss: 7.228173732757568\n",
"epoch: 0, update in batch 1200/???, loss: 6.513087272644043\n",
"epoch: 0, update in batch 1300/???, loss: 7.142991542816162\n",
"epoch: 0, update in batch 1400/???, loss: 7.711663246154785\n",
"epoch: 0, update in batch 1500/???, loss: 6.894327640533447\n",
"epoch: 0, update in batch 1600/???, loss: 7.723884582519531\n",
"epoch: 0, update in batch 1700/???, loss: 8.409640312194824\n",
"epoch: 0, update in batch 1800/???, loss: 6.570927619934082\n",
"epoch: 0, update in batch 1900/???, loss: 6.906421661376953\n",
"epoch: 0, update in batch 2000/???, loss: 7.197023868560791\n",
"epoch: 0, update in batch 2100/???, loss: 6.892503261566162\n",
"epoch: 0, update in batch 2200/???, loss: 7.109471321105957\n",
"epoch: 0, update in batch 2300/???, loss: 8.84702205657959\n",
"epoch: 0, update in batch 2400/???, loss: 7.394454002380371\n",
"epoch: 0, update in batch 2500/???, loss: 7.380859375\n",
"epoch: 0, update in batch 2600/???, loss: 6.635237693786621\n",
"epoch: 0, update in batch 2700/???, loss: 6.869620323181152\n",
"epoch: 0, update in batch 2800/???, loss: 6.656294822692871\n",
"epoch: 0, update in batch 2900/???, loss: 8.090291976928711\n",
"epoch: 0, update in batch 3000/???, loss: 7.012345314025879\n",
"epoch: 0, update in batch 3100/???, loss: 6.7099809646606445\n",
"epoch: 0, update in batch 3200/???, loss: 6.798626899719238\n",
"epoch: 0, update in batch 3300/???, loss: 6.510752201080322\n",
"epoch: 0, update in batch 3400/???, loss: 7.742552757263184\n",
"epoch: 0, update in batch 3500/???, loss: 7.3319292068481445\n",
"epoch: 0, update in batch 3600/???, loss: 8.022462844848633\n",
"epoch: 0, update in batch 3700/???, loss: 5.883602619171143\n",
"epoch: 0, update in batch 3800/???, loss: 6.235389232635498\n",
"epoch: 0, update in batch 3900/???, loss: 7.012289524078369\n",
"epoch: 0, update in batch 4000/???, loss: 7.005420684814453\n",
"epoch: 0, update in batch 4100/???, loss: 6.595402717590332\n",
"epoch: 0, update in batch 4200/???, loss: 6.7428154945373535\n",
"epoch: 0, update in batch 4300/???, loss: 6.358878135681152\n",
"epoch: 0, update in batch 4400/???, loss: 6.6188201904296875\n",
"epoch: 0, update in batch 4500/???, loss: 7.08281946182251\n",
"epoch: 0, update in batch 4600/???, loss: 5.705609321594238\n",
"epoch: 0, update in batch 4700/???, loss: 7.1878180503845215\n",
"epoch: 0, update in batch 4800/???, loss: 7.071160793304443\n",
"epoch: 0, update in batch 4900/???, loss: 6.768280029296875\n",
"epoch: 0, update in batch 5000/???, loss: 6.507267951965332\n",
"epoch: 0, update in batch 5100/???, loss: 6.6431379318237305\n",
"epoch: 0, update in batch 5200/???, loss: 6.719052314758301\n",
"epoch: 0, update in batch 5300/???, loss: 7.172060489654541\n",
"epoch: 0, update in batch 5400/???, loss: 5.98638916015625\n",
"epoch: 0, update in batch 5500/???, loss: 5.674165725708008\n",
"epoch: 0, update in batch 5600/???, loss: 5.612569808959961\n",
"epoch: 0, update in batch 5700/???, loss: 6.307109832763672\n",
"epoch: 0, update in batch 5800/???, loss: 5.382391452789307\n",
"epoch: 0, update in batch 5900/???, loss: 5.712988376617432\n",
"epoch: 0, update in batch 6000/???, loss: 6.371735572814941\n",
"epoch: 0, update in batch 6100/???, loss: 6.417542457580566\n",
"epoch: 0, update in batch 6200/???, loss: 7.14879846572876\n",
"epoch: 0, update in batch 6300/???, loss: 7.0701189041137695\n",
"epoch: 0, update in batch 6400/???, loss: 7.048495292663574\n",
"epoch: 0, update in batch 6500/???, loss: 7.3384833335876465\n",
"epoch: 0, update in batch 6600/???, loss: 6.561330318450928\n",
"epoch: 0, update in batch 6700/???, loss: 6.839573860168457\n",
"epoch: 0, update in batch 6800/???, loss: 6.5179548263549805\n",
"epoch: 0, update in batch 6900/???, loss: 7.246607303619385\n",
"epoch: 0, update in batch 7000/???, loss: 6.5699052810668945\n",
"epoch: 0, update in batch 7100/???, loss: 7.202715873718262\n",
"epoch: 0, update in batch 7200/???, loss: 6.1833648681640625\n",
"epoch: 0, update in batch 7300/???, loss: 5.977782249450684\n",
"epoch: 0, update in batch 7400/???, loss: 6.717446327209473\n",
"epoch: 0, update in batch 7500/???, loss: 6.574376583099365\n",
"epoch: 0, update in batch 7600/???, loss: 5.8418450355529785\n",
"epoch: 0, update in batch 7700/???, loss: 6.282655715942383\n",
"epoch: 0, update in batch 7800/???, loss: 6.065321922302246\n",
"epoch: 0, update in batch 7900/???, loss: 6.415077209472656\n",
"epoch: 0, update in batch 8000/???, loss: 6.482673645019531\n",
"epoch: 0, update in batch 8100/???, loss: 6.670407772064209\n",
"epoch: 0, update in batch 8200/???, loss: 6.799211025238037\n",
"epoch: 0, update in batch 8300/???, loss: 7.299313545227051\n",
"epoch: 0, update in batch 8400/???, loss: 7.42974328994751\n",
"epoch: 0, update in batch 8500/???, loss: 8.549559593200684\n",
"epoch: 0, update in batch 8600/???, loss: 6.794680118560791\n",
"epoch: 0, update in batch 8700/???, loss: 7.390380859375\n",
"epoch: 0, update in batch 8800/???, loss: 7.552660942077637\n",
"epoch: 0, update in batch 8900/???, loss: 6.663547515869141\n",
"epoch: 0, update in batch 9000/???, loss: 6.5236711502075195\n",
"epoch: 0, update in batch 9100/???, loss: 7.666424751281738\n",
"epoch: 0, update in batch 9200/???, loss: 6.479496955871582\n",
"epoch: 0, update in batch 9300/???, loss: 5.5056304931640625\n",
"epoch: 0, update in batch 9400/???, loss: 6.6904096603393555\n",
"epoch: 0, update in batch 9500/???, loss: 6.9318037033081055\n",
"epoch: 0, update in batch 9600/???, loss: 6.521365165710449\n",
"epoch: 0, update in batch 9700/???, loss: 6.376631736755371\n",
"epoch: 0, update in batch 9800/???, loss: 6.4104766845703125\n",
"epoch: 0, update in batch 9900/???, loss: 7.3995232582092285\n",
"epoch: 0, update in batch 10000/???, loss: 6.510337829589844\n",
"epoch: 0, update in batch 10100/???, loss: 6.2512407302856445\n",
"epoch: 0, update in batch 10200/???, loss: 6.048404216766357\n",
"epoch: 0, update in batch 10300/???, loss: 6.832150936126709\n",
"epoch: 0, update in batch 10400/???, loss: 6.7485456466674805\n",
"epoch: 0, update in batch 10500/???, loss: 5.385656833648682\n",
"epoch: 0, update in batch 10600/???, loss: 6.769070625305176\n",
"epoch: 0, update in batch 10700/???, loss: 6.857029914855957\n",
"epoch: 0, update in batch 10800/???, loss: 5.991332530975342\n",
"epoch: 0, update in batch 10900/???, loss: 6.5500006675720215\n",
"epoch: 0, update in batch 11000/???, loss: 6.951509952545166\n",
"epoch: 0, update in batch 11100/???, loss: 6.396986961364746\n",
"epoch: 0, update in batch 11200/???, loss: 6.639346122741699\n",
"epoch: 0, update in batch 11300/???, loss: 5.87351655960083\n",
"epoch: 0, update in batch 11400/???, loss: 5.996974945068359\n",
"epoch: 0, update in batch 11500/???, loss: 7.103158473968506\n",
"epoch: 0, update in batch 11600/???, loss: 6.429941654205322\n",
"epoch: 0, update in batch 11700/???, loss: 5.597273826599121\n",
"epoch: 0, update in batch 11800/???, loss: 7.112508296966553\n",
"epoch: 0, update in batch 11900/???, loss: 6.745194911956787\n",
"epoch: 0, update in batch 12000/???, loss: 7.47100305557251\n",
"epoch: 0, update in batch 12100/???, loss: 6.847914695739746\n",
"epoch: 0, update in batch 12200/???, loss: 6.876992702484131\n",
"epoch: 0, update in batch 12300/???, loss: 6.499053955078125\n",
"epoch: 0, update in batch 12400/???, loss: 7.196413993835449\n",
"epoch: 0, update in batch 12500/???, loss: 6.593430995941162\n",
"epoch: 0, update in batch 12600/???, loss: 6.368945121765137\n",
"epoch: 0, update in batch 12700/???, loss: 6.362246513366699\n",
"epoch: 0, update in batch 12800/???, loss: 7.209506034851074\n",
"epoch: 0, update in batch 12900/???, loss: 6.8092780113220215\n",
"epoch: 0, update in batch 13000/???, loss: 8.273663520812988\n",
"epoch: 0, update in batch 13100/???, loss: 7.061187744140625\n",
"epoch: 0, update in batch 13200/???, loss: 5.778809547424316\n",
"epoch: 0, update in batch 13300/???, loss: 5.650263786315918\n",
"epoch: 0, update in batch 13400/???, loss: 5.9032440185546875\n",
"epoch: 0, update in batch 13500/???, loss: 6.629636287689209\n",
"epoch: 0, update in batch 13600/???, loss: 6.577019691467285\n",
"epoch: 0, update in batch 13700/???, loss: 5.953114032745361\n",
"epoch: 0, update in batch 13800/???, loss: 6.630902290344238\n",
"epoch: 0, update in batch 13900/???, loss: 7.593966484069824\n",
"epoch: 0, update in batch 14000/???, loss: 6.636081695556641\n",
"epoch: 0, update in batch 14100/???, loss: 5.772985458374023\n",
"epoch: 0, update in batch 14200/???, loss: 5.907249450683594\n",
"epoch: 0, update in batch 14300/???, loss: 7.863391876220703\n",
"epoch: 0, update in batch 14400/???, loss: 7.275572776794434\n",
"epoch: 0, update in batch 14500/???, loss: 6.818984031677246\n",
"epoch: 0, update in batch 14600/???, loss: 6.0456342697143555\n",
"epoch: 0, update in batch 14700/???, loss: 6.281990051269531\n",
"epoch: 0, update in batch 14800/???, loss: 6.197850227355957\n",
"epoch: 0, update in batch 14900/???, loss: 5.851240634918213\n",
"epoch: 0, update in batch 15000/???, loss: 6.826748847961426\n",
"epoch: 0, update in batch 15100/???, loss: 7.2189483642578125\n",
"epoch: 0, update in batch 15200/???, loss: 6.609204292297363\n",
"epoch: 0, update in batch 15300/???, loss: 6.947709560394287\n",
"epoch: 0, update in batch 15400/???, loss: 6.604478359222412\n",
"epoch: 0, update in batch 15500/???, loss: 6.222006797790527\n",
"epoch: 0, update in batch 15600/???, loss: 6.515635013580322\n",
"epoch: 0, update in batch 15700/???, loss: 6.40108585357666\n",
"epoch: 0, update in batch 15800/???, loss: 6.36106014251709\n",
"epoch: 0, update in batch 15900/???, loss: 6.533608436584473\n",
"epoch: 0, update in batch 16000/???, loss: 6.662516117095947\n",
"epoch: 0, update in batch 16100/???, loss: 7.284195899963379\n",
"epoch: 0, update in batch 16200/???, loss: 6.6524176597595215\n",
"epoch: 0, update in batch 16300/???, loss: 6.430756568908691\n",
"epoch: 0, update in batch 16400/???, loss: 7.515387058258057\n",
"epoch: 0, update in batch 16500/???, loss: 6.938241481781006\n",
"epoch: 0, update in batch 16600/???, loss: 5.860864162445068\n",
"epoch: 0, update in batch 16700/???, loss: 6.451329231262207\n",
"epoch: 0, update in batch 16800/???, loss: 6.5510663986206055\n",
"epoch: 0, update in batch 16900/???, loss: 7.3591437339782715\n",
"epoch: 0, update in batch 17000/???, loss: 6.158746719360352\n",
"epoch: 0, update in batch 17100/???, loss: 7.202520847320557\n",
"epoch: 0, update in batch 17200/???, loss: 6.80673885345459\n",
"epoch: 0, update in batch 17300/???, loss: 6.698304653167725\n",
"epoch: 0, update in batch 17400/???, loss: 5.743161201477051\n",
"epoch: 0, update in batch 17500/???, loss: 6.518529415130615\n",
"epoch: 0, update in batch 17600/???, loss: 6.021708011627197\n",
"epoch: 0, update in batch 17700/???, loss: 6.354712963104248\n",
"epoch: 0, update in batch 17800/???, loss: 6.323357582092285\n",
"epoch: 0, update in batch 17900/???, loss: 6.61548376083374\n",
"epoch: 0, update in batch 18000/???, loss: 6.600308895111084\n",
"epoch: 0, update in batch 18100/???, loss: 6.794068336486816\n",
"epoch: 0, update in batch 18200/???, loss: 7.487390041351318\n",
"epoch: 0, update in batch 18300/???, loss: 5.973461627960205\n",
"epoch: 0, update in batch 18400/???, loss: 6.891515254974365\n",
"epoch: 0, update in batch 18500/???, loss: 5.897144317626953\n",
"epoch: 0, update in batch 18600/???, loss: 6.6016364097595215\n",
"epoch: 0, update in batch 18700/???, loss: 6.948650360107422\n",
"epoch: 0, update in batch 18800/???, loss: 7.221627235412598\n",
"epoch: 0, update in batch 18900/???, loss: 6.817994117736816\n",
"epoch: 0, update in batch 19000/???, loss: 5.730655193328857\n",
"epoch: 0, update in batch 19100/???, loss: 6.236818790435791\n",
"epoch: 0, update in batch 19200/???, loss: 7.178666114807129\n",
"epoch: 0, update in batch 19300/???, loss: 6.77465295791626\n",
"epoch: 0, update in batch 19400/???, loss: 6.996792793273926\n",
"epoch: 0, update in batch 19500/???, loss: 6.80951452255249\n",
"epoch: 0, update in batch 19600/???, loss: 7.1757965087890625\n",
"epoch: 0, update in batch 19700/???, loss: 8.400952339172363\n",
"epoch: 0, update in batch 19800/???, loss: 7.1904473304748535\n",
"epoch: 0, update in batch 19900/???, loss: 6.339241981506348\n",
"epoch: 0, update in batch 20000/???, loss: 7.078637599945068\n",
"epoch: 0, update in batch 20100/???, loss: 5.015235900878906\n",
"epoch: 0, update in batch 20200/???, loss: 6.763777732849121\n",
"epoch: 0, update in batch 20300/???, loss: 6.543915748596191\n",
"epoch: 0, update in batch 20400/???, loss: 6.027902603149414\n",
"epoch: 0, update in batch 20500/???, loss: 6.710694789886475\n",
"epoch: 0, update in batch 20600/???, loss: 6.800978660583496\n",
"epoch: 0, update in batch 20700/???, loss: 6.371827125549316\n",
"epoch: 0, update in batch 20800/???, loss: 5.952463626861572\n",
"epoch: 0, update in batch 20900/???, loss: 6.317960739135742\n",
"epoch: 0, update in batch 21000/???, loss: 7.178386688232422\n",
"epoch: 0, update in batch 21100/???, loss: 6.887454986572266\n",
"epoch: 0, update in batch 21200/???, loss: 6.468400478363037\n",
"epoch: 0, update in batch 21300/???, loss: 7.8383684158325195\n",
"epoch: 0, update in batch 21400/???, loss: 5.850740909576416\n",
"epoch: 0, update in batch 21500/???, loss: 6.065464973449707\n",
"epoch: 0, update in batch 21600/???, loss: 7.537625312805176\n",
"epoch: 0, update in batch 21700/???, loss: 6.095994472503662\n",
"epoch: 0, update in batch 21800/???, loss: 6.342766761779785\n",
"epoch: 0, update in batch 21900/???, loss: 5.810301780700684\n",
"epoch: 0, update in batch 22000/???, loss: 6.447206974029541\n",
"epoch: 0, update in batch 22100/???, loss: 7.0662946701049805\n",
"epoch: 0, update in batch 22200/???, loss: 6.535088539123535\n",
"epoch: 0, update in batch 22300/???, loss: 7.017588138580322\n",
"epoch: 0, update in batch 22400/???, loss: 5.067782402038574\n",
"epoch: 0, update in batch 22500/???, loss: 6.493170738220215\n",
"epoch: 0, update in batch 22600/???, loss: 5.642627716064453\n",
"epoch: 0, update in batch 22700/???, loss: 7.200662136077881\n",
"epoch: 0, update in batch 22800/???, loss: 6.137134075164795\n",
"epoch: 0, update in batch 22900/???, loss: 6.367280006408691\n",
"epoch: 0, update in batch 23000/???, loss: 7.458652496337891\n",
"epoch: 0, update in batch 23100/???, loss: 6.515708923339844\n",
"epoch: 0, update in batch 23200/???, loss: 7.526422023773193\n",
"epoch: 0, update in batch 23300/???, loss: 6.653852939605713\n",
"epoch: 0, update in batch 23400/???, loss: 6.737251281738281\n",
"epoch: 0, update in batch 23500/???, loss: 6.493605136871338\n",
"epoch: 0, update in batch 23600/???, loss: 6.132809638977051\n",
"epoch: 0, update in batch 23700/???, loss: 6.406940460205078\n",
"epoch: 0, update in batch 23800/???, loss: 6.84005880355835\n",
"epoch: 0, update in batch 23900/???, loss: 6.830739498138428\n",
"epoch: 0, update in batch 24000/???, loss: 5.862464427947998\n",
"epoch: 0, update in batch 24100/???, loss: 6.382696628570557\n",
"epoch: 0, update in batch 24200/???, loss: 5.722895622253418\n",
"epoch: 0, update in batch 24300/???, loss: 6.697083473205566\n",
"epoch: 0, update in batch 24400/???, loss: 6.56771183013916\n",
"epoch: 0, update in batch 24500/???, loss: 7.566462516784668\n",
"epoch: 0, update in batch 24600/???, loss: 6.217026710510254\n",
"epoch: 0, update in batch 24700/???, loss: 7.164259433746338\n",
"epoch: 0, update in batch 24800/???, loss: 6.460946083068848\n",
"epoch: 0, update in batch 24900/???, loss: 6.333778381347656\n",
"epoch: 0, update in batch 25000/???, loss: 6.522342681884766\n",
"epoch: 0, update in batch 25100/???, loss: 6.270648002624512\n",
"epoch: 0, update in batch 25200/???, loss: 7.118265628814697\n",
"epoch: 0, update in batch 25300/???, loss: 5.8695197105407715\n",
"epoch: 0, update in batch 25400/???, loss: 5.92995023727417\n",
"epoch: 0, update in batch 25500/???, loss: 6.202570915222168\n",
"epoch: 0, update in batch 25600/???, loss: 6.4268975257873535\n",
"epoch: 0, update in batch 25700/???, loss: 6.710567474365234\n",
"epoch: 0, update in batch 25800/???, loss: 6.130914688110352\n",
"epoch: 0, update in batch 25900/???, loss: 6.082686424255371\n",
"epoch: 0, update in batch 26000/???, loss: 6.111697196960449\n",
"epoch: 0, update in batch 26100/???, loss: 7.320557594299316\n",
"epoch: 0, update in batch 26200/???, loss: 6.227985858917236\n",
"epoch: 0, update in batch 26300/???, loss: 6.204974174499512\n",
"epoch: 0, update in batch 26400/???, loss: 6.658400058746338\n",
"epoch: 0, update in batch 26500/???, loss: 5.911742687225342\n",
"epoch: 0, update in batch 26600/???, loss: 6.891500949859619\n",
"epoch: 0, update in batch 26700/???, loss: 5.763737201690674\n",
"epoch: 0, update in batch 26800/???, loss: 5.757307529449463\n",
"epoch: 0, update in batch 26900/???, loss: 6.076601982116699\n",
"epoch: 0, update in batch 27000/???, loss: 6.193032264709473\n",
"epoch: 0, update in batch 27100/???, loss: 6.120661735534668\n",
"epoch: 0, update in batch 27200/???, loss: 6.5425519943237305\n",
"epoch: 0, update in batch 27300/???, loss: 6.511394500732422\n",
"epoch: 0, update in batch 27400/???, loss: 7.127263069152832\n",
"epoch: 0, update in batch 27500/???, loss: 6.134243488311768\n",
"epoch: 0, update in batch 27600/???, loss: 6.5747809410095215\n",
"epoch: 0, update in batch 27700/???, loss: 6.351634979248047\n",
"epoch: 0, update in batch 27800/???, loss: 5.589611530303955\n",
"epoch: 0, update in batch 27900/???, loss: 6.916817665100098\n",
"epoch: 0, update in batch 28000/???, loss: 5.711864948272705\n",
"epoch: 0, update in batch 28100/???, loss: 6.921398162841797\n",
"epoch: 0, update in batch 28200/???, loss: 6.785823822021484\n",
"epoch: 0, update in batch 28300/???, loss: 6.007838249206543\n",
"epoch: 0, update in batch 28400/???, loss: 6.338862419128418\n",
"epoch: 0, update in batch 28500/???, loss: 6.9078168869018555\n",
"epoch: 0, update in batch 28600/???, loss: 6.710842132568359\n",
"epoch: 0, update in batch 28700/???, loss: 6.592329502105713\n",
"epoch: 0, update in batch 28800/???, loss: 6.184128761291504\n",
"epoch: 0, update in batch 28900/???, loss: 6.209361553192139\n",
"epoch: 0, update in batch 29000/???, loss: 7.067984104156494\n",
"epoch: 0, update in batch 29100/???, loss: 6.479236602783203\n",
"epoch: 0, update in batch 29200/???, loss: 6.413198947906494\n",
"epoch: 0, update in batch 29300/???, loss: 6.638579368591309\n",
"epoch: 0, update in batch 29400/???, loss: 5.938233375549316\n",
"epoch: 0, update in batch 29500/???, loss: 6.8490891456604\n",
"epoch: 0, update in batch 29600/???, loss: 6.111110210418701\n",
"epoch: 0, update in batch 29700/???, loss: 6.959462642669678\n",
"epoch: 0, update in batch 29800/???, loss: 6.964720726013184\n",
"epoch: 0, update in batch 29900/???, loss: 6.2007527351379395\n",
"epoch: 0, update in batch 30000/???, loss: 6.803907871246338\n",
"epoch: 0, update in batch 30100/???, loss: 5.665301322937012\n",
"epoch: 0, update in batch 30200/???, loss: 6.913702487945557\n",
"epoch: 0, update in batch 30300/???, loss: 6.824265956878662\n",
"epoch: 0, update in batch 30400/???, loss: 6.131905555725098\n",
"epoch: 0, update in batch 30500/???, loss: 5.799595832824707\n",
"epoch: 0, update in batch 30600/???, loss: 6.846949100494385\n",
"epoch: 0, update in batch 30700/???, loss: 6.481771945953369\n",
"epoch: 0, update in batch 30800/???, loss: 6.5581254959106445\n",
"epoch: 0, update in batch 30900/???, loss: 6.111696720123291\n",
"epoch: 0, update in batch 31000/???, loss: 4.8547563552856445\n",
"epoch: 0, update in batch 31100/???, loss: 6.5503740310668945\n",
"epoch: 0, update in batch 31200/???, loss: 6.212404251098633\n",
"epoch: 0, update in batch 31300/???, loss: 5.761624336242676\n",
"epoch: 0, update in batch 31400/???, loss: 7.043508052825928\n",
"epoch: 0, update in batch 31500/???, loss: 8.301980018615723\n",
"epoch: 0, update in batch 31600/???, loss: 5.655745506286621\n",
"epoch: 0, update in batch 31700/???, loss: 7.116888999938965\n",
"epoch: 0, update in batch 31800/???, loss: 6.237078666687012\n",
"epoch: 0, update in batch 31900/???, loss: 6.990937232971191\n",
"epoch: 0, update in batch 32000/???, loss: 6.327075958251953\n",
"epoch: 0, update in batch 32100/???, loss: 6.831456184387207\n",
"epoch: 0, update in batch 32200/???, loss: 6.511493682861328\n",
"epoch: 0, update in batch 32300/???, loss: 6.719797611236572\n",
"epoch: 0, update in batch 32400/???, loss: 6.46258544921875\n",
"epoch: 0, update in batch 32500/???, loss: 7.349535942077637\n",
"epoch: 0, update in batch 32600/???, loss: 5.773186683654785\n",
"epoch: 0, update in batch 32700/???, loss: 6.072037696838379\n",
"epoch: 0, update in batch 32800/???, loss: 7.044579982757568\n",
"epoch: 0, update in batch 32900/???, loss: 6.290024757385254\n",
"epoch: 0, update in batch 33000/???, loss: 7.101686000823975\n",
"epoch: 0, update in batch 33100/???, loss: 6.590539455413818\n",
"epoch: 0, update in batch 33200/???, loss: 6.944089412689209\n",
"epoch: 0, update in batch 33300/???, loss: 6.6709442138671875\n",
"epoch: 0, update in batch 33400/???, loss: 7.119935035705566\n",
"epoch: 0, update in batch 33500/???, loss: 6.845646858215332\n",
"epoch: 0, update in batch 33600/???, loss: 6.941410064697266\n",
"epoch: 0, update in batch 33700/???, loss: 6.341822624206543\n",
"epoch: 0, update in batch 33800/???, loss: 6.98660945892334\n",
"epoch: 0, update in batch 33900/???, loss: 7.544371128082275\n",
"epoch: 0, update in batch 34000/???, loss: 6.844598293304443\n",
"epoch: 0, update in batch 34100/???, loss: 6.958268642425537\n",
"epoch: 0, update in batch 34200/???, loss: 6.6372880935668945\n"
2022-04-23 21:35:59 +02:00
]
}
],
"source": [
2022-06-06 11:25:46 +02:00
"train(train_dataset_front, model_front, 1, 64)"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "f0512b6e-098d-4264-b52e-1d9ce88311f0",
"metadata": {},
"outputs": [],
"source": [
"def predict_probs(left_tokens, right_tokens):\n",
" model_front.eval()\n",
2022-04-23 21:35:59 +02:00
"\n",
2022-06-06 11:25:46 +02:00
" x_left = torch.tensor([[train_dataset_front.key_to_index[w] if w in key_to_index else train_dataset_front.key_to_index['<unk>'] for w in left_tokens]]).to(device)\n",
" x_right = torch.tensor([[train_dataset_front.key_to_index[w] if w in key_to_index else train_dataset_front.key_to_index['<unk>'] for w in right_tokens]]).to(device)\n",
" y_pred_left, (state_h_left, state_c_left) = model_front(x_left)\n",
" y_pred_right, (state_h_right, state_c_right) = model_back(x_right)\n",
2022-04-23 21:35:59 +02:00
"\n",
2022-06-06 11:25:46 +02:00
" last_word_logits_left = y_pred_left[0][-1]\n",
" last_word_logits_right = y_pred_right[0][-1]\n",
" probs_left = torch.nn.functional.softmax(last_word_logits_left, dim=0).detach().cpu().numpy()\n",
" probs_right = torch.nn.functional.softmax(last_word_logits_right, dim=0).detach().cpu().numpy()\n",
" \n",
" probs = [np.mean(k) for k in zip(probs_left, probs_right)]\n",
" \n",
" top_words = []\n",
" for index in range(len(probs)):\n",
" if len(top_words) < 30:\n",
" top_words.append((probs[index], [index]))\n",
" else:\n",
" worst_word = None\n",
" for word in top_words:\n",
" if not worst_word:\n",
" worst_word = word\n",
" else:\n",
" if word[0] < worst_word[0]:\n",
" worst_word = word\n",
" if worst_word[0] < probs[index] and index != len(probs) - 1:\n",
" top_words.remove(worst_word)\n",
" top_words.append((probs[index], [index]))\n",
" \n",
" prediction = ''\n",
" sum_prob = 0.0\n",
" for word in top_words:\n",
" sum_prob += word[0]\n",
" word_index = word[0]\n",
" word_text = index_to_key[word[1][0]]\n",
" prediction += f'{word_text}:{word_index} '\n",
" prediction += f':{1 - sum_prob}'\n",
" \n",
" return prediction"
2022-04-23 21:35:59 +02:00
]
},
{
"cell_type": "code",
2022-06-06 11:25:46 +02:00
"execution_count": 23,
"id": "90938496-416a-4618-bf88-eddb3cc6e8da",
2022-04-23 21:35:59 +02:00
"metadata": {},
"outputs": [],
"source": [
2022-06-06 11:25:46 +02:00
"dev_data = pd.read_csv('dev-0/in.tsv.xz', sep='\\t', on_bad_lines='skip', header=None, quoting=csv.QUOTE_NONE)\n",
"test_data = pd.read_csv('test-A/in.tsv.xz', sep='\\t', on_bad_lines='skip', header=None, quoting=csv.QUOTE_NONE)"
2022-04-23 21:35:59 +02:00
]
},
{
"cell_type": "code",
2022-06-06 11:25:46 +02:00
"execution_count": 24,
"id": "a0d4bd42-0499-46af-ab0e-5dd23dacf47a",
2022-04-23 21:35:59 +02:00
"metadata": {},
"outputs": [],
"source": [
2022-06-06 11:25:46 +02:00
"with open('dev-0/out.tsv', 'w') as file:\n",
" for index, row in dev_data.iterrows():\n",
" left_text = clean_text(str(row[6]))\n",
" right_text = clean_text(str(row[7]))\n",
" left_words = word_tokenize(left_text)\n",
" right_words = word_tokenize(right_text)\n",
" right_words.reverse()\n",
" if len(left_words) < 6 or len(right_words) < 6:\n",
" prediction = ':1.0'\n",
" else:\n",
" prediction = predict_probs(left_words[-5:], right_words[-5:])\n",
" file.write(prediction + '\\n')"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "819be88f-6336-4853-8015-33c9f3392aa4",
"metadata": {},
"outputs": [],
"source": [
"with open('test-A/out.tsv', 'w') as file:\n",
" for index, row in test_data.iterrows():\n",
" left_text = clean_text(str(row[6]))\n",
" right_text = clean_text(str(row[7]))\n",
" left_words = word_tokenize(left_text)\n",
" right_words = word_tokenize(right_text)\n",
" right_words.reverse()\n",
" if len(left_words) < 6 or len(right_words) < 6:\n",
" prediction = ':1.0'\n",
" else:\n",
" prediction = predict_probs(left_words[-5:], right_words[-5:])\n",
" file.write(prediction + '\\n')"
2022-04-23 21:35:59 +02:00
]
},
{
"cell_type": "code",
"execution_count": null,
2022-06-06 11:25:46 +02:00
"id": "8de69da7-999e-4653-8308-b9d7e6fe24c1",
2022-04-23 21:35:59 +02:00
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.7"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
