fix missing correct files
This commit is contained in:
kpierzynski
parent
b13b78e58e
commit
1f224ccd28
@ -1,3 +1,3 @@
|
||||
## challenging-america-word-gap-prediction
|
||||
### using simple trigram nn
|
||||
calculated perplexity: 583.35
|
||||
calculated perplexity: 653.89
|
||||
|
||||
21038
dev-0/out.tsv
21038
dev-0/out.tsv
File diff suppressed because it is too large
Load Diff
59
run.py
59
run.py
@ -17,12 +17,12 @@ import torch
|
||||
|
||||
from tqdm.notebook import tqdm
|
||||
|
||||
embed_size = 100
|
||||
vocab_size = 25_000
|
||||
num_epochs = 1
|
||||
embed_size = 30
|
||||
vocab_size = 10_000
|
||||
num_epochs = 2
|
||||
device = 'cuda'
|
||||
batch_size = 2048
|
||||
train_file_path = 'train/train.txt'
|
||||
batch_size = 8192
|
||||
train_file_path = 'train/nano.txt'
|
||||
|
||||
with open(train_file_path, 'r', encoding='utf-8') as file:
|
||||
total = len(file.readlines())
|
||||
@ -43,7 +43,7 @@ def get_words_from_line(line):
|
||||
def get_word_lines_from_file(file_name):
|
||||
limit = total * 2
|
||||
with open(file_name, 'r', encoding='utf8') as fh:
|
||||
for line in tqdm(fh, total=total):
|
||||
for line in fh:
|
||||
limit -= 1
|
||||
if not limit:
|
||||
break
|
||||
@ -82,7 +82,6 @@ train_dataset = Trigrams(train_file_path, vocab_size)
|
||||
# In[3]:
|
||||
|
||||
|
||||
# Neural network model for trigram language modeling
|
||||
class SimpleTrigramNeuralLanguageModel(nn.Module):
|
||||
def __init__(self, vocabulary_size, embedding_size):
|
||||
super(SimpleTrigramNeuralLanguageModel, self).__init__()
|
||||
@ -93,48 +92,50 @@ class SimpleTrigramNeuralLanguageModel(nn.Module):
|
||||
self.embedding_size = embedding_size
|
||||
|
||||
def forward(self, x):
|
||||
embeds = self.embedding(x).view(-1, self.embedding_size * 2)
|
||||
out = torch.relu(self.linear1(embeds))
|
||||
embeds = self.embedding(x).view(x.size(0), -1)
|
||||
out = self.linear1(embeds)
|
||||
out = self.linear2(out)
|
||||
return self.softmax(out)
|
||||
|
||||
# Instantiate the model
|
||||
model = SimpleTrigramNeuralLanguageModel(vocab_size, embed_size)
|
||||
model = SimpleTrigramNeuralLanguageModel(vocab_size, embed_size).to(device)
|
||||
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
model = SimpleTrigramNeuralLanguageModel(vocab_size, embed_size).to(device)
|
||||
data = DataLoader(train_dataset, batch_size=batch_size)
|
||||
optimizer = torch.optim.Adam(model.parameters())
|
||||
criterion = torch.nn.NLLLoss()
|
||||
criterion = torch.nn.CrossEntropyLoss()
|
||||
|
||||
|
||||
# In[5]:
|
||||
|
||||
|
||||
model.train()
|
||||
step = 0
|
||||
for _ in range(num_epochs):
|
||||
for x1,x2,y in tqdm(data, total=total):
|
||||
x = torch.cat((x1,x2), dim=0).to(device)
|
||||
for x1,x2,y in tqdm(data, desc="Train loop"):
|
||||
y = y.to(device)
|
||||
x = torch.cat((x1.unsqueeze(1),x2.unsqueeze(1)), dim=1).to(device)
|
||||
optimizer.zero_grad()
|
||||
ypredicted = model(x)
|
||||
|
||||
loss = criterion(torch.log(ypredicted), y)
|
||||
if step % 5000 == 0:
|
||||
print(step, loss)
|
||||
step += 1
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
step = 0
|
||||
model.eval()
|
||||
|
||||
|
||||
# In[10]:
|
||||
# In[6]:
|
||||
|
||||
|
||||
def get_gap_candidates(words, n=10, vocab=train_dataset.vocab):
|
||||
ixs = vocab.forward(words)
|
||||
ixs = torch.tensor(ixs)
|
||||
ixs = torch.cat(tuple([ixs]), dim=0).to(device)
|
||||
ixs = vocab(words)
|
||||
ixs = torch.tensor(ixs).unsqueeze(0).to(device)
|
||||
|
||||
out = model(ixs)
|
||||
top = torch.topk(out[0], n)
|
||||
@ -144,7 +145,7 @@ def get_gap_candidates(words, n=10, vocab=train_dataset.vocab):
|
||||
return list(zip(top_words, top_probs))
|
||||
|
||||
|
||||
# In[11]:
|
||||
# In[7]:
|
||||
|
||||
|
||||
def clean(text):
|
||||
@ -172,13 +173,7 @@ def predictor(prefix):
|
||||
return output
|
||||
|
||||
|
||||
# In[12]:
|
||||
|
||||
|
||||
predictor("I really bug")
|
||||
|
||||
|
||||
# In[13]:
|
||||
# In[8]:
|
||||
|
||||
|
||||
def generate_result(input_path, output_path='out.tsv'):
|
||||
@ -191,14 +186,8 @@ def generate_result(input_path, output_path='out.tsv'):
|
||||
output_file.write(result + '\n')
|
||||
|
||||
|
||||
# In[14]:
|
||||
# In[9]:
|
||||
|
||||
|
||||
generate_result('dev-0/in.tsv', output_path='dev-0/out.tsv')
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
169
trigram.ipynb
169
trigram.ipynb
@ -10,11 +10,11 @@
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"S:\\WENV\\Lib\\site-packages\\torchtext\\vocab\\__init__.py:4: UserWarning: \n",
|
||||
"S:\\WENV_TORCHTEXT\\Lib\\site-packages\\torchtext\\vocab\\__init__.py:4: UserWarning: \n",
|
||||
"/!\\ IMPORTANT WARNING ABOUT TORCHTEXT STATUS /!\\ \n",
|
||||
"Torchtext is deprecated and the last released version will be 0.18 (this one). You can silence this warning by calling the following at the beginnign of your scripts: `import torchtext; torchtext.disable_torchtext_deprecation_warning()`\n",
|
||||
" warnings.warn(torchtext._TORCHTEXT_DEPRECATION_MSG)\n",
|
||||
"S:\\WENV\\Lib\\site-packages\\torchtext\\utils.py:4: UserWarning: \n",
|
||||
"S:\\WENV_TORCHTEXT\\Lib\\site-packages\\torchtext\\utils.py:4: UserWarning: \n",
|
||||
"/!\\ IMPORTANT WARNING ABOUT TORCHTEXT STATUS /!\\ \n",
|
||||
"Torchtext is deprecated and the last released version will be 0.18 (this one). You can silence this warning by calling the following at the beginnign of your scripts: `import torchtext; torchtext.disable_torchtext_deprecation_warning()`\n",
|
||||
" warnings.warn(torchtext._TORCHTEXT_DEPRECATION_MSG)\n"
|
||||
@ -35,12 +35,12 @@
|
||||
"\n",
|
||||
"from tqdm.notebook import tqdm\n",
|
||||
"\n",
|
||||
"embed_size = 100\n",
|
||||
"vocab_size = 25_000\n",
|
||||
"num_epochs = 1\n",
|
||||
"embed_size = 30\n",
|
||||
"vocab_size = 10_000\n",
|
||||
"num_epochs = 2\n",
|
||||
"device = 'cuda'\n",
|
||||
"batch_size = 2048\n",
|
||||
"train_file_path = 'train/train.txt'\n",
|
||||
"batch_size = 8192\n",
|
||||
"train_file_path = 'train/nano.txt'\n",
|
||||
"\n",
|
||||
"with open(train_file_path, 'r', encoding='utf-8') as file:\n",
|
||||
" total = len(file.readlines())"
|
||||
@ -51,22 +51,7 @@
|
||||
"execution_count": 2,
|
||||
"id": "40392665-79bc-4032-a5de-9d189545c9f7",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "5049d1e295954b7baf71ac05a793071a",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/432022 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Function to extract words from a line of text\n",
|
||||
"def get_words_from_line(line):\n",
|
||||
@ -80,7 +65,7 @@
|
||||
"def get_word_lines_from_file(file_name):\n",
|
||||
" limit = total * 2\n",
|
||||
" with open(file_name, 'r', encoding='utf8') as fh:\n",
|
||||
" for line in tqdm(fh, total=total):\n",
|
||||
" for line in fh:\n",
|
||||
" limit -= 1\n",
|
||||
" if not limit:\n",
|
||||
" break\n",
|
||||
@ -123,7 +108,6 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Neural network model for trigram language modeling\n",
|
||||
"class SimpleTrigramNeuralLanguageModel(nn.Module):\n",
|
||||
" def __init__(self, vocabulary_size, embedding_size):\n",
|
||||
" super(SimpleTrigramNeuralLanguageModel, self).__init__()\n",
|
||||
@ -134,44 +118,41 @@
|
||||
" self.embedding_size = embedding_size\n",
|
||||
"\n",
|
||||
" def forward(self, x):\n",
|
||||
" embeds = self.embedding(x).view(-1, self.embedding_size * 2)\n",
|
||||
" out = torch.relu(self.linear1(embeds))\n",
|
||||
" embeds = self.embedding(x).view(x.size(0), -1)\n",
|
||||
" out = self.linear1(embeds)\n",
|
||||
" out = self.linear2(out)\n",
|
||||
" return self.softmax(out)\n",
|
||||
"\n",
|
||||
"# Instantiate the model\n",
|
||||
"model = SimpleTrigramNeuralLanguageModel(vocab_size, embed_size)"
|
||||
"model = SimpleTrigramNeuralLanguageModel(vocab_size, embed_size).to(device)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "32ea22db-7259-4549-a9d5-4781d9bc99bc",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"data = DataLoader(train_dataset, batch_size=batch_size)\n",
|
||||
"optimizer = torch.optim.Adam(model.parameters())\n",
|
||||
"criterion = torch.nn.CrossEntropyLoss()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "0858967e-5143-4253-921d-a009dbbdca27",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "c422bb888518406e9f6a4a8f10f2b473",
|
||||
"model_id": "5e4b6ce6edf94b90a70d415d75be7eb6",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/432022 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "4020a53c31544ef3b4017c43798aa305",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/432022 [00:00<?, ?it/s]"
|
||||
"Train loop: 0it [00:00, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
@ -181,73 +162,76 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"0 tensor(10.1654, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||||
"5000 tensor(6.5147, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||||
"10000 tensor(6.6747, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||||
"15000 tensor(6.9061, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||||
"20000 tensor(6.8899, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||||
"25000 tensor(6.8373, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||||
"30000 tensor(6.8942, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||||
"35000 tensor(6.9564, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||||
"40000 tensor(6.9709, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||||
"45000 tensor(6.9592, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||||
"50000 tensor(6.8195, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||||
"55000 tensor(6.7074, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||||
"60000 tensor(6.8755, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||||
"65000 tensor(6.9605, device='cuda:0', grad_fn=<NllLossBackward0>)\n"
|
||||
"0 tensor(9.2450, device='cuda:0', grad_fn=<NllLossBackward0>)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "044c2ea05e344306881002e34d89bd54",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Train loop: 0it [00:00, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"0 tensor(6.2669, device='cuda:0', grad_fn=<NllLossBackward0>)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"SimpleTrigramNeuralLanguageModel(\n",
|
||||
" (embedding): Embedding(25000, 100)\n",
|
||||
" (linear1): Linear(in_features=200, out_features=100, bias=True)\n",
|
||||
" (linear2): Linear(in_features=100, out_features=25000, bias=True)\n",
|
||||
" (embedding): Embedding(10000, 30)\n",
|
||||
" (linear1): Linear(in_features=60, out_features=30, bias=True)\n",
|
||||
" (linear2): Linear(in_features=30, out_features=10000, bias=True)\n",
|
||||
" (softmax): Softmax(dim=1)\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model = SimpleTrigramNeuralLanguageModel(vocab_size, embed_size).to(device)\n",
|
||||
"data = DataLoader(train_dataset, batch_size=batch_size)\n",
|
||||
"optimizer = torch.optim.Adam(model.parameters())\n",
|
||||
"criterion = torch.nn.NLLLoss()\n",
|
||||
"\n",
|
||||
"model.train()\n",
|
||||
"step = 0\n",
|
||||
"for _ in range(num_epochs):\n",
|
||||
" for x1,x2,y in tqdm(data, total=total):\n",
|
||||
" x = torch.cat((x1,x2), dim=0).to(device)\n",
|
||||
" for x1,x2,y in tqdm(data, desc=\"Train loop\"):\n",
|
||||
" y = y.to(device)\n",
|
||||
" x = torch.cat((x1.unsqueeze(1),x2.unsqueeze(1)), dim=1).to(device)\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
" ypredicted = model(x)\n",
|
||||
" \n",
|
||||
" loss = criterion(torch.log(ypredicted), y)\n",
|
||||
" if step % 5000 == 0:\n",
|
||||
" print(step, loss)\n",
|
||||
" step += 1\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
"\n",
|
||||
" step = 0\n",
|
||||
"model.eval()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"execution_count": 6,
|
||||
"id": "da4d116c-beec-436d-84d8-577282507226",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def get_gap_candidates(words, n=10, vocab=train_dataset.vocab):\n",
|
||||
" ixs = vocab.forward(words)\n",
|
||||
" ixs = torch.tensor(ixs)\n",
|
||||
" ixs = torch.cat(tuple([ixs]), dim=0).to(device)\n",
|
||||
" ixs = vocab(words)\n",
|
||||
" ixs = torch.tensor(ixs).unsqueeze(0).to(device)\n",
|
||||
"\n",
|
||||
" out = model(ixs)\n",
|
||||
" top = torch.topk(out[0], n)\n",
|
||||
@ -259,7 +243,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"execution_count": 7,
|
||||
"id": "0cafd70a-29b3-4a49-b40f-b8ce3143084a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@ -291,28 +275,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "2084fb5f-6405-4e44-a06f-953db852e526",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'the:0.07267232984304428 of:0.043321046978235245 and:0.032147664576768875 to:0.02692588046193123 a:0.020654045045375824 in:0.020213929936289787 that:0.010836434550583363 is:0.00959325022995472 it:0.008407277055084705 :0.755228141322732'"
|
||||
]
|
||||
},
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"predictor(\"I really bug\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"execution_count": 8,
|
||||
"id": "965ebaf3-4c0b-4462-8ac5-4746ec9489ab",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@ -329,21 +292,13 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"execution_count": 9,
|
||||
"id": "80547ba7-9d01-4d2b-9e83-269919513de9",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"generate_result('dev-0/in.tsv', output_path='dev-0/out.tsv')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "1d7d72b0-d629-487e-bcec-4756fa88ae49",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user