Zadanie RNN

Untitled.py

@@ -0,0 +1,248 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[ ]:
+
+
+import nltk
+from nltk.tokenize import word_tokenize
+import torch
+from tqdm.notebook import tqdm
+from sklearn.model_selection import train_test_split
+import numpy as np
+nltk.download('punkt')
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+print('Using {} device'.format(device))
+
+with open("train/train.tsv", "r", encoding='utf-8') as lalka_path:
+    lines = lalka_path.readlines()
+train, test = train_test_split(lines, test_size = 0.2)
+with open("train/train_train.tsv", "w", encoding='utf-8') as out_train_file:
+    for i in train:
+        out_train_file.write(i)
+with open("train/train_test.tsv", "w", encoding='utf-8') as out_test_file:
+    for i in test:
+        out_test_file.write(i)
+
+lalka_path_train= 'train/train_train.tsv'
+lalka_path_valid= 'train/train_test.tsv'
+
+corpora_train = open(lalka_path_train, encoding='utf-8').read()
+
+corpora_train_tokenized = list(word_tokenize(corpora_train))
+corpora_train_tokenized = [token.lower() for token in corpora_train_tokenized]
+
+vocab_itos = sorted(set(corpora_train_tokenized))
+print(len(vocab_itos))
+
+vocab_itos = vocab_itos[:15005]
+vocab_itos[15001] = "<UNK>"
+vocab_itos[15002] = "<BOS>"
+vocab_itos[15003] = "<EOS>"
+vocab_itos[15004] = "<PAD>"
+
+print(len(vocab_itos))
+
+vocab_stoi = dict()
+for i, token in enumerate(vocab_itos):
+    vocab_stoi[token] = i
+
+NGRAMS = 5
+
+def get_token_id(dataset):
+    token_ids = [vocab_stoi['<PAD>']] * (NGRAMS-1) + [vocab_stoi['<BOS>']]
+    for token in dataset:
+        try:
+            token_ids.append(vocab_stoi[token])
+        except KeyError:
+            token_ids.append(vocab_stoi['<UNK>'])
+    token_ids.append(vocab_stoi['<EOS>'])
+    return token_ids
+
+train_ids = get_token_id(corpora_train_tokenized)
+
+def get_samples(dataset):
+    samples = []
+    for i in range(len(dataset)-NGRAMS):
+        samples.append(dataset[i:i+NGRAMS])
+    return samples
+
+train_ids = get_samples(train_ids)
+
+train_ids = torch.tensor(train_ids, device = device)
+
+corpora_valid = open(lalka_path_valid, encoding='utf-8').read()
+
+corpora_valid_tokenized = list(word_tokenize(corpora_valid))
+corpora_valid_tokenized = [token.lower() for token in corpora_valid_tokenized]
+
+valid_ids = get_token_id(corpora_valid_tokenized)
+
+valid_ids = torch.tensor(get_samples(valid_ids), dtype = torch.long, device = device)
+
+class GRU(torch.nn.Module):
+
+    def __init__(self):
+        super(GRU, self).__init__()
+        self.emb = torch.nn.Embedding(len(vocab_itos),100)
+        self.rec = torch.nn.GRU(100, 256, 1, batch_first = True)
+        self.fc1 = torch.nn.Linear( 256 ,len(vocab_itos))
+        self.dropout = torch.nn.Dropout(0.5)
+
+    def forward(self, x):
+        emb = self.emb(x)
+        #emb = self.dropout(emb)
+        output, h_n = self.rec(emb)
+        hidden = h_n.squeeze(0)
+        out = self.fc1(hidden)
+        out = self.dropout(out)
+        return out
+
+lm = GRU().to(device)
+
+criterion = torch.nn.CrossEntropyLoss()
+
+optimizer = torch.optim.Adam(lm.parameters(),lr=0.0001)
+
+BATCH_SIZE = 128
+EPOCHS = 15
+
+def get_ppl(dataset_ids):
+    lm.eval()
+    batches = 0
+    loss_sum =0
+    acc_score = 0
+    for i in range(0, len(dataset_ids)-BATCH_SIZE+1, BATCH_SIZE):
+        X = dataset_ids[i:i+BATCH_SIZE,:NGRAMS-1]
+        Y = dataset_ids[i:i+BATCH_SIZE,NGRAMS-1]
+        predictions = lm(X)
+        
+        # equally distributted
+        # predictions = torch.zeros_like(predictions)
+        
+        loss = criterion(predictions,Y)
+
+        loss_sum += loss.item()
+        batches += 1
+
+    return np.exp(loss_sum / batches)
+
+history_ppl_train  = []
+history_ppl_valid  = []
+for epoch in range(EPOCHS):
+    
+    batches = 0
+    loss_sum =0
+    acc_score = 0
+    lm.train()
+    #for i in range(0, len(train_ids)-BATCH_SIZE+1, BATCH_SIZE):
+    for i in tqdm(range(0, len(train_ids)-BATCH_SIZE+1, BATCH_SIZE)):
+        X = train_ids[i:i+BATCH_SIZE,:NGRAMS-1]
+        Y = train_ids[i:i+BATCH_SIZE,NGRAMS-1]
+        predictions = lm(X)
+        loss = criterion(predictions,Y)
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+        loss_sum += loss.item()
+        batches += 1
+        
+    ppl_train = get_ppl(train_ids)
+    ppl_valid = get_ppl(valid_ids)
+    
+    history_ppl_train.append(ppl_train)
+    history_ppl_valid.append(ppl_valid)    
+    print('epoch: ', epoch)
+    print('train ppl: ', ppl_train)
+    print('valid ppl: ', ppl_valid)
+    print()
+
+tokenized = list(word_tokenize('Gości innych nie widział oprócz spółleśników'))
+tokenized = [token.lower() for token in tokenized]
+
+ids = []
+for word in tokenized:
+    if word in vocab_stoi:
+        ids.append(vocab_stoi[word])
+    else:
+        ids.append(vocab_stoi['<UNK>'])
+
+lm.eval()
+
+ids = torch.tensor(ids, dtype = torch.long, device = device)
+
+preds= lm(ids.unsqueeze(0))
+
+vocab_itos[torch.argmax(torch.softmax(preds,1),1).item()]
+
+tokenized = list(word_tokenize('Lalka'))
+tokenized = [token.lower() for token in tokenized]
+
+ids = []
+for word in tokenized:
+    if word in vocab_stoi:
+        ids.append(vocab_stoi[word])
+    else:
+        ids.append(vocab_stoi['<UNK>'])
+ids = torch.tensor([ids], dtype = torch.long, device = device)
+
+candidates_number = 10
+for i in range(30):
+    preds= lm(ids)
+    candidates = torch.topk(torch.softmax(preds,1),candidates_number)[1][0].cpu().numpy()
+    candidate = 15001
+    while candidate > 15000:
+        candidate = candidates[np.random.randint(candidates_number)]
+    print(vocab_itos[candidate])
+    ids =  torch.cat((ids, torch.tensor([[candidate]], device = device)), 1)
+
+with open("dev-0/in.tsv", "r", encoding='utf-8') as dev_path:
+    nr_of_dev_lines = len(dev_path.readlines())
+with open("test-A/in.tsv", "r", encoding='utf-8') as test_a_path:
+    nr_of_test_a_lines = len(test_a_path.readlines())
+with open("dev-0/out.tsv", "w", encoding='utf-8') as out_dev_file:
+    for i in range(nr_of_dev_lines):
+        preds= lm(ids)
+        candidates = torch.topk(torch.softmax(preds,1),candidates_number)[1][0].cpu().numpy()
+        candidate = 15001
+        while candidate > 15000:
+            candidate = candidates[np.random.randint(candidates_number)]
+        print(vocab_itos[candidate])
+        ids =  torch.cat((ids, torch.tensor([[candidate]], device = device)), 1)
+        out_dev_file.write(vocab_itos[candidate] + '\n')
+with open("test-A/out.tsv", "w", encoding='utf-8') as out_test_file:
+    for i in range(nr_of_dev_lines):
+        preds= lm(ids)
+        candidates = torch.topk(torch.softmax(preds,1),candidates_number)[1][0].cpu().numpy()
+        candidate = 15001
+        while candidate > 15000:
+            candidate = candidates[np.random.randint(candidates_number)]
+        print(vocab_itos[candidate])
+        ids =  torch.cat((ids, torch.tensor([[candidate]], device = "cuda")), 1)
+        out_test_file.write(vocab_itos[candidate]  + '\n')
+
+
+# In[10]:
+
+
+get_ipython().system('pip install ipywidgets')
+
+
+# In[12]:
+
+
+get_ipython().system('pip install notebook --upgrade ')
+
+
+# In[14]:
+
+
+get_ipython().system('jupyter nbextension enable --py widgetsnbextension')
+
+
+# In[ ]:
+
+
+
+

dev-0/out.tsv

@@ -0,0 +1,471 @@
+na
+,
+jak
+ja
+.
+po
+cóż
+,
+na
+której
+!
+o
+i
+nawet
+już
+na
+mnie
+.
+panna
+,
+czy
+jeszcze
+nie
+jest
+na
+niego
+nie
+.
+,
+nie
+na
+mnie
+nie
+o
+czym
+od
+dwudziestu
+na
+nią
+.
+,
+a
+na
+jego
+nie
+i
+po
+co
+.
+pan
+,
+panie
+?
+,
+a
+i
+nawet
+nie
+było
+,
+nie
+mógł
+od
+.
+po
+co
+.
+na
+lewo
+i
+nie
+,
+i
+ja
+mam
+do
+końca
+!
+nie
+o
+mnie
+.
+na
+chwilę
+o
+mnie
+nie
+do
+paryża
+i
+po
+chwili
+od
+niego
+:
+nie
+,
+co
+pan
+ignacy
+!
+,
+a
+do
+końca
+,
+kiedy
+pan
+do
+pani
+,
+która
+go
+nie
+jest
+bardzo
+?
+do
+i
+nawet
+,
+ażeby
+go
+:
+,
+ale
+nawet
+nie
+na
+mnie
+?
+po
+o
+jej
+nie
+ma
+do
+domu
+,
+ażeby
+mnie
+,
+na
+której
+;
+już
+mi
+na
+,
+ale
+gdyby
+nawet
+i
+nie
+,
+o
+czym
+,
+co
+na
+obiad
+;
+a
+gdy
+już
+o
+.
+i
+ja
+,
+a
+do
+jego
+i
+mówi
+na
+,
+ażeby
+:
+pani
+małgorzata
+do
+panny
+,
+nie
+mogła
+,
+ażeby
+mnie
+!
+co
+do
+domu
+;
+na
+mnie
+?
+i
+nie
+jest
+mnie
+!
+na
+mnie
+od
+.
+,
+a
+on
+już
+o
+panu
+nie
+na
+do
+i
+o
+jej
+nie
+na
+co
+!
+,
+nie
+mógł
+,
+co
+nawet
+nie
+.
+nawet
+po
+francusku
+nie
+było
+?
+o
+co
+do
+nas
+!
+na
+jej
+!
+o
+.
+.
+gdy
+i
+,
+co
+mu
+nie
+było
+do
+nas
+?
+nie
+jest
+?
+a
+potem
+na
+jej
+:
+co
+pan
+?
+)
+do
+niej
+i
+po
+chwili
+,
+bo
+,
+nie
+jest
+kobietą
+!
+)
+?
+na
+i
+ja
+po
+na
+co
+?
+i
+jak
+mnie
+.
+panna
+izabela
+do
+.
+,
+jeżeli
+o
+,
+ale
+o
+co
+do
+mnie
+i
+o
+nim
+,
+a
+i
+jego
+już
+.
+ale
+na
+nią
+na
+miejscu
+do
+i
+po
+co
+dzień
+?
+i
+nawet
+,
+a
+i
+jej
+.
+o
+ile
+mu
+bardzo
+o
+nie
+,
+na
+co
+i
+,
+i
+nareszcie
+go
+już
+od
+,
+na
+mnie
+na
+i
+na
+jej
+,
+a
+potem
+i
+jego
+drzwi
+na
+powrót
+:
+,
+ale
+on
+.
+ale
+na
+niego
+do
+panny
+izabeli
+.
+po
+nie
+,
+co
+na
+jego
+!
+nie
+jest
+i
+na
+.
+,
+nie
+był
+nie
+mam
+do
+nas
+.
+na
+,
+ażeby
+go
+do
+.
+i
+dopiero
+.
+ale
+czy
+do
+,
+a
+ja
+do
+panny
+;
+ale
+nie
+może
+pan
+do
+pani
+;
+i
+ja
+do
+mnie
+do
+drzwi
+,
+a
+pan
+mnie
+:
+co
+pan
+ignacy
+.
+i
+ja
+go
+.
+pan
+,
+i
+nie
+ma
+.
+od
+nas
+i
+już
+o
+mnie
+.
+czy
+ja
+na
+niego
+i
+na
+mnie
+na
+i
+nie
+jest
+o
+.
+a
+pani
+baronowa
+i

test-A/out.tsv

@@ -0,0 +1,471 @@
+ja
+,
+nie
+na
+co
+,
+bo
+nie
+,
+co
+na
+niego
+,
+ale
+ja
+nie
+mam
+?
+i
+już
+do
+końca
+?
+a
+może
+na
+jego
+:
+czy
+,
+kiedy
+mi
+do
+mnie
+nie
+było
+do
+domu
+nie
+jest
+.
+czy
+może
+mnie
+,
+i
+,
+i
+już
+nic
+mnie
+na
+i
+.
+do
+nas
+pani
+nie
+,
+a
+nie
+o
+nim
+od
+niej
+;
+a
+może
+na
+niego
+od
+,
+kiedy
+już
+o
+nie
+było
+,
+do
+mnie
+,
+ale
+jeszcze
+i
+jej
+i
+.
+już
+już
+,
+jak
+mnie
+!
+i
+jeszcze
+dalej
+;
+ale
+nie
+do
+.
+nie
+ma
+na
+nią
+:
+co
+do
+nas
+!
+)
+i
+nie
+.
+i
+nawet
+mu
+i
+nawet
+już
+od
+do
+paryża
+i
+o
+mnie
+nie
+było
+na
+myśl
+o
+!
+.
+pani
+meliton
+na
+myśl
+.
+nie
+był
+mu
+.
+o
+ile
+mu
+o
+.
+.
+i
+ja
+.
+na
+jej
+do
+kościoła
+nie
+i
+już
+na
+myśl
+i
+na
+lewo
+nie
+mógł
+na
+mnie
+na
+i
+.
+gdy
+na
+niego
+.
+na
+,
+jak
+mnie
+.
+po
+południu
+na
+myśl
+,
+nie
+,
+i
+jeszcze
+dalej
+:
+i
+ja
+go
+nie
+.
+na
+jej
+.
+i
+dopiero
+:
+dla
+nich
+,
+ażeby
+go
+pod
+,
+na
+których
+,
+na
+którego
+:
+na
+prawo
+nie
+mógł
+ani
+na
+myśl
+!
+.
+nie
+,
+panie
+.
+po
+chwili
+i
+już
+o
+,
+ale
+ja
+.
+a
+jeżeli
+i
+na
+jego
+na
+myśl
+o
+nie
+i
+nie
+.
+po
+dziś
+na
+obiad
+?
+na
+jej
+,
+ażeby
+mnie
+do
+pani
+baronowej
+nie
+jest
+o
+do
+i
+.
+a
+jeżeli
+do
+mego
+i
+po
+na
+obiad
+?
+,
+i
+ja
+.
+pan
+,
+i
+po
+południu
+,
+ale
+nawet
+mi
+o
+mnie
+!
+co
+na
+co
+do
+,
+o
+której
+,
+ale
+nie
+mógł
+mnie
+?
+)
+!
+co
+na
+co
+,
+na
+i
+,
+co
+chwilę
+na
+myśl
+i
+,
+ale
+nie
+jest
+?
+czy
+może
+i
+pan
+.
+o
+nie
+,
+a
+pani
+,
+ale
+ja
+jestem
+kupcem
+,
+nie
+,
+a
+nawet
+,
+kiedy
+?
+a
+,
+jak
+ona
+:
+nie
+mam
+od
+dwudziestu
+i
+i
+do
+pani
+;
+lecz
+i
+o
+jego
+do
+i
+nawet
+i
+po
+chwili
+do
+domu
+;
+lecz
+gdy
+nie
+,
+nie
+może
+nie
+ma
+?
+czy
+do
+.
+na
+co
+i
+o
+mnie
+;
+a
+potem
+?
+,
+czy
+nie
+było
+na
+niej
+.
+,
+czy
+do
+ciebie
+?
+nie
+o
+,
+czy
+na
+jego
+i
+nie
+na
+jej
+:
+na
+,
+i
+jeszcze
+dalej
+do
+głowy
+nie
+było
+i
+po
+pokoju
+do
+,
+czy
+ja
+,
+i
+już
+o
+mnie
+i
+po
+francusku
+:
+co
+?
+,
+który
+i
+ja
+i
+nie
+jest
+o
+.
+czy
+już
+był
+bardzo
+od
+pani
+krzeszowskiej
+nie
+było
+do
+,
+nie
+na
+jej
+,
+ale
+jeszcze