32 KiB
32 KiB
Modelowanie Języka
10. Model rekurencyjny z atencją [ćwiczenia]
Jakub Pokrywka (2022)
# https://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html
from __future__ import unicode_literals, print_function, division
from io import open
import unicodedata
import string
import re
import random
import torch
import torch.nn as nn
from torch import optim
import torch.nn.functional as F
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
SOS_token = 0
EOS_token = 1
class Lang:
def __init__(self):
self.word2index = {}
self.word2count = {}
self.index2word = {0: "SOS", 1: "EOS"}
self.n_words = 2 # Count SOS and EOS
def addSentence(self, sentence):
for word in sentence.split(' '):
self.addWord(word)
def addWord(self, word):
if word not in self.word2index:
self.word2index[word] = self.n_words
self.word2count[word] = 1
self.index2word[self.n_words] = word
self.n_words += 1
else:
self.word2count[word] += 1
def unicodeToAscii(s):
return ''.join(
c for c in unicodedata.normalize('NFD', s)
if unicodedata.category(c) != 'Mn'
)
pairs = []
with open('data/eng-fra.txt') as f:
for line in f:
eng_line, fra_line = line.lower().rstrip().split('\t')
eng_line = re.sub(r"([.!?])", r" \1", eng_line)
eng_line = re.sub(r"[^a-zA-Z.!?]+", r" ", eng_line)
fra_line = re.sub(r"([.!?])", r" \1", fra_line)
fra_line = re.sub(r"[^a-zA-Z.!?]+", r" ", fra_line)
eng_line = unicodeToAscii(eng_line)
fra_line = unicodeToAscii(fra_line)
pairs.append([eng_line, fra_line])
pairs[1]
['run !', 'cours !']
MAX_LENGTH = 10
eng_prefixes = (
"i am ", "i m ",
"he is", "he s ",
"she is", "she s ",
"you are", "you re ",
"we are", "we re ",
"they are", "they re "
)
pairs = [p for p in pairs if len(p[0].split(' ')) < MAX_LENGTH and len(p[1].split(' ')) < MAX_LENGTH]
pairs = [p for p in pairs if p[0].startswith(eng_prefixes)]
eng_lang = Lang()
fra_lang = Lang()
for pair in pairs:
eng_lang.addSentence(pair[0])
fra_lang.addSentence(pair[1])
pairs[0]
['i m .', 'j ai ans .']
pairs[1]
['i m ok .', 'je vais bien .']
pairs[2]
['i m ok .', ' a va .']
class EncoderRNN(nn.Module):
def __init__(self, input_size, hidden_size):
super(EncoderRNN, self).__init__()
self.hidden_size = hidden_size
self.embedding = nn.Embedding(input_size, hidden_size)
self.gru = nn.GRU(hidden_size, hidden_size)
def forward(self, input, hidden):
embedded = self.embedding(input).view(1, 1, -1)
output = embedded
output, hidden = self.gru(output, hidden)
return output, hidden
def initHidden(self):
return torch.zeros(1, 1, self.hidden_size, device=device)
class DecoderRNN(nn.Module):
def __init__(self, hidden_size, output_size):
super(DecoderRNN, self).__init__()
self.hidden_size = hidden_size
self.embedding = nn.Embedding(output_size, hidden_size)
self.gru = nn.GRU(hidden_size, hidden_size)
self.out = nn.Linear(hidden_size, output_size)
self.softmax = nn.LogSoftmax(dim=1)
def forward(self, input, hidden):
output = self.embedding(input).view(1, 1, -1)
output = F.relu(output)
output, hidden = self.gru(output, hidden)
output = self.softmax(self.out(output[0]))
return output, hidden
def initHidden(self):
return torch.zeros(1, 1, self.hidden_size, device=device)
class AttnDecoderRNN(nn.Module):
def __init__(self, hidden_size, output_size, dropout_p=0.1, max_length=MAX_LENGTH):
super(AttnDecoderRNN, self).__init__()
self.hidden_size = hidden_size
self.output_size = output_size
self.dropout_p = dropout_p
self.max_length = max_length
self.embedding = nn.Embedding(self.output_size, self.hidden_size)
self.attn = nn.Linear(self.hidden_size * 2, self.max_length)
self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)
self.dropout = nn.Dropout(self.dropout_p)
self.gru = nn.GRU(self.hidden_size, self.hidden_size)
self.out = nn.Linear(self.hidden_size, self.output_size)
def forward(self, input, hidden, encoder_outputs):
embedded = self.embedding(input).view(1, 1, -1)
embedded = self.dropout(embedded)
attn_weights = F.softmax(
self.attn(torch.cat((embedded[0], hidden[0]), 1)), dim=1)
attn_applied = torch.bmm(attn_weights.unsqueeze(0),
encoder_outputs.unsqueeze(0))
output = torch.cat((embedded[0], attn_applied[0]), 1)
output = self.attn_combine(output).unsqueeze(0)
output = F.relu(output)
output, hidden = self.gru(output, hidden)
output = F.log_softmax(self.out(output[0]), dim=1)
return output, hidden, attn_weights
def initHidden(self):
return torch.zeros(1, 1, self.hidden_size, device=device)
def tensorFromSentence(sentence, lang):
indexes = [lang.word2index[word] for word in sentence.split(' ')]
indexes.append(EOS_token)
return torch.tensor(indexes, dtype=torch.long, device=device).view(-1, 1)
teacher_forcing_ratio = 0.5
def train_one_batch(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, max_length=MAX_LENGTH):
encoder_hidden = encoder.initHidden()
encoder_optimizer.zero_grad()
decoder_optimizer.zero_grad()
input_length = input_tensor.size(0)
target_length = target_tensor.size(0)
encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)
loss = 0
for ei in range(input_length):
encoder_output, encoder_hidden = encoder(
input_tensor[ei], encoder_hidden)
encoder_outputs[ei] = encoder_output[0, 0]
decoder_input = torch.tensor([[SOS_token]], device=device)
decoder_hidden = encoder_hidden
use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False
if use_teacher_forcing:
# Teacher forcing: Feed the target as the next input
for di in range(target_length):
decoder_output, decoder_hidden, decoder_attention = decoder(
decoder_input, decoder_hidden, encoder_outputs)
loss += criterion(decoder_output, target_tensor[di])
decoder_input = target_tensor[di] # Teacher forcing
else:
# Without teacher forcing: use its own predictions as the next input
for di in range(target_length):
decoder_output, decoder_hidden, decoder_attention = decoder(
decoder_input, decoder_hidden, encoder_outputs)
topv, topi = decoder_output.topk(1)
decoder_input = topi.squeeze().detach() # detach from history as input
loss += criterion(decoder_output, target_tensor[di])
if decoder_input.item() == EOS_token:
break
loss.backward()
encoder_optimizer.step()
decoder_optimizer.step()
return loss.item() / target_length
def trainIters(encoder, decoder, n_iters, print_every=1000, learning_rate=0.01):
print_loss_total = 0 # Reset every print_every
encoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate)
decoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate)
training_pairs = [random.choice(pairs) for _ in range(n_iters)]
training_pairs = [(tensorFromSentence(p[0], eng_lang), tensorFromSentence(p[1], fra_lang)) for p in training_pairs]
criterion = nn.NLLLoss()
for i in range(1, n_iters + 1):
training_pair = training_pairs[i - 1]
input_tensor = training_pair[0]
target_tensor = training_pair[1]
loss = train_one_batch(input_tensor,
target_tensor,
encoder,
encoder,
encoder_optimizer,
decoder_optimizer,
criterion)
print_loss_total += loss
if i % print_every == 0:
print_loss_avg = print_loss_total / print_every
print_loss_total = 0
print(f'iter: {i}, loss: {print_loss_avg}')
def evaluate(encoder, decoder, sentence, max_length=MAX_LENGTH):
with torch.no_grad():
input_tensor = tensorFromSentence(sentence, eng_lang)
input_length = input_tensor.size()[0]
encoder_hidden = encoder.initHidden()
encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)
for ei in range(input_length):
encoder_output, encoder_hidden = encoder(input_tensor[ei],
encoder_hidden)
encoder_outputs[ei] += encoder_output[0, 0]
decoder_input = torch.tensor([[SOS_token]], device=device) # SOS
decoder_hidden = encoder_hidden
decoded_words = []
decoder_attentions = torch.zeros(max_length, max_length)
for di in range(max_length):
decoder_output, decoder_hidden, decoder_attention = decoder(
decoder_input, decoder_hidden, encoder_outputs)
decoder_attentions[di] = decoder_attention.data
topv, topi = decoder_output.data.topk(1)
if topi.item() == EOS_token:
decoded_words.append('<EOS>')
break
else:
decoded_words.append(fra_lang.index2word[topi.item()])
decoder_input = topi.squeeze().detach()
return decoded_words, decoder_attentions[:di + 1]
def evaluateRandomly(encoder, decoder, n=10):
for i in range(n):
pair = random.choice(pairs)
print('>', pair[0])
print('=', pair[1])
output_words, attentions = evaluate(encoder, decoder, pair[0])
output_sentence = ' '.join(output_words)
print('<', output_sentence)
print('')
hidden_size = 256
encoder1 = EncoderRNN(eng_lang.n_words, hidden_size).to(device)
attn_decoder1 = AttnDecoderRNN(hidden_size, fra_lang.n_words, dropout_p=0.1).to(device)
trainIters(encoder1, attn_decoder1, 75000, print_every=50)
iter: 50, loss: 4.78930813773473 iter: 100, loss: 4.554949267220875 iter: 150, loss: 4.238516052685087 iter: 200, loss: 4.279887475513276 iter: 250, loss: 4.1802274973884455 iter: 300, loss: 4.2113521892305394 iter: 350, loss: 4.266180963228619 iter: 400, loss: 4.225914733432588 iter: 450, loss: 4.1369073431075565 iter: 500, loss: 3.9906799076019768 iter: 550, loss: 3.842005534717016 iter: 600, loss: 4.081443620484972 iter: 650, loss: 4.030401878296383 iter: 700, loss: 3.869014380984837 iter: 750, loss: 3.8505467753031906 iter: 800, loss: 3.855170104072209 iter: 850, loss: 3.675745445599631 iter: 900, loss: 3.9147777624584386 iter: 950, loss: 3.766264297788106 iter: 1000, loss: 3.6813155986997814 iter: 1050, loss: 3.9307321495934144 iter: 1100, loss: 3.9047770059525027 iter: 1150, loss: 3.655722749588981 iter: 1200, loss: 3.540693810886806 iter: 1250, loss: 3.790360960324605 iter: 1300, loss: 3.7472636015907153 iter: 1350, loss: 3.641857419574072 iter: 1400, loss: 3.717327400631375 iter: 1450, loss: 3.4848567311423166 iter: 1500, loss: 3.56774485397339 iter: 1550, loss: 3.460277635226175 iter: 1600, loss: 3.241899683013796 iter: 1650, loss: 3.50151977614751 iter: 1700, loss: 3.621569488313462 iter: 1750, loss: 3.3851226735947626 iter: 1800, loss: 3.346289497057597 iter: 1850, loss: 3.5180823354569695 iter: 1900, loss: 3.433616197676886 iter: 1950, loss: 3.6162788327080864 iter: 2000, loss: 3.4990604458763492 iter: 2050, loss: 3.3144700173423405 iter: 2100, loss: 3.2962356294980135 iter: 2150, loss: 3.1448448797861728 iter: 2200, loss: 3.6958242581534018 iter: 2250, loss: 3.5269318538241925 iter: 2300, loss: 3.180744191850934 iter: 2350, loss: 3.317159715145354 iter: 2400, loss: 3.638545340795366 iter: 2450, loss: 3.7591161967988995 iter: 2500, loss: 3.3513535446742218 iter: 2550, loss: 3.4554441847271393 iter: 2600, loss: 2.9394915195343994 iter: 2650, loss: 3.370902210848673 iter: 2700, loss: 3.4259227318839423 iter: 2750, loss: 3.4058353806904393 iter: 2800, loss: 3.467306881359647 iter: 2850, loss: 3.222254538074372 iter: 2900, loss: 3.3392559226808087 iter: 2950, loss: 3.4203980594362533 iter: 3000, loss: 3.3507530433563955 iter: 3050, loss: 3.4326547555317966 iter: 3100, loss: 3.1755515496390205 iter: 3150, loss: 3.3925877854634847 iter: 3200, loss: 3.223531436912598 iter: 3250, loss: 3.3089625614862603 iter: 3300, loss: 3.367763715501815 iter: 3350, loss: 3.4278301871163497 iter: 3400, loss: 3.373292277381534 iter: 3450, loss: 3.3497054475829717 iter: 3500, loss: 3.402910869681646 iter: 3550, loss: 3.072571641732776 iter: 3600, loss: 3.2611226563832116 iter: 3650, loss: 3.231520605495998 iter: 3700, loss: 3.3788801974569043 iter: 3750, loss: 3.176644308181036 iter: 3800, loss: 3.2255533708693496 iter: 3850, loss: 3.2362594686387083 iter: 3900, loss: 3.095807164230044 iter: 3950, loss: 3.2343999077024916 iter: 4000, loss: 3.3681417366512245 iter: 4050, loss: 3.0732023419879737 iter: 4100, loss: 3.0663742440617283 iter: 4150, loss: 3.396770855048347 iter: 4200, loss: 3.4262332421522292 iter: 4250, loss: 3.060121847773354 iter: 4300, loss: 2.895130627753243 iter: 4350, loss: 3.017712699065133 iter: 4400, loss: 3.1289404028559487 iter: 4450, loss: 3.163725920904249 iter: 4500, loss: 3.3627441662606743 iter: 4550, loss: 3.409984823173947 iter: 4600, loss: 2.8944704760899618 iter: 4650, loss: 3.0016444209568083 iter: 4700, loss: 2.8574393688837683 iter: 4750, loss: 3.1946328716656525 iter: 4800, loss: 2.768447057353125 iter: 4850, loss: 3.075327144675784 iter: 4900, loss: 3.268370175997416 iter: 4950, loss: 3.1798231331053235 iter: 5000, loss: 3.3217560536218063 iter: 5050, loss: 3.006732604223585 iter: 5100, loss: 3.3575944598061698 iter: 5150, loss: 2.9057663469655175 iter: 5200, loss: 2.8928466574502374 iter: 5250, loss: 3.061066797528948 iter: 5300, loss: 3.35562970057745 iter: 5350, loss: 2.9118076042901895 iter: 5400, loss: 2.9514354321918783 iter: 5450, loss: 2.9334804391406832 iter: 5500, loss: 3.204634138440329 iter: 5550, loss: 2.8140748963961526 iter: 5600, loss: 3.011708143741365 iter: 5650, loss: 3.323859388586074 iter: 5700, loss: 2.8442912295810756 iter: 5750, loss: 2.80684267281729 iter: 5800, loss: 3.1174840584860903 iter: 5850, loss: 2.6991389470478837 iter: 5900, loss: 2.9698236653237116 iter: 5950, loss: 3.0238281039586137 iter: 6000, loss: 2.8812837354947645 iter: 6050, loss: 3.1709352504639394 iter: 6100, loss: 2.937920509209709 iter: 6150, loss: 3.178728113076043 iter: 6200, loss: 2.8974244089429337 iter: 6250, loss: 2.809626478180052 iter: 6300, loss: 2.781241159703996 iter: 6350, loss: 2.9004218400395105 iter: 6400, loss: 2.9118271145669246 iter: 6450, loss: 2.8842602037096787 iter: 6500, loss: 2.9489114957536966 iter: 6550, loss: 2.9503131193130736 iter: 6600, loss: 2.8961831474304187 iter: 6650, loss: 3.002027267266834 iter: 6700, loss: 3.0047303264103236 iter: 6750, loss: 2.958453589060949 iter: 6800, loss: 2.9524990789852446 iter: 6850, loss: 2.935619188210321 iter: 6900, loss: 2.9734530233807033 iter: 6950, loss: 2.785320390822396 iter: 7000, loss: 3.1911680922054106 iter: 7050, loss: 2.7732513120363635 iter: 7100, loss: 2.7432456348282948 iter: 7150, loss: 2.823985375283256 iter: 7200, loss: 2.927504679808541 iter: 7250, loss: 3.0693400076760184 iter: 7300, loss: 2.666468213043515 iter: 7350, loss: 2.808132514378382 iter: 7400, loss: 2.558679431067573 iter: 7450, loss: 2.6974468813850763 iter: 7500, loss: 2.8497490201223457 iter: 7550, loss: 2.7490190564337236 iter: 7600, loss: 2.8300208840067427 iter: 7650, loss: 2.793417969741518
[0;31m---------------------------------------------------------------------------[0m [0;31mKeyboardInterrupt[0m Traceback (most recent call last) Input [0;32mIn [19][0m, in [0;36m<cell line: 5>[0;34m()[0m [1;32m 2[0m encoder1 [38;5;241m=[39m EncoderRNN(eng_lang[38;5;241m.[39mn_words, hidden_size)[38;5;241m.[39mto(device) [1;32m 3[0m attn_decoder1 [38;5;241m=[39m AttnDecoderRNN(hidden_size, fra_lang[38;5;241m.[39mn_words, dropout_p[38;5;241m=[39m[38;5;241m0.1[39m)[38;5;241m.[39mto(device) [0;32m----> 5[0m [43mtrainIters[49m[43m([49m[43mencoder1[49m[43m,[49m[43m [49m[43mattn_decoder1[49m[43m,[49m[43m [49m[38;5;241;43m75000[39;49m[43m,[49m[43m [49m[43mprint_every[49m[38;5;241;43m=[39;49m[38;5;241;43m50[39;49m[43m)[49m Input [0;32mIn [16][0m, in [0;36mtrainIters[0;34m(encoder, decoder, n_iters, print_every, plot_every, learning_rate)[0m [1;32m 16[0m input_tensor [38;5;241m=[39m training_pair[[38;5;241m0[39m] [1;32m 17[0m target_tensor [38;5;241m=[39m training_pair[[38;5;241m1[39m] [0;32m---> 19[0m loss [38;5;241m=[39m [43mtrain[49m[43m([49m[43minput_tensor[49m[43m,[49m[43m [49m[43mtarget_tensor[49m[43m,[49m[43m [49m[43mencoder[49m[43m,[49m [1;32m 20[0m [43m [49m[43mdecoder[49m[43m,[49m[43m [49m[43mencoder_optimizer[49m[43m,[49m[43m [49m[43mdecoder_optimizer[49m[43m,[49m[43m [49m[43mcriterion[49m[43m)[49m [1;32m 21[0m print_loss_total [38;5;241m+[39m[38;5;241m=[39m loss [1;32m 22[0m plot_loss_total [38;5;241m+[39m[38;5;241m=[39m loss Input [0;32mIn [15][0m, in [0;36mtrain[0;34m(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, max_length)[0m [1;32m 45[0m [38;5;28;01mif[39;00m decoder_input[38;5;241m.[39mitem() [38;5;241m==[39m EOS_token: [1;32m 46[0m [38;5;28;01mbreak[39;00m [0;32m---> 48[0m [43mloss[49m[38;5;241;43m.[39;49m[43mbackward[49m[43m([49m[43m)[49m [1;32m 50[0m encoder_optimizer[38;5;241m.[39mstep() [1;32m 51[0m decoder_optimizer[38;5;241m.[39mstep() File [0;32m~/anaconda3/envs/zajeciaei/lib/python3.10/site-packages/torch/_tensor.py:363[0m, in [0;36mTensor.backward[0;34m(self, gradient, retain_graph, create_graph, inputs)[0m [1;32m 354[0m [38;5;28;01mif[39;00m has_torch_function_unary([38;5;28mself[39m): [1;32m 355[0m [38;5;28;01mreturn[39;00m handle_torch_function( [1;32m 356[0m Tensor[38;5;241m.[39mbackward, [1;32m 357[0m ([38;5;28mself[39m,), [0;32m (...)[0m [1;32m 361[0m create_graph[38;5;241m=[39mcreate_graph, [1;32m 362[0m inputs[38;5;241m=[39minputs) [0;32m--> 363[0m [43mtorch[49m[38;5;241;43m.[39;49m[43mautograd[49m[38;5;241;43m.[39;49m[43mbackward[49m[43m([49m[38;5;28;43mself[39;49m[43m,[49m[43m [49m[43mgradient[49m[43m,[49m[43m [49m[43mretain_graph[49m[43m,[49m[43m [49m[43mcreate_graph[49m[43m,[49m[43m [49m[43minputs[49m[38;5;241;43m=[39;49m[43minputs[49m[43m)[49m File [0;32m~/anaconda3/envs/zajeciaei/lib/python3.10/site-packages/torch/autograd/__init__.py:173[0m, in [0;36mbackward[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)[0m [1;32m 168[0m retain_graph [38;5;241m=[39m create_graph [1;32m 170[0m [38;5;66;03m# The reason we repeat same the comment below is that[39;00m [1;32m 171[0m [38;5;66;03m# some Python versions print out the first line of a multi-line function[39;00m [1;32m 172[0m [38;5;66;03m# calls in the traceback and some print out the last line[39;00m [0;32m--> 173[0m [43mVariable[49m[38;5;241;43m.[39;49m[43m_execution_engine[49m[38;5;241;43m.[39;49m[43mrun_backward[49m[43m([49m[43m [49m[38;5;66;43;03m# Calls into the C++ engine to run the backward pass[39;49;00m [1;32m 174[0m [43m [49m[43mtensors[49m[43m,[49m[43m [49m[43mgrad_tensors_[49m[43m,[49m[43m [49m[43mretain_graph[49m[43m,[49m[43m [49m[43mcreate_graph[49m[43m,[49m[43m [49m[43minputs[49m[43m,[49m [1;32m 175[0m [43m [49m[43mallow_unreachable[49m[38;5;241;43m=[39;49m[38;5;28;43;01mTrue[39;49;00m[43m,[49m[43m [49m[43maccumulate_grad[49m[38;5;241;43m=[39;49m[38;5;28;43;01mTrue[39;49;00m[43m)[49m [0;31mKeyboardInterrupt[0m:
evaluateRandomly(encoder1, attn_decoder1)
> you re sad . = tu es triste . < vous tes . . <EOS> > she is sewing a dress . = elle coud une robe . < elle est une une . . <EOS> > he is suffering from a headache . = il souffre d un mal de t te . < il est un un un un . <EOS> > i m glad to see you . = je suis heureux de vous voir . < je suis content de vous voir . <EOS> > you are only young once . = on n est jeune qu une fois . < vous tes trop plus une enfant . <EOS> > you re so sweet . = vous tes si gentille ! < vous tes trop si . <EOS> > i m running out of closet space . = je manque d espace dans mon placard . < je suis un de de <EOS> > i m sort of an extrovert . = je suis en quelque sorte extraverti . < je suis un un . . <EOS> > i m out of practice . = je manque de pratique . < j ai ai pas de <EOS> > you re the last hope for humanity . = tu es le dernier espoir de l humanit . < vous tes le la la . . <EOS>