119 KiB
119 KiB
Seq2Seq Fiński --> Angielski
https://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html
from __future__ import unicode_literals, print_function, division
from io import open
import unicodedata
import re
import random
import torch
import torch.nn as nn
from torch import optim
import torch.nn.functional as F
import numpy as np
from torch.utils.data import TensorDataset, DataLoader, RandomSampler
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")torch.cuda.device_count()2
Konwersja słów na index
SOS_token = 0
EOS_token = 1
class Lang:
def __init__(self, name):
self.name = name
self.word2index = {}
self.word2count = {}
self.index2word = {0: "SOS", 1: "EOS"}
self.n_words = 2 # Count SOS and EOS
def addSentence(self, sentence):
for word in sentence.split(' '):
self.addWord(word)
def addWord(self, word):
if word not in self.word2index:
self.word2index[word] = self.n_words
self.word2count[word] = 1
self.index2word[self.n_words] = word
self.n_words += 1
else:
self.word2count[word] += 1Normalizacja tekstu
# Turn a Unicode string to plain ASCII, thanks to
# https://stackoverflow.com/a/518232/2809427
def unicodeToAscii(s):
return ''.join(
c for c in unicodedata.normalize('NFD', s)
if unicodedata.category(c) != 'Mn'
)
# Lowercase, trim, and remove non-letter characters
def normalizeString(s):
s = unicodeToAscii(s.lower().strip())
s = re.sub(r"([.!?])", r" \1", s)
s = re.sub(r"[^a-zA-Z!?]+", r" ", s)
return s.strip()Wczytywanie danych (zmodyfikowane ze względu na ścieżkę w kaggle)
# Zmodyfikowana wersja ze względu na użycie pojedynczego pliku przesłanego na Kaggle
def readLangs(reverse=False):
print("Reading lines...")
lang1="en"
lang2="fin"
# Read the file and split into lines
lines = open('/kaggle/input/anki-en-fin/fin.txt', encoding='utf-8').\
read().strip().split('\n')
# Split every line into pairs and normalize
pairs = [[normalizeString(s) for s in l.split('\t')[:-1]] for l in lines] # +Usuwanie licencji CC z linii
# Reverse pairs, make Lang instances
if reverse:
pairs = [list(reversed(p)) for p in pairs]
input_lang = Lang(lang2)
output_lang = Lang(lang1)
else:
input_lang = Lang(lang1)
output_lang = Lang(lang2)
return input_lang, output_lang, pairsOgraniczenie do zdań max 10 słów, formy I am / You are / He is etc. bez interpunkcji
MAX_LENGTH = 10
eng_prefixes = (
"i am ", "i m ",
"he is", "he s ",
"she is", "she s ",
"you are", "you re ",
"we are", "we re ",
"they are", "they re "
)
def filterPair(p):
return len(p[0].split(' ')) < MAX_LENGTH and \
len(p[1].split(' ')) < MAX_LENGTH and \
p[1].startswith(eng_prefixes)
def filterPairs(pairs):
return [pair for pair in pairs if filterPair(pair)]def prepareData(reverse=False):
input_lang, output_lang, pairs = readLangs(reverse)
print("Read %s sentence pairs" % len(pairs))
pairs = filterPairs(pairs)
print("Trimmed to %s sentence pairs" % len(pairs))
print("Counting words...")
for pair in pairs:
input_lang.addSentence(pair[0])
output_lang.addSentence(pair[1])
print("Counted words:")
print(input_lang.name, input_lang.n_words)
print(output_lang.name, output_lang.n_words)
return input_lang, output_lang, pairs
input_lang, output_lang, pairs = prepareData(True)
print(random.choice(pairs))Reading lines... Read 72258 sentence pairs Trimmed to 5005 sentence pairs Counting words... Counted words: fin 3686 en 1971 ['mina odotan joulua innolla', 'i am looking forward to christmas']
Definicja modelu
class EncoderRNN(nn.Module):
def __init__(self, input_size, hidden_size, dropout_p=0.1):
super(EncoderRNN, self).__init__()
self.hidden_size = hidden_size
self.embedding = nn.Embedding(input_size, hidden_size)
self.gru = nn.GRU(hidden_size, hidden_size, batch_first=True)
self.dropout = nn.Dropout(dropout_p)
def forward(self, input):
embedded = self.dropout(self.embedding(input))
output, hidden = self.gru(embedded)
return output, hiddenclass DecoderRNN(nn.Module):
def __init__(self, hidden_size, output_size):
super(DecoderRNN, self).__init__()
self.embedding = nn.Embedding(output_size, hidden_size)
self.gru = nn.GRU(hidden_size, hidden_size, batch_first=True)
self.out = nn.Linear(hidden_size, output_size)
def forward(self, encoder_outputs, encoder_hidden, target_tensor=None):
batch_size = encoder_outputs.size(0)
decoder_input = torch.empty(batch_size, 1, dtype=torch.long, device=device).fill_(SOS_token)
decoder_hidden = encoder_hidden
decoder_outputs = []
for i in range(MAX_LENGTH):
decoder_output, decoder_hidden = self.forward_step(decoder_input, decoder_hidden)
decoder_outputs.append(decoder_output)
if target_tensor is not None:
# Teacher forcing: Feed the target as the next input
decoder_input = target_tensor[:, i].unsqueeze(1) # Teacher forcing
else:
# Without teacher forcing: use its own predictions as the next input
_, topi = decoder_output.topk(1)
decoder_input = topi.squeeze(-1).detach() # detach from history as input
decoder_outputs = torch.cat(decoder_outputs, dim=1)
decoder_outputs = F.log_softmax(decoder_outputs, dim=-1)
return decoder_outputs, decoder_hidden, None # We return `None` for consistency in the training loop
def forward_step(self, input, hidden):
output = self.embedding(input)
output = F.relu(output)
output, hidden = self.gru(output, hidden)
output = self.out(output)
return output, hiddenclass BahdanauAttention(nn.Module):
def __init__(self, hidden_size):
super(BahdanauAttention, self).__init__()
self.Wa = nn.Linear(hidden_size, hidden_size)
self.Ua = nn.Linear(hidden_size, hidden_size)
self.Va = nn.Linear(hidden_size, 1)
def forward(self, query, keys):
scores = self.Va(torch.tanh(self.Wa(query) + self.Ua(keys)))
scores = scores.squeeze(2).unsqueeze(1)
weights = F.softmax(scores, dim=-1)
context = torch.bmm(weights, keys)
return context, weights
class AttnDecoderRNN(nn.Module):
def __init__(self, hidden_size, output_size, dropout_p=0.1):
super(AttnDecoderRNN, self).__init__()
self.embedding = nn.Embedding(output_size, hidden_size)
self.attention = BahdanauAttention(hidden_size)
self.gru = nn.GRU(2 * hidden_size, hidden_size, batch_first=True)
self.out = nn.Linear(hidden_size, output_size)
self.dropout = nn.Dropout(dropout_p)
def forward(self, encoder_outputs, encoder_hidden, target_tensor=None):
batch_size = encoder_outputs.size(0)
decoder_input = torch.empty(batch_size, 1, dtype=torch.long, device=device).fill_(SOS_token)
decoder_hidden = encoder_hidden
decoder_outputs = []
attentions = []
for i in range(MAX_LENGTH):
decoder_output, decoder_hidden, attn_weights = self.forward_step(
decoder_input, decoder_hidden, encoder_outputs
)
decoder_outputs.append(decoder_output)
attentions.append(attn_weights)
if target_tensor is not None:
# Teacher forcing: Feed the target as the next input
decoder_input = target_tensor[:, i].unsqueeze(1) # Teacher forcing
else:
# Without teacher forcing: use its own predictions as the next input
_, topi = decoder_output.topk(1)
decoder_input = topi.squeeze(-1).detach() # detach from history as input
decoder_outputs = torch.cat(decoder_outputs, dim=1)
decoder_outputs = F.log_softmax(decoder_outputs, dim=-1)
attentions = torch.cat(attentions, dim=1)
return decoder_outputs, decoder_hidden, attentions
def forward_step(self, input, hidden, encoder_outputs):
embedded = self.dropout(self.embedding(input))
query = hidden.permute(1, 0, 2)
context, attn_weights = self.attention(query, encoder_outputs)
input_gru = torch.cat((embedded, context), dim=2)
output, hidden = self.gru(input_gru, hidden)
output = self.out(output)
return output, hidden, attn_weightsdef indexesFromSentence(lang, sentence):
return [lang.word2index[word] for word in sentence.split(' ')]
def tensorFromSentence(lang, sentence):
indexes = indexesFromSentence(lang, sentence)
indexes.append(EOS_token)
return torch.tensor(indexes, dtype=torch.long, device=device).view(1, -1)
def tensorsFromPair(pair):
input_tensor = tensorFromSentence(input_lang, pair[0])
target_tensor = tensorFromSentence(output_lang, pair[1])
return (input_tensor, target_tensor)
def get_dataloader(batch_size):
input_lang, output_lang, pairs = prepareData(True)
n = len(pairs)
input_ids = np.zeros((n, MAX_LENGTH), dtype=np.int32)
target_ids = np.zeros((n, MAX_LENGTH), dtype=np.int32)
for idx, (inp, tgt) in enumerate(pairs):
inp_ids = indexesFromSentence(input_lang, inp)
tgt_ids = indexesFromSentence(output_lang, tgt)
inp_ids.append(EOS_token)
tgt_ids.append(EOS_token)
input_ids[idx, :len(inp_ids)] = inp_ids
target_ids[idx, :len(tgt_ids)] = tgt_ids
train_data = TensorDataset(torch.LongTensor(input_ids).to(device),
torch.LongTensor(target_ids).to(device))
train_sampler = RandomSampler(train_data)
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
return input_lang, output_lang, train_dataloaderdef train_epoch(dataloader, encoder, decoder, encoder_optimizer,
decoder_optimizer, criterion):
total_loss = 0
for data in dataloader:
input_tensor, target_tensor = data
encoder_optimizer.zero_grad()
decoder_optimizer.zero_grad()
encoder_outputs, encoder_hidden = encoder(input_tensor)
decoder_outputs, _, _ = decoder(encoder_outputs, encoder_hidden, target_tensor)
loss = criterion(
decoder_outputs.view(-1, decoder_outputs.size(-1)),
target_tensor.view(-1)
)
loss.backward()
encoder_optimizer.step()
decoder_optimizer.step()
total_loss += loss.item()
return total_loss / len(dataloader)import time
import math
def asMinutes(s):
m = math.floor(s / 60)
s -= m * 60
return '%dm %ds' % (m, s)
def timeSince(since, percent):
now = time.time()
s = now - since
es = s / (percent)
rs = es - s
return '%s (- %s)' % (asMinutes(s), asMinutes(rs))def train(train_dataloader, encoder, decoder, n_epochs, learning_rate=0.001,
print_every=100, plot_every=100):
start = time.time()
plot_losses = []
print_loss_total = 0 # Reset every print_every
plot_loss_total = 0 # Reset every plot_every
encoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate)
decoder_optimizer = optim.Adam(decoder.parameters(), lr=learning_rate)
criterion = nn.NLLLoss()
for epoch in range(1, n_epochs + 1):
loss = train_epoch(train_dataloader, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion)
print_loss_total += loss
plot_loss_total += loss
if epoch % print_every == 0:
print_loss_avg = print_loss_total / print_every
print_loss_total = 0
print('%s (%d %d%%) %.4f' % (timeSince(start, epoch / n_epochs),
epoch, epoch / n_epochs * 100, print_loss_avg))
if epoch % plot_every == 0:
plot_loss_avg = plot_loss_total / plot_every
plot_losses.append(plot_loss_avg)
plot_loss_total = 0
showPlot(plot_losses)import matplotlib.pyplot as plt
plt.switch_backend('agg')
import matplotlib.ticker as ticker
import numpy as np
%matplotlib inline
def showPlot(points):
plt.figure()
fig, ax = plt.subplots()
# this locator puts ticks at regular intervals
loc = ticker.MultipleLocator(base=0.2)
ax.yaxis.set_major_locator(loc)
plt.plot(points)Ewaluacja
def evaluate(encoder, decoder, sentence, input_lang, output_lang):
with torch.no_grad():
input_tensor = tensorFromSentence(input_lang, sentence)
encoder_outputs, encoder_hidden = encoder(input_tensor)
decoder_outputs, decoder_hidden, decoder_attn = decoder(encoder_outputs, encoder_hidden)
_, topi = decoder_outputs.topk(1)
decoded_ids = topi.squeeze()
decoded_words = []
for idx in decoded_ids:
if idx.item() == EOS_token:
decoded_words.append('<EOS>')
break
decoded_words.append(output_lang.index2word[idx.item()])
return decoded_words, decoder_attndef evaluateRandomly(encoder, decoder, n=10):
for i in range(n):
pair = random.choice(pairs)
print('Input sentence: ', pair[0])
print('Target (true) translation:' , pair[1])
output_words, _ = evaluate(encoder, decoder, pair[0], input_lang, output_lang)
output_sentence = ' '.join(output_words)
print('Output sentence: ', output_sentence)
print('')Wykorzystanie zdefiniowanych wyżej funkcji
Trenowanie modelu
hidden_size = 128
batch_size = 32
input_lang, output_lang, train_dataloader = get_dataloader(batch_size)
encoder = EncoderRNN(input_lang.n_words, hidden_size).to(device)
decoder = AttnDecoderRNN(hidden_size, output_lang.n_words).to(device)
train(train_dataloader, encoder, decoder, 80, print_every=5, plot_every=5)Reading lines... Read 72258 sentence pairs Trimmed to 5005 sentence pairs Counting words... Counted words: fin 3686 en 1971 0m 21s (- 5m 21s) (5 6%) 1.9364 0m 36s (- 4m 17s) (10 12%) 1.0355 0m 51s (- 3m 45s) (15 18%) 0.6313 1m 7s (- 3m 21s) (20 25%) 0.3787 1m 22s (- 3m 1s) (25 31%) 0.2243 1m 37s (- 2m 42s) (30 37%) 0.1371 1m 52s (- 2m 25s) (35 43%) 0.0903 2m 7s (- 2m 7s) (40 50%) 0.0668 2m 23s (- 1m 51s) (45 56%) 0.0538 2m 38s (- 1m 34s) (50 62%) 0.0471 2m 53s (- 1m 18s) (55 68%) 0.0410 3m 8s (- 1m 2s) (60 75%) 0.0381 3m 23s (- 0m 47s) (65 81%) 0.0343 3m 38s (- 0m 31s) (70 87%) 0.0342 3m 54s (- 0m 15s) (75 93%) 0.0322 4m 9s (- 0m 0s) (80 100%) 0.0307
evaluateRandomly(encoder, decoder)Input sentence: olen hyvin hyvin vihainen Target (true) translation: i m very very angry Output sentence: i am very angry today <EOS> Input sentence: han valehtelee Target (true) translation: he s lying Output sentence: he is telling a lie <EOS> Input sentence: olen myohassa Target (true) translation: i m late Output sentence: i m late <EOS> Input sentence: han on linja autonkuljettaja Target (true) translation: he is a bus driver Output sentence: he is a bus driver <EOS> Input sentence: mukava tavata sinut taas Target (true) translation: i m glad to see you again Output sentence: i m glad to see you again <EOS> Input sentence: olet kuumeessa Target (true) translation: you re running a fever Output sentence: you re so predictable <EOS> Input sentence: anteeksi mutta unohdin tehda laksyt Target (true) translation: i m sorry i forgot to do my homework Output sentence: i m sorry i forgot to do my homework <EOS> Input sentence: mina olen tyoton Target (true) translation: i m unemployed Output sentence: i m unemployed <EOS> Input sentence: olen taynna Target (true) translation: i am full Output sentence: i am full of french <EOS> Input sentence: ma kuolen nalkaan ! Target (true) translation: i m dying of hunger Output sentence: i m dying of hunger <EOS>
def showAttention(input_sentence, output_words, attentions):
fig = plt.figure()
ax = fig.add_subplot(111)
cax = ax.matshow(attentions.cpu().numpy(), cmap='bone')
fig.colorbar(cax)
# Set up axes
ax.set_xticklabels([''] + input_sentence.split(' ') +
['<EOS>'], rotation=90)
ax.set_yticklabels([''] + output_words)
# Show label at every tick
ax.xaxis.set_major_locator(ticker.MultipleLocator(1))
ax.yaxis.set_major_locator(ticker.MultipleLocator(1))
plt.show()
def evaluateAndShowAttention(input_sentence):
input_sentence = normalizeString(input_sentence)
output_words, attentions = evaluate(encoder, decoder, input_sentence, input_lang, output_lang)
print('input =', input_sentence)
print('output =', ' '.join(output_words))
showAttention(input_sentence, output_words, attentions[0, :len(output_words), :])
def translate(input_sentence, tokenized=False):
input_sentence = normalizeString(input_sentence)
output_words, attentions = evaluate(encoder, decoder, input_sentence, input_lang, output_lang)
if tokenized:
if "<EOS>" in output_words:
output_words.remove("<EOS>")
return output_words
return ' '.join(output_words)translate("Meillä on nälkä", tokenized=True)['we', 'are', 'hungry']
evaluateAndShowAttention('Olet liian naivi')input = olet liian naivi output = you re too naive <EOS>
/tmp/ipykernel_34/2052950992.py:8: UserWarning: FixedFormatter should only be used together with FixedLocator
ax.set_xticklabels([''] + input_sentence.split(' ') +
/tmp/ipykernel_34/2052950992.py:10: UserWarning: FixedFormatter should only be used together with FixedLocator
ax.set_yticklabels([''] + output_words)
evaluateAndShowAttention('Olen todella pahoillani')input = olen todella pahoillani output = i am truly sorry <EOS>
/tmp/ipykernel_34/2052950992.py:8: UserWarning: FixedFormatter should only be used together with FixedLocator
ax.set_xticklabels([''] + input_sentence.split(' ') +
/tmp/ipykernel_34/2052950992.py:10: UserWarning: FixedFormatter should only be used together with FixedLocator
ax.set_yticklabels([''] + output_words)
evaluateAndShowAttention('Olet minun isäni')input = olet minun isani output = you re my father <EOS>
/tmp/ipykernel_34/2052950992.py:8: UserWarning: FixedFormatter should only be used together with FixedLocator
ax.set_xticklabels([''] + input_sentence.split(' ') +
/tmp/ipykernel_34/2052950992.py:10: UserWarning: FixedFormatter should only be used together with FixedLocator
ax.set_yticklabels([''] + output_words)
evaluateAndShowAttention('Hän on opettaja')input = han on opettaja output = he is a teacher <EOS>
/tmp/ipykernel_34/2052950992.py:8: UserWarning: FixedFormatter should only be used together with FixedLocator
ax.set_xticklabels([''] + input_sentence.split(' ') +
/tmp/ipykernel_34/2052950992.py:10: UserWarning: FixedFormatter should only be used together with FixedLocator
ax.set_yticklabels([''] + output_words)
torch.save(encoder.state_dict(), "encoder.pt")
torch.save(decoder.state_dict(), "decoder.pt")BLEU score
Jako że korzystaliśmy z okrojonej wersji zbioru danych, słownik nie zawiera wszystkich słów pojawiających się w przykładach więc do ewaluacji wykorzystujemy część przykładów z treningu
import pandas as pd
def filter_rows(row):
return len(row["English"].split(' '))<MAX_LENGTH and \
len(row["Finnish"].split(' '))<MAX_LENGTH and \
row["English"].startswith(eng_prefixes)
data_file = pd.read_csv("/kaggle/input/anki-en-fin/fin.txt", sep='\t', names=["English","Finnish","attribution"])
data_file["English"] = data_file["English"].apply(normalizeString)
data_file["Finnish"] = data_file["Finnish"].apply(normalizeString)
filter_list = data_file.apply(filter_rows, axis=1)
test_section = data_file[filter_list]
test_section = test_section.sample(frac=1).head(500)
test_section.head()| English | Finnish | attribution | |
|---|---|---|---|
| 38027 | i m very serious about this | olen hyvin tosissani tasta | CC-BY 2.0 (France) Attribution: tatoeba.org #2... |
| 3803 | i m not tired | en ole vasynyt | CC-BY 2.0 (France) Attribution: tatoeba.org #1... |
| 26924 | i m not married either | minakaan en ole naimisissa | CC-BY 2.0 (France) Attribution: tatoeba.org #6... |
| 32009 | he s sleeping like a baby | han nukkuu kuin pikkuvauva | CC-BY 2.0 (France) Attribution: tatoeba.org #2... |
| 21339 | i m joking of course | se oli vitsi tietenkin | CC-BY 2.0 (France) Attribution: tatoeba.org #2... |
test_section["English_tokenized"] = test_section["English"].apply(lambda x: x.split())test_section.head()["English_tokenized"]38027 [i, m, very, serious, about, this] 3803 [i, m, not, tired] 26924 [i, m, not, married, either] 32009 [he, s, sleeping, like, a, baby] 21339 [i, m, joking, of, course] Name: English_tokenized, dtype: object
test_section["English_translated"] = test_section["Finnish"].apply(lambda x: translate(x, tokenized=True))test_section.head()| English | Finnish | attribution | English_tokenized | English_translated | |
|---|---|---|---|---|---|
| 38027 | i m very serious about this | olen hyvin tosissani tasta | CC-BY 2.0 (France) Attribution: tatoeba.org #2... | [i, m, very, serious, about, this] | [i, m, in, french] |
| 3803 | i m not tired | en ole vasynyt | CC-BY 2.0 (France) Attribution: tatoeba.org #1... | [i, m, not, tired] | [i, not, tired] |
| 26924 | i m not married either | minakaan en ole naimisissa | CC-BY 2.0 (France) Attribution: tatoeba.org #6... | [i, m, not, married, either] | [i, m, not, married, either] |
| 32009 | he s sleeping like a baby | han nukkuu kuin pikkuvauva | CC-BY 2.0 (France) Attribution: tatoeba.org #2... | [he, s, sleeping, like, a, baby] | [he, is, as, a, pianist] |
| 21339 | i m joking of course | se oli vitsi tietenkin | CC-BY 2.0 (France) Attribution: tatoeba.org #2... | [i, m, joking, of, course] | [i, m, joking, of, course] |
candidate_corpus = test_section["English_translated"].values
references_corpus = test_section["English_tokenized"].values.tolist()
x = candidate_corpus.tolist()
y = [[el] for el in references_corpus]
#print(references_corpus[:5])
#print(candidate_corpus[:5])y[:5][[['i', 'm', 'very', 'serious', 'about', 'this']], [['i', 'm', 'not', 'tired']], [['i', 'm', 'not', 'married', 'either']], [['he', 's', 'sleeping', 'like', 'a', 'baby']], [['i', 'm', 'joking', 'of', 'course']]]
from torchtext.data.metrics import bleu_score
bleu_score(x, y)0.5885258316993713