242 lines
9.9 KiB
Python
242 lines
9.9 KiB
Python
# https://arxiv.org/pdf/2001.11692.pdf
|
||
|
||
|
||
import numpy as np
|
||
import unicodedata
|
||
from time import sleep
|
||
|
||
import torch
|
||
import torch.nn as nn
|
||
import torch.nn.functional as F
|
||
import torch.optim as optim
|
||
from sacred.observers import FileStorageObserver, MongoObserver
|
||
from torch.utils.data import Dataset, DataLoader
|
||
import re
|
||
import random
|
||
import os
|
||
import sys
|
||
from tqdm import tqdm
|
||
from Levenshtein import distance as levenshtein_distance
|
||
from sacred import Experiment
|
||
import traceback
|
||
from mlflow import log_metric, log_param, log_artifacts
|
||
import mlflow
|
||
|
||
import logging
|
||
|
||
logging.basicConfig(level=logging.WARN)
|
||
logger = logging.getLogger(__name__)
|
||
|
||
mlflow.set_tracking_uri("http://172.17.0.1:5000")
|
||
mlflow.set_experiment("s434749")
|
||
|
||
ex = Experiment("CNN")
|
||
ex.observers.append(FileStorageObserver('sacred_file_observer'))
|
||
try:
|
||
ex.observers.append(MongoObserver(url='mongodb://mongo_user:mongo_password_IUM_2021@172.17.0.1:27017',
|
||
db_name='sacred'))
|
||
except Exception as e:
|
||
traceback.print_exc()
|
||
|
||
device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu')
|
||
|
||
|
||
class CNN(nn.Module):
|
||
def __init__(self, kernel_size, hidden_layers, channels, embedding_size, in_alphabet, max_len):
|
||
super(CNN, self).__init__()
|
||
self.input_conv = nn.Conv1d(in_channels=len(in_alphabet), out_channels=channels, kernel_size=kernel_size)
|
||
self.conv_hidden = nn.ModuleList(
|
||
[nn.Conv1d(in_channels=channels, out_channels=channels, kernel_size=kernel_size) for _ in
|
||
range(hidden_layers)])
|
||
self.last_layer_size = (max_len - (kernel_size - 1) * (hidden_layers + 1)) * channels
|
||
self.lin = nn.Linear(self.last_layer_size, embedding_size)
|
||
|
||
def forward(self, x):
|
||
x = self.input_conv(x)
|
||
x = F.relu(x, inplace=True)
|
||
for c in self.conv_hidden:
|
||
x = c(x)
|
||
x = F.relu(x, inplace=True)
|
||
x = x.view(x.size()[0], self.last_layer_size)
|
||
x = self.lin(x)
|
||
return x
|
||
|
||
|
||
def dist(a: [str], b: [str]):
|
||
return torch.tensor([levenshtein_distance(a[i], b[i]) for i in range(len(a))], dtype=torch.float, device=device)
|
||
|
||
|
||
def encode(batch: [(torch.tensor, str)], in_alphabet, max_len):
|
||
batch_text = torch.zeros((len(batch), len(in_alphabet), max_len))
|
||
batch_phonemes = list(map(lambda x: x[1], batch))
|
||
for i, (sample, _) in enumerate(batch):
|
||
for chr_pos, index in enumerate(sample):
|
||
batch_text[i, index, chr_pos] = 1
|
||
return batch_text, batch_phonemes
|
||
|
||
|
||
def encode_str(batch: [(str, str)], in_alphabet, max_len):
|
||
batch = [(torch.tensor([in_alphabet[letter] for letter in in_str], dtype=torch.int), out_str) for in_str, out_str in
|
||
batch]
|
||
return encode(batch, in_alphabet, max_len)
|
||
|
||
|
||
def train_model(model, learning_rate, in_alphabet, max_len, data, epochs, batch_size):
|
||
optimizer = optim.Adam(filter(lambda x: x.requires_grad, model.parameters()),
|
||
lr=learning_rate)
|
||
outer_bar = tqdm(total=epochs, position=0)
|
||
inner_bar = tqdm(total=len(data), position=1)
|
||
outer_bar.reset()
|
||
outer_bar.set_description("Epochs")
|
||
|
||
def collate(batch: [(torch.tensor, str)]):
|
||
return encode(batch, in_alphabet, max_len)
|
||
|
||
data_loader = DataLoader(dataset=data, drop_last=True,
|
||
batch_size=3 * batch_size,
|
||
collate_fn=collate,
|
||
shuffle=True)
|
||
for epoch in range(epochs):
|
||
total_loss = 0
|
||
inner_bar.reset()
|
||
|
||
for batch_text, batch_phonemes in data_loader:
|
||
optimizer.zero_grad()
|
||
anchor, positive, negative = batch_text.to(device).split(batch_size)
|
||
ph_anchor = batch_phonemes[:batch_size]
|
||
ph_positive = batch_phonemes[batch_size:2 * batch_size]
|
||
ph_negative = batch_phonemes[2 * batch_size:]
|
||
embedded_anchor = model(anchor)
|
||
embedded_positive = model(positive)
|
||
embedded_negative = model(negative)
|
||
estimated_pos_dist = torch.linalg.norm(embedded_anchor - embedded_positive, dim=1)
|
||
estimated_neg_dist = torch.linalg.norm(embedded_anchor - embedded_negative, dim=1)
|
||
estimated_pos_neg_dist = torch.linalg.norm(embedded_positive - embedded_negative, dim=1)
|
||
actual_pos_dist = dist(ph_anchor, ph_positive)
|
||
actual_neg_dist = dist(ph_anchor, ph_negative)
|
||
actual_pos_neg_dist = dist(ph_positive, ph_negative)
|
||
loss = sum(abs(estimated_neg_dist - actual_neg_dist)
|
||
+ abs(estimated_pos_dist - actual_pos_dist)
|
||
+ abs(estimated_pos_neg_dist - actual_pos_neg_dist)
|
||
+ (estimated_pos_dist - estimated_neg_dist - (actual_pos_dist - actual_neg_dist)).clip(min=0))
|
||
loss.backward()
|
||
optimizer.step()
|
||
inner_bar.update(3 * batch_size)
|
||
loss_scalar = loss.item()
|
||
total_loss += loss_scalar
|
||
inner_bar.set_description("loss %.2f" % loss_scalar)
|
||
ex.log_scalar("avg_loss", total_loss / len(data) * 3)
|
||
log_metric("avg_loss", total_loss / len(data) * 3)
|
||
# print()
|
||
# print("Total epoch loss:", total_loss)
|
||
# print("Total epoch avg loss:", total_loss / TOTAL_TRAINING_OUT_LEN)
|
||
# print("Training snapshots:", train_snapshots)
|
||
# print("Training snapshots(%):", train_snapshots_percentage)
|
||
# print("Evaluation snapshots:", eval_snapshots)
|
||
# print("Evaluation snapshots(%):", eval_snapshots_percentage)
|
||
outer_bar.set_description("Epochs")
|
||
outer_bar.update(1)
|
||
|
||
|
||
def evaluate_monte_carlo(model, repeats, data, batch_size, in_alphabet, max_len):
|
||
with torch.no_grad():
|
||
i = 0
|
||
diff = 0
|
||
outer_bar = tqdm(total=repeats, position=0)
|
||
inner_bar = tqdm(total=len(data), position=1)
|
||
outer_bar.set_description("Epochs")
|
||
|
||
def collate(batch: [(torch.tensor, str)]):
|
||
return encode(batch, in_alphabet, max_len)
|
||
|
||
for _ in range(repeats):
|
||
data_loader = DataLoader(dataset=data, drop_last=True,
|
||
batch_size=2 * batch_size,
|
||
collate_fn=collate,
|
||
shuffle=True)
|
||
inner_bar.reset()
|
||
for batch_text, batch_phonemes in data_loader:
|
||
positive, negative = batch_text.to(device).split(batch_size)
|
||
ph_positive = batch_phonemes[0:batch_size]
|
||
ph_negative = batch_phonemes[batch_size:]
|
||
embedded_positive = model(positive)
|
||
embedded_negative = model(negative)
|
||
estimated_dist = torch.linalg.norm(embedded_negative - embedded_positive, dim=1)
|
||
actual_dist = dist(ph_negative, ph_positive)
|
||
diff += sum(abs(estimated_dist - actual_dist))
|
||
i += batch_size
|
||
inner_bar.update(2 * batch_size)
|
||
outer_bar.update(1)
|
||
with open('results.txt', 'w+') as r:
|
||
print("Average estimation error " + str(diff.item() / i))
|
||
r.write("Average estimation error " + str(diff.item() / i) + "\n")
|
||
ex.log_scalar("avg_estim_error", diff.item() / i)
|
||
|
||
|
||
@ex.config
|
||
def cfg():
|
||
kernel_size = 3
|
||
hidden_layers = 14
|
||
data_file = 'preprocessed.tsv'
|
||
epochs = 14
|
||
mode = 'train'
|
||
teacher_forcing_probability = 0.4
|
||
learning_rate = 0.01
|
||
batch_size = 512
|
||
max_len = 32
|
||
total_out_len = 0
|
||
model_file = 'cnn.pth'
|
||
out_lookup = ['', 'b', 'a', 'ʊ', 't', 'k', 'ə', 'z', 'ɔ', 'ɹ', 's', 'j', 'u', 'm', 'f', 'ɪ', 'o', 'ɡ', 'ɛ', 'n',
|
||
'e', 'd',
|
||
'ɫ', 'w', 'i', 'p', 'ɑ', 'ɝ', 'θ', 'v', 'h', 'æ', 'ŋ', 'ʃ', 'ʒ', 'ð']
|
||
in_lookup = ['', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't',
|
||
'u', 'v', 'w', 'x', 'y', 'z']
|
||
|
||
|
||
def signature(model, in_alphabet, max_len):
|
||
mock_x = [('abc', 'xyz')]
|
||
mock_text, _ = encode_str(mock_x, in_alphabet, max_len)
|
||
mock_y = model(mock_text)
|
||
return mlflow.models.signature.infer_signature(mock_text.detach().numpy(), mock_y.detach().numpy())
|
||
|
||
|
||
@ex.automain
|
||
def run(kernel_size, hidden_layers, data_file, epochs, teacher_forcing_probability, learning_rate, batch_size, max_len,
|
||
total_out_len, model_file, out_lookup, in_lookup, mode):
|
||
with mlflow.start_run():
|
||
log_param("kernel_size", kernel_size)
|
||
log_param("hidden_layers", hidden_layers)
|
||
log_param("data_file", data_file)
|
||
log_param("epochs", epochs)
|
||
log_param("learning_rate", learning_rate)
|
||
log_param("batch_size", batch_size)
|
||
log_param("max_len", max_len)
|
||
in_alphabet = {letter: idx for idx, letter in enumerate(in_lookup)}
|
||
|
||
out_alphabet = {letter: idx for idx, letter in enumerate(out_lookup)}
|
||
|
||
data: [(torch.tensor, torch.tensor)] = []
|
||
|
||
texts: [str] = []
|
||
|
||
with open(data_file) as f:
|
||
for line in f:
|
||
text, phonemes = line.split("\t")
|
||
texts.append(text)
|
||
assert len(text) <= max_len, text
|
||
text = torch.tensor([in_alphabet[letter] for letter in text], dtype=torch.int)
|
||
data.append((text, phonemes))
|
||
|
||
cnn = CNN(kernel_size=kernel_size, hidden_layers=hidden_layers, channels=max_len, embedding_size=max_len,
|
||
in_alphabet=in_alphabet, max_len=max_len).to(device)
|
||
if mode == 'train':
|
||
train_model(cnn, learning_rate, in_alphabet, max_len, data, epochs, batch_size)
|
||
torch.save(cnn.state_dict(), model_file)
|
||
ex.add_artifact(model_file)
|
||
mlflow.pytorch.log_model(cnn, "cnn-model", registered_model_name="PhoneticEdDistEmbeddings",
|
||
signature=signature(cnn, in_alphabet, max_len))
|
||
|
||
if mode == 'eval':
|
||
cnn.eval()
|
||
evaluate_monte_carlo(cnn, 1, data, batch_size, in_alphabet, max_len)
|