348 lines
8.9 KiB
Plaintext
348 lines
8.9 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"source": [
|
|
"## IMPORTS"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"outputs": [],
|
|
"source": [
|
|
"import regex as re\n",
|
|
"import sys\n",
|
|
"from torchtext.vocab import build_vocab_from_iterator\n",
|
|
"import lzma\n",
|
|
"from torch.utils.data import IterableDataset\n",
|
|
"import itertools\n",
|
|
"from torch import nn\n",
|
|
"import torch\n",
|
|
"import pickle\n",
|
|
"from torch.utils.data import DataLoader\n",
|
|
"\n",
|
|
"print(torch.backends.mps.is_available())\n",
|
|
"print(torch.backends.mps.is_built())"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"source": [
|
|
"## FUNCTIONS"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"outputs": [],
|
|
"source": [
|
|
"def get_words_from_line(line):\n",
|
|
" line = line.rstrip()\n",
|
|
" yield '<s>'\n",
|
|
" for t in line.split(' '):\n",
|
|
" yield t\n",
|
|
" yield '</s>'\n",
|
|
"\n",
|
|
"def get_word_lines_from_file(file_name):\n",
|
|
" n = 0\n",
|
|
" with lzma.open(file_name, 'r') as fh:\n",
|
|
" for line in fh:\n",
|
|
" n += 1\n",
|
|
" if n % 1000 == 0:\n",
|
|
" print(n ,file=sys.stderr)\n",
|
|
" yield get_words_from_line(line.decode('utf-8'))\n",
|
|
"\n",
|
|
"def look_ahead_iterator(gen):\n",
|
|
" prev = None\n",
|
|
" for item in gen:\n",
|
|
" if prev is not None:\n",
|
|
" yield (prev, item)\n",
|
|
" prev = item\n",
|
|
"\n",
|
|
"def clean(text):\n",
|
|
" text = str(text).lower().replace('-\\\\n', '').replace('\\\\n', ' ').replace('-', '').replace('\\'s', ' is').replace('\\'re', ' are').replace('\\'m', ' am').replace('\\'ve', ' have').replace('\\'ll', ' will')\n",
|
|
" text = re.sub(r'\\p{P}', '', text)\n",
|
|
" return text\n",
|
|
"\n",
|
|
"def predict(word, model, vocab):\n",
|
|
" try:\n",
|
|
" ixs = torch.tensor(vocab.forward([word])).to(device)\n",
|
|
" except:\n",
|
|
" ixs = torch.tensor(vocab.forward(['<unk>'])).to(device)\n",
|
|
" word = '<unk>'\n",
|
|
" out = model(ixs)\n",
|
|
" top = torch.topk(out[0], 300)\n",
|
|
" top_indices = top.indices.tolist()\n",
|
|
" top_probs = top.values.tolist()\n",
|
|
" top_words = vocab.lookup_tokens(top_indices)\n",
|
|
" prob_list = list(zip(top_words, top_probs))\n",
|
|
" for index, element in enumerate(prob_list):\n",
|
|
" unk = None\n",
|
|
" if '<unk>' in element:\n",
|
|
" unk = prob_list.pop(index)\n",
|
|
" prob_list.append(('', unk[1]))\n",
|
|
" break\n",
|
|
" if unk is None:\n",
|
|
" prob_list[-1] = ('', prob_list[-1][1])\n",
|
|
" return ' '.join([f'{x[0]}:{x[1]}' for x in prob_list])\n",
|
|
"\n",
|
|
"def predicition_for_file(model, vocab, folder, file):\n",
|
|
" print('=' * 10, f' do prediction for {folder}/{file} ', '=' * 10)\n",
|
|
" with lzma.open(f'{folder}/in.tsv.xz', mode='rt', encoding='utf-8') as f:\n",
|
|
" with open(f'{folder}/out.tsv', 'w', encoding='utf-8') as fid:\n",
|
|
"\n",
|
|
" for line in f:\n",
|
|
" separated = line.split('\\t')\n",
|
|
" head = clean(separated[6]).split()[-1]\n",
|
|
" new_line = predict(head, model, vocab)\n",
|
|
" fid.write(new_line + '\\n')"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"source": [
|
|
"## CLASSES"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"outputs": [],
|
|
"source": [
|
|
"class Bigrams(IterableDataset):\n",
|
|
" def __init__(self, text_file, vocabulary_size):\n",
|
|
" self.vocab = build_vocab_from_iterator(\n",
|
|
" get_word_lines_from_file(text_file),\n",
|
|
" max_tokens = vocabulary_size,\n",
|
|
" specials = ['<unk>'])\n",
|
|
" self.vocab.set_default_index(self.vocab['<unk>'])\n",
|
|
" self.vocabulary_size = vocabulary_size\n",
|
|
" self.text_file = text_file\n",
|
|
"\n",
|
|
" def __iter__(self):\n",
|
|
" return look_ahead_iterator(\n",
|
|
" (self.vocab[t] for t in itertools.chain.from_iterable(get_word_lines_from_file(self.text_file))))\n",
|
|
"\n",
|
|
"class SimpleBigramNeuralLanguageModel(nn.Module):\n",
|
|
" def __init__(self, vocabulary_size, embedding_size):\n",
|
|
" super(SimpleBigramNeuralLanguageModel, self).__init__()\n",
|
|
" self.model = nn.Sequential(\n",
|
|
" nn.Embedding(vocabulary_size, embedding_size),\n",
|
|
" nn.Linear(embedding_size, vocabulary_size),\n",
|
|
" nn.Softmax()\n",
|
|
" )\n",
|
|
"\n",
|
|
" def forward(self, x):\n",
|
|
" return self.model(x)"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"source": [
|
|
"## PARAMETERS"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"outputs": [],
|
|
"source": [
|
|
"vocab_size = 30000\n",
|
|
"embed_size = 1000\n",
|
|
"batch_size = 5000\n",
|
|
"device = 'mps'\n",
|
|
"path_to_training_file = './train/in.tsv.xz'\n",
|
|
"path_to_model_file = 'model_neural_network.bin'\n",
|
|
"folder_dev_0, file_dev_0 = 'dev-0', 'in.tsv.xz'\n",
|
|
"folder_test_a, file_test_a = 'test-A', 'in.tsv.xz'\n",
|
|
"path_to_vocabulary_file = 'vocabulary_neural_network.pickle'"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"source": [
|
|
"## VOCAB"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"outputs": [],
|
|
"source": [
|
|
"vocab = build_vocab_from_iterator(\n",
|
|
" get_word_lines_from_file(path_to_training_file),\n",
|
|
" max_tokens = vocab_size,\n",
|
|
" specials = ['<unk>'])\n",
|
|
"\n",
|
|
"with open(path_to_vocabulary_file, 'wb') as handle:\n",
|
|
" pickle.dump(vocab, handle, protocol=pickle.HIGHEST_PROTOCOL)"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"source": [
|
|
"## TRAIN MODEL"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"outputs": [],
|
|
"source": [
|
|
"train_dataset = Bigrams(path_to_training_file, vocab_size)\n",
|
|
"model = SimpleBigramNeuralLanguageModel(vocab_size, embed_size).to(device)\n",
|
|
"data = DataLoader(train_dataset, batch_size=batch_size)\n",
|
|
"optimizer = torch.optim.Adam(model.parameters())\n",
|
|
"criterion = torch.nn.NLLLoss()\n",
|
|
"\n",
|
|
"model.train()\n",
|
|
"step = 0\n",
|
|
"for x, y in data:\n",
|
|
" x = x.to(device)\n",
|
|
" y = y.to(device)\n",
|
|
" optimizer.zero_grad()\n",
|
|
" ypredicted = model(x)\n",
|
|
" loss = criterion(torch.log(ypredicted), y)\n",
|
|
" if step % 100 == 0:\n",
|
|
" print(step, loss)\n",
|
|
" step += 1\n",
|
|
" loss.backward()\n",
|
|
" optimizer.step()\n",
|
|
"\n",
|
|
"torch.save(model.state_dict(), path_to_model_file)"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"source": [
|
|
"## LOAD MODEL AND VOCAB"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"outputs": [],
|
|
"source": [
|
|
"with open(path_to_vocabulary_file, 'rb') as handle:\n",
|
|
" vocab = pickle.load(handle)\n",
|
|
"model = SimpleBigramNeuralLanguageModel(vocab_size, embed_size).to(device)\n",
|
|
"model.load_state_dict(torch.load(path_to_model_file))\n",
|
|
"model.eval()"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"source": [
|
|
"## CREATE OUTPUTS FILES"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"source": [
|
|
"### DEV-0"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"outputs": [],
|
|
"source": [
|
|
"predicition_for_file(model, vocab, folder_dev_0, file_dev_0)"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"source": [
|
|
"### TEST-A"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"outputs": [],
|
|
"source": [
|
|
"predicition_for_file(model, vocab, folder_test_a, file_test_a)"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 2
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython2",
|
|
"version": "2.7.6"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 0
|
|
}
|