{ "cells": [ { "cell_type": "markdown", "source": [ "## Seq2Seq translation" ], "metadata": { "collapsed": false }, "id": "9748a77fd2a3cac3" }, { "cell_type": "code", "execution_count": 3, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "\n", "# Pytorch\n", "import torch\n", "import torch.nn as nn\n", "import torch.optim as optim\n", "\n", "from torch.utils.data import TensorDataset, DataLoader, RandomSampler\n", "import torch.nn.functional as F\n", "\n", "import torchtext\n", "torchtext.disable_torchtext_deprecation_warning()\n", "\n", "from torchtext.data.metrics import bleu_score\n", "\n", "from unidecode import unidecode\n", "\n", "import regex as re\n", "from string import punctuation\n", "import random\n", "\n", "from tqdm.notebook import tqdm" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-05-23T07:51:30.799418700Z", "start_time": "2024-05-23T07:51:30.786186900Z" } }, "id": "initial_id" }, { "cell_type": "code", "execution_count": 4, "outputs": [], "source": [ "# Use GPU if available\n", "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-05-23T07:51:31.808275400Z", "start_time": "2024-05-23T07:51:31.791795200Z" } }, "id": "373fcd790697fce0" }, { "cell_type": "markdown", "source": [ "### Load data corpus" ], "metadata": { "collapsed": false }, "id": "81adf3bad0a07802" }, { "cell_type": "code", "execution_count": 5, "outputs": [], "source": [ "# Load the data\n", "df = pd.read_csv('pol-eng/pol.txt', sep='\\t', header=None, usecols=[0, 1], names=['source', 'target'])\n", "df_1000 = df.head(1000)\n", "df = df_1000" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-05-23T07:51:33.322595100Z", "start_time": "2024-05-23T07:51:33.202326400Z" } }, "id": "695222e59fb7c9fd" }, { "cell_type": "code", "execution_count": 6, "outputs": [ { "data": { "text/plain": " source target\n0 Go. Idź.\n1 Hi. Cześć.\n2 Run! Uciekaj!\n3 Run. Biegnij.\n4 Run. Uciekaj.\n.. ... ...\n995 We walked. Poszliśmy pieszo.\n996 We yawned. Ziewaliśmy.\n997 We'll see. Zobaczymy.\n998 We're men. Jesteśmy mężczyznami.\n999 We're sad. Jesteśmy smutni.\n\n[1000 rows x 2 columns]", "text/html": "
\n | source | \ntarget | \n
---|---|---|
0 | \nGo. | \nIdź. | \n
1 | \nHi. | \nCześć. | \n
2 | \nRun! | \nUciekaj! | \n
3 | \nRun. | \nBiegnij. | \n
4 | \nRun. | \nUciekaj. | \n
... | \n... | \n... | \n
995 | \nWe walked. | \nPoszliśmy pieszo. | \n
996 | \nWe yawned. | \nZiewaliśmy. | \n
997 | \nWe'll see. | \nZobaczymy. | \n
998 | \nWe're men. | \nJesteśmy mężczyznami. | \n
999 | \nWe're sad. | \nJesteśmy smutni. | \n
1000 rows × 2 columns
\n