machine_translation.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/imancn/nlp/blob/main/machine_translation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# English to Persian Translation "
      ],
      "metadata": {
        "id": "v-4pTrYWQThZ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Imports"
      ],
      "metadata": {
        "id": "t0q6hUMFQThd"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt \n",
        "import torch\n",
        "import random\n",
        "import re\n",
        "import unicodedata\n",
        "import os"
      ],
      "outputs": [],
      "metadata": {
        "id": "eDgqL980QThe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "y_UcMkZBRIK1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "098dcbf1-a916-47b5-d4b7-cdfac3c5efea"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Data Pre Processing"
      ],
      "metadata": {
        "id": "bgeXr7HLQThf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def preProcessing(w):\n",
        "    word = ''.join(c for c in unicodedata.normalize('NFD', word.lower().strip())\n",
        "                   if unicodedata.category(c) != 'Mn')\n",
        "    word = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n",
        "    word = re.sub(r'[\" \"]+', \" \", w)\n",
        "return '<start> ' + word.strip() + ' <end>'"
      ],
      "metadata": {
        "id": "s3JRrYzxNLjM"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "source": [
        "def create_dataset():\n",
        "  fileEN = \"/content/drive/MyDrive/Colab Notebooks/NLP/machin-translation/content/TEP.en-fa.en\"\n",
        "  fileFA = \"/content/drive/MyDrive/Colab Notebooks/NLP/machin-translation/content/TEP.en-fa.fa\"\n",
        "  enList = []\n",
        "  faList = []\n",
        "  count = 0\n",
        "  with open(fileEN,\"r\") as fen:\n",
        "    for line in fen:\n",
        "      enList.append(preProcessing(line))\n",
        "      count += 1\n",
        "      if (count > 150000) : \n",
        "        break\n",
        "  count = 0\n",
        "  with open(fileFA,\"r\") as ffa:\n",
        "    for line in ffa:\n",
        "      faList.append(preProcessing(line))\n",
        "      count += 1\n",
        "      if (count > 150000) : \n",
        "        break\n",
        "  return faList, enList\n"
      ],
      "outputs": [],
      "metadata": {
        "id": "lLzWrEYFQThg"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# tokenize"
      ],
      "metadata": {
        "id": "uAH6XWkKEVTj"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "def tokenize(lang):\n",
        "    lang_tokenizer = tf.keras.preprocessing.text.Tokenizer(filters='')\n",
        "    lang_tokenizer.fit_on_texts(lang)\n",
        "    tensor = tf.keras.preprocessing.sequence.pad_sequences(lang_tokenizer.texts_to_sequences(lang), padding='post')\n",
        "    return tensor, lang_tokenizer"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "-eJTE3p6FC1N"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "outputs": [],
      "source": [
        "def load_dataset():\n",
        "    targ_lang, inp_lang = create_dataset()\n",
        "    input_tensor, inp_lang_tokenizer = tokenize(inp_lang)\n",
        "    target_tensor, targ_lang_tokenizer = tokenize(targ_lang)\n",
        "\n",
        "    return input_tensor, target_tensor, inp_lang_tokenizer, targ_lang_tokenizer"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "XtAFv0hYFC1P"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Create dataset (target language: Farsi, input language: English)"
      ],
      "metadata": {
        "id": "A-cF4K9XEag0"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "outputs": [],
      "source": [
        "input_tensor, target_tensor, inp_lang, targ_lang = load_dataset()\n",
        "max_length_targ, max_length_inp = target_tensor.shape[1], input_tensor.shape[1]"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "j7BZ2vlKFC1R"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: sklearn in /usr/local/lib/python3.7/dist-packages (0.0)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from sklearn) (1.0.2)\n",
            "Requirement already satisfied: numpy>=1.14.6 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->sklearn) (1.21.6)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->sklearn) (1.1.0)\n",
            "Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->sklearn) (1.7.3)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->sklearn) (3.1.0)\n",
            "120000 120000 30001 30001\n"
          ]
        }
      ],
      "source": [
        "!pip3 install sklearn\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Create training and validation sets using an 80/20 split\n",
        "input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor, target_tensor, test_size=0.2)\n",
        "\n",
        "print(len(input_tensor_train), len(target_tensor_train), len(input_tensor_val), len(target_tensor_val))"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "DQcGIS-eFC1S",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0bf033a0-014d-408c-e337-33f88a7445d4"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Show the mapping word index and language tokenizer\n"
      ],
      "metadata": {
        "id": "7PkmyUqGE1yd"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Input Lang; index to word mapping\n",
            "2 -> <start>\n",
            "11 -> and\n",
            "102 -> then\n",
            "32 -> he\n",
            "568 -> started\n",
            "342 -> saying\n",
            "2148 -> nasty\n",
            "185 -> things\n",
            "8 -> to\n",
            "14 -> that\n",
            "5929 -> stripper\n",
            "1 -> .\n",
            "3 -> <end>\n",
            "\n",
            "Target Lang; index to word mapping\n",
            "2 -> <start>\n",
            "99 -> بعد\n",
            "180 -> شروع\n",
            "121 -> کرد\n",
            "7 -> به\n",
            "5257 -> فحش\n",
            "454 -> دادن\n",
            "7 -> به\n",
            "13 -> اون\n",
            "8603 -> رقاصه\n",
            "1 -> .\n",
            "3 -> <end>\n"
          ]
        }
      ],
      "source": [
        "def convert(lang, tensor):\n",
        "    for t in tensor:\n",
        "        if t != 0:\n",
        "            print (\"%d -> %s\" % (t, lang.index_word[t]))\n",
        "\n",
        "print (\"Input Lang; index to word mapping\")\n",
        "convert(inp_lang, input_tensor_train[0])\n",
        "print ()\n",
        "print (\"Target Lang; index to word mapping\")\n",
        "convert(targ_lang, target_tensor_train[0])"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "poCJ9FzBFC1T",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0c258b91-ab00-47ce-813d-bd9ada1b154d"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# model parameters\n"
      ],
      "metadata": {
        "id": "Cai34mnNE5F7"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "outputs": [],
      "source": [
        "BUFFER_SIZE = len(input_tensor_train)\n",
        "BATCH_SIZE = 256\n",
        "steps_per_epoch = len(input_tensor_train)//BATCH_SIZE\n",
        "embedding_dim = 256\n",
        "units = 1024\n",
        "vocab_inp_size = len(inp_lang.word_index) + 1\n",
        "vocab_tar_size = len(targ_lang.word_index) + 1"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "mvKT_2heFC1V"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "outputs": [],
      "source": [
        "dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train)).shuffle(BUFFER_SIZE)\n",
        "dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "78I603Z6FC1W"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(TensorShape([256, 39]), TensorShape([256, 35]))"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ],
      "source": [
        "example_input_batch, example_target_batch = next(iter(dataset))\n",
        "example_input_batch.shape, example_target_batch.shape"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "zErjDfLJFC1X",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "04bc3dd2-f409-4c9e-9350-ad3195efd78e"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Encoder"
      ],
      "metadata": {
        "id": "QRekFZs_E9xV"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "outputs": [],
      "source": [
        "class Encoder(tf.keras.Model):\n",
        "    def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n",
        "        super(Encoder, self).__init__()\n",
        "        self.batch_sz = batch_sz\n",
        "        self.enc_units = enc_units\n",
        "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
        "        self.gru = tf.keras.layers.GRU(self.enc_units, return_sequences=True, return_state=True, recurrent_initializer='glorot_uniform')\n",
        "\n",
        "    def call(self, x, hidden):\n",
        "        x = self.embedding(x)\n",
        "        output, state = self.gru(x, initial_state=hidden)\n",
        "        return output, state\n",
        "\n",
        "    def initialize_hidden_state(self):\n",
        "        return tf.zeros((self.batch_sz, self.enc_units))"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "PVKfUGDVFC1Z"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Encoder output shape: (batch size, sequence length, units) (256, 39, 1024)\n",
            "Encoder Hidden state shape: (batch size, units) (256, 1024)\n"
          ]
        }
      ],
      "source": [
        "encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)\n",
        "\n",
        "sample_hidden = encoder.initialize_hidden_state()\n",
        "sample_output, sample_hidden = encoder(example_input_batch, sample_hidden)\n",
        "\n",
        "print ('Encoder output shape: (batch size, sequence length, units) {}'.format(sample_output.shape))\n",
        "print ('Encoder Hidden state shape: (batch size, units) {}'.format(sample_hidden.shape))"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "isIRFEGxFC1a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fb91c865-b993-49b4-b741-216f114e7d0e"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Attention Mechanism"
      ],
      "metadata": {
        "id": "1KiBPaikFMRj"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "outputs": [],
      "source": [
        "class Attention(tf.keras.layers.Layer):\n",
        "    def __init__(self, units):\n",
        "        super(Attention, self).__init__()\n",
        "        self.W1 = tf.keras.layers.Dense(units)\n",
        "        self.W2 = tf.keras.layers.Dense(units)\n",
        "        self.V = tf.keras.layers.Dense(1)\n",
        "\n",
        "    def call(self, query, values):\n",
        "        query_with_time_axis = tf.expand_dims(query, 1)\n",
        "        score = self.V(tf.nn.tanh(\n",
        "            self.W1(query_with_time_axis) + self.W2(values)))\n",
        "        attention_weights = tf.nn.softmax(score, axis=1)\n",
        "        context_vector = attention_weights * values\n",
        "        context_vector = tf.reduce_sum(context_vector, axis=1)\n",
        "\n",
        "        return context_vector, attention_weights"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "1ffByHc8FC1a"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Attention result shape: (batch size, units) (256, 1024)\n",
            "Attention weights shape: (batch_size, sequence_length, 1) (256, 39, 1)\n"
          ]
        }
      ],
      "source": [
        "attention_layer = Attention(10)\n",
        "attention_result, attention_weights = attention_layer(sample_hidden, sample_output)\n",
        "\n",
        "print(\"Attention result shape: (batch size, units) {}\".format(attention_result.shape))\n",
        "print(\"Attention weights shape: (batch_size, sequence_length, 1) {}\".format(attention_weights.shape))"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "YowqILutFC1b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c5c2100d-b0a8-48e0-ac1a-b18e75c59ed7"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Decoder"
      ],
      "metadata": {
        "id": "UH8Nqu2eFJr5"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "outputs": [],
      "source": [
        "class Decoder(tf.keras.Model):\n",
        "    def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n",
        "        super(Decoder, self).__init__()\n",
        "        self.batch_sz = batch_sz\n",
        "        self.dec_units = dec_units\n",
        "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
        "        self.gru = tf.keras.layers.GRU(self.dec_units, return_sequences=True, return_state=True, recurrent_initializer='glorot_uniform')\n",
        "        self.fc = tf.keras.layers.Dense(vocab_size)\n",
        "        self.attention = Attention(self.dec_units)\n",
        "\n",
        "    def call(self, x, hidden, enc_output):\n",
        "        context_vector, attention_weights = self.attention(hidden, enc_output)\n",
        "        x = self.embedding(x)\n",
        "        x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
        "        output, state = self.gru(x)\n",
        "        output = tf.reshape(output, (-1, output.shape[2]))\n",
        "        x = self.fc(output)\n",
        "\n",
        "        return x, state, attention_weights"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "GK0bGvCCFC1b"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Decoder output shape: (batch_size, vocab size) (256, 51819)\n"
          ]
        }
      ],
      "source": [
        "decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE)\n",
        "sample_decoder_output, _, _ = decoder(tf.random.uniform((BATCH_SIZE, 1)), sample_hidden, sample_output)\n",
        "\n",
        "print ('Decoder output shape: (batch_size, vocab size) {}'.format(sample_decoder_output.shape))"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "kQ7uy7C0FC1c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6f279ddb-40a3-44ee-8371-43a94b708259"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Initialize optimizer and loss functions"
      ],
      "metadata": {
        "id": "09-syKrdD5jj"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "outputs": [],
      "source": [
        "optimizer = tf.keras.optimizers.Adam()\n",
        "\n",
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction='none')\n",
        "\n",
        "def lossFun(real, pred):\n",
        "    mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
        "    loss_ = loss_object(real, pred)\n",
        "    mask = tf.cast(mask, dtype=loss_.dtype)\n",
        "    loss_ *= mask\n",
        "    return tf.reduce_mean(loss_)"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "EhW0D7O2FC1d"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# training_checkpoints"
      ],
      "metadata": {
        "id": "9yarnytuD1SZ"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "outputs": [],
      "source": [
        "import os\n",
        "\n",
        "cp_dir = '/content/drive/MyDrive/Colab Notebooks/NLP/machin-translation/content/training_checkpoints'\n",
        "cp_prefix = os.path.join(cp_dir, \"ckpt\")\n",
        "cp = tf.train.Checkpoint(optimizer=optimizer, encoder=encoder, decoder=decoder)"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "qKTC9zoyFC1d"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# train step"
      ],
      "metadata": {
        "id": "mdXKzKjIKSDl"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "outputs": [],
      "source": [
        "@tf.function\n",
        "def train_step(inp, targ, enc_hidden):\n",
        "    loss = 0\n",
        "\n",
        "    with tf.GradientTape() as tape:\n",
        "        enc_output, enc_hidden = encoder(inp, enc_hidden)\n",
        "        dec_hidden = enc_hidden\n",
        "        # <start> token is the initial decoder input\n",
        "        dec_input = tf.expand_dims([targ_lang.word_index['<start>']] * BATCH_SIZE, 1)\n",
        "\n",
        "        for t in range(1, targ.shape[1]):\n",
        "            predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n",
        "            loss += lossFun(targ[:, t], predictions)\n",
        "            dec_input = tf.expand_dims(targ[:, t], 1)\n",
        "\n",
        "    batch_loss = (loss / int(targ.shape[1]))\n",
        "    variables = encoder.trainable_variables + decoder.trainable_variables\n",
        "    gradients = tape.gradient(loss, variables)\n",
        "    optimizer.apply_gradients(zip(gradients, variables))\n",
        "\n",
        "    return batch_loss"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n"
        },
        "id": "6LT_kSTmFC1e"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Start Training"
      ],
      "metadata": {
        "id": "Ycx2zW1VKWjj"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1 Loss 1.6320\n",
            "Time taken for 1 epoch 908.2921917438507 sec\n",
            "\n",
            "Epoch 2 Loss 1.3534\n",
            "Time taken for 1 epoch 890.5865161418915 sec\n",
            "\n",
            "Epoch 3 Loss 1.1964\n",
            "Time taken for 1 epoch 889.5785422325134 sec\n",
            "\n",
            "Epoch 4 Loss 1.0470\n",
            "Time taken for 1 epoch 894.8115491867065 sec\n",
            "\n",
            "Epoch 5 Loss 0.9070\n",
            "Time taken for 1 epoch 890.4350519180298 sec\n",
            "\n",
            "Epoch 6 Loss 0.7785\n",
            "Time taken for 1 epoch 926.2395641803741 sec\n",
            "\n",
            "Epoch 7 Loss 0.6659\n",
            "Time taken for 1 epoch 890.8751950263977 sec\n",
            "\n",
            "Epoch 8 Loss 0.5738\n",
            "Time taken for 1 epoch 895.5262868404388 sec\n",
            "\n",
            "Epoch 9 Loss 0.5015\n",
            "Time taken for 1 epoch 921.9271593093872 sec\n",
            "\n",
            "Epoch 10 Loss 0.4388\n",
            "Time taken for 1 epoch 894.3808171749115 sec\n",
            "\n"
          ]
        }
      ],
      "source": [
        " import time\n",
        "\n",
        "EPOCHS = 10\n",
        "\n",
        "for epoch in range(EPOCHS):\n",
        "    start = time.time()\n",
        "    enc_hidden = encoder.initialize_hidden_state()\n",
        "    total_loss = 0\n",
        "\n",
        "    for (batch, (inp, targ)) in enumerate(dataset.take(steps_per_epoch)):\n",
        "        # Call the train method\n",
        "        batch_loss = train_step(inp, targ, enc_hidden)\n",
        "        # Compute the loss (per batch)\n",
        "        total_loss += batch_loss\n",
        "\n",
        "    # Save (checkpoint)\n",
        "    if (epoch + 1) % 2 == 0:\n",
        "        cp.save(file_prefix = cp_prefix)\n",
        "    print('Epoch {} Loss {:.4f}'.format(epoch + 1,total_loss / steps_per_epoch))\n",
        "\n",
        "    print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n",
          "is_executing": true
        },
        "id": "hioCuDSPFC1e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "eef86ddf-0dd6-44cd-b17c-df68bd8af2c2"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Evaluate function"
      ],
      "metadata": {
        "id": "6JkG-G5JKlV6"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "\n",
        "def evaluate(sentence):\n",
        "  attention_plot = np.zeros((max_length_targ, max_length_inp))\n",
        "  # Preprocess the sentence given\n",
        "  sentence = preProcessing(sentence)\n",
        "\n",
        "  # Fetch the indices concerning the words in the sentence and pad the sequence\n",
        "  inputs = [inp_lang.word_index[i] for i in sentence.split(' ')]\n",
        "  inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs], maxlen=max_length_inp, padding='post')\n",
        "  # Convert the inputs to tensors\n",
        "  inputs = tf.convert_to_tensor(inputs)\n",
        "\n",
        "  result = ''\n",
        "\n",
        "  hidden = [tf.zeros((1, units))]\n",
        "  enc_out, enc_hidden = encoder(inputs, hidden)\n",
        "  dec_hidden = enc_hidden\n",
        "  dec_input = tf.expand_dims([targ_lang.word_index['<start>']], 0)\n",
        "\n",
        "  # Loop until the max_length is reached for the target lang (ENGLISH)\n",
        "  for t in range(max_length_targ):\n",
        "    predictions, dec_hidden, attention_weights = decoder(dec_input, dec_hidden, enc_out)\n",
        "    attention_weights = tf.reshape(attention_weights, (-1, ))\n",
        "    attention_plot[t] = attention_weights.numpy()\n",
        "    predicted_id = tf.argmax(predictions[0]).numpy()\n",
        "    result += targ_lang.index_word[predicted_id] + ' '\n",
        "    # If <end> token is reached, return\n",
        "    if targ_lang.index_word[predicted_id] == '<end>':\n",
        "      return result, sentence, attention_plot\n",
        "    # The predicted ID is fed back into the model\n",
        "    dec_input = tf.expand_dims([predicted_id], 0)\n",
        "\n",
        "  return result, sentence, attention_plot"
      ],
      "metadata": {
        "pycharm": {
          "name": "#%%\n",
          "is_executing": true
        },
        "id": "9bicVC4LFC1f"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Translate function"
      ],
      "metadata": {
        "id": "_1brPL93T9IN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def translate(sentence):\n",
        "  result, sentence, attention_plot = evaluate(sentence)\n",
        "\n",
        "  print('Input: %s' % (sentence))\n",
        "  print('Output: {}'.format(result))"
      ],
      "metadata": {
        "id": "76dg1Q7Jhp7B"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Restore the latest checkpoint"
      ],
      "metadata": {
        "id": "dZlBJK9wUBlm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Restore the latest checkpoint in checkpoint_dir\n",
        "cp.restore(tf.train.latest_checkpoint(cp_dir))"
      ],
      "metadata": {
        "id": "7LCjWvyShty-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b095323c-3076-4c5f-f933-f886b81d8389"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x7f88ca7e0e90>"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "translate(\"i was very bad\")\n",
        "translate(\"im at home\")\n",
        "translate(\"i have a cold\")\n",
        "translate(\"take me to the tunnel\")\n",
        "translate(\"go away\")\n",
        "translate(\"i walk everyday\")"
      ],
      "metadata": {
        "id": "ehoNDbbnhxxB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bc05e63c-5ad3-40d3-b325-bf1129cb39f3"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Input: <start> i was very bad <end>\n",
            "Output: من خيلي بد بود . <end> \n",
            "Input: <start> im at home <end>\n",
            "Output: من تو خونه ام . <end> \n",
            "Input: <start> i have a cold <end>\n",
            "Output: يه خورده . <end> \n",
            "Input: <start> take me to the tunnel <end>\n",
            "Output: منو ببر به تونل تونل بگيريد . <end> \n",
            "Input: <start> go away <end>\n",
            "Output: برو . <end> \n",
            "Input: <start> i walk everyday <end>\n",
            "Output: من فردا روز میرم . <end> \n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "interpreter": {
      "hash": "4bb0fe8ced3cf0716ac3718fe834e829af40e8ba0fef1c4cadecb390da29a017"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3.7.11 64-bit ('torch': conda)"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.11"
    },
    "colab": {
      "name": "machine-translation.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}