Sequence Models/Neural_machine_translation_with_attention_v4a.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 14508,
     "status": "ok",
     "timestamp": 1612468511651,
     "user": {
      "displayName": "Mubsi K",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gip7OjOkdNkKxKDyWEQAq1o8ccGN_HrBTGdqjgQ=s64",
      "userId": "08094225471505108399"
     },
     "user_tz": -300
    },
    "id": "RcBRMzPiiMmp",
    "outputId": "17e9a429-5bb6-4401-a23a-f8f756d6113d"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Bidirectional, Concatenate, Permute, Dot, Input, LSTM, Multiply\n",
    "from tensorflow.keras.layers import RepeatVector, Dense, Activation, Lambda\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.models import load_model, Model\n",
    "import tensorflow.keras.backend as K\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "from faker import Faker\n",
    "import random\n",
    "from tqdm import tqdm\n",
    "from babel.dates import format_date\n",
    "from nmt_utils import *\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 16981,
     "status": "ok",
     "timestamp": 1612468514155,
     "user": {
      "displayName": "Mubsi K",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gip7OjOkdNkKxKDyWEQAq1o8ccGN_HrBTGdqjgQ=s64",
      "userId": "08094225471505108399"
     },
     "user_tz": -300
    },
    "id": "gwIf5l17h3Mg",
    "outputId": "1fca5fb8-3a9b-4a78-f726-7aef8e14ee41"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10000/10000 [00:00<00:00, 24261.36it/s]\n"
     ]
    }
   ],
   "source": [
    "m = 10000\n",
    "dataset, human_vocab, machine_vocab, inv_machine_vocab = load_dataset(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 16972,
     "status": "ok",
     "timestamp": 1612468514156,
     "user": {
      "displayName": "Mubsi K",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gip7OjOkdNkKxKDyWEQAq1o8ccGN_HrBTGdqjgQ=s64",
      "userId": "08094225471505108399"
     },
     "user_tz": -300
    },
    "id": "zCTqMyPch3Mg",
    "outputId": "42c9d8aa-d07b-4618-ab8a-4db4e1b971e2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('9 may 1998', '1998-05-09'),\n",
       " ('10.11.19', '2019-11-10'),\n",
       " ('9/10/70', '1970-09-10'),\n",
       " ('saturday april 28 1990', '1990-04-28'),\n",
       " ('thursday january 26 1995', '1995-01-26'),\n",
       " ('monday march 7 1983', '1983-03-07'),\n",
       " ('sunday may 22 1988', '1988-05-22'),\n",
       " ('08 jul 2008', '2008-07-08'),\n",
       " ('8 sep 1999', '1999-09-08'),\n",
       " ('thursday january 1 1981', '1981-01-01')]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 16962,
     "status": "ok",
     "timestamp": 1612468514157,
     "user": {
      "displayName": "Mubsi K",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gip7OjOkdNkKxKDyWEQAq1o8ccGN_HrBTGdqjgQ=s64",
      "userId": "08094225471505108399"
     },
     "user_tz": -300
    },
    "id": "Qdso90EBh3Mg",
    "outputId": "0a364ad8-8b25-4de3-f036-d5d8e40bdf8c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X.shape: (10000, 30)\n",
      "Y.shape: (10000, 10)\n",
      "Xoh.shape: (10000, 30, 37)\n",
      "Yoh.shape: (10000, 10, 11)\n"
     ]
    }
   ],
   "source": [
    "Tx = 30\n",
    "Ty = 10\n",
    "X, Y, Xoh, Yoh = preprocess_data(dataset, human_vocab, machine_vocab, Tx, Ty)\n",
    "\n",
    "print(\"X.shape:\", X.shape)\n",
    "print(\"Y.shape:\", Y.shape)\n",
    "print(\"Xoh.shape:\", Xoh.shape)\n",
    "print(\"Yoh.shape:\", Yoh.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "executionInfo": {
     "elapsed": 16950,
     "status": "ok",
     "timestamp": 1612468514158,
     "user": {
      "displayName": "Mubsi K",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gip7OjOkdNkKxKDyWEQAq1o8ccGN_HrBTGdqjgQ=s64",
      "userId": "08094225471505108399"
     },
     "user_tz": -300
    },
    "id": "Cvop5Apyh3Mm"
   },
   "outputs": [],
   "source": [
    "# Defined shared layers as global variables\n",
    "repeator = RepeatVector(Tx)\n",
    "concatenator = Concatenate(axis=-1)\n",
    "densor1 = Dense(10, activation = \"tanh\")\n",
    "densor2 = Dense(1, activation = \"relu\")\n",
    "activator = Activation(softmax, name='attention_weights') # We are using a custom softmax(axis = 1) loaded in this notebook\n",
    "dotor = Dot(axes = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "executionInfo": {
     "elapsed": 16950,
     "status": "ok",
     "timestamp": 1612468514159,
     "user": {
      "displayName": "Mubsi K",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gip7OjOkdNkKxKDyWEQAq1o8ccGN_HrBTGdqjgQ=s64",
      "userId": "08094225471505108399"
     },
     "user_tz": -300
    },
    "id": "mZuMOnTDh3Mn"
   },
   "outputs": [],
   "source": [
    "def one_step_attention(a, s_prev):\n",
    "    \"\"\"\n",
    "    Performs one step of attention: Outputs a context vector computed as a dot product of the attention weights\n",
    "    \"alphas\" and the hidden states \"a\" of the Bi-LSTM.\n",
    "    \n",
    "    Arguments:\n",
    "    a -- hidden state output of the Bi-LSTM, numpy-array of shape (m, Tx, 2*n_a)\n",
    "    s_prev -- previous hidden state of the (post-attention) LSTM, numpy-array of shape (m, n_s)\n",
    "    \n",
    "    Returns:\n",
    "    context -- context vector, input of the next (post-attention) LSTM cell\n",
    "    \"\"\"\n",
    "    \n",
    "    ### START CODE HERE ###\n",
    "    # Use repeator to repeat s_prev to be of shape (m, Tx, n_s) so that you can concatenate it with all hidden states \"a\" (≈ 1 line)\n",
    "    s_prev = repeator(s_prev)\n",
    "    # Use concatenator to concatenate a and s_prev on the last axis (≈ 1 line)\n",
    "    # For grading purposes, please list 'a' first and 's_prev' second, in this order.\n",
    "    concat = concatenator([a,s_prev])\n",
    "    # Use densor1 to propagate concat through a small fully-connected neural network to compute the \"intermediate energies\" variable e. (≈1 lines)\n",
    "    e = densor1(concat)\n",
    "    # Use densor2 to propagate e through a small fully-connected neural network to compute the \"energies\" variable energies. (≈1 lines)\n",
    "    energies = densor2(e)\n",
    "    # Use \"activator\" on \"energies\" to compute the attention weights \"alphas\" (≈ 1 line)\n",
    "    alphas = activator(energies)\n",
    "    # Use dotor together with \"alphas\" and \"a\", in this order, to compute the context vector to be given to the next (post-attention) LSTM-cell (≈ 1 line)\n",
    "    context = dotor([alphas,a])\n",
    "    ### END CODE HERE ###\n",
    "    \n",
    "    return context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "executionInfo": {
     "elapsed": 16949,
     "status": "ok",
     "timestamp": 1612468514159,
     "user": {
      "displayName": "Mubsi K",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gip7OjOkdNkKxKDyWEQAq1o8ccGN_HrBTGdqjgQ=s64",
      "userId": "08094225471505108399"
     },
     "user_tz": -300
    },
    "id": "5RHgmZrVh3Mo"
   },
   "outputs": [],
   "source": [
    "n_a = 32 # number of units for the pre-attention, bi-directional LSTM's hidden state 'a'\n",
    "n_s = 64 # number of units for the post-attention LSTM's hidden state \"s\"\n",
    "\n",
    "# Please note, this is the post attention LSTM cell.  \n",
    "post_activation_LSTM_cell = LSTM(n_s, return_state = True) # Please do not modify this global variable.\n",
    "output_layer = Dense(len(machine_vocab), activation=softmax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "executionInfo": {
     "elapsed": 16948,
     "status": "ok",
     "timestamp": 1612468514160,
     "user": {
      "displayName": "Mubsi K",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gip7OjOkdNkKxKDyWEQAq1o8ccGN_HrBTGdqjgQ=s64",
      "userId": "08094225471505108399"
     },
     "user_tz": -300
    },
    "id": "qeKbeDOvh3Mo"
   },
   "outputs": [],
   "source": [
    "def modelf(Tx, Ty, n_a, n_s, human_vocab_size, machine_vocab_size):\n",
    "    \"\"\"\n",
    "    Arguments:\n",
    "    Tx -- length of the input sequence\n",
    "    Ty -- length of the output sequence\n",
    "    n_a -- hidden state size of the Bi-LSTM\n",
    "    n_s -- hidden state size of the post-attention LSTM\n",
    "    human_vocab_size -- size of the python dictionary \"human_vocab\"\n",
    "    machine_vocab_size -- size of the python dictionary \"machine_vocab\"\n",
    "\n",
    "    Returns:\n",
    "    model -- Keras model instance\n",
    "    \"\"\"\n",
    "    \n",
    "    # Define the inputs of your model with a shape (Tx,)\n",
    "    # Define s0 (initial hidden state) and c0 (initial cell state)\n",
    "    # for the decoder LSTM with shape (n_s,)\n",
    "    X = Input(shape=(Tx, human_vocab_size))\n",
    "    s0 = Input(shape=(n_s,), name='s0')\n",
    "    c0 = Input(shape=(n_s,), name='c0')\n",
    "    s = s0\n",
    "    c = c0\n",
    "    \n",
    "    # Initialize empty list of outputs\n",
    "    outputs = []\n",
    "    \n",
    "    ### START CODE HERE ###\n",
    "    \n",
    "    # Step 1: Define your pre-attention Bi-LSTM. (≈ 1 line)\n",
    "    a = Bidirectional(LSTM(units=n_a, return_sequences=True))(X)\n",
    "    \n",
    "    # Step 2: Iterate for Ty steps\n",
    "    for t in range(Ty):\n",
    "    \n",
    "        # Step 2.A: Perform one step of the attention mechanism to get back the context vector at step t (≈ 1 line)\n",
    "        context = one_step_attention(a, s)\n",
    "        \n",
    "        # Step 2.B: Apply the post-attention LSTM cell to the \"context\" vector.\n",
    "        # Don't forget to pass: initial_state = [hidden state, cell state] (≈ 1 line)\n",
    "        s, _, c = post_activation_LSTM_cell(context,initial_state =[s,c])\n",
    "        \n",
    "        # Step 2.C: Apply Dense layer to the hidden state output of the post-attention LSTM (≈ 1 line)\n",
    "        out = output_layer(s)\n",
    "        \n",
    "        # Step 2.D: Append \"out\" to the \"outputs\" list (≈ 1 line)\n",
    "        outputs.append(out)\n",
    "    \n",
    "    # Step 3: Create model instance taking three inputs and returning the list of outputs. (≈ 1 line)\n",
    "    model = Model(inputs=[X,s0,c0], outputs=outputs)\n",
    "    \n",
    "    ### END CODE HERE ###\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "executionInfo": {
     "elapsed": 20837,
     "status": "ok",
     "timestamp": 1612468518050,
     "user": {
      "displayName": "Mubsi K",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gip7OjOkdNkKxKDyWEQAq1o8ccGN_HrBTGdqjgQ=s64",
      "userId": "08094225471505108399"
     },
     "user_tz": -300
    },
    "id": "psdd-Ac6h3Mp"
   },
   "outputs": [],
   "source": [
    "model = modelf(Tx, Ty, n_a, n_s, len(human_vocab), len(machine_vocab))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VgeU_I9_h3Mp"
   },
   "source": [
    "Let's get a summary of the model to check if it matches the expected output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 20835,
     "status": "ok",
     "timestamp": 1612468518050,
     "user": {
      "displayName": "Mubsi K",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gip7OjOkdNkKxKDyWEQAq1o8ccGN_HrBTGdqjgQ=s64",
      "userId": "08094225471505108399"
     },
     "user_tz": -300
    },
    "id": "tX0vaYmPh3Mq",
    "outputId": "336b9248-70b0-4379-be95-95366874c02a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_3\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_8 (InputLayer)            [(None, 30, 37)]     0                                            \n",
      "__________________________________________________________________________________________________\n",
      "s0 (InputLayer)                 [(None, 64)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_7 (Bidirectional) (None, 30, 64)       17920       input_8[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "repeat_vector (RepeatVector)    (None, 30, 64)       0           s0[0][0]                         \n",
      "                                                                 lstm[10][0]                      \n",
      "                                                                 lstm[11][0]                      \n",
      "                                                                 lstm[12][0]                      \n",
      "                                                                 lstm[13][0]                      \n",
      "                                                                 lstm[14][0]                      \n",
      "                                                                 lstm[15][0]                      \n",
      "                                                                 lstm[16][0]                      \n",
      "                                                                 lstm[17][0]                      \n",
      "                                                                 lstm[18][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 30, 128)      0           bidirectional_7[0][0]            \n",
      "                                                                 repeat_vector[16][0]             \n",
      "                                                                 bidirectional_7[0][0]            \n",
      "                                                                 repeat_vector[17][0]             \n",
      "                                                                 bidirectional_7[0][0]            \n",
      "                                                                 repeat_vector[18][0]             \n",
      "                                                                 bidirectional_7[0][0]            \n",
      "                                                                 repeat_vector[19][0]             \n",
      "                                                                 bidirectional_7[0][0]            \n",
      "                                                                 repeat_vector[20][0]             \n",
      "                                                                 bidirectional_7[0][0]            \n",
      "                                                                 repeat_vector[21][0]             \n",
      "                                                                 bidirectional_7[0][0]            \n",
      "                                                                 repeat_vector[22][0]             \n",
      "                                                                 bidirectional_7[0][0]            \n",
      "                                                                 repeat_vector[23][0]             \n",
      "                                                                 bidirectional_7[0][0]            \n",
      "                                                                 repeat_vector[24][0]             \n",
      "                                                                 bidirectional_7[0][0]            \n",
      "                                                                 repeat_vector[25][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 30, 10)       1290        concatenate[16][0]               \n",
      "                                                                 concatenate[17][0]               \n",
      "                                                                 concatenate[18][0]               \n",
      "                                                                 concatenate[19][0]               \n",
      "                                                                 concatenate[20][0]               \n",
      "                                                                 concatenate[21][0]               \n",
      "                                                                 concatenate[22][0]               \n",
      "                                                                 concatenate[23][0]               \n",
      "                                                                 concatenate[24][0]               \n",
      "                                                                 concatenate[25][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 30, 1)        11          dense[16][0]                     \n",
      "                                                                 dense[17][0]                     \n",
      "                                                                 dense[18][0]                     \n",
      "                                                                 dense[19][0]                     \n",
      "                                                                 dense[20][0]                     \n",
      "                                                                 dense[21][0]                     \n",
      "                                                                 dense[22][0]                     \n",
      "                                                                 dense[23][0]                     \n",
      "                                                                 dense[24][0]                     \n",
      "                                                                 dense[25][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "attention_weights (Activation)  (None, 30, 1)        0           dense_1[16][0]                   \n",
      "                                                                 dense_1[17][0]                   \n",
      "                                                                 dense_1[18][0]                   \n",
      "                                                                 dense_1[19][0]                   \n",
      "                                                                 dense_1[20][0]                   \n",
      "                                                                 dense_1[21][0]                   \n",
      "                                                                 dense_1[22][0]                   \n",
      "                                                                 dense_1[23][0]                   \n",
      "                                                                 dense_1[24][0]                   \n",
      "                                                                 dense_1[25][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dot (Dot)                       (None, 1, 64)        0           attention_weights[16][0]         \n",
      "                                                                 bidirectional_7[0][0]            \n",
      "                                                                 attention_weights[17][0]         \n",
      "                                                                 bidirectional_7[0][0]            \n",
      "                                                                 attention_weights[18][0]         \n",
      "                                                                 bidirectional_7[0][0]            \n",
      "                                                                 attention_weights[19][0]         \n",
      "                                                                 bidirectional_7[0][0]            \n",
      "                                                                 attention_weights[20][0]         \n",
      "                                                                 bidirectional_7[0][0]            \n",
      "                                                                 attention_weights[21][0]         \n",
      "                                                                 bidirectional_7[0][0]            \n",
      "                                                                 attention_weights[22][0]         \n",
      "                                                                 bidirectional_7[0][0]            \n",
      "                                                                 attention_weights[23][0]         \n",
      "                                                                 bidirectional_7[0][0]            \n",
      "                                                                 attention_weights[24][0]         \n",
      "                                                                 bidirectional_7[0][0]            \n",
      "                                                                 attention_weights[25][0]         \n",
      "                                                                 bidirectional_7[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "c0 (InputLayer)                 [(None, 64)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lstm (LSTM)                     [(None, 64), (None,  33024       dot[16][0]                       \n",
      "                                                                 s0[0][0]                         \n",
      "                                                                 c0[0][0]                         \n",
      "                                                                 dot[17][0]                       \n",
      "                                                                 lstm[10][0]                      \n",
      "                                                                 lstm[10][2]                      \n",
      "                                                                 dot[18][0]                       \n",
      "                                                                 lstm[11][0]                      \n",
      "                                                                 lstm[11][2]                      \n",
      "                                                                 dot[19][0]                       \n",
      "                                                                 lstm[12][0]                      \n",
      "                                                                 lstm[12][2]                      \n",
      "                                                                 dot[20][0]                       \n",
      "                                                                 lstm[13][0]                      \n",
      "                                                                 lstm[13][2]                      \n",
      "                                                                 dot[21][0]                       \n",
      "                                                                 lstm[14][0]                      \n",
      "                                                                 lstm[14][2]                      \n",
      "                                                                 dot[22][0]                       \n",
      "                                                                 lstm[15][0]                      \n",
      "                                                                 lstm[15][2]                      \n",
      "                                                                 dot[23][0]                       \n",
      "                                                                 lstm[16][0]                      \n",
      "                                                                 lstm[16][2]                      \n",
      "                                                                 dot[24][0]                       \n",
      "                                                                 lstm[17][0]                      \n",
      "                                                                 lstm[17][2]                      \n",
      "                                                                 dot[25][0]                       \n",
      "                                                                 lstm[18][0]                      \n",
      "                                                                 lstm[18][2]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 11)           715         lstm[10][0]                      \n",
      "                                                                 lstm[11][0]                      \n",
      "                                                                 lstm[12][0]                      \n",
      "                                                                 lstm[13][0]                      \n",
      "                                                                 lstm[14][0]                      \n",
      "                                                                 lstm[15][0]                      \n",
      "                                                                 lstm[16][0]                      \n",
      "                                                                 lstm[17][0]                      \n",
      "                                                                 lstm[18][0]                      \n",
      "                                                                 lstm[19][0]                      \n",
      "==================================================================================================\n",
      "Total params: 52,960\n",
      "Trainable params: 52,960\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "executionInfo": {
     "elapsed": 20835,
     "status": "ok",
     "timestamp": 1612468518051,
     "user": {
      "displayName": "Mubsi K",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gip7OjOkdNkKxKDyWEQAq1o8ccGN_HrBTGdqjgQ=s64",
      "userId": "08094225471505108399"
     },
     "user_tz": -300
    },
    "id": "sBFRJ49rh3Ms"
   },
   "outputs": [],
   "source": [
    "### START CODE HERE ### (≈2 lines)\n",
    "opt = Adam(lr=0.005, beta_1=0.9, beta_2=0.999, decay=0.01)  \n",
    "model.compile(loss = 'categorical_crossentropy', optimizer = opt, metrics = ['accuracy'])\n",
    "### END CODE HERE ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "executionInfo": {
     "elapsed": 20833,
     "status": "ok",
     "timestamp": 1612468518051,
     "user": {
      "displayName": "Mubsi K",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gip7OjOkdNkKxKDyWEQAq1o8ccGN_HrBTGdqjgQ=s64",
      "userId": "08094225471505108399"
     },
     "user_tz": -300
    },
    "id": "USFiNKYhh3Mt"
   },
   "outputs": [],
   "source": [
    "s0 = np.zeros((m, n_s))\n",
    "c0 = np.zeros((m, n_s))\n",
    "outputs = list(Yoh.swapaxes(0,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 47944,
     "status": "ok",
     "timestamp": 1612468545172,
     "user": {
      "displayName": "Mubsi K",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gip7OjOkdNkKxKDyWEQAq1o8ccGN_HrBTGdqjgQ=s64",
      "userId": "08094225471505108399"
     },
     "user_tz": -300
    },
    "id": "tPuwY45bh3Mt",
    "outputId": "ec9dfc4c-1dcb-4577-d872-474f79c60d5f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100/100 [==============================] - 12s 120ms/step - loss: 16.7390 - dense_2_loss: 1.1682 - dense_2_1_loss: 1.0289 - dense_2_2_loss: 1.7609 - dense_2_3_loss: 2.6727 - dense_2_4_loss: 0.8051 - dense_2_5_loss: 1.2987 - dense_2_6_loss: 2.7369 - dense_2_7_loss: 0.9716 - dense_2_8_loss: 1.7380 - dense_2_9_loss: 2.5579 - dense_2_accuracy: 0.5530 - dense_2_1_accuracy: 0.7018 - dense_2_2_accuracy: 0.2826 - dense_2_3_accuracy: 0.0800 - dense_2_4_accuracy: 0.9739 - dense_2_5_accuracy: 0.3233 - dense_2_6_accuracy: 0.0395 - dense_2_7_accuracy: 0.9567 - dense_2_8_accuracy: 0.2242 - dense_2_9_accuracy: 0.0987\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f2c88233510>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit([Xoh, s0, c0], outputs, epochs=1, batch_size=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 53835,
     "status": "ok",
     "timestamp": 1612468551077,
     "user": {
      "displayName": "Mubsi K",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gip7OjOkdNkKxKDyWEQAq1o8ccGN_HrBTGdqjgQ=s64",
      "userId": "08094225471505108399"
     },
     "user_tz": -300
    },
    "id": "rQ8sd_cuh3Mv",
    "outputId": "c37e92ac-5c60-4caf-b843-6aaeaa37be25"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "source: 3 May 1979\n",
      "output: 1979-05-33 \n",
      "\n",
      "source: 5 April 09\n",
      "output: 2009-04-05 \n",
      "\n",
      "source: 21th of August 2016\n",
      "output: 2016-08-20 \n",
      "\n",
      "source: Tue 10 Jul 2007\n",
      "output: 2007-07-10 \n",
      "\n",
      "source: Saturday May 9 2018\n",
      "output: 2018-05-09 \n",
      "\n",
      "source: March 3 2001\n",
      "output: 2001-03-03 \n",
      "\n",
      "source: March 3rd 2001\n",
      "output: 2001-03-03 \n",
      "\n",
      "source: 1 March 2001\n",
      "output: 2001-03-01 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "EXAMPLES = ['3 May 1979', '5 April 09', '21th of August 2016', 'Tue 10 Jul 2007', 'Saturday May 9 2018', 'March 3 2001', 'March 3rd 2001', '1 March 2001']\n",
    "s00 = np.zeros((1, n_s))\n",
    "c00 = np.zeros((1, n_s))\n",
    "for example in EXAMPLES:\n",
    "    source = string_to_int(example, Tx, human_vocab)\n",
    "    #print(source)\n",
    "    source = np.array(list(map(lambda x: to_categorical(x, num_classes=len(human_vocab)), source))).swapaxes(0,1)\n",
    "    source = np.swapaxes(source, 0, 1)\n",
    "    source = np.expand_dims(source, axis=0)\n",
    "    prediction = model.predict([source, s00, c00])\n",
    "    prediction = np.argmax(prediction, axis = -1)\n",
    "    output = [inv_machine_vocab[int(i)] for i in prediction]\n",
    "    print(\"source:\", example)\n",
    "    print(\"output:\", ''.join(output),\"\\n\")"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Solution_Neural_machine_translation_with_attention_v4a.ipynb",
   "provenance": []
  },
  "coursera": {
   "schema_names": [
    "DLSC5W3-1A"
   ]
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}