e12_transformer_based_translation/Code/Hinglish to english translation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1725145f-dbb2-492b-86fd-0a567a8c03d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from keras_transformer import get_model, decode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2f7e79ec-e96d-494d-8af4-58e7588d3eab",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../Dataset/English_Hindi_Hinglish.txt', mode = 'r') as f:\n",
    "    data = f.readlines()\n",
    "\n",
    "data = data[0:195] # 195Because we have that many labeled data points for Hinglish to English translation.\n",
    "\n",
    "source_tokens0 = [i.split(',')[1].strip().split(' ') for i in data]\n",
    "target_tokens0 = [i.split('\t')[0].strip().split(' ') for i in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f153acec-9e87-4078-bef1-b9273e38fbe4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.read_csv('../Dataset/Moni.csv')\n",
    "df1.dropna(inplace = True)\n",
    "source_tokens1 = [i.split(' ') for i in df1['Hinglish'].values.tolist()]\n",
    "target_tokens1 = [i.split(' ') for i in df1['English'].values.tolist()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f7aa2aee-fa42-46d5-8382-3833fe99e1b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.read_csv('../Dataset/Ashish.csv')\n",
    "df2.dropna(inplace = True)\n",
    "source_tokens2 = [i.split(' ') for i in df2['Hinglish'].values.tolist()]\n",
    "target_tokens2 = [i.split(' ') for i in df2['English'].values.tolist()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "778914dc-0a68-4df5-9f27-0e275f528bca",
   "metadata": {},
   "outputs": [],
   "source": [
    "source_tokens = source_tokens0 + source_tokens1 + source_tokens2\n",
    "target_tokens = target_tokens0 + target_tokens1 + target_tokens2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f16a0fdf-fb5c-4b39-8656-32540b2e853b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ashish/anaconda3/envs/mt/lib/python3.10/site-packages/keras/initializers/initializers_v2.py:120: UserWarning: The initializer GlorotNormal is unseeded and being called multiple times, which will return identical values  each time (even if the initializer is unseeded). Please update your code to provide a seed to the initializer, or avoid using the same initalizer instance more than once.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " Encoder-Input (InputLayer)     [(None, None)]       0           []                               \n",
      "                                                                                                  \n",
      " Encoder-Token-Embedding (Embed  [(None, None, 32),  21408       ['Encoder-Input[0][0]']          \n",
      " dingRet)                        (669, 32)]                                                       \n",
      "                                                                                                  \n",
      " Encoder-Embedding (TrigPosEmbe  (None, None, 32)    0           ['Encoder-Token-Embedding[0][0]']\n",
      " dding)                                                                                           \n",
      "                                                                                                  \n",
      " Encoder-1-MultiHeadSelfAttenti  (None, None, 32)    4224        ['Encoder-Embedding[0][0]']      \n",
      " on (MultiHeadAttention)                                                                          \n",
      "                                                                                                  \n",
      " Encoder-1-MultiHeadSelfAttenti  (None, None, 32)    0           ['Encoder-1-MultiHeadSelfAttentio\n",
      " on-Dropout (Dropout)                                            n[0][0]']                        \n",
      "                                                                                                  \n",
      " Encoder-1-MultiHeadSelfAttenti  (None, None, 32)    0           ['Encoder-Embedding[0][0]',      \n",
      " on-Add (Add)                                                     'Encoder-1-MultiHeadSelfAttentio\n",
      "                                                                 n-Dropout[0][0]']                \n",
      "                                                                                                  \n",
      " Encoder-1-MultiHeadSelfAttenti  (None, None, 32)    64          ['Encoder-1-MultiHeadSelfAttentio\n",
      " on-Norm (LayerNormalization)                                    n-Add[0][0]']                    \n",
      "                                                                                                  \n",
      " Encoder-1-FeedForward (FeedFor  (None, None, 32)    8352        ['Encoder-1-MultiHeadSelfAttentio\n",
      " ward)                                                           n-Norm[0][0]']                   \n",
      "                                                                                                  \n",
      " Encoder-1-FeedForward-Dropout   (None, None, 32)    0           ['Encoder-1-FeedForward[0][0]']  \n",
      " (Dropout)                                                                                        \n",
      "                                                                                                  \n",
      " Encoder-1-FeedForward-Add (Add  (None, None, 32)    0           ['Encoder-1-MultiHeadSelfAttentio\n",
      " )                                                               n-Norm[0][0]',                   \n",
      "                                                                  'Encoder-1-FeedForward-Dropout[0\n",
      "                                                                 ][0]']                           \n",
      "                                                                                                  \n",
      " Encoder-1-FeedForward-Norm (La  (None, None, 32)    64          ['Encoder-1-FeedForward-Add[0][0]\n",
      " yerNormalization)                                               ']                               \n",
      "                                                                                                  \n",
      " Encoder-2-MultiHeadSelfAttenti  (None, None, 32)    4224        ['Encoder-1-FeedForward-Norm[0][0\n",
      " on (MultiHeadAttention)                                         ]']                              \n",
      "                                                                                                  \n",
      " Decoder-Input (InputLayer)     [(None, None)]       0           []                               \n",
      "                                                                                                  \n",
      " Encoder-2-MultiHeadSelfAttenti  (None, None, 32)    0           ['Encoder-2-MultiHeadSelfAttentio\n",
      " on-Dropout (Dropout)                                            n[0][0]']                        \n",
      "                                                                                                  \n",
      " Decoder-Token-Embedding (Embed  [(None, None, 32),  21408       ['Decoder-Input[0][0]']          \n",
      " dingRet)                        (669, 32)]                                                       \n",
      "                                                                                                  \n",
      " Encoder-2-MultiHeadSelfAttenti  (None, None, 32)    0           ['Encoder-1-FeedForward-Norm[0][0\n",
      " on-Add (Add)                                                    ]',                              \n",
      "                                                                  'Encoder-2-MultiHeadSelfAttentio\n",
      "                                                                 n-Dropout[0][0]']                \n",
      "                                                                                                  \n",
      " Decoder-Embedding (TrigPosEmbe  (None, None, 32)    0           ['Decoder-Token-Embedding[0][0]']\n",
      " dding)                                                                                           \n",
      "                                                                                                  \n",
      " Encoder-2-MultiHeadSelfAttenti  (None, None, 32)    64          ['Encoder-2-MultiHeadSelfAttentio\n",
      " on-Norm (LayerNormalization)                                    n-Add[0][0]']                    \n",
      "                                                                                                  \n",
      " Decoder-1-MultiHeadSelfAttenti  (None, None, 32)    4224        ['Decoder-Embedding[0][0]']      \n",
      " on (MultiHeadAttention)                                                                          \n",
      "                                                                                                  \n",
      " Encoder-2-FeedForward (FeedFor  (None, None, 32)    8352        ['Encoder-2-MultiHeadSelfAttentio\n",
      " ward)                                                           n-Norm[0][0]']                   \n",
      "                                                                                                  \n",
      " Decoder-1-MultiHeadSelfAttenti  (None, None, 32)    0           ['Decoder-1-MultiHeadSelfAttentio\n",
      " on-Dropout (Dropout)                                            n[0][0]']                        \n",
      "                                                                                                  \n",
      " Encoder-2-FeedForward-Dropout   (None, None, 32)    0           ['Encoder-2-FeedForward[0][0]']  \n",
      " (Dropout)                                                                                        \n",
      "                                                                                                  \n",
      " Decoder-1-MultiHeadSelfAttenti  (None, None, 32)    0           ['Decoder-Embedding[0][0]',      \n",
      " on-Add (Add)                                                     'Decoder-1-MultiHeadSelfAttentio\n",
      "                                                                 n-Dropout[0][0]']                \n",
      "                                                                                                  \n",
      " Encoder-2-FeedForward-Add (Add  (None, None, 32)    0           ['Encoder-2-MultiHeadSelfAttentio\n",
      " )                                                               n-Norm[0][0]',                   \n",
      "                                                                  'Encoder-2-FeedForward-Dropout[0\n",
      "                                                                 ][0]']                           \n",
      "                                                                                                  \n",
      " Decoder-1-MultiHeadSelfAttenti  (None, None, 32)    64          ['Decoder-1-MultiHeadSelfAttentio\n",
      " on-Norm (LayerNormalization)                                    n-Add[0][0]']                    \n",
      "                                                                                                  \n",
      " Encoder-2-FeedForward-Norm (La  (None, None, 32)    64          ['Encoder-2-FeedForward-Add[0][0]\n",
      " yerNormalization)                                               ']                               \n",
      "                                                                                                  \n",
      " Decoder-1-MultiHeadQueryAttent  (None, None, 32)    4224        ['Decoder-1-MultiHeadSelfAttentio\n",
      " ion (MultiHeadAttention)                                        n-Norm[0][0]',                   \n",
      "                                                                  'Encoder-2-FeedForward-Norm[0][0\n",
      "                                                                 ]',                              \n",
      "                                                                  'Encoder-2-FeedForward-Norm[0][0\n",
      "                                                                 ]']                              \n",
      "                                                                                                  \n",
      " Decoder-1-MultiHeadQueryAttent  (None, None, 32)    0           ['Decoder-1-MultiHeadQueryAttenti\n",
      " ion-Dropout (Dropout)                                           on[0][0]']                       \n",
      "                                                                                                  \n",
      " Decoder-1-MultiHeadQueryAttent  (None, None, 32)    0           ['Decoder-1-MultiHeadSelfAttentio\n",
      " ion-Add (Add)                                                   n-Norm[0][0]',                   \n",
      "                                                                  'Decoder-1-MultiHeadQueryAttenti\n",
      "                                                                 on-Dropout[0][0]']               \n",
      "                                                                                                  \n",
      " Decoder-1-MultiHeadQueryAttent  (None, None, 32)    64          ['Decoder-1-MultiHeadQueryAttenti\n",
      " ion-Norm (LayerNormalization)                                   on-Add[0][0]']                   \n",
      "                                                                                                  \n",
      " Decoder-1-FeedForward (FeedFor  (None, None, 32)    8352        ['Decoder-1-MultiHeadQueryAttenti\n",
      " ward)                                                           on-Norm[0][0]']                  \n",
      "                                                                                                  \n",
      " Decoder-1-FeedForward-Dropout   (None, None, 32)    0           ['Decoder-1-FeedForward[0][0]']  \n",
      " (Dropout)                                                                                        \n",
      "                                                                                                  \n",
      " Decoder-1-FeedForward-Add (Add  (None, None, 32)    0           ['Decoder-1-MultiHeadQueryAttenti\n",
      " )                                                               on-Norm[0][0]',                  \n",
      "                                                                  'Decoder-1-FeedForward-Dropout[0\n",
      "                                                                 ][0]']                           \n",
      "                                                                                                  \n",
      " Decoder-1-FeedForward-Norm (La  (None, None, 32)    64          ['Decoder-1-FeedForward-Add[0][0]\n",
      " yerNormalization)                                               ']                               \n",
      "                                                                                                  \n",
      " Decoder-2-MultiHeadSelfAttenti  (None, None, 32)    4224        ['Decoder-1-FeedForward-Norm[0][0\n",
      " on (MultiHeadAttention)                                         ]']                              \n",
      "                                                                                                  \n",
      " Decoder-2-MultiHeadSelfAttenti  (None, None, 32)    0           ['Decoder-2-MultiHeadSelfAttentio\n",
      " on-Dropout (Dropout)                                            n[0][0]']                        \n",
      "                                                                                                  \n",
      " Decoder-2-MultiHeadSelfAttenti  (None, None, 32)    0           ['Decoder-1-FeedForward-Norm[0][0\n",
      " on-Add (Add)                                                    ]',                              \n",
      "                                                                  'Decoder-2-MultiHeadSelfAttentio\n",
      "                                                                 n-Dropout[0][0]']                \n",
      "                                                                                                  \n",
      " Decoder-2-MultiHeadSelfAttenti  (None, None, 32)    64          ['Decoder-2-MultiHeadSelfAttentio\n",
      " on-Norm (LayerNormalization)                                    n-Add[0][0]']                    \n",
      "                                                                                                  \n",
      " Decoder-2-MultiHeadQueryAttent  (None, None, 32)    4224        ['Decoder-2-MultiHeadSelfAttentio\n",
      " ion (MultiHeadAttention)                                        n-Norm[0][0]',                   \n",
      "                                                                  'Encoder-2-FeedForward-Norm[0][0\n",
      "                                                                 ]',                              \n",
      "                                                                  'Encoder-2-FeedForward-Norm[0][0\n",
      "                                                                 ]']                              \n",
      "                                                                                                  \n",
      " Decoder-2-MultiHeadQueryAttent  (None, None, 32)    0           ['Decoder-2-MultiHeadQueryAttenti\n",
      " ion-Dropout (Dropout)                                           on[0][0]']                       \n",
      "                                                                                                  \n",
      " Decoder-2-MultiHeadQueryAttent  (None, None, 32)    0           ['Decoder-2-MultiHeadSelfAttentio\n",
      " ion-Add (Add)                                                   n-Norm[0][0]',                   \n",
      "                                                                  'Decoder-2-MultiHeadQueryAttenti\n",
      "                                                                 on-Dropout[0][0]']               \n",
      "                                                                                                  \n",
      " Decoder-2-MultiHeadQueryAttent  (None, None, 32)    64          ['Decoder-2-MultiHeadQueryAttenti\n",
      " ion-Norm (LayerNormalization)                                   on-Add[0][0]']                   \n",
      "                                                                                                  \n",
      " Decoder-2-FeedForward (FeedFor  (None, None, 32)    8352        ['Decoder-2-MultiHeadQueryAttenti\n",
      " ward)                                                           on-Norm[0][0]']                  \n",
      "                                                                                                  \n",
      " Decoder-2-FeedForward-Dropout   (None, None, 32)    0           ['Decoder-2-FeedForward[0][0]']  \n",
      " (Dropout)                                                                                        \n",
      "                                                                                                  \n",
      " Decoder-2-FeedForward-Add (Add  (None, None, 32)    0           ['Decoder-2-MultiHeadQueryAttenti\n",
      " )                                                               on-Norm[0][0]',                  \n",
      "                                                                  'Decoder-2-FeedForward-Dropout[0\n",
      "                                                                 ][0]']                           \n",
      "                                                                                                  \n",
      " Decoder-2-FeedForward-Norm (La  (None, None, 32)    64          ['Decoder-2-FeedForward-Add[0][0]\n",
      " yerNormalization)                                               ']                               \n",
      "                                                                                                  \n",
      " Decoder-Output (EmbeddingSim)  (None, None, 669)    669         ['Decoder-2-FeedForward-Norm[0][0\n",
      "                                                                 ]',                              \n",
      "                                                                  'Decoder-Token-Embedding[0][1]']\n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 102,877\n",
      "Trainable params: 102,877\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Generate dictionaries\n",
    "def build_token_dict(token_list):\n",
    "    token_dict = {\n",
    "        '<PAD>': 0,\n",
    "        '<START>': 1,\n",
    "        '<END>': 2,\n",
    "    }\n",
    "    for tokens in token_list:\n",
    "        for token in tokens:\n",
    "            if token not in token_dict:\n",
    "                token_dict[token] = len(token_dict)\n",
    "    return token_dict\n",
    "\n",
    "source_token_dict = build_token_dict(source_tokens)\n",
    "target_token_dict = build_token_dict(target_tokens)\n",
    "target_token_dict_inv = {v: k for k, v in target_token_dict.items()}\n",
    "\n",
    "# Add special tokens\n",
    "encode_tokens = [['<START>'] + tokens + ['<END>'] for tokens in source_tokens]\n",
    "decode_tokens = [['<START>'] + tokens + ['<END>'] for tokens in target_tokens]\n",
    "output_tokens = [tokens + ['<END>', '<PAD>'] for tokens in target_tokens]\n",
    "\n",
    "# Padding\n",
    "source_max_len = max(map(len, encode_tokens))\n",
    "target_max_len = max(map(len, decode_tokens))\n",
    "\n",
    "encode_tokens = [tokens + ['<PAD>'] * (source_max_len - len(tokens)) for tokens in encode_tokens]\n",
    "decode_tokens = [tokens + ['<PAD>'] * (target_max_len - len(tokens)) for tokens in decode_tokens]\n",
    "output_tokens = [tokens + ['<PAD>'] * (target_max_len - len(tokens)) for tokens in output_tokens]\n",
    "\n",
    "encode_input = [list(map(lambda x: source_token_dict[x], tokens)) for tokens in encode_tokens]\n",
    "decode_input = [list(map(lambda x: target_token_dict[x], tokens)) for tokens in decode_tokens]\n",
    "decode_output = [list(map(lambda x: [target_token_dict[x]], tokens)) for tokens in output_tokens]\n",
    "\n",
    "# Build & fit model\n",
    "model = get_model(\n",
    "    token_num=max(len(source_token_dict), len(target_token_dict)),\n",
    "    embed_dim=32,\n",
    "    encoder_num=2,\n",
    "    decoder_num=2,\n",
    "    head_num=4,\n",
    "    hidden_dim=128,\n",
    "    dropout_rate=0.05,\n",
    "    use_same_embed=False,  # Use different embeddings for different languages\n",
    ")\n",
    "model.compile('adam', 'sparse_categorical_crossentropy')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6298a5e4-9e8c-4513-bc3e-149b7c1a7ef3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "13728/13728 [==============================] - 268s 19ms/step - loss: 0.1462\n",
      "Epoch 2/5\n",
      "13728/13728 [==============================] - 278s 20ms/step - loss: 0.0071\n",
      "Epoch 3/5\n",
      "13728/13728 [==============================] - 261s 19ms/step - loss: 0.0066\n",
      "Epoch 4/5\n",
      "13728/13728 [==============================] - 264s 19ms/step - loss: 0.0062\n",
      "Epoch 5/5\n",
      "13728/13728 [==============================] - 264s 19ms/step - loss: 0.0060\n",
      "CPU times: user 50min 9s, sys: 3min 38s, total: 53min 47s\n",
      "Wall time: 22min 16s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f9bcafc5090>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "model.fit(\n",
    "    x=[np.array(encode_input * 1024), np.array(decode_input * 1024)],\n",
    "    y=np.array(decode_output * 1024),\n",
    "    epochs=5,\n",
    "    batch_size=32,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "932a17cf-eb43-4961-9fbf-c75d6c2d668e",
   "metadata": {},
   "source": [
    "# Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f1e46380",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14/14 [==============================] - 2s 7ms/step\n",
      "14/14 [==============================] - 2s 7ms/step\n",
      "13/13 [==============================] - 0s 7ms/step\n",
      "12/12 [==============================] - 0s 8ms/step\n",
      "7/7 [==============================] - 0s 8ms/step\n",
      "1/1 [==============================] - 0s 23ms/step\n",
      "Wow!\n",
      "Help!\n",
      "Jump.\n",
      "Jump.\n",
      "Jump.\n",
      "Hello!\n",
      "Hello!\n",
      "Cheers!\n",
      "Cheers!\n",
      "Got it?\n",
      "I'm OK.\n",
      "Awesome!\n",
      "Come in.\n",
      "Get out!\n",
      "Go away!\n",
      "Goodbye!\n",
      "Perfect!\n",
      "Perfect!\n",
      "Welcome.\n",
      "Welcome.\n",
      "Have fun.\n",
      "Have fun.\n",
      "Have fun.\n",
      "I forgot.\n",
      "I forgot.\n",
      "I'll pay.\n",
      "I'm fine.\n",
      "I'm full.\n",
      "Let's go!\n",
      "Answer me.\n",
      "Birds fly.\n",
      "Excuse me.\n",
      "Fantastic!\n",
      "I fainted.\n",
      "I fear so.\n",
      "I laughed.\n",
      "I'm bored.\n",
      "I'm broke.\n",
      "I'm tired.\n",
      "It's cold.\n",
      "Who knows?\n",
      "Who knows?\n",
      "Who knows?\n",
      "Who knows?\n",
      "Wonderful!\n",
      "Birds sing.\n",
      "Come on in.\n",
      "Definitely!\n",
      "Don't move.\n",
      "Fire burns.\n",
      "Follow him.\n",
      "I'm tired.\n",
      "I can swim.\n",
      "I can swim.\n",
      "I love you.\n",
      "I love you.\n",
      "I love you.\n",
      "I love you.\n",
      "I love you.\n",
      "I will try.\n",
      "I'm coming.\n",
      "I'm hungry!\n",
      "Let him in.\n",
      "Let him in.\n",
      "Let me out!\n",
      "Once again.\n",
      "Please sit.\n",
      "What's new?\n",
      "What's new?\n",
      "Who's that?\n",
      "Don't shout.\n",
      "Don't shout.\n",
      "He stood up.\n",
      "He's strong.\n",
      "How are you?\n",
      "How are you?\n",
      "How are you?\n",
      "How are you?\n",
      "How are you?\n",
      "How are you?\n",
      "How are you?\n",
      "How are you?\n",
      "I like both.\n",
      "I like cake.\n",
      "I like dogs.\n",
      "I like math.\n",
      "I'll attend.\n",
      "Nobody came.\n",
      "Was I wrong?\n",
      "What is this?\n",
      "Are you sick?\n",
      "Bring him in.\n",
      "Come with us.\n",
      "Happy Easter!\n",
      "Has Tom left?\n",
      "He is French.\n",
      "I am at home.\n",
      "I can't move.\n",
      "I don't know.\n",
      "I don't know.\n",
      "I have a car.\n",
      "I have a dog.\n",
      "I understand.\n",
      "I'm a doctor.\n",
      "I'm starving!\n",
      "It is a book.\n",
      "It's snowing.\n",
      "It's too big.\n",
      "Please leave.\n",
      "Unbelievable!\n",
      "We are happy.\n",
      "What is this?\n",
      "Are you tired?\n",
      "Can you drive?\n",
      "Don't get fat.\n",
      "Don't give in.\n",
      "Drink it down.\n",
      "Everyone dies.\n",
      "Flowers bloom.\n",
      "I am who I am.\n",
      "I'll take him.\n",
      "I'm tired now.\n",
      "I'm very busy.\n",
      "Is that a cat?\n",
      "It's for free.\n",
      "It's for free.\n",
      "Let me try it.\n",
      "Let me try it.\n",
      "Let me try it.\n",
      "Make it quick.\n",
      "May I come in?\n",
      "Open the door.\n",
      "Open the door.\n",
      "Please get in.\n",
      "Read it again.\n",
      "Read it aloud.\n",
      "She bent down.\n",
      "Some fish fly.\n",
      "This is a map.\n",
      "Tom is my son.\n",
      "We're in town.\n",
      "Were you shot?\n",
      "What about us?\n",
      "Can I help you?\n",
      "Can I help you?\n",
      "Clean the room.\n",
      "Don't touch it.\n",
      "Get out of bed!\n",
      "Happy New Year!\n",
      "Happy New Year!\n",
      "Happy birthday!\n",
      "He has a beard.\n",
      "He is an actor.\n",
      "He needs money.\n",
      "He was hard up.\n",
      "I like history.\n",
      "I like the dog.\n",
      "I must buy one.\n",
      "I'll come back.\n",
      "I'll phone you.\n",
      "I'll stay home.\n",
      "I'm an atheist.\n",
      "I'm an atheist.\n",
      "I'm very tired.\n",
      "It's hot today.\n",
      "This is a book.\n",
      "It's your move.\n",
      "Only God knows.\n",
      "Summer is over.\n",
      "Take your time.\n",
      "Think about it.\n",
      "This is a book.\n",
      "This is a book.\n",
      "This is my bag.\n",
      "This is my dog.\n",
      "Wash your feet.\n",
      "We study music.\n",
      "We'll call you.\n",
      "We'll call you.\n",
      "Where were you?\n",
      "Where were you?\n",
      "You have to go.\n",
      "You look tired.\n",
      "You're kidding!\n",
      "A man must work.\n",
      "Are you at home?\n",
      "Are you guys OK?\n",
      "Come home early.\n",
      "Come if you can.\n",
      "Congratulations!\n",
      "Congratulations!\n",
      "Did he go there?\n",
      "Did you miss me?\n",
      "Don't be absurd.\n",
      "He came running.\n",
      "He can't be ill.\n",
      "He has a hat on.\n",
      "He reads Arabic.\n",
      "He's my husband.\n",
      "How is everyone?\n",
      "How is it going?\n",
      "How old are you?\n",
      "How rude of you!\n",
      "How rude of you!\n",
      "I cooked dinner.\n",
      "I feel nauseous.\n",
      "I go every year.\n",
      "I heard a noise.\n",
      "I like this one.\n",
      "I run every day.\n",
      "I saw her again.\n",
      "I want a guitar.\n",
      "I was assaulted.\n",
      "I'm a good cook.\n",
      "I'm a good cook.\n",
      "I'm lucky today.\n",
      "Is anybody here?\n",
      "Is it not black?\n",
      "Is there a cafe?\n",
      "Let's fly kites.\n",
      "No, I didn't go.\n",
      "Nobody's around.\n",
      "Now stop crying.\n",
      "Open your mouth!\n",
      "Sit down, please.\n",
      "She is a beauty.\n",
      "She talks a lot.\n",
      "She talks a lot.\n",
      "That isn't fair.\n",
      "That's my fault.\n",
      "The dog is mine.\n",
      "The power's out.\n",
      "These are birds.\n",
      "This is for you.\n",
      "We lack nothing.\n",
      "We study Arabic.\n",
      "What time is it?\n",
      "What time is it?\n",
      "Where's my book?\n",
      "You startled me!\n",
      "You're a person.\n",
      "Are you a doctor?\n",
      "Can you teach me?\n",
      "Climb to the top.\n",
      "Come and join us.\n",
      "Come and join us.\n",
      "Cut the potatoes.\n",
      "Do you have rice?\n",
      "Don't make faces.\n",
      "Don't make noise.\n",
      "Have a good time.\n",
      "Have a good time.\n",
      "He has long legs.\n",
      "He is her friend.\n",
      "He likes oranges.\n",
      "He likes oranges.\n",
      "He must love you.\n",
      "He speaks Arabic.\n",
      "Here is your bag.\n",
      "I can't remember.\n",
      "I don't know him.\n",
      "I have a problem.\n",
      "I have to go now.\n",
      "I live near here.\n",
      "I love my mother.\n",
      "I must leave now.\n",
      "I'm really tired.\n",
      "I'm very thirsty.\n",
      "It's April first.\n",
      "It's now my turn.\n",
      "Let me try again.\n",
      "Nice to meet you.\n",
      "Please follow me.\n",
      "Please sign here.\n",
      "She betrayed you.\n",
      "Sit down, please.\n",
      "The car is ready.\n",
      "This is your dog.\n",
      "This is your key.\n",
      "This isn't right.\n",
      "Tom is my friend.\n",
      "We're going home.\n",
      "We're in a hurry.\n",
      "Welcome to Japan.\n",
      "What is your age?\n",
      "What's that bird?\n",
      "What's the story?\n",
      "What is your name?\n",
      "Where did you go?\n",
      "Where is the bar?\n",
      "Whose turn is it?\n",
      "You may be right.\n",
      "Are your eyes bad?\n",
      "Birds build nests.\n",
      "Birds build nests.\n",
      "Boys will be boys.\n",
      "Can I do anything?\n",
      "Didn't you go out?\n",
      "Didn't you go out?\n",
      "Didn't you go out?\n",
      "Do you believe me?\n",
      "Do you believe me?\n",
      "Do you believe me?\n",
      "Don't deceive him.\n",
      "He began to shout.\n",
      "He came to see me.\n",
      "He employs a maid.\n",
      "He is after a job.\n",
      "He is walking now.\n",
      "He studied abroad.\n",
      "He turned traitor.\n",
      "He's already left.\n",
      "I can drive a car.\n",
      "I cried all night.\n",
      "I got very sleepy.\n",
      "I have a headache.\n",
      "I have a headache.\n",
      "I have some money.\n",
      "I haven't met him.\n",
      "I listen to music.\n",
      "I saw him running.\n",
      "I saw him running.\n",
      "I truly loved her.\n",
      "I work for a bank.\n",
      "I'm getting happy.\n",
      "I'm not a student.\n",
      "I'm not like that.\n",
      "I'm really hungry.\n",
      "It's not my fault.\n",
      "It's starting now.\n",
      "My bag was stolen.\n",
      "My eyes are tired.\n",
      "Nobody's around.\n",
      "Put your hands up!\n",
      "She began to sing.\n",
      "She has blue eyes.\n",
      "She isn't married.\n",
      "Tell me the truth.\n",
      "The earth rotates.\n",
      "Try it once again.\n",
      "Try it once again.\n",
      "Turn on the radio.\n",
      "We don't know him.\n",
      "We don't know him.\n",
      "We have good news.\n",
      "What is happiness?\n",
      "What is happiness?\n",
      "What is your name?\n",
      "What's the matter?\n",
      "Where do you live?\n",
      "Where do you live?\n",
      "Where do you live?\n",
      "Where do you live?\n",
      "Where do you live?\n",
      "Where is the book?\n",
      "Where's the beach?\n",
      "Who made this pie?\n",
      "Yes. That's right.\n",
      "You have to leave.\n",
      "Are you busy today?\n",
      "Are you busy today?\n",
      "Can we have a talk?\n",
      "Come along with us.\n",
      "Did you sleep well?\n",
      "Do you remember me?\n",
      "Don't give up hope.\n",
      "Don't throw stones.\n",
      "Everyone likes him.\n",
      "Everyone likes him.\n",
      "Have you gone nuts?\n",
      "He breathed deeply.\n",
      "He closed his eyes.\n",
      "He cried and cried.\n",
      "He is a simple man.\n",
      "He is sure to come.\n",
      "He isn't my cousin.\n",
      "He isn't my cousin.\n",
      "He waited his turn.\n",
      "He works at a bank.\n",
      "He's a good person.\n",
      "I admit my mistake.\n",
      "I can read English.\n",
      "I can read English.\n",
      "I can't believe it!\n",
      "I heard him go out.\n",
      "I know her address.\n",
      "I know his address.\n",
      "I know those women.\n",
      "I know your father.\n",
      "I sort of like him.\n",
      "I still don't know.\n",
      "I was born in 1960.\n",
      "I was born in 1979.\n",
      "I watch television.\n",
      "I worked all night.\n",
      "I worked all night.\n",
      "I'll see you later.\n",
      "I'll see you later.\n",
      "I'm as tall as you.\n",
      "I'm coming at once.\n",
      "Is this book yours?\n",
      "It is already dark.\n",
      "It looks like snow.\n",
      "It started to snow.\n",
      "It was sort of fun.\n",
      "It's mine, not his.\n",
      "It's mine, not his.\n",
      "It's too expensive!\n",
      "It's too expensive!\n",
      "It's very hot here.\n",
      "Let me handle this.\n",
      "Let's take a train.\n",
      "My car is a Toyota.\n",
      "My family is small.\n",
      "No one can help me.\n",
      "She is very pretty.\n",
      "She leaped for joy.\n",
      "She smiled happily.\n",
      "She's not a doctor.\n",
      "That guy annoys me.\n",
      "The batter was out.\n",
      "The crow flew away.\n",
      "The power went out.\n",
      "These dogs are big.\n",
      "This is Mary's dog.\n",
      "Was his story true?\n",
      "How are you?\n",
      "Do you love me?\n",
      "Do you love me?\n"
     ]
    }
   ],
   "source": [
    "decoded = decode(\n",
    "    model,\n",
    "    encode_input,\n",
    "    start_token=target_token_dict['<START>'],\n",
    "    end_token=target_token_dict['<END>'],\n",
    "    pad_token=target_token_dict['<PAD>'],\n",
    ")\n",
    "for i in decoded:\n",
    "    pass\n",
    "    #print(' '.join(map(lambda x: target_token_dict_inv[x], i[1:-1])))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "115972a8-4f67-4aba-a726-7edb51e1c044",
   "metadata": {},
   "source": [
    "# Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "9890f24c-1f85-4edb-b367-61108409fb90",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sents = [\n",
    "    'kaise ho?',\n",
    "    'kya tum mujhse pyar karte ho?',\n",
    "    'kya tum mujhe pyar karte ho?'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c8dc7e12-c0bd-4aee-b665-dfedc2e9abfd",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_tokens = [i.split() for i in test_sents]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "10df85f1-1b12-4261-921b-2d942b169b7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_token_dict = build_token_dict(test_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a78b347a-7fe8-4926-bccc-25eb45bfd8a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_token_dict_inv = {v: k for k, v in test_token_dict.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "24d65795-01db-4798-8b2b-6d42a4c32cd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_enc_tokens = [['<START>'] + tokens + ['<END>'] for tokens in test_tokens]\n",
    "test_enc_tokens = [tokens + ['<PAD>'] * (target_max_len - len(tokens)) for tokens in test_enc_tokens]\n",
    "test_input = [list(map(lambda x: test_token_dict[x], tokens)) for tokens in test_enc_tokens]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "21be3596-52cd-4761-b2a9-595f54cb0df4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 2s 2s/step\n",
      "1/1 [==============================] - 0s 22ms/step\n",
      "1/1 [==============================] - 0s 25ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "Wow!\n",
      "I believe you?\n",
      "I'm really tired.\n"
     ]
    }
   ],
   "source": [
    "decoded = decode(\n",
    "    model,\n",
    "    test_input,\n",
    "    start_token=test_token_dict['<START>'],\n",
    "    end_token=test_token_dict['<END>'],\n",
    "    pad_token=test_token_dict['<PAD>'],\n",
    ")\n",
    "for i in decoded:\n",
    "    print(' '.join(map(lambda x: target_token_dict_inv[x], i[1:-1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bd4e4ca-58f8-4715-902b-d872b59aa6dc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1982e90c-3e54-4623-9595-c8a65639029a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mt",
   "language": "python",
   "name": "mt"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}