Sequence Models/Dinosaurus_Island_Character_level_language_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "r0Nj4psY01SJ"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from utils import *\n",
    "import random\n",
    "import pprint\n",
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qB2XWVg_01SO"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 19909 total characters and 27 unique characters in your data.\n"
     ]
    }
   ],
   "source": [
    "data = open('dinos.txt', 'r').read()\n",
    "data= data.lower()\n",
    "chars = list(set(data))\n",
    "data_size, vocab_size = len(data), len(chars)\n",
    "print('There are %d total characters and %d unique characters in your data.' % (data_size, vocab_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bh3QcYpr01SQ"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\\n', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']\n"
     ]
    }
   ],
   "source": [
    "chars = sorted(chars)\n",
    "print(chars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2YltsxeZ01SU"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{   0: '\\n',\n",
      "    1: 'a',\n",
      "    2: 'b',\n",
      "    3: 'c',\n",
      "    4: 'd',\n",
      "    5: 'e',\n",
      "    6: 'f',\n",
      "    7: 'g',\n",
      "    8: 'h',\n",
      "    9: 'i',\n",
      "    10: 'j',\n",
      "    11: 'k',\n",
      "    12: 'l',\n",
      "    13: 'm',\n",
      "    14: 'n',\n",
      "    15: 'o',\n",
      "    16: 'p',\n",
      "    17: 'q',\n",
      "    18: 'r',\n",
      "    19: 's',\n",
      "    20: 't',\n",
      "    21: 'u',\n",
      "    22: 'v',\n",
      "    23: 'w',\n",
      "    24: 'x',\n",
      "    25: 'y',\n",
      "    26: 'z'}\n"
     ]
    }
   ],
   "source": [
    "char_to_ix = { ch:i for i,ch in enumerate(chars) }\n",
    "ix_to_char = { i:ch for i,ch in enumerate(chars) }\n",
    "pp = pprint.PrettyPrinter(indent=4)\n",
    "pp.pprint(ix_to_char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2yYvYeI501SX"
   },
   "outputs": [],
   "source": [
    "def clip(gradients, maxValue):\n",
    "    '''\n",
    "    Clips the gradients' values between minimum and maximum.\n",
    "    \n",
    "    Arguments:\n",
    "    gradients -- a dictionary containing the gradients \"dWaa\", \"dWax\", \"dWya\", \"db\", \"dby\"\n",
    "    maxValue -- everything above this number is set to this number, and everything less than -maxValue is set to -maxValue\n",
    "    \n",
    "    Returns: \n",
    "    gradients -- a dictionary with the clipped gradients.\n",
    "    '''\n",
    "    gradients = copy.deepcopy(gradients)\n",
    "    \n",
    "    dWaa, dWax, dWya, db, dby = gradients['dWaa'], gradients['dWax'], gradients['dWya'], gradients['db'], gradients['dby']\n",
    "   \n",
    "    ### START CODE HERE ###\n",
    "    # Clip to mitigate exploding gradients, loop over [dWax, dWaa, dWya, db, dby]. (≈2 lines)\n",
    "    for gradient in [dWax, dWaa, dWya, db, dby]:\n",
    "        np.clip(gradient, -maxValue, maxValue, out = gradient)\n",
    "    ### END CODE HERE ###\n",
    "    \n",
    "    gradients = {\"dWaa\": dWaa, \"dWax\": dWax, \"dWya\": dWya, \"db\": db, \"dby\": dby}\n",
    "    \n",
    "    return gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UIkYdtBx01Su"
   },
   "outputs": [],
   "source": [
    "def sample(parameters, char_to_ix, seed):\n",
    "    \"\"\"\n",
    "    Sample a sequence of characters according to a sequence of probability distributions output of the RNN\n",
    "\n",
    "    Arguments:\n",
    "    parameters -- Python dictionary containing the parameters Waa, Wax, Wya, by, and b. \n",
    "    char_to_ix -- Python dictionary mapping each character to an index.\n",
    "    seed -- Used for grading purposes. Do not worry about it.\n",
    "\n",
    "    Returns:\n",
    "    indices -- A list of length n containing the indices of the sampled characters.\n",
    "    \"\"\"\n",
    "    \n",
    "    # Retrieve parameters and relevant shapes from \"parameters\" dictionary\n",
    "    Waa, Wax, Wya, by, b = parameters['Waa'], parameters['Wax'], parameters['Wya'], parameters['by'], parameters['b']\n",
    "    vocab_size = by.shape[0]\n",
    "    n_a = Waa.shape[1]\n",
    "    \n",
    "    ### START CODE HERE ###\n",
    "    # Step 1: Create the a zero vector x that can be used as the one-hot vector \n",
    "    # Representing the first character (initializing the sequence generation). (≈1 line)\n",
    "    x = np.zeros((vocab_size,1),dtype=float)\n",
    "    # Step 1': Initialize a_prev as zeros (≈1 line)\n",
    "    a_prev = np.zeros((n_a,1),dtype=float)\n",
    "    \n",
    "    # Create an empty list of indices. This is the list which will contain the list of indices of the characters to generate (≈1 line)\n",
    "    indices = []\n",
    "    \n",
    "    # idx is the index of the one-hot vector x that is set to 1\n",
    "    # All other positions in x are zero.\n",
    "    # Initialize idx to -1\n",
    "    idx =  -1\n",
    "    \n",
    "    # Loop over time-steps t. At each time-step:\n",
    "    # Sample a character from a probability distribution \n",
    "    # And append its index (`idx`) to the list \"indices\". \n",
    "    # You'll stop if you reach 50 characters \n",
    "    # (which should be very unlikely with a well-trained model).\n",
    "    # Setting the maximum number of characters helps with debugging and prevents infinite loops. \n",
    "    counter = 0\n",
    "    newline_character = char_to_ix['\\n']\n",
    "    \n",
    "    while (idx != newline_character and counter != 50):\n",
    "        \n",
    "        # Step 2: Forward propagate x using the equations (1), (2) and (3)\n",
    "        a = np.tanh(np.dot(Wax,x)+np.dot(Waa,a_prev)+b)\n",
    "        z = np.dot(Wya,a)+by\n",
    "        y = softmax(z)\n",
    "        \n",
    "        # For grading purposes\n",
    "        np.random.seed(counter + seed) \n",
    "        \n",
    "        # Step 3: Sample the index of a character within the vocabulary from the probability distribution y\n",
    "        # (see additional hints above)\n",
    "        idx = np.random.choice(range(len(y)),p=y.ravel())\n",
    "\n",
    "        # Append the index to \"indices\"\n",
    "        indices.append(idx)\n",
    "        \n",
    "        # Step 4: Overwrite the input x with one that corresponds to the sampled index `idx`.\n",
    "        # (see additional hints above)\n",
    "        x = np.zeros((vocab_size,1),dtype=float)\n",
    "        x[idx] = 1\n",
    "        \n",
    "        # Update \"a_prev\" to be \"a\"\n",
    "        a_prev = a\n",
    "        \n",
    "        # for grading purposes\n",
    "        seed += 1\n",
    "        counter +=1\n",
    "        \n",
    "    ### END CODE HERE ###\n",
    "\n",
    "    if (counter == 50):\n",
    "        indices.append(char_to_ix['\\n'])\n",
    "    \n",
    "    return indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_BbEdIgY01S3"
   },
   "outputs": [],
   "source": [
    "def optimize(X, Y, a_prev, parameters, learning_rate = 0.01):\n",
    "    \"\"\"\n",
    "    Execute one step of the optimization to train the model.\n",
    "    \n",
    "    Arguments:\n",
    "    X -- list of integers, where each integer is a number that maps to a character in the vocabulary.\n",
    "    Y -- list of integers, exactly the same as X but shifted one index to the left.\n",
    "    a_prev -- previous hidden state.\n",
    "    parameters -- python dictionary containing:\n",
    "                        Wax -- Weight matrix multiplying the input, numpy array of shape (n_a, n_x)\n",
    "                        Waa -- Weight matrix multiplying the hidden state, numpy array of shape (n_a, n_a)\n",
    "                        Wya -- Weight matrix relating the hidden-state to the output, numpy array of shape (n_y, n_a)\n",
    "                        b --  Bias, numpy array of shape (n_a, 1)\n",
    "                        by -- Bias relating the hidden-state to the output, numpy array of shape (n_y, 1)\n",
    "    learning_rate -- learning rate for the model.\n",
    "    \n",
    "    Returns:\n",
    "    loss -- value of the loss function (cross-entropy)\n",
    "    gradients -- python dictionary containing:\n",
    "                        dWax -- Gradients of input-to-hidden weights, of shape (n_a, n_x)\n",
    "                        dWaa -- Gradients of hidden-to-hidden weights, of shape (n_a, n_a)\n",
    "                        dWya -- Gradients of hidden-to-output weights, of shape (n_y, n_a)\n",
    "                        db -- Gradients of bias vector, of shape (n_a, 1)\n",
    "                        dby -- Gradients of output bias vector, of shape (n_y, 1)\n",
    "    a[len(X)-1] -- the last hidden state, of shape (n_a, 1)\n",
    "    \"\"\"\n",
    "    \n",
    "    ### START CODE HERE ###\n",
    "    \n",
    "    # Forward propagate through time (≈1 line)\n",
    "    loss, cache = rnn_forward(X, Y, a_prev, parameters)\n",
    "    \n",
    "    # Backpropagate through time (≈1 line)\n",
    "    gradients, a = rnn_backward(X, Y, parameters, cache)\n",
    "    \n",
    "    # Clip your gradients between -5 (min) and 5 (max) (≈1 line)\n",
    "    gradients = clip(gradients, 5)\n",
    "    \n",
    "    # Update parameters (≈1 line)\n",
    "    parameters = update_parameters(parameters, gradients, learning_rate)\n",
    "    \n",
    "    ### END CODE HERE ###\n",
    "    \n",
    "    return loss, gradients, a[len(X)-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "l214uNun01S_"
   },
   "outputs": [],
   "source": [
    "def model(data_x, ix_to_char, char_to_ix, num_iterations = 35000, n_a = 50, dino_names = 7, vocab_size = 27, verbose = False):\n",
    "    \"\"\"\n",
    "    Trains the model and generates dinosaur names. \n",
    "    \n",
    "    Arguments:\n",
    "    data_x -- text corpus, divided in words\n",
    "    ix_to_char -- dictionary that maps the index to a character\n",
    "    char_to_ix -- dictionary that maps a character to an index\n",
    "    num_iterations -- number of iterations to train the model for\n",
    "    n_a -- number of units of the RNN cell\n",
    "    dino_names -- number of dinosaur names you want to sample at each iteration. \n",
    "    vocab_size -- number of unique characters found in the text (size of the vocabulary)\n",
    "    \n",
    "    Returns:\n",
    "    parameters -- learned parameters\n",
    "    \"\"\"\n",
    "    \n",
    "    # Retrieve n_x and n_y from vocab_size\n",
    "    n_x, n_y = vocab_size, vocab_size\n",
    "    \n",
    "    # Initialize parameters\n",
    "    parameters = initialize_parameters(n_a, n_x, n_y)\n",
    "    \n",
    "    # Initialize loss (this is required because we want to smooth our loss)\n",
    "    loss = get_initial_loss(vocab_size, dino_names)\n",
    "    \n",
    "    # Build list of all dinosaur names (training examples).\n",
    "    examples = [x.strip() for x in data_x]\n",
    "    \n",
    "    # Shuffle list of all dinosaur names\n",
    "    np.random.seed(0)\n",
    "    np.random.shuffle(examples)\n",
    "    \n",
    "    # Initialize the hidden state of your LSTM\n",
    "    a_prev = np.zeros((n_a, 1))\n",
    "    \n",
    "    # for grading purposes\n",
    "    last_dino_name = \"abc\"\n",
    "    \n",
    "    # Optimization loop\n",
    "    for j in range(num_iterations):\n",
    "        \n",
    "        ### START CODE HERE ###\n",
    "        \n",
    "        # Set the index `idx` (see instructions above)\n",
    "        idx = j % len(examples)\n",
    "        \n",
    "        # Set the input X (see instructions above)\n",
    "        single_example = examples[idx]\n",
    "        single_example_chars = [c for c in single_example]\n",
    "        single_example_ix = [char_to_ix[c] for c in single_example_chars]\n",
    "        X = [None] + [char_to_ix[ch] for ch in examples[idx]] \n",
    "        \n",
    "        # Set the labels Y (see instructions above)\n",
    "        ix_newline = char_to_ix[\"\\n\"]\n",
    "        Y = X[1:] + [ix_newline]\n",
    "\n",
    "        # Perform one optimization step: Forward-prop -> Backward-prop -> Clip -> Update parameters\n",
    "        # Choose a learning rate of 0.01\n",
    "        curr_loss, gradients, a_prev = optimize(X, Y, a_prev, parameters, learning_rate = 0.01)\n",
    "        \n",
    "        ### END CODE HERE ###\n",
    "        \n",
    "        # debug statements to aid in correctly forming X, Y\n",
    "        if verbose and j in [0, len(examples) -1, len(examples)]:\n",
    "            print(\"j = \" , j, \"idx = \", idx,) \n",
    "        if verbose and j in [0]:\n",
    "            print(\"single_example =\", single_example)\n",
    "            print(\"single_example_chars\", single_example_chars)\n",
    "            print(\"single_example_ix\", single_example_ix)\n",
    "            print(\" X = \", X, \"\\n\", \"Y =       \", Y, \"\\n\")\n",
    "        \n",
    "        # to keep the loss smooth.\n",
    "        loss = smooth(loss, curr_loss)\n",
    "\n",
    "        # Every 2000 Iteration, generate \"n\" characters thanks to sample() to check if the model is learning properly\n",
    "        if j % 2000 == 0:\n",
    "            \n",
    "            print('Iteration: %d, Loss: %f' % (j, loss) + '\\n')\n",
    "            \n",
    "            # The number of dinosaur names to print\n",
    "            seed = 0\n",
    "            for name in range(dino_names):\n",
    "                \n",
    "                # Sample indices and print them\n",
    "                sampled_indices = sample(parameters, char_to_ix, seed)\n",
    "                last_dino_name = get_sample(sampled_indices, ix_to_char)\n",
    "                print(last_dino_name.replace('\\n', ''))\n",
    "                \n",
    "                seed += 1  # To get the same result (for grading purposes), increment the seed by one. \n",
    "      \n",
    "            print('\\n')\n",
    "        \n",
    "    return parameters, last_dino_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3EH8Edc001TC",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "j =  0 idx =  0\n",
      "single_example = turiasaurus\n",
      "single_example_chars ['t', 'u', 'r', 'i', 'a', 's', 'a', 'u', 'r', 'u', 's']\n",
      "single_example_ix [20, 21, 18, 9, 1, 19, 1, 21, 18, 21, 19]\n",
      " X =  [None, 20, 21, 18, 9, 1, 19, 1, 21, 18, 21, 19] \n",
      " Y =        [20, 21, 18, 9, 1, 19, 1, 21, 18, 21, 19, 0] \n",
      "\n",
      "Iteration: 0, Loss: 23.087336\n",
      "\n",
      "Nkzxwtdmfqoeyhsqwasjkjvu\n",
      "Kneb\n",
      "Kzxwtdmfqoeyhsqwasjkjvu\n",
      "Neb\n",
      "Zxwtdmfqoeyhsqwasjkjvu\n",
      "Eb\n",
      "Xwtdmfqoeyhsqwasjkjvu\n",
      "\n",
      "\n",
      "j =  1535 idx =  1535\n",
      "j =  1536 idx =  0\n",
      "Iteration: 2000, Loss: 27.884160\n",
      "\n",
      "Liusskeomnolxeros\n",
      "Hmdaairus\n",
      "Hytroligoraurus\n",
      "Lecalosapaus\n",
      "Xusicikoraurus\n",
      "Abalpsamantisaurus\n",
      "Tpraneronxeros\n",
      "\n",
      "\n",
      "Iteration: 4000, Loss: 25.901815\n",
      "\n",
      "Mivrosaurus\n",
      "Inee\n",
      "Ivtroplisaurus\n",
      "Mbaaisaurus\n",
      "Wusichisaurus\n",
      "Cabaselachus\n",
      "Toraperlethosdarenitochusthiamamumamaon\n",
      "\n",
      "\n",
      "Iteration: 6000, Loss: 24.608779\n",
      "\n",
      "Onwusceomosaurus\n",
      "Lieeaerosaurus\n",
      "Lxussaurus\n",
      "Oma\n",
      "Xusteonosaurus\n",
      "Eeahosaurus\n",
      "Toreonosaurus\n",
      "\n",
      "\n",
      "Iteration: 8000, Loss: 24.070350\n",
      "\n",
      "Onxusichepriuon\n",
      "Kilabersaurus\n",
      "Lutrodon\n",
      "Omaaerosaurus\n",
      "Xutrcheps\n",
      "Edaksoje\n",
      "Trodiktonus\n",
      "\n",
      "\n",
      "Iteration: 10000, Loss: 23.844446\n",
      "\n",
      "Onyusaurus\n",
      "Klecalosaurus\n",
      "Lustodon\n",
      "Ola\n",
      "Xusodonia\n",
      "Eeaeosaurus\n",
      "Troceosaurus\n",
      "\n",
      "\n",
      "Iteration: 12000, Loss: 23.291971\n",
      "\n",
      "Onyxosaurus\n",
      "Kica\n",
      "Lustrepiosaurus\n",
      "Olaagrraiansaurus\n",
      "Yuspangosaurus\n",
      "Eealosaurus\n",
      "Trognesaurus\n",
      "\n",
      "\n",
      "Iteration: 14000, Loss: 23.382338\n",
      "\n",
      "Meutromodromurus\n",
      "Inda\n",
      "Iutroinatorsaurus\n",
      "Maca\n",
      "Yusteratoptititan\n",
      "Ca\n",
      "Troclosaurus\n",
      "\n",
      "\n",
      "Iteration: 16000, Loss: 23.224544\n",
      "\n",
      "Meusspanchodtashuarhiaspantaxia\n",
      "Indaa\n",
      "Iuspsauhosaurus\n",
      "Macacosaurus\n",
      "Yusoconikaulrit\n",
      "Cacasoceimurus\n",
      "Trrasaurus\n",
      "\n",
      "\n",
      "Iteration: 18000, Loss: 22.904954\n",
      "\n",
      "Pivrrong\n",
      "Llecanosaurus\n",
      "Myssocilindus\n",
      "Peeaishidanagtallsaurus\n",
      "Ytrong\n",
      "Eg\n",
      "Trojichus\n",
      "\n",
      "\n",
      "Iteration: 20000, Loss: 23.005394\n",
      "\n",
      "Nkytrohelosaurus\n",
      "Lolaagosaurus\n",
      "Lyusochosaurus\n",
      "Necakson\n",
      "Yussangosaurus\n",
      "Eiagosaurus\n",
      "Trodon\n",
      "\n",
      "\n",
      "Iteration: 22000, Loss: 22.728886\n",
      "\n",
      "Onustreofkelus\n",
      "Llecagosaurus\n",
      "Mystolojmiaterltasaurus\n",
      "Ola\n",
      "Yuskeolongus\n",
      "Eiacosaurus\n",
      "Trodonosaurus\n",
      "\n",
      "\n",
      "\u001b[92mAll tests passed!\n"
     ]
    }
   ],
   "source": [
    "parameters, last_name = model(data.split(\"\\n\"), ix_to_char, char_to_ix, 22001, verbose = True)\n",
    "\n",
    "assert last_name == 'Trodonosaurus\\n', \"Wrong expected output\"\n",
    "print(\"\\033[92mAll tests passed!\")"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "Dinosaurus_Island_Character_level_language_model_final_v3b+Proposed.ipynb",
   "provenance": []
  },
  "coursera": {
   "schema_names": [
    "DLSC5W1-A2"
   ]
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}