07.3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/udlbook/udlbook/blob/main/Notebooks/Chap07/7_3_Initialization.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "L6chybAVFJW2"
   },
   "source": [
    "# **Notebook 7.3: Initialization**\n",
    "\n",
    "This notebook explores weight initialization in deep neural networks as described in section 7.5 of the book.\n",
    "\n",
    "Work through the cells below, running each cell in turn. In various places you will see the words \"TO DO\". Follow the instructions at these places and make predictions about what is going to happen or write code to complete the functions.\n",
    "\n",
    "Contact me at udlbookmail@gmail.com if you find any mistakes or have any suggestions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "LdIDglk1FFcG"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nnUoI0m6GyjC"
   },
   "source": [
    "First let's define a neural network.  We'll just choose the weights and biases randomly for now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "WVM4Tc_jGI0Q"
   },
   "outputs": [],
   "source": [
    "def init_params(K, D, sigma_sq_omega):\n",
    "  # Set seed so we always get the same random numbers\n",
    "  np.random.seed(0)\n",
    "\n",
    "  # Input layer\n",
    "  D_i = 1\n",
    "  # Output layer\n",
    "  D_o = 1\n",
    "\n",
    "  # Make empty lists\n",
    "  all_weights = [None] * (K+1)\n",
    "  all_biases = [None] * (K+1)\n",
    "\n",
    "  # Create input and output layers\n",
    "  all_weights[0] = np.random.normal(size=(D, D_i))*np.sqrt(sigma_sq_omega)\n",
    "  all_weights[-1] = np.random.normal(size=(D_o, D)) * np.sqrt(sigma_sq_omega)\n",
    "  all_biases[0] = np.zeros((D,1))\n",
    "  all_biases[-1]= np.zeros((D_o,1))\n",
    "\n",
    "  # Create intermediate layers\n",
    "  for layer in range(1,K):\n",
    "    all_weights[layer] = np.random.normal(size=(D,D))*np.sqrt(sigma_sq_omega)\n",
    "    all_biases[layer] = np.zeros((D,1))\n",
    "\n",
    "  return all_weights, all_biases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "jZh-7bPXIDq4"
   },
   "outputs": [],
   "source": [
    "# Define the Rectified Linear Unit (ReLU) function\n",
    "def ReLU(preactivation):\n",
    "  activation = preactivation.clip(0.0)\n",
    "  return activation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "LgquJUJvJPaN"
   },
   "outputs": [],
   "source": [
    "def compute_network_output(net_input, all_weights, all_biases):\n",
    "\n",
    "  # Retrieve number of layers\n",
    "  K = len(all_weights)-1\n",
    "\n",
    "  # We'll store the pre-activations at each layer in a list \"all_f\"\n",
    "  # and the activations in a second list \"all_h\".\n",
    "  all_f = [None] * (K+1)\n",
    "  all_h = [None] * (K+1)\n",
    "\n",
    "  #For convenience, we'll set\n",
    "  # all_h[0] to be the input, and all_f[K] will be the output\n",
    "  all_h[0] = net_input\n",
    "\n",
    "  # Run through the layers, calculating all_f[0...K-1] and all_h[1...K]\n",
    "  for layer in range(K):\n",
    "      # Update preactivations and activations at this layer according to eqn 7.5\n",
    "      all_f[layer] = all_biases[layer] + np.matmul(all_weights[layer], all_h[layer])\n",
    "      all_h[layer+1] = ReLU(all_f[layer])\n",
    "\n",
    "  # Compute the output from the last hidden layer\n",
    "  all_f[K] = all_biases[K] + np.matmul(all_weights[K], all_h[K])\n",
    "\n",
    "  # Retrieve the output\n",
    "  net_output = all_f[K]\n",
    "\n",
    "  return net_output, all_f, all_h"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bIUrcXnOqChl"
   },
   "source": [
    "Now let's investigate how the size of the outputs vary as we change the initialization variance:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "A55z3rKBqO7M"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Layer 1, std of hidden units = 0.098\n",
      "Layer 2, std of hidden units = 0.078\n",
      "Layer 3, std of hidden units = 0.083\n",
      "Layer 4, std of hidden units = 0.101\n",
      "Layer 5, std of hidden units = 0.111\n",
      "Layer 6, std of hidden units = 0.099\n",
      "Layer 7, std of hidden units = 0.092\n",
      "Layer 8, std of hidden units = 0.095\n",
      "Layer 9, std of hidden units = 0.083\n",
      "Layer 10, std of hidden units = 0.072\n",
      "Layer 11, std of hidden units = 0.076\n",
      "Layer 12, std of hidden units = 0.067\n",
      "Layer 13, std of hidden units = 0.065\n",
      "Layer 14, std of hidden units = 0.065\n",
      "Layer 15, std of hidden units = 0.064\n",
      "Layer 16, std of hidden units = 0.066\n",
      "Layer 17, std of hidden units = 0.070\n",
      "Layer 18, std of hidden units = 0.075\n",
      "Layer 19, std of hidden units = 0.073\n",
      "Layer 20, std of hidden units = 0.095\n",
      "Layer 21, std of hidden units = 0.080\n",
      "Layer 22, std of hidden units = 0.081\n",
      "Layer 23, std of hidden units = 0.085\n",
      "Layer 24, std of hidden units = 0.097\n",
      "Layer 25, std of hidden units = 0.106\n",
      "Layer 26, std of hidden units = 0.101\n",
      "Layer 27, std of hidden units = 0.086\n",
      "Layer 28, std of hidden units = 0.083\n",
      "Layer 29, std of hidden units = 0.104\n",
      "Layer 30, std of hidden units = 0.119\n",
      "Layer 31, std of hidden units = 0.099\n",
      "Layer 32, std of hidden units = 0.070\n",
      "Layer 33, std of hidden units = 0.066\n",
      "Layer 34, std of hidden units = 0.066\n",
      "Layer 35, std of hidden units = 0.068\n",
      "Layer 36, std of hidden units = 0.061\n",
      "Layer 37, std of hidden units = 0.067\n",
      "Layer 38, std of hidden units = 0.058\n",
      "Layer 39, std of hidden units = 0.057\n",
      "Layer 40, std of hidden units = 0.058\n",
      "Layer 41, std of hidden units = 0.063\n",
      "Layer 42, std of hidden units = 0.069\n",
      "Layer 43, std of hidden units = 0.064\n",
      "Layer 44, std of hidden units = 0.068\n",
      "Layer 45, std of hidden units = 0.080\n",
      "Layer 46, std of hidden units = 0.076\n",
      "Layer 47, std of hidden units = 0.081\n",
      "Layer 48, std of hidden units = 0.068\n",
      "Layer 49, std of hidden units = 0.063\n",
      "Layer 50, std of hidden units = 0.067\n"
     ]
    }
   ],
   "source": [
    "# Number of layers\n",
    "K = 50\n",
    "# Number of neurons per layer\n",
    "D = 80\n",
    "# Input layer\n",
    "D_i = 1\n",
    "# Output layer\n",
    "D_o = 1\n",
    "# Set variance of initial weights to 1\n",
    "sigma_sq_omega = 2/80\n",
    "# Initialize parameters\n",
    "all_weights, all_biases = init_params(K,D,sigma_sq_omega)\n",
    "\n",
    "n_data = 1000\n",
    "data_in = np.random.normal(size=(1,n_data))\n",
    "net_output, all_f, all_h = compute_network_output(data_in, all_weights, all_biases)\n",
    "\n",
    "for layer in range(1,K+1):\n",
    "  print(\"Layer %d, std of hidden units = %3.3f\"%(layer, np.std(all_h[layer])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "VL_SO4tar3DC"
   },
   "outputs": [],
   "source": [
    "# You can see that the values of the hidden units are increasing on average (the variance is across all hidden units at the layer\n",
    "# and the 1000 training examples\n",
    "\n",
    "# TO DO\n",
    "# Change this to 50 layers with 80 hidden units per layer\n",
    "\n",
    "# TO DO\n",
    "# Now experiment with sigma_sq_omega to try to stop the variance of the forward computation exploding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SxVTKp3IcoBF"
   },
   "source": [
    "Now let's define a loss function.  We'll just use the least squares loss function. We'll also write a function to compute dloss_doutput\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "6XqWSYWJdhQR"
   },
   "outputs": [],
   "source": [
    "def least_squares_loss(net_output, y):\n",
    "  return np.sum((net_output-y) * (net_output-y))\n",
    "\n",
    "def d_loss_d_output(net_output, y):\n",
    "    return 2*(net_output -y);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "98WmyqFYWA-0"
   },
   "source": [
    "Here's the code for the backward pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "LJng7WpRPLMz"
   },
   "outputs": [],
   "source": [
    "# We'll need the indicator function\n",
    "def indicator_function(x):\n",
    "  x_in = np.array(x)\n",
    "  x_in[x_in>=0] = 1\n",
    "  x_in[x_in<0] = 0\n",
    "  return x_in\n",
    "\n",
    "# Main backward pass routine\n",
    "def backward_pass(all_weights, all_biases, all_f, all_h, y):\n",
    "  # Retrieve number of layers\n",
    "  K = len(all_weights) - 1\n",
    "\n",
    "  # We'll store the derivatives dl_dweights and dl_dbiases in lists as well\n",
    "  all_dl_dweights = [None] * (K+1)\n",
    "  all_dl_dbiases = [None] * (K+1)\n",
    "  # And we'll store the derivatives of the loss with respect to the activation and preactivations in lists\n",
    "  all_dl_df = [None] * (K+1)\n",
    "  all_dl_dh = [None] * (K+1)\n",
    "  # Again for convenience we'll stick with the convention that all_h[0] is the net input and all_f[k] in the net output\n",
    "\n",
    "  # Compute derivatives of net output with respect to loss\n",
    "  all_dl_df[K] = np.array(d_loss_d_output(all_f[K],y))\n",
    "\n",
    "  # Now work backwards through the network\n",
    "  for layer in range(K,-1,-1):\n",
    "    # Calculate the derivatives of biases at layer from all_dl_df[K]. (eq 7.13, line 1)\n",
    "    all_dl_dbiases[layer] = np.array(all_dl_df[layer])\n",
    "    # Calculate the derivatives of weight at layer from all_dl_df[K] and all_h[K] (eq 7.13, line 2)\n",
    "    all_dl_dweights[layer] = np.matmul(all_dl_df[layer], all_h[layer].transpose())\n",
    "\n",
    "    # Calculate the derivatives of activations from weight and derivatives of next preactivations (eq 7.13, line 3 second part)\n",
    "    all_dl_dh[layer] = np.matmul(all_weights[layer].transpose(), all_dl_df[layer])\n",
    "    # Calculate the derivatives of the pre-activation f with respect to activation h (eq 7.13, line 3, first part)\n",
    "    if layer > 0:\n",
    "      all_dl_df[layer-1] = indicator_function(all_f[layer-1]) * all_dl_dh[layer]\n",
    "\n",
    "  return all_dl_dweights, all_dl_dbiases, all_dl_dh, all_dl_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "phFnbthqwhFi"
   },
   "source": [
    "Now let's look at what happens to the magnitude of the gradients on the way back."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "9A9MHc4sQvbp"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Layer 1, std of dl_dh = 0.030\n",
      "Layer 2, std of dl_dh = 0.032\n",
      "Layer 3, std of dl_dh = 0.037\n",
      "Layer 4, std of dl_dh = 0.036\n",
      "Layer 5, std of dl_dh = 0.034\n",
      "Layer 6, std of dl_dh = 0.035\n",
      "Layer 7, std of dl_dh = 0.036\n",
      "Layer 8, std of dl_dh = 0.033\n",
      "Layer 9, std of dl_dh = 0.033\n",
      "Layer 10, std of dl_dh = 0.037\n",
      "Layer 11, std of dl_dh = 0.038\n",
      "Layer 12, std of dl_dh = 0.038\n",
      "Layer 13, std of dl_dh = 0.035\n",
      "Layer 14, std of dl_dh = 0.036\n",
      "Layer 15, std of dl_dh = 0.038\n",
      "Layer 16, std of dl_dh = 0.035\n",
      "Layer 17, std of dl_dh = 0.030\n",
      "Layer 18, std of dl_dh = 0.030\n",
      "Layer 19, std of dl_dh = 0.027\n",
      "Layer 20, std of dl_dh = 0.033\n",
      "Layer 21, std of dl_dh = 0.030\n",
      "Layer 22, std of dl_dh = 0.027\n",
      "Layer 23, std of dl_dh = 0.028\n",
      "Layer 24, std of dl_dh = 0.024\n",
      "Layer 25, std of dl_dh = 0.025\n",
      "Layer 26, std of dl_dh = 0.035\n",
      "Layer 27, std of dl_dh = 0.029\n",
      "Layer 28, std of dl_dh = 0.032\n",
      "Layer 29, std of dl_dh = 0.035\n",
      "Layer 30, std of dl_dh = 0.037\n",
      "Layer 31, std of dl_dh = 0.038\n",
      "Layer 32, std of dl_dh = 0.037\n",
      "Layer 33, std of dl_dh = 0.039\n",
      "Layer 34, std of dl_dh = 0.044\n",
      "Layer 35, std of dl_dh = 0.041\n",
      "Layer 36, std of dl_dh = 0.042\n",
      "Layer 37, std of dl_dh = 0.037\n",
      "Layer 38, std of dl_dh = 0.035\n",
      "Layer 39, std of dl_dh = 0.032\n",
      "Layer 40, std of dl_dh = 0.035\n",
      "Layer 41, std of dl_dh = 0.035\n",
      "Layer 42, std of dl_dh = 0.042\n",
      "Layer 43, std of dl_dh = 0.046\n",
      "Layer 44, std of dl_dh = 0.051\n",
      "Layer 45, std of dl_dh = 0.055\n",
      "Layer 46, std of dl_dh = 0.053\n",
      "Layer 47, std of dl_dh = 0.054\n",
      "Layer 48, std of dl_dh = 0.049\n",
      "Layer 49, std of dl_dh = 0.042\n"
     ]
    }
   ],
   "source": [
    "# Number of layers\n",
    "K = 50\n",
    "# Number of neurons per layer\n",
    "D = 80\n",
    "# Input layer\n",
    "D_i = 1\n",
    "# Output layer\n",
    "D_o = 1\n",
    "# Set variance of initial weights to 1\n",
    "sigma_sq_omega = 2/80\n",
    "# Initialize parameters\n",
    "all_weights, all_biases = init_params(K,D,sigma_sq_omega)\n",
    "\n",
    "# For simplicity we'll just consider the gradients of the weights and biases between the first and last hidden layer\n",
    "n_data = 100\n",
    "aggregate_dl_df = [None] * (K+1)\n",
    "for layer in range(1,K):\n",
    "  # These 3D arrays will store the gradients for every data point\n",
    "  aggregate_dl_df[layer] = np.zeros((D,n_data))\n",
    "\n",
    "\n",
    "# We'll have to compute the derivatives of the parameters for each data point separately\n",
    "for c_data in range(n_data):\n",
    "  data_in = np.random.normal(size=(1,1))\n",
    "  y = np.zeros((1,1))\n",
    "  net_output, all_f, all_h = compute_network_output(data_in, all_weights, all_biases)\n",
    "  all_dl_dweights, all_dl_dbiases, all_dl_dh, all_dl_df = backward_pass(all_weights, all_biases, all_f, all_h, y)\n",
    "  for layer in range(1,K):\n",
    "    aggregate_dl_df[layer][:,c_data] = np.squeeze(all_dl_df[layer])\n",
    "\n",
    "for layer in range(1,K):\n",
    "  print(\"Layer %d, std of dl_dh = %3.3f\"%(layer, np.std(aggregate_dl_df[layer].ravel())))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "gtokc0VX0839"
   },
   "outputs": [],
   "source": [
    "# You can see that the values of the hidden units are increasing on average (the variance is across all hidden units at the layer\n",
    "# and the 1000 training examples\n",
    "\n",
    "# TO DO\n",
    "# Change this to 50 layers with 80 hidden units per layer\n",
    "\n",
    "# TO DO\n",
    "# Now experiment with sigma_sq_omega to try to stop the variance of the gradients exploding\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "include_colab_link": true,
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}