Neural Networks and Deep Learning/Building_your_Deep_Neural_Network_Step_by_Step.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import h5py\n",
    "import matplotlib.pyplot as plt\n",
    "from testCases import *\n",
    "from dnn_utils import sigmoid, sigmoid_backward, relu, relu_backward\n",
    "from public_tests import *\n",
    "\n",
    "%matplotlib inline\n",
    "plt.rcParams['figure.figsize'] = (5.0, 4.0) # set default size of plots\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "plt.rcParams['image.cmap'] = 'gray'\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "np.random.seed(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "deletable": false,
    "nbgrader": {
     "cell_type": "code",
     "checksum": "c468c89deb6d0cacf2ade5ab4151d26e",
     "grade": false,
     "grade_id": "cell-96d4e144d9419b32",
     "locked": false,
     "schema_version": 3,
     "solution": true,
     "task": false
    }
   },
   "outputs": [],
   "source": [
    "def initialize_parameters(n_x, n_h, n_y):\n",
    "    \"\"\"\n",
    "    Argument:\n",
    "    n_x -- size of the input layer\n",
    "    n_h -- size of the hidden layer\n",
    "    n_y -- size of the output layer\n",
    "    \n",
    "    Returns:\n",
    "    parameters -- python dictionary containing your parameters:\n",
    "                    W1 -- weight matrix of shape (n_h, n_x)\n",
    "                    b1 -- bias vector of shape (n_h, 1)\n",
    "                    W2 -- weight matrix of shape (n_y, n_h)\n",
    "                    b2 -- bias vector of shape (n_y, 1)\n",
    "    \"\"\"\n",
    "    \n",
    "    np.random.seed(1)\n",
    "    \n",
    "    #(≈ 4 lines of code)\n",
    "    # W1 = ...\n",
    "    # b1 = ...\n",
    "    # W2 = ...\n",
    "    # b2 = ...\n",
    "    # YOUR CODE STARTS HERE\n",
    "    W1 = np.random.randn(n_h,n_x)*0.01\n",
    "    b1 = np.zeros((n_h,1),dtype=float)\n",
    "    W2 = np.random.randn(n_y,n_h)*0.01\n",
    "    b2 = np.zeros((n_y,1),dtype=float)   \n",
    "    \n",
    "    # YOUR CODE ENDS HERE\n",
    "    \n",
    "    parameters = {\"W1\": W1,\n",
    "                  \"b1\": b1,\n",
    "                  \"W2\": W2,\n",
    "                  \"b2\": b2}\n",
    "    \n",
    "    return parameters    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "deletable": false,
    "nbgrader": {
     "cell_type": "code",
     "checksum": "1773f5c69d941998dc8da88f4151e8d3",
     "grade": false,
     "grade_id": "cell-37b22e0664a4949e",
     "locked": false,
     "schema_version": 3,
     "solution": true,
     "task": false
    }
   },
   "outputs": [],
   "source": [
    "def initialize_parameters_deep(layer_dims):\n",
    "    \"\"\"\n",
    "    Arguments:\n",
    "    layer_dims -- python array (list) containing the dimensions of each layer in our network\n",
    "    \n",
    "    Returns:\n",
    "    parameters -- python dictionary containing your parameters \"W1\", \"b1\", ..., \"WL\", \"bL\":\n",
    "                    Wl -- weight matrix of shape (layer_dims[l], layer_dims[l-1])\n",
    "                    bl -- bias vector of shape (layer_dims[l], 1)\n",
    "    \"\"\"\n",
    "    \n",
    "    np.random.seed(3)\n",
    "    parameters = {}\n",
    "    L = len(layer_dims) # number of layers in the network\n",
    "\n",
    "    for l in range(1, L):\n",
    "        #(≈ 2 lines of code)\n",
    "        # parameters['W' + str(l)] = ...\n",
    "        # parameters['b' + str(l)] = ...\n",
    "        # YOUR CODE STARTS HERE\n",
    "        parameters['W' + str(l)] = np.random.randn(layer_dims[l],layer_dims[l-1])*0.01\n",
    "        parameters['b' + str(l)] = np.zeros((layer_dims[l],1),dtype=float)        \n",
    "        \n",
    "        # YOUR CODE ENDS HERE\n",
    "        \n",
    "        assert(parameters['W' + str(l)].shape == (layer_dims[l], layer_dims[l - 1]))\n",
    "        assert(parameters['b' + str(l)].shape == (layer_dims[l], 1))\n",
    "\n",
    "        \n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "deletable": false,
    "nbgrader": {
     "cell_type": "code",
     "checksum": "770763ab229ee87e8f5dfd520428caa3",
     "grade": false,
     "grade_id": "cell-4d6e09486a53f4c4",
     "locked": false,
     "schema_version": 3,
     "solution": true,
     "task": false
    }
   },
   "outputs": [],
   "source": [
    "def linear_forward(A, W, b):\n",
    "    \"\"\"\n",
    "    Implement the linear part of a layer's forward propagation.\n",
    "\n",
    "    Arguments:\n",
    "    A -- activations from previous layer (or input data): (size of previous layer, number of examples)\n",
    "    W -- weights matrix: numpy array of shape (size of current layer, size of previous layer)\n",
    "    b -- bias vector, numpy array of shape (size of the current layer, 1)\n",
    "\n",
    "    Returns:\n",
    "    Z -- the input of the activation function, also called pre-activation parameter \n",
    "    cache -- a python tuple containing \"A\", \"W\" and \"b\" ; stored for computing the backward pass efficiently\n",
    "    \"\"\"\n",
    "    \n",
    "    #(≈ 1 line of code)\n",
    "    # Z = ...\n",
    "    # YOUR CODE STARTS HERE\n",
    "    Z = np.dot(W,A)+b\n",
    "    \n",
    "    # YOUR CODE ENDS HERE\n",
    "    cache = (A, W, b)\n",
    "    \n",
    "    return Z, cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "deletable": false,
    "nbgrader": {
     "cell_type": "code",
     "checksum": "f09e76f2a56c8ee77db3e89214a676b2",
     "grade": false,
     "grade_id": "cell-eb48903dd8e48a90",
     "locked": false,
     "schema_version": 3,
     "solution": true,
     "task": false
    }
   },
   "outputs": [],
   "source": [
    "def linear_activation_forward(A_prev, W, b, activation):\n",
    "    \"\"\"\n",
    "    Implement the forward propagation for the LINEAR->ACTIVATION layer\n",
    "\n",
    "    Arguments:\n",
    "    A_prev -- activations from previous layer (or input data): (size of previous layer, number of examples)\n",
    "    W -- weights matrix: numpy array of shape (size of current layer, size of previous layer)\n",
    "    b -- bias vector, numpy array of shape (size of the current layer, 1)\n",
    "    activation -- the activation to be used in this layer, stored as a text string: \"sigmoid\" or \"relu\"\n",
    "\n",
    "    Returns:\n",
    "    A -- the output of the activation function, also called the post-activation value \n",
    "    cache -- a python tuple containing \"linear_cache\" and \"activation_cache\";\n",
    "             stored for computing the backward pass efficiently\n",
    "    \"\"\"\n",
    "    \n",
    "    if activation == \"sigmoid\":\n",
    "        #(≈ 2 lines of code)\n",
    "        # Z, linear_cache = ...\n",
    "        # A, activation_cache = ...\n",
    "        # YOUR CODE STARTS HERE\n",
    "        Z, linear_cache = linear_forward(A_prev, W, b)\n",
    "        A, activation_cache = sigmoid(Z)        \n",
    "        \n",
    "        # YOUR CODE ENDS HERE\n",
    "    \n",
    "    elif activation == \"relu\":\n",
    "        #(≈ 2 lines of code)\n",
    "        # Z, linear_cache = ...\n",
    "        # A, activation_cache = ...\n",
    "        # YOUR CODE STARTS HERE\n",
    "        Z, linear_cache = linear_forward(A_prev, W, b)\n",
    "        A, activation_cache = relu(Z)        \n",
    "        \n",
    "        # YOUR CODE ENDS HERE\n",
    "    cache = (linear_cache, activation_cache)\n",
    "\n",
    "    return A, cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "deletable": false,
    "nbgrader": {
     "cell_type": "code",
     "checksum": "a0071c19f83d4b851dc8a67e66545262",
     "grade": false,
     "grade_id": "cell-9a8ec52ec8f6e04a",
     "locked": false,
     "schema_version": 3,
     "solution": true,
     "task": false
    }
   },
   "outputs": [],
   "source": [
    "def L_model_forward(X, parameters):\n",
    "    \"\"\"\n",
    "    Implement forward propagation for the [LINEAR->RELU]*(L-1)->LINEAR->SIGMOID computation\n",
    "    \n",
    "    Arguments:\n",
    "    X -- data, numpy array of shape (input size, number of examples)\n",
    "    parameters -- output of initialize_parameters_deep()\n",
    "    \n",
    "    Returns:\n",
    "    AL -- activation value from the output (last) layer\n",
    "    caches -- list of caches containing:\n",
    "                every cache of linear_activation_forward() (there are L of them, indexed from 0 to L-1)\n",
    "    \"\"\"\n",
    "\n",
    "    caches = []\n",
    "    A = X\n",
    "    L = len(parameters) // 2                  # number of layers in the neural network\n",
    "    \n",
    "    # Implement [LINEAR -> RELU]*(L-1). Add \"cache\" to the \"caches\" list.\n",
    "    # The for loop starts at 1 because layer 0 is the input\n",
    "    for l in range(1, L):\n",
    "        A_prev = A \n",
    "        #(≈ 2 lines of code)\n",
    "        # A, cache = ...\n",
    "        # caches ...\n",
    "        # YOUR CODE STARTS HERE\n",
    "        A, cache = linear_activation_forward(A_prev, parameters[\"W\"+str(l)], parameters[\"b\"+str(l)], \"relu\")\n",
    "        caches.append(cache)       \n",
    "        \n",
    "        # YOUR CODE ENDS HERE\n",
    "    \n",
    "    # Implement LINEAR -> SIGMOID. Add \"cache\" to the \"caches\" list.\n",
    "    #(≈ 2 lines of code)\n",
    "    # AL, cache = ...\n",
    "    # caches ...\n",
    "    # YOUR CODE STARTS HERE\n",
    "    AL, cache = linear_activation_forward(A, parameters[\"W\"+str(L)], parameters[\"b\"+str(L)], \"sigmoid\")\n",
    "    caches.append(cache)    \n",
    "    \n",
    "    # YOUR CODE ENDS HERE\n",
    "          \n",
    "    return AL, caches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "deletable": false,
    "nbgrader": {
     "cell_type": "code",
     "checksum": "17919bb7d82635554b52aed7e96e8d9b",
     "grade": false,
     "grade_id": "cell-abad606772066f14",
     "locked": false,
     "schema_version": 3,
     "solution": true,
     "task": false
    }
   },
   "outputs": [],
   "source": [
    "def compute_cost(AL, Y):\n",
    "    \"\"\"\n",
    "    Implement the cost function defined by equation (7).\n",
    "\n",
    "    Arguments:\n",
    "    AL -- probability vector corresponding to your label predictions, shape (1, number of examples)\n",
    "    Y -- true \"label\" vector (for example: containing 0 if non-cat, 1 if cat), shape (1, number of examples)\n",
    "\n",
    "    Returns:\n",
    "    cost -- cross-entropy cost\n",
    "    \"\"\"\n",
    "    \n",
    "    m = Y.shape[1]\n",
    "\n",
    "    # Compute loss from aL and y.\n",
    "    # (≈ 1 lines of code)\n",
    "    # cost = ...\n",
    "    # YOUR CODE STARTS HERE\n",
    "    cost = -(1/m)*np.sum(Y*np.log(AL)+(1-Y)*np.log(1-AL))\n",
    "    \n",
    "    # YOUR CODE ENDS HERE\n",
    "    \n",
    "    cost = np.squeeze(cost)      # To make sure your cost's shape is what we expect (e.g. this turns [[17]] into 17).\n",
    "\n",
    "    \n",
    "    return cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "deletable": false,
    "nbgrader": {
     "cell_type": "code",
     "checksum": "137d11e28068848079eb6c315a59f2be",
     "grade": false,
     "grade_id": "cell-418e156a9203fe72",
     "locked": false,
     "schema_version": 3,
     "solution": true,
     "task": false
    }
   },
   "outputs": [],
   "source": [
    "def linear_backward(dZ, cache):\n",
    "    \"\"\"\n",
    "    Implement the linear portion of backward propagation for a single layer (layer l)\n",
    "\n",
    "    Arguments:\n",
    "    dZ -- Gradient of the cost with respect to the linear output (of current layer l)\n",
    "    cache -- tuple of values (A_prev, W, b) coming from the forward propagation in the current layer\n",
    "\n",
    "    Returns:\n",
    "    dA_prev -- Gradient of the cost with respect to the activation (of the previous layer l-1), same shape as A_prev\n",
    "    dW -- Gradient of the cost with respect to W (current layer l), same shape as W\n",
    "    db -- Gradient of the cost with respect to b (current layer l), same shape as b\n",
    "    \"\"\"\n",
    "    A_prev, W, b = cache\n",
    "    m = A_prev.shape[1]\n",
    "\n",
    "    ### START CODE HERE ### (≈ 3 lines of code)\n",
    "    # dW = ...\n",
    "    # db = ... sum by the rows of dZ with keepdims=True\n",
    "    # dA_prev = ...\n",
    "    # YOUR CODE STARTS HERE\n",
    "    dW=(1/m)*np.dot(dZ,A_prev.T)\n",
    "    db=(1/m)*np.sum(dZ,axis=1,keepdims=True)\n",
    "    dA_prev =np.dot(W.T,dZ)\n",
    "    \n",
    "    # YOUR CODE ENDS HERE\n",
    "    \n",
    "    return dA_prev, dW, db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "deletable": false,
    "nbgrader": {
     "cell_type": "code",
     "checksum": "3497ac4aa36a57278edbfb84a44e1d72",
     "grade": false,
     "grade_id": "cell-6c59263d69168c17",
     "locked": false,
     "schema_version": 3,
     "solution": true,
     "task": false
    }
   },
   "outputs": [],
   "source": [
    "def linear_activation_backward(dA, cache, activation):\n",
    "    \"\"\"\n",
    "    Implement the backward propagation for the LINEAR->ACTIVATION layer.\n",
    "    \n",
    "    Arguments:\n",
    "    dA -- post-activation gradient for current layer l \n",
    "    cache -- tuple of values (linear_cache, activation_cache) we store for computing backward propagation efficiently\n",
    "    activation -- the activation to be used in this layer, stored as a text string: \"sigmoid\" or \"relu\"\n",
    "    \n",
    "    Returns:\n",
    "    dA_prev -- Gradient of the cost with respect to the activation (of the previous layer l-1), same shape as A_prev\n",
    "    dW -- Gradient of the cost with respect to W (current layer l), same shape as W\n",
    "    db -- Gradient of the cost with respect to b (current layer l), same shape as b\n",
    "    \"\"\"\n",
    "    linear_cache, activation_cache = cache\n",
    "    \n",
    "    if activation == \"relu\":\n",
    "        #(≈ 2 lines of code)\n",
    "        # dZ =  ...\n",
    "        # dA_prev, dW, db =  ...\n",
    "        # YOUR CODE STARTS HERE\n",
    "        dZ =  relu_backward(dA, activation_cache)\n",
    "        dA_prev, dW, db =  linear_backward(dZ, linear_cache)     \n",
    "        \n",
    "        # YOUR CODE ENDS HERE\n",
    "        \n",
    "    elif activation == \"sigmoid\":\n",
    "        #(≈ 2 lines of code)\n",
    "        # dZ =  ...\n",
    "        # dA_prev, dW, db =  ...\n",
    "        # YOUR CODE STARTS HERE\n",
    "        dZ =  sigmoid_backward(dA, activation_cache)\n",
    "        dA_prev, dW, db =  linear_backward(dZ, linear_cache)       \n",
    "        \n",
    "        # YOUR CODE ENDS HERE\n",
    "    \n",
    "    return dA_prev, dW, db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "deletable": false,
    "nbgrader": {
     "cell_type": "code",
     "checksum": "d3e23a2b5f3b33e264a122b3c4b0d760",
     "grade": false,
     "grade_id": "cell-9eec96b6d83ff809",
     "locked": false,
     "schema_version": 3,
     "solution": true,
     "task": false
    }
   },
   "outputs": [],
   "source": [
    "def L_model_backward(AL, Y, caches):\n",
    "    \"\"\"\n",
    "    Implement the backward propagation for the [LINEAR->RELU] * (L-1) -> LINEAR -> SIGMOID group\n",
    "    \n",
    "    Arguments:\n",
    "    AL -- probability vector, output of the forward propagation (L_model_forward())\n",
    "    Y -- true \"label\" vector (containing 0 if non-cat, 1 if cat)\n",
    "    caches -- list of caches containing:\n",
    "                every cache of linear_activation_forward() with \"relu\" (it's caches[l], for l in range(L-1) i.e l = 0...L-2)\n",
    "                the cache of linear_activation_forward() with \"sigmoid\" (it's caches[L-1])\n",
    "    \n",
    "    Returns:\n",
    "    grads -- A dictionary with the gradients\n",
    "             grads[\"dA\" + str(l)] = ... \n",
    "             grads[\"dW\" + str(l)] = ...\n",
    "             grads[\"db\" + str(l)] = ... \n",
    "    \"\"\"\n",
    "    grads = {}\n",
    "    L = len(caches) # the number of layers\n",
    "    m = AL.shape[1]\n",
    "    Y = Y.reshape(AL.shape) # after this line, Y is the same shape as AL\n",
    "    \n",
    "    # Initializing the backpropagation\n",
    "    #(1 line of code)\n",
    "    # dAL = ...\n",
    "    # YOUR CODE STARTS HERE\n",
    "    dAL =  - (np.divide(Y, AL) - np.divide(1 - Y, 1 - AL))\n",
    "    \n",
    "    # YOUR CODE ENDS HERE\n",
    "    \n",
    "    # Lth layer (SIGMOID -> LINEAR) gradients. Inputs: \"dAL, current_cache\". Outputs: \"grads[\"dAL-1\"], grads[\"dWL\"], grads[\"dbL\"]\n",
    "    #(approx. 5 lines)\n",
    "    # current_cache = ...\n",
    "    # dA_prev_temp, dW_temp, db_temp = ...\n",
    "    # grads[\"dA\" + str(L-1)] = ...\n",
    "    # grads[\"dW\" + str(L)] = ...\n",
    "    # grads[\"db\" + str(L)] = ...\n",
    "    # YOUR CODE STARTS HERE\n",
    "    current_cache = caches[L]\n",
    "    dA_prev_temp, dW_temp, db_temp = linear_activation_backward(dAL, current_cache, \"sigmoid\")\n",
    "    grads[\"dA\" + str(L-1)] = dA_prev_temp\n",
    "    grads[\"dW\" + str(L)] = dW_temp\n",
    "    grads[\"db\" + str(L)] = db_temp    \n",
    "    \n",
    "    # YOUR CODE ENDS HERE\n",
    "    \n",
    "    # Loop from l=L-2 to l=0\n",
    "    for l in reversed(range(L-1)):\n",
    "        # lth layer: (RELU -> LINEAR) gradients.\n",
    "        # Inputs: \"grads[\"dA\" + str(l + 1)], current_cache\". Outputs: \"grads[\"dA\" + str(l)] , grads[\"dW\" + str(l + 1)] , grads[\"db\" + str(l + 1)] \n",
    "        #(approx. 5 lines)\n",
    "        # current_cache = ...\n",
    "        # dA_prev_temp, dW_temp, db_temp = ...\n",
    "        # grads[\"dA\" + str(l)] = ...\n",
    "        # grads[\"dW\" + str(l + 1)] = ...\n",
    "        # grads[\"db\" + str(l + 1)] = ...\n",
    "        # YOUR CODE STARTS HERE\n",
    "        current_cache = caches[l]\n",
    "        dA_prev_temp, dW_temp, db_temp = linear_activation_backward(dA_prev_temp, current_cache, \"relu\")\n",
    "        grads[\"dA\" + str(l)] = dA_prev_temp\n",
    "        grads[\"dW\" + str(l + 1)] = dW_temp\n",
    "        grads[\"db\" + str(l + 1)] = db_temp        \n",
    "        \n",
    "        # YOUR CODE ENDS HERE\n",
    "\n",
    "    return grads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "deletable": false,
    "nbgrader": {
     "cell_type": "code",
     "checksum": "a497191ef80b70967006d707cac91c20",
     "grade": false,
     "grade_id": "cell-3cb535f16aba3339",
     "locked": false,
     "schema_version": 3,
     "solution": true,
     "task": false
    }
   },
   "outputs": [],
   "source": [
    "def update_parameters(params, grads, learning_rate):\n",
    "    \"\"\"\n",
    "    Update parameters using gradient descent\n",
    "    \n",
    "    Arguments:\n",
    "    params -- python dictionary containing your parameters \n",
    "    grads -- python dictionary containing your gradients, output of L_model_backward\n",
    "    \n",
    "    Returns:\n",
    "    parameters -- python dictionary containing your updated parameters \n",
    "                  parameters[\"W\" + str(l)] = ... \n",
    "                  parameters[\"b\" + str(l)] = ...\n",
    "    \"\"\"\n",
    "    parameters = params.copy()\n",
    "    L = len(parameters) // 2 # number of layers in the neural network\n",
    "\n",
    "    # Update rule for each parameter. Use a for loop.\n",
    "    #(≈ 2 lines of code)\n",
    "    for l in range(L):\n",
    "        # parameters[\"W\" + str(l+1)] = ...\n",
    "        # parameters[\"b\" + str(l+1)] = ...\n",
    "        # YOUR CODE STARTS HERE\n",
    "        parameters[\"W\" + str(l+1)] = parameters[\"W\" + str(l+1)]-learning_rate*grads[\"dW\" + str(l+1)]\n",
    "        parameters[\"b\" + str(l+1)] = parameters[\"b\" + str(l+1)]-learning_rate*grads[\"db\" + str(l+1)]     \n",
    "        \n",
    "        # YOUR CODE ENDS HERE\n",
    "    return parameters"
   ]
  }
 ],
 "metadata": {
  "coursera": {
   "course_slug": "neural-networks-deep-learning",
   "graded_item_id": "c4HO0",
   "launcher_item_id": "lSYZM"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}