NN_Algo_And_TensorFlow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Neural Network for Binary Classification\n",
    "\n",
    "## N layer with N hidden units are designed with following options\n",
    "\n",
    "  **1 ** ) **Hyper Paremters Initialization**\n",
    "                 - zero Initialization\n",
    "                 - random Initialization\n",
    "                 - HE Initialization\n",
    "  **2 ** ) **Forward Propogation**\n",
    "                 - Out of L layers first L-1 layers use Relu activation function\n",
    "                 - Lth layer uses sigmoid activation function\n",
    "  **3 ** ) **Regulariztion**\n",
    "                 - Dropout\n",
    "                 - lambda with L2 regularization\n",
    "  **4 ** )  **Cost**\n",
    "                 - L2 Regularization\n",
    "  **5 ** ) **Optimization**\n",
    "                 - Gradient Descent Algorithm\n",
    "                 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1234, 1)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "import h5py\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy\n",
    "from PIL import Image\n",
    "from scipy import ndimage\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "data = pd.read_table(\"C:/Users/gokul/Documents/Projects/Neural Networks/data_banknote_authentication.txt\", sep=',', header=None)\n",
    "Xpd = data[[0,1,2,3]]\n",
    "ypd = data[4]\n",
    "#X_train, X_test, y_train, y_test = train_test_split(Xpd, ypd, test_size=0.10, random_state=324)\n",
    "X_el = np.matrix(Xpd)\n",
    "Y_el = np.matrix(ypd)\n",
    "X, X_test, Y, y_test = train_test_split(X_el, Y_el.T, test_size=0.10, random_state=324)\n",
    "X = X.T\n",
    "Y.shape #(m,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyper Parameter (W, b) Initialization\n",
    "\n",
    "**HE Initiallization ** : This function is similar to the previous `initialize_parameters_random(...)`. The only difference is that instead of multiplying `np.random.randn(..,..)` by 10, you will multiply it by $\\sqrt{\\frac{2}{\\text{dimension of the previous layer}}}$, which is what He initialization recommends for layers with a ReLU activation. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def initialize_parameters_deep(layers_dims,initialization=\"he\"):\n",
    "    \"\"\"\n",
    "    Arguments:\n",
    "    layer_dims -- python array (list) containing the dimensions of each layer in our network\n",
    "    \n",
    "    Returns:\n",
    "    parameters -- python dictionary containing your parameters \"W1\", \"b1\", ..., \"WL\", \"bL\":\n",
    "                    Wl -- weight matrix of shape (layer_dims[l], layer_dims[l-1])\n",
    "                    bl -- bias vector of shape (layer_dims[l], 1)\n",
    "    \"\"\"\n",
    "    \n",
    "    np.random.seed(3)\n",
    "    parameters = {}\n",
    "    L = len(layers_dims)            # number of layers in the network\n",
    "    \n",
    "\n",
    "    for l in range(1, L):\n",
    "        \n",
    "        if initialization == \"zeros\":\n",
    "            parameters['W' + str(l)] = np.zeros((layers_dims[l], layers_dims[l-1]))\n",
    "            parameters['b' + str(l)] = np.zeros((layers_dims[l],1))\n",
    "        elif initialization == \"random\":\n",
    "            parameters['W' + str(l)] = np.random.randn(layers_dims[l],layers_dims[l-1]) * 0.01\n",
    "            parameters['b' + str(l)] = np.zeros((layers_dims[l],1))\n",
    "        elif initialization == \"he\":\n",
    "            parameters['W' + str(l)] = np.random.randn(layers_dims[l],layers_dims[l-1]) * np.sqrt(2/layers_dims[l-1])\n",
    "            parameters['b' + str(l)] = np.zeros((layers_dims[l],1))\n",
    "            #print(parameters['W' + str(l)].shape)\n",
    "            #print(parameters['b' + str(l)].shape)\n",
    "        \n",
    "        assert(parameters['W' + str(l)].shape == (layers_dims[l], layers_dims[l-1]))\n",
    "        assert(parameters['b' + str(l)].shape == (layers_dims[l], 1))\n",
    "\n",
    "        \n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Foward Propogation Algorithm starts here\n",
    "\n",
    "."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sigmoid(z):\n",
    "    \"\"\"\n",
    "    Compute the sigmoid of z\n",
    "\n",
    "    Arguments:\n",
    "    z -- A scalar or numpy array of any size.\n",
    "\n",
    "    Return:\n",
    "    s -- sigmoid(z)\n",
    "    \"\"\"\n",
    "    s = 1/(1+np.exp(-z))\n",
    "    return s,z\n",
    "\n",
    "def relu(z):\n",
    "    s = (abs(z) + z) / 2\n",
    "    return s,z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def linear_forward(A, W, b):\n",
    "    \"\"\"\n",
    "    Implement the linear part of a layer's forward propagation.\n",
    "\n",
    "    Arguments:\n",
    "    A -- activations from previous layer (or input data): (size of previous layer, number of examples)\n",
    "    W -- weights matrix: numpy array of shape (size of current layer, size of previous layer)\n",
    "    b -- bias vector, numpy array of shape (size of the current layer, 1)\n",
    "\n",
    "    Returns:\n",
    "    Z -- the input of the activation function, also called pre-activation parameter \n",
    "    cache -- a python dictionary containing \"A\", \"W\" and \"b\" ; stored for computing the backward pass efficiently\n",
    "    \"\"\"\n",
    "    \n",
    "    Z = np.dot(W,A) + b\n",
    "    assert(Z.shape == (W.shape[0], A.shape[1]))\n",
    "    cache = (A, W, b)\n",
    "    \n",
    "    return Z, cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def linear_activation_forward(A_prev, W, b, activation):\n",
    "    \"\"\"\n",
    "    Implement the forward propagation for the LINEAR->ACTIVATION layer\n",
    "\n",
    "    Arguments:\n",
    "    A_prev -- activations from previous layer (or input data): (size of previous layer, number of examples)\n",
    "    W -- weights matrix: numpy array of shape (size of current layer, size of previous layer)\n",
    "    b -- bias vector, numpy array of shape (size of the current layer, 1)\n",
    "    activation -- the activation to be used in this layer, stored as a text string: \"sigmoid\" or \"relu\"\n",
    "\n",
    "    Returns:\n",
    "    A -- the output of the activation function, also called the post-activation value \n",
    "    cache -- a python dictionary containing \"linear_cache\" and \"activation_cache\";\n",
    "             stored for computing the backward pass efficiently\n",
    "    \"\"\"\n",
    "    \n",
    "    if activation == \"sigmoid\":\n",
    "        # Inputs: \"A_prev, W, b\". Outputs: \"A, activation_cache\".\n",
    "        Z, linear_cache = linear_forward(A_prev,W,b)\n",
    "        #Z = np.dot(W,A_prev) + b\n",
    "        A, activation_cache = sigmoid(Z)\n",
    "        #print(\"sigmoid\")\n",
    "    \n",
    "    elif activation == \"relu\":\n",
    "        # Inputs: \"A_prev, W, b\". Outputs: \"A, activation_cache\".\n",
    "        Z, linear_cache = linear_forward(A_prev,W,b)\n",
    "        #Z = np.dot(W,A_prev) + b\n",
    "        A, activation_cache = relu(Z)\n",
    "        #print(\"relu\")\n",
    "    \n",
    "    assert (A.shape == (W.shape[0], A_prev.shape[1]))\n",
    "    cache = (linear_cache, activation_cache)\n",
    "\n",
    "    return A, cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def L_model_forward(X, parameters,keep_prob):\n",
    "    \"\"\"\n",
    "    Implement forward propagation for the [LINEAR->RELU]*(L-1)->LINEAR->SIGMOID computation\n",
    "    \n",
    "    Arguments:\n",
    "    X -- data, numpy array of shape (input size, number of examples)\n",
    "    parameters -- output of initialize_parameters_deep()\n",
    "    \n",
    "    Returns:\n",
    "    AL -- last post-activation value\n",
    "    caches -- list of caches containing:\n",
    "                every cache of linear_activation_forward() (there are L-1 of them, indexed from 0 to L-1)\n",
    "    \"\"\"\n",
    "\n",
    "    caches = []\n",
    "    A = X\n",
    "    L = len(parameters) // 2                  # number of layers in the neural network\n",
    "    # Implement [LINEAR -> RELU]*(L-1). Add \"cache\" to the \"caches\" list.\n",
    "    for l in range(1, L):\n",
    "        A_prev = A \n",
    "        #print(A_prev.shape)\n",
    "        A, cache = linear_activation_forward(A_prev, parameters['W' + str(l)] ,parameters['b' + str(l)], \"relu\")\n",
    "        D = np.random.rand(A.shape[0],A.shape[1])\n",
    "        D = D < keep_prob\n",
    "        #print(A.shape[0])\n",
    "        A = np.multiply(A,D )\n",
    "        A = A/keep_prob\n",
    "        cache = cache + (D,)\n",
    "        caches.append(cache)\n",
    "    \n",
    "    # Implement LINEAR -> SIGMOID. Add \"cache\" to the \"caches\" list.\n",
    "    AL, cache = linear_activation_forward(A,parameters['W' + str(L)] ,parameters['b' + str(L)],\"sigmoid\" )\n",
    "    caches.append(cache)\n",
    "    assert(AL.shape == (1,X.shape[1]))\n",
    "            \n",
    "    return AL, caches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1, 1234)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parameters = initialize_parameters_deep([4,5,2,1], \"he\")\n",
    "print(parameters[\"W1\"].shape)\n",
    "AL, caches = L_model_forward(X, parameters,1)\n",
    "AL.shape\n",
    "#caches[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cost Computation \n",
    "## L2 Regularization\n",
    "\n",
    "The standard way to avoid overfitting is called **L2 regularization**. It consists of appropriately modifying your cost function, from:\n",
    "$$J = -\\frac{1}{m} \\sum\\limits_{i = 1}^{m} \\large{(}\\small  y^{(i)}\\log\\left(a^{[L](i)}\\right) + (1-y^{(i)})\\log\\left(1- a^{[L](i)}\\right) \\large{)} \\tag{1}$$\n",
    "To:\n",
    "$$J_{regularized} = \\small \\underbrace{-\\frac{1}{m} \\sum\\limits_{i = 1}^{m} \\large{(}\\small y^{(i)}\\log\\left(a^{[L](i)}\\right) + (1-y^{(i)})\\log\\left(1- a^{[L](i)}\\right) \\large{)} }_\\text{cross-entropy cost} + \\underbrace{\\frac{1}{m} \\frac{\\lambda}{2} \\sum\\limits_l\\sum\\limits_k\\sum\\limits_j W_{k,j}^{[l]2} }_\\text{L2 regularization cost} \\tag{2}$$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_cost(AL, Y,parameters,lambd,regularized):\n",
    "    \"\"\"\n",
    "    Implement the cost function defined by equation (7).\n",
    "\n",
    "    Arguments:\n",
    "    AL -- probability vector corresponding to your label predictions, shape (1, number of examples)\n",
    "    Y -- true \"label\" vector (for example: containing 0 if non-cat, 1 if cat), shape (1, number of examples)\n",
    "    parameters -- python dictionary containing parameters of the model\n",
    "\n",
    "    Returns:\n",
    "    cost --  value of the regularized loss function (formula (2))\n",
    "    \"\"\"\n",
    "    \n",
    "    m = Y.shape[1]\n",
    "    cross_entropy_cost = -(1/m)* np.sum(np.multiply(np.log(AL),Y) + np.multiply((1-Y),np.log(1-AL)))\n",
    "    loop_len = len(parameters)//2\n",
    "    #print(AL.shape)\n",
    "    \n",
    "    if regularized == True:\n",
    "        temp = 0\n",
    "        for i in range(1,loop_len+1):\n",
    "            temp += np.sum(np.square(parameters['W' + str(i)]))\n",
    "        L2_regularization_cost = (1/m) * (lambd/2) * temp\n",
    "        #print(\"L2_regularization_cost {}\".format(L2_regularization_cost))\n",
    "        #L2_regularization_cost = (1/m) * (lambd/2) * (np.sum(np.square(parameters[\"W1\"])) + np.sum(np.square(parameters[\"W2\"])) + np.sum(np.square(parameters[\"W3\"])))\n",
    "        #print(L2_regularization_cost)\n",
    "        cost = cross_entropy_cost + L2_regularization_cost\n",
    "    else:\n",
    "        cost = cross_entropy_cost\n",
    "        \n",
    "    #print(\"Cross Entropy cost {}\".format(cross_entropy_cost))\n",
    "    \n",
    "    \n",
    "    cost = np.squeeze(cost)      # To make sure your cost's shape is what we expect (e.g. this turns [[17]] into 17).\n",
    "    assert(cost.shape == ())\n",
    "    \n",
    "    return cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.670889593999\n",
      "kljdf\n"
     ]
    }
   ],
   "source": [
    "cost = compute_cost(AL, Y.T,parameters,0.01,False)\n",
    "print(cost)\n",
    "kldjf = 4\n",
    "if kldjf is None:\n",
    "    print(\"he\")\n",
    "elif kldjf is not(None):\n",
    "    print(\"kljdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Back Propogation Alogorithm starts here\n",
    "\n",
    "."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def linear_backward(dZ, cache,D,lambd,keep_prob,val):\n",
    "    \"\"\"\n",
    "    Implement the linear portion of backward propagation for a single layer (layer l)\n",
    "\n",
    "    Arguments:\n",
    "    dZ -- Gradient of the cost with respect to the linear output (of current layer l)\n",
    "    cache -- tuple of values (A_prev, W, b) coming from the forward propagation in the current layer\n",
    "\n",
    "    Returns:\n",
    "    dA_prev -- Gradient of the cost with respect to the activation (of the previous layer l-1), same shape as A_prev\n",
    "    dW -- Gradient of the cost with respect to W (current layer l), same shape as W\n",
    "    db -- Gradient of the cost with respect to b (current layer l), same shape as b\n",
    "    \"\"\"\n",
    "    #linear_cache, activation_cache,D = cache\n",
    "\n",
    "    linear_cache, activation_cache, dummy= cache\n",
    "    A_prev, W, b = linear_cache\n",
    "    m = A_prev.shape[1]\n",
    "    #print(\"W shape: {}\".format(W.shape))\n",
    "    #print(\"DZ shape: {}\".format(dZ.shape))\n",
    "    #print(dZ.shape)\n",
    "    dA_prev = np.dot(W.T, dZ)\n",
    "    #print(\"DA_prev shape: {}\".format(dA_prev.shape))\n",
    "    #print(\"D shape: {}\".format(D.shape))\n",
    "    if val ==0:\n",
    "        dZ_prev = None\n",
    "    else:\n",
    "        dA_prev = np.multiply(dA_prev,D )\n",
    "        dA_prev = dA_prev/keep_prob\n",
    "        dZ_prev = np.multiply(dA_prev, np.int64(A_prev > 0))\n",
    "    \n",
    "    dW = (1./m) * np.dot(dZ,A_prev.T) + ((lambd/m) * W)\n",
    "    db = (1./m) * np.sum(dZ,axis=1)\n",
    "    assert (dA_prev.shape == A_prev.shape)\n",
    "    assert (dW.shape == W.shape)\n",
    "    assert (db.shape == b.shape)\n",
    "    return dZ_prev, dA_prev, dW, db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def L_model_backward(AL, Y, caches, lambd,keep_prob):\n",
    "    \"\"\"\n",
    "    Implement the backward propagation for the [LINEAR->RELU] * (L-1) -> LINEAR -> SIGMOID group\n",
    "    \n",
    "    Arguments:\n",
    "    AL -- probability vector, output of the forward propagation (L_model_forward())\n",
    "    Y -- true \"label\" vector (containing 0 if non-cat, 1 if cat)\n",
    "    caches -- list of caches containing:\n",
    "                every cache of linear_activation_forward() with \"relu\" (it's caches[l], for l in range(L-1) i.e l = 0...L-2)\n",
    "                the cache of linear_activation_forward() with \"sigmoid\" (it's caches[L-1])\n",
    "    \n",
    "    Returns:\n",
    "    grads -- A dictionary with the gradients\n",
    "             grads[\"dA\" + str(l)] = ... \n",
    "             grads[\"dW\" + str(l)] = ...\n",
    "             grads[\"db\" + str(l)] = ... \n",
    "    \"\"\"\n",
    "    grads = {}\n",
    "    L = len(caches) # the number of layers\n",
    "    #print(L)\n",
    "    m = AL.shape[1]\n",
    "    Y = Y.reshape(AL.shape) # after this line, Y is the same shape as AL\n",
    "    \n",
    "    # Initializing the backpropagation and dZL for last layer \n",
    "    dAL = - (np.divide(Y, AL) - np.divide(1 - Y, 1 - AL))\n",
    "    #print(\"AL shape {}\".format(AL.shape))\n",
    "    linear_cache, activation_cache = caches[L-1]\n",
    "    A_prev2, W3, b3 = linear_cache\n",
    "    #print(\"A_prev2 {}\".format(A_prev2.shape))\n",
    "    \n",
    "    dZ_prev3 = AL - Y\n",
    "    #print(\"dZ_prev {}\".format(dZ_prev3.shape))\n",
    "    grads[\"dW\" + str(L)] = 1./m * np.dot(dZ_prev3, A_prev2.T)+((lambd/m) * W3)\n",
    "    grads[\"db\" + str(L)] = 1./m * np.sum(dZ_prev3, axis=1)\n",
    "    #print(\"DWL shape: {}\".format( grads[\"dW\" + str(L)].shape))\n",
    "    #print(\"DbL Shape: {}\".format(grads[\"db\" + str(L)].shape))\n",
    "    \n",
    "    dA_prev2 = np.dot(W3.T, dZ_prev3)\n",
    "    #print(\"dA_prev2 shape: {}\".format(dA_prev2.shape))\n",
    "    linear_cache, activation_cache,D = caches[L-2]\n",
    "    #A_prev, W, b = linear_cache\n",
    "    #print(\"D2 shape {}\".format(D.shape))\n",
    "    #print(dA_prev.shape)\n",
    "    dA_prev2 = np.multiply(D,dA_prev2)\n",
    "    dA_prev2 = dA_prev2/keep_prob\n",
    "    #print(\"After dA_prev2 shape: {}\".format(dA_prev2.shape))\n",
    "    dZ = np.multiply(dA_prev2, np.int64(A_prev2 > 0))\n",
    "    #print(\"DZ shape last: {}\".format(dZ.shape))\n",
    "    x, y,D_prev = caches[L-3]\n",
    "    \n",
    "    #print(dZ_prev.shape)\n",
    "    #grads[\"dA\" + str(L)] = dAL\n",
    "    \n",
    "    # Loop from l=L-2 to l=0\n",
    "    for l in reversed(range(L)):\n",
    "        # lth layer: (RELU -> LINEAR) gradients.\n",
    "        # Inputs: \"grads[\"dA\" + str(l + 1)], current_cache\". Outputs: \"grads[\"dA\" + str(l)] , grads[\"dW\" + str(l + 1)] , grads[\"db\" + str(l + 1)] \n",
    "        current_cache = caches[l-1]\n",
    "        #print(l)\n",
    "        D = D_prev\n",
    "        #print(l-1)\n",
    "        val = l - 1\n",
    "        dZ_prev, dA_prev_temp, dW_temp, db_temp = linear_backward(dZ, current_cache, D,lambd,keep_prob, val)\n",
    "        if val ==0:\n",
    "            pass\n",
    "        else:\n",
    "            x, y, D_prev = caches[l-2]\n",
    "        dZ = dZ_prev\n",
    "        #grads[\"dA\" + str(l)] = dA_prev_temp\n",
    "        grads[\"dW\" + str(l)] = dW_temp\n",
    "        grads[\"db\" + str(l)] = db_temp\n",
    "        if val ==0:\n",
    "            break\n",
    "    \n",
    "    return grads"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Updating Hyper Parameters Based on Gradient Descent\n",
    "\n",
    "."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def update_parameters(parameters, grads, learning_rate):\n",
    "    \"\"\"\n",
    "    Update parameters using gradient descent\n",
    "    \n",
    "    Arguments:\n",
    "    parameters -- python dictionary containing your parameters \n",
    "    grads -- python dictionary containing your gradients, output of L_model_backward\n",
    "    \n",
    "    Returns:\n",
    "    parameters -- python dictionary containing your updated parameters \n",
    "                  parameters[\"W\" + str(l)] = ... \n",
    "                  parameters[\"b\" + str(l)] = ...\n",
    "    \"\"\"\n",
    "    \n",
    "    L = len(parameters) // 2 # number of layers in the neural network\n",
    "\n",
    "    # Update rule for each parameter. Use a for loop.\n",
    "    ### START CODE HERE ### (≈ 3 lines of code)\n",
    "    for l in range(L):\n",
    "        parameters[\"W\" + str(l+1)] = parameters[\"W\" + str(l+1)] - learning_rate * grads[\"dW\" + str(l + 1)]\n",
    "        parameters[\"b\" + str(l+1)] = parameters[\"b\" + str(l+1)] - learning_rate * grads[\"db\" + str(l + 1)]\n",
    "    ### END CODE HERE ###\n",
    "    return parameters\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model controlling Function\n",
    "\n",
    "."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def model(X, Y, learning_rate = 0.3, num_iterations = 10000, print_cost = True, lambd = 0.01, keep_prob = 1):\n",
    "    \n",
    "    \"\"\"\n",
    "    Implements a three-layer neural network: LINEAR->RELU->LINEAR->RELU->LINEAR->SIGMOID.\n",
    "    \n",
    "    Arguments:\n",
    "    X -- input data, of shape (input size, number of examples)\n",
    "    Y -- true \"label\" vector (1 for blue dot / 0 for red dot), of shape (output size, number of examples)\n",
    "    learning_rate -- learning rate of the optimization\n",
    "    num_iterations -- number of iterations of the optimization loop\n",
    "    print_cost -- If True, print the cost every 10000 iterations\n",
    "    lambd -- regularization hyperparameter, scalar\n",
    "    keep_prob - probability of keeping a neuron active during drop-out, scalar.\n",
    "    \n",
    "    Returns:\n",
    "    parameters -- parameters learned by the model. They can then be used to predict.\n",
    "    \"\"\"\n",
    "        \n",
    "    grads = {}\n",
    "    costs = []                            # to keep track of the cost\n",
    "    m = X.shape[1]                        # number of examples\n",
    "    layers_dims = [X.shape[0],5,3, 1]\n",
    "    #print(layers_dims)\n",
    "    # Initialize parameters dictionary.\n",
    "    parameters = initialize_parameters_deep(layers_dims,\"random\")\n",
    "    #print(parameters.keys())\n",
    "\n",
    "    # Loop (gradient descent)\n",
    "    for i in range(0, num_iterations):\n",
    "        AL, cache = L_model_forward(X, parameters,keep_prob)\n",
    "        cost = compute_cost(AL, Y.T,parameters,lambd,True)\n",
    "        grads = L_model_backward(AL, Y, cache, lambd,keep_prob)\n",
    "        #print(grads.keys())\n",
    "        \n",
    "        parameters = update_parameters(parameters, grads, learning_rate)\n",
    "        \n",
    "        if print_cost and i % 1000 == 0:\n",
    "            print(\"Cost after iteration {}: {}\".format(i, cost))\n",
    "        if print_cost and i % 1000 == 0:\n",
    "            costs.append(cost)\n",
    "    \n",
    "    # plot the cost\n",
    "    plt.plot(costs)\n",
    "    plt.ylabel('cost')\n",
    "    plt.xlabel('iterations (x1,000)')\n",
    "    plt.title(\"Learning rate =\" + str(learning_rate))\n",
    "    plt.show()\n",
    "    \n",
    "    return parameters,grads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cost after iteration 0: 0.6931475034011116\n",
      "Cost after iteration 1000: 0.006110831942679048\n",
      "Cost after iteration 2000: 0.0024430686962044817\n",
      "Cost after iteration 3000: 0.0015414471554590387\n",
      "Cost after iteration 4000: 0.0011370673124825952\n",
      "Cost after iteration 5000: 0.0009093745916969098\n",
      "Cost after iteration 6000: 0.000763799678626909\n",
      "Cost after iteration 7000: 0.0006629272015578252\n",
      "Cost after iteration 8000: 0.0005890747442191828\n",
      "Cost after iteration 9000: 0.0005328199407976464\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xt4HHd97/H3R5Itx9baiW1ZCrYT\nO1Ra11AgxSRwaCk9XOocaNJyKc4pJfTmwqkLBXpoaPuENJw+D4Xe4BB6CCkptISQJr241MUn5XoI\nhVpAEnAcJ45zsRpsy7FjO5ZjWdb3/DGzykjZlWRbo9nVfl7Po8c7M7+Z/Woh+uzMb+b3U0RgZmYG\n0FJ0AWZmVj8cCmZmNsqhYGZmoxwKZmY2yqFgZmajHApmZjbKoWBNR9K/Srqq6DrM6pFDwWaMpIcl\nvbLoOiLisoj4dNF1AEj6qqRfm4H3aZf0KUlHJO2V9O4J2m6QtFPSYUn7JX1a0sK8a7T64FCwWUVS\nW9E1VNRTLcC1QA9wIfDTwHslra/R9k7gpRGxCLgIaAP+10wUacVzKFhdkPRaSXdJekLSNyU9L7Pt\nakkPSjoq6V5JP5/Z9lZJd0r6c0kHgWvTdd+Q9CeSDkl6SNJlmX1Gv51Poe1qSV9P3/vfJF0v6W9r\n/A4vl9Qv6Xcl7QVuknSepC9IGkiP/wVJK9L2fwT8JPAxSU9K+li6fo2kOyQdTL+x/8I0fMRvAT4Q\nEYciYgfwSeCt1RpGxJ6IOJBZdQr4kWmowRqAQ8EKJ+nHgU8BvwEsAT4BbJbUnjZ5kOSP5yLgD4G/\nlXR+5hCXAruBZcAfZdbtBJYCHwL+SpJqlDBR25uB/0jruhb4pUl+nW5gMck38o0k/43dlC5fABwH\nPgYQEb8P/D9gU0R0RMQmSQuAO9L3XQZcCXxc0nOqvZmkj6dBWu3nnrTNecCzgLszu94NVD1mus9P\nSDoMHAVeD/zFJL+3zRIOBasHvw58IiK+HRGn0uv9J4AXA0TE30XEYxExEhGfBx4ALsns/1hE/O+I\nGI6I4+m6RyLikxFxCvg0cD7QVeP9q7aVdAHwIuCaiBiKiG8Amyf5XUaA90fEiYg4HhGPR8TtETEY\nEUdJQuunJtj/tcDDEXFT+vt8F7gdeEO1xhHxPyLi3Bo/lbOtjvTfw5ldDwOlWkVExDfSy0crgA8D\nD0/ye9ss4VCwenAh8J7st1xgJcm3WyS9JXNp6QnguSTf6iv2VDnm3sqLiBhMX3ZUaTdR22cBBzPr\nar1X1kBEPFVZkDRf0ickPSLpCPB14FxJrTX2vxC4dNxn8YskZyBn6sn032xn8UKSs4AJRcR/Al8E\nbjmL97cG4lCwerAH+KNx33LnR8TnJF1Icv17E7AkIs4FfgBkLwXlNdTvD4HFkuZn1q2cZJ/xtbwH\nKAOXRsRC4GXpetVovwf42rjPoiMi3l7tzST9n7Q/otrPdoCIOJT+Ls/P7Pp8YPskv0tFG/DsKba1\nBudQsJk2R9K8zE8byR/9t0m6VIkFkl4jqQQsIPnDOQAg6ZdJzhRyFxGPAH0knddzJb0E+NnTPEyJ\npB/hCUmLgfeP276P5A6fii8AvZJ+SdKc9OdFkn60Ro1vS0Oj2k+2z+AzwB+kHd9rSC7Z/XW1Y0r6\nRUkXpP9bXEhyyetLp/l7W4NyKNhM20LyR7Lyc21E9JH8kfoYcAjYRXpnTETcC/wp8O8kf0B/jOSW\nyZnyi8BLgMdJbsv8PEl/x1T9BXAOcAD4FsmlmKyPAG9I70z6aNrv8GpgA/AYyaWtPwbaOTvvJ+mw\nfwT4GvDhiPgiQBoAT6Z9KABrgW+SXHa6k6QT/tfP8v2tQciT7JhNnaTPA/dFxPhv/Gazgs8UzCaQ\nXrp5tqQWJQ97XQH8Y9F1meWlnp64NKtH3cDfkzyn0A+8PSK+V2xJZvnx5SMzMxvly0dmZjaq4S4f\nLV26NFatWlV0GWZmDeU73/nOgYjonKxdw4XCqlWr6OvrK7oMM7OGIumRqbTz5SMzMxvlUDAzs1EO\nBTMzG5VrKEhan04SskvS1VW2/3k6+uVdku5PR4Q0M7OC5NbRnA4NfD3wKpKHfrZJ2pyOZQNARLwr\n0/63gIvzqsfMzCaX55nCJcCuiNgdEUMk47FfMUH7K4HP5ViPmZlNIs9QWM7YCUn603XPkA7Puxr4\nco3tGyX1SeobGBiY9kLNzCyRZyhUmw+31pgaG4Db0ukQn7lTxA0RsS4i1nV2TvrsRVV9Dx/kj794\nHx7Ww8ystjxDoZ+xs1StIBkfvpoN5Hzp6Af/eZi//OqDDBw9naHwzcyaS56hsA3okbRa0lySP/zP\nmPRcUhk4j2QSldz0didzlO/cN+m0tGZmTSu3UIiIYZJ5dbcCO4BbI2K7pOskXZ5peiVwS+R8Xafc\nlYbCXoeCmVktuY59FBFbSKZfzK67ZtzytXnWULGko52lHXMdCmZmE2iqJ5p7u0rc78tHZmY1NWEo\nPMnIiO9AMjOrpqlCYU13ieMnT9F/6HjRpZiZ1aWmCgXfgWRmNrGmCoWeZR0A7lcwM6uhqUKhNG8O\ny889x3cgmZnV0FShAFDuLjkUzMxqaLpQ6O0q8eDAkwwNjxRdiplZ3Wm6UFjTXWJ4JHj48WNFl2Jm\nVneaLhR6PdyFmVlNTRcKF3UuoLVFvgPJzKyKpguFeXNaWbVkvs8UzMyqaLpQgOQOJJ8pmJk9U1OG\nQm9XiUcODjI4NFx0KWZmdaUpQ6HcVSICdu1/suhSzMzqSnOGQrfvQDIzq6YpQ+HCJQuY29bifgUz\ns3GaMhRaW0TPsg527vPlIzOzrKYMBUj6Fe735SMzszGaNhR6u0vsPfIUhwdPFl2KmVndyDUUJK2X\ntFPSLklX12jzC5LulbRd0s151pNV7vKEO2Zm4+UWCpJageuBy4C1wJWS1o5r0wO8D3hpRDwH+O28\n6hmv7FnYzMyeIc8zhUuAXRGxOyKGgFuAK8a1+XXg+og4BBAR+3OsZ4zzF82j1N7mfgUzs4w8Q2E5\nsCez3J+uy+oFeiXdKelbktZXO5CkjZL6JPUNDAxMS3GS6O0u+UzBzCwjz1BQlXUxbrkN6AFeDlwJ\n3Cjp3GfsFHFDRKyLiHWdnZ3TVmBvVzIGUsT4sszMmlOeodAPrMwsrwAeq9LmnyLiZEQ8BOwkCYkZ\nUe7q4InBkwwcPTFTb2lmVtfyDIVtQI+k1ZLmAhuAzePa/CPw0wCSlpJcTtqdY01j9Kadzfe5X8HM\nDMgxFCJiGNgEbAV2ALdGxHZJ10m6PG22FXhc0r3AV4D/GRGP51XTeJXbUj3chZlZoi3Pg0fEFmDL\nuHXXZF4H8O70Z8Yt6WhnaUe7B8YzM0s17RPNFeXuDp8pmJmlmj4UkjuQnmRkxHcgmZk1fSiUu0oc\nP3mK/kPHiy7FzKxwTR8KvR7uwsxsVNOHQs+yDgB27j1ScCVmZsVr+lAozZvD8nPP8YQ7ZmY4FABY\n0+0Jd8zMwKEAJP0KDw48ydDwSNGlmJkVyqFAcgfS8Ejw8OPHii7FzKxQDgWSZxUAP9lsZk3PoQBc\n1LmA1hY5FMys6TkUgHlzWlm1ZL6fVTCzpudQSK3pXugxkMys6TkUUr1dJR49OMjg0HDRpZiZFcah\nkCp3dxABu/b7ITYza14OhZTvQDIzcyiMunDJAua2tbhfwcyamkMh1doiepZ1eL5mM2tqDoWMclfJ\nZwpm1tRyDQVJ6yXtlLRL0tVVtr9V0oCku9KfX8uznsmUu0vsO3KCJwaHiizDzKwwuYWCpFbgeuAy\nYC1wpaS1VZp+PiJekP7cmFc9U1GZcOd+D6NtZk0qzzOFS4BdEbE7IoaAW4Arcny/s1bu8ixsZtbc\n8gyF5cCezHJ/um6810u6R9JtklZWO5CkjZL6JPUNDAzkUSsA5y+aR6m9zXMrmFnTyjMUVGVdjFv+\nZ2BVRDwP+Dfg09UOFBE3RMS6iFjX2dk5zWU+TRK93SU/q2BmTSvPUOgHst/8VwCPZRtExOMRcSJd\n/CTwwhzrmZLerhI79x0lYnx+mZnNfnmGwjagR9JqSXOBDcDmbANJ52cWLwd25FjPlKzpLnH4+En2\nHz0xeWMzs1mmLa8DR8SwpE3AVqAV+FREbJd0HdAXEZuBd0i6HBgGDgJvzaueqcoOd9G1cF7B1ZiZ\nzazcQgEgIrYAW8atuybz+n3A+/Ks4XT1dnUAcP++o7ysN7/+CzOzeuQnmsdZ0tHO0o52dzabWVNy\nKFRR7u7wcBdm1pQcClX0dpW4f9+TjIz4DiQzay4OhSrKXSWOnzzFnkODRZdiZjajHApVlLs94Y6Z\nNSeHQhU9XZWB8RwKZtZcHApVdLS3seK8c9jp0VLNrMk4FGood5U8MJ6ZNR2HQg293SUeHHiSoeGR\noksxM5sxDoUayl0lhkeChw4cK7oUM7MZ41CoYfQOJHc2m1kTcSjUcFHnAlpb5H4FM2sqDoUa2tta\nWb10gc8UzKypOBQmUO4q+VkFM2sqDoUJ9HaVePTgIINDw0WXYmY2IxwKEyh3dxABD/ghNjNrEg6F\nCYzOwuZLSGbWJBwKE7hwyQLa21p8B5KZNQ2HwgRaW0RPV4fPFMysaTgUJtHrO5DMrInkGgqS1kva\nKWmXpKsnaPcGSSFpXZ71nIlyV4l9R07wxOBQ0aWYmeUut1CQ1ApcD1wGrAWulLS2SrsS8A7g23nV\ncjZ6uytzK/gOJDOb/fI8U7gE2BURuyNiCLgFuKJKuw8AHwKeyrGWM1au3IG090jBlZiZ5S/PUFgO\n7Mks96frRkm6GFgZEV+Y6ECSNkrqk9Q3MDAw/ZVO4PxF8yjNa3Nns5k1hTxDQVXWxehGqQX4c+A9\nkx0oIm6IiHURsa6zs3MaS5ycpHTCHV8+MrPZL89Q6AdWZpZXAI9llkvAc4GvSnoYeDGwuR47m3u7\nS+zcd5SImLyxmVkDyzMUtgE9klZLmgtsADZXNkbE4YhYGhGrImIV8C3g8ojoy7GmM1LuKnH4+En2\nHz1RdClmZrnKLRQiYhjYBGwFdgC3RsR2SddJujyv983D6HAXfrLZzGa5tjwPHhFbgC3j1l1To+3L\n86zlbPR2dQBw/76jvKx3Zvs0zMxmkp9onoIlHe0s7WjnPp8pmNksN6VQkPTGqaybzdZ0e7gLM5v9\npnqm8L4prpu1KmMgjYz4DiQzm70m7FOQdBnw34Dlkj6a2bQQaKrpyMrdHTx1coQ9hwa5cMmCossx\nM8vFZGcKjwF9JENQfCfzsxn4mXxLqy++A8nMmsGEZwoRcTdwt6SbI+IkgKTzSIamODQTBdaLnq7K\nwHhHefVzuguuxswsH1PtU7hD0kJJi4G7gZsk/VmOddWdjvY2Vpx3ju9AMrNZbaqhsCgijgCvA26K\niBcCr8yvrPpU9oQ7ZjbLTTUU2iSdD/wCMOGIprNZubvE7oFjDA2PFF2KmVkuphoK15EMV/FgRGyT\ndBHwQH5l1adyd4nhkeChA8eKLsXMLBdTCoWI+LuIeF5EvD1d3h0Rr8+3tPozegeSLyGZ2Sw11Sea\nV0j6B0n7Je2TdLukFXkXV28u6lxAa4u4353NZjZLTfXy0U0kzyY8i2T2tH9O1zWV9rZWVi9d4DMF\nM5u1phoKnRFxU0QMpz9/DTTlcKHlrpIfYDOzWWuqoXBA0psltaY/bwYez7OwelXuLvHowUEGh5pq\nlA8zaxJTDYVfIbkddS/wQ+ANwC/nVVQ9q3Q2P7DPczab2ewz1VD4AHBVRHRGxDKSkLg2t6rqWLnb\ndyCZ2ew11VB4Xnaso4g4CFycT0n17YLF82lva/EdSGY2K001FFrSgfAASMdAynUqz3rV2iJ6ujp8\npmBms9JUQ+FPgW9K+oCk64BvAh+abCdJ6yXtlLRL0tVVtr9N0vcl3SXpG5LWnl75xej1GEhmNktN\n9YnmzwCvB/YBA8DrIuJvJtpHUitwPXAZsBa4ssof/Zsj4sci4gUkIdMQI6+Wu0rsO3KCJwaHii7F\nzGxaTfkSUETcC9x7Gse+BNgVEbsBJN0CXJE9RjryasUCoCHmuhztbN57lEsvWlJwNWZm02eql4/O\nxHJgT2a5P103hqTflPQgyZnCO6odSNJGSX2S+gYGBnIp9nRUQsGXkMxstskzFFRl3TPOBCLi+oh4\nNvC7wB9UO1BE3BAR6yJiXWdn8Q9Sdy+cR2lemzubzWzWyTMU+oGVmeUVJHM+13IL8HM51jNtJCUT\n7uz1A2xmNrvkGQrbgB5JqyXNBTaQDKo3SlJPZvE1NNAcDb3dJXbuO0pEQ3SDmJlNSW6hEBHDwCaS\nyXl2ALdGxHZJ10m6PG22SdJ2SXcB7wauyque6VbuKnH4+En2HTlRdClmZtMm1wfQImILsGXcumsy\nr9+Z5/vnKTvhTveieQVXY2Y2PfK8fDSrjd6B5OEuzGwWcSicocUL5tJZavcdSGY2qzgUzkLZw12Y\n2SzjUDgLlTGQRkZ8B5KZzQ4OhbNQ7u7gqZMj7Dk0WHQpZmbTwqFwFip3IN3nzmYzmyUcCmehEgq+\nA8nMZguHwllY0N7GysXn+A4kM5s1HApnyXcgmdls4lA4S71dJXYPHGNoeKToUszMzppD4SyVu0sM\njwQPHThWdClmZmfNoXCWsmMgmZk1OofCWbqocwGtLWLn3iOTNzYzq3MOhbPU3tbKRUsXsNMT7pjZ\nLOBQmAa93b4DycxmB4fCNCh3lXj04CCDQ8NFl2JmdlYcCtOg0tn8wD5fQjKzxuZQmAaVCXd8B5KZ\nNTqHwjS4YPF85s1pYafHQDKzBudQmAatLaJnmTubzazx5RoKktZL2ilpl6Srq2x/t6R7Jd0j6UuS\nLsyznjz1dpV8pmBmDS+3UJDUClwPXAasBa6UtHZcs+8B6yLiecBtwIfyqidv5e4O9h89waFjQ0WX\nYmZ2xvI8U7gE2BURuyNiCLgFuCLbICK+EhGVacu+BazIsZ5cjc6t4EtIZtbA8gyF5cCezHJ/uq6W\nXwX+tdoGSRsl9UnqGxgYmMYSp0/lDiSHgpk1sjxDQVXWVZ3hXtKbgXXAh6ttj4gbImJdRKzr7Oyc\nxhKnT/fCeZTmtfm2VDNraG05HrsfWJlZXgE8Nr6RpFcCvw/8VEScyLGeXEmi7M5mM2tweZ4pbAN6\nJK2WNBfYAGzONpB0MfAJ4PKI2J9jLTOi3J2EQkTVEyIzs7qXWyhExDCwCdgK7ABujYjtkq6TdHna\n7MNAB/B3ku6StLnG4RpCubvEkaeG2XekYU94zKzJ5Xn5iIjYAmwZt+6azOtX5vn+My074U73onkF\nV2Nmdvr8RPM0Gr0t1f0KZtagHArTaPGCuXSW2n0Hkpk1LIfCNPMdSGbWyBwK06zcXeKB/Uc5NeI7\nkMys8TgUplm5q8RTJ0fYc3Bw8sZmZnXGoTDNej3hjpk1MIfCNOtZ1gH4DiQza0wOhWm2oL2NlYvP\n8ZmCmTUkh0IOyl2ehc3MGpNDIQe9XSV2DxxjaHik6FLMzE6LQyEH5e4SwyPB7gNPFl2KmdlpcSjk\noDLhjh9iM7NG41DIwUVLO2hrkfsVzKzhOBRyMLethdVLF7Bzry8fmVljcSjkpLfbdyCZWeNxKOSk\n3FXi0YODDA4NF12KmdmUORRyMjq3wj5fQjKzxuFQyMmabk+4Y2aNx6GQk5WL5zNvTouHuzCzhpJr\nKEhaL2mnpF2Srq6y/WWSvitpWNIb8qxlprW2iJ5l7mw2s8aSWyhIagWuBy4D1gJXSlo7rtmjwFuB\nm/Oqo0i9noXNzBpMnmcKlwC7ImJ3RAwBtwBXZBtExMMRcQ8wKwcJKnd3sP/oCQ4dGyq6FDOzKckz\nFJYDezLL/em6plG5A8n9CmbWKPIMBVVZd0YTF0vaKKlPUt/AwMBZljVz1nQvBHC/gpk1jDxDoR9Y\nmVleATx2JgeKiBsiYl1ErOvs7JyW4mZC18J2Fs5rc7+CmTWMPENhG9AjabWkucAGYHOO71d3JFH2\ncBdm1kByC4WIGAY2AVuBHcCtEbFd0nWSLgeQ9CJJ/cAbgU9I2p5XPUWp3IEUcUZXzszMZlRbngeP\niC3AlnHrrsm83kZyWWnWKneX+Oy3h9l35ATdi+YVXY6Z2YT8RHPOfAeSmTUSh0LOypVQ2Huk4ErM\nzCbnUMjZeQvmsqzU7gl3zKwhOBRmgO9AMrNG4VCYAb1dJR7Yf5RTI74Dyczqm0NhBpS7Sjx1coQ9\nBweLLsXMbEIOhRnQ2+07kMysMTgUZkDPsg7As7CZWf1zKMyABe1tXLB4Pvf5TMHM6pxDYYb0dpV8\npmBmdc+hMEPK3R08dOAYJ4ZPFV2KmVlNDoUZ0ttVYngkeOjAsaJLMTOryaEwQ8qVO5B8CcnM6phD\nYYZctLSDthb5yWYzq2sOhRkyt62F1UsX+EzBzOqaQ2EGlbtLfoDNzOqaQ2EGlbtK7Dl4nGMnhosu\nxcysKofCDKoMd/HAfg+jbWb1yaEwgyoT7vghNjOrVw6FGbRy8XzmzWlxv4KZ1a22PA8uaT3wEaAV\nuDEiPjhuezvwGeCFwOPAmyLi4TxrKlJri+hZVuLOXQf4/LZHaW9rpb2thfY5LU+/bmtNl1vGbJ/b\n2kJbqzPczPKVWyhIagWuB14F9APbJG2OiHszzX4VOBQRPyJpA/DHwJvyqqkeXLp6MTd+4yF+9/bv\nn/a+rS1Kw6J2eDwdLi3MnWBb+5xW2lpEi4TEmH+TH1D6b4tES0tl+el1E7Wf0jHT7UrXA4h0mXRb\nZSVj10kabVvZRuVYme2MO152+enXz2wPT9dk1kzyPFO4BNgVEbsBJN0CXAFkQ+EK4Nr09W3AxyQp\nImbtFGW//5of5W0vfzYnhkc4cfIUJ4ZHGBoeSZaHT3HiZOZ1ps347UPZNpltR44PZ/Yd28Yzv52d\npwOosqxxy5XtYxvW2j7Z8cYvZNdnA0tTaVPl96i2V61j1dpXVVpVy9Lqx5pa6NZqNtHu1eqa2n61\n9pngeLUPN+HGifar9X7vfEUPP/v8Z030jmctz1BYDuzJLPcDl9ZqExHDkg4DS4AD2UaSNgIbAS64\n4IK86p0Rklja0V7Iew+fGmHoVBIWTw2fYvhUEAEjEYxEEEBEMFJZN5L8m20zEuPajNleaZ/sG2SW\nM22icqwROBWRNASC5FhJHU8vJ9uSlaPb0teMtk3Wkdl37LaxxyNtn91eqYExy5k3ySxP1n78dp6x\nfeL9xuw7bn1W9vtTjFk/tePU2mfslhr7VqkpprBf9aNP/Xg1DzD5Jib6vllry0RfUaf7vSbbuOic\nORPtOS3yDIVqUTf+151KGyLiBuAGgHXr1vnr7hlqS/sl5s8tuhIzq1d59lz2AyszyyuAx2q1kdQG\nLAIO5liTmZlNIM9Q2Ab0SFotaS6wAdg8rs1m4Kr09RuAL8/m/gQzs3qX2+WjtI9gE7CV5JbUT0XE\ndknXAX0RsRn4K+BvJO0iOUPYkFc9ZmY2uVyfU4iILcCWceuuybx+CnhjnjWYmdnU+WkoMzMb5VAw\nM7NRDgUzMxvlUDAzs1FqtDtAJQ0Aj5zh7ksZ97R0k/PnMZY/j6f5sxhrNnweF0ZE52SNGi4Uzoak\nvohYV3Qd9cKfx1j+PJ7mz2KsZvo8fPnIzMxGORTMzGxUs4XCDUUXUGf8eYzlz+Np/izGaprPo6n6\nFMzMbGLNdqZgZmYTcCiYmdmopgkFSesl7ZS0S9LVRddTFEkrJX1F0g5J2yW9s+ia6oGkVknfk/SF\nomspmqRzJd0m6b70/ycvKbqmokh6V/rfyQ8kfU7SvKJryltThIKkVuB64DJgLXClpLXFVlWYYeA9\nEfGjwIuB32zizyLrncCOoouoEx8BvhgRa4Dn06Sfi6TlwDuAdRHxXJIpAGb98P5NEQrAJcCuiNgd\nEUPALcAVBddUiIj4YUR8N319lOQ/+OXFVlUsSSuA1wA3Fl1L0SQtBF5GMtcJETEUEU8UW1Wh2oBz\n0pkh5/PM2SNnnWYJheXAnsxyP03+hxBA0irgYuDbxVZSuL8A3guMFF1IHbgIGABuSi+n3ShpQdFF\nFSEi/hP4E+BR4IfA4Yj4v8VWlb9mCQVVWdfU9+JK6gBuB347Io4UXU9RJL0W2B8R3ym6ljrRBvw4\n8JcRcTFwDGjKPjhJ55FcUVgNPAtYIOnNxVaVv2YJhX5gZWZ5BU1wGliLpDkkgfDZiPj7ousp2EuB\nyyU9THJZ8b9K+ttiSypUP9AfEZWzx9tIQqIZvRJ4KCIGIuIk8PfAfym4ptw1SyhsA3okrZY0l6Sz\naHPBNRVCkkiuF++IiD8rup6iRcT7ImJFRKwi+f/FlyNi1n8brCUi9gJ7JJXTVa8A7i2wpCI9CrxY\n0vz0v5tX0ASd7rnO0VwvImJY0iZgK8kdBJ+KiO0Fl1WUlwK/BHxf0l3put9L59M2A/gt4LPpF6jd\nwC8XXE8hIuLbkm4Dvkty1973aILhLjzMhZmZjWqWy0dmZjYFDgUzMxvlUDAzs1EOBTMzG+VQMDOz\nUQ4Fq0uSvpn+u0rSf5/mY/9etffKi6Sfk3TNJG3emI7GOSKp5gTxkq6S9ED6c1Vm/QslfT8dBfij\n6X31SFos6Y60/R3pU7pIeq2kP5yu39FmD4eC1aWIqDw5ugo4rVBIR8WdyJhQyLxXXt4LfHySNj8A\nXgd8vVYDSYuB9wOXkgzy+P7KH3ngL4GNQE/6sz5dfzXwpYjoAb7E00NW/AvJk9zzT/u3sVnNoWB1\nSdKT6csPAj8p6a50bPtWSR+WtE3SPZJ+I23/8nSeiJuB76fr/lHSd9Jv4BvTdR8kGfXyLkmfzb6X\nEh9Ox87/vqQ3ZY791cwcA5/NfBP/oKR701r+pMrv0QuciIgD6fI/SXpL+vo3KjVExI6I2DnJx/Iz\nwB0RcTAiDgF3AOslnQ8sjIh/j+TBo88AP5fucwXw6fT1pyvr03ZfBV47hf85rIk0xRPN1tCuBn4n\nIl4LkP5xPxwRL5LUDtwpqTJy5SXAcyPioXT5VyLioKRzgG2Sbo+IqyVtiogXVHmv1wEvIJlDYGm6\nT+Wb+8XAc0jGzLoTeKmke4GfB9ZEREg6t8oxX0ryRGzFxrTmh4D3kMxpMVW1Rvtdnr4evx6gKyJ+\nCMmw6ZKWZdr1AT8J3HoaNdgXBvlzAAACRUlEQVQs5zMFazSvBt6SDtHxbWAJyeUSgP/IBALAOyTd\nDXyLZEDEHib2E8DnIuJUROwDvga8KHPs/ogYAe4iuax1BHgKuFHS64DBKsc8n2QoagDS414DfIVk\nsqODU/u1gdqj/Z7pKMD7SUb/NBvlULBGI+C3IuIF6c/qzBj3x0YbSS8nGeXyJRHxfJJxayabSrHa\nH9eKE5nXp4C2iBgmOTu5neSyzBer7He8yvv+GPA4p/8HudZov/3p6/HrAfall5dI/92faTcvrc9s\nlEPB6t1RoJRZ3gq8PR3+G0m9NSaBWQQciohBSWsYe5nmZGX/cb4OvCntt+gkmYHsP2oVls5JsSgd\nTPC3SS49jbcD+JHMPpeQTAt7MfA7klbXOn7afrmkL6WLW4FXSzov7WB+NbA1vTx0VNKL076OtwD/\nlO6zGajcpXRVZj1AL0kHt9koh4LVu3uAYUl3S3oXyZSZ9wLflfQD4BNU7xv7ItAm6R7gAySXkCpu\nAO6pdPJm/EP6fncDXwbemw4lXUsJ+EL6Hl8D3lWlzdeBi9NO7HbgkyR9HY+R9Cl8Kt3285L6gZcA\n/yJpa7r/+SQjdJJeavoAyVDw24DrMpef3p5+NruAB4F/Tdd/EHiVpAeAV6XLFT9NcheS2SiPkmqW\nM0kfAf45Iv7tDPbdBDwaEdM6/4ekLuDmiHjFdB7XGp9DwSxn6R/gS6f7D/vZkPQi4GRE3DVpY2sq\nDgUzMxvlPgUzMxvlUDAzs1EOBTMzG+VQMDOzUQ4FMzMb9f8Bwa65+7ItMvQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x22233043ef0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "On the train set:\n",
      "1.0\n",
      "On the test set:\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "keep_prob=0.2\n",
    "learning_rate = 0.5\n",
    "parameters,grads = model(X, Y)\n",
    "print (\"On the train set:\")\n",
    "AL, xx = L_model_forward(X, parameters,1)\n",
    "predicted = np.matrix(np.zeros(AL.shape[1]))\n",
    "predicted[AL>0.5] = 1\n",
    "Accuracy = accuracy_score(y_true=Y.T,y_pred = predicted,normalize=True)\n",
    "print(Accuracy)\n",
    "print (\"On the test set:\")\n",
    "AT, xx = L_model_forward(X_test.T, parameters,1)\n",
    "Tpredicted = np.matrix(np.zeros(AT.shape[1]))\n",
    "Tpredicted[AT>0.5] = 1\n",
    "TAccuracy = accuracy_score(y_true=y_test.T,y_pred = Tpredicted,normalize=True)\n",
    "print(TAccuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import numpy as np\n",
    "import h5py\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.framework import ops\n",
    "#from tf_utils import load_dataset, random_mini_batches, convert_to_one_hot, predict\n",
    "\n",
    "%matplotlib inline\n",
    "np.random.seed(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_placeholders(n_x, n_y):\n",
    "    \"\"\"\n",
    "    Creates the placeholders for the tensorflow session.\n",
    "    \n",
    "    Arguments:\n",
    "    n_x -- scalar, size of an image vector (num_px * num_px = 64 * 64 * 3 = 12288)\n",
    "    n_y -- scalar, number of classes (from 0 to 5, so -> 6)\n",
    "    \n",
    "    Returns:\n",
    "    X -- placeholder for the data input, of shape [n_x, None] and dtype \"float\"\n",
    "    Y -- placeholder for the input labels, of shape [n_y, None] and dtype \"float\"\n",
    "    \n",
    "    Tips:\n",
    "    - You will use None because it let's us be flexible on the number of examples you will for the placeholders.\n",
    "      In fact, the number of examples during test/train is different.\n",
    "    \"\"\"\n",
    "\n",
    "    ### START CODE HERE ### (approx. 2 lines)\n",
    "    X = tf.placeholder(shape=[n_x,None], dtype= \"float\", name = \"X\")\n",
    "    Y = tf.placeholder(shape=[n_y, None], dtype = \"float\", name = \"Y\")\n",
    "    ### END CODE HERE ###\n",
    "    \n",
    "    return X, Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def initialize_parameters():\n",
    "    \"\"\"\n",
    "    Initializes parameters to build a neural network with tensorflow. The shapes are:\n",
    "                        W1 : [25, 12288]\n",
    "                        b1 : [25, 1]\n",
    "                        W2 : [12, 25]\n",
    "                        b2 : [12, 1]\n",
    "                        W3 : [6, 12]\n",
    "                        b3 : [6, 1]\n",
    "    \n",
    "    Returns:\n",
    "    parameters -- a dictionary of tensors containing W1, b1, W2, b2, W3, b3\n",
    "    \"\"\"\n",
    "    \n",
    "    tf.set_random_seed(1)                   # so that your \"random\" numbers match ours\n",
    "        \n",
    "    ### START CODE HERE ### (approx. 6 lines of code)\n",
    "    W1 = tf.get_variable(\"W1\", [25, 12288], initializer= tf.contrib.layers.xavier_initializer(seed = 1))\n",
    "    b1 = tf.get_variable(\"b1\", [25,1], initializer = tf.zeros_initializer())\n",
    "    W2 = tf.get_variable(\"W2\", [12, 25], initializer= tf.contrib.layers.xavier_initializer(seed = 1))\n",
    "    b2 = tf.get_variable(\"b2\", [12,1], initializer = tf.zeros_initializer())\n",
    "    W3 = tf.get_variable(\"W3\", [6, 12], initializer= tf.contrib.layers.xavier_initializer(seed = 1))\n",
    "    b3 = tf.get_variable(\"b3\", [6,1], initializer = tf.zeros_initializer())\n",
    "    ### END CODE HERE ###\n",
    "\n",
    "    parameters = {\"W1\": W1,\n",
    "                  \"b1\": b1,\n",
    "                  \"W2\": W2,\n",
    "                  \"b2\": b2,\n",
    "                  \"W3\": W3,\n",
    "                  \"b3\": b3}\n",
    "    \n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def forward_propagation(X, parameters):\n",
    "    \"\"\"\n",
    "    Implements the forward propagation for the model: LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SOFTMAX\n",
    "    \n",
    "    Arguments:\n",
    "    X -- input dataset placeholder, of shape (input size, number of examples)\n",
    "    parameters -- python dictionary containing your parameters \"W1\", \"b1\", \"W2\", \"b2\", \"W3\", \"b3\"\n",
    "                  the shapes are given in initialize_parameters\n",
    "\n",
    "    Returns:\n",
    "    Z3 -- the output of the last LINEAR unit\n",
    "    \"\"\"\n",
    "    \n",
    "    # Retrieve the parameters from the dictionary \"parameters\" \n",
    "    W1 = parameters['W1']\n",
    "    b1 = parameters['b1']\n",
    "    W2 = parameters['W2']\n",
    "    b2 = parameters['b2']\n",
    "    W3 = parameters['W3']\n",
    "    b3 = parameters['b3']\n",
    "    \n",
    "    ### START CODE HERE ### (approx. 5 lines)              # Numpy Equivalents:\n",
    "    Z1 = tf.add(tf.matmul(W1,X),b1 )                                             # Z1 = np.dot(W1, X) + b1\n",
    "    A1 = tf.nn.relu(Z1)                                              # A1 = relu(Z1)\n",
    "    Z2 = tf.add(tf.matmul(W2,A1),b2 )                                               # Z2 = np.dot(W2, a1) + b2\n",
    "    A2 = tf.nn.relu(Z2)                                              # A2 = relu(Z2)\n",
    "    Z3 = tf.add(tf.matmul(W3,A2),b3 )                                               # Z3 = np.dot(W3,Z2) + b3\n",
    "    ### END CODE HERE ###\n",
    "    \n",
    "    return Z3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_cost(Z3, Y):\n",
    "    \"\"\"\n",
    "    Computes the cost\n",
    "    \n",
    "    Arguments:\n",
    "    Z3 -- output of forward propagation (output of the last LINEAR unit), of shape (6, number of examples)\n",
    "    Y -- \"true\" labels vector placeholder, same shape as Z3\n",
    "    \n",
    "    Returns:\n",
    "    cost - Tensor of the cost function\n",
    "    \"\"\"\n",
    "    \n",
    "    # to fit the tensorflow requirement for tf.nn.softmax_cross_entropy_with_logits(...,...)\n",
    "    logits = tf.transpose(Z3)\n",
    "    labels = tf.transpose(Y)\n",
    "    \n",
    "    ### START CODE HERE ### (1 line of code)\n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = logits,labels = labels))\n",
    "    ### END CODE HERE ###\n",
    "    \n",
    "    return cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def model(X_train, Y_train, X_test, Y_test, learning_rate = 0.0001,\n",
    "          num_epochs = 1500, minibatch_size = 32, print_cost = True):\n",
    "    \"\"\"\n",
    "    Implements a three-layer tensorflow neural network: LINEAR->RELU->LINEAR->RELU->LINEAR->SOFTMAX.\n",
    "    \n",
    "    Arguments:\n",
    "    X_train -- training set, of shape (input size = 12288, number of training examples = 1080)\n",
    "    Y_train -- test set, of shape (output size = 6, number of training examples = 1080)\n",
    "    X_test -- training set, of shape (input size = 12288, number of training examples = 120)\n",
    "    Y_test -- test set, of shape (output size = 6, number of test examples = 120)\n",
    "    learning_rate -- learning rate of the optimization\n",
    "    num_epochs -- number of epochs of the optimization loop\n",
    "    minibatch_size -- size of a minibatch\n",
    "    print_cost -- True to print the cost every 100 epochs\n",
    "    \n",
    "    Returns:\n",
    "    parameters -- parameters learnt by the model. They can then be used to predict.\n",
    "    \"\"\"\n",
    "    \n",
    "    ops.reset_default_graph()                         # to be able to rerun the model without overwriting tf variables\n",
    "    tf.set_random_seed(1)                             # to keep consistent results\n",
    "    seed = 3                                          # to keep consistent results\n",
    "    (n_x, m) = X_train.shape                          # (n_x: input size, m : number of examples in the train set)\n",
    "    n_y = Y_train.shape[0]                            # n_y : output size\n",
    "    costs = []                                        # To keep track of the cost\n",
    "    \n",
    "    # Create Placeholders of shape (n_x, n_y)\n",
    "    ### START CODE HERE ### (1 line)\n",
    "    X, Y = create_placeholders(n_x, n_y)\n",
    "    ### END CODE HERE ###\n",
    "\n",
    "    # Initialize parameters\n",
    "    ### START CODE HERE ### (1 line)\n",
    "    parameters = initialize_parameters()\n",
    "    ### END CODE HERE ###\n",
    "    \n",
    "    # Forward propagation: Build the forward propagation in the tensorflow graph\n",
    "    ### START CODE HERE ### (1 line)\n",
    "    Z3 = forward_propagation(X,parameters)\n",
    "    ### END CODE HERE ###\n",
    "    \n",
    "    # Cost function: Add cost function to tensorflow graph\n",
    "    ### START CODE HERE ### (1 line)\n",
    "    cost = compute_cost(Z3,Y)\n",
    "    ### END CODE HERE ###\n",
    "    \n",
    "    # Backpropagation: Define the tensorflow optimizer. Use an AdamOptimizer.\n",
    "    ### START CODE HERE ### (1 line)\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(cost)\n",
    "    ### END CODE HERE ###\n",
    "    \n",
    "    # Initialize all the variables\n",
    "    init = tf.global_variables_initializer()\n",
    "\n",
    "    # Start the session to compute the tensorflow graph\n",
    "    with tf.Session() as sess:\n",
    "        \n",
    "        # Run the initialization\n",
    "        sess.run(init)\n",
    "        \n",
    "        # Do the training loop\n",
    "        for epoch in range(num_epochs):\n",
    "\n",
    "            epoch_cost = 0.                       # Defines a cost related to an epoch\n",
    "            num_minibatches = int(m / minibatch_size) # number of minibatches of size minibatch_size in the train set\n",
    "            seed = seed + 1\n",
    "            minibatches = random_mini_batches(X_train, Y_train, minibatch_size, seed)\n",
    "\n",
    "            for minibatch in minibatches:\n",
    "\n",
    "                # Select a minibatch\n",
    "                (minibatch_X, minibatch_Y) = minibatch\n",
    "                \n",
    "                # IMPORTANT: The line that runs the graph on a minibatch.\n",
    "                # Run the session to execute the \"optimizer\" and the \"cost\", the feedict should contain a minibatch for (X,Y).\n",
    "                ### START CODE HERE ### (1 line)\n",
    "                _ , minibatch_cost = sess.run([optimizer, cost], feed_dict={X: minibatch_X, Y: minibatch_Y})\n",
    "                ### END CODE HERE ###\n",
    "                \n",
    "                epoch_cost += minibatch_cost / num_minibatches\n",
    "\n",
    "            # Print the cost every epoch\n",
    "            if print_cost == True and epoch % 100 == 0:\n",
    "                print (\"Cost after epoch %i: %f\" % (epoch, epoch_cost))\n",
    "            if print_cost == True and epoch % 5 == 0:\n",
    "                costs.append(epoch_cost)\n",
    "                \n",
    "        # plot the cost\n",
    "        plt.plot(np.squeeze(costs))\n",
    "        plt.ylabel('cost')\n",
    "        plt.xlabel('iterations (per tens)')\n",
    "        plt.title(\"Learning rate =\" + str(learning_rate))\n",
    "        plt.show()\n",
    "\n",
    "        # lets save the parameters in a variable\n",
    "        parameters = sess.run(parameters)\n",
    "        print (\"Parameters have been trained!\")\n",
    "\n",
    "        # Calculate the correct predictions\n",
    "        correct_prediction = tf.equal(tf.argmax(Z3), tf.argmax(Y))\n",
    "\n",
    "        # Calculate accuracy on the test set\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
    "\n",
    "        print (\"Train Accuracy:\", accuracy.eval({X: X_train, Y: Y_train}))\n",
    "        print (\"Test Accuracy:\", accuracy.eval({X: X_test, Y: Y_test}))\n",
    "        \n",
    "        return parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "parameters = model(X_train, Y_train, X_test, Y_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}