tf_learn.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "tf_learn.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/imransalam/463b75beecdbff629c2deca204c83bca/tf_learn.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "dd-jrA-KvEfk",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# **some tensorflow stuff....**\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "9X3u3UXkvkbx",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Please Go to Runtime type and change environment to Python3.\n",
        "\n",
        "---\n",
        "\n",
        "\n",
        "You can change the hardware accelerator to GPU too (if your luck is good enough)\n",
        "\n",
        "\n",
        "\n",
        "---\n",
        "\n",
        "The code written below does not follow \"Good Coding Practices\". This was made for a step by step guide. Hence a lot of things would be repeating."
      ]
    },
    {
      "metadata": {
        "id": "Fe4ViPhuvbtu",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from sklearn import metrics"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "EWDxbeDci1_Q",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## **Constants**"
      ]
    },
    {
      "metadata": {
        "id": "Oa0kaU-tveuA",
        "colab_type": "code",
        "outputId": "8aa15daa-06ab-4418-d111-a0bc0cf2ae1b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "cell_type": "code",
      "source": [
        "'''\n",
        "    Some basic Mathematics Operations Over Constants\n",
        "'''\n",
        "\n",
        "# CREATE GRAPH\n",
        "basic_addition_graph = tf.Graph()\n",
        "with basic_addition_graph.as_default():\n",
        "    a = tf.constant(value=[ 2, 5, 7], \n",
        "                    name=\"a\", \n",
        "                    dtype=tf.int8, \n",
        "                    shape=(3,), \n",
        "                    verify_shape=True)\n",
        "    \n",
        "    b = tf.constant(value=[ 1, -2, -4], \n",
        "                    name=\"b\", \n",
        "                    dtype=tf.int8, \n",
        "                    shape=(3,), \n",
        "                    verify_shape=True)\n",
        "    \n",
        "    c = tf.constant(value=[[ -4, -1, 2], [ -4, -1, 2]], \n",
        "                name=\"c\", \n",
        "                dtype=tf.int8, \n",
        "                shape=(2,3), \n",
        "                verify_shape=True)\n",
        "    \n",
        "    \n",
        "    x = tf.add(a, b)\n",
        "    y = tf.multiply(x , c) # BROADCASTING, (3, ) X (2,3)\n",
        "    \n",
        "\n",
        "# CREATE A SESSION TO RUN THE GRAPH\n",
        "with tf.Session(graph=basic_addition_graph) as sess:\n",
        "    \n",
        "    # writer = tf.summary.FileWriter('basic_addition_graph', sess.graph)\n",
        "    # tensorboard --logdir=\"basic_addition_graph\" --port 8000\n",
        "    \n",
        "    print(sess.run([x, y])) # Providing No feed_dict, No Placeholder in the graph"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[array([3, 3, 3], dtype=int8), array([[-12,  -3,   6],\n",
            "       [-12,  -3,   6]], dtype=int8)]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "tX0vouVXjFW8",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## **Placeholders**"
      ]
    },
    {
      "metadata": {
        "id": "pGkFDkHSy1iO",
        "colab_type": "code",
        "outputId": "f9c695a7-c174-4b87-c519-42e7221dba8b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "cell_type": "code",
      "source": [
        "'''\n",
        "    Some basic Mathematics Operations Over Placeholders\n",
        "'''\n",
        "\n",
        "# CREATE GRAPH\n",
        "basic_addition_graph = tf.Graph()\n",
        "with basic_addition_graph.as_default():\n",
        "    a = tf.placeholder(name=\"a\", \n",
        "                    dtype=tf.int8, \n",
        "                    shape=(3,))\n",
        "    \n",
        "    b = tf.placeholder(name=\"b\", \n",
        "                    dtype=tf.int8, \n",
        "                    shape=(3,))\n",
        "    \n",
        "    c = tf.constant(value=[[ -4, -1, 2], [ -4, -1, 2]], \n",
        "                name=\"c\", \n",
        "                dtype=tf.int8, \n",
        "                shape=(2,3), \n",
        "                verify_shape=True)\n",
        "    \n",
        "    \n",
        "    x = tf.add(a, b)\n",
        "    y = tf.multiply(x , c) # BROADCASTING, (3, ) X (2,3)\n",
        "    \n",
        "\n",
        "# CREATE A SESSION TO RUN THE GRAPH\n",
        "with tf.Session(graph=basic_addition_graph) as sess:\n",
        "    \n",
        "    # writer = tf.summary.FileWriter('basic_addition_graph', sess.graph)\n",
        "    # tensorboard --logdir=\"basic_addition_graph\" --port 8000\n",
        "    \n",
        "    print( sess.run([x, y], feed_dict={a:[ 2, 5, 7], b:[ 1, -2, -4]}) )"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[array([3, 3, 3], dtype=int8), array([[-12,  -3,   6],\n",
            "       [-12,  -3,   6]], dtype=int8)]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "q2oT0iPaklAI",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## **Variables**"
      ]
    },
    {
      "metadata": {
        "id": "1HH9n06Ikoa7",
        "colab_type": "code",
        "outputId": "818b090b-4569-4197-86dd-a0feda580d70",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 179
        }
      },
      "cell_type": "code",
      "source": [
        "'''\n",
        "    Some basic Mathematics Operations Over Placeholders and Variables\n",
        "'''\n",
        "\n",
        "# WHY VARIABLES?\n",
        "# WE WANT A MATRIX/VECTOR THAT CAN BE OPTIMIZED.\n",
        "\n",
        "\n",
        "# DIFFERENCE IN VARIABLES TYPE THAN PLACEHOLDERS OR CONSTANTS?\n",
        "# CONSTANTS AND PLACEHOLDERS ARE OPS. THEY ARE SIMPLY SCALARS / VECTORS / MATRICES / TENSORS. \n",
        "# VARIABLE ON THE OTHER HAND IS A CLASS. IT HAS ITS OWN METHODS DEFINED INSIDE IT.\n",
        "# HENCE YOU WOULD NOTICE THE INITIAL OF VARIABLE IS IN CAPITAL.\n",
        "\n",
        "# CREATE GRAPH\n",
        "basic_graph = tf.Graph()\n",
        "with basic_graph.as_default():\n",
        "    a = tf.placeholder(name=\"a\",\n",
        "                    dtype=tf.float32,\n",
        "                    shape=(None,5))\n",
        "    \n",
        "    \n",
        "    W = tf.Variable(tf.random_normal((5, 3)), name=\"W\") # INITIALIZE IT WITH RANDOM\n",
        "    b = tf.Variable(tf.random_normal((3,)), name=\"b\") # INITIALIZE IT WITH RANDOM\n",
        "    \n",
        "    y = tf.add(tf.matmul(a, W), b)\n",
        "    \n",
        "\n",
        "# CREATE A SESSION TO RUN THE GRAPH\n",
        "with tf.Session(graph=basic_graph) as sess:\n",
        "    \n",
        "    # writer = tf.summary.FileWriter('basic_addition_graph', sess.graph)\n",
        "    # tensorboard --logdir=\"basic_addition_graph\" --port 8000\n",
        "    init = tf.global_variables_initializer()\n",
        "    sess.run(init)\n",
        "    print( sess.run(y, feed_dict={a: np.random.uniform(size=(500, 5)) }) )\n",
        "    print('-' * 20)\n",
        "    print(b.eval())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[1.739159   1.1435039  1.5349289 ]\n",
            " [1.1710117  1.4256032  1.5753393 ]\n",
            " [2.264703   0.75571954 1.3597436 ]\n",
            " ...\n",
            " [2.8382485  0.7648671  1.9334944 ]\n",
            " [2.087041   1.2666237  2.5543098 ]\n",
            " [1.6948155  2.1569772  2.3694232 ]]\n",
            "--------------------\n",
            "[ 1.5039895  -0.19186825  0.3403521 ]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "FvOU2I1m3nkM",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## **Start Some Training**"
      ]
    },
    {
      "metadata": {
        "id": "M_WbpUOm3wJB",
        "colab_type": "code",
        "outputId": "f6f1e952-7e77-44a5-827b-19fcfea09002",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 593
        }
      },
      "cell_type": "code",
      "source": [
        "'''\n",
        "    Training Over Boston Housing. Its a simple Linear regression :/\n",
        "'''\n",
        "\n",
        "# LOAD DATASET\n",
        "dataset = tf.keras.datasets.boston_housing\n",
        "(x_train, y_train), (x_val, y_val) = dataset.load_data()\n",
        "\n",
        "# CONFIGURE\n",
        "lr = 1e-7\n",
        "show_every = 1000\n",
        "epochs = 10000\n",
        "\n",
        "# CREATE GRAPH\n",
        "basic_linear_regression_graph = tf.Graph()\n",
        "with basic_linear_regression_graph.as_default():\n",
        "    \n",
        "    x = tf.placeholder(name=\"x\",\n",
        "                    dtype=tf.float64)\n",
        "    \n",
        "    y = tf.placeholder(name=\"y\",\n",
        "                    dtype=tf.float64)\n",
        "    \n",
        "    W = tf.Variable(tf.truncated_normal([13, 1], mean=0.0, stddev=1.0, dtype=tf.float64), name=\"W\") # INITIALIZE IT WITH RANDOM\n",
        "    b = tf.Variable(tf.zeros(1, dtype = tf.float64), name=\"b\") # INITIALIZE IT WITH RANDOM\n",
        "    \n",
        "    y_hat = tf.add(tf.matmul(x, W), b)\n",
        "    loss = tf.reduce_mean(tf.square(y - y_hat))\n",
        "    \n",
        "    optimizer = tf.train.GradientDescentOptimizer(lr).minimize(loss) # Try Adam with bigger lr\n",
        "    \n",
        "\n",
        "# CREATE A SESSION TO RUN THE GRAPH\n",
        "with tf.Session(graph=basic_linear_regression_graph) as sess:\n",
        "    \n",
        "    init = tf.global_variables_initializer()\n",
        "    sess.run(init)\n",
        "    for i in range(epochs): # EPOCHS  1ep is one iteration over complete Dataset.\n",
        "        sess.run(optimizer, feed_dict={x: x_train, y: y_train})\n",
        "        if i % show_every == 0:\n",
        "            cur_loss_train = sess.run(loss, feed_dict={x: x_train, y: y_train})\n",
        "            cur_loss_val = sess.run(loss, feed_dict={x: x_val, y: y_val})\n",
        "            print(\"Training loss at Epoch # \", i, \" is : \", cur_loss_train / y_train.shape[0])\n",
        "            print(\"Validation loss at Epoch # \", i, \" is : \", cur_loss_val / y_val.shape[0])\n",
        "            print('-' * 45)\n",
        "            \n",
        "        "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/boston_housing.npz\n",
            "57344/57026 [==============================] - 0s 0us/step\n",
            "Training loss at Epoch #  0  is :  370.0406052246812\n",
            "Validation loss at Epoch #  0  is :  1483.314488482391\n",
            "---------------------------------------------\n",
            "Training loss at Epoch #  1000  is :  1.6799104722119447\n",
            "Validation loss at Epoch #  1000  is :  5.530671860368268\n",
            "---------------------------------------------\n",
            "Training loss at Epoch #  2000  is :  1.5568439210871277\n",
            "Validation loss at Epoch #  2000  is :  5.028803758578608\n",
            "---------------------------------------------\n",
            "Training loss at Epoch #  3000  is :  1.4561101405198131\n",
            "Validation loss at Epoch #  3000  is :  4.593574454026101\n",
            "---------------------------------------------\n",
            "Training loss at Epoch #  4000  is :  1.3690914593086776\n",
            "Validation loss at Epoch #  4000  is :  4.224590411287281\n",
            "---------------------------------------------\n",
            "Training loss at Epoch #  5000  is :  1.29333636534445\n",
            "Validation loss at Epoch #  5000  is :  3.909977421834976\n",
            "---------------------------------------------\n",
            "Training loss at Epoch #  6000  is :  1.2269264104019602\n",
            "Validation loss at Epoch #  6000  is :  3.6401772191359867\n",
            "---------------------------------------------\n",
            "Training loss at Epoch #  7000  is :  1.1683436221761334\n",
            "Validation loss at Epoch #  7000  is :  3.4075891887138345\n",
            "---------------------------------------------\n",
            "Training loss at Epoch #  8000  is :  1.1163735949964446\n",
            "Validation loss at Epoch #  8000  is :  3.2061189278629767\n",
            "---------------------------------------------\n",
            "Training loss at Epoch #  9000  is :  1.0700341336523187\n",
            "Validation loss at Epoch #  9000  is :  3.0308385492066696\n",
            "---------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "vO18TxdSLnIH",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## **Define Custom loss with Condition**\n",
        "**Huber Loss**\n",
        "\n",
        "A little Intuituin behind it:\n",
        "\n",
        "If the distance between predicted and original is lesser than some delta use the squared loss, otherwise use the absolute difference to cater for the outliers.\n",
        "\n",
        "Let's see how we can make a conditional statement like this in tensorflow.\n",
        "\n",
        "\\begin{equation*}\n",
        "L_\\delta (y,f(x)) = \\begin{vmatrix}\n",
        "1/2(y-f(x))^2 for |y - f(x)| \\leq \\delta, \\\\\n",
        "\\delta|y-f(x)| - 1/2\\delta^2 Otherwise\\\\\n",
        "\\end{vmatrix}\n",
        "\\end{equation*}"
      ]
    },
    {
      "metadata": {
        "id": "kVdY0J1oJlim",
        "colab_type": "code",
        "outputId": "31bdcd71-8262-4c74-ca1b-592a6d2ca4a2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 557
        }
      },
      "cell_type": "code",
      "source": [
        "'''\n",
        "    Training Over Boston Housing. Its a simple Linear regression with conditional Loss\n",
        "'''\n",
        "\n",
        "# LOAD DATASET\n",
        "dataset = tf.keras.datasets.boston_housing\n",
        "(x_train, y_train), (x_val, y_val) = dataset.load_data()\n",
        "\n",
        "# CONFIGURE\n",
        "lr = 1e-7\n",
        "show_every = 1000\n",
        "epochs = 10000\n",
        "\n",
        "# CREATE GRAPH\n",
        "basic_linear_regression_graph = tf.Graph()\n",
        "with basic_linear_regression_graph.as_default():\n",
        "    \n",
        "    def huber_loss(labels, predicted, delta=1.0):\n",
        "        tf_delta = tf.constant(value=delta, dtype=tf.float64)\n",
        "        residual = tf.reduce_mean(tf.abs(predicted - labels))\n",
        "        condition = tf.less(residual, tf_delta)\n",
        "        small_res = 0.5 * tf.square(residual)\n",
        "        large_res = tf_delta * residual - 0.5 * tf.square(tf_delta)\n",
        "        return tf.where(condition, small_res, large_res)\n",
        "        \n",
        "    \n",
        "    x = tf.placeholder(name=\"x\",\n",
        "                    dtype=tf.float64)\n",
        "    \n",
        "    y = tf.placeholder(name=\"y\",\n",
        "                    dtype=tf.float64)\n",
        "    \n",
        "    W = tf.Variable(tf.truncated_normal([13, 1], mean=0.0, stddev=1.0, dtype=tf.float64), name=\"W\") # INITIALIZE IT WITH RANDOM\n",
        "    b = tf.Variable(tf.zeros(1, dtype = tf.float64), name=\"b\") # INITIALIZE IT WITH RANDOM\n",
        "    \n",
        "    y_hat = tf.add(tf.matmul(x, W), b)\n",
        "    loss = huber_loss(y, y_hat, delta=1.0)\n",
        "#     print(loss)\n",
        "    \n",
        "    optimizer = tf.train.GradientDescentOptimizer(lr).minimize(loss) # Try Adam with bigger lr\n",
        "    \n",
        "\n",
        "# CREATE A SESSION TO RUN THE GRAPH\n",
        "with tf.Session(graph=basic_linear_regression_graph) as sess:\n",
        "    \n",
        "    init = tf.global_variables_initializer()\n",
        "    sess.run(init)\n",
        "    for i in range(epochs): # EPOCHS  1ep is one iteration over complete Dataset.\n",
        "        sess.run(optimizer, feed_dict={x: x_train, y: y_train})\n",
        "        if i % show_every == 0:\n",
        "            cur_loss_train = sess.run(loss, feed_dict={x: x_train, y: y_train})\n",
        "            cur_loss_val = sess.run(loss, feed_dict={x: x_val, y: y_val})\n",
        "            print(\"Training loss at Epoch # \", i, \" is : \", cur_loss_train / y_train.shape[0])\n",
        "            print(\"Validation loss at Epoch # \", i, \" is : \", cur_loss_val / y_val.shape[0])\n",
        "            print('-' * 45)\n",
        "            \n",
        "        "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training loss at Epoch #  0  is :  2.1908339990975128\n",
            "Validation loss at Epoch #  0  is :  8.907859305382098\n",
            "---------------------------------------------\n",
            "Training loss at Epoch #  1000  is :  2.1174914995179175\n",
            "Validation loss at Epoch #  1000  is :  8.609640281803092\n",
            "---------------------------------------------\n",
            "Training loss at Epoch #  2000  is :  2.0441489999382774\n",
            "Validation loss at Epoch #  2000  is :  8.311421258224081\n",
            "---------------------------------------------\n",
            "Training loss at Epoch #  3000  is :  1.9708065003587472\n",
            "Validation loss at Epoch #  3000  is :  8.013202234645041\n",
            "---------------------------------------------\n",
            "Training loss at Epoch #  4000  is :  1.8974640007792105\n",
            "Validation loss at Epoch #  4000  is :  7.714983211066019\n",
            "---------------------------------------------\n",
            "Training loss at Epoch #  5000  is :  1.8241215011995957\n",
            "Validation loss at Epoch #  5000  is :  7.416764187486983\n",
            "---------------------------------------------\n",
            "Training loss at Epoch #  6000  is :  1.750779001620034\n",
            "Validation loss at Epoch #  6000  is :  7.118545163907945\n",
            "---------------------------------------------\n",
            "Training loss at Epoch #  7000  is :  1.6774365020404278\n",
            "Validation loss at Epoch #  7000  is :  6.820326140328926\n",
            "---------------------------------------------\n",
            "Training loss at Epoch #  8000  is :  1.6040940024608479\n",
            "Validation loss at Epoch #  8000  is :  6.522107116749929\n",
            "---------------------------------------------\n",
            "Training loss at Epoch #  9000  is :  1.5307515028812708\n",
            "Validation loss at Epoch #  9000  is :  6.223888093170887\n",
            "---------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "RH-zUGns0VCr",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## **Classification with Logistic Regression**"
      ]
    },
    {
      "metadata": {
        "id": "gq0dpUgllmPb",
        "colab_type": "code",
        "outputId": "2616d3e4-897a-41a4-fd28-bfd40bf96f88",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 8189
        }
      },
      "cell_type": "code",
      "source": [
        "'''\n",
        "    Training Over MNIST\n",
        "'''\n",
        "\n",
        "# LOAD DATASET\n",
        "from tensorflow.examples.tutorials.mnist import input_data\n",
        "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n",
        "\n",
        "\n",
        "# CONFIGURE\n",
        "lr = 1e-3\n",
        "batch_size = 128\n",
        "epochs = 150\n",
        "\n",
        "# CREATE GRAPH\n",
        "basic_logistic_regression_graph = tf.Graph()\n",
        "with basic_logistic_regression_graph.as_default():\n",
        "    \n",
        "    x = tf.placeholder(name=\"x\",\n",
        "                    shape=(None, 784),\n",
        "                    dtype=tf.float32)\n",
        "    \n",
        "    y = tf.placeholder(name=\"y\",\n",
        "                    shape=(None, 10),\n",
        "                    dtype=tf.float32)\n",
        "    \n",
        "    W = tf.Variable(tf.truncated_normal([ 784, 10], mean=0.0, stddev=1.0, dtype=tf.float32), name=\"W\")\n",
        "    b = tf.Variable(tf.zeros(10, dtype = tf.float32), name=\"b\") \n",
        "    \n",
        "    logits = tf.add(tf.matmul(x, W), b)\n",
        "    \n",
        "    loss_for_each_sample = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y)\n",
        "    \n",
        "    loss = tf.reduce_mean(loss_for_each_sample)\n",
        "    \n",
        "    optimizer = tf.train.GradientDescentOptimizer(lr).minimize(loss) \n",
        "    \n",
        "\n",
        "# CREATE A SESSION TO RUN THE GRAPH\n",
        "with tf.Session(graph=basic_logistic_regression_graph) as sess:\n",
        "    \n",
        "    init = tf.global_variables_initializer()\n",
        "    sess.run(init)\n",
        "    for i in range(epochs): # EPOCHS  1ep is one iteration over complete Dataset.\n",
        "        \n",
        "        n_batches = int(mnist.train.num_examples / batch_size)\n",
        "        total_loss = 0\n",
        "        for j in range(n_batches):\n",
        "            x_batch , y_batch = mnist.train.next_batch(batch_size)\n",
        "            _, cur_loss_train = sess.run([optimizer, loss], feed_dict={x: x_batch, y: y_batch})\n",
        "            total_loss = total_loss + cur_loss_train\n",
        "        \n",
        "        print(\"Training loss at Epoch #\", i, \" is :\", total_loss / n_batches)\n",
        "        \n",
        "        n_batches = int(mnist.test.num_examples / batch_size)\n",
        "        total_acc = 0.0\n",
        "        for j in range(n_batches):\n",
        "            x_batch , y_batch = mnist.test.next_batch(batch_size)\n",
        "            preds = sess.run(logits, feed_dict={x: x_batch, y: y_batch})            \n",
        "            correct_preds = np.zeros_like(preds)\n",
        "            correct_preds[np.arange(len(preds)), preds.argmax(1)] = 1\n",
        "            acc = metrics.accuracy_score(y_true=y_batch.astype(int), y_pred=correct_preds.astype(int))\n",
        "            total_acc = total_acc + acc\n",
        "            \n",
        "            \n",
        "        print(\"Validation Accuracy at Epoch #\", i, \" is :\", total_acc / n_batches)\n",
        "        print('-' * 45)\n",
        "            \n",
        "        "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
            "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
            "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
            "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
            "Training loss at Epoch # 0  is : 10.488791933704368\n",
            "Validation Accuracy at Epoch # 0  is : 0.11187900641025642\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 1  is : 9.121935548760119\n",
            "Validation Accuracy at Epoch # 1  is : 0.1233974358974359\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 2  is : 8.240222829760928\n",
            "Validation Accuracy at Epoch # 2  is : 0.13872195512820512\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 3  is : 7.532380922135218\n",
            "Validation Accuracy at Epoch # 3  is : 0.1539463141025641\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 4  is : 6.943778738275275\n",
            "Validation Accuracy at Epoch # 4  is : 0.17197516025641027\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 5  is : 6.441345850626628\n",
            "Validation Accuracy at Epoch # 5  is : 0.19270833333333334\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 6  is : 6.009191625323885\n",
            "Validation Accuracy at Epoch # 6  is : 0.21574519230769232\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 7  is : 5.637731507663682\n",
            "Validation Accuracy at Epoch # 7  is : 0.2405849358974359\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 8  is : 5.29623446653495\n",
            "Validation Accuracy at Epoch # 8  is : 0.2646233974358974\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 9  is : 4.999421996670169\n",
            "Validation Accuracy at Epoch # 9  is : 0.28495592948717946\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 10  is : 4.742652555723568\n",
            "Validation Accuracy at Epoch # 10  is : 0.30679086538461536\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 11  is : 4.502618627948361\n",
            "Validation Accuracy at Epoch # 11  is : 0.3288261217948718\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 12  is : 4.298879750680812\n",
            "Validation Accuracy at Epoch # 12  is : 0.34655448717948717\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 13  is : 4.104512695388083\n",
            "Validation Accuracy at Epoch # 13  is : 0.3691907051282051\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 14  is : 3.9240520528424314\n",
            "Validation Accuracy at Epoch # 14  is : 0.38331330128205127\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 15  is : 3.7699905408845913\n",
            "Validation Accuracy at Epoch # 15  is : 0.4011418269230769\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 16  is : 3.6276254426071417\n",
            "Validation Accuracy at Epoch # 16  is : 0.4133613782051282\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 17  is : 3.499072088228239\n",
            "Validation Accuracy at Epoch # 17  is : 0.4282852564102564\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 18  is : 3.367276456250455\n",
            "Validation Accuracy at Epoch # 18  is : 0.43990384615384615\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 19  is : 3.259353832765059\n",
            "Validation Accuracy at Epoch # 19  is : 0.45622996794871795\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 20  is : 3.1551413486053894\n",
            "Validation Accuracy at Epoch # 20  is : 0.4638421474358974\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 21  is : 3.0615264168986074\n",
            "Validation Accuracy at Epoch # 21  is : 0.4775641025641026\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 22  is : 2.9787657861109382\n",
            "Validation Accuracy at Epoch # 22  is : 0.483974358974359\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 23  is : 2.8840983578097292\n",
            "Validation Accuracy at Epoch # 23  is : 0.49459134615384615\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 24  is : 2.8162601338677753\n",
            "Validation Accuracy at Epoch # 24  is : 0.5036057692307693\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 25  is : 2.739902269590151\n",
            "Validation Accuracy at Epoch # 25  is : 0.5166266025641025\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 26  is : 2.6723250517478356\n",
            "Validation Accuracy at Epoch # 26  is : 0.5268429487179487\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 27  is : 2.608515443501773\n",
            "Validation Accuracy at Epoch # 27  is : 0.5297475961538461\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 28  is : 2.5509591072033615\n",
            "Validation Accuracy at Epoch # 28  is : 0.5416666666666666\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 29  is : 2.4942758336211694\n",
            "Validation Accuracy at Epoch # 29  is : 0.5472756410256411\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 30  is : 2.4415368025119486\n",
            "Validation Accuracy at Epoch # 30  is : 0.5557892628205128\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 31  is : 2.3930445164789407\n",
            "Validation Accuracy at Epoch # 31  is : 0.5641025641025641\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 32  is : 2.3409871829973232\n",
            "Validation Accuracy at Epoch # 32  is : 0.5668068910256411\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 33  is : 2.3047476801283153\n",
            "Validation Accuracy at Epoch # 33  is : 0.5753205128205128\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 34  is : 2.2612099164015764\n",
            "Validation Accuracy at Epoch # 34  is : 0.5796274038461539\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 35  is : 2.2137144217680107\n",
            "Validation Accuracy at Epoch # 35  is : 0.5868389423076923\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 36  is : 2.178320642395731\n",
            "Validation Accuracy at Epoch # 36  is : 0.5930488782051282\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 37  is : 2.145300518382679\n",
            "Validation Accuracy at Epoch # 37  is : 0.5947516025641025\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 38  is : 2.1064270367989173\n",
            "Validation Accuracy at Epoch # 38  is : 0.6006610576923077\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 39  is : 2.0898551927024114\n",
            "Validation Accuracy at Epoch # 39  is : 0.6040665064102564\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 40  is : 2.0317007325468084\n",
            "Validation Accuracy at Epoch # 40  is : 0.6072716346153846\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 41  is : 2.0161661240326496\n",
            "Validation Accuracy at Epoch # 41  is : 0.6122796474358975\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 42  is : 1.9896762974056608\n",
            "Validation Accuracy at Epoch # 42  is : 0.6176883012820513\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 43  is : 1.9571956846263858\n",
            "Validation Accuracy at Epoch # 43  is : 0.6227964743589743\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 44  is : 1.931422362516532\n",
            "Validation Accuracy at Epoch # 44  is : 0.6227964743589743\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 45  is : 1.8932863562812894\n",
            "Validation Accuracy at Epoch # 45  is : 0.628104967948718\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 46  is : 1.8883250462703216\n",
            "Validation Accuracy at Epoch # 46  is : 0.6354166666666666\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 47  is : 1.8618623540673778\n",
            "Validation Accuracy at Epoch # 47  is : 0.6376201923076923\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 48  is : 1.833889427579644\n",
            "Validation Accuracy at Epoch # 48  is : 0.6419270833333334\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 49  is : 1.8150718220464\n",
            "Validation Accuracy at Epoch # 49  is : 0.6443309294871795\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 50  is : 1.7876383330160643\n",
            "Validation Accuracy at Epoch # 50  is : 0.6487379807692307\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 51  is : 1.772726978177513\n",
            "Validation Accuracy at Epoch # 51  is : 0.6540464743589743\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 52  is : 1.7526110402353994\n",
            "Validation Accuracy at Epoch # 52  is : 0.6526442307692307\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 53  is : 1.7268499964044923\n",
            "Validation Accuracy at Epoch # 53  is : 0.6569511217948718\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 54  is : 1.7211000920731427\n",
            "Validation Accuracy at Epoch # 54  is : 0.6618589743589743\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 55  is : 1.6956722133365267\n",
            "Validation Accuracy at Epoch # 55  is : 0.660957532051282\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 56  is : 1.68734239212005\n",
            "Validation Accuracy at Epoch # 56  is : 0.6675681089743589\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 57  is : 1.6556328615664324\n",
            "Validation Accuracy at Epoch # 57  is : 0.6736778846153846\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 58  is : 1.6408236974324935\n",
            "Validation Accuracy at Epoch # 58  is : 0.6737780448717948\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 59  is : 1.635276474597015\n",
            "Validation Accuracy at Epoch # 59  is : 0.6741786858974359\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 60  is : 1.619842742428635\n",
            "Validation Accuracy at Epoch # 60  is : 0.6817908653846154\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 61  is : 1.5950922226850248\n",
            "Validation Accuracy at Epoch # 61  is : 0.6800881410256411\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 62  is : 1.586020954700061\n",
            "Validation Accuracy at Epoch # 62  is : 0.6831931089743589\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 63  is : 1.5805520283314454\n",
            "Validation Accuracy at Epoch # 63  is : 0.6855969551282052\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 64  is : 1.556159074489887\n",
            "Validation Accuracy at Epoch # 64  is : 0.6888020833333334\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 65  is : 1.549780938730929\n",
            "Validation Accuracy at Epoch # 65  is : 0.6923076923076923\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 66  is : 1.541831859341868\n",
            "Validation Accuracy at Epoch # 66  is : 0.6932091346153846\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 67  is : 1.517578222951689\n",
            "Validation Accuracy at Epoch # 67  is : 0.6952123397435898\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 68  is : 1.5085725824593943\n",
            "Validation Accuracy at Epoch # 68  is : 0.6962139423076923\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 69  is : 1.501255289380089\n",
            "Validation Accuracy at Epoch # 69  is : 0.6987179487179487\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 70  is : 1.4837712481583194\n",
            "Validation Accuracy at Epoch # 70  is : 0.6967147435897436\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 71  is : 1.4713793435852567\n",
            "Validation Accuracy at Epoch # 71  is : 0.7034254807692307\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 72  is : 1.4691686721948476\n",
            "Validation Accuracy at Epoch # 72  is : 0.7028245192307693\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 73  is : 1.4527158346765248\n",
            "Validation Accuracy at Epoch # 73  is : 0.7112379807692307\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 74  is : 1.4533171168851964\n",
            "Validation Accuracy at Epoch # 74  is : 0.7040264423076923\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 75  is : 1.42250700291498\n",
            "Validation Accuracy at Epoch # 75  is : 0.7115384615384616\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 76  is : 1.4194726923128942\n",
            "Validation Accuracy at Epoch # 76  is : 0.7080328525641025\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 77  is : 1.4234166571866105\n",
            "Validation Accuracy at Epoch # 77  is : 0.7136418269230769\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 78  is : 1.3956228825317951\n",
            "Validation Accuracy at Epoch # 78  is : 0.7157451923076923\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 79  is : 1.3970418383469392\n",
            "Validation Accuracy at Epoch # 79  is : 0.7131410256410257\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 80  is : 1.3877214344787152\n",
            "Validation Accuracy at Epoch # 80  is : 0.7199519230769231\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 81  is : 1.3767320249742006\n",
            "Validation Accuracy at Epoch # 81  is : 0.7150440705128205\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 82  is : 1.3673828390094784\n",
            "Validation Accuracy at Epoch # 82  is : 0.7229567307692307\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 83  is : 1.3638645534193043\n",
            "Validation Accuracy at Epoch # 83  is : 0.717948717948718\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 84  is : 1.3480365291619911\n",
            "Validation Accuracy at Epoch # 84  is : 0.7295673076923077\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 85  is : 1.3500464710044415\n",
            "Validation Accuracy at Epoch # 85  is : 0.7206530448717948\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 86  is : 1.325639934111864\n",
            "Validation Accuracy at Epoch # 86  is : 0.727363782051282\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 87  is : 1.332061895262667\n",
            "Validation Accuracy at Epoch # 87  is : 0.7291666666666666\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 88  is : 1.319003914063905\n",
            "Validation Accuracy at Epoch # 88  is : 0.7284655448717948\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 89  is : 1.313950159332969\n",
            "Validation Accuracy at Epoch # 89  is : 0.7331730769230769\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 90  is : 1.3039261451412192\n",
            "Validation Accuracy at Epoch # 90  is : 0.731270032051282\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 91  is : 1.2957516136147202\n",
            "Validation Accuracy at Epoch # 91  is : 0.7359775641025641\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 92  is : 1.2941536988133873\n",
            "Validation Accuracy at Epoch # 92  is : 0.734375\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 93  is : 1.2815871961189038\n",
            "Validation Accuracy at Epoch # 93  is : 0.7375801282051282\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 94  is : 1.2755638747226385\n",
            "Validation Accuracy at Epoch # 94  is : 0.7365785256410257\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 95  is : 1.266969377622182\n",
            "Validation Accuracy at Epoch # 95  is : 0.7433894230769231\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 96  is : 1.2644873913073595\n",
            "Validation Accuracy at Epoch # 96  is : 0.7385817307692307\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 97  is : 1.2597370802105723\n",
            "Validation Accuracy at Epoch # 97  is : 0.7409855769230769\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 98  is : 1.250517668007137\n",
            "Validation Accuracy at Epoch # 98  is : 0.7463942307692307\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 99  is : 1.2495806050467324\n",
            "Validation Accuracy at Epoch # 99  is : 0.7447916666666666\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 100  is : 1.2461114416033516\n",
            "Validation Accuracy at Epoch # 100  is : 0.7504006410256411\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 101  is : 1.2212982605665157\n",
            "Validation Accuracy at Epoch # 101  is : 0.7434895833333334\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 102  is : 1.2238450668630623\n",
            "Validation Accuracy at Epoch # 102  is : 0.7504006410256411\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 103  is : 1.2311392237534333\n",
            "Validation Accuracy at Epoch # 103  is : 0.7480969551282052\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 104  is : 1.1981691697260717\n",
            "Validation Accuracy at Epoch # 104  is : 0.7476963141025641\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 105  is : 1.2174986345228893\n",
            "Validation Accuracy at Epoch # 105  is : 0.7536057692307693\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 106  is : 1.2012427767395695\n",
            "Validation Accuracy at Epoch # 106  is : 0.7529046474358975\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 107  is : 1.2055456620274168\n",
            "Validation Accuracy at Epoch # 107  is : 0.7532051282051282\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 108  is : 1.1982794788889675\n",
            "Validation Accuracy at Epoch # 108  is : 0.7527043269230769\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 109  is : 1.1787081146851563\n",
            "Validation Accuracy at Epoch # 109  is : 0.7592147435897436\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 110  is : 1.1824511820341879\n",
            "Validation Accuracy at Epoch # 110  is : 0.7543068910256411\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 111  is : 1.1808953162951348\n",
            "Validation Accuracy at Epoch # 111  is : 0.7540064102564102\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 112  is : 1.1671357275722745\n",
            "Validation Accuracy at Epoch # 112  is : 0.7594150641025641\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 113  is : 1.1721637346249916\n",
            "Validation Accuracy at Epoch # 113  is : 0.7601161858974359\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 114  is : 1.1601026286056269\n",
            "Validation Accuracy at Epoch # 114  is : 0.7592147435897436\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 115  is : 1.1661535798808633\n",
            "Validation Accuracy at Epoch # 115  is : 0.7602163461538461\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 116  is : 1.1421069883522177\n",
            "Validation Accuracy at Epoch # 116  is : 0.7620192307692307\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 117  is : 1.1533486993162783\n",
            "Validation Accuracy at Epoch # 117  is : 0.760917467948718\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 118  is : 1.1377696872869016\n",
            "Validation Accuracy at Epoch # 118  is : 0.7643229166666666\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 119  is : 1.1482344253079875\n",
            "Validation Accuracy at Epoch # 119  is : 0.7608173076923077\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 120  is : 1.132576035313951\n",
            "Validation Accuracy at Epoch # 120  is : 0.7635216346153846\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 121  is : 1.1334951772834316\n",
            "Validation Accuracy at Epoch # 121  is : 0.7669270833333334\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 122  is : 1.121805268845636\n",
            "Validation Accuracy at Epoch # 122  is : 0.7621193910256411\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 123  is : 1.121686808851771\n",
            "Validation Accuracy at Epoch # 123  is : 0.7701322115384616\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 124  is : 1.1210395146361043\n",
            "Validation Accuracy at Epoch # 124  is : 0.7646233974358975\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 125  is : 1.1147253784266384\n",
            "Validation Accuracy at Epoch # 125  is : 0.76953125\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 126  is : 1.1076955807792557\n",
            "Validation Accuracy at Epoch # 126  is : 0.7671274038461539\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 127  is : 1.1080940838182445\n",
            "Validation Accuracy at Epoch # 127  is : 0.7696314102564102\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 128  is : 1.090166272816958\n",
            "Validation Accuracy at Epoch # 128  is : 0.7698317307692307\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 129  is : 1.1082758140035998\n",
            "Validation Accuracy at Epoch # 129  is : 0.7733373397435898\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 130  is : 1.0954913556714714\n",
            "Validation Accuracy at Epoch # 130  is : 0.7732371794871795\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 131  is : 1.090421071141472\n",
            "Validation Accuracy at Epoch # 131  is : 0.7728365384615384\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 132  is : 1.0751639765737218\n",
            "Validation Accuracy at Epoch # 132  is : 0.7762419871794872\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 133  is : 1.086643960787144\n",
            "Validation Accuracy at Epoch # 133  is : 0.7733373397435898\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 134  is : 1.0778810458305554\n",
            "Validation Accuracy at Epoch # 134  is : 0.7784455128205128\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 135  is : 1.080574429674304\n",
            "Validation Accuracy at Epoch # 135  is : 0.7730368589743589\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 136  is : 1.0647468127848663\n",
            "Validation Accuracy at Epoch # 136  is : 0.7783453525641025\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 137  is : 1.0655342182079395\n",
            "Validation Accuracy at Epoch # 137  is : 0.7809495192307693\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 138  is : 1.0690659877859352\n",
            "Validation Accuracy at Epoch # 138  is : 0.7744391025641025\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 139  is : 1.0583848703157652\n",
            "Validation Accuracy at Epoch # 139  is : 0.7762419871794872\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 140  is : 1.058330219942373\n",
            "Validation Accuracy at Epoch # 140  is : 0.7772435897435898\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 141  is : 1.0537785301397453\n",
            "Validation Accuracy at Epoch # 141  is : 0.7803485576923077\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 142  is : 1.0486959518391492\n",
            "Validation Accuracy at Epoch # 142  is : 0.7841546474358975\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 143  is : 1.0543269602568832\n",
            "Validation Accuracy at Epoch # 143  is : 0.7777443910256411\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 144  is : 1.0433078993172635\n",
            "Validation Accuracy at Epoch # 144  is : 0.7817508012820513\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 145  is : 1.0359705137761879\n",
            "Validation Accuracy at Epoch # 145  is : 0.7827524038461539\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 146  is : 1.0339653653142613\n",
            "Validation Accuracy at Epoch # 146  is : 0.7818509615384616\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 147  is : 1.0408826869128744\n",
            "Validation Accuracy at Epoch # 147  is : 0.7836538461538461\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 148  is : 1.031022960628385\n",
            "Validation Accuracy at Epoch # 148  is : 0.7840544871794872\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 149  is : 1.0336335875215508\n",
            "Validation Accuracy at Epoch # 149  is : 0.7832532051282052\n",
            "---------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "AcjI988f01YJ",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# **One Hidden Layer...**"
      ]
    },
    {
      "metadata": {
        "id": "AkzAFZ3guqOZ",
        "colab_type": "code",
        "outputId": "8ac556d5-ed1c-4919-b94c-7df73390a05e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1709
        }
      },
      "cell_type": "code",
      "source": [
        "'''\n",
        "    Training Over MNIST\n",
        "'''\n",
        "\n",
        "# LOAD DATASET\n",
        "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n",
        "\n",
        "# CONFIGURE\n",
        "lr = 1e-3\n",
        "batch_size = 128\n",
        "epochs = 30\n",
        "\n",
        "# CREATE GRAPH\n",
        "one_layered_nn_graph = tf.Graph()\n",
        "with one_layered_nn_graph.as_default():\n",
        "    \n",
        "    with tf.name_scope(\"placeholders\"):\n",
        "        x = tf.placeholder(name=\"x\",\n",
        "                        shape=(None, 784),\n",
        "                        dtype=tf.float32)\n",
        "        y = tf.placeholder(name=\"y\",\n",
        "                        shape=(None, 10),\n",
        "                        dtype=tf.float32)\n",
        "    \n",
        "    with tf.name_scope(\"Hidden-Layer\"):\n",
        "        W1 = tf.Variable(tf.truncated_normal([ 784, 128], mean=0.0, stddev=1.0, dtype=tf.float32), name=\"W1\")\n",
        "        b1 = tf.Variable(tf.zeros(128, dtype = tf.float32), name=\"b1\")\n",
        "        h1 = tf.nn.relu(tf.add(tf.matmul(x, W1), b1))\n",
        "        \n",
        "    with tf.name_scope(\"Output-Layer\"):\n",
        "        W2 = tf.Variable(tf.truncated_normal([ 128, 10], mean=0.0, stddev=1.0, dtype=tf.float32), name=\"W2\")\n",
        "        b2 = tf.Variable(tf.zeros(10, dtype = tf.float32), name=\"b2\")        \n",
        "        logits = tf.add(tf.matmul(h1, W2), b2)\n",
        "   \n",
        "    \n",
        "    loss_for_each_sample = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y) \n",
        "    loss = tf.reduce_mean(loss_for_each_sample)\n",
        "    optimizer = tf.train.AdamOptimizer(lr).minimize(loss) # Try Adam with bigger lr\n",
        "        \n",
        "# CREATE A SESSION TO RUN THE GRAPH\n",
        "with tf.Session(graph=one_layered_nn_graph) as sess:    \n",
        "    init = tf.global_variables_initializer()\n",
        "    sess.run(init)\n",
        "    for i in range(epochs): # EPOCHS  1ep is one iteration over complete Dataset.\n",
        "        \n",
        "        n_batches = int(mnist.train.num_examples / batch_size)\n",
        "        total_loss = 0\n",
        "        for j in range(n_batches):\n",
        "            x_batch , y_batch = mnist.train.next_batch(batch_size)\n",
        "            _, cur_loss_train = sess.run([optimizer, loss], feed_dict={x: x_batch, y: y_batch})\n",
        "            total_loss = total_loss + cur_loss_train\n",
        "        \n",
        "        print(\"Training loss at Epoch #\", i, \" is :\", total_loss / n_batches)\n",
        "        \n",
        "        n_batches = int(mnist.test.num_examples / batch_size)\n",
        "        total_acc = 0.0\n",
        "        for j in range(n_batches):\n",
        "            x_batch , y_batch = mnist.test.next_batch(batch_size)\n",
        "            preds = sess.run(logits, feed_dict={x: x_batch, y: y_batch})            \n",
        "            correct_preds = np.zeros_like(preds)\n",
        "            correct_preds[np.arange(len(preds)), preds.argmax(1)] = 1\n",
        "            acc = metrics.accuracy_score(y_true=y_batch.astype(int), y_pred=correct_preds.astype(int))\n",
        "            total_acc = total_acc + acc\n",
        "            \n",
        "            \n",
        "        print(\"Validation Accuracy at Epoch #\", i, \" is :\", total_acc / n_batches)\n",
        "        print('-' * 45)\n",
        "        "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
            "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
            "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
            "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
            "Training loss at Epoch # 0  is : 14.337095459580144\n",
            "Validation Accuracy at Epoch # 0  is : 0.8018830128205128\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 1  is : 3.512636625127637\n",
            "Validation Accuracy at Epoch # 1  is : 0.8633814102564102\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 2  is : 2.2932865445986215\n",
            "Validation Accuracy at Epoch # 2  is : 0.8822115384615384\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 3  is : 1.7213295957286319\n",
            "Validation Accuracy at Epoch # 3  is : 0.8964342948717948\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 4  is : 1.3752113148604794\n",
            "Validation Accuracy at Epoch # 4  is : 0.9066506410256411\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 5  is : 1.1322491291241767\n",
            "Validation Accuracy at Epoch # 5  is : 0.9137620192307693\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 6  is : 0.940759841639262\n",
            "Validation Accuracy at Epoch # 6  is : 0.9156650641025641\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 7  is : 0.799675856662658\n",
            "Validation Accuracy at Epoch # 7  is : 0.9195713141025641\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 8  is : 0.6822917355715117\n",
            "Validation Accuracy at Epoch # 8  is : 0.9235777243589743\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 9  is : 0.5875063279865088\n",
            "Validation Accuracy at Epoch # 9  is : 0.9267828525641025\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 10  is : 0.5032054275873449\n",
            "Validation Accuracy at Epoch # 10  is : 0.9293870192307693\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 11  is : 0.4358317202135122\n",
            "Validation Accuracy at Epoch # 11  is : 0.9311899038461539\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 12  is : 0.37351340729751925\n",
            "Validation Accuracy at Epoch # 12  is : 0.9338942307692307\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 13  is : 0.328104122793671\n",
            "Validation Accuracy at Epoch # 13  is : 0.9338942307692307\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 14  is : 0.28560141774029024\n",
            "Validation Accuracy at Epoch # 14  is : 0.9347956730769231\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 15  is : 0.2448418754072087\n",
            "Validation Accuracy at Epoch # 15  is : 0.9378004807692307\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 16  is : 0.2149087799016152\n",
            "Validation Accuracy at Epoch # 16  is : 0.9370993589743589\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 17  is : 0.1873212875429289\n",
            "Validation Accuracy at Epoch # 17  is : 0.9376001602564102\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 18  is : 0.16293393633147107\n",
            "Validation Accuracy at Epoch # 18  is : 0.9410056089743589\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 19  is : 0.1410655081451226\n",
            "Validation Accuracy at Epoch # 19  is : 0.940604967948718\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 20  is : 0.12447429602392591\n",
            "Validation Accuracy at Epoch # 20  is : 0.9424078525641025\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 21  is : 0.10443782632234822\n",
            "Validation Accuracy at Epoch # 21  is : 0.9424078525641025\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 22  is : 0.09386628561833382\n",
            "Validation Accuracy at Epoch # 22  is : 0.9433092948717948\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 23  is : 0.07885064271271625\n",
            "Validation Accuracy at Epoch # 23  is : 0.9425080128205128\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 24  is : 0.0679988590312529\n",
            "Validation Accuracy at Epoch # 24  is : 0.9441105769230769\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 25  is : 0.058726180370463954\n",
            "Validation Accuracy at Epoch # 25  is : 0.9452123397435898\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 26  is : 0.048976756512995545\n",
            "Validation Accuracy at Epoch # 26  is : 0.9451121794871795\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 27  is : 0.04128994989031948\n",
            "Validation Accuracy at Epoch # 27  is : 0.9459134615384616\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 28  is : 0.03472492881931988\n",
            "Validation Accuracy at Epoch # 28  is : 0.9465144230769231\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 29  is : 0.028523440622439118\n",
            "Validation Accuracy at Epoch # 29  is : 0.9456129807692307\n",
            "---------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "grBVaFFNEfcP",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# **Stack More Layers....**"
      ]
    },
    {
      "metadata": {
        "id": "2ZqvfHDj3vbV",
        "colab_type": "code",
        "outputId": "49cfe633-7d5b-4f07-86ea-776ef6c80535",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1709
        }
      },
      "cell_type": "code",
      "source": [
        "'''\n",
        "    Training Over MNIST\n",
        "'''\n",
        "\n",
        "# LOAD DATASET\n",
        "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n",
        "\n",
        "# CONFIGURE\n",
        "lr = 1e-3\n",
        "batch_size = 128\n",
        "epochs = 30\n",
        "\n",
        "# CREATE GRAPH\n",
        "mult_layered_nn_graph = tf.Graph()\n",
        "with mult_layered_nn_graph.as_default():\n",
        "    \n",
        "    def get_dense_hidden_layer(x_temp, in_shape=0, out_shape=0):\n",
        "        W_temp = tf.Variable(tf.truncated_normal([in_shape, out_shape], mean=0.0, stddev=1.0, dtype=tf.float32))\n",
        "        b_temp = tf.Variable(tf.zeros(out_shape, dtype = tf.float32))\n",
        "        return tf.nn.relu(tf.add(tf.matmul(x_temp, W_temp), b_temp))\n",
        "    \n",
        "    with tf.name_scope(\"placeholders\"):\n",
        "        x = tf.placeholder(name=\"x\",\n",
        "                        shape=(None, 784),\n",
        "                        dtype=tf.float32)\n",
        "        y = tf.placeholder(name=\"y\",\n",
        "                        shape=(None, 10),\n",
        "                        dtype=tf.float32)\n",
        "    \n",
        "    with tf.name_scope(\"Hidden-Layers\"):\n",
        "        h1 = get_dense_hidden_layer(x, in_shape=784, out_shape=256)\n",
        "        h2 = get_dense_hidden_layer(h1, in_shape=256, out_shape=256)\n",
        "        #h2 = tf.nn.dropout(h2)\n",
        "        h3 = get_dense_hidden_layer(h2, in_shape=256, out_shape=128)\n",
        "  \n",
        "    with tf.name_scope(\"Output-Layer\"):\n",
        "        W = tf.Variable(tf.truncated_normal([ 128, 10], mean=0.0, stddev=1.0, dtype=tf.float32), name=\"W\")\n",
        "        b = tf.Variable(tf.zeros(10, dtype = tf.float32), name=\"b\")        \n",
        "        logits = tf.add(tf.matmul(h3, W), b)\n",
        "   \n",
        "    loss_for_each_sample = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y) \n",
        "    loss = tf.reduce_mean(loss_for_each_sample)\n",
        "    optimizer = tf.train.AdamOptimizer(lr).minimize(loss) # Try Adam with bigger lr\n",
        "\n",
        "    \n",
        "    \n",
        "# CREATE A SESSION TO RUN THE GRAPH\n",
        "with tf.Session(graph=mult_layered_nn_graph) as sess:\n",
        "    init = tf.global_variables_initializer()\n",
        "    sess.run(init)\n",
        "    for i in range(epochs): # EPOCHS  1ep is one iteration over complete Dataset.\n",
        "        \n",
        "        n_batches = int(mnist.train.num_examples / batch_size)\n",
        "        total_loss = 0\n",
        "        for j in range(n_batches):\n",
        "            x_batch , y_batch = mnist.train.next_batch(batch_size)\n",
        "            _, cur_loss_train = sess.run([optimizer, loss], feed_dict={x: x_batch, y: y_batch})\n",
        "            total_loss = total_loss + cur_loss_train\n",
        "        \n",
        "        print(\"Training loss at Epoch #\", i, \" is :\", total_loss / n_batches)\n",
        "        \n",
        "        n_batches = int(mnist.test.num_examples / batch_size)\n",
        "        total_acc = 0.0\n",
        "        for j in range(n_batches):\n",
        "            x_batch , y_batch = mnist.test.next_batch(batch_size)\n",
        "            preds = sess.run(logits, feed_dict={x: x_batch, y: y_batch})            \n",
        "            correct_preds = np.zeros_like(preds)\n",
        "            correct_preds[np.arange(len(preds)), preds.argmax(1)] = 1\n",
        "            acc = metrics.accuracy_score(y_true=y_batch.astype(int), y_pred=correct_preds.astype(int))\n",
        "            total_acc = total_acc + acc\n",
        "            \n",
        "            \n",
        "        print(\"Validation Accuracy at Epoch #\", i, \" is :\", total_acc / n_batches)\n",
        "        print('-' * 45)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
            "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
            "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
            "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
            "Training loss at Epoch # 0  is : 625.074664278186\n",
            "Validation Accuracy at Epoch # 0  is : 0.8506610576923077\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 1  is : 161.51604860923786\n",
            "Validation Accuracy at Epoch # 1  is : 0.8849158653846154\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 2  is : 98.80372516258613\n",
            "Validation Accuracy at Epoch # 2  is : 0.9020432692307693\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 3  is : 67.28131949373615\n",
            "Validation Accuracy at Epoch # 3  is : 0.9120592948717948\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 4  is : 47.128619471907896\n",
            "Validation Accuracy at Epoch # 4  is : 0.9212740384615384\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 5  is : 33.958961840231616\n",
            "Validation Accuracy at Epoch # 5  is : 0.9243790064102564\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 6  is : 24.811649915587623\n",
            "Validation Accuracy at Epoch # 6  is : 0.9279847756410257\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 7  is : 18.685724080658012\n",
            "Validation Accuracy at Epoch # 7  is : 0.9287860576923077\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 8  is : 13.237552088748833\n",
            "Validation Accuracy at Epoch # 8  is : 0.9303886217948718\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 9  is : 10.063158452271152\n",
            "Validation Accuracy at Epoch # 9  is : 0.9326923076923077\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 10  is : 7.600745243424099\n",
            "Validation Accuracy at Epoch # 10  is : 0.9320913461538461\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 11  is : 5.914852345244186\n",
            "Validation Accuracy at Epoch # 11  is : 0.9354967948717948\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 12  is : 4.376567939286929\n",
            "Validation Accuracy at Epoch # 12  is : 0.9347956730769231\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 13  is : 3.6074758290393185\n",
            "Validation Accuracy at Epoch # 13  is : 0.9379006410256411\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 14  is : 2.923575993895008\n",
            "Validation Accuracy at Epoch # 14  is : 0.9402043269230769\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 15  is : 2.822014109846838\n",
            "Validation Accuracy at Epoch # 15  is : 0.9427083333333334\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 16  is : 2.8297010918525425\n",
            "Validation Accuracy at Epoch # 16  is : 0.9418068910256411\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 17  is : 1.8480470217593723\n",
            "Validation Accuracy at Epoch # 17  is : 0.9421073717948718\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 18  is : 1.9586131005537977\n",
            "Validation Accuracy at Epoch # 18  is : 0.9463141025641025\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 19  is : 2.4853893305080663\n",
            "Validation Accuracy at Epoch # 19  is : 0.9481169871794872\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 20  is : 1.6754423893526027\n",
            "Validation Accuracy at Epoch # 20  is : 0.946113782051282\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 21  is : 1.7978802843731339\n",
            "Validation Accuracy at Epoch # 21  is : 0.9446113782051282\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 22  is : 1.688811390717331\n",
            "Validation Accuracy at Epoch # 22  is : 0.9490184294871795\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 23  is : 1.5301093231686602\n",
            "Validation Accuracy at Epoch # 23  is : 0.948417467948718\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 24  is : 1.3112036228974964\n",
            "Validation Accuracy at Epoch # 24  is : 0.9498197115384616\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 25  is : 1.1827830741708762\n",
            "Validation Accuracy at Epoch # 25  is : 0.9493189102564102\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 26  is : 1.4635615734856726\n",
            "Validation Accuracy at Epoch # 26  is : 0.9521233974358975\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 27  is : 1.4679878471998815\n",
            "Validation Accuracy at Epoch # 27  is : 0.94921875\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 28  is : 1.4037162992649768\n",
            "Validation Accuracy at Epoch # 28  is : 0.9548277243589743\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 29  is : 1.1982791037382918\n",
            "Validation Accuracy at Epoch # 29  is : 0.9532251602564102\n",
            "---------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "RHXkcHb14sVS",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## **ConvNets -  Saving a Model**"
      ]
    },
    {
      "metadata": {
        "id": "CncZRJlwRQxL",
        "colab_type": "code",
        "outputId": "c30f031b-9028-407b-fbfa-8a5a8738b885",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 721
        }
      },
      "cell_type": "code",
      "source": [
        "'''\n",
        "    Training Over MNIST\n",
        "'''\n",
        "\n",
        "# LOAD DATASET\n",
        "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n",
        "\n",
        "import os\n",
        "if not os.path.exists('my_model_dir'):\n",
        "    os.makedirs('my_model_dir')\n",
        "\n",
        "# CONFIGURE\n",
        "lr = 1e-3\n",
        "batch_size = 128\n",
        "epochs = 3\n",
        "\n",
        "# CREATE GRAPH\n",
        "alexnet_graph = tf.Graph()\n",
        "with alexnet_graph.as_default():\n",
        "    \n",
        "    def get_dense_hidden_layer(x_temp, in_shape=0, out_shape=0):\n",
        "        W_temp = tf.Variable(tf.truncated_normal([in_shape, out_shape], mean=0.0, stddev=1.0, dtype=tf.float32))\n",
        "        b_temp = tf.Variable(tf.zeros(out_shape, dtype = tf.float32))\n",
        "        return tf.nn.relu(tf.add(tf.matmul(x_temp, W_temp), b_temp))\n",
        "    \n",
        "    def get_conv_layer(x_temp, filters=None, strides=None, padding='SAME', name=None):\n",
        "        conv_temp = tf.nn.conv2d(input=x_temp, filter=tf.Variable(tf.truncated_normal(filters, mean=0.0, stddev=0.01, dtype=tf.float32)), strides=strides, padding=padding, name=name)\n",
        "        conv_temp = tf.nn.bias_add(conv_temp, tf.Variable(tf.constant(0.0, shape=[filters[-1]])), name=name+'_bias')\n",
        "        return conv_temp\n",
        "    \n",
        "    with tf.name_scope(\"placeholders\"):\n",
        "        x = tf.placeholder(name=\"x\",\n",
        "                        shape=(None, 28, 28, 1),\n",
        "                        dtype=tf.float32)\n",
        "        y = tf.placeholder(name=\"y\",\n",
        "                        shape=(None, 10),\n",
        "                        dtype=tf.float32)\n",
        "    \n",
        "    with tf.name_scope(\"Hidden-Conv-Layers\"):\n",
        "        # 1st Conv Layer\n",
        "        conv1 = get_conv_layer(x, filters=[5,5,1,32], strides=[1,1,1,1], padding='SAME', name='conv1')\n",
        "        conv1 = tf.nn.relu(conv1)\n",
        "        conv1 = tf.nn.max_pool(conv1, ksize=[1,3,3,1], strides=[1,2,2,1], padding='VALID')\n",
        "        \n",
        "        # 2nd Conv Layer\n",
        "        conv2 = get_conv_layer(conv1, filters=[5,5,32,64], strides=[1,1,1,1], padding='SAME', name='conv2')\n",
        "        conv2 = tf.nn.relu(conv2)\n",
        "        conv2 = tf.nn.max_pool(conv2, ksize=[1,3,3,1], strides=[1,2,2,1], padding='VALID')\n",
        "        \n",
        "  \n",
        "    with tf.name_scope(\"Hidden-Dense-Layers\"):            \n",
        "        flatten = tf.reshape(conv2, [-1, 6*6*64])\n",
        "        fc1 = get_dense_hidden_layer(flatten, in_shape=6*6*64, out_shape=1000)\n",
        "        fc2 = get_dense_hidden_layer(fc1, in_shape=1000, out_shape=128)\n",
        "        \n",
        "        \n",
        "    with tf.name_scope(\"Output-Layer\"):\n",
        "        W = tf.Variable(tf.truncated_normal([128, 10], mean=0.0, stddev=1.0, dtype=tf.float32), name=\"W\")\n",
        "        b = tf.Variable(tf.zeros(10, dtype = tf.float32), name=\"b\")\n",
        "        logits = tf.add(tf.matmul(fc2, W), b)\n",
        "   \n",
        "    loss_for_each_sample = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y) \n",
        "    loss = tf.reduce_mean(loss_for_each_sample)\n",
        "    optimizer = tf.train.AdamOptimizer(lr).minimize(loss) # Try Adam with bigger lr\n",
        "\n",
        "    \n",
        "# CREATE A SESSION TO RUN THE GRAPH\n",
        "with tf.Session(graph=alexnet_graph) as sess:\n",
        "    \n",
        "    init = tf.global_variables_initializer()\n",
        "    sess.run(init)\n",
        "    for i in range(epochs): # EPOCHS  1 epoch is one iteration over complete Dataset.\n",
        "        \n",
        "        n_batches = int(mnist.train.num_examples / batch_size)\n",
        "        total_loss = 0\n",
        "        for j in range(n_batches):\n",
        "            x_batch , y_batch = mnist.train.next_batch(batch_size)\n",
        "            x_batch = x_batch.reshape(x_batch.shape[0], 28, 28, 1)\n",
        "            _, cur_loss_train = sess.run([optimizer, loss], feed_dict={x: x_batch, y: y_batch})\n",
        "            total_loss = total_loss + cur_loss_train\n",
        "            \n",
        "        print(\"Training loss at Epoch #\", i, \" is :\", total_loss / n_batches)\n",
        "        saver = tf.train.Saver()\n",
        "        saver.save(sess, 'my_model_dir/' + 'model.ckpt')\n",
        "        \n",
        "        \n",
        "        n_batches = int(mnist.test.num_examples / batch_size)\n",
        "        total_acc = 0.0\n",
        "        for j in range(n_batches):\n",
        "            x_batch , y_batch = mnist.test.next_batch(batch_size)\n",
        "            x_batch = x_batch.reshape(x_batch.shape[0], 28, 28, 1)\n",
        "            preds = sess.run(logits, feed_dict={x: x_batch, y: y_batch})            \n",
        "            correct_preds = np.zeros_like(preds)\n",
        "            correct_preds[np.arange(len(preds)), preds.argmax(1)] = 1\n",
        "            acc = metrics.accuracy_score(y_true=y_batch.astype(int), y_pred=correct_preds.astype(int))\n",
        "            total_acc = total_acc + acc\n",
        "            \n",
        "            \n",
        "        print(\"Validation Accuracy at Epoch #\", i, \" is :\", total_acc / n_batches)\n",
        "        print('-' * 45)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-4-1c3ec398d6d1>:7: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please write your own downloading logic.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/base.py:252: _internal_retry.<locals>.wrap.<locals>.wrapped_fn (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use urllib or similar directly.\n",
            "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.data to implement this functionality.\n",
            "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
            "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.data to implement this functionality.\n",
            "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.one_hot on tensors.\n",
            "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
            "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
            "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
            "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
            "Training loss at Epoch # 0  is : 1.0199407245573047\n",
            "Validation Accuracy at Epoch # 0  is : 0.9517227564102564\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 1  is : 0.14487366156874004\n",
            "Validation Accuracy at Epoch # 1  is : 0.9663461538461539\n",
            "---------------------------------------------\n",
            "Training loss at Epoch # 2  is : 0.11579370234348314\n",
            "Validation Accuracy at Epoch # 2  is : 0.9711538461538461\n",
            "---------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "aHk8qqUoFmdO",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## **Restoring a Model**"
      ]
    },
    {
      "metadata": {
        "id": "1ogfWeG7Ftov",
        "colab_type": "code",
        "outputId": "a621ffea-2deb-4db9-851a-3798e830d110",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "cell_type": "code",
      "source": [
        "'''\n",
        "1.tensorflowModel.ckpt.meta: Tenosrflow stores the graph structure separately from the variable values. The file .ckpt.meta contains the complete graph. It includes GraphDef, SaverDef, and so on.\n",
        "\n",
        "2.tensorflowModel.ckpt.data-00000-of-00001: This contains the values of variables(weights, biases, placeholders, gradients, hyper-parameters etc).\n",
        "\n",
        "3.tensorflowModel.ckpt.index: It is a table where Each key is the name of a tensor and its value is a serialized BundleEntryProto.\n",
        "'''\n",
        "\n",
        "\n",
        "batch_size = 128\n",
        "lr = 1e-3\n",
        "\n",
        "# CREATE GRAPH\n",
        "alexnet_graph = tf.Graph()\n",
        "with alexnet_graph.as_default():\n",
        "    def get_dense_hidden_layer(x_temp, in_shape=0, out_shape=0):\n",
        "        W_temp = tf.Variable(tf.truncated_normal([in_shape, out_shape], mean=0.0, stddev=1.0, dtype=tf.float32))\n",
        "        b_temp = tf.Variable(tf.zeros(out_shape, dtype = tf.float32))\n",
        "        return tf.nn.relu(tf.add(tf.matmul(x_temp, W_temp), b_temp))\n",
        "    \n",
        "    def get_conv_layer(x_temp, filters=None, strides=None, padding='SAME', name=None):\n",
        "        conv_temp = tf.nn.conv2d(input=x_temp, filter=tf.Variable(tf.truncated_normal(filters, mean=0.0, stddev=0.01, dtype=tf.float32)), strides=strides, padding=padding, name=name)\n",
        "        conv_temp = tf.nn.bias_add(conv_temp, tf.Variable(tf.constant(0.0, shape=[filters[-1]])), name=name+'_bias')\n",
        "        return conv_temp\n",
        "    \n",
        "    with tf.name_scope(\"placeholders\"):\n",
        "        x = tf.placeholder(name=\"x\",\n",
        "                        shape=(None, 28, 28, 1),\n",
        "                        dtype=tf.float32)\n",
        "        y = tf.placeholder(name=\"y\",\n",
        "                        shape=(None, 10),\n",
        "                        dtype=tf.float32)\n",
        "    \n",
        "    with tf.name_scope(\"Hidden-Conv-Layers\"):\n",
        "        # 1st Conv Layer\n",
        "        conv1 = get_conv_layer(x, filters=[5,5,1,32], strides=[1,1,1,1], padding='SAME', name='conv1')\n",
        "        conv1 = tf.nn.relu(conv1)\n",
        "        conv1 = tf.nn.max_pool(conv1, ksize=[1,3,3,1], strides=[1,2,2,1], padding='VALID')\n",
        "        \n",
        "        # 2nd Conv Layer\n",
        "        conv2 = get_conv_layer(conv1, filters=[5,5,32,64], strides=[1,1,1,1], padding='SAME', name='conv2')\n",
        "        conv2 = tf.nn.relu(conv2)\n",
        "        conv2 = tf.nn.max_pool(conv2, ksize=[1,3,3,1], strides=[1,2,2,1], padding='VALID')\n",
        "        \n",
        "  \n",
        "    with tf.name_scope(\"Hidden-Dense-Layers\"):            \n",
        "        flatten = tf.reshape(conv2, [-1, 6*6*64])\n",
        "        fc1 = get_dense_hidden_layer(flatten, in_shape=6*6*64, out_shape=1000)\n",
        "        fc2 = get_dense_hidden_layer(fc1, in_shape=1000, out_shape=128)\n",
        "        \n",
        "        \n",
        "    with tf.name_scope(\"Output-Layer\"):\n",
        "        W = tf.Variable(tf.truncated_normal([128, 10], mean=0.0, stddev=1.0, dtype=tf.float32), name=\"W\")\n",
        "        b = tf.Variable(tf.zeros(10, dtype = tf.float32), name=\"b\")\n",
        "        logits = tf.add(tf.matmul(fc2, W), b)\n",
        "   \n",
        "    loss_for_each_sample = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y) \n",
        "    loss = tf.reduce_mean(loss_for_each_sample)\n",
        "    optimizer = tf.train.AdamOptimizer(lr).minimize(loss) # Try Adam with bigger lr\n",
        "\n",
        "    \n",
        "# CREATE A SESSION TO RUN THE GRAPH\n",
        "with tf.Session(graph=alexnet_graph) as sess:\n",
        "    sess.run(tf.initialize_all_variables())\n",
        "    \n",
        "    ckpt = tf.train.get_checkpoint_state('my_model_dir')\n",
        "    print(ckpt.model_checkpoint_path)\n",
        "    \n",
        "    saver = tf.train.Saver()\n",
        "    saver.restore(sess, ckpt.model_checkpoint_path)\n",
        "        \n",
        "    x_batch , y_batch = mnist.test.next_batch(batch_size)\n",
        "    x_batch = x_batch.reshape(x_batch.shape[0], 28, 28, 1)\n",
        "    preds = sess.run(logits, feed_dict={x: x_batch, y: y_batch})\n",
        "    \n",
        "    correct_preds = np.zeros_like(preds)\n",
        "    correct_preds[np.arange(len(preds)), preds.argmax(1)] = 1\n",
        "    acc = metrics.accuracy_score(y_true=y_batch.astype(int), y_pred=correct_preds.astype(int)) * 100\n",
        "    \n",
        "    print(acc)\n",
        "    \n",
        "    \n",
        "            \n",
        "\n",
        "        \n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "my_model_dir/model.ckpt\n",
            "INFO:tensorflow:Restoring parameters from my_model_dir/model.ckpt\n",
            "96.09375\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "ESz-NGJ_pO97",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## **Layers Visualizations**"
      ]
    },
    {
      "metadata": {
        "id": "f2bLLGMfNLT2",
        "colab_type": "code",
        "outputId": "3e52989c-b182-425f-e7ac-828da5c45fcb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 706
        }
      },
      "cell_type": "code",
      "source": [
        "import matplotlib as mp\n",
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "\n",
        "def plot_nn_filter(image, units):\n",
        "    filters = units.shape[3]\n",
        "    \n",
        "    plt.figure(1, figsize=(15,15))\n",
        "    n_columns = 6\n",
        "    n_rows = np.ceil(filters / n_columns) + 2\n",
        "\n",
        "    plt.subplot(n_rows, n_columns, 1)\n",
        "    plt.title('Original Image ')\n",
        "    plt.imshow(x_batch[0,:,:,0], interpolation=\"nearest\", cmap=\"gray\")\n",
        "    \n",
        "    for i in range(filters):\n",
        "        plt.subplot(n_rows, n_columns, i+2)\n",
        "        plt.title('Filter ' + str(i))\n",
        "        plt.imshow(units[0,:,:,i], interpolation=\"nearest\", cmap=\"gray\")\n",
        "        \n",
        "batch_size = 1\n",
        "lr = 1e-3\n",
        "\n",
        "# CREATE GRAPH\n",
        "alexnet_graph = tf.Graph()\n",
        "with alexnet_graph.as_default():\n",
        "    def get_dense_hidden_layer(x_temp, in_shape=0, out_shape=0):\n",
        "        W_temp = tf.Variable(tf.truncated_normal([in_shape, out_shape], mean=0.0, stddev=1.0, dtype=tf.float32))\n",
        "        b_temp = tf.Variable(tf.zeros(out_shape, dtype = tf.float32))\n",
        "        return tf.nn.relu(tf.add(tf.matmul(x_temp, W_temp), b_temp))\n",
        "    \n",
        "    def get_conv_layer(x_temp, filters=None, strides=None, padding='SAME', name=None):\n",
        "        conv_temp = tf.nn.conv2d(input=x_temp, filter=tf.Variable(tf.truncated_normal(filters, mean=0.0, stddev=0.01, dtype=tf.float32)), strides=strides, padding=padding, name=name)\n",
        "        conv_temp = tf.nn.bias_add(conv_temp, tf.Variable(tf.constant(0.0, shape=[filters[-1]])), name=name+'_bias')\n",
        "        return conv_temp\n",
        "    \n",
        "    with tf.name_scope(\"placeholders\"):\n",
        "        x = tf.placeholder(name=\"x\",\n",
        "                        shape=(None, 28, 28, 1),\n",
        "                        dtype=tf.float32)\n",
        "        y = tf.placeholder(name=\"y\",\n",
        "                        shape=(None, 10),\n",
        "                        dtype=tf.float32)\n",
        "    \n",
        "    with tf.name_scope(\"Hidden-Conv-Layers\"):\n",
        "        # 1st Conv Layer\n",
        "        conv1 = get_conv_layer(x, filters=[5,5,1,32], strides=[1,1,1,1], padding='SAME', name='conv1')\n",
        "        conv1 = tf.nn.relu(conv1)\n",
        "        conv1 = tf.nn.max_pool(conv1, ksize=[1,3,3,1], strides=[1,2,2,1], padding='VALID')\n",
        "        \n",
        "        # 2nd Conv Layer\n",
        "        conv2 = get_conv_layer(conv1, filters=[5,5,32,64], strides=[1,1,1,1], padding='SAME', name='conv2')\n",
        "        conv2 = tf.nn.relu(conv2)\n",
        "        conv2 = tf.nn.max_pool(conv2, ksize=[1,3,3,1], strides=[1,2,2,1], padding='VALID')\n",
        "        \n",
        "  \n",
        "    with tf.name_scope(\"Hidden-Dense-Layers\"):            \n",
        "        flatten = tf.reshape(conv2, [-1, 6*6*64])\n",
        "        fc1 = get_dense_hidden_layer(flatten, in_shape=6*6*64, out_shape=1000)\n",
        "        fc2 = get_dense_hidden_layer(fc1, in_shape=1000, out_shape=128)\n",
        "        \n",
        "        \n",
        "    with tf.name_scope(\"Output-Layer\"):\n",
        "        W = tf.Variable(tf.truncated_normal([128, 10], mean=0.0, stddev=1.0, dtype=tf.float32), name=\"W\")\n",
        "        b = tf.Variable(tf.zeros(10, dtype = tf.float32), name=\"b\")\n",
        "        logits = tf.add(tf.matmul(fc2, W), b)\n",
        "   \n",
        "    loss_for_each_sample = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y) \n",
        "    loss = tf.reduce_mean(loss_for_each_sample)\n",
        "    optimizer = tf.train.AdamOptimizer(lr).minimize(loss) # Try Adam with bigger lr\n",
        "\n",
        "    \n",
        "# CREATE A SESSION TO RUN THE GRAPH\n",
        "with tf.Session(graph=alexnet_graph) as sess:\n",
        "    sess.run(tf.initialize_all_variables())\n",
        "    \n",
        "    ckpt = tf.train.get_checkpoint_state('my_model_dir')\n",
        "    print(ckpt.model_checkpoint_path)\n",
        "    \n",
        "    saver = tf.train.Saver()\n",
        "    saver.restore(sess, ckpt.model_checkpoint_path)\n",
        "        \n",
        "    x_batch , y_batch = mnist.test.next_batch(batch_size)\n",
        "    x_batch = x_batch.reshape(x_batch.shape[0], 28, 28, 1)\n",
        "    conv1_results, conv2_results  = sess.run([conv1, conv2], feed_dict={x: x_batch, y: y_batch})\n",
        "    \n",
        "    plot_nn_filter(x_batch, conv1_results)\n",
        "    "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "my_model_dir/model.ckpt\n",
            "INFO:tensorflow:Restoring parameters from my_model_dir/model.ckpt\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0QAAAKNCAYAAAAAiavmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3XmcLHV97//XCCgIIuLCniBCPiTO\niVe54pKgRxFFjKJhCQpcRVxikEsW9d4k+ovwc4kLEgLE5UqEi2IEBUFjlICgGHCJXpMh4gdFhchB\ngSDoMVzkYN8/qqdPzzndMz1V1Wu9no8HD6qru6uqv/M+357P1Le+NddqtZAkSZKkJnrAuA9AkiRJ\nksbFgkiSJElSY1kQSZIkSWosCyJJkiRJjWVBJEmSJKmxLIgkSZIkNdaW49hpRMwB/x14BbAVRWF2\nJfDGzLy9z3uuAF6fmd9YZrtvB27KzPeVPK43A7tn5is2Wb8n8N3MHEt7aTQiogXcCGzoWn1TZj5n\nMX/A9sAHM3PviNgJeFJmXlpxv48D3gs8ArgD+P3M/Ncq29RojTE7c8DrgLcBz8jML1XZnsZjjPn5\nLeA97W3/J/BHmfnFKtvUaI0xO08H3gk8lCI7f2h2ps+48tO1/8cB/wwclJlX1bHNssb1C/5bgWcB\nz83MH0bElu11V0XEf83MezZ9Q2YeuNJGM/NP6z9UNczazPzhpisX8xcRa7tWP4Mix1U7hr8D/jQz\nPxkRLwA+AqypuE2N3jiy815gC+C2itvR+I00PxHxIOAS4IjMvDIiDgE+CuxWdpsam1FnZxvgE8Bz\nMvPrEXEocEFE7JKZ3txy+ozju4uIeADFd9iPqm6rDiMviCJiR+APgf+y+APIzA3A/4iIA4FjgQ9E\nxA+AvwWOBg4Cvggck5lfiog/a2/jJuBDwBsyc8+IOIfiTM5b2u9/O3A8sAdwfmb+SfsYXgH8CcXn\nvxU4NjNvWsVnaAGvojjLtQPwUuCVwFOBbwHPz8wN7V9u3wo8EFgPHJ+Z32yH4HTgCOC7wKcoisO1\nEbEDcAbwpPbx/f+Z+aFBj03D0c7TMV2PnwCcCWwZEdtl5lHtL4W3ANtS/Fxfkpl3tM887gY8jiKH\nf9W1nTXADpn5SYDMvDQi/ldE/HpmXj+aT6dhGlZ22s7NzGvb+9AMGmJ+tgJelZlXth9/Cdg1InbI\nzLuG/LE0AkPMzgMpfp/5evvxFcBOFL8P/WSoH0ojM+TvLoDfB74J/GKYn2NQ47iG6MnAzZl5Q4/n\nPgU8vevx7pkZmXnz4oqIeCzwBopGPgA4cpl9PQ14CrAfcGJE7B4Rj6L4gR6UmftQ/ADfVOJzPCIz\n1wAfo/hLyV8Av0bxl/2nt896nQu8MjOD4i9x726/9xDgucDewAuAl3Vt91Tgl8C+FEXRyRExX+L4\nNETtoZtnAh9vdwp7AecBL87MvSiGgHYP3TwEOKRHp/BrwPc2Wfc9ip+/ZlCN2SEzrx3FMWty1JWf\nzFyfmRd1rXoucIPF0OyqMTt3Z+Yl0Bm2ezxwdWZaDM2wOr+7ImJn4CTgz4Z/5IMZx5C5HYGe1wkB\nP6YomBZ9usdrngZclZm3AkTE3wKn9Nne+Zl5P7AuIn4M7NH+a+r2mblYkV5NcVZqtT7Z/v8CcONi\ngRcR3wF2bZ8helRm3te1n5e1lw8APp2Z69vv+SjwzPZzzwcOzsxfArdHxEXA7wLXlThGrd5VEdE9\nlvbqzHzlAO87mCKXiz+n9wE/jogt2o+/kpl39Hjfg4H/u8m6eyj+2qLpMursaLaMLT8R8ZvAacBL\nVnvQmghjyU5EHE7xC/JdFL+naDqNIz9/BZySmXdFRLmjrtk4CqI7gF37PLcTS8fC39njNQ/bZP0t\ny+zr7q7l+4Et2j+oU9rD2bYAHgL0Olu1kp91bXf9pvtpL//3iHgp8CBga2BxbO3DgO7xmt2fYQeK\nsbiL4dwGuLDE8amcnmNpB7AD8LSI+HbXuruBh7eXe2UZ4OcU2ej2YJZmStNh1NnRbBlLfiLiqcAF\nwCvGfVGzShtLdjLz48DHI+KZwJUR8bjMnIjrQbQqI81PRDwHeHhmfqTEPodmHAXRtcCO7X84/7LJ\nc79Dcf3Mcn4KbNf1eJdV7v/3KIapPa09zvGVFNcp1ar9JfM/gP0z8wcRcRDwv9pPL/cZ1gEv7Kq4\nNR3WAZdn5uGbPrHCXz++DTym67VzFEMpv1X3AWpilc2OBBXy0z4zdCFwVGZePZzD0wQrlZ2I2APY\nr+va189HxA8pRvh8su8bNWvK9j0vAh4fEYvF847ARRHxh5n5v+s/zMGM/BqizLybYqKB8yLi0QAR\nsWUUU2ZvQTHj1nK+CjwjIh7RniXnpas8hEcBP2gXQw+nuAZpuxXeU8ajKM523RwRD6Y4zm3bv/B+\nFfidiNimPYlC93VQl1BcaLbYLqe1L2TT5LmP4i8kAJ8DDmiPqSUi9o+I01faQGZ+i2Jo5OJQlZdS\nTHlZ5qylpkfl7KjRKuen/V10LvAHFkONUkff80DgnPY13UTEPhR/yPu3IRyvJksdv/f8fmY+IjN3\nzsydgWuA3x1nMQRjujFrZr4b+ADwqfaptm9RVIjP6rq2p997v0rRif8f4PMUEzGsZprHjwIPj4jv\ntpffCOwREaeu+oMs77MU1fONwGUU4yXvBj4OXEwx73pSTMhwARs/w5uAh0ZEUnQuWwDek2YyXQY8\nMyK+1r6m7ZXAxRFxPcW46o8NuJ2XUAyv/A7FvblqP2OpiVNLdiLiunYfuhvwkYj4dkTsP7Sj1qSo\nIz9PBn4TeEc7N4v/+Qe42VY5O5l5Y/t9H233P5cCJ2Xmd4Z43JoMdf3eM3HmWq3pmzI+IuayPdd9\nRDwPeEtmPn7Mh7Uqm3yGEyiKwReN+bAkSZKkRhnXjVlLi4hHAot/xbqZYrjZVE09GxH/BfhkRDye\nYnKG36U49ShJkiRphMYyZK6KzLwd+HOKG4HdQDHU7s3jPKbVysxvUgz7+zpwPcUsc2eO9aAkSZKk\nBprKIXOSJEmSVIfSQ+Yi4jSKizJbFBfTfa22o5IkSZKkESg1ZC4ing7sk5lPAY4H/rrWo5IkSZKk\nESh7huhA2jffyszrI+JhEbF9Zv6014vn5uZaAAsLC6xZs6bkLqfXMD93q9WaG8qGh2w1ZxgX8wPN\nzVAvdbTFNOanbHbA/CxqanbAvqcOTc2PfU91Tc0O2PfUYZj5KTupws7A7V2Pb2+v62lhYYFWq8X8\n/DytVqtx/w3zc0+jKmcY5+fnh3Zc06aJbVH17HQT26yXpraDfU89mtgW9j31aGo72PfUY5htUdcs\nc8tW62vWrGFurnjJ3Nxc4/4b5ueeUkvOMAIPi4jtx3tImhJmR1WYH5VldlSF+ZlwZYfMrWPpGaFd\ngVurH44aYmeKKccXLZ5h7DnkcmFhYclfBab1zNgwNLAtKmUHGtlmPTW0Hex7atLAtrDvqUlD28G+\npybDaouyBdFlwMnA+9s3SF2XmT+r77DUMCueYVzUarWm+cxYrepoixnoZAfODpifRWanw76nBPMD\n2PeUYnY67HtKGGZ+Sg2Zy8xrgK9HxDUU4yBPKH9oaiDPMKoss6MqzI/KMjuqwvxMuNLXEGXm/8zM\np2bmb2fmv9R5UJp5lwGHA3iGUatkdlSF+VFZZkdVmJ8JNzeKU4+L0wc29bTfMD/3FE8/+ZfA04Bf\nAicsV1R3Tz/Z1Az1UtOp46lrzLLZAfOzqKnZAfueOjQ1P/Y91TU1O2DfU4dh5seCqGaPe9zjOssn\nn3wyAIceeuiS17z2ta/tLJ911lmV9jetHcNq2DH01uQvlkH5S0lvZmcw9j29mZ+V2ff0ZnYGY9/T\n2zDzU9e025IkSZI0dSyIJEmSJDVW2Wm3G+8BD9hYS5555pmd5eOPP76zvNVWW3WWu4cmbrPNNkM+\nOkmSJEmDsCCSJElSaYcddlip11599dU9X/PTn/a8X6k0NA6ZkyRJktRYFkSSJEmSGsshc6uw++67\nd5YvvvjizvJ+++234nvf//73d5ZPP/30eg9MkiRJUimeIZIkSZLUWBZEkiRJkhrLIXMr2HfffTvL\nF154YWf5sY99bGd5/fr1neXuoXFnn302ANdffz0nnXRSZ/199903lGOVJEmStDoWRJIkSVrRgx/8\n4J7r5+bm+r6n+z6Mm+q+X6M0Tg6ZkyRJktRYniHaxN57773k8RVXXNFZ3mWXXTrLd955Z2f5iCOO\n6CxfeeWVPbf7i1/8oq5DlCRJklQTzxBJkiRJaiwLIkmSJEmN5ZA5YNddd+0sX3vttUuee/jDH95Z\n/vnPf95ZXrt2bWf5uuuuG97BSZIkSRoazxBJkiRJaizPEEmSJGlF/abQvuaaawZ+T/drb7311noO\nTKqosQXRzjvv3Fm+/PLLO8vdQ+Rg6TC5F77whZ1lh8lJkiRJ088hc5IkSZIaa6AzRBExD1wCnJaZ\nZ0bEHsB5wBbArcCxmXnv8A5TkiRJkuq3YkEUEdsCZwBXdK0+BTgrMy+MiLcBLwfeO5xDrE/3MLkv\nfOELneV99tmn73uOPvroznL3TVolSZIkTb9BhszdCxwCrOtatxa4tL38KeBZ9R6WJEmSJA3fimeI\nMnMDsCEiuldv2zVE7jZgl+W2sbCwwPz8PNB/hpJJdckll9SynWn73JIkSYP4yU9+0ve5e+65Z8lj\nZ5bTJKpjlrm5lV6wZs0aoCgK5uZWfHmtHvjAB3aWv/nNb3aW9913356vP+mkk5Y8/pu/+ZvO8v33\n31/qGIb5uaex0IqItcCFwL+1Vy1k5onjOyJNC7OjKsyPyjI7qsL8TL6yBdH6iNgmM+8BdmPpcDpp\nEF/IzMPHfRCaSmZHVZgflWV2VIX5mWBlp92+HDisvXwY8Nl6DkeSJEmSRmeQWeb2A04F9gTui4jD\ngaOBcyLi1cBNwLnDPMjV2nLLjR/r3e9+d2e53zC5U089tbP83vcunSyv3zC5hz3sYZ3l3XbbrbP8\nnve8p7O84447dpZf9KIXdZYvvvjivsfeIL8REZcCOwInZ+Y/9nth9zVoMJ3DBIeloW1ROjvQ2Dbb\nTIPbwb6nBg1tC/ueGjS4Hex7ajCsthhkUoWvU8wqt6mDaj8aNcV3gJOBC4C9gCsjYu/M/EWvFy9e\ngwbjuQ5tUtXRFlPYyZbODpifRQ3NDtj31KKh+bHvAbbZZptVv6d7UoWGZgfse2oxzPzUMamCtCqZ\neQvwsfbDGyPiRxTXon1/fEelaWB2VIX5UVlmR1WYn8k3kwXRwQcf3Fl+7Wtf2/M13X+xeMtb3tJZ\n3rBhQ9/t/uqv/mpn+YILLugsP/GJT1zxmM4///zO8jHHHNNZ/sQnPrHie2dNRBwN7JKZ746InYGd\ngFvGfFiaAmZHVZgflWV2CptOoa3BmJ/JN5MFkSbepcD5EXEo8EDgNf1OG0ubMDuqwvyoLLOjKszP\nhLMg0shl5s+A54/7ODR9zI6qMD8qy+yoCvMz+WamIHrMYx7TWe4entbtF7/YWIw/6UlP6izffffd\nneXuGeoAjjrqqM7yySef3Fl+9KMfvarje9CDHtRZ3n///TvLTRwyJ0mSJE2KsvchkiRJkqSpZ0Ek\nSZIkqbEsiCRJkiQ11lRfQ9R9c6Y3vvGNneXtttuus9w9jfbhhx/eWb7uuus6yw94wMa6sHs6bYAX\nvvCFPff9ne98p7N87bXXdpY/9rGPdZZf9apXLbsNSZIkSePlGSJJkiRJjWVBJEmSJKmxpnrI3OMf\n//jO8ktf+tKer/ne977XWf70pz/d8zUXX3xxZ/n5z+8/Tfyb3vSmzvLZZ5/dWf7Rj37UWX7uc5/b\nWX7Oc57TWf7Zz37WWf7MZz7Tdx+SJEmSRsczRJIkSZIay4JIkiRJUmNN9ZC5I444ouf6n/70p53l\ngw46qOdrjjnmmM7ys5/97L77OOOMMzrLb3/72zvLv/zlLzvL8/PzneV3v/vdneWbb74ZgIjgBS94\nQWf9F77whb77kyRJkjQ6U10QSZIkaXJtvfXWfR9vv/32Pd/Tfd21NAoOmZMkSZLUWFN9hqjfkLl7\n7rmns/zv//7vneUDDzyws3zOOed0lrtvzHr66acv2dbrX//6nq/bf//9O8vnnntuZ3nLLTc26dFH\nHw0UN251mJwkSZI0eTxDJEmSJKmxLIgkSZIkNdZUD5m74YYbOst77bVXZ/kRj3hEZ7n7ZqpveMMb\nOsvdw9+6XXrppUsen3TSSZ3lI488srP8xCc+sbP85S9/ubP8vOc9r7P8k5/8ZPkPIEmSJGmsprog\nkiRJmgXHHXdcqfd96EMfqvlI+uu+TrrbQx7ykL7v2WKLLZY83m677Vbcz7333ru6A5MqGqggioh3\nAge0X/924GvAecAWwK3AsZlpeiVJkiRNlRULooh4BjCfmU+JiIcD/we4AjgrMy+MiLcBLwfeO9xD\n3dypp57aWT744IM7y91/jTj55JNXtc0rrrii73PdN2P94he/2Fl+4Qtf2Fm+6667VrU/SZIkSeMz\nyKQKXwQW57e+C9gWWAssXmzzKeBZtR+ZJEmSJA3ZimeIMvN+4Ofth8cDnwGe0zVE7jZgl+W2sbCw\nwPz8PACtVqv0wY5b95mnpz/96Z3lQSZPmObPLUmSJM2qgSdViIhDKQqiZwPf6XpqbqX3rlmzBiiK\ngrm5FV8+sO4L80444YTO8l/8xV90lrfeeutVbfP+++9f8vi2227rLL/mNa/pLG86G91y6v7cm25b\nkiRJUjkD3YcoIp4D/Dnw3My8G1gfEdu0n94NWDek45MkSZKkoRlkUoWHAu8CnpWZd7ZXXw4cBny4\n/f/PDu0INdUiYh64BDgtM8+MiD1whkINyPyoLLOjKsaRnyc96Ul9n9t///37Pve3f/u3q97XunX9\n/479D//wD32f+6d/+qee6y+77LK+71luCu077rij5/ruSaymjX3PdBpkyNzvAY8ALoiIxXUvBT4Y\nEa8GbgLOHc7hLW/9+vWd5Xe84x2d5QsvvLCzvDhcD+Dwww/vLL/4xS/uLF900UWd5U3n81+uY9Dy\nImJb4AyKWQkXncIEzFCoyWd+VJbZURXmR2WZnem14pC5zPxAZu6amWu7/rspMw/KzAMy85jMvG8U\nB6upcy9wCEuHVK7FGQo1GPOjssyOqjA/KsvsTKmBJ1WQViszNwAbus4sAmy7mhkK1VzmR2WZHVVh\nflSW2ZleM1kQfe973+u5fMkll3SWjz322JEek3oaaOq97mnbwZn1ujW8LVbMz6bZgca3WUfD28G+\np6KGt8XU9z277rpr3+eOP/74Us8N6vbbb6+8jSlm31PRsNpiJgsiTbT1EbFNZt7DgDMUdl8HNswp\nzKdNHW0xhZ3sqvLTnR0wP4vMjn1PFeZnOH3P+973vr7PLTepwuMf//hlt9vLuCZVuP3223nkIx/Z\neXznnXf2esuykyrMenbAvqefYfY9A027LdVocYZCcIZCrZ75UVlmR1WYH5VldqbA3Cgq7bm5uRY0\nt8od8o1ZJ7ZBI2I/4FRgT+A+4BbgaOAcYGuKGQqPW2lSjsX8QHMz1EtNfymZ2MasIz/d2QHzs8js\n2PdUYX6G0/ccd9xxfZ878cQT+z5X5gzRcs4+++y+z5U5Q3TLLbd0ls2OfU8Vw8yPQ+Y0NJn5dYrZ\nVTZ10IgPRVPI/Kgss6MqzI/KMjvTyyFzkiRJkhrLgkiSJElSY1kQSZIkSWosCyJJkiRJjeUscyPQ\n1Fnm6uJsK73N+mw9dXCWud7MzmDse3ozPyuru+/Zfffd+z73kpe8pOf6hz70oX3f88///M99n7v2\n2mv7Ptfvpqr3339/3/d0MzuDse/pbZj58QyRJEmSpMayIJIkSZLUWBZEkiRJkhrLgkiSJElSY1kQ\nSZIkSWosCyJJkiRJjTWSabclSZIkaRJ5hkiSJElSY1kQSZIkSWqsLcd9AHWKiBZwI7Cha/VNmfmc\niLgCeD2wPfDBzNw7InYCnpSZl9aw3+xadUtmHlhlmxq9MebnIcDZwJOB/wT+PDM/UWWbGq1xZCci\nngJ8aJPVjwGekJkLZber0Rtj3/M84K3A1sB/AH+UmV+tsk2N1hizczDwl8AOwL8Bx2bmnVW2qeEb\nY17mgNcBbwOekZlf6nruKOCNwFbAdcDLM/PuKvsrY6YKora1mfnDTVcuFigRsbZr9TOAZwGVftDt\n7e9bdRuaCOPIz3uAW4FfBX4NeF9EXJKZG5Z/mybMSLOTmdcCnX4nIp4EnEHxhaLpM9L8RMQOwPnA\nAZn5r+1fcD8B7FF2mxqbUWfnkcBHKX6x/WZEvB14F3B82W1qpMbxe857gS2A27pXRsSvUHxv7ZeZ\nN0fEqRR/pHltxf2t2iwWRD1FxA+AY7oePwE4E9gyIrbLzKMi4lDgLcC2wHeBl2TmHRHxZmA34HHA\n+Zn5VyM+fI3ZsPITEQ8CXgzslZmLZxqfMZIPpZEYYd9zOvAn7RxpRgwxP3sB/5mZ/9p+/Hlg94jY\nITPvGvLH0ggMMTtPAb6Tmd9sPz4NuAELoqk25O+qczPz2vY+uh0KXJGZN7cfnw1cyRgKosZeQ5SZ\n36D4QX+8/UPeCzgPeHFm7kXxA3lf11sOAQ7p9wtJRHw4Ir4VEV+MiKcO+/g1XjXmZx/gHuBl7fx8\nNSKeNYKPoDGpu++BztCnezLz6iEeuiZAjfm5Hrg/Ip7Zfnw48M8WQ7Orxuy0KP7av+jnwEMj4hHD\nO3qNWp3fVe0RDb38GsUQvkU3Ao+KiIfV8RlWYxbPEF0VEd1Dja7OzFcO8L6Dgasyc3G4yfuAH0fE\n4j/6r2TmHX3e+7+AM9vDDo4EPhURj/GLZSqNOj87tP/7v5n5GxHxHODjEbGX47Gnzjj6nkVvAN65\nusPVhBlpfjLznoh4FfD3EXEPxR9ID65w/BqfUfc91wL7RMSBFGcW/5jimpStyx2+Rmyc31WbejBd\nw+gy8972dU7bAj9Z5bYqGVlBFBGnUVw03gJOysyvDWlXPcdGDmAH4GkR8e2udXcDD28v9/3lNDNf\n1bV8QUScCVwbEeuBtwNfo6iqt6C4VuTYzLy3xDE22ogyNOr83E2Ri/cCZObnIuJmis/5me4XRsQ8\ncAlwWmaeGRF7YK4GMqPZASAidgfmgc+u8DrzU9Is5icidqUYnrJ/Zi60rxu4OCL2ycz1m7zW7JQ0\ni9lpD5E6kuK6oa2AD1KMdOh5Ibz5KW9I+RnLd1UfP6erkI6IrYE5YH378ciyM5IhcxHxdGCfzHwK\nxRjTvx7FfldpHXB5Zu7b9d8jM/O25d4UEdtFRHQ9fgawDfDfKarpvwJOAc7KzAMoxly+fGifYkZN\nQYZK5Qf49/b/H9K17v72fx0RsS3FhYdXdK02VwOY4ewseh7wj5l5f78XmJ/yZjg/TwW+l+0ZCTPz\nKop+59e7X2R2ypvh7JCZn83MJ2TmGuCTwH9k5s82fZ35KW8C81P1u6qXbwN7dz3eB7g1M+8adXZG\ndQ3RgRT/YMjM64GHRcT2I9r3cu6jqHgBPgcc0B4jSUTsHxGnD7CNPSjOBi3+QB9E8ZeSrwB3UZz2\nW8vGGTo+RTFjh1ZnEjNUOT/tYZWfo5iOcnGmsD0pzip2u5difO66rnVrMVeDmMnsdHkcxfUgyzE/\n5c1qfm4AHhsRe7bf9wTgoSwdzw9mp4qZzE5EbB8RGRG/EsV0ym8CzunzcvNT3iTkp87vql4uAQ7s\nOrHwxxQzGMKIszOqgmhn4Paux7e3143bZcAzI+JrmXkr8EqKIQPXU1xI9rGVNtAO6R8Cl7ZPI/4F\ncGhm/pSiov8MsG3XKb3bgF3q/ygzbxIzVDk/bccD+7dnX/kA8HubXj+UmRsy855N3meuBjPL2QHY\nHfjRci8wP5XMZH6ymF3ufwL/EBFJ8QvtMfY9tZrV7PyUYma5LwA/aK9+W5/Xmp/yJiE/tXxXRcR1\n7d+RdwM+EhHfjoj9M/MW4A+AT0bEdyiuKfoLGH12xjWpwtwwNpqZfbebmXt2Pdy7ve5rwI5dr7mU\nHnOtZ+abV9jv/wb+d/e6KKYmPB54NvCdrqeG8tkbqPZ2HGN+1lH9rxzmanAzk532a16wikPtx/wM\nbmbyk5nvY+ksUWWYncGZnc2Zn8FVbqsx5mV+mecuAC5Y7v191JqdUZ0hWsfSqnZXiouhZlJ7prA/\nB56bxd1210fENu2nd2Pp6T8NplEZGpC5GozZ6c38DMb8bM7sDMbs9GZ+BmN+Nje07IyqILqM4h4H\ni+OU1/W6+G4WRMRDKWZe+Z2uoQeXA4e1lw9jhdmg1FNjMrQK5mowZqc38zMY87M5szMYs9Ob+RmM\n+dnc0LIz12qN5qbmEfGXwNOAXwInZOa/jGTHI9a+r8ObKS5YXfRSimkptwZuAo7LzPtGf3TTrSkZ\n6iUi9gNOpZhw4T7gFuBoinH/5moFTc4OmJ+qmpwfs1NNk7MD5qeqJudn1NkZWUEkSZIkSZNmVEPm\nJEmSJGnilJ5lbjV3z52bm1tyGmphYYE1a9aU3fVMqdoWrVZrKmdoKZsfs7NRHW0xjfmx76muqdkB\n+546NDU/9j39feITn+j73O/+7u/2fW5ubvUxmMbsgH1PHYbZ95Q6Q1T17rnz831n32ucJrZFlfw0\nsb36aWJb2PfUo6ntYN9Tjya2hX2PqrDvqccw26LskLlJuHuuppf5UVlmR1WYH5VldlSF+ZlwZYfM\n7Qx8vevx4t1zf9rrxQsLC5tVdU7msFED26JSfhrYXn01sC3se2rS0Haw76lJA9vCvmcIGtQm9j01\nGVZblL6GaBPLjufcdLxfq9UqNW50FlVtixn5RzJwfszORnW0xQzkx76nBLPTYd9TgvkB7HuWGPE1\nRKt+zwSy7ylhmH1P2SFz3j1XVZgflWV2VIX5UVlmR1WYnwlX9gzRZcDJwPu9e65KMD8qy+yoCvOj\nsszOMg477LC+z73oRS/qLF900UXLnjGaYeZnwpU6Q5SZ1wBfj4hrKGbKOKHWo9JMMz8qy+yoCvOj\nssyOqjA/k29uFGMxN52P3/GQG9VwDdHMN2R3fszORjWNpZ3pxrTv6c3sDMa+pzfzszL7no2WO0N0\n8cUXr3p7s54dsO/pZ5h9T9n11tqCAAAgAElEQVRriCRJkiRp6lkQSZIkSWosCyJJkiRJjeU1RGPm\nNUQrcyxtb47jX5l9T29mZzD2Pb2Zn5XZ9/RmdgZj39Ob1xBJkiRJ0hBYEEmSJElqLAsiSZIkSY1l\nQSRJkiSpsSyIJEmSJDWWBZEkSZKkxrIgkiRJktRYFkSSJEmSGsuCSJIkSVJjWRBJkiRJaiwLIkmS\nJEmNZUEkSZIkqbEsiCRJkiQ11pbjPgBJkiRNrwsuuKDUa4888shhHI60ap4hkiRJktRYFkSSJEmS\nGsuCSJIkSVJjWRBJkiRJaiwLIkmSJEmNZUEkSZIkqbFKTbsdEWuBC4F/a69ayMwT6zoozTbzo7LM\njqowPyrL7KgK8zP5qtyH6AuZeXhtR6KmMT8qy+yoCvOjssyOqjA/E8whc5IkSZIaq8oZot+IiEuB\nHYGTM/Mf+71wYWGB+fn5JetarVaFXc+WhrZF6fw0tL16amhb2PfUoMHtYN9Tg4a2hX1PDY444ojO\ncsPaxL6nBsNqi7kyG46I3YDfBi4A9gKuBPbOzF/03Mnc3JKdtFot5ubmVn+0M6hqW7RaralryCr5\nMTsb1dEW05Yf+556NDE7YN9Tlybmx75neRdccMFArzviiCO48MILO4+PPPLIVe9r2rID9j11GWbf\nU+oMUWbeAnys/fDGiPgRsBvw/XKHpyYxPyrL7KgK86OyzI6qMD+Tr+wsc0cDu2TmuyNiZ2An4JZa\nj0wzy/yoLLOjKsyPyjI7yxv0TE+r1Sp1VmjamZ/JV3bI3EOA84EdgAdSjIX8TN+dNOzU8Wo0dMhc\n6fyYnY0aOmzFvqcGTcwO2PfUpYn5se+pRxOzA/Y9dRlmfkoVRKtlx9BfEwui1bJj6K2pXyyrYd/T\nm9kZjH1Pb+ZnZfY9vZmdwdj39DbM/DjttiRJkqTGsiCSJEmS1FgWRJIkSZIay4JIkiRJUmNZEEmS\nJElqLAsiSZIkSY1lQSRJkiSpsSyIJEmSJDWWBZEkSZKkxrIgkiRJktRYFkSSJEmSGsuCSJIkSVJj\nWRBJkiRJaiwLIkmSJEmNZUEkSZIkqbEsiCRJkiQ1lgWRJEmSpMayIJIkSZLUWBZEkiRJkhrLgkiS\nJElSY1kQSZIkSWosCyJJkiRJjWVBJEmSJKmxLIgkSZIkNdaWg7woIuaBS4DTMvPMiNgDOA/YArgV\nODYz7x3eYWpamR1VYX5UltlRFeZHZZmd6bTiGaKI2BY4A7iia/UpwFmZeQDwXeDlwzk8TTOzoyrM\nj8oyO6rC/KgsszO9Bhkydy9wCLCua91a4NL28qeAZ9V7WJoRZkdVmB+VZXZUhflRWWZnSq04ZC4z\nNwAbIqJ79bZdp/tuA3YZwrFpypkdVWF+VJbZURXmR2WZnek10DVEK5hb6QULCwvMz88vWddqtWrY\n9WxocFusmB3YPD8Nbq/NNLwt7HsqaHg72PdU1PC2sO+poOHtYN9T0bDaomxBtD4itsnMe4DdWHpq\ncDNr1qxZ8rjVajE3N1AmZl7VtpjCfySryg4szY/Z2aiOtpj1/Nj39GZ27HuqMD/2PWWZHfueKoaZ\nn7LTbl8OHNZePgz4bMntqHnMjqowPyrL7KgK86OyzM4UmFup0o6I/YBTgT2B+4BbgKOBc4CtgZuA\n4zLzvr47mZtbshOr3Y1qOEM0sQ1ZR3ZgaX7MzkY1/aVkYhvTvmd4zI59TxXmx76nLLNj31PFMPOz\nYkFUBzuG/ma5IKqLHUNvs/7FUgf7nt7MzmDse3ozPyuz7+nN7AzGvqe3Yean7JA5SZIkSZp6FkSS\nJEmSGsuCSJIkSVJjWRBJkiRJaiwLIkmSJEmNZUEkSZIkqbEsiCRJkiQ1lgWRJEmSpMayIJIkSZLU\nWBZEkiRJkhrLgkiSJElSY821Wq1xH4MkSZIkjYVniCRJkiQ1lgWRJEmSpMbactwHUFVEtIAbgQ1d\nq2/KzOdExBXA64HtgQ9m5t4RsRPwpMy8tOJ+54DXAW8DnpGZX+p6bjvg/cDvZebUt/GsmtDsvBL4\nQ2AL4AfAKzLzh1X2p+GY0Py8BngtRd/+feCVmfnvVfan4ZjE/HS95rXAGZk5V2VfGo5Jy05ErAU+\nA9zc9fKLM/NPq+xP9Zu07LSf+w3gQ8AjgP8AXpaZ36qyvzJm5Zf1tb1+aczMA6Hzj3XRM4BnAZV+\nuMB7KX5pva3Hc9cAn664fY3GxGQnIp4InAzsl5m3RsS7gHcAR1fcn4ZnkvLzVIovnP+amT+JiNOA\nU4EjK+5PwzMx+VkUEbsAr6q4Dw3fpGXnq5m5tsd6TZ6JyU5EbAFcBPxZZl4UEccArwD+uOL+Vm1W\nCqKeIuIHwDFdj58AnAlsGRHbZeZREXEo8BZgW+C7wEsy846IeDOwG/A44PzM/KtNNn9uZl7b3sem\nXg3cCryh1g+kkRlTdm4HjsrMW9uPrwZOqfNzaTTGlJ/bgGMz8yftx1cAb63zc2k0xvjdBXB6e7sf\nq+0DaWTGnB1NsTFl56nAhsy8CCAzPwx8uOaPNpBGXUOUmd+g+OF+vP2D3Qs4D3hxZu4FXAm8r+st\nhwCH9PjBkpnXLrOfvs9pOo0iO5n5g8z8Yteq5wJfqeszaHxGlJ/vZuY1ABGxDcWZxUvq/SQah1F9\nd0XEc4HtM/OCWj+AxmZU2QF+JSI+FxEZER+PiN1q/BgagxFl53HATRFxTkTcEBF/HxGPrveTDGZW\nzhBdFRHd4yGvzsxXDvC+g4GrMvO69uP3AT9un8ID+Epm3lHngWriTGR2IuJYioLoyWW3oZGYuPxE\nxDuB3we+BLyzzDY0MhOTn3YRfSrw/NW8T2MzMdmhGBFzEcUQ77uAd1P84vzMVW5HozFJ2dkBeBrF\nsLyXU4yKOQ/47VVup7KRFkTtMe1PBlrASZn5tZo23XM85AB2AJ4WEd/uWnc38PD28p2Vj2wTETFP\n8Vfb0zLzzIjYg+KHvwVFp3JsZt5b935nwZDyM3HZiYg/oBg/+8zM/FHXerNTUpP6nsx8Q0T8GUWG\nLqddVJuf8hqSn/8P+Ehm3rjpE2anvCZkJzOT4vpFACLiZOCOiNg2M39ufsprwO89dwPfzMyvAETE\ne4A/G0d2RlYQRcTTgX0y8ykR8evA3wJPGdX++1gHXJ6Zh2/6RETUvrOI2BY4g2Js/6JTgLMy88KI\neBtFhfze2nc+5SYwP0PJTkS8jGKWsKdl5rqu9WanpAnMDgwhPxGxP/CAzPxyZm6IiPcC74iIHYD7\nMD+lNCU/wAuAR0TEiV3b+hFwEPDXmJ1Va0p2opiJbMvMvKW9akuKX+A3+N1V3gTmZxj9zk3AQ7se\n37/4/1FnZ5TXEB0IfBIgM68HHhYR249w/4vuo6hyAT4HHNAeF0lE7B8Rpw9x3/dSjLFc17VuLRtn\n7/gUxWlDbW4S8jPU7LTHXL8dOLi7GGozO+VNQnZg+H3PvsAHImLxy+X5wM2ZeRfmp4pG5CczH5uZ\nO2Xmzpm5c3vdzsD1mJ2yGpEd4FDgoihuOQJwEnBF+y/39j3lTUJ+hp2dK4BdIuLZ7cevAv4pM/8v\nI87OKIfM7Qx8vevx7e11Px3hMQBcBvxJRHwtM58YxX1fLo6IBwI/o7gHzIoi4jqK9tsN+EhE3AP8\nN4q53c8HtgK2WDy1mJn7ZuYGir+YdG9q267TfbcBu1T+hLNpEvIz7OwcCGwHXNaVkQ2ZOW92KpmE\n7MDw83MesA/wlSju+XAX7Sm3zU8ljchPZn611+vNTiWNyA7wQeDXgG9GxP3At4DjwPxUNAn5GXq/\nExEvAt4fEQ+iOGP0Mhh9dsY5qUItN3zLZW4cl5l7dj3cu73ua8COXa+5lB7zq2fmm1fY7/wyT++7\n3HuX4U3wBle5rSYwO1+lOENUhtkZ3Cz3PW9q/7da5mdws5yfgY5vE2ZncLOcndfRdR3RKpifwc3i\n7z1kcZPWxy73/j5qzc4oh8yto6hsF+1KcUFU062PYnYfKCrnTYdKqWB+Nmd2BmN2ejM/gzE/mzM7\ngzE7vZmfwZifzQ0tO6MsiC4DDofOzZ7WZebPRrj/SXU5cFh7+TDgs2M8lklmfjZndgZjdnozP4Mx\nP5szO4MxO72Zn8GYn80NLTtzrVarrm2tKCL+kmK+8V8CJ2Tmv4xs5xMgIvajuM/DnhQXqt1CcfPE\nc4CtKcZOHpeZ943pECdak/NjdqppcnbA/FTV5PyYnWqanB0wP1U1OT+jzs5ICyJJkiRJmiSjHDIn\nSZIkSROl9CxzsYq7587NzS05DbWwsMCaNWvK7nqmVG2LVqs1lTO0lM2P2dmojraYxvzY91TX1OyA\nfU8dmpof+57qmpodsO8BuPjii3uuf+ELX9j3Pd/+9rc7y49+9KP5/ve/33n84Q9/uOd73vrWt/bd\nXr/8lDpD1H33XOB4irtYD2x+fqBZPxuhiW1RJT9NbK9+mtgW9j31aGo72PfUo4ltYd9Tj6a2g31P\nPR70oAcNbdtlh8xNwt1zNb3Mj8oyO6rC/Kgss6MqzM+EKztkblV3z11YWNiswnUyh40a2BaV8tPA\n9uqrgW1h31OThraDfU9NGtgW9j01aWg72PeUtO+++/Z9/Ja3vKXne/qtX07pa4g2sex4zk3HPrZa\nLebmpnIIaO2qtsWM/CMZOD9mZ6M62mIG8mPfU4LZ6bDvKcH8APY9pZidjkb2PVWvIdp3332XPC55\nDVHP9WWHzHn3XFVhflSW2VEV5kdlmR1VYX4mXNmCyLvnqgrzo7LMjqowPyrL7KgK8zPhSg2Zy8xr\nIuLrEXEN7bvn1ntYmmXmR2WZHVVhflSW2VEVTcrP2Wef3fe5Rz3qUSM8ktUpfQ1RZv7POg9EzWJ+\nVJbZURXmR2WZHVVhfiZb2SFzkiRJkjT1LIgkSZIkNZYFkSRJkqTGsiCSJEmS1Fh13ZhVkiRJ0ow7\n8cQTS72v+6aq3X7rt35roPe3Wi1+/dd/vfO4zhvWeoZIkiRJUmNZEEmSJElqLAsiSZIkSY1lQSRJ\nkiSpsSyIJEmSJDWWBZEkSZKkxpprtVrD38nc3JKdtFqtWqfKm2ZV26LVas18Q3bnx+xsVEdbzHp+\n7Ht6MzuDse/pzfyszL6nN7MzmEnoe0444YS+zz3gAeXOp5xxxhllDwcYbn48QyRJkiSpsSyIJEmS\nJDWWBZEkSZKkxrIgkiRJktRYFkSSJEmSGsuCSJIkSVJjbTnuA5AkSZI0es9+9rN7rl9uau1f/vKX\nfZ8766yz+j63xx579Fz/rne9q+97NvV3f/d3neWjjjpq4PetxDNEkiRJkhrLgkiSJElSY1kQSZIk\nSWosCyJJkiRJjWVBJEmSJKmx5lqt1qrfFBFrgQuBf2uvWsjME/vuZG5uyU5arRZzc3Or3u8sqtoW\nrVZr6hqySn7MzkZ1tMW05ce+px5NzA7Y99Slifmx76lHE7MD4+97nvzkJ/d9brvttlv19i6//PK+\nz5133nl9nzvyyCNXva/777+/s7zNNttwzz33dB7//d//fc/3HHHEEX231y8/Vabd/kJmHl7h/Wo2\n86OyzI6qMD8qy+yoCvMzwRwyJ0mSJKmxqpwh+o2IuBTYETg5M/+x3wsXFhaYn59fsq7MUL1Z1dC2\nKJ2fhrZXTw1tC/ueGjS4Hex7atDQtrDvqUGD28G+pwbbbLNNZ/nww3ufcCvTXmWvIdoN+G3gAmAv\n4Epg78z8Rc+dOJa2r4ZeQ1Q6P2ZnoyaOxbbvqUcTswP2PXVpYn7se+rRxOzA+PseryHaqNZriDLz\nFuBj7Yc3RsSPgN2A75fZnprF/Kgss6MqzI/KMjuqwvxMvlLXEEXE0RHxuvbyzsBOwC11Hphml/lR\nWWZHVZgflWV2VIX5mXxlryG6FDg/Ig4FHgi8pt9pP6kH86OyzI6qMD8qy+yoionNz/r163uu//KX\nv9z3PR/4wAdK7euCCy7ouX6HHXbo+54DDzyw1L5Wq+yQuZ8Bz6/5WNQQ5kdlmR1VYX5UltlRFeZn\n8jnttiRJkqTGsiCSJEmS1FgWRJIkSZIay4JIkiRJUmNZEEmSJElqrLLTbkuSJEmacMtNod3PVltt\n1fe5z3/+832f+/GPf9z3uSuvvHLVx9Gt1Wrx4Ac/uPP4MY95TKXtdfMMkSRJkqTGsiCSJEmS1FgW\nRJIkSZIay4JIkiRJUmNZEEmSJElqLAsiSZIkSY0112q1hr+TubklO2m1WszNzQ19v9Ogalu0Wq2Z\nb8ju/Jidjepoi1nPj31Pb2ZnMPY9vZmfldn39GZ2BjPKvufVr371qt+z8847l9rXDTfc0HP9Rz/6\n0YHeP8z8eIZIkiRJUmNZEEmSJElqLAsiSZIkSY1lQSRJkiSpsSyIJEmSJDXWluM+AEmSJEmz7cYb\nbxz3IfTlGSJJkiRJjWVBJEmSJKmxLIgkSZIkNZYFkSRJkqTGsiCSJEmS1FgDzTIXEfPAJcBpmXlm\nROwBnAdsAdwKHJuZ9w7vMDWtzI6qMD8qy+yoCvOjsszOdFqxIIqIbYEzgCu6Vp8CnJWZF0bE24CX\nA+8dziFqWpkdVWF+VJbZURXmR2VNY3be//73j/sQJsIgQ+buBQ4B1nWtWwtc2l7+FPCseg9LM8Ls\nqArzo7LMjqowPyrL7EypFc8QZeYGYENEdK/etut0323ALkM4Nk05s6MqzI/KMjuqwvyoLLMzvQa6\nhmgFcyu9YGFhgfn5+SXrWq1WDbueDQ1uixWzA5vnp8HttZmGt4V9TwUNbwf7nooa3hb2PRU0vB3s\neyoaVluULYjWR8Q2mXkPsBtLTw1uZs2aNUset1ot5uYGysTMq9oWU/iPZFXZgaX5MTsb1dEWs54f\n+57ezI59TxXmx76nLLNj31PFMPNTdtrty4HD2suHAZ8tuR01j9lRFeZHZZkdVWF+VJbZmQJzK1Xa\nEbEfcCqwJ3AfcAtwNHAOsDVwE3BcZt7Xdydzc0t2YrW7UQ1niCa2IevIDizNj9nZqKa/lExsY9r3\nDI/Zse+pwvzY95Rldux7qhhmflYsiOpgx9DfLBdEdbFj6G3Wv1jqYN/Tm9kZjH1Pb+ZnZfY9vZmd\nwdj39DbM/JQdMidJkiRJU8+CSJIkSVJjWRBJkiRJaiwLIkmSJEmNZUEkSZIkqbEsiCRJkiQ1lgWR\nJEmSpMayIJIkSZLUWBZEkiRJkhrLgkiSJElSY1kQSZIkSWosCyJJkiRJjTXXarXGfQySJEmSNBae\nIZIkSZLUWFuO+wCqiogWcCOwoWv1TZn5nIi4Ang9sD3wwczcOyJ2Ap6UmZdW3O8c8DrgbcAzMvNL\n7fUnAa/peulWwA6Z+fAq+9NwTFp+2s/9KfBSoAVcD/xBZv6oyv5UvwnNzv+gyM52wMeBP8lMhwFM\nmDFm57eA97S3/Z/AH2XmF9vPHQW8keI76zrg5Zl5d5X9aTgmND/bAe8Hfi8zp/53y1k1odl5JfCH\nwBbAD4BXZOYPq+yvjFkJ7dpejZeZBwJExNqu1c8AngVU+uEC76X44d22yT5PB05ffBwRbwB2qrgv\nDdfE5CciDgJeDvzXzLw7Iv4SeDdwTMX9aTgmKTvPBV4B7A/8HPhHitycV3F/Go6RZiciHgRcAhyR\nmVdGxCHAR4HdIuJXgDOA/TLz5og4FXgr8Nqy+9PQTUx+2i+5Bvh02e1rpCYmOxHxROBkir7n1oh4\nF/AO4Oiy+ytrVgqiniLiB3T9IhkRTwDOBLaMiO0y86iIOBR4C7At8F3gJZl5R0S8meIf+uOA8zPz\nrzbZ/LmZeW17H/32vxPF2aLH1/WZNDpjys8a4J+7/jL7eeCddX4uDd+YsnMQcHFm/qS9z7OAl2BB\nNFWGmJ2tgFdl5pXtx18Cdo2IHYBDgSsy8+b2c2cDV2JBNHXGkZ/MvAt4NXAr8IZhfj4Nz5j6ntuB\nozLz1vZzVwOnDOkjLqtR1xBl5jcofrgfb/9g96L4ZeHFmbkXxRfA+7recghwSI9fSMjMawfY5euA\nc9qdhabciPJzFfDUiNg9IrYEXkTxl35NsRFlp0Vx5mjRemDvOo5f41NXdjJzfWZe1LXqucAN7e+n\nX6MYRrPoRuBREfGw+j+RRmlE+Rn0dyJNkVFkJzN/sDh0ruu5rwzj86xkVs4QXRUR3eMhr87MVw7w\nvoOBqzLzuvbj9wE/jojFXyq+kpl3lDmgiHgo8N+Ax5Z5v0ZqYvKTmd+IiHMpxtH+HPghcMBqtqGR\nmpjsUBTOH4yI04A7gVcCW69yGxqdsWUnIn4TOI3iDCLAg+kagpmZ97avNdgW+MkAx6TRm6T8aLpM\nZHYi4liKgujJAxxL7UZaELW/qJ9M8ZfMkzLzazVtuud4yAHsADwtIr7dte5uYHEChDsrHNPv0CMc\nETFPMZbytMw8MyL2oKi4t6A43XxsZt5bYb8zqwn5iYgXAM+juO7sTuDPgA8Dh5id8pqQncz8bET8\nNXA5xS+xFwF7LD5vfsobUn7Gkp2IeCpwAcWFy1e1V/+cruI5IrYG5ijOMpqdCmat7+mTn2WZn/Ia\n0PcsPvcHwB8Dz8yuSaRGmZ2RDZmLiKcD+2TmU4Djgb8e1b6XsQ64PDP37frvkZl524rvXNnvAJ/p\nXhER21JcuHpF1+pTgLMy8wCK8Zgvr2HfM6dB+Xk28NnM/I8sZgf7GPB0s1Neg7JDZr6zva2nAD8G\nFsC+p4oJzE/p7LT/OnshxZCX7u+nb7N0eOU+wK2ZeZfZKW8CswPDyc9y7zE/JU1gfoaSnYh4GcX1\nik/LzO91rR9pdkZ5DdGBwCcBMvN64GERsf0I97/oPooqF+BzwAHtcZFExP4RcXrfd67O4yimTO52\nL8UYy3Vd69aycfaOT1HM5qHNNSU/CRwYEQ9uP34exRS4Zqe8RmQnItZGxJUR8cCIeAjwR8C57afN\nT3mTkJ/K2YliuvZzKabxv3qTpy+h6Hei/fiPKWaBArNTxSRkB4afn+WYn/ImIT9DzU5E7Aa8HTg4\nM9dt8taRZmeUQ+Z2Br7e9fj29rqfjvAYAC4D/iQivpaZT4xi/vOLI+KBwM8o5kJfUURcR9F+uwEf\niYh7gP+WmV9tv2R3YMm9YzJzA7Bh43cOANt2ne67Ddil5OeadY3ID8WY3AD+NSLup8jQcWankqZk\n52rgBuA7wC8phhhcBfY9FU1CfurIzpOB3wTeERHv6Fr/kva1i38AfDKKyVy+AZwIZqeiScgODDk/\n7f+fTzGb2BaLQ6raZxDMT3mTkJ9hZ+c5FPfNu6wrIxsyc37U2RnnpApzdWwkM/tuJzP37Hq4d3vd\n14Adu15zKT3mV8/MN6+w3/kVni9TxdfSJg0xy/kpM9Wt2RncLGfn1cu9dxnmZ3CV22oc2cliFrAt\nlnn+Aorx/atldgY3tX3PSvkB9l3mueWYn8HNYt/zDYozRGXUmp1RDplbR1HZLtqV4oKoplsfEdu0\nl3dj6alBbWR+Nmd2BmN2ejM/gzE/mzM7gzE7vZmfwZifzQ0tO6MsiC4DDofOzZ7WZebPRrj/SXU5\ncFh7+TDgs2M8lklmfjZndgZjdnozP4MxP5szO4MxO72Zn8GYn80NLTtzrVarrm2tKCL+Engaxfj2\nEzLzX0a28wkQEfsBpwJ7UlyodgtwNHAOxZSnN1FcL3LfmA5xojU5P2anmiZnB8xPVU3Oj9mppsnZ\nAfNTVZPzM+rsjLQgkiRJkqRJMsohc5IkSZI0UUrPMreau+fOzc0tOQ21sLDAmjVryu56plRti1ar\nNZUztJTNj9nZqI62mMb82PdU19TsgH1PHZqaH/ue6pqaHbDvqcMw81PqDFHVu+fOzy87Y3WjNLEt\nquSnie3VTxPbwr6nHk1tB/ueejSxLex76tHUdrDvqccw26LskLlJuHuuppf5UVlmR1WYH5VldlSF\n+ZlwZYfMreruuQsLC5tVdU7msFED26JSfhrYXn01sC3se2rS0Haw76lJA9vCvqcmDW0H+56aDKst\nSl9DtIllx3NuOt6v1WoxNzeVQ0BrV7UtZuQfycD5MTsb1dEWM5Af+54SzE6HfU8J5gew7ynF7HTY\n95QwzPyUHTLn3XNVhflRWWZHVZgflWV2VIX5mXBlCyLvnqsqzI/KMjuqwvyoLLOjKszPhCt9Y9bV\n3D130+knPf23UQ1D5qayIcvmx+xsVNOp46lrTPue6pqaHbDvqUNT82PfU11TswP2PXUYZn5KF0Sr\nYcfQX1MLotWwY+ityV8sg7Lv6c3sDMa+pzfzszL7nt7MzmDse3obZn7KDpmTJEmSpKlnQSRJkiSp\nsSyIJEmSJDWWBZEkSZKkxrIgkiRJktRYFkSSJEmSGsuCSJIkSVJjWRBJkiRJaiwLIkmSJEmNZUEk\nSZIkqbEsiCRJkiQ1lgWRJEmSpMayIJIkSZLUWBZEkiRJkhrLgkiSJElSY1kQSZIkSWosCyJJkiRJ\njWVBJEmSJKmxLIgkSZIkNZYFkSRJkqTGsiCSJEmS1FgWRJIkSZIay4JIkiRJUmNZEEmSJElqrC3L\nvCki1gIXAv/WXrWQmSfWdVCabeZHZZkdVWF+VJbZURXmZ/KVKojavpCZh9d2JGoa86OyzI6qMD8q\ny+yoCvMzwRwyJ0mSJKmxqpwh+o2IuBTYETg5M/+xpmNSM5gflWV2VIX5UVlmR1WYnwk212q1Vv2m\niNgN+G3gAmAv4Epg78z8Ra/XX3fdda35+fkqx6n+5sZ9AKtlfibKVOXH7EyUqcoOmJ8JM1X5MTsT\nZaqyA+ZnwvTMT6mCaFMR8VXg9zLz+z13Mje3ZCetVou5uanL81BUbYtWqzX1Dbma/Jidjepoi2nP\nj31POWanYN9Tjvmx7ynL7BTse8oZZn5KXUMUEUdHxOvayzsDOwG3lD88NYn5UVlmR1WYH5VldlSF\n+Zl8ZYfMPQQ4H9gBeCDFWMjP9N2Jfynpq4lniKrkx+xs1MS/tNn31KOJ2QH7nro0MT/2PfVoYnbA\nvqcuw8xPLUPmVmLH0InOpU8AACAASURBVF8TC6LVsmPoralfLKth39Ob2RmMfU9v5mdl9j29mZ3B\n2Pf0NnFD5iRJkiRpFlgQSZIkSWosCyJJkiRJjWVBJEmSJKmxLIgkSZIkNZYFkSRJkqTGsiCSJEmS\n1FgWRJIkSZIay4JIkiRJUmNZEEmSJElqLAsiSZIkSY1lQSRJkiSpsSyIJEmSJDWWBZEkSZKkxrIg\nkiRJktRYFkSSJEmSGsuCSJIkSVJjWRBJkiRJaiwLIkmSJEmNZUEkSZIkqbEsiCRJkiQ1lgWRJEmS\npMayIJIkSZLUWBZEkiRJkhrLgkiSJElSY205yIsiYh64BDgtM8+MiD2A84AtgFuBYzPz3uEdpqaV\n2VEV5kdlmR1VYX5UltmZTiueIYqIbYEzgCu6Vp8CnJWZBwDfBV4+nMPTNDM7qsL8qCyzoyrMj8oy\nO9NrkCFz9wKHAOu61q0FLm0vfwp4Vr2HpRlhdlSF+VFZZkdVmB+VZXam1IpD5jJzA7AhIrpXb9t1\nuu82YJfltrGwsMD8/PySda1Wa3VHOsNmtS3qyA5snp9Zba8yZrkt7HuGa5bbwb5n+Ga5Lex7hmuW\n28G+Z/iG1RYDXUO0grmVXrBmzZolj1utFnNzK76tEaq2xZT/Ixnog3fnx+xsVEdbzHp+7Ht6Mzv2\nPVWYH/uessyOfU8Vw8xP2Vnm1kfENu3l3Vh6alBajtlRFeZHZZkdVWF+VJbZmQJlC6LLgcPay4cB\nn63ncNQAZkdVmB+VZXZUhflRWWZnCsytdOoxIvYDTgX2BO4DbgGOBs4BtgZuAo7LzPv67mRubslO\nPP23UQ1D5ia2IevIDizNj9nZqKZTxxPbmPY9w2N27HuqMD/2PWWZHfueKoaZnxULojrYMfQ3ywVR\nXewYepv1L5Y62Pf0ZnYGY9/Tm/lZmX1Pb2ZnMPY9vQ0zP2WHzEmSJEnS1LMgkiRJktRYFkSSJEmS\nGsuCSJIkSVJjWRBJkiRJaiwLIkmSJEmNZUEkSZIkqbEsiCRJkiQ1lgWRJEmSpMayIJL0/9q7+2DJ\n6jq/4+9eRoQd5UGzzgCSItSwX4IzYMRVYHmYDe6q7IPZgIEFzS4obPAh4PqQWitJKbub8iHIIlBh\nozFYGKmAQYFogTIL7hApJVhL7qB8XSwdhQHBlRnB1ZHBzh/dw+17p/vevuf08+/9qqLsPuf0Ob/+\nzcff7W+f3zktSZJULAsiSZIkScVqNJvNcbdBkiRJksbCM0SSJEmSimVBJEmSJKlYq8bdgLoiogl8\nG9jVsXhrZr46IjYB7wb2Az6emesiYg3wysy8ueZxfx34SHvf/wC8IzP/pr3uQuBttPr3O8D5mfn9\nOsfTcExifjq2eRtwRWY26hxLwzFp2YmIjcAXgO91bP7ZzPzTOsfT4E1adtrrjgL+O/CPgL8H/igz\nv1HneBqOSctPRFwEXNix6XOAAzLzhXWOp+GYtPy01/0p8IdAE/gm8JbMfLTO8VZq6guito2Z+dDi\nhZl5KkD7g8JuvwG8Cqj8DxsRzwVuAl6fmXdExGnAdcAhEXEC8C7g5Zn5RERcBlwK/Kuqx9PQTUx+\nOrY5CLig6jE0MpOWna9l5sZer9dEmZjsRMRewI3AezPzxoh4A/Bm4E+qHk9DNzH5yczLgcs7tn0P\nsKbqsTQSE5OfiPhN4Dxan5t3RMQHgP8MvKHq8aqYlYKoq4j4Lh0dGhEvA64EVkXE8zLzrIh4HfDn\nwGrgQeDszPxhRLyP1oeMY4BPZ+Zfduz6OcAFmXlH+/ldwMERcQDwGPDGzHyivW4T8BdDeosaonHk\nJzO3t5dd3t7v/xzW+9PwjGns0QwYU3Y2ALsy80aAzPwU8KnhvUsNy5j/btE+m3Ah8M+G9BY1RGMc\nf/5vZu5or/tr4ENDeos9FXUNUWZ+ndY/7Gfa/6iHA9cCf5CZhwN3AFd3vOQ04LRF/6hk5lO7/3C0\nvRb4VmZuz8wHM/MrABGxL3AOrapYU24U+QGIiNcC+2Xm9UN8OxqhUWUH+McRcVtEZER8JiIOQVNt\nRNk5BtgaEddExLci4vMR8U+G+b40GiMce3Z7F3BNl+WaQiPKz53ACRHx4ohYBfw+8KWhvakeRnqG\nqD197DhacwQvysx7BrTrOyOicy7k5sw8v4/XvQa4MzO3tJ9fDfygPX0A4KuZ+cOldhARRwOXAWcv\nWv4h4N/QqoI/1LF8Pa0C6bLMvDIiDqUVrr2AR2idXdrZR9uLU0J+2kX0pcDvdtnW7FRUQnZoZeBG\n4IPAdlpTDq4F/nl7e/NT0ZDyM0nZOQA4mda0mPOAS2hl48T29manokLGnt3L9wf+NfCSRcvNT0Ul\n5Cczvx4RnwS+C/wEeAg4qb3tyLIzsoIoIk4BjsjM4yPinwKfAI4f0O67zoXswwHAyRHxQMeyHcDu\nCwF/tNSL29cLXQ+8OTPv7FyXme+JiPfSmoN9O3BcRKwGrqA1jW63S4CrMvOGiPhPtP4Y/ZcK72Wm\nFZSf/wj8j8z89qJtzU5FpWQnM5PWt7O7t3k/8MN2dsD8VDLE/ExMdtqv/9vM/Gp7m48A7zU79ZQy\n9nT4HRZ9IPZvV3Wl5Ccifg/4bVrXnf0IeC/wqYh4PSPMziinzJ0KfA4gM78JHBgR+43w+N1sA27P\nzCM7/vuVzHxsuRe2K9wbaJ02/ELH8ldExHEAmbmL1j/UK9vzJHfSOp24rWNXG5m/UO0WWt/QaU9F\n5Af4PeDfRsSjEfFoe9tHgRdjdqoqIjsRsWbRFLlVtL5V3IVjTx2Tlp9hjDtbgf07nj/T8b9mp7pJ\nyw4MJz+7/Q6tO112Mj/VlZKf3wJuzcy/z8wmrWunT2HE2RllQbQWeLzj+ePtZaP2NK0KF+A24KT2\nnMjdxczlPV/ZFhEN4JO0bgu4edHqI4H/2j51DK2pT9/L1vVFuzLzp4u2X91xuu8x4KCVv6UiFJGf\nzHxJZq7JzLWZuba9bG22mJ1qisgO8Drgxoh4Xvv5RcCmzNzp2FPLJORn2NnZBBwUEb/Vfn4B8H8y\n82dmp5ZJyA4MPz+7HUPrlsnPMj+1lJKfBE6NiF9uP/9tYMuoszPOu8yN67dVvgi8MyLuycxfi4jz\ngc9GxN7Ak8DFfezjOOBo4IMR8cGO5WfTmtt4BPDVdgC20/8tt/29mf7NZH6ydQFjFWanfzOZHeDj\nwK8CfxsRzwDfAM7ts23mp3/j6KuhjzsR8fvAX0Xr9rhbgT/qs21mp38zOfZ0/N16MbDS344xP/2b\nyfzQug4pgP/X/tv1KP397Rpof4yyINrGwsr2YFoXRNWSS/xoZWYe1vF0XXvZPcALOra5mS73Vs/M\n9y2x37tpXdDVy39o/9ePpyJi33YVfAgLTw1qXkn56at9mJ1+lZSdd9FxHdEyzE9/Bp6fScxOZt7F\noovhl2B2+lPS2ENm9judy/z0p6T8vG2JdZ2Glp1RTpn7InAGPHtf822Z+eQIjz+pbgdObz8+Hbh1\njG2ZZOZnT2anP2anO/PTH/OzJ7PTH7PTnfnpj/nZ09Cy02g2m4Pa17Ki9euzJwO/AN6amfeN7OAT\nICKOpXVL5cNozcl8mNbvFF0D7ENrmsK5mfn0mJo40UrOj9mpp+TsgPmpq+T8mJ16Ss4OmJ+6Ss7P\nqLMz0oJIkiRJkibJKKfMSZIkSdJEqXxThVjBr+c2Go0Fp6Hm5ubYsGFD1UPPlLp90Ww2p/IOLVXz\nY3bmDaIvpjE/jj31lZodcOwZhFLzU8rY88IXvrDnuquuumrF+7vggguefXz33Xdz/PHzvy364x//\neMX7m8bsgGPPIAxz7Kl0hqjz13OBNwEfXcnr169fX+WwM6nEvqiTnxL7q5cS+8KxZzBK7QfHnsEo\nsS8cewbjqKOOGncTxsKxZzCG2RdVp8xN4q/nanqYH1VldlSH+VFVZkd1mJ8JV3XK3Frg3o7nu389\nt+u5z7m5uT2qOm/mMK/AvqiVnwL7q6cC+8KxZ0AK7QfHngEpsC8ceyo688wzFzzfsWPHmFoyVo49\nAzKsvhjUD7MuOZ9z8Xy/ZrNJozGVU0AHrm5fzMj/SfrOj9mZN4i+mIH8OPZUYHae5dhTgfkBZnjs\nGeY1RDt27GD//fd/9nnFa4hW/JoJ5NhTwTDHnqpT5oby67kqhvlRVWZHdZgfVWV2VIf5mXBVCyJ/\nPVd1mB9VZXZUh/lRVWZHdZifCVf5h1lX8uu5i28/6em/eQOYMjeVHVk1P2Zn3oBOHU9dZzr21Fdq\ndsCxZxBKzc8sjT0vf/nLe677xCc+seL9feMb3+i57qyzznr28eJ++KVfWvn38s8888zkdOQKOPbU\nN8yxp3JBtBKTPjCMU6kF0Uo4MHRX6oeSlXDs6c7s9Mexpzvzs7xJH3ssiCabY093wxx7qk6ZkyRJ\nkqSpZ0EkSZIkqVgWRJIkSZKKZUEkSZIkqViD+mFWSZIkTYgTTjih57qPfexjPdc988wzKz5W540T\nFttrr716Pj/xxBO7vmbr1q0rboNUh2eIJEmSJBXLgkiSJElSsSyIJEmSJBXLgkiSJElSsSyIJEmS\nJBXLgkiSJElSsbzttiRJ0pR66Utf2nX51VdfXWl/P/nJT3quO/7447suX7duXc/XPPe5z13w/Mgj\nj1y2Ddu3b192G2mQPEMkSZIkqVgWRJIkSZKKZUEkSZIkqVgWRJIkSZKKZUEkSZIkqVgWRJIkSZKK\n5W23JUmSJtiLXvSinutuueWWFe/voYce6rmu1621AS666KIVH+vzn//8guc7d+589vGXv/zlFe9P\nGgbPEEmSJEkqlgWRJEmSpGJZEEmSJEkqlgWRJEmSpGJZEEmSJEkqVqW7zEXERuAG4P72ornMfPug\nGqXZZn5UldlRHeZHVY07O/fee+9A97fUneSazeaK93fxxRf3XPfggw8u+bwE486Pllfntttfzswz\nBtYSlcb8qCqzozrMj6oyO6rD/Ewwp8xJkiRJKladM0RHRcTNwAuA92fmlwbUJpXB/Kgqs6M6zI+q\nMjuqw/xMsEaVuaIRcQhwInA9cDhwB7AuM3/ebfstW7Y0169fX6ed6q0x7gaslPmZKFOVH7MzUaYq\nO2B+JsxU5cfsTJSpyg6YnwnTNT+VCqLFIuJrwJmZ+Z2uB2k0Fhyk2WzSaExdnoeibl80m82p78iV\n5MfszBtEX0x7fhx7qjE7LY491Zif0Y893//+9yu/tptDDz2057pB31Th8ssvX7Dv0rMDjj1VDTM/\nla4hiohzIuJd7cdrgTXAw9Wbp5KYH1VldlSH+VFVZkd1mJ/JV/UaopuBT0fE64C9gQt7nfaTujA/\nqsrsqA7zo6qGnp0/+7M/G+TuljwLdOuttw70WJ1ngdSVY8+Eq1QQZeaTwO8OuC0qhPlRVWZHdZgf\nVWV2VIf5mXzedluSJElSsSyIJEmSJBXLgkiSJElSsSyIJEmSJBXLgkiSJElSsaredluSJEkDcsop\np/Rc98QTT/Rcd/TRR3dd/u53v7tSO2677bae617zmtdU2mcvETHQ/UlVeYZIkiRJUrEsiCRJkiQV\ny4JIkiRJUrEsiCRJkiQVy4JIkiRJUrEsiCRJkiQVy9tuS5IkjdnWrVsrrati06ZNPdd9+MMfHuix\n1q1b1/P5gQce2PU1S91mXBoGzxBJkiRJKpYFkSRJkqRiWRBJkiRJKpYFkSRJkqRiWRBJkiRJKlaj\n2WwO/yCNxoKDNJtNGo3G0I87Der2RbPZnPmO7MyP2Zk3iL6Y9fw49nRndvrj2NOd+VlelbHn4IMP\n7rlu27Ztg2nYkKxfv76v7ebm5tiwYcOzz1et6n6z4/vvv7/nPn7+85/PdHbAsaeXYY49niGSJEmS\nVCwLIkmSJEnFsiCSJEmSVCwLIkmSJEnFsiCSJEmSVCwLIkmSJEnF6n6/w0UiYj1wE3BZZl4ZEYcC\n1wJ7AY8Ab8zMncNrpqaV2VEd5kdVmR3VMY78TPqttQGOOeaYrsuf//zn93zN9u3be67bsmVL1+W7\ndu1aWcMmiGPPdFr2DFFErAauADZ1LL4EuCozTwIeBM4bTvM0zcyO6jA/qsrsqA7zo6rMzvTqZ8rc\nTuA0oPOri43Aze3HtwCvGmyzNCPMjuowP6rK7KgO86OqzM6UWnbKXGbuAnZFROfi1R2n+x4DDlpq\nH3Nzc3v8inGz2eyxdXlmtS8GkR3YMz+z2l9VzHJfOPYM1yz3g2PP8M1yXzj2DNfc3Ny4mzA0jj3D\nN6y+6OsaomU0lttgw4YNC543m00ajWVfVoS6fTHl/yfp64135sfszBtEX8x6fhx7ujM7jj11mJ9y\nx5661xDNzc0t6JsHHnig62uWuoZo1rMDjj29DHPsqXqXuaciYt/240NYeGpQWorZUR3mR1WZHdVh\nflSV2ZkCVQui24HT249PB24dTHNUALOjOsyPqjI7qsP8qCqzMwUay516jIhjgUuBw4CngYeBc4Br\ngH2ArcC5mfl0z4M0GgsO4um/eQOYMjexHTmI7MDC/JideQM6dTyxnenYMzxmx7GnDvMz/WPPmjVr\neq7bd999e67bf//9V3ys++6779nHZsexp45h5mfZgmgQJn1gGKdZLogGxYGhu1n/wzIIjj3dmZ3+\nOPZ0Z36WN+ljjwXRZHPs6W6Y+ak6ZU6SJEmSpp4FkSRJkqRiWRBJkiRJKpYFkSRJkqRiDeKHWSVJ\nkjQDfvazn/Vc98gjj3RdvnPnzmE1RxoJzxBJkiRJKpYFkSRJkqRiWRBJkiRJKpYFkSRJkqRiWRBJ\nkiRJKpYFkSRJkqRiNZrN5rjbIEmSJElj4RkiSZIkScWyIJIkSZJUrFXjbkBdEdEEvg3s6li8NTNf\nHRGbgHcD+wEfz8x1EbEGeGVm3lzzuL8OfKS9738A3pGZf9Ne9++APwSeB3wGeGdmOjdxwowxO6cA\nHwL2p5Wdizuycxbw74HnAFuA8zJzR53jaTgmND/PA/4KODMzp358lyRpFGblD+bGzHxo8cLMPBUg\nIjZ2LP4N4FVA5Q8lEfFc4Cbg9Zl5R0ScBlwHHBIRrwXeDLwC+AnwJeANwLVVj6ehGnV29gX+F/Dq\nzLw3Il4HXB8RBwGHAlcAx2bm9yLiUuAvgLdVPZ6GbmLy0/7S5SvA/666f0mSSjQrBVFXEfFdWsXI\n7ucvA64EVkXE8zLzrPYHij8HVgMPAmdn5g8j4n3AIcAxwKcz8y87dv0c4ILMvKP9/C7g4Ig4APhN\n4LOZ+UT7mFcBZ2NBNFWGmJ29gTdl5r3t55uANcABwOuATZn5vfa6/wbcgQXR1BlTfp4A/hh4BHjP\n8N6dJEmzpahriDLz67Q+lHym/YHkcFqFyh9k5uG0Pnxe3fGS04DTFn0gITOfyswbOxa9FvhWZm4H\nmsBeHeueAtYN/t1olAaYnR2ZeRNARDSANwGb2wX0r9KagrXbt4EXRcSBw3pfGo0R5YfMvHv470aS\npNky0jNEEXEZcBytouGizLxnQLu+MyI65/Fvzszz+3jda4A7M3NL+/nVwA8iYndB89XM/OFSO4iI\no4HLaJ0FgtYUuY+33+uPgPOBfTq2X09rut1lmXllRBxK64PRXrS+2X1jZu7so+3FGVJ+xpKdiDiD\n1gfk7cC/bC/+ZeCx3dtk5s72dSqrgSfMTnWzNvb0yM+SzE91Q8zPVDA71ZWeHTA/dZSen1FmZ2Rn\niNoXAh+RmcfT+lbzowPc/cbMPLLjv34+kEBrmsnJEfFARDwA3A3sAF7YXv+jpV4cEScAXwDenJl3\nAmTmrbTe2+20prN8hdaHFiJiNa1rRDZ17OYS4KrMPInWtJnz+mx7UYaYn7FkJzM/k5lrgbcAd0TE\nWlrXnHUWz/sADeAps1PdLI49PfLTk/mpbsj5mXhmp7rSswPmp47S8zPq7IxyytypwOcAMvObwIER\nsd8Ij9/NNuD2RR9ofiUzH1vuhe0zQzfQmvLyhc51mfmh9r6OB34AzLVX7aQ1FWZbx+Ybmb/I+hZa\nF11rT5OWn0rZiYhDI+Jf7H6emX8NPETrG6AHWDi98gjgkfZUTLNT3aRlB4aTn6WYn+omMT+jZHaq\nKz07YH7qKD0/I83OKAuitcDjHc8fby8btadpfTsLcBtwUns+PxHxioi4fLkdtOfufxJ4S2ZuXrRu\nY0TcERF7R8TzgXe0tyUzd2XmTxftbnXH6b7HgIOqvrEZNwn5qZ0dWhfFXxMRL2m/7ghaRdD9tE4L\nnxoR0d72T2jdvdDs1DMJ2YHh56cn81PLpORnLMxOLUVnB8xPTUXnZ9TZGedd5hpjOu4XgXdGxD2Z\n+WsRcT7w2YjYG3gSuLiPfRwHHA18MCI+2LH8bGAz8C3g74Bf0Jr3eGefbRtXn0yjcfRV7exk5rfb\nr7uu/brd84L/DiAi3gJ8LiJWAV8H3t5n28xO/6Z27FkqP+072X2a1l0w92pPxSMzj+yjbeanf/bV\nQvZH/+yrPdkn/bOvFhpof4yyINrGwsr2YFoXRNWSmT07JDMP63i6rr3sHuAFHdvcTJffBcnM9y2x\n37tZeCe5xf54iXWLPRUR+7ar4ENYeGpQ8waen3Fkp73+BlrTLbutux64fqnXdzA7/ZmZsae9vmt+\nsnUnu36Kn93MT3+Gkp8pZ3b6Y3a6Mz/9MT97Glp2Rjll7ovAGfDsb3Jsy8wnR3j8SXU7cHr78enA\nrWNsyyQzP3syO/0xO92Zn/6Ynz2Znf6Yne7MT3/Mz56Glp1Gs9kc1L6WFREfAE6mNZXsrZl538gO\nPgEi4ljgUuAwWtcTPAycA1xD6+5iW4FzM/PpMTVxopWcH7NTT8nZAfNTV8n5MTv1lJwdMD91lZyf\nUWdnpAWRJEmSJE2SUU6ZkyRJkqSJUvmmCiv59dxGo7HgNNTc3BwbNmyoeuiZUrcvms3mVN51pGp+\nzM68xX1x0003dd3u4Ycf7rmPCy+8cOry49hT3yD6YVrHHkmSFqt0hqjur+euX7++ymFnUol9USc/\nJfZXLyX2hWPPYNgPkiTNqzplrvRfz1U95kdVmR1JkjRQVafMrQXu7Xi++9dzf9xt47m5uT2+kfRm\nDvMK7Ita+Smwv3oqsC8cewbEfpAkqWVQP8y65FzyxXPVm80mjYbTz6F+X8zIh5q+82N25i3ui4rX\nEA28XSPm2FPBIPphRsYeSZIqT5nz13NVh/lRVWZHkiQNVNWCyF/PVR3mR1WZHUmSNFCVCqLM/Apw\nb0R8hdZdnt460FZpppkfVWV2JEnSoDVGMQ988W+BOI9/3gCuIZr5juzMj9mZN6BriGa6Mx17uhvQ\nNUR2pCRpJlSdMidJkiRJU8+CSJIkSVKxLIgkSZIkFcuCSJIkSVKxvKnCmHlTheV5U4XuvDB+eY49\n3ZkdSZLmeYZIkiRJUrEsiCRJkiQVy4JIkiRJUrEsiCRJkiQVy4JIkiRJUrEsiCRJkiQVy4JIkiRJ\nUrEsiCRJkiQVy4JIkiRJUrEsiCRJkiQVy4JIkiRJUrEsiCRJkiQVy4JIkiRJUrFWjbsBkvp34okn\nLvm8m7vuumtYzZEkSZp6niGSJEmSVCwLIkmSJEnFsiCSJEmSVCwLIkmSJEnFsiCSJEmSVKxKd5mL\niI3ADcD97UVzmfn2QTVKs838DMfmzZu7Lr/88stH3JLhMTuSJGnQ6tx2+8uZecbAWqLSmB9VZXYk\nSdLAOGVOkiRJUrHqnCE6KiJuBl4AvD8zvzSgNqkM5kdVmR1JkjQwjWazueIXRcQhwInA9cDhwB3A\nusz8ebftt2zZ0ly/fn2ddqq3xrgbsFLmZ6JMVX7MzkSZquxIktRLpYJosYj4GnBmZn6n60EajQUH\naTabNBr+LYX6fdFsNqe+I1eSn9Kzc+KJJz77ePPmzZx00kkLnnez1E0VLrrooqnuTMeeagbRD7Mw\n9kiSBBWvIYqIcyLiXe3Ha4E1wMODbJhml/lRVWZHkiQNWtUpc88HPg0cAOxNax7/F3oexG9peyrx\nDFGd/JideYv74tFHH13xPtasWTNVnenYMxieIZIkad5ApswtexA/lPRUYkG0UhZE3ZVYEK2UY093\nFkSSJM3zttuSJEmSimVBJEmSJKlYFkSSJEmSimVBJEmSJKlYFkSSJEmSirVq3A2QNBjXXXfdil9z\n8cUXD6ElkiRJ08MzRJIkSZKKZUEkSZIkqVgWRJIkSZKKZUEkSZIkqVgWRJIkSZKKZUEkSZIkqViN\nZrM5/IM0GgsO0mw2aTQaQz/uNKjbF81mc+Y7sjM/ZmfeIPpi1vPj2NOd2ZEkaZ5niCRJkiQVy4JI\nkiRJUrEsiCRJkiQVy4JIkiRJUrEsiCRJkiQVy4JIkiRJUrEsiCRJkiQVy4JIkiRJUrEsiCRJkiQV\ny4JIkiRJUrEsiCRJkiQVy4JIkiRJUrFW9bNRRKwHbgIuy8wrI+JQ4FpgL+AR4I2ZuXN4zdS0Mjuq\nw/xIkqRhW/YMUUSsBq4ANnUsvgS4KjNPAh4EzhtO8zTNzI7qMD+SJGkU+pkytxM4DdjWsWwjcHP7\n8S3AqwbbLM0Is6M6zI8kSRq6ZafMZeYuYFdEdC5e3TFN5THgoKX2MTc3x/r16xcsazabK2vpDJvV\nvhhEdmDP/Mxqf1Uxy33h2DNc9oMkSS19XUO0jMZyG2zYsGHB82azSaOx7MuKULcvpvxDTV9vvDM/\nZmfeIPpi1vPj2NOd2ZEkaV7Vu8w9FRH7th8fwsIpLdJSzI7qMD+SJGmgqhZEtwOntx+fDtw6mOao\nAGZHdZgfSZI0UI3lpj1ExLHApcBhwNPAw8A5wDXAPsBW4NzMfLrnQRqNBQdx2sq8AUyZm9iOHER2\nYGF+zM68AU17mtjOdOwZnlnPjiRJK7FsQTSQg/ihpKdZLogGxYKoOz/ULs+xpzuzI0nSvKpT5iRJ\nkiRp6lkQSZIkSSqWBZEkSZKkYlkQSZIkSSqWBZEkSZKkYlkQSZIkSSqWBZEkSZKkYlkQSZIkSSqW\nBZEkSZKkYlkQ+cnA9QAAAh1JREFUSZIkSSqWBZEkSZKkYjWazea42yBJkiRJY+EZIkmSJEnFsiCS\nJEmSVCwLIkmSJEnFsiCSJEmSVCwLIkmSJEnFsiCSJEmSVKxVozxYRFwGHAc0gYsy855RHn8SRMR6\n4Cbgssy8MiIOBa4F9gIeAd6YmTvH2cZJVXp+zE51pWcHzI8kSb2M7AxRRJwCHJGZxwNvAj46qmNP\niohYDVwBbOpYfAlwVWaeBDwInDeOtk260vNjdqorPTtgfiRJWsoop8ydCnwOIDO/CRwYEfuN8PiT\nYCdwGrCtY9lG4Ob241uAV424TdOi9PyYnepKzw6YH0mSehplQbQWeLzj+ePtZcXIzF2Z+dNFi1d3\nTFN5DDhoxM2aFkXnx+zUUnR2wPxIkrSUcd5UoTHGY08q+6R/9tVC9kf/7Ks92SeSpGKNsiDaxsJv\nZQ+mdSFv6Z6KiH3bjw9h4ZQWzTM/ezI7/TE73ZkfSZIYbUH0ReAMgIh4GbAtM58c4fEn1e3A6e3H\npwO3jrEtk8z87Mns9MfsdGd+JEkCGs1mc2QHi4gPACcDvwDempn3jezgEyAijgUuBQ4DngYeBs4B\nrgH2AbYC52bm02Nq4kQrOT9mp56SswPmR5KkpYy0IJIkSZKkSTLOmypIkiRJ0lhZEEmSJEkqlgWR\nJEmSpGJZEEmSJEkqlgWRJEmSpGJZEEmSJEkqlgWRJEmSpGJZEEmSJEkq1v8HMYU1M4IGMZ0AAAAA\nSUVORK5CYII=\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7f3b4d8fd8d0>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "k_EOu76kNQQX",
        "colab_type": "code",
        "outputId": "56cbc2c5-b93e-43c6-eb3c-aef1a313075a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "x_batch.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1, 28, 28, 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "metadata": {
        "id": "3zEfm6xGwgYs",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}