10_introduction_to_artificial_neural_networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Chapter 10 – Introduction to Artificial Neural Networks**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_This notebook contains all the sample code and solutions to the exercises in chapter 10._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To support both python 2 and python 3\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "def reset_graph(seed=42):\n",
    "    tf.reset_default_graph()\n",
    "    tf.set_random_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"ann\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perceptrons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.linear_model import Perceptron\n",
    "\n",
    "iris = load_iris()\n",
    "X = iris.data[:, (2, 3)]  # petal length, petal width\n",
    "y = (iris.target == 0).astype(np.int)\n",
    "\n",
    "per_clf = Perceptron(random_state=42)\n",
    "per_clf.fit(X, y)\n",
    "\n",
    "y_pred = per_clf.predict([[2, 0.5]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure perceptron_iris_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsgAAAEYCAYAAABBfQDEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XdcleX/x/HXBQo4UZy5M0dOcpQ5cqappQ2tNMVE/aoN\n7Wc5cg8QFUXUMs1c5R5p2ddyTyxF6luau9yaI/dCGffvD/S4QBHh3Iz38/HgIedz7uu+P+dA9OHm\nuq6PsSwLERERERGJ4WJ3AiIiIiIiyYkKZBERERGRO6hAFhERERG5gwpkEREREZE7qEAWEREREbmD\nCmQRERERkTuoQBYRERERuYNTC2RjjJsxZrIx5qAx5oIx5ldjTMMHHN/NGPOPMebczXHpnZmviIiI\niKQ9zr6DnA44DLxgWZYnMACYb4wpdO+BxpiXgJ5AHaAI8BQw2HmpioiIiEhaZOzupGeM+QMYZFnW\n4nvis4ADlmX1u/m4LjDLsqwnbEhTRERERNIIW+cgG2PyAMWBHbE8XQb4447HfwC5jTHZnZGbiIiI\niKRNthXIxph0wExgumVZe2M5JDNw4Y7HFwADZHFCeiIiIiKSRqWz46LGGENMcXwd6BLHYZeBrHc8\nzgpYwKVYzmfvPBERERERSREsyzIPO8aWAhmYAuQEGluWFRXHMTsAb2DhzcfPACctyzoX28E3bnyX\n6EkmtZMnz9OhwziWL//NEXN3z0zLll/w/PM+NmaWcv3wwyCaNBlkdxpiM30fyC36XhDQ90FqEhQ0\niH37Bt0XL158EJ98cn/8/jEPrY0BG6ZYGGMmAk8DTS3LuvGAQ78B2htjSt2cd9wXmOaMHJ0lT55s\nfP99PwIDfUmfPuZ3levXLzN9ehumTfPh2rWLNmcoIiIikvY49Q7yze3cOgLhwMmYmRZYQCcghJi7\nxqUtyzpqWdZyY0wgsBbwIOZO8iBn5usMLi4u/N//vUqtWmVp1SqIv/46DsCWLTPZv/8X2refQ5Ei\nz9qcpYiIiIj9cuf2ILZyMCb+8DH79sXvOrZv85YYjDFWSpxica/Ll6/Rrdtkvv56tSOWLp0rTZoE\nUL9+d1xc1PjwYfbsWUfJkrXtTkNspu8DuUXfCwL6PpDbOnc28ZqDrAI5GZo7dwMffDCBS5euOWJP\nP/0ivr7f4OmpbaBFREREEiK+BbJuSSZDLVrUZOvWYJ57roQjtnv3Kvz9vdm+/UcbMxMRERFJ/VQg\nJ1NFi+Zl7doAevZsxs252ly6dJrx419m/vxuRERctzlDERERkdRJUyxSgLVrt9G2bTD//HN7h7uC\nBZ+hffu55M1b0sbMRERERJK3GTOGc+pUOAD79g3WFIvUok6d8vz661gaN67siB058jsBARXZtGkq\nqeGXHBEREZGkcOpUOPv2xb5/clxUIKcQOXNmZfHivowZ8x/c3dMDcOPGVWbMaM+UKS25evW8zRmK\niIiIpA4qkFMQYwzvv/8yISGBlCxZwBEPC5vH0KEV2L//FxuzExEREUkdVCCnQN7eT7J58yjat6/v\niJ05c5BRo17gxx+HEh0dV/duEREREXkYFcgpVKZMHkyY8AFz5vQkW7ZMAERHR7FkST/GjHmRc+eO\n2ZyhiIiISMqkAjmFa9asGlu3BlOtWilHbO/edfj7l+ePP5bYmJmIiIiI/XLn9qB48UEULz4o3mO0\nzVsqERkZxdCh8xk2bAHR0dGOeK1aH9Cs2Ujc3DLYmJ2IiIiI/dRJL41Jl86VgQNbsnKlHwUK5HDE\n168fz4gRVTh+fKeN2YmIiIikHCqQU5kXXihDWNgYXn31eUfs2LHtDBtWmY0bJ2nPZBEREZGHUIGc\nCnl5ZWH+/F58/nlnPDzcAIiIuMasWZ2YNKk5V66ctTlDERERkeRLc5BTuT//PISPTxA7dhx2xLJn\nL0i7drMoXvwFGzMTERGRtKxbt/qEh+e4L+7hcYbg4JWxjhk4sAUXLnjcF/f0DGfw4LkPvWZ85yCn\ne+iZJEUrW7YwP/88kl69pjNx4k8AnDt3hNGja9O4cX8aN+6Hq6u+DURERMS5wsNzYFn3F7Xh4S3i\nHHPhggfh4dNjeaZtouUFmmKRJmTI4M64cZ1YsOBTvLyyAGBZ0SxdOpjg4DqcPXv4IWcQERERSTtU\nIKchr776PGFhwdSsWcYR++uvEPz9vfntt29tzExEREQk+VCBnMYUKJCT5cuHMGjQO7i6xnz5r149\nz6RJzZk1qzM3bly1OUMRERERe6lAToNcXV3p0+ctVq8eSqFCuRzxjRu/ZNiwZzl6dJuN2YmIiIjY\nS6uz0rBq1UoRFhbMe+99wbff/gzAP//sZPjw52jePIhatd7HmIcu9BQRERF5ZB4eZ2JdkOfhcSbO\nMZ6e4cS2IC8mnni0zZtgWRbTp6+iW7fJXL163REvX74pbdpMIXPmnDZmJyIiIpI41Gpa4s0Yg69v\nfTZvDqJ8+SKO+LZtS/Dz82bPnrX2JSciIiLiZCqQxeHppwsQEhJIly6vOGIXLhxnzJh6fP99P6Ki\nImzMTkRERMQ5VCDLXTw83AgK6sDixX3JmTMrEDMF46efhhIUVIt//z1ob4IiIiIiSUxzkCVOx4+f\npV27MaxZc3tXCw+PrLRuPYnKld+2MTMRERGJjxkzhnPq1P0L2HLn9sDH51MbMnK+O9+DffsGq9W0\nPJ58+bz48cdBBAV9x8CBs4iMjCI8/CKTJ7dg584VvP32ONzdM9mdpoiIiMTh1Klw9u0bFMszscVS\np7vfg8HxGqMpFvJALi4u9OjxBuvWDePJJ/M44j//PJWAgIocPvw/G7MTERERSXwqkCVennuuBFu3\nBtOiRU1H7OTJvQQGPs/q1WNIDVN1REREREAFsjyCrFkz8vXX3Zgy5SMyZfIAIDLyBgsWdGP8+Fe4\nePGUzRmKiIiIPD4VyPJIjDH4+NRhy5YgKlQo6oj/+eeP+Pt7s3PnShuzExEREXl8WqQnCVKiRH42\nbhxB//4zCQ7+HoCLF08wblwDGjToSdOmfqRL52ZzliIiImlb7twexLYgLyaeNtz5HuzbF78x2uZN\nHtuKFf+jXbsxnDp1wRErXPhZOnSYQ65cT9mYmYiIiMhtajUtTtOgQQV+/XUsDRpUcMQOHdqKv/8z\nbNky08bMRERERB6dCmRJFHnyZGPJkv4EBvqSPn3MzJ3r1y8zbZoP06a1ITz8ks0ZioiIiMSPCmRJ\nNC4uLvzf/73Kxo3DKVbsCUd8y5YZDB1agUOHwmzMTkRERCR+VCBLoqtYsRhbtozGx6eOI3b69N+M\nGFGVFStGEh0dbWN2IiIiIg+mRXqSpObMWc+HH07k0qVrjlipUvVp2/YbPD3z2piZiIhI6jdjxnBO\nnQq/L547twc+Pp+muOs8rvgu0tM2b5KkWrasRZUqJfHxCWLr1pi9VXbtWom/f3neffdrypZtZHOG\nIiIiqdepU+Hs2zcolmdiiyX/6ziLplhIkitaNC/r1g2jR49mGBPzS9ulS6f5/PPGLFjwMRER123O\nUEREROQ2FcjiFOnTp2PoUB9++mkQTzyR3RFfvTqYwMCqnDy518bsRERERG5TgSxOVbeuN2FhY2jc\nuLIjduTI/wgM9Obnn6eTGubEi4iISMqmAlmcLlcuTxYv7svo0R1wc4uZBn/lSjjffOPLlCnvcO3a\nhYecQURERCTpaJGe2MIYw4cfvkKNGqXx8RnNnj1HAQgLm8uBA5tp334ORYs+b3OWIiIiKVvu3B7E\ntlAuJp7yruMs2uZNbHflSjiffDKFqVNXOmIuLq40aTKEl17qhYuLq43ZiYiISGoR323eNMVCbJcp\nkwcTJ37A7Nk98PTMCEB0dBTff9+XsWPrc+7cMZszFBERkbTE6QWyMeYDY8xWY0y4MWbqA4571xgT\naYy5aIy5dPPfms7MVZyrefPqhIWNoWrVpx2xPXvW4u/vzbZtP9iYmYiIiKQldtxBPgb4AVPicezP\nlmVltSwry81/NyRxbmKzwoVzs3r1UPr0ecuxZ/KVK2f44oumzJvXlYiI+7v0iIiIiCQmpy/Ssyzr\nOwBjzLNAfmdfX5K/dOlcGTToHerUKUfbtmM4duwMAGvXfsbevetp334O+fKVtjlLERGR5N9i+b33\nqmFZ95dbxhxjwoSfYx2TkNfkrPdh4MAWXLhw/8I/T89wBg+e+0i5PUhy38WigjHmFHAWmAkEWJYV\nbXNO4iS1apUjLCyYTp3Gs2TJFgCOHdvGsGGVeeutsdSo0cFxl1lERMQOyb3FckxxvCCW+JtxjknI\na3LW+3Dhggfh4dNjeaZtnGPuzm1wvK6TnBfprQfKWpaVG2gGtAR62JuSOFuOHFlZsOBTPvusEx4e\nbgBERFxj1qyOTJr0JleunLM5QxEREUltku0dZMuyDt7x+Q5jzBCgOzAituOHDJnj+LxWrbLUqlUu\nqVMUJzHG0KlTI6pXL03r1kHs3HkYgP/971sOHgylffvZFCtWw+YsRUREJLnZs2cdZ86s41HvZCfb\nAjkOcf49fcCAls7MQ2xQtmxhfvllJD17TuPLL5cBcO7cEYKCavHKKwNp1Kiv9kwWERERh5Ila5Mj\nR23Onh10M5JMp1gYY1yNMR6AK5DOGONujLmvqjHGNDTG5L75+dNAP0DdQNK4DBnc+eyzzsyf/ynZ\ns2cGwLKi+eGHgQQH1+Xs2SM2ZygiIiIpnR13kPsBA4FbLfxaAYONMdOAnUApy7KOAvWA6caYTMBJ\nYAYwzIZ8JRl67bXnqVTpKdq2HcPGjTsA2LdvA/7+3vj4TKZChTdszlBERNKC5N5i2ZhjsS7IMybu\nJlwJeU3Oeh88PcOJbUFeTDx2d+a2b1/8rqNW05KiRUVFMXz4Qvz85hEdfXuDkxde6MSbb47GzS2j\njdmJiIhIcqJW05ImuLq60rfv26xZM5RChXI54hs3fsmwYc9y7Nh2G7MTERGRlEgFsqQK1aqVYuvW\nYJo1q+aI/fPPToYNe5Z168aTGv5SIiIiIs6hAllSjezZMzN7dg8mTvyADBli9kyOjLzO3LkfMnHi\n61y+fMbmDEVERCQl0BxkSZV27TpC69ZBbN9+0BHLli0/vr4zKVmytm15iYjIo0vu7ZydJSFtlhMi\nIe93YraATsqva3znIKe0fZBF4qVUqYJs2hRInz7f8Pnn/wXg/PljjBlTl0aN+vLyywNxddW3v4hI\nSpDc2zk7S0LaLCdEQt7vx28BHb/rOIumWEiq5eHhxujRHVi0qA85cmQBwLIsfvzRn6Cgmvz770F7\nExQREZFkSQWypHqvvPIcv/46ljp1brcf37//F4YOfYawsHk2ZiYiIiLJkQpkSRPy5fPixx8H4e/v\ng6trzLf9tWsXmDy5BTNmdOD69Ss2ZygiIiLJhQpkSTNcXV3p2bMZ69YN48kn8zjimzZNISCgEkeO\n/G5jdiIiIpJcaJWSpDlVqpQkNHQ0H344kXnzNgJw8uQeRoyowuuvB1K3bleMeegCVxERcZLk3s7Z\nWRLSZjkhEvJ+P24L6Phex1m0zZukWZZlMWPGWj76aBJXrtz+D7hcuZdp02YaWbLkesBoERERSWnU\nalrkIYwxtGlTly1bgqhQoagjvn37Uvz8yrNr1yobsxMRERG7qECWNK9Eifxs2DCCbt1edcQuXjzB\nuHENWLz4U6KiImzMTkRERJxNBbII4O6enhEjfPnhhwHkzu0JxEzBWL58BCNH1uD06b9tzlBERESc\nJd5zkI0xGYFngNzcU1hblrUo8VOLP81BlsR04sQ52rcfy8qVt3e18PDIQsuWE6hSpZWNmYmIJJ3U\n1s65W7f6hIfnuC/u4XGG4OCVsY5JSLtkZ42BhH2NnDUmpUjUVtPGmBeBOcD932lgAa6Plp5I8pU3\nb3Z++GEAY8YsoX//mURERBIefolp01qza9cKWrT4HA+PLHanKSKSqJJz29+ECA/PgWXdX2yGh7eI\nc0xC2iU7awwk7GvkrDGpTXynWIwFlgIFLMtyuedDxbGkOi4uLnz88Wts2DCcYsWecMQ3b/6GoUMr\ncuhQmI3ZiYiISFKKb4FcBPCzLOt4EuYikuxUqlSMLVtG07p1HUfs9Om/CAysxooVo4iOjrYxOxER\nEUkK8S2QNwElkzIRkeQqS5YMTJ36EdOndyNLlgwAREVFsGhRDz7/vBEXLpywOUMRERFJTHEWyMaY\nirc+gInAKGNMB2NMlTufu/m8SKr3zju1CA0dTeXKxR2xnTtX4O/vzY4dy2zMTERERBLTgxbphRGz\nAO/OlX6TYjlOi/QkzXjqqSdYty6AQYPmMGpUzOYtly6d4rPPGvHiix/z6qsBpE/vbnOWIiKPLjm3\n/U0ID48zsS7I8/A4E+eYhLRLdtYYSNjXyFljUps4t3kzxhSO70ksyzqUaBklgLZ5EzusXv0Hvr5j\nOHHinCNWqFBF2refQ548JWzMTERERGLz2K2mLcs6dOsDKAwcuzN2M37s5nMiaU69et78+usYGjWq\n5IgdPvwbAQEV+fnn6cR3j3ERERFJXuK7SG8t4BVL3PPmcyJpUq5cnnz3XT+Cgtrj5hYzY+n69St8\n840vU6e24tq1CzZnKCIiIo8qvgWyIWau8b1yAFcSLx2RlMcYQ5cuTQgJCaREifyO+Natcxg6tAL7\n92+2MTsRERF5VA9sNW2MWXLz05eBVcD1O552BcoCuyzLaphkGcaD5iBLcnHlSjgffzyZadNWOWIu\nLq40bepHgwa9cHGJ7++kIiJyS3Jul+zMtszJuQV0cs7tTonVavrWUk8DnAOu3fHcDSAE+CpBGYqk\nQpkyefDllx9Sr94zvP/+F1y8eJXo6Ci++64Pu3atwtd3Btmy5bM7TRGRFCU5t0t2Zlvm5NwCOjnn\nlhAPLJAty/IFMMYcBEZZlqXpFCLx8NZbNXjuueK0aTOazZv3ALBnzxr8/Mrz7rvTKV/+FZszFBER\nkbjE6++9lmUNVnEs8miKFMnDmjUB9O79JsbE/DXnypUzfPFFE+bN60pExIP3uxQRERF7PKiT3gFj\nzP74fDgzYZGUJF06VwYPbsWKFUPInz+HI7527WcMH16Ff/7ZZWN2IiIiEpsH3UH+HBh/8+NrYnas\n+BuYefPj75ux6UmbokjKV6tWOcLCgmnS5DlH7NixbQQEVCIkZLL2TBYREUlG4pyDbFlW0K3PjTHT\ngRGWZQXceYwxpjdQJsmyE0lFcuTIysKFvfnyy5/o0WMa169HEBFxjZkz/8POnSto3XoSGTNmsztN\nEZFkJzm3S3ZmW+bk3AI6OeeWEA/c5s1xkDEXgYqWZf11T7wY8JtlWVmTKL940TZvktJs336Q1q2D\n2LXriCPm5VWIdu1mU6xYdRszExERSb0eu9X0Pa4AtWOJ1wauxj8tEQEoV64Iv/wyio4db28hfvbs\nYUaPrsnSpX5ER0fZmJ2IiEjaFt87yD0BP2AacKst2PPAu8Agy7JGJFmG8aA7yJKSLV78C507j+fc\nucuOWPHiNfH1nYmXV0EbMxMREUld4nsHOV4FMoAx5i3gI6DUzdAuYKxlWfMTnGUiUYEsKd2RI6d5\n991gQkJ2OmIZM2bHx2cKFSq8bmNmIiIiqUeiF8jJmQpkSQ2ioqIYNmwh/v7ziI6OdsRr1nyP5s2D\ncHPLYGN2IiIiKZ8KZJEUatOmnbRpM5ojR/51xPLlK0P79nPJn7+sjZmJpAwzZgzn1Kn7G/Hkzu2B\nj8+nNmQkd0rI10dfU0ks8S2Q49zm7ebOFUUty/rXGHMJiLOStnsXC5HUpHr10oSFjaFz5/EsXvwL\nAMeP72D48Gdp3nw0NWt2dnTmE5H7nToVzr59g2J5JraYOFtCvj76moqzxVkgA12AS3d8nvJvNYuk\nENmzZ2bu3J5MnbqSjz+ezLVrN4iICGfOnPfZuXMFPj6TyZw5x8NPJCIiIo/sQY1Cvr7j8+lOyUZE\nHIwxtG/fgKpVS9G69Sj+/PMQAH/88R2HDm2lXbtZlChRy+YsRUREUp947YNsjOltjHneGOOa1AmJ\nyN1Kly7Izz+P5IMPXnbEzp8/RnBwHZYsGUBUVKSN2YmIiKQ+8W0U8jKwHjhvjFl+s2CuqoJZxDk8\nPNwIDv4Pixb1IUeOLABYlsWPP/oRFFSLM2cO2ZyhiIhI6vGgOcgOlmXVMMZkAGoAtYgpmAcCEcaY\nTZZlNXzgCUQkUbzyynOEhY3B13cM69ZtB2D//p/x9/emdeuvqFTpTZszFLFf7twexLZ4KyYudkvI\n10dfU3G2R97mzRiTF6hDTJH8NhBhWVbGJMjtUXLSNm+SpkRFRTFy5CIGD55DVNTtPZOrV+/AW2+N\nwd09k43ZiYiIJE/x3eYtvnOQ3zTGfGGM2QX8DXQE/gLqA9kfK1MReWSurq58+umbrFs3jCJFcjvi\nmzZNZtiwyhw9+oeN2YmIiKRs8Z2DPA9oBkwDclmWVceyrEGWZa2zLOv6o1zQGPOBMWarMSbcGDP1\nIcd2M8b8Y4w5Z4yZbIxJ/yjXEkntqlQpydatwbz11guO2IkTuwkMrMyaNeNIDY2AREREnC2+BXIn\nYCUx+yEfN8b8YIz5xBhT0Tx6x4JjgB8w5UEHGWNeAnoSM52jCPAUMPgRryWS6nl6ZmLGjI+ZPLkL\nmTLFzMe7cSOS+fM/4osvmnDp0mmbMxQREUlZEjIHuRhQm5jpFa8Dly3L8nrkCxvjB+S3LKtdHM/P\nAg5YltXv5uO6wCzLsp6I5VjNQRYB9uw5ho9PEL//vt8R8/R8Al/fGTz9dD0bMxMRgIEDW3Dhwv0L\nyzw9wxk8eG6Ku05ybgGtltYSm8duNX0vY4wL8CwxxXFdoPrNp/YkJMF4KAPcWfX+AeQ2xmS3LOtc\nEl1TJEUrWTI/GzeOoF+/GYwduwSACxf+YezY+jRo0IumTYfg6qqZSiJ2uXDBg/Dw6bE80zZFXic5\nt4BWS2t5HPFdpPcjcA7YSMxd4/8BzYHslmVVTaLcMgMX7nh8ATBAliS6nkiq4O6enpEj27FkSX9y\n5fIEYvZMXr58OCNH1uD06f0POYOIiEjaFt87yNuAccBGy7KuJGE+d7oMZL3jcVbAAi7FdvCQIXMc\nn9eqVZZatcolaXIiyV3DhpX49dcxtGs3hlWrYna1OHgwlMDAsrz55mSee+4dmzMUERFJWnv2rGPv\n3nWPPC6+jULsmHizA/AGFt58/AxwMq7pFQMGtHRWXiIpRt682fnvfwcyZswS+vWbQWRkFJcuXWPq\n1Fbs3LmCFi0+w8NDf5QREZHUqWTJ2pQsWdvxeOnS+O33EN9dLBKNMcbVGOMBuALpjDHucbSs/gZo\nb4wpZYzJDvQlZps5EXkELi4ufPzxa2zYMJynnsrriG/e/DUBAZU4dOhXG7MTERFJfuK9SC8R9SOm\nTfWt7TNaAYONMdOAnUApy7KOWpa13BgTCKwFPIi5kzzIhnxFUoXKlYsTGhpM165fMmvWOgBOndpH\nYGBVXnttGPXqdcPFxem/M4ukKZ6e4cS2UC4mnvKuk5xbQKultTyOR97mLTnSNm8ij2bWrHV06TKR\ny5dv/8+ydOmXaNv2a7JmzWNjZiIiIkknUVtNi0jq0qpVbUJDg6lcubgjtnPncvz8yrNjx3IbMxMR\nEbGfCmSRNKpYsSdYty6ATz553RG7dOkUn33WkIULuxMZecPG7EREROwT5xQLY8wlbs8TfiDLsrI+\n/KikoykWIo9n1arfadduLCdO3N4kplChSrRvP4c8eYo/YKSIiEjKEd8pFg8qkN+N78Usy/r6EXJL\ndCqQRR7fqVPn6dBhHMuW/eaIubtnomXLL6hSxQdjHvrzREREJFl77AI5JVGBLJI4oqOj+fzz/9K7\n9zdEREQ64s8++w7vvDOBDBls/WORiIjIY9EiPRF5ZC4uLnTt2pSQkECKF8/niG/dOpuhQytw4MAW\nG7MTERFxjngVyMYYN2PMYGPMXmNMuDEm6s6PpE5SRJyrQoWibNkSRNu29Ryxf//dz8iRNVi2bDjR\n0dE2ZiciIpK04nsH2Q94FwgCooEewHjgDPB+0qQmInbKnDkDkyZ1YebM7mTNmhGA6OhIvvuuN+PG\nNeD8+eM2ZygiIpI04lsgvwV0tizrSyAK+N6yrK7EdMSrn1TJiYj93nqrBlu3BlOlSklHbPfu1fj7\ne7Nt239tzExERCRpxLdAzkNMG2iAy0C2m58vAxokdlIikrw8+WQe1qwZyqefvunYzeLy5X/54osm\nzJv3ERER123OUEREJPHEt0A+DNxasfMX8NLNz6sC1xI7KRFJftKnT8eQIa1YvnwI+fJ5OeJr145j\nxIgqnDix28bsREREEk98C+TFwK3VOmOBwcaYA8B0YHIS5CUiyVTt2uUICxvDK68854gdPfoHAQGV\n2LRpCqlh60gREUnbErQPsjGmClAd2GtZlu2TELUPsojzWZbFxIk/0bPnNK5fj3DEK1Z8k9atJ5Ex\nY7YHjBYREXG+RG0UYoypCfxsWVbkPfF0QDXLsjYkONNEoAJZxD7bth3ExyeIXbuOOGJeXoVp3342\nTz1VzcbMRERE7pbYjULWAl6xxD1vPiciaVT58kX45ZdR/Oc/LzliZ88eIiioJkuX+hEdra3SRUQk\nZYlvgWyA2G415wCuJF46IpISZczozvjx7zF3bk+yZcsEQHR0FD/8MIDg4HqcO3fU5gxFRETi74FT\nLIwxS25++jKwCrhzLydXoCywy7KshkmWYTxoioVI8nH48GnefXc0mzbtcsQyZfLCx2cKzzzzmo2Z\niYhIWpdYUyzO3PwwwLk7Hp8BjgITgdaPl6qIpCaFCuVi5Up/+vdvgYtLzI+YK1fOMnHi68yZ8wE3\nbmhnSBERSd7iu0hvIDDKsqxkOZ1Cd5BFkqeQkB28+24wR47864jly1eW9u3nkD9/WRszExGRtChR\nF+lZljXYsqwrxpjKxpi3jTGZAIwxmW7uZCEicp8aNcoQFjaG11+v6ogdP/4nw4c/y4YNE7VnsoiI\nJEvxKpCNMXmMMVuAUGA2Ma2nAUYDQUmUm4ikAtmzZ2bu3J588cV7eHi4ARAREc7s2e/x5ZfNuHLl\nrM0Zioift/M7AAAgAElEQVSI3C2+u1gEAyeI2bXi6h3xBUCDxE5KRFIXYwwdOrzEL7+MomzZwo74\n778vxt/fm337bN1KXURE5C7xLZDrAX0tyzp3T/xvoFDipiQiqVWZMoXYtCmQ995r7IidO3eU0aPr\n8MMPA4mKinzAaBEREeeIb4GcAbgRSzwXEJ546YhIapchgztjx3Zk4cLeeHllAcCyolm6dAijR9fm\nzJlDNmcoIiJpXXwL5A1A2zseW8YYV6AXsDqxkxKR1K9p0yr8+usYatW6vZvF339vwt/fm19/XWBj\nZiIiktbFt0DuCfzHGLMScCdmYd5OoDrQO4lyE5FULn/+HCxbNpghQ1rh6hrz4+jatQt89dVbzJzZ\nkevXk+XOkiIiksrFd5u3nUB54BdgBeBBzAK9CpZl/Z106YlIaufq6sqnn77J2rUBFC6cyxEPCfmK\nYcMqc/ToHzZmJyIiaVG8GoUkd2oUIpI6nD9/mQ8+mMiCBSGOWLp07jRrNpLatT/EmIfu7S4iIhKn\nRGkUYozJaIwZb4w5Zow5ZYyZbYzJmXhpiojcli1bZmbO/IRJkz4kY0Z3ACIjrzNvXlcmTHiVy5f/\nfcgZREREHt/DplgMJmZx3lJgLlAfmJDEOYlIGmaMoW3bF9myZTTe3k864tu2/YCfX3l2715jY3Yi\nIpIWPKxAfgNob1lWR8uyugIvA6/d3MFCRCTJlCyZn5CQQLp2beKIXbjwD2PHvsh33/UhKirCxuxE\nRCQ1e1iBXBDYeOuBZVmhQCSQLymTEhEBcHdPz6hR7fn++37kyuUJgGVZLFs2jFGjXuD06f02Zygi\nIqnRwwpkV+5vEBIJpEuadERE7teoUWXCwoKpV8/bETtwYAtDhz7D1q1zbMxMRERSowfuYmGMiQZW\nAtfvCDcC1gNXbwUsy2qaVAnGh3axEEkboqOjGT36OwYMmEVkZJQjXrVqW95++zM8PDLbmJ2IiCR3\n8d3F4mEF8rT4XMyyLN9HyC3RqUAWSVu2bt1Lmzaj+fvvE45Y7tzF6dBhLoUKVbQxMxERSc4SpUBO\nKVQgi6Q9Fy9epWvXL5k9e70j5uqantdfH07duv+Hi0t8G4WKiEhakSj7IIuIJFdZs2Zk+vRuTJ36\nEZkzewAQFRXBwoWfMH78y1y8eNLmDEVEJKXSHWSRNOzgwRNMmjSb69fP4u7uRceO71CkSF6703pk\nf/31Dz4+Qfz661+OWNaseWjb9htKl25gY2YiIpKc6A6yiDzQwYMn8PcfRN26G3jjjT+pW3cD/v6D\nOHjwxIMHJkPFij3B+vXD+Pjj1xyxixdPMm7cS3z7bQ8iI+/djEdERCRuKpBF0qhJk2bTosUJMmSI\neZwhA7RoEXNHOSVyc0vP8OFtWbp0IHnyZHPEV64cxciR1Tl5cp+N2YmISEqiAlkkjbp+/ayjOL4l\nQwa4fv2cPQklkvr1K/Drr2N46aXbu1kcOhRGQEBFNm+eYWNmIiKSUqhAFkmj3N29uHbt7ti1a+Du\nnt2ehBJR7tzZ+P77fowc2Y706WP6Gl2/fpnp09swdWprrl27aHOGIiKSnKlAFkmjOnZ8h7lz8zqK\n5GvXYO7cvHTs+I69iSUSFxcXPvqoKSEhIyhePJ8jHho6i6FDK3DgQKiN2YmISHKmXSxE0rDbu1ic\nw909e4rdxeJhLl++Rrduk/n669WOmItLOl59dSj163fXnskiImmEGoWIiNxj7twNfPjhRC5evOqI\nPf30i/j6foOn5xM2ZiYiIs6gbd5ERO7RokVNtm4NpkqVko7Y7t2r8PMrz/btP9qYmYiIJCdOL5CN\nMdmNMYuNMZeNMQeMMS3jOG6gMeaGMeaiMebSzX+LODdbEUltnnwyD2vWDKVXr+YYE3MT4fLlfxk/\n/mXmz/8/IiKu25yhiIjYzY47yF8A4UAuoDUwwRhTKo5j51qWldWyrCw3/z3orCRFJPVKnz4dfn6t\nWbZsMPnyeTnia9aMJTDweU6c2G1jdiIiYjenzkE2xmQEzgGlLcv6+2bsG+CoZVl97jl2IPCUZVlt\n4nFezUEWcZLU0p76ln//vch//vMZS5dudcTc3DLy9tvjqFatneMus4iIpHzJdQ5yCSDyVnF80x9A\nmTiOb2KM+dcYs90Y0znp0xORB0lN7alvyZkzK4sW9WHMmP/g7p4egBs3rjJjRgemTGnJ1avnbc5Q\nRESczdkFcmbgwj2xC0CWWI6dB5QiZipGR2CAMebtpE1PRB4ktbWnvsUYw/vvv0xISCAlSxZwxMPC\n5uHv/wx///2zjdmJiIizpXPy9S4DWe+JZQUu3XugZVl3TgL8xRgzFmhOTOF8nyFD5jg+r1WrLLVq\nlXvsZEXkbqm1PfUt3t5PsmVLEN27T2Hy5BUAnD17iNGjX+Dll4fQsOGnuLi42pyliIjE154969i7\nd90jj3N2gbwXSGeMeeqOaRbewI54jLWAOOeMDBgQ62YYIpKIbrWnvrNITi3tqW/JmNGdL754n3r1\nnuG998Zz/vwVoqKiWbKkH7t3r8LXdwbZsxd4+IlERMR2JUvWpmTJ2o7HS5cOjtc4p06xsCzrKrAI\nGGKMyWiMqQ40BWbce6wxpqkxJtvNz58DugJaiSdio9TenvpOzZpVIyxsDNWr395kZ+/edfj7e/P7\n79/bmJmIiCQ1p3fSM8ZkB6YC9YF/gV6WZc0zxtQAfrQsK+vN42YDDQA34Cgw3rKs8XGcU7tYiDhJ\nWmlPfUtkZBQBAfMJCFhAdHS0I16r1vs0azYKN7cMDxgtIvcaMKAIp04dsjsNScVy5y7MkCEHY31O\nraZFRBLRxo07ePfd0Rw9esYRy5evLB06zCVfvrg24hGRe90sUOxOQ1IxYwwTJ8b+PZZct3kTEUmR\nXnihDGFhY3j11ecdsePH/yQwsCIbNnyp/+GLiKQiKpBFROLJyysL8+f3Yvz49/DwcAMgPPwGs2d3\nZtKk5ly5ctbmDEVEJDGoQBYReQTGGP7zn5f45ZdRlClTyBH/3/8W4e/vzb59G2zMTkREEoOzt3kT\nSXOc1Zp54cL1BAV9QZYsEVy6lJ5PPnmf5s1rJXpuzno9yb2ldZkyhfj555F8+unXTJjwIwDnzh1l\n9Og6NG7cn8aN++Hqqh+xIiIpkRbpiSShW62Zb3Wfu7UtWr9+gxK12Fu4cD3TpgXTrRuO6wQHg69v\ntziL5ITk5qzX46zrJJYlS7bQsePnnD17u+fRU09Vp127WeTIUdjGzESSHy3SS3x16tShXLlyjBs3\nzu5UkgUt0hNJ5pzVmjko6AtHcXzrOt26xcQTMzdnvZ6U1tK6adMqhIUFU7Pm7d0s/v57E0OHPsNv\nv31rY2Yikljatm2Li4sLAQEBd8XXr1+Pi4sLZ8/Gfw1CnTp16Nq160OP8/X1pWnTpg89bvHixQwb\nNize17/XtWvX6NOnD8WLFydDhgzkypWLGjVqMG9erM2LY3Xo0CFcXFz47bffEpxHcqK//4kkIWe1\nZs6SJSLW62TOHJGouTnr9aTEltYFCuRk+fIhjBjxLX5+c4mKiubq1fNMmtScF17oyJtvBuPmltHu\nNEWStY4dh7N3b/h98RIlPJg06VPbzgUxdyUzZMhAYGAgnTp1IkeOHHc9Z4eIiAjSp09PtmzZHus8\nnTp14pdffmHcuHGUKVOGs2fPsmXLlkcq+i3Lsu19SAq6gyyShG61Zr5TUrRmvnQpfazXuXw5faLm\n5qzX46zrJDZXV1f69HmLNWsCKFw4lyO+ceMkAgIqc/ToNhuzE0n+9u4NZ/36Qfd9xFboOvNct9Sp\nU4ciRYowZMiQBx63YcMGnn/+eTJkyEDevHn5+OOPiYyMBGLuCq9fv57x48fj4uKCq6srhw8fjtf1\nfX19adKkCYGBgRQsWJCCBQsCULt27bvuSC9atAhvb28yZsxIjhw5qFOnDqdPn47zvD/88AO9e/em\nUaNGFCpUiGeeeYZOnTrx3nvv3XVcYGAgxYoVI2PGjHh7ezNr1izHc0WLFgWgcuXKuLi4ULduXSCm\ncPbz86NQoUJ4eHhQvnx5lixZctd5hwwZQpEiRfDw8OCJJ56gbdu2jueWL19OzZo18fLyIkeOHDRs\n2JDdu3fH6/16HCqQRZKQs1ozf/LJ+wQHc9d1goNj4omZm7NeT0pvaV216tNs3RpM8+bVHbETJ3Yx\nfPhzrF37ueZfiqRQLi4uDB8+nIkTJ3LgwIFYjzl+/DiNGzemUqVK/P7770ydOpU5c+bQu3dvAMaO\nHUvVqlXx9fXl5MmT/PPPP45CNz7Wr1/P9u3bWb58OatXrwbuvoN98uRJWrZsia+vL7t372bjxo34\n+Pg88Jx58+Zl2bJlXLx4Mc5j+vbty7Rp05gwYQK7du2id+/edO7cmZ9++gmA0NBQLMtixYoVnDhx\ngkWLFgEwZswYgoKCGDlyJH/++Sevv/46b7zxBtu2xdww+PbbbwkKCmLixIn89ddfLF26lOeee85x\n3StXrtCtWzfCwsJYv3492bJlo0mTJo5fOJKKpliIJKEiRWIWlt3Zmrlfv8TfjeHWQrz+/b8gc+YI\nLl9++C4WCcnNWa/HWddJStmyZWbWrO40aFCB//u/r7h69TqRkdeZN68Lu3atoE2bqWTOnNPuNEXk\nETVs2JDq1avTt29fZs++f13E+PHjyZcvH+PHjwegZMmSDB8+nM6dO+Pn50fWrFlxc3MjY8aM5MqV\n677xD5MhQwamTZtGunSxl3DHjx8nMjKSZs2aOQrv0qVLP/CckyZNonXr1uTMmZNy5cpRrVo1Xn31\nVV588UUArl69SnBwMCtXrqR69Zhf/AsXLsyWLVsYP348jRo1crwWLy8vcufO7Th3UFAQPXr04O23\n3wZg8ODBbNiwgVGjRvHNN99w+PBh8uXLR/369XF1daVAgQJUrFjRMf6NN964K9cpU6bg6elJaGgo\n1apVe5S37pGoQBZJYkWK5CUg4OMkv07z5rUeuq3bvRKSm7Nej7Ouk5SMMbRt+yLPP/80rVuPYtu2\ngwBs2/YDfn7etGs3k5Il69ibpIg8ssDAQKpWrUr37t3ve2737t1UrVr1rliNGjW4ceMGf/31F2XL\nln2sa5ctWzbO4hjA29ubevXqUaZMGRo0aMCLL75I8+bNyZkzJ0eOHHEUy8YY+vTpw6effsoLL7zA\n/v372bx5M5s2bWLNmjU0aNCATp06MWHCBHbu3El4eDgNGza861qRkZE8+eSTceZy6dIljh8/fl8h\nW6NGDced5zfffJOxY8dSpEgRXnrpJRo2bEjTpk1xc4tpxrR//3769etHaGgop0+fJjo6GsuyOHz4\ncJIWyJpiISKSxJ5+ugAhIYF06fKKI3bhwnHGjKnHd9/1JSoq7sWUIpL8VK5cmTfeeINevXrd91xc\ni9USaxFbpkyZHvi8i4sLK1asYOXKlXh7ezNlyhSKFy/O9u3byZ8/P3/88Qd//PEHv//+O507d3aM\nc3V1pXr16vTs2ZNly5bh5+fHpEmTOHz4MNHR0QD897//dYz/448/2LFjB8uXL39ozrG97luxAgUK\nsHfvXiZNmoSnpyfdu3enUqVKXLs5x+6VV17hzJkzTJo0idDQUH7//XdcXV25ceNGvN+zhNAdZBER\nJ/DwcCMoqAP16j1Dhw7j+Pffi1iWxbJlAezZs4b27WeTM2fcd2JE0oISJTyAQXHE7TtXbAICAihd\nujTLli27K166dGkWLFhwV2zjxo24u7vz1FNPAeDm5kZUVFSi5BGXKlWqUKVKFfr370+ZMmWYN28e\n/v7+jsV0D1OqVCkALl++TOnSpXF3d+fgwYPUqhX7Xypv3fG983VlyZKFfPnyERISQu3atR3xkJCQ\nu6Z9uLm50ahRIxo1akSvXr3ImzcvmzZtomLFiuzevZsJEyY4rvvbb78l+fxjUIEsIuJUjRtX5tdf\nx+DrO4Y1a2IWqRw4sBl//2do1epLnn22hc0ZitgnIduvOeNcsXnqqafo1KkTY8eOvSv+/vvvM3bs\nWN577z0++ugj/v77b3r37k2XLl3w8IgpzosUKUJoaCiHDh0ic+bMeHl5JdoWaVu2bGHVqlW89NJL\n5MmTh99++42jR49SpkyZOMfUqVOHli1bUrlyZXLkyMGOHTvo27cvTz/9NKVKlcIYQ/fu3enevTvR\n0dHUrFmTy5cvs3nzZlxdXenQoQO5c+cmQ4YMLF++nMKFC+Ph4UHWrFnp0aMHAwcOpFixYlSqVIkZ\nM2YQEhLi2C/566+/JjIykipVqpA5c2bmzp2Lm5sbJUqUIHv27OTMmZOvvvqKAgUKcPToUXr27En6\n9HHv0JRYNMVCJImFhGynceOOvPrqOzRu3JGQkO0PHbNw4XqqVn2bBg3eoGrVt1m4cP1Dxxw8eII+\nfUbzySf96NNnNAcPnkiM9G27Tmr2xBNe/PjjIIYObUO6dK4AhIdfZMqUlnzzTTvCwy/bnKGIxEf/\n/v1Jly7dXcVtvnz5+Omnn/j999+pUKECHTp0oFWrVgwdOtRxTPfu3XFzc6N06dLkzp2bI0eOPFYe\nd17f09OTTZs20aRJE0qUKEGPHj0YMGAALVu2jHN8w4YNmTlzJg0bNqRUqVJ8+OGH1KpVixUrVjjO\n7efnx6BBgwgKCqJs2bI0aNCARYsWOeYgu7q68tlnnzF58mTy58/Pa6+9BkDXrl3p0aMHvXr1oly5\ncnz//fcsWrSIcuXKAZAtWzamTJlCzZo1KVeuHIsXL2bx4sUUKlQIYwzz589n27ZtlCtXji5duuDv\n74+7u/tjvV/xoVbTIkkoJGQ7gYGD6No1ytEyedw4V3r2HESNGuViHeOsttEJkdJaQKcEW7fuxccn\niP37TzpiefKUoH37ORQqVPEBI0VSJrWalqSmVtMiyVxAwGeO4hhiCt6uXaMICPgszjHOahudECmt\nBXRK8OyzJQgNDaZFi5qO2MmTexkx4nlWrQpWISEiYgMVyCJJKH36y7G2TE6f/kqcY5zVNjohUmIL\n6JQga9aMfP11N6ZM+YhMmWLmKEZFRbBw4cd8/vnLXLx4yuYMRUTSFhXIIkkoIiJzrC2TIyLi3qbH\nWW2jEyKltoBOCYwx+PjUITR0NBUrPuWI79jxE/7+5dm5c6WN2YmIpC0qkEWSUJ8+XRg3zvWulsnj\nxrnSp0+XOMc4q210QqT0FtApQfHi+diwYTgff/yaI3bx4knGjWvAt9/2JDIyaff+FBERLdITSXIh\nIdsJCPiM9OmvEBGRiT59usS5QO+WhQvXExQU/7bRELOA7s7WzB07Jk1rZmddR2DFiv/Rvv1YTp48\n74gVLlyZ9u3nkDt3MRszE0k4LdKTpJYYi/RUIIuIJGMnT56nQ4dxLF/+myPm7p6Zli2/4PnnfWzM\nTCRhVCBLUtMuFiIiqVyePNn4/vt+BAb6kj59TG+n69cvM316G6ZN8yE8/JLNGYqIpD4qkEVEkjkX\nFxf+7/9eZePG4RQrls8R37JlJkOHVuDgwa02ZicikvqoQBYRSSEqVixGaGgQbdrUdcROn/6bwMBq\nrFgxkujoaBuzExFJPdLZnYBIYri9cOws7u5eSbZw7PaCu8tERGSO14K7CROWMHnydLJli+b8eRc6\ndGjLe+81feCYVq0C2LEjlBw54MwZKFPmOWbN6vPAMT16fMnKlT85xtSv34iRIzs9cMzQobNYuHAB\nXl5w9iw0b/4mffu2euAYZ73XzrpOSpM5cwYmT+5KvXrefPjhRC5dukZ0dCSLFvVk166VtG37NZ6e\nT9idpkiaVKdOHcqVK8e4cePsTkUekxbpSYrnrPbHCWkbPWHCEv7736n3tY1+5ZV2cRbJrVoFcP58\n6H1jsmWLu0ju0eNLdu786b4xpUvHXSQPHTqLn39ecN+YatXiLpLV0jp52b//BG3ajCY0dK8jliVL\nLtq0mU65co1tzEwkbil1kZ6vry9nzpxhyZIlcR5z/vx50qdPT6ZMce91/yDXrl3Dz8+PBQsWcPTo\nUTJnzkzJkiXp0qULb7/9drzOcejQIZ588knCwsKoWDFttqvXIj0RnNf+OCFtoydPnh5r2+jJk6fH\nOWbHjtBYx+zYERrnmJUrf4p1zMqVP8U5ZuHCBbGOWbhwQZxj1NI6eSlaNC9r1wbQs2czjIn5eX/p\n0mnGj3+Z+fO7ERFx3eYMRR7dwYMH6NWrNR99VIdevVpz8OCBZHGuB4mIiOl0mi1btgQXxwCdOnVi\nwYIFjBs3jj179rBy5Up8fHw4e/ZsvM9hWZbj54EknApkSfGc1f44IW2js2WLjnVMtmxxzxXNkYNY\nx3h5xZ1bQsZ4ecU+JvsDmuKppXXykz59Ovz9ffjpp0E88cTtL96aNWMIDHyeEyf22JidyKM5ePAA\nAwfWp3btWbz++jpq157FwIH1E1TYJua57uXr60uTJk0IDAykYMGCFCxYEIDatWvTtWtXx3GLFi3C\n29ubjBkzkiNHDurUqcPp06fjPO8PP/xA7969adSoEYUKFeKZZ56hU6dOvPfee3cdFxgYSLFixciY\nMSPe3t7MmjXL8VzRokUBqFy5Mi4uLtStG7NmwbIs/Pz8KFSoEB4eHpQvX/6+u+FDhgyhSJEieHh4\n8MQTT9C2bVvHc8uXL6dmzZp4eXmRI0cOGjZsyO7duxP2BqYAKpAlxXNW++OEtI0+f94l1jHnz8f9\nn96ZM8Q65kE3EBIy5uzZ2Mece0ANqpbWyVfdut6EhY2hcePKjtiRI78TEFCRn3+eliL/pC1pz4QJ\n/WnR4u97/nr0NxMm9Lf1XLFZv34927dvZ/ny5axevRrgrju3J0+epGXLlvj6+rJ79242btyIj8+D\n9y7Pmzcvy5Yt4+LFi3Ee07dvX6ZNm8aECRPYtWsXvXv3pnPnzvz0U8xfDENDQ7EsixUrVnDixAkW\nLVoEwJgxYwgKCmLkyJH8+eefvP7667zxxhts27YNgG+//ZagoCAmTpzIX3/9xdKlS3nuuecc171y\n5QrdunUjLCyM9evXky1bNpo0aUJkZGTC3sBkTgWypHjOan+ckLbRHTq0jbVtdIcObeMcU6bMc7GO\nKVPmuTjH1K/fKNYx9es3inNM8+ZvxjqmefM34xyjltbJW65cnixe3Jfg4A64ucWswb5x4yrffNOO\nKVPe4dq1CzZnKPJg4eHHYv3rUXj4cVvPFZsMGTIwbdo0SpcuTZkyZe57/vjx40RGRtKsWTMKFSpE\n6dKladeuHbly5YrznJMmTWLLli3kzJmTSpUq0aVLF1atWuV4/urVqwQHBzN58mTq169P4cKFadGi\nBR06dGD8+PEAjvN7eXmRO3dusmXLBkBQUBA9evTg7bffplixYgwePJgXXniBUaNGAXD48GHy5ctH\n/fr1KVCgABUrVuT99993XPuNN97g9ddfp2jRopQtW5YpU6Zw4MABQkPjnv6XkmkXC0nxihSJWbx1\nZ/vjfv0Sf8eDmIV4gx6pbfSthXi9esV/F4tZs/rQqlUA3buHOnaXeNguFiNHdqJHD+je/SfHmIft\nYtG3byuGDoUePRaQPXvMneOH7WLhrPfaWddJjYwxfPDBK9SoUYbWrYPYs+coAGFhczlwYDPt28+m\naNGqNmcpEjsPj/xcu3b39K9r18DDI1/cg5xwrtiULVuWdOniLqO8vb2pV68eZcqUoUGDBrz44os0\nb96cnDlzcuTIEUqXLg3E/Dfbp08fPv30U1544QX279/P5s2b2bRpE2vWrKFBgwZ06tSJCRMmsHPn\nTsLDw2nYsOFd14qMjOTJJ5+MM5dLly5x/PhxqlWrdle8Ro0ajjvPb775JmPHjqVIkSK89NJLNGzY\nkKZNm+Lm5gbA/v376devH6GhoZw+fZro6Ggsy+Lw4cP3nTc10C4WIiKp1JUr4XTvPoUpU1Y6Yi4u\nrrzyymAaNvwUFxdXG7OTtOpBu1jcmjd8a2pEzF+PnmLw4JUUKRJ3AZjU54K7d7GIa0eL2LZ527Jl\nCytWrGDJkiX89ddfbNiwgTJlynDw4EHHMV5eXo47vfcaOnQoAwYM4MCBA5w4cYLnn3+etWvXOuY9\n35I+fXoKFiwY6y4Wly5dwtPTkzVr1lC7dm3HmP79+7Ns2TK2bo1pNnTjxg1Wr17NqlWr+Pbbb8mS\nJQuhoaFkyJCB0qVLU7BgQXr16kX+/PlJly4dpUqVYvLkybRp0+aR38+kpF0sREQkTpkyeTBhwgfM\nnt0DT8+MAERHR7FkST/Gjq3PuXPHbM5Q5G5FijzJ4MErWbeuFYsX12HdulYJLmgT81yPo0qVKvTv\n35+tW7eSL18+5s2bh4uLC0WLFnV8xFUcA5QqVQqAy5cvU7p0adzd3Tl48OBd44sWLeoomG/d8Y2K\ninKcI0uWLOTLl4+QkJC7zh0SEuK4k31rbKNGjQgKCiI0NJQdO3awadMmzp49y+7du+nTpw9169al\nZMmSXLhwIdXOPwZNsRARSfWaN6/Os88Wp02b0fzyS8yq8z171uLv7827706jfPkmNmcocluRIk8y\nYsTMZHeuR7VlyxZWrVrFSy+9RJ48efjtt984evRorPOVb6lTpw4tW7akcuXK5MiRgx07dtC3b1+e\nfvppSpUqhTGG7t270717d6Kjo6lZsyaXL19m8+bNuLq60qFDB3Lnzk2GDBlYvnw5hQsXxsPDg6xZ\ns9KjRw8GDhxIsWLFqFSpEjNmzCAkJITffvsNgK+//prIyEiqVKlC5syZmTt3Lm5ubpQoUYLs2bOT\nM2dOvvrqKwoUKMDRo0fp2bMn6dOnd9bb6XS6gywikgYULpyb1auH0rfv27i4xPzov3LlDF980ZS5\nc7sQERFuc4Yiyd/D9he+83lPT082bdpEkyZNKFGiBD169GDAgAG0bNkyzvENGzZk5syZNGzYkFKl\nSvHhhx9Sq1YtVqxY4Ti3n58fgwYNIigoiLJly9KgQQMWLVrkmIPs6urKZ599xuTJk8mfPz+vvfYa\nALig8owAAAxTSURBVF27dqVHjx706tWLcuXK8f3337No0SLKlYtZR5MtWzamTJlCzZo1KVeuHIsX\nL2bx4sUUKlQIYwzz589n27ZtlCtXji5duuDv74+7u/tjvZ/JmeYgi4ikMRs2/EnbtsEcPXrGEcuf\nvxzt288lX77SDxgp8vhSaic9STkSYw6yplhImnXw4ImbuyScxd3di44dk88uCQnJbcKEJUyeHP/d\nMiTtqlmzLGFhY+jUaTzff78ZgGPHtjNyZAXeeOMzatT4jzpxiUiapgJZ0qSDB0/g7z/I0c742jXw\n999Lv36DbC+SE5LbhAlL+O9/pzJiBDfHRBMcPBVARbLEyssrC/Pn9+Krr5bTvftUwsNvcO3aDWbN\n6sTOnctp3forMmV6QCtGEZFUTHOQJU2aNGm2owCFWx2WYu7a2i0huU2ePJ1u3bhrTLduMXGRuBhj\n6NixIT//PJLSpQs54v/73yL8/7+9ew/Sqq7jOP7+wMoKEkJ5LwMzHW8hkWmlRJGZaQNN6WjmhdQg\nxHKGJrtomLhpqFwaU1FTy0pJx0zdtIs3UtFycxJ0IDQhTExAXBGBVdZvf5yzcNqeXfbG89tn9/Oa\nYWbP7/zO4XNmHni+z29/z+9XM4Jnn304YTozs3RcIFuv1NCwpuQOSw0NreyzXCYdyTZ48Nslrxk8\n+O1tkNB6moMOGspjj13GxIlbNh949dUXmDnzE9TWXkhjY89dysnMrBQXyNYrVVe/c/M2xk02bIDq\n6iFpAhV0JFt9fZ+S19TX+5+4tU3//tVcccXXuO227zBkyEAAIt6mtvYHzJo1hjVrlidOaGZWPn73\ntF5pwoSTmDt3t81FZbbD0m5MmHBS2mB0LNuZZ45n1iz+55pZs7J2s/YYN+4j1NXNYtSoLWu1Pvfc\nw9TUHMyTT96eMJmZWfl4mTfrtbasFPEq1dVDuukqFm3P5lUsrCs1NjYyffrtXHTRXBobt0zVGTVq\nIscfP5N+/QYkTGeVzMu82bbWFcu8uUA2M7MWzZ+/iFNPncny5as2t+2++wGcccYtvOc9wxMms0o1\ndeowVq78V+oY1oPtsstQpk1bVvKcC2QzM+sS9fXrmDTpKm6/ff7mtqqqao47bgajR5/lNZPNrGK0\ntUAu+xxkSUMk3SFpnaSlklrcc1HSdEmrJa2SNL2cOc3MLDN48EBuvvlbzJkzmf79+wGwaVMDc+ee\nzdVXf551617Zyh3MzCpLii/pXQVsBHYGTgaulrR/806SJgJjgQ8Aw4HPSZpQzqBWeebNW5g6gnUD\nfh10PUmcfvqnefzxGQwfPmxz+4IFd1FTczD/+MdDybK1prvmsvLy68Daq6wFsqQBwBeA8yNiQ0Q8\nCtwFnFKi+6nAjIh4KSJeAmYA48sW1irSvHlPp45g3YBfB9vO/vvvySOPXMrZZ39uc1t9/YvMnj2G\nO+88n8bGtxKm+39LljyUOoJ1A34dWHuVewR5X2BTRPyz0PYUcGCJvgfm57bWz8zMymj77fsxc+aZ\n3HHHeey00yAAIoJ77/0hM2aMZvXqZWkDmpl1UrkL5IHAa83aXgPe0Ya+r+VtZmbWDRx77Iepq5vN\nmDFbVrN4/vnHqKk5mLq6XydMZmbWOWVdxULSCOCRiBhYaJsCjI6Icc361gNHRkRdfjwSeDAidixx\n38pfisPMzMzMtrm2rGJRVY4gBUuAKkl7F6ZZHAw8U6LvM/m5uvx4RAv92vSgZmZmZmZtUdYpFhGx\nHvgNME3SAEmHk61U8YsS3W8CpkjaQ9IewBTgxvKlNTMzM7PeKMUyb5OBAcBK4FfA1yJikaQjJK1t\n6hQR1wB3AwuBBcDdEXFdgrxmZmZm1ov0iJ30zMzMzMy6SooRZDMzMzOzbquiC+T2bFttPZekyZKe\nkLRR0g2p81gakvpJ+qmkZZJek/Q3SUenzmVpSPqFpBX5a2GxpDNSZ7J0JO0jaYOkm1JnsTQkPZS/\nBtZKel3Sotb6V3SBTBu3rbYe70XgIuD61EEsqSpgOTAqXw5yKnCrpPemjWWJXAwMzV8LY4EaSR9M\nnMnS+Qnw19QhLKkAzoqIQRHxjohotV6s2AK5ndtWWw8WEb+NiLuANamzWDoRsT4ipkXEC/nx74Cl\nwIfSJrMUImJRRDTtey2yN8e9E0ayRCSdCLwK3J86iyXX5mWBK7ZApn3bVptZLyNpV2AfWlg/3Xo+\nSVdKegNYBKwA7kkcycpM0iDgQuCbtKM4sh7rEkkrJT0saXRrHSu5QG7PttVm1otIqgJ+CfwsIpak\nzmNpRMRksveKI8jW4G9Im8gSmAZcFxEvpg5iyZ0LvA94N3AdcLekvVrqXMkF8jpgULO2QcDrCbKY\nWTchSWTFcQPw9cRxLLHIzAf2BCalzmPlI2kEcCQwO3UWSy8inoiINyLirYi4CXgUOKal/uXearor\ntWfbajPrPa4HdgKOiYjG1GGs26jCc5B7m9HAUGB5/sF5INBX0gERcUjaaNYNBK1Mu6nYEeR2bltt\nPZikvpK2B/qSfWiqltQ3dS4rP0lzgP2AsRHxZuo8loaknSWdIGkHSX0kfQY4EX9Jq7e5huxD0Qiy\nAbQ5QC1wVMpQVn6SdpR0VFN9IOnLwCjgDy1dU7EFcq7kttVpI1kC5wPrgW8DX85/Pi9pIiu7fDm3\nCWRvhi/n61yu9frovVKQTad4gWx1m0uBcyKiNmkqK6uI2BgRK5v+kE3N3BgRXvGo99kOqCGrF1eR\n1Y/jIuLZli7wVtNmZmZmZgWVPoJsZmZmZtalXCCbmZmZmRW4QDYzMzMzK3CBbGZmZmZW4ALZzMzM\nzKzABbKZmZmZWYELZDMzMzOzAhfIZmYVRNJpkl7fSp+lkqaUK1NrJA2V9LakkamzmJm1lQtkM7N2\nknRjXvQ1SnpT0j8lXSZpQDvvcVcHI3TLHZ5aeaZumdfMrCVVqQOYmVWoPwEnA/2AUcD1wACyLUzt\nfyl1ADOz9vAIsplZxzRExKqIeDEi5gK/Aj7fdFLSAZJqJa2V9LKkmyXtmp+7ADgNOLYwEv3x/Nwl\nkhZLWp9PlZguqV9ngkoaJOnaPMdaSQ9K+lDh/GmSXpc0RtJCSeskPSBpaLP7fFfSf/J7/EzSVElL\nt/ZMuWGS/ijpDUnPSDqyM89kZrYtuUA2M+saG4HtACTtDswDFgCHAJ8CdgCaph9cDtwK3AfsCuwO\nzM/PrQPGA/sBk4ATgPM6me0eYDfgGGAE8Gfg/qaCPVcNfCf/uz8CDAbmNJ2UdCIwFfguMBJYDExh\ny/SJ1p4JoAaYDQwHngBuac+UFDOzcvIUCzOzTpJ0KPAlsmkXkBW2f4+I7xX6jAdekXRIRNRJ2gAM\niIhVxXtFxA8Lh8slXQJ8E7igg9nGkBWlO0dEQ958gaSxwClkhS1AX+CsiHguv+5y4IbCrb4B3BAR\nN+bHP5L0SWCfPPcbpZ5J2jy7YmZE3JO3fQ84laxYLxbRZmbdggtkM7OO+Wy+mkRV/ue3ZEUkZCOs\no0usNhHA3kBdSzeVdBxwDvB+YCBZ4dqZ3/aNJBu9Xl0oViEbMd67cNzQVBznVgDbSRocEfVkI9rX\nNrv3X8gL5DZY2PRDRKzIs+zSxmvNzMrKBbKZWcfMA74KbAJWRERj4VwfoJZs5Lf5F9RebumGkg4D\nbiEbLf4DUA+MAy7rRM4+wH+AI0pkWVv4eVOzc01TJ/qUaOuIt1rIZmbW7bhANjPrmPURsbSFc08C\nxwPLmxXORW+SjQ4XHQ78OyIubmqQNKyTOZ8kmxMcreRti8XAocDPC22HNetT6pnMzCqOP72bmXW9\nK4EdgVslHSppL0lHSrpG0g55n2XAQZL2lfQuSVXAEuDdkk7Kr5kEnNiZIBFxH/AocKekoyUNk/RR\nST+QdPhWLi+OOP8YGC/pK5LeL+lcsoK5OKpc6pnMzCqOC2Qzsy4WES+RjQY3AvcCTwNXkK100fRF\nueuARWTzkVcCH4uIWrLpFLOAp8hWv/h+RyI0Oz4GeIBsDvFiYC6wL9k84zbdJyJ+DVwEXEI2Kn0A\n2SoXGwv9/++ZWsjTUpuZWbegCP8fZWZm7SfpN0DfiBiXOouZWVfyr7/MzGyrJPUnW77u92Qj418E\nxgJfSJnLzGxb8AiymZltlaTtgbvJ1i7uDzwLTM93ETQz61FcIJuZmZmZFfhLemZmZmZmBS6QzczM\nzMwKXCCbmZmZmRW4QDYzMzMzK3CBbGZmZmZW8F8oWao9tA0FUQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f67fd7be588>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = -per_clf.coef_[0][0] / per_clf.coef_[0][1]\n",
    "b = -per_clf.intercept_ / per_clf.coef_[0][1]\n",
    "\n",
    "axes = [0, 5, 0, 2]\n",
    "\n",
    "x0, x1 = np.meshgrid(\n",
    "        np.linspace(axes[0], axes[1], 500).reshape(-1, 1),\n",
    "        np.linspace(axes[2], axes[3], 200).reshape(-1, 1),\n",
    "    )\n",
    "X_new = np.c_[x0.ravel(), x1.ravel()]\n",
    "y_predict = per_clf.predict(X_new)\n",
    "zz = y_predict.reshape(x0.shape)\n",
    "\n",
    "plt.figure(figsize=(10, 4))\n",
    "plt.plot(X[y==0, 0], X[y==0, 1], \"bs\", label=\"Not Iris-Setosa\")\n",
    "plt.plot(X[y==1, 0], X[y==1, 1], \"yo\", label=\"Iris-Setosa\")\n",
    "\n",
    "plt.plot([axes[0], axes[1]], [a * axes[0] + b, a * axes[1] + b], \"k-\", linewidth=3)\n",
    "from matplotlib.colors import ListedColormap\n",
    "custom_cmap = ListedColormap(['#9898ff', '#fafab0'])\n",
    "\n",
    "plt.contourf(x0, x1, zz, cmap=custom_cmap, linewidth=5)\n",
    "plt.xlabel(\"Petal length\", fontsize=14)\n",
    "plt.ylabel(\"Petal width\", fontsize=14)\n",
    "plt.legend(loc=\"lower right\", fontsize=14)\n",
    "plt.axis(axes)\n",
    "\n",
    "save_fig(\"perceptron_iris_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Activation functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def logit(z):\n",
    "    return 1 / (1 + np.exp(-z))\n",
    "\n",
    "def relu(z):\n",
    "    return np.maximum(0, z)\n",
    "\n",
    "def derivative(f, z, eps=0.000001):\n",
    "    return (f(z + eps) - f(z - eps))/(2 * eps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure activation_functions_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxAAAAEYCAYAAADMNRC5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd0FNXbwPHvzaaRBoQeEAEB6U2KFCF0BRHBgiIKYkOx\nIPb2w/basGLBCijFggIiWEAhAtIEpCMdpENIIJCe3fv+cXdDSDYhZWvyfM7Zszuzd2buPpns7J3b\nlNYaIYQQQgghhCiMAG9nQAghhBBCCOE/pAAhhBBCCCGEKDQpQAghhBBCCCEKTQoQQgghhBBCiEKT\nAoQQQgghhBCi0KQAIYQQQgghhCg0KUAIn6aU2quUGuuB44xTSm30wHGUUuoTpVS8UsqqlOrq7mNe\nID+TlVJzvZkHIYTwJ0qp4UqpJA8dy6aUGuyJYwlRFErmgRCuopRqBawBVmitryjituOA67XWzXOt\nrwQka63TXJTHi4G9QFut9boc68OAEK11oiuOU8Dx+wM/AN3s+UjQWme585j243YDFgOVtdYJOdZH\nYr4HPHIxFEIId1NKTQaGAxqwAonAFuB74NOSfucqpUKASK11fEnzmmOfk4FKWutrcq2vCiRqrTNd\ndSwhXEFqIIQr3QV8CDRTSl1ajO3zlGa11iddVXiwU/kcJ8XdhQe7BsARrfUqrfVxTxQe7ByfW+Vc\nqbU+I4UHIUQptBCoDlwM9AbmAi8AS5VS5Yq7U6VUoNY63ZWFh4LYrxNSeBA+RwoQwiWUUqHAUOAz\nzF2eO52kqaGUmm5vvpOslFqnlOqmlBoOjAOa2qtrrUqp2+zbZDdhUkrNUEp9n2ufSin1n1LqIfty\nX6XUEqVUglLqpFLqV6VUoxyb7LE/r7Efa5F9u+eVUpty7fc5+77TlFIblVLX5Hj/YkfVslJqgf3z\nbFFK9SogRpOBt4Ha9m332NfHKaUm5E6bs2mRUmqxUupDpdT/KaVOKKWOKaXG59omSCn1ilJqnz3P\nu5RS99trXRbZk52wx3eSfZspuY4TrJR6Vyl1VCmVqpRaoZTqnOP9bva891BKrbR/7r+VUq1zpIlS\nSk215zHVno8H84uLEEK4QbrW+oTW+ojWeqPW+l0gFmgDPA7Z35mvK6UOKKXOKqVWKaX6OHaQ4/vu\nKvt7aUAfexOmM/Y0DexpmuY8uFLqbvt3tUUpFaCU+lwptUcplaKU2qGUeixH2nGYGpP+Oa6BXe3v\nZTdhUkotd/K9H2nf58BCfqZApdQEpdQh+3Viv1LqFRfGXZQRUoAQrnIDsE9rvRmYBtymlLI43lSm\nidASoDYwEGgGvGh/+xvgLWA7UA2oAXzr5BjTgH5Kqagc62Ixd5lm2JfDgXeAtphmQqeAn5RSgfb3\n22Puwvexb+doW6o5v2ZiDPAI8Jg9r7OBWUqpFrny9DLwLtAC+Bv42v5ZnXnQ/pkP2j9nuxzHLoyh\nQCbQERgNjFFKDcnx/lfAMHveGwF3YD7/f8B19jSNMfF9KJ9jj8f8LUcArYBNwK9KqWq50r2CuQi3\nBk5i/jYO/wc0BfoBlwIjgUOF/IxCCOEWWustwK+c+z6cAlwB3IT5nv8SmKuUap5r09eAZzDfq6sc\nu7Pvcyfmu/+WXNsMBb7WWlsxv7UOAtfb9/E08JRS6nZ72jeB74DfOXcNXO7kI0yz5zWn64EUYH4h\nP9NDmGvwjUB9YAjm2itE0Wit5SGPEj+AOODhHMt7gEE5lu8CTgMV89l+HLDRyfq9wFj7awtwDLg9\nx/ufAb8UkK9wIAvoZF++GLABbQo6PubL/plcaRYDX+Xaz5053o+xr+tUQH4eAfY42e+EXOsmA3Nz\npfkrV5oFmPa8YJpG2YDe+Ry3G6YtcHR+xwHCgHTglhzvBwC7gBdz7McG9MqRppN93zH25R+BL7x9\nTspDHvIom4/c35+53nsVOAvUs39v1cr1/mzgA/trx/fdtbnSDAeSciw/COzNsVzLvu/2BeTxVWDB\nhfJsP/5g++to+3d09xzvLwQm2l9fUojP9B6w0Nt/I3n4/0NqIESJKaXqA52Br3OsnoEpNDi0wvxA\nL3Y/A23u5HyL/U6PUioYcydpao681FOmqdMupdRp4CimxqF2YY+jTMfiGPLeAVoGNMm1LrvZk9b6\nsP1l1cIeq4hyjxJ1OMexWmMuHHEl2P8lQCA5PrfW2gas4PzPrcnxue35UDnyMhEYopRar5Qar7w8\n0pQQQuTg6A/Wxv56q1LqjOOBqTm9JEd6Day9wD6/BmoqpbrYl28BdmutV2cfVKlR9uaex+3HeZgi\nXJcAtBkA4zfOXQNrAN05dw1sXYjPNAVobW9G9YFSqp9S6ry+cUIURuCFkwhxQXdi7lQfyP09pJSq\nqbU+RK7OuyUwDfjL/sXZEQgC5uR4fx5wALgb02wmC9gGBBfjWM6aFuVe56xzW1EL5jbyxifISbrc\nx9LFOFZBHHko6ud2vBcAoLX+VSlVG7gK6AnMV0p9p7W+w4V5FUKI4miCqSEPwHz3tsVcJ3JKzbWc\nXNAOtdYnlFK/Y37YL8M0X8pu1mlvavoOMBZzQyYJuB+4thj5nwZ8opS6D7gZ+E9r7bjpc8HPpLX+\nx94v7kqgB6aJ03pMR3MhCk1qIESJ2Ps53AY8CbTM9dgIONp4rgNaKKWi89lVBqaJUoHsd3R2Y76g\nhwJztNYp9rxEY9qXvqK1XqS13g6U5/yCcob9Od9jaa3PYO6qd8n1Vhdg64XyWAwnMG1ec2pZxH2s\nw/w/d8/n/Qt+bkxTpUxyfG6lVACmoLalKJnRWidoradrrUdi+mIMV0o5KxQJIYRHKKWaYX44zwT+\nwXxn1tBa78n1OFKM3U8DblBKtQGaA9NzvNcZWKm1nqi1Xq+13oPpf5BToa6BmCaiAAMw18Ccx/kH\ncyOowM+ktU7WWv+gtR4N9Ad62lsSCFFoUgMhSupqoBLwee7mSUqpb4F7MR2NZwBPAHOUUk9j+hg0\nx7Qj/RPYB1xsH83nP+CM1joD52Zgaj0uBgblWJ8IxAN3KaUOYtqhvsH5d8uPY+7E9FVK7QfStPNh\nTMcDLyildmGqr2/F/LBuc8GIFN0i4B2l1ABMZ7Z7gIsw/T8KRWu9Syk1E/hcKTUGU6CoBdTRWk8D\n9mNqCvorpeYBqVrr5Fz7SFFKTQReU0qdtB9/LKZp0sQcSQusTVJKvWA//hZMTcp1mOp8GYpQCOEp\nIfbBHwKAKkAv4ClMh+e3tNapSqnpwBSl1KOY76xozMAcu7XWjprtwtaezwY+Br4AVmmtd+d4bwfm\nJsqVmBs1NwNdgYQcafYBVyqlGmIGpjhlb7Z7Hq11ulJqNvAsZvCOW3K8t1MpNaOgz6SUehg4gql1\nyLJvfxpzTRai0KQGQpTUSGBRPn0bZmKGLO1lryXohmlWNBfYDDzPueYvPwA/A39gfuQ7Rppw1pxm\nGmZ0n1OYDmQmodYaM7JEC0wb/fcxX7LpOdJYgQcwBZBDnN/8KacJmELE6/Z9DcR0ZMvZ9r8wTX0K\nY5L98QWm+vsMMKsY+70VU7h6D9NsazIQBdn9M8ZhRkg6iomNM09gRgOZhLmb1Qzoq7U+doG85FyX\njik0rgeWYjqyX+NkGyGEcJdemJrk/ZjRja7GfAd201o7miiNwHxPvo75zvwJM4LR/hz7KdR3un2f\nszHXn6m53v4E8706HViN6fvwZq40n9nzsAZzDXQMn+3s+FPtx1lrr2nP6UKf6QxmdMFV9mO1AK7U\nrp1vSZQBMhO1EEIIIYQQotCkBkIIIYQQQghRaFKAEEIIIYQQQhSaFCCEEEIIIYQQhebzozAppaST\nhhBCeIDW2m8mlJJrgxBCeIaza4Nf1EB4e7pux2P48OFez4OvPSQm/heX+PnxLGYxqxqtwmazSUy8\n+PClmPgjb8fM1/6GvvSQuEhMJC6lIyb58YsChBDCdU7OPwlAtWHVyD1zuBBCCCHEhfh8EyZfUqdO\nHW9nwedITJzz5bg0eL8BVYdUpVz9ch49ri/HxFskJv5P/obOSVzykpg4J3HJyx9iIgWIIoiNjfV2\nFnyOxMQ5X46LClBU6FrB48f15Zh4i8TE/8nf0DmJS14SE+ckLnn5Q0ykCZMQQgghhBCi0KQAIYQQ\nQgghhCg0VVAPa1+glNK+nkch/MHpFacJLB9IeJNwb2dF+CClFNrPhnGVa4MQQrhXftcGqYEQoow4\n+89ZNvTZkD0KkxBCCCFEcUgBogji4uK8nQWfIzFxzhfjUvO+mnTc35GKfSt65fi+GBNvk5j4P/kb\nOidxyUti4pzEJS9/iImMwiREGaIsCoXftFIRQgghhA+SPhBCCCGkD4QQQog8pA+EEEIIIYQQosSk\nAFEE/tAmzdMkJs75Ulz2v7afvc/tJe1gmlfz4Usx8RUSE/8nf0PnJC55SUyck7jk5Q8xkT4QQpRi\ntkwbB98+SOaJTCpfW5nQWqHezpIQQggh/JxL+0AopUYDI4DmwAyt9cgC0j4MPA6EAj8A92qtM52k\nk3auQhTTyV9OsqnfJspdWo7229qjlN80cRceJn0ghBBC5JbftcHVBYhrARvQFyiXXwFCKdUXmAJ0\nB44Ac4AVWuunnaSVi4QQxWRNs5IwPwGtNVWvr+rt7AgfJgUIIYQQuXmkE7XWeo7Wei6QcIGktwFf\naK3/1VqfBl4CbndlXtzBH9qkeZrExDlfiYsl1EKV66r4ROHBV2LiSyQm/k/+hs5JXPKSmDgnccnL\nH2LirT4QTTG1Dg4bgKpKqYpa60Qv5UkI1zl2DHbs8HYufMuBA6U6JlqD1QpWm8JmM68dN8g16txr\nfW590rZDnCy364LpznttT1PYdM7yKYQQQpSEW+aBUEq9BNQsoAnTLuA+rfUC+3IgkAHU0Vr/lyut\nVFML//LBB/DAA97ORammgSSiSCCaJKI4SwTJhOf7nE4I6YSQQTAZBOf72rGcRSBWLOc9bAQUuE77\n/aB20oRJCCHE+fJrwuStGoizQFSO5SjMb4IzzhKPGDGCOnXqAFChQgVatWpFbGwscK6aR5Zl2WeW\nf/uNWIAqVYgLDjbvh4WZ91NSPLLcJaQSVh3MiowjXjl+cZaztIXZZwM5aa1I9cBmHM6qyvLU08Rb\nK2BR3Uiwledg5jaSdATJtn5YCQRM/DERL8Fy5xJub5YVfxCAjSB1BQHYsOk/AbAEXAForCwhQCuC\nVBcUmky9FAUEq86kBtrIsi0DBQGWTmjAaltKVFYAwaozCk2GXgZAaEAnDpfLAutyc/jAy0FpyFpB\ntVQL5QIuByDNtgKlNOUCOrInIhMyV5r0QR0gcxWk/0BEVgBBqhaJNoQQQohC8VYNxHRgj9b6Ofty\nD2Ca1jrGSVqfucsUFxeX/UNRGBITJ+66i7jPPyf2s8/gzju9koWE3xPYesNWaj5Qk7ov1vVKHnKL\ni4uja9dY9u6Fbdtg585zjx074OBBsBXhR2xkJFSsCOXLQ0QEhIefe879ulw5CA42j5CQC78ODASL\n5fyHCrCRlH6KxIx4EtJOkJB2gtMZCdzZdiQWCwQEgGOQqyxbFnXfq0tiaiLJmcnZeVYorP+zZo+G\n5fj/sWkblhctTj+n9X9WAtT5tRtaawJeDCAwIJDQwNDzHtvv3+40/ZDvhxBsCSbIEkRwQHD267f6\nvIVSSjpRF5N8BzoncclLYuKcxCUvX4qJR2oglFIWIAiwAIFKqRAgS2ttzZX0K2CyUmoGcBR4Bpjs\nyrwI4TVW++ke4L0mLdG9oul4uCOZ8XlGRvYYrWH7dlizBtatg0WLYO9eSEpynl4pqFYNYmKgZs3z\nn6tVg0qVIDraPCpWhKAg1+TTarNyLPkYB5MOsvfMYQZeOjDPcLc2bSPs/8JIt6bn2X5k+6EEW86f\nXyMwIJCTKSdJzUolQAVQIbQCFUIrUD6kPBnWDEICQ85LH6ACGN97PKGBoYQHhRMRHEFEcAThweFO\n86yUIvO5TAIDCvcVrpTiuxu+K1RaIYQQ4kJcPYzrOGAcpjmSwwuYwsFWoLHW+qA97RjgScw8EN8j\n80CI0mLECPjyS5g82bwuIxwFht9/h7g4WLIETpzIm65GDWjaFBo2hAYNzKNhQ7j4YnP331N6fNmD\nXQm7OHzmMNYc9zgSHk+gYrmKedJXf7M6aVlpVA2vSpXwKlQNr0rVsKqM7zOeqJCoPOkPJh0kKiSK\nyOBIv5h/Q2oghBBC5OaRGgit9QuYAoMzkbnSvgu868rjC+ETHO1wLM6bpJQmVqspKMydC/Pmwa5d\n579fowZcfjm0aWMerVubde6QmJrIlhNb2HpiK//G/8uuhF3sTtzNotsWUS2iWp70/53+jwNJBwCo\nGl6VmpE1qRVVi9SsVCqStwBx4OEDBFkKX+1RK6pW8T+MEEII4cO81YnaL/lSmzRfITFxwmolDoj1\nYhMmd9u4EaZOhRkz4PDhc+ujo+HKK6F7d+jWDerXP9cvIC4ujho1Yt2Wp45fdGT7ye151u9K2OW0\nADFryCwigyOJiYzJ06TImaIUHgpL/n/8n/wNnZO45CUxcU7ikpc/xEQKEEK4mqMGwgsFiLObz3Jq\n8SmqDqlKcFXXtgfKyIDvvoP33jP9Ghzq1YMbboCrr4aOHV1b8ZKamcqGYxtYe3gt646sY+2RtXx8\n9cdcXuvyPGnb12xPRHAETas2pXHlxtSPrk/96Po0qtzI6b5bVGvhuowKIYQQZYhbRmFyJWnnKvzO\nkCHml/a338KNN3r00DvH7OTQe4eo+VBNGrzbwCX7PH0a3n8fPvwQjh416ypWhJtugmHDTKHBHU38\nH/j5ASaumXhe/wSACVdO4IEOMs+Gq0kfCCGEELn52jwQQpReXhqFyZZl4/jXxwGoNixvk52iSkkx\nc+K99hok2ueHb9YMxoyBoUPN0Kglte/UPqw2K5dEX5LnvWoR1dBomlVtxmU1LjOPmMtoWa1lyQ8s\nhBBCiGIrvY203cAxaZg4R2LihM1mphfzdBMmK9R9sS5Vb6lK5GWRF06fD5sNJk0y/ReeeMIUHrp2\nNaMrbdwId9xRvMJDXFwcZzPOMm/HPB74+QEu/eBS6r5Xl7dWvOU0/f3t7yfpySQ23buJKddO4YEO\nD9Dpok75Dm3qj8rC/49SarRS6m+lVJpSatIF0j6slDqilEpUSn2ulHJ9xxMXKwt/w+KQuOQlMXFO\n4pKXP8REaiCEcDUv9YEICAkg5p4YYu7JMx9joW3dCqNGwdKlZvmyy+CVV6B375I3U1p9aDV9Xu9D\npu3caM3lQ8pTLtB5aaRCaIWSHVD4ikPAS0BfIN+ip1KqL/A40B04AszBjOr3tAfyKIQQogikD4QQ\nrnbNNfDTT2Zs0wEDvJ2bQsnIgJdfNs2VMjOhalV4+23TVMlV/RuOnj1Krbdr0TamLX0v6Uvf+n1p\nX7N9oSdDE+7l7j4QSqmXgJpa65H5vD8d2Ku1fta+3AOYrrV2OvCvXBuEEML9pA+EEJ7iAzNRF8W+\nfabf9+rVZvnuu01BomLeqRDyZdM2VhxYwYxNM1i8bzEbRm3IM+xp9YjqnHz8JOVDy7su86I0aYqp\ndXDYAFRVSlXUWid6KU9ClFhmJuzdaybbjImByBwtTDNOZJBxJIOIFhHnbZNxIoPkTckABFUOcvq+\nbCfbuXu7gvjHLxwf4Q9t0jxNYuKEF/pA2DJsxdpuzhwzudvq1VC7tpkU7pNPCl942HRsE0/9/hT1\n3qtHl8ld+GjNR2yL38ZfB/7KkzYuLk4KD7nI/895IoDTOZZPA4pck5D6GvkbOidxMQWHjz4y/cku\nvRQaNYqjRg145BE4bsa7IGlFEnuf3Ztn26QVSWzouYENPTfk+35p2e67Ud/5RT49ud2XPb/0iXwW\nRGoghHA1D89EnXUmi1X1VhHdL5pGUxqhCtHmyGaDJ5+E8ePN8jXXwOTJZiK4onhkwSMs3LMQMDMv\n39zsZm5qdhOtq7cu6scQ4iwQlWM5CtDAmfw2GDFiBHXq1AGgQoUKtGrVKnvyJccPWHcvO3jqeP6y\nvH79ep/Kj6eXf/stjnHjYNUqs1yx4mJuSJzFgGQLA97uwvTpf/LWW3B13daENw/Ps/3K/1ZytNVR\n2lZsm+/7SVFJNKf5ecdvXbk1FbpXYE3iGg5HHXb6vq8dLzQmtFR/vuIc79B/h9gUtckrny8uLo4p\nU6YAZH+/OiN9IIRwtV694I8/YOFC89oDMk5kkLQiicrXVL5g2rQ0GD7cTFURGGgKEQ89VLy+DjO3\nzOSPvX8wtPlQutTuQoCSSk1/5SN9IPZorZ+zL/cApmmtnY4KINcG4avS0qB/f1i0CCpVgo8/hkED\nNUuD/4QAePyyWP7+G2rUgD//hAaumbJHCLeQPhBCeIoXRmEKrhJcqMJDYiJce61pqhQVBbNmQc+e\nBaRPTWTSP5M4m3GWcbHj8rx/Q9MbuKHpDSXJuijllFIWIAiwAIFKqRAgS+tcMwTCV8BkpdQM4Cjw\nDDDZo5kVwgWeesoUHqpXN8NfN20KtgxT2FUWxR9/mFrfuDgz1+jKlRAS4t08C1FUcruwCHJXWQuJ\niVOOPhAeasJUWEeOQJcupvAQE2OGas2v8LD1xFbunXcvtd6pxaMLH+XNFW9yNuNsiY4v50peZSQm\nzwIpwBPALfbXzyilLlJKnVFK1QLQWv8GvAEsBvbaH897JcdFUEb+hkVWVuPy22/w7rumdnfuXFN4\nAFBBCut8K52OdyIy0rxXrx6sXw/PPuvdPHtbWT1XCuIPMZEaCCFczUvzQBTkxAnTmmrrVmjSBH75\nxXSazi3TmsmArwfw2+7fstf1rtebBzs8mO98DUIURGv9AmY+B2cic6V9F3jX7ZkSwg2Sk2GkvYHe\niy9Cu3bn3lNKYQmzEFTBjE4XGQnTp5ubOm++CdddB5df7oVMC1FM0gdCCFfr3BmWL4dly8xrN9Fa\nE/9jPNF9orGE5V/bkZgIPXqYO11Nmpg2t5ULaO1048wbmb9zPsNbDuf+9vfTpEoTN+Re+Bp394Fw\nNbk2CF/z0kvwv/9B27amWVJhKqGfesoMm+2oHXbVvDtCuEp+1wYpQAjhah07mqvH8uXmtZucWXuG\ntW3XEnpJKB12dnA6+lJSkplFevVq01FvyRLTLrcgB5MOEh4UTsVyRZgIQvg9KUAIUXzHjpnhWs+e\nNX0bunUr3HZJSXDJJRAfb4bVHjjQrdkUosjyuzb4ThsLP+APbdI8TWLihIf6QBybdgyASv0rOS08\nZGaaavHVq6FOHTMwVPXqkGHN4OM1H/Pk70863W+tqFpuKTzIuZKXxMT/yd/QubIWl5dfNoWHAQPy\nLzw4i0lUlKm1ADO0tjX30AJlQFk7VwrDH2IiBQghXM1DM1GXq1+OsEZhVLulmtP3x441I4BUq2Yv\nPMRk8sW6L2j4fkPunX8v45ePZ1fCLrfmUQghSrv4ePj8c/P6lVecp8k4kcHGqzay8pKVed675x64\n+GL491/46Sc3ZlQIF5ImTEK4Wps28M8/sHatee1Gjv+N3DUQn35qLkrBwaY6fV/E1zy3+Dl2J+4G\noHHlxrwQ+wLXNblO5m4QgDRhEqK4HH0f+vWD+fOdp9E2jTXZCjYILJ93/Jr33oMxY0xfiKVL3Zxh\nIYpAmjAJ4SkenIlaKZWn8LBkCYwebV5/8onphrFk/xJ2J+6mQXQDpg+ezqZ7N3FD0xuk8CCEECWQ\nlgYffGBeP/JI/ulUgCIwMtBp4QHM6E3ly5uxN1avdkNGhXAx+fVQBP7QJs3TJCZOWK2mD4QXhnE9\neND0e8jKMk2YRoww65+PfZ5J10xi6+itDG0+FEuA5+eokHMlL4mJ/5O/oXNlJS4zZsDx49C6NXTv\nXnDagmISGQmjRpnX77zjuvz5g7JyrhSFP8REChBCuJqX5oGwWuGWW7OIj4c+feCNN869Vy2iGre3\nvp3AAJn6RQghXOWzz8zzgw+WfAjW0aPNZWPWLNOvQghfJn0ghHC1xo1Nb7itW81rF/v3jn+xRFio\n/URtQmJCADMB3OCHVjLvwyuoWDmTbZuDqOa8b7UQTkkfCCGKZssWaNbM1B4cOQLh4SXfZ79+ZqLP\nd94xfSKE8DbpAyGEp7i5D0TtJ2sTFB2Espj/58V7F9PomVuYN7EDAD3GTJXCgxBCuNkXX5jnoUMv\nXHhI2ZXC0silrGmzpsB0d955bt9SPha+TAoQReAPbdI8TWLihJv7QIQ1CKPOuDokhicy5Psh9Pj8\navZ8/iLYgrl62F6+f2akW45bUnKu5CUx8X/yN3SutMclPR2++sq8vuOOC6fXmZq1Z9diTSl4ooer\nr4YqVWDz5rLTmbq0nyvF4Q8xkQKEEK7moT4QQZYg/tjzB5Y/3oaTjWjcxMZ3n9Z16zGFEEKYZkYn\nT0Lz5tC2bSE2sF8WHDXH+QkOhttuM68dBRQhfJH0gRDC1erUgf37Yd8+MzuQG034YTVjbmiHxaL4\n+29o1cqthxOlmPSBEKLwhgyB776D11+Hxx+/cHpt1dm1D4GRBQ9msX69GdWpShU4dAiCglyRYyGK\nR/pACOEpbpqJOm1/Gtp67gdTVhZMeqk9WiseeUQKD0II4QlnzpybMfqmmwq3jbLY54G4QOEBoGVL\naNQITpyAP/4oQUaFcCMpQBSBP7RJ8zSJiRM2m0v7QPz4748M+mYQG67cwIqLVpD8bzJgZi7dsMFU\nePzvfy45lFvJuZKXxMT/yd/QudIclx9/hNRU6NwZatcu/HaFjYlScPPN5vXXXxc9f/6mNJ8rxeUP\nMZEChBCu5qJRmA6fOcz1313Ptd9ey5bFW0j9NxWdpSl3STn+++9coeHDDyEsrIR5FkIIUSiOH/WO\nH/nu4Nj37NmmsCKEr5E+EEK4WrVqZmrSY8egatUib661ZsamGYz+eTSn008THhTOB4Ef0OCtBlS9\nsSoN3m/Atdeau2DXXw8zZ7rhM4gyR/pACHFhp06Zvgk2m5n7oRhf8YV22WWwbh3MmQMDB7rvOEIU\nJL9rg0xLK4SrlbAPxLwd8xg2exgA/Rr0Y2L/idQuXxvbWBu2ZBu//WYKD5GRphmTEEIIz5g3z/Q/\n6969aIX90X8vAAAgAElEQVSH0ytPs7H3RqI6RtFyQctCbTNokClA/PijFCCE75EmTEXgD23SPE1i\n4kQJ+0D0b9ifgZcO5ItrvmDezfOoXd40sg0IDEBFBGaP+PHccxAT45Ice4ScK3lJTPyf/A2dK61x\nmT3bPA8aVLTtotpFkfFNBk1/aFroba691jzPm3fuvlRpVFrPlZLwh5hIDYQQrlbCPhABKoDZQ2aj\nVN7WJFOnwsaNpuPeAw+UJJNCCCGKIiXFzP8A537cF5ayKCzhlkKNwuTQtCnUqwd79sCKFdClS9GO\nKYQ7SR8IIVwtMhLOnoWkJPO6AKfSTlEhtEKhdpuaCg0amHHBp06FYcNckVkhDOkDIUTBZs+GwYOh\nfXtYtcozxxw7Ft55Bx59FMaP98wxhchJ5oEQwlMKMRN1elY6Tyx8gvoT6nMo6VC+6RIWJLDvpX2k\n7kvlvfdM4aFVKxg61NWZFkIIUZDiNl8qCUffhx9/BCkvC18iBYgi8Ic2aZ4mMXHC0QcinyZMG45u\noN1n7Xhj+RskpiWyaO+ifHcVUiuEzOOZHJx7ildfNevGj3f5HHUeIedKXhIT/yd/Q+dKW1ysVvj5\nZ/O6uB2aixOTzp0hOhp27oR//y3ecX1daTtXXMEfYuKHP0OE8HH5jMJktVl5bdlrtPusHZuOb6J+\ndH2W3b6MW1vemu+uwpuE0+D9Bny0twZJSdC3L/Tq5c7MC+FaSqmKSqnZSqmzSqm9Simno+crpYKV\nUh8rpY4qpeKVUj8qpWp4Or9COPP333DyJNSta2aJLqqTv55k41Ub2XLDliJtFxgIV19tXs+ZU/Tj\nCuEu0gdCCFcLDDSFiMxM89pu07FNtP6kNVZt5d629zK+93jCg8MvuLujR81FKy0N/vnHNGESwtXc\n1QdCKeWYS3ck0AaYD3TUWm/Lle5x4GagN5AEfA6Eaa2vz2e/cm0QHvPcc/Dyy3D//fD++0XfPn5u\nPJsHbqbS1ZVo/lPzIm07axZcdx106AArVxb92EKUhEf6QBThTtM4pVSGUipJKXXG/lzHlXkRwmvy\nGYWpebXmvN33bX695Vc+6v9RoQoPAG+9ZQoPAwdK4UH4F6VUGDAYeFZrnaq1/guYCzirdqsD/Ka1\njtdaZwDfAIUf81IIN5o/3zz371+87bXNXtgtxuB8ffpASIjpuH3kSPGOL4SruboJ00dAGlAFGAZM\nVEo1ziftN1rrKK11pP15n4vz4nL+0CbN0yQmuWgNWps+EE6GYX2ww4P0rd/3gruxZdnQWnPiBHz0\nkVn33HMuzanHybmSVxmISUMgS2u9O8e6DTgvGHwBdFFK1bAXPG4BfvZAHkukDPwNi6U0xeXwYVP7\nGxYGsbHF20elqythnW+lyfQmRd42IuJc09Wffire8X1ZaTpXXMUfYuKyAkQR7zQJUTrZax+2Vy7Z\nbg68cYC/m//NtPviSUmBfv3gsstckD8hPCsCOJ1r3WnA2fjGO4D/gEPAKaAR8JJbcydEITg6T/fs\nCaGhxdtHQGAAljALlvDizQ/k6Lg9d27xji+Eq7lyIrn87jR1zSf9AKVUPHAE+FBr/bEL8+IWscW9\n9VCKSUzOl5h8kvsHw4wWcNHOn+nXoF+R96G15ti0Y6RsS2HublOL4e+1DyDnijNlICZngahc66KA\nM07SfgyEABWBFOAJ4Ffg8vx2PmLECOrUqQNAhQoVaNWqVXZMHXfwZNk7y451vpKfkiyb5ktx1K8P\nULL9ORR1+4oVzfLixbGkp8OKFZ77/O5ejo2N9an8+MKyY503jh8XF8eUKVMAsr9fnXFZJ2qlVBfg\nO611TI51dwJDtdY9cqVthLnDdAxzcfgBeFhr/a2T/erhw4fLRUKWfX55we4F3PLWLcSnxBNWEz4Z\nMpVaCbWKvL+MYxmEjgwlDQtXpaTT+rIA1qzx/ueT5dK1HJfrIvHCCy+4vBO1vWY6AWjquLmklPoS\nOKS1fjpX2k3A01rrn+zL5YFEoLLWOsHJvqUTtXC79HSoVAmSk+G//+Cii7yXl5YtYeNG+P13Uxsi\nhCfkO8CG1tolD6AVcDbXurHAj4XY9glgZj7vaV+xePFib2fB50hMtE7OSNaj54/WPI/meXTHO9DT\nooNKtM+EQ5n6svDTGrRessRFGfUyOVfy8qWY2L9rXXZN0Oe+w2cA04EwoDOmUNDYSbpJwExMDUUQ\n8DRwoID9uj8oheBLf0NfUlrismCB6djWokXJ91XSmDz+uMnLo4+WPC++pLScK67kSzHJ79oQ4MJC\nyg4gUCl1SY51LYHCDHqsAZcPHyiEJ2RaM5m7fS5BAUG80mUcSyZDzZTitXN1mPJtIGuTo+jaFa64\nwkUZFcI7RmMKD8cxBYlRWuttSqkuSqmkHOkeBdKBnZja6SsBD875K0Rejv4PxR19yeHo1KNsvGoj\nO+7bUex9XHmlef7115LlRQhXcOk8EEqpGZjCwF1Aa2Ae0EnnHe/7GmCJ1vqUUqo9MAt4Ums9zck+\ntSvzKIQ7LD+wnLCgMFqF1YPy5SEqCk7n7jtaOFlZUL8+7N9vOswNGODizArhhLvmgXAXuTYIT2jY\n0MwCvWyZmRW6uGyZNmypNlSgwhJWvBtMGRlmVurkZDhwAGrVKn5+hCgsj8wDQeHvNN0E7LKvmwK8\n6qzwIIS/6HRRJ1pVb3VuDoiA4v9rzZ5tCg8NGpT8rpcQQoji2bXLFB6io+HyfLvyF05AUACBUYHF\nLjwABAef6/vw228ly48QJeXSAoTWOlFrPUhrHaG1rqPtnaK11su01lE50g3VWlfWZv6HJlrrD12Z\nD3dxdEAU55SlmFhtVj5b+xnpWekFJLICEGd/LtL+U60c/uQwH7+eCcCYMSUqh/icwp4rderUQSkl\nDzc8ChpRQxRPWfoOLIrSEJcFC8xz79555gUtFlfEpDQ2YyoN54qr+UNMXDmMqxCl1q6EXYyYM4K/\nDvzF7sTdvNbrNecJHTUQTiaRu5CsU1nsmplI/7XH+adiK4YPL0GG/dj+/fuRpinuoYpxXgpRVjkK\nEH36eDcfOTkKEAsXmuaugfIrTniJS/tAuIO0cxXeZNM2Jv49kcd/f5yUzBRqRNTg82s+z39+h2PH\noHp1qFYNjh4t8vFuvBG+n6l58inFK6+UMPN+yt7e0tvZKJUKiq30gRDinMxMM3zrmTPeH741t0aN\nYPv2kvfLEKIwPNUHQohSIzkjmT5T+3D/L/eTkpnC0OZD2Xzf5oInh3M0XSpG26N9++CHHyAwSHH/\n/cXLsxBCiJJbtcoUHho3dk3h4eAHB1kauZQ9T+0p8b5KYzMm4X+kAFEE/tAmzdNKc0zCgsKIDImk\nclhlvr/he6YPnk50ueiCN7I3YYrLzCzy8d5/32w+ZAjExFw4vb8pzeeKKLvkvHbO3+Pi6uZLtjQb\na8+uxZZhK/G+SlsBwt/PFXfwh5hI6zkh8qGU4tOrP8WmbVSLqFa4jYo5ClNqKkyaZF4/9FCRNhVC\nCOFiLu//4OgeZyl5K8Fu3SA0FNasgePHoWrVEu9SiCKTPhBCuNLevVCvHtSpY14XQtr+NOL6buPt\n7TU41a46q1e7N4u+TvpAuI/0gRDiwhISoEoVM/JSYiKEh5d8n7YMG7Y0GypIYSlX8iGd+vY1hZzp\n02Ho0JLnT4j8SB8IIfKxN3EvA74ewPb47SXfWTFqII7NOEbo9tN0IIF77y15FoT3xMfHc99991G3\nbl1CQ0OpXr06vXr14o8//gCgbt26vP32217OpRCiIIsWma/yLl1cU3gACAi2zwPhgsIDmAIEnKsp\nEcLTpABRBP7QJs3T/DkmVpuVd1a8Q7OJzZi3Yx5P/P5EyXfq6AORXsBcEbnsn3wcgJXhVRkypORZ\n8FX+fK4U1uDBg1mzZg2TJ09m586dzJ8/n379+nHy5ElvZ024SVk4r4vDn+PiruFbXRkTR94WLAB/\nr4jz53PFXfwhJlKAEGXSpmOb6DSpE2MXjCUlM4Wbm93MpwM+LfmOi1EDMbNdCyZQn+YjowkLK3kW\nhHecPn2aZcuW8dprrxEbG8tFF13EZZddxtixY7nxxhvp3r07+/fv57HHHiMgIABLjpmpli9fTmxs\nLOHh4dSqVYv77ruPM2fOZL/fvXt37r33XsaMGUN0dDTR0dE8/vjj3viYQpRqWvvm/A+5NW0KNWrA\nkSOwebO3cyPKIilAFEFsbKy3s+Bz/DEmZ9LPcMXkK1h9aDU1I2vy080/MeO6GVQNd0FPNPswrrER\nEYVKfvo0TJoTwmxqcc/o0v3v6I/nSlFEREQQERHB3LlzSXdSAzVr1ixq1arFuHHjOHr0KEeOHAFg\n06ZN9O3bl2uvvZZNmzYxe/ZsNmzYwMiRI8/bfsaMGWitWblyJZ9++imffvop7777rkc+m8hfaT+v\ni8tf47JzJ+zfD5UrQ6tWrt23K2Oi1Pm1EP7MX88Vd/KHmJTuXyxCOBEZEsm4buO4t+29bB29lasb\nXu26nRexBuKrryAlBXr2hEsvdV02Sj2l3PMoAYvFwpdffsm0adOoUKECnTp14rHHHmO1vVd8xYoV\nsVgsREREULVqVarah0558803uemmmxgzZgz16tWjXbt2fPjhh/zwww/Ex8dn779GjRq89957NGzY\nkOuvv57HHntM+lMI4WKOH+O9exdrOp987X1+L0sjl3LgrQMu22dpKUAI/yQFiCLwhzZpnuavMXm4\n48N81P8jokKiXLtjRx+I1NQLJtUaJk40r8tC52l/PVeKYtCgQRw+fJh58+bRr18/VqxYweWXX85r\nr72W7zZr165l2rRpREZGZj+6dOmCUordu3dnp7v88svP265jx44cOnSIs2fPuu3ziAsrC+d1cfhr\nXNzVfOnipy4m45sMYka7bpKfXr3M85IlZihwf+Wv54o7+UNMpAAhSq20rDS+XP+lZ4cEdcxEXYi7\n2csmnWHXNis1asA117g5X6WN1u55uEBwcDA9e/bk2WefZdmyZdxxxx08//zzZOYzuaDNZuPOO+9k\n48aNbNiwgQ0bNrBx40Z27txJK1e3oRBC5CsjAxYvNq9793btvgNCArCEW7CEumYUJjDzP7RuDWlp\nsGyZy3YrRKHIRHJF4A9t0jzNV2Py665feeCXB9iVsIuQwBBuanaTZw5sr4GILV/+gkn3vLif7znF\nuhvaExQU7O6ceZ2vnivu1rhxY7KyskhPTyc4OBiro5Bp16ZNG7Zs2ULdunUL3M+qVavOW16xYgUx\nMTFEFLK/jXCPsnpeX4g/xmXlSjh7Fpo0gZo1Xb9/d8Skb1/45x9Tc+LqQo+n+OO54m7+EBOpgRCl\nypbjW7hq+lVcNf0qdiXsonHlxtSMdMOVID+OPhCWgu8ynTkD98U3YwTtGfZA6S88lAUJCQn07NmT\n6dOns2nTJvbt28fMmTMZP348vXr1IiIigjp16rB06VIOHz6cPbTrE088werVq7n33ntZv349u3fv\nZt68eYwaNeq8/R8+fJiHH36YHTt28P333/Pmm28yduxYb3xUIUolfxh9KTfpByG8RQoQReAPbdI8\nzZdiErcvjhYft+DXXb8SFRLFG73eYP2o9Vxx8RWey4SjD8QF2qV/+63pPN28azD163siY97nS+eK\nO0RERNCxY0cmTJhAbGwszZo149lnn2XYsGF88803ALz44oscOHCASy65JLsTdfPmzVmyZAn79+8n\nNjaWVq1a8cwzz1C9evXz9n/LLbdgtVrp0KED99xzD3fddRdjxozx+OcU5yvt53Vx+WNc3F2AcEdM\nOnWCsDDYuNEM6eqP/PFccTd/iIk0YRKlRueLOtO4cmO6XdyN52Ofp0p4Fc9nopB9IL74wjzfcYeb\n8yM8Jjg4mJdffpmXX3453zQdOnTgn3/+ybO+TZs2/PzzzwXuPzAwkAkTJjBhwoQS51UIcb6TJ2HN\nGggOhq5dXb//XQ/vYuPHG2n4fkNi7nRdR+qQEIiNhZ9/hoUL4bbbXLZrIQokNRBF4A9t0jzNl2IS\nZAli7d1r+bD/h94pPMC5PhAVK+abZOtW09Y2MhKuu85TGfM+XzpXhHAVOa+d87e4/PGHGUehSxcI\nD3f9/q0pVlqktUBnuX5QD39vxuRv54on+ENMpAZC+BWrzcqMTTMItgQzpNmQPO+HBIZ4IVc5FKIP\nxLI79tKZCJrfWInwcCnDiwtTJZyjQghRsN9+M89u6/9gvzQoi+v/lx15XrjQXIJcOX+FEPmR06wI\n/KFNmqd5KiZWm5VvN39Ly49bctuc2xi7YCypmT448LW9CVPc6dNO3z6zK42GK/fzHNsYPtTmyZx5\nnfz/FN+iRYuk6ZKPkvPaOX+Ki9bnChB9+7rnGPXfr491npXqt1W/cOIiatQIatWC48dNXwh/40/n\niqf4Q0ykACF8mk3b+HL9lzT5qAk3/XATW05s4eLyF/NKj1cItvjg6EUXmIl66QvHAdgcVYkO3aUC\nUAghvG3bNjh0yMyr0KKFe45hCbVgCbcQEOL6n11KnSv4+GszJuF/lEcn2SoGpZT29TwK99Fac/kX\nl7P60GrqVqjLk12eZHjL4d5vqpSfhQtNfXLv3k6/yYf3TCFw0VE6jKrI3RPz7ydRlimlPDv5XxlS\nUGzt7/lNWym5NghXeecdGDsWhg2DqVO9nZvi+e47GDIEevaE33/3dm5EaZLftUFugQqfppTitZ6v\ncSDpADc3u5kgS5C3s1QwxyhMTmog4uNhxpIwtKUerzzv2WwJ4U1KqYrAJKA3cAJ4Wmv9dT5p2wDv\nAG2As8ArWuv3PZVXUfb44/wPufXsaWoili41Q4SHhXk7R6K0kyZMReAPbdI8zVUx2Xx8M99t+c7p\ne93rdue2lrf5fuEBzs0DkZiY563vvoOsLFM5Ua2apzPmffL/U6Z9BKQBVYBhwESlVOPciZRSlYBf\ngIlARaA+4NONMuS8ds5f4pKWBn/+aV67eyZnd8akUiVo2xYyMmDJErcdxi385VzxJH+IiRQghNdk\nWDOYuWUmvb7qRfOJzblz7p0kpSd5O1slU0AfCEfV+K23ejA/QniZUioMGAw8q7VO1Vr/BcwFnP0n\njAV+1Vp/o7XO0lona623ezK/omxZtgxSU6FlS6ju+v7N2bYO28rGqzZyYs4Jtx3DUYPi6BAuhDtJ\nHwjhFeMWj+PjtR9zPNl0Kg4LCuP2Vrczrts4783h4Apz58LAgXDNNfDjj9mrd/yTRaM2FsLCFceO\nuWec8dJC+kC4jzf6QCilWgF/aa3Dc6x7BOiqtR6YK+0fwCagHab2YSVwv9b6gJP9yrVBlNhjj8Gb\nb8Ljj8Prr7vvONY0KzpDExAaQECwe+7dLlkC3bpBkyawZYtbDiHKIOkDIXzK5hObOZ58nKZVmnLP\nZfdwa8tbqRBawdvZKrl8+kAsH3OArznK+ssvJTw82gsZE6XBCy+8wA8//MBG/xqrMQLIPa7xaSDS\nSdpaQGugF7AZGA98DXRxtuMRI0ZQp04dACpUqECrVq2yJ2ByNAGQZVkuaPm338xytWpxxMW573hL\nVy51++fJzISIiFi2boWZM+OoUsX78ZVl/1uOi4tjypQpANnfr05prX36YbLoGxYvXuztLPicgmKS\nZc3S8cnxTt9bf2S9XrZ/mbbZbG7KmZd8/73WoBdfcUX2KptN60su0bouZ/SCGWlezJx3Ffb/x5f+\n54tqxIgResCAAW7bf3Jysk5ISCj28QqKrf09d3yHtwLO5lo3FvjRSdr1wBc5lqMxU3BFOklb6M/t\nTnJdcM4f4nL4sNagdViY1mke+Gr2REwGDDCfadIktx/KZfzhXPE0X4pJftcG6QMhXMpqs7J0/1Ie\n+uUhLn73Yu766S6n6VpWb0nn2p1L3wy7TvpArFwJu3dDWo0Ietzoo8PPCr8QFhZGxYp+N/zvDiBQ\nKXVJjnUtAWeNLDYCudslaaCUfVEIX+AYfSk2FkJKyVezzAchPEUKEEXgqOoR5zhiEp8Sz+j5o6n1\nTi26TunKhNUTOHTmENvit5Fly/JuJj3J3oQpNkdvvGnTzPPQoWCxeCNTvqGs//8cOHCAQYMGERUV\nRVRUFNdddx2HDh06L82rr75K9erViYqKYsSIEbz44ovUrVs3+/0XXniB5s2bZ7/+8ssvmT9/PgEB\nAVgsFpb44PArWusUYBbwolIqTCnVGbgGcDbi/mRgkFKqhVIqCHgOWKa19tnRFcr6eZ0ff4iLp4dv\n9URMHJ9l4cJz97N8nT+cK57mDzGRAoRwiYjgCL7a+BVHzx6lboW6PNbpMVbesZKt920lMKAMdbXJ\nVQORkQHffGNWyehLZdvAgQM5ceIEcXFxxMXFcfjwYQYNGpT9/jfffMOLL77Iq6++yrp162jUqBFv\nv/12nlo6x/Kjjz7KjTfeSK9evTh27BhHjhyhU6dOHv1MRTAaCAOOA9OBUVrrbUqpLkqp7MKB1nox\n8DTwM3AUqAcM9UJ+RSlns50rQDju2rvTxqs3sjRyKYmL8w7x7Ur160OdOnDyJPzzj1sPJcq4MvTL\nruTi4uL8olToDmlZaaw4sILF+xbzQPsHskdKcsQkNDCUzwd8TsNKDWlVvVXpa5pUWI55IE6cIBZY\n+M5pYhIgpmkULVqU0ZjYufr/R73gPJ56XD6jDBUxvSstXLiQTZs2sWfPHi666CIAZsyYQf369Vm0\naBE9evRgwoQJjBw5kttvvx2AJ598ksWLF7Nz506n+wwPD6dcuXKkpKRQpYpvj1ymtU4EBjlZvwyI\nyrXuE+ATD2WtxMrydaEgvh6X9evN5J4XXQSXXur+49mSbaw9u5Zmuplbj6OUqYX49FMznOtll7n1\ncC7h6+eKN/hDTKQGQuRr3ZF1vPTnS3T/sjsVXqtAj6968NKSl/hl1y9O0w9pNoTWNVqX3cID5KmB\nSHxrL+/zD2OaH6csh6Ws+/fff4mJickuPADUrVuXmJgYtm7dmp2mXbt2523XoUMHj+ZTiLLCMVdC\n37545LtZ28yNCmVx/8EczZikH4RwJ6mBKAJfLw262hfrvuCjNR9lL7es1pLudbrTpEqT7HVlLSYX\n5OgDERPD8S1pxJw4RQaKXs9V8nLGvM/V50pRaw48UdOQ77G1zrdgnXN9mS58+yn5DnTO1+OSswDh\nCS1+bUHzzOYEhLn/vm2PHuYe1vLlcOYMRDobMNmH+Pq54g3+EBMpQJRBiamJbDy2kbVH1rLq0Co6\n1erEQ5c/lCfdoMaDUErRo24Pul3cjUph8iP4gnLUQPwyHzZSg5ox0KeJ/KuVZU2aNOHQoUP8999/\n1K5dG4A9e/Zw+PBhmjZtCkCjRo1YvXo1w4cPz95u1apVBe43ODgYq2PuESFEoZw5Y35cBwRAz56e\nOaalnAXKeeZYFStC+/ZmBMA//4Srr/bMcUXZIk2YisAx0Ya/WrB7AbXfqU30G9HEfhnLIwse4bst\n3zF/53yn6XvV68UH/T5gcOPB+RYe/D0mLufoA3HsGJN/DuVtLqXiyx5oYOsHysq5kpSUxIYNG857\n1K9fn5YtW3LLLbewbt061qxZw7Bhw2jbtm32naaHHnqIKVOmMHnyZHbt2sUbb7zB6tWrC6yVqFOn\nDps3b2bHjh2cPHmSrKwyNOKZjygr53VR+XJc4uIgM9P8yPbkqMiejIk/Defqy+eKt/hDTOS2aCmQ\nkJrA9vjtbD+5ne3x2ykXVI7/dftfnnTR5aI5kHSAcoHlaFa1Ga2qt6JDzQ50ushnR27xP/a7wcfS\novjzTwgNheuu83KehEctXbqUNm3anLfuuuuuY86cOTzwwAN0794dgN69ezNhwoTsNEOGDGHv3r08\n9dRTpKSkMHjwYEaNGsXcuXPzPdZdd93Fn3/+Sdu2bUlOTmbx4sV07drVPR9MiFLC0XzJU8O3ekOf\nPvDCC+c+qxCupswkc75LKaV9PY/ulpKZQlhQWJ71O0/upMPnHUhMO39YuIvLX8y+MfvypM+wZrAn\ncQ8NohtgCSjDExK400cfwejRvNphDk+vGsiQIeeGcRWFo5SirP/POwwePBir1cqPP/7okv0VFFv7\ne37TCUOuDaI4tIZLLoG9e00zpo4dvZ0j98jKgipV4NQp2LnTDO8qRHHkd21waQ2EUqoiMAnoDZwA\nntZaf51P2teBOzCzjE7SWj/hyrz4o+SMZF5a8hKHzxzmyNkjHD5zmENJhwgNDOXoo0fzpK8WUY3E\ntETCg8JpWKkhl1a+lEsrXUrjyo2d7j/YEkyjyo3c/THKNpsNGzB1R3tA5n4QhZeamsrEiRO58sor\nsVgs/PDDD8ydO5dZs2Z5O2tClBrbtpnCQ+XKpgmTp6zruI7kLcm0/qs1Ec0j3H68wEC48kpzA2v+\nfHgobzdHIUrE1U2YPgLSgCpAG2C+Umq91npbzkRKqXswM5E2t6/6XSm1W2v9qYvz41LOxuXVWpOS\nmUJ4cHie9Fm2LF768yXiU+KJT40nPiWekyknSUpPYveDu/O0bQ6yBPH6X6873U96VjohgSHnrY8K\nieL4o8epHFbZa6O3+MNYxR5ltbKGwfRLnEuzyGvo06eGt3PkM+RcKZhSil9++YVXX32V1NRUGjRo\nwLRp07jmmmu8nTVRADmvnfPVuMy3d/m76iqweLAivuUfLYlbHEd4k7y/Fdylf3//KED46rniTf4Q\nE5cVIJRSYcBgoInWOhX4Syk1F7gVM7NoTrcBb2mtj9i3fQu4E/B4AeKv//4iOTOZlMyU8x73tbsv\nzwzKWmu6Tu7KqbRTnE4/TVJ6EknpSdi0jfRn0wm2BJ+X3qIsvPbXa2RYM/IcNzkzmYjg8+9CBFuC\nebP3m0SXiyYmMoYakTWoEVGjwAKCY0I34SNsNr6hMz8Rwx1dAgkK8naGhL8IDQ1l4cKF3s6GEKWa\nowDh6ZGJLGEWLOEWj8wD4XDllWaOiz//9I/hXIV/cVkfCKVUK+AvrXV4jnWPAF211gNzpT0F9NZa\n/0wci8AAACAASURBVG1fvgxYpLUu72S/etRPo8iwZpBuTSfDmkGGNYMZ181w2i+g5cctSUhNMOmz\nzqVPeCKBqJCoPOmjXo3iTMaZPOtPPXGK8qF5skOF1ypwOv30eevCgsI48PABostF50k//q/xhAaG\nUjmsMpXCKlE5rDKVwypTK6oWAUoGwSptst54m1pPDOUY1Vm1yrNV5KWF9IFwH+kDIcqyxETTLwDM\nLNQVKng3P57QubPp6zFrFgzKMxe8EBfmiT4QEcDpXOtOA87KvLnTnravc6rnzWag5pH3jeRMmPmx\n7+hYvLrpalovbU1QtLnVeyjpECdTTzL5w8lEpEZkb5e7FsCxXbc63UjLSiMsKIywoDCGPDyEkOQQ\nNn+ymfab22fv1+HHyT8SOSeSitUrUj60PJHBkQRZgljddDVZiWYIxXab22Vv91jnx847Xu795bdd\n7nzKdv6x3aIdF3GM6jSseIJ27aR2SAghfMWCBWagvG7dykbhAUwzpuXLTc2LFCCEK7myAHEWyH2L\nPwrIe3s/b9oo+zqnPj/7OdWpTq9jvYioFkHDpg2zm/+sOrCKpKVJ9BrYC4D3G79PgAogJisG61kr\n61nPzI4zqVTOzGPgGFs38GggaHikxiPAuVn/3r/9faxnrLSiFehz6WNjY4mLi2PTkU003tGYNo3b\n5NlfVkIW61lPytKU7PzkPl7O/Tnyn9/xnH0+Xzveu+++S6tWrUrt5yvq8d6MOwvE0bnm3yj1WJ70\nZXk557jWF0ov3MfRtjYuLo4pU6YAZj4JUTz+0FbZG3wxLt5qvuTgjZj07w/PPAM//2ymKQrwwYYP\nvniueJs/xMSVTZjCgASgqdZ6t33dl8AhrfXTudL+hRl56Qv78kjgTq11ngkJlFI67VAaAMHVg1EB\n59eipB9JJ7ia8/XYP5qrtouLi6PjpR09djx/2M5xkvt6Pj2xXUZEMNUrZ5KSGcz0AaMYOvdjxDmF\n/UKUJkzuI02YXM8fLvTe4GtxsVqhWjU4eRK2boXGzgcrdJvVTVazct9K7v73bkJrh3rsuFpD7dpw\n8CCsWQOXXeaxQxear50rvsCXYpLftcGl80AopWZgfk7dBbQG5gGd8hmF6UHMcK8AC4D3tNafOdmn\nT1wkhLiQqVPhttugM8tY9uxv8NJL3s6SX5IChPtIAUKUVStWQKdOUK8e7NplOhd70vJay8k4lMHl\n/11O6EWeK0AAjBoFn3xiJpb7X945ZoUoUH7XBldXZo0GwoDjwHRglNZ6m1Kqi1IqyZFIa/0J8BOw\nCdgI/OSs8CCEP5k2zTzfylTfrCcWQogyytF8qX9/zxceALCZJ0+OwuTQv795dsRACFdw6a8crXWi\n1nqQ1jpCa11Ha/2tff0yrXVUrrRPaq0raa0ra62fcmU+3EXaZ+clMTGOHIHff4dgSxY3MJO4/fu9\nnSWfI+eKa9x8883ceOON3s6GsJPz2jlfi0vOAoQ3dNjZAes8K8HVgy+c2MV69ICQEPj7bzh2zOOH\nvyBfO1d8gT/ERG6TCuECX39tOqj1r7+DaBI9O0OR8LqAgAAsFgsBAQF5HhaLhZEjR3o7i0KUWQcO\nwPr1EBZmRmDyBku4fR6IAM/XQISHm0KE1qYztRCuIAWIIvCVDi2+RGJiOJovDWv6DwCxl1zixdz4\nptJ8rhw9epQjR45w9OhRPvvsM5RSHDt2LHv9e++95+0sCjcpzed1SfhSXObMMc9XXgmhnu1+cB5v\nxmSgfTau2bO9loV8+dK54iv8ISZSgBCihLZsgX/+MeOK96+71ayUPhBlStWqVbMfFewDzFepUiV7\nXaR9CthHHnmEhg0bEhYWRr169Xj22WfJysrK3s9TTz1Fu3btmDp1KvXq1aN8+fLccMMNnD6de4od\nePPNN4mJiaFy5crcfffdZGZmeubDCuFnZs0yz4MHezcf3jRwoOn7sWCBmZVaiJKSXzlF4A9t0jxN\nYnKu9uHGGyEkwPyIi9u714s58k1yrkCFChWYNm0a//77L++//z5Tpkxh/Pjx56XZsWMH/9/efYdH\nVaUPHP+eVEiBUEITQ6gioBQLCCiIoIIuxdXFgtgQCxbsKLsq1sXVRXFVLFhWLCD+VEBhxTIgIIhg\nQAFBejEJNQlJSAKZ8/vjzJAyk5BJZubOzbyf57lP5t45c+fknTv3zrmnzZs3j3nz5jF//nx+/PFH\nHn/88TJpvv76a3bs2IHD4WDGjBl8/PHHvPrqq0H8T4SbHNfehUpc9u2DxYshOtq6/g9uVsakWTMz\nClVhIcyfb1k2vAqVYyWU2CEmUoAQogacTvjgA/P42mtdG0BqIAJMqcAsgfaPf/yDs88+m5SUFC65\n5BIeeOABPvroI4907777Lp06daJ3797ceOONfPvtt2WeT05O5uWXX6ZDhw5cfPHFDB8+3CONEALm\nzjWn5QEDrJ19emnyUtYOWUtxfrFleXDXwIRiMyZhP/Irxwd2aJMWbOEek8WLTQe91FRzd4dic3Ho\n36GDpfkKReF+rAB89NFH9OnTh+bNm5OYmMiECRPYuXNnmTRt2rShbt26x9dbtGjB3r17y6Tp0qVL\nmXVvaURwyHHtXajEJVSaL/Xc1pPbM24noq51P7tGjDB/582DggLLsuEhVI6VUGKHmEgBQogaeP99\n83fUKFelg9RABIXWgVkCadGiRYwePZrhw4fz5ZdfkpaWxqOPPkpRUVGZdNHR0WXWlVI43ceVD2mE\nCHc5ObBwoalddHcitkpUQhRR9aJQlkxCYbRuDd26QW4uSIWlqCn5leMDO7RJC7Zwjkl+PnzyiXk8\napRro+tHnGPrVmsyFcLC+VgBWLp0Ke3ateOBBx6gR48etG3blm1h0ldGKdVAKfWZUipXKbVNKXXV\nCdJHK6V+V0rtrCxdKAj347oioRCXr76CoiLo2xeaNrU6N6ERE3dNjLtmJhSEQlxCjR1iIgUIIarp\n88/NaBY9e8Ipp7g2upowWTPVqQhlHTp0YNu2bXzyySds3bqVqVOn8n+hdBUPrFeBAiAZGAW8ppQ6\ntZL0DwIZwciYqL3cXy930x1RUoD44gsoNQCcED6TAoQP7NAmLdjCOSbvvmv+XnddqY2uGoj+HTsG\nPT+hLpyPFYDLL7+cO++8k3HjxtG9e3eWLVvmMbpSbaSUigMuA/6utT6itV4KzAGurSB9a+Bq4Nng\n5bL6wv24rojVcSkoKJk0LVQKEFbHBKBTJ+jQAQ4cgCVLrM6NEQpxCTV2iInSgW74W0NKKR3qeRTh\nZ/duSEkxQwOmp0PDhq4nxo6FN9+EN96Am2+2NI92pZRCvvOBUVlsXc/5vepMKdUNWKq1ji+17T7g\nPK21R8t0pdRc4E0gG3hfa51SwX7l2iAqNHcuDB0KPXrAqlXW5sVZ6GRJoyVE1Img7/6+1mYGmDAB\nJk+GO++EqVOtzo0IdRVdG6QGwgd2aJMWbOEakxkzTKfbYcNKFR6gpA/Epk3WZCyEheuxIkjAFAZK\nywYSyydUSo0AIrXWc4KRMX+Q49o7q+Mye7b5Gwq1D7pY48xzsirH4pKMi7sZ06eflrS6tZLVx0oo\nskNMoqzOgBB2ozW89555XKb5EpScjWUUJiHccoF65bbVA8rMh+tq6jQZGOzedKIdX3/99aSmpgJm\nkr5u3bodr/p3X4ADve4WrPezy3paWppl73/kCMyaZdZHjrTm/y+9rp2aNNLYzGbcrMzPWWdB06YO\n/vwTfvihP/37W3+8yHrZdSu/Pw6Hg3ddbbTd51dvpAmTED5asQJ69TKjeuzeDVGli+HXXQf//a/p\nIOFRuhBVIU2YAseiJkxxwEGgs9Z6i2vbe8AerfUjpdJ1BX4CDmAKDzFAfWAv0EtrvbPcfuXaILya\nNQtGjoSzzoKffrI6N6C1pjivGJwQVS807ttOnAjPPANjxphWt0JURJowCeEn7tqHa64pV3iAknkg\nIiODmichQpXWOh/4P+AJpVScUqoPMBR4v1zSX4GTgW5AV2AMZiSmrsCu4OVY2N2HH5q/V19tbT7c\nlFLH54EIFddcY/7Ong2FhdbmRdiTFCB84K7iESXCLSaFhfDxx+ax1woGVxMmx++/By9TNhFux4oo\nYxwQh6lN+AC4VWu9QSnVVymVA6C1dmqt97oXTK2FU2u9L5SrGuS49s6quBw8aEZfioiAK6+0JAsV\nCqVjpVMnM6lcVhbMn29tXkIpLqHCDjGRAoQQPpg7Fw4dgu7d4fTTvSSQmaiF8KC1PqS1HqG1TtBa\np2qtZ7q2L9Fal+8f4X7NoopGYBKiIp9+CkePwgUXQLNmVucmtLlraNw1NkL4QvpACOGDSy+FL7+E\nF1+Eu+/2kuBvfzPTU8+aBVdcEfT81QbSByJwrOgDEShybRDe9O8PixbBO+/A9ddbnZvQ5h6OPDYW\nMjOhnteivAh30gdCiBrKzIQFC0y/hwrb1koNhBBCWGLXLlN4qFOnZKjSUFC4p5DF8YtZ0XGF1Vkp\no2VLOO88M+mee9ZuIapKfuX4wA5t0oItnGLywQemi8OQIZCcXEEidx+I9euDlzGbCKdjRYQPOa69\nsyIu7v5pf/lLaN1Nj2keQ+/M3uT+O9fqrHhwd6a2shmTfIc82SEmUoAQogpKz/1QabW4uwZC2aYl\niBBC1AoffGD+hsroS24qwozCFBkXeqPz/fWvEB0N334LGRlW50bYiRQgfOCecEOUCJeYrFoFa9dC\no0ZwySWVJHQVIPp36xacjNlIOBwrN9xwAxEREURGRhIdHU2rVq24/fbbycrKqvI+Fi1aREREBAcP\nHqzwPYYOHerz60RghMNxXR3BjsuaNWZJSoLBg0+c3gqheKw0bGhq1Z1OmDHDmjyEYlysZoeYSAFC\niCpwT7QzejTExFSSUGaiDnuDBg0iIyODHTt2MH36dObOncu4ceN82oeqZg1WdV8nhN298Yb5e+21\nplOwqLqbbjJ/33jD1LYLURXyK8cHdmiTFmzhEJPc3JL2oTfffILErhoIx2+/BTZTNhQOxwpAbGws\nycnJtGjRgoEDBzJy5Ei+/vrr48/n5OQwduxYmjZtSr169Tj//PNZtWqVhTkWNREux7WvghmXvLyS\nu+cnPEdbKFSPlcGD4aST4I8/TCf0YAvVuFjJDjGRAoQQJ/Dxx6YQ0bcvnHrqCRLLKEyilK1bt7Jg\nwQKio6OPbxsyZAgZGRl89dVXpKWlcd5553HBBReQmZlpYU6FsK9ZsyAnB845B047zerceMpbn8fi\n+MVsvG2j1VnxKiqqbC2EEFUROvOq24Ad2qQFWzjExH1CHTu2ColdTZj6d+8euAzZlL+PFYdylOxb\ne+7boRwVbq/sdTU1f/58EhMTKS4upqCgAKUUU6ZMAeC7775j7dq17Nu3j1hXO4tJkyYxZ84c3n//\nfe6//36/50cEVjicA6sjmHHx6RxtAX1U48x30iO6h9VZqdBNN8FTT5mJ+Pbvh8aNg/fe8h3yZIeY\nyG1SISqRlgYrV5qOeZdfXoUXSA1E2OvXrx9r165l5cqV3HXXXQwZMoQ777wTgNWrV5OXl0fjxo1J\nTEw8vqxbt44tW7ZYnHMh7GftWli+HOrXN/N4hiLtNB0LVGTo9lFKSTFNmYqKSkYcFKIy8ivHB3Zo\nkxZstT0m7s7T114LdetW4QXuPhBr1wYuUzbl72Olv+5/fKno+eq8rqbi4uJo3bo1nTt35sUXXyQv\nL48nnngCAKfTSbNmzVi7di1r1qw5vvz+++88+eSTVdp/vXr1yM7O9tielZVFREQEiYmJfv1/ROVq\n+zmwuoIVl9Kdp+PigvKWPkvomkDfw33JfsbzextK3DU4we5MLd8hT3aIiRQghKhAfn7JuOJV7pgn\nNRCinMcee4zJkyeTkZFBjx49yMzMRClFmzZtyiyNq9hm4JRTTmH9+vUUFhaW2b5q1SpatWpVpr+F\nELVZfj68/755HKrNl6DUPBB1Q28eiNKGDIEWLWDTJms6Uwt7kV85PrBDm7Rgq80xmTULsrOhVy8f\nOua5+0CccUbgMmZTtflYqUy/fv3o3LkzTz31FAMHDqR3794MGzaMBQsWsH37dn788Ucef/xxli5d\nevw1Wmt+/fXXMrUUa9asAWDUqFFERUUxevRoVq9ezZYtW3jnnXeYOnUqDz74oFX/ZtgK1+P6RIIR\nF3fnaZ/O0RYK9WOldGfq118P3vuGelysYIeYSAFCCC+0hldeMY99urMlNRDCi3vvvZfp06eza9cu\n5s+fz4ABAxg7diwdO3bkyiuvZNOmTbRo0eJ4eqUUAwYMoEePHmWW/Px86tWrxw8//EBxcTHDhg2j\ne/fuvPzyy0yZMoWxoXwbVgg/0hqmTjWP5bD3nzFjzOVr9mzYtcvq3IhQpnSIzxqilNKhkkeHw2GL\nUmEw1daYLF9uhgRs1MicRKvU/wHg7LNh5Uocr71G/1tvDWge7aaqx4pSilD5ztc2lcXW9Vzo9vIs\nJ1SuDbX1HFhTgY7LokXQvz8kJ8POnVCnTsDeym/scqxceSXMnAn33QfPPx/497NLXIIplGJS0bVB\nbpMK4cXLL5u/Y8b4UHiAkpmoZUZgIYQIGNfIyNx2W+gXHrIWZbE4fjFbJthjpLUHHjB/33jDNOMV\nwhupgRCinIwMM6RdcTFs3QqtWvnw4u7dzdivq1ebx8JnUgMROFIDIWqDzZuhQweIjja1D02bWp2j\nyuliTfGRYpRSRMaHdkdqtwED4PvvYfJkkK5V4U1qIISootdfh6NHYdgwHwsPUNIHItIeFwkhhLCb\n5583fSCuuSb0Cw9g5n+ISoiyTeEBwD2n5UsvmbkhhChPChA+sMO4vMFW22JSVATTppnHrrm/fOOe\nB2LVKv9lqpaobceKECDHdUUCFZf0dHjnHdNK1G53xt/9/F1Wp69m4/6N7MnZQ05hTsjWtg4eDJ07\nw59/wkcfBfa95DvkyQ4xibI6A0KEktmzTROmLl1MBz2fSR8IIYQImClTzI2eyy6Djh2tzcvBIwdJ\ny0jj18xf2XJoC1sObWHroa3MGDGDM1p4DuX9wo8v8Nua38psqxtVl7lXzeWCNhcEK9tVopSphbjh\nBlPjM3q0XNZEWdIHQohSzjnHjMA0bRrccks1dtCxI2zcCBs2WH91synpAxE40gdC2NmBA5CaCrm5\nsHIlnHmmtfnp83Yflu1a5rH9y6u/ZEj7IR7bb513Kyv2rCCvKI/colyyC7PJP5rPqrGr6NG8h0f6\nK2dfSXZhNj1P6kmfk/vQN6UvdaN9GdWjZoqKoHVrUwsxdy5cemnQ3lqEkIquDVIDIYTLsmWm8JCU\nBKNGVXMn0gdCCCEC4vnnTeHhoouCU3jIyM1gweYFnNniTLo06eLxfJ+T+1DsLKZbs250aNSBtg3a\n0qZBG9o3al8m3f4v9rP+6vXcNfwuOn3QqcxzhwsPey0UOLWTBZsXkF2YzYLNCwCIjYylb0pf3hv+\nHifVO8mP/6l3MTFmKNf77oPHH4dLLpFaCFHCb30glFINlFKfKaVylVLblFJXVZL2MaVUkVIqRyl1\n2PU31V95CRQ7tEkLttoUk8mTzd9x4yA+vpo7cTVhcvz0k38yVYtU9Vhp1aoVSilZArC08nlUAHEi\ntekc6E/+jsvevSXDaz/xhF93XcbunN38+8d/0+utXjR/oTk3fHEDM9bO8Jr2uUHPsXzMcqZdOo17\nz7mXYR2HcVrT06gTVXZcWWeRE2e+kxW7V3jsIzE2kagIz3u5CsXa29Yy6/JZ3NPrHro3605hcSHL\ndy8nOT7ZP/9sFdx6KzRrBqtWwZw5gXkP+Q55skNM/FkD8SpQACQDPYAvlVJpWusNFaT/WGs92o/v\nL0S1rVtnTo516sBdd9VgRzITdY1t377d6iwETShNFhRISqkGwNvAIGAf8IjW2qNrplLqfuA6oJUr\n3Wta6yBMZSVC3XPPQV6eaUZz9tmBeY/317zP6M9LfpbUjarL+a3P54zmnv0ZfOK6LPhyy1YpRUr9\nFFLqp3BF5ysA2J+/n/X71hMTGeORPqcwh38u+Seju46mY2P/NZ+Ni4OHH4a774ZHH4W//EUub8Lw\nSx8IpVQccAjopLXe4tr2X2C31voRL+kfA9pWpQAh7VxFMFx/Pbz3npmU6NVXa7CjVq3MwOQ7dpjJ\nJISwiUD2gVBKuQsLN+K6wQScU/4Gk6sA8Q2wFmgHfA08qLWe5WWfcm0IEzt2wCmnQGGhuRPew7O7\ngF9sz9pOp1c6MaT9EEZ2HsklHS4hLjquxvt1HnPiLHCiIhSRcYFp3jp99XTGzB0DQK+Wvbi+6/WM\n7DKSpDpJNd53QQG0bw+7d8OMGWb4XBE+Kro2+KsA0Q1YqrWOL7XtPuA8rfUwL+kfA8YDxUA68IrW\neloF+5aLhAioXbugTRtTefDHH+ZxtbVsCXv2mJ22bOm3PAoRaIEqQPh6g6nca18C0Frf7eU5uTaE\niWuvNT9cr7oKPvywZvvaemgrn67/lPt734/y0qA//2i+XwoNwZaWkcZ/fvoPs9bN4nDRYcD0mZg8\ncDJ39/L4+vjsnXfgxhvNfbGNG0N/9m/hP4GeSC4BKD/heTaQWEH6mcCpmOZOY4FHlVIj/ZSXgLFD\nm7Rgqw0xmTIFjh2DK66oYeEBSuaBWL685hmrZWrDseJvYRKTDsAxd+HBZQ3QuQqvPRdYF5Bc+UmY\nfIY+81dcVq82hYeYGHj66ertQ2vN4h2LGfbxMNpNbceD3zzIkp1LvKYNZOEhkMdKt2bdeGvoW6Tf\nl877I95nQOsBFBYX0rlJVb5mJzZ6NJx2mqlgd/dF8Rf5DnmyQ0yq1AdCKfU90A/wdrtnKXAXUL/c\n9nrAYW/701r/Xmr1R9ddpssxBQsP119/PampqQAkJSXRrVu34+2G3UGWdWvW09LSQio/vq7PmePg\ntdcA+vPQQ37Yf0EBwPFGolb/f7Ie2utWfn8cDgfvvvsuwPHza4D4eoMJAKXUJEAB7wQoXyLEaW3a\n3gPccYcZUtRX8/+Yz2OOx1j550rA3JW/ovMVNKzb0I85DR3xMfGMOn0Uo04fxY6sHaTU996U9tCR\nQzSo26DK+42MNP1QBg+Gp54ytULNmvkr18KO/NkH4iDQuVQV9XvAnhNVUbvSPgicrbW+3MtzUk0t\nAubxx2HSJLjwQvjf//ywwyZNYN8+M2RIcvBGyhCipgLYhKkbsERrnVBq271AP29NXF3P3wHcA/TV\nWqdXkEZfd911cnOpFq8vXAjPPNOf5GR4+20HCQm+729t3bXcveBu6v1Zj+Edh/Pczc/RNKFpSPx/\nVq1n5GaQcncKvVr2YvKYyZxz8jlVer3W8Pzz/fnqK7joIgcTJoTG/yPrgb25NGnSpMD1gQBQSn2I\nqaG4GegOzAN6exuFSSk1FFistc5SSp0N/B8wQWvtMVaaFCBEoBw4YJos5eTAokVw3nl+2GmjRnDw\nIOzfbx4LYRMB7gNR5RtMSqkbgceBc7XWOyrZr1wbarHDh03H6fR0ePttMyNydeQW5fLB2g+4tuu1\nlvVtSJ+ezh93/UHzm5rTfmr7E78gwD7b8BkjZ4/kqPMoAL1P7s0DvR9g6ClDiVARlb5282bo3NlM\nMrd0KfTuHYwcCysFug8EwDggDtgLfADc6i48KKX6KqVySqW9Etjs2vYu8Ky3wkOocZfQRAk7x+Rf\n/zKFh0GD/FR4gJI+EMuW+WmHtYedj5VACYeYaK3zMTeJnlBKxSml+gBDgffLp1VKXQM8DQyqrPAQ\nSsLhM6yOmsblkUdM4aFnT7juusrTpmWkcdf8uzjmPObxXEJMAreceYulHaObjm5K78ze7Lxkp2V5\nKG3EqSPYMX4HD/d9mKQ6SSzbtYwRM0fw0MKHTvjadu3ggQfM41tuMQWJmpLvkCc7xMRvBQit9SGt\n9QitdYLWOlVrPbPUc0u01vVKrV+ttW6sta6nte6ktX7FX/kQoioyMmDqVPP4qaf8uGOZB0IIb7ze\nYPJyc+lJoCGwUpVMMlqTgZWFDS1bBq+8AlFR8Prr3k+nTu3kqz++4oL/XkD317vz8k8vM3v97OBn\ntgoioiOISogiMjYwQ7hWR/PE5jxzwTPsumcXL170IqlJqVzX7QQlNZeJE01B4rff4J//DHBGRcjy\nWxOmQJFqahEI48fDSy/B0KHwxRd+3HFCgpnt6PBh81gImwjkPBCBINeG2unIETjjDNiwwdRCeBt5\nad6meTy48EE27DctpBNiEhjTfQzje42nVZLMtl4dTu2ssPlSZm4mTROaltn2/fcwYABER5uRsrp0\nCUYuhRWC0YRJCFvYtQvXyEvw5JN+3rnUQAghRLU98ogpPJxyCvzjH97T5B/NZ8P+DZyUeBLPDXyO\nXffsYsrFU6TwUAMVFR427NtAyykt+dsnf2PF7hXHt59/PowdC0ePmhGZ/NGUSdiL/MrxgR3apAWb\nHWPy1FPmZDdyJJx+up937u4DscT7GOPhzI7HSqBJTOxPPkPvqhOXb76BF180TZdmzKh4srLLTr2M\nj//6MVvv3soDfR7wy2zLwWDHY2X57uUoFJ+s/4Re03tx7jvn8sXvX+DUTp5/3gxEkpYGjz5a/few\nY1wCzQ4xkQKECCvr18P06aaCYNKkALxBcbH5KzUQQghRZZmZ5k42wKOPagqaLOHqT6/mcKHndFJR\nEVGM7DKSmMiYIOeyenb9exeLExaT/rbXEYlD2g3db2Db3dt4qM9D1I+tz5KdSxg+czjPLX2OxERT\n0IuIMHNELFxodW5FMEkfCBE2tIaLLjInuVtvLWnG5FeRkaYW4tgx81gIm5A+EMIqxcVmLp7vvoNT\nz8wkYcwIVmb8CMCLF73I3b3utjiHNbPjmR1sm7iNlAkptHm2jdXZqbbDhYeZ/st0Xl35Ko7rHbRI\nbAGYpsCPPmqmPvrlFzjpJIszKvxK+kCIsDdnjik8JCUFoO+Dm/SBEEIIn0ycaAoPEYl72dCvx3bm\nvAAAH99JREFUGyszfqRh3Yb8/dy/M7LLSKuzV2Pa6Sro2vyeUmJsIuN7jWfjHRuPFx7AfH4XXmjm\nUB06ooDCQgszKYJGfuX4wA5t0oLNLjEpKIB77zWPn3gCGjcOwJu474YqhWPRogC8gb3Z5VgJJomJ\n/cln6F1V4zJjBkyeDBGRGudlI+nYOolpl0xj1z27eHLAkzRLaBbYjAZByoMp9D3cl23nbbM6K36h\nVNmb0RER5nNs0qKA1Svr0H7Qd3y/zUFVawjlO+TJDjGRAoQIC1OmwNatZgbN224L0JtI/wchhKiy\nJUtgzBjzeOpL4Hj8cdbdvs7yid/8LSLGNQ9EjM2rICqRnAw3PvcFROex64cBDLjpG0577TSm/TyN\n3KJcq7MnAkD6QIha788/oUMHMz3DN9/ABRcE6I2KiiA21gyMLWPaCZuRPhAikA7kH+C9Ne/xTto7\nLLhmAYd2nsS550JWFtx+u5k4TtjfOx9mceOo+qAVDL4Tev6H/wz+D+POHmd11kQ1VXRtiLIiM0IE\ni9Zwxx2m8DBiRAALD1DS/0E6TwshBFprlu1axrRV0/hk3ScUFpvG8c/P+5xZD4wjK8ucl6dOtTij\nwm9uuDqJ4ny4+WZg/st0bpnCtV2vtTpbIgCkrYUP7NAmLdhCPSazZsFnn0FiohlfPKBKNWEK9bhY\nQWLiSWJif/IZeudwOHj6h6fp+05fZqydQVFxERe3u5hpvb9m9kO38+ef0K8ffPhh+NxzCZdjZcwY\neOEF83jD9AdYOK+eR5qi4iIe+fYR1u9bHzZx8YUdYiIFCFFr7d0L41y1pi+8ACkpAX5DGYFJCCGO\nG9FxBE3im/Bw34fZctcWnu08n0nXDWL3bkWfPjBvXsWTxdUmWx/eyuKExez7dJ/VWQmae++Fxx4z\nl8Urr4R33y37/JyNc3h2ybN0frUzt395O2+tfsvrnB8idEkfCFFrXXEFzJ4NAwfC11+DCnTr7uxs\nM0Zs/fqmYa8QNiJ9IISvtNakZaSxcOtCHuzzoNc0x5zHiIqI4ptv4LLL4PBhU/Mwd66pGQ4HziIn\nziInEdERRMSGzw0mreHvf4dnnjHrTz5phnxVCtbtXcdLK17i498+5nCRKTjER8fz9ICnbT/vR21T\n0bVBChCiVvrkE/jb3yAhAX79FVJTg/Cmhw5Bw4bQoAEcPBiENxTCf6QAIapq04FNzF4/m49++4jf\n9v4GwK+3/UqXJl28pp8xA264wcyvOXIkvPeeGW9ChIdXXoE77zQFirFjzXqUqwduXlEen6z/hOm/\nTGfJziV8/NePa8XcH7WJTCTnB3ZokxZsoRiT3btLhmr917+CVHgA6QNxAhITTxIT+wu3z/CKT67g\nlP+cwsTvJvLb3t9oVLcRd5x1B/HR8WXSORwOCgth/Hi49lpTeLjvPtPnIVwLD+F2rLiNGweffmqa\nq73xhmkVsGePeS4+Jp7UrFR+uOEHNt+5maGnDPW6j/fXvM/SnUspdhYHMefWscOxIqMwiVrl6FFT\n83DggJkZc+zYIL659IEQQtRynZM7szB2IcM6DuPyUy/nonYXERMZ45Fuzx7o0wdWrTJ3m//9b3MX\nWoSnESPMMOqXXw6LFkG3bqYmasiQkjRtG7b1+trcolxumXcLR44dITkumb90+AtDTxnKoLaDatV8\nIXYjTZhErXLvvWbSuJYtYfVqM7lN0GRkQPPm0KwZpKcH8Y2FqDlpwiQOHTnEwq0Lmb95Pqc0OoUJ\nfSd4pMkpzKFOVB2vhQYwzVTeew/uusv0d0hNhY8/hp49A5x5YQuZmTB6tOmXCKaG6qmnID6+4tfs\ny9vHs0ue5YuNX7D10Nbj25vEN+HPe/8kMiJMhvGyiPSBELXep5+auxtRUbB4MZxzTpAzsGePKbm0\naFFSPyuETUgBIjzty9vHW6vfYv7m+SzbtYxibZqIdE7uzG+3/+bTvtavN81HFy8265dfDm++acaW\nCGebbttExvsZdHi1A81GN7M6O5ZzOk3z4okTTcvfk0+Gl1+GYcMqf53WmnX71vHF718wZ9McUpNS\nmXn5TI90eUV5OLWTxNgw6aUfYNIHwg/s0CYt2EIlJps2wY03msfPP29B4QHKNGEKlbiEEomJJ4mJ\n/dn9M8w7mscj3z3CDzt/QClFv1b9mDxwMh/99aMq7yMrCyZMgK5dTeEhORkmTHAwa5YUHgCKjxTj\nzHOydN1Sq7MSEiIi4KGHYPly6NEDdu1yMHw4DB0KGzdW/DqlFF2adGHieRNZMWYFH1z2gdd0s9bN\nouFzDTn3nXOZ5JjE99u+J68oL0D/TWDY4bwifSCE7WVmwuDBkJNj7njddZdFGZGZqIUQISSnMIeV\ne1ayYs8Kfsn4hY//+rFHc4/UpFTuP+d+erXsxcA2A6lfp37V958DL71k5tnJzjbDc95yCzz7LKxZ\nE4Shs+3CdWlQERKQ0s48E376Ce65x8wTMXcufPkljBoF//gHtGtX+eujIrz/hN2etR2tNUt2LmHJ\nziUARKpInr/wecb3Gu/n/yJ8SRMmYWu5uXD++fDzz3DGGeBwmKFbLbF1K7RtC61bm8dC2Ig0Yao9\nxi8Yz8KtC9mwbwOakhitGruKHs171Hj/e/bA66+b4TjdI1YPGGAKDmefXePd1zrOQtc8ELERRMRI\nww9v/vwTJk2Ct982I3ZFRsI115iO92ee6fv+sguycWx38N2271i6aym/ZPzCZyM/8zrK0zdbv0Fr\nzRktzqBh3YZ++G9qF+kDIWqdo0dNm8n586FNG1i2DJo2tTBDmzdD+/amELF5s4UZEcJ3UoCwj315\n+1iTuYauTbuSHO85UsQF/72A77Z9R0xkDN2adaPnST3peVJPhrQfQoO6Dar1nk6nGT3n1Vfhs89K\nRq0+91wzQVi/fjX5j4Qwtm41nar/+9+SY6xnTzMU7F//CnHVHHTpcOFhoiOjqRPlOfV5n7f7sGzX\nMsDUyJ3R/Ax6NO/BDd1uoHli8+r+K7WG9IHwAzu0SQs2q2LidJohWufPh8aNYcECiwsP7kyB9IGo\ngMTEk8TE/oLxGX7+++fcNu82Brw3gOYvNKfJ800Y9P4gvt32rdf0T57/JMtvWk7OhBxWjFnB1MFT\nueb0a3wuPGhtanfvuw9SUkwtw+zZ5rkrrjAFikWLvBce5Nj2JDHxrnRc2rQxtRAbN5pRFZOSYMUK\nM3JTkyamVmLePCgq8u09EmMTvRYeAPqe3JdzWp5D3ai6bM/azqcbPmXidxOPz5Bd3pKdS9iwbwMF\nxwp8y4QP7HCsSB8IYTvHjplZTWfMgLp1zcmkfXurc4X0gRBC+ORw4WF2ZO9gR9YOdmTv4IzmZ9Cz\nped4p99t+45pq6YdX0+ISeD0pqdX+IOo98m9q52nnBz49lv46itzY2b37pLnWrUyP+RuuQVOOqna\nbyHECbVta/rWPPmkmXzwrbdMQeLDD82SmGgmpBs8GC6+2IzkVF2TB00G4JjzGL/v/51Vf65iTeYa\n2jbwnJdCa82lH15KdmE2CkXLei1p17AdbRu0ZcrFU0iIsaoNdfBJEyZhK0VFcPXVZsjW+HhTeOjf\n3+pcuaxbB126QKdO5rEQNiJNmPxHa012YTZO7fTapvql5S8xadEkDhUcKrN94rkTeWrAUx7pF21f\nRFpGGqc0PoUOjTqQmpRKhKp5AwKnE7ZvN7UMS5fCkiWQllZyLwTMtDZXXAFXXQW9eknH6Cr78ksz\nk15lw1BlZZnAX3JJ8PJlY1u3wsyZZl6RtWvLPte6NfTta5aePeHUUyHG+1QlNZJblMtlMy9jy6Et\n7MjacXzY49jIWPIn5nt8L4udxVz60aW0SGhBy3otObn+ybSs15KU+il0Su7k/wwGgPSBELZ35Ii5\nkH35JdSvb5ovWTJca0V+/RVOP90UIn791ercCOGTQBYglFINgLeBQcA+4BGttddxQpVSk4GbAA28\nrbV+qIJ0Qbs2OLWTrIIstNY0imvk8fzCLQuZtmoa6YfTSc9NJyM3g4JjBYzvOZ4pF0/xSP/ayte4\n/avbqRNVh5T6KaQmpdKqfisu7XCp106eNaW1medy82bYsMGMkLRmjfkRdrhcK42oKFNQGDzYLF27\nmmE3hY+yssxEB08/zbqb93Bg/gE6z+xMo0saeTwvY936budOU0M2f76pMSt/HEdHm0JE165m6dLF\njOqUkmKe84ejxUfZkb2DzQc3sz9/P6NOH+WRJv1wOi3+3cJje4M6DTj40EGP7XlFefzj+3/QJL7J\n8aVxXGMaxzWmXcMTDEsVIFKA8AOHw0H/kLndHRqCFZPdu2HECHOnrFEjM4tlj5oPJuJfa9ZAt27Q\ntSuOF1+UY6Uc+f54CqWYBLgA4S4s3Aj0AL4EztFabyiX7hZgPDDAtekb4CWt9Rte9lnla0NuUS5/\nHv6TvKI8DhcdJqcwh5zCHFrWa8l5rc7zSP/lpi95fNHj5BTmcPDIQQ4eOYhTOxnTfQxvDn2zTFqH\nw8G2+tu4cc6NZbYnxiRyY/cbefHiFz32n12QTcGxAprEN0HV8Ja+1nDgAKSnmyUjo+Tx9u2wZYu5\nc5uf7/31TZua01afPiV3b6vbUbW0UDq2LeMqJBT/40kOFhbz4f99yOjrR9NAKSk8lFLTY6W42Nyz\nW7IEfvgBVq82x72300NkpGmK17at6W/RogU0b26WZs3M36ZNTUHaH/KP5uPY7mB3zm52Ze9i92Hz\nNyYyhq+u+coj/ZaDW2j3cjvYBrQu2Z5SP4Ud43d4pN+bt5cxc8bQoG4DGtRpQL3YeiTEJNAsoRmj\nu472SH/MeYysgiwSYhKIjYyt0vmnomuD9IEQIW/ZMrjsMjPfQ+vWptlSp1Cs+XMPGSG364Q4TikV\nB1wGdNJaHwGWKqXmANcCj5RLPhp4QWud7nrtC8AYwKMAAXDRjIvIP5p/fLmwzYW8NPglj3RzNs7h\nmv+7xmP7lV2u9FqAyC3K5ec/fy6zrX5sfY85FNz6p/Zn5uUzaZ7QnOaJzWme0Jz4mHivabWGWOrj\nPFqf9HTzw/7IkbKLe1tenvkNWnrJzi67vm+fGZHuRBo2NHdg27cvuSvbtWsIDD5RmyUlwdNPs+XS\nS/l8504idu/mzSkvMDwlhQ7z5knhwU8iI00huFs3uOMOsy031xQq3LVtGzaYQsXu3aZAXdlI60qZ\nwVkaNDBLUlLZx0lJpgl1XFzZxdu22Ng4Lmw9hMjIqjX/qxdbj38N+hc/L/uZOu3qsDdvL/vy99Es\nwfsM5pm5mczdNNdje8fGHb0WIDYf3Mypr5xq4qYiiYuOIy46ji5NuvDN6G9OnMHScQqVu/sVCaUa\nCBFcWsMbb5hxoI8eNSOAzJplaiBC0s8/w1lnmQkpfv75xOmFCCGBqoFQSnUDlmqt40ttuw84T2s9\nrFzaLGCQ1nqla/0M4DuttcfsZkopfUvTT9hWt4ivG+WAjqBVlmLgVkV9YkmiDlvIQ6PYk6D46aQI\n2uRHMyAnnggNeyKL+V0V0u6gQqNwEkEjomhMJGmRx8iNjiCyWBHlVLQojqAz0WhgL7AGxTGiOEo0\nx4iiMRE0B5aSwDGijj/XgmP04TBOIthKAv+jOccoaT/RhlzakcvXlP1x0IZcBpEJwDbivT5f+nX1\nyaI56XQlnXPII4E8Ikgnnh9pyxbasoUkssl1vbIZX5fZXy5tyGQQAPFs8/q8vK56r4NtfMrXPFjq\n+cnAzYDMOBB8BcSyjdZsoS3baE06zcmgGek0P77sIxkdgEFKoykimqNel5hyz0VSTATOEy7FEU4O\nxDk5Fqk5GumkONJsi3U6OW1f2bQKzcE68FUHzdFIjTPC/dta0+iI5or1YM6GZrv78X+4S2oghH2k\np8PNN5v+DgDjx8O//uW/asWAkFGYhPAmAcguty0bSKxC2mzXNq+2Z75FEW1gexMgiR10Yzr9OY99\ntGEe79Ea6A+5wEYHmiyGunb3HttIozVp9HftzcHpZPFX2rCguAsUO1zbzf568AEASQzkO7oADqAI\n6M157MPJPDa638+1v8ZkMZwk1/ut4Jjr+RgKiWYhzTjEIDqwmwMc4wdiKaQ5HTmNY7TkDwD60Ypz\nWcU+1pFALucQSz1asJYYbmImfyGXuhTgALI4nSReAhLZxv9owieccTw3kEUC7ehDM76m5L+DI7Rk\nLh0BGEgLr8/Pp0W5/87sL4krAdhWEm15v1Lv140WdC/3fA/gA+C0cvtD1oOwXsip/E4mv3MacKeX\n9EeJYg6J5JJAe1qSRRJLKSKXBJLpQjb1+YPdFFCHepxBPnHsYROF1CGKvuQRzyHSKKAOTi7gKNE4\n+YGjwFF//0fO/ub85uX5Nd7SF/SHtZ7pD+Bg2vF1B/Cu6/lUKiI1ED6QNp2eAhGTmTPh9tvNDKdJ\nSWbioquu8utbBMby5aZXd69eOJ59Vo6VcuT74ymUYhLgGoglWuuEUtvuBfpVUAMxUGv9s2u9B/B9\nRTUQHw3fwdEmdcnrnoxSHF9i9+YTk57HkTPM9ogIsz06I5+6q/ejFBQ3r0vR2WWfj0zPJ2p3HsXn\nlN1fZHo+Ucv3gwJOqos6N5noaHNDY+1aB2c2Oxu1I4/4i5KJiuL4c86d+eR/vZ/ISIjrUJcmlyUT\nGVnSyjF/Uz556/NIHl52Mrj8Tfns/2I/AHXb1/X6fKi/zuFwcHaLs0M+n4F+nbO5k1mPnMeDu3bh\nwPw8m3zyyYxds4YGDao3qV9tE0rnwUBxOk1LCm9LUZHntlWrHHTt2h+nE58Wrb1vd7ew1rpkKb1e\n0WOt4e67pQ+ECHEbNpiJYxYsMOsXXQTTp9tovHHpAyGEN5uAKKVUW631Fte2roC3sY7XuZ5ztwHs\nVkE6AK78LKWCZ+Jci7ftFb3mBK+72fvrjh6Fs/pX8LqWcdC74veL6xBHXAfP18V1iCPlAXldrXhd\nVhbDp6UwGYjcvZsVLVsyIiXFdKQWYSMiAmJjzVIVRUVwnmf3LEvcfbf37VIDISx38CA8/ripaSgu\nNkO0/vOfZrIiW51jf/jBfOP79jWPhbCRAI/C9CFmWNabge7APKB3BaMw3QXHG5B/jRmFqezQR8i1\nQdhAqaFaDzqdrF+/ns6dO8soTMJWKro2yK1SYZnMTJgwAVJT4eWXTVXZLbfAH3/ArbfarPAA0gdC\niIqNw9yi34tp/n2r1nqDUqqvUirHnUhr/TowF/gV01J3rrfCgxAhr9w8Dw0bNqRv376m2ZJrdCYm\nTjTphLAhKUD4wOFwWJ2FkFOdmPz+uxlZKTUVJk82E8AMGgS//ALTpkFy8gl3EZpKNWGSY8WTxMRT\nuMREa31Iaz1Ca52gtU7VWs90bV+ita5XLu0ErXUjrXVjrfXD1uS46sLlM/RV2Mdl6VKPGoYyMXEX\nIpYuDX7eQkzYHyte2CEm0gdCBMWRIzB7Nrz5ZtnWPcOGwcMPm8mLbM9dAyF9IIQQIrxdcsmJ0yQl\nVS2dECFI+kCIgMnPN9PMf/KJmfwtL89sj4+Hq682tRCnnWZtHv3q669Nz+8LL4T//c/q3Ajhk0D2\ngQgEuTYIIUTgyUzUIuC0hvXrze/ohQvB4TA1D25nnWXmdrjySkj0NgK83ckoTEIIIYQIA/JLxwd2\naJMWTIcOwXPPOXjiCVML26QJdOlihmKdP98UHnr2NBPAbd0KP/1kChC1svAAZZowybHiSWLiSWJi\nf/IZeidx8SQx8U7i4skOMfFLDYRSahxwPWZixQ+11jeeIP09wINAHeBT4Dat9VF/5EX436FDsHGj\nqV3YsKFk2brVM22zZqZD9IUXwsCBZj1sSB8IIYQQQoQBv/SBUEoNB5zARUDdygoQSqmLMHNknw+k\nA58DP2qtH6kgvbRzDZAjR2DfPti7t2TJzISdO2HHDrPs3Ak5Od5fHxsLPXqYWoaePaFXL2jVyobD\nr/rLF1/A8OGmZ/jnn1udGyF8In0ghBBClBfQPhBa689db3IWcKJ5g0cD07XWv7te8yRmXHCvBYhw\nprVpVl9YCAUFZin9uKJtubnmR39FS3a2KTjk5lYtH3Fx0KEDnHpq2aV9e4iJCWwMbEX6QAghhBAi\nDFjRibozptbBbQ3QRCnVQGt9yNsLrkhZAZhpTN2FIA1oXI+1wn0fSqNOnEarku1l9lk+Tdl9ZhX9\nRP2Ys6v8Pk4dQbGO4Jj7rzOS4ipsM48jcerA/hCNjjhGk9hsmtTJITn2ME3qZNMkNoeU+AOkxO2n\nVbxZGsbkltQqbHAtLo7MTPo3bRrQfNrGzp3mr6sPRP/+/S3NTqiRmHiSmNiffIbeSVw8SUy8k7h4\nskNMrChAJADZpdazAQUkAl4LELN3vQakutaSgG5Af9e6w/U3GOuZriVY7weKb4nmKPGcTR0K0DiI\n5iiNOJ06FHCE5cRQRAtOoQ4FHCKNOI5wKk2pRw4ZbCCefHpSl3rksIk9xJPPpeRSz5nDoiPAkern\nNi2o0bDJursmgpKOUO4TgazLeun1tLQ0y97f4XDw7rvvApCamooQQghRVSfsA6GU+h7oB3hLuFRr\nfV6ptE8CJ52gD0Qa8JTWerZrvSGwD2jsrQZCKaVn3vkDSpm3V5S0sVfKfd/fbHPf+1eqVBp0qfRV\nSVO1fVblfZSCqEgnkUoTFamJjDCLe5t5XPG2yAgdvv0J7ComxswFUWuHmhK1lfSBEEIIUV61+0Bo\nrc/3c17WAV2B2a71bkBmRc2XAP42ta+fsyCEEEIIIYSoDr80sldKRSql6gCRQJRSKlYpFVlB8v8C\nNymlTlVKNQAmAu/4Ix+BZodxeYNNYuKdxMWTxMSTxMT+5DP0TuLiSWLincTFkx1i4q9eun8H8oGH\ngGtcjycCKKVOVkrlKKVaAmit/wc8B3wPbHMtj/spH0IIIYQQQogA8ss8EIEk7VyFECLwpA+EEEKI\n8iq6NsiA9UIIIYQQQogqkwKED+zQJi3YJCbeSVw8SUw8SUzsTz5D7yQuniQm3klcPNkhJlKAEEII\nIYQQQlSZ9IEQQgghfSCEEEJ4kD4QQgghhBBCiBqTAoQP7NAmLdgkJt5JXDxJTDxJTOxPPkPvJC6e\nJCbeSVw82SEmUoAQQgghhBBCVJn0gRBCCCF9IIQQQniQPhBCCCGEEEKIGpMChA/s0CYt2CQm3klc\nPElMPNX2mCilGiilPlNK5Sqltimlrqok7f1KqV+VUjlKqS1KqfuDmdfqqu2fYXVJXDxJTLyTuHiy\nQ0ykAOGDtLQ0q7MQciQm3klcPElMPIVBTF4FCoBkYBTwmlLq1ErSXwskAYOBO5RSfwt8FmsmDD7D\napG4eJKYeCdx8WSHmEgBwgdZWVlWZyHkSEy8k7h4kph4qs0xUUrFAZcBf9daH9FaLwXmYAoJHrTW\nz2ut07TWTq31JuALoE/wclw9tfkzrAmJiyeJiXcSF092iIkUIIQQQgRCB+CY1npLqW1rgM5VfP25\nwDq/50oIIUSNSQHCB9u3b7c6CyFHYuKdxMWTxMRTLY9JApBdbls2kHiiFyqlJgEKeCcA+fKrWv4Z\nVpvExZPExDuJiyc7xMQWw7hanQchhAgHvgzjqpT6HugHeDtHLwXuApZqreNLveZeoJ/Welgl+70D\nuAfoq7VOrySdXBuEECIIvF0boqzIiC/sNC65EEKEC631+ZU97+oDEamUaluqGVNXKmmWpJS6EXgQ\nOLeywoPr/eXaIIQQFgn5GgghhBD2pJT6EFNDcTPQHZgH9NZab/CS9hrgeaC/1npjUDMqhBDCJ9IH\nQgghRKCMA+KAvcAHwK3uwoNSqq9SKqdU2ieBhsBKpdRh13wQrwY9x0IIIU5IaiCEEEIIIYQQVSY1\nEEIIIYQQQogqkwJENSml2iuljiil/mt1XqymlIpRSr2llNqulMpWSq1SSl1sdb6soJRqoJT6TCmV\nq5TappS6yuo8WUmOjcrJeaT2kc/UkO9+CbkueJLjo3J2OI9IAaL6/gP8ZHUmQkQUsBMzckp94FFg\nllIqxdpsWeJVoABIBkYBrymlTrU2S5aSY6Nych6pfeQzNeS7X0KuC57k+KhcyJ9HpABRDUqpK4FD\nwLdW5yUUaK3ztdZPaK13uda/BLYBZ1ibs+ByDVt5GfB3rfURrfVSYA5wrbU5s44cGxWT80jtI59p\nCfnuG3Jd8E6Oj4rZ5TwiBQgfKaXqAZOA+zAzpYpylFJNgfZUMt57LdUBOFZqzHuANUBni/ITcsL4\n2ChDziO1j3ymlQvj775cF6ogjI+PMux0HpEChO+eAN7UWu+xOiOhSCkVBcwA3tVab7I6P0GWAGSX\n25YNJFqQl5AT5sdGeXIeqX3kM61AmH/35bpwAmF+fJRnm/OIFCBKUUp9r5RyKqWKvSyLlVJdgYHA\ni1bnNZhOFJdS6RTmJFAI3GlZhq2TC9Qrt60ecNiCvIQUOTZKKKW6EYbnETuTa4MnuS5UmVwXKiHH\nRwm7XRuirM5AKNFan1/Z80qpu4FWwE7XQZ8ARCqlOmmtzwxGHq1woriUMh1oDAzRWhcHMEuhahMQ\npZRqW6q6uithXiXrEu7HRmn9CMPziJ3JtcGTXBeqTK4LlQv346M0W10bZCI5Hyil6lD2TsIDmA/7\nVq31QWtyFRqUUtOA04GBWut8q/NjFaXUh4AGbga6A/OA3u7Zd8ORHBtlyXmk9pHP1Dv57htyXfBO\njo+y7HYekRoIH2itCzBDsQGglMoFCkLxgw0m17BrYzGxyTQFZzRwi9b6IyvzZoFxwNvAXmA/5osf\nthcJOTY8yXmk9pHP1JN898uQ60I5cnx4stt5RGoghBBCCCGEEFUmnaiFEEIIIYQQVSYFCCGEEEII\nIUSVSQFCCCGEEEIIUWVSgBBCCCGEEEJUmRQghBBCCCGEEFUmBQghhBBCCCFElUkBQgghhBBCCFFl\nUoAQQgghhBBCVNn/A/DkBetHy0ZRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f67fb4c1e10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "z = np.linspace(-5, 5, 200)\n",
    "\n",
    "plt.figure(figsize=(11,4))\n",
    "\n",
    "plt.subplot(121)\n",
    "plt.plot(z, np.sign(z), \"r-\", linewidth=2, label=\"Step\")\n",
    "plt.plot(z, logit(z), \"g--\", linewidth=2, label=\"Logit\")\n",
    "plt.plot(z, np.tanh(z), \"b-\", linewidth=2, label=\"Tanh\")\n",
    "plt.plot(z, relu(z), \"m-.\", linewidth=2, label=\"ReLU\")\n",
    "plt.grid(True)\n",
    "plt.legend(loc=\"center right\", fontsize=14)\n",
    "plt.title(\"Activation functions\", fontsize=14)\n",
    "plt.axis([-5, 5, -1.2, 1.2])\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.plot(z, derivative(np.sign, z), \"r-\", linewidth=2, label=\"Step\")\n",
    "plt.plot(0, 0, \"ro\", markersize=5)\n",
    "plt.plot(0, 0, \"rx\", markersize=10)\n",
    "plt.plot(z, derivative(logit, z), \"g--\", linewidth=2, label=\"Logit\")\n",
    "plt.plot(z, derivative(np.tanh, z), \"b-\", linewidth=2, label=\"Tanh\")\n",
    "plt.plot(z, derivative(relu, z), \"m-.\", linewidth=2, label=\"ReLU\")\n",
    "plt.grid(True)\n",
    "#plt.legend(loc=\"center right\", fontsize=14)\n",
    "plt.title(\"Derivatives\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.2, 1.2])\n",
    "\n",
    "save_fig(\"activation_functions_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def heaviside(z):\n",
    "    return (z >= 0).astype(z.dtype)\n",
    "\n",
    "def sigmoid(z):\n",
    "    return 1/(1+np.exp(-z))\n",
    "\n",
    "def mlp_xor(x1, x2, activation=heaviside):\n",
    "    return activation(-activation(x1 + x2 - 1.5) + activation(x1 + x2 - 0.5) - 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmYAAAEOCAYAAAAjY9sgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztvXu4HGWV7/9ZkstMDAlJjPwAkYCIXCRcDMgYhMBRGRwH\nhOFnCBdFRgdm0GeOkGdwApENGeKg5MxRznAZuSgM7B8j4oA8w8icI1skZ8Q4mOxwU4RsQC4hhmSH\nEMgm8P7+qKqkdqW6u6r6rXrfql6f5+ln7+p+u2r1292rvr3qW6vEGIOiKIqiKIrinne4DkBRFEVR\nFEUJUGGmKIqiKIriCSrMFEVRFEVRPEGFmaIoiqIoiieoMFMURVEURfEEFWaKoiiKoiieoMKsJERk\nlYicX8F2LhGRwQq2IyJynYj8XkTeEpGjyt5mh3huEpG7HWx3DxF5W0QOrXrbKbFkngMROTp836a2\nGXOKiLxtL0KliWhuKz0eJ7mtEz7FlSUWEfmRiNxYVUw2Ee1jBiJyMPBL4D+NMR/N+dxLgFOMMQcm\n7p8GvGaMecNSjHsAq4BZxpiHY/dPAMYbY9bZ2E6b7f8J8APg6DCOV4wxW8rcZrjdo4H7gXcZY16J\n3b8jwed3Q9kxJOLZA3gaOCz+PrggzxyIyBhgqjHm5TZj/gz4F2PMDhbDVByiuS3T9jW3ZcCnuLLE\nIiI/AtYYY86uLjI7jHEdgCd8EfhH4LMi8gFjzK9zPn87dWuMWWslsm1Ii+1sAjZZ3lYa7wdeNMY8\nVMG24kSvW+J3GmNerTiOONJ5SPnkmYNwR9NSlCmNRXNbZzS3ZcCnuHyKpRSMMT19A/4AWAd8EPgO\n8M2UMbsAtwK/B14DHib4dfU54G3grdjfz4bPWQWcH/5/G3BHYp0CPAv8dbh8HPAA8AqwFvh3YN/Y\n+Ph23gZ+Et7fB6xMrHdhuO43gEHghNjje4TPPxm4L3w9jwIfazNHNyW2/3R4/wDw7ZSxd8eW7yfY\nMVwOrAFWJ+cYGAssBobCmH8LfCkWa3x+bwyf893EdsYB/xN4CXgd+E9gduzxo8N1HAv8PHzdy4BD\ncn5eMs0fsD9wD7AhfM23ATvHHp8F/Dick2HgZ8ARscezfGaSc31U+LpfBdaH/++feP1TY+M/G875\nRuBu4K+AtxLb/FOCisvrwFPA3wFjXX9v9Zbps6q5TXNb3s/M12Kxvgh8t83rnwDcTJBvXgS+Cvwo\neh2xz8rC8LkbwvfuM8BkoD987m+AjyfiOCp8La+Hr/t/EMs7KbH8YThvUSx/m4ylTjfnAbi+AWcC\nvwr/Pzr8cu2Q+PA9SbDjnA3sCXw6HDse+CbwGDAdeDdB6T36QEbJ65MEv/wmxdZ7DDACTA+XTwZO\nAvYiSKT/X7jdMeHjs8Iv38fC7ewU3n8JMBhb71cIdspzgb2BS4EtwMzw8SghPBbG9b7wA70GmNBi\njnYkSJLPhK9zWnj//WRLXuvC5+8NnAK8CcyNjekPv7CfBmaEc3sGQSI+iSBpfSB83Tu22M63gOeB\nPw7H/lP4Jd059t6+TfBlPwrYh2AH8Wgi/reBr7X5vHScP+D/CZcXh9v5IHAX8FDi/T89fHwf4NsE\nO66pOT4zW+cA2CF8/hXhHO4DnAp8IPb634qt/8Ph8lfD9+WLBDvnt2LbO45ANH429r48DnzD9fdW\nb5rb0NxmO7f9GcH3/Y+B9wCHAn/V5vVfG34WjgX2C1/rerYXZr8Hzgnfj28SiK17wnnYC7ieQHyN\nC5+zK8GPxX8MX+8nCcTWN9vEcjXwXPgZ2h/4l/C1qDCr443gl9FXYstPAyfFlr8YvsFTWjx/VPJI\nfCCj5LUDQVL8fOzx7wD3tonrnQRJ5yPhcpR0Dm23feB3wEWJMfcDNyfW84XY47uG932kTTwXEP6a\nTKw3S/JamhhzH/BP4f/vD7f98RbbHSUo0rZDsIPZDJwee/wdBL9OL4ut521iv56Bj4Tr3jV232PA\nX7aZh47zR7DD+I/E86aEY2a1WK8ALwCnZf3MJOZgSvhaPpplHgmqJD9OjPkOo4XZT1M+SycCr5b5\nndSbnRua27b7braIR3Ob2Sp8Hycm3tvE9c4wrv839vgEgh+HSWF2a+K9fxv4h9h9o95/ggrkbxLb\n/hyBoPuDFrG8AZya2M46airMevqsTBHZm+CXYn/s7tsIElbEwQTJobAB1RjzFnA7QYUEERlH8Ovk\nllgse4nIbSLyWxEZJvgFIcB7s24nNETuCvzfxEMPEvyKiLMyFt8L4b/vzrqtnCTPrHohtq1DCBLI\nQBfrfx+BX3Lr6zbGvE14KC82zhB73WEcEosFY8z+xphrMmyz3fx9CDhaRF6NbgS/mk0YKyIyPTwT\n7Ncisp6gzD+d8P3O8pmJE34+vwfcJyL3iMhXROQ9beLfj2B+4iSXPwRclHgdtwF/KCI7t1m34hjN\nbVvj09y27Xmdctv3CQ4JDonI9eFZ2uM6xLUstv5NwCMpYwdjY14jqLDGx60O/0ax7sv2uehBgkO6\ne7eIZSxBxTC+nZUpY2tBr5v/v0Dw6+M5kdF+bhHZzRjzPPaM3v8MLBWRXYA/Ivgg/Wvs8XsISrF/\nQVC23kLw66XVF6MdJsN9b6aMySvU32b7+RmbMi65LVNgW+2IYsj7uqPHisTSbv7eQfB+XsD28xMl\noZsJhNhfExxG2Qz8hNHvd6fPzCiMMWeLyD8QHIo4AbhcRE40xvxHyvAsn+t3EFT/vp/y2JoMz1fc\nobltNJrbOmCM+Z2I7AP8N4JDglcCl4jI4caY13PElSRtjtrFmnoySIf7G0XPVsxEZAcC78xXgYMS\nt0Hg8+HQh4GZbfo/jRCU89tijPkFgXn6tPD2r+EvDMJ17wssNsb8xARnTk1mtHAeCf+23JYJzlR5\nATgy8dCRBGVs26whMA/HOSjnOh4m+Bwe0+Lxjq+boKz/JrHXLSLvINhJPJozHhs8DBwAPGuMeTpx\ney0cMxu4yhjz78aYxwkMu6Pmst1nphXGmJXGmG8aY44h+KX+uRZDHwOOSNz3RymvY9+U1/B0+Ktd\n8RDNbVboydxmjBkxxtxrjLkAOJwgj81uEdeWcEwU1wQCD2G3PMb2ueijBD9en2oTy9Z8JiLvtBSL\nE3pWmAGfAqYB1xtjHovfCErzfx6Ou42gzcC/isiRIjJDRP407EEDwRkse4jIISIyrU3pN1rXFwjM\njP8cu38dgUHyiyLyvnDd1zD6V8XLBMfYjxORd4vIpBbb+CYwX0ROFZH3i8hlBF/qK7NMSk5+Ahwf\nzsc+IrIE2D3PCowxvyWoyFwvIieH83ukiJwRDnmG4FfSn4jIu8IvXHIdmwjm6+9F5HgR2ZfAmPru\n8P6Ijr+sROQJEfmrPK8hhX8k2Pn8i4gcLiJ7isjHwkOXUfy/Ac4Qkf1E5DCCQ06bU9bV6jOTjHuG\niHxdRP5IRN4rIscAMxmdvOOv/9vAx0TkqyKyt4h8kcCgHOcy4DQRuVREDhCRD4jIn4nIFdmnQnGA\n5rbu6bncJiKfE5E/F5EPisgM4GwC8fhkSlyvATcC3xCRY0VkfwJvYauqVh6uBnYVkWtEZF8J+sx9\nneCH7Ha988JYbgCuCPPsAeFybfVNbQO3wNkEp2Wn+Su+D7xXRD4WfjGOJijB301wbLyPbR++HwD/\nBvwfggRzanh/2ofznwnOMlkPbD28ZAK34mcIdqQrgauAi4ntqEMvx5cJkt/ztD6k9W2CBHZFuK4T\ngZONMfHj7VnK4lm4MbzdQOABeBW4s8B6zyRI7N8iOMRxEzAJtnpELiEwhL5EMDdpXEhwJs6NwK8I\nfi0dZ4xZHRuT5XW/n2Cn1o626zHGvEjwK/Mt4F6Cz8xVBAbV6D39PDCRoBXFbQRzOJSy3tTPTAqb\nCM7G+hfg1wRzeAvwjRYxPkSwgz4XWEEgyi4Z9YKMuQ/4E2AO8FB4u5Bgh6L4i+a20WhuC+iU29YT\n5IQHCOb3JIKTRVp93+eHY+8i+IysAP6LIM/liWvUfeG8HE/ggfwVwVmbtwIXtYl9PsHJGHeGsawM\nY6sl2vlfURRFUZSuCCuqzxC00/kH1/HUmV43/yuKoiiKkhMJLve1H/ALgirghQRHAW53GVcTUGGm\nKIqiKEoRziewUGwBlhP0UXyh/VOUTuihTEVRFEVRFE/oZfO/oiiKoiiKV3h/KFNEtKSnKD2IMab2\njSM1fylKb9JN/vJemEFwDq4vXAP8pesgEtiOaeYFMH7BMCPThjyJqFs0nvb4Fg/k7+XpLw+6DiDB\nDWxrZFY2s+d1HtO3EvoOLD+WrPRNgj7HX4e7D/rE1v/7+37LvL60KxHZ41rOyTX+yb7beX/f3Nzb\nufeBk3M/JxM39cHn+4L/+8rZxCjuf6jDgGTv7nzoocycPO86gBRsxzS4BDYvnsy4tTMKrsG3WdJ4\n2uNbPEqZvFjhtpb2B7d2DG2sJpasDP3YdQRwwor7tv6/eih5NST7nMt1ucZvGnq50HaOP+pOjj8q\n2Q7OAi8Obfu/z/7qt+OYD5e6ehVmSiqROFto9MxnRVG6o5M4844rwptD4uKsCs7lutwCrSiliLM4\nfeWuHihVnKkwy8mJrgNIoayYBpfAKbK4gDjzbZY0nvb4Fo9SJp90tN1W4uysvaqNoxOj4vFAnP23\ns3ardJtZxNl7zmp1+c/sWBVnx5+1/X19lC/QShJn3rfLEBHjk8esV7nDLGDRaZdD/6DrUJSe4KDG\nmP9985i5JovvzDsudB3AaN9ZFeT1nRWlNN9ZnL6S17+d5+yIrvKXVsxyssx1AClUEdMpspjhW8bD\nvJkZRvs2SxpPe3yLRymThx1vP1k5G1idPs4VqfE4rJwNhF9PF4c2W7F24BFr27FSOfvVQPvH+7rf\nRFssV85UmCmZGRozkkOcKYqipFM7zxk4P6wJfokzm5TuOYNaiTM9lKnkZsaWcUwfXtNFOw1F6YQe\nyuwF9LBmMfSwZhf0lbz++x9CD2UqlTM0ZqTLdhqKoijZ2ml4h1bOSkMrZwEqzHLioxvHRUzt22n4\nNksaT3t8i0cpE9cesyQP45c4y+R5q1CcDbT4erpsp2HTY5akUK+zTh6zJH35hleNCjOlMINL4LL5\nRdppKIqijMYncZaJHux1Bg2qnvXhrUBTj5liBW2nodhFPWa9ivrOiqG+sy7os7y++0U9Zop78rXT\nUBRFSad2lTNwXjkD9Z11RV/5m8iDCrOc+OjG8SWmre00jn3FdSgJfJmhCI1HcYePHrMkLsVZ4b5q\nJYmzVh6zNKoSZysHghzvjTjL6zFLo6/7VdhChZlilaExI9yz4ON6xqaiKF2hlbNiaOWsC/rK30QW\n1GOmlMLMC2D8gmHtdaYURD1myjZq5ztTz1lp1MJz1qXHTIWZUhoqzpTiqDBTRqPiLD9VizNokEDr\n6+K5av6vFh/dOL7FFMXTvtdZlfg6Q77gWzxKmdTBY5ZGVYc2rV2701I7jTwesyRlHdaMPGZpODm0\nacNjlqQPZ4c2VZgppTK4JDhjc6G5Xc/YVBSlK9R3lp8TVtynvrNu6Ct/E0n0UKZSGdrrTMmOHspU\nWlO7w5rQk4c2G3NYE/IJNPWYVYcBbpwIZ2+E2u8xHDFjyzgmn7lZxVkV7HQpTH7G3vqG94D1l9hb\nX1tUmJWBAW6dCKc3IIepOCtGncTZyguv4bXVL2Qau2799M6DJu0DX/inwvFkFmddCrMxRZ/Yizw4\nHlbsD0tXwJGbXUezjWXAYa6DiNEunqExIwxvGc/0/7WmwpMC6jRDFpn8DHz+vzqPWwXsmWF9NwHr\nu4xJccpD4+Gx/eGhFXCEJznsYeDQAs9b2l+OOBtYDXN2tr9eIDismVOcDSyDORbTxQkr7utanK0c\neIUD50zNNPZcrisszl5b/QLr9nys88BVwCEZVvirQmFso49KDm2qxywjBvjZXnDiqfDAXsGyUoyh\nMSNsXjxZe50pSoUY4KG94NOnBn+bkMPUc1aMpnrOKqGv/E2oMMvIg+PhQ6fCIYfAoXNh6XjXEW3D\np1oQZIsnOmOzGnFWxxmqkCzVMqX2PDQeZoU5bNbcYNkHilTL4izttyvQSquWxckhzmxWy+J0I86y\nVsvilCrOqs5hfeWuXoVZBqJq2ayPBsuHHaVVMxv4005DUZpNVC07LJbDmlI1i6hd9cxSO41ucFE5\na0z1rK+8Vaswy0BULROB5cuDvz5VzXzrQpUnnlHtNEqjzjNUAatcB6CUTVQti+cwX6pmNvuq2RBn\n1vqYZaWDOOumj1kWioizdn3MsmBdnLnKYX2UItCsCjMROU9ElonIGyJyY4exXxGRF0VknYhcLyJj\nbcZii2S1LEKrZnbRXmeKa5qYv2D7allEE6tmUMPKGXhROVPfWRf02V2d7YrZ88Ai4IZ2g0TkOOBv\ngGOAGcD7gEstx2KFeLUM4OCDg78+Vc08cywVjucUWczwLeNLEGdNmaGSUI9ZROPyF4yulsHoHOZD\n1axbj1ka3YizSjxmabQQZ2V5zNLIKs6KeMzSsCbOfMhhffZWZVWYGWP+1RhzN9CpzvlZ4AZjzBPG\nmGGCZPh5m7HYoFW1LEKrZvYZGjNSkjhTlPY0LX9B62pZRFOrZqCVs6Jo5awL+uysxpXH7AAg3jd2\nBfBuEZniKJ5UktUyCPwZEb5UzTxzLHUdTyTO7J2x2bQZsox6zPJSi/wF21fLYPsc5rpqVua1O4uI\ns8o9ZkkS4qxsj1kancRZtx6zJF2Ls4w5bMpOa7rbThb6ul+FqwazE4Hh2PIwQSPqHYF1ycEXA7vF\nnrgv2w4GRZ9Z28uzCKplH94xSGRR+f+3vw3+RstjJ8EdO8PsZ4MXUFY87ZafqHh7VcTDmBE2XzCZ\nsXfdw5YTX/IgIt9mKONylLCiUn/a8ksdHo8vlxYvwC8JjiZ6T6789XfALrEnvp9th+8iUVLGsgF+\ntDOcuOO2WJYvD3JYlL+WLw9y2H/uBR9+fFv/zSrii5afLHn9D/cHy7PnbRNd0eHKtOXl69o/Xsly\n2Ih2YBks//W2w5mRSKti+YQV93H5ulnAtkOXSUEWLScfL7J8Ltdx+cBsAKbN+SAAb770WrChTvmJ\nDo+Hy2++9BqHv30Zv3jH14I7ooufHzKnu2WA5QPw4hA2KOWSTCKyCNjNGHN2i8eXA39njLkjXJ4K\nrAHeZYxZlxjr5JJMPxsPmxYEpf5O/OKnMPHrfl0NoCnMvAC+duUCFslc16HUjz3Oztb5Pys3fQie\naeuJt4i7SzLZzl+uLsn08/Hw1gI4PGMO2+Hr/lwNoCxqdxmnHryEE2y7jNPPz1qYrfN/Rqas2p8j\nvrto63Jp19g8urtLMrk6lPkocFBs+WBgdTKpuaKTtyyJes3KY3AJXDa/7HYaipILr/MXdPaWJWmy\n1yxO7XxnPdjrDKrznR1/1J2VbCcvtttl7CAifwDsAIwRkfEiskPK0JuBPxeR/UJfxkUEV+LzgjRv\nWUTcnxHh2mvmmWPJejyjep0VOimg6TPUJeoxA5qTvyDdWxbRKoe58pqV6TFLo5M4c+4xSzCwGi/E\nWVyg2faYpZFLnHWRw3wUZ7YrZhcDmwgKsKeH/18kIruLyKsi8h4AY8yPgW8A9xNM6SoquQJVZ/JW\nyyK0alY+5bXTUBSgAfkL8lfLInqlagY1rJyBc3EG1VfPduWFSrbjmzgrxWNmk6o9Znm8ZUnUa1YN\nM7aMY/KZm6F/0HUofqMeM+e48Jjl8ZYl6RWvWUTtPGfQU76zBWf9gkf3XG9tfUmPWRJrnrOaesy8\npGi1LEKrZtVgv52GojSDotWyiF6qmoFWzoriwndWBb5UzlSYxWjnLYtI82dEuPKaeeZYqiSeoTEj\nbF48OaM468UZyoF6zBpDO29ZRKccVrXXrGqPWZKl/aMFmpcesySOxdnAMs/EmcUc5oM4U2EW0m21\nLEKrZtUxuIQc4kxRmk231bKIXquaRdSueqaVs9I4/qg7nQo0FWYhWaplsK0xYytcVM08u/JipfFE\n4qx9O41enqEM+HCdOaVrslTLIFsOq7JqVsa1MouytN/htTJb0DYeR+004tfv9EKclZTDXIkzNf+H\n3DgVXt09aN/dLQbY8Tk4u/wzipUYd5gFLDrtcj0pIGKnS2HyM/bWN7wHrL/E3vraoub/vNw6FTZZ\nzGETnoPTezSH6UkBxbB9UsD/uvBRXlj9WsvHX2DXXOt75867cuAVf5k7jtwnBXRp/ldhlpNleFfv\n8C4ml/GkizOdofb4Fg+oMCuPh/GrSuVrPL6Is4HVOap4FYmzgWWjq2Zxqr5SwLWcw9qBR7Zexqks\ncokzPStTUbahvc4URbFB7Txn0JO+syZeJUArZkojmbFlHNOH1zAybch1KEohtGKm+IEvlbNcNPCw\nZiei62uWTabKmVbMFGV78rXTUBRFSUcrZ8XQyllxVJjlxLMOVIB/MfkST3TG5pi7nnMdSgJfZijC\nt3iUMnHdNyxJHeJJ9jqrksJ91UoUZwMZU0ZV4iy6dmeV4qxMgabCTGk0g0vgP279VId2GoqiKJ2p\nXfXMUTuNOC4qZ3WvnqnHTOkZ7jALWCRzXYehZEI9Zoq/qO8sP1V7zsCh70w9ZoqSjVNkcVA50zM2\nFUXpgtpVzsCLypn6zrKhwiwnPrpxfIvJ53j8aKfh8wwpTacOni6XZI2nKnFm9dqdlsRZVo9ZGmWI\ns8hjlkYdxZkKM6XnGBoz4oE4UxSl7mjlrBhaOWuPesyUnkV7nfmMesyU+qCes2I0tteZ/Jl6zBSl\nCNrrTFEUG7hsp1EYrZx5iwqznPjoxvEtpjrFE/U6q7adRp1mSGkadfV0VUU38ZQhzqx6zJIUFGfd\neMyS2BBn7TxmSapsp1EUFWZKzzO4BC6bv1h7nSmlUMvDXEphalk58+CMzarxWZypx0xRYtxhFrDo\ntMuhf9B1KD1OczxmJhRmtdthK11RS0GuvjMrqMdMUSziRzsNpYnUcketFKaWQlx9Z15gVZiJyBQR\n+aGIbBSRVSKSmopEZJyIXCsiL4nI70XkLhHZxWYsZeGjG8e3mOoeT/ntNOo+Q82kivzlgzhrkqer\nDGzGY0OcleoxSyODOLPpMUsjrzjL4zFLwzdxZrtidjXwBjAdOAO4RkT2Sxn334EPAx8EdgWGgass\nx6IohYnEmZ6x2VNUkr9mz/NDoCnVoJWzYvRy5cyax0xEJgDrgP2NMU+F990M/M4YsyAx9mpggzHm\nq+HyJ4ElxpjtkqB6zBSXzLwAxi8Y1l5nlVOtx6zM/GXaiLBa7rSVwtROkKvnrBA+ecz2AbZESS1k\nBXBAytgbgCNFZJcwIZ4O/JvFWBTFClE7Da2cNR4n+at2O2qlK2onxHu0cua6emZTmE0kKOnHGQZ2\nTBn7G+BZ4HlgPbAvsMhiLKXhoxvHt5iaFo/9XmdNm6FG4Cx/VS3OmuzpskHZ8eQVZ5V7zJKktNMo\n22OWpJM469ZjloZLcTbG4ro2ApMS900CXk0Zey0wHpgCbCIomP47cETaii8Gdgv/n0iQBQ8Ll6PP\nR1XLT1S8vSzLT2g8pcfDEjhlyWKG7n+GWxZ9AX4y1XFEvs1Q1zMM/JJA6zihtPx11n/CjInB/zuN\nhYOnwJydg+VopztnXrDDjkTBoeHfMpafLHn9Gk/nZfoDQb71/U9+HmLLy9e1f7yy5Stg4FhGEQm0\nOYeVv3zCivsYWAZL957FgXOC/JsUZNFy8vGiy7MHvs7d/CnT5nwQgLUDjwBstwzwysCjbBp6GRvY\n9pi9AhwQ82h8D3g+xaOxElhgjPlRuDyZwN/xLmPMK4mx6jFTvEJ7nVWBE49ZKfmrnccsSe0OdSld\nUctD2eo764g3HjNjzCbgTuAyEZkgIrOBE4BbUoYvAz4rIpNEZCxwHkECtF+PVBTLaK+z5uFL/qrl\njlopTC2FeI/6zqrEdruM84AJwMvArcC5xpjHReRIEdkQGzcf2ExQNV4N/DFwkuVYSsFHN45vMfVC\nPN210+iFGaolXuSvsttp9JqnKy9Vx9NJnDn3mKUwcL7rCEaLszI8ZkmqFGdWhZkxZp0x5iRjzERj\nzAxjAre0MeZBY8yk2LhXjDFnGGN2NsZMNcYcZYz5pc1YFKVshsaM6BmbDcK3/KXVs95BK2fFaGrl\nTK+VqShdor3OyqB518osSi132kphaifI1XO2Hd54zBSlV7HfTkNRtlG7HbXSFbUT4intNKqmab3O\nVJjlxEc3jm8x9WI8g0uCkwKyibNenCGlG2yKs173dHXCh3ji4sxLj1laTA7F2cCy6sUZlHdoU4WZ\nolhkqzjTMzYVy2jlrLeoXeUMvKicNcF3ph4zRSmBGVvGMfnMzdrrrDDqMWtFLXfYSmFqKch73Hem\nHjNF8ZConYZWzhTblN1OQ/GLWgpxPWOzK1SY5cRHN45vMWk8Aa17nekMKd1TVJz54KGKo/G052H8\nE2eZfG8VirNW1+6sqzhTYaYoJaK9zpQy0cpZ77C03z+B1hGtnBVCPWaKUgEzL4CvXbmARTLXdSg1\nQT1meajdDlvpitoJ8h7znJ0o96nHTFF8Z3AJXDY/azsNRclH7XbUSlfUToj3YK+zblBhlhMf3Ti+\nxaTxpBP1Ojvz/m94dlKALzOkdENWceajh8on6hKPS3FWuLdaSeKslccsiYt2GkVQYaYoFXP0Mbfq\nGZtKKWjlrLeoXeUMnFfOwP/qmXrMFMUR2uusHeox64Za7rCVwtRSkDfYd9YTHrMZW8a5DkFRrNO6\nnYaidIf2OustainEtXLWkloIs8lnbuYOs8B1GICfbhzfYtJ42hOPx492Gr7NkGKLNHFWFw+VK+oa\nT5XtNKxdv9OSOMvqMUvDR3FWC2FG/yCLZC53mAXMvMB1MIpil8EleCDOlKailbPeonbVM62cbUct\nPGawzWW20NzOZfMXM7jEYVCKUgLa6yyOesxsU7sdttIVtRTkjn1ntjxn3XrMaifMAMatncHmxZNV\nnCmN5A6zgEWnXd7jJwWoMCsDFWe9hYqzYnQr0HrC/J9kZNoQ4xcMOzkpwEc3jm8xaTzt6RTPKbKY\nhbddVGEDLosjAAAgAElEQVQ7Dd9mSCmL2fPq66GqiibFU5YQt+YxS6PAoc1uPGZpuD60WUthBoE4\nm3zmZj1jU2kkp8hi7XWmlMKBx7qOQKmSWlZJe9x3VstDmaOYN5PhW8YzNGakuqAUpSJmbBnH9OE1\njEwbch1KxeihzCqo5U5bKYQe1ixGkcOaPXkocxT9g16101AUm/jRTkNpKrXcWSuFqKUI79HKmVVh\nJiJTROSHIrJRRFaJSMuvvYgcKiI/FZFXReRFEfly4Q3H2mmUjY9uHN9i0njakzee8ttp+DZDbnCW\nvyom6Q9yLc6a5OkqA5vx2Op1VqrHLEkGcWbbY5akanFmu2J2NfAGMB04A7hGRPZLDhKRacC9wDXA\nFGBvoOtXrr3OlKYSibOF5nbXoTQZp/nLJa7FmVIttaueXYHz6lmV4syax0xEJgDrgP2NMU+F990M\n/M6Y0aUsEbkceI8x5nMZ1tveY5aCttNQmswdphd6nVXrMSszf/nsMUtSux220hW1FOQ16HXmk8ds\nH2BLlNRCVgAHpIw9AlgnIktFZLWI3CUiu9sKJGqnoZUzpYmcIouDypmesWkTb/KXS2q5o1YKU0sh\n7kHlrOzqmU1hNhEYTtw3DOyYMvY9wGeBLwO7A0OA1Y9IWb3OfHTj+BaTxtMeG/HYbafh2ww5obz8\n5cGZZXE6+YOqFmdN9nTZoOx4ioizSj1maSTEWdkeszTKFGdjLK5rIzApcd8k4NWUsa8DPzTGPAwg\nIpcCvxeRHY0xKeMvBnYL/58I7AscFi5H78j2yyPThph87I+5539/nN0/tqXD6GzLT3T5/DKWn9B4\nejKeoTEj3PO/x/Kp1f8BP5nqQUTdLAP8EngeR5SWv85aCDN2BZbCTmPh4CkwZ+fgsWgHV+Xy8nUZ\nxofi7Kpwp31o+FoikWBz+cmS16/xdF6mPxDkWT9PES4+v1uXr4CBRE++SKDNOaya5UnX38fSvWcB\n8MjAK6weeh0b2PaYvQIcEPNofA94PsWjcTMwYoz5Qrg8FVgDTDHGbEiMze0x2455M1l420WcIou7\nW4+ieEgze5058ZiVkr/M8tgdHpz+n5daHu5SClHLQ9keVKSTvjNvPGbGmE3AncBlIjJBRGYDJwC3\npAy/CThJRGaKyFhgIfBgMqlZo8J2GopSNdrrrHsqy18e7ETyUsudtVIIW+00KsWDHzu2D2vabpdx\nHjABeBm4FTjXGPO4iBwpIluTljHmfmAB8G/AS8BewGmWY9kOG+00fHTj+BaTxtOeMuLprp2GbzPk\njGryl2NxVsQfVKY46zVPV15cxNNJnDn3mCUYON91BHbFmVVhZoxZZ4w5yRgz0Rgzw5hgL2GMedAY\nMykx9jpjzHuMMdOMMScaYyoxlyySuXrGptJIBpfAZfMXa6+zglSav7RypnhOLStnHpyxaYP6Xyuz\nINrrTGkyd5gFLDrtcugfdB1KQRp0rczlbQZ4cBgmL7XbYStdUUtB7viHjxxMV/mrZ4UZBOJszeTp\negF0pZHM2DKOyWdurqk46xFhFlEzgabirLdQcZaPboVZ/S9i3gUj04aYfObmXL3OfHTj+BaTxtOe\nquIZGjOSsdeZbzPUg1S4E7HhD7K5o1ZPV3t8iCcpxL3zmKXFU7MfO3F6WpgB0D+YW5wpSl2IxJme\nsVkDauY7mz2vppUUpRC1rJLWVJz19KHMUWivM6XBzLwAxi8YrlGvsx47lBmnhjuTWu60lcLUTpBX\n/KNHD2XaItbrTM/YVJpG1E5DK2c1oGaVM6jhjlrpitoJ8Zr92FFhlqBTOw0f3Ti+xaTxtMdVPK17\nnfk2Q0qZ4qwsf1BRceaDhyqOxtOZh/FLnGX6THvQTiMrKsxSiC6ArpUzpWkMLgkugL7Q3G7pAuhK\naWjlTPEcn8RZZmogztRj1gZtp6E0Gb97nfWwxyyNGuxM4tRyh60UppaCvMQfPuoxK5Ei7TQUpS6c\nIoszttNQnFOz6lktd9RKYWopxD3+saPCrBOJdho+unF8i0njaY9P8QyNGeGePx+rJwXUAUvirKoe\nVFnbafjmodJ4OpMWk0txVvgz7ak4U2GWhVCc3WEWuI5EUazz0se26BmbdaFmlTPQ6lkvoZUzO6jH\nLCcLze3a60xpJH71OlOPWVs83Jl0opY7baUwtRPkFn/0qMesYrTXmdJUWrfTULxDK2eK59ROiHvU\nTkOFWW6Wdex1VjU+eZZA4+mEz/GMaqeh+E1BcebyOodp4sw3D5XG05msMVUlzqx+pj0QZyrMCqK9\nzpQmo73OaoJWzhTPqV3lDJyLM/WYdYn2OlOazIwt45h85mYHvc7UY5YbD37p56GWO2ylMLUU5AV/\n+KjHzDHa60xpMkNjRrTXWV2oWfUsazsNpRnUUog7+rGjwiw3KQ4hx+00fPYs+YDG055O8UTiTNtp\n1IAM4sylxyyNN491HcFofPN0+RYPFI+pLHFW6mfagThTYWaL/sGtZ2wqStMYGjOivc7qQs0qZ6CV\ns15iaX8Nq2cVizP1mJXAQnM7l81fzOAS15Eoil1mXgBfu3IBi2RuyVtSj1nX1MxzBjXcYStdUTtB\nnvFHT7ceMxVmJTFu7Qw2L56s4kxpHNWIMxVmVlBxpnhO7cQZdBRoXpn/RWSKiPxQRDaKyCoRaTvl\nIjJWRJ4QkWdtxlEu2RxCVbbTqJtnqWo0nvbkjWdUr7MGnRTQyPyVsgPxzWOWjMf1jto3T5dv8YDd\nmGwI8co/0yX/4LHtMbsaeAOYDpwBXCMi+7UZ/zfAS5Zj8IZInOkZm0oTOUUWN+2MzVLy190HfcJO\ndEW5kNr5zlyLM6VaalklLVGcWTuUKSITgHXA/saYp8L7bgZ+Z8z2jngR2RO4Bzgf+I4x5r0t1lvL\nQ5mjmDeT4VvGa68zpZGU0+us2kOZZeavu0wgzE5YcV9J0eegZoc2a7nDVgpTS0Ge8qPHp0OZ+wBb\noqQWsgI4oMX4bwN/S/ALtdmE7TS0cqY0kYa00yg9fzmvnEEtK2e13FkrhailEC/hx45NYTYRGE7c\nNwzsmBwoIicBOxhj7ra4/Yoo6BAqsddZ3T1LZaPxtMdGPA1op1FJ/vJBnA141jcsiz+oSnHmm6fL\nt3ig3JiKtNNw7pu0LM7GWFzXRmBS4r5JwKvxO8JDBlcAx0d3dV71xcBu4f8TgX2Bw8LlaLdS1fIT\nxZ/fP8ii/r0Yuv90br7nVgaX2InuCYuvTuPReIqub3AJ/Pi5yXz8rnvYcuLuOdcA8EvgeRxRWv76\nn2etZOcZfwjAhJ3GsPLgWVw05ZcADIQvf044HVUsL/81zLkweBXRDm3OzuHjDpaXr8s2fvY8uCrc\nYR8a/NkqEGwuP1ny+useT5wyt7e0f1vz4U6fjwhXn2eAgTNgaE+sYNtj9gpwQMyj8T3g+bhHQ0QO\nAn4BrCVIauOAycDLwBHGmGcT662/xywFbaehNBU77TSceMxKyV+RxyyJes6KUcvDXUph6ngoW/o9\n6mMmIrcBBvgicAiBOfYjxpjHY2PeAbwr9rTZwFXh+N+bREBNFWag4kxpNneYBSw67fKCJwVU38es\nrPzVSphFOBdoKs4Uz6mbOOtWmNlul3EeMIHg1+OtwLnGmMdF5EgR2QBgjHnbGPNydCP4lfq2MWZN\nMqn5iT2HkK12Gk30LNlE42lPWfGcIotZeNtFdWqn4SR/Ve07G0i+4Y7baRTxB5W5o/bN0+VbPFB9\nTJ2EuHOPmWWsCjNjzDpjzEnGmInGmBnGmNvD+x80xiT9G9FzftrqVPNeYGTakJ6xqTSWOvU6c5m/\nfDgpoI5nbCq9Qy9VSfWSTL6gvc6UBjNjyzimD69hZNpQxmc055JMnQ5lxnF+WBP00KbiNXUQ5L4d\nylSKUmI7DUVxTQPaaVSCVs6KUYedtWKHXhDhKsxyU6JDqH+QRTI3tzjrFc9SUTSe9lQVz+ASVJxl\noGxxtp3HLI0KxZktf5Atceabp8u3eMB9TMleZ+oxU0onEmdVXABdUaokEmcLA/uW0gKtnBVDK2e9\nRVOrZ+ox8xhtp6E0mTtMu15nvekxS8O570w9Z4rn+CbI1WPWYKJ2Glo5U5rIKbI4qJzV4IxNlziv\nnmnlTPGcpglxFWa5qdYhlKXXWa96lrKi8bTHZTx1aqfhEpviLJPHLEmJvc7K8gcVFWeu/VNJfIsH\n/IvpYZolzlSY1QDtdaY0maExIyrOMuC8cga1q57NnqfVs16iKeJMPWZ1Yt5MFt52EafIYteRKIp1\nRvc6U49ZK5x7zkB9Z4rXuBbj6jHrJQq201CUOqC9zrKhlbNiuN5ZK9WRbKdRN1SY5ca9QyjZTsN9\nRKPReNqj8bQmaqehtKcbcVbIY5aGJXFWZQ+qLOLMR/+Ub/gWU6t46irOVJjVlEUyV8/YVBqJtofJ\nxt0HfcJ99UwrZ4rn1FGcqces5mivM6WJHER3Hg1fKMNjloZz35l6zhTPqVKQq8esx8nSTkNRlGbj\nReWsZtUzrZz1FnUS4irMcuOTIydgZNr3vWqn4dsMaTzt8S0epRhZxZk1j1kaBcSZy+scpomzuvin\nXOJbTFnjqYs4U2HWFPoHvRJniqJUj/PKGdSycqbVs96hDuJMPWZNQ3udKQ2gSR6z480POJfrKt2u\nc88ZqO9M8Z6yBLl6zJTRxHqd6RmbiuIH13JOpdvTylkxtHLWW/gqxFWY5cZHR872MS2SuXztSjfi\nzLcZ0nja41s8TcUXcVaqxyxJBnHm0mOWxpvHuo5gNL75ucC/mLqJx0dxpsKswWivM0XxCxfizHn1\nTCtniuf4Js7UY9YDjFs7gzWTpzM0ZsR1KIqSiaZ5zJJU7TkD9Z0VwbcdtlIutgS5esyUjoxMG9Iz\nNhXFI67lHG8ObVZKzapnWjnrLXwR4laFmYhMEZEfishGEVklIqkfaxGZLyIrRWSDiDwlIvNtxlEu\nPjpyMsRUYTsN32ZI42mPb/G4wkX+ciHOKvWYpZEQZ755zJLxuG6n4ZufC/yLyWY8Pogz2xWzq4E3\ngOnAGcA1IrJfi7FnAjsBxwNfEpHPWI5FSRKKszvMAteRKIqPOMlfVYuzpXvPqnR7qdSscgZaPesl\nXIszax4zEZkArAP2N8Y8Fd53M/A7Y9orARH5FoAx5q9THlOPWQksNLdrrzPFW6r2mJWZv9I8Zmlo\nr7N64HqnrVRLEUHuk8dsH2BLlNRCVgAHZHjuR4FHLcaidEB7nSnKKJznL/Wc1QOtnPUWLoS4TWE2\nERhO3DcM7NjuSSJyKSDATRZjKRHXBo00isVUVjsN32ZI42mPb/E4wov8VYU4Wznwytb/fWinMeBZ\n37AsnrcqxZlvfi7wL6ay46lanI2xuK6NwKTEfZOAV1s9QUS+RODlONIY82brVV8M7Bb+PxHYFzgs\nXI52K1UtP1Hx9rIsP1H4+SPTvs/Yu+7hTT7F4BLX0fg2OxpPVcsAvwSexxml5a8VZ13FhBnvBmDM\nThOYdPCeTJvzQQDWDjwCMGr5cmZz0ZylwDYRdeCcqdaWn16+YbvHmfMJTlhx39YTA+aEb1AVy8t/\nDXMuBK7YJorm7Bw+7mB5+bps42fPg6vCHfahwZ+tAsHm8pMlr7/IMh0eb2I8S/u3NR9Ofh4ABl6G\noY1YwbbH7BXggJhH43vA82keDRE5G+gDPmqMeabNetVjVgHa60zxCUces1LyV1aPWRrqO/Mf9Zz1\nFlmqpd54zIwxm4A7gctEZIKIzAZOAG5JjhWR04HLgY+3S2pKdWivM6WX8TV/qe/Mf1y301CqpQoh\nbrtdxnnABOBl4FbgXGPM4yJypIhsiI1bBEwFlonIq2E/oKstx1ISPjpyLMVkqZ2GbzOk8bTHt3gc\n4mX+si3O4h6zNKoWZ6l91RyKs6J91coSZ775ucC/mKqOp2xxZlWYGWPWGWNOMsZMNMbMMMbcHt7/\noDFmUmzcXsaY8caYScaYHcO/f2UzFqUg/YNbz9hUlF7C5/yllbN6oJWz3mFpf3kCTa+VqbRkobmd\ny+YvZnCJ60iUXqPp18osinrO6oH6znqLpCD3xmOmNI+y2mkoilIMF5Uz59UzrZwpnmNbiKswy42P\njpzyYhqZNpRbnPk2QxpPe3yLR2lPt+Ksk8csjTLFWaZrd1Yozmxdu9OWOPPNzwX+xeRDPDbFmQoz\npSORONMzNhXFD6qunIEHvrMLqV31TCtnvYUtcaYeMyU782YyfMt47XWmlI56zLKjvjP/Uc9Zb3Ek\n6jFTqiJsp6GVM0XxBz1j03+015mSBxVmufHRkVNhTBl6nfk2QxpPe3yLR8lPHnFWxGOWxKY4y+Qx\nS6MkcWbLY5ZGEXHmg38qiW8x+RZPt6gwU/IT63WmZ2wqSnvufeDkSrajlbN6oJUzpRPqMVO6Ytza\nGWxePFl7nSlWaZLHjJ8GOfb4o+6sZJvqOasH6jtrLuoxU5xSpJ2GovQiTa6cOa+eaeVMaRAqzHLj\noyPHbUzJdhq+zZDG0x7f4mkyPogzGx6zNIqKs8IesySW2mmU6TFLkkWc+eif8i0m3+LpFhVmihVG\npg3pGZuKkoF7Hzi5EoF2Leeo76wGaOVMSaIeM8Uu2utMsUATPWZpqO+sRNR3pjhCPWaKX2Rop6Eo\nSoAPhzbLQCtnxdDqmQIqzArgoyPHs5j6b9jaTsMHPJsdjUcZRdXirCyPWZKs4syaxyyNAuKsSo9Z\nGklx5qN/yreYfIunW1SYKaWhvc4UJRtaOSsRrZwpNUM9ZkrpaK8zJS+94jFL0lTPGXjgO1PPmVIR\n3XrMVJhF7HQpTH7G3vqG94D1l9hbX81RceYnBrhxIpy9EXxSQb0qzKC4OFt54TW8tvqFzON3pf3Y\nXXd+J1+64oBCsbRCxVl+VJy1xwC3ToTTPcph3QqzMRZjqTeTn4HP/1fncauAPTOs7yZgfZcxZWYZ\ncFhVG8vA9vGMTBti/Nph1lwxvfIzNv2fHXc8OB5W7A9LV8CRm11Ho8C2w5p5Bdprq19g3Z6PdR4Y\n5rB1WcZZ5u6DPrGdOBtYBnOq+kJEhzXbCLSB1TBn50qiycSbx8LYn7iOYjQPA4e6DiLkofHw2P7w\n0Ao4oiE5TD1mSmVorzO/MMDP9oITT4UH9gqWFX+oyndWNeo7y8/seeo7S8MAD+0Fnz41+NuUHKbC\nLC9ZqmWV40v9JaJNPA7aadRodirlwfHwoVPhkEPg0LmwdLzriJQkpYgzD3JYXJxVVi1L0kKc+VQt\ng9Hx+CLOfKqWzQpz2Ky5wXITUGGmVE//oFftNHqRqFo266PB8mFHadXMV7RyViI1q5yBP+LMNVG1\n7LBYDmtK1cyqMBORKSLyQxHZKCKrRKTlR0hErhCR34vIGhGpjyWzBN9F9/jWiSpbPFW106jn7JRL\nVC0TgeXLg7+9XjXzOX9ZFWce5bC7D/pEuX3MspAQZ677mCVJi8e1OPOhb1hULYvnsKZUzWxXzK4G\n3gCmA2cA14jIfslBInIOcAJwIDAT+JSI/IXlWJQasEjmMn7BsPY6q5BktSxCq2Z+56+mVs6W7j3L\nffVMK2e1Ilkti2hK1cyaMBORCcDJwMXGmNeNMUuBu4EzU4Z/FlhijHnRGPMisAQ4y1YspeKBP2N7\nfHEtReSLZ2TaUKnirN6zY594tQzg4IODv71cNatL/rIizjzLYQfOmQp4cGgzFGc+e8ySuBJnrj1m\n8WoZjM5hTaia2ayY7QNsMcY8FbtvBZDWCOcARjcnazVO6REicaZnbJZLq2pZRA9XzWqTv6qqnL3A\nrpVsJ44X4qxm1bNeq5y1qpZFNKFqZlOYTQSGE/cNAztmGDsc3uc/HvkztuHapJGkWDxltdNoxuzY\nIVktg8CfEdHDVbNa5a97Hzi5uEDLkcOu5ZzSL+OUvHana3E2sAyvxFkWz1vV4sylxyxZLYPtc1jd\nq2Y2G8xuBCYl7psEvJph7KTwvhZcDOwW/j8R2JdtB4Si3Vy3yyFR0tqzxfJLHR7fLunZiq/d8hMl\nr7/CePpvYPLqHzO85TiGxoy4jsa32elq2QA/2Bn+NCY1li+H3/5226GA5cth7KSgajb7cfhlRfFB\nsK3ncUZ5+WvxWbDLjOD/d+4E7z8YDpkTLP9qIPhbcPneb03l8IMeYNqcDwb3d8pfqwhyWKf8FS6v\nHXgEgGvnnMO5XLdVREWHH20sP718w3aPMydoRBudGBC11Khiefmvw+ULYeD88PHwcGIkkqpcXr4u\n2/jZ84LllT/ZdqgxElC2l+nweJnLP9oZLgmrZXFBFl8+7Ci45nYY93g18QH8CngRO1i7JFPo0XgF\nOCA6HCAi3wOeN2Z0XwQRWQrcaIy5IVw+G/iCMeYjKeut5pJMe5ydrfN/Vm76EDxzo7319RrzZrLw\ntos4RRa7jqQx/Gw8bFoQJK1O/OKnMPHr7q4GUPUlmUrNXzkvyVSE44+6k5+ftTBb5/+MTFm1P0d8\nd9Go+6q+xqbzSziBXsbJI34+Ht5aAIdnzGE7fN3N1QC6vSSTtUOZxphNwJ3AZSIyQURmE5y5dEvK\n8JuB80VkVxHZFTif4CJGihIQ63WmZ2x2TydvWZJe85qVmr/67MebpCrfWdmHNZO4PqwJeHVYMytN\n9J118pYlqbPXzHa7jPOACcDLwK3AucaYx0XkSBHZEA0yxlwH/AhYCQwCPzLGfMdyLOWgHrMM2Itn\nkczla1d2J86aOzvZSfOWRSQPB0DPes3Ky199ZYW8jXXrp2cb2GUOsy3Okh6zJFWLs9S+ag7FWdG+\namWKMxceszRvWUSrHFZXr5lVYWaMWWeMOckYM9EYM8MYc3t4/4PGmEmJsV81xkwzxrzLGPO3NuNQ\nmoX2OuuOvNWyiB6smpWbv/rsx+wKF5Uz59UzrZw5I2+1LKKuVTO9JFNePOsBFOC6M1YS+/F0006j\n+bPTnnbVMthm/E/So1WzcunDvUCzlMNsibPI+J+FKsRZ22t3Omin0W1ftTLEWdV9zNpVy6B9Dqtj\n1UyFmVIbymqn0WSKVssieq1qVhl9rgOwQxXtNJI4r5xB7apnda6cFa2WRdSxaqbCLC/qMctAifH0\nD+YWZz00O9vRqVoG6f6MCK2alUifo+2WkMO6EWedPGZplCnOMl+7syJxZuvanbPn2RNoVXrMOlXL\noHMOq1vVTIWZUj9CcXbH6C4GSoJuq2URWjUrkT7XAdhDK2f1oE7Vs26rZRF1q5qpMMuLeswyUEE8\nsXYaHkSTi6riyVItg9b+jAitmpVMX8XbKzGHFRFneTxmScoQZ209ZmmULM7KuHZnt+KsKo9ZlmoZ\nZMthdaqaWWswWxaVNZjd6VKY/Iy99Q3vAesvsbc+pSULze1cNn8xg0tcR+IXN06FV3cHG11aDbDj\nc3B2/qNOhai6wWxZiIjhmIw5tq/LjV3/F7DhN22HTNlpTebVvXPnXTnwir8sFIo2oq0HvjeivXUq\nbLKYwyY8B6dXkMO6bTCrwiw3y/CvBuNbTNXHM27tDDYvnpwqznR22uNbPNCjwgwqqZ4d/vZl2y7h\nVCJZxdnKgVe6qprFsSHQBpYVqJpFlCDOBlaXUzWLKCLOHqb6MzPb4Vs83nT+VxSXRO00tNeZUmv6\nKF2c/WJFhuvZWKBqzxl44DtTz5liAa2YKY1i3NoZrJk8naExI65DUbqgZytmcfqshpLK8UfdWf5G\n0EObdcD3w5p1QitmihJDe50pjaGv/E3oNTZLpGbVM5vtNJTuUGGWG9+6YoF/MTmOJ9FOQ2enPb7F\no8ToK2GdvxoYtehanBXpY5aFouIscx+zLFgQZ7b6mGUlizhzca3MdvgWT7eoMFOaSY52GoriNX3l\nb8K1OCsLrZwVQytnblGPmdJ4tJ1G/VCPWQp9dlbTDvWclUjNPGegvrOiaLsMRclAu3Yain+oMGtB\nn71VtaKp4gw8EGgqznoCNf9Xjo+OHN9i8i8en9pp+Dc7Sm3os7COhMcsSdWHNcvymKWR5dCmVY9Z\nkgKHNav2mCVJO6zpm6fLt3i6RYWZ0jNE4kzP2FRqTR+lV86a6jkDD3xnF1I735l6zqpFD2Uqvce8\nmQzfMl57nXmMHsrMSF95q45o6qFN54c1oXaHNvWwZjb0UKai5CVsp6GVM6X29JW/iaZWz5xXzqCW\nlTOtnpWPCrPc+OjI8S2mGsST6HXmOBqn+BZP47j/oXLX35dzfAePWRplirO1A49s/d8HcVaqxyyN\nDuLMtccsjTePdR3BaNRjpihNIdbrzIeTApQG45s4K4BWzkqkZpUz0MpZmajHTFHQXme+0SiPGT/f\ndscxHy53g33lrh7Uc1YqNfOcgfrO0tA+ZopiCe115g+NFWag4iwH2uusHqg4G40X5n8RmSIiPxSR\njSKySkRaFjlFZL6IrBSRDSLylIjMtxFDdfjoyPEtpnrGU1U7jXrOTrOpNIdVcVizr83jBTxmSWwe\n1ox7zJK4aKdx+bpZlW9zFIl2Gj56zJIxuT6sqR6zdK4G3gCmA2cA14jIfm3GnwnsBBwPfElEPmMp\njgp4wnUAKfgWU33jGZk2VPoZm/WdnUZTbQ4rW5xBa3H25HIrq7/3gZOtCLQNy1e1ffxazqlUoD29\nfINXvrPl69yGkUZaTC7F2ZPuNl0KXQszEZkAnAxcbIx53RizFLibIHFthzHmSmPMcmPM28aY3wB3\nAbO7jaM6NroOIAXfYqp5PCW306j57DQOZzns/ofcnBTw2nqrm+hWnG1ZvynTuKrE2ab1WwB/TgpY\n/6brILanVUyu2mk0LYfZqJjtA2wxxjwVu28FcEDG538UeNRCHIpiD4ftNJTKcZvD9IzNzPTkGZs1\nKltEuD60WXdsCLOJwHDivmFgx05PFJFLAQFushBHRTzvOoAUfIupIfHE2ml4EE1p+BaPA9znsCrF\n2YtDpWyiqDjbNPRyrvFli7PVQ6+PWnYtzoZewLt2GkMZSlRVirMXq9tUNRhj2t6A+4G3gbdSbg8A\nB91a12AAAAV4SURBVAOvJZ5zPnBXh/V+CXgK2KXDOKM3vemt926dclPWGw5zmOs51Jve9Obm1k3O\nGkMHjDHHtHs89GfsICLvix0KOIg2pX0RORv4G+Cjxpi2YrcJp8wriuIOlzlM85eiKHmx0sdMRG4j\nUIlfBA4B7gE+Yox5PGXs6cCVwBxjzK+73riiKEqXaA5TFMUXbLXLOA+YALwM3AqcGyU0ETlSRDbE\nxi4CpgLLROTVsBfQ1ZbiUBRFKYLmMEVRvMD7zv+KoiiKoii9gncXMffhKgI5Y7hCRH4vImtEpJSL\naWSNp6qrKuSZn3D8WBF5QkSedR2PiBwqIj8NKx0visiXXcUjIuNE5FoReSn8DN0lIruUEM95IrJM\nRN4QkRs7jP1KOC/rROR6ERlrO54mo/mreDxV5a88McXGaw5LH6c5rAxsnflk8Qyq/vD2hwQdXNYD\n+7UYO5/gjKp3EPQiGgI+U1UMwDnA48Au4e1R4C9czUlZ89HNexSOvwgYAJ51+ZkBpgGrgVOBMcA7\ngQ84jOdvgF8B7wLGATcDd5QQz6eBE4B/BG5sM+44gjPP9wUmE5zNuLiM96ypN81fXcVTSf7K+z6F\n4zWHaQ6r7OY8gMSkTgA2A++L3Xdz1okFvgV8q6oYgKXAF2LLZwP/15c5sTEf3cYD7Bkm/OPKSGo5\n36/Lge/ZjqGLeK4G/j62/Eng8RJjW9Qhqd0K/F1s+VjgxTLnq0k3zV9256SM/FUkJs1hmsOqvvl2\nKNOHqwjkieGA8LFO46qKJ0kZV1XIG8+3gb8luA5hGeSJ5whgnYgsFZHVYdl9d4fx3AAcKSK7SNCy\n4XTg3yzHk4e0z/O7RWSKo3jqhuav7uJJUtZVYTSH2YtHc1gJ+CbM3HfgzhdDcuxweJ9NCs2Jxfko\nHI+InATsYIy523IMheIB3gN8FvgysDvBoZJ+h/H8BniWoPn+eoLy+yLL8eQh7fMsZPj+KYDmr27j\n2UqJ+StXTJrDOsajOawEKhVmInK/iLwtIm+l3B4guBbp5MTTJgGvdljvl4AzgE8aY7q95OvGcJtZ\nYkiOnYT966nmiQewPh+F4gl/PV1BkEAg+HKUQZ75eR34oTHmYWPMCHAp8BERsfmlzRPPtcB4YAqB\nV+SHwL9bjCUvaZ9nQ4fvX6+g+av0eIDS81fmmDSHZYpHc1gJVCrMjDHHGGPeYYzZIeV2FIH63kFE\n3hd7WtYO3MeaDlcRyMhvgDEZY3g0fCzi4HaxVhBPGfNRNJ73A3sAPxORF4EfALuKyAsi8l4H8QAM\nEnxJ4xjsJtw88cwEvmuMGQ53QFcBh4vIVIvx5CHt87zaGLPOUTxeofmr9HiqyF95YtIc1jkezWFl\n4NrklrwBtxEY+CYQnA2yjtZnNZ1OcAaG1bNSssZAcFbTo8Cu4e0R4Iuu5qSs+SgSD4Hof3fsdhLw\nO2A6Yf88B/NzDLCWIJmMBf4B+KnD9+tG4PsEv+rGAguA50qIZwfgD4DFBCbe8QSHZ5LjjgNeAPYj\n+AX8f4DLy/wsNe2m+aureCrJX1lj0hymOczVzXkAKRM7haAcupHg+Pnc2GNHAhtiy08TnD2ygaBU\nuQG4uqwYktsP7/v78Ivye+DrVc5JVfPRzfzEnnM05Z1qnuf9OidMrmuBu4DdHL5fU4F/Jjj9/RWC\nC2rPKiGeS9j+It5fI/CovAq8Jzb2vwMvEfhFrgfGlvGeNfWm+at4PFXlr7xzFHuO5jDNYZXctPO/\noiiKoiiKJ/h2VqaiKIqiKErPosJMURRFURTFE1SYKYqiKIqieIIKM0VRFEVRFE9QYaYoiqIoiuIJ\nKswURVEURVE8QYWZoiiKoiiKJ6gwUxRFURRF8QQVZoqiKIqiKJ7w/wOiAZBmyw4BkAAAAABJRU5E\nrkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f67fb42bc50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x1s = np.linspace(-0.2, 1.2, 100)\n",
    "x2s = np.linspace(-0.2, 1.2, 100)\n",
    "x1, x2 = np.meshgrid(x1s, x2s)\n",
    "\n",
    "z1 = mlp_xor(x1, x2, activation=heaviside)\n",
    "z2 = mlp_xor(x1, x2, activation=sigmoid)\n",
    "\n",
    "plt.figure(figsize=(10,4))\n",
    "\n",
    "plt.subplot(121)\n",
    "plt.contourf(x1, x2, z1)\n",
    "plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
    "plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
    "plt.title(\"Activation function: heaviside\", fontsize=14)\n",
    "plt.grid(True)\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.contourf(x1, x2, z2)\n",
    "plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
    "plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
    "plt.title(\"Activation function: sigmoid\", fontsize=14)\n",
    "plt.grid(True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FNN for MNIST"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## using tf.learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = mnist.train.images\n",
    "X_test = mnist.test.images\n",
    "y_train = mnist.train.labels.astype(\"int\")\n",
    "y_test = mnist.test.labels.astype(\"int\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using config: {'_master': '', '_num_ps_replicas': 0, '_task_type': None, '_save_checkpoints_steps': None, '_tf_config': gpu_options {\n",
      "  per_process_gpu_memory_fraction: 1.0\n",
      "}\n",
      ", '_evaluation_master': '', '_save_summary_steps': 100, '_save_checkpoints_secs': 600, '_num_worker_replicas': 0, '_model_dir': None, '_environment': 'local', '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f67f1f57d30>, '_is_chief': True, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_tf_random_seed': 42, '_task_id': 0}\n",
      "WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpecewk7kb\n",
      "WARNING:tensorflow:From /home/ageron/dev/py/envs/ml/lib/python3.5/site-packages/tensorflow/contrib/learn/python/learn/estimators/head.py:615: scalar_summary (from tensorflow.python.ops.logging_ops) is deprecated and will be removed after 2016-11-30.\n",
      "Instructions for updating:\n",
      "Please switch to tf.summary.scalar. Note that tf.summary.scalar uses the node name instead of the tag. This means that TensorFlow will automatically de-duplicate summary names based on the scope they are created in. Also, passing a tensor or list of tags to a scalar summary op is no longer supported.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /tmp/tmpecewk7kb/model.ckpt.\n",
      "INFO:tensorflow:loss = 2.36404, step = 1\n",
      "INFO:tensorflow:global_step/sec: 260.817\n",
      "INFO:tensorflow:loss = 0.311432, step = 101 (0.383 sec)\n",
      "INFO:tensorflow:global_step/sec: 260.391\n",
      "INFO:tensorflow:loss = 0.265409, step = 201 (0.384 sec)\n",
      "INFO:tensorflow:global_step/sec: 262.476\n",
      "INFO:tensorflow:loss = 0.408733, step = 301 (0.381 sec)\n",
      "INFO:tensorflow:global_step/sec: 261.314\n",
      "INFO:tensorflow:loss = 0.244357, step = 401 (0.383 sec)\n",
      "INFO:tensorflow:global_step/sec: 258.889\n",
      "INFO:tensorflow:loss = 0.238858, step = 501 (0.386 sec)\n",
      "INFO:tensorflow:global_step/sec: 258.784\n",
      "INFO:tensorflow:loss = 0.0918271, step = 601 (0.386 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.338\n",
      "INFO:tensorflow:loss = 0.123374, step = 701 (0.609 sec)\n",
      "INFO:tensorflow:global_step/sec: 184.472\n",
      "INFO:tensorflow:loss = 0.196473, step = 801 (0.542 sec)\n",
      "INFO:tensorflow:global_step/sec: 187.728\n",
      "INFO:tensorflow:loss = 0.0932024, step = 901 (0.533 sec)\n",
      "INFO:tensorflow:global_step/sec: 189.618\n",
      "INFO:tensorflow:loss = 0.196834, step = 1001 (0.527 sec)\n",
      "INFO:tensorflow:global_step/sec: 264.9\n",
      "INFO:tensorflow:loss = 0.194408, step = 1101 (0.377 sec)\n",
      "INFO:tensorflow:global_step/sec: 262.289\n",
      "INFO:tensorflow:loss = 0.152817, step = 1201 (0.381 sec)\n",
      "INFO:tensorflow:global_step/sec: 262.723\n",
      "INFO:tensorflow:loss = 0.149922, step = 1301 (0.381 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.842\n",
      "INFO:tensorflow:loss = 0.0659786, step = 1401 (0.607 sec)\n",
      "INFO:tensorflow:global_step/sec: 228.232\n",
      "INFO:tensorflow:loss = 0.0721019, step = 1501 (0.438 sec)\n",
      "INFO:tensorflow:global_step/sec: 202.979\n",
      "INFO:tensorflow:loss = 0.120699, step = 1601 (0.493 sec)\n",
      "INFO:tensorflow:global_step/sec: 228.385\n",
      "INFO:tensorflow:loss = 0.0398833, step = 1701 (0.437 sec)\n",
      "INFO:tensorflow:global_step/sec: 262.751\n",
      "INFO:tensorflow:loss = 0.151045, step = 1801 (0.381 sec)\n",
      "INFO:tensorflow:global_step/sec: 259.859\n",
      "INFO:tensorflow:loss = 0.0709309, step = 1901 (0.385 sec)\n",
      "INFO:tensorflow:global_step/sec: 260.716\n",
      "INFO:tensorflow:loss = 0.0622928, step = 2001 (0.384 sec)\n",
      "INFO:tensorflow:global_step/sec: 259.09\n",
      "INFO:tensorflow:loss = 0.0229075, step = 2101 (0.386 sec)\n",
      "INFO:tensorflow:global_step/sec: 259.745\n",
      "INFO:tensorflow:loss = 0.0303926, step = 2201 (0.385 sec)\n",
      "INFO:tensorflow:global_step/sec: 260.825\n",
      "INFO:tensorflow:loss = 0.0458983, step = 2301 (0.383 sec)\n",
      "INFO:tensorflow:global_step/sec: 259.816\n",
      "INFO:tensorflow:loss = 0.0586828, step = 2401 (0.385 sec)\n",
      "INFO:tensorflow:global_step/sec: 243.524\n",
      "INFO:tensorflow:loss = 0.0920379, step = 2501 (0.411 sec)\n",
      "INFO:tensorflow:global_step/sec: 224.828\n",
      "INFO:tensorflow:loss = 0.0315541, step = 2601 (0.445 sec)\n",
      "INFO:tensorflow:global_step/sec: 233.629\n",
      "INFO:tensorflow:loss = 0.0114352, step = 2701 (0.428 sec)\n",
      "INFO:tensorflow:global_step/sec: 185.194\n",
      "INFO:tensorflow:loss = 0.0563824, step = 2801 (0.540 sec)\n",
      "INFO:tensorflow:global_step/sec: 178.371\n",
      "INFO:tensorflow:loss = 0.0811181, step = 2901 (0.561 sec)\n",
      "INFO:tensorflow:global_step/sec: 173.41\n",
      "INFO:tensorflow:loss = 0.0140813, step = 3001 (0.576 sec)\n",
      "INFO:tensorflow:global_step/sec: 194.442\n",
      "INFO:tensorflow:loss = 0.0322334, step = 3101 (0.514 sec)\n",
      "INFO:tensorflow:global_step/sec: 186.116\n",
      "INFO:tensorflow:loss = 0.0132405, step = 3201 (0.538 sec)\n",
      "INFO:tensorflow:global_step/sec: 192.012\n",
      "INFO:tensorflow:loss = 0.0339669, step = 3301 (0.520 sec)\n",
      "INFO:tensorflow:global_step/sec: 182.959\n",
      "INFO:tensorflow:loss = 0.157921, step = 3401 (0.547 sec)\n",
      "INFO:tensorflow:global_step/sec: 182.487\n",
      "INFO:tensorflow:loss = 0.0848148, step = 3501 (0.548 sec)\n",
      "INFO:tensorflow:global_step/sec: 142.784\n",
      "INFO:tensorflow:loss = 0.155988, step = 3601 (0.700 sec)\n",
      "INFO:tensorflow:global_step/sec: 184.358\n",
      "INFO:tensorflow:loss = 0.0346186, step = 3701 (0.542 sec)\n",
      "INFO:tensorflow:global_step/sec: 169.954\n",
      "INFO:tensorflow:loss = 0.00991458, step = 3801 (0.589 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.217\n",
      "INFO:tensorflow:loss = 0.150861, step = 3901 (0.623 sec)\n",
      "INFO:tensorflow:global_step/sec: 211.631\n",
      "INFO:tensorflow:loss = 0.110915, step = 4001 (0.473 sec)\n",
      "INFO:tensorflow:global_step/sec: 177.961\n",
      "INFO:tensorflow:loss = 0.0493516, step = 4101 (0.562 sec)\n",
      "INFO:tensorflow:global_step/sec: 178.605\n",
      "INFO:tensorflow:loss = 0.0546287, step = 4201 (0.560 sec)\n",
      "INFO:tensorflow:global_step/sec: 183.171\n",
      "INFO:tensorflow:loss = 0.159896, step = 4301 (0.546 sec)\n",
      "INFO:tensorflow:global_step/sec: 207.876\n",
      "INFO:tensorflow:loss = 0.112916, step = 4401 (0.481 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.187\n",
      "INFO:tensorflow:loss = 0.0161132, step = 4501 (0.632 sec)\n",
      "INFO:tensorflow:global_step/sec: 241.573\n",
      "INFO:tensorflow:loss = 0.0171333, step = 4601 (0.414 sec)\n",
      "INFO:tensorflow:global_step/sec: 233.973\n",
      "INFO:tensorflow:loss = 0.00835642, step = 4701 (0.427 sec)\n",
      "INFO:tensorflow:global_step/sec: 236.421\n",
      "INFO:tensorflow:loss = 0.01708, step = 4801 (0.423 sec)\n",
      "INFO:tensorflow:global_step/sec: 237.868\n",
      "INFO:tensorflow:loss = 0.0864718, step = 4901 (0.420 sec)\n",
      "INFO:tensorflow:global_step/sec: 239.187\n",
      "INFO:tensorflow:loss = 0.0439425, step = 5001 (0.418 sec)\n",
      "INFO:tensorflow:global_step/sec: 238.716\n",
      "INFO:tensorflow:loss = 0.00764107, step = 5101 (0.419 sec)\n",
      "INFO:tensorflow:global_step/sec: 237.782\n",
      "INFO:tensorflow:loss = 0.0232963, step = 5201 (0.421 sec)\n",
      "INFO:tensorflow:global_step/sec: 238.707\n",
      "INFO:tensorflow:loss = 0.046633, step = 5301 (0.419 sec)\n",
      "INFO:tensorflow:global_step/sec: 239.04\n",
      "INFO:tensorflow:loss = 0.066787, step = 5401 (0.418 sec)\n",
      "INFO:tensorflow:global_step/sec: 252.49\n",
      "INFO:tensorflow:loss = 0.0494149, step = 5501 (0.396 sec)\n",
      "INFO:tensorflow:global_step/sec: 258.641\n",
      "INFO:tensorflow:loss = 0.0707151, step = 5601 (0.387 sec)\n",
      "INFO:tensorflow:global_step/sec: 241.49\n",
      "INFO:tensorflow:loss = 0.0192079, step = 5701 (0.414 sec)\n",
      "INFO:tensorflow:global_step/sec: 229.017\n",
      "INFO:tensorflow:loss = 0.00933775, step = 5801 (0.437 sec)\n",
      "INFO:tensorflow:global_step/sec: 235.112\n",
      "INFO:tensorflow:loss = 0.106693, step = 5901 (0.425 sec)\n",
      "INFO:tensorflow:global_step/sec: 233.677\n",
      "INFO:tensorflow:loss = 0.0908673, step = 6001 (0.428 sec)\n",
      "INFO:tensorflow:global_step/sec: 236.366\n",
      "INFO:tensorflow:loss = 0.01711, step = 6101 (0.423 sec)\n",
      "INFO:tensorflow:global_step/sec: 186.503\n",
      "INFO:tensorflow:loss = 0.0224653, step = 6201 (0.536 sec)\n",
      "INFO:tensorflow:global_step/sec: 167.806\n",
      "INFO:tensorflow:loss = 0.0684235, step = 6301 (0.596 sec)\n",
      "INFO:tensorflow:global_step/sec: 210.251\n",
      "INFO:tensorflow:loss = 0.0247326, step = 6401 (0.476 sec)\n",
      "INFO:tensorflow:global_step/sec: 197.723\n",
      "INFO:tensorflow:loss = 0.00986267, step = 6501 (0.506 sec)\n",
      "INFO:tensorflow:global_step/sec: 266.76\n",
      "INFO:tensorflow:loss = 0.0220192, step = 6601 (0.375 sec)\n",
      "INFO:tensorflow:global_step/sec: 264.179\n",
      "INFO:tensorflow:loss = 0.0224969, step = 6701 (0.379 sec)\n",
      "INFO:tensorflow:global_step/sec: 257.535\n",
      "INFO:tensorflow:loss = 0.0130047, step = 6801 (0.388 sec)\n",
      "INFO:tensorflow:global_step/sec: 260.336\n",
      "INFO:tensorflow:loss = 0.0126607, step = 6901 (0.384 sec)\n",
      "INFO:tensorflow:global_step/sec: 268.145\n",
      "INFO:tensorflow:loss = 0.015101, step = 7001 (0.373 sec)\n",
      "INFO:tensorflow:global_step/sec: 251.59\n",
      "INFO:tensorflow:loss = 0.00366965, step = 7101 (0.397 sec)\n",
      "INFO:tensorflow:global_step/sec: 237.158\n",
      "INFO:tensorflow:loss = 0.044044, step = 7201 (0.422 sec)\n",
      "INFO:tensorflow:global_step/sec: 232.033\n",
      "INFO:tensorflow:loss = 0.00571019, step = 7301 (0.431 sec)\n",
      "INFO:tensorflow:global_step/sec: 236.205\n",
      "INFO:tensorflow:loss = 0.013417, step = 7401 (0.423 sec)\n",
      "INFO:tensorflow:global_step/sec: 225.404\n",
      "INFO:tensorflow:loss = 0.0043693, step = 7501 (0.444 sec)\n",
      "INFO:tensorflow:global_step/sec: 238.452\n",
      "INFO:tensorflow:loss = 0.0141966, step = 7601 (0.419 sec)\n",
      "INFO:tensorflow:global_step/sec: 240.771\n",
      "INFO:tensorflow:loss = 0.00819222, step = 7701 (0.415 sec)\n",
      "INFO:tensorflow:global_step/sec: 229.499\n",
      "INFO:tensorflow:loss = 0.00547122, step = 7801 (0.436 sec)\n",
      "INFO:tensorflow:global_step/sec: 222.969\n",
      "INFO:tensorflow:loss = 0.012388, step = 7901 (0.448 sec)\n",
      "INFO:tensorflow:global_step/sec: 234.443\n",
      "INFO:tensorflow:loss = 0.0043136, step = 8001 (0.427 sec)\n",
      "INFO:tensorflow:global_step/sec: 181.467\n",
      "INFO:tensorflow:loss = 0.0393611, step = 8101 (0.552 sec)\n",
      "INFO:tensorflow:global_step/sec: 178.792\n",
      "INFO:tensorflow:loss = 0.0268929, step = 8201 (0.559 sec)\n",
      "INFO:tensorflow:global_step/sec: 185.438\n",
      "INFO:tensorflow:loss = 0.0534401, step = 8301 (0.539 sec)\n",
      "INFO:tensorflow:global_step/sec: 151.617\n",
      "INFO:tensorflow:loss = 0.017818, step = 8401 (0.660 sec)\n",
      "INFO:tensorflow:global_step/sec: 155.867\n",
      "INFO:tensorflow:loss = 0.0116427, step = 8501 (0.642 sec)\n",
      "INFO:tensorflow:global_step/sec: 181.382\n",
      "INFO:tensorflow:loss = 0.00533878, step = 8601 (0.551 sec)\n",
      "INFO:tensorflow:global_step/sec: 187.288\n",
      "INFO:tensorflow:loss = 0.00569211, step = 8701 (0.534 sec)\n",
      "INFO:tensorflow:global_step/sec: 205.664\n",
      "INFO:tensorflow:loss = 0.00665956, step = 8801 (0.486 sec)\n",
      "INFO:tensorflow:global_step/sec: 204.953\n",
      "INFO:tensorflow:loss = 0.00315022, step = 8901 (0.488 sec)\n",
      "INFO:tensorflow:global_step/sec: 194.252\n",
      "INFO:tensorflow:loss = 0.0133965, step = 9001 (0.515 sec)\n",
      "INFO:tensorflow:global_step/sec: 188.951\n",
      "INFO:tensorflow:loss = 0.00906787, step = 9101 (0.529 sec)\n",
      "INFO:tensorflow:global_step/sec: 173.39\n",
      "INFO:tensorflow:loss = 0.00360787, step = 9201 (0.577 sec)\n",
      "INFO:tensorflow:global_step/sec: 198.042\n",
      "INFO:tensorflow:loss = 0.01566, step = 9301 (0.505 sec)\n",
      "INFO:tensorflow:global_step/sec: 220.507\n",
      "INFO:tensorflow:loss = 0.0403218, step = 9401 (0.454 sec)\n",
      "INFO:tensorflow:global_step/sec: 220.935\n",
      "INFO:tensorflow:loss = 0.00779154, step = 9501 (0.452 sec)\n",
      "INFO:tensorflow:global_step/sec: 239.463\n",
      "INFO:tensorflow:loss = 0.017734, step = 9601 (0.418 sec)\n",
      "INFO:tensorflow:global_step/sec: 238.71\n",
      "INFO:tensorflow:loss = 0.00776719, step = 9701 (0.419 sec)\n",
      "INFO:tensorflow:global_step/sec: 236.017\n",
      "INFO:tensorflow:loss = 0.00336712, step = 9801 (0.424 sec)\n",
      "INFO:tensorflow:global_step/sec: 230.414\n",
      "INFO:tensorflow:loss = 0.0176963, step = 9901 (0.434 sec)\n",
      "INFO:tensorflow:global_step/sec: 229.642\n",
      "INFO:tensorflow:loss = 0.0169004, step = 10001 (0.435 sec)\n",
      "INFO:tensorflow:global_step/sec: 218.107\n",
      "INFO:tensorflow:loss = 0.00473883, step = 10101 (0.458 sec)\n",
      "INFO:tensorflow:global_step/sec: 202.282\n",
      "INFO:tensorflow:loss = 0.00767333, step = 10201 (0.494 sec)\n",
      "INFO:tensorflow:global_step/sec: 221.607\n",
      "INFO:tensorflow:loss = 0.00469218, step = 10301 (0.451 sec)\n",
      "INFO:tensorflow:global_step/sec: 211.051\n",
      "INFO:tensorflow:loss = 0.00941437, step = 10401 (0.474 sec)\n",
      "INFO:tensorflow:global_step/sec: 213.276\n",
      "INFO:tensorflow:loss = 0.00360837, step = 10501 (0.469 sec)\n",
      "INFO:tensorflow:global_step/sec: 204.939\n",
      "INFO:tensorflow:loss = 0.00824566, step = 10601 (0.488 sec)\n",
      "INFO:tensorflow:global_step/sec: 199.485\n",
      "INFO:tensorflow:loss = 0.034342, step = 10701 (0.501 sec)\n",
      "INFO:tensorflow:global_step/sec: 220.138\n",
      "INFO:tensorflow:loss = 0.0160151, step = 10801 (0.454 sec)\n",
      "INFO:tensorflow:global_step/sec: 232.804\n",
      "INFO:tensorflow:loss = 0.00446511, step = 10901 (0.429 sec)\n",
      "INFO:tensorflow:global_step/sec: 238.091\n",
      "INFO:tensorflow:loss = 0.0278223, step = 11001 (0.420 sec)\n",
      "INFO:tensorflow:global_step/sec: 236.775\n",
      "INFO:tensorflow:loss = 0.00451434, step = 11101 (0.422 sec)\n",
      "INFO:tensorflow:global_step/sec: 213.7\n",
      "INFO:tensorflow:loss = 0.000942479, step = 11201 (0.468 sec)\n",
      "INFO:tensorflow:global_step/sec: 215.302\n",
      "INFO:tensorflow:loss = 0.0130732, step = 11301 (0.464 sec)\n",
      "INFO:tensorflow:global_step/sec: 234.367\n",
      "INFO:tensorflow:loss = 0.0126658, step = 11401 (0.427 sec)\n",
      "INFO:tensorflow:global_step/sec: 201.051\n",
      "INFO:tensorflow:loss = 0.0185032, step = 11501 (0.497 sec)\n",
      "INFO:tensorflow:global_step/sec: 211.212\n",
      "INFO:tensorflow:loss = 0.000567014, step = 11601 (0.473 sec)\n",
      "INFO:tensorflow:global_step/sec: 170.486\n",
      "INFO:tensorflow:loss = 0.00216249, step = 11701 (0.587 sec)\n",
      "INFO:tensorflow:global_step/sec: 194.085\n",
      "INFO:tensorflow:loss = 0.000577293, step = 11801 (0.515 sec)\n",
      "INFO:tensorflow:global_step/sec: 215.625\n",
      "INFO:tensorflow:loss = 0.00549651, step = 11901 (0.464 sec)\n",
      "INFO:tensorflow:global_step/sec: 191.043\n",
      "INFO:tensorflow:loss = 0.000167048, step = 12001 (0.524 sec)\n",
      "INFO:tensorflow:global_step/sec: 154.278\n",
      "INFO:tensorflow:loss = 0.00373405, step = 12101 (0.648 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.362\n",
      "INFO:tensorflow:loss = 0.00398317, step = 12201 (0.631 sec)\n",
      "INFO:tensorflow:global_step/sec: 195.115\n",
      "INFO:tensorflow:loss = 0.00490612, step = 12301 (0.513 sec)\n",
      "INFO:tensorflow:global_step/sec: 180.842\n",
      "INFO:tensorflow:loss = 0.000254854, step = 12401 (0.553 sec)\n",
      "INFO:tensorflow:global_step/sec: 239.513\n",
      "INFO:tensorflow:loss = 0.00370345, step = 12501 (0.417 sec)\n",
      "INFO:tensorflow:global_step/sec: 229.337\n",
      "INFO:tensorflow:loss = 0.00152101, step = 12601 (0.436 sec)\n",
      "INFO:tensorflow:global_step/sec: 235.719\n",
      "INFO:tensorflow:loss = 0.00526951, step = 12701 (0.424 sec)\n",
      "INFO:tensorflow:global_step/sec: 238.127\n",
      "INFO:tensorflow:loss = 0.00592783, step = 12801 (0.420 sec)\n",
      "INFO:tensorflow:global_step/sec: 224.962\n",
      "INFO:tensorflow:loss = 0.00324891, step = 12901 (0.445 sec)\n",
      "INFO:tensorflow:global_step/sec: 133.41\n",
      "INFO:tensorflow:loss = 0.00343657, step = 13001 (0.749 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.258\n",
      "INFO:tensorflow:loss = 0.00340272, step = 13101 (0.620 sec)\n",
      "INFO:tensorflow:global_step/sec: 237.132\n",
      "INFO:tensorflow:loss = 0.00556199, step = 13201 (0.422 sec)\n",
      "INFO:tensorflow:global_step/sec: 228.502\n",
      "INFO:tensorflow:loss = 0.0042778, step = 13301 (0.438 sec)\n",
      "INFO:tensorflow:global_step/sec: 232.921\n",
      "INFO:tensorflow:loss = 0.00246822, step = 13401 (0.429 sec)\n",
      "INFO:tensorflow:global_step/sec: 238.2\n",
      "INFO:tensorflow:loss = 0.00610874, step = 13501 (0.420 sec)\n",
      "INFO:tensorflow:global_step/sec: 203.273\n",
      "INFO:tensorflow:loss = 0.00441995, step = 13601 (0.492 sec)\n",
      "INFO:tensorflow:global_step/sec: 238.859\n",
      "INFO:tensorflow:loss = 0.0023879, step = 13701 (0.419 sec)\n",
      "INFO:tensorflow:global_step/sec: 250.859\n",
      "INFO:tensorflow:loss = 0.0074444, step = 13801 (0.399 sec)\n",
      "INFO:tensorflow:global_step/sec: 260.437\n",
      "INFO:tensorflow:loss = 0.00315427, step = 13901 (0.384 sec)\n",
      "INFO:tensorflow:global_step/sec: 248.155\n",
      "INFO:tensorflow:loss = 0.00219044, step = 14001 (0.403 sec)\n",
      "INFO:tensorflow:global_step/sec: 236.323\n",
      "INFO:tensorflow:loss = 0.00482443, step = 14101 (0.423 sec)\n",
      "INFO:tensorflow:global_step/sec: 247.825\n",
      "INFO:tensorflow:loss = 0.00988796, step = 14201 (0.404 sec)\n",
      "INFO:tensorflow:global_step/sec: 254.179\n",
      "INFO:tensorflow:loss = 0.00231401, step = 14301 (0.393 sec)\n",
      "INFO:tensorflow:global_step/sec: 248.802\n",
      "INFO:tensorflow:loss = 0.000743411, step = 14401 (0.402 sec)\n",
      "INFO:tensorflow:global_step/sec: 255.943\n",
      "INFO:tensorflow:loss = 0.00100101, step = 14501 (0.391 sec)\n",
      "INFO:tensorflow:global_step/sec: 231.248\n",
      "INFO:tensorflow:loss = 0.00342037, step = 14601 (0.432 sec)\n",
      "INFO:tensorflow:global_step/sec: 225.419\n",
      "INFO:tensorflow:loss = 0.00153701, step = 14701 (0.444 sec)\n",
      "INFO:tensorflow:global_step/sec: 236.466\n",
      "INFO:tensorflow:loss = 0.00118163, step = 14801 (0.423 sec)\n",
      "INFO:tensorflow:global_step/sec: 233.832\n",
      "INFO:tensorflow:loss = 0.00225318, step = 14901 (0.428 sec)\n",
      "INFO:tensorflow:global_step/sec: 227.359\n",
      "INFO:tensorflow:loss = 0.00111509, step = 15001 (0.440 sec)\n",
      "INFO:tensorflow:global_step/sec: 233.517\n",
      "INFO:tensorflow:loss = 0.00252178, step = 15101 (0.428 sec)\n",
      "INFO:tensorflow:global_step/sec: 232.674\n",
      "INFO:tensorflow:loss = 0.00225656, step = 15201 (0.430 sec)\n",
      "INFO:tensorflow:global_step/sec: 221.965\n",
      "INFO:tensorflow:loss = 0.00173806, step = 15301 (0.451 sec)\n",
      "INFO:tensorflow:global_step/sec: 228.725\n",
      "INFO:tensorflow:loss = 0.00382904, step = 15401 (0.437 sec)\n",
      "INFO:tensorflow:global_step/sec: 220.263\n",
      "INFO:tensorflow:loss = 0.00451967, step = 15501 (0.454 sec)\n",
      "INFO:tensorflow:global_step/sec: 185.685\n",
      "INFO:tensorflow:loss = 0.0047358, step = 15601 (0.538 sec)\n",
      "INFO:tensorflow:global_step/sec: 214.146\n",
      "INFO:tensorflow:loss = 0.0145779, step = 15701 (0.467 sec)\n",
      "INFO:tensorflow:global_step/sec: 211.497\n",
      "INFO:tensorflow:loss = 0.00161293, step = 15801 (0.473 sec)\n",
      "INFO:tensorflow:global_step/sec: 200.982\n",
      "INFO:tensorflow:loss = 0.000548923, step = 15901 (0.498 sec)\n",
      "INFO:tensorflow:global_step/sec: 215.662\n",
      "INFO:tensorflow:loss = 0.00568894, step = 16001 (0.464 sec)\n",
      "INFO:tensorflow:global_step/sec: 208.946\n",
      "INFO:tensorflow:loss = 0.00496605, step = 16101 (0.479 sec)\n",
      "INFO:tensorflow:global_step/sec: 219.495\n",
      "INFO:tensorflow:loss = 7.30671e-05, step = 16201 (0.456 sec)\n",
      "INFO:tensorflow:global_step/sec: 207.948\n",
      "INFO:tensorflow:loss = 0.00304547, step = 16301 (0.481 sec)\n",
      "INFO:tensorflow:global_step/sec: 204.766\n",
      "INFO:tensorflow:loss = 0.00170099, step = 16401 (0.488 sec)\n",
      "INFO:tensorflow:global_step/sec: 235.146\n",
      "INFO:tensorflow:loss = 0.00151268, step = 16501 (0.425 sec)\n",
      "INFO:tensorflow:global_step/sec: 234.786\n",
      "INFO:tensorflow:loss = 0.00499173, step = 16601 (0.426 sec)\n",
      "INFO:tensorflow:global_step/sec: 233.865\n",
      "INFO:tensorflow:loss = 0.00237819, step = 16701 (0.428 sec)\n",
      "INFO:tensorflow:global_step/sec: 224.686\n",
      "INFO:tensorflow:loss = 0.00212857, step = 16801 (0.445 sec)\n",
      "INFO:tensorflow:global_step/sec: 237.371\n",
      "INFO:tensorflow:loss = 0.0025386, step = 16901 (0.421 sec)\n",
      "INFO:tensorflow:global_step/sec: 228.135\n",
      "INFO:tensorflow:loss = 0.00303927, step = 17001 (0.438 sec)\n",
      "INFO:tensorflow:global_step/sec: 234.993\n",
      "INFO:tensorflow:loss = 0.000547433, step = 17101 (0.425 sec)\n",
      "INFO:tensorflow:global_step/sec: 222.667\n",
      "INFO:tensorflow:loss = 0.0022543, step = 17201 (0.449 sec)\n",
      "INFO:tensorflow:global_step/sec: 225.896\n",
      "INFO:tensorflow:loss = 0.000823148, step = 17301 (0.442 sec)\n",
      "INFO:tensorflow:global_step/sec: 210.336\n",
      "INFO:tensorflow:loss = 0.00171766, step = 17401 (0.475 sec)\n",
      "INFO:tensorflow:global_step/sec: 203.11\n",
      "INFO:tensorflow:loss = 0.00192135, step = 17501 (0.492 sec)\n",
      "INFO:tensorflow:global_step/sec: 212.637\n",
      "INFO:tensorflow:loss = 0.000309051, step = 17601 (0.470 sec)\n",
      "INFO:tensorflow:global_step/sec: 208.182\n",
      "INFO:tensorflow:loss = 0.000621968, step = 17701 (0.480 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.634\n",
      "INFO:tensorflow:loss = 0.000520324, step = 17801 (0.607 sec)\n",
      "INFO:tensorflow:global_step/sec: 210.295\n",
      "INFO:tensorflow:loss = 0.00209451, step = 17901 (0.475 sec)\n",
      "INFO:tensorflow:global_step/sec: 174.574\n",
      "INFO:tensorflow:loss = 0.00160342, step = 18001 (0.573 sec)\n",
      "INFO:tensorflow:global_step/sec: 132.402\n",
      "INFO:tensorflow:loss = 0.00103904, step = 18101 (0.756 sec)\n",
      "INFO:tensorflow:global_step/sec: 155.235\n",
      "INFO:tensorflow:loss = 0.00412339, step = 18201 (0.645 sec)\n",
      "INFO:tensorflow:global_step/sec: 147.223\n",
      "INFO:tensorflow:loss = 0.014485, step = 18301 (0.678 sec)\n",
      "INFO:tensorflow:global_step/sec: 147.963\n",
      "INFO:tensorflow:loss = 0.0034526, step = 18401 (0.676 sec)\n",
      "INFO:tensorflow:global_step/sec: 152.085\n",
      "INFO:tensorflow:loss = 0.00173879, step = 18501 (0.658 sec)\n",
      "INFO:tensorflow:global_step/sec: 143.207\n",
      "INFO:tensorflow:loss = 0.00422458, step = 18601 (0.698 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.083\n",
      "INFO:tensorflow:loss = 0.000858988, step = 18701 (0.609 sec)\n",
      "INFO:tensorflow:global_step/sec: 132.525\n",
      "INFO:tensorflow:loss = 0.00143733, step = 18801 (0.755 sec)\n",
      "INFO:tensorflow:global_step/sec: 176.179\n",
      "INFO:tensorflow:loss = 0.00293665, step = 18901 (0.568 sec)\n",
      "INFO:tensorflow:global_step/sec: 224.261\n",
      "INFO:tensorflow:loss = 0.001538, step = 19001 (0.446 sec)\n",
      "INFO:tensorflow:global_step/sec: 228.922\n",
      "INFO:tensorflow:loss = 0.000944429, step = 19101 (0.437 sec)\n",
      "INFO:tensorflow:global_step/sec: 231.406\n",
      "INFO:tensorflow:loss = 0.00041447, step = 19201 (0.432 sec)\n",
      "INFO:tensorflow:global_step/sec: 230.819\n",
      "INFO:tensorflow:loss = 0.00526123, step = 19301 (0.433 sec)\n",
      "INFO:tensorflow:global_step/sec: 231.187\n",
      "INFO:tensorflow:loss = 0.000478011, step = 19401 (0.433 sec)\n",
      "INFO:tensorflow:global_step/sec: 235.88\n",
      "INFO:tensorflow:loss = 0.00260038, step = 19501 (0.424 sec)\n",
      "INFO:tensorflow:global_step/sec: 230.394\n",
      "INFO:tensorflow:loss = 0.000504177, step = 19601 (0.434 sec)\n",
      "INFO:tensorflow:global_step/sec: 232.283\n",
      "INFO:tensorflow:loss = 0.000130909, step = 19701 (0.430 sec)\n",
      "INFO:tensorflow:global_step/sec: 229.826\n",
      "INFO:tensorflow:loss = 0.000504942, step = 19801 (0.435 sec)\n",
      "INFO:tensorflow:global_step/sec: 227.287\n",
      "INFO:tensorflow:loss = 0.00158035, step = 19901 (0.440 sec)\n",
      "INFO:tensorflow:global_step/sec: 233.08\n",
      "INFO:tensorflow:loss = 0.0020706, step = 20001 (0.429 sec)\n",
      "INFO:tensorflow:global_step/sec: 229.328\n",
      "INFO:tensorflow:loss = 0.000324922, step = 20101 (0.436 sec)\n",
      "INFO:tensorflow:global_step/sec: 205.141\n",
      "INFO:tensorflow:loss = 0.00199532, step = 20201 (0.487 sec)\n",
      "INFO:tensorflow:global_step/sec: 216.646\n",
      "INFO:tensorflow:loss = 0.00110691, step = 20301 (0.461 sec)\n",
      "INFO:tensorflow:global_step/sec: 234.478\n",
      "INFO:tensorflow:loss = 0.0001444, step = 20401 (0.426 sec)\n",
      "INFO:tensorflow:global_step/sec: 251.37\n",
      "INFO:tensorflow:loss = 0.000286913, step = 20501 (0.398 sec)\n",
      "INFO:tensorflow:global_step/sec: 223.94\n",
      "INFO:tensorflow:loss = 0.00139893, step = 20601 (0.447 sec)\n",
      "INFO:tensorflow:global_step/sec: 241.293\n",
      "INFO:tensorflow:loss = 0.00105777, step = 20701 (0.414 sec)\n",
      "INFO:tensorflow:global_step/sec: 262.75\n",
      "INFO:tensorflow:loss = 0.00169489, step = 20801 (0.381 sec)\n",
      "INFO:tensorflow:global_step/sec: 249.916\n",
      "INFO:tensorflow:loss = 0.00156369, step = 20901 (0.400 sec)\n",
      "INFO:tensorflow:global_step/sec: 226.122\n",
      "INFO:tensorflow:loss = 0.00274591, step = 21001 (0.443 sec)\n",
      "INFO:tensorflow:global_step/sec: 224.531\n",
      "INFO:tensorflow:loss = 0.00134583, step = 21101 (0.445 sec)\n",
      "INFO:tensorflow:global_step/sec: 257.902\n",
      "INFO:tensorflow:loss = 0.00222729, step = 21201 (0.387 sec)\n",
      "INFO:tensorflow:global_step/sec: 217.25\n",
      "INFO:tensorflow:loss = 0.000948768, step = 21301 (0.460 sec)\n",
      "INFO:tensorflow:global_step/sec: 205.733\n",
      "INFO:tensorflow:loss = 0.00258286, step = 21401 (0.488 sec)\n",
      "INFO:tensorflow:global_step/sec: 208.844\n",
      "INFO:tensorflow:loss = 0.000207113, step = 21501 (0.477 sec)\n",
      "INFO:tensorflow:global_step/sec: 147.089\n",
      "INFO:tensorflow:loss = 0.00130515, step = 21601 (0.680 sec)\n",
      "INFO:tensorflow:global_step/sec: 217.076\n",
      "INFO:tensorflow:loss = 0.000956831, step = 21701 (0.460 sec)\n",
      "INFO:tensorflow:global_step/sec: 231.529\n",
      "INFO:tensorflow:loss = 0.000395721, step = 21801 (0.432 sec)\n",
      "INFO:tensorflow:global_step/sec: 147.527\n",
      "INFO:tensorflow:loss = 0.000597151, step = 21901 (0.678 sec)\n",
      "INFO:tensorflow:global_step/sec: 227.008\n",
      "INFO:tensorflow:loss = 0.000139627, step = 22001 (0.440 sec)\n",
      "INFO:tensorflow:global_step/sec: 152.091\n",
      "INFO:tensorflow:loss = 0.000522552, step = 22101 (0.658 sec)\n",
      "INFO:tensorflow:global_step/sec: 122.338\n",
      "INFO:tensorflow:loss = 0.00138648, step = 22201 (0.819 sec)\n",
      "INFO:tensorflow:global_step/sec: 166.695\n",
      "INFO:tensorflow:loss = 0.00254255, step = 22301 (0.598 sec)\n",
      "INFO:tensorflow:global_step/sec: 182.221\n",
      "INFO:tensorflow:loss = 0.00194729, step = 22401 (0.549 sec)\n",
      "INFO:tensorflow:global_step/sec: 200.809\n",
      "INFO:tensorflow:loss = 0.00153244, step = 22501 (0.498 sec)\n",
      "INFO:tensorflow:global_step/sec: 195.102\n",
      "INFO:tensorflow:loss = 0.00344794, step = 22601 (0.513 sec)\n",
      "INFO:tensorflow:global_step/sec: 190.696\n",
      "INFO:tensorflow:loss = 0.00100069, step = 22701 (0.525 sec)\n",
      "INFO:tensorflow:global_step/sec: 174.991\n",
      "INFO:tensorflow:loss = 0.00124635, step = 22801 (0.571 sec)\n",
      "INFO:tensorflow:global_step/sec: 175.633\n",
      "INFO:tensorflow:loss = 0.00231376, step = 22901 (0.569 sec)\n",
      "INFO:tensorflow:global_step/sec: 195.749\n",
      "INFO:tensorflow:loss = 0.000602279, step = 23001 (0.511 sec)\n",
      "INFO:tensorflow:global_step/sec: 214.824\n",
      "INFO:tensorflow:loss = 0.00305156, step = 23101 (0.466 sec)\n",
      "INFO:tensorflow:global_step/sec: 204.676\n",
      "INFO:tensorflow:loss = 0.00235002, step = 23201 (0.489 sec)\n",
      "INFO:tensorflow:global_step/sec: 228.647\n",
      "INFO:tensorflow:loss = 0.00154166, step = 23301 (0.437 sec)\n",
      "INFO:tensorflow:global_step/sec: 226.05\n",
      "INFO:tensorflow:loss = 0.000704969, step = 23401 (0.442 sec)\n",
      "INFO:tensorflow:global_step/sec: 231.15\n",
      "INFO:tensorflow:loss = 0.000767915, step = 23501 (0.433 sec)\n",
      "INFO:tensorflow:global_step/sec: 199.699\n",
      "INFO:tensorflow:loss = 0.00063945, step = 23601 (0.501 sec)\n",
      "INFO:tensorflow:global_step/sec: 223.919\n",
      "INFO:tensorflow:loss = 0.000301056, step = 23701 (0.447 sec)\n",
      "INFO:tensorflow:global_step/sec: 245.216\n",
      "INFO:tensorflow:loss = 0.00175696, step = 23801 (0.408 sec)\n",
      "INFO:tensorflow:global_step/sec: 240.635\n",
      "INFO:tensorflow:loss = 0.00106804, step = 23901 (0.416 sec)\n",
      "INFO:tensorflow:global_step/sec: 235.93\n",
      "INFO:tensorflow:loss = 0.000982589, step = 24001 (0.424 sec)\n",
      "INFO:tensorflow:global_step/sec: 236.769\n",
      "INFO:tensorflow:loss = 0.000534493, step = 24101 (0.422 sec)\n",
      "INFO:tensorflow:global_step/sec: 213.622\n",
      "INFO:tensorflow:loss = 0.000924729, step = 24201 (0.468 sec)\n",
      "INFO:tensorflow:global_step/sec: 221.864\n",
      "INFO:tensorflow:loss = 0.000206531, step = 24301 (0.451 sec)\n",
      "INFO:tensorflow:global_step/sec: 199.399\n",
      "INFO:tensorflow:loss = 0.00184635, step = 24401 (0.501 sec)\n",
      "INFO:tensorflow:global_step/sec: 219.656\n",
      "INFO:tensorflow:loss = 0.000563592, step = 24501 (0.455 sec)\n",
      "INFO:tensorflow:global_step/sec: 215.674\n",
      "INFO:tensorflow:loss = 0.000240078, step = 24601 (0.464 sec)\n",
      "INFO:tensorflow:global_step/sec: 202.946\n",
      "INFO:tensorflow:loss = 0.000911916, step = 24701 (0.493 sec)\n",
      "INFO:tensorflow:global_step/sec: 203.138\n",
      "INFO:tensorflow:loss = 0.00135369, step = 24801 (0.492 sec)\n",
      "INFO:tensorflow:global_step/sec: 228.396\n",
      "INFO:tensorflow:loss = 0.00251554, step = 24901 (0.438 sec)\n",
      "INFO:tensorflow:global_step/sec: 241.946\n",
      "INFO:tensorflow:loss = 0.000153224, step = 25001 (0.413 sec)\n",
      "INFO:tensorflow:global_step/sec: 235.858\n",
      "INFO:tensorflow:loss = 0.00190746, step = 25101 (0.424 sec)\n",
      "INFO:tensorflow:global_step/sec: 184.528\n",
      "INFO:tensorflow:loss = 0.00279237, step = 25201 (0.542 sec)\n",
      "INFO:tensorflow:global_step/sec: 186.822\n",
      "INFO:tensorflow:loss = 0.000147187, step = 25301 (0.536 sec)\n",
      "INFO:tensorflow:global_step/sec: 187.126\n",
      "INFO:tensorflow:loss = 0.000728114, step = 25401 (0.534 sec)\n",
      "INFO:tensorflow:global_step/sec: 197.134\n",
      "INFO:tensorflow:loss = 0.00135055, step = 25501 (0.507 sec)\n",
      "INFO:tensorflow:global_step/sec: 200.474\n",
      "INFO:tensorflow:loss = 0.00101404, step = 25601 (0.499 sec)\n",
      "INFO:tensorflow:global_step/sec: 217.685\n",
      "INFO:tensorflow:loss = 0.000952221, step = 25701 (0.459 sec)\n",
      "INFO:tensorflow:global_step/sec: 220.52\n",
      "INFO:tensorflow:loss = 0.00164703, step = 25801 (0.453 sec)\n",
      "INFO:tensorflow:global_step/sec: 221.992\n",
      "INFO:tensorflow:loss = 0.00165784, step = 25901 (0.450 sec)\n",
      "INFO:tensorflow:global_step/sec: 221.295\n",
      "INFO:tensorflow:loss = 5.12721e-05, step = 26001 (0.452 sec)\n",
      "INFO:tensorflow:global_step/sec: 223.473\n",
      "INFO:tensorflow:loss = 0.00105168, step = 26101 (0.447 sec)\n",
      "INFO:tensorflow:global_step/sec: 242.589\n",
      "INFO:tensorflow:loss = 0.000811974, step = 26201 (0.412 sec)\n",
      "INFO:tensorflow:global_step/sec: 243.138\n",
      "INFO:tensorflow:loss = 0.000871437, step = 26301 (0.411 sec)\n",
      "INFO:tensorflow:global_step/sec: 243.198\n",
      "INFO:tensorflow:loss = 0.00176584, step = 26401 (0.411 sec)\n",
      "INFO:tensorflow:global_step/sec: 247.072\n",
      "INFO:tensorflow:loss = 0.000849402, step = 26501 (0.405 sec)\n",
      "INFO:tensorflow:global_step/sec: 240.846\n",
      "INFO:tensorflow:loss = 0.000518591, step = 26601 (0.415 sec)\n",
      "INFO:tensorflow:global_step/sec: 241.065\n",
      "INFO:tensorflow:loss = 1.50678e-05, step = 26701 (0.415 sec)\n",
      "INFO:tensorflow:global_step/sec: 242.412\n",
      "INFO:tensorflow:loss = 0.000841992, step = 26801 (0.413 sec)\n",
      "INFO:tensorflow:global_step/sec: 243.623\n",
      "INFO:tensorflow:loss = 0.000838306, step = 26901 (0.411 sec)\n",
      "INFO:tensorflow:global_step/sec: 243.662\n",
      "INFO:tensorflow:loss = 0.000587201, step = 27001 (0.410 sec)\n",
      "INFO:tensorflow:global_step/sec: 232.392\n",
      "INFO:tensorflow:loss = 0.00046221, step = 27101 (0.430 sec)\n",
      "INFO:tensorflow:global_step/sec: 217.556\n",
      "INFO:tensorflow:loss = 0.000655521, step = 27201 (0.460 sec)\n",
      "INFO:tensorflow:global_step/sec: 218.291\n",
      "INFO:tensorflow:loss = 0.00101177, step = 27301 (0.458 sec)\n",
      "INFO:tensorflow:global_step/sec: 219.019\n",
      "INFO:tensorflow:loss = 0.000149555, step = 27401 (0.457 sec)\n",
      "INFO:tensorflow:global_step/sec: 215.76\n",
      "INFO:tensorflow:loss = 0.00147278, step = 27501 (0.463 sec)\n",
      "INFO:tensorflow:global_step/sec: 217.106\n",
      "INFO:tensorflow:loss = 0.000980559, step = 27601 (0.461 sec)\n",
      "INFO:tensorflow:global_step/sec: 217.779\n",
      "INFO:tensorflow:loss = 0.000261994, step = 27701 (0.459 sec)\n",
      "INFO:tensorflow:global_step/sec: 216.324\n",
      "INFO:tensorflow:loss = 0.000522326, step = 27801 (0.462 sec)\n",
      "INFO:tensorflow:global_step/sec: 219.423\n",
      "INFO:tensorflow:loss = 0.000382858, step = 27901 (0.456 sec)\n",
      "INFO:tensorflow:global_step/sec: 228.358\n",
      "INFO:tensorflow:loss = 0.00150058, step = 28001 (0.438 sec)\n",
      "INFO:tensorflow:global_step/sec: 237.938\n",
      "INFO:tensorflow:loss = 0.000560936, step = 28101 (0.420 sec)\n",
      "INFO:tensorflow:global_step/sec: 236.436\n",
      "INFO:tensorflow:loss = 0.00132494, step = 28201 (0.423 sec)\n",
      "INFO:tensorflow:global_step/sec: 238.36\n",
      "INFO:tensorflow:loss = 0.000744157, step = 28301 (0.420 sec)\n",
      "INFO:tensorflow:global_step/sec: 230.558\n",
      "INFO:tensorflow:loss = 0.000808092, step = 28401 (0.434 sec)\n",
      "INFO:tensorflow:global_step/sec: 236.566\n",
      "INFO:tensorflow:loss = 0.000633052, step = 28501 (0.423 sec)\n",
      "INFO:tensorflow:global_step/sec: 236.807\n",
      "INFO:tensorflow:loss = 0.00040288, step = 28601 (0.422 sec)\n",
      "INFO:tensorflow:global_step/sec: 236.69\n",
      "INFO:tensorflow:loss = 0.000790712, step = 28701 (0.423 sec)\n",
      "INFO:tensorflow:global_step/sec: 239.467\n",
      "INFO:tensorflow:loss = 0.00100996, step = 28801 (0.418 sec)\n",
      "INFO:tensorflow:global_step/sec: 227.861\n",
      "INFO:tensorflow:loss = 0.000233286, step = 28901 (0.439 sec)\n",
      "INFO:tensorflow:global_step/sec: 217.61\n",
      "INFO:tensorflow:loss = 0.00175366, step = 29001 (0.460 sec)\n",
      "INFO:tensorflow:global_step/sec: 216.918\n",
      "INFO:tensorflow:loss = 0.00104872, step = 29101 (0.461 sec)\n",
      "INFO:tensorflow:global_step/sec: 219.206\n",
      "INFO:tensorflow:loss = 0.00168296, step = 29201 (0.456 sec)\n",
      "INFO:tensorflow:global_step/sec: 219.362\n",
      "INFO:tensorflow:loss = 0.00105643, step = 29301 (0.456 sec)\n",
      "INFO:tensorflow:global_step/sec: 222.068\n",
      "INFO:tensorflow:loss = 0.000981568, step = 29401 (0.450 sec)\n",
      "INFO:tensorflow:global_step/sec: 219.518\n",
      "INFO:tensorflow:loss = 0.000960704, step = 29501 (0.456 sec)\n",
      "INFO:tensorflow:global_step/sec: 218.85\n",
      "INFO:tensorflow:loss = 0.000378366, step = 29601 (0.457 sec)\n",
      "INFO:tensorflow:global_step/sec: 222.507\n",
      "INFO:tensorflow:loss = 0.000236591, step = 29701 (0.449 sec)\n",
      "INFO:tensorflow:global_step/sec: 230.935\n",
      "INFO:tensorflow:loss = 0.000619082, step = 29801 (0.433 sec)\n",
      "INFO:tensorflow:global_step/sec: 243.681\n",
      "INFO:tensorflow:loss = 0.000708774, step = 29901 (0.410 sec)\n",
      "INFO:tensorflow:global_step/sec: 242.447\n",
      "INFO:tensorflow:loss = 9.85982e-05, step = 30001 (0.412 sec)\n",
      "INFO:tensorflow:global_step/sec: 241.814\n",
      "INFO:tensorflow:loss = 0.000597217, step = 30101 (0.414 sec)\n",
      "INFO:tensorflow:global_step/sec: 237.015\n",
      "INFO:tensorflow:loss = 0.000242105, step = 30201 (0.422 sec)\n",
      "INFO:tensorflow:global_step/sec: 235.772\n",
      "INFO:tensorflow:loss = 0.000666237, step = 30301 (0.424 sec)\n",
      "INFO:tensorflow:global_step/sec: 240.741\n",
      "INFO:tensorflow:loss = 0.00107709, step = 30401 (0.415 sec)\n",
      "INFO:tensorflow:global_step/sec: 239.014\n",
      "INFO:tensorflow:loss = 0.000853609, step = 30501 (0.418 sec)\n",
      "INFO:tensorflow:global_step/sec: 238.176\n",
      "INFO:tensorflow:loss = 0.00153748, step = 30601 (0.420 sec)\n",
      "INFO:tensorflow:global_step/sec: 240.268\n",
      "INFO:tensorflow:loss = 0.00100649, step = 30701 (0.416 sec)\n",
      "INFO:tensorflow:global_step/sec: 257.441\n",
      "INFO:tensorflow:loss = 0.00119904, step = 30801 (0.388 sec)\n",
      "INFO:tensorflow:global_step/sec: 258.798\n",
      "INFO:tensorflow:loss = 0.00121068, step = 30901 (0.386 sec)\n",
      "INFO:tensorflow:global_step/sec: 267.911\n",
      "INFO:tensorflow:loss = 0.000979147, step = 31001 (0.373 sec)\n",
      "INFO:tensorflow:global_step/sec: 264.721\n",
      "INFO:tensorflow:loss = 0.00142901, step = 31101 (0.378 sec)\n",
      "INFO:tensorflow:global_step/sec: 266.221\n",
      "INFO:tensorflow:loss = 0.000594258, step = 31201 (0.376 sec)\n",
      "INFO:tensorflow:global_step/sec: 265.146\n",
      "INFO:tensorflow:loss = 0.000413715, step = 31301 (0.377 sec)\n",
      "INFO:tensorflow:global_step/sec: 267.588\n",
      "INFO:tensorflow:loss = 0.000751181, step = 31401 (0.374 sec)\n",
      "INFO:tensorflow:global_step/sec: 265.785\n",
      "INFO:tensorflow:loss = 0.000252517, step = 31501 (0.376 sec)\n",
      "INFO:tensorflow:global_step/sec: 260.745\n",
      "INFO:tensorflow:loss = 8.00835e-05, step = 31601 (0.384 sec)\n",
      "INFO:tensorflow:global_step/sec: 271.475\n",
      "INFO:tensorflow:loss = 0.000599239, step = 31701 (0.368 sec)\n",
      "INFO:tensorflow:global_step/sec: 257.765\n",
      "INFO:tensorflow:loss = 0.00010314, step = 31801 (0.388 sec)\n",
      "INFO:tensorflow:global_step/sec: 244.79\n",
      "INFO:tensorflow:loss = 0.000702046, step = 31901 (0.408 sec)\n",
      "INFO:tensorflow:global_step/sec: 236.558\n",
      "INFO:tensorflow:loss = 0.000189627, step = 32001 (0.423 sec)\n",
      "INFO:tensorflow:global_step/sec: 236.523\n",
      "INFO:tensorflow:loss = 0.000453534, step = 32101 (0.423 sec)\n",
      "INFO:tensorflow:global_step/sec: 238.548\n",
      "INFO:tensorflow:loss = 0.00113025, step = 32201 (0.419 sec)\n",
      "INFO:tensorflow:global_step/sec: 237.276\n",
      "INFO:tensorflow:loss = 0.000369316, step = 32301 (0.421 sec)\n",
      "INFO:tensorflow:global_step/sec: 239.846\n",
      "INFO:tensorflow:loss = 9.61032e-05, step = 32401 (0.417 sec)\n",
      "INFO:tensorflow:global_step/sec: 240.832\n",
      "INFO:tensorflow:loss = 0.000561119, step = 32501 (0.415 sec)\n",
      "INFO:tensorflow:global_step/sec: 245.837\n",
      "INFO:tensorflow:loss = 0.00012762, step = 32601 (0.407 sec)\n",
      "INFO:tensorflow:global_step/sec: 244.065\n",
      "INFO:tensorflow:loss = 0.00137255, step = 32701 (0.410 sec)\n",
      "INFO:tensorflow:global_step/sec: 241.211\n",
      "INFO:tensorflow:loss = 0.000887949, step = 32801 (0.415 sec)\n",
      "INFO:tensorflow:global_step/sec: 246.688\n",
      "INFO:tensorflow:loss = 0.000337199, step = 32901 (0.405 sec)\n",
      "INFO:tensorflow:global_step/sec: 237.944\n",
      "INFO:tensorflow:loss = 0.000573135, step = 33001 (0.420 sec)\n",
      "INFO:tensorflow:global_step/sec: 247.035\n",
      "INFO:tensorflow:loss = 0.00195268, step = 33101 (0.405 sec)\n",
      "INFO:tensorflow:global_step/sec: 241.959\n",
      "INFO:tensorflow:loss = 0.000947158, step = 33201 (0.413 sec)\n",
      "INFO:tensorflow:global_step/sec: 239.726\n",
      "INFO:tensorflow:loss = 0.00028633, step = 33301 (0.417 sec)\n",
      "INFO:tensorflow:global_step/sec: 214.33\n",
      "INFO:tensorflow:loss = 0.00124017, step = 33401 (0.467 sec)\n",
      "INFO:tensorflow:global_step/sec: 193.288\n",
      "INFO:tensorflow:loss = 0.000209001, step = 33501 (0.517 sec)\n",
      "INFO:tensorflow:global_step/sec: 191.715\n",
      "INFO:tensorflow:loss = 0.0010908, step = 33601 (0.522 sec)\n",
      "INFO:tensorflow:global_step/sec: 193.058\n",
      "INFO:tensorflow:loss = 0.000751158, step = 33701 (0.518 sec)\n",
      "INFO:tensorflow:global_step/sec: 193.318\n",
      "INFO:tensorflow:loss = 0.00117691, step = 33801 (0.517 sec)\n",
      "INFO:tensorflow:global_step/sec: 192.555\n",
      "INFO:tensorflow:loss = 0.00109041, step = 33901 (0.519 sec)\n",
      "INFO:tensorflow:global_step/sec: 194.089\n",
      "INFO:tensorflow:loss = 0.000558409, step = 34001 (0.515 sec)\n",
      "INFO:tensorflow:global_step/sec: 190.68\n",
      "INFO:tensorflow:loss = 0.000955204, step = 34101 (0.524 sec)\n",
      "INFO:tensorflow:global_step/sec: 212.115\n",
      "INFO:tensorflow:loss = 0.00114629, step = 34201 (0.471 sec)\n",
      "INFO:tensorflow:global_step/sec: 220.762\n",
      "INFO:tensorflow:loss = 0.000116086, step = 34301 (0.453 sec)\n",
      "INFO:tensorflow:global_step/sec: 220.499\n",
      "INFO:tensorflow:loss = 0.000376291, step = 34401 (0.454 sec)\n",
      "INFO:tensorflow:global_step/sec: 220.746\n",
      "INFO:tensorflow:loss = 0.000596384, step = 34501 (0.453 sec)\n",
      "INFO:tensorflow:global_step/sec: 219.763\n",
      "INFO:tensorflow:loss = 0.000287235, step = 34601 (0.455 sec)\n",
      "INFO:tensorflow:global_step/sec: 166.882\n",
      "INFO:tensorflow:loss = 0.0017038, step = 34701 (0.599 sec)\n",
      "INFO:tensorflow:global_step/sec: 193.247\n",
      "INFO:tensorflow:loss = 0.000414888, step = 34801 (0.518 sec)\n",
      "INFO:tensorflow:global_step/sec: 168.67\n",
      "INFO:tensorflow:loss = 0.000323204, step = 34901 (0.593 sec)\n",
      "INFO:tensorflow:global_step/sec: 215.312\n",
      "INFO:tensorflow:loss = 0.000240949, step = 35001 (0.464 sec)\n",
      "INFO:tensorflow:global_step/sec: 209.156\n",
      "INFO:tensorflow:loss = 0.00105711, step = 35101 (0.478 sec)\n",
      "INFO:tensorflow:global_step/sec: 214.048\n",
      "INFO:tensorflow:loss = 0.000331689, step = 35201 (0.467 sec)\n",
      "INFO:tensorflow:global_step/sec: 172.105\n",
      "INFO:tensorflow:loss = 0.000143141, step = 35301 (0.581 sec)\n",
      "INFO:tensorflow:global_step/sec: 218.348\n",
      "INFO:tensorflow:loss = 0.00099728, step = 35401 (0.458 sec)\n",
      "INFO:tensorflow:global_step/sec: 216.362\n",
      "INFO:tensorflow:loss = 0.000150676, step = 35501 (0.462 sec)\n",
      "INFO:tensorflow:global_step/sec: 229.565\n",
      "INFO:tensorflow:loss = 0.000298942, step = 35601 (0.435 sec)\n",
      "INFO:tensorflow:global_step/sec: 172.278\n",
      "INFO:tensorflow:loss = 0.000777814, step = 35701 (0.580 sec)\n",
      "INFO:tensorflow:global_step/sec: 189.537\n",
      "INFO:tensorflow:loss = 0.000844667, step = 35801 (0.528 sec)\n",
      "INFO:tensorflow:global_step/sec: 236.462\n",
      "INFO:tensorflow:loss = 0.000575924, step = 35901 (0.423 sec)\n",
      "INFO:tensorflow:global_step/sec: 208.91\n",
      "INFO:tensorflow:loss = 0.000289464, step = 36001 (0.478 sec)\n",
      "INFO:tensorflow:global_step/sec: 180.32\n",
      "INFO:tensorflow:loss = 0.00115901, step = 36101 (0.555 sec)\n",
      "INFO:tensorflow:global_step/sec: 169.349\n",
      "INFO:tensorflow:loss = 0.000844626, step = 36201 (0.591 sec)\n",
      "INFO:tensorflow:global_step/sec: 156.569\n",
      "INFO:tensorflow:loss = 8.0961e-05, step = 36301 (0.639 sec)\n",
      "INFO:tensorflow:global_step/sec: 133.603\n",
      "INFO:tensorflow:loss = 0.000652663, step = 36401 (0.748 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.481\n",
      "INFO:tensorflow:loss = 0.000233111, step = 36501 (0.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 171.56\n",
      "INFO:tensorflow:loss = 0.000246441, step = 36601 (0.583 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.011\n",
      "INFO:tensorflow:loss = 0.000453849, step = 36701 (0.621 sec)\n",
      "INFO:tensorflow:global_step/sec: 198.293\n",
      "INFO:tensorflow:loss = 0.000899995, step = 36801 (0.504 sec)\n",
      "INFO:tensorflow:global_step/sec: 181.652\n",
      "INFO:tensorflow:loss = 0.000713761, step = 36901 (0.550 sec)\n",
      "INFO:tensorflow:global_step/sec: 190.299\n",
      "INFO:tensorflow:loss = 0.000382187, step = 37001 (0.525 sec)\n",
      "INFO:tensorflow:global_step/sec: 200.515\n",
      "INFO:tensorflow:loss = 0.000168937, step = 37101 (0.499 sec)\n",
      "INFO:tensorflow:global_step/sec: 205.368\n",
      "INFO:tensorflow:loss = 0.000345245, step = 37201 (0.487 sec)\n",
      "INFO:tensorflow:global_step/sec: 195.98\n",
      "INFO:tensorflow:loss = 0.000750656, step = 37301 (0.510 sec)\n",
      "INFO:tensorflow:global_step/sec: 212.979\n",
      "INFO:tensorflow:loss = 0.000352557, step = 37401 (0.470 sec)\n",
      "INFO:tensorflow:global_step/sec: 210.174\n",
      "INFO:tensorflow:loss = 0.000393667, step = 37501 (0.476 sec)\n",
      "INFO:tensorflow:global_step/sec: 156.95\n",
      "INFO:tensorflow:loss = 0.000323634, step = 37601 (0.637 sec)\n",
      "INFO:tensorflow:global_step/sec: 172.972\n",
      "INFO:tensorflow:loss = 0.00011668, step = 37701 (0.578 sec)\n",
      "INFO:tensorflow:global_step/sec: 209.986\n",
      "INFO:tensorflow:loss = 0.000419136, step = 37801 (0.476 sec)\n",
      "INFO:tensorflow:global_step/sec: 242.398\n",
      "INFO:tensorflow:loss = 0.00133944, step = 37901 (0.413 sec)\n",
      "INFO:tensorflow:global_step/sec: 232.131\n",
      "INFO:tensorflow:loss = 0.000302842, step = 38001 (0.431 sec)\n",
      "INFO:tensorflow:global_step/sec: 192.622\n",
      "INFO:tensorflow:loss = 0.00128635, step = 38101 (0.519 sec)\n",
      "INFO:tensorflow:global_step/sec: 228.307\n",
      "INFO:tensorflow:loss = 0.000813133, step = 38201 (0.438 sec)\n",
      "INFO:tensorflow:global_step/sec: 235.862\n",
      "INFO:tensorflow:loss = 0.000126317, step = 38301 (0.424 sec)\n",
      "INFO:tensorflow:global_step/sec: 239.058\n",
      "INFO:tensorflow:loss = 0.000201125, step = 38401 (0.418 sec)\n",
      "INFO:tensorflow:global_step/sec: 214.99\n",
      "INFO:tensorflow:loss = 0.000416319, step = 38501 (0.465 sec)\n",
      "INFO:tensorflow:global_step/sec: 216.723\n",
      "INFO:tensorflow:loss = 0.000711198, step = 38601 (0.461 sec)\n",
      "INFO:tensorflow:global_step/sec: 240.79\n",
      "INFO:tensorflow:loss = 0.000432415, step = 38701 (0.415 sec)\n",
      "INFO:tensorflow:global_step/sec: 239.449\n",
      "INFO:tensorflow:loss = 0.000230968, step = 38801 (0.418 sec)\n",
      "INFO:tensorflow:global_step/sec: 224.749\n",
      "INFO:tensorflow:loss = 0.00131089, step = 38901 (0.445 sec)\n",
      "INFO:tensorflow:global_step/sec: 185.815\n",
      "INFO:tensorflow:loss = 0.000195044, step = 39001 (0.538 sec)\n",
      "INFO:tensorflow:global_step/sec: 186.065\n",
      "INFO:tensorflow:loss = 0.000720432, step = 39101 (0.537 sec)\n",
      "INFO:tensorflow:global_step/sec: 189.428\n",
      "INFO:tensorflow:loss = 0.000604851, step = 39201 (0.528 sec)\n",
      "INFO:tensorflow:global_step/sec: 190.859\n",
      "INFO:tensorflow:loss = 0.000287668, step = 39301 (0.524 sec)\n",
      "INFO:tensorflow:global_step/sec: 188.808\n",
      "INFO:tensorflow:loss = 0.000637942, step = 39401 (0.530 sec)\n",
      "INFO:tensorflow:global_step/sec: 180.328\n",
      "INFO:tensorflow:loss = 0.000344775, step = 39501 (0.554 sec)\n",
      "INFO:tensorflow:global_step/sec: 191.627\n",
      "INFO:tensorflow:loss = 0.000685066, step = 39601 (0.522 sec)\n",
      "INFO:tensorflow:global_step/sec: 212.621\n",
      "INFO:tensorflow:loss = 0.000204588, step = 39701 (0.470 sec)\n",
      "INFO:tensorflow:global_step/sec: 216.98\n",
      "INFO:tensorflow:loss = 0.00110618, step = 39801 (0.461 sec)\n",
      "INFO:tensorflow:global_step/sec: 215.497\n",
      "INFO:tensorflow:loss = 0.0012423, step = 39901 (0.464 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 40000 into /tmp/tmpecewk7kb/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.000481317.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SKCompat()"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "config = tf.contrib.learn.RunConfig(tf_random_seed=42) # not shown in the config\n",
    "\n",
    "feature_cols = tf.contrib.learn.infer_real_valued_columns_from_input(X_train)\n",
    "dnn_clf = tf.contrib.learn.DNNClassifier(hidden_units=[300,100], n_classes=10,\n",
    "                                         feature_columns=feature_cols, config=config)\n",
    "dnn_clf = tf.contrib.learn.SKCompat(dnn_clf) # if TensorFlow >= 1.1\n",
    "dnn_clf.fit(X_train, y_train, batch_size=50, steps=40000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /tmp/tmpecewk7kb/model.ckpt-40000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.98250000000000004"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "y_pred = dnn_clf.predict(X_test)\n",
    "accuracy_score(y_test, y_pred['classes'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.071487383848950564"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import log_loss\n",
    "\n",
    "y_pred_proba = y_pred['probabilities']\n",
    "log_loss(y_test, y_pred_proba)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Using plain TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def neuron_layer(X, n_neurons, name, activation=None):\n",
    "    with tf.name_scope(name):\n",
    "        n_inputs = int(X.get_shape()[1])\n",
    "        stddev = 2 / np.sqrt(n_inputs)\n",
    "        init = tf.truncated_normal((n_inputs, n_neurons), stddev=stddev)\n",
    "        W = tf.Variable(init, name=\"kernel\")\n",
    "        b = tf.Variable(tf.zeros([n_neurons]), name=\"bias\")\n",
    "        Z = tf.matmul(X, W) + b\n",
    "        if activation is not None:\n",
    "            return activation(Z)\n",
    "        else:\n",
    "            return Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = neuron_layer(X, n_hidden1, name=\"hidden1\",\n",
    "                           activation=tf.nn.relu)\n",
    "    hidden2 = neuron_layer(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                           activation=tf.nn.relu)\n",
    "    logits = neuron_layer(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,\n",
    "                                                              logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 40\n",
    "batch_size = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.9 Val accuracy: 0.9146\n",
      "1 Train accuracy: 0.94 Val accuracy: 0.9348\n",
      "2 Train accuracy: 0.92 Val accuracy: 0.9466\n",
      "3 Train accuracy: 0.96 Val accuracy: 0.9508\n",
      "4 Train accuracy: 0.92 Val accuracy: 0.9586\n",
      "5 Train accuracy: 0.94 Val accuracy: 0.9584\n",
      "6 Train accuracy: 0.98 Val accuracy: 0.9608\n",
      "7 Train accuracy: 0.96 Val accuracy: 0.9636\n",
      "8 Train accuracy: 0.92 Val accuracy: 0.9638\n",
      "9 Train accuracy: 0.96 Val accuracy: 0.965\n",
      "10 Train accuracy: 0.98 Val accuracy: 0.9686\n",
      "11 Train accuracy: 0.94 Val accuracy: 0.9686\n",
      "12 Train accuracy: 1.0 Val accuracy: 0.9702\n",
      "13 Train accuracy: 0.94 Val accuracy: 0.9686\n",
      "14 Train accuracy: 1.0 Val accuracy: 0.9716\n",
      "15 Train accuracy: 1.0 Val accuracy: 0.973\n",
      "16 Train accuracy: 1.0 Val accuracy: 0.9736\n",
      "17 Train accuracy: 0.98 Val accuracy: 0.9736\n",
      "18 Train accuracy: 1.0 Val accuracy: 0.9752\n",
      "19 Train accuracy: 1.0 Val accuracy: 0.975\n",
      "20 Train accuracy: 0.98 Val accuracy: 0.9748\n",
      "21 Train accuracy: 1.0 Val accuracy: 0.975\n",
      "22 Train accuracy: 1.0 Val accuracy: 0.9756\n",
      "23 Train accuracy: 1.0 Val accuracy: 0.9772\n",
      "24 Train accuracy: 1.0 Val accuracy: 0.978\n",
      "25 Train accuracy: 1.0 Val accuracy: 0.9772\n",
      "26 Train accuracy: 1.0 Val accuracy: 0.9778\n",
      "27 Train accuracy: 1.0 Val accuracy: 0.9762\n",
      "28 Train accuracy: 0.98 Val accuracy: 0.9774\n",
      "29 Train accuracy: 1.0 Val accuracy: 0.9784\n",
      "30 Train accuracy: 1.0 Val accuracy: 0.9776\n",
      "31 Train accuracy: 1.0 Val accuracy: 0.9786\n",
      "32 Train accuracy: 0.98 Val accuracy: 0.9776\n",
      "33 Train accuracy: 0.98 Val accuracy: 0.9794\n",
      "34 Train accuracy: 0.98 Val accuracy: 0.978\n",
      "35 Train accuracy: 0.98 Val accuracy: 0.9768\n",
      "36 Train accuracy: 0.98 Val accuracy: 0.9794\n",
      "37 Train accuracy: 1.0 Val accuracy: 0.9784\n",
      "38 Train accuracy: 1.0 Val accuracy: 0.979\n",
      "39 Train accuracy: 1.0 Val accuracy: 0.9788\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_val = accuracy.eval(feed_dict={X: mnist.validation.images,\n",
    "                                            y: mnist.validation.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Val accuracy:\", acc_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_final.ckpt\") # or better, use save_path\n",
    "    X_new_scaled = mnist.test.images[:20]\n",
    "    Z = logits.eval(feed_dict={X: X_new_scaled})\n",
    "    y_pred = np.argmax(Z, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted classes: [7 2 1 0 4 1 4 9 6 9 0 6 9 0 1 5 9 7 3 4]\n",
      "Actual classes:    [7 2 1 0 4 1 4 9 5 9 0 6 9 0 1 5 9 7 3 4]\n"
     ]
    }
   ],
   "source": [
    "print(\"Predicted classes:\", y_pred)\n",
    "print(\"Actual classes:   \", mnist.test.labels[:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from IPython.display import clear_output, Image, display, HTML\n",
    "\n",
    "def strip_consts(graph_def, max_const_size=32):\n",
    "    \"\"\"Strip large constant values from graph_def.\"\"\"\n",
    "    strip_def = tf.GraphDef()\n",
    "    for n0 in graph_def.node:\n",
    "        n = strip_def.node.add() \n",
    "        n.MergeFrom(n0)\n",
    "        if n.op == 'Const':\n",
    "            tensor = n.attr['value'].tensor\n",
    "            size = len(tensor.tensor_content)\n",
    "            if size > max_const_size:\n",
    "                tensor.tensor_content = b\"<stripped %d bytes>\"%size\n",
    "    return strip_def\n",
    "\n",
    "def show_graph(graph_def, max_const_size=32):\n",
    "    \"\"\"Visualize TensorFlow graph.\"\"\"\n",
    "    if hasattr(graph_def, 'as_graph_def'):\n",
    "        graph_def = graph_def.as_graph_def()\n",
    "    strip_def = strip_consts(graph_def, max_const_size=max_const_size)\n",
    "    code = \"\"\"\n",
    "        <script>\n",
    "          function load() {{\n",
    "            document.getElementById(\"{id}\").pbtxt = {data};\n",
    "          }}\n",
    "        </script>\n",
    "        <link rel=\"import\" href=\"https://tensorboard.appspot.com/tf-graph-basic.build.html\" onload=load()>\n",
    "        <div style=\"height:600px\">\n",
    "          <tf-graph-basic id=\"{id}\"></tf-graph-basic>\n",
    "        </div>\n",
    "    \"\"\".format(data=repr(str(strip_def)), id='graph'+str(np.random.rand()))\n",
    "\n",
    "    iframe = \"\"\"\n",
    "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"{}\"></iframe>\n",
    "    \"\"\".format(code.replace('\"', '&quot;'))\n",
    "    display(HTML(iframe))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.2851015593374667&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0714285746216774\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 5\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/mul&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;dnn/hidden1/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;dnn/hidden1/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;dnn/hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;dnn/hidden1/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\000d\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.1154700517654419\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 21\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/mul&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;dnn/hidden2/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;dnn/hidden2/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;dnn/hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;dnn/hidden2/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;d\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.20000000298023224\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/outputs/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 37\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/outputs/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/outputs/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/outputs/truncated_normal/mul&quot;\\n  input: &quot;dnn/outputs/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;dnn/outputs/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;dnn/outputs/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;dnn/outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;dnn/outputs/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/add&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;train/gradients/Shape&quot;\\n  input: &quot;train/gradients/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/Fill&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;message&quot;\\n    value {\\n      s: &quot;Currently there is no way to take the second derivative of sparse_softmax_cross_entropy_with_logits due to the fused implementation\\\\\\'s interaction with tf.gradients()&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 10\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 100\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 300\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden1/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden1/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden2/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden2/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/outputs/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/outputs/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden1/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden1/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden2/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden2/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/outputs/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/outputs/bias/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;eval/InTopK&quot;\\n  op: &quot;InTopK&quot;\\n  input: &quot;dnn/outputs/add&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;k&quot;\\n    value {\\n      i: 1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/InTopK&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^dnn/hidden1/kernel/Assign&quot;\\n  input: &quot;^dnn/hidden1/bias/Assign&quot;\\n  input: &quot;^dnn/hidden2/kernel/Assign&quot;\\n  input: &quot;^dnn/hidden2/bias/Assign&quot;\\n  input: &quot;^dnn/outputs/kernel/Assign&quot;\\n  input: &quot;^dnn/outputs/bias/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;dnn/hidden1/bias&quot;\\n        string_val: &quot;dnn/hidden1/kernel&quot;\\n        string_val: &quot;dnn/hidden2/bias&quot;\\n        string_val: &quot;dnn/hidden2/kernel&quot;\\n        string_val: &quot;dnn/outputs/bias&quot;\\n        string_val: &quot;dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_1/tensor_names&quot;\\n  input: &quot;save/RestoreV2_1/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;save/RestoreV2_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_2/tensor_names&quot;\\n  input: &quot;save/RestoreV2_2/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;save/RestoreV2_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_3/tensor_names&quot;\\n  input: &quot;save/RestoreV2_3/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;save/RestoreV2_3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_4/tensor_names&quot;\\n  input: &quot;save/RestoreV2_4/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;save/RestoreV2_4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_5/tensor_names&quot;\\n  input: &quot;save/RestoreV2_5/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;save/RestoreV2_5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.2851015593374667&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using `dense()` instead of `neuron_layer()`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: the book uses `tensorflow.contrib.layers.fully_connected()` rather than `tf.layers.dense()` (which did not exist when this chapter was written). It is now preferable to use `tf.layers.dense()`, because anything in the contrib module may change or be deleted without notice. The `dense()` function is almost identical to the `fully_connected()` function, except for a few minor differences:\n",
    "* several parameters are renamed: `scope` becomes `name`, `activation_fn` becomes `activation` (and similarly the `_fn` suffix is removed from other parameters such as `normalizer_fn`), `weights_initializer` becomes `kernel_initializer`, etc.\n",
    "* the default `activation` is now `None` rather than `tf.nn.relu`.\n",
    "* a few more differences are presented in chapter 11."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\",\n",
    "                              activation=tf.nn.relu)\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                              activation=tf.nn.relu)\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.9 Test accuracy: 0.9053\n",
      "1 Train accuracy: 0.88 Test accuracy: 0.9206\n",
      "2 Train accuracy: 0.94 Test accuracy: 0.9301\n",
      "3 Train accuracy: 0.94 Test accuracy: 0.9397\n",
      "4 Train accuracy: 0.92 Test accuracy: 0.9451\n",
      "5 Train accuracy: 0.94 Test accuracy: 0.9476\n",
      "6 Train accuracy: 0.92 Test accuracy: 0.9515\n",
      "7 Train accuracy: 0.98 Test accuracy: 0.9546\n",
      "8 Train accuracy: 0.96 Test accuracy: 0.9569\n",
      "9 Train accuracy: 0.94 Test accuracy: 0.9605\n",
      "10 Train accuracy: 0.92 Test accuracy: 0.9619\n",
      "11 Train accuracy: 0.96 Test accuracy: 0.9631\n",
      "12 Train accuracy: 1.0 Test accuracy: 0.9661\n",
      "13 Train accuracy: 0.94 Test accuracy: 0.9657\n",
      "14 Train accuracy: 1.0 Test accuracy: 0.9669\n",
      "15 Train accuracy: 0.94 Test accuracy: 0.9682\n",
      "16 Train accuracy: 0.96 Test accuracy: 0.9701\n",
      "17 Train accuracy: 0.98 Test accuracy: 0.9696\n",
      "18 Train accuracy: 1.0 Test accuracy: 0.97\n",
      "19 Train accuracy: 1.0 Test accuracy: 0.971\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "n_batches = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.7224827313584268&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 5\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Initializer/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/bias/Initializer/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;hidden1/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\000d\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 22\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Initializer/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/bias/Initializer/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;hidden2/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;d\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 39\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Initializer/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/bias/Initializer/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;outputs/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;train/gradients/Shape&quot;\\n  input: &quot;train/gradients/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/Fill&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;message&quot;\\n    value {\\n      s: &quot;Currently there is no way to take the second derivative of sparse_softmax_cross_entropy_with_logits due to the fused implementation\\\\\\'s interaction with tf.gradients()&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;eval/InTopK&quot;\\n  op: &quot;InTopK&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;k&quot;\\n    value {\\n      i: 1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/InTopK&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^hidden1/kernel/Assign&quot;\\n  input: &quot;^hidden1/bias/Assign&quot;\\n  input: &quot;^hidden2/kernel/Assign&quot;\\n  input: &quot;^hidden2/bias/Assign&quot;\\n  input: &quot;^outputs/kernel/Assign&quot;\\n  input: &quot;^outputs/bias/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n        string_val: &quot;hidden1/kernel&quot;\\n        string_val: &quot;hidden2/bias&quot;\\n        string_val: &quot;hidden2/kernel&quot;\\n        string_val: &quot;outputs/bias&quot;\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_1/tensor_names&quot;\\n  input: &quot;save/RestoreV2_1/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;save/RestoreV2_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_2/tensor_names&quot;\\n  input: &quot;save/RestoreV2_2/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;save/RestoreV2_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_3/tensor_names&quot;\\n  input: &quot;save/RestoreV2_3/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;save/RestoreV2_3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_4/tensor_names&quot;\\n  input: &quot;save/RestoreV2_4/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;save/RestoreV2_4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_5/tensor_names&quot;\\n  input: &quot;save/RestoreV2_5/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;save/RestoreV2_5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.7224827313584268&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Exercise solutions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. to 8."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "See appendix A."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Train a deep MLP on the MNIST dataset and see if you can get over 98% precision. Just like in the last exercise of chapter 9, try adding all the bells and whistles (i.e., save checkpoints, restore the last checkpoint in case of an interruption, add summaries, plot learning curves using TensorBoard, and so on)._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First let's create the deep net. It's exactly the same as earlier, with just one addition: we add a `tf.summary.scalar()` to track the loss and the accuracy during training, so we can view nice learning curves using TensorBoard."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\",\n",
    "                              activation=tf.nn.relu)\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                              activation=tf.nn.relu)\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "    loss_summary = tf.summary.scalar('log_loss', loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    accuracy_summary = tf.summary.scalar('accuracy', accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we need to define the directory to write the TensorBoard logs to:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "def log_dir(prefix=\"\"):\n",
    "    now = datetime.utcnow().strftime(\"%Y%m%d%H%M%S\")\n",
    "    root_logdir = \"tf_logs\"\n",
    "    if prefix:\n",
    "        prefix += \"-\"\n",
    "    name = prefix + \"run-\" + now\n",
    "    return \"{}/{}/\".format(root_logdir, name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "logdir = log_dir(\"mnist_dnn\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can create the `FileWriter` that we will use to write the TensorBoard logs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Hey! Why don't we implement early stopping? For this, we are going to need a validation set. Luckily, the dataset returned by TensorFlow's `input_data()` function (see above) is already split into a training set (60,000 instances, already shuffled for us), a validation set (5,000 instances) and a test set (5,000 instances). So we can easily define `X_valid` and `y_valid`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_valid = mnist.validation.images\n",
    "y_valid = mnist.validation.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "m, n = X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0 \tValidation accuracy: 90.440% \tLoss: 0.35228\n",
      "Epoch: 5 \tValidation accuracy: 95.060% \tLoss: 0.17539\n",
      "Epoch: 10 \tValidation accuracy: 96.680% \tLoss: 0.12546\n",
      "Epoch: 15 \tValidation accuracy: 97.220% \tLoss: 0.10438\n",
      "Epoch: 20 \tValidation accuracy: 97.600% \tLoss: 0.08914\n",
      "Epoch: 25 \tValidation accuracy: 97.740% \tLoss: 0.08115\n",
      "Epoch: 30 \tValidation accuracy: 97.780% \tLoss: 0.07788\n",
      "Epoch: 35 \tValidation accuracy: 97.920% \tLoss: 0.07094\n",
      "Epoch: 40 \tValidation accuracy: 97.920% \tLoss: 0.06983\n",
      "Epoch: 45 \tValidation accuracy: 97.880% \tLoss: 0.06778\n",
      "Epoch: 50 \tValidation accuracy: 98.100% \tLoss: 0.06649\n",
      "Epoch: 55 \tValidation accuracy: 98.080% \tLoss: 0.06642\n",
      "Epoch: 60 \tValidation accuracy: 98.220% \tLoss: 0.06510\n",
      "Epoch: 65 \tValidation accuracy: 98.060% \tLoss: 0.06588\n",
      "Epoch: 70 \tValidation accuracy: 98.080% \tLoss: 0.06762\n",
      "Epoch: 75 \tValidation accuracy: 98.160% \tLoss: 0.06705\n",
      "Epoch: 80 \tValidation accuracy: 98.160% \tLoss: 0.06705\n",
      "Epoch: 85 \tValidation accuracy: 98.200% \tLoss: 0.06709\n",
      "Epoch: 90 \tValidation accuracy: 98.180% \tLoss: 0.06698\n",
      "Epoch: 95 \tValidation accuracy: 98.180% \tLoss: 0.06890\n",
      "Epoch: 100 \tValidation accuracy: 98.220% \tLoss: 0.06838\n",
      "Epoch: 105 \tValidation accuracy: 98.120% \tLoss: 0.06893\n",
      "Epoch: 110 \tValidation accuracy: 98.180% \tLoss: 0.06980\n",
      "Epoch: 115 \tValidation accuracy: 98.240% \tLoss: 0.07049\n",
      "Early stopping\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10001\n",
    "batch_size = 50\n",
    "n_batches = int(np.ceil(m / batch_size))\n",
    "\n",
    "checkpoint_path = \"/tmp/my_deep_mnist_model.ckpt\"\n",
    "checkpoint_epoch_path = checkpoint_path + \".epoch\"\n",
    "final_model_path = \"./my_deep_mnist_model\"\n",
    "\n",
    "best_loss = np.infty\n",
    "epochs_without_progress = 0\n",
    "max_epochs_without_progress = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    if os.path.isfile(checkpoint_epoch_path):\n",
    "        # if the checkpoint file exists, restore the model and load the epoch number\n",
    "        with open(checkpoint_epoch_path, \"rb\") as f:\n",
    "            start_epoch = int(f.read())\n",
    "        print(\"Training was interrupted. Continuing at epoch\", start_epoch)\n",
    "        saver.restore(sess, checkpoint_path)\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        sess.run(init)\n",
    "\n",
    "    for epoch in range(start_epoch, n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val, loss_val, accuracy_summary_str, loss_summary_str = sess.run([accuracy, loss, accuracy_summary, loss_summary], feed_dict={X: X_valid, y: y_valid})\n",
    "        file_writer.add_summary(accuracy_summary_str, epoch)\n",
    "        file_writer.add_summary(loss_summary_str, epoch)\n",
    "        if epoch % 5 == 0:\n",
    "            print(\"Epoch:\", epoch,\n",
    "                  \"\\tValidation accuracy: {:.3f}%\".format(accuracy_val * 100),\n",
    "                  \"\\tLoss: {:.5f}\".format(loss_val))\n",
    "            saver.save(sess, checkpoint_path)\n",
    "            with open(checkpoint_epoch_path, \"wb\") as f:\n",
    "                f.write(b\"%d\" % (epoch + 1))\n",
    "            if loss_val < best_loss:\n",
    "                saver.save(sess, final_model_path)\n",
    "                best_loss = loss_val\n",
    "            else:\n",
    "                epochs_without_progress += 5\n",
    "                if epochs_without_progress > max_epochs_without_progress:\n",
    "                    print(\"Early stopping\")\n",
    "                    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.remove(checkpoint_epoch_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_deep_mnist_model\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, final_model_path)\n",
    "    accuracy_val = accuracy.eval(feed_dict={X: X_test, y: y_test})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.97839999"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  },
  "nav_menu": {
   "height": "264px",
   "width": "369px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}