Notebooks/C11_N1_The Vanishing Exploding Gradients Problems.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 11 - Training Deep Neural Networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part I: The Vanishing/Exploding Gradients Problems"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:root:Line magic function `%tensorflow_version` not found.\n",
      "/Users/dongdongdongdong/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The tensorboard module is not an IPython extension.\n"
     ]
    }
   ],
   "source": [
    "# Python ≥3.5 is required\n",
    "import sys\n",
    "assert sys.version_info >= (3, 5)\n",
    "\n",
    "# Scikit-Learn ≥0.20 is required\n",
    "import sklearn\n",
    "assert sklearn.__version__ >= \"0.20\"\n",
    "\n",
    "try:\n",
    "    # %tensorflow_version only exists in Colab.\n",
    "    %tensorflow_version 2.x\n",
    "except Exception:\n",
    "    pass\n",
    "\n",
    "# TensorFlow ≥2.0 is required\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "assert tf.__version__ >= \"2.0\"\n",
    "\n",
    "%load_ext tensorboard\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "np.random.seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "mpl.rc('axes', labelsize=14)\n",
    "mpl.rc('xtick', labelsize=12)\n",
    "mpl.rc('ytick', labelsize=12)\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"deep\"\n",
    "IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID)\n",
    "os.makedirs(IMAGES_PATH, exist_ok=True)\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True, fig_extension=\"png\", resolution=300):\n",
    "    path = os.path.join(IMAGES_PATH, fig_id + \".\" + fig_extension)\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format=fig_extension, dpi=resolution)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vanishing/Exploding Gradients Problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def logit(z):\n",
    "    return 1 / (1 + np.exp(-z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure sigmoid_saturation_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xl4Tdf6wPHvisyDOVJEDTXGPLWGW2KuoobQUrO2itZP\ni9Jr6JVqtTXGvdVBJ62gSs1qLNEoSmgorSg1hQRBSGSSZP3+2BEZTiLhJOckeT/Ps59k773OXu/Z\njrxn7b32WkprjRBCCGFtbCwdgBBCCGGKJCghhBBWSRKUEEIIqyQJSgghhFWSBCWEEMIqSYISQghh\nlSRBiYeilApQSn1s6TggZ7EopY4rpWbkU0hp612ilNqUD/V4K6W0UqpsPtQ1Uil1QSmVbIlzmiGW\nYUqpaEvGIPKOkuegREZKKXfAF3gWKA9EAseBD7XWO1LKlAbuaq2jLBZoipzEopQ6DqzWWs/Ioxi8\ngd2Au9Y6Is32Ehj/zyLNWNc54GOt9dw02+yB0sAVnYf/qZVSpYCrwHhgNRCltc6XBKGU0kA/rfXq\nNNucADet9dX8iEHkL1tLByCs0o+AM/AScBooB7QFytwroLW+YZnQMrOmWDLSWt/Kp3oSgPB8qKoy\nxt+NTVrrsHyoL1ta61gg1tJxiDyitZZFltQFKAlooOMDygVgfIu/t+4BbMD4Y3EeGI7R6pqRpowG\nRgPrgRjgFNAO8AS2AXeAYKBJhrr6AH8A8cBFYCoprf8sYimXUse9WEZkjMXE+3ki5TXhKXEcAbpn\nKGMPzEo5ZjzwD/B/QJWU95Z2WZLymiUYf8wBXgWuALYZjrscWJ+TOFLea7q6UrZ7p6yXzcV5OwdM\nAz4HbgOhwFvZnKNhJt5nFWAGcNxE2eg06zNS/g36A2eAKGBd2nhTyg1NE/OVNOfxXIZ6z5mqJ815\nPg0kpPx8JcN+DYwEVqWc43+AQZb+vydL5kXuQYmMolOW55RSjrl43bcY367bAz2BQSnrGU0Dvgca\nAkHACuAr4BOgMXAZ4486AEqpphh/SNYA9YG3gX8Dr2cTyxKgOtAR6AUMwfhDmh1XYAvQKSW2H4E1\nSqnaGd7jEIzLW3UwWpiRGH/8fVLK1MW4LDrORB0/YHwB6Jjm/blgnC//HMbRByORvJtST3lTbyYX\n5+1NjITQBPgImK2UamnqmMBK4JmU359MqftiFmVNqQK8APQGOmP8e7+fJuZXMZLlN0ADjEvMJ1J2\nN0/5+UpKvffW01FK9QY+BvyAesBC4BOlVI8MRd/B+CLQMOV9fa2UMvV5FZZk6Qwpi/UtGH9sbwBx\nwH5gLvBUhjIBpLRagFoY30pbpNlfCUgicwvqgzTr9VK2jU+zzZs0LQFgGbArQ90zgNAsYqmZ8vrW\nafZXzhhLDs/DAWBayu81Uo77TBZl08WdZvsSUlpQKetrgaVp1gcBtwDHnMSRsn4OmJhd/Tk8b+eA\nFRnK/J22LhOxNEupp0qG4+akBRUHlEizbSpwOs16KMZ9zqzq1kDfB9TzK/C1iX+Dvdl8Dm0xWvTS\nirKyRVpQIhOt9Y9ABaAHxrf5VsABpdSULF5SG0jGaBHdO8ZFjNZQRsfS/H4l5ecfJraVS/lZB+OP\nTlp7gYpKqeImjl8nJZaDaWI5n0UsqZRSLkqp2UqpP5VSN1N6hjUDHk8p0jjluLuzO04O+AO9lFLO\nKesDMTpvxOUwjpzK6Xk7lqHMZe6fe3M7r9Pfk0utSylVDqgI/PyIdWT1vr0ybEt931rrROAaefe+\nxUOSBCVM0lrHaa13aK3f1Vq3wrgMNyOlt1hGKheHvpu2mmy23ftsqjTbMoX5iLGkNRfoB0zH6BDS\nCCPJ3Xu/D3vcjDYBiUDPlD/KHbl/eS8nceRUTs/bXRP7cvt3IZnM58fORLns6jLX+b133AdtM8f7\nFnlM/kFETv2JcSnE1H2pvzA+S03vbVBKeWK0wsxR778ybPsXxqUqU93K78WSeo9CKfV4DmL5F/Cd\n1vpHrfUxjMtNT6TZfyTluO2yeH1Cys9i2VWitY7H6J49EON+TDiwJxdx3Ksr23rI/Xl7FNcAD6VU\n2iTTKDcH0FpfAS4BHbIpdpcHv++/MP2+/8xNPMI6SIIS6SilyiildimlBimlGiilqiql+gGTgJ+1\n1rczvkZrHYLRC+8zpVQLpVQjjBvdMWT9LT6n5gFtlVIzlFI1lVIDgQnAbFOFU2LZCnyulGqZEssS\nHtwV+RTQWynVRClVH6NVk5qMtdZ/Y3Ry+FIp5ZNyXp5WSg1OKXIe4712U0q5K6Vcs6nLH+gCjAKW\na62TcxpHinPA00qpitk8mJur8/aIAjCewZqilHpCKfUS0PchjvM+8IZS6s2UmBsppSak2X8O6KCU\neizleSxT5gCDlVKvKaVqKKXGYnwZyIv3LfKYJCiRUTTGTflxGN/sT2B0rV6O8Y0/K8Mwvu0HYHQ3\nX4bxQGfcowSjtT6CccnLh5SHhVOW7EaOGAacBXYBG1NiP/eAqsanxBuIcd/tQMrvaQ1JOdZ/gZMY\nia9ESpyXgP9g/JG98oD4fsFoLXiR/vJeTuN4B6MTyhmM1ksmD3neHorW+i+MxwdGYtzb6YTxmcnt\ncT4FXsPoqXcc44tG3TRFJmC0YC8Cv2dxjHXAWIzeiX9ifI7HaK035jYeYXkykoTIEynf7C8DA1I6\nXQghRK7ISBLCLJRS7QE3jB555TBaEhEY34KFECLXzHaJTyn1ulIqSCkVr5Rakk25oUqpw0qp20qp\n0JQutZIoCz474D2MBLUR455PG631HYtGJYQosMx2iU8p1Qeju2kXwElrPSyLcqMxri//Brhj3K9Y\npbX+0CyBCCGEKBTM1nLRWq8BUEo1wxhbLatyn6ZZvaSUWkbWXXeFEEIUUdZwaa0N98fbykQpNRKj\ndxBOTk5NK1WqlF9x5UhycjI2NtIZ8kHkPOXMxYsX0Vrz+OO5HTii6Mnvz1R4XDjOxZwpbmdqABPr\nZY3/906dOhWhtXZ/UDmLJiil1HCMYVxezqqM1noxsBigWbNmOigoKKuiFhEQEIC3t7elw7B6cp5y\nxtvbm8jISIKDgy0ditXLz8/U5B2Tmb1vNhO8J/BO23fypU5zscb/e0qp8zkpZ7EEpZTqhfFcRked\nZoI3IYSwJvP3z2f2vtmMaTaG6W2mWzqcIsUiCUop9QzwBdBNa/3Hg8oLIYQlLDu2jAnbJ9DXqy//\n7fpf0o/mJPKa2RJUSldxW4yxsoqlzCWUmDJScNpy7TFGGeittT6Y+UhCCGEdzkaepV2Vdvj39qeY\nzYOGARTmZs47Z9Mwnn15G2OOm1hgmlLqcaVUdMqAnWCM0lwC+Clle7RSaosZ4xBCiEeSlJwEwLQ2\n09g2aBsOtg4WjqhoMluC0lrP0FqrDMsMrfUFrbWr1vpCSrl2WmvblG33lq7mikMIIR5FSEQI9T6t\nx8FLxgUeu2KmZg4R+cEaupkLIYRVuBx1mS7+XYi5G0Mpx6wGTBf5RRKUEEIAkXGRPOP/DNdjrxMw\nNIAaZWpYOqQiTxKUEKLIi0uMo+f3PTkZcZLNL26maYWmD36RyHPW9XixEEJYSFnnsnzX+zs6PdHJ\n0qGIFNKCEkIUWVprYu7G4GLvwup+q+U5JysjLSghRJHlu8eXll+1JDIuUpKTFZIEJYQokj4L+gzf\nPb40q9CMEg4lLB2OMEESlBCiyPnxzx8Zs3kM3Wt2Z3GPxdJ6slKSoIQQRUrg+UBeXPMiLTxbsLLv\nSmxt5Fa8tZIEJYQoUqqWqspztZ5j04ubcLZztnQ4Ihvy1UEIUSRcib5CWeeyeBb3ZFW/VZYOR+SA\ntKCEEIXe1TtX+dc3/+LVTa9aOhSRC5KghBCFWlR8FN2Wd+PS7UuMaDzC0uGIXJBLfEKIQishKQGf\nH3z4Pex31r6wllaVWlk6JJELkqCEEIXW6E2j2fHPDr5+7mt61Oph6XBELkmCEkIUWi83eZkGHg0Y\n3ni4pUMRD0ESlBCi0Pnjyh/U96hPy0otaVmppaXDEQ9JOkkIIQqVb4O/pcFnDVj711pLhyIekSQo\nIUShsfnUZl7a8BIdq3WkW81ulg5HPCJJUEKIQuFA6AH6repHo8caseb5NdgXs7d0SOIRSYISQhR4\nN2Jv0H15dyoWr8hPA3/CzcHN0iEJM5BOEkKIAq+0U2kWdFlA68dbU86lnKXDEWYiLSghRIF1I/YG\nv4X+BsDghoOpVqqahSMS5mTWBKWUel0pFaSUildKLXlA2TeVUuFKqVtKqa+VUg7mjEUIUbjFJcXR\nY0UPnln2DJFxkZYOR+QBc7egLgPvAV9nV0gp1QV4G+gAVAGqAb5mjkUIUUglJicy86+Z7L+4ny96\nfEFJx5KWDknkAaW1Nv9BlXoP8NRaD8ti/3LgnNZ6Ssp6B2CZ1vqx7I7r5uammzZtmm7b888/z5gx\nY4iJieHZZ5/N9Jphw4YxbNgwIiIi6Nu3b6b9o0eP5oUXXuDixYsMHjw40/4JEybQo0cPQkJCePXV\nzCMh9+jRgwkTJhAcHMwbb7yRaf+sWbNo1aoV+/btY8qUKZn2+/n50ahRI3bu3Ml7772Xaf/nn39O\nrVq12LhxI/Pmzcu0f+nSpVSqVImVK1fy6aefZtq/evVqypYty5IlS1iyZEmm/T/99BPOzs588skn\n/PDDD5n2BwQEADB37lw2bdqUbp+TkxNbtmwBYObMmfz888/p9pcpU4Yff/wRgIEDB3Lp0qV0+z09\nPfH39wfgjTfeIDg4ON3+mjVrsnjxYgBGjhzJqVOn0u1v1KgRfn5+AAwaNIjQ0NB0+1u2bMkHH3wA\ngI+PD9evX0+3v0OHDkyfPh2Arl27Ehsbm25/9+7dmThxIgDe3t5klBefveDgYBITE2nWrNkDP3vT\npk2jY8eORe6zp9GcqXuGS+Uu8cmznxCxNSLbz96///1v9u/fn25/UfrsdezYkZIl0yfwR/2796if\nvT179hzWWjfLtCMDS3WSqAusT7N+FPBQSpXRWqf7l1RKjQRGAtjZ2REZmb4pf+rUKQICAoiLi8u0\nD+DkyZMEBARw69Ytk/tPnDhBQEAAV69eNbn/jz/+wM3NjQsXLpjcHxsbS0BAAKdPnza5/8iRIyQk\nJHD8+HGT+4OCgoiMjOTo0aMm9//222+EhYXxxx9/mNy/f/9+zpw5w4kTJ0zu//XXXylRogQnT540\nuf+XX37B0dGRU6dOmdx/74/EmTNnMu2/994Bzp49m2l/cnJy6v6EhIRM++3s7FL3h4aGZtp/+fLl\n1P2XL1/OtD80NDR1/5UrVzLtv3DhQur+a9eucfv27XT7z549m7r/xo0bxMfHp9t/5syZ1P2mzk1e\nfPYSExPRWhMZGfnAz97Ro0extbUtcp+9m543uVTuEv3L96fOnTp8d/a7bD97ps5fUfrsJSUlZSrz\nMH/3tLYhOdmFpCRntm27yN9/H+aff8K4cKEuyckOaO1IcrIDycmOfPQRlCp1jkuXHDhxYiTJyY4k\nJzuitQPJyQ7A05nqNMVSLagzwGta660p63ZAAlBVa30uq+M2a9ZMBwUFmT3eRxEQEGDyG45IT85T\nznh7exMZGZnpG724Lyk5iVV/rsLjmgft2rWzdDhWLyAggLZtvblzB65fhxs3jCXt7zdvQlSUsdy+\nff/3tOsxMeaMSll1CyoaKJ5m/d7vURaIRQhRAGw6tYmGHg2pVKIS/ev1T21hFFWxsXDlCoSH3/+Z\ndomIMJJQeHgroqPh7t1Hr9PN7f7i6grOzuDkdH/JuJ7Vtu7dc1afpRLUCaAhcO/Cc0PgSsbLe0II\nAbDzn530WdkHHy8fVvissHQ4eS4hAUJD4fx5uHDBWO79fvEihIXBrVs5PZoxooaTE5QuDWXKGD/v\nLWXKQMmSULz4/eRj6ncXF7B5hG51p06d4sKFC3Ts2DHHrzFrglJK2aYcsxhQTCnlCCRqrRMzFP0O\nWKKUWgaEAdOAJeaMRQhROBy+fJjeK3tTu2xtPu2WuTNGQRUZCX//bSynThk/z5wxklB4ODzo7oud\nHXh4wGOP3V/SrpctaySfkJB9dOvWCien/Hlfpixfvpzhw4fTpEkTyyUojETznzTrgwBfpdTXwJ+A\nl9b6gtZ6q1JqNrAbcAJ+zPA6IYTg9I3TPLv8Wco4lWHroK0Frju51sbltz/+MJbjxyEkxEhG165l\n/bpixaBiRXj8cahc2fh5b6lUydhXqhQo9eAYrl9PsFhyiouLY8yYMaxcuZKEhASSk5Nz9XqzJiit\n9QxgRha7XTOUnQ/MN2f9QojC5a0db5GUnMS2Qduo4FbB0uFkKzHRSECHD8OxY/eTUkSE6fJOTlCj\nhrHUrGn8rF4dqlSB8uXBtoAPRPfPP//w7LPPcuHChdRu9LntlFfAT4EQojBb0nMJ52+dp1bZWpYO\nJR2t4exZOHjw/nLkiNFxIaPixaFePahf31jq1DESUoUKj3ZPx5qtWbOGoUOHEhMTk67VZNEWlBBC\nPKr4xHg+3Pshk1pPooRjCRo4NrB0SCQmGglozx745Rc4cMB0y+iJJ6B5c2jQ4H5CevzxnF2KKwzu\n3r3Lm2++yddff53p4WOQFpQQogBLSk5i0NpBrP5zNc0qNLPYpIPJyUZC2rnTSEp790J0dPoy7u7w\n5JP3l+bNjU4JRdWFCxfo3r07p0+fNpmcQBKUEKKA0lozbus4Vv+5mrmd5uZ7crp6FbZvh61bjZ8Z\nOzHUqAFt2xpL69bGvaKi0jJ6kM2bNzNgwABiYmJISkrKspxc4hNCFEizAmex6NAiJracyIRWE/Kl\nzr/+gjVrYN06yDhITeXK0LkztG8PbdoY94xEZlOmTMHPzy/LVlNa0oISQhQ412Ou4/ebH4MbDOaj\nTh/lWT1aG73s1qyBtWvh5Mn7+xwdjdbRM88YS61a0kLKidDQULTWFCtWLNvWE0iCEkIUQGWcy3Dw\n5YN4FvfERpm/a9vff8PSpeDvb/S+u6d0aXjuOejTBzp0MIblEbnz3XffMXXqVKZNm8amTZuIj4/P\nMhFJghJCFBiB5wPZfW4309tMp2qpqmY9dkQErFxpJKbffru/vXx5IyH16WNcuivozxtZg1q1avHD\nDz/QoEEDjh8/nmU5SVBCiALhjyt/8Nz3z+Hh4sEbLd6guEPxB7/oAbSGgAD47DPjEt69AVJdXcHH\nBwYPBm9vY6QGYV67du3ibNrmKcaccXfv3iUx0RjtTjpJCCGs3vnI8zyz7Bmc7ZzZNmjbIyenyEj4\n9lsjMd27r2RjY9xLGjwYevWSy3d5bdKkSdy5cyfdtnLlyuHt7c3KlSu5e/eutKCEENYtIiaCLv5d\nuJNwh8DhgVQuWfmhj3XyJMybB8uW3R/FoXx5eOUVY/H0NFPQIlt79uwhJCQk3TZXV1dmz57N888/\nz8yZM/H19c1RT7+0JEEJIfLVgdADXIq6xE8v/kR9j/oPdYxff4Vp0+rx66/3t3XsCKNHQ48exkjf\nIv+89dZbmVpPpUuXpm/fvgBUqlSJL7/8MtfHlQQlhMhX3Wt259y4c5Rxzt2wC8nJsHEjzJ4N+/YB\nlMXBAYYNgzffNLqFi/wXGBjIiRMn0m1zcXHhww8/xOYRBxsspEMVCiGsSbJOZuTGkaz5aw1ArpKT\n1rB+PTRqZNxL2rfPmGpi8OBznD9v3HeS5GQ5kyZNIibDfPClSpXi+eeff+RjS4ISQuS5t3e+zRdH\nvuDPa3/m+DVaG8MOPfmkkZj++MO4p+TnZ0zqN2LEOTw88jBo8UD79u3j2LFj6ba5urrywQcfUMwM\nXSXlEp8QIk/N2zePOfvm8Frz15j69NQcvWbPHpg2zRikFYyZYqdONTo+ODrmYbAiV0y1nooXL07/\n/v3NcnxJUEKIPON/zJ+JOybSz6sfC59ZiHrA2EGnT8PEicYlPTBGB588GV57TbqJW5vffvuN33//\nPd02V1dXZs2aha2Znn6WBCWEyDNHw4/Srko7lvZeSjGbrC/53LoF770HCxcaD9e6uMCkSfDGG8aE\nf8L6TJ48OVPrycXFhYEDB5qtDklQQgizS9bJ2CgbZneaTUJSAg62DibLJSXB118bl/OuXjW2DRsG\ns2YZzzMJ6xQUFMTBgwfTbXN1deX99983W+sJpJOEEMLMQiJCaPx5Y/648gdKqSyT0++/w1NPwciR\nRnJq3RoOHYJvvpHkZO0mT55MXFxcum1OTk4MGTLErPVIghJCmM2l25fo7N+ZsKgwnOycTJaJiTEu\n3zVvbkx9UakSfP89BAZCs2b5HLDItd9//539+/enG7bIxcWFmTNnYmfmJ6TlEp8Qwiwi4yJ5Ztkz\n3Ii9QcDQAKqXrp6pzPbtMGqUMeWFjY1xj2nmTGMwV1EwvP322yZbT8OHDzd7XZKghBCPLPZuLM+t\neI6QiBB+GvgTTSs0Tbf/xg0YN86YjwmgQQP48kujFSUKjmPHjhEYGJip9eTr64u9vb3Z6zPrJT6l\nVGml1Fql1B2l1Hml1ItZlHNQSn2mlLqilLqhlNqolKpozliEEPknMTkRR1tHlvZeSsdqHdPt274d\n6tUzkpOjI3z4oTG9uiSnguftt98mPj4+3TYHBwdeeumlPKnP3C2oRUAC4AE0AjYrpY5qrU9kKDcO\naAk0AG4BXwD/A/qYOR4hRB7SWhOfFI+bgxvbBm1L95xTTIzxDNPHHxvrrVvDkiVQPfOVP1EA3Lhx\ng61bt2ZqPc2YMQMHB9MdYR6V2VpQSikXwAeYrrWO1lrvBTYAg00Urwps01pf0VrHAd8Ddc0VixAi\nf/ju8cV7iTdR8VHpktPhw9C0qZGcbG2NbuN79khyKshKly7Njh07aNy4MS4uLgDY29vzyiuv5Fmd\n5mxB1QSStNan0mw7CrQ1UfYrYKFSqgIQCQwEtpg6qFJqJDASwMPDg4CAADOG/Oiio6OtLiZrJOcp\nZyIjI0lKSioQ52r95fX4/e1H18e6ErQvCKUUycmwYsXjfPNNFZKSbKhc+Q5TpvxFzZrRBAaat375\nTOWMOc9TsWLFmD9/PsHBwXzxxRd07dqVAwcOmOXYJmmtzbIATwPhGba9AgSYKFscWAFoIBH4HSj9\noDqaNm2qrc3u3bstHUKBIOcpZ9q2basbNmxo6TAeaNWJVVrNULr78u76btJdrbXW165p/cwzWhvD\nvGr9f/+ndUxM3sUgn6mcscbzBATpHOQVc3aSiE5JPGkVB6JMlP0UcATKAC7AGrJoQQkhrMuec3sY\nuGYgLSu1ZGXfldja2LJ/PzRubIw+XqYM/PSTMWyRk+lHoYTIEXMmqFOArVKqRpptDYGMHSTubV+i\ntb6htY7H6CDxpFKqrBnjEULkgYrFK9L5ic5sHLARJ1tn/PygTRsIDYWWLY0RIrp2tXSUojAwW4LS\nWt/BaAm9q5RyUUq1BnoCS00UPwQMUUqVUErZAWOAy1rrCHPFI4Qwr4iYCLTWVC9dnY0DNlIsoTT9\n+hmz2SYmGj8DAoyRIYQwB3MPdTQGcAKuYtxjGq21PqGUelopFZ2m3EQgDvgbuAY8C/Q2cyxCCDO5\neucqLb9qyfht4wH4+29o0QJ+/NEYbXz1apg/H/LgWU1RhJn1OSit9Q2gl4ntgYBrmvXrGD33hBBW\nLio+imeXPcul25d4vu7z/Pwz9OsHN28aD+CuXSvdx62Rt7c39erVo2/fvpYO5aHJYLFCiCwlJCXQ\n54c+BIcH80PfVRxe35IuXYzk1KMH7NtXuJLTtWvXGDNmDFWqVMHBwQEPDw86dOjAjh07cvT6gIAA\nlFJEROTf3YolS5bgamIwwzVr1vDBBx/kWxx5QcbiE0Jk6ZWNr7Dzn5188ey3bJzfjcWLje3//rcx\nwaBNIfuK6+PjQ0xMDF999RXVq1fn6tWr7Nmzh+vXr+d7LAkJCY80vl3p0qXNGI1lFLKPlxDCnAbV\nH8TMFotYNmkIixeDgwMsW2aMDFHYklNkZCSBgYF8+OGHdOjQgcqVK9O8eXMmTpxI//79AfD396d5\n8+a4ublRrlw5+vXrx6VLlwA4d+4c7dq1A8Dd3R2lFMOGDQOMy22vv/56uvqGDRtG9+7dU9e9vb0Z\nPXo0EydOxN3dndatWwMwf/58GjRogIuLCxUrVuTll18mMjISMFpsw4cP586dOyilUEoxY8YMk3VW\nqVKF9957j1dffZXixYvj6enJnDlz0sV06tQp2rZti6OjI7Vq1eKnn37C1dWVJUuWmOck51Ih+4gJ\nIcwhJCIEgJq2nVj+5hgCAoxJBAMD4UWTQ0AXfK6urri6urJhw4ZM00nck5CQgK+vL0ePHmXTpk1E\nREQwYMAAACpVqsSPP/4IwIkTJwgLC2PhwoW5isHf3x+tNYGBgXz33XcA2NjY4Ofnx4kTJ1i+fDkH\nDx5k7NixALRq1Qo/Pz+cnZ0JCwsjLCyMiRMnZnn8BQsWUL9+fY4cOcLkyZOZNGkS+/fvByA5OZne\nvXtja2vLgQMHWLJkCb6+vpkGh81PcolPCJHOkuAlvLThJf7b6Bfee7U14eFGZ4gtW8DT09LR5R1b\nW1uWLFnCK6+8wuLFi2ncuDGtW7emX79+PPXUUwCMGDEitXy1atX49NNPqVOnDqGhoXh6eqZeVitX\nrhxly+b+sc6qVasyb968dNveeOON1N+rVKnC7Nmz6dmzJ99++y329vaUKFECpRSPPfbYA4/fuXPn\n1FbV2LFj+e9//8vPP/9My5Yt2bFjByEhIWzfvp2KFY3JJRYsWJDakrMEaUEJIVJtPrWZlze8TKM7\nb/H2wFaEh4O3t9FyKszJ6R4fHx8uX77Mxo0b6dq1K/v27aNFixbMmjULgCNHjtCzZ08qV66Mm5sb\nzVKmAL5w4YJZ6m/atGmmbbt27aJTp054enri5uZGnz59SEhIIDw8PNfHb9CgQbr1ChUqcPXqVQBO\nnjxJhQoVUpMTQPPmzbGx4LVcSVBCCAD2X9xPv1X9qHRuKsfmf0B0tKJ/f2P4opIlLR1d/nF0dKRT\np06888477Nu3j5deeokZM2Zw69YtunTpgrOzM0uXLuXQoUNs3boVMC79ZcfGxibdNBUAd+/ezVTu\n3ijh95zNKuXSAAAgAElEQVQ/f55u3bpRp04dVq1axeHDh/n6669zVKcpGadkNwb4TQaMcVnTjkhv\nDSRBCSG4eucq3Vd0x/m3dzn3jS+JiYq33jI6ROTRVD8FhpeXF4mJiQQHBxMREcGsWbNo06YNtWvX\nTm193HOv111SUlK67e7u7oSFhaXbdvTo0QfWHRQUREJCAgsWLKBly5bUrFmTy5cvZ6ozY30Po06d\nOly6dCnd8YOCglITmCVIghJC4O5cjmYndnB940SUMgZ6nT278PXUy87169dp3749/v7+HDt2jLNn\nz7Jq1Spmz55Nhw4d8PLywsHBgY8//ph//vmHzZs3M3369HTHqFy5MkopNm/ezLVr14iONgbQad++\nPVu2bGHDhg2EhIQwfvx4Ll68+MCYatSoQXJyMn5+fpw9e5YVK1bg5+eXrkyVKlWIi4tjx44dRERE\nEBMT81Dvv1OnTtSqVYuhQ4dy9OhRDhw4wPjx47G1tbVYy6oIffyEEBndiL1BcNgxxoyB7d81wdbW\naDX93/9ZOrL85+rqSosWLVi4cCFt27albt26TJkyhRdffJGVK1fi7u7Ot99+y7p16/Dy8sLX15f5\n8+enO0bFihXx9fVl6tSpeHh4pHZIGDFiROrSunVrXF1d6d37waO7NWjQgIULFzJ//ny8vLz48ssv\nmTt3broyrVq1YtSoUQwYMAB3d3dmz579UO/fxsaGtWvXEh8fz5NPPsnQoUOZOnUqSikcHR0f6piP\nLCdzcljLIvNBFVxynnImP+eDupNwR7f4/F/aockPGrR2cNB648Z8qdos5DOVM49ynoKDgzWgg4KC\nzBeQzvl8UNLNXIgiKDE5kb4rBnJgwXg42RtXV9iwAVKeMxVF1Nq1a3FxcaFGjRqcO3eO8ePH07Bh\nQ5o0aWKReCRBCVHEaK0Zvup1tviOhn86U6qU8YxTyqM+ogiLiopi8uTJXLx4kVKlSuHt7c2CBQss\ndg9KEpQQRczn+/3xn9wfznvj4QHbt0OGx2NEETVkyBCGDBli6TBSSYISogiJjoblbw+E8zZUqKDZ\nvVtRs6aloxLCNOnFJ0QRsfr3bXTsnEBgoA0VK8KePZKchHWTFpQQRcD6o7t4vpcr+oI9np6we3fh\nmsdJFE6SoIQo5PaE/E6f55zRF1pQ0TOZgAAbnnjC0lEJ8WByiU+IQiz4/Gk6drlL8oUWVPBM5Jc9\nkpxEwSEJSohCKjYWOnSNJvH8k5SveJfAPbZUq2bpqITIOUlQQhRC8fHQpw/c+KsRZT3u8kuAnSQn\nUeBIghKikImOjadp5xC2boWyZWHPLjvpECEKJElQQhQiCXeT8Op0iBO/1MK1+F127AAvL0tHJcTD\nMWuCUkqVVkqtVUrdUUqdV0q9mE3ZJkqpX5RS0UqpK0qpceaMRYiiJilJ06j7b1z89V84OCewc7sd\njRpZOiohHp65u5kvAhIAD6ARsFkpdVRrfSJtIaVUWWAr8CawGrAHisCE0kLkDa2hdb8g/treCluH\nBLZvsZex9USBZ7YWlFLKBfABpmuto7XWe4ENwGATxccD27TWy7TW8VrrKK31X+aKRYiiRGsYOz6a\n39Y2x8b2Lps22NKmjaWjEuLRmbMFVRNI0lqfSrPtKNDWRNkWwB9KqX1AdeA34DWt9YWMBZVSI4GR\nAB4eHgQEBJgx5EcXHR1tdTFZIzlPORMZGUlSUlKuzpW//+N89VU1ihVL5j//OYGDfSRF4VTLZypn\nCvJ5MmeCcgVuZdh2C3AzUdYTaAJ0Av4AZgMrgNYZC2qtFwOLAZo1a6a9vb3NF7EZBAQEYG0xWSM5\nTzlTsmRJIiMjc3yuJn34N199VQ2lYNkyG154oejcdJLPVM4U5PNkzgQVDRTPsK04EGWibCywVmt9\nCEAp5QtEKKVKaK0zJjkhhAkLvj7PnCnGw03zF8bzwgsOFo5ICPMyZy++U4CtUqpGmm0NgRMmyh4D\ndJr1e79bZlYsIQqYFRvDGT/yMdDFGP/vSN4YK8lJFD5mS1Ba6zvAGuBdpZSLUqo10BNYaqL4N0Bv\npVQjpZQdMB3Yq7WONFc8QhRWu/bdZNDzrpDkQP8R15n7fklLhyREnjD3g7pjACfgKsY9pdFa6xNK\nqaeVUtH3CmmtdwFTgM0pZasDWT4zJYQwnD4NfXo4kRznSvse11j2RRksNBu3EHnOrM9Baa1vAL1M\nbA/E6ESRdtunwKfmrF+IwiwsDDp3hls3HGnTPp4tq92xkbFgRCEmH28hCoAbN5Op3yqUs2eheXPY\ntM4Be3tLRyVE3pIEJYSVi42Fhm3Ocf2cJ2Ufj2DzZnAz9fCGEIWMJCghrFhiIjTv8jehx6vhUuYm\nh/aUwd3d0lEJkT8kQQlhpbSGDv1OcyKwBnYu0ezbXZwqVaRHhCg6JEEJYaWmTIFf1lXHxj6OHVvs\naVC/mKVDEiJfSYISwgrNm6f58EOwtdWsXg1tn5YeEaLokQQlhJW5EtOZiRONS3nffKPo3cPRwhEJ\nYRmSoISwIuG3mhJ+ehYAU96/xqBBFg5ICAuSBCWEldi6+zYhx2eCtmX42HDenyLd9UTRJglKCCsQ\n9HscPborSHTGrcJKvlr4mKVDEsLiJEEJYWHnz8Nz3exJjHHDtcJOqpadLePrCYGZx+ITQuTO1aua\nTp0hLMyGtm01ycmzuX07ydJhCWEVJEEJYSFRUdC4zWUun6pI/QbJrF9vQ8+eCZnKbdiwgeDgYOrX\nr0/dunV54oknKFZMnokShZ8kKCEsID4enux4kcshlXDzuMq2re6UKGG67JkzZ/D19cXV1ZWkpCQS\nEhLw9PSkfv36PPnkk9SrV4+6detStWpVSVyiUJEEJUQ+S0oC7+dCOXmwEg4lbnIosDTly2d902n0\n6NG899573LhxI3Xb2bNnOXv2LD/99BPOzs7pEleDBg148skn6d+/P9WqVcuPtyREnpBOEkLkI62h\n1+BLHNjuSTGnKAJ2OlKrRvbfEx0dHXnvvfdwcXHJtC8xMZHbt29z584d7t69y9mzZ1m/fj3Tpk1j\n//79efU2hMgXkqCEyEfTpsGmFRWxsYtn/QZNi2ZOOXrdyy+/jKur64MLAvb29nTp0oUXX5RJqkXB\nJglKiHzy/uwYZs2CYsVg3Y8OdOtYPMevtbOz46OPPjLZisqoePHiLFu2DCV91UUBJwlKiHzw8Re3\nmTbZGYCvv4YePXJ/jEGDBlG6dOlsyzg4OFCjRo2HCVEIqyMJSog89sOaGMaOMpLTuP+cY8iQhztO\nsWLFmDt3bratqPj4eA4fPkytWrXYu3fvw1UkhJWQBCVEHvp5910G9C8Gyba8MPo0fjOqPNLx+vbt\nS/ny5bMtk5CQQEREBJ07d2b69OkkJcmDv6JgkgQlRB4JDoau3e+SfNcBb58QViyq/sjHtLGxYcGC\nBZlaUY6OmafkiI2NZf78+Tz11FOEhoY+ct1C5DezJiilVGml1Fql1B2l1HmlVLbdiJRS9kqpk0op\n+d8jCpW//4YuXeBujDMN24ewc2Uts42v161bN6pWrZq67uzszGuvvYarqys2Nun/S8fExBAcHIyX\nlxfr1q0zTwBC5BNzt6AWAQmABzAQ+FQpVTeb8m8BV80cgxAWdfkytO94l6tXoVMn+O2nWphzgAel\nFH5+fjg7O+Pk5MSAAQOYO3cux48fp379+jg7O6crn5SURFRUFAMHDuTll18mNjbWfMEIkYfMlqCU\nUi6ADzBdax2ttd4LbAAGZ1G+KjAI+MBcMQhhaVevQrPWkYResKNOw9usWQMODuavp0OHDtStW5fy\n5cvzv//9D4DKlSsTFBTE2LFjcXLK/HxVTEwMy5cvp169evz555/mD0oIM1Naa/McSKnGwD6ttVOa\nbROBtlrrTJ1qlVKbgK+Am4C/1tozi+OOBEYCeHh4NP3+++/NEq+5REdH5/gByqKsKJyn27dtGTWu\nBmHnPHAqf5qli0IpUyp3x3jjjTdISkpKTTrZuTf0kamu58HBwbzzzjvExsaSmJiYbp9SCnt7e8aM\nGUOPHj0K7PNSReEzZQ7WeJ7atWt3WGvd7IEFtdZmWYCngfAM214BAkyU7Q1sTfndGwjNSR1NmzbV\n1mb37t2WDqFAKOzn6dYtres0vK1Ba0ePc/rMheiHOk7btm11w4YNzRLTtWvXdPv27bWzs7MGMi3O\nzs66W7du+ubNm2apL78V9s+UuVjjeQKCdA7+5pvzHlQ0kPHR+OJAVNoNKZcCZwNjzVi3EBZz5w50\n7hrPX0fdsC1zkQO/uFKt0oNHfMhrZcuWZefOnbz//vtZXvLbuXMnNWvWZN++fRaIUIjsmTNBnQJs\nlVJpH2NvCJzIUK4GUAUIVEqFA2uA8kqpcKVUFTPGI0Sei4uDXr3gt30OlHCP4ued0LBmGUuHlUop\nxRtvvMH+/fupVKlSpu7o8fHxXLt2jY4dOzJjxgx5ZkpYFbMlKK31HYxk865SykUp1RroCSzNUPQ4\nUAlolLK8DFxJ+f2iueIRIq8lJECvPgns3AnlysFvgW60aVTJ0mGZ1LBhQ/766y98fHwy9fID45mp\nOXPm0KpVKy5dumSBCIXIzNzdzMcAThhdx1cAo7XWJ5RSTyulogG01ola6/B7C3ADSE5Zl69vokBI\nTIQBLyaybYs9yvkmm7bEUauWpaPKnouLC/7+/nz55ZdZPjN15MgRvLy82LBhg4WiFOI+syYorfUN\nrXUvrbWL1vpxrfXylO2BWmuT3Ui01gE6ix58QlijpCQYNjyZNT/agsMtPlxyhOZNMo/kYK0GDBjA\nsWPHqFu3bqbW1L35pQYMGMCrr75KXFychaIUQoY6KlS8vb15/fXXLR1GoZaUBMOHa5b524BdNBMX\n7WZSvw6WDivXqlatyuHDhxkzZkyWHSiWLl1K/fr1OXnypAUiFEISFNeuXWPMmDFUqVIFBwcHPDw8\n6NChAzt27MjR64ODg1FKERERkceR3rdkyRKTzzWsWbOGDz6Q557zSlISDB0KS5cqsItm2NwfmPNS\nL0uH9dDs7OyYM2cOGzdupFSpUtjZ2aXbHxsby5kzZ2jatClffvnlvUdEhMg3RT5B+fj4cPDgQb76\n6itOnTrFpk2b6Nq1K9evX8/3WBISEh7p9aVLl8bNzc1M0Yi0EhNhyBBYtgxcXTVvffozX48dbumw\nzKJDhw6EhITQunXrTJf8tNbExMQwbtw4evbsya1btywUpSiScvKwlLUs5n5Q9+bNmxrQO3bsyLLM\n0qVLdbNmzbSrq6t2d3fXffv21aGhoVprrc+ePZvp4cehQ4dqrY0HLl977bV0xxo6dKju1q1b6nrb\ntm31qFGj9IQJE3TZsmV1s2bNtNZaz5s3T9evX187OzvrChUq6Jdeein1Ycrdu3dnqvM///mPyTor\nV66sZ86cqUeOHKnd3Nx0xYoV9ezZs9PFFBISotu0aaMdHBx0zZo19ebNm7WLi4v+5ptvHuqcZsUa\nHxbMqbt3tR4wQGvQ2tU1We/dm3d1mfNB3dxKTk7Wc+fO1U5OTiYf7HVwcNAeHh56//79Fokvo4L8\nmcpP1niesMCDugWOq6srrq6ubNiwIcubwQkJCfj6+nL06FE2bdpEREQEAwYMAKBSpUr4+voCcOLE\nCcLCwli4cGGuYvD390drTWBgIN999x1gTKng5+fHiRMnWL58OQcPHmTsWOO55latWqUOFBoWFkZY\nWBgTJ07M8vgLFiygfv36HDlyhMmTJzNp0iT2798PQHJyMr1798bW1pYDBw6wZMkSfH19iY+Pz9V7\nKMwSE2HwYFixArCPosP0ubRubemo8oZSigkTJvDrr79SsWJFk89MXblyhfbt2zNz5kySk5MtFKko\nMnKSxaxlyYuhjlavXq1LlSqlHRwcdIsWLfSECRP0gQMHsiz/119/aUBfvHhRa631ggULNKCvXbuW\nrlxOW1D169d/YIxbtmzR9vb2OikpSWut9TfffKNdXFwylTPVgurfv3+6MtWrV9czZ87UWmu9detW\nXaxYsdQWodZa//rrrxqQFpTWOi5O6969jZYTDrf0ExMH68jYyDyt05ItqLSioqJ0//79sx0mqUWL\nFvry5csWi7EgfqYswRrPE9KCyhkfHx8uX77Mxo0b6dq1K/v27aNFixbMmjULgCNHjtCzZ08qV66M\nm5sbzZoZ4xteuHDBLPU3bdo007Zdu3bRqVMnPD09cXNzo0+fPiQkJBAeHp7r4zdo0CDdeoUKFbh6\n1Zjh5OTJk1SoUIGKFSum7m/evHmm52OKojt34LnnYO1aUI63eGz0UH5550NKOJawdGj5wtXVlRUr\nVrB48WJcXFxMPjMVFBRE7dq12bx5s4WiFIWd/CXCmI20U6dOvPPOO+zbt4+XXnqJGTNmcOvWLbp0\n6YKzszNLly7l0KFDbN26FXhwhwYbG5tMvZ7u3r2bqVzGmVHPnz9Pt27dqFOnDqtWreLw4cN8/fXX\nOarTlIw9s5RSqZdmtNYFdiTrvBQZaUw2uH072LndpMSoXuyZ9hEV3CpYOrR8N3DgQI4dO0adOnWy\nfGaqX79+vPbaa3JpWJidJCgTvLy8SExMJDg4mIiICGbNmkWbNm2oXbt2auvjHltbW4BMY5i5u7sT\nFhaWbtvRo0cfWHdQUBAJCQksWLCAli1bUrNmTS5fvpyujL29vVnGTKtTpw6XLl1Kd/ygoKAifW/h\n2jVo3x5+/RU8PWHH7nh+njSPmmVqWjo0i6lWrRq///47I0eONPnMVGxsLIsXL2b79u0WiE4UZkU6\nQV2/fp327dvj7+/PsWPHOHv2LKtWrWL27Nl06NABLy8vHBwc+Pjjj/nnn3/YvHkz06dPT3cMDw8P\nlFJs3ryZa9euER0dDUD79u3ZsmULGzZsICQkhPHjx3Px4oOHGqxRowbJycn4+flx9uxZVqxYgZ+f\nX7oyVapUIS4ujh07dhAREUFMTMxDvf9OnTpRq1Ythg4dytGjRzlw4ADjx4/H1ta2SLasQkOhTRv4\n/Xco6xnJnl+SaNv0MZqUb2Lp0CzOzs6OBQsWsG7dOkqWLJmuZW5nZ0erVq3o1q2bBSMUhVGRTlCu\nrq60aNGChQsX0rZtW+rWrcuUKVN48cUXWblyJe7u7nz77besW7cOLy8vfH19mT9/frpjuLu74+vr\ny9SpU/Hw8EgdyWHEiBGpS+vWrXF1daV3794PjKlBgwYsXLiQ+fPn4+XlxZdffsncuXPTlWnVqhWj\nRo1iwIABuLu7M3v27Id6/zY2Nqxdu5b4+HiefPJJhg4dytSpU1FKZerBVdiFhMDTT8PJk1Di8fNE\nvFCbi2qvpcOyOp07dyYkJIQWLVqkXvJzcXFh1apVcu9SmF9OelJYyyITFua94OBgDeigoCCzHtea\nz9PevVqXLm301vOo9Y9mUik9f998i8RiLb34HiQpKUl/9NFH2s7OTm/fvt0iMVjzZ8qaWON5Ioe9\n+GwtnSCFZa1duxYXFxdq1KjBuXPnGD9+PA0bNqRJk6JxWWvtWnjxRWNep5otT3GqXWMmeb/Omy3f\ntHRoVs3GxoZJkyYxbtw4HBwcLB2OKKSkTV7ERUVF8frrr+Pl5cXAgQOpU6cO27ZtKxL3oD7+GHx8\njOQ0cHg0F55pytDm/fiw44eWDq3AkOQk8pK0oIq4IUOGMGTIEEuHka+Sk+Hf/4Z7t+7eew+mTHFl\n0tVfqVO2TpFIzkIUBJKgRJESEwPDh8MPP4CtLbz1YQiPtduLUi/RwKPBgw8ghMg3colPFBmhoUZP\nvR9+ADc3+O/Sf/jk7lPM2z+PuESZmM9SqlSpkqmnqhAgLShRRBw4AL17Q3g4PPEEfOp/iaG//gtX\ne1e2DtqKo23R6laf34YNG0ZERASbNm3KtO/QoUOZRlQRAopACyo8PJyuXbuybNkyGYqliFq6FLy9\njeTUrh1s3hXB60HtiU2MZdugbTxe4nFLh1ikubu7ZxpGyRIedT42YX6FPkF98skn7Ny5k1GjRuHu\n7s6bb76ZaQgiUTglJcHkycZEg/HxMGYMbNsG+29s4uKti2wasIm65epaOswiL+MlPqUUixcvpl+/\nfri4uFCtWjX8/f3TvebSpUu8++67lCpVilKlStGtWzf+/vvv1P1nzpyhZ8+ePPbYY7i4uNCkSZNM\nrbcqVaowY8YMRowYQcmSJRk4cGDevlGRa4U6QSUmJrJo0SISExOJjo4mKiqKRYsW8e2331o6NJHH\nrlyBzp2NnnrFisEnn8CiRWBnB8MaDSPk9RBaP15IJ3YqBN5991169uzJ0aNHeeGFFxgxYgTnz58H\njJHU27Vrh729PXv27GH//v2UL1+ejh07pg77FR0dTdeuXdmxYwdHjx7Fx8eHPn36cPLkyXT1zJ8/\nn9q1axMUFJQ6g4GwHoU6QW3evDnTCOI2NjYMGjTIQhGJ/PDLL9C4MezaBR4esHMnvDoqmdd/ep19\nF/cBUKlEJQtHKbIzePBgBg0aRPXq1Zk5cya2trYEBgYC8P3336O1ZvLkyTRo0IDatWvz+eefEx0d\nndpKatiwIaNGjaJ+/fpUr16dqVOn0qRJE1avXp2unrZt2zJp0iSqV69OjRo18v19iuwV6gQ1e/Zs\noqKi0m17+umn8fT0tFBEIi8lJ8OHHxr3mcLC7g/86u0Nk3ZMYtGhRfxy/hdLhylyIO08Zra2tri7\nu6fOJHD48GHOnj3Ls88+mzordokSJbh58yZnzpwB4M6dO0yaNAkvLy9KlSqFq6srQUFBmeZxuze/\nm7BOZu3Fp5QqDXwFdAYigH9rrZebKPcWMBSonFLuE631HHPG8s8//3DkyJF029zc3LKdHl0UXDdu\nwNChcO82w9tvw8yZxrNOc/fNZd7+ebze/HUmt55s2UBFjmQ3j1lycjKNGjXizTff5KmnnkpXrnTp\n0gBMnDiRrVu3MnfuXGrUqIGzszNDhgzJ1BFCeg9aN3N3M18EJAAeQCNgs1LqqNb6RIZyChgCHAOe\nALYrpS5qrb83VyD/+9//Ms2Z5OzsTKdOncxVhbASu3YZySk0FEqVgu++g+7djX3fHf2Ot3a8xfN1\nn8fvGT8ZJaIQaNKkCStWrKBEiRJUr17dZJm9e/cyZMgQfHx8AIiLi+PMmTPUrFl05/UqiMyWoJRS\nLoAPUE9rHQ3sVUptAAYDb6ctq7VOOz9EiFJqPdAaMEuCio+P56uvvkp3/8nJyYlx48bJlACFSFwc\nTJkCCxYY6089Bd9/D1WqGOtaa7ac3kL7qu35rtd3FLMpZrFYBdy+fZvg4OB020qWLJnr4wwcOJC5\nc+cydepU3NzcePzxx7l48SLr169n1KhR1KhRg5o1a7J27Vp69uyJnZ0dvr6+xMXJw9gFjTlbUDWB\nJK31qTTbjgJts3uRMr7SPg18nsX+kcBIMCYHDAgIeGAgO3fuJDExMd22xMRE6tSpk6PX50Z0dLTZ\nj1kYmfs8nT7twvvve3HunAs2NpohQ84xaNAFzp3TnDt3fzr7l0u/TELJBPbv3W+2uvNSZGQkSUlJ\nhe4zFR4eTmBgII0bN063vU2bNqmtm7Tv+cSJE5QtWzZ1PWOZDz74gE8++YRevXpx584dypQpQ6NG\njfjzzz+5dOkS/fr1Y86cOalzsfXt2xcvLy/Cw8NTj2Gq3sKoQP+NysmcHDlZMJJMeIZtrwABD3id\nL0Yic3hQHTmdD6pRo0YaSF2UUrpnz545naokV6xxrhVrZK7zlJio9UcfaW1nZ8zfVKOG1r/9lr7M\nX9f+0m2/aasv3rpoljrzU0GZD8oayP+9nLHG84QF5oOKBopn2FYciDJRFgCl1OsY96Ke1lqbZZiH\n48ePExISkm6bs7OzdI4oBI4dg1degYMHjfXRo2HOHEh7n/vS7Ut08e9CXGIc8YkycogQBZk5b8ic\nAmyVUmkfJmgIZOwgAYBSagTGvakOWutQcwXh5+eXqadO2bJlad1aHsosqGJjjekxmjY1klPFikZv\nvU8+SZ+cbsbe5Jllz3Az9iZbB27lidJPWC5oIcQjM1uC0lrfAdYA7yqlXJRSrYGewNKMZZVSA4FZ\nQCet9T/miuHOnTssX748Xe89Z2dnJkyYIL23Cqiff4b69Y3nm5KS4LXX4M8/oVu39OVi78by3PfP\ncer6Kdb1X0fj8o1NH1AIUWCYu0vbGMAJuAqsAEZrrU8opZ5WSkWnKfceUAY4pJSKTlk+e9TKly9f\nnqmXXnJycpGbkK8wCAszuo537AhnzkDduvDrr8YsuMUzXkgGohKiiE6IZmnvpbSv2j7/AxZCmJ1Z\nn4PSWt8AepnYHgi4plmvas56U47JnDlzuHPnTuo2GxsbfHx8KFGihLmrE3kkLs7oNj5rFkRHg4MD\nTJ8Ob70F9vaZy2utSdbJlHMpx6FXDmFrIzPICFFYFJqHgoKCgrh8+XK6bY6OjowfP95CEYnc0Bp+\n/BG8vIxnm6KjoWdPOH4cpk41nZwA/hPwH3x+8CEhKUGSkxCFTKFJUPPmzSM2NjbdtsqVK9OkSRML\nRSRyKijIGD+vb184exbq1TMGeF23DrIYKACARQcXMfOXmZR1LoudjV3WBYUQBVKhSFA3b95k/fr1\nqWN1Abi6ukrXcit37Jgxy23z5rBnD5QpY/TM+/136NAh+9euOrGKsVvG8lyt5/is+2fSCUaIQqhQ\nXBNZsmRJps4RWmv69+9voYhEdk6ehBkzYOVKY93JCcaONQZ4LVXqwa/fdXYXg9YOovXjrfne53u5\ntCdEIVXg/2drrZk/f37qRGVgDM8/ePBgq5hGWtz355/w0Ufg729MjWFvbzxs+/bb8NhjOT+Oi50L\nLT1bsvaFtTjZOeVdwEIIiyrwCWrPnj1ERkam22ZnZ8e4ceMsFJFIS2vYuxemTKnH/pTh8GxtjREh\npk6FSrmYNzAqPgo3Bzee8nyK3UN3y2U9IQq5An8Pat68eURHR6fb5uXlRe3atS0UkQDjodo1a6BV\nK7sM3PUAAA7ISURBVGPiwP37y+LoCGPGQEgIfPZZ7pLTlegrNP68MbN/NQbCl+QkROFXoBJUbGws\n27dvT+0MceXKFXbu3JmujKurK5MmTbJEeAK4etW4jFejBvj4wIEDULo0DBlyjgsXYNEiqFYtd8e8\nHX+brsu6cjnqMm0qt8mbwIUQVqdAXeK7fv06Xbt2pVy5cowbN46IiIhMZWxsbOjVK9OzwiIP3buM\n9+mnsHo13JuGq0oVGD8eRoyAQ4fO4e5eJdfHjk+Mp8/KPhy7cowNAzbQwrOFWWMXQlivApWgbG1t\nsbGxITw8nHfffZeEhIR04+7Z29szcuRI7LN6qlOY1aVLsHw5fPstnEgZElgpYzbb0aOhSxco9ghz\nBGqtGbZ+GD+f/Zlve33LszWeNU/gQogCocAlKAcHBxITEzM9lAvGfYnBgwdbILKiIzrauLe0dKkx\nkKsxpRd4eMDLLxudHypXNk9dSimeeeIZmpVvxpCGMp6iEEVNgUtQxbL5Sl6sWDGeeuop+vbty/jx\n4zPN3ikeTnQ0bN1qDEW0YQPc69Fvb2+0lgYNMkYXN2fDNfR2KJ7FPRnaaKj5DiqEKFAKVCcJW1vb\nbHtvxcTEEBcXx/Lly2nSpAlffPFFPkZXuNy4YVy669kT3N2hXz/4/nsjObVubfTCCwszklbv/2/v\n7oOrqu88jr+/uSGRkAdBXAR5EFlYV2oJksJSSoliNWy1ih2llrpluxXXAh1mS32o44zVPux0OqUd\n60ip7BbBYrGlu2DEqusGpR1lYZuorAiliOAU5SmQhEBI8t0/zr2SxDzckBvOubmf18xvcs/J7958\nc+bkfPO753e/v9mpTU4r/ncFYx8Zy2v7X0vdi4pI2km7EVTLe04dOe+885gyZQq33XbbOYiqb2hu\nhqqqYKT03HPB0hYtD/XUqXDzzUHr7iy87lj/9nrmPzOfay69Rms6iWS4tEtQbVfLbSsvL49bbrmF\nxx9/nOzstPr1zrn9+4MaeM8/D7/7Hbz//pnvxWLBWkyzZ8NNN8GwYb0fz+/f/T1zfj2HSUMn8Ztb\nf0NOTJNdRDJZWl3BY7EYpxNzmNuRl5fHfffdx/33368PcrZj794gISXa7t2tvz98OJSVBW3mTDj/\n/HMYW/VeblhzAyMKR1D+xXLyc/K7fpKI9GlplaDMjLy8vFaLEibk5eWxfPly5s6dG0Jk0XPsGGzb\nBlu2nGnvvde6T0EBfOpTcPXVMGtWsBZTWHl9RNEIFk1exLzieVw44MJwghCRSEmrBAVQVFT0kQSV\nn5/Phg0bKC0tDSeokB05Am+8EbStW4NktGPHmSngCUVFMH06zJgBpaVQXBzUxQvT4ROHqW2oZdT5\no/j2Vd8ONxgRiZS0S1ADBw78cOXc7OxsBg4cSEVFBZdffnnIkfW+EyeCOnaJZJRobRYSBoJZdcXF\nwVpLkycHbdw4yIrQvM26hjquX3M9H9R9wFsL3tI9JxFpJe0S1ODBgwHIzc1l1KhRVFRUMHTo0JCj\nSp1Tp4JVZXfuhF27gpZ4vH9/+8/Jy4Px4+GKK+DKK4Nk9PGPQ27uuY29O043nWbOr+ew5b0tPH3L\n00pOIvIRaZegLrroIrKyspgyZQrl5eXk56fPzXT34O24d99t3fbuPfP4wIGPvjWXkJ0NY8YEiahl\nGz26ZyWFzjV3544Nd1C+q5xln13GzX97c9ghiUgEpV2CmjZtGvn5+Sxbtiwy08jr6+HQoSC5tGzv\nv996+y9/OVOFoSNZWUGpoHHjgjZ2bNDGjQv2R+RX7pFHtjzCyqqVPDjjQe4suTPscEQkotLucrdo\n0aKUv2ZzM9TVwfHjUFNzplVXByOew4eDr20fHzkCBw9Op4uPZrVSUBAkmpEjg9by8ciRweeN+kIS\n6sy84nlkWRYLPrEg7FBEJMJSeik0s0HACuBa4BBwn7v/sp1+Bvwr8NX4rhXAPe4dvbkVaGyEd94J\nRiyJduJEctt1dUHSaZmEEo/brHfYTTFycmDw4GDZ8kQbMqT1dmJfUVFPflZ6e2nPS0y5eAqFuYUs\nnLww7HBEJOJS/b/6o0ADMAQoBsrNrMrdt7fpNx+4CZgAOPAC8GdgWWcvXlUV3G/pDQMGBKObwsLg\na0FBkEwuuCBYcC/xNdES22+++TJlZZ8O7fND6WLrka1865VvseATC1hatjTscEQkDVgXg5bkX8hs\nAHAU+Ji774zvWwW85+73tun7B+AX7r48vv1PwB3u3ulqdFlZEz0nZyNZWQ3EYqfIyjrTzmw3xLdP\nfvg4sZ2dXUcsVk8sdoJYrI7s7MTjesyaz+r3rq6u5vxzWXIhDdUU1FA5oZL+J/tT/Mdispv6+HuY\nPVBZWUljYyMlJSVhhxJ5+ttLThSP06ZNm7a5e5cneSqvFOOApkRyiqsCZrTTd3z8ey37jW/vRc1s\nPsGIi379+nHZZWU9DrS5OWidVE1KWlNTE9XV1T1/oT7q1IBT/OmKPxFriDHqlVHUnurR+6l9XmNj\nI+6ucyoJ+ttLTjofp1QmqHzgWJt9x4CCJPoeA/LNzNreh4qPspYDlJSU+NatW1MXcQpUVFRkbAWL\nrrg7U1dMZeDRgfxo/I/40g++FHZIkVdaWkp1dTWVlZVhhxJ5+ttLThSPU7K1UlOZoGqBwjb7CoGa\nJPoWArVdTZKQ9GJmrJq9iuOnjlOzs73TQESkY6ksfLMTyDazsS32TQDaTpAgvm9CEv0kDZ1sPMnP\nt/0cd2fsBWOZNGxS2CGJSBpKWYJy9zpgHfCQmQ0ws2nAjcCqdro/AfyLmV1sZsOAbwC/SFUsEp6m\n5ibmrpvL/Gfm8+r+V8MOR0TSWKpLh34N6A98AKwB7nL37WY23cxa3h3/GbABeAN4EyiP75M05u4s\nfHYh695ax9LrljJ1xNSwQxKRNJbS+b7ufoTg801t979CMDEise3A3fEmfcTDLz/Msm3LuGfaPSz+\nu8VhhyMiaS5Ciy9IOtt9ZDffefk7fHnCl/n+zO+HHY6I9AH6xKSkxJhBY9j8lc1MvGhi0lNIRUQ6\noxGU9MimdzaxdvtaACZfPJl+sX4hRyQifYVGUHLWqg5U8bmnPseIwhHMvmy2kpOIpJRGUHJW9hzd\nQ9mTZRTkFPDs3GeVnEQk5TSCkm47WHeQ61Zfx8nGk2z+x82MLBoZdkgi0gcpQUm3/Wr7r9h3fB8v\n3v4i4/+q3Rq/IiI9pgQl3bZw8kJm/fUsxgwaE3YoItKH6R6UJKXZm1n83GIqDwRVtpWcRKS3KUFJ\nl9ydJc8v4Sev/YQXdr8QdjgikiGUoKRLP/zDD1n66lIWTV7Ekk8uCTscEckQSlDSqZWVK7n7xbu5\ndfyt/Ljsx6oSISLnjBKUdMjdefr/nmbm6Jk8cdMTZJlOFxE5dzSLTzpkZqybs46GpgZys3PDDkdE\nMoz+JZaP2HFoB7OenMXBuoPkxHLIz8nv+kkiIimmEZS0sv/4fq5ddS0NTQ3UNNRw4YALww5JRDKU\nEpR86Gj9UcpWl1F9sppN8zZx6cBLww5JRDKYEpQAUH+6nhvW3MCuI7vYOHcjE4dODDskEclwugcl\nAByuP8yhE4dYPXs1V4++OuxwREQ0gsp07o7jDC8czut3vU5OLCfskEREAI2gMt4D//0A8/5jHo3N\njUpOIhIpSlAZ7Kdbfsp3X/kuubFcYhYLOxwRkVaUoDLU2u1r+frGr3Pj39zIY9c/phJGIhI5KUlQ\nZjbIzH5rZnVmttfMvthJ32+a2ZtmVmNme8zsm6mIQZL30p6XuP23tzNt5DTWfH4N2Vm6FSki0ZOq\nK9OjQAMwBCgGys2syt23t9PXgH8AXgfGAM+b2T53fypFsUgXDKNkWAnrv7Ce/v36hx2OiEi7epyg\nzGwA8HngY+5eC2w2s/XA7cC9bfu7+w9abL5tZv8JTAOUoHpZ/el6+vfrz1Wjr2LzJZv1tp6IRFoq\nRlDjgCZ339liXxUwo6snWnCFnA78rJM+84H58c1aM3u7B7H2hsHAobCDSAM6TskbbGY6Vl3TOZWc\nKB6nUcl0SkWCygeOtdl3DChI4rkPEtwH+/eOOrj7cmD52QbX28xsq7uXhB1H1Ok4JU/HKjk6TslJ\n5+PU5SQJM6swM++gbQZqgcI2TysEarp43YUE96I+6+6nzvYXEBGRvqnLEZS7l3b2/fg9qGwzG+vu\nu+K7JwDtTZBIPOcrBPenPu3u+5MPV0REMkWPp5m7ex2wDnjIzAaY2TTgRmBVe/3NbC7wPeAz7v7n\nnv78CIjs248Ro+OUPB2r5Og4JSdtj5O5e89fxGwQ8G/AZ4DDwL3u/sv496YDG909P769BxgOtHxb\nb7W7/3OPAxERkT4jJQlKREQk1VTqSEREIkkJSkREIkkJKsXMbKyZnTSz1WHHEjVmlmtmK+L1GmvM\n7I9mNivsuKKiOzUtM5XOoe5L52uSElTqPQr8T9hBRFQ2sI+gykgR8ACw1swuCTGmKGlZ03Iu8JiZ\njQ83pMjROdR9aXtNUoJKITP7AlAN/FfYsUSRu9e5+4Pu/o67N7v7M8AeYFLYsYWtRU3LB9y91t03\nA4malhKnc6h70v2apASVImZWCDwEfCPsWNKFmQ0hqOXY4Ye6M0hHNS01guqEzqGO9YVrkhJU6jwM\nrHD3fWEHkg7MrB/wJLDS3XeEHU8E9KSmZUbSOdSltL8mKUEloat6hGZWDFwDLA071jAlUbcx0S+L\noNJIA7AwtICj5axqWmYqnUOd6yvXJC2lmoQk6hEuBi4B3o2vsZQPxMzscne/stcDjIiujhN8uMTK\nCoKJAH/v7qd7O640sZNu1rTMVDqHklJKH7gmqZJECphZHq3/+11CcHLc5e4HQwkqosxsGcGqy9fE\nF7iUODN7CnDgqwTH6Fngkx2sTJ2xdA51ra9ckzSCSgF3PwGcSGybWS1wMp1OhHPBzEYBdxLUYTzQ\nYkXfO939ydACi46vEdS0/ICgpuVdSk6t6RxKTl+5JmkEJSIikaRJEiIiEklKUCIiEklKUCIiEklK\nUCIiEklKUCIiEklKUCIiEklKUCIiEklKUCIiEkn/D2tkNeRZHFMhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a298cbf28>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "z = np.linspace(-5, 5, 200)\n",
    "\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([-5, 5], [1, 1], 'k--')\n",
    "plt.plot([0, 0], [-0.2, 1.2], 'k-')\n",
    "plt.plot([-5, 5], [-3/4, 7/4], 'g--')\n",
    "\n",
    "plt.plot(z, logit(z), 'b-', linewidth=2)\n",
    "\n",
    "props = dict(facecolor='black', shrink=0.1)\n",
    "\n",
    "plt.annotate('Saturating', xytext=(3.5, 0.7), xy=(5, 1),\n",
    "             arrowprops=props, fontsize=14, ha='center')\n",
    "plt.annotate('Saturating', xytext=(-3.5, 0.3), xy=(-5, 0), \n",
    "             arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.annotate('Linear', xytext=(2, 0.2), xy=(0, 0.5), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.grid(True)\n",
    "plt.title(\"Sigmoid activation function\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.2, 1.2])\n",
    "\n",
    "save_fig(\"sigmoid_saturation_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Xavier and He Initializations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Constant',\n",
       " 'GlorotNormal',\n",
       " 'GlorotUniform',\n",
       " 'Identity',\n",
       " 'Initializer',\n",
       " 'Ones',\n",
       " 'Orthogonal',\n",
       " 'RandomNormal',\n",
       " 'RandomUniform',\n",
       " 'TruncatedNormal',\n",
       " 'VarianceScaling',\n",
       " 'Zeros',\n",
       " 'constant',\n",
       " 'deserialize',\n",
       " 'get',\n",
       " 'glorot_normal',\n",
       " 'glorot_uniform',\n",
       " 'he_normal',\n",
       " 'he_uniform',\n",
       " 'identity',\n",
       " 'lecun_normal',\n",
       " 'lecun_uniform',\n",
       " 'ones',\n",
       " 'orthogonal',\n",
       " 'serialize',\n",
       " 'zeros']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[name for name in dir(keras.initializers) if not name.startswith('_')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.layers.core.Dense at 0x1a29a9d828>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keras.layers.Dense(10, activation='relu', kernel_initializer='he_normal')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "init = keras.initializers.VarianceScaling(scale=2.,\n",
    "                                          mode='fan_avg',\n",
    "                                          distribution='uniform')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.layers.core.Dense at 0x1a336262e8>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keras.layers.Dense(10, activation='relu', kernel_initializer=init)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Nonsaturating Activation Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Leaky ReLU:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def leaky_relu(z, alpha=0.01):\n",
    "    return np.maximum(alpha * z, z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure leaky_relu_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8FPX9x/HXh3AlHCJyVMGCeKDgBcSTXzEepd6ioIJo\nxQNQq9YDrVcFxLNiLSoqWBQRQZRT8WgVjIpXBcVboBQUUAGBhISQAMn398d3kRBy7CbZzOzm/Xw8\n9sEek533Dpt9Z2a+O2POOURERMKmTtABRERESqOCEhGRUFJBiYhIKKmgREQklFRQIiISSiooEREJ\nJRWUlMnMMs3ssaBzJAMzyzAzZ2YtamBey81sSA3M50Az+9DM8s1sebznF0UeZ2Z9gs4h1UcFlaDM\nbLyZzQ46R6wipecily1mttTM7jOzBjE+zwAzy61gPruUa0U/Vx3KKIgPgD2BddU4n2Fm9lUpDx0B\nPF5d8ynH3UAecGBknjWinPf+nsArNZVD4q9u0AGkVnoGuA2oj/9geyZy/62BJYoz59wW4Ocamtfa\nmpgPsB8wyzm3vIbmVy7nXI0sX6k5WoNKUma2m5mNNbM1ZpZjZu+YWXqxx/cws8lmttLMNpvZ12Z2\nSQXPeaKZZZnZYDPrYWZbzew3Jaa5x8y+qCBennPuZ+fcD865acCbQM8Sz9PGzF4wsw2Ry6tmtn+M\ni6FSzOx+M1sUWS7LzexvZtawxDSnmdnHkWnWmdkrZtbQzDKBdsCD29cUI9P/uokv8n+z2czOKPGc\nPSPLtFVFOcxsADAU6FxsjXRA5LGd1uDM7LdmNiPyPsgxs+lm1rbY48PM7Csz6xtZo80xs5nlbY6M\nvK7DgDsj8x5mZu0j19NLTrt901uxaXqb2Ztmlmdm35jZ70v8zIFm9rKZZZtZbmRT4iFmNgy4GDit\n2OvOKDmfyO1DzOytyPJbH1nz2q3Y4+PNbLaZ/dnMVkXeZ8+YWVpZr1tqlgoqCZmZAa8CbYDTgS7A\nu8BcM9szMllD4NPI452BUcAYMzuxjOfsDcwABjnnxjjn3gWWAn8sNk2dyO1xMWQ9DOgObC12Xxrw\nNpAPHAccA/wEvFVDHx6bgEuBg4CrgL7A7cXynQzMwhdrN+B44B3879M5wErgLvwmpz0pwTmXDcwG\n+pd4qD/wb+fcmihyTAEeAhYVm8+UkvOKvBdmAq2BEyJZ9wJmRh7brj1wPnA2/o+FLsA9ZSwfIvNb\nFMmwJzCynGlLcw/wCL7kPgFeMLPGkcx7AfMAB/we6AqMBlIi83kReKvY6/6glNedBrwB5AJHRl7X\nscDTJSb9HXAwcBI7Xv+fY3wtEi/OOV0S8AKMB2aX8dgJ+F/M1BL3LwRuLuc5XwD+Wex2JvAYMAjI\nBnqWmH4I8G2x26cABcAe5cwjE9gSyVeA/xAqBHoXm+ZSYAlgxe5Lwe+/OS9yewCQW8F8Hivl/nJ/\nroznugL4b7Hb7wMvlDP9cmBIifsyIq+1ReT2Wfj9N00it1OBjUC/GHIMA74qb/74D/hCoH2xxzsA\nRcBJxZ4nH9it2DS3F59XGXm+AoYVu90+8hrTS0zngD4lphlc7PE2kfv+L3L7HuB7oH4s7/0S8xkY\nec82KeX/YL9iz7MCqFtsmqeAtyrzO6lL9V+0BpWcugFpwNrI5pFc8wMDDgb2BTCzFDO73cy+iGyi\nysX/9f/bEs91Fv6v15Odc/8u8dizQAczOzZy+1JgpnOuooEAU4DD8WtGLwJPOb+pr3j+fYCcYtmz\ngd23548nM+tjZvPM7OfIvB9m5+XSBZhTxdm8hi+osyO3zwQMv2YWbY5oHAT86IrtJ3LO/Q/4EehU\nbLrvnV+z2+5HoFWM84pF8c3AP0b+3T6/LsA85/fbVdZBwBfOuZxi932AL+bir/sb59y2Elni+bol\nBhokkZzqAKvxmy9K2hj5dwhwI35zxpf4NZp72fWX8wv8X52XmdlHLvJnJvid8Wb2MnCpmS3Cf8ie\nQcWynXP/BTCzC4GvzWyAc258sfwL8Zu0SlofxfODf527lXJ/M3zZlcrMjsavSQ4Hrgey8K8r1k1Y\n5XLObTWzl/Cb9SZE/p3unMur5hyG//8rNUax61tLeSzWP2CLis3TXzGrV8a0v87POeciWxu3z89K\n/YnY1OTrljhRQSWnT/H7HIoify2X5v+AV5xzz8Gv+yoOwH8QFrcMuAa/yWysmQ0qXlL4TSJTgf/h\nS/GtWIJGPqjvBe4zsxcjH9CfAv2AX5xzJfNEaxFwqplZibxdI4+VpTuwyjk3YvsdZtauxDSfASfi\nX3tptuA3SVZkIvCOmXUCTgZOizFHNPP5BmhjZu23r0WZWQf8fqhvosgYi+2jB4vvdzu8Es/zKXCh\nmdUvYy0q2td9qZk1KbYWdSy+fL6tRCYJgP5SSGxNzezwEpf2+JJ4H5hlZqeY2T5mdoyZDTez7WtV\ni4ETzez/zOxA/L6mfUqbSaTkjsd/iI4tsXP9Tfy+oaHAM865olKeoiKT8H+5Xh25/Ty+7GaZ2XGR\n/D3M7CHbeSRfnVJe/8GRx57A72t51MwOM7OOZnY9vvjKWwtZjP9A729mHczsysjPFHcPcK6Z3W1m\nncyss5ldX2wAx3Lgd+ZHIpY5Es459z5+X8sk4Bdgbow5lgPtzKyr+dGBpX2X7C3gc+B5M+tmfoTd\n8/gSmFvK9JXmnNsMfAT8JbJMjqVya56PA42BF83sCDPbz8z6mdn2slsOHBz5P21Rxlra8/hBJhPM\nj+brAYzBr6X+txKZJAAqqMT2O/xf88UvIyNrDKfiP4Cewq8xvAh0ZMf2/ruB/wCv40f4bcL/UpfK\nObcUv5P5ZPxoP4vc7/DfY6rHju8zxSTyV/JjwM2Rv3jzgB74tbKXgO/w+7t2BzYU+9HUUl5/ZuQ5\n/xd5jv2Bf0dea1/gXOfca+VkeQV4EPgHfvPm74E7S0zzGn7f0SmReb6DL/Dt5XwnsDd+lGNF30l6\nHj+SbbJzrjCWHMA0/L6sOZH5lCyw7f8/vSKPZ+JHR/4M9CqxZlldLo38+wm+EO6I9Qmcc6vw/3f1\n8Xk/w6/Fb99X9BR+LWg+/nV1L+U58oA/AE3x//ezgA+L5ZMEYPF5j0ptYmZP4EdG/b7CiUVEoqR9\nUFJp5r/02A3/3afzAo4jIklGBSVVMQv/JchxzrlXgw4jIslFm/hERCSUNEhCRERCKW6b+Fq0aOHa\nt28fr6evkk2bNtGoUaOgYyQkLbvYLVq0iMLCQjp16lTxxLITvd8qr6xlt2wZrF8PDRrAQQdBSjTf\n2KtmCxYs+MU517Ki6eJWUO3bt2f+/PnxevoqyczMJCMjI+gYCUnLLnYZGRlkZWWF9vchzPR+q7zS\nlt1DD8GQIdCoEXz8MXTuHEw2M/s+mum0iU9EpBZ48024+WZ/fcKE4MopFiooEZEk97//wfnnQ1ER\n/PWvcM45QSeKjgpKRCSJbdoEvXrBhg1w+ukwbFjQiaKnghIRSVLOwSWXwJdfQseOMHEi1EmgT/0E\niioiIrF44AF46SVo0gRmzoTdSjsBTYjFVFBmtr+Z5ZvZxHgFEhGRqvv44+bcdpu//vzzcOCBweap\njFjXoEbjj1IsIiIhtWQJ3H33QTgHw4fDGdGcRjSEoi4oM+uLP5ldVU91LSIicZKT4wdF5ObWo1cv\nuCPmE56ER1Rf1DWzpsBd+LOIXlbOdIOAQQCtW7cmMzOzGiJWv9zc3NBmCzstu9hlZWVRWFio5VYJ\ner/FpqgIhg7tzDfftGTvvXMYOHAh775bWPEPhlS0R5IYgT9i9YqdT6a6M+fcWGAsQHp6ugvrN8D1\n7fTK07KLXbNmzcjKytJyqwS932IzYgTMm+cHQ9x77zeceurvKv6hEKuwoCKnWT4J6BL/OCIiUhmv\nvAJDh4IZTJ4Mqambg45UZdGsQWUA7YEfImtPjYEUM+vknOsav2giIhKN776DCy/033u691445RRI\nhi2j0RTUWOCFYreH4AvryngEEhGR6GVn+0ERGzdCnz5wyy1BJ6o+FRaUcy4PyNt+28xygXzn3Np4\nBhMRkfIVFfk1p0WL4JBD4Jln/Ca+ZBHz6Tacc8PikENERGI0fDjMng277+6PFNG4cdCJqpcOdSQi\nkoBmzIC77vLH1nvhBejQIehE1U8FJSKSYL7+Gv74R3/9gQegZ89g88SLCkpEJIFs2LD9SBHQrx/c\neGPQieJHBSUikiAKC6F/f/jvf+Hww+Gf/0yuQRElqaBERBLEX/8Kr78Oe+zh90GlpQWdKL5UUCIi\nCeDFF+G++yAlxV9v3z7oRPGnghIRCbkvvvBnxgV46CE44YRg89QUFZSISIitX+8HReTl+ZF7114b\ndKKao4ISEQmpbdugb19Ytgy6dYMnn0zuQRElqaBERELq1lvhzTehZUs/KCI1NehENUsFJSISQpMm\nwciRULcuTJ0Ke+8ddKKap4ISEQmZzz6Dyy/31//xD+jRI9g8QVFBiYiEyNq1flDE5s1w6aVw1VVB\nJwqOCkpEJCS2boXzz4cffoCjjoLRo2vXoIiSVFAiIiFx003w9tvwm9/AtGnQsGHQiYKlghIRCYEJ\nE2DUKKhXz5dTmzZBJwqeCkpEJGDz58OgQf76Y4/BsccGmycsVFAiIgFavRrOPhsKCmDw4B1FJSoo\nEZHAbNkC554LK1dC9+7wyCNBJwoXFZSISEBuuAHeew/22st/Gbd+/aAThYsKSkQkAOPG+WHk9evD\n9Ol+5J7sTAUlIlLDPvpoxxdwn3zSf+dJdqWCEhGpQT/9BOec4/c/XX31jvM8ya5UUCIiNaSgAHr3\n9iXVowf8/e9BJwo3FZSISA259lr48ENo2xZeesl/KVfKpoISEakBY8bA2LH+8EUzZ0KrVkEnCj8V\nlIhInL3/Plxzjb8+dqw/O65UTAUlIhJHq1b5/U5bt8J118FFFwWdKHGooERE4iQ/34/YW70aTjgB\nHnww6ESJRQUlIhIHzsGVV8J//gPt2sGUKf707RI9FZSISByMHg3jx0Nqqh8U0aJF0IkSjwpKRKSa\nvfMOXH+9v/7003D44cHmSVQqKBGRavTDD/4I5du2+TPk9u0bdKLEpYISEakmmzf7czutXQs9e8J9\n9wWdKLGpoEREqoFz/mSDn34KHTrA5MmQkhJ0qsSmghIRqQajRsHEidCokR8U0bx50IkSnwpKRKSK\n5syBIUP89fHj4ZBDAo2TNFRQIiJVsGwZnH8+FBbCbbdBnz5BJ0oeKigRkUrKy/ODItatg1NPhbvu\nCjpRcomqoMxsopn9ZGYbzWyxmV0e72AiImHmHFx2GXz+Oey/Pzz/vAZFVLdo16DuA9o755oCZwJ3\nm5mOxysitdbIkfDCC9C4sR8U0axZ0ImST1QF5Zz72jlXsP1m5LJv3FKJiITYv/4Ft9zirz/3HHTq\nFGyeZBX1oQvN7HFgAJAKfAa8Vso0g4BBAK1btyYzM7NaQla33Nzc0GYLOy272GVlZVFYWKjlVglh\nfL+tWtWQK67oRlFRPS6+eDnNmi0nZBGBcC67WJlzLvqJzVKAY4AM4AHn3Naypk1PT3fz58+vcsB4\nyMzMJCMjI+gYCUnLLnYZGRlkZWWxcOHCoKMknLC933Jz4Zhj4Kuv4MwzYcYMqBPSoWZhW3bFmdkC\n51x6RdPFtGidc4XOuXlAW+DKyoYTEUk0zsGAAb6cDjzQb9oLazkli8ou3rpoH5SI1CL33QfTpkHT\npn5QRNOmQSdKfhUWlJm1MrO+ZtbYzFLM7A9AP2Bu/OOJiATv1VfhjjvADCZNgo4dg05UO0QzSMLh\nN+c9iS+074HrnHOz4hlMRCQMFi+G/v39Jr4RI+C004JOVHtUWFDOubXAcTWQRUQkVDZuhF69IDsb\nzjnHH8pIao528YmIlKKoCP74R/j2W+jc2R8EVoMiapYWt4hIKUaMgFmz/BEiZs6EJk2CTlT7qKBE\nREp4+WUYNswPipg8GfbbL+hEtZMKSkSkmG+/hQsv9Nfvuw9OPjnYPLWZCkpEJCIrC846C3Jy4Lzz\n4Oabg05Uu6mgRETwgyIuvBCWLIFDD4Wnn/ab+CQ4KigREWDoUP+F3ObN/aCIRo2CTiQqKBGp9aZN\ng7vv9sPIp0yBffYJOpGACkpEarmvvoKLL/bXH3wQTjop2DyygwpKRGqtDRv8kSI2bfKHM7r++qAT\nSXEqKBGplQoLoV8/WLoUunSBsWM1KCJsVFAiUivdfrs/dXuLFv7Eg2lpQSeSklRQIlLrTJkCDzwA\nKSnw0kvQrl3QiaQ0KigRqVU+/xwuucRff/hhCOlZ0QUVlIjUIuvW+UERmzf7kXtXXx10IimPCkpE\naoVt2+D882H5cjjiCHjySQ2KCDsVlIjUCn/5C8yZA61awfTp0LBh0ImkIiooEUl6EyfC3/8Odev6\no0a0bRt0IomGCkpEktqnn8LAgf76I4/A//1fsHkkeiooEUlaa9b4QRH5+XD55XDFFUEnkliooEQk\nKW3d6s/ptGIFHH00PPaYBkUkGhWUiCSlG2+Ed96BPff0+50aNAg6kcRKBSUiSWf8eHj0UahXz5fT\nXnsFnUgqQwUlIknlP//Zsa/p8cfhmGOCzSOVp4ISkaTx889wzjlQUABXXukHRkjiUkGJSFLYsgX6\n9IFVq/xQ8n/8I+hEUlUqKBFJCtddB++/D23awNSpUL9+0ImkqlRQIpLwnnoKnnjCj9SbMQNatw46\nkVQHFZSIJLQPPoA//clff/JJfyBYSQ4qKBFJWD/+CL17+y/lXnstDBgQdCKpTiooEUlIBQW+nH7+\nGY47DkaODDqRVDcVlIgkHOf8Zr2PPoLf/taftr1evaBTSXVTQYlIwnnySRg3zp/TacYMaNky6EQS\nDyooEUko773n9zcB/POf0LVrsHkkflRQIpIwVq70X8bdtg1uuAH69w86kcSTCkpEEkJ+Ppx9tj/H\n04knwgMPBJ1I4k0FJSKh55w/AOz8+dC+PUyZ4k/fLslNBSUioffoo/Dss5CWBjNnwh57BJ1IaoIK\nSkRCLTPT728CeOYZOOywQONIDaqwoMysgZmNM7PvzSzHzD4zs1NqIpyI1G4//9yAc8+FwkL4y1/8\nKdyl9ohmDaousAI4DtgN+Cvwopm1j18sEant8vLgzjsP5pdf4A9/gHvuCTqR1LQKdzM65zYBw4rd\nNdvMlgHdgOXxiSUitZlzMHAgLFnShH33hcmTISUl6FRS02IeB2NmrYEDgK9LeWwQMAigdevWZGZm\nVjVfXOTm5oY2W9hp2cUuKyuLwsJCLbcYvPhiWyZN2o+GDbdx++2f8fnnm4KOlHCS4XfVnHPRT2xW\nD3gdWOqcG1zetOnp6W7+/PlVjBcfmZmZZGRkBB0jIWnZxS4jI4OsrCwWLlwYdJSE8NZbfpNeUREM\nH/4Vd955cNCRElKYf1fNbIFzLr2i6aIexWdmdYDngC3A1VXIJiJSqv/9D84/35fTHXdAjx6/BB1J\nAhRVQZmZAeOA1kBv59zWuKYSkVpn0ybo1QvWr4fTT4fhw4NOJEGLdh/UE8BBwEnOuc1xzCMitZBz\ncOml8OWXcMABMHEi1NG3NGu9aL4H1Q4YDBwO/GxmuZGLDtMoItXib3+DF1+EJk38kSJ22y3oRBIG\n0Qwz/x6wGsgiIrXQG2/Arbf66xMnwkEHBZtHwkMr0SISmP/+F/r185v4hg+HM88MOpGEiQpKRAKR\nk+MHRWRl+X/vuCPoRBI2KigRqXFFRXDxxfD1136T3rPPalCE7EpvCRGpcffeCzNm+MEQM2dC06ZB\nJ5IwUkGJSI2aPRvuvBPMYNIkP6xcpDQ6J6WI1JhFi6B/fz8o4p574NRTg04kYaY1KBGpEdnZcNZZ\nsHEj9OmzY2i5SFlUUCISd0VFcNFFfg3q4IP9mXFN366UCqigRCTuhg+HV16B3Xf3gyIaNw46kSQC\nFZSIxNXMmXDXXX4Y+QsvwL77Bp1IEoUKSkTi5ptv/KY9gPvvh549g80jiUUFJSJxkZXlB0Xk5kLf\nvjBkSNCJJNGooESk2hUWwgUX+GPtHX44jBunQRESOxWUiFS7O++E11+HPfbwR4xISws6kSQiFZSI\nVKuXXvKHMkpJgSlToH37oBNJolJBiUi1+eILGDDAXx85Ek48MdA4kuBUUCJSLdav96fNyMvzI/f+\n/OegE0miU0GJSJVt2+ZH6i1bBt26wZgxGhQhVaeCEpEqu+02ePNNaNkSpk+H1NSgE0kyUEGJSJVM\nngwPPgh168LUqfDb3wadSJKFCkpEKm3hQrjsMn/9H/+AHj2CzSPJRQUlIpXyyy9+UMTmzXDJJXDV\nVUEnkmSjghKRmG3bBuedB99/D0ceCY8/rkERUv1UUCISs5tugrffhtat/aCIhg2DTiTJSAUlIjF5\n7jm/v6lePZg2Ddq0CTqRJCsVlIhEbf58GDjQX3/0UejePdg8ktxUUCISldWr4eyzoaAABg2CwYOD\nTiTJTgUlIhXauhXOPRdWroRjj4VHHgk6kdQGKigRqdD118N778Fee/kv4zZoEHQiqQ1UUCJSrqef\nhtGjoX59P2Jvzz2DTiS1hQpKRMr08cdw5ZX++hNPwFFHBZtHahcVlIiU6qef4JxzYMsW+NOf4NJL\ng04ktY0KSkR2sWUL9OkDP/7oj6/38MNBJ5LaSAUlIru49lr44ANo29afwr1evaATSW2kghKRnYwZ\n4y8NGsCMGdCqVdCJpLZSQYnIr95/H665xl8fOxbS04PNI7WbCkpEAFi1Cnr39l/Kve46+OMfg04k\ntZ0KSkTIz/cj9lavhuOP92fIFQlaVAVlZleb2XwzKzCz8XHOJCI1yDk/jPw//4F27WDKFH/6dpGg\nRfs2/BG4G/gDkBq/OCJS0x5/3B8tIjXVD4po2TLoRCJeVAXlnJsOYGbpQNu4JhKRGvPuu35/E8C4\ncdClS7B5RIrTPiiRWmrFCv9l3G3bYMgQ6Ncv6EQiO6vWLc1mNggYBNC6dWsyMzOr8+mrTW5ubmiz\nhZ2WXeyysrIoLCwM1XIrKKjDtdd2Ye3aJqSnr+fkk78kM9MFHWsXer9VXjIsu2otKOfcWGAsQHp6\nusvIyKjOp682mZmZhDVb2GnZxa5Zs2ZkZWWFZrk554eQL14MHTrAv/7VnObNjws6Vqn0fqu8ZFh2\n2sQnUsuMGgUTJ0JaGsycCc2bB51IpHRRrUGZWd3ItClAipk1BLY557bFM5yIVK+5c/3+JoDx4+GQ\nQwKNI1KuaNeg7gA2A7cAF0au3xGvUCJS/ZYvh/POg8JCuPVWfwp3kTCLdpj5MGBYXJOISNzk5UGv\nXrBuHZxyCowYEXQikYppH5RIknMOLrsMPv8c9t8fJk2ClJSgU4lUTAUlkuQeegheeAEaN/aDIpo1\nCzqRSHRUUCJJ7N//hr/8xV+fMAE6dQo2j0gsVFAiSWrpUujbF4qK4M474eyzg04kEhsVlEgSys31\ngyI2bIAzzoChQ4NOJBI7FZRIknEOLrkEvvoKOnb0X8qto990SUB624okmfvvh6lToWlTmDXL/yuS\niFRQIknktdfg9tvBDJ5/3q9BiSQqFVQNyMjI4Oqrrw46hiS5JUvgggv8Jr677oLTTw86kUjVqKCA\nAQMGcLp+myWB5eTAWWdBdrYfrXfbbUEnEqk6FZRIgisq8qfP+PZb/z2nZ5/VoAhJDnobVyA7O5tB\ngwbRqlUrmjRpwnHHHcf8+fN/fXzdunX069ePtm3bkpqaSufOnXnmmWfKfc45c+bQrFkzxowZE+/4\nUgvcffeOI0TMmgVNmgSdSKR6qKDK4ZzjtNNOY9WqVcyePZvPPvuMHj16cMIJJ/DTTz8BkJ+fT9eu\nXZk9ezZff/01f/7znxk8eDBz5swp9TmnTZvG2WefzdixYxk8eHBNvhxJQi+/7L/jZAaTJ8N++wWd\nSKT6VOsZdZPN22+/zcKFC1m7di2pqakAjBgxgldeeYXnnnuOm2++mTZt2nDTTTf9+jODBg1i7ty5\nTJ48mRNPPHGn5xs7diw33XQTU6dOpWfPnjX6WiT5fPcdXHihv37vvXDyycHmEaluKqhyLFiwgLy8\nPFq2bLnT/fn5+SxduhSAwsJC7r//fqZMmcKqVasoKChgy5Ytu5xqedasWYwZM4Z3332XY445pqZe\ngiSp7Gw/KCInx5/Xafvx9kSSiQqqHEVFRbRu3Zr33ntvl8eaRr79OHLkSB566CFGjRrFIYccQuPG\njbnttttYs2bNTtMfeuihmBnjxo3j6KOPxsxq5DVI8ikqgv79YfFif0bcZ57xm/hEko0Kqhxdu3Zl\n9erV1KlThw4dOpQ6zbx58zjjjDO46KKLAL/favHixTQrcU6DffbZh0cffZSMjAwGDRrE2LFjVVJS\nKUOHwquvQvPmfnBEo0ZBJxKJDw2SiNi4cSMLFy7c6bLffvvRvXt3zjrrLF5//XWWLVvGhx9+yNCh\nQ39dqzrggAOYM2cO8+bN47vvvuPqq69m2bJlpc6jQ4cOvP3227zxxhsMGjQI51xNvkRJAtOn+1F7\nderAlClQxt9NIklBBRXx3nvv0aVLl50uN910E6+99honnHACAwcOpGPHjpx33nksWrSIvfbaC4A7\n7riDI488klNOOYUePXrQqFEj+vfvX+Z89t13XzIzM3njjTcYPHiwSkqi9tVX/vtOAH/7G5x0UrB5\nROJNm/iA8ePHM378+DIfHzVqFKNGjSr1sd13353p06eX+/yZmZk73d53331ZsWJFrDGlFtuwwZ8+\nY9MmfzijG24IOpFI/GkNSiTkCguhXz9/AsIuXeCppzQoQmoHFZRIyN1xB/zrX9CiBcyYAWlpQScS\nqRkqKJEQe/FFf36nlBR/vV27oBOJ1JykLqht27YxZswY1q1bF3QUkZh9/rk/My7A3/8Oxx8fbB6R\nmpa0BbVixQqOPPJIrrnmGs4991yNlpOEsm6dHxSRlwcXXwzXXBN0IpGal5QFNWvWLDp37swXX3zB\n1q1b+fjjjxk5cmTQsUSism0b9O0Ly5dDejo8+aQGRUjtlFQFVVBQwBVXXMEFF1xATk4OhYWFAOTl\n5TF06NBtOIEYAAAKaklEQVSdTpMhEla33AJvvQWtWvkv5jZsGHQikWAkTUEtWbKEww47jAkTJpCX\nl7fL4845lixZEkAykeg9/zw89BDUrQtTp8LeewedSCQ4SfFF3YkTJ3LFFVeQl5e3y76mevXq0bRp\nU2bMmMHvfve7gBKKVOzTT+Hyy/31Rx4BvV2ltkvogtq0aRMDBw5k1qxZpa41paWlcdRRR/HSSy+x\nxx57BJBQJDpr18LZZ0N+Plx2GVxxRdCJRIKXsJv4vvzySzp16sSMGTNKLafU1FSGDx/OnDlzVE4S\nalu3wnnnwQ8/wNFHw+jRGhQhAgm4BuWc44knnmDIkCFs3rx5l8cbNGhA8+bNefnll0lPTw8goUhs\nhgyBzEz4zW9g2jRo0CDoRCLhkFAFlZWVxYUXXsjbb79dajmlpaXx+9//ngkTJvx6QkGRMBs/3u9v\nqlfPj9iLHCRfREiggvr4448588wzyc7OpqCgYJfH09LSePjhhxk4cKBOBCgJ4ZNPduxrGj0ajjkm\n2DwiYRP6gioqKuL+++/n7rvvLnWtqWHDhuy5557Mnj2bTp06BZBQJHarV/tBEQUFvqQGDgw6kUj4\nhLqg1qxZQ58+fViwYEGZm/R69+7NmDFjSE1NDSChSOy2bIE+fWDVKujeHco41ZhIrRfoKL5169bx\n6aeflvrY3LlzOfDAA/noo492GaVXp04dGjVqxLhx45gwYYLKSRLKddfBvHnQpo3/Mm79+kEnEgmn\nQAvqqquuonv37ixduvTX+7Zt28Ytt9zC6aefzoYNG9i6detOP5OWlsaBBx7IF198Qd++fWs6skiV\n/POf8MQTfqTe9Ol+5J6IlC6wglq8eDEvv/wyW7Zs4YwzzmDLli2sXLmSo446ikcffbTUTXqpqalc\ndtllfPbZZ3To0CGA1CKV9+GH8Kc/+etPPAFHHhlsHpGwi2oflJk1B8YBPYFfgFudc5OqMuNbbrmF\nrVu3UlRUxPLly+nVqxfz5s0jLy/v14O8/hqybl3S0tKYNGkSp512WlVmKxKIrVvr0Lu33/90zTU7\nzvMkImWLdpDEaGAL0Bo4HHjVzD53zn1dmZl+8803vP76678W0ebNm5k7d26Zw8c7d+7MjBkzaNOm\nTWVmJxKo/HxYtqwRmzfDccf5g8GKSMWsohP5mVkjYANwsHNuceS+54BVzrlbyvq5Jk2auG7dupX6\n2Jdffsn69esrDFenTh3atm1L+/btq/W7TVlZWTRr1qzanq820bLbmXP+/E1lXbZsgZUrFwLQoMHh\ndOvmv5Qr0dH7rfLCvOzeeeedBc65Cg/1E80a1AFA4fZyivgcOK7khGY2CBgE/ijiWVlZuzzZ5s2b\n2bBhQ4UzTUlJoX379jRu3Jjs7OwoYkavsLCw1GxSsWRbds5BYaFV+uJcdH84paQUsd9+2WzapDM7\nxyLZ3m81KRmWXTQF1Rgo2RDZQJOSEzrnxgJjAdLT011pJwjs2bNnhedlateuHfPnz6dFixZRxItd\nZmYmGRkZcXnuZBe2ZVdQAFlZ5V+ys8t+rJSxODFJSYHddoNmzcq+TJ2agVkWCxd+Vj0vuhYJ2/st\nkYR52UW7RSyagsoFSh7YrimQE2MmFixYwLx583Y5Z1NJa9as4csvv+T444+PdRaSYPLzKy6Y8som\nP79q809Jgd1390VSUdGUdmnUqOIjj8+Z47OKSGyiKajFQF0z2985t33V5zAg5gESN954I/lRfKJs\n3ryZ3r17s2jRIlq2bBnrbKSGOBd7wZQsmlLGxcSkbt0dBVP8Em3ZpKXp1BYiYVVhQTnnNpnZdOAu\nM7scP4rvLODYWGb00Ucf8cknn1S49rRdTk4O119/PRMnToxlNhID5yAvL7pNYdsvK1Z0pbBwx+0S\n36OOWb16pRdMtGWTmqqCEUlW0Q4zvwp4GlgDrAOujHWI+Q033FDqiQUbNGhAgwYNKCgooH79+hxw\nwAEcccQRdOvWTZv4KuAcbNoU2z6Xkpdt22Kd685be+vXr7hgyiuahg1VMCJSuqgKyjm3HuhV2Zl8\n8MEHfPjhhzRp0oTCwkIKCwvp0KEDXbt25YgjjuDQQw+lc+fOtGrVqrKzSEjOQW5u5XfwZ2VBie80\nx6xhw9j2uSxduoATT+z2a9k0bFg9y0JEpKQaOZr5Hnvswb333sshhxzCwQcfTLt27ZLinE1FRRUX\nTEVFU1RUtQxpabHvdyk+faxnb83MzKFjx6plFhGJRo0UVMeOHbn11ltrYlYxKSqCnJzK7+DPzq56\nwTRqFPt+l+LT6EjYIpKsQn0+qIoUFsLGjbHvd/n556PJz/c/G+WYjTI1bly5fS/b79dRBUREShdo\nQRUW7lossRTNxo2VnfOOHSdNmsS2Sazk7boJXfEiIuEVt4/X1avhzjvLL5icmL/qu6vddot938t3\n333EH/5wNE2bqmBERMIqbh/PK1fCiBHlT2O2c7nEWjRNmvgjAcQqOzuf5s0r97pERKRmxK2gWrWC\nq66quGDqBHpOXxERCau4FdTee8PQofF6dhERSXZafxERkVBSQYmISCipoEREJJRUUCIiEkoqKBER\nCSUVlIiIhJIKSkREQkkFJSIioaSCEhGRUFJBiYhIKJmr6gmRynpis7XA93F58qprAfwSdIgEpWVX\nOVpulaPlVnlhXnbtnHMtK5oobgUVZmY23zmXHnSORKRlVzlabpWj5VZ5ybDstIlPRERCSQUlIiKh\nVFsLamzQARKYll3laLlVjpZb5SX8squV+6BERCT8ausalIiIhJwKSkREQkkFJSIioaSCAsxsfzPL\nN7OJQWcJOzNrYGbjzOx7M8sxs8/M7JSgc4WVmTU3sxlmtimyzC4IOlPY6T1WPZLhc00F5Y0GPgk6\nRIKoC6wAjgN2A/4KvGhm7QPMFGajgS1Aa6A/8ISZdQ42UujpPVY9Ev5zrdYXlJn1BbKAOUFnSQTO\nuU3OuWHOueXOuSLn3GxgGdAt6GxhY2aNgN7AX51zuc65ecDLwEXBJgs3vceqLlk+12p1QZlZU+Au\n4MagsyQqM2sNHAB8HXSWEDoAKHTOLS523+eA1qBioPdYbJLpc61WFxQwAhjnnFsRdJBEZGb1gOeB\nZ51z3wWdJ4QaA9kl7ssGmgSQJSHpPVYpSfO5lrQFZWaZZubKuMwzs8OBk4CHg84aJhUtt2LT1QGe\nw+9fuTqwwOGWCzQtcV9TICeALAlH77HYJdvnWt2gA8SLcy6jvMfN7DqgPfCDmYH/azfFzDo557rG\nPWBIVbTcAMwvsHH4Hf+nOue2xjtXgloM1DWz/Z1zSyL3HYY2VVVI77FKyyCJPtdq7aGOzCyNnf+6\nHYL/j73SObc2kFAJwsyeBA4HTnLO5QadJ8zM7AXAAZfjl9lrwLHOOZVUOfQeq5xk+1xL2jWoijjn\n8oC87bfNLBfIT8T/xJpkZu2AwUAB8HPkrzSAwc655wMLFl5XAU8Da4B1+A8KlVM59B6rvGT7XKu1\na1AiIhJuSTtIQkREEpsKSkREQkkFJSIioaSCEhGRUFJBiYhIKKmgREQklFRQIiISSiooEREJpf8H\nfTTHcBq1QyYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a33480b00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(z, leaky_relu(z, 0.05), 'b-', linewidth=2)\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([0, 0], [-0.5, 4.2], 'k-')\n",
    "plt.grid(True)\n",
    "props = dict(facecolor='black', shrink=0.1)\n",
    "plt.annotate('Leak', xytext=(-3.5, 0.5), xy=(-5, -0.2), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.title(\"Leaky ReLU activation function\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.5, 4.2])\n",
    "\n",
    "save_fig(\"leaky_relu_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['deserialize',\n",
       " 'elu',\n",
       " 'exponential',\n",
       " 'get',\n",
       " 'hard_sigmoid',\n",
       " 'linear',\n",
       " 'relu',\n",
       " 'selu',\n",
       " 'serialize',\n",
       " 'sigmoid',\n",
       " 'softmax',\n",
       " 'softplus',\n",
       " 'softsign',\n",
       " 'tanh']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[m for m in dir(keras.activations) if not m.startswith('_')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['LeakyReLU', 'PReLU', 'ReLU', 'ThresholdedReLU']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[m for m in dir(keras.layers) if 'relu' in m.lower()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's train a neural network on Fashion MNIST using the Leaky ReLU:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.fashion_mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_full = X_train_full / 255.0\n",
    "X_test = X_test / 255.0\n",
    "X_valid, X_train = X_train_full[:5000], X_train_full[5000:]\n",
    "y_valid, y_train = y_train_full[:5000], y_train_full[5000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(30, kernel_initializer='he_normal'),\n",
    "    keras.layers.LeakyReLU(),\n",
    "    keras.layers.Dense(100, kernel_initializer='he_normal'),\n",
    "    keras.layers.LeakyReLU(),\n",
    "    keras.layers.Dense(10, activation='softmax')\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss='sparse_categorical_crossentropy',\n",
    "              optimizer=keras.optimizers.SGD(lr=1e-3),\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 55000 samples, validate on 5000 samples\n",
      "Epoch 1/10\n",
      "55000/55000 [==============================] - 2s 41us/sample - loss: 1.3667 - accuracy: 0.5693 - val_loss: 0.9051 - val_accuracy: 0.7082\n",
      "Epoch 2/10\n",
      "55000/55000 [==============================] - 2s 41us/sample - loss: 0.8126 - accuracy: 0.7205 - val_loss: 0.7256 - val_accuracy: 0.7500\n",
      "Epoch 3/10\n",
      "55000/55000 [==============================] - 2s 33us/sample - loss: 0.6966 - accuracy: 0.7577 - val_loss: 0.6478 - val_accuracy: 0.7772\n",
      "Epoch 4/10\n",
      "55000/55000 [==============================] - 2s 38us/sample - loss: 0.6344 - accuracy: 0.7808 - val_loss: 0.5995 - val_accuracy: 0.8028\n",
      "Epoch 5/10\n",
      "55000/55000 [==============================] - 2s 33us/sample - loss: 0.5932 - accuracy: 0.7967 - val_loss: 0.5667 - val_accuracy: 0.8096\n",
      "Epoch 6/10\n",
      "55000/55000 [==============================] - 2s 34us/sample - loss: 0.5643 - accuracy: 0.8073 - val_loss: 0.5435 - val_accuracy: 0.8172\n",
      "Epoch 7/10\n",
      "55000/55000 [==============================] - 2s 34us/sample - loss: 0.5426 - accuracy: 0.8142 - val_loss: 0.5250 - val_accuracy: 0.8250\n",
      "Epoch 8/10\n",
      "55000/55000 [==============================] - 2s 32us/sample - loss: 0.5257 - accuracy: 0.8193 - val_loss: 0.5089 - val_accuracy: 0.8324\n",
      "Epoch 9/10\n",
      "55000/55000 [==============================] - 2s 32us/sample - loss: 0.5121 - accuracy: 0.8231 - val_loss: 0.4973 - val_accuracy: 0.8348\n",
      "Epoch 10/10\n",
      "55000/55000 [==============================] - 2s 32us/sample - loss: 0.5010 - accuracy: 0.8258 - val_loss: 0.4912 - val_accuracy: 0.8348\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train, y_train,\n",
    "                    epochs=10,\n",
    "                    validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PReLU:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's try PReLU:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(300, kernel_initializer=\"he_normal\"),\n",
    "    keras.layers.PReLU(),\n",
    "    keras.layers.Dense(100, kernel_initializer=\"he_normal\"),\n",
    "    keras.layers.PReLU(),\n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "              optimizer=keras.optimizers.SGD(lr=1e-3),\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 55000 samples, validate on 5000 samples\n",
      "Epoch 1/10\n",
      "55000/55000 [==============================] - 5s 86us/sample - loss: 1.3460 - accuracy: 0.6233 - val_loss: 0.9251 - val_accuracy: 0.7208\n",
      "Epoch 2/10\n",
      "55000/55000 [==============================] - 5s 84us/sample - loss: 0.8208 - accuracy: 0.7359 - val_loss: 0.7318 - val_accuracy: 0.7628\n",
      "Epoch 3/10\n",
      "55000/55000 [==============================] - 4s 77us/sample - loss: 0.6974 - accuracy: 0.7695 - val_loss: 0.6500 - val_accuracy: 0.7888\n",
      "Epoch 4/10\n",
      "55000/55000 [==============================] - 4s 79us/sample - loss: 0.6339 - accuracy: 0.7903 - val_loss: 0.6001 - val_accuracy: 0.8068\n",
      "Epoch 5/10\n",
      "55000/55000 [==============================] - 4s 80us/sample - loss: 0.5920 - accuracy: 0.8045 - val_loss: 0.5661 - val_accuracy: 0.8174\n",
      "Epoch 6/10\n",
      "55000/55000 [==============================] - 4s 74us/sample - loss: 0.5620 - accuracy: 0.8138 - val_loss: 0.5417 - val_accuracy: 0.8230\n",
      "Epoch 7/10\n",
      "55000/55000 [==============================] - 4s 73us/sample - loss: 0.5393 - accuracy: 0.8203 - val_loss: 0.5218 - val_accuracy: 0.8306\n",
      "Epoch 8/10\n",
      "55000/55000 [==============================] - 4s 73us/sample - loss: 0.5216 - accuracy: 0.8248 - val_loss: 0.5052 - val_accuracy: 0.8340\n",
      "Epoch 9/10\n",
      "55000/55000 [==============================] - 4s 73us/sample - loss: 0.5069 - accuracy: 0.8288 - val_loss: 0.4922 - val_accuracy: 0.8382\n",
      "Epoch 10/10\n",
      "55000/55000 [==============================] - 4s 80us/sample - loss: 0.4949 - accuracy: 0.8323 - val_loss: 0.4848 - val_accuracy: 0.8370\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train, y_train,\n",
    "                    epochs=10,\n",
    "                    validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ELU:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The exponential linear unit(ELU):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def elu(z, alpha=1):\n",
    "    return np.where(z < 0, alpha * (np.exp(z) - 1), z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure elu_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8FPX9x/HXJwkql4KgscplPVDrQSXVaj1StSrgWc9W\nqXhBPdpStfWkP61WrWKFqqC0WhQ8UMAqKOAP/S14oBQEBFq5BIRyHxsIR4Dk+/vju0DIuUkmmdnd\n9/Px2Ec2M7Mzn/1msu+d6zvmnENERCRqssIuQEREpCIKKBERiSQFlIiIRJICSkREIkkBJSIikaSA\nEhGRSFJAiYhIJCmgREQkkhRQkjLMbLCZjU6j5WSZ2QtmttbMnJnl1/cyq6ilQd5zYlktzWylmR3W\nEMurKTMbbmZ3hF2HgKknifRkZoOB6yoY9YVz7oeJ8a2dcxdU8voYMMs5d3uZ4T2AZ51zzQItOLll\n74dfZ+OptJwqln8BMBLIB74B1jnnttXnMhPLjVHmfTfUe04s60n8und9fS+rgmWfAdwFdAYOBq53\nzg0uM81xwATgUOdcQUPXKLvlhF2A1KvxQPcyw+r9A7C+NNSHRQN+KB0OLHfOfdZAy6tUQ71nM2sC\n3ARc2BDLq0AzYBbwSuJRjnNuppl9A1wLPNeAtUkZ2sWX3oqccyvKPNbV90LN7Hwz+9jM1pvZOjMb\nZ2ZHlxpvZnanmc0zsyIzW2pmjyXGDQbOBG5L7PZyZtZh5zgzG21mvRK7iHLKLPc1M3snmTqSWU6p\n+extZv0Sy9xqZp+b2WmlxsfMbICZPWpma8xslZn1NbNK/78Sy38aaJdY9qJS83q27LQ760lmWbVp\n35q+59q+b6ArUAJ8WkGbdDazD81si5nNN7MzzOxKMys3bW055953zt3nnBueqKMy7wI/C2q5UjsK\nKKkPTYF+wEn43VcFwCgz2ysx/lGgD/AY8D3gCmBJYtxvgEnAP4DvJB47x+30JtACOGfnADNrClwM\nDE2yjmSWs9MTwFXADcD3gZnAWDP7TqlprgF2AKcCtwO9E6+pzG+APwJLE8v+QRXTllXdsuravpDc\ne06mlrJOB6a6MscWzOwHwMfA/wHHA58DDwH3J94LZaa/z8wKq3mcXkUd1ZkMnGRmjeswD6kj7eJL\nb+ebWWGZYc855+6uz4U650aU/t3Mrgc24P/hpwO/BXo7515KTDIf/6GJc67AzLYBm51zKyqZ/3oz\nex//4Tg2MfhS/AflqGTqcM59Ut1yEq9pCtwC3OScey8x7JfAWcBtwAOJSf/tnPtD4vlcM7sZOBt4\nvZL3UGBmG4HiqpZfiUqXZWbNqEX7mllt3nON3zfQHlhewfCngFHOuUcSy3sN/7ec6Jz7qILpn8d/\nUanKf6sZX5VlQCP8caoFdZiP1IECKr1NBHqWGdYQB8EPAx4GTgYOwG+pZwHt8MfA9gY+rONihgKD\nzayJc24zPqyGO+e2JllHsg7Df1Dt2s3knCs2s0nAMaWm+6rM65YBB9ZgOTVR1bKOoe7tm+x7rq6W\nijQGVpYeYGYH4besflxq8Db836rc1lOinnVAfe6u3pL4qS2oECmg0ttm59z8Wr52A7BfBcNb4HeV\nVWUU/ttrr8TPHcC/gb0Aq+J1NTE6Md+LzexD/O6+c2tQR7J21lvR6a6lh22vYFxtdqGXUL6NGpX5\nvaplBdG+yb7n6mqpyBqgZZlhO49P/qvUsI7AHOfcJxUWaHYfcF8VywHo4pz7uJppKrN/4ufqWr5e\nAqCAksrMAbqamZU5XnBiYlyFzKwV/gPnNufc/yWGncjude3fQBF+N9C8SmazDciuqjjnXJGZDcdv\nObUGVuBPDU62jqSWg989tg04DX8qOGaWDZwCvFbNa2tjNf64UGknAIuSfH0Q7Vuf73ka0KPMsBb4\nYCtJLKs5/thTVbs+63sX37HAMufcymqnlHqjgEpveyd2n5RW7Jzb+a1wXzPrVGZ83Dm3CBiIP+j9\njJn9DdiKPwPrZ/iTESqzHv8t+WYzWwIcAjyJ33rBObfRzPoDj5lZEX43ZCugs3NuYGIei/DHqzoA\nhfjrgyo642oo/lT6Q4HXykxTZR3JLsc5t8nMBgKPm9kaYCH+GE8uMKCKdqitj4B+ZnYR/otAL6At\nSQZUbdu3zDzq8z2PA/5sZq2cc2sTw6bjt9ruNbNX8X+n5cDhZnaEc65c0NZ2F1/iGN3hiV+z8GdR\ndsL/7b8tNenp7D6+KSHRWXzp7Rz8P3rpx7RS409P/F760RfAOfcNcAZwBPAB/qymq4ErnHPvV7bA\nxAf8VfgzsWbhryPpg/9Wv9O9wJ8Tw/8DjADalBrfF/8N/t/4LYrKjhlNxH9LPoY9z95Lto5kl3M3\n/tv6P/AfpscD5zvnKjrYX1cvlXp8ig+Qt2s4jyDat17es3NuJrvXpZ3DFuK3mG4BZgAb8evuLCDo\na8Ty2L2uN8afKTgNf0YlAGa2D/6km78FvGypIfUkISINyszOB/oDxzjnisOupywzuw242DlX9pim\nNDBtQYlIg3LOjcVv0bapbtqQbAd+FXYRoi0oERGJKG1BiYhIJCmgREQkkkI/zbx169auQ4cOYZdR\nzqZNm2jatGnYZaQUtVny5syZQ3FxMcccU7ZjBqlMqq1fixfDmjWQnQ0dO0LjEPqkiGqbTZ06dY1z\n7oDqpgs9oDp06MCUKVPCLqOcWCxGfn5+2GWkFLVZ8vLz84nH45Fc96MqldavP/wBHn4Y9tkHxo+H\nH/0onDqi2mZmtjiZ6bSLT0QkQM8958MpOxvefDO8cEoHCigRkYC89Rb8KnGC+qBBcGFYt2VMEwoo\nEZEAfPQRXHstOAePPgo33BB2Rakv0IAys6FmttzMNpjZXDO7Kcj5i4hE0bRpcMklsG0b/PrXcM89\nYVeUHoLegnoM6OCc2xe4CHjEzDoHvAwRkchYsAC6dIGNG+Gqq+Dpp8GCuqlMhgs0oJxzs51zOzvj\ndInHYUEuQ0QkKlauhPPO8z/POQdefhmydOAkMIGfZm5mA/D3e2mM7yW4XM/XZtaTxJ1ec3NzicVi\nQZdRZ4WFhZGsK8rUZsmLx+MUFxervWogauvX5s3Z9O7diQULmnPEERv57W+nM2lStPq+jVqb1VS9\n9MVX6uZm+cCfnXNl77q5S15enovitSBRvX4gytRmydt5HdT06dPDLiVlRGn9KiqCbt3gww/hsMPg\n008hNzfsqsqLUpuVZmZTnXN51U1XLxujzrnixK2a2+Dv8SIikhZKSuC663w45ebCuHHRDKd0UN97\nS3PQMSgRSRPOQe/eMGwYNG8OY8b4LSipH4EFlJkdaGZXm1kzM8s2s/Pwtwf/KKhliIiE6fHH4Zln\nYK+94J//hO9/P+yK0luQJ0k4/O685/HBtxjo7Zx7J8BliIiE4sUX4b77/CnkQ4fCWWeFXVH6Cyyg\nnHOrgTODmp+ISFS8+y707OmfP/ssXHFFuPVkCp2xLyJShU8/9RfglpRAnz5w661hV5Q5FFAiIpWY\nPRsuuAC2boWbb4aHHgq7osyigBIRqcCSJXD++RCPw8UXw4AB6sKooSmgRETKWLvWd2G0dCmcdhq8\n/jrkhH5718yjgBIRKWXTJr9b7z//gWOP9SdIhHG7dlFAiYjssn27PyHi88+hXTsYOxZatgy7qsyl\ngBIRwfcScfPN8N570KqV78LokEPCriqzKaBERIB77/W3y2jSxIfUUUeFXZEooEQk4z39NPz5z/5E\niBEj4OSTw65IQAElIhnu1Vfhjjv885de8qeWSzQooEQkY33wAfTo4Z/37Qvdu4dajpShgBKRjPSv\nf8FPfwo7dsCdd/qHRIsCSkQyzty50LWrv+bp2mvhiSfCrkgqooASkYyyfLnvJWLNGn+86aWXIEuf\nhJGkP4uIZIyCAh9KixbBSSfBW29Bo0ZhVyWVUUCJSEbYutV3+vrVV9Cxo7/WqVmzsKuSqiigRCTt\nFRfDNdfAhAlw8MG+l4jWrcOuSqqjgBKRtOYc3HYbjBwJ++3n+9dr3z7sqiQZCigRSWt//CO88ALs\nvTeMGgXHHRd2RZIsBZSIpK0XXoAHH/Rn6b3xBpx+etgVSU0ooEQkLY0cCbfe6p8PHAiXXBJuPVJz\nCigRSTsTJsDPfw4lJX4XX8+eYVcktaGAEpG0MmMGXHQRFBX5LagHHgi7IqktBZSIpI2FC/2FuBs2\nwOWXw1//CmZhVyW1pYASkbSwerXvwmjFCvjxj2HoUMjODrsqqQsFlIikvMJC3/nrvHnQqRO8/bY/\nrVxSmwJKRFLatm1w2WUwZQoceiiMGeMvyJXUp4ASkZRVUgLXX+9vPHjAAf7nQQeFXZUERQElIinJ\nObjrLnjtNd/p65gxcPjhYVclQVJAiUhK6tsXnn7a3y5j5Ejo3DnsiiRoCigRSTkvvwy//71//sor\n8JOfhFuP1A8FlIiklPfegxtv9M/794errw63Hqk/gQWUme1tZi+a2WIz22hm08ysS1DzFxH5/HO4\n4gp/f6d774Vf/zrsiqQ+BbkFlQMsAc4E9gP6AG+aWYcAlyEiGWrx4iZ06wZbtsANN8Cf/hR2RVLf\ncoKakXNuE/BgqUGjzWwh0BlYFNRyRCTzLF0Kv//98axbBxdc4G+joS6M0l+9HYMys1zgSGB2fS1D\nRNLf+vW+f71Vq/bh1FNh2DDICeyrtURZvfyZzawR8CrwsnPu6wrG9wR6AuTm5hKLxeqjjDopLCyM\nZF1RpjZLXjwep7i4WO1VjaKiLO666wRmz96Ptm03cvfdM5g8eUfYZaWMVP+fDDygzCwLGAJsA26v\naBrn3CBgEEBeXp7Lz88Puow6i8ViRLGuKFObJa9FixbE43G1VxV27PBdGM2aBW3aQN++s7jootPC\nLiulpPr/ZKABZWYGvAjkAl2dc9uDnL+IZAbn4Je/hHffhZYtYdw4WLWqKOyypIEFfQxqIHA0cKFz\nbkvA8xaRDNGnD7z4IjRu7K97OuaYsCuSMAR5HVR7oBfQCVhhZoWJxzVBLUNE0t8zz/hTyLOz4c03\n4ZRTwq5IwhLkaeaLAZ34KSK1NmwY/OY3/vnf/+5PKZfMpa6ORCQSxo+H7t398afHH4cePcKuSMKm\ngBKR0H35JVx6KWzfDr177+4IVjKbAkpEQrVgAXTp4m/b/rOfwVNPqZcI8RRQIhKalSvh3HNh1Sp/\ny4zBgyFLn0qSoFVBREKxYYPfcvrmG3+zwREjYK+9wq5KokQBJSINrqjIH3OaNs3fpv3996F587Cr\nkqhRQIlIgyou9mfrffQRHHQQfPABHHhg2FVJFCmgRKTBOOevc3rrLdh3XxgzBg49NOyqJKoUUCLS\nYB59FJ57zh9reucd6NQp7IokyhRQItIg/v53eOABfwr5a69BCneyLQ1EASUi9e6dd6BXL/98wAB/\nGw2R6iigRKReffIJXH01lJTA//yPv42GSDIUUCJSb2bNggsvhK1boWdPH1AiyVJAiUi9WLwYzjsP\n4nF/zdOAAerCSGpGASUigVuzxofTsmVwxhn+pIjs7LCrklSjgBKRQG3a5O/jNGcOHHecP0Fin33C\nrkpSkQJKRAKzfTtccQV88QW0bw9jx0KLFmFXJalKASUigSgpgRtv9L1DtG7tuzA6+OCwq5JUpoAS\nkUDccw8MGQJNm8J778GRR4ZdkaQ6BZSI1Nlf/gJPPgk5Of62GSedFHZFkg4UUCJSJ6++Cnfe6Z8P\nHuzP3hMJggJKRGpt3Djo0cM/f+opuOaaUMuRNKOAEpFamTzZ96m3Ywf87ndwxx1hVyTpRgElIjU2\nZw506+avefrFL+Dxx8OuSNKRAkpEamTZMn+cac0a6NLF30YjS58kUg+0WolI0uJxOP9838/eySf7\nO+M2ahR2VZKuFFAikpQtW+Cii2DmTDjqKH+tU9OmYVcl6UwBJSLVKi72Z+h9/DEccog/e69Vq7Cr\nknSngBKRKjkHt94Kb7/t+9UbOxbatQu7KskECigRqdJDD8GgQb5H8lGj4Nhjw65IMoUCSkQqNXCg\nD6isLBg2DE47LeyKJJMooESkQsOHw223+ecvvOBPkBBpSAooESknFvMnRTgHjzwCN90UdkWSiQIN\nKDO73cymmFmRmQ0Oct4i0jCmT4eLL4Zt2+D22+G++8KuSDJVTsDzWwY8ApwHNA543iJSz775xvcO\nsWEDXHkl9OsHZmFXJZkq0IByzo0EMLM8oE2Q8xaR+rVqle/CaMUKOOsseOUVyM4OuyrJZEFvQSXF\nzHoCPQFyc3OJxWJhlFGlwsLCSNYVZWqz5MXjcYqLiyPTXps3Z3PHHScwf/6+HHHERu64YzqTJhWH\nXdYetH7VXKq3WSgB5ZwbBAwCyMvLc/n5+WGUUaVYLEYU64oytVnyWrRoQTwej0R7bdsGF1zgeyg/\n7DD4+OPm5OaeHnZZ5Wj9qrlUbzOdxSeSwUpK/A0H//d/4cADfRdGublhVyXiKaBEMpRz/iaDr78O\nzZrBmDF+C0okKgLdxWdmOYl5ZgPZZrYPsMM5tyPI5YhI3T3xBPTv72+X8c9/woknhl2RyJ6C3oJ6\nANgC3ANcm3j+QMDLEJE6+sc/4J57/CnkQ4fC2WeHXZFIeUGfZv4g8GCQ8xSRYI0eDTff7J/37++v\ndxKJIh2DEskgkyb5QCouhvvvh1/9KuyKRCqngBLJEP/+N3Tr5u+Me+ON8PDDYVckUjUFlEgGWLLE\n9xKxfr3vlfz559WFkUSfAkokza1bB+efD0uX+vs5vfEG5IRyib5IzSigRNLY5s1w4YV+9973vgfv\nvguN1Y2zpAgFlEia2rEDrroKPvsM2raFsWOhZcuwqxJJngJKJA05Bz17+lPK99/fd2HURvcXkBSj\ngBJJQ/ff7y/GbdwY3nsPjj467IpEak4BJZJm+veHxx7z93IaPhx++MOwKxKpHQWUSBp54w3o3ds/\nf+kl6No13HpE6kIBJZImxo+HX/zCP3/iid3PRVKVAkokDUydCpdeCtu3+1to3HVX2BWJ1J0CSiTF\nzZsHXbpAYSFccw08+aR6iZD0oIASSWErVvgujFavhnPP9cedsvRfLWlCq7JIiioo8F0YLVwIP/gB\njBgBe+0VdlUiwVFAiaSgrVvhkktgxgw44gh/rVOzZmFXJRIsBZRIiikuhu7dIRaD73wHPvgADjgg\n7KpEgqeAEkkhzsGvf+0vwN13X9+/XocOYVclUj8UUCIp5E9/ggEDYO+9fc/kxx8fdkUi9UcBJZIi\n/vY36NPHn6X32mtw5plhVyRSvxRQIingn/+EX/7SPx8wAH7603DrEWkICiiRiJs4Ea6+GkpK4KGH\noFevsCsSaRgKKJEImzkTLroIior8FlSfPmFXJNJwFFAiEbVoke8loqDA79J79ll1YSSZRQElEkFr\n1vhwWr7cnwzx6qv+/k4imUQBJRIxhYXQrRvMnQsnnADvvAP77BN2VSINTwElEiHbt8Pll8Pkyf4C\n3DFjYL/9wq5KJBwKKJGIKCmBG26AceN810UffOC7MhLJVAookYi4+24YOhSaNoX33/edwIpkMgWU\nSAT07esfjRrB229DXl7YFYmETwElErIhQ+B3v/PPX34ZfvKTcOsRiQoFlEiIxozxx50Ann4afvaz\ncOsRiZJAA8rM9jezt81sk5ktNrOfBzl/kXSyeXM2l18OO3b440+9e4ddkUi05AQ8v+eAbUAu0Al4\nz8xmOOdmB7wckZS2eTN8800ziovhuuvgscfCrkgkesw5F8yMzJoC64FjnXNzE8OGAP91zt1T2eua\nN2/uOnfuHEgNQYrH47Ro0SLsMlKK2iw5W7fC5MnTcQ72378Txx6rLoySofWr5qLaZhMmTJjqnKv2\nVKAgt6COBIp3hlPCDKDcXWvMrCfQE6BRo0bE4/EAywhGcXFxJOuKMrVZ9XbsyGLevGY4B1lZjkMO\nKaCgIJgvielO61fNpXqbBRlQzYCCMsMKgOZlJ3TODQIGAeTl5bkpU6YEWEYwYrEY+fn5YZeRUtRm\nVYvHfb9627ZBs2b5dOhQwFdfTQu7rJSh9avmotpmluQugyADqhDYt8ywfYGNAS5DJCUVFEDXrvDV\nV9CxI7RqBZs2actJpCpBnsU3F8gxs9LXv58A6AQJyWjr1/trmyZNgnbtfBdGjRqFXZVI9AUWUM65\nTcBI4I9m1tTMfgRcDAwJahkiqWbNGjj7bPjXv+DQQ2HCBB9SIlK9oC/UvRVoDKwCXgdu0SnmkqlW\nrYKzzoJp03y/ehMm+B7KRSQ5gV4H5ZxbB1wS5DxFUtGCBdClC8ybB0cdBR99pJ7JRWpKXR2JBGzy\nZDjlFB9OnTpBLKZwEqkNBZRIgEaNgvx8WL0azj0XJk6E3NywqxJJTQookQA4B3/9K1xyCWzZAtdf\nD6NHQ/NyVwGKSLIUUCJ1tGUL9OgBv/mNvyvuH/4AL76oU8lF6irozmJFMsrixfDTn8KXX0KTJj6Y\nrr467KpE0oMCSqSWxo2Da6/11zp997v+TrjHHx92VSLpQ7v4RGqoqAjuvBPOP9+H03nn+QtxFU4i\nwdIWlEgNzJnj73o7bRpkZ8PDD8Pvf++fi0iwFFAiSSguhmefhfvu8zcb/O534bXX4OSTw65MJH0p\noESq8Z//wI03+s5eAbp392G1b9m++0UkUDoGJVKJoiJ45BHfG8SkSXDwwfDOO/DKKwonkYagLSiR\nCrz3HvTuDfPn+99vugmefBIiePdskbSlgBIpZd48+O1vfUABHH2035131lnh1iWSibSLTwRYsQJu\nuw2OOcaHU/Pm8Je/wIwZCieRsGgLSjJaPA59+8LTT/uz87KyfD96jz4KBx0UdnUimU0BJRlpzRro\n3x+eeQYKCvywiy+GP/0Jvve9cGsTEU8BJRnlv//1W0sDB/otJoAf/9gH0ymnhFubiOxJASUZYfJk\n6NcP3noLduzww7p2hfvvh1NPDbc2EamYAkrS1qZNMHw4PP88fP65H5adDVdeCXffDSeeGG59IlI1\nBZSkFefgiy/8bS+GDYONG/3wli2hZ09/pl7btuHWKCLJUUBJWvj2W3jzTXjpJd810U6nngo33ODv\n0dS0aXj1iUjNKaAkZS1a5HfhvfWWP8a004EHwnXX+WA66qjQyhOROlJAScpwzl84O2YMjBwJU6bs\nHtekCXTrBj//uf+p262LpD4FlETa+vUwfrwPpbFjYfny3eOaNoULLoArroAuXXxIiUj6UEBJpBQU\nwCefwIQJMHGi30oqLt49/uCD/Z1su3XzPxVKIulLASWhcQ4WL/bHjyZN8qE0YwaUlOyeJicHzjzT\nbyF16QLHHQdm4dUsIg1HASUNwjnfi8NXX/mtosmT/WP16j2na9QIfvhDH0pnnAE/+pHvuFVEMo8C\nSgK3YQPMmgUzZ+75WL++/LStW8MPfgAnnQSnn+67G9JuOxEBBZTUUlERfPONv3/SzsfkySewZg0s\nWVLxa/bf3++i69zZB9JJJ0GHDtplJyIVU0BJhTZs8Be/LllS/ueiRf556WNFXksA9trL31fpuON2\nP44/Hr7zHYWRiCRPAZVBioth7VpYuXLPx6pV/ueKFbB0qQ+fDRuqnldWFnz3u3DEEbsfW7Z8xWWX\nHU/79roOSUTqTgGVgoqKfB9z69f7x7p1Vf9cv96fjLB6dUVbPRVr3BjatfP91lX089BD/ZZSabHY\nOg4/PPj3KyKZKZCAMrPbgR7AccDrzrkeQcw3FTkH27fDli3+sXVr9c83b4bCQh86Gzfufl7ZsO3b\na19fy5aQm7v7cdBBe/5+yCE+gPbfX7vjRCRcQW1BLQMeAc4DGtfkhUVFMHeu3/1U9lFSUvHw6sZV\nN3779t2Pbdv2/Lnz+X//ewz9+1c/3c7nOwNn69bkt1JqKyfHn3rdsqUPkpYt93xe0bBWrXwfdWW3\nekREoiqQgHLOjQQwszygTU1eO2vWHDp2zC8z9ErgVmAz0LWCV/VIPNYAl1cw/hbgKmAJ0L2C8XcC\nFwJzgF4VjH8AOAeYDvSuYPyjwKnAZ8B95cZmZ/ejSZNOZGWNZ+vWR8jKYtcjOxuOO+4FDjigI2vX\njmLOnKfIzmaPxy23DKF9+7ZMnTqMsWMHlhs/cuRwWrduzeDBgxk8eDDbtu0+ngTw/vvv06RJEwYM\nGMBf//pmufpisRgAffv2ZfTo0XuMa9y4MWPGjAHg4Ycf5sMPP9xjfKtWrRgxYgQA9957L5MmTdo1\nLh6Pc+yxxzJ06FAAevfuzfTp0/d4/ZFHHsmgQYMA6NmzJ3Pnzt1jfKdOnejXrx8A1157LUuXLt1j\n/CmnnMJjjz0GwGWXXcbatWv3GH/22WfTp08fALp06cKWLVv2GH/BBRdw1113AZCfn1+uba688kpu\nvfVWNm/eTNeu5de9Hj160KNHD9asWcPll5df92655RauuuoqlixZQvfu5de9O++8kwsvvJDNmzcz\nf/78cjU88MADnHPOOUyfPp3evcuve48++iinnnoqn332GffdV37d69evH506dWL8+PE88sgj5ca/\n8MILdOzYkVGjRvHUU0+VGz9kyBDatm3LsGHDGDhwYLnxw4fvue6VVXrde/PNYNe9kpISJk6cCJRf\n9wDatGmjda/MuhePx2nRogWwe92bM2cOvXqV/9xryHUvWaEcgzKznkBP/1tT9tqrJLE7yWEGzZtv\npWXLjcAmli7dkXjN7l1OBxxQyIEHrqW4eC1z527fNdzMAdCuXZyDD15OUdFKZszYhpkrNQ0cffRq\n2rdfxKZNS5k0aeuu8TtrOO20xbRt+yUbN37DuHGbEuPcrp+XXvofOnZsxMKFsxkxYsOu4VlZ/uev\nfjWFww+PM3XqDIYMiZd7/zfd9AXt2i3ns89mEo+XH9+mzSRatVpATs5sSkrilJTsuVvv008/Zb/9\n9uPrr7+u8PUTJ05kn332Ye7cuRWO3/khsWDBgnLjt2zZsmv8woULy40vKSnZNf7bb7/dY3xxcTEr\nV67cNX7p0qXlXr9s2bJd45ctW1Zu/NKlS3eNX7lyZbnx33777a7xq1evZkOZszkWLly4a/y6deso\nKiraY/zBuSoMAAAF8klEQVSCBQt2ja+obebOnUssFmPr1q0Vjv/666+JxWIUFBRUOH727NnEYjFW\nrVpV4fiZM2fSvHlzNm7ciHOu3DQzZswgJyeH+fPnV/j6L7/8km3btjFr1qwKx0+ZMoV4PM6MGTMq\nHP/FF1+wfPlyZs6seN2bNGkSCxYsYPbs2RWOD3Pda9KkSaXrHkCjRo207pVZ94qLi3c937nuVdR2\n0LDrXrLMOZf0xNXOzOwRoE1NjkHl5eW5KaW7pY6IWCxW4bccqZzaLHn5+fnE4/Fy3/Klclq/ai6q\nbWZmU51zedVNl5XEjGJm5ip5fBJMuSIiInuqdhefcy6/AeoQERHZQ1Cnmeck5pUNZJvZPsAO59yO\nIOYvIiKZp9pdfEl6ANgC3ANcm3j+QEDzFhGRDBTUaeYPAg8GMS8REREIbgtKREQkUAooERGJJAWU\niIhEkgJKREQiSQElIiKRpIASEZFIUkCJiEgkKaBERCSSFFAiIhJJCigREYkkBZSIiESSAkpERCJJ\nASUiIpGkgBIRkUhSQImISCQpoEREJJIUUCIiEkkKKBERiSQFlIiIRJICSkREIkkBJSIikaSAEhGR\nSFJAiYhIJCmgREQkkhRQIiISSQooERGJJAWUiIhEkgJKREQiSQElIiKRpIASEZFIUkCJiEgkKaBE\nRCSS6hxQZra3mb1oZovNbKOZTTOzLkEUJyIimSuILagcYAlwJrAf0Ad408w6BDBvERHJUDl1nYFz\nbhPwYKlBo81sIdAZWFTX+YuISGaqc0CVZWa5wJHA7Cqm6Qn0BMjNzSUWiwVdRp0VFhZGsq4oU5sl\nLx6PU1xcrPaqAa1fNZfqbWbOueBmZtYIGAMscM71SuY1eXl5bsqUKYHVEJRYLEZ+fn7YZaQUtVny\n8vPzicfjTJ8+PexSUobWr5qLapuZ2VTnXF5101V7DMrMYmbmKnl8Umq6LGAIsA24vU7Vi4hIxqt2\nF59zLr+6aczMgBeBXKCrc2573UsTEZFMFtQxqIHA0cA5zrktAc1TREQyWBDXQbUHegGdgBVmVph4\nXFPn6kREJGMFcZr5YsACqEVERGQXdXUkIiKRpIASEZFICvQ6qFoVYLYaWBxqERVrDawJu4gUozar\nGbVXzai9ai6qbdbeOXdAdROFHlBRZWZTkrmQTHZTm9WM2qtm1F41l+ptpl18IiISSQooERGJJAVU\n5QaFXUAKUpvVjNqrZtReNZfSbaZjUCIiEknaghIRkUhSQImISCQpoEREJJIUUEkysyPMbKuZDQ27\nlqgys73N7EUzW2xmG81smpl1CbuuqDGz/c3sbTPblGirn4ddU1RpnaqbVP/cUkAl7zngX2EXEXE5\nwBLgTGA/oA/wppl1CLGmKHoOf2PPXOAaYKCZfS/ckiJL61TdpPTnlgIqCWZ2NRAHPgy7lihzzm1y\nzj3onFvknCtxzo0GFgKdw64tKsysKXAZ0Mc5V+ic+wR4F+gebmXRpHWq9tLhc0sBVQ0z2xf4I3Bn\n2LWkGjPLBY4EZoddS4QcCRQ75+aWGjYD0BZUErROJSddPrcUUNV7GHjRObck7EJSiZk1Al4FXnbO\nfR12PRHSDCgoM6wAaB5CLSlF61SNpMXnVkYHlJnFzMxV8vjEzDoB5wBPh11rFFTXXqWmywKG4I+z\n3B5awdFUCOxbZti+wMYQakkZWqeSl06fW3W+o24qc87lVzXezHoDHYBvzQz8t99sMzvGOXdivRcY\nMdW1F4D5hnoRfwJAV+fc9vquK8XMBXLM7Ajn3LzEsBPQLqtKaZ2qsXzS5HNLXR1VwcyasOe33bvw\nf/hbnHOrQykq4szseaATcI5zrjDseqLIzN4AHHATvq3eB051zimkKqB1qmbS6XMro7egquOc2wxs\n3vm7mRUCW1Ptj9xQzKw90AsoAlYkvr0B9HLOvRpaYdFzK/ASsApYi//gUDhVQOtUzaXT55a2oERE\nJJIy+iQJERGJLgWUiIhEkgJKREQiSQElIiKRpIASEZFIUkCJiEgkKaBERCSSFFAiIhJJ/w/GblHg\nCvYWgwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a3665aac8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(z, elu(z), \"b-\", linewidth=2)\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([-5, 5], [-1, -1], 'k--')\n",
    "plt.plot([0, 0], [-2.2, 3.2], 'k-')\n",
    "plt.grid(True)\n",
    "plt.title(r\"ELU activation function ($\\alpha=1$)\", fontsize=14)\n",
    "plt.axis([-5, 5, -2.2, 3.2])\n",
    "\n",
    "save_fig(\"elu_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implementing ELU in TensorFlow is trivial, just specify the activation function when building each layer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.layers.core.Dense at 0x1a35103dd8>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keras.layers.Dense(10, activation='elu')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SELU:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This activation function was proposed in this great paper by Günter Klambauer, Thomas Unterthiner and Andreas Mayr, published in June 2017. During training, a neural network composed exclusively of a stack of dense layers using the SELU activation function and LeCun initialization will self-normalize: the output of each layer will tend to preserve the same mean and variance during training, which solves the vanishing/exploding gradients problem. As a result, this activation function outperforms the other activation functions very significantly for such neural nets, so you should really try it out. Unfortunately, the self-normalizing property of the SELU activation function is easily broken: you cannot use ℓ1 or ℓ2 regularization, regular dropout, max-norm, skip connections or other non-sequential topologies (so recurrent neural networks won't self-normalize). However, in practice it works quite well with sequential CNNs. If you break self-normalization, SELU will not necessarily outperform other activation functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.special import erfc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# alpha and scale to self normalize with mean 0 and standard deviation 1\n",
    "# (see equation 14 in the paper):\n",
    "alpha_0_1 = -np.sqrt(2 / np.pi) / (erfc(1/np.sqrt(2)) * np.exp(1/2) - 1)\n",
    "scale_0_1 = (1 - erfc(1 / np.sqrt(2)) * np.sqrt(np.e)) * np.sqrt(2 * np.pi) * (2 * erfc(np.sqrt(2))*np.e**2 + np.pi*erfc(1/np.sqrt(2))**2*np.e - 2*(2+np.pi)*erfc(1/np.sqrt(2))*np.sqrt(np.e)+np.pi+2)**(-1/2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def selu(z, scale=scale_0_1, alpha=alpha_0_1):\n",
    "    return scale * elu(z, alpha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure selu_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt4FPXZxvHvEwIIiEZB8iqo8YTnihDbivY1rVgVD7XF\nai2gWC2IVYtAqyIIL1BsLVW0BSyCoKBVqoiKaKu00VrECoJn8VQQROWgCyYcQsLv/eO3McuSkE0y\nm5nd3J/rmivLzGTm2WGy987sszPmnENERCRqcsIuQEREpDoKKBERiSQFlIiIRJICSkREIkkBJSIi\nkaSAEhGRSFJAieyGma0ws6GNsJ5RZvZmI6wnx8z+bGYbzMyZWVG611lLPTPMbF6YNUh0KaAkJWa2\nn5lNir9gbzOzz81sgZmdkTBPcfxFL3l4KGEeZ2YX1rCOfmZWUsO0Gn8vCLsJiJOASQGupyD+XAqT\nJo0HTgtqPbvRE7gcOA/YH1jYCOvEzIriz7t90qRfAn0aowbJPLlhFyAZ41GgNXAF8AHQAf+C2i5p\nvunAsKRxW9JeXZo459Y10npKgGrDOWCHA5865xolmGrjnNsYdg0SXTqCklqZWR7wHeBG59wC59xK\n59wrzrnxzrmHkmbf7Jz7LGlI+4uQmZ1lZv8ysy/N7Asz+5uZHZ00zwFm9kD89NZmM1tmZt81s37A\nSODYhKO+fvHf+foUn5n9xcweTVpmjpmtMrPrU6zjv/Gfr8TXUxz/vZ2O4OLLHRFf9jYze8PMfpAw\nvfJIrJeZPRt/Pm8nHtFWs41mAHcAB8V/d0V8fLGZ/Sl53sRTb/F5JpnZODNbb2ZrzWy8meUkzNMi\nPn1lvOaPzOw6MysA/hmfbV183TNqWE9LM5sQP0LfamaLzOzUhOmVR2Knm9nL8ee92My61vS8JXMp\noCQVle/uzzezPcIupgZtgAnAN4EiYCPwpJm1ADCzNsDzQAHwQ+B4YHT8dx8G/gAsx5/22j8+Ltks\n4Jx4YFc6LT7/X1KpIz4e4Kz47/2ohufzS+BXwA3xWh8D5phZl6T5fgPcBZwAvAI8ZGZ77maZo4HV\n8XWfVMN8NekNlAPdgWuAQcDFCdPvAy4FBgNH44+2Y8AqoFd8nmPj6/5lDeu4Lb7MnwEnAm8Az5jZ\n/knz3QrcCHQFNgAPmJnV8flI1DnnNGiodcC/wHwBbAVewn9m8q2keYqBMqoCrXK4OmEeB1xYwzr6\nASU1TKvx92qYvw1QAZwa//fPga+A9jXMPwp4s5rxK4Ch8ce5wFrgioTpU4G/1aGOgvhzKdzd+oFP\ngFuq2b6zkpYzIGF6x/i4U3dTz1BgRTXL/VPSuBnAvKR5Xkqa51lgavzxEfF1n1XDeovi09vXtJ74\ntioDLk2Y3gz4EBibtJwzE+Y5JT6uU9h/JxqCHXQEJSlxzj0KHID/cP1p/LvoRWaW/HnTw0CXpOGB\ndNdnZoeZ2YNm9qGZbQI+x58hOCg+y4nA68659fVdh3OuHP/8esfX2RIf3LPqUEcqz2Uv/Lb+d9Kk\nF4Fjksa9nvB4Tfxnh1TXVUevJ/17TcK6TgR2UHUqrz4OA5qT8LydcxX4N0RhPm8JiZokJGXOua34\nd83PAqPNbCowyszGO+fK4rNtdM59UM9VbAJamVlz59z2ypEJp9R291nWk/ijjgHxn+XA20DlqbWg\nTv/MAhaaWUfgW/HlP1aHOuqiulsNJI/7ejs551z8LFdd33juYNft07ya+bYn/dslrCuI7Vu5jDo9\n74RpesOdZfQfKg3xNv5NTlCfSy3H75MnJo3vmjB9F2bWDv+Zxzjn3HPOuXeAtuz8BuxV4BvVtDlX\nKsOfTtot59zL+FNOl+CPpOY634GXah2VQV7jupxzm/BHBacmTToVv82Dtg7/uVCiE+q4jFfx/3ff\nrWF6rc8b3x1aRsLzNrNmwMmk53lLxOkISmoVf+H9K3Av/tTKV0Ah8GtgQfwFtVJrM/ufpEWUOee+\nSPh3QTUf9n/knHvLzP4OTDWzwfgg6AzcCcx2zn1cQ4lfAuuBn5vZKvxnMb/HH71UehD/ofpcM7sJ\n3yhwPPCVc+6f+M+aDo53g30cH7+thvU9AFxJVcNFXepYi2+7PzPeRbfVVd/l+Hv8Uer7wBL8d4W+\nA3SroaaG+AcwwczOx78JGAAciN8mKXHOvW9ms/H/d7/EB1YnoMA5NxNYiT/SOcfMngS2VAZ7wjJK\nzWwy8FszW4/veLweyCfA76JJBgn7QzAN0R+AlsA4fJfYl8Bm4H3gdmDfhPmK8S9CycOLCfNUN90B\n58an5+ED6YP4et4DfgfsWUuN3wPexDdxvAmciW/Q6JcwTyf8Z0ix+LKXAkUJz/GR+PNzlb9HQpNE\nwnIOi8/zOZBbjzquxIdgBVAcHzeKnZskcoAR+A64Mnw32wUJ0wuovtlit80kVN8k0RyYiA/X9fhO\nvxns2iRRWyNFS3wX3ifANvwbjGsSpo8APsWfUpyxm2VMiG/bbcAiEpo+qKbZoqZtoSHzB4v/B4uI\niESKPoMSEZFIUkCJiEgkKaBERCSSFFAiIhJJobeZt2/f3hUUFIRdxi5KS0tp06ZN2GVkFG2z1C1f\nvpyKigqOOSb5AglSk6juX2Vl8M47UF4O7dpBlF7OorrNlixZst45t19t84UeUAUFBSxevDjsMnZR\nXFxMUVFR2GVkFG2z1BUVFRGLxSK570dVFPevTZvglFN8OH33u/DMM9CiPtcMSZMobjMAM1uZynw6\nxSciUg/l5XDxxfDmm3DUUfDoo9EKp2yggBIRqSPn4Lrr/BFT+/bw1FOwzz5hV5V9FFAiInU0YQJM\nngwtW8Ljj8Ohh4ZdUXZSQImI1MHcuTBkiH98333QvXu49WSzQAPKzGaZ2admtsnM3jOzK4NcvohI\nmJYsgd69/Sm+sWP9Z1CSPkEfQd2Kv3rxXsD5wFgzS8fVl0VEGtWqVXDeebB5M1x2GQxLvlWnBC7Q\ngHLOveWqblFQeZXqw4Jch4hIY9u0Cc45Bz79FIqKYMoUsKBugSk1Cvx7UGY2CegHtMLfzmB+NfP0\nB/oD5OfnU1xcHHQZDVZSUhLJuqJM2yx1sViMiooKba86CGv/qqgwhg07jjfeaMeBB25m8OBXWbiw\nvPZfjIBM/5tMy+02Eu6CWQT8ziXcvjtZYWGhi+KXFaP6Bbco0zZLXeUXdZctWxZ2KRkjjP3LObjm\nGpg0ybeTL1oEh2XQOaGo/k2a2RLnXGFt86Wli885V+GcexF/g7iB6ViHiEi63XmnD6cWLXz3XiaF\nUzZId5t5LvoMSkQy0OOPw+DB/vGMGf6SRtK4AgsoM+tgZj8xsz3NrJmZnQlcAvwjqHWIiDSGJUvg\npz/1p/jGjIFLLgm7oqYpyCYJhz+ddzc++FYCg5xzjwe4DhGRtEpuJ7/55rAraroCCyjn3DrgtKCW\nJyLS2L76Cs4917eTn3aa2snDpksdiYjgr07+k5/A669D584wZ46uTh42BZSINHnOwaBBMH++v+ng\n/Pmw775hVyUKKBFp8u66CyZOVDt51CigRKRJe/JJuP56/3j6dDj11HDrkSoKKBFpsl591X/u5ByM\nHu1byyU6FFAi0iStXl3VTn7ppTB8eNgVSTIFlIg0OZXt5GvWwP/+r9rJo0oBJSJNSnm5vzLEa6/B\nEUf4dvKWLcOuSqqjgBKRJmXwYHjqqap28nbtwq5IaqKAEpEm46674I9/rGonP/zwsCuS3VFAiUiT\nMG9eVTv5vfeqnTwTKKBEJOstXerbyXfsgFGjoHfvsCuSVCigRCSrrV7tO/ZKS6FPH7jllrArklQp\noEQka5WU+O86VbaTT52qdvJMooASkaxUUeHbyZctUzt5plJAiUhWGjzYN0bsu29VW7lkFgWUiGSd\nP/7Rt5RXtpMfcUTYFUl9KKBEJKs89ZS/txPAtGnwne+EW4/UnwJKRLLGsmVw8cW+nXzkSN+1J5lL\nASUiWeGTT6rayXv39gElmU0BJSIZr7Kd/JNP/BUipk1TO3k2UECJSEarqPA3Gly61F9bb+5ctZNn\nCwWUiGS0IUP8bdvVTp59FFAikrEmToQ774TmzeGxx6Bz57ArkiApoEQkI82fD9dd5x9Pm+YvZSTZ\nRQElIhnntdeq2slvuQX69g27IkkHBZSIZJQ1a3w7eUmJb44YNSrsiiRdFFAikjFKS307+erVcMop\naifPdgooEckIle3kr74Khx3m28n32CPsqiSdFFAikhHuvvswnngC9tnHN0i0bx92RZJuCigRibxJ\nk+CRRw5UO3kTo4ASkUh7+mm49lr/eOpUOO20cOuRxhNYQJlZSzObZmYrzewrM1tqZmcHtXwRaXpe\new0uusi3k/ftu4JLLw27ImlMQR5B5QKrgNOAvYERwGwzKwhwHSLSRCS2k19yCVx++YqwS5JGFlhA\nOedKnXOjnHMrnHM7nHPzgP8C3YJah4g0Dcnt5Pfeq3bypig3XQs2s3ygM/BWNdP6A/0B8vPzKS4u\nTlcZ9VZSUhLJuqJM2yx1sViMiooKba9qVFTAyJHH8eqr7TnggC0MHfoqixZt1/5VD5m+zcw5F/xC\nzZoDTwMfOucG7G7ewsJCt3jx4sBraKji4mKKiorCLiOjaJulrqioiFgsxrJly8IuJXKGDIHbb/ft\n5C+9BEce6cdr/6q7qG4zM1vinCusbb7Au/jMLAeYCZQB1wS9fBHJXpMn+3Bq3hzmzKkKJ2maAj3F\nZ2YGTAPygZ7Oue1BLl9Estczz1S1k99zD0Twjb80sqA/g5oMHA30cM5tCXjZIpKl3njDt5NXVMDN\nN8Nll4VdkURBkN+DOhgYAHQBPjOzkvjQO6h1iEj2+fRTOOcc+OorfwuN0aPDrkiiIrAjKOfcSkCN\noCKSssp28lWroHt3mDEDcnR9G4nTriAioaiogD59YMkSOPRQXZ1cdqWAEpFQ3HCDD6W8PHjqKdhv\nv7ArkqhRQIlIo7v7bvjDHyA3Fx59FI46KuyKJIoUUCLSqP72N7gm/g3JKVPge98Ltx6JLgWUiDSa\nN96AH//Yf/40bBhcfnnYFUmUKaBEpFF89pm/OnllO/mYMWFXJFGngBKRtNu8Gc4/Hz7+GL79bZg+\nXe3kUjvtIiKSVjt2+HbyV16BQw6Bxx+HVq3CrkoygQJKRNLqhhvgscdg7719O3mHDmFXJJlCASUi\naTNlCowf79vJ58yBo48OuyLJJAooEUmLv/8drr7aP/7zn9VOLnWngBKRwL35ZlU7+U03wc9+FnZF\nkokUUCISqM8+81cn37TJh9TYsWFXJJlKASUigUluJ7/vPrWTS/1p1xGRQOzYAZde6tvJCwrUTi4N\np4ASkUDcdJO/8KvaySUoCigRabB77oHbbqu6Ovkxx4RdkWQDBZSINMizz8LAgf7x3XfD6aeHW49k\nDwWUiNTbW2/BhRf6dvIbboArrgi7IskmCigRqZfPP69qJ7/wQhg3LuyKJNsooESkzirbyVeuhG99\nC+6/X+3kEjztUiJSJ5Xt5P/5j9rJJb0UUCJSJ8OG+U69vfby7eT5+WFXJNlKASUiKZs6FX73O7WT\nS+NQQIlISp57Dq66yj+ePBl69Ai3Hsl+CigRqdXbb1e1k//613DllWFXJE2BAkpEdquynXzjRujV\nC269NeyKpKlQQIlIjbZsgR/8AFasgG9+U+3k0ri0q4lItSrbyV9+GQ4+GJ54Alq3DrsqaUoUUCJS\nrZtvhkceUTu5hEcBJSK7uPde+O1voVkzH1LHHht2RdIUKaBEZCcLFsCAAf7x5Mlwxhnh1iNNlwJK\nRL729tu+U6+8HH71K/j5z8OuSJqyQAPKzK4xs8Vmts3MZgS5bBFJr7Vrq9rJf/Qjf4pPJEy5AS9v\nDTAWOBPQ5SNFMkRiO/lJJ8HMmWonl/AFGlDOuTkAZlYIdApy2SKSHjt2QL9+sGgRHHSQ2sklOoI+\ngkqJmfUH+gPk5+dTXFwcRhm7VVJSEsm6okzbLHWxWIyKiopIbK977jmE2bMPpk2bckaNWsq775by\n7rthV7Ur7V91l+nbLJSAcs5NAaYAFBYWuqKiojDK2K3i4mKiWFeUaZulLi8vj1gsFvr2mj4dHnzQ\nt5PPmZPL979/Uqj17I72r7rL9G2ms8wiTdQ//gH9+/vHEyfC978fbj0iyRRQIk3QO+9UtZMPHVr1\nvSeRKAn0FJ+Z5caX2QxoZmZ7AOXOufIg1yMi9VfZTh6LwQ9/6G9AKBJFQR9BDQe2ADcCfeKPhwe8\nDhGpp61b4YIL4L//hcJCmDVL7eQSXUG3mY8CRgW5TBEJRmU7+UsvwYEHqp1cok/vnUSaiFtugYcf\nhrZt/dXJ998/7IpEdk8BJdIETJ8Ov/mNbyf/61/h+OPDrkikdgookSz3z39WtZP/6U9w5pnh1iOS\nKgWUSBZ7911/4dfychg8GK66KuyKRFKngBLJUuvWVbWTX3AB3HZb2BWJ1I0CSiQLVbaTf/QRdOvm\n28mbNQu7KpG6UUCJZJkdO+Dyy2HhQt9O/uST0KZN2FWJ1J0CSiTLjBwJDz3k28nnzVM7uWQuBZRI\nFrnvPhg71p/Omz0bvvGNsCsSqT8FlEiWKC6Gn//cP/7jH+Gss0ItR6TBFFAiWWD5ct9Ovn07XH89\nDBwYdkUiDaeAEslw69f7dvIvv4Tzz4ff/z7sikSCoYASyWCV7eQffghdu1bdHVckGyigRDKUc/Cz\nn8G//w2dOqmdXLKPAkokQ40cCX/5C+y5p786+QEHhF2RSLAUUCIZ6P77YcwYf7PBhx9WO7lkJwWU\nSIZ5/nm48kr/+K67oGfPcOsRSRcFlEgGWb4cfvhD304+aBD84hdhVySSPgookQyR3E4+fnzYFYmk\nlwJKJANs2+aPnD78EE48ER54QO3kkv0UUCIRV9lO/uKL0LGjbyffc8+wqxJJPwWUSMSNGuW/gFvZ\nTt6xY9gViTQOBZRIhM2cCaNHV7WTn3BC2BWJNB4FlEhEvfACXHGFf3znnWonl6ZHASUSQe+/X9VO\nft11cM01YVck0vgUUCIRs2GDP1r64gs47zy4/fawKxIJhwJKJEK2bfNXJ//gA99OrquTS1OmgBKJ\nCOf8JYzUTi7iKaBEImL0aJg1y98yY948tZOLKKBEImDWLP99p5wceOgh6NIl7IpEwqeAEgnZv/5V\n1U4+YQKce2649YhEhQJKJETvv++bIsrK4Npr/SAiXqABZWb7mtljZlZqZivN7KdBLl8km5SXG+ec\n49vJzzkH7rgj7IpEoiU34OVNBMqAfKAL8JSZveaceyvg9YhkNOdgxYo2lJb6z5seekjt5CLJzDkX\nzILM2gBfAsc5596Lj5sJfOKcu7Gm32vbtq3r1q1bIDUEKRaLkZeXF3YZGUXbLHUvv7yMrVuhRYsu\ndO0KLVuGXVH0af+qu6hus+eff36Jc66wtvmCPILqDFRUhlPca8BpyTOaWX+gP0Dz5s2JxWIBlhGM\nioqKSNYVZdpmqdm2LYetW/3jjh1L2bJlO1u2hFtTJtD+VXeZvs2CDKg9gY1J4zYCbZNndM5NAaYA\nFBYWusWLFwdYRjCKi4spKioKu4yMom1WO+egRw94990i8vLK+OijhWGXlDG0f9VdVLeZmaU0X5BN\nEiXAXknj9gK+CnAdIhlt3jz4xz8gNxc6dtRhk8juBBlQ7wG5ZnZEwrgTADVIiAAVFXBj/NPYgw+G\n3NxgPv8VyVaBBZRzrhSYA4w2szZmdgrwA2BmUOsQyWT33w9vvw0FBXDAAWFXIxJ9QX9R92qgFbAW\n+AswUC3mIlBaCrfc4h+PHesvaSQiuxfon4lz7gvn3AXOuTbOuYOccw8GuXyRTHXbbbB6NXTtCpdc\nEnY1IplB7+NE0mzlSh9QAHfdpaMnkVTpT0UkzX71K9i61R85nXJK2NWIZA4FlEgaFRfDX/8KrVtX\nHUWJSGoUUCJpUlEBv/ylf3zjjdCpU7j1iGQaBZRImtxzD7z+uv/O09ChYVcjknkUUCJp8NlncNNN\n/vH48dCqVbj1iGQiBZRIGlx7LcRicPbZ0KtX2NWIZCYFlEjA5s6FRx6BNm1g8mRI8bqYIpJEASUS\noI0b4Re/8I/HjfOfP4lI/SigRAJ0442wZg18+9tVQSUi9aOAEgnICy/A3XdD8+Ywdapu4S7SUAoo\nkQBs2gSXXeYf33QTHHtsuPWIZAMFlEgArr0WVqzwF4O9+eawqxHJDgookQZ6+GF/r6dWreCBB6BF\ni7ArEskOCiiRBvj4Y7jqKv/49tvhqKPCrUckmyigROqpogIuvdR/Ife882DAgLArEskuCiiReho3\nDp5/HvLzYdo0fSFXJGgKKJF6ePppGDnSh9L998N++4VdkUj2yQ27AJFM89FH0Ls3OAdjxsD3vx92\nRSLZSUdQInWwebO/+OuXX/rPnYYNC7sikeylgBJJkXMwcCAsWwaHH+5P7eXoL0gkbfTnJZKiO+7w\nodS6NcyZA3l5YVckkt0UUCIpePTRqrviTp8Oxx8fbj0iTYECSqQWixZBnz7+FN9vfwsXXRR2RSJN\ngwJKZDc+/BDOPx+2boX+/eHXvw67IpGmQwElUoO1a6FnT1i3Ds46CyZO1JdxRRqTAkqkGl984b/f\n9N57cMIJ/oKwufrWoEijUkCJJNm0Cc4+G157DTp3hr/9DfbaK+yqRJoeBZRIgtJSOPdc+M9/4JBD\nYMECf609EWl8CiiRuNJS+MEP4F//go4dfTh16hR2VSJNl86qi+BvmXHuufDvf0OHDj6cDjkk7KpE\nmjYdQUmTt24dfPe7PpwOPNAfQR15ZNhViYiOoKRJW70azjgD3n3XX19vwQI46KCwqxIRCOgIysyu\nMbPFZrbNzGYEsUyRdHv9deje3YfT8cf7IyeFk0h0BHWKbw0wFrg3oOWJpNX8+XDKKbBqlQ+p4mL4\nn/8JuyoRSRRIQDnn5jjn5gIbglieSDpNnOjv5VRSApdc4k/r7btv2FWJSLJQPoMys/5Af4D8/HyK\ni4vDKGO3SkpKIllXlEV9m5WVGRMnHs4TT3QE4NJLV9Cv3woWLWr8WmKxGBUVFZHeXlET9f0rijJ9\nm4USUM65KcAUgMLCQldUVBRGGbtVXFxMFOuKsihvs5Ur4cc/hldegRYtYOpU6Nu3ACgIpZ68vDxi\nsVhkt1cURXn/iqpM32a1nuIzs2IzczUMLzZGkSIN8cwz0LWrD6eDD/bt5H37hl2ViNSm1iMo51xR\nI9QhEriyMrjlFrjtNn8vp7PPhpkzoV27sCsTkVQEcorPzHLjy2oGNDOzPYBy51x5EMsXqau33oLe\nvf0FX3Ny4P/+D26+2T8WkcwQ1J/rcGALcCPQJ/54eEDLFknZjh0wYQJ06+bD6dBD/febRoxQOIlk\nmkCOoJxzo4BRQSxLpL7efBMGDICFC/2/r7gC7rgD2rYNty4RqR+9p5SMt3UrDB8OJ57ow2n//WHu\nXN+pp3ASyVy6Fp9kLOdg3jwYPBg++MCPu+oquPVWyMsLtzYRaTgFlGSkN96A66/3V4EAOOYYmDLF\nX75IRLKDTvFJRlm9Gvr3hy5dfDjtsw/ceScsW6ZwEsk2OoKSjPD55/7U3d13w7Zt0KwZXHstjByp\n7zWJZCsFlETap5/6TryJE2HzZj/uoov895qOOirc2kQkvRRQEknvvw+//z3cd5+/IgT4K5CPGQMn\nnBBubSLSOBRQEhk7dsCzz8KkSfDkk75Lzwx69YIbboCTTgq7QhFpTAooCd2GDTB9uv986cMP/bjm\nzeGyy2DoUDjyyHDrE5FwKKAkFBUV8PzzMGMGzJ7tGx/A33J9wAB/FYj8/FBLFJGQKaCk0TgHS5fC\nAw/AQw/BmjV+vJm/0vjVV/ufzZqFW6eIRIMCStLKOf+l2rlz4cEHYfnyqmmHHgo//Slcfrl/LCKS\nSAElgSsr86fvnnjCDx9/XDVtv/3g4ov9rTC+9S1/9CQiUh0FlDSYc/5aeE88cQB/+pPvxNu0qWp6\nhw6+RbxXL+jRwzdAiIjURgEl9bJqFbzwAjz3nL/k0KpVAJ2/nn7ccXD++T6YvvlN3YtJROpOASW1\nKivz17pbuNAPL73kr4mXqF07OO64tVxySQfOOEOfKYlIwymgZCdbt/ob/736qh+WLvV3pq1sA6+U\nlwcnnwynn+6Hb3wDXnjhbYqKOoRTuIhkHQVUE1VW5i8n9M478PbbVcM770B5+a7zH3UUdO/uh5NP\n9v/WaTsRSScFVBYrL/efDX30UdWwfLkPog8+8F+WTZaTA0cfDV27Vg1duugGgCLS+BRQGco53yn3\nySf+86BPPvFDYiCtXFl9CIFv7z7sMB9Gxxzjfx59tG9uaNOmcZ+LiEh1FFARs2ULrFsHa9f6n4nD\nmjU7B1Jpae3L69jRNyxUDocf7gPpyCOhVav0Px8RkfpSQAXMOd9QsHEjxGKpDRs2VIVQKqFTqXVr\n6NTJh1Dl0KlTVRgVFMAee6TtqYqIpFWTCagdO3xwbN3qh8TH1Y1bujSf5cv9EU1JiQ+OxJ81PS4t\nrb7JIFUtWvirLVQOHTpUPd5//53DaO+9dSUGEcleoQfUp5/CiBH+RX379qqfiY/rO27btqrQqbzp\nXeqOrvdzys31TQXVDfvsU/24yjBq21ahIyICEQioNWuWM3ZsUdLYi4Crgc1Az2p+q198WA9cWM30\ngcDFwCqg79djzXyXWtu2Q9h77/PIyVnO2rUDyMlhp+HYY4fTsuVxtG37GS+/PIhmzfwVtnNy/M/e\nvcfRtWt3VqxYyPTpw76eXjnceecEunTpwnPPPcfYsWPZvr3qFB7An//8Z4488kiefPJJbrvtD7tU\nP3PmTA488EAefvhhJk+evMv0Rx55hPbt2zNjxgxmzJixy/T58+fTunVrJk2axOzZs3eZXlxcDMD4\n8eOZN2/eTtNatWrF008/DcCYMWNYsGDBTtPbtWvHo48+CsBNN93ESy+99PW0WCzGcccdx6xZswAY\nNGgQy5Yt2+n3O3fuzJQpUwDo378/77333k7Tu3TpwoQJEwDo06cPq5O+EXzyySdz6623AtCrVy82\nbNiw0/S0rVP4AAAFVElEQVTTTz+dESNGAHD22WezZcuWnaafe+65DB06FICioqJdts1FF13E1Vdf\nzebNm+nZc9d9r1+/fvTr14/169dz4YW77nsDBw7k4osvZtWqVfTt23eX6UOGDOG8885j8+bNfPDB\nB7vUMHz4cHr06MGyZcsYNGjQLr8/btw4unfvzsKFCxk2bNgu0ydM2HnfS5a47/3hD5m17+3YsYMX\nXngB2HXfA+jUqZP2vaR9LxaLkRdvwa3c95YvX86AAQN2+f3G3PdSFXpAtWgBBxzgw6Ny6NYNvvc9\nf1rurrt2nmYGZ54JPXv602kjR+48LScH+vSBCy6A9evhuuv8uMSjkiFD/CV4li/39x5KNnw45Oa+\nS15eHtX8P9Gjh/8+0MKF8Mgj6ds2IiJNmTnnQi2gsLDQLV68ONQaqlNcXFztuxypmbZZ6oqKiojF\nYru8y5eaaf+qu6huMzNb4pwrrG0+XQtAREQiSQElIiKRpIASEZFIUkCJiEgkKaBERCSSGhxQZtbS\nzKaZ2Uoz+8rMlprZ2UEUJyIiTVcQR1C5+G/EngbsDYwAZptZQQDLFhGRJqrBX9R1zpUCoxJGzTOz\n/wLdgBUNXb6IiDRNgV9Jwszygc7AW7uZpz/QHyA/P//ry59ESUlJSSTrijJts9TFYjEqKiq0vepA\n+1fdZfo2C/RKEmbWHHga+NA5V81FhHalK0lkD22z1OlKEnWn/avuorrNAruShJkVm5mrYXgxYb4c\nYCZQBlzToOpFRKTJq/UUn3OuqLZ5zMyAaUA+0NM5t73hpYmISFMW1GdQk/E3UOrhnNtS28wiIiK1\nCeJ7UAcDA4AuwGdmVhIfeje4OhERabKCaDNfCegesCIiEihd6khERCJJASUiIpEU+h11zWwdsDLU\nIqrXHlgfdhEZRtusbrS96kbbq+6ius0Ods7tV9tMoQdUVJnZ4lS+SCZVtM3qRturbrS96i7Tt5lO\n8YmISCQpoEREJJIUUDWbEnYBGUjbrG60vepG26vuMnqb6TMoERGJJB1BiYhIJCmgREQkkhRQIiIS\nSQqoFJnZEWa21cxmhV1LVJlZSzObZmYrzewrM1tqZmeHXVfUmNm+ZvaYmZXGt9VPw64pqrRPNUym\nv24poFI3EXgl7CIiLhdYBZwG7A2MAGabWUGINUXRRPyNPfOB3sBkMzs23JIiS/tUw2T065YCKgVm\n9hMgBiwIu5Yoc86VOudGOedWOOd2OOfmAf8FuoVdW1SYWRugFzDCOVfinHsReALoG25l0aR9qv6y\n4XVLAVULM9sLGA0MCbuWTGNm+UBn4K2wa4mQzkCFc+69hHGvATqCSoH2qdRky+uWAqp2Y4BpzrlV\nYReSScysOfAAcJ9z7t2w64mQPYGNSeM2Am1DqCWjaJ+qk6x43WrSAWVmxWbmahheNLMuQA/gjrBr\njYLatlfCfDnATPznLNeEVnA0lQB7JY3bC/gqhFoyhvap1GXT61aD76ibyZxzRbubbmaDgALgYzMD\n/+63mZkd45zrmvYCI6a27QVgfkNNwzcA9HTObU93XRnmPSDXzI5wzr0fH3cCOmVVI+1TdVZElrxu\n6VJHu2Fmrdn53e5Q/H/8QOfculCKijgzuxvoAvRwzpWEXU8UmdlDgAOuxG+r+UB355xCqhrap+om\nm163mvQRVG2cc5uBzZX/NrMSYGum/Sc3FjM7GBgAbAM+i797AxjgnHsgtMKi52rgXmAtsAH/wqFw\nqob2qbrLptctHUGJiEgkNekmCRERiS4FlIiIRJICSkREIkkBJSIikaSAEhGRSFJAiYhIJCmgREQk\nkhRQIiISSf8PkVQyL4R+s08AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a341571d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(z, selu(z), \"b-\", linewidth=2)\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([-5, 5], [-1.758, -1.758], 'k--')\n",
    "plt.plot([0, 0], [-2.2, 3.2], 'k-')\n",
    "plt.grid(True)\n",
    "plt.title(\"SELU activation function\", fontsize=14)\n",
    "plt.axis([-5, 5, -2.2, 3.2])\n",
    "\n",
    "save_fig(\"selu_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "By default, the SELU hyperparameters (`scale` and `alpha`) are tuned in such a way that the mean output of each neuron remains close to 0, and the standard deviation remains close to 1 (assuming the inputs are standardized with mean 0 and standard deviation 1 too). Using this activation function, even a 1,000 layer deep neural network preserves roughly mean 0 and standard deviation 1 across all layers, avoiding the exploding/vanishing gradients problem:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Layer 0: mean -0.00, std deviation 1.00\n",
      "Layer 100: mean 0.02, std deviation 0.96\n",
      "Layer 200: mean 0.01, std deviation 0.90\n",
      "Layer 300: mean -0.02, std deviation 0.92\n",
      "Layer 400: mean 0.05, std deviation 0.89\n",
      "Layer 500: mean 0.01, std deviation 0.93\n",
      "Layer 600: mean 0.02, std deviation 0.92\n",
      "Layer 700: mean -0.02, std deviation 0.90\n",
      "Layer 800: mean 0.05, std deviation 0.83\n",
      "Layer 900: mean 0.02, std deviation 1.00\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "Z = np.random.normal(size=(500, 100)) # standardized inputs\n",
    "for layer in range(1000):\n",
    "    W = np.random.normal(size=(100, 100), scale=np.sqrt(1 / 100)) # LeCun initialization\n",
    "    Z = selu(np.dot(Z, W))\n",
    "    means = np.mean(Z, axis=0).mean()\n",
    "    stds = np.std(Z, axis=0).mean()\n",
    "    if layer % 100 == 0:\n",
    "        print(\"Layer {}: mean {:.2f}, std deviation {:.2f}\".format(layer, means, stds))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using SELU is easy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.layers.core.Dense at 0x1a3547ccf8>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keras.layers.Dense(10, activation=\"selu\",\n",
    "                   kernel_initializer=\"lecun_normal\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's create a neural net for Fashion MNIST with 100 hidden layers, using the SELU activation function:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(300, activation='selu',\n",
    "                       kernel_initializer='lecun_normal')\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for layer in range(99):\n",
    "    model.add(keras.layers.Dense(100, activation='selu',\n",
    "                                 kernel_initializer='lecun_normal'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.add(keras.layers.Dense(10, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss='sparse_categorical_crossentropy',\n",
    "              optimizer=keras.optimizers.SGD(lr=1e-3),\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's train it. Do not forget to scale the inputs to mean 0 and standard deviation 1:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pixel_means = X_train.mean(axis=0, keepdims=True)\n",
    "pixel_stds = X_train.std(axis=0, keepdims=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_scaled = (X_train - pixel_means) / pixel_stds\n",
    "X_valid_scaled = (X_valid - pixel_means) / pixel_stds\n",
    "X_test_scaled = (X_test - pixel_means) / pixel_stds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 55000 samples, validate on 5000 samples\n",
      "Epoch 1/5\n",
      "55000/55000 [==============================] - 31s 565us/sample - loss: 1.0197 - accuracy: 0.6154 - val_loss: 0.7386 - val_accuracy: 0.7348\n",
      "Epoch 2/5\n",
      "55000/55000 [==============================] - 26s 470us/sample - loss: 0.7149 - accuracy: 0.7401 - val_loss: 0.6187 - val_accuracy: 0.7774\n",
      "Epoch 3/5\n",
      "55000/55000 [==============================] - 26s 468us/sample - loss: 0.6193 - accuracy: 0.7803 - val_loss: 0.5926 - val_accuracy: 0.8036\n",
      "Epoch 4/5\n",
      "55000/55000 [==============================] - 26s 477us/sample - loss: 0.5555 - accuracy: 0.8043 - val_loss: 0.5208 - val_accuracy: 0.8262\n",
      "Epoch 5/5\n",
      "55000/55000 [==============================] - 27s 483us/sample - loss: 0.5159 - accuracy: 0.8238 - val_loss: 0.4790 - val_accuracy: 0.8358\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train_scaled, y_train,\n",
    "                    epochs=5,\n",
    "                    validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now look at what happens if we try to use the ReLU activation function instead:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = keras.models.Sequential()\n",
    "model.add(keras.layers.Flatten(input_shape=[28, 28]))\n",
    "model.add(keras.layers.Dense(300, activation=\"relu\", kernel_initializer=\"he_normal\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for layer in range(99):\n",
    "    model.add(keras.layers.Dense(100, activation=\"relu\", kernel_initializer=\"he_normal\"))\n",
    "model.add(keras.layers.Dense(10, activation=\"softmax\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "              optimizer=keras.optimizers.SGD(lr=1e-3),\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 55000 samples, validate on 5000 samples\n",
      "Epoch 1/5\n",
      "55000/55000 [==============================] - 18s 325us/sample - loss: 1.9174 - accuracy: 0.2242 - val_loss: 1.3856 - val_accuracy: 0.3846\n",
      "Epoch 2/5\n",
      "55000/55000 [==============================] - 17s 305us/sample - loss: 1.2147 - accuracy: 0.4750 - val_loss: 1.0691 - val_accuracy: 0.5510\n",
      "Epoch 3/5\n",
      "55000/55000 [==============================] - 17s 307us/sample - loss: 0.9576 - accuracy: 0.6025 - val_loss: 0.7688 - val_accuracy: 0.7036\n",
      "Epoch 4/5\n",
      "55000/55000 [==============================] - 17s 312us/sample - loss: 0.8116 - accuracy: 0.6762 - val_loss: 0.7276 - val_accuracy: 0.7288\n",
      "Epoch 5/5\n",
      "55000/55000 [==============================] - 18s 320us/sample - loss: 0.8167 - accuracy: 0.6862 - val_loss: 0.7697 - val_accuracy: 0.7032\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train_scaled, y_train, \n",
    "                    epochs=5,\n",
    "                    validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Batch Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Dense(300, activation='relu'),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Dense(100, activation='relu'),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Dense(10, activation='softmax')\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_4\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_4 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_v2 (Batc (None, 784)               3136      \n",
      "_________________________________________________________________\n",
      "dense_212 (Dense)            (None, 300)               235500    \n",
      "_________________________________________________________________\n",
      "batch_normalization_v2_1 (Ba (None, 300)               1200      \n",
      "_________________________________________________________________\n",
      "dense_213 (Dense)            (None, 100)               30100     \n",
      "_________________________________________________________________\n",
      "batch_normalization_v2_2 (Ba (None, 100)               400       \n",
      "_________________________________________________________________\n",
      "dense_214 (Dense)            (None, 10)                1010      \n",
      "=================================================================\n",
      "Total params: 271,346\n",
      "Trainable params: 268,978\n",
      "Non-trainable params: 2,368\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('batch_normalization_v2/gamma:0', True),\n",
       " ('batch_normalization_v2/beta:0', True),\n",
       " ('batch_normalization_v2/moving_mean:0', False),\n",
       " ('batch_normalization_v2/moving_variance:0', False)]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bn1 = model.layers[1]\n",
    "\n",
    "[(var.name, var.trainable) for var in bn1.variables]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ListWrapper([<tf.Operation 'batch_normalization_v2/cond_2/Identity' type=Identity>, <tf.Operation 'batch_normalization_v2/cond_3/Identity' type=Identity>])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bn1.updates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "              optimizer=keras.optimizers.SGD(lr=1e-3),\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 55000 samples, validate on 5000 samples\n",
      "Epoch 1/10\n",
      "55000/55000 [==============================] - 7s 122us/sample - loss: 0.8756 - accuracy: 0.7140 - val_loss: 0.5514 - val_accuracy: 0.8212\n",
      "Epoch 2/10\n",
      "55000/55000 [==============================] - 6s 112us/sample - loss: 0.5765 - accuracy: 0.8033 - val_loss: 0.4742 - val_accuracy: 0.8436\n",
      "Epoch 3/10\n",
      "55000/55000 [==============================] - 6s 115us/sample - loss: 0.5146 - accuracy: 0.8216 - val_loss: 0.4382 - val_accuracy: 0.8530\n",
      "Epoch 4/10\n",
      "55000/55000 [==============================] - 6s 114us/sample - loss: 0.4821 - accuracy: 0.8322 - val_loss: 0.4170 - val_accuracy: 0.8604\n",
      "Epoch 5/10\n",
      "55000/55000 [==============================] - 6s 110us/sample - loss: 0.4589 - accuracy: 0.8402 - val_loss: 0.4003 - val_accuracy: 0.8658\n",
      "Epoch 6/10\n",
      "55000/55000 [==============================] - 6s 109us/sample - loss: 0.4428 - accuracy: 0.8459 - val_loss: 0.3883 - val_accuracy: 0.8698\n",
      "Epoch 7/10\n",
      "55000/55000 [==============================] - 6s 111us/sample - loss: 0.4220 - accuracy: 0.8521 - val_loss: 0.3792 - val_accuracy: 0.8720\n",
      "Epoch 8/10\n",
      "55000/55000 [==============================] - 6s 110us/sample - loss: 0.4150 - accuracy: 0.8546 - val_loss: 0.3696 - val_accuracy: 0.8754\n",
      "Epoch 9/10\n",
      "55000/55000 [==============================] - 6s 117us/sample - loss: 0.4013 - accuracy: 0.8589 - val_loss: 0.3629 - val_accuracy: 0.8746\n",
      "Epoch 10/10\n",
      "55000/55000 [==============================] - 6s 111us/sample - loss: 0.3931 - accuracy: 0.8615 - val_loss: 0.3581 - val_accuracy: 0.8766\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train, y_train, epochs=10,\n",
    "                    validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gradient Clipping"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All Keras optimizers accept `clipnorm` or `clipvalue` arguments:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "optimizer = keras.optimizers.SGD(clipvalue=1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "optimizer = keras.optimizers.SGD(clipnorm=1.0)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}