mnist_cnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-22T21:22:03.433987Z",
     "iopub.status.busy": "2020-12-22T21:22:03.433649Z",
     "iopub.status.idle": "2020-12-22T21:22:43.334101Z",
     "shell.execute_reply": "2020-12-22T21:22:43.333346Z",
     "shell.execute_reply.started": "2020-12-22T21:22:03.433916Z"
    }
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "import cv2\n",
    "import jax\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "import trax\n",
    "from trax import layers as tl\n",
    "from trax.shapes import signature\n",
    "from trax.supervised import lr_schedules, training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-22T21:22:43.336111Z",
     "iopub.status.busy": "2020-12-22T21:22:43.335816Z",
     "iopub.status.idle": "2020-12-22T21:22:43.340675Z",
     "shell.execute_reply": "2020-12-22T21:22:43.339706Z",
     "shell.execute_reply.started": "2020-12-22T21:22:43.336081Z"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-22T21:22:43.344856Z",
     "iopub.status.busy": "2020-12-22T21:22:43.344269Z",
     "iopub.status.idle": "2020-12-22T21:22:43.352811Z",
     "shell.execute_reply": "2020-12-22T21:22:43.351386Z",
     "shell.execute_reply.started": "2020-12-22T21:22:43.344819Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensorflow-numpy\n"
     ]
    }
   ],
   "source": [
    "# Use the tensorflow-numpy backend.\n",
    "trax.fastmath.set_backend(\"tensorflow-numpy\")\n",
    "print(trax.fastmath.backend_name())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-22T21:22:43.357286Z",
     "iopub.status.busy": "2020-12-22T21:22:43.356804Z",
     "iopub.status.idle": "2020-12-22T21:22:43.365891Z",
     "shell.execute_reply": "2020-12-22T21:22:43.364895Z",
     "shell.execute_reply.started": "2020-12-22T21:22:43.357245Z"
    }
   },
   "outputs": [],
   "source": [
    "# https://github.com/google/flax/blob/9349a3d740918f8df49058e853f521d28181677d/linen_examples/mnist/mnist.ipynb\n",
    "# Helper functions for images.\n",
    "def show_img(img, ax=None, title=None):\n",
    "    \"\"\"Shows a single image.\"\"\"\n",
    "    if ax is None:\n",
    "        ax = plt.gca()\n",
    "    ax.imshow(img[..., 0], cmap=\"gray\")\n",
    "    ax.set_xticks([])\n",
    "    ax.set_yticks([])\n",
    "    if title:\n",
    "        ax.set_title(title)\n",
    "\n",
    "\n",
    "def show_img_grid(imgs, titles):\n",
    "    \"\"\"Shows a grid of images.\"\"\"\n",
    "    n = int(np.ceil(len(imgs) ** 0.5))\n",
    "    _, axs = plt.subplots(n, n, figsize=(3 * n, 3 * n))\n",
    "    for i, (img, title) in enumerate(zip(imgs, titles)):\n",
    "        show_img(img, axs[i // n][i % n], title)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-22T21:22:43.368363Z",
     "iopub.status.busy": "2020-12-22T21:22:43.367832Z",
     "iopub.status.idle": "2020-12-22T21:22:43.779180Z",
     "shell.execute_reply": "2020-12-22T21:22:43.778523Z",
     "shell.execute_reply.started": "2020-12-22T21:22:43.368320Z"
    }
   },
   "outputs": [],
   "source": [
    "train_stream = trax.data.TFDS(\n",
    "    \"mnist\", data_dir=\"./mnist_data2/\", keys=(\"image\", \"label\"), train=True\n",
    ")()\n",
    "eval_stream = trax.data.TFDS(\n",
    "    \"mnist\", data_dir=\"./mnist_data2/\", keys=(\"image\", \"label\"), train=False\n",
    ")()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`train_stream` and `eval_stream` are generators. `next(train_stream)` return a tuple *(image, label)*. \n",
    "\n",
    "Let check an image."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-22T21:22:43.780866Z",
     "iopub.status.busy": "2020-12-22T21:22:43.780415Z",
     "iopub.status.idle": "2020-12-22T21:22:43.989527Z",
     "shell.execute_reply": "2020-12-22T21:22:43.988701Z",
     "shell.execute_reply.started": "2020-12-22T21:22:43.780775Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOsAAAD7CAYAAACL3GNOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAGX0lEQVR4nO3dz2uVVx7H8XOm/sAyVMhSBbupi4GEqeg6gv0HxCyG/APqRoTpXzCzceequJAKQlchuBZlQFcG4kIJgitxGlG0IgOWWBB8ZjEUWib33NybJ958bl+vnfnmkS/ImyM5PDe167oC7H5/mvQCwNaIFUKIFUKIFUKIFUKIFUKIFUKIdYrVWr+qtf5Sa/1h0ruwfWKdbt+VUlYnvQT9EOuUqrX+rZTyn1LKvya8Cj0R6xSqtX5RSvlHKeXvk96F/oh1Ov2zlPJ913Xrk16E/uyZ9AL0q9b611LKN6WUrye8Cj0T6/Q5VUr5spTyY621lFL+XEr5rNb6l67rjk9wL7apekVuutRaPy+lfPGbL31b/hfvha7rfprIUvTCyTpluq7bKKVs/PrnWuvPpZRfhJrPyQoh/DQYQogVQogVQogVQogVQox0dVNr9aNj2GFd19XNvu5khRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRBihRB7Jr0AuRYWFprzpaWl5vzcuXPN+bVr10beaZo5WSGEWCGEWCGEWCGEWCGEWCGEWCGEe1bGtri42Jx3Xdecz8zM9LnO1HOyQgixQgixQgixQgixQgixQog67Mfrv/vmWrf+zUyFo0ePDpw9efKk+eza2lpzfvbs2eZ8fX29OZ9WXdfVzb7uZIUQYoUQYoUQYoUQYoUQYoUQYoUQXpHrQa2bXott2Sh33Z/axYsXB8727dvXfPbp06fN+R/1HnVcTlYIIVYIIVYIIVYIIVYIIVYIIVYI4Z61B6dOnWrOr1y50pyfP39+4GxlZWWclXozOzs79rMPHz7sbxGcrJBCrBBCrBBCrBBCrBBCrBBCrBDCPWsP3r9/35wPu6ucn58fONvpe9YjR440563d3r1713z2xo0bY+3E5pysEEKsEEKsEEKsEEKsEEKsEEKsEMI9aw9ev3496RXGdubMmeZ87969A2cPHjxoPvvy5cuxdmJzTlYIIVYIIVYIIVYIIVYIIVYI4eqmBzMzM5NeYWyHDh0a+9m7d+/2twhDOVkhhFghhFghhFghhFghhFghhFghhHvWHgx7zazW+ok2+X+HDx9uzi9cuNCct3a/fv36WDsxHicrhBArhBArhBArhBArhBArhBArhKhd1239m2vd+jdPkf379zfnz58/b86Hve+6trY2cHb//v1t/d1zc3PN+bFjx5rzR48eDZydOHGi+ezHjx+bczbXdd2ml9tOVgghVgghVgghVgghVgghVgghVgjhfdYtWFxcbM63+7nBs7OzA2fD7klHuScfx+XLlwfO3KN+Wk5WCCFWCCFWCCFWCCFWCCFWCCFWCOGedQtOnjzZnG9sbDTnwz5f98WLFwNnb9++bT775s2b5nx5ebk5H+bWrVvbep7+OFkhhFghhFghhFghhFghhFghhI8iDbewsNCcLy0tNec3b97c1t9P/3wUKYQTK4QQK4QQK4QQK4QQK4QQK4Twily4YR+TOuwefXV1tc912EFOVgghVgghVgghVgghVgghVgghVgjhnjXc/Px8cz7snvXevXt9rsMOcrJCCLFCCLFCCLFCCLFCCLFCCLFCCPesu9zx48eb8z172v+Et2/fbs5XVlZG3onJcLJCCLFCCLFCCLFCCLFCCLFCCL/ycZe7c+dOc3769Onm/MOHD835pUuXmvOrV6825/TPr3yEcGKFEGKFEGKFEGKFEGKFEGKFEF6R2+WG3YMPmz9+/Lg5X15eHnknJsPJCiHECiHECiHECiHECiHECiHECiG8z7rLra+vN+cHDx5szufm5przZ8+ejboSO8z7rBBOrBBCrBBCrBBCrBBCrBBCrBDC+6y73IEDB5rzV69eNefuUaeHkxVCiBVCiBVCiBVCiBVCiBVCiBVCeJ8Vdhnvs0I4sUIIsUIIsUIIsUIIsUIIsUIIsUIIsUIIsUIIsUIIsUIIsUIIsUIIsUIIsUIIsUIIsUIIsUIIsUIIsUIIsUKIUX/l45tSyr93YhGglFLK0UGDkT43GJgc/w2GEGKFEGKFEGKFEGKFEGKFEGKFEGKFEGKFEP8FtBQBj8sIM5gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "img, label = next(train_stream)\n",
    "show_img(img, title=str(label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-22T21:22:43.992070Z",
     "iopub.status.busy": "2020-12-22T21:22:43.991561Z",
     "iopub.status.idle": "2020-12-22T21:22:43.996730Z",
     "shell.execute_reply": "2020-12-22T21:22:43.995928Z",
     "shell.execute_reply.started": "2020-12-22T21:22:43.992033Z"
    }
   },
   "outputs": [],
   "source": [
    "def random_shift(X):\n",
    "    dx, dy = np.random.choice(np.arange(-2, 3), size=2, p=[0.1, 0.1, 0.6, 0.1, 0.1])\n",
    "    X = np.roll(X, dy, axis=0)\n",
    "    X = np.roll(X, dx, axis=1)\n",
    "    return X"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function to rotate images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-22T21:22:53.820741Z",
     "iopub.status.busy": "2020-12-22T21:22:53.820344Z",
     "iopub.status.idle": "2020-12-22T21:22:53.826782Z",
     "shell.execute_reply": "2020-12-22T21:22:53.825783Z",
     "shell.execute_reply.started": "2020-12-22T21:22:53.820693Z"
    }
   },
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/9041681/opencv-python-rotate-image-by-x-degrees-around-specific-point\n",
    "def rotate_image(g):\n",
    "    \"\"\"\n",
    "    Randomly rotate a image from (-10 degree to 10 degree).\n",
    "    \"\"\"\n",
    "    for tup in g:\n",
    "        image, label = tup\n",
    "\n",
    "        angle = np.random.choice(\n",
    "            np.arange(-10, 15, 5), size=1, p=[0.1, 0.1, 0.6, 0.1, 0.1]\n",
    "        )\n",
    "\n",
    "        row, col, _ = image.shape\n",
    "        center = tuple(np.array([row, col]) / 2)\n",
    "        rot_mat = cv2.getRotationMatrix2D(center, angle, 1.0)\n",
    "        new_image = cv2.warpAffine(image, rot_mat, (col, row))\n",
    "\n",
    "        yield (new_image[:, :, None], label)\n",
    "\n",
    "\n",
    "# def rotateimage():\n",
    "#     return lambda g: rotate_image(g)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function to shift images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-22T21:22:55.946219Z",
     "iopub.status.busy": "2020-12-22T21:22:55.945917Z",
     "iopub.status.idle": "2020-12-22T21:22:55.952025Z",
     "shell.execute_reply": "2020-12-22T21:22:55.951038Z",
     "shell.execute_reply.started": "2020-12-22T21:22:55.946192Z"
    }
   },
   "outputs": [],
   "source": [
    "def shift_image(g):\n",
    "    \"\"\"\n",
    "    Randomly shift the image up/down/left or right base on the given probability.\n",
    "    \"\"\"\n",
    "    dx, dy = np.random.choice(\n",
    "        np.arange(-2, 3), size=2, p=[0.075, 0.075, 0.7, 0.075, 0.075]\n",
    "    )\n",
    "\n",
    "    for tup in g:\n",
    "        X, y = tup\n",
    "\n",
    "        X = np.roll(X, dy, axis=0)\n",
    "\n",
    "        X = np.roll(X, dx, axis=1)\n",
    "        for dxy in [dx, dy]:\n",
    "            if dxy < 0:\n",
    "                X[dxy:, :] = 0\n",
    "            if dxy > 0:\n",
    "                X[:dxy, :] = 0\n",
    "\n",
    "        yield tuple((X, y))\n",
    "\n",
    "\n",
    "# def shiftimage():\n",
    "#     return lambda g: shift_image(g)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also use lambda function shown [here](https://trax-ml.readthedocs.io/en/latest/trax.data.html).\n",
    "\n",
    "```python\n",
    "def random_shift(X):\n",
    "    dx, dy = np.random.choice(\n",
    "        np.arange(-2, 3), size=2, p=[0.075, 0.075, 0.7, 0.075, 0.075]\n",
    "    )\n",
    "    \n",
    "    X = np.roll(X, dy, axis=0)\n",
    "    X = np.roll(X, dx, axis=1)\n",
    "    \n",
    "    for dxy in [dx, dy]:\n",
    "        if dxy < 0:\n",
    "            X[dxy:, :] = 0\n",
    "        if dxy > 0:\n",
    "            X[:dxy, :] = 0\n",
    "            \n",
    "    return X\n",
    "```\n",
    "\n",
    "And the data pipeline can be written using a lambda function:\n",
    "```python\n",
    "train_data_pipeline = trax.data.Serial(\n",
    "        lambda g: map(\n",
    "            lambda x: (random_shift(x[0]), x[1]), g\n",
    "        ),  # shift image using random_shift function: x[0]=image,x[1]=label\n",
    "    trax.data.Batch(3),\n",
    "    trax.data.Shuffle(),\n",
    "    trax.data.AddLossWeights(),\n",
    ")\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-22T21:23:43.176853Z",
     "iopub.status.busy": "2020-12-22T21:23:43.176549Z",
     "iopub.status.idle": "2020-12-22T21:23:43.180646Z",
     "shell.execute_reply": "2020-12-22T21:23:43.179722Z",
     "shell.execute_reply.started": "2020-12-22T21:23:43.176825Z"
    }
   },
   "outputs": [],
   "source": [
    "train_data_pipeline = trax.data.Serial(\n",
    "    shift_image,\n",
    "    rotate_image,\n",
    "    trax.data.Batch(40),\n",
    "    trax.data.Shuffle(),\n",
    "    trax.data.AddLossWeights(),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-22T21:23:44.648242Z",
     "iopub.status.busy": "2020-12-22T21:23:44.647938Z",
     "iopub.status.idle": "2020-12-22T21:23:44.651730Z",
     "shell.execute_reply": "2020-12-22T21:23:44.650927Z",
     "shell.execute_reply.started": "2020-12-22T21:23:44.648215Z"
    }
   },
   "outputs": [],
   "source": [
    "eval_data_pipeline = trax.data.Serial(\n",
    "    trax.data.Batch(20), trax.data.AddLossWeights(),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-22T21:23:48.192364Z",
     "iopub.status.busy": "2020-12-22T21:23:48.192057Z",
     "iopub.status.idle": "2020-12-22T21:23:48.195677Z",
     "shell.execute_reply": "2020-12-22T21:23:48.194748Z",
     "shell.execute_reply.started": "2020-12-22T21:23:48.192338Z"
    }
   },
   "outputs": [],
   "source": [
    "train_batch = train_data_pipeline(train_stream)\n",
    "eval_batch = eval_data_pipeline(eval_stream)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-22T21:23:49.575878Z",
     "iopub.status.busy": "2020-12-22T21:23:49.575595Z",
     "iopub.status.idle": "2020-12-22T21:24:03.441612Z",
     "shell.execute_reply": "2020-12-22T21:24:03.440993Z",
     "shell.execute_reply.started": "2020-12-22T21:23:49.575852Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAILCAYAAACXVIRDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAvZklEQVR4nO3deZRV5ZX+8WcDVYrMyqSgMgmKCohEWrRRUdMgzi00gSaKUxDngQ6SNspyxKFpRRNpR4ydYCuySMQp0XQkKDiCDAIigoIgM1SBDFWc3x9V/rpk70ouVUXde+t+P2vdtaoezz1nF75VbI+73mNJkggAAOS2WukuAAAApB8NAQAAoCEAAAA0BAAAQDQEAABANAQAAEA52BCY2TIzOyOF4xIz61DBa1T4vUAqWMfIdqzhzJNzDUGmshJjzWx96et+M7N01wXsDTO7wcyWmtkWM/vGzMaZWZ101wWkyswam9lEM1tT+roj3TVVFxqCzHGlpPMldZXURdLZkn6WzoKACviDpO5JkjSUdIxK1vN16S0J2CvjJB0gqY2kEyQNNbNhaa2omuRsQ2BmJ5jZe2a2ycxWmdmjZpa/x2Fnlf7Xzjoze8DMapV5/6Vm9pmZbTSzN8zs8EqWdLGkh5IkWZEkyUpJD0m6pJLnRA2Xaes4SZIvkiTZ9P3pJe2WxC1blCvT1rCkcyTdnyTJtiRJlkl6StKllTxnVsjZhkBSsaQbJTWVdKKk0yWN2OOYCyT1kNRd0nkqXRRmdr6k0ZIulNRM0nRJv4suYmajShd6+Cpz6NGS5pT5fE5pBvwtmbaOZWaDzWyLpHUquUMwoSq+UNRYGbeGVdLMlv34mIp/eVkkSZKceklaJumMIL9B0pQynyeS+pb5fISkt0o/fk3SZWX+WS1J2yQdXua9HfayrmJJR5b5/IjS81i6/8x4Zd4rU9fxHrUcIelOSS3T/efFK/NembqGJT0v6WVJDVRyd+sLSTvS/edVHa+cvUNgZh3N7BUzW136XzP3qKRDLevrMh8vl3RI6ceHS3q4TGe5QSVdZKtKlFQoqWGZzxtKKkxKVygQycB1/P8lSfK5pPmSflUV50PNlIFr+DpJ30n6XNJUldxxWFGJ82WNnG0IJP1a0kJJRyQlA1Cj9cPbRJJ0aJmPD5P0TenHX0v6WZIkjcu86iZJ8u6eFzGz0WZWWN6rzKHzVXJ79XtdSzPgb8m0dbynOpLaV/irQy7IqDWcJMmGJEmGJEnSMkmSo1Xy9+T7Vfj1ZqxcbggaSNoiqdDMjpR0VXDMSDNrYmaHSrpe0gul+eOSbjWzoyXJzBqZ2YDoIkmS3JMkSf3yXmUOfU7STWbWyswOkXSzpGer5CtFTZZR69jMLjez5qUfd5Z0q6S3quqLRY2UaWu4vZkdZGa1zayfSn4D7K6q+3IzVy43BLdIGiypQNIT+r8FVtZUSR9Jmi1pmkqmTZUkyRRJYyVNKr3FNU9Sv0rWM0Elv7I1t/R808QwFv6+TFvHJ0maa2ZbJb1a+hpdyXOiZsu0NXy8Sn4OF0i6V9KQJEly4m6t8b+oAQBALt8hAAAApWgIAAAADQEAAKAhAAAAoiEAAAAq2TQkZWbGrySgMtYlSdIsnQWwhlFJrGFku3LXMHcIUJ2Wp7sAoJJYw8h25a5hGgIAAEBDAAAAaAgAAIBoCAAAgGgIAACAaAgAAIBoCAAAgGgIAACAaAgAAIBoCAAAgGgIAACAaAgAAIBoCAAAgGgIAACAaAgAAIBoCAAAgGgIAACAaAgAAIBoCAAAgKQ66S6gJqldu7bL6tev77JatXwftmXLlvCcxcXFlS8MAIC/gzsEAACAhgAAANAQAAAA0RAAAAAxVFhhBxxwgMuGDBnisvbt27tsw4YNLnv11VfD63z++ecu27FjRyolAgCQMu4QAAAAGgIAAEBDAAAAREMAAADEUKHTqFEjl91www0uu+iii1zWqVMnl+Xl5bmsqKjIZd26dQvreeqpp1z2/vvvu6ygoCB8P2qGxo0bu2zQoEHVX0ipadOmuezrr79OQyUAqgp3CAAAAA0BAACgIQAAAKIhAAAAypGhwtatW7usZcuW4bF9+/Z12b/+67+6rF27di6LHlW8fv16l+Xn57vs/PPPD+tp0KCByx588EGXzZgxw2XR8CKy0+jRo112yy23pKGSEtEQ6+uvvx4ee9lll7mssLCwymsCUDncIQAAADQEAACAhgAAAIiGAAAAKMuHCs3MZW3btnXZsGHDXHbGGWeE54wGEJs2beqy6HHFTz/9tMuinQr/8R//0WW9e/cO6znxxBNdFn09W7duddns2bNdluuDhi1atNBPf/rTH2QPPPBAmqpJXZ06/lt18eLF4bGpDsHWrl3bZZ999pnL/umf/sllv/zlL102YMCAsJ6OHTu67J//+Z9dtnTp0vD9yD6pPh4+WgeS1LNnT5c98sgjLps8ebLLPv3001RKRIA7BAAAgIYAAADQEAAAANEQAAAASZYkSeoHm6V+cBWLBgij3QYvvfRSlw0fPtxl0fBgeT744AOX3XbbbS574403Ujpf/fr1XXbFFVeEx0bXifzP//yPy8aNG+eyRYsWpXS+feSjJEl6pLOAjh07JuPHj/9BFu1OmWnq1avnsocffjg89sknn3TZzJkzq7Sexx9/3GXRDpySdMkll7jsrbfectmZZ55Z6bqqQdrXcDp/DkeiodHnnnvOZSeccILL9ubvn1Tdc889Lrv77rtdtn379iq/dpYodw1zhwAAANAQAAAAGgIAACAaAgAAoAzdqTAaIGzSpInL+vfv77LBgwe7LBoeeeGFF8JrR4OK0SDeJ598Er4/FdGjX997773w2FmzZrns1FNPdVmfPn1Sem+ahwrTLi8vT61atUp3GXst2ony8ssvT0MlJaJB3RYtWoTHRkOFp59+elWXhGpw3nnnuWzChAkua9asWZVfO9rpsFevXi675pprXNa5c2eXXX311S5bvXp1BaurGbhDAAAAaAgAAAANAQAAEA0BAAAQDQEAAFAG/JZBrVq+J2natKnLLrjgApdF08vRpPPEiRNd9sQTT6RYobRp0yaXrV27NuX3p6K831p48MEHXRZ9jZ06dXLZoEGDXPbll1+67H//939TqLBmWLt2rX71q1+lu4wa6dBDD0352FWrVu3DSrCvRP+OU/2Nguhn3NixY8Njo597d9xxh8uOO+44l/32t7912bRp01z2zDPPuCz6e0bKnW2OuUMAAABoCAAAAA0BAAAQDQEAAFAGDBVGz3mPth++/fbbXXbAAQe4bMaMGS6LhkcWLlyYaonVYseOHWEeDeK88sorLouGa370ox+5LNr2OJeGCnfu3KkVK1aku4ysN2zYMJc99dRT4bE7d+502dNPP13lNWHf++KLLyr83hEjRrgs2l5dkkaNGuWyrl27umy//fZz2cyZM102cOBAl7355psuGz9+fFjPFVdcEeY1DXcIAAAADQEAAKAhAAAAoiEAAADKgKHCaNe9G2+80WWNGzd22fr16132+9//3mXz58+vWHEZYMuWLS4rbxBnT40aNXLZgAEDXBbtAFZTFRQU5NQQ5d6KBrd++ctfuuycc85x2a5du8JzRjuKTpo0ae+LQ9r17NnTZWbmshdffNFl0c+tVq1ahddp27ZtStdJ1VtvveWyaF3feeed4fuj2p988skK15OpuEMAAABoCAAAAA0BAAAQDQEAAFA1DhXWqRNfqm/fvi6LBk2i3c6mT5/usmioMEmSVErMSEVFRS7bvHmzy6I/n/z8fJd17tzZZfvvv7/LaurjPnfv3q2CgoK0XDsaoP3www9dFu22GQ3sLViwILxO9IjrDh06uOzkk092WZ8+fVwWrcFnn33WZSNHjgzridYrao7o5+urr76a0nt//vOfh3nDhg1Tuk5lPPzwwy6L/j6SpGuuucZl0WOWt23bVvnC0og7BAAAgIYAAADQEAAAANEQAAAAVeNQ4WGHHRbmV155pcuiHanWrVvnsmgoZPny5RWoLrvUrl3bZRs3bnRZNMQWadmypcuWLVu213Xhb4t2nZwyZYrLfvzjH7usSZMmLhsyZEjVFFbGr3/9a5eNHj3aZQwKYm9FP4/OPvvslN//2GOPuawyA8KFhYUue/DBB8NjX375ZZddd911LrvvvvsqXE8m4A4BAACgIQAAADQEAABANAQAAEDVOFTYr1+/MI+GDWvV8n1KtCvb4sWLXVZcXFyB6jJXtGNXtMNcXl5eSuebN2+ey6KdClH1vvvuO5dFg0mRaKiwWbNm4bHNmzd3WYMGDVw2btw4l/Xv399lTz/9tMs++uij8NpAeSZOnOiyww8/PDx2zpw5Lvv3f/93l1X1z/s33ngjzN99912XXX311S5jqBAAAGQ9GgIAAEBDAAAAaAgAAICqcajwH/7hH8I8ekTvjh07XPb222+7LBrSylZ169YN87POOstlgwYNclmjRo1cFg1dPvDAAy77+uuvUykRaRTtRBllUvzvPRJ9T02bNs1l0aDVZZdd5rLo0eNSdj9+HD+0atWqlI6LBgDbt2/vsvLWxu233+6yaKfPqlbeY99ffPFFl0VDudHA91//+tfKF1ZNuEMAAABoCAAAAA0BAAAQDQEAABANAQAAUDX+lkG0Ba8UT9cvXbrUZXPnznVZ9DzrbFCvXj2X9e7dOzz22muvddmRRx7pspUrV7rsqaeectlrr73msq1bt4bXRs0W/TZPtMV4tHXxlClTXHbUUUeF11m0aFEFqkMmmjRpksui3zjp0aOHy8zMZY899lh4nXfeeacC1e07X331lcuiLfYvvPBCl/FbBgAAIKvQEAAAABoCAABAQwAAAFSNQ4WpbnkpxUNuRUVFLtu1a1elaqoO0dBkNLh1yy23hO8//vjjU7rO+++/77JXX33VZevWrUvpfMhN0ffUjTfe6LL+/fu7LNpuVpIGDx5c+cKQETZv3uyy6667zmX33nuvy6It0m+77baUr5NOS5YscVlN3JKbOwQAAICGAAAA0BAAAADREAAAAFXjUOH69evDPNotrUGDBi474YQTXLZ69WqXLV++3GXRc+PL2zkx2n1q06ZN4bF7Ovjgg102aNAgl11xxRUu69ixY3jOqPZPPvnEZdHzupctW+aymjgIg30rWv/R4G+fPn2qoRpkmpkzZ7rstNNOS0Ml+868efNctnDhQpdFu8hmE+4QAAAAGgIAAEBDAAAAREMAAABUjUOF//3f/x3mLVu2dNnAgQNdNmLECJeddNJJLps+fbrLosG8tm3bhvW0aNEizPcUDT62adPGZe3bt3dZ9PjjFStWhNeZOnWqy1566SWXffzxxy7L1sdD55Lyhlv3tGXLln1cSfmGDh3qslatWrlsxowZ1VEOkBGiocJomDL6XokeV58JuEMAAABoCAAAAA0BAAAQDQEAAFA1DhUuWLAgzJ977jmXRbv2RY8BPuOMM1zWq1cvlxUUFLhs//33D+uJ8uLiYpfl5eW5LD8/32XRToxfffWVy1577bWwnnvuucdle/MoaaTHT3/6U5dFO1RGj6h+7LHH9klNqejZs6fLojUYPUZ7+PDh+6QmIFtEQ8JNmzZ1GUOFAAAgY9EQAAAAGgIAAEBDAAAAVI1DheWJHp156623uuz666932XHHHeeyQw45xGXRroLlPQZ4586dKR0bHRc9bjjaoTHaVXDu3LlhPQwQZqe7777bZY0bN3ZZtLNmtFNn5NNPPw3zLl26uKxdu3YuGzBggMuinTWLiopcFg3vzp8/P6wHqInMzGVr1651WfT3QqbiDgEAAKAhAAAANAQAAEA0BAAAQBkwVBjt5BcNGkaPBz7llFNc1qFDB5dFgx5r1qwJ69m8eXNKx6aaRV8far5Bgwa57K677nLZqFGjUjpfNMBUnvIGZvcUfU9FuyROmDDBZfPmzUu5HqAmir7Ptm7d6rLo75RMxR0CAABAQwAAAGgIAACAaAgAAIBoCAAAgDLgtwwi0Vap0faP2bQlJHLLjBkzXHbmmWe6rHnz5i7r3r27y1q1alWpepYsWeKyWbNmuaywsLBS1wFyxcKFC13WtWvXNFRSdbhDAAAAaAgAAAANAQAAEA0BAABQhg4VAjVRNCz7zTffpJQByCyLFy9OdwlVjjsEAACAhgAAANAQAAAA0RAAAABJluqz0yXJzFI/GPA+SpKkRzoLYA2jkljDyHblrmHuEAAAABoCAABAQwAAAERDAAAAtPc7Fa6TtHxfFIKccHi6CxBrGJXDGka2K3cN79VvGQAAgJqJ/2UAAABoCAAAAA0BAAAQDQEAABANAQAAEA0BAAAQDQEAABANAQAAEA0BAAAQDQEAABANAQAAUA42BGa2zMzOSOG4xMw6VPAaFX4vkArWMbIdazjz5FxDkKnMrLGZTTSzNaWvO9JdE7C3WMfIdlZirJmtL33db2aW7rqqw94+/hj7zjhJB0hqI6m5pLfMbHmSJM+ktSpg77COke2ulHS+pK6SEkl/lLRU0uNprKla5OwdAjM7wczeM7NNZrbKzB41s/w9DjvLzJaa2Toze8DMapV5/6Vm9pmZbTSzN8ysss9JP0fS/UmSbEuSZJmkpyRdWslzooZjHSPbZeAavljSQ0mSrEiSZKWkhyRdUslzZoWcbQgkFUu6UVJTSSdKOl3SiD2OuUBSD0ndJZ2n0h9sZna+pNGSLpTUTNJ0Sb+LLmJmo0oXevja8/A9Pj6m4l8ecgTrGNku09bw0ZLmlPl8TmlW8yVJklMvScsknRHkN0iaUubzRFLfMp+PkPRW6cevSbqszD+rJWmbpMPLvLfDXtb1vKSXJTWQ1EHSF5J2pPvPi1dmvljHvLL9lcFruFjSkWU+P6L0PJbuP7N9/crZOwRm1tHMXjGz1Wa2RdI9KulQy/q6zMfLJR1S+vHhkh4u01luUMl/CbWqREnXSfpO0ueSpqqky11RifMhB7COke0ycA0XSmpY5vOGkgqT0u6gJsvZhkDSryUtlHREkiQNVXLbac9J0kPLfHyYpG9KP/5a0s+SJGlc5lU3SZJ397yImY02s8LyXt8flyTJhiRJhiRJ0jJJkqNV8u/m/Sr8elEzsY6R7TJqDUuar5KBwu91Lc1qvFxuCBpI2iKp0MyOlHRVcMxIM2tiZodKul7SC6X545JuNbOjJcnMGpnZgOgiSZLckyRJ/fJe3x9nZu3N7CAzq21m/VQy6XpX1X25qKFYx8h2GbWGJT0n6SYza2Vmh0i6WdKzVfKVZrhcbghukTRYUoGkJ/R/C6ysqZI+kjRb0jSVTEwrSZIpksZKmlR6i2uepH6VrOd4SXNL67lX0pAkSXKiK0WlsI6R7TJtDU+Q9AeVrON5pdebUMlzZgXLgf8tAgAA/o5cvkMAAABK0RAAAAAaAgAAQEMAAABEQwAAALSXTzs0M34lAZWxLkmSZuksgDWMSmINI9uVu4a5Q4DqtDzdBQCVxBpGtit3DdMQAAAAGgIAAEBDAAAAREMAAABEQwAAAERDAAAAREMAAABEQwAAAERDAAAAREMAAABEQwAAAERDAAAAREMAAABEQwAAAERDAAAAREMAAABEQwAAAERDAAAAREMAAABEQwAAAERDAAAAREMAAABEQwAAAERDAAAAJNVJdwEAAFSV/Px8l+Xl5bmsT58+Ljv55JPDc7733nsumz59usvWr1+fSokZizsEAACAhgAAANAQAAAA0RAAAAAxVAgAyFLRAOHpp5/ushEjRrise/fuLmvQoEF4nWgAccyYMS57/fXXXVZUVBSeMxNxhwAAANAQAAAAGgIAACAaAgAAoBweKox2rpKkn/3sZyllxxxzjMtuuukml40bN64C1QEV16hRI5cdddRRLqtdu7bLlixZ4rJvv/22agoDUtS8eXOXde7c2WUtWrRw2ejRo1N678aNG11mZmE9Rx99tMs6dOjgsuh7L5t2L+QOAQAAoCEAAAA0BAAAQDQEAABAOTJUeMABB7hs6tSp4bHRLlczZsxw2apVq1wWPTqToUL8LdFgX8uWLV120kknuey4444Lz9m+fXuXRbulHXHEESm9d8eOHS578sknXfbYY4+F9URDiUmShMei5qpVy//3Z7RboCSNHDnSZW3btnVZs2bNXLZ48WKXjR8/3mULFixw2TnnnBPWU79+fZe9//77LisoKAjfny24QwAAAGgIAAAADQEAABANAQAAUA0cKqxXr57LpkyZ4rJoeFCS7r77bpeNHTvWZQceeKDLUt3RbciQIS7bvn17eOzkyZNTOiey07HHHuuyaACqTZs2LnvzzTfDc86fP99la9ascdlbb73lsg0bNrisU6dOLrvyyitdtnTp0rCeZ555JsxRc0UDhO3atXPZqFGjwvdHA9rRev+P//gPl02aNMll0d8LBx10kMteeOGFsJ5obW/ZsiU8NptxhwAAANAQAAAAGgIAACAaAgAAoCwfKmzdurXLoh3UogHC8oZZot3Wtm7d6rLCwkKXnXrqqS576KGHXNatWzeXrVu3LqyHocKarUuXLi6LdiqMhqeiYVkpfqzr5s2bXVa3bl2XNWnSxGXRMFg0ULVp06awHuSeVq1auWzw4MEu69WrV/j+v/zlLy4bM2aMy7788kuX5efnuyza6fOGG25w2axZs8J6/uu//stlDBUCAIAaiYYAAADQEAAAABoCAAAgGgIAAKAs+i2D6HnUv/3tb10WbXl53333uez+++9P+dqHHXaYy26++WaXXXXVVS6LtrycO3euyw4++OCU60HNEW0VHE0vX3jhhS5r1KhReM5ly5a5bNu2bS5r2LChy84880yXNWjQwGVPPPGEy95+++2wHuSevn37uiz6LYP169eH7//973/vsq+++splu3fvdlm0bfLRRx/tsp49e7os+u0ISfr0009dNnXq1JTqySbcIQAAADQEAACAhgAAAIiGAAAAKIuGCu+55x6XRQOEf/rTn1x21113pXydaPvh3/zmNy6Lhhwvvvhil0Xby0YDKshNr7zyistatGjhsj59+risc+fO4TmjPBpAjAZZO3bs6LK1a9e6LNriO9oeGTVfvXr1XHbccce5rHnz5i575JFHwnNOnz7dZbt27Uqpnu3bt7ts4cKFLlu0aJHLOnXqFJ4z+p6Khmiz/XuAOwQAAICGAAAA0BAAAADREAAAAGXoUGH0PPihQ4e6bM2aNS6LnnEdPXM+2jVLkkaMGOGyaEeqK6+80mXl7bq1pzZt2rgseoY9ctNTTz3lsmjntvJ2VYvWV7Qj4vLly11WXFzssmhQNxpynDlzZlhPqt8XyE69evVy2cCBA102e/Zslz3//PPhOVeuXFnpusqKhhSj3Tbvvvvu8P3t2rVzWbTTJ0OFAAAg69EQAAAAGgIAAEBDAAAAlKFDhdEugNFOa++8847LxowZ47ILLrjAZUVFReG1X3rpJZcNHz7cZZs2bQrfD+wL0W6BUSbFw1uVEe3eFu1ON3r06Cq9LjLPgQce6LJzzz3XZXXq+L9aXn75ZZd98cUX4XWSJKlAdeUrLCx0WbRj7EEHHRS+v3v37imdM9txhwAAANAQAAAAGgIAACAaAgAAoAwdKvzmm29cNn/+fJf17t3bZdGOf9EjiG+77bbw2tFjMoFc0a1bt5SOiwaqduzYUcXVIJ3MzGUdOnRIKYseGR8NFVb18GB59ttvP5cdeuihLsvLywvfHz0qPBq2LW+3zmzBHQIAAEBDAAAAaAgAAIBoCAAAgDJ0qHDbtm0u69u3r8uiQY/oka7l7egG4IeOPPLIlI5j+Lbma9CggcuiRx1HO/5NmDDBZatXr66awiogeqx3lO3N3xWnnXaayxgqBAAAWY+GAAAA0BAAAAAaAgAAoAwdKoysXLkypQxAxXXt2tVl0a6E2T48hb8vetRxv379XPbhhx+6bOvWrS7bvXt31RRWAdGOiE2bNnVZQUFB+P5du3a5bNWqVZUvLMNwhwAAANAQAAAAGgIAACAaAgAAIBoCAACgLPotAwBVa//993dZtHXx+++/77IZM2bsk5qQOaL10aZNG5dNmjTJZdH28+nUsmVLl0VbD7do0SJ8/9y5c122dOnSyheWYbhDAAAAaAgAAAANAQAAEA0BAAAQQ4VAzjr99NNdFj3vfvz48S6bP3/+PqkJ6VGrlv9vw8MOO8xlHTt2dNmyZctcFm1dvC/Url3bZa1bt3bZhRde6LIf//jHLou2KJak9957z2XR153tuEMAAABoCAAAAA0BAAAQDQEAABBDhUBOyM/Pd1m0e1uTJk1c9tlnn7ls48aNVVMYMsJBBx3ksmgnvxUrVrgsGsTbvXt3peqJ1mu7du1cFg0QXn311S7r2bOny3bu3OmyqVOnhvXceeedLtu0aVN4bDbjDgEAAKAhAAAANAQAAEA0BAAAQAwVAjkhGr7q3r27y6IdCOfMmbNPakLmiB51XK9ePZdFw4LnnXeey9q2beuydevWuay8nQGj9Tpw4ECXde3a1WV5eXkumz59ust+97vfuWz27NlhPTVxgDDCHQIAAEBDAAAAaAgAAIBoCAAAgBgqBHJCjx49XHbWWWe57J133nHZ8uXL90lNyBwbNmxwWTRM2rt3b5dde+21Ltu+fbvLVq9e7bLy1la0E+YXX3zhsvvuu89l27Ztc1k0FBg9vrioqCisJ1dwhwAAANAQAAAAGgIAACAaAgAAIIYK02LhwoUua9WqVXhstFvY1q1bq7wm1GzRo2zbtGnjsocffthl5e0mh5ojGsSbPHmyy9asWeOy4cOHuyzaLfDPf/6zy8p7jPa8efNc9sEHH7hsx44dLqtdu7bLkiRxWWUf0VwTcYcAAADQEAAAABoCAAAgGgIAACCGCtNizJgxLnvxxRfDY4cOHeqyxx9/vMprQs3WrVu3lI4r7/GvqNmiobtod78//OEPKWVNmzZ1WTQA+N1334X1FBcXp1Rjqu9FarhDAAAAaAgAAAANAQAAEA0BAAAQDQEAAJBkqU5uSpKZpX4wylW3bl2Xlbcd8dq1a1327LPPuuzOO+90WWFh4d4Xt299lCRJj3QWkKtrONoitnHjxi47+OCDXRY9xz6HsYaR7cpdw9whAAAANAQAAICGAAAAiIYAAACIrYvTYvv27S67/vrrw2Oj59M3a9bMZRk4QIg0idZHNBgYrUMGCIHcxR0CAABAQwAAAGgIAACAaAgAAIDYqRDVi13eMkh+fr7Ldu7cmYZKsgprGNmOnQoBAED5aAgAAAANAQAAoCEAAADa+50K10lavi8KQU44PN0FiDX8/zFAWCGsYWS7ctfwXv2WAQAAqJn4XwYAAICGAAAA0BAAAADREAAAANEQAAAA0RAAAADREAAAANEQAAAA0RAAAADREAAAANEQAAAA5WBDYGbLzOyMFI5LzKxDBa9R4fcCqWAdI9uxhjNPzjUEmc7M8s1soZmtSHctwN4yszvMbJeZFZZ5tUt3XUCqzOw0M/uzmW02s2Xprqc60RBknpGS1qS7CKASXkiSpH6Z19J0FwTsha2SnlbJz+KckrMNgZmdYGbvmdkmM1tlZo+aWf4eh51lZkvNbJ2ZPWBmtcq8/1Iz+8zMNprZG2ZW6eekm1lbSf8q6d7Kngu5IRPXMbA3Mm0NJ0nyfpIkv5GUc41szjYEkool3SipqaQTJZ0uacQex1wgqYek7pLOk3SpJJnZ+ZJGS7pQUjNJ0yX9LrqImY0qXejha4/Dx5ee97vKf3nIEZm4js8xsw1mNt/MrqqKLxI1Wiau4dyUJElOvSQtk3RGkN8gaUqZzxNJfct8PkLSW6UfvybpsjL/rJakbZIOL/PeDntZ1wWSXi/9+FRJK9L9Z8Urc18ZvI47SzpEUm1JvSStkvSTdP958cq8V6au4TLnOkPSsnT/OVXnK2fvEJhZRzN7xcxWm9kWSfeopEMt6+syHy9XyQ86STpc0sNlOssNkkxSqwrWUk/S/ZKurcj7kbsyaR1LUpIkC5Ik+SZJkuIkSd6V9LCkiyp6PtR8mbaGc1nONgSSfi1poaQjkiRpqJLbTrbHMYeW+fgwSd+Ufvy1pJ8lSdK4zKtu6Q/AHzCz0XtMXP/gVXrYEZLaSJpuZqslvSzp4NJvkDZV9QWjRsqkdRxJgnqAsjJ9DeeMXG4IGkjaIqnQzI6UFP2/zpFm1sTMDpV0vaQXSvPHJd1qZkdLkpk1MrMB0UWSJLkn+eHE9Q9epYfNU8mC71b6ulzSt6Uffx2cFvheJq1jmdl5pdcyMztB0nWSplbdl4saKNPWcC0z219SXsmntn8w5Fgj5XJDcIukwZIKJD2h/1tgZU2V9JGk2ZKmSXpKkpIkmSJprKRJpbe45knqV9FCkiQpSpJk9fcvldz22l36eXFFz4uckDHruNQgSUtK63lO0tgkSSZW8pyo2TJtDfdWyWD3qyq5G/GdpDcrec6sYKXDEwAAIIfl8h0CAABQioYAAADQEAAAABoCAAAgqc7eHGxmTCCiMtYlSdIsnQWwhlFJrGFku3LXMHcIUJ2Wp7sAoJJYw8h25a5hGgIAAEBDAAAAaAgAAIBoCAAAgGgIAACAaAgAAIBoCAAAgGgIAACAaAgAAIBoCAAAgGgIAACAaAgAAIBoCAAAgGgIAACAaAgAAIBoCAAAgGgIAACAaAgAAIBoCAAAgGgIAACAaAgAAIBoCAAAgGgIAACAaAgAAICkOukuAACA6ta4cWOXnX/++eGxJ598ssseffRRl82ePbuSVaUXdwgAAAANAQAAoCEAAACiIQAAAGKoEECWqVevnstq1fL/bVNYWBi+P0mSKq8Jma1+/fouO/300102atSo8P0HHnigy7788kuXLViwwGU7d+5MpcSMwB0CAABAQwAAAGgIAACAaAgAAIAYKgSwj5mZy1q2bOmy3r17u6xXr14uu/zyy11Wt25dl918881hPePGjQtz1FyNGjVyWc+ePV3WoUOH8P2bN292WTSoWKeO/yuVoUIAAJBVaAgAAAANAQAAoCEAAABiqDArRUNaRx11lMv+7d/+zWUdO3Z02bXXXuuyjz76qILVoaZp2rSpy37yk5+4rG3btuH7o8HAE044ofKF/R09evTY59dA5snLy3NZly5dXHbiiSe6rHbt2uE5161b57LFixe7LJsGCCPcIQAAADQEAACAhgAAAIiGAAAAiKFC55hjjnFZ9JjLrVu3Vkc5oejxr4899pjLTjnlFJd9/vnnLhs6dKjLGCrMfA0bNgzzQYMGuSwa+Dv77LNTuk6LFi1cFg0a7gvbt2932cKFC1127733umzWrFn7pCZkjuix1+eee67LRo4c6bJu3bq5rKioKLzO7NmzXTZ37tyU358tuEMAAABoCAAAAA0BAAAQDQEAAFAODxW2a9cuzP/617+6bMqUKS4bNmxYldeUqhtvvNFl0QDhxx9/7LL+/fu77KCDDqqawlCtonUpSaeddlo1V/K3Rd9TO3bscNkjjzzismgNr1y5smoKQ9Y77LDDXDZgwACXRTsV7rfffi774IMPwutMnjzZZdFwa7bjDgEAAKAhAAAANAQAAEA0BAAAQDQEAABAOfJbBtHUdXkT2nXr1nXZ66+/XuU1pSrahnbUqFEui7bRPOuss1y2Zs0al3377bcVrA7ptDfbB0dbqv7xj39M6b1/+tOfXFZYWOiyV155JXz/6tWrXZYkSUrXBr6Xl5fnsosuushlJ510ksuin+vRttgvvPBCeO0///nPLou+B7IddwgAAAANAQAAoCEAAACiIQAAAKqBQ4WpDhCW9yz5u+66y2XlDZpUh0cffdRlu3btctmZZ57psmiAEJnPzFx2//33u+zYY48N3x+thVS3DwYyVbTehwwZ4rKWLVumdL63337bZeUNxq5fvz6lc2Y77hAAAAAaAgAAQEMAAABEQwAAAJRFQ4W1a9d22ciRI132i1/8wmX5+fku69OnT3idaPgq0rx585SOKygocNngwYNdFj0LXpKKi4tdduKJJ7qMAcLsFK3rSy65xGXDhw932Zdffhme85133nFZNIgKZKrGjRu7bNiwYS7r1KmTy+rU8X+tRT+Hx48f77Lyvqd2794d5jUNdwgAAAANAQAAoCEAAACiIQAAAMrQocLoMZcvvfSSy84555yUzhcN5k2YMGHvCyujQYMGLot2mPvuu+9c1qZNm5SvEw059uvXz2XRI2ZzZXetbHbHHXe4LBqMjZQ3KPjGG2+47L333nPZzp07Xfb000+7LFpbDCmiKkRDtZJ0/vnnu2zo0KEuix5rHHnxxRddFj3SOPqeyCXcIQAAADQEAACAhgAAAIiGAAAASLIkSVI/2Cz1gyth3LhxLrv++uur49JZKxqa+Zd/+Zc0VPI3fZQkSY90FlBda7hRo0Yu+8///E+XXXTRRS6rV69eSteIhlglaeHChSkd27Fjx5SOmzhxosuuuuoql0UDtDVQzqzhqhatrdatW4fHTpo0yWW9evVyWbSD4NKlS112wQUXuGzevHnhtXNAuWuYOwQAAICGAAAA0BAAAADREAAAAGXAToVDhgxxWaoDhNFOfHPmzHHZggULXDZ58uTwnH/5y19SunZkyZIlLmvfvr3Lot3pxowZU+HrIvPceuutLrv44osrfL4HHnjAZdFOa5L05ptvuiwa6DriiCNc1qOHnzWKHs39+eefu2zs2LFhPUVFRWGO3BIN2p599tnhsd26dUvpnNHfAdFum5999llK58t13CEAAAA0BAAAgIYAAACIhgAAACgDhgqHDx/usnXr1rks2i0tGmKK3rsvnHrqqS5r166dy6Ihxfvuu29flIQMMmDAgJSO+/jjj112zTXXuOzDDz90WWWH9aIdDVeuXOmyn/zkJy6LBn+jnRglhgpzUfQI+y5durisvEHbAw44wGXROlq2bJnLpk2b5rLi4uLwOvgh7hAAAAAaAgAAQEMAAABEQwAAAERDAAAAlAG/ZTBs2DCXRc9Vj6afq8shhxzisilTprhs165dLou2Zt6xY0fVFIaMFT1//fjjj3fZJ5984rLZs2fvi5JSUlBQ4LJnnnnGZf369XNZ9DVL0vPPP1/5wpBVmjRp4rLoN7OOO+648P27d+92WbRddrRN8aJFi1KoEBHuEAAAABoCAABAQwAAAERDAAAAlAFDhUuWLEl3CT9w5ZVXuizaSjYaBrv00ktdls5hSKTPp59+mlKWDRYvXuyyjRs3uqxnz57h+xkqrNn2339/l/Xu3dtl0SBqfn5+eM5ouPWPf/yjy6Lhboa2K447BAAAgIYAAADQEAAAANEQAAAAZcBQYbp069YtzEeMGOGy6NncY8eOdVn0bG7kpmjQqm/fvi6L1kw6dyqsU8f/SOjfv7/L6tev77J33313n9SEzNalSxeXDR482GU/+tGPUj5n9H0xffp0l3377bcpnxN/H3cIAAAADQEAAKAhAAAAoiEAAADK4aHCY489NsyPOuool/3mN79x2RtvvFHlNaHm6N69u8tefvlll0W7F06cONFl0WNeN2/eXMHqSkTDsj//+c9ddtttt7ls3rx5LnvppZcqVQ+yU/Qzs3Pnzi6rXbu2y6JHxkvxz9dZs2ZVoDrsDe4QAAAAGgIAAEBDAAAAREMAAACUw0OFAwcODPPCwkKXNWjQwGUXX3yxy6JhMOSmOXPmuCx6FHa0y9tDDz3ksptuusllzz77bHjtaFArLy/PZWeffbbLOnbs6LJNmza5bPTo0SldFzVfkyZNXFavXr2U3lveo4oXLVrksuiR26ha3CEAAAA0BAAAgIYAAACIhgAAACiHhwpfffXVMC8uLnbZuHHjXDZz5swqrwk1x9atW112zDHHuGzAgAEuu/POO13WqlUrl/3iF7+oYHXliwYfb7nlFpe9/vrrVX5tZKdGjRq5LNoFM7Jt27Ywjx7DHe10iKrFHQIAAEBDAAAAaAgAAIBoCAAAgCRLkiT1g81SPxjwPkqSpEc6C8iGNdy0aVOX9e/f32XlDW6de+65LqtVy/f+8+fPd9ntt9/usoKCgvA6OYo1vIeTTz7ZZaeccorLWrdu7bJoiFWKHxW+ePFilxUVFaVSIn6o3DXMHQIAAEBDAAAAaAgAAIBoCAAAgGgIAACA+C0DVC8mtJHtWMPIdvyWAQAAKB8NAQAAoCEAAAA0BAAAQDQEAABANAQAAEA0BAAAQDQEAABANAQAAEA0BAAAQDQEAABANAQAAEA0BAAAQDQEAABAUp29PH6dpOX7ohDkhMPTXYBYw6gc1jCyXblr2JKER2sDAJDr+F8GAACAhgAAANAQAAAA0RAAAADREAAAANEQAAAA0RAAAADREAAAANEQAAAASf8PMN5losFxCgEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 648x648 with 9 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "images = next(train_batch)\n",
    "show_img_grid(\n",
    "    [images[0][idx] for idx in range(9)],\n",
    "    [f\"label={images[1][idx]}\" for idx in range(9)],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-08T17:32:49.116788Z",
     "iopub.status.busy": "2020-12-08T17:32:49.110946Z",
     "iopub.status.idle": "2020-12-08T17:32:49.163285Z",
     "shell.execute_reply": "2020-12-08T17:32:49.161977Z",
     "shell.execute_reply.started": "2020-12-08T17:32:49.115856Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_model(n_output_classes=10):\n",
    "    model = tl.Serial(\n",
    "        tl.ToFloat(),\n",
    "        tl.Conv(32, (3, 3), (1, 1), \"SAME\"),\n",
    "        tl.LayerNorm(),\n",
    "        tl.Relu(),\n",
    "        tl.AvgPool(strides=(2,2)),\n",
    "        tl.Conv(64, (3, 3), (1, 1), \"SAME\"),\n",
    "        tl.LayerNorm(),\n",
    "        tl.Relu(),\n",
    "        tl.AvgPool(strides=(2,2)),\n",
    "        tl.Flatten(),\n",
    "        tl.Dense(256),\n",
    "        tl.Relu(),\n",
    "        tl.Dense(n_output_classes),\n",
    "        tl.LogSoftmax(),\n",
    "    )\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-08T17:32:58.167017Z",
     "iopub.status.busy": "2020-12-08T17:32:58.166718Z",
     "iopub.status.idle": "2020-12-08T17:32:58.180970Z",
     "shell.execute_reply": "2020-12-08T17:32:58.180203Z",
     "shell.execute_reply.started": "2020-12-08T17:32:58.166988Z"
    }
   },
   "outputs": [],
   "source": [
    "model = get_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-08T17:30:18.503847Z",
     "iopub.status.busy": "2020-12-08T17:30:18.503174Z",
     "iopub.status.idle": "2020-12-08T17:30:18.512542Z",
     "shell.execute_reply": "2020-12-08T17:30:18.511781Z",
     "shell.execute_reply.started": "2020-12-08T17:30:18.503819Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Serial[\n",
      "  ToFloat\n",
      "  Conv\n",
      "  LayerNorm\n",
      "  Serial[\n",
      "    Relu\n",
      "  ]\n",
      "  MaxPool(2,2)\n",
      "  Conv\n",
      "  LayerNorm\n",
      "  Serial[\n",
      "    Relu\n",
      "  ]\n",
      "  MaxPool(2,2)\n",
      "  Flatten_keep1\n",
      "  Dense_256\n",
      "  Serial[\n",
      "    Relu\n",
      "  ]\n",
      "  Dense_10\n",
      "  LogSoftmax\n",
      "]\n"
     ]
    }
   ],
   "source": [
    "print(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make sure the model is working."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-07T03:53:37.179437Z",
     "iopub.status.busy": "2020-12-07T03:53:37.179197Z",
     "iopub.status.idle": "2020-12-07T03:53:37.567329Z",
     "shell.execute_reply": "2020-12-07T03:53:37.566693Z",
     "shell.execute_reply.started": "2020-12-07T03:53:37.179410Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(((),\n",
       "  (ndarray<<tf.Tensor: shape=(5, 5, 1, 16), dtype=float32, numpy=\n",
       "   array([[[[ 5.91100194e-02,  7.42365792e-02,  4.12834734e-01,\n",
       "             -6.59424663e-01, -2.39922386e-02, -3.18918377e-01,\n",
       "             -7.88308159e-02,  3.63800019e-01,  6.30915016e-02,\n",
       "             -9.55872890e-03, -3.47371221e-01, -1.22353315e-01,\n",
       "              4.70524102e-01, -1.15175627e-01, -1.00883290e-01,\n",
       "             -3.54151309e-01]],\n",
       "   \n",
       "           [[-4.66780335e-01,  1.81568116e-01,  8.73525560e-01,\n",
       "             -8.62190500e-02,  4.61799741e-01,  1.51982412e-01,\n",
       "             -1.15936518e-01, -4.46387142e-01, -1.73292667e-01,\n",
       "             -1.17932297e-01,  1.38963908e-01,  1.95878986e-02,\n",
       "             -2.54022032e-01, -5.04785292e-02, -1.48370132e-01,\n",
       "             -1.77843884e-01]],\n",
       "   \n",
       "           [[-7.58526385e-01,  1.34381980e-01, -6.34019613e-01,\n",
       "             -3.96193862e-01, -4.17205662e-01,  9.24890190e-02,\n",
       "             -5.73916286e-02,  4.91443694e-01, -2.68169612e-01,\n",
       "              1.50136769e-01, -1.53146788e-01, -4.42513674e-01,\n",
       "             -2.74810553e-01,  1.39818266e-01,  3.20293933e-01,\n",
       "             -6.30552351e-01]],\n",
       "   \n",
       "           [[ 7.01668739e-01,  2.73835093e-01, -7.29688883e-01,\n",
       "              1.67338803e-01, -6.23908825e-02, -8.27975124e-02,\n",
       "              1.38063729e-01, -1.51194915e-01,  8.34025979e-01,\n",
       "             -3.19617003e-01, -5.06067276e-01, -1.21243652e-02,\n",
       "             -2.84737229e-01, -2.46785298e-01,  1.63351759e-01,\n",
       "             -2.95551032e-01]],\n",
       "   \n",
       "           [[-1.54490069e-01,  2.83491999e-01, -4.14964333e-02,\n",
       "              2.39120275e-01, -1.56554077e-02,  4.96514201e-01,\n",
       "              3.83904904e-01, -7.25178197e-02,  4.25563097e-01,\n",
       "             -8.95712674e-01,  1.52626604e-01,  2.38353387e-01,\n",
       "              6.14620924e-01,  3.43144573e-02,  3.51073384e-01,\n",
       "             -4.85653020e-02]]],\n",
       "   \n",
       "   \n",
       "          [[[ 3.04332614e-01,  5.97579330e-02,  1.77448258e-01,\n",
       "             -2.95625538e-01, -3.45032215e-01, -1.51740760e-01,\n",
       "             -9.53051090e-01,  2.59518325e-01,  1.97441876e-01,\n",
       "              8.51124167e-01,  3.39812428e-01,  3.78138006e-01,\n",
       "              3.81053478e-01, -1.92718178e-01,  3.72094154e-01,\n",
       "              3.64964575e-01]],\n",
       "   \n",
       "           [[-7.37880945e-01, -1.42514586e-01, -1.88900858e-01,\n",
       "             -5.27029932e-02, -3.09059303e-02,  3.26338202e-01,\n",
       "              2.87440062e-01,  2.25212082e-01, -5.24917543e-01,\n",
       "              7.71345437e-01,  2.90411096e-02,  5.56010485e-01,\n",
       "             -1.37831718e-01,  1.87138992e-03,  2.78250039e-01,\n",
       "             -5.97221553e-02]],\n",
       "   \n",
       "           [[ 6.92932692e-04, -3.65827739e-01, -3.65341961e-01,\n",
       "              5.78978984e-03,  1.60679519e-01, -8.30364406e-01,\n",
       "              4.42780882e-01,  1.87110037e-01, -1.21564500e-01,\n",
       "              1.65757492e-01, -3.88141125e-02, -5.32510616e-02,\n",
       "             -3.05630356e-01,  4.40547734e-01, -2.32972875e-01,\n",
       "              6.75459921e-01]],\n",
       "   \n",
       "           [[ 1.95531175e-02, -1.90795034e-01, -8.92201718e-03,\n",
       "              1.83468387e-01, -4.19106930e-01,  2.79776305e-01,\n",
       "              2.20764011e-01, -2.30778441e-01, -1.49071142e-01,\n",
       "             -6.35861874e-01,  1.31946877e-01, -1.48703501e-01,\n",
       "             -4.34533864e-01, -3.74043733e-01, -8.21218312e-01,\n",
       "             -1.52887538e-01]],\n",
       "   \n",
       "           [[-1.93876043e-01, -3.91662344e-02,  5.49007177e-01,\n",
       "              3.97634059e-02,  1.45579383e-01,  3.33017595e-02,\n",
       "              4.43150401e-01,  8.09273198e-02,  1.55932888e-01,\n",
       "             -2.15972155e-01,  6.53401017e-01, -5.74724283e-03,\n",
       "             -4.87592041e-01,  1.75204769e-01,  9.30190757e-02,\n",
       "              6.71166554e-02]]],\n",
       "   \n",
       "   \n",
       "          [[[ 6.20768499e-03,  9.15260196e-01,  7.20706284e-01,\n",
       "              2.05833897e-01, -2.29682669e-01,  1.80067658e-01,\n",
       "              1.45886436e-01, -2.00192094e-01, -3.32993329e-01,\n",
       "              6.94626391e-01, -8.43389481e-02, -1.35783449e-01,\n",
       "             -5.47387540e-01, -6.03111722e-02,  3.61897230e-01,\n",
       "             -1.75255552e-01]],\n",
       "   \n",
       "           [[ 1.99269935e-01, -1.91950127e-02,  5.70408881e-01,\n",
       "              1.94201738e-01, -6.49681091e-02,  1.44760057e-01,\n",
       "             -3.70976068e-02,  3.31765801e-01, -4.07307982e-01,\n",
       "              9.22431469e-01,  2.66881883e-01, -3.95279974e-01,\n",
       "             -5.93972743e-01,  3.43125105e-01, -1.54550165e-01,\n",
       "              1.78090632e-02]],\n",
       "   \n",
       "           [[ 2.99658448e-01, -4.41315651e-01, -8.60226527e-03,\n",
       "              6.72416165e-02,  4.77829069e-01, -3.17705274e-01,\n",
       "              1.41881555e-01,  1.83187246e-01, -3.05728972e-01,\n",
       "              4.31260496e-01,  1.30052164e-01, -2.03397855e-01,\n",
       "             -6.12314522e-01, -9.27937686e-01,  8.38187099e-01,\n",
       "             -1.99584469e-01]],\n",
       "   \n",
       "           [[-3.80365580e-01,  6.15867198e-01, -3.10880691e-01,\n",
       "              1.83834287e-03, -2.70501543e-02,  8.79398510e-02,\n",
       "              3.35366279e-02, -3.81441474e-01,  1.43057004e-01,\n",
       "              3.04721415e-01,  4.60808367e-01,  1.36495661e-02,\n",
       "              1.07702076e-01, -8.37641239e-01, -1.06918879e-01,\n",
       "              1.06870025e-01]],\n",
       "   \n",
       "           [[-9.29007232e-01,  1.01307213e-01,  1.49133146e-01,\n",
       "              2.97238976e-01, -1.18152305e-01, -1.17041752e-01,\n",
       "             -1.52347833e-01,  5.14263451e-01,  5.23390993e-02,\n",
       "              1.81693006e-02, -5.21665275e-01, -4.42925692e-02,\n",
       "             -2.64706880e-01,  1.72500715e-01, -2.35033855e-01,\n",
       "             -2.55883396e-01]]],\n",
       "   \n",
       "   \n",
       "          [[[-1.05946876e-01,  7.08002523e-02, -4.08434361e-01,\n",
       "              3.99322331e-01, -2.30537698e-01,  1.33596417e-02,\n",
       "              6.18868589e-01, -1.54470369e-01,  3.03524047e-01,\n",
       "             -4.94861096e-01,  6.46665633e-01,  1.61837742e-01,\n",
       "             -1.68359112e-02, -1.42563716e-01, -2.41790920e-01,\n",
       "              4.39592570e-01]],\n",
       "   \n",
       "           [[ 4.18401808e-01,  2.56853193e-01,  2.74789155e-01,\n",
       "             -5.02839029e-01,  2.71119326e-01,  1.17832743e-01,\n",
       "              1.16093671e+00, -1.27621936e-02, -3.66566777e-01,\n",
       "              2.78004974e-01, -6.71633184e-01,  8.81149713e-03,\n",
       "             -2.16328248e-01,  3.70458126e-01,  3.80389810e-01,\n",
       "             -2.02269897e-01]],\n",
       "   \n",
       "           [[-3.70294064e-01,  2.25975573e-01, -3.21797699e-01,\n",
       "             -2.64860213e-01,  8.03270489e-02, -3.58026326e-01,\n",
       "              2.70997912e-01,  2.30767459e-01, -2.30331913e-01,\n",
       "              4.08002883e-01, -6.23384416e-02,  2.91239116e-02,\n",
       "              7.43637607e-02,  6.05036616e-01,  2.29266599e-01,\n",
       "              1.99364677e-01]],\n",
       "   \n",
       "           [[-2.63734072e-01, -2.19796121e-01, -9.78281498e-01,\n",
       "              1.55627087e-01,  1.91124305e-01,  5.01526415e-01,\n",
       "              1.32469386e-01,  1.00554608e-01,  2.50187248e-01,\n",
       "              1.09933361e-01, -1.62153766e-01,  5.92341661e-01,\n",
       "              2.41400376e-01,  7.42704928e-01,  6.90018594e-01,\n",
       "              2.54399151e-01]],\n",
       "   \n",
       "           [[ 2.58032888e-01,  5.50760031e-01,  3.38504016e-01,\n",
       "              2.27334917e-01, -2.92674243e-01,  3.41133952e-01,\n",
       "              5.58284342e-01,  2.34534249e-01,  1.29865438e-01,\n",
       "              5.45597732e-01, -4.51780289e-01,  4.33931947e-02,\n",
       "              2.89547592e-01, -3.01179260e-01, -5.37072420e-01,\n",
       "             -1.70432866e-01]]],\n",
       "   \n",
       "   \n",
       "          [[[-9.51891243e-02, -4.99543875e-01,  4.66196686e-01,\n",
       "             -1.57642975e-01,  1.98591530e-01,  6.27549231e-01,\n",
       "             -1.77125916e-01, -2.09613606e-01,  9.33057666e-01,\n",
       "              5.56538880e-01, -8.57270658e-01, -2.29085952e-01,\n",
       "             -5.50114930e-01,  5.26293032e-02,  4.54981238e-01,\n",
       "              2.15309009e-01]],\n",
       "   \n",
       "           [[ 7.60514021e-01,  1.21709846e-01,  4.11168516e-01,\n",
       "              2.84744143e-01,  1.73727021e-01, -3.57502028e-02,\n",
       "              4.89090979e-01,  4.93398219e-01,  1.78068310e-01,\n",
       "             -4.20131356e-01,  8.97865295e-02,  1.81275398e-01,\n",
       "              1.37995481e-01,  5.41491061e-02, -1.20132349e-01,\n",
       "             -3.35372686e-01]],\n",
       "   \n",
       "           [[ 1.17116347e-01, -3.19998324e-01,  1.84099704e-01,\n",
       "             -2.68930584e-01,  3.15472603e-01, -3.52102458e-01,\n",
       "              1.97902158e-01, -1.67161062e-01,  1.10882908e-01,\n",
       "             -2.51789838e-01, -9.39427316e-02, -4.53720719e-01,\n",
       "             -5.14703095e-01,  2.14130193e-01,  2.30037540e-01,\n",
       "              3.31199206e-02]],\n",
       "   \n",
       "           [[-1.08246177e-01,  4.73884761e-01, -1.47892758e-01,\n",
       "             -2.61939883e-01,  5.17636836e-01, -2.06760779e-01,\n",
       "              2.57584363e-01, -1.95466787e-01,  3.14623684e-01,\n",
       "              2.25939900e-01,  3.28460485e-01,  7.68203512e-02,\n",
       "             -3.72105360e-01, -2.88258381e-02, -3.02884489e-01,\n",
       "              3.44763070e-01]],\n",
       "   \n",
       "           [[ 3.79295528e-01,  1.36414438e-01, -5.98849431e-02,\n",
       "              4.84224081e-01,  3.53869319e-01, -3.79811078e-01,\n",
       "             -5.21581411e-01,  2.45805711e-01, -5.93008697e-01,\n",
       "              2.24767327e-01,  4.26107734e-01,  1.14657283e-02,\n",
       "              9.73459333e-02,  2.84145623e-01, -2.86572546e-01,\n",
       "             -1.33496866e-01]]]], dtype=float32)>>,\n",
       "   ndarray<<tf.Tensor: shape=(16,), dtype=float32, numpy=\n",
       "   array([-3.0721506e-07, -1.9611741e-07, -2.2513261e-06,  9.6569215e-07,\n",
       "           8.7837122e-08, -4.8932236e-07,  7.0473442e-07, -4.2480599e-07,\n",
       "          -8.1587740e-07,  2.1119077e-07,  3.2771709e-07,  7.4243491e-07,\n",
       "          -7.6050748e-07,  2.2316328e-07,  3.3240076e-07, -8.7253619e-07],\n",
       "         dtype=float32)>>),\n",
       "  (ndarray<<tf.Tensor: shape=(16,), dtype=float32, numpy=\n",
       "   array([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         dtype=float32)>>,\n",
       "   ndarray<<tf.Tensor: shape=(16,), dtype=float32, numpy=\n",
       "   array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "         dtype=float32)>>),\n",
       "  ((), (), ()),\n",
       "  (),\n",
       "  (ndarray<<tf.Tensor: shape=(5, 5, 16, 32), dtype=float32, numpy=\n",
       "   array([[[[ 2.95689762e-01,  6.62079155e-02, -1.98664099e-01, ...,\n",
       "             -5.07130444e-01,  7.34216198e-02, -5.78257218e-02],\n",
       "            [ 2.74971485e-01, -5.06651700e-01,  4.61498424e-02, ...,\n",
       "             -5.35000801e-01,  2.11436059e-02, -1.98559523e-01],\n",
       "            [-3.94377321e-01,  2.32908696e-01, -1.65530547e-01, ...,\n",
       "             -9.64086205e-02,  2.52304256e-01,  1.26737431e-01],\n",
       "            ...,\n",
       "            [-8.15186091e-03, -2.53345743e-02,  2.17264473e-01, ...,\n",
       "              2.37894833e-01,  2.87563175e-01, -2.17661276e-01],\n",
       "            [ 5.64976297e-02, -3.73212844e-01,  7.06396773e-02, ...,\n",
       "             -1.11579522e-01,  8.18011165e-02,  6.76721931e-02],\n",
       "            [ 1.43143684e-01, -1.22533269e-01,  1.13467671e-01, ...,\n",
       "             -1.70256987e-01, -3.09241824e-02, -7.78710842e-02]],\n",
       "   \n",
       "           [[ 2.31574178e-01, -6.07966818e-02, -1.97442308e-01, ...,\n",
       "              3.89131805e-04, -3.62486362e-01, -4.20972139e-01],\n",
       "            [-2.82048136e-01,  2.71581680e-01, -1.13102168e-01, ...,\n",
       "             -2.68476963e-01, -1.97737068e-01, -2.15139925e-01],\n",
       "            [-3.96390706e-02,  1.31343767e-01, -1.65589735e-01, ...,\n",
       "             -5.20528853e-02, -3.99649516e-02,  1.17750786e-01],\n",
       "            ...,\n",
       "            [-1.96722791e-01,  3.23065221e-01, -9.45684537e-02, ...,\n",
       "              1.79297134e-01,  7.08314776e-02,  2.67685294e-01],\n",
       "            [ 1.61489800e-01,  2.92875588e-01,  2.94968247e-01, ...,\n",
       "             -1.30648002e-01, -7.20544606e-02, -2.99815476e-01],\n",
       "            [-1.27852201e-01, -1.36185154e-01, -2.24859081e-03, ...,\n",
       "              2.43887380e-01,  2.03305110e-01, -4.74144548e-01]],\n",
       "   \n",
       "           [[-2.10846424e-01, -3.63863975e-01,  1.67650744e-01, ...,\n",
       "              1.48581147e-01,  4.15309727e-01, -1.69740453e-01],\n",
       "            [ 1.23436093e-01, -5.72722964e-02,  2.12081298e-01, ...,\n",
       "              1.68566331e-01, -1.33173645e-01, -2.44808242e-01],\n",
       "            [ 1.84780270e-01,  1.68121308e-01,  7.02617085e-03, ...,\n",
       "             -2.11633369e-01,  2.30879843e-01,  3.28543991e-01],\n",
       "            ...,\n",
       "            [-8.69032834e-03,  2.92023987e-01, -2.14149907e-01, ...,\n",
       "             -3.55483472e-01, -1.74012423e-01, -4.50377852e-01],\n",
       "            [ 2.23237038e-01,  1.01585098e-01, -2.59171993e-01, ...,\n",
       "              1.07475840e-01,  1.84249982e-01, -8.23494941e-02],\n",
       "            [ 4.59792465e-02,  3.26119810e-01,  6.57260492e-02, ...,\n",
       "              4.64323431e-01,  5.63986748e-02, -2.72184610e-02]],\n",
       "   \n",
       "           [[ 1.46023870e-01, -1.47620887e-01, -5.99272512e-02, ...,\n",
       "              3.47116649e-01, -8.83995369e-02,  2.24698558e-02],\n",
       "            [ 1.71272397e-01, -8.11562613e-02,  3.83241922e-01, ...,\n",
       "              4.18659717e-01,  4.38081384e-01, -1.15761431e-02],\n",
       "            [ 3.21402520e-01,  3.07987910e-02,  5.00324607e-01, ...,\n",
       "             -2.88480613e-03,  8.59532654e-02,  2.00994194e-01],\n",
       "            ...,\n",
       "            [ 8.55379179e-02,  7.74726868e-02,  4.45672832e-02, ...,\n",
       "             -1.18018594e-03, -2.31375396e-01,  4.44452390e-02],\n",
       "            [ 1.83626398e-01, -2.50489563e-01,  1.18793920e-01, ...,\n",
       "             -9.41509828e-02,  3.25258791e-01,  1.14698417e-01],\n",
       "            [ 1.11238845e-01,  1.20104529e-01,  5.80673516e-01, ...,\n",
       "              2.18439803e-01, -3.23837489e-01, -2.65071094e-01]],\n",
       "   \n",
       "           [[ 1.28051743e-01, -2.56665796e-01,  7.90473190e-04, ...,\n",
       "             -5.12548089e-02,  2.50991131e-03,  9.87858418e-03],\n",
       "            [-1.56203538e-01,  9.71109495e-02, -2.60851592e-01, ...,\n",
       "             -1.48398593e-01, -1.81322366e-01,  7.47953206e-02],\n",
       "            [-2.32805282e-01, -1.50102928e-01, -3.80863063e-02, ...,\n",
       "             -7.67705739e-02, -2.77508665e-02, -1.51123941e-01],\n",
       "            ...,\n",
       "            [-3.15659307e-02, -3.34298849e-01, -4.66463834e-01, ...,\n",
       "              1.37234971e-01, -1.54712573e-02, -3.34953249e-01],\n",
       "            [-3.09682161e-01, -2.02320158e-01, -2.95418561e-01, ...,\n",
       "             -2.35428557e-01, -1.01067446e-01,  9.30207502e-03],\n",
       "            [-9.31343362e-02, -6.99635223e-02, -2.03395590e-01, ...,\n",
       "              2.07644761e-01,  3.62870693e-01,  2.01926693e-01]]],\n",
       "   \n",
       "   \n",
       "          [[[-1.62447348e-01, -9.04930085e-02,  3.35225761e-02, ...,\n",
       "              9.35433805e-03, -1.01897836e-01,  2.31956825e-01],\n",
       "            [-9.72747877e-02,  4.09823060e-01,  6.66131079e-02, ...,\n",
       "             -1.82389438e-01, -5.49266413e-02,  7.27145970e-02],\n",
       "            [-1.24949247e-01, -7.24006593e-02, -4.02542740e-01, ...,\n",
       "             -3.79502386e-01,  3.05942774e-01, -7.20708072e-02],\n",
       "            ...,\n",
       "            [-6.91689998e-02, -8.44089240e-02, -2.78491169e-01, ...,\n",
       "              3.47173601e-01,  3.82001221e-01,  3.45299691e-01],\n",
       "            [-3.10018547e-02, -5.85449152e-02,  5.95057532e-02, ...,\n",
       "             -3.41281503e-01,  2.49418393e-01, -7.53465816e-02],\n",
       "            [-3.46832633e-01,  1.25844881e-01, -6.39341027e-03, ...,\n",
       "              1.78984180e-01, -5.72188795e-02,  2.18711972e-01]],\n",
       "   \n",
       "           [[ 1.96677655e-01, -1.92795634e-01, -4.15255763e-02, ...,\n",
       "              8.49761907e-03,  1.56237381e-02, -8.16685930e-02],\n",
       "            [ 1.18770398e-01,  7.17683285e-02,  1.29281834e-01, ...,\n",
       "             -3.10128450e-01, -1.91193074e-01,  1.98516883e-02],\n",
       "            [ 4.68341321e-01,  1.28111303e-01,  5.45552492e-01, ...,\n",
       "             -2.36943290e-01, -4.33292612e-02,  4.30657744e-01],\n",
       "            ...,\n",
       "            [-1.22881845e-01, -2.51961827e-01, -3.18700492e-01, ...,\n",
       "             -7.64626265e-02,  1.58411860e-01,  7.71563202e-02],\n",
       "            [ 4.52831462e-02, -1.17914699e-01,  6.42744377e-02, ...,\n",
       "             -1.21004179e-01,  1.04202203e-01,  3.12465757e-01],\n",
       "            [-4.05466147e-02,  4.18109447e-01,  9.22586769e-02, ...,\n",
       "              1.07969321e-01, -5.18649220e-01,  4.28276323e-02]],\n",
       "   \n",
       "           [[ 1.21930249e-01, -1.19736172e-01, -2.75697887e-01, ...,\n",
       "              3.35219055e-02, -4.30324525e-01, -5.45033701e-02],\n",
       "            [-4.29296531e-02, -4.37524736e-01,  3.70974123e-01, ...,\n",
       "             -1.83510244e-01,  4.14408557e-02, -1.13123201e-01],\n",
       "            [ 1.23664446e-01,  1.06463999e-01,  1.63854703e-01, ...,\n",
       "             -2.00581238e-01, -1.09870091e-01,  3.75765890e-01],\n",
       "            ...,\n",
       "            [-1.81339905e-02, -1.36265069e-01,  3.01938951e-01, ...,\n",
       "             -1.82516441e-01, -2.54841715e-01, -5.08369841e-02],\n",
       "            [ 9.82709527e-02,  6.86439732e-03,  1.61885843e-02, ...,\n",
       "              4.48756032e-02, -3.05187497e-02, -1.42918676e-02],\n",
       "            [ 9.68233198e-02,  4.99065846e-01,  1.20540068e-01, ...,\n",
       "              3.40810567e-01,  6.19495288e-02,  6.94423467e-02]],\n",
       "   \n",
       "           [[-1.03755027e-01, -2.48894483e-01, -4.52848710e-02, ...,\n",
       "              2.19400123e-01, -2.89039761e-02,  3.72298099e-02],\n",
       "            [-8.40552244e-03,  6.43256977e-02,  7.90388361e-02, ...,\n",
       "              2.53872156e-01, -7.41980076e-02, -1.66643262e-01],\n",
       "            [ 1.50448918e-01, -1.86867639e-02, -2.36409321e-01, ...,\n",
       "              2.22116470e-01, -2.44245842e-01,  2.44531333e-02],\n",
       "            ...,\n",
       "            [ 1.39712309e-02,  1.13230078e-02,  4.86174375e-02, ...,\n",
       "             -8.79071653e-02,  2.25506350e-01,  1.44121438e-01],\n",
       "            [-6.92209303e-02,  2.04311833e-01,  4.04695421e-01, ...,\n",
       "             -1.35400109e-02,  6.43226877e-02, -3.73912066e-01],\n",
       "            [ 2.93134183e-01, -6.40677363e-02,  5.42861931e-02, ...,\n",
       "             -1.76533133e-01, -2.36330688e-01, -2.28882238e-01]],\n",
       "   \n",
       "           [[-3.03107351e-01, -7.92052895e-02,  9.13772359e-02, ...,\n",
       "              2.27078319e-01,  6.74604252e-02,  1.33207947e-01],\n",
       "            [-3.06978375e-01,  9.31198746e-02,  2.27015298e-02, ...,\n",
       "             -1.02937639e-01, -2.34754935e-01,  1.09271951e-01],\n",
       "            [ 6.91666156e-02, -1.42824456e-01, -3.72475013e-02, ...,\n",
       "              4.47277911e-03, -1.37859195e-01, -1.48187935e-01],\n",
       "            ...,\n",
       "            [ 1.27938852e-01, -2.92800426e-01, -3.33507538e-01, ...,\n",
       "             -8.15749094e-02,  5.87563179e-02, -1.90046802e-01],\n",
       "            [ 1.79103270e-01, -2.03421399e-01, -2.15261858e-02, ...,\n",
       "              7.10068420e-02, -1.39760375e-01,  6.31047995e-04],\n",
       "            [-3.64535749e-02,  1.29360199e-01, -1.97108641e-01, ...,\n",
       "             -1.46763548e-01,  5.03456414e-01,  2.63188332e-01]]],\n",
       "   \n",
       "   \n",
       "          [[[ 3.69727284e-01,  1.29731447e-01, -5.37255816e-02, ...,\n",
       "              8.48643631e-02,  1.66118771e-01,  2.27962747e-01],\n",
       "            [-3.14718962e-01, -2.94495225e-01,  5.71407378e-01, ...,\n",
       "              4.15142119e-01,  2.46840179e-01,  2.74532456e-02],\n",
       "            [-1.17075657e-02,  4.70657721e-02,  2.19597891e-01, ...,\n",
       "              1.21120833e-01, -5.55096231e-02, -2.03220442e-01],\n",
       "            ...,\n",
       "            [-2.52451479e-01, -1.33824110e-01,  2.19124630e-02, ...,\n",
       "              2.69093305e-01, -4.51246500e-01,  1.67064071e-01],\n",
       "            [ 3.44184458e-01, -2.87850976e-01, -1.32894635e-01, ...,\n",
       "             -7.53876492e-02,  1.35177523e-01, -2.07646918e-02],\n",
       "            [ 1.92317471e-01, -8.00824091e-02, -1.17123179e-01, ...,\n",
       "              1.28217533e-01, -8.57008249e-02,  1.16724633e-01]],\n",
       "   \n",
       "           [[-6.00472353e-02, -2.00659007e-01, -5.13662621e-02, ...,\n",
       "              2.26109043e-01,  3.40309320e-03, -1.47473201e-01],\n",
       "            [ 1.49146482e-01, -5.43261580e-02,  1.40942767e-01, ...,\n",
       "              8.71982202e-02,  2.87221950e-02, -2.58694608e-02],\n",
       "            [-3.53943408e-02,  3.20093155e-01, -1.99767604e-01, ...,\n",
       "              1.44532114e-01, -2.57970691e-01,  9.43849832e-02],\n",
       "            ...,\n",
       "            [ 2.04686493e-01,  3.37923914e-02,  2.28657238e-02, ...,\n",
       "              3.33618134e-01,  1.78884283e-01,  1.25592470e-01],\n",
       "            [-1.99656725e-01, -9.25944969e-02,  2.03005821e-01, ...,\n",
       "             -2.19093114e-01, -5.25723733e-02,  2.89226025e-01],\n",
       "            [-8.75724554e-02, -3.16875398e-01, -1.83332637e-02, ...,\n",
       "             -8.33303202e-04,  1.92311294e-02,  1.86264947e-01]],\n",
       "   \n",
       "           [[-1.68815777e-01,  3.06443740e-02,  2.69209653e-01, ...,\n",
       "              1.48058832e-01,  1.01436190e-02, -2.70417452e-01],\n",
       "            [-1.55076265e-01,  4.96424884e-02,  3.18171322e-01, ...,\n",
       "             -2.06512764e-01, -4.66666855e-02,  9.81324911e-02],\n",
       "            [-3.23583893e-02,  3.46207470e-02,  2.29689941e-01, ...,\n",
       "             -9.37327147e-02, -2.67351456e-02,  8.87936279e-02],\n",
       "            ...,\n",
       "            [-2.17470407e-01,  3.45301896e-01,  1.12814300e-01, ...,\n",
       "             -3.60195130e-01, -2.85796881e-01,  7.16971010e-02],\n",
       "            [ 2.81583875e-01, -4.77254353e-02, -8.09855945e-03, ...,\n",
       "              7.07269162e-02,  4.70883042e-01,  1.13878638e-01],\n",
       "            [ 3.37584883e-01,  2.40475535e-01,  2.32185647e-02, ...,\n",
       "              2.59220004e-01,  1.61167100e-01, -1.00966059e-01]],\n",
       "   \n",
       "           [[-2.25555018e-01,  1.70022264e-01, -1.66938663e-01, ...,\n",
       "              1.60995889e-02, -2.38434538e-01,  7.68032074e-02],\n",
       "            [ 1.97264820e-01, -1.89908996e-01,  3.82455066e-02, ...,\n",
       "              2.11017318e-02,  3.04610223e-01,  6.29017130e-02],\n",
       "            [-1.60390586e-01, -1.07728407e-01,  8.60561132e-02, ...,\n",
       "             -2.55772829e-01, -1.18058048e-01, -5.38448468e-02],\n",
       "            ...,\n",
       "            [ 1.57929927e-01,  5.12894273e-01,  1.37648776e-01, ...,\n",
       "             -4.71801698e-01, -2.02381209e-01, -6.55831620e-02],\n",
       "            [ 2.15728417e-01, -1.76596150e-01, -3.36297393e-01, ...,\n",
       "              1.99205890e-01, -2.58331358e-01,  9.86026824e-02],\n",
       "            [-1.51692405e-01,  9.26795155e-02,  9.22621340e-02, ...,\n",
       "             -1.04784757e-01, -2.81181514e-01, -1.07847312e-02]],\n",
       "   \n",
       "           [[ 1.71703622e-01,  1.42451391e-01, -1.39081731e-01, ...,\n",
       "              2.21401826e-01, -1.17748283e-01, -7.14854524e-02],\n",
       "            [-7.88805187e-02, -2.12676719e-01, -7.64802098e-03, ...,\n",
       "              2.35465318e-01,  1.98144853e-01, -2.00841993e-01],\n",
       "            [ 3.74778770e-02, -6.20396547e-02,  1.25901043e-01, ...,\n",
       "             -3.76146346e-01,  1.11308567e-01, -5.37561513e-02],\n",
       "            ...,\n",
       "            [ 2.08964765e-01, -3.03061455e-01, -8.07060450e-02, ...,\n",
       "              7.49775916e-02,  8.09011161e-02, -1.21475518e-01],\n",
       "            [-7.24246725e-02, -1.78630322e-01, -2.96344489e-01, ...,\n",
       "             -1.08458050e-01,  4.55838628e-02, -5.97684346e-02],\n",
       "            [ 1.20279985e-02,  1.12294316e-01, -9.84323397e-02, ...,\n",
       "             -3.59249562e-01,  1.33221984e-01,  5.41142784e-02]]],\n",
       "   \n",
       "   \n",
       "          [[[-9.61110219e-02,  1.69471443e-01, -1.94371194e-01, ...,\n",
       "             -1.33597657e-01,  2.38102391e-01, -9.92622823e-02],\n",
       "            [-4.41119671e-01,  7.42692649e-02, -7.25764781e-02, ...,\n",
       "             -3.12513769e-01,  9.37430412e-02,  2.98953336e-02],\n",
       "            [ 1.22273780e-01, -7.63044208e-02, -6.87909648e-02, ...,\n",
       "             -4.86998782e-02, -1.74154863e-01, -3.65438730e-01],\n",
       "            ...,\n",
       "            [ 1.29873127e-01, -6.50904030e-02, -1.55978441e-01, ...,\n",
       "              4.53277677e-01,  9.01983213e-03,  2.79871345e-01],\n",
       "            [-1.81007236e-01,  2.69723266e-01, -6.60620093e-01, ...,\n",
       "             -1.04580335e-01, -7.59884641e-02, -3.55659962e-01],\n",
       "            [-9.89237577e-02, -8.88276100e-02,  1.09674744e-01, ...,\n",
       "             -1.20349973e-01, -1.38183847e-01,  2.02843875e-01]],\n",
       "   \n",
       "           [[-1.69616282e-01, -6.09441549e-02, -4.50303070e-02, ...,\n",
       "              1.31041735e-01, -1.86644569e-01,  3.88329536e-01],\n",
       "            [-1.14144236e-01,  1.51319683e-01,  3.43286574e-01, ...,\n",
       "              2.32842028e-01,  2.77273417e-01,  8.49968493e-02],\n",
       "            [-3.27775806e-01, -2.08921075e-01,  2.58307070e-01, ...,\n",
       "             -4.16688859e-01,  1.85773388e-01, -1.55357793e-01],\n",
       "            ...,\n",
       "            [ 1.09629154e-01, -2.93375790e-01,  2.98076905e-02, ...,\n",
       "              4.67290636e-03,  2.40354747e-01, -1.41759604e-01],\n",
       "            [-1.56936139e-01, -6.32938296e-02, -1.28223985e-01, ...,\n",
       "             -1.44296169e-01, -1.51989833e-01,  9.30998623e-02],\n",
       "            [-1.97159231e-01, -1.39620811e-01,  3.56241800e-02, ...,\n",
       "             -1.97572514e-01,  1.55606419e-01,  3.08439910e-01]],\n",
       "   \n",
       "           [[-1.69365987e-01, -1.79356355e-02, -1.62956625e-01, ...,\n",
       "              2.30908871e-01,  4.10611838e-01, -3.00955866e-02],\n",
       "            [-2.49175280e-01,  1.82883620e-01, -2.65825242e-01, ...,\n",
       "              1.73960909e-01, -2.14924783e-01, -2.43090078e-01],\n",
       "            [-1.14580922e-01, -3.16884249e-01, -2.99593490e-02, ...,\n",
       "              1.56455010e-01,  1.85528174e-01,  2.67557412e-01],\n",
       "            ...,\n",
       "            [-1.70245796e-01,  9.03630927e-02,  1.30872577e-01, ...,\n",
       "              3.67808342e-02,  2.46099159e-01, -1.81777462e-01],\n",
       "            [-2.32871827e-02,  3.22212994e-01,  1.11882523e-01, ...,\n",
       "             -1.15714587e-01, -2.22410157e-01, -2.15815052e-01],\n",
       "            [ 4.32661831e-01, -5.67369312e-02,  1.90786853e-01, ...,\n",
       "              2.62447651e-02, -8.53329971e-02,  2.44647667e-01]],\n",
       "   \n",
       "           [[-4.61788744e-01,  4.42463189e-01, -2.79436439e-01, ...,\n",
       "              9.55307856e-03,  3.16784441e-01, -8.39788839e-02],\n",
       "            [ 6.14072382e-02,  8.64192937e-03,  5.33985943e-02, ...,\n",
       "              3.21995378e-01, -1.95606396e-01,  2.60174245e-01],\n",
       "            [-1.81638762e-01, -7.13192448e-02,  1.06130034e-01, ...,\n",
       "             -2.76662201e-01,  2.63233393e-01,  1.17232963e-01],\n",
       "            ...,\n",
       "            [-7.68263862e-02,  9.37624872e-02, -3.04292012e-02, ...,\n",
       "              1.79134652e-01, -3.46810341e-01, -1.69565156e-01],\n",
       "            [ 7.54548237e-02, -2.17738777e-01, -1.12682708e-01, ...,\n",
       "              1.18433712e-02, -3.20397407e-01, -3.85079324e-01],\n",
       "            [ 9.42471400e-02,  8.13565180e-02, -1.76150545e-01, ...,\n",
       "             -1.91916034e-01,  8.76053572e-02,  9.05700848e-02]],\n",
       "   \n",
       "           [[ 5.17899618e-02,  2.41614297e-01, -1.67617962e-01, ...,\n",
       "             -1.74946278e-01, -3.19641940e-02,  2.92924047e-01],\n",
       "            [ 2.61956871e-01,  5.70435524e-02,  9.44453105e-02, ...,\n",
       "             -6.96243765e-03,  2.64123995e-02,  1.16686031e-01],\n",
       "            [-5.45624830e-02,  5.77520877e-02,  1.35531604e-01, ...,\n",
       "             -3.33280414e-02, -3.71521175e-01, -2.04627797e-01],\n",
       "            ...,\n",
       "            [-3.33946347e-01,  2.03452155e-01, -4.89454642e-02, ...,\n",
       "             -2.00137123e-03, -1.19860649e-01,  9.04711559e-02],\n",
       "            [-2.73662329e-01, -2.39322618e-01, -8.45204294e-02, ...,\n",
       "             -1.26208320e-01,  1.25528112e-01, -1.55517861e-01],\n",
       "            [-3.86384204e-02, -3.58285918e-03, -2.23181754e-01, ...,\n",
       "             -1.32323653e-01,  9.99704599e-02, -1.44708142e-01]]],\n",
       "   \n",
       "   \n",
       "          [[[ 3.89815420e-01, -1.91631526e-01,  1.25546724e-01, ...,\n",
       "              3.27181637e-01, -7.27228448e-02,  1.16955519e-01],\n",
       "            [-1.65510178e-02, -2.93107271e-01,  6.61652833e-02, ...,\n",
       "             -1.63607031e-01,  8.73364359e-02,  6.99537545e-02],\n",
       "            [-2.89597452e-01,  1.47610754e-01, -3.79333466e-01, ...,\n",
       "             -6.42380655e-01,  2.43922010e-01, -3.18215102e-01],\n",
       "            ...,\n",
       "            [-1.94643795e-01,  7.46915460e-01,  2.93644547e-01, ...,\n",
       "             -1.46706790e-01, -3.66323031e-02,  7.69934654e-02],\n",
       "            [ 4.37117033e-02, -6.99826255e-02, -2.27253154e-01, ...,\n",
       "             -2.56237358e-01,  6.64421245e-02,  3.25812548e-02],\n",
       "            [-6.67922711e-03,  7.92114586e-02, -2.52220463e-02, ...,\n",
       "             -1.64260194e-01,  4.94103692e-02,  9.50684100e-02]],\n",
       "   \n",
       "           [[-3.03515851e-01, -1.20786831e-01,  2.09761247e-01, ...,\n",
       "             -4.90321964e-02,  8.49436447e-02,  1.46961585e-01],\n",
       "            [ 3.03556800e-01, -1.04182810e-01,  1.02444328e-01, ...,\n",
       "             -3.88114601e-02,  1.90128863e-01, -1.21002950e-01],\n",
       "            [-4.17629719e-01,  8.92618001e-02,  9.81254131e-02, ...,\n",
       "              2.62371302e-01,  1.84841797e-01, -4.52746414e-02],\n",
       "            ...,\n",
       "            [ 1.88620821e-01,  2.72341013e-01,  2.38417611e-01, ...,\n",
       "             -7.29448199e-02, -6.51768595e-02, -3.22508544e-01],\n",
       "            [ 1.90343291e-01, -1.89750001e-01,  1.19686216e-01, ...,\n",
       "              1.25317559e-01, -4.45067167e-01, -7.52470875e-03],\n",
       "            [ 1.37779430e-01,  1.86547101e-01,  3.07920873e-01, ...,\n",
       "              9.61425453e-02, -1.73623502e-01,  3.38871479e-02]],\n",
       "   \n",
       "           [[-9.98537317e-02, -2.57822961e-01,  1.05756186e-01, ...,\n",
       "             -1.01895683e-01,  2.47422487e-01, -7.51796663e-02],\n",
       "            [ 3.61355394e-01,  1.19307257e-01, -1.69676512e-01, ...,\n",
       "              1.44173997e-02,  3.02993596e-01,  4.33130227e-02],\n",
       "            [-2.38071784e-01,  3.28083821e-02, -2.43736938e-01, ...,\n",
       "             -3.25636357e-01, -1.52028963e-01, -5.57533085e-01],\n",
       "            ...,\n",
       "            [ 1.21078044e-01,  2.54583120e-01,  1.13467328e-01, ...,\n",
       "              2.36544773e-01, -4.04001474e-01,  1.42183080e-01],\n",
       "            [-1.08600602e-01, -1.59723219e-02, -7.50271007e-02, ...,\n",
       "             -1.56637743e-01,  2.42668428e-02,  2.51764953e-02],\n",
       "            [-1.55130997e-01, -4.76707816e-02,  2.46290769e-02, ...,\n",
       "             -3.79102141e-01,  3.23072702e-01,  2.55645234e-02]],\n",
       "   \n",
       "           [[-3.12144727e-01, -6.61290064e-02, -6.90099597e-02, ...,\n",
       "             -1.72910288e-01, -9.67031121e-02,  2.70878285e-01],\n",
       "            [-5.16062319e-01,  7.47922659e-02,  1.28542081e-01, ...,\n",
       "              5.33966534e-02,  9.54298899e-02, -2.83338368e-01],\n",
       "            [-7.57523626e-02,  5.60546070e-02,  7.36310473e-03, ...,\n",
       "             -1.81265891e-01, -9.08946991e-02, -1.97712705e-01],\n",
       "            ...,\n",
       "            [-9.91135389e-02, -1.08654676e-02,  2.24425465e-01, ...,\n",
       "              2.65776545e-01,  9.54855978e-02,  1.63602322e-01],\n",
       "            [ 2.21342161e-01,  7.46598169e-02,  4.83048558e-02, ...,\n",
       "              2.73890138e-01, -9.24030244e-02, -7.92904571e-02],\n",
       "            [-2.80936986e-01,  1.82007216e-02, -3.15428555e-01, ...,\n",
       "             -2.61404127e-01,  5.36557697e-02, -1.62718650e-02]],\n",
       "   \n",
       "           [[-1.17229745e-01, -2.94855684e-01,  5.02559841e-02, ...,\n",
       "              2.81066298e-01,  1.60408631e-01,  1.09233506e-01],\n",
       "            [ 2.01587409e-01,  2.89567858e-01,  1.81748256e-01, ...,\n",
       "              1.39669254e-01, -4.64646369e-01,  1.80622518e-01],\n",
       "            [ 1.68294966e-01, -2.00932816e-01, -3.38616282e-01, ...,\n",
       "             -2.31434032e-01, -2.51236707e-01,  2.63717920e-01],\n",
       "            ...,\n",
       "            [-3.77996951e-01, -3.87146324e-02, -2.59434551e-01, ...,\n",
       "              1.05622672e-01,  1.78867027e-01,  1.82590052e-01],\n",
       "            [-8.03529099e-02,  1.65361892e-02,  2.39503682e-01, ...,\n",
       "             -1.01086283e-02,  2.70105481e-01,  5.72460182e-02],\n",
       "            [-2.72007525e-01,  1.12238400e-01, -4.55772988e-02, ...,\n",
       "              3.47214371e-01,  2.42115855e-01, -3.49192113e-01]]]],\n",
       "         dtype=float32)>>,\n",
       "   ndarray<<tf.Tensor: shape=(32,), dtype=float32, numpy=\n",
       "   array([ 7.0956651e-07,  7.2064569e-07, -1.6689721e-06,  3.1689916e-07,\n",
       "           1.4270959e-06, -7.6400454e-07, -2.2747654e-06, -1.3545406e-06,\n",
       "           1.1254360e-06, -4.3908848e-08, -3.1721672e-07, -6.0135591e-07,\n",
       "          -1.1000159e-06,  2.8713811e-07, -3.2474247e-07, -1.0354346e-06,\n",
       "           4.3793315e-07,  1.4196331e-06,  1.4449873e-06, -1.6563995e-07,\n",
       "          -9.8372709e-07, -1.2033893e-06, -1.8388236e-06,  3.3182931e-07,\n",
       "           2.6371023e-07, -1.5292293e-06,  1.9469026e-06,  9.4161641e-07,\n",
       "           1.6143292e-06,  2.7788496e-07,  8.0314749e-07, -2.2827714e-06],\n",
       "         dtype=float32)>>),\n",
       "  (ndarray<<tf.Tensor: shape=(32,), dtype=float32, numpy=\n",
       "   array([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "          1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         dtype=float32)>>,\n",
       "   ndarray<<tf.Tensor: shape=(32,), dtype=float32, numpy=\n",
       "   array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "         dtype=float32)>>),\n",
       "  ((), (), ()),\n",
       "  (),\n",
       "  (),\n",
       "  (ndarray<<tf.Tensor: shape=(21632, 10), dtype=float32, numpy=\n",
       "   array([[-4.68489900e-03, -2.98018940e-03, -1.38967698e-02, ...,\n",
       "           -8.40274524e-03,  6.40190206e-03,  1.58676691e-02],\n",
       "          [-7.19273463e-04,  5.69133088e-03,  2.59321555e-03, ...,\n",
       "            2.55225413e-03, -1.55899068e-02,  3.27689946e-03],\n",
       "          [ 2.92265601e-03,  5.48369065e-03, -1.52157946e-02, ...,\n",
       "            9.28519666e-03, -9.60628130e-03, -1.53968679e-02],\n",
       "          ...,\n",
       "          [ 2.95396894e-05,  3.06324475e-03, -9.90241952e-03, ...,\n",
       "            1.06119104e-02, -1.58535000e-02,  9.64261219e-03],\n",
       "          [ 5.77823259e-03,  5.92214987e-03,  2.69953161e-04, ...,\n",
       "            9.19976272e-03, -1.46838985e-02, -1.45543292e-02],\n",
       "          [-7.02929869e-03,  8.32364336e-03, -1.44729558e-02, ...,\n",
       "           -1.55110555e-02,  3.24561261e-03, -9.07444954e-03]], dtype=float32)>>,\n",
       "   ndarray<<tf.Tensor: shape=(10,), dtype=float32, numpy=\n",
       "   array([ 1.2078609e-06, -1.8373109e-07,  6.2526959e-07,  1.5330471e-07,\n",
       "          -5.3556181e-07,  7.0492143e-07,  7.3059783e-07,  1.2054099e-06,\n",
       "           1.5587167e-06, -3.9170763e-07], dtype=float32)>>),\n",
       "  ()),\n",
       " ((), (), (), ((), (), ()), (), (), (), ((), (), ()), (), (), (), ()))"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.init(trax.shapes.signature(images))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-07T03:53:37.911101Z",
     "iopub.status.busy": "2020-12-07T03:53:37.910872Z",
     "iopub.status.idle": "2020-12-07T03:53:37.992656Z",
     "shell.execute_reply": "2020-12-07T03:53:37.991926Z",
     "shell.execute_reply.started": "2020-12-07T03:53:37.911076Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(ndarray<<tf.Tensor: shape=(40, 10), dtype=float32, numpy=\n",
       " array([[-2.9722693 , -4.252158  , -3.0531437 , -1.0045846 , -2.2159855 ,\n",
       "         -3.5954227 , -2.9320014 , -3.8534117 , -4.923906  , -1.1943221 ],\n",
       "        [-2.3031063 , -4.557092  , -3.1261766 , -0.8969562 , -2.524972  ,\n",
       "         -3.3742464 , -2.6220512 , -3.849649  , -5.0705404 , -1.4990557 ],\n",
       "        [-3.3077242 , -4.269676  , -2.9998207 , -0.5171206 , -2.7020864 ,\n",
       "         -3.1602228 , -3.5374167 , -3.8792336 , -4.1650896 , -2.0509052 ],\n",
       "        [-3.3614984 , -3.9202263 , -3.9911637 , -0.4495175 , -2.990291  ,\n",
       "         -3.720287  , -2.9372625 , -4.8209248 , -5.682685  , -1.8965079 ],\n",
       "        [-3.60244   , -4.605439  , -2.4567509 , -0.5783191 , -3.7254725 ,\n",
       "         -3.6583397 , -2.4172654 , -2.9666986 , -6.1480474 , -2.0911927 ],\n",
       "        [-3.7094529 , -4.290477  , -2.8947887 , -0.692044  , -3.7405124 ,\n",
       "         -3.086965  , -2.6704342 , -1.9327414 , -4.0642033 , -2.2498388 ],\n",
       "        [-3.9933312 , -4.930169  , -3.1608465 , -0.33524108, -3.4737546 ,\n",
       "         -3.8321884 , -3.5417638 , -4.2843256 , -4.7912207 , -2.1797385 ],\n",
       "        [-1.7264374 , -3.9659326 , -2.852391  , -1.3206072 , -1.6733639 ,\n",
       "         -3.6234505 , -2.2928047 , -3.3175395 , -5.712831  , -2.0905433 ],\n",
       "        [-2.3328538 , -4.6805153 , -4.0966525 , -0.68692493, -3.648747  ,\n",
       "         -3.663448  , -3.1403582 , -2.466387  , -5.1374025 , -1.6699351 ],\n",
       "        [-2.2404358 , -3.5629077 , -2.390588  , -1.2034519 , -3.174119  ,\n",
       "         -1.9271631 , -1.7491415 , -2.9250636 , -5.049557  , -2.9543738 ],\n",
       "        [-2.8669326 , -4.3459873 , -3.7840042 , -0.81139445, -2.1404684 ,\n",
       "         -2.9713066 , -2.9865384 , -3.6310391 , -4.9204173 , -1.5600957 ],\n",
       "        [-1.953346  , -3.119027  , -3.0460682 , -0.75808144, -2.763587  ,\n",
       "         -2.8381584 , -2.4783356 , -3.2126648 , -4.472621  , -3.2004685 ],\n",
       "        [-2.4150589 , -5.115882  , -2.3196602 , -1.1495609 , -2.4610033 ,\n",
       "         -2.9167256 , -1.8485054 , -4.059542  , -5.343852  , -1.7685049 ],\n",
       "        [-2.4247327 , -4.7504907 , -3.9624398 , -0.52635837, -3.2897131 ,\n",
       "         -3.802434  , -3.2934997 , -4.0161414 , -6.048645  , -1.7373073 ],\n",
       "        [-2.5471802 , -4.3158116 , -4.1041155 , -0.90170956, -2.0842395 ,\n",
       "         -2.6252778 , -2.626844  , -3.507339  , -4.3532777 , -1.7487363 ],\n",
       "        [-1.9055791 , -4.765352  , -2.5791855 , -0.8505211 , -3.4925222 ,\n",
       "         -3.7603025 , -2.6609833 , -2.6953337 , -5.12427   , -1.9472557 ],\n",
       "        [-1.7495186 , -4.6317596 , -2.8115547 , -0.8647487 , -2.5460505 ,\n",
       "         -3.4830346 , -2.701921  , -3.7384439 , -5.587037  , -2.0293756 ],\n",
       "        [-2.5901306 , -4.775737  , -3.1521168 , -1.4080908 , -3.1348667 ,\n",
       "         -3.1572523 , -2.8103147 , -4.08446   , -5.376302  , -0.773283  ],\n",
       "        [-3.5649836 , -3.2681048 , -3.2716584 , -1.1117676 , -2.483621  ,\n",
       "         -2.7466998 , -3.7866118 , -4.4501324 , -5.72108   , -0.9636483 ],\n",
       "        [-2.8396595 , -5.18596   , -3.6091263 , -0.8606355 , -3.0430837 ,\n",
       "         -2.3978403 , -2.6181965 , -4.224134  , -4.981596  , -1.3745469 ],\n",
       "        [-2.739169  , -4.3371143 , -1.9011589 , -1.2633102 , -3.4375193 ,\n",
       "         -2.3110967 , -1.9970317 , -3.1469717 , -6.4310303 , -1.7228167 ],\n",
       "        [-2.4503522 , -4.878055  , -2.8564188 , -0.9012563 , -2.6954932 ,\n",
       "         -2.5646703 , -3.5563464 , -3.9974926 , -4.017678  , -1.4556928 ],\n",
       "        [-2.0043342 , -4.708591  , -2.7035375 , -1.6753722 , -2.6639886 ,\n",
       "         -2.1171708 , -2.3293705 , -2.9627917 , -5.1098156 , -1.3590071 ],\n",
       "        [-2.9909952 , -4.195113  , -3.705008  , -0.38240075, -4.350149  ,\n",
       "         -4.15299   , -2.9260824 , -3.7303877 , -6.0153947 , -2.1267967 ],\n",
       "        [-2.9466302 , -4.003836  , -4.2071257 , -0.43131423, -3.9444194 ,\n",
       "         -2.9552078 , -2.9627745 , -3.4571419 , -5.9073954 , -2.2316656 ],\n",
       "        [-3.3471196 , -5.303431  , -3.949458  , -0.23679471, -4.616018  ,\n",
       "         -4.173697  , -3.1529331 , -5.7799625 , -6.540862  , -2.5398874 ],\n",
       "        [-1.6488215 , -3.7898052 , -2.9851556 , -1.5212013 , -2.4279165 ,\n",
       "         -1.4924707 , -2.884916  , -3.7015753 , -4.701148  , -2.1761558 ],\n",
       "        [-2.5786924 , -4.6989856 , -1.6003362 , -1.1290832 , -2.5256598 ,\n",
       "         -4.3858585 , -3.773461  , -3.4368043 , -4.746641  , -1.4542056 ],\n",
       "        [-3.3213043 , -4.8432484 , -4.5197725 , -0.54303193, -4.2670217 ,\n",
       "         -3.1698241 , -2.628005  , -3.4528537 , -6.897337  , -1.5934912 ],\n",
       "        [-2.338185  , -5.037846  , -4.978675  , -0.93730235, -2.548047  ,\n",
       "         -3.5396278 , -3.5497272 , -3.21263   , -5.5526557 , -1.1446829 ],\n",
       "        [-2.0355668 , -4.089279  , -3.8702087 , -0.9307723 , -3.2404609 ,\n",
       "         -2.3743167 , -3.115237  , -3.6596434 , -5.8451896 , -1.4596751 ],\n",
       "        [-3.494752  , -4.6763916 , -4.037882  , -0.46178222, -3.9263794 ,\n",
       "         -3.4986737 , -2.8058324 , -2.712659  , -6.3022327 , -2.0103955 ],\n",
       "        [-2.5591245 , -5.048099  , -2.9131827 , -0.7499895 , -2.517412  ,\n",
       "         -3.460964  , -3.0217292 , -3.2119374 , -4.380456  , -1.7375599 ],\n",
       "        [-1.8940799 , -4.162657  , -2.6339579 , -1.1291171 , -2.1345325 ,\n",
       "         -2.866146  , -4.0663786 , -3.1117315 , -4.4635086 , -1.6583033 ],\n",
       "        [-2.2394204 , -3.398016  , -3.0035574 , -0.89338315, -2.61334   ,\n",
       "         -2.7053316 , -2.308841  , -2.796013  , -4.0394187 , -2.4890766 ],\n",
       "        [-3.2842677 , -4.1025085 , -3.835548  , -0.88852096, -2.4757297 ,\n",
       "         -3.844922  , -2.9252765 , -3.7579641 , -5.5359373 , -1.1186073 ],\n",
       "        [-4.112044  , -3.9705973 , -4.086546  , -0.86779404, -2.0864074 ,\n",
       "         -3.6558428 , -2.0335536 , -3.560401  , -4.2991886 , -1.5835696 ],\n",
       "        [-2.6972885 , -5.2218266 , -4.3753357 , -0.38092303, -4.2726154 ,\n",
       "         -4.579382  , -3.2961044 , -3.468296  , -4.8272    , -2.0326524 ],\n",
       "        [-4.285282  , -5.36203   , -3.3203855 , -0.23628116, -4.1923237 ,\n",
       "         -3.79296   , -3.8055303 , -4.0418944 , -5.1250205 , -2.624947  ],\n",
       "        [-2.7856255 , -3.645289  , -3.3304327 , -0.72640395, -3.1069143 ,\n",
       "         -3.640039  , -2.1844115 , -2.942385  , -5.767602  , -1.8748902 ]],\n",
       "       dtype=float32)>>,\n",
       " array([2, 9, 4, 2, 2, 4, 2, 8, 0, 4, 4, 1, 3, 0, 9, 0, 8, 5, 6, 0, 1, 4,\n",
       "        7, 3, 3, 2, 1, 5, 3, 9, 5, 7, 9, 1, 1, 6, 4, 0, 6, 3]),\n",
       " array([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 1.], dtype=float32))"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model(images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-09T00:24:25.745005Z",
     "iopub.status.busy": "2020-12-09T00:24:25.744765Z",
     "iopub.status.idle": "2020-12-09T00:24:25.772382Z",
     "shell.execute_reply": "2020-12-09T00:24:25.771781Z",
     "shell.execute_reply.started": "2020-12-09T00:24:25.744976Z"
    }
   },
   "outputs": [],
   "source": [
    "train_task = training.TrainTask(\n",
    "    labeled_data=train_batch,\n",
    "    loss_layer=tl.WeightedCategoryCrossEntropy(),\n",
    "    optimizer=trax.optimizers.Adam(0.00005),\n",
    "    #     lr_schedule=lr_schedules.warmup_and_rsqrt_decay(5, 0.1),\n",
    "    n_steps_per_checkpoint=100,\n",
    ")\n",
    "\n",
    "eval_task = training.EvalTask(\n",
    "    labeled_data=eval_batch,\n",
    "    metrics=[tl.WeightedCategoryCrossEntropy(), tl.Accuracy()],\n",
    "    n_eval_batches=20,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-09T00:24:26.277704Z",
     "iopub.status.busy": "2020-12-09T00:24:26.277473Z",
     "iopub.status.idle": "2020-12-09T00:24:26.433803Z",
     "shell.execute_reply": "2020-12-09T00:24:26.432179Z",
     "shell.execute_reply.started": "2020-12-09T00:24:26.277676Z"
    }
   },
   "outputs": [],
   "source": [
    "!rm -r mnist_cnn_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-09T00:35:34.435966Z",
     "iopub.status.busy": "2020-12-09T00:35:34.435563Z",
     "iopub.status.idle": "2020-12-09T00:37:05.165093Z",
     "shell.execute_reply": "2020-12-09T00:37:05.164544Z",
     "shell.execute_reply.started": "2020-12-09T00:35:34.435897Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Step   4100: Ran 100 train steps in 10.86 secs\n",
      "Step   4100: train WeightedCategoryCrossEntropy |  0.11098009\n",
      "Step   4100: eval  WeightedCategoryCrossEntropy |  0.04259281\n",
      "Step   4100: eval                      Accuracy |  0.98750000\n",
      "\n",
      "Step   4200: Ran 100 train steps in 8.36 secs\n",
      "Step   4200: train WeightedCategoryCrossEntropy |  0.06507728\n",
      "Step   4200: eval  WeightedCategoryCrossEntropy |  0.05611138\n",
      "Step   4200: eval                      Accuracy |  0.98000000\n",
      "\n",
      "Step   4300: Ran 100 train steps in 8.51 secs\n",
      "Step   4300: train WeightedCategoryCrossEntropy |  0.06066537\n",
      "Step   4300: eval  WeightedCategoryCrossEntropy |  0.06836465\n",
      "Step   4300: eval                      Accuracy |  0.97750000\n",
      "\n",
      "Step   4400: Ran 100 train steps in 8.23 secs\n",
      "Step   4400: train WeightedCategoryCrossEntropy |  0.07124132\n",
      "Step   4400: eval  WeightedCategoryCrossEntropy |  0.03294150\n",
      "Step   4400: eval                      Accuracy |  0.99500000\n",
      "\n",
      "Step   4500: Ran 100 train steps in 8.27 secs\n",
      "Step   4500: train WeightedCategoryCrossEntropy |  0.06107776\n",
      "Step   4500: eval  WeightedCategoryCrossEntropy |  0.06733363\n",
      "Step   4500: eval                      Accuracy |  0.98250000\n",
      "\n",
      "Step   4600: Ran 100 train steps in 8.22 secs\n",
      "Step   4600: train WeightedCategoryCrossEntropy |  0.05941249\n",
      "Step   4600: eval  WeightedCategoryCrossEntropy |  0.06753404\n",
      "Step   4600: eval                      Accuracy |  0.98000000\n",
      "\n",
      "Step   4700: Ran 100 train steps in 8.22 secs\n",
      "Step   4700: train WeightedCategoryCrossEntropy |  0.06282330\n",
      "Step   4700: eval  WeightedCategoryCrossEntropy |  0.02977730\n",
      "Step   4700: eval                      Accuracy |  0.99750000\n",
      "\n",
      "Step   4800: Ran 100 train steps in 8.20 secs\n",
      "Step   4800: train WeightedCategoryCrossEntropy |  0.06035573\n",
      "Step   4800: eval  WeightedCategoryCrossEntropy |  0.03933088\n",
      "Step   4800: eval                      Accuracy |  0.99000000\n",
      "\n",
      "Step   4900: Ran 100 train steps in 8.21 secs\n",
      "Step   4900: train WeightedCategoryCrossEntropy |  0.05716214\n",
      "Step   4900: eval  WeightedCategoryCrossEntropy |  0.04026582\n",
      "Step   4900: eval                      Accuracy |  0.99000000\n",
      "\n",
      "Step   5000: Ran 100 train steps in 8.34 secs\n",
      "Step   5000: train WeightedCategoryCrossEntropy |  0.06911314\n",
      "Step   5000: eval  WeightedCategoryCrossEntropy |  0.06031746\n",
      "Step   5000: eval                      Accuracy |  0.98000000\n"
     ]
    }
   ],
   "source": [
    "training_loop = training.Loop(\n",
    "    model, train_task, eval_tasks=[eval_task], output_dir=\"./mnist_cnn_model\"\n",
    ")\n",
    "\n",
    "training_loop.run(1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make predictions for a batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-07T04:04:04.455989Z",
     "iopub.status.busy": "2020-12-07T04:04:04.455084Z",
     "iopub.status.idle": "2020-12-07T04:04:04.470262Z",
     "shell.execute_reply": "2020-12-07T04:04:04.469542Z",
     "shell.execute_reply.started": "2020-12-07T04:04:04.455955Z"
    }
   },
   "outputs": [],
   "source": [
    "images = next(eval_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-07T04:04:05.402174Z",
     "iopub.status.busy": "2020-12-07T04:04:05.401874Z",
     "iopub.status.idle": "2020-12-07T04:04:05.458663Z",
     "shell.execute_reply": "2020-12-07T04:04:05.458019Z",
     "shell.execute_reply.started": "2020-12-07T04:04:05.402141Z"
    }
   },
   "outputs": [],
   "source": [
    "y_pred = np.argmax(model(images[0]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-07T04:04:06.350628Z",
     "iopub.status.busy": "2020-12-07T04:04:06.350319Z",
     "iopub.status.idle": "2020-12-07T04:04:06.354059Z",
     "shell.execute_reply": "2020-12-07T04:04:06.353255Z",
     "shell.execute_reply.started": "2020-12-07T04:04:06.350594Z"
    }
   },
   "outputs": [],
   "source": [
    "y = images[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-07T04:04:07.544878Z",
     "iopub.status.busy": "2020-12-07T04:04:07.544635Z",
     "iopub.status.idle": "2020-12-07T04:04:07.549814Z",
     "shell.execute_reply": "2020-12-07T04:04:07.549181Z",
     "shell.execute_reply.started": "2020-12-07T04:04:07.544850Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([12]),)"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.where(y != y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-07T04:05:38.147798Z",
     "iopub.status.busy": "2020-12-07T04:05:38.147559Z",
     "iopub.status.idle": "2020-12-07T04:05:38.151523Z",
     "shell.execute_reply": "2020-12-07T04:05:38.150715Z",
     "shell.execute_reply.started": "2020-12-07T04:05:38.147771Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "predicted = [9 6 7 9 4 3 0 6 4 6 4 1 8 2 4 4 9 1 7 3],\n",
      " actual   = [9 6 7 9 4 3 0 6 4 6 4 1 9 2 4 4 9 1 7 3]\n"
     ]
    }
   ],
   "source": [
    "print(f\"predicted = {y_pred},\\n actual   = {y}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-07T04:07:02.418368Z",
     "iopub.status.busy": "2020-12-07T04:07:02.418135Z",
     "iopub.status.idle": "2020-12-07T04:07:02.989679Z",
     "shell.execute_reply": "2020-12-07T04:07:02.988967Z",
     "shell.execute_reply.started": "2020-12-07T04:07:02.418341Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqsAAAKuCAYAAACG1lVCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABJiUlEQVR4nO3debzXc/r/8etqU7SNipQWKiEMTUgT5msZoiSUkT1bmGjRCMMkW2SL7EPDIIlJSDIyiMIIozK2mlK29pQWdc7r98c5fs50vc54n89yPtf5fB732+1zu52e3st18jqfrt6d67w0hCAAAACAR9VyXQAAAABQHppVAAAAuEWzCgAAALdoVgEAAOAWzSoAAADcolkFAACAWwXVrKrqAlU9LMFxQVXbpniPlM8FysPaRVXF2kVVxLr1paCaVa9UtaGqPqyqS0pfw3NdE5CUqnZU1ddVda2qfquqF+e6JuDnqOqU0jX74+sHVZ2d67qA/6VQ+4UauS4AIiJym4hsLSKtRWQ7EZmmqgtDCGNzWhXwM1S1sYi8KCKDROQpEaklIjvmtCgggRBCt7K/VtVXReSV3FQDJFaQ/UJBPllV1f1UdaaqrlLVr1V1jKrW2uKwo1R1vqouU9VRqlqtzPn9VPXfqrpSVaeqaqs0S+ohIjeFENaFEBaIyIMi0i/NayIPOVy7g0VkagjhsRDCxhDCmhDCv9O8JvKQw7VbtrbWInKgiPw1U9dEfnC4bguyXyjIZlVEiqTkSVBjETlARA4VkQu2OKaXiHQSkY4i0lNKF4OqHisil4vIcSLSRESmi8i42E1UdVjpAo++tjx8i4/3SP3TQx7ztnY7i8gKVZ1R+k9Sz6lqywx9rsgv3tZuWaeJyPQQwn/S+PyQnzyu28LrF0IIBfMSkQUiclgkHygiE8v8OojIkWV+fYGITCv9eIqInFXmv1UTkXUi0qrMuW0rWNejIvI3EaknIm1FZJ6IbMz17xcvPy/Ha/dTEVklIvuKSG0RuUNE3sz17xcvPy+va3eLWj4XkTNy/XvFy8/L67ot1H6hIJ+squouqvq8qn6jqt+JyPVS8remshaV+XihiDQr/biViIwu87edFVLyN5vmaZR0kYisF5HPRGSSlPzNa3Ea10Oecrh210vJG/c/QwgbRORqEemiqg3SuCbykMO1+2NdXUWkqZR8zzXwXxyu24LsFwqyWRWRe0TkYxFpF0KoLyWP6XWLY1qU+biliHxV+vEiETkvhNCwzKtOCGHGljdR1cv1v6dN/+v143EhhBUhhJNDCE1DCB2k5P/LOxn8fJE/XK1dEflQSp4O/OjHj7esCfC2dn90uoj8LYQQ+2+Aq3VbqP1CoTar9UTkOxFZq6q7isj5kWOGquovVLWFiFwsIuNL83tF5DJV7SAioqoNVLV37CYhhOtDCHXLe/14nKq2UdVGqlpdVbuJyLkicm3mPl3kEVdrV0TGikgvVd1bVWuKyJUi8kYIYVVGPlvkE29rV1S1joj0FpG/ZOQzRD5ytW4LtV8o1Gb1EhHpKyJrROQB+WlhlTVJRGaJyAciMllKJu4khDBRRG4UkSdK/0lgjoh0i5xfEb8Skdml9dwgIieHEOameU3kJ1drN4TwipQ8aZgsIkuk5Huo+qZzTeQtV2u31LEislpE/pGBayE/eVu3BdkvaAjh548CAAAAcqBQn6wCAACgCqBZBQAAgFs0qwAAAHCLZhUAAABu0awCAADArRoVOVhV+dEBSMWyEEKTXN2cdYsU5XTdirB2kTLWLqqq6NrlySoqw8JcFwCkgHWLqoq1i6oqunZpVgEAAOAWzSoAAADcolkFAACAWzSrAAAAcItmFQAAAG7RrAIAAMAtmlUAAAC4RbMKAAAAtyq0gxWAwtSwYUOTzZ4922TLly832eGHH26ypUuXZqQuAED+48kqAAAA3KJZBQAAgFs0qwAAAHCLZhUAAABuMWAF4L8cdNBBJpswYYLJGjdubLK33nrLZFtvvXVmCgMAFCSerAIAAMAtmlUAAAC4RbMKAAAAt2hWAQAA4BYDVkABq1+/vsmee+45k9WoYd8qDj74YJO9++67JtuwYUOK1QEAwJNVAAAAOEazCgAAALdoVgEAAOAWzSoAAADcYsAKKBC1a9c2WWyYql69eiY7+eSTTfbGG29kpjAAAP4HnqwCAADALZpVAAAAuEWzCgAAALdoVgEAAOAWA1YZ0rZtW5MNGTLEZP379zfZyJEjTfbHP/7RZEVFRSlWB4h0797dZF27djXZl19+abLYIBYAAJWBJ6sAAABwi2YVAAAAbtGsAgAAwC2aVQAAALjFgFUKrrvuOpOdddZZJttuu+1MFkIw2aWXXmqy1157zWQvvvhi0hJR4LbZZhuTXXPNNSZTVZNdddVVJlu7dm1mCkOV1aZNm2hevXr1jN5n1apVJmvYsGHK14sNtdaoYf/o69u3r8kaNWpksptuuslksfdwQESkc+fOJhs8eLDJdtxxR5MdcMABGa1l5syZJlu8eHH02KeeespkTz75ZEbrqQierAIAAMAtmlUAAAC4RbMKAAAAt2hWAQAA4BbNKgAAANzipwGUUb9+fZO98847Jtt5551NFpsu/eSTT0y2ww47mGz16tVJSwQS2XPPPU22yy67mCz20ynGjh2blZqQe9Wq2ecTp556qslOOukkkx100EHRa2611VbpF1bGokWLTNaiRYuM3iOp4uJik+211145qAS5FFt/AwcONFnv3r0TnZtU7KcGxLbD3n///RPdN/aTCcr7iQOxz6V58+Ymu+2226LnZxpPVgEAAOAWzSoAAADcolkFAACAWzSrAAAAcKtgB6xig04zZswwWatWrRJdb8qUKSY7//zzTRYbznrllVdMNmbMGJN17drVZN98802i+pC/ateubbLLLrss0blr1qzJdDlw4vjjjzdZbCvdPfbYw2Sxwbu33347ep/Y+2ZMbBArNvDx8ccfm+zbb781Wey976OPPjLZZ599ZrLNmzebLLaNauz38P333zcZ8kdsTca2GU06OJV0i9MhQ4aYLDZsGJN0G9Q+ffqYbPz48YnOFYkPXTFgBQAAgIJHswoAAAC3aFYBAADgFs0qAAAA3CqIAavtt9/eZHPmzDHZL37xC5P98MMPJhs9erTJrr76apOtW7fOZCtXriy3zrJig1gnnnhiolpQWK644gqTde/e3WRLly412a9//eus1ITcO+ecc0wWG6bauHGjyS688EKT5dPOZl26dDHZIYccYrJNmzaZbOrUqVmpCdlT3jBUbBeq2K5RMRMmTDBZOkNSmRYbprr55pvTuubtt9+e1vnp4MkqAAAA3KJZBQAAgFs0qwAAAHCLZhUAAABu5d2AVWxIau7cuYmOmz9/vslOO+00kyXdsSUm9g3YQFKdOnUy2dChQxOdO3LkSJPNmzcv7Zrg0z333GOytWvXmmzy5Mkmy6dhqpjY+3Dsz4QFCxaY7LXXXstGSciQ2A5UsV2kyhM7NjZYlHTXqEyLDYvdcsstJovtNlURsd+HXH3OIjxZBQAAgGM0qwAAAHCLZhUAAABu0awCAADArbwbsGrTpo3Jtt12W5PFdpVo3769yYqKijJTWKn9998/o9dDYfnlL39pslq1apnss88+M1kudx9B5Zs0aVKiDOWL7WAFPwYNGpQoK8+tt95qstj7pPddqMrboSuJ2O+BiL8/L3iyCgAAALdoVgEAAOAWzSoAAADcolkFAACAW3k3YNW1a1eTbd682WTHH3+8yTp06GCyZs2amaxhw4Ym69atm8kOP/xwk22//fYmi3n55ZdNNnXq1ETnIj80btzYZGPGjDFZCMFkL774YlZq+jknnXSSye644w6Tff311yaL7TwjIrJu3br0C0PBiu1M1bFjx0TnxgZckBux94fyhoO2dMABB0Tzt956K62aUpWrwamqvIMmT1YBAADgFs0qAAAA3KJZBQAAgFs0qwAAAHAr7wasDjroIJPVqGE/zVdeecVkNWvWNNlWW22VmcL+h6uvvtpkI0eONNnGjRuzXgv8OOKII0wWW4/Tp0832SWXXJKVmsraY489THbjjTeaLLaDXKNGjUxWu3bt6H0YsEI6RowYYbKWLVuaLLa715w5c7JSEyou6TBV7LhsDFLFhp9ig1yZHpyKmTlzpsmq8jBVDE9WAQAA4BbNKgAAANyiWQUAAIBbNKsAAABwK+8GrPr372+yZcuWmWyHHXYw2ZtvvpnoHs8++6zJYrtavfHGGyYrKioy2fjx403GMFVhqVbN/r2xe/fuic4966yzTLZp06a0ayorNpAybdo0k8V23YoZO3asyb777ruKFwaUERtAPOOMMxKd+9xzz5msuLg43ZKQIbfffrvJYgNNgwcPTnTc/8q3NGHCBJP17t3bZIsWLTJZbLgrNgAb6wOS1lIIO63xZBUAAABu0awCAADALZpVAAAAuEWzCgAAALfybsBqyZIlJjv33HOzft8777wz0XHPP/+8yT7++ONMl4Mq5rjjjjNZ7Jvm3333XZMtXLgwo7U0b97cZC+99JLJkg5TxcR2kNu8eXPK1wNERP7yl7+YbOuttzbZRx99ZLKJEydmoyRkyJNPPpnouIEDB5os6SCVSHxIKia2U1ZsCCx2vc6dOyc6LrbTVb7tTJUUT1YBAADgFs0qAAAA3KJZBQAAgFs0qwAAAHAr7wasKkOjRo1Mduyxx5ostvvJuHHjslESqrhmzZqZTFVNFtuNLdO7VS1evNhkIYSM3iPpbnFAefbee2+TJd317fXXXzfZqlWr0qwIlS02dJV0EKuyxIakYjXGjosNhiUdAMs3PFkFAACAWzSrAAAAcItmFQAAAG7RrAIAAMAtBqxScOCBB5ostuvPU089ZTJv3/wNH2IDTEmzdBx55JFZv++wYcNMtmDBgpSvB4iI1K1b12Sx3apig66xnQSBbIjtqBUbpooNTr311lvZKKlK4skqAAAA3KJZBQAAgFs0qwAAAHCLZhUAAABuMWCVgttuuy3Rcbfffnt2C0HeGD9+vMl+//vfm6xt27Yma9OmjcnmzZuX6L4DBgxIdFxSsftOnDgxo/cARESOOOKIRMe99957JpsyZUqmywGkT58+Jhs8eHDK5+InPFkFAACAWzSrAAAAcItmFQAAAG7RrAIAAMAtBqx+RmygpVGjRiZbv369yVavXp2VmpB/lixZYrKLLrrIZM8884zJYgMkH330UaL7tm/fPtFxMatWrTLZ4YcfbrKFCxemfA9ARKRr164mu+CCCxKde+2112a6HCDq5ptvTnRcbOiK3ar+N56sAgAAwC2aVQAAALhFswoAAAC3aFYBAADgFgNWP+P+++83Wd26dU12xRVXmGzu3LlZqQmFYerUqSY77rjjTHbSSSeZbM899zRZvXr1TNagQYMUqxMZNWqUyRimQjYMHz7cZA0bNkx07rRp0zJbDCAiTz75pMlatGhhskWLFpks6S6Y+AlPVgEAAOAWzSoAAADcolkFAACAWzSrAAAAcItmFQAAAG7x0wDKiG2tuv/++5ts48aNJpswYUJWagLKmjJlSqIMqKpOPvlkkx144IGJzn3kkUdM9sMPP6RdEwpb586dE2Uxl1xySabLKUg8WQUAAIBbNKsAAABwi2YVAAAAbtGsAgAAwC0GrMp4/vnnTVanTh2TxbZZ+/zzz7NSEwAUkgEDBpisRg37R9X7779vsvPPP99kmzdvzkxhKAixLVNvvfXWRMfFBq1j/QIqjierAAAAcItmFQAAAG7RrAIAAMAtmlUAAAC4VbADVm3atEmUFRUVmWzUqFFZqQkACsmgQYNMtu+++yY6d+rUqSbbsGFD2jWhsJ1wwgkmO+CAA0y2aNEik/Xp0ycrNYEnqwAAAHCMZhUAAABu0awCAADALZpVAAAAuFWwA1ZNmjQxmaqa7IILLjDZrFmzslITAOSrunXrmmzw4MGJzp09e7bJ7rrrrrRrArbUu3fvRMcxTFW5eLIKAAAAt2hWAQAA4BbNKgAAANyiWQUAAIBbBTtg9dZbb5msRo2C/e0AgKwqLi422Zo1axKde9VVV5nsq6++SrsmFLYWLVqYbPHixSabMGGCyWI9BLKHJ6sAAABwi2YVAAAAbtGsAgAAwC2aVQAAALilIYTkB6smPxj4yawQQqdc3Zx1ixTldN2KsHaRMtYuqqro2uXJKgAAANyiWQUAAIBbNKsAAABwi2YVAAAAblV0y6ZlIrIwG4Ugr7XK8f1Zt0hFrtetCGsXqWHtoqqKrt0K/TQAAAAAoDLxbQAAAABwi2YVAAAAbtGsAgAAwC2aVQAAALhFswoAAAC3aFYBAADgFs0qAAAA3KJZBQAAgFs0qwAAAHCLZhUAAABuFVSzqqoLVPWwBMcFVW2b4j1SPhcoD2sXVRVrF1UR69aXgmpWvVPVWqr6saouznUtQBKqOlBV56vqd6r6larepqo1cl0X8HO0xI2qurz0dZOqaq7rAn6OqnZU1ddVda2qfquqF+e6pmyjWfVlqIgsyXURQAU8JyIdQwj1RWQPEfmliFyU25KARM4VkWOlZM3uJSLdReS8XBYE/BxVbSwiL4rIfSLSSETaishLOS2qEhRks6qq+6nqTFVdpapfq+oYVa21xWFHlT4xWqaqo1S1Wpnz+6nqv1V1papOVdVWGahpJxE5RURuSPdayF/e1m4IYV4IYdWPlxeRYil58wT+i7e1KyKni8gtIYTFIYQvReQWETkjzWsizzhct4NFZGoI4bEQwsYQwpoQwr/TvKZ7BdmsikiRiAwSkcYicoCIHCoiF2xxTC8R6SQiHUWkp4j0ExFR1WNF5HIROU5EmojIdBEZF7uJqg4rXeDR1xaH31l63fXpf3rIY+7Wrqr2VdXvRGSZlDylui8Tnyjyjre120FE/lXm1/8qzYCyvK3bziKyQlVnqOoSVX1OVVtm6HP1K4RQMC8RWSAih0XygSIyscyvg4gcWebXF4jItNKPp4jIWWX+WzURWScircqc27aCdfUSkRdLP/6NiCzO9e8VL18vr2t3i1raicg1ItI0179fvPy8vK5dKWlCdi3z63al19Fc/57xyv3L8br9VERWici+IlJbRO4QkTdz/fuV7VdBPllV1V1U9XlV/ab0idD1UvK3prIWlfl4oYg0K/24lYiMLvO3nRVS8s+fzVOsZRsRuUlEBqRyPgqLp7W7pRDCZyIyV0TuzsT1kF8crt21IlK/zK/ri8jaUNoRACIu1+16KWmW/xlC2CAiV4tIF1VtkMY13SvIZlVE7hGRj0WkXSgZDLlcShZQWS3KfNxSRL4q/XiRiJwXQmhY5lUnhDBjy5uo6uVaMq0XfZUe1k5EWovIdFX9RkT+JiI7lH5htM7UJ4y84WntxtQQkTYpf3bIZ97W7lwp+baVH/2yNAPK8rZuP5SSJ7I/+vHjvP5JFoXarNYTke9EZK2q7ioi50eOGaqqv1DVFiJysYiML83vFZHLVLWDiIiqNlDV3rGbhBCuDyHULe9VetgcKVnoe5e+zhaRb0s/XhS5LAqbp7Urqnq2qm5X+vHuInKZiEzL1CeLvOJq7YrIIyIyWFWbq2ozERkiIn/JyGeKfOJt3Y4VkV6qureq1hSRK0XkjfDToGteKtRm9RIR6Ssia0TkAflpYZU1SURmicgHIjJZRB4UEQkhTBSRG0XkidJ/EpgjIt1SLSSEsDmE8M2PLyn5Z4Li0l8XpXpd5C03a7fUr0Vktqp+LyIvlL4uT/OayE/e1u59UvKj12aXXm+yMBwIy9W6DSG8IiXvsZOl5Eddti2tL68p354DAAAArwr1ySoAAACqAJpVAAAAuEWzCgAAALdoVgEAAOAWzSoAAADcqlGRg1WVHx2AVCwLITTJ1c1Zt0hRTtetCGsXKWPtoqqKrl2erKIyLMx1AUAKWLeoqli7qKqia5dmFQAAAG7RrAIAAMAtmlUAAAC4RbMKAAAAt2hWAQAA4BbNKgAAANyiWQUAAIBbNKsAAABwq0I7WAEAAMCPX/7ylyabMWOGyf785z+b7OKLL85KTZnGk1UAAAC4RbMKAAAAt2hWAQAA4BbNKgAAANxiwMqpu+66y2T9+/c32fXXX2+yK6+8Mis1AQAAXwYOHGiyOnXqmKxatar7fLLqVg4AAIC8R7MKAAAAt2hWAQAA4BbNKgAAANzSEELyg1WTH4y0FBUVmSz2/+rrr782WZcuXaLXXLRoUfqFpWZWCKFTrm7OukWKcrpuRVi7SBlrN0/ttddeJnvvvfdMFhum6t27t8mefvrpzBSWOdG1y5NVAAAAuEWzCgAAALdoVgEAAOAWzSoAAADcYgcrB1q3bp3yuTvssIPJGjRoED02hwNW+Bl77723yXr06JHy9dq2bWuyU089NeXrqarJkg5nrlmzxmQjRoww2S233FLxwgAgT7Vs2dJkEyZMMFlsmOr999832fTp0zNTWA7wZBUAAABu0awCAADALZpVAAAAuEWzCgAAALcYsHLghBNOSPnc2A5Wq1evTqccZFFskEpEZNq0aSZr2LBhRu9dkd3qMnlu3bp1TXbTTTeZbOPGjdHzx4wZk/K9kR9q1qxpsvr16yc697vvvjPZpk2b0q4JyKTYYPTQoUNN1q5du0TXGz58uMmWLFlS4bq84MkqAAAA3KJZBQAAgFs0qwAAAHCLZhUAAABuMWBVxc2ZM8dk7FTl1zbbbBPNYwMkmbZ48WKTPf300ylf7+ijjzZZbOespGK7sKDwxL4WrrvuOpPFhk9iYrul/elPf6p4Yf9DeV/X1atXN1ls4Au45pprTHbhhRcmOvepp54y2d///ve0a/KEPx0AAADgFs0qAAAA3KJZBQAAgFs0qwAAAHCLZhUAAABu8dMAnIpNRhcXF5vs1ltvrYxykCFvvvlmND/uuONMFpvwfO6550wWmy6+5557TPbFF18kOjep2ET1n//8Z5Ols50w8luHDh1MFpv879mzZ6LrzZw502Tlfc1l0hlnnBHNO3bsaLJzzjnHZLH3duSH1q1bm2zChAkma9++faLrzZ8/32Snn366yTZs2JDoelUFT1YBAADgFs0qAAAA3KJZBQAAgFs0qwAAAHCLASsHGjdubLLYN9yHECqjHOTAyy+/bLJddtnFZCtWrDDZ5s2bs1JTWdtuu63Jxo0bZ7IuXbokul7s83jwwQcrXhiqjNgw1eTJk03WqlUrk8WGRW644QaT3XfffSb79ttvk5aYcf369TPZK6+8YrLHHnusMspBltWoYVuqadOmmWynnXZKdL2lS5eaLDawun79+kTXq8p4sgoAAAC3aFYBAADgFs0qAAAA3KJZBQAAgFtVYsDq7LPPNtm7774bPfaDDz7IcjWZd9ppp+W6BDi0ZMmSRMe1aNHCZAcffHDK9z311FNNtvvuu5usWbNmKd/jlltuMdn333+f8vXgS//+/U02bNgwk8WGqWI7rb3xxhsmGzFiRIrVZV5sYFBEpKioyGSHHHKIyR5//HGTMVDrW/PmzU02fPhwkyUdpnr99ddNdskll5isKvY4mcCTVQAAALhFswoAAAC3aFYBAADgFs0qAAAA3HI3YBUb2vjjH/9ostq1a0fPb9q0acZryqSdd97ZZOV9LigcrVu3NtlZZ51lstjuJXXr1jVZOsNP6Xj77bdNdscdd5jsxRdfrIxykEGx3XlE4oNT55xzjslatmxpsnfeecdkvXr1MtlXX32VpMScie3mJiJy1113mSy2q9XVV19tstigGfyIDX7H3rOTuvfee00WGyTfaqutTLbnnnsmukdsGHfVqlXRY//xj38kumZl4ckqAAAA3KJZBQAAgFs0qwAAAHCLZhUAAABuuRuw2mOPPUy24447Jj7/oYceMlnsG9pz5aKLLjJZ/fr1E527dOlSk/3nP/9Juybk3qGHHmqyyy+/PAeVpGfGjBkmW7lypcnK+6Z++FC9enWTxdaoiMg111yT6JqxYapjjz3WZF9//XWi6+WT2EDalVdemYNKEHPmmWeaLPZneVKnn366yWIDdc8++6zJYsO4sb4pqeLi4mjes2dPk02ePDnl+6SLJ6sAAABwi2YVAAAAbtGsAgAAwC2aVQAAALjlbsDqwAMPNJmqJj7/4IMPzmQ5Gfeb3/zGZLHPr1o1+/eI8ePHm+zzzz/PSF3IrUmTJpnskEMOMVmPHj1M9v3335sstita0kG+dAwaNMhkv//97002fPhwk40cOTIbJSEF5557rsnuvvvuxOcvWbLEZIU4TLVw4UKT/eIXv8hBJUhHbGfMpP8fY4PR++23n8liw+GxQcdMi/UaIiKNGjXK+r0rgierAAAAcItmFQAAAG7RrAIAAMAtmlUAAAC45W7Aat999zVZCCHx+SNGjMhkORkX+1xiWWyHn9tvvz0LFcGDZcuWmezkk0822d57722yDRs2mKxWrVomq1u3bqJaTj31VJO1a9fOZLFdU5o0aWKymjVrmuzqq682WXlf5zfeeGM0R/bstNNOiY/94YcfTHbVVVeZLN+HqWIeeeQRk8W+hmO7EsGPY445JuVzY++JF154YTrlZFRRUVE0nz9/fiVX8r/xZBUAAABu0awCAADALZpVAAAAuEWzCgAAALdyOmC18847m6xjx45pXfPhhx9O6/wtxb7xfcGCBYmO+9WvfmWypIML69evT3RfFJYPPvgg6/eYMWNGouM6dOhgsoEDB5qsX79+JqtRw771XHPNNdH7xNZ9bDc3pGb33Xc32SmnnGKyTZs2Rc8fMGCAye6///70C8sDsbUf07lz5yxXgqR69eplstiOU5UhtjvhqFGjTBbbhWrIkCEm22abbUx2zz33RO/9xhtvJCmx0vBkFQAAAG7RrAIAAMAtmlUAAAC4RbMKAAAAt3I6YFWvXj2TbbvttmldMzaAUpEdsLbUuHFjky1fvtxkjRo1Mlns80u6ixDg3dy5c0128cUXm2zHHXc02W9/+1uTVa9ePXqfK664wmQMWGXOP/7xD5Ntt912JluxYkX0/HwepmrZsqXJzjnnHJN17949ev5uu+2W6D7Dhw+vUF3IjOOPP95kd999t8lUNeu1xHY7u+2220xWu3Ztk8Vqjg1T/fOf/zTZY489lrTEnOLJKgAAANyiWQUAAIBbNKsAAABwi2YVAAAAbuV0wGrp0qUmW7hwoclatWqV+Jp77rmnydIZsIpp1qxZ1u8Bv7bffvtEWWwAqaioKCs1ebFu3TqT3XTTTSaLDViVp127dmnVhP/tmWeeMdm5555b+YVkwKGHHmqyLl26mOx3v/tdous1bNjQZLH3/88//zx6/lNPPWWy2A5xjz/+eKJ6kJrYLm0iIn/5y19MFhuCTvrne3Fxscmuu+46k/3tb38z2cqVK00W29XvuOOOM1lsmCo2EHnmmWea7KOPPjKZRzxZBQAAgFs0qwAAAHCLZhUAAABu0awCAADArZwOWH311VcmGzZsmMmuvPJKk8V2lhKJ7zQR++bo2CDX008/Hb3mll599VWTvfvuuybr1auXyWLfcB/z+uuvJzoOlS/2De533nmnydq2bWuyBQsWZKMk12LfwL9q1SqTxYZZROJf07Gv/2XLllW4NsTfX2NDSeUNqdxxxx0mi70fJtW7d2+T7brrronOje2WFtvxJyb2tTlt2jSTjR492mSLFi2KXnPJkiWJ7o3M6dSpk8li788i8cGkdIalY+dOnz7dZP379zfZYYcdZrI2bdokum/sve+oo44yWVUZporhySoAAADcolkFAACAWzSrAAAAcItmFQAAAG7ldMAqZsKECYmyquC8884zWdJv3k5nQAHZtWHDBpPF/r9edNFFJhs8eHBWavKiZs2aJqtTp47JXnjhBZP17ds3es3YMFbTpk1NxoBVamJDQLFhj5tvvjl6/oABAzJe05ZiNW7evNlkL7/8ssm++eYbk33yyScmi+1mxJryrU+fPia77777TNagQYPKKEeqV69uspdeeimj94jtTNWjRw+T5VsPwZNVAAAAuEWzCgAAALdoVgEAAOAWzSoAAADccjdgBXgX+2b2atXs3/sOP/xwk+2www7Ra3799dfpF1bJYsNUf/jDH0w2YsSIRNcrb/jw6quvNtmcOXMSXROp+fbbb012+umnR48dOnSoyWK7kbVs2dJkH374YaJ6Nm3aZLLYelm7dq3Jfvjhh0T3QNUzcOBAk1VkmOq1114zWf369U22zz77VKiuTJk7d67JYl+H7733XmWUk1M8WQUAAIBbNKsAAABwi2YVAAAAbtGsAgAAwC0GrLLoxhtvNFmnTp1MFhtGgF9vvvmmyXr27Gmy3Xff3WTl7WYyffp0k916660mi+3mVBlOPvlkk+2///4mO/HEExNdLzYc89BDD0WPveeeexJdE9lVXFwczWM7RMWyjz/+OOM1AVtatGiRyWJDsSIiH330kcnOOecck8X+LK9bt24K1ZUvNjg1ceJEk8WGCAsBT1YBAADgFs0qAAAA3KJZBQAAgFs0qwAAAHCLZhUAAABuaXlbHEYPVk1+MKIGDRpksu7du5vsmGOOMdn333+flZoqwawQgv0xCJWkMtbt6tWrTZbpadGqKunk/7nnnlsZ5VRETtetCO+5SBlrF1VVdO3yZBUAAABu0awCAADALZpVAAAAuEWzCgAAALcYsEJlyPsBq/32289kBx98sMmGDRsWPb8qbrkbe+/YtGmTyf7617+azOEwVQxDKqiqWLuoqhiwAgAAQNVCswoAAAC3aFYBAADgFs0qAAAA3GLACpUh7wes0nXxxRebLOkOWLvuuqvJtt9+e5MdeuihJovtJLVgwYJE912+fLnJ7r333kTnVhEMqaCqYu2iqmLACgAAAFULzSoAAADcolkFAACAWzSrAAAAcKtGrgsAIDJ69OhclwAAgEs8WQUAAIBbNKsAAABwi2YVAAAAbtGsAgAAwC2aVQAAALhFswoAAAC3aFYBAADgFs0qAAAA3KJZBQAAgFs0qwAAAHCLZhUAAABu0awCAADALZpVAAAAuEWzCgAAALdqVPD4ZSKyMBuFIK+1yvH9WbdIRa7XrQhrF6lh7aKqiq5dDSFUdiEAAABAInwbAAAAANyiWQUAAIBbNKsAAABwi2YVAAAAbtGsAgAAwC2aVQAAALhFswoAAAC3aFYBAADgFs0qAAAA3KJZBQAAgFs0qwAAAHCroJpVVV2gqoclOC6oatsU75HyuUB5WLuoqli7qIpYt74UVLPqnarWUtWPVXVxrmsBklLVjqr6uqquVdVvVfXiXNcEJMX7LqoSVf0/Vf2Hqq5W1QW5rqey0Kz6MlREluS6CCApVW0sIi+KyH0i0khE2orISzktCqgY3ndRlXwvIg9JybotGAXZrKrqfqo6U1VXqerXqjpGVWttcdhRqjpfVZep6ihVrVbm/H6q+m9VXamqU1W1VQZq2klEThGRG9K9FvKXw7U7WESmhhAeCyFsDCGsCSH8O81rIg85XLu87+JneVu3IYR3Qgh/FZH56VynqinIZlVEikRkkIg0FpEDRORQEblgi2N6iUgnEekoIj1FpJ+IiKoeKyKXi8hxItJERKaLyLjYTVR1WOkCj762OPzO0uuuT//TQx7ztnY7i8gKVZ2hqktU9TlVbZmhzxX5xdvaFeF9Fz/P47otPCGEgnmJyAIROSySDxSRiWV+HUTkyDK/vkBEppV+PEVEzirz36qJyDoRaVXm3LYVrKuXiLxY+vFvRGRxrn+vePl6OV67n4rIKhHZV0Rqi8gdIvJmrn+/ePl5OV67vO/yKvfldd2WudZhIrIg179PlfUqyCerqrqLqj6vqt+o6ncicr2U/K2prEVlPl4oIs1KP24lIqPL/G1nhYioiDRPsZZtROQmERmQyvkoLJ7Wbqn1UvLG/c8QwgYRuVpEuqhqgzSuiTzkae3yvoukPK3bQlaQzaqI3CMiH4tIuxBCfSl5TK9bHNOizMctReSr0o8Xich5IYSGZV51QggztryJql6uJRPS0VfpYe1EpLWITFfVb0TkbyKyQ+kXRutMfcLIG57WrojIh1LydOBHP368ZU2Ap7XL+y6S8rRuC1ahNqv1ROQ7EVmrqruKyPmRY4aq6i9UtYWIXCwi40vze0XkMlXtICKiqg1UtXfsJiGE60MIdct7lR42R0oW+t6lr7NF5NvSjxdFLovC5mntioiMFZFeqrq3qtYUkStF5I0QwqqMfLbIJ57WLu+7SMrTuhVVraaqtUWkZskvtbbaga+8U6jN6iUi0ldE1ojIA/LTwiprkojMEpEPRGSyiDwoIhJCmCgiN4rIE6X/JDBHRLqlWkgIYXMI4ZsfX1LyzwTFpb8uSvW6yFtu1m7pNV+RkicNk6Xkx/+0La0P2JKbtcv7LirAzbotdZCUfPvVC1LyFHe9FMCPC9QQws8fBQAAAORAoT5ZBQAAQBVAswoAAAC3aFYBAADgFs0qAAAA3KpRkYNVlWkspGJZCKFJrm7OukWKcrpuRVi7SBlrF1VVdO3yZBWVYWGuCwBSwLpFVcXaRVUVXbs0qwAAAHCLZhUAAABu0awCAADALZpVAAAAuEWzCgAAALdoVgEAAOAWzSoAAADcolkFAACAWzSrAAAAcItmFQAAAG7RrAIAAMAtmlUAAAC4RbMKAAAAt2rkuoBCU79+fZM99thjJrv77rtNNmXKlKzUBFS2bbfd1mTnn39+9NgRI0aYbOnSpSZr2rRp+oUBANzhySoAAADcolkFAACAWzSrAAAAcItmFQAAAG4xYJVFqmqy0047zWRHH320yT799FOTMWCFqmi77bYz2aRJk0y23377Rc8PIZjslltuSb8wAHCsQ4cOJps9e7bJmjRpYrLly5dnpaZc4ckqAAAA3KJZBQAAgFs0qwAAAHCLZhUAAABuMWCVRS1btjTZHXfcYbKioiKTPfvss1mpCciU2rVrm2zYsGEmGzBggMkaNGhgshdeeCF6nyeeeMJk48aNS1IikJb+/fub7J577jHZ1ltvHT1//fr1Ga8JhSM2fB0bON1zzz1N9uqrr2ajpJzhySoAAADcolkFAACAWzSrAAAAcItmFQAAAG4xYJUhTZs2Ndno0aMTnXvRRReZ7LXXXku7JiBTdtllF5Ndd911JuvVq1ei640dO9ZkI0eOjB47b968RNcE0tG5c2eTxd7DYwMuQLoaNWpksh49eiQ6d9999zUZA1YAAABAJaFZBQAAgFs0qwAAAHCLZhUAAABuMWCVIbFvzj/mmGNMtnLlSpM999xzWakJSMUee+xhsjFjxpisa9euJovt2NO3b1+TvfTSSybbuHFj0hLhXGx3s//7v/8z2YsvvmiyyhhgatWqlcmuueYak9WsWdNkmzdvzkpNKGy77rqrybp06ZLo3MceeyzT5bjDk1UAAAC4RbMKAAAAt2hWAQAA4BbNKgAAANyiWQUAAIBb/DSAFNStW9dkl19+ucmKi4tNNmTIEJMtXrw4M4UBFbT//vubbPLkySaLbQW4atUqk8W+DvhpF/mtQYMGJrv99ttNdvrpp5vsqKOOMlnsJwRk2oUXXmiyQw891GRr1qwx2RFHHGGy2E/BACpLIfwkFZ6sAgAAwC2aVQAAALhFswoAAAC3aFYBAADgFgNWKTjhhBNM1qlTJ5Nt2rTJZH/5y1+yURLws2JrdNKkSSZr2LChyWbNmmWyYcOGmezll19OrThUWR07djRZbJgqZunSpZkuxzjooINMdtFFFyU698033zTZW2+9lXZNwJZi76eqmoNKfOLJKgAAANyiWQUAAIBbNKsAAABwi2YVAAAAbjFg9TOOOeYYk1111VUmi+0gcfPNN2elJuDndOjQwWQ33XSTyRo3bmyylStXmqx///4me/fdd1OsDlXRzjvvHM0ffvjhROfH3iPnzp2bVk1JxAZia9Wqlejc2NcMkK7jjz/eZEcffbTJQgiVUU6VwJNVAAAAuEWzCgAAALdoVgEAAOAWzSoAAADcYsCqjNatW5vs/PPPT3Tce++9Z7Irr7wyE2X9f7/85S9Nts8++5jsiSeeiJ6/YcOGjNYDH3bZZReTDR8+3GSxnXxWrVplsthOVwsXLkypNuSP3XffPZrvuOOOJosNU5144okmy/R7Ur169Uz229/+NtG58+fPN1nsfR1I15577pnyuW+//bbJ1qxZk045VQJPVgEAAOAWzSoAAADcolkFAACAWzSrAAAAcIsBqzIGDhxosiOOOMJkK1asMFm/fv2yUdLP3mPAgAEm22233aLnX3rppRmvCbl3//33m6xr164mi+1MxTAVkurbt2/iY9euXWuyZ599NpPlRF1yySUmiw0gxsybN89k3333Xdo1obB169bNZMOGDUt0blFRkcmuueYak/3www8VL6yK4ckqAAAA3KJZBQAAgFs0qwAAAHCLZhUAAABuFeyAVfPmzU125plnJjr3xhtvNNmHH36Ydk1lnXXWWSbr379/onPL22mmWjX7d5MQQqIMPvzxj3802cEHH2yy2M5UV111lcmKi4tNNmLECJPNnTvXZM8//7zJvv/+e5Oh6qlTp47Jjj766MTn/+c//0l0zfXr11essDJq165tstguWTGfffaZyWLrHkhX7M/jmjVrJjo3NhQ7ZcqUtGuqiniyCgAAALdoVgEAAOAWzSoAAADcolkFAACAWwU7YPX444+brF69eia77777TDZq1CiTNW3aNFEWGzyI7Qzzhz/8wWRJvyn7oIMOiuavvvqqyT744AOTXXHFFSZbs2ZNonsjc0444QSTxdZFbEhqzJgxJvvnP/+ZKGvcuLHJVNVko0ePNtngwYNNhqontnPO8uXLo8fG3jdjO6PNnDnTZP/+979Ndt5555kstpPU8OHDTZZ0t6oXXnjBZG+++Waic4GKOO6440wWez+NZWPHjs1KTVURT1YBAADgFs0qAAAA3KJZBQAAgFs0qwAAAHAr7wasYrs0devWzWRdunRJdL3YTlexQaX27dubrEGDBiZbunSpyVq0aJGolqRiAw8iIl27djVZbBDikUceMdm7776bfmEo1zbbbGOyP/3pTybbeuutTfboo4+a7OmnnzbZSy+9ZLLYWlm7dm2i45o1a2Yy5IcffvjBZLFBUBGR8ePHmyz2nrbXXnslypLuQjV//vxEx2X6XKA8W221lcliQ3+xXSI3bdpksieffDIzheUBnqwCAADALZpVAAAAuEWzCgAAALdoVgEAAOBW3g1Yxb5h/7nnnkv5et27d0/53Lfffttk+++/f8rXiw09PPzwwyaL7ZwlIjJt2jSTxXZy+fzzz1OoDumIDVjttttuic6dOHGiyX7zm9+YLDYkdcEFF5isZ8+eJuvRo0eiWpC/3nrrrWh+8MEHm+yMM84w2e9+97tE99l2221NFttVbeedd050vZiRI0eabOjQoSlfr7ydszZs2JDyNVH1nHvuuSaLreeYd955x2Tvvfde2jXlC56sAgAAwC2aVQAAALhFswoAAAC3aFYBAADgVpUdsIrtFCEi8te//jXla65YscJkN998s8kef/zxRNf79ttvTXb99debbNCgQSaLDVPFhgKGDx+eqBb4lnT4ZNy4cSZ75plnTPbKK6+Y7I033jDZ2LFjTTZ48OBEtTzxxBOJjkN+W7Bggcli70tJ36tiuwbG1vivfvWrRNeLDTnFhk1ju/nNnj3bZPPmzTNZcXFxolqQ3w455JCUz03aVxQqnqwCAADALZpVAAAAuEWzCgAAALdoVgEAAOBWlR2w6tu3bzTv0KFDovOfeuopk1144YUmW7p0acUKK6NWrVomiw1TxVx22WUmu+2221KuBT507Ngxmt9www0mU9VEx3Xt2tVksV2FjjzySJONGjXKZLvvvrvJ7rvvPpPFhl6AdH355ZcmW716dcrXO/PMM002fvx4k7Vu3dpkseExQESkdu3aJmvZsmXK12O3s/+NJ6sAAABwi2YVAAAAbtGsAgAAwC2aVQAAALhVJQaszjvvPJONHj06emxsKOWRRx4xWWyYau3atSlUV74LLrgg0XGxgYK77747o7XAhzZt2kTz2I5sIYRE1zzxxBNN9vnnn5vsmGOOMdn555+f6L6TJk1KVAuQrp49e5rswAMPTHRubFe1p59+OtG5DFOhImLv5fvss0+ic6dPn26y2G6C+AlPVgEAAOAWzSoAAADcolkFAACAWzSrAAAAcMvdgFXnzp1N1q9fP5PFdocSEVm1apXJhgwZYrJMD1PFdOnSJdFxp512msk2btyY6XLgQHn/X2NDTbFhwdhAVJ06dUwW++b/2LkxseHDqVOnJjoXSFds976aNWsmOnf27Nkm27x5c9o1AVu69NJLTZZ0KPbTTz/NdDl5jyerAAAAcItmFQAAAG7RrAIAAMAtmlUAAAC45W7A6qyzzjLZvvvua7Lvv/8+ev4pp5xismXLlqVfWApGjBhhsjPOOMNkGzZsqIRq4MGzzz4bzW+++WaTDR061GSxIanYIFbsG/03bdpksgkTJphs3Lhx0RqBTGvUqJHJWrRokfL1vvjii3TKARLr1KlTyucuWrQog5UUBp6sAgAAwC2aVQAAALhFswoAAAC3aFYBAADgFs0qAAAA3HL30wBi25AVFRWZ7KSTToqeP3ny5IzXlKo5c+bkugRUEVdeeaXJlixZYrJjjjkm0fVia+/Pf/6zyf71r38luh6QDb/+9a9NtsMOOyQ695NPPjHZpEmT0q4JyKQffvjBZOX9VBiUjyerAAAAcItmFQAAAG7RrAIAAMAtmlUAAAC45W7AatSoUYkyIJ9s3rzZZLfddluiDKiqunXrlvK5d999t8nWrl2bTjlAYrFh7vbt25vszjvvNNkHH3yQjZLyGk9WAQAA4BbNKgAAANyiWQUAAIBbNKsAAABwy92AFQCgMMyfPz/RcVOnTjXZ+++/n+lygMSGDh2aKENm8GQVAAAAbtGsAgAAwC2aVQAAALhFswoAAAC3NISQ/GDV5AcDP5kVQuiUq5uzbpGinK5bEdYuUsbaRVUVXbs8WQUAAIBbNKsAAABwi2YVAAAAbtGsAgAAwC2aVQAAALhFswoAAAC3aFYBAADgFs0qAAAA3KJZBQAAgFs0qwAAAHCLZhUAAABu0awCAADALZpVAAAAuEWzCgAAALdqVPD4ZSKyMBuFIK+1yvH9WbdIRa7XrQhrF6lh7aKqiq5dDSFUdiEAAABAInwbAAAAANyiWQUAAIBbNKsAAABwi2YVAAAAbtGsAgAAwC2aVQAAALhFswoAAAC3aFYBAADgFs0qAAAA3KJZBQAAgFs0qwAAAHCroJpVVV2gqoclOC6oatsU75HyuUB5WLuoqli7qIpYt74UVLPqlao2VNWHVXVJ6Wt4rmsCklDVoao6R1XXqOp/VHVormsCKkJVa6nqx6q6ONe1AEkV2rqtkesCICIit4nI1iLSWkS2E5FpqrowhDA2p1UBP09F5DQR+VBE2ojIS6q6KITwRG7LAhIbKiJLRKRurgsBKqCg1m1BPllV1f1UdaaqrlLVr1V1jKrW2uKwo1R1vqouU9VRqlqtzPn9VPXfqrpSVaeqaqs0S+ohIjeFENaFEBaIyIMi0i/NayIPeVu7IYSbQgjvhRA2hxA+EZFJIvLrdK6J/ORt7ZZecycROUVEbkj3WshPrFsfCrJZFZEiERkkIo1F5AAROVRELtjimF4i0klEOopITyltHlX1WBG5XESOE5EmIjJdRMbFbqKqw0oXePS15eFbfLxH6p8e8pjHtfvjOSoiB4rI3LQ+Q+Qrj2v3ztLrrk//00OeYt16EEIomJeILBCRwyL5QBGZWObXQUSOLPPrC0RkWunHU0TkrDL/rZqIrBORVmXObVvBuh4Vkb+JSD0RaSsi80RkY65/v3j5eXldu1vUcrWI/EtEtsr17xcvPy+va1dKGowXSz/+jYgszvXvFS8/L9atr1dBPllV1V1U9XlV/UZVvxOR66Xkb01lLSrz8UIRaVb6cSsRGV3mbzsrpORJaPM0SrpISv6G9JmU/DPqOBEpiG+aRsU4XLs/1vV7Kfne1aNDCBvTvR7yj6e1q6rbiMhNIjIglfNROFi3PhRksyoi94jIxyLSLoRQX0oep+sWx7Qo83FLEfmq9ONFInJeCKFhmVedEMKMLW+iqper6tryXj8eF0JYEUI4OYTQNITQQUr+v7yTwc8X+cPV2i09tp+IDBORQ0MI/CUL5fG0dttJyUDrdFX9Rkr+ZWuH0oakdaY+YeQF1q0Dhdqs1hOR70RkraruKiLnR44Zqqq/UNUWInKxiIwvze8VkctUtYOIiKo2UNXesZuEEK4PIdQt7/XjcaraRlUbqWp1Ve0mIueKyLWZ+3SRR7yt3ZOl5EnD4SGE+Zn7NJGHPK3dOVLSYOxd+jpbRL4t/XhR5LIoXKxbBwq1Wb1ERPqKyBoReUB+WlhlTRKRWSLygYhMlpIJfQkhTBSRG0XkidJ/EpgjIt3SrOdXIjK7tJ4bROTkEAJDKojxtnavFZFGIvLPMk8B7k3zmshPbtZuKPnpFd/8+JKSf54tLv11UarXRV5i3TqgoeSbdAEAAAB3CvXJKgAAAKoAmlUAAAC4RbMKAAAAt2hWAQAA4FaNihysqkxjIRXLQghNcnVz1i1SlNN1K8LaRcpYu6iqomuXJ6uoDAtzXQCQAtYtqirWLqqq6NqlWQUAAIBbNKsAAABwi2YVAAAAbtGsAgAAwC2aVQAAALhFswoAAAC3aFYBAADgFs0qAAAA3KJZBQAAgFs0qwAAAHCLZhUAAABu0awCAADALZpVAAAAuEWzCgAAALdoVgEAAOAWzSoAAADcolkFAACAWzSrAAAAcKtGrgsAUDlq1LBf7rNnz0507quvvmqyefPmmeyRRx4x2ZIlSxLdAwByrUmTJiZ74IEHTParX/3KZEcccYTJPvroo8wUVuB4sgoAAAC3aFYBAADgFs0qAAAA3KJZBQAAgFsFO2DVuXNnk916662Jzp0wYYLJbrvttrRrArLp8ssvN1n79u0TnZv0uNNPP91kRx55pMm+/PLLRNcDgMoUG6Y6+uijE5179tlnm2zw4MFp1wSerAIAAMAxmlUAAAC4RbMKAAAAt2hWAQAA4JaGEJIfrJr8YOdmzJhhsgMOOCDl68W+iZqhq/9vVgihU65unk/rNqZp06Ymiw0LnnjiiSZT1azUVFbs6+DSSy812ebNm7NeSwXldN2KVN21W7duXZMNHTrUZDfffLPJ1qxZk5Wafk7r1q1NFvtz4vPPPzfZQQcdlI2S0sHaTVFRUZHJiouLE527cOFCk8Xed2fNmlXxwgpHdO3yZBUAAABu0awCAADALZpVAAAAuEWzCgAAALcKYgerPn36mCw2TBUbkood17t370QZA1bItP32289kjz32mMnatGmT8j2mTJlisnfeecdk3bp1M1msvkGDBpns008/Ndl9992XtEQ4F/t/GRs02WuvvUzWq1evrNT0c/bdd1+TbbfddiZbvnx5ZZSDStCzZ8+Uz121apXJYkOEuRqmig0M9ujRw2Sxz0NE5K9//WuGK0oPT1YBAADgFs0qAAAA3KJZBQAAgFs0qwAAAHCLZhUAAABuFcRPA4ht6RcTm95/6qmnTBab/I/91IAWLVqYbNGiRYlqAS677DKTxbYprV+/fsr3+Oyzz0w2YMAAk82fP99kI0eONNmSJUtMVq9ePZO1a9cuaYlwLjbRf8wxxyQ6d+bMmZkuJ2XHHntsouOeeeaZrNaBynP22WenfO6FF15osokTJ6ZTTspGjBhhsiOPPNJk++yzj8li2weL8NMAAAAAgMRoVgEAAOAWzSoAAADcolkFAACAWwUxYJWO2EBULGOYCplWp04dkzVo0MBkIYRE14sdd9FFF5ksNkwVs3HjxpRrOe2000x2ySWXJDoXvgwZMsRksbW7bt06k916661ZqennVKtmn9PUrl070bn33HNPpstBBm2zzTYmGzVqVPTYo446ymSxtRGTq21UY+/ZV1xxhclin0dxcbHJVDUzhWUZT1YBAADgFs0qAAAA3KJZBQAAgFs0qwAAAHCrIAasFi9ebLLYQFQsO+GEExIdFzNo0CCTxXbJAmJOOeUUkyUdYIoZOHCgyaZOnZry9WKefPJJk8V2iYkNZ8G3mjVrRvPtt98+0fljx4412ebNm9OqKVVNmjQxWc+ePU02Y8YMk61YsSIrNSEzdthhB5Odc8450WNjA0fpHFcZYn8GpPN5pPNnSmXiySoAAADcolkFAACAWzSrAAAAcItmFQAAAG4VxIDVhAkTTHbAAQeY7IsvvsjofXv37m0yBqyQVL169RIdV1RUZLLYmn/kkUfSrunn9OnTJ9FxDz30UJYrQabtv//+0fywww4z2d///neTedqhLOk6Xbhwock2bNiQ6XKQQc2bN8/4Na+99lqTZbpfSGrVqlUmi+0OV7du3UqopvLwZBUAAABu0awCAADALZpVAAAAuEWzCgAAALcKYsAqNtS04447mmzw4MEmiw2qzJw502S33nqryWJDXEBSsR3Q9t13X5PNmjXLZJUxTNW+fXuT1apVK9G5VWXXFPzkyiuvTHzssmXLTLZp06ZMlpOW7bbbLtFxDz74YJYrQabdf//9Gb+mp/X84Ycfmiw27LX77rtXRjmVhierAAAAcItmFQAAAG7RrAIAAMAtmlUAAAC4VRADVjFDhgxJlCUVG7AC0vHoo48mypLaY489TFa7dm2TxYYJFixYYLKOHTsmul5sx5UxY8aUUyW8yveB0fXr15ss9rUA5NJBBx1ksl133TUHlVQunqwCAADALZpVAAAAuEWzCgAAALdoVgEAAOBWwQ5Y5UqfPn1M9uSTT+agEhSa2DqLfWP+8uXLTTZ58mSTHXjggYnuO3DgQJMxuOJbt27dTBYbnivPM888k8FqMu/44483WWxnoDlz5lRGOcigo48+2mSffPJJ4vOrVbPP8GK7QW299dYmW7duXeL7JLlHbKjx9ttvT3S92OcRo6qJjss1nqwCAADALZpVAAAAuEWzCgAAALdoVgEAAOAWA1ZAFRfbmWr48OEmS7rLSaNGjUx22mmnVbiuHy1dujTlc5EbK1euNFlxcXH02G+//dZk7777bsZr2lKDBg1MdsYZZ5isb9++JmvXrp3J3nnnnYzUBX/KW7tJnXvuuSaLDVitXr060fViQ02xgdU999zTZOl8LrFzQwgpX68y8WQVAAAAbtGsAgAAwC2aVQAAALhFswoAAAC3GLDKkAkTJpisd+/eJjvhhBNMxg5WiOncubPJYoNOZ555psm22mqrrNSEwvDWW2+ZLLazmYhIs2bNTDZ//nyTbd682WSxnYU6dOiQpMToDj3pDJ989913KZ8LPzZt2mSy8oY8mzRpkvJ9TjnllJTPzfTaLQQ8WQUAAIBbNKsAAABwi2YVAAAAbtGsAgAAwC0GrDJk5syZJosNWMUyYJtttjHZtddea7JDDjnEZLEdSB588EGTzZs3z2SxHVfuuuuucutMRWyA8NRTTzXZM888k9H7IrN69OgRzUeOHGmyTp06max+/fomi+3kM23atET1xAaiHn74YZONHj3aZC1btkx0LqqehQsXmux3v/td9NiHHnrIZDvttFPGa0L6eLIKAAAAt2hWAQAA4BbNKgAAANyiWQUAAIBbDFhlyJdffpnrElCF3XnnnSaLDVMtWbIk0bnXXXedyWK7pjzwwANJSzRiAymxHdpiw2P77LOPyRiw8u29996L5r/97W9NtvPOO5ts6623NlnsfXPlypUpVFe+2KAiCsvrr78ezU888USTvfPOO9kuJ+ree+812fPPP2+y7t27m6x///5ZqckTnqwCAADALZpVAAAAuEWzCgAAALdoVgEAAOAWA1YZEtvBKqk+ffqYLLbrD6q+bbfdNpoffPDBJisuLjZZbHep2DBVrVq1TDZmzBiTnXnmmdF6tvThhx+aLPZN/bEBsKFDh5rs/PPPN9mkSZOi9y5vsAd+zZ8/P9cl/E+xr60NGzbkoBLk0qxZs0xWvXr1HFSSXPv27U0WG56NZTGxXeQ84skqAAAA3KJZBQAAgFs0qwAAAHCLZhUAAABuMWCVIYsWLUr53IEDB5osNrCVzj3gQ9++faP5TjvtZLLY7j7XXHONyWrWrGmyu+++22T9+vVLUmL0vrGdqTZu3GiyESNGmOzwww832d57722y8nbTOvDAA022bt266LFAEsuWLTMZO6ihKgghmCw2MBgTOy52PY94sgoAAAC3aFYBAADgFs0qAAAA3KJZBQAAgFsMWGVRbCCqRYsWJjvggANMFhu6GjJkSEbqQu7EBqnK07RpU5N99tlnJovtVJL0PpMnTzbZsGHDTPb5558nut73339vslGjRpns0UcfNdk+++wTvWZsQKu83a5Q2GJfMw0aNMhBJUB2PPLIIybr1auXyWK7IlZlPFkFAACAWzSrAAAAcItmFQAAAG7RrAIAAMAtmlUAAAC4xU8DyKIJEyaYbPDgwYnOjf3UABSW6tWrm6xNmzaJzo391IA777zTZGPGjKl4YRU0btw4k8V+2sWCBQui57/yyisZrgj5qkOHDibbcccdTbZkyZLKKAfIuNWrV5ts/PjxiY7r3r17VmqqDDxZBQAAgFs0qwAAAHCLZhUAAABu0awCAADALQassujtt9/OdQlw5oEHHojmffr0MVnz5s1N9uKLL5rsuuuuM9ncuXNNtmrVqgQVVo6jjjrKZMuXL89BJShE06ZNy3UJQMbcd999Jvviiy9MFhuw+tOf/pSVmjKNJ6sAAABwi2YVAAAAbtGsAgAAwC2aVQAAALjFgFUWPfnkkyaL7TQxc+ZMk8UGblD1ffzxx9G80HYsY5gKufTRRx/lugQgq6ZMmWKymjVr5qCSzODJKgAAANyiWQUAAIBbNKsAAABwi2YVAAAAbjFgVclUNdclAEBemjFjhsk+++wzk33wwQeVUA2ATOHJKgAAANyiWQUAAIBbNKsAAABwi2YVAAAAbjFgBQDIC+vXrzfZbrvtloNKAGQST1YBAADgFs0qAAAA3KJZBQAAgFs0qwAAAHCLZhUAAABu0awCAADALZpVAAAAuEWzCgAAALdoVgEAAOBWRXewWiYiC7NRCPJaqxzfn3WLVOR63YqwdpEa1i6qquja1RBCZRcCAAAAJMK3AQAAAMAtmlUAAAC4RbMKAAAAt2hWAQAA4BbNKgAAANyiWQUAAIBbNKsAAABwi2YVAAAAbtGsAgAAwK3/BxGL9A7kX7X7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x864 with 16 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_img_grid(\n",
    "    [images[0][idx] for idx in range(16)],\n",
    "    [f\"label={images[1][idx]}\" for idx in range(16)],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make a prediction for a single image."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-07T04:07:28.216159Z",
     "iopub.status.busy": "2020-12-07T04:07:28.215909Z",
     "iopub.status.idle": "2020-12-07T04:07:28.240369Z",
     "shell.execute_reply": "2020-12-07T04:07:28.239597Z",
     "shell.execute_reply.started": "2020-12-07T04:07:28.216131Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(model(images[0][2][None, :]))  # Add a batch dimension"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check Accuracy of Evaluation Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-09T00:40:59.078572Z",
     "iopub.status.busy": "2020-12-09T00:40:59.076824Z",
     "iopub.status.idle": "2020-12-09T00:43:44.766668Z",
     "shell.execute_reply": "2020-12-09T00:43:44.765736Z",
     "shell.execute_reply.started": "2020-12-09T00:40:59.078455Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finish 1000 image prediction in 16.671 secs\n",
      "Finish 2000 image prediction in 16.584 secs\n",
      "Finish 3000 image prediction in 16.616 secs\n",
      "Finish 4000 image prediction in 16.385 secs\n",
      "Finish 5000 image prediction in 17.079 secs\n",
      "Finish 6000 image prediction in 16.649 secs\n",
      "Finish 7000 image prediction in 16.576 secs\n",
      "Finish 8000 image prediction in 16.493 secs\n",
      "Finish 9000 image prediction in 17.668 secs\n",
      "Finish 10000 image prediction in 14.524 secs\n"
     ]
    }
   ],
   "source": [
    "eval_stream = trax.data.TFDS(\n",
    "    \"mnist\", data_dir=\"./mnist_data2/\", keys=(\"image\", \"label\"), train=False\n",
    ")()\n",
    "\n",
    "y_pred = []\n",
    "y = []\n",
    "count = 0\n",
    "start_time=time.time()\n",
    "while True:\n",
    "    count += 1\n",
    "   \n",
    "    image, label = next(eval_stream)\n",
    "    y.append(label)\n",
    "    y_pred.append(np.argmax(model(image[None, :]), axis=1)[0])\n",
    "\n",
    "    if count % 1000 == 0:\n",
    "        end_time=time.time()\n",
    "        print(f\"Finish {count} image prediction in {end_time-start_time:.3f} secs\")\n",
    "        start_time=end_time\n",
    "    \n",
    "    if count == 10000:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-09T00:43:44.768480Z",
     "iopub.status.busy": "2020-12-09T00:43:44.768320Z",
     "iopub.status.idle": "2020-12-09T00:43:44.773435Z",
     "shell.execute_reply": "2020-12-09T00:43:44.772864Z",
     "shell.execute_reply.started": "2020-12-09T00:43:44.768461Z"
    }
   },
   "outputs": [],
   "source": [
    "y_pred = np.array(y_pred)\n",
    "y=np.array(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-09T00:43:44.774971Z",
     "iopub.status.busy": "2020-12-09T00:43:44.774808Z",
     "iopub.status.idle": "2020-12-09T00:43:44.781950Z",
     "shell.execute_reply": "2020-12-09T00:43:44.781090Z",
     "shell.execute_reply.started": "2020-12-09T00:43:44.774952Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "98.56"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(np.where(y_pred==y)[0])/len(y)*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-09T00:37:13.059223Z",
     "iopub.status.busy": "2020-12-09T00:37:13.057922Z",
     "iopub.status.idle": "2020-12-09T00:37:13.082665Z",
     "shell.execute_reply": "2020-12-09T00:37:13.081836Z",
     "shell.execute_reply.started": "2020-12-09T00:37:13.059176Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The tensorboard extension is already loaded. To reload it, use:\n",
      "  %reload_ext tensorboard\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Reusing TensorBoard on port 6008 (pid 83629), started 19:46:18 ago. (Use '!kill 83629' to kill it.)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "      <iframe id=\"tensorboard-frame-907c7fdb2f6ba542\" width=\"100%\" height=\"800\" frameborder=\"0\">\n",
       "      </iframe>\n",
       "      <script>\n",
       "        (function() {\n",
       "          const frame = document.getElementById(\"tensorboard-frame-907c7fdb2f6ba542\");\n",
       "          const url = new URL(\"/\", window.location);\n",
       "          const port = 6008;\n",
       "          if (port) {\n",
       "            url.port = port;\n",
       "          }\n",
       "          frame.src = url;\n",
       "        })();\n",
       "      </script>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Load the TensorBoard notebook extension\n",
    "%load_ext tensorboard\n",
    "%tensorboard --logdir mnist_cnn_model --reload_multifile True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.6 64-bit ('anaconda3': virtualenv)",
   "language": "python",
   "name": "python37664bitanaconda3virtualenv8d2cff73d4434f4db794d807b7c026ac"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}