Chapter10 - Intro to Convolutional Neural Networks - Learning Edges and Corners.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Upgrading our MNIST Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I: 1 Test accuracy: 0.0462 Train accuracy: 0.075 \n",
      "I: 2 Test accuracy: 0.0505 Train accuracy: 0.039 \n",
      "I: 3 Test accuracy: 0.0558 Train accuracy: 0.045 \n",
      "I: 4 Test accuracy: 0.0656 Train accuracy: 0.047 \n",
      "I: 5 Test accuracy: 0.079 Train accuracy: 0.052 \n",
      "I: 6 Test accuracy: 0.1011 Train accuracy: 0.062 \n",
      "I: 7 Test accuracy: 0.1236 Train accuracy: 0.08 \n",
      "I: 8 Test accuracy: 0.1469 Train accuracy: 0.11 \n",
      "I: 9 Test accuracy: 0.174 Train accuracy: 0.144 \n",
      "I: 10 Test accuracy: 0.2044 Train accuracy: 0.167 \n",
      "I: 11 Test accuracy: 0.2285 Train accuracy: 0.205 \n",
      "I: 12 Test accuracy: 0.2572 Train accuracy: 0.243 \n",
      "I: 14 Test accuracy: 0.2553 Train accuracy: 0.256 \n",
      "I: 15 Test accuracy: 0.2114 Train accuracy: 0.214 \n",
      "I: 16 Test accuracy: 0.1275 Train accuracy: 0.162 \n",
      "I: 17 Test accuracy: 0.0781 Train accuracy: 0.089 \n",
      "I: 18 Test accuracy: 0.052 Train accuracy: 0.047 \n",
      "I: 19 Test accuracy: 0.045 Train accuracy: 0.05 \n",
      "I: 20 Test accuracy: 0.043 Train accuracy: 0.031 \n",
      "I: 21 Test accuracy: 0.0544 Train accuracy: 0.046 \n",
      "I: 22 Test accuracy: 0.069 Train accuracy: 0.056 \n",
      "I: 23 Test accuracy: 0.1025 Train accuracy: 0.074 \n",
      "I: 24 Test accuracy: 0.1459 Train accuracy: 0.119 \n",
      "I: 25 Test accuracy: 0.2024 Train accuracy: 0.163 \n",
      "I: 26 Test accuracy: 0.2519 Train accuracy: 0.218 \n",
      "I: 27 Test accuracy: 0.3063 Train accuracy: 0.287 \n",
      "I: 28 Test accuracy: 0.351 Train accuracy: 0.331 \n",
      "I: 29 Test accuracy: 0.3853 Train accuracy: 0.344 \n",
      "I: 30 Test accuracy: 0.4122 Train accuracy: 0.379 \n",
      "I: 31 Test accuracy: 0.4345 Train accuracy: 0.401 \n",
      "I: 32 Test accuracy: 0.448 Train accuracy: 0.435 \n",
      "I: 33 Test accuracy: 0.4586 Train accuracy: 0.448 \n",
      "I: 34 Test accuracy: 0.4565 Train accuracy: 0.443 \n",
      "I: 35 Test accuracy: 0.447 Train accuracy: 0.441 \n",
      "I: 36 Test accuracy: 0.4268 Train accuracy: 0.429 \n",
      "I: 37 Test accuracy: 0.4043 Train accuracy: 0.431 \n",
      "I: 38 Test accuracy: 0.3657 Train accuracy: 0.375 \n",
      "I: 39 Test accuracy: 0.3363 Train accuracy: 0.337 \n",
      "I: 40 Test accuracy: 0.3113 Train accuracy: 0.311 \n",
      "I: 41 Test accuracy: 0.315 Train accuracy: 0.303 \n",
      "I: 42 Test accuracy: 0.3232 Train accuracy: 0.289 \n",
      "I: 43 Test accuracy: 0.3459 Train accuracy: 0.321 \n",
      "I: 44 Test accuracy: 0.3695 Train accuracy: 0.338 \n",
      "I: 45 Test accuracy: 0.4088 Train accuracy: 0.36 \n",
      "I: 46 Test accuracy: 0.4372 Train accuracy: 0.409 \n",
      "I: 47 Test accuracy: 0.4704 Train accuracy: 0.448 \n",
      "I: 48 Test accuracy: 0.4891 Train accuracy: 0.459 \n",
      "I: 49 Test accuracy: 0.5127 Train accuracy: 0.472 \n",
      "I: 50 Test accuracy: 0.5477 Train accuracy: 0.495 \n",
      "I: 51 Test accuracy: 0.5675 Train accuracy: 0.516 \n",
      "I: 52 Test accuracy: 0.5788 Train accuracy: 0.537 \n",
      "I: 53 Test accuracy: 0.6032 Train accuracy: 0.546 \n",
      "I: 54 Test accuracy: 0.6068 Train accuracy: 0.593 \n",
      "I: 55 Test accuracy: 0.62 Train accuracy: 0.59 \n",
      "I: 56 Test accuracy: 0.6279 Train accuracy: 0.598 \n",
      "I: 57 Test accuracy: 0.6356 Train accuracy: 0.61 \n",
      "I: 58 Test accuracy: 0.6474 Train accuracy: 0.619 \n",
      "I: 59 Test accuracy: 0.6544 Train accuracy: 0.649 \n",
      "I: 60 Test accuracy: 0.6565 Train accuracy: 0.621 \n",
      "I: 61 Test accuracy: 0.6635 Train accuracy: 0.632 \n",
      "I: 62 Test accuracy: 0.6742 Train accuracy: 0.64 \n",
      "I: 63 Test accuracy: 0.6844 Train accuracy: 0.647 \n",
      "I: 64 Test accuracy: 0.6799 Train accuracy: 0.639 \n",
      "I: 65 Test accuracy: 0.6885 Train accuracy: 0.668 \n",
      "I: 66 Test accuracy: 0.6923 Train accuracy: 0.657 \n",
      "I: 67 Test accuracy: 0.696 Train accuracy: 0.663 \n",
      "I: 68 Test accuracy: 0.7054 Train accuracy: 0.665 \n",
      "I: 69 Test accuracy: 0.7061 Train accuracy: 0.679 \n",
      "I: 70 Test accuracy: 0.7116 Train accuracy: 0.695 \n",
      "I: 71 Test accuracy: 0.7159 Train accuracy: 0.673 \n",
      "I: 72 Test accuracy: 0.7199 Train accuracy: 0.708 \n",
      "I: 73 Test accuracy: 0.7191 Train accuracy: 0.698 \n",
      "I: 74 Test accuracy: 0.7292 Train accuracy: 0.694 \n",
      "I: 75 Test accuracy: 0.7309 Train accuracy: 0.704 \n",
      "I: 76 Test accuracy: 0.7316 Train accuracy: 0.72 \n",
      "I: 77 Test accuracy: 0.7338 Train accuracy: 0.701 \n",
      "I: 78 Test accuracy: 0.7388 Train accuracy: 0.708 \n",
      "I: 79 Test accuracy: 0.7355 Train accuracy: 0.699 \n",
      "I: 80 Test accuracy: 0.7453 Train accuracy: 0.732 \n",
      "I: 81 Test accuracy: 0.7342 Train accuracy: 0.73 \n",
      "I: 82 Test accuracy: 0.742 Train accuracy: 0.704 \n",
      "I: 83 Test accuracy: 0.7509 Train accuracy: 0.739 \n",
      "I: 84 Test accuracy: 0.7502 Train accuracy: 0.714 \n",
      "I: 85 Test accuracy: 0.7495 Train accuracy: 0.717 \n",
      "I: 86 Test accuracy: 0.7524 Train accuracy: 0.732 \n",
      "I: 87 Test accuracy: 0.7531 Train accuracy: 0.721 \n",
      "I: 88 Test accuracy: 0.7593 Train accuracy: 0.733 \n",
      "I: 89 Test accuracy: 0.7557 Train accuracy: 0.735 \n",
      "I: 90 Test accuracy: 0.758 Train accuracy: 0.739 \n",
      "I: 91 Test accuracy: 0.763 Train accuracy: 0.721 \n",
      "I: 92 Test accuracy: 0.7633 Train accuracy: 0.738 \n",
      "I: 93 Test accuracy: 0.7684 Train accuracy: 0.725 \n",
      "I: 94 Test accuracy: 0.7648 Train accuracy: 0.747 \n",
      "I: 95 Test accuracy: 0.774 Train accuracy: 0.746 \n",
      "I: 96 Test accuracy: 0.7711 Train accuracy: 0.722 \n",
      "I: 97 Test accuracy: 0.7697 Train accuracy: 0.72 \n",
      "I: 98 Test accuracy: 0.7728 Train accuracy: 0.733 \n",
      "I: 99 Test accuracy: 0.7767 Train accuracy: 0.733 \n",
      "I: 100 Test accuracy: 0.7761 Train accuracy: 0.759 \n",
      "I: 101 Test accuracy: 0.7774 Train accuracy: 0.756 \n",
      "I: 102 Test accuracy: 0.7736 Train accuracy: 0.757 \n",
      "I: 103 Test accuracy: 0.7786 Train accuracy: 0.727 \n",
      "I: 104 Test accuracy: 0.7729 Train accuracy: 0.753 \n",
      "I: 105 Test accuracy: 0.782 Train accuracy: 0.764 \n",
      "I: 106 Test accuracy: 0.7759 Train accuracy: 0.748 \n",
      "I: 107 Test accuracy: 0.7809 Train accuracy: 0.746 \n",
      "I: 108 Test accuracy: 0.7761 Train accuracy: 0.753 \n",
      "I: 109 Test accuracy: 0.7799 Train accuracy: 0.763 \n",
      "I: 110 Test accuracy: 0.7839 Train accuracy: 0.751 \n",
      "I: 111 Test accuracy: 0.7885 Train accuracy: 0.768 \n",
      "I: 112 Test accuracy: 0.7875 Train accuracy: 0.754 \n",
      "I: 113 Test accuracy: 0.7904 Train accuracy: 0.754 \n",
      "I: 114 Test accuracy: 0.7888 Train accuracy: 0.757 \n",
      "I: 115 Test accuracy: 0.7893 Train accuracy: 0.759 \n",
      "I: 116 Test accuracy: 0.7912 Train accuracy: 0.76 \n",
      "I: 117 Test accuracy: 0.7868 Train accuracy: 0.757 \n",
      "I: 118 Test accuracy: 0.7891 Train accuracy: 0.756 \n",
      "I: 119 Test accuracy: 0.7916 Train accuracy: 0.769 \n",
      "I: 120 Test accuracy: 0.7946 Train accuracy: 0.771 \n",
      "I: 121 Test accuracy: 0.7924 Train accuracy: 0.765 \n",
      "I: 122 Test accuracy: 0.7952 Train accuracy: 0.753 \n",
      "I: 123 Test accuracy: 0.7932 Train accuracy: 0.765 \n",
      "I: 124 Test accuracy: 0.7966 Train accuracy: 0.768 \n",
      "I: 125 Test accuracy: 0.7966 Train accuracy: 0.772 \n",
      "I: 126 Test accuracy: 0.799 Train accuracy: 0.763 \n",
      "I: 127 Test accuracy: 0.7902 Train accuracy: 0.77 \n",
      "I: 128 Test accuracy: 0.7969 Train accuracy: 0.77 \n",
      "I: 129 Test accuracy: 0.8005 Train accuracy: 0.783 \n",
      "I: 130 Test accuracy: 0.7958 Train accuracy: 0.76 \n",
      "I: 131 Test accuracy: 0.8019 Train accuracy: 0.761 \n",
      "I: 132 Test accuracy: 0.7985 Train accuracy: 0.778 \n",
      "I: 133 Test accuracy: 0.7996 Train accuracy: 0.784 \n",
      "I: 134 Test accuracy: 0.8017 Train accuracy: 0.762 \n",
      "I: 135 Test accuracy: 0.7965 Train accuracy: 0.773 \n",
      "I: 136 Test accuracy: 0.804 Train accuracy: 0.767 \n",
      "I: 137 Test accuracy: 0.8002 Train accuracy: 0.769 \n",
      "I: 138 Test accuracy: 0.8069 Train accuracy: 0.781 \n",
      "I: 139 Test accuracy: 0.7985 Train accuracy: 0.776 \n",
      "I: 140 Test accuracy: 0.7981 Train accuracy: 0.77 \n",
      "I: 141 Test accuracy: 0.7999 Train accuracy: 0.787 \n",
      "I: 142 Test accuracy: 0.806 Train accuracy: 0.783 \n",
      "I: 143 Test accuracy: 0.8059 Train accuracy: 0.769 \n",
      "I: 144 Test accuracy: 0.8012 Train accuracy: 0.783 \n",
      "I: 145 Test accuracy: 0.8012 Train accuracy: 0.786 \n",
      "I: 146 Test accuracy: 0.8023 Train accuracy: 0.778 \n",
      "I: 147 Test accuracy: 0.8049 Train accuracy: 0.782 \n",
      "I: 148 Test accuracy: 0.8022 Train accuracy: 0.783 \n",
      "I: 149 Test accuracy: 0.8042 Train accuracy: 0.779 \n",
      "I: 150 Test accuracy: 0.8021 Train accuracy: 0.77 \n",
      "I: 151 Test accuracy: 0.8094 Train accuracy: 0.781 \n",
      "I: 152 Test accuracy: 0.8047 Train accuracy: 0.795 \n",
      "I: 153 Test accuracy: 0.8096 Train accuracy: 0.794 \n",
      "I: 154 Test accuracy: 0.8062 Train accuracy: 0.794 \n",
      "I: 155 Test accuracy: 0.8081 Train accuracy: 0.789 \n",
      "I: 156 Test accuracy: 0.8038 Train accuracy: 0.779 \n",
      "I: 157 Test accuracy: 0.8036 Train accuracy: 0.796 \n",
      "I: 158 Test accuracy: 0.8034 Train accuracy: 0.796 \n",
      "I: 159 Test accuracy: 0.8065 Train accuracy: 0.786 \n",
      "I: 160 Test accuracy: 0.8126 Train accuracy: 0.789 \n",
      "I: 161 Test accuracy: 0.8104 Train accuracy: 0.798 \n",
      "I: 162 Test accuracy: 0.8084 Train accuracy: 0.782 \n",
      "I: 163 Test accuracy: 0.8087 Train accuracy: 0.782 \n",
      "I: 164 Test accuracy: 0.8096 Train accuracy: 0.786 \n",
      "I: 165 Test accuracy: 0.8145 Train accuracy: 0.789 \n",
      "I: 166 Test accuracy: 0.81 Train accuracy: 0.764 \n",
      "I: 167 Test accuracy: 0.8131 Train accuracy: 0.787 \n",
      "I: 168 Test accuracy: 0.8118 Train accuracy: 0.776 \n",
      "I: 169 Test accuracy: 0.8139 Train accuracy: 0.791 \n",
      "I: 170 Test accuracy: 0.8126 Train accuracy: 0.79 \n",
      "I: 171 Test accuracy: 0.8136 Train accuracy: 0.799 \n",
      "I: 172 Test accuracy: 0.8142 Train accuracy: 0.793 \n",
      "I: 173 Test accuracy: 0.8105 Train accuracy: 0.79 \n",
      "I: 174 Test accuracy: 0.8145 Train accuracy: 0.778 \n",
      "I: 175 Test accuracy: 0.812 Train accuracy: 0.79 \n",
      "I: 176 Test accuracy: 0.8147 Train accuracy: 0.784 \n",
      "I: 177 Test accuracy: 0.8095 Train accuracy: 0.801 \n",
      "I: 178 Test accuracy: 0.8151 Train accuracy: 0.793 \n",
      "I: 179 Test accuracy: 0.8172 Train accuracy: 0.786 \n",
      "I: 180 Test accuracy: 0.8167 Train accuracy: 0.794 \n",
      "I: 181 Test accuracy: 0.8127 Train accuracy: 0.792 \n",
      "I: 182 Test accuracy: 0.8207 Train accuracy: 0.793 \n",
      "I: 183 Test accuracy: 0.8146 Train accuracy: 0.798 \n",
      "I: 184 Test accuracy: 0.8158 Train accuracy: 0.782 \n",
      "I: 185 Test accuracy: 0.8163 Train accuracy: 0.803 \n",
      "I: 186 Test accuracy: 0.8178 Train accuracy: 0.797 \n",
      "I: 187 Test accuracy: 0.8133 Train accuracy: 0.799 \n",
      "I: 188 Test accuracy: 0.8166 Train accuracy: 0.801 \n",
      "I: 189 Test accuracy: 0.8142 Train accuracy: 0.804 \n",
      "I: 190 Test accuracy: 0.8144 Train accuracy: 0.804 \n",
      "I: 191 Test accuracy: 0.8178 Train accuracy: 0.795 \n",
      "I: 192 Test accuracy: 0.8197 Train accuracy: 0.805 \n",
      "I: 193 Test accuracy: 0.8164 Train accuracy: 0.804 \n",
      "I: 194 Test accuracy: 0.815 Train accuracy: 0.805 \n",
      "I: 195 Test accuracy: 0.8213 Train accuracy: 0.798 \n",
      "I: 196 Test accuracy: 0.8193 Train accuracy: 0.804 \n",
      "I: 197 Test accuracy: 0.8134 Train accuracy: 0.797 \n",
      "I: 198 Test accuracy: 0.8186 Train accuracy: 0.81 \n",
      "I: 199 Test accuracy: 0.8191 Train accuracy: 0.801 \n",
      "I: 200 Test accuracy: 0.8156 Train accuracy: 0.809 \n",
      "I: 201 Test accuracy: 0.8167 Train accuracy: 0.803 \n",
      "I: 202 Test accuracy: 0.8255 Train accuracy: 0.809 \n",
      "I: 203 Test accuracy: 0.8176 Train accuracy: 0.81 \n",
      "I: 204 Test accuracy: 0.8191 Train accuracy: 0.813 \n",
      "I: 205 Test accuracy: 0.8207 Train accuracy: 0.814 \n",
      "I: 206 Test accuracy: 0.8171 Train accuracy: 0.797 \n",
      "I: 207 Test accuracy: 0.818 Train accuracy: 0.809 \n",
      "I: 208 Test accuracy: 0.8205 Train accuracy: 0.793 \n",
      "I: 209 Test accuracy: 0.8137 Train accuracy: 0.817 \n",
      "I: 210 Test accuracy: 0.8193 Train accuracy: 0.804 \n",
      "I: 211 Test accuracy: 0.8166 Train accuracy: 0.815 \n",
      "I: 212 Test accuracy: 0.8167 Train accuracy: 0.81 \n",
      "I: 213 Test accuracy: 0.8193 Train accuracy: 0.827 \n",
      "I: 214 Test accuracy: 0.8209 Train accuracy: 0.799 \n",
      "I: 215 Test accuracy: 0.8217 Train accuracy: 0.811 \n",
      "I: 216 Test accuracy: 0.8214 Train accuracy: 0.81 \n",
      "I: 217 Test accuracy: 0.8185 Train accuracy: 0.806 \n",
      "I: 218 Test accuracy: 0.8165 Train accuracy: 0.799 \n",
      "I: 219 Test accuracy: 0.8168 Train accuracy: 0.805 \n",
      "I: 220 Test accuracy: 0.8187 Train accuracy: 0.825 \n",
      "I: 221 Test accuracy: 0.8229 Train accuracy: 0.811 \n",
      "I: 222 Test accuracy: 0.8204 Train accuracy: 0.813 \n",
      "I: 223 Test accuracy: 0.8224 Train accuracy: 0.808 \n",
      "I: 224 Test accuracy: 0.8188 Train accuracy: 0.811 \n",
      "I: 225 Test accuracy: 0.8206 Train accuracy: 0.798 \n",
      "I: 226 Test accuracy: 0.8201 Train accuracy: 0.818 \n",
      "I: 227 Test accuracy: 0.8256 Train accuracy: 0.805 \n",
      "I: 228 Test accuracy: 0.8227 Train accuracy: 0.801 \n",
      "I: 229 Test accuracy: 0.818 Train accuracy: 0.804 \n",
      "I: 230 Test accuracy: 0.8223 Train accuracy: 0.803 \n",
      "I: 231 Test accuracy: 0.8172 Train accuracy: 0.81 \n",
      "I: 232 Test accuracy: 0.8213 Train accuracy: 0.806 \n",
      "I: 233 Test accuracy: 0.819 Train accuracy: 0.817 \n",
      "I: 234 Test accuracy: 0.8206 Train accuracy: 0.803 \n",
      "I: 235 Test accuracy: 0.821 Train accuracy: 0.8 \n",
      "I: 236 Test accuracy: 0.8213 Train accuracy: 0.812 \n",
      "I: 237 Test accuracy: 0.8238 Train accuracy: 0.801 \n",
      "I: 238 Test accuracy: 0.8259 Train accuracy: 0.823 \n",
      "I: 239 Test accuracy: 0.8233 Train accuracy: 0.805 \n",
      "I: 240 Test accuracy: 0.8221 Train accuracy: 0.813 \n",
      "I: 241 Test accuracy: 0.8165 Train accuracy: 0.811 \n",
      "I: 242 Test accuracy: 0.8193 Train accuracy: 0.82 \n",
      "I: 243 Test accuracy: 0.8225 Train accuracy: 0.829 \n",
      "I: 244 Test accuracy: 0.8274 Train accuracy: 0.807 \n",
      "I: 245 Test accuracy: 0.8254 Train accuracy: 0.818 \n",
      "I: 246 Test accuracy: 0.8255 Train accuracy: 0.818 \n",
      "I: 247 Test accuracy: 0.824 Train accuracy: 0.811 \n",
      "I: 248 Test accuracy: 0.8242 Train accuracy: 0.814 \n",
      "I: 249 Test accuracy: 0.8213 Train accuracy: 0.808 \n",
      "I: 250 Test accuracy: 0.8215 Train accuracy: 0.816 \n",
      "I: 251 Test accuracy: 0.8245 Train accuracy: 0.82 \n",
      "I: 252 Test accuracy: 0.8179 Train accuracy: 0.823 \n",
      "I: 253 Test accuracy: 0.8246 Train accuracy: 0.808 \n",
      "I: 254 Test accuracy: 0.8235 Train accuracy: 0.818 \n",
      "I: 255 Test accuracy: 0.8255 Train accuracy: 0.815 \n",
      "I: 256 Test accuracy: 0.8244 Train accuracy: 0.811 \n",
      "I: 257 Test accuracy: 0.8249 Train accuracy: 0.819 \n",
      "I: 258 Test accuracy: 0.8213 Train accuracy: 0.809 \n",
      "I: 259 Test accuracy: 0.8252 Train accuracy: 0.818 \n",
      "I: 260 Test accuracy: 0.8274 Train accuracy: 0.807 \n",
      "I: 261 Test accuracy: 0.8221 Train accuracy: 0.814 \n",
      "I: 262 Test accuracy: 0.8248 Train accuracy: 0.814 \n",
      "I: 263 Test accuracy: 0.8223 Train accuracy: 0.812 \n",
      "I: 264 Test accuracy: 0.8194 Train accuracy: 0.821 \n",
      "I: 265 Test accuracy: 0.8236 Train accuracy: 0.816 \n",
      "I: 266 Test accuracy: 0.8251 Train accuracy: 0.818 \n",
      "I: 267 Test accuracy: 0.8228 Train accuracy: 0.825 \n",
      "I: 268 Test accuracy: 0.8257 Train accuracy: 0.816 \n",
      "I: 269 Test accuracy: 0.824 Train accuracy: 0.818 \n",
      "I: 270 Test accuracy: 0.8287 Train accuracy: 0.822 \n",
      "I: 271 Test accuracy: 0.8261 Train accuracy: 0.807 \n",
      "I: 272 Test accuracy: 0.8284 Train accuracy: 0.816 \n",
      "I: 273 Test accuracy: 0.8233 Train accuracy: 0.821 \n",
      "I: 274 Test accuracy: 0.8311 Train accuracy: 0.818 \n",
      "I: 275 Test accuracy: 0.8242 Train accuracy: 0.808 \n",
      "I: 276 Test accuracy: 0.8227 Train accuracy: 0.812 \n",
      "I: 277 Test accuracy: 0.8282 Train accuracy: 0.814 \n",
      "I: 278 Test accuracy: 0.8277 Train accuracy: 0.82 \n",
      "I: 279 Test accuracy: 0.8239 Train accuracy: 0.811 \n",
      "I: 280 Test accuracy: 0.822 Train accuracy: 0.813 \n",
      "I: 281 Test accuracy: 0.8203 Train accuracy: 0.816 \n",
      "I: 282 Test accuracy: 0.8263 Train accuracy: 0.821 \n",
      "I: 283 Test accuracy: 0.8254 Train accuracy: 0.815 \n",
      "I: 284 Test accuracy: 0.8265 Train accuracy: 0.824 \n",
      "I: 285 Test accuracy: 0.8201 Train accuracy: 0.817 \n",
      "I: 286 Test accuracy: 0.82 Train accuracy: 0.81 \n",
      "I: 287 Test accuracy: 0.8246 Train accuracy: 0.825 \n",
      "I: 288 Test accuracy: 0.8246 Train accuracy: 0.823 \n",
      "I: 289 Test accuracy: 0.8304 Train accuracy: 0.833 \n",
      "I: 290 Test accuracy: 0.8302 Train accuracy: 0.813 \n",
      "I: 291 Test accuracy: 0.8247 Train accuracy: 0.826 \n",
      "I: 292 Test accuracy: 0.8211 Train accuracy: 0.823 \n",
      "I: 293 Test accuracy: 0.8255 Train accuracy: 0.815 \n",
      "I: 294 Test accuracy: 0.8271 Train accuracy: 0.818 \n",
      "I: 295 Test accuracy: 0.8312 Train accuracy: 0.813 \n",
      "I: 296 Test accuracy: 0.8246 Train accuracy: 0.824 \n",
      "I: 297 Test accuracy: 0.8222 Train accuracy: 0.812 \n",
      "I: 298 Test accuracy: 0.8296 Train accuracy: 0.825 \n",
      "I: 299 Test accuracy: 0.828 Train accuracy: 0.82 \n",
      "I: 300 Test accuracy: 0.825 Train accuracy: 0.817 \n"
     ]
    }
   ],
   "source": [
    "using MLDatasets\n",
    "train_x, train_y = MNIST.traindata()\n",
    "test_x,  test_y  = MNIST.testdata();\n",
    "(images, labels) = (train_x[:,:,1:1000], train_y[1:1000]); #(col,row,batch)\n",
    "\n",
    "one_hot_labels = zeros(10, length(labels))\n",
    "for (i,l) in enumerate(labels)\n",
    "    one_hot_labels[l+1,i] = 1.0\n",
    "end\n",
    "\n",
    "labels = one_hot_labels\n",
    "\n",
    "test_labels = zeros(10, length(test_y))\n",
    "for (i,l) in enumerate(test_y)\n",
    "    test_labels[l+1, i] = 1.0\n",
    "end\n",
    "\n",
    "using Random\n",
    "Random.seed!(1)\n",
    "\n",
    "tanh2deriv(output) = 1 - output^2\n",
    "\n",
    "function softmax(x)\n",
    "    temp = exp.(x)\n",
    "    return temp ./ sum(temp, dims=1)\n",
    "end\n",
    "\n",
    "alpha, iterations = (2, 300)\n",
    "pixels_per_image, num_labels = (784, 10)\n",
    "batch_size = 128\n",
    "\n",
    "input_rows = 28\n",
    "input_cols = 28\n",
    "\n",
    "kernel_rows = 3\n",
    "kernel_cols = 3\n",
    "num_kernels = 16\n",
    "\n",
    "hidden_size = ((input_rows - kernel_rows) * \n",
    "               (input_cols - kernel_cols)) * num_kernels\n",
    "\n",
    "kernels = 0.02 .* rand(num_kernels, kernel_rows*kernel_cols) .- 0.01  \n",
    "weights_1_2 = 0.2 .* rand(num_labels, hidden_size) .- 0.1\n",
    "\n",
    "function get_image_section(layer,row_from, row_to, col_from, col_to)\n",
    "    section = layer[col_from:col_to, row_from:row_to, :]\n",
    "    return reshape(section, (col_to-col_from+1, row_to-row_from+1, 1, :))\n",
    "end\n",
    "\n",
    "\n",
    "for j=1:iterations\n",
    "    Correct_cnt = 0\n",
    "    for i = 1:batch_size:size(images, 3)-batch_size\n",
    "        batch_start, batch_end = i, i+batch_size-1\n",
    "        layer_0 = images[:,:, batch_start:batch_end]\n",
    "        sects = []\n",
    "        for col_start=1:size(layer_0, 1) - kernel_cols\n",
    "            for row_start=1:size(layer_0, 2)-kernel_rows\n",
    "                sect = get_image_section(layer_0, col_start, col_start+kernel_cols-1, row_start, row_start+kernel_rows-1)\n",
    "                push!(sects, sect)\n",
    "            end\n",
    "        end\n",
    "        expanded_input = cat(sects...,dims=3) \n",
    "        es = size(expanded_input)\n",
    "        flattened_input = reshape(expanded_input, (:,es[3]*es[4]))\n",
    "        kernel_output = kernels * flattened_input\n",
    "        layer_1 = tanh.(reshape(kernel_output, (:, es[4])))\n",
    "        dropout_mask = bitrand(size(layer_1))\n",
    "        layer_1 .*= dropout_mask .* 2\n",
    "        layer_2 = softmax(weights_1_2 * layer_1)\n",
    "\n",
    "        Correct_cnt += sum(argmax(layer_2;dims=1) .== argmax(labels[:,batch_start:batch_end];dims=1))\n",
    "        layer_2_delta = (labels[:,batch_start:batch_end] .- layer_2) ./ (batch_size * size(layer_2, 2))\n",
    "        layer_1_delta = (weights_1_2' * layer_2_delta) .* tanh2deriv.(layer_1)\n",
    "        layer_1_delta .*= dropout_mask\n",
    "        weights_1_2 += alpha .* layer_2_delta * layer_1'\n",
    "        l1d_reshape = reshape(layer_1_delta, size(kernel_output)) \n",
    "        k_update = l1d_reshape * flattened_input'\n",
    "        kernels .-= alpha .* k_update\n",
    "    end\n",
    "    \n",
    "    test_correct_cnt = 0\n",
    "    \n",
    "    for i=1:size(test_x, 3)\n",
    "        layer_0 = test_x[:,:, i]\n",
    "        sects = []\n",
    "        for col_start=1:size(layer_0, 1) - kernel_cols\n",
    "            for row_start=1:size(layer_0, 2)-kernel_rows\n",
    "                sect = get_image_section(layer_0, col_start, col_start+kernel_cols-1, row_start, row_start+kernel_rows-1)\n",
    "                push!(sects, sect)\n",
    "            end\n",
    "        end\n",
    "        expanded_input = cat(sects...,dims=3) ##\n",
    "        es = size(expanded_input)\n",
    "        flattened_input = reshape(expanded_input, (:,es[3]*es[4]))\n",
    "        kernel_output = kernels * flattened_input\n",
    "        layer_1 = tanh.(reshape(kernel_output, (:, es[4])))\n",
    "        dropout_mask = bitrand(size(layer_1))\n",
    "        layer_1 .*= dropout_mask .* 2\n",
    "        layer_2 = softmax(weights_1_2 * layer_1)\n",
    "        \n",
    "        test_correct_cnt += Int(argmax(dropdims(layer_2;dims=2)) == argmax(test_labels[:,i]))\n",
    "    end\n",
    "    \n",
    "    if (j%1 == 0)\n",
    "        println(\"I: $(j) Test accuracy: $(test_correct_cnt/size(test_x, 3)) Train accuracy: $(Correct_cnt/size(images, 3)) \")\n",
    "    end  \n",
    "end"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.4.1",
   "language": "julia",
   "name": "julia-1.4"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.4.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}