Chapter8 - Intro to Regularization - Learning Signal and Ignoring Noise.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3 Layer Network on MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: Precompiling MLDatasets [eb30cadb-4394-5ae3-aed4-317e484a6458]\n",
      "└ @ Base loading.jl:1260\n"
     ]
    }
   ],
   "source": [
    "using Pkg;Pkg.add(\"MLDatasets\")\n",
    "using MLDatasets\n",
    "train_x, train_y = MNIST.traindata()\n",
    "test_x,  test_y  = MNIST.testdata();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I : 40 Train error: 0.0682271128273786 Train accuracy: 0.993    \r"
     ]
    }
   ],
   "source": [
    "(images, labels) = (reshape(train_x[:,:,1:1000], (28*28, 1000)), train_y[1:1000])\n",
    "one_hot_labels = zeros(10,length(labels))\n",
    "for (i,l) in enumerate(labels)\n",
    "    one_hot_labels[l+1, i] = 1.0\n",
    "end\n",
    "labels = one_hot_labels\n",
    "\n",
    "test_images = reshape(test_x, (28*28, size(test_x,3)))\n",
    "test_labels = zeros((10, size(test_x,3)))\n",
    "\n",
    "for (i,l) in enumerate(test_y)\n",
    "    test_labels[l+1, i] = 1.0\n",
    "end\n",
    "\n",
    "using Random\n",
    "Random.seed!(1)\n",
    "\n",
    "\n",
    "relu(x) = x > 0 ? x : 0\n",
    "relu2deriv(x) = x > 0 ? 1 : 0\n",
    "\n",
    "alpha, iterations, hidden_size, pixels_per_image, num_labels = (0.005, 40, 40, 784, 10)\n",
    "\n",
    "weights_0_1 = 0.2 .* rand(pixels_per_image,hidden_size) .- 0.1\n",
    "weights_1_2 = 0.2 .* rand(hidden_size,num_labels) .- 0.1\n",
    "\n",
    "for j = 1:iterations\n",
    "    Error, Correct_cnt = (0.0, 0)\n",
    "    for i = 1:size(images,2)\n",
    "        layer_0 = images[:,i]\n",
    "        layer_1 = relu.(layer_0' * weights_0_1)\n",
    "        layer_2 = layer_1 * weights_1_2\n",
    "        Error += sum((labels[:, i]' .- layer_2) .^ 2)\n",
    "        \n",
    "        Correct_cnt += Int(argmax(layer_2) == argmax(labels[:, i]'))\n",
    "        \n",
    "        layer_2_delta = (labels[:, i]' .- layer_2)\n",
    "        layer_1_delta = (layer_2_delta * weights_1_2') .* relu2deriv.(layer_1)\n",
    "        \n",
    "        weights_1_2 += alpha .* layer_1' * layer_2_delta\n",
    "        weights_0_1 += alpha .* layer_0 * layer_1_delta\n",
    "    end\n",
    "    print(\"I : $(j) Train error: $(Error/size(images, 2)) Train accuracy: $(Correct_cnt/size(images, 2))   \\r\")\n",
    "    \n",
    "end\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test-Err:: 0.27701831632546603 Test-Acc:: 0.8727   \r"
     ]
    }
   ],
   "source": [
    "if (j % 10 == 0) || (j == iterations)\n",
    "    Error, Correct_cnt = (0.0, 0)\n",
    "\n",
    "    for i = 1:size(test_images, 2)\n",
    "        \n",
    "        layer_0 = test_images[:, i]\n",
    "        layer_1 = relu.(layer_0' * weights_0_1)\n",
    "        layer_2 = layer_1 * weights_1_2\n",
    "        \n",
    "        Error += sum((test_labels[:, i]' .- layer_2) .^ 2)\n",
    "        Correct_cnt += Int(argmax(layer_2) == argmax(test_labels[:, i]'))\n",
    "\n",
    "    end\n",
    "    print(\"Test-Err:: $(Error/size(test_images,2)) Test-Acc:: $(Correct_cnt/size(test_images,2))   \\r\")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I : 5 Train error: 0.29739439558840397 Train accuracy: 0.878   \n",
      "Test-Err:: 0.3767939044628232 Test-Acc:: 0.8162\n",
      "I : 10 Train error: 0.18814760092583613 Train accuracy: 0.942   \n",
      "Test-Err:: 0.30468604953956663 Test-Acc:: 0.8629\n",
      "I : 15 Train error: 0.14439583078193777 Train accuracy: 0.967   \n",
      "Test-Err:: 0.2836949460454106 Test-Acc:: 0.8731\n",
      "I : 20 Train error: 0.11903362765796817 Train accuracy: 0.979   \n",
      "Test-Err:: 0.27646178423027645 Test-Acc:: 0.8724\n",
      "I : 25 Train error: 0.10114517964328593 Train accuracy: 0.987   \n",
      "Test-Err:: 0.2739270039563274 Test-Acc:: 0.8723\n",
      "I : 30 Train error: 0.0875160775763943 Train accuracy: 0.991    \n",
      "Test-Err:: 0.2733526287620866 Test-Acc:: 0.8738\n",
      "I : 35 Train error: 0.07688726309627121 Train accuracy: 0.992   \n",
      "Test-Err:: 0.274741321852286 Test-Acc:: 0.8726\n",
      "I : 40 Train error: 0.06822711282737862 Train accuracy: 0.993   \n",
      "Test-Err:: 0.27701831632546603 Test-Acc:: 0.8727\n"
     ]
    }
   ],
   "source": [
    "(images, labels) = (reshape(train_x[:,:,1:1000], (28*28, 1000)), train_y[1:1000])\n",
    "one_hot_labels = zeros(10,length(labels))\n",
    "for (i,l) in enumerate(labels)\n",
    "    one_hot_labels[l+1, i] = 1.0\n",
    "end\n",
    "labels = one_hot_labels\n",
    "\n",
    "test_images = reshape(test_x, (28*28, size(test_x,3)))\n",
    "test_labels = zeros((10, size(test_x,3)))\n",
    "\n",
    "for (i,l) in enumerate(test_y)\n",
    "    test_labels[l+1, i] = 1.0\n",
    "end\n",
    "\n",
    "using Random\n",
    "Random.seed!(1)\n",
    "\n",
    "\n",
    "relu(x) = x > 0 ? x : 0\n",
    "relu2deriv(x) = x > 0 ? 1 : 0\n",
    "\n",
    "alpha, iterations, hidden_size, pixels_per_image, num_labels = (0.005, 40, 40, 784, 10)\n",
    "\n",
    "weights_0_1 = 0.2 .* rand(pixels_per_image,hidden_size) .- 0.1\n",
    "weights_1_2 = 0.2 .* rand(hidden_size,num_labels) .- 0.1\n",
    "\n",
    "for j = 1:iterations\n",
    "    Error, Correct_cnt = (0.0, 0)\n",
    "    for i = 1:size(images,2)\n",
    "        layer_0 = images[:,i]\n",
    "        layer_1 = relu.(layer_0' * weights_0_1)\n",
    "        layer_2 = layer_1 * weights_1_2\n",
    "        Error += sum((labels[:, i]' .- layer_2) .^ 2)\n",
    "        \n",
    "        Correct_cnt += Int(argmax(layer_2) == argmax(labels[:, i]'))\n",
    "        \n",
    "        layer_2_delta = (labels[:, i]' .- layer_2)\n",
    "        layer_1_delta = (layer_2_delta * weights_1_2') .* relu2deriv.(layer_1)\n",
    "        \n",
    "        weights_1_2 += alpha .* layer_1' * layer_2_delta\n",
    "        weights_0_1 += alpha .* layer_0 * layer_1_delta\n",
    "    end\n",
    "    print(\"I : $(j) Train error: $(Error/size(images, 2)) Train accuracy: $(Correct_cnt/size(images,2))   \\r\")\n",
    "    \n",
    "    if (j % 5 == 0) || (j == iterations)\n",
    "        Error, Correct_cnt = (0.0, 0)\n",
    "        for i = 1:size(test_images, 2)\n",
    "        \n",
    "            layer_0 = test_images[:, i]\n",
    "            layer_1 = relu.(layer_0' * weights_0_1)\n",
    "            layer_2 = layer_1 * weights_1_2\n",
    "\n",
    "            Error += sum((test_labels[:, i]' .- layer_2) .^ 2)\n",
    "            Correct_cnt += Int(argmax(layer_2) == argmax(test_labels[:, i]'))\n",
    "        end\n",
    "    println()\n",
    "    println(\"Test-Err:: $(Error/size(test_images, 2)) Test-Acc:: $(Correct_cnt/size(test_images, 2))\")\n",
    "    end\n",
    "end\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dropout In Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "i = 1\n",
    "layer_0 = images[:, i]\n",
    "dropout_mask = bitrand(size(layer_1))\n",
    "layer_1 = relu.(layer_0' * weights_0_1)\n",
    "layer_1 .*= dropout_mask .* 2\n",
    "layer_2 = layer_1 * weights_1_2\n",
    "\n",
    "Error += sum((labels[:, i]' .- layer_2) .^ 2)\n",
    "Correct_cnt += Int(argmax(layer_2) == argmax(labels[:, i]'))\n",
    "\n",
    "layer_2_delta = (labels[i]' .- layer_2)\n",
    "layer_1_delta = (layer_2_delta * weights_1_2') .* relu2deriv.(layer_1)\n",
    "\n",
    "layer_1_delta .*= dropout_mask\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I: 10 Train error: 0.40356431055896214 Train accuracy: 0.821 Test-Err:: 0.37596035249840204 Test-Acc:: 0.8267\n",
      "I: 20 Train error: 0.3316619242484017 Train accuracy: 0.864 Test-Err:: 0.3210854633568354 Test-Acc:: 0.8565\n",
      "I: 30 Train error: 0.28404975000597044 Train accuracy: 0.909 Test-Err:: 0.297259838898316 Test-Acc:: 0.8695\n",
      "I: 40 Train error: 0.2498115288206865 Train accuracy: 0.938 Test-Err:: 0.2893557843097215 Test-Acc:: 0.872\n",
      "I: 50 Train error: 0.22600793670063843 Train accuracy: 0.94 Test-Err:: 0.28712980676338556 Test-Acc:: 0.8747\n",
      "I: 60 Train error: 0.22531916559425316 Train accuracy: 0.955 Test-Err:: 0.28186890220650884 Test-Acc:: 0.875\n",
      "I: 70 Train error: 0.22171653734668323 Train accuracy: 0.95 Test-Err:: 0.2829342462900369 Test-Acc:: 0.8761\n",
      "I: 80 Train error: 0.19041724520488393 Train accuracy: 0.964 Test-Err:: 0.2810256102909915 Test-Acc:: 0.8742\n",
      "I: 90 Train error: 0.19813206766203406 Train accuracy: 0.961 Test-Err:: 0.28149859920679754 Test-Acc:: 0.8777\n",
      "I: 100 Train error: 0.18926961276782234 Train accuracy: 0.972 Test-Err:: 0.2819067210751134 Test-Acc:: 0.8739\n",
      "I: 110 Train error: 0.19372927456550987 Train accuracy: 0.967 Test-Err:: 0.28927875885794363 Test-Acc:: 0.8704\n",
      "I: 120 Train error: 0.1831997845386575 Train accuracy: 0.971 Test-Err:: 0.2809491380007914 Test-Acc:: 0.873\n",
      "I: 130 Train error: 0.17850438982105696 Train accuracy: 0.967 Test-Err:: 0.2838860525795159 Test-Acc:: 0.8733\n",
      "I: 140 Train error: 0.17670248273226621 Train accuracy: 0.98 Test-Err:: 0.2823073186893131 Test-Acc:: 0.8715\n",
      "I: 150 Train error: 0.17215894385792782 Train accuracy: 0.977 Test-Err:: 0.2904828830075241 Test-Acc:: 0.8727\n",
      "I: 160 Train error: 0.17305531641370137 Train accuracy: 0.976 Test-Err:: 0.28424541500244616 Test-Acc:: 0.8744\n",
      "I: 170 Train error: 0.16314605128673285 Train accuracy: 0.974 Test-Err:: 0.2858407221895154 Test-Acc:: 0.8714\n",
      "I: 180 Train error: 0.18126053551539842 Train accuracy: 0.965 Test-Err:: 0.2863702153853667 Test-Acc:: 0.8724\n",
      "I: 190 Train error: 0.16747228743757442 Train accuracy: 0.976 Test-Err:: 0.2846326076435637 Test-Acc:: 0.8719\n",
      "I: 200 Train error: 0.1588838652755072 Train accuracy: 0.981 Test-Err:: 0.2919092953635126 Test-Acc:: 0.8717\n",
      "I: 210 Train error: 0.1696327475878241 Train accuracy: 0.978 Test-Err:: 0.2890941732654742 Test-Acc:: 0.873\n",
      "I: 220 Train error: 0.16007505368406827 Train accuracy: 0.975 Test-Err:: 0.2896464591344745 Test-Acc:: 0.8703\n",
      "I: 230 Train error: 0.16155303310002603 Train accuracy: 0.973 Test-Err:: 0.29286989225311844 Test-Acc:: 0.869\n",
      "I: 240 Train error: 0.15622801192838798 Train accuracy: 0.975 Test-Err:: 0.29359598474328436 Test-Acc:: 0.868\n",
      "I: 250 Train error: 0.16142636859339965 Train accuracy: 0.979 Test-Err:: 0.2861386783957469 Test-Acc:: 0.8716\n",
      "I: 260 Train error: 0.14900963086692945 Train accuracy: 0.985 Test-Err:: 0.291710859957472 Test-Acc:: 0.8695\n",
      "I: 270 Train error: 0.15065591041036916 Train accuracy: 0.986 Test-Err:: 0.2904821096457524 Test-Acc:: 0.8689\n",
      "I: 280 Train error: 0.15472367318938593 Train accuracy: 0.981 Test-Err:: 0.2852670112409462 Test-Acc:: 0.8707\n",
      "I: 290 Train error: 0.16576813389330905 Train accuracy: 0.978 Test-Err:: 0.292761099753508 Test-Acc:: 0.8697\n",
      "I: 300 Train error: 0.15642359754004118 Train accuracy: 0.977 Test-Err:: 0.29269829887865706 Test-Acc:: 0.8677\n"
     ]
    }
   ],
   "source": [
    "using Random\n",
    "Random.seed!(1)\n",
    "\n",
    "relu(x) = x > 0 ? x : 0\n",
    "relu2deriv(x) = x > 0 ? 1 : 0\n",
    "\n",
    "alpha, iterations, hidden_size = (0.005, 300, 100)\n",
    "pixels_per_image, num_labels = (784, 10)\n",
    "\n",
    "weights_0_1 = 0.2 .* rand(pixels_per_image,hidden_size) .- 0.1\n",
    "weights_1_2 = 0.2 .* rand(hidden_size,num_labels) .- 0.1\n",
    "\n",
    "for j = 1:iterations\n",
    "    Error, Correct_cnt = (0.0, 0)\n",
    "    for i = 1:size(images, 2)\n",
    "        layer_0 = images[:, i]\n",
    "        layer_1 = relu.(layer_0' * weights_0_1)\n",
    "        dropout_mask = bitrand(size(layer_1))\n",
    "        layer_1 .*= dropout_mask .* 2\n",
    "        layer_2 = layer_1 * weights_1_2\n",
    "\n",
    "        Error += sum((labels[:, i]' .- layer_2) .^ 2)\n",
    "        Correct_cnt += Int(argmax(layer_2) == argmax(labels[:, i]'))\n",
    "\n",
    "        layer_2_delta = (labels[:, i]' .- layer_2)\n",
    "        layer_1_delta = (layer_2_delta * weights_1_2') .* relu2deriv.(layer_1)\n",
    "\n",
    "        layer_1_delta .*= dropout_mask\n",
    "        \n",
    "        weights_1_2 += alpha .* layer_1' * layer_2_delta\n",
    "        weights_0_1 += alpha .* layer_0 * layer_1_delta\n",
    "    end\n",
    "    \n",
    "    if (j % 10 == 0) || (j == iterations)\n",
    "        test_Error, test_Correct_cnt = (0.0, 0)\n",
    "        for i = 1:size(test_images, 2)\n",
    "        \n",
    "            layer_0 = test_images[:, i]\n",
    "            layer_1 = relu.(layer_0' * weights_0_1)\n",
    "            layer_2 = layer_1 * weights_1_2\n",
    "\n",
    "            test_Error += sum((test_labels[:, i]' .- layer_2) .^ 2)\n",
    "            test_Correct_cnt += Int(argmax(layer_2) == argmax(test_labels[:, i]'))\n",
    "        end\n",
    "        println(\"I: $(j) Train error: $(Error/size(images, 2)) Train accuracy: $(Correct_cnt/size(images, 2)) Test-Err:: $(test_Error/size(test_images, 2)) Test-Acc:: $(test_Correct_cnt/size(test_images, 2))\")\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Batch Gradient Descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I: 10 Train error: 0.44005504903415854 Train accuracy: 0.447 Test-Err:: 0.5814515684878412 Test-Acc:: 0.6908\n",
      "I: 20 Train error: 0.3821944487765399 Train accuracy: 0.509 Test-Err:: 0.5082174537259565 Test-Acc:: 0.7475\n",
      "I: 30 Train error: 0.3337470974674489 Train accuracy: 0.531 Test-Err:: 0.46339433903535265 Test-Acc:: 0.7747\n",
      "I: 40 Train error: 0.3065819905145321 Train accuracy: 0.563 Test-Err:: 0.43126136608968446 Test-Acc:: 0.7865\n",
      "I: 50 Train error: 0.2838652991377701 Train accuracy: 0.579 Test-Err:: 0.4063259786700258 Test-Acc:: 0.8045\n",
      "I: 60 Train error: 0.2685204515587329 Train accuracy: 0.584 Test-Err:: 0.39027730787014797 Test-Acc:: 0.8141\n",
      "I: 70 Train error: 0.26185819369436936 Train accuracy: 0.585 Test-Err:: 0.37423314258394824 Test-Acc:: 0.8195\n",
      "I: 80 Train error: 0.24019714807604417 Train accuracy: 0.612 Test-Err:: 0.36445480684902587 Test-Acc:: 0.8267\n",
      "I: 90 Train error: 0.2275176534585732 Train accuracy: 0.616 Test-Err:: 0.353904891368789 Test-Acc:: 0.8342\n",
      "I: 100 Train error: 0.22529505852927106 Train accuracy: 0.617 Test-Err:: 0.3495655126718038 Test-Acc:: 0.8397\n",
      "I: 110 Train error: 0.21115677315186077 Train accuracy: 0.631 Test-Err:: 0.34191692396048395 Test-Acc:: 0.8439\n",
      "I: 120 Train error: 0.20762139312136615 Train accuracy: 0.628 Test-Err:: 0.33587268550914884 Test-Acc:: 0.8466\n",
      "I: 130 Train error: 0.20344866977953838 Train accuracy: 0.636 Test-Err:: 0.33273429687381567 Test-Acc:: 0.8473\n",
      "I: 140 Train error: 0.19255611528566674 Train accuracy: 0.647 Test-Err:: 0.32690125481756244 Test-Acc:: 0.8528\n",
      "I: 150 Train error: 0.1948592035399143 Train accuracy: 0.63 Test-Err:: 0.32492704986784 Test-Acc:: 0.8534\n",
      "I: 160 Train error: 0.18600160653290107 Train accuracy: 0.643 Test-Err:: 0.31970813382688595 Test-Acc:: 0.8553\n",
      "I: 170 Train error: 0.1941665256814844 Train accuracy: 0.654 Test-Err:: 0.31639326355639197 Test-Acc:: 0.8556\n",
      "I: 180 Train error: 0.1737731031927613 Train accuracy: 0.653 Test-Err:: 0.30937537766704415 Test-Acc:: 0.8569\n",
      "I: 190 Train error: 0.17001643943309752 Train accuracy: 0.662 Test-Err:: 0.30863058629524626 Test-Acc:: 0.8598\n",
      "I: 200 Train error: 0.17741207200592904 Train accuracy: 0.655 Test-Err:: 0.3096403402579495 Test-Acc:: 0.8592\n",
      "I: 210 Train error: 0.16833655645297455 Train accuracy: 0.66 Test-Err:: 0.30734775224989064 Test-Acc:: 0.8612\n",
      "I: 220 Train error: 0.16699638190282845 Train accuracy: 0.662 Test-Err:: 0.3060050914338937 Test-Acc:: 0.8613\n",
      "I: 230 Train error: 0.1523256889182157 Train accuracy: 0.668 Test-Err:: 0.3050336599157115 Test-Acc:: 0.86\n",
      "I: 240 Train error: 0.16434981146431005 Train accuracy: 0.66 Test-Err:: 0.3023542695802737 Test-Acc:: 0.8656\n",
      "I: 250 Train error: 0.16060003107436369 Train accuracy: 0.66 Test-Err:: 0.30148534649273734 Test-Acc:: 0.8618\n",
      "I: 260 Train error: 0.15909602507394205 Train accuracy: 0.665 Test-Err:: 0.29846944215759913 Test-Acc:: 0.8648\n",
      "I: 270 Train error: 0.16328862414474352 Train accuracy: 0.646 Test-Err:: 0.2996312673426734 Test-Acc:: 0.8648\n",
      "I: 280 Train error: 0.16016463987648774 Train accuracy: 0.664 Test-Err:: 0.3002910238937812 Test-Acc:: 0.8634\n",
      "I: 290 Train error: 0.1490086245254712 Train accuracy: 0.673 Test-Err:: 0.29881714940788345 Test-Acc:: 0.8662\n",
      "I: 300 Train error: 0.15275096249423778 Train accuracy: 0.669 Test-Err:: 0.2995182609772385 Test-Acc:: 0.8656\n"
     ]
    }
   ],
   "source": [
    "using Random\n",
    "Random.seed!(1)\n",
    "\n",
    "relu(x) = x > 0 ? x : 0\n",
    "relu2deriv(x) = x > 0 ? 1 : 0\n",
    "\n",
    "batch_size = 100\n",
    "alpha, iterations = (0.001, 300)\n",
    "pixels_per_image, num_labels, hidden_size = (784, 10, 100)\n",
    "\n",
    "weights_0_1 = 0.2 .* rand(pixels_per_image,hidden_size) .- 0.1\n",
    "weights_1_2 = 0.2 .* rand(hidden_size,num_labels) .- 0.1\n",
    "\n",
    "for j = 1:iterations\n",
    "    Error, Correct_cnt = (0.0, 0)\n",
    "    for i = 1:batch_size:size(images,2)-batch_size\n",
    "        batch_start, batch_end = i, i+batch_size-1\n",
    "        layer_0 = images[:, batch_start:batch_end]\n",
    "        layer_1 = relu.(layer_0' * weights_0_1)\n",
    "        \n",
    "        dropout_mask = bitrand(size(layer_1))\n",
    "        layer_1 .*= (dropout_mask .* 2)\n",
    "        layer_2 = layer_1 * weights_1_2\n",
    "        \n",
    "        Error += sum((labels[:, batch_start:batch_end]' .- layer_2) .^ 2)\n",
    "        \n",
    "        for k=1:batch_size\n",
    "            Correct_cnt += Int(argmax(layer_2[k, :]) == argmax(labels[:, batch_start+k-1]))\n",
    "            layer_2_delta = (labels[:, batch_start:batch_end]' .- layer_2) ./batch_size\n",
    "            layer_1_delta = (layer_2_delta * weights_1_2') .* relu2deriv.(layer_1)\n",
    "\n",
    "            layer_1_delta .*= dropout_mask\n",
    "\n",
    "            weights_1_2 += alpha .* layer_1' * layer_2_delta\n",
    "            weights_0_1 += alpha .* layer_0 * layer_1_delta\n",
    "        end\n",
    "    end\n",
    "        \n",
    "    if (j % 10 == 0)\n",
    "        test_Error, test_Correct_cnt = (0.0, 0)\n",
    "        for i = 1:size(test_images, 2)\n",
    "\n",
    "            layer_0 = test_images[:, i]\n",
    "            layer_1 = relu.(layer_0' * weights_0_1)\n",
    "            layer_2 = layer_1 * weights_1_2\n",
    "\n",
    "            test_Error += sum((test_labels[:, i]' .- layer_2) .^ 2)\n",
    "            test_Correct_cnt += Int(argmax(layer_2[1,:]) == argmax(test_labels[:, i]))\n",
    "        end\n",
    "        println(\"I: $(j) Train error: $(Error/size(images, 2)) Train accuracy: $(Correct_cnt/size(images, 2)) Test-Err:: $(test_Error/size(test_images, 2)) Test-Acc:: $(test_Correct_cnt/size(test_images, 2))\")\n",
    "    end\n",
    "end\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.4.1",
   "language": "julia",
   "name": "julia-1.4"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.4.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}