Chapter6 - Intro to Backpropagation - Building Your First DEEP Neural Network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating a Matrix or Two in Julia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error:0.03999999999999998 Prediction:-0.19999999999999996\n",
      "Error:0.025599999999999973 Prediction:-0.15999999999999992\n",
      "Error:0.01638399999999997 Prediction:-0.1279999999999999\n",
      "Error:0.010485759999999964 Prediction:-0.10239999999999982\n",
      "Error:0.006710886399999962 Prediction:-0.08191999999999977\n",
      "Error:0.004294967295999976 Prediction:-0.06553599999999982\n",
      "Error:0.002748779069439994 Prediction:-0.05242879999999994\n",
      "Error:0.0017592186044416036 Prediction:-0.04194304000000004\n",
      "Error:0.0011258999068426293 Prediction:-0.03355443200000008\n",
      "Error:0.0007205759403792803 Prediction:-0.02684354560000002\n",
      "Error:0.0004611686018427356 Prediction:-0.021474836479999926\n",
      "Error:0.0002951479051793508 Prediction:-0.01717986918399994\n",
      "Error:0.00018889465931478573 Prediction:-0.013743895347199997\n",
      "Error:0.00012089258196146188 Prediction:-0.010995116277759953\n",
      "Error:7.737125245533561e-5 Prediction:-0.008796093022207963\n",
      "Error:4.951760157141604e-5 Prediction:-0.007036874417766459\n",
      "Error:3.169126500570676e-5 Prediction:-0.0056294995342132115\n",
      "Error:2.028240960365233e-5 Prediction:-0.004503599627370569\n",
      "Error:1.298074214633813e-5 Prediction:-0.003602879701896544\n",
      "Error:8.307674973656916e-6 Prediction:-0.002882303761517324\n"
     ]
    }
   ],
   "source": [
    "weights = [0.5,0.48,-0.7]\n",
    "alpha = 0.1\n",
    "\n",
    "streetlights = [ [ 1, 0, 1 ],\n",
    "                           [ 0, 1, 1 ],\n",
    "                           [ 0, 0, 1 ],\n",
    "                           [ 1, 1, 1 ],\n",
    "                           [ 0, 1, 1 ],\n",
    "                           [ 1, 0, 1 ] ] \n",
    "\n",
    "walk_vs_stop = [ 0, 1, 0, 1, 1, 0 ] \n",
    "\n",
    "input = streetlights[1] # [1,0,1]\n",
    "goal_prediction = walk_vs_stop[1] # equals 0... i.e. \"stop\"\n",
    "\n",
    "for iteration =1:20\n",
    "    prediction = input' * weights\n",
    "    error = (goal_prediction - prediction) ^ 2\n",
    "    delta = prediction - goal_prediction\n",
    "    weights = weights - (alpha * (input * delta))\n",
    "\n",
    "    println(\"Error:\",error, \" Prediction:\" ,prediction)\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building Our Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 2, 4, 3]\n",
      "[2, 3, 4, 4]\n",
      "[0.0, 0.5, 1.0, 0.5]\n",
      "[0.5, 1.5, 2.5, 1.5]\n"
     ]
    }
   ],
   "source": [
    "a = [0,1,2,1]\n",
    "b = [2,2,2,3]\n",
    "\n",
    "println(a .* b) #elementwise multiplication\n",
    "println(a .+ b) #elementwise addition\n",
    "println(a .* 0.5) # vector-scalar multiplication\n",
    "println(a .+ 0.5) # vector-scalar addition"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learning the whole dataset!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction:-0.19999999999999996\n",
      "Prediction:-0.19999999999999996\n",
      "Prediction:-0.5599999999999999\n",
      "Prediction:0.6160000000000001\n",
      "Prediction:0.17279999999999995\n",
      "Prediction:0.17552\n",
      "Error:2.6561231104\n",
      "\n",
      "Prediction:0.14041599999999999\n",
      "Prediction:0.3066464\n",
      "Prediction:-0.34513824\n",
      "Prediction:1.006637344\n",
      "Prediction:0.4785034751999999\n",
      "Prediction:0.26700416768\n",
      "Error:0.9628701776715985\n",
      "\n",
      "Prediction:0.213603334144\n",
      "Prediction:0.5347420299776\n",
      "Prediction:-0.26067345110016\n",
      "Prediction:1.1319428845096962\n",
      "Prediction:0.6274723921901568\n",
      "Prediction:0.25433999330650114\n",
      "Error:0.5509165866836797\n",
      "\n",
      "Prediction:0.20347199464520088\n",
      "Prediction:0.6561967149569552\n",
      "Prediction:-0.221948503950995\n",
      "Prediction:1.166258650532124\n",
      "Prediction:0.7139004922542389\n",
      "Prediction:0.21471099528371604\n",
      "Error:0.36445836852222424\n",
      "\n",
      "Prediction:0.17176879622697283\n",
      "Prediction:0.7324724146523222\n",
      "Prediction:-0.19966478845083285\n",
      "Prediction:1.1697769945341199\n",
      "Prediction:0.7719890116601171\n",
      "Prediction:0.17297997428859369\n",
      "Error:0.2516768662079895\n",
      "\n",
      "Prediction:0.13838397943087496\n",
      "Prediction:0.7864548139561468\n",
      "Prediction:-0.1836567869927348\n",
      "Prediction:1.163248019006011\n",
      "Prediction:0.8148799260629888\n",
      "Prediction:0.1362897844408577\n",
      "Error:0.17797575048089034\n",
      "\n",
      "Prediction:0.10903182755268614\n",
      "Prediction:0.8273717796510367\n",
      "Prediction:-0.17037324196481937\n",
      "Prediction:1.1537962739591756\n",
      "Prediction:0.8481754931254761\n",
      "Prediction:0.1059488041691444\n",
      "Error:0.12864460733422164\n",
      "\n",
      "Prediction:0.0847590433353155\n",
      "Prediction:0.859469609749935\n",
      "Prediction:-0.1585508402022421\n",
      "Prediction:1.1438418857156731\n",
      "Prediction:0.8746623946770374\n",
      "Prediction:0.08148074110264475\n",
      "Error:0.09511036950476208\n",
      "\n",
      "Prediction:0.06518459288211581\n",
      "Prediction:0.8850633823431538\n",
      "Prediction:-0.14771905585408038\n",
      "Prediction:1.1341830033853888\n",
      "Prediction:0.8959860107828534\n",
      "Prediction:0.0619780399014222\n",
      "Error:0.07194564247043436\n",
      "\n",
      "Prediction:0.04958243192113776\n",
      "Prediction:0.9056327614440267\n",
      "Prediction:-0.13768337501215525\n",
      "Prediction:1.1250605910610996\n",
      "Prediction:0.9132624284442169\n",
      "Prediction:0.04653264583708144\n",
      "Error:0.05564914990717743\n",
      "\n",
      "Prediction:0.03722611666966513\n",
      "Prediction:0.922234066504699\n",
      "Prediction:-0.12834662236261596\n",
      "Prediction:1.116526024487899\n",
      "Prediction:0.9273167105424409\n",
      "Prediction:0.03435527296969987\n",
      "Error:0.04394763937673939\n",
      "\n",
      "Prediction:0.027484218375759886\n",
      "Prediction:0.9356694192994068\n",
      "Prediction:-0.11964712469387503\n",
      "Prediction:1.1085678053734553\n",
      "Prediction:0.9387866868342218\n",
      "Prediction:0.024792915481941458\n",
      "Error:0.035357967050948465\n",
      "\n",
      "Prediction:0.019834332385553155\n",
      "Prediction:0.946566624680628\n",
      "Prediction:-0.11153724870006754\n",
      "Prediction:1.1011550767549563\n",
      "Prediction:0.948176009263518\n",
      "Prediction:0.017315912033043404\n",
      "Error:0.02890700056547436\n",
      "\n",
      "Prediction:0.013852729626434732\n",
      "Prediction:0.9554239432448665\n",
      "Prediction:-0.10397589092234266\n",
      "Prediction:1.0942524239871314\n",
      "Prediction:0.9558862588907013\n",
      "Prediction:0.011498267782398985\n",
      "Error:0.023951660591138853\n",
      "\n",
      "Prediction:0.009198614225919194\n",
      "Prediction:0.9626393189117293\n",
      "Prediction:-0.09692579020989642\n",
      "Prediction:1.087824783849832\n",
      "Prediction:0.9622390773804066\n",
      "Prediction:0.006998674002545002\n",
      "Error:0.020063105176016144\n",
      "\n",
      "Prediction:0.005598939202035996\n",
      "Prediction:0.9685315005838672\n",
      "Prediction:-0.09035250869077546\n",
      "Prediction:1.0818389613301889\n",
      "Prediction:0.9674926590701334\n",
      "Prediction:0.003544193999268516\n",
      "Error:0.016952094519447087\n",
      "\n",
      "Prediction:0.0028353551994148157\n",
      "Prediction:0.9733561723362383\n",
      "Prediction:-0.0842239920152223\n",
      "Prediction:1.0762639960116431\n",
      "Prediction:0.9718545378681842\n",
      "Prediction:0.0009168131382832068\n",
      "Error:0.014420818295271236\n",
      "\n",
      "Prediction:0.0007334505106265654\n",
      "Prediction:0.9773186039296565\n",
      "Prediction:-0.07851033295953944\n",
      "Prediction:1.0710711494147542\n",
      "Prediction:0.9754916865567282\n",
      "Prediction:-0.0010574652271341245\n",
      "Error:0.012331739998443648\n",
      "\n",
      "Prediction:-0.0008459721817072885\n",
      "Prediction:0.9805836929862668\n",
      "Prediction:-0.07318360881847627\n",
      "Prediction:1.066233777045345\n",
      "Prediction:0.9785385598617921\n",
      "Prediction:-0.0025173975573930946\n",
      "Error:0.010587393171639842\n",
      "\n",
      "Prediction:-0.002013918045914484\n",
      "Prediction:0.9832839794497644\n",
      "Prediction:-0.06821774801198803\n",
      "Prediction:1.0617271739912904\n",
      "Prediction:0.9811035235627523\n",
      "Prediction:-0.0035735447350425317\n",
      "Error:0.009117233405426495\n",
      "\n",
      "Prediction:-0.002858835788034024\n",
      "Prediction:0.9855260569025094\n",
      "Prediction:-0.06358841060413677\n",
      "Prediction:1.05752842286588\n",
      "Prediction:0.9832740020092452\n",
      "Prediction:-0.004313918034364962\n",
      "Error:0.00786904226904208\n",
      "\n",
      "Prediction:-0.003451134427491974\n",
      "Prediction:0.9873957068535818\n",
      "Prediction:-0.059272877470408075\n",
      "Prediction:1.0536162524729626\n",
      "Prediction:0.9851206027353137\n",
      "Prediction:-0.004808501248434842\n",
      "Error:0.006803273214640502\n",
      "\n",
      "Prediction:-0.0038468009987478735\n",
      "Prediction:0.9889620124129692\n",
      "Prediction:-0.05524994626077355\n",
      "Prediction:1.049970908776931\n",
      "Prediction:0.9867004228010665\n",
      "Prediction:-0.005112871449710697\n",
      "Error:0.005889303541837786\n",
      "\n",
      "Prediction:-0.004090297159768559\n",
      "Prediction:0.9902806551018011\n",
      "Prediction:-0.051499833441728114\n",
      "Prediction:1.0465740376293469\n",
      "Prediction:0.9880596998997442\n",
      "Prediction:-0.0052710974096659285\n",
      "Error:0.0051029252561172675\n",
      "\n",
      "Prediction:-0.004216877927732746\n",
      "Prediction:0.9913965574535352\n",
      "Prediction:-0.048004082062078055\n",
      "Prediction:1.043408578143574\n",
      "Prediction:0.9892359385403211\n",
      "Prediction:-0.005318059364078823\n",
      "Error:0.004424644608684828\n",
      "\n",
      "Prediction:-0.0042544474912630525\n",
      "Prediction:0.992346001517791\n",
      "Prediction:-0.044745474990504665\n",
      "Prediction:1.0404586655589985\n",
      "Prediction:0.9902596156014837\n",
      "Prediction:-0.005281305317687134\n",
      "Error:0.0038385124412518303\n",
      "\n",
      "Prediction:-0.0042250442541497055\n",
      "Prediction:0.9931583274383705\n",
      "Prediction:-0.041707953394155776\n",
      "Prediction:1.0377095425371112\n",
      "Prediction:0.9911555487826897\n",
      "Prediction:-0.005182536193432452\n",
      "Error:0.0033313054558089675\n",
      "\n",
      "Prediction:-0.004146028954745959\n",
      "Prediction:0.9938572955409696\n",
      "Prediction:-0.03887654022599941\n",
      "Prediction:1.0351474779634813\n",
      "Prediction:0.9919439948626794\n",
      "Prediction:-0.00503879377425797\n",
      "Error:0.0028919416227737734\n",
      "\n",
      "Prediction:-0.004031035019406375\n",
      "Prediction:0.9944621787695098\n",
      "Prediction:-0.03623726848360008\n",
      "Prediction:1.032759692455092\n",
      "Prediction:0.9926415313729495\n",
      "Prediction:-0.004863410672429416\n",
      "Error:0.002511053608117256\n",
      "\n",
      "Prediction:-0.003890728537943533\n",
      "Prediction:0.9949886390193969\n",
      "Prediction:-0.03377711399894662\n",
      "Prediction:1.0305342898820642\n",
      "Prediction:0.9932617646389992\n",
      "Prediction:-0.004666769772712614\n",
      "Error:0.0021806703520253884\n",
      "\n",
      "Prediction:-0.003733415818170091\n",
      "Prediction:0.9954494302702878\n",
      "Prediction:-0.03148393251909879\n",
      "Prediction:1.0284601943056741\n",
      "Prediction:0.9938158986070053\n",
      "Prediction:-0.004456911151490314\n",
      "Error:0.0018939739123713475\n",
      "\n",
      "Prediction:-0.003565528921192253\n",
      "Prediction:0.9958549628928723\n",
      "Prediction:-0.029346400840475826\n",
      "Prediction:1.0265270918125804\n",
      "Prediction:0.9943131920358295\n",
      "Prediction:-0.004240016908292479\n",
      "Error:0.0016451096996342332\n",
      "\n",
      "Prediction:-0.0033920135266339822\n",
      "Prediction:0.9962137566721563\n",
      "Prediction:-0.02735396176499221\n",
      "Prediction:1.0247253767906936\n",
      "Prediction:0.9947613261560856\n",
      "Prediction:-0.004020798285770878\n",
      "Error:0.0014290353984827077\n",
      "\n",
      "Prediction:-0.003216638628616701\n",
      "Prediction:0.9965328046163073\n",
      "Prediction:-0.025496772653362886\n",
      "Prediction:1.0230461022472208\n",
      "Prediction:0.9951667005089379\n",
      "Prediction:-0.0038028045995257484\n",
      "Error:0.0012413985592149145\n",
      "\n",
      "Prediction:-0.003042243679620596\n",
      "Prediction:0.996817865235065\n",
      "Prediction:-0.023765657359234325\n",
      "Prediction:1.0214809338160067\n",
      "Prediction:0.995534671160774\n",
      "Prediction:-0.0035886696105582767\n",
      "Error:0.0010784359268087556\n",
      "\n",
      "Prediction:-0.0028709356884466207\n",
      "Prediction:0.9970736974585198\n",
      "Prediction:-0.022152061336940452\n",
      "Prediction:1.0200221071408409\n",
      "Prediction:0.9958697426723416\n",
      "Prediction:-0.0033803078583175654\n",
      "Error:0.0009368896209360312\n",
      "\n",
      "Prediction:-0.0027042462866540516\n",
      "Prediction:0.9973042495523706\n",
      "Prediction:-0.02064800972530455\n",
      "Prediction:1.018662388355171\n",
      "Prediction:0.9961757229433927\n",
      "Prediction:-0.0031790709774033414\n",
      "Error:0.0008139366504753339\n",
      "\n",
      "Prediction:-0.002543256781922673\n",
      "Prediction:0.9975128111306469\n",
      "Prediction:-0.019246068219762574\n",
      "Prediction:1.0173950374076535\n",
      "Prediction:0.9964558482449631\n",
      "Prediction:-0.0029858720226535913\n",
      "Error:0.0007071291752624441\n",
      "\n",
      "Prediction:-0.002388697618122871\n",
      "Prediction:0.9977021355600483\n",
      "Prediction:-0.01793930655497516\n",
      "Prediction:1.0162137740080082\n",
      "Prediction:0.9967128843019345\n",
      "Prediction:-0.0028012842268006904\n",
      "Error:0.0006143435674831474\n",
      "\n",
      "Prediction:-0.0022410273814405524\n",
      "Prediction:0.9978745386023716\n",
      "Prediction:-0.016721264429884947\n",
      "Prediction:1.0151127459893812\n",
      "Prediction:0.9969492081270097\n",
      "Prediction:-0.0026256193329783125\n",
      "Error:0.00053373677328488\n",
      "\n"
     ]
    }
   ],
   "source": [
    "weights = [0.5,0.48,-0.7]\n",
    "alpha = 0.1\n",
    "\n",
    "streetlights =  [[ 1, 0, 1 ],\n",
    "                          [ 0, 1, 1 ],\n",
    "                          [ 0, 0, 1 ],\n",
    "                          [ 1, 1, 1 ],\n",
    "                          [ 0, 1, 1 ],\n",
    "                          [ 1, 0, 1 ] ] \n",
    "\n",
    "walk_vs_stop = [ 0, 1, 0, 1, 1, 0 ]\n",
    "\n",
    "input = streetlights[1] # [1,0,1]\n",
    "goal_prediction = walk_vs_stop[1] # equals 0... i.e. \"stop\"\n",
    "\n",
    "for iteration =1:40\n",
    "    error_for_all_lights = 0\n",
    "    for row_index =1:length(walk_vs_stop)\n",
    "        input = streetlights[row_index]\n",
    "        goal_prediction = walk_vs_stop[row_index]\n",
    "        prediction = (input' * weights)\n",
    "        \n",
    "        error = (goal_prediction - prediction) ^ 2\n",
    "        error_for_all_lights += error\n",
    "        \n",
    "        delta = prediction - goal_prediction\n",
    "        weights = weights - (alpha * (input * delta))\n",
    "        println(\"Prediction:\" ,prediction)\n",
    "    end\n",
    "    println(\"Error:\" ,error_for_all_lights,\"\\n\")\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Our First \"Deep\" Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1×1 Array{Any,2}:\n",
       " -0.6706524921531596"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "using Random\n",
    "\n",
    "Random.seed!(1)\n",
    "\n",
    "relu(x) = x > 0 ? x : 0\n",
    "\n",
    "alpha = 0.2\n",
    "hidden_size = 4\n",
    "\n",
    "streetlights = [1 0 1;\n",
    "                0 1 1;\n",
    "                0 0 1;\n",
    "                1 1 1;] \n",
    "\n",
    "walk_vs_stop = [ 1; 0; 0; 0]\n",
    "weights_0_1 = 2 .* rand(3,hidden_size) .- 1\n",
    "weights_1_2 = 2 .* rand(hidden_size,1) .- 1\n",
    "\n",
    "layer_0 = streetlights[1,:]\n",
    "layer_1 = relu.(layer_0' * weights_0_1)\n",
    "layer_2 = layer_1 * weights_1_2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Backpropagation in Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error:0.9291584161474236\n",
      "Error:0.2689477130783834\n",
      "Error:0.02684840715626828\n",
      "Error:0.003409539705999161\n",
      "Error:0.00033592786527769387\n",
      "Error:2.9218547775875146e-5\n"
     ]
    }
   ],
   "source": [
    "using Random\n",
    "Random.seed!(1)\n",
    "\n",
    "relu(x) = x > 0 ? x : 0\n",
    "relu2deriv(output) = output>0 ? 1 : 0\n",
    "\n",
    "alpha = 0.2\n",
    "hidden_size = 4\n",
    "\n",
    "weights_0_1 = 2 .* rand(3,hidden_size) .- 1\n",
    "weights_1_2 = 2 .* rand(hidden_size,1) .- 1\n",
    "\n",
    "for iteration = 1:60\n",
    "   layer_2_error = 0\n",
    "   for i = 1:size(streetlights, 1)\n",
    "      layer_0 = streetlights[i,:]'\n",
    "      layer_1 = relu.(layer_0 * weights_0_1)\n",
    "      layer_2 = layer_1 * weights_1_2\n",
    "      layer_2_error += sum((layer_2 .- walk_vs_stop[i]).^ 2)\n",
    "\n",
    "      layer_2_delta = (walk_vs_stop[i] .- layer_2)\n",
    "      layer_1_delta= (layer_2_delta * weights_1_2') .* relu2deriv.(layer_1)\n",
    "\n",
    "      weights_1_2 += alpha .* (layer_1' * layer_2_delta)\n",
    "      weights_0_1 += alpha .* (layer_0' * layer_1_delta)\n",
    "    end\n",
    "        \n",
    "\n",
    "   if (iteration % 10 == 9)\n",
    "      println(\"Error:\", layer_2_error)\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# One Iteration of Backpropagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3×3 Array{Float64,2}:\n",
       " -0.527933  -0.984181    0.877387\n",
       " -0.306966  -0.0227743   0.999809\n",
       " -0.374586  -0.578064   -0.523121"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "using Random\n",
    "Random.seed!(1)\n",
    "\n",
    "relu(x) = x > 0 ? x : 0\n",
    "relu2deriv(output) = output>0 ? 1 : 0\n",
    "\n",
    "lights = [1 0 1;\n",
    "                0 1 1;\n",
    "                0 0 1;\n",
    "                1 1 1;] \n",
    "\n",
    "walk_stop = [ 1; 0; 0; 0]\n",
    "\n",
    "\n",
    "alpha = 0.2\n",
    "hidden_size = 3\n",
    "\n",
    "weights_0_1 = 2 .* rand(3,hidden_size) .- 1\n",
    "weights_1_2 = 2 .* rand(hidden_size,1) .- 1\n",
    "\n",
    "layer_0 = lights[1,:]'\n",
    "layer_1 = layer_0 * weights_0_1\n",
    "layer_1 = relu.(layer_1)\n",
    "layer_2 = layer_1 * weights_1_2\n",
    "\n",
    "error = (layer_2 .- walk_stop[1]) .^2\n",
    "\n",
    "layer_2_delta=(layer_2 .- walk_stop[1])\n",
    "layer_1_delta=layer_2_delta * weights_1_2'\n",
    "\n",
    "layer_1_delta .*= relu2deriv.(layer_1)\n",
    "\n",
    "weight_delta_1_2 = layer_1' * layer_2_delta\n",
    "weight_delta_0_1 = layer_0' * layer_1_delta\n",
    "\n",
    "weights_1_2 -= alpha * weight_delta_1_2\n",
    "weights_0_1 -= alpha * weight_delta_0_1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Putting it all Together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error:0.9291584161474236\n",
      "Error:0.2689477130783834\n",
      "Error:0.026848407156268274\n",
      "Error:0.003409539705999169\n",
      "Error:0.0003359278652776877\n",
      "Error:2.9218547775875068e-5\n"
     ]
    }
   ],
   "source": [
    "using Random\n",
    "Random.seed!(1)\n",
    "\n",
    "relu(x) = x > 0 ? x : 0\n",
    "relu2deriv(output) = output>0 ? 1 : 0\n",
    "\n",
    "streetlights = [1 0 1;\n",
    "                0 1 1;\n",
    "                0 0 1;\n",
    "                1 1 1;] \n",
    "\n",
    "walk_vs_stop = [ 1; 0; 0; 0]\n",
    "\n",
    "alpha = 0.2\n",
    "hidden_size = 4\n",
    "\n",
    "weights_0_1 = 2 .* rand(3,hidden_size) .- 1\n",
    "weights_1_2 = 2 .* rand(hidden_size,1) .- 1\n",
    "\n",
    "for iteration = 1:60\n",
    "   layer_2_error = 0\n",
    "   for i = 1:size(streetlights)[1]\n",
    "      layer_0 = streetlights[i,:]'\n",
    "      layer_1 = relu.(layer_0 * weights_0_1)\n",
    "      layer_2 = layer_1 * weights_1_2\n",
    "\n",
    "      layer_2_error += sum((layer_2 .- walk_vs_stop[i]) .^ 2)\n",
    "\n",
    "      layer_2_delta = (layer_2 .- walk_vs_stop[i])\n",
    "      layer_1_delta=(layer_2_delta * weights_1_2') .* relu2deriv.(layer_1)\n",
    "\n",
    "      weights_1_2 -= alpha * layer_1' * layer_2_delta\n",
    "      weights_0_1 -= alpha * layer_0' * layer_1_delta\n",
    "    end\n",
    "\n",
    "   if(iteration % 10 == 9)\n",
    "      println(\"Error:\",layer_2_error)\n",
    "   end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.4.1",
   "language": "julia",
   "name": "julia-1.4"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.4.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}