train_ONN_CNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "deluxe-connection",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting weights for Conv2d(1, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n",
      "Setting weights for Conv2d(32, 64, kernel_size=(5, 5), stride=(1, 1), bias=False)\n",
      "Setting weights for Linear(in_features=1024, out_features=128, bias=False)\n",
      "Setting weights for Linear(in_features=128, out_features=10, bias=False)\n",
      "Prepared classifier with network:\n",
      "\n",
      " ConvNet(\n",
      "  (layers_conv): ModuleList(\n",
      "    (0): Sequential(\n",
      "      (0): Conv2d(1, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n",
      "      (1): SatAbsNL(encoding=Encoding.AMPLITUDE, gradient=Gradient.APPROXIMATE, OD=50, I_sat=1)\n",
      "      (2): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
      "    )\n",
      "    (1): Sequential(\n",
      "      (0): Conv2d(32, 64, kernel_size=(5, 5), stride=(1, 1), bias=False)\n",
      "      (1): SatAbsNL(encoding=Encoding.AMPLITUDE, gradient=Gradient.APPROXIMATE, OD=50, I_sat=1)\n",
      "      (2): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
      "    )\n",
      "  )\n",
      "  (dropout): Dropout(p=0.4, inplace=False)\n",
      "  (layers_fc): ModuleList(\n",
      "    (0): Sequential(\n",
      "      (0): Linear(in_features=1024, out_features=128, bias=False)\n",
      "      (1): SatAbsNL(encoding=Encoding.AMPLITUDE, gradient=Gradient.APPROXIMATE, OD=50, I_sat=1)\n",
      "    )\n",
      "    (1): Sequential(\n",
      "      (0): Linear(in_features=128, out_features=10, bias=False)\n",
      "    )\n",
      "  )\n",
      ")\n",
      "dir already exists:  cnn/ONN\\seed0\n",
      "Size of input to first linear layer is torch.Size([64, 1024])\n",
      "Train Epoch: 1 [0/60000 (0%)]\tLoss: 4.008453\n",
      "Train Epoch: 1 [1600/60000 (3%)]\tLoss: 0.623479\n",
      "Train Epoch: 1 [3200/60000 (5%)]\tLoss: 0.227588\n",
      "Train Epoch: 1 [4800/60000 (8%)]\tLoss: 0.140693\n",
      "Train Epoch: 1 [6400/60000 (11%)]\tLoss: 0.127189\n",
      "Train Epoch: 1 [8000/60000 (13%)]\tLoss: 0.105860\n",
      "Train Epoch: 1 [9600/60000 (16%)]\tLoss: 0.100375\n",
      "Train Epoch: 1 [11200/60000 (19%)]\tLoss: 0.093921\n",
      "Train Epoch: 1 [12800/60000 (21%)]\tLoss: 0.070018\n",
      "Train Epoch: 1 [14400/60000 (24%)]\tLoss: 0.078810\n",
      "Train Epoch: 1 [16000/60000 (27%)]\tLoss: 0.080630\n",
      "Train Epoch: 1 [17600/60000 (29%)]\tLoss: 0.058761\n",
      "Train Epoch: 1 [19200/60000 (32%)]\tLoss: 0.070694\n",
      "Train Epoch: 1 [20800/60000 (35%)]\tLoss: 0.058153\n",
      "Train Epoch: 1 [22400/60000 (37%)]\tLoss: 0.065823\n",
      "Train Epoch: 1 [24000/60000 (40%)]\tLoss: 0.056488\n",
      "Train Epoch: 1 [25600/60000 (43%)]\tLoss: 0.058062\n",
      "Train Epoch: 1 [27200/60000 (45%)]\tLoss: 0.054208\n",
      "Train Epoch: 1 [28800/60000 (48%)]\tLoss: 0.053606\n",
      "Train Epoch: 1 [30400/60000 (51%)]\tLoss: 0.052786\n",
      "Train Epoch: 1 [32000/60000 (53%)]\tLoss: 0.050717\n",
      "Train Epoch: 1 [33600/60000 (56%)]\tLoss: 0.044071\n",
      "Train Epoch: 1 [35200/60000 (59%)]\tLoss: 0.055125\n",
      "Train Epoch: 1 [36800/60000 (61%)]\tLoss: 0.044670\n",
      "Train Epoch: 1 [38400/60000 (64%)]\tLoss: 0.055291\n",
      "Train Epoch: 1 [40000/60000 (67%)]\tLoss: 0.047266\n",
      "Train Epoch: 1 [41600/60000 (69%)]\tLoss: 0.047706\n",
      "Train Epoch: 1 [43200/60000 (72%)]\tLoss: 0.045127\n",
      "Train Epoch: 1 [44800/60000 (75%)]\tLoss: 0.050618\n",
      "Train Epoch: 1 [46400/60000 (77%)]\tLoss: 0.037048\n",
      "Train Epoch: 1 [48000/60000 (80%)]\tLoss: 0.042365\n",
      "Train Epoch: 1 [49600/60000 (83%)]\tLoss: 0.043987\n",
      "Train Epoch: 1 [51200/60000 (85%)]\tLoss: 0.036008\n",
      "Train Epoch: 1 [52800/60000 (88%)]\tLoss: 0.042717\n",
      "Train Epoch: 1 [54400/60000 (91%)]\tLoss: 0.040782\n",
      "Train Epoch: 1 [56000/60000 (93%)]\tLoss: 0.038924\n",
      "Train Epoch: 1 [57600/60000 (96%)]\tLoss: 0.041330\n",
      "Train Epoch: 1 [59200/60000 (99%)]\tLoss: 0.038705\n",
      "\n",
      "Test set: Avg. loss: 0.3378, Accuracy: 4404/5000 (88.08%)\n",
      "\n",
      "Train Epoch: 2 [0/60000 (0%)]\tLoss: 0.039841\n",
      "Train Epoch: 2 [1600/60000 (3%)]\tLoss: 0.038202\n",
      "Train Epoch: 2 [3200/60000 (5%)]\tLoss: 0.035853\n",
      "Train Epoch: 2 [4800/60000 (8%)]\tLoss: 0.036126\n",
      "Train Epoch: 2 [6400/60000 (11%)]\tLoss: 0.034938\n",
      "Train Epoch: 2 [8000/60000 (13%)]\tLoss: 0.042053\n",
      "Train Epoch: 2 [9600/60000 (16%)]\tLoss: 0.047658\n",
      "Train Epoch: 2 [11200/60000 (19%)]\tLoss: 0.037041\n",
      "Train Epoch: 2 [12800/60000 (21%)]\tLoss: 0.032398\n",
      "Train Epoch: 2 [14400/60000 (24%)]\tLoss: 0.038856\n",
      "Train Epoch: 2 [16000/60000 (27%)]\tLoss: 0.041290\n",
      "Train Epoch: 2 [17600/60000 (29%)]\tLoss: 0.033546\n",
      "Train Epoch: 2 [19200/60000 (32%)]\tLoss: 0.035915\n",
      "Train Epoch: 2 [20800/60000 (35%)]\tLoss: 0.034052\n",
      "Train Epoch: 2 [22400/60000 (37%)]\tLoss: 0.042246\n",
      "Train Epoch: 2 [24000/60000 (40%)]\tLoss: 0.032878\n",
      "Train Epoch: 2 [25600/60000 (43%)]\tLoss: 0.036289\n",
      "Train Epoch: 2 [27200/60000 (45%)]\tLoss: 0.038135\n",
      "Train Epoch: 2 [28800/60000 (48%)]\tLoss: 0.031409\n",
      "Train Epoch: 2 [30400/60000 (51%)]\tLoss: 0.031557\n",
      "Train Epoch: 2 [32000/60000 (53%)]\tLoss: 0.035910\n",
      "Train Epoch: 2 [33600/60000 (56%)]\tLoss: 0.035194\n",
      "Train Epoch: 2 [35200/60000 (59%)]\tLoss: 0.036450\n",
      "Train Epoch: 2 [36800/60000 (61%)]\tLoss: 0.031418\n",
      "Train Epoch: 2 [38400/60000 (64%)]\tLoss: 0.029191\n",
      "Train Epoch: 2 [40000/60000 (67%)]\tLoss: 0.039542\n",
      "Train Epoch: 2 [41600/60000 (69%)]\tLoss: 0.035199\n",
      "Train Epoch: 2 [43200/60000 (72%)]\tLoss: 0.031270\n",
      "Train Epoch: 2 [44800/60000 (75%)]\tLoss: 0.032210\n",
      "Train Epoch: 2 [46400/60000 (77%)]\tLoss: 0.031132\n",
      "Train Epoch: 2 [48000/60000 (80%)]\tLoss: 0.035699\n",
      "Train Epoch: 2 [49600/60000 (83%)]\tLoss: 0.033596\n",
      "Train Epoch: 2 [51200/60000 (85%)]\tLoss: 0.025205\n",
      "Train Epoch: 2 [52800/60000 (88%)]\tLoss: 0.031431\n",
      "Train Epoch: 2 [54400/60000 (91%)]\tLoss: 0.034313\n",
      "Train Epoch: 2 [56000/60000 (93%)]\tLoss: 0.027207\n",
      "Train Epoch: 2 [57600/60000 (96%)]\tLoss: 0.036770\n",
      "Train Epoch: 2 [59200/60000 (99%)]\tLoss: 0.038454\n",
      "\n",
      "Test set: Avg. loss: 0.2512, Accuracy: 4609/5000 (92.18%)\n",
      "\n",
      "Train Epoch: 3 [0/60000 (0%)]\tLoss: 0.032329\n",
      "Train Epoch: 3 [1600/60000 (3%)]\tLoss: 0.025263\n",
      "Train Epoch: 3 [3200/60000 (5%)]\tLoss: 0.031263\n",
      "Train Epoch: 3 [4800/60000 (8%)]\tLoss: 0.033006\n",
      "Train Epoch: 3 [6400/60000 (11%)]\tLoss: 0.032973\n",
      "Train Epoch: 3 [8000/60000 (13%)]\tLoss: 0.033281\n",
      "Train Epoch: 3 [9600/60000 (16%)]\tLoss: 0.021258\n",
      "Train Epoch: 3 [11200/60000 (19%)]\tLoss: 0.026904\n",
      "Train Epoch: 3 [12800/60000 (21%)]\tLoss: 0.029506\n",
      "Train Epoch: 3 [14400/60000 (24%)]\tLoss: 0.027688\n",
      "Train Epoch: 3 [16000/60000 (27%)]\tLoss: 0.029964\n",
      "Train Epoch: 3 [17600/60000 (29%)]\tLoss: 0.026145\n",
      "Train Epoch: 3 [19200/60000 (32%)]\tLoss: 0.024275\n",
      "Train Epoch: 3 [20800/60000 (35%)]\tLoss: 0.027040\n",
      "Train Epoch: 3 [22400/60000 (37%)]\tLoss: 0.026944\n",
      "Train Epoch: 3 [24000/60000 (40%)]\tLoss: 0.027609\n",
      "Train Epoch: 3 [25600/60000 (43%)]\tLoss: 0.026205\n",
      "Train Epoch: 3 [27200/60000 (45%)]\tLoss: 0.032588\n",
      "Train Epoch: 3 [28800/60000 (48%)]\tLoss: 0.027122\n",
      "Train Epoch: 3 [30400/60000 (51%)]\tLoss: 0.029842\n",
      "Train Epoch: 3 [32000/60000 (53%)]\tLoss: 0.020958\n",
      "Train Epoch: 3 [33600/60000 (56%)]\tLoss: 0.030836\n",
      "Train Epoch: 3 [35200/60000 (59%)]\tLoss: 0.024738\n",
      "Train Epoch: 3 [36800/60000 (61%)]\tLoss: 0.020556\n",
      "Train Epoch: 3 [38400/60000 (64%)]\tLoss: 0.024739\n",
      "Train Epoch: 3 [40000/60000 (67%)]\tLoss: 0.022050\n",
      "Train Epoch: 3 [41600/60000 (69%)]\tLoss: 0.026591\n",
      "Train Epoch: 3 [43200/60000 (72%)]\tLoss: 0.030456\n",
      "Train Epoch: 3 [44800/60000 (75%)]\tLoss: 0.028559\n",
      "Train Epoch: 3 [46400/60000 (77%)]\tLoss: 0.022778\n",
      "Train Epoch: 3 [48000/60000 (80%)]\tLoss: 0.019264\n",
      "Train Epoch: 3 [49600/60000 (83%)]\tLoss: 0.021395\n",
      "Train Epoch: 3 [51200/60000 (85%)]\tLoss: 0.027853\n",
      "Train Epoch: 3 [52800/60000 (88%)]\tLoss: 0.021027\n",
      "Train Epoch: 3 [54400/60000 (91%)]\tLoss: 0.018257\n",
      "Train Epoch: 3 [56000/60000 (93%)]\tLoss: 0.022584\n",
      "Train Epoch: 3 [57600/60000 (96%)]\tLoss: 0.027267\n",
      "Train Epoch: 3 [59200/60000 (99%)]\tLoss: 0.025715\n",
      "\n",
      "Test set: Avg. loss: 0.1880, Accuracy: 4712/5000 (94.24%)\n",
      "\n",
      "Train Epoch: 4 [0/60000 (0%)]\tLoss: 0.026614\n",
      "Train Epoch: 4 [1600/60000 (3%)]\tLoss: 0.021864\n",
      "Train Epoch: 4 [3200/60000 (5%)]\tLoss: 0.024145\n",
      "Train Epoch: 4 [4800/60000 (8%)]\tLoss: 0.022906\n",
      "Train Epoch: 4 [6400/60000 (11%)]\tLoss: 0.022345\n",
      "Train Epoch: 4 [8000/60000 (13%)]\tLoss: 0.024407\n",
      "Train Epoch: 4 [9600/60000 (16%)]\tLoss: 0.023343\n",
      "Train Epoch: 4 [11200/60000 (19%)]\tLoss: 0.023025\n",
      "Train Epoch: 4 [12800/60000 (21%)]\tLoss: 0.019843\n",
      "Train Epoch: 4 [14400/60000 (24%)]\tLoss: 0.017935\n",
      "Train Epoch: 4 [16000/60000 (27%)]\tLoss: 0.025236\n",
      "Train Epoch: 4 [17600/60000 (29%)]\tLoss: 0.024230\n",
      "Train Epoch: 4 [19200/60000 (32%)]\tLoss: 0.022897\n",
      "Train Epoch: 4 [20800/60000 (35%)]\tLoss: 0.016236\n",
      "Train Epoch: 4 [22400/60000 (37%)]\tLoss: 0.021539\n",
      "Train Epoch: 4 [24000/60000 (40%)]\tLoss: 0.020934\n",
      "Train Epoch: 4 [25600/60000 (43%)]\tLoss: 0.016252\n",
      "Train Epoch: 4 [27200/60000 (45%)]\tLoss: 0.021245\n",
      "Train Epoch: 4 [28800/60000 (48%)]\tLoss: 0.018553\n",
      "Train Epoch: 4 [30400/60000 (51%)]\tLoss: 0.023825\n",
      "Train Epoch: 4 [32000/60000 (53%)]\tLoss: 0.018646\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 4 [33600/60000 (56%)]\tLoss: 0.020545\n",
      "Train Epoch: 4 [35200/60000 (59%)]\tLoss: 0.020060\n",
      "Train Epoch: 4 [36800/60000 (61%)]\tLoss: 0.022768\n",
      "Train Epoch: 4 [38400/60000 (64%)]\tLoss: 0.020752\n",
      "Train Epoch: 4 [40000/60000 (67%)]\tLoss: 0.021938\n",
      "Train Epoch: 4 [41600/60000 (69%)]\tLoss: 0.017222\n",
      "Train Epoch: 4 [43200/60000 (72%)]\tLoss: 0.025602\n",
      "Train Epoch: 4 [44800/60000 (75%)]\tLoss: 0.017366\n",
      "Train Epoch: 4 [46400/60000 (77%)]\tLoss: 0.023110\n",
      "Train Epoch: 4 [48000/60000 (80%)]\tLoss: 0.017368\n",
      "Train Epoch: 4 [49600/60000 (83%)]\tLoss: 0.015987\n",
      "Train Epoch: 4 [51200/60000 (85%)]\tLoss: 0.017864\n",
      "Train Epoch: 4 [52800/60000 (88%)]\tLoss: 0.016239\n",
      "Train Epoch: 4 [54400/60000 (91%)]\tLoss: 0.019042\n",
      "Train Epoch: 4 [56000/60000 (93%)]\tLoss: 0.021173\n",
      "Train Epoch: 4 [57600/60000 (96%)]\tLoss: 0.019131\n",
      "Train Epoch: 4 [59200/60000 (99%)]\tLoss: 0.020759\n",
      "\n",
      "Test set: Avg. loss: 0.1443, Accuracy: 4769/5000 (95.38%)\n",
      "\n",
      "Train Epoch: 5 [0/60000 (0%)]\tLoss: 0.016285\n",
      "Train Epoch: 5 [1600/60000 (3%)]\tLoss: 0.018721\n",
      "Train Epoch: 5 [3200/60000 (5%)]\tLoss: 0.014401\n",
      "Train Epoch: 5 [4800/60000 (8%)]\tLoss: 0.018880\n",
      "Train Epoch: 5 [6400/60000 (11%)]\tLoss: 0.018646\n",
      "Train Epoch: 5 [8000/60000 (13%)]\tLoss: 0.018501\n",
      "Train Epoch: 5 [9600/60000 (16%)]\tLoss: 0.019008\n",
      "Train Epoch: 5 [11200/60000 (19%)]\tLoss: 0.014319\n",
      "Train Epoch: 5 [12800/60000 (21%)]\tLoss: 0.018785\n",
      "Train Epoch: 5 [14400/60000 (24%)]\tLoss: 0.016668\n",
      "Train Epoch: 5 [16000/60000 (27%)]\tLoss: 0.014963\n",
      "Train Epoch: 5 [17600/60000 (29%)]\tLoss: 0.018603\n",
      "Train Epoch: 5 [19200/60000 (32%)]\tLoss: 0.018771\n",
      "Train Epoch: 5 [20800/60000 (35%)]\tLoss: 0.012206\n",
      "Train Epoch: 5 [22400/60000 (37%)]\tLoss: 0.014743\n",
      "Train Epoch: 5 [24000/60000 (40%)]\tLoss: 0.013325\n",
      "Train Epoch: 5 [25600/60000 (43%)]\tLoss: 0.015542\n",
      "Train Epoch: 5 [27200/60000 (45%)]\tLoss: 0.018471\n",
      "Train Epoch: 5 [28800/60000 (48%)]\tLoss: 0.014500\n",
      "Train Epoch: 5 [30400/60000 (51%)]\tLoss: 0.015579\n",
      "Train Epoch: 5 [32000/60000 (53%)]\tLoss: 0.017978\n",
      "Train Epoch: 5 [33600/60000 (56%)]\tLoss: 0.016384\n",
      "Train Epoch: 5 [35200/60000 (59%)]\tLoss: 0.010969\n",
      "Train Epoch: 5 [36800/60000 (61%)]\tLoss: 0.013790\n",
      "Train Epoch: 5 [38400/60000 (64%)]\tLoss: 0.018196\n",
      "Train Epoch: 5 [40000/60000 (67%)]\tLoss: 0.009892\n",
      "Train Epoch: 5 [41600/60000 (69%)]\tLoss: 0.017066\n",
      "Train Epoch: 5 [43200/60000 (72%)]\tLoss: 0.018915\n",
      "Train Epoch: 5 [44800/60000 (75%)]\tLoss: 0.014798\n",
      "Train Epoch: 5 [46400/60000 (77%)]\tLoss: 0.019674\n",
      "Train Epoch: 5 [48000/60000 (80%)]\tLoss: 0.014664\n",
      "Train Epoch: 5 [49600/60000 (83%)]\tLoss: 0.018771\n",
      "Train Epoch: 5 [51200/60000 (85%)]\tLoss: 0.020146\n",
      "Train Epoch: 5 [52800/60000 (88%)]\tLoss: 0.014124\n",
      "Train Epoch: 5 [54400/60000 (91%)]\tLoss: 0.016829\n",
      "Train Epoch: 5 [56000/60000 (93%)]\tLoss: 0.016971\n",
      "Train Epoch: 5 [57600/60000 (96%)]\tLoss: 0.016656\n",
      "Train Epoch: 5 [59200/60000 (99%)]\tLoss: 0.013648\n",
      "\n",
      "Test set: Avg. loss: 0.1210, Accuracy: 4809/5000 (96.18%)\n",
      "\n",
      "Train Epoch: 6 [0/60000 (0%)]\tLoss: 0.012406\n",
      "Train Epoch: 6 [1600/60000 (3%)]\tLoss: 0.020257\n",
      "Train Epoch: 6 [3200/60000 (5%)]\tLoss: 0.015272\n",
      "Train Epoch: 6 [4800/60000 (8%)]\tLoss: 0.011987\n",
      "Train Epoch: 6 [6400/60000 (11%)]\tLoss: 0.013973\n",
      "Train Epoch: 6 [8000/60000 (13%)]\tLoss: 0.011731\n",
      "Train Epoch: 6 [9600/60000 (16%)]\tLoss: 0.015133\n",
      "Train Epoch: 6 [11200/60000 (19%)]\tLoss: 0.015285\n",
      "Train Epoch: 6 [12800/60000 (21%)]\tLoss: 0.012030\n",
      "Train Epoch: 6 [14400/60000 (24%)]\tLoss: 0.017802\n",
      "Train Epoch: 6 [16000/60000 (27%)]\tLoss: 0.018818\n",
      "Train Epoch: 6 [17600/60000 (29%)]\tLoss: 0.016206\n",
      "Train Epoch: 6 [19200/60000 (32%)]\tLoss: 0.011310\n",
      "Train Epoch: 6 [20800/60000 (35%)]\tLoss: 0.014177\n",
      "Train Epoch: 6 [22400/60000 (37%)]\tLoss: 0.018276\n",
      "Train Epoch: 6 [24000/60000 (40%)]\tLoss: 0.012128\n",
      "Train Epoch: 6 [25600/60000 (43%)]\tLoss: 0.012722\n",
      "Train Epoch: 6 [27200/60000 (45%)]\tLoss: 0.015860\n",
      "Train Epoch: 6 [28800/60000 (48%)]\tLoss: 0.018200\n",
      "Train Epoch: 6 [30400/60000 (51%)]\tLoss: 0.013752\n",
      "Train Epoch: 6 [32000/60000 (53%)]\tLoss: 0.014063\n",
      "Train Epoch: 6 [33600/60000 (56%)]\tLoss: 0.013568\n",
      "Train Epoch: 6 [35200/60000 (59%)]\tLoss: 0.014445\n",
      "Train Epoch: 6 [36800/60000 (61%)]\tLoss: 0.012401\n",
      "Train Epoch: 6 [38400/60000 (64%)]\tLoss: 0.018136\n",
      "Train Epoch: 6 [40000/60000 (67%)]\tLoss: 0.012867\n",
      "Train Epoch: 6 [41600/60000 (69%)]\tLoss: 0.010721\n",
      "Train Epoch: 6 [43200/60000 (72%)]\tLoss: 0.018598\n",
      "Train Epoch: 6 [44800/60000 (75%)]\tLoss: 0.014325\n",
      "Train Epoch: 6 [46400/60000 (77%)]\tLoss: 0.014189\n",
      "Train Epoch: 6 [48000/60000 (80%)]\tLoss: 0.012661\n",
      "Train Epoch: 6 [49600/60000 (83%)]\tLoss: 0.012883\n",
      "Train Epoch: 6 [51200/60000 (85%)]\tLoss: 0.010607\n",
      "Train Epoch: 6 [52800/60000 (88%)]\tLoss: 0.009987\n",
      "Train Epoch: 6 [54400/60000 (91%)]\tLoss: 0.010009\n",
      "Train Epoch: 6 [56000/60000 (93%)]\tLoss: 0.016144\n",
      "Train Epoch: 6 [57600/60000 (96%)]\tLoss: 0.017120\n",
      "Train Epoch: 6 [59200/60000 (99%)]\tLoss: 0.013869\n",
      "\n",
      "Test set: Avg. loss: 0.1008, Accuracy: 4844/5000 (96.88%)\n",
      "\n",
      "Train Epoch: 7 [0/60000 (0%)]\tLoss: 0.012797\n",
      "Train Epoch: 7 [1600/60000 (3%)]\tLoss: 0.015418\n",
      "Train Epoch: 7 [3200/60000 (5%)]\tLoss: 0.012595\n",
      "Train Epoch: 7 [4800/60000 (8%)]\tLoss: 0.012815\n",
      "Train Epoch: 7 [6400/60000 (11%)]\tLoss: 0.018384\n",
      "Train Epoch: 7 [8000/60000 (13%)]\tLoss: 0.017105\n",
      "Train Epoch: 7 [9600/60000 (16%)]\tLoss: 0.016967\n",
      "Train Epoch: 7 [11200/60000 (19%)]\tLoss: 0.011174\n",
      "Train Epoch: 7 [12800/60000 (21%)]\tLoss: 0.011806\n",
      "Train Epoch: 7 [14400/60000 (24%)]\tLoss: 0.014321\n",
      "Train Epoch: 7 [16000/60000 (27%)]\tLoss: 0.015182\n",
      "Train Epoch: 7 [17600/60000 (29%)]\tLoss: 0.013826\n",
      "Train Epoch: 7 [19200/60000 (32%)]\tLoss: 0.013583\n",
      "Train Epoch: 7 [20800/60000 (35%)]\tLoss: 0.014329\n",
      "Train Epoch: 7 [22400/60000 (37%)]\tLoss: 0.014750\n",
      "Train Epoch: 7 [24000/60000 (40%)]\tLoss: 0.010279\n",
      "Train Epoch: 7 [25600/60000 (43%)]\tLoss: 0.014042\n",
      "Train Epoch: 7 [27200/60000 (45%)]\tLoss: 0.014708\n",
      "Train Epoch: 7 [28800/60000 (48%)]\tLoss: 0.012507\n",
      "Train Epoch: 7 [30400/60000 (51%)]\tLoss: 0.010079\n",
      "Train Epoch: 7 [32000/60000 (53%)]\tLoss: 0.011062\n",
      "Train Epoch: 7 [33600/60000 (56%)]\tLoss: 0.009176\n",
      "Train Epoch: 7 [35200/60000 (59%)]\tLoss: 0.017174\n",
      "Train Epoch: 7 [36800/60000 (61%)]\tLoss: 0.008939\n",
      "Train Epoch: 7 [38400/60000 (64%)]\tLoss: 0.016621\n",
      "Train Epoch: 7 [40000/60000 (67%)]\tLoss: 0.009450\n",
      "Train Epoch: 7 [41600/60000 (69%)]\tLoss: 0.011810\n",
      "Train Epoch: 7 [43200/60000 (72%)]\tLoss: 0.016581\n",
      "Train Epoch: 7 [44800/60000 (75%)]\tLoss: 0.012943\n",
      "Train Epoch: 7 [46400/60000 (77%)]\tLoss: 0.013616\n",
      "Train Epoch: 7 [48000/60000 (80%)]\tLoss: 0.012336\n",
      "Train Epoch: 7 [49600/60000 (83%)]\tLoss: 0.007621\n",
      "Train Epoch: 7 [51200/60000 (85%)]\tLoss: 0.013265\n",
      "Train Epoch: 7 [52800/60000 (88%)]\tLoss: 0.014665\n",
      "Train Epoch: 7 [54400/60000 (91%)]\tLoss: 0.015206\n",
      "Train Epoch: 7 [56000/60000 (93%)]\tLoss: 0.014082\n",
      "Train Epoch: 7 [57600/60000 (96%)]\tLoss: 0.014932\n",
      "Train Epoch: 7 [59200/60000 (99%)]\tLoss: 0.009866\n",
      "\n",
      "Test set: Avg. loss: 0.0922, Accuracy: 4862/5000 (97.24%)\n",
      "\n",
      "Train Epoch: 8 [0/60000 (0%)]\tLoss: 0.013529\n",
      "Train Epoch: 8 [1600/60000 (3%)]\tLoss: 0.009697\n",
      "Train Epoch: 8 [3200/60000 (5%)]\tLoss: 0.010771\n",
      "Train Epoch: 8 [4800/60000 (8%)]\tLoss: 0.011721\n",
      "Train Epoch: 8 [6400/60000 (11%)]\tLoss: 0.008590\n",
      "Train Epoch: 8 [8000/60000 (13%)]\tLoss: 0.015901\n",
      "Train Epoch: 8 [9600/60000 (16%)]\tLoss: 0.014557\n",
      "Train Epoch: 8 [11200/60000 (19%)]\tLoss: 0.010868\n",
      "Train Epoch: 8 [12800/60000 (21%)]\tLoss: 0.010113\n",
      "Train Epoch: 8 [14400/60000 (24%)]\tLoss: 0.010809\n",
      "Train Epoch: 8 [16000/60000 (27%)]\tLoss: 0.009019\n",
      "Train Epoch: 8 [17600/60000 (29%)]\tLoss: 0.015088\n",
      "Train Epoch: 8 [19200/60000 (32%)]\tLoss: 0.014453\n",
      "Train Epoch: 8 [20800/60000 (35%)]\tLoss: 0.014923\n",
      "Train Epoch: 8 [22400/60000 (37%)]\tLoss: 0.015737\n",
      "Train Epoch: 8 [24000/60000 (40%)]\tLoss: 0.014215\n",
      "Train Epoch: 8 [25600/60000 (43%)]\tLoss: 0.007659\n",
      "Train Epoch: 8 [27200/60000 (45%)]\tLoss: 0.013837\n",
      "Train Epoch: 8 [28800/60000 (48%)]\tLoss: 0.012856\n",
      "Train Epoch: 8 [30400/60000 (51%)]\tLoss: 0.013433\n",
      "Train Epoch: 8 [32000/60000 (53%)]\tLoss: 0.010582\n",
      "Train Epoch: 8 [33600/60000 (56%)]\tLoss: 0.011891\n",
      "Train Epoch: 8 [35200/60000 (59%)]\tLoss: 0.012189\n",
      "Train Epoch: 8 [36800/60000 (61%)]\tLoss: 0.012439\n",
      "Train Epoch: 8 [38400/60000 (64%)]\tLoss: 0.010869\n",
      "Train Epoch: 8 [40000/60000 (67%)]\tLoss: 0.010247\n",
      "Train Epoch: 8 [41600/60000 (69%)]\tLoss: 0.014947\n",
      "Train Epoch: 8 [43200/60000 (72%)]\tLoss: 0.012063\n",
      "Train Epoch: 8 [44800/60000 (75%)]\tLoss: 0.007967\n",
      "Train Epoch: 8 [46400/60000 (77%)]\tLoss: 0.015806\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 8 [48000/60000 (80%)]\tLoss: 0.012159\n",
      "Train Epoch: 8 [49600/60000 (83%)]\tLoss: 0.010067\n",
      "Train Epoch: 8 [51200/60000 (85%)]\tLoss: 0.011884\n",
      "Train Epoch: 8 [52800/60000 (88%)]\tLoss: 0.012926\n",
      "Train Epoch: 8 [54400/60000 (91%)]\tLoss: 0.009708\n",
      "Train Epoch: 8 [56000/60000 (93%)]\tLoss: 0.009350\n",
      "Train Epoch: 8 [57600/60000 (96%)]\tLoss: 0.010637\n",
      "Train Epoch: 8 [59200/60000 (99%)]\tLoss: 0.012001\n",
      "\n",
      "Test set: Avg. loss: 0.0860, Accuracy: 4880/5000 (97.60%)\n",
      "\n",
      "Train Epoch: 9 [0/60000 (0%)]\tLoss: 0.006918\n",
      "Train Epoch: 9 [1600/60000 (3%)]\tLoss: 0.010533\n",
      "Train Epoch: 9 [3200/60000 (5%)]\tLoss: 0.009726\n",
      "Train Epoch: 9 [4800/60000 (8%)]\tLoss: 0.013511\n",
      "Train Epoch: 9 [6400/60000 (11%)]\tLoss: 0.011219\n",
      "Train Epoch: 9 [8000/60000 (13%)]\tLoss: 0.011337\n",
      "Train Epoch: 9 [9600/60000 (16%)]\tLoss: 0.007727\n",
      "Train Epoch: 9 [11200/60000 (19%)]\tLoss: 0.011680\n",
      "Train Epoch: 9 [12800/60000 (21%)]\tLoss: 0.012445\n",
      "Train Epoch: 9 [14400/60000 (24%)]\tLoss: 0.010547\n",
      "Train Epoch: 9 [16000/60000 (27%)]\tLoss: 0.012974\n",
      "Train Epoch: 9 [17600/60000 (29%)]\tLoss: 0.011191\n",
      "Train Epoch: 9 [19200/60000 (32%)]\tLoss: 0.012625\n",
      "Train Epoch: 9 [20800/60000 (35%)]\tLoss: 0.010863\n",
      "Train Epoch: 9 [22400/60000 (37%)]\tLoss: 0.011922\n",
      "Train Epoch: 9 [24000/60000 (40%)]\tLoss: 0.012808\n",
      "Train Epoch: 9 [25600/60000 (43%)]\tLoss: 0.012654\n",
      "Train Epoch: 9 [27200/60000 (45%)]\tLoss: 0.011300\n",
      "Train Epoch: 9 [28800/60000 (48%)]\tLoss: 0.015305\n",
      "Train Epoch: 9 [30400/60000 (51%)]\tLoss: 0.007440\n",
      "Train Epoch: 9 [32000/60000 (53%)]\tLoss: 0.011679\n",
      "Train Epoch: 9 [33600/60000 (56%)]\tLoss: 0.013227\n",
      "Train Epoch: 9 [35200/60000 (59%)]\tLoss: 0.014624\n",
      "Train Epoch: 9 [36800/60000 (61%)]\tLoss: 0.011592\n",
      "Train Epoch: 9 [38400/60000 (64%)]\tLoss: 0.010332\n",
      "Train Epoch: 9 [40000/60000 (67%)]\tLoss: 0.009266\n",
      "Train Epoch: 9 [41600/60000 (69%)]\tLoss: 0.013161\n",
      "Train Epoch: 9 [43200/60000 (72%)]\tLoss: 0.010414\n",
      "Train Epoch: 9 [44800/60000 (75%)]\tLoss: 0.013692\n",
      "Train Epoch: 9 [46400/60000 (77%)]\tLoss: 0.008432\n",
      "Train Epoch: 9 [48000/60000 (80%)]\tLoss: 0.007371\n",
      "Train Epoch: 9 [49600/60000 (83%)]\tLoss: 0.012028\n",
      "Train Epoch: 9 [51200/60000 (85%)]\tLoss: 0.013527\n",
      "Train Epoch: 9 [52800/60000 (88%)]\tLoss: 0.009290\n",
      "Train Epoch: 9 [54400/60000 (91%)]\tLoss: 0.011764\n",
      "Train Epoch: 9 [56000/60000 (93%)]\tLoss: 0.013439\n",
      "Train Epoch: 9 [57600/60000 (96%)]\tLoss: 0.009408\n",
      "Train Epoch: 9 [59200/60000 (99%)]\tLoss: 0.011742\n",
      "\n",
      "Test set: Avg. loss: 0.0878, Accuracy: 4891/5000 (97.82%)\n",
      "\n",
      "Train Epoch: 10 [0/60000 (0%)]\tLoss: 0.011244\n",
      "Train Epoch: 10 [1600/60000 (3%)]\tLoss: 0.007092\n",
      "Train Epoch: 10 [3200/60000 (5%)]\tLoss: 0.011441\n",
      "Train Epoch: 10 [4800/60000 (8%)]\tLoss: 0.012558\n",
      "Train Epoch: 10 [6400/60000 (11%)]\tLoss: 0.009315\n",
      "Train Epoch: 10 [8000/60000 (13%)]\tLoss: 0.011680\n",
      "Train Epoch: 10 [9600/60000 (16%)]\tLoss: 0.009665\n",
      "Train Epoch: 10 [11200/60000 (19%)]\tLoss: 0.008540\n",
      "Train Epoch: 10 [12800/60000 (21%)]\tLoss: 0.008662\n",
      "Train Epoch: 10 [14400/60000 (24%)]\tLoss: 0.009879\n",
      "Train Epoch: 10 [16000/60000 (27%)]\tLoss: 0.009068\n",
      "Train Epoch: 10 [17600/60000 (29%)]\tLoss: 0.008141\n",
      "Train Epoch: 10 [19200/60000 (32%)]\tLoss: 0.011816\n",
      "Train Epoch: 10 [20800/60000 (35%)]\tLoss: 0.011398\n",
      "Train Epoch: 10 [22400/60000 (37%)]\tLoss: 0.016163\n",
      "Train Epoch: 10 [24000/60000 (40%)]\tLoss: 0.007386\n",
      "Train Epoch: 10 [25600/60000 (43%)]\tLoss: 0.007313\n",
      "Train Epoch: 10 [27200/60000 (45%)]\tLoss: 0.012913\n",
      "Train Epoch: 10 [28800/60000 (48%)]\tLoss: 0.010675\n",
      "Train Epoch: 10 [30400/60000 (51%)]\tLoss: 0.012249\n",
      "Train Epoch: 10 [32000/60000 (53%)]\tLoss: 0.011597\n",
      "Train Epoch: 10 [33600/60000 (56%)]\tLoss: 0.013799\n",
      "Train Epoch: 10 [35200/60000 (59%)]\tLoss: 0.010481\n",
      "Train Epoch: 10 [36800/60000 (61%)]\tLoss: 0.007022\n",
      "Train Epoch: 10 [38400/60000 (64%)]\tLoss: 0.008391\n",
      "Train Epoch: 10 [40000/60000 (67%)]\tLoss: 0.013793\n",
      "Train Epoch: 10 [41600/60000 (69%)]\tLoss: 0.008352\n",
      "Train Epoch: 10 [43200/60000 (72%)]\tLoss: 0.012536\n",
      "Train Epoch: 10 [44800/60000 (75%)]\tLoss: 0.011706\n",
      "Train Epoch: 10 [46400/60000 (77%)]\tLoss: 0.011848\n",
      "Train Epoch: 10 [48000/60000 (80%)]\tLoss: 0.010271\n",
      "Train Epoch: 10 [49600/60000 (83%)]\tLoss: 0.014658\n",
      "Train Epoch: 10 [51200/60000 (85%)]\tLoss: 0.008124\n",
      "Train Epoch: 10 [52800/60000 (88%)]\tLoss: 0.010045\n",
      "Train Epoch: 10 [54400/60000 (91%)]\tLoss: 0.012244\n",
      "Train Epoch: 10 [56000/60000 (93%)]\tLoss: 0.008717\n",
      "Train Epoch: 10 [57600/60000 (96%)]\tLoss: 0.008008\n",
      "Train Epoch: 10 [59200/60000 (99%)]\tLoss: 0.010223\n",
      "\n",
      "Test set: Avg. loss: 0.0835, Accuracy: 4886/5000 (97.72%)\n",
      "\n",
      "Train Epoch: 11 [0/60000 (0%)]\tLoss: 0.011999\n",
      "Train Epoch: 11 [1600/60000 (3%)]\tLoss: 0.015248\n",
      "Train Epoch: 11 [3200/60000 (5%)]\tLoss: 0.010421\n",
      "Train Epoch: 11 [4800/60000 (8%)]\tLoss: 0.011943\n",
      "Train Epoch: 11 [6400/60000 (11%)]\tLoss: 0.008372\n",
      "Train Epoch: 11 [8000/60000 (13%)]\tLoss: 0.007739\n",
      "Train Epoch: 11 [9600/60000 (16%)]\tLoss: 0.008176\n",
      "Train Epoch: 11 [11200/60000 (19%)]\tLoss: 0.010471\n",
      "Train Epoch: 11 [12800/60000 (21%)]\tLoss: 0.009466\n",
      "Train Epoch: 11 [14400/60000 (24%)]\tLoss: 0.011964\n",
      "Train Epoch: 11 [16000/60000 (27%)]\tLoss: 0.011306\n",
      "Train Epoch: 11 [17600/60000 (29%)]\tLoss: 0.009892\n",
      "Train Epoch: 11 [19200/60000 (32%)]\tLoss: 0.010034\n",
      "Train Epoch: 11 [20800/60000 (35%)]\tLoss: 0.008563\n",
      "Train Epoch: 11 [22400/60000 (37%)]\tLoss: 0.008523\n",
      "Train Epoch: 11 [24000/60000 (40%)]\tLoss: 0.013762\n",
      "Train Epoch: 11 [25600/60000 (43%)]\tLoss: 0.007584\n",
      "Train Epoch: 11 [27200/60000 (45%)]\tLoss: 0.007247\n",
      "Train Epoch: 11 [28800/60000 (48%)]\tLoss: 0.009936\n",
      "Train Epoch: 11 [30400/60000 (51%)]\tLoss: 0.008295\n",
      "Train Epoch: 11 [32000/60000 (53%)]\tLoss: 0.009406\n",
      "Train Epoch: 11 [33600/60000 (56%)]\tLoss: 0.012149\n",
      "Train Epoch: 11 [35200/60000 (59%)]\tLoss: 0.011392\n",
      "Train Epoch: 11 [36800/60000 (61%)]\tLoss: 0.007860\n",
      "Train Epoch: 11 [38400/60000 (64%)]\tLoss: 0.008549\n",
      "Train Epoch: 11 [40000/60000 (67%)]\tLoss: 0.012057\n",
      "Train Epoch: 11 [41600/60000 (69%)]\tLoss: 0.012197\n",
      "Train Epoch: 11 [43200/60000 (72%)]\tLoss: 0.007666\n",
      "Train Epoch: 11 [44800/60000 (75%)]\tLoss: 0.006482\n",
      "Train Epoch: 11 [46400/60000 (77%)]\tLoss: 0.011390\n",
      "Train Epoch: 11 [48000/60000 (80%)]\tLoss: 0.012638\n",
      "Train Epoch: 11 [49600/60000 (83%)]\tLoss: 0.011439\n",
      "Train Epoch: 11 [51200/60000 (85%)]\tLoss: 0.010865\n",
      "Train Epoch: 11 [52800/60000 (88%)]\tLoss: 0.012579\n",
      "Train Epoch: 11 [54400/60000 (91%)]\tLoss: 0.008487\n",
      "Train Epoch: 11 [56000/60000 (93%)]\tLoss: 0.008076\n",
      "Train Epoch: 11 [57600/60000 (96%)]\tLoss: 0.010567\n",
      "Train Epoch: 11 [59200/60000 (99%)]\tLoss: 0.008480\n",
      "\n",
      "Test set: Avg. loss: 0.0866, Accuracy: 4903/5000 (98.06%)\n",
      "\n",
      "Train Epoch: 12 [0/60000 (0%)]\tLoss: 0.010463\n",
      "Train Epoch: 12 [1600/60000 (3%)]\tLoss: 0.007954\n",
      "Train Epoch: 12 [3200/60000 (5%)]\tLoss: 0.009251\n",
      "Train Epoch: 12 [4800/60000 (8%)]\tLoss: 0.010092\n",
      "Train Epoch: 12 [6400/60000 (11%)]\tLoss: 0.010375\n",
      "Train Epoch: 12 [8000/60000 (13%)]\tLoss: 0.007028\n",
      "Train Epoch: 12 [9600/60000 (16%)]\tLoss: 0.011699\n",
      "Train Epoch: 12 [11200/60000 (19%)]\tLoss: 0.008479\n",
      "Train Epoch: 12 [12800/60000 (21%)]\tLoss: 0.008372\n",
      "Train Epoch: 12 [14400/60000 (24%)]\tLoss: 0.007745\n",
      "Train Epoch: 12 [16000/60000 (27%)]\tLoss: 0.009331\n",
      "Train Epoch: 12 [17600/60000 (29%)]\tLoss: 0.010369\n",
      "Train Epoch: 12 [19200/60000 (32%)]\tLoss: 0.007292\n",
      "Train Epoch: 12 [20800/60000 (35%)]\tLoss: 0.009746\n",
      "Train Epoch: 12 [22400/60000 (37%)]\tLoss: 0.010145\n",
      "Train Epoch: 12 [24000/60000 (40%)]\tLoss: 0.008325\n",
      "Train Epoch: 12 [25600/60000 (43%)]\tLoss: 0.012190\n",
      "Train Epoch: 12 [27200/60000 (45%)]\tLoss: 0.008141\n",
      "Train Epoch: 12 [28800/60000 (48%)]\tLoss: 0.010684\n",
      "Train Epoch: 12 [30400/60000 (51%)]\tLoss: 0.008000\n",
      "Train Epoch: 12 [32000/60000 (53%)]\tLoss: 0.007108\n",
      "Train Epoch: 12 [33600/60000 (56%)]\tLoss: 0.011783\n",
      "Train Epoch: 12 [35200/60000 (59%)]\tLoss: 0.005155\n",
      "Train Epoch: 12 [36800/60000 (61%)]\tLoss: 0.011849\n",
      "Train Epoch: 12 [38400/60000 (64%)]\tLoss: 0.013677\n",
      "Train Epoch: 12 [40000/60000 (67%)]\tLoss: 0.009626\n",
      "Train Epoch: 12 [41600/60000 (69%)]\tLoss: 0.009825\n",
      "Train Epoch: 12 [43200/60000 (72%)]\tLoss: 0.009021\n",
      "Train Epoch: 12 [44800/60000 (75%)]\tLoss: 0.007021\n",
      "Train Epoch: 12 [46400/60000 (77%)]\tLoss: 0.009655\n",
      "Train Epoch: 12 [48000/60000 (80%)]\tLoss: 0.013375\n",
      "Train Epoch: 12 [49600/60000 (83%)]\tLoss: 0.010887\n",
      "Train Epoch: 12 [51200/60000 (85%)]\tLoss: 0.011296\n",
      "Train Epoch: 12 [52800/60000 (88%)]\tLoss: 0.010094\n",
      "Train Epoch: 12 [54400/60000 (91%)]\tLoss: 0.011764\n",
      "Train Epoch: 12 [56000/60000 (93%)]\tLoss: 0.011285\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 12 [57600/60000 (96%)]\tLoss: 0.009489\n",
      "Train Epoch: 12 [59200/60000 (99%)]\tLoss: 0.011034\n",
      "\n",
      "Test set: Avg. loss: 0.0723, Accuracy: 4921/5000 (98.42%)\n",
      "\n",
      "Train Epoch: 13 [0/60000 (0%)]\tLoss: 0.009831\n",
      "Train Epoch: 13 [1600/60000 (3%)]\tLoss: 0.010524\n",
      "Train Epoch: 13 [3200/60000 (5%)]\tLoss: 0.009459\n",
      "Train Epoch: 13 [4800/60000 (8%)]\tLoss: 0.009026\n",
      "Train Epoch: 13 [6400/60000 (11%)]\tLoss: 0.006170\n",
      "Train Epoch: 13 [8000/60000 (13%)]\tLoss: 0.007801\n",
      "Train Epoch: 13 [9600/60000 (16%)]\tLoss: 0.007212\n",
      "Train Epoch: 13 [11200/60000 (19%)]\tLoss: 0.011008\n",
      "Train Epoch: 13 [12800/60000 (21%)]\tLoss: 0.007212\n",
      "Train Epoch: 13 [14400/60000 (24%)]\tLoss: 0.005619\n",
      "Train Epoch: 13 [16000/60000 (27%)]\tLoss: 0.009251\n",
      "Train Epoch: 13 [17600/60000 (29%)]\tLoss: 0.010139\n",
      "Train Epoch: 13 [19200/60000 (32%)]\tLoss: 0.007586\n",
      "Train Epoch: 13 [20800/60000 (35%)]\tLoss: 0.007868\n",
      "Train Epoch: 13 [22400/60000 (37%)]\tLoss: 0.011582\n",
      "Train Epoch: 13 [24000/60000 (40%)]\tLoss: 0.010673\n",
      "Train Epoch: 13 [25600/60000 (43%)]\tLoss: 0.010814\n",
      "Train Epoch: 13 [27200/60000 (45%)]\tLoss: 0.006007\n",
      "Train Epoch: 13 [28800/60000 (48%)]\tLoss: 0.012265\n",
      "Train Epoch: 13 [30400/60000 (51%)]\tLoss: 0.009912\n",
      "Train Epoch: 13 [32000/60000 (53%)]\tLoss: 0.009239\n",
      "Train Epoch: 13 [33600/60000 (56%)]\tLoss: 0.011886\n",
      "Train Epoch: 13 [35200/60000 (59%)]\tLoss: 0.008123\n",
      "Train Epoch: 13 [36800/60000 (61%)]\tLoss: 0.012491\n",
      "Train Epoch: 13 [38400/60000 (64%)]\tLoss: 0.007504\n",
      "Train Epoch: 13 [40000/60000 (67%)]\tLoss: 0.007760\n",
      "Train Epoch: 13 [41600/60000 (69%)]\tLoss: 0.009401\n",
      "Train Epoch: 13 [43200/60000 (72%)]\tLoss: 0.006507\n",
      "Train Epoch: 13 [44800/60000 (75%)]\tLoss: 0.007356\n",
      "Train Epoch: 13 [46400/60000 (77%)]\tLoss: 0.010413\n",
      "Train Epoch: 13 [48000/60000 (80%)]\tLoss: 0.012436\n",
      "Train Epoch: 13 [49600/60000 (83%)]\tLoss: 0.007955\n",
      "Train Epoch: 13 [51200/60000 (85%)]\tLoss: 0.008832\n",
      "Train Epoch: 13 [52800/60000 (88%)]\tLoss: 0.007460\n",
      "Train Epoch: 13 [54400/60000 (91%)]\tLoss: 0.006439\n",
      "Train Epoch: 13 [56000/60000 (93%)]\tLoss: 0.008561\n",
      "Train Epoch: 13 [57600/60000 (96%)]\tLoss: 0.014296\n",
      "Train Epoch: 13 [59200/60000 (99%)]\tLoss: 0.009775\n",
      "\n",
      "Test set: Avg. loss: 0.0799, Accuracy: 4902/5000 (98.04%)\n",
      "\n",
      "Train Epoch: 14 [0/60000 (0%)]\tLoss: 0.010326\n",
      "Train Epoch: 14 [1600/60000 (3%)]\tLoss: 0.008880\n",
      "Train Epoch: 14 [3200/60000 (5%)]\tLoss: 0.009875\n",
      "Train Epoch: 14 [4800/60000 (8%)]\tLoss: 0.006485\n",
      "Train Epoch: 14 [6400/60000 (11%)]\tLoss: 0.006997\n",
      "Train Epoch: 14 [8000/60000 (13%)]\tLoss: 0.011220\n",
      "Train Epoch: 14 [9600/60000 (16%)]\tLoss: 0.007686\n",
      "Train Epoch: 14 [11200/60000 (19%)]\tLoss: 0.011095\n",
      "Train Epoch: 14 [12800/60000 (21%)]\tLoss: 0.008709\n",
      "Train Epoch: 14 [14400/60000 (24%)]\tLoss: 0.011304\n",
      "Train Epoch: 14 [16000/60000 (27%)]\tLoss: 0.009414\n",
      "Train Epoch: 14 [17600/60000 (29%)]\tLoss: 0.007694\n",
      "Train Epoch: 14 [19200/60000 (32%)]\tLoss: 0.008004\n",
      "Train Epoch: 14 [20800/60000 (35%)]\tLoss: 0.008742\n",
      "Train Epoch: 14 [22400/60000 (37%)]\tLoss: 0.009859\n",
      "Train Epoch: 14 [24000/60000 (40%)]\tLoss: 0.009966\n",
      "Train Epoch: 14 [25600/60000 (43%)]\tLoss: 0.010564\n",
      "Train Epoch: 14 [27200/60000 (45%)]\tLoss: 0.010844\n",
      "Train Epoch: 14 [28800/60000 (48%)]\tLoss: 0.009259\n",
      "Train Epoch: 14 [30400/60000 (51%)]\tLoss: 0.007362\n",
      "Train Epoch: 14 [32000/60000 (53%)]\tLoss: 0.009741\n",
      "Train Epoch: 14 [33600/60000 (56%)]\tLoss: 0.011627\n",
      "Train Epoch: 14 [35200/60000 (59%)]\tLoss: 0.008631\n",
      "Train Epoch: 14 [36800/60000 (61%)]\tLoss: 0.004919\n",
      "Train Epoch: 14 [38400/60000 (64%)]\tLoss: 0.009703\n",
      "Train Epoch: 14 [40000/60000 (67%)]\tLoss: 0.008241\n",
      "Train Epoch: 14 [41600/60000 (69%)]\tLoss: 0.008355\n",
      "Train Epoch: 14 [43200/60000 (72%)]\tLoss: 0.007199\n",
      "Train Epoch: 14 [44800/60000 (75%)]\tLoss: 0.006579\n",
      "Train Epoch: 14 [46400/60000 (77%)]\tLoss: 0.006204\n",
      "Train Epoch: 14 [48000/60000 (80%)]\tLoss: 0.007546\n",
      "Train Epoch: 14 [49600/60000 (83%)]\tLoss: 0.010524\n",
      "Train Epoch: 14 [51200/60000 (85%)]\tLoss: 0.008011\n",
      "Train Epoch: 14 [52800/60000 (88%)]\tLoss: 0.010307\n",
      "Train Epoch: 14 [54400/60000 (91%)]\tLoss: 0.010137\n",
      "Train Epoch: 14 [56000/60000 (93%)]\tLoss: 0.007996\n",
      "Train Epoch: 14 [57600/60000 (96%)]\tLoss: 0.010638\n",
      "Train Epoch: 14 [59200/60000 (99%)]\tLoss: 0.010418\n",
      "\n",
      "Test set: Avg. loss: 0.0631, Accuracy: 4936/5000 (98.72%)\n",
      "\n",
      "Train Epoch: 15 [0/60000 (0%)]\tLoss: 0.009433\n",
      "Train Epoch: 15 [1600/60000 (3%)]\tLoss: 0.008112\n",
      "Train Epoch: 15 [3200/60000 (5%)]\tLoss: 0.007810\n",
      "Train Epoch: 15 [4800/60000 (8%)]\tLoss: 0.008510\n",
      "Train Epoch: 15 [6400/60000 (11%)]\tLoss: 0.007561\n",
      "Train Epoch: 15 [8000/60000 (13%)]\tLoss: 0.012231\n",
      "Train Epoch: 15 [9600/60000 (16%)]\tLoss: 0.007593\n",
      "Train Epoch: 15 [11200/60000 (19%)]\tLoss: 0.010599\n",
      "Train Epoch: 15 [12800/60000 (21%)]\tLoss: 0.009574\n",
      "Train Epoch: 15 [14400/60000 (24%)]\tLoss: 0.010993\n",
      "Train Epoch: 15 [16000/60000 (27%)]\tLoss: 0.009876\n",
      "Train Epoch: 15 [17600/60000 (29%)]\tLoss: 0.006076\n",
      "Train Epoch: 15 [19200/60000 (32%)]\tLoss: 0.008921\n",
      "Train Epoch: 15 [20800/60000 (35%)]\tLoss: 0.007634\n",
      "Train Epoch: 15 [22400/60000 (37%)]\tLoss: 0.007154\n",
      "Train Epoch: 15 [24000/60000 (40%)]\tLoss: 0.006003\n",
      "Train Epoch: 15 [25600/60000 (43%)]\tLoss: 0.006949\n",
      "Train Epoch: 15 [27200/60000 (45%)]\tLoss: 0.006118\n",
      "Train Epoch: 15 [28800/60000 (48%)]\tLoss: 0.008495\n",
      "Train Epoch: 15 [30400/60000 (51%)]\tLoss: 0.008916\n",
      "Train Epoch: 15 [32000/60000 (53%)]\tLoss: 0.006797\n",
      "Train Epoch: 15 [33600/60000 (56%)]\tLoss: 0.014203\n",
      "Train Epoch: 15 [35200/60000 (59%)]\tLoss: 0.006593\n",
      "Train Epoch: 15 [36800/60000 (61%)]\tLoss: 0.008479\n",
      "Train Epoch: 15 [38400/60000 (64%)]\tLoss: 0.010121\n",
      "Train Epoch: 15 [40000/60000 (67%)]\tLoss: 0.007335\n",
      "Train Epoch: 15 [41600/60000 (69%)]\tLoss: 0.011657\n",
      "Train Epoch: 15 [43200/60000 (72%)]\tLoss: 0.009416\n",
      "Train Epoch: 15 [44800/60000 (75%)]\tLoss: 0.012194\n",
      "Train Epoch: 15 [46400/60000 (77%)]\tLoss: 0.005437\n",
      "Train Epoch: 15 [48000/60000 (80%)]\tLoss: 0.009121\n",
      "Train Epoch: 15 [49600/60000 (83%)]\tLoss: 0.007166\n",
      "Train Epoch: 15 [51200/60000 (85%)]\tLoss: 0.007267\n",
      "Train Epoch: 15 [52800/60000 (88%)]\tLoss: 0.009047\n",
      "Train Epoch: 15 [54400/60000 (91%)]\tLoss: 0.007807\n",
      "Train Epoch: 15 [56000/60000 (93%)]\tLoss: 0.006629\n",
      "Train Epoch: 15 [57600/60000 (96%)]\tLoss: 0.008672\n",
      "Train Epoch: 15 [59200/60000 (99%)]\tLoss: 0.007019\n",
      "\n",
      "Test set: Avg. loss: 0.0675, Accuracy: 4936/5000 (98.72%)\n",
      "\n",
      "Train Epoch: 16 [0/60000 (0%)]\tLoss: 0.007983\n",
      "Train Epoch: 16 [1600/60000 (3%)]\tLoss: 0.010454\n",
      "Train Epoch: 16 [3200/60000 (5%)]\tLoss: 0.006398\n",
      "Train Epoch: 16 [4800/60000 (8%)]\tLoss: 0.008687\n",
      "Train Epoch: 16 [6400/60000 (11%)]\tLoss: 0.007271\n",
      "Train Epoch: 16 [8000/60000 (13%)]\tLoss: 0.008055\n",
      "Train Epoch: 16 [9600/60000 (16%)]\tLoss: 0.010402\n",
      "Train Epoch: 16 [11200/60000 (19%)]\tLoss: 0.007651\n",
      "Train Epoch: 16 [12800/60000 (21%)]\tLoss: 0.008953\n",
      "Train Epoch: 16 [14400/60000 (24%)]\tLoss: 0.009616\n",
      "Train Epoch: 16 [16000/60000 (27%)]\tLoss: 0.008609\n",
      "Train Epoch: 16 [17600/60000 (29%)]\tLoss: 0.005343\n",
      "Train Epoch: 16 [19200/60000 (32%)]\tLoss: 0.008115\n",
      "Train Epoch: 16 [20800/60000 (35%)]\tLoss: 0.007892\n",
      "Train Epoch: 16 [22400/60000 (37%)]\tLoss: 0.009229\n",
      "Train Epoch: 16 [24000/60000 (40%)]\tLoss: 0.013295\n",
      "Train Epoch: 16 [25600/60000 (43%)]\tLoss: 0.005918\n",
      "Train Epoch: 16 [27200/60000 (45%)]\tLoss: 0.009456\n",
      "Train Epoch: 16 [28800/60000 (48%)]\tLoss: 0.006913\n",
      "Train Epoch: 16 [30400/60000 (51%)]\tLoss: 0.009100\n",
      "Train Epoch: 16 [32000/60000 (53%)]\tLoss: 0.010490\n",
      "Train Epoch: 16 [33600/60000 (56%)]\tLoss: 0.006540\n",
      "Train Epoch: 16 [35200/60000 (59%)]\tLoss: 0.007697\n",
      "Train Epoch: 16 [36800/60000 (61%)]\tLoss: 0.010010\n",
      "Train Epoch: 16 [38400/60000 (64%)]\tLoss: 0.005334\n",
      "Train Epoch: 16 [40000/60000 (67%)]\tLoss: 0.009838\n",
      "Train Epoch: 16 [41600/60000 (69%)]\tLoss: 0.010561\n",
      "Train Epoch: 16 [43200/60000 (72%)]\tLoss: 0.005945\n",
      "Train Epoch: 16 [44800/60000 (75%)]\tLoss: 0.008636\n",
      "Train Epoch: 16 [46400/60000 (77%)]\tLoss: 0.006742\n",
      "Train Epoch: 16 [48000/60000 (80%)]\tLoss: 0.008350\n",
      "Train Epoch: 16 [49600/60000 (83%)]\tLoss: 0.006917\n",
      "Train Epoch: 16 [51200/60000 (85%)]\tLoss: 0.008662\n",
      "Train Epoch: 16 [52800/60000 (88%)]\tLoss: 0.010230\n",
      "Train Epoch: 16 [54400/60000 (91%)]\tLoss: 0.007675\n",
      "Train Epoch: 16 [56000/60000 (93%)]\tLoss: 0.006198\n",
      "Train Epoch: 16 [57600/60000 (96%)]\tLoss: 0.008695\n",
      "Train Epoch: 16 [59200/60000 (99%)]\tLoss: 0.005655\n",
      "\n",
      "Test set: Avg. loss: 0.0590, Accuracy: 4944/5000 (98.88%)\n",
      "\n",
      "Train Epoch: 17 [0/60000 (0%)]\tLoss: 0.009806\n",
      "Train Epoch: 17 [1600/60000 (3%)]\tLoss: 0.006638\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 17 [3200/60000 (5%)]\tLoss: 0.006648\n",
      "Train Epoch: 17 [4800/60000 (8%)]\tLoss: 0.006619\n",
      "Train Epoch: 17 [6400/60000 (11%)]\tLoss: 0.008909\n",
      "Train Epoch: 17 [8000/60000 (13%)]\tLoss: 0.006439\n",
      "Train Epoch: 17 [9600/60000 (16%)]\tLoss: 0.006811\n",
      "Train Epoch: 17 [11200/60000 (19%)]\tLoss: 0.011399\n",
      "Train Epoch: 17 [12800/60000 (21%)]\tLoss: 0.007088\n",
      "Train Epoch: 17 [14400/60000 (24%)]\tLoss: 0.007756\n",
      "Train Epoch: 17 [16000/60000 (27%)]\tLoss: 0.005377\n",
      "Train Epoch: 17 [17600/60000 (29%)]\tLoss: 0.009249\n",
      "Train Epoch: 17 [19200/60000 (32%)]\tLoss: 0.007580\n",
      "Train Epoch: 17 [20800/60000 (35%)]\tLoss: 0.007189\n",
      "Train Epoch: 17 [22400/60000 (37%)]\tLoss: 0.007289\n",
      "Train Epoch: 17 [24000/60000 (40%)]\tLoss: 0.008484\n",
      "Train Epoch: 17 [25600/60000 (43%)]\tLoss: 0.010651\n",
      "Train Epoch: 17 [27200/60000 (45%)]\tLoss: 0.011748\n",
      "Train Epoch: 17 [28800/60000 (48%)]\tLoss: 0.010991\n",
      "Train Epoch: 17 [30400/60000 (51%)]\tLoss: 0.007409\n",
      "Train Epoch: 17 [32000/60000 (53%)]\tLoss: 0.006627\n",
      "Train Epoch: 17 [33600/60000 (56%)]\tLoss: 0.005092\n",
      "Train Epoch: 17 [35200/60000 (59%)]\tLoss: 0.007330\n",
      "Train Epoch: 17 [36800/60000 (61%)]\tLoss: 0.005662\n",
      "Train Epoch: 17 [38400/60000 (64%)]\tLoss: 0.011697\n",
      "Train Epoch: 17 [40000/60000 (67%)]\tLoss: 0.006793\n",
      "Train Epoch: 17 [41600/60000 (69%)]\tLoss: 0.011777\n",
      "Train Epoch: 17 [43200/60000 (72%)]\tLoss: 0.009917\n",
      "Train Epoch: 17 [44800/60000 (75%)]\tLoss: 0.007288\n",
      "Train Epoch: 17 [46400/60000 (77%)]\tLoss: 0.007049\n",
      "Train Epoch: 17 [48000/60000 (80%)]\tLoss: 0.008325\n",
      "Train Epoch: 17 [49600/60000 (83%)]\tLoss: 0.005962\n",
      "Train Epoch: 17 [51200/60000 (85%)]\tLoss: 0.009122\n",
      "Train Epoch: 17 [52800/60000 (88%)]\tLoss: 0.011331\n",
      "Train Epoch: 17 [54400/60000 (91%)]\tLoss: 0.009802\n",
      "Train Epoch: 17 [56000/60000 (93%)]\tLoss: 0.008797\n",
      "Train Epoch: 17 [57600/60000 (96%)]\tLoss: 0.005998\n",
      "Train Epoch: 17 [59200/60000 (99%)]\tLoss: 0.007024\n",
      "\n",
      "Test set: Avg. loss: 0.0611, Accuracy: 4929/5000 (98.58%)\n",
      "\n",
      "Train Epoch: 18 [0/60000 (0%)]\tLoss: 0.007309\n",
      "Train Epoch: 18 [1600/60000 (3%)]\tLoss: 0.006826\n",
      "Train Epoch: 18 [3200/60000 (5%)]\tLoss: 0.007436\n",
      "Train Epoch: 18 [4800/60000 (8%)]\tLoss: 0.005066\n",
      "Train Epoch: 18 [6400/60000 (11%)]\tLoss: 0.006718\n",
      "Train Epoch: 18 [8000/60000 (13%)]\tLoss: 0.009060\n",
      "Train Epoch: 18 [9600/60000 (16%)]\tLoss: 0.006410\n",
      "Train Epoch: 18 [11200/60000 (19%)]\tLoss: 0.004677\n",
      "Train Epoch: 18 [12800/60000 (21%)]\tLoss: 0.007848\n",
      "Train Epoch: 18 [14400/60000 (24%)]\tLoss: 0.008963\n",
      "Train Epoch: 18 [16000/60000 (27%)]\tLoss: 0.005134\n",
      "Train Epoch: 18 [17600/60000 (29%)]\tLoss: 0.005349\n",
      "Train Epoch: 18 [19200/60000 (32%)]\tLoss: 0.007682\n",
      "Train Epoch: 18 [20800/60000 (35%)]\tLoss: 0.004482\n",
      "Train Epoch: 18 [22400/60000 (37%)]\tLoss: 0.005704\n",
      "Train Epoch: 18 [24000/60000 (40%)]\tLoss: 0.007817\n",
      "Train Epoch: 18 [25600/60000 (43%)]\tLoss: 0.007758\n",
      "Train Epoch: 18 [27200/60000 (45%)]\tLoss: 0.005993\n",
      "Train Epoch: 18 [28800/60000 (48%)]\tLoss: 0.006504\n",
      "Train Epoch: 18 [30400/60000 (51%)]\tLoss: 0.008505\n",
      "Train Epoch: 18 [32000/60000 (53%)]\tLoss: 0.012100\n",
      "Train Epoch: 18 [33600/60000 (56%)]\tLoss: 0.006524\n",
      "Train Epoch: 18 [35200/60000 (59%)]\tLoss: 0.007365\n",
      "Train Epoch: 18 [36800/60000 (61%)]\tLoss: 0.007325\n",
      "Train Epoch: 18 [38400/60000 (64%)]\tLoss: 0.005502\n",
      "Train Epoch: 18 [40000/60000 (67%)]\tLoss: 0.006970\n",
      "Train Epoch: 18 [41600/60000 (69%)]\tLoss: 0.009756\n",
      "Train Epoch: 18 [43200/60000 (72%)]\tLoss: 0.006155\n",
      "Train Epoch: 18 [44800/60000 (75%)]\tLoss: 0.006014\n",
      "Train Epoch: 18 [46400/60000 (77%)]\tLoss: 0.004959\n",
      "Train Epoch: 18 [48000/60000 (80%)]\tLoss: 0.009720\n",
      "Train Epoch: 18 [49600/60000 (83%)]\tLoss: 0.009810\n",
      "Train Epoch: 18 [51200/60000 (85%)]\tLoss: 0.007556\n",
      "Train Epoch: 18 [52800/60000 (88%)]\tLoss: 0.010947\n",
      "Train Epoch: 18 [54400/60000 (91%)]\tLoss: 0.005065\n",
      "Train Epoch: 18 [56000/60000 (93%)]\tLoss: 0.009267\n",
      "Train Epoch: 18 [57600/60000 (96%)]\tLoss: 0.006224\n",
      "Train Epoch: 18 [59200/60000 (99%)]\tLoss: 0.007792\n",
      "\n",
      "Test set: Avg. loss: 0.0611, Accuracy: 4945/5000 (98.90%)\n",
      "\n",
      "Train Epoch: 19 [0/60000 (0%)]\tLoss: 0.007879\n",
      "Train Epoch: 19 [1600/60000 (3%)]\tLoss: 0.008495\n",
      "Train Epoch: 19 [3200/60000 (5%)]\tLoss: 0.008043\n",
      "Train Epoch: 19 [4800/60000 (8%)]\tLoss: 0.005702\n",
      "Train Epoch: 19 [6400/60000 (11%)]\tLoss: 0.006685\n",
      "Train Epoch: 19 [8000/60000 (13%)]\tLoss: 0.011047\n",
      "Train Epoch: 19 [9600/60000 (16%)]\tLoss: 0.006634\n",
      "Train Epoch: 19 [11200/60000 (19%)]\tLoss: 0.007692\n",
      "Train Epoch: 19 [12800/60000 (21%)]\tLoss: 0.007511\n",
      "Train Epoch: 19 [14400/60000 (24%)]\tLoss: 0.006459\n",
      "Train Epoch: 19 [16000/60000 (27%)]\tLoss: 0.006355\n",
      "Train Epoch: 19 [17600/60000 (29%)]\tLoss: 0.009335\n",
      "Train Epoch: 19 [19200/60000 (32%)]\tLoss: 0.004130\n",
      "Train Epoch: 19 [20800/60000 (35%)]\tLoss: 0.005404\n",
      "Train Epoch: 19 [22400/60000 (37%)]\tLoss: 0.005125\n",
      "Train Epoch: 19 [24000/60000 (40%)]\tLoss: 0.009651\n",
      "Train Epoch: 19 [25600/60000 (43%)]\tLoss: 0.007322\n",
      "Train Epoch: 19 [27200/60000 (45%)]\tLoss: 0.006392\n",
      "Train Epoch: 19 [28800/60000 (48%)]\tLoss: 0.006122\n",
      "Train Epoch: 19 [30400/60000 (51%)]\tLoss: 0.006705\n",
      "Train Epoch: 19 [32000/60000 (53%)]\tLoss: 0.011117\n",
      "Train Epoch: 19 [33600/60000 (56%)]\tLoss: 0.006279\n",
      "Train Epoch: 19 [35200/60000 (59%)]\tLoss: 0.009551\n",
      "Train Epoch: 19 [36800/60000 (61%)]\tLoss: 0.006921\n",
      "Train Epoch: 19 [38400/60000 (64%)]\tLoss: 0.008799\n",
      "Train Epoch: 19 [40000/60000 (67%)]\tLoss: 0.009100\n",
      "Train Epoch: 19 [41600/60000 (69%)]\tLoss: 0.008397\n",
      "Train Epoch: 19 [43200/60000 (72%)]\tLoss: 0.006271\n",
      "Train Epoch: 19 [44800/60000 (75%)]\tLoss: 0.006727\n",
      "Train Epoch: 19 [46400/60000 (77%)]\tLoss: 0.007166\n",
      "Train Epoch: 19 [48000/60000 (80%)]\tLoss: 0.005531\n",
      "Train Epoch: 19 [49600/60000 (83%)]\tLoss: 0.008678\n",
      "Train Epoch: 19 [51200/60000 (85%)]\tLoss: 0.009153\n",
      "Train Epoch: 19 [52800/60000 (88%)]\tLoss: 0.006381\n",
      "Train Epoch: 19 [54400/60000 (91%)]\tLoss: 0.009788\n",
      "Train Epoch: 19 [56000/60000 (93%)]\tLoss: 0.008064\n",
      "Train Epoch: 19 [57600/60000 (96%)]\tLoss: 0.008010\n",
      "Train Epoch: 19 [59200/60000 (99%)]\tLoss: 0.007027\n",
      "\n",
      "Test set: Avg. loss: 0.0582, Accuracy: 4941/5000 (98.82%)\n",
      "\n",
      "Train Epoch: 20 [0/60000 (0%)]\tLoss: 0.007283\n",
      "Train Epoch: 20 [1600/60000 (3%)]\tLoss: 0.007159\n",
      "Train Epoch: 20 [3200/60000 (5%)]\tLoss: 0.007191\n",
      "Train Epoch: 20 [4800/60000 (8%)]\tLoss: 0.007452\n",
      "Train Epoch: 20 [6400/60000 (11%)]\tLoss: 0.010378\n",
      "Train Epoch: 20 [8000/60000 (13%)]\tLoss: 0.004408\n",
      "Train Epoch: 20 [9600/60000 (16%)]\tLoss: 0.004455\n",
      "Train Epoch: 20 [11200/60000 (19%)]\tLoss: 0.008402\n",
      "Train Epoch: 20 [12800/60000 (21%)]\tLoss: 0.006008\n",
      "Train Epoch: 20 [14400/60000 (24%)]\tLoss: 0.009823\n",
      "Train Epoch: 20 [16000/60000 (27%)]\tLoss: 0.005978\n",
      "Train Epoch: 20 [17600/60000 (29%)]\tLoss: 0.007498\n",
      "Train Epoch: 20 [19200/60000 (32%)]\tLoss: 0.008415\n",
      "Train Epoch: 20 [20800/60000 (35%)]\tLoss: 0.007856\n",
      "Train Epoch: 20 [22400/60000 (37%)]\tLoss: 0.005274\n",
      "Train Epoch: 20 [24000/60000 (40%)]\tLoss: 0.006788\n",
      "Train Epoch: 20 [25600/60000 (43%)]\tLoss: 0.006018\n",
      "Train Epoch: 20 [27200/60000 (45%)]\tLoss: 0.004316\n",
      "Train Epoch: 20 [28800/60000 (48%)]\tLoss: 0.008027\n",
      "Train Epoch: 20 [30400/60000 (51%)]\tLoss: 0.006041\n",
      "Train Epoch: 20 [32000/60000 (53%)]\tLoss: 0.008369\n",
      "Train Epoch: 20 [33600/60000 (56%)]\tLoss: 0.007883\n",
      "Train Epoch: 20 [35200/60000 (59%)]\tLoss: 0.004886\n",
      "Train Epoch: 20 [36800/60000 (61%)]\tLoss: 0.007850\n",
      "Train Epoch: 20 [38400/60000 (64%)]\tLoss: 0.007366\n",
      "Train Epoch: 20 [40000/60000 (67%)]\tLoss: 0.006516\n",
      "Train Epoch: 20 [41600/60000 (69%)]\tLoss: 0.007749\n",
      "Train Epoch: 20 [43200/60000 (72%)]\tLoss: 0.007685\n",
      "Train Epoch: 20 [44800/60000 (75%)]\tLoss: 0.005283\n",
      "Train Epoch: 20 [46400/60000 (77%)]\tLoss: 0.009313\n",
      "Train Epoch: 20 [48000/60000 (80%)]\tLoss: 0.006663\n",
      "Train Epoch: 20 [49600/60000 (83%)]\tLoss: 0.007617\n",
      "Train Epoch: 20 [51200/60000 (85%)]\tLoss: 0.007986\n",
      "Train Epoch: 20 [52800/60000 (88%)]\tLoss: 0.008280\n",
      "Train Epoch: 20 [54400/60000 (91%)]\tLoss: 0.006380\n",
      "Train Epoch: 20 [56000/60000 (93%)]\tLoss: 0.007313\n",
      "Train Epoch: 20 [57600/60000 (96%)]\tLoss: 0.006234\n",
      "Train Epoch: 20 [59200/60000 (99%)]\tLoss: 0.011970\n",
      "\n",
      "Test set: Avg. loss: 0.0639, Accuracy: 4930/5000 (98.60%)\n",
      "\n",
      "Train Epoch: 21 [0/60000 (0%)]\tLoss: 0.009210\n",
      "Train Epoch: 21 [1600/60000 (3%)]\tLoss: 0.007637\n",
      "Train Epoch: 21 [3200/60000 (5%)]\tLoss: 0.006583\n",
      "Train Epoch: 21 [4800/60000 (8%)]\tLoss: 0.006582\n",
      "Train Epoch: 21 [6400/60000 (11%)]\tLoss: 0.006301\n",
      "Train Epoch: 21 [8000/60000 (13%)]\tLoss: 0.007303\n",
      "Train Epoch: 21 [9600/60000 (16%)]\tLoss: 0.006633\n",
      "Train Epoch: 21 [11200/60000 (19%)]\tLoss: 0.008154\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 21 [12800/60000 (21%)]\tLoss: 0.007348\n",
      "Train Epoch: 21 [14400/60000 (24%)]\tLoss: 0.005353\n",
      "Train Epoch: 21 [16000/60000 (27%)]\tLoss: 0.006996\n",
      "Train Epoch: 21 [17600/60000 (29%)]\tLoss: 0.006406\n",
      "Train Epoch: 21 [19200/60000 (32%)]\tLoss: 0.007763\n",
      "Train Epoch: 21 [20800/60000 (35%)]\tLoss: 0.009701\n",
      "Train Epoch: 21 [22400/60000 (37%)]\tLoss: 0.005291\n",
      "Train Epoch: 21 [24000/60000 (40%)]\tLoss: 0.006842\n",
      "Train Epoch: 21 [25600/60000 (43%)]\tLoss: 0.007047\n",
      "Train Epoch: 21 [27200/60000 (45%)]\tLoss: 0.006183\n",
      "Train Epoch: 21 [28800/60000 (48%)]\tLoss: 0.007196\n",
      "Train Epoch: 21 [30400/60000 (51%)]\tLoss: 0.006002\n",
      "Train Epoch: 21 [32000/60000 (53%)]\tLoss: 0.006648\n",
      "Train Epoch: 21 [33600/60000 (56%)]\tLoss: 0.006683\n",
      "Train Epoch: 21 [35200/60000 (59%)]\tLoss: 0.010135\n",
      "Train Epoch: 21 [36800/60000 (61%)]\tLoss: 0.006942\n",
      "Train Epoch: 21 [38400/60000 (64%)]\tLoss: 0.006987\n",
      "Train Epoch: 21 [40000/60000 (67%)]\tLoss: 0.005422\n",
      "Train Epoch: 21 [41600/60000 (69%)]\tLoss: 0.008366\n",
      "Train Epoch: 21 [43200/60000 (72%)]\tLoss: 0.012015\n",
      "Train Epoch: 21 [44800/60000 (75%)]\tLoss: 0.006474\n",
      "Train Epoch: 21 [46400/60000 (77%)]\tLoss: 0.006923\n",
      "Train Epoch: 21 [48000/60000 (80%)]\tLoss: 0.007463\n",
      "Train Epoch: 21 [49600/60000 (83%)]\tLoss: 0.008362\n",
      "Train Epoch: 21 [51200/60000 (85%)]\tLoss: 0.008592\n",
      "Train Epoch: 21 [52800/60000 (88%)]\tLoss: 0.005339\n",
      "Train Epoch: 21 [54400/60000 (91%)]\tLoss: 0.007316\n",
      "Train Epoch: 21 [56000/60000 (93%)]\tLoss: 0.006076\n",
      "Train Epoch: 21 [57600/60000 (96%)]\tLoss: 0.007067\n",
      "Train Epoch: 21 [59200/60000 (99%)]\tLoss: 0.007302\n",
      "\n",
      "Test set: Avg. loss: 0.0539, Accuracy: 4945/5000 (98.90%)\n",
      "\n",
      "Train Epoch: 22 [0/60000 (0%)]\tLoss: 0.007824\n",
      "Train Epoch: 22 [1600/60000 (3%)]\tLoss: 0.005863\n",
      "Train Epoch: 22 [3200/60000 (5%)]\tLoss: 0.007374\n",
      "Train Epoch: 22 [4800/60000 (8%)]\tLoss: 0.006994\n",
      "Train Epoch: 22 [6400/60000 (11%)]\tLoss: 0.004486\n",
      "Train Epoch: 22 [8000/60000 (13%)]\tLoss: 0.008101\n",
      "Train Epoch: 22 [9600/60000 (16%)]\tLoss: 0.007188\n",
      "Train Epoch: 22 [11200/60000 (19%)]\tLoss: 0.007865\n",
      "Train Epoch: 22 [12800/60000 (21%)]\tLoss: 0.007954\n",
      "Train Epoch: 22 [14400/60000 (24%)]\tLoss: 0.006633\n",
      "Train Epoch: 22 [16000/60000 (27%)]\tLoss: 0.006414\n",
      "Train Epoch: 22 [17600/60000 (29%)]\tLoss: 0.006607\n",
      "Train Epoch: 22 [19200/60000 (32%)]\tLoss: 0.004317\n",
      "Train Epoch: 22 [20800/60000 (35%)]\tLoss: 0.009370\n",
      "Train Epoch: 22 [22400/60000 (37%)]\tLoss: 0.006848\n",
      "Train Epoch: 22 [24000/60000 (40%)]\tLoss: 0.007330\n",
      "Train Epoch: 22 [25600/60000 (43%)]\tLoss: 0.008110\n",
      "Train Epoch: 22 [27200/60000 (45%)]\tLoss: 0.006293\n",
      "Train Epoch: 22 [28800/60000 (48%)]\tLoss: 0.008854\n",
      "Train Epoch: 22 [30400/60000 (51%)]\tLoss: 0.006862\n",
      "Train Epoch: 22 [32000/60000 (53%)]\tLoss: 0.004960\n",
      "Train Epoch: 22 [33600/60000 (56%)]\tLoss: 0.005416\n",
      "Train Epoch: 22 [35200/60000 (59%)]\tLoss: 0.006777\n",
      "Train Epoch: 22 [36800/60000 (61%)]\tLoss: 0.006690\n",
      "Train Epoch: 22 [38400/60000 (64%)]\tLoss: 0.008052\n",
      "Train Epoch: 22 [40000/60000 (67%)]\tLoss: 0.004105\n",
      "Train Epoch: 22 [41600/60000 (69%)]\tLoss: 0.005645\n",
      "Train Epoch: 22 [43200/60000 (72%)]\tLoss: 0.004492\n",
      "Train Epoch: 22 [44800/60000 (75%)]\tLoss: 0.008188\n",
      "Train Epoch: 22 [46400/60000 (77%)]\tLoss: 0.006525\n",
      "Train Epoch: 22 [48000/60000 (80%)]\tLoss: 0.005657\n",
      "Train Epoch: 22 [49600/60000 (83%)]\tLoss: 0.005590\n",
      "Train Epoch: 22 [51200/60000 (85%)]\tLoss: 0.004551\n",
      "Train Epoch: 22 [52800/60000 (88%)]\tLoss: 0.006365\n",
      "Train Epoch: 22 [54400/60000 (91%)]\tLoss: 0.006735\n",
      "Train Epoch: 22 [56000/60000 (93%)]\tLoss: 0.005198\n",
      "Train Epoch: 22 [57600/60000 (96%)]\tLoss: 0.006642\n",
      "Train Epoch: 22 [59200/60000 (99%)]\tLoss: 0.006105\n",
      "\n",
      "Test set: Avg. loss: 0.0618, Accuracy: 4932/5000 (98.64%)\n",
      "\n",
      "Train Epoch: 23 [0/60000 (0%)]\tLoss: 0.007694\n",
      "Train Epoch: 23 [1600/60000 (3%)]\tLoss: 0.005897\n",
      "Train Epoch: 23 [3200/60000 (5%)]\tLoss: 0.004863\n",
      "Train Epoch: 23 [4800/60000 (8%)]\tLoss: 0.005326\n",
      "Train Epoch: 23 [6400/60000 (11%)]\tLoss: 0.006503\n",
      "Train Epoch: 23 [8000/60000 (13%)]\tLoss: 0.006279\n",
      "Train Epoch: 23 [9600/60000 (16%)]\tLoss: 0.006299\n",
      "Train Epoch: 23 [11200/60000 (19%)]\tLoss: 0.006776\n",
      "Train Epoch: 23 [12800/60000 (21%)]\tLoss: 0.006582\n",
      "Train Epoch: 23 [14400/60000 (24%)]\tLoss: 0.006865\n",
      "Train Epoch: 23 [16000/60000 (27%)]\tLoss: 0.006540\n",
      "Train Epoch: 23 [17600/60000 (29%)]\tLoss: 0.007985\n",
      "Train Epoch: 23 [19200/60000 (32%)]\tLoss: 0.005795\n",
      "Train Epoch: 23 [20800/60000 (35%)]\tLoss: 0.004544\n",
      "Train Epoch: 23 [22400/60000 (37%)]\tLoss: 0.006927\n",
      "Train Epoch: 23 [24000/60000 (40%)]\tLoss: 0.007305\n",
      "Train Epoch: 23 [25600/60000 (43%)]\tLoss: 0.007973\n",
      "Train Epoch: 23 [27200/60000 (45%)]\tLoss: 0.005781\n",
      "Train Epoch: 23 [28800/60000 (48%)]\tLoss: 0.006218\n",
      "Train Epoch: 23 [30400/60000 (51%)]\tLoss: 0.007915\n",
      "Train Epoch: 23 [32000/60000 (53%)]\tLoss: 0.009767\n",
      "Train Epoch: 23 [33600/60000 (56%)]\tLoss: 0.006635\n",
      "Train Epoch: 23 [35200/60000 (59%)]\tLoss: 0.009693\n",
      "Train Epoch: 23 [36800/60000 (61%)]\tLoss: 0.006639\n",
      "Train Epoch: 23 [38400/60000 (64%)]\tLoss: 0.005091\n",
      "Train Epoch: 23 [40000/60000 (67%)]\tLoss: 0.006023\n",
      "Train Epoch: 23 [41600/60000 (69%)]\tLoss: 0.006301\n",
      "Train Epoch: 23 [43200/60000 (72%)]\tLoss: 0.006814\n",
      "Train Epoch: 23 [44800/60000 (75%)]\tLoss: 0.005143\n",
      "Train Epoch: 23 [46400/60000 (77%)]\tLoss: 0.006670\n",
      "Train Epoch: 23 [48000/60000 (80%)]\tLoss: 0.006494\n",
      "Train Epoch: 23 [49600/60000 (83%)]\tLoss: 0.005395\n",
      "Train Epoch: 23 [51200/60000 (85%)]\tLoss: 0.005940\n",
      "Train Epoch: 23 [52800/60000 (88%)]\tLoss: 0.003719\n",
      "Train Epoch: 23 [54400/60000 (91%)]\tLoss: 0.007041\n",
      "Train Epoch: 23 [56000/60000 (93%)]\tLoss: 0.007219\n",
      "Train Epoch: 23 [57600/60000 (96%)]\tLoss: 0.008080\n",
      "Train Epoch: 23 [59200/60000 (99%)]\tLoss: 0.003850\n",
      "\n",
      "Test set: Avg. loss: 0.0497, Accuracy: 4946/5000 (98.92%)\n",
      "\n",
      "Train Epoch: 24 [0/60000 (0%)]\tLoss: 0.006874\n",
      "Train Epoch: 24 [1600/60000 (3%)]\tLoss: 0.004309\n",
      "Train Epoch: 24 [3200/60000 (5%)]\tLoss: 0.006268\n",
      "Train Epoch: 24 [4800/60000 (8%)]\tLoss: 0.003929\n",
      "Train Epoch: 24 [6400/60000 (11%)]\tLoss: 0.008667\n",
      "Train Epoch: 24 [8000/60000 (13%)]\tLoss: 0.007673\n",
      "Train Epoch: 24 [9600/60000 (16%)]\tLoss: 0.006155\n",
      "Train Epoch: 24 [11200/60000 (19%)]\tLoss: 0.007394\n",
      "Train Epoch: 24 [12800/60000 (21%)]\tLoss: 0.005983\n",
      "Train Epoch: 24 [14400/60000 (24%)]\tLoss: 0.005078\n",
      "Train Epoch: 24 [16000/60000 (27%)]\tLoss: 0.010827\n",
      "Train Epoch: 24 [17600/60000 (29%)]\tLoss: 0.004530\n",
      "Train Epoch: 24 [19200/60000 (32%)]\tLoss: 0.008197\n",
      "Train Epoch: 24 [20800/60000 (35%)]\tLoss: 0.006110\n",
      "Train Epoch: 24 [22400/60000 (37%)]\tLoss: 0.007823\n",
      "Train Epoch: 24 [24000/60000 (40%)]\tLoss: 0.005272\n",
      "Train Epoch: 24 [25600/60000 (43%)]\tLoss: 0.006214\n",
      "Train Epoch: 24 [27200/60000 (45%)]\tLoss: 0.008848\n",
      "Train Epoch: 24 [28800/60000 (48%)]\tLoss: 0.006271\n",
      "Train Epoch: 24 [30400/60000 (51%)]\tLoss: 0.009518\n",
      "Train Epoch: 24 [32000/60000 (53%)]\tLoss: 0.007358\n",
      "Train Epoch: 24 [33600/60000 (56%)]\tLoss: 0.006714\n",
      "Train Epoch: 24 [35200/60000 (59%)]\tLoss: 0.004975\n",
      "Train Epoch: 24 [36800/60000 (61%)]\tLoss: 0.004475\n",
      "Train Epoch: 24 [38400/60000 (64%)]\tLoss: 0.005888\n",
      "Train Epoch: 24 [40000/60000 (67%)]\tLoss: 0.005586\n",
      "Train Epoch: 24 [41600/60000 (69%)]\tLoss: 0.005933\n",
      "Train Epoch: 24 [43200/60000 (72%)]\tLoss: 0.004846\n",
      "Train Epoch: 24 [44800/60000 (75%)]\tLoss: 0.007866\n",
      "Train Epoch: 24 [46400/60000 (77%)]\tLoss: 0.006280\n",
      "Train Epoch: 24 [48000/60000 (80%)]\tLoss: 0.008154\n",
      "Train Epoch: 24 [49600/60000 (83%)]\tLoss: 0.007586\n",
      "Train Epoch: 24 [51200/60000 (85%)]\tLoss: 0.007237\n",
      "Train Epoch: 24 [52800/60000 (88%)]\tLoss: 0.005171\n",
      "Train Epoch: 24 [54400/60000 (91%)]\tLoss: 0.007459\n",
      "Train Epoch: 24 [56000/60000 (93%)]\tLoss: 0.006007\n",
      "Train Epoch: 24 [57600/60000 (96%)]\tLoss: 0.003231\n",
      "Train Epoch: 24 [59200/60000 (99%)]\tLoss: 0.007158\n",
      "\n",
      "Test set: Avg. loss: 0.0452, Accuracy: 4956/5000 (99.12%)\n",
      "\n",
      "Train Epoch: 25 [0/60000 (0%)]\tLoss: 0.004063\n",
      "Train Epoch: 25 [1600/60000 (3%)]\tLoss: 0.006354\n",
      "Train Epoch: 25 [3200/60000 (5%)]\tLoss: 0.006132\n",
      "Train Epoch: 25 [4800/60000 (8%)]\tLoss: 0.006790\n",
      "Train Epoch: 25 [6400/60000 (11%)]\tLoss: 0.006770\n",
      "Train Epoch: 25 [8000/60000 (13%)]\tLoss: 0.004062\n",
      "Train Epoch: 25 [9600/60000 (16%)]\tLoss: 0.004954\n",
      "Train Epoch: 25 [11200/60000 (19%)]\tLoss: 0.005077\n",
      "Train Epoch: 25 [12800/60000 (21%)]\tLoss: 0.007085\n",
      "Train Epoch: 25 [14400/60000 (24%)]\tLoss: 0.007645\n",
      "Train Epoch: 25 [16000/60000 (27%)]\tLoss: 0.004675\n",
      "Train Epoch: 25 [17600/60000 (29%)]\tLoss: 0.007354\n",
      "Train Epoch: 25 [19200/60000 (32%)]\tLoss: 0.008134\n",
      "Train Epoch: 25 [20800/60000 (35%)]\tLoss: 0.005889\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 25 [22400/60000 (37%)]\tLoss: 0.005491\n",
      "Train Epoch: 25 [24000/60000 (40%)]\tLoss: 0.004139\n",
      "Train Epoch: 25 [25600/60000 (43%)]\tLoss: 0.004248\n",
      "Train Epoch: 25 [27200/60000 (45%)]\tLoss: 0.005234\n",
      "Train Epoch: 25 [28800/60000 (48%)]\tLoss: 0.005533\n",
      "Train Epoch: 25 [30400/60000 (51%)]\tLoss: 0.003975\n",
      "Train Epoch: 25 [32000/60000 (53%)]\tLoss: 0.007704\n",
      "Train Epoch: 25 [33600/60000 (56%)]\tLoss: 0.003831\n",
      "Train Epoch: 25 [35200/60000 (59%)]\tLoss: 0.007733\n",
      "Train Epoch: 25 [36800/60000 (61%)]\tLoss: 0.006999\n",
      "Train Epoch: 25 [38400/60000 (64%)]\tLoss: 0.006670\n",
      "Train Epoch: 25 [40000/60000 (67%)]\tLoss: 0.006490\n",
      "Train Epoch: 25 [41600/60000 (69%)]\tLoss: 0.013393\n",
      "Train Epoch: 25 [43200/60000 (72%)]\tLoss: 0.006822\n",
      "Train Epoch: 25 [44800/60000 (75%)]\tLoss: 0.004681\n",
      "Train Epoch: 25 [46400/60000 (77%)]\tLoss: 0.006869\n",
      "Train Epoch: 25 [48000/60000 (80%)]\tLoss: 0.004941\n",
      "Train Epoch: 25 [49600/60000 (83%)]\tLoss: 0.007052\n",
      "Train Epoch: 25 [51200/60000 (85%)]\tLoss: 0.004645\n",
      "Train Epoch: 25 [52800/60000 (88%)]\tLoss: 0.005241\n",
      "Train Epoch: 25 [54400/60000 (91%)]\tLoss: 0.004632\n",
      "Train Epoch: 25 [56000/60000 (93%)]\tLoss: 0.006007\n",
      "Train Epoch: 25 [57600/60000 (96%)]\tLoss: 0.005215\n",
      "Train Epoch: 25 [59200/60000 (99%)]\tLoss: 0.008789\n",
      "\n",
      "Test set: Avg. loss: 0.0505, Accuracy: 4934/5000 (98.68%)\n",
      "\n",
      "Train Epoch: 26 [0/60000 (0%)]\tLoss: 0.005250\n",
      "Train Epoch: 26 [1600/60000 (3%)]\tLoss: 0.003462\n",
      "Train Epoch: 26 [3200/60000 (5%)]\tLoss: 0.005159\n",
      "Train Epoch: 26 [4800/60000 (8%)]\tLoss: 0.006238\n",
      "Train Epoch: 26 [6400/60000 (11%)]\tLoss: 0.005484\n",
      "Train Epoch: 26 [8000/60000 (13%)]\tLoss: 0.009039\n",
      "Train Epoch: 26 [9600/60000 (16%)]\tLoss: 0.004417\n",
      "Train Epoch: 26 [11200/60000 (19%)]\tLoss: 0.003798\n",
      "Train Epoch: 26 [12800/60000 (21%)]\tLoss: 0.004323\n",
      "Train Epoch: 26 [14400/60000 (24%)]\tLoss: 0.008420\n",
      "Train Epoch: 26 [16000/60000 (27%)]\tLoss: 0.006602\n",
      "Train Epoch: 26 [17600/60000 (29%)]\tLoss: 0.006344\n",
      "Train Epoch: 26 [19200/60000 (32%)]\tLoss: 0.005240\n",
      "Train Epoch: 26 [20800/60000 (35%)]\tLoss: 0.004057\n",
      "Train Epoch: 26 [22400/60000 (37%)]\tLoss: 0.005244\n",
      "Train Epoch: 26 [24000/60000 (40%)]\tLoss: 0.004159\n",
      "Train Epoch: 26 [25600/60000 (43%)]\tLoss: 0.004676\n",
      "Train Epoch: 26 [27200/60000 (45%)]\tLoss: 0.003433\n",
      "Train Epoch: 26 [28800/60000 (48%)]\tLoss: 0.005338\n",
      "Train Epoch: 26 [30400/60000 (51%)]\tLoss: 0.003161\n",
      "Train Epoch: 26 [32000/60000 (53%)]\tLoss: 0.005587\n",
      "Train Epoch: 26 [33600/60000 (56%)]\tLoss: 0.005646\n",
      "Train Epoch: 26 [35200/60000 (59%)]\tLoss: 0.010494\n",
      "Train Epoch: 26 [36800/60000 (61%)]\tLoss: 0.004356\n",
      "Train Epoch: 26 [38400/60000 (64%)]\tLoss: 0.003006\n",
      "Train Epoch: 26 [40000/60000 (67%)]\tLoss: 0.006789\n",
      "Train Epoch: 26 [41600/60000 (69%)]\tLoss: 0.004174\n",
      "Train Epoch: 26 [43200/60000 (72%)]\tLoss: 0.003379\n",
      "Train Epoch: 26 [44800/60000 (75%)]\tLoss: 0.005596\n",
      "Train Epoch: 26 [46400/60000 (77%)]\tLoss: 0.004315\n",
      "Train Epoch: 26 [48000/60000 (80%)]\tLoss: 0.004459\n",
      "Train Epoch: 26 [49600/60000 (83%)]\tLoss: 0.004487\n",
      "Train Epoch: 26 [51200/60000 (85%)]\tLoss: 0.005790\n",
      "Train Epoch: 26 [52800/60000 (88%)]\tLoss: 0.005862\n",
      "Train Epoch: 26 [54400/60000 (91%)]\tLoss: 0.009592\n",
      "Train Epoch: 26 [56000/60000 (93%)]\tLoss: 0.005182\n",
      "Train Epoch: 26 [57600/60000 (96%)]\tLoss: 0.005463\n",
      "Train Epoch: 26 [59200/60000 (99%)]\tLoss: 0.005276\n",
      "\n",
      "Test set: Avg. loss: 0.0451, Accuracy: 4950/5000 (99.00%)\n",
      "\n",
      "Train Epoch: 27 [0/60000 (0%)]\tLoss: 0.008208\n",
      "Train Epoch: 27 [1600/60000 (3%)]\tLoss: 0.005249\n",
      "Train Epoch: 27 [3200/60000 (5%)]\tLoss: 0.002626\n",
      "Train Epoch: 27 [4800/60000 (8%)]\tLoss: 0.004059\n",
      "Train Epoch: 27 [6400/60000 (11%)]\tLoss: 0.006931\n",
      "Train Epoch: 27 [8000/60000 (13%)]\tLoss: 0.006873\n",
      "Train Epoch: 27 [9600/60000 (16%)]\tLoss: 0.005514\n",
      "Train Epoch: 27 [11200/60000 (19%)]\tLoss: 0.006654\n",
      "Train Epoch: 27 [12800/60000 (21%)]\tLoss: 0.007449\n",
      "Train Epoch: 27 [14400/60000 (24%)]\tLoss: 0.005006\n",
      "Train Epoch: 27 [16000/60000 (27%)]\tLoss: 0.006124\n",
      "Train Epoch: 27 [17600/60000 (29%)]\tLoss: 0.003261\n",
      "Train Epoch: 27 [19200/60000 (32%)]\tLoss: 0.006142\n",
      "Train Epoch: 27 [20800/60000 (35%)]\tLoss: 0.003187\n",
      "Train Epoch: 27 [22400/60000 (37%)]\tLoss: 0.004633\n",
      "Train Epoch: 27 [24000/60000 (40%)]\tLoss: 0.005269\n",
      "Train Epoch: 27 [25600/60000 (43%)]\tLoss: 0.005867\n",
      "Train Epoch: 27 [27200/60000 (45%)]\tLoss: 0.003502\n",
      "Train Epoch: 27 [28800/60000 (48%)]\tLoss: 0.006460\n",
      "Train Epoch: 27 [30400/60000 (51%)]\tLoss: 0.008364\n",
      "Train Epoch: 27 [32000/60000 (53%)]\tLoss: 0.005743\n",
      "Train Epoch: 27 [33600/60000 (56%)]\tLoss: 0.007429\n",
      "Train Epoch: 27 [35200/60000 (59%)]\tLoss: 0.004215\n",
      "Train Epoch: 27 [36800/60000 (61%)]\tLoss: 0.005966\n",
      "Train Epoch: 27 [38400/60000 (64%)]\tLoss: 0.004657\n",
      "Train Epoch: 27 [40000/60000 (67%)]\tLoss: 0.007366\n",
      "Train Epoch: 27 [41600/60000 (69%)]\tLoss: 0.010809\n",
      "Train Epoch: 27 [43200/60000 (72%)]\tLoss: 0.004276\n",
      "Train Epoch: 27 [44800/60000 (75%)]\tLoss: 0.007073\n",
      "Train Epoch: 27 [46400/60000 (77%)]\tLoss: 0.004245\n",
      "Train Epoch: 27 [48000/60000 (80%)]\tLoss: 0.004315\n",
      "Train Epoch: 27 [49600/60000 (83%)]\tLoss: 0.005893\n",
      "Train Epoch: 27 [51200/60000 (85%)]\tLoss: 0.011334\n",
      "Train Epoch: 27 [52800/60000 (88%)]\tLoss: 0.005812\n",
      "Train Epoch: 27 [54400/60000 (91%)]\tLoss: 0.004755\n",
      "Train Epoch: 27 [56000/60000 (93%)]\tLoss: 0.004413\n",
      "Train Epoch: 27 [57600/60000 (96%)]\tLoss: 0.006066\n",
      "Train Epoch: 27 [59200/60000 (99%)]\tLoss: 0.002848\n",
      "\n",
      "Test set: Avg. loss: 0.0501, Accuracy: 4949/5000 (98.98%)\n",
      "\n",
      "Train Epoch: 28 [0/60000 (0%)]\tLoss: 0.006772\n",
      "Train Epoch: 28 [1600/60000 (3%)]\tLoss: 0.004396\n",
      "Train Epoch: 28 [3200/60000 (5%)]\tLoss: 0.004284\n",
      "Train Epoch: 28 [4800/60000 (8%)]\tLoss: 0.006554\n",
      "Train Epoch: 28 [6400/60000 (11%)]\tLoss: 0.005867\n",
      "Train Epoch: 28 [8000/60000 (13%)]\tLoss: 0.005543\n",
      "Train Epoch: 28 [9600/60000 (16%)]\tLoss: 0.005347\n",
      "Train Epoch: 28 [11200/60000 (19%)]\tLoss: 0.004294\n",
      "Train Epoch: 28 [12800/60000 (21%)]\tLoss: 0.006420\n",
      "Train Epoch: 28 [14400/60000 (24%)]\tLoss: 0.005718\n",
      "Train Epoch: 28 [16000/60000 (27%)]\tLoss: 0.004424\n",
      "Train Epoch: 28 [17600/60000 (29%)]\tLoss: 0.005327\n",
      "Train Epoch: 28 [19200/60000 (32%)]\tLoss: 0.005214\n",
      "Train Epoch: 28 [20800/60000 (35%)]\tLoss: 0.005023\n",
      "Train Epoch: 28 [22400/60000 (37%)]\tLoss: 0.007281\n",
      "Train Epoch: 28 [24000/60000 (40%)]\tLoss: 0.005471\n",
      "Train Epoch: 28 [25600/60000 (43%)]\tLoss: 0.005247\n",
      "Train Epoch: 28 [27200/60000 (45%)]\tLoss: 0.006360\n",
      "Train Epoch: 28 [28800/60000 (48%)]\tLoss: 0.004583\n",
      "Train Epoch: 28 [30400/60000 (51%)]\tLoss: 0.006825\n",
      "Train Epoch: 28 [32000/60000 (53%)]\tLoss: 0.005472\n",
      "Train Epoch: 28 [33600/60000 (56%)]\tLoss: 0.004302\n",
      "Train Epoch: 28 [35200/60000 (59%)]\tLoss: 0.006691\n",
      "Train Epoch: 28 [36800/60000 (61%)]\tLoss: 0.004117\n",
      "Train Epoch: 28 [38400/60000 (64%)]\tLoss: 0.006291\n",
      "Train Epoch: 28 [40000/60000 (67%)]\tLoss: 0.005814\n",
      "Train Epoch: 28 [41600/60000 (69%)]\tLoss: 0.006711\n",
      "Train Epoch: 28 [43200/60000 (72%)]\tLoss: 0.005242\n",
      "Train Epoch: 28 [44800/60000 (75%)]\tLoss: 0.006273\n",
      "Train Epoch: 28 [46400/60000 (77%)]\tLoss: 0.006538\n",
      "Train Epoch: 28 [48000/60000 (80%)]\tLoss: 0.005327\n",
      "Train Epoch: 28 [49600/60000 (83%)]\tLoss: 0.004223\n",
      "Train Epoch: 28 [51200/60000 (85%)]\tLoss: 0.007604\n",
      "Train Epoch: 28 [52800/60000 (88%)]\tLoss: 0.003560\n",
      "Train Epoch: 28 [54400/60000 (91%)]\tLoss: 0.006180\n",
      "Train Epoch: 28 [56000/60000 (93%)]\tLoss: 0.007754\n",
      "Train Epoch: 28 [57600/60000 (96%)]\tLoss: 0.007007\n",
      "Train Epoch: 28 [59200/60000 (99%)]\tLoss: 0.003526\n",
      "\n",
      "Test set: Avg. loss: 0.0397, Accuracy: 4949/5000 (98.98%)\n",
      "\n",
      "Train Epoch: 29 [0/60000 (0%)]\tLoss: 0.005900\n",
      "Train Epoch: 29 [1600/60000 (3%)]\tLoss: 0.004030\n",
      "Train Epoch: 29 [3200/60000 (5%)]\tLoss: 0.004021\n",
      "Train Epoch: 29 [4800/60000 (8%)]\tLoss: 0.006044\n",
      "Train Epoch: 29 [6400/60000 (11%)]\tLoss: 0.004971\n",
      "Train Epoch: 29 [8000/60000 (13%)]\tLoss: 0.006334\n",
      "Train Epoch: 29 [9600/60000 (16%)]\tLoss: 0.009327\n",
      "Train Epoch: 29 [11200/60000 (19%)]\tLoss: 0.006488\n",
      "Train Epoch: 29 [12800/60000 (21%)]\tLoss: 0.003591\n",
      "Train Epoch: 29 [14400/60000 (24%)]\tLoss: 0.006362\n",
      "Train Epoch: 29 [16000/60000 (27%)]\tLoss: 0.006369\n",
      "Train Epoch: 29 [17600/60000 (29%)]\tLoss: 0.004967\n",
      "Train Epoch: 29 [19200/60000 (32%)]\tLoss: 0.004993\n",
      "Train Epoch: 29 [20800/60000 (35%)]\tLoss: 0.005695\n",
      "Train Epoch: 29 [22400/60000 (37%)]\tLoss: 0.004778\n",
      "Train Epoch: 29 [24000/60000 (40%)]\tLoss: 0.006457\n",
      "Train Epoch: 29 [25600/60000 (43%)]\tLoss: 0.007952\n",
      "Train Epoch: 29 [27200/60000 (45%)]\tLoss: 0.004661\n",
      "Train Epoch: 29 [28800/60000 (48%)]\tLoss: 0.005777\n",
      "Train Epoch: 29 [30400/60000 (51%)]\tLoss: 0.004264\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 29 [32000/60000 (53%)]\tLoss: 0.004560\n",
      "Train Epoch: 29 [33600/60000 (56%)]\tLoss: 0.008928\n",
      "Train Epoch: 29 [35200/60000 (59%)]\tLoss: 0.003786\n",
      "Train Epoch: 29 [36800/60000 (61%)]\tLoss: 0.005205\n",
      "Train Epoch: 29 [38400/60000 (64%)]\tLoss: 0.004415\n",
      "Train Epoch: 29 [40000/60000 (67%)]\tLoss: 0.005945\n",
      "Train Epoch: 29 [41600/60000 (69%)]\tLoss: 0.003993\n",
      "Train Epoch: 29 [43200/60000 (72%)]\tLoss: 0.006298\n",
      "Train Epoch: 29 [44800/60000 (75%)]\tLoss: 0.005229\n",
      "Train Epoch: 29 [46400/60000 (77%)]\tLoss: 0.007346\n",
      "Train Epoch: 29 [48000/60000 (80%)]\tLoss: 0.005479\n",
      "Train Epoch: 29 [49600/60000 (83%)]\tLoss: 0.005069\n",
      "Train Epoch: 29 [51200/60000 (85%)]\tLoss: 0.005763\n",
      "Train Epoch: 29 [52800/60000 (88%)]\tLoss: 0.003221\n",
      "Train Epoch: 29 [54400/60000 (91%)]\tLoss: 0.004127\n",
      "Train Epoch: 29 [56000/60000 (93%)]\tLoss: 0.004861\n",
      "Train Epoch: 29 [57600/60000 (96%)]\tLoss: 0.003752\n",
      "Train Epoch: 29 [59200/60000 (99%)]\tLoss: 0.003490\n",
      "\n",
      "Test set: Avg. loss: 0.0435, Accuracy: 4956/5000 (99.12%)\n",
      "\n",
      "Train Epoch: 30 [0/60000 (0%)]\tLoss: 0.006034\n",
      "Train Epoch: 30 [1600/60000 (3%)]\tLoss: 0.004838\n",
      "Train Epoch: 30 [3200/60000 (5%)]\tLoss: 0.004651\n",
      "Train Epoch: 30 [4800/60000 (8%)]\tLoss: 0.005443\n",
      "Train Epoch: 30 [6400/60000 (11%)]\tLoss: 0.002779\n",
      "Train Epoch: 30 [8000/60000 (13%)]\tLoss: 0.004942\n",
      "Train Epoch: 30 [9600/60000 (16%)]\tLoss: 0.005394\n",
      "Train Epoch: 30 [11200/60000 (19%)]\tLoss: 0.004053\n",
      "Train Epoch: 30 [12800/60000 (21%)]\tLoss: 0.003768\n",
      "Train Epoch: 30 [14400/60000 (24%)]\tLoss: 0.004237\n",
      "Train Epoch: 30 [16000/60000 (27%)]\tLoss: 0.004758\n",
      "Train Epoch: 30 [17600/60000 (29%)]\tLoss: 0.004760\n",
      "Train Epoch: 30 [19200/60000 (32%)]\tLoss: 0.005040\n",
      "Train Epoch: 30 [20800/60000 (35%)]\tLoss: 0.005926\n",
      "Train Epoch: 30 [22400/60000 (37%)]\tLoss: 0.003796\n",
      "Train Epoch: 30 [24000/60000 (40%)]\tLoss: 0.003566\n",
      "Train Epoch: 30 [25600/60000 (43%)]\tLoss: 0.003770\n",
      "Train Epoch: 30 [27200/60000 (45%)]\tLoss: 0.006168\n",
      "Train Epoch: 30 [28800/60000 (48%)]\tLoss: 0.003906\n",
      "Train Epoch: 30 [30400/60000 (51%)]\tLoss: 0.003636\n",
      "Train Epoch: 30 [32000/60000 (53%)]\tLoss: 0.006213\n",
      "Train Epoch: 30 [33600/60000 (56%)]\tLoss: 0.005150\n",
      "Train Epoch: 30 [35200/60000 (59%)]\tLoss: 0.004781\n",
      "Train Epoch: 30 [36800/60000 (61%)]\tLoss: 0.006939\n",
      "Train Epoch: 30 [38400/60000 (64%)]\tLoss: 0.005909\n",
      "Train Epoch: 30 [40000/60000 (67%)]\tLoss: 0.003489\n",
      "Train Epoch: 30 [41600/60000 (69%)]\tLoss: 0.008320\n",
      "Train Epoch: 30 [43200/60000 (72%)]\tLoss: 0.003487\n",
      "Train Epoch: 30 [44800/60000 (75%)]\tLoss: 0.005519\n",
      "Train Epoch: 30 [46400/60000 (77%)]\tLoss: 0.007962\n",
      "Train Epoch: 30 [48000/60000 (80%)]\tLoss: 0.005028\n",
      "Train Epoch: 30 [49600/60000 (83%)]\tLoss: 0.006475\n",
      "Train Epoch: 30 [51200/60000 (85%)]\tLoss: 0.005379\n",
      "Train Epoch: 30 [52800/60000 (88%)]\tLoss: 0.004132\n",
      "Train Epoch: 30 [54400/60000 (91%)]\tLoss: 0.006964\n",
      "Train Epoch: 30 [56000/60000 (93%)]\tLoss: 0.007047\n",
      "Train Epoch: 30 [57600/60000 (96%)]\tLoss: 0.006137\n",
      "Train Epoch: 30 [59200/60000 (99%)]\tLoss: 0.004543\n",
      "\n",
      "Test set: Avg. loss: 0.0425, Accuracy: 4950/5000 (99.00%)\n",
      "\n",
      "Losses saved to cnn/ONN\\seed0\\loss.pkl\n",
      "Scores saved to cnn/ONN\\seed0\\scores.pkl\n",
      "Validation accuracy : 99.12%\n",
      "Validation scores saved to cnn/ONN\\seed0\\validation_score.pkl\n",
      "Validation scores are:\n",
      "\tseed0 : 99.12%\n",
      "seed0: Best score 4956/5\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmoAAAD9CAYAAAAMLlutAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAA9hAAAPYQGoP6dpAABL9klEQVR4nO3dd3hUZdoG8Ht6CpmEFAIpdAJkQhKaFBd0IQpIEyT7qYsBBEFXEFfYVValiBpXxC4Ki1jAla4gKEUQFgHpPUGKQArpkJkhZer7/REZiJQUJjknk/t3XXOZM3POnGfGMy/PeatCCCFARERERLKjlDoAIiIiIro5JmpEREREMsVEjYiIiEimmKgRERERyRQTNSIiIiKZYqJGREREJFNM1IiIiIhkSi11AHLgdDpx8eJF+Pn5QaFQSB0OEd2GEAJmsxlhYWFQKnmveT2WZUR1Q1XKMSZqAC5evIjIyEipwyCiKkhPT0dERITUYcgKyzKiuqUy5RgTNQB+fn4Ayr4wvV4vcTREdDsmkwmRkZGu3y1dw7KMqG6oSjnGRA1wNRHo9XoWbkR1BJv2bsSyjKhuqUw5xg4eRERERDLFRI2IiIhIptj0SSQTdrsdDodD6jBkQaVSQa1m8eROvL6u4fVFdQlr1IhkwGg0wmKxSB2GbFgsFhiNRqnD8Bi8vsrj9UV1CW8piCRmt9uhVqvh6+srdSiyodPpUFRU5PpuqPp4fd2I1xfVJbxCq2jD8Sx0bxmEAB+t1KGQh3A4HPzH4ibUajW/Gzfgd3hzvL7k7XB6Ib7YdR7HM42Y/WAMurcMkjSezMISbE3NwY+puThfUIS4iAD0bBWEHq2C0DTQp0ZHofMKraLVBzMRFerHRI2IiMiNrHYnfjiehc92nsfh9ELX86MW7cUnIzvjz+0aVen9hBDVTqCcToEjGYXYkpqLH1NzcDLbXO71CwXFWHvkIgAgPMAbPVoFuRK3Jv7e1TrnrTBRqwYhdQBEREQeIs9swX/3pGHJngvIM5f1pWys98LI7k1hKrVjwf9+wxNf7sd7D3fEwNgmFb7f+fwivLD6KPacuwSdWgkfrRreGhV8tGUPb60K3hoVlLdI4pxC4FimCflXrvXrDPLV4s/tGiGhfSO0a6zHofTL2HWmALvOFiCzsAQrD2Rg5YEMAMC9bUPw+Zi73PDNlJFForZixQrMnDkTVqsVI0eOxIwZM8q9fvDgQUycOBFFRUXw8fHBJ598gri4OBQXF6NRo0Zo3bq1a98DBw5ApVLVWKycY5Ooepo3b45t27ahefPm+OCDDzBv3jzY7Xb84x//wPjx46UOj+q4+nJ9OZxlVQUqZdX/MTpw4RLSL5WgTWgDtAppAC+Ne/6tPJltwvn8ItwX3bhKceWaS/HWxl/xzaFM2Bxln6tLs4YYfXdz9DM0hkZVNt4xwEeDNzf8iklfH0SRJRZ/6XrzZdKEEPhqTxpeW5+KEpsDWpUSVrsTpTZrtT5Xu8Z+6Nu+Efq2D0VcREC5z9Y82BfDOkZACIH0SyXYdTYfu38rS9wiGnpYjVp2djamTp2K/fv3IyAgAAMGDMDGjRvRr18/1z6PPfYYPv74Y/Tu3RubN29GUlISjhw5ggMHDqBv375Ys2ZNrcYsWKVGVG2HDh3CggULsH//fjidTvTs2RO9e/dGu3btpA6NPIAnXl9Op8C+85fw3dGL+OFYNoqsdjzTtw2e6NXSlczcjrnUhtnrUrB8f4brOaWiLNloG+qHNqF+aBvqh+gwPVoEV37QSYnVgXe3nMLCHefgcAq0DfXDvwa2xz1RIbc9zu5w4svdF/DO5lMwW+zQqpR4qFM4Rvdsjg4R/jfs/7d7W8NPp8bLa07gn6uOwmyxY+yfWpTbJ9tYin+uOor/ncoDACT1aIYXBrSDt0YFi92JEqsDxTYHSqx2FFsdKLE6bts6FtHQGxENfSr8DhQKBZoG+aBpUFM8fFdTCCFQanNWeFxVSJ6obd68GX369EFISNn/2KSkJCxbtsyVqDmdTkyZMgW9e/cGAHTs2BFpaWkAgH379iEzMxPdunWDSqXCm2++iT/96U8VntNisZQbqm4ymdz9sYjuiBDCdedcE1RKxW37buTn5+ORRx7BpUuXoFAo8PLLLyMmJgZPPfUU8vPzodPpMHfuXPTs2RN5eXmYMGECLly4AACYOXMmBg8ejEuXLmHkyJFIS0uDwWBAaWkpAGDdunUYMWKEaxRiYmIili9fjunTp9fY56XyDNM3uGow3E2jUuDEK/1vuw+vr4oJIXAkw4jvjlzE+qNZyDaVlnv9zQ2/Yv3RLPz7oVjEhN+Y3Fy1+2wBpq44gszCEvjp1EiIDsVv+UU4nWPGb3lF+C2vCD8cz3bt36NlEP7251b4U+vg25YRu87kY9o3x3ChoBhalRJNg33wa44ZoxbtRe+oELz4QHu0bXzjOpb7zl/Cy98ed/X5GtYxHC8MaIdQvddtv4/HejSHr06Nf6w8itnrUlBksWNSn7LWtLVHLuLlb4/DVGpHY70X5iTGoleba8mil0YFL40KDW97BvdQKBTw1rq3VU/yRC0zMxPh4eGu7bCwMGRkXJf1K5V4/PHHXdsvvfQSHnzwQddriYmJ+Oc//4lDhw5h0KBBOHbsGIKCbj86JDk5GbNmzXLvByFyI4dTYNi8XTX2/t/8rSfUqlsXwl999RViY2Mxd+5cpKam4j//+Q/mzJmDd999F126dMHZs2eRkJCA06dPY/LkyRg5ciSGDx+O/Px8dO/eHd26dcMrr7yCuLg4fP/999ixYweWL18OoOw336VLF9e5wsLCsHfv3hr7rCQ/9f36Sisoxhe7z8PuuHnNi9Uh8POZPKRfKnE9166xHwbHhWFwbBjyiyx4YdVRnLhowtCPdmJcrxZ4tm9UuQSh1ObAmxt+xaKd5wAAPVsFYU5iHMIDyprlnE6BzMIS/Jptxq85ZpzOMePnM2XNd7t/K0CHcH88dW8r9DOUb840Ftvw+vepWLY/HQDQuVlDvDG8A1oE+2LFgQzM3XQK/zuVh59P5+EvXSLx3P1RaOTnhTyzBck/pGL1wUwAQNtQP7wy1IBuVRjNObxTBHy0ajzz9SG8vfkULhVZkWe2YP2xLABlSd/MwQb4+2gq/Z51geSJmtN544WqVN5YletwOPDss89i//792LJlCwDg2Wefdb3eqVMndO3aFTt37sSQIUNue85p06bhueeec21fXcW+8tj2STVLpVTgm7/1rNH3v517770X/fr1w7lz5zBgwADMnDkTwcHBGDdunGsfm82GtLQ0bNq0CSdOnMArr7ziej41NRXbtm3Df//7XwBAr1690LJlSwCV/81Tzamoxqum1efry2p3YsKSA0jNqrglp2Wwb1lyFtcErRtdq51qGuSDdZN64eNtZ/HhT6cxf/tv2HA8G8nDO6Bnq2AczSjE35cdxtm8IujUSrwwoB1G9WgO5XW/e6VSgchAH0QG+iAhOhRAWXK34kAGFvzvLI5lGvG3rw6iZbAvJtzTEsM6RmBLag6mrz2BPLMFvloVnh/QDiO7NXO97yN3NcXguDAs2H4WC3b8hqX70rH2yEUMiQvD+qNZMFvsaKBT4+/3RSGpR7NKNdv+Uf+Yxlg4qgvGL96Pz3edBwA09NHg9WEdMKBDxQMN6iLJE7WIiAhs377dtZ2VlYWIiIhy+1gsFvzf//0frly5gq1bt0Kv1wMAFi5ciH79+rmSLKfTWak5cXQ6HXQ6XbXiVYCjCajmKRSK29Z41bS4uDicOnUKP/zwA9atW4fXX38dOp0Ohw8fdu2TmZmJJk2awOFw4KeffkJgYCCAst9wSEgIFAoFxHUdOjWasrvciIgIZGVluZ6/2W+ePFt9vr7e23IKqVkmtGvshyd6tbzlfu2a+CG6if6WzY9atRKTE9rggQ6N8fyqoziYVohH/7MHvdoEY9fZAjicAnER/pj7l3i0btSgUrF5aVR4rHszPNI1EuuPZeHjbWdxMtuM51cdw6vrU2EutQMA/tw2BK8O6+CqnbteA50az93fFo90a4q5m05h1cEMLN1XVvs2rGM4pg1oh0YVNHNWpHdUCJaM7YZnvj6EDhH+mP1gDBr53dl7ypqQWGZmpmjWrJnIzs4WVqtV3HfffWL16tXl9klKShKJiYnCarWWe378+PFiypQpQgghTpw4IcLDw4XJZKpyDEajUQAQRqOxwn0nfLlfnMqu+jmIbqW0tFSUlpZKHUY5ycnJ4qWXXhJCCGE2m0XDhg1Fy5YtxeLFi4UQQuzYsUOEhoYKi8Uihg8fLmbPni2EEOLUqVOiYcOGIicnRzz77LPi73//uxBCiP379wulUinOnTsn9u3bJ2JjY4XZbBZms1l06NBBHDhw4IYYbvW9VOX3Wt/c7Lvh9VW166sm7T9/SbR4YZ1o/a/1IuWi+65fh8MpPt95TkS//INo9vw60WraevHej6eEze64o/d1Op3ix5RsMXzeTtHs+XWi4yubxLeHMoTT6az0exzLKBQz1hwXv5zNv6NYPE1VyjHJEzUhhFi+fLmIiYkRbdq0EVOnThVCCDF27FixZs0acerUKQFAtG3bVsTFxbkedrtd5Ofni8GDB4vo6GjRoUMHsXXr1mqdvypf2JOLmaiRe8nxH9K8vDxx3333iQ4dOoiOHTuK999/X6Smpop7771XdOjQQcTHx4sdO3YIIcputgYPHiw6dOggDAaD60bLaDSKYcOGiejoaDF8+HDRunVrce7cOSGEEO+//76Ijo4Wbdq0EXPnzr1pDEzUqq6uJGpyvr5qSpHFJu6d85No9vw6Me+nMzVyjozLxeKtjSfFsYxCt793ykWjKCy2VrwjVUpVyjGFEJxswmQywd/fH0aj0dWseitPLTmA5+6LQpvQG0ezEFXH1RHI1W2O91S3+l6q8nutb2723fD6urk7/V72n7+Ed388jSFxYbec1+t609ccx5e7L6Bzs4ZYPqFHteZBI89RlXJM8j5qdQ0nvCUiqr+EEFi08zySv0+F3Snw85l8HMs0Yvrg6Ft2jt9xOg9f7r4Ab40KcxPjmKRRlchnKEwdUu+rIImI6qErFjsm/vcQZq9LgVMIPHJXU/hqVVj8ywWMXLgHBdctOXSVsdiGf6w4CgD418D2aF6FCWWJACZqVcZRn0RE9c+pHDOGfPgz1h/LQpCvFovHdkPy8A745um70SzIB3vOXcKQD3ci5WL5aTdmfncC2aZS9I4KwchuTSWKnuoyJmrVwF595E4qlQp2u13qMGTHbrfX6Lq99QWvr5uryvW15nAmhn64E7/lFaFT0wCse+ZPuLt1MAAgKtQPa56+G39qHYzMwhI89PEufP/7BKw/HMvCN4cyofdS482HYm870z/RrbCPGpHE1Go1ioqKUFRUVKl5AOsDu90Ou93uWgaorlixYgVmzpwJq9WKkSNHYsaMGeVe37t3L55++mlYLBY0bdoUCxcuROPGjWG1WjFmzBgcOXIEKpUKc+fORUJCglti4vV1o8peX1a7E6+tT8EXu8uWrxrdszn+9UB7aNXl6zgCfLT4fExXvP79SSzaeQ5/++ognujVAisPlK2y88rQGDT29+B5vqhGsUaNSAb8/f05Ku86Op0O/v63Xr9QjrKzszF16lRs27YNKSkp2LFjBzZu3Oh6XQiBESNGIDk5GUePHkVSUhLGjx8PAFi8eDFsNhuOHz+OJUuWYPTo0W6NjddXeZW5vi4UFCHxk134YvcF+GhV+OCRjpg5xHBDknaVWqXE9MHReHNELLQqJf6z4xwuF9vwQIfGGBofVhMfg+oJ3l5Vg+BwAqoBarWaNR512ObNm9GnTx+EhJQtBp2UlIRly5ahX79+AMoWIi8pKXHVlA0aNAgjR46ExWKBw+FASUmJ67/e3jfO+H4zFovFNc0EUDbk/1Z4fVXet4cy8dK3x3HFYkebRg3w8chO5ZZwup2/dIlE60YN8NSSA9ColHj1wQ5s8qQ7wl9tVfH3RkQ3kZmZifDwcNd2WFgYMjIyXNvBwcHw9fXFpk2bcP/992Pp0qWw2WwoKCjA6NGj8fnnnyMsLAyFhYX4+uuvK3XO5ORkzJo1y+2fpb4qstgxfc0JrDpY9v/tkbuaYvqg6HKLnVdGp6YNsfP5PrA7Bbw07GdJd4ZNn9XAwQRE9EcVLQauUCiwatUqvPbaa+jYsSMKCwsRFBQErVaLmTNnokePHsjOzsaxY8cwefJkXLhwocJzTps2DUaj0fVIT09362fyBEUWO3adzUeWseS2+x3PNGLQBz9j1cEM6L3UmPfXTkge3qHKSdpVapWSSRq5BWvUqogVakR0MxEREdi+fbtr+2aLgWs0Gtc+ly5dwuzZsxEYGIg1a9Zg2bJlUCgUiIqKQvfu3bF37140a9bstufU6XTse3YLl4us+HzXeXyx+zwKi20AgFC9DvGRAYiPbIj4yAB0iPCHr1aFRTvP440fUmFzCHRu1hDvPRyPiIY+En8CojJM1IiI3CAhIQHTp09HTk4OAgMDsXjxYjz11FPl9hkzZgw+/PBD9OjRA2+99RYSExOhVCoRFxeHlStXIiYmBnl5edi3bx9ee+01iT5J3ZZlLMHCHefw9d40FFsdAID4yADkmkpx0ViKjSdysPFEDgBAqQBC9V7IMpZCoQAm9WmNyX3bQH2LFQaIpMBErYrYKZSIbiYsLAxz5sxBQkICLBYLhg4dimHDhmHcuHEYMmQIhgwZgvnz52PChAkoKipCbGwsPv30UwDAO++8g/HjxyM6OhpqtRqvvvoqoqKiJP5EdctveVfwyfaz+OZQJmwOAaUCGBTbBE/d2wqGsLIRnrmmUhxKL8Th9EIcTivE0YxCZBlLEarX4Z3/i0fPVsESfwqiG3FRdlRtcdRJXx/CU/e0QnQYF4MmkgIXZb+1+vjdFFnsmLG2bACAEIBWpcRDncMxoXerCpdrcjgF0i4Vo4m/F/uTUa3iouw1iPVpRETycCbXjCeXHMSZ3Cvw1arw1+7NMPZPLRCqr9zksiqlAi249ibJHBO1auA8akRE0lp75CJeWHUUxVYH4iIDMO+vnRAeULn554jqEiZqRERUZ1jtTrz+fSo+33UeAJDUoxleHNgeOjWbLskzMVEjIqI6IctYgqe/OoiDaYXw1qjwxkMdMDQ+vOIDieowJmrVwOEXRES1a+eZfEz6+hAuFVnRMsQXn4zsjKjQyi3rRFSXMVGrIs7OQURUe5xOgXnbzuDtzafgFMDA2Cb490OxaKDjP19UP/BKJyIiWTIW2/D35Yex9WQu1EoFXhrYHmPubs75LKleYaJWRSweiIhq3rEMI5766gAyLpcgVK/DR492QpfmgVKHRVTrmKgREZFsCCGwdF86Zqw9AavdiZ6tgvD+Ix0R3IBrmlL9xEStiljlTkRUM0qsDry85jhWHsgAADz951Z47r62UClZ7lL9xUStGjjqk4jIvdIKijFhyQGkZpng56XGO3+JR0J0qNRhEUmOiVoV8b6OiMi9Sm0OPP7FPpzJvQJDmB4f/7Uzmgb5SB0WkSwopQ4AAFasWAGDwYA2bdpg1qxZN7x+8OBB9OzZE3FxcejRoweOHDkCALDZbBg7diyio6NhMBiwZ8+eWomXS0gREbnPuz+expncK+gQ7o9VT/VkkkZ0HckTtezsbEydOhXbtm1DSkoKduzYgY0bN5bb57HHHsMbb7yBI0eO4JVXXkFSUhIA4KOPPoIQAikpKVixYgUee+wx2O12KT4GERFVw+H0Qiz431loVUq8lRgHLw2XgiK6nuSJ2ubNm9GnTx+EhIRAo9EgKSkJy5Ytc73udDoxZcoU9O7dGwDQsWNHpKWlAQC+++47jBo1CgAQHR2NiIgI7Nq1q/Y/BBERVVmpzYGpK47AKYDJCW3QtjFXGiD6I8n7qGVmZiI8/NpabWFhYcjIyHBtK5VKPP74467tl156CQ8++GCljr0Vi8UCi8Xi2jaZTFWKmYMJiIju3HtbrjV5TujdUupwiGRJ8ho1p9N5w3NK5Y1hORwOTJo0Cfv378e7775bpWP/KDk5Gf7+/q5HZGRk5QPmaAIiojt2OL0Q87dfa/JUqyT/54hIliT/ZURERCArK8u1nZWVhYiIiHL7WCwWPPTQQ0hNTcXWrVvh7+9f6WNvZtq0aTAaja5Henp6lWJmhRoRUfWxyZOo8iRP1BISErBlyxbk5OTAZrNh8eLFGDhwYLl9xo8fD61Wix9++AF6vd71/MCBA/HZZ58BAE6ePIkzZ86ga9euFZ5Tp9NBr9eXe1SWglVqRER3hE2eRJUneR+1sLAwzJkzBwkJCbBYLBg6dCiGDRuGcePGYciQIWjfvj2+/PJLtG3btlwSduDAAUycOBFPP/00DAYDFAoFFi1aBJ2Oy4wQEcnV1SZPjUrBJk+iSpA8UQOAxMREJCYmlntu4cKFrr/FLXrvq1SqcvvVBq4gRURUPaU2B/7xe5Pnc33Z5ElUGbyVqYZbJY5ERHRr7205jdO/N3k+eU8rqcMhqhOYqFURK9SI6FYqWmVl79696Nq1K2JjYzFo0CBkZ2cDAKxWK5555hnEx8fDYDBg06ZNtR16jUvNMrHJk6ga+EupBtanEdEfVbTKihACI0aMQHJyMo4ePYqkpCSMHz8eAPDmm28iPz8fhw4dwvLlyzF69OibTj9UVwkhMOu7E3AK4Ml7WrHJk6gKmKgREblBRaus5Ofno6SkBAkJCQCAQYMGYcOGDbBYLFi2bBleeOEFKBQKGAwG/Pjjjx7VxeKH49n45bdLaOLvhafuZZMnUVUwUSMicoOKVkoJDg6Gr6+vq1lz6dKlsNlsKCgowJkzZ7Bjxw507doV3bt3R05ODlSqite8tFgsMJlM5R5yU2J14LX1qQCAfz3QHj5aWYxhI6ozmKhVgwfd6BKRm1S0UopCocCqVavw2muvoWPHjigsLERQUBC0Wi3sdjvOnTuHvXv3YsGCBXj00UdhNBorPOcdrbJSS+b/7ywyC0twV4tADIptInU4RHUOE7Uq4vQcRHQzlVkpRaPRYPv27Th06BBGjx4Nh8OBwMBANG7cGP/3f/8HhUKB2NhYREZG4tdff63wnHe6ykpNy7hcjI+3nYVSAcwYHA0FC1CiKmOiVi2sUiOi8iqzysqYMWOwe/duAMBbb72FxMREKJVKDB482NWf7fz580hLS0Pbtm0rPOedrLJSG5K/PwmL3YlH7moKQ5i/1OEQ1UlM1KqIS0gR0c1cv8qKwWBAXFyca5WVtWvXAgDmz5+PiRMnol27djhz5gzmzp0LAHjjjTeQm5sLg8GAgQMHYsGCBa41jeuq3WcLsP5YFvReaky5v+Kkk4hujr06iYjcpKJVVrp06YIDBw7ccJxer8eXX35Z4/HVFrvDiVnfnQAAPHdfFAJ9tRJHRFR3sUatitjFgojo9r7em4aT2WZEhTbAyO7NpA6HqE5jolYNHPVJRHRzhcVWzN18CgAwc7CBKxAQ3SH+gqqINWpERLf29uZTKCy2ob+hMXq2DpY6HKI6j4kaERG5xclsE5b8cgE6tRIvDmwvdThEHoGJWjWw5ZOI6EafbDsLpwDG926JyEAfqcMh8ghM1IiI6I4VXLHg+2PZ0KqUGN2zudThEHkMJmrVwMEERETlrTiQAavDiQEdGiOogU7qcIg8BhO1KuNoAiKi6zmdAv/dkwYAnI6DyM2YqFWDYJUakccpKCiQOoQ6a8eZfKRdKkbbUD90adZQ6nCIPAoTtSri9BxEnikqKgqjRo3Cnj17pA6lzlnyywUAwMjuTbnwOpGbMVEjIgJw7tw5dO7cGY8//ji6dOmCzz77DKWlpVKHJXsXC0uwJTUHPloVHuwYLnU4RB6HiVoV8V6RyDPp9Xo888wzOHHiBF5//XXMnj0b4eHheOGFF2AymaQOT7aW7kuHUwBD48Ph56WROhwij8NErRrYQ43IM+3fvx9PPPEEHnnkEfTs2RMrV66E0+nEsGHDpA5NlmwOJ5buLRtE8NduTSWOhsgzqaUOoK5h9wsiz9SpUyfk5uZiwoQJSE1NRaNGjQAA9957L4KCgiSOTp5+TMlBrtmC+MgAxIT7Sx0OkUdiokZEBGDatGkYPnw4VCoVnE4nHA4HVCoVFAoFMjMzpQ5PlpbsuTqIgFNyENUUNn1WA2fnIPI8oaGh6Nq1KwDgxIkTiIyMxP79+wEA3t7eUoYmS7/lXcHOMwXw99ZgUGwTqcMh8lhuTdScTicA4PTp09i0aVOlj1uxYgUMBgPatGmDWbNm3XK/GTNmYObMma7tX3/9FXq9HvHx8YiPj0e/fv2qHTsR1W9TpkzBe++9BwDo0KEDVq1ahWeeeUbiqOTr6gS3iZ0j4KVRSRwNkedyW6L28ccf49FHH0VeXh7+9Kc/YdKkSfjHP/5R4XHZ2dmYOnUqtm3bhpSUFOzYsQMbN24st09hYSEef/xxzJkzp9zze/fuxZgxY3D48GEcPnz4huNqiuBwAiKPY7FY0KtXL9d2jx49OD3HLZTaHFhxIAMA8CgHERDVKLclap9++inee+89rFq1CkOHDsWJEyfw008/VXjc5s2b0adPH4SEhECj0SApKQnLli0rt88333yDtm3bYsqUKeWe37dvHw4ePIjOnTujb9++OHHiRKVitVgsMJlM5R6VpeAEHUQeSa1Wu5o6AeDIkSPQaDjdxM2sO5oFY4kNd7cOQsuQBlKHQ+TR3DaYQKFQIDQ0FFu3bsXw4cOhVqvhcDgqPC4zMxPh4dcmSQwLC0NGRka5fcaMGQMA5Zo9AcDHxwdjx47F6NGj8f3332Po0KFITU2tsHBNTk6+bRNrhVihRuRxXn/9dSQkJCAqKgoKhQLnzp3D8uXLpQ5Llr66OoigGwcRENU0tyVqGo0G8+fPx5YtWzBv3jx88cUX0Ol0FR53tV/b9ZTKylX0vfHGG66/H3jgAUybNg2pqamIjY297XHTpk3Dc88959o2mUyIjIys1Dk5PQeRZ+rfvz/OnDmDHTt2QKPRoHv37ggODpY6LNk5cdGIQ2mFaOSnQ0J0qNThEHk8tzV9fvTRR9i8eTPmzZuH4OBgLF++HJ988kmFx0VERCArK8u1nZWVhYiIiEqdc86cOTCbza5tp9MJtbri3FOn00Gv15d7EBGlp6cjODgYer0ex44dw4IFC6QOSXaW/FI2iODhrpHQqDhxAFFNc1uNWseOHbFkyRKcPn0aQgisXLmyUkPaExISMH36dOTk5CAwMBCLFy/GU089Valz/vjjj/Dy8sKkSZOwdetWOBwOtGvX7k4/ym2xQo3IM02YMAErV66ExWJBcHAw0tPT0bNnT4wfP17q0GTD6RT4/ljZjfXDd3EQAVFtcNvt0N69e9GiRQsMHDgQGRkZaNq0Kfbu3VvhcWFhYZgzZw4SEhJgMBgQFxeHYcOGYdy4cVi7du1tj/3kk0+wevVqxMTE4IUXXsCyZcsq3Wx6J9hFjcjzbNiwAefOncOwYcPw008/4fvvv0dgYGCV3qOiqYb27t2Lrl27IjY2FoMGDUJ2dna5181mM1q1aoVt27bdyUepMb/lX4GxxIY2jRogLIBzyxHVCuEm99xzjzh48KCIj48XQgixatUqcdddd7nr7WuU0WgUAITRaKxw3xlrjoufT+fVQlREdDNV+b1WRbdu3YQQQsyePVusXbtWCCFEly5dKn18VlaWaNq0qcjNzRVWq1X07dtXbNiwwfW60+kUkZGRYvPmzUIIIZYtWyYGDx5c7j2SkpJEw4YNxU8//VStz1BT381VS/deEM2eXyeeX3mkRt6fqL6oym/VbdVPZrMZHTt2dG0PHz4cFovFXW9PRFSj1Go1UlJS0LZtW2zfvh0WiwUFBQWVPr6iqYby8/NRUlKChIQEAMCgQYOwYcMGVzm5bNky+Pn5VTgY6np3MtVQdRy4cBkA0KlZwxo9DxFd47ZETalUori4GIrfh0VmZGTcdESnJ+ASUkSe5+WXX8azzz6LgQMHYu3atQgLC8P9999f6eMrmmooODgYvr6+rlVbli5dCpvNhoKCAqSlpeHdd9+9YVLviiQnJ8Pf39/1qOzo9eq6mqh1ZqJGVGvclqhNmjQJCQkJyM7Oxt///nd069YNkyZNctfbExHVqBYtWmDTpk3w8fHB4cOHsXXrVnz88ceVPr6iqYYUCgVWrVqF1157DR07dkRhYSGCgoKgVqsxduxYfPjhh1VeU3TatGkwGo2uR3p6epWOr4rLRVaczStCgI8GLYN9a+w8RFSe20Z9JiUloUWLFli3bh0cDge++OILVxW/p+ESUkSeZ8iQITh58iSAssm04+LiqnR8REQEtm/f7tq+2VRDGo3Gtc+lS5cwe/Zs5Ofn4+TJkxg7diwA4MyZMxg3bhw++eSTCstQnU5Xqfkq3eFQ+u+1aU0bulpOiKjmuXWI5N13341///vfmDBhgsc2exKRZ7o62tJqtVbr+ISEBGzZsgU5OTmw2WxYvHgxBg4cWG6fMWPGYPfu3QCAt956C4mJiYiOjkZ6erprzeIuXbpg4cKFsrvRZf80ImlIvih7XcQ+akSe5+DBg+jTpw98fHyg1Wqh0Wig1WorfXxlphqaP38+Jk6ciHbt2uHMmTOYO3duTX0ct2P/NCJpuK3p89NPP8X69euxatUqDBkyBB9//DG6d+/urreXDdb4E3mmX3755Y7fIzExEYmJieWeW7hwoevvLl264MCBA7d9DznOoWZzOHEk3QiVUoG4iACpwyGqVyRflJ2ISA4uXLhw0+ebNePC4yezzCixORAb4Q9vrUrqcIjqFckXZa9rFFxEisgjXe3MDwBWqxUZGRno0aMHfv75ZwmjkocDFy4BADo1ZbMnUW1zWx+1Dz/8EJs3b8ZHH33kWpS9KkPb6xJ2USPyPKdPn3Y9Lly4gF9++QVRUVFShyULB9IKAbB/GpEU3Faj1qlTJ6xcuRIAUFhYiHfeeccjCzn2USOqH7p27coF2X93kAMJiCTjtkTtm2++waZNm/Dmm28iNjYWJpMJL730EqZOnequUxAR1Zhdu3a5/hZC4NChQygtLZUwInnIMpYgs7AETfy9uBA7kQTclqglJyfj008/xerVq9GzZ08sWLAAffr08chETXB+DiKP89e//tX1t0KhQEhICN5//30JI5KHgxcKAXD+NCKpuC1RA4AOHTpgzpw56N+/P/R6PRMaIqozzp07h6KiIvj6+sJiseDKlSsICgqSOizJueZP40ACIkm4bTCBEAIbN27EDz/8gPvuuw/btm3z2NUJmH4SeZ5Vq1YhNjYWQFnSFhUVhQ0bNkgclfQOpLF/GpGU3Faj9uqrr+LFF1/E888/j/DwcPTv3x/vvPOOu95eNjiWgMgzvfLKK9i4cSMAoF27dti3bx8SExPRv39/iSOTTqnNgROZRnhplIgO00sdDlG95LZErV+/fujXrx9KS0tRVFSEY8eOueut5YdVakQex+FwoHXr1q7tli1b1vtJu49mGGF3CnRq1hAalVuXhiaiSnLbLy8nJwf3338/GjRoAL1ej169eiEzM9Ndby8bnJ6DyDPp9Xp8++23ru3169cjICBAsnjkwLUQO/unEUnGbYna5MmT0a1bN+Tn5yM3Nxf33HMPJk6c6K63JyKqUe+//z4mTpyIRo0aITQ0FJMmTar3oz4Psn8akeTc1vSZmpqKpUuXurZfffVVREdHu+vtZUPBKjUij9SlSxecP38ex44dg0ajQdu2baHRaKQOSzJCCNdEt52aBkgbDFE95rYaNbvdDrvd7tq2Wq1QKj2zT4NgJzUij/O///0Pd911Fzp27AghBJo1a4b9+/dLHZZkLhQUo6DIihbBvghq4HnrNhPVFW7LpPr164cHH3wQ69evx/r16zF8+HD069fPXW8vG6xPI/JMU6ZMwXvvvQegbE7IVatW4ZlnnpE4KumwfxqRPLit6XPOnDl49dVXMXv2bDidTgwYMABdunRx19sTEdUoq9WKXr16ubZ79OhRr5eQ4vxpRPLgtkRNpVJhxowZmDFjhus5vV4Pk8nkrlPIBhdcIPI8KpUK+/fvd91gHj58uF73UeNC7ETy4NYlpP6IS0gRUV3x+uuvIyEhAVFRUQCA8+fP44svvpA4KmmYSm34NccMP50abRo1kDoconqtRhM1Tx0hyfyTyPP0798fZ86cwY4dO2A2m/HLL79g1KhRyM3NlTq0Wnc4rRBCAB2bNYRS6ZnlOFFdIYthmStWrIDBYECbNm0wa9asW+43Y8YMzJw507VtMpkwdOhQREdHo2vXrjh9+nTNB8syi8hj5eXlYf369XjyySfx448/Ys6cOVKHJAkuxE4kH3ecqGk0Gmi12hseGo0GRUVFFR6fnZ2NqVOnYtu2bUhJScGOHTtc6+1dVVhYiMcff/yGQnP69Ono1KkTUlJS8O9//xujRo26049TKaxQI/IsW7duxQMPPIC4uDikpaVBr9cjNTW11soUueFEt0TyccdNn2fOnLmj4zdv3ow+ffogJCQEAJCUlIRly5aVm9rjm2++Qdu2bTFlypRyx3733XfYunUrAKBPnz7Izs5GWloamjZtettzWiwWWCwW13ZVBjwoWKVG5FE6deqE4uJiJCUlYf78+YiMjETLli2hUqmkDk0SDqfAobRCKBVAXKS/1OEQ1Xt3nKg1a9bsjo7PzMxEeHi4azssLAwZGRnl9hkzZgwAlGv2vN2xFSVqycnJt21iJaL6w2w2o1GjRlCpVPU2ObveqRwzrljsaN9EDz+v+jvqlUguJO+j5nQ6b3iusisaVPfYadOmwWg0uh7p6emVOh8AKBVld5xE5BlOnz6Nl19+GTt27ECLFi3Qv39/FBcX37R8qUhF/W337t2Lrl27IjY2FoMGDUJ2djYAoKCgAA8++CDi4uLQoUOHcsvx1bbTuVcAAIYwvWQxENE1kidqERERyMrKcm1nZWUhIiKiRo/V6XTQ6/XlHpWlUio47QiRh+nfvz/WrVuH1NRUxMTEwG63o0WLFnj33Xcr/R4V9bcVQmDEiBFITk7G0aNHkZSUhPHjxwO41t/2yJEj2Lx5M5577jnk5OS4+2NWSlZhCQAgLMBbkvMTUXmSJ2oJCQnYsmULcnJyYLPZsHjxYgwcOLBSxw4cOBCLFi0CAGzbtg0NGjSodJJXXQqFAg4makQeqWXLlnjrrbeQkZGB6dOnY/HixZU+9vr+thqNxtXf9qr8/HyUlJQgISEBADBo0CBs2LABFosFAwYMcCVtjRs3RmBgoKu27XYsFgtMJlO5x53KMpatxhDm73XH70VEd07yRC0sLAxz5sxBQkICDAYD4uLiMGzYMIwbNw5r16697bGzZ8/G0aNHERMTgylTpuDLL7+s8XhVCgXY8knk2by8vDB27FgcOHCg0sdU1N82ODgYvr6+2LRpEwBg6dKlsNlsKCgowKBBg9C4cWPX8xaLBQaDocJzJicnw9/f3/WIjIysdLy3cvH3GrUmrFEjkoUanfC2shITE5GYmFjuuYULF96w3x8HEwQEBGDVqlU1GdoNlAquuEBEN6qoz6xCocCqVavw3HPP4fnnn8djjz2GoKAgaLVa1z5LlizBP//5T2zYsAFqdcXF87Rp0/Dcc8+5tk0m0x0na6xRI5IXWSRqdYlSqeBgAiK6QUREBLZv3+7avlmfWY1G49rn0qVLmD17NgIDAwEAb7zxBj755BNs2bIF7du3r9Q5dToddDqdmz7B73EbWaNGJCeSN33WNSolmz6J6EaV6W87ZswY7N69GwDw1ltvITExEUqlEp9//jm++OIL/PLLL5VO0mpCqc2B/CtW6L3UaKDjfTyRHDBRqyKlAnAyUyOiP6hMf9v58+dj4sSJaNeuHc6cOYO5c+cCAF588UWYzWb0798f8fHxiI+Px549e2r9M2RfbfZkbRqRbPCWqYqUCgWc7KNGRDdRUX/bLl263HSAQmZmZo3HVhkXrzZ7sn8akWywRq2KlBz1SUQeKquwrEaN/dOI5IOJWhUpFeA8akTkka4OJOCITyL5YKJWRVyZgIg81cXf+6g18WeNGpFcMFGrIoWC03MQkWfKck12yxo1IrlgolZFSiZqROShrk52G84+akSywUStitRKBdjySUSeKPP3GrXG7KNGJBtM1KpIqeSi7ETkea5Y7DCX2hHcQAudWiV1OET0OyZqVeSlUaLU5pA6DCIit3L1T+NAAiJZYaJWRd4aFUqsTNSIyLNcG/HJZk8iOWGiVkXeGhWKmagRkYe5WqPG5aOI5IWJWhX56NQoYdMnEXkY1qgRyRMTtSpi0ycReaJrc6ixRo1ITpioVVFZ06dd6jCIiNzq6hxqXD6KSF6YqFWRt1bFpk8i8jgX2UeNSJaYqFVRA50axVYHVycgIo8hhMBFYwmUCqCRn07qcIjoOkzUqshbq4JWpYSpxCZ1KEREblFYbEOpzYlQvRfUKv6zQCQn/EVWQ4CPFpeLrVKHQUTkFheNVye7Zf80IrlholYNDX00KGSNGhF5iKzC36fmYP80ItlholYNgb5aFFxhjRoReYas32vUOOKTSH6YqFVDWIC3q2AjIqrrrk12yxo1IrlholYNYQHeyLzMRI2IPMO15aNYo0YkN7JI1FasWAGDwYA2bdpg1qxZN7yemZmJP//5z2jfvj369OmD3NxcAEBxcTEaNGiA+Ph418PhqPk5zsIDvJFZyESNiDzDxd/7qHEONSL5kTxRy87OxtSpU7Ft2zakpKRgx44d2LhxY7l9nn76aYwePRqpqakYOXIkJk+eDAA4cOAA+vbti8OHD7seKpWqxmOODPTGxcIS2BzOGj8XEdUdFd107t27F127dkVsbCwGDRqE7OxsAIDNZsPYsWMRHR0Ng8GAPXv21Grc10Z9MlEjkhvJE7XNmzejT58+CAkJgUajQVJSEpYtW+Z63Waz4aeffsKjjz4KAEhKSsL69eths9mwb98+ZGZmolu3bujZsyd+/vnnWonZz0uD4AY6nM8vqpXzEZH8VXTTKYTAiBEjkJycjKNHjyIpKQnjx48HAHz00UcQQiAlJQUrVqzAY489Bru9dpaqczoFckyl0KqUCPLV1so5iajyJE/UMjMzER4e7toOCwtDRkaGa7ugoAB+fn7QaDQAALVaDb1ej7y8PCiVSiQmJuKXX37Bhx9+iL/85S8oKCio8JwWiwUmk6nco6raNvZDara5yscRkWeq6KYzPz8fJSUlSEhIAAAMGjQIGzZsgMViwXfffYdRo0YBAKKjoxEREYFdu3bVStz5VyywOQQa+3tBqVTUyjmJqPIkT9SczhubD5VK5W1fv7rPs88+i+effx4KhQKdOnVC165dsXPnzgrPmZycDH9/f9cjMjKyynEbwvQ4kWms8nFE5JkquukMDg6Gr68vNm3aBABYunQpbDYbCgoKKjz2Vtxx03ltxCcHEhDJkeSJWkREBLKyslzbWVlZiIiIcG2HhITAZDK5mgHsdjvMZjOCgoKwcOFCpKenu/Z1Op1Qq9UVnnPatGkwGo2ux/XvUVlxkQE4klEIJ9f8JCJUfNOpUCiwatUqvPbaa+jYsSMKCwsRFBQErVZb4bG34o6bziwuxk4ka5InagkJCdiyZQtycnJgs9mwePFiDBw40PW6RqPBPffcg6+++goA8NVXX+Hee++FRqPBvn378N577wEAUlJScOjQIfTq1avCc+p0Ouj1+nKPqmri742GPlocZa0aEaHim06grDzbvn07Dh06hNGjR8PhcCAwMLBSx96MO246WaNGJG+SJ2phYWGYM2cOEhISYDAYEBcXh2HDhmHcuHFYu3YtAGDevHlYvHgxDAYDFi5ciA8++AAA8Prrr+PUqVMwGAx4+OGHsXjxYvj5+dVa7H3aNcLGE9m1dj4ikq+KbjoBYMyYMdi9ezcA4K233kJiYiKUSiUGDhyIzz77DABw8uRJnDlzBl27dq3wnO646bxao8blo4jkqeJ2wlqQmJiIxMTEcs8tXLjQ9XdkZCR+/PHHG44LCgpyJXNSSGgfir99dRCnc8xoE1p7CSIRyc/1N50WiwVDhw513XQOGTIEQ4YMwfz58zFhwgQUFRUhNjYWn376KQBg4sSJePrpp2EwGKBQKLBo0SLodLpaifsil48ikjWFEKLed7IymUzw9/eH0Wis8h3p6oMZ2PZrHt57OB4KBUdMEdW0O/m9errqfDcPfrQTh9ML8cPkXmjfhN8nUW2oym9V8qbPuu7B+HAUW+343+l8qUMhIqqyawuys+mTSI6YqN0hpVKB8b1b4b0fT+HAhctSh0NEVGk2hxO5Zgt8tCrovWXRE4aI/oCJmhvc1SIQLw6MxpyNJ/H9sSxO2UFEdUKOqRRClI34ZNcNInliouYmnZs1xIsPROPHlBz8e+NJJmtEJHtZRi7GTiR3TNTcqEOEP14b1gG/5RXh4f/8gtM5XGKKiOTr4tWpOTjik0i2mKi5mbdWhTkjYtGnXSM8t/wI51kjItnKck12yxo1Irli79EaEOCjxZP3tEJC+1C8+M0xOJ0CAzo0kTosIqJyLrqWj2KNGpFcsUatBrVu1ACvDYvBkj0XMGX5EeRfsUgdEhGRy8VC9lEjkjsmajWsdSM/zPtrZwT4aPDvH07CVGqTOiQiIgDX5lBj0yeRfDFRqwX+3hpMuT8KOo0Szy07gszfmxuIiKR0bdQnmz6J5IqJWi3x0aoxe2gM/tQ6CH9bcgBvb/oVZtauEZFESm0OXCqywt9bAx8tuysTyRUTtVqkUCgw+u4WmDeyM5wCePQ/e/DRT2c45xoR1bprIz5Zm0YkZ0zUJBAe4I2p/dpi5pBopGaZ8LevDmL5/nQUW+1Sh0ZE9USWa8Qn+6cRyRnruyXUuVkgDGH+2Hf+EjaeyMZXv1zAYz2aY1BsE3hpVFKHR0Qe7CJr1IjqBCZqEvPSqNCrTQh6tQnBqRwzPtt5Dl/sOo+erYIwonME2oT6SR0iEXmgi6xRI6oTmKjJSFSoH5KHxyLHVIqfT+fj5TXHMTQ+HH3bNUIjPe96ich9rk3NwbKFSM6YqMlQqN4LD3WOQNfmgViw4yzWHM5E76gQNNCpEeCjRbCvFn5eGoQ39Eagr1bqcImoDuJkt0R1AxM1GWsa5INXH+yA1CwT/ncqDxcLS/FbXhFyzaXIv2KF0ykQqvdCqxBfNAnwRp92jeCjVcHhFPD31kChUEj9EYhIpq7WqIVxslsiWWOiVge0b6JH+yb6G5632p04lWPGzjP5uFhYgieXHIDDKSAEoFSULRDfQKdGfGQAYiMC0KpRAwT6aKFQAAoFoFEqf/+bCR1RfZP1e41aqL9O4kiI6HaYqNVhWrUSMeH+iAn3BwA8d18UAEAIwFRqQ7HVgXP5RTieacQPx7Pxa7YJNse1Odsa6NS4YrHD31sDQ5ge9xtCEeirg8XuQInVgczCEjicAj5aNdo0aoDIQB+olDcmdUIIJntEdYi51AazxY7gBjro1BxhTiRnTNQ8yNVkSaEAAny0CPAp639yd+tgAGU1cFaHExqVAkIAuSYLLHYHVEoFvjuShQX/+w02h0CRxQ4frQqGMH/YHE44hcCnP/8GBRQIC/CGVq1AQx8tCq5YkWUqxeUiK6JC/dBAp0JcZAAa673g56WB2WJDA50aGpUSgb5a+HtrXNOOFFnsOJ17BSVWB1o18kWwrw7KmySBROR+XDqKqO5golaPaNVKaNXX5jhuGuTj+ntyQpsKj08rKMbRzELovTS4XGxFkK8OzYJ84KVR4ddsM65YbDiUVohDaYXIMpYgxE8Hi82JKxY7ckylcAqgRbAvbA4nLhaWoEWwL5QKBc7mF/3e306H1o38UGpzIMtYAovdifAAbygUQL7ZimKbA3c1bwhfnRoOp0BDHy1MpTYE+moR4KNFYbEVei8NWob4Iv+KBQAQ2dAHVyx2FFsd8NIo4aVRIchXB29tWcIohHDVQOaaLfDWqBDR0Js1hOTRrk7NwRGfRPLHRI0qrWmQT7nk7nohfmX9XPrHNLnp63ZHWW3eqZwrUCsVaB7siwa6ssvP5nCi4IoVl4utuFBQBABoFuQLnVqJXLMFTqeAv48GhcU25JpLUWRxQK0q6wztpVEhraAYR9KN0GmUKLE68Pmu8wjy1aLIasflYhv0Xhr46lQosthhKrHD7nQi0FcLH60aGZeLoVQoYHcK+HmVJYB+XmqE6r3Qq00IghtooVUrcSb3Cs7lFyHHVIoGOjX8vDRo4u8Fh1PAYneisNiGzMJiNPTVommgDxr5eaGJvxdUSgUuFZV9tstFVnhr1fDVquCjU8Nic8DhFHAIAYdTQKVUIFTvBX9vDUIa6OCrU5dLrAHA6RQotjmgUSmgVSnLJZROp4DdKaBQAA6n4KTJdEtXR3w24UACItljoka1Qq1SQq1SIj4y4IbXNColGvt7obG/1w2DJlqGNHB7LKZSGzIvl+CKxY6wAG+YS21o08gPKqXCNUDjWKYRv/xWgEtFVjicAkENtIgJ80enpg1RWGKDsdiKXLMFOrUSOrUSEQ290aV5Q5hL7ci4XIz95y8hy1jqSvwa+enQ0FeL/CsWXLDYYS4tWy7Mz0sNpUIBlVKBUpsDW1NzUWJzlCWoQiAswBsNtGoE+GhgsTtxNu8KTCU2OAWgUSngq1ND76WBTqNExqUSWOwOXF06tmmgD6JC/WCxO9DAS40WQb44mHYZJTYHAn11uFJa1sQdE65HdJOyfo5Xm8KMJTakXy6Br1aFwhIbnE6BC5eKy95YAEqlAkG+WoT46RDoq8WlIitMJTZY7E74aFUo+n05NCEAvbcGTqdAVKgfAnxuPhq5xOqA3emEn5fG7f+/a9OKFSswc+ZMWK1WjBw5EjNmzCj3+vnz55GUlASTyQR/f398+eWXaNasGaxWK8aMGYMjR45ApVJh7ty5SEhIqLE4XSM+2fRJJHtM1Kje0XtpoG9yfUJwrVbhjwM0pCKEgKnEjovGEhRZ7CgstkGrVmJ0z+ZoFuQDpwCulNqRd6Wsn6HNIRAe4I0GOrVrVO/pnCs4nWuGRqWEudSOw+mFMIT5o5GfDgVFVvh5qVFwxYrtp/Lw1Z40WGxltZ4Op4BSqUConw5KhQL+3ho4hUAjvQ4qZVliKoTAiYtG5JosKCyxItBXhwBvDTQqJUpsdlfCVWpzoNTmgBDAb/lF8NIoEernhWA/HRr56ZBZWILMyyXIv2LB/YbGePKeVpJ+73ciOzsbU6dOxf79+xEQEIABAwZg48aN6Nevn2ufl19+GQ8//DD+9re/4YMPPsCLL76IJUuWYPHixbDZbDh+/DiOHTuGAQMGICMjo8Zi5RxqRHWHLBK1iu5CMzMzMXLkSGRnZ6NJkyZYunQpGjVqBJvNhieffBK7d++GQqHAokWL0K1bN4k+BZH7KBQK+Pto4O9z8xomlQK3fR1ApRPOhzpHuP42lthctYTu7qfndArkmEuRVlAMc2lZEtqtRRDCO3qjZYgvfHWyKI6qbfPmzejTpw9CQkIAAElJSVi2bFm5RM3hcMBsNgMASkpK4O3t7Xq+pKTE9d+rz1fEYrHAYrG4tk0mU6WOu1xsBcCmT6K6QPKSsTJ3oU8//TRGjx6NUaNGYdGiRZg8eTK+/vprfPTRRxBCICUlBSkpKXjwwQeRkpICtVryj0VUJ/l711zTo1KpQBN/b49NDjIzMxEeHu7aDgsLu6FWbPbs2ejZsyfef/992Gw27Nq1CwAwevRofP755wgLC0NhYSG+/vrrSp0zOTkZs2bNqnKsi0Z3hanUBm/2YySSPWXFu9Ss6+9CNRqN6y70KpvNhp9++gmPPvoogLK71PXr18Nms+G7777DqFGjAADR0dGIiIhwFXy3Y7FYYDKZyj2IiO6E0+m84TmlsnwRO2rUKCxYsACZmZn4+OOPMWzYMAghMHPmTPTo0QPZ2dk4duwYJk+ejAsXLlR4zmnTpsFoNLoe6enplY5X71XWVE1E8ib5r7Siu9CCggL4+flBoym701er1dDr9cjLy6vUHezNJCcnw9/f3/WIjIx04yciovooIiICWVlZru2srCxERFxrVs7Ly8PJkycxdOhQAMBDDz2E7Oxs5OfnY82aNRgzZgwUCgWioqLQvXt37N27t8Jz6nQ66PX6cg8i8iySJ2oV3YXe7PWr+1TmDvZm7uQulIjoZhISErBlyxbk5OTAZrNh8eLFGDhwoOv14OBgeHl5Ydu2bQCAnTt3okGDBggODkZcXBxWrlwJoCyh27dvH+Li4qT4GEQkM5J35oqIiMD27dtd23+8Cw0JCYHJZILdbodarYbdbofZbEZQUJDrDrZ169Y3PfZWdDoddDqub0dE7hMWFoY5c+YgISEBFosFQ4cOxbBhwzBu3DgMGTIEQ4YMwerVqzFp0iQUFxfDz88PK1euhEKhwDvvvIPx48cjOjoaarUar776KqKioqT+SEQkAwohhKh4t5pz8eJF9OzZE3v27EFgYCAGDhyIp556CsOGDXPtM3jwYIwYMQKjRo3CF198gdWrV2PNmjWYO3cuTpw4gUWLFuHkyZPo168fTp06VeUk7OqcRkajkU0HRDLH3+ut8bshqhuq8luVvOnz+rtQg8GAuLg4113o2rVrAQDz5s3D4sWLYTAYsHDhQnzwwQcAgIkTJ0KpVMJgMGDEiBFYtGgRa8qIiIjIY0heoyYHRqMRAQEBSE9P510okcyZTCZERkaisLAQ/v7STkwsNyzLiOqGqpRjkvdRk4OrE1By9CdR3WE2m5mo/QHLMqK6pTLlGGvUUDay9OLFi/Dz87vpbOxXM1853aXKLSa5xQPILya5xQPUzZiEEDCbzQgLC6vUKO/65PqyzGw217n/t1JgTJXDmCqnsjFVpRxjjRrKpvSozGhROc5TJLeY5BYPIL+Y5BYPUPdiYk3azV1fll296axr/2+lwpgqhzFVTmViqmw5xttRIiIiIpliokZEREQkU0zUKkGn02HGjBmymvpDbjHJLR5AfjHJLR6AMXkyOX6PjKlyGFPl1JeYOJiAiIiISKZYo0ZEREQkU0zUiIiIiGSKiRoRERGRTDFRIyIiIpIpJmpEREREMsVEjYiIiEimmKhVwooVK2AwGNCmTRvMmjVL6nDwyCOPICoqCvHx8YiPj8c333wjSRwmkwkdOnTA+fPnAQDbt29HfHw8oqKi8NRTT8Fut0se07Rp09C8eXPXd/XRRx/Vajxvv/02YmJiEBMTgzFjxsBqteL48ePo3r072rVrhxEjRqCoqEjSeObPn4+wsDDXd/Tiiy/WWjwA8MILLyA6OhoGgwFvv/02AHlcS3WZ3MosQD7lFsCyqzLkVnbdKqZ6UX4Juq2srCzRtGlTkZubK6xWq+jbt6/YsGGDpDG1bt1aFBQUSBrDrl27RExMjNBoNOLcuXOitLRUNG3aVPz666/C6XSKpKQk8cknn0gakxBC9OnTRxw8eLBW47hqz549IiYmRly5ckU4nU4xcuRI8fbbb4u4uDixbds2IYQQL7/8snjhhRckjefxxx8Xq1evrpUY/mjdunWid+/ewm63i+LiYtG8eXNx8uRJya+lukyOZZYQ8ii3hGDZVRlyK7tuF1N9KL9Yo1aBzZs3o0+fPggJCYFGo0FSUhKWLVsmWTwFBQXIy8vDY489htjYWMyaNQtCgjmLFyxYgHnz5iEsLAwAsHfvXrRs2RJRUVFQKBQYO3ZsrX9Pf4xJCIFDhw7hlVdeQWxsLCZPngyLxVJr8TRs2BAffvghfH19oVAoEBcXh6NHj8JoNOKee+4BAIwbN67WvqebxZOWloZ9+/Zh4cKFiIuLQ1JSEgoLC2slHgAYOHAgfvzxR6hUKuTm5sJutyM3N1fya6kuk1uZBcin3AJYdlWG3MquW8VUX8ovJmoVyMzMRHh4uGs7LCwMGRkZksWTk5OD++67D19++SV2796N7du347PPPqv1OD777DP06tXLtS2H7+mPMRUUFKBnz5545513cPDgQeTn5+O1116rtXjatGnjKtRyc3Px4Ycfom3btpJ9TzeLZ/DgwYiMjMTs2bNx+PBhhIWF4ZlnnqmVeK7SaDR46aWX0L59e/Tt2xcZGRmSX0t1mRx+i38kl3ILYNlVGXIru24VU30pv5ioVcDpdN7wnFIp3dcWHR2NFStWICgoCL6+vpg0aRLWrVsnWTxXye17AoDg4GCsW7cOzZs3h1qtxtSpUyX5rs6fP48///nPeOKJJ9C7d+8bXq/t7+n6ePr06YP169ejU6dOUCgUeP755yX5jl599VXk5eUhLS0Np0+fvuF1qa+lukSOv0W5lluAPL8vll2Vi6m+lF8s/SoQERGBrKws13ZWVhYiIiIki2f//v347rvvXNtOpxNqtVqyeK6S2/cEAKdPn8aSJUtc21J8V4cPH8bdd9+NJ598Ei+++KLk39Mf48nNzS3XSbm2v6MTJ07g2LFjAABfX1889NBD2LZtm+yupbpE6mvsZuRabgHy/L5YdlUupnpTfrmhP51Hy8zMFM2aNRPZ2dnCarWK++67T7KOi0IIsXPnTtGyZUthNBqFxWIRCQkJ4r///a9k8TRr1szVITciIkKkpqYKIYQYM2aMePvttyWN6fz58yIkJESkp6cLp9MpRo0aJV5//fVaiyM3N1eEhoaKVatWlXs+JibG1SF35syZ4plnnpEsnqKiIhEcHCwOHTokhBBi+vTpYvz48bUSjxBCrFixQtx9993CarWK0tJS0bdvX7F06VLZXEt1kdzKLCHkV24JwbLrduRWdt0qpvpSfjFRq4Tly5eLmJgY0aZNGzF16lSpwxFvvfWWaNeunWjdunWtjrq5masFixBCbNu2TcTHx4u2bduKxx57TFgsFsljWrJkiYiOjhZt2rQRjz/+eK3G9K9//Ut4e3uLuLg41+Nf//qXOHbsmOjevbto3769eOCBB0RhYaGk8WzatEnExcWJtm3biqFDh9ZaPFdNmzZNtG/fXsTExIhXXnlFCCGfa6mukluZJYS8yi0hWHbdjtzKrtvFVB/KL4UQEg29ISIiIqLbYh81IiIiIpliokZEREQkU0zUiIiIiGSKiRoRERGRTDFRIyIiIpIpJmpEREREMsVEjYiIiEimmKiRrCkUChgMBsTHx5d77Nq1y+3n+vzzz5GQkOD29yWi+o3lGN0JeSy2RnQbGzdulHztPSKiO8FyjKqLNWpUZ50/fx5NmjTBuHHj0LFjR7Rv3x5r1651vZ6cnAyDwYC4uDj0798fp0+fBgA4HA7885//RFRUFAwGA0aMGAGTyQQAyMvLw9ChQxEXF4d27dph+/btknw2IqofWI5RRVijRrLXr18/aDQa17aPj4+rySA7Oxs9e/bEwoULsWfPHtx33304e/Ysvv/+e6xYsQK7du2Cv78/Fi5ciMGDB+PEiROYP38+duzYgUOHDsHX1xdPPvkkPv74Y4SGhuK3337D119/jejoaMydOxcvvfQSduzYIdVHJyIPwXKMqu3OlyQlqjkARHp6+k1fO3funPDz8yv3XOfOncXKlSvFX/7yF/H++++Xe02v14uUlBQxePBg8cEHH9zwfp999pno3bu3a3vr1q2idevWbvgURFSfsRyjO8EaNarT1Oryl7AQAiqVCk6n84Z9nU4n7HY7NBoNFAqF6/lLly7BbDYDQLk7XoVCASFEDUVORFSG5RjdDvuoUZ12+fJlfPvttwCA3bt349y5c7jnnnswYMAAfPbZZzAajQCA//znP2jYsCGio6Nx//334+uvv0ZJSQkAYNq0aXj//fel+ghEVM+xHKPbYY0ayd4f+3YAwJNPPon+/ftDqVTi22+/xaxZs6BQKLB69Wo0bNgQY8aMwcWLF3H33XfD6XQiNDQU33//PVQqFZ544glkZGSga9euAID4+Hi88847WL58uRQfj4jqAZZjVF0KwTpRqqPOnz+P1q1bw263Sx0KEVG1sByjirDpk4iIiEimWKNGREREJFOsUSMiIiKSKSZqRERERDLFRI2IiIhIppioEREREckUEzUiIiIimWKiRkRERCRTTNSIiIiIZIqJGhEREZFMMVEjIiIikikmakREREQyxUSNiIiISKb+Hw1hkJ/j57smAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 700x250 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "import pickle\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "\n",
    "from experiments.utils import Loss\n",
    "from src.classifier import Classifier, weight_norm\n",
    "from src.network import ConvNet\n",
    "from src.optical_nls import SatAbsNL, Encoding, Gradient\n",
    "from src.utils import Dataset, get_dataset_loaders\n",
    "\n",
    "try:\n",
    "    import seaborn as sns\n",
    "    plt.style.use('seaborn-paper')\n",
    "except ImportError:\n",
    "    pass\n",
    "\n",
    "def run(save_loc=\"cnn/ONN\", loss=Loss.MSE, OD=50, gradient=Gradient.APPROXIMATE):\n",
    "\n",
    "#     print(\"\\n----- Running {} -----\".format(os.path.basename(__file__)))\n",
    "\n",
    "    ####################################################\n",
    "    # Configure datasets.\n",
    "    ####################################################\n",
    "\n",
    "    dataset = Dataset.MNIST\n",
    "\n",
    "    if dataset != Dataset.MNIST:\n",
    "        save_loc += \"_{}\".format(str(dataset).split(\".\")[-1])\n",
    "\n",
    "    batch_size_train = 64\n",
    "    batch_size_test = 1000\n",
    "\n",
    "    ####################################################\n",
    "    # Configure Networks.\n",
    "    ####################################################\n",
    "\n",
    "    sat_abs_nl_args = {'I_sat': 1,\n",
    "                       'OD': OD,\n",
    "                       'encoding': Encoding.AMPLITUDE,\n",
    "                       'gradient': gradient}\n",
    "\n",
    "    SANL = lambda: SatAbsNL(**sat_abs_nl_args)\n",
    "\n",
    "    if loss==Loss.MSE:\n",
    "        output = None\n",
    "        loss_str = \"mse\"\n",
    "    elif loss==Loss.CCE:\n",
    "        output = lambda: nn.LogSoftmax(-1)\n",
    "        loss_str = \"nll\"\n",
    "    else:\n",
    "        raise ValueError(\"Unrecognised loss :\", loss)\n",
    "\n",
    "    net_args = {\n",
    "        'n_ch_conv': [32, 64],\n",
    "        'kernel_size_conv': [5, 5],\n",
    "        'n_in_fc': 1024,\n",
    "        'n_hid_fc': [128],\n",
    "        'activation_conv': [SANL, SANL],\n",
    "        'activation_fc': SANL,\n",
    "        'dropout': lambda: nn.Dropout(0.4),\n",
    "        'conv_args': {'stride': 1, 'padding': 0, 'bias': False},\n",
    "        'pool_conv': lambda: nn.AvgPool2d(kernel_size=2, stride=2),\n",
    "        'n_out': 10 if dataset != Dataset.EMNIST else 47,\n",
    "        'bias_fc': False,\n",
    "        'output': output\n",
    "    }\n",
    "\n",
    "    ####################################################\n",
    "    # Train classifiers\n",
    "    ####################################################\n",
    "\n",
    "    n_seeds = 1\n",
    "\n",
    "    losses = {}\n",
    "    corrects = {}\n",
    "    valid_scores = {}\n",
    "\n",
    "    for i in range(n_seeds):\n",
    "        lab = 'seed{}'.format(i)\n",
    "\n",
    "        network = ConvNet(**net_args)\n",
    "\n",
    "        train_loader, test_loader, validation_loader = get_dataset_loaders(\n",
    "            dataset=dataset,\n",
    "            train_batch=batch_size_train,\n",
    "            test_batch=batch_size_test,\n",
    "            unroll_img=False,\n",
    "            max_value=100 if OD > 10 else 5,\n",
    "            get_validation=True)\n",
    "\n",
    "        classifier = Classifier(network, train_loader, test_loader,\n",
    "                                n_epochs=30 if dataset == Dataset.MNIST else 1,\n",
    "                                learning_rate=5e-4,\n",
    "                                init_weight_mean=0., init_weight_std=0.01, init_conv_weight_std=0.1,\n",
    "                                loss=loss_str,\n",
    "                                weight_range=None,\n",
    "                                weight_normalisation=weight_norm.NONE,\n",
    "                                log_interval=25, n_test_per_epoch=0,\n",
    "                                save_path=os.path.join(save_loc, lab))\n",
    "\n",
    "        train_losses, test_correct = classifier.train()\n",
    "\n",
    "        losses[lab] = train_losses\n",
    "        corrects[lab] = test_correct\n",
    "\n",
    "        ####################################################\n",
    "        # Validation\n",
    "        ####################################################\n",
    "\n",
    "        classifier.load(classifier.network_save_path)\n",
    "\n",
    "        valid_loss, valid_correct = classifier.validate(validation_loader)\n",
    "\n",
    "        print(\"Validation accuracy : {:.2f}%\".format(100. * valid_correct / len(validation_loader.dataset)))\n",
    "        valid_scores[lab] = 100. * valid_correct / len(validation_loader.dataset)\n",
    "\n",
    "        validation_save_path = os.path.join(classifier.save_path, \"validation_score.pkl\")\n",
    "        with open(validation_save_path, 'wb+') as output:\n",
    "            pickle.dump(np.array([valid_loss, valid_correct]), output, pickle.HIGHEST_PROTOCOL)\n",
    "            print('Validation scores saved to {}'.format(validation_save_path))\n",
    "\n",
    "    print(\"Validation scores are:\")\n",
    "    for lab, score in valid_scores.items():\n",
    "        print(\"\\t{} : {:.2f}%\".format(lab, score))\n",
    "\n",
    "    ####################################################\n",
    "    # Plot results\n",
    "    ####################################################\n",
    "\n",
    "    fig_fname = os.path.join(save_loc, \"training_performance\")\n",
    "\n",
    "    with plt.style.context('seaborn-paper', after_reset=True):\n",
    "\n",
    "        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(7, 2.5), gridspec_kw={'wspace': 0.3})\n",
    "\n",
    "        window = 25\n",
    "        avg_mask = np.ones(window) / window\n",
    "\n",
    "        for lab, data in losses.items():\n",
    "            ax1.plot(np.convolve(data[:, 0], avg_mask, 'valid'),\n",
    "                     np.convolve(data[:, 1], avg_mask, 'valid'),\n",
    "                     label=lab, linewidth=0.75, alpha=0.8)\n",
    "        ax1.legend()\n",
    "        ax1.set_xlabel(\"Epoch\")\n",
    "        ax1.set_ylabel(\"Losses\")\n",
    "\n",
    "        for lab, data in corrects.items():\n",
    "            ax2.plot(data[:, 0], data[:, 1] / len(test_loader.dataset), label=lab)\n",
    "            print(\"{}: Best score {}/{}\".format(lab, np.max(data), len(test_loader)))\n",
    "        ax2.legend()\n",
    "        ax2.set_xlabel(\"Epoch\")\n",
    "        ax2.set_ylabel(\"Accuracy\")\n",
    "\n",
    "        plt.savefig(fig_fname + \".png\", bbox_inches='tight')\n",
    "        plt.savefig(fig_fname + \".pdf\", bbox_inches='tight')\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "wrapped-directive",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}