FCN(STN).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from dmnist import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "...loading the dataset\n",
      "...Applying rotation to dataset\n",
      "Current training data size is 50000\n",
      "Current validation data size is 10000\n",
      "Current test data size is 10000\n",
      "...building the model\n",
      "...training\n",
      "('training @ iter = ', 0)\n",
      "('training @ iter = ', 100)\n",
      "epoch 1, minibatch 195/195, validation error 84.395032 %\n",
      "     epoch 1, minibatch 195/195, test error of best model 84.955929 %\n",
      "('training @ iter = ', 200)\n",
      "('training @ iter = ', 300)\n",
      "epoch 2, minibatch 195/195, validation error 76.231971 %\n",
      "     epoch 2, minibatch 195/195, test error of best model 78.215144 %\n",
      "('training @ iter = ', 400)\n",
      "('training @ iter = ', 500)\n",
      "epoch 3, minibatch 195/195, validation error 49.799679 %\n",
      "     epoch 3, minibatch 195/195, test error of best model 51.492388 %\n",
      "('training @ iter = ', 600)\n",
      "('training @ iter = ', 700)\n",
      "epoch 4, minibatch 195/195, validation error 42.247596 %\n",
      "     epoch 4, minibatch 195/195, test error of best model 43.249199 %\n",
      "('training @ iter = ', 800)\n",
      "('training @ iter = ', 900)\n",
      "epoch 5, minibatch 195/195, validation error 39.683494 %\n",
      "     epoch 5, minibatch 195/195, test error of best model 41.095753 %\n",
      "('training @ iter = ', 1000)\n",
      "('training @ iter = ', 1100)\n",
      "epoch 6, minibatch 195/195, validation error 37.379808 %\n",
      "     epoch 6, minibatch 195/195, test error of best model 38.581731 %\n",
      "('training @ iter = ', 1200)\n",
      "('training @ iter = ', 1300)\n",
      "epoch 7, minibatch 195/195, validation error 35.606971 %\n",
      "     epoch 7, minibatch 195/195, test error of best model 36.618590 %\n",
      "('training @ iter = ', 1400)\n",
      "('training @ iter = ', 1500)\n",
      "epoch 8, minibatch 195/195, validation error 34.445112 %\n",
      "     epoch 8, minibatch 195/195, test error of best model 35.256410 %\n",
      "('training @ iter = ', 1600)\n",
      "('training @ iter = ', 1700)\n",
      "epoch 9, minibatch 195/195, validation error 33.593750 %\n",
      "     epoch 9, minibatch 195/195, test error of best model 34.274840 %\n",
      "('training @ iter = ', 1800)\n",
      "('training @ iter = ', 1900)\n",
      "epoch 10, minibatch 195/195, validation error 32.522035 %\n",
      "     epoch 10, minibatch 195/195, test error of best model 33.503606 %\n",
      "('training @ iter = ', 2000)\n",
      "('training @ iter = ', 2100)\n",
      "epoch 11, minibatch 195/195, validation error 31.710737 %\n",
      "     epoch 11, minibatch 195/195, test error of best model 32.612179 %\n",
      "('training @ iter = ', 2200)\n",
      "('training @ iter = ', 2300)\n",
      "epoch 12, minibatch 195/195, validation error 30.809295 %\n",
      "     epoch 12, minibatch 195/195, test error of best model 31.760817 %\n",
      "('training @ iter = ', 2400)\n",
      "('training @ iter = ', 2500)\n",
      "epoch 13, minibatch 195/195, validation error 30.178285 %\n",
      "     epoch 13, minibatch 195/195, test error of best model 30.929487 %\n",
      "('training @ iter = ', 2600)\n",
      "('training @ iter = ', 2700)\n",
      "epoch 14, minibatch 195/195, validation error 29.206731 %\n",
      "     epoch 14, minibatch 195/195, test error of best model 30.258413 %\n",
      "('training @ iter = ', 2800)\n",
      "('training @ iter = ', 2900)\n",
      "epoch 15, minibatch 195/195, validation error 28.285256 %\n",
      "     epoch 15, minibatch 195/195, test error of best model 29.397035 %\n",
      "('training @ iter = ', 3000)\n",
      "('training @ iter = ', 3100)\n",
      "epoch 16, minibatch 195/195, validation error 27.574119 %\n",
      "     epoch 16, minibatch 195/195, test error of best model 28.575721 %\n",
      "('training @ iter = ', 3200)\n",
      "('training @ iter = ', 3300)\n",
      "epoch 17, minibatch 195/195, validation error 26.862981 %\n",
      "     epoch 17, minibatch 195/195, test error of best model 27.714343 %\n",
      "('training @ iter = ', 3400)\n",
      "('training @ iter = ', 3500)\n",
      "epoch 18, minibatch 195/195, validation error 26.081731 %\n",
      "     epoch 18, minibatch 195/195, test error of best model 27.023237 %\n",
      "('training @ iter = ', 3600)\n",
      "('training @ iter = ', 3700)\n",
      "epoch 19, minibatch 195/195, validation error 25.390625 %\n",
      "     epoch 19, minibatch 195/195, test error of best model 26.322115 %\n",
      "('training @ iter = ', 3800)\n",
      "epoch 20, minibatch 195/195, validation error 24.469151 %\n",
      "     epoch 20, minibatch 195/195, test error of best model 25.370593 %\n",
      "('training @ iter = ', 3900)\n",
      "('training @ iter = ', 4000)\n",
      "epoch 21, minibatch 195/195, validation error 23.778045 %\n",
      "     epoch 21, minibatch 195/195, test error of best model 24.589343 %\n",
      "('training @ iter = ', 4100)\n",
      "('training @ iter = ', 4200)\n",
      "epoch 22, minibatch 195/195, validation error 22.986779 %\n",
      "     epoch 22, minibatch 195/195, test error of best model 23.657853 %\n",
      "('training @ iter = ', 4300)\n",
      "('training @ iter = ', 4400)\n",
      "epoch 23, minibatch 195/195, validation error 22.325721 %\n",
      "     epoch 23, minibatch 195/195, test error of best model 22.946715 %\n",
      "('training @ iter = ', 4500)\n",
      "('training @ iter = ', 4600)\n",
      "epoch 24, minibatch 195/195, validation error 21.694712 %\n",
      "     epoch 24, minibatch 195/195, test error of best model 22.085337 %\n",
      "('training @ iter = ', 4700)\n",
      "('training @ iter = ', 4800)\n",
      "epoch 25, minibatch 195/195, validation error 20.953526 %\n",
      "     epoch 25, minibatch 195/195, test error of best model 21.424279 %\n",
      "('training @ iter = ', 4900)\n",
      "('training @ iter = ', 5000)\n",
      "epoch 26, minibatch 195/195, validation error 20.232372 %\n",
      "     epoch 26, minibatch 195/195, test error of best model 20.693109 %\n",
      "('training @ iter = ', 5100)\n",
      "('training @ iter = ', 5200)\n",
      "epoch 27, minibatch 195/195, validation error 19.561298 %\n",
      "     epoch 27, minibatch 195/195, test error of best model 19.941907 %\n",
      "('training @ iter = ', 5300)\n",
      "('training @ iter = ', 5400)\n",
      "epoch 28, minibatch 195/195, validation error 19.010417 %\n",
      "     epoch 28, minibatch 195/195, test error of best model 19.230769 %\n",
      "('training @ iter = ', 5500)\n",
      "('training @ iter = ', 5600)\n",
      "epoch 29, minibatch 195/195, validation error 18.249199 %\n",
      "     epoch 29, minibatch 195/195, test error of best model 18.659856 %\n",
      "('training @ iter = ', 5700)\n",
      "('training @ iter = ', 5800)\n",
      "epoch 30, minibatch 195/195, validation error 17.798478 %\n",
      "     epoch 30, minibatch 195/195, test error of best model 18.169071 %\n",
      "('training @ iter = ', 5900)\n",
      "('training @ iter = ', 6000)\n",
      "epoch 31, minibatch 195/195, validation error 17.317708 %\n",
      "     epoch 31, minibatch 195/195, test error of best model 17.778446 %\n",
      "('training @ iter = ', 6100)\n",
      "('training @ iter = ', 6200)\n",
      "epoch 32, minibatch 195/195, validation error 16.947115 %\n",
      "     epoch 32, minibatch 195/195, test error of best model 17.277644 %\n",
      "('training @ iter = ', 6300)\n",
      "('training @ iter = ', 6400)\n",
      "epoch 33, minibatch 195/195, validation error 16.556490 %\n",
      "     epoch 33, minibatch 195/195, test error of best model 16.957131 %\n",
      "('training @ iter = ', 6500)\n",
      "('training @ iter = ', 6600)\n",
      "epoch 34, minibatch 195/195, validation error 16.185897 %\n",
      "     epoch 34, minibatch 195/195, test error of best model 16.576522 %\n",
      "('training @ iter = ', 6700)\n",
      "('training @ iter = ', 6800)\n",
      "epoch 35, minibatch 195/195, validation error 15.895433 %\n",
      "     epoch 35, minibatch 195/195, test error of best model 16.155849 %\n",
      "('training @ iter = ', 6900)\n",
      "('training @ iter = ', 7000)\n",
      "epoch 36, minibatch 195/195, validation error 15.574920 %\n",
      "     epoch 36, minibatch 195/195, test error of best model 15.745192 %\n",
      "('training @ iter = ', 7100)\n",
      "('training @ iter = ', 7200)\n",
      "epoch 37, minibatch 195/195, validation error 15.314503 %\n",
      "     epoch 37, minibatch 195/195, test error of best model 15.444712 %\n",
      "('training @ iter = ', 7300)\n",
      "('training @ iter = ', 7400)\n",
      "epoch 38, minibatch 195/195, validation error 15.134215 %\n",
      "     epoch 38, minibatch 195/195, test error of best model 15.094151 %\n",
      "('training @ iter = ', 7500)\n",
      "('training @ iter = ', 7600)\n",
      "epoch 39, minibatch 195/195, validation error 14.833734 %\n",
      "     epoch 39, minibatch 195/195, test error of best model 14.893830 %\n",
      "('training @ iter = ', 7700)\n",
      "epoch 40, minibatch 195/195, validation error 14.593349 %\n",
      "     epoch 40, minibatch 195/195, test error of best model 14.603365 %\n",
      "('training @ iter = ', 7800)\n",
      "('training @ iter = ', 7900)\n",
      "epoch 41, minibatch 195/195, validation error 14.413061 %\n",
      "     epoch 41, minibatch 195/195, test error of best model 14.403045 %\n",
      "('training @ iter = ', 8000)\n",
      "('training @ iter = ', 8100)\n",
      "epoch 42, minibatch 195/195, validation error 14.042468 %\n",
      "     epoch 42, minibatch 195/195, test error of best model 14.172676 %\n",
      "('training @ iter = ', 8200)\n",
      "('training @ iter = ', 8300)\n",
      "epoch 43, minibatch 195/195, validation error 13.852163 %\n",
      "     epoch 43, minibatch 195/195, test error of best model 14.022436 %\n",
      "('training @ iter = ', 8400)\n",
      "('training @ iter = ', 8500)\n",
      "epoch 44, minibatch 195/195, validation error 13.701923 %\n",
      "     epoch 44, minibatch 195/195, test error of best model 13.902244 %\n",
      "('training @ iter = ', 8600)\n",
      "('training @ iter = ', 8700)\n",
      "epoch 45, minibatch 195/195, validation error 13.471554 %\n",
      "     epoch 45, minibatch 195/195, test error of best model 13.601763 %\n",
      "('training @ iter = ', 8800)\n",
      "('training @ iter = ', 8900)\n",
      "epoch 46, minibatch 195/195, validation error 13.251202 %\n",
      "     epoch 46, minibatch 195/195, test error of best model 13.381410 %\n",
      "('training @ iter = ', 9000)\n",
      "('training @ iter = ', 9100)\n",
      "epoch 47, minibatch 195/195, validation error 13.080929 %\n",
      "     epoch 47, minibatch 195/195, test error of best model 13.241186 %\n",
      "('training @ iter = ', 9200)\n",
      "('training @ iter = ', 9300)\n",
      "epoch 48, minibatch 195/195, validation error 12.870593 %\n",
      "     epoch 48, minibatch 195/195, test error of best model 13.141026 %\n",
      "('training @ iter = ', 9400)\n",
      "('training @ iter = ', 9500)\n",
      "epoch 49, minibatch 195/195, validation error 12.740385 %\n",
      "     epoch 49, minibatch 195/195, test error of best model 13.060897 %\n",
      "('training @ iter = ', 9600)\n",
      "('training @ iter = ', 9700)\n",
      "epoch 50, minibatch 195/195, validation error 12.600160 %\n",
      "     epoch 50, minibatch 195/195, test error of best model 12.940705 %\n",
      "('training @ iter = ', 9800)\n",
      "('training @ iter = ', 9900)\n",
      "epoch 51, minibatch 195/195, validation error 12.419872 %\n",
      "     epoch 51, minibatch 195/195, test error of best model 12.930689 %\n",
      "('training @ iter = ', 10000)\n",
      "('training @ iter = ', 10100)\n",
      "epoch 52, minibatch 195/195, validation error 12.309696 %\n",
      "     epoch 52, minibatch 195/195, test error of best model 12.780449 %\n",
      "('training @ iter = ', 10200)\n",
      "('training @ iter = ', 10300)\n",
      "epoch 53, minibatch 195/195, validation error 12.149439 %\n",
      "     epoch 53, minibatch 195/195, test error of best model 12.670272 %\n",
      "('training @ iter = ', 10400)\n",
      "('training @ iter = ', 10500)\n",
      "epoch 54, minibatch 195/195, validation error 11.979167 %\n",
      "     epoch 54, minibatch 195/195, test error of best model 12.550080 %\n",
      "('training @ iter = ', 10600)\n",
      "('training @ iter = ', 10700)\n",
      "epoch 55, minibatch 195/195, validation error 11.818910 %\n",
      "     epoch 55, minibatch 195/195, test error of best model 12.469952 %\n",
      "('training @ iter = ', 10800)\n",
      "('training @ iter = ', 10900)\n",
      "epoch 56, minibatch 195/195, validation error 11.678686 %\n",
      "     epoch 56, minibatch 195/195, test error of best model 12.429888 %\n",
      "('training @ iter = ', 11000)\n",
      "('training @ iter = ', 11100)\n",
      "epoch 57, minibatch 195/195, validation error 11.538462 %\n",
      "     epoch 57, minibatch 195/195, test error of best model 12.379808 %\n",
      "('training @ iter = ', 11200)\n",
      "('training @ iter = ', 11300)\n",
      "epoch 58, minibatch 195/195, validation error 11.388221 %\n",
      "     epoch 58, minibatch 195/195, test error of best model 12.369792 %\n",
      "('training @ iter = ', 11400)\n",
      "('training @ iter = ', 11500)\n",
      "epoch 59, minibatch 195/195, validation error 11.258013 %\n",
      "     epoch 59, minibatch 195/195, test error of best model 12.309696 %\n",
      "('training @ iter = ', 11600)\n",
      "epoch 60, minibatch 195/195, validation error 11.187901 %\n",
      "     epoch 60, minibatch 195/195, test error of best model 12.229567 %\n",
      "('training @ iter = ', 11700)\n",
      "('training @ iter = ', 11800)\n",
      "epoch 61, minibatch 195/195, validation error 11.167869 %\n",
      "     epoch 61, minibatch 195/195, test error of best model 12.139423 %\n",
      "('training @ iter = ', 11900)\n",
      "('training @ iter = ', 12000)\n",
      "epoch 62, minibatch 195/195, validation error 11.127804 %\n",
      "     epoch 62, minibatch 195/195, test error of best model 12.059295 %\n",
      "('training @ iter = ', 12100)\n",
      "('training @ iter = ', 12200)\n",
      "epoch 63, minibatch 195/195, validation error 11.047676 %\n",
      "     epoch 63, minibatch 195/195, test error of best model 11.969151 %\n",
      "('training @ iter = ', 12300)\n",
      "('training @ iter = ', 12400)\n",
      "epoch 64, minibatch 195/195, validation error 10.897436 %\n",
      "     epoch 64, minibatch 195/195, test error of best model 11.919071 %\n",
      "('training @ iter = ', 12500)\n",
      "('training @ iter = ', 12600)\n",
      "epoch 65, minibatch 195/195, validation error 10.867388 %\n",
      "     epoch 65, minibatch 195/195, test error of best model 11.879006 %\n",
      "('training @ iter = ', 12700)\n",
      "('training @ iter = ', 12800)\n",
      "epoch 66, minibatch 195/195, validation error 10.757212 %\n",
      "     epoch 66, minibatch 195/195, test error of best model 11.788862 %\n",
      "('training @ iter = ', 12900)\n",
      "('training @ iter = ', 13000)\n",
      "epoch 67, minibatch 195/195, validation error 10.687099 %\n",
      "     epoch 67, minibatch 195/195, test error of best model 11.648638 %\n",
      "('training @ iter = ', 13100)\n",
      "('training @ iter = ', 13200)\n",
      "epoch 68, minibatch 195/195, validation error 10.637019 %\n",
      "     epoch 68, minibatch 195/195, test error of best model 11.568510 %\n",
      "('training @ iter = ', 13300)\n",
      "('training @ iter = ', 13400)\n",
      "epoch 69, minibatch 195/195, validation error 10.526843 %\n",
      "     epoch 69, minibatch 195/195, test error of best model 11.418269 %\n",
      "('training @ iter = ', 13500)\n",
      "('training @ iter = ', 13600)\n",
      "epoch 70, minibatch 195/195, validation error 10.466747 %\n",
      "     epoch 70, minibatch 195/195, test error of best model 11.338141 %\n",
      "('training @ iter = ', 13700)\n",
      "('training @ iter = ', 13800)\n",
      "epoch 71, minibatch 195/195, validation error 10.386619 %\n",
      "     epoch 71, minibatch 195/195, test error of best model 11.298077 %\n",
      "('training @ iter = ', 13900)\n",
      "('training @ iter = ', 14000)\n",
      "epoch 72, minibatch 195/195, validation error 10.316506 %\n",
      "     epoch 72, minibatch 195/195, test error of best model 11.207933 %\n",
      "('training @ iter = ', 14100)\n",
      "('training @ iter = ', 14200)\n",
      "epoch 73, minibatch 195/195, validation error 10.206330 %\n",
      "     epoch 73, minibatch 195/195, test error of best model 11.147837 %\n",
      "('training @ iter = ', 14300)\n",
      "('training @ iter = ', 14400)\n",
      "epoch 74, minibatch 195/195, validation error 10.116186 %\n",
      "     epoch 74, minibatch 195/195, test error of best model 11.107772 %\n",
      "('training @ iter = ', 14500)\n",
      "('training @ iter = ', 14600)\n",
      "epoch 75, minibatch 195/195, validation error 10.026042 %\n",
      "     epoch 75, minibatch 195/195, test error of best model 11.047676 %\n",
      "('training @ iter = ', 14700)\n",
      "('training @ iter = ', 14800)\n",
      "epoch 76, minibatch 195/195, validation error 10.016026 %\n",
      "     epoch 76, minibatch 195/195, test error of best model 10.957532 %\n",
      "('training @ iter = ', 14900)\n",
      "('training @ iter = ', 15000)\n",
      "epoch 77, minibatch 195/195, validation error 9.935897 %\n",
      "     epoch 77, minibatch 195/195, test error of best model 10.877404 %\n",
      "('training @ iter = ', 15100)\n",
      "('training @ iter = ', 15200)\n",
      "epoch 78, minibatch 195/195, validation error 9.895833 %\n",
      "     epoch 78, minibatch 195/195, test error of best model 10.777244 %\n",
      "('training @ iter = ', 15300)\n",
      "('training @ iter = ', 15400)\n",
      "epoch 79, minibatch 195/195, validation error 9.895833 %\n",
      "('training @ iter = ', 15500)\n",
      "epoch 80, minibatch 195/195, validation error 9.885817 %\n",
      "     epoch 80, minibatch 195/195, test error of best model 10.697115 %\n",
      "('training @ iter = ', 15600)\n",
      "('training @ iter = ', 15700)\n",
      "epoch 81, minibatch 195/195, validation error 9.825721 %\n",
      "     epoch 81, minibatch 195/195, test error of best model 10.647035 %\n",
      "('training @ iter = ', 15800)\n",
      "('training @ iter = ', 15900)\n",
      "epoch 82, minibatch 195/195, validation error 9.775641 %\n",
      "     epoch 82, minibatch 195/195, test error of best model 10.556891 %\n",
      "('training @ iter = ', 16000)\n",
      "('training @ iter = ', 16100)\n",
      "epoch 83, minibatch 195/195, validation error 9.645433 %\n",
      "     epoch 83, minibatch 195/195, test error of best model 10.516827 %\n",
      "('training @ iter = ', 16200)\n",
      "('training @ iter = ', 16300)\n",
      "epoch 84, minibatch 195/195, validation error 9.545272 %\n",
      "     epoch 84, minibatch 195/195, test error of best model 10.496795 %\n",
      "('training @ iter = ', 16400)\n",
      "('training @ iter = ', 16500)\n",
      "epoch 85, minibatch 195/195, validation error 9.505208 %\n",
      "     epoch 85, minibatch 195/195, test error of best model 10.446715 %\n",
      "('training @ iter = ', 16600)\n",
      "('training @ iter = ', 16700)\n",
      "epoch 86, minibatch 195/195, validation error 9.465144 %\n",
      "     epoch 86, minibatch 195/195, test error of best model 10.356571 %\n",
      "('training @ iter = ', 16800)\n",
      "('training @ iter = ', 16900)\n",
      "epoch 87, minibatch 195/195, validation error 9.395032 %\n",
      "     epoch 87, minibatch 195/195, test error of best model 10.316506 %\n",
      "('training @ iter = ', 17000)\n",
      "('training @ iter = ', 17100)\n",
      "epoch 88, minibatch 195/195, validation error 9.385016 %\n",
      "     epoch 88, minibatch 195/195, test error of best model 10.206330 %\n",
      "('training @ iter = ', 17200)\n",
      "('training @ iter = ', 17300)\n",
      "epoch 89, minibatch 195/195, validation error 9.304888 %\n",
      "     epoch 89, minibatch 195/195, test error of best model 10.096154 %\n",
      "('training @ iter = ', 17400)\n",
      "('training @ iter = ', 17500)\n",
      "epoch 90, minibatch 195/195, validation error 9.284856 %\n",
      "     epoch 90, minibatch 195/195, test error of best model 10.056090 %\n",
      "('training @ iter = ', 17600)\n",
      "('training @ iter = ', 17700)\n",
      "epoch 91, minibatch 195/195, validation error 9.254808 %\n",
      "     epoch 91, minibatch 195/195, test error of best model 9.995994 %\n",
      "('training @ iter = ', 17800)\n",
      "('training @ iter = ', 17900)\n",
      "epoch 92, minibatch 195/195, validation error 9.144631 %\n",
      "     epoch 92, minibatch 195/195, test error of best model 9.925881 %\n",
      "('training @ iter = ', 18000)\n",
      "('training @ iter = ', 18100)\n",
      "epoch 93, minibatch 195/195, validation error 9.124599 %\n",
      "     epoch 93, minibatch 195/195, test error of best model 9.855769 %\n",
      "('training @ iter = ', 18200)\n",
      "('training @ iter = ', 18300)\n",
      "epoch 94, minibatch 195/195, validation error 9.114583 %\n",
      "     epoch 94, minibatch 195/195, test error of best model 9.795673 %\n",
      "('training @ iter = ', 18400)\n",
      "('training @ iter = ', 18500)\n",
      "epoch 95, minibatch 195/195, validation error 9.024439 %\n",
      "     epoch 95, minibatch 195/195, test error of best model 9.785657 %\n",
      "('training @ iter = ', 18600)\n",
      "('training @ iter = ', 18700)\n",
      "epoch 96, minibatch 195/195, validation error 8.984375 %\n",
      "     epoch 96, minibatch 195/195, test error of best model 9.705529 %\n",
      "('training @ iter = ', 18800)\n",
      "('training @ iter = ', 18900)\n",
      "epoch 97, minibatch 195/195, validation error 8.974359 %\n",
      "     epoch 97, minibatch 195/195, test error of best model 9.705529 %\n",
      "('training @ iter = ', 19000)\n",
      "('training @ iter = ', 19100)\n",
      "epoch 98, minibatch 195/195, validation error 8.934295 %\n",
      "     epoch 98, minibatch 195/195, test error of best model 9.675481 %\n",
      "('training @ iter = ', 19200)\n",
      "('training @ iter = ', 19300)\n",
      "epoch 99, minibatch 195/195, validation error 8.934295 %\n",
      "('training @ iter = ', 19400)\n",
      "epoch 100, minibatch 195/195, validation error 8.854167 %\n",
      "     epoch 100, minibatch 195/195, test error of best model 9.565304 %\n",
      "('training @ iter = ', 19500)\n",
      "('training @ iter = ', 19600)\n",
      "epoch 101, minibatch 195/195, validation error 8.844151 %\n",
      "     epoch 101, minibatch 195/195, test error of best model 9.515224 %\n",
      "('training @ iter = ', 19700)\n",
      "('training @ iter = ', 19800)\n",
      "epoch 102, minibatch 195/195, validation error 8.844151 %\n",
      "('training @ iter = ', 19900)\n",
      "('training @ iter = ', 20000)\n",
      "epoch 103, minibatch 195/195, validation error 8.804087 %\n",
      "     epoch 103, minibatch 195/195, test error of best model 9.445112 %\n",
      "('training @ iter = ', 20100)\n",
      "('training @ iter = ', 20200)\n",
      "epoch 104, minibatch 195/195, validation error 8.784054 %\n",
      "     epoch 104, minibatch 195/195, test error of best model 9.405048 %\n",
      "('training @ iter = ', 20300)\n",
      "('training @ iter = ', 20400)\n",
      "epoch 105, minibatch 195/195, validation error 8.774038 %\n",
      "     epoch 105, minibatch 195/195, test error of best model 9.364984 %\n",
      "('training @ iter = ', 20500)\n",
      "('training @ iter = ', 20600)\n",
      "epoch 106, minibatch 195/195, validation error 8.754006 %\n",
      "     epoch 106, minibatch 195/195, test error of best model 9.334936 %\n",
      "('training @ iter = ', 20700)\n",
      "('training @ iter = ', 20800)\n",
      "epoch 107, minibatch 195/195, validation error 8.673878 %\n",
      "     epoch 107, minibatch 195/195, test error of best model 9.274840 %\n",
      "('training @ iter = ', 20900)\n",
      "('training @ iter = ', 21000)\n",
      "epoch 108, minibatch 195/195, validation error 8.683894 %\n",
      "('training @ iter = ', 21100)\n",
      "('training @ iter = ', 21200)\n",
      "epoch 109, minibatch 195/195, validation error 8.653846 %\n",
      "     epoch 109, minibatch 195/195, test error of best model 9.204728 %\n",
      "('training @ iter = ', 21300)\n",
      "('training @ iter = ', 21400)\n",
      "epoch 110, minibatch 195/195, validation error 8.603766 %\n",
      "     epoch 110, minibatch 195/195, test error of best model 9.164663 %\n",
      "('training @ iter = ', 21500)\n",
      "('training @ iter = ', 21600)\n",
      "epoch 111, minibatch 195/195, validation error 8.613782 %\n",
      "('training @ iter = ', 21700)\n",
      "('training @ iter = ', 21800)\n",
      "epoch 112, minibatch 195/195, validation error 8.573718 %\n",
      "     epoch 112, minibatch 195/195, test error of best model 9.144631 %\n",
      "('training @ iter = ', 21900)\n",
      "('training @ iter = ', 22000)\n",
      "epoch 113, minibatch 195/195, validation error 8.573718 %\n",
      "('training @ iter = ', 22100)\n",
      "('training @ iter = ', 22200)\n",
      "epoch 114, minibatch 195/195, validation error 8.573718 %\n",
      "('training @ iter = ', 22300)\n",
      "('training @ iter = ', 22400)\n",
      "epoch 115, minibatch 195/195, validation error 8.543670 %\n",
      "     epoch 115, minibatch 195/195, test error of best model 9.054487 %\n",
      "('training @ iter = ', 22500)\n",
      "('training @ iter = ', 22600)\n",
      "epoch 116, minibatch 195/195, validation error 8.493590 %\n",
      "     epoch 116, minibatch 195/195, test error of best model 9.054487 %\n",
      "('training @ iter = ', 22700)\n",
      "('training @ iter = ', 22800)\n",
      "epoch 117, minibatch 195/195, validation error 8.453526 %\n",
      "     epoch 117, minibatch 195/195, test error of best model 9.044471 %\n",
      "('training @ iter = ', 22900)\n",
      "('training @ iter = ', 23000)\n",
      "epoch 118, minibatch 195/195, validation error 8.453526 %\n",
      "('training @ iter = ', 23100)\n",
      "('training @ iter = ', 23200)\n",
      "epoch 119, minibatch 195/195, validation error 8.433494 %\n",
      "     epoch 119, minibatch 195/195, test error of best model 8.934295 %\n",
      "('training @ iter = ', 23300)\n",
      "epoch 120, minibatch 195/195, validation error 8.423478 %\n",
      "     epoch 120, minibatch 195/195, test error of best model 8.934295 %\n",
      "('training @ iter = ', 23400)\n",
      "('training @ iter = ', 23500)\n",
      "epoch 121, minibatch 195/195, validation error 8.383413 %\n",
      "     epoch 121, minibatch 195/195, test error of best model 8.914263 %\n",
      "('training @ iter = ', 23600)\n",
      "('training @ iter = ', 23700)\n",
      "epoch 122, minibatch 195/195, validation error 8.373397 %\n",
      "     epoch 122, minibatch 195/195, test error of best model 8.914263 %\n",
      "('training @ iter = ', 23800)\n",
      "('training @ iter = ', 23900)\n",
      "epoch 123, minibatch 195/195, validation error 8.373397 %\n",
      "('training @ iter = ', 24000)\n",
      "('training @ iter = ', 24100)\n",
      "epoch 124, minibatch 195/195, validation error 8.383413 %\n",
      "('training @ iter = ', 24200)\n",
      "('training @ iter = ', 24300)\n",
      "epoch 125, minibatch 195/195, validation error 8.373397 %\n",
      "('training @ iter = ', 24400)\n",
      "('training @ iter = ', 24500)\n",
      "epoch 126, minibatch 195/195, validation error 8.353365 %\n",
      "     epoch 126, minibatch 195/195, test error of best model 8.784054 %\n",
      "('training @ iter = ', 24600)\n",
      "('training @ iter = ', 24700)\n",
      "epoch 127, minibatch 195/195, validation error 8.383413 %\n",
      "('training @ iter = ', 24800)\n",
      "('training @ iter = ', 24900)\n",
      "epoch 128, minibatch 195/195, validation error 8.403446 %\n",
      "('training @ iter = ', 25000)\n",
      "('training @ iter = ', 25100)\n",
      "epoch 129, minibatch 195/195, validation error 8.373397 %\n",
      "('training @ iter = ', 25200)\n",
      "('training @ iter = ', 25300)\n",
      "epoch 130, minibatch 195/195, validation error 8.373397 %\n",
      "('training @ iter = ', 25400)\n",
      "('training @ iter = ', 25500)\n",
      "epoch 131, minibatch 195/195, validation error 8.313301 %\n",
      "     epoch 131, minibatch 195/195, test error of best model 8.633814 %\n",
      "('training @ iter = ', 25600)\n",
      "('training @ iter = ', 25700)\n",
      "epoch 132, minibatch 195/195, validation error 8.263221 %\n",
      "     epoch 132, minibatch 195/195, test error of best model 8.623798 %\n",
      "('training @ iter = ', 25800)\n",
      "('training @ iter = ', 25900)\n",
      "epoch 133, minibatch 195/195, validation error 8.253205 %\n",
      "     epoch 133, minibatch 195/195, test error of best model 8.623798 %\n",
      "('training @ iter = ', 26000)\n",
      "('training @ iter = ', 26100)\n",
      "epoch 134, minibatch 195/195, validation error 8.203125 %\n",
      "     epoch 134, minibatch 195/195, test error of best model 8.573718 %\n",
      "('training @ iter = ', 26200)\n",
      "('training @ iter = ', 26300)\n",
      "epoch 135, minibatch 195/195, validation error 8.173077 %\n",
      "     epoch 135, minibatch 195/195, test error of best model 8.573718 %\n",
      "('training @ iter = ', 26400)\n",
      "('training @ iter = ', 26500)\n",
      "epoch 136, minibatch 195/195, validation error 8.112981 %\n",
      "     epoch 136, minibatch 195/195, test error of best model 8.553686 %\n",
      "('training @ iter = ', 26600)\n",
      "('training @ iter = ', 26700)\n",
      "epoch 137, minibatch 195/195, validation error 8.062901 %\n",
      "     epoch 137, minibatch 195/195, test error of best model 8.543670 %\n",
      "('training @ iter = ', 26800)\n",
      "('training @ iter = ', 26900)\n",
      "epoch 138, minibatch 195/195, validation error 8.022837 %\n",
      "     epoch 138, minibatch 195/195, test error of best model 8.503606 %\n",
      "('training @ iter = ', 27000)\n",
      "('training @ iter = ', 27100)\n",
      "epoch 139, minibatch 195/195, validation error 8.012821 %\n",
      "     epoch 139, minibatch 195/195, test error of best model 8.493590 %\n",
      "('training @ iter = ', 27200)\n",
      "epoch 140, minibatch 195/195, validation error 8.012821 %\n",
      "('training @ iter = ', 27300)\n",
      "('training @ iter = ', 27400)\n",
      "epoch 141, minibatch 195/195, validation error 8.002804 %\n",
      "     epoch 141, minibatch 195/195, test error of best model 8.473558 %\n",
      "('training @ iter = ', 27500)\n",
      "('training @ iter = ', 27600)\n",
      "epoch 142, minibatch 195/195, validation error 7.992788 %\n",
      "     epoch 142, minibatch 195/195, test error of best model 8.403446 %\n",
      "('training @ iter = ', 27700)\n",
      "('training @ iter = ', 27800)\n",
      "epoch 143, minibatch 195/195, validation error 7.962740 %\n",
      "     epoch 143, minibatch 195/195, test error of best model 8.363381 %\n",
      "('training @ iter = ', 27900)\n",
      "('training @ iter = ', 28000)\n",
      "epoch 144, minibatch 195/195, validation error 7.932692 %\n",
      "     epoch 144, minibatch 195/195, test error of best model 8.343349 %\n",
      "('training @ iter = ', 28100)\n",
      "('training @ iter = ', 28200)\n",
      "epoch 145, minibatch 195/195, validation error 7.942708 %\n",
      "('training @ iter = ', 28300)\n",
      "('training @ iter = ', 28400)\n",
      "epoch 146, minibatch 195/195, validation error 7.922676 %\n",
      "     epoch 146, minibatch 195/195, test error of best model 8.353365 %\n",
      "('training @ iter = ', 28500)\n",
      "('training @ iter = ', 28600)\n",
      "epoch 147, minibatch 195/195, validation error 7.882612 %\n",
      "     epoch 147, minibatch 195/195, test error of best model 8.353365 %\n",
      "('training @ iter = ', 28700)\n",
      "('training @ iter = ', 28800)\n",
      "epoch 148, minibatch 195/195, validation error 7.872596 %\n",
      "     epoch 148, minibatch 195/195, test error of best model 8.353365 %\n",
      "('training @ iter = ', 28900)\n",
      "('training @ iter = ', 29000)\n",
      "epoch 149, minibatch 195/195, validation error 7.832532 %\n",
      "     epoch 149, minibatch 195/195, test error of best model 8.343349 %\n",
      "('training @ iter = ', 29100)\n",
      "('training @ iter = ', 29200)\n",
      "epoch 150, minibatch 195/195, validation error 7.832532 %\n",
      "('training @ iter = ', 29300)\n",
      "('training @ iter = ', 29400)\n",
      "epoch 151, minibatch 195/195, validation error 7.782452 %\n",
      "     epoch 151, minibatch 195/195, test error of best model 8.313301 %\n",
      "('training @ iter = ', 29500)\n",
      "('training @ iter = ', 29600)\n",
      "epoch 152, minibatch 195/195, validation error 7.792468 %\n",
      "('training @ iter = ', 29700)\n",
      "('training @ iter = ', 29800)\n",
      "epoch 153, minibatch 195/195, validation error 7.782452 %\n",
      "('training @ iter = ', 29900)\n",
      "('training @ iter = ', 30000)\n",
      "epoch 154, minibatch 195/195, validation error 7.762420 %\n",
      "     epoch 154, minibatch 195/195, test error of best model 8.243189 %\n",
      "('training @ iter = ', 30100)\n",
      "('training @ iter = ', 30200)\n",
      "epoch 155, minibatch 195/195, validation error 7.762420 %\n",
      "('training @ iter = ', 30300)\n",
      "('training @ iter = ', 30400)\n",
      "epoch 156, minibatch 195/195, validation error 7.742388 %\n",
      "     epoch 156, minibatch 195/195, test error of best model 8.223157 %\n",
      "('training @ iter = ', 30500)\n",
      "('training @ iter = ', 30600)\n",
      "epoch 157, minibatch 195/195, validation error 7.732372 %\n",
      "     epoch 157, minibatch 195/195, test error of best model 8.213141 %\n",
      "('training @ iter = ', 30700)\n",
      "('training @ iter = ', 30800)\n",
      "epoch 158, minibatch 195/195, validation error 7.752404 %\n",
      "('training @ iter = ', 30900)\n",
      "('training @ iter = ', 31000)\n",
      "epoch 159, minibatch 195/195, validation error 7.752404 %\n",
      "('training @ iter = ', 31100)\n",
      "epoch 160, minibatch 195/195, validation error 7.722356 %\n",
      "     epoch 160, minibatch 195/195, test error of best model 8.153045 %\n",
      "('training @ iter = ', 31200)\n",
      "('training @ iter = ', 31300)\n",
      "epoch 161, minibatch 195/195, validation error 7.732372 %\n",
      "('training @ iter = ', 31400)\n",
      "('training @ iter = ', 31500)\n",
      "epoch 162, minibatch 195/195, validation error 7.682292 %\n",
      "     epoch 162, minibatch 195/195, test error of best model 8.143029 %\n",
      "('training @ iter = ', 31600)\n",
      "('training @ iter = ', 31700)\n",
      "epoch 163, minibatch 195/195, validation error 7.692308 %\n",
      "('training @ iter = ', 31800)\n",
      "('training @ iter = ', 31900)\n",
      "epoch 164, minibatch 195/195, validation error 7.662260 %\n",
      "     epoch 164, minibatch 195/195, test error of best model 8.112981 %\n",
      "('training @ iter = ', 32000)\n",
      "('training @ iter = ', 32100)\n",
      "epoch 165, minibatch 195/195, validation error 7.652244 %\n",
      "     epoch 165, minibatch 195/195, test error of best model 8.092949 %\n",
      "('training @ iter = ', 32200)\n",
      "('training @ iter = ', 32300)\n",
      "epoch 166, minibatch 195/195, validation error 7.652244 %\n",
      "('training @ iter = ', 32400)\n",
      "('training @ iter = ', 32500)\n",
      "epoch 167, minibatch 195/195, validation error 7.662260 %\n",
      "('training @ iter = ', 32600)\n",
      "('training @ iter = ', 32700)\n",
      "epoch 168, minibatch 195/195, validation error 7.622196 %\n",
      "     epoch 168, minibatch 195/195, test error of best model 8.082933 %\n",
      "('training @ iter = ', 32800)\n",
      "('training @ iter = ', 32900)\n",
      "epoch 169, minibatch 195/195, validation error 7.622196 %\n",
      "('training @ iter = ', 33000)\n",
      "('training @ iter = ', 33100)\n",
      "epoch 170, minibatch 195/195, validation error 7.622196 %\n",
      "('training @ iter = ', 33200)\n",
      "('training @ iter = ', 33300)\n",
      "epoch 171, minibatch 195/195, validation error 7.622196 %\n",
      "('training @ iter = ', 33400)\n",
      "('training @ iter = ', 33500)\n",
      "epoch 172, minibatch 195/195, validation error 7.622196 %\n",
      "('training @ iter = ', 33600)\n",
      "('training @ iter = ', 33700)\n",
      "epoch 173, minibatch 195/195, validation error 7.642228 %\n",
      "('training @ iter = ', 33800)\n",
      "('training @ iter = ', 33900)\n",
      "epoch 174, minibatch 195/195, validation error 7.622196 %\n",
      "('training @ iter = ', 34000)\n",
      "('training @ iter = ', 34100)\n",
      "epoch 175, minibatch 195/195, validation error 7.642228 %\n",
      "('training @ iter = ', 34200)\n",
      "('training @ iter = ', 34300)\n",
      "epoch 176, minibatch 195/195, validation error 7.632212 %\n",
      "('training @ iter = ', 34400)\n",
      "('training @ iter = ', 34500)\n",
      "epoch 177, minibatch 195/195, validation error 7.622196 %\n",
      "('training @ iter = ', 34600)\n",
      "('training @ iter = ', 34700)\n",
      "epoch 178, minibatch 195/195, validation error 7.632212 %\n",
      "('training @ iter = ', 34800)\n",
      "('training @ iter = ', 34900)\n",
      "epoch 179, minibatch 195/195, validation error 7.622196 %\n",
      "('training @ iter = ', 35000)\n",
      "epoch 180, minibatch 195/195, validation error 7.632212 %\n",
      "('training @ iter = ', 35100)\n",
      "('training @ iter = ', 35200)\n",
      "epoch 181, minibatch 195/195, validation error 7.642228 %\n",
      "('training @ iter = ', 35300)\n",
      "('training @ iter = ', 35400)\n",
      "epoch 182, minibatch 195/195, validation error 7.602163 %\n",
      "     epoch 182, minibatch 195/195, test error of best model 7.972756 %\n",
      "('training @ iter = ', 35500)\n",
      "('training @ iter = ', 35600)\n",
      "epoch 183, minibatch 195/195, validation error 7.592147 %\n",
      "     epoch 183, minibatch 195/195, test error of best model 7.972756 %\n",
      "('training @ iter = ', 35700)\n",
      "('training @ iter = ', 35800)\n",
      "epoch 184, minibatch 195/195, validation error 7.582131 %\n",
      "     epoch 184, minibatch 195/195, test error of best model 7.962740 %\n",
      "('training @ iter = ', 35900)\n",
      "('training @ iter = ', 36000)\n",
      "epoch 185, minibatch 195/195, validation error 7.582131 %\n",
      "('training @ iter = ', 36100)\n",
      "('training @ iter = ', 36200)\n",
      "epoch 186, minibatch 195/195, validation error 7.592147 %\n",
      "('training @ iter = ', 36300)\n",
      "('training @ iter = ', 36400)\n",
      "epoch 187, minibatch 195/195, validation error 7.582131 %\n",
      "('training @ iter = ', 36500)\n",
      "('training @ iter = ', 36600)\n",
      "epoch 188, minibatch 195/195, validation error 7.552083 %\n",
      "     epoch 188, minibatch 195/195, test error of best model 7.922676 %\n",
      "('training @ iter = ', 36700)\n",
      "('training @ iter = ', 36800)\n",
      "epoch 189, minibatch 195/195, validation error 7.552083 %\n",
      "('training @ iter = ', 36900)\n",
      "('training @ iter = ', 37000)\n",
      "epoch 190, minibatch 195/195, validation error 7.542067 %\n",
      "     epoch 190, minibatch 195/195, test error of best model 7.862580 %\n",
      "('training @ iter = ', 37100)\n",
      "('training @ iter = ', 37200)\n",
      "epoch 191, minibatch 195/195, validation error 7.522035 %\n",
      "     epoch 191, minibatch 195/195, test error of best model 7.892628 %\n",
      "('training @ iter = ', 37300)\n",
      "('training @ iter = ', 37400)\n",
      "epoch 192, minibatch 195/195, validation error 7.512019 %\n",
      "     epoch 192, minibatch 195/195, test error of best model 7.872596 %\n",
      "('training @ iter = ', 37500)\n",
      "('training @ iter = ', 37600)\n",
      "epoch 193, minibatch 195/195, validation error 7.512019 %\n",
      "('training @ iter = ', 37700)\n",
      "('training @ iter = ', 37800)\n",
      "epoch 194, minibatch 195/195, validation error 7.532051 %\n",
      "('training @ iter = ', 37900)\n",
      "('training @ iter = ', 38000)\n",
      "epoch 195, minibatch 195/195, validation error 7.532051 %\n",
      "('training @ iter = ', 38100)\n",
      "('training @ iter = ', 38200)\n",
      "epoch 196, minibatch 195/195, validation error 7.512019 %\n",
      "('training @ iter = ', 38300)\n",
      "('training @ iter = ', 38400)\n",
      "epoch 197, minibatch 195/195, validation error 7.491987 %\n",
      "     epoch 197, minibatch 195/195, test error of best model 7.852564 %\n",
      "('training @ iter = ', 38500)\n",
      "('training @ iter = ', 38600)\n",
      "epoch 198, minibatch 195/195, validation error 7.502003 %\n",
      "('training @ iter = ', 38700)\n",
      "('training @ iter = ', 38800)\n",
      "epoch 199, minibatch 195/195, validation error 7.491987 %\n",
      "('training @ iter = ', 38900)\n",
      "epoch 200, minibatch 195/195, validation error 7.512019 %\n",
      "Optimization complete.\n",
      "Best validation score of 7.491987 % obtained at iteration 38415, with test performance 7.852564 %\n",
      "The training process for function run ran for 3.76m\n"
     ]
    }
   ],
   "source": [
    "# FCN Rotation\n",
    "run(network_type = 'fcn', include_st = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "...loading the dataset\n",
      "...Applying rotation to dataset\n",
      "Current training data size is 50000\n",
      "Current validation data size is 10000\n",
      "Current test data size is 10000\n",
      "...building the model\n",
      "...Apply STN before CNN\n",
      "...Init localization to identity\n",
      "...training\n",
      "('training @ iter = ', 0)\n",
      "('training @ iter = ', 100)\n",
      "epoch 1, minibatch 195/195, validation error 85.777244 %\n",
      "     epoch 1, minibatch 195/195, test error of best model 86.408253 %\n",
      "('training @ iter = ', 200)\n",
      "('training @ iter = ', 300)\n",
      "epoch 2, minibatch 195/195, validation error 58.042869 %\n",
      "     epoch 2, minibatch 195/195, test error of best model 60.116186 %\n",
      "('training @ iter = ', 400)\n",
      "('training @ iter = ', 500)\n",
      "epoch 3, minibatch 195/195, validation error 42.708333 %\n",
      "     epoch 3, minibatch 195/195, test error of best model 43.950321 %\n",
      "('training @ iter = ', 600)\n",
      "('training @ iter = ', 700)\n",
      "epoch 4, minibatch 195/195, validation error 36.618590 %\n",
      "     epoch 4, minibatch 195/195, test error of best model 37.339744 %\n",
      "('training @ iter = ', 800)\n",
      "('training @ iter = ', 900)\n",
      "epoch 5, minibatch 195/195, validation error 31.500401 %\n",
      "     epoch 5, minibatch 195/195, test error of best model 33.012821 %\n",
      "('training @ iter = ', 1000)\n",
      "('training @ iter = ', 1100)\n",
      "epoch 6, minibatch 195/195, validation error 28.695913 %\n",
      "     epoch 6, minibatch 195/195, test error of best model 29.527244 %\n",
      "('training @ iter = ', 1200)\n",
      "('training @ iter = ', 1300)\n",
      "epoch 7, minibatch 195/195, validation error 27.003205 %\n",
      "     epoch 7, minibatch 195/195, test error of best model 28.515625 %\n",
      "('training @ iter = ', 1400)\n",
      "('training @ iter = ', 1500)\n",
      "epoch 8, minibatch 195/195, validation error 25.631010 %\n",
      "     epoch 8, minibatch 195/195, test error of best model 26.762821 %\n",
      "('training @ iter = ', 1600)\n",
      "('training @ iter = ', 1700)\n",
      "epoch 9, minibatch 195/195, validation error 22.946715 %\n",
      "     epoch 9, minibatch 195/195, test error of best model 23.617788 %\n",
      "('training @ iter = ', 1800)\n",
      "('training @ iter = ', 1900)\n",
      "epoch 10, minibatch 195/195, validation error 24.298878 %\n",
      "('training @ iter = ', 2000)\n",
      "('training @ iter = ', 2100)\n",
      "epoch 11, minibatch 195/195, validation error 20.092147 %\n",
      "     epoch 11, minibatch 195/195, test error of best model 20.683093 %\n",
      "('training @ iter = ', 2200)\n",
      "('training @ iter = ', 2300)\n",
      "epoch 12, minibatch 195/195, validation error 18.599760 %\n",
      "     epoch 12, minibatch 195/195, test error of best model 19.190705 %\n",
      "('training @ iter = ', 2400)\n",
      "('training @ iter = ', 2500)\n",
      "epoch 13, minibatch 195/195, validation error 22.716346 %\n",
      "('training @ iter = ', 2600)\n",
      "('training @ iter = ', 2700)\n",
      "epoch 14, minibatch 195/195, validation error 19.901843 %\n",
      "('training @ iter = ', 2800)\n",
      "('training @ iter = ', 2900)\n",
      "epoch 15, minibatch 195/195, validation error 16.225962 %\n",
      "     epoch 15, minibatch 195/195, test error of best model 16.726763 %\n",
      "('training @ iter = ', 3000)\n",
      "('training @ iter = ', 3100)\n",
      "epoch 16, minibatch 195/195, validation error 29.667468 %\n",
      "('training @ iter = ', 3200)\n",
      "('training @ iter = ', 3300)\n",
      "epoch 17, minibatch 195/195, validation error 21.985176 %\n",
      "('training @ iter = ', 3400)\n",
      "('training @ iter = ', 3500)\n",
      "epoch 18, minibatch 195/195, validation error 24.198718 %\n",
      "('training @ iter = ', 3600)\n",
      "('training @ iter = ', 3700)\n",
      "epoch 19, minibatch 195/195, validation error 18.589744 %\n",
      "('training @ iter = ', 3800)\n",
      "epoch 20, minibatch 195/195, validation error 13.962340 %\n",
      "     epoch 20, minibatch 195/195, test error of best model 14.282853 %\n",
      "('training @ iter = ', 3900)\n",
      "('training @ iter = ', 4000)\n",
      "epoch 21, minibatch 195/195, validation error 13.381410 %\n",
      "     epoch 21, minibatch 195/195, test error of best model 13.701923 %\n",
      "('training @ iter = ', 4100)\n",
      "('training @ iter = ', 4200)\n",
      "epoch 22, minibatch 195/195, validation error 15.124199 %\n",
      "('training @ iter = ', 4300)\n",
      "('training @ iter = ', 4400)\n",
      "epoch 23, minibatch 195/195, validation error 34.214744 %\n",
      "('training @ iter = ', 4500)\n",
      "('training @ iter = ', 4600)\n",
      "epoch 24, minibatch 195/195, validation error 12.339744 %\n",
      "     epoch 24, minibatch 195/195, test error of best model 12.700321 %\n",
      "('training @ iter = ', 4700)\n",
      "('training @ iter = ', 4800)\n",
      "epoch 25, minibatch 195/195, validation error 11.728766 %\n",
      "     epoch 25, minibatch 195/195, test error of best model 11.919071 %\n",
      "('training @ iter = ', 4900)\n",
      "('training @ iter = ', 5000)\n",
      "epoch 26, minibatch 195/195, validation error 10.827324 %\n",
      "     epoch 26, minibatch 195/195, test error of best model 11.047676 %\n",
      "('training @ iter = ', 5100)\n",
      "('training @ iter = ', 5200)\n",
      "epoch 27, minibatch 195/195, validation error 11.678686 %\n",
      "('training @ iter = ', 5300)\n",
      "('training @ iter = ', 5400)\n",
      "epoch 28, minibatch 195/195, validation error 12.409856 %\n",
      "('training @ iter = ', 5500)\n",
      "('training @ iter = ', 5600)\n",
      "epoch 29, minibatch 195/195, validation error 10.106170 %\n",
      "     epoch 29, minibatch 195/195, test error of best model 10.546875 %\n",
      "('training @ iter = ', 5700)\n",
      "('training @ iter = ', 5800)\n",
      "epoch 30, minibatch 195/195, validation error 10.006010 %\n",
      "     epoch 30, minibatch 195/195, test error of best model 10.066106 %\n",
      "('training @ iter = ', 5900)\n",
      "('training @ iter = ', 6000)\n",
      "epoch 31, minibatch 195/195, validation error 21.554487 %\n",
      "('training @ iter = ', 6100)\n",
      "('training @ iter = ', 6200)\n",
      "epoch 32, minibatch 195/195, validation error 23.447516 %\n",
      "('training @ iter = ', 6300)\n",
      "('training @ iter = ', 6400)\n",
      "epoch 33, minibatch 195/195, validation error 19.531250 %\n",
      "('training @ iter = ', 6500)\n",
      "('training @ iter = ', 6600)\n",
      "epoch 34, minibatch 195/195, validation error 21.434295 %\n",
      "('training @ iter = ', 6700)\n",
      "('training @ iter = ', 6800)\n",
      "epoch 35, minibatch 195/195, validation error 21.344151 %\n",
      "('training @ iter = ', 6900)\n",
      "('training @ iter = ', 7000)\n",
      "epoch 36, minibatch 195/195, validation error 16.826923 %\n",
      "('training @ iter = ', 7100)\n",
      "('training @ iter = ', 7200)\n",
      "epoch 37, minibatch 195/195, validation error 19.731571 %\n",
      "('training @ iter = ', 7300)\n",
      "('training @ iter = ', 7400)\n",
      "epoch 38, minibatch 195/195, validation error 18.179087 %\n",
      "('training @ iter = ', 7500)\n",
      "('training @ iter = ', 7600)\n",
      "epoch 39, minibatch 195/195, validation error 17.357772 %\n",
      "('training @ iter = ', 7700)\n",
      "epoch 40, minibatch 195/195, validation error 15.594952 %\n",
      "('training @ iter = ', 7800)\n",
      "('training @ iter = ', 7900)\n",
      "epoch 41, minibatch 195/195, validation error 10.176282 %\n",
      "('training @ iter = ', 8000)\n",
      "('training @ iter = ', 8100)\n",
      "epoch 42, minibatch 195/195, validation error 40.314503 %\n",
      "('training @ iter = ', 8200)\n",
      "('training @ iter = ', 8300)\n",
      "epoch 43, minibatch 195/195, validation error 10.586939 %\n",
      "('training @ iter = ', 8400)\n",
      "('training @ iter = ', 8500)\n",
      "epoch 44, minibatch 195/195, validation error 9.665465 %\n",
      "     epoch 44, minibatch 195/195, test error of best model 10.046074 %\n",
      "('training @ iter = ', 8600)\n",
      "('training @ iter = ', 8700)\n",
      "epoch 45, minibatch 195/195, validation error 18.599760 %\n",
      "('training @ iter = ', 8800)\n",
      "('training @ iter = ', 8900)\n",
      "epoch 46, minibatch 195/195, validation error 24.759615 %\n",
      "('training @ iter = ', 9000)\n",
      "('training @ iter = ', 9100)\n",
      "epoch 47, minibatch 195/195, validation error 21.734776 %\n",
      "('training @ iter = ', 9200)\n",
      "('training @ iter = ', 9300)\n",
      "epoch 48, minibatch 195/195, validation error 15.474760 %\n",
      "('training @ iter = ', 9400)\n",
      "('training @ iter = ', 9500)\n",
      "epoch 49, minibatch 195/195, validation error 14.833734 %\n",
      "('training @ iter = ', 9600)\n",
      "('training @ iter = ', 9700)\n",
      "epoch 50, minibatch 195/195, validation error 11.318109 %\n",
      "('training @ iter = ', 9800)\n",
      "('training @ iter = ', 9900)\n",
      "epoch 51, minibatch 195/195, validation error 12.550080 %\n",
      "('training @ iter = ', 10000)\n",
      "('training @ iter = ', 10100)\n",
      "epoch 52, minibatch 195/195, validation error 11.748798 %\n",
      "('training @ iter = ', 10200)\n",
      "('training @ iter = ', 10300)\n",
      "epoch 53, minibatch 195/195, validation error 18.840144 %\n",
      "('training @ iter = ', 10400)\n",
      "('training @ iter = ', 10500)\n",
      "epoch 54, minibatch 195/195, validation error 11.368189 %\n",
      "('training @ iter = ', 10600)\n",
      "('training @ iter = ', 10700)\n",
      "epoch 55, minibatch 195/195, validation error 33.423478 %\n",
      "('training @ iter = ', 10800)\n",
      "('training @ iter = ', 10900)\n",
      "epoch 56, minibatch 195/195, validation error 13.561699 %\n",
      "('training @ iter = ', 11000)\n",
      "('training @ iter = ', 11100)\n",
      "epoch 57, minibatch 195/195, validation error 14.282853 %\n",
      "('training @ iter = ', 11200)\n",
      "('training @ iter = ', 11300)\n",
      "epoch 58, minibatch 195/195, validation error 22.445913 %\n",
      "('training @ iter = ', 11400)\n",
      "('training @ iter = ', 11500)\n",
      "epoch 59, minibatch 195/195, validation error 12.950721 %\n",
      "('training @ iter = ', 11600)\n",
      "epoch 60, minibatch 195/195, validation error 13.271234 %\n",
      "('training @ iter = ', 11700)\n",
      "('training @ iter = ', 11800)\n",
      "epoch 61, minibatch 195/195, validation error 12.319712 %\n",
      "('training @ iter = ', 11900)\n",
      "('training @ iter = ', 12000)\n",
      "epoch 62, minibatch 195/195, validation error 15.404647 %\n",
      "('training @ iter = ', 12100)\n",
      "('training @ iter = ', 12200)\n",
      "epoch 63, minibatch 195/195, validation error 14.733574 %\n",
      "('training @ iter = ', 12300)\n",
      "('training @ iter = ', 12400)\n",
      "epoch 64, minibatch 195/195, validation error 27.483974 %\n",
      "('training @ iter = ', 12500)\n",
      "('training @ iter = ', 12600)\n",
      "epoch 65, minibatch 195/195, validation error 10.076122 %\n",
      "('training @ iter = ', 12700)\n",
      "('training @ iter = ', 12800)\n",
      "epoch 66, minibatch 195/195, validation error 12.019231 %\n",
      "('training @ iter = ', 12900)\n",
      "('training @ iter = ', 13000)\n",
      "epoch 67, minibatch 195/195, validation error 13.010817 %\n",
      "('training @ iter = ', 13100)\n",
      "('training @ iter = ', 13200)\n",
      "epoch 68, minibatch 195/195, validation error 13.371394 %\n",
      "('training @ iter = ', 13300)\n",
      "('training @ iter = ', 13400)\n",
      "epoch 69, minibatch 195/195, validation error 10.416667 %\n",
      "('training @ iter = ', 13500)\n",
      "('training @ iter = ', 13600)\n",
      "epoch 70, minibatch 195/195, validation error 17.788462 %\n",
      "('training @ iter = ', 13700)\n",
      "('training @ iter = ', 13800)\n",
      "epoch 71, minibatch 195/195, validation error 8.723958 %\n",
      "     epoch 71, minibatch 195/195, test error of best model 9.054487 %\n",
      "('training @ iter = ', 13900)\n",
      "('training @ iter = ', 14000)\n",
      "epoch 72, minibatch 195/195, validation error 15.174279 %\n",
      "('training @ iter = ', 14100)\n",
      "('training @ iter = ', 14200)\n",
      "epoch 73, minibatch 195/195, validation error 14.312901 %\n",
      "('training @ iter = ', 14300)\n",
      "('training @ iter = ', 14400)\n",
      "epoch 74, minibatch 195/195, validation error 26.292067 %\n",
      "('training @ iter = ', 14500)\n",
      "('training @ iter = ', 14600)\n",
      "epoch 75, minibatch 195/195, validation error 8.002804 %\n",
      "     epoch 75, minibatch 195/195, test error of best model 8.603766 %\n",
      "('training @ iter = ', 14700)\n",
      "('training @ iter = ', 14800)\n",
      "epoch 76, minibatch 195/195, validation error 6.700721 %\n",
      "     epoch 76, minibatch 195/195, test error of best model 7.341747 %\n",
      "('training @ iter = ', 14900)\n",
      "('training @ iter = ', 15000)\n",
      "epoch 77, minibatch 195/195, validation error 15.594952 %\n",
      "('training @ iter = ', 15100)\n",
      "('training @ iter = ', 15200)\n",
      "epoch 78, minibatch 195/195, validation error 7.722356 %\n",
      "('training @ iter = ', 15300)\n",
      "('training @ iter = ', 15400)\n",
      "epoch 79, minibatch 195/195, validation error 16.686699 %\n",
      "('training @ iter = ', 15500)\n",
      "epoch 80, minibatch 195/195, validation error 13.932292 %\n",
      "('training @ iter = ', 15600)\n",
      "('training @ iter = ', 15700)\n",
      "epoch 81, minibatch 195/195, validation error 7.381811 %\n",
      "('training @ iter = ', 15800)\n",
      "('training @ iter = ', 15900)\n",
      "epoch 82, minibatch 195/195, validation error 6.550481 %\n",
      "     epoch 82, minibatch 195/195, test error of best model 7.351763 %\n",
      "('training @ iter = ', 16000)\n",
      "('training @ iter = ', 16100)\n",
      "epoch 83, minibatch 195/195, validation error 12.409856 %\n",
      "('training @ iter = ', 16200)\n",
      "('training @ iter = ', 16300)\n",
      "epoch 84, minibatch 195/195, validation error 7.061298 %\n",
      "('training @ iter = ', 16400)\n",
      "('training @ iter = ', 16500)\n",
      "epoch 85, minibatch 195/195, validation error 16.887019 %\n",
      "('training @ iter = ', 16600)\n",
      "('training @ iter = ', 16700)\n",
      "epoch 86, minibatch 195/195, validation error 7.161458 %\n",
      "('training @ iter = ', 16800)\n",
      "('training @ iter = ', 16900)\n",
      "epoch 87, minibatch 195/195, validation error 45.432692 %\n",
      "('training @ iter = ', 17000)\n",
      "('training @ iter = ', 17100)\n",
      "epoch 88, minibatch 195/195, validation error 6.189904 %\n",
      "     epoch 88, minibatch 195/195, test error of best model 7.001202 %\n",
      "('training @ iter = ', 17200)\n",
      "('training @ iter = ', 17300)\n",
      "epoch 89, minibatch 195/195, validation error 7.532051 %\n",
      "('training @ iter = ', 17400)\n",
      "('training @ iter = ', 17500)\n",
      "epoch 90, minibatch 195/195, validation error 6.290064 %\n",
      "('training @ iter = ', 17600)\n",
      "('training @ iter = ', 17700)\n",
      "epoch 91, minibatch 195/195, validation error 7.361779 %\n",
      "('training @ iter = ', 17800)\n",
      "('training @ iter = ', 17900)\n",
      "epoch 92, minibatch 195/195, validation error 6.330128 %\n",
      "('training @ iter = ', 18000)\n",
      "('training @ iter = ', 18100)\n",
      "epoch 93, minibatch 195/195, validation error 6.700721 %\n",
      "('training @ iter = ', 18200)\n",
      "('training @ iter = ', 18300)\n",
      "epoch 94, minibatch 195/195, validation error 9.925881 %\n",
      "('training @ iter = ', 18400)\n",
      "('training @ iter = ', 18500)\n",
      "epoch 95, minibatch 195/195, validation error 8.343349 %\n",
      "('training @ iter = ', 18600)\n",
      "('training @ iter = ', 18700)\n",
      "epoch 96, minibatch 195/195, validation error 6.790865 %\n",
      "('training @ iter = ', 18800)\n",
      "('training @ iter = ', 18900)\n",
      "epoch 97, minibatch 195/195, validation error 5.869391 %\n",
      "     epoch 97, minibatch 195/195, test error of best model 6.850962 %\n",
      "('training @ iter = ', 19000)\n",
      "('training @ iter = ', 19100)\n",
      "epoch 98, minibatch 195/195, validation error 6.250000 %\n",
      "('training @ iter = ', 19200)\n",
      "('training @ iter = ', 19300)\n",
      "epoch 99, minibatch 195/195, validation error 6.059696 %\n",
      "('training @ iter = ', 19400)\n",
      "epoch 100, minibatch 195/195, validation error 6.049679 %\n",
      "('training @ iter = ', 19500)\n",
      "('training @ iter = ', 19600)\n",
      "epoch 101, minibatch 195/195, validation error 8.293269 %\n",
      "('training @ iter = ', 19700)\n",
      "('training @ iter = ', 19800)\n",
      "epoch 102, minibatch 195/195, validation error 9.545272 %\n",
      "('training @ iter = ', 19900)\n",
      "('training @ iter = ', 20000)\n",
      "epoch 103, minibatch 195/195, validation error 6.209936 %\n",
      "('training @ iter = ', 20100)\n",
      "('training @ iter = ', 20200)\n",
      "epoch 104, minibatch 195/195, validation error 6.049679 %\n",
      "('training @ iter = ', 20300)\n",
      "('training @ iter = ', 20400)\n",
      "epoch 105, minibatch 195/195, validation error 6.089744 %\n",
      "('training @ iter = ', 20500)\n",
      "('training @ iter = ', 20600)\n",
      "epoch 106, minibatch 195/195, validation error 6.109776 %\n",
      "('training @ iter = ', 20700)\n",
      "('training @ iter = ', 20800)\n",
      "epoch 107, minibatch 195/195, validation error 6.239984 %\n",
      "('training @ iter = ', 20900)\n",
      "('training @ iter = ', 21000)\n",
      "epoch 108, minibatch 195/195, validation error 7.832532 %\n",
      "('training @ iter = ', 21100)\n",
      "('training @ iter = ', 21200)\n",
      "epoch 109, minibatch 195/195, validation error 6.119792 %\n",
      "('training @ iter = ', 21300)\n",
      "('training @ iter = ', 21400)\n",
      "epoch 110, minibatch 195/195, validation error 6.049679 %\n",
      "('training @ iter = ', 21500)\n",
      "('training @ iter = ', 21600)\n",
      "epoch 111, minibatch 195/195, validation error 70.412660 %\n",
      "('training @ iter = ', 21700)\n",
      "('training @ iter = ', 21800)\n",
      "epoch 112, minibatch 195/195, validation error 5.909455 %\n",
      "('training @ iter = ', 21900)\n",
      "('training @ iter = ', 22000)\n",
      "epoch 113, minibatch 195/195, validation error 6.009615 %\n",
      "('training @ iter = ', 22100)\n",
      "('training @ iter = ', 22200)\n",
      "epoch 114, minibatch 195/195, validation error 5.989583 %\n",
      "('training @ iter = ', 22300)\n",
      "('training @ iter = ', 22400)\n",
      "epoch 115, minibatch 195/195, validation error 6.209936 %\n",
      "('training @ iter = ', 22500)\n",
      "('training @ iter = ', 22600)\n",
      "epoch 116, minibatch 195/195, validation error 6.039663 %\n",
      "('training @ iter = ', 22700)\n",
      "('training @ iter = ', 22800)\n",
      "epoch 117, minibatch 195/195, validation error 6.480369 %\n",
      "('training @ iter = ', 22900)\n",
      "('training @ iter = ', 23000)\n",
      "epoch 118, minibatch 195/195, validation error 5.809295 %\n",
      "     epoch 118, minibatch 195/195, test error of best model 6.300080 %\n",
      "('training @ iter = ', 23100)\n",
      "('training @ iter = ', 23200)\n",
      "epoch 119, minibatch 195/195, validation error 6.099760 %\n",
      "('training @ iter = ', 23300)\n",
      "epoch 120, minibatch 195/195, validation error 6.119792 %\n",
      "('training @ iter = ', 23400)\n",
      "('training @ iter = ', 23500)\n",
      "epoch 121, minibatch 195/195, validation error 6.450321 %\n",
      "('training @ iter = ', 23600)\n",
      "('training @ iter = ', 23700)\n",
      "epoch 122, minibatch 195/195, validation error 5.869391 %\n",
      "('training @ iter = ', 23800)\n",
      "('training @ iter = ', 23900)\n",
      "epoch 123, minibatch 195/195, validation error 10.076122 %\n",
      "('training @ iter = ', 24000)\n",
      "('training @ iter = ', 24100)\n",
      "epoch 124, minibatch 195/195, validation error 15.464744 %\n",
      "('training @ iter = ', 24200)\n",
      "('training @ iter = ', 24300)\n",
      "epoch 125, minibatch 195/195, validation error 6.139824 %\n",
      "('training @ iter = ', 24400)\n",
      "('training @ iter = ', 24500)\n",
      "epoch 126, minibatch 195/195, validation error 6.510417 %\n",
      "('training @ iter = ', 24600)\n",
      "('training @ iter = ', 24700)\n",
      "epoch 127, minibatch 195/195, validation error 6.109776 %\n",
      "('training @ iter = ', 24800)\n",
      "('training @ iter = ', 24900)\n",
      "epoch 128, minibatch 195/195, validation error 10.747196 %\n",
      "('training @ iter = ', 25000)\n",
      "('training @ iter = ', 25100)\n",
      "epoch 129, minibatch 195/195, validation error 6.119792 %\n",
      "('training @ iter = ', 25200)\n",
      "('training @ iter = ', 25300)\n",
      "epoch 130, minibatch 195/195, validation error 5.979567 %\n",
      "('training @ iter = ', 25400)\n",
      "('training @ iter = ', 25500)\n",
      "epoch 131, minibatch 195/195, validation error 6.680689 %\n",
      "('training @ iter = ', 25600)\n",
      "('training @ iter = ', 25700)\n",
      "epoch 132, minibatch 195/195, validation error 5.799279 %\n",
      "     epoch 132, minibatch 195/195, test error of best model 6.440304 %\n",
      "('training @ iter = ', 25800)\n",
      "('training @ iter = ', 25900)\n",
      "epoch 133, minibatch 195/195, validation error 6.119792 %\n",
      "('training @ iter = ', 26000)\n",
      "('training @ iter = ', 26100)\n",
      "epoch 134, minibatch 195/195, validation error 6.089744 %\n",
      "('training @ iter = ', 26200)\n",
      "('training @ iter = ', 26300)\n",
      "epoch 135, minibatch 195/195, validation error 5.889423 %\n",
      "('training @ iter = ', 26400)\n",
      "('training @ iter = ', 26500)\n",
      "epoch 136, minibatch 195/195, validation error 35.637019 %\n",
      "('training @ iter = ', 26600)\n",
      "('training @ iter = ', 26700)\n",
      "epoch 137, minibatch 195/195, validation error 6.089744 %\n",
      "('training @ iter = ', 26800)\n",
      "('training @ iter = ', 26900)\n",
      "epoch 138, minibatch 195/195, validation error 6.300080 %\n",
      "('training @ iter = ', 27000)\n",
      "('training @ iter = ', 27100)\n",
      "epoch 139, minibatch 195/195, validation error 6.099760 %\n",
      "('training @ iter = ', 27200)\n",
      "epoch 140, minibatch 195/195, validation error 6.109776 %\n",
      "('training @ iter = ', 27300)\n",
      "('training @ iter = ', 27400)\n",
      "epoch 141, minibatch 195/195, validation error 5.639022 %\n",
      "     epoch 141, minibatch 195/195, test error of best model 6.159856 %\n",
      "('training @ iter = ', 27500)\n",
      "('training @ iter = ', 27600)\n",
      "epoch 142, minibatch 195/195, validation error 19.481170 %\n",
      "('training @ iter = ', 27700)\n",
      "('training @ iter = ', 27800)\n",
      "epoch 143, minibatch 195/195, validation error 6.059696 %\n",
      "('training @ iter = ', 27900)\n",
      "('training @ iter = ', 28000)\n",
      "epoch 144, minibatch 195/195, validation error 5.749199 %\n",
      "('training @ iter = ', 28100)\n",
      "('training @ iter = ', 28200)\n",
      "epoch 145, minibatch 195/195, validation error 5.899439 %\n",
      "('training @ iter = ', 28300)\n",
      "('training @ iter = ', 28400)\n",
      "epoch 146, minibatch 195/195, validation error 5.819311 %\n",
      "('training @ iter = ', 28500)\n",
      "('training @ iter = ', 28600)\n",
      "epoch 147, minibatch 195/195, validation error 5.799279 %\n",
      "('training @ iter = ', 28700)\n",
      "('training @ iter = ', 28800)\n",
      "epoch 148, minibatch 195/195, validation error 5.869391 %\n",
      "('training @ iter = ', 28900)\n",
      "('training @ iter = ', 29000)\n",
      "epoch 149, minibatch 195/195, validation error 5.829327 %\n",
      "('training @ iter = ', 29100)\n",
      "('training @ iter = ', 29200)\n",
      "epoch 150, minibatch 195/195, validation error 5.749199 %\n",
      "('training @ iter = ', 29300)\n",
      "('training @ iter = ', 29400)\n",
      "epoch 151, minibatch 195/195, validation error 5.829327 %\n",
      "('training @ iter = ', 29500)\n",
      "('training @ iter = ', 29600)\n",
      "epoch 152, minibatch 195/195, validation error 5.829327 %\n",
      "('training @ iter = ', 29700)\n",
      "('training @ iter = ', 29800)\n",
      "epoch 153, minibatch 195/195, validation error 55.629006 %\n",
      "('training @ iter = ', 29900)\n",
      "('training @ iter = ', 30000)\n",
      "epoch 154, minibatch 195/195, validation error 5.588942 %\n",
      "     epoch 154, minibatch 195/195, test error of best model 5.789263 %\n",
      "('training @ iter = ', 30100)\n",
      "('training @ iter = ', 30200)\n",
      "epoch 155, minibatch 195/195, validation error 5.598958 %\n",
      "('training @ iter = ', 30300)\n",
      "('training @ iter = ', 30400)\n",
      "epoch 156, minibatch 195/195, validation error 5.659054 %\n",
      "('training @ iter = ', 30500)\n",
      "('training @ iter = ', 30600)\n",
      "epoch 157, minibatch 195/195, validation error 5.588942 %\n",
      "('training @ iter = ', 30700)\n",
      "('training @ iter = ', 30800)\n",
      "epoch 158, minibatch 195/195, validation error 5.608974 %\n",
      "('training @ iter = ', 30900)\n",
      "('training @ iter = ', 31000)\n",
      "epoch 159, minibatch 195/195, validation error 5.689103 %\n",
      "('training @ iter = ', 31100)\n",
      "epoch 160, minibatch 195/195, validation error 84.755609 %\n",
      "('training @ iter = ', 31200)\n",
      "('training @ iter = ', 31300)\n",
      "epoch 161, minibatch 195/195, validation error 5.338542 %\n",
      "     epoch 161, minibatch 195/195, test error of best model 5.398638 %\n",
      "('training @ iter = ', 31400)\n",
      "('training @ iter = ', 31500)\n",
      "epoch 162, minibatch 195/195, validation error 5.358574 %\n",
      "('training @ iter = ', 31600)\n",
      "('training @ iter = ', 31700)\n",
      "epoch 163, minibatch 195/195, validation error 5.438702 %\n",
      "('training @ iter = ', 31800)\n",
      "('training @ iter = ', 31900)\n",
      "epoch 164, minibatch 195/195, validation error 5.488782 %\n",
      "('training @ iter = ', 32000)\n",
      "('training @ iter = ', 32100)\n",
      "epoch 165, minibatch 195/195, validation error 5.518830 %\n",
      "('training @ iter = ', 32200)\n",
      "('training @ iter = ', 32300)\n",
      "epoch 166, minibatch 195/195, validation error 5.558894 %\n",
      "('training @ iter = ', 32400)\n",
      "('training @ iter = ', 32500)\n",
      "epoch 167, minibatch 195/195, validation error 5.659054 %\n",
      "('training @ iter = ', 32600)\n",
      "('training @ iter = ', 32700)\n",
      "epoch 168, minibatch 195/195, validation error 5.478766 %\n",
      "('training @ iter = ', 32800)\n",
      "('training @ iter = ', 32900)\n",
      "epoch 169, minibatch 195/195, validation error 5.558894 %\n",
      "('training @ iter = ', 33000)\n",
      "('training @ iter = ', 33100)\n",
      "epoch 170, minibatch 195/195, validation error 5.639022 %\n",
      "('training @ iter = ', 33200)\n",
      "('training @ iter = ', 33300)\n",
      "epoch 171, minibatch 195/195, validation error 5.528846 %\n",
      "('training @ iter = ', 33400)\n",
      "('training @ iter = ', 33500)\n",
      "epoch 172, minibatch 195/195, validation error 5.588942 %\n",
      "('training @ iter = ', 33600)\n",
      "('training @ iter = ', 33700)\n",
      "epoch 173, minibatch 195/195, validation error 5.598958 %\n",
      "('training @ iter = ', 33800)\n",
      "('training @ iter = ', 33900)\n",
      "epoch 174, minibatch 195/195, validation error 5.588942 %\n",
      "('training @ iter = ', 34000)\n",
      "('training @ iter = ', 34100)\n",
      "epoch 175, minibatch 195/195, validation error 5.438702 %\n",
      "('training @ iter = ', 34200)\n",
      "('training @ iter = ', 34300)\n",
      "epoch 176, minibatch 195/195, validation error 5.518830 %\n",
      "('training @ iter = ', 34400)\n",
      "('training @ iter = ', 34500)\n",
      "epoch 177, minibatch 195/195, validation error 5.588942 %\n",
      "('training @ iter = ', 34600)\n",
      "('training @ iter = ', 34700)\n",
      "epoch 178, minibatch 195/195, validation error 5.598958 %\n",
      "('training @ iter = ', 34800)\n",
      "('training @ iter = ', 34900)\n",
      "epoch 179, minibatch 195/195, validation error 5.578926 %\n",
      "('training @ iter = ', 35000)\n",
      "epoch 180, minibatch 195/195, validation error 5.598958 %\n",
      "('training @ iter = ', 35100)\n",
      "('training @ iter = ', 35200)\n",
      "epoch 181, minibatch 195/195, validation error 5.528846 %\n",
      "('training @ iter = ', 35300)\n",
      "('training @ iter = ', 35400)\n",
      "epoch 182, minibatch 195/195, validation error 5.468750 %\n",
      "('training @ iter = ', 35500)\n",
      "('training @ iter = ', 35600)\n",
      "epoch 183, minibatch 195/195, validation error 5.538862 %\n",
      "('training @ iter = ', 35700)\n",
      "('training @ iter = ', 35800)\n",
      "epoch 184, minibatch 195/195, validation error 5.528846 %\n",
      "('training @ iter = ', 35900)\n",
      "('training @ iter = ', 36000)\n",
      "epoch 185, minibatch 195/195, validation error 5.478766 %\n",
      "('training @ iter = ', 36100)\n",
      "('training @ iter = ', 36200)\n",
      "epoch 186, minibatch 195/195, validation error 5.418670 %\n",
      "('training @ iter = ', 36300)\n",
      "('training @ iter = ', 36400)\n",
      "epoch 187, minibatch 195/195, validation error 5.478766 %\n",
      "('training @ iter = ', 36500)\n",
      "('training @ iter = ', 36600)\n",
      "epoch 188, minibatch 195/195, validation error 5.488782 %\n",
      "('training @ iter = ', 36700)\n",
      "('training @ iter = ', 36800)\n",
      "epoch 189, minibatch 195/195, validation error 5.448718 %\n",
      "('training @ iter = ', 36900)\n",
      "('training @ iter = ', 37000)\n",
      "epoch 190, minibatch 195/195, validation error 5.488782 %\n",
      "('training @ iter = ', 37100)\n",
      "('training @ iter = ', 37200)\n",
      "epoch 191, minibatch 195/195, validation error 5.398638 %\n",
      "('training @ iter = ', 37300)\n",
      "('training @ iter = ', 37400)\n",
      "epoch 192, minibatch 195/195, validation error 5.518830 %\n",
      "('training @ iter = ', 37500)\n",
      "('training @ iter = ', 37600)\n",
      "epoch 193, minibatch 195/195, validation error 5.388622 %\n",
      "('training @ iter = ', 37700)\n",
      "('training @ iter = ', 37800)\n",
      "epoch 194, minibatch 195/195, validation error 5.158253 %\n",
      "     epoch 194, minibatch 195/195, test error of best model 5.378606 %\n",
      "('training @ iter = ', 37900)\n",
      "('training @ iter = ', 38000)\n",
      "epoch 195, minibatch 195/195, validation error 5.238381 %\n",
      "('training @ iter = ', 38100)\n",
      "('training @ iter = ', 38200)\n",
      "epoch 196, minibatch 195/195, validation error 5.228365 %\n",
      "('training @ iter = ', 38300)\n",
      "('training @ iter = ', 38400)\n",
      "epoch 197, minibatch 195/195, validation error 5.338542 %\n",
      "('training @ iter = ', 38500)\n",
      "('training @ iter = ', 38600)\n",
      "epoch 198, minibatch 195/195, validation error 5.328526 %\n",
      "('training @ iter = ', 38700)\n",
      "('training @ iter = ', 38800)\n",
      "epoch 199, minibatch 195/195, validation error 5.378606 %\n",
      "('training @ iter = ', 38900)\n",
      "epoch 200, minibatch 195/195, validation error 5.358574 %\n",
      "Optimization complete.\n",
      "Best validation score of 5.158253 % obtained at iteration 37830, with test performance 5.378606 %\n",
      "The training process for function run ran for 26.43m\n"
     ]
    }
   ],
   "source": [
    "## STN-FCN Rotation\n",
    "run(network_type = 'fcn', include_st = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#CNN Rotation\n",
    "run(network_type='cnn', include_st=False, transformation=rotation, iterations=1.5e5, n_epochs=200, \n",
    "    batch_size=256, learning_rate=0.01, decay_param=0.1, activation=T.nnet.relu, verbose=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "...loading the dataset\n",
    "...applying rotation to dataset\n",
    "Current training data size is 50000\n",
    "Current validation data size is 10000\n",
    "Current test data size is 10000\n",
    "...building the model\n",
    "...training\n",
    "('training @ iter = ', 0)\n",
    "('training @ iter = ', 100)\n",
    "epoch 1, minibatch 195/195, validation error 72.105369 %\n",
    "     epoch 1, minibatch 195/195, test error of best model 73.056891 %\n",
    "('training @ iter = ', 200)\n",
    "('training @ iter = ', 300)\n",
    "epoch 2, minibatch 195/195, validation error 52.143429 %\n",
    "     epoch 2, minibatch 195/195, test error of best model 53.305288 %\n",
    "('training @ iter = ', 400)\n",
    "('training @ iter = ', 500)\n",
    "epoch 3, minibatch 195/195, validation error 42.327724 %\n",
    "     epoch 3, minibatch 195/195, test error of best model 43.309295 %\n",
    "('training @ iter = ', 600)\n",
    "('training @ iter = ', 700)\n",
    "epoch 4, minibatch 195/195, validation error 33.884215 %\n",
    "     epoch 4, minibatch 195/195, test error of best model 35.106170 %\n",
    "('training @ iter = ', 800)\n",
    "('training @ iter = ', 900)\n",
    "epoch 5, minibatch 195/195, validation error 28.485577 %\n",
    "     epoch 5, minibatch 195/195, test error of best model 29.106571 %\n",
    "('training @ iter = ', 1000)\n",
    "('training @ iter = ', 1100)\n",
    "epoch 6, minibatch 195/195, validation error 24.649439 %\n",
    "     epoch 6, minibatch 195/195, test error of best model 25.280449 %\n",
    "('training @ iter = ', 1200)\n",
    "('training @ iter = ', 1300)\n",
    "epoch 7, minibatch 195/195, validation error 22.105369 %\n",
    "     epoch 7, minibatch 195/195, test error of best model 22.405849 %\n",
    "('training @ iter = ', 1400)\n",
    "('training @ iter = ', 1500)\n",
    "epoch 8, minibatch 195/195, validation error 19.811699 %\n",
    "     epoch 8, minibatch 195/195, test error of best model 20.242388 %\n",
    "('training @ iter = ', 1600)\n",
    "('training @ iter = ', 1700)\n",
    "epoch 9, minibatch 195/195, validation error 18.269231 %\n",
    "     epoch 9, minibatch 195/195, test error of best model 18.309295 %\n",
    "('training @ iter = ', 1800)\n",
    "('training @ iter = ', 1900)\n",
    "epoch 10, minibatch 195/195, validation error 16.846955 %\n",
    "     epoch 10, minibatch 195/195, test error of best model 16.826923 %\n",
    "('training @ iter = ', 2000)\n",
    "('training @ iter = ', 2100)\n",
    "epoch 11, minibatch 195/195, validation error 15.775240 %\n",
    "     epoch 11, minibatch 195/195, test error of best model 16.055689 %\n",
    "('training @ iter = ', 2200)\n",
    "('training @ iter = ', 2300)\n",
    "epoch 12, minibatch 195/195, validation error 15.034054 %\n",
    "     epoch 12, minibatch 195/195, test error of best model 15.204327 %\n",
    "('training @ iter = ', 2400)\n",
    "('training @ iter = ', 2500)\n",
    "epoch 13, minibatch 195/195, validation error 14.282853 %\n",
    "     epoch 13, minibatch 195/195, test error of best model 14.613381 %\n",
    "('training @ iter = ', 2600)\n",
    "('training @ iter = ', 2700)\n",
    "epoch 14, minibatch 195/195, validation error 13.641827 %\n",
    "     epoch 14, minibatch 195/195, test error of best model 13.912260 %\n",
    "('training @ iter = ', 2800)\n",
    "('training @ iter = ', 2900)\n",
    "epoch 15, minibatch 195/195, validation error 13.040865 %\n",
    "     epoch 15, minibatch 195/195, test error of best model 13.421474 %\n",
    "('training @ iter = ', 3000)\n",
    "('training @ iter = ', 3100)\n",
    "epoch 16, minibatch 195/195, validation error 12.469952 %\n",
    "     epoch 16, minibatch 195/195, test error of best model 13.110978 %\n",
    "('training @ iter = ', 3200)\n",
    "('training @ iter = ', 3300)\n",
    "epoch 17, minibatch 195/195, validation error 12.009215 %\n",
    "     epoch 17, minibatch 195/195, test error of best model 12.660256 %\n",
    "('training @ iter = ', 3400)\n",
    "('training @ iter = ', 3500)\n",
    "epoch 18, minibatch 195/195, validation error 11.598558 %\n",
    "     epoch 18, minibatch 195/195, test error of best model 12.199519 %\n",
    "('training @ iter = ', 3600)\n",
    "('training @ iter = ', 3700)\n",
    "epoch 19, minibatch 195/195, validation error 11.278045 %\n",
    "     epoch 19, minibatch 195/195, test error of best model 11.778846 %\n",
    "('training @ iter = ', 3800)\n",
    "epoch 20, minibatch 195/195, validation error 10.897436 %\n",
    "     epoch 20, minibatch 195/195, test error of best model 11.428285 %\n",
    "('training @ iter = ', 3900)\n",
    "('training @ iter = ', 4000)\n",
    "epoch 21, minibatch 195/195, validation error 10.657051 %\n",
    "     epoch 21, minibatch 195/195, test error of best model 11.167869 %\n",
    "('training @ iter = ', 4100)\n",
    "('training @ iter = ', 4200)\n",
    "epoch 22, minibatch 195/195, validation error 10.356571 %\n",
    "     epoch 22, minibatch 195/195, test error of best model 10.937500 %\n",
    "('training @ iter = ', 4300)\n",
    "('training @ iter = ', 4400)\n",
    "epoch 23, minibatch 195/195, validation error 10.016026 %\n",
    "     epoch 23, minibatch 195/195, test error of best model 10.777244 %\n",
    "('training @ iter = ', 4500)\n",
    "('training @ iter = ', 4600)\n",
    "epoch 24, minibatch 195/195, validation error 9.785657 %\n",
    "     epoch 24, minibatch 195/195, test error of best model 10.406651 %\n",
    "('training @ iter = ', 4700)\n",
    "('training @ iter = ', 4800)\n",
    "epoch 25, minibatch 195/195, validation error 9.625401 %\n",
    "     epoch 25, minibatch 195/195, test error of best model 10.236378 %\n",
    "('training @ iter = ', 4900)\n",
    "('training @ iter = ', 5000)\n",
    "epoch 26, minibatch 195/195, validation error 9.425080 %\n",
    "     epoch 26, minibatch 195/195, test error of best model 9.975962 %\n",
    "('training @ iter = ', 5100)\n",
    "('training @ iter = ', 5200)\n",
    "epoch 27, minibatch 195/195, validation error 9.224760 %\n",
    "     epoch 27, minibatch 195/195, test error of best model 9.745593 %\n",
    "('training @ iter = ', 5300)\n",
    "('training @ iter = ', 5400)\n",
    "epoch 28, minibatch 195/195, validation error 9.114583 %\n",
    "     epoch 28, minibatch 195/195, test error of best model 9.505208 %\n",
    "('training @ iter = ', 5500)\n",
    "('training @ iter = ', 5600)\n",
    "epoch 29, minibatch 195/195, validation error 8.934295 %\n",
    "     epoch 29, minibatch 195/195, test error of best model 9.344952 %\n",
    "('training @ iter = ', 5700)\n",
    "('training @ iter = ', 5800)\n",
    "epoch 30, minibatch 195/195, validation error 8.804087 %\n",
    "     epoch 30, minibatch 195/195, test error of best model 9.124599 %\n",
    "('training @ iter = ', 5900)\n",
    "('training @ iter = ', 6000)\n",
    "epoch 31, minibatch 195/195, validation error 8.653846 %\n",
    "     epoch 31, minibatch 195/195, test error of best model 8.844151 %\n",
    "('training @ iter = ', 6100)\n",
    "('training @ iter = ', 6200)\n",
    "epoch 32, minibatch 195/195, validation error 8.493590 %\n",
    "     epoch 32, minibatch 195/195, test error of best model 8.703926 %\n",
    "('training @ iter = ', 6300)\n",
    "('training @ iter = ', 6400)\n",
    "epoch 33, minibatch 195/195, validation error 8.373397 %\n",
    "     epoch 33, minibatch 195/195, test error of best model 8.593750 %\n",
    "('training @ iter = ', 6500)\n",
    "('training @ iter = ', 6600)\n",
    "epoch 34, minibatch 195/195, validation error 8.253205 %\n",
    "     epoch 34, minibatch 195/195, test error of best model 8.453526 %\n",
    "('training @ iter = ', 6700)\n",
    "('training @ iter = ', 6800)\n",
    "epoch 35, minibatch 195/195, validation error 8.052885 %\n",
    "     epoch 35, minibatch 195/195, test error of best model 8.313301 %\n",
    "('training @ iter = ', 6900)\n",
    "('training @ iter = ', 7000)\n",
    "epoch 36, minibatch 195/195, validation error 7.842548 %\n",
    "     epoch 36, minibatch 195/195, test error of best model 8.193109 %\n",
    "('training @ iter = ', 7100)\n",
    "('training @ iter = ', 7200)\n",
    "epoch 37, minibatch 195/195, validation error 7.692308 %\n",
    "     epoch 37, minibatch 195/195, test error of best model 8.072917 %\n",
    "('training @ iter = ', 7300)\n",
    "('training @ iter = ', 7400)\n",
    "epoch 38, minibatch 195/195, validation error 7.552083 %\n",
    "     epoch 38, minibatch 195/195, test error of best model 7.852564 %\n",
    "('training @ iter = ', 7500)\n",
    "('training @ iter = ', 7600)\n",
    "epoch 39, minibatch 195/195, validation error 7.401843 %\n",
    "     epoch 39, minibatch 195/195, test error of best model 7.772436 %\n",
    "('training @ iter = ', 7700)\n",
    "epoch 40, minibatch 195/195, validation error 7.331731 %\n",
    "     epoch 40, minibatch 195/195, test error of best model 7.662260 %\n",
    "('training @ iter = ', 7800)\n",
    "('training @ iter = ', 7900)\n",
    "epoch 41, minibatch 195/195, validation error 7.271635 %\n",
    "     epoch 41, minibatch 195/195, test error of best model 7.572115 %\n",
    "('training @ iter = ', 8000)\n",
    "('training @ iter = ', 8100)\n",
    "epoch 42, minibatch 195/195, validation error 7.201522 %\n",
    "     epoch 42, minibatch 195/195, test error of best model 7.491987 %\n",
    "('training @ iter = ', 8200)\n",
    "('training @ iter = ', 8300)\n",
    "epoch 43, minibatch 195/195, validation error 7.141426 %\n",
    "     epoch 43, minibatch 195/195, test error of best model 7.401843 %\n",
    "('training @ iter = ', 8400)\n",
    "('training @ iter = ', 8500)\n",
    "epoch 44, minibatch 195/195, validation error 7.051282 %\n",
    "     epoch 44, minibatch 195/195, test error of best model 7.341747 %\n",
    "('training @ iter = ', 8600)\n",
    "('training @ iter = ', 8700)\n",
    "epoch 45, minibatch 195/195, validation error 6.991186 %\n",
    "     epoch 45, minibatch 195/195, test error of best model 7.221554 %\n",
    "('training @ iter = ', 8800)\n",
    "('training @ iter = ', 8900)\n",
    "epoch 46, minibatch 195/195, validation error 6.901042 %\n",
    "     epoch 46, minibatch 195/195, test error of best model 7.171474 %\n",
    "('training @ iter = ', 9000)\n",
    "('training @ iter = ', 9100)\n",
    "epoch 47, minibatch 195/195, validation error 6.790865 %\n",
    "     epoch 47, minibatch 195/195, test error of best model 7.111378 %\n",
    "('training @ iter = ', 9200)\n",
    "('training @ iter = ', 9300)\n",
    "epoch 48, minibatch 195/195, validation error 6.670673 %\n",
    "     epoch 48, minibatch 195/195, test error of best model 7.051282 %\n",
    "('training @ iter = ', 9400)\n",
    "('training @ iter = ', 9500)\n",
    "epoch 49, minibatch 195/195, validation error 6.620593 %\n",
    "     epoch 49, minibatch 195/195, test error of best model 6.991186 %\n",
    "('training @ iter = ', 9600)\n",
    "('training @ iter = ', 9700)\n",
    "epoch 50, minibatch 195/195, validation error 6.530449 %\n",
    "     epoch 50, minibatch 195/195, test error of best model 6.951122 %\n",
    "('training @ iter = ', 9800)\n",
    "('training @ iter = ', 9900)\n",
    "epoch 51, minibatch 195/195, validation error 6.430288 %\n",
    "     epoch 51, minibatch 195/195, test error of best model 6.850962 %\n",
    "('training @ iter = ', 10000)\n",
    "('training @ iter = ', 10100)\n",
    "epoch 52, minibatch 195/195, validation error 6.390224 %\n",
    "     epoch 52, minibatch 195/195, test error of best model 6.810897 %\n",
    "('training @ iter = ', 10200)\n",
    "('training @ iter = ', 10300)\n",
    "epoch 53, minibatch 195/195, validation error 6.280048 %\n",
    "     epoch 53, minibatch 195/195, test error of best model 6.710737 %\n",
    "('training @ iter = ', 10400)\n",
    "('training @ iter = ', 10500)\n",
    "epoch 54, minibatch 195/195, validation error 6.189904 %\n",
    "     epoch 54, minibatch 195/195, test error of best model 6.610577 %\n",
    "('training @ iter = ', 10600)\n",
    "('training @ iter = ', 10700)\n",
    "epoch 55, minibatch 195/195, validation error 6.159856 %\n",
    "     epoch 55, minibatch 195/195, test error of best model 6.530449 %\n",
    "('training @ iter = ', 10800)\n",
    "('training @ iter = ', 10900)\n",
    "epoch 56, minibatch 195/195, validation error 6.059696 %\n",
    "     epoch 56, minibatch 195/195, test error of best model 6.510417 %\n",
    "('training @ iter = ', 11000)\n",
    "('training @ iter = ', 11100)\n",
    "epoch 57, minibatch 195/195, validation error 6.019631 %\n",
    "     epoch 57, minibatch 195/195, test error of best model 6.460337 %\n",
    "('training @ iter = ', 11200)\n",
    "('training @ iter = ', 11300)\n",
    "epoch 58, minibatch 195/195, validation error 5.939503 %\n",
    "     epoch 58, minibatch 195/195, test error of best model 6.390224 %\n",
    "('training @ iter = ', 11400)\n",
    "('training @ iter = ', 11500)\n",
    "epoch 59, minibatch 195/195, validation error 5.929487 %\n",
    "     epoch 59, minibatch 195/195, test error of best model 6.350160 %\n",
    "('training @ iter = ', 11600)\n",
    "epoch 60, minibatch 195/195, validation error 5.889423 %\n",
    "     epoch 60, minibatch 195/195, test error of best model 6.300080 %\n",
    "('training @ iter = ', 11700)\n",
    "('training @ iter = ', 11800)\n",
    "epoch 61, minibatch 195/195, validation error 5.839343 %\n",
    "     epoch 61, minibatch 195/195, test error of best model 6.270032 %\n",
    "('training @ iter = ', 11900)\n",
    "('training @ iter = ', 12000)\n",
    "epoch 62, minibatch 195/195, validation error 5.769231 %\n",
    "     epoch 62, minibatch 195/195, test error of best model 6.229968 %\n",
    "('training @ iter = ', 12100)\n",
    "('training @ iter = ', 12200)\n",
    "epoch 63, minibatch 195/195, validation error 5.739183 %\n",
    "     epoch 63, minibatch 195/195, test error of best model 6.179888 %\n",
    "('training @ iter = ', 12300)\n",
    "('training @ iter = ', 12400)\n",
    "epoch 64, minibatch 195/195, validation error 5.719151 %\n",
    "     epoch 64, minibatch 195/195, test error of best model 6.129808 %\n",
    "('training @ iter = ', 12500)\n",
    "('training @ iter = ', 12600)\n",
    "epoch 65, minibatch 195/195, validation error 5.639022 %\n",
    "     epoch 65, minibatch 195/195, test error of best model 6.109776 %\n",
    "('training @ iter = ', 12700)\n",
    "('training @ iter = ', 12800)\n",
    "epoch 66, minibatch 195/195, validation error 5.588942 %\n",
    "     epoch 66, minibatch 195/195, test error of best model 6.069712 %\n",
    "('training @ iter = ', 12900)\n",
    "('training @ iter = ', 13000)\n",
    "epoch 67, minibatch 195/195, validation error 5.538862 %\n",
    "     epoch 67, minibatch 195/195, test error of best model 6.049679 %\n",
    "('training @ iter = ', 13100)\n",
    "('training @ iter = ', 13200)\n",
    "epoch 68, minibatch 195/195, validation error 5.518830 %\n",
    "     epoch 68, minibatch 195/195, test error of best model 5.989583 %\n",
    "('training @ iter = ', 13300)\n",
    "('training @ iter = ', 13400)\n",
    "epoch 69, minibatch 195/195, validation error 5.438702 %\n",
    "     epoch 69, minibatch 195/195, test error of best model 5.969551 %\n",
    "('training @ iter = ', 13500)\n",
    "('training @ iter = ', 13600)\n",
    "epoch 70, minibatch 195/195, validation error 5.398638 %\n",
    "     epoch 70, minibatch 195/195, test error of best model 5.909455 %\n",
    "('training @ iter = ', 13700)\n",
    "('training @ iter = ', 13800)\n",
    "epoch 71, minibatch 195/195, validation error 5.358574 %\n",
    "     epoch 71, minibatch 195/195, test error of best model 5.889423 %\n",
    "('training @ iter = ', 13900)\n",
    "('training @ iter = ', 14000)\n",
    "epoch 72, minibatch 195/195, validation error 5.318510 %\n",
    "     epoch 72, minibatch 195/195, test error of best model 5.859375 %\n",
    "('training @ iter = ', 14100)\n",
    "('training @ iter = ', 14200)\n",
    "epoch 73, minibatch 195/195, validation error 5.298478 %\n",
    "     epoch 73, minibatch 195/195, test error of best model 5.809295 %\n",
    "('training @ iter = ', 14300)\n",
    "('training @ iter = ', 14400)\n",
    "epoch 74, minibatch 195/195, validation error 5.318510 %\n",
    "('training @ iter = ', 14500)\n",
    "('training @ iter = ', 14600)\n",
    "epoch 75, minibatch 195/195, validation error 5.318510 %\n",
    "('training @ iter = ', 14700)\n",
    "('training @ iter = ', 14800)\n",
    "epoch 76, minibatch 195/195, validation error 5.328526 %\n",
    "('training @ iter = ', 14900)\n",
    "('training @ iter = ', 15000)\n",
    "epoch 77, minibatch 195/195, validation error 5.238381 %\n",
    "     epoch 77, minibatch 195/195, test error of best model 5.679087 %\n",
    "('training @ iter = ', 15100)\n",
    "('training @ iter = ', 15200)\n",
    "epoch 78, minibatch 195/195, validation error 5.208333 %\n",
    "     epoch 78, minibatch 195/195, test error of best model 5.629006 %\n",
    "('training @ iter = ', 15300)\n",
    "('training @ iter = ', 15400)\n",
    "epoch 79, minibatch 195/195, validation error 5.168269 %\n",
    "     epoch 79, minibatch 195/195, test error of best model 5.598958 %\n",
    "('training @ iter = ', 15500)\n",
    "epoch 80, minibatch 195/195, validation error 5.128205 %\n",
    "     epoch 80, minibatch 195/195, test error of best model 5.608974 %\n",
    "('training @ iter = ', 15600)\n",
    "('training @ iter = ', 15700)\n",
    "epoch 81, minibatch 195/195, validation error 5.108173 %\n",
    "     epoch 81, minibatch 195/195, test error of best model 5.578926 %\n",
    "('training @ iter = ', 15800)\n",
    "('training @ iter = ', 15900)\n",
    "epoch 82, minibatch 195/195, validation error 5.058093 %\n",
    "     epoch 82, minibatch 195/195, test error of best model 5.538862 %\n",
    "('training @ iter = ', 16000)\n",
    "('training @ iter = ', 16100)\n",
    "epoch 83, minibatch 195/195, validation error 4.997997 %\n",
    "     epoch 83, minibatch 195/195, test error of best model 5.498798 %\n",
    "('training @ iter = ', 16200)\n",
    "('training @ iter = ', 16300)\n",
    "epoch 84, minibatch 195/195, validation error 4.987981 %\n",
    "     epoch 84, minibatch 195/195, test error of best model 5.468750 %\n",
    "('training @ iter = ', 16400)\n",
    "('training @ iter = ', 16500)\n",
    "epoch 85, minibatch 195/195, validation error 4.977965 %\n",
    "     epoch 85, minibatch 195/195, test error of best model 5.448718 %\n",
    "('training @ iter = ', 16600)\n",
    "('training @ iter = ', 16700)\n",
    "epoch 86, minibatch 195/195, validation error 4.977965 %\n",
    "('training @ iter = ', 16800)\n",
    "('training @ iter = ', 16900)\n",
    "epoch 87, minibatch 195/195, validation error 4.987981 %\n",
    "('training @ iter = ', 17000)\n",
    "('training @ iter = ', 17100)\n",
    "epoch 88, minibatch 195/195, validation error 4.947917 %\n",
    "     epoch 88, minibatch 195/195, test error of best model 5.398638 %\n",
    "('training @ iter = ', 17200)\n",
    "('training @ iter = ', 17300)\n",
    "epoch 89, minibatch 195/195, validation error 4.907853 %\n",
    "     epoch 89, minibatch 195/195, test error of best model 5.358574 %\n",
    "('training @ iter = ', 17400)\n",
    "('training @ iter = ', 17500)\n",
    "epoch 90, minibatch 195/195, validation error 4.887821 %\n",
    "     epoch 90, minibatch 195/195, test error of best model 5.358574 %\n",
    "('training @ iter = ', 17600)\n",
    "('training @ iter = ', 17700)\n",
    "epoch 91, minibatch 195/195, validation error 4.887821 %\n",
    "('training @ iter = ', 17800)\n",
    "('training @ iter = ', 17900)\n",
    "epoch 92, minibatch 195/195, validation error 4.887821 %\n",
    "('training @ iter = ', 18000)\n",
    "('training @ iter = ', 18100)\n",
    "epoch 93, minibatch 195/195, validation error 4.827724 %\n",
    "     epoch 93, minibatch 195/195, test error of best model 5.288462 %\n",
    "('training @ iter = ', 18200)\n",
    "('training @ iter = ', 18300)\n",
    "epoch 94, minibatch 195/195, validation error 4.817708 %\n",
    "     epoch 94, minibatch 195/195, test error of best model 5.278446 %\n",
    "('training @ iter = ', 18400)\n",
    "('training @ iter = ', 18500)\n",
    "epoch 95, minibatch 195/195, validation error 4.817708 %\n",
    "('training @ iter = ', 18600)\n",
    "('training @ iter = ', 18700)\n",
    "epoch 96, minibatch 195/195, validation error 4.777644 %\n",
    "     epoch 96, minibatch 195/195, test error of best model 5.218349 %\n",
    "('training @ iter = ', 18800)\n",
    "('training @ iter = ', 18900)\n",
    "epoch 97, minibatch 195/195, validation error 4.797676 %\n",
    "('training @ iter = ', 19000)\n",
    "('training @ iter = ', 19100)\n",
    "epoch 98, minibatch 195/195, validation error 4.767628 %\n",
    "     epoch 98, minibatch 195/195, test error of best model 5.178285 %\n",
    "('training @ iter = ', 19200)\n",
    "('training @ iter = ', 19300)\n",
    "epoch 99, minibatch 195/195, validation error 4.717548 %\n",
    "     epoch 99, minibatch 195/195, test error of best model 5.148237 %\n",
    "('training @ iter = ', 19400)\n",
    "epoch 100, minibatch 195/195, validation error 4.697516 %\n",
    "     epoch 100, minibatch 195/195, test error of best model 5.138221 %\n",
    "('training @ iter = ', 19500)\n",
    "('training @ iter = ', 19600)\n",
    "epoch 101, minibatch 195/195, validation error 4.677484 %\n",
    "     epoch 101, minibatch 195/195, test error of best model 5.128205 %\n",
    "('training @ iter = ', 19700)\n",
    "('training @ iter = ', 19800)\n",
    "epoch 102, minibatch 195/195, validation error 4.687500 %\n",
    "('training @ iter = ', 19900)\n",
    "('training @ iter = ', 20000)\n",
    "epoch 103, minibatch 195/195, validation error 4.667468 %\n",
    "     epoch 103, minibatch 195/195, test error of best model 5.138221 %\n",
    "('training @ iter = ', 20100)\n",
    "('training @ iter = ', 20200)\n",
    "epoch 104, minibatch 195/195, validation error 4.637420 %\n",
    "     epoch 104, minibatch 195/195, test error of best model 5.118189 %\n",
    "('training @ iter = ', 20300)\n",
    "('training @ iter = ', 20400)\n",
    "epoch 105, minibatch 195/195, validation error 4.637420 %\n",
    "('training @ iter = ', 20500)\n",
    "('training @ iter = ', 20600)\n",
    "epoch 106, minibatch 195/195, validation error 4.637420 %\n",
    "('training @ iter = ', 20700)\n",
    "('training @ iter = ', 20800)\n",
    "epoch 107, minibatch 195/195, validation error 4.607372 %\n",
    "     epoch 107, minibatch 195/195, test error of best model 5.128205 %\n",
    "('training @ iter = ', 20900)\n",
    "('training @ iter = ', 21000)\n",
    "epoch 108, minibatch 195/195, validation error 4.607372 %\n",
    "('training @ iter = ', 21100)\n",
    "('training @ iter = ', 21200)\n",
    "epoch 109, minibatch 195/195, validation error 4.567308 %\n",
    "     epoch 109, minibatch 195/195, test error of best model 5.048077 %\n",
    "('training @ iter = ', 21300)\n",
    "('training @ iter = ', 21400)\n",
    "epoch 110, minibatch 195/195, validation error 4.567308 %\n",
    "('training @ iter = ', 21500)\n",
    "('training @ iter = ', 21600)\n",
    "epoch 111, minibatch 195/195, validation error 4.547276 %\n",
    "     epoch 111, minibatch 195/195, test error of best model 5.038061 %\n",
    "('training @ iter = ', 21700)\n",
    "('training @ iter = ', 21800)\n",
    "epoch 112, minibatch 195/195, validation error 4.527244 %\n",
    "     epoch 112, minibatch 195/195, test error of best model 4.997997 %\n",
    "('training @ iter = ', 21900)\n",
    "('training @ iter = ', 22000)\n",
    "epoch 113, minibatch 195/195, validation error 4.537260 %\n",
    "('training @ iter = ', 22100)\n",
    "('training @ iter = ', 22200)\n",
    "epoch 114, minibatch 195/195, validation error 4.527244 %\n",
    "('training @ iter = ', 22300)\n",
    "('training @ iter = ', 22400)\n",
    "epoch 115, minibatch 195/195, validation error 4.517228 %\n",
    "     epoch 115, minibatch 195/195, test error of best model 4.917869 %\n",
    "('training @ iter = ', 22500)\n",
    "('training @ iter = ', 22600)\n",
    "epoch 116, minibatch 195/195, validation error 4.497196 %\n",
    "     epoch 116, minibatch 195/195, test error of best model 4.917869 %\n",
    "('training @ iter = ', 22700)\n",
    "('training @ iter = ', 22800)\n",
    "epoch 117, minibatch 195/195, validation error 4.477163 %\n",
    "     epoch 117, minibatch 195/195, test error of best model 4.917869 %\n",
    "('training @ iter = ', 22900)\n",
    "('training @ iter = ', 23000)\n",
    "epoch 118, minibatch 195/195, validation error 4.467147 %\n",
    "     epoch 118, minibatch 195/195, test error of best model 4.917869 %\n",
    "('training @ iter = ', 23100)\n",
    "('training @ iter = ', 23200)\n",
    "epoch 119, minibatch 195/195, validation error 4.447115 %\n",
    "     epoch 119, minibatch 195/195, test error of best model 4.857772 %\n",
    "('training @ iter = ', 23300)\n",
    "epoch 120, minibatch 195/195, validation error 4.477163 %\n",
    "('training @ iter = ', 23400)\n",
    "('training @ iter = ', 23500)\n",
    "epoch 121, minibatch 195/195, validation error 4.467147 %\n",
    "('training @ iter = ', 23600)\n",
    "('training @ iter = ', 23700)\n",
    "epoch 122, minibatch 195/195, validation error 4.447115 %\n",
    "('training @ iter = ', 23800)\n",
    "('training @ iter = ', 23900)\n",
    "epoch 123, minibatch 195/195, validation error 4.437099 %\n",
    "     epoch 123, minibatch 195/195, test error of best model 4.807692 %\n",
    "('training @ iter = ', 24000)\n",
    "('training @ iter = ', 24100)\n",
    "epoch 124, minibatch 195/195, validation error 4.437099 %\n",
    "('training @ iter = ', 24200)\n",
    "('training @ iter = ', 24300)\n",
    "epoch 125, minibatch 195/195, validation error 4.387019 %\n",
    "     epoch 125, minibatch 195/195, test error of best model 4.797676 %\n",
    "('training @ iter = ', 24400)\n",
    "('training @ iter = ', 24500)\n",
    "epoch 126, minibatch 195/195, validation error 4.356971 %\n",
    "     epoch 126, minibatch 195/195, test error of best model 4.797676 %\n",
    "('training @ iter = ', 24600)\n",
    "('training @ iter = ', 24700)\n",
    "epoch 127, minibatch 195/195, validation error 4.366987 %\n",
    "('training @ iter = ', 24800)\n",
    "('training @ iter = ', 24900)\n",
    "epoch 128, minibatch 195/195, validation error 4.346955 %\n",
    "     epoch 128, minibatch 195/195, test error of best model 4.777644 %\n",
    "('training @ iter = ', 25000)\n",
    "('training @ iter = ', 25100)\n",
    "epoch 129, minibatch 195/195, validation error 4.326923 %\n",
    "     epoch 129, minibatch 195/195, test error of best model 4.767628 %\n",
    "('training @ iter = ', 25200)\n",
    "('training @ iter = ', 25300)\n",
    "epoch 130, minibatch 195/195, validation error 4.316907 %\n",
    "     epoch 130, minibatch 195/195, test error of best model 4.787660 %\n",
    "('training @ iter = ', 25400)\n",
    "('training @ iter = ', 25500)\n",
    "epoch 131, minibatch 195/195, validation error 4.306891 %\n",
    "     epoch 131, minibatch 195/195, test error of best model 4.767628 %\n",
    "('training @ iter = ', 25600)\n",
    "('training @ iter = ', 25700)\n",
    "epoch 132, minibatch 195/195, validation error 4.286859 %\n",
    "     epoch 132, minibatch 195/195, test error of best model 4.737580 %\n",
    "('training @ iter = ', 25800)\n",
    "('training @ iter = ', 25900)\n",
    "epoch 133, minibatch 195/195, validation error 4.296875 %\n",
    "('training @ iter = ', 26000)\n",
    "('training @ iter = ', 26100)\n",
    "epoch 134, minibatch 195/195, validation error 4.276843 %\n",
    "     epoch 134, minibatch 195/195, test error of best model 4.707532 %\n",
    "('training @ iter = ', 26200)\n",
    "('training @ iter = ', 26300)\n",
    "epoch 135, minibatch 195/195, validation error 4.276843 %\n",
    "('training @ iter = ', 26400)\n",
    "('training @ iter = ', 26500)\n",
    "epoch 136, minibatch 195/195, validation error 4.246795 %\n",
    "     epoch 136, minibatch 195/195, test error of best model 4.657452 %\n",
    "('training @ iter = ', 26600)\n",
    "('training @ iter = ', 26700)\n",
    "epoch 137, minibatch 195/195, validation error 4.236779 %\n",
    "     epoch 137, minibatch 195/195, test error of best model 4.647436 %\n",
    "('training @ iter = ', 26800)\n",
    "('training @ iter = ', 26900)\n",
    "epoch 138, minibatch 195/195, validation error 4.236779 %\n",
    "('training @ iter = ', 27000)\n",
    "('training @ iter = ', 27100)\n",
    "epoch 139, minibatch 195/195, validation error 4.236779 %\n",
    "('training @ iter = ', 27200)\n",
    "epoch 140, minibatch 195/195, validation error 4.206731 %\n",
    "     epoch 140, minibatch 195/195, test error of best model 4.647436 %\n",
    "('training @ iter = ', 27300)\n",
    "('training @ iter = ', 27400)\n",
    "epoch 141, minibatch 195/195, validation error 4.216747 %\n",
    "('training @ iter = ', 27500)\n",
    "('training @ iter = ', 27600)\n",
    "epoch 142, minibatch 195/195, validation error 4.206731 %\n",
    "('training @ iter = ', 27700)\n",
    "('training @ iter = ', 27800)\n",
    "epoch 143, minibatch 195/195, validation error 4.176683 %\n",
    "     epoch 143, minibatch 195/195, test error of best model 4.647436 %\n",
    "('training @ iter = ', 27900)\n",
    "('training @ iter = ', 28000)\n",
    "epoch 144, minibatch 195/195, validation error 4.236779 %\n",
    "('training @ iter = ', 28100)\n",
    "('training @ iter = ', 28200)\n",
    "epoch 145, minibatch 195/195, validation error 4.216747 %\n",
    "('training @ iter = ', 28300)\n",
    "('training @ iter = ', 28400)\n",
    "epoch 146, minibatch 195/195, validation error 4.196715 %\n",
    "('training @ iter = ', 28500)\n",
    "('training @ iter = ', 28600)\n",
    "epoch 147, minibatch 195/195, validation error 4.186699 %\n",
    "('training @ iter = ', 28700)\n",
    "('training @ iter = ', 28800)\n",
    "epoch 148, minibatch 195/195, validation error 4.156651 %\n",
    "     epoch 148, minibatch 195/195, test error of best model 4.557292 %\n",
    "('training @ iter = ', 28900)\n",
    "('training @ iter = ', 29000)\n",
    "epoch 149, minibatch 195/195, validation error 4.156651 %\n",
    "('training @ iter = ', 29100)\n",
    "('training @ iter = ', 29200)\n",
    "epoch 150, minibatch 195/195, validation error 4.126603 %\n",
    "     epoch 150, minibatch 195/195, test error of best model 4.547276 %\n",
    "('training @ iter = ', 29300)\n",
    "('training @ iter = ', 29400)\n",
    "epoch 151, minibatch 195/195, validation error 4.116587 %\n",
    "     epoch 151, minibatch 195/195, test error of best model 4.557292 %\n",
    "('training @ iter = ', 29500)\n",
    "('training @ iter = ', 29600)\n",
    "epoch 152, minibatch 195/195, validation error 4.106571 %\n",
    "     epoch 152, minibatch 195/195, test error of best model 4.557292 %\n",
    "('training @ iter = ', 29700)\n",
    "('training @ iter = ', 29800)\n",
    "epoch 153, minibatch 195/195, validation error 4.116587 %\n",
    "('training @ iter = ', 29900)\n",
    "('training @ iter = ', 30000)\n",
    "epoch 154, minibatch 195/195, validation error 4.126603 %\n",
    "('training @ iter = ', 30100)\n",
    "('training @ iter = ', 30200)\n",
    "epoch 155, minibatch 195/195, validation error 4.116587 %\n",
    "('training @ iter = ', 30300)\n",
    "('training @ iter = ', 30400)\n",
    "epoch 156, minibatch 195/195, validation error 4.096554 %\n",
    "     epoch 156, minibatch 195/195, test error of best model 4.507212 %\n",
    "('training @ iter = ', 30500)\n",
    "('training @ iter = ', 30600)\n",
    "epoch 157, minibatch 195/195, validation error 4.086538 %\n",
    "     epoch 157, minibatch 195/195, test error of best model 4.497196 %\n",
    "('training @ iter = ', 30700)\n",
    "('training @ iter = ', 30800)\n",
    "epoch 158, minibatch 195/195, validation error 4.096554 %\n",
    "('training @ iter = ', 30900)\n",
    "('training @ iter = ', 31000)\n",
    "epoch 159, minibatch 195/195, validation error 4.096554 %\n",
    "('training @ iter = ', 31100)\n",
    "epoch 160, minibatch 195/195, validation error 4.056490 %\n",
    "     epoch 160, minibatch 195/195, test error of best model 4.487179 %\n",
    "('training @ iter = ', 31200)\n",
    "('training @ iter = ', 31300)\n",
    "epoch 161, minibatch 195/195, validation error 4.076522 %\n",
    "('training @ iter = ', 31400)\n",
    "('training @ iter = ', 31500)\n",
    "epoch 162, minibatch 195/195, validation error 4.056490 %\n",
    "('training @ iter = ', 31600)\n",
    "('training @ iter = ', 31700)\n",
    "epoch 163, minibatch 195/195, validation error 4.056490 %\n",
    "('training @ iter = ', 31800)\n",
    "('training @ iter = ', 31900)\n",
    "epoch 164, minibatch 195/195, validation error 4.056490 %\n",
    "('training @ iter = ', 32000)\n",
    "('training @ iter = ', 32100)\n",
    "epoch 165, minibatch 195/195, validation error 4.056490 %\n",
    "('training @ iter = ', 32200)\n",
    "('training @ iter = ', 32300)\n",
    "epoch 166, minibatch 195/195, validation error 4.036458 %\n",
    "     epoch 166, minibatch 195/195, test error of best model 4.527244 %\n",
    "('training @ iter = ', 32400)\n",
    "('training @ iter = ', 32500)\n",
    "epoch 167, minibatch 195/195, validation error 4.066506 %\n",
    "('training @ iter = ', 32600)\n",
    "('training @ iter = ', 32700)\n",
    "epoch 168, minibatch 195/195, validation error 4.066506 %\n",
    "('training @ iter = ', 32800)\n",
    "('training @ iter = ', 32900)\n",
    "epoch 169, minibatch 195/195, validation error 4.086538 %\n",
    "('training @ iter = ', 33000)\n",
    "('training @ iter = ', 33100)\n",
    "epoch 170, minibatch 195/195, validation error 4.076522 %\n",
    "('training @ iter = ', 33200)\n",
    "('training @ iter = ', 33300)\n",
    "epoch 171, minibatch 195/195, validation error 4.066506 %\n",
    "('training @ iter = ', 33400)\n",
    "('training @ iter = ', 33500)\n",
    "epoch 172, minibatch 195/195, validation error 4.046474 %\n",
    "('training @ iter = ', 33600)\n",
    "('training @ iter = ', 33700)\n",
    "epoch 173, minibatch 195/195, validation error 4.046474 %\n",
    "('training @ iter = ', 33800)\n",
    "('training @ iter = ', 33900)\n",
    "epoch 174, minibatch 195/195, validation error 4.066506 %\n",
    "('training @ iter = ', 34000)\n",
    "('training @ iter = ', 34100)\n",
    "epoch 175, minibatch 195/195, validation error 4.026442 %\n",
    "     epoch 175, minibatch 195/195, test error of best model 4.417067 %\n",
    "('training @ iter = ', 34200)\n",
    "('training @ iter = ', 34300)\n",
    "epoch 176, minibatch 195/195, validation error 4.046474 %\n",
    "('training @ iter = ', 34400)\n",
    "('training @ iter = ', 34500)\n",
    "epoch 177, minibatch 195/195, validation error 4.046474 %\n",
    "('training @ iter = ', 34600)\n",
    "('training @ iter = ', 34700)\n",
    "epoch 178, minibatch 195/195, validation error 4.046474 %\n",
    "('training @ iter = ', 34800)\n",
    "('training @ iter = ', 34900)\n",
    "epoch 179, minibatch 195/195, validation error 4.026442 %\n",
    "('training @ iter = ', 35000)\n",
    "epoch 180, minibatch 195/195, validation error 4.046474 %\n",
    "('training @ iter = ', 35100)\n",
    "('training @ iter = ', 35200)\n",
    "epoch 181, minibatch 195/195, validation error 4.026442 %\n",
    "('training @ iter = ', 35300)\n",
    "('training @ iter = ', 35400)\n",
    "epoch 182, minibatch 195/195, validation error 4.046474 %\n",
    "('training @ iter = ', 35500)\n",
    "('training @ iter = ', 35600)\n",
    "epoch 183, minibatch 195/195, validation error 4.056490 %\n",
    "('training @ iter = ', 35700)\n",
    "('training @ iter = ', 35800)\n",
    "epoch 184, minibatch 195/195, validation error 4.016426 %\n",
    "     epoch 184, minibatch 195/195, test error of best model 4.407051 %\n",
    "('training @ iter = ', 35900)\n",
    "('training @ iter = ', 36000)\n",
    "epoch 185, minibatch 195/195, validation error 4.046474 %\n",
    "('training @ iter = ', 36100)\n",
    "('training @ iter = ', 36200)\n",
    "epoch 186, minibatch 195/195, validation error 4.056490 %\n",
    "('training @ iter = ', 36300)\n",
    "('training @ iter = ', 36400)\n",
    "epoch 187, minibatch 195/195, validation error 4.046474 %\n",
    "('training @ iter = ', 36500)\n",
    "('training @ iter = ', 36600)\n",
    "epoch 188, minibatch 195/195, validation error 4.046474 %\n",
    "('training @ iter = ', 36700)\n",
    "('training @ iter = ', 36800)\n",
    "epoch 189, minibatch 195/195, validation error 4.046474 %\n",
    "('training @ iter = ', 36900)\n",
    "('training @ iter = ', 37000)\n",
    "epoch 190, minibatch 195/195, validation error 4.036458 %\n",
    "('training @ iter = ', 37100)\n",
    "('training @ iter = ', 37200)\n",
    "epoch 191, minibatch 195/195, validation error 4.056490 %\n",
    "('training @ iter = ', 37300)\n",
    "('training @ iter = ', 37400)\n",
    "epoch 192, minibatch 195/195, validation error 4.036458 %\n",
    "('training @ iter = ', 37500)\n",
    "('training @ iter = ', 37600)\n",
    "epoch 193, minibatch 195/195, validation error 4.036458 %\n",
    "('training @ iter = ', 37700)\n",
    "('training @ iter = ', 37800)\n",
    "epoch 194, minibatch 195/195, validation error 4.036458 %\n",
    "('training @ iter = ', 37900)\n",
    "('training @ iter = ', 38000)\n",
    "epoch 195, minibatch 195/195, validation error 4.036458 %\n",
    "('training @ iter = ', 38100)\n",
    "('training @ iter = ', 38200)\n",
    "epoch 196, minibatch 195/195, validation error 4.036458 %\n",
    "('training @ iter = ', 38300)\n",
    "('training @ iter = ', 38400)\n",
    "epoch 197, minibatch 195/195, validation error 4.046474 %\n",
    "('training @ iter = ', 38500)\n",
    "('training @ iter = ', 38600)\n",
    "epoch 198, minibatch 195/195, validation error 4.036458 %\n",
    "('training @ iter = ', 38700)\n",
    "('training @ iter = ', 38800)\n",
    "epoch 199, minibatch 195/195, validation error 4.036458 %\n",
    "('training @ iter = ', 38900)\n",
    "epoch 200, minibatch 195/195, validation error 4.036458 %\n",
    "Optimization complete.\n",
    "Best validation score of 4.016426 % obtained at iteration 35880, with test performance 4.407051 %\n",
    "The training process for function run ran for 40.55m\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# STN-CNN Rotation\n",
    "run(network_type = 'cnn', include_st = True, runallmode = True, n_epochs = 200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "...loading the dataset\n",
    "...Applying rotation to dataset\n",
    "Current training data size is 50000\n",
    "Current validation data size is 10000\n",
    "Current test data size is 10000\n",
    "...building the model\n",
    "...training with include_st = True\n",
    "...Apply STN before CNN\n",
    "...Init localization to identity\n",
    "...training\n",
    "('training @ iter = ', 0)\n",
    "('training @ iter = ', 100)\n",
    "epoch 1, minibatch 195/195, validation error 68.301683 %\n",
    "     epoch 1, minibatch 195/195, test error of best model 69.573718 %\n",
    "('training @ iter = ', 200)\n",
    "('training @ iter = ', 300)\n",
    "epoch 2, minibatch 195/195, validation error 51.454728 %\n",
    "     epoch 2, minibatch 195/195, test error of best model 51.024038 %\n",
    "('training @ iter = ', 400)\n",
    "('training @ iter = ', 500)\n",
    "epoch 3, minibatch 195/195, validation error 42.510417 %\n",
    "     epoch 3, minibatch 195/195, test error of best model 41.639022 %\n",
    "('training @ iter = ', 600)\n",
    "('training @ iter = ', 700)\n",
    "epoch 4, minibatch 195/195, validation error 39.826122 %\n",
    "     epoch 4, minibatch 195/195, test error of best model 39.425481 %\n",
    "('training @ iter = ', 800)\n",
    "('training @ iter = ', 900)\n",
    "epoch 5, minibatch 195/195, validation error 32.674679 %\n",
    "     epoch 5, minibatch 195/195, test error of best model 31.903446 %\n",
    "('training @ iter = ', 1000)\n",
    "('training @ iter = ', 1100)\n",
    "epoch 6, minibatch 195/195, validation error 28.888622 %\n",
    "     epoch 6, minibatch 195/195, test error of best model 28.457933 %\n",
    "('training @ iter = ', 1200)\n",
    "('training @ iter = ', 1300)\n",
    "epoch 7, minibatch 195/195, validation error 24.591747 %\n",
    "     epoch 7, minibatch 195/195, test error of best model 24.261218 %\n",
    "('training @ iter = ', 1400)\n",
    "('training @ iter = ', 1500)\n",
    "epoch 8, minibatch 195/195, validation error 19.774038 %\n",
    "     epoch 8, minibatch 195/195, test error of best model 19.673878 %\n",
    "('training @ iter = ', 1600)\n",
    "('training @ iter = ', 1700)\n",
    "epoch 9, minibatch 195/195, validation error 18.622196 %\n",
    "     epoch 9, minibatch 195/195, test error of best model 19.042869 %\n",
    "('training @ iter = ', 1800)\n",
    "('training @ iter = ', 1900)\n",
    "epoch 10, minibatch 195/195, validation error 16.148237 %\n",
    "     epoch 10, minibatch 195/195, test error of best model 16.719151 %\n",
    "('training @ iter = ', 2000)\n",
    "('training @ iter = ', 2100)\n",
    "epoch 11, minibatch 195/195, validation error 15.577324 %\n",
    "     epoch 11, minibatch 195/195, test error of best model 15.917869 %\n",
    "('training @ iter = ', 2200)\n",
    "('training @ iter = ', 2300)\n",
    "epoch 12, minibatch 195/195, validation error 13.774439 %\n",
    "     epoch 12, minibatch 195/195, test error of best model 14.355369 %\n",
    "('training @ iter = ', 2400)\n",
    "('training @ iter = ', 2500)\n",
    "epoch 13, minibatch 195/195, validation error 12.171875 %\n",
    "     epoch 13, minibatch 195/195, test error of best model 12.672676 %\n",
    "('training @ iter = ', 2600)\n",
    "('training @ iter = ', 2700)\n",
    "epoch 14, minibatch 195/195, validation error 10.258814 %\n",
    "     epoch 14, minibatch 195/195, test error of best model 11.100160 %\n",
    "('training @ iter = ', 2800)\n",
    "('training @ iter = ', 2900)\n",
    "epoch 15, minibatch 195/195, validation error 9.838141 %\n",
    "     epoch 15, minibatch 195/195, test error of best model 10.779647 %\n",
    "('training @ iter = ', 3000)\n",
    "('training @ iter = ', 3100)\n",
    "epoch 16, minibatch 195/195, validation error 9.617788 %\n",
    "     epoch 16, minibatch 195/195, test error of best model 10.679487 %\n",
    "('training @ iter = ', 3200)\n",
    "('training @ iter = ', 3300)\n",
    "epoch 17, minibatch 195/195, validation error 8.746394 %\n",
    "     epoch 17, minibatch 195/195, test error of best model 9.667869 %\n",
    "('training @ iter = ', 3400)\n",
    "('training @ iter = ', 3500)\n",
    "epoch 18, minibatch 195/195, validation error 8.145433 %\n",
    "     epoch 18, minibatch 195/195, test error of best model 9.197115 %\n",
    "('training @ iter = ', 3600)\n",
    "('training @ iter = ', 3700)\n",
    "epoch 19, minibatch 195/195, validation error 7.644631 %\n",
    "     epoch 19, minibatch 195/195, test error of best model 8.696314 %\n",
    "('training @ iter = ', 3800)\n",
    "epoch 20, minibatch 195/195, validation error 7.464343 %\n",
    "     epoch 20, minibatch 195/195, test error of best model 8.485978 %\n",
    "('training @ iter = ', 3900)\n",
    "('training @ iter = ', 4000)\n",
    "epoch 21, minibatch 195/195, validation error 7.374199 %\n",
    "     epoch 21, minibatch 195/195, test error of best model 8.305689 %\n",
    "('training @ iter = ', 4100)\n",
    "('training @ iter = ', 4200)\n",
    "epoch 22, minibatch 195/195, validation error 7.254006 %\n",
    "     epoch 22, minibatch 195/195, test error of best model 7.915064 %\n",
    "('training @ iter = ', 4300)\n",
    "('training @ iter = ', 4400)\n",
    "epoch 23, minibatch 195/195, validation error 6.592949 %\n",
    "     epoch 23, minibatch 195/195, test error of best model 7.314103 %\n",
    "('training @ iter = ', 4500)\n",
    "('training @ iter = ', 4600)\n",
    "epoch 24, minibatch 195/195, validation error 6.312500 %\n",
    "     epoch 24, minibatch 195/195, test error of best model 7.163862 %\n",
    "('training @ iter = ', 4700)\n",
    "('training @ iter = ', 4800)\n",
    "epoch 25, minibatch 195/195, validation error 5.991987 %\n",
    "     epoch 25, minibatch 195/195, test error of best model 6.773237 %\n",
    "('training @ iter = ', 4900)\n",
    "('training @ iter = ', 5000)\n",
    "epoch 26, minibatch 195/195, validation error 5.831731 %\n",
    "     epoch 26, minibatch 195/195, test error of best model 6.472756 %\n",
    "('training @ iter = ', 5100)\n",
    "('training @ iter = ', 5200)\n",
    "epoch 27, minibatch 195/195, validation error 5.681490 %\n",
    "     epoch 27, minibatch 195/195, test error of best model 6.472756 %\n",
    "('training @ iter = ', 5300)\n",
    "('training @ iter = ', 5400)\n",
    "epoch 28, minibatch 195/195, validation error 5.370994 %\n",
    "     epoch 28, minibatch 195/195, test error of best model 6.082131 %\n",
    "('training @ iter = ', 5500)\n",
    "('training @ iter = ', 5600)\n",
    "epoch 29, minibatch 195/195, validation error 5.220753 %\n",
    "     epoch 29, minibatch 195/195, test error of best model 6.092147 %\n",
    "('training @ iter = ', 5700)\n",
    "('training @ iter = ', 5800)\n",
    "epoch 30, minibatch 195/195, validation error 5.090545 %\n",
    "     epoch 30, minibatch 195/195, test error of best model 5.931891 %\n",
    "('training @ iter = ', 5900)\n",
    "('training @ iter = ', 6000)\n",
    "epoch 31, minibatch 195/195, validation error 5.000401 %\n",
    "     epoch 31, minibatch 195/195, test error of best model 5.761619 %\n",
    "('training @ iter = ', 6100)\n",
    "('training @ iter = ', 6200)\n",
    "epoch 32, minibatch 195/195, validation error 4.810096 %\n",
    "     epoch 32, minibatch 195/195, test error of best model 5.611378 %\n",
    "('training @ iter = ', 6300)\n",
    "('training @ iter = ', 6400)\n",
    "epoch 33, minibatch 195/195, validation error 4.659856 %\n",
    "     epoch 33, minibatch 195/195, test error of best model 5.441106 %\n",
    "('training @ iter = ', 6500)\n",
    "('training @ iter = ', 6600)\n",
    "epoch 34, minibatch 195/195, validation error 4.609776 %\n",
    "     epoch 34, minibatch 195/195, test error of best model 5.461138 %\n",
    "('training @ iter = ', 6700)\n",
    "('training @ iter = ', 6800)\n",
    "epoch 35, minibatch 195/195, validation error 4.559696 %\n",
    "     epoch 35, minibatch 195/195, test error of best model 5.320913 %\n",
    "('training @ iter = ', 6900)\n",
    "('training @ iter = ', 7000)\n",
    "epoch 36, minibatch 195/195, validation error 4.429487 %\n",
    "     epoch 36, minibatch 195/195, test error of best model 5.180689 %\n",
    "('training @ iter = ', 7100)\n",
    "('training @ iter = ', 7200)\n",
    "epoch 37, minibatch 195/195, validation error 4.489583 %\n",
    "('training @ iter = ', 7300)\n",
    "('training @ iter = ', 7400)\n",
    "epoch 38, minibatch 195/195, validation error 4.429487 %\n",
    "('training @ iter = ', 7500)\n",
    "('training @ iter = ', 7600)\n",
    "epoch 39, minibatch 195/195, validation error 4.459535 %\n",
    "('training @ iter = ', 7700)\n",
    "epoch 40, minibatch 195/195, validation error 4.339343 %\n",
    "     epoch 40, minibatch 195/195, test error of best model 5.080529 %\n",
    "('training @ iter = ', 7800)\n",
    "('training @ iter = ', 7900)\n",
    "epoch 41, minibatch 195/195, validation error 4.199119 %\n",
    "     epoch 41, minibatch 195/195, test error of best model 5.060497 %\n",
    "('training @ iter = ', 8000)\n",
    "('training @ iter = ', 8100)\n",
    "epoch 42, minibatch 195/195, validation error 4.209135 %\n",
    "('training @ iter = ', 8200)\n",
    "('training @ iter = ', 8300)\n",
    "epoch 43, minibatch 195/195, validation error 4.299279 %\n",
    "('training @ iter = ', 8400)\n",
    "('training @ iter = ', 8500)\n",
    "epoch 44, minibatch 195/195, validation error 4.219151 %\n",
    "('training @ iter = ', 8600)\n",
    "('training @ iter = ', 8700)\n",
    "epoch 45, minibatch 195/195, validation error 4.118990 %\n",
    "     epoch 45, minibatch 195/195, test error of best model 4.980369 %\n",
    "('training @ iter = ', 8800)\n",
    "('training @ iter = ', 8900)\n",
    "epoch 46, minibatch 195/195, validation error 4.018830 %\n",
    "     epoch 46, minibatch 195/195, test error of best model 4.800080 %\n",
    "('training @ iter = ', 9000)\n",
    "('training @ iter = ', 9100)\n",
    "epoch 47, minibatch 195/195, validation error 4.098958 %\n",
    "('training @ iter = ', 9200)\n",
    "('training @ iter = ', 9300)\n",
    "epoch 48, minibatch 195/195, validation error 3.928686 %\n",
    "     epoch 48, minibatch 195/195, test error of best model 4.579728 %\n",
    "('training @ iter = ', 9400)\n",
    "('training @ iter = ', 9500)\n",
    "epoch 49, minibatch 195/195, validation error 3.938702 %\n",
    "('training @ iter = ', 9600)\n",
    "('training @ iter = ', 9700)\n",
    "epoch 50, minibatch 195/195, validation error 3.978766 %\n",
    "('training @ iter = ', 9800)\n",
    "('training @ iter = ', 9900)\n",
    "epoch 51, minibatch 195/195, validation error 3.898638 %\n",
    "     epoch 51, minibatch 195/195, test error of best model 4.679888 %\n",
    "('training @ iter = ', 10000)\n",
    "('training @ iter = ', 10100)\n",
    "epoch 52, minibatch 195/195, validation error 3.778446 %\n",
    "     epoch 52, minibatch 195/195, test error of best model 4.479567 %\n",
    "('training @ iter = ', 10200)\n",
    "('training @ iter = ', 10300)\n",
    "epoch 53, minibatch 195/195, validation error 3.808494 %\n",
    "('training @ iter = ', 10400)\n",
    "('training @ iter = ', 10500)\n",
    "epoch 54, minibatch 195/195, validation error 3.788462 %\n",
    "('training @ iter = ', 10600)\n",
    "('training @ iter = ', 10700)\n",
    "epoch 55, minibatch 195/195, validation error 3.658253 %\n",
    "     epoch 55, minibatch 195/195, test error of best model 4.349359 %\n",
    "('training @ iter = ', 10800)\n",
    "('training @ iter = ', 10900)\n",
    "epoch 56, minibatch 195/195, validation error 3.778446 %\n",
    "('training @ iter = ', 11000)\n",
    "('training @ iter = ', 11100)\n",
    "epoch 57, minibatch 195/195, validation error 3.808494 %\n",
    "('training @ iter = ', 11200)\n",
    "('training @ iter = ', 11300)\n",
    "epoch 58, minibatch 195/195, validation error 3.828526 %\n",
    "('training @ iter = ', 11400)\n",
    "('training @ iter = ', 11500)\n",
    "epoch 59, minibatch 195/195, validation error 3.688301 %\n",
    "('training @ iter = ', 11600)\n",
    "epoch 60, minibatch 195/195, validation error 3.738381 %\n",
    "('training @ iter = ', 11700)\n",
    "('training @ iter = ', 11800)\n",
    "epoch 61, minibatch 195/195, validation error 3.758413 %\n",
    "('training @ iter = ', 11900)\n",
    "('training @ iter = ', 12000)\n",
    "epoch 62, minibatch 195/195, validation error 3.638221 %\n",
    "     epoch 62, minibatch 195/195, test error of best model 4.078926 %\n",
    "('training @ iter = ', 12100)\n",
    "('training @ iter = ', 12200)\n",
    "epoch 63, minibatch 195/195, validation error 3.508013 %\n",
    "     epoch 63, minibatch 195/195, test error of best model 4.179087 %\n",
    "('training @ iter = ', 12300)\n",
    "('training @ iter = ', 12400)\n",
    "epoch 64, minibatch 195/195, validation error 3.608173 %\n",
    "('training @ iter = ', 12500)\n",
    "('training @ iter = ', 12600)\n",
    "epoch 65, minibatch 195/195, validation error 3.578125 %\n",
    "('training @ iter = ', 12700)\n",
    "('training @ iter = ', 12800)\n",
    "epoch 66, minibatch 195/195, validation error 8.566106 %\n",
    "('training @ iter = ', 12900)\n",
    "('training @ iter = ', 13000)\n",
    "epoch 67, minibatch 195/195, validation error 3.558093 %\n",
    "('training @ iter = ', 13100)\n",
    "('training @ iter = ', 13200)\n",
    "epoch 68, minibatch 195/195, validation error 3.538061 %\n",
    "('training @ iter = ', 13300)\n",
    "('training @ iter = ', 13400)\n",
    "epoch 69, minibatch 195/195, validation error 3.407853 %\n",
    "     epoch 69, minibatch 195/195, test error of best model 3.868590 %\n",
    "('training @ iter = ', 13500)\n",
    "('training @ iter = ', 13600)\n",
    "epoch 70, minibatch 195/195, validation error 3.548077 %\n",
    "('training @ iter = ', 13700)\n",
    "('training @ iter = ', 13800)\n",
    "epoch 71, minibatch 195/195, validation error 3.417869 %\n",
    "('training @ iter = ', 13900)\n",
    "('training @ iter = ', 14000)\n",
    "epoch 72, minibatch 195/195, validation error 3.447917 %\n",
    "('training @ iter = ', 14100)\n",
    "('training @ iter = ', 14200)\n",
    "epoch 73, minibatch 195/195, validation error 3.267628 %\n",
    "     epoch 73, minibatch 195/195, test error of best model 3.708333 %\n",
    "('training @ iter = ', 14300)\n",
    "('training @ iter = ', 14400)\n",
    "epoch 74, minibatch 195/195, validation error 3.518029 %\n",
    "('training @ iter = ', 14500)\n",
    "('training @ iter = ', 14600)\n",
    "epoch 75, minibatch 195/195, validation error 3.558093 %\n",
    "('training @ iter = ', 14700)\n",
    "('training @ iter = ', 14800)\n",
    "epoch 76, minibatch 195/195, validation error 3.518029 %\n",
    "('training @ iter = ', 14900)\n",
    "('training @ iter = ', 15000)\n",
    "epoch 77, minibatch 195/195, validation error 3.327724 %\n",
    "('training @ iter = ', 15100)\n",
    "('training @ iter = ', 15200)\n",
    "epoch 78, minibatch 195/195, validation error 3.447917 %\n",
    "('training @ iter = ', 15300)\n",
    "('training @ iter = ', 15400)\n",
    "epoch 79, minibatch 195/195, validation error 3.477965 %\n",
    "('training @ iter = ', 15500)\n",
    "epoch 80, minibatch 195/195, validation error 3.367788 %\n",
    "('training @ iter = ', 15600)\n",
    "('training @ iter = ', 15700)\n",
    "epoch 81, minibatch 195/195, validation error 3.538061 %\n",
    "('training @ iter = ', 15800)\n",
    "('training @ iter = ', 15900)\n",
    "epoch 82, minibatch 195/195, validation error 3.487981 %\n",
    "('training @ iter = ', 16000)\n",
    "('training @ iter = ', 16100)\n",
    "epoch 83, minibatch 195/195, validation error 3.397837 %\n",
    "('training @ iter = ', 16200)\n",
    "('training @ iter = ', 16300)\n",
    "epoch 84, minibatch 195/195, validation error 3.588141 %\n",
    "('training @ iter = ', 16400)\n",
    "('training @ iter = ', 16500)\n",
    "epoch 85, minibatch 195/195, validation error 3.688301 %\n",
    "('training @ iter = ', 16600)\n",
    "('training @ iter = ', 16700)\n",
    "epoch 86, minibatch 195/195, validation error 3.688301 %\n",
    "('training @ iter = ', 16800)\n",
    "('training @ iter = ', 16900)\n",
    "epoch 87, minibatch 195/195, validation error 3.688301 %\n",
    "('training @ iter = ', 17000)\n",
    "('training @ iter = ', 17100)\n",
    "epoch 88, minibatch 195/195, validation error 3.407853 %\n",
    "('training @ iter = ', 17200)\n",
    "('training @ iter = ', 17300)\n",
    "epoch 89, minibatch 195/195, validation error 3.778446 %\n",
    "('training @ iter = ', 17400)\n",
    "('training @ iter = ', 17500)\n",
    "epoch 90, minibatch 195/195, validation error 3.487981 %\n",
    "('training @ iter = ', 17600)\n",
    "('training @ iter = ', 17700)\n",
    "epoch 91, minibatch 195/195, validation error 3.748397 %\n",
    "('training @ iter = ', 17800)\n",
    "('training @ iter = ', 17900)\n",
    "epoch 92, minibatch 195/195, validation error 3.688301 %\n",
    "('training @ iter = ', 18000)\n",
    "('training @ iter = ', 18100)\n",
    "epoch 93, minibatch 195/195, validation error 3.407853 %\n",
    "('training @ iter = ', 18200)\n",
    "('training @ iter = ', 18300)\n",
    "epoch 94, minibatch 195/195, validation error 3.267628 %\n",
    "('training @ iter = ', 18400)\n",
    "('training @ iter = ', 18500)\n",
    "epoch 95, minibatch 195/195, validation error 3.187500 %\n",
    "     epoch 95, minibatch 195/195, test error of best model 3.758413 %\n",
    "('training @ iter = ', 18600)\n",
    "('training @ iter = ', 18700)\n",
    "epoch 96, minibatch 195/195, validation error 3.548077 %\n",
    "('training @ iter = ', 18800)\n",
    "('training @ iter = ', 18900)\n",
    "epoch 97, minibatch 195/195, validation error 3.457933 %\n",
    "('training @ iter = ', 19000)\n",
    "('training @ iter = ', 19100)\n",
    "epoch 98, minibatch 195/195, validation error 3.878606 %\n",
    "('training @ iter = ', 19200)\n",
    "('training @ iter = ', 19300)\n",
    "epoch 99, minibatch 195/195, validation error 3.608173 %\n",
    "('training @ iter = ', 19400)\n",
    "epoch 100, minibatch 195/195, validation error 3.427885 %\n",
    "('training @ iter = ', 19500)\n",
    "('training @ iter = ', 19600)\n",
    "epoch 101, minibatch 195/195, validation error 4.098958 %\n",
    "('training @ iter = ', 19700)\n",
    "('training @ iter = ', 19800)\n",
    "epoch 102, minibatch 195/195, validation error 3.367788 %\n",
    "('training @ iter = ', 19900)\n",
    "('training @ iter = ', 20000)\n",
    "epoch 103, minibatch 195/195, validation error 3.417869 %\n",
    "('training @ iter = ', 20100)\n",
    "('training @ iter = ', 20200)\n",
    "epoch 104, minibatch 195/195, validation error 6.683093 %\n",
    "('training @ iter = ', 20300)\n",
    "('training @ iter = ', 20400)\n",
    "epoch 105, minibatch 195/195, validation error 3.908654 %\n",
    "('training @ iter = ', 20500)\n",
    "('training @ iter = ', 20600)\n",
    "epoch 106, minibatch 195/195, validation error 3.457933 %\n",
    "('training @ iter = ', 20700)\n",
    "('training @ iter = ', 20800)\n",
    "epoch 107, minibatch 195/195, validation error 3.548077 %\n",
    "('training @ iter = ', 20900)\n",
    "('training @ iter = ', 21000)\n",
    "epoch 108, minibatch 195/195, validation error 3.938702 %\n",
    "('training @ iter = ', 21100)\n",
    "('training @ iter = ', 21200)\n",
    "epoch 109, minibatch 195/195, validation error 4.078926 %\n",
    "('training @ iter = ', 21300)\n",
    "('training @ iter = ', 21400)\n",
    "epoch 110, minibatch 195/195, validation error 3.477965 %\n",
    "('training @ iter = ', 21500)\n",
    "('training @ iter = ', 21600)\n",
    "epoch 111, minibatch 195/195, validation error 3.387821 %\n",
    "('training @ iter = ', 21700)\n",
    "('training @ iter = ', 21800)\n",
    "epoch 112, minibatch 195/195, validation error 4.008814 %\n",
    "('training @ iter = ', 21900)\n",
    "('training @ iter = ', 22000)\n",
    "epoch 113, minibatch 195/195, validation error 3.477965 %\n",
    "('training @ iter = ', 22100)\n",
    "('training @ iter = ', 22200)\n",
    "epoch 114, minibatch 195/195, validation error 3.578125 %\n",
    "('training @ iter = ', 22300)\n",
    "('training @ iter = ', 22400)\n",
    "epoch 115, minibatch 195/195, validation error 4.980369 %\n",
    "('training @ iter = ', 22500)\n",
    "('training @ iter = ', 22600)\n",
    "epoch 116, minibatch 195/195, validation error 3.437901 %\n",
    "('training @ iter = ', 22700)\n",
    "('training @ iter = ', 22800)\n",
    "epoch 117, minibatch 195/195, validation error 4.619792 %\n",
    "('training @ iter = ', 22900)\n",
    "('training @ iter = ', 23000)\n",
    "epoch 118, minibatch 195/195, validation error 4.770032 %\n",
    "('training @ iter = ', 23100)\n",
    "('training @ iter = ', 23200)\n",
    "epoch 119, minibatch 195/195, validation error 3.668269 %\n",
    "('training @ iter = ', 23300)\n",
    "epoch 120, minibatch 195/195, validation error 3.898638 %\n",
    "('training @ iter = ', 23400)\n",
    "('training @ iter = ', 23500)\n",
    "epoch 121, minibatch 195/195, validation error 3.638221 %\n",
    "('training @ iter = ', 23600)\n",
    "('training @ iter = ', 23700)\n",
    "epoch 122, minibatch 195/195, validation error 9.006811 %\n",
    "('training @ iter = ', 23800)\n",
    "('training @ iter = ', 23900)\n",
    "epoch 123, minibatch 195/195, validation error 3.267628 %\n",
    "('training @ iter = ', 24000)\n",
    "('training @ iter = ', 24100)\n",
    "epoch 124, minibatch 195/195, validation error 3.177484 %\n",
    "     epoch 124, minibatch 195/195, test error of best model 3.367788 %\n",
    "('training @ iter = ', 24200)\n",
    "('training @ iter = ', 24300)\n",
    "epoch 125, minibatch 195/195, validation error 3.337740 %\n",
    "('training @ iter = ', 24400)\n",
    "('training @ iter = ', 24500)\n",
    "epoch 126, minibatch 195/195, validation error 3.287660 %\n",
    "('training @ iter = ', 24600)\n",
    "('training @ iter = ', 24700)\n",
    "epoch 127, minibatch 195/195, validation error 3.117388 %\n",
    "     epoch 127, minibatch 195/195, test error of best model 3.377804 %\n",
    "('training @ iter = ', 24800)\n",
    "('training @ iter = ', 24900)\n",
    "epoch 128, minibatch 195/195, validation error 3.467949 %\n",
    "('training @ iter = ', 25000)\n",
    "('training @ iter = ', 25100)\n",
    "epoch 129, minibatch 195/195, validation error 3.437901 %\n",
    "('training @ iter = ', 25200)\n",
    "('training @ iter = ', 25300)\n",
    "epoch 130, minibatch 195/195, validation error 4.139022 %\n",
    "('training @ iter = ', 25400)\n",
    "('training @ iter = ', 25500)\n",
    "epoch 131, minibatch 195/195, validation error 3.497997 %\n",
    "('training @ iter = ', 25600)\n",
    "('training @ iter = ', 25700)\n",
    "epoch 132, minibatch 195/195, validation error 3.508013 %\n",
    "('training @ iter = ', 25800)\n",
    "('training @ iter = ', 25900)\n",
    "epoch 133, minibatch 195/195, validation error 3.417869 %\n",
    "('training @ iter = ', 26000)\n",
    "('training @ iter = ', 26100)\n",
    "epoch 134, minibatch 195/195, validation error 3.588141 %\n",
    "('training @ iter = ', 26200)\n",
    "('training @ iter = ', 26300)\n",
    "epoch 135, minibatch 195/195, validation error 5.401042 %\n",
    "('training @ iter = ', 26400)\n",
    "('training @ iter = ', 26500)\n",
    "epoch 136, minibatch 195/195, validation error 8.776442 %\n",
    "('training @ iter = ', 26600)\n",
    "('training @ iter = ', 26700)\n",
    "epoch 137, minibatch 195/195, validation error 3.307692 %\n",
    "('training @ iter = ', 26800)\n",
    "('training @ iter = ', 26900)\n",
    "epoch 138, minibatch 195/195, validation error 4.259215 %\n",
    "('training @ iter = ', 27000)\n",
    "('training @ iter = ', 27100)\n",
    "epoch 139, minibatch 195/195, validation error 3.317708 %\n",
    "('training @ iter = ', 27200)\n",
    "epoch 140, minibatch 195/195, validation error 3.407853 %\n",
    "('training @ iter = ', 27300)\n",
    "('training @ iter = ', 27400)\n",
    "epoch 141, minibatch 195/195, validation error 4.309295 %\n",
    "('training @ iter = ', 27500)\n",
    "('training @ iter = ', 27600)\n",
    "epoch 142, minibatch 195/195, validation error 3.788462 %\n",
    "('training @ iter = ', 27700)\n",
    "('training @ iter = ', 27800)\n",
    "epoch 143, minibatch 195/195, validation error 3.477965 %\n",
    "('training @ iter = ', 27900)\n",
    "('training @ iter = ', 28000)\n",
    "epoch 144, minibatch 195/195, validation error 3.327724 %\n",
    "('training @ iter = ', 28100)\n",
    "('training @ iter = ', 28200)\n",
    "epoch 145, minibatch 195/195, validation error 3.437901 %\n",
    "('training @ iter = ', 28300)\n",
    "('training @ iter = ', 28400)\n",
    "epoch 146, minibatch 195/195, validation error 3.598157 %\n",
    "('training @ iter = ', 28500)\n",
    "('training @ iter = ', 28600)\n",
    "epoch 147, minibatch 195/195, validation error 3.538061 %\n",
    "('training @ iter = ', 28700)\n",
    "('training @ iter = ', 28800)\n",
    "epoch 148, minibatch 195/195, validation error 3.768429 %\n",
    "('training @ iter = ', 28900)\n",
    "('training @ iter = ', 29000)\n",
    "epoch 149, minibatch 195/195, validation error 3.948718 %\n",
    "('training @ iter = ', 29100)\n",
    "('training @ iter = ', 29200)\n",
    "epoch 150, minibatch 195/195, validation error 3.728365 %\n",
    "('training @ iter = ', 29300)\n",
    "('training @ iter = ', 29400)\n",
    "epoch 151, minibatch 195/195, validation error 3.758413 %\n",
    "('training @ iter = ', 29500)\n",
    "('training @ iter = ', 29600)\n",
    "epoch 152, minibatch 195/195, validation error 3.377804 %\n",
    "('training @ iter = ', 29700)\n",
    "('training @ iter = ', 29800)\n",
    "epoch 153, minibatch 195/195, validation error 3.548077 %\n",
    "('training @ iter = ', 29900)\n",
    "('training @ iter = ', 30000)\n",
    "epoch 154, minibatch 195/195, validation error 3.297676 %\n",
    "('training @ iter = ', 30100)\n",
    "('training @ iter = ', 30200)\n",
    "epoch 155, minibatch 195/195, validation error 3.387821 %\n",
    "('training @ iter = ', 30300)\n",
    "('training @ iter = ', 30400)\n",
    "epoch 156, minibatch 195/195, validation error 3.558093 %\n",
    "('training @ iter = ', 30500)\n",
    "('training @ iter = ', 30600)\n",
    "epoch 157, minibatch 195/195, validation error 3.287660 %\n",
    "('training @ iter = ', 30700)\n",
    "('training @ iter = ', 30800)\n",
    "epoch 158, minibatch 195/195, validation error 3.407853 %\n",
    "('training @ iter = ', 30900)\n",
    "('training @ iter = ', 31000)\n",
    "epoch 159, minibatch 195/195, validation error 3.447917 %\n",
    "('training @ iter = ', 31100)\n",
    "epoch 160, minibatch 195/195, validation error 3.237580 %\n",
    "('training @ iter = ', 31200)\n",
    "('training @ iter = ', 31300)\n",
    "epoch 161, minibatch 195/195, validation error 3.437901 %\n",
    "('training @ iter = ', 31400)\n",
    "('training @ iter = ', 31500)\n",
    "epoch 162, minibatch 195/195, validation error 3.307692 %\n",
    "('training @ iter = ', 31600)\n",
    "('training @ iter = ', 31700)\n",
    "epoch 163, minibatch 195/195, validation error 3.467949 %\n",
    "('training @ iter = ', 31800)\n",
    "('training @ iter = ', 31900)\n",
    "epoch 164, minibatch 195/195, validation error 3.718349 %\n",
    "('training @ iter = ', 32000)\n",
    "('training @ iter = ', 32100)\n",
    "epoch 165, minibatch 195/195, validation error 3.377804 %\n",
    "('training @ iter = ', 32200)\n",
    "('training @ iter = ', 32300)\n",
    "epoch 166, minibatch 195/195, validation error 3.427885 %\n",
    "('training @ iter = ', 32400)\n",
    "('training @ iter = ', 32500)\n",
    "epoch 167, minibatch 195/195, validation error 3.387821 %\n",
    "('training @ iter = ', 32600)\n",
    "('training @ iter = ', 32700)\n",
    "epoch 168, minibatch 195/195, validation error 3.387821 %\n",
    "('training @ iter = ', 32800)\n",
    "('training @ iter = ', 32900)\n",
    "epoch 169, minibatch 195/195, validation error 3.487981 %\n",
    "('training @ iter = ', 33000)\n",
    "('training @ iter = ', 33100)\n",
    "epoch 170, minibatch 195/195, validation error 3.347756 %\n",
    "('training @ iter = ', 33200)\n",
    "('training @ iter = ', 33300)\n",
    "epoch 171, minibatch 195/195, validation error 3.788462 %\n",
    "('training @ iter = ', 33400)\n",
    "('training @ iter = ', 33500)\n",
    "epoch 172, minibatch 195/195, validation error 3.437901 %\n",
    "('training @ iter = ', 33600)\n",
    "('training @ iter = ', 33700)\n",
    "epoch 173, minibatch 195/195, validation error 14.235176 %\n",
    "('training @ iter = ', 33800)\n",
    "('training @ iter = ', 33900)\n",
    "epoch 174, minibatch 195/195, validation error 3.447917 %\n",
    "('training @ iter = ', 34000)\n",
    "('training @ iter = ', 34100)\n",
    "epoch 175, minibatch 195/195, validation error 3.558093 %\n",
    "('training @ iter = ', 34200)\n",
    "('training @ iter = ', 34300)\n",
    "epoch 176, minibatch 195/195, validation error 3.347756 %\n",
    "('training @ iter = ', 34400)\n",
    "('training @ iter = ', 34500)\n",
    "epoch 177, minibatch 195/195, validation error 3.578125 %\n",
    "('training @ iter = ', 34600)\n",
    "('training @ iter = ', 34700)\n",
    "epoch 178, minibatch 195/195, validation error 3.317708 %\n",
    "('training @ iter = ', 34800)\n",
    "('training @ iter = ', 34900)\n",
    "epoch 179, minibatch 195/195, validation error 3.357772 %\n",
    "('training @ iter = ', 35000)\n",
    "epoch 180, minibatch 195/195, validation error 3.337740 %\n",
    "('training @ iter = ', 35100)\n",
    "('training @ iter = ', 35200)\n",
    "epoch 181, minibatch 195/195, validation error 3.297676 %\n",
    "('training @ iter = ', 35300)\n",
    "('training @ iter = ', 35400)\n",
    "epoch 182, minibatch 195/195, validation error 3.568109 %\n",
    "('training @ iter = ', 35500)\n",
    "('training @ iter = ', 35600)\n",
    "epoch 183, minibatch 195/195, validation error 3.427885 %\n",
    "('training @ iter = ', 35700)\n",
    "('training @ iter = ', 35800)\n",
    "epoch 184, minibatch 195/195, validation error 3.437901 %\n",
    "('training @ iter = ', 35900)\n",
    "('training @ iter = ', 36000)\n",
    "epoch 185, minibatch 195/195, validation error 3.317708 %\n",
    "('training @ iter = ', 36100)\n",
    "('training @ iter = ', 36200)\n",
    "epoch 186, minibatch 195/195, validation error 3.558093 %\n",
    "('training @ iter = ', 36300)\n",
    "('training @ iter = ', 36400)\n",
    "epoch 187, minibatch 195/195, validation error 3.197516 %\n",
    "('training @ iter = ', 36500)\n",
    "('training @ iter = ', 36600)\n",
    "epoch 188, minibatch 195/195, validation error 3.357772 %\n",
    "('training @ iter = ', 36700)\n",
    "('training @ iter = ', 36800)\n",
    "epoch 189, minibatch 195/195, validation error 3.407853 %\n",
    "('training @ iter = ', 36900)\n",
    "('training @ iter = ', 37000)\n",
    "epoch 190, minibatch 195/195, validation error 3.427885 %\n",
    "('training @ iter = ', 37100)\n",
    "('training @ iter = ', 37200)\n",
    "epoch 191, minibatch 195/195, validation error 3.447917 %\n",
    "('training @ iter = ', 37300)\n",
    "('training @ iter = ', 37400)\n",
    "epoch 192, minibatch 195/195, validation error 7.724760 %\n",
    "('training @ iter = ', 37500)\n",
    "('training @ iter = ', 37600)\n",
    "epoch 193, minibatch 195/195, validation error 3.137420 %\n",
    "('training @ iter = ', 37700)\n",
    "('training @ iter = ', 37800)\n",
    "epoch 194, minibatch 195/195, validation error 3.357772 %\n",
    "('training @ iter = ', 37900)\n",
    "('training @ iter = ', 38000)\n",
    "epoch 195, minibatch 195/195, validation error 3.357772 %\n",
    "('training @ iter = ', 38100)\n",
    "('training @ iter = ', 38200)\n",
    "epoch 196, minibatch 195/195, validation error 3.337740 %\n",
    "('training @ iter = ', 38300)\n",
    "('training @ iter = ', 38400)\n",
    "epoch 197, minibatch 195/195, validation error 3.437901 %\n",
    "('training @ iter = ', 38500)\n",
    "('training @ iter = ', 38600)\n",
    "epoch 198, minibatch 195/195, validation error 3.457933 %\n",
    "('training @ iter = ', 38700)\n",
    "('training @ iter = ', 38800)\n",
    "epoch 199, minibatch 195/195, validation error 3.467949 %\n",
    "('training @ iter = ', 38900)\n",
    "epoch 200, minibatch 195/195, validation error 3.738381 %\n",
    "Optimization complete.\n",
    "Best validation score of 3.17388 % obtained at iteration 24765, with test performance 3.377804 %\n",
    "The training process for function run ran for 78.38m\n",
    "----------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "...loading the dataset\n",
      "...Applying rts_combo to dataset\n",
      "Current training data size is 50000\n",
      "Current validation data size is 10000\n",
      "Current test data size is 10000\n",
      "...building the model\n",
      "...training fcn with include_st = False\n",
      "...training\n",
      "('training @ iter = ', 0)\n",
      "('training @ iter = ', 100)\n",
      "epoch 1, minibatch 195/195, validation error 90.084135 %\n",
      "     epoch 1, minibatch 195/195, test error of best model 90.174279 %\n",
      "('training @ iter = ', 200)\n",
      "('training @ iter = ', 300)\n",
      "epoch 2, minibatch 195/195, validation error 89.763622 %\n",
      "     epoch 2, minibatch 195/195, test error of best model 89.953926 %\n",
      "('training @ iter = ', 400)\n",
      "('training @ iter = ', 500)\n",
      "epoch 3, minibatch 195/195, validation error 88.782051 %\n",
      "     epoch 3, minibatch 195/195, test error of best model 88.812099 %\n",
      "('training @ iter = ', 600)\n",
      "('training @ iter = ', 700)\n",
      "epoch 4, minibatch 195/195, validation error 87.199519 %\n",
      "     epoch 4, minibatch 195/195, test error of best model 87.199519 %\n",
      "('training @ iter = ', 800)\n",
      "('training @ iter = ', 900)\n",
      "epoch 5, minibatch 195/195, validation error 84.945913 %\n",
      "     epoch 5, minibatch 195/195, test error of best model 84.975962 %\n",
      "('training @ iter = ', 1000)\n",
      "('training @ iter = ', 1100)\n",
      "epoch 6, minibatch 195/195, validation error 82.471955 %\n",
      "     epoch 6, minibatch 195/195, test error of best model 82.982772 %\n",
      "('training @ iter = ', 1200)\n",
      "('training @ iter = ', 1300)\n",
      "epoch 7, minibatch 195/195, validation error 80.218349 %\n",
      "     epoch 7, minibatch 195/195, test error of best model 80.769231 %\n",
      "('training @ iter = ', 1400)\n",
      "('training @ iter = ', 1500)\n",
      "epoch 8, minibatch 195/195, validation error 78.535657 %\n",
      "     epoch 8, minibatch 195/195, test error of best model 78.615785 %\n",
      "('training @ iter = ', 1600)\n",
      "('training @ iter = ', 1700)\n",
      "epoch 9, minibatch 195/195, validation error 77.273638 %\n",
      "     epoch 9, minibatch 195/195, test error of best model 77.103365 %\n",
      "('training @ iter = ', 1800)\n",
      "('training @ iter = ', 1900)\n",
      "epoch 10, minibatch 195/195, validation error 76.392228 %\n",
      "     epoch 10, minibatch 195/195, test error of best model 75.851362 %\n",
      "('training @ iter = ', 2000)\n",
      "('training @ iter = ', 2100)\n",
      "epoch 11, minibatch 195/195, validation error 75.450721 %\n",
      "     epoch 11, minibatch 195/195, test error of best model 75.090144 %\n",
      "('training @ iter = ', 2200)\n",
      "('training @ iter = ', 2300)\n",
      "epoch 12, minibatch 195/195, validation error 74.939904 %\n",
      "     epoch 12, minibatch 195/195, test error of best model 74.579327 %\n",
      "('training @ iter = ', 2400)\n",
      "('training @ iter = ', 2500)\n",
      "epoch 13, minibatch 195/195, validation error 74.028446 %\n",
      "     epoch 13, minibatch 195/195, test error of best model 73.858173 %\n",
      "('training @ iter = ', 2600)\n",
      "('training @ iter = ', 2700)\n",
      "epoch 14, minibatch 195/195, validation error 73.106971 %\n",
      "     epoch 14, minibatch 195/195, test error of best model 73.026843 %\n",
      "('training @ iter = ', 2800)\n",
      "('training @ iter = ', 2900)\n",
      "epoch 15, minibatch 195/195, validation error 71.975160 %\n",
      "     epoch 15, minibatch 195/195, test error of best model 71.935096 %\n",
      "('training @ iter = ', 3000)\n",
      "('training @ iter = ', 3100)\n",
      "epoch 16, minibatch 195/195, validation error 70.933494 %\n",
      "     epoch 16, minibatch 195/195, test error of best model 71.003606 %\n",
      "('training @ iter = ', 3200)\n",
      "('training @ iter = ', 3300)\n",
      "epoch 17, minibatch 195/195, validation error 69.851763 %\n",
      "     epoch 17, minibatch 195/195, test error of best model 69.961939 %\n",
      "('training @ iter = ', 3400)\n",
      "('training @ iter = ', 3500)\n",
      "epoch 18, minibatch 195/195, validation error 68.880208 %\n",
      "     epoch 18, minibatch 195/195, test error of best model 68.709936 %\n",
      "('training @ iter = ', 3600)\n",
      "('training @ iter = ', 3700)\n",
      "epoch 19, minibatch 195/195, validation error 67.698317 %\n",
      "     epoch 19, minibatch 195/195, test error of best model 67.798478 %\n",
      "('training @ iter = ', 3800)\n",
      "epoch 20, minibatch 195/195, validation error 66.426282 %\n",
      "     epoch 20, minibatch 195/195, test error of best model 66.726763 %\n",
      "('training @ iter = ', 3900)\n",
      "('training @ iter = ', 4000)\n",
      "epoch 21, minibatch 195/195, validation error 65.494792 %\n",
      "     epoch 21, minibatch 195/195, test error of best model 65.574920 %\n",
      "('training @ iter = ', 4100)\n",
      "('training @ iter = ', 4200)\n",
      "epoch 22, minibatch 195/195, validation error 64.383013 %\n",
      "     epoch 22, minibatch 195/195, test error of best model 64.473157 %\n",
      "('training @ iter = ', 4300)\n",
      "('training @ iter = ', 4400)\n",
      "epoch 23, minibatch 195/195, validation error 63.301282 %\n",
      "     epoch 23, minibatch 195/195, test error of best model 63.401442 %\n",
      "('training @ iter = ', 4500)\n",
      "('training @ iter = ', 4600)\n",
      "epoch 24, minibatch 195/195, validation error 61.909054 %\n",
      "     epoch 24, minibatch 195/195, test error of best model 62.239583 %\n",
      "('training @ iter = ', 4700)\n",
      "('training @ iter = ', 4800)\n",
      "epoch 25, minibatch 195/195, validation error 60.717147 %\n",
      "     epoch 25, minibatch 195/195, test error of best model 60.907452 %\n",
      "('training @ iter = ', 4900)\n",
      "('training @ iter = ', 5000)\n",
      "epoch 26, minibatch 195/195, validation error 59.495192 %\n",
      "     epoch 26, minibatch 195/195, test error of best model 60.096154 %\n",
      "('training @ iter = ', 5100)\n",
      "('training @ iter = ', 5200)\n",
      "epoch 27, minibatch 195/195, validation error 58.433494 %\n",
      "     epoch 27, minibatch 195/195, test error of best model 58.884215 %\n",
      "('training @ iter = ', 5300)\n",
      "('training @ iter = ', 5400)\n",
      "epoch 28, minibatch 195/195, validation error 57.061298 %\n",
      "     epoch 28, minibatch 195/195, test error of best model 57.932692 %\n",
      "('training @ iter = ', 5500)\n",
      "('training @ iter = ', 5600)\n",
      "epoch 29, minibatch 195/195, validation error 56.149840 %\n",
      "     epoch 29, minibatch 195/195, test error of best model 56.921074 %\n",
      "('training @ iter = ', 5700)\n",
      "('training @ iter = ', 5800)\n",
      "epoch 30, minibatch 195/195, validation error 55.138221 %\n",
      "     epoch 30, minibatch 195/195, test error of best model 55.819311 %\n",
      "('training @ iter = ', 5900)\n",
      "('training @ iter = ', 6000)\n",
      "epoch 31, minibatch 195/195, validation error 54.016426 %\n",
      "     epoch 31, minibatch 195/195, test error of best model 54.717548 %\n",
      "('training @ iter = ', 6100)\n",
      "('training @ iter = ', 6200)\n",
      "epoch 32, minibatch 195/195, validation error 53.345353 %\n",
      "     epoch 32, minibatch 195/195, test error of best model 53.836138 %\n",
      "('training @ iter = ', 6300)\n",
      "('training @ iter = ', 6400)\n",
      "epoch 33, minibatch 195/195, validation error 52.353766 %\n",
      "     epoch 33, minibatch 195/195, test error of best model 52.764423 %\n",
      "('training @ iter = ', 6500)\n",
      "('training @ iter = ', 6600)\n",
      "epoch 34, minibatch 195/195, validation error 51.322115 %\n",
      "     epoch 34, minibatch 195/195, test error of best model 52.073317 %\n",
      "('training @ iter = ', 6700)\n",
      "('training @ iter = ', 6800)\n",
      "epoch 35, minibatch 195/195, validation error 50.510817 %\n",
      "     epoch 35, minibatch 195/195, test error of best model 51.061699 %\n",
      "('training @ iter = ', 6900)\n",
      "('training @ iter = ', 7000)\n",
      "epoch 36, minibatch 195/195, validation error 49.268830 %\n",
      "     epoch 36, minibatch 195/195, test error of best model 49.959936 %\n",
      "('training @ iter = ', 7100)\n",
      "('training @ iter = ', 7200)\n",
      "epoch 37, minibatch 195/195, validation error 48.227163 %\n",
      "     epoch 37, minibatch 195/195, test error of best model 48.978365 %\n",
      "('training @ iter = ', 7300)\n",
      "('training @ iter = ', 7400)\n",
      "epoch 38, minibatch 195/195, validation error 47.405849 %\n",
      "     epoch 38, minibatch 195/195, test error of best model 48.076923 %\n",
      "('training @ iter = ', 7500)\n",
      "('training @ iter = ', 7600)\n",
      "epoch 39, minibatch 195/195, validation error 46.444311 %\n",
      "     epoch 39, minibatch 195/195, test error of best model 47.135417 %\n",
      "('training @ iter = ', 7700)\n",
      "epoch 40, minibatch 195/195, validation error 45.502804 %\n",
      "     epoch 40, minibatch 195/195, test error of best model 46.294071 %\n",
      "('training @ iter = ', 7800)\n",
      "('training @ iter = ', 7900)\n",
      "epoch 41, minibatch 195/195, validation error 44.621394 %\n",
      "     epoch 41, minibatch 195/195, test error of best model 45.412660 %\n",
      "('training @ iter = ', 8000)\n",
      "('training @ iter = ', 8100)\n",
      "epoch 42, minibatch 195/195, validation error 43.579728 %\n",
      "     epoch 42, minibatch 195/195, test error of best model 44.631410 %\n",
      "('training @ iter = ', 8200)\n",
      "('training @ iter = ', 8300)\n",
      "epoch 43, minibatch 195/195, validation error 42.738381 %\n",
      "     epoch 43, minibatch 195/195, test error of best model 43.729968 %\n",
      "('training @ iter = ', 8400)\n",
      "('training @ iter = ', 8500)\n",
      "epoch 44, minibatch 195/195, validation error 41.907051 %\n",
      "     epoch 44, minibatch 195/195, test error of best model 42.828526 %\n",
      "('training @ iter = ', 8600)\n",
      "('training @ iter = ', 8700)\n",
      "epoch 45, minibatch 195/195, validation error 41.276042 %\n",
      "     epoch 45, minibatch 195/195, test error of best model 42.017228 %\n",
      "('training @ iter = ', 8800)\n",
      "('training @ iter = ', 8900)\n",
      "epoch 46, minibatch 195/195, validation error 40.504808 %\n",
      "     epoch 46, minibatch 195/195, test error of best model 41.175881 %\n",
      "('training @ iter = ', 9000)\n",
      "('training @ iter = ', 9100)\n",
      "epoch 47, minibatch 195/195, validation error 39.783654 %\n",
      "     epoch 47, minibatch 195/195, test error of best model 40.434696 %\n",
      "('training @ iter = ', 9200)\n",
      "('training @ iter = ', 9300)\n",
      "epoch 48, minibatch 195/195, validation error 39.222756 %\n",
      "     epoch 48, minibatch 195/195, test error of best model 39.803686 %\n",
      "('training @ iter = ', 9400)\n",
      "('training @ iter = ', 9500)\n",
      "epoch 49, minibatch 195/195, validation error 38.641827 %\n",
      "     epoch 49, minibatch 195/195, test error of best model 39.012420 %\n",
      "('training @ iter = ', 9600)\n",
      "('training @ iter = ', 9700)\n",
      "epoch 50, minibatch 195/195, validation error 37.870593 %\n",
      "     epoch 50, minibatch 195/195, test error of best model 38.431490 %\n",
      "('training @ iter = ', 9800)\n",
      "('training @ iter = ', 9900)\n",
      "epoch 51, minibatch 195/195, validation error 37.299679 %\n",
      "     epoch 51, minibatch 195/195, test error of best model 37.820513 %\n",
      "('training @ iter = ', 10000)\n",
      "('training @ iter = ', 10100)\n",
      "epoch 52, minibatch 195/195, validation error 36.678686 %\n",
      "     epoch 52, minibatch 195/195, test error of best model 37.199519 %\n",
      "('training @ iter = ', 10200)\n",
      "('training @ iter = ', 10300)\n",
      "epoch 53, minibatch 195/195, validation error 36.057692 %\n",
      "     epoch 53, minibatch 195/195, test error of best model 36.568510 %\n",
      "('training @ iter = ', 10400)\n",
      "('training @ iter = ', 10500)\n",
      "epoch 54, minibatch 195/195, validation error 35.526843 %\n",
      "     epoch 54, minibatch 195/195, test error of best model 36.087740 %\n",
      "('training @ iter = ', 10600)\n",
      "('training @ iter = ', 10700)\n",
      "epoch 55, minibatch 195/195, validation error 35.126202 %\n",
      "     epoch 55, minibatch 195/195, test error of best model 35.536859 %\n",
      "('training @ iter = ', 10800)\n",
      "('training @ iter = ', 10900)\n",
      "epoch 56, minibatch 195/195, validation error 34.705529 %\n",
      "     epoch 56, minibatch 195/195, test error of best model 34.965946 %\n",
      "('training @ iter = ', 11000)\n",
      "('training @ iter = ', 11100)\n",
      "epoch 57, minibatch 195/195, validation error 34.084535 %\n",
      "     epoch 57, minibatch 195/195, test error of best model 34.445112 %\n",
      "('training @ iter = ', 11200)\n",
      "('training @ iter = ', 11300)\n",
      "epoch 58, minibatch 195/195, validation error 33.613782 %\n",
      "     epoch 58, minibatch 195/195, test error of best model 33.904247 %\n",
      "('training @ iter = ', 11400)\n",
      "('training @ iter = ', 11500)\n",
      "epoch 59, minibatch 195/195, validation error 33.173077 %\n",
      "     epoch 59, minibatch 195/195, test error of best model 33.353365 %\n",
      "('training @ iter = ', 11600)\n",
      "epoch 60, minibatch 195/195, validation error 32.842548 %\n",
      "     epoch 60, minibatch 195/195, test error of best model 32.752404 %\n",
      "('training @ iter = ', 11700)\n",
      "('training @ iter = ', 11800)\n",
      "epoch 61, minibatch 195/195, validation error 32.391827 %\n",
      "     epoch 61, minibatch 195/195, test error of best model 32.391827 %\n",
      "('training @ iter = ', 11900)\n",
      "('training @ iter = ', 12000)\n",
      "epoch 62, minibatch 195/195, validation error 32.181490 %\n",
      "     epoch 62, minibatch 195/195, test error of best model 31.931090 %\n",
      "('training @ iter = ', 12100)\n",
      "('training @ iter = ', 12200)\n",
      "epoch 63, minibatch 195/195, validation error 31.650641 %\n",
      "     epoch 63, minibatch 195/195, test error of best model 31.610577 %\n",
      "('training @ iter = ', 12300)\n",
      "('training @ iter = ', 12400)\n",
      "epoch 64, minibatch 195/195, validation error 31.219952 %\n",
      "     epoch 64, minibatch 195/195, test error of best model 31.300080 %\n",
      "('training @ iter = ', 12500)\n",
      "('training @ iter = ', 12600)\n",
      "epoch 65, minibatch 195/195, validation error 30.839343 %\n",
      "     epoch 65, minibatch 195/195, test error of best model 30.979567 %\n",
      "('training @ iter = ', 12700)\n",
      "('training @ iter = ', 12800)\n",
      "epoch 66, minibatch 195/195, validation error 30.598958 %\n",
      "     epoch 66, minibatch 195/195, test error of best model 30.679087 %\n",
      "('training @ iter = ', 12900)\n",
      "('training @ iter = ', 13000)\n",
      "epoch 67, minibatch 195/195, validation error 30.278446 %\n",
      "     epoch 67, minibatch 195/195, test error of best model 30.438702 %\n",
      "('training @ iter = ', 13100)\n",
      "('training @ iter = ', 13200)\n",
      "epoch 68, minibatch 195/195, validation error 30.218349 %\n",
      "     epoch 68, minibatch 195/195, test error of best model 30.078125 %\n",
      "('training @ iter = ', 13300)\n",
      "('training @ iter = ', 13400)\n",
      "epoch 69, minibatch 195/195, validation error 29.927885 %\n",
      "     epoch 69, minibatch 195/195, test error of best model 29.697516 %\n",
      "('training @ iter = ', 13500)\n",
      "('training @ iter = ', 13600)\n",
      "epoch 70, minibatch 195/195, validation error 29.727564 %\n",
      "     epoch 70, minibatch 195/195, test error of best model 29.497196 %\n",
      "('training @ iter = ', 13700)\n",
      "('training @ iter = ', 13800)\n",
      "epoch 71, minibatch 195/195, validation error 29.537260 %\n",
      "     epoch 71, minibatch 195/195, test error of best model 29.206731 %\n",
      "('training @ iter = ', 13900)\n",
      "('training @ iter = ', 14000)\n",
      "epoch 72, minibatch 195/195, validation error 29.176683 %\n",
      "     epoch 72, minibatch 195/195, test error of best model 28.856170 %\n",
      "('training @ iter = ', 14100)\n",
      "('training @ iter = ', 14200)\n",
      "epoch 73, minibatch 195/195, validation error 28.986378 %\n",
      "     epoch 73, minibatch 195/195, test error of best model 28.635817 %\n",
      "('training @ iter = ', 14300)\n",
      "('training @ iter = ', 14400)\n",
      "epoch 74, minibatch 195/195, validation error 28.776042 %\n",
      "     epoch 74, minibatch 195/195, test error of best model 28.275240 %\n",
      "('training @ iter = ', 14500)\n",
      "('training @ iter = ', 14600)\n",
      "epoch 75, minibatch 195/195, validation error 28.505609 %\n",
      "     epoch 75, minibatch 195/195, test error of best model 28.175080 %\n",
      "('training @ iter = ', 14700)\n",
      "('training @ iter = ', 14800)\n",
      "epoch 76, minibatch 195/195, validation error 28.275240 %\n",
      "     epoch 76, minibatch 195/195, test error of best model 27.814503 %\n",
      "('training @ iter = ', 14900)\n",
      "('training @ iter = ', 15000)\n",
      "epoch 77, minibatch 195/195, validation error 28.014824 %\n",
      "     epoch 77, minibatch 195/195, test error of best model 27.483974 %\n",
      "('training @ iter = ', 15100)\n",
      "('training @ iter = ', 15200)\n",
      "epoch 78, minibatch 195/195, validation error 27.694311 %\n",
      "     epoch 78, minibatch 195/195, test error of best model 27.183494 %\n",
      "('training @ iter = ', 15300)\n",
      "('training @ iter = ', 15400)\n",
      "epoch 79, minibatch 195/195, validation error 27.524038 %\n",
      "     epoch 79, minibatch 195/195, test error of best model 27.083333 %\n",
      "('training @ iter = ', 15500)\n",
      "epoch 80, minibatch 195/195, validation error 27.333734 %\n",
      "     epoch 80, minibatch 195/195, test error of best model 26.872997 %\n",
      "('training @ iter = ', 15600)\n",
      "('training @ iter = ', 15700)\n",
      "epoch 81, minibatch 195/195, validation error 27.163462 %\n",
      "     epoch 81, minibatch 195/195, test error of best model 26.722756 %\n",
      "('training @ iter = ', 15800)\n",
      "('training @ iter = ', 15900)\n",
      "epoch 82, minibatch 195/195, validation error 26.953125 %\n",
      "     epoch 82, minibatch 195/195, test error of best model 26.502404 %\n",
      "('training @ iter = ', 16000)\n",
      "('training @ iter = ', 16100)\n",
      "epoch 83, minibatch 195/195, validation error 26.692708 %\n",
      "     epoch 83, minibatch 195/195, test error of best model 26.312099 %\n",
      "('training @ iter = ', 16200)\n",
      "('training @ iter = ', 16300)\n",
      "epoch 84, minibatch 195/195, validation error 26.512420 %\n",
      "     epoch 84, minibatch 195/195, test error of best model 26.111779 %\n",
      "('training @ iter = ', 16400)\n",
      "('training @ iter = ', 16500)\n",
      "epoch 85, minibatch 195/195, validation error 26.262019 %\n",
      "     epoch 85, minibatch 195/195, test error of best model 25.961538 %\n",
      "('training @ iter = ', 16600)\n",
      "('training @ iter = ', 16700)\n",
      "epoch 86, minibatch 195/195, validation error 26.151843 %\n",
      "     epoch 86, minibatch 195/195, test error of best model 25.791266 %\n",
      "('training @ iter = ', 16800)\n",
      "('training @ iter = ', 16900)\n",
      "epoch 87, minibatch 195/195, validation error 26.031651 %\n",
      "     epoch 87, minibatch 195/195, test error of best model 25.681090 %\n",
      "('training @ iter = ', 17000)\n",
      "('training @ iter = ', 17100)\n",
      "epoch 88, minibatch 195/195, validation error 25.791266 %\n",
      "     epoch 88, minibatch 195/195, test error of best model 25.520833 %\n",
      "('training @ iter = ', 17200)\n",
      "('training @ iter = ', 17300)\n",
      "epoch 89, minibatch 195/195, validation error 25.691106 %\n",
      "     epoch 89, minibatch 195/195, test error of best model 25.390625 %\n",
      "('training @ iter = ', 17400)\n",
      "('training @ iter = ', 17500)\n",
      "epoch 90, minibatch 195/195, validation error 25.550881 %\n",
      "     epoch 90, minibatch 195/195, test error of best model 25.210337 %\n",
      "('training @ iter = ', 17600)\n",
      "('training @ iter = ', 17700)\n",
      "epoch 91, minibatch 195/195, validation error 25.280449 %\n",
      "     epoch 91, minibatch 195/195, test error of best model 25.070112 %\n",
      "('training @ iter = ', 17800)\n",
      "('training @ iter = ', 17900)\n",
      "epoch 92, minibatch 195/195, validation error 25.170272 %\n",
      "     epoch 92, minibatch 195/195, test error of best model 24.929888 %\n",
      "('training @ iter = ', 18000)\n",
      "('training @ iter = ', 18100)\n",
      "epoch 93, minibatch 195/195, validation error 25.040064 %\n",
      "     epoch 93, minibatch 195/195, test error of best model 24.709535 %\n",
      "('training @ iter = ', 18200)\n",
      "('training @ iter = ', 18300)\n",
      "epoch 94, minibatch 195/195, validation error 24.879808 %\n",
      "     epoch 94, minibatch 195/195, test error of best model 24.559295 %\n",
      "('training @ iter = ', 18400)\n",
      "('training @ iter = ', 18500)\n",
      "epoch 95, minibatch 195/195, validation error 24.789663 %\n",
      "     epoch 95, minibatch 195/195, test error of best model 24.529247 %\n",
      "('training @ iter = ', 18600)\n",
      "('training @ iter = ', 18700)\n",
      "epoch 96, minibatch 195/195, validation error 24.649439 %\n",
      "     epoch 96, minibatch 195/195, test error of best model 24.348958 %\n",
      "('training @ iter = ', 18800)\n",
      "('training @ iter = ', 18900)\n",
      "epoch 97, minibatch 195/195, validation error 24.659455 %\n",
      "('training @ iter = ', 19000)\n",
      "('training @ iter = ', 19100)\n",
      "epoch 98, minibatch 195/195, validation error 24.499199 %\n",
      "     epoch 98, minibatch 195/195, test error of best model 24.158654 %\n",
      "('training @ iter = ', 19200)\n",
      "('training @ iter = ', 19300)\n",
      "epoch 99, minibatch 195/195, validation error 24.449119 %\n",
      "     epoch 99, minibatch 195/195, test error of best model 24.038462 %\n",
      "('training @ iter = ', 19400)\n",
      "epoch 100, minibatch 195/195, validation error 24.288862 %\n",
      "     epoch 100, minibatch 195/195, test error of best model 23.938301 %\n",
      "('training @ iter = ', 19500)\n",
      "('training @ iter = ', 19600)\n",
      "epoch 101, minibatch 195/195, validation error 24.188702 %\n",
      "     epoch 101, minibatch 195/195, test error of best model 23.768029 %\n",
      "('training @ iter = ', 19700)\n",
      "('training @ iter = ', 19800)\n",
      "epoch 102, minibatch 195/195, validation error 24.098558 %\n",
      "     epoch 102, minibatch 195/195, test error of best model 23.727965 %\n",
      "('training @ iter = ', 19900)\n",
      "('training @ iter = ', 20000)\n",
      "epoch 103, minibatch 195/195, validation error 23.958333 %\n",
      "     epoch 103, minibatch 195/195, test error of best model 23.567708 %\n",
      "('training @ iter = ', 20100)\n",
      "('training @ iter = ', 20200)\n",
      "epoch 104, minibatch 195/195, validation error 23.918269 %\n",
      "     epoch 104, minibatch 195/195, test error of best model 23.547676 %\n",
      "('training @ iter = ', 20300)\n",
      "('training @ iter = ', 20400)\n",
      "epoch 105, minibatch 195/195, validation error 23.818109 %\n",
      "     epoch 105, minibatch 195/195, test error of best model 23.447516 %\n",
      "('training @ iter = ', 20500)\n",
      "('training @ iter = ', 20600)\n",
      "epoch 106, minibatch 195/195, validation error 23.667869 %\n",
      "     epoch 106, minibatch 195/195, test error of best model 23.377404 %\n",
      "('training @ iter = ', 20700)\n",
      "('training @ iter = ', 20800)\n",
      "epoch 107, minibatch 195/195, validation error 23.527644 %\n",
      "     epoch 107, minibatch 195/195, test error of best model 23.377404 %\n",
      "('training @ iter = ', 20900)\n",
      "('training @ iter = ', 21000)\n",
      "epoch 108, minibatch 195/195, validation error 23.457532 %\n",
      "     epoch 108, minibatch 195/195, test error of best model 23.337340 %\n",
      "('training @ iter = ', 21100)\n",
      "('training @ iter = ', 21200)\n",
      "epoch 109, minibatch 195/195, validation error 23.337340 %\n",
      "     epoch 109, minibatch 195/195, test error of best model 23.277244 %\n",
      "('training @ iter = ', 21300)\n",
      "('training @ iter = ', 21400)\n",
      "epoch 110, minibatch 195/195, validation error 23.267228 %\n",
      "     epoch 110, minibatch 195/195, test error of best model 23.197115 %\n",
      "('training @ iter = ', 21500)\n",
      "('training @ iter = ', 21600)\n",
      "epoch 111, minibatch 195/195, validation error 23.237179 %\n",
      "     epoch 111, minibatch 195/195, test error of best model 23.147035 %\n",
      "('training @ iter = ', 21700)\n",
      "('training @ iter = ', 21800)\n",
      "epoch 112, minibatch 195/195, validation error 23.106971 %\n",
      "     epoch 112, minibatch 195/195, test error of best model 23.207131 %\n",
      "('training @ iter = ', 21900)\n",
      "('training @ iter = ', 22000)\n",
      "epoch 113, minibatch 195/195, validation error 23.036859 %\n",
      "     epoch 113, minibatch 195/195, test error of best model 23.197115 %\n",
      "('training @ iter = ', 22100)\n",
      "('training @ iter = ', 22200)\n",
      "epoch 114, minibatch 195/195, validation error 22.986779 %\n",
      "     epoch 114, minibatch 195/195, test error of best model 23.127003 %\n",
      "('training @ iter = ', 22300)\n",
      "('training @ iter = ', 22400)\n",
      "epoch 115, minibatch 195/195, validation error 22.996795 %\n",
      "('training @ iter = ', 22500)\n",
      "('training @ iter = ', 22600)\n",
      "epoch 116, minibatch 195/195, validation error 22.906651 %\n",
      "     epoch 116, minibatch 195/195, test error of best model 23.046875 %\n",
      "('training @ iter = ', 22700)\n",
      "('training @ iter = ', 22800)\n",
      "epoch 117, minibatch 195/195, validation error 22.856571 %\n",
      "     epoch 117, minibatch 195/195, test error of best model 22.996795 %\n",
      "('training @ iter = ', 22900)\n",
      "('training @ iter = ', 23000)\n",
      "epoch 118, minibatch 195/195, validation error 22.776442 %\n",
      "     epoch 118, minibatch 195/195, test error of best model 22.876603 %\n",
      "('training @ iter = ', 23100)\n",
      "('training @ iter = ', 23200)\n",
      "epoch 119, minibatch 195/195, validation error 22.746394 %\n",
      "     epoch 119, minibatch 195/195, test error of best model 22.786458 %\n",
      "('training @ iter = ', 23300)\n",
      "epoch 120, minibatch 195/195, validation error 22.686298 %\n",
      "     epoch 120, minibatch 195/195, test error of best model 22.736378 %\n",
      "('training @ iter = ', 23400)\n",
      "('training @ iter = ', 23500)\n",
      "epoch 121, minibatch 195/195, validation error 22.646234 %\n",
      "     epoch 121, minibatch 195/195, test error of best model 22.686298 %\n",
      "('training @ iter = ', 23600)\n",
      "('training @ iter = ', 23700)\n",
      "epoch 122, minibatch 195/195, validation error 22.506010 %\n",
      "     epoch 122, minibatch 195/195, test error of best model 22.646234 %\n",
      "('training @ iter = ', 23800)\n",
      "('training @ iter = ', 23900)\n",
      "epoch 123, minibatch 195/195, validation error 22.395833 %\n",
      "     epoch 123, minibatch 195/195, test error of best model 22.616186 %\n",
      "('training @ iter = ', 24000)\n",
      "('training @ iter = ', 24100)\n",
      "epoch 124, minibatch 195/195, validation error 22.365785 %\n",
      "     epoch 124, minibatch 195/195, test error of best model 22.606170 %\n",
      "('training @ iter = ', 24200)\n",
      "('training @ iter = ', 24300)\n",
      "epoch 125, minibatch 195/195, validation error 22.315705 %\n",
      "     epoch 125, minibatch 195/195, test error of best model 22.576122 %\n",
      "('training @ iter = ', 24400)\n",
      "('training @ iter = ', 24500)\n",
      "epoch 126, minibatch 195/195, validation error 22.255609 %\n",
      "     epoch 126, minibatch 195/195, test error of best model 22.586138 %\n",
      "('training @ iter = ', 24600)\n",
      "('training @ iter = ', 24700)\n",
      "epoch 127, minibatch 195/195, validation error 22.165465 %\n",
      "     epoch 127, minibatch 195/195, test error of best model 22.546074 %\n",
      "('training @ iter = ', 24800)\n",
      "('training @ iter = ', 24900)\n",
      "epoch 128, minibatch 195/195, validation error 22.105369 %\n",
      "     epoch 128, minibatch 195/195, test error of best model 22.485978 %\n",
      "('training @ iter = ', 25000)\n",
      "('training @ iter = ', 25100)\n",
      "epoch 129, minibatch 195/195, validation error 22.105369 %\n",
      "('training @ iter = ', 25200)\n",
      "('training @ iter = ', 25300)\n",
      "epoch 130, minibatch 195/195, validation error 22.045272 %\n",
      "     epoch 130, minibatch 195/195, test error of best model 22.465946 %\n",
      "('training @ iter = ', 25400)\n",
      "('training @ iter = ', 25500)\n",
      "epoch 131, minibatch 195/195, validation error 22.025240 %\n",
      "     epoch 131, minibatch 195/195, test error of best model 22.415865 %\n",
      "('training @ iter = ', 25600)\n",
      "('training @ iter = ', 25700)\n",
      "epoch 132, minibatch 195/195, validation error 21.975160 %\n",
      "     epoch 132, minibatch 195/195, test error of best model 22.355769 %\n",
      "('training @ iter = ', 25800)\n",
      "('training @ iter = ', 25900)\n",
      "epoch 133, minibatch 195/195, validation error 21.975160 %\n",
      "('training @ iter = ', 26000)\n",
      "('training @ iter = ', 26100)\n",
      "epoch 134, minibatch 195/195, validation error 21.895032 %\n",
      "     epoch 134, minibatch 195/195, test error of best model 22.375801 %\n",
      "('training @ iter = ', 26200)\n",
      "('training @ iter = ', 26300)\n",
      "epoch 135, minibatch 195/195, validation error 21.895032 %\n",
      "('training @ iter = ', 26400)\n",
      "('training @ iter = ', 26500)\n",
      "epoch 136, minibatch 195/195, validation error 21.824920 %\n",
      "     epoch 136, minibatch 195/195, test error of best model 22.355769 %\n",
      "('training @ iter = ', 26600)\n",
      "('training @ iter = ', 26700)\n",
      "epoch 137, minibatch 195/195, validation error 21.714744 %\n",
      "     epoch 137, minibatch 195/195, test error of best model 22.295673 %\n",
      "('training @ iter = ', 26800)\n",
      "('training @ iter = ', 26900)\n",
      "epoch 138, minibatch 195/195, validation error 21.634615 %\n",
      "     epoch 138, minibatch 195/195, test error of best model 22.205529 %\n",
      "('training @ iter = ', 27000)\n",
      "('training @ iter = ', 27100)\n",
      "epoch 139, minibatch 195/195, validation error 21.544471 %\n",
      "     epoch 139, minibatch 195/195, test error of best model 22.185497 %\n",
      "('training @ iter = ', 27200)\n",
      "epoch 140, minibatch 195/195, validation error 21.504407 %\n",
      "     epoch 140, minibatch 195/195, test error of best model 22.175481 %\n",
      "('training @ iter = ', 27300)\n",
      "('training @ iter = ', 27400)\n",
      "epoch 141, minibatch 195/195, validation error 21.454327 %\n",
      "     epoch 141, minibatch 195/195, test error of best model 22.095353 %\n",
      "('training @ iter = ', 27500)\n",
      "('training @ iter = ', 27600)\n",
      "epoch 142, minibatch 195/195, validation error 21.434295 %\n",
      "     epoch 142, minibatch 195/195, test error of best model 22.045272 %\n",
      "('training @ iter = ', 27700)\n",
      "('training @ iter = ', 27800)\n",
      "epoch 143, minibatch 195/195, validation error 21.354167 %\n",
      "     epoch 143, minibatch 195/195, test error of best model 22.035256 %\n",
      "('training @ iter = ', 27900)\n",
      "('training @ iter = ', 28000)\n",
      "epoch 144, minibatch 195/195, validation error 21.344151 %\n",
      "     epoch 144, minibatch 195/195, test error of best model 21.965144 %\n",
      "('training @ iter = ', 28100)\n",
      "('training @ iter = ', 28200)\n",
      "epoch 145, minibatch 195/195, validation error 21.304087 %\n",
      "     epoch 145, minibatch 195/195, test error of best model 21.905048 %\n",
      "('training @ iter = ', 28300)\n",
      "('training @ iter = ', 28400)\n",
      "epoch 146, minibatch 195/195, validation error 21.274038 %\n",
      "     epoch 146, minibatch 195/195, test error of best model 21.804888 %\n",
      "('training @ iter = ', 28500)\n",
      "('training @ iter = ', 28600)\n",
      "epoch 147, minibatch 195/195, validation error 21.233974 %\n",
      "     epoch 147, minibatch 195/195, test error of best model 21.764824 %\n",
      "('training @ iter = ', 28700)\n",
      "('training @ iter = ', 28800)\n",
      "epoch 148, minibatch 195/195, validation error 21.213942 %\n",
      "     epoch 148, minibatch 195/195, test error of best model 21.744792 %\n",
      "('training @ iter = ', 28900)\n",
      "('training @ iter = ', 29000)\n",
      "epoch 149, minibatch 195/195, validation error 21.123798 %\n",
      "     epoch 149, minibatch 195/195, test error of best model 21.674679 %\n",
      "('training @ iter = ', 29100)\n",
      "('training @ iter = ', 29200)\n",
      "epoch 150, minibatch 195/195, validation error 21.073718 %\n",
      "     epoch 150, minibatch 195/195, test error of best model 21.614583 %\n",
      "('training @ iter = ', 29300)\n",
      "('training @ iter = ', 29400)\n",
      "epoch 151, minibatch 195/195, validation error 21.023638 %\n",
      "     epoch 151, minibatch 195/195, test error of best model 21.594551 %\n",
      "('training @ iter = ', 29500)\n",
      "('training @ iter = ', 29600)\n",
      "epoch 152, minibatch 195/195, validation error 21.003606 %\n",
      "     epoch 152, minibatch 195/195, test error of best model 21.534455 %\n",
      "('training @ iter = ', 29700)\n",
      "('training @ iter = ', 29800)\n",
      "epoch 153, minibatch 195/195, validation error 20.983574 %\n",
      "     epoch 153, minibatch 195/195, test error of best model 21.494391 %\n",
      "('training @ iter = ', 29900)\n",
      "('training @ iter = ', 30000)\n",
      "epoch 154, minibatch 195/195, validation error 20.983574 %\n",
      "('training @ iter = ', 30100)\n",
      "('training @ iter = ', 30200)\n",
      "epoch 155, minibatch 195/195, validation error 20.903446 %\n",
      "     epoch 155, minibatch 195/195, test error of best model 21.424279 %\n",
      "('training @ iter = ', 30300)\n",
      "('training @ iter = ', 30400)\n",
      "epoch 156, minibatch 195/195, validation error 20.833333 %\n",
      "     epoch 156, minibatch 195/195, test error of best model 21.394231 %\n",
      "('training @ iter = ', 30500)\n",
      "('training @ iter = ', 30600)\n",
      "epoch 157, minibatch 195/195, validation error 20.783253 %\n",
      "     epoch 157, minibatch 195/195, test error of best model 21.404247 %\n",
      "('training @ iter = ', 30700)\n",
      "('training @ iter = ', 30800)\n",
      "epoch 158, minibatch 195/195, validation error 20.713141 %\n",
      "     epoch 158, minibatch 195/195, test error of best model 21.404247 %\n",
      "('training @ iter = ', 30900)\n",
      "('training @ iter = ', 31000)\n",
      "epoch 159, minibatch 195/195, validation error 20.693109 %\n",
      "     epoch 159, minibatch 195/195, test error of best model 21.394231 %\n",
      "('training @ iter = ', 31100)\n",
      "epoch 160, minibatch 195/195, validation error 20.602965 %\n",
      "     epoch 160, minibatch 195/195, test error of best model 21.394231 %\n",
      "('training @ iter = ', 31200)\n",
      "('training @ iter = ', 31300)\n",
      "epoch 161, minibatch 195/195, validation error 20.592949 %\n",
      "     epoch 161, minibatch 195/195, test error of best model 21.414263 %\n",
      "('training @ iter = ', 31400)\n",
      "('training @ iter = ', 31500)\n",
      "epoch 162, minibatch 195/195, validation error 20.522837 %\n",
      "     epoch 162, minibatch 195/195, test error of best model 21.324119 %\n",
      "('training @ iter = ', 31600)\n",
      "('training @ iter = ', 31700)\n",
      "epoch 163, minibatch 195/195, validation error 20.492788 %\n",
      "     epoch 163, minibatch 195/195, test error of best model 21.284054 %\n",
      "('training @ iter = ', 31800)\n",
      "('training @ iter = ', 31900)\n",
      "epoch 164, minibatch 195/195, validation error 20.462740 %\n",
      "     epoch 164, minibatch 195/195, test error of best model 21.264022 %\n",
      "('training @ iter = ', 32000)\n",
      "('training @ iter = ', 32100)\n",
      "epoch 165, minibatch 195/195, validation error 20.412660 %\n",
      "     epoch 165, minibatch 195/195, test error of best model 21.274038 %\n",
      "('training @ iter = ', 32200)\n",
      "('training @ iter = ', 32300)\n",
      "epoch 166, minibatch 195/195, validation error 20.442708 %\n",
      "('training @ iter = ', 32400)\n",
      "('training @ iter = ', 32500)\n",
      "epoch 167, minibatch 195/195, validation error 20.442708 %\n",
      "('training @ iter = ', 32600)\n",
      "('training @ iter = ', 32700)\n",
      "epoch 168, minibatch 195/195, validation error 20.462740 %\n",
      "('training @ iter = ', 32800)\n",
      "('training @ iter = ', 32900)\n",
      "epoch 169, minibatch 195/195, validation error 20.422676 %\n",
      "('training @ iter = ', 33000)\n",
      "('training @ iter = ', 33100)\n",
      "epoch 170, minibatch 195/195, validation error 20.432692 %\n",
      "('training @ iter = ', 33200)\n",
      "('training @ iter = ', 33300)\n",
      "epoch 171, minibatch 195/195, validation error 20.372596 %\n",
      "     epoch 171, minibatch 195/195, test error of best model 21.314103 %\n",
      "('training @ iter = ', 33400)\n",
      "('training @ iter = ', 33500)\n",
      "epoch 172, minibatch 195/195, validation error 20.382612 %\n",
      "('training @ iter = ', 33600)\n",
      "('training @ iter = ', 33700)\n",
      "epoch 173, minibatch 195/195, validation error 20.322516 %\n",
      "     epoch 173, minibatch 195/195, test error of best model 21.183894 %\n",
      "('training @ iter = ', 33800)\n",
      "('training @ iter = ', 33900)\n",
      "epoch 174, minibatch 195/195, validation error 20.312500 %\n",
      "     epoch 174, minibatch 195/195, test error of best model 21.163862 %\n",
      "('training @ iter = ', 34000)\n",
      "('training @ iter = ', 34100)\n",
      "epoch 175, minibatch 195/195, validation error 20.302484 %\n",
      "     epoch 175, minibatch 195/195, test error of best model 21.153846 %\n",
      "('training @ iter = ', 34200)\n",
      "('training @ iter = ', 34300)\n",
      "epoch 176, minibatch 195/195, validation error 20.252404 %\n",
      "     epoch 176, minibatch 195/195, test error of best model 21.113782 %\n",
      "('training @ iter = ', 34400)\n",
      "('training @ iter = ', 34500)\n",
      "epoch 177, minibatch 195/195, validation error 20.272436 %\n",
      "('training @ iter = ', 34600)\n",
      "('training @ iter = ', 34700)\n",
      "epoch 178, minibatch 195/195, validation error 20.242388 %\n",
      "     epoch 178, minibatch 195/195, test error of best model 21.083734 %\n",
      "('training @ iter = ', 34800)\n",
      "('training @ iter = ', 34900)\n",
      "epoch 179, minibatch 195/195, validation error 20.212340 %\n",
      "     epoch 179, minibatch 195/195, test error of best model 21.073718 %\n",
      "('training @ iter = ', 35000)\n",
      "epoch 180, minibatch 195/195, validation error 20.152244 %\n",
      "     epoch 180, minibatch 195/195, test error of best model 21.023638 %\n",
      "('training @ iter = ', 35100)\n",
      "('training @ iter = ', 35200)\n",
      "epoch 181, minibatch 195/195, validation error 20.112179 %\n",
      "     epoch 181, minibatch 195/195, test error of best model 20.973558 %\n",
      "('training @ iter = ', 35300)\n",
      "('training @ iter = ', 35400)\n",
      "epoch 182, minibatch 195/195, validation error 20.102163 %\n",
      "     epoch 182, minibatch 195/195, test error of best model 20.963542 %\n",
      "('training @ iter = ', 35500)\n",
      "('training @ iter = ', 35600)\n",
      "epoch 183, minibatch 195/195, validation error 20.012019 %\n",
      "     epoch 183, minibatch 195/195, test error of best model 20.883413 %\n",
      "('training @ iter = ', 35700)\n",
      "('training @ iter = ', 35800)\n",
      "epoch 184, minibatch 195/195, validation error 20.012019 %\n",
      "('training @ iter = ', 35900)\n",
      "('training @ iter = ', 36000)\n",
      "epoch 185, minibatch 195/195, validation error 19.961939 %\n",
      "     epoch 185, minibatch 195/195, test error of best model 20.803285 %\n",
      "('training @ iter = ', 36100)\n",
      "('training @ iter = ', 36200)\n",
      "epoch 186, minibatch 195/195, validation error 19.971955 %\n",
      "('training @ iter = ', 36300)\n",
      "('training @ iter = ', 36400)\n",
      "epoch 187, minibatch 195/195, validation error 19.981971 %\n",
      "('training @ iter = ', 36500)\n",
      "('training @ iter = ', 36600)\n",
      "epoch 188, minibatch 195/195, validation error 19.951923 %\n",
      "     epoch 188, minibatch 195/195, test error of best model 20.823317 %\n",
      "('training @ iter = ', 36700)\n",
      "('training @ iter = ', 36800)\n",
      "epoch 189, minibatch 195/195, validation error 19.921875 %\n",
      "     epoch 189, minibatch 195/195, test error of best model 20.823317 %\n",
      "('training @ iter = ', 36900)\n",
      "('training @ iter = ', 37000)\n",
      "epoch 190, minibatch 195/195, validation error 19.851763 %\n",
      "     epoch 190, minibatch 195/195, test error of best model 20.793269 %\n",
      "('training @ iter = ', 37100)\n",
      "('training @ iter = ', 37200)\n",
      "epoch 191, minibatch 195/195, validation error 19.811699 %\n",
      "     epoch 191, minibatch 195/195, test error of best model 20.713141 %\n",
      "('training @ iter = ', 37300)\n",
      "('training @ iter = ', 37400)\n",
      "epoch 192, minibatch 195/195, validation error 19.831731 %\n",
      "('training @ iter = ', 37500)\n",
      "('training @ iter = ', 37600)\n",
      "epoch 193, minibatch 195/195, validation error 19.771635 %\n",
      "     epoch 193, minibatch 195/195, test error of best model 20.723157 %\n",
      "('training @ iter = ', 37700)\n",
      "('training @ iter = ', 37800)\n",
      "epoch 194, minibatch 195/195, validation error 19.751603 %\n",
      "     epoch 194, minibatch 195/195, test error of best model 20.713141 %\n",
      "('training @ iter = ', 37900)\n",
      "('training @ iter = ', 38000)\n",
      "epoch 195, minibatch 195/195, validation error 19.731571 %\n",
      "     epoch 195, minibatch 195/195, test error of best model 20.683093 %\n",
      "('training @ iter = ', 38100)\n",
      "('training @ iter = ', 38200)\n",
      "epoch 196, minibatch 195/195, validation error 19.701522 %\n",
      "     epoch 196, minibatch 195/195, test error of best model 20.673077 %\n",
      "('training @ iter = ', 38300)\n",
      "('training @ iter = ', 38400)\n",
      "epoch 197, minibatch 195/195, validation error 19.701522 %\n",
      "('training @ iter = ', 38500)\n",
      "('training @ iter = ', 38600)\n",
      "epoch 198, minibatch 195/195, validation error 19.671474 %\n",
      "     epoch 198, minibatch 195/195, test error of best model 20.582933 %\n",
      "('training @ iter = ', 38700)\n",
      "('training @ iter = ', 38800)\n",
      "epoch 199, minibatch 195/195, validation error 19.651442 %\n",
      "     epoch 199, minibatch 195/195, test error of best model 20.582933 %\n",
      "('training @ iter = ', 38900)\n",
      "epoch 200, minibatch 195/195, validation error 19.671474 %\n",
      "Optimization complete.\n",
      "Best validation score of 19.651442 % obtained at iteration 38805, with test performance 20.582933 %\n",
      "The training process for function run ran for 4.03m\n",
      "----------------------------------------------------------------------\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# FCN RTS\n",
    "run(network_type = 'fcn', include_st = False, transformation = rts_combo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "...loading the dataset\n",
      "...Applying rts_combo to dataset\n",
      "Current training data size is 50000\n",
      "Current validation data size is 10000\n",
      "Current test data size is 10000\n",
      "...building the model\n",
      "...Apply STN before CNN\n",
      "...Init localization to identity\n",
      "...training\n",
      "('training @ iter = ', 0)\n",
      "('training @ iter = ', 100)\n",
      "epoch 1, minibatch 195/195, validation error 90.074119 %\n",
      "     epoch 1, minibatch 195/195, test error of best model 90.194311 %\n",
      "('training @ iter = ', 200)\n",
      "('training @ iter = ', 300)\n",
      "epoch 2, minibatch 195/195, validation error 89.673478 %\n",
      "     epoch 2, minibatch 195/195, test error of best model 89.903846 %\n",
      "('training @ iter = ', 400)\n",
      "('training @ iter = ', 500)\n",
      "epoch 3, minibatch 195/195, validation error 88.701923 %\n",
      "     epoch 3, minibatch 195/195, test error of best model 89.002404 %\n",
      "('training @ iter = ', 600)\n",
      "('training @ iter = ', 700)\n",
      "epoch 4, minibatch 195/195, validation error 87.059295 %\n",
      "     epoch 4, minibatch 195/195, test error of best model 87.389824 %\n",
      "('training @ iter = ', 800)\n",
      "('training @ iter = ', 900)\n",
      "epoch 5, minibatch 195/195, validation error 84.795673 %\n",
      "     epoch 5, minibatch 195/195, test error of best model 85.536859 %\n",
      "('training @ iter = ', 1000)\n",
      "('training @ iter = ', 1100)\n",
      "epoch 6, minibatch 195/195, validation error 82.191506 %\n",
      "     epoch 6, minibatch 195/195, test error of best model 83.463542 %\n",
      "('training @ iter = ', 1200)\n",
      "('training @ iter = ', 1300)\n",
      "epoch 7, minibatch 195/195, validation error 79.807692 %\n",
      "     epoch 7, minibatch 195/195, test error of best model 81.029647 %\n",
      "('training @ iter = ', 1400)\n",
      "('training @ iter = ', 1500)\n",
      "epoch 8, minibatch 195/195, validation error 77.353766 %\n",
      "     epoch 8, minibatch 195/195, test error of best model 78.766026 %\n",
      "('training @ iter = ', 1600)\n",
      "('training @ iter = ', 1700)\n",
      "epoch 9, minibatch 195/195, validation error 74.989984 %\n",
      "     epoch 9, minibatch 195/195, test error of best model 75.951522 %\n",
      "('training @ iter = ', 1800)\n",
      "('training @ iter = ', 1900)\n",
      "epoch 10, minibatch 195/195, validation error 71.254006 %\n",
      "     epoch 10, minibatch 195/195, test error of best model 71.734776 %\n",
      "('training @ iter = ', 2000)\n",
      "('training @ iter = ', 2100)\n",
      "epoch 11, minibatch 195/195, validation error 63.331330 %\n",
      "     epoch 11, minibatch 195/195, test error of best model 64.102564 %\n",
      "('training @ iter = ', 2200)\n",
      "('training @ iter = ', 2300)\n",
      "epoch 12, minibatch 195/195, validation error 55.228365 %\n",
      "     epoch 12, minibatch 195/195, test error of best model 55.829327 %\n",
      "('training @ iter = ', 2400)\n",
      "('training @ iter = ', 2500)\n",
      "epoch 13, minibatch 195/195, validation error 47.546074 %\n",
      "     epoch 13, minibatch 195/195, test error of best model 47.435897 %\n",
      "('training @ iter = ', 2600)\n",
      "('training @ iter = ', 2700)\n",
      "epoch 14, minibatch 195/195, validation error 39.763622 %\n",
      "     epoch 14, minibatch 195/195, test error of best model 39.723558 %\n",
      "('training @ iter = ', 2800)\n",
      "('training @ iter = ', 2900)\n",
      "epoch 15, minibatch 195/195, validation error 32.471955 %\n",
      "     epoch 15, minibatch 195/195, test error of best model 33.153045 %\n",
      "('training @ iter = ', 3000)\n",
      "('training @ iter = ', 3100)\n",
      "epoch 16, minibatch 195/195, validation error 28.876202 %\n",
      "     epoch 16, minibatch 195/195, test error of best model 29.747596 %\n",
      "('training @ iter = ', 3200)\n",
      "('training @ iter = ', 3300)\n",
      "epoch 17, minibatch 195/195, validation error 25.140224 %\n",
      "     epoch 17, minibatch 195/195, test error of best model 25.661058 %\n",
      "('training @ iter = ', 3400)\n",
      "('training @ iter = ', 3500)\n",
      "epoch 18, minibatch 195/195, validation error 23.477564 %\n",
      "     epoch 18, minibatch 195/195, test error of best model 23.527644 %\n",
      "('training @ iter = ', 3600)\n",
      "('training @ iter = ', 3700)\n",
      "epoch 19, minibatch 195/195, validation error 21.985176 %\n",
      "     epoch 19, minibatch 195/195, test error of best model 22.536058 %\n",
      "('training @ iter = ', 3800)\n",
      "epoch 20, minibatch 195/195, validation error 21.294071 %\n",
      "     epoch 20, minibatch 195/195, test error of best model 21.594551 %\n",
      "('training @ iter = ', 3900)\n",
      "('training @ iter = ', 4000)\n",
      "epoch 21, minibatch 195/195, validation error 20.873397 %\n",
      "     epoch 21, minibatch 195/195, test error of best model 21.223958 %\n",
      "('training @ iter = ', 4100)\n",
      "('training @ iter = ', 4200)\n",
      "epoch 22, minibatch 195/195, validation error 21.774840 %\n",
      "('training @ iter = ', 4300)\n",
      "('training @ iter = ', 4400)\n",
      "epoch 23, minibatch 195/195, validation error 18.820112 %\n",
      "     epoch 23, minibatch 195/195, test error of best model 19.110577 %\n",
      "('training @ iter = ', 4500)\n",
      "('training @ iter = ', 4600)\n",
      "epoch 24, minibatch 195/195, validation error 19.160657 %\n",
      "('training @ iter = ', 4700)\n",
      "('training @ iter = ', 4800)\n",
      "epoch 25, minibatch 195/195, validation error 20.683093 %\n",
      "('training @ iter = ', 4900)\n",
      "('training @ iter = ', 5000)\n",
      "epoch 26, minibatch 195/195, validation error 19.471154 %\n",
      "('training @ iter = ', 5100)\n",
      "('training @ iter = ', 5200)\n",
      "epoch 27, minibatch 195/195, validation error 20.643029 %\n",
      "('training @ iter = ', 5300)\n",
      "('training @ iter = ', 5400)\n",
      "epoch 28, minibatch 195/195, validation error 24.348958 %\n",
      "('training @ iter = ', 5500)\n",
      "('training @ iter = ', 5600)\n",
      "epoch 29, minibatch 195/195, validation error 19.200721 %\n",
      "('training @ iter = ', 5700)\n",
      "('training @ iter = ', 5800)\n",
      "epoch 30, minibatch 195/195, validation error 19.961939 %\n",
      "('training @ iter = ', 5900)\n",
      "('training @ iter = ', 6000)\n",
      "epoch 31, minibatch 195/195, validation error 18.299279 %\n",
      "     epoch 31, minibatch 195/195, test error of best model 18.329327 %\n",
      "('training @ iter = ', 6100)\n",
      "('training @ iter = ', 6200)\n",
      "epoch 32, minibatch 195/195, validation error 21.173878 %\n",
      "('training @ iter = ', 6300)\n",
      "('training @ iter = ', 6400)\n",
      "epoch 33, minibatch 195/195, validation error 16.306090 %\n",
      "     epoch 33, minibatch 195/195, test error of best model 16.636619 %\n",
      "('training @ iter = ', 6500)\n",
      "('training @ iter = ', 6600)\n",
      "epoch 34, minibatch 195/195, validation error 17.928686 %\n",
      "('training @ iter = ', 6700)\n",
      "('training @ iter = ', 6800)\n",
      "epoch 35, minibatch 195/195, validation error 16.957131 %\n",
      "('training @ iter = ', 6900)\n",
      "('training @ iter = ', 7000)\n",
      "epoch 36, minibatch 195/195, validation error 13.241186 %\n",
      "     epoch 36, minibatch 195/195, test error of best model 13.661859 %\n",
      "('training @ iter = ', 7100)\n",
      "('training @ iter = ', 7200)\n",
      "epoch 37, minibatch 195/195, validation error 13.381410 %\n",
      "('training @ iter = ', 7300)\n",
      "('training @ iter = ', 7400)\n",
      "epoch 38, minibatch 195/195, validation error 12.730369 %\n",
      "     epoch 38, minibatch 195/195, test error of best model 12.970753 %\n",
      "('training @ iter = ', 7500)\n",
      "('training @ iter = ', 7600)\n",
      "epoch 39, minibatch 195/195, validation error 13.090946 %\n",
      "('training @ iter = ', 7700)\n",
      "epoch 40, minibatch 195/195, validation error 13.882212 %\n",
      "('training @ iter = ', 7800)\n",
      "('training @ iter = ', 7900)\n",
      "epoch 41, minibatch 195/195, validation error 12.950721 %\n",
      "('training @ iter = ', 8000)\n",
      "('training @ iter = ', 8100)\n",
      "epoch 42, minibatch 195/195, validation error 11.157853 %\n",
      "     epoch 42, minibatch 195/195, test error of best model 11.478365 %\n",
      "('training @ iter = ', 8200)\n",
      "('training @ iter = ', 8300)\n",
      "epoch 43, minibatch 195/195, validation error 11.818910 %\n",
      "('training @ iter = ', 8400)\n",
      "('training @ iter = ', 8500)\n",
      "epoch 44, minibatch 195/195, validation error 12.239583 %\n",
      "('training @ iter = ', 8600)\n",
      "('training @ iter = ', 8700)\n",
      "epoch 45, minibatch 195/195, validation error 11.738782 %\n",
      "('training @ iter = ', 8800)\n",
      "('training @ iter = ', 8900)\n",
      "epoch 46, minibatch 195/195, validation error 11.348157 %\n",
      "('training @ iter = ', 9000)\n",
      "('training @ iter = ', 9100)\n",
      "epoch 47, minibatch 195/195, validation error 13.431490 %\n",
      "('training @ iter = ', 9200)\n",
      "('training @ iter = ', 9300)\n",
      "epoch 48, minibatch 195/195, validation error 15.204327 %\n",
      "('training @ iter = ', 9400)\n",
      "('training @ iter = ', 9500)\n",
      "epoch 49, minibatch 195/195, validation error 12.469952 %\n",
      "('training @ iter = ', 9600)\n",
      "('training @ iter = ', 9700)\n",
      "epoch 50, minibatch 195/195, validation error 13.391426 %\n",
      "('training @ iter = ', 9800)\n",
      "('training @ iter = ', 9900)\n",
      "epoch 51, minibatch 195/195, validation error 14.022436 %\n",
      "('training @ iter = ', 10000)\n",
      "('training @ iter = ', 10100)\n",
      "epoch 52, minibatch 195/195, validation error 14.823718 %\n",
      "('training @ iter = ', 10200)\n",
      "('training @ iter = ', 10300)\n",
      "epoch 53, minibatch 195/195, validation error 14.042468 %\n",
      "('training @ iter = ', 10400)\n",
      "('training @ iter = ', 10500)\n",
      "epoch 54, minibatch 195/195, validation error 13.010817 %\n",
      "('training @ iter = ', 10600)\n",
      "('training @ iter = ', 10700)\n",
      "epoch 55, minibatch 195/195, validation error 15.655048 %\n",
      "('training @ iter = ', 10800)\n",
      "('training @ iter = ', 10900)\n",
      "epoch 56, minibatch 195/195, validation error 11.979167 %\n",
      "('training @ iter = ', 11000)\n",
      "('training @ iter = ', 11100)\n",
      "epoch 57, minibatch 195/195, validation error 13.551683 %\n",
      "('training @ iter = ', 11200)\n",
      "('training @ iter = ', 11300)\n",
      "epoch 58, minibatch 195/195, validation error 13.591747 %\n",
      "('training @ iter = ', 11400)\n",
      "('training @ iter = ', 11500)\n",
      "epoch 59, minibatch 195/195, validation error 13.561699 %\n",
      "('training @ iter = ', 11600)\n",
      "epoch 60, minibatch 195/195, validation error 13.822115 %\n",
      "('training @ iter = ', 11700)\n",
      "('training @ iter = ', 11800)\n",
      "epoch 61, minibatch 195/195, validation error 13.611779 %\n",
      "('training @ iter = ', 11900)\n",
      "('training @ iter = ', 12000)\n",
      "epoch 62, minibatch 195/195, validation error 9.375000 %\n",
      "     epoch 62, minibatch 195/195, test error of best model 9.745593 %\n",
      "('training @ iter = ', 12100)\n",
      "('training @ iter = ', 12200)\n",
      "epoch 63, minibatch 195/195, validation error 15.504808 %\n",
      "('training @ iter = ', 12300)\n",
      "('training @ iter = ', 12400)\n",
      "epoch 64, minibatch 195/195, validation error 10.987580 %\n",
      "('training @ iter = ', 12500)\n",
      "('training @ iter = ', 12600)\n",
      "epoch 65, minibatch 195/195, validation error 8.934295 %\n",
      "     epoch 65, minibatch 195/195, test error of best model 9.224760 %\n",
      "('training @ iter = ', 12700)\n",
      "('training @ iter = ', 12800)\n",
      "epoch 66, minibatch 195/195, validation error 10.216346 %\n",
      "('training @ iter = ', 12900)\n",
      "('training @ iter = ', 13000)\n",
      "epoch 67, minibatch 195/195, validation error 9.995994 %\n",
      "('training @ iter = ', 13100)\n",
      "('training @ iter = ', 13200)\n",
      "epoch 68, minibatch 195/195, validation error 10.486779 %\n",
      "('training @ iter = ', 13300)\n",
      "('training @ iter = ', 13400)\n",
      "epoch 69, minibatch 195/195, validation error 8.834135 %\n",
      "     epoch 69, minibatch 195/195, test error of best model 9.074519 %\n",
      "('training @ iter = ', 13500)\n",
      "('training @ iter = ', 13600)\n",
      "epoch 70, minibatch 195/195, validation error 8.133013 %\n",
      "     epoch 70, minibatch 195/195, test error of best model 8.533654 %\n",
      "('training @ iter = ', 13700)\n",
      "('training @ iter = ', 13800)\n",
      "epoch 71, minibatch 195/195, validation error 12.289663 %\n",
      "('training @ iter = ', 13900)\n",
      "('training @ iter = ', 14000)\n",
      "epoch 72, minibatch 195/195, validation error 9.765625 %\n",
      "('training @ iter = ', 14100)\n",
      "('training @ iter = ', 14200)\n",
      "epoch 73, minibatch 195/195, validation error 10.176282 %\n",
      "('training @ iter = ', 14300)\n",
      "('training @ iter = ', 14400)\n",
      "epoch 74, minibatch 195/195, validation error 8.193109 %\n",
      "('training @ iter = ', 14500)\n",
      "('training @ iter = ', 14600)\n",
      "epoch 75, minibatch 195/195, validation error 8.193109 %\n",
      "('training @ iter = ', 14700)\n",
      "('training @ iter = ', 14800)\n",
      "epoch 76, minibatch 195/195, validation error 8.683894 %\n",
      "('training @ iter = ', 14900)\n",
      "('training @ iter = ', 15000)\n",
      "epoch 77, minibatch 195/195, validation error 20.703125 %\n",
      "('training @ iter = ', 15100)\n",
      "('training @ iter = ', 15200)\n",
      "epoch 78, minibatch 195/195, validation error 8.082933 %\n",
      "     epoch 78, minibatch 195/195, test error of best model 8.143029 %\n",
      "('training @ iter = ', 15300)\n",
      "('training @ iter = ', 15400)\n",
      "epoch 79, minibatch 195/195, validation error 8.313301 %\n",
      "('training @ iter = ', 15500)\n",
      "epoch 80, minibatch 195/195, validation error 11.508413 %\n",
      "('training @ iter = ', 15600)\n",
      "('training @ iter = ', 15700)\n",
      "epoch 81, minibatch 195/195, validation error 7.912660 %\n",
      "     epoch 81, minibatch 195/195, test error of best model 8.213141 %\n",
      "('training @ iter = ', 15800)\n",
      "('training @ iter = ', 15900)\n",
      "epoch 82, minibatch 195/195, validation error 8.353365 %\n",
      "('training @ iter = ', 16000)\n",
      "('training @ iter = ', 16100)\n",
      "epoch 83, minibatch 195/195, validation error 33.343349 %\n",
      "('training @ iter = ', 16200)\n",
      "('training @ iter = ', 16300)\n",
      "epoch 84, minibatch 195/195, validation error 8.573718 %\n",
      "('training @ iter = ', 16400)\n",
      "('training @ iter = ', 16500)\n",
      "epoch 85, minibatch 195/195, validation error 10.867388 %\n",
      "('training @ iter = ', 16600)\n",
      "('training @ iter = ', 16700)\n",
      "epoch 86, minibatch 195/195, validation error 14.072516 %\n",
      "('training @ iter = ', 16800)\n",
      "('training @ iter = ', 16900)\n",
      "epoch 87, minibatch 195/195, validation error 7.592147 %\n",
      "     epoch 87, minibatch 195/195, test error of best model 7.962740 %\n",
      "('training @ iter = ', 17000)\n",
      "('training @ iter = ', 17100)\n",
      "epoch 88, minibatch 195/195, validation error 11.768830 %\n",
      "('training @ iter = ', 17200)\n",
      "('training @ iter = ', 17300)\n",
      "epoch 89, minibatch 195/195, validation error 8.433494 %\n",
      "('training @ iter = ', 17400)\n",
      "('training @ iter = ', 17500)\n",
      "epoch 90, minibatch 195/195, validation error 9.435096 %\n",
      "('training @ iter = ', 17600)\n",
      "('training @ iter = ', 17700)\n",
      "epoch 91, minibatch 195/195, validation error 10.536859 %\n",
      "('training @ iter = ', 17800)\n",
      "('training @ iter = ', 17900)\n",
      "epoch 92, minibatch 195/195, validation error 11.618590 %\n",
      "('training @ iter = ', 18000)\n",
      "('training @ iter = ', 18100)\n",
      "epoch 93, minibatch 195/195, validation error 7.291667 %\n",
      "     epoch 93, minibatch 195/195, test error of best model 7.692308 %\n",
      "('training @ iter = ', 18200)\n",
      "('training @ iter = ', 18300)\n",
      "epoch 94, minibatch 195/195, validation error 8.353365 %\n",
      "('training @ iter = ', 18400)\n",
      "('training @ iter = ', 18500)\n",
      "epoch 95, minibatch 195/195, validation error 8.173077 %\n",
      "('training @ iter = ', 18600)\n",
      "('training @ iter = ', 18700)\n",
      "epoch 96, minibatch 195/195, validation error 8.062901 %\n",
      "('training @ iter = ', 18800)\n",
      "('training @ iter = ', 18900)\n",
      "epoch 97, minibatch 195/195, validation error 12.489984 %\n",
      "('training @ iter = ', 19000)\n",
      "('training @ iter = ', 19100)\n",
      "epoch 98, minibatch 195/195, validation error 7.391827 %\n",
      "('training @ iter = ', 19200)\n",
      "('training @ iter = ', 19300)\n",
      "epoch 99, minibatch 195/195, validation error 8.844151 %\n",
      "('training @ iter = ', 19400)\n",
      "epoch 100, minibatch 195/195, validation error 10.516827 %\n",
      "('training @ iter = ', 19500)\n",
      "('training @ iter = ', 19600)\n",
      "epoch 101, minibatch 195/195, validation error 7.732372 %\n",
      "('training @ iter = ', 19700)\n",
      "('training @ iter = ', 19800)\n",
      "epoch 102, minibatch 195/195, validation error 9.655449 %\n",
      "('training @ iter = ', 19900)\n",
      "('training @ iter = ', 20000)\n",
      "epoch 103, minibatch 195/195, validation error 10.506811 %\n",
      "('training @ iter = ', 20100)\n",
      "('training @ iter = ', 20200)\n",
      "epoch 104, minibatch 195/195, validation error 8.493590 %\n",
      "('training @ iter = ', 20300)\n",
      "('training @ iter = ', 20400)\n",
      "epoch 105, minibatch 195/195, validation error 18.659856 %\n",
      "('training @ iter = ', 20500)\n",
      "('training @ iter = ', 20600)\n",
      "epoch 106, minibatch 195/195, validation error 7.221554 %\n",
      "     epoch 106, minibatch 195/195, test error of best model 7.351763 %\n",
      "('training @ iter = ', 20700)\n",
      "('training @ iter = ', 20800)\n",
      "epoch 107, minibatch 195/195, validation error 7.632212 %\n",
      "('training @ iter = ', 20900)\n",
      "('training @ iter = ', 21000)\n",
      "epoch 108, minibatch 195/195, validation error 7.231571 %\n",
      "('training @ iter = ', 21100)\n",
      "('training @ iter = ', 21200)\n",
      "epoch 109, minibatch 195/195, validation error 7.491987 %\n",
      "('training @ iter = ', 21300)\n",
      "('training @ iter = ', 21400)\n",
      "epoch 110, minibatch 195/195, validation error 8.092949 %\n",
      "('training @ iter = ', 21500)\n",
      "('training @ iter = ', 21600)\n",
      "epoch 111, minibatch 195/195, validation error 13.191106 %\n",
      "('training @ iter = ', 21700)\n",
      "('training @ iter = ', 21800)\n",
      "epoch 112, minibatch 195/195, validation error 7.431891 %\n",
      "('training @ iter = ', 21900)\n",
      "('training @ iter = ', 22000)\n",
      "epoch 113, minibatch 195/195, validation error 7.101362 %\n",
      "     epoch 113, minibatch 195/195, test error of best model 7.301683 %\n",
      "('training @ iter = ', 22100)\n",
      "('training @ iter = ', 22200)\n",
      "epoch 114, minibatch 195/195, validation error 8.293269 %\n",
      "('training @ iter = ', 22300)\n",
      "('training @ iter = ', 22400)\n",
      "epoch 115, minibatch 195/195, validation error 7.281651 %\n",
      "('training @ iter = ', 22500)\n",
      "('training @ iter = ', 22600)\n",
      "epoch 116, minibatch 195/195, validation error 6.650641 %\n",
      "     epoch 116, minibatch 195/195, test error of best model 7.171474 %\n",
      "('training @ iter = ', 22700)\n",
      "('training @ iter = ', 22800)\n",
      "epoch 117, minibatch 195/195, validation error 6.931090 %\n",
      "('training @ iter = ', 22900)\n",
      "('training @ iter = ', 23000)\n",
      "epoch 118, minibatch 195/195, validation error 7.181490 %\n",
      "('training @ iter = ', 23100)\n",
      "('training @ iter = ', 23200)\n",
      "epoch 119, minibatch 195/195, validation error 6.971154 %\n",
      "('training @ iter = ', 23300)\n",
      "epoch 120, minibatch 195/195, validation error 6.740785 %\n",
      "('training @ iter = ', 23400)\n",
      "('training @ iter = ', 23500)\n",
      "epoch 121, minibatch 195/195, validation error 7.361779 %\n",
      "('training @ iter = ', 23600)\n",
      "('training @ iter = ', 23700)\n",
      "epoch 122, minibatch 195/195, validation error 7.051282 %\n",
      "('training @ iter = ', 23800)\n",
      "('training @ iter = ', 23900)\n",
      "epoch 123, minibatch 195/195, validation error 22.776442 %\n",
      "('training @ iter = ', 24000)\n",
      "('training @ iter = ', 24100)\n",
      "epoch 124, minibatch 195/195, validation error 6.490385 %\n",
      "     epoch 124, minibatch 195/195, test error of best model 6.700721 %\n",
      "('training @ iter = ', 24200)\n",
      "('training @ iter = ', 24300)\n",
      "epoch 125, minibatch 195/195, validation error 7.121394 %\n",
      "('training @ iter = ', 24400)\n",
      "('training @ iter = ', 24500)\n",
      "epoch 126, minibatch 195/195, validation error 6.780849 %\n",
      "('training @ iter = ', 24600)\n",
      "('training @ iter = ', 24700)\n",
      "epoch 127, minibatch 195/195, validation error 7.101362 %\n",
      "('training @ iter = ', 24800)\n",
      "('training @ iter = ', 24900)\n",
      "epoch 128, minibatch 195/195, validation error 6.780849 %\n",
      "('training @ iter = ', 25000)\n",
      "('training @ iter = ', 25100)\n",
      "epoch 129, minibatch 195/195, validation error 6.680689 %\n",
      "('training @ iter = ', 25200)\n",
      "('training @ iter = ', 25300)\n",
      "epoch 130, minibatch 195/195, validation error 6.590545 %\n",
      "('training @ iter = ', 25400)\n",
      "('training @ iter = ', 25500)\n",
      "epoch 131, minibatch 195/195, validation error 6.870994 %\n",
      "('training @ iter = ', 25600)\n",
      "('training @ iter = ', 25700)\n",
      "epoch 132, minibatch 195/195, validation error 6.590545 %\n",
      "('training @ iter = ', 25800)\n",
      "('training @ iter = ', 25900)\n",
      "epoch 133, minibatch 195/195, validation error 7.231571 %\n",
      "('training @ iter = ', 26000)\n",
      "('training @ iter = ', 26100)\n",
      "epoch 134, minibatch 195/195, validation error 6.610577 %\n",
      "('training @ iter = ', 26200)\n",
      "('training @ iter = ', 26300)\n",
      "epoch 135, minibatch 195/195, validation error 6.680689 %\n",
      "('training @ iter = ', 26400)\n",
      "('training @ iter = ', 26500)\n",
      "epoch 136, minibatch 195/195, validation error 23.387420 %\n",
      "('training @ iter = ', 26600)\n",
      "('training @ iter = ', 26700)\n",
      "epoch 137, minibatch 195/195, validation error 7.732372 %\n",
      "('training @ iter = ', 26800)\n",
      "('training @ iter = ', 26900)\n",
      "epoch 138, minibatch 195/195, validation error 7.131410 %\n",
      "('training @ iter = ', 27000)\n",
      "('training @ iter = ', 27100)\n",
      "epoch 139, minibatch 195/195, validation error 6.790865 %\n",
      "('training @ iter = ', 27200)\n",
      "epoch 140, minibatch 195/195, validation error 6.760817 %\n",
      "('training @ iter = ', 27300)\n",
      "('training @ iter = ', 27400)\n",
      "epoch 141, minibatch 195/195, validation error 6.440304 %\n",
      "     epoch 141, minibatch 195/195, test error of best model 6.911058 %\n",
      "('training @ iter = ', 27500)\n",
      "('training @ iter = ', 27600)\n",
      "epoch 142, minibatch 195/195, validation error 6.270032 %\n",
      "     epoch 142, minibatch 195/195, test error of best model 6.901042 %\n",
      "('training @ iter = ', 27700)\n",
      "('training @ iter = ', 27800)\n",
      "epoch 143, minibatch 195/195, validation error 7.041266 %\n",
      "('training @ iter = ', 27900)\n",
      "('training @ iter = ', 28000)\n",
      "epoch 144, minibatch 195/195, validation error 7.712340 %\n",
      "('training @ iter = ', 28100)\n",
      "('training @ iter = ', 28200)\n",
      "epoch 145, minibatch 195/195, validation error 37.940705 %\n",
      "('training @ iter = ', 28300)\n",
      "('training @ iter = ', 28400)\n",
      "epoch 146, minibatch 195/195, validation error 50.090144 %\n",
      "('training @ iter = ', 28500)\n",
      "('training @ iter = ', 28600)\n",
      "epoch 147, minibatch 195/195, validation error 7.051282 %\n",
      "('training @ iter = ', 28700)\n",
      "('training @ iter = ', 28800)\n",
      "epoch 148, minibatch 195/195, validation error 7.011218 %\n",
      "('training @ iter = ', 28900)\n",
      "('training @ iter = ', 29000)\n",
      "epoch 149, minibatch 195/195, validation error 6.500401 %\n",
      "('training @ iter = ', 29100)\n",
      "('training @ iter = ', 29200)\n",
      "epoch 150, minibatch 195/195, validation error 6.600561 %\n",
      "('training @ iter = ', 29300)\n",
      "('training @ iter = ', 29400)\n",
      "epoch 151, minibatch 195/195, validation error 6.640625 %\n",
      "('training @ iter = ', 29500)\n",
      "('training @ iter = ', 29600)\n",
      "epoch 152, minibatch 195/195, validation error 6.750801 %\n",
      "('training @ iter = ', 29700)\n",
      "('training @ iter = ', 29800)\n",
      "epoch 153, minibatch 195/195, validation error 6.520433 %\n",
      "('training @ iter = ', 29900)\n",
      "('training @ iter = ', 30000)\n",
      "epoch 154, minibatch 195/195, validation error 74.629407 %\n",
      "('training @ iter = ', 30100)\n",
      "('training @ iter = ', 30200)\n",
      "epoch 155, minibatch 195/195, validation error 12.650240 %\n",
      "('training @ iter = ', 30300)\n",
      "('training @ iter = ', 30400)\n",
      "epoch 156, minibatch 195/195, validation error 6.951122 %\n",
      "('training @ iter = ', 30500)\n",
      "('training @ iter = ', 30600)\n",
      "epoch 157, minibatch 195/195, validation error 6.510417 %\n",
      "('training @ iter = ', 30700)\n",
      "('training @ iter = ', 30800)\n",
      "epoch 158, minibatch 195/195, validation error 7.301683 %\n",
      "('training @ iter = ', 30900)\n",
      "('training @ iter = ', 31000)\n",
      "epoch 159, minibatch 195/195, validation error 8.052885 %\n",
      "('training @ iter = ', 31100)\n",
      "epoch 160, minibatch 195/195, validation error 6.981170 %\n",
      "('training @ iter = ', 31200)\n",
      "('training @ iter = ', 31300)\n",
      "epoch 161, minibatch 195/195, validation error 6.490385 %\n",
      "('training @ iter = ', 31400)\n",
      "('training @ iter = ', 31500)\n",
      "epoch 162, minibatch 195/195, validation error 6.560497 %\n",
      "('training @ iter = ', 31600)\n",
      "('training @ iter = ', 31700)\n",
      "epoch 163, minibatch 195/195, validation error 6.570513 %\n",
      "('training @ iter = ', 31800)\n",
      "('training @ iter = ', 31900)\n",
      "epoch 164, minibatch 195/195, validation error 6.470353 %\n",
      "('training @ iter = ', 32000)\n",
      "('training @ iter = ', 32100)\n",
      "epoch 165, minibatch 195/195, validation error 6.820913 %\n",
      "('training @ iter = ', 32200)\n",
      "('training @ iter = ', 32300)\n",
      "epoch 166, minibatch 195/195, validation error 6.370192 %\n",
      "('training @ iter = ', 32400)\n",
      "('training @ iter = ', 32500)\n",
      "epoch 167, minibatch 195/195, validation error 7.071314 %\n",
      "('training @ iter = ', 32600)\n",
      "('training @ iter = ', 32700)\n",
      "epoch 168, minibatch 195/195, validation error 6.330128 %\n",
      "('training @ iter = ', 32800)\n",
      "('training @ iter = ', 32900)\n",
      "epoch 169, minibatch 195/195, validation error 6.540465 %\n",
      "('training @ iter = ', 33000)\n",
      "('training @ iter = ', 33100)\n",
      "epoch 170, minibatch 195/195, validation error 6.360176 %\n",
      "('training @ iter = ', 33200)\n",
      "('training @ iter = ', 33300)\n",
      "epoch 171, minibatch 195/195, validation error 6.550481 %\n",
      "('training @ iter = ', 33400)\n",
      "('training @ iter = ', 33500)\n",
      "epoch 172, minibatch 195/195, validation error 7.091346 %\n",
      "('training @ iter = ', 33600)\n",
      "('training @ iter = ', 33700)\n",
      "epoch 173, minibatch 195/195, validation error 6.510417 %\n",
      "('training @ iter = ', 33800)\n",
      "('training @ iter = ', 33900)\n",
      "epoch 174, minibatch 195/195, validation error 7.321715 %\n",
      "('training @ iter = ', 34000)\n",
      "('training @ iter = ', 34100)\n",
      "epoch 175, minibatch 195/195, validation error 6.490385 %\n",
      "('training @ iter = ', 34200)\n",
      "('training @ iter = ', 34300)\n",
      "epoch 176, minibatch 195/195, validation error 6.981170 %\n",
      "('training @ iter = ', 34400)\n",
      "('training @ iter = ', 34500)\n",
      "epoch 177, minibatch 195/195, validation error 35.797276 %\n",
      "('training @ iter = ', 34600)\n",
      "('training @ iter = ', 34700)\n",
      "epoch 178, minibatch 195/195, validation error 8.713942 %\n",
      "('training @ iter = ', 34800)\n",
      "('training @ iter = ', 34900)\n",
      "epoch 179, minibatch 195/195, validation error 6.951122 %\n",
      "('training @ iter = ', 35000)\n",
      "epoch 180, minibatch 195/195, validation error 6.330128 %\n",
      "('training @ iter = ', 35100)\n",
      "('training @ iter = ', 35200)\n",
      "epoch 181, minibatch 195/195, validation error 6.380208 %\n",
      "('training @ iter = ', 35300)\n",
      "('training @ iter = ', 35400)\n",
      "epoch 182, minibatch 195/195, validation error 6.610577 %\n",
      "('training @ iter = ', 35500)\n",
      "('training @ iter = ', 35600)\n",
      "epoch 183, minibatch 195/195, validation error 6.620593 %\n",
      "('training @ iter = ', 35700)\n",
      "('training @ iter = ', 35800)\n",
      "epoch 184, minibatch 195/195, validation error 8.633814 %\n",
      "('training @ iter = ', 35900)\n",
      "('training @ iter = ', 36000)\n",
      "epoch 185, minibatch 195/195, validation error 6.300080 %\n",
      "('training @ iter = ', 36100)\n",
      "('training @ iter = ', 36200)\n",
      "epoch 186, minibatch 195/195, validation error 6.380208 %\n",
      "('training @ iter = ', 36300)\n",
      "('training @ iter = ', 36400)\n",
      "epoch 187, minibatch 195/195, validation error 6.350160 %\n",
      "('training @ iter = ', 36500)\n",
      "('training @ iter = ', 36600)\n",
      "epoch 188, minibatch 195/195, validation error 6.640625 %\n",
      "('training @ iter = ', 36700)\n",
      "('training @ iter = ', 36800)\n",
      "epoch 189, minibatch 195/195, validation error 6.360176 %\n",
      "('training @ iter = ', 36900)\n",
      "('training @ iter = ', 37000)\n",
      "epoch 190, minibatch 195/195, validation error 6.330128 %\n",
      "('training @ iter = ', 37100)\n",
      "('training @ iter = ', 37200)\n",
      "epoch 191, minibatch 195/195, validation error 6.420272 %\n",
      "('training @ iter = ', 37300)\n",
      "('training @ iter = ', 37400)\n",
      "epoch 192, minibatch 195/195, validation error 6.440304 %\n",
      "('training @ iter = ', 37500)\n",
      "('training @ iter = ', 37600)\n",
      "epoch 193, minibatch 195/195, validation error 6.780849 %\n",
      "('training @ iter = ', 37700)\n",
      "('training @ iter = ', 37800)\n",
      "epoch 194, minibatch 195/195, validation error 6.530449 %\n",
      "('training @ iter = ', 37900)\n",
      "('training @ iter = ', 38000)\n",
      "epoch 195, minibatch 195/195, validation error 7.421875 %\n",
      "('training @ iter = ', 38100)\n",
      "('training @ iter = ', 38200)\n",
      "epoch 196, minibatch 195/195, validation error 6.330128 %\n",
      "('training @ iter = ', 38300)\n",
      "('training @ iter = ', 38400)\n",
      "epoch 197, minibatch 195/195, validation error 6.420272 %\n",
      "('training @ iter = ', 38500)\n",
      "('training @ iter = ', 38600)\n",
      "epoch 198, minibatch 195/195, validation error 6.330128 %\n",
      "('training @ iter = ', 38700)\n",
      "('training @ iter = ', 38800)\n",
      "epoch 199, minibatch 195/195, validation error 60.146234 %\n",
      "('training @ iter = ', 38900)\n",
      "epoch 200, minibatch 195/195, validation error 8.133013 %\n",
      "Optimization complete.\n",
      "Best validation score of 6.270032 % obtained at iteration 27690, with test performance 6.901042 %\n",
      "The training process for function run ran for 25.25m\n"
     ]
    }
   ],
   "source": [
    "# STN-FCN RTS\n",
    "run(network_type = 'fcn', include_st = True, transformation = rts_combo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# STN-CNN AND CNN RTS\n",
    "from dmnist import *\n",
    "run(network_type = 'cnn', include_st = True, transformation=rts_combo, runallmode=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Current validation data size is 10000\n",
    "Current test data size is 10000\n",
    "...building the model\n",
    "...training cnn with include_st = True\n",
    "...Apply STN before CNN\n",
    "Init localization to identity\n",
    "...training\n",
    "('training @ iter = ', 0)\n",
    "('training @ iter = ', 100)\n",
    "epoch 1, minibatch 195/195, validation error 88.534054 %\n",
    "     epoch 1, minibatch 195/195, test error of best model 88.574119 %\n",
    "('training @ iter = ', 200)\n",
    "('training @ iter = ', 300)\n",
    "epoch 2, minibatch 195/195, validation error 87.792869 %\n",
    "     epoch 2, minibatch 195/195, test error of best model 87.812901 %\n",
    "('training @ iter = ', 400)\n",
    "('training @ iter = ', 500)\n",
    "epoch 3, minibatch 195/195, validation error 82.484375 %\n",
    "     epoch 3, minibatch 195/195, test error of best model 82.464343 %\n",
    "('training @ iter = ', 600)\n",
    "('training @ iter = ', 700)\n",
    "epoch 4, minibatch 195/195, validation error 76.364583 %\n",
    "     epoch 4, minibatch 195/195, test error of best model 76.064103 %\n",
    "('training @ iter = ', 800)\n",
    "('training @ iter = ', 900)\n",
    "epoch 5, minibatch 195/195, validation error 72.358173 %\n",
    "     epoch 5, minibatch 195/195, test error of best model 72.458333 %\n",
    "('training @ iter = ', 1000)\n",
    "('training @ iter = ', 1100)\n",
    "epoch 6, minibatch 195/195, validation error 67.750801 %\n",
    "     epoch 6, minibatch 195/195, test error of best model 68.231571 %\n",
    "('training @ iter = ', 1200)\n",
    "('training @ iter = ', 1300)\n",
    "epoch 7, minibatch 195/195, validation error 45.364984 %\n",
    "     epoch 7, minibatch 195/195, test error of best model 46.026042 %\n",
    "('training @ iter = ', 1400)\n",
    "('training @ iter = ', 1500)\n",
    "epoch 8, minibatch 195/195, validation error 39.034856 %\n",
    "     epoch 8, minibatch 195/195, test error of best model 40.246795 %\n",
    "('training @ iter = ', 1600)\n",
    "('training @ iter = ', 1700)\n",
    "epoch 9, minibatch 195/195, validation error 34.187099 %\n",
    "     epoch 9, minibatch 195/195, test error of best model 35.829728 %\n",
    "('training @ iter = ', 1800)\n",
    "('training @ iter = ', 1900)\n",
    "epoch 10, minibatch 195/195, validation error 30.931891 %\n",
    "     epoch 10, minibatch 195/195, test error of best model 32.123798 %\n",
    "('training @ iter = ', 2000)\n",
    "('training @ iter = ', 2100)\n",
    "epoch 11, minibatch 195/195, validation error 29.078926 %\n",
    "     epoch 11, minibatch 195/195, test error of best model 30.961939 %\n",
    "('training @ iter = ', 2200)\n",
    "('training @ iter = ', 2300)\n",
    "epoch 12, minibatch 195/195, validation error 23.530048 %\n",
    "     epoch 12, minibatch 195/195, test error of best model 24.561699 %\n",
    "('training @ iter = ', 2400)\n",
    "('training @ iter = ', 2500)\n",
    "epoch 13, minibatch 195/195, validation error 25.122596 %\n",
    "('training @ iter = ', 2600)\n",
    "('training @ iter = ', 2700)\n",
    "epoch 14, minibatch 195/195, validation error 20.174679 %\n",
    "     epoch 14, minibatch 195/195, test error of best model 21.166266 %\n",
    "('training @ iter = ', 2800)\n",
    "('training @ iter = ', 2900)\n",
    "epoch 15, minibatch 195/195, validation error 18.712340 %\n",
    "     epoch 15, minibatch 195/195, test error of best model 19.924279 %\n",
    "('training @ iter = ', 3000)\n",
    "('training @ iter = ', 3100)\n",
    "epoch 16, minibatch 195/195, validation error 17.810897 %\n",
    "     epoch 16, minibatch 195/195, test error of best model 19.173077 %\n",
    "('training @ iter = ', 3200)\n",
    "('training @ iter = ', 3300)\n",
    "epoch 17, minibatch 195/195, validation error 18.972756 %\n",
    "('training @ iter = ', 3400)\n",
    "('training @ iter = ', 3500)\n",
    "epoch 18, minibatch 195/195, validation error 14.695913 %\n",
    "     epoch 18, minibatch 195/195, test error of best model 15.857772 %\n",
    "('training @ iter = ', 3600)\n",
    "('training @ iter = ', 3700)\n",
    "epoch 19, minibatch 195/195, validation error 20.004407 %\n",
    "('training @ iter = ', 3800)\n",
    "epoch 20, minibatch 195/195, validation error 18.812500 %\n",
    "('training @ iter = ', 3900)\n",
    "('training @ iter = ', 4000)\n",
    "epoch 21, minibatch 195/195, validation error 18.011218 %\n",
    "('training @ iter = ', 4100)\n",
    "('training @ iter = ', 4200)\n",
    "epoch 22, minibatch 195/195, validation error 19.112981 %\n",
    "('training @ iter = ', 4300)\n",
    "('training @ iter = ', 4400)\n",
    "epoch 23, minibatch 195/195, validation error 12.031651 %\n",
    "     epoch 23, minibatch 195/195, test error of best model 12.682692 %\n",
    "('training @ iter = ', 4500)\n",
    "('training @ iter = ', 4600)\n",
    "epoch 24, minibatch 195/195, validation error 14.495593 %\n",
    "('training @ iter = ', 4700)\n",
    "('training @ iter = ', 4800)\n",
    "epoch 25, minibatch 195/195, validation error 11.761218 %\n",
    "     epoch 25, minibatch 195/195, test error of best model 12.462340 %\n",
    "('training @ iter = ', 4900)\n",
    "('training @ iter = ', 5000)\n",
    "epoch 26, minibatch 195/195, validation error 9.878205 %\n",
    "     epoch 26, minibatch 195/195, test error of best model 10.228766 %\n",
    "('training @ iter = ', 5100)\n",
    "('training @ iter = ', 5200)\n",
    "epoch 27, minibatch 195/195, validation error 9.717949 %\n",
    "     epoch 27, minibatch 195/195, test error of best model 10.238782 %\n",
    "('training @ iter = ', 5300)\n",
    "('training @ iter = ', 5400)\n",
    "epoch 28, minibatch 195/195, validation error 9.247196 %\n",
    "     epoch 28, minibatch 195/195, test error of best model 9.948317 %\n",
    "('training @ iter = ', 5500)\n",
    "('training @ iter = ', 5600)\n",
    "epoch 29, minibatch 195/195, validation error 9.697917 %\n",
    "('training @ iter = ', 5700)\n",
    "('training @ iter = ', 5800)\n",
    "epoch 30, minibatch 195/195, validation error 9.026843 %\n",
    "     epoch 30, minibatch 195/195, test error of best model 10.018429 %\n",
    "('training @ iter = ', 5900)\n",
    "('training @ iter = ', 6000)\n",
    "epoch 31, minibatch 195/195, validation error 9.998397 %\n",
    "('training @ iter = ', 6100)\n",
    "('training @ iter = ', 6200)\n",
    "epoch 32, minibatch 195/195, validation error 9.106971 %\n",
    "('training @ iter = ', 6300)\n",
    "('training @ iter = ', 6400)\n",
    "epoch 33, minibatch 195/195, validation error 9.848157 %\n",
    "('training @ iter = ', 6500)\n",
    "('training @ iter = ', 6600)\n",
    "epoch 34, minibatch 195/195, validation error 9.647837 %\n",
    "('training @ iter = ', 6700)\n",
    "('training @ iter = ', 6800)\n",
    "epoch 35, minibatch 195/195, validation error 9.257212 %\n",
    "('training @ iter = ', 6900)\n",
    "('training @ iter = ', 7000)\n",
    "epoch 36, minibatch 195/195, validation error 9.918269 %\n",
    "('training @ iter = ', 7100)\n",
    "('training @ iter = ', 7200)\n",
    "epoch 37, minibatch 195/195, validation error 8.686298 %\n",
    "     epoch 37, minibatch 195/195, test error of best model 9.367388 %\n",
    "('training @ iter = ', 7300)\n",
    "('training @ iter = ', 7400)\n",
    "epoch 38, minibatch 195/195, validation error 7.895032 %\n",
    "     epoch 38, minibatch 195/195, test error of best model 8.546074 %\n",
    "('training @ iter = ', 7500)\n",
    "('training @ iter = ', 7600)\n",
    "epoch 39, minibatch 195/195, validation error 9.647837 %\n",
    "('training @ iter = ', 7700)\n",
    "epoch 40, minibatch 195/195, validation error 8.746394 %\n",
    "('training @ iter = ', 7800)\n",
    "('training @ iter = ', 7900)\n",
    "epoch 41, minibatch 195/195, validation error 10.268830 %\n",
    "('training @ iter = ', 8000)\n",
    "('training @ iter = ', 8100)\n",
    "epoch 42, minibatch 195/195, validation error 7.644631 %\n",
    "     epoch 42, minibatch 195/195, test error of best model 8.025240 %\n",
    "('training @ iter = ', 8200)\n",
    "('training @ iter = ', 8300)\n",
    "epoch 43, minibatch 195/195, validation error 7.334135 %\n",
    "     epoch 43, minibatch 195/195, test error of best model 7.834936 %\n",
    "('training @ iter = ', 8400)\n",
    "('training @ iter = ', 8500)\n",
    "epoch 44, minibatch 195/195, validation error 8.886619 %\n",
    "('training @ iter = ', 8600)\n",
    "('training @ iter = ', 8700)\n",
    "epoch 45, minibatch 195/195, validation error 14.335337 %\n",
    "('training @ iter = ', 8800)\n",
    "('training @ iter = ', 8900)\n",
    "epoch 46, minibatch 195/195, validation error 9.147035 %\n",
    "('training @ iter = ', 9000)\n",
    "('training @ iter = ', 9100)\n",
    "epoch 47, minibatch 195/195, validation error 9.547676 %\n",
    "('training @ iter = ', 9200)\n",
    "('training @ iter = ', 9300)\n",
    "epoch 48, minibatch 195/195, validation error 8.886619 %\n",
    "('training @ iter = ', 9400)\n",
    "('training @ iter = ', 9500)\n",
    "epoch 49, minibatch 195/195, validation error 10.108574 %\n",
    "('training @ iter = ', 9600)\n",
    "('training @ iter = ', 9700)\n",
    "epoch 50, minibatch 195/195, validation error 6.312500 %\n",
    "     epoch 50, minibatch 195/195, test error of best model 7.163862 %\n",
    "('training @ iter = ', 9800)\n",
    "('training @ iter = ', 9900)\n",
    "epoch 51, minibatch 195/195, validation error 6.302484 %\n",
    "     epoch 51, minibatch 195/195, test error of best model 7.163862 %\n",
    "('training @ iter = ', 10000)\n",
    "('training @ iter = ', 10100)\n",
    "epoch 52, minibatch 195/195, validation error 6.953526 %\n",
    "('training @ iter = ', 10200)\n",
    "('training @ iter = ', 10300)\n",
    "epoch 53, minibatch 195/195, validation error 6.152244 %\n",
    "     epoch 53, minibatch 195/195, test error of best model 6.833333 %\n",
    "('training @ iter = ', 10400)\n",
    "('training @ iter = ', 10500)\n",
    "epoch 54, minibatch 195/195, validation error 7.824920 %\n",
    "('training @ iter = ', 10600)\n",
    "('training @ iter = ', 10700)\n",
    "epoch 55, minibatch 195/195, validation error 6.142228 %\n",
    "     epoch 55, minibatch 195/195, test error of best model 7.003606 %\n",
    "('training @ iter = ', 10800)\n",
    "('training @ iter = ', 10900)\n",
    "epoch 56, minibatch 195/195, validation error 6.432692 %\n",
    "('training @ iter = ', 11000)\n",
    "('training @ iter = ', 11100)\n",
    "epoch 57, minibatch 195/195, validation error 6.813301 %\n",
    "('training @ iter = ', 11200)\n",
    "('training @ iter = ', 11300)\n",
    "epoch 58, minibatch 195/195, validation error 6.262420 %\n",
    "('training @ iter = ', 11400)\n",
    "('training @ iter = ', 11500)\n",
    "epoch 59, minibatch 195/195, validation error 6.272436 %\n",
    "('training @ iter = ', 11600)\n",
    "epoch 60, minibatch 195/195, validation error 6.142228 %\n",
    "('training @ iter = ', 11700)\n",
    "('training @ iter = ', 11800)\n",
    "epoch 61, minibatch 195/195, validation error 5.531250 %\n",
    "     epoch 61, minibatch 195/195, test error of best model 6.633013 %\n",
    "('training @ iter = ', 11900)\n",
    "('training @ iter = ', 12000)\n",
    "epoch 62, minibatch 195/195, validation error 5.911859 %\n",
    "('training @ iter = ', 12100)\n",
    "('training @ iter = ', 12200)\n",
    "epoch 63, minibatch 195/195, validation error 7.274038 %\n",
    "('training @ iter = ', 12300)\n",
    "('training @ iter = ', 12400)\n",
    "epoch 64, minibatch 195/195, validation error 6.362580 %\n",
    "('training @ iter = ', 12500)\n",
    "('training @ iter = ', 12600)\n",
    "epoch 65, minibatch 195/195, validation error 6.002003 %\n",
    "('training @ iter = ', 12700)\n",
    "('training @ iter = ', 12800)\n",
    "epoch 66, minibatch 195/195, validation error 6.102163 %\n",
    "('training @ iter = ', 12900)\n",
    "('training @ iter = ', 13000)\n",
    "epoch 67, minibatch 195/195, validation error 5.881811 %\n",
    "('training @ iter = ', 13100)\n",
    "('training @ iter = ', 13200)\n",
    "epoch 68, minibatch 195/195, validation error 6.372596 %\n",
    "('training @ iter = ', 13300)\n",
    "('training @ iter = ', 13400)\n",
    "epoch 69, minibatch 195/195, validation error 5.821715 %\n",
    "('training @ iter = ', 13500)\n",
    "('training @ iter = ', 13600)\n",
    "epoch 70, minibatch 195/195, validation error 5.391026 %\n",
    "     epoch 70, minibatch 195/195, test error of best model 6.322516 %\n",
    "('training @ iter = ', 13700)\n",
    "('training @ iter = ', 13800)\n",
    "epoch 71, minibatch 195/195, validation error 5.441106 %\n",
    "('training @ iter = ', 13900)\n",
    "('training @ iter = ', 14000)\n",
    "epoch 72, minibatch 195/195, validation error 5.511218 %\n",
    "('training @ iter = ', 14100)\n",
    "('training @ iter = ', 14200)\n",
    "epoch 73, minibatch 195/195, validation error 5.761619 %\n",
    "('training @ iter = ', 14300)\n",
    "('training @ iter = ', 14400)\n",
    "epoch 74, minibatch 195/195, validation error 6.472756 %\n",
    "('training @ iter = ', 14500)\n",
    "('training @ iter = ', 14600)\n",
    "epoch 75, minibatch 195/195, validation error 6.082131 %\n",
    "('training @ iter = ', 14700)\n",
    "('training @ iter = ', 14800)\n",
    "epoch 76, minibatch 195/195, validation error 5.801683 %\n",
    "('training @ iter = ', 14900)\n",
    "('training @ iter = ', 15000)\n",
    "epoch 77, minibatch 195/195, validation error 5.280849 %\n",
    "     epoch 77, minibatch 195/195, test error of best model 5.931891 %\n",
    "('training @ iter = ', 15100)\n",
    "('training @ iter = ', 15200)\n",
    "epoch 78, minibatch 195/195, validation error 6.402644 %\n",
    "('training @ iter = ', 15300)\n",
    "('training @ iter = ', 15400)\n",
    "epoch 79, minibatch 195/195, validation error 6.502804 %\n",
    "('training @ iter = ', 15500)\n",
    "epoch 80, minibatch 195/195, validation error 6.482772 %\n",
    "('training @ iter = ', 15600)\n",
    "('training @ iter = ', 15700)\n",
    "epoch 81, minibatch 195/195, validation error 7.364183 %\n",
    "('training @ iter = ', 15800)\n",
    "('training @ iter = ', 15900)\n",
    "epoch 82, minibatch 195/195, validation error 6.052083 %\n",
    "('training @ iter = ', 16000)\n",
    "('training @ iter = ', 16100)\n",
    "epoch 83, minibatch 195/195, validation error 5.310897 %\n",
    "('training @ iter = ', 16200)\n",
    "('training @ iter = ', 16300)\n",
    "epoch 84, minibatch 195/195, validation error 5.531250 %\n",
    "('training @ iter = ', 16400)\n",
    "('training @ iter = ', 16500)\n",
    "epoch 85, minibatch 195/195, validation error 7.434295 %\n",
    "('training @ iter = ', 16600)\n",
    "('training @ iter = ', 16700)\n",
    "epoch 86, minibatch 195/195, validation error 6.022035 %\n",
    "('training @ iter = ', 16800)\n",
    "('training @ iter = ', 16900)\n",
    "epoch 87, minibatch 195/195, validation error 5.360978 %\n",
    "('training @ iter = ', 17000)\n",
    "('training @ iter = ', 17100)\n",
    "epoch 88, minibatch 195/195, validation error 5.941907 %\n",
    "('training @ iter = ', 17200)\n",
    "('training @ iter = ', 17300)\n",
    "epoch 89, minibatch 195/195, validation error 4.860176 %\n",
    "     epoch 89, minibatch 195/195, test error of best model 5.671474 %\n",
    "('training @ iter = ', 17400)\n",
    "('training @ iter = ', 17500)\n",
    "epoch 90, minibatch 195/195, validation error 7.043670 %\n",
    "('training @ iter = ', 17600)\n",
    "('training @ iter = ', 17700)\n",
    "epoch 91, minibatch 195/195, validation error 5.050481 %\n",
    "('training @ iter = ', 17800)\n",
    "('training @ iter = ', 17900)\n",
    "epoch 92, minibatch 195/195, validation error 5.200721 %\n",
    "('training @ iter = ', 18000)\n",
    "('training @ iter = ', 18100)\n",
    "epoch 93, minibatch 195/195, validation error 5.230769 %\n",
    "('training @ iter = ', 18200)\n",
    "('training @ iter = ', 18300)\n",
    "epoch 94, minibatch 195/195, validation error 7.514423 %\n",
    "('training @ iter = ', 18400)\n",
    "('training @ iter = ', 18500)\n",
    "epoch 95, minibatch 195/195, validation error 4.970353 %\n",
    "('training @ iter = ', 18600)\n",
    "('training @ iter = ', 18700)\n",
    "epoch 96, minibatch 195/195, validation error 4.950321 %\n",
    "('training @ iter = ', 18800)\n",
    "('training @ iter = ', 18900)\n",
    "epoch 97, minibatch 195/195, validation error 4.830128 %\n",
    "     epoch 97, minibatch 195/195, test error of best model 5.901843 %\n",
    "('training @ iter = ', 19000)\n",
    "('training @ iter = ', 19100)\n",
    "epoch 98, minibatch 195/195, validation error 4.990385 %\n",
    "('training @ iter = ', 19200)\n",
    "('training @ iter = ', 19300)\n",
    "epoch 99, minibatch 195/195, validation error 4.739984 %\n",
    "     epoch 99, minibatch 195/195, test error of best model 5.661458 %\n",
    "('training @ iter = ', 19400)\n",
    "epoch 100, minibatch 195/195, validation error 5.100561 %\n",
    "('training @ iter = ', 19500)\n",
    "('training @ iter = ', 19600)\n",
    "epoch 101, minibatch 195/195, validation error 4.729968 %\n",
    "     epoch 101, minibatch 195/195, test error of best model 5.691506 %\n",
    "('training @ iter = ', 19700)\n",
    "('training @ iter = ', 19800)\n",
    "epoch 102, minibatch 195/195, validation error 5.040465 %\n",
    "('training @ iter = ', 19900)\n",
    "('training @ iter = ', 20000)\n",
    "epoch 103, minibatch 195/195, validation error 4.369391 %\n",
    "     epoch 103, minibatch 195/195, test error of best model 5.571314 %\n",
    "('training @ iter = ', 20100)\n",
    "('training @ iter = ', 20200)\n",
    "epoch 104, minibatch 195/195, validation error 4.619792 %\n",
    "('training @ iter = ', 20300)\n",
    "('training @ iter = ', 20400)\n",
    "epoch 105, minibatch 195/195, validation error 5.651442 %\n",
    "('training @ iter = ', 20500)\n",
    "('training @ iter = ', 20600)\n",
    "epoch 106, minibatch 195/195, validation error 5.120593 %\n",
    "('training @ iter = ', 20700)\n",
    "('training @ iter = ', 20800)\n",
    "epoch 107, minibatch 195/195, validation error 4.569712 %\n",
    "('training @ iter = ', 20900)\n",
    "('training @ iter = ', 21000)\n",
    "epoch 108, minibatch 195/195, validation error 4.669872 %\n",
    "('training @ iter = ', 21100)\n",
    "('training @ iter = ', 21200)\n",
    "epoch 109, minibatch 195/195, validation error 4.639824 %\n",
    "('training @ iter = ', 21300)\n",
    "('training @ iter = ', 21400)\n",
    "epoch 110, minibatch 195/195, validation error 4.369391 %\n",
    "('training @ iter = ', 21500)\n",
    "('training @ iter = ', 21600)\n",
    "epoch 111, minibatch 195/195, validation error 4.509615 %\n",
    "('training @ iter = ', 21700)\n",
    "('training @ iter = ', 21800)\n",
    "epoch 112, minibatch 195/195, validation error 4.599760 %\n",
    "('training @ iter = ', 21900)\n",
    "('training @ iter = ', 22000)\n",
    "epoch 113, minibatch 195/195, validation error 4.840144 %\n",
    "('training @ iter = ', 22100)\n",
    "('training @ iter = ', 22200)\n",
    "epoch 114, minibatch 195/195, validation error 4.599760 %\n",
    "('training @ iter = ', 22300)\n",
    "('training @ iter = ', 22400)\n",
    "epoch 115, minibatch 195/195, validation error 4.359375 %\n",
    "     epoch 115, minibatch 195/195, test error of best model 5.170673 %\n",
    "('training @ iter = ', 22500)\n",
    "('training @ iter = ', 22600)\n",
    "epoch 116, minibatch 195/195, validation error 4.459535 %\n",
    "('training @ iter = ', 22700)\n",
    "('training @ iter = ', 22800)\n",
    "epoch 117, minibatch 195/195, validation error 4.389423 %\n",
    "('training @ iter = ', 22900)\n",
    "('training @ iter = ', 23000)\n",
    "epoch 118, minibatch 195/195, validation error 4.509615 %\n",
    "('training @ iter = ', 23100)\n",
    "('training @ iter = ', 23200)\n",
    "epoch 119, minibatch 195/195, validation error 4.639824 %\n",
    "('training @ iter = ', 23300)\n",
    "epoch 120, minibatch 195/195, validation error 4.339343 %\n",
    "     epoch 120, minibatch 195/195, test error of best model 5.421074 %\n",
    "('training @ iter = ', 23400)\n",
    "('training @ iter = ', 23500)\n",
    "epoch 121, minibatch 195/195, validation error 4.279247 %\n",
    "     epoch 121, minibatch 195/195, test error of best model 5.571314 %\n",
    "('training @ iter = ', 23600)\n",
    "('training @ iter = ', 23700)\n",
    "epoch 122, minibatch 195/195, validation error 4.719952 %\n",
    "('training @ iter = ', 23800)\n",
    "('training @ iter = ', 23900)\n",
    "epoch 123, minibatch 195/195, validation error 4.539663 %\n",
    "('training @ iter = ', 24000)\n",
    "('training @ iter = ', 24100)\n",
    "epoch 124, minibatch 195/195, validation error 4.559696 %\n",
    "('training @ iter = ', 24200)\n",
    "('training @ iter = ', 24300)\n",
    "epoch 125, minibatch 195/195, validation error 4.589744 %\n",
    "('training @ iter = ', 24400)\n",
    "('training @ iter = ', 24500)\n",
    "epoch 126, minibatch 195/195, validation error 4.750000 %\n",
    "('training @ iter = ', 24600)\n",
    "('training @ iter = ', 24700)\n",
    "epoch 127, minibatch 195/195, validation error 5.080529 %\n",
    "('training @ iter = ', 24800)\n",
    "('training @ iter = ', 24900)\n",
    "epoch 128, minibatch 195/195, validation error 4.529647 %\n",
    "('training @ iter = ', 25000)\n",
    "('training @ iter = ', 25100)\n",
    "epoch 129, minibatch 195/195, validation error 4.920272 %\n",
    "('training @ iter = ', 25200)\n",
    "('training @ iter = ', 25300)\n",
    "epoch 130, minibatch 195/195, validation error 4.669872 %\n",
    "('training @ iter = ', 25400)\n",
    "('training @ iter = ', 25500)\n",
    "epoch 131, minibatch 195/195, validation error 5.020433 %\n",
    "('training @ iter = ', 25600)\n",
    "('training @ iter = ', 25700)\n",
    "epoch 132, minibatch 195/195, validation error 4.559696 %\n",
    "('training @ iter = ', 25800)\n",
    "('training @ iter = ', 25900)\n",
    "epoch 133, minibatch 195/195, validation error 4.469551 %\n",
    "('training @ iter = ', 26000)\n",
    "('training @ iter = ', 26100)\n",
    "epoch 134, minibatch 195/195, validation error 4.489583 %\n",
    "('training @ iter = ', 26200)\n",
    "('training @ iter = ', 26300)\n",
    "epoch 135, minibatch 195/195, validation error 5.060497 %\n",
    "('training @ iter = ', 26400)\n",
    "('training @ iter = ', 26500)\n",
    "epoch 136, minibatch 195/195, validation error 4.489583 %\n",
    "('training @ iter = ', 26600)\n",
    "('training @ iter = ', 26700)\n",
    "epoch 137, minibatch 195/195, validation error 4.469551 %\n",
    "('training @ iter = ', 26800)\n",
    "('training @ iter = ', 26900)\n",
    "epoch 138, minibatch 195/195, validation error 4.629808 %\n",
    "('training @ iter = ', 27000)\n",
    "('training @ iter = ', 27100)\n",
    "epoch 139, minibatch 195/195, validation error 4.770032 %\n",
    "('training @ iter = ', 27200)\n",
    "epoch 140, minibatch 195/195, validation error 4.509615 %\n",
    "('training @ iter = ', 27300)\n",
    "('training @ iter = ', 27400)\n",
    "epoch 141, minibatch 195/195, validation error 5.200721 %\n",
    "('training @ iter = ', 27500)\n",
    "('training @ iter = ', 27600)\n",
    "epoch 142, minibatch 195/195, validation error 4.499599 %\n",
    "('training @ iter = ', 27700)\n",
    "('training @ iter = ', 27800)\n",
    "epoch 143, minibatch 195/195, validation error 4.409455 %\n",
    "('training @ iter = ', 27900)\n",
    "('training @ iter = ', 28000)\n",
    "epoch 144, minibatch 195/195, validation error 4.770032 %\n",
    "('training @ iter = ', 28100)\n",
    "('training @ iter = ', 28200)\n",
    "epoch 145, minibatch 195/195, validation error 4.619792 %\n",
    "('training @ iter = ', 28300)\n",
    "('training @ iter = ', 28400)\n",
    "epoch 146, minibatch 195/195, validation error 4.559696 %\n",
    "('training @ iter = ', 28500)\n",
    "('training @ iter = ', 28600)\n",
    "epoch 147, minibatch 195/195, validation error 5.381010 %\n",
    "('training @ iter = ', 28700)\n",
    "('training @ iter = ', 28800)\n",
    "epoch 148, minibatch 195/195, validation error 4.709936 %\n",
    "('training @ iter = ', 28900)\n",
    "('training @ iter = ', 29000)\n",
    "epoch 149, minibatch 195/195, validation error 4.900240 %\n",
    "('training @ iter = ', 29100)\n",
    "('training @ iter = ', 29200)\n",
    "epoch 150, minibatch 195/195, validation error 4.709936 %\n",
    "('training @ iter = ', 29300)\n",
    "('training @ iter = ', 29400)\n",
    "epoch 151, minibatch 195/195, validation error 4.689904 %\n",
    "('training @ iter = ', 29500)\n",
    "('training @ iter = ', 29600)\n",
    "epoch 152, minibatch 195/195, validation error 4.649840 %\n",
    "('training @ iter = ', 29700)\n",
    "('training @ iter = ', 29800)\n",
    "epoch 153, minibatch 195/195, validation error 4.589744 %\n",
    "('training @ iter = ', 29900)\n",
    "('training @ iter = ', 30000)\n",
    "epoch 154, minibatch 195/195, validation error 4.930288 %\n",
    "('training @ iter = ', 30100)\n",
    "('training @ iter = ', 30200)\n",
    "epoch 155, minibatch 195/195, validation error 4.850160 %\n",
    "('training @ iter = ', 30300)\n",
    "('training @ iter = ', 30400)\n",
    "epoch 156, minibatch 195/195, validation error 4.599760 %\n",
    "('training @ iter = ', 30500)\n",
    "('training @ iter = ', 30600)\n",
    "epoch 157, minibatch 195/195, validation error 4.549679 %\n",
    "('training @ iter = ', 30700)\n",
    "('training @ iter = ', 30800)\n",
    "epoch 158, minibatch 195/195, validation error 5.050481 %\n",
    "('training @ iter = ', 30900)\n",
    "('training @ iter = ', 31000)\n",
    "epoch 159, minibatch 195/195, validation error 4.319311 %\n",
    "('training @ iter = ', 31100)\n",
    "epoch 160, minibatch 195/195, validation error 4.920272 %\n",
    "('training @ iter = ', 31200)\n",
    "('training @ iter = ', 31300)\n",
    "epoch 161, minibatch 195/195, validation error 5.250801 %\n",
    "('training @ iter = ', 31400)\n",
    "('training @ iter = ', 31500)\n",
    "epoch 162, minibatch 195/195, validation error 4.679888 %\n",
    "('training @ iter = ', 31600)\n",
    "('training @ iter = ', 31700)\n",
    "epoch 163, minibatch 195/195, validation error 4.299279 %\n",
    "('training @ iter = ', 31800)\n",
    "('training @ iter = ', 31900)\n",
    "epoch 164, minibatch 195/195, validation error 4.559696 %\n",
    "('training @ iter = ', 32000)\n",
    "('training @ iter = ', 32100)\n",
    "epoch 165, minibatch 195/195, validation error 4.599760 %\n",
    "('training @ iter = ', 32200)\n",
    "('training @ iter = ', 32300)\n",
    "epoch 166, minibatch 195/195, validation error 4.709936 %\n",
    "('training @ iter = ', 32400)\n",
    "('training @ iter = ', 32500)\n",
    "epoch 167, minibatch 195/195, validation error 5.521234 %\n",
    "('training @ iter = ', 32600)\n",
    "('training @ iter = ', 32700)\n",
    "epoch 168, minibatch 195/195, validation error 5.210737 %\n",
    "('training @ iter = ', 32800)\n",
    "('training @ iter = ', 32900)\n",
    "epoch 169, minibatch 195/195, validation error 4.880208 %\n",
    "('training @ iter = ', 33000)\n",
    "('training @ iter = ', 33100)\n",
    "epoch 170, minibatch 195/195, validation error 7.133814 %\n",
    "('training @ iter = ', 33200)\n",
    "('training @ iter = ', 33300)\n",
    "epoch 171, minibatch 195/195, validation error 6.182292 %\n",
    "('training @ iter = ', 33400)\n",
    "('training @ iter = ', 33500)\n",
    "epoch 172, minibatch 195/195, validation error 4.669872 %\n",
    "('training @ iter = ', 33600)\n",
    "('training @ iter = ', 33700)\n",
    "epoch 173, minibatch 195/195, validation error 5.421074 %\n",
    "('training @ iter = ', 33800)\n",
    "('training @ iter = ', 33900)\n",
    "epoch 174, minibatch 195/195, validation error 5.481170 %\n",
    "('training @ iter = ', 34000)\n",
    "('training @ iter = ', 34100)\n",
    "epoch 175, minibatch 195/195, validation error 5.260817 %\n",
    "('training @ iter = ', 34200)\n",
    "('training @ iter = ', 34300)\n",
    "epoch 176, minibatch 195/195, validation error 4.169071 %\n",
    "     epoch 176, minibatch 195/195, test error of best model 4.830128 %\n",
    "('training @ iter = ', 34400)\n",
    "('training @ iter = ', 34500)\n",
    "epoch 177, minibatch 195/195, validation error 4.750000 %\n",
    "('training @ iter = ', 34600)\n",
    "('training @ iter = ', 34700)\n",
    "epoch 178, minibatch 195/195, validation error 4.289263 %\n",
    "('training @ iter = ', 34800)\n",
    "('training @ iter = ', 34900)\n",
    "epoch 179, minibatch 195/195, validation error 5.280849 %\n",
    "('training @ iter = ', 35000)\n",
    "epoch 180, minibatch 195/195, validation error 4.579728 %\n",
    "('training @ iter = ', 35100)\n",
    "('training @ iter = ', 35200)\n",
    "epoch 181, minibatch 195/195, validation error 5.100561 %\n",
    "('training @ iter = ', 35300)\n",
    "('training @ iter = ', 35400)\n",
    "epoch 182, minibatch 195/195, validation error 4.880208 %\n",
    "('training @ iter = ', 35500)\n",
    "('training @ iter = ', 35600)\n",
    "epoch 183, minibatch 195/195, validation error 4.790064 %\n",
    "('training @ iter = ', 35700)\n",
    "('training @ iter = ', 35800)\n",
    "epoch 184, minibatch 195/195, validation error 5.230769 %\n",
    "('training @ iter = ', 35900)\n",
    "('training @ iter = ', 36000)\n",
    "epoch 185, minibatch 195/195, validation error 6.773237 %\n",
    "('training @ iter = ', 36100)\n",
    "('training @ iter = ', 36200)\n",
    "epoch 186, minibatch 195/195, validation error 4.880208 %\n",
    "('training @ iter = ', 36300)\n",
    "('training @ iter = ', 36400)\n",
    "epoch 187, minibatch 195/195, validation error 5.481170 %\n",
    "('training @ iter = ', 36500)\n",
    "('training @ iter = ', 36600)\n",
    "epoch 188, minibatch 195/195, validation error 4.960337 %\n",
    "('training @ iter = ', 36700)\n",
    "('training @ iter = ', 36800)\n",
    "epoch 189, minibatch 195/195, validation error 6.372596 %\n",
    "('training @ iter = ', 36900)\n",
    "('training @ iter = ', 37000)\n",
    "epoch 190, minibatch 195/195, validation error 4.389423 %\n",
    "('training @ iter = ', 37100)\n",
    "('training @ iter = ', 37200)\n",
    "epoch 191, minibatch 195/195, validation error 4.910256 %\n",
    "('training @ iter = ', 37300)\n",
    "('training @ iter = ', 37400)\n",
    "epoch 192, minibatch 195/195, validation error 4.870192 %\n",
    "('training @ iter = ', 37500)\n",
    "('training @ iter = ', 37600)\n",
    "epoch 193, minibatch 195/195, validation error 5.451122 %\n",
    "('training @ iter = ', 37700)\n",
    "('training @ iter = ', 37800)\n",
    "epoch 194, minibatch 195/195, validation error 5.441106 %\n",
    "('training @ iter = ', 37900)\n",
    "('training @ iter = ', 38000)\n",
    "epoch 195, minibatch 195/195, validation error 5.801683 %\n",
    "('training @ iter = ', 38100)\n",
    "('training @ iter = ', 38200)\n",
    "epoch 196, minibatch 195/195, validation error 4.579728 %\n",
    "('training @ iter = ', 38300)\n",
    "('training @ iter = ', 38400)\n",
    "epoch 197, minibatch 195/195, validation error 5.981971 %\n",
    "('training @ iter = ', 38500)\n",
    "('training @ iter = ', 38600)\n",
    "epoch 198, minibatch 195/195, validation error 5.541266 %\n",
    "('training @ iter = ', 38700)\n",
    "('training @ iter = ', 38800)\n",
    "epoch 199, minibatch 195/195, validation error 5.140625 %\n",
    "('training @ iter = ', 38900)\n",
    "epoch 200, minibatch 195/195, validation error 4.729968 %\n",
    "Optimization complete.\n",
    "Best validation score of 5.669071 % obtained at iteration 34320, with test performance 6.330128 %\n",
    "The training process for function run ran for 85.12m\n",
    "----------------------------------------------------------------------\n",
    "\n",
    "\n",
    "...training cnn with include_st = False\n",
    "...training\n",
    "('training @ iter = ', 0)\n",
    "('training @ iter = ', 100)\n",
    "epoch 1, minibatch 195/195, validation error 87.520032 %\n",
    "     epoch 1, minibatch 195/195, test error of best model 87.690304 %\n",
    "('training @ iter = ', 200)\n",
    "('training @ iter = ', 300)\n",
    "epoch 2, minibatch 195/195, validation error 87.439904 %\n",
    "     epoch 2, minibatch 195/195, test error of best model 87.650240 %\n",
    "('training @ iter = ', 400)\n",
    "('training @ iter = ', 500)\n",
    "epoch 3, minibatch 195/195, validation error 85.486779 %\n",
    "     epoch 3, minibatch 195/195, test error of best model 85.536859 %\n",
    "('training @ iter = ', 600)\n",
    "('training @ iter = ', 700)\n",
    "epoch 4, minibatch 195/195, validation error 81.971154 %\n",
    "     epoch 4, minibatch 195/195, test error of best model 81.991186 %\n",
    "('training @ iter = ', 800)\n",
    "('training @ iter = ', 900)\n",
    "epoch 5, minibatch 195/195, validation error 77.483974 %\n",
    "     epoch 5, minibatch 195/195, test error of best model 77.754407 %\n",
    "('training @ iter = ', 1000)\n",
    "('training @ iter = ', 1100)\n",
    "epoch 6, minibatch 195/195, validation error 74.238782 %\n",
    "     epoch 6, minibatch 195/195, test error of best model 74.729567 %\n",
    "('training @ iter = ', 1200)\n",
    "('training @ iter = ', 1300)\n",
    "epoch 7, minibatch 195/195, validation error 71.314103 %\n",
    "     epoch 7, minibatch 195/195, test error of best model 72.245593 %\n",
    "('training @ iter = ', 1400)\n",
    "('training @ iter = ', 1500)\n",
    "epoch 8, minibatch 195/195, validation error 67.137420 %\n",
    "     epoch 8, minibatch 195/195, test error of best model 67.978766 %\n",
    "('training @ iter = ', 1600)\n",
    "('training @ iter = ', 1700)\n",
    "epoch 9, minibatch 195/195, validation error 61.107772 %\n",
    "     epoch 9, minibatch 195/195, test error of best model 62.049279 %\n",
    "('training @ iter = ', 1800)\n",
    "('training @ iter = ', 1900)\n",
    "epoch 10, minibatch 195/195, validation error 55.228365 %\n",
    "     epoch 10, minibatch 195/195, test error of best model 55.588942 %\n",
    "('training @ iter = ', 2000)\n",
    "('training @ iter = ', 2100)\n",
    "epoch 11, minibatch 195/195, validation error 50.771234 %\n",
    "     epoch 11, minibatch 195/195, test error of best model 50.370593 %\n",
    "('training @ iter = ', 2200)\n",
    "('training @ iter = ', 2300)\n",
    "epoch 12, minibatch 195/195, validation error 45.562901 %\n",
    "     epoch 12, minibatch 195/195, test error of best model 45.532853 %\n",
    "('training @ iter = ', 2400)\n",
    "('training @ iter = ', 2500)\n",
    "epoch 13, minibatch 195/195, validation error 42.217548 %\n",
    "     epoch 13, minibatch 195/195, test error of best model 42.447917 %\n",
    "('training @ iter = ', 2600)\n",
    "('training @ iter = ', 2700)\n",
    "epoch 14, minibatch 195/195, validation error 40.294471 %\n",
    "     epoch 14, minibatch 195/195, test error of best model 40.134215 %\n",
    "('training @ iter = ', 2800)\n",
    "('training @ iter = ', 2900)\n",
    "epoch 15, minibatch 195/195, validation error 38.701923 %\n",
    "     epoch 15, minibatch 195/195, test error of best model 38.361378 %\n",
    "('training @ iter = ', 3000)\n",
    "('training @ iter = ', 3100)\n",
    "epoch 16, minibatch 195/195, validation error 36.889022 %\n",
    "     epoch 16, minibatch 195/195, test error of best model 36.698718 %\n",
    "('training @ iter = ', 3200)\n",
    "('training @ iter = ', 3300)\n",
    "epoch 17, minibatch 195/195, validation error 35.046074 %\n",
    "     epoch 17, minibatch 195/195, test error of best model 34.885817 %\n",
    "('training @ iter = ', 3400)\n",
    "('training @ iter = ', 3500)\n",
    "epoch 18, minibatch 195/195, validation error 33.563702 %\n",
    "     epoch 18, minibatch 195/195, test error of best model 33.443510 %\n",
    "('training @ iter = ', 3600)\n",
    "('training @ iter = ', 3700)\n",
    "epoch 19, minibatch 195/195, validation error 32.021234 %\n",
    "     epoch 19, minibatch 195/195, test error of best model 32.151442 %\n",
    "('training @ iter = ', 3800)\n",
    "epoch 20, minibatch 195/195, validation error 30.709135 %\n",
    "     epoch 20, minibatch 195/195, test error of best model 30.719151 %\n",
    "('training @ iter = ', 3900)\n",
    "('training @ iter = ', 4000)\n",
    "epoch 21, minibatch 195/195, validation error 29.607372 %\n",
    "     epoch 21, minibatch 195/195, test error of best model 29.667468 %\n",
    "('training @ iter = ', 4100)\n",
    "('training @ iter = ', 4200)\n",
    "epoch 22, minibatch 195/195, validation error 28.655849 %\n",
    "     epoch 22, minibatch 195/195, test error of best model 28.565705 %\n",
    "('training @ iter = ', 4300)\n",
    "('training @ iter = ', 4400)\n",
    "epoch 23, minibatch 195/195, validation error 27.534054 %\n",
    "     epoch 23, minibatch 195/195, test error of best model 27.724359 %\n",
    "('training @ iter = ', 4500)\n",
    "('training @ iter = ', 4600)\n",
    "epoch 24, minibatch 195/195, validation error 26.422276 %\n",
    "     epoch 24, minibatch 195/195, test error of best model 26.822917 %\n",
    "('training @ iter = ', 4700)\n",
    "('training @ iter = ', 4800)\n",
    "epoch 25, minibatch 195/195, validation error 25.691106 %\n",
    "     epoch 25, minibatch 195/195, test error of best model 25.911458 %\n",
    "('training @ iter = ', 4900)\n",
    "('training @ iter = ', 5000)\n",
    "epoch 26, minibatch 195/195, validation error 25.020032 %\n",
    "     epoch 26, minibatch 195/195, test error of best model 25.250401 %\n",
    "('training @ iter = ', 5100)\n",
    "('training @ iter = ', 5200)\n",
    "epoch 27, minibatch 195/195, validation error 24.379006 %\n",
    "     epoch 27, minibatch 195/195, test error of best model 24.609375 %\n",
    "('training @ iter = ', 5300)\n",
    "('training @ iter = ', 5400)\n",
    "epoch 28, minibatch 195/195, validation error 23.557692 %\n",
    "     epoch 28, minibatch 195/195, test error of best model 24.088542 %\n",
    "('training @ iter = ', 5500)\n",
    "('training @ iter = ', 5600)\n",
    "epoch 29, minibatch 195/195, validation error 23.026843 %\n",
    "     epoch 29, minibatch 195/195, test error of best model 23.317308 %\n",
    "('training @ iter = ', 5700)\n",
    "('training @ iter = ', 5800)\n",
    "epoch 30, minibatch 195/195, validation error 22.495994 %\n",
    "     epoch 30, minibatch 195/195, test error of best model 22.626202 %\n",
    "('training @ iter = ', 5900)\n",
    "('training @ iter = ', 6000)\n",
    "epoch 31, minibatch 195/195, validation error 22.045272 %\n",
    "     epoch 31, minibatch 195/195, test error of best model 22.235577 %\n",
    "('training @ iter = ', 6100)\n",
    "('training @ iter = ', 6200)\n",
    "epoch 32, minibatch 195/195, validation error 21.424279 %\n",
    "     epoch 32, minibatch 195/195, test error of best model 21.744792 %\n",
    "('training @ iter = ', 6300)\n",
    "('training @ iter = ', 6400)\n",
    "epoch 33, minibatch 195/195, validation error 21.153846 %\n",
    "     epoch 33, minibatch 195/195, test error of best model 21.354167 %\n",
    "('training @ iter = ', 6500)\n",
    "('training @ iter = ', 6600)\n",
    "epoch 34, minibatch 195/195, validation error 20.803285 %\n",
    "     epoch 34, minibatch 195/195, test error of best model 20.943510 %\n",
    "('training @ iter = ', 6700)\n",
    "('training @ iter = ', 6800)\n",
    "epoch 35, minibatch 195/195, validation error 20.522837 %\n",
    "     epoch 35, minibatch 195/195, test error of best model 20.512821 %\n",
    "('training @ iter = ', 6900)\n",
    "('training @ iter = ', 7000)\n",
    "epoch 36, minibatch 195/195, validation error 20.112179 %\n",
    "     epoch 36, minibatch 195/195, test error of best model 20.192308 %\n",
    "('training @ iter = ', 7100)\n",
    "('training @ iter = ', 7200)\n",
    "epoch 37, minibatch 195/195, validation error 19.791667 %\n",
    "     epoch 37, minibatch 195/195, test error of best model 19.821715 %\n",
    "('training @ iter = ', 7300)\n",
    "('training @ iter = ', 7400)\n",
    "epoch 38, minibatch 195/195, validation error 19.310897 %\n",
    "     epoch 38, minibatch 195/195, test error of best model 19.441106 %\n",
    "('training @ iter = ', 7500)\n",
    "('training @ iter = ', 7600)\n",
    "epoch 39, minibatch 195/195, validation error 19.120593 %\n",
    "     epoch 39, minibatch 195/195, test error of best model 19.391026 %\n",
    "('training @ iter = ', 7700)\n",
    "epoch 40, minibatch 195/195, validation error 18.820112 %\n",
    "     epoch 40, minibatch 195/195, test error of best model 19.050481 %\n",
    "('training @ iter = ', 7800)\n",
    "('training @ iter = ', 7900)\n",
    "epoch 41, minibatch 195/195, validation error 18.649840 %\n",
    "     epoch 41, minibatch 195/195, test error of best model 18.890224 %\n",
    "('training @ iter = ', 8000)\n",
    "('training @ iter = ', 8100)\n",
    "epoch 42, minibatch 195/195, validation error 18.469551 %\n",
    "     epoch 42, minibatch 195/195, test error of best model 18.629808 %\n",
    "('training @ iter = ', 8200)\n",
    "('training @ iter = ', 8300)\n",
    "epoch 43, minibatch 195/195, validation error 18.309295 %\n",
    "     epoch 43, minibatch 195/195, test error of best model 18.569712 %\n",
    "('training @ iter = ', 8400)\n",
    "('training @ iter = ', 8500)\n",
    "epoch 44, minibatch 195/195, validation error 18.179087 %\n",
    "     epoch 44, minibatch 195/195, test error of best model 18.319311 %\n",
    "('training @ iter = ', 8600)\n",
    "('training @ iter = ', 8700)\n",
    "epoch 45, minibatch 195/195, validation error 17.938702 %\n",
    "     epoch 45, minibatch 195/195, test error of best model 18.108974 %\n",
    "('training @ iter = ', 8800)\n",
    "('training @ iter = ', 8900)\n",
    "epoch 46, minibatch 195/195, validation error 17.828526 %\n",
    "     epoch 46, minibatch 195/195, test error of best model 18.038862 %\n",
    "('training @ iter = ', 9000)\n",
    "('training @ iter = ', 9100)\n",
    "epoch 47, minibatch 195/195, validation error 17.648237 %\n",
    "     epoch 47, minibatch 195/195, test error of best model 17.788462 %\n",
    "('training @ iter = ', 9200)\n",
    "('training @ iter = ', 9300)\n",
    "epoch 48, minibatch 195/195, validation error 17.447917 %\n",
    "     epoch 48, minibatch 195/195, test error of best model 17.487981 %\n",
    "('training @ iter = ', 9400)\n",
    "('training @ iter = ', 9500)\n",
    "epoch 49, minibatch 195/195, validation error 17.207532 %\n",
    "     epoch 49, minibatch 195/195, test error of best model 17.377804 %\n",
    "('training @ iter = ', 9600)\n",
    "('training @ iter = ', 9700)\n",
    "epoch 50, minibatch 195/195, validation error 17.037260 %\n",
    "     epoch 50, minibatch 195/195, test error of best model 17.227564 %\n",
    "('training @ iter = ', 9800)\n",
    "('training @ iter = ', 9900)\n",
    "epoch 51, minibatch 195/195, validation error 16.897035 %\n",
    "     epoch 51, minibatch 195/195, test error of best model 17.067308 %\n",
    "('training @ iter = ', 10000)\n",
    "('training @ iter = ', 10100)\n",
    "epoch 52, minibatch 195/195, validation error 16.726763 %\n",
    "     epoch 52, minibatch 195/195, test error of best model 16.957131 %\n",
    "('training @ iter = ', 10200)\n",
    "('training @ iter = ', 10300)\n",
    "epoch 53, minibatch 195/195, validation error 16.636619 %\n",
    "     epoch 53, minibatch 195/195, test error of best model 16.957131 %\n",
    "('training @ iter = ', 10400)\n",
    "('training @ iter = ', 10500)\n",
    "epoch 54, minibatch 195/195, validation error 16.486378 %\n",
    "     epoch 54, minibatch 195/195, test error of best model 16.897035 %\n",
    "('training @ iter = ', 10600)\n",
    "('training @ iter = ', 10700)\n",
    "epoch 55, minibatch 195/195, validation error 16.306090 %\n",
    "     epoch 55, minibatch 195/195, test error of best model 16.736779 %\n",
    "('training @ iter = ', 10800)\n",
    "('training @ iter = ', 10900)\n",
    "epoch 56, minibatch 195/195, validation error 16.115785 %\n",
    "     epoch 56, minibatch 195/195, test error of best model 16.586538 %\n",
    "('training @ iter = ', 11000)\n",
    "('training @ iter = ', 11100)\n",
    "epoch 57, minibatch 195/195, validation error 16.045673 %\n",
    "     epoch 57, minibatch 195/195, test error of best model 16.426282 %\n",
    "('training @ iter = ', 11200)\n",
    "('training @ iter = ', 11300)\n",
    "epoch 58, minibatch 195/195, validation error 15.945513 %\n",
    "     epoch 58, minibatch 195/195, test error of best model 16.356170 %\n",
    "('training @ iter = ', 11400)\n",
    "('training @ iter = ', 11500)\n",
    "epoch 59, minibatch 195/195, validation error 15.835337 %\n",
    "     epoch 59, minibatch 195/195, test error of best model 16.165865 %\n",
    "('training @ iter = ', 11600)\n",
    "epoch 60, minibatch 195/195, validation error 15.795272 %\n",
    "     epoch 60, minibatch 195/195, test error of best model 16.075721 %\n",
    "('training @ iter = ', 11700)\n",
    "('training @ iter = ', 11800)\n",
    "epoch 61, minibatch 195/195, validation error 15.584936 %\n",
    "     epoch 61, minibatch 195/195, test error of best model 15.875401 %\n",
    "('training @ iter = ', 11900)\n",
    "('training @ iter = ', 12000)\n",
    "epoch 62, minibatch 195/195, validation error 15.454728 %\n",
    "     epoch 62, minibatch 195/195, test error of best model 15.755208 %\n",
    "('training @ iter = ', 12100)\n",
    "('training @ iter = ', 12200)\n",
    "epoch 63, minibatch 195/195, validation error 15.384615 %\n",
    "     epoch 63, minibatch 195/195, test error of best model 15.625000 %\n",
    "('training @ iter = ', 12300)\n",
    "('training @ iter = ', 12400)\n",
    "epoch 64, minibatch 195/195, validation error 15.214343 %\n",
    "     epoch 64, minibatch 195/195, test error of best model 15.464744 %\n",
    "('training @ iter = ', 12500)\n",
    "('training @ iter = ', 12600)\n",
    "epoch 65, minibatch 195/195, validation error 15.064103 %\n",
    "     epoch 65, minibatch 195/195, test error of best model 15.454728 %\n",
    "('training @ iter = ', 12700)\n",
    "('training @ iter = ', 12800)\n",
    "epoch 66, minibatch 195/195, validation error 14.983974 %\n",
    "     epoch 66, minibatch 195/195, test error of best model 15.404647 %\n",
    "('training @ iter = ', 12900)\n",
    "('training @ iter = ', 13000)\n",
    "epoch 67, minibatch 195/195, validation error 15.034054 %\n",
    "('training @ iter = ', 13100)\n",
    "('training @ iter = ', 13200)\n",
    "epoch 68, minibatch 195/195, validation error 14.993990 %\n",
    "('training @ iter = ', 13300)\n",
    "('training @ iter = ', 13400)\n",
    "epoch 69, minibatch 195/195, validation error 14.873798 %\n",
    "     epoch 69, minibatch 195/195, test error of best model 15.214343 %\n",
    "('training @ iter = ', 13500)\n",
    "('training @ iter = ', 13600)\n",
    "epoch 70, minibatch 195/195, validation error 14.773638 %\n",
    "     epoch 70, minibatch 195/195, test error of best model 15.094151 %\n",
    "('training @ iter = ', 13700)\n",
    "('training @ iter = ', 13800)\n",
    "epoch 71, minibatch 195/195, validation error 14.693510 %\n",
    "     epoch 71, minibatch 195/195, test error of best model 15.024038 %\n",
    "('training @ iter = ', 13900)\n",
    "('training @ iter = ', 14000)\n",
    "epoch 72, minibatch 195/195, validation error 14.553285 %\n",
    "     epoch 72, minibatch 195/195, test error of best model 14.873798 %\n",
    "('training @ iter = ', 14100)\n",
    "('training @ iter = ', 14200)\n",
    "epoch 73, minibatch 195/195, validation error 14.443109 %\n",
    "     epoch 73, minibatch 195/195, test error of best model 14.673478 %\n",
    "('training @ iter = ', 14300)\n",
    "('training @ iter = ', 14400)\n",
    "epoch 74, minibatch 195/195, validation error 14.332933 %\n",
    "     epoch 74, minibatch 195/195, test error of best model 14.573317 %\n",
    "('training @ iter = ', 14500)\n",
    "('training @ iter = ', 14600)\n",
    "epoch 75, minibatch 195/195, validation error 14.302885 %\n",
    "     epoch 75, minibatch 195/195, test error of best model 14.593349 %\n",
    "('training @ iter = ', 14700)\n",
    "('training @ iter = ', 14800)\n",
    "epoch 76, minibatch 195/195, validation error 14.182692 %\n",
    "     epoch 76, minibatch 195/195, test error of best model 14.393029 %\n",
    "('training @ iter = ', 14900)\n",
    "('training @ iter = ', 15000)\n",
    "epoch 77, minibatch 195/195, validation error 14.152644 %\n",
    "     epoch 77, minibatch 195/195, test error of best model 14.322917 %\n",
    "('training @ iter = ', 15100)\n",
    "('training @ iter = ', 15200)\n",
    "epoch 78, minibatch 195/195, validation error 14.082532 %\n",
    "     epoch 78, minibatch 195/195, test error of best model 14.272837 %\n",
    "('training @ iter = ', 15300)\n",
    "('training @ iter = ', 15400)\n",
    "epoch 79, minibatch 195/195, validation error 14.092548 %\n",
    "('training @ iter = ', 15500)\n",
    "epoch 80, minibatch 195/195, validation error 14.082532 %\n",
    "('training @ iter = ', 15600)\n",
    "('training @ iter = ', 15700)\n",
    "epoch 81, minibatch 195/195, validation error 13.872196 %\n",
    "     epoch 81, minibatch 195/195, test error of best model 14.042468 %\n",
    "('training @ iter = ', 15800)\n",
    "('training @ iter = ', 15900)\n",
    "epoch 82, minibatch 195/195, validation error 13.772035 %\n",
    "     epoch 82, minibatch 195/195, test error of best model 13.982372 %\n",
    "('training @ iter = ', 16000)\n",
    "('training @ iter = ', 16100)\n",
    "epoch 83, minibatch 195/195, validation error 13.711939 %\n",
    "     epoch 83, minibatch 195/195, test error of best model 13.952324 %\n",
    "('training @ iter = ', 16200)\n",
    "('training @ iter = ', 16300)\n",
    "epoch 84, minibatch 195/195, validation error 13.671875 %\n",
    "     epoch 84, minibatch 195/195, test error of best model 13.862179 %\n",
    "('training @ iter = ', 16400)\n",
    "('training @ iter = ', 16500)\n",
    "epoch 85, minibatch 195/195, validation error 13.551683 %\n",
    "     epoch 85, minibatch 195/195, test error of best model 13.802083 %\n",
    "('training @ iter = ', 16600)\n",
    "('training @ iter = ', 16700)\n",
    "epoch 86, minibatch 195/195, validation error 13.341346 %\n",
    "     epoch 86, minibatch 195/195, test error of best model 13.671875 %\n",
    "('training @ iter = ', 16800)\n",
    "('training @ iter = ', 16900)\n",
    "epoch 87, minibatch 195/195, validation error 13.231170 %\n",
    "     epoch 87, minibatch 195/195, test error of best model 13.561699 %\n",
    "('training @ iter = ', 17000)\n",
    "('training @ iter = ', 17100)\n",
    "epoch 88, minibatch 195/195, validation error 13.110978 %\n",
    "     epoch 88, minibatch 195/195, test error of best model 13.451522 %\n",
    "('training @ iter = ', 17200)\n",
    "('training @ iter = ', 17300)\n",
    "epoch 89, minibatch 195/195, validation error 12.930689 %\n",
    "     epoch 89, minibatch 195/195, test error of best model 13.341346 %\n",
    "('training @ iter = ', 17400)\n",
    "('training @ iter = ', 17500)\n",
    "epoch 90, minibatch 195/195, validation error 12.850561 %\n",
    "     epoch 90, minibatch 195/195, test error of best model 13.241186 %\n",
    "('training @ iter = ', 17600)\n",
    "('training @ iter = ', 17700)\n",
    "epoch 91, minibatch 195/195, validation error 12.770433 %\n",
    "     epoch 91, minibatch 195/195, test error of best model 13.131010 %\n",
    "('training @ iter = ', 17800)\n",
    "('training @ iter = ', 17900)\n",
    "epoch 92, minibatch 195/195, validation error 12.660256 %\n",
    "     epoch 92, minibatch 195/195, test error of best model 13.100962 %\n",
    "('training @ iter = ', 18000)\n",
    "('training @ iter = ', 18100)\n",
    "epoch 93, minibatch 195/195, validation error 12.620192 %\n",
    "     epoch 93, minibatch 195/195, test error of best model 13.050881 %\n",
    "('training @ iter = ', 18200)\n",
    "('training @ iter = ', 18300)\n",
    "epoch 94, minibatch 195/195, validation error 12.489984 %\n",
    "     epoch 94, minibatch 195/195, test error of best model 12.960737 %\n",
    "('training @ iter = ', 18400)\n",
    "('training @ iter = ', 18500)\n",
    "epoch 95, minibatch 195/195, validation error 12.419872 %\n",
    "     epoch 95, minibatch 195/195, test error of best model 12.930689 %\n",
    "('training @ iter = ', 18600)\n",
    "('training @ iter = ', 18700)\n",
    "epoch 96, minibatch 195/195, validation error 12.209535 %\n",
    "     epoch 96, minibatch 195/195, test error of best model 12.840545 %\n",
    "('training @ iter = ', 18800)\n",
    "('training @ iter = ', 18900)\n",
    "epoch 97, minibatch 195/195, validation error 12.209535 %\n",
    "('training @ iter = ', 19000)\n",
    "('training @ iter = ', 19100)\n",
    "epoch 98, minibatch 195/195, validation error 12.059295 %\n",
    "     epoch 98, minibatch 195/195, test error of best model 12.660256 %\n",
    "('training @ iter = ', 19200)\n",
    "('training @ iter = ', 19300)\n",
    "epoch 99, minibatch 195/195, validation error 12.029247 %\n",
    "     epoch 99, minibatch 195/195, test error of best model 12.630208 %\n",
    "('training @ iter = ', 19400)\n",
    "epoch 100, minibatch 195/195, validation error 12.009215 %\n",
    "     epoch 100, minibatch 195/195, test error of best model 12.620192 %\n",
    "('training @ iter = ', 19500)\n",
    "('training @ iter = ', 19600)\n",
    "epoch 101, minibatch 195/195, validation error 11.919071 %\n",
    "     epoch 101, minibatch 195/195, test error of best model 12.520032 %\n",
    "('training @ iter = ', 19700)\n",
    "('training @ iter = ', 19800)\n",
    "epoch 102, minibatch 195/195, validation error 11.909054 %\n",
    "     epoch 102, minibatch 195/195, test error of best model 12.500000 %\n",
    "('training @ iter = ', 19900)\n",
    "('training @ iter = ', 20000)\n",
    "epoch 103, minibatch 195/195, validation error 11.879006 %\n",
    "     epoch 103, minibatch 195/195, test error of best model 12.399840 %\n",
    "('training @ iter = ', 20100)\n",
    "('training @ iter = ', 20200)\n",
    "epoch 104, minibatch 195/195, validation error 11.748798 %\n",
    "     epoch 104, minibatch 195/195, test error of best model 12.239583 %\n",
    "('training @ iter = ', 20300)\n",
    "('training @ iter = ', 20400)\n",
    "epoch 105, minibatch 195/195, validation error 11.698718 %\n",
    "     epoch 105, minibatch 195/195, test error of best model 12.139423 %\n",
    "('training @ iter = ', 20500)\n",
    "('training @ iter = ', 20600)\n",
    "epoch 106, minibatch 195/195, validation error 11.728766 %\n",
    "('training @ iter = ', 20700)\n",
    "('training @ iter = ', 20800)\n",
    "epoch 107, minibatch 195/195, validation error 11.628606 %\n",
    "     epoch 107, minibatch 195/195, test error of best model 12.059295 %\n",
    "('training @ iter = ', 20900)\n",
    "('training @ iter = ', 21000)\n",
    "epoch 108, minibatch 195/195, validation error 11.568510 %\n",
    "     epoch 108, minibatch 195/195, test error of best model 11.979167 %\n",
    "('training @ iter = ', 21100)\n",
    "('training @ iter = ', 21200)\n",
    "epoch 109, minibatch 195/195, validation error 11.558494 %\n",
    "     epoch 109, minibatch 195/195, test error of best model 11.949119 %\n",
    "('training @ iter = ', 21300)\n",
    "('training @ iter = ', 21400)\n",
    "epoch 110, minibatch 195/195, validation error 11.518429 %\n",
    "     epoch 110, minibatch 195/195, test error of best model 11.899038 %\n",
    "('training @ iter = ', 21500)\n",
    "('training @ iter = ', 21600)\n",
    "epoch 111, minibatch 195/195, validation error 11.478365 %\n",
    "     epoch 111, minibatch 195/195, test error of best model 11.828926 %\n",
    "('training @ iter = ', 21700)\n",
    "('training @ iter = ', 21800)\n",
    "epoch 112, minibatch 195/195, validation error 11.408253 %\n",
    "     epoch 112, minibatch 195/195, test error of best model 11.768830 %\n",
    "('training @ iter = ', 21900)\n",
    "('training @ iter = ', 22000)\n",
    "epoch 113, minibatch 195/195, validation error 11.318109 %\n",
    "     epoch 113, minibatch 195/195, test error of best model 11.688702 %\n",
    "('training @ iter = ', 22100)\n",
    "('training @ iter = ', 22200)\n",
    "epoch 114, minibatch 195/195, validation error 11.388221 %\n",
    "('training @ iter = ', 22300)\n",
    "('training @ iter = ', 22400)\n",
    "epoch 115, minibatch 195/195, validation error 11.338141 %\n",
    "('training @ iter = ', 22500)\n",
    "('training @ iter = ', 22600)\n",
    "epoch 116, minibatch 195/195, validation error 11.338141 %\n",
    "('training @ iter = ', 22700)\n",
    "('training @ iter = ', 23400)\n",
    "('training @ iter = ', 23500)\n",
    "epoch 121, minibatch 195/195, validation error 11.187901 %\n",
    "     epoch 121, minibatch 195/195, test error of best model 11.548478 %\n",
    "('training @ iter = ', 23600)\n",
    "('training @ iter = ', 23700)\n",
    "epoch 122, minibatch 195/195, validation error 11.197917 %\n",
    "('training @ iter = ', 23800)\n",
    "('training @ iter = ', 23900)\n",
    "epoch 123, minibatch 195/195, validation error 11.157853 %\n",
    "     epoch 123, minibatch 195/195, test error of best model 11.478365 %\n",
    "('training @ iter = ', 24000)\n",
    "('training @ iter = ', 24100)\n",
    "epoch 124, minibatch 195/195, validation error 11.097756 %\n",
    "     epoch 124, minibatch 195/195, test error of best model 11.448317 %\n",
    "('training @ iter = ', 24200)\n",
    "('training @ iter = ', 24300)\n",
    "epoch 125, minibatch 195/195, validation error 11.037660 %\n",
    "     epoch 125, minibatch 195/195, test error of best model 11.328125 %\n",
    "('training @ iter = ', 24400)\n",
    "('training @ iter = ', 24500)\n",
    "epoch 126, minibatch 195/195, validation error 10.947516 %\n",
    "     epoch 126, minibatch 195/195, test error of best model 11.237981 %\n",
    "('training @ iter = ', 24600)\n",
    "('training @ iter = ', 24700)\n",
    "epoch 127, minibatch 195/195, validation error 10.817308 %\n",
    "     epoch 127, minibatch 195/195, test error of best model 11.187901 %\n",
    "('training @ iter = ', 24800)\n",
    "('training @ iter = ', 24900)\n",
    "epoch 128, minibatch 195/195, validation error 10.737179 %\n",
    "     epoch 128, minibatch 195/195, test error of best model 11.117788 %\n",
    "('training @ iter = ', 25000)\n",
    "('training @ iter = ', 25100)\n",
    "epoch 129, minibatch 195/195, validation error 10.787260 %\n",
    "('training @ iter = ', 25200)\n",
    "('training @ iter = ', 25300)\n",
    "epoch 130, minibatch 195/195, validation error 10.797276 %\n",
    "('training @ iter = ', 25400)\n",
    "('training @ iter = ', 25500)\n",
    "epoch 131, minibatch 195/195, validation error 10.717147 %\n",
    "     epoch 131, minibatch 195/195, test error of best model 10.987580 %\n",
    "('training @ iter = ', 25600)\n",
    "('training @ iter = ', 25700)\n",
    "epoch 132, minibatch 195/195, validation error 10.727163 %\n",
    "('training @ iter = ', 25800)\n",
    "('training @ iter = ', 25900)\n",
    "epoch 133, minibatch 195/195, validation error 10.737179 %\n",
    "('training @ iter = ', 26000)\n",
    "('training @ iter = ', 26100)\n",
    "epoch 134, minibatch 195/195, validation error 10.596955 %\n",
    "     epoch 134, minibatch 195/195, test error of best model 10.877404 %\n",
    "('training @ iter = ', 26200)\n",
    "('training @ iter = ', 26300)\n",
    "epoch 135, minibatch 195/195, validation error 10.616987 %\n",
    "('training @ iter = ', 26400)\n",
    "('training @ iter = ', 26500)\n",
    "epoch 136, minibatch 195/195, validation error 10.556891 %\n",
    "     epoch 136, minibatch 195/195, test error of best model 10.707131 %\n",
    "('training @ iter = ', 26600)\n",
    "('training @ iter = ', 26700)\n",
    "epoch 137, minibatch 195/195, validation error 10.586939 %\n",
    "('training @ iter = ', 26800)\n",
    "('training @ iter = ', 26900)\n",
    "epoch 138, minibatch 195/195, validation error 10.546875 %\n",
    "     epoch 138, minibatch 195/195, test error of best model 10.747196 %\n",
    "('training @ iter = ', 27000)\n",
    "('training @ iter = ', 27100)\n",
    "epoch 139, minibatch 195/195, validation error 10.516827 %\n",
    "     epoch 139, minibatch 195/195, test error of best model 10.767228 %\n",
    "('training @ iter = ', 27200)\n",
    "epoch 140, minibatch 195/195, validation error 10.406651 %\n",
    "     epoch 140, minibatch 195/195, test error of best model 10.647035 %\n",
    "('training @ iter = ', 27300)\n",
    "('training @ iter = ', 27400)\n",
    "epoch 141, minibatch 195/195, validation error 10.376603 %\n",
    "     epoch 141, minibatch 195/195, test error of best model 10.606971 %\n",
    "('training @ iter = ', 27500)\n",
    "('training @ iter = ', 27600)\n",
    "epoch 142, minibatch 195/195, validation error 10.336538 %\n",
    "     epoch 142, minibatch 195/195, test error of best model 10.536859 %\n",
    "('training @ iter = ', 27700)\n",
    "('training @ iter = ', 27800)\n",
    "epoch 143, minibatch 195/195, validation error 10.336538 %\n",
    "('training @ iter = ', 27900)\n",
    "('training @ iter = ', 28000)\n",
    "epoch 144, minibatch 195/195, validation error 10.256410 %\n",
    "     epoch 144, minibatch 195/195, test error of best model 10.346554 %\n",
    "('training @ iter = ', 28100)\n",
    "('training @ iter = ', 28200)\n",
    "epoch 145, minibatch 195/195, validation error 10.146234 %\n",
    "     epoch 145, minibatch 195/195, test error of best model 10.216346 %\n",
    "('training @ iter = ', 28300)\n",
    "('training @ iter = ', 28400)\n",
    "epoch 146, minibatch 195/195, validation error 10.086138 %\n",
    "     epoch 146, minibatch 195/195, test error of best model 10.136218 %\n",
    "('training @ iter = ', 28500)\n",
    "('training @ iter = ', 28600)\n",
    "epoch 147, minibatch 195/195, validation error 10.006010 %\n",
    "     epoch 147, minibatch 195/195, test error of best model 10.026042 %\n",
    "('training @ iter = ', 28700)\n",
    "('training @ iter = ', 28800)\n",
    "epoch 148, minibatch 195/195, validation error 9.975962 %\n",
    "     epoch 148, minibatch 195/195, test error of best model 9.945913 %\n",
    "('training @ iter = ', 28900)\n",
    "('training @ iter = ', 29000)\n",
    "epoch 149, minibatch 195/195, validation error 9.935897 %\n",
    "     epoch 149, minibatch 195/195, test error of best model 9.835737 %\n",
    "('training @ iter = ', 29100)\n",
    "('training @ iter = ', 29200)\n",
    "epoch 150, minibatch 195/195, validation error 9.835737 %\n",
    "     epoch 150, minibatch 195/195, test error of best model 9.725561 %\n",
    "('training @ iter = ', 29300)\n",
    "('training @ iter = ', 29400)\n",
    "epoch 151, minibatch 195/195, validation error 9.735577 %\n",
    "     epoch 151, minibatch 195/195, test error of best model 9.675481 %\n",
    "('training @ iter = ', 29500)\n",
    "('training @ iter = ', 29600)\n",
    "epoch 152, minibatch 195/195, validation error 9.715545 %\n",
    "     epoch 152, minibatch 195/195, test error of best model 9.695513 %\n",
    "('training @ iter = ', 29700)\n",
    "('training @ iter = ', 29800)\n",
    "epoch 153, minibatch 195/195, validation error 9.655449 %\n",
    "     epoch 153, minibatch 195/195, test error of best model 9.655449 %\n",
    "('training @ iter = ', 29900)\n",
    "('training @ iter = ', 30000)\n",
    "epoch 154, minibatch 195/195, validation error 9.615385 %\n",
    "     epoch 154, minibatch 195/195, test error of best model 9.595353 %\n",
    "('training @ iter = ', 30100)\n",
    "('training @ iter = ', 30200)\n",
    "epoch 155, minibatch 195/195, validation error 9.545272 %\n",
    "     epoch 155, minibatch 195/195, test error of best model 9.515224 %\n",
    "('training @ iter = ', 30300)\n",
    "('training @ iter = ', 30400)\n",
    "epoch 156, minibatch 195/195, validation error 9.525240 %\n",
    "     epoch 156, minibatch 195/195, test error of best model 9.495192 %\n",
    "('training @ iter = ', 30500)\n",
    "('training @ iter = ', 30600)\n",
    "epoch 157, minibatch 195/195, validation error 9.545272 %\n",
    "('training @ iter = ', 30700)\n",
    "('training @ iter = ', 30800)\n",
    "epoch 158, minibatch 195/195, validation error 9.505208 %\n",
    "     epoch 158, minibatch 195/195, test error of best model 9.475160 %\n",
    "('training @ iter = ', 30900)\n",
    "('training @ iter = ', 31000)\n",
    "epoch 159, minibatch 195/195, validation error 9.515224 %\n",
    "('training @ iter = ', 31100)\n",
    "epoch 160, minibatch 195/195, validation error 9.415064 %\n",
    "     epoch 160, minibatch 195/195, test error of best model 9.415064 %\n",
    "('training @ iter = ', 31200)\n",
    "('training @ iter = ', 31300)\n",
    "epoch 161, minibatch 195/195, validation error 9.445112 %\n",
    "('training @ iter = ', 31400)\n",
    "('training @ iter = ', 31500)\n",
    "epoch 162, minibatch 195/195, validation error 9.375000 %\n",
    "     epoch 162, minibatch 195/195, test error of best model 9.375000 %\n",
    "('training @ iter = ', 31600)\n",
    "('training @ iter = ', 31700)\n",
    "epoch 163, minibatch 195/195, validation error 9.395032 %\n",
    "('training @ iter = ', 31800)\n",
    "('training @ iter = ', 31900)\n",
    "epoch 164, minibatch 195/195, validation error 9.274840 %\n",
    "     epoch 164, minibatch 195/195, test error of best model 9.264824 %\n",
    "('training @ iter = ', 32000)\n",
    "('training @ iter = ', 32100)\n",
    "epoch 165, minibatch 195/195, validation error 9.234776 %\n",
    "     epoch 165, minibatch 195/195, test error of best model 9.274840 %\n",
    "('training @ iter = ', 32200)\n",
    "('training @ iter = ', 32300)\n",
    "epoch 166, minibatch 195/195, validation error 9.134615 %\n",
    "     epoch 166, minibatch 195/195, test error of best model 9.204728 %\n",
    "('training @ iter = ', 32400)\n",
    "('training @ iter = ', 32500)\n",
    "epoch 167, minibatch 195/195, validation error 9.144631 %\n",
    "('training @ iter = ', 32600)\n",
    "('training @ iter = ', 32700)\n",
    "epoch 168, minibatch 195/195, validation error 9.114583 %\n",
    "     epoch 168, minibatch 195/195, test error of best model 9.214744 %\n",
    "('training @ iter = ', 32800)\n",
    "('training @ iter = ', 32900)\n",
    "epoch 169, minibatch 195/195, validation error 9.024439 %\n",
    "     epoch 169, minibatch 195/195, test error of best model 9.084535 %\n",
    "('training @ iter = ', 33000)\n",
    "('training @ iter = ', 33100)\n",
    "epoch 170, minibatch 195/195, validation error 8.984375 %\n",
    "     epoch 170, minibatch 195/195, test error of best model 9.044471 %\n",
    "('training @ iter = ', 33200)\n",
    "('training @ iter = ', 33300)\n",
    "epoch 171, minibatch 195/195, validation error 8.974359 %\n",
    "     epoch 171, minibatch 195/195, test error of best model 9.044471 %\n",
    "('training @ iter = ', 33400)\n",
    "('training @ iter = ', 33500)\n",
    "epoch 172, minibatch 195/195, validation error 8.824119 %\n",
    "     epoch 172, minibatch 195/195, test error of best model 8.924279 %\n",
    "('training @ iter = ', 33600)\n",
    "('training @ iter = ', 33700)\n",
    "epoch 173, minibatch 195/195, validation error 8.814103 %\n",
    "     epoch 173, minibatch 195/195, test error of best model 8.884215 %\n",
    "('training @ iter = ', 33800)\n",
    "('training @ iter = ', 33900)\n",
    "epoch 174, minibatch 195/195, validation error 8.794071 %\n",
    "     epoch 174, minibatch 195/195, test error of best model 8.834135 %\n",
    "('training @ iter = ', 34000)\n",
    "('training @ iter = ', 34100)\n",
    "epoch 175, minibatch 195/195, validation error 8.834135 %\n",
    "('training @ iter = ', 34200)\n",
    "('training @ iter = ', 34300)\n",
    "epoch 176, minibatch 195/195, validation error 8.774038 %\n",
    "     epoch 176, minibatch 195/195, test error of best model 8.794071 %\n",
    "('training @ iter = ', 34400)\n",
    "('training @ iter = ', 34500)\n",
    "epoch 177, minibatch 195/195, validation error 8.713942 %\n",
    "     epoch 177, minibatch 195/195, test error of best model 8.774038 %\n",
    "('training @ iter = ', 34600)\n",
    "('training @ iter = ', 34700)\n",
    "epoch 178, minibatch 195/195, validation error 8.683894 %\n",
    "     epoch 178, minibatch 195/195, test error of best model 8.723958 %\n",
    "('training @ iter = ', 34800)\n",
    "('training @ iter = ', 34900)\n",
    "epoch 179, minibatch 195/195, validation error 8.573718 %\n",
    "     epoch 179, minibatch 195/195, test error of best model 8.643830 %\n",
    "('training @ iter = ', 35000)\n",
    "epoch 180, minibatch 195/195, validation error 8.563702 %\n",
    "     epoch 180, minibatch 195/195, test error of best model 8.603766 %\n",
    "('training @ iter = ', 35100)\n",
    "('training @ iter = ', 35200)\n",
    "epoch 181, minibatch 195/195, validation error 8.563702 %\n",
    "('training @ iter = ', 35300)\n",
    "('training @ iter = ', 35400)\n",
    "epoch 182, minibatch 195/195, validation error 8.513622 %\n",
    "     epoch 182, minibatch 195/195, test error of best model 8.503606 %\n",
    "('training @ iter = ', 35500)\n",
    "('training @ iter = ', 35600)\n",
    "epoch 183, minibatch 195/195, validation error 8.523638 %\n",
    "('training @ iter = ', 35700)\n",
    "('training @ iter = ', 35800)\n",
    "epoch 184, minibatch 195/195, validation error 8.483574 %\n",
    "     epoch 184, minibatch 195/195, test error of best model 8.443510 %\n",
    "('training @ iter = ', 35900)\n",
    "('training @ iter = ', 36000)\n",
    "epoch 185, minibatch 195/195, validation error 8.443510 %\n",
    "     epoch 185, minibatch 195/195, test error of best model 8.433494 %\n",
    "('training @ iter = ', 36100)\n",
    "('training @ iter = ', 36200)\n",
    "epoch 186, minibatch 195/195, validation error 8.413462 %\n",
    "     epoch 186, minibatch 195/195, test error of best model 8.403446 %\n",
    "('training @ iter = ', 36300)\n",
    "('training @ iter = ', 36400)\n",
    "epoch 187, minibatch 195/195, validation error 8.403446 %\n",
    "     epoch 187, minibatch 195/195, test error of best model 8.353365 %\n",
    "('training @ iter = ', 36500)\n",
    "('training @ iter = ', 36600)\n",
    "epoch 188, minibatch 195/195, validation error 8.403446 %\n",
    "('training @ iter = ', 36700)\n",
    "('training @ iter = ', 36800)\n",
    "epoch 189, minibatch 195/195, validation error 8.383413 %\n",
    "     epoch 189, minibatch 195/195, test error of best model 8.373397 %\n",
    "('training @ iter = ', 36900)\n",
    "('training @ iter = ', 37000)\n",
    "epoch 190, minibatch 195/195, validation error 8.323317 %\n",
    "     epoch 190, minibatch 195/195, test error of best model 8.333333 %\n",
    "('training @ iter = ', 37100)\n",
    "('training @ iter = ', 37200)\n",
    "epoch 191, minibatch 195/195, validation error 8.223157 %\n",
    "     epoch 191, minibatch 195/195, test error of best model 8.323317 %\n",
    "('training @ iter = ', 37300)\n",
    "('training @ iter = ', 37400)\n",
    "epoch 192, minibatch 195/195, validation error 8.203125 %\n",
    "     epoch 192, minibatch 195/195, test error of best model 8.273237 %\n",
    "('training @ iter = ', 37500)\n",
    "('training @ iter = ', 37600)\n",
    "epoch 193, minibatch 195/195, validation error 8.163061 %\n",
    "     epoch 193, minibatch 195/195, test error of best model 8.213141 %\n",
    "('training @ iter = ', 37700)\n",
    "('training @ iter = ', 37800)\n",
    "epoch 194, minibatch 195/195, validation error 8.183093 %\n",
    "('training @ iter = ', 37900)\n",
    "('training @ iter = ', 38000)\n",
    "epoch 195, minibatch 195/195, validation error 8.163061 %\n",
    "('training @ iter = ', 38100)\n",
    "('training @ iter = ', 38200)\n",
    "epoch 196, minibatch 195/195, validation error 8.153045 %\n",
    "     epoch 196, minibatch 195/195, test error of best model 8.193109 %\n",
    "('training @ iter = ', 38300)\n",
    "('training @ iter = ', 38400)\n",
    "epoch 197, minibatch 195/195, validation error 8.102965 %\n",
    "     epoch 197, minibatch 195/195, test error of best model 8.133013 %\n",
    "('training @ iter = ', 38500)\n",
    "('training @ iter = ', 38600)\n",
    "epoch 198, minibatch 195/195, validation error 8.092949 %\n",
    "     epoch 198, minibatch 195/195, test error of best model 8.173077 %\n",
    "('training @ iter = ', 38700)\n",
    "('training @ iter = ', 38800)\n",
    "epoch 199, minibatch 195/195, validation error 8.112981 %\n",
    "('training @ iter = ', 38900)\n",
    "epoch 200, minibatch 195/195, validation error 8.082933 %\n",
    "     epoch 200, minibatch 195/195, test error of best model 8.112981 %\n",
    "Optimization complete.\n",
    "Best validation score of 8.082933 % obtained at iteration 39000, with test performance 8.112981 %\n",
    "The training process for function run ran for 40.85m\n",
    "----------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "...loading the dataset\n",
      "...Applying proj_transform to dataset\n",
      "Current training data size is 50000\n",
      "Current validation data size is 10000\n",
      "Current test data size is 10000\n",
      "...building the model\n",
      "...training fcn with include_st = False\n",
      "...training\n",
      "('training @ iter = ', 0)\n",
      "('training @ iter = ', 100)\n",
      "epoch 1, minibatch 195/195, validation error 89.963942 %\n",
      "     epoch 1, minibatch 195/195, test error of best model 89.963942 %\n",
      "('training @ iter = ', 200)\n",
      "('training @ iter = ', 300)\n",
      "epoch 2, minibatch 195/195, validation error 89.433093 %\n",
      "     epoch 2, minibatch 195/195, test error of best model 89.312901 %\n",
      "('training @ iter = ', 400)\n",
      "('training @ iter = ', 500)\n",
      "epoch 3, minibatch 195/195, validation error 87.850561 %\n",
      "     epoch 3, minibatch 195/195, test error of best model 87.770433 %\n",
      "('training @ iter = ', 600)\n",
      "('training @ iter = ', 700)\n",
      "epoch 4, minibatch 195/195, validation error 85.456731 %\n",
      "     epoch 4, minibatch 195/195, test error of best model 85.306490 %\n",
      "('training @ iter = ', 800)\n",
      "('training @ iter = ', 900)\n",
      "epoch 5, minibatch 195/195, validation error 81.300080 %\n",
      "     epoch 5, minibatch 195/195, test error of best model 81.800881 %\n",
      "('training @ iter = ', 1000)\n",
      "('training @ iter = ', 1100)\n",
      "epoch 6, minibatch 195/195, validation error 77.343750 %\n",
      "     epoch 6, minibatch 195/195, test error of best model 78.024840 %\n",
      "('training @ iter = ', 1200)\n",
      "('training @ iter = ', 1300)\n",
      "epoch 7, minibatch 195/195, validation error 73.838141 %\n",
      "     epoch 7, minibatch 195/195, test error of best model 74.549279 %\n",
      "('training @ iter = ', 1400)\n",
      "('training @ iter = ', 1500)\n",
      "epoch 8, minibatch 195/195, validation error 70.022035 %\n",
      "     epoch 8, minibatch 195/195, test error of best model 71.304087 %\n",
      "('training @ iter = ', 1600)\n",
      "('training @ iter = ', 1700)\n",
      "epoch 9, minibatch 195/195, validation error 66.436298 %\n",
      "     epoch 9, minibatch 195/195, test error of best model 68.249199 %\n",
      "('training @ iter = ', 1800)\n",
      "('training @ iter = ', 1900)\n",
      "epoch 10, minibatch 195/195, validation error 63.351362 %\n",
      "     epoch 10, minibatch 195/195, test error of best model 65.104167 %\n",
      "('training @ iter = ', 2000)\n",
      "('training @ iter = ', 2100)\n",
      "epoch 11, minibatch 195/195, validation error 60.026042 %\n",
      "     epoch 11, minibatch 195/195, test error of best model 62.129407 %\n",
      "('training @ iter = ', 2200)\n",
      "('training @ iter = ', 2300)\n",
      "epoch 12, minibatch 195/195, validation error 57.211538 %\n",
      "     epoch 12, minibatch 195/195, test error of best model 59.044471 %\n",
      "('training @ iter = ', 2400)\n",
      "('training @ iter = ', 2500)\n",
      "epoch 13, minibatch 195/195, validation error 54.377003 %\n",
      "     epoch 13, minibatch 195/195, test error of best model 56.270032 %\n",
      "('training @ iter = ', 2600)\n",
      "('training @ iter = ', 2700)\n",
      "epoch 14, minibatch 195/195, validation error 51.492388 %\n",
      "     epoch 14, minibatch 195/195, test error of best model 53.405449 %\n",
      "('training @ iter = ', 2800)\n",
      "('training @ iter = ', 2900)\n",
      "epoch 15, minibatch 195/195, validation error 48.988381 %\n",
      "     epoch 15, minibatch 195/195, test error of best model 50.721154 %\n",
      "('training @ iter = ', 3000)\n",
      "('training @ iter = ', 3100)\n",
      "epoch 16, minibatch 195/195, validation error 47.425881 %\n",
      "     epoch 16, minibatch 195/195, test error of best model 48.447516 %\n",
      "('training @ iter = ', 3200)\n",
      "('training @ iter = ', 3300)\n",
      "epoch 17, minibatch 195/195, validation error 45.833333 %\n",
      "     epoch 17, minibatch 195/195, test error of best model 46.834936 %\n",
      "('training @ iter = ', 3400)\n",
      "('training @ iter = ', 3500)\n",
      "epoch 18, minibatch 195/195, validation error 44.421074 %\n",
      "     epoch 18, minibatch 195/195, test error of best model 45.032051 %\n",
      "('training @ iter = ', 3600)\n",
      "('training @ iter = ', 3700)\n",
      "epoch 19, minibatch 195/195, validation error 43.149038 %\n",
      "     epoch 19, minibatch 195/195, test error of best model 43.669872 %\n",
      "('training @ iter = ', 3800)\n",
      "epoch 20, minibatch 195/195, validation error 41.917067 %\n",
      "     epoch 20, minibatch 195/195, test error of best model 42.377804 %\n",
      "('training @ iter = ', 3900)\n",
      "('training @ iter = ', 4000)\n",
      "epoch 21, minibatch 195/195, validation error 40.625000 %\n",
      "     epoch 21, minibatch 195/195, test error of best model 41.235978 %\n",
      "('training @ iter = ', 4100)\n",
      "('training @ iter = ', 4200)\n",
      "epoch 22, minibatch 195/195, validation error 39.322917 %\n",
      "     epoch 22, minibatch 195/195, test error of best model 40.254407 %\n",
      "('training @ iter = ', 4300)\n",
      "('training @ iter = ', 4400)\n",
      "epoch 23, minibatch 195/195, validation error 38.281250 %\n",
      "     epoch 23, minibatch 195/195, test error of best model 38.982372 %\n",
      "('training @ iter = ', 4500)\n",
      "('training @ iter = ', 4600)\n",
      "epoch 24, minibatch 195/195, validation error 37.339744 %\n",
      "     epoch 24, minibatch 195/195, test error of best model 37.640224 %\n",
      "('training @ iter = ', 4700)\n",
      "('training @ iter = ', 4800)\n",
      "epoch 25, minibatch 195/195, validation error 36.187901 %\n",
      "     epoch 25, minibatch 195/195, test error of best model 36.418269 %\n",
      "('training @ iter = ', 4900)\n",
      "('training @ iter = ', 5000)\n",
      "epoch 26, minibatch 195/195, validation error 35.036058 %\n",
      "     epoch 26, minibatch 195/195, test error of best model 35.436699 %\n",
      "('training @ iter = ', 5100)\n",
      "('training @ iter = ', 5200)\n",
      "epoch 27, minibatch 195/195, validation error 33.944311 %\n",
      "     epoch 27, minibatch 195/195, test error of best model 34.385016 %\n",
      "('training @ iter = ', 5300)\n",
      "('training @ iter = ', 5400)\n",
      "epoch 28, minibatch 195/195, validation error 32.962740 %\n",
      "     epoch 28, minibatch 195/195, test error of best model 33.543670 %\n",
      "('training @ iter = ', 5500)\n",
      "('training @ iter = ', 5600)\n",
      "epoch 29, minibatch 195/195, validation error 31.911058 %\n",
      "     epoch 29, minibatch 195/195, test error of best model 32.461939 %\n",
      "('training @ iter = ', 5700)\n",
      "('training @ iter = ', 5800)\n",
      "epoch 30, minibatch 195/195, validation error 30.949519 %\n",
      "     epoch 30, minibatch 195/195, test error of best model 31.580529 %\n",
      "('training @ iter = ', 5900)\n",
      "('training @ iter = ', 6000)\n",
      "epoch 31, minibatch 195/195, validation error 30.338542 %\n",
      "     epoch 31, minibatch 195/195, test error of best model 30.979567 %\n",
      "('training @ iter = ', 6100)\n",
      "('training @ iter = ', 6200)\n",
      "epoch 32, minibatch 195/195, validation error 29.246795 %\n",
      "     epoch 32, minibatch 195/195, test error of best model 29.907853 %\n",
      "('training @ iter = ', 6300)\n",
      "('training @ iter = ', 6400)\n",
      "epoch 33, minibatch 195/195, validation error 28.395433 %\n",
      "     epoch 33, minibatch 195/195, test error of best model 29.166667 %\n",
      "('training @ iter = ', 6500)\n",
      "('training @ iter = ', 6600)\n",
      "epoch 34, minibatch 195/195, validation error 27.674279 %\n",
      "     epoch 34, minibatch 195/195, test error of best model 28.515625 %\n",
      "('training @ iter = ', 6700)\n",
      "('training @ iter = ', 6800)\n",
      "epoch 35, minibatch 195/195, validation error 26.963141 %\n",
      "     epoch 35, minibatch 195/195, test error of best model 27.944712 %\n",
      "('training @ iter = ', 6900)\n",
      "('training @ iter = ', 7000)\n",
      "epoch 36, minibatch 195/195, validation error 26.352163 %\n",
      "     epoch 36, minibatch 195/195, test error of best model 27.313702 %\n",
      "('training @ iter = ', 7100)\n",
      "('training @ iter = ', 7200)\n",
      "epoch 37, minibatch 195/195, validation error 25.711138 %\n",
      "     epoch 37, minibatch 195/195, test error of best model 26.842949 %\n",
      "('training @ iter = ', 7300)\n",
      "('training @ iter = ', 7400)\n",
      "epoch 38, minibatch 195/195, validation error 25.180288 %\n",
      "     epoch 38, minibatch 195/195, test error of best model 26.322115 %\n",
      "('training @ iter = ', 7500)\n",
      "('training @ iter = ', 7600)\n",
      "epoch 39, minibatch 195/195, validation error 24.679487 %\n",
      "     epoch 39, minibatch 195/195, test error of best model 25.751202 %\n",
      "('training @ iter = ', 7700)\n",
      "epoch 40, minibatch 195/195, validation error 24.278846 %\n",
      "     epoch 40, minibatch 195/195, test error of best model 25.380609 %\n",
      "('training @ iter = ', 7800)\n",
      "('training @ iter = ', 7900)\n",
      "epoch 41, minibatch 195/195, validation error 23.938301 %\n",
      "     epoch 41, minibatch 195/195, test error of best model 25.020032 %\n",
      "('training @ iter = ', 8000)\n",
      "('training @ iter = ', 8100)\n",
      "epoch 42, minibatch 195/195, validation error 23.587740 %\n",
      "     epoch 42, minibatch 195/195, test error of best model 24.409054 %\n",
      "('training @ iter = ', 8200)\n",
      "('training @ iter = ', 8300)\n",
      "epoch 43, minibatch 195/195, validation error 23.227163 %\n",
      "     epoch 43, minibatch 195/195, test error of best model 24.018429 %\n",
      "('training @ iter = ', 8400)\n",
      "('training @ iter = ', 8500)\n",
      "epoch 44, minibatch 195/195, validation error 22.906651 %\n",
      "     epoch 44, minibatch 195/195, test error of best model 23.627804 %\n",
      "('training @ iter = ', 8600)\n",
      "('training @ iter = ', 8700)\n",
      "epoch 45, minibatch 195/195, validation error 22.606170 %\n",
      "     epoch 45, minibatch 195/195, test error of best model 23.427484 %\n",
      "('training @ iter = ', 8800)\n",
      "('training @ iter = ', 8900)\n",
      "epoch 46, minibatch 195/195, validation error 22.315705 %\n",
      "     epoch 46, minibatch 195/195, test error of best model 22.916667 %\n",
      "('training @ iter = ', 9000)\n",
      "('training @ iter = ', 9100)\n",
      "epoch 47, minibatch 195/195, validation error 22.035256 %\n",
      "     epoch 47, minibatch 195/195, test error of best model 22.576122 %\n",
      "('training @ iter = ', 9200)\n",
      "('training @ iter = ', 9300)\n",
      "epoch 48, minibatch 195/195, validation error 21.744792 %\n",
      "     epoch 48, minibatch 195/195, test error of best model 22.275641 %\n",
      "('training @ iter = ', 9400)\n",
      "('training @ iter = ', 9500)\n",
      "epoch 49, minibatch 195/195, validation error 21.334135 %\n",
      "     epoch 49, minibatch 195/195, test error of best model 21.885016 %\n",
      "('training @ iter = ', 9600)\n",
      "('training @ iter = ', 9700)\n",
      "epoch 50, minibatch 195/195, validation error 21.093750 %\n",
      "     epoch 50, minibatch 195/195, test error of best model 21.644631 %\n",
      "('training @ iter = ', 9800)\n",
      "('training @ iter = ', 9900)\n",
      "epoch 51, minibatch 195/195, validation error 20.813301 %\n",
      "     epoch 51, minibatch 195/195, test error of best model 21.344151 %\n",
      "('training @ iter = ', 10000)\n",
      "('training @ iter = ', 10100)\n",
      "epoch 52, minibatch 195/195, validation error 20.442708 %\n",
      "     epoch 52, minibatch 195/195, test error of best model 20.993590 %\n",
      "('training @ iter = ', 10200)\n",
      "('training @ iter = ', 10300)\n",
      "epoch 53, minibatch 195/195, validation error 20.192308 %\n",
      "     epoch 53, minibatch 195/195, test error of best model 20.733173 %\n",
      "('training @ iter = ', 10400)\n",
      "('training @ iter = ', 10500)\n",
      "epoch 54, minibatch 195/195, validation error 19.851763 %\n",
      "     epoch 54, minibatch 195/195, test error of best model 20.602965 %\n",
      "('training @ iter = ', 10600)\n",
      "('training @ iter = ', 10700)\n",
      "epoch 55, minibatch 195/195, validation error 19.641426 %\n",
      "     epoch 55, minibatch 195/195, test error of best model 20.182292 %\n",
      "('training @ iter = ', 10800)\n",
      "('training @ iter = ', 10900)\n",
      "epoch 56, minibatch 195/195, validation error 19.481170 %\n",
      "     epoch 56, minibatch 195/195, test error of best model 20.002003 %\n",
      "('training @ iter = ', 11000)\n",
      "('training @ iter = ', 11100)\n",
      "epoch 57, minibatch 195/195, validation error 19.220753 %\n",
      "     epoch 57, minibatch 195/195, test error of best model 19.821715 %\n",
      "('training @ iter = ', 11200)\n",
      "('training @ iter = ', 11300)\n",
      "epoch 58, minibatch 195/195, validation error 19.070513 %\n",
      "     epoch 58, minibatch 195/195, test error of best model 19.611378 %\n",
      "('training @ iter = ', 11400)\n",
      "('training @ iter = ', 11500)\n",
      "epoch 59, minibatch 195/195, validation error 18.850160 %\n",
      "     epoch 59, minibatch 195/195, test error of best model 19.391026 %\n",
      "('training @ iter = ', 11600)\n",
      "epoch 60, minibatch 195/195, validation error 18.599760 %\n",
      "     epoch 60, minibatch 195/195, test error of best model 19.050481 %\n",
      "('training @ iter = ', 11700)\n",
      "('training @ iter = ', 11800)\n",
      "epoch 61, minibatch 195/195, validation error 18.399439 %\n",
      "     epoch 61, minibatch 195/195, test error of best model 18.820112 %\n",
      "('training @ iter = ', 11900)\n",
      "('training @ iter = ', 12000)\n",
      "epoch 62, minibatch 195/195, validation error 18.259215 %\n",
      "     epoch 62, minibatch 195/195, test error of best model 18.589744 %\n",
      "('training @ iter = ', 12100)\n",
      "('training @ iter = ', 12200)\n",
      "epoch 63, minibatch 195/195, validation error 18.108974 %\n",
      "     epoch 63, minibatch 195/195, test error of best model 18.379407 %\n",
      "('training @ iter = ', 12300)\n",
      "('training @ iter = ', 12400)\n",
      "epoch 64, minibatch 195/195, validation error 17.848558 %\n",
      "     epoch 64, minibatch 195/195, test error of best model 18.149038 %\n",
      "('training @ iter = ', 12500)\n",
      "('training @ iter = ', 12600)\n",
      "epoch 65, minibatch 195/195, validation error 17.718349 %\n",
      "     epoch 65, minibatch 195/195, test error of best model 17.838542 %\n",
      "('training @ iter = ', 12700)\n",
      "('training @ iter = ', 12800)\n",
      "epoch 66, minibatch 195/195, validation error 17.698317 %\n",
      "     epoch 66, minibatch 195/195, test error of best model 17.598157 %\n",
      "('training @ iter = ', 12900)\n",
      "('training @ iter = ', 13000)\n",
      "epoch 67, minibatch 195/195, validation error 17.518029 %\n",
      "     epoch 67, minibatch 195/195, test error of best model 17.497997 %\n",
      "('training @ iter = ', 13100)\n",
      "('training @ iter = ', 13200)\n",
      "epoch 68, minibatch 195/195, validation error 17.347756 %\n",
      "     epoch 68, minibatch 195/195, test error of best model 17.287660 %\n",
      "('training @ iter = ', 13300)\n",
      "('training @ iter = ', 13400)\n",
      "epoch 69, minibatch 195/195, validation error 17.107372 %\n",
      "     epoch 69, minibatch 195/195, test error of best model 17.187500 %\n",
      "('training @ iter = ', 13500)\n",
      "('training @ iter = ', 13600)\n",
      "epoch 70, minibatch 195/195, validation error 17.027244 %\n",
      "     epoch 70, minibatch 195/195, test error of best model 17.027244 %\n",
      "('training @ iter = ', 13700)\n",
      "('training @ iter = ', 13800)\n",
      "epoch 71, minibatch 195/195, validation error 16.866987 %\n",
      "     epoch 71, minibatch 195/195, test error of best model 16.866987 %\n",
      "('training @ iter = ', 13900)\n",
      "('training @ iter = ', 14000)\n",
      "epoch 72, minibatch 195/195, validation error 16.706731 %\n",
      "     epoch 72, minibatch 195/195, test error of best model 16.766827 %\n",
      "('training @ iter = ', 14100)\n",
      "('training @ iter = ', 14200)\n",
      "epoch 73, minibatch 195/195, validation error 16.606571 %\n",
      "     epoch 73, minibatch 195/195, test error of best model 16.576522 %\n",
      "('training @ iter = ', 14300)\n",
      "('training @ iter = ', 14400)\n",
      "epoch 74, minibatch 195/195, validation error 16.416266 %\n",
      "     epoch 74, minibatch 195/195, test error of best model 16.436298 %\n",
      "('training @ iter = ', 14500)\n",
      "('training @ iter = ', 14600)\n",
      "epoch 75, minibatch 195/195, validation error 16.245994 %\n",
      "     epoch 75, minibatch 195/195, test error of best model 16.266026 %\n",
      "('training @ iter = ', 14700)\n",
      "('training @ iter = ', 14800)\n",
      "epoch 76, minibatch 195/195, validation error 16.155849 %\n",
      "     epoch 76, minibatch 195/195, test error of best model 16.075721 %\n",
      "('training @ iter = ', 14900)\n",
      "('training @ iter = ', 15000)\n",
      "epoch 77, minibatch 195/195, validation error 16.045673 %\n",
      "     epoch 77, minibatch 195/195, test error of best model 15.985577 %\n",
      "('training @ iter = ', 15100)\n",
      "('training @ iter = ', 15200)\n",
      "epoch 78, minibatch 195/195, validation error 15.915465 %\n",
      "     epoch 78, minibatch 195/195, test error of best model 15.815304 %\n",
      "('training @ iter = ', 15300)\n",
      "('training @ iter = ', 15400)\n",
      "epoch 79, minibatch 195/195, validation error 15.825321 %\n",
      "     epoch 79, minibatch 195/195, test error of best model 15.655048 %\n",
      "('training @ iter = ', 15500)\n",
      "epoch 80, minibatch 195/195, validation error 15.655048 %\n",
      "     epoch 80, minibatch 195/195, test error of best model 15.484776 %\n",
      "('training @ iter = ', 15600)\n",
      "('training @ iter = ', 15700)\n",
      "epoch 81, minibatch 195/195, validation error 15.534856 %\n",
      "     epoch 81, minibatch 195/195, test error of best model 15.314503 %\n",
      "('training @ iter = ', 15800)\n",
      "('training @ iter = ', 15900)\n",
      "epoch 82, minibatch 195/195, validation error 15.504808 %\n",
      "     epoch 82, minibatch 195/195, test error of best model 15.234375 %\n",
      "('training @ iter = ', 16000)\n",
      "('training @ iter = ', 16100)\n",
      "epoch 83, minibatch 195/195, validation error 15.424679 %\n",
      "     epoch 83, minibatch 195/195, test error of best model 15.114183 %\n",
      "('training @ iter = ', 16200)\n",
      "('training @ iter = ', 16300)\n",
      "epoch 84, minibatch 195/195, validation error 15.274439 %\n",
      "     epoch 84, minibatch 195/195, test error of best model 14.993990 %\n",
      "('training @ iter = ', 16400)\n",
      "('training @ iter = ', 16500)\n",
      "epoch 85, minibatch 195/195, validation error 15.134215 %\n",
      "     epoch 85, minibatch 195/195, test error of best model 14.883814 %\n",
      "('training @ iter = ', 16600)\n",
      "('training @ iter = ', 16700)\n",
      "epoch 86, minibatch 195/195, validation error 15.034054 %\n",
      "     epoch 86, minibatch 195/195, test error of best model 14.763622 %\n",
      "('training @ iter = ', 16800)\n",
      "('training @ iter = ', 16900)\n",
      "epoch 87, minibatch 195/195, validation error 14.903846 %\n",
      "     epoch 87, minibatch 195/195, test error of best model 14.653446 %\n",
      "('training @ iter = ', 17000)\n",
      "('training @ iter = ', 17100)\n",
      "epoch 88, minibatch 195/195, validation error 14.863782 %\n",
      "     epoch 88, minibatch 195/195, test error of best model 14.513221 %\n",
      "('training @ iter = ', 17200)\n",
      "('training @ iter = ', 17300)\n",
      "epoch 89, minibatch 195/195, validation error 14.763622 %\n",
      "     epoch 89, minibatch 195/195, test error of best model 14.403045 %\n",
      "('training @ iter = ', 17400)\n",
      "('training @ iter = ', 17500)\n",
      "epoch 90, minibatch 195/195, validation error 14.643429 %\n",
      "     epoch 90, minibatch 195/195, test error of best model 14.403045 %\n",
      "('training @ iter = ', 17600)\n",
      "('training @ iter = ', 17700)\n",
      "epoch 91, minibatch 195/195, validation error 14.603365 %\n",
      "     epoch 91, minibatch 195/195, test error of best model 14.272837 %\n",
      "('training @ iter = ', 17800)\n",
      "('training @ iter = ', 17900)\n",
      "epoch 92, minibatch 195/195, validation error 14.563301 %\n",
      "     epoch 92, minibatch 195/195, test error of best model 14.202724 %\n",
      "('training @ iter = ', 18000)\n",
      "('training @ iter = ', 18100)\n",
      "epoch 93, minibatch 195/195, validation error 14.493189 %\n",
      "     epoch 93, minibatch 195/195, test error of best model 14.082532 %\n",
      "('training @ iter = ', 18200)\n",
      "('training @ iter = ', 18300)\n",
      "epoch 94, minibatch 195/195, validation error 14.413061 %\n",
      "     epoch 94, minibatch 195/195, test error of best model 14.012420 %\n",
      "('training @ iter = ', 18400)\n",
      "('training @ iter = ', 18500)\n",
      "epoch 95, minibatch 195/195, validation error 14.342949 %\n",
      "     epoch 95, minibatch 195/195, test error of best model 13.912260 %\n",
      "('training @ iter = ', 18600)\n",
      "('training @ iter = ', 18700)\n",
      "epoch 96, minibatch 195/195, validation error 14.292869 %\n",
      "     epoch 96, minibatch 195/195, test error of best model 13.952324 %\n",
      "('training @ iter = ', 18800)\n",
      "('training @ iter = ', 18900)\n",
      "epoch 97, minibatch 195/195, validation error 14.222756 %\n",
      "     epoch 97, minibatch 195/195, test error of best model 13.842147 %\n",
      "('training @ iter = ', 19000)\n",
      "('training @ iter = ', 19100)\n",
      "epoch 98, minibatch 195/195, validation error 14.152644 %\n",
      "     epoch 98, minibatch 195/195, test error of best model 13.822115 %\n",
      "('training @ iter = ', 19200)\n",
      "('training @ iter = ', 19300)\n",
      "epoch 99, minibatch 195/195, validation error 14.062500 %\n",
      "     epoch 99, minibatch 195/195, test error of best model 13.741987 %\n",
      "('training @ iter = ', 19400)\n",
      "epoch 100, minibatch 195/195, validation error 14.002404 %\n",
      "     epoch 100, minibatch 195/195, test error of best model 13.681891 %\n",
      "('training @ iter = ', 19500)\n",
      "('training @ iter = ', 19600)\n",
      "epoch 101, minibatch 195/195, validation error 13.922276 %\n",
      "     epoch 101, minibatch 195/195, test error of best model 13.651843 %\n",
      "('training @ iter = ', 19700)\n",
      "('training @ iter = ', 19800)\n",
      "epoch 102, minibatch 195/195, validation error 13.892228 %\n",
      "     epoch 102, minibatch 195/195, test error of best model 13.581731 %\n",
      "('training @ iter = ', 19900)\n",
      "('training @ iter = ', 20000)\n",
      "epoch 103, minibatch 195/195, validation error 13.932292 %\n",
      "('training @ iter = ', 20100)\n",
      "('training @ iter = ', 20200)\n",
      "epoch 104, minibatch 195/195, validation error 13.932292 %\n",
      "('training @ iter = ', 20300)\n",
      "('training @ iter = ', 20400)\n",
      "epoch 105, minibatch 195/195, validation error 13.862179 %\n",
      "     epoch 105, minibatch 195/195, test error of best model 13.421474 %\n",
      "('training @ iter = ', 20500)\n",
      "('training @ iter = ', 20600)\n",
      "epoch 106, minibatch 195/195, validation error 13.772035 %\n",
      "     epoch 106, minibatch 195/195, test error of best model 13.411458 %\n",
      "('training @ iter = ', 20700)\n",
      "('training @ iter = ', 20800)\n",
      "epoch 107, minibatch 195/195, validation error 13.681891 %\n",
      "     epoch 107, minibatch 195/195, test error of best model 13.371394 %\n",
      "('training @ iter = ', 20900)\n",
      "('training @ iter = ', 21000)\n",
      "epoch 108, minibatch 195/195, validation error 13.571715 %\n",
      "     epoch 108, minibatch 195/195, test error of best model 13.411458 %\n",
      "('training @ iter = ', 21100)\n",
      "('training @ iter = ', 21200)\n",
      "epoch 109, minibatch 195/195, validation error 13.501603 %\n",
      "     epoch 109, minibatch 195/195, test error of best model 13.361378 %\n",
      "('training @ iter = ', 21300)\n",
      "('training @ iter = ', 21400)\n",
      "epoch 110, minibatch 195/195, validation error 13.381410 %\n",
      "     epoch 110, minibatch 195/195, test error of best model 13.251202 %\n",
      "('training @ iter = ', 21500)\n",
      "('training @ iter = ', 21600)\n",
      "epoch 111, minibatch 195/195, validation error 13.371394 %\n",
      "     epoch 111, minibatch 195/195, test error of best model 13.181090 %\n",
      "('training @ iter = ', 21700)\n",
      "('training @ iter = ', 21800)\n",
      "epoch 112, minibatch 195/195, validation error 13.261218 %\n",
      "     epoch 112, minibatch 195/195, test error of best model 13.161058 %\n",
      "('training @ iter = ', 21900)\n",
      "('training @ iter = ', 22000)\n",
      "epoch 113, minibatch 195/195, validation error 13.201122 %\n",
      "     epoch 113, minibatch 195/195, test error of best model 13.141026 %\n",
      "('training @ iter = ', 22100)\n",
      "('training @ iter = ', 22200)\n",
      "epoch 114, minibatch 195/195, validation error 13.141026 %\n",
      "     epoch 114, minibatch 195/195, test error of best model 13.120994 %\n",
      "('training @ iter = ', 22300)\n",
      "('training @ iter = ', 22400)\n",
      "epoch 115, minibatch 195/195, validation error 13.141026 %\n",
      "('training @ iter = ', 22500)\n",
      "('training @ iter = ', 22600)\n",
      "epoch 116, minibatch 195/195, validation error 13.100962 %\n",
      "     epoch 116, minibatch 195/195, test error of best model 13.040865 %\n",
      "('training @ iter = ', 22700)\n",
      "('training @ iter = ', 22800)\n",
      "epoch 117, minibatch 195/195, validation error 13.040865 %\n",
      "     epoch 117, minibatch 195/195, test error of best model 13.000801 %\n",
      "('training @ iter = ', 22900)\n",
      "('training @ iter = ', 23000)\n",
      "epoch 118, minibatch 195/195, validation error 12.920673 %\n",
      "     epoch 118, minibatch 195/195, test error of best model 12.950721 %\n",
      "('training @ iter = ', 23100)\n",
      "('training @ iter = ', 23200)\n",
      "epoch 119, minibatch 195/195, validation error 12.860577 %\n",
      "     epoch 119, minibatch 195/195, test error of best model 12.940705 %\n",
      "('training @ iter = ', 23300)\n",
      "epoch 120, minibatch 195/195, validation error 12.760417 %\n",
      "     epoch 120, minibatch 195/195, test error of best model 12.940705 %\n",
      "('training @ iter = ', 23400)\n",
      "('training @ iter = ', 23500)\n",
      "epoch 121, minibatch 195/195, validation error 12.710337 %\n",
      "     epoch 121, minibatch 195/195, test error of best model 12.870593 %\n",
      "('training @ iter = ', 23600)\n",
      "('training @ iter = ', 23700)\n",
      "epoch 122, minibatch 195/195, validation error 12.660256 %\n",
      "     epoch 122, minibatch 195/195, test error of best model 12.860577 %\n",
      "('training @ iter = ', 23800)\n",
      "('training @ iter = ', 23900)\n",
      "epoch 123, minibatch 195/195, validation error 12.600160 %\n",
      "     epoch 123, minibatch 195/195, test error of best model 12.810497 %\n",
      "('training @ iter = ', 24000)\n",
      "('training @ iter = ', 24100)\n",
      "epoch 124, minibatch 195/195, validation error 12.520032 %\n",
      "     epoch 124, minibatch 195/195, test error of best model 12.770433 %\n",
      "('training @ iter = ', 24200)\n",
      "('training @ iter = ', 24300)\n",
      "epoch 125, minibatch 195/195, validation error 12.449920 %\n",
      "     epoch 125, minibatch 195/195, test error of best model 12.700321 %\n",
      "('training @ iter = ', 24400)\n",
      "('training @ iter = ', 24500)\n",
      "epoch 126, minibatch 195/195, validation error 12.409856 %\n",
      "     epoch 126, minibatch 195/195, test error of best model 12.580128 %\n",
      "('training @ iter = ', 24600)\n",
      "('training @ iter = ', 24700)\n",
      "epoch 127, minibatch 195/195, validation error 12.379808 %\n",
      "     epoch 127, minibatch 195/195, test error of best model 12.550080 %\n",
      "('training @ iter = ', 24800)\n",
      "('training @ iter = ', 24900)\n",
      "epoch 128, minibatch 195/195, validation error 12.359776 %\n",
      "     epoch 128, minibatch 195/195, test error of best model 12.500000 %\n",
      "('training @ iter = ', 25000)\n",
      "('training @ iter = ', 25100)\n",
      "epoch 129, minibatch 195/195, validation error 12.299679 %\n",
      "     epoch 129, minibatch 195/195, test error of best model 12.479968 %\n",
      "('training @ iter = ', 25200)\n",
      "('training @ iter = ', 25300)\n",
      "epoch 130, minibatch 195/195, validation error 12.299679 %\n",
      "('training @ iter = ', 25400)\n",
      "('training @ iter = ', 25500)\n",
      "epoch 131, minibatch 195/195, validation error 12.229567 %\n",
      "     epoch 131, minibatch 195/195, test error of best model 12.429888 %\n",
      "('training @ iter = ', 25600)\n",
      "('training @ iter = ', 25700)\n",
      "epoch 132, minibatch 195/195, validation error 12.169471 %\n",
      "     epoch 132, minibatch 195/195, test error of best model 12.409856 %\n",
      "('training @ iter = ', 25800)\n",
      "('training @ iter = ', 25900)\n",
      "epoch 133, minibatch 195/195, validation error 12.129407 %\n",
      "     epoch 133, minibatch 195/195, test error of best model 12.419872 %\n",
      "('training @ iter = ', 26000)\n",
      "('training @ iter = ', 26100)\n",
      "epoch 134, minibatch 195/195, validation error 12.149439 %\n",
      "('training @ iter = ', 26200)\n",
      "('training @ iter = ', 26300)\n",
      "epoch 135, minibatch 195/195, validation error 12.099359 %\n",
      "     epoch 135, minibatch 195/195, test error of best model 12.339744 %\n",
      "('training @ iter = ', 26400)\n",
      "('training @ iter = ', 26500)\n",
      "epoch 136, minibatch 195/195, validation error 12.079327 %\n",
      "     epoch 136, minibatch 195/195, test error of best model 12.309696 %\n",
      "('training @ iter = ', 26600)\n",
      "('training @ iter = ', 26700)\n",
      "epoch 137, minibatch 195/195, validation error 12.039263 %\n",
      "     epoch 137, minibatch 195/195, test error of best model 12.199519 %\n",
      "('training @ iter = ', 26800)\n",
      "('training @ iter = ', 26900)\n",
      "epoch 138, minibatch 195/195, validation error 11.989183 %\n",
      "     epoch 138, minibatch 195/195, test error of best model 12.149439 %\n",
      "('training @ iter = ', 27000)\n",
      "('training @ iter = ', 27100)\n",
      "epoch 139, minibatch 195/195, validation error 11.969151 %\n",
      "     epoch 139, minibatch 195/195, test error of best model 12.069311 %\n",
      "('training @ iter = ', 27200)\n",
      "epoch 140, minibatch 195/195, validation error 11.969151 %\n",
      "('training @ iter = ', 27300)\n",
      "('training @ iter = ', 27400)\n",
      "epoch 141, minibatch 195/195, validation error 11.949119 %\n",
      "     epoch 141, minibatch 195/195, test error of best model 12.009215 %\n",
      "('training @ iter = ', 27500)\n",
      "('training @ iter = ', 27600)\n",
      "epoch 142, minibatch 195/195, validation error 11.899038 %\n",
      "     epoch 142, minibatch 195/195, test error of best model 11.959135 %\n",
      "('training @ iter = ', 27700)\n",
      "('training @ iter = ', 27800)\n",
      "epoch 143, minibatch 195/195, validation error 11.858974 %\n",
      "     epoch 143, minibatch 195/195, test error of best model 11.919071 %\n",
      "('training @ iter = ', 27900)\n",
      "('training @ iter = ', 28000)\n",
      "epoch 144, minibatch 195/195, validation error 11.818910 %\n",
      "     epoch 144, minibatch 195/195, test error of best model 11.919071 %\n",
      "('training @ iter = ', 28100)\n",
      "('training @ iter = ', 28200)\n",
      "epoch 145, minibatch 195/195, validation error 11.788862 %\n",
      "     epoch 145, minibatch 195/195, test error of best model 11.879006 %\n",
      "('training @ iter = ', 28300)\n",
      "('training @ iter = ', 28400)\n",
      "epoch 146, minibatch 195/195, validation error 11.748798 %\n",
      "     epoch 146, minibatch 195/195, test error of best model 11.868990 %\n",
      "('training @ iter = ', 28500)\n",
      "('training @ iter = ', 28600)\n",
      "epoch 147, minibatch 195/195, validation error 11.678686 %\n",
      "     epoch 147, minibatch 195/195, test error of best model 11.858974 %\n",
      "('training @ iter = ', 28700)\n",
      "('training @ iter = ', 28800)\n",
      "epoch 148, minibatch 195/195, validation error 11.638622 %\n",
      "     epoch 148, minibatch 195/195, test error of best model 11.838942 %\n",
      "('training @ iter = ', 28900)\n",
      "('training @ iter = ', 29000)\n",
      "epoch 149, minibatch 195/195, validation error 11.608574 %\n",
      "     epoch 149, minibatch 195/195, test error of best model 11.798878 %\n",
      "('training @ iter = ', 29100)\n",
      "('training @ iter = ', 29200)\n",
      "epoch 150, minibatch 195/195, validation error 11.608574 %\n",
      "('training @ iter = ', 29300)\n",
      "('training @ iter = ', 29400)\n",
      "epoch 151, minibatch 195/195, validation error 11.578526 %\n",
      "     epoch 151, minibatch 195/195, test error of best model 11.748798 %\n",
      "('training @ iter = ', 29500)\n",
      "('training @ iter = ', 29600)\n",
      "epoch 152, minibatch 195/195, validation error 11.518429 %\n",
      "     epoch 152, minibatch 195/195, test error of best model 11.738782 %\n",
      "('training @ iter = ', 29700)\n",
      "('training @ iter = ', 29800)\n",
      "epoch 153, minibatch 195/195, validation error 11.458333 %\n",
      "     epoch 153, minibatch 195/195, test error of best model 11.688702 %\n",
      "('training @ iter = ', 29900)\n",
      "('training @ iter = ', 30000)\n",
      "epoch 154, minibatch 195/195, validation error 11.448317 %\n",
      "     epoch 154, minibatch 195/195, test error of best model 11.678686 %\n",
      "('training @ iter = ', 30100)\n",
      "('training @ iter = ', 30200)\n",
      "epoch 155, minibatch 195/195, validation error 11.448317 %\n",
      "('training @ iter = ', 30300)\n",
      "('training @ iter = ', 30400)\n",
      "epoch 156, minibatch 195/195, validation error 11.418269 %\n",
      "     epoch 156, minibatch 195/195, test error of best model 11.588542 %\n",
      "('training @ iter = ', 30500)\n",
      "('training @ iter = ', 30600)\n",
      "epoch 157, minibatch 195/195, validation error 11.408253 %\n",
      "     epoch 157, minibatch 195/195, test error of best model 11.548478 %\n",
      "('training @ iter = ', 30700)\n",
      "('training @ iter = ', 30800)\n",
      "epoch 158, minibatch 195/195, validation error 11.408253 %\n",
      "('training @ iter = ', 30900)\n",
      "('training @ iter = ', 31000)\n",
      "epoch 159, minibatch 195/195, validation error 11.408253 %\n",
      "('training @ iter = ', 31100)\n",
      "epoch 160, minibatch 195/195, validation error 11.418269 %\n",
      "('training @ iter = ', 31200)\n",
      "('training @ iter = ', 31300)\n",
      "epoch 161, minibatch 195/195, validation error 11.408253 %\n",
      "('training @ iter = ', 31400)\n",
      "('training @ iter = ', 31500)\n",
      "epoch 162, minibatch 195/195, validation error 11.368189 %\n",
      "     epoch 162, minibatch 195/195, test error of best model 11.448317 %\n",
      "('training @ iter = ', 31600)\n",
      "('training @ iter = ', 31700)\n",
      "epoch 163, minibatch 195/195, validation error 11.368189 %\n",
      "('training @ iter = ', 31800)\n",
      "('training @ iter = ', 31900)\n",
      "epoch 164, minibatch 195/195, validation error 11.348157 %\n",
      "     epoch 164, minibatch 195/195, test error of best model 11.408253 %\n",
      "('training @ iter = ', 32000)\n",
      "('training @ iter = ', 32100)\n",
      "epoch 165, minibatch 195/195, validation error 11.348157 %\n",
      "('training @ iter = ', 32200)\n",
      "('training @ iter = ', 32300)\n",
      "epoch 166, minibatch 195/195, validation error 11.348157 %\n",
      "('training @ iter = ', 32400)\n",
      "('training @ iter = ', 32500)\n",
      "epoch 167, minibatch 195/195, validation error 11.278045 %\n",
      "     epoch 167, minibatch 195/195, test error of best model 11.338141 %\n",
      "('training @ iter = ', 32600)\n",
      "('training @ iter = ', 32700)\n",
      "epoch 168, minibatch 195/195, validation error 11.258013 %\n",
      "     epoch 168, minibatch 195/195, test error of best model 11.308093 %\n",
      "('training @ iter = ', 32800)\n",
      "('training @ iter = ', 32900)\n",
      "epoch 169, minibatch 195/195, validation error 11.207933 %\n",
      "     epoch 169, minibatch 195/195, test error of best model 11.368189 %\n",
      "('training @ iter = ', 33000)\n",
      "('training @ iter = ', 33100)\n",
      "epoch 170, minibatch 195/195, validation error 11.157853 %\n",
      "     epoch 170, minibatch 195/195, test error of best model 11.338141 %\n",
      "('training @ iter = ', 33200)\n",
      "('training @ iter = ', 33300)\n",
      "epoch 171, minibatch 195/195, validation error 11.117788 %\n",
      "     epoch 171, minibatch 195/195, test error of best model 11.338141 %\n",
      "('training @ iter = ', 33400)\n",
      "('training @ iter = ', 33500)\n",
      "epoch 172, minibatch 195/195, validation error 11.077724 %\n",
      "     epoch 172, minibatch 195/195, test error of best model 11.328125 %\n",
      "('training @ iter = ', 33600)\n",
      "('training @ iter = ', 33700)\n",
      "epoch 173, minibatch 195/195, validation error 11.087740 %\n",
      "('training @ iter = ', 33800)\n",
      "('training @ iter = ', 33900)\n",
      "epoch 174, minibatch 195/195, validation error 11.077724 %\n",
      "('training @ iter = ', 34000)\n",
      "('training @ iter = ', 34100)\n",
      "epoch 175, minibatch 195/195, validation error 11.087740 %\n",
      "('training @ iter = ', 34200)\n",
      "('training @ iter = ', 34300)\n",
      "epoch 176, minibatch 195/195, validation error 11.097756 %\n",
      "('training @ iter = ', 34400)\n",
      "('training @ iter = ', 34500)\n",
      "epoch 177, minibatch 195/195, validation error 11.057692 %\n",
      "     epoch 177, minibatch 195/195, test error of best model 11.187901 %\n",
      "('training @ iter = ', 34600)\n",
      "('training @ iter = ', 34700)\n",
      "epoch 178, minibatch 195/195, validation error 11.077724 %\n",
      "('training @ iter = ', 34800)\n",
      "('training @ iter = ', 34900)\n",
      "epoch 179, minibatch 195/195, validation error 11.067708 %\n",
      "('training @ iter = ', 35000)\n",
      "epoch 180, minibatch 195/195, validation error 11.057692 %\n",
      "('training @ iter = ', 35100)\n",
      "('training @ iter = ', 35200)\n",
      "epoch 181, minibatch 195/195, validation error 11.037660 %\n",
      "     epoch 181, minibatch 195/195, test error of best model 11.037660 %\n",
      "('training @ iter = ', 35300)\n",
      "('training @ iter = ', 35400)\n",
      "epoch 182, minibatch 195/195, validation error 10.977564 %\n",
      "     epoch 182, minibatch 195/195, test error of best model 11.027644 %\n",
      "('training @ iter = ', 35500)\n",
      "('training @ iter = ', 35600)\n",
      "epoch 183, minibatch 195/195, validation error 10.947516 %\n",
      "     epoch 183, minibatch 195/195, test error of best model 11.047676 %\n",
      "('training @ iter = ', 35700)\n",
      "('training @ iter = ', 35800)\n",
      "epoch 184, minibatch 195/195, validation error 10.937500 %\n",
      "     epoch 184, minibatch 195/195, test error of best model 11.037660 %\n",
      "('training @ iter = ', 35900)\n",
      "('training @ iter = ', 36000)\n",
      "epoch 185, minibatch 195/195, validation error 10.917468 %\n",
      "     epoch 185, minibatch 195/195, test error of best model 11.047676 %\n",
      "('training @ iter = ', 36100)\n",
      "('training @ iter = ', 36200)\n",
      "epoch 186, minibatch 195/195, validation error 10.937500 %\n",
      "('training @ iter = ', 36300)\n",
      "('training @ iter = ', 36400)\n",
      "epoch 187, minibatch 195/195, validation error 10.937500 %\n",
      "('training @ iter = ', 36500)\n",
      "('training @ iter = ', 36600)\n",
      "epoch 188, minibatch 195/195, validation error 10.927484 %\n",
      "('training @ iter = ', 36700)\n",
      "('training @ iter = ', 36800)\n",
      "epoch 189, minibatch 195/195, validation error 10.957532 %\n",
      "('training @ iter = ', 36900)\n",
      "('training @ iter = ', 37000)\n",
      "epoch 190, minibatch 195/195, validation error 10.937500 %\n",
      "('training @ iter = ', 37100)\n",
      "('training @ iter = ', 37200)\n",
      "epoch 191, minibatch 195/195, validation error 10.917468 %\n",
      "('training @ iter = ', 37300)\n",
      "('training @ iter = ', 37400)\n",
      "epoch 192, minibatch 195/195, validation error 10.907452 %\n",
      "     epoch 192, minibatch 195/195, test error of best model 10.947516 %\n",
      "('training @ iter = ', 37500)\n",
      "('training @ iter = ', 37600)\n",
      "epoch 193, minibatch 195/195, validation error 10.857372 %\n",
      "     epoch 193, minibatch 195/195, test error of best model 10.947516 %\n",
      "('training @ iter = ', 37700)\n",
      "('training @ iter = ', 37800)\n",
      "epoch 194, minibatch 195/195, validation error 10.827324 %\n",
      "     epoch 194, minibatch 195/195, test error of best model 10.927484 %\n",
      "('training @ iter = ', 37900)\n",
      "('training @ iter = ', 38000)\n",
      "epoch 195, minibatch 195/195, validation error 10.817308 %\n",
      "     epoch 195, minibatch 195/195, test error of best model 10.927484 %\n",
      "('training @ iter = ', 38100)\n",
      "('training @ iter = ', 38200)\n",
      "epoch 196, minibatch 195/195, validation error 10.807292 %\n",
      "     epoch 196, minibatch 195/195, test error of best model 10.947516 %\n",
      "('training @ iter = ', 38300)\n",
      "('training @ iter = ', 38400)\n",
      "epoch 197, minibatch 195/195, validation error 10.817308 %\n",
      "('training @ iter = ', 38500)\n",
      "('training @ iter = ', 38600)\n",
      "epoch 198, minibatch 195/195, validation error 10.817308 %\n",
      "('training @ iter = ', 38700)\n",
      "('training @ iter = ', 38800)\n",
      "epoch 199, minibatch 195/195, validation error 10.807292 %\n",
      "('training @ iter = ', 38900)\n",
      "epoch 200, minibatch 195/195, validation error 10.787260 %\n",
      "     epoch 200, minibatch 195/195, test error of best model 10.937500 %\n",
      "Optimization complete.\n",
      "Best validation score of 10.787260 % obtained at iteration 39000, with test performance 10.937500 %\n",
      "The training process for function run ran for 3.83m\n",
      "----------------------------------------------------------------------\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "run(network_type = 'fcn', include_st = False, transformation = proj_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "...loading the dataset\n",
      "...Applying proj_transform to dataset\n",
      "Current training data size is 50000\n",
      "Current validation data size is 10000\n",
      "Current test data size is 10000\n",
      "...building the model\n",
      "...Apply STN before CNN\n",
      "...Init localization to identity\n",
      "...training\n",
      "('training @ iter = ', 0)\n",
      "('training @ iter = ', 100)\n",
      "epoch 1, minibatch 195/195, validation error 89.683494 %\n",
      "     epoch 1, minibatch 195/195, test error of best model 89.733574 %\n",
      "('training @ iter = ', 200)\n",
      "('training @ iter = ', 300)\n",
      "epoch 2, minibatch 195/195, validation error 88.912260 %\n",
      "     epoch 2, minibatch 195/195, test error of best model 89.032452 %\n",
      "('training @ iter = ', 400)\n",
      "('training @ iter = ', 500)\n",
      "epoch 3, minibatch 195/195, validation error 87.910657 %\n",
      "     epoch 3, minibatch 195/195, test error of best model 88.261218 %\n",
      "('training @ iter = ', 600)\n",
      "('training @ iter = ', 700)\n",
      "epoch 4, minibatch 195/195, validation error 86.318109 %\n",
      "     epoch 4, minibatch 195/195, test error of best model 86.578526 %\n",
      "('training @ iter = ', 800)\n",
      "('training @ iter = ', 900)\n",
      "epoch 5, minibatch 195/195, validation error 82.662260 %\n",
      "     epoch 5, minibatch 195/195, test error of best model 83.102965 %\n",
      "('training @ iter = ', 1000)\n",
      "('training @ iter = ', 1100)\n",
      "epoch 6, minibatch 195/195, validation error 78.425481 %\n",
      "     epoch 6, minibatch 195/195, test error of best model 79.036458 %\n",
      "('training @ iter = ', 1200)\n",
      "('training @ iter = ', 1300)\n",
      "epoch 7, minibatch 195/195, validation error 73.958333 %\n",
      "     epoch 7, minibatch 195/195, test error of best model 75.020032 %\n",
      "('training @ iter = ', 1400)\n",
      "('training @ iter = ', 1500)\n",
      "epoch 8, minibatch 195/195, validation error 68.209135 %\n",
      "     epoch 8, minibatch 195/195, test error of best model 69.821715 %\n",
      "('training @ iter = ', 1600)\n",
      "('training @ iter = ', 1700)\n",
      "epoch 9, minibatch 195/195, validation error 57.041266 %\n",
      "     epoch 9, minibatch 195/195, test error of best model 59.344952 %\n",
      "('training @ iter = ', 1800)\n",
      "('training @ iter = ', 1900)\n",
      "epoch 10, minibatch 195/195, validation error 44.220753 %\n",
      "     epoch 10, minibatch 195/195, test error of best model 46.223958 %\n",
      "('training @ iter = ', 2000)\n",
      "('training @ iter = ', 2100)\n",
      "epoch 11, minibatch 195/195, validation error 37.690304 %\n",
      "     epoch 11, minibatch 195/195, test error of best model 38.281250 %\n",
      "('training @ iter = ', 2200)\n",
      "('training @ iter = ', 2300)\n",
      "epoch 12, minibatch 195/195, validation error 31.590545 %\n",
      "     epoch 12, minibatch 195/195, test error of best model 32.461939 %\n",
      "('training @ iter = ', 2400)\n",
      "('training @ iter = ', 2500)\n",
      "epoch 13, minibatch 195/195, validation error 27.504006 %\n",
      "     epoch 13, minibatch 195/195, test error of best model 28.485577 %\n",
      "('training @ iter = ', 2600)\n",
      "('training @ iter = ', 2700)\n",
      "epoch 14, minibatch 195/195, validation error 40.474760 %\n",
      "('training @ iter = ', 2800)\n",
      "('training @ iter = ', 2900)\n",
      "epoch 15, minibatch 195/195, validation error 38.812099 %\n",
      "('training @ iter = ', 3000)\n",
      "('training @ iter = ', 3100)\n",
      "epoch 16, minibatch 195/195, validation error 28.886218 %\n",
      "('training @ iter = ', 3200)\n",
      "('training @ iter = ', 3300)\n",
      "epoch 17, minibatch 195/195, validation error 26.662660 %\n",
      "     epoch 17, minibatch 195/195, test error of best model 26.572516 %\n",
      "('training @ iter = ', 3400)\n",
      "('training @ iter = ', 3500)\n",
      "epoch 18, minibatch 195/195, validation error 21.654647 %\n",
      "     epoch 18, minibatch 195/195, test error of best model 22.055288 %\n",
      "('training @ iter = ', 3600)\n",
      "('training @ iter = ', 3700)\n",
      "epoch 19, minibatch 195/195, validation error 21.163862 %\n",
      "     epoch 19, minibatch 195/195, test error of best model 21.564503 %\n",
      "('training @ iter = ', 3800)\n",
      "epoch 20, minibatch 195/195, validation error 22.626202 %\n",
      "('training @ iter = ', 3900)\n",
      "('training @ iter = ', 4000)\n",
      "epoch 21, minibatch 195/195, validation error 20.783253 %\n",
      "     epoch 21, minibatch 195/195, test error of best model 21.193910 %\n",
      "('training @ iter = ', 4100)\n",
      "('training @ iter = ', 4200)\n",
      "epoch 22, minibatch 195/195, validation error 19.881811 %\n",
      "     epoch 22, minibatch 195/195, test error of best model 20.372596 %\n",
      "('training @ iter = ', 4300)\n",
      "('training @ iter = ', 4400)\n",
      "epoch 23, minibatch 195/195, validation error 19.491186 %\n",
      "     epoch 23, minibatch 195/195, test error of best model 19.831731 %\n",
      "('training @ iter = ', 4500)\n",
      "('training @ iter = ', 4600)\n",
      "epoch 24, minibatch 195/195, validation error 19.370994 %\n",
      "     epoch 24, minibatch 195/195, test error of best model 19.290865 %\n",
      "('training @ iter = ', 4700)\n",
      "('training @ iter = ', 4800)\n",
      "epoch 25, minibatch 195/195, validation error 21.554487 %\n",
      "('training @ iter = ', 4900)\n",
      "('training @ iter = ', 5000)\n",
      "epoch 26, minibatch 195/195, validation error 20.112179 %\n",
      "('training @ iter = ', 5100)\n",
      "('training @ iter = ', 5200)\n",
      "epoch 27, minibatch 195/195, validation error 18.539663 %\n",
      "     epoch 27, minibatch 195/195, test error of best model 18.499599 %\n",
      "('training @ iter = ', 5300)\n",
      "('training @ iter = ', 5400)\n",
      "epoch 28, minibatch 195/195, validation error 17.788462 %\n",
      "     epoch 28, minibatch 195/195, test error of best model 17.698317 %\n",
      "('training @ iter = ', 5500)\n",
      "('training @ iter = ', 5600)\n",
      "epoch 29, minibatch 195/195, validation error 20.152244 %\n",
      "('training @ iter = ', 5700)\n",
      "('training @ iter = ', 5800)\n",
      "epoch 30, minibatch 195/195, validation error 16.576522 %\n",
      "     epoch 30, minibatch 195/195, test error of best model 16.205929 %\n",
      "('training @ iter = ', 5900)\n",
      "('training @ iter = ', 6000)\n",
      "epoch 31, minibatch 195/195, validation error 17.497997 %\n",
      "('training @ iter = ', 6100)\n",
      "('training @ iter = ', 6200)\n",
      "epoch 32, minibatch 195/195, validation error 17.457933 %\n",
      "('training @ iter = ', 6300)\n",
      "('training @ iter = ', 6400)\n",
      "epoch 33, minibatch 195/195, validation error 17.027244 %\n",
      "('training @ iter = ', 6500)\n",
      "('training @ iter = ', 6600)\n",
      "epoch 34, minibatch 195/195, validation error 19.501202 %\n",
      "('training @ iter = ', 6700)\n",
      "('training @ iter = ', 6800)\n",
      "epoch 35, minibatch 195/195, validation error 16.195913 %\n",
      "     epoch 35, minibatch 195/195, test error of best model 15.875401 %\n",
      "('training @ iter = ', 6900)\n",
      "('training @ iter = ', 7000)\n",
      "epoch 36, minibatch 195/195, validation error 15.965545 %\n",
      "     epoch 36, minibatch 195/195, test error of best model 15.594952 %\n",
      "('training @ iter = ', 7100)\n",
      "('training @ iter = ', 7200)\n",
      "epoch 37, minibatch 195/195, validation error 13.932292 %\n",
      "     epoch 37, minibatch 195/195, test error of best model 13.882212 %\n",
      "('training @ iter = ', 7300)\n",
      "('training @ iter = ', 7400)\n",
      "epoch 38, minibatch 195/195, validation error 14.453125 %\n",
      "('training @ iter = ', 7500)\n",
      "('training @ iter = ', 7600)\n",
      "epoch 39, minibatch 195/195, validation error 13.371394 %\n",
      "     epoch 39, minibatch 195/195, test error of best model 13.211138 %\n",
      "('training @ iter = ', 7700)\n",
      "epoch 40, minibatch 195/195, validation error 16.937099 %\n",
      "('training @ iter = ', 7800)\n",
      "('training @ iter = ', 7900)\n",
      "epoch 41, minibatch 195/195, validation error 12.359776 %\n",
      "     epoch 41, minibatch 195/195, test error of best model 12.059295 %\n",
      "('training @ iter = ', 8000)\n",
      "('training @ iter = ', 8100)\n",
      "epoch 42, minibatch 195/195, validation error 16.416266 %\n",
      "('training @ iter = ', 8200)\n",
      "('training @ iter = ', 8300)\n",
      "epoch 43, minibatch 195/195, validation error 12.590144 %\n",
      "('training @ iter = ', 8400)\n",
      "('training @ iter = ', 8500)\n",
      "epoch 44, minibatch 195/195, validation error 13.511619 %\n",
      "('training @ iter = ', 8600)\n",
      "('training @ iter = ', 8700)\n",
      "epoch 45, minibatch 195/195, validation error 15.464744 %\n",
      "('training @ iter = ', 8800)\n",
      "('training @ iter = ', 8900)\n",
      "epoch 46, minibatch 195/195, validation error 20.943510 %\n",
      "('training @ iter = ', 9000)\n",
      "('training @ iter = ', 9100)\n",
      "epoch 47, minibatch 195/195, validation error 13.601763 %\n",
      "('training @ iter = ', 9200)\n",
      "('training @ iter = ', 9300)\n",
      "epoch 48, minibatch 195/195, validation error 12.159455 %\n",
      "     epoch 48, minibatch 195/195, test error of best model 11.648638 %\n",
      "('training @ iter = ', 9400)\n",
      "('training @ iter = ', 9500)\n",
      "epoch 49, minibatch 195/195, validation error 19.040465 %\n",
      "('training @ iter = ', 9600)\n",
      "('training @ iter = ', 9700)\n",
      "epoch 50, minibatch 195/195, validation error 16.306090 %\n",
      "('training @ iter = ', 9800)\n",
      "('training @ iter = ', 9900)\n",
      "epoch 51, minibatch 195/195, validation error 14.823718 %\n",
      "('training @ iter = ', 10000)\n",
      "('training @ iter = ', 10100)\n",
      "epoch 52, minibatch 195/195, validation error 10.857372 %\n",
      "     epoch 52, minibatch 195/195, test error of best model 10.316506 %\n",
      "('training @ iter = ', 10200)\n",
      "('training @ iter = ', 10300)\n",
      "epoch 53, minibatch 195/195, validation error 21.814904 %\n",
      "('training @ iter = ', 10400)\n",
      "('training @ iter = ', 10500)\n",
      "epoch 54, minibatch 195/195, validation error 13.601763 %\n",
      "('training @ iter = ', 10600)\n",
      "('training @ iter = ', 10700)\n",
      "epoch 55, minibatch 195/195, validation error 13.782051 %\n",
      "('training @ iter = ', 10800)\n",
      "('training @ iter = ', 10900)\n",
      "epoch 56, minibatch 195/195, validation error 22.956731 %\n",
      "('training @ iter = ', 11000)\n",
      "('training @ iter = ', 11100)\n",
      "epoch 57, minibatch 195/195, validation error 11.458333 %\n",
      "('training @ iter = ', 11200)\n",
      "('training @ iter = ', 11300)\n",
      "epoch 58, minibatch 195/195, validation error 16.586538 %\n",
      "('training @ iter = ', 11400)\n",
      "('training @ iter = ', 11500)\n",
      "epoch 59, minibatch 195/195, validation error 12.209535 %\n",
      "('training @ iter = ', 11600)\n",
      "epoch 60, minibatch 195/195, validation error 16.396234 %\n",
      "('training @ iter = ', 11700)\n",
      "('training @ iter = ', 11800)\n",
      "epoch 61, minibatch 195/195, validation error 19.671474 %\n",
      "('training @ iter = ', 11900)\n",
      "('training @ iter = ', 12000)\n",
      "epoch 62, minibatch 195/195, validation error 12.379808 %\n",
      "('training @ iter = ', 12100)\n",
      "('training @ iter = ', 12200)\n",
      "epoch 63, minibatch 195/195, validation error 17.257612 %\n",
      "('training @ iter = ', 12300)\n",
      "('training @ iter = ', 12400)\n",
      "epoch 64, minibatch 195/195, validation error 15.895433 %\n",
      "('training @ iter = ', 12500)\n",
      "('training @ iter = ', 12600)\n",
      "epoch 65, minibatch 195/195, validation error 11.157853 %\n",
      "('training @ iter = ', 12700)\n",
      "('training @ iter = ', 12800)\n",
      "epoch 66, minibatch 195/195, validation error 13.251202 %\n",
      "('training @ iter = ', 12900)\n",
      "('training @ iter = ', 13000)\n",
      "epoch 67, minibatch 195/195, validation error 14.793670 %\n",
      "('training @ iter = ', 13100)\n",
      "('training @ iter = ', 13200)\n",
      "epoch 68, minibatch 195/195, validation error 14.543269 %\n",
      "('training @ iter = ', 13300)\n",
      "('training @ iter = ', 13400)\n",
      "epoch 69, minibatch 195/195, validation error 10.496795 %\n",
      "     epoch 69, minibatch 195/195, test error of best model 10.116186 %\n",
      "('training @ iter = ', 13500)\n",
      "('training @ iter = ', 13600)\n",
      "epoch 70, minibatch 195/195, validation error 14.663462 %\n",
      "('training @ iter = ', 13700)\n",
      "('training @ iter = ', 13800)\n",
      "epoch 71, minibatch 195/195, validation error 11.137821 %\n",
      "('training @ iter = ', 13900)\n",
      "('training @ iter = ', 14000)\n",
      "epoch 72, minibatch 195/195, validation error 10.927484 %\n",
      "('training @ iter = ', 14100)\n",
      "('training @ iter = ', 14200)\n",
      "epoch 73, minibatch 195/195, validation error 15.985577 %\n",
      "('training @ iter = ', 14300)\n",
      "('training @ iter = ', 14400)\n",
      "epoch 74, minibatch 195/195, validation error 12.369792 %\n",
      "('training @ iter = ', 14500)\n",
      "('training @ iter = ', 14600)\n",
      "epoch 75, minibatch 195/195, validation error 11.818910 %\n",
      "('training @ iter = ', 14700)\n",
      "('training @ iter = ', 14800)\n",
      "epoch 76, minibatch 195/195, validation error 24.268830 %\n",
      "('training @ iter = ', 14900)\n",
      "('training @ iter = ', 15000)\n",
      "epoch 77, minibatch 195/195, validation error 11.638622 %\n",
      "('training @ iter = ', 15100)\n",
      "('training @ iter = ', 15200)\n",
      "epoch 78, minibatch 195/195, validation error 10.627003 %\n",
      "('training @ iter = ', 15300)\n",
      "('training @ iter = ', 15400)\n",
      "epoch 79, minibatch 195/195, validation error 11.979167 %\n",
      "('training @ iter = ', 15500)\n",
      "epoch 80, minibatch 195/195, validation error 11.889022 %\n",
      "('training @ iter = ', 15600)\n",
      "('training @ iter = ', 15700)\n",
      "epoch 81, minibatch 195/195, validation error 12.189503 %\n",
      "('training @ iter = ', 15800)\n",
      "('training @ iter = ', 15900)\n",
      "epoch 82, minibatch 195/195, validation error 12.349760 %\n",
      "('training @ iter = ', 16000)\n",
      "('training @ iter = ', 16100)\n",
      "epoch 83, minibatch 195/195, validation error 10.877404 %\n",
      "('training @ iter = ', 16200)\n",
      "('training @ iter = ', 16300)\n",
      "epoch 84, minibatch 195/195, validation error 13.731971 %\n",
      "('training @ iter = ', 16400)\n",
      "('training @ iter = ', 16500)\n",
      "epoch 85, minibatch 195/195, validation error 11.768830 %\n",
      "('training @ iter = ', 16600)\n",
      "('training @ iter = ', 16700)\n",
      "epoch 86, minibatch 195/195, validation error 10.156250 %\n",
      "     epoch 86, minibatch 195/195, test error of best model 10.026042 %\n",
      "('training @ iter = ', 16800)\n",
      "('training @ iter = ', 16900)\n",
      "epoch 87, minibatch 195/195, validation error 13.371394 %\n",
      "('training @ iter = ', 17000)\n",
      "('training @ iter = ', 17100)\n",
      "epoch 88, minibatch 195/195, validation error 11.909054 %\n",
      "('training @ iter = ', 17200)\n",
      "('training @ iter = ', 17300)\n",
      "epoch 89, minibatch 195/195, validation error 10.747196 %\n",
      "('training @ iter = ', 17400)\n",
      "('training @ iter = ', 17500)\n",
      "epoch 90, minibatch 195/195, validation error 11.217949 %\n",
      "('training @ iter = ', 17600)\n",
      "('training @ iter = ', 17700)\n",
      "epoch 91, minibatch 195/195, validation error 9.815705 %\n",
      "     epoch 91, minibatch 195/195, test error of best model 9.114583 %\n",
      "('training @ iter = ', 17800)\n",
      "('training @ iter = ', 17900)\n",
      "epoch 92, minibatch 195/195, validation error 9.154647 %\n",
      "     epoch 92, minibatch 195/195, test error of best model 8.583734 %\n",
      "('training @ iter = ', 18000)\n",
      "('training @ iter = ', 18100)\n",
      "epoch 93, minibatch 195/195, validation error 10.657051 %\n",
      "('training @ iter = ', 18200)\n",
      "('training @ iter = ', 18300)\n",
      "epoch 94, minibatch 195/195, validation error 9.174679 %\n",
      "('training @ iter = ', 18400)\n",
      "('training @ iter = ', 18500)\n",
      "epoch 95, minibatch 195/195, validation error 11.077724 %\n",
      "('training @ iter = ', 18600)\n",
      "('training @ iter = ', 18700)\n",
      "epoch 96, minibatch 195/195, validation error 8.804087 %\n",
      "     epoch 96, minibatch 195/195, test error of best model 8.383413 %\n",
      "('training @ iter = ', 18800)\n",
      "('training @ iter = ', 18900)\n",
      "epoch 97, minibatch 195/195, validation error 11.428285 %\n",
      "('training @ iter = ', 19000)\n",
      "('training @ iter = ', 19100)\n",
      "epoch 98, minibatch 195/195, validation error 8.733974 %\n",
      "     epoch 98, minibatch 195/195, test error of best model 8.343349 %\n",
      "('training @ iter = ', 19200)\n",
      "('training @ iter = ', 19300)\n",
      "epoch 99, minibatch 195/195, validation error 8.673878 %\n",
      "     epoch 99, minibatch 195/195, test error of best model 8.423478 %\n",
      "('training @ iter = ', 19400)\n",
      "epoch 100, minibatch 195/195, validation error 9.855769 %\n",
      "('training @ iter = ', 19500)\n",
      "('training @ iter = ', 19600)\n",
      "epoch 101, minibatch 195/195, validation error 9.835737 %\n",
      "('training @ iter = ', 19700)\n",
      "('training @ iter = ', 19800)\n",
      "epoch 102, minibatch 195/195, validation error 10.056090 %\n",
      "('training @ iter = ', 19900)\n",
      "('training @ iter = ', 20000)\n",
      "epoch 103, minibatch 195/195, validation error 8.814103 %\n",
      "('training @ iter = ', 20100)\n",
      "('training @ iter = ', 20200)\n",
      "epoch 104, minibatch 195/195, validation error 9.455128 %\n",
      "('training @ iter = ', 20300)\n",
      "('training @ iter = ', 20400)\n",
      "epoch 105, minibatch 195/195, validation error 9.064503 %\n",
      "('training @ iter = ', 20500)\n",
      "('training @ iter = ', 20600)\n",
      "epoch 106, minibatch 195/195, validation error 9.324920 %\n",
      "('training @ iter = ', 20700)\n",
      "('training @ iter = ', 20800)\n",
      "epoch 107, minibatch 195/195, validation error 9.284856 %\n",
      "('training @ iter = ', 20900)\n",
      "('training @ iter = ', 21000)\n",
      "epoch 108, minibatch 195/195, validation error 9.284856 %\n",
      "('training @ iter = ', 21100)\n",
      "('training @ iter = ', 21200)\n",
      "epoch 109, minibatch 195/195, validation error 8.894231 %\n",
      "('training @ iter = ', 21300)\n",
      "('training @ iter = ', 21400)\n",
      "epoch 110, minibatch 195/195, validation error 9.184696 %\n",
      "('training @ iter = ', 21500)\n",
      "('training @ iter = ', 21600)\n",
      "epoch 111, minibatch 195/195, validation error 8.523638 %\n",
      "     epoch 111, minibatch 195/195, test error of best model 8.223157 %\n",
      "('training @ iter = ', 21700)\n",
      "('training @ iter = ', 21800)\n",
      "epoch 112, minibatch 195/195, validation error 8.633814 %\n",
      "('training @ iter = ', 21900)\n",
      "('training @ iter = ', 22000)\n",
      "epoch 113, minibatch 195/195, validation error 11.318109 %\n",
      "('training @ iter = ', 22100)\n",
      "('training @ iter = ', 22200)\n",
      "epoch 114, minibatch 195/195, validation error 8.453526 %\n",
      "     epoch 114, minibatch 195/195, test error of best model 8.193109 %\n",
      "('training @ iter = ', 22300)\n",
      "('training @ iter = ', 22400)\n",
      "epoch 115, minibatch 195/195, validation error 10.376603 %\n",
      "('training @ iter = ', 22500)\n",
      "('training @ iter = ', 22600)\n",
      "epoch 116, minibatch 195/195, validation error 8.393429 %\n",
      "     epoch 116, minibatch 195/195, test error of best model 8.293269 %\n",
      "('training @ iter = ', 22700)\n",
      "('training @ iter = ', 22800)\n",
      "epoch 117, minibatch 195/195, validation error 9.294872 %\n",
      "('training @ iter = ', 22900)\n",
      "('training @ iter = ', 23000)\n",
      "epoch 118, minibatch 195/195, validation error 8.713942 %\n",
      "('training @ iter = ', 23100)\n",
      "('training @ iter = ', 23200)\n",
      "epoch 119, minibatch 195/195, validation error 9.184696 %\n",
      "('training @ iter = ', 23300)\n",
      "epoch 120, minibatch 195/195, validation error 8.523638 %\n",
      "('training @ iter = ', 23400)\n",
      "('training @ iter = ', 23500)\n",
      "epoch 121, minibatch 195/195, validation error 8.383413 %\n",
      "     epoch 121, minibatch 195/195, test error of best model 8.313301 %\n",
      "('training @ iter = ', 23600)\n",
      "('training @ iter = ', 23700)\n",
      "epoch 122, minibatch 195/195, validation error 8.814103 %\n",
      "('training @ iter = ', 23800)\n",
      "('training @ iter = ', 23900)\n",
      "epoch 123, minibatch 195/195, validation error 9.475160 %\n",
      "('training @ iter = ', 24000)\n",
      "('training @ iter = ', 24100)\n",
      "epoch 124, minibatch 195/195, validation error 8.473558 %\n",
      "('training @ iter = ', 24200)\n",
      "('training @ iter = ', 24300)\n",
      "epoch 125, minibatch 195/195, validation error 8.533654 %\n",
      "('training @ iter = ', 24400)\n",
      "('training @ iter = ', 24500)\n",
      "epoch 126, minibatch 195/195, validation error 8.513622 %\n",
      "('training @ iter = ', 24600)\n",
      "('training @ iter = ', 24700)\n",
      "epoch 127, minibatch 195/195, validation error 8.443510 %\n",
      "('training @ iter = ', 24800)\n",
      "('training @ iter = ', 24900)\n",
      "epoch 128, minibatch 195/195, validation error 8.533654 %\n",
      "('training @ iter = ', 25000)\n",
      "('training @ iter = ', 25100)\n",
      "epoch 129, minibatch 195/195, validation error 8.623798 %\n",
      "('training @ iter = ', 25200)\n",
      "('training @ iter = ', 25300)\n",
      "epoch 130, minibatch 195/195, validation error 9.965946 %\n",
      "('training @ iter = ', 25400)\n",
      "('training @ iter = ', 25500)\n",
      "epoch 131, minibatch 195/195, validation error 8.443510 %\n",
      "('training @ iter = ', 25600)\n",
      "('training @ iter = ', 25700)\n",
      "epoch 132, minibatch 195/195, validation error 8.814103 %\n",
      "('training @ iter = ', 25800)\n",
      "('training @ iter = ', 25900)\n",
      "epoch 133, minibatch 195/195, validation error 8.513622 %\n",
      "('training @ iter = ', 26000)\n",
      "('training @ iter = ', 26100)\n",
      "epoch 134, minibatch 195/195, validation error 8.473558 %\n",
      "('training @ iter = ', 26200)\n",
      "('training @ iter = ', 26300)\n",
      "epoch 135, minibatch 195/195, validation error 8.583734 %\n",
      "('training @ iter = ', 26400)\n",
      "('training @ iter = ', 26500)\n",
      "epoch 136, minibatch 195/195, validation error 8.513622 %\n",
      "('training @ iter = ', 26600)\n",
      "('training @ iter = ', 26700)\n",
      "epoch 137, minibatch 195/195, validation error 8.553686 %\n",
      "('training @ iter = ', 26800)\n",
      "('training @ iter = ', 26900)\n",
      "epoch 138, minibatch 195/195, validation error 8.613782 %\n",
      "('training @ iter = ', 27000)\n",
      "('training @ iter = ', 27100)\n",
      "epoch 139, minibatch 195/195, validation error 9.014423 %\n",
      "('training @ iter = ', 27200)\n",
      "epoch 140, minibatch 195/195, validation error 8.353365 %\n",
      "     epoch 140, minibatch 195/195, test error of best model 8.133013 %\n",
      "('training @ iter = ', 27300)\n",
      "('training @ iter = ', 27400)\n",
      "epoch 141, minibatch 195/195, validation error 8.413462 %\n",
      "('training @ iter = ', 27500)\n",
      "('training @ iter = ', 27600)\n",
      "epoch 142, minibatch 195/195, validation error 8.663862 %\n",
      "('training @ iter = ', 27700)\n",
      "('training @ iter = ', 27800)\n",
      "epoch 143, minibatch 195/195, validation error 9.425080 %\n",
      "('training @ iter = ', 27900)\n",
      "('training @ iter = ', 28000)\n",
      "epoch 144, minibatch 195/195, validation error 8.283253 %\n",
      "     epoch 144, minibatch 195/195, test error of best model 8.273237 %\n",
      "('training @ iter = ', 28100)\n",
      "('training @ iter = ', 28200)\n",
      "epoch 145, minibatch 195/195, validation error 8.463542 %\n",
      "('training @ iter = ', 28300)\n",
      "('training @ iter = ', 28400)\n",
      "epoch 146, minibatch 195/195, validation error 8.633814 %\n",
      "('training @ iter = ', 28500)\n",
      "('training @ iter = ', 28600)\n",
      "epoch 147, minibatch 195/195, validation error 8.743990 %\n",
      "('training @ iter = ', 28700)\n",
      "('training @ iter = ', 28800)\n",
      "epoch 148, minibatch 195/195, validation error 9.375000 %\n",
      "('training @ iter = ', 28900)\n",
      "('training @ iter = ', 29000)\n",
      "epoch 149, minibatch 195/195, validation error 8.413462 %\n",
      "('training @ iter = ', 29100)\n",
      "('training @ iter = ', 29200)\n",
      "epoch 150, minibatch 195/195, validation error 8.553686 %\n",
      "('training @ iter = ', 29300)\n",
      "('training @ iter = ', 29400)\n",
      "epoch 151, minibatch 195/195, validation error 8.423478 %\n",
      "('training @ iter = ', 29500)\n",
      "('training @ iter = ', 29600)\n",
      "epoch 152, minibatch 195/195, validation error 8.673878 %\n",
      "('training @ iter = ', 29700)\n",
      "('training @ iter = ', 29800)\n",
      "epoch 153, minibatch 195/195, validation error 8.814103 %\n",
      "('training @ iter = ', 29900)\n",
      "('training @ iter = ', 30000)\n",
      "epoch 154, minibatch 195/195, validation error 8.553686 %\n",
      "('training @ iter = ', 30100)\n",
      "('training @ iter = ', 30200)\n",
      "epoch 155, minibatch 195/195, validation error 8.663862 %\n",
      "('training @ iter = ', 30300)\n",
      "('training @ iter = ', 30400)\n",
      "epoch 156, minibatch 195/195, validation error 8.543670 %\n",
      "('training @ iter = ', 30500)\n",
      "('training @ iter = ', 30600)\n",
      "epoch 157, minibatch 195/195, validation error 9.004407 %\n",
      "('training @ iter = ', 30700)\n",
      "('training @ iter = ', 30800)\n",
      "epoch 158, minibatch 195/195, validation error 8.533654 %\n",
      "('training @ iter = ', 30900)\n",
      "('training @ iter = ', 31000)\n",
      "epoch 159, minibatch 195/195, validation error 8.593750 %\n",
      "('training @ iter = ', 31100)\n",
      "epoch 160, minibatch 195/195, validation error 8.593750 %\n",
      "('training @ iter = ', 31200)\n",
      "('training @ iter = ', 31300)\n",
      "epoch 161, minibatch 195/195, validation error 8.603766 %\n",
      "('training @ iter = ', 31400)\n",
      "('training @ iter = ', 31500)\n",
      "epoch 162, minibatch 195/195, validation error 8.683894 %\n",
      "('training @ iter = ', 31600)\n",
      "('training @ iter = ', 31700)\n",
      "epoch 163, minibatch 195/195, validation error 8.573718 %\n",
      "('training @ iter = ', 31800)\n",
      "('training @ iter = ', 31900)\n",
      "epoch 164, minibatch 195/195, validation error 8.423478 %\n",
      "('training @ iter = ', 32000)\n",
      "('training @ iter = ', 32100)\n",
      "epoch 165, minibatch 195/195, validation error 8.894231 %\n",
      "('training @ iter = ', 32200)\n",
      "('training @ iter = ', 32300)\n",
      "epoch 166, minibatch 195/195, validation error 8.483574 %\n",
      "('training @ iter = ', 32400)\n",
      "('training @ iter = ', 32500)\n",
      "epoch 167, minibatch 195/195, validation error 8.794071 %\n",
      "('training @ iter = ', 32600)\n",
      "('training @ iter = ', 32700)\n",
      "epoch 168, minibatch 195/195, validation error 8.433494 %\n",
      "('training @ iter = ', 32800)\n",
      "('training @ iter = ', 32900)\n",
      "epoch 169, minibatch 195/195, validation error 9.014423 %\n",
      "('training @ iter = ', 33000)\n",
      "('training @ iter = ', 33100)\n",
      "epoch 170, minibatch 195/195, validation error 8.653846 %\n",
      "('training @ iter = ', 33200)\n",
      "('training @ iter = ', 33300)\n",
      "epoch 171, minibatch 195/195, validation error 8.814103 %\n",
      "('training @ iter = ', 33400)\n",
      "('training @ iter = ', 33500)\n",
      "epoch 172, minibatch 195/195, validation error 9.184696 %\n",
      "('training @ iter = ', 33600)\n",
      "('training @ iter = ', 33700)\n",
      "epoch 173, minibatch 195/195, validation error 8.643830 %\n",
      "('training @ iter = ', 33800)\n",
      "('training @ iter = ', 33900)\n",
      "epoch 174, minibatch 195/195, validation error 8.613782 %\n",
      "('training @ iter = ', 34000)\n",
      "('training @ iter = ', 34100)\n",
      "epoch 175, minibatch 195/195, validation error 8.723958 %\n",
      "('training @ iter = ', 34200)\n",
      "('training @ iter = ', 34300)\n",
      "epoch 176, minibatch 195/195, validation error 8.583734 %\n",
      "('training @ iter = ', 34400)\n",
      "('training @ iter = ', 34500)\n",
      "epoch 177, minibatch 195/195, validation error 8.764022 %\n",
      "('training @ iter = ', 34600)\n",
      "('training @ iter = ', 34700)\n",
      "epoch 178, minibatch 195/195, validation error 8.673878 %\n",
      "('training @ iter = ', 34800)\n",
      "('training @ iter = ', 34900)\n",
      "epoch 179, minibatch 195/195, validation error 8.663862 %\n",
      "('training @ iter = ', 35000)\n",
      "epoch 180, minibatch 195/195, validation error 8.583734 %\n",
      "('training @ iter = ', 35100)\n",
      "('training @ iter = ', 35200)\n",
      "epoch 181, minibatch 195/195, validation error 8.743990 %\n",
      "('training @ iter = ', 35300)\n",
      "('training @ iter = ', 35400)\n",
      "epoch 182, minibatch 195/195, validation error 8.583734 %\n",
      "('training @ iter = ', 35500)\n",
      "('training @ iter = ', 35600)\n",
      "epoch 183, minibatch 195/195, validation error 8.463542 %\n",
      "('training @ iter = ', 35700)\n",
      "('training @ iter = ', 35800)\n",
      "epoch 184, minibatch 195/195, validation error 8.513622 %\n",
      "('training @ iter = ', 35900)\n",
      "('training @ iter = ', 36000)\n",
      "epoch 185, minibatch 195/195, validation error 8.513622 %\n",
      "('training @ iter = ', 36100)\n",
      "('training @ iter = ', 36200)\n",
      "epoch 186, minibatch 195/195, validation error 8.543670 %\n",
      "('training @ iter = ', 36300)\n",
      "('training @ iter = ', 36400)\n",
      "epoch 187, minibatch 195/195, validation error 9.415064 %\n",
      "('training @ iter = ', 36500)\n",
      "('training @ iter = ', 36600)\n",
      "epoch 188, minibatch 195/195, validation error 18.159054 %\n",
      "('training @ iter = ', 36700)\n",
      "('training @ iter = ', 36800)\n",
      "epoch 189, minibatch 195/195, validation error 8.403446 %\n",
      "('training @ iter = ', 36900)\n",
      "('training @ iter = ', 37000)\n",
      "epoch 190, minibatch 195/195, validation error 8.423478 %\n",
      "('training @ iter = ', 37100)\n",
      "('training @ iter = ', 37200)\n",
      "epoch 191, minibatch 195/195, validation error 10.256410 %\n",
      "('training @ iter = ', 37300)\n",
      "('training @ iter = ', 37400)\n",
      "epoch 192, minibatch 195/195, validation error 8.433494 %\n",
      "('training @ iter = ', 37500)\n",
      "('training @ iter = ', 37600)\n",
      "epoch 193, minibatch 195/195, validation error 8.623798 %\n",
      "('training @ iter = ', 37700)\n",
      "('training @ iter = ', 37800)\n",
      "epoch 194, minibatch 195/195, validation error 8.533654 %\n",
      "('training @ iter = ', 37900)\n",
      "('training @ iter = ', 38000)\n",
      "epoch 195, minibatch 195/195, validation error 9.014423 %\n",
      "('training @ iter = ', 38100)\n",
      "('training @ iter = ', 38200)\n",
      "epoch 196, minibatch 195/195, validation error 8.463542 %\n",
      "('training @ iter = ', 38300)\n",
      "('training @ iter = ', 38400)\n",
      "epoch 197, minibatch 195/195, validation error 8.473558 %\n",
      "('training @ iter = ', 38500)\n",
      "('training @ iter = ', 38600)\n",
      "epoch 198, minibatch 195/195, validation error 8.914263 %\n",
      "('training @ iter = ', 38700)\n",
      "('training @ iter = ', 38800)\n",
      "epoch 199, minibatch 195/195, validation error 8.303285 %\n",
      "('training @ iter = ', 38900)\n",
      "epoch 200, minibatch 195/195, validation error 8.393429 %\n",
      "Optimization complete.\n",
      "Best validation score of 8.283253 % obtained at iteration 28080, with test performance 8.273237 %\n",
      "The training process for function run ran for 25.81m\n"
     ]
    }
   ],
   "source": [
    "run(network_type = 'fcn', include_st = True, transformation = proj_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "run(network_type = 'cnn', transformation = proj_transform, include_st = True, runallmode = True, n_epochs = 200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "...loading the dataset\n",
    "...Applying proj_transform to dataset\n",
    "Current training data size is 50000\n",
    "Current validation data size is 10000\n",
    "Current test data size is 10000\n",
    "...building the model\n",
    "...training cnn with include_st = True\n",
    "...Apply STN before CNN\n",
    "...Init localization to identity\n",
    "...training\n",
    "('training @ iter = ', 0)\n",
    "('training @ iter = ', 100)\n",
    "epoch 1, minibatch 195/195, validation error 81.482772 %\n",
    "     epoch 1, minibatch 195/195, test error of best model 81.502804 %\n",
    "('training @ iter = ', 200)\n",
    "('training @ iter = ', 300)\n",
    "epoch 2, minibatch 195/195, validation error 79.750000 %\n",
    "     epoch 2, minibatch 195/195, test error of best model 79.970353 %\n",
    "('training @ iter = ', 400)\n",
    "('training @ iter = ', 500)\n",
    "epoch 3, minibatch 195/195, validation error 75.953926 %\n",
    "     epoch 3, minibatch 195/195, test error of best model 75.953926 %\n",
    "('training @ iter = ', 600)\n",
    "('training @ iter = ', 700)\n",
    "epoch 4, minibatch 195/195, validation error 70.745593 %\n",
    "     epoch 4, minibatch 195/195, test error of best model 71.186298 %\n",
    "('training @ iter = ', 800)\n",
    "('training @ iter = ', 900)\n",
    "epoch 5, minibatch 195/195, validation error 57.634615 %\n",
    "     epoch 5, minibatch 195/195, test error of best model 58.305689 %\n",
    "('training @ iter = ', 1000)\n",
    "('training @ iter = ', 1100)\n",
    "epoch 6, minibatch 195/195, validation error 40.407051 %\n",
    "     epoch 6, minibatch 195/195, test error of best model 41.328526 %\n",
    "('training @ iter = ', 1200)\n",
    "('training @ iter = ', 1300)\n",
    "epoch 7, minibatch 195/195, validation error 38.203526 %\n",
    "     epoch 7, minibatch 195/195, test error of best model 39.325321 %\n",
    "('training @ iter = ', 1400)\n",
    "('training @ iter = ', 1500)\n",
    "epoch 8, minibatch 195/195, validation error 31.993590 %\n",
    "     epoch 8, minibatch 195/195, test error of best model 33.255609 %\n",
    "('training @ iter = ', 1600)\n",
    "('training @ iter = ', 1700)\n",
    "epoch 9, minibatch 195/195, validation error 27.165865 %\n",
    "     epoch 9, minibatch 195/195, test error of best model 28.598157 %\n",
    "('training @ iter = ', 1800)\n",
    "('training @ iter = ', 1900)\n",
    "epoch 10, minibatch 195/195, validation error 24.161058 %\n",
    "     epoch 10, minibatch 195/195, test error of best model 25.553285 %\n",
    "('training @ iter = ', 2000)\n",
    "('training @ iter = ', 2100)\n",
    "epoch 11, minibatch 195/195, validation error 20.995994 %\n",
    "     epoch 11, minibatch 195/195, test error of best model 22.237981 %\n",
    "('training @ iter = ', 2200)\n",
    "('training @ iter = ', 2300)\n",
    "epoch 12, minibatch 195/195, validation error 18.361779 %\n",
    "     epoch 12, minibatch 195/195, test error of best model 19.183093 %\n",
    "('training @ iter = ', 2400)\n",
    "('training @ iter = ', 2500)\n",
    "epoch 13, minibatch 195/195, validation error 16.819311 %\n",
    "     epoch 13, minibatch 195/195, test error of best model 17.560497 %\n",
    "('training @ iter = ', 2600)\n",
    "('training @ iter = ', 2700)\n",
    "epoch 14, minibatch 195/195, validation error 15.837740 %\n",
    "     epoch 14, minibatch 195/195, test error of best model 16.048077 %\n",
    "('training @ iter = ', 2800)\n",
    "('training @ iter = ', 2900)\n",
    "epoch 15, minibatch 195/195, validation error 14.495593 %\n",
    "     epoch 15, minibatch 195/195, test error of best model 15.006410 %\n",
    "('training @ iter = ', 3000)\n",
    "('training @ iter = ', 3100)\n",
    "epoch 16, minibatch 195/195, validation error 13.694311 %\n",
    "     epoch 16, minibatch 195/195, test error of best model 14.094952 %\n",
    "('training @ iter = ', 3200)\n",
    "('training @ iter = ', 3300)\n",
    "epoch 17, minibatch 195/195, validation error 12.522436 %\n",
    "     epoch 17, minibatch 195/195, test error of best model 13.073317 %\n",
    "('training @ iter = ', 3400)\n",
    "('training @ iter = ', 3500)\n",
    "epoch 18, minibatch 195/195, validation error 11.911458 %\n",
    "     epoch 18, minibatch 195/195, test error of best model 12.252003 %\n",
    "('training @ iter = ', 3600)\n",
    "('training @ iter = ', 3700)\n",
    "epoch 19, minibatch 195/195, validation error 11.470753 %\n",
    "     epoch 19, minibatch 195/195, test error of best model 11.671074 %\n",
    "('training @ iter = ', 3800)\n",
    "epoch 20, minibatch 195/195, validation error 10.869792 %\n",
    "     epoch 20, minibatch 195/195, test error of best model 10.869792 %\n",
    "('training @ iter = ', 3900)\n",
    "('training @ iter = ', 4000)\n",
    "epoch 21, minibatch 195/195, validation error 10.429087 %\n",
    "     epoch 21, minibatch 195/195, test error of best model 10.338942 %\n",
    "('training @ iter = ', 4100)\n",
    "('training @ iter = ', 4200)\n",
    "epoch 22, minibatch 195/195, validation error 10.058494 %\n",
    "     epoch 22, minibatch 195/195, test error of best model 10.348958 %\n",
    "('training @ iter = ', 4300)\n",
    "('training @ iter = ', 4400)\n",
    "epoch 23, minibatch 195/195, validation error 9.657853 %\n",
    "     epoch 23, minibatch 195/195, test error of best model 9.888221 %\n",
    "('training @ iter = ', 4500)\n",
    "('training @ iter = ', 4600)\n",
    "epoch 24, minibatch 195/195, validation error 9.417468 %\n",
    "     epoch 24, minibatch 195/195, test error of best model 9.477564 %\n",
    "('training @ iter = ', 4700)\n",
    "('training @ iter = ', 4800)\n",
    "epoch 25, minibatch 195/195, validation error 9.116987 %\n",
    "     epoch 25, minibatch 195/195, test error of best model 9.106971 %\n",
    "('training @ iter = ', 4900)\n",
    "('training @ iter = ', 5000)\n",
    "epoch 26, minibatch 195/195, validation error 8.976763 %\n",
    "     epoch 26, minibatch 195/195, test error of best model 8.856571 %\n",
    "('training @ iter = ', 5100)\n",
    "('training @ iter = ', 5200)\n",
    "epoch 27, minibatch 195/195, validation error 8.766426 %\n",
    "     epoch 27, minibatch 195/195, test error of best model 8.556090 %\n",
    "('training @ iter = ', 5300)\n",
    "('training @ iter = ', 5400)\n",
    "epoch 28, minibatch 195/195, validation error 8.596154 %\n",
    "     epoch 28, minibatch 195/195, test error of best model 8.355769 %\n",
    "('training @ iter = ', 5500)\n",
    "('training @ iter = ', 5600)\n",
    "epoch 29, minibatch 195/195, validation error 8.405849 %\n",
    "     epoch 29, minibatch 195/195, test error of best model 8.085337 %\n",
    "('training @ iter = ', 5700)\n",
    "('training @ iter = ', 5800)\n",
    "epoch 30, minibatch 195/195, validation error 8.205529 %\n",
    "     epoch 30, minibatch 195/195, test error of best model 7.895032 %\n",
    "('training @ iter = ', 5900)\n",
    "('training @ iter = ', 6000)\n",
    "epoch 31, minibatch 195/195, validation error 8.155449 %\n",
    "     epoch 31, minibatch 195/195, test error of best model 7.975160 %\n",
    "('training @ iter = ', 6100)\n",
    "('training @ iter = ', 6200)\n",
    "epoch 32, minibatch 195/195, validation error 8.105369 %\n",
    "     epoch 32, minibatch 195/195, test error of best model 7.704728 %\n",
    "('training @ iter = ', 6300)\n",
    "('training @ iter = ', 6400)\n",
    "epoch 33, minibatch 195/195, validation error 7.864984 %\n",
    "     epoch 33, minibatch 195/195, test error of best model 7.744792 %\n",
    "('training @ iter = ', 6500)\n",
    "('training @ iter = ', 6600)\n",
    "epoch 34, minibatch 195/195, validation error 7.814904 %\n",
    "     epoch 34, minibatch 195/195, test error of best model 7.704728 %\n",
    "('training @ iter = ', 6700)\n",
    "('training @ iter = ', 6800)\n",
    "epoch 35, minibatch 195/195, validation error 8.015224 %\n",
    "('training @ iter = ', 6900)\n",
    "('training @ iter = ', 7000)\n",
    "epoch 36, minibatch 195/195, validation error 7.814904 %\n",
    "('training @ iter = ', 7100)\n",
    "('training @ iter = ', 7200)\n",
    "epoch 37, minibatch 195/195, validation error 7.484375 %\n",
    "     epoch 37, minibatch 195/195, test error of best model 7.264022 %\n",
    "('training @ iter = ', 7300)\n",
    "('training @ iter = ', 7400)\n",
    "epoch 38, minibatch 195/195, validation error 7.314103 %\n",
    "     epoch 38, minibatch 195/195, test error of best model 7.093750 %\n",
    "('training @ iter = ', 7500)\n",
    "('training @ iter = ', 7600)\n",
    "epoch 39, minibatch 195/195, validation error 7.514423 %\n",
    "('training @ iter = ', 7700)\n",
    "epoch 40, minibatch 195/195, validation error 7.223958 %\n",
    "     epoch 40, minibatch 195/195, test error of best model 7.073718 %\n",
    "('training @ iter = ', 7800)\n",
    "('training @ iter = ', 7900)\n",
    "epoch 41, minibatch 195/195, validation error 7.173878 %\n",
    "     epoch 41, minibatch 195/195, test error of best model 6.993590 %\n",
    "('training @ iter = ', 8000)\n",
    "('training @ iter = ', 8100)\n",
    "epoch 42, minibatch 195/195, validation error 7.143830 %\n",
    "     epoch 42, minibatch 195/195, test error of best model 6.963542 %\n",
    "('training @ iter = ', 8200)\n",
    "('training @ iter = ', 8300)\n",
    "epoch 43, minibatch 195/195, validation error 7.324119 %\n",
    "('training @ iter = ', 8400)\n",
    "('training @ iter = ', 8500)\n",
    "epoch 44, minibatch 195/195, validation error 7.053686 %\n",
    "     epoch 44, minibatch 195/195, test error of best model 6.643029 %\n",
    "('training @ iter = ', 8600)\n",
    "('training @ iter = ', 8700)\n",
    "epoch 45, minibatch 195/195, validation error 6.993590 %\n",
    "     epoch 45, minibatch 195/195, test error of best model 6.673077 %\n",
    "('training @ iter = ', 8800)\n",
    "('training @ iter = ', 8900)\n",
    "epoch 46, minibatch 195/195, validation error 6.993590 %\n",
    "('training @ iter = ', 9000)\n",
    "('training @ iter = ', 9100)\n",
    "epoch 47, minibatch 195/195, validation error 6.993590 %\n",
    "('training @ iter = ', 9200)\n",
    "('training @ iter = ', 9300)\n",
    "epoch 48, minibatch 195/195, validation error 6.913462 %\n",
    "     epoch 48, minibatch 195/195, test error of best model 6.763221 %\n",
    "('training @ iter = ', 9400)\n",
    "('training @ iter = ', 9500)\n",
    "epoch 49, minibatch 195/195, validation error 6.833333 %\n",
    "     epoch 49, minibatch 195/195, test error of best model 6.582933 %\n",
    "('training @ iter = ', 9600)\n",
    "('training @ iter = ', 9700)\n",
    "epoch 50, minibatch 195/195, validation error 6.953526 %\n",
    "('training @ iter = ', 9800)\n",
    "('training @ iter = ', 9900)\n",
    "epoch 51, minibatch 195/195, validation error 7.023638 %\n",
    "('training @ iter = ', 10000)\n",
    "('training @ iter = ', 10100)\n",
    "epoch 52, minibatch 195/195, validation error 6.863381 %\n",
    "('training @ iter = ', 10200)\n",
    "('training @ iter = ', 10300)\n",
    "epoch 53, minibatch 195/195, validation error 6.883413 %\n",
    "('training @ iter = ', 10400)\n",
    "('training @ iter = ', 10500)\n",
    "epoch 54, minibatch 195/195, validation error 6.723157 %\n",
    "     epoch 54, minibatch 195/195, test error of best model 6.502804 %\n",
    "('training @ iter = ', 10600)\n",
    "('training @ iter = ', 10700)\n",
    "epoch 55, minibatch 195/195, validation error 6.693109 %\n",
    "     epoch 55, minibatch 195/195, test error of best model 6.502804 %\n",
    "('training @ iter = ', 10800)\n",
    "('training @ iter = ', 10900)\n",
    "epoch 56, minibatch 195/195, validation error 6.693109 %\n",
    "('training @ iter = ', 11000)\n",
    "('training @ iter = ', 11100)\n",
    "epoch 57, minibatch 195/195, validation error 6.492788 %\n",
    "     epoch 57, minibatch 195/195, test error of best model 6.392628 %\n",
    "('training @ iter = ', 11200)\n",
    "('training @ iter = ', 11300)\n",
    "epoch 58, minibatch 195/195, validation error 6.653045 %\n",
    "('training @ iter = ', 11400)\n",
    "('training @ iter = ', 11500)\n",
    "epoch 59, minibatch 195/195, validation error 6.733173 %\n",
    "('training @ iter = ', 11600)\n",
    "epoch 60, minibatch 195/195, validation error 6.602965 %\n",
    "('training @ iter = ', 11700)\n",
    "('training @ iter = ', 11800)\n",
    "epoch 61, minibatch 195/195, validation error 6.532853 %\n",
    "('training @ iter = ', 11900)\n",
    "('training @ iter = ', 12000)\n",
    "epoch 62, minibatch 195/195, validation error 6.823317 %\n",
    "('training @ iter = ', 12100)\n",
    "('training @ iter = ', 12200)\n",
    "epoch 63, minibatch 195/195, validation error 6.673077 %\n",
    "('training @ iter = ', 12300)\n",
    "('training @ iter = ', 12400)\n",
    "epoch 64, minibatch 195/195, validation error 6.512821 %\n",
    "('training @ iter = ', 12500)\n",
    "('training @ iter = ', 12600)\n",
    "epoch 65, minibatch 195/195, validation error 6.683093 %\n",
    "('training @ iter = ', 12700)\n",
    "('training @ iter = ', 12800)\n",
    "epoch 66, minibatch 195/195, validation error 6.683093 %\n",
    "('training @ iter = ', 12900)\n",
    "('training @ iter = ', 13000)\n",
    "epoch 67, minibatch 195/195, validation error 6.673077 %\n",
    "('training @ iter = ', 13100)\n",
    "('training @ iter = ', 13200)\n",
    "epoch 68, minibatch 195/195, validation error 6.733173 %\n",
    "('training @ iter = ', 13300)\n",
    "('training @ iter = ', 13400)\n",
    "epoch 69, minibatch 195/195, validation error 6.532853 %\n",
    "('training @ iter = ', 13500)\n",
    "('training @ iter = ', 13600)\n",
    "epoch 70, minibatch 195/195, validation error 6.542869 %\n",
    "('training @ iter = ', 13700)\n",
    "('training @ iter = ', 13800)\n",
    "epoch 71, minibatch 195/195, validation error 6.382612 %\n",
    "     epoch 71, minibatch 195/195, test error of best model 6.212340 %\n",
    "('training @ iter = ', 13900)\n",
    "('training @ iter = ', 14000)\n",
    "epoch 72, minibatch 195/195, validation error 6.442708 %\n",
    "('training @ iter = ', 14100)\n",
    "('training @ iter = ', 14200)\n",
    "epoch 73, minibatch 195/195, validation error 6.612981 %\n",
    "('training @ iter = ', 14300)\n",
    "('training @ iter = ', 14400)\n",
    "epoch 74, minibatch 195/195, validation error 6.582933 %\n",
    "('training @ iter = ', 14500)\n",
    "('training @ iter = ', 14600)\n",
    "epoch 75, minibatch 195/195, validation error 6.542869 %\n",
    "('training @ iter = ', 14700)\n",
    "('training @ iter = ', 14800)\n",
    "epoch 76, minibatch 195/195, validation error 6.582933 %\n",
    "('training @ iter = ', 14900)\n",
    "('training @ iter = ', 15000)\n",
    "epoch 77, minibatch 195/195, validation error 6.542869 %\n",
    "('training @ iter = ', 15100)\n",
    "('training @ iter = ', 15200)\n",
    "epoch 78, minibatch 195/195, validation error 6.723157 %\n",
    "('training @ iter = ', 15300)\n",
    "('training @ iter = ', 15400)\n",
    "epoch 79, minibatch 195/195, validation error 6.653045 %\n",
    "('training @ iter = ', 15500)\n",
    "epoch 80, minibatch 195/195, validation error 6.502804 %\n",
    "('training @ iter = ', 15600)\n",
    "('training @ iter = ', 15700)\n",
    "epoch 81, minibatch 195/195, validation error 6.743189 %\n",
    "('training @ iter = ', 15800)\n",
    "('training @ iter = ', 15900)\n",
    "epoch 82, minibatch 195/195, validation error 6.643029 %\n",
    "('training @ iter = ', 16000)\n",
    "('training @ iter = ', 16100)\n",
    "epoch 83, minibatch 195/195, validation error 6.612981 %\n",
    "('training @ iter = ', 16200)\n",
    "('training @ iter = ', 16300)\n",
    "epoch 84, minibatch 195/195, validation error 6.542869 %\n",
    "('training @ iter = ', 16400)\n",
    "('training @ iter = ', 16500)\n",
    "epoch 85, minibatch 195/195, validation error 6.713141 %\n",
    "('training @ iter = ', 16600)\n",
    "('training @ iter = ', 16700)\n",
    "epoch 86, minibatch 195/195, validation error 6.743189 %\n",
    "('training @ iter = ', 16800)\n",
    "('training @ iter = ', 16900)\n",
    "epoch 87, minibatch 195/195, validation error 6.472756 %\n",
    "('training @ iter = ', 17000)\n",
    "('training @ iter = ', 17100)\n",
    "epoch 88, minibatch 195/195, validation error 6.342548 %\n",
    "     epoch 88, minibatch 195/195, test error of best model 6.162260 %\n",
    "('training @ iter = ', 17200)\n",
    "('training @ iter = ', 17300)\n",
    "epoch 89, minibatch 195/195, validation error 6.422676 %\n",
    "('training @ iter = ', 17400)\n",
    "('training @ iter = ', 17500)\n",
    "epoch 90, minibatch 195/195, validation error 6.372596 %\n",
    "('training @ iter = ', 17600)\n",
    "('training @ iter = ', 17700)\n",
    "epoch 91, minibatch 195/195, validation error 6.683093 %\n",
    "('training @ iter = ', 17800)\n",
    "('training @ iter = ', 17900)\n",
    "epoch 92, minibatch 195/195, validation error 6.542869 %\n",
    "('training @ iter = ', 18000)\n",
    "('training @ iter = ', 18100)\n",
    "epoch 93, minibatch 195/195, validation error 6.542869 %\n",
    "('training @ iter = ', 18200)\n",
    "('training @ iter = ', 18300)\n",
    "epoch 94, minibatch 195/195, validation error 6.622997 %\n",
    "('training @ iter = ', 18400)\n",
    "('training @ iter = ', 18500)\n",
    "epoch 95, minibatch 195/195, validation error 6.713141 %\n",
    "('training @ iter = ', 18600)\n",
    "('training @ iter = ', 18700)\n",
    "epoch 96, minibatch 195/195, validation error 6.863381 %\n",
    "('training @ iter = ', 18800)\n",
    "('training @ iter = ', 18900)\n",
    "epoch 97, minibatch 195/195, validation error 6.512821 %\n",
    "('training @ iter = ', 19000)\n",
    "('training @ iter = ', 19100)\n",
    "epoch 98, minibatch 195/195, validation error 6.663061 %\n",
    "('training @ iter = ', 19200)\n",
    "('training @ iter = ', 19300)\n",
    "epoch 99, minibatch 195/195, validation error 6.562901 %\n",
    "('training @ iter = ', 19400)\n",
    "epoch 100, minibatch 195/195, validation error 6.472756 %\n",
    "('training @ iter = ', 19500)\n",
    "('training @ iter = ', 19600)\n",
    "epoch 101, minibatch 195/195, validation error 6.643029 %\n",
    "('training @ iter = ', 19700)\n",
    "('training @ iter = ', 19800)\n",
    "epoch 102, minibatch 195/195, validation error 6.482772 %\n",
    "('training @ iter = ', 19900)\n",
    "('training @ iter = ', 20000)\n",
    "epoch 103, minibatch 195/195, validation error 6.923478 %\n",
    "('training @ iter = ', 20100)\n",
    "('training @ iter = ', 20200)\n",
    "epoch 104, minibatch 195/195, validation error 6.502804 %\n",
    "('training @ iter = ', 20300)\n",
    "('training @ iter = ', 20400)\n",
    "epoch 105, minibatch 195/195, validation error 6.542869 %\n",
    "('training @ iter = ', 20500)\n",
    "('training @ iter = ', 20600)\n",
    "epoch 106, minibatch 195/195, validation error 6.783253 %\n",
    "('training @ iter = ', 20700)\n",
    "('training @ iter = ', 20800)\n",
    "epoch 107, minibatch 195/195, validation error 6.582933 %\n",
    "('training @ iter = ', 20900)\n",
    "('training @ iter = ', 21000)\n",
    "epoch 108, minibatch 195/195, validation error 6.673077 %\n",
    "('training @ iter = ', 21100)\n",
    "('training @ iter = ', 21200)\n",
    "epoch 109, minibatch 195/195, validation error 6.332532 %\n",
    "     epoch 109, minibatch 195/195, test error of best model 6.272436 %\n",
    "('training @ iter = ', 21300)\n",
    "('training @ iter = ', 21400)\n",
    "epoch 110, minibatch 195/195, validation error 6.542869 %\n",
    "('training @ iter = ', 21500)\n",
    "('training @ iter = ', 21600)\n",
    "epoch 111, minibatch 195/195, validation error 6.422676 %\n",
    "('training @ iter = ', 21700)\n",
    "('training @ iter = ', 21800)\n",
    "epoch 112, minibatch 195/195, validation error 7.003606 %\n",
    "('training @ iter = ', 21900)\n",
    "('training @ iter = ', 22000)\n",
    "epoch 113, minibatch 195/195, validation error 6.582933 %\n",
    "('training @ iter = ', 22100)\n",
    "('training @ iter = ', 22200)\n",
    "epoch 114, minibatch 195/195, validation error 7.173878 %\n",
    "('training @ iter = ', 22300)\n",
    "('training @ iter = ', 22400)\n",
    "epoch 115, minibatch 195/195, validation error 6.472756 %\n",
    "('training @ iter = ', 22500)\n",
    "('training @ iter = ', 22600)\n",
    "epoch 116, minibatch 195/195, validation error 6.823317 %\n",
    "('training @ iter = ', 22700)\n",
    "('training @ iter = ', 22800)\n",
    "epoch 117, minibatch 195/195, validation error 6.973558 %\n",
    "('training @ iter = ', 22900)\n",
    "('training @ iter = ', 23000)\n",
    "epoch 118, minibatch 195/195, validation error 6.713141 %\n",
    "('training @ iter = ', 23100)\n",
    "('training @ iter = ', 23200)\n",
    "epoch 119, minibatch 195/195, validation error 6.392628 %\n",
    "('training @ iter = ', 23300)\n",
    "epoch 120, minibatch 195/195, validation error 6.683093 %\n",
    "('training @ iter = ', 23400)\n",
    "('training @ iter = ', 23500)\n",
    "epoch 121, minibatch 195/195, validation error 6.743189 %\n",
    "('training @ iter = ', 23600)\n",
    "('training @ iter = ', 23700)\n",
    "epoch 122, minibatch 195/195, validation error 6.552885 %\n",
    "('training @ iter = ', 23800)\n",
    "('training @ iter = ', 23900)\n",
    "epoch 123, minibatch 195/195, validation error 6.322516 %\n",
    "     epoch 123, minibatch 195/195, test error of best model 6.142228 %\n",
    "('training @ iter = ', 24000)\n",
    "('training @ iter = ', 24100)\n",
    "epoch 124, minibatch 195/195, validation error 6.592949 %\n",
    "('training @ iter = ', 24200)\n",
    "('training @ iter = ', 24300)\n",
    "epoch 125, minibatch 195/195, validation error 6.392628 %\n",
    "('training @ iter = ', 24400)\n",
    "('training @ iter = ', 24500)\n",
    "epoch 126, minibatch 195/195, validation error 6.983574 %\n",
    "('training @ iter = ', 24600)\n",
    "('training @ iter = ', 24700)\n",
    "epoch 127, minibatch 195/195, validation error 6.883413 %\n",
    "('training @ iter = ', 24800)\n",
    "('training @ iter = ', 24900)\n",
    "epoch 128, minibatch 195/195, validation error 6.903446 %\n",
    "('training @ iter = ', 25000)\n",
    "('training @ iter = ', 25100)\n",
    "epoch 129, minibatch 195/195, validation error 6.382612 %\n",
    "('training @ iter = ', 25200)\n",
    "('training @ iter = ', 25300)\n",
    "epoch 130, minibatch 195/195, validation error 6.462740 %\n",
    "('training @ iter = ', 25400)\n",
    "('training @ iter = ', 25500)\n",
    "epoch 131, minibatch 195/195, validation error 6.612981 %\n",
    "('training @ iter = ', 25600)\n",
    "('training @ iter = ', 25700)\n",
    "epoch 132, minibatch 195/195, validation error 6.392628 %\n",
    "('training @ iter = ', 25800)\n",
    "('training @ iter = ', 25900)\n",
    "epoch 133, minibatch 195/195, validation error 6.432692 %\n",
    "('training @ iter = ', 26000)\n",
    "('training @ iter = ', 26100)\n",
    "epoch 134, minibatch 195/195, validation error 6.312500 %\n",
    "     epoch 134, minibatch 195/195, test error of best model 6.272436 %\n",
    "('training @ iter = ', 26200)\n",
    "('training @ iter = ', 26300)\n",
    "epoch 135, minibatch 195/195, validation error 6.472756 %\n",
    "('training @ iter = ', 26400)\n",
    "('training @ iter = ', 26500)\n",
    "epoch 136, minibatch 195/195, validation error 6.803285 %\n",
    "('training @ iter = ', 26600)\n",
    "('training @ iter = ', 26700)\n",
    "epoch 137, minibatch 195/195, validation error 6.402644 %\n",
    "('training @ iter = ', 26800)\n",
    "('training @ iter = ', 26900)\n",
    "epoch 138, minibatch 195/195, validation error 6.412660 %\n",
    "('training @ iter = ', 27000)\n",
    "('training @ iter = ', 27100)\n",
    "epoch 139, minibatch 195/195, validation error 6.602965 %\n",
    "('training @ iter = ', 27200)\n",
    "epoch 140, minibatch 195/195, validation error 6.552885 %\n",
    "('training @ iter = ', 27300)\n",
    "('training @ iter = ', 27400)\n",
    "epoch 141, minibatch 195/195, validation error 6.873397 %\n",
    "('training @ iter = ', 27500)\n",
    "('training @ iter = ', 27600)\n",
    "epoch 142, minibatch 195/195, validation error 6.472756 %\n",
    "('training @ iter = ', 27700)\n",
    "('training @ iter = ', 27800)\n",
    "epoch 143, minibatch 195/195, validation error 6.933494 %\n",
    "('training @ iter = ', 27900)\n",
    "('training @ iter = ', 28000)\n",
    "epoch 144, minibatch 195/195, validation error 6.432692 %\n",
    "('training @ iter = ', 28100)\n",
    "('training @ iter = ', 28200)\n",
    "epoch 145, minibatch 195/195, validation error 6.673077 %\n",
    "('training @ iter = ', 28300)\n",
    "('training @ iter = ', 28400)\n",
    "epoch 146, minibatch 195/195, validation error 6.552885 %\n",
    "('training @ iter = ', 28500)\n",
    "('training @ iter = ', 28600)\n",
    "epoch 147, minibatch 195/195, validation error 6.683093 %\n",
    "('training @ iter = ', 28700)\n",
    "('training @ iter = ', 28800)\n",
    "epoch 148, minibatch 195/195, validation error 7.103766 %\n",
    "('training @ iter = ', 28900)\n",
    "('training @ iter = ', 29000)\n",
    "epoch 149, minibatch 195/195, validation error 6.803285 %\n",
    "('training @ iter = ', 29100)\n",
    "('training @ iter = ', 29200)\n",
    "epoch 150, minibatch 195/195, validation error 6.542869 %\n",
    "('training @ iter = ', 29300)\n",
    "('training @ iter = ', 29400)\n",
    "epoch 151, minibatch 195/195, validation error 6.763221 %\n",
    "('training @ iter = ', 29500)\n",
    "('training @ iter = ', 29600)\n",
    "epoch 152, minibatch 195/195, validation error 6.392628 %\n",
    "('training @ iter = ', 29700)\n",
    "('training @ iter = ', 29800)\n",
    "epoch 153, minibatch 195/195, validation error 6.923478 %\n",
    "('training @ iter = ', 29900)\n",
    "('training @ iter = ', 30000)\n",
    "epoch 154, minibatch 195/195, validation error 7.093750 %\n",
    "('training @ iter = ', 30100)\n",
    "('training @ iter = ', 30200)\n",
    "epoch 155, minibatch 195/195, validation error 6.542869 %\n",
    "('training @ iter = ', 30300)\n",
    "('training @ iter = ', 30400)\n",
    "epoch 156, minibatch 195/195, validation error 6.622997 %\n",
    "('training @ iter = ', 30500)\n",
    "('training @ iter = ', 30600)\n",
    "epoch 157, minibatch 195/195, validation error 6.392628 %\n",
    "('training @ iter = ', 30700)\n",
    "('training @ iter = ', 30800)\n",
    "epoch 158, minibatch 195/195, validation error 6.302484 %\n",
    "     epoch 158, minibatch 195/195, test error of best model 6.442708 %\n",
    "('training @ iter = ', 30900)\n",
    "('training @ iter = ', 31000)\n",
    "epoch 159, minibatch 195/195, validation error 6.663061 %\n",
    "('training @ iter = ', 31100)\n",
    "epoch 160, minibatch 195/195, validation error 6.442708 %\n",
    "('training @ iter = ', 31200)\n",
    "('training @ iter = ', 31300)\n",
    "epoch 161, minibatch 195/195, validation error 6.673077 %\n",
    "('training @ iter = ', 31400)\n",
    "('training @ iter = ', 31500)\n",
    "epoch 162, minibatch 195/195, validation error 6.342548 %\n",
    "('training @ iter = ', 31600)\n",
    "('training @ iter = ', 31700)\n",
    "epoch 163, minibatch 195/195, validation error 6.853365 %\n",
    "('training @ iter = ', 31800)\n",
    "('training @ iter = ', 31900)\n",
    "epoch 164, minibatch 195/195, validation error 6.352564 %\n",
    "('training @ iter = ', 32000)\n",
    "('training @ iter = ', 32100)\n",
    "epoch 165, minibatch 195/195, validation error 6.643029 %\n",
    "('training @ iter = ', 32200)\n",
    "('training @ iter = ', 32300)\n",
    "epoch 166, minibatch 195/195, validation error 6.452724 %\n",
    "('training @ iter = ', 32400)\n",
    "('training @ iter = ', 32500)\n",
    "epoch 167, minibatch 195/195, validation error 6.823317 %\n",
    "('training @ iter = ', 32600)\n",
    "('training @ iter = ', 32700)\n",
    "epoch 168, minibatch 195/195, validation error 6.522837 %\n",
    "('training @ iter = ', 32800)\n",
    "('training @ iter = ', 32900)\n",
    "epoch 169, minibatch 195/195, validation error 6.783253 %\n",
    "('training @ iter = ', 33000)\n",
    "('training @ iter = ', 33100)\n",
    "epoch 170, minibatch 195/195, validation error 6.442708 %\n",
    "('training @ iter = ', 33200)\n",
    "('training @ iter = ', 33300)\n",
    "epoch 171, minibatch 195/195, validation error 6.753205 %\n",
    "('training @ iter = ', 33400)\n",
    "('training @ iter = ', 33500)\n",
    "epoch 172, minibatch 195/195, validation error 6.272436 %\n",
    "     epoch 172, minibatch 195/195, test error of best model 6.502804 %\n",
    "('training @ iter = ', 33600)\n",
    "('training @ iter = ', 33700)\n",
    "epoch 173, minibatch 195/195, validation error 6.602965 %\n",
    "('training @ iter = ', 33800)\n",
    "('training @ iter = ', 33900)\n",
    "epoch 174, minibatch 195/195, validation error 6.793269 %\n",
    "('training @ iter = ', 34000)\n",
    "('training @ iter = ', 34100)\n",
    "epoch 175, minibatch 195/195, validation error 6.342548 %\n",
    "('training @ iter = ', 34200)\n",
    "('training @ iter = ', 34300)\n",
    "epoch 176, minibatch 195/195, validation error 6.302484 %\n",
    "('training @ iter = ', 34400)\n",
    "('training @ iter = ', 34500)\n",
    "epoch 177, minibatch 195/195, validation error 6.122196 %\n",
    "     epoch 177, minibatch 195/195, test error of best model 6.452724 %\n",
    "('training @ iter = ', 34600)\n",
    "('training @ iter = ', 34700)\n",
    "epoch 178, minibatch 195/195, validation error 6.793269 %\n",
    "('training @ iter = ', 34800)\n",
    "('training @ iter = ', 34900)\n",
    "epoch 179, minibatch 195/195, validation error 6.192308 %\n",
    "('training @ iter = ', 35000)\n",
    "epoch 180, minibatch 195/195, validation error 6.903446 %\n",
    "('training @ iter = ', 35100)\n",
    "('training @ iter = ', 35200)\n",
    "epoch 181, minibatch 195/195, validation error 6.362580 %\n",
    "('training @ iter = ', 35300)\n",
    "('training @ iter = ', 35400)\n",
    "epoch 182, minibatch 195/195, validation error 6.803285 %\n",
    "('training @ iter = ', 35500)\n",
    "('training @ iter = ', 35600)\n",
    "epoch 183, minibatch 195/195, validation error 6.352564 %\n",
    "('training @ iter = ', 35700)\n",
    "('training @ iter = ', 35800)\n",
    "epoch 184, minibatch 195/195, validation error 6.392628 %\n",
    "('training @ iter = ', 35900)\n",
    "('training @ iter = ', 36000)\n",
    "epoch 185, minibatch 195/195, validation error 6.312500 %\n",
    "('training @ iter = ', 36100)\n",
    "('training @ iter = ', 36200)\n",
    "epoch 186, minibatch 195/195, validation error 6.312500 %\n",
    "('training @ iter = ', 36300)\n",
    "('training @ iter = ', 36400)\n",
    "epoch 187, minibatch 195/195, validation error 6.332532 %\n",
    "('training @ iter = ', 36500)\n",
    "('training @ iter = ', 36600)\n",
    "epoch 188, minibatch 195/195, validation error 6.582933 %\n",
    "('training @ iter = ', 36700)\n",
    "('training @ iter = ', 36800)\n",
    "epoch 189, minibatch 195/195, validation error 6.512821 %\n",
    "('training @ iter = ', 36900)\n",
    "('training @ iter = ', 37000)\n",
    "epoch 190, minibatch 195/195, validation error 6.372596 %\n",
    "('training @ iter = ', 37100)\n",
    "('training @ iter = ', 37200)\n",
    "epoch 191, minibatch 195/195, validation error 6.262420 %\n",
    "('training @ iter = ', 37300)\n",
    "('training @ iter = ', 37400)\n",
    "epoch 192, minibatch 195/195, validation error 6.192308 %\n",
    "('training @ iter = ', 37500)\n",
    "('training @ iter = ', 37600)\n",
    "epoch 193, minibatch 195/195, validation error 6.332532 %\n",
    "('training @ iter = ', 37700)\n",
    "('training @ iter = ', 37800)\n",
    "epoch 194, minibatch 195/195, validation error 6.703125 %\n",
    "('training @ iter = ', 37900)\n",
    "('training @ iter = ', 38000)\n",
    "epoch 195, minibatch 195/195, validation error 6.052083 %\n",
    "     epoch 195, minibatch 195/195, test error of best model 6.372596 %\n",
    "('training @ iter = ', 38100)\n",
    "('training @ iter = ', 38200)\n",
    "epoch 196, minibatch 195/195, validation error 6.062099 %\n",
    "('training @ iter = ', 38300)\n",
    "('training @ iter = ', 38400)\n",
    "epoch 197, minibatch 195/195, validation error 6.432692 %\n",
    "('training @ iter = ', 38500)\n",
    "('training @ iter = ', 38600)\n",
    "epoch 198, minibatch 195/195, validation error 6.432692 %\n",
    "('training @ iter = ', 38700)\n",
    "('training @ iter = ', 38800)\n",
    "epoch 199, minibatch 195/195, validation error 6.092147 %\n",
    "('training @ iter = ', 38900)\n",
    "epoch 200, minibatch 195/195, validation error 6.152244 %\n",
    "Optimization complete.\n",
    "Best validation score of 6.31083 % obtained at iteration 38025, with test performance 6.432596 %\n",
    "The training process for function run ran for 78.59m\n",
    "----------------------------------------------------------------------\n",
    "\n",
    "\n",
    "...training cnn with include_st = False\n",
    "...training\n",
    "('training @ iter = ', 0)\n",
    "('training @ iter = ', 100)\n",
    "epoch 1, minibatch 195/195, validation error 84.104567 %\n",
    "     epoch 1, minibatch 195/195, test error of best model 84.875801 %\n",
    "('training @ iter = ', 200)\n",
    "('training @ iter = ', 300)\n",
    "epoch 2, minibatch 195/195, validation error 81.610577 %\n",
    "     epoch 2, minibatch 195/195, test error of best model 82.381811 %\n",
    "('training @ iter = ', 400)\n",
    "('training @ iter = ', 500)\n",
    "epoch 3, minibatch 195/195, validation error 78.655849 %\n",
    "     epoch 3, minibatch 195/195, test error of best model 78.735978 %\n",
    "('training @ iter = ', 600)\n",
    "('training @ iter = ', 700)\n",
    "epoch 4, minibatch 195/195, validation error 74.859776 %\n",
    "     epoch 4, minibatch 195/195, test error of best model 75.220353 %\n",
    "('training @ iter = ', 800)\n",
    "('training @ iter = ', 900)\n",
    "epoch 5, minibatch 195/195, validation error 67.548077 %\n",
    "     epoch 5, minibatch 195/195, test error of best model 68.669872 %\n",
    "('training @ iter = ', 1000)\n",
    "('training @ iter = ', 1100)\n",
    "epoch 6, minibatch 195/195, validation error 59.014423 %\n",
    "     epoch 6, minibatch 195/195, test error of best model 59.725561 %\n",
    "('training @ iter = ', 1200)\n",
    "('training @ iter = ', 1300)\n",
    "epoch 7, minibatch 195/195, validation error 52.293670 %\n",
    "     epoch 7, minibatch 195/195, test error of best model 52.724359 %\n",
    "('training @ iter = ', 1400)\n",
    "('training @ iter = ', 1500)\n",
    "epoch 8, minibatch 195/195, validation error 46.264022 %\n",
    "     epoch 8, minibatch 195/195, test error of best model 46.474359 %\n",
    "('training @ iter = ', 1600)\n",
    "('training @ iter = ', 1700)\n",
    "epoch 9, minibatch 195/195, validation error 40.204327 %\n",
    "     epoch 9, minibatch 195/195, test error of best model 40.935497 %\n",
    "('training @ iter = ', 1800)\n",
    "('training @ iter = ', 1900)\n",
    "epoch 10, minibatch 195/195, validation error 35.777244 %\n",
    "     epoch 10, minibatch 195/195, test error of best model 36.027644 %\n",
    "('training @ iter = ', 2000)\n",
    "('training @ iter = ', 2100)\n",
    "epoch 11, minibatch 195/195, validation error 31.911058 %\n",
    "     epoch 11, minibatch 195/195, test error of best model 32.341747 %\n",
    "('training @ iter = ', 2200)\n",
    "('training @ iter = ', 2300)\n",
    "epoch 12, minibatch 195/195, validation error 29.437099 %\n",
    "     epoch 12, minibatch 195/195, test error of best model 29.557292 %\n",
    "('training @ iter = ', 2400)\n",
    "('training @ iter = ', 2500)\n",
    "epoch 13, minibatch 195/195, validation error 27.213542 %\n",
    "     epoch 13, minibatch 195/195, test error of best model 27.584135 %\n",
    "('training @ iter = ', 2600)\n",
    "('training @ iter = ', 2700)\n",
    "epoch 14, minibatch 195/195, validation error 25.190304 %\n",
    "     epoch 14, minibatch 195/195, test error of best model 25.861378 %\n",
    "('training @ iter = ', 2800)\n",
    "('training @ iter = ', 2900)\n",
    "epoch 15, minibatch 195/195, validation error 23.667869 %\n",
    "     epoch 15, minibatch 195/195, test error of best model 24.298878 %\n",
    "('training @ iter = ', 3000)\n",
    "('training @ iter = ', 3100)\n",
    "epoch 16, minibatch 195/195, validation error 22.155449 %\n",
    "     epoch 16, minibatch 195/195, test error of best model 22.946715 %\n",
    "('training @ iter = ', 3200)\n",
    "('training @ iter = ', 3300)\n",
    "epoch 17, minibatch 195/195, validation error 20.933494 %\n",
    "     epoch 17, minibatch 195/195, test error of best model 21.654647 %\n",
    "('training @ iter = ', 3400)\n",
    "('training @ iter = ', 3500)\n",
    "epoch 18, minibatch 195/195, validation error 19.821715 %\n",
    "     epoch 18, minibatch 195/195, test error of best model 20.602965 %\n",
    "('training @ iter = ', 3600)\n",
    "('training @ iter = ', 3700)\n",
    "epoch 19, minibatch 195/195, validation error 19.000401 %\n",
    "     epoch 19, minibatch 195/195, test error of best model 19.641426 %\n",
    "('training @ iter = ', 3800)\n",
    "epoch 20, minibatch 195/195, validation error 18.279247 %\n",
    "     epoch 20, minibatch 195/195, test error of best model 18.780048 %\n",
    "('training @ iter = ', 3900)\n",
    "('training @ iter = ', 4000)\n",
    "epoch 21, minibatch 195/195, validation error 17.558093 %\n",
    "     epoch 21, minibatch 195/195, test error of best model 18.028846 %\n",
    "('training @ iter = ', 4100)\n",
    "('training @ iter = ', 4200)\n",
    "epoch 22, minibatch 195/195, validation error 16.977163 %\n",
    "     epoch 22, minibatch 195/195, test error of best model 17.548077 %\n",
    "('training @ iter = ', 4300)\n",
    "('training @ iter = ', 4400)\n",
    "epoch 23, minibatch 195/195, validation error 16.386218 %\n",
    "     epoch 23, minibatch 195/195, test error of best model 16.666667 %\n",
    "('training @ iter = ', 4500)\n",
    "('training @ iter = ', 4600)\n",
    "epoch 24, minibatch 195/195, validation error 15.705128 %\n",
    "     epoch 24, minibatch 195/195, test error of best model 16.075721 %\n",
    "('training @ iter = ', 4700)\n",
    "('training @ iter = ', 4800)\n",
    "epoch 25, minibatch 195/195, validation error 15.304487 %\n",
    "     epoch 25, minibatch 195/195, test error of best model 15.614984 %\n",
    "('training @ iter = ', 4900)\n",
    "('training @ iter = ', 5000)\n",
    "epoch 26, minibatch 195/195, validation error 14.963942 %\n",
    "     epoch 26, minibatch 195/195, test error of best model 15.024038 %\n",
    "('training @ iter = ', 5100)\n",
    "('training @ iter = ', 5200)\n",
    "epoch 27, minibatch 195/195, validation error 14.593349 %\n",
    "     epoch 27, minibatch 195/195, test error of best model 14.693510 %\n",
    "('training @ iter = ', 5300)\n",
    "('training @ iter = ', 5400)\n",
    "epoch 28, minibatch 195/195, validation error 14.202724 %\n",
    "     epoch 28, minibatch 195/195, test error of best model 14.463141 %\n",
    "('training @ iter = ', 5500)\n",
    "('training @ iter = ', 5600)\n",
    "epoch 29, minibatch 195/195, validation error 13.822115 %\n",
    "     epoch 29, minibatch 195/195, test error of best model 14.152644 %\n",
    "('training @ iter = ', 5700)\n",
    "('training @ iter = ', 5800)\n",
    "epoch 30, minibatch 195/195, validation error 13.511619 %\n",
    "     epoch 30, minibatch 195/195, test error of best model 13.962340 %\n",
    "('training @ iter = ', 5900)\n",
    "('training @ iter = ', 6000)\n",
    "epoch 31, minibatch 195/195, validation error 13.331330 %\n",
    "     epoch 31, minibatch 195/195, test error of best model 13.611779 %\n",
    "('training @ iter = ', 6100)\n",
    "('training @ iter = ', 6200)\n",
    "epoch 32, minibatch 195/195, validation error 13.050881 %\n",
    "     epoch 32, minibatch 195/195, test error of best model 13.331330 %\n",
    "('training @ iter = ', 6300)\n",
    "('training @ iter = ', 6400)\n",
    "epoch 33, minibatch 195/195, validation error 12.860577 %\n",
    "     epoch 33, minibatch 195/195, test error of best model 13.110978 %\n",
    "('training @ iter = ', 6500)\n",
    "('training @ iter = ', 6600)\n",
    "epoch 34, minibatch 195/195, validation error 12.680288 %\n",
    "     epoch 34, minibatch 195/195, test error of best model 12.930689 %\n",
    "('training @ iter = ', 6700)\n",
    "('training @ iter = ', 6800)\n",
    "epoch 35, minibatch 195/195, validation error 12.580128 %\n",
    "     epoch 35, minibatch 195/195, test error of best model 12.830529 %\n",
    "('training @ iter = ', 6900)\n",
    "('training @ iter = ', 7000)\n",
    "epoch 36, minibatch 195/195, validation error 12.469952 %\n",
    "     epoch 36, minibatch 195/195, test error of best model 12.710337 %\n",
    "('training @ iter = ', 7100)\n",
    "('training @ iter = ', 7200)\n",
    "epoch 37, minibatch 195/195, validation error 12.219551 %\n",
    "     epoch 37, minibatch 195/195, test error of best model 12.520032 %\n",
    "('training @ iter = ', 7300)\n",
    "('training @ iter = ', 7400)\n",
    "epoch 38, minibatch 195/195, validation error 12.129407 %\n",
    "     epoch 38, minibatch 195/195, test error of best model 12.359776 %\n",
    "('training @ iter = ', 7500)\n",
    "('training @ iter = ', 7600)\n",
    "epoch 39, minibatch 195/195, validation error 11.999199 %\n",
    "     epoch 39, minibatch 195/195, test error of best model 12.199519 %\n",
    "('training @ iter = ', 7700)\n",
    "epoch 40, minibatch 195/195, validation error 11.788862 %\n",
    "     epoch 40, minibatch 195/195, test error of best model 12.029247 %\n",
    "('training @ iter = ', 7800)\n",
    "('training @ iter = ', 7900)\n",
    "epoch 41, minibatch 195/195, validation error 11.718750 %\n",
    "     epoch 41, minibatch 195/195, test error of best model 11.858974 %\n",
    "('training @ iter = ', 8000)\n",
    "('training @ iter = ', 8100)\n",
    "epoch 42, minibatch 195/195, validation error 11.578526 %\n",
    "     epoch 42, minibatch 195/195, test error of best model 11.708734 %\n",
    "('training @ iter = ', 8200)\n",
    "('training @ iter = ', 8300)\n",
    "epoch 43, minibatch 195/195, validation error 11.478365 %\n",
    "     epoch 43, minibatch 195/195, test error of best model 11.578526 %\n",
    "('training @ iter = ', 8400)\n",
    "('training @ iter = ', 8500)\n",
    "epoch 44, minibatch 195/195, validation error 11.318109 %\n",
    "     epoch 44, minibatch 195/195, test error of best model 11.438301 %\n",
    "('training @ iter = ', 8600)\n",
    "('training @ iter = ', 8700)\n",
    "epoch 45, minibatch 195/195, validation error 11.258013 %\n",
    "     epoch 45, minibatch 195/195, test error of best model 11.308093 %\n",
    "('training @ iter = ', 8800)\n",
    "('training @ iter = ', 8900)\n",
    "epoch 46, minibatch 195/195, validation error 11.177885 %\n",
    "     epoch 46, minibatch 195/195, test error of best model 11.207933 %\n",
    "('training @ iter = ', 9000)\n",
    "('training @ iter = ', 9100)\n",
    "epoch 47, minibatch 195/195, validation error 11.067708 %\n",
    "     epoch 47, minibatch 195/195, test error of best model 11.147837 %\n",
    "('training @ iter = ', 9200)\n",
    "('training @ iter = ', 9300)\n",
    "epoch 48, minibatch 195/195, validation error 11.037660 %\n",
    "     epoch 48, minibatch 195/195, test error of best model 11.067708 %\n",
    "('training @ iter = ', 9400)\n",
    "('training @ iter = ', 9500)\n",
    "epoch 49, minibatch 195/195, validation error 10.957532 %\n",
    "     epoch 49, minibatch 195/195, test error of best model 10.937500 %\n",
    "('training @ iter = ', 9600)\n",
    "('training @ iter = ', 9700)\n",
    "epoch 50, minibatch 195/195, validation error 10.877404 %\n",
    "     epoch 50, minibatch 195/195, test error of best model 10.947516 %\n",
    "('training @ iter = ', 9800)\n",
    "('training @ iter = ', 9900)\n",
    "epoch 51, minibatch 195/195, validation error 10.787260 %\n",
    "     epoch 51, minibatch 195/195, test error of best model 10.867388 %\n",
    "('training @ iter = ', 10000)\n",
    "('training @ iter = ', 10100)\n",
    "epoch 52, minibatch 195/195, validation error 10.717147 %\n",
    "     epoch 52, minibatch 195/195, test error of best model 10.737179 %\n",
    "('training @ iter = ', 10200)\n",
    "('training @ iter = ', 10300)\n",
    "epoch 53, minibatch 195/195, validation error 10.677083 %\n",
    "     epoch 53, minibatch 195/195, test error of best model 10.647035 %\n",
    "('training @ iter = ', 10400)\n",
    "('training @ iter = ', 10500)\n",
    "epoch 54, minibatch 195/195, validation error 10.546875 %\n",
    "     epoch 54, minibatch 195/195, test error of best model 10.566907 %\n",
    "('training @ iter = ', 10600)\n",
    "('training @ iter = ', 10700)\n",
    "epoch 55, minibatch 195/195, validation error 10.396635 %\n",
    "     epoch 55, minibatch 195/195, test error of best model 10.486779 %\n",
    "('training @ iter = ', 10800)\n",
    "('training @ iter = ', 10900)\n",
    "epoch 56, minibatch 195/195, validation error 10.336538 %\n",
    "     epoch 56, minibatch 195/195, test error of best model 10.426683 %\n",
    "('training @ iter = ', 11000)\n",
    "('training @ iter = ', 11100)\n",
    "epoch 57, minibatch 195/195, validation error 10.266426 %\n",
    "     epoch 57, minibatch 195/195, test error of best model 10.386619 %\n",
    "('training @ iter = ', 11200)\n",
    "('training @ iter = ', 11300)\n",
    "epoch 58, minibatch 195/195, validation error 10.126202 %\n",
    "     epoch 58, minibatch 195/195, test error of best model 10.366587 %\n",
    "('training @ iter = ', 11400)\n",
    "('training @ iter = ', 11500)\n",
    "epoch 59, minibatch 195/195, validation error 10.096154 %\n",
    "     epoch 59, minibatch 195/195, test error of best model 10.276442 %\n",
    "('training @ iter = ', 11600)\n",
    "epoch 60, minibatch 195/195, validation error 10.076122 %\n",
    "     epoch 60, minibatch 195/195, test error of best model 10.176282 %\n",
    "('training @ iter = ', 11700)\n",
    "('training @ iter = ', 11800)\n",
    "epoch 61, minibatch 195/195, validation error 9.995994 %\n",
    "     epoch 61, minibatch 195/195, test error of best model 10.106170 %\n",
    "('training @ iter = ', 11900)\n",
    "('training @ iter = ', 12000)\n",
    "epoch 62, minibatch 195/195, validation error 9.925881 %\n",
    "     epoch 62, minibatch 195/195, test error of best model 10.066106 %\n",
    "('training @ iter = ', 12100)\n",
    "('training @ iter = ', 12200)\n",
    "epoch 63, minibatch 195/195, validation error 9.845753 %\n",
    "     epoch 63, minibatch 195/195, test error of best model 9.985978 %\n",
    "('training @ iter = ', 12300)\n",
    "('training @ iter = ', 12400)\n",
    "epoch 64, minibatch 195/195, validation error 9.855769 %\n",
    "('training @ iter = ', 12500)\n",
    "('training @ iter = ', 12600)\n",
    "epoch 65, minibatch 195/195, validation error 9.715545 %\n",
    "     epoch 65, minibatch 195/195, test error of best model 9.825721 %\n",
    "('training @ iter = ', 12700)\n",
    "('training @ iter = ', 12800)\n",
    "epoch 66, minibatch 195/195, validation error 9.675481 %\n",
    "     epoch 66, minibatch 195/195, test error of best model 9.795673 %\n",
    "('training @ iter = ', 12900)\n",
    "('training @ iter = ', 13000)\n",
    "epoch 67, minibatch 195/195, validation error 9.585337 %\n",
    "     epoch 67, minibatch 195/195, test error of best model 9.725561 %\n",
    "('training @ iter = ', 13100)\n",
    "('training @ iter = ', 13200)\n",
    "epoch 68, minibatch 195/195, validation error 9.525240 %\n",
    "     epoch 68, minibatch 195/195, test error of best model 9.665465 %\n",
    "('training @ iter = ', 13300)\n",
    "('training @ iter = ', 13400)\n",
    "epoch 69, minibatch 195/195, validation error 9.445112 %\n",
    "     epoch 69, minibatch 195/195, test error of best model 9.645433 %\n",
    "('training @ iter = ', 13500)\n",
    "('training @ iter = ', 13600)\n",
    "epoch 70, minibatch 195/195, validation error 9.385016 %\n",
    "     epoch 70, minibatch 195/195, test error of best model 9.575321 %\n",
    "('training @ iter = ', 13700)\n",
    "('training @ iter = ', 13800)\n",
    "epoch 71, minibatch 195/195, validation error 9.364984 %\n",
    "     epoch 71, minibatch 195/195, test error of best model 9.525240 %\n",
    "('training @ iter = ', 13900)\n",
    "('training @ iter = ', 14000)\n",
    "epoch 72, minibatch 195/195, validation error 9.314904 %\n",
    "     epoch 72, minibatch 195/195, test error of best model 9.475160 %\n",
    "('training @ iter = ', 14100)\n",
    "('training @ iter = ', 14200)\n",
    "epoch 73, minibatch 195/195, validation error 9.264824 %\n",
    "     epoch 73, minibatch 195/195, test error of best model 9.425080 %\n",
    "('training @ iter = ', 14300)\n",
    "('training @ iter = ', 14400)\n",
    "epoch 74, minibatch 195/195, validation error 9.234776 %\n",
    "     epoch 74, minibatch 195/195, test error of best model 9.395032 %\n",
    "('training @ iter = ', 14500)\n",
    "('training @ iter = ', 14600)\n",
    "epoch 75, minibatch 195/195, validation error 9.214744 %\n",
    "     epoch 75, minibatch 195/195, test error of best model 9.344952 %\n",
    "('training @ iter = ', 14700)\n",
    "('training @ iter = ', 14800)\n",
    "epoch 76, minibatch 195/195, validation error 9.144631 %\n",
    "     epoch 76, minibatch 195/195, test error of best model 9.284856 %\n",
    "('training @ iter = ', 14900)\n",
    "('training @ iter = ', 15000)\n",
    "epoch 77, minibatch 195/195, validation error 9.084535 %\n",
    "     epoch 77, minibatch 195/195, test error of best model 9.284856 %\n",
    "('training @ iter = ', 15100)\n",
    "('training @ iter = ', 15200)\n",
    "epoch 78, minibatch 195/195, validation error 9.084535 %\n",
    "('training @ iter = ', 15300)\n",
    "('training @ iter = ', 15400)\n",
    "epoch 79, minibatch 195/195, validation error 8.994391 %\n",
    "     epoch 79, minibatch 195/195, test error of best model 9.174679 %\n",
    "('training @ iter = ', 15500)\n",
    "epoch 80, minibatch 195/195, validation error 8.914263 %\n",
    "     epoch 80, minibatch 195/195, test error of best model 9.114583 %\n",
    "('training @ iter = ', 15600)\n",
    "('training @ iter = ', 15700)\n",
    "epoch 81, minibatch 195/195, validation error 8.834135 %\n",
    "     epoch 81, minibatch 195/195, test error of best model 9.064503 %\n",
    "('training @ iter = ', 15800)\n",
    "('training @ iter = ', 15900)\n",
    "epoch 82, minibatch 195/195, validation error 8.794071 %\n",
    "     epoch 82, minibatch 195/195, test error of best model 9.024439 %\n",
    "('training @ iter = ', 16000)\n",
    "('training @ iter = ', 16100)\n",
    "epoch 83, minibatch 195/195, validation error 8.774038 %\n",
    "     epoch 83, minibatch 195/195, test error of best model 8.994391 %\n",
    "('training @ iter = ', 16200)\n",
    "('training @ iter = ', 16300)\n",
    "epoch 84, minibatch 195/195, validation error 8.723958 %\n",
    "     epoch 84, minibatch 195/195, test error of best model 8.934295 %\n",
    "('training @ iter = ', 16400)\n",
    "('training @ iter = ', 16500)\n",
    "epoch 85, minibatch 195/195, validation error 8.643830 %\n",
    "     epoch 85, minibatch 195/195, test error of best model 8.924279 %\n",
    "('training @ iter = ', 16600)\n",
    "('training @ iter = ', 16700)\n",
    "epoch 86, minibatch 195/195, validation error 8.613782 %\n",
    "     epoch 86, minibatch 195/195, test error of best model 8.874199 %\n",
    "('training @ iter = ', 16800)\n",
    "('training @ iter = ', 16900)\n",
    "epoch 87, minibatch 195/195, validation error 8.563702 %\n",
    "     epoch 87, minibatch 195/195, test error of best model 8.844151 %\n",
    "('training @ iter = ', 17000)\n",
    "('training @ iter = ', 17100)\n",
    "epoch 88, minibatch 195/195, validation error 8.533654 %\n",
    "     epoch 88, minibatch 195/195, test error of best model 8.804087 %\n",
    "('training @ iter = ', 17200)\n",
    "('training @ iter = ', 17300)\n",
    "epoch 89, minibatch 195/195, validation error 8.493590 %\n",
    "     epoch 89, minibatch 195/195, test error of best model 8.774038 %\n",
    "('training @ iter = ', 17400)\n",
    "('training @ iter = ', 17500)\n",
    "epoch 90, minibatch 195/195, validation error 8.463542 %\n",
    "     epoch 90, minibatch 195/195, test error of best model 8.713942 %\n",
    "('training @ iter = ', 17600)\n",
    "('training @ iter = ', 17700)\n",
    "epoch 91, minibatch 195/195, validation error 8.423478 %\n",
    "     epoch 91, minibatch 195/195, test error of best model 8.643830 %\n",
    "('training @ iter = ', 17800)\n",
    "('training @ iter = ', 17900)\n",
    "epoch 92, minibatch 195/195, validation error 8.333333 %\n",
    "     epoch 92, minibatch 195/195, test error of best model 8.633814 %\n",
    "('training @ iter = ', 18000)\n",
    "('training @ iter = ', 18100)\n",
    "epoch 93, minibatch 195/195, validation error 8.303285 %\n",
    "     epoch 93, minibatch 195/195, test error of best model 8.583734 %\n",
    "('training @ iter = ', 18200)\n",
    "('training @ iter = ', 18300)\n",
    "epoch 94, minibatch 195/195, validation error 8.343349 %\n",
    "('training @ iter = ', 18400)\n",
    "('training @ iter = ', 18500)\n",
    "epoch 95, minibatch 195/195, validation error 8.303285 %\n",
    "('training @ iter = ', 18600)\n",
    "('training @ iter = ', 18700)\n",
    "epoch 96, minibatch 195/195, validation error 8.313301 %\n",
    "('training @ iter = ', 18800)\n",
    "('training @ iter = ', 18900)\n",
    "epoch 97, minibatch 195/195, validation error 8.293269 %\n",
    "     epoch 97, minibatch 195/195, test error of best model 8.423478 %\n",
    "('training @ iter = ', 19000)\n",
    "('training @ iter = ', 19100)\n",
    "epoch 98, minibatch 195/195, validation error 8.273237 %\n",
    "     epoch 98, minibatch 195/195, test error of best model 8.373397 %\n",
    "('training @ iter = ', 19200)\n",
    "('training @ iter = ', 19300)\n",
    "epoch 99, minibatch 195/195, validation error 8.253205 %\n",
    "     epoch 99, minibatch 195/195, test error of best model 8.343349 %\n",
    "('training @ iter = ', 19400)\n",
    "epoch 100, minibatch 195/195, validation error 8.253205 %\n",
    "('training @ iter = ', 19500)\n",
    "('training @ iter = ', 19600)\n",
    "epoch 101, minibatch 195/195, validation error 8.243189 %\n",
    "     epoch 101, minibatch 195/195, test error of best model 8.303285 %\n",
    "('training @ iter = ', 19700)\n",
    "('training @ iter = ', 19800)\n",
    "epoch 102, minibatch 195/195, validation error 8.183093 %\n",
    "     epoch 102, minibatch 195/195, test error of best model 8.303285 %\n",
    "('training @ iter = ', 19900)\n",
    "('training @ iter = ', 20000)\n",
    "epoch 103, minibatch 195/195, validation error 8.143029 %\n",
    "     epoch 103, minibatch 195/195, test error of best model 8.263221 %\n",
    "('training @ iter = ', 20100)\n",
    "('training @ iter = ', 20200)\n",
    "epoch 104, minibatch 195/195, validation error 8.143029 %\n",
    "('training @ iter = ', 20300)\n",
    "('training @ iter = ', 20400)\n",
    "epoch 105, minibatch 195/195, validation error 8.072917 %\n",
    "     epoch 105, minibatch 195/195, test error of best model 8.233173 %\n",
    "('training @ iter = ', 20500)\n",
    "('training @ iter = ', 20600)\n",
    "epoch 106, minibatch 195/195, validation error 8.042869 %\n",
    "     epoch 106, minibatch 195/195, test error of best model 8.213141 %\n",
    "('training @ iter = ', 20700)\n",
    "('training @ iter = ', 20800)\n",
    "epoch 107, minibatch 195/195, validation error 7.952724 %\n",
    "     epoch 107, minibatch 195/195, test error of best model 8.193109 %\n",
    "('training @ iter = ', 20900)\n",
    "('training @ iter = ', 21000)\n",
    "epoch 108, minibatch 195/195, validation error 7.932692 %\n",
    "     epoch 108, minibatch 195/195, test error of best model 8.122997 %\n",
    "('training @ iter = ', 21100)\n",
    "('training @ iter = ', 21200)\n",
    "epoch 109, minibatch 195/195, validation error 7.912660 %\n",
    "     epoch 109, minibatch 195/195, test error of best model 8.112981 %\n",
    "('training @ iter = ', 21300)\n",
    "('training @ iter = ', 21400)\n",
    "epoch 110, minibatch 195/195, validation error 7.902644 %\n",
    "     epoch 110, minibatch 195/195, test error of best model 8.092949 %\n",
    "('training @ iter = ', 21500)\n",
    "('training @ iter = ', 21600)\n",
    "epoch 111, minibatch 195/195, validation error 7.892628 %\n",
    "     epoch 111, minibatch 195/195, test error of best model 8.092949 %\n",
    "('training @ iter = ', 21700)\n",
    "('training @ iter = ', 21800)\n",
    "epoch 112, minibatch 195/195, validation error 7.902644 %\n",
    "('training @ iter = ', 21900)\n",
    "('training @ iter = ', 22000)\n",
    "epoch 113, minibatch 195/195, validation error 7.882612 %\n",
    "     epoch 113, minibatch 195/195, test error of best model 8.012821 %\n",
    "('training @ iter = ', 22100)\n",
    "('training @ iter = ', 22200)\n",
    "epoch 114, minibatch 195/195, validation error 7.842548 %\n",
    "     epoch 114, minibatch 195/195, test error of best model 7.982772 %\n",
    "('training @ iter = ', 22300)\n",
    "('training @ iter = ', 22400)\n",
    "epoch 115, minibatch 195/195, validation error 7.782452 %\n",
    "     epoch 115, minibatch 195/195, test error of best model 7.982772 %\n",
    "('training @ iter = ', 22500)\n",
    "('training @ iter = ', 22600)\n",
    "epoch 116, minibatch 195/195, validation error 7.782452 %\n",
    "('training @ iter = ', 22700)\n",
    "('training @ iter = ', 22800)\n",
    "epoch 117, minibatch 195/195, validation error 7.792468 %\n",
    "('training @ iter = ', 22900)\n",
    "('training @ iter = ', 23000)\n",
    "epoch 118, minibatch 195/195, validation error 7.752404 %\n",
    "     epoch 118, minibatch 195/195, test error of best model 7.872596 %\n",
    "('training @ iter = ', 23100)\n",
    "('training @ iter = ', 23200)\n",
    "epoch 119, minibatch 195/195, validation error 7.732372 %\n",
    "     epoch 119, minibatch 195/195, test error of best model 7.832532 %\n",
    "('training @ iter = ', 23300)\n",
    "epoch 120, minibatch 195/195, validation error 7.702324 %\n",
    "     epoch 120, minibatch 195/195, test error of best model 7.802484 %\n",
    "('training @ iter = ', 23400)\n",
    "('training @ iter = ', 23500)\n",
    "epoch 121, minibatch 195/195, validation error 7.652244 %\n",
    "     epoch 121, minibatch 195/195, test error of best model 7.762420 %\n",
    "('training @ iter = ', 23600)\n",
    "('training @ iter = ', 23700)\n",
    "epoch 122, minibatch 195/195, validation error 7.642228 %\n",
    "     epoch 122, minibatch 195/195, test error of best model 7.742388 %\n",
    "('training @ iter = ', 23800)\n",
    "('training @ iter = ', 23900)\n",
    "epoch 123, minibatch 195/195, validation error 7.612179 %\n",
    "     epoch 123, minibatch 195/195, test error of best model 7.702324 %\n",
    "('training @ iter = ', 24000)\n",
    "('training @ iter = ', 24100)\n",
    "epoch 124, minibatch 195/195, validation error 7.582131 %\n",
    "     epoch 124, minibatch 195/195, test error of best model 7.692308 %\n",
    "('training @ iter = ', 24200)\n",
    "('training @ iter = ', 24300)\n",
    "epoch 125, minibatch 195/195, validation error 7.542067 %\n",
    "     epoch 125, minibatch 195/195, test error of best model 7.662260 %\n",
    "('training @ iter = ', 24400)\n",
    "('training @ iter = ', 24500)\n",
    "epoch 126, minibatch 195/195, validation error 7.512019 %\n",
    "     epoch 126, minibatch 195/195, test error of best model 7.652244 %\n",
    "('training @ iter = ', 24600)\n",
    "('training @ iter = ', 24700)\n",
    "epoch 127, minibatch 195/195, validation error 7.491987 %\n",
    "     epoch 127, minibatch 195/195, test error of best model 7.642228 %\n",
    "('training @ iter = ', 24800)\n",
    "('training @ iter = ', 24900)\n",
    "epoch 128, minibatch 195/195, validation error 7.471955 %\n",
    "     epoch 128, minibatch 195/195, test error of best model 7.642228 %\n",
    "('training @ iter = ', 25000)\n",
    "('training @ iter = ', 25100)\n",
    "epoch 129, minibatch 195/195, validation error 7.441907 %\n",
    "     epoch 129, minibatch 195/195, test error of best model 7.632212 %\n",
    "('training @ iter = ', 25200)\n",
    "('training @ iter = ', 25300)\n",
    "epoch 130, minibatch 195/195, validation error 7.421875 %\n",
    "     epoch 130, minibatch 195/195, test error of best model 7.622196 %\n",
    "('training @ iter = ', 25400)\n",
    "('training @ iter = ', 25500)\n",
    "epoch 131, minibatch 195/195, validation error 7.421875 %\n",
    "('training @ iter = ', 25600)\n",
    "('training @ iter = ', 25700)\n",
    "epoch 132, minibatch 195/195, validation error 7.381811 %\n",
    "     epoch 132, minibatch 195/195, test error of best model 7.572115 %\n",
    "('training @ iter = ', 25800)\n",
    "('training @ iter = ', 25900)\n",
    "epoch 133, minibatch 195/195, validation error 7.351763 %\n",
    "     epoch 133, minibatch 195/195, test error of best model 7.572115 %\n",
    "('training @ iter = ', 26000)\n",
    "('training @ iter = ', 26100)\n",
    "epoch 134, minibatch 195/195, validation error 7.351763 %\n",
    "('training @ iter = ', 26200)\n",
    "('training @ iter = ', 26300)\n",
    "epoch 135, minibatch 195/195, validation error 7.331731 %\n",
    "     epoch 135, minibatch 195/195, test error of best model 7.582131 %\n",
    "('training @ iter = ', 26400)\n",
    "('training @ iter = ', 26500)\n",
    "epoch 136, minibatch 195/195, validation error 7.311699 %\n",
    "     epoch 136, minibatch 195/195, test error of best model 7.582131 %\n",
    "('training @ iter = ', 26600)\n",
    "('training @ iter = ', 26700)\n",
    "epoch 137, minibatch 195/195, validation error 7.291667 %\n",
    "     epoch 137, minibatch 195/195, test error of best model 7.602163 %\n",
    "('training @ iter = ', 26800)\n",
    "('training @ iter = ', 26900)\n",
    "epoch 138, minibatch 195/195, validation error 7.281651 %\n",
    "     epoch 138, minibatch 195/195, test error of best model 7.562099 %\n",
    "('training @ iter = ', 27000)\n",
    "('training @ iter = ', 27100)\n",
    "epoch 139, minibatch 195/195, validation error 7.251603 %\n",
    "     epoch 139, minibatch 195/195, test error of best model 7.542067 %\n",
    "('training @ iter = ', 27200)\n",
    "epoch 140, minibatch 195/195, validation error 7.231571 %\n",
    "     epoch 140, minibatch 195/195, test error of best model 7.532051 %\n",
    "('training @ iter = ', 27300)\n",
    "('training @ iter = ', 27400)\n",
    "epoch 141, minibatch 195/195, validation error 7.201522 %\n",
    "     epoch 141, minibatch 195/195, test error of best model 7.502003 %\n",
    "('training @ iter = ', 27500)\n",
    "('training @ iter = ', 27600)\n",
    "epoch 142, minibatch 195/195, validation error 7.171474 %\n",
    "     epoch 142, minibatch 195/195, test error of best model 7.491987 %\n",
    "('training @ iter = ', 27700)\n",
    "('training @ iter = ', 27800)\n",
    "epoch 143, minibatch 195/195, validation error 7.181490 %\n",
    "('training @ iter = ', 27900)\n",
    "('training @ iter = ', 28000)\n",
    "epoch 144, minibatch 195/195, validation error 7.181490 %\n",
    "('training @ iter = ', 28100)\n",
    "('training @ iter = ', 28200)\n",
    "epoch 145, minibatch 195/195, validation error 7.131410 %\n",
    "     epoch 145, minibatch 195/195, test error of best model 7.441907 %\n",
    "('training @ iter = ', 28300)\n",
    "('training @ iter = ', 28400)\n",
    "epoch 146, minibatch 195/195, validation error 7.111378 %\n",
    "     epoch 146, minibatch 195/195, test error of best model 7.421875 %\n",
    "('training @ iter = ', 28500)\n",
    "('training @ iter = ', 28600)\n",
    "epoch 147, minibatch 195/195, validation error 7.081330 %\n",
    "     epoch 147, minibatch 195/195, test error of best model 7.401843 %\n",
    "('training @ iter = ', 28700)\n",
    "('training @ iter = ', 28800)\n",
    "epoch 148, minibatch 195/195, validation error 7.081330 %\n",
    "('training @ iter = ', 28900)\n",
    "('training @ iter = ', 29000)\n",
    "epoch 149, minibatch 195/195, validation error 7.081330 %\n",
    "('training @ iter = ', 29100)\n",
    "('training @ iter = ', 29200)\n",
    "epoch 150, minibatch 195/195, validation error 7.081330 %\n",
    "('training @ iter = ', 29300)\n",
    "('training @ iter = ', 29400)\n",
    "epoch 151, minibatch 195/195, validation error 7.091346 %\n",
    "('training @ iter = ', 29500)\n",
    "('training @ iter = ', 29600)\n",
    "epoch 152, minibatch 195/195, validation error 7.091346 %\n",
    "('training @ iter = ', 29700)\n",
    "('training @ iter = ', 29800)\n",
    "epoch 153, minibatch 195/195, validation error 7.101362 %\n",
    "('training @ iter = ', 29900)\n",
    "('training @ iter = ', 30000)\n",
    "epoch 154, minibatch 195/195, validation error 7.081330 %\n",
    "('training @ iter = ', 30100)\n",
    "('training @ iter = ', 30200)\n",
    "epoch 155, minibatch 195/195, validation error 7.061298 %\n",
    "     epoch 155, minibatch 195/195, test error of best model 7.271635 %\n",
    "('training @ iter = ', 30300)\n",
    "('training @ iter = ', 30400)\n",
    "epoch 156, minibatch 195/195, validation error 7.031250 %\n",
    "     epoch 156, minibatch 195/195, test error of best model 7.231571 %\n",
    "('training @ iter = ', 30500)\n",
    "('training @ iter = ', 30600)\n",
    "epoch 157, minibatch 195/195, validation error 6.981170 %\n",
    "     epoch 157, minibatch 195/195, test error of best model 7.231571 %\n",
    "('training @ iter = ', 30700)\n",
    "('training @ iter = ', 30800)\n",
    "epoch 158, minibatch 195/195, validation error 6.971154 %\n",
    "     epoch 158, minibatch 195/195, test error of best model 7.251603 %\n",
    "('training @ iter = ', 30900)\n",
    "('training @ iter = ', 31000)\n",
    "epoch 159, minibatch 195/195, validation error 6.951122 %\n",
    "     epoch 159, minibatch 195/195, test error of best model 7.251603 %\n",
    "('training @ iter = ', 31100)\n",
    "epoch 160, minibatch 195/195, validation error 6.941106 %\n",
    "     epoch 160, minibatch 195/195, test error of best model 7.251603 %\n",
    "('training @ iter = ', 31200)\n",
    "('training @ iter = ', 31300)\n",
    "epoch 161, minibatch 195/195, validation error 6.921074 %\n",
    "     epoch 161, minibatch 195/195, test error of best model 7.231571 %\n",
    "('training @ iter = ', 31400)\n",
    "('training @ iter = ', 31500)\n",
    "epoch 162, minibatch 195/195, validation error 6.971154 %\n",
    "('training @ iter = ', 31600)\n",
    "('training @ iter = ', 31700)\n",
    "epoch 163, minibatch 195/195, validation error 6.961138 %\n",
    "('training @ iter = ', 31800)\n",
    "('training @ iter = ', 31900)\n",
    "epoch 164, minibatch 195/195, validation error 6.951122 %\n",
    "('training @ iter = ', 32000)\n",
    "('training @ iter = ', 32100)\n",
    "epoch 165, minibatch 195/195, validation error 6.951122 %\n",
    "('training @ iter = ', 32200)\n",
    "('training @ iter = ', 32300)\n",
    "epoch 166, minibatch 195/195, validation error 6.921074 %\n",
    "('training @ iter = ', 32400)\n",
    "('training @ iter = ', 32500)\n",
    "epoch 167, minibatch 195/195, validation error 6.921074 %\n",
    "('training @ iter = ', 32600)\n",
    "('training @ iter = ', 32700)\n",
    "epoch 168, minibatch 195/195, validation error 6.901042 %\n",
    "     epoch 168, minibatch 195/195, test error of best model 7.261619 %\n",
    "('training @ iter = ', 32800)\n",
    "('training @ iter = ', 32900)\n",
    "epoch 169, minibatch 195/195, validation error 6.911058 %\n",
    "('training @ iter = ', 33000)\n",
    "('training @ iter = ', 33100)\n",
    "epoch 170, minibatch 195/195, validation error 6.911058 %\n",
    "('training @ iter = ', 33200)\n",
    "('training @ iter = ', 33300)\n",
    "epoch 171, minibatch 195/195, validation error 6.891026 %\n",
    "     epoch 171, minibatch 195/195, test error of best model 7.171474 %\n",
    "('training @ iter = ', 33400)\n",
    "('training @ iter = ', 33500)\n",
    "epoch 172, minibatch 195/195, validation error 6.881010 %\n",
    "     epoch 172, minibatch 195/195, test error of best model 7.181490 %\n",
    "('training @ iter = ', 33600)\n",
    "('training @ iter = ', 33700)\n",
    "epoch 173, minibatch 195/195, validation error 6.860978 %\n",
    "     epoch 173, minibatch 195/195, test error of best model 7.161458 %\n",
    "('training @ iter = ', 33800)\n",
    "('training @ iter = ', 33900)\n",
    "epoch 174, minibatch 195/195, validation error 6.840946 %\n",
    "     epoch 174, minibatch 195/195, test error of best model 7.161458 %\n",
    "('training @ iter = ', 34000)\n",
    "('training @ iter = ', 34100)\n",
    "epoch 175, minibatch 195/195, validation error 6.830929 %\n",
    "     epoch 175, minibatch 195/195, test error of best model 7.131410 %\n",
    "('training @ iter = ', 34200)\n",
    "('training @ iter = ', 34300)\n",
    "epoch 176, minibatch 195/195, validation error 6.830929 %\n",
    "('training @ iter = ', 34400)\n",
    "('training @ iter = ', 34500)\n",
    "epoch 177, minibatch 195/195, validation error 6.790865 %\n",
    "     epoch 177, minibatch 195/195, test error of best model 7.131410 %\n",
    "('training @ iter = ', 34600)\n",
    "('training @ iter = ', 34700)\n",
    "epoch 178, minibatch 195/195, validation error 6.800881 %\n",
    "('training @ iter = ', 34800)\n",
    "('training @ iter = ', 34900)\n",
    "epoch 179, minibatch 195/195, validation error 6.800881 %\n",
    "('training @ iter = ', 35000)\n",
    "epoch 180, minibatch 195/195, validation error 6.780849 %\n",
    "     epoch 180, minibatch 195/195, test error of best model 7.091346 %\n",
    "('training @ iter = ', 35100)\n",
    "('training @ iter = ', 35200)\n",
    "epoch 181, minibatch 195/195, validation error 6.790865 %\n",
    "('training @ iter = ', 35300)\n",
    "('training @ iter = ', 35400)\n",
    "epoch 182, minibatch 195/195, validation error 6.750801 %\n",
    "     epoch 182, minibatch 195/195, test error of best model 7.121394 %\n",
    "('training @ iter = ', 35500)\n",
    "('training @ iter = ', 35600)\n",
    "epoch 183, minibatch 195/195, validation error 6.750801 %\n",
    "('training @ iter = ', 35700)\n",
    "('training @ iter = ', 35800)\n",
    "epoch 184, minibatch 195/195, validation error 6.750801 %\n",
    "('training @ iter = ', 35900)\n",
    "('training @ iter = ', 36000)\n",
    "epoch 185, minibatch 195/195, validation error 6.720753 %\n",
    "     epoch 185, minibatch 195/195, test error of best model 7.081330 %\n",
    "('training @ iter = ', 36100)\n",
    "('training @ iter = ', 36200)\n",
    "epoch 186, minibatch 195/195, validation error 6.730769 %\n",
    "('training @ iter = ', 36300)\n",
    "('training @ iter = ', 36400)\n",
    "epoch 187, minibatch 195/195, validation error 6.730769 %\n",
    "('training @ iter = ', 36500)\n",
    "('training @ iter = ', 36600)\n",
    "epoch 188, minibatch 195/195, validation error 6.720753 %\n",
    "('training @ iter = ', 36700)\n",
    "('training @ iter = ', 36800)\n",
    "epoch 189, minibatch 195/195, validation error 6.730769 %\n",
    "('training @ iter = ', 36900)\n",
    "('training @ iter = ', 37000)\n",
    "epoch 190, minibatch 195/195, validation error 6.700721 %\n",
    "     epoch 190, minibatch 195/195, test error of best model 7.031250 %\n",
    "('training @ iter = ', 37100)\n",
    "('training @ iter = ', 37200)\n",
    "epoch 191, minibatch 195/195, validation error 6.690705 %\n",
    "     epoch 191, minibatch 195/195, test error of best model 7.001202 %\n",
    "('training @ iter = ', 37300)\n",
    "('training @ iter = ', 37400)\n",
    "epoch 192, minibatch 195/195, validation error 6.700721 %\n",
    "('training @ iter = ', 37500)\n",
    "('training @ iter = ', 37600)\n",
    "epoch 193, minibatch 195/195, validation error 6.700721 %\n",
    "('training @ iter = ', 37700)\n",
    "('training @ iter = ', 37800)\n",
    "epoch 194, minibatch 195/195, validation error 6.680689 %\n",
    "     epoch 194, minibatch 195/195, test error of best model 7.001202 %\n",
    "('training @ iter = ', 37900)\n",
    "('training @ iter = ', 38000)\n",
    "epoch 195, minibatch 195/195, validation error 6.660657 %\n",
    "     epoch 195, minibatch 195/195, test error of best model 6.971154 %\n",
    "('training @ iter = ', 38100)\n",
    "('training @ iter = ', 38200)\n",
    "epoch 196, minibatch 195/195, validation error 6.650641 %\n",
    "     epoch 196, minibatch 195/195, test error of best model 6.981170 %\n",
    "('training @ iter = ', 38300)\n",
    "('training @ iter = ', 38400)\n",
    "epoch 197, minibatch 195/195, validation error 6.630609 %\n",
    "     epoch 197, minibatch 195/195, test error of best model 6.981170 %\n",
    "('training @ iter = ', 38500)\n",
    "('training @ iter = ', 38600)\n",
    "epoch 198, minibatch 195/195, validation error 6.640625 %\n",
    "('training @ iter = ', 38700)\n",
    "('training @ iter = ', 38800)\n",
    "epoch 199, minibatch 195/195, validation error 6.590545 %\n",
    "     epoch 199, minibatch 195/195, test error of best model 6.991186 %\n",
    "('training @ iter = ', 38900)\n",
    "epoch 200, minibatch 195/195, validation error 6.600561 %\n",
    "Optimization complete.\n",
    "Best validation score of 6.590545 % obtained at iteration 38805, with test performance 6.991186 %\n",
    "The training process for function run ran for 39.12m\n",
    "----------------------------------------------------------------------"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}