HAR-CNN_LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HAR CNN + LSTM training "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import numpy as np\n",
    "import os\n",
    "from utils.utilities import *\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gabriele/Documents/TestingArea/DeepLearningHAR/utils/utilities.py:37: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  X[:,:,i_ch] = dat_.as_matrix()\n"
     ]
    }
   ],
   "source": [
    "X_train, labels_train, list_ch_train = read_data(data_path=\"./UCIHAR/\", split=\"train\") # train\n",
    "X_test, labels_test, list_ch_test = read_data(data_path=\"./UCIHAR/\", split=\"test\") # test\n",
    "\n",
    "assert list_ch_train == list_ch_test, \"Mistmatch in channels!\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Standardize\n",
    "X_train, X_test = standardize(X_train, X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train/Validation Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_tr, X_vld, lab_tr, lab_vld = train_test_split(X_train, labels_train,\n",
    "                                                stratify = labels_train, random_state = 123)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "One-hot encoding:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_tr = one_hot(lab_tr)\n",
    "y_vld = one_hot(lab_vld)\n",
    "y_test = one_hot(labels_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_size = 27         # 3 times the amount of channels\n",
    "lstm_layers = 2        # Number of layers\n",
    "batch_size = 600       # Batch size\n",
    "seq_len = 128          # Number of steps\n",
    "learning_rate = 0.0001  # Learning rate (default is 0.001)\n",
    "epochs = 1000\n",
    "\n",
    "# Fixed\n",
    "n_classes = 6\n",
    "n_channels = 9"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Construct the graph\n",
    "Placeholders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "\n",
    "# Construct placeholders\n",
    "with graph.as_default():\n",
    "    inputs_ = tf.placeholder(tf.float32, [None, seq_len, n_channels], name = 'inputs')\n",
    "    labels_ = tf.placeholder(tf.float32, [None, n_classes], name = 'labels')\n",
    "    keep_prob_ = tf.placeholder(tf.float32, name = 'keep')\n",
    "    learning_rate_ = tf.placeholder(tf.float32, name = 'learning_rate')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build Convolutional Layer(s)\n",
    "\n",
    "Questions: \n",
    "* Should we use a different activation? Like tf.nn.tanh?\n",
    "* Should we use pooling? average or max?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convolutional layers\n",
    "with graph.as_default():\n",
    "    # (batch, 128, 9) --> (batch, 128, 18)\n",
    "    conv1 = tf.layers.conv1d(inputs=inputs_, filters=18, kernel_size=2, strides=1, \n",
    "                             padding='same', activation = tf.nn.relu)\n",
    "    n_ch = n_channels *2\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, pass to LSTM cells"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "with graph.as_default():\n",
    "    # Construct the LSTM inputs and LSTM cells\n",
    "    lstm_in = tf.transpose(conv1, [1,0,2]) # reshape into (seq_len, batch, channels)\n",
    "    lstm_in = tf.reshape(lstm_in, [-1, n_ch]) # Now (seq_len*N, n_channels)\n",
    "    \n",
    "    # To cells\n",
    "    lstm_in = tf.layers.dense(lstm_in, lstm_size, activation=None) # or tf.nn.relu, tf.nn.sigmoid, tf.nn.tanh?\n",
    "    \n",
    "    # Open up the tensor into a list of seq_len pieces\n",
    "    lstm_in = tf.split(lstm_in, seq_len, 0)\n",
    "    \n",
    "    # Add LSTM layers\n",
    "    lstm = tf.contrib.rnn.BasicLSTMCell(lstm_size)\n",
    "    drop = tf.contrib.rnn.DropoutWrapper(lstm, output_keep_prob=keep_prob_)\n",
    "    cell = tf.contrib.rnn.MultiRNNCell([drop] * lstm_layers)\n",
    "    initial_state = cell.zero_state(batch_size, tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define forward pass and cost function:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "with graph.as_default():\n",
    "    outputs, final_state = tf.contrib.rnn.static_rnn(cell, lstm_in, dtype=tf.float32,\n",
    "                                                     initial_state = initial_state)\n",
    "    \n",
    "    # We only need the last output tensor to pass into a classifier\n",
    "    logits = tf.layers.dense(outputs[-1], n_classes, name='logits')\n",
    "    \n",
    "    # Cost function and optimizer\n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels_))\n",
    "    #optimizer = tf.train.AdamOptimizer(learning_rate_).minimize(cost) # No grad clipping\n",
    "    \n",
    "    # Grad clipping\n",
    "    train_op = tf.train.AdamOptimizer(learning_rate_)\n",
    "\n",
    "    gradients = train_op.compute_gradients(cost)\n",
    "    capped_gradients = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in gradients]\n",
    "    optimizer = train_op.apply_gradients(capped_gradients)\n",
    "    \n",
    "    # Accuracy\n",
    "    correct_pred = tf.equal(tf.argmax(logits, 1), tf.argmax(labels_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32), name='accuracy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "if (os.path.exists('checkpoints-crnn') == False):\n",
    "    !mkdir checkpoints-crnn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0/1000 Iteration: 5 Train loss: 1.774352 Train acc: 0.216667\n",
      "Epoch: 1/1000 Iteration: 10 Train loss: 1.762559 Train acc: 0.235000\n",
      "Epoch: 1/1000 Iteration: 15 Train loss: 1.754483 Train acc: 0.270000\n",
      "Epoch: 2/1000 Iteration: 20 Train loss: 1.749729 Train acc: 0.275000\n",
      "Epoch: 2/1000 Iteration: 25 Train loss: 1.740363 Train acc: 0.260000\n",
      "Epoch: 2/1000 Iteration: 25 Validation loss: 1.734952 Validation acc: 0.303333\n",
      "Epoch: 3/1000 Iteration: 30 Train loss: 1.746327 Train acc: 0.246667\n",
      "Epoch: 3/1000 Iteration: 35 Train loss: 1.729172 Train acc: 0.281667\n",
      "Epoch: 4/1000 Iteration: 40 Train loss: 1.732908 Train acc: 0.281667\n",
      "Epoch: 4/1000 Iteration: 45 Train loss: 1.736876 Train acc: 0.256667\n",
      "Epoch: 5/1000 Iteration: 50 Train loss: 1.699082 Train acc: 0.325000\n",
      "Epoch: 5/1000 Iteration: 50 Validation loss: 1.692070 Validation acc: 0.322222\n",
      "Epoch: 6/1000 Iteration: 55 Train loss: 1.695403 Train acc: 0.315000\n",
      "Epoch: 6/1000 Iteration: 60 Train loss: 1.690983 Train acc: 0.315000\n",
      "Epoch: 7/1000 Iteration: 65 Train loss: 1.674535 Train acc: 0.350000\n",
      "Epoch: 7/1000 Iteration: 70 Train loss: 1.673686 Train acc: 0.318333\n",
      "Epoch: 8/1000 Iteration: 75 Train loss: 1.681741 Train acc: 0.313333\n",
      "Epoch: 8/1000 Iteration: 75 Validation loss: 1.643774 Validation acc: 0.336667\n",
      "Epoch: 8/1000 Iteration: 80 Train loss: 1.664194 Train acc: 0.351667\n",
      "Epoch: 9/1000 Iteration: 85 Train loss: 1.652401 Train acc: 0.343333\n",
      "Epoch: 9/1000 Iteration: 90 Train loss: 1.661009 Train acc: 0.341667\n",
      "Epoch: 10/1000 Iteration: 95 Train loss: 1.604457 Train acc: 0.385000\n",
      "Epoch: 11/1000 Iteration: 100 Train loss: 1.616532 Train acc: 0.395000\n",
      "Epoch: 11/1000 Iteration: 100 Validation loss: 1.587901 Validation acc: 0.386111\n",
      "Epoch: 11/1000 Iteration: 105 Train loss: 1.597598 Train acc: 0.375000\n",
      "Epoch: 12/1000 Iteration: 110 Train loss: 1.577428 Train acc: 0.405000\n",
      "Epoch: 12/1000 Iteration: 115 Train loss: 1.574137 Train acc: 0.396667\n",
      "Epoch: 13/1000 Iteration: 120 Train loss: 1.587901 Train acc: 0.383333\n",
      "Epoch: 13/1000 Iteration: 125 Train loss: 1.573870 Train acc: 0.396667\n",
      "Epoch: 13/1000 Iteration: 125 Validation loss: 1.527435 Validation acc: 0.453333\n",
      "Epoch: 14/1000 Iteration: 130 Train loss: 1.574305 Train acc: 0.385000\n",
      "Epoch: 14/1000 Iteration: 135 Train loss: 1.559772 Train acc: 0.410000\n",
      "Epoch: 15/1000 Iteration: 140 Train loss: 1.501554 Train acc: 0.456667\n",
      "Epoch: 16/1000 Iteration: 145 Train loss: 1.502274 Train acc: 0.481667\n",
      "Epoch: 16/1000 Iteration: 150 Train loss: 1.478691 Train acc: 0.443333\n",
      "Epoch: 16/1000 Iteration: 150 Validation loss: 1.465121 Validation acc: 0.513333\n",
      "Epoch: 17/1000 Iteration: 155 Train loss: 1.464293 Train acc: 0.470000\n",
      "Epoch: 17/1000 Iteration: 160 Train loss: 1.467205 Train acc: 0.491667\n",
      "Epoch: 18/1000 Iteration: 165 Train loss: 1.471306 Train acc: 0.471667\n",
      "Epoch: 18/1000 Iteration: 170 Train loss: 1.484908 Train acc: 0.440000\n",
      "Epoch: 19/1000 Iteration: 175 Train loss: 1.463877 Train acc: 0.476667\n",
      "Epoch: 19/1000 Iteration: 175 Validation loss: 1.401332 Validation acc: 0.560556\n",
      "Epoch: 19/1000 Iteration: 180 Train loss: 1.471329 Train acc: 0.450000\n",
      "Epoch: 20/1000 Iteration: 185 Train loss: 1.410011 Train acc: 0.473333\n",
      "Epoch: 21/1000 Iteration: 190 Train loss: 1.405807 Train acc: 0.493333\n",
      "Epoch: 21/1000 Iteration: 195 Train loss: 1.365806 Train acc: 0.536667\n",
      "Epoch: 22/1000 Iteration: 200 Train loss: 1.360165 Train acc: 0.543333\n",
      "Epoch: 22/1000 Iteration: 200 Validation loss: 1.335482 Validation acc: 0.623333\n",
      "Epoch: 22/1000 Iteration: 205 Train loss: 1.366619 Train acc: 0.498333\n",
      "Epoch: 23/1000 Iteration: 210 Train loss: 1.374779 Train acc: 0.518333\n",
      "Epoch: 23/1000 Iteration: 215 Train loss: 1.361756 Train acc: 0.535000\n",
      "Epoch: 24/1000 Iteration: 220 Train loss: 1.337710 Train acc: 0.528333\n",
      "Epoch: 24/1000 Iteration: 225 Train loss: 1.352646 Train acc: 0.543333\n",
      "Epoch: 24/1000 Iteration: 225 Validation loss: 1.266084 Validation acc: 0.666111\n",
      "Epoch: 25/1000 Iteration: 230 Train loss: 1.293312 Train acc: 0.541667\n",
      "Epoch: 26/1000 Iteration: 235 Train loss: 1.295915 Train acc: 0.526667\n",
      "Epoch: 26/1000 Iteration: 240 Train loss: 1.256767 Train acc: 0.550000\n",
      "Epoch: 27/1000 Iteration: 245 Train loss: 1.248808 Train acc: 0.598333\n",
      "Epoch: 27/1000 Iteration: 250 Train loss: 1.258486 Train acc: 0.581667\n",
      "Epoch: 27/1000 Iteration: 250 Validation loss: 1.193087 Validation acc: 0.706111\n",
      "Epoch: 28/1000 Iteration: 255 Train loss: 1.252826 Train acc: 0.590000\n",
      "Epoch: 28/1000 Iteration: 260 Train loss: 1.228057 Train acc: 0.618333\n",
      "Epoch: 29/1000 Iteration: 265 Train loss: 1.223364 Train acc: 0.608333\n",
      "Epoch: 29/1000 Iteration: 270 Train loss: 1.213675 Train acc: 0.606667\n",
      "Epoch: 30/1000 Iteration: 275 Train loss: 1.164593 Train acc: 0.606667\n",
      "Epoch: 30/1000 Iteration: 275 Validation loss: 1.111755 Validation acc: 0.740556\n",
      "Epoch: 31/1000 Iteration: 280 Train loss: 1.155850 Train acc: 0.630000\n",
      "Epoch: 31/1000 Iteration: 285 Train loss: 1.109280 Train acc: 0.658333\n",
      "Epoch: 32/1000 Iteration: 290 Train loss: 1.121311 Train acc: 0.636667\n",
      "Epoch: 32/1000 Iteration: 295 Train loss: 1.096382 Train acc: 0.665000\n",
      "Epoch: 33/1000 Iteration: 300 Train loss: 1.134897 Train acc: 0.608333\n",
      "Epoch: 33/1000 Iteration: 300 Validation loss: 1.021628 Validation acc: 0.770000\n",
      "Epoch: 33/1000 Iteration: 305 Train loss: 1.080634 Train acc: 0.681667\n",
      "Epoch: 34/1000 Iteration: 310 Train loss: 1.067508 Train acc: 0.660000\n",
      "Epoch: 34/1000 Iteration: 315 Train loss: 1.074250 Train acc: 0.650000\n",
      "Epoch: 35/1000 Iteration: 320 Train loss: 1.023030 Train acc: 0.683333\n",
      "Epoch: 36/1000 Iteration: 325 Train loss: 1.015239 Train acc: 0.703333\n",
      "Epoch: 36/1000 Iteration: 325 Validation loss: 0.926729 Validation acc: 0.783889\n",
      "Epoch: 36/1000 Iteration: 330 Train loss: 0.970559 Train acc: 0.715000\n",
      "Epoch: 37/1000 Iteration: 335 Train loss: 0.951481 Train acc: 0.716667\n",
      "Epoch: 37/1000 Iteration: 340 Train loss: 0.951436 Train acc: 0.728333\n",
      "Epoch: 38/1000 Iteration: 345 Train loss: 0.997367 Train acc: 0.675000\n",
      "Epoch: 38/1000 Iteration: 350 Train loss: 0.931992 Train acc: 0.756667\n",
      "Epoch: 38/1000 Iteration: 350 Validation loss: 0.842397 Validation acc: 0.798333\n",
      "Epoch: 39/1000 Iteration: 355 Train loss: 0.952045 Train acc: 0.716667\n",
      "Epoch: 39/1000 Iteration: 360 Train loss: 0.925879 Train acc: 0.735000\n",
      "Epoch: 40/1000 Iteration: 365 Train loss: 0.891628 Train acc: 0.733333\n",
      "Epoch: 41/1000 Iteration: 370 Train loss: 0.878607 Train acc: 0.768333\n",
      "Epoch: 41/1000 Iteration: 375 Train loss: 0.856503 Train acc: 0.753333\n",
      "Epoch: 41/1000 Iteration: 375 Validation loss: 0.765768 Validation acc: 0.811111\n",
      "Epoch: 42/1000 Iteration: 380 Train loss: 0.848646 Train acc: 0.755000\n",
      "Epoch: 42/1000 Iteration: 385 Train loss: 0.845570 Train acc: 0.745000\n",
      "Epoch: 43/1000 Iteration: 390 Train loss: 0.900317 Train acc: 0.715000\n",
      "Epoch: 43/1000 Iteration: 395 Train loss: 0.803566 Train acc: 0.791667\n",
      "Epoch: 44/1000 Iteration: 400 Train loss: 0.830378 Train acc: 0.741667\n",
      "Epoch: 44/1000 Iteration: 400 Validation loss: 0.703516 Validation acc: 0.822222\n",
      "Epoch: 44/1000 Iteration: 405 Train loss: 0.830952 Train acc: 0.748333\n",
      "Epoch: 45/1000 Iteration: 410 Train loss: 0.770233 Train acc: 0.783333\n",
      "Epoch: 46/1000 Iteration: 415 Train loss: 0.756428 Train acc: 0.793333\n",
      "Epoch: 46/1000 Iteration: 420 Train loss: 0.763450 Train acc: 0.765000\n",
      "Epoch: 47/1000 Iteration: 425 Train loss: 0.777654 Train acc: 0.773333\n",
      "Epoch: 47/1000 Iteration: 425 Validation loss: 0.652679 Validation acc: 0.835556\n",
      "Epoch: 47/1000 Iteration: 430 Train loss: 0.775647 Train acc: 0.766667\n",
      "Epoch: 48/1000 Iteration: 435 Train loss: 0.773107 Train acc: 0.778333\n",
      "Epoch: 48/1000 Iteration: 440 Train loss: 0.721872 Train acc: 0.805000\n",
      "Epoch: 49/1000 Iteration: 445 Train loss: 0.765114 Train acc: 0.768333\n",
      "Epoch: 49/1000 Iteration: 450 Train loss: 0.743614 Train acc: 0.776667\n",
      "Epoch: 49/1000 Iteration: 450 Validation loss: 0.609974 Validation acc: 0.851667\n",
      "Epoch: 50/1000 Iteration: 455 Train loss: 0.692468 Train acc: 0.820000\n",
      "Epoch: 51/1000 Iteration: 460 Train loss: 0.695136 Train acc: 0.795000\n",
      "Epoch: 51/1000 Iteration: 465 Train loss: 0.696379 Train acc: 0.803333\n",
      "Epoch: 52/1000 Iteration: 470 Train loss: 0.684752 Train acc: 0.795000\n",
      "Epoch: 52/1000 Iteration: 475 Train loss: 0.719835 Train acc: 0.801667\n",
      "Epoch: 52/1000 Iteration: 475 Validation loss: 0.575105 Validation acc: 0.868333\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 53/1000 Iteration: 480 Train loss: 0.740274 Train acc: 0.751667\n",
      "Epoch: 53/1000 Iteration: 485 Train loss: 0.643521 Train acc: 0.826667\n",
      "Epoch: 54/1000 Iteration: 490 Train loss: 0.685337 Train acc: 0.795000\n",
      "Epoch: 54/1000 Iteration: 495 Train loss: 0.690596 Train acc: 0.793333\n",
      "Epoch: 55/1000 Iteration: 500 Train loss: 0.631418 Train acc: 0.821667\n",
      "Epoch: 55/1000 Iteration: 500 Validation loss: 0.545098 Validation acc: 0.876667\n",
      "Epoch: 56/1000 Iteration: 505 Train loss: 0.645311 Train acc: 0.816667\n",
      "Epoch: 56/1000 Iteration: 510 Train loss: 0.621686 Train acc: 0.836667\n",
      "Epoch: 57/1000 Iteration: 515 Train loss: 0.649746 Train acc: 0.803333\n",
      "Epoch: 57/1000 Iteration: 520 Train loss: 0.629695 Train acc: 0.820000\n",
      "Epoch: 58/1000 Iteration: 525 Train loss: 0.688416 Train acc: 0.776667\n",
      "Epoch: 58/1000 Iteration: 525 Validation loss: 0.510790 Validation acc: 0.888333\n",
      "Epoch: 58/1000 Iteration: 530 Train loss: 0.617387 Train acc: 0.831667\n",
      "Epoch: 59/1000 Iteration: 535 Train loss: 0.659254 Train acc: 0.801667\n",
      "Epoch: 59/1000 Iteration: 540 Train loss: 0.609662 Train acc: 0.823333\n",
      "Epoch: 60/1000 Iteration: 545 Train loss: 0.588621 Train acc: 0.828333\n",
      "Epoch: 61/1000 Iteration: 550 Train loss: 0.611814 Train acc: 0.818333\n",
      "Epoch: 61/1000 Iteration: 550 Validation loss: 0.486540 Validation acc: 0.897222\n",
      "Epoch: 61/1000 Iteration: 555 Train loss: 0.589234 Train acc: 0.833333\n",
      "Epoch: 62/1000 Iteration: 560 Train loss: 0.558520 Train acc: 0.841667\n",
      "Epoch: 62/1000 Iteration: 565 Train loss: 0.584348 Train acc: 0.818333\n",
      "Epoch: 63/1000 Iteration: 570 Train loss: 0.622947 Train acc: 0.820000\n",
      "Epoch: 63/1000 Iteration: 575 Train loss: 0.549234 Train acc: 0.850000\n",
      "Epoch: 63/1000 Iteration: 575 Validation loss: 0.455045 Validation acc: 0.910000\n",
      "Epoch: 64/1000 Iteration: 580 Train loss: 0.579277 Train acc: 0.830000\n",
      "Epoch: 64/1000 Iteration: 585 Train loss: 0.573633 Train acc: 0.845000\n",
      "Epoch: 65/1000 Iteration: 590 Train loss: 0.554658 Train acc: 0.838333\n",
      "Epoch: 66/1000 Iteration: 595 Train loss: 0.511522 Train acc: 0.870000\n",
      "Epoch: 66/1000 Iteration: 600 Train loss: 0.530897 Train acc: 0.836667\n",
      "Epoch: 66/1000 Iteration: 600 Validation loss: 0.426109 Validation acc: 0.919445\n",
      "Epoch: 67/1000 Iteration: 605 Train loss: 0.524594 Train acc: 0.845000\n",
      "Epoch: 67/1000 Iteration: 610 Train loss: 0.543784 Train acc: 0.841667\n",
      "Epoch: 68/1000 Iteration: 615 Train loss: 0.561724 Train acc: 0.840000\n",
      "Epoch: 68/1000 Iteration: 620 Train loss: 0.503730 Train acc: 0.885000\n",
      "Epoch: 69/1000 Iteration: 625 Train loss: 0.547411 Train acc: 0.856667\n",
      "Epoch: 69/1000 Iteration: 625 Validation loss: 0.397802 Validation acc: 0.920000\n",
      "Epoch: 69/1000 Iteration: 630 Train loss: 0.536461 Train acc: 0.861667\n",
      "Epoch: 70/1000 Iteration: 635 Train loss: 0.544333 Train acc: 0.843333\n",
      "Epoch: 71/1000 Iteration: 640 Train loss: 0.511225 Train acc: 0.860000\n",
      "Epoch: 71/1000 Iteration: 645 Train loss: 0.504796 Train acc: 0.866667\n",
      "Epoch: 72/1000 Iteration: 650 Train loss: 0.477556 Train acc: 0.860000\n",
      "Epoch: 72/1000 Iteration: 650 Validation loss: 0.372052 Validation acc: 0.925556\n",
      "Epoch: 72/1000 Iteration: 655 Train loss: 0.511101 Train acc: 0.861667\n",
      "Epoch: 73/1000 Iteration: 660 Train loss: 0.517316 Train acc: 0.856667\n",
      "Epoch: 73/1000 Iteration: 665 Train loss: 0.456759 Train acc: 0.906667\n",
      "Epoch: 74/1000 Iteration: 670 Train loss: 0.495134 Train acc: 0.860000\n",
      "Epoch: 74/1000 Iteration: 675 Train loss: 0.490987 Train acc: 0.880000\n",
      "Epoch: 74/1000 Iteration: 675 Validation loss: 0.349590 Validation acc: 0.930556\n",
      "Epoch: 75/1000 Iteration: 680 Train loss: 0.476344 Train acc: 0.873333\n",
      "Epoch: 76/1000 Iteration: 685 Train loss: 0.443014 Train acc: 0.900000\n",
      "Epoch: 76/1000 Iteration: 690 Train loss: 0.443913 Train acc: 0.885000\n",
      "Epoch: 77/1000 Iteration: 695 Train loss: 0.441570 Train acc: 0.901667\n",
      "Epoch: 77/1000 Iteration: 700 Train loss: 0.465672 Train acc: 0.886667\n",
      "Epoch: 77/1000 Iteration: 700 Validation loss: 0.332838 Validation acc: 0.931667\n",
      "Epoch: 78/1000 Iteration: 705 Train loss: 0.478176 Train acc: 0.865000\n",
      "Epoch: 78/1000 Iteration: 710 Train loss: 0.425903 Train acc: 0.918333\n",
      "Epoch: 79/1000 Iteration: 715 Train loss: 0.472539 Train acc: 0.856667\n",
      "Epoch: 79/1000 Iteration: 720 Train loss: 0.453139 Train acc: 0.898333\n",
      "Epoch: 80/1000 Iteration: 725 Train loss: 0.444365 Train acc: 0.896667\n",
      "Epoch: 80/1000 Iteration: 725 Validation loss: 0.321652 Validation acc: 0.931667\n",
      "Epoch: 81/1000 Iteration: 730 Train loss: 0.408843 Train acc: 0.916667\n",
      "Epoch: 81/1000 Iteration: 735 Train loss: 0.403988 Train acc: 0.900000\n",
      "Epoch: 82/1000 Iteration: 740 Train loss: 0.402525 Train acc: 0.908333\n",
      "Epoch: 82/1000 Iteration: 745 Train loss: 0.419001 Train acc: 0.911667\n",
      "Epoch: 83/1000 Iteration: 750 Train loss: 0.453172 Train acc: 0.890000\n",
      "Epoch: 83/1000 Iteration: 750 Validation loss: 0.301978 Validation acc: 0.935000\n",
      "Epoch: 83/1000 Iteration: 755 Train loss: 0.377438 Train acc: 0.928333\n",
      "Epoch: 84/1000 Iteration: 760 Train loss: 0.434001 Train acc: 0.891667\n",
      "Epoch: 84/1000 Iteration: 765 Train loss: 0.412722 Train acc: 0.900000\n",
      "Epoch: 85/1000 Iteration: 770 Train loss: 0.401966 Train acc: 0.905000\n",
      "Epoch: 86/1000 Iteration: 775 Train loss: 0.381582 Train acc: 0.920000\n",
      "Epoch: 86/1000 Iteration: 775 Validation loss: 0.288737 Validation acc: 0.936667\n",
      "Epoch: 86/1000 Iteration: 780 Train loss: 0.390268 Train acc: 0.920000\n",
      "Epoch: 87/1000 Iteration: 785 Train loss: 0.416144 Train acc: 0.905000\n",
      "Epoch: 87/1000 Iteration: 790 Train loss: 0.391961 Train acc: 0.920000\n",
      "Epoch: 88/1000 Iteration: 795 Train loss: 0.425695 Train acc: 0.903333\n",
      "Epoch: 88/1000 Iteration: 800 Train loss: 0.368180 Train acc: 0.923333\n",
      "Epoch: 88/1000 Iteration: 800 Validation loss: 0.280233 Validation acc: 0.937778\n",
      "Epoch: 89/1000 Iteration: 805 Train loss: 0.410215 Train acc: 0.896667\n",
      "Epoch: 89/1000 Iteration: 810 Train loss: 0.383799 Train acc: 0.923333\n",
      "Epoch: 90/1000 Iteration: 815 Train loss: 0.370177 Train acc: 0.906667\n",
      "Epoch: 91/1000 Iteration: 820 Train loss: 0.385790 Train acc: 0.916667\n",
      "Epoch: 91/1000 Iteration: 825 Train loss: 0.362791 Train acc: 0.926667\n",
      "Epoch: 91/1000 Iteration: 825 Validation loss: 0.269283 Validation acc: 0.938333\n",
      "Epoch: 92/1000 Iteration: 830 Train loss: 0.362254 Train acc: 0.930000\n",
      "Epoch: 92/1000 Iteration: 835 Train loss: 0.381875 Train acc: 0.913333\n",
      "Epoch: 93/1000 Iteration: 840 Train loss: 0.401292 Train acc: 0.896667\n",
      "Epoch: 93/1000 Iteration: 845 Train loss: 0.358330 Train acc: 0.916667\n",
      "Epoch: 94/1000 Iteration: 850 Train loss: 0.404827 Train acc: 0.891667\n",
      "Epoch: 94/1000 Iteration: 850 Validation loss: 0.253903 Validation acc: 0.939444\n",
      "Epoch: 94/1000 Iteration: 855 Train loss: 0.372470 Train acc: 0.911667\n",
      "Epoch: 95/1000 Iteration: 860 Train loss: 0.365299 Train acc: 0.905000\n",
      "Epoch: 96/1000 Iteration: 865 Train loss: 0.365397 Train acc: 0.918333\n",
      "Epoch: 96/1000 Iteration: 870 Train loss: 0.334916 Train acc: 0.930000\n",
      "Epoch: 97/1000 Iteration: 875 Train loss: 0.343055 Train acc: 0.930000\n",
      "Epoch: 97/1000 Iteration: 875 Validation loss: 0.247374 Validation acc: 0.940556\n",
      "Epoch: 97/1000 Iteration: 880 Train loss: 0.367514 Train acc: 0.906667\n",
      "Epoch: 98/1000 Iteration: 885 Train loss: 0.379007 Train acc: 0.901667\n",
      "Epoch: 98/1000 Iteration: 890 Train loss: 0.361927 Train acc: 0.926667\n",
      "Epoch: 99/1000 Iteration: 895 Train loss: 0.382880 Train acc: 0.898333\n",
      "Epoch: 99/1000 Iteration: 900 Train loss: 0.347149 Train acc: 0.930000\n",
      "Epoch: 99/1000 Iteration: 900 Validation loss: 0.245649 Validation acc: 0.940000\n",
      "Epoch: 100/1000 Iteration: 905 Train loss: 0.338089 Train acc: 0.918333\n",
      "Epoch: 101/1000 Iteration: 910 Train loss: 0.356099 Train acc: 0.920000\n",
      "Epoch: 101/1000 Iteration: 915 Train loss: 0.314747 Train acc: 0.931667\n",
      "Epoch: 102/1000 Iteration: 920 Train loss: 0.330036 Train acc: 0.936667\n",
      "Epoch: 102/1000 Iteration: 925 Train loss: 0.348187 Train acc: 0.925000\n",
      "Epoch: 102/1000 Iteration: 925 Validation loss: 0.231338 Validation acc: 0.943333\n",
      "Epoch: 103/1000 Iteration: 930 Train loss: 0.333212 Train acc: 0.915000\n",
      "Epoch: 103/1000 Iteration: 935 Train loss: 0.327244 Train acc: 0.938333\n",
      "Epoch: 104/1000 Iteration: 940 Train loss: 0.363566 Train acc: 0.903333\n",
      "Epoch: 104/1000 Iteration: 945 Train loss: 0.325021 Train acc: 0.933333\n",
      "Epoch: 105/1000 Iteration: 950 Train loss: 0.316346 Train acc: 0.925000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 105/1000 Iteration: 950 Validation loss: 0.224644 Validation acc: 0.946111\n",
      "Epoch: 106/1000 Iteration: 955 Train loss: 0.323835 Train acc: 0.916667\n",
      "Epoch: 106/1000 Iteration: 960 Train loss: 0.296360 Train acc: 0.941667\n",
      "Epoch: 107/1000 Iteration: 965 Train loss: 0.317126 Train acc: 0.941667\n",
      "Epoch: 107/1000 Iteration: 970 Train loss: 0.315627 Train acc: 0.936667\n",
      "Epoch: 108/1000 Iteration: 975 Train loss: 0.343680 Train acc: 0.908333\n",
      "Epoch: 108/1000 Iteration: 975 Validation loss: 0.219351 Validation acc: 0.945000\n",
      "Epoch: 108/1000 Iteration: 980 Train loss: 0.293513 Train acc: 0.946667\n",
      "Epoch: 109/1000 Iteration: 985 Train loss: 0.332561 Train acc: 0.925000\n",
      "Epoch: 109/1000 Iteration: 990 Train loss: 0.315858 Train acc: 0.933333\n",
      "Epoch: 110/1000 Iteration: 995 Train loss: 0.324163 Train acc: 0.920000\n",
      "Epoch: 111/1000 Iteration: 1000 Train loss: 0.301981 Train acc: 0.931667\n",
      "Epoch: 111/1000 Iteration: 1000 Validation loss: 0.211062 Validation acc: 0.945556\n",
      "Epoch: 111/1000 Iteration: 1005 Train loss: 0.296191 Train acc: 0.928333\n",
      "Epoch: 112/1000 Iteration: 1010 Train loss: 0.310750 Train acc: 0.931667\n",
      "Epoch: 112/1000 Iteration: 1015 Train loss: 0.293864 Train acc: 0.936667\n",
      "Epoch: 113/1000 Iteration: 1020 Train loss: 0.331917 Train acc: 0.918333\n",
      "Epoch: 113/1000 Iteration: 1025 Train loss: 0.298214 Train acc: 0.935000\n",
      "Epoch: 113/1000 Iteration: 1025 Validation loss: 0.204572 Validation acc: 0.946667\n",
      "Epoch: 114/1000 Iteration: 1030 Train loss: 0.334511 Train acc: 0.911667\n",
      "Epoch: 114/1000 Iteration: 1035 Train loss: 0.328953 Train acc: 0.931667\n",
      "Epoch: 115/1000 Iteration: 1040 Train loss: 0.294689 Train acc: 0.928333\n",
      "Epoch: 116/1000 Iteration: 1045 Train loss: 0.294781 Train acc: 0.936667\n",
      "Epoch: 116/1000 Iteration: 1050 Train loss: 0.270885 Train acc: 0.940000\n",
      "Epoch: 116/1000 Iteration: 1050 Validation loss: 0.200226 Validation acc: 0.949444\n",
      "Epoch: 117/1000 Iteration: 1055 Train loss: 0.277045 Train acc: 0.948333\n",
      "Epoch: 117/1000 Iteration: 1060 Train loss: 0.305102 Train acc: 0.921667\n",
      "Epoch: 118/1000 Iteration: 1065 Train loss: 0.338336 Train acc: 0.910000\n",
      "Epoch: 118/1000 Iteration: 1070 Train loss: 0.278539 Train acc: 0.941667\n",
      "Epoch: 119/1000 Iteration: 1075 Train loss: 0.328429 Train acc: 0.910000\n",
      "Epoch: 119/1000 Iteration: 1075 Validation loss: 0.194778 Validation acc: 0.947778\n",
      "Epoch: 119/1000 Iteration: 1080 Train loss: 0.309748 Train acc: 0.925000\n",
      "Epoch: 120/1000 Iteration: 1085 Train loss: 0.288523 Train acc: 0.930000\n",
      "Epoch: 121/1000 Iteration: 1090 Train loss: 0.263499 Train acc: 0.945000\n",
      "Epoch: 121/1000 Iteration: 1095 Train loss: 0.282433 Train acc: 0.940000\n",
      "Epoch: 122/1000 Iteration: 1100 Train loss: 0.278637 Train acc: 0.936667\n",
      "Epoch: 122/1000 Iteration: 1100 Validation loss: 0.191320 Validation acc: 0.951111\n",
      "Epoch: 122/1000 Iteration: 1105 Train loss: 0.291065 Train acc: 0.933333\n",
      "Epoch: 123/1000 Iteration: 1110 Train loss: 0.318196 Train acc: 0.925000\n",
      "Epoch: 123/1000 Iteration: 1115 Train loss: 0.289017 Train acc: 0.943333\n",
      "Epoch: 124/1000 Iteration: 1120 Train loss: 0.323827 Train acc: 0.903333\n",
      "Epoch: 124/1000 Iteration: 1125 Train loss: 0.314260 Train acc: 0.913333\n",
      "Epoch: 124/1000 Iteration: 1125 Validation loss: 0.189092 Validation acc: 0.949444\n",
      "Epoch: 125/1000 Iteration: 1130 Train loss: 0.289045 Train acc: 0.941667\n",
      "Epoch: 126/1000 Iteration: 1135 Train loss: 0.279324 Train acc: 0.933333\n",
      "Epoch: 126/1000 Iteration: 1140 Train loss: 0.261333 Train acc: 0.943333\n",
      "Epoch: 127/1000 Iteration: 1145 Train loss: 0.272119 Train acc: 0.930000\n",
      "Epoch: 127/1000 Iteration: 1150 Train loss: 0.286600 Train acc: 0.935000\n",
      "Epoch: 127/1000 Iteration: 1150 Validation loss: 0.189215 Validation acc: 0.946667\n",
      "Epoch: 128/1000 Iteration: 1155 Train loss: 0.301794 Train acc: 0.923333\n",
      "Epoch: 128/1000 Iteration: 1160 Train loss: 0.261402 Train acc: 0.946667\n",
      "Epoch: 129/1000 Iteration: 1165 Train loss: 0.307782 Train acc: 0.906667\n",
      "Epoch: 129/1000 Iteration: 1170 Train loss: 0.287183 Train acc: 0.930000\n",
      "Epoch: 130/1000 Iteration: 1175 Train loss: 0.278953 Train acc: 0.926667\n",
      "Epoch: 130/1000 Iteration: 1175 Validation loss: 0.183725 Validation acc: 0.947222\n",
      "Epoch: 131/1000 Iteration: 1180 Train loss: 0.274392 Train acc: 0.938333\n",
      "Epoch: 131/1000 Iteration: 1185 Train loss: 0.257889 Train acc: 0.930000\n",
      "Epoch: 132/1000 Iteration: 1190 Train loss: 0.271371 Train acc: 0.930000\n",
      "Epoch: 132/1000 Iteration: 1195 Train loss: 0.261097 Train acc: 0.941667\n",
      "Epoch: 133/1000 Iteration: 1200 Train loss: 0.296482 Train acc: 0.918333\n",
      "Epoch: 133/1000 Iteration: 1200 Validation loss: 0.181769 Validation acc: 0.950556\n",
      "Epoch: 133/1000 Iteration: 1205 Train loss: 0.258014 Train acc: 0.958333\n",
      "Epoch: 134/1000 Iteration: 1210 Train loss: 0.323696 Train acc: 0.915000\n",
      "Epoch: 134/1000 Iteration: 1215 Train loss: 0.275041 Train acc: 0.941667\n",
      "Epoch: 135/1000 Iteration: 1220 Train loss: 0.272161 Train acc: 0.931667\n",
      "Epoch: 136/1000 Iteration: 1225 Train loss: 0.240734 Train acc: 0.938333\n",
      "Epoch: 136/1000 Iteration: 1225 Validation loss: 0.182904 Validation acc: 0.948889\n",
      "Epoch: 136/1000 Iteration: 1230 Train loss: 0.244923 Train acc: 0.946667\n",
      "Epoch: 137/1000 Iteration: 1235 Train loss: 0.255857 Train acc: 0.943333\n",
      "Epoch: 137/1000 Iteration: 1240 Train loss: 0.261823 Train acc: 0.936667\n",
      "Epoch: 138/1000 Iteration: 1245 Train loss: 0.296736 Train acc: 0.918333\n",
      "Epoch: 138/1000 Iteration: 1250 Train loss: 0.272964 Train acc: 0.933333\n",
      "Epoch: 138/1000 Iteration: 1250 Validation loss: 0.177054 Validation acc: 0.949444\n",
      "Epoch: 139/1000 Iteration: 1255 Train loss: 0.283968 Train acc: 0.928333\n",
      "Epoch: 139/1000 Iteration: 1260 Train loss: 0.281248 Train acc: 0.935000\n",
      "Epoch: 140/1000 Iteration: 1265 Train loss: 0.271431 Train acc: 0.941667\n",
      "Epoch: 141/1000 Iteration: 1270 Train loss: 0.251907 Train acc: 0.936667\n",
      "Epoch: 141/1000 Iteration: 1275 Train loss: 0.234001 Train acc: 0.943333\n",
      "Epoch: 141/1000 Iteration: 1275 Validation loss: 0.174632 Validation acc: 0.949444\n",
      "Epoch: 142/1000 Iteration: 1280 Train loss: 0.250404 Train acc: 0.935000\n",
      "Epoch: 142/1000 Iteration: 1285 Train loss: 0.242402 Train acc: 0.946667\n",
      "Epoch: 143/1000 Iteration: 1290 Train loss: 0.311101 Train acc: 0.916667\n",
      "Epoch: 143/1000 Iteration: 1295 Train loss: 0.262108 Train acc: 0.938333\n",
      "Epoch: 144/1000 Iteration: 1300 Train loss: 0.282779 Train acc: 0.931667\n",
      "Epoch: 144/1000 Iteration: 1300 Validation loss: 0.171613 Validation acc: 0.950556\n",
      "Epoch: 144/1000 Iteration: 1305 Train loss: 0.263813 Train acc: 0.941667\n",
      "Epoch: 145/1000 Iteration: 1310 Train loss: 0.252302 Train acc: 0.931667\n",
      "Epoch: 146/1000 Iteration: 1315 Train loss: 0.264385 Train acc: 0.935000\n",
      "Epoch: 146/1000 Iteration: 1320 Train loss: 0.227093 Train acc: 0.941667\n",
      "Epoch: 147/1000 Iteration: 1325 Train loss: 0.245107 Train acc: 0.941667\n",
      "Epoch: 147/1000 Iteration: 1325 Validation loss: 0.167993 Validation acc: 0.949444\n",
      "Epoch: 147/1000 Iteration: 1330 Train loss: 0.252022 Train acc: 0.938333\n",
      "Epoch: 148/1000 Iteration: 1335 Train loss: 0.272530 Train acc: 0.926667\n",
      "Epoch: 148/1000 Iteration: 1340 Train loss: 0.242906 Train acc: 0.950000\n",
      "Epoch: 149/1000 Iteration: 1345 Train loss: 0.286042 Train acc: 0.920000\n",
      "Epoch: 149/1000 Iteration: 1350 Train loss: 0.278301 Train acc: 0.931667\n",
      "Epoch: 149/1000 Iteration: 1350 Validation loss: 0.168481 Validation acc: 0.951111\n",
      "Epoch: 150/1000 Iteration: 1355 Train loss: 0.232799 Train acc: 0.948333\n",
      "Epoch: 151/1000 Iteration: 1360 Train loss: 0.242568 Train acc: 0.940000\n",
      "Epoch: 151/1000 Iteration: 1365 Train loss: 0.230404 Train acc: 0.940000\n",
      "Epoch: 152/1000 Iteration: 1370 Train loss: 0.251097 Train acc: 0.945000\n",
      "Epoch: 152/1000 Iteration: 1375 Train loss: 0.243426 Train acc: 0.943333\n",
      "Epoch: 152/1000 Iteration: 1375 Validation loss: 0.165270 Validation acc: 0.950556\n",
      "Epoch: 153/1000 Iteration: 1380 Train loss: 0.289252 Train acc: 0.921667\n",
      "Epoch: 153/1000 Iteration: 1385 Train loss: 0.246769 Train acc: 0.950000\n",
      "Epoch: 154/1000 Iteration: 1390 Train loss: 0.285357 Train acc: 0.918333\n",
      "Epoch: 154/1000 Iteration: 1395 Train loss: 0.256565 Train acc: 0.931667\n",
      "Epoch: 155/1000 Iteration: 1400 Train loss: 0.242397 Train acc: 0.931667\n",
      "Epoch: 155/1000 Iteration: 1400 Validation loss: 0.164408 Validation acc: 0.949444\n",
      "Epoch: 156/1000 Iteration: 1405 Train loss: 0.225924 Train acc: 0.948333\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 156/1000 Iteration: 1410 Train loss: 0.238378 Train acc: 0.950000\n",
      "Epoch: 157/1000 Iteration: 1415 Train loss: 0.243154 Train acc: 0.943333\n",
      "Epoch: 157/1000 Iteration: 1420 Train loss: 0.242585 Train acc: 0.940000\n",
      "Epoch: 158/1000 Iteration: 1425 Train loss: 0.276927 Train acc: 0.921667\n",
      "Epoch: 158/1000 Iteration: 1425 Validation loss: 0.162822 Validation acc: 0.950556\n",
      "Epoch: 158/1000 Iteration: 1430 Train loss: 0.246895 Train acc: 0.955000\n",
      "Epoch: 159/1000 Iteration: 1435 Train loss: 0.275149 Train acc: 0.921667\n",
      "Epoch: 159/1000 Iteration: 1440 Train loss: 0.250822 Train acc: 0.935000\n",
      "Epoch: 160/1000 Iteration: 1445 Train loss: 0.245951 Train acc: 0.935000\n",
      "Epoch: 161/1000 Iteration: 1450 Train loss: 0.235161 Train acc: 0.951667\n",
      "Epoch: 161/1000 Iteration: 1450 Validation loss: 0.158865 Validation acc: 0.950556\n",
      "Epoch: 161/1000 Iteration: 1455 Train loss: 0.212606 Train acc: 0.953333\n",
      "Epoch: 162/1000 Iteration: 1460 Train loss: 0.235797 Train acc: 0.961667\n",
      "Epoch: 162/1000 Iteration: 1465 Train loss: 0.240918 Train acc: 0.933333\n",
      "Epoch: 163/1000 Iteration: 1470 Train loss: 0.278542 Train acc: 0.921667\n",
      "Epoch: 163/1000 Iteration: 1475 Train loss: 0.229696 Train acc: 0.951667\n",
      "Epoch: 163/1000 Iteration: 1475 Validation loss: 0.158595 Validation acc: 0.951667\n",
      "Epoch: 164/1000 Iteration: 1480 Train loss: 0.286186 Train acc: 0.908333\n",
      "Epoch: 164/1000 Iteration: 1485 Train loss: 0.265661 Train acc: 0.941667\n",
      "Epoch: 165/1000 Iteration: 1490 Train loss: 0.237526 Train acc: 0.930000\n",
      "Epoch: 166/1000 Iteration: 1495 Train loss: 0.222780 Train acc: 0.945000\n",
      "Epoch: 166/1000 Iteration: 1500 Train loss: 0.218400 Train acc: 0.950000\n",
      "Epoch: 166/1000 Iteration: 1500 Validation loss: 0.156981 Validation acc: 0.953333\n",
      "Epoch: 167/1000 Iteration: 1505 Train loss: 0.214563 Train acc: 0.946667\n",
      "Epoch: 167/1000 Iteration: 1510 Train loss: 0.238555 Train acc: 0.941667\n",
      "Epoch: 168/1000 Iteration: 1515 Train loss: 0.268172 Train acc: 0.925000\n",
      "Epoch: 168/1000 Iteration: 1520 Train loss: 0.229324 Train acc: 0.936667\n",
      "Epoch: 169/1000 Iteration: 1525 Train loss: 0.273842 Train acc: 0.926667\n",
      "Epoch: 169/1000 Iteration: 1525 Validation loss: 0.154535 Validation acc: 0.953333\n",
      "Epoch: 169/1000 Iteration: 1530 Train loss: 0.241191 Train acc: 0.928333\n",
      "Epoch: 170/1000 Iteration: 1535 Train loss: 0.232512 Train acc: 0.940000\n",
      "Epoch: 171/1000 Iteration: 1540 Train loss: 0.206488 Train acc: 0.956667\n",
      "Epoch: 171/1000 Iteration: 1545 Train loss: 0.218518 Train acc: 0.950000\n",
      "Epoch: 172/1000 Iteration: 1550 Train loss: 0.219468 Train acc: 0.946667\n",
      "Epoch: 172/1000 Iteration: 1550 Validation loss: 0.149810 Validation acc: 0.952778\n",
      "Epoch: 172/1000 Iteration: 1555 Train loss: 0.221369 Train acc: 0.950000\n",
      "Epoch: 173/1000 Iteration: 1560 Train loss: 0.287471 Train acc: 0.918333\n",
      "Epoch: 173/1000 Iteration: 1565 Train loss: 0.205667 Train acc: 0.958333\n",
      "Epoch: 174/1000 Iteration: 1570 Train loss: 0.250904 Train acc: 0.928333\n",
      "Epoch: 174/1000 Iteration: 1575 Train loss: 0.264100 Train acc: 0.933333\n",
      "Epoch: 174/1000 Iteration: 1575 Validation loss: 0.146598 Validation acc: 0.952222\n",
      "Epoch: 175/1000 Iteration: 1580 Train loss: 0.206245 Train acc: 0.945000\n",
      "Epoch: 176/1000 Iteration: 1585 Train loss: 0.217803 Train acc: 0.953333\n",
      "Epoch: 176/1000 Iteration: 1590 Train loss: 0.203903 Train acc: 0.950000\n",
      "Epoch: 177/1000 Iteration: 1595 Train loss: 0.232004 Train acc: 0.955000\n",
      "Epoch: 177/1000 Iteration: 1600 Train loss: 0.240806 Train acc: 0.940000\n",
      "Epoch: 177/1000 Iteration: 1600 Validation loss: 0.146346 Validation acc: 0.953333\n",
      "Epoch: 178/1000 Iteration: 1605 Train loss: 0.267496 Train acc: 0.928333\n",
      "Epoch: 178/1000 Iteration: 1610 Train loss: 0.227102 Train acc: 0.946667\n",
      "Epoch: 179/1000 Iteration: 1615 Train loss: 0.255806 Train acc: 0.923333\n",
      "Epoch: 179/1000 Iteration: 1620 Train loss: 0.251288 Train acc: 0.938333\n",
      "Epoch: 180/1000 Iteration: 1625 Train loss: 0.243382 Train acc: 0.945000\n",
      "Epoch: 180/1000 Iteration: 1625 Validation loss: 0.144271 Validation acc: 0.953333\n",
      "Epoch: 181/1000 Iteration: 1630 Train loss: 0.206195 Train acc: 0.951667\n",
      "Epoch: 181/1000 Iteration: 1635 Train loss: 0.196975 Train acc: 0.948333\n",
      "Epoch: 182/1000 Iteration: 1640 Train loss: 0.206133 Train acc: 0.953333\n",
      "Epoch: 182/1000 Iteration: 1645 Train loss: 0.215223 Train acc: 0.948333\n",
      "Epoch: 183/1000 Iteration: 1650 Train loss: 0.248639 Train acc: 0.928333\n",
      "Epoch: 183/1000 Iteration: 1650 Validation loss: 0.146560 Validation acc: 0.952778\n",
      "Epoch: 183/1000 Iteration: 1655 Train loss: 0.202354 Train acc: 0.945000\n",
      "Epoch: 184/1000 Iteration: 1660 Train loss: 0.249433 Train acc: 0.928333\n",
      "Epoch: 184/1000 Iteration: 1665 Train loss: 0.226793 Train acc: 0.943333\n",
      "Epoch: 185/1000 Iteration: 1670 Train loss: 0.208474 Train acc: 0.943333\n",
      "Epoch: 186/1000 Iteration: 1675 Train loss: 0.192969 Train acc: 0.951667\n",
      "Epoch: 186/1000 Iteration: 1675 Validation loss: 0.142842 Validation acc: 0.952778\n",
      "Epoch: 186/1000 Iteration: 1680 Train loss: 0.218684 Train acc: 0.938333\n",
      "Epoch: 187/1000 Iteration: 1685 Train loss: 0.225031 Train acc: 0.945000\n",
      "Epoch: 187/1000 Iteration: 1690 Train loss: 0.229937 Train acc: 0.933333\n",
      "Epoch: 188/1000 Iteration: 1695 Train loss: 0.248050 Train acc: 0.940000\n",
      "Epoch: 188/1000 Iteration: 1700 Train loss: 0.228021 Train acc: 0.938333\n",
      "Epoch: 188/1000 Iteration: 1700 Validation loss: 0.146660 Validation acc: 0.952778\n",
      "Epoch: 189/1000 Iteration: 1705 Train loss: 0.245516 Train acc: 0.921667\n",
      "Epoch: 189/1000 Iteration: 1710 Train loss: 0.224051 Train acc: 0.946667\n",
      "Epoch: 190/1000 Iteration: 1715 Train loss: 0.199553 Train acc: 0.950000\n",
      "Epoch: 191/1000 Iteration: 1720 Train loss: 0.209142 Train acc: 0.941667\n",
      "Epoch: 191/1000 Iteration: 1725 Train loss: 0.193050 Train acc: 0.950000\n",
      "Epoch: 191/1000 Iteration: 1725 Validation loss: 0.140678 Validation acc: 0.953889\n",
      "Epoch: 192/1000 Iteration: 1730 Train loss: 0.213039 Train acc: 0.946667\n",
      "Epoch: 192/1000 Iteration: 1735 Train loss: 0.225926 Train acc: 0.936667\n",
      "Epoch: 193/1000 Iteration: 1740 Train loss: 0.246203 Train acc: 0.936667\n",
      "Epoch: 193/1000 Iteration: 1745 Train loss: 0.197006 Train acc: 0.951667\n",
      "Epoch: 194/1000 Iteration: 1750 Train loss: 0.235766 Train acc: 0.921667\n",
      "Epoch: 194/1000 Iteration: 1750 Validation loss: 0.141046 Validation acc: 0.953889\n",
      "Epoch: 194/1000 Iteration: 1755 Train loss: 0.224098 Train acc: 0.940000\n",
      "Epoch: 195/1000 Iteration: 1760 Train loss: 0.226402 Train acc: 0.925000\n",
      "Epoch: 196/1000 Iteration: 1765 Train loss: 0.191108 Train acc: 0.950000\n",
      "Epoch: 196/1000 Iteration: 1770 Train loss: 0.194495 Train acc: 0.956667\n",
      "Epoch: 197/1000 Iteration: 1775 Train loss: 0.193694 Train acc: 0.955000\n",
      "Epoch: 197/1000 Iteration: 1775 Validation loss: 0.141509 Validation acc: 0.953333\n",
      "Epoch: 197/1000 Iteration: 1780 Train loss: 0.215182 Train acc: 0.941667\n",
      "Epoch: 198/1000 Iteration: 1785 Train loss: 0.251144 Train acc: 0.933333\n",
      "Epoch: 198/1000 Iteration: 1790 Train loss: 0.199947 Train acc: 0.950000\n",
      "Epoch: 199/1000 Iteration: 1795 Train loss: 0.243768 Train acc: 0.923333\n",
      "Epoch: 199/1000 Iteration: 1800 Train loss: 0.230786 Train acc: 0.935000\n",
      "Epoch: 199/1000 Iteration: 1800 Validation loss: 0.144254 Validation acc: 0.952222\n",
      "Epoch: 200/1000 Iteration: 1805 Train loss: 0.207987 Train acc: 0.946667\n",
      "Epoch: 201/1000 Iteration: 1810 Train loss: 0.215103 Train acc: 0.935000\n",
      "Epoch: 201/1000 Iteration: 1815 Train loss: 0.197053 Train acc: 0.950000\n",
      "Epoch: 202/1000 Iteration: 1820 Train loss: 0.203209 Train acc: 0.941667\n",
      "Epoch: 202/1000 Iteration: 1825 Train loss: 0.213737 Train acc: 0.940000\n",
      "Epoch: 202/1000 Iteration: 1825 Validation loss: 0.153962 Validation acc: 0.950000\n",
      "Epoch: 203/1000 Iteration: 1830 Train loss: 0.240864 Train acc: 0.935000\n",
      "Epoch: 203/1000 Iteration: 1835 Train loss: 0.203768 Train acc: 0.951667\n",
      "Epoch: 204/1000 Iteration: 1840 Train loss: 0.231615 Train acc: 0.926667\n",
      "Epoch: 204/1000 Iteration: 1845 Train loss: 0.225919 Train acc: 0.941667\n",
      "Epoch: 205/1000 Iteration: 1850 Train loss: 0.219169 Train acc: 0.940000\n",
      "Epoch: 205/1000 Iteration: 1850 Validation loss: 0.139582 Validation acc: 0.953333\n",
      "Epoch: 206/1000 Iteration: 1855 Train loss: 0.192803 Train acc: 0.950000\n",
      "Epoch: 206/1000 Iteration: 1860 Train loss: 0.202486 Train acc: 0.936667\n",
      "Epoch: 207/1000 Iteration: 1865 Train loss: 0.204622 Train acc: 0.961667\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 207/1000 Iteration: 1870 Train loss: 0.197908 Train acc: 0.950000\n",
      "Epoch: 208/1000 Iteration: 1875 Train loss: 0.253983 Train acc: 0.926667\n",
      "Epoch: 208/1000 Iteration: 1875 Validation loss: 0.136577 Validation acc: 0.953333\n",
      "Epoch: 208/1000 Iteration: 1880 Train loss: 0.185973 Train acc: 0.951667\n",
      "Epoch: 209/1000 Iteration: 1885 Train loss: 0.247681 Train acc: 0.923333\n",
      "Epoch: 209/1000 Iteration: 1890 Train loss: 0.196706 Train acc: 0.940000\n",
      "Epoch: 210/1000 Iteration: 1895 Train loss: 0.204549 Train acc: 0.943333\n",
      "Epoch: 211/1000 Iteration: 1900 Train loss: 0.192013 Train acc: 0.945000\n",
      "Epoch: 211/1000 Iteration: 1900 Validation loss: 0.140184 Validation acc: 0.951667\n",
      "Epoch: 211/1000 Iteration: 1905 Train loss: 0.183279 Train acc: 0.950000\n",
      "Epoch: 212/1000 Iteration: 1910 Train loss: 0.208579 Train acc: 0.946667\n",
      "Epoch: 212/1000 Iteration: 1915 Train loss: 0.194827 Train acc: 0.946667\n",
      "Epoch: 213/1000 Iteration: 1920 Train loss: 0.222093 Train acc: 0.930000\n",
      "Epoch: 213/1000 Iteration: 1925 Train loss: 0.187638 Train acc: 0.961667\n",
      "Epoch: 213/1000 Iteration: 1925 Validation loss: 0.141482 Validation acc: 0.951667\n",
      "Epoch: 214/1000 Iteration: 1930 Train loss: 0.233646 Train acc: 0.931667\n",
      "Epoch: 214/1000 Iteration: 1935 Train loss: 0.207293 Train acc: 0.938333\n",
      "Epoch: 215/1000 Iteration: 1940 Train loss: 0.198899 Train acc: 0.951667\n",
      "Epoch: 216/1000 Iteration: 1945 Train loss: 0.193256 Train acc: 0.946667\n",
      "Epoch: 216/1000 Iteration: 1950 Train loss: 0.178028 Train acc: 0.958333\n",
      "Epoch: 216/1000 Iteration: 1950 Validation loss: 0.143690 Validation acc: 0.951667\n",
      "Epoch: 217/1000 Iteration: 1955 Train loss: 0.196738 Train acc: 0.951667\n",
      "Epoch: 217/1000 Iteration: 1960 Train loss: 0.213886 Train acc: 0.945000\n",
      "Epoch: 218/1000 Iteration: 1965 Train loss: 0.241355 Train acc: 0.931667\n",
      "Epoch: 218/1000 Iteration: 1970 Train loss: 0.183464 Train acc: 0.956667\n",
      "Epoch: 219/1000 Iteration: 1975 Train loss: 0.235131 Train acc: 0.928333\n",
      "Epoch: 219/1000 Iteration: 1975 Validation loss: 0.139408 Validation acc: 0.952222\n",
      "Epoch: 219/1000 Iteration: 1980 Train loss: 0.194092 Train acc: 0.950000\n",
      "Epoch: 220/1000 Iteration: 1985 Train loss: 0.184533 Train acc: 0.951667\n",
      "Epoch: 221/1000 Iteration: 1990 Train loss: 0.188717 Train acc: 0.945000\n",
      "Epoch: 221/1000 Iteration: 1995 Train loss: 0.187182 Train acc: 0.946667\n",
      "Epoch: 222/1000 Iteration: 2000 Train loss: 0.186486 Train acc: 0.961667\n",
      "Epoch: 222/1000 Iteration: 2000 Validation loss: 0.131787 Validation acc: 0.954444\n",
      "Epoch: 222/1000 Iteration: 2005 Train loss: 0.194418 Train acc: 0.943333\n",
      "Epoch: 223/1000 Iteration: 2010 Train loss: 0.229215 Train acc: 0.925000\n",
      "Epoch: 223/1000 Iteration: 2015 Train loss: 0.176044 Train acc: 0.963333\n",
      "Epoch: 224/1000 Iteration: 2020 Train loss: 0.231436 Train acc: 0.926667\n",
      "Epoch: 224/1000 Iteration: 2025 Train loss: 0.209791 Train acc: 0.941667\n",
      "Epoch: 224/1000 Iteration: 2025 Validation loss: 0.132230 Validation acc: 0.953333\n",
      "Epoch: 225/1000 Iteration: 2030 Train loss: 0.185956 Train acc: 0.936667\n",
      "Epoch: 226/1000 Iteration: 2035 Train loss: 0.175401 Train acc: 0.958333\n",
      "Epoch: 226/1000 Iteration: 2040 Train loss: 0.174406 Train acc: 0.948333\n",
      "Epoch: 227/1000 Iteration: 2045 Train loss: 0.180207 Train acc: 0.956667\n",
      "Epoch: 227/1000 Iteration: 2050 Train loss: 0.187349 Train acc: 0.950000\n",
      "Epoch: 227/1000 Iteration: 2050 Validation loss: 0.130194 Validation acc: 0.953889\n",
      "Epoch: 228/1000 Iteration: 2055 Train loss: 0.239674 Train acc: 0.928333\n",
      "Epoch: 228/1000 Iteration: 2060 Train loss: 0.184337 Train acc: 0.945000\n",
      "Epoch: 229/1000 Iteration: 2065 Train loss: 0.239966 Train acc: 0.925000\n",
      "Epoch: 229/1000 Iteration: 2070 Train loss: 0.200624 Train acc: 0.941667\n",
      "Epoch: 230/1000 Iteration: 2075 Train loss: 0.174286 Train acc: 0.950000\n",
      "Epoch: 230/1000 Iteration: 2075 Validation loss: 0.134025 Validation acc: 0.952222\n",
      "Epoch: 231/1000 Iteration: 2080 Train loss: 0.190472 Train acc: 0.951667\n",
      "Epoch: 231/1000 Iteration: 2085 Train loss: 0.170027 Train acc: 0.955000\n",
      "Epoch: 232/1000 Iteration: 2090 Train loss: 0.189653 Train acc: 0.961667\n",
      "Epoch: 232/1000 Iteration: 2095 Train loss: 0.208553 Train acc: 0.943333\n",
      "Epoch: 233/1000 Iteration: 2100 Train loss: 0.223338 Train acc: 0.926667\n",
      "Epoch: 233/1000 Iteration: 2100 Validation loss: 0.135164 Validation acc: 0.952778\n",
      "Epoch: 233/1000 Iteration: 2105 Train loss: 0.177020 Train acc: 0.961667\n",
      "Epoch: 234/1000 Iteration: 2110 Train loss: 0.229873 Train acc: 0.931667\n",
      "Epoch: 234/1000 Iteration: 2115 Train loss: 0.202057 Train acc: 0.946667\n",
      "Epoch: 235/1000 Iteration: 2120 Train loss: 0.187498 Train acc: 0.948333\n",
      "Epoch: 236/1000 Iteration: 2125 Train loss: 0.168282 Train acc: 0.951667\n",
      "Epoch: 236/1000 Iteration: 2125 Validation loss: 0.126606 Validation acc: 0.954444\n",
      "Epoch: 236/1000 Iteration: 2130 Train loss: 0.178117 Train acc: 0.948333\n",
      "Epoch: 237/1000 Iteration: 2135 Train loss: 0.174230 Train acc: 0.950000\n",
      "Epoch: 237/1000 Iteration: 2140 Train loss: 0.179127 Train acc: 0.946667\n",
      "Epoch: 238/1000 Iteration: 2145 Train loss: 0.234368 Train acc: 0.930000\n",
      "Epoch: 238/1000 Iteration: 2150 Train loss: 0.168842 Train acc: 0.960000\n",
      "Epoch: 238/1000 Iteration: 2150 Validation loss: 0.126356 Validation acc: 0.955555\n",
      "Epoch: 239/1000 Iteration: 2155 Train loss: 0.213577 Train acc: 0.931667\n",
      "Epoch: 239/1000 Iteration: 2160 Train loss: 0.196715 Train acc: 0.945000\n",
      "Epoch: 240/1000 Iteration: 2165 Train loss: 0.195122 Train acc: 0.946667\n",
      "Epoch: 241/1000 Iteration: 2170 Train loss: 0.165071 Train acc: 0.951667\n",
      "Epoch: 241/1000 Iteration: 2175 Train loss: 0.170163 Train acc: 0.953333\n",
      "Epoch: 241/1000 Iteration: 2175 Validation loss: 0.133496 Validation acc: 0.953333\n",
      "Epoch: 242/1000 Iteration: 2180 Train loss: 0.183648 Train acc: 0.955000\n",
      "Epoch: 242/1000 Iteration: 2185 Train loss: 0.216024 Train acc: 0.950000\n",
      "Epoch: 243/1000 Iteration: 2190 Train loss: 0.230191 Train acc: 0.930000\n",
      "Epoch: 243/1000 Iteration: 2195 Train loss: 0.151344 Train acc: 0.965000\n",
      "Epoch: 244/1000 Iteration: 2200 Train loss: 0.216941 Train acc: 0.930000\n",
      "Epoch: 244/1000 Iteration: 2200 Validation loss: 0.131449 Validation acc: 0.953889\n",
      "Epoch: 244/1000 Iteration: 2205 Train loss: 0.196665 Train acc: 0.946667\n",
      "Epoch: 245/1000 Iteration: 2210 Train loss: 0.201029 Train acc: 0.951667\n",
      "Epoch: 246/1000 Iteration: 2215 Train loss: 0.159357 Train acc: 0.950000\n",
      "Epoch: 246/1000 Iteration: 2220 Train loss: 0.175017 Train acc: 0.950000\n",
      "Epoch: 247/1000 Iteration: 2225 Train loss: 0.185183 Train acc: 0.950000\n",
      "Epoch: 247/1000 Iteration: 2225 Validation loss: 0.131545 Validation acc: 0.952778\n",
      "Epoch: 247/1000 Iteration: 2230 Train loss: 0.180484 Train acc: 0.943333\n",
      "Epoch: 248/1000 Iteration: 2235 Train loss: 0.242586 Train acc: 0.930000\n",
      "Epoch: 248/1000 Iteration: 2240 Train loss: 0.162373 Train acc: 0.955000\n",
      "Epoch: 249/1000 Iteration: 2245 Train loss: 0.211585 Train acc: 0.933333\n",
      "Epoch: 249/1000 Iteration: 2250 Train loss: 0.205234 Train acc: 0.940000\n",
      "Epoch: 249/1000 Iteration: 2250 Validation loss: 0.125074 Validation acc: 0.954444\n",
      "Epoch: 250/1000 Iteration: 2255 Train loss: 0.179952 Train acc: 0.955000\n",
      "Epoch: 251/1000 Iteration: 2260 Train loss: 0.159933 Train acc: 0.956667\n",
      "Epoch: 251/1000 Iteration: 2265 Train loss: 0.163869 Train acc: 0.948333\n",
      "Epoch: 252/1000 Iteration: 2270 Train loss: 0.191796 Train acc: 0.948333\n",
      "Epoch: 252/1000 Iteration: 2275 Train loss: 0.182284 Train acc: 0.951667\n",
      "Epoch: 252/1000 Iteration: 2275 Validation loss: 0.130750 Validation acc: 0.953333\n",
      "Epoch: 253/1000 Iteration: 2280 Train loss: 0.222258 Train acc: 0.923333\n",
      "Epoch: 253/1000 Iteration: 2285 Train loss: 0.172729 Train acc: 0.948333\n",
      "Epoch: 254/1000 Iteration: 2290 Train loss: 0.211567 Train acc: 0.921667\n",
      "Epoch: 254/1000 Iteration: 2295 Train loss: 0.183543 Train acc: 0.948333\n",
      "Epoch: 255/1000 Iteration: 2300 Train loss: 0.180742 Train acc: 0.946667\n",
      "Epoch: 255/1000 Iteration: 2300 Validation loss: 0.125115 Validation acc: 0.953889\n",
      "Epoch: 256/1000 Iteration: 2305 Train loss: 0.170196 Train acc: 0.956667\n",
      "Epoch: 256/1000 Iteration: 2310 Train loss: 0.177157 Train acc: 0.938333\n",
      "Epoch: 257/1000 Iteration: 2315 Train loss: 0.182312 Train acc: 0.950000\n",
      "Epoch: 257/1000 Iteration: 2320 Train loss: 0.166287 Train acc: 0.950000\n",
      "Epoch: 258/1000 Iteration: 2325 Train loss: 0.210731 Train acc: 0.935000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 258/1000 Iteration: 2325 Validation loss: 0.129432 Validation acc: 0.952778\n",
      "Epoch: 258/1000 Iteration: 2330 Train loss: 0.158713 Train acc: 0.955000\n",
      "Epoch: 259/1000 Iteration: 2335 Train loss: 0.209134 Train acc: 0.933333\n",
      "Epoch: 259/1000 Iteration: 2340 Train loss: 0.198359 Train acc: 0.936667\n",
      "Epoch: 260/1000 Iteration: 2345 Train loss: 0.182193 Train acc: 0.958333\n",
      "Epoch: 261/1000 Iteration: 2350 Train loss: 0.170314 Train acc: 0.945000\n",
      "Epoch: 261/1000 Iteration: 2350 Validation loss: 0.127622 Validation acc: 0.953889\n",
      "Epoch: 261/1000 Iteration: 2355 Train loss: 0.157096 Train acc: 0.958333\n",
      "Epoch: 262/1000 Iteration: 2360 Train loss: 0.169357 Train acc: 0.955000\n",
      "Epoch: 262/1000 Iteration: 2365 Train loss: 0.184164 Train acc: 0.943333\n",
      "Epoch: 263/1000 Iteration: 2370 Train loss: 0.216137 Train acc: 0.938333\n",
      "Epoch: 263/1000 Iteration: 2375 Train loss: 0.150279 Train acc: 0.961667\n",
      "Epoch: 263/1000 Iteration: 2375 Validation loss: 0.130314 Validation acc: 0.954444\n",
      "Epoch: 264/1000 Iteration: 2380 Train loss: 0.214962 Train acc: 0.935000\n",
      "Epoch: 264/1000 Iteration: 2385 Train loss: 0.197955 Train acc: 0.943333\n",
      "Epoch: 265/1000 Iteration: 2390 Train loss: 0.166448 Train acc: 0.945000\n",
      "Epoch: 266/1000 Iteration: 2395 Train loss: 0.167927 Train acc: 0.946667\n",
      "Epoch: 266/1000 Iteration: 2400 Train loss: 0.171945 Train acc: 0.945000\n",
      "Epoch: 266/1000 Iteration: 2400 Validation loss: 0.124839 Validation acc: 0.953889\n",
      "Epoch: 267/1000 Iteration: 2405 Train loss: 0.175199 Train acc: 0.960000\n",
      "Epoch: 267/1000 Iteration: 2410 Train loss: 0.183653 Train acc: 0.946667\n",
      "Epoch: 268/1000 Iteration: 2415 Train loss: 0.194716 Train acc: 0.940000\n",
      "Epoch: 268/1000 Iteration: 2420 Train loss: 0.157947 Train acc: 0.961667\n",
      "Epoch: 269/1000 Iteration: 2425 Train loss: 0.207224 Train acc: 0.933333\n",
      "Epoch: 269/1000 Iteration: 2425 Validation loss: 0.124556 Validation acc: 0.953889\n",
      "Epoch: 269/1000 Iteration: 2430 Train loss: 0.180154 Train acc: 0.951667\n",
      "Epoch: 270/1000 Iteration: 2435 Train loss: 0.169294 Train acc: 0.941667\n",
      "Epoch: 271/1000 Iteration: 2440 Train loss: 0.159723 Train acc: 0.953333\n",
      "Epoch: 271/1000 Iteration: 2445 Train loss: 0.153912 Train acc: 0.961667\n",
      "Epoch: 272/1000 Iteration: 2450 Train loss: 0.169489 Train acc: 0.953333\n",
      "Epoch: 272/1000 Iteration: 2450 Validation loss: 0.125628 Validation acc: 0.953333\n",
      "Epoch: 272/1000 Iteration: 2455 Train loss: 0.175410 Train acc: 0.960000\n",
      "Epoch: 273/1000 Iteration: 2460 Train loss: 0.199761 Train acc: 0.931667\n",
      "Epoch: 273/1000 Iteration: 2465 Train loss: 0.147634 Train acc: 0.956667\n",
      "Epoch: 274/1000 Iteration: 2470 Train loss: 0.205670 Train acc: 0.926667\n",
      "Epoch: 274/1000 Iteration: 2475 Train loss: 0.191951 Train acc: 0.951667\n",
      "Epoch: 274/1000 Iteration: 2475 Validation loss: 0.129818 Validation acc: 0.951111\n",
      "Epoch: 275/1000 Iteration: 2480 Train loss: 0.187815 Train acc: 0.941667\n",
      "Epoch: 276/1000 Iteration: 2485 Train loss: 0.158557 Train acc: 0.948333\n",
      "Epoch: 276/1000 Iteration: 2490 Train loss: 0.173178 Train acc: 0.950000\n",
      "Epoch: 277/1000 Iteration: 2495 Train loss: 0.165173 Train acc: 0.963333\n",
      "Epoch: 277/1000 Iteration: 2500 Train loss: 0.162154 Train acc: 0.948333\n",
      "Epoch: 277/1000 Iteration: 2500 Validation loss: 0.125288 Validation acc: 0.950000\n",
      "Epoch: 278/1000 Iteration: 2505 Train loss: 0.207719 Train acc: 0.926667\n",
      "Epoch: 278/1000 Iteration: 2510 Train loss: 0.158595 Train acc: 0.950000\n",
      "Epoch: 279/1000 Iteration: 2515 Train loss: 0.209440 Train acc: 0.930000\n",
      "Epoch: 279/1000 Iteration: 2520 Train loss: 0.185372 Train acc: 0.941667\n",
      "Epoch: 280/1000 Iteration: 2525 Train loss: 0.186033 Train acc: 0.948333\n",
      "Epoch: 280/1000 Iteration: 2525 Validation loss: 0.130442 Validation acc: 0.948889\n",
      "Epoch: 281/1000 Iteration: 2530 Train loss: 0.170612 Train acc: 0.951667\n",
      "Epoch: 281/1000 Iteration: 2535 Train loss: 0.156010 Train acc: 0.951667\n",
      "Epoch: 282/1000 Iteration: 2540 Train loss: 0.176368 Train acc: 0.950000\n",
      "Epoch: 282/1000 Iteration: 2545 Train loss: 0.164434 Train acc: 0.950000\n",
      "Epoch: 283/1000 Iteration: 2550 Train loss: 0.203112 Train acc: 0.930000\n",
      "Epoch: 283/1000 Iteration: 2550 Validation loss: 0.127754 Validation acc: 0.951111\n",
      "Epoch: 283/1000 Iteration: 2555 Train loss: 0.166401 Train acc: 0.960000\n",
      "Epoch: 284/1000 Iteration: 2560 Train loss: 0.206961 Train acc: 0.935000\n",
      "Epoch: 284/1000 Iteration: 2565 Train loss: 0.178589 Train acc: 0.955000\n",
      "Epoch: 285/1000 Iteration: 2570 Train loss: 0.158963 Train acc: 0.958333\n",
      "Epoch: 286/1000 Iteration: 2575 Train loss: 0.149753 Train acc: 0.958333\n",
      "Epoch: 286/1000 Iteration: 2575 Validation loss: 0.117487 Validation acc: 0.952778\n",
      "Epoch: 286/1000 Iteration: 2580 Train loss: 0.152900 Train acc: 0.955000\n",
      "Epoch: 287/1000 Iteration: 2585 Train loss: 0.153626 Train acc: 0.960000\n",
      "Epoch: 287/1000 Iteration: 2590 Train loss: 0.157583 Train acc: 0.953333\n",
      "Epoch: 288/1000 Iteration: 2595 Train loss: 0.189689 Train acc: 0.941667\n",
      "Epoch: 288/1000 Iteration: 2600 Train loss: 0.170601 Train acc: 0.948333\n",
      "Epoch: 288/1000 Iteration: 2600 Validation loss: 0.116839 Validation acc: 0.953889\n",
      "Epoch: 289/1000 Iteration: 2605 Train loss: 0.194581 Train acc: 0.928333\n",
      "Epoch: 289/1000 Iteration: 2610 Train loss: 0.201221 Train acc: 0.945000\n",
      "Epoch: 290/1000 Iteration: 2615 Train loss: 0.172785 Train acc: 0.951667\n",
      "Epoch: 291/1000 Iteration: 2620 Train loss: 0.152700 Train acc: 0.953333\n",
      "Epoch: 291/1000 Iteration: 2625 Train loss: 0.152930 Train acc: 0.958333\n",
      "Epoch: 291/1000 Iteration: 2625 Validation loss: 0.119598 Validation acc: 0.952778\n",
      "Epoch: 292/1000 Iteration: 2630 Train loss: 0.171507 Train acc: 0.960000\n",
      "Epoch: 292/1000 Iteration: 2635 Train loss: 0.150077 Train acc: 0.955000\n",
      "Epoch: 293/1000 Iteration: 2640 Train loss: 0.188345 Train acc: 0.935000\n",
      "Epoch: 293/1000 Iteration: 2645 Train loss: 0.139095 Train acc: 0.965000\n",
      "Epoch: 294/1000 Iteration: 2650 Train loss: 0.198392 Train acc: 0.933333\n",
      "Epoch: 294/1000 Iteration: 2650 Validation loss: 0.119696 Validation acc: 0.951667\n",
      "Epoch: 294/1000 Iteration: 2655 Train loss: 0.183575 Train acc: 0.946667\n",
      "Epoch: 295/1000 Iteration: 2660 Train loss: 0.178949 Train acc: 0.958333\n",
      "Epoch: 296/1000 Iteration: 2665 Train loss: 0.149078 Train acc: 0.955000\n",
      "Epoch: 296/1000 Iteration: 2670 Train loss: 0.148100 Train acc: 0.961667\n",
      "Epoch: 297/1000 Iteration: 2675 Train loss: 0.175078 Train acc: 0.963333\n",
      "Epoch: 297/1000 Iteration: 2675 Validation loss: 0.118639 Validation acc: 0.951667\n",
      "Epoch: 297/1000 Iteration: 2680 Train loss: 0.165185 Train acc: 0.958333\n",
      "Epoch: 298/1000 Iteration: 2685 Train loss: 0.215558 Train acc: 0.920000\n",
      "Epoch: 298/1000 Iteration: 2690 Train loss: 0.155808 Train acc: 0.956667\n",
      "Epoch: 299/1000 Iteration: 2695 Train loss: 0.177362 Train acc: 0.933333\n",
      "Epoch: 299/1000 Iteration: 2700 Train loss: 0.183831 Train acc: 0.946667\n",
      "Epoch: 299/1000 Iteration: 2700 Validation loss: 0.121571 Validation acc: 0.952222\n",
      "Epoch: 300/1000 Iteration: 2705 Train loss: 0.164235 Train acc: 0.950000\n",
      "Epoch: 301/1000 Iteration: 2710 Train loss: 0.140068 Train acc: 0.958333\n",
      "Epoch: 301/1000 Iteration: 2715 Train loss: 0.156471 Train acc: 0.945000\n",
      "Epoch: 302/1000 Iteration: 2720 Train loss: 0.156239 Train acc: 0.945000\n",
      "Epoch: 302/1000 Iteration: 2725 Train loss: 0.165721 Train acc: 0.953333\n",
      "Epoch: 302/1000 Iteration: 2725 Validation loss: 0.123855 Validation acc: 0.951111\n",
      "Epoch: 303/1000 Iteration: 2730 Train loss: 0.192068 Train acc: 0.931667\n",
      "Epoch: 303/1000 Iteration: 2735 Train loss: 0.145356 Train acc: 0.956667\n",
      "Epoch: 304/1000 Iteration: 2740 Train loss: 0.186348 Train acc: 0.933333\n",
      "Epoch: 304/1000 Iteration: 2745 Train loss: 0.185014 Train acc: 0.955000\n",
      "Epoch: 305/1000 Iteration: 2750 Train loss: 0.150055 Train acc: 0.956667\n",
      "Epoch: 305/1000 Iteration: 2750 Validation loss: 0.119699 Validation acc: 0.952222\n",
      "Epoch: 306/1000 Iteration: 2755 Train loss: 0.150542 Train acc: 0.950000\n",
      "Epoch: 306/1000 Iteration: 2760 Train loss: 0.148211 Train acc: 0.948333\n",
      "Epoch: 307/1000 Iteration: 2765 Train loss: 0.167683 Train acc: 0.953333\n",
      "Epoch: 307/1000 Iteration: 2770 Train loss: 0.148840 Train acc: 0.956667\n",
      "Epoch: 308/1000 Iteration: 2775 Train loss: 0.192649 Train acc: 0.936667\n",
      "Epoch: 308/1000 Iteration: 2775 Validation loss: 0.123726 Validation acc: 0.952778\n",
      "Epoch: 308/1000 Iteration: 2780 Train loss: 0.135663 Train acc: 0.961667\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 309/1000 Iteration: 2785 Train loss: 0.198235 Train acc: 0.933333\n",
      "Epoch: 309/1000 Iteration: 2790 Train loss: 0.172862 Train acc: 0.950000\n",
      "Epoch: 310/1000 Iteration: 2795 Train loss: 0.154007 Train acc: 0.953333\n",
      "Epoch: 311/1000 Iteration: 2800 Train loss: 0.155767 Train acc: 0.960000\n",
      "Epoch: 311/1000 Iteration: 2800 Validation loss: 0.119605 Validation acc: 0.951667\n",
      "Epoch: 311/1000 Iteration: 2805 Train loss: 0.174888 Train acc: 0.943333\n",
      "Epoch: 312/1000 Iteration: 2810 Train loss: 0.155708 Train acc: 0.958333\n",
      "Epoch: 312/1000 Iteration: 2815 Train loss: 0.158572 Train acc: 0.940000\n",
      "Epoch: 313/1000 Iteration: 2820 Train loss: 0.192403 Train acc: 0.935000\n",
      "Epoch: 313/1000 Iteration: 2825 Train loss: 0.138233 Train acc: 0.963333\n",
      "Epoch: 313/1000 Iteration: 2825 Validation loss: 0.119868 Validation acc: 0.951667\n",
      "Epoch: 314/1000 Iteration: 2830 Train loss: 0.188463 Train acc: 0.941667\n",
      "Epoch: 314/1000 Iteration: 2835 Train loss: 0.169028 Train acc: 0.948333\n",
      "Epoch: 315/1000 Iteration: 2840 Train loss: 0.156876 Train acc: 0.948333\n",
      "Epoch: 316/1000 Iteration: 2845 Train loss: 0.147820 Train acc: 0.958333\n",
      "Epoch: 316/1000 Iteration: 2850 Train loss: 0.141170 Train acc: 0.963333\n",
      "Epoch: 316/1000 Iteration: 2850 Validation loss: 0.121933 Validation acc: 0.953333\n",
      "Epoch: 317/1000 Iteration: 2855 Train loss: 0.156525 Train acc: 0.961667\n",
      "Epoch: 317/1000 Iteration: 2860 Train loss: 0.170795 Train acc: 0.951667\n",
      "Epoch: 318/1000 Iteration: 2865 Train loss: 0.189998 Train acc: 0.940000\n",
      "Epoch: 318/1000 Iteration: 2870 Train loss: 0.131915 Train acc: 0.963333\n",
      "Epoch: 319/1000 Iteration: 2875 Train loss: 0.202734 Train acc: 0.930000\n",
      "Epoch: 319/1000 Iteration: 2875 Validation loss: 0.121937 Validation acc: 0.951111\n",
      "Epoch: 319/1000 Iteration: 2880 Train loss: 0.164689 Train acc: 0.951667\n",
      "Epoch: 320/1000 Iteration: 2885 Train loss: 0.160004 Train acc: 0.948333\n",
      "Epoch: 321/1000 Iteration: 2890 Train loss: 0.154432 Train acc: 0.943333\n",
      "Epoch: 321/1000 Iteration: 2895 Train loss: 0.145156 Train acc: 0.958333\n",
      "Epoch: 322/1000 Iteration: 2900 Train loss: 0.158014 Train acc: 0.960000\n",
      "Epoch: 322/1000 Iteration: 2900 Validation loss: 0.118908 Validation acc: 0.951111\n",
      "Epoch: 322/1000 Iteration: 2905 Train loss: 0.160292 Train acc: 0.950000\n",
      "Epoch: 323/1000 Iteration: 2910 Train loss: 0.188464 Train acc: 0.938333\n",
      "Epoch: 323/1000 Iteration: 2915 Train loss: 0.136037 Train acc: 0.955000\n",
      "Epoch: 324/1000 Iteration: 2920 Train loss: 0.180546 Train acc: 0.931667\n",
      "Epoch: 324/1000 Iteration: 2925 Train loss: 0.156347 Train acc: 0.958333\n",
      "Epoch: 324/1000 Iteration: 2925 Validation loss: 0.117038 Validation acc: 0.952778\n",
      "Epoch: 325/1000 Iteration: 2930 Train loss: 0.159567 Train acc: 0.950000\n",
      "Epoch: 326/1000 Iteration: 2935 Train loss: 0.150668 Train acc: 0.945000\n",
      "Epoch: 326/1000 Iteration: 2940 Train loss: 0.142754 Train acc: 0.953333\n",
      "Epoch: 327/1000 Iteration: 2945 Train loss: 0.172714 Train acc: 0.951667\n",
      "Epoch: 327/1000 Iteration: 2950 Train loss: 0.165531 Train acc: 0.943333\n",
      "Epoch: 327/1000 Iteration: 2950 Validation loss: 0.115414 Validation acc: 0.952778\n",
      "Epoch: 328/1000 Iteration: 2955 Train loss: 0.195345 Train acc: 0.935000\n",
      "Epoch: 328/1000 Iteration: 2960 Train loss: 0.135711 Train acc: 0.965000\n",
      "Epoch: 329/1000 Iteration: 2965 Train loss: 0.176931 Train acc: 0.938333\n",
      "Epoch: 329/1000 Iteration: 2970 Train loss: 0.169620 Train acc: 0.948333\n",
      "Epoch: 330/1000 Iteration: 2975 Train loss: 0.157307 Train acc: 0.950000\n",
      "Epoch: 330/1000 Iteration: 2975 Validation loss: 0.119482 Validation acc: 0.952778\n",
      "Epoch: 331/1000 Iteration: 2980 Train loss: 0.139426 Train acc: 0.951667\n",
      "Epoch: 331/1000 Iteration: 2985 Train loss: 0.152551 Train acc: 0.951667\n",
      "Epoch: 332/1000 Iteration: 2990 Train loss: 0.144609 Train acc: 0.958333\n",
      "Epoch: 332/1000 Iteration: 2995 Train loss: 0.158151 Train acc: 0.953333\n",
      "Epoch: 333/1000 Iteration: 3000 Train loss: 0.193837 Train acc: 0.936667\n",
      "Epoch: 333/1000 Iteration: 3000 Validation loss: 0.119991 Validation acc: 0.952222\n",
      "Epoch: 333/1000 Iteration: 3005 Train loss: 0.130325 Train acc: 0.965000\n",
      "Epoch: 334/1000 Iteration: 3010 Train loss: 0.187766 Train acc: 0.933333\n",
      "Epoch: 334/1000 Iteration: 3015 Train loss: 0.159425 Train acc: 0.948333\n",
      "Epoch: 335/1000 Iteration: 3020 Train loss: 0.153192 Train acc: 0.956667\n",
      "Epoch: 336/1000 Iteration: 3025 Train loss: 0.145759 Train acc: 0.955000\n",
      "Epoch: 336/1000 Iteration: 3025 Validation loss: 0.118620 Validation acc: 0.952778\n",
      "Epoch: 336/1000 Iteration: 3030 Train loss: 0.146404 Train acc: 0.951667\n",
      "Epoch: 337/1000 Iteration: 3035 Train loss: 0.135182 Train acc: 0.955000\n",
      "Epoch: 337/1000 Iteration: 3040 Train loss: 0.154912 Train acc: 0.951667\n",
      "Epoch: 338/1000 Iteration: 3045 Train loss: 0.165802 Train acc: 0.946667\n",
      "Epoch: 338/1000 Iteration: 3050 Train loss: 0.144744 Train acc: 0.958333\n",
      "Epoch: 338/1000 Iteration: 3050 Validation loss: 0.117526 Validation acc: 0.952778\n",
      "Epoch: 339/1000 Iteration: 3055 Train loss: 0.180518 Train acc: 0.931667\n",
      "Epoch: 339/1000 Iteration: 3060 Train loss: 0.170689 Train acc: 0.948333\n",
      "Epoch: 340/1000 Iteration: 3065 Train loss: 0.144308 Train acc: 0.950000\n",
      "Epoch: 341/1000 Iteration: 3070 Train loss: 0.158090 Train acc: 0.948333\n",
      "Epoch: 341/1000 Iteration: 3075 Train loss: 0.138987 Train acc: 0.953333\n",
      "Epoch: 341/1000 Iteration: 3075 Validation loss: 0.121905 Validation acc: 0.951111\n",
      "Epoch: 342/1000 Iteration: 3080 Train loss: 0.152924 Train acc: 0.963333\n",
      "Epoch: 342/1000 Iteration: 3085 Train loss: 0.153206 Train acc: 0.951667\n",
      "Epoch: 343/1000 Iteration: 3090 Train loss: 0.170612 Train acc: 0.941667\n",
      "Epoch: 343/1000 Iteration: 3095 Train loss: 0.115316 Train acc: 0.965000\n",
      "Epoch: 344/1000 Iteration: 3100 Train loss: 0.182103 Train acc: 0.931667\n",
      "Epoch: 344/1000 Iteration: 3100 Validation loss: 0.116997 Validation acc: 0.952778\n",
      "Epoch: 344/1000 Iteration: 3105 Train loss: 0.168857 Train acc: 0.945000\n",
      "Epoch: 345/1000 Iteration: 3110 Train loss: 0.157719 Train acc: 0.951667\n",
      "Epoch: 346/1000 Iteration: 3115 Train loss: 0.144585 Train acc: 0.958333\n",
      "Epoch: 346/1000 Iteration: 3120 Train loss: 0.142215 Train acc: 0.951667\n",
      "Epoch: 347/1000 Iteration: 3125 Train loss: 0.129935 Train acc: 0.963333\n",
      "Epoch: 347/1000 Iteration: 3125 Validation loss: 0.118282 Validation acc: 0.952778\n",
      "Epoch: 347/1000 Iteration: 3130 Train loss: 0.143481 Train acc: 0.955000\n",
      "Epoch: 348/1000 Iteration: 3135 Train loss: 0.188708 Train acc: 0.933333\n",
      "Epoch: 348/1000 Iteration: 3140 Train loss: 0.144870 Train acc: 0.953333\n",
      "Epoch: 349/1000 Iteration: 3145 Train loss: 0.179912 Train acc: 0.930000\n",
      "Epoch: 349/1000 Iteration: 3150 Train loss: 0.162133 Train acc: 0.945000\n",
      "Epoch: 349/1000 Iteration: 3150 Validation loss: 0.114680 Validation acc: 0.952778\n",
      "Epoch: 350/1000 Iteration: 3155 Train loss: 0.142730 Train acc: 0.953333\n",
      "Epoch: 351/1000 Iteration: 3160 Train loss: 0.141082 Train acc: 0.951667\n",
      "Epoch: 351/1000 Iteration: 3165 Train loss: 0.138959 Train acc: 0.956667\n",
      "Epoch: 352/1000 Iteration: 3170 Train loss: 0.164919 Train acc: 0.955000\n",
      "Epoch: 352/1000 Iteration: 3175 Train loss: 0.161646 Train acc: 0.953333\n",
      "Epoch: 352/1000 Iteration: 3175 Validation loss: 0.114231 Validation acc: 0.952222\n",
      "Epoch: 353/1000 Iteration: 3180 Train loss: 0.180147 Train acc: 0.943333\n",
      "Epoch: 353/1000 Iteration: 3185 Train loss: 0.130808 Train acc: 0.956667\n",
      "Epoch: 354/1000 Iteration: 3190 Train loss: 0.174338 Train acc: 0.936667\n",
      "Epoch: 354/1000 Iteration: 3195 Train loss: 0.159507 Train acc: 0.953333\n",
      "Epoch: 355/1000 Iteration: 3200 Train loss: 0.146653 Train acc: 0.956667\n",
      "Epoch: 355/1000 Iteration: 3200 Validation loss: 0.114860 Validation acc: 0.951111\n",
      "Epoch: 356/1000 Iteration: 3205 Train loss: 0.141467 Train acc: 0.960000\n",
      "Epoch: 356/1000 Iteration: 3210 Train loss: 0.144333 Train acc: 0.956667\n",
      "Epoch: 357/1000 Iteration: 3215 Train loss: 0.137108 Train acc: 0.955000\n",
      "Epoch: 357/1000 Iteration: 3220 Train loss: 0.153502 Train acc: 0.950000\n",
      "Epoch: 358/1000 Iteration: 3225 Train loss: 0.182194 Train acc: 0.931667\n",
      "Epoch: 358/1000 Iteration: 3225 Validation loss: 0.113086 Validation acc: 0.951667\n",
      "Epoch: 358/1000 Iteration: 3230 Train loss: 0.118116 Train acc: 0.961667\n",
      "Epoch: 359/1000 Iteration: 3235 Train loss: 0.192479 Train acc: 0.931667\n",
      "Epoch: 359/1000 Iteration: 3240 Train loss: 0.171068 Train acc: 0.958333\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 360/1000 Iteration: 3245 Train loss: 0.154844 Train acc: 0.950000\n",
      "Epoch: 361/1000 Iteration: 3250 Train loss: 0.123391 Train acc: 0.953333\n",
      "Epoch: 361/1000 Iteration: 3250 Validation loss: 0.111424 Validation acc: 0.952778\n",
      "Epoch: 361/1000 Iteration: 3255 Train loss: 0.137787 Train acc: 0.958333\n",
      "Epoch: 362/1000 Iteration: 3260 Train loss: 0.135507 Train acc: 0.965000\n",
      "Epoch: 362/1000 Iteration: 3265 Train loss: 0.144411 Train acc: 0.960000\n",
      "Epoch: 363/1000 Iteration: 3270 Train loss: 0.187780 Train acc: 0.941667\n",
      "Epoch: 363/1000 Iteration: 3275 Train loss: 0.123403 Train acc: 0.970000\n",
      "Epoch: 363/1000 Iteration: 3275 Validation loss: 0.112533 Validation acc: 0.952778\n",
      "Epoch: 364/1000 Iteration: 3280 Train loss: 0.183308 Train acc: 0.940000\n",
      "Epoch: 364/1000 Iteration: 3285 Train loss: 0.168189 Train acc: 0.953333\n",
      "Epoch: 365/1000 Iteration: 3290 Train loss: 0.145025 Train acc: 0.951667\n",
      "Epoch: 366/1000 Iteration: 3295 Train loss: 0.135079 Train acc: 0.960000\n",
      "Epoch: 366/1000 Iteration: 3300 Train loss: 0.139670 Train acc: 0.955000\n",
      "Epoch: 366/1000 Iteration: 3300 Validation loss: 0.117244 Validation acc: 0.951667\n",
      "Epoch: 367/1000 Iteration: 3305 Train loss: 0.145412 Train acc: 0.960000\n",
      "Epoch: 367/1000 Iteration: 3310 Train loss: 0.153948 Train acc: 0.955000\n",
      "Epoch: 368/1000 Iteration: 3315 Train loss: 0.176020 Train acc: 0.945000\n",
      "Epoch: 368/1000 Iteration: 3320 Train loss: 0.123280 Train acc: 0.961667\n",
      "Epoch: 369/1000 Iteration: 3325 Train loss: 0.179110 Train acc: 0.933333\n",
      "Epoch: 369/1000 Iteration: 3325 Validation loss: 0.114934 Validation acc: 0.954444\n",
      "Epoch: 369/1000 Iteration: 3330 Train loss: 0.156404 Train acc: 0.950000\n",
      "Epoch: 370/1000 Iteration: 3335 Train loss: 0.132103 Train acc: 0.960000\n",
      "Epoch: 371/1000 Iteration: 3340 Train loss: 0.140381 Train acc: 0.945000\n",
      "Epoch: 371/1000 Iteration: 3345 Train loss: 0.132481 Train acc: 0.958333\n",
      "Epoch: 372/1000 Iteration: 3350 Train loss: 0.147624 Train acc: 0.953333\n",
      "Epoch: 372/1000 Iteration: 3350 Validation loss: 0.113778 Validation acc: 0.952222\n",
      "Epoch: 372/1000 Iteration: 3355 Train loss: 0.146762 Train acc: 0.951667\n",
      "Epoch: 373/1000 Iteration: 3360 Train loss: 0.177120 Train acc: 0.936667\n",
      "Epoch: 373/1000 Iteration: 3365 Train loss: 0.122540 Train acc: 0.958333\n",
      "Epoch: 374/1000 Iteration: 3370 Train loss: 0.171100 Train acc: 0.940000\n",
      "Epoch: 374/1000 Iteration: 3375 Train loss: 0.163290 Train acc: 0.951667\n",
      "Epoch: 374/1000 Iteration: 3375 Validation loss: 0.117097 Validation acc: 0.952778\n",
      "Epoch: 375/1000 Iteration: 3380 Train loss: 0.152840 Train acc: 0.948333\n",
      "Epoch: 376/1000 Iteration: 3385 Train loss: 0.128028 Train acc: 0.960000\n",
      "Epoch: 376/1000 Iteration: 3390 Train loss: 0.142327 Train acc: 0.956667\n",
      "Epoch: 377/1000 Iteration: 3395 Train loss: 0.145167 Train acc: 0.956667\n",
      "Epoch: 377/1000 Iteration: 3400 Train loss: 0.144563 Train acc: 0.950000\n",
      "Epoch: 377/1000 Iteration: 3400 Validation loss: 0.115878 Validation acc: 0.952222\n",
      "Epoch: 378/1000 Iteration: 3405 Train loss: 0.176798 Train acc: 0.936667\n",
      "Epoch: 378/1000 Iteration: 3410 Train loss: 0.124320 Train acc: 0.968333\n",
      "Epoch: 379/1000 Iteration: 3415 Train loss: 0.173520 Train acc: 0.936667\n",
      "Epoch: 379/1000 Iteration: 3420 Train loss: 0.163472 Train acc: 0.948333\n",
      "Epoch: 380/1000 Iteration: 3425 Train loss: 0.148360 Train acc: 0.956667\n",
      "Epoch: 380/1000 Iteration: 3425 Validation loss: 0.116258 Validation acc: 0.951111\n",
      "Epoch: 381/1000 Iteration: 3430 Train loss: 0.126776 Train acc: 0.960000\n",
      "Epoch: 381/1000 Iteration: 3435 Train loss: 0.139910 Train acc: 0.951667\n",
      "Epoch: 382/1000 Iteration: 3440 Train loss: 0.139508 Train acc: 0.963333\n",
      "Epoch: 382/1000 Iteration: 3445 Train loss: 0.141842 Train acc: 0.958333\n",
      "Epoch: 383/1000 Iteration: 3450 Train loss: 0.170742 Train acc: 0.938333\n",
      "Epoch: 383/1000 Iteration: 3450 Validation loss: 0.116107 Validation acc: 0.952222\n",
      "Epoch: 383/1000 Iteration: 3455 Train loss: 0.120196 Train acc: 0.958333\n",
      "Epoch: 384/1000 Iteration: 3460 Train loss: 0.155742 Train acc: 0.938333\n",
      "Epoch: 384/1000 Iteration: 3465 Train loss: 0.156648 Train acc: 0.950000\n",
      "Epoch: 385/1000 Iteration: 3470 Train loss: 0.136874 Train acc: 0.965000\n",
      "Epoch: 386/1000 Iteration: 3475 Train loss: 0.113470 Train acc: 0.960000\n",
      "Epoch: 386/1000 Iteration: 3475 Validation loss: 0.136961 Validation acc: 0.948889\n",
      "Epoch: 386/1000 Iteration: 3480 Train loss: 0.143940 Train acc: 0.955000\n",
      "Epoch: 387/1000 Iteration: 3485 Train loss: 0.139509 Train acc: 0.953333\n",
      "Epoch: 387/1000 Iteration: 3490 Train loss: 0.146766 Train acc: 0.958333\n",
      "Epoch: 388/1000 Iteration: 3495 Train loss: 0.172043 Train acc: 0.931667\n",
      "Epoch: 388/1000 Iteration: 3500 Train loss: 0.123294 Train acc: 0.960000\n",
      "Epoch: 388/1000 Iteration: 3500 Validation loss: 0.108654 Validation acc: 0.956111\n",
      "Epoch: 389/1000 Iteration: 3505 Train loss: 0.156800 Train acc: 0.943333\n",
      "Epoch: 389/1000 Iteration: 3510 Train loss: 0.147240 Train acc: 0.953333\n",
      "Epoch: 390/1000 Iteration: 3515 Train loss: 0.149107 Train acc: 0.945000\n",
      "Epoch: 391/1000 Iteration: 3520 Train loss: 0.129177 Train acc: 0.956667\n",
      "Epoch: 391/1000 Iteration: 3525 Train loss: 0.130814 Train acc: 0.953333\n",
      "Epoch: 391/1000 Iteration: 3525 Validation loss: 0.114290 Validation acc: 0.953333\n",
      "Epoch: 392/1000 Iteration: 3530 Train loss: 0.141436 Train acc: 0.956667\n",
      "Epoch: 392/1000 Iteration: 3535 Train loss: 0.140231 Train acc: 0.956667\n",
      "Epoch: 393/1000 Iteration: 3540 Train loss: 0.149617 Train acc: 0.941667\n",
      "Epoch: 393/1000 Iteration: 3545 Train loss: 0.125355 Train acc: 0.960000\n",
      "Epoch: 394/1000 Iteration: 3550 Train loss: 0.163900 Train acc: 0.936667\n",
      "Epoch: 394/1000 Iteration: 3550 Validation loss: 0.112788 Validation acc: 0.953889\n",
      "Epoch: 394/1000 Iteration: 3555 Train loss: 0.142564 Train acc: 0.958333\n",
      "Epoch: 395/1000 Iteration: 3560 Train loss: 0.149294 Train acc: 0.945000\n",
      "Epoch: 396/1000 Iteration: 3565 Train loss: 0.127590 Train acc: 0.960000\n",
      "Epoch: 396/1000 Iteration: 3570 Train loss: 0.116617 Train acc: 0.956667\n",
      "Epoch: 397/1000 Iteration: 3575 Train loss: 0.146013 Train acc: 0.956667\n",
      "Epoch: 397/1000 Iteration: 3575 Validation loss: 0.114475 Validation acc: 0.953333\n",
      "Epoch: 397/1000 Iteration: 3580 Train loss: 0.136141 Train acc: 0.963333\n",
      "Epoch: 398/1000 Iteration: 3585 Train loss: 0.168418 Train acc: 0.938333\n",
      "Epoch: 398/1000 Iteration: 3590 Train loss: 0.100305 Train acc: 0.978333\n",
      "Epoch: 399/1000 Iteration: 3595 Train loss: 0.158189 Train acc: 0.941667\n",
      "Epoch: 399/1000 Iteration: 3600 Train loss: 0.156215 Train acc: 0.955000\n",
      "Epoch: 399/1000 Iteration: 3600 Validation loss: 0.112594 Validation acc: 0.955000\n",
      "Epoch: 400/1000 Iteration: 3605 Train loss: 0.145983 Train acc: 0.950000\n",
      "Epoch: 401/1000 Iteration: 3610 Train loss: 0.124260 Train acc: 0.970000\n",
      "Epoch: 401/1000 Iteration: 3615 Train loss: 0.133471 Train acc: 0.946667\n",
      "Epoch: 402/1000 Iteration: 3620 Train loss: 0.134887 Train acc: 0.968333\n",
      "Epoch: 402/1000 Iteration: 3625 Train loss: 0.146092 Train acc: 0.956667\n",
      "Epoch: 402/1000 Iteration: 3625 Validation loss: 0.111239 Validation acc: 0.953889\n",
      "Epoch: 403/1000 Iteration: 3630 Train loss: 0.172659 Train acc: 0.943333\n",
      "Epoch: 403/1000 Iteration: 3635 Train loss: 0.119510 Train acc: 0.963333\n",
      "Epoch: 404/1000 Iteration: 3640 Train loss: 0.156182 Train acc: 0.941667\n",
      "Epoch: 404/1000 Iteration: 3645 Train loss: 0.140756 Train acc: 0.953333\n",
      "Epoch: 405/1000 Iteration: 3650 Train loss: 0.142795 Train acc: 0.951667\n",
      "Epoch: 405/1000 Iteration: 3650 Validation loss: 0.110842 Validation acc: 0.954444\n",
      "Epoch: 406/1000 Iteration: 3655 Train loss: 0.122213 Train acc: 0.958333\n",
      "Epoch: 406/1000 Iteration: 3660 Train loss: 0.137625 Train acc: 0.946667\n",
      "Epoch: 407/1000 Iteration: 3665 Train loss: 0.146807 Train acc: 0.955000\n",
      "Epoch: 407/1000 Iteration: 3670 Train loss: 0.143940 Train acc: 0.966667\n",
      "Epoch: 408/1000 Iteration: 3675 Train loss: 0.161697 Train acc: 0.940000\n",
      "Epoch: 408/1000 Iteration: 3675 Validation loss: 0.111031 Validation acc: 0.953333\n",
      "Epoch: 408/1000 Iteration: 3680 Train loss: 0.107397 Train acc: 0.968333\n",
      "Epoch: 409/1000 Iteration: 3685 Train loss: 0.169320 Train acc: 0.938333\n",
      "Epoch: 409/1000 Iteration: 3690 Train loss: 0.142899 Train acc: 0.950000\n",
      "Epoch: 410/1000 Iteration: 3695 Train loss: 0.130963 Train acc: 0.961667\n",
      "Epoch: 411/1000 Iteration: 3700 Train loss: 0.116536 Train acc: 0.961667\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 411/1000 Iteration: 3700 Validation loss: 0.111677 Validation acc: 0.954444\n",
      "Epoch: 411/1000 Iteration: 3705 Train loss: 0.145494 Train acc: 0.948333\n",
      "Epoch: 412/1000 Iteration: 3710 Train loss: 0.143117 Train acc: 0.956667\n",
      "Epoch: 412/1000 Iteration: 3715 Train loss: 0.137851 Train acc: 0.958333\n",
      "Epoch: 413/1000 Iteration: 3720 Train loss: 0.154965 Train acc: 0.950000\n",
      "Epoch: 413/1000 Iteration: 3725 Train loss: 0.108125 Train acc: 0.966667\n",
      "Epoch: 413/1000 Iteration: 3725 Validation loss: 0.110309 Validation acc: 0.952778\n",
      "Epoch: 414/1000 Iteration: 3730 Train loss: 0.170913 Train acc: 0.936667\n",
      "Epoch: 414/1000 Iteration: 3735 Train loss: 0.136517 Train acc: 0.956667\n",
      "Epoch: 415/1000 Iteration: 3740 Train loss: 0.137235 Train acc: 0.958333\n",
      "Epoch: 416/1000 Iteration: 3745 Train loss: 0.110348 Train acc: 0.966667\n",
      "Epoch: 416/1000 Iteration: 3750 Train loss: 0.130755 Train acc: 0.953333\n",
      "Epoch: 416/1000 Iteration: 3750 Validation loss: 0.113179 Validation acc: 0.954444\n",
      "Epoch: 417/1000 Iteration: 3755 Train loss: 0.120491 Train acc: 0.966667\n",
      "Epoch: 417/1000 Iteration: 3760 Train loss: 0.133225 Train acc: 0.958333\n",
      "Epoch: 418/1000 Iteration: 3765 Train loss: 0.166985 Train acc: 0.940000\n",
      "Epoch: 418/1000 Iteration: 3770 Train loss: 0.101676 Train acc: 0.965000\n",
      "Epoch: 419/1000 Iteration: 3775 Train loss: 0.166532 Train acc: 0.936667\n",
      "Epoch: 419/1000 Iteration: 3775 Validation loss: 0.113183 Validation acc: 0.954444\n",
      "Epoch: 419/1000 Iteration: 3780 Train loss: 0.144396 Train acc: 0.955000\n",
      "Epoch: 420/1000 Iteration: 3785 Train loss: 0.133897 Train acc: 0.960000\n",
      "Epoch: 421/1000 Iteration: 3790 Train loss: 0.130204 Train acc: 0.960000\n",
      "Epoch: 421/1000 Iteration: 3795 Train loss: 0.135394 Train acc: 0.956667\n",
      "Epoch: 422/1000 Iteration: 3800 Train loss: 0.143808 Train acc: 0.958333\n",
      "Epoch: 422/1000 Iteration: 3800 Validation loss: 0.110854 Validation acc: 0.955000\n",
      "Epoch: 422/1000 Iteration: 3805 Train loss: 0.143337 Train acc: 0.948333\n",
      "Epoch: 423/1000 Iteration: 3810 Train loss: 0.167879 Train acc: 0.933333\n",
      "Epoch: 423/1000 Iteration: 3815 Train loss: 0.104308 Train acc: 0.971667\n",
      "Epoch: 424/1000 Iteration: 3820 Train loss: 0.143244 Train acc: 0.940000\n",
      "Epoch: 424/1000 Iteration: 3825 Train loss: 0.138084 Train acc: 0.960000\n",
      "Epoch: 424/1000 Iteration: 3825 Validation loss: 0.108477 Validation acc: 0.955555\n",
      "Epoch: 425/1000 Iteration: 3830 Train loss: 0.153905 Train acc: 0.950000\n",
      "Epoch: 426/1000 Iteration: 3835 Train loss: 0.122451 Train acc: 0.961667\n",
      "Epoch: 426/1000 Iteration: 3840 Train loss: 0.121641 Train acc: 0.965000\n",
      "Epoch: 427/1000 Iteration: 3845 Train loss: 0.133789 Train acc: 0.963333\n",
      "Epoch: 427/1000 Iteration: 3850 Train loss: 0.136897 Train acc: 0.965000\n",
      "Epoch: 427/1000 Iteration: 3850 Validation loss: 0.107493 Validation acc: 0.956667\n",
      "Epoch: 428/1000 Iteration: 3855 Train loss: 0.162526 Train acc: 0.943333\n",
      "Epoch: 428/1000 Iteration: 3860 Train loss: 0.107926 Train acc: 0.973333\n",
      "Epoch: 429/1000 Iteration: 3865 Train loss: 0.175370 Train acc: 0.940000\n",
      "Epoch: 429/1000 Iteration: 3870 Train loss: 0.134542 Train acc: 0.960000\n",
      "Epoch: 430/1000 Iteration: 3875 Train loss: 0.122158 Train acc: 0.956667\n",
      "Epoch: 430/1000 Iteration: 3875 Validation loss: 0.109388 Validation acc: 0.956111\n",
      "Epoch: 431/1000 Iteration: 3880 Train loss: 0.126667 Train acc: 0.958333\n",
      "Epoch: 431/1000 Iteration: 3885 Train loss: 0.120158 Train acc: 0.961667\n",
      "Epoch: 432/1000 Iteration: 3890 Train loss: 0.128629 Train acc: 0.960000\n",
      "Epoch: 432/1000 Iteration: 3895 Train loss: 0.137102 Train acc: 0.963333\n",
      "Epoch: 433/1000 Iteration: 3900 Train loss: 0.143360 Train acc: 0.948333\n",
      "Epoch: 433/1000 Iteration: 3900 Validation loss: 0.108982 Validation acc: 0.955555\n",
      "Epoch: 433/1000 Iteration: 3905 Train loss: 0.095293 Train acc: 0.971667\n",
      "Epoch: 434/1000 Iteration: 3910 Train loss: 0.159153 Train acc: 0.940000\n",
      "Epoch: 434/1000 Iteration: 3915 Train loss: 0.137690 Train acc: 0.953333\n",
      "Epoch: 435/1000 Iteration: 3920 Train loss: 0.141248 Train acc: 0.945000\n",
      "Epoch: 436/1000 Iteration: 3925 Train loss: 0.124193 Train acc: 0.963333\n",
      "Epoch: 436/1000 Iteration: 3925 Validation loss: 0.107695 Validation acc: 0.955555\n",
      "Epoch: 436/1000 Iteration: 3930 Train loss: 0.123097 Train acc: 0.956667\n",
      "Epoch: 437/1000 Iteration: 3935 Train loss: 0.130670 Train acc: 0.958333\n",
      "Epoch: 437/1000 Iteration: 3940 Train loss: 0.127421 Train acc: 0.960000\n",
      "Epoch: 438/1000 Iteration: 3945 Train loss: 0.161765 Train acc: 0.943333\n",
      "Epoch: 438/1000 Iteration: 3950 Train loss: 0.102344 Train acc: 0.971667\n",
      "Epoch: 438/1000 Iteration: 3950 Validation loss: 0.108437 Validation acc: 0.955000\n",
      "Epoch: 439/1000 Iteration: 3955 Train loss: 0.174206 Train acc: 0.936667\n",
      "Epoch: 439/1000 Iteration: 3960 Train loss: 0.133124 Train acc: 0.958333\n",
      "Epoch: 440/1000 Iteration: 3965 Train loss: 0.131729 Train acc: 0.948333\n",
      "Epoch: 441/1000 Iteration: 3970 Train loss: 0.129106 Train acc: 0.946667\n",
      "Epoch: 441/1000 Iteration: 3975 Train loss: 0.121073 Train acc: 0.955000\n",
      "Epoch: 441/1000 Iteration: 3975 Validation loss: 0.109502 Validation acc: 0.955555\n",
      "Epoch: 442/1000 Iteration: 3980 Train loss: 0.135947 Train acc: 0.965000\n",
      "Epoch: 442/1000 Iteration: 3985 Train loss: 0.114016 Train acc: 0.965000\n",
      "Epoch: 443/1000 Iteration: 3990 Train loss: 0.153510 Train acc: 0.940000\n",
      "Epoch: 443/1000 Iteration: 3995 Train loss: 0.107582 Train acc: 0.965000\n",
      "Epoch: 444/1000 Iteration: 4000 Train loss: 0.157788 Train acc: 0.945000\n",
      "Epoch: 444/1000 Iteration: 4000 Validation loss: 0.108380 Validation acc: 0.955000\n",
      "Epoch: 444/1000 Iteration: 4005 Train loss: 0.135270 Train acc: 0.958333\n",
      "Epoch: 445/1000 Iteration: 4010 Train loss: 0.128068 Train acc: 0.956667\n",
      "Epoch: 446/1000 Iteration: 4015 Train loss: 0.108387 Train acc: 0.973333\n",
      "Epoch: 446/1000 Iteration: 4020 Train loss: 0.110802 Train acc: 0.960000\n",
      "Epoch: 447/1000 Iteration: 4025 Train loss: 0.119463 Train acc: 0.965000\n",
      "Epoch: 447/1000 Iteration: 4025 Validation loss: 0.106974 Validation acc: 0.957778\n",
      "Epoch: 447/1000 Iteration: 4030 Train loss: 0.134521 Train acc: 0.960000\n",
      "Epoch: 448/1000 Iteration: 4035 Train loss: 0.194448 Train acc: 0.930000\n",
      "Epoch: 448/1000 Iteration: 4040 Train loss: 0.126746 Train acc: 0.960000\n",
      "Epoch: 449/1000 Iteration: 4045 Train loss: 0.173137 Train acc: 0.928333\n",
      "Epoch: 449/1000 Iteration: 4050 Train loss: 0.150682 Train acc: 0.960000\n",
      "Epoch: 449/1000 Iteration: 4050 Validation loss: 0.105464 Validation acc: 0.954444\n",
      "Epoch: 450/1000 Iteration: 4055 Train loss: 0.137048 Train acc: 0.948333\n",
      "Epoch: 451/1000 Iteration: 4060 Train loss: 0.101211 Train acc: 0.976667\n",
      "Epoch: 451/1000 Iteration: 4065 Train loss: 0.123993 Train acc: 0.955000\n",
      "Epoch: 452/1000 Iteration: 4070 Train loss: 0.128061 Train acc: 0.963333\n",
      "Epoch: 452/1000 Iteration: 4075 Train loss: 0.127906 Train acc: 0.956667\n",
      "Epoch: 452/1000 Iteration: 4075 Validation loss: 0.103351 Validation acc: 0.957222\n",
      "Epoch: 453/1000 Iteration: 4080 Train loss: 0.144236 Train acc: 0.941667\n",
      "Epoch: 453/1000 Iteration: 4085 Train loss: 0.099236 Train acc: 0.963333\n",
      "Epoch: 454/1000 Iteration: 4090 Train loss: 0.144431 Train acc: 0.945000\n",
      "Epoch: 454/1000 Iteration: 4095 Train loss: 0.134420 Train acc: 0.956667\n",
      "Epoch: 455/1000 Iteration: 4100 Train loss: 0.132587 Train acc: 0.955000\n",
      "Epoch: 455/1000 Iteration: 4100 Validation loss: 0.102729 Validation acc: 0.956667\n",
      "Epoch: 456/1000 Iteration: 4105 Train loss: 0.111285 Train acc: 0.968333\n",
      "Epoch: 456/1000 Iteration: 4110 Train loss: 0.120991 Train acc: 0.966667\n",
      "Epoch: 457/1000 Iteration: 4115 Train loss: 0.118051 Train acc: 0.968333\n",
      "Epoch: 457/1000 Iteration: 4120 Train loss: 0.114363 Train acc: 0.960000\n",
      "Epoch: 458/1000 Iteration: 4125 Train loss: 0.148973 Train acc: 0.945000\n",
      "Epoch: 458/1000 Iteration: 4125 Validation loss: 0.105519 Validation acc: 0.956667\n",
      "Epoch: 458/1000 Iteration: 4130 Train loss: 0.104855 Train acc: 0.968333\n",
      "Epoch: 459/1000 Iteration: 4135 Train loss: 0.133542 Train acc: 0.943333\n",
      "Epoch: 459/1000 Iteration: 4140 Train loss: 0.133837 Train acc: 0.951667\n",
      "Epoch: 460/1000 Iteration: 4145 Train loss: 0.132288 Train acc: 0.958333\n",
      "Epoch: 461/1000 Iteration: 4150 Train loss: 0.110281 Train acc: 0.958333\n",
      "Epoch: 461/1000 Iteration: 4150 Validation loss: 0.103605 Validation acc: 0.958333\n",
      "Epoch: 461/1000 Iteration: 4155 Train loss: 0.106726 Train acc: 0.968333\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 462/1000 Iteration: 4160 Train loss: 0.122656 Train acc: 0.966667\n",
      "Epoch: 462/1000 Iteration: 4165 Train loss: 0.125956 Train acc: 0.963333\n",
      "Epoch: 463/1000 Iteration: 4170 Train loss: 0.142622 Train acc: 0.951667\n",
      "Epoch: 463/1000 Iteration: 4175 Train loss: 0.108908 Train acc: 0.963333\n",
      "Epoch: 463/1000 Iteration: 4175 Validation loss: 0.103420 Validation acc: 0.957222\n",
      "Epoch: 464/1000 Iteration: 4180 Train loss: 0.140689 Train acc: 0.951667\n",
      "Epoch: 464/1000 Iteration: 4185 Train loss: 0.113082 Train acc: 0.958333\n",
      "Epoch: 465/1000 Iteration: 4190 Train loss: 0.139829 Train acc: 0.951667\n",
      "Epoch: 466/1000 Iteration: 4195 Train loss: 0.126063 Train acc: 0.955000\n",
      "Epoch: 466/1000 Iteration: 4200 Train loss: 0.127992 Train acc: 0.950000\n",
      "Epoch: 466/1000 Iteration: 4200 Validation loss: 0.106367 Validation acc: 0.956111\n",
      "Epoch: 467/1000 Iteration: 4205 Train loss: 0.133380 Train acc: 0.960000\n",
      "Epoch: 467/1000 Iteration: 4210 Train loss: 0.138388 Train acc: 0.963333\n",
      "Epoch: 468/1000 Iteration: 4215 Train loss: 0.151882 Train acc: 0.945000\n",
      "Epoch: 468/1000 Iteration: 4220 Train loss: 0.104989 Train acc: 0.968333\n",
      "Epoch: 469/1000 Iteration: 4225 Train loss: 0.148895 Train acc: 0.943333\n",
      "Epoch: 469/1000 Iteration: 4225 Validation loss: 0.104196 Validation acc: 0.958333\n",
      "Epoch: 469/1000 Iteration: 4230 Train loss: 0.142759 Train acc: 0.950000\n",
      "Epoch: 470/1000 Iteration: 4235 Train loss: 0.126279 Train acc: 0.956667\n",
      "Epoch: 471/1000 Iteration: 4240 Train loss: 0.110986 Train acc: 0.966667\n",
      "Epoch: 471/1000 Iteration: 4245 Train loss: 0.121700 Train acc: 0.960000\n",
      "Epoch: 472/1000 Iteration: 4250 Train loss: 0.120803 Train acc: 0.961667\n",
      "Epoch: 472/1000 Iteration: 4250 Validation loss: 0.109084 Validation acc: 0.957778\n",
      "Epoch: 472/1000 Iteration: 4255 Train loss: 0.124329 Train acc: 0.966667\n",
      "Epoch: 473/1000 Iteration: 4260 Train loss: 0.149695 Train acc: 0.948333\n",
      "Epoch: 473/1000 Iteration: 4265 Train loss: 0.104207 Train acc: 0.963333\n",
      "Epoch: 474/1000 Iteration: 4270 Train loss: 0.138892 Train acc: 0.945000\n",
      "Epoch: 474/1000 Iteration: 4275 Train loss: 0.123098 Train acc: 0.960000\n",
      "Epoch: 474/1000 Iteration: 4275 Validation loss: 0.103413 Validation acc: 0.957222\n",
      "Epoch: 475/1000 Iteration: 4280 Train loss: 0.128538 Train acc: 0.946667\n",
      "Epoch: 476/1000 Iteration: 4285 Train loss: 0.117089 Train acc: 0.960000\n",
      "Epoch: 476/1000 Iteration: 4290 Train loss: 0.107171 Train acc: 0.966667\n",
      "Epoch: 477/1000 Iteration: 4295 Train loss: 0.130165 Train acc: 0.960000\n",
      "Epoch: 477/1000 Iteration: 4300 Train loss: 0.130609 Train acc: 0.961667\n",
      "Epoch: 477/1000 Iteration: 4300 Validation loss: 0.106708 Validation acc: 0.957222\n",
      "Epoch: 478/1000 Iteration: 4305 Train loss: 0.144479 Train acc: 0.955000\n",
      "Epoch: 478/1000 Iteration: 4310 Train loss: 0.100066 Train acc: 0.963333\n",
      "Epoch: 479/1000 Iteration: 4315 Train loss: 0.148116 Train acc: 0.941667\n",
      "Epoch: 479/1000 Iteration: 4320 Train loss: 0.127835 Train acc: 0.958333\n",
      "Epoch: 480/1000 Iteration: 4325 Train loss: 0.127355 Train acc: 0.965000\n",
      "Epoch: 480/1000 Iteration: 4325 Validation loss: 0.103811 Validation acc: 0.957222\n",
      "Epoch: 481/1000 Iteration: 4330 Train loss: 0.109369 Train acc: 0.966667\n",
      "Epoch: 481/1000 Iteration: 4335 Train loss: 0.103798 Train acc: 0.960000\n",
      "Epoch: 482/1000 Iteration: 4340 Train loss: 0.118395 Train acc: 0.961667\n",
      "Epoch: 482/1000 Iteration: 4345 Train loss: 0.137152 Train acc: 0.965000\n",
      "Epoch: 483/1000 Iteration: 4350 Train loss: 0.148991 Train acc: 0.955000\n",
      "Epoch: 483/1000 Iteration: 4350 Validation loss: 0.104330 Validation acc: 0.958333\n",
      "Epoch: 483/1000 Iteration: 4355 Train loss: 0.086216 Train acc: 0.971667\n",
      "Epoch: 484/1000 Iteration: 4360 Train loss: 0.153550 Train acc: 0.941667\n",
      "Epoch: 484/1000 Iteration: 4365 Train loss: 0.125065 Train acc: 0.958333\n",
      "Epoch: 485/1000 Iteration: 4370 Train loss: 0.117984 Train acc: 0.956667\n",
      "Epoch: 486/1000 Iteration: 4375 Train loss: 0.120677 Train acc: 0.966667\n",
      "Epoch: 486/1000 Iteration: 4375 Validation loss: 0.104626 Validation acc: 0.957778\n",
      "Epoch: 486/1000 Iteration: 4380 Train loss: 0.116560 Train acc: 0.970000\n",
      "Epoch: 487/1000 Iteration: 4385 Train loss: 0.126475 Train acc: 0.958333\n",
      "Epoch: 487/1000 Iteration: 4390 Train loss: 0.111617 Train acc: 0.971667\n",
      "Epoch: 488/1000 Iteration: 4395 Train loss: 0.169380 Train acc: 0.948333\n",
      "Epoch: 488/1000 Iteration: 4400 Train loss: 0.107127 Train acc: 0.966667\n",
      "Epoch: 488/1000 Iteration: 4400 Validation loss: 0.104003 Validation acc: 0.958333\n",
      "Epoch: 489/1000 Iteration: 4405 Train loss: 0.136283 Train acc: 0.950000\n",
      "Epoch: 489/1000 Iteration: 4410 Train loss: 0.128505 Train acc: 0.960000\n",
      "Epoch: 490/1000 Iteration: 4415 Train loss: 0.118918 Train acc: 0.956667\n",
      "Epoch: 491/1000 Iteration: 4420 Train loss: 0.112452 Train acc: 0.951667\n",
      "Epoch: 491/1000 Iteration: 4425 Train loss: 0.113821 Train acc: 0.958333\n",
      "Epoch: 491/1000 Iteration: 4425 Validation loss: 0.103421 Validation acc: 0.957222\n",
      "Epoch: 492/1000 Iteration: 4430 Train loss: 0.132205 Train acc: 0.965000\n",
      "Epoch: 492/1000 Iteration: 4435 Train loss: 0.115394 Train acc: 0.961667\n",
      "Epoch: 493/1000 Iteration: 4440 Train loss: 0.162268 Train acc: 0.951667\n",
      "Epoch: 493/1000 Iteration: 4445 Train loss: 0.092206 Train acc: 0.971667\n",
      "Epoch: 494/1000 Iteration: 4450 Train loss: 0.141315 Train acc: 0.943333\n",
      "Epoch: 494/1000 Iteration: 4450 Validation loss: 0.102692 Validation acc: 0.957778\n",
      "Epoch: 494/1000 Iteration: 4455 Train loss: 0.136635 Train acc: 0.953333\n",
      "Epoch: 495/1000 Iteration: 4460 Train loss: 0.125331 Train acc: 0.955000\n",
      "Epoch: 496/1000 Iteration: 4465 Train loss: 0.123025 Train acc: 0.960000\n",
      "Epoch: 496/1000 Iteration: 4470 Train loss: 0.102206 Train acc: 0.968333\n",
      "Epoch: 497/1000 Iteration: 4475 Train loss: 0.132954 Train acc: 0.960000\n",
      "Epoch: 497/1000 Iteration: 4475 Validation loss: 0.106004 Validation acc: 0.956667\n",
      "Epoch: 497/1000 Iteration: 4480 Train loss: 0.129366 Train acc: 0.965000\n",
      "Epoch: 498/1000 Iteration: 4485 Train loss: 0.155794 Train acc: 0.948333\n",
      "Epoch: 498/1000 Iteration: 4490 Train loss: 0.090141 Train acc: 0.978333\n",
      "Epoch: 499/1000 Iteration: 4495 Train loss: 0.135859 Train acc: 0.943333\n",
      "Epoch: 499/1000 Iteration: 4500 Train loss: 0.119340 Train acc: 0.955000\n",
      "Epoch: 499/1000 Iteration: 4500 Validation loss: 0.104479 Validation acc: 0.958889\n",
      "Epoch: 500/1000 Iteration: 4505 Train loss: 0.132055 Train acc: 0.951667\n",
      "Epoch: 501/1000 Iteration: 4510 Train loss: 0.104738 Train acc: 0.963333\n",
      "Epoch: 501/1000 Iteration: 4515 Train loss: 0.108526 Train acc: 0.966667\n",
      "Epoch: 502/1000 Iteration: 4520 Train loss: 0.134647 Train acc: 0.958333\n",
      "Epoch: 502/1000 Iteration: 4525 Train loss: 0.116800 Train acc: 0.963333\n",
      "Epoch: 502/1000 Iteration: 4525 Validation loss: 0.105181 Validation acc: 0.957222\n",
      "Epoch: 503/1000 Iteration: 4530 Train loss: 0.148209 Train acc: 0.955000\n",
      "Epoch: 503/1000 Iteration: 4535 Train loss: 0.094782 Train acc: 0.970000\n",
      "Epoch: 504/1000 Iteration: 4540 Train loss: 0.132916 Train acc: 0.945000\n",
      "Epoch: 504/1000 Iteration: 4545 Train loss: 0.123211 Train acc: 0.963333\n",
      "Epoch: 505/1000 Iteration: 4550 Train loss: 0.131156 Train acc: 0.955000\n",
      "Epoch: 505/1000 Iteration: 4550 Validation loss: 0.103580 Validation acc: 0.958889\n",
      "Epoch: 506/1000 Iteration: 4555 Train loss: 0.112315 Train acc: 0.955000\n",
      "Epoch: 506/1000 Iteration: 4560 Train loss: 0.093220 Train acc: 0.971667\n",
      "Epoch: 507/1000 Iteration: 4565 Train loss: 0.121949 Train acc: 0.965000\n",
      "Epoch: 507/1000 Iteration: 4570 Train loss: 0.125535 Train acc: 0.960000\n",
      "Epoch: 508/1000 Iteration: 4575 Train loss: 0.144484 Train acc: 0.950000\n",
      "Epoch: 508/1000 Iteration: 4575 Validation loss: 0.106500 Validation acc: 0.958333\n",
      "Epoch: 508/1000 Iteration: 4580 Train loss: 0.091991 Train acc: 0.971667\n",
      "Epoch: 509/1000 Iteration: 4585 Train loss: 0.153420 Train acc: 0.941667\n",
      "Epoch: 509/1000 Iteration: 4590 Train loss: 0.118823 Train acc: 0.970000\n",
      "Epoch: 510/1000 Iteration: 4595 Train loss: 0.124508 Train acc: 0.955000\n",
      "Epoch: 511/1000 Iteration: 4600 Train loss: 0.107375 Train acc: 0.961667\n",
      "Epoch: 511/1000 Iteration: 4600 Validation loss: 0.107102 Validation acc: 0.957222\n",
      "Epoch: 511/1000 Iteration: 4605 Train loss: 0.094448 Train acc: 0.968333\n",
      "Epoch: 512/1000 Iteration: 4610 Train loss: 0.127472 Train acc: 0.958333\n",
      "Epoch: 512/1000 Iteration: 4615 Train loss: 0.110304 Train acc: 0.965000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 513/1000 Iteration: 4620 Train loss: 0.148825 Train acc: 0.945000\n",
      "Epoch: 513/1000 Iteration: 4625 Train loss: 0.090833 Train acc: 0.975000\n",
      "Epoch: 513/1000 Iteration: 4625 Validation loss: 0.102922 Validation acc: 0.960556\n",
      "Epoch: 514/1000 Iteration: 4630 Train loss: 0.142116 Train acc: 0.948333\n",
      "Epoch: 514/1000 Iteration: 4635 Train loss: 0.123836 Train acc: 0.960000\n",
      "Epoch: 515/1000 Iteration: 4640 Train loss: 0.120939 Train acc: 0.953333\n",
      "Epoch: 516/1000 Iteration: 4645 Train loss: 0.105833 Train acc: 0.966667\n",
      "Epoch: 516/1000 Iteration: 4650 Train loss: 0.110317 Train acc: 0.960000\n",
      "Epoch: 516/1000 Iteration: 4650 Validation loss: 0.100383 Validation acc: 0.959444\n",
      "Epoch: 517/1000 Iteration: 4655 Train loss: 0.118794 Train acc: 0.966667\n",
      "Epoch: 517/1000 Iteration: 4660 Train loss: 0.110561 Train acc: 0.965000\n",
      "Epoch: 518/1000 Iteration: 4665 Train loss: 0.158074 Train acc: 0.941667\n",
      "Epoch: 518/1000 Iteration: 4670 Train loss: 0.090057 Train acc: 0.971667\n",
      "Epoch: 519/1000 Iteration: 4675 Train loss: 0.135260 Train acc: 0.943333\n",
      "Epoch: 519/1000 Iteration: 4675 Validation loss: 0.105273 Validation acc: 0.959444\n",
      "Epoch: 519/1000 Iteration: 4680 Train loss: 0.115072 Train acc: 0.963333\n",
      "Epoch: 520/1000 Iteration: 4685 Train loss: 0.114013 Train acc: 0.963333\n",
      "Epoch: 521/1000 Iteration: 4690 Train loss: 0.099140 Train acc: 0.961667\n",
      "Epoch: 521/1000 Iteration: 4695 Train loss: 0.104815 Train acc: 0.966667\n",
      "Epoch: 522/1000 Iteration: 4700 Train loss: 0.118103 Train acc: 0.963333\n",
      "Epoch: 522/1000 Iteration: 4700 Validation loss: 0.109200 Validation acc: 0.956111\n",
      "Epoch: 522/1000 Iteration: 4705 Train loss: 0.123861 Train acc: 0.960000\n",
      "Epoch: 523/1000 Iteration: 4710 Train loss: 0.137255 Train acc: 0.955000\n",
      "Epoch: 523/1000 Iteration: 4715 Train loss: 0.089685 Train acc: 0.978333\n",
      "Epoch: 524/1000 Iteration: 4720 Train loss: 0.134745 Train acc: 0.956667\n",
      "Epoch: 524/1000 Iteration: 4725 Train loss: 0.125301 Train acc: 0.956667\n",
      "Epoch: 524/1000 Iteration: 4725 Validation loss: 0.104922 Validation acc: 0.959444\n",
      "Epoch: 525/1000 Iteration: 4730 Train loss: 0.125149 Train acc: 0.951667\n",
      "Epoch: 526/1000 Iteration: 4735 Train loss: 0.101907 Train acc: 0.963333\n",
      "Epoch: 526/1000 Iteration: 4740 Train loss: 0.109214 Train acc: 0.963333\n",
      "Epoch: 527/1000 Iteration: 4745 Train loss: 0.130742 Train acc: 0.963333\n",
      "Epoch: 527/1000 Iteration: 4750 Train loss: 0.120484 Train acc: 0.968333\n",
      "Epoch: 527/1000 Iteration: 4750 Validation loss: 0.103639 Validation acc: 0.958889\n",
      "Epoch: 528/1000 Iteration: 4755 Train loss: 0.123073 Train acc: 0.958333\n",
      "Epoch: 528/1000 Iteration: 4760 Train loss: 0.096458 Train acc: 0.968333\n",
      "Epoch: 529/1000 Iteration: 4765 Train loss: 0.153548 Train acc: 0.936667\n",
      "Epoch: 529/1000 Iteration: 4770 Train loss: 0.131271 Train acc: 0.961667\n",
      "Epoch: 530/1000 Iteration: 4775 Train loss: 0.124599 Train acc: 0.948333\n",
      "Epoch: 530/1000 Iteration: 4775 Validation loss: 0.105011 Validation acc: 0.958889\n",
      "Epoch: 531/1000 Iteration: 4780 Train loss: 0.115607 Train acc: 0.956667\n",
      "Epoch: 531/1000 Iteration: 4785 Train loss: 0.106157 Train acc: 0.966667\n",
      "Epoch: 532/1000 Iteration: 4790 Train loss: 0.116106 Train acc: 0.970000\n",
      "Epoch: 532/1000 Iteration: 4795 Train loss: 0.113917 Train acc: 0.968333\n",
      "Epoch: 533/1000 Iteration: 4800 Train loss: 0.140395 Train acc: 0.955000\n",
      "Epoch: 533/1000 Iteration: 4800 Validation loss: 0.103492 Validation acc: 0.958889\n",
      "Epoch: 533/1000 Iteration: 4805 Train loss: 0.080722 Train acc: 0.978333\n",
      "Epoch: 534/1000 Iteration: 4810 Train loss: 0.135309 Train acc: 0.946667\n",
      "Epoch: 534/1000 Iteration: 4815 Train loss: 0.112079 Train acc: 0.963333\n",
      "Epoch: 535/1000 Iteration: 4820 Train loss: 0.119323 Train acc: 0.956667\n",
      "Epoch: 536/1000 Iteration: 4825 Train loss: 0.107645 Train acc: 0.958333\n",
      "Epoch: 536/1000 Iteration: 4825 Validation loss: 0.103783 Validation acc: 0.958333\n",
      "Epoch: 536/1000 Iteration: 4830 Train loss: 0.099660 Train acc: 0.968333\n",
      "Epoch: 537/1000 Iteration: 4835 Train loss: 0.130465 Train acc: 0.965000\n",
      "Epoch: 537/1000 Iteration: 4840 Train loss: 0.112889 Train acc: 0.966667\n",
      "Epoch: 538/1000 Iteration: 4845 Train loss: 0.139522 Train acc: 0.953333\n",
      "Epoch: 538/1000 Iteration: 4850 Train loss: 0.082501 Train acc: 0.966667\n",
      "Epoch: 538/1000 Iteration: 4850 Validation loss: 0.103237 Validation acc: 0.960556\n",
      "Epoch: 539/1000 Iteration: 4855 Train loss: 0.129328 Train acc: 0.945000\n",
      "Epoch: 539/1000 Iteration: 4860 Train loss: 0.113847 Train acc: 0.960000\n",
      "Epoch: 540/1000 Iteration: 4865 Train loss: 0.122475 Train acc: 0.956667\n",
      "Epoch: 541/1000 Iteration: 4870 Train loss: 0.099304 Train acc: 0.973333\n",
      "Epoch: 541/1000 Iteration: 4875 Train loss: 0.097330 Train acc: 0.971667\n",
      "Epoch: 541/1000 Iteration: 4875 Validation loss: 0.105547 Validation acc: 0.956667\n",
      "Epoch: 542/1000 Iteration: 4880 Train loss: 0.120109 Train acc: 0.960000\n",
      "Epoch: 542/1000 Iteration: 4885 Train loss: 0.100424 Train acc: 0.966667\n",
      "Epoch: 543/1000 Iteration: 4890 Train loss: 0.130857 Train acc: 0.946667\n",
      "Epoch: 543/1000 Iteration: 4895 Train loss: 0.096009 Train acc: 0.966667\n",
      "Epoch: 544/1000 Iteration: 4900 Train loss: 0.173815 Train acc: 0.938333\n",
      "Epoch: 544/1000 Iteration: 4900 Validation loss: 0.108712 Validation acc: 0.958333\n",
      "Epoch: 544/1000 Iteration: 4905 Train loss: 0.119349 Train acc: 0.965000\n",
      "Epoch: 545/1000 Iteration: 4910 Train loss: 0.119315 Train acc: 0.958333\n",
      "Epoch: 546/1000 Iteration: 4915 Train loss: 0.093721 Train acc: 0.975000\n",
      "Epoch: 546/1000 Iteration: 4920 Train loss: 0.109749 Train acc: 0.963333\n",
      "Epoch: 547/1000 Iteration: 4925 Train loss: 0.101225 Train acc: 0.968333\n",
      "Epoch: 547/1000 Iteration: 4925 Validation loss: 0.101965 Validation acc: 0.962778\n",
      "Epoch: 547/1000 Iteration: 4930 Train loss: 0.113608 Train acc: 0.966667\n",
      "Epoch: 548/1000 Iteration: 4935 Train loss: 0.154065 Train acc: 0.953333\n",
      "Epoch: 548/1000 Iteration: 4940 Train loss: 0.084597 Train acc: 0.973333\n",
      "Epoch: 549/1000 Iteration: 4945 Train loss: 0.145551 Train acc: 0.940000\n",
      "Epoch: 549/1000 Iteration: 4950 Train loss: 0.111309 Train acc: 0.955000\n",
      "Epoch: 549/1000 Iteration: 4950 Validation loss: 0.104352 Validation acc: 0.961667\n",
      "Epoch: 550/1000 Iteration: 4955 Train loss: 0.121274 Train acc: 0.948333\n",
      "Epoch: 551/1000 Iteration: 4960 Train loss: 0.106658 Train acc: 0.963333\n",
      "Epoch: 551/1000 Iteration: 4965 Train loss: 0.108281 Train acc: 0.956667\n",
      "Epoch: 552/1000 Iteration: 4970 Train loss: 0.111847 Train acc: 0.963333\n",
      "Epoch: 552/1000 Iteration: 4975 Train loss: 0.106616 Train acc: 0.965000\n",
      "Epoch: 552/1000 Iteration: 4975 Validation loss: 0.104666 Validation acc: 0.961111\n",
      "Epoch: 553/1000 Iteration: 4980 Train loss: 0.136967 Train acc: 0.950000\n",
      "Epoch: 553/1000 Iteration: 4985 Train loss: 0.086061 Train acc: 0.976667\n",
      "Epoch: 554/1000 Iteration: 4990 Train loss: 0.150484 Train acc: 0.941667\n",
      "Epoch: 554/1000 Iteration: 4995 Train loss: 0.122261 Train acc: 0.961667\n",
      "Epoch: 555/1000 Iteration: 5000 Train loss: 0.142281 Train acc: 0.951667\n",
      "Epoch: 555/1000 Iteration: 5000 Validation loss: 0.103355 Validation acc: 0.961111\n",
      "Epoch: 556/1000 Iteration: 5005 Train loss: 0.114428 Train acc: 0.961667\n",
      "Epoch: 556/1000 Iteration: 5010 Train loss: 0.100449 Train acc: 0.966667\n",
      "Epoch: 557/1000 Iteration: 5015 Train loss: 0.127785 Train acc: 0.965000\n",
      "Epoch: 557/1000 Iteration: 5020 Train loss: 0.115510 Train acc: 0.958333\n",
      "Epoch: 558/1000 Iteration: 5025 Train loss: 0.144891 Train acc: 0.948333\n",
      "Epoch: 558/1000 Iteration: 5025 Validation loss: 0.102907 Validation acc: 0.960000\n",
      "Epoch: 558/1000 Iteration: 5030 Train loss: 0.101219 Train acc: 0.971667\n",
      "Epoch: 559/1000 Iteration: 5035 Train loss: 0.131628 Train acc: 0.946667\n",
      "Epoch: 559/1000 Iteration: 5040 Train loss: 0.113250 Train acc: 0.966667\n",
      "Epoch: 560/1000 Iteration: 5045 Train loss: 0.123128 Train acc: 0.953333\n",
      "Epoch: 561/1000 Iteration: 5050 Train loss: 0.116314 Train acc: 0.953333\n",
      "Epoch: 561/1000 Iteration: 5050 Validation loss: 0.103521 Validation acc: 0.961111\n",
      "Epoch: 561/1000 Iteration: 5055 Train loss: 0.097366 Train acc: 0.965000\n",
      "Epoch: 562/1000 Iteration: 5060 Train loss: 0.120743 Train acc: 0.970000\n",
      "Epoch: 562/1000 Iteration: 5065 Train loss: 0.102145 Train acc: 0.971667\n",
      "Epoch: 563/1000 Iteration: 5070 Train loss: 0.131940 Train acc: 0.955000\n",
      "Epoch: 563/1000 Iteration: 5075 Train loss: 0.077163 Train acc: 0.978333\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 563/1000 Iteration: 5075 Validation loss: 0.105176 Validation acc: 0.958889\n",
      "Epoch: 564/1000 Iteration: 5080 Train loss: 0.131385 Train acc: 0.946667\n",
      "Epoch: 564/1000 Iteration: 5085 Train loss: 0.108637 Train acc: 0.966667\n",
      "Epoch: 565/1000 Iteration: 5090 Train loss: 0.123450 Train acc: 0.953333\n",
      "Epoch: 566/1000 Iteration: 5095 Train loss: 0.095356 Train acc: 0.971667\n",
      "Epoch: 566/1000 Iteration: 5100 Train loss: 0.087739 Train acc: 0.968333\n",
      "Epoch: 566/1000 Iteration: 5100 Validation loss: 0.103831 Validation acc: 0.959444\n",
      "Epoch: 567/1000 Iteration: 5105 Train loss: 0.113428 Train acc: 0.961667\n",
      "Epoch: 567/1000 Iteration: 5110 Train loss: 0.117566 Train acc: 0.960000\n",
      "Epoch: 568/1000 Iteration: 5115 Train loss: 0.141562 Train acc: 0.950000\n",
      "Epoch: 568/1000 Iteration: 5120 Train loss: 0.075005 Train acc: 0.978333\n",
      "Epoch: 569/1000 Iteration: 5125 Train loss: 0.128583 Train acc: 0.956667\n",
      "Epoch: 569/1000 Iteration: 5125 Validation loss: 0.102913 Validation acc: 0.959444\n",
      "Epoch: 569/1000 Iteration: 5130 Train loss: 0.124191 Train acc: 0.968333\n",
      "Epoch: 570/1000 Iteration: 5135 Train loss: 0.113095 Train acc: 0.960000\n",
      "Epoch: 571/1000 Iteration: 5140 Train loss: 0.095604 Train acc: 0.965000\n",
      "Epoch: 571/1000 Iteration: 5145 Train loss: 0.093006 Train acc: 0.965000\n",
      "Epoch: 572/1000 Iteration: 5150 Train loss: 0.108419 Train acc: 0.970000\n",
      "Epoch: 572/1000 Iteration: 5150 Validation loss: 0.102196 Validation acc: 0.960000\n",
      "Epoch: 572/1000 Iteration: 5155 Train loss: 0.109676 Train acc: 0.963333\n",
      "Epoch: 573/1000 Iteration: 5160 Train loss: 0.131118 Train acc: 0.956667\n",
      "Epoch: 573/1000 Iteration: 5165 Train loss: 0.082516 Train acc: 0.973333\n",
      "Epoch: 574/1000 Iteration: 5170 Train loss: 0.126608 Train acc: 0.945000\n",
      "Epoch: 574/1000 Iteration: 5175 Train loss: 0.125781 Train acc: 0.965000\n",
      "Epoch: 574/1000 Iteration: 5175 Validation loss: 0.102334 Validation acc: 0.960000\n",
      "Epoch: 575/1000 Iteration: 5180 Train loss: 0.112550 Train acc: 0.960000\n",
      "Epoch: 576/1000 Iteration: 5185 Train loss: 0.104040 Train acc: 0.960000\n",
      "Epoch: 576/1000 Iteration: 5190 Train loss: 0.093368 Train acc: 0.968333\n",
      "Epoch: 577/1000 Iteration: 5195 Train loss: 0.101302 Train acc: 0.966667\n",
      "Epoch: 577/1000 Iteration: 5200 Train loss: 0.101076 Train acc: 0.968333\n",
      "Epoch: 577/1000 Iteration: 5200 Validation loss: 0.102376 Validation acc: 0.960556\n",
      "Epoch: 578/1000 Iteration: 5205 Train loss: 0.137504 Train acc: 0.951667\n",
      "Epoch: 578/1000 Iteration: 5210 Train loss: 0.093581 Train acc: 0.965000\n",
      "Epoch: 579/1000 Iteration: 5215 Train loss: 0.135555 Train acc: 0.950000\n",
      "Epoch: 579/1000 Iteration: 5220 Train loss: 0.111860 Train acc: 0.968333\n",
      "Epoch: 580/1000 Iteration: 5225 Train loss: 0.117754 Train acc: 0.960000\n",
      "Epoch: 580/1000 Iteration: 5225 Validation loss: 0.101973 Validation acc: 0.960556\n",
      "Epoch: 581/1000 Iteration: 5230 Train loss: 0.086282 Train acc: 0.966667\n",
      "Epoch: 581/1000 Iteration: 5235 Train loss: 0.101959 Train acc: 0.965000\n",
      "Epoch: 582/1000 Iteration: 5240 Train loss: 0.109730 Train acc: 0.965000\n",
      "Epoch: 582/1000 Iteration: 5245 Train loss: 0.093488 Train acc: 0.973333\n",
      "Epoch: 583/1000 Iteration: 5250 Train loss: 0.147737 Train acc: 0.958333\n",
      "Epoch: 583/1000 Iteration: 5250 Validation loss: 0.102753 Validation acc: 0.961111\n",
      "Epoch: 583/1000 Iteration: 5255 Train loss: 0.086636 Train acc: 0.971667\n",
      "Epoch: 584/1000 Iteration: 5260 Train loss: 0.135827 Train acc: 0.946667\n",
      "Epoch: 584/1000 Iteration: 5265 Train loss: 0.105245 Train acc: 0.968333\n",
      "Epoch: 585/1000 Iteration: 5270 Train loss: 0.118467 Train acc: 0.953333\n",
      "Epoch: 586/1000 Iteration: 5275 Train loss: 0.094486 Train acc: 0.963333\n",
      "Epoch: 586/1000 Iteration: 5275 Validation loss: 0.105046 Validation acc: 0.958889\n",
      "Epoch: 586/1000 Iteration: 5280 Train loss: 0.090638 Train acc: 0.968333\n",
      "Epoch: 587/1000 Iteration: 5285 Train loss: 0.118581 Train acc: 0.958333\n",
      "Epoch: 587/1000 Iteration: 5290 Train loss: 0.094644 Train acc: 0.973333\n",
      "Epoch: 588/1000 Iteration: 5295 Train loss: 0.132465 Train acc: 0.953333\n",
      "Epoch: 588/1000 Iteration: 5300 Train loss: 0.076790 Train acc: 0.978333\n",
      "Epoch: 588/1000 Iteration: 5300 Validation loss: 0.104640 Validation acc: 0.958889\n",
      "Epoch: 589/1000 Iteration: 5305 Train loss: 0.133296 Train acc: 0.951667\n",
      "Epoch: 589/1000 Iteration: 5310 Train loss: 0.133466 Train acc: 0.956667\n",
      "Epoch: 590/1000 Iteration: 5315 Train loss: 0.111574 Train acc: 0.953333\n",
      "Epoch: 591/1000 Iteration: 5320 Train loss: 0.109881 Train acc: 0.960000\n",
      "Epoch: 591/1000 Iteration: 5325 Train loss: 0.090258 Train acc: 0.970000\n",
      "Epoch: 591/1000 Iteration: 5325 Validation loss: 0.102571 Validation acc: 0.960000\n",
      "Epoch: 592/1000 Iteration: 5330 Train loss: 0.119488 Train acc: 0.965000\n",
      "Epoch: 592/1000 Iteration: 5335 Train loss: 0.110581 Train acc: 0.955000\n",
      "Epoch: 593/1000 Iteration: 5340 Train loss: 0.125806 Train acc: 0.958333\n",
      "Epoch: 593/1000 Iteration: 5345 Train loss: 0.085311 Train acc: 0.978333\n",
      "Epoch: 594/1000 Iteration: 5350 Train loss: 0.143342 Train acc: 0.948333\n",
      "Epoch: 594/1000 Iteration: 5350 Validation loss: 0.104456 Validation acc: 0.959444\n",
      "Epoch: 594/1000 Iteration: 5355 Train loss: 0.122122 Train acc: 0.966667\n",
      "Epoch: 595/1000 Iteration: 5360 Train loss: 0.101240 Train acc: 0.970000\n",
      "Epoch: 596/1000 Iteration: 5365 Train loss: 0.098538 Train acc: 0.968333\n",
      "Epoch: 596/1000 Iteration: 5370 Train loss: 0.086448 Train acc: 0.970000\n",
      "Epoch: 597/1000 Iteration: 5375 Train loss: 0.124049 Train acc: 0.960000\n",
      "Epoch: 597/1000 Iteration: 5375 Validation loss: 0.104232 Validation acc: 0.958889\n",
      "Epoch: 597/1000 Iteration: 5380 Train loss: 0.099073 Train acc: 0.973333\n",
      "Epoch: 598/1000 Iteration: 5385 Train loss: 0.131816 Train acc: 0.956667\n",
      "Epoch: 598/1000 Iteration: 5390 Train loss: 0.083177 Train acc: 0.975000\n",
      "Epoch: 599/1000 Iteration: 5395 Train loss: 0.128427 Train acc: 0.950000\n",
      "Epoch: 599/1000 Iteration: 5400 Train loss: 0.106552 Train acc: 0.966667\n",
      "Epoch: 599/1000 Iteration: 5400 Validation loss: 0.103881 Validation acc: 0.958333\n",
      "Epoch: 600/1000 Iteration: 5405 Train loss: 0.109642 Train acc: 0.965000\n",
      "Epoch: 601/1000 Iteration: 5410 Train loss: 0.109224 Train acc: 0.956667\n",
      "Epoch: 601/1000 Iteration: 5415 Train loss: 0.092023 Train acc: 0.966667\n",
      "Epoch: 602/1000 Iteration: 5420 Train loss: 0.114645 Train acc: 0.965000\n",
      "Epoch: 602/1000 Iteration: 5425 Train loss: 0.098266 Train acc: 0.975000\n",
      "Epoch: 602/1000 Iteration: 5425 Validation loss: 0.103422 Validation acc: 0.959444\n",
      "Epoch: 603/1000 Iteration: 5430 Train loss: 0.133968 Train acc: 0.953333\n",
      "Epoch: 603/1000 Iteration: 5435 Train loss: 0.082079 Train acc: 0.981667\n",
      "Epoch: 604/1000 Iteration: 5440 Train loss: 0.144255 Train acc: 0.948333\n",
      "Epoch: 604/1000 Iteration: 5445 Train loss: 0.112732 Train acc: 0.965000\n",
      "Epoch: 605/1000 Iteration: 5450 Train loss: 0.120986 Train acc: 0.963333\n",
      "Epoch: 605/1000 Iteration: 5450 Validation loss: 0.103057 Validation acc: 0.962222\n",
      "Epoch: 606/1000 Iteration: 5455 Train loss: 0.099911 Train acc: 0.963333\n",
      "Epoch: 606/1000 Iteration: 5460 Train loss: 0.086119 Train acc: 0.971667\n",
      "Epoch: 607/1000 Iteration: 5465 Train loss: 0.108503 Train acc: 0.970000\n",
      "Epoch: 607/1000 Iteration: 5470 Train loss: 0.105088 Train acc: 0.961667\n",
      "Epoch: 608/1000 Iteration: 5475 Train loss: 0.153630 Train acc: 0.951667\n",
      "Epoch: 608/1000 Iteration: 5475 Validation loss: 0.106739 Validation acc: 0.957778\n",
      "Epoch: 608/1000 Iteration: 5480 Train loss: 0.072776 Train acc: 0.971667\n",
      "Epoch: 609/1000 Iteration: 5485 Train loss: 0.132992 Train acc: 0.950000\n",
      "Epoch: 609/1000 Iteration: 5490 Train loss: 0.111952 Train acc: 0.966667\n",
      "Epoch: 610/1000 Iteration: 5495 Train loss: 0.113241 Train acc: 0.958333\n",
      "Epoch: 611/1000 Iteration: 5500 Train loss: 0.103171 Train acc: 0.966667\n",
      "Epoch: 611/1000 Iteration: 5500 Validation loss: 0.102020 Validation acc: 0.961667\n",
      "Epoch: 611/1000 Iteration: 5505 Train loss: 0.090565 Train acc: 0.966667\n",
      "Epoch: 612/1000 Iteration: 5510 Train loss: 0.102244 Train acc: 0.971667\n",
      "Epoch: 612/1000 Iteration: 5515 Train loss: 0.106583 Train acc: 0.965000\n",
      "Epoch: 613/1000 Iteration: 5520 Train loss: 0.131194 Train acc: 0.951667\n",
      "Epoch: 613/1000 Iteration: 5525 Train loss: 0.073540 Train acc: 0.976667\n",
      "Epoch: 613/1000 Iteration: 5525 Validation loss: 0.102292 Validation acc: 0.960556\n",
      "Epoch: 614/1000 Iteration: 5530 Train loss: 0.122752 Train acc: 0.955000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 614/1000 Iteration: 5535 Train loss: 0.103466 Train acc: 0.968333\n",
      "Epoch: 615/1000 Iteration: 5540 Train loss: 0.124510 Train acc: 0.953333\n",
      "Epoch: 616/1000 Iteration: 5545 Train loss: 0.112090 Train acc: 0.950000\n",
      "Epoch: 616/1000 Iteration: 5550 Train loss: 0.091683 Train acc: 0.976667\n",
      "Epoch: 616/1000 Iteration: 5550 Validation loss: 0.104745 Validation acc: 0.959444\n",
      "Epoch: 617/1000 Iteration: 5555 Train loss: 0.105364 Train acc: 0.963333\n",
      "Epoch: 617/1000 Iteration: 5560 Train loss: 0.109389 Train acc: 0.965000\n",
      "Epoch: 618/1000 Iteration: 5565 Train loss: 0.146145 Train acc: 0.950000\n",
      "Epoch: 618/1000 Iteration: 5570 Train loss: 0.082990 Train acc: 0.975000\n",
      "Epoch: 619/1000 Iteration: 5575 Train loss: 0.134065 Train acc: 0.961667\n",
      "Epoch: 619/1000 Iteration: 5575 Validation loss: 0.111476 Validation acc: 0.961667\n",
      "Epoch: 619/1000 Iteration: 5580 Train loss: 0.108106 Train acc: 0.968333\n",
      "Epoch: 620/1000 Iteration: 5585 Train loss: 0.108230 Train acc: 0.965000\n",
      "Epoch: 621/1000 Iteration: 5590 Train loss: 0.092561 Train acc: 0.966667\n",
      "Epoch: 621/1000 Iteration: 5595 Train loss: 0.093418 Train acc: 0.973333\n",
      "Epoch: 622/1000 Iteration: 5600 Train loss: 0.107805 Train acc: 0.973333\n",
      "Epoch: 622/1000 Iteration: 5600 Validation loss: 0.106727 Validation acc: 0.959444\n",
      "Epoch: 622/1000 Iteration: 5605 Train loss: 0.120888 Train acc: 0.961667\n",
      "Epoch: 623/1000 Iteration: 5610 Train loss: 0.134190 Train acc: 0.956667\n",
      "Epoch: 623/1000 Iteration: 5615 Train loss: 0.079181 Train acc: 0.971667\n",
      "Epoch: 624/1000 Iteration: 5620 Train loss: 0.140807 Train acc: 0.946667\n",
      "Epoch: 624/1000 Iteration: 5625 Train loss: 0.120151 Train acc: 0.956667\n",
      "Epoch: 624/1000 Iteration: 5625 Validation loss: 0.105786 Validation acc: 0.959444\n",
      "Epoch: 625/1000 Iteration: 5630 Train loss: 0.111482 Train acc: 0.965000\n",
      "Epoch: 626/1000 Iteration: 5635 Train loss: 0.100044 Train acc: 0.963333\n",
      "Epoch: 626/1000 Iteration: 5640 Train loss: 0.085929 Train acc: 0.971667\n",
      "Epoch: 627/1000 Iteration: 5645 Train loss: 0.121648 Train acc: 0.973333\n",
      "Epoch: 627/1000 Iteration: 5650 Train loss: 0.112931 Train acc: 0.961667\n",
      "Epoch: 627/1000 Iteration: 5650 Validation loss: 0.105251 Validation acc: 0.958889\n",
      "Epoch: 628/1000 Iteration: 5655 Train loss: 0.128546 Train acc: 0.943333\n",
      "Epoch: 628/1000 Iteration: 5660 Train loss: 0.086502 Train acc: 0.970000\n",
      "Epoch: 629/1000 Iteration: 5665 Train loss: 0.129357 Train acc: 0.946667\n",
      "Epoch: 629/1000 Iteration: 5670 Train loss: 0.120971 Train acc: 0.960000\n",
      "Epoch: 630/1000 Iteration: 5675 Train loss: 0.122744 Train acc: 0.963333\n",
      "Epoch: 630/1000 Iteration: 5675 Validation loss: 0.105415 Validation acc: 0.960000\n",
      "Epoch: 631/1000 Iteration: 5680 Train loss: 0.099885 Train acc: 0.960000\n",
      "Epoch: 631/1000 Iteration: 5685 Train loss: 0.088356 Train acc: 0.965000\n",
      "Epoch: 632/1000 Iteration: 5690 Train loss: 0.115236 Train acc: 0.965000\n",
      "Epoch: 632/1000 Iteration: 5695 Train loss: 0.109536 Train acc: 0.978333\n",
      "Epoch: 633/1000 Iteration: 5700 Train loss: 0.182044 Train acc: 0.946667\n",
      "Epoch: 633/1000 Iteration: 5700 Validation loss: 0.118031 Validation acc: 0.955000\n",
      "Epoch: 633/1000 Iteration: 5705 Train loss: 0.098866 Train acc: 0.970000\n",
      "Epoch: 634/1000 Iteration: 5710 Train loss: 0.143411 Train acc: 0.938333\n",
      "Epoch: 634/1000 Iteration: 5715 Train loss: 0.124622 Train acc: 0.956667\n",
      "Epoch: 635/1000 Iteration: 5720 Train loss: 0.126345 Train acc: 0.956667\n",
      "Epoch: 636/1000 Iteration: 5725 Train loss: 0.108747 Train acc: 0.955000\n",
      "Epoch: 636/1000 Iteration: 5725 Validation loss: 0.109902 Validation acc: 0.957222\n",
      "Epoch: 636/1000 Iteration: 5730 Train loss: 0.079886 Train acc: 0.968333\n",
      "Epoch: 637/1000 Iteration: 5735 Train loss: 0.115953 Train acc: 0.966667\n",
      "Epoch: 637/1000 Iteration: 5740 Train loss: 0.102839 Train acc: 0.968333\n",
      "Epoch: 638/1000 Iteration: 5745 Train loss: 0.128365 Train acc: 0.951667\n",
      "Epoch: 638/1000 Iteration: 5750 Train loss: 0.082262 Train acc: 0.978333\n",
      "Epoch: 638/1000 Iteration: 5750 Validation loss: 0.097880 Validation acc: 0.959444\n",
      "Epoch: 639/1000 Iteration: 5755 Train loss: 0.131199 Train acc: 0.946667\n",
      "Epoch: 639/1000 Iteration: 5760 Train loss: 0.116094 Train acc: 0.970000\n",
      "Epoch: 640/1000 Iteration: 5765 Train loss: 0.113128 Train acc: 0.961667\n",
      "Epoch: 641/1000 Iteration: 5770 Train loss: 0.109109 Train acc: 0.961667\n",
      "Epoch: 641/1000 Iteration: 5775 Train loss: 0.082896 Train acc: 0.966667\n",
      "Epoch: 641/1000 Iteration: 5775 Validation loss: 0.098553 Validation acc: 0.961111\n",
      "Epoch: 642/1000 Iteration: 5780 Train loss: 0.102265 Train acc: 0.968333\n",
      "Epoch: 642/1000 Iteration: 5785 Train loss: 0.097458 Train acc: 0.963333\n",
      "Epoch: 643/1000 Iteration: 5790 Train loss: 0.115397 Train acc: 0.960000\n",
      "Epoch: 643/1000 Iteration: 5795 Train loss: 0.088318 Train acc: 0.975000\n",
      "Epoch: 644/1000 Iteration: 5800 Train loss: 0.125336 Train acc: 0.958333\n",
      "Epoch: 644/1000 Iteration: 5800 Validation loss: 0.092546 Validation acc: 0.960556\n",
      "Epoch: 644/1000 Iteration: 5805 Train loss: 0.114993 Train acc: 0.961667\n",
      "Epoch: 645/1000 Iteration: 5810 Train loss: 0.115893 Train acc: 0.950000\n",
      "Epoch: 646/1000 Iteration: 5815 Train loss: 0.098381 Train acc: 0.965000\n",
      "Epoch: 646/1000 Iteration: 5820 Train loss: 0.097767 Train acc: 0.965000\n",
      "Epoch: 647/1000 Iteration: 5825 Train loss: 0.116784 Train acc: 0.965000\n",
      "Epoch: 647/1000 Iteration: 5825 Validation loss: 0.097959 Validation acc: 0.961111\n",
      "Epoch: 647/1000 Iteration: 5830 Train loss: 0.103968 Train acc: 0.968333\n",
      "Epoch: 648/1000 Iteration: 5835 Train loss: 0.136768 Train acc: 0.950000\n",
      "Epoch: 648/1000 Iteration: 5840 Train loss: 0.075643 Train acc: 0.975000\n",
      "Epoch: 649/1000 Iteration: 5845 Train loss: 0.126360 Train acc: 0.951667\n",
      "Epoch: 649/1000 Iteration: 5850 Train loss: 0.096360 Train acc: 0.963333\n",
      "Epoch: 649/1000 Iteration: 5850 Validation loss: 0.092835 Validation acc: 0.960000\n",
      "Epoch: 650/1000 Iteration: 5855 Train loss: 0.118473 Train acc: 0.965000\n",
      "Epoch: 651/1000 Iteration: 5860 Train loss: 0.099259 Train acc: 0.953333\n",
      "Epoch: 651/1000 Iteration: 5865 Train loss: 0.091817 Train acc: 0.963333\n",
      "Epoch: 652/1000 Iteration: 5870 Train loss: 0.106217 Train acc: 0.968333\n",
      "Epoch: 652/1000 Iteration: 5875 Train loss: 0.109682 Train acc: 0.968333\n",
      "Epoch: 652/1000 Iteration: 5875 Validation loss: 0.094683 Validation acc: 0.961111\n",
      "Epoch: 653/1000 Iteration: 5880 Train loss: 0.121113 Train acc: 0.958333\n",
      "Epoch: 653/1000 Iteration: 5885 Train loss: 0.079964 Train acc: 0.971667\n",
      "Epoch: 654/1000 Iteration: 5890 Train loss: 0.124423 Train acc: 0.953333\n",
      "Epoch: 654/1000 Iteration: 5895 Train loss: 0.101635 Train acc: 0.971667\n",
      "Epoch: 655/1000 Iteration: 5900 Train loss: 0.115493 Train acc: 0.961667\n",
      "Epoch: 655/1000 Iteration: 5900 Validation loss: 0.091966 Validation acc: 0.963333\n",
      "Epoch: 656/1000 Iteration: 5905 Train loss: 0.100988 Train acc: 0.963333\n",
      "Epoch: 656/1000 Iteration: 5910 Train loss: 0.072768 Train acc: 0.978333\n",
      "Epoch: 657/1000 Iteration: 5915 Train loss: 0.111635 Train acc: 0.961667\n",
      "Epoch: 657/1000 Iteration: 5920 Train loss: 0.097440 Train acc: 0.971667\n",
      "Epoch: 658/1000 Iteration: 5925 Train loss: 0.137177 Train acc: 0.953333\n",
      "Epoch: 658/1000 Iteration: 5925 Validation loss: 0.093108 Validation acc: 0.963889\n",
      "Epoch: 658/1000 Iteration: 5930 Train loss: 0.082345 Train acc: 0.981667\n",
      "Epoch: 659/1000 Iteration: 5935 Train loss: 0.132610 Train acc: 0.943333\n",
      "Epoch: 659/1000 Iteration: 5940 Train loss: 0.109232 Train acc: 0.966667\n",
      "Epoch: 660/1000 Iteration: 5945 Train loss: 0.106431 Train acc: 0.961667\n",
      "Epoch: 661/1000 Iteration: 5950 Train loss: 0.094740 Train acc: 0.966667\n",
      "Epoch: 661/1000 Iteration: 5950 Validation loss: 0.095984 Validation acc: 0.961111\n",
      "Epoch: 661/1000 Iteration: 5955 Train loss: 0.090282 Train acc: 0.968333\n",
      "Epoch: 662/1000 Iteration: 5960 Train loss: 0.115051 Train acc: 0.966667\n",
      "Epoch: 662/1000 Iteration: 5965 Train loss: 0.100310 Train acc: 0.970000\n",
      "Epoch: 663/1000 Iteration: 5970 Train loss: 0.120597 Train acc: 0.951667\n",
      "Epoch: 663/1000 Iteration: 5975 Train loss: 0.085705 Train acc: 0.973333\n",
      "Epoch: 663/1000 Iteration: 5975 Validation loss: 0.091445 Validation acc: 0.962778\n",
      "Epoch: 664/1000 Iteration: 5980 Train loss: 0.130493 Train acc: 0.948333\n",
      "Epoch: 664/1000 Iteration: 5985 Train loss: 0.115146 Train acc: 0.956667\n",
      "Epoch: 665/1000 Iteration: 5990 Train loss: 0.105133 Train acc: 0.965000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 666/1000 Iteration: 5995 Train loss: 0.102681 Train acc: 0.958333\n",
      "Epoch: 666/1000 Iteration: 6000 Train loss: 0.090306 Train acc: 0.970000\n",
      "Epoch: 666/1000 Iteration: 6000 Validation loss: 0.091987 Validation acc: 0.964444\n",
      "Epoch: 667/1000 Iteration: 6005 Train loss: 0.104717 Train acc: 0.965000\n",
      "Epoch: 667/1000 Iteration: 6010 Train loss: 0.090850 Train acc: 0.976667\n",
      "Epoch: 668/1000 Iteration: 6015 Train loss: 0.116062 Train acc: 0.960000\n",
      "Epoch: 668/1000 Iteration: 6020 Train loss: 0.069937 Train acc: 0.981667\n",
      "Epoch: 669/1000 Iteration: 6025 Train loss: 0.126879 Train acc: 0.943333\n",
      "Epoch: 669/1000 Iteration: 6025 Validation loss: 0.092333 Validation acc: 0.965556\n",
      "Epoch: 669/1000 Iteration: 6030 Train loss: 0.105808 Train acc: 0.966667\n",
      "Epoch: 670/1000 Iteration: 6035 Train loss: 0.102350 Train acc: 0.966667\n",
      "Epoch: 671/1000 Iteration: 6040 Train loss: 0.089089 Train acc: 0.971667\n",
      "Epoch: 671/1000 Iteration: 6045 Train loss: 0.083957 Train acc: 0.973333\n",
      "Epoch: 672/1000 Iteration: 6050 Train loss: 0.101869 Train acc: 0.971667\n",
      "Epoch: 672/1000 Iteration: 6050 Validation loss: 0.099742 Validation acc: 0.962778\n",
      "Epoch: 672/1000 Iteration: 6055 Train loss: 0.094373 Train acc: 0.968333\n",
      "Epoch: 673/1000 Iteration: 6060 Train loss: 0.121386 Train acc: 0.956667\n",
      "Epoch: 673/1000 Iteration: 6065 Train loss: 0.085566 Train acc: 0.970000\n",
      "Epoch: 674/1000 Iteration: 6070 Train loss: 0.122826 Train acc: 0.963333\n",
      "Epoch: 674/1000 Iteration: 6075 Train loss: 0.118269 Train acc: 0.968333\n",
      "Epoch: 674/1000 Iteration: 6075 Validation loss: 0.092175 Validation acc: 0.961111\n",
      "Epoch: 675/1000 Iteration: 6080 Train loss: 0.111711 Train acc: 0.953333\n",
      "Epoch: 676/1000 Iteration: 6085 Train loss: 0.087345 Train acc: 0.970000\n",
      "Epoch: 676/1000 Iteration: 6090 Train loss: 0.074169 Train acc: 0.971667\n",
      "Epoch: 677/1000 Iteration: 6095 Train loss: 0.108180 Train acc: 0.971667\n",
      "Epoch: 677/1000 Iteration: 6100 Train loss: 0.106644 Train acc: 0.971667\n",
      "Epoch: 677/1000 Iteration: 6100 Validation loss: 0.092104 Validation acc: 0.959444\n",
      "Epoch: 678/1000 Iteration: 6105 Train loss: 0.113983 Train acc: 0.975000\n",
      "Epoch: 678/1000 Iteration: 6110 Train loss: 0.075338 Train acc: 0.978333\n",
      "Epoch: 679/1000 Iteration: 6115 Train loss: 0.121393 Train acc: 0.948333\n",
      "Epoch: 679/1000 Iteration: 6120 Train loss: 0.113610 Train acc: 0.955000\n",
      "Epoch: 680/1000 Iteration: 6125 Train loss: 0.103360 Train acc: 0.958333\n",
      "Epoch: 680/1000 Iteration: 6125 Validation loss: 0.094516 Validation acc: 0.963333\n",
      "Epoch: 681/1000 Iteration: 6130 Train loss: 0.099773 Train acc: 0.966667\n",
      "Epoch: 681/1000 Iteration: 6135 Train loss: 0.086373 Train acc: 0.965000\n",
      "Epoch: 682/1000 Iteration: 6140 Train loss: 0.102772 Train acc: 0.975000\n",
      "Epoch: 682/1000 Iteration: 6145 Train loss: 0.099544 Train acc: 0.963333\n",
      "Epoch: 683/1000 Iteration: 6150 Train loss: 0.127222 Train acc: 0.958333\n",
      "Epoch: 683/1000 Iteration: 6150 Validation loss: 0.099680 Validation acc: 0.961111\n",
      "Epoch: 683/1000 Iteration: 6155 Train loss: 0.072729 Train acc: 0.976667\n",
      "Epoch: 684/1000 Iteration: 6160 Train loss: 0.112843 Train acc: 0.950000\n",
      "Epoch: 684/1000 Iteration: 6165 Train loss: 0.113348 Train acc: 0.961667\n",
      "Epoch: 685/1000 Iteration: 6170 Train loss: 0.104370 Train acc: 0.961667\n",
      "Epoch: 686/1000 Iteration: 6175 Train loss: 0.101389 Train acc: 0.966667\n",
      "Epoch: 686/1000 Iteration: 6175 Validation loss: 0.096122 Validation acc: 0.962222\n",
      "Epoch: 686/1000 Iteration: 6180 Train loss: 0.085867 Train acc: 0.966667\n",
      "Epoch: 687/1000 Iteration: 6185 Train loss: 0.112516 Train acc: 0.968333\n",
      "Epoch: 687/1000 Iteration: 6190 Train loss: 0.091216 Train acc: 0.971667\n",
      "Epoch: 688/1000 Iteration: 6195 Train loss: 0.131694 Train acc: 0.955000\n",
      "Epoch: 688/1000 Iteration: 6200 Train loss: 0.073029 Train acc: 0.978333\n",
      "Epoch: 688/1000 Iteration: 6200 Validation loss: 0.095001 Validation acc: 0.963333\n",
      "Epoch: 689/1000 Iteration: 6205 Train loss: 0.123583 Train acc: 0.946667\n",
      "Epoch: 689/1000 Iteration: 6210 Train loss: 0.108332 Train acc: 0.970000\n",
      "Epoch: 690/1000 Iteration: 6215 Train loss: 0.098229 Train acc: 0.965000\n",
      "Epoch: 691/1000 Iteration: 6220 Train loss: 0.097467 Train acc: 0.963333\n",
      "Epoch: 691/1000 Iteration: 6225 Train loss: 0.075442 Train acc: 0.975000\n",
      "Epoch: 691/1000 Iteration: 6225 Validation loss: 0.093581 Validation acc: 0.964444\n",
      "Epoch: 692/1000 Iteration: 6230 Train loss: 0.100377 Train acc: 0.973333\n",
      "Epoch: 692/1000 Iteration: 6235 Train loss: 0.090984 Train acc: 0.975000\n",
      "Epoch: 693/1000 Iteration: 6240 Train loss: 0.126871 Train acc: 0.963333\n",
      "Epoch: 693/1000 Iteration: 6245 Train loss: 0.087614 Train acc: 0.971667\n",
      "Epoch: 694/1000 Iteration: 6250 Train loss: 0.117641 Train acc: 0.951667\n",
      "Epoch: 694/1000 Iteration: 6250 Validation loss: 0.095782 Validation acc: 0.963889\n",
      "Epoch: 694/1000 Iteration: 6255 Train loss: 0.094152 Train acc: 0.968333\n",
      "Epoch: 695/1000 Iteration: 6260 Train loss: 0.105147 Train acc: 0.963333\n",
      "Epoch: 696/1000 Iteration: 6265 Train loss: 0.091068 Train acc: 0.966667\n",
      "Epoch: 696/1000 Iteration: 6270 Train loss: 0.080839 Train acc: 0.968333\n",
      "Epoch: 697/1000 Iteration: 6275 Train loss: 0.106418 Train acc: 0.968333\n",
      "Epoch: 697/1000 Iteration: 6275 Validation loss: 0.099910 Validation acc: 0.964444\n",
      "Epoch: 697/1000 Iteration: 6280 Train loss: 0.101986 Train acc: 0.968333\n",
      "Epoch: 698/1000 Iteration: 6285 Train loss: 0.119615 Train acc: 0.955000\n",
      "Epoch: 698/1000 Iteration: 6290 Train loss: 0.072901 Train acc: 0.981667\n",
      "Epoch: 699/1000 Iteration: 6295 Train loss: 0.112240 Train acc: 0.955000\n",
      "Epoch: 699/1000 Iteration: 6300 Train loss: 0.121424 Train acc: 0.965000\n",
      "Epoch: 699/1000 Iteration: 6300 Validation loss: 0.101357 Validation acc: 0.963333\n",
      "Epoch: 700/1000 Iteration: 6305 Train loss: 0.105139 Train acc: 0.965000\n",
      "Epoch: 701/1000 Iteration: 6310 Train loss: 0.108940 Train acc: 0.960000\n",
      "Epoch: 701/1000 Iteration: 6315 Train loss: 0.085949 Train acc: 0.971667\n",
      "Epoch: 702/1000 Iteration: 6320 Train loss: 0.105541 Train acc: 0.975000\n",
      "Epoch: 702/1000 Iteration: 6325 Train loss: 0.097425 Train acc: 0.970000\n",
      "Epoch: 702/1000 Iteration: 6325 Validation loss: 0.090891 Validation acc: 0.966111\n",
      "Epoch: 703/1000 Iteration: 6330 Train loss: 0.109622 Train acc: 0.963333\n",
      "Epoch: 703/1000 Iteration: 6335 Train loss: 0.064830 Train acc: 0.981667\n",
      "Epoch: 704/1000 Iteration: 6340 Train loss: 0.106943 Train acc: 0.961667\n",
      "Epoch: 704/1000 Iteration: 6345 Train loss: 0.100288 Train acc: 0.965000\n",
      "Epoch: 705/1000 Iteration: 6350 Train loss: 0.103429 Train acc: 0.956667\n",
      "Epoch: 705/1000 Iteration: 6350 Validation loss: 0.090643 Validation acc: 0.963889\n",
      "Epoch: 706/1000 Iteration: 6355 Train loss: 0.103833 Train acc: 0.961667\n",
      "Epoch: 706/1000 Iteration: 6360 Train loss: 0.082045 Train acc: 0.966667\n",
      "Epoch: 707/1000 Iteration: 6365 Train loss: 0.103398 Train acc: 0.973333\n",
      "Epoch: 707/1000 Iteration: 6370 Train loss: 0.100751 Train acc: 0.963333\n",
      "Epoch: 708/1000 Iteration: 6375 Train loss: 0.117557 Train acc: 0.958333\n",
      "Epoch: 708/1000 Iteration: 6375 Validation loss: 0.094624 Validation acc: 0.965000\n",
      "Epoch: 708/1000 Iteration: 6380 Train loss: 0.082097 Train acc: 0.983333\n",
      "Epoch: 709/1000 Iteration: 6385 Train loss: 0.115855 Train acc: 0.951667\n",
      "Epoch: 709/1000 Iteration: 6390 Train loss: 0.119049 Train acc: 0.966667\n",
      "Epoch: 710/1000 Iteration: 6395 Train loss: 0.100507 Train acc: 0.971667\n",
      "Epoch: 711/1000 Iteration: 6400 Train loss: 0.093416 Train acc: 0.970000\n",
      "Epoch: 711/1000 Iteration: 6400 Validation loss: 0.091048 Validation acc: 0.966111\n",
      "Epoch: 711/1000 Iteration: 6405 Train loss: 0.075727 Train acc: 0.976667\n",
      "Epoch: 712/1000 Iteration: 6410 Train loss: 0.107909 Train acc: 0.966667\n",
      "Epoch: 712/1000 Iteration: 6415 Train loss: 0.096817 Train acc: 0.973333\n",
      "Epoch: 713/1000 Iteration: 6420 Train loss: 0.117107 Train acc: 0.951667\n",
      "Epoch: 713/1000 Iteration: 6425 Train loss: 0.069239 Train acc: 0.981667\n",
      "Epoch: 713/1000 Iteration: 6425 Validation loss: 0.091187 Validation acc: 0.966111\n",
      "Epoch: 714/1000 Iteration: 6430 Train loss: 0.118516 Train acc: 0.955000\n",
      "Epoch: 714/1000 Iteration: 6435 Train loss: 0.109272 Train acc: 0.966667\n",
      "Epoch: 715/1000 Iteration: 6440 Train loss: 0.102071 Train acc: 0.966667\n",
      "Epoch: 716/1000 Iteration: 6445 Train loss: 0.102056 Train acc: 0.968333\n",
      "Epoch: 716/1000 Iteration: 6450 Train loss: 0.092162 Train acc: 0.966667\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 716/1000 Iteration: 6450 Validation loss: 0.092264 Validation acc: 0.963889\n",
      "Epoch: 717/1000 Iteration: 6455 Train loss: 0.112871 Train acc: 0.965000\n",
      "Epoch: 717/1000 Iteration: 6460 Train loss: 0.109513 Train acc: 0.965000\n",
      "Epoch: 718/1000 Iteration: 6465 Train loss: 0.131746 Train acc: 0.956667\n",
      "Epoch: 718/1000 Iteration: 6470 Train loss: 0.071166 Train acc: 0.981667\n",
      "Epoch: 719/1000 Iteration: 6475 Train loss: 0.117552 Train acc: 0.956667\n",
      "Epoch: 719/1000 Iteration: 6475 Validation loss: 0.098604 Validation acc: 0.960556\n",
      "Epoch: 719/1000 Iteration: 6480 Train loss: 0.096095 Train acc: 0.966667\n",
      "Epoch: 720/1000 Iteration: 6485 Train loss: 0.095695 Train acc: 0.968333\n",
      "Epoch: 721/1000 Iteration: 6490 Train loss: 0.084060 Train acc: 0.965000\n",
      "Epoch: 721/1000 Iteration: 6495 Train loss: 0.080790 Train acc: 0.970000\n",
      "Epoch: 722/1000 Iteration: 6500 Train loss: 0.116613 Train acc: 0.965000\n",
      "Epoch: 722/1000 Iteration: 6500 Validation loss: 0.091487 Validation acc: 0.963889\n",
      "Epoch: 722/1000 Iteration: 6505 Train loss: 0.091271 Train acc: 0.970000\n",
      "Epoch: 723/1000 Iteration: 6510 Train loss: 0.119034 Train acc: 0.963333\n",
      "Epoch: 723/1000 Iteration: 6515 Train loss: 0.076999 Train acc: 0.978333\n",
      "Epoch: 724/1000 Iteration: 6520 Train loss: 0.128189 Train acc: 0.951667\n",
      "Epoch: 724/1000 Iteration: 6525 Train loss: 0.108983 Train acc: 0.968333\n",
      "Epoch: 724/1000 Iteration: 6525 Validation loss: 0.093141 Validation acc: 0.961111\n",
      "Epoch: 725/1000 Iteration: 6530 Train loss: 0.100564 Train acc: 0.968333\n",
      "Epoch: 726/1000 Iteration: 6535 Train loss: 0.097692 Train acc: 0.963333\n",
      "Epoch: 726/1000 Iteration: 6540 Train loss: 0.088271 Train acc: 0.966667\n",
      "Epoch: 727/1000 Iteration: 6545 Train loss: 0.102846 Train acc: 0.966667\n",
      "Epoch: 727/1000 Iteration: 6550 Train loss: 0.096134 Train acc: 0.965000\n",
      "Epoch: 727/1000 Iteration: 6550 Validation loss: 0.094264 Validation acc: 0.962222\n",
      "Epoch: 728/1000 Iteration: 6555 Train loss: 0.124616 Train acc: 0.956667\n",
      "Epoch: 728/1000 Iteration: 6560 Train loss: 0.083655 Train acc: 0.976667\n",
      "Epoch: 729/1000 Iteration: 6565 Train loss: 0.106644 Train acc: 0.958333\n",
      "Epoch: 729/1000 Iteration: 6570 Train loss: 0.097780 Train acc: 0.965000\n",
      "Epoch: 730/1000 Iteration: 6575 Train loss: 0.112544 Train acc: 0.963333\n",
      "Epoch: 730/1000 Iteration: 6575 Validation loss: 0.096537 Validation acc: 0.961667\n",
      "Epoch: 731/1000 Iteration: 6580 Train loss: 0.108395 Train acc: 0.961667\n",
      "Epoch: 731/1000 Iteration: 6585 Train loss: 0.079004 Train acc: 0.975000\n",
      "Epoch: 732/1000 Iteration: 6590 Train loss: 0.094182 Train acc: 0.968333\n",
      "Epoch: 732/1000 Iteration: 6595 Train loss: 0.089054 Train acc: 0.973333\n",
      "Epoch: 733/1000 Iteration: 6600 Train loss: 0.110227 Train acc: 0.958333\n",
      "Epoch: 733/1000 Iteration: 6600 Validation loss: 0.098985 Validation acc: 0.960556\n",
      "Epoch: 733/1000 Iteration: 6605 Train loss: 0.076552 Train acc: 0.976667\n",
      "Epoch: 734/1000 Iteration: 6610 Train loss: 0.142690 Train acc: 0.946667\n",
      "Epoch: 734/1000 Iteration: 6615 Train loss: 0.097181 Train acc: 0.966667\n",
      "Epoch: 735/1000 Iteration: 6620 Train loss: 0.091421 Train acc: 0.973333\n",
      "Epoch: 736/1000 Iteration: 6625 Train loss: 0.088563 Train acc: 0.971667\n",
      "Epoch: 736/1000 Iteration: 6625 Validation loss: 0.096650 Validation acc: 0.963333\n",
      "Epoch: 736/1000 Iteration: 6630 Train loss: 0.082172 Train acc: 0.971667\n",
      "Epoch: 737/1000 Iteration: 6635 Train loss: 0.095935 Train acc: 0.970000\n",
      "Epoch: 737/1000 Iteration: 6640 Train loss: 0.089364 Train acc: 0.976667\n",
      "Epoch: 738/1000 Iteration: 6645 Train loss: 0.120063 Train acc: 0.958333\n",
      "Epoch: 738/1000 Iteration: 6650 Train loss: 0.076397 Train acc: 0.980000\n",
      "Epoch: 738/1000 Iteration: 6650 Validation loss: 0.097286 Validation acc: 0.964444\n",
      "Epoch: 739/1000 Iteration: 6655 Train loss: 0.110654 Train acc: 0.958333\n",
      "Epoch: 739/1000 Iteration: 6660 Train loss: 0.109719 Train acc: 0.968333\n",
      "Epoch: 740/1000 Iteration: 6665 Train loss: 0.100285 Train acc: 0.961667\n",
      "Epoch: 741/1000 Iteration: 6670 Train loss: 0.097309 Train acc: 0.963333\n",
      "Epoch: 741/1000 Iteration: 6675 Train loss: 0.082674 Train acc: 0.970000\n",
      "Epoch: 741/1000 Iteration: 6675 Validation loss: 0.099032 Validation acc: 0.961111\n",
      "Epoch: 742/1000 Iteration: 6680 Train loss: 0.094135 Train acc: 0.965000\n",
      "Epoch: 742/1000 Iteration: 6685 Train loss: 0.106146 Train acc: 0.973333\n",
      "Epoch: 743/1000 Iteration: 6690 Train loss: 0.122248 Train acc: 0.961667\n",
      "Epoch: 743/1000 Iteration: 6695 Train loss: 0.070716 Train acc: 0.980000\n",
      "Epoch: 744/1000 Iteration: 6700 Train loss: 0.106123 Train acc: 0.953333\n",
      "Epoch: 744/1000 Iteration: 6700 Validation loss: 0.093594 Validation acc: 0.965000\n",
      "Epoch: 744/1000 Iteration: 6705 Train loss: 0.100589 Train acc: 0.970000\n",
      "Epoch: 745/1000 Iteration: 6710 Train loss: 0.097668 Train acc: 0.970000\n",
      "Epoch: 746/1000 Iteration: 6715 Train loss: 0.089223 Train acc: 0.961667\n",
      "Epoch: 746/1000 Iteration: 6720 Train loss: 0.073470 Train acc: 0.971667\n",
      "Epoch: 747/1000 Iteration: 6725 Train loss: 0.098820 Train acc: 0.966667\n",
      "Epoch: 747/1000 Iteration: 6725 Validation loss: 0.094740 Validation acc: 0.963333\n",
      "Epoch: 747/1000 Iteration: 6730 Train loss: 0.098630 Train acc: 0.970000\n",
      "Epoch: 748/1000 Iteration: 6735 Train loss: 0.118341 Train acc: 0.956667\n",
      "Epoch: 748/1000 Iteration: 6740 Train loss: 0.067971 Train acc: 0.983333\n",
      "Epoch: 749/1000 Iteration: 6745 Train loss: 0.119580 Train acc: 0.961667\n",
      "Epoch: 749/1000 Iteration: 6750 Train loss: 0.106812 Train acc: 0.966667\n",
      "Epoch: 749/1000 Iteration: 6750 Validation loss: 0.102585 Validation acc: 0.963889\n",
      "Epoch: 750/1000 Iteration: 6755 Train loss: 0.110225 Train acc: 0.968333\n",
      "Epoch: 751/1000 Iteration: 6760 Train loss: 0.093162 Train acc: 0.961667\n",
      "Epoch: 751/1000 Iteration: 6765 Train loss: 0.077560 Train acc: 0.973333\n",
      "Epoch: 752/1000 Iteration: 6770 Train loss: 0.096514 Train acc: 0.970000\n",
      "Epoch: 752/1000 Iteration: 6775 Train loss: 0.095343 Train acc: 0.970000\n",
      "Epoch: 752/1000 Iteration: 6775 Validation loss: 0.094510 Validation acc: 0.963333\n",
      "Epoch: 753/1000 Iteration: 6780 Train loss: 0.127774 Train acc: 0.963333\n",
      "Epoch: 753/1000 Iteration: 6785 Train loss: 0.078137 Train acc: 0.975000\n",
      "Epoch: 754/1000 Iteration: 6790 Train loss: 0.115277 Train acc: 0.958333\n",
      "Epoch: 754/1000 Iteration: 6795 Train loss: 0.098442 Train acc: 0.970000\n",
      "Epoch: 755/1000 Iteration: 6800 Train loss: 0.096241 Train acc: 0.965000\n",
      "Epoch: 755/1000 Iteration: 6800 Validation loss: 0.094815 Validation acc: 0.963889\n",
      "Epoch: 756/1000 Iteration: 6805 Train loss: 0.092789 Train acc: 0.965000\n",
      "Epoch: 756/1000 Iteration: 6810 Train loss: 0.072697 Train acc: 0.971667\n",
      "Epoch: 757/1000 Iteration: 6815 Train loss: 0.105788 Train acc: 0.971667\n",
      "Epoch: 757/1000 Iteration: 6820 Train loss: 0.105467 Train acc: 0.973333\n",
      "Epoch: 758/1000 Iteration: 6825 Train loss: 0.109071 Train acc: 0.958333\n",
      "Epoch: 758/1000 Iteration: 6825 Validation loss: 0.097250 Validation acc: 0.963889\n",
      "Epoch: 758/1000 Iteration: 6830 Train loss: 0.063311 Train acc: 0.983333\n",
      "Epoch: 759/1000 Iteration: 6835 Train loss: 0.106747 Train acc: 0.953333\n",
      "Epoch: 759/1000 Iteration: 6840 Train loss: 0.117046 Train acc: 0.966667\n",
      "Epoch: 760/1000 Iteration: 6845 Train loss: 0.098030 Train acc: 0.961667\n",
      "Epoch: 761/1000 Iteration: 6850 Train loss: 0.083804 Train acc: 0.975000\n",
      "Epoch: 761/1000 Iteration: 6850 Validation loss: 0.097338 Validation acc: 0.965555\n",
      "Epoch: 761/1000 Iteration: 6855 Train loss: 0.084087 Train acc: 0.973333\n",
      "Epoch: 762/1000 Iteration: 6860 Train loss: 0.104944 Train acc: 0.975000\n",
      "Epoch: 762/1000 Iteration: 6865 Train loss: 0.084748 Train acc: 0.976667\n",
      "Epoch: 763/1000 Iteration: 6870 Train loss: 0.118143 Train acc: 0.963333\n",
      "Epoch: 763/1000 Iteration: 6875 Train loss: 0.064683 Train acc: 0.990000\n",
      "Epoch: 763/1000 Iteration: 6875 Validation loss: 0.096706 Validation acc: 0.967778\n",
      "Epoch: 764/1000 Iteration: 6880 Train loss: 0.105803 Train acc: 0.966667\n",
      "Epoch: 764/1000 Iteration: 6885 Train loss: 0.114147 Train acc: 0.966667\n",
      "Epoch: 765/1000 Iteration: 6890 Train loss: 0.096382 Train acc: 0.970000\n",
      "Epoch: 766/1000 Iteration: 6895 Train loss: 0.083802 Train acc: 0.971667\n",
      "Epoch: 766/1000 Iteration: 6900 Train loss: 0.079970 Train acc: 0.976667\n",
      "Epoch: 766/1000 Iteration: 6900 Validation loss: 0.094599 Validation acc: 0.965556\n",
      "Epoch: 767/1000 Iteration: 6905 Train loss: 0.103001 Train acc: 0.971667\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 767/1000 Iteration: 6910 Train loss: 0.085482 Train acc: 0.971667\n",
      "Epoch: 768/1000 Iteration: 6915 Train loss: 0.117030 Train acc: 0.960000\n",
      "Epoch: 768/1000 Iteration: 6920 Train loss: 0.073721 Train acc: 0.970000\n",
      "Epoch: 769/1000 Iteration: 6925 Train loss: 0.136828 Train acc: 0.940000\n",
      "Epoch: 769/1000 Iteration: 6925 Validation loss: 0.099951 Validation acc: 0.966667\n",
      "Epoch: 769/1000 Iteration: 6930 Train loss: 0.098751 Train acc: 0.968333\n",
      "Epoch: 770/1000 Iteration: 6935 Train loss: 0.089624 Train acc: 0.971667\n",
      "Epoch: 771/1000 Iteration: 6940 Train loss: 0.098262 Train acc: 0.963333\n",
      "Epoch: 771/1000 Iteration: 6945 Train loss: 0.072429 Train acc: 0.975000\n",
      "Epoch: 772/1000 Iteration: 6950 Train loss: 0.098995 Train acc: 0.968333\n",
      "Epoch: 772/1000 Iteration: 6950 Validation loss: 0.100095 Validation acc: 0.966667\n",
      "Epoch: 772/1000 Iteration: 6955 Train loss: 0.106256 Train acc: 0.966667\n",
      "Epoch: 773/1000 Iteration: 6960 Train loss: 0.128305 Train acc: 0.953333\n",
      "Epoch: 773/1000 Iteration: 6965 Train loss: 0.076649 Train acc: 0.973333\n",
      "Epoch: 774/1000 Iteration: 6970 Train loss: 0.114145 Train acc: 0.965000\n",
      "Epoch: 774/1000 Iteration: 6975 Train loss: 0.109413 Train acc: 0.968333\n",
      "Epoch: 774/1000 Iteration: 6975 Validation loss: 0.106286 Validation acc: 0.961111\n",
      "Epoch: 775/1000 Iteration: 6980 Train loss: 0.104754 Train acc: 0.963333\n",
      "Epoch: 776/1000 Iteration: 6985 Train loss: 0.090227 Train acc: 0.966667\n",
      "Epoch: 776/1000 Iteration: 6990 Train loss: 0.079682 Train acc: 0.976667\n",
      "Epoch: 777/1000 Iteration: 6995 Train loss: 0.112435 Train acc: 0.973333\n",
      "Epoch: 777/1000 Iteration: 7000 Train loss: 0.093170 Train acc: 0.973333\n",
      "Epoch: 777/1000 Iteration: 7000 Validation loss: 0.101053 Validation acc: 0.962778\n",
      "Epoch: 778/1000 Iteration: 7005 Train loss: 0.114650 Train acc: 0.963333\n",
      "Epoch: 778/1000 Iteration: 7010 Train loss: 0.068851 Train acc: 0.981667\n",
      "Epoch: 779/1000 Iteration: 7015 Train loss: 0.120790 Train acc: 0.951667\n",
      "Epoch: 779/1000 Iteration: 7020 Train loss: 0.098837 Train acc: 0.973333\n",
      "Epoch: 780/1000 Iteration: 7025 Train loss: 0.095925 Train acc: 0.963333\n",
      "Epoch: 780/1000 Iteration: 7025 Validation loss: 0.093141 Validation acc: 0.965555\n",
      "Epoch: 781/1000 Iteration: 7030 Train loss: 0.089334 Train acc: 0.965000\n",
      "Epoch: 781/1000 Iteration: 7035 Train loss: 0.075056 Train acc: 0.978333\n",
      "Epoch: 782/1000 Iteration: 7040 Train loss: 0.095243 Train acc: 0.966667\n",
      "Epoch: 782/1000 Iteration: 7045 Train loss: 0.096526 Train acc: 0.970000\n",
      "Epoch: 783/1000 Iteration: 7050 Train loss: 0.120525 Train acc: 0.961667\n",
      "Epoch: 783/1000 Iteration: 7050 Validation loss: 0.096562 Validation acc: 0.966667\n",
      "Epoch: 783/1000 Iteration: 7055 Train loss: 0.067297 Train acc: 0.978333\n",
      "Epoch: 784/1000 Iteration: 7060 Train loss: 0.108355 Train acc: 0.951667\n",
      "Epoch: 784/1000 Iteration: 7065 Train loss: 0.109812 Train acc: 0.961667\n",
      "Epoch: 785/1000 Iteration: 7070 Train loss: 0.099288 Train acc: 0.960000\n",
      "Epoch: 786/1000 Iteration: 7075 Train loss: 0.091317 Train acc: 0.968333\n",
      "Epoch: 786/1000 Iteration: 7075 Validation loss: 0.095635 Validation acc: 0.964444\n",
      "Epoch: 786/1000 Iteration: 7080 Train loss: 0.075800 Train acc: 0.973333\n",
      "Epoch: 787/1000 Iteration: 7085 Train loss: 0.095803 Train acc: 0.968333\n",
      "Epoch: 787/1000 Iteration: 7090 Train loss: 0.086761 Train acc: 0.970000\n",
      "Epoch: 788/1000 Iteration: 7095 Train loss: 0.109408 Train acc: 0.963333\n",
      "Epoch: 788/1000 Iteration: 7100 Train loss: 0.066535 Train acc: 0.978333\n",
      "Epoch: 788/1000 Iteration: 7100 Validation loss: 0.094938 Validation acc: 0.963333\n",
      "Epoch: 789/1000 Iteration: 7105 Train loss: 0.112724 Train acc: 0.953333\n",
      "Epoch: 789/1000 Iteration: 7110 Train loss: 0.101201 Train acc: 0.971667\n",
      "Epoch: 790/1000 Iteration: 7115 Train loss: 0.100320 Train acc: 0.968333\n",
      "Epoch: 791/1000 Iteration: 7120 Train loss: 0.082826 Train acc: 0.971667\n",
      "Epoch: 791/1000 Iteration: 7125 Train loss: 0.071446 Train acc: 0.975000\n",
      "Epoch: 791/1000 Iteration: 7125 Validation loss: 0.096872 Validation acc: 0.963333\n",
      "Epoch: 792/1000 Iteration: 7130 Train loss: 0.102372 Train acc: 0.971667\n",
      "Epoch: 792/1000 Iteration: 7135 Train loss: 0.092387 Train acc: 0.971667\n",
      "Epoch: 793/1000 Iteration: 7140 Train loss: 0.111004 Train acc: 0.958333\n",
      "Epoch: 793/1000 Iteration: 7145 Train loss: 0.069504 Train acc: 0.985000\n",
      "Epoch: 794/1000 Iteration: 7150 Train loss: 0.107877 Train acc: 0.960000\n",
      "Epoch: 794/1000 Iteration: 7150 Validation loss: 0.099362 Validation acc: 0.965000\n",
      "Epoch: 794/1000 Iteration: 7155 Train loss: 0.095371 Train acc: 0.975000\n",
      "Epoch: 795/1000 Iteration: 7160 Train loss: 0.098727 Train acc: 0.965000\n",
      "Epoch: 796/1000 Iteration: 7165 Train loss: 0.079911 Train acc: 0.965000\n",
      "Epoch: 796/1000 Iteration: 7170 Train loss: 0.071870 Train acc: 0.975000\n",
      "Epoch: 797/1000 Iteration: 7175 Train loss: 0.098573 Train acc: 0.970000\n",
      "Epoch: 797/1000 Iteration: 7175 Validation loss: 0.105930 Validation acc: 0.961111\n",
      "Epoch: 797/1000 Iteration: 7180 Train loss: 0.095435 Train acc: 0.973333\n",
      "Epoch: 798/1000 Iteration: 7185 Train loss: 0.125294 Train acc: 0.953333\n",
      "Epoch: 798/1000 Iteration: 7190 Train loss: 0.073046 Train acc: 0.975000\n",
      "Epoch: 799/1000 Iteration: 7195 Train loss: 0.123347 Train acc: 0.955000\n",
      "Epoch: 799/1000 Iteration: 7200 Train loss: 0.101081 Train acc: 0.971667\n",
      "Epoch: 799/1000 Iteration: 7200 Validation loss: 0.094230 Validation acc: 0.965000\n",
      "Epoch: 800/1000 Iteration: 7205 Train loss: 0.101147 Train acc: 0.966667\n",
      "Epoch: 801/1000 Iteration: 7210 Train loss: 0.091375 Train acc: 0.961667\n",
      "Epoch: 801/1000 Iteration: 7215 Train loss: 0.082376 Train acc: 0.970000\n",
      "Epoch: 802/1000 Iteration: 7220 Train loss: 0.095557 Train acc: 0.971667\n",
      "Epoch: 802/1000 Iteration: 7225 Train loss: 0.093812 Train acc: 0.975000\n",
      "Epoch: 802/1000 Iteration: 7225 Validation loss: 0.099226 Validation acc: 0.965000\n",
      "Epoch: 803/1000 Iteration: 7230 Train loss: 0.113801 Train acc: 0.955000\n",
      "Epoch: 803/1000 Iteration: 7235 Train loss: 0.068722 Train acc: 0.978333\n",
      "Epoch: 804/1000 Iteration: 7240 Train loss: 0.108997 Train acc: 0.955000\n",
      "Epoch: 804/1000 Iteration: 7245 Train loss: 0.095654 Train acc: 0.973333\n",
      "Epoch: 805/1000 Iteration: 7250 Train loss: 0.086261 Train acc: 0.971667\n",
      "Epoch: 805/1000 Iteration: 7250 Validation loss: 0.097182 Validation acc: 0.965555\n",
      "Epoch: 806/1000 Iteration: 7255 Train loss: 0.085490 Train acc: 0.965000\n",
      "Epoch: 806/1000 Iteration: 7260 Train loss: 0.076317 Train acc: 0.976667\n",
      "Epoch: 807/1000 Iteration: 7265 Train loss: 0.082968 Train acc: 0.975000\n",
      "Epoch: 807/1000 Iteration: 7270 Train loss: 0.080018 Train acc: 0.975000\n",
      "Epoch: 808/1000 Iteration: 7275 Train loss: 0.112911 Train acc: 0.965000\n",
      "Epoch: 808/1000 Iteration: 7275 Validation loss: 0.103716 Validation acc: 0.966667\n",
      "Epoch: 808/1000 Iteration: 7280 Train loss: 0.068280 Train acc: 0.981667\n",
      "Epoch: 809/1000 Iteration: 7285 Train loss: 0.114165 Train acc: 0.965000\n",
      "Epoch: 809/1000 Iteration: 7290 Train loss: 0.096739 Train acc: 0.966667\n",
      "Epoch: 810/1000 Iteration: 7295 Train loss: 0.106860 Train acc: 0.953333\n",
      "Epoch: 811/1000 Iteration: 7300 Train loss: 0.088938 Train acc: 0.968333\n",
      "Epoch: 811/1000 Iteration: 7300 Validation loss: 0.107950 Validation acc: 0.958333\n",
      "Epoch: 811/1000 Iteration: 7305 Train loss: 0.079380 Train acc: 0.975000\n",
      "Epoch: 812/1000 Iteration: 7310 Train loss: 0.110697 Train acc: 0.973333\n",
      "Epoch: 812/1000 Iteration: 7315 Train loss: 0.093631 Train acc: 0.970000\n",
      "Epoch: 813/1000 Iteration: 7320 Train loss: 0.125088 Train acc: 0.960000\n",
      "Epoch: 813/1000 Iteration: 7325 Train loss: 0.063386 Train acc: 0.983333\n",
      "Epoch: 813/1000 Iteration: 7325 Validation loss: 0.102356 Validation acc: 0.964444\n",
      "Epoch: 814/1000 Iteration: 7330 Train loss: 0.110926 Train acc: 0.956667\n",
      "Epoch: 814/1000 Iteration: 7335 Train loss: 0.096418 Train acc: 0.970000\n",
      "Epoch: 815/1000 Iteration: 7340 Train loss: 0.095955 Train acc: 0.963333\n",
      "Epoch: 816/1000 Iteration: 7345 Train loss: 0.092022 Train acc: 0.965000\n",
      "Epoch: 816/1000 Iteration: 7350 Train loss: 0.079068 Train acc: 0.971667\n",
      "Epoch: 816/1000 Iteration: 7350 Validation loss: 0.095312 Validation acc: 0.966111\n",
      "Epoch: 817/1000 Iteration: 7355 Train loss: 0.094957 Train acc: 0.973333\n",
      "Epoch: 817/1000 Iteration: 7360 Train loss: 0.091369 Train acc: 0.971667\n",
      "Epoch: 818/1000 Iteration: 7365 Train loss: 0.120399 Train acc: 0.956667\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 818/1000 Iteration: 7370 Train loss: 0.068612 Train acc: 0.971667\n",
      "Epoch: 819/1000 Iteration: 7375 Train loss: 0.120431 Train acc: 0.958333\n",
      "Epoch: 819/1000 Iteration: 7375 Validation loss: 0.095039 Validation acc: 0.966667\n",
      "Epoch: 819/1000 Iteration: 7380 Train loss: 0.100865 Train acc: 0.966667\n",
      "Epoch: 820/1000 Iteration: 7385 Train loss: 0.082009 Train acc: 0.973333\n",
      "Epoch: 821/1000 Iteration: 7390 Train loss: 0.087075 Train acc: 0.963333\n",
      "Epoch: 821/1000 Iteration: 7395 Train loss: 0.068129 Train acc: 0.973333\n",
      "Epoch: 822/1000 Iteration: 7400 Train loss: 0.102721 Train acc: 0.968333\n",
      "Epoch: 822/1000 Iteration: 7400 Validation loss: 0.102878 Validation acc: 0.963333\n",
      "Epoch: 822/1000 Iteration: 7405 Train loss: 0.082046 Train acc: 0.966667\n",
      "Epoch: 823/1000 Iteration: 7410 Train loss: 0.130989 Train acc: 0.953333\n",
      "Epoch: 823/1000 Iteration: 7415 Train loss: 0.070452 Train acc: 0.980000\n",
      "Epoch: 824/1000 Iteration: 7420 Train loss: 0.150311 Train acc: 0.941667\n",
      "Epoch: 824/1000 Iteration: 7425 Train loss: 0.099576 Train acc: 0.971667\n",
      "Epoch: 824/1000 Iteration: 7425 Validation loss: 0.098666 Validation acc: 0.965000\n",
      "Epoch: 825/1000 Iteration: 7430 Train loss: 0.100830 Train acc: 0.965000\n",
      "Epoch: 826/1000 Iteration: 7435 Train loss: 0.079803 Train acc: 0.966667\n",
      "Epoch: 826/1000 Iteration: 7440 Train loss: 0.078226 Train acc: 0.970000\n",
      "Epoch: 827/1000 Iteration: 7445 Train loss: 0.088977 Train acc: 0.973333\n",
      "Epoch: 827/1000 Iteration: 7450 Train loss: 0.100790 Train acc: 0.968333\n",
      "Epoch: 827/1000 Iteration: 7450 Validation loss: 0.099428 Validation acc: 0.966111\n",
      "Epoch: 828/1000 Iteration: 7455 Train loss: 0.111849 Train acc: 0.961667\n",
      "Epoch: 828/1000 Iteration: 7460 Train loss: 0.070532 Train acc: 0.976667\n",
      "Epoch: 829/1000 Iteration: 7465 Train loss: 0.123108 Train acc: 0.950000\n",
      "Epoch: 829/1000 Iteration: 7470 Train loss: 0.094087 Train acc: 0.978333\n",
      "Epoch: 830/1000 Iteration: 7475 Train loss: 0.099332 Train acc: 0.961667\n",
      "Epoch: 830/1000 Iteration: 7475 Validation loss: 0.097604 Validation acc: 0.966111\n",
      "Epoch: 831/1000 Iteration: 7480 Train loss: 0.099227 Train acc: 0.948333\n",
      "Epoch: 831/1000 Iteration: 7485 Train loss: 0.082640 Train acc: 0.970000\n",
      "Epoch: 832/1000 Iteration: 7490 Train loss: 0.103390 Train acc: 0.973333\n",
      "Epoch: 832/1000 Iteration: 7495 Train loss: 0.086149 Train acc: 0.971667\n",
      "Epoch: 833/1000 Iteration: 7500 Train loss: 0.108037 Train acc: 0.968333\n",
      "Epoch: 833/1000 Iteration: 7500 Validation loss: 0.102383 Validation acc: 0.963333\n",
      "Epoch: 833/1000 Iteration: 7505 Train loss: 0.071093 Train acc: 0.980000\n",
      "Epoch: 834/1000 Iteration: 7510 Train loss: 0.101446 Train acc: 0.961667\n",
      "Epoch: 834/1000 Iteration: 7515 Train loss: 0.086252 Train acc: 0.970000\n",
      "Epoch: 835/1000 Iteration: 7520 Train loss: 0.097336 Train acc: 0.970000\n",
      "Epoch: 836/1000 Iteration: 7525 Train loss: 0.089002 Train acc: 0.966667\n",
      "Epoch: 836/1000 Iteration: 7525 Validation loss: 0.110833 Validation acc: 0.963333\n",
      "Epoch: 836/1000 Iteration: 7530 Train loss: 0.068024 Train acc: 0.978333\n",
      "Epoch: 837/1000 Iteration: 7535 Train loss: 0.107211 Train acc: 0.971667\n",
      "Epoch: 837/1000 Iteration: 7540 Train loss: 0.088343 Train acc: 0.973333\n",
      "Epoch: 838/1000 Iteration: 7545 Train loss: 0.133558 Train acc: 0.961667\n",
      "Epoch: 838/1000 Iteration: 7550 Train loss: 0.064208 Train acc: 0.980000\n",
      "Epoch: 838/1000 Iteration: 7550 Validation loss: 0.107506 Validation acc: 0.962222\n",
      "Epoch: 839/1000 Iteration: 7555 Train loss: 0.113495 Train acc: 0.958333\n",
      "Epoch: 839/1000 Iteration: 7560 Train loss: 0.084973 Train acc: 0.970000\n",
      "Epoch: 840/1000 Iteration: 7565 Train loss: 0.105339 Train acc: 0.968333\n",
      "Epoch: 841/1000 Iteration: 7570 Train loss: 0.081375 Train acc: 0.965000\n",
      "Epoch: 841/1000 Iteration: 7575 Train loss: 0.074362 Train acc: 0.978333\n",
      "Epoch: 841/1000 Iteration: 7575 Validation loss: 0.096992 Validation acc: 0.966667\n",
      "Epoch: 842/1000 Iteration: 7580 Train loss: 0.105102 Train acc: 0.973333\n",
      "Epoch: 842/1000 Iteration: 7585 Train loss: 0.090609 Train acc: 0.971667\n",
      "Epoch: 843/1000 Iteration: 7590 Train loss: 0.113850 Train acc: 0.961667\n",
      "Epoch: 843/1000 Iteration: 7595 Train loss: 0.062114 Train acc: 0.983333\n",
      "Epoch: 844/1000 Iteration: 7600 Train loss: 0.103044 Train acc: 0.961667\n",
      "Epoch: 844/1000 Iteration: 7600 Validation loss: 0.114231 Validation acc: 0.963333\n",
      "Epoch: 844/1000 Iteration: 7605 Train loss: 0.103125 Train acc: 0.966667\n",
      "Epoch: 845/1000 Iteration: 7610 Train loss: 0.084600 Train acc: 0.973333\n",
      "Epoch: 846/1000 Iteration: 7615 Train loss: 0.088029 Train acc: 0.960000\n",
      "Epoch: 846/1000 Iteration: 7620 Train loss: 0.068421 Train acc: 0.976667\n",
      "Epoch: 847/1000 Iteration: 7625 Train loss: 0.095244 Train acc: 0.973333\n",
      "Epoch: 847/1000 Iteration: 7625 Validation loss: 0.102463 Validation acc: 0.965000\n",
      "Epoch: 847/1000 Iteration: 7630 Train loss: 0.084450 Train acc: 0.975000\n",
      "Epoch: 848/1000 Iteration: 7635 Train loss: 0.116534 Train acc: 0.958333\n",
      "Epoch: 848/1000 Iteration: 7640 Train loss: 0.080202 Train acc: 0.976667\n",
      "Epoch: 849/1000 Iteration: 7645 Train loss: 0.102135 Train acc: 0.958333\n",
      "Epoch: 849/1000 Iteration: 7650 Train loss: 0.097964 Train acc: 0.971667\n",
      "Epoch: 849/1000 Iteration: 7650 Validation loss: 0.105536 Validation acc: 0.963333\n",
      "Epoch: 850/1000 Iteration: 7655 Train loss: 0.094184 Train acc: 0.973333\n",
      "Epoch: 851/1000 Iteration: 7660 Train loss: 0.080414 Train acc: 0.968333\n",
      "Epoch: 851/1000 Iteration: 7665 Train loss: 0.073573 Train acc: 0.975000\n",
      "Epoch: 852/1000 Iteration: 7670 Train loss: 0.099229 Train acc: 0.970000\n",
      "Epoch: 852/1000 Iteration: 7675 Train loss: 0.086891 Train acc: 0.970000\n",
      "Epoch: 852/1000 Iteration: 7675 Validation loss: 0.100737 Validation acc: 0.964444\n",
      "Epoch: 853/1000 Iteration: 7680 Train loss: 0.117805 Train acc: 0.958333\n",
      "Epoch: 853/1000 Iteration: 7685 Train loss: 0.070905 Train acc: 0.976667\n",
      "Epoch: 854/1000 Iteration: 7690 Train loss: 0.098926 Train acc: 0.956667\n",
      "Epoch: 854/1000 Iteration: 7695 Train loss: 0.112375 Train acc: 0.961667\n",
      "Epoch: 855/1000 Iteration: 7700 Train loss: 0.099640 Train acc: 0.966667\n",
      "Epoch: 855/1000 Iteration: 7700 Validation loss: 0.118727 Validation acc: 0.963889\n",
      "Epoch: 856/1000 Iteration: 7705 Train loss: 0.093978 Train acc: 0.960000\n",
      "Epoch: 856/1000 Iteration: 7710 Train loss: 0.078152 Train acc: 0.973333\n",
      "Epoch: 857/1000 Iteration: 7715 Train loss: 0.095800 Train acc: 0.976667\n",
      "Epoch: 857/1000 Iteration: 7720 Train loss: 0.102676 Train acc: 0.968333\n",
      "Epoch: 858/1000 Iteration: 7725 Train loss: 0.105554 Train acc: 0.960000\n",
      "Epoch: 858/1000 Iteration: 7725 Validation loss: 0.110625 Validation acc: 0.959444\n",
      "Epoch: 858/1000 Iteration: 7730 Train loss: 0.066895 Train acc: 0.983333\n",
      "Epoch: 859/1000 Iteration: 7735 Train loss: 0.105156 Train acc: 0.960000\n",
      "Epoch: 859/1000 Iteration: 7740 Train loss: 0.096979 Train acc: 0.970000\n",
      "Epoch: 860/1000 Iteration: 7745 Train loss: 0.102858 Train acc: 0.970000\n",
      "Epoch: 861/1000 Iteration: 7750 Train loss: 0.090319 Train acc: 0.970000\n",
      "Epoch: 861/1000 Iteration: 7750 Validation loss: 0.117968 Validation acc: 0.961667\n",
      "Epoch: 861/1000 Iteration: 7755 Train loss: 0.076244 Train acc: 0.968333\n",
      "Epoch: 862/1000 Iteration: 7760 Train loss: 0.120751 Train acc: 0.968333\n",
      "Epoch: 862/1000 Iteration: 7765 Train loss: 0.094394 Train acc: 0.971667\n",
      "Epoch: 863/1000 Iteration: 7770 Train loss: 0.113327 Train acc: 0.961667\n",
      "Epoch: 863/1000 Iteration: 7775 Train loss: 0.072787 Train acc: 0.976667\n",
      "Epoch: 863/1000 Iteration: 7775 Validation loss: 0.098229 Validation acc: 0.965000\n",
      "Epoch: 864/1000 Iteration: 7780 Train loss: 0.110558 Train acc: 0.951667\n",
      "Epoch: 864/1000 Iteration: 7785 Train loss: 0.111340 Train acc: 0.970000\n",
      "Epoch: 865/1000 Iteration: 7790 Train loss: 0.078250 Train acc: 0.973333\n",
      "Epoch: 866/1000 Iteration: 7795 Train loss: 0.087858 Train acc: 0.970000\n",
      "Epoch: 866/1000 Iteration: 7800 Train loss: 0.074774 Train acc: 0.975000\n",
      "Epoch: 866/1000 Iteration: 7800 Validation loss: 0.106530 Validation acc: 0.964444\n",
      "Epoch: 867/1000 Iteration: 7805 Train loss: 0.103057 Train acc: 0.973333\n",
      "Epoch: 867/1000 Iteration: 7810 Train loss: 0.082649 Train acc: 0.975000\n",
      "Epoch: 868/1000 Iteration: 7815 Train loss: 0.108814 Train acc: 0.963333\n",
      "Epoch: 868/1000 Iteration: 7820 Train loss: 0.070654 Train acc: 0.976667\n",
      "Epoch: 869/1000 Iteration: 7825 Train loss: 0.110926 Train acc: 0.956667\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 869/1000 Iteration: 7825 Validation loss: 0.104867 Validation acc: 0.965000\n",
      "Epoch: 869/1000 Iteration: 7830 Train loss: 0.117411 Train acc: 0.963333\n",
      "Epoch: 870/1000 Iteration: 7835 Train loss: 0.075849 Train acc: 0.971667\n",
      "Epoch: 871/1000 Iteration: 7840 Train loss: 0.077148 Train acc: 0.971667\n",
      "Epoch: 871/1000 Iteration: 7845 Train loss: 0.072425 Train acc: 0.975000\n",
      "Epoch: 872/1000 Iteration: 7850 Train loss: 0.081115 Train acc: 0.978333\n",
      "Epoch: 872/1000 Iteration: 7850 Validation loss: 0.097555 Validation acc: 0.965000\n",
      "Epoch: 872/1000 Iteration: 7855 Train loss: 0.089222 Train acc: 0.970000\n",
      "Epoch: 873/1000 Iteration: 7860 Train loss: 0.105607 Train acc: 0.966667\n",
      "Epoch: 873/1000 Iteration: 7865 Train loss: 0.067445 Train acc: 0.981667\n",
      "Epoch: 874/1000 Iteration: 7870 Train loss: 0.100232 Train acc: 0.965000\n",
      "Epoch: 874/1000 Iteration: 7875 Train loss: 0.102002 Train acc: 0.966667\n",
      "Epoch: 874/1000 Iteration: 7875 Validation loss: 0.090454 Validation acc: 0.966667\n",
      "Epoch: 875/1000 Iteration: 7880 Train loss: 0.080049 Train acc: 0.975000\n",
      "Epoch: 876/1000 Iteration: 7885 Train loss: 0.085168 Train acc: 0.970000\n",
      "Epoch: 876/1000 Iteration: 7890 Train loss: 0.069925 Train acc: 0.973333\n",
      "Epoch: 877/1000 Iteration: 7895 Train loss: 0.097541 Train acc: 0.976667\n",
      "Epoch: 877/1000 Iteration: 7900 Train loss: 0.097764 Train acc: 0.970000\n",
      "Epoch: 877/1000 Iteration: 7900 Validation loss: 0.091105 Validation acc: 0.964444\n",
      "Epoch: 878/1000 Iteration: 7905 Train loss: 0.115080 Train acc: 0.963333\n",
      "Epoch: 878/1000 Iteration: 7910 Train loss: 0.065376 Train acc: 0.980000\n",
      "Epoch: 879/1000 Iteration: 7915 Train loss: 0.102663 Train acc: 0.955000\n",
      "Epoch: 879/1000 Iteration: 7920 Train loss: 0.099265 Train acc: 0.975000\n",
      "Epoch: 880/1000 Iteration: 7925 Train loss: 0.088258 Train acc: 0.968333\n",
      "Epoch: 880/1000 Iteration: 7925 Validation loss: 0.091648 Validation acc: 0.965555\n",
      "Epoch: 881/1000 Iteration: 7930 Train loss: 0.088923 Train acc: 0.970000\n",
      "Epoch: 881/1000 Iteration: 7935 Train loss: 0.072286 Train acc: 0.975000\n",
      "Epoch: 882/1000 Iteration: 7940 Train loss: 0.087073 Train acc: 0.976667\n",
      "Epoch: 882/1000 Iteration: 7945 Train loss: 0.080273 Train acc: 0.975000\n",
      "Epoch: 883/1000 Iteration: 7950 Train loss: 0.113944 Train acc: 0.953333\n",
      "Epoch: 883/1000 Iteration: 7950 Validation loss: 0.094489 Validation acc: 0.966111\n",
      "Epoch: 883/1000 Iteration: 7955 Train loss: 0.064943 Train acc: 0.975000\n",
      "Epoch: 884/1000 Iteration: 7960 Train loss: 0.112994 Train acc: 0.953333\n",
      "Epoch: 884/1000 Iteration: 7965 Train loss: 0.098741 Train acc: 0.970000\n",
      "Epoch: 885/1000 Iteration: 7970 Train loss: 0.080356 Train acc: 0.968333\n",
      "Epoch: 886/1000 Iteration: 7975 Train loss: 0.086942 Train acc: 0.963333\n",
      "Epoch: 886/1000 Iteration: 7975 Validation loss: 0.090127 Validation acc: 0.965000\n",
      "Epoch: 886/1000 Iteration: 7980 Train loss: 0.068099 Train acc: 0.975000\n",
      "Epoch: 887/1000 Iteration: 7985 Train loss: 0.101218 Train acc: 0.968333\n",
      "Epoch: 887/1000 Iteration: 7990 Train loss: 0.076927 Train acc: 0.978333\n",
      "Epoch: 888/1000 Iteration: 7995 Train loss: 0.114172 Train acc: 0.956667\n",
      "Epoch: 888/1000 Iteration: 8000 Train loss: 0.061108 Train acc: 0.986667\n",
      "Epoch: 888/1000 Iteration: 8000 Validation loss: 0.101210 Validation acc: 0.965556\n",
      "Epoch: 889/1000 Iteration: 8005 Train loss: 0.109448 Train acc: 0.958333\n",
      "Epoch: 889/1000 Iteration: 8010 Train loss: 0.093784 Train acc: 0.976667\n",
      "Epoch: 890/1000 Iteration: 8015 Train loss: 0.080943 Train acc: 0.978333\n",
      "Epoch: 891/1000 Iteration: 8020 Train loss: 0.084794 Train acc: 0.973333\n",
      "Epoch: 891/1000 Iteration: 8025 Train loss: 0.073234 Train acc: 0.971667\n",
      "Epoch: 891/1000 Iteration: 8025 Validation loss: 0.101576 Validation acc: 0.962778\n",
      "Epoch: 892/1000 Iteration: 8030 Train loss: 0.094753 Train acc: 0.971667\n",
      "Epoch: 892/1000 Iteration: 8035 Train loss: 0.083358 Train acc: 0.975000\n",
      "Epoch: 893/1000 Iteration: 8040 Train loss: 0.097063 Train acc: 0.958333\n",
      "Epoch: 893/1000 Iteration: 8045 Train loss: 0.071661 Train acc: 0.975000\n",
      "Epoch: 894/1000 Iteration: 8050 Train loss: 0.118826 Train acc: 0.958333\n",
      "Epoch: 894/1000 Iteration: 8050 Validation loss: 0.096257 Validation acc: 0.966111\n",
      "Epoch: 894/1000 Iteration: 8055 Train loss: 0.098215 Train acc: 0.971667\n",
      "Epoch: 895/1000 Iteration: 8060 Train loss: 0.083079 Train acc: 0.975000\n",
      "Epoch: 896/1000 Iteration: 8065 Train loss: 0.087283 Train acc: 0.971667\n",
      "Epoch: 896/1000 Iteration: 8070 Train loss: 0.070227 Train acc: 0.971667\n",
      "Epoch: 897/1000 Iteration: 8075 Train loss: 0.090689 Train acc: 0.971667\n",
      "Epoch: 897/1000 Iteration: 8075 Validation loss: 0.097791 Validation acc: 0.965000\n",
      "Epoch: 897/1000 Iteration: 8080 Train loss: 0.074838 Train acc: 0.976667\n",
      "Epoch: 898/1000 Iteration: 8085 Train loss: 0.100393 Train acc: 0.961667\n",
      "Epoch: 898/1000 Iteration: 8090 Train loss: 0.062141 Train acc: 0.980000\n",
      "Epoch: 899/1000 Iteration: 8095 Train loss: 0.102487 Train acc: 0.961667\n",
      "Epoch: 899/1000 Iteration: 8100 Train loss: 0.095013 Train acc: 0.976667\n",
      "Epoch: 899/1000 Iteration: 8100 Validation loss: 0.099710 Validation acc: 0.963333\n",
      "Epoch: 900/1000 Iteration: 8105 Train loss: 0.089049 Train acc: 0.963333\n",
      "Epoch: 901/1000 Iteration: 8110 Train loss: 0.087597 Train acc: 0.963333\n",
      "Epoch: 901/1000 Iteration: 8115 Train loss: 0.071291 Train acc: 0.973333\n",
      "Epoch: 902/1000 Iteration: 8120 Train loss: 0.084503 Train acc: 0.978333\n",
      "Epoch: 902/1000 Iteration: 8125 Train loss: 0.080054 Train acc: 0.976667\n",
      "Epoch: 902/1000 Iteration: 8125 Validation loss: 0.100812 Validation acc: 0.964444\n",
      "Epoch: 903/1000 Iteration: 8130 Train loss: 0.116790 Train acc: 0.958333\n",
      "Epoch: 903/1000 Iteration: 8135 Train loss: 0.068574 Train acc: 0.983333\n",
      "Epoch: 904/1000 Iteration: 8140 Train loss: 0.098998 Train acc: 0.961667\n",
      "Epoch: 904/1000 Iteration: 8145 Train loss: 0.092949 Train acc: 0.970000\n",
      "Epoch: 905/1000 Iteration: 8150 Train loss: 0.086186 Train acc: 0.973333\n",
      "Epoch: 905/1000 Iteration: 8150 Validation loss: 0.101526 Validation acc: 0.963889\n",
      "Epoch: 906/1000 Iteration: 8155 Train loss: 0.097704 Train acc: 0.958333\n",
      "Epoch: 906/1000 Iteration: 8160 Train loss: 0.066073 Train acc: 0.978333\n",
      "Epoch: 907/1000 Iteration: 8165 Train loss: 0.092184 Train acc: 0.965000\n",
      "Epoch: 907/1000 Iteration: 8170 Train loss: 0.081455 Train acc: 0.973333\n",
      "Epoch: 908/1000 Iteration: 8175 Train loss: 0.117809 Train acc: 0.965000\n",
      "Epoch: 908/1000 Iteration: 8175 Validation loss: 0.098196 Validation acc: 0.964444\n",
      "Epoch: 908/1000 Iteration: 8180 Train loss: 0.065084 Train acc: 0.981667\n",
      "Epoch: 909/1000 Iteration: 8185 Train loss: 0.112701 Train acc: 0.956667\n",
      "Epoch: 909/1000 Iteration: 8190 Train loss: 0.096414 Train acc: 0.970000\n",
      "Epoch: 910/1000 Iteration: 8195 Train loss: 0.088744 Train acc: 0.968333\n",
      "Epoch: 911/1000 Iteration: 8200 Train loss: 0.085232 Train acc: 0.961667\n",
      "Epoch: 911/1000 Iteration: 8200 Validation loss: 0.104606 Validation acc: 0.961111\n",
      "Epoch: 911/1000 Iteration: 8205 Train loss: 0.068967 Train acc: 0.973333\n",
      "Epoch: 912/1000 Iteration: 8210 Train loss: 0.092156 Train acc: 0.973333\n",
      "Epoch: 912/1000 Iteration: 8215 Train loss: 0.092232 Train acc: 0.970000\n",
      "Epoch: 913/1000 Iteration: 8220 Train loss: 0.105225 Train acc: 0.961667\n",
      "Epoch: 913/1000 Iteration: 8225 Train loss: 0.069734 Train acc: 0.978333\n",
      "Epoch: 913/1000 Iteration: 8225 Validation loss: 0.093099 Validation acc: 0.966111\n",
      "Epoch: 914/1000 Iteration: 8230 Train loss: 0.105574 Train acc: 0.963333\n",
      "Epoch: 914/1000 Iteration: 8235 Train loss: 0.096662 Train acc: 0.970000\n",
      "Epoch: 915/1000 Iteration: 8240 Train loss: 0.084774 Train acc: 0.973333\n",
      "Epoch: 916/1000 Iteration: 8245 Train loss: 0.083658 Train acc: 0.976667\n",
      "Epoch: 916/1000 Iteration: 8250 Train loss: 0.066620 Train acc: 0.978333\n",
      "Epoch: 916/1000 Iteration: 8250 Validation loss: 0.100633 Validation acc: 0.963333\n",
      "Epoch: 917/1000 Iteration: 8255 Train loss: 0.082633 Train acc: 0.971667\n",
      "Epoch: 917/1000 Iteration: 8260 Train loss: 0.075706 Train acc: 0.976667\n",
      "Epoch: 918/1000 Iteration: 8265 Train loss: 0.108304 Train acc: 0.963333\n",
      "Epoch: 918/1000 Iteration: 8270 Train loss: 0.061624 Train acc: 0.980000\n",
      "Epoch: 919/1000 Iteration: 8275 Train loss: 0.103912 Train acc: 0.960000\n",
      "Epoch: 919/1000 Iteration: 8275 Validation loss: 0.100950 Validation acc: 0.965000\n",
      "Epoch: 919/1000 Iteration: 8280 Train loss: 0.081838 Train acc: 0.978333\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 920/1000 Iteration: 8285 Train loss: 0.085784 Train acc: 0.968333\n",
      "Epoch: 921/1000 Iteration: 8290 Train loss: 0.078234 Train acc: 0.975000\n",
      "Epoch: 921/1000 Iteration: 8295 Train loss: 0.070400 Train acc: 0.973333\n",
      "Epoch: 922/1000 Iteration: 8300 Train loss: 0.092852 Train acc: 0.973333\n",
      "Epoch: 922/1000 Iteration: 8300 Validation loss: 0.097236 Validation acc: 0.968333\n",
      "Epoch: 922/1000 Iteration: 8305 Train loss: 0.089074 Train acc: 0.978333\n",
      "Epoch: 923/1000 Iteration: 8310 Train loss: 0.130010 Train acc: 0.951667\n",
      "Epoch: 923/1000 Iteration: 8315 Train loss: 0.074303 Train acc: 0.975000\n",
      "Epoch: 924/1000 Iteration: 8320 Train loss: 0.121350 Train acc: 0.951667\n",
      "Epoch: 924/1000 Iteration: 8325 Train loss: 0.091539 Train acc: 0.975000\n",
      "Epoch: 924/1000 Iteration: 8325 Validation loss: 0.108046 Validation acc: 0.959444\n",
      "Epoch: 925/1000 Iteration: 8330 Train loss: 0.092219 Train acc: 0.965000\n",
      "Epoch: 926/1000 Iteration: 8335 Train loss: 0.078387 Train acc: 0.975000\n",
      "Epoch: 926/1000 Iteration: 8340 Train loss: 0.075978 Train acc: 0.976667\n",
      "Epoch: 927/1000 Iteration: 8345 Train loss: 0.085048 Train acc: 0.976667\n",
      "Epoch: 927/1000 Iteration: 8350 Train loss: 0.096346 Train acc: 0.970000\n",
      "Epoch: 927/1000 Iteration: 8350 Validation loss: 0.103012 Validation acc: 0.962222\n",
      "Epoch: 928/1000 Iteration: 8355 Train loss: 0.119890 Train acc: 0.955000\n",
      "Epoch: 928/1000 Iteration: 8360 Train loss: 0.065912 Train acc: 0.975000\n",
      "Epoch: 929/1000 Iteration: 8365 Train loss: 0.107635 Train acc: 0.953333\n",
      "Epoch: 929/1000 Iteration: 8370 Train loss: 0.094699 Train acc: 0.975000\n",
      "Epoch: 930/1000 Iteration: 8375 Train loss: 0.083026 Train acc: 0.973333\n",
      "Epoch: 930/1000 Iteration: 8375 Validation loss: 0.108343 Validation acc: 0.962222\n",
      "Epoch: 931/1000 Iteration: 8380 Train loss: 0.080210 Train acc: 0.971667\n",
      "Epoch: 931/1000 Iteration: 8385 Train loss: 0.074399 Train acc: 0.970000\n",
      "Epoch: 932/1000 Iteration: 8390 Train loss: 0.099283 Train acc: 0.971667\n",
      "Epoch: 932/1000 Iteration: 8395 Train loss: 0.087005 Train acc: 0.973333\n",
      "Epoch: 933/1000 Iteration: 8400 Train loss: 0.112584 Train acc: 0.968333\n",
      "Epoch: 933/1000 Iteration: 8400 Validation loss: 0.095583 Validation acc: 0.965000\n",
      "Epoch: 933/1000 Iteration: 8405 Train loss: 0.067059 Train acc: 0.983333\n",
      "Epoch: 934/1000 Iteration: 8410 Train loss: 0.101040 Train acc: 0.966667\n",
      "Epoch: 934/1000 Iteration: 8415 Train loss: 0.083085 Train acc: 0.975000\n",
      "Epoch: 935/1000 Iteration: 8420 Train loss: 0.078529 Train acc: 0.968333\n",
      "Epoch: 936/1000 Iteration: 8425 Train loss: 0.084233 Train acc: 0.965000\n",
      "Epoch: 936/1000 Iteration: 8425 Validation loss: 0.096317 Validation acc: 0.965000\n",
      "Epoch: 936/1000 Iteration: 8430 Train loss: 0.066931 Train acc: 0.973333\n",
      "Epoch: 937/1000 Iteration: 8435 Train loss: 0.089783 Train acc: 0.978333\n",
      "Epoch: 937/1000 Iteration: 8440 Train loss: 0.085136 Train acc: 0.971667\n",
      "Epoch: 938/1000 Iteration: 8445 Train loss: 0.100453 Train acc: 0.971667\n",
      "Epoch: 938/1000 Iteration: 8450 Train loss: 0.064028 Train acc: 0.976667\n",
      "Epoch: 938/1000 Iteration: 8450 Validation loss: 0.099957 Validation acc: 0.962222\n",
      "Epoch: 939/1000 Iteration: 8455 Train loss: 0.131055 Train acc: 0.951667\n",
      "Epoch: 939/1000 Iteration: 8460 Train loss: 0.098968 Train acc: 0.971667\n",
      "Epoch: 940/1000 Iteration: 8465 Train loss: 0.102700 Train acc: 0.968333\n",
      "Epoch: 941/1000 Iteration: 8470 Train loss: 0.082402 Train acc: 0.970000\n",
      "Epoch: 941/1000 Iteration: 8475 Train loss: 0.068227 Train acc: 0.970000\n",
      "Epoch: 941/1000 Iteration: 8475 Validation loss: 0.105131 Validation acc: 0.958333\n",
      "Epoch: 942/1000 Iteration: 8480 Train loss: 0.096597 Train acc: 0.971667\n",
      "Epoch: 942/1000 Iteration: 8485 Train loss: 0.092286 Train acc: 0.975000\n",
      "Epoch: 943/1000 Iteration: 8490 Train loss: 0.107046 Train acc: 0.960000\n",
      "Epoch: 943/1000 Iteration: 8495 Train loss: 0.067994 Train acc: 0.978333\n",
      "Epoch: 944/1000 Iteration: 8500 Train loss: 0.104612 Train acc: 0.956667\n",
      "Epoch: 944/1000 Iteration: 8500 Validation loss: 0.108252 Validation acc: 0.963889\n",
      "Epoch: 944/1000 Iteration: 8505 Train loss: 0.089557 Train acc: 0.970000\n",
      "Epoch: 945/1000 Iteration: 8510 Train loss: 0.078397 Train acc: 0.975000\n",
      "Epoch: 946/1000 Iteration: 8515 Train loss: 0.079958 Train acc: 0.973333\n",
      "Epoch: 946/1000 Iteration: 8520 Train loss: 0.068567 Train acc: 0.973333\n",
      "Epoch: 947/1000 Iteration: 8525 Train loss: 0.092594 Train acc: 0.971667\n",
      "Epoch: 947/1000 Iteration: 8525 Validation loss: 0.105351 Validation acc: 0.963889\n",
      "Epoch: 947/1000 Iteration: 8530 Train loss: 0.080771 Train acc: 0.978333\n",
      "Epoch: 948/1000 Iteration: 8535 Train loss: 0.117414 Train acc: 0.966667\n",
      "Epoch: 948/1000 Iteration: 8540 Train loss: 0.065406 Train acc: 0.981667\n",
      "Epoch: 949/1000 Iteration: 8545 Train loss: 0.097945 Train acc: 0.958333\n",
      "Epoch: 949/1000 Iteration: 8550 Train loss: 0.087580 Train acc: 0.973333\n",
      "Epoch: 949/1000 Iteration: 8550 Validation loss: 0.105047 Validation acc: 0.966667\n",
      "Epoch: 950/1000 Iteration: 8555 Train loss: 0.077382 Train acc: 0.975000\n",
      "Epoch: 951/1000 Iteration: 8560 Train loss: 0.084965 Train acc: 0.966667\n",
      "Epoch: 951/1000 Iteration: 8565 Train loss: 0.064809 Train acc: 0.980000\n",
      "Epoch: 952/1000 Iteration: 8570 Train loss: 0.079290 Train acc: 0.976667\n",
      "Epoch: 952/1000 Iteration: 8575 Train loss: 0.095053 Train acc: 0.971667\n",
      "Epoch: 952/1000 Iteration: 8575 Validation loss: 0.109899 Validation acc: 0.964444\n",
      "Epoch: 953/1000 Iteration: 8580 Train loss: 0.105679 Train acc: 0.970000\n",
      "Epoch: 953/1000 Iteration: 8585 Train loss: 0.059226 Train acc: 0.981667\n",
      "Epoch: 954/1000 Iteration: 8590 Train loss: 0.110239 Train acc: 0.956667\n",
      "Epoch: 954/1000 Iteration: 8595 Train loss: 0.090632 Train acc: 0.976667\n",
      "Epoch: 955/1000 Iteration: 8600 Train loss: 0.097975 Train acc: 0.963333\n",
      "Epoch: 955/1000 Iteration: 8600 Validation loss: 0.108536 Validation acc: 0.963333\n",
      "Epoch: 956/1000 Iteration: 8605 Train loss: 0.082935 Train acc: 0.966667\n",
      "Epoch: 956/1000 Iteration: 8610 Train loss: 0.065164 Train acc: 0.976667\n",
      "Epoch: 957/1000 Iteration: 8615 Train loss: 0.091045 Train acc: 0.975000\n",
      "Epoch: 957/1000 Iteration: 8620 Train loss: 0.081072 Train acc: 0.970000\n",
      "Epoch: 958/1000 Iteration: 8625 Train loss: 0.126712 Train acc: 0.956667\n",
      "Epoch: 958/1000 Iteration: 8625 Validation loss: 0.122654 Validation acc: 0.963889\n",
      "Epoch: 958/1000 Iteration: 8630 Train loss: 0.060181 Train acc: 0.976667\n",
      "Epoch: 959/1000 Iteration: 8635 Train loss: 0.123236 Train acc: 0.955000\n",
      "Epoch: 959/1000 Iteration: 8640 Train loss: 0.102329 Train acc: 0.965000\n",
      "Epoch: 960/1000 Iteration: 8645 Train loss: 0.106783 Train acc: 0.956667\n",
      "Epoch: 961/1000 Iteration: 8650 Train loss: 0.085758 Train acc: 0.968333\n",
      "Epoch: 961/1000 Iteration: 8650 Validation loss: 0.104616 Validation acc: 0.966111\n",
      "Epoch: 961/1000 Iteration: 8655 Train loss: 0.064991 Train acc: 0.971667\n",
      "Epoch: 962/1000 Iteration: 8660 Train loss: 0.097075 Train acc: 0.971667\n",
      "Epoch: 962/1000 Iteration: 8665 Train loss: 0.089501 Train acc: 0.975000\n",
      "Epoch: 963/1000 Iteration: 8670 Train loss: 0.116298 Train acc: 0.960000\n",
      "Epoch: 963/1000 Iteration: 8675 Train loss: 0.095840 Train acc: 0.971667\n",
      "Epoch: 963/1000 Iteration: 8675 Validation loss: 0.111974 Validation acc: 0.964444\n",
      "Epoch: 964/1000 Iteration: 8680 Train loss: 0.105047 Train acc: 0.963333\n",
      "Epoch: 964/1000 Iteration: 8685 Train loss: 0.101577 Train acc: 0.965000\n",
      "Epoch: 965/1000 Iteration: 8690 Train loss: 0.088077 Train acc: 0.966667\n",
      "Epoch: 966/1000 Iteration: 8695 Train loss: 0.072366 Train acc: 0.973333\n",
      "Epoch: 966/1000 Iteration: 8700 Train loss: 0.072768 Train acc: 0.975000\n",
      "Epoch: 966/1000 Iteration: 8700 Validation loss: 0.089579 Validation acc: 0.968889\n",
      "Epoch: 967/1000 Iteration: 8705 Train loss: 0.105066 Train acc: 0.966667\n",
      "Epoch: 967/1000 Iteration: 8710 Train loss: 0.095245 Train acc: 0.971667\n",
      "Epoch: 968/1000 Iteration: 8715 Train loss: 0.104111 Train acc: 0.960000\n",
      "Epoch: 968/1000 Iteration: 8720 Train loss: 0.071535 Train acc: 0.975000\n",
      "Epoch: 969/1000 Iteration: 8725 Train loss: 0.104204 Train acc: 0.958333\n",
      "Epoch: 969/1000 Iteration: 8725 Validation loss: 0.086579 Validation acc: 0.967222\n",
      "Epoch: 969/1000 Iteration: 8730 Train loss: 0.096244 Train acc: 0.975000\n",
      "Epoch: 970/1000 Iteration: 8735 Train loss: 0.094508 Train acc: 0.966667\n",
      "Epoch: 971/1000 Iteration: 8740 Train loss: 0.086304 Train acc: 0.965000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 971/1000 Iteration: 8745 Train loss: 0.063238 Train acc: 0.978333\n",
      "Epoch: 972/1000 Iteration: 8750 Train loss: 0.102519 Train acc: 0.973333\n",
      "Epoch: 972/1000 Iteration: 8750 Validation loss: 0.087862 Validation acc: 0.965556\n",
      "Epoch: 972/1000 Iteration: 8755 Train loss: 0.086379 Train acc: 0.968333\n",
      "Epoch: 973/1000 Iteration: 8760 Train loss: 0.114018 Train acc: 0.963333\n",
      "Epoch: 973/1000 Iteration: 8765 Train loss: 0.061843 Train acc: 0.975000\n",
      "Epoch: 974/1000 Iteration: 8770 Train loss: 0.110969 Train acc: 0.958333\n",
      "Epoch: 974/1000 Iteration: 8775 Train loss: 0.101678 Train acc: 0.965000\n",
      "Epoch: 974/1000 Iteration: 8775 Validation loss: 0.078183 Validation acc: 0.968333\n",
      "Epoch: 975/1000 Iteration: 8780 Train loss: 0.090325 Train acc: 0.971667\n",
      "Epoch: 976/1000 Iteration: 8785 Train loss: 0.077971 Train acc: 0.966667\n",
      "Epoch: 976/1000 Iteration: 8790 Train loss: 0.074086 Train acc: 0.976667\n",
      "Epoch: 977/1000 Iteration: 8795 Train loss: 0.099275 Train acc: 0.970000\n",
      "Epoch: 977/1000 Iteration: 8800 Train loss: 0.073003 Train acc: 0.983333\n",
      "Epoch: 977/1000 Iteration: 8800 Validation loss: 0.089361 Validation acc: 0.965556\n",
      "Epoch: 978/1000 Iteration: 8805 Train loss: 0.108436 Train acc: 0.960000\n",
      "Epoch: 978/1000 Iteration: 8810 Train loss: 0.061164 Train acc: 0.976667\n",
      "Epoch: 979/1000 Iteration: 8815 Train loss: 0.098416 Train acc: 0.960000\n",
      "Epoch: 979/1000 Iteration: 8820 Train loss: 0.093326 Train acc: 0.973333\n",
      "Epoch: 980/1000 Iteration: 8825 Train loss: 0.079092 Train acc: 0.973333\n",
      "Epoch: 980/1000 Iteration: 8825 Validation loss: 0.092570 Validation acc: 0.965556\n",
      "Epoch: 981/1000 Iteration: 8830 Train loss: 0.081421 Train acc: 0.970000\n",
      "Epoch: 981/1000 Iteration: 8835 Train loss: 0.068517 Train acc: 0.975000\n",
      "Epoch: 982/1000 Iteration: 8840 Train loss: 0.089785 Train acc: 0.968333\n",
      "Epoch: 982/1000 Iteration: 8845 Train loss: 0.081964 Train acc: 0.978333\n",
      "Epoch: 983/1000 Iteration: 8850 Train loss: 0.109579 Train acc: 0.956667\n",
      "Epoch: 983/1000 Iteration: 8850 Validation loss: 0.089705 Validation acc: 0.966111\n",
      "Epoch: 983/1000 Iteration: 8855 Train loss: 0.065082 Train acc: 0.980000\n",
      "Epoch: 984/1000 Iteration: 8860 Train loss: 0.113623 Train acc: 0.960000\n",
      "Epoch: 984/1000 Iteration: 8865 Train loss: 0.094210 Train acc: 0.966667\n",
      "Epoch: 985/1000 Iteration: 8870 Train loss: 0.084936 Train acc: 0.975000\n",
      "Epoch: 986/1000 Iteration: 8875 Train loss: 0.079864 Train acc: 0.971667\n",
      "Epoch: 986/1000 Iteration: 8875 Validation loss: 0.085945 Validation acc: 0.967778\n",
      "Epoch: 986/1000 Iteration: 8880 Train loss: 0.074253 Train acc: 0.971667\n",
      "Epoch: 987/1000 Iteration: 8885 Train loss: 0.107833 Train acc: 0.973333\n",
      "Epoch: 987/1000 Iteration: 8890 Train loss: 0.076366 Train acc: 0.975000\n",
      "Epoch: 988/1000 Iteration: 8895 Train loss: 0.109893 Train acc: 0.961667\n",
      "Epoch: 988/1000 Iteration: 8900 Train loss: 0.071025 Train acc: 0.980000\n",
      "Epoch: 988/1000 Iteration: 8900 Validation loss: 0.082895 Validation acc: 0.970000\n",
      "Epoch: 989/1000 Iteration: 8905 Train loss: 0.107576 Train acc: 0.966667\n",
      "Epoch: 989/1000 Iteration: 8910 Train loss: 0.088182 Train acc: 0.970000\n",
      "Epoch: 990/1000 Iteration: 8915 Train loss: 0.081758 Train acc: 0.973333\n",
      "Epoch: 991/1000 Iteration: 8920 Train loss: 0.077944 Train acc: 0.970000\n",
      "Epoch: 991/1000 Iteration: 8925 Train loss: 0.064937 Train acc: 0.978333\n",
      "Epoch: 991/1000 Iteration: 8925 Validation loss: 0.091577 Validation acc: 0.966111\n",
      "Epoch: 992/1000 Iteration: 8930 Train loss: 0.098500 Train acc: 0.966667\n",
      "Epoch: 992/1000 Iteration: 8935 Train loss: 0.078241 Train acc: 0.976667\n",
      "Epoch: 993/1000 Iteration: 8940 Train loss: 0.091339 Train acc: 0.963333\n",
      "Epoch: 993/1000 Iteration: 8945 Train loss: 0.069632 Train acc: 0.976667\n",
      "Epoch: 994/1000 Iteration: 8950 Train loss: 0.108118 Train acc: 0.960000\n",
      "Epoch: 994/1000 Iteration: 8950 Validation loss: 0.089557 Validation acc: 0.966111\n",
      "Epoch: 994/1000 Iteration: 8955 Train loss: 0.083917 Train acc: 0.971667\n",
      "Epoch: 995/1000 Iteration: 8960 Train loss: 0.078314 Train acc: 0.966667\n",
      "Epoch: 996/1000 Iteration: 8965 Train loss: 0.091319 Train acc: 0.966667\n",
      "Epoch: 996/1000 Iteration: 8970 Train loss: 0.063785 Train acc: 0.983333\n",
      "Epoch: 997/1000 Iteration: 8975 Train loss: 0.098082 Train acc: 0.968333\n",
      "Epoch: 997/1000 Iteration: 8975 Validation loss: 0.089292 Validation acc: 0.966111\n",
      "Epoch: 997/1000 Iteration: 8980 Train loss: 0.080445 Train acc: 0.973333\n",
      "Epoch: 998/1000 Iteration: 8985 Train loss: 0.099712 Train acc: 0.966667\n",
      "Epoch: 998/1000 Iteration: 8990 Train loss: 0.057408 Train acc: 0.983333\n",
      "Epoch: 999/1000 Iteration: 8995 Train loss: 0.090879 Train acc: 0.958333\n",
      "Epoch: 999/1000 Iteration: 9000 Train loss: 0.080198 Train acc: 0.975000\n",
      "Epoch: 999/1000 Iteration: 9000 Validation loss: 0.090981 Validation acc: 0.966667\n"
     ]
    }
   ],
   "source": [
    "validation_acc = []\n",
    "validation_loss = []\n",
    "\n",
    "train_acc = []\n",
    "train_loss = []\n",
    "\n",
    "with graph.as_default():\n",
    "    saver = tf.train.Saver()\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    iteration = 1\n",
    "    \n",
    "    for e in range(epochs):\n",
    "        # Initialize \n",
    "        state = sess.run(initial_state)\n",
    "        \n",
    "        # Loop over batches\n",
    "        for x,y in get_batches(X_tr, y_tr, batch_size):\n",
    "            \n",
    "            # Feed dictionary\n",
    "            feed = {inputs_ : x, labels_ : y, keep_prob_ : 0.5, \n",
    "                    initial_state : state, learning_rate_ : learning_rate}\n",
    "            \n",
    "            loss, _ , state, acc = sess.run([cost, optimizer, final_state, accuracy], feed_dict = feed)\n",
    "            \n",
    "            train_acc.append(acc)\n",
    "            train_loss.append(loss)\n",
    "            \n",
    "            # Print at each 5 iters\n",
    "            if (iteration % 5 == 0):\n",
    "                print(\"Epoch: {}/{}\".format(e, epochs),\n",
    "                      \"Iteration: {:d}\".format(iteration),\n",
    "                      \"Train loss: {:6f}\".format(loss),\n",
    "                      \"Train acc: {:.6f}\".format(acc))\n",
    "            \n",
    "            # Compute validation loss at every 25 iterations\n",
    "            if (iteration%25 == 0):\n",
    "                \n",
    "                # Initiate for validation set\n",
    "                val_state = sess.run(cell.zero_state(batch_size, tf.float32))\n",
    "                \n",
    "                val_acc_ = []\n",
    "                val_loss_ = []\n",
    "                for x_v, y_v in get_batches(X_vld, y_vld, batch_size):\n",
    "                    # Feed\n",
    "                    feed = {inputs_ : x_v, labels_ : y_v, keep_prob_ : 1.0, initial_state : val_state}\n",
    "                    \n",
    "                    # Loss\n",
    "                    loss_v, state_v, acc_v = sess.run([cost, final_state, accuracy], feed_dict = feed)\n",
    "                    \n",
    "                    val_acc_.append(acc_v)\n",
    "                    val_loss_.append(loss_v)\n",
    "                \n",
    "                # Print info\n",
    "                print(\"Epoch: {}/{}\".format(e, epochs),\n",
    "                      \"Iteration: {:d}\".format(iteration),\n",
    "                      \"Validation loss: {:6f}\".format(np.mean(val_loss_)),\n",
    "                      \"Validation acc: {:.6f}\".format(np.mean(val_acc_)))\n",
    "                \n",
    "                # Store\n",
    "                validation_acc.append(np.mean(val_acc_))\n",
    "                validation_loss.append(np.mean(val_loss_))\n",
    "            \n",
    "            # Iterate \n",
    "            iteration += 1\n",
    "    \n",
    "    saver.save(sess,\"checkpoints-crnn/har.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAF3CAYAAAC2bHyQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8nWWd///XJ+mStFDovrIURSiUrgFhQEpHhQIqtICAIII6le2HG+Ogzldn6IwyiohCKVZElN2BsohFEYdSSql2oS2lbKUtkDZdaKFbuiX5/P647uSck5wkJ8k5uU9y3s/H437kPvd65U5yf3Ld1319LnN3REREmlMUdwFERKRjUMAQEZGMKGCIiEhGFDBERCQjChgiIpIRBQwREcmIAoaIiGREAUNERDKigCEiIhlRwBARkYx0ibsA2dSvXz8//PDD4y6GiEiHsXjx4vfdvX8m23aqgHH44YezaNGiuIshItJhmNk7mW6rR1IiIpIRBQwREcmIAoaIiGSkU7VhiEjnsX//fsrLy9mzZ0/cRekUSkpKGDZsGF27dm31MRQwRCQvlZeXc+CBB3L44YdjZnEXp0Nzd7Zs2UJ5eTnDhw9v9XH0SEpE8tKePXvo27evgkUWmBl9+/Ztc21NAUNE8paCRfZk41oqYIiIpPHhhx9yxx13tHi/s846iw8//DAHJYqfAoaISBqNBYyqqqom95s9ezYHH3xwrooVKzV6i4ikccMNN/D2228zZswYunbtSklJCb179+b111/nzTff5Nxzz+W9995jz549fP3rX2fq1KlAIuPEzp07OfPMMznllFOYP38+Q4cO5YknnqC0tDTm76z1FDBEJP994xuwdGl2jzlmDNx6a6Orb7rpJlasWMHSpUuZM2cOZ599NitWrKh7y+juu++mT58+7N69m+OPP57zzjuPvn37phzjrbfe4sEHH+TXv/41n//853n00Ue59NJLs/t9tCM9kgLYuhU2bIi7FCKSx0444YSUV1J/+ctfMnr0aE488UTee+893nrrrQb7DB8+nDFjxgAwfvx41q5d217FzQnVMABq/ytwj7ccIpJeEzWB9tKzZ8+6+Tlz5vDss8/y0ksv0aNHD0477bS0r6x27969br64uJjdu3e3S1lzRTUMEZE0DjzwQHbs2JF23bZt2+jduzc9evTg9ddfZ8GCBe1cuniohiEikkbfvn05+eSTGTlyJKWlpQwcOLBu3aRJk7jzzjsZMWIERx11FCeeeGKMJW0/5p3oMUxZWZm3ajyM2g4tNTWJeRGJ1WuvvcaIESPiLkanku6amtlidy/LZP+cPZIys7vNbJOZrWhk/b+a2dJoWmFm1WbWJ1q31sxeida134hIv/hFu51KRKSjyWUbxj3ApMZWuvtP3X2Mu48Bvgs87+5bkzaZGK3PKPJlxTe/2W6nEhHpaHIWMNx9LrC12Q2Di4EHc1UWERFpu9jfkjKzHoSayKNJix141swWm9nUXJehgkFMYA4bGNj8xiIiBSr2gAF8Fnix3uOoU6JHVWcC15jZqY3tbGZTzWyRmS3avHlzqwow7cTZzOMUbuQHrdpfRKQQ5EPAuIh6j6PcfV30dRPwGHBCYzu7+0x3L3P3sv79+7foxKWl4aWoGQvGUkMxM7gas7BcRERSxRowzOwgYALwRNKynmZ2YO08cDqQ9k2rtlq9Gr7wBehRUgNAD3ZxyQX7WLMmF2cTkc7sgAMOAGD9+vWcf/75abc57bTTaO7V/1tvvZXKysq6z/mULj2Xr9U+CLwEHGVm5Wb2FTO70syuTNpsMvCMu+9KWjYQmGdmy4B/AH9y9z/nooyDB0OvXrBnn1HCbvZQQq83FzJoUC7OJiK5VlEBEybEmxpuyJAhPPLII63ev37AyKd06bl8S+pidx/s7l3dfZi7/8bd73T3O5O2ucfdL6q332p3Hx1Nx7r7f+eqjAAbN8KVVxoLOJEruZMNy5SEUKSjmjYN5s2DG29s+7FuuOEGpk+fXvf5P/7jP/iv//ovPvnJTzJu3DiOO+44nnjiiQb7rV27lpEjRwKwe/duLrroIkaMGMHkyZNTckldddVVlJWVceyxx/LDH/4QCAkN169fz8SJE5k4cSIQ0qW///77ANxyyy2MHDmSkSNHcmuUX2vt2rWMGDGCf/mXf+HYY4/l9NNPz13OKnfvNNP48eO91ULqwTCJSOxWrlyZ8bYlJal/wrVTSUnrz79kyRI/9dRT6z6PGDHC3333Xd+2bZu7u2/evNk/8pGPeE1Njbu79+zZ093d16xZ48cee6y7u//sZz/zK664wt3dly1b5sXFxb5w4UJ3d9+yZYu7u1dVVfmECRN82bJl7u5+2GGH+ebNm+vOW/t50aJFPnLkSN+5c6fv2LHDjznmGF+yZImvWbPGi4uL/eWXX3Z39wsuuMDvvffetN9TumsKLPIM77H50OgtItImde2RPcLnHj3gkktoU3vk2LFj2bRpE+vXr2fZsmX07t2bQYMG8b3vfY9Ro0bxqU99inXr1rFx48ZGjzF37ty68S9GjRrFqFGj6tb94Q9/YNy4cYwdO5ZXX32VlStXNlmeefPmMXnyZHr27MkBBxzAlClTeOGFF4D2S6Ou5IORil5HcdH2X/EwF6ImDJGOpa49cg+UlISvvXrR5vbICy64gEceeYQNGzZw4YUXcv/997N582YWL15M165dOfzww9OmNW/OmjVruPnmm1m4cCG9e/fm8ssvb9VxarVXGnXVMCLThsxQXwyRDiy0R8KCBeFrNhq+L7zwQh566CEeeeQRLrjgArZt28aAAQPo2rUrzz33HO+8806T+5966qk88MADAKxYsYLly5cDsH37dnr27MlBBx3Exo0befrpp+v2aSyt+ic+8Qkef/xxKisr2bVrF4899hif+MQn2v5NtkDB1zBKS8N/IxAamGZwNTMs/JfSwcc6ESkos2Yl5pPaqtvk2GOPZceOHQwdOpTBgwdzySWX8NnPfpbjjjuOsrIyjj766Cb3v+qqq7jiiisYMWIEI0aMYPz48QCMHj2asWPHcvTRR3PIIYdw8skn1+0zdepUJk2axJAhQ3juuefqlo8bN47LL7+cE04I3dK++tWvMnbs2HYdxa/g05tXVMD118Pjj1ZTubeYHuxi8iU9ufnmtldnRaT1lN48+/I2vXlHUffsc39Roi9GFp59ioh0NgUfMCB69vk1En0xKmriLpKISN4p+DYMqH32aTBjOdO5Fu6/AugRc6lERPKLahjpTJgQdwlEhNCxWLIjG9dSASNSUUFiTIzWjAsuIllVUlLCli1bFDSywN3ZsmULJSUlbTqOHklFpk2jrh/GHVwTd3FECt6wYcMoLy+ntePcSKqSkhKGDRvWpmMU/Gu1iX4YqdQPQ0QKgV6rbYEGOWjYxSXcpzExRETqKfiAkZKDpktV6IfBdvXDEBGpp+ADBiTloHlme+iHwcC4iyQiknfU6E1yDpo+oR8GAJ2nbUdEJBtUw2hMnoyhKyKSLxQwGnP55XGXQEQkryhgJEnpvJdmrF4RkUKmgJEkufOeiIikUsAgdN4zgxkzoIZiZnA1hlNaGnfJRETyhwIG6rwnIpIJBQzqdd6zPeq8JyKShvphRGo7703t+ydmTttABYoWIiLJFDAidZ33Nn2C6dNqe3qr856ISC09kqpvwIDEfCfK5Csi0lYKGE15++24SyAikjcUMJpSVRV3CURE8oYCRlO2bo27BCIieUMBo56U9CAnnxx3cURE8oYCRj1KDyIikp4CRkTpQUREmpazgGFmd5vZJjNb0cj608xsm5ktjaYfJK2bZGZvmNkqM7shV2VMpvQgIiJNy2UN4x5gUjPbvODuY6LpRgAzKwamA2cCxwAXm9kxOSwnUC89SPE+pQcREaknZwHD3ecCrXnN6ARglbuvdvd9wEPAOVktXCPqxvY++780treISD1xpwb5JzNbDqwDrnf3V4GhwHtJ25QDH2+PwtSlB7niPaYzLfqg3t4iIhBvo/cS4FB3HwXcBjzemoOY2VQzW2RmizZv3pydkv3kJ9k5johIJxJbwHD37e6+M5qfDXQ1s36E2sYhSZsOi5Y1dpyZ7l7m7mX9+/fPTuGSj6N8UiIiQIwBw8wGmZlF8ydEZdkCLASONLPhZtYNuAh4Mq5y8vrrsZ1aRCSf5PK12geBl4CjzKzczL5iZlea2ZXRJucDK8xsGfBL4CIPqoBrgb8ArwF/iNo22k1Kb+8f/rA9Ty0ikrfMO9Ejl7KyMl+0aFGbj3P11fCrGdV8jV9xR/F1SkIoIp2WmS1297JMtlVP7yRpe3tXV6m3t4gIChgp1NtbRKRxChhJUnp7s1u9vUVEkihg1FPX2/uYr6i3t4hIkrh7euedut7ec69k+oQJ0YfO82KAiEhrqYbRmH794i6BiEheUcBoTOhTKCIiEQWMxhQlXZp3342vHCIieUIBozHV1Yn5p56KrxwiInlCAaMRFQd+LJEepLIy7uKIiMROAaMR037chXmcwo38QO0ZIiIol1QDpaWh4159JSWwe3ebDi0ikneUS6oN0qYHGfuq0oOISMFTwKgnbXqQZfOUHkRECp4CRhp16UE4MaQHqVEnPhERtWE0JbmxuxNdJxGRWmrDEBGRrFPAEBGRjChgNOWII+IugYhI3lDAaMqvfhV3CURE8oYCRhMqPvqJRHqQlSvjLo6ISKwUMJow7SfdE+lBPvww7uKIiMRKI+6lkZoepJgZXM2Mk5UeREQKm2oYaaRND9LlYaUHEZGCpoCRRtr0IFVblB5ERAqaAkYjGqQHYWDcRRIRiZVSgzRH6UFEpBNTahAREck6BQwREcmIAoaIiGREAUNERDKigNGcIUMS8+kG+xYRKRAKGM2ouO7HiXxSe/fGXRwRkdgoYDRj2sIzEvmkREQKWM4ChpndbWabzGxFI+svMbPlZvaKmc03s9FJ69ZGy5eaWZY7VmSmtDR0wZjx6EBqonxSdvBBlJbGURoRkfjlsoZxDzCpifVrgAnufhwwDZhZb/1Edx+TaYeSbEubT+qCfconJSIFK2cBw93nAlubWD/f3T+IPi4AhuWqLK2RNp/UljXKJyUiBStf2jC+Ajyd9NmBZ81ssZlNjalMDfNJrdgcV1FERGIX+3gYZjaREDBOSVp8iruvM7MBwF/N7PWoxpJu/6nAVIBDDz00q2WbNSuauWM507kWDvwIsCqr5xAR6ShirWGY2SjgLuAcd99Su9zd10VfNwGPASc0dgx3n+nuZe5e1r9//9wW+O23c3t8EZE8FlvAMLNDgVnAF939zaTlPc3swNp54HQg7ZtWIiLSfnL2SMrMHgROA/qZWTnwQ6ArgLvfCfwA6AvcYSGFeFX0RtRA4LFoWRfgAXf/c67KKSIimclZwHD3i5tZ/1Xgq2mWrwZGN9xDRETilC9vSYmISJ5TwBARkYwoYDSjogImDHhNY3qLSMFTwGjGtGkwb/NRSj4oIgVPAaMRdckHZ0CNW0g+iCv5oIgULAWMRqRNPsh9Sj4oIgVLAaMRKckHSwjJB9nOoN4aRElECpMCRhPqkg8uICQfZCB88EHzO4qIdEKxJx/MZ3XJByEkHwTw9fEURkQkZqphtJR73CUQEYmFAkZL1dTEXQIRkVgoYLTUnj1xl0BEJBYKGC319a/HXQIRkVgoYLTU7Nlxl0BEJBYKGCIikhEFjAxUVMAE5igBoYgUNAWMDEybBvM4RQkIRaSgKWA0ISUBIcVKQCgiBU0BowlKQCgikqCA0YSUBITdaxIJCAfFXTIRkfangNGMugSET3+YSEAoIlKAlHywGXUJCDfsSyQgRPmkRKTwqIaRqT594i6BiEisFDAy1a1bYn7+/PjKISISEwWM1rjzzrhLICLS7hQwWuOll+IugYhIu1PAEBGRjChgtEZlZdwlEBFpdwoYrbFe43qLSOFRwBARkYwoYIiISEYUMDKkMTFEpNApYGRIY2KISKHLWcAws7vNbJOZrWhkvZnZL81slZktN7NxSesmmdkb0bobclXGTGhMDBGRIJc1jHuASU2sPxM4MpqmAjMAzKwYmB6tPwa42MyOyWE5m6QxMUREgpwFDHefC2xtYpNzgN97sAA42MwGAycAq9x9tbvvAx6Kto2FxsQQEQnibMMYCryX9Lk8WtbY8tjUjYnx9yKNiSEiBavDj4dhZlMJj7Q49NBDc3KOujExIDEmxqaNMGBATs4nIpKP4qxhrAMOSfo8LFrW2PK03H2mu5e5e1n//v1zUtC0nnmm/c4lIpIH4gwYTwKXRW9LnQhsc/cKYCFwpJkNN7NuwEXRtvmlpibuEoiItKucPZIysweB04B+ZlYO/BDoCuDudwKzgbOAVUAlcEW0rsrMrgX+AhQDd7v7q7kqZ6u5hmkVkcKSs4Dh7hc3s96BaxpZN5sQUPLXokXwpS/FXQoRkXaT0SMpM/uImXWP5k8zs+vM7ODcFi3P3X573CUQEWlXmbZhPApUm9lHgZmERukHclYqERHJO5kGjBp3rwImA7e5+78Cg3NXLBERyTeZBoz9ZnYx8CXgqWhZ19wUSURE8lGmAeMK4CTgv919jZkNB+7NXbFERCTfZBQw3H2lu1/n7g+aWW/gQHf/nxyXLe9oTAwRKWSZviU1x8x6mVkfYAnwazO7JbdFyz8aE0NEClmmj6QOcvftwBRChtmPA5/KXbHyi8bEEBHJPGB0iVKPf55Eo3fBaHRMjBfK4y2YiEg7yjRg3EhI1fG2uy80syOAt3JXrPySOiaGJ8bE6Lol7qKJiLSbTBu9/9fdR7n7VdHn1e5+Xm6Lll/qxsR4ZntiTIzq6riLJSLSbjLKJWVmw4DbgJOjRS8AX3f3gnkmkxgT46DEmBhrH4Vx4xrbRUSkU8n0kdRvCSnGh0TTH6Nlhe28gqpkiUiByzRg9Hf337p7VTTdA7TjaEUiIhK3TAPGFjO71MyKo+lSQC2+IiIFJNOA8WXCK7UbgArgfODyHJVJRETyUKZvSb3j7p9z9/7uPsDdzwX0AF9EpIC0ZUzvb2WtFCIikvfaEjAsa6UQEZG815aA4VkrRQeijLUiUqiaDBhmtsPMtqeZdhD6YxScadNgnp2ayFjrBRk3RaQAmXeiG15ZWZkvWrQoJ8cuLQ25pOorKXF279bTORHpmMxssbuXZbJtWx5JFZRGM9Z+99fxFkxEpJ0oYGQoJWNtCYmMta/PibtoIiLtQgGjBeoy1i4gkbH2wQfjLpaISLvIKFutBImMtSQy1oqIFAjVMEREJCMKGCIikhEFDBERyYgChoiIZEQBQ0REMqKAkQ0HHxx3CUREck4BoxUqKmBC6d8TCQi3bYu3QCIi7UABoxWmTYN5u8sSCQhFRApATgOGmU0yszfMbJWZ3ZBm/b+a2dJoWmFm1WbWJ1q31sxeidblJqNgC5WWghnMmAE1FDGDqzGcUirjLpqISM7lLGCYWTEwHTgTOAa42MyOSd7G3X/q7mPcfQzwXeB5d9+atMnEaH1GmRRzrdEEhAyPt2AiIu0glzWME4BV7r7a3fcBDwHnNLH9xUBeJ2ZKSUDYZX8iASEb4y6aiEjO5TJgDAXeS/pcHi1rwMx6AJOAR5MWO/CsmS02s6k5K2UL1SUgvOa+RAJCgN274y2YiEiO5Uvywc8CL9Z7HHWKu68zswHAX83sdXefW3/HKJhMBTj00ENzXtC6BIS3bmM630ysmDMHzjwz5+cXEYlLLmsY64BDkj4Pi5alcxH1Hke5+7ro6ybgMcIjrgbcfaa7l7l7Wf/+/dtc6IxdeWXq57POar9zi4jEIJcBYyFwpJkNN7NuhKDwZP2NzOwgYALwRNKynmZ2YO08cDqwIodlbbmSEnjoobhLISLSbnL2SMrdq8zsWuAvQDFwt7u/amZXRuvvjDadDDzj7ruSdh8IPGZmtWV8wN3/nKuytlq/fnGXQESk3eS0DcPdZwOz6y27s97ne4B76i1bDYzOZdmyoj0fgYmIxEw9vVuhogImTIANA0alrqiqiqdAIiLtQAGjFaZNg3nz4MYb66343vdiKY+ISHvIl9dqO4TS0tBpr9aMGTADp4Td7KYHPP98fIUTEckx1TBaoEFqkB6kpgb5xz/iK5yISI4pYLRASmqQkvC1QWoQ9/gKKCKSQwoYLVSXGmRB+FqXGqTWAw/EUzARkRxTG0YL1aUGAaZPB+b8AFYmbbByZf1dREQ6BdUw2mp0ve4i8+bFUw4RkRxTwGirX/869fPcBvkRRUQ6BQWMVqrrvLejZ9xFERFpFwoYrdRo5z0RkU5Kjd4t1GznPRGRTko1jBZqtvMewI4d8RRORCSHFDBaKKPOe+eeG18BRURyRAGjFRp03hs9KXUDvVorIp2Q2jBaoUHnvb1DoSRpg3372rtIIiI5pxpGG9S9WvtB97iLIiKScwoYbdDkq7Xr1rV7eUREckkBoxVKS8EsvFJbUxO+Gk4plYmNbrghvgKKiOSAAkYrpH21dsru1Fdr77svnsKJiOSIAkYrpH21dmBp6qu1IiKdjAJGKzV4tXZDmo3uvbfdyyUikivmnWiEuLKyMl+0aFG7nrOiAi66CB5+GAYNtoYbdKLrKyKdj5ktdveyTLZVDaONlIRQRAqFOu61kpIQikihUQ2jlTJKQigi0okoYLRS2jelLj1Hb0qJSKelgNEGtW9K/fGPMHAgrN2gFCEi0nmpDaMNapMQXn11CB6Hn9sNnq23UXU1FBe3e9lERLJNNYw2yChFyPz58RVQRCSLFDDaIG3D94Ty1Ibvqqp4CicikmUKGG2QtuG7l6c2fP/qV/EVUEQkixQw2qhBipDiYakbbNkST8FERLJMjd5tVNvwXVEBK1bAww8bPJ60wbP1W8FFRDqmnNYwzGySmb1hZqvMrMEAEWZ2mpltM7Ol0fSDTPfNN0oRIiKdXc5qGGZWDEwHPg2UAwvN7El3X1lv0xfc/TOt3Dd2GaUIMVMSQhHp8HJZwzgBWOXuq919H/AQcE477Nuu0r4pNXG9UoSISKeTy4AxFHgv6XN5tKy+fzKz5Wb2tJkd28J9Y5f8plT37lBZCV0OHaIUISLS6cT9ltQS4FB3HwXcRmpzcUbMbKqZLTKzRZs3b856ATNR+6bU5z4XPs+dm2aj5cvbtUwiItmWy7ek1gGHJH0eFi2r4+7bk+Znm9kdZtYvk32T9psJzIQwgFJ2it4yTz+d2o6xZk3o8Z3SjjF6NDz+OJyTl0/WRESalcsaxkLgSDMbbmbdgIuAJ5M3MLNBZmbR/AlRebZksm8+yTjV+UsvtX/hRESyJGcBw92rgGuBvwCvAX9w91fN7EozuzLa7HxghZktA34JXORB2n1zVda2Stvj+4IzGrZj6E0pEenActpxz91nA7PrLbszaf524PZM981nGzfCF78Ir7wCo0bBhm39G260d2/7F0xEJEvibvTuNGbNCo+ili4NfTNqe4Cn+MUv2r1cIiLZooCRBWnTnBupac5rrVrV/gUUEckCBYwsSNvofQmsmXJ9w42PPDJEkxdfbN9Cioi0kQJGFqRPcw6DHm7iEdRdd7VfAUVEskABI0sajO+9FujSxDsFRbr0ItKxKL15ljQY3/vwZnYI3U9ERDoM/ZubJS1q+AYFDBHpcBQwsqTRhu/GstYqYIhIB6OAkSWNNnw3lrVWQ7eKSAejgJFFaRu+GzNrFvzoR7B1a3sVT0SkTRQwsmjWLJg+PXyta/j+z/9sfIfvfx/OPru9iici0ibmnSghXllZmS9atCi289cfrrVWSfF+dld3a3zHTvQzEJGOxcwWu3tZJtuqhpFFjTZ8L90Wb8FERLJA/TCyKO1wrV1g0Mh+Te+4ciX06RM68w0Y0D6FFRFpIdUwsiyj4VrrO/bYEG0GDoQNG3JaPhGR1lINI8vSDtdqUEJlYrjWpmzYAIMG5a6AIiKtpBpGljXajnFzugEy0lADuIjkKQWMLGu0HePbl8D77zd/gJqa3BdSRKQVFDByoNF2jL59m9957lz45Cdh+/aclU9EpDXUhpEDjbZjlMDuj38c/v73xnf+1rfC15/+FKZNy21BRURaQDWMHKjfjlFUBFOmhMDBT36S2UEqK8MkIpInFDByILkdo7g4NEu88Ub08tOpp2Z2kFtugZ494c034Ykn4L33clpmEZHm6JFUjsycmdp+/eqrSY+lWnKgo44KXwcNgoqKbBZRRKRFVMPIkfLy8FiquDh8Li6OXq9d08oDbtgA1dVZK5+ISEspYOTIEUfAAw8k7vHV1XD//TB8OHD77fCPf7T8oN/5TlbLKCLSEgoYObJ6NQwbFvpg1Bo8OKphXHMNHH98yw96yy3wzDNZK6OISEsoYOTI4MHwmc+Edozax1J9+tTL+rFkScsPfMYZoTGkvDwr5RQRyZQCRg7VNnzXPpaqbfguLY02GDu29Qd/8sk2l09EpCUUMHKotuG7R1LOwSOPrNfw/cc/tu7g11wD990Ht92WunzZMvja15RiRESyTiPu5ViXLulfbiopgd2179eate0kyT/D2mO9915oRBERaYJG3Msjp58eahUlJYllDWoZ7qFm0FpmYbrkksSyQw6BF19s/TFFROpRwMix2bPDG1PJuaXeeis0ite1ZQCMGgUvv9y2kz3wQOrnH/0oNJz8+MdtO66ICAoY7SKjWgbAmDHZPfHs2TBxInzve6EG8sYb2T2+iBSUnAYMM5tkZm+Y2SozuyHN+kvMbLmZvWJm881sdNK6tdHypWaWXw0TLZRxLQPgwguze/LNmxPzTzwRHn89+CDs25e6XUWFUo+ISJNyFjDMrBiYDpwJHANcbGbH1NtsDTDB3Y8DpgEz662f6O5jMm2QyWcZ1zJ+97vwBlQubNgAd9wRXt0aOjRELQiRbMiQMImINCKXNYwTgFXuvtrd9wEPAeckb+Du8939g+jjAqDTvtaTcS2je/eQOuSuu7JfiJ//HK69Nsy//z587GPw3e+mFuDHP4Zu3cIjrE9/OjzG+uCD9McTkYKSy4AxFEjOyV0eLWvMV4Cnkz478KyZLTazqTkoX7tLV8vo1auRhIRf+Qrs3QtbtuS2UDfdlPr5e9+D/fvD/LPPwtGrOKUtAAAYSUlEQVRHwwkn5LYMItIh5EWjt5lNJASMf0tafIq7jyE80rrGzNIOJGFmU81skZkt2pz8vD4PzZ4dahXJtYzt20MtI21XjG7dQj6RCy5otzKmtWpVYv5b34KPfjS+sohIbHIZMNYBhyR9HhYtS2Fmo4C7gHPcve7faXdfF33dBDxGeMTVgLvPdPcydy/r379/FoufGxMnwgEHpC4rKoK//a2Jnf7wh5yWKSNPPhle+/35z+Htt8OA5b/7XdylEpF2lLOe3mbWBXgT+CQhUCwEvuDuryZtcyjwf8Bl7j4/aXlPoMjdd0TzfwVudPc/N3XOfOzpnU5RUWrn7Frdu6fWPlLccw9ccUUui9U6td3Yi/KisioiLdSSnt45G3HP3avM7FrgL0AxcLe7v2pmV0br7wR+APQF7rDwTKYqKvhA4LFoWRfggeaCRUfSvz9s2tRw+d69of15d7oh+S6/PD8DRmlpeEX38cfD17fegosvhr59QwONiHQayiUVk8sug3vvTb8uJc9UsvXrw9tNp53WMd5ceuopOPvsxtevWwcLFsB557VfmUQkhXJJdQA7d4a3WusrKYG//72RnYYMCSlEtm4Nz7ReeAG++c2clrNNPvOZ0Ei+axf8z/+Eb+z00xOvhZ12Gpx/PlRVxVpMEcmMahgxmjIlPMlpcXtGfW3NdptrF18cepfXuvTSUL3q2jUEi7174d13wzvHEyfCT34CZR2+r6ZIh6AaRgcxaxZMmpT+fr93b1h+0kmhg3aTKivD20v5KjlYQEiIWFSUqFnU1MB114X5555revja225L9FCvr7IyVN1EJCdUw8gDjY2ZUWvw4DCaa8rwruksXgxTp7Zu6Nc4bdwIAwemLnv//dCQc/nlIZXJsGHQu3eiE2FVVWLs21oHHwzbtqWvsrXFvn2hw0y/ftk9rkgeaEkNQwEjD5x1FsybBzt2NL2dWWj3bjJw7N8fOvwViq1bQ+Do1StRVcv27/R554XqYLrj1tQkXm8T6YD0SKqDmT07vIXaHPdQ22jyMVXXruE/86VLw8YQGpwBTjklK+XNK336wEEHNayi7dgRbvLZUHucbdsarrvuujAGb1NVxI6gE/3jKLmjgJEnxo6F4cPhiCOa33bBgmYCR0kJjB4Nc+aEZILf+U64IcyeHRIbdkZHHZWY/8Y3Qo3jvPPCY7rf/z7kyKod53zePBg5MjS0t/YctWZGCZY78pteS5eGNqU/d5quTpIr7t5ppvHjx3tHN3my+/Dh7kOHupeWuoc7fdPTiSe6V1S04CTLlrn/7W/uTz7p3qtXZifpDNMXv9hwWWWle1WV+zvvuC9dGpa98kqYLrzQfdOm1O2fey71WnbtGpbv3p3NXwP37dvdr7nGfefO7B43ndtvD9/DVVfl/lySd4BFnuE9VjWMPDNrVkiDXl4e3qDq3bv5fWprHOPHh6nZN6tGjaJixD8z4ebPsuHpNg4L25Gk6ynZo0d46+CwwxIjHh53XJgefhgGDEjdfuLERG3ittsSmX2TH0m98EJoT0lOhllTA7fc0nxDVa2f/ASmT++8NULpkBQw8tisWeHp0oEHhpeEmrNkSZgWLIAbGoxvmGratPBk5sb7jgg3O096hl3b5iHp/fWvIfjUvgoMoR2l1k9/Gr6ecUYYqOrDD+FPf4Jvfxuuvz6s27AhpI/fsSP946zkV47bS77355HYKWDkufXrwxudxx8f2jgOPTSz/X73u/D3bxY6h/fqBcuXh5d5zGDGjHAvmjEDrLiIkhKYcOIeNqzaGdo83n47RJ8dO5SVtr6zzgq5XZJVV4cLe/XVoU0AQnbf9etDNfFznwvLZs4MVcjBg8MAVb16hbFP3n8//KDre/nl5ntwvvJKqBGla5SvtWlTCFxN2bWr6fWFZvny0GdIEjJ9dtURps7QhtGcyZPdDzywdY/wS0vdjzvOvXv31OUHHeTer1+YHzy4kfaQdAfs1i3+dok003oG+anM8QoGxl6WFk+VleF6f/e7Dddt3eq+dq37pz8dPr/xRtj2nHPC57PPdr/nHveamvQ/v27d0v9STZ+eOMeuXZn9Iq5c6X7++e5792a2fVx+8AP3hQvDfE1N+LxyZWb71l6TTo4WtGFktFFHmQohYLinNoy3NnhkMo0bl2hQf3nQGX4gH/r4fmt96eU/91N7L/OKd/b6+inXhJtzyeHuX/mK58MN+yqmexFVfhXTYzl/m6d/+7fMt509u+GyLl3cP/9592eeCZ+PPDKxrrra/WMfc+/bN9xA5893/+//Tqz/whfCCwDJPvggEZxqffzjYfuXXsr8F/edd9yXLEldtnOn+/PPh1+y115r3R9EfTfd5P7mm2E++aa/dWuYHzAgs+MoYDSYMtqoo0yFEjCSJQePoqLc3L8GDHAvKalxqHFw79EjLO/bt/YloRofMCAEmHEs9H5sdKjxgVSEoPHjH+emYPWmEirTriqhsl3O3yGmAw7IbLu1a8ObYu7uRx0Vli1fnqiBlJWFZX//e+a/rLXHdne/9towf955qedtyuuvu69a1fQ2H3wQjjN0aMNzvv9+mO/dO9Q6Xn/d/aGHmi/vjh3h689/ntn32cEoYBSoyZPdjz02/ntS/al7d/exQzd4zy6VPmr4dh/XdamPY6Efx1LvyTYfRfh8IvO9goH+MqNCbYaFvpTjMq6trGeQf4H7vAc7Hdx7sNMv4d6O+WgqX6Zf/rLhsilTEvNf/Wr45auoCDffqqrwecuW1NrH/PmJfWpv3BBu3snHfv759L/cH36Y2Obllxu+3uzuvmaN+9tvJ7Z76qnEvHsINuDep0/qOS+8sO4Q69e7n3pq9Fi2dv3q1eHrYYdl/sdYXe3+9NOJx4MbNiQeN9Z6/vkQ4JKknL+dKGAUsOQaR21NID+nmrTTANZ7CTvrPvdgh0OV92VDg+CylON8LAtTltfWboxqhyofwHofx8KUgFRbiPUM8o8zv8HyQpty+gixVy/3F18M/51nus+uXYn5q692P+GE1CBVO+3eHdokKivd9+9v+pjpHt0lf//D3/WKL3zLr7r4Ay+yGr/s5DcT12TJksT206Yl/tjefdf9r38N8/PnJ2pkW7cmHvM9/HBYlnzO3/42PIoD9/79U/5+r/rKHi+i2q/66j73669PtLfs3eu+b19O7hkKGJIiOYh85CPuQ4aEx1dduoTHTb16hc9m2b9fZH9KBJcQTNIHHqjxYvY2WNaFPXXBZQAVDtUONd6PjXWBpX7tp6mA0tKgk3xzbupGnc2beP1jvcwoP4gPfBkj3QltPkaVD6a8UwXO5q5v7c/tS9zttY9b60/F7HcfPz514Y4doQZR+3nt2sS8e+q2P/uZ+/XXN/w92bo1Ub5XNnv37unPX0Jl6rmOP95948as3h8UMKRVBg8OjegDBoRgkgd/8zFMjdd86j9CayropNt2HAu9b1QDuoAHfTDlnq72lDhuVUqAa+q46ZbXLgu1rkRtK7kG1+hNKkc36rbu21SArh+Ma6/vYMobPNq8jN82+v2nm4yqBuerH3hry5Zcm10/4p/9VOb4BTxYd82vYrp7WVldGfqxse73IlGmGh/OqsavYe2jrunT3U86qU1vq7UkYChbrTRqypTQpaB2eIqnngpZyIcPh7lzwyBPPXuGDs1moZPhnj3hN9osm33OHMiXTmXJfy+Nlamxv6nmvofa/dJt19q/09ZdN6Oa9QxlEBupYBAX8RAPc2GDz44xmVnsp2vdvuUMYzP9uZJfcQfX1C1vbr9u7OcoXudeLuNS7uUNjqKSHqzlcB7jXL7Pj1jDcDYxADAGsIFhrEs57yb6E7qXNXUNm/q5Nb7fl7iHe/hy3dKRLOdVRlJKJSN4LTr/gLpjXMbv+D1fauJ8mSlhN7vp0fRGL7+cyFTQQkpvLrG76iq4887UZcXFDZO6FhW1NbDU/yOvf0Ov//td/483n4JRPnCKqaKaYvqymT2UUsoe3qcvA9jEMNbxDoeyhf70YzNF1KTcJBu74Y5iORsYyCb605f32c7B7Kd+Gv7mAmVLb/S50r7n685uFnAipzKXj/EWT/EZHONs/sibfIyjeZOnxv2QQYv/1KrjtyRgdGnVGUSasXFjqInU1k4WLgz/ADWWcby52ox7yNzesNOzpQSioiLDLDUwdSmqoaoGwn+e6f5ByuSfpqZuELm6gWTzuOkCa7pjG9XRf/1bCINa7aIXAJsYzCYG1235PgMa7p72vLCc0XVLao+bmUy+/zgCfi5+5g3/2SmiipG8ykU8xA56sZjxjGMxp/MMLzMegMWM58YlZ3NHlkuTjmoY0uFMmRIya0ydGjJtVFSkBqJ06xe8VMPOHU5pz2K2bgWvrqFrcTVVNUV06WoUmbNnL2AW1XicIpwajNxk0ElX+4lbZ6tppattpquNNvWzaOk1aSqQpHsklu68rf85lJSE4XBaQo+kRNqisjKMWlhTw5TPd2Hp8iIqKkI+wJoaqKkJfzM9ehg1NU7VfkLQoZo9e2qooZjUgBPWARhe9x98+FxDETV0ZR97KAGMHuxiD6U4Rlf2U0MRVXUPAxoeN935arctYQ/76EYNRZSwl6GsYx1DqaILXahiKOuoYBCVHJB0ARq7uWZyI2zuEVLycRq7WTrpj5Xu8WLTyxLXpynhfD3YyWA2sJsSdtCLHVHNKvXY6fZtTGb/FPRiGwewg+7sYxc92MxAWppI3KjhzLOK+M1vMhjKuf6+eiQl0gY9Eg2Msx5Pt4E1Ml8MTzwVkvxddllILnj88fBv/wY33gj//u9M+fJBDD5oF1M/t4GZd1RR8X+vMmtOv5CgcPTo1NPMm9cuoyRO4RGWMgajhtV8lMRNrv7NrrmbX1hfzP4oKDa2fWK5UYNRQw1FpD4y9Gh9NR4FRICPsIp3OIwqugI1hOtfQzHVKef8CKuooYj1DOGjvEYPKlnKGKopZjhr2E0JH9KbgWzkMaYwk6lUMIhZnF93TQazgdc5ipcZywf0afT7qQ006xhKV/ZzIDtYz9CofNCbrYxlKWsYzkYG0psPKGEPuymhO/sYw9K68wJcxj3cyxej7y2T2objGIcd1vJg0VKqYYjkg507Qx77k06C3/wGhgxJpEzfuLHxO8EZZ8Bf/pJ+3bPPwqc+lXERkm+SczkVxxjMetYzpK6GUkR1VBOCEvZwMB82uPE+xrmUsIcKBlNFF2oowqiJ6kBGDyo5mRd5iZM4gJ2sZxhDKKcPW+lBJav4CMVU83ke4THOZTKPM5WZdTd1gMFsaHZZ8k24LWoD6j668QEHs4eSuu893Q2/9jq2tixTeIQ3+RivMjJpaQiMqS8XOKXsqvt0xuQDWjUqcUtqGBm9e9tRJvXDkE5r69aQHmPXLvd//MP95pvd584N6154IfTOBPc5c9w/9akw7x7SddR/h/8Pf0jM16bsGDs2bJ+83eDBLe5HoSk702Qe8eGs8iN40xOZC2q8N++7Ue3F7Pe6Ph21+7US6rgnUmBqahKduaqqGuYtcg+JAufPb/44ixaFr9//fuJm9L//G5bddJP76aeH/Eru7r/7nfuXvxy2+fjHQ14mcL/rrsS+/+//JeZPOy18LSsLaTP+/d9jvznn8zSZR/xqbveljPKrud0HU57yeTKPJLZvJQUMEWm7t98OKYlrg0NLvfdeSJtRXZ2oDbmnT2O+fXtIuTFypPutt4ZgUnsjHDs2JOr79a9DWf72t8QY6uXl7gsWhJ7Ou3a5v/pq2OcXv3Dfti1kpE2+CR99dNjvuuvcv/1t90mTEuuWLQtB8uCD3f/858Zv5KtXpwZEcB82LP22yQndbrqp4fpDDslegGklBQwR6dhqatzvuy87Cff27w+J1O6/P/36O+4ImW7rW7s2NctuOtXVDWtzq1eHpIRvvdVw+/rZf2uTEH72syHR4JVXuv/xj2HZrbcmyrF2rfvmzeF8I0cm9p8/P6QHWbYs48tRX0sChhq9RUSa8pvfQN++cO652TtmVVUYEjd5LPhkixbB+PGNj7NeWQl//nPodNRG6ochIiIZaUnAyEUXVhER6YQUMEREJCMKGCIikpGcBgwzm2Rmb5jZKjO7Ic16M7NfRuuXm9m4TPcVEZH2lbOAYWbFwHTgTOAY4GIzO6beZmcCR0bTVGBGC/YVEZF2lMsaxgnAKndf7e77gIeAc+ptcw7w++h14AXAwWY2OMN9RUSkHeUyYAwF3kv6XB4ty2SbTPYVEZF21OEbvc1sqpktMrNFmzdvjrs4IiKdVi4DxjrgkKTPw6JlmWyTyb4AuPtMdy9z97L+/fu3udAiIpJeLgPGQuBIMxtuZt2Ai4An623zJHBZ9LbUicA2d6/IcF8REWlHORtxz92rzOxa4C9AMXC3u79qZldG6+8EZgNnAauASuCKpvbNVVlFRKR5yiUlIlLAlEtKRESyrlPVMMxsM/BOK3fvB7yfxeJ0ZLoWqXQ9Uul6JHSGa3GYu2f0xlCnChhtYWaLMq2WdXa6Fql0PVLpeiQU2rXQIykREcmIAoaIiGREASNhZtwFyCO6Fql0PVLpeiQU1LVQG4aIiGRENQwREclIwQeMQhioycwOMbPnzGylmb1qZl+Plvcxs7+a2VvR195J+3w3uiZvmNkZScvHm9kr0bpfmpnF8T1lg5kVm9nLZvZU9Llgr4eZHWxmj5jZ62b2mpmdVKjXw8y+Gf2drDCzB82spFCvRQPuXrATIe3I28ARQDdgGXBM3OXKwfc5GBgXzR8IvEkYmOonwA3R8huA/4nmj4muRXdgeHSNiqN1/wBOBAx4Gjgz7u+vDdflW8ADwFPR54K9HsDvgK9G892AgwvxehCGUVgDlEaf/wBcXojXIt1U6DWMghioyd0r3H1JNL8DeI3wh3EO4UZB9PXcaP4c4CF33+vuawi5vk6IBrfq5e4LPPxF/D5pnw7FzIYBZwN3JS0uyOthZgcBpwK/AXD3fe7+IQV6PQg59krNrAvQA1hP4V6LFIUeMApuoCYzOxwYC/wdGOghOzDABmBgNN/UwFblaZZ3RLcC3wFqkpYV6vUYDmwGfhs9orvLzHpSgNfD3dcBNwPvAhWEDNrPUIDXIp1CDxgFxcwOAB4FvuHu25PXRf8FFcQrc2b2GWCTuy9ubJtCuh6E/6jHATPcfSywi/DYpU6hXI+obeIcQhAdAvQ0s0uTtymUa5FOoQeMjAdq6ujMrCshWNzv7rOixRujqjPR103R8qYGthqWZnlHczLwOTNbS3gM+c9mdh+Fez3KgXJ3/3v0+RFCACnE6/EpYI27b3b3/cAs4J8ozGvRQKEHjIIYqCl6O+M3wGvufkvSqieBL0XzXwKeSFp+kZl1N7PhwJHAP6Iq+XYzOzE65mVJ+3QY7v5ddx/m7ocTfub/5+6XUrjXYwPwnpkdFS36JLCSwrwe7wInmlmP6Hv4JKHNrxCvRUNxt7rHPREGcHqT8HbD9+MuT46+x1MIVejlwNJoOgvoC/wNeAt4FuiTtM/3o2vyBklvdwBlwIpo3e1EnT876gScRuItqYK9HsAYYFH0O/I40LtQrwfwn8Dr0fdxL+ENqIK8FvUn9fQWEZGMFPojKRERyZAChoiIZEQBQ0REMqKAISIiGVHAEBGRjChgiKRhZvOjr4eb2ReyfOzvpTuXSL7Ta7UiTTCz04Dr3f0zLdini7tXNbF+p7sfkI3yibQn1TBE0jCzndHsTcAnzGxpNE5CsZn91MwWmtlyM/tatP1pZvaCmT1J6CWNmT1uZoujsRWmRstuImRCXWpm9yefy4KfRuMwvGJmFyYde44lxqu4v1OMrSAdTpe4CyCS524gqYYR3fi3ufvxZtYdeNHMnom2HQeM9JDmGuDL7r7VzEqBhWb2qLvfYGbXuvuYNOeaQuhxPRroF+0zN1o3FjiWkGr7RUI+rHnZ/3ZFGqcahkjLnA5cZmZLCSni+xLyB0HIIbQmadvrzGwZsICQoO5ImnYK8KC7V7v7RuB54PikY5e7ew0htcvhWfluRFpANQyRljHg/3P3v6QsDG0du+p9/hRwkrtXmtkcoKQN592bNF+N/nYlBqphiDRtB2FY21p/Aa6K0sVjZh+LBhuq7yDggyhYHE0YqrPW/tr963kBuDBqJ+lPGAXvH1n5LkSyQP+liDRtOVAdPVq6B/gF4XHQkqjheTPph978M3Clmb1GyGK6IGndTGC5mS1x90uSlj8GnEQYI9qB77j7hijgiMROr9WKiEhG9EhKREQyooAhIiIZUcAQEZGMKGCIiEhGFDBERCQjChgiIpIRBQwREcmIAoaIiGTk/wfHMBfsZfga8wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7ff8596fa240>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot training and test loss\n",
    "t = np.arange(iteration-1)\n",
    "\n",
    "plt.figure(figsize = (6,6))\n",
    "plt.plot(t, np.array(train_loss), 'r-', t[t % 25 == 0], np.array(validation_loss), 'b*')\n",
    "plt.xlabel(\"iteration\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.legend(['train', 'validation'], loc='upper right')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAF3CAYAAABKeVdaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XucVXW9//HXZ4bLDDe5jYAgDIo3NEUFsvR4SUu0FLESL2Xq8SDesk7+Cq1OHS2zY3XSI0JUmqXmFS+ZplmaWWKgIiheuAroMFwUEJiBGebz++O777Nn9h6YNXtm9vv5eOzH3uuy1/rsxfD9rPX9rvX9mrsjIiICUFLoAEREpP1QUhARkQQlBRERSVBSEBGRBCUFERFJUFIQEZEEJQUREUlQUhARkQQlBRERSVBSEBGRhC6FDqClBg4c6JWVlYUOQ0SkQ3n55ZfXu3tFrvU6XFKorKxk3rx5hQ5DRKRDMbN381lP1UciIpKgpCAiIglKCiIiktDh2hREpHOpq6tj9erV1NbWFjqUTqGsrIxhw4bRtWvXXfq+koKIFNTq1avp3bs3lZWVmFmhw+nQ3J0NGzawevVqRo4cuUvbiKz6yMxuN7O1ZvZ6E8vNzG4xsyVmtsDMjogqFhFpv2praxkwYIASQiswMwYMGLBbV11Rtin8BpjQzPJTgP1irynAjAhjEZF2TAmh9ezusYwsKbj788AHzawyEfitB3OAvmY2JKp4RESy2bhxI7fddluLv3fqqaeycePGCCIqrELefTQUWJUyvTo2T0SkzTSVFOrr65v93hNPPEHfvn2jCqtgOkRDs5lNIVQxMXz48AJHIyKdybRp01i6dCljxoyha9eulJWV0a9fP9566y3eeecdzjjjDFatWkVtbS1XXXUVU6ZMAZK9K2zZsoVTTjmFY445hn/+858MHTqURx99lPLy8gL/sl1TyKTwHrB3yvSw2LxG3H0WMAtg7NixHn1oIlIQX/sazJ/futscMwZ+/vMmF9944428/vrrzJ8/n+eee47PfvazvP7664m7d26//Xb69+9PTU0N48aN4/Of/zwDBgxI28bixYv5/e9/zy9/+UvOOussHnroIb70pS+17u9oI4WsPnoMOD92F9JRwCZ3rypgPCJSSDt2QENDoaNg/Pjxabdz3nLLLRx22GEcddRRrFq1isWLFzf6zsiRIxkzZgwARx55JCtWrGircFtdZFcKZvZ74HhgoJmtBr4HdAVw95nAE8CpwBJgG3BhVLGISAewYAFccAEcUdi703v27Jn4/Nxzz/HMM8/w4osv0qNHD44//vjGt3tu2UL3LsmitLS0lJqamrYKt9VFlhTc/Zwcyx24PKr9i3RIjz0GJ58M3bvv+jZeew169ID99stv/XffhbVrYdy4Xd/n7lqzJrzHrxQ2bYJevaCuLszr0SPMr61Nn45raIDNm2GPPaCFt2T27t2bjzZvDtvOsGnTJvqVltLj3Xd5y4w5c+aEK5q4jz6CZcvCvIUL4WMfS8azaVOIJ27btvDaYw/I52njjz6CsrL81m1FHaKhWaQoPP88TJwIX/86/Oxn6cv+9jf4n/8JSaO0tPF3d+wIhciAAaEOHcBTmt/cQ8H72mvwi1/Aww+H9c0gPj7Jpk1h2z17hoKuSxcYODAUwN/+NtxwQ1heXx/Wfe89ePllmDYNLrkEVqyA7343JLVbb4XPfCYU1KWlcOedYZtXXw233x5iGDw4vLvD6tXJWDdsgOXLoX9/+CB2V/vHPhZifT32LOz++8PKlXDAAWH6zTfDMaishK1bw28YODDEun59eNXWQp8+IVm+/TZs2QLduzPAjKNHj+aQgw+mvG9fBvXpA/PmQc+eTKioYGZdHQedeioHHHooRx18cIht3rywv6VLk3Fv3x7i2LAhvOLVTAcdFJL8okXJdSsrwzEcMSIc5+3bw3eqqsLxGDEiJOtM++wTjkuEzL1jtduOHTvWNZ6CdDr33gvvvAPf+x6cfjo8+mhy2R13wEUXhc9r10JFlnFSPv1peOYZ+OMf4bOfDfPcwxnrj34EP/0pfPhhcv2dO7MnF4DbboPLLguf99gjfK8kpfnxhhvg2mtz/6Zbb4Urrsi52ptPPslBAwfm3l42vXqFwj0bs/TEGLfPPiFBtRdduoTkla+xY3Ou8uabb3LQQQelzTOzl90955fVS6p0Ln37wve/3/Tye+4JhcXWrdmXz53b9LJUW7aEs8W4n/0sveAEeOkleOIJWLIkFMBf/3qYP2RIOEu+/fbkuuecExICwKuvhhjjr3hCANhzz/Rl8dczz4Tl8YQAcM01Yb/f+U56QoCmEwIkEwKEs9nM35VPQoC8EsJuayohQPaEAO0rIUDLEkIbUFKQzmXTJvjv/w6fd+wIr0svDZfujz0G550XlsXvDtm+PTRurlwZCubx48N73M6dybrmLVvgrLPCJf6kSaEOfv16eOMN+MY3QiF0//2wcWM4Qz/qqFBI77dfmP75z0N1ypo1UF0N//7vyUI91apVtIobb2yd7UhOO+jKWxxAHS2r/49/bxvljb6/q9vcXWpTkOgsXRoK49NOS85zh1/9KhS8vXrl3sbKleGM/Mwzm15n0yaYPTtZNx7XvXt4bd8Of/lLso4X4JBDQgy//32YvvPO5LJ//Su833VXqLr561/hxBPDNgAeeCC5bmZVzuTJ4f2aa7LHmtrw2E5VMZizuZf7mMxgqvNazzEmMRsDHmZSs9/LtJNS3uRAnGRyNJwRvMu7jABgFEvpSl3a93bQlaXsC9Bo3Tq68DYHcABv04MadtCVZezDvixrtJ3MbS5jH/bifZayL/uylPfZi+GsZCXDG30/vn53trOFXrzPEEawstlt78syHNK+F35/CW8wmm7siK3fjXq6JOYZHjsO0VKbgrSce2jwi99p0ZRu3cLdI6l/Y/H68SOPDIXvli0wciQ8+GA4w/7d70IVyW23wbHHwj/+Eb539NHwwgvhDo+99goPOB19dLg7Y/LkcIae6oQT4NlnW/d3RyDfAjjqGOIF+kwu4TM8zVoGMYhq5jMGxzibe7mFK7mEXyTOXFcynPVUMJB1lNDAWvYEjD1Zwz6sSEsO8zmU4/kbDzGJb3NDYht1dOV/nuzKwIEHNYqrhAYaYpUZXaijK/VspxujWMp7DGU73amPnddmrruTUpwSSmignBrKqGUDAzCcMmoxPFHI1tGFtziA7uyI7aUL4IAl3uPb70I9DZTQne2U0MBWesbWS2c00INtQDJhxeM1PJYAW9pxnVPR5UNGjMnd0Lw7bQpKCtJyt90Gl18Os2bBV74C3/pWqIeeMQN+/OPQqDlyJJx9dlj/O9+BCy8MDXypVSXf+EaoV8+s7wb44Q/DHS+p/vKXcMYeN2oUHHggPP546//G3ZSrsI8vH8kyfsv5VLCWYbEH+rtRl1agxgvtOro2Wpa5ve/y33yB2TzEJK7j+1n3n7o9gNUMSxTo2QqqEuppwGKFWbwtIp8CrYEjeAWARRxELT0opY6ddEn5vvHkk29mTQrZxcur3e1VNWxnABv4kH6xhNKSbcaTRuP5XQhtBPVpFTGt2wusWTivaoqSQmfy8Y+HhskNG6Ldj3uoBjnzzHD3wwcfhDP3ww4LZ+LXXRfqwx99NFS1nHUWnHtuODu/vPM9XpJZiMen42fHqdUiVQzms/yBJezHw5zBdXw/7Sy6G3UcwFuNCvtUr3AkTRUqAANZx/BYNUT8jDxuT9YwjPeooyvLGMm+LGcNg1hLBaXsZCddY4VvKXtmJJuZXMIpPEkVQ2gqCbS1liWFYuf07WuMGNH84wtKCp1J/Ey6qX+XU0+FKVPgjDOS89xhwgQ4/vhQYPfpE25d7NMnVK8sWxYK/x494MUXw73sEBLP5MlQW0vVoy+FQvH/vczgm74R6U9sTZn12k2dnaeeHacWpl1i9cPhbLkiUYjGp8uopZbwoFS8MA7LBgEkCt8e1LCNHjRd0Kb+e7ZWQdySs+bWOsPOtY+Wbz8khQNjU00du6a2m/q7Mn/jrsWTWXWU+X7ssb14/vktrFv3Pj/5yVf58Y8fbLSFSy45nquu+gmjR2eWwcmY7rnnfznzzEsoKwsd51111Wf5wQ/upnfv5nteragISaE5u5MU1NDcnqQmu9T7zeOee46qJ19l0pODqDsC2LEdPvyQuvI+LFvyAPs+vZwu174DR4yl7pUqVtCdh58p49vnOrZ2Bg8zCWcwkzYkqw64L9TrvsWB1NGVw27an2EcC5C18KyjKyuo5AWO4VDCg0SpZ84vcAwVrE+rynief0vMu4/JrGEQx/I8I1mR2G6m+Flt/Cw9/jn1Toz4GfnfOYZDeI3N9A2/gVcTZ8fx31BOLesZSGohsYBDY5+S89YyhLUkh/WopWeTywB20g2AbeRqNI+iMG7JNvNNHNkK18x1cm2jue+mbj/X9z3L/LBs/Xrn2muNG24wko84xL+T7XuZ28v12zzne0XFkFhCaGnyCdu4996bOfXUL1FWVk4JDfzfzX+goYnjU4JTyk5KcOrqduNp9zwoKbS1116jqmQoZ18xkFtuCQ+CmsHMmfDVcVu5hY9xCb9g2+d6sKwn7LtvqN2hfgd1C/rxFiuooxu84kA3iJ2xQkoh94pD7PPJJ9Wxk5HAPhzGqzRQmlYVEWQrFJN/mMnCM+ko5nAYC1KqI/ZKzO/NR6ylggn8iZ10TZt3GK+ymT7U0iPrdtP38WLiLP2T/DPlTDwZ9xw+AcCGlOPQ+DcYW+nT7L4KL58riVxn+02d4Ta3z+yFbxd2UB9LeJnrGA0YyY7ryqilLxvZSL/YHTNdabrgj2+/jnq6UEo9JeykhJ2U0kBfNrKOisR6Jewk3ItE7D3E8qtfwfz54X3aNIvFFNbag01soVeiUTe0F8S30JB2lxMY//d/32LQoL2ZfNZlOMYvZ32PLqWlvDzvWTZ9tImd9Tu49NLrOPa4SWm/6v33V/D1r5/GffctZGftJr573SUsXvwalZUHsn17DfEEdeONl7Fo0Vxqa2s48cTPc/kl3+Xue6ezbt37XDb1eAb07cvvZj7OSacfzLzf/paBffvys7vv5vbHHgPg4okT+dq557Li/fc55eqrOea44yLtoltJoY1VjZnAkfYKVQ6f/GToCgXgE5+AGo5JL/y2OgsWxL/ZFTisxfvbmXJmnXmW27zmz3xq6MkcPsEYFjSaXxM7u94Za5RMndeSGFLP0nOfiWfTGo2RTVVFZJ6BNvVOxnqZ8YV55WyL3fnSXIGaue/mlmW+x/eXOS9ZSO/NKh7mTGYxhYc5g/58wCIOTqz9aZ7mffbiA/rzPsOy7v1MHmQ+Y9hBNz6kL7WUUUYtDZTSlTrGMZcDeZsqBjObLwDwJk9yEOFZkCXsSwXrqGAd66igjq6MYmli2SFH92X7juS/w0MPhVe3bsY7/1jf6HbQJexLV+oabe9dhieSz2c+M5mf//Qqppx1HqNYyvnP3MVTt9zCHpOPo0+vXqzfuJGjLryQrx07HDOjxJy+vXdSywa6s50K1vF/D91BWVk5DzywiMWLF/DlLx+ZSFKXXvoD9tijP/13ruGiyz5HyeKDuOnsY3jgnoH8c+bNDOzbF0h2l/HykiXc8eSTvPSb3+DufPyCCzjuyCPp17s3i5ct4/f33x9pF91KCm2kqgqGDgWnKvF/Mp4QAEKniqW7WPi1ll2tgy205grvzPVyaa7aItv3m69mCGfEqQV9Q6IxOD7vYF5nf94BaFSgQvqZOEA/PqSKwTRQQgkeO/stoSdb6c52BlFNNYPYTvdYG0qXtP0B9GYzA1mfSAKphfR0rmA6V3AmD3IczzOFWYl1Xs9xdRffxi4pL2dUTbJwzCzgR/Vey/K3yrn60q088lxftm0vpay7M/GETXzjqnrqrHujf6JRZN9eHV0TyafigGH814fV9Fg3h9c+/JB+vXsz+DOf4etXXsnzf/0rJWa8t24d1Zs2MTg20tqoA7rQ5e33KaGBEazktVef56LJFzGaRVTsN4QDRh1ML7ZQUbaFfz50M7MefoT6nTvZsHEti5Yv59DUzgq7dk228wEvLFzIpC9+kZ6VlbB9O2eecAJ/f/VVTj/zzDbpoltJoQ1UVYUbelpPS24OaKqOuPnL+/y2m+27mYVzUwV2tv00Ve/b1Jl3tvWa2nZ8zQacUEfbEGsU7sFWaimjIXF/ergNswRnUKzBuppBdKGeobxHDWVspB+DqGYky6lmENWx+/rj74Oo5kDe5mHOYBKPpBWsAENYkzZvtwrTHOJn7+OYC8BcxjGG+WlJIJvUmLKu89BD8PnPh88vvRTunNtVpaWxetKY+EOHEO6I27kTysoYAvTpuZnaHSWUdWtgxw6jf88djBu4EoaPgCx9yGWTSBb9+jHiw5Wcd+KxPPjqq6xZvJjJp5/O3ffcw7qNG3l54UK6LltG5UknUdslo7hMuf2nJ1sZxFp6MIIRrKS77WDv/ttoKF/LLfffzdxf/pJ+ffpwwa23Upvay+rAgcnO/uJdj5SUhOm9946fLYauUfbai+4pvedG1UW3kkIE4r0g1NWFbmyav8GrqYK0efF63QZKAU8r5JLbStalJuelagCMUuoppYE6usYe7qlhGz3z3G4+Z9bZlyfrdzO32dBk7PGz4sz99+IjulPLB4kHlJK/oYxaSmK/9WSeyloAn8mDkRTUqYVptoK1qQI5b9//fva+nm65Bb76VSCPs/eSkpYPbrNtW7izLW78+PCHnqvb6vh/BrPQe+iiRaFn0YMOCj2XxsWTQklJKHxTCuDq+v5M/fw6pkxax6yHK6ha3zX5nbFjQ7cjNTXhiXn39L6sRo0K/WPFb+oYOhQ+/JDJ55/Pf0ybxvrqav723HPcP3s2e+65J13Lynj21Vd5t6oq3M03dGjyNx54YKJfqGNPOIF75szhU+PG8fqSJSxYvBgGD2bzli307NWLPT7+capXr+bJJ5/k+B/8AIDePXrw0R57MDDet1RpKQwdyr+ddhoXXHwx06ZNw915+KWX+F1b9CMVo6TQyqqqQs/Fa9dC01UZmYVz9sLPMgrkUurpzwfUUk4vtjRZrwvphdwkZlPNIPrxIWXUUkMZ3dmRdraYr/hZp9HAMkYl4u3NZnZSyud4nH9wNJvpQ3e2s53u9GEzQOIs+mj+kXa2mlkgP8wZlFHLOOYmthWvi46fea+lggf5YqwhEr7MXdy2G8Nz5DwrLpSLLgqF2n33henXXgtnznHf+14yKTz9dOiu+pJL4Mor4Te/gVdeCV1ZP/VU0/vYsiWciffu3fQ6Dz8ctve734XpXI2bP/gB/Md/wKDYDQDvvw/9+iWXb92afmUAobvoeNfYcZnjJgCzH+0Cb6yDmhqmf3dNKJgbD4UAw4eH71dVhf2VlISEkKqsDMaM4eDSUj766COGDh/OkL335rzzzuO0007jYx/7GGMPOIADKytDMhiUvKEBs5CIxozh0oMP5sILL+SgL36RgyorOTL2ZNlhhx3G4YcfzoGHH87ee+/N0UcfHRLcEUcw5cormTBhAnvttRfPxp++r6jgiIMO4oILLmD8+PEAXHzxxRx++OFtN5qbu3eo15FHHuntVffu7uHUpLlXg0ODd2ObD6Tae7PRh7LSh7LSe7PRP8Wf/TJu9Uk8mM/GCvaaxIN+Gbf6fA4tSLyF3n+jl5n7I4+kz3viiabXv+228EfT1PJbb3VftSqs8847Yd7f/x6mhw1Lrpe6DXf3P//ZfceO8PmDD9xffDF8vuuuxvt46in3V15J/gHH5z/3nHtVVfg9b7zR/B/9s8+6z56dnL7ySveePZPTpaXu3/hGs5tYtGhRcmLjRve5c92XLnX/6CP3urrsX1q/PqxXV+e+eHH4vHlz42VNmTs3vPKxaVNYt7Y297q1te7btuW33QilHdMYYJ577jI25wrt7dXeksL777t//OOhTGi+3AjJYAirvDcbfQirC1uI6ZXfa/To9OlTTw3/8A0N6fO/8IUw/1OfCtOptm1rvN1bbw3Lhg8P0z/8ofuaNe7XXRemlyxp/g8vvp3Mz815/PH0GBYsSF/+61+7z5+fezutLK0Aa2hwf++95gv0TDt2hO80NOT/nblz3RcuzH/9DkZJoYAuvbS5MqUh7XUwCwp/RluMr1Gjdu17l1/u/vzz7mPHJuelni2eeGJyfryA2bYtnGVn+uMfk+t26+ZeXR3mf/SR+7p1yfV27nR/993cf3g33+z+la+Ez/kmBXf35cvd+/ZNj7nAshVgkduxw72+vu3320Z2JymoTWEXlZdnHdI1hQOhG4RBVNOdHezPO5HeZSJNiHeZffDB6UMi3ndf6HZ7773DGAZTpiSXXXBBGDkMwsA7ZqG+PnXs5GeeCXcSdOkS+oeC8IeRrb791FPD3TonnxyGiozr1Su9C/GSklAXnkusERkIw0OuzN5dcyOVleE1f37uRuHOrI3HPe5Q8skc7enVXq4U3n/ffdKkbCeY4apgBEt9JEt0ZZDPK37m2tzr2mtDHXnqvJtvbrzeCSekT2dWQ4B7nz7Z/1FTv5dvfXNHtHSp+7e+1bLqlggtWrTIG9pJLJ1BQ0PDbl0paOS1XTRyZLghI5uDeZ0jeJVljNKVQS733ps++PnBB4f3k05KzjvxxNCVdr9+4S6S/v3hD39IP1uO++tfk5/HjWt8h8uKFeHMujnz5+c1Dm6Htc8+YVS2dnKlUFZWxoYNGwjlluwOd2fDhg2Upd4u3EKqPtpFkyfDb3+bPq+sDPasfbdjVxO99FK45/zuu2F3H5//z/8Mheu554bp+H/6srLkg0nDh4dCPi71lsR4oRUffxjCLYap3Yp36xaqXz74oPH+4yOopWque8k//Sk8ZZhr8CBpVcOGDWP16tWsW7eu0KF0CmVlZQwb1vTt6rmo6+wWaq4tobSkgfqGZgZEL4T//M/w/rOf5V53y5b0+u7hw5PjBdfWJh9WqqlJ1pvfdVfTySP+t5XZHfh3vxvuY4fw4NKBB2bvMjxXN+IQkkv8fvF+/UJyuOGGsM3mhvAUKTL5dp29W/X7hXgVuk0h3pZQWpqsfi4pcR85eKufQsYtf4V+rV6dDPz888O8iy4Kd9CA+yGHhPfx47P/2A8+cL/jjuSdMldd5b7nnuHzD38Yvuvufsst6fsdN869vDy5nREj3C++ODldV+f+pz8l77t3D7dgLl+evv/49vKxYEEyThFpBN19FI0hQ0Jtxc6dyXkNDc6ENb/ZrSdqd1v8DppUQ4cmP8fPuo85JpxVu4eR1846K3w3m379wl04cT//eXgBXHtteEF4enbgQBg8OFQ/ffObicf/gVCPn6pLl3AXTqp99228/4ULwxO5+VCVj0irUFLYBS+8EN733TeMkzr3/jAcYqtasSIUyps3pxfaX/5ysqsBCFU+O3aEKp2TTgq3Pd55Z+PH+b/zndAnTOqIbSefHBpjr79+9+M955zwfsIJu7+tuEMOSd7qKSJtQm0KLdBUe0IZNdTQuI+WXXLUUXDPPeH2pri//jWceQ8cCD/5SSj0AY47Dp57rnX2KyKdWr5tCpHekmpmE8zsbTNbYmbTsizvZ2YPm9kCM/uXmbXr08Jly8KNNPE+unr0gPPOg+WMbP6LLfHii+kJAeBTnwrjLx9ySDizB/jb35QQRKTVRZYUzKwUmA6cAowGzjGz0RmrXQvMd/dDgfOBm6OKpzUMGRJ6z43fiFNbG6YzB4nfJQcdBL/4Re71Lrss1LUfe+zu71NEJEOUVwrjgSXuvszddwD3AhMz1hkN/BXA3d8CKs2slSvnW9e774becx9/HKZOhTVrdnOD8QewFi5M72ahKWaqZxeRyESZFIYCqbfDrI7NS/UacCaAmY0HRkAzgwS0A5WVUF0d2nOnT4fZs1u4ge9/PySC+H3+//u/4ZKjtJ093yAiRanQdx/dCNxsZvOBhcCrwM7MlcxsCjAFYHg+nYVFILORecaM8Crr7uQ9IN7zz4dbQs3g298OD1qVlKR3siYiUkBRXim8B6TeAD8sNi/B3Te7+4XuPobQplABLMvckLvPcvex7j62oqIiwpCbtmxZGGIzfvt9opF5+5D8NrBkCfzbvyWfF9hzz/DUrYhIOxJlUpgL7GdmI82sG3A28FjqCmbWN7YM4GLgeXffHGFMu2zIkDCEbENDqOlpUSPzmjXZH84SEWlnIqs+cvd6M7sCeAooBW539zfMbGps+UzgIOBOM3PgDeDfo4pnd2RWHcWfZv7FL+C2XF+eOzd9XFcRkXYs0jYFd38CeCJj3syUzy8C+0cZQ2tYtgyuvhoeeQS2bQtVR5MmwU8ufANOyvHlztwFs4h0OhpPIQ9DhoQqo23bQptwouroggmFDk1EpFUV+u6jDiPe39Hpp0NFBVRVEbJEc/70p8jjEhFpTUoKOWS2JzzwQHgvKwN65hi5KrMnUBGRdk7VRznE+zuKP2tWXh67FXU56SOAZUrtjVREpIPQlUIO8f6OamJPqNXUxNoTBjfzpbq6xmMDi4h0ACq5csjWXfaMGXDHrxuafpJZCUFEOihVH+XQZHfZO/YqbGAiIhFQUsgha3fZVW+3TnfZIiLtjJJCHqqrQzfZc+bEuste3kwXeOef33aBiYi0MlV+52H6dDj77NBbxfSf1kLt/8HyJla2HLepioi0Y7pSyMP114eH1667DujXD26/vemVDz20zeISEWlt5u6FjqFFxo4d6/PmzWuTfWW78wigjBpq6NF4gRnU1yf71xYRaSfM7GV3z9kZm0qvZmS984i7WM7I7F844gglBBHp0FSCNSPrnUdsbvrOow521SUikklJIYdGdx7RzNgI3/hG2wUmIhIB3X2UQ9qdR9OB276QfUVdJYhIJ6ArhRzS7jxqyjXXtFk8IiJRUlJoQnl5uJloxowwLvOMGWG6nCxjKPTs2fYBiohEQEmhCS2682j06LYNTkQkIkoKTcj7zqOZM8OAzSIinYCSQjPefTc0MD/+OEz9j53Z7zz6QhMNzyIiHZCSQjMqK8MtqQ89BNNPepjZZEkAelhNRDoRdXORRYu6t/jwQ+jbN9J4RER2l7q52A2NGpm71TfdyKyiwxYuAAAbYElEQVQrBRHpRFSiZTFkCJSWwrZt0L071NaVNN29hbrKFpFOREmhCS+8EN5PPx2m+oymu7dQUhCRTkTdXGTIbE944AGAyykjy2hrJ5wAvXq1VWgiIpHTlUKGRu0J5d50e8K0aW0bnIhIxCJNCmY2wczeNrMlZtaoBDWzPczsD2b2mpm9YWYXRhlPPho9tFbT0Hx32SIinUhkScHMSoHpwCnAaOAcM8vsD+JyYJG7HwYcD/zUzLpFFVO+0h5aY2bT7Qkd7HZeEZFcomxTGA8scfdlAGZ2LzARWJSyjgO9zcyAXsAHQH2EMeWlshKeeio8tHYbVxQ6HBGRNhNlUhgKrEqZXg18PGOdW4HHgPeB3sBkd2/I3JCZTQGmAAwfPjySYKFxI/OMGTADb3pMZhGRTqbQDc0nA/OBvYAxwK1m1idzJXef5e5j3X1sRUVFZMG0eEzm/v0ji0VEpBCiTArvAXunTA+LzUt1ITDbgyXAcuDACGNqVovGZL7rLhg3ru2DFBGJUJRJYS6wn5mNjDUen02oKkq1EjgRwMwGAQcAyyKMKafqavjyl8MQCV/+UkPTjcznnde2gYmItIHIkoK71wNXAE8BbwL3u/sbZjbVzKbGVrse+KSZLQT+AnzL3ddHFVM+Zs8O1Ubz54dnFLL2jCoi0kmpl9QULeodtYMdNxEpbuoldRe06GlmEZFOSEkhRYsamkVEOiElhQxpTzP7bU03NIuIdELqJTVDXk8zl5W1aUwiIm1FSSGmRU8zb93atsGJiLQRVR/FtOhpZg3BKSKdlEq3mLwbmY89tjABioi0ASWFFNXVMHUqzJkT3rM2MusqQUQ6MbUppJg9O/l5+nTgtixPM2tMZhHpxHTa25TNmwsdgYhIm1NSaMonPpF9vq4URKQTU1JoyqJF2ecrKYhIJ6ak0FJXXVXoCEREIqOkkKKqCo47DtasaWKFFSvgtNPaMiQRkTalpJDi+uvhhRfguuuyLBwyBEaMaPOYRETakm5JJc8uLl5+uTDBiYi0IV0pkGcXF0OGFCY4EZE2pKSAxlEQEYlTUojJq4sLEZFOTmM0NyXzeYQOdpxERFJpjObWNG5coSMQEWkTSgr5GKSqJBEpDkoK+VDXFiJSJJQU8qGkICJFQkkhHxpYR0SKhEq7bBYvTp/ea6/CxCEi0saUFGLSOsO74470hT/9aUFiEhFpa5EmBTObYGZvm9kSM5uWZfn/M7P5sdfrZrbTzPpHGVNT0jrD+9GP0heWlRUiJBGRNhfZw2tmVgq8A3waWA3MBc5x96yj15jZacDX3f1TzW23tR9ey+wMLy6tMzw9uCYiHVx7eHhtPLDE3Ze5+w7gXmBiM+ufA/w+wniyytkZ3oABbR2SiEjBRJkUhgKrUqZXx+Y1YmY9gAnAQxHGk1XOzvD+/Oe2DklEpGDaS0PzacA/3P2DbAvNbIqZzTOzeevWrWv1nTfbGd7hh7f6/kRE2qso2xQ+AXzf3U+OTV8D4O4/yrLuw8AD7n5Pru22SYd4qQ+rqT1BRDqB9tCmMBfYz8xGmlk34GzgscyVzGwP4Djg0QhjERGRPEQ2HKe715vZFcBTQClwu7u/YWZTY8tnxladBDzt7lujikVERPKj8RSyUfWRiHQy7aH6qGNSEhCRIqakkOmenG3dIiKdlpJCpkVZH7gWESkKSgqZPsj6qISISFFQUsj07LOFjkBEpGCUFMjoNju1ofmGGwoWk4hIISgpkNFtdkNDckH/gvTiLSJSMJE9vNYRZHabPWMGzGBxerfZIiJFpKivFHJ2m536EJuISBEo6qSQs9tsJQURKTJFnRQgR7fZIiJFpqjbFABmz05+nj4duO0LyRm6UhCRIlP0VwoiIpKkpNCcPfcsdAQiIm1KSaE5p51W6AhERNqUkkJz1KYgIkVGSUFERBKUFEREJCFnUjCz0rYIpN2prCx0BCIibS6fK4XFZnaTmY2OPJr25MQTCx2BiEibyycpHAa8A/zKzOaY2RQz6xNxXIWxcGHyc4lq1kSk+OQs+dz9I3f/pbt/EvgW8D2gyszuNLNRkUcYsbSxFMaOTS4oLc5aMxEpbnm1KZjZ6Wb2MPBz4KfAPsAfgCciji9yaWMp7NiRXKArBREpQvn0fbQYeBa4yd3/mTL/QTM7Npqwopd9LAVPjqWgpCAiRSifku9Qd//3jIQAgLt/NYKY2kTOsRSUFESkCOVzpVBvZpcDBwNl8ZnuflFkUbWBnGMpiIgUoXxOh38HDAZOBv4GDAM+ijKotqKxFERE0uVzpTDK3b9oZhPd/U4zuwf4e9SBtQWNpSAiki6fK4W62PtGMzsE2APIq09pM5tgZm+b2RIzm9bEOseb2Xwze8PM/pZf2G1ASUFEilA+VwqzzKwf8B3gMaAX8N1cX4p1jzEd+DSwGphrZo+5+6KUdfoCtwET3H2lmbWfAQy++c1CRyAi0uaaTQpmVgJsdvcPgecJzyfkazywxN2XxbZ1LzARWJSyzrnAbHdfCeDua1uw/WgNGVLoCERE2lyz1Ufu3gDs6inzUGBVyvTq2LxU+wP9zOw5M3vZzM7fxX2JiEgryKf66Bkzuxq4D9gan+nuH7TS/o8ETgTKgRfNbI67v5O6kplNAaYADB8+vBV2m8VHKTdUqT1BRIpUPklhcuz98pR5Tu6qpPeAvVOmh8XmpVoNbHD3rcBWM3ueZAd8yZ25zwJmAYwdO9bziLnlVqVc1PTtG8kuRETau5xJwd1H7uK25wL7mdlIQjI4m9CGkOpR4FYz6wJ0Az4O/O8u7m/3NDQkP6szPBEpUjmTQlP1/O7+2+a+5+71ZnYF8BRQCtzu7m+Y2dTY8pnu/qaZ/QlYADQAv3L311v6I1qFp1yAdO1akBBERAotn+qjcSmfywj1/68AzSYFAHd/goyeVN19Zsb0TcBNecQRrdQrhaGZ7eEiIsUhn+qjK1OnY88W3BtZRIWSeqWgNgURKVK70hXoVmBX2xnar9QrBRGRIpVPm8IfCHcbQUgio4H7owyqIJ5+utARiIgUXD5tCj9J+VwPvOvuqyOKp3CuuabQEYiIFFw+SWElUOXutQBmVm5mle6+ItLICkl3H4lIkcqnTeEBwu2icTtj8zqvX/+60BGIiBREPkmhi7snRrSPfe4WXUjtgDrDE5EilU9SWGdmp8cnzGwisD66kNpOVRUcdxysWVPoSERE2od8ksJU4FozW2lmK4FvAZdEG1bbuP56eOEFuO66QkciItI+5PPw2lLgKDPrFZveEnlUESsvh9ra5PSMGTADp4waauhRuMBERAos55WCmd1gZn3dfYu7bzGzfmb2g7YILirLlsG550KPWPnfo6yB87iL5Z3wmTwRkZbIp/roFHffGJ+IjcJ2anQhRW/IEOjTJ1wtlJVBba3Th80MprrQoYmIFFQ+SaHUzLrHJ8ysHOjezPodQnU1TJ0Kc+bAVGayhkGFDklEpODyeXjtbuAvZnYHYMAFwJ1RBtUWZs9Ofp7OFYULRESkHcmnofnHZvYacBKhD6SngBFRB1Ywl1+eex0RkU4q315SqwkJ4YvAp4A3I4uo0Pr0KXQEIiIF0+SVgpntD5wTe60H7gPM3U9oo9gKo0s+NWoiIp1TcyXgW8Dfgc+5+xIAM/t6m0RVSCW7MsSEiEjn0FwJeCZQBTxrZr80sxMJDc2dm5KCiBSxJktAd3/E3c8GDgSeBb4G7GlmM8zsM20VYJuzzp/3RESakvO02N23uvs97n4aMAx4ldD/UeekpCAiRaxFdSXu/qG7z3L3E6MKqOBUfSQiRUwlYKZRowodgYhIwSgpZCovL3QEIiIFU9RJITHITmq/R0OHFi4gEZECK+qkkBhkh/9KzjziiMIFJCJSYEWZFMrLw01GM2ZAQwPM4DIMp5xthQ5NRKSgijIpNBpkh60aZEdEhIiTgplNMLO3zWyJmU3Lsvx4M9tkZvNjr//Ktp3WljbITrcGainTIDsiIuQ3nsIuMbNSYDrwaWA1MNfMHnP3RRmr/t3dPxdVHE2JD7IzZd2PmPVAX6oY3NYhiIi0O1F2CToeWOLuywDM7F5gIpCZFAoiMcjO5AVM5/6CxiIi0l5EWX00FFiVMr06Ni/TJ81sgZk9aWYHRxhPdvenJITTTmvz3YuItCeFHjzgFWC4u28xs1OBR4D9MlcysynAFIDhw4dHF83550e3bRGRDiDKK4X3gL1TpofF5iW4+2Z33xL7/ATQ1cwGZm4o1t/SWHcfW1FREV3EPXtGt20RkQ4gyqQwF9jPzEaaWTfgbOCx1BXMbLBZ6JbUzMbH4tkQYUzNmzChYLsWEWkPIqs+cvd6M7sCeAooBW539zfMbGps+UzgC8ClZlYP1ABnu7tHFVNO6jZbRIqcFbIM3hVjx471efPmtd4GUxNBBzsWIiL5MrOX3X1srvWK8olmERHJrriTwk03FToCEZF2pbiTwo03FjoCEZF2pbiTghqWRUTSFHdSEBGRNMWdFOrrCx2BiEi7UtxJYdOmQkcgItKuFHdSEBGRNEoKIiKSULRJoWrFdo7jOdYwqNChiIi0G0WbFK6/znmBY7iONhkBVESkQyi6pFBeHh5PmHFHGQ2UMoPLMJxyqyl0aCIiBVd0SWHZMjj3XOhRHjq/68FWzuMuln/vzgJHJiJSeEWXFIYMgT59oHY7lFFDLWX0YTODTz2i0KGJiBRc0SUFgOpqmHrBduZwFFOZGRqbx40rdFgiIgVXvOMpfPABDBiQnO5gx0FEpCU0noKIiLSYkoKIiCQoKYiISIKSgoiIJCgpiIhIQnEmhfXr4dhjCx2FiEi7U5xJ4eqr4Y03Ch2FiEi7U5xJYfnyQkcgItIuFWdSMCt0BCIi7ZKSgoiIJBRnUhARkayUFEREJCHSpGBmE8zsbTNbYmbTmllvnJnVm9kXoownZYdtshsRkY4msqRgZqXAdOAUYDRwjpmNbmK9HwNPRxVLTj16FGzXIiLtSZRXCuOBJe6+zN13APcCE7OsdyXwELA2wliad9llBdu1iEh7EmVSGAqsSpleHZuXYGZDgUnAjAjjaOzZZ9On+/dv092LiLRXhW5o/jnwLXdvaG4lM5tiZvPMbN66detaP4qrr279bYqIdEBdItz2e8DeKdPDYvNSjQXutdDwOxA41czq3f2R1JXcfRYwC8LIa60eadeurb5JEZGOKMorhbnAfmY20sy6AWcDj6Wu4O4j3b3S3SuBB4HLMhNCFKoYzHE8F8ZmFhGRhMiSgrvXA1cATwFvAve7+xtmNtXMpka133xcz3d5gWO4jv8qZBgiIu2OeQcbsH7s2LE+b968XfpueTnU1jaeX1YGNTW7GZiISDtmZi+7+9hc6xW6oblNLVsG554LPdgKhPfzuEudpoqIxBRVUhgyBPr0gVrKKKOGWsrow2YGDy50ZCIi7UNRJQWA6mqYykzmcBRTmanGZhGRFEXVppCQ2vfRHnvAxo27tz0RkXZObQr56mBJUUQkSkoKIiKSoKQgIiIJSgqqPhIRSVBSuOKKQkcgItJuKCl86UuFjkBEpN0ovqSwaVOhIxARabeKLymoDUFEpEnFlxQyKUmIiCQUX1J44on0aSUFEZGE4ksKW7cWOgIRkXar6JJC1eaeGnVNRKQJRZcUrn/88PRR11R9JCKSUDS9pDY56lp3p6bWGi8QEelE1EtqhsSoaxbG3ezBVs47YyvLVyghiIjEdSl0AG0lMeqad0uOujakVKOuiYikKJorBYB334VBrOFxPhtGXVtT6IhERNqXokoKlZVQzWAe4gtM5wpmzy50RCIi7UtRJIXy8jAC54wZ0EApM7gMwykvL3RkIiLtS1EkhUQjc48w3YOtnMddLF9e2LhERNqbokgKiUbmWpKNzGxWI7OISIaiSAoA1dUwdSrM4ajQyKwnmkVEGimah9cSLOW5hA7220VEdpUeXstQVQXHHYeuEEREmhFpUjCzCWb2tpktMbNpWZZPNLMFZjbfzOaZ2TFRxXL99fDCC57s80hERBqJrPrIzEqBd4BPA6uBucA57r4oZZ1ewFZ3dzM7FLjf3Q9sbrstrT5qss8jaqhx3ZMqIsWhPVQfjQeWuPsyd98B3AtMTF3B3bd4Miv1BFo9QzV5OyojW3tXIiIdXpRJYSiwKmV6dWxeGjObZGZvAX8ELmrtINJuR+3ekLwdlerW3pWISIdX8IZmd384VmV0BnB9tnXMbEqszWHeunXrWryPxO2o969M3o565ZW7GbmISOcTZS+p7wF7p0wPi83Lyt2fN7N9zGygu6/PWDYLmAWhTaGlgST6OFpcx3SuCJ/7qcFZRCRTlFcKc4H9zGykmXUDzgYeS13BzEaZhQcHzOwIoDuwIbKITGMniIg0J7IrBXevN7MrgKeAUuB2d3/DzKbGls8EPg+cb2Z1QA0w2aN8mq4kJQfqwTURkUYiHWTH3Z8AnsiYNzPl84+BH0cZQ5pFi5KfGxrabLciIh1FwRua29RppyU/60pBRKSR4koKqZQUREQaKd6ksP/+hY5ARKTdKd6k8JWvFDoCEZF2p3iTgm5PFRFppHiTgoiINKKkICIiCUoKIiKSoKQgIiIJSgoiIpKgpCAiIglKCiIiklCcSeGOOwodgYhIu1ScSUFERLIqzqRw5JGFjkBEpF0qnqSQ2ivqoEGFi0NEpB0rnqQgIiI5FU9SWLMm+VljKYiIZFU8SeGnP01+3nPPwsUhItKOFU9SSB2TWd1mi4hkVTxJQVVGIiI5FU9SePvtQkcgItLuFU9SWLiw0BGIiLR7xZMUREQkp+JJCmpTEBHJqXiSQurdRyIikpWSgoiIJCgpiIhIQqRJwcwmmNnbZrbEzKZlWX6emS0ws4Vm9k8zOyyyYNSmICKSU2RJwcxKgenAKcBo4BwzG52x2nLgOHf/GHA9MCuqeJQURERyi/JKYTywxN2XufsO4F5gYuoK7v5Pd/8wNjkHGBZZNBMn5l5HRKTIRZkUhgKrUqZXx+Y15d+BJ7MtMLMpZjbPzOatW7du16LZf/9d+56ISBFpFw3NZnYCISl8K9tyd5/l7mPdfWxFRcWu7UTVRyIiOXWJcNvvAXunTA+LzUtjZocCvwJOcfcNEcYjIiI5RHmlMBfYz8xGmlk34GzgsdQVzGw4MBv4sru/E2EsIiKSh8iuFNy93syuAJ4CSoHb3f0NM5saWz4T+C9gAHCbhTEO6t19bCQBHXdceN/V6icRkSIQZfUR7v4E8ETGvJkpny8GLo4yhoTBg8N7nz5tsjsRkY6oXTQ0t4nKSrj6avjjHwsdiYhIuxXplUK7YgY33VToKERE2rXiuVIQEZGclBRERCRBSUFERBKUFEREJEFJQUREEpQUREQkQUlBREQSlBRERCRBSUFERBKUFEREJEFJQUREEpQUREQkQUlBREQSzDvY2MVmtg54dxe/PhBY34rhdHQ6Hul0PJJ0LNJ1huMxwt1zjjLW4ZLC7jCzeZGN7NYB6Xik0/FI0rFIV0zHQ9VHIiKSoKQgIiIJxZYUZhU6gHZGxyOdjkeSjkW6ojkeRdWmICIizSu2KwUREWlG0SQFM5tgZm+b2RIzm1boeKJgZnub2bNmtsjM3jCzq2Lz+5vZn81scey9X8p3rokdk7fN7OSU+Uea2cLYslvMzArxm3aXmZWa2atm9nhsupiPRV8ze9DM3jKzN83sE0V+PL4e+3/yupn93szKivl4JLh7p38BpcBSYB+gG/AaMLrQcUXwO4cAR8Q+9wbeAUYD/wNMi82fBvw49nl07Fh0B0bGjlFpbNm/gKMAA54ETin079vFY/KfwD3A47HpYj4WdwIXxz53A/oW6/EAhgLLgfLY9P3ABcV6PFJfxXKlMB5Y4u7L3H0HcC8wscAxtTp3r3L3V2KfPwLeJPzxTyQUCMTez4h9ngjc6+7b3X05sAQYb2ZDgD7uPsfDX/1vU77TYZjZMOCzwK9SZhfrsdgDOBb4NYC773D3jRTp8YjpApSbWRegB/A+xX08gOKpPhoKrEqZXh2b12mZWSVwOPASMMjdq2KL1gCDYp+bOi5DY58z53c0Pwe+CTSkzCvWYzESWAfcEatO+5WZ9aRIj4e7vwf8BFgJVAGb3P1pivR4pCqWpFBUzKwX8BDwNXffnLosdjbT6W85M7PPAWvd/eWm1imWYxHTBTgCmOHuhwNbCdUjCcV0PGJtBRMJyXIvoKeZfSl1nWI6HqmKJSm8B+ydMj0sNq/TMbOuhIRwt7vPjs2ujl3mEntfG5vf1HF5L/Y5c35HcjRwupmtIFQXfsrM7qI4jwWEM9jV7v5SbPpBQpIo1uNxErDc3de5ex0wG/gkxXs8EoolKcwF9jOzkWbWDTgbeKzAMbW62F0PvwbedPefpSx6DPhK7PNXgEdT5p9tZt3NbCSwH/Cv2OXzZjM7KrbN81O+0yG4+zXuPszdKwn/3n919y9RhMcCwN3XAKvM7IDYrBOBRRTp8SBUGx1lZj1iv+NEQhtcsR6PpEK3dLfVCziVcDfOUuDbhY4not94DOFydwEwP/Y6FRgA/AVYDDwD9E/5zrdjx+RtUu6aAMYCr8eW3UrsQceO+AKOJ3n3UdEeC2AMMC/29/EI0K/Ij8d/A2/FfsvvCHcWFe3xiL/0RLOIiCQUS/WRiIjkQUlBREQSlBRERCRBSUFERBKUFEREJEFJQYqWmf0z9l5pZue28ravzbYvkfZOt6RK0TOz44Gr3f1zLfhOF3evb2b5Fnfv1RrxibQlXSlI0TKzLbGPNwL/ZmbzY33sl5rZTWY218wWmNklsfWPN7O/m9ljhKeBMbNHzOzlWL/8U2LzbiT0vjnfzO5O3ZcFN8X68F9oZpNTtv2cJcc7uLvD98svHVKXQgcg0g5MI+VKIVa4b3L3cWbWHfiHmT0dW/cI4BAP3ScDXOTuH5hZOTDXzB5y92lmdoW7j8myrzMJTxYfBgyMfef52LLDgYMJXTj/g9B/0wut/3NFmqYrBZHGPgOcb2bzCV2PDyD0dQOhv5vlKet+1cxeA+YQOkzbj+YdA/ze3Xe6ezXwN2BcyrZXu3sDoYuSylb5NSItoCsFkcYMuNLdn0qbGdoetmZMnwR8wt23mdlzQNlu7Hd7yued6P+nFICuFETgI8LwpXFPAZfGuiHHzPaPDUiTaQ/gw1hCOJAwJGNcXfz7Gf4OTI61W1QQRkP7V6v8CpFWoDMRkdBr6M5YNdBvgJsJVTevxBp715F9iMU/AVPN7E1Cz5lzUpbNAhaY2Svufl7K/IeBTxDG+3Xgm+6+JpZURApOt6SKiEiCqo9ERCRBSUFERBKUFEREJEFJQUREEpQUREQkQUlBREQSlBRERCRBSUFERBL+P3Oyy4su+OAoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7ff859739588>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot Accuracies\n",
    "plt.figure(figsize = (6,6))\n",
    "\n",
    "plt.plot(t, np.array(train_acc), 'r-', t[t % 25 == 0], validation_acc, 'b*')\n",
    "plt.xlabel(\"iteration\")\n",
    "plt.ylabel(\"Accuray\")\n",
    "plt.legend(['train', 'validation'], loc='upper right')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate on test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from checkpoints-crnn/har.ckpt\n",
      "Test accuracy: 0.882917\n"
     ]
    }
   ],
   "source": [
    "test_acc = []\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    # Restore\n",
    "    saver.restore(sess, tf.train.latest_checkpoint('checkpoints-crnn'))\n",
    "    \n",
    "    for x_t, y_t in get_batches(X_test, y_test, batch_size):\n",
    "        feed = {inputs_: x_t,\n",
    "                labels_: y_t,\n",
    "                keep_prob_: 1}\n",
    "        \n",
    "        batch_acc = sess.run(accuracy, feed_dict=feed)\n",
    "        test_acc.append(batch_acc)\n",
    "    print(\"Test accuracy: {:.6f}\".format(np.mean(test_acc)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}