lasagne_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Lasagne example\n",
    "codes from [**lasagne tutorial**](http://lasagne.readthedocs.org/en/latest/user/tutorial.html#before-we-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "\n",
    "\"\"\"\n",
    "This example is deliberately structured as a long flat file, focusing on how\n",
    "to use Lasagne, instead of focusing on writing maximally modular and reusable\n",
    "code. It is used as the foundation for the introductory Lasagne tutorial:\n",
    "http://lasagne.readthedocs.org/en/latest/user/tutorial.html\n",
    "\n",
    "More in-depth examples and reproductions of paper results are maintained in\n",
    "a separate repository: https://github.com/Lasagne/Recipes\n",
    "\"\"\"\n",
    "\n",
    "from __future__ import print_function\n",
    "\n",
    "import sys\n",
    "import os\n",
    "import time\n",
    "\n",
    "import numpy as np\n",
    "import theano\n",
    "import theano.tensor as T\n",
    "import sklearn\n",
    "from sklearn import datasets\n",
    "import lasagne\n",
    "\n",
    "\n",
    "def load_data():\n",
    "    \n",
    "    face=sklearn.datasets.fetch_olivetti_faces(shuffle=True)\n",
    "    train_set=(face.data[0:200,].reshape((200,1,64,64)),face.target[0:200,].astype(np.int32))\n",
    "    valid_set=(face.data[200:300,].reshape((100,1,64,64)),face.target[200:300,].astype(np.int32))\n",
    "    test_set =(face.data[300:400,].reshape((100,1,64,64)),face.target[300:400,].astype(np.int32))\n",
    "    #train_set, valid_set, test_set format: tuple(input, target)\n",
    "    #input is an np.ndarray of 2 dimensions (a matrix)\n",
    "    #witch row's correspond to an example. target is a\n",
    "    #np.ndarray of 1 dimensions (vector)) that have the same length as\n",
    "    #the number of rows in the input. It should give the target\n",
    "    #target to the example with the same index in the input.\n",
    "\n",
    "    rval = [train_set,valid_set, test_set]\n",
    "    return rval\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_cnn(input_var=None):\n",
    "    # As a third model, we'll create a CNN of two convolution + pooling stages\n",
    "    # and a fully-connected hidden layer in front of the output layer.\n",
    "\n",
    "    # Input layer, as usual:\n",
    "    network = lasagne.layers.InputLayer(shape=(20, 1, 64, 64),\n",
    "                                        input_var=input_var)\n",
    "    # This time we do not apply input dropout, as it tends to work less well\n",
    "    # for convolutional layers.\n",
    "\n",
    "    # Convolutional layer with 32 kernels of size 5x5. Strided and padded\n",
    "    # convolutions are supported as well; see the docstring.\n",
    "    network = lasagne.layers.Conv2DLayer(\n",
    "            network, num_filters=32, filter_size=(5, 5),\n",
    "            nonlinearity=lasagne.nonlinearities.rectify,\n",
    "            W=lasagne.init.GlorotUniform())\n",
    "    # Expert note: Lasagne provides alternative convolutional layers that\n",
    "    # override Theano's choice of which implementation to use; for details\n",
    "    # please see http://lasagne.readthedocs.org/en/latest/user/tutorial.html.\n",
    "\n",
    "    # Max-pooling layer of factor 2 in both dimensions:\n",
    "    network = lasagne.layers.MaxPool2DLayer(network, pool_size=(2, 2))\n",
    "\n",
    "    # Another convolution with 32 5x5 kernels, and another 2x2 pooling:\n",
    "    network = lasagne.layers.Conv2DLayer(\n",
    "            network, num_filters=32, filter_size=(5, 5),\n",
    "            nonlinearity=lasagne.nonlinearities.rectify)\n",
    "    network = lasagne.layers.MaxPool2DLayer(network, pool_size=(2, 2))\n",
    "\n",
    "    # A fully-connected layer of 256 units with 50% dropout on its inputs:\n",
    "    network = lasagne.layers.DenseLayer(\n",
    "            lasagne.layers.dropout(network, p=.5),\n",
    "            num_units=256,\n",
    "            nonlinearity=lasagne.nonlinearities.rectify)\n",
    "\n",
    "    # And, finally, the 10-unit output layer with 50% dropout on its inputs:\n",
    "    network = lasagne.layers.DenseLayer(\n",
    "            lasagne.layers.dropout(network, p=.5),\n",
    "            num_units=40,\n",
    "            nonlinearity=lasagne.nonlinearities.softmax)\n",
    "\n",
    "    return network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def iterate_minibatches(inputs, targets, batchsize, shuffle=False):\n",
    "    assert len(inputs) == len(targets)\n",
    "    if shuffle:\n",
    "        indices = np.arange(len(inputs))\n",
    "        np.random.shuffle(indices)\n",
    "    for start_idx in range(0, len(inputs) - batchsize + 1, batchsize):\n",
    "        if shuffle:\n",
    "            excerpt = indices[start_idx:start_idx + batchsize]\n",
    "        else:\n",
    "            excerpt = slice(start_idx, start_idx + batchsize)\n",
    "        yield inputs[excerpt], targets[excerpt]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def main(num_epochs=5):\n",
    "    # Load the dataset\n",
    "    print(\"Loading data...\")\n",
    "    datasets = load_data()\n",
    "    X_train, y_train = datasets[0]\n",
    "    X_val, y_val = datasets[1]\n",
    "    X_test, y_test = datasets[2]\n",
    "    # Prepare Theano variables for inputs and targets\n",
    "    input_var = T.tensor4('inputs')\n",
    "    target_var = T.ivector('targets')\n",
    "\n",
    "    # Create neural network model (depending on first command line parameter)\n",
    "    print(\"Building model and compiling functions...\")\n",
    "\n",
    "    network = build_cnn(input_var)\n",
    "    # Create a loss expression for training, i.e., a scalar objective we want\n",
    "    # to minimize (for our multi-class problem, it is the cross-entropy loss):\n",
    "    prediction = lasagne.layers.get_output(network)\n",
    "    loss = lasagne.objectives.categorical_crossentropy(prediction, target_var)\n",
    "    loss = loss.mean()\n",
    "    # We could add some weight decay as well here, see lasagne.regularization.\n",
    "\n",
    "    # Create update expressions for training, i.e., how to modify the\n",
    "    # parameters at each training step. Here, we'll use Stochastic Gradient\n",
    "    # Descent (SGD) with Nesterov momentum, but Lasagne offers plenty more.\n",
    "    params = lasagne.layers.get_all_params(network, trainable=True)\n",
    "    updates = lasagne.updates.nesterov_momentum(\n",
    "            loss, params, learning_rate=0.01, momentum=0.9)\n",
    "\n",
    "    # Create a loss expression for validation/testing. The crucial difference\n",
    "    # here is that we do a deterministic forward pass through the network,\n",
    "    # disabling dropout layers.\n",
    "    test_prediction = lasagne.layers.get_output(network, deterministic=True)\n",
    "    test_loss = lasagne.objectives.categorical_crossentropy(test_prediction,\n",
    "                                                            target_var)\n",
    "    test_loss = test_loss.mean()\n",
    "    # As a bonus, also create an expression for the classification accuracy:\n",
    "    test_acc = T.mean(T.eq(T.argmax(test_prediction, axis=1), target_var),\n",
    "                      dtype=theano.config.floatX)\n",
    "\n",
    "    # Compile a function performing a training step on a mini-batch (by giving\n",
    "    # the updates dictionary) and returning the corresponding training loss:\n",
    "    train_fn = theano.function([input_var, target_var], loss, updates=updates)\n",
    "\n",
    "    # Compile a second function computing the validation loss and accuracy:\n",
    "    val_fn = theano.function([input_var, target_var], [test_loss, test_acc])\n",
    "\n",
    "    # Finally, launch the training loop.\n",
    "    print(\"Starting training...\")\n",
    "    # We iterate over epochs:\n",
    "    for epoch in range(num_epochs):\n",
    "        # In each epoch, we do a full pass over the training data:\n",
    "        train_err = 0\n",
    "        train_batches = 0\n",
    "        start_time = time.time()\n",
    "        for batch in iterate_minibatches(X_train, y_train, 20, shuffle=False):\n",
    "            inputs, targets = batch\n",
    "            train_err += train_fn(inputs, targets)\n",
    "            train_batches += 1\n",
    "\n",
    "        # And a full pass over the validation data:\n",
    "        val_err = 0\n",
    "        val_acc = 0\n",
    "        val_batches = 0\n",
    "        for batch in iterate_minibatches(X_val, y_val, 20, shuffle=False):\n",
    "            inputs, targets = batch\n",
    "            err, acc = val_fn(inputs, targets)\n",
    "            val_err += err\n",
    "            val_acc += acc\n",
    "            val_batches += 1\n",
    "\n",
    "        # Then we print the results for this epoch:\n",
    "        print(\"Epoch {} of {} took {:.3f}s\".format(\n",
    "            epoch + 1, num_epochs, time.time() - start_time))\n",
    "        print(\"  training loss:\\t\\t{:.6f}\".format(train_err / train_batches))\n",
    "        print(\"  validation loss:\\t\\t{:.6f}\".format(val_err / val_batches))\n",
    "        print(\"  validation accuracy:\\t\\t{:.2f} %\".format(\n",
    "            val_acc / val_batches * 100))\n",
    "\n",
    "    # After training, we compute and print the test error:\n",
    "    test_err = 0\n",
    "    test_acc = 0\n",
    "    test_batches = 0\n",
    "    for batch in iterate_minibatches(X_test, y_test, 20, shuffle=False):\n",
    "        inputs, targets = batch\n",
    "        err, acc = val_fn(inputs, targets)\n",
    "        test_err += err\n",
    "        test_acc += acc\n",
    "        test_batches += 1\n",
    "    print(\"Final results:\")\n",
    "    print(\"  test loss:\\t\\t\\t{:.6f}\".format(test_err / test_batches))\n",
    "    print(\"  test accuracy:\\t\\t{:.2f} %\".format(\n",
    "        test_acc / test_batches * 100))\n",
    "\n",
    "    # Optionally, you could now dump the network weights to a file like this:\n",
    "    # np.savez('model.npz', *lasagne.layers.get_all_param_values(network))\n",
    "    #\n",
    "    # And load them again later on like this:\n",
    "    # with np.load('model.npz') as f:\n",
    "    #     param_values = [f['arr_%d' % i] for i in range(len(f.files))]\n",
    "    # lasagne.layers.set_all_param_values(network, param_values)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data...\n",
      "Building model and compiling functions...\n",
      "Starting training...\n",
      "Epoch 1 of 100 took 4.293s\n",
      "  training loss:\t\t3.721591\n",
      "  validation loss:\t\t3.692663\n",
      "  validation accuracy:\t\t5.00 %\n",
      "Epoch 2 of 100 took 4.414s\n",
      "  training loss:\t\t3.695866\n",
      "  validation loss:\t\t3.689976\n",
      "  validation accuracy:\t\t3.00 %\n",
      "Epoch 3 of 100 took 4.331s\n",
      "  training loss:\t\t3.688406\n",
      "  validation loss:\t\t3.710782\n",
      "  validation accuracy:\t\t2.00 %\n",
      "Epoch 4 of 100 took 4.291s\n",
      "  training loss:\t\t3.688380\n",
      "  validation loss:\t\t3.705348\n",
      "  validation accuracy:\t\t2.00 %\n",
      "Epoch 5 of 100 took 4.705s\n",
      "  training loss:\t\t3.677979\n",
      "  validation loss:\t\t3.717669\n",
      "  validation accuracy:\t\t2.00 %\n",
      "Epoch 6 of 100 took 4.366s\n",
      "  training loss:\t\t3.667840\n",
      "  validation loss:\t\t3.737466\n",
      "  validation accuracy:\t\t2.00 %\n",
      "Epoch 7 of 100 took 4.228s\n",
      "  training loss:\t\t3.671016\n",
      "  validation loss:\t\t3.744931\n",
      "  validation accuracy:\t\t2.00 %\n",
      "Epoch 8 of 100 took 4.239s\n",
      "  training loss:\t\t3.673022\n",
      "  validation loss:\t\t3.723901\n",
      "  validation accuracy:\t\t3.00 %\n",
      "Epoch 9 of 100 took 4.485s\n",
      "  training loss:\t\t3.662701\n",
      "  validation loss:\t\t3.740178\n",
      "  validation accuracy:\t\t4.00 %\n",
      "Epoch 10 of 100 took 4.769s\n",
      "  training loss:\t\t3.650825\n",
      "  validation loss:\t\t3.748749\n",
      "  validation accuracy:\t\t2.00 %\n",
      "Epoch 11 of 100 took 4.773s\n",
      "  training loss:\t\t3.637479\n",
      "  validation loss:\t\t3.730272\n",
      "  validation accuracy:\t\t3.00 %\n",
      "Epoch 12 of 100 took 4.751s\n",
      "  training loss:\t\t3.641955\n",
      "  validation loss:\t\t3.731977\n",
      "  validation accuracy:\t\t2.00 %\n",
      "Epoch 13 of 100 took 4.341s\n",
      "  training loss:\t\t3.619490\n",
      "  validation loss:\t\t3.704992\n",
      "  validation accuracy:\t\t6.00 %\n",
      "Epoch 14 of 100 took 4.297s\n",
      "  training loss:\t\t3.601250\n",
      "  validation loss:\t\t3.699573\n",
      "  validation accuracy:\t\t8.00 %\n",
      "Epoch 15 of 100 took 4.264s\n",
      "  training loss:\t\t3.579416\n",
      "  validation loss:\t\t3.675743\n",
      "  validation accuracy:\t\t8.00 %\n",
      "Epoch 16 of 100 took 4.267s\n",
      "  training loss:\t\t3.500350\n",
      "  validation loss:\t\t3.619044\n",
      "  validation accuracy:\t\t14.00 %\n",
      "Epoch 17 of 100 took 4.251s\n",
      "  training loss:\t\t3.448962\n",
      "  validation loss:\t\t3.598465\n",
      "  validation accuracy:\t\t22.00 %\n",
      "Epoch 18 of 100 took 4.253s\n",
      "  training loss:\t\t3.379564\n",
      "  validation loss:\t\t3.446823\n",
      "  validation accuracy:\t\t25.00 %\n",
      "Epoch 19 of 100 took 4.246s\n",
      "  training loss:\t\t3.252572\n",
      "  validation loss:\t\t3.264270\n",
      "  validation accuracy:\t\t25.00 %\n",
      "Epoch 20 of 100 took 4.485s\n",
      "  training loss:\t\t3.059002\n",
      "  validation loss:\t\t3.299073\n",
      "  validation accuracy:\t\t33.00 %\n",
      "Epoch 21 of 100 took 4.439s\n",
      "  training loss:\t\t2.879454\n",
      "  validation loss:\t\t2.658152\n",
      "  validation accuracy:\t\t39.00 %\n",
      "Epoch 22 of 100 took 4.276s\n",
      "  training loss:\t\t2.610487\n",
      "  validation loss:\t\t2.265993\n",
      "  validation accuracy:\t\t44.00 %\n",
      "Epoch 23 of 100 took 4.279s\n",
      "  training loss:\t\t2.253483\n",
      "  validation loss:\t\t2.038647\n",
      "  validation accuracy:\t\t54.00 %\n",
      "Epoch 24 of 100 took 4.270s\n",
      "  training loss:\t\t1.944388\n",
      "  validation loss:\t\t1.684463\n",
      "  validation accuracy:\t\t62.00 %\n",
      "Epoch 25 of 100 took 4.471s\n",
      "  training loss:\t\t1.696766\n",
      "  validation loss:\t\t1.532427\n",
      "  validation accuracy:\t\t60.00 %\n",
      "Epoch 26 of 100 took 4.804s\n",
      "  training loss:\t\t1.425939\n",
      "  validation loss:\t\t1.217002\n",
      "  validation accuracy:\t\t69.00 %\n",
      "Epoch 27 of 100 took 4.613s\n",
      "  training loss:\t\t1.309493\n",
      "  validation loss:\t\t1.196059\n",
      "  validation accuracy:\t\t70.00 %\n",
      "Epoch 28 of 100 took 4.272s\n",
      "  training loss:\t\t1.037437\n",
      "  validation loss:\t\t0.977555\n",
      "  validation accuracy:\t\t71.00 %\n",
      "Epoch 29 of 100 took 4.275s\n",
      "  training loss:\t\t0.803960\n",
      "  validation loss:\t\t0.757116\n",
      "  validation accuracy:\t\t78.00 %\n",
      "Epoch 30 of 100 took 4.269s\n",
      "  training loss:\t\t0.882325\n",
      "  validation loss:\t\t0.756757\n",
      "  validation accuracy:\t\t79.00 %\n",
      "Epoch 31 of 100 took 4.229s\n",
      "  training loss:\t\t0.766812\n",
      "  validation loss:\t\t0.717067\n",
      "  validation accuracy:\t\t80.00 %\n",
      "Epoch 32 of 100 took 4.273s\n",
      "  training loss:\t\t0.439427\n",
      "  validation loss:\t\t0.709337\n",
      "  validation accuracy:\t\t75.00 %\n",
      "Epoch 33 of 100 took 4.271s\n",
      "  training loss:\t\t0.641647\n",
      "  validation loss:\t\t0.550490\n",
      "  validation accuracy:\t\t83.00 %\n",
      "Epoch 34 of 100 took 4.255s\n",
      "  training loss:\t\t0.552287\n",
      "  validation loss:\t\t0.617265\n",
      "  validation accuracy:\t\t80.00 %\n",
      "Epoch 35 of 100 took 4.277s\n",
      "  training loss:\t\t0.580104\n",
      "  validation loss:\t\t0.554974\n",
      "  validation accuracy:\t\t84.00 %\n",
      "Epoch 36 of 100 took 4.258s\n",
      "  training loss:\t\t0.425969\n",
      "  validation loss:\t\t0.587648\n",
      "  validation accuracy:\t\t84.00 %\n",
      "Epoch 37 of 100 took 4.269s\n",
      "  training loss:\t\t0.548458\n",
      "  validation loss:\t\t0.572581\n",
      "  validation accuracy:\t\t83.00 %\n",
      "Epoch 38 of 100 took 4.268s\n",
      "  training loss:\t\t0.448673\n",
      "  validation loss:\t\t0.496881\n",
      "  validation accuracy:\t\t81.00 %\n",
      "Epoch 39 of 100 took 4.565s\n",
      "  training loss:\t\t0.416294\n",
      "  validation loss:\t\t0.756943\n",
      "  validation accuracy:\t\t79.00 %\n",
      "Epoch 40 of 100 took 4.689s\n",
      "  training loss:\t\t0.361111\n",
      "  validation loss:\t\t0.540397\n",
      "  validation accuracy:\t\t84.00 %\n",
      "Epoch 41 of 100 took 4.754s\n",
      "  training loss:\t\t0.303926\n",
      "  validation loss:\t\t0.619301\n",
      "  validation accuracy:\t\t81.00 %\n",
      "Epoch 42 of 100 took 4.795s\n",
      "  training loss:\t\t0.439340\n",
      "  validation loss:\t\t0.558110\n",
      "  validation accuracy:\t\t86.00 %\n",
      "Epoch 43 of 100 took 4.475s\n",
      "  training loss:\t\t0.310041\n",
      "  validation loss:\t\t0.518717\n",
      "  validation accuracy:\t\t85.00 %\n",
      "Epoch 44 of 100 took 4.312s\n",
      "  training loss:\t\t0.226533\n",
      "  validation loss:\t\t0.412455\n",
      "  validation accuracy:\t\t87.00 %\n",
      "Epoch 45 of 100 took 4.349s\n",
      "  training loss:\t\t0.179341\n",
      "  validation loss:\t\t0.493323\n",
      "  validation accuracy:\t\t87.00 %\n",
      "Epoch 46 of 100 took 4.340s\n",
      "  training loss:\t\t0.306249\n",
      "  validation loss:\t\t0.530543\n",
      "  validation accuracy:\t\t85.00 %\n",
      "Epoch 47 of 100 took 4.324s\n",
      "  training loss:\t\t0.253440\n",
      "  validation loss:\t\t0.478068\n",
      "  validation accuracy:\t\t88.00 %\n",
      "Epoch 48 of 100 took 4.336s\n",
      "  training loss:\t\t0.210451\n",
      "  validation loss:\t\t0.480414\n",
      "  validation accuracy:\t\t86.00 %\n",
      "Epoch 49 of 100 took 4.491s\n",
      "  training loss:\t\t0.223456\n",
      "  validation loss:\t\t0.498337\n",
      "  validation accuracy:\t\t85.00 %\n",
      "Epoch 50 of 100 took 4.963s\n",
      "  training loss:\t\t0.192537\n",
      "  validation loss:\t\t0.516269\n",
      "  validation accuracy:\t\t89.00 %\n",
      "Epoch 51 of 100 took 4.791s\n",
      "  training loss:\t\t0.207454\n",
      "  validation loss:\t\t0.435553\n",
      "  validation accuracy:\t\t89.00 %\n",
      "Epoch 52 of 100 took 4.832s\n",
      "  training loss:\t\t0.276494\n",
      "  validation loss:\t\t0.406562\n",
      "  validation accuracy:\t\t87.00 %\n",
      "Epoch 53 of 100 took 4.575s\n",
      "  training loss:\t\t0.301388\n",
      "  validation loss:\t\t0.384212\n",
      "  validation accuracy:\t\t92.00 %\n",
      "Epoch 54 of 100 took 4.532s\n",
      "  training loss:\t\t0.245528\n",
      "  validation loss:\t\t0.465385\n",
      "  validation accuracy:\t\t89.00 %\n",
      "Epoch 55 of 100 took 4.866s\n",
      "  training loss:\t\t0.327787\n",
      "  validation loss:\t\t0.452307\n",
      "  validation accuracy:\t\t88.00 %\n",
      "Epoch 56 of 100 took 4.597s\n",
      "  training loss:\t\t0.131601\n",
      "  validation loss:\t\t0.430082\n",
      "  validation accuracy:\t\t88.00 %\n",
      "Epoch 57 of 100 took 4.560s\n",
      "  training loss:\t\t0.207806\n",
      "  validation loss:\t\t0.409128\n",
      "  validation accuracy:\t\t88.00 %\n",
      "Epoch 58 of 100 took 4.892s\n",
      "  training loss:\t\t0.198680\n",
      "  validation loss:\t\t0.462455\n",
      "  validation accuracy:\t\t87.00 %\n",
      "Epoch 59 of 100 took 4.815s\n",
      "  training loss:\t\t0.203264\n",
      "  validation loss:\t\t0.393605\n",
      "  validation accuracy:\t\t89.00 %\n",
      "Epoch 60 of 100 took 4.523s\n",
      "  training loss:\t\t0.209203\n",
      "  validation loss:\t\t0.469160\n",
      "  validation accuracy:\t\t88.00 %\n",
      "Epoch 61 of 100 took 4.363s\n",
      "  training loss:\t\t0.125102\n",
      "  validation loss:\t\t0.440717\n",
      "  validation accuracy:\t\t87.00 %\n",
      "Epoch 62 of 100 took 4.688s\n",
      "  training loss:\t\t0.133135\n",
      "  validation loss:\t\t0.696846\n",
      "  validation accuracy:\t\t84.00 %\n",
      "Epoch 63 of 100 took 4.409s\n",
      "  training loss:\t\t0.192402\n",
      "  validation loss:\t\t0.526055\n",
      "  validation accuracy:\t\t88.00 %\n",
      "Epoch 64 of 100 took 4.642s\n",
      "  training loss:\t\t0.126597\n",
      "  validation loss:\t\t0.362099\n",
      "  validation accuracy:\t\t88.00 %\n",
      "Epoch 65 of 100 took 4.673s\n",
      "  training loss:\t\t0.122431\n",
      "  validation loss:\t\t0.387167\n",
      "  validation accuracy:\t\t88.00 %\n",
      "Epoch 66 of 100 took 4.764s\n",
      "  training loss:\t\t0.084898\n",
      "  validation loss:\t\t0.429614\n",
      "  validation accuracy:\t\t88.00 %\n",
      "Epoch 67 of 100 took 4.852s\n",
      "  training loss:\t\t0.089336\n",
      "  validation loss:\t\t0.467646\n",
      "  validation accuracy:\t\t87.00 %\n",
      "Epoch 68 of 100 took 4.802s\n",
      "  training loss:\t\t0.110936\n",
      "  validation loss:\t\t0.465257\n",
      "  validation accuracy:\t\t86.00 %\n",
      "Epoch 69 of 100 took 4.880s\n",
      "  training loss:\t\t0.127098\n",
      "  validation loss:\t\t0.421448\n",
      "  validation accuracy:\t\t89.00 %\n",
      "Epoch 70 of 100 took 4.751s\n",
      "  training loss:\t\t0.120550\n",
      "  validation loss:\t\t0.380999\n",
      "  validation accuracy:\t\t88.00 %\n",
      "Epoch 71 of 100 took 4.891s\n",
      "  training loss:\t\t0.107396\n",
      "  validation loss:\t\t0.432842\n",
      "  validation accuracy:\t\t88.00 %\n",
      "Epoch 72 of 100 took 4.801s\n",
      "  training loss:\t\t0.121358\n",
      "  validation loss:\t\t0.386833\n",
      "  validation accuracy:\t\t91.00 %\n",
      "Epoch 73 of 100 took 4.853s\n",
      "  training loss:\t\t0.119595\n",
      "  validation loss:\t\t0.397856\n",
      "  validation accuracy:\t\t91.00 %\n",
      "Epoch 74 of 100 took 4.797s\n",
      "  training loss:\t\t0.077589\n",
      "  validation loss:\t\t0.357235\n",
      "  validation accuracy:\t\t90.00 %\n",
      "Epoch 75 of 100 took 4.835s\n",
      "  training loss:\t\t0.047637\n",
      "  validation loss:\t\t0.377338\n",
      "  validation accuracy:\t\t89.00 %\n",
      "Epoch 76 of 100 took 4.532s\n",
      "  training loss:\t\t0.050064\n",
      "  validation loss:\t\t0.372710\n",
      "  validation accuracy:\t\t90.00 %\n",
      "Epoch 77 of 100 took 4.743s\n",
      "  training loss:\t\t0.163816\n",
      "  validation loss:\t\t0.331150\n",
      "  validation accuracy:\t\t91.00 %\n",
      "Epoch 78 of 100 took 4.717s\n",
      "  training loss:\t\t0.097790\n",
      "  validation loss:\t\t0.353226\n",
      "  validation accuracy:\t\t91.00 %\n",
      "Epoch 79 of 100 took 4.627s\n",
      "  training loss:\t\t0.195441\n",
      "  validation loss:\t\t0.597856\n",
      "  validation accuracy:\t\t84.00 %\n",
      "Epoch 80 of 100 took 4.878s\n",
      "  training loss:\t\t0.087183\n",
      "  validation loss:\t\t0.507091\n",
      "  validation accuracy:\t\t88.00 %\n",
      "Epoch 81 of 100 took 4.777s\n",
      "  training loss:\t\t0.115755\n",
      "  validation loss:\t\t0.395191\n",
      "  validation accuracy:\t\t90.00 %\n",
      "Epoch 82 of 100 took 4.780s\n",
      "  training loss:\t\t0.095672\n",
      "  validation loss:\t\t0.398740\n",
      "  validation accuracy:\t\t92.00 %\n",
      "Epoch 83 of 100 took 4.853s\n",
      "  training loss:\t\t0.133013\n",
      "  validation loss:\t\t0.342108\n",
      "  validation accuracy:\t\t89.00 %\n",
      "Epoch 84 of 100 took 4.701s\n",
      "  training loss:\t\t0.113313\n",
      "  validation loss:\t\t0.445730\n",
      "  validation accuracy:\t\t89.00 %\n",
      "Epoch 85 of 100 took 4.860s\n",
      "  training loss:\t\t0.112590\n",
      "  validation loss:\t\t0.396763\n",
      "  validation accuracy:\t\t89.00 %\n",
      "Epoch 86 of 100 took 4.886s\n",
      "  training loss:\t\t0.075681\n",
      "  validation loss:\t\t0.416154\n",
      "  validation accuracy:\t\t89.00 %\n",
      "Epoch 87 of 100 took 4.300s\n",
      "  training loss:\t\t0.109752\n",
      "  validation loss:\t\t0.460249\n",
      "  validation accuracy:\t\t89.00 %\n",
      "Epoch 88 of 100 took 5.021s\n",
      "  training loss:\t\t0.128729\n",
      "  validation loss:\t\t0.576113\n",
      "  validation accuracy:\t\t88.00 %\n",
      "Epoch 89 of 100 took 5.155s\n",
      "  training loss:\t\t0.213554\n",
      "  validation loss:\t\t0.324095\n",
      "  validation accuracy:\t\t89.00 %\n",
      "Epoch 90 of 100 took 5.136s\n",
      "  training loss:\t\t0.106966\n",
      "  validation loss:\t\t0.323657\n",
      "  validation accuracy:\t\t89.00 %\n",
      "Epoch 91 of 100 took 5.117s\n",
      "  training loss:\t\t0.065203\n",
      "  validation loss:\t\t0.350304\n",
      "  validation accuracy:\t\t91.00 %\n",
      "Epoch 92 of 100 took 5.151s\n",
      "  training loss:\t\t0.054691\n",
      "  validation loss:\t\t0.361161\n",
      "  validation accuracy:\t\t91.00 %\n",
      "Epoch 93 of 100 took 4.957s\n",
      "  training loss:\t\t0.101208\n",
      "  validation loss:\t\t0.346509\n",
      "  validation accuracy:\t\t90.00 %\n",
      "Epoch 94 of 100 took 4.456s\n",
      "  training loss:\t\t0.075093\n",
      "  validation loss:\t\t0.371776\n",
      "  validation accuracy:\t\t89.00 %\n",
      "Epoch 95 of 100 took 5.035s\n",
      "  training loss:\t\t0.064359\n",
      "  validation loss:\t\t0.348829\n",
      "  validation accuracy:\t\t88.00 %\n",
      "Epoch 96 of 100 took 5.109s\n",
      "  training loss:\t\t0.081307\n",
      "  validation loss:\t\t0.243640\n",
      "  validation accuracy:\t\t91.00 %\n",
      "Epoch 97 of 100 took 5.136s\n",
      "  training loss:\t\t0.096665\n",
      "  validation loss:\t\t0.351910\n",
      "  validation accuracy:\t\t92.00 %\n",
      "Epoch 98 of 100 took 4.951s\n",
      "  training loss:\t\t0.099899\n",
      "  validation loss:\t\t0.336466\n",
      "  validation accuracy:\t\t88.00 %\n",
      "Epoch 99 of 100 took 4.662s\n",
      "  training loss:\t\t0.074292\n",
      "  validation loss:\t\t0.423613\n",
      "  validation accuracy:\t\t91.00 %\n",
      "Epoch 100 of 100 took 4.816s\n",
      "  training loss:\t\t0.038253\n",
      "  validation loss:\t\t0.326549\n",
      "  validation accuracy:\t\t90.00 %\n",
      "Final results:\n",
      "  test loss:\t\t\t0.506846\n",
      "  test accuracy:\t\t88.00 %\n"
     ]
    }
   ],
   "source": [
    " main(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}