CNTK_CIFAR_highAPI.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# High-level CNTK Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "import cntk\n",
    "from cntk.layers import Convolution2D, MaxPooling, Dense, Dropout\n",
    "from common.params import *\n",
    "from common.utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OS:  linux\n",
      "Python:  3.5.2 |Anaconda custom (64-bit)| (default, Jul  2 2016, 17:53:06) \n",
      "[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)]\n",
      "Numpy:  1.13.1\n",
      "CNTK:  2.1\n",
      "GPU:  ['Tesla K80', 'Tesla K80']\n"
     ]
    }
   ],
   "source": [
    "print(\"OS: \", sys.platform)\n",
    "print(\"Python: \", sys.version)\n",
    "print(\"Numpy: \", np.__version__)\n",
    "print(\"CNTK: \", cntk.__version__)\n",
    "print(\"GPU: \", get_gpu_name())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_symbol():\n",
    "    # Weight initialiser from uniform distribution\n",
    "    # Activation (unless states) is None\n",
    "    with cntk.layers.default_options(init = cntk.glorot_uniform(), activation = cntk.relu):\n",
    "        x = Convolution2D(filter_shape=(3, 3), num_filters=50, pad=True)(features)\n",
    "        x = Convolution2D(filter_shape=(3, 3), num_filters=50, pad=True)(x)\n",
    "        x = MaxPooling((2, 2), strides=(2, 2), pad=False)(x)\n",
    "        x = Dropout(0.25)(x)\n",
    "\n",
    "        x = Convolution2D(filter_shape=(3, 3), num_filters=100, pad=True)(x)\n",
    "        x = Convolution2D(filter_shape=(3, 3), num_filters=100, pad=True)(x)\n",
    "        x = MaxPooling((2, 2), strides=(2, 2), pad=False)(x)\n",
    "        x = Dropout(0.25)(x)    \n",
    "        \n",
    "        x = Dense(512)(x)\n",
    "        x = Dropout(0.5)(x)\n",
    "        x = Dense(N_CLASSES, activation=None)(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def init_model(m):\n",
    "    # Loss (dense labels); check if support for sparse labels\n",
    "    loss = cntk.cross_entropy_with_softmax(m, labels)\n",
    "    # Momentum SGD\n",
    "    # https://github.com/Microsoft/CNTK/blob/master/Manual/Manual_How_to_use_learners.ipynb\n",
    "    # unit_gain=False: momentum_direction = momentum*old_momentum_direction + gradient\n",
    "    # if unit_gain=True then ...(1-momentum)*gradient\n",
    "    learner = cntk.momentum_sgd(m.parameters, \n",
    "                                lr=cntk.learning_rate_schedule(LR, cntk.UnitType.minibatch) , \n",
    "                                momentum=cntk.momentum_schedule(MOMENTUM),\n",
    "                                unit_gain=False)\n",
    "    return loss, learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz\n",
      "Done.\n",
      "Extracting files...\n",
      "Done.\n",
      "Preparing train set...\n",
      "Preparing test set...\n",
      "Done.\n",
      "(50000, 3, 32, 32) (10000, 3, 32, 32) (50000, 10) (10000, 10)\n",
      "float32 float32 float32 float32\n",
      "CPU times: user 2.8 s, sys: 1.55 s, total: 4.34 s\n",
      "Wall time: 21.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Data into format for library\n",
    "x_train, x_test, y_train, y_test = cifar_for_library(channel_first=True, one_hot=True)\n",
    "# CNTK format\n",
    "y_train = y_train.astype(np.float32)\n",
    "y_test = y_test.astype(np.float32)\n",
    "print(x_train.shape, x_test.shape, y_train.shape, y_test.shape)\n",
    "print(x_train.dtype, x_test.dtype, y_train.dtype, y_test.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 36 ms, sys: 516 ms, total: 552 ms\n",
      "Wall time: 587 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Placeholders\n",
    "features = cntk.input_variable((3, 32, 32), np.float32)\n",
    "labels = cntk.input_variable(N_CLASSES, np.float32)\n",
    "# Load symbol\n",
    "sym = create_symbol()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 52 ms, sys: 368 ms, total: 420 ms\n",
      "Wall time: 439 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "loss, learner = init_model(sym)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 11s, sys: 28.8 s, total: 2min 40s\n",
      "Wall time: 2min 43s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'epoch_summaries': [{'loss': 1.8209584375, 'metric': 0.0, 'samples': 50000},\n",
       "  {'loss': 1.3497275, 'metric': 0.0, 'samples': 50000},\n",
       "  {'loss': 1.112234453125, 'metric': 0.0, 'samples': 50000},\n",
       "  {'loss': 0.964451953125, 'metric': 0.0, 'samples': 50000},\n",
       "  {'loss': 0.858155625, 'metric': 0.0, 'samples': 50000},\n",
       "  {'loss': 0.773041875, 'metric': 0.0, 'samples': 50000},\n",
       "  {'loss': 0.713242734375, 'metric': 0.0, 'samples': 50000},\n",
       "  {'loss': 0.64646359375, 'metric': 0.0, 'samples': 50000},\n",
       "  {'loss': 0.5934486328125, 'metric': 0.0, 'samples': 50000},\n",
       "  {'loss': 0.558412578125, 'metric': 0.0, 'samples': 50000}],\n",
       " 'updates': [{'loss': 1.8209670844670294, 'metric': 0.0, 'samples': 49984},\n",
       "  {'loss': 1.349637821602913, 'metric': 0.0, 'samples': 49984},\n",
       "  {'loss': 1.112269207396367, 'metric': 0.0, 'samples': 49984},\n",
       "  {'loss': 0.9643642997009043, 'metric': 0.0, 'samples': 49984},\n",
       "  {'loss': 0.8580787258172615, 'metric': 0.0, 'samples': 49984},\n",
       "  {'loss': 0.7730779318131802, 'metric': 0.0, 'samples': 49984},\n",
       "  {'loss': 0.7133127913432298, 'metric': 0.0, 'samples': 49984},\n",
       "  {'loss': 0.6465474420564581, 'metric': 0.0, 'samples': 49984},\n",
       "  {'loss': 0.5934441599436819, 'metric': 0.0, 'samples': 49984},\n",
       "  {'loss': 0.5584908655144646, 'metric': 0.0, 'samples': 49984}]}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "loss.train((x_train, y_train), \n",
    "           minibatch_size=BATCHSIZE, \n",
    "           max_epochs=EPOCHS,\n",
    "           parameter_learners=[learner])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 928 ms, sys: 300 ms, total: 1.23 s\n",
      "Wall time: 1.51 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Predict and then score accuracy\n",
    "# Apply softmax since that is only applied at training\n",
    "# with cross-entropy loss\n",
    "z = cntk.softmax(sym)\n",
    "n_samples = (y_test.shape[0]//BATCHSIZE)*BATCHSIZE\n",
    "y_guess = np.zeros(n_samples, dtype=np.int)\n",
    "y_truth = np.argmax(y_test[:n_samples], axis=-1)\n",
    "c = 0\n",
    "for data, label in yield_mb(x_test, y_test, BATCHSIZE):\n",
    "    predicted_label_probs = z.eval({features : data})\n",
    "    y_guess[c*BATCHSIZE:(c+1)*BATCHSIZE] = np.argmax(predicted_label_probs, axis=-1)\n",
    "    c += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.768329326923\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy: \", sum(y_guess == y_truth)/len(y_guess))"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}