multi_framework/ConvNet_Comparisons.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convolutional Neural Networks on MNIST\n",
    "\n",
    "MNIST:\n",
    "training set of 60,000 examples, and a test set of 10,000 examples - images of handwritten digits (0-9) that are 28x28 pixels (grayscale).\n",
    "\n",
    "The CPU time is only an estimate and highly dependent upon the way in which the training is being evaluated as moving through epochs.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/micheleenharris/anaconda36/bin/python'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Current kernel\n",
    "import sys\n",
    "sys.executable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Keras ConvNet Example\n",
    "\n",
    "Adapted from:  https://github.com/keras-team/keras/blob/master/examples/mnist_cnn.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (60000, 28, 28, 1)\n",
      "60000 train samples\n",
      "10000 test samples\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "60000/60000 [==============================] - 31s 523us/step - loss: 0.2335 - acc: 0.9350 - val_loss: 0.0692 - val_acc: 0.9773\n",
      "Epoch 2/10\n",
      "60000/60000 [==============================] - 31s 519us/step - loss: 0.0573 - acc: 0.9829 - val_loss: 0.0466 - val_acc: 0.9852\n",
      "Epoch 3/10\n",
      "60000/60000 [==============================] - 31s 524us/step - loss: 0.0428 - acc: 0.9868 - val_loss: 0.0451 - val_acc: 0.9855\n",
      "Epoch 4/10\n",
      "60000/60000 [==============================] - 32s 529us/step - loss: 0.0346 - acc: 0.9892 - val_loss: 0.0382 - val_acc: 0.9868\n",
      "Epoch 5/10\n",
      "60000/60000 [==============================] - 32s 535us/step - loss: 0.0293 - acc: 0.9911 - val_loss: 0.0302 - val_acc: 0.9901\n",
      "Epoch 6/10\n",
      "60000/60000 [==============================] - 33s 546us/step - loss: 0.0250 - acc: 0.9921 - val_loss: 0.0340 - val_acc: 0.9887\n",
      "Epoch 7/10\n",
      "60000/60000 [==============================] - 33s 545us/step - loss: 0.0214 - acc: 0.9934 - val_loss: 0.0337 - val_acc: 0.9887\n",
      "Epoch 8/10\n",
      "60000/60000 [==============================] - 33s 545us/step - loss: 0.0181 - acc: 0.9943 - val_loss: 0.0302 - val_acc: 0.9901\n",
      "Epoch 9/10\n",
      "60000/60000 [==============================] - 33s 546us/step - loss: 0.0161 - acc: 0.9949 - val_loss: 0.0376 - val_acc: 0.9878\n",
      "Epoch 10/10\n",
      "60000/60000 [==============================] - 34s 565us/step - loss: 0.0151 - acc: 0.9947 - val_loss: 0.0307 - val_acc: 0.9900\n",
      "CPU times: user 26min 5s, sys: 2min 14s, total: 28min 19s\n",
      "Wall time: 5min 24s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras import backend as K\n",
    "from keras.optimizers import SGD\n",
    "\n",
    "# For training\n",
    "batch_size = 128\n",
    "num_classes = 10\n",
    "num_epochs = 10\n",
    "num_samples = 60000\n",
    "\n",
    "# input image dimensions\n",
    "img_rows, img_cols = 28, 28\n",
    "\n",
    "# the data, split between train and test sets\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)\n",
    "    x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)\n",
    "    x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)\n",
    "\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "# convert class vectors to binary class matrices\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Conv2D(32, kernel_size=(3, 3),\n",
    "                 strides=(1, 1),\n",
    "                 padding='same',\n",
    "                 activation='relu',\n",
    "                 input_shape=input_shape))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Conv2D(64, kernel_size=(3, 3),\n",
    "                 strides=(1, 1),\n",
    "                 padding='same',\n",
    "                 activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "# model.add(Dropout(0.25))\n",
    "model.add(Flatten())\n",
    "# model.add(Dense(128, activation='relu'))\n",
    "# model.add(Dropout(0.5))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "sgd = SGD(lr=0.05, decay=1e-6, momentum=0.9, nesterov=True)\n",
    "model.compile(loss='categorical_crossentropy', \n",
    "              optimizer=sgd,\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "#           steps_per_epoch=int(num_samples/batch_size),\n",
    "          epochs=num_epochs,\n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test))\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 99.14%\n"
     ]
    }
   ],
   "source": [
    "keras_score = score[1]\n",
    "print('Test accuracy: {:.2%}'.format(keras_score))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch ConvNet Example\n",
    "\n",
    "Apated from:  https://github.com/rasbt/deep-learning-book/blob/master/code/model_zoo/pytorch_ipynb/convnet.ipynb by Sebastian Raschka"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image batch dimensions: torch.Size([128, 1, 28, 28])\n",
      "Image label dimensions: torch.Size([128])\n",
      "Epoch: 001/010 | Batch 000/468 | Cost: 2.3102\n",
      "Epoch: 001/010 | Batch 468/468 | Cost: 0.1679\n",
      "Epoch: 001/010 training accuracy: 95.56%\n",
      "Epoch: 002/010 | Batch 000/468 | Cost: 0.1277\n",
      "Epoch: 002/010 | Batch 468/468 | Cost: 0.0510\n",
      "Epoch: 002/010 training accuracy: 97.10%\n",
      "Epoch: 003/010 | Batch 000/468 | Cost: 0.0759\n",
      "Epoch: 003/010 | Batch 468/468 | Cost: 0.1045\n",
      "Epoch: 003/010 training accuracy: 97.74%\n",
      "Epoch: 004/010 | Batch 000/468 | Cost: 0.1485\n",
      "Epoch: 004/010 | Batch 468/468 | Cost: 0.0432\n",
      "Epoch: 004/010 training accuracy: 98.03%\n",
      "Epoch: 005/010 | Batch 000/468 | Cost: 0.0380\n",
      "Epoch: 005/010 | Batch 468/468 | Cost: 0.0330\n",
      "Epoch: 005/010 training accuracy: 98.03%\n",
      "Epoch: 006/010 | Batch 000/468 | Cost: 0.0979\n",
      "Epoch: 006/010 | Batch 468/468 | Cost: 0.0725\n",
      "Epoch: 006/010 training accuracy: 98.24%\n",
      "Epoch: 007/010 | Batch 000/468 | Cost: 0.0755\n",
      "Epoch: 007/010 | Batch 468/468 | Cost: 0.0556\n",
      "Epoch: 007/010 training accuracy: 98.60%\n",
      "Epoch: 008/010 | Batch 000/468 | Cost: 0.0739\n",
      "Epoch: 008/010 | Batch 468/468 | Cost: 0.0373\n",
      "Epoch: 008/010 training accuracy: 98.60%\n",
      "Epoch: 009/010 | Batch 000/468 | Cost: 0.0414\n",
      "Epoch: 009/010 | Batch 468/468 | Cost: 0.0950\n",
      "Epoch: 009/010 training accuracy: 98.65%\n",
      "Epoch: 010/010 | Batch 000/468 | Cost: 0.0126\n",
      "Epoch: 010/010 | Batch 468/468 | Cost: 0.0402\n",
      "Epoch: 010/010 training accuracy: 98.31%\n",
      "CPU times: user 1h 12min 19s, sys: 2min 5s, total: 1h 14min 25s\n",
      "Wall time: 12min 17s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torchvision import datasets\n",
    "from torchvision import transforms\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "##########################\n",
    "### SETTINGS\n",
    "##########################\n",
    "\n",
    "# Device\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# Hyperparameters\n",
    "random_seed = 1\n",
    "learning_rate = 0.05\n",
    "num_samples = 60000\n",
    "num_epochs = 10\n",
    "num_steps = num_samples//batch_size\n",
    "batch_size = 128\n",
    "\n",
    "# Architecture\n",
    "num_classes = 10\n",
    "\n",
    "\n",
    "##########################\n",
    "### MNIST DATASET\n",
    "##########################\n",
    "\n",
    "# Note transforms.ToTensor() scales input images\n",
    "# to 0-1 range\n",
    "train_dataset = datasets.MNIST(root='data', \n",
    "                               train=True, \n",
    "                               transform=transforms.ToTensor(),\n",
    "                               download=True)\n",
    "\n",
    "test_dataset = datasets.MNIST(root='data', \n",
    "                              train=False, \n",
    "                              transform=transforms.ToTensor())\n",
    "\n",
    "\n",
    "train_loader = DataLoader(dataset=train_dataset, \n",
    "                          batch_size=batch_size, \n",
    "                          shuffle=True)\n",
    "\n",
    "test_loader = DataLoader(dataset=test_dataset, \n",
    "                         batch_size=batch_size, \n",
    "                         shuffle=False)\n",
    "\n",
    "# Checking the dataset\n",
    "for images, labels in train_loader:  \n",
    "    print('Image batch dimensions:', images.shape)\n",
    "    print('Image label dimensions:', labels.shape)\n",
    "    break\n",
    "\n",
    "##########################\n",
    "### MODEL\n",
    "##########################\n",
    "\n",
    "    \n",
    "class ConvNetPyTorch(nn.Module):\n",
    "    \"\"\"Adapted from:\n",
    "    https://github.com/rasbt/deep-learning-book/blob/master/code/model_zoo/pytorch_ipynb/convnet.ipynb\n",
    "    \"\"\"\n",
    "    def __init__(self, num_classes=10):\n",
    "        super(ConvNetPyTorch, self).__init__()\n",
    "        self.layer1 = nn.Sequential(\n",
    "            # 28x28x1 => 28x28x32\n",
    "            nn.Conv2d(in_channels=1,\n",
    "                      out_channels=32,\n",
    "                      kernel_size=(3, 3),\n",
    "                      stride=(1, 1),\n",
    "                      padding=1), # (1(28-1) - 28 + 3) / 2 = 1\n",
    "            nn.ReLU(),\n",
    "            # 28x28x32 => 14x14x32\n",
    "            nn.MaxPool2d(kernel_size=(2, 2),\n",
    "                         stride=(2, 2),\n",
    "                         padding=0)) # (2(14-1) - 28 + 2) = 0    \n",
    "        self.layer2 = nn.Sequential(\n",
    "            # 14x14x32 => 14x14x64\n",
    "            nn.Conv2d(in_channels=32,\n",
    "                      out_channels=64,\n",
    "                      kernel_size=(3, 3),\n",
    "                      stride=(1, 1),\n",
    "                      padding=1), # (1(14-1) - 14 + 3) / 2 = 1   \n",
    "            nn.ReLU(),\n",
    "            # 14x14x64 => 7x7x64\n",
    "            nn.MaxPool2d(kernel_size=(2, 2),\n",
    "                         stride=(2, 2),\n",
    "                         padding=0)) # (2(7-1) - 14 + 2) = 0\n",
    "        self.linear_1 = nn.Linear(7*7*64, num_classes)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        out = self.layer1(x)\n",
    "        out = self.layer2(out)\n",
    "        out = out.reshape(out.size(0), -1)\n",
    "        logits = self.linear_1(out.view(-1, 7*7*64))\n",
    "        probas = F.softmax(logits, dim=1)\n",
    "        return logits, probas\n",
    "\n",
    "torch.manual_seed(random_seed)\n",
    "model = ConvNetPyTorch(num_classes=num_classes)\n",
    "model = model.to(device)\n",
    "    \n",
    "\n",
    "##########################\n",
    "### COST AND OPTIMIZER\n",
    "##########################\n",
    "\n",
    "cost_fn = torch.nn.CrossEntropyLoss()  \n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n",
    "\n",
    "def compute_accuracy(model, data_loader):\n",
    "    correct_pred, num_examples = 0, 0\n",
    "    for features, targets in data_loader:\n",
    "        features = features.to(device)\n",
    "        targets = targets.to(device)\n",
    "        logits, probas = model(features)\n",
    "        _, predicted_labels = torch.max(probas.data, 1)\n",
    "        num_examples += targets.size(0)\n",
    "        correct_pred += (predicted_labels == targets).sum()\n",
    "    return correct_pred.float()/num_examples\n",
    "    \n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    model = model.train()\n",
    "    for batch_idx, (features, targets) in enumerate(train_loader):\n",
    "        \n",
    "        features = features.to(device)\n",
    "        targets = targets.to(device)\n",
    "\n",
    "        ### FORWARD AND BACK PROP\n",
    "        logits, probas = model(features)\n",
    "        cost = cost_fn(logits, targets)\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        cost.backward()\n",
    "        \n",
    "        ### UPDATE MODEL PARAMETERS\n",
    "        optimizer.step()\n",
    "        \n",
    "        ### LOGGING\n",
    "        if not batch_idx % num_steps:\n",
    "            print ('Epoch: %03d/%03d | Batch %03d/%03d | Cost: %.4f' \n",
    "                   %(epoch+1, num_epochs, batch_idx, \n",
    "                     num_steps, cost.data))\n",
    "    \n",
    "    model = model.eval()\n",
    "    print('Epoch: %03d/%03d training accuracy: %.2f%%' % (\n",
    "          epoch+1, num_epochs, \n",
    "          compute_accuracy(model, train_loader)))\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 98.28%\n"
     ]
    }
   ],
   "source": [
    "pytorch_score = (compute_accuracy(model, test_loader))\n",
    "print('Test accuracy: {:.2%}'.format(pytorch_score))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TensorFlow ConvNet Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n",
      "Epoch = 1/10 | Loss = 0.1832001656293869 | Accuracy = 0.9459999799728394\n",
      "Epoch = 2/10 | Loss = 0.10717695951461792 | Accuracy = 0.9688000082969666\n",
      "Epoch = 3/10 | Loss = 0.08500224351882935 | Accuracy = 0.9753999710083008\n",
      "Epoch = 4/10 | Loss = 0.07122006267309189 | Accuracy = 0.9789999723434448\n",
      "Epoch = 5/10 | Loss = 0.06446825712919235 | Accuracy = 0.9802908897399902\n",
      "Epoch = 6/10 | Loss = 0.0528169721364975 | Accuracy = 0.9838545322418213\n",
      "Epoch = 7/10 | Loss = 0.06288313865661621 | Accuracy = 0.9801999926567078\n",
      "Epoch = 8/10 | Loss = 0.04890061169862747 | Accuracy = 0.9851999878883362\n",
      "Epoch = 9/10 | Loss = 0.044429562985897064 | Accuracy = 0.9868909120559692\n",
      "Epoch = 10/10 | Loss = 0.041722942143678665 | Accuracy = 0.9875817894935608\n",
      "CPU times: user 23min 8s, sys: 1min 28s, total: 24min 37s\n",
      "Wall time: 5min 10s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\"\"\" Convolutional Neural Network.\n",
    "Build and train a convolutional neural network with TensorFlow.\n",
    "This example is using the MNIST database of handwritten digits\n",
    "(http://yann.lecun.com/exdb/mnist/)\n",
    "This example is using TensorFlow layers API, see 'convolutional_network_raw' \n",
    "example for a raw implementation with variables.\n",
    "Author: Aymeric Damien\n",
    "Project: https://github.com/aymericdamien/TensorFlow-Examples/\n",
    "\"\"\"\n",
    "from __future__ import division, print_function, absolute_import\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import tensorflow as tf\n",
    "import os\n",
    "\n",
    "tf.logging.set_verbosity(tf.logging.ERROR)\n",
    "\n",
    "# Import MNIST data\n",
    "DATA_DIR = os.path.join(os.sep + \"tmp\", \"data\")\n",
    "mnist = input_data.read_data_sets(DATA_DIR, one_hot=False)\n",
    "\n",
    "# Training Parameters\n",
    "num_samples = 60000\n",
    "batch_size = 128\n",
    "num_steps = int(num_samples/batch_size)\n",
    "learning_rate = 0.05\n",
    "num_epochs = 10\n",
    "\n",
    "# Network Parameters\n",
    "num_input = 784 # MNIST data input (img shape: 28*28)\n",
    "num_classes = 10 # MNIST total classes (0-9 digits)\n",
    "\n",
    "# Create the neural network\n",
    "def convNetTensorFlow(x_dict, n_classes, reuse, is_training):\n",
    "    \"\"\"Adapted from:\n",
    "    https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/3_NeuralNetworks/convolutional_network.py\n",
    "    \"\"\"\n",
    "    # Define a scope for reusing the variables\n",
    "    with tf.variable_scope('ConvNet', reuse=reuse):\n",
    "        # TF Estimator input is a dict, in case of multiple inputs\n",
    "        x = x_dict['images']\n",
    "\n",
    "        # MNIST data input is a 1-D vector of 784 features (28*28 pixels)\n",
    "        # Reshape to match picture format [Height x Width x Channel]\n",
    "        # Tensor input become 4-D: [Batch Size, Height, Width, Channel]\n",
    "        x = tf.reshape(x, shape=[-1, 28, 28, 1])\n",
    "\n",
    "        # Convolution Layer with 32 filters and a kernel size of 5\n",
    "        conv1 = tf.layers.conv2d(x, 32, 5, activation=tf.nn.relu)\n",
    "        # Max Pooling (down-sampling) with strides of 2 and kernel size of 2\n",
    "        conv1 = tf.layers.max_pooling2d(conv1, 2, 2)\n",
    "\n",
    "        # Convolution Layer with 64 filters and a kernel size of 3\n",
    "        conv2 = tf.layers.conv2d(conv1, 64, 3, activation=tf.nn.relu)\n",
    "        # Max Pooling (down-sampling) with strides of 2 and kernel size of 2\n",
    "        conv2 = tf.layers.max_pooling2d(conv2, 2, 2)\n",
    "\n",
    "        # Flatten the data to a 1-D vector for the fully connected layer\n",
    "        fc1 = tf.contrib.layers.flatten(conv2)\n",
    "\n",
    "        # Output layer, class prediction\n",
    "        logits = tf.layers.dense(fc1, n_classes, activation=None)\n",
    "        \n",
    "    return logits\n",
    "\n",
    "# Define the model function (following TF Estimator Template)\n",
    "def model_fn(features, labels, mode):\n",
    "    # Build the neural network\n",
    "    # Because Dropout have different behavior at training and prediction time, we\n",
    "    # need to create 2 distinct computation graphs that still share the same weights.\n",
    "    logits_train = convNetTensorFlow(features, num_classes, reuse=False,\n",
    "                            is_training=True)\n",
    "    logits_test = convNetTensorFlow(features, num_classes, reuse=True,\n",
    "                           is_training=False)\n",
    "\n",
    "    # Predictions\n",
    "    pred_classes = tf.argmax(logits_test, axis=1)\n",
    "    pred_probas = tf.nn.softmax(logits_test)\n",
    "\n",
    "    # If prediction mode, early return\n",
    "    if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "        return tf.estimator.EstimatorSpec(mode, predictions=pred_classes)\n",
    "\n",
    "        # Define loss and optimizer\n",
    "    loss_op = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "        logits=logits_train, labels=tf.cast(labels, dtype=tf.int32)))\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n",
    "    train_op = optimizer.minimize(loss_op,\n",
    "                                  global_step=tf.train.get_global_step())\n",
    "\n",
    "    # Evaluate the accuracy of the model\n",
    "    acc_op = tf.metrics.accuracy(labels=labels, predictions=pred_classes)\n",
    "    \n",
    "    # TF Estimators requires to return a EstimatorSpec, that specify\n",
    "    # the different ops for training, evaluating, ...\n",
    "    estim_specs = tf.estimator.EstimatorSpec(\n",
    "        mode=mode,\n",
    "        predictions=pred_classes,\n",
    "        loss=loss_op,\n",
    "        train_op=train_op,\n",
    "        eval_metric_ops={'accuracy': acc_op})\n",
    "\n",
    "    return estim_specs\n",
    "\n",
    "# Build the Estimator\n",
    "model = tf.estimator.Estimator(model_fn)\n",
    "\n",
    "for epoch_idx in range(num_epochs):\n",
    "    print('Epoch = {}/{} | '.format(epoch_idx+1, num_epochs), end='')\n",
    "    # Define the input function for training\n",
    "    input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "        x={'images': mnist.train.images}, y=mnist.train.labels,\n",
    "        batch_size=batch_size, num_epochs=None, shuffle=True)\n",
    "    # Train the Model\n",
    "    model.train(input_fn, steps=num_steps)\n",
    "\n",
    "    # Evaluate the Model on training data\n",
    "    input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "        x={'images': mnist.train.images}, y=mnist.train.labels,\n",
    "        batch_size=batch_size, num_epochs=1, shuffle=True)\n",
    "    e = model.evaluate(input_fn)\n",
    "    print('Loss = {} | Accuracy = {}'.format(e['loss'], e['accuracy']))\n",
    "\n",
    "# Evaluate the Model on test data\n",
    "# Define the input function for evaluating\n",
    "input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x={'images': mnist.test.images}, y=mnist.test.labels,\n",
    "    batch_size=batch_size, shuffle=False)\n",
    "# Use the Estimator 'evaluate' method\n",
    "e = model.evaluate(input_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 98.62%\n"
     ]
    }
   ],
   "source": [
    "print('Test accuracy: {:.2%}'.format(e['accuracy']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNTK ConvNet Example\n",
    "\n",
    "Adapted from:\n",
    "https://cntk.ai/pythondocs/CNTK_103A_MNIST_DataLoader.html and https://cntk.ai/pythondocs/CNTK_103D_MNIST_ConvolutionalNeuralNetwork.html\n",
    "\n",
    "NOT RUN, ONLY FOR REFERENCE - there is _no distribution_ that natively works on macOS."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function # Use a function definition from future version (say 3.x from 2.7 interpreter)\n",
    "import matplotlib.image as mpimg\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "\n",
    "import cntk as C\n",
    "import cntk.tests.test_utils\n",
    "cntk.tests.test_utils.set_device_from_pytest_env() # (only needed for our build system)\n",
    "C.cntk_py.set_fixed_random_seed(1) # fix a random seed for CNTK components\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "# From TF example above (this is location the \n",
    "# MNIST .gz files were downloaded to)\n",
    "DATA_DIR = os.path.join(os.sep + \"tmp\", \"data\")\n",
    "\n",
    "# Import MNIST data using TensorFlow downloader (if files exist, does nothing)\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(DATA_DIR, one_hot=False)\n",
    "\n",
    "# Save the data files into a format compatible with CNTK text reader\n",
    "def savetxt(filename, ndarray):\n",
    "    dir = os.path.dirname(filename)\n",
    "\n",
    "    if not os.path.exists(dir):\n",
    "        os.makedirs(dir)\n",
    "\n",
    "    if not os.path.isfile(filename):\n",
    "        print(\"Saving\", filename )\n",
    "        with open(filename, 'w') as f:\n",
    "            labels = list(map(' '.join, np.eye(10, dtype=np.uint).astype(str)))\n",
    "            for row in ndarray:\n",
    "                row_str = row.astype(str)\n",
    "                label_str = labels[row[-1]]\n",
    "                feature_str = ' '.join(row_str[:-1])\n",
    "                f.write('|labels {} |features {}\\n'.format(label_str, feature_str))\n",
    "    else:\n",
    "        print(\"File already exists\", filename)\n",
    "\n",
    "print ('Writing train text file...')\n",
    "savetxt(os.path.join(DATA_DIR, \"Train-28x28_cntk_text.txt\"), train)\n",
    "\n",
    "print ('Writing test text file...')\n",
    "savetxt(os.path.join(DATA_DIR, \"Test-28x28_cntk_text.txt\"), test)\n",
    "\n",
    "print('Done')\n",
    "        \n",
    "# Define the data dimensions\n",
    "input_dim_model = (1, 28, 28)    # images are 28 x 28 with 1 channel of color (gray)\n",
    "input_dim = 28*28                # used by readers to treat input data as a vector\n",
    "num_output_classes = 10\n",
    "\n",
    "# Train parameters\n",
    "learning_rate = 0.05\n",
    "train_minibatch_size = 128\n",
    "num_samples_per_sweep = 60000\n",
    "num_minibatches_to_train = (num_samples_per_sweep * num_sweeps_to_train_with) / train_minibatch_size\n",
    "\n",
    "# Test parameters\n",
    "test_minibatch_size = 128\n",
    "num_samples = 10000\n",
    "num_minibatches_to_test = num_samples // test_minibatch_size\n",
    "\n",
    "# Read a CTF formatted text (as mentioned above) using the CTF deserializer from a file\n",
    "def create_reader(path, is_training, input_dim, num_label_classes):\n",
    "\n",
    "    ctf = C.io.CTFDeserializer(path, C.io.StreamDefs(\n",
    "          labels=C.io.StreamDef(field='labels', shape=num_label_classes, is_sparse=False),\n",
    "          features=C.io.StreamDef(field='features', shape=input_dim, is_sparse=False)))\n",
    "\n",
    "    return C.io.MinibatchSource(ctf,\n",
    "        randomize = is_training, max_sweeps = C.io.INFINITELY_REPEAT if is_training else 1)\n",
    "\n",
    "def create_criterion_function(model, labels):\n",
    "    loss = C.cross_entropy_with_softmax(model, labels)\n",
    "    errs = C.classification_error(model, labels)\n",
    "    return loss, errs # (model, labels) -> (loss, error metric)\n",
    "\n",
    "# Define a utility function to compute the moving average sum.\n",
    "# A more efficient implementation is possible with np.cumsum() function\n",
    "def moving_average(a, w=5):\n",
    "    if len(a) < w:\n",
    "        return a[:]    # Need to send a copy of the array\n",
    "    return [val if idx < w else sum(a[(idx-w):idx])/w for idx, val in enumerate(a)]\n",
    "\n",
    "\n",
    "# Defines a utility that prints the training progress\n",
    "def print_training_progress(trainer, mb, frequency, verbose=1):\n",
    "    training_loss = \"NA\"\n",
    "    eval_error = \"NA\"\n",
    "\n",
    "    if mb%frequency == 0:\n",
    "        training_loss = trainer.previous_minibatch_loss_average\n",
    "        eval_error = trainer.previous_minibatch_evaluation_average\n",
    "        if verbose:\n",
    "            print (\"Minibatch: {0}, Loss: {1:.4f}, Error: {2:.2f}%\".format(mb, \n",
    "                                                                           training_loss, \n",
    "                                                                           eval_error*100))\n",
    "\n",
    "    return mb, training_loss, eval_error\n",
    "\n",
    "def convNetCNTK(features, num_output_classes):\n",
    "    \"\"\"https://cntk.ai/pythondocs/CNTK_103D_MNIST_ConvolutionalNeuralNetwork.html\"\"\"\n",
    "    with C.layers.default_options(init=C.glorot_uniform(), activation=C.relu):\n",
    "        model = C.layers.Sequential([\n",
    "            C.layers.For(range(2), lambda i: [\n",
    "                C.layers.Convolution((3,3), [32,64][i], pad=True),\n",
    "                C.layers.MaxPooling((2,2), strides=(2,2))\n",
    "                ]),\n",
    "            C.layers.Dense(64),\n",
    "            C.layers.Dense(out_dims, activation=None)\n",
    "        ])\n",
    "\n",
    "    return model(features)\n",
    "\n",
    "def train_test(train_reader, test_reader, model_func, num_sweeps_to_train_with=10):\n",
    "\n",
    "    # Instantiate the model function; x is the input (feature) variable\n",
    "    # We will scale the input image pixels within 0-1 range by dividing all input value by 255.\n",
    "    model = convNetCNTK(x/255, num_output_classes)\n",
    "\n",
    "    # Instantiate the loss and error function\n",
    "    loss, label_error = create_criterion_function(model, y)\n",
    "\n",
    "    # Instantiate the trainer object to drive the model training\n",
    "    lr_schedule = C.learning_parameter_schedule(learning_rate)\n",
    "    learner = C.sgd(z.parameters, lr_schedule)\n",
    "    trainer = C.Trainer(z, (loss, label_error), [learner])\n",
    "\n",
    "    # Map the data streams to the input and labels.\n",
    "    input_map={\n",
    "        y  : train_reader.streams.labels,\n",
    "        x  : train_reader.streams.features\n",
    "    }\n",
    "\n",
    "    # Uncomment below for more detailed logging\n",
    "    training_progress_output_freq = 500\n",
    "\n",
    "    # Start a timer\n",
    "    start = time.time()\n",
    "\n",
    "    for i in range(0, int(num_minibatches_to_train)):\n",
    "        # Read a mini batch from the training data file\n",
    "        data=train_reader.next_minibatch(train_minibatch_size, input_map=input_map)\n",
    "        trainer.train_minibatch(data)\n",
    "        print_training_progress(trainer, i, training_progress_output_freq, verbose=1)\n",
    "\n",
    "    # Print training time\n",
    "    print(\"Training took {:.1f} sec\".format(time.time() - start))\n",
    "\n",
    "    # Test the model\n",
    "    test_input_map = {\n",
    "        y  : test_reader.streams.labels,\n",
    "        x  : test_reader.streams.features\n",
    "    }\n",
    "\n",
    "    test_result = 0.0\n",
    "\n",
    "    for i in range(num_minibatches_to_test):\n",
    "\n",
    "        # We are loading test data in batches specified by test_minibatch_size\n",
    "        # Each data point in the minibatch is a MNIST digit image of 784 dimensions\n",
    "        # with one pixel per dimension that we will encode / decode with the\n",
    "        # trained model.\n",
    "        data = test_reader.next_minibatch(test_minibatch_size, input_map=test_input_map)\n",
    "        eval_error = trainer.test_minibatch(data)\n",
    "        test_result = test_result + eval_error\n",
    "\n",
    "    # Average of evaluation errors of all test minibatches\n",
    "    print(\"Average test error: {0:.2f}%\".format(test_result*100 / num_minibatches_to_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "nav_menu": {},
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}