notebooks/samples/DeepLearning - Flower Image Classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml import Transformer, Estimator, Pipeline\n",
    "from pyspark.ml.classification import LogisticRegression\n",
    "from mmlspark.downloader import ModelDownloader\n",
    "import os, sys, time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "mml-deploy": "local"
   },
   "outputs": [],
   "source": [
    "model = ModelDownloader(spark, \"dbfs:/models/\").downloadByName(\"ResNet50\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the images\n",
    "# use flowers_and_labels.parquet on larger cluster in order to get better results\n",
    "imagesWithLabels = spark.read.parquet(\"wasbs://publicwasb@mmlspark.blob.core.windows.net/flowers_and_labels2.parquet\") \\\n",
    "    .withColumnRenamed(\"bytes\",\"image\").sample(.1)\n",
    "\n",
    "imagesWithLabels.printSchema()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Smiley face](https://i.imgur.com/p2KgdYL.jpg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mmlspark.opencv import ImageTransformer\n",
    "from mmlspark.image import UnrollImage, ImageFeaturizer\n",
    "from mmlspark.stages import *\n",
    "\n",
    "# Make some featurizers\n",
    "it = ImageTransformer()\\\n",
    "    .setOutputCol(\"scaled\")\\\n",
    "    .resize(height = 60, width = 60)\n",
    "\n",
    "ur = UnrollImage()\\\n",
    "    .setInputCol(\"scaled\")\\\n",
    "    .setOutputCol(\"features\")\n",
    "    \n",
    "dc1 = DropColumns().setCols([\"scaled\", \"image\"])\n",
    "\n",
    "lr1 = LogisticRegression().setMaxIter(8).setFeaturesCol(\"features\").setLabelCol(\"labels\")\n",
    "\n",
    "dc2 = DropColumns().setCols([\"features\"])\n",
    "\n",
    "basicModel = Pipeline(stages=[it, ur, dc1, lr1, dc2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "resnet = ImageFeaturizer()\\\n",
    "    .setInputCol(\"image\")\\\n",
    "    .setOutputCol(\"features\")\\\n",
    "    .setModelLocation(model.uri)\\\n",
    "    .setLayerNames(model.layerNames)\\\n",
    "    .setCutOutputLayers(1)\n",
    "    \n",
    "dc3 = DropColumns().setCols([\"image\"])\n",
    "    \n",
    "lr2 = LogisticRegression().setMaxIter(8).setFeaturesCol(\"features\").setLabelCol(\"labels\")\n",
    "\n",
    "dc4 = DropColumns().setCols([\"features\"])\n",
    "\n",
    "deepModel = Pipeline(stages=[resnet, dc3, lr2, dc4])    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Resnet 18](https://i.imgur.com/Mb4Dyou.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How does it work?\n\n![Convolutional network weights](http://i.stack.imgur.com/Hl2H6.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def timedExperiment(model, train, test):\n",
    "  start = time.time()\n",
    "  result =  model.fit(train).transform(test).toPandas()\n",
    "  print(\"Experiment took {}s\".format(time.time() - start))\n",
    "  return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, test = imagesWithLabels.randomSplit([.8,.2])\n",
    "train.count(), test.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "basicResults = timedExperiment(basicModel, train, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deepResults = timedExperiment(deepModel, train, test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot confusion matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import numpy as np\n",
    "\n",
    "def evaluate(results, name):\n",
    "    y, y_hat = results[\"labels\"],results[\"prediction\"]\n",
    "    y = [int(l) for l in y]\n",
    "\n",
    "    accuracy = np.mean([1. if pred==true else 0. for (pred,true) in zip(y_hat,y)])\n",
    "    cm = confusion_matrix(y, y_hat)\n",
    "    cm = cm.astype(\"float\") / cm.sum(axis=1)[:, np.newaxis]\n",
    "\n",
    "    plt.text(40, 10,\"$Accuracy$ $=$ ${}\\%$\".format(round(accuracy*100,1)),fontsize=14)\n",
    "    plt.imshow(cm, interpolation=\"nearest\", cmap=plt.cm.Blues)\n",
    "    plt.colorbar()\n",
    "    plt.xlabel(\"$Predicted$ $label$\", fontsize=18)\n",
    "    plt.ylabel(\"$True$ $Label$\", fontsize=18)\n",
    "    plt.title(\"$Normalized$ $CM$ $for$ ${}$\".format(name))\n",
    "\n",
    "plt.figure(figsize=(12,5))\n",
    "plt.subplot(1,2,1)\n",
    "evaluate(deepResults,\"CNTKModel + LR\")\n",
    "plt.subplot(1,2,2)\n",
    "evaluate(basicResults,\"LR\")\n",
    "# Note that on the larger dataset the accuracy will bump up from 44% to >90%\n",
    "display(plt.show())"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}