[NN] Lab session 3 - TensorFlow eager execution.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Neural Networks\n",
    "## Lab session 3: TensorFlow eager"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Basic mechanics of TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "6A1QaLgXkI3Q",
    "outputId": "c0661465-73e0-4773-8ec7-482b869a34dc",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[2.1477053 2.7965155]\n",
      " [2.6615286 2.8650625]\n",
      " [2.0061755 2.0236745]], shape=(3, 2), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "a = tf.random_uniform(shape=[3, 2])\n",
    "b = a + 2.0\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "Oa54tf5wBkST",
    "outputId": "d4ea272c-f6c6-4375-b1c4-9c5ec57133a9",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "with sess.as_default():\n",
    "    print(sess.run(b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "UZsEuv93Blm2",
    "outputId": "e13a369e-a563-407c-b9a3-a0956fb20fc8",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"add:0\", shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "with sess.as_default():\n",
    "    c = sess.run(1.5*b)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Lr791RZ7BmoL",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\simon\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "# IMPORTANT: restart the runtime before carrying on\n",
    "import tensorflow as tf\n",
    "tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "-nyHSJHpBuXk",
    "outputId": "99c148b3-eb41-497e-b49a-a16ab8081b68",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# Try again\n",
    "a = tf.random_uniform(shape=[3, 2])\n",
    "b = a + 2.0\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "Yr04sfwBB03a",
    "outputId": "36c7712d-49d4-4333-b0f7-65a8d75b0f92"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: id=28, shape=(), dtype=float32, numpy=-5.4667816>]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Auto-differencing\n",
    "import tensorflow.contrib.eager as tfe\n",
    "\n",
    "def f(x):\n",
    "    return tf.sin(x ** 2.0)\n",
    "\n",
    "f_grad = tfe.gradients_function(f, ['x'])\n",
    "\n",
    "f_grad(tf.constant(3.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "VqzuumEjB6zN",
    "outputId": "dd94c9d1-d612-4582-aa31-ddbe163c55ef"
   },
   "outputs": [],
   "source": [
    "ff = tfe.gradients_function(f_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "zZ-llIHWB-n8",
    "outputId": "030eca2f-8ce3-42da-e659-ea72ee8875ae",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(<tf.Tensor: id=57, shape=(), dtype=float32, numpy=0.9553365>,\n",
       "  <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.3>)]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Using variables and implicit gradients.\n",
    "w = tfe.Variable(0.3)\n",
    "\n",
    "def f():\n",
    "    return tf.sin(w)\n",
    "\n",
    "f_grad = tfe.implicit_gradients(f)\n",
    "\n",
    "f_grad()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Classification on the Iris dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "zS_Aj1ziBxOU",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn import datasets, model_selection\n",
    "dataset = datasets.load_iris()\n",
    "X = dataset['data']\n",
    "y = dataset['target']\n",
    "Xtrain, Xtest, ytrain, ytest = \\\n",
    "            model_selection.train_test_split(X, y, stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "class NN(tf.keras.Model):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super(NN, self).__init__()\n",
    "        self.hidden = tf.layers.Dense(15, activation=tf.nn.tanh)\n",
    "        self.out = tf.layers.Dense(3)\n",
    "        \n",
    "    def call(self, x):\n",
    "        return self.out(self.hidden(x))\n",
    "    \n",
    "net = NN()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "cQjfFftsCPOY",
    "outputId": "bdc6ea3b-6000-4b7a-f04d-0b5c6f58a1dd",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Play with the model variables.\n",
    "net.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "anWl97SgCQZT",
    "outputId": "cc5babb2-d72c-4d0d-9673-b2f850ebe126"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=117, shape=(1, 3), dtype=float64, numpy=array([[ 2.36962899,  0.55703203, -1.55184501]])>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net(tf.constant(X[0:1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "moH8t6lBCSCu",
    "outputId": "e4eeec36-45ab-47c7-adb4-6e61e050f85c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(net.variables)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "ofy1cZfeCX8W",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# Define the loss function\n",
    "def loss(x, y):\n",
    "    ypred = net(x)\n",
    "    l = tf.losses.sparse_softmax_cross_entropy(\\\n",
    "                                               labels=y, \\\n",
    "                                               logits=ypred\\\n",
    "                                              )\n",
    "    return l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "rWYnBsE2C1CH",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# Instatiate loss and optimizer\n",
    "loss_grad = tfe.implicit_gradients(loss)\n",
    "opt = tf.train.GradientDescentOptimizer(0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 107
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "JsTRx0quC3nS",
    "outputId": "75079a02-8452-410d-b9fb-e9ff14254ee2",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# Training\n",
    "Xtrain_t = tf.constant(Xtrain)\n",
    "ytrain_t = tf.constant(ytrain)\n",
    "\n",
    "for i in range(500):\n",
    "    \n",
    "    grads = loss_grad(Xtrain_t, ytrain_t)\n",
    "    opt.apply_gradients(grads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "eWAwo78DDUta",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=45232, shape=(), dtype=float32, numpy=0.7894737>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Accuracy\n",
    "import numpy as np\n",
    "ypred = net(tf.constant(Xtest))\n",
    "ypred = tf.argmax(ypred, axis=1)\n",
    "tmp = tf.cast(tf.equal(ypred, tf.constant(ytest.astype(np.int64))), tf.float32)\n",
    "tf.reduce_mean(tmp)"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "colab": {
   "collapsed_sections": [],
   "name": "Untitled3.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}