lab-06-1-softmax_classifier-eager.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 06-1 Softmax Classification Eager\n",
    "* Softmax를 사용하여분류를 진행합니다.\n",
    "\n",
    "### 기본 Library 선언 및 Tensorflow 버전 확인"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "tf.enable_eager_execution()\n",
    "tf.set_random_seed(777)  # for reproducibility\n",
    "tfe = tf.contrib.eager"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_data = [[1, 2, 1, 1],\n",
    "          [2, 1, 3, 2],\n",
    "          [3, 1, 3, 4],\n",
    "          [4, 1, 5, 5],\n",
    "          [1, 7, 5, 5],\n",
    "          [1, 2, 5, 6],\n",
    "          [1, 6, 6, 6],\n",
    "          [1, 7, 7, 7]]\n",
    "y_data = [[0, 0, 1],\n",
    "          [0, 0, 1],\n",
    "          [0, 0, 1],\n",
    "          [0, 1, 0],\n",
    "          [0, 1, 0],\n",
    "          [0, 1, 0],\n",
    "          [1, 0, 0],\n",
    "          [1, 0, 0]]\n",
    "\n",
    "#convert into numpy and float format\n",
    "x_data = np.asarray(x_data, dtype=np.float32)\n",
    "y_data = np.asarray(y_data, dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8, 4)\n",
      "(8, 3)\n"
     ]
    }
   ],
   "source": [
    "#dataset을 선언합니다.\n",
    "# dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data))\n",
    "# dataset = dataset.repeat().batch(2)\n",
    "\n",
    "nb_classes = 3 #class의 개수입니다.\n",
    "\n",
    "print(x_data.shape)\n",
    "print(y_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.Variable 'weight:0' shape=(4, 3) dtype=float32, numpy=\n",
      "array([[ 0.7706481 ,  0.37335402, -0.05576323],\n",
      "       [ 0.00358377, -0.5898363 ,  1.5702795 ],\n",
      "       [ 0.2460895 , -0.09918973,  1.4418385 ],\n",
      "       [ 0.3200988 ,  0.526784  , -0.7703731 ]], dtype=float32)> <tf.Variable 'bias:0' shape=(3,) dtype=float32, numpy=array([-1.3080608 , -0.13253094,  0.5513761 ], dtype=float32)>\n"
     ]
    }
   ],
   "source": [
    "#Weight and bias setting\n",
    "W = tfe.Variable(tf.random_normal([4, nb_classes]), name='weight')\n",
    "b = tfe.Variable(tf.random_normal([nb_classes]), name='bias')\n",
    "variables = [W, b]\n",
    "\n",
    "print(W,b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[1.36571955e-02 7.90162385e-03 9.78441238e-01]\n",
      " [3.92597765e-02 1.70347411e-02 9.43705440e-01]\n",
      " [3.80385160e-01 1.67723209e-01 4.51891690e-01]\n",
      " [3.23390663e-01 5.90759777e-02 6.17533386e-01]\n",
      " [3.62997412e-06 6.20727292e-08 9.99996305e-01]\n",
      " [2.62520462e-02 1.07279727e-02 9.63019967e-01]\n",
      " [1.56525111e-05 4.21802781e-07 9.99983907e-01]\n",
      " [2.94076904e-06 3.81133276e-08 9.99997020e-01]], shape=(8, 3), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# tf.nn.softmax computes softmax activations\n",
    "# softmax = exp(logits) / reduce_sum(exp(logits), dim)\n",
    "def hypothesis(X):\n",
    "    return tf.nn.softmax(tf.matmul(X, W) + b)\n",
    "\n",
    "print(hypothesis(x_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([[0.9302204  0.06200533 0.00777428]], shape=(1, 3), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# Softmax onehot test\n",
    "sample_db = [[8,2,1,4]]\n",
    "sample_db = np.asarray(sample_db, dtype=np.float32)\n",
    "\n",
    "print(hypothesis(sample_db))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(6.07932, shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "def cost_fn(X, Y):\n",
    "    logits = hypothesis(X)\n",
    "    cost = -tf.reduce_sum(Y * tf.log(logits), axis=1)\n",
    "    cost_mean = tf.reduce_mean(cost)\n",
    "    \n",
    "    return cost_mean\n",
    "\n",
    "print(cost_fn(x_data, y_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(6.0, shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "x = tf.constant(3.0)\n",
    "with tf.GradientTape() as g:\n",
    "    g.watch(x)\n",
    "    y = x * x # x^2\n",
    "dy_dx = g.gradient(y, x) # Will compute to 6.0\n",
    "print(dy_dx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<tf.Tensor: id=118, shape=(4, 3), dtype=float32, numpy=\n",
      "array([[ 0.06914616, -0.6509784 ,  0.58183223],\n",
      "       [-1.5221257 , -1.214863  ,  2.7369888 ],\n",
      "       [-1.2473826 , -1.7611003 ,  3.008483  ],\n",
      "       [-1.2014605 , -1.8659232 ,  3.0673838 ]], dtype=float32)>, <tf.Tensor: id=116, shape=(3,), dtype=float32, numpy=array([-0.15212911, -0.342192  ,  0.49432108], dtype=float32)>]\n"
     ]
    }
   ],
   "source": [
    "def grad_fn(X, Y):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss = cost_fn(X, Y)\n",
    "        grads = tape.gradient(loss, variables)\n",
    "\n",
    "        return grads\n",
    "\n",
    "print(grad_fn(x_data, y_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at epoch 1: 0.126001\n",
      "Loss at epoch 100: 0.124082\n",
      "Loss at epoch 200: 0.122203\n",
      "Loss at epoch 300: 0.120380\n",
      "Loss at epoch 400: 0.118610\n",
      "Loss at epoch 500: 0.116891\n",
      "Loss at epoch 600: 0.115221\n",
      "Loss at epoch 700: 0.113598\n",
      "Loss at epoch 800: 0.112019\n",
      "Loss at epoch 900: 0.110482\n",
      "Loss at epoch 1000: 0.108987\n",
      "Loss at epoch 1100: 0.107531\n",
      "Loss at epoch 1200: 0.106112\n",
      "Loss at epoch 1300: 0.104730\n",
      "Loss at epoch 1400: 0.103382\n",
      "Loss at epoch 1500: 0.102067\n",
      "Loss at epoch 1600: 0.100785\n",
      "Loss at epoch 1700: 0.099534\n",
      "Loss at epoch 1800: 0.098313\n",
      "Loss at epoch 1900: 0.097120\n",
      "Loss at epoch 2000: 0.095956\n"
     ]
    }
   ],
   "source": [
    "def fit(X, Y, epochs=2000, verbose=100):\n",
    "    optimizer =  tf.train.GradientDescentOptimizer(learning_rate=0.1)\n",
    "\n",
    "    for i in range(epochs):\n",
    "        grads = grad_fn(X, Y)\n",
    "        optimizer.apply_gradients(zip(grads, variables))\n",
    "        if (i==0) | ((i+1)%verbose==0):\n",
    "            print('Loss at epoch %d: %f' %(i+1, cost_fn(X, Y).numpy()))\n",
    "            \n",
    "fit(x_data, y_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prediction Check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[0.00783868 0.08934675 0.90281457]\n",
      " [0.00837632 0.10334261 0.88828105]\n",
      " [0.00890698 0.11949147 0.8716016 ]\n",
      " [0.01300187 0.34645918 0.64053893]\n",
      " [0.01916472 0.3473155  0.6335198 ]\n",
      " [0.01627325 0.38224185 0.6014849 ]\n",
      " [0.03545809 0.24856271 0.7159792 ]\n",
      " [0.04392094 0.21500538 0.74107367]], shape=(8, 3), dtype=float32)\n",
      "tf.Tensor([2 2 2 2 2 2 2 2], shape=(8,), dtype=int64)\n"
     ]
    }
   ],
   "source": [
    "sample_data = [[2,1,3,2]] # answer_label [[0,0,1]]\n",
    "sample_data = np.asarray(sample_data, dtype=np.float32)\n",
    "\n",
    "a = hypothesis(sample_data)\n",
    "\n",
    "print(a)\n",
    "print(tf.argmax(a, 1)) #index: 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[1.17311458e-06 4.64814686e-04 9.99534011e-01]\n",
      " [8.37631337e-03 1.03342585e-01 8.88281107e-01]\n",
      " [1.04251918e-09 1.33687243e-01 8.66312802e-01]\n",
      " [5.01113000e-08 8.58640432e-01 1.41359538e-01]\n",
      " [1.16531663e-01 8.71398807e-01 1.20694786e-02]\n",
      " [5.55007122e-02 9.44499254e-01 7.49097243e-11]\n",
      " [8.76283109e-01 1.23716690e-01 2.13872141e-07]\n",
      " [9.74358737e-01 2.56412514e-02 1.90751481e-09]], shape=(8, 3), dtype=float32)\n",
      "tf.Tensor([2 2 2 1 1 1 0 0], shape=(8,), dtype=int64)\n",
      "tf.Tensor([2 2 2 1 1 1 0 0], shape=(8,), dtype=int64)\n"
     ]
    }
   ],
   "source": [
    "b = hypothesis(x_data)\n",
    "print(b)\n",
    "print(tf.argmax(b, 1))\n",
    "print(tf.argmax(y_data, 1)) # matches with y_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert as Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class softmax_classifer(tf.keras.Model):\n",
    "    def __init__(self, nb_classes):\n",
    "        super(softmax_classifer, self).__init__()\n",
    "        self.W = tfe.Variable(tf.random_normal([4, nb_classes]), name='weight')\n",
    "        self.b = tfe.Variable(tf.random_normal([nb_classes]), name='bias')\n",
    "        \n",
    "    def softmax_regression(self, X):\n",
    "        return tf.nn.softmax(tf.matmul(X, self.W) + self.b)\n",
    "    \n",
    "    def cost_fn(self, X, Y):\n",
    "        logits = self.softmax_regression(X)\n",
    "        cost = tf.reduce_mean(-tf.reduce_sum(Y * tf.log(logits), axis=1))\n",
    "        \n",
    "        return cost\n",
    "    \n",
    "    def grad_fn(self, X, Y):\n",
    "        with tf.GradientTape() as tape:\n",
    "            cost = self.cost_fn(x_data, y_data)\n",
    "            grads = tape.gradient(cost, self.variables)\n",
    "            \n",
    "            return grads\n",
    "    \n",
    "def fit(self, X, Y, epochs=2000, verbose=500):\n",
    "    optimizer =  tf.train.GradientDescentOptimizer(learning_rate=0.1)\n",
    "\n",
    "    for i in range(epochs):\n",
    "        grads = self.grad_fn(X, Y)\n",
    "        optimizer.apply_gradients(zip(grads, self.variables))\n",
    "        if (i==0) | ((i+1)%verbose==0):\n",
    "            print('Loss at epoch %d: %f' %(i+1, self.loss_fn(X, Y).numpy()))\n",
    "            \n",
    "model = softmax_classifer(nb_classes)\n",
    "model.fit(x_data, y_data)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}