deprecated/06-ranknet/RankNet.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Network Architecture"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Network](architecture.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Theory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Probability of RankNet's problem\n",
    "$$ P_{ij} = P(x_j > x_i) $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Output Node's result\n",
    "$$ o_{ij} = f(x_i) - f(x_j)$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- re-define probability of output node\n",
    "$$ \\overline{P_{ij}} = {\\frac{e^{o_{ij}}}{1 + e^{o_{ij}}}} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- cost function\n",
    "$$ C_{ij} = -\\overline{P_{ij}}log(P_{ij}) - (1-\\overline{P_{ij}}log(1-P_{ij})) $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "import keras\n",
    "from keras import backend\n",
    "from keras.layers import Activation, Dense, Input, Subtract\n",
    "from keras.models import Model, Sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def base_network(input_dim):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(input_dim, input_shape=(input_dim,), activation='relu'))\n",
    "    model.add(Dense(100, activation='relu'))\n",
    "    model.add(Dense(50, activation='relu'))\n",
    "    model.add(Dense(1))\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/gshs/anaconda3/envs/gryta/lib/python3.7/site-packages/tensorflow/python/ops/nn_impl.py:180: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /Users/gshs/anaconda3/envs/gryta/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "Train on 80 samples, validate on 20 samples\n",
      "Epoch 1/10\n",
      "80/80 [==============================] - 0s 2ms/step - loss: 0.9398 - accuracy: 0.0125 - val_loss: 0.3847 - val_accuracy: 1.0000\n",
      "Epoch 2/10\n",
      "80/80 [==============================] - 0s 84us/step - loss: 0.3763 - accuracy: 1.0000 - val_loss: 0.2081 - val_accuracy: 1.0000\n",
      "Epoch 3/10\n",
      "80/80 [==============================] - 0s 54us/step - loss: 0.1981 - accuracy: 1.0000 - val_loss: 0.1079 - val_accuracy: 1.0000\n",
      "Epoch 4/10\n",
      "80/80 [==============================] - 0s 73us/step - loss: 0.1017 - accuracy: 1.0000 - val_loss: 0.0601 - val_accuracy: 1.0000\n",
      "Epoch 5/10\n",
      "80/80 [==============================] - 0s 53us/step - loss: 0.0560 - accuracy: 1.0000 - val_loss: 0.0316 - val_accuracy: 1.0000\n",
      "Epoch 6/10\n",
      "80/80 [==============================] - 0s 49us/step - loss: 0.0296 - accuracy: 1.0000 - val_loss: 0.0165 - val_accuracy: 1.0000\n",
      "Epoch 7/10\n",
      "80/80 [==============================] - 0s 44us/step - loss: 0.0155 - accuracy: 1.0000 - val_loss: 0.0086 - val_accuracy: 1.0000\n",
      "Epoch 8/10\n",
      "80/80 [==============================] - 0s 52us/step - loss: 0.0080 - accuracy: 1.0000 - val_loss: 0.0046 - val_accuracy: 1.0000\n",
      "Epoch 9/10\n",
      "80/80 [==============================] - 0s 45us/step - loss: 0.0042 - accuracy: 1.0000 - val_loss: 0.0024 - val_accuracy: 1.0000\n",
      "Epoch 10/10\n",
      "80/80 [==============================] - 0s 46us/step - loss: 0.0022 - accuracy: 1.0000 - val_loss: 0.0013 - val_accuracy: 1.0000\n"
     ]
    }
   ],
   "source": [
    "input_dim = 200\n",
    "n = 100\n",
    "\n",
    "# define input\n",
    "pre_pair_input = Input(shape=(input_dim,))\n",
    "post_pair_input = Input(shape=(input_dim,))\n",
    "\n",
    "# get pair network\n",
    "base_model = base_network(input_dim=input_dim)\n",
    "rel_score = base_model(pre_pair_input)\n",
    "irr_score = base_model(post_pair_input)\n",
    "\n",
    "# subtract each score\n",
    "diff = Subtract()([rel_score, irr_score])\n",
    "\n",
    "# activate sigmoid function\n",
    "prob = Activation(\"sigmoid\")(diff)\n",
    "\n",
    "# build\n",
    "model = Model(inputs=[pre_pair_input, post_pair_input], outputs=prob)\n",
    "model.compile(optimizer=\"adam\", loss=\"binary_crossentropy\", metrics=['accuracy'])\n",
    "\n",
    "# data : a is always higher than b\n",
    "doc_a_feature = np.random.uniform(size=(n, input_dim)) + 1\n",
    "doc_b_feature = np.random.uniform(size=(n, input_dim))\n",
    "y = np.ones((n, 1))\n",
    "\n",
    "# train\n",
    "history = model.fit([doc_a_feature, doc_b_feature], y, epochs=10, batch_size=100, validation_split=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3df3xU9Z3v8dcnISEQIr8SiiVoqAYw/gLMtbZgLbU/wNuF1qVW1HJ169qyrb+669VtlVa3uw9324dX29IftFbXXSprdbXeW6x1K121rdYAghg6ExTUABkCAk5+QAh87h8zSUNIYAI5OTM57+fjkYdzzpw58zmDyXvO+X7P92vujoiIRFde2AWIiEi4FAQiIhGnIBARiTgFgYhIxCkIREQibkjYBfRVaWmpV1RUhF2GiEhOWb169U53L+vpuZwLgoqKCmpqasIuQ0Qkp5jZm709p0tDIiIRpyAQEYk4BYGISMQpCEREIk5BICIScYEFgZn91Mx2mNmGXp43M/uOmW0ys/VmNiOoWkREpHdBnhE8CMw5yvNzgcr0z3XADwKsRUREehHYfQTu/pyZVRxlk/nAQ54aB/tFMxtlZie7+/ZACnrqNmh4NZBd90VzWzvvNLeFXYaI5KCiidMo+8z/6ff9hnlD2QTg7S7L9el1RwSBmV1H6qyBU045ZUCKC0r97hZ2txwIuwwRyUEjR++nx1uDT1BO3Fns7suAZQDV1dXHN5PO3Lv7s6Tjdu2/PMu0M0bz3YXTwy5FRAQIt9fQVmBil+Xy9LpBq3l/O2+/08rkcSPCLkVEpFOYQfAksCjde+gCYG9g7QNZom5HEwCTx5eEXImIyJ8FdmnIzB4GPgyUmlk98HWgAMDdfwisBC4BNgEtwDVB1ZIt4g1JAKa8R0EgItkjyF5DC4/xvANfCur9s1E8kaSoII+JY4aHXYqISCfdWTyAYokkleNKyM+zsEsREemkIBhA8USSyveooVhEsouCYIDsaWkj8e5+tQ+ISNZREAyQeEI9hkQkOykIBkg8oR5DIpKdFAQDJJ5IUjJ0CCePLAq7FBGRwygIBkisIdVQbKYeQyKSXRQEA8DdiSeSTFH7gIhkIQXBAGhs2s/ulgNMVvuAiGQhBcEAqEv3GFJDsYhkIwXBAIilxxhS11ERyUYKggEQTyQZU1xI6YihYZciInIEBcEAiCWSTNbQEiKSpRQEAXN36hJNah8QkaylIAjYtr37aNrfrvYBEclaCoKAaTIaEcl2CoKAxdJjDFUqCEQkSykIAhZvSDL+pCJGDisIuxQRkR4pCAIWSyTVPiAiWU1BEKCDh5xNO5qYoq6jIpLFFAQBeuudFva3H1L7gIhkNQVBgGLqMSQiOUBBEKB4Z48hXRoSkeylIAhQLJHklDHDGV44JOxSRER6pSAIUF0iqTkIRCTrKQgC0tZ+iDcamzXYnIhkPQVBQDbvbKb9kGt6ShHJegqCgHQMLaFLQyKS7RQEAYk3JMnPM95XVhx2KSIiR6UgCEg8kWRSaTFDh+SHXYqIyFEpCAIS16xkIpIjAg0CM5tjZjEz22Rmt/Xw/ClmtsrM1prZejO7JMh6Bkpr20HefKdF7QMikhMCCwIzyweWAnOBKmChmVV12+x24BF3nw5cDnw/qHoG0qYdTbhraAkRyQ1BnhGcD2xy9zfcvQ1YAczvto0DJ6UfjwS2BVjPgOkYWkLDT4tILggyCCYAb3dZrk+v6+obwFVmVg+sBK7vaUdmdp2Z1ZhZTWNjYxC19qt4IknhkDxOHTM87FJERI4p7MbihcCD7l4OXAL8m5kdUZO7L3P3anevLisrG/Ai+yqWSHJa2QiG5If98YqIHFuQf6m2AhO7LJen13X1eeARAHf/A1AElAZY04CINyQ1GY2I5Iwgg+BloNLMJplZIanG4Ce7bfMWcDGAmZ1BKgiy/9rPUby77wDb9u5T+4CI5IzAgsDd24EvA08DG0n1DnrNzO4ys3npzf4W+GszWwc8DFzt7h5UTQOhLtEEqMeQiOSOQAfKd/eVpBqBu65b0uVxLTAzyBoGWlxjDIlIjlFrZj+LNSQZXpjPhFHDwi5FRCQjCoJ+Fk8kqXxPCXl5FnYpIiIZURD0s3hCPYZEJLcoCPrRrqb97GxqU/uAiOQUBUE/inf0GFLXURHJIQqCfqQeQyKSixQE/SiWSDJyWAHjSoaGXYqISMYUBP0oNbRECWbqMSQiuUNB0E/cPTUr2Xj1GBKR3KIg6CeJd/fz7r52DS0hIjlHQdBPYumG4koFgYjkGAVBP4k3qMeQiOQmBUE/iSWSlJUMZUxxYdiliIj0iYKgn9QlkmofEJGcpCDoB4cOOfFEE5UaY0hEcpCCoB/U726l9cBBnRGISE5SEPSDjh5Dmp5SRHKRgqAfdIwxVDlOl4ZEJPcoCPpBPJFkwqhhlBQVhF2KiEifKQj6QawhyWQ1FItIjlIQnKADBw/xRmOz2gdEJGcpCE7Qm7uaaTt4SD2GRCRnKQhOUKwhNSuZhpYQkVylIDhB8USSPIPT1WNIRHKUguAExRNJTh1bTFFBftiliIgcFwXBCYol1GNIRHKbguAE7DtwkC07m9VQLCI5TUFwAl5vbOKQa2gJEcltCoITUJdI9RjSGYGI5DIFwQmIJZIU5BsVpcVhlyIictwUBCcg3pDkfaUjKMjXxygiuSvQv2BmNsfMYma2ycxu62Wby8ys1sxeM7OfBVlPf4slkmofEJGcF1gQmFk+sBSYC1QBC82sqts2lcDfAzPd/UzgpqDq6W/N+9up393KFHUdFZEcF+QZwfnAJnd/w93bgBXA/G7b/DWw1N13A7j7jgDr6Vd1OzS0hIgMDkEGwQTg7S7L9el1XU0GJpvZ78zsRTOb09OOzOw6M6sxs5rGxsaAyu2beEN6VjIFgYjkuLBbOYcAlcCHgYXAj81sVPeN3H2Zu1e7e3VZWdkAl9izWCJJUUEeE8cMD7sUEZETcswgMLOxx7nvrcDELsvl6XVd1QNPuvsBd98MxEkFQ9aLJ5JUjishP8/CLkVE5IRkckbwopn93MwuMbO+/NV7Gag0s0lmVghcDjzZbZsnSJ0NYGalpC4VvdGH9whNalYyXRYSkdyXSRBMBpYBnwPqzOyfzGzysV7k7u3Al4GngY3AI+7+mpndZWbz0ps9Dewys1pgFXCLu+86ngMZSHta2tiR3M+U8eoxJCK5b8ixNnB3B54BnjGz2cC/A39jZuuA29z9D0d57UpgZbd1S7rt+yvpn5wRTw8tUakzAhEZBI4ZBOk2gqtInREkgOtJXeKZBvwcmBRkgdkolkj1GNIYQyIyGBwzCIA/AP8GfMrd67usrzGzHwZTVnaLNyQpGTqEk0cWhV2KiMgJyyQIpqQv4RzB3f+5n+vJCR1DS/St7VxEJDtl0lj86659+81stJk9HWBNWc3dqUuox5CIDB6ZBEGZu+/pWEgPBzEuuJKyW2PTfna3HND0lCIyaGQSBAfN7JSOBTM7FejxUlEUxBs0GY2IDC6ZtBF8DXjBzP4bMOBC4LpAq8piHT2GNPy0iAwWmdxH8CszmwFckF51k7vvDLas7BVvSDK2uJDSEUPDLkVEpF9kckYAcBDYARQBVWaGuz8XXFnZK74jSaXaB0RkEMnkhrJrgRtJDRr3Cqkzgz8AHwm2tOzj7sQbkiw4rzzsUkRE+k0mjcU3Av8DeNPdZwPTgT1Hf8ngtHVPK81tB9U+ICKDSiZBsM/d9wGY2VB3/xMwJdiyslNcQ0uIyCCUSRtBffqGsidIDTy3G3gz2LKyU6xBg82JyOCTSa+hT6cffsPMVgEjgV8FWlWWqkskGX9SESOHFYRdiohIvzlqEJhZPvCau08FcPf/HpCqslTHGEMiIoPJUdsI3P0gEOt6Z3FUHTzk1O1oYoq6jorIIJNJG8Fo4DUz+yPQ3LHS3ef1/pLB581dzbS1H9JgcyIy6GQSBHcEXkUO6OwxpEtDIjLIZBIEbwHbu3QhHQa8J9CqslDH9JSnj9OlIREZXDK5j+DnwKEuywfT6yIllkhyypjhDC/MdFQOEZHckEkQDHH3to6F9OPC4ErKTvEGTUYjIoNTJkHQaGadDcNmNh+I1Oij+9sPsnlnM1PG67KQiAw+mVzn+CKw3My+l16uBxYFV1L22byzmfZDrjMCERmUMrmz+HXgAjMbkV5uCryqLNPRUKwgEJHB6JiXhszsn8xslLs3uXtTevL6bw5Ecdki3pAkP894X1lx2KWIiPS7TNoI5vYwef0lwZWUfWKJJJNKixk6JD/sUkRE+l0mQZBvZp3zMqbvI4jUPI3xRFJDT4vIoJVJECwHfmNmn0/PVvYM8K/BlpU9WtsO8tY7LWofEJFBK5PG4n82s3XARwEHngZODbqwbLFpRxPuMFmDzYnIIJXJGQFAglQIfIbUXMUbA6soy8TSYwxp+GkRGax6PSMws8nAwvTPTuA/AEvPWxwZ8USSwiF5nDpmeNiliIgE4mhnBH8i9e3/k+4+y92/S2qcoYyZ2Rwzi5nZJjO77Sjb/aWZuZlV92X/AyHWkOT0shEMyc/05ElEJLcc7a/bpcB2YJWZ/djMLgYs0x2nZzdbCswFqoCFZlbVw3YlwI3AS30pfKDEE0kNPS0ig1qvQeDuT7j75cBUYBVwEzDOzH5gZh/PYN/nA5vc/Y30QHUrgPk9bPcPwD8D+/pcfcDe3XeA7Xv3UamGYhEZxI55vcPdm939Z+7+F0A5sBa4NYN9TwDe7rJcn17XycxmABPd/ZdH25GZXWdmNWZW09jYmMFb94+6jslo1HVURAaxPl34dvfd7r7M3S8+0Tc2szzgHuBvM3jfZe5e7e7VZWVlJ/rWGYs1aIwhERn8gmwB3QpM7LJcnl7XoQQ4C/itmW0BLgCezKYG43giSXFhPhNGDQu7FBGRwAQZBC8DlWY2ycwKgcuBJzuedPe97l7q7hXuXgG8CMxz95oAa+qTWEOSyveUkJeXcRu5iEjOCSwI3L0d+DKpO5E3Ao+4+2tmdlfXiW6yWd2OpO4oFpFBL9AJeN19JbCy27olvWz74SBr6audTfvZ2dSm9gERGfR0l1Qv4h09hnQPgYgMcgqCXsQb1HVURKJBQdCLWKKJkcMKKCuJ1NQLIhJBCoJe1KUnozFTjyERGdwUBD1wd2KJJJPHq8eQiAx+CoIeNLy7j+S+drUPiEgkKAh6EEs3FKvrqIhEgYKgBx1dRxUEIhIFCoIexBNNlJUMZXRxYdiliIgETkHQg3i6x5CISBQoCLo5dMiJJ5K6LCQikaEg6Obt3S3sO3CIKeo6KiIRoSDopqPHUKXOCEQkIhQE3dTtSM1KVjlOZwQiEg0Kgm5iDUkmjBpGSVFB2KWIiAwIBUE38URSQ0+LSKQoCLo4cPAQrzc2qceQiESKgqCLLTubOXDQNT2liESKgqCLeCLVUKwzAhGJEgVBF7FEkjyD09VjSEQiREHQRbwhScXYYooK8sMuRURkwCgIutDQEiISRQqCtH0HDrJlV7MaikUkchQEaa83NnHIYbLuIRCRiFEQpHVMRqPhp0UkahQEabGGJgryjYrS4rBLEREZUAqCtHgiyWllIyjI10ciItGiv3pp8URSQ0+LSCQpCICm/e3U725linoMiUgEKQiAunRDse4hEJEoUhDQpceQuo6KSAQFGgRmNsfMYma2ycxu6+H5r5hZrZmtN7PfmNmpQdbTm1hDE0UFeUwcPTyMtxcRCVVgQWBm+cBSYC5QBSw0s6pum60Fqt39HOBR4F+Cqudo6nYkqRxXQl6ehfH2IiKhCvKM4Hxgk7u/4e5twApgftcN3H2Vu7ekF18EygOsp1exBo0xJCLRFWQQTADe7rJcn17Xm88DT/X0hJldZ2Y1ZlbT2NjYjyXC7uY2diT3M2W8egyJSDRlRWOxmV0FVAPf6ul5d1/m7tXuXl1WVtav7x1XjyERibghAe57KzCxy3J5et1hzOyjwNeAi9x9f4D19EhBICJRF+QZwctApZlNMrNC4HLgya4bmNl04EfAPHffEWAtvYonmigZOoSTRxaF8fYiIqELLAjcvR34MvA0sBF4xN1fM7O7zGxeerNvASOAn5vZK2b2ZC+7C0wskWTy+BLM1GNIRKIpyEtDuPtKYGW3dUu6PP5okO9/LO5OPJFk7lknh1mGiEiosqKxOCyNyf3saTmgMYZEJNIiHQQxNRSLiEQ7COKJJkDTU4pItEU7CBqSjC0upHTE0LBLEREJTaCNxdkultDQEiI9OXDgAPX19ezbty/sUqSPioqKKC8vp6CgIOPXRDYIDh1y6hJJPlM98dgbi0RMfX09JSUlVFRUqGt1DnF3du3aRX19PZMmTcr4dZG9NLR1TyvNbQepVI8hkSPs27ePsWPHKgRyjJkxduzYPp/JRTYI6nakJ6PRpSGRHikEctPx/LtFNghiDakeQ5qwXkSiLrJBEE8kOXlkESOHZd6gIiIDY9euXUybNo1p06Yxfvx4JkyY0Lnc1taW0T6uueYaYrHYUbdZunQpy5cv74+SD/Nf//VffOpTnzrqNmvWrOFXv/pVv7/38YhsY7EmoxHJXmPHjuWVV14B4Bvf+AYjRozg7/7u7w7bxt1xd/Lyev4++8ADDxzzfb70pS+deLHHac2aNWzYsIE5c+aEVkOHSAbBwUPOpsYmZp4+NuxSRLLenf/3NWq3vduv+6x670l8/S/O7PPrNm3axLx585g+fTpr167lmWee4c4772TNmjW0trby2c9+liVLUsOZzZo1i+9973ucddZZlJaW8sUvfpGnnnqK4cOH84tf/IJx48Zx++23U1payk033cSsWbOYNWsWzz77LHv37uWBBx7ggx/8IM3NzSxatIiNGzdSVVXFli1b+MlPfsK0adMOq+2Xv/wlX/nKVyguLmbmzJmd61988UVuvvlm9u3bx/Dhw3nwwQcpLy/nrrvuorW1ld/+9rfcfvvtlJeXH7FdZWXliX3QGYrkpaE3dzXT1n5IZwQiOehPf/oTN998M7W1tUyYMIG7776bmpoa1q1bxzPPPENtbe0Rr9m7dy8XXXQR69at4wMf+AA//elPe9y3u/PHP/6Rb33rW9x1110AfPe732X8+PHU1tZyxx13sHbt2iNe19LSwhe+8AVWrlzJ6tWr2bZtW+dzZ5xxBs8//zxr167ljjvu4Pbbb2fYsGEsWbKEK6+8kldeeYUFCxb0uN1AieQZQcdkNFM0tITIMR3PN/cgnXbaaVRXV3cuP/zww9x///20t7ezbds2amtrqaqqOuw1w4YNY+7cuQCcd955PP/88z3u+9JLL+3cZsuWLQC88MIL3HrrrQCce+65nHnmkZ9HbW0tkydP5rTTTgPgyiuv5KGHHgJgz549LFq0iNdff/2ox5XpdkGI5BlBrKEJMzh9nO4hEMk1xcXFnY/r6uq47777ePbZZ1m/fj1z5szpsQ99YWFh5+P8/Hza29t73PfQoUOPuU1ffe1rX+MTn/gEGzZs4Iknnui1j3+m2wUhkkEQTyQ5ZcxwhhdG8oRIZNB49913KSkp4aSTTmL79u08/fTT/f4eM2fO5JFHHgHg1Vdf7fHSU1VVFXV1dWzevBl35+GHH+58bu/evUyYMAGABx98sHN9SUkJyWTymNsNhEgGQSyRpHKcLguJ5LoZM2ZQVVXF1KlTWbRo0WGNtP3l+uuvZ+vWrVRVVXHnnXdSVVXFyJEjD9tm+PDh/PCHP2Tu3LlUV1dz8sl/nuzq1ltv5ZZbbmHGjBm4e+f6j3zkI6xbt47p06fz6KOP9rrdQLCBfsMTVV1d7TU1Ncf9+v3tBzlzydN84aL3ccsnpvZjZSKDx8aNGznjjDPCLiMrtLe3097eTlFREXV1dXz84x+nrq6OIUOy94pCT/9+Zrba3at72j57jyQgm3c2037I1WNIRDLS1NTExRdfTHt7O+7Oj370o6wOgeMxuI4mA7EG9RgSkcyNGjWK1atXh11GoCLXRhBPJBmSZ7yvVD2GREQggkEQa2iiorSYwiGRO3QRkR5F7q9h3Y6khp4WEekiUkHQ0tbOW++0qKFYRKSLSAXBph1NuMOU8WofEMlms2fPPuLmsHvvvZfFixcf9XUjRqR+t7dt28aCBQt63ObDH/4wx+qCfu+999LS0tK5fMkll7Bnz55MSu+Tjnp7s2fPHr7//e/3+/t2F6kg6OgxpDMCkey2cOFCVqxYcdi6FStWsHDhwoxe/973vpdHH330uN+/exCsXLmSUaNGHff+jtdABUGkuo/GE0kKh+Rx6tjiY28sIilP3QYNr/bvPsefDXPv7vXpBQsWcPvtt9PW1kZhYSFbtmxh27ZtXHjhhTQ1NTF//nx2797NgQMH+OY3v8n8+fMPe/2WLVv45Cc/yYYNG2htbeWaa65h3bp1TJ06ldbW1s7tFi9ezMsvv0xraysLFizgzjvv5Dvf+Q7btm1j9uzZlJaWsmrVKioqKqipqaG0tJR77rmnc/TSa6+9lptuuoktW7Ywd+5cZs2axe9//3smTJjAL37xC4YNG3ZYXZs3b+aKK67oPIYOvR3Tbbfdxuuvv860adP42Mc+xte//vVjHvvxiFgQNHF62Qjy8zQXq0g2GzNmDOeffz5PPfUU8+fPZ8WKFVx22WWYGUVFRTz++OOcdNJJ7Ny5kwsuuIB58+b1OlfvD37wA4YPH87GjRtZv349M2bM6HzuH//xHxkzZgwHDx7k4osvZv369dxwww3cc889rFq1itLS0sP2tXr1ah544AFeeukl3J33v//9XHTRRYwePZq6ujoefvhhfvzjH3PZZZfx2GOPcdVVVx32+htvvJHFixezaNEili5d2rm+t2O6++672bBhQ+ckPe3t7X069kxFLAiSXPA+TUYj0idH+eYepI7LQx1BcP/99wOpOQO++tWv8txzz5GXl8fWrVtJJBKMHz++x/0899xz3HDDDQCcc845nHPOOZ3PPfLIIyxbtoz29na2b99ObW3tYc9398ILL/DpT3+6cwTUSy+9lOeff5558+YxadKkzslqug5j3dXvfvc7HnvsMQA+97nPdQ5v3dsxddfXY89UZIJgb+sBtu/dp/YBkRwxf/58br75ZtasWUNLSwvnnXceAMuXL6exsZHVq1dTUFBARUXFcQ3ZvHnzZr797W/z8ssvM3r0aK6++uoTGvq5YwhrSA1j3fUSVFc9fXvP9Jj669i7i0xjcV2io6FYPYZEcsGIESOYPXs2f/VXf3VYI/HevXsZN24cBQUFrFq1ijfffPOo+/nQhz7Ez372MwA2bNjA+vXrgdQQ1sXFxYwcOZJEIsFTTz3V+ZruQ0R3uPDCC3niiSdoaWmhubmZxx9/nAsvvDDjY5o5c2ZnI/jy5cuPeUw9DVXdl2PPVKBBYGZzzCxmZpvM7LYenh9qZv+Rfv4lM6sIqpZYQj2GRHLNwoULWbdu3WFBcOWVV1JTU8PZZ5/NQw89xNSpRx9FePHixTQ1NXHGGWewZMmSzjOLc889l+nTpzN16lSuuOKKw4awvu6665gzZw6zZ88+bF8zZszg6quv5vzzz+f9738/1157LdOnT8/4eO677z6WLl3K2WefzdatW495TGPHjmXmzJmcddZZ3HLLLX0+9kwFNgy1meUDceBjQD3wMrDQ3Wu7bPM3wDnu/kUzuxz4tLt/9mj7Pd5hqH/9WgM/X13Pj646jzw1FosclYahzm3ZNAz1+cAmd38jXcQKYD7QdXqf+cA30o8fBb5nZuYBpNPHzxzPx888sQYVEZHBKMhLQxOAt7ss16fX9biNu7cDe4EjuvWY2XVmVmNmNY2NjQGVKyISTTnRWOzuy9y92t2ry8rKwi5HJBJybfZCSTmef7cgg2ArMLHLcnl6XY/bmNkQYCSwK8CaRCQDRUVF7Nq1S2GQY9ydXbt2UVRU1KfXBdlG8DJQaWaTSP3Bvxy4ots2TwL/C/gDsAB4Noj2ARHpm/Lycurr69Gl2NxTVFREeXl5n14TWBC4e7uZfRl4GsgHfurur5nZXUCNuz8J3A/8m5ltAt4hFRYiErKCggImTZoUdhkyQAK9s9jdVwIru61b0uXxPuAzQdYgIiJHlxONxSIiEhwFgYhIxAV2Z3FQzKwRON4BNkqBnf1YTq7T53E4fR5/ps/icIPh8zjV3Xvsf59zQXAizKymt1uso0ifx+H0efyZPovDDfbPQ5eGREQiTkEgIhJxUQuCZWEXkGX0eRxOn8ef6bM43KD+PCLVRiAiIkeK2hmBiIh0oyAQEYm4yATBsabNjAozm2hmq8ys1sxeM7Mbw64pG5hZvpmtNbP/F3YtYTOzUWb2qJn9ycw2mtkHwq4pLGZ2c/r3ZIOZPWxmfRvWM0dEIgjS02YuBeYCVcBCM6sKt6rQtAN/6+5VwAXAlyL8WXR1I7Ax7CKyxH3Ar9x9KnAuEf1czGwCcANQ7e5nkRo8c1AOjBmJIKDLtJnu3gZ0TJsZOe6+3d3XpB8nSf2Sd585LlLMrBz4n8BPwq4lbGY2EvgQqZGBcfc2d98TblWhGgIMS8+XMhzYFnI9gYhKEGQybWbkmFkFMB14KdxKQncv8L+BQ2EXkgUmAY3AA+lLZT8xs+KwiwqDu28Fvg28BWwH9rr7r8OtKhhRCQLpxsxGAI8BN7n7u2HXExYz+ySww91Xh11LlhgCzAB+4O7TgWYgkm1qZjaa1JWDScB7gWIzuyrcqoIRlSDIZNrMyDCzAlIhsNzd/zPsekI2E5hnZltIXTL8iJn9e7glhaoeqHf3jrPER0kFQxR9FNjs7o3ufgD4T+CDIdcUiKgEQee0mWZWSKrB58mQawqFmRmp678b3f2esOsJm7v/vbuXu3sFqf8vnnX3QfmtLxPu3gC8bWZT0qsuBmpDLClMbwEXmNnw9O/NxQzShvNAZyjLFr1NmxlyWWGZCXwOeNXMXkmv+2p6NjkRgOuB5ekvTW8A14RcTyjc/SUzexRYQ6q33VoG6VATGmJCRCTionJpSEREeqEgEBGJOAWBiEjEKQhERCJOQSAiEnEKApFuzOygmb3S5aff7qw1swoz29Bf+xPpD5G4j0Ckj1rdfVrYRYgMFJ0RiEZEYP0AAAFmSURBVGTIzLaY2b+Y2atm9kczOz29vsLMnjWz9Wb2GzM7Jb3+PWb2uJmtS/90DE+Qb2Y/To9z/2szGxbaQYmgIBDpybBul4Y+2+W5ve5+NvA9UqOWAnwX+Fd3PwdYDnwnvf47wH+7+7mkxuvpuJu9Eljq7mcCe4C/DPh4RI5KdxaLdGNmTe4+oof1W4CPuPsb6YH7Gtx9rJntBE529wPp9dvdvdTMGoFyd9/fZR8VwDPuXplevhUocPdvBn9kIj3TGYFI33gvj/tif5fHB1FbnYRMQSDSN5/t8t8/pB//nj9PYXgl8Hz68W+AxdA5J/LIgSpSpC/0TUTkSMO6jMwKqfl7O7qQjjaz9aS+1S9Mr7ue1Ixet5Ca3atjtM4bgWVm9nlS3/wXk5rpSiSrqI1AJEPpNoJqd98Zdi0i/UmXhkREIk5nBCIiEaczAhGRiFMQiIhEnIJARCTiFAQiIhGnIBARibj/DxdUJqvhjmwEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# visualize model acc\n",
    "history = history.history\n",
    "plt.plot(history['accuracy'])\n",
    "plt.plot(history['val_accuracy'])\n",
    "plt.ylabel('Accracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Training data', 'Validation data'], loc=0)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}