Evaluating Performance of Deep Learning Models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "periodic-electric",
   "metadata": {},
   "source": [
    "# Few ways to evaluate model performance using keras\n",
    "1. Using an automatic verification dataset\n",
    "2. Using a manual verification dataset\n",
    "3. Using a k-fold cross validation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "polish-western",
   "metadata": {},
   "source": [
    "## 1. Using a Automatic Verification Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "defensive-ferry",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/150\n",
      "52/52 [==============================] - 1s 8ms/step - loss: 3.2024 - accuracy: 0.5059 - val_loss: 1.4554 - val_accuracy: 0.5630\n",
      "Epoch 2/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 1.7146 - accuracy: 0.5270 - val_loss: 1.2068 - val_accuracy: 0.5709\n",
      "Epoch 3/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 1.3252 - accuracy: 0.6079 - val_loss: 1.1104 - val_accuracy: 0.6102\n",
      "Epoch 4/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 1.3701 - accuracy: 0.5825 - val_loss: 1.0143 - val_accuracy: 0.5906\n",
      "Epoch 5/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 1.1676 - accuracy: 0.6266 - val_loss: 0.9963 - val_accuracy: 0.5748\n",
      "Epoch 6/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 1.0063 - accuracy: 0.6115 - val_loss: 1.0297 - val_accuracy: 0.6772\n",
      "Epoch 7/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.9980 - accuracy: 0.6351 - val_loss: 0.9051 - val_accuracy: 0.6299\n",
      "Epoch 8/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 1.0795 - accuracy: 0.6201 - val_loss: 0.9571 - val_accuracy: 0.5748\n",
      "Epoch 9/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.9113 - accuracy: 0.6436 - val_loss: 0.8557 - val_accuracy: 0.6339\n",
      "Epoch 10/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.8755 - accuracy: 0.6497 - val_loss: 0.8565 - val_accuracy: 0.6457\n",
      "Epoch 11/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.8391 - accuracy: 0.6640 - val_loss: 0.8945 - val_accuracy: 0.6654\n",
      "Epoch 12/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.8331 - accuracy: 0.6489 - val_loss: 0.7431 - val_accuracy: 0.6339\n",
      "Epoch 13/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.7772 - accuracy: 0.6815 - val_loss: 0.8000 - val_accuracy: 0.6693\n",
      "Epoch 14/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.8243 - accuracy: 0.6437 - val_loss: 0.7313 - val_accuracy: 0.6142\n",
      "Epoch 15/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.7142 - accuracy: 0.6678 - val_loss: 0.7713 - val_accuracy: 0.6063\n",
      "Epoch 16/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.7687 - accuracy: 0.6427 - val_loss: 0.6880 - val_accuracy: 0.6496\n",
      "Epoch 17/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6453 - accuracy: 0.7241 - val_loss: 0.6976 - val_accuracy: 0.6378\n",
      "Epoch 18/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.7944 - accuracy: 0.6574 - val_loss: 0.6897 - val_accuracy: 0.6535\n",
      "Epoch 19/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6729 - accuracy: 0.7119 - val_loss: 0.7495 - val_accuracy: 0.5945\n",
      "Epoch 20/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6501 - accuracy: 0.6756 - val_loss: 0.6720 - val_accuracy: 0.6378\n",
      "Epoch 21/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6778 - accuracy: 0.6581 - val_loss: 0.6700 - val_accuracy: 0.6654\n",
      "Epoch 22/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6190 - accuracy: 0.7031 - val_loss: 0.6467 - val_accuracy: 0.6614\n",
      "Epoch 23/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6200 - accuracy: 0.7234 - val_loss: 0.6376 - val_accuracy: 0.6850\n",
      "Epoch 24/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6201 - accuracy: 0.7043 - val_loss: 0.7030 - val_accuracy: 0.6850\n",
      "Epoch 25/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6438 - accuracy: 0.6772 - val_loss: 0.7516 - val_accuracy: 0.6654\n",
      "Epoch 26/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6774 - accuracy: 0.6940 - val_loss: 0.7344 - val_accuracy: 0.6693\n",
      "Epoch 27/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6090 - accuracy: 0.7027 - val_loss: 0.7176 - val_accuracy: 0.6181\n",
      "Epoch 28/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6124 - accuracy: 0.7030 - val_loss: 0.6842 - val_accuracy: 0.6693\n",
      "Epoch 29/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.6062 - accuracy: 0.7120 - val_loss: 0.6232 - val_accuracy: 0.6654\n",
      "Epoch 30/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6048 - accuracy: 0.7234 - val_loss: 0.6744 - val_accuracy: 0.6890\n",
      "Epoch 31/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6055 - accuracy: 0.7183 - val_loss: 0.6263 - val_accuracy: 0.6772\n",
      "Epoch 32/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6216 - accuracy: 0.6983 - val_loss: 0.6215 - val_accuracy: 0.6614\n",
      "Epoch 33/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6211 - accuracy: 0.6795 - val_loss: 0.6256 - val_accuracy: 0.6969\n",
      "Epoch 34/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.6339 - accuracy: 0.6725 - val_loss: 0.6113 - val_accuracy: 0.6772\n",
      "Epoch 35/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5799 - accuracy: 0.6958 - val_loss: 0.6332 - val_accuracy: 0.6929\n",
      "Epoch 36/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.6573 - accuracy: 0.6840 - val_loss: 0.6293 - val_accuracy: 0.6732\n",
      "Epoch 37/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5808 - accuracy: 0.7434 - val_loss: 0.6385 - val_accuracy: 0.6850\n",
      "Epoch 38/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5628 - accuracy: 0.7320 - val_loss: 0.6390 - val_accuracy: 0.6811\n",
      "Epoch 39/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5775 - accuracy: 0.7424 - val_loss: 0.6440 - val_accuracy: 0.6654\n",
      "Epoch 40/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5969 - accuracy: 0.7071 - val_loss: 0.6185 - val_accuracy: 0.6929\n",
      "Epoch 41/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5804 - accuracy: 0.7181 - val_loss: 0.6706 - val_accuracy: 0.6535\n",
      "Epoch 42/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6254 - accuracy: 0.7037 - val_loss: 0.6224 - val_accuracy: 0.6732\n",
      "Epoch 43/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5951 - accuracy: 0.7118 - val_loss: 0.6064 - val_accuracy: 0.6969\n",
      "Epoch 44/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5935 - accuracy: 0.7147 - val_loss: 0.6173 - val_accuracy: 0.6654\n",
      "Epoch 45/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5934 - accuracy: 0.7113 - val_loss: 0.7636 - val_accuracy: 0.6614\n",
      "Epoch 46/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5881 - accuracy: 0.7120 - val_loss: 0.7630 - val_accuracy: 0.6811\n",
      "Epoch 47/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6295 - accuracy: 0.7128 - val_loss: 0.6892 - val_accuracy: 0.6614\n",
      "Epoch 48/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.6481 - accuracy: 0.7233 - val_loss: 0.6050 - val_accuracy: 0.6969\n",
      "Epoch 49/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5327 - accuracy: 0.7457 - val_loss: 0.5964 - val_accuracy: 0.6969\n",
      "Epoch 50/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5477 - accuracy: 0.7653 - val_loss: 0.6055 - val_accuracy: 0.6890\n",
      "Epoch 51/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5429 - accuracy: 0.7312 - val_loss: 0.5968 - val_accuracy: 0.6850\n",
      "Epoch 52/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5093 - accuracy: 0.7551 - val_loss: 0.6080 - val_accuracy: 0.6693\n",
      "Epoch 53/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5609 - accuracy: 0.7147 - val_loss: 0.6370 - val_accuracy: 0.6496\n",
      "Epoch 54/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5925 - accuracy: 0.6939 - val_loss: 0.6057 - val_accuracy: 0.6772\n",
      "Epoch 55/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5441 - accuracy: 0.7494 - val_loss: 0.5957 - val_accuracy: 0.6890\n",
      "Epoch 56/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5124 - accuracy: 0.7357 - val_loss: 0.5930 - val_accuracy: 0.7087\n",
      "Epoch 57/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5171 - accuracy: 0.7733 - val_loss: 0.6763 - val_accuracy: 0.6850\n",
      "Epoch 58/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5481 - accuracy: 0.7253 - val_loss: 0.6068 - val_accuracy: 0.6890\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 59/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5696 - accuracy: 0.7328 - val_loss: 0.6021 - val_accuracy: 0.6969\n",
      "Epoch 60/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5262 - accuracy: 0.7389 - val_loss: 0.6849 - val_accuracy: 0.6732\n",
      "Epoch 61/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5889 - accuracy: 0.7113 - val_loss: 0.5949 - val_accuracy: 0.6929\n",
      "Epoch 62/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5320 - accuracy: 0.7427 - val_loss: 0.6479 - val_accuracy: 0.6969\n",
      "Epoch 63/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5131 - accuracy: 0.7523 - val_loss: 0.6087 - val_accuracy: 0.6929\n",
      "Epoch 64/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5572 - accuracy: 0.7338 - val_loss: 0.6253 - val_accuracy: 0.6811\n",
      "Epoch 65/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5102 - accuracy: 0.7606 - val_loss: 0.7097 - val_accuracy: 0.6890\n",
      "Epoch 66/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5886 - accuracy: 0.6977 - val_loss: 0.5829 - val_accuracy: 0.6969\n",
      "Epoch 67/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5188 - accuracy: 0.7451 - val_loss: 0.6259 - val_accuracy: 0.7087\n",
      "Epoch 68/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5284 - accuracy: 0.7646 - val_loss: 0.5835 - val_accuracy: 0.7165\n",
      "Epoch 69/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4955 - accuracy: 0.7651 - val_loss: 0.6631 - val_accuracy: 0.6732\n",
      "Epoch 70/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5170 - accuracy: 0.7493 - val_loss: 0.5970 - val_accuracy: 0.6929\n",
      "Epoch 71/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5225 - accuracy: 0.7357 - val_loss: 0.6184 - val_accuracy: 0.6811\n",
      "Epoch 72/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5277 - accuracy: 0.7281 - val_loss: 0.6159 - val_accuracy: 0.6969\n",
      "Epoch 73/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5330 - accuracy: 0.7474 - val_loss: 0.7103 - val_accuracy: 0.6732\n",
      "Epoch 74/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5645 - accuracy: 0.7048 - val_loss: 0.6513 - val_accuracy: 0.6850\n",
      "Epoch 75/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6686 - accuracy: 0.7230 - val_loss: 0.7298 - val_accuracy: 0.6654\n",
      "Epoch 76/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5597 - accuracy: 0.7270 - val_loss: 0.6099 - val_accuracy: 0.7165\n",
      "Epoch 77/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5185 - accuracy: 0.7342 - val_loss: 0.5797 - val_accuracy: 0.6969\n",
      "Epoch 78/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5116 - accuracy: 0.7547 - val_loss: 0.5864 - val_accuracy: 0.6772\n",
      "Epoch 79/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5258 - accuracy: 0.7335 - val_loss: 0.6098 - val_accuracy: 0.6693\n",
      "Epoch 80/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5158 - accuracy: 0.7688 - val_loss: 0.5991 - val_accuracy: 0.7087\n",
      "Epoch 81/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4551 - accuracy: 0.8011 - val_loss: 0.5856 - val_accuracy: 0.7283\n",
      "Epoch 82/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5685 - accuracy: 0.7124 - val_loss: 0.5747 - val_accuracy: 0.7323\n",
      "Epoch 83/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5582 - accuracy: 0.7410 - val_loss: 0.6120 - val_accuracy: 0.6811\n",
      "Epoch 84/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5388 - accuracy: 0.7133 - val_loss: 0.5654 - val_accuracy: 0.7283\n",
      "Epoch 85/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5027 - accuracy: 0.7546 - val_loss: 0.5946 - val_accuracy: 0.6772\n",
      "Epoch 86/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5271 - accuracy: 0.7387 - val_loss: 0.6475 - val_accuracy: 0.6890\n",
      "Epoch 87/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.6450 - accuracy: 0.7117 - val_loss: 0.5509 - val_accuracy: 0.7402\n",
      "Epoch 88/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5813 - accuracy: 0.6993 - val_loss: 0.5806 - val_accuracy: 0.6772\n",
      "Epoch 89/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5855 - accuracy: 0.7197 - val_loss: 0.6098 - val_accuracy: 0.7205\n",
      "Epoch 90/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5348 - accuracy: 0.7241 - val_loss: 0.5614 - val_accuracy: 0.7244\n",
      "Epoch 91/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5078 - accuracy: 0.7689 - val_loss: 0.5710 - val_accuracy: 0.7362\n",
      "Epoch 92/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.4994 - accuracy: 0.7406 - val_loss: 0.6023 - val_accuracy: 0.7205\n",
      "Epoch 93/150\n",
      "52/52 [==============================] - 0s 5ms/step - loss: 0.5323 - accuracy: 0.7363 - val_loss: 0.5735 - val_accuracy: 0.7244\n",
      "Epoch 94/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5448 - accuracy: 0.7101 - val_loss: 0.5594 - val_accuracy: 0.7165\n",
      "Epoch 95/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5475 - accuracy: 0.7609 - val_loss: 0.6155 - val_accuracy: 0.7165\n",
      "Epoch 96/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5135 - accuracy: 0.7510 - val_loss: 0.5769 - val_accuracy: 0.7126\n",
      "Epoch 97/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5354 - accuracy: 0.7404 - val_loss: 0.5611 - val_accuracy: 0.7244\n",
      "Epoch 98/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5114 - accuracy: 0.7515 - val_loss: 0.5689 - val_accuracy: 0.7362\n",
      "Epoch 99/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5163 - accuracy: 0.7547 - val_loss: 0.5635 - val_accuracy: 0.7244\n",
      "Epoch 100/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4601 - accuracy: 0.7935 - val_loss: 0.5726 - val_accuracy: 0.7126\n",
      "Epoch 101/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5662 - accuracy: 0.7428 - val_loss: 0.5596 - val_accuracy: 0.7244\n",
      "Epoch 102/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.4933 - accuracy: 0.7774 - val_loss: 0.6509 - val_accuracy: 0.6929\n",
      "Epoch 103/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5944 - accuracy: 0.7474 - val_loss: 0.6665 - val_accuracy: 0.6693\n",
      "Epoch 104/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5302 - accuracy: 0.7504 - val_loss: 0.5625 - val_accuracy: 0.7441\n",
      "Epoch 105/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.4798 - accuracy: 0.7779 - val_loss: 0.5876 - val_accuracy: 0.7087\n",
      "Epoch 106/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5169 - accuracy: 0.7623 - val_loss: 0.5573 - val_accuracy: 0.7677\n",
      "Epoch 107/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.4943 - accuracy: 0.7764 - val_loss: 0.5681 - val_accuracy: 0.7480\n",
      "Epoch 108/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.4947 - accuracy: 0.7467 - val_loss: 0.5647 - val_accuracy: 0.7638\n",
      "Epoch 109/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5709 - accuracy: 0.7072 - val_loss: 0.5688 - val_accuracy: 0.7441\n",
      "Epoch 110/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5386 - accuracy: 0.7332 - val_loss: 0.5688 - val_accuracy: 0.7205\n",
      "Epoch 111/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5284 - accuracy: 0.7070 - val_loss: 0.6003 - val_accuracy: 0.6929\n",
      "Epoch 112/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4996 - accuracy: 0.7493 - val_loss: 0.5666 - val_accuracy: 0.7520\n",
      "Epoch 113/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4846 - accuracy: 0.7641 - val_loss: 0.6043 - val_accuracy: 0.7323\n",
      "Epoch 114/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5223 - accuracy: 0.7468 - val_loss: 0.6001 - val_accuracy: 0.7441\n",
      "Epoch 115/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5032 - accuracy: 0.7306 - val_loss: 0.7373 - val_accuracy: 0.6339\n",
      "Epoch 116/150\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5311 - accuracy: 0.7289 - val_loss: 0.5641 - val_accuracy: 0.7559\n",
      "Epoch 117/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4763 - accuracy: 0.7676 - val_loss: 0.5689 - val_accuracy: 0.7441\n",
      "Epoch 118/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5263 - accuracy: 0.7331 - val_loss: 0.5641 - val_accuracy: 0.7205\n",
      "Epoch 119/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5379 - accuracy: 0.7347 - val_loss: 0.6041 - val_accuracy: 0.7165\n",
      "Epoch 120/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5379 - accuracy: 0.7270 - val_loss: 0.5762 - val_accuracy: 0.7441\n",
      "Epoch 121/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5226 - accuracy: 0.7788 - val_loss: 0.5624 - val_accuracy: 0.7441\n",
      "Epoch 122/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4533 - accuracy: 0.7796 - val_loss: 0.6090 - val_accuracy: 0.7244\n",
      "Epoch 123/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5045 - accuracy: 0.7447 - val_loss: 0.5685 - val_accuracy: 0.7087\n",
      "Epoch 124/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5043 - accuracy: 0.7770 - val_loss: 0.6124 - val_accuracy: 0.6969\n",
      "Epoch 125/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5097 - accuracy: 0.7530 - val_loss: 0.5662 - val_accuracy: 0.7244\n",
      "Epoch 126/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5282 - accuracy: 0.7350 - val_loss: 0.5574 - val_accuracy: 0.7520\n",
      "Epoch 127/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4895 - accuracy: 0.7334 - val_loss: 0.5671 - val_accuracy: 0.7402\n",
      "Epoch 128/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4693 - accuracy: 0.7834 - val_loss: 0.5782 - val_accuracy: 0.6969\n",
      "Epoch 129/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4970 - accuracy: 0.7461 - val_loss: 0.5477 - val_accuracy: 0.7244\n",
      "Epoch 130/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5037 - accuracy: 0.7567 - val_loss: 0.5654 - val_accuracy: 0.7402\n",
      "Epoch 131/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5082 - accuracy: 0.7482 - val_loss: 0.5701 - val_accuracy: 0.7244\n",
      "Epoch 132/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5113 - accuracy: 0.7390 - val_loss: 0.6095 - val_accuracy: 0.6929\n",
      "Epoch 133/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.4620 - accuracy: 0.7882 - val_loss: 0.5477 - val_accuracy: 0.7638\n",
      "Epoch 134/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.4655 - accuracy: 0.7735 - val_loss: 0.5394 - val_accuracy: 0.7205\n",
      "Epoch 135/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5069 - accuracy: 0.7482 - val_loss: 0.6040 - val_accuracy: 0.7402\n",
      "Epoch 136/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5692 - accuracy: 0.7092 - val_loss: 0.5431 - val_accuracy: 0.7559\n",
      "Epoch 137/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4716 - accuracy: 0.7887 - val_loss: 0.5563 - val_accuracy: 0.7362\n",
      "Epoch 138/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4822 - accuracy: 0.7508 - val_loss: 0.5440 - val_accuracy: 0.7323\n",
      "Epoch 139/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4896 - accuracy: 0.7550 - val_loss: 0.5555 - val_accuracy: 0.7559\n",
      "Epoch 140/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5060 - accuracy: 0.7523 - val_loss: 0.6530 - val_accuracy: 0.7047\n",
      "Epoch 141/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5807 - accuracy: 0.6926 - val_loss: 0.5427 - val_accuracy: 0.7441\n",
      "Epoch 142/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4655 - accuracy: 0.7973 - val_loss: 0.6027 - val_accuracy: 0.6890\n",
      "Epoch 143/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5499 - accuracy: 0.7460 - val_loss: 0.5459 - val_accuracy: 0.7283\n",
      "Epoch 144/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4633 - accuracy: 0.7660 - val_loss: 0.5497 - val_accuracy: 0.7244\n",
      "Epoch 145/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5126 - accuracy: 0.7301 - val_loss: 0.5734 - val_accuracy: 0.7441\n",
      "Epoch 146/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4642 - accuracy: 0.7737 - val_loss: 0.5712 - val_accuracy: 0.7677\n",
      "Epoch 147/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4695 - accuracy: 0.7505 - val_loss: 0.5423 - val_accuracy: 0.7441\n",
      "Epoch 148/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4958 - accuracy: 0.7620 - val_loss: 0.5454 - val_accuracy: 0.7480\n",
      "Epoch 149/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4731 - accuracy: 0.7893 - val_loss: 0.5720 - val_accuracy: 0.7323\n",
      "Epoch 150/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4955 - accuracy: 0.7589 - val_loss: 0.5805 - val_accuracy: 0.7598\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fa04fafea50>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# MLP with automatic validation dataset\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "import numpy\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 7\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# load pima indians dataset\n",
    "dataset = numpy.loadtxt('pima-indians-diabetes.data.csv',delimiter=\",\")\n",
    "\n",
    "# split into input(X) and output(Y) variables\n",
    "X = dataset[:,0:8]\n",
    "Y = dataset[:,8]\n",
    "\n",
    "# create model\n",
    "model = Sequential()\n",
    "model.add(Dense(12,input_dim=8,activation='relu'))\n",
    "model.add(Dense(8,activation='relu'))\n",
    "model.add(Dense(1,activation='sigmoid'))\n",
    "\n",
    "# compile model\n",
    "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n",
    "\n",
    "# Fit the model\n",
    "model.fit(X,Y,validation_split=0.33,epochs=150,batch_size=10)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "thermal-kansas",
   "metadata": {},
   "source": [
    "## 2. Using a Manual Verification Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "applied-negotiation",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/150\n",
      "52/52 [==============================] - 0s 3ms/step - loss: 3.8902 - accuracy: 0.5403 - val_loss: 2.4088 - val_accuracy: 0.5118\n",
      "Epoch 2/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 2.0698 - accuracy: 0.5067 - val_loss: 1.4675 - val_accuracy: 0.5630\n",
      "Epoch 3/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 1.1825 - accuracy: 0.5822 - val_loss: 1.2050 - val_accuracy: 0.5512\n",
      "Epoch 4/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.9207 - accuracy: 0.6414 - val_loss: 1.0366 - val_accuracy: 0.5945\n",
      "Epoch 5/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 1.0039 - accuracy: 0.6336 - val_loss: 1.0545 - val_accuracy: 0.6102\n",
      "Epoch 6/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.7972 - accuracy: 0.7017 - val_loss: 0.8823 - val_accuracy: 0.5945\n",
      "Epoch 7/150\n",
      "52/52 [==============================] - 0s 7ms/step - loss: 0.7615 - accuracy: 0.7095 - val_loss: 0.8408 - val_accuracy: 0.6378\n",
      "Epoch 8/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6673 - accuracy: 0.6827 - val_loss: 0.8967 - val_accuracy: 0.6417\n",
      "Epoch 9/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6594 - accuracy: 0.6885 - val_loss: 0.7846 - val_accuracy: 0.6417\n",
      "Epoch 10/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6218 - accuracy: 0.6869 - val_loss: 0.8235 - val_accuracy: 0.6260\n",
      "Epoch 11/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6262 - accuracy: 0.6794 - val_loss: 0.8030 - val_accuracy: 0.6299\n",
      "Epoch 12/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6386 - accuracy: 0.6819 - val_loss: 0.7247 - val_accuracy: 0.6142\n",
      "Epoch 13/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.7123 - accuracy: 0.6579 - val_loss: 0.6802 - val_accuracy: 0.6496\n",
      "Epoch 14/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6119 - accuracy: 0.6830 - val_loss: 0.7143 - val_accuracy: 0.6378\n",
      "Epoch 15/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6085 - accuracy: 0.6812 - val_loss: 0.6701 - val_accuracy: 0.6732\n",
      "Epoch 16/150\n",
      "52/52 [==============================] - 0s 3ms/step - loss: 0.5219 - accuracy: 0.7402 - val_loss: 0.8077 - val_accuracy: 0.6339\n",
      "Epoch 17/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5734 - accuracy: 0.7130 - val_loss: 0.8956 - val_accuracy: 0.5827\n",
      "Epoch 18/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5992 - accuracy: 0.7173 - val_loss: 0.7996 - val_accuracy: 0.6063\n",
      "Epoch 19/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5844 - accuracy: 0.7149 - val_loss: 0.8260 - val_accuracy: 0.5984\n",
      "Epoch 20/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6292 - accuracy: 0.7224 - val_loss: 0.6653 - val_accuracy: 0.6654\n",
      "Epoch 21/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5535 - accuracy: 0.7198 - val_loss: 0.7402 - val_accuracy: 0.6378\n",
      "Epoch 22/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6563 - accuracy: 0.6696 - val_loss: 0.7201 - val_accuracy: 0.6142\n",
      "Epoch 23/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5875 - accuracy: 0.6827 - val_loss: 0.6774 - val_accuracy: 0.6496\n",
      "Epoch 24/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6002 - accuracy: 0.6828 - val_loss: 0.6659 - val_accuracy: 0.6496\n",
      "Epoch 25/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5331 - accuracy: 0.7423 - val_loss: 0.7424 - val_accuracy: 0.6457\n",
      "Epoch 26/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5911 - accuracy: 0.7073 - val_loss: 0.6897 - val_accuracy: 0.6260\n",
      "Epoch 27/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.7683 - accuracy: 0.6703 - val_loss: 0.6489 - val_accuracy: 0.6772\n",
      "Epoch 28/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5478 - accuracy: 0.7468 - val_loss: 0.7070 - val_accuracy: 0.6535\n",
      "Epoch 29/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6203 - accuracy: 0.7158 - val_loss: 0.8016 - val_accuracy: 0.6654\n",
      "Epoch 30/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5492 - accuracy: 0.7457 - val_loss: 0.6715 - val_accuracy: 0.6457\n",
      "Epoch 31/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5964 - accuracy: 0.6939 - val_loss: 0.7128 - val_accuracy: 0.6496\n",
      "Epoch 32/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6337 - accuracy: 0.7125 - val_loss: 0.7655 - val_accuracy: 0.6378\n",
      "Epoch 33/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5765 - accuracy: 0.7082 - val_loss: 0.6470 - val_accuracy: 0.6496\n",
      "Epoch 34/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5851 - accuracy: 0.7191 - val_loss: 0.8433 - val_accuracy: 0.6575\n",
      "Epoch 35/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6917 - accuracy: 0.6741 - val_loss: 0.6379 - val_accuracy: 0.6772\n",
      "Epoch 36/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5535 - accuracy: 0.7198 - val_loss: 0.8914 - val_accuracy: 0.6378\n",
      "Epoch 37/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6654 - accuracy: 0.6946 - val_loss: 0.6325 - val_accuracy: 0.6772\n",
      "Epoch 38/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5561 - accuracy: 0.7428 - val_loss: 0.8195 - val_accuracy: 0.6260\n",
      "Epoch 39/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6475 - accuracy: 0.7146 - val_loss: 0.6396 - val_accuracy: 0.6575\n",
      "Epoch 40/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5865 - accuracy: 0.6996 - val_loss: 0.6352 - val_accuracy: 0.7047\n",
      "Epoch 41/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5629 - accuracy: 0.7478 - val_loss: 0.6488 - val_accuracy: 0.6654\n",
      "Epoch 42/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5690 - accuracy: 0.7134 - val_loss: 0.6246 - val_accuracy: 0.6850\n",
      "Epoch 43/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5982 - accuracy: 0.7056 - val_loss: 0.7151 - val_accuracy: 0.6535\n",
      "Epoch 44/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5813 - accuracy: 0.7214 - val_loss: 0.6521 - val_accuracy: 0.6654\n",
      "Epoch 45/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5392 - accuracy: 0.7449 - val_loss: 0.6396 - val_accuracy: 0.6850\n",
      "Epoch 46/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5515 - accuracy: 0.7344 - val_loss: 0.7283 - val_accuracy: 0.6457\n",
      "Epoch 47/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5838 - accuracy: 0.7016 - val_loss: 0.7712 - val_accuracy: 0.6220\n",
      "Epoch 48/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5229 - accuracy: 0.7583 - val_loss: 0.6268 - val_accuracy: 0.6772\n",
      "Epoch 49/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5785 - accuracy: 0.7062 - val_loss: 0.6675 - val_accuracy: 0.6732\n",
      "Epoch 50/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5714 - accuracy: 0.7484 - val_loss: 0.6733 - val_accuracy: 0.6457\n",
      "Epoch 51/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5322 - accuracy: 0.7462 - val_loss: 0.6239 - val_accuracy: 0.6890\n",
      "Epoch 52/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6033 - accuracy: 0.7087 - val_loss: 0.7552 - val_accuracy: 0.6693\n",
      "Epoch 53/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5878 - accuracy: 0.7496 - val_loss: 0.6160 - val_accuracy: 0.7008\n",
      "Epoch 54/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5922 - accuracy: 0.7412 - val_loss: 0.6287 - val_accuracy: 0.6969\n",
      "Epoch 55/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5082 - accuracy: 0.7551 - val_loss: 0.6667 - val_accuracy: 0.6654\n",
      "Epoch 56/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5771 - accuracy: 0.7260 - val_loss: 0.9162 - val_accuracy: 0.6654\n",
      "Epoch 57/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.6523 - accuracy: 0.7001 - val_loss: 0.6125 - val_accuracy: 0.7402\n",
      "Epoch 58/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5377 - accuracy: 0.7268 - val_loss: 0.6871 - val_accuracy: 0.6614\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 59/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5168 - accuracy: 0.7548 - val_loss: 0.6195 - val_accuracy: 0.7008\n",
      "Epoch 60/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5435 - accuracy: 0.7134 - val_loss: 0.6394 - val_accuracy: 0.6575\n",
      "Epoch 61/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5772 - accuracy: 0.7242 - val_loss: 0.7495 - val_accuracy: 0.6575\n",
      "Epoch 62/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5559 - accuracy: 0.7372 - val_loss: 0.6183 - val_accuracy: 0.7165\n",
      "Epoch 63/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5460 - accuracy: 0.7392 - val_loss: 0.6457 - val_accuracy: 0.6614\n",
      "Epoch 64/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5650 - accuracy: 0.7459 - val_loss: 0.6403 - val_accuracy: 0.7126\n",
      "Epoch 65/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5641 - accuracy: 0.7363 - val_loss: 0.6883 - val_accuracy: 0.6535\n",
      "Epoch 66/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5206 - accuracy: 0.7649 - val_loss: 0.6641 - val_accuracy: 0.6575\n",
      "Epoch 67/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5000 - accuracy: 0.7509 - val_loss: 0.6075 - val_accuracy: 0.7087\n",
      "Epoch 68/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5862 - accuracy: 0.7200 - val_loss: 0.9644 - val_accuracy: 0.6535\n",
      "Epoch 69/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6230 - accuracy: 0.7540 - val_loss: 0.7130 - val_accuracy: 0.6535\n",
      "Epoch 70/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5549 - accuracy: 0.7204 - val_loss: 0.6177 - val_accuracy: 0.6929\n",
      "Epoch 71/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5026 - accuracy: 0.7592 - val_loss: 0.7213 - val_accuracy: 0.6457\n",
      "Epoch 72/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5985 - accuracy: 0.7446 - val_loss: 0.6883 - val_accuracy: 0.6496\n",
      "Epoch 73/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4758 - accuracy: 0.7618 - val_loss: 0.6261 - val_accuracy: 0.6850\n",
      "Epoch 74/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5150 - accuracy: 0.7798 - val_loss: 0.6501 - val_accuracy: 0.6732\n",
      "Epoch 75/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4969 - accuracy: 0.7606 - val_loss: 0.6824 - val_accuracy: 0.6732\n",
      "Epoch 76/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5794 - accuracy: 0.7076 - val_loss: 0.6195 - val_accuracy: 0.7008\n",
      "Epoch 77/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5354 - accuracy: 0.7562 - val_loss: 0.6184 - val_accuracy: 0.7205\n",
      "Epoch 78/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5208 - accuracy: 0.7696 - val_loss: 0.6092 - val_accuracy: 0.7165\n",
      "Epoch 79/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5274 - accuracy: 0.7715 - val_loss: 0.6002 - val_accuracy: 0.7323\n",
      "Epoch 80/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5060 - accuracy: 0.7740 - val_loss: 0.6093 - val_accuracy: 0.6969\n",
      "Epoch 81/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5121 - accuracy: 0.7701 - val_loss: 0.8113 - val_accuracy: 0.6732\n",
      "Epoch 82/150\n",
      "52/52 [==============================] - 0s 3ms/step - loss: 0.5353 - accuracy: 0.7523 - val_loss: 0.6198 - val_accuracy: 0.6890\n",
      "Epoch 83/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4981 - accuracy: 0.7383 - val_loss: 0.9315 - val_accuracy: 0.6378\n",
      "Epoch 84/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6022 - accuracy: 0.7188 - val_loss: 0.6071 - val_accuracy: 0.7047\n",
      "Epoch 85/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5140 - accuracy: 0.7316 - val_loss: 0.6134 - val_accuracy: 0.6850\n",
      "Epoch 86/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5094 - accuracy: 0.7284 - val_loss: 0.6200 - val_accuracy: 0.7047\n",
      "Epoch 87/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5296 - accuracy: 0.7482 - val_loss: 0.6939 - val_accuracy: 0.6811\n",
      "Epoch 88/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5564 - accuracy: 0.7442 - val_loss: 0.6130 - val_accuracy: 0.6969\n",
      "Epoch 89/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5558 - accuracy: 0.7352 - val_loss: 0.8515 - val_accuracy: 0.6535\n",
      "Epoch 90/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6648 - accuracy: 0.6830 - val_loss: 0.6238 - val_accuracy: 0.6929\n",
      "Epoch 91/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4867 - accuracy: 0.7785 - val_loss: 0.6524 - val_accuracy: 0.6732\n",
      "Epoch 92/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5171 - accuracy: 0.7757 - val_loss: 0.6061 - val_accuracy: 0.7047\n",
      "Epoch 93/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4993 - accuracy: 0.7685 - val_loss: 0.7559 - val_accuracy: 0.6535\n",
      "Epoch 94/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5375 - accuracy: 0.7673 - val_loss: 0.6033 - val_accuracy: 0.6929\n",
      "Epoch 95/150\n",
      "52/52 [==============================] - 0s 6ms/step - loss: 0.4863 - accuracy: 0.7822 - val_loss: 0.6007 - val_accuracy: 0.7362\n",
      "Epoch 96/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5331 - accuracy: 0.7253 - val_loss: 0.7577 - val_accuracy: 0.6575\n",
      "Epoch 97/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5736 - accuracy: 0.7476 - val_loss: 0.5959 - val_accuracy: 0.7165\n",
      "Epoch 98/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4852 - accuracy: 0.7762 - val_loss: 0.6917 - val_accuracy: 0.6969\n",
      "Epoch 99/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5519 - accuracy: 0.7438 - val_loss: 0.6765 - val_accuracy: 0.6811\n",
      "Epoch 100/150\n",
      "52/52 [==============================] - 0s 4ms/step - loss: 0.5251 - accuracy: 0.7631 - val_loss: 0.8775 - val_accuracy: 0.6378\n",
      "Epoch 101/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6463 - accuracy: 0.6928 - val_loss: 0.6018 - val_accuracy: 0.6969\n",
      "Epoch 102/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4817 - accuracy: 0.7942 - val_loss: 0.6244 - val_accuracy: 0.7205\n",
      "Epoch 103/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5158 - accuracy: 0.7603 - val_loss: 0.8457 - val_accuracy: 0.6732\n",
      "Epoch 104/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5278 - accuracy: 0.7582 - val_loss: 0.5929 - val_accuracy: 0.7480\n",
      "Epoch 105/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5053 - accuracy: 0.7838 - val_loss: 0.7172 - val_accuracy: 0.6772\n",
      "Epoch 106/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5012 - accuracy: 0.7761 - val_loss: 0.6541 - val_accuracy: 0.6850\n",
      "Epoch 107/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5337 - accuracy: 0.7599 - val_loss: 0.5997 - val_accuracy: 0.7126\n",
      "Epoch 108/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5043 - accuracy: 0.7669 - val_loss: 0.7120 - val_accuracy: 0.6654\n",
      "Epoch 109/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5191 - accuracy: 0.7357 - val_loss: 0.6144 - val_accuracy: 0.7047\n",
      "Epoch 110/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4847 - accuracy: 0.7768 - val_loss: 0.6848 - val_accuracy: 0.6535\n",
      "Epoch 111/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5859 - accuracy: 0.7203 - val_loss: 0.5876 - val_accuracy: 0.7244\n",
      "Epoch 112/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4843 - accuracy: 0.7687 - val_loss: 0.6104 - val_accuracy: 0.7047\n",
      "Epoch 113/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4919 - accuracy: 0.7641 - val_loss: 0.5926 - val_accuracy: 0.7087\n",
      "Epoch 114/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5118 - accuracy: 0.7730 - val_loss: 0.6478 - val_accuracy: 0.6772\n",
      "Epoch 115/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5364 - accuracy: 0.7596 - val_loss: 0.6219 - val_accuracy: 0.6811\n",
      "Epoch 116/150\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4823 - accuracy: 0.7740 - val_loss: 0.5966 - val_accuracy: 0.7283\n",
      "Epoch 117/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5155 - accuracy: 0.7388 - val_loss: 0.5985 - val_accuracy: 0.7244\n",
      "Epoch 118/150\n",
      "52/52 [==============================] - 0s 1ms/step - loss: 0.5225 - accuracy: 0.7542 - val_loss: 0.6016 - val_accuracy: 0.7205\n",
      "Epoch 119/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4444 - accuracy: 0.8003 - val_loss: 0.6465 - val_accuracy: 0.6654\n",
      "Epoch 120/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5004 - accuracy: 0.7505 - val_loss: 0.5902 - val_accuracy: 0.7008\n",
      "Epoch 121/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5161 - accuracy: 0.7596 - val_loss: 0.6590 - val_accuracy: 0.6811\n",
      "Epoch 122/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4613 - accuracy: 0.7822 - val_loss: 0.6203 - val_accuracy: 0.7126\n",
      "Epoch 123/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5102 - accuracy: 0.7407 - val_loss: 0.5886 - val_accuracy: 0.7205\n",
      "Epoch 124/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5389 - accuracy: 0.7907 - val_loss: 0.6263 - val_accuracy: 0.7087\n",
      "Epoch 125/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5014 - accuracy: 0.7632 - val_loss: 0.6015 - val_accuracy: 0.7244\n",
      "Epoch 126/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4579 - accuracy: 0.7755 - val_loss: 0.7449 - val_accuracy: 0.6654\n",
      "Epoch 127/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5655 - accuracy: 0.7312 - val_loss: 0.6001 - val_accuracy: 0.7244\n",
      "Epoch 128/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4876 - accuracy: 0.7668 - val_loss: 0.5902 - val_accuracy: 0.7205\n",
      "Epoch 129/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5562 - accuracy: 0.7345 - val_loss: 0.6496 - val_accuracy: 0.7008\n",
      "Epoch 130/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5282 - accuracy: 0.7702 - val_loss: 0.7391 - val_accuracy: 0.6693\n",
      "Epoch 131/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5973 - accuracy: 0.7483 - val_loss: 0.6823 - val_accuracy: 0.6654\n",
      "Epoch 132/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4489 - accuracy: 0.7791 - val_loss: 0.6398 - val_accuracy: 0.6850\n",
      "Epoch 133/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4801 - accuracy: 0.7690 - val_loss: 0.7349 - val_accuracy: 0.6614\n",
      "Epoch 134/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.6217 - accuracy: 0.7137 - val_loss: 0.6029 - val_accuracy: 0.7205\n",
      "Epoch 135/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5287 - accuracy: 0.7553 - val_loss: 0.6464 - val_accuracy: 0.7047\n",
      "Epoch 136/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4692 - accuracy: 0.8074 - val_loss: 0.5831 - val_accuracy: 0.7323\n",
      "Epoch 137/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4954 - accuracy: 0.7551 - val_loss: 0.6108 - val_accuracy: 0.6890\n",
      "Epoch 138/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4705 - accuracy: 0.7684 - val_loss: 0.6105 - val_accuracy: 0.7126\n",
      "Epoch 139/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5495 - accuracy: 0.7406 - val_loss: 0.6045 - val_accuracy: 0.7244\n",
      "Epoch 140/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4316 - accuracy: 0.7938 - val_loss: 0.7311 - val_accuracy: 0.6732\n",
      "Epoch 141/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4710 - accuracy: 0.7518 - val_loss: 0.6007 - val_accuracy: 0.7244\n",
      "Epoch 142/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5032 - accuracy: 0.7597 - val_loss: 0.5969 - val_accuracy: 0.7205\n",
      "Epoch 143/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4558 - accuracy: 0.7714 - val_loss: 0.5894 - val_accuracy: 0.7165\n",
      "Epoch 144/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5447 - accuracy: 0.7285 - val_loss: 0.6676 - val_accuracy: 0.6772\n",
      "Epoch 145/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5389 - accuracy: 0.7477 - val_loss: 0.6038 - val_accuracy: 0.7323\n",
      "Epoch 146/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4636 - accuracy: 0.7776 - val_loss: 0.5860 - val_accuracy: 0.7205\n",
      "Epoch 147/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4809 - accuracy: 0.7730 - val_loss: 0.5899 - val_accuracy: 0.7244\n",
      "Epoch 148/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4359 - accuracy: 0.7849 - val_loss: 0.7245 - val_accuracy: 0.6772\n",
      "Epoch 149/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.4883 - accuracy: 0.7851 - val_loss: 0.6078 - val_accuracy: 0.6969\n",
      "Epoch 150/150\n",
      "52/52 [==============================] - 0s 2ms/step - loss: 0.5404 - accuracy: 0.7158 - val_loss: 0.5886 - val_accuracy: 0.7362\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fa0347c6250>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# MLP with manual validation set\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "import numpy\n",
    "from sklearn.model_selection import train_test_split\n",
    "seed = 7\n",
    "numpy.random.seed(seed)\n",
    "dataset = numpy.loadtxt('pima-indians-diabetes.data.csv',delimiter=\",\")\n",
    "X = dataset[:,0:8]\n",
    "Y = dataset[:,8]\n",
    "X_train,X_test,y_train,y_test = train_test_split(X,Y,test_size=0.33,random_state=seed)\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(12,input_dim=8,activation='relu'))\n",
    "model.add(Dense(8,activation='relu'))\n",
    "model.add(Dense(1,activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n",
    "\n",
    "model.fit(X_train,y_train,validation_data=(X_test,y_test),epochs=150,batch_size=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "latin-pipeline",
   "metadata": {},
   "source": [
    "## 3. Using k-fold Cross Validation\n",
    "* cross validation is often not used for evaluating deep learning models because of the greater computational expense.\n",
    "* The folds are stratified,meaning that the algorithm attempts to balance the number of instances of each class in each fold."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "psychological-original",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy : 71.43\n",
      "accuracy : 77.92\n",
      "accuracy : 79.22\n",
      "WARNING:tensorflow:5 out of the last 3910 calls to <function Model.make_test_function.<locals>.test_function at 0x7f9fce793560> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "accuracy : 72.73\n",
      "WARNING:tensorflow:5 out of the last 13 calls to <function Model.make_test_function.<locals>.test_function at 0x7f9fcd6a6830> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "accuracy : 61.04\n",
      "WARNING:tensorflow:5 out of the last 13 calls to <function Model.make_test_function.<locals>.test_function at 0x7f9fcc532560> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "accuracy : 63.64\n",
      "WARNING:tensorflow:5 out of the last 13 calls to <function Model.make_test_function.<locals>.test_function at 0x7f9fcc4097a0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "accuracy : 74.03\n",
      "WARNING:tensorflow:5 out of the last 13 calls to <function Model.make_test_function.<locals>.test_function at 0x7f9fcc2aea70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "accuracy : 76.62\n",
      "WARNING:tensorflow:5 out of the last 13 calls to <function Model.make_test_function.<locals>.test_function at 0x7f9fcc0a0c20> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "accuracy : 72.37\n",
      "WARNING:tensorflow:5 out of the last 13 calls to <function Model.make_test_function.<locals>.test_function at 0x7f9fa86907a0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "accuracy : 82.89\n",
      "73.19 (+/- 6.39)\n"
     ]
    }
   ],
   "source": [
    "# MLP for pima Indians Dataset with 10-fold cross validation\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "import numpy\n",
    "\n",
    "seed = 7\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "dataset = numpy.loadtxt(\"pima-indians-diabetes.data.csv\",delimiter=\",\")\n",
    "\n",
    "X = dataset[:,0:8]\n",
    "Y = dataset[:,8]\n",
    "\n",
    "# define 10-fold cross validation test harness\n",
    "kfold = StratifiedKFold(n_splits=10,shuffle=True,random_state=seed)\n",
    "cvscores = []\n",
    "for train,test in kfold.split(X,Y):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(12,input_dim=8,activation='relu'))\n",
    "    model.add(Dense(8,activation='relu'))\n",
    "    model.add(Dense(1,activation='sigmoid'))\n",
    "    \n",
    "    model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n",
    "    \n",
    "    model.fit(X[train],Y[train],epochs=150,batch_size=10,verbose=0)\n",
    "    scores = model.evaluate(X[test],Y[test],verbose=0)\n",
    "    print(\"%s : %.2f\" %(model.metrics_names[1],scores[1]*100))\n",
    "    cvscores.append(scores[1]*100)\n",
    "    \n",
    "print(\"%.2f (+/- %.2f)\" % (numpy.mean(cvscores),numpy.std(cvscores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "chicken-worry",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}