solutions/dataEfficiency.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_openml\n",
    "from sklearn.preprocessing import LabelEncoder, StandardScaler\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "seed = 0\n",
    "np.random.seed(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fetch the jet tagging dataset from Open ML (run this only once, this can take a while)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = fetch_openml('hls4ml_lhc_jets_hlf')\n",
    "X, y = data['data'], data['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "le = LabelEncoder()\n",
    "yl = le.fit_transform(y)\n",
    "yc = to_categorical(yl, len(le.classes_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_val, X_test, y_train_val, y_test = train_test_split(X, yc, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "setGPU: Setting GPU to: 2\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from sklearn.metrics import accuracy_score\n",
    "import setGPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def History(label, h):\n",
    "    plt.plot( h.history['loss'], label ='Training Loss: {}'.format(label))\n",
    "    plt.plot( h.history['val_loss'], label = 'Validation Loss: {}'.format(label))\n",
    "    plt.plot( h.history['accuracy'], label = 'Training Accuracy: {}'.format(label))\n",
    "    plt.plot( h.history['val_accuracy'], label = 'Validation Accuracy: {}'.format(label))\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Accuracy(m):\n",
    "    pred = m.predict( X_test)\n",
    "    a = accuracy_score(np.argmax(y_test, axis=1), np.argmax(pred, axis=1))\n",
    "    print(\"Accuracy: {}: {}\".format(m.name, a))\n",
    "    return pred, a"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Construct a simple dense model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll use 3 hidden layers with 32, then 64, then 64 neurons. Each layer will use `relu` activation.\n",
    "Add an output layer with 5 neurons (one for each class), then finish with Softmax activation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import Input,Dense, Activation, BatchNormalization, Conv1D, Dropout, Reshape, Flatten\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.regularizers import l1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model(label=\"\"):\n",
    "    i = Input(shape=(16,))\n",
    "    d = Dense(32, name='fc1', activation='relu')(i)\n",
    "    d = Dense(64, name='fc2', activation='relu')(d)\n",
    "    d = Dense(64, name='fc3', activation='relu')(d)\n",
    "    o = Dense(len(le.classes_), activation='softmax', name='output', kernel_initializer='lecun_uniform', kernel_regularizer=l1(0.0001))(d)\n",
    "    model = Model(inputs=i, outputs=o, name='model_DNN{}'.format(label))\n",
    "    model.summary()\n",
    "    model.compile(optimizer=Adam(lr=0.0001), loss=['categorical_crossentropy'], metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the model\n",
    "with an increasing amount of the training dataset, reporting the performance on the fixed test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_steps = 10\n",
    "increase_factor = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_size = int(X_train_val.shape[0] / (increase_factor**(n_steps-1)))\n",
    "steps = np.asarray(range(n_steps) )\n",
    "n_samples = start_size*(increase_factor**steps)\n",
    "fractions = n_samples / max(n_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_DNN_0\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 1296 random samples\n",
      "Epoch 1/40\n",
      "1/1 [==============================] - 2s 2s/step - loss: 8.4446 - accuracy: 0.1811 - val_loss: 8.1151 - val_accuracy: 0.1821\n",
      "Epoch 2/40\n",
      "1/1 [==============================] - 0s 55ms/step - loss: 8.2906 - accuracy: 0.1811 - val_loss: 7.9637 - val_accuracy: 0.1821\n",
      "Epoch 3/40\n",
      "1/1 [==============================] - 0s 155ms/step - loss: 8.1375 - accuracy: 0.1811 - val_loss: 7.8134 - val_accuracy: 0.1821\n",
      "Epoch 4/40\n",
      "1/1 [==============================] - 0s 151ms/step - loss: 7.9854 - accuracy: 0.1811 - val_loss: 7.6643 - val_accuracy: 0.1821\n",
      "Epoch 5/40\n",
      "1/1 [==============================] - 0s 242ms/step - loss: 7.8344 - accuracy: 0.1811 - val_loss: 7.5165 - val_accuracy: 0.1821\n",
      "Epoch 6/40\n",
      "1/1 [==============================] - 0s 228ms/step - loss: 7.6844 - accuracy: 0.1811 - val_loss: 7.3697 - val_accuracy: 0.1821\n",
      "Epoch 7/40\n",
      "1/1 [==============================] - 0s 266ms/step - loss: 7.5356 - accuracy: 0.1811 - val_loss: 7.2241 - val_accuracy: 0.1821\n",
      "Epoch 8/40\n",
      "1/1 [==============================] - 0s 143ms/step - loss: 7.3878 - accuracy: 0.1811 - val_loss: 7.0796 - val_accuracy: 0.1821\n",
      "Epoch 9/40\n",
      "1/1 [==============================] - 0s 194ms/step - loss: 7.2411 - accuracy: 0.1811 - val_loss: 6.9361 - val_accuracy: 0.1821\n",
      "Epoch 10/40\n",
      "1/1 [==============================] - 0s 178ms/step - loss: 7.0956 - accuracy: 0.1811 - val_loss: 6.7938 - val_accuracy: 0.1821\n",
      "Epoch 11/40\n",
      "1/1 [==============================] - 0s 172ms/step - loss: 6.9512 - accuracy: 0.1811 - val_loss: 6.6532 - val_accuracy: 0.1821\n",
      "Epoch 12/40\n",
      "1/1 [==============================] - ETA: 0s - loss: 6.8082 - accuracy: 0.18 - 0s 178ms/step - loss: 6.8082 - accuracy: 0.1811 - val_loss: 6.5145 - val_accuracy: 0.1821\n",
      "Epoch 13/40\n",
      "1/1 [==============================] - 0s 266ms/step - loss: 6.6667 - accuracy: 0.1811 - val_loss: 6.3776 - val_accuracy: 0.1821\n",
      "Epoch 14/40\n",
      "1/1 [==============================] - 0s 268ms/step - loss: 6.5266 - accuracy: 0.1811 - val_loss: 6.2420 - val_accuracy: 0.1821\n",
      "Epoch 15/40\n",
      "1/1 [==============================] - 0s 204ms/step - loss: 6.3876 - accuracy: 0.1811 - val_loss: 6.1080 - val_accuracy: 0.1821\n",
      "Epoch 16/40\n",
      "1/1 [==============================] - 0s 308ms/step - loss: 6.2499 - accuracy: 0.1811 - val_loss: 5.9754 - val_accuracy: 0.1821\n",
      "Epoch 17/40\n",
      "1/1 [==============================] - 0s 144ms/step - loss: 6.1138 - accuracy: 0.1811 - val_loss: 5.8441 - val_accuracy: 0.1821\n",
      "Epoch 18/40\n",
      "1/1 [==============================] - 0s 134ms/step - loss: 5.9790 - accuracy: 0.1811 - val_loss: 5.7138 - val_accuracy: 0.1821\n",
      "Epoch 19/40\n",
      "1/1 [==============================] - 0s 64ms/step - loss: 5.8452 - accuracy: 0.1811 - val_loss: 5.5846 - val_accuracy: 0.1821\n",
      "Epoch 20/40\n",
      "1/1 [==============================] - 0s 134ms/step - loss: 5.7125 - accuracy: 0.1811 - val_loss: 5.4564 - val_accuracy: 0.1821\n",
      "Epoch 21/40\n",
      "1/1 [==============================] - 0s 226ms/step - loss: 5.5808 - accuracy: 0.1811 - val_loss: 5.3295 - val_accuracy: 0.1821\n",
      "Epoch 22/40\n",
      "1/1 [==============================] - 0s 189ms/step - loss: 5.4503 - accuracy: 0.1811 - val_loss: 5.2039 - val_accuracy: 0.1821\n",
      "Epoch 23/40\n",
      "1/1 [==============================] - 0s 193ms/step - loss: 5.3209 - accuracy: 0.1811 - val_loss: 5.0795 - val_accuracy: 0.1821\n",
      "Epoch 24/40\n",
      "1/1 [==============================] - 0s 175ms/step - loss: 5.1924 - accuracy: 0.1811 - val_loss: 4.9564 - val_accuracy: 0.1821\n",
      "Epoch 25/40\n",
      "1/1 [==============================] - 0s 232ms/step - loss: 5.0651 - accuracy: 0.1811 - val_loss: 4.8346 - val_accuracy: 0.1821\n",
      "Epoch 26/40\n",
      "1/1 [==============================] - 0s 169ms/step - loss: 4.9390 - accuracy: 0.1811 - val_loss: 4.7140 - val_accuracy: 0.1821\n",
      "Epoch 27/40\n",
      "1/1 [==============================] - 0s 158ms/step - loss: 4.8143 - accuracy: 0.1811 - val_loss: 4.5951 - val_accuracy: 0.1821\n",
      "Epoch 28/40\n",
      "1/1 [==============================] - 0s 124ms/step - loss: 4.6914 - accuracy: 0.1811 - val_loss: 4.4785 - val_accuracy: 0.1821\n",
      "Epoch 29/40\n",
      "1/1 [==============================] - 0s 200ms/step - loss: 4.5708 - accuracy: 0.1831 - val_loss: 4.3639 - val_accuracy: 0.1821\n",
      "Epoch 30/40\n",
      "1/1 [==============================] - 0s 154ms/step - loss: 4.4522 - accuracy: 0.1831 - val_loss: 4.2514 - val_accuracy: 0.1852\n",
      "Epoch 31/40\n",
      "1/1 [==============================] - 0s 186ms/step - loss: 4.3354 - accuracy: 0.1872 - val_loss: 4.1408 - val_accuracy: 0.1914\n",
      "Epoch 32/40\n",
      "1/1 [==============================] - 0s 255ms/step - loss: 4.2206 - accuracy: 0.1934 - val_loss: 4.0322 - val_accuracy: 0.2006\n",
      "Epoch 33/40\n",
      "1/1 [==============================] - 0s 68ms/step - loss: 4.1079 - accuracy: 0.1975 - val_loss: 3.9259 - val_accuracy: 0.2068\n",
      "Epoch 34/40\n",
      "1/1 [==============================] - 0s 262ms/step - loss: 3.9973 - accuracy: 0.2006 - val_loss: 3.8217 - val_accuracy: 0.2068\n",
      "Epoch 35/40\n",
      "1/1 [==============================] - 0s 203ms/step - loss: 3.8887 - accuracy: 0.2037 - val_loss: 3.7198 - val_accuracy: 0.2006\n",
      "Epoch 36/40\n",
      "1/1 [==============================] - 0s 187ms/step - loss: 3.7823 - accuracy: 0.2058 - val_loss: 3.6200 - val_accuracy: 0.1944\n",
      "Epoch 37/40\n",
      "1/1 [==============================] - 0s 143ms/step - loss: 3.6780 - accuracy: 0.2027 - val_loss: 3.5221 - val_accuracy: 0.1975\n",
      "Epoch 38/40\n",
      "1/1 [==============================] - 0s 158ms/step - loss: 3.5757 - accuracy: 0.2058 - val_loss: 3.4262 - val_accuracy: 0.1852\n",
      "Epoch 39/40\n",
      "1/1 [==============================] - 0s 106ms/step - loss: 3.4755 - accuracy: 0.2068 - val_loss: 3.3321 - val_accuracy: 0.1821\n",
      "Epoch 40/40\n",
      "1/1 [==============================] - 0s 158ms/step - loss: 3.3770 - accuracy: 0.2078 - val_loss: 3.2396 - val_accuracy: 0.1759\n",
      "Accuracy: model_DNN_0: 0.2200421686746988\n",
      "Model: \"model_DNN_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 2592 random samples\n",
      "Epoch 1/40\n",
      "2/2 [==============================] - 1s 570ms/step - loss: 13.0356 - accuracy: 0.1908 - val_loss: 12.2813 - val_accuracy: 0.2207\n",
      "Epoch 2/40\n",
      "2/2 [==============================] - 0s 47ms/step - loss: 12.6393 - accuracy: 0.1908 - val_loss: 11.9005 - val_accuracy: 0.2207\n",
      "Epoch 3/40\n",
      "2/2 [==============================] - 0s 31ms/step - loss: 12.2452 - accuracy: 0.1908 - val_loss: 11.5240 - val_accuracy: 0.2207\n",
      "Epoch 4/40\n",
      "2/2 [==============================] - 0s 40ms/step - loss: 11.8512 - accuracy: 0.1908 - val_loss: 11.1524 - val_accuracy: 0.2207\n",
      "Epoch 5/40\n",
      "2/2 [==============================] - 0s 94ms/step - loss: 11.4680 - accuracy: 0.1908 - val_loss: 10.7903 - val_accuracy: 0.2207\n",
      "Epoch 6/40\n",
      "2/2 [==============================] - 0s 101ms/step - loss: 11.0944 - accuracy: 0.1908 - val_loss: 10.4470 - val_accuracy: 0.2207\n",
      "Epoch 7/40\n",
      "2/2 [==============================] - 0s 77ms/step - loss: 10.7430 - accuracy: 0.1908 - val_loss: 10.1101 - val_accuracy: 0.2207\n",
      "Epoch 8/40\n",
      "2/2 [==============================] - 0s 83ms/step - loss: 10.3912 - accuracy: 0.1908 - val_loss: 9.7771 - val_accuracy: 0.2207\n",
      "Epoch 9/40\n",
      "2/2 [==============================] - 0s 60ms/step - loss: 10.0446 - accuracy: 0.1908 - val_loss: 9.4476 - val_accuracy: 0.2207\n",
      "Epoch 10/40\n",
      "2/2 [==============================] - 0s 59ms/step - loss: 9.7029 - accuracy: 0.1908 - val_loss: 9.1206 - val_accuracy: 0.2207\n",
      "Epoch 11/40\n",
      "2/2 [==============================] - 0s 102ms/step - loss: 9.3602 - accuracy: 0.1908 - val_loss: 8.7967 - val_accuracy: 0.2207\n",
      "Epoch 12/40\n",
      "2/2 [==============================] - 0s 60ms/step - loss: 9.0247 - accuracy: 0.1908 - val_loss: 8.4769 - val_accuracy: 0.2207\n",
      "Epoch 13/40\n",
      "2/2 [==============================] - 0s 48ms/step - loss: 8.6905 - accuracy: 0.1908 - val_loss: 8.1611 - val_accuracy: 0.2207\n",
      "Epoch 14/40\n",
      "2/2 [==============================] - 0s 35ms/step - loss: 8.3631 - accuracy: 0.1908 - val_loss: 7.8466 - val_accuracy: 0.2207\n",
      "Epoch 15/40\n",
      "2/2 [==============================] - 0s 80ms/step - loss: 8.0355 - accuracy: 0.1908 - val_loss: 7.5336 - val_accuracy: 0.2207\n",
      "Epoch 16/40\n",
      "2/2 [==============================] - 0s 55ms/step - loss: 7.7140 - accuracy: 0.1908 - val_loss: 7.2221 - val_accuracy: 0.2207\n",
      "Epoch 17/40\n",
      "2/2 [==============================] - 0s 28ms/step - loss: 7.3887 - accuracy: 0.1908 - val_loss: 6.9132 - val_accuracy: 0.2207\n",
      "Epoch 18/40\n",
      "2/2 [==============================] - 0s 122ms/step - loss: 7.0657 - accuracy: 0.1908 - val_loss: 6.6072 - val_accuracy: 0.2207\n",
      "Epoch 19/40\n",
      "2/2 [==============================] - 0s 136ms/step - loss: 6.7483 - accuracy: 0.1908 - val_loss: 6.3044 - val_accuracy: 0.2207\n",
      "Epoch 20/40\n",
      "2/2 [==============================] - 0s 23ms/step - loss: 6.4371 - accuracy: 0.1908 - val_loss: 6.0053 - val_accuracy: 0.2207\n",
      "Epoch 21/40\n",
      "2/2 [==============================] - 0s 119ms/step - loss: 6.1262 - accuracy: 0.1908 - val_loss: 5.7113 - val_accuracy: 0.2207\n",
      "Epoch 22/40\n",
      "2/2 [==============================] - 0s 106ms/step - loss: 5.8195 - accuracy: 0.1908 - val_loss: 5.4242 - val_accuracy: 0.2207\n",
      "Epoch 23/40\n",
      "2/2 [==============================] - 0s 155ms/step - loss: 5.5264 - accuracy: 0.1908 - val_loss: 5.1446 - val_accuracy: 0.2207\n",
      "Epoch 24/40\n",
      "2/2 [==============================] - 0s 138ms/step - loss: 5.2372 - accuracy: 0.1903 - val_loss: 4.8754 - val_accuracy: 0.2207\n",
      "Epoch 25/40\n",
      "2/2 [==============================] - 0s 119ms/step - loss: 4.9601 - accuracy: 0.1908 - val_loss: 4.6186 - val_accuracy: 0.2222\n",
      "Epoch 26/40\n",
      "2/2 [==============================] - 0s 91ms/step - loss: 4.6972 - accuracy: 0.1950 - val_loss: 4.3751 - val_accuracy: 0.2253\n",
      "Epoch 27/40\n",
      "2/2 [==============================] - 0s 81ms/step - loss: 4.4453 - accuracy: 0.1980 - val_loss: 4.1455 - val_accuracy: 0.2207\n",
      "Epoch 28/40\n",
      "2/2 [==============================] - 0s 119ms/step - loss: 4.2123 - accuracy: 0.2027 - val_loss: 3.9289 - val_accuracy: 0.2269\n",
      "Epoch 29/40\n",
      "2/2 [==============================] - 0s 107ms/step - loss: 3.9895 - accuracy: 0.2047 - val_loss: 3.7244 - val_accuracy: 0.2361\n",
      "Epoch 30/40\n",
      "2/2 [==============================] - 0s 148ms/step - loss: 3.7801 - accuracy: 0.2088 - val_loss: 3.5310 - val_accuracy: 0.2315\n",
      "Epoch 31/40\n",
      "2/2 [==============================] - 0s 110ms/step - loss: 3.5800 - accuracy: 0.2124 - val_loss: 3.3498 - val_accuracy: 0.2377\n",
      "Epoch 32/40\n",
      "2/2 [==============================] - 0s 115ms/step - loss: 3.3953 - accuracy: 0.2160 - val_loss: 3.1798 - val_accuracy: 0.2438\n",
      "Epoch 33/40\n",
      "2/2 [==============================] - 0s 72ms/step - loss: 3.2220 - accuracy: 0.2227 - val_loss: 3.0213 - val_accuracy: 0.2454\n",
      "Epoch 34/40\n",
      "2/2 [==============================] - 0s 96ms/step - loss: 3.0625 - accuracy: 0.2305 - val_loss: 2.8749 - val_accuracy: 0.2485\n",
      "Epoch 35/40\n",
      "2/2 [==============================] - 0s 81ms/step - loss: 2.9143 - accuracy: 0.2356 - val_loss: 2.7429 - val_accuracy: 0.2608\n",
      "Epoch 36/40\n",
      "2/2 [==============================] - 0s 119ms/step - loss: 2.7840 - accuracy: 0.2407 - val_loss: 2.6270 - val_accuracy: 0.2685\n",
      "Epoch 37/40\n",
      "2/2 [==============================] - 0s 130ms/step - loss: 2.6693 - accuracy: 0.2541 - val_loss: 2.5278 - val_accuracy: 0.2809\n",
      "Epoch 38/40\n",
      "2/2 [==============================] - 0s 110ms/step - loss: 2.5694 - accuracy: 0.2572 - val_loss: 2.4442 - val_accuracy: 0.2809\n",
      "Epoch 39/40\n",
      "2/2 [==============================] - 0s 127ms/step - loss: 2.4881 - accuracy: 0.2464 - val_loss: 2.3731 - val_accuracy: 0.2994\n",
      "Epoch 40/40\n",
      "2/2 [==============================] - 0s 130ms/step - loss: 2.4194 - accuracy: 0.2659 - val_loss: 2.3108 - val_accuracy: 0.3040\n",
      "Accuracy: model_DNN_1: 0.2928855421686747\n",
      "Model: \"model_DNN_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_3 (InputLayer)         [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 5184 random samples\n",
      "Epoch 1/40\n",
      "4/4 [==============================] - 1s 340ms/step - loss: 5.5877 - accuracy: 0.1968 - val_loss: 5.2409 - val_accuracy: 0.2029\n",
      "Epoch 2/40\n",
      "4/4 [==============================] - 0s 65ms/step - loss: 4.9595 - accuracy: 0.1968 - val_loss: 4.6219 - val_accuracy: 0.2029\n",
      "Epoch 3/40\n",
      "4/4 [==============================] - 0s 51ms/step - loss: 4.3460 - accuracy: 0.1968 - val_loss: 4.0246 - val_accuracy: 0.2029\n",
      "Epoch 4/40\n",
      "4/4 [==============================] - 0s 59ms/step - loss: 3.7560 - accuracy: 0.1968 - val_loss: 3.4651 - val_accuracy: 0.2029\n",
      "Epoch 5/40\n",
      "4/4 [==============================] - 0s 34ms/step - loss: 3.2181 - accuracy: 0.1968 - val_loss: 2.9736 - val_accuracy: 0.2029\n",
      "Epoch 6/40\n",
      "4/4 [==============================] - 0s 47ms/step - loss: 2.7598 - accuracy: 0.1968 - val_loss: 2.6006 - val_accuracy: 0.2006\n",
      "Epoch 7/40\n",
      "4/4 [==============================] - 0s 27ms/step - loss: 2.4345 - accuracy: 0.2086 - val_loss: 2.3704 - val_accuracy: 0.2400\n",
      "Epoch 8/40\n",
      "4/4 [==============================] - 0s 27ms/step - loss: 2.2339 - accuracy: 0.2263 - val_loss: 2.2372 - val_accuracy: 0.2315\n",
      "Epoch 9/40\n",
      "4/4 [==============================] - 0s 29ms/step - loss: 2.1206 - accuracy: 0.2284 - val_loss: 2.1428 - val_accuracy: 0.2253\n",
      "Epoch 10/40\n",
      "4/4 [==============================] - 0s 31ms/step - loss: 2.0306 - accuracy: 0.2361 - val_loss: 2.0603 - val_accuracy: 0.2847\n",
      "Epoch 11/40\n",
      "4/4 [==============================] - 0s 34ms/step - loss: 1.9520 - accuracy: 0.2834 - val_loss: 1.9831 - val_accuracy: 0.2901\n",
      "Epoch 12/40\n",
      "4/4 [==============================] - 0s 40ms/step - loss: 1.8822 - accuracy: 0.2912 - val_loss: 1.9111 - val_accuracy: 0.2963\n",
      "Epoch 13/40\n",
      "4/4 [==============================] - 0s 56ms/step - loss: 1.8180 - accuracy: 0.2971 - val_loss: 1.8467 - val_accuracy: 0.3063\n",
      "Epoch 14/40\n",
      "4/4 [==============================] - 0s 40ms/step - loss: 1.7623 - accuracy: 0.3027 - val_loss: 1.7914 - val_accuracy: 0.3117\n",
      "Epoch 15/40\n",
      "4/4 [==============================] - 0s 70ms/step - loss: 1.7153 - accuracy: 0.3079 - val_loss: 1.7440 - val_accuracy: 0.3202\n",
      "Epoch 16/40\n",
      "4/4 [==============================] - 0s 36ms/step - loss: 1.6768 - accuracy: 0.3179 - val_loss: 1.7023 - val_accuracy: 0.3233\n",
      "Epoch 17/40\n",
      "4/4 [==============================] - 0s 27ms/step - loss: 1.6416 - accuracy: 0.3264 - val_loss: 1.6649 - val_accuracy: 0.3279\n",
      "Epoch 18/40\n",
      "4/4 [==============================] - 0s 28ms/step - loss: 1.6099 - accuracy: 0.3287 - val_loss: 1.6302 - val_accuracy: 0.3380\n",
      "Epoch 19/40\n",
      "4/4 [==============================] - 0s 34ms/step - loss: 1.5809 - accuracy: 0.3328 - val_loss: 1.5981 - val_accuracy: 0.3441\n",
      "Epoch 20/40\n",
      "4/4 [==============================] - 0s 26ms/step - loss: 1.5537 - accuracy: 0.3364 - val_loss: 1.5693 - val_accuracy: 0.3465\n",
      "Epoch 21/40\n",
      "4/4 [==============================] - 0s 24ms/step - loss: 1.5297 - accuracy: 0.3436 - val_loss: 1.5437 - val_accuracy: 0.3526\n",
      "Epoch 22/40\n",
      "4/4 [==============================] - 0s 27ms/step - loss: 1.5088 - accuracy: 0.3465 - val_loss: 1.5209 - val_accuracy: 0.3580\n",
      "Epoch 23/40\n",
      "4/4 [==============================] - 0s 35ms/step - loss: 1.4896 - accuracy: 0.3526 - val_loss: 1.5007 - val_accuracy: 0.3603\n",
      "Epoch 24/40\n",
      "4/4 [==============================] - 0s 30ms/step - loss: 1.4735 - accuracy: 0.3555 - val_loss: 1.4827 - val_accuracy: 0.3619\n",
      "Epoch 25/40\n",
      "4/4 [==============================] - 0s 29ms/step - loss: 1.4592 - accuracy: 0.3588 - val_loss: 1.4669 - val_accuracy: 0.3665\n",
      "Epoch 26/40\n",
      "4/4 [==============================] - 0s 32ms/step - loss: 1.4469 - accuracy: 0.3611 - val_loss: 1.4531 - val_accuracy: 0.3688\n",
      "Epoch 27/40\n",
      "4/4 [==============================] - 0s 31ms/step - loss: 1.4360 - accuracy: 0.3650 - val_loss: 1.4414 - val_accuracy: 0.3711\n",
      "Epoch 28/40\n",
      "4/4 [==============================] - 0s 34ms/step - loss: 1.4276 - accuracy: 0.3688 - val_loss: 1.4312 - val_accuracy: 0.3719\n",
      "Epoch 29/40\n",
      "4/4 [==============================] - 0s 34ms/step - loss: 1.4197 - accuracy: 0.3701 - val_loss: 1.4229 - val_accuracy: 0.3742\n",
      "Epoch 30/40\n",
      "4/4 [==============================] - 0s 27ms/step - loss: 1.4130 - accuracy: 0.3737 - val_loss: 1.4161 - val_accuracy: 0.3765\n",
      "Epoch 31/40\n",
      "4/4 [==============================] - 0s 28ms/step - loss: 1.4079 - accuracy: 0.3765 - val_loss: 1.4107 - val_accuracy: 0.3773\n",
      "Epoch 32/40\n",
      "4/4 [==============================] - 0s 40ms/step - loss: 1.4039 - accuracy: 0.3737 - val_loss: 1.4065 - val_accuracy: 0.3812\n",
      "Epoch 33/40\n",
      "4/4 [==============================] - 0s 42ms/step - loss: 1.4007 - accuracy: 0.3745 - val_loss: 1.4035 - val_accuracy: 0.3819\n",
      "Epoch 34/40\n",
      "4/4 [==============================] - 0s 69ms/step - loss: 1.3979 - accuracy: 0.3758 - val_loss: 1.4013 - val_accuracy: 0.3843\n",
      "Epoch 35/40\n",
      "4/4 [==============================] - 0s 17ms/step - loss: 1.3959 - accuracy: 0.3794 - val_loss: 1.3996 - val_accuracy: 0.3835\n",
      "Epoch 36/40\n",
      "4/4 [==============================] - 0s 18ms/step - loss: 1.3943 - accuracy: 0.3789 - val_loss: 1.3984 - val_accuracy: 0.3781\n",
      "Epoch 37/40\n",
      "4/4 [==============================] - 0s 42ms/step - loss: 1.3929 - accuracy: 0.3773 - val_loss: 1.3973 - val_accuracy: 0.3804\n",
      "Epoch 38/40\n",
      "4/4 [==============================] - 0s 30ms/step - loss: 1.3915 - accuracy: 0.3773 - val_loss: 1.3964 - val_accuracy: 0.3812\n",
      "Epoch 39/40\n",
      "4/4 [==============================] - 0s 40ms/step - loss: 1.3902 - accuracy: 0.3778 - val_loss: 1.3957 - val_accuracy: 0.3843\n",
      "Epoch 40/40\n",
      "4/4 [==============================] - 0s 38ms/step - loss: 1.3893 - accuracy: 0.3768 - val_loss: 1.3950 - val_accuracy: 0.3881\n",
      "Accuracy: model_DNN_2: 0.3902831325301205\n",
      "Model: \"model_DNN_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_4 (InputLayer)         [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 10368 random samples\n",
      "Epoch 1/40\n",
      "8/8 [==============================] - 1s 165ms/step - loss: 9.9560 - accuracy: 0.1856 - val_loss: 8.7882 - val_accuracy: 0.1968\n",
      "Epoch 2/40\n",
      "8/8 [==============================] - 0s 36ms/step - loss: 8.6032 - accuracy: 0.1798 - val_loss: 7.5169 - val_accuracy: 0.2014\n",
      "Epoch 3/40\n",
      "8/8 [==============================] - 0s 22ms/step - loss: 7.2777 - accuracy: 0.1901 - val_loss: 6.2547 - val_accuracy: 0.2199\n",
      "Epoch 4/40\n",
      "8/8 [==============================] - 0s 32ms/step - loss: 5.9835 - accuracy: 0.1988 - val_loss: 5.0825 - val_accuracy: 0.2242\n",
      "Epoch 5/40\n",
      "8/8 [==============================] - 0s 21ms/step - loss: 4.8287 - accuracy: 0.2034 - val_loss: 4.0425 - val_accuracy: 0.2288\n",
      "Epoch 6/40\n",
      "8/8 [==============================] - 0s 22ms/step - loss: 3.7729 - accuracy: 0.2130 - val_loss: 3.0719 - val_accuracy: 0.2342\n",
      "Epoch 7/40\n",
      "8/8 [==============================] - 0s 37ms/step - loss: 2.7654 - accuracy: 0.2230 - val_loss: 2.2103 - val_accuracy: 0.2515\n",
      "Epoch 8/40\n",
      "8/8 [==============================] - 0s 29ms/step - loss: 1.9701 - accuracy: 0.2438 - val_loss: 1.7053 - val_accuracy: 0.2465\n",
      "Epoch 9/40\n",
      "8/8 [==============================] - 0s 24ms/step - loss: 1.6304 - accuracy: 0.2586 - val_loss: 1.6395 - val_accuracy: 0.2643\n",
      "Epoch 10/40\n",
      "8/8 [==============================] - 0s 17ms/step - loss: 1.6008 - accuracy: 0.3054 - val_loss: 1.6166 - val_accuracy: 0.3044\n",
      "Epoch 11/40\n",
      "8/8 [==============================] - 0s 19ms/step - loss: 1.5680 - accuracy: 0.3189 - val_loss: 1.5772 - val_accuracy: 0.2975\n",
      "Epoch 12/40\n",
      "8/8 [==============================] - 0s 26ms/step - loss: 1.5429 - accuracy: 0.3198 - val_loss: 1.5450 - val_accuracy: 0.2978\n",
      "Epoch 13/40\n",
      "8/8 [==============================] - 0s 33ms/step - loss: 1.5152 - accuracy: 0.3211 - val_loss: 1.5143 - val_accuracy: 0.3090\n",
      "Epoch 14/40\n",
      "8/8 [==============================] - 0s 25ms/step - loss: 1.4948 - accuracy: 0.3234 - val_loss: 1.4974 - val_accuracy: 0.3075\n",
      "Epoch 15/40\n",
      "8/8 [==============================] - 0s 17ms/step - loss: 1.4810 - accuracy: 0.3203 - val_loss: 1.4844 - val_accuracy: 0.3044\n",
      "Epoch 16/40\n",
      "8/8 [==============================] - ETA: 0s - loss: 1.4688 - accuracy: 0.32 - 0s 33ms/step - loss: 1.4688 - accuracy: 0.3224 - val_loss: 1.4727 - val_accuracy: 0.3040\n",
      "Epoch 17/40\n",
      "8/8 [==============================] - 0s 31ms/step - loss: 1.4574 - accuracy: 0.3275 - val_loss: 1.4602 - val_accuracy: 0.3083\n",
      "Epoch 18/40\n",
      "8/8 [==============================] - 0s 37ms/step - loss: 1.4466 - accuracy: 0.3281 - val_loss: 1.4473 - val_accuracy: 0.3137\n",
      "Epoch 19/40\n",
      "8/8 [==============================] - 0s 36ms/step - loss: 1.4349 - accuracy: 0.3273 - val_loss: 1.4339 - val_accuracy: 0.3125\n",
      "Epoch 20/40\n",
      "8/8 [==============================] - 0s 28ms/step - loss: 1.4256 - accuracy: 0.3308 - val_loss: 1.4225 - val_accuracy: 0.3160\n",
      "Epoch 21/40\n",
      "8/8 [==============================] - 0s 36ms/step - loss: 1.4179 - accuracy: 0.3373 - val_loss: 1.4145 - val_accuracy: 0.3306\n",
      "Epoch 22/40\n",
      "8/8 [==============================] - 0s 37ms/step - loss: 1.4128 - accuracy: 0.3480 - val_loss: 1.4081 - val_accuracy: 0.3414\n",
      "Epoch 23/40\n",
      "8/8 [==============================] - 0s 32ms/step - loss: 1.4082 - accuracy: 0.3588 - val_loss: 1.4038 - val_accuracy: 0.3445\n",
      "Epoch 24/40\n",
      "8/8 [==============================] - 0s 28ms/step - loss: 1.4049 - accuracy: 0.3711 - val_loss: 1.4017 - val_accuracy: 0.3526\n",
      "Epoch 25/40\n",
      "8/8 [==============================] - 0s 28ms/step - loss: 1.4023 - accuracy: 0.3783 - val_loss: 1.3990 - val_accuracy: 0.3573\n",
      "Epoch 26/40\n",
      "8/8 [==============================] - 0s 39ms/step - loss: 1.3999 - accuracy: 0.3830 - val_loss: 1.3960 - val_accuracy: 0.3627\n",
      "Epoch 27/40\n",
      "8/8 [==============================] - 0s 29ms/step - loss: 1.3978 - accuracy: 0.3868 - val_loss: 1.3940 - val_accuracy: 0.3657\n",
      "Epoch 28/40\n",
      "8/8 [==============================] - 0s 36ms/step - loss: 1.3960 - accuracy: 0.3881 - val_loss: 1.3921 - val_accuracy: 0.3661\n",
      "Epoch 29/40\n",
      "8/8 [==============================] - 0s 50ms/step - loss: 1.3941 - accuracy: 0.3904 - val_loss: 1.3902 - val_accuracy: 0.3654\n",
      "Epoch 30/40\n",
      "8/8 [==============================] - 0s 38ms/step - loss: 1.3924 - accuracy: 0.3911 - val_loss: 1.3891 - val_accuracy: 0.3665\n",
      "Epoch 31/40\n",
      "8/8 [==============================] - 0s 27ms/step - loss: 1.3906 - accuracy: 0.3916 - val_loss: 1.3883 - val_accuracy: 0.3681\n",
      "Epoch 32/40\n",
      "8/8 [==============================] - 0s 15ms/step - loss: 1.3891 - accuracy: 0.3926 - val_loss: 1.3863 - val_accuracy: 0.3696\n",
      "Epoch 33/40\n",
      "8/8 [==============================] - 0s 19ms/step - loss: 1.3875 - accuracy: 0.3931 - val_loss: 1.3847 - val_accuracy: 0.3700\n",
      "Epoch 34/40\n",
      "8/8 [==============================] - 0s 9ms/step - loss: 1.3857 - accuracy: 0.3944 - val_loss: 1.3833 - val_accuracy: 0.3715\n",
      "Epoch 35/40\n",
      "8/8 [==============================] - 0s 10ms/step - loss: 1.3843 - accuracy: 0.3949 - val_loss: 1.3820 - val_accuracy: 0.3715\n",
      "Epoch 36/40\n",
      "8/8 [==============================] - 0s 11ms/step - loss: 1.3826 - accuracy: 0.3970 - val_loss: 1.3801 - val_accuracy: 0.3711\n",
      "Epoch 37/40\n",
      "8/8 [==============================] - 0s 18ms/step - loss: 1.3809 - accuracy: 0.3976 - val_loss: 1.3788 - val_accuracy: 0.3742\n",
      "Epoch 38/40\n",
      "8/8 [==============================] - 0s 33ms/step - loss: 1.3793 - accuracy: 0.3996 - val_loss: 1.3772 - val_accuracy: 0.3719\n",
      "Epoch 39/40\n",
      "8/8 [==============================] - 0s 45ms/step - loss: 1.3777 - accuracy: 0.3997 - val_loss: 1.3761 - val_accuracy: 0.3715\n",
      "Epoch 40/40\n",
      "8/8 [==============================] - 0s 47ms/step - loss: 1.3764 - accuracy: 0.4008 - val_loss: 1.3746 - val_accuracy: 0.3765\n",
      "Accuracy: model_DNN_3: 0.4014819277108434\n",
      "Model: \"model_DNN_4\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_5 (InputLayer)         [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 20736 random samples\n",
      "Epoch 1/40\n",
      "16/16 [==============================] - 2s 106ms/step - loss: 5.2948 - accuracy: 0.1969 - val_loss: 4.4739 - val_accuracy: 0.1989\n",
      "Epoch 2/40\n",
      "16/16 [==============================] - 0s 14ms/step - loss: 3.9159 - accuracy: 0.1977 - val_loss: 3.2897 - val_accuracy: 0.2008\n",
      "Epoch 3/40\n",
      "16/16 [==============================] - 0s 13ms/step - loss: 2.8573 - accuracy: 0.1972 - val_loss: 2.4012 - val_accuracy: 0.1860\n",
      "Epoch 4/40\n",
      "16/16 [==============================] - 0s 10ms/step - loss: 2.1957 - accuracy: 0.1825 - val_loss: 1.9731 - val_accuracy: 0.2091\n",
      "Epoch 5/40\n",
      "16/16 [==============================] - 0s 11ms/step - loss: 1.8620 - accuracy: 0.2428 - val_loss: 1.6888 - val_accuracy: 0.3110\n",
      "Epoch 6/40\n",
      "16/16 [==============================] - 0s 12ms/step - loss: 1.6179 - accuracy: 0.3267 - val_loss: 1.5016 - val_accuracy: 0.3547\n",
      "Epoch 7/40\n",
      "16/16 [==============================] - 0s 10ms/step - loss: 1.4806 - accuracy: 0.3470 - val_loss: 1.4234 - val_accuracy: 0.3571\n",
      "Epoch 8/40\n",
      "16/16 [==============================] - 0s 11ms/step - loss: 1.4298 - accuracy: 0.3445 - val_loss: 1.3987 - val_accuracy: 0.3486\n",
      "Epoch 9/40\n",
      "16/16 [==============================] - 0s 10ms/step - loss: 1.4102 - accuracy: 0.3349 - val_loss: 1.3888 - val_accuracy: 0.3499\n",
      "Epoch 10/40\n",
      "16/16 [==============================] - 0s 11ms/step - loss: 1.4017 - accuracy: 0.3391 - val_loss: 1.3839 - val_accuracy: 0.3505\n",
      "Epoch 11/40\n",
      "16/16 [==============================] - 0s 13ms/step - loss: 1.3971 - accuracy: 0.3360 - val_loss: 1.3798 - val_accuracy: 0.3459\n",
      "Epoch 12/40\n",
      "16/16 [==============================] - 0s 10ms/step - loss: 1.3933 - accuracy: 0.3366 - val_loss: 1.3759 - val_accuracy: 0.3480\n",
      "Epoch 13/40\n",
      "16/16 [==============================] - 0s 14ms/step - loss: 1.3898 - accuracy: 0.3379 - val_loss: 1.3727 - val_accuracy: 0.3517\n",
      "Epoch 14/40\n",
      "16/16 [==============================] - 0s 9ms/step - loss: 1.3870 - accuracy: 0.3414 - val_loss: 1.3698 - val_accuracy: 0.3563\n",
      "Epoch 15/40\n",
      "16/16 [==============================] - 0s 13ms/step - loss: 1.3839 - accuracy: 0.3499 - val_loss: 1.3670 - val_accuracy: 0.3665\n",
      "Epoch 16/40\n",
      "16/16 [==============================] - 0s 17ms/step - loss: 1.3810 - accuracy: 0.3582 - val_loss: 1.3642 - val_accuracy: 0.3657\n",
      "Epoch 17/40\n",
      "16/16 [==============================] - 0s 14ms/step - loss: 1.3783 - accuracy: 0.3547 - val_loss: 1.3617 - val_accuracy: 0.3727\n",
      "Epoch 18/40\n",
      "16/16 [==============================] - 0s 11ms/step - loss: 1.3756 - accuracy: 0.3690 - val_loss: 1.3592 - val_accuracy: 0.3850\n",
      "Epoch 19/40\n",
      "16/16 [==============================] - 0s 11ms/step - loss: 1.3731 - accuracy: 0.3697 - val_loss: 1.3566 - val_accuracy: 0.3839\n",
      "Epoch 20/40\n",
      "16/16 [==============================] - 0s 12ms/step - loss: 1.3709 - accuracy: 0.3891 - val_loss: 1.3545 - val_accuracy: 0.3987\n",
      "Epoch 21/40\n",
      "16/16 [==============================] - 0s 13ms/step - loss: 1.3681 - accuracy: 0.3886 - val_loss: 1.3518 - val_accuracy: 0.3985\n",
      "Epoch 22/40\n",
      "16/16 [==============================] - 0s 15ms/step - loss: 1.3654 - accuracy: 0.3856 - val_loss: 1.3487 - val_accuracy: 0.4061\n",
      "Epoch 23/40\n",
      "16/16 [==============================] - 0s 16ms/step - loss: 1.3626 - accuracy: 0.3912 - val_loss: 1.3465 - val_accuracy: 0.4080\n",
      "Epoch 24/40\n",
      "16/16 [==============================] - 0s 15ms/step - loss: 1.3600 - accuracy: 0.3963 - val_loss: 1.3434 - val_accuracy: 0.4165\n",
      "Epoch 25/40\n",
      "16/16 [==============================] - 0s 13ms/step - loss: 1.3573 - accuracy: 0.4035 - val_loss: 1.3413 - val_accuracy: 0.4223\n",
      "Epoch 26/40\n",
      "16/16 [==============================] - 0s 15ms/step - loss: 1.3546 - accuracy: 0.4072 - val_loss: 1.3384 - val_accuracy: 0.4228\n",
      "Epoch 27/40\n",
      "16/16 [==============================] - 0s 18ms/step - loss: 1.3521 - accuracy: 0.4058 - val_loss: 1.3362 - val_accuracy: 0.4236\n",
      "Epoch 28/40\n",
      "16/16 [==============================] - 0s 12ms/step - loss: 1.3499 - accuracy: 0.4126 - val_loss: 1.3340 - val_accuracy: 0.4257\n",
      "Epoch 29/40\n",
      "16/16 [==============================] - 0s 11ms/step - loss: 1.3474 - accuracy: 0.4131 - val_loss: 1.3318 - val_accuracy: 0.4271\n",
      "Epoch 30/40\n",
      "16/16 [==============================] - 0s 21ms/step - loss: 1.3449 - accuracy: 0.4122 - val_loss: 1.3293 - val_accuracy: 0.4261\n",
      "Epoch 31/40\n",
      "16/16 [==============================] - 0s 16ms/step - loss: 1.3427 - accuracy: 0.4156 - val_loss: 1.3270 - val_accuracy: 0.4313\n",
      "Epoch 32/40\n",
      "16/16 [==============================] - 0s 12ms/step - loss: 1.3406 - accuracy: 0.4122 - val_loss: 1.3249 - val_accuracy: 0.4309\n",
      "Epoch 33/40\n",
      "16/16 [==============================] - 0s 12ms/step - loss: 1.3382 - accuracy: 0.4198 - val_loss: 1.3224 - val_accuracy: 0.4340\n",
      "Epoch 34/40\n",
      "16/16 [==============================] - 0s 10ms/step - loss: 1.3353 - accuracy: 0.4200 - val_loss: 1.3197 - val_accuracy: 0.4317\n",
      "Epoch 35/40\n",
      "16/16 [==============================] - 0s 9ms/step - loss: 1.3327 - accuracy: 0.4196 - val_loss: 1.3171 - val_accuracy: 0.4367\n",
      "Epoch 36/40\n",
      "16/16 [==============================] - 0s 12ms/step - loss: 1.3301 - accuracy: 0.4227 - val_loss: 1.3144 - val_accuracy: 0.4381\n",
      "Epoch 37/40\n",
      "16/16 [==============================] - 0s 13ms/step - loss: 1.3279 - accuracy: 0.4201 - val_loss: 1.3124 - val_accuracy: 0.4435\n",
      "Epoch 38/40\n",
      "16/16 [==============================] - 0s 11ms/step - loss: 1.3254 - accuracy: 0.4287 - val_loss: 1.3097 - val_accuracy: 0.4365\n",
      "Epoch 39/40\n",
      "16/16 [==============================] - 0s 15ms/step - loss: 1.3232 - accuracy: 0.4264 - val_loss: 1.3073 - val_accuracy: 0.4414\n",
      "Epoch 40/40\n",
      "16/16 [==============================] - 0s 11ms/step - loss: 1.3202 - accuracy: 0.4243 - val_loss: 1.3047 - val_accuracy: 0.4429\n",
      "Accuracy: model_DNN_4: 0.4385301204819277\n",
      "Model: \"model_DNN_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_6 (InputLayer)         [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 41472 random samples\n",
      "Epoch 1/40\n",
      "31/31 [==============================] - 2s 56ms/step - loss: 6.2567 - accuracy: 0.1665 - val_loss: 4.4178 - val_accuracy: 0.2297\n",
      "Epoch 2/40\n",
      "31/31 [==============================] - 0s 13ms/step - loss: 3.5052 - accuracy: 0.2115 - val_loss: 2.7299 - val_accuracy: 0.2031\n",
      "Epoch 3/40\n",
      "31/31 [==============================] - 1s 17ms/step - loss: 2.4837 - accuracy: 0.2978 - val_loss: 2.2158 - val_accuracy: 0.3364\n",
      "Epoch 4/40\n",
      "31/31 [==============================] - 0s 13ms/step - loss: 2.0211 - accuracy: 0.3551 - val_loss: 1.7767 - val_accuracy: 0.3544\n",
      "Epoch 5/40\n",
      "31/31 [==============================] - 0s 15ms/step - loss: 1.6127 - accuracy: 0.3476 - val_loss: 1.4714 - val_accuracy: 0.3325\n",
      "Epoch 6/40\n",
      "31/31 [==============================] - 0s 11ms/step - loss: 1.4398 - accuracy: 0.3310 - val_loss: 1.4111 - val_accuracy: 0.3469\n",
      "Epoch 7/40\n",
      "31/31 [==============================] - 1s 18ms/step - loss: 1.3974 - accuracy: 0.3609 - val_loss: 1.3803 - val_accuracy: 0.3702\n",
      "Epoch 8/40\n",
      "31/31 [==============================] - 0s 12ms/step - loss: 1.3740 - accuracy: 0.3725 - val_loss: 1.3639 - val_accuracy: 0.3909\n",
      "Epoch 9/40\n",
      "31/31 [==============================] - 1s 19ms/step - loss: 1.3626 - accuracy: 0.3924 - val_loss: 1.3550 - val_accuracy: 0.4133\n",
      "Epoch 10/40\n",
      "31/31 [==============================] - 1s 17ms/step - loss: 1.3553 - accuracy: 0.4063 - val_loss: 1.3489 - val_accuracy: 0.4193\n",
      "Epoch 11/40\n",
      "31/31 [==============================] - 0s 9ms/step - loss: 1.3504 - accuracy: 0.4060 - val_loss: 1.3448 - val_accuracy: 0.3989\n",
      "Epoch 12/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3451 - accuracy: 0.4144 - val_loss: 1.3385 - val_accuracy: 0.4224\n",
      "Epoch 13/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3406 - accuracy: 0.4161 - val_loss: 1.3341 - val_accuracy: 0.4257\n",
      "Epoch 14/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3362 - accuracy: 0.4197 - val_loss: 1.3292 - val_accuracy: 0.4258\n",
      "Epoch 15/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3308 - accuracy: 0.4210 - val_loss: 1.3247 - val_accuracy: 0.4251\n",
      "Epoch 16/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3258 - accuracy: 0.4225 - val_loss: 1.3190 - val_accuracy: 0.4307\n",
      "Epoch 17/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3213 - accuracy: 0.4274 - val_loss: 1.3149 - val_accuracy: 0.4328\n",
      "Epoch 18/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3168 - accuracy: 0.4302 - val_loss: 1.3103 - val_accuracy: 0.4334\n",
      "Epoch 19/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3102 - accuracy: 0.4382 - val_loss: 1.3033 - val_accuracy: 0.4522\n",
      "Epoch 20/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3046 - accuracy: 0.4426 - val_loss: 1.2973 - val_accuracy: 0.4487\n",
      "Epoch 21/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2995 - accuracy: 0.4490 - val_loss: 1.2933 - val_accuracy: 0.4745\n",
      "Epoch 22/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2943 - accuracy: 0.4534 - val_loss: 1.2874 - val_accuracy: 0.4659\n",
      "Epoch 23/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2889 - accuracy: 0.4619 - val_loss: 1.2817 - val_accuracy: 0.4649\n",
      "Epoch 24/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2834 - accuracy: 0.4701 - val_loss: 1.2768 - val_accuracy: 0.4991\n",
      "Epoch 25/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2774 - accuracy: 0.4740 - val_loss: 1.2701 - val_accuracy: 0.4872\n",
      "Epoch 26/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2712 - accuracy: 0.4927 - val_loss: 1.2651 - val_accuracy: 0.5186\n",
      "Epoch 27/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2651 - accuracy: 0.4950 - val_loss: 1.2576 - val_accuracy: 0.5154\n",
      "Epoch 28/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2584 - accuracy: 0.5067 - val_loss: 1.2522 - val_accuracy: 0.5319\n",
      "Epoch 29/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2526 - accuracy: 0.5113 - val_loss: 1.2459 - val_accuracy: 0.5449\n",
      "Epoch 30/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2464 - accuracy: 0.5284 - val_loss: 1.2397 - val_accuracy: 0.5214\n",
      "Epoch 31/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2402 - accuracy: 0.5336 - val_loss: 1.2326 - val_accuracy: 0.5358\n",
      "Epoch 32/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2328 - accuracy: 0.5434 - val_loss: 1.2258 - val_accuracy: 0.5622\n",
      "Epoch 33/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2265 - accuracy: 0.5473 - val_loss: 1.2200 - val_accuracy: 0.5299\n",
      "Epoch 34/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2200 - accuracy: 0.5507 - val_loss: 1.2134 - val_accuracy: 0.5729\n",
      "Epoch 35/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2126 - accuracy: 0.5589 - val_loss: 1.2069 - val_accuracy: 0.5779\n",
      "Epoch 36/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2060 - accuracy: 0.5630 - val_loss: 1.2004 - val_accuracy: 0.5780\n",
      "Epoch 37/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.1993 - accuracy: 0.5699 - val_loss: 1.1941 - val_accuracy: 0.5644\n",
      "Epoch 38/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.1921 - accuracy: 0.5660 - val_loss: 1.1857 - val_accuracy: 0.5666\n",
      "Epoch 39/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.1855 - accuracy: 0.5669 - val_loss: 1.1794 - val_accuracy: 0.5673\n",
      "Epoch 40/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.1791 - accuracy: 0.5705 - val_loss: 1.1720 - val_accuracy: 0.5777\n",
      "Accuracy: model_DNN_5: 0.5772289156626506\n",
      "Model: \"model_DNN_6\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_7 (InputLayer)         [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 82944 random samples\n",
      "Epoch 1/40\n",
      "61/61 [==============================] - 0s 5ms/step - loss: 4.7408 - accuracy: 0.3054 - val_loss: 2.2046 - val_accuracy: 0.3086\n",
      "Epoch 2/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.5388 - accuracy: 0.3263 - val_loss: 1.3986 - val_accuracy: 0.3312\n",
      "Epoch 3/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.3876 - accuracy: 0.3336 - val_loss: 1.3839 - val_accuracy: 0.3303\n",
      "Epoch 4/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.3780 - accuracy: 0.3341 - val_loss: 1.3776 - val_accuracy: 0.3339\n",
      "Epoch 5/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.3717 - accuracy: 0.3370 - val_loss: 1.3709 - val_accuracy: 0.3364\n",
      "Epoch 6/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.3649 - accuracy: 0.3520 - val_loss: 1.3631 - val_accuracy: 0.3484\n",
      "Epoch 7/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.3574 - accuracy: 0.3694 - val_loss: 1.3569 - val_accuracy: 0.3664\n",
      "Epoch 8/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.3503 - accuracy: 0.3849 - val_loss: 1.3499 - val_accuracy: 0.3817\n",
      "Epoch 9/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.3427 - accuracy: 0.3934 - val_loss: 1.3422 - val_accuracy: 0.3976\n",
      "Epoch 10/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.3345 - accuracy: 0.4044 - val_loss: 1.3333 - val_accuracy: 0.4037\n",
      "Epoch 11/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.3257 - accuracy: 0.4151 - val_loss: 1.3243 - val_accuracy: 0.4110\n",
      "Epoch 12/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.3162 - accuracy: 0.4207 - val_loss: 1.3147 - val_accuracy: 0.4236\n",
      "Epoch 13/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.3068 - accuracy: 0.4364 - val_loss: 1.3054 - val_accuracy: 0.4195\n",
      "Epoch 14/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.2971 - accuracy: 0.4412 - val_loss: 1.2956 - val_accuracy: 0.4449\n",
      "Epoch 15/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.2870 - accuracy: 0.4562 - val_loss: 1.2846 - val_accuracy: 0.4443\n",
      "Epoch 16/40\n",
      "61/61 [==============================] - 0s 2ms/step - loss: 1.2760 - accuracy: 0.4641 - val_loss: 1.2735 - val_accuracy: 0.4651\n",
      "Epoch 17/40\n",
      "61/61 [==============================] - 0s 2ms/step - loss: 1.2641 - accuracy: 0.4746 - val_loss: 1.2586 - val_accuracy: 0.4757\n",
      "Epoch 18/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.2487 - accuracy: 0.4923 - val_loss: 1.2424 - val_accuracy: 0.5051\n",
      "Epoch 19/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.2327 - accuracy: 0.5156 - val_loss: 1.2285 - val_accuracy: 0.5299\n",
      "Epoch 20/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.2182 - accuracy: 0.5355 - val_loss: 1.2129 - val_accuracy: 0.5497\n",
      "Epoch 21/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.2038 - accuracy: 0.5530 - val_loss: 1.1987 - val_accuracy: 0.5539\n",
      "Epoch 22/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.1897 - accuracy: 0.5600 - val_loss: 1.1866 - val_accuracy: 0.5631\n",
      "Epoch 23/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.1766 - accuracy: 0.5635 - val_loss: 1.1745 - val_accuracy: 0.5756\n",
      "Epoch 24/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.1640 - accuracy: 0.5687 - val_loss: 1.1620 - val_accuracy: 0.5815\n",
      "Epoch 25/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.1524 - accuracy: 0.5737 - val_loss: 1.1504 - val_accuracy: 0.5752\n",
      "Epoch 26/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.1422 - accuracy: 0.5776 - val_loss: 1.1414 - val_accuracy: 0.5741\n",
      "Epoch 27/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.1324 - accuracy: 0.5809 - val_loss: 1.1310 - val_accuracy: 0.5738\n",
      "Epoch 28/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.1239 - accuracy: 0.5878 - val_loss: 1.1213 - val_accuracy: 0.5926\n",
      "Epoch 29/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.1131 - accuracy: 0.5977 - val_loss: 1.1123 - val_accuracy: 0.6035\n",
      "Epoch 30/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.1047 - accuracy: 0.6041 - val_loss: 1.1050 - val_accuracy: 0.5905\n",
      "Epoch 31/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0969 - accuracy: 0.6077 - val_loss: 1.0987 - val_accuracy: 0.5961\n",
      "Epoch 32/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0898 - accuracy: 0.6087 - val_loss: 1.0920 - val_accuracy: 0.6068\n",
      "Epoch 33/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0840 - accuracy: 0.6109 - val_loss: 1.0867 - val_accuracy: 0.6044\n",
      "Epoch 34/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0792 - accuracy: 0.6129 - val_loss: 1.0799 - val_accuracy: 0.6120\n",
      "Epoch 35/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0729 - accuracy: 0.6160 - val_loss: 1.0746 - val_accuracy: 0.6162\n",
      "Epoch 36/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0682 - accuracy: 0.6174 - val_loss: 1.0689 - val_accuracy: 0.6198\n",
      "Epoch 37/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0633 - accuracy: 0.6233 - val_loss: 1.0652 - val_accuracy: 0.6226\n",
      "Epoch 38/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0586 - accuracy: 0.6224 - val_loss: 1.0593 - val_accuracy: 0.6227\n",
      "Epoch 39/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0543 - accuracy: 0.6257 - val_loss: 1.0561 - val_accuracy: 0.6238\n",
      "Epoch 40/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0506 - accuracy: 0.6245 - val_loss: 1.0515 - val_accuracy: 0.6259\n",
      "Accuracy: model_DNN_6: 0.6276626506024097\n",
      "Model: \"model_DNN_7\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_8 (InputLayer)         [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 165888 random samples\n",
      "Epoch 1/40\n",
      "122/122 [==============================] - 0s 4ms/step - loss: 1.7998 - accuracy: 0.3250 - val_loss: 1.3885 - val_accuracy: 0.3573\n",
      "Epoch 2/40\n",
      "122/122 [==============================] - 0s 3ms/step - loss: 1.3714 - accuracy: 0.3673 - val_loss: 1.3554 - val_accuracy: 0.3858\n",
      "Epoch 3/40\n",
      "122/122 [==============================] - 0s 3ms/step - loss: 1.3475 - accuracy: 0.4009 - val_loss: 1.3355 - val_accuracy: 0.4073\n",
      "Epoch 4/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.3286 - accuracy: 0.4258 - val_loss: 1.3177 - val_accuracy: 0.4422\n",
      "Epoch 5/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.3100 - accuracy: 0.4482 - val_loss: 1.2984 - val_accuracy: 0.4576\n",
      "Epoch 6/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.2901 - accuracy: 0.4701 - val_loss: 1.2776 - val_accuracy: 0.4807\n",
      "Epoch 7/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.2687 - accuracy: 0.5018 - val_loss: 1.2545 - val_accuracy: 0.5185\n",
      "Epoch 8/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.2444 - accuracy: 0.5337 - val_loss: 1.2298 - val_accuracy: 0.5389\n",
      "Epoch 9/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.2140 - accuracy: 0.5612 - val_loss: 1.1943 - val_accuracy: 0.5745\n",
      "Epoch 10/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.1822 - accuracy: 0.5790 - val_loss: 1.1654 - val_accuracy: 0.5714\n",
      "Epoch 11/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.1549 - accuracy: 0.5887 - val_loss: 1.1401 - val_accuracy: 0.5827\n",
      "Epoch 12/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.1314 - accuracy: 0.5920 - val_loss: 1.1173 - val_accuracy: 0.6039\n",
      "Epoch 13/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.1104 - accuracy: 0.5952 - val_loss: 1.0979 - val_accuracy: 0.5950\n",
      "Epoch 14/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0918 - accuracy: 0.5967 - val_loss: 1.0809 - val_accuracy: 0.6066\n",
      "Epoch 15/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0766 - accuracy: 0.6045 - val_loss: 1.0679 - val_accuracy: 0.6098\n",
      "Epoch 16/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0649 - accuracy: 0.6103 - val_loss: 1.0574 - val_accuracy: 0.6191\n",
      "Epoch 17/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0542 - accuracy: 0.6165 - val_loss: 1.0480 - val_accuracy: 0.6252\n",
      "Epoch 18/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0450 - accuracy: 0.6214 - val_loss: 1.0401 - val_accuracy: 0.6259\n",
      "Epoch 19/40\n",
      "122/122 [==============================] - 0s 3ms/step - loss: 1.0376 - accuracy: 0.6275 - val_loss: 1.0333 - val_accuracy: 0.6292\n",
      "Epoch 20/40\n",
      "122/122 [==============================] - 0s 3ms/step - loss: 1.0305 - accuracy: 0.6313 - val_loss: 1.0263 - val_accuracy: 0.6362\n",
      "Epoch 21/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0249 - accuracy: 0.6333 - val_loss: 1.0225 - val_accuracy: 0.6399\n",
      "Epoch 22/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0195 - accuracy: 0.6383 - val_loss: 1.0161 - val_accuracy: 0.6360\n",
      "Epoch 23/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0148 - accuracy: 0.6391 - val_loss: 1.0126 - val_accuracy: 0.6406\n",
      "Epoch 24/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0103 - accuracy: 0.6407 - val_loss: 1.0062 - val_accuracy: 0.6469\n",
      "Epoch 25/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0059 - accuracy: 0.6426 - val_loss: 1.0014 - val_accuracy: 0.6469\n",
      "Epoch 26/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0014 - accuracy: 0.6446 - val_loss: 0.9989 - val_accuracy: 0.6471\n",
      "Epoch 27/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9978 - accuracy: 0.6456 - val_loss: 0.9953 - val_accuracy: 0.6467\n",
      "Epoch 28/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9948 - accuracy: 0.6462 - val_loss: 0.9913 - val_accuracy: 0.6516\n",
      "Epoch 29/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9910 - accuracy: 0.6480 - val_loss: 0.9889 - val_accuracy: 0.6529\n",
      "Epoch 30/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9879 - accuracy: 0.6489 - val_loss: 0.9857 - val_accuracy: 0.6507\n",
      "Epoch 31/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9853 - accuracy: 0.6499 - val_loss: 0.9823 - val_accuracy: 0.6547\n",
      "Epoch 32/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9824 - accuracy: 0.6512 - val_loss: 0.9792 - val_accuracy: 0.6555\n",
      "Epoch 33/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9799 - accuracy: 0.6525 - val_loss: 0.9767 - val_accuracy: 0.6570\n",
      "Epoch 34/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9776 - accuracy: 0.6532 - val_loss: 0.9760 - val_accuracy: 0.6557\n",
      "Epoch 35/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9748 - accuracy: 0.6544 - val_loss: 0.9733 - val_accuracy: 0.6585\n",
      "Epoch 36/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9726 - accuracy: 0.6552 - val_loss: 0.9728 - val_accuracy: 0.6496\n",
      "Epoch 37/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9704 - accuracy: 0.6562 - val_loss: 0.9680 - val_accuracy: 0.6583\n",
      "Epoch 38/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9678 - accuracy: 0.6571 - val_loss: 0.9654 - val_accuracy: 0.6617\n",
      "Epoch 39/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9658 - accuracy: 0.6583 - val_loss: 0.9631 - val_accuracy: 0.6621\n",
      "Epoch 40/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9636 - accuracy: 0.6591 - val_loss: 0.9620 - val_accuracy: 0.6639\n",
      "Accuracy: model_DNN_7: 0.6597831325301204\n",
      "Model: \"model_DNN_8\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_9 (InputLayer)         [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 331776 random samples\n",
      "Epoch 1/40\n",
      "243/243 [==============================] - 1s 3ms/step - loss: 2.0141 - accuracy: 0.3054 - val_loss: 1.3721 - val_accuracy: 0.3416\n",
      "Epoch 2/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.3601 - accuracy: 0.3676 - val_loss: 1.3428 - val_accuracy: 0.3886\n",
      "Epoch 3/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.3280 - accuracy: 0.4060 - val_loss: 1.3059 - val_accuracy: 0.4283\n",
      "Epoch 4/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.2858 - accuracy: 0.4635 - val_loss: 1.2639 - val_accuracy: 0.5007\n",
      "Epoch 5/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.2443 - accuracy: 0.5317 - val_loss: 1.2239 - val_accuracy: 0.5729\n",
      "Epoch 6/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.2035 - accuracy: 0.5727 - val_loss: 1.1839 - val_accuracy: 0.5784\n",
      "Epoch 7/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.1659 - accuracy: 0.5819 - val_loss: 1.1504 - val_accuracy: 0.5853\n",
      "Epoch 8/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.1352 - accuracy: 0.5853 - val_loss: 1.1219 - val_accuracy: 0.5867\n",
      "Epoch 9/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.1120 - accuracy: 0.5919 - val_loss: 1.1014 - val_accuracy: 0.5985\n",
      "Epoch 10/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.0952 - accuracy: 0.6020 - val_loss: 1.0869 - val_accuracy: 0.6042\n",
      "Epoch 11/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.0817 - accuracy: 0.6105 - val_loss: 1.0746 - val_accuracy: 0.6127\n",
      "Epoch 12/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.0702 - accuracy: 0.6155 - val_loss: 1.0642 - val_accuracy: 0.6200\n",
      "Epoch 13/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.0597 - accuracy: 0.6217 - val_loss: 1.0533 - val_accuracy: 0.6264\n",
      "Epoch 14/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.0497 - accuracy: 0.6274 - val_loss: 1.0429 - val_accuracy: 0.6339\n",
      "Epoch 15/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.0405 - accuracy: 0.6320 - val_loss: 1.0337 - val_accuracy: 0.6358\n",
      "Epoch 16/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.0310 - accuracy: 0.6376 - val_loss: 1.0242 - val_accuracy: 0.6364\n",
      "Epoch 17/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.0218 - accuracy: 0.6429 - val_loss: 1.0160 - val_accuracy: 0.6453\n",
      "Epoch 18/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.0128 - accuracy: 0.6471 - val_loss: 1.0072 - val_accuracy: 0.6485\n",
      "Epoch 19/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.0051 - accuracy: 0.6504 - val_loss: 1.0005 - val_accuracy: 0.6488\n",
      "Epoch 20/40\n",
      "243/243 [==============================] - 1s 2ms/step - loss: 0.9976 - accuracy: 0.6536 - val_loss: 0.9929 - val_accuracy: 0.6569\n",
      "Epoch 21/40\n",
      "243/243 [==============================] - 1s 2ms/step - loss: 0.9908 - accuracy: 0.6565 - val_loss: 0.9855 - val_accuracy: 0.6568\n",
      "Epoch 22/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9847 - accuracy: 0.6578 - val_loss: 0.9798 - val_accuracy: 0.6606\n",
      "Epoch 23/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9783 - accuracy: 0.6611 - val_loss: 0.9731 - val_accuracy: 0.6620\n",
      "Epoch 24/40\n",
      "243/243 [==============================] - 1s 2ms/step - loss: 0.9722 - accuracy: 0.6633 - val_loss: 0.9664 - val_accuracy: 0.6652\n",
      "Epoch 25/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9664 - accuracy: 0.6646 - val_loss: 0.9614 - val_accuracy: 0.6672\n",
      "Epoch 26/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9612 - accuracy: 0.6668 - val_loss: 0.9567 - val_accuracy: 0.6689\n",
      "Epoch 27/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9564 - accuracy: 0.6683 - val_loss: 0.9527 - val_accuracy: 0.6700\n",
      "Epoch 28/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9518 - accuracy: 0.6702 - val_loss: 0.9507 - val_accuracy: 0.6687\n",
      "Epoch 29/40\n",
      "243/243 [==============================] - 1s 2ms/step - loss: 0.9470 - accuracy: 0.6713 - val_loss: 0.9438 - val_accuracy: 0.6729\n",
      "Epoch 30/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9426 - accuracy: 0.6732 - val_loss: 0.9388 - val_accuracy: 0.6750\n",
      "Epoch 31/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9384 - accuracy: 0.6749 - val_loss: 0.9346 - val_accuracy: 0.6766\n",
      "Epoch 32/40\n",
      "243/243 [==============================] - 1s 2ms/step - loss: 0.9343 - accuracy: 0.6765 - val_loss: 0.9300 - val_accuracy: 0.6779\n",
      "Epoch 33/40\n",
      "243/243 [==============================] - 1s 2ms/step - loss: 0.9305 - accuracy: 0.6773 - val_loss: 0.9310 - val_accuracy: 0.6715\n",
      "Epoch 34/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9262 - accuracy: 0.6793 - val_loss: 0.9229 - val_accuracy: 0.6805\n",
      "Epoch 35/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9226 - accuracy: 0.6808 - val_loss: 0.9189 - val_accuracy: 0.6816\n",
      "Epoch 36/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9189 - accuracy: 0.6816 - val_loss: 0.9155 - val_accuracy: 0.6832\n",
      "Epoch 37/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9147 - accuracy: 0.6839 - val_loss: 0.9112 - val_accuracy: 0.6854\n",
      "Epoch 38/40\n",
      "243/243 [==============================] - 1s 2ms/step - loss: 0.9114 - accuracy: 0.6848 - val_loss: 0.9076 - val_accuracy: 0.6864\n",
      "Epoch 39/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9073 - accuracy: 0.6867 - val_loss: 0.9053 - val_accuracy: 0.6877\n",
      "Epoch 40/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9040 - accuracy: 0.6881 - val_loss: 0.9019 - val_accuracy: 0.6888\n",
      "Accuracy: model_DNN_8: 0.6868012048192771\n",
      "Model: \"model_DNN_9\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_10 (InputLayer)        [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 663552 random samples\n",
      "Epoch 1/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 1.5296 - accuracy: 0.3384 - val_loss: 1.3131 - val_accuracy: 0.4250\n",
      "Epoch 2/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 1.2614 - accuracy: 0.5105 - val_loss: 1.2105 - val_accuracy: 0.5689\n",
      "Epoch 3/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 1.1512 - accuracy: 0.5875 - val_loss: 1.1026 - val_accuracy: 0.6011\n",
      "Epoch 4/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 1.0726 - accuracy: 0.6113 - val_loss: 1.0519 - val_accuracy: 0.6194\n",
      "Epoch 5/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 1.0360 - accuracy: 0.6281 - val_loss: 1.0242 - val_accuracy: 0.6327\n",
      "Epoch 6/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 1.0143 - accuracy: 0.6394 - val_loss: 1.0074 - val_accuracy: 0.6422\n",
      "Epoch 7/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9989 - accuracy: 0.6468 - val_loss: 0.9939 - val_accuracy: 0.6424\n",
      "Epoch 8/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9874 - accuracy: 0.6517 - val_loss: 0.9819 - val_accuracy: 0.6540\n",
      "Epoch 9/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9782 - accuracy: 0.6559 - val_loss: 0.9738 - val_accuracy: 0.6568\n",
      "Epoch 10/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9701 - accuracy: 0.6599 - val_loss: 0.9675 - val_accuracy: 0.6616\n",
      "Epoch 11/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9629 - accuracy: 0.6628 - val_loss: 0.9599 - val_accuracy: 0.6611\n",
      "Epoch 12/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9558 - accuracy: 0.6659 - val_loss: 0.9518 - val_accuracy: 0.6683\n",
      "Epoch 13/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9492 - accuracy: 0.6690 - val_loss: 0.9456 - val_accuracy: 0.6713\n",
      "Epoch 14/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9418 - accuracy: 0.6720 - val_loss: 0.9376 - val_accuracy: 0.6737\n",
      "Epoch 15/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9351 - accuracy: 0.6751 - val_loss: 0.9304 - val_accuracy: 0.6776\n",
      "Epoch 16/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9283 - accuracy: 0.6779 - val_loss: 0.9239 - val_accuracy: 0.6802\n",
      "Epoch 17/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9215 - accuracy: 0.6810 - val_loss: 0.9184 - val_accuracy: 0.6830\n",
      "Epoch 18/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9155 - accuracy: 0.6834 - val_loss: 0.9134 - val_accuracy: 0.6837\n",
      "Epoch 19/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9091 - accuracy: 0.6864 - val_loss: 0.9054 - val_accuracy: 0.6877\n",
      "Epoch 20/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9025 - accuracy: 0.6888 - val_loss: 0.8997 - val_accuracy: 0.6894\n",
      "Epoch 21/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8966 - accuracy: 0.6917 - val_loss: 0.8924 - val_accuracy: 0.6929\n",
      "Epoch 22/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8904 - accuracy: 0.6942 - val_loss: 0.8878 - val_accuracy: 0.6957\n",
      "Epoch 23/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8847 - accuracy: 0.6960 - val_loss: 0.8822 - val_accuracy: 0.6986\n",
      "Epoch 24/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8785 - accuracy: 0.6989 - val_loss: 0.8744 - val_accuracy: 0.6999\n",
      "Epoch 25/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8731 - accuracy: 0.7007 - val_loss: 0.8693 - val_accuracy: 0.7025\n",
      "Epoch 26/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8676 - accuracy: 0.7030 - val_loss: 0.8648 - val_accuracy: 0.7036\n",
      "Epoch 27/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8624 - accuracy: 0.7048 - val_loss: 0.8585 - val_accuracy: 0.7060\n",
      "Epoch 28/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8573 - accuracy: 0.7066 - val_loss: 0.8560 - val_accuracy: 0.7073\n",
      "Epoch 29/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8525 - accuracy: 0.7080 - val_loss: 0.8496 - val_accuracy: 0.7092\n",
      "Epoch 30/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8480 - accuracy: 0.7097 - val_loss: 0.8482 - val_accuracy: 0.7078\n",
      "Epoch 31/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8441 - accuracy: 0.7109 - val_loss: 0.8401 - val_accuracy: 0.7129\n",
      "Epoch 32/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8399 - accuracy: 0.7124 - val_loss: 0.8381 - val_accuracy: 0.7121\n",
      "Epoch 33/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8366 - accuracy: 0.7134 - val_loss: 0.8334 - val_accuracy: 0.7140\n",
      "Epoch 34/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8329 - accuracy: 0.7144 - val_loss: 0.8309 - val_accuracy: 0.7150\n",
      "Epoch 35/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8296 - accuracy: 0.7158 - val_loss: 0.8258 - val_accuracy: 0.7169\n",
      "Epoch 36/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8263 - accuracy: 0.7167 - val_loss: 0.8232 - val_accuracy: 0.7169\n",
      "Epoch 37/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8236 - accuracy: 0.7175 - val_loss: 0.8231 - val_accuracy: 0.7169\n",
      "Epoch 38/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8211 - accuracy: 0.7182 - val_loss: 0.8182 - val_accuracy: 0.7186\n",
      "Epoch 39/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8183 - accuracy: 0.7191 - val_loss: 0.8182 - val_accuracy: 0.7189\n",
      "Epoch 40/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8156 - accuracy: 0.7195 - val_loss: 0.8135 - val_accuracy: 0.7197\n",
      "Accuracy: model_DNN_9: 0.7182590361445783\n"
     ]
    }
   ],
   "source": [
    "accuracies=[]\n",
    "histories=[]\n",
    "models=[]\n",
    "for i_step,n_sub_sample in enumerate(n_samples):\n",
    "        models.append( get_model(\"_{}\".format(i_step)) )\n",
    "        print(\"training on {} random samples\".format(n_sub_sample ))\n",
    "        sub_sample = np.random.choice(range(X_train_val.shape[0]), n_sub_sample)\n",
    "        histories.append(models[-1].fit( X_train_val[sub_sample], y_train_val[sub_sample], batch_size=1024,\n",
    "                        epochs=40, validation_split=0.25, shuffle=True) )\n",
    "        _, a = Accuracy(models[-1])\n",
    "        accuracies.append( a )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAevElEQVR4nO3deXRV5b3/8feXQCCBEKaEQEIYJMxDgQBOpQ4gqBVRqaJtrcgVbdVOv3bVVut8r9auamuvt/eiAtpasQXFaEVEax1wIgFUEuYxCSQEAmHInPP8/kikIQZzQs7JPufk81orC/beT/b54hM+bvb57ueYcw4REQl/7bwuQEREAkOBLiISIRToIiIRQoEuIhIhFOgiIhFCgS4iEiHae/XCvXr1cgMGDPDq5UVEwlJWVtYB51xCY8c8C/QBAwaQmZnp1cuLiIQlM9t9qmO65SIiEiEU6CIiEUKBLiISIRToIiIRQoEuIhIhFOgiIhFCgS4i0kqqany8mVNI4ZHyoJzfsz50EZG2wDlH9t4jLFubR8b6vRw8XskdFw/jlm+cEfDXUqCLiARB4ZFylq/L58W1+WwuPEp0VDumjkjkqvEpTBnS6IOeLaZAFxEJkLLKGt7IKWDZ2nze31qEz8G41G48OGsUl43pS3xsh6C+vgJdRKQFfD7Hml3FvLg2n398vo9jFdUkd4vh1vMHc8W4ZAYldGm1WvwKdDObAfwBiAKecs493OD4Y8D5dZuxQKJzrlsA6xQRCSm7Dx5n2dp8XlqXR25xGZ2jo7h4dB+uGp/C5IE9aNfOWr2mJgPdzKKAJ4BpQB6wxswynHM5X4xxzv2k3vjbgXFBqFVExFMlZVW89vk+lmXlkbn7EGZw7uBe/HTaEKaPTCI22tubHv68+iRgm3NuB4CZLQEuB3JOMf5a4J7AlCci4q3qGh/vbT3A0rV5rMoppLLax+DELvxixjBmjetLn/gYr0s8wZ9ATwZy623nAZMbG2hm/YGBwD9bXpqIiHdy9h7hxbV5LF+/lwPHKuge24FrJ/bjqgkpjE6Ox6z1b6k0JdD/PpgDLHXO1TR20MzmA/MBUlNTA/zSIiItU3S0gpfX57NsbT4b9x2hQ5RxwbDaVsPzhiYS3T60n8X0J9DzgX71tlPq9jVmDnDrqU7knFsALABIT093ftYoIhI05VU1vLmxkGVZeby79QA1PsfYft24//KRXDamL907R3tdot/8CfQ1QJqZDaQ2yOcA1zUcZGbDgO7AhwGtUEQkwJxzZO0+xLK1+bz62V6OllfTJ74TN08ZxJXjkxmcGOd1iaelyUB3zlWb2W3ASmrbFhc657LN7H4g0zmXUTd0DrDEOacrbxEJSbnFpby4Np8X1+Wx+2ApMR2iuHhUEldNSOHMQT2J8qDVMJDMq/xNT093+kxREQm23OJSVmYXsDK7gDW7alsNzxrUkyvHp3DxqCQ6dwyv5yvNLMs5l97YsfD6k4iINME5x+bCo6zcUMjK7AJy9h0BYFhSHD+7aAhXjE8huVvotBoGkgJdRMKez+dYl3uYN+quxHcdLMUMJqR2585LhnPRyN7079nZ6zKDToEuImGpstrHRzsOsjK7gFU5hew/WkGHKOOsM3px05RBTBvRm8S4Tl6X2aoU6CISNkorq3l3SxErswt5c2MhR8uriY2O4ryhCUwfmcR5QxOJjwnuioahTIEuIiHtcGklb27cz8rsAt7bWkR5lY9usR2YPjKJ6SOT+HpaLzp1iPK6zJCgQBeRkFNQUs4bObX3wz/aUUyNz9EnvhNzJqZy0cjeTBrQg/ZRof3UphcU6CISErYXHatrLyzk09zDAJyR0Jmbpwxi+sgkxqSE5vopoUSBLiKecM6xIf/IiR7xrfuPATA2JZ6fTx/K9JFJDE5svQ+HiAQKdBFpNdU1PtbsOnSiMyX/cBlR7YxJA3rwnTP7M21Eb/pGaI94a1Cgi0hQlVfVsHrbAVZmF/Dmxv0UH6+kY/t2fD0tgR9PTWPq8N5htQBWKFOgi0jAOef4NK+EpVm5ZKzfy5HyauI6tefCYYlMH5nElCEJYffIfTjQf1ERCZj9R8t5aW0+S7Py2Lr/GJ06tOPiUX24/Gt9OfuMXiG/nni4U6CLSItUVNfwz437+XtWHu9sKaLG55jQvzsPXzmaS8b0oWuntvugT2tToItIsznnyN57hKVZeSxfn8/h0iqSutauJz57QgqDEtSd4gUFuoj47cCxCpavq72lsqngKNHt2zF9ZBKzJ6Rw7uBeYb+eeLhToIvIV6qq8fHPTftZmpXH25v2U133EW0PzhrFZWP6Eh+rWyqhQoEuIo3auO8If8/M4+X1+Rw8XklCXEfmnTuQ2RNSSOsdnh/RFukU6CJyQvHxSjLW5/P3rDyy9x4hOqodU0ckMntCClPSErR+SohToIu0cdU1Pt7ZUsTSrDze3FhIVY1jdHI8980cycyx4fWp922dAl2kjdpSeJSlWXm8uDafA8cq6Nk5muvPGsDsCSkM79PV6/LkNCjQRdqQktIqMj6t7VL5NK+E9u2MC4Yl8q30fpw3NIEOuqUS1hToIhGuxud4b2sRf8/KY1VOIZXVPoYlxfHrb45g1tf60rNLR69LlABRoItEqL2Hy/jzR7t5cW0ehUcq6B7bgesmpTJ7QgqjkuO9Lk+CQIEuEmHKq2p46r0d/Pfb26iqcZw3JIH7ZqZw/rBEOrbXR7VFMgW6SAR5a2Mh972Sw57iUi4elcSvLhlOvx6xXpclrUSBLhIBdh44zv2vZPP25iLOSOjMX+ZN5ty0Xl6XJa1MgS4Sxkorq3ni7W08+e5Ootu3485LhvO9swdomdo2SoEuEoacc7z62T7+67WN7Csp58rxydwxYxiJXTt5XZp4SIEuEmY2FxzlnowNfLSjmBF9uvLHa8eRPqCH12VJCFCgi4SJkrIqfv/mFp79cDdxndrz4KxRXDspVUvWygkKdJEQ5/M5lq7N45HXN3HweCXXTUrlZxcN1Ror8iV+BbqZzQD+AEQBTznnHm5kzNXAvYADPnXOXRfAOkXapM/yDnP3y9mszz3MhP7dWTx3kh4KklNqMtDNLAp4ApgG5AFrzCzDOZdTb0wa8EvgHOfcITNLDFbBIm3BwWMV/HblZl7IzKVn5448evVYrhiXjJlur8ip+XOFPgnY5pzbAWBmS4DLgZx6Y24CnnDOHQJwzu0PdKEibUF1jY/nPt7D797YTGllDfPOGciPpqYRpw9aFj/4E+jJQG697TxgcoMxQwDMbDW1t2Xudc693vBEZjYfmA+Qmpp6OvWKRKxPdhZz98sb2FRwlHMH9+LemSMYnKhPBhL/BepN0fZAGnAekAK8a2ajnXOH6w9yzi0AFgCkp6e7AL22SFgrKCnnoRUbeXn9XpK7xfCnb49nxqgk3V6RZvMn0POBfvW2U+r21ZcHfOycqwJ2mtkWagN+TUCqFIlAldU+Fq7eyeNvbaXa5/jhBYP5/nmDiYnWAlpyevwJ9DVAmpkNpDbI5wANO1iWA9cCi8ysF7W3YHYEsE6RiPLOliLuy8hmx4HjTB3em7u/OYLUnlpES1qmyUB3zlWb2W3ASmrvjy90zmWb2f1ApnMuo+7YRWaWA9QAP3fOHQxm4SLhKLe4lPtfzWFVTiEDe3Vm0dyJnD9UTWESGOacN7ey09PTXWZmpievLdLayipr+NM72/nfd7bTvp1x+wVp3HjuAK1PLs1mZlnOufTGjulJUZEgcs6xMruAB17dSP7hMmaO7cuvLhlOUrwW0ZLAU6CLBMm2/ce475Vs3tt6gGFJcSyZfyZnDurpdVkSwRToIgFWXlXDo6u2sPD9ncRGR3HvZSP4zpn9aR+lNcoluBToIgH2yOubWbh6J9ek9+PnM4bSq0tHr0uSNkKBLhJAG/JLWPzBTr5zZioPzhrtdTnSxujfgCIB4vM57lq+gR6do/n59GFelyNtkAJdJECeX7OH9bmHufPS4cTHaDEtaX0KdJEAKDpawW9WbOKsQT2Z9bVkr8uRNkqBLhIAD722kbKqGh6YNUqLaolnFOgiLfTB9gO8uC6fm6ecweDELl6XI22YAl2kBSqrffx6+QZSe8Ry2wWDvS5H2ji1LYq0wJPv7WB70XEWzZ1Ipw5al0W8pSt0kdO052Apj7+1lYtHJWnFRAkJCnSR0+Cc456MDbRvZ9x92QivyxEBFOgip2VldgFvby7iJ9OG0Cc+xutyRAAFukizHauo5r5Xchjepys3nD3A63JETlCgizTT71dtoeBIOf95xSitoCghRT+NIs2wcd8RFn2wizkTUxmf2t3rckROokAX8ZPP57jzpc/pFtOBX8wY6nU5Il+iQBfx0wuZuazdc5hfXTKcbrHRXpcj8iUKdBE/HDxWwcMrNjF5YA+uHK/FtyQ0KdBF/PDQik0cr6jmQS2+JSFMgS7ShI93HGRpVh7zpwwirXec1+WInJICXeQrVFb7uGv5BlK6x3D7BWlelyPylbQ4l8hXePr9nWzdf4ynv5dOTLQW35LQpit0kVPILS7lD29tYfrI3lw4vLfX5Yg0SYEucgr3vZJNOzPuuWyk16WI+EWBLtKIN7ILeHPjfn48NY2+3bT4loQHBbpIA8crqrk3I5thSXHMPWeg1+WI+E2BLtLA429tZW9JOQ/OGkUHLb4lYUQ/rSL1bC44ytPv7+Sa9H6kD+jhdTkizeJXoJvZDDPbbGbbzOyORo7fYGZFZra+7us/Al+qSHD5fI67ln9OXKf23HHxMK/LEWm2JvvQzSwKeAKYBuQBa8wswzmX02DoC86524JQo0irWJqVx5pdh3hk9hi6d9biWxJ+/LlCnwRsc87tcM5VAkuAy4NblkjrOnS8kodWbGTigO7MHp/idTkip8WfQE8Gcutt59Xta+gqM/vMzJaaWb/GTmRm880s08wyi4qKTqNckeB4eMUmjpZX8+Cs0bRrp8W3JDwF6k3RV4ABzrkxwCrgmcYGOecWOOfSnXPpCQkJAXppkZbJ3FXMC5m5zPv6QIYmafEtCV/+BHo+UP+KO6Vu3wnOuYPOuYq6zaeACYEpTyS4qmp83PnSBpK7xfCjC7X4loQ3fwJ9DZBmZgPNLBqYA2TUH2BmfeptzgQ2Bq5EkeBZtHonmwuPcu/MkcRGa606CW9N/gQ756rN7DZgJRAFLHTOZZvZ/UCmcy4D+KGZzQSqgWLghiDWLBIQ+YfLeGzVVqYO7820EVp8S8KfX5ckzrnXgNca7Lu73u9/CfwysKWJBNd9GdkA3DtzhMeViASGnhSVNunNnELeyCnkR1PTSOke63U5IgGhQJc2p7SymnsyshnSuwvzztXiWxI59C6QtDl//Oc28g+X8bebz9LiWxJR9NMsbcqWwqM8+e4OvjUhhUkDtfiWRBYFurQZzjnuWr6BLp3a88tLhntdjkjAKdClzVi2Np9PdhZzx4xh9NDiWxKBFOjSJhwureS/XtvIhP7duTq90aWGRMKeAl3ahN+8vomSsioenDVKi29JxFKgS8TL2n2I5z/J5cZzBjC8T1evyxEJGgW6RLTqGh93vvQ5feI78eOpQ7wuRySoFOgS0RZ/sItNBUe557KRdO6oxy4ksinQJWLtKynjsVVbuGBYItNHavEtiXwKdIlY97+SQ41z3DdzJGZ6I1QinwJdItLbm/azYkMBt1+QRr8eWnxL2gYFukScssoa7s7YwODELtz09UFelyPSavQukUScJ97eRm5xGUvmn0l0e12zSNuhn3aJKNv2H+X/3t3OleOTOXNQT6/LEWlVCnSJGBXVNdy1fAOx0e35lRbfkjZIt1wk7BUdreCvH+/hzx/t5sCxCh66cjS9unT0uiyRVqdAl7C1Ib+ERat38cqne6ms8XHe0ARuPGcgU4YkeF2aiCcU6BJWanyOVTkFLFy9i092FhMbHcWcSf343tkDOCOhi9fliXhKgS5hoaSsihfW7OGZD3aTf7iMlO4x3HnJcK6e2I/4mA5elycSEhToEtK2Fx1j8epdLFubR2llDZMH9uDX3xzBtBG9idIyuCInUaBLyHHO8e7WAyx8fyfvbCkiOqodM7/Wl7nnDGBk33ivyxMJWQp0CRmlldUsW5vP4tU72V50nIS4jvx02hCum5yqrhURPyjQxXN5h0r584e7ef6TPRwpr2ZMSjyPXTOWS0f31ZOeIs2gQBdPOOfI3H2Ihe/vZGV2AWbGjJFJzD1nABP6d9fqiCKnQYEuraqiuoZXP93Hog92siH/CPExHZg/5Qy+e1Z/krvFeF2eSFhToEurKDpawXMf7+YvH+3hwLEKBid24T+vGMUV45KJjdaPoUgg6G+SBNWG/BIWrt7Jq5/uo7LGx/lDE7jx3IGcO7iXbquIBJhfgW5mM4A/AFHAU865h08x7ipgKTDROZcZsColrFTX+FiVU8ii1bv4ZFft05zX1j3NOUhPc4oETZOBbmZRwBPANCAPWGNmGc65nAbj4oAfAR8Ho1AJfSWlVbyQefLTnHddOpxvpetpTpHW4M8V+iRgm3NuB4CZLQEuB3IajHsA+A3w84BWKCHHOcfh0ipyD5Wyp7j2a9v+Y6z4vICyqtqnOe++bARTh+tpTpHW5E+gJwO59bbzgMn1B5jZeKCfc+4fZqZAjwDlVTXkHy5jT3EpecX/Du7c4jJyi0s5WlF90vienaO5dEwfPc0p4qEWvylqZu2AR4Eb/Bg7H5gPkJqa2tKXlhbw+RxFxyrqQvrfgZ1XXBviBUfKTxrfsX07+vWIJbVHLJMG9iClewypPWJJ7RlLSvdYunTU++siXvPnb2E+0K/edkrdvi/EAaOAf9V1LSQBGWY2s+Ebo865BcACgPT0dNeCusUPR8uraq+oD/07tL/4Ne9QGRXVvhNjzSCpayf69Yjl3LRe9OseS2rPmNpfe8SSENdRXSkiIc6fQF8DpJnZQGqDfA5w3RcHnXMlQK8vts3sX8DP1OXSOkrKqtiQX3JSWOcWl5J7qIzi45UnjY3r1J7UHrGkJcZx4fDe9OsRS7+6K+3k7jF0bB/l0Z9CRAKhyUB3zlWb2W3ASmrbFhc657LN7H4g0zmXEewipXF7D5dxxf+spvBIBQDt2xkp3WPo1yOWGcnxpPaIPXGFndojlvhYdZqIRDK/bnw6514DXmuw7+5TjD2v5WVJU0rKqrhh0SeUVtTw1PXpDOsTR5/4GHWViLRheicrDFVW+/j+X7LYUXScZ26cxDmDezX9TSIS8RToYcY5xx3LPuOD7Qf53bfGKsxF5AQtNh1mHlu1hRfX5fPTaUO4akKK1+WISAhRoIeRF9bs4fF/buOa9H7cfsFgr8sRkRCjQA8T72wp4lcvbWDKkAQevGKUesJF5EsU6GEge28JP/hLFkN7x/E/3x5PhyhNm4h8mZIhxOUfLmPuojXEx3Rg0dyJesReRE5J6RDCSsqqmLvoE8oqa1j6/bPp3bWT1yWJSAhToIeoL3rNdx44zjNzJzE0Kc7rkkQkxCnQQ1D9XvNHrx7L2eo1FxE/6B56CPqi1/z/TRvClePVay4i/lGgh5j6vea3qddcRJpBgR5C1GsuIi2hQA8RG/LVay4iLaPUCAH5h8u4cbF6zUWkZZQcHlOvuYgEigLdQ5XVPm75s3rNRSQwFOgecc7xi2Wf8eEO9ZqLSGDoHrpHHl21hZfUay4iAaRA98CST/bwR/Wai0iAKdBb2b827+fO5eo1F5HAU6C3og35Jdz63Fr1motIUChRWol6zUUk2JQqreBEr3lVDUtvUa+5iASHrtCDrH6v+f99Z4J6zUUkaHSFHkT1e80fu0a95iISXLpCD6Ives1/dtEQrhinXnMRCS4FepB80Ws+Z2I/bj1fveYiEnwK9CD4otf8G0MSeGCWes1FpHUo0AOsfq/5E+o1F5FWpLQJIPWai4iX/Ap0M5thZpvNbJuZ3dHI8VvM7HMzW29m75vZiMCXGtrq95ovvnGSes1FpNU1GehmFgU8AVwMjACubSSw/+qcG+2c+xrwCPBooAsNZQ17zYf0Vq+5iLQ+f67QJwHbnHM7nHOVwBLg8voDnHNH6m12BlzgSgxt9XvNH5k9Rr3mIuIZf27yJgO59bbzgMkNB5nZrcBPgWjggsZOZGbzgfkAqampza01JKnXXERCRcDeFHXOPeGcOwP4BXDXKcYscM6lO+fSExISAvXSnlGvuYiEEn8CPR/oV287pW7fqSwBZrWgprDw1sZC9ZqLSEjxJ9DXAGlmNtDMooE5QEb9AWaWVm/zUmBr4EoMPX/9eA/z/5zF8D7qNReR0NHkPXTnXLWZ3QasBKKAhc65bDO7H8h0zmUAt5nZVKAKOAR8L5hFe6XG53h4xUaefG8n5w1N4L+vG69ecxEJGX6lkXPuNeC1Bvvurvf7HwW4rpBTWlnNj5es542cQq4/qz93f3ME7XVlLiIhRJeXfig8Us5/PJNJ9t4S7rlsBHPPGeh1SSIiX6JAb8LGfUe4cfEaSsqqePL6dC4c3tvrkkREGqVA/wpvb9rPbX9dS1ynDvzt5rMYlRzvdUkiIqekQD+FZz/cxb0Z2Qzv05WnvzeRpHitzSIioU2B3kCNz/HgP3JYtHoXU4cn8oc54+isThYRCQNKqnqOV1Tzw+fX8dam/dx4zkDuvHQ4Ue30wJCIhAcFep19JWXMW5zJpoIjPHD5SL571gCvSxIRaRYFOrWfMjTvmTUcK6/m6Rsmcv7QRK9LEhFptjYf6G/mFPLDJevoFtOBpd8/m+F9unpdkojIaWmzge6cY9HqXTzwjxxGJ8fz1PXpJOpThkQkjLXJQK+u8XH/qzk8++Fupo/sze+vGUdMdJTXZYmItEibC/Sj5VXc/vw6/rW5iJunDOIXM4bRTp0sIhIB2lSg5x8uY97iNWzdf4yHrhzNtZMi41OTRESgDQX6Z3mHmfdMJuWVNSyeO5Gvp4X/JyaJiNTXJgL99Q0F/PiFdfTs3JHnfjCZIb3jvC5JRCTgIjrQnXM8+d4OHlqxibEp3Xjy+nQS4jp6XZaISFBEbKBX1fi4++Vsnv9kD5eO7sPvrh5Lpw7qZBGRyBWRgX6kvIpbn1vLe1sP8IPzzuBnFw1VJ4uIRLyIC/Tc4lJuXLyGnQeO88jsMVyd3s/rkkREWkVEBfq6PYe46dlMKqt9PDtvEmef0cvrkkREWk3EBPprn+/jJy+sp3fXTiyZP5HBiV28LklEpFWFfaA75/jTO9t55PXNTOjfnQXfnUDPLupkEZG2J6wDvbLax13LP+dvmXnMHNuXR2aPUSeLiLRZYRvoJaVVfP+5LD7YfpAfXpjGT6amYaZOFhFpu8Iy0PccLGXu4k/YU1zKo1eP5crxKV6XJCLiubAL9Kzdxdz0bBY+5/jLvMlMHtTT65JEREJC2AX6rgOlxMd0YOENExnYq7PX5YiIhIywC/SrJqRw6Zg+evNTRKSBdl4XcDoU5iIiXxaWgS4iIl+mQBcRiRB+BbqZzTCzzWa2zczuaOT4T80sx8w+M7O3zKx/4EsVEZGv0mSgm1kU8ARwMTACuNbMRjQYtg5Id86NAZYCjwS6UBER+Wr+XKFPArY553Y45yqBJcDl9Qc45952zpXWbX4E6EkfEZFW5k+gJwO59bbz6vadyjxgRWMHzGy+mWWaWWZRUZH/VYqISJMC+qaomX0HSAd+29hx59wC51y6cy49ISEhkC8tItLm+fNgUT5Q/2N/Uur2ncTMpgJ3At9wzlU0ddKsrKwDZrbb30IbEQ+UtOD7A3W+5nyfP2ObGnOq483Z3ws40EQdrSHc5lDzd7Jwmz9/x37VmNM5Fug5PHXTiXPuK7+oDf0dwEAgGvgUGNlgzDhgO5DW1PkC9QUsCIXzNef7/Bnb1JhTHW/OfiCzteYpkuZQ8xfe8xeIOTydY605h03ecnHOVQO3ASuBjcDfnHPZZna/mc2sG/ZboAvwdzNbb2YZTZ03AF4JkfM15/v8GdvUmFMdb+7+UBBuc6j5O1m4zZ+/Y79qzOkca7U5tLr/W0gbYmaZzrl0r+uQ06P5C3/BmkM9Kdo2LfC6AGkRzV/4C8oc6gpdRCRC6ApdRCRCKNBFRCKEAl1EJEIo0OUkZjbLzJ40sxfM7CKv65HmMbNBZva0mS31uhbxj5l1NrNn6v7efbsl51KgRxAzW2hm+81sQ4P9X7n8cX3OueXOuZuAW4BrglmvnCxA87fDOTcvuJVKU5o5l1cCS+v+3s380smaQYEeWRYDM+rvONXyx2Y22sxebfCVWO9b76r7Pmk9iwnc/Im3FuPnXFK7nMoXCyDWtORFw+5DouXUnHPvmtmABrtPLH8MYGZLgMudcw8B32x4DjMz4GFghXNubZBLlnoCMX8SGpozl9SuYJsCrKeFF9m6Qo98zV3++HZgKjDbzG4JZmHil2bNn5n1NLP/BcaZ2S+DXZw0y6nm8kXgKjP7Ey1cJkBX6HIS59zjwONe1yGnxzl3kNr3PyRMOOeOA3MDcS5doUc+v5Y/lpCl+YscQZ9LBXrkWwOkmdlAM4sG5gCtsRqmBIbmL3IEfS4V6BHEzJ4HPgSGmlmemc071fLHXtYpjdP8RQ6v5lKLc4mIRAhdoYuIRAgFuohIhFCgi4hECAW6iEiEUKCLiEQIBbqISIRQoIuIRAgFuohIhFCgi4hEiP8Pq2lhd8sX3M4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fractions,accuracies)\n",
    "plt.xscale('log')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The non-zero derivative might indicate that this model type is not reaching its best performance, because it could use more data for training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise:\n",
    "reproduce this with proper convergence using callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.callbacks import Callback, EarlyStopping,History,ModelCheckpoint,TensorBoard,ReduceLROnPlateau"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_DNN_0\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_11 (InputLayer)        [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 1296 random samples\n",
      "Epoch 1/40\n",
      "1/1 [==============================] - 0s 169ms/step - loss: 7.2880 - accuracy: 0.1872 - val_loss: 6.8614 - val_accuracy: 0.1975\n",
      "Epoch 2/40\n",
      "1/1 [==============================] - 0s 25ms/step - loss: 7.1292 - accuracy: 0.1872 - val_loss: 6.7098 - val_accuracy: 0.1975\n",
      "Epoch 3/40\n",
      "1/1 [==============================] - 0s 25ms/step - loss: 6.9715 - accuracy: 0.1872 - val_loss: 6.5589 - val_accuracy: 0.1975\n",
      "Epoch 4/40\n",
      "1/1 [==============================] - 0s 25ms/step - loss: 6.8146 - accuracy: 0.1872 - val_loss: 6.4088 - val_accuracy: 0.1975\n",
      "Epoch 5/40\n",
      "1/1 [==============================] - 0s 25ms/step - loss: 6.6586 - accuracy: 0.1872 - val_loss: 6.2595 - val_accuracy: 0.1975\n",
      "Epoch 6/40\n",
      "1/1 [==============================] - 0s 23ms/step - loss: 6.5033 - accuracy: 0.1883 - val_loss: 6.1110 - val_accuracy: 0.1975\n",
      "Epoch 7/40\n",
      "1/1 [==============================] - 0s 24ms/step - loss: 6.3488 - accuracy: 0.1893 - val_loss: 5.9633 - val_accuracy: 0.1975\n",
      "Epoch 8/40\n",
      "1/1 [==============================] - 0s 23ms/step - loss: 6.1951 - accuracy: 0.1903 - val_loss: 5.8164 - val_accuracy: 0.2006\n",
      "Epoch 9/40\n",
      "1/1 [==============================] - 0s 24ms/step - loss: 6.0421 - accuracy: 0.1903 - val_loss: 5.6706 - val_accuracy: 0.2006\n",
      "Epoch 10/40\n",
      "1/1 [==============================] - 0s 25ms/step - loss: 5.8899 - accuracy: 0.1934 - val_loss: 5.5255 - val_accuracy: 0.2130\n",
      "Epoch 11/40\n",
      "1/1 [==============================] - 0s 24ms/step - loss: 5.7387 - accuracy: 0.1965 - val_loss: 5.3814 - val_accuracy: 0.2191\n",
      "Epoch 12/40\n",
      "1/1 [==============================] - 0s 25ms/step - loss: 5.5883 - accuracy: 0.2037 - val_loss: 5.2384 - val_accuracy: 0.2191\n",
      "Epoch 13/40\n",
      "1/1 [==============================] - 0s 24ms/step - loss: 5.4391 - accuracy: 0.2078 - val_loss: 5.0963 - val_accuracy: 0.2191\n",
      "Epoch 14/40\n",
      "1/1 [==============================] - 0s 24ms/step - loss: 5.2913 - accuracy: 0.2078 - val_loss: 4.9552 - val_accuracy: 0.2253\n",
      "Epoch 15/40\n",
      "1/1 [==============================] - 0s 23ms/step - loss: 5.1445 - accuracy: 0.2078 - val_loss: 4.8149 - val_accuracy: 0.2253\n",
      "Epoch 16/40\n",
      "1/1 [==============================] - 0s 24ms/step - loss: 4.9988 - accuracy: 0.2099 - val_loss: 4.6756 - val_accuracy: 0.2253\n",
      "Epoch 17/40\n",
      "1/1 [==============================] - 0s 23ms/step - loss: 4.8542 - accuracy: 0.2109 - val_loss: 4.5376 - val_accuracy: 0.2253\n",
      "Epoch 18/40\n",
      "1/1 [==============================] - 0s 24ms/step - loss: 4.7109 - accuracy: 0.2119 - val_loss: 4.4009 - val_accuracy: 0.2222\n",
      "Epoch 19/40\n",
      "1/1 [==============================] - 0s 23ms/step - loss: 4.5690 - accuracy: 0.2119 - val_loss: 4.2657 - val_accuracy: 0.2222\n",
      "Epoch 20/40\n",
      "1/1 [==============================] - 0s 24ms/step - loss: 4.4287 - accuracy: 0.2140 - val_loss: 4.1319 - val_accuracy: 0.2222\n",
      "Epoch 21/40\n",
      "1/1 [==============================] - 0s 23ms/step - loss: 4.2899 - accuracy: 0.2140 - val_loss: 3.9991 - val_accuracy: 0.2222\n",
      "Epoch 22/40\n",
      "1/1 [==============================] - 0s 23ms/step - loss: 4.1525 - accuracy: 0.2150 - val_loss: 3.8678 - val_accuracy: 0.2222\n",
      "Epoch 23/40\n",
      "1/1 [==============================] - 0s 25ms/step - loss: 4.0165 - accuracy: 0.2150 - val_loss: 3.7380 - val_accuracy: 0.2222\n",
      "Epoch 24/40\n",
      "1/1 [==============================] - 0s 24ms/step - loss: 3.8823 - accuracy: 0.2160 - val_loss: 3.6099 - val_accuracy: 0.2253\n",
      "Epoch 25/40\n",
      "1/1 [==============================] - 0s 35ms/step - loss: 3.7499 - accuracy: 0.2171 - val_loss: 3.4837 - val_accuracy: 0.2222\n",
      "Epoch 26/40\n",
      "1/1 [==============================] - 0s 23ms/step - loss: 3.6195 - accuracy: 0.2171 - val_loss: 3.3594 - val_accuracy: 0.2222\n",
      "Epoch 27/40\n",
      "1/1 [==============================] - 0s 25ms/step - loss: 3.4914 - accuracy: 0.2171 - val_loss: 3.2371 - val_accuracy: 0.2191\n",
      "Epoch 28/40\n",
      "1/1 [==============================] - 0s 25ms/step - loss: 3.3658 - accuracy: 0.2171 - val_loss: 3.1176 - val_accuracy: 0.2191\n",
      "Epoch 29/40\n",
      "1/1 [==============================] - 0s 25ms/step - loss: 3.2429 - accuracy: 0.2171 - val_loss: 3.0012 - val_accuracy: 0.2191\n",
      "Epoch 30/40\n",
      "1/1 [==============================] - 0s 23ms/step - loss: 3.1232 - accuracy: 0.2171 - val_loss: 2.8886 - val_accuracy: 0.2191\n",
      "Epoch 31/40\n",
      "1/1 [==============================] - 0s 24ms/step - loss: 3.0072 - accuracy: 0.2191 - val_loss: 2.7803 - val_accuracy: 0.2191\n",
      "Epoch 32/40\n",
      "1/1 [==============================] - 0s 24ms/step - loss: 2.8955 - accuracy: 0.2202 - val_loss: 2.6764 - val_accuracy: 0.2191\n",
      "Epoch 33/40\n",
      "1/1 [==============================] - 0s 23ms/step - loss: 2.7887 - accuracy: 0.2202 - val_loss: 2.5780 - val_accuracy: 0.2160\n",
      "Epoch 34/40\n",
      "1/1 [==============================] - 0s 23ms/step - loss: 2.6872 - accuracy: 0.2202 - val_loss: 2.4854 - val_accuracy: 0.2160\n",
      "Epoch 35/40\n",
      "1/1 [==============================] - 0s 23ms/step - loss: 2.5915 - accuracy: 0.2212 - val_loss: 2.3993 - val_accuracy: 0.2160\n",
      "Epoch 36/40\n",
      "1/1 [==============================] - 0s 23ms/step - loss: 2.5019 - accuracy: 0.2212 - val_loss: 2.3201 - val_accuracy: 0.2160\n",
      "Epoch 37/40\n",
      "1/1 [==============================] - 0s 24ms/step - loss: 2.4192 - accuracy: 0.2222 - val_loss: 2.2478 - val_accuracy: 0.2160\n",
      "Epoch 38/40\n",
      "1/1 [==============================] - 0s 25ms/step - loss: 2.3430 - accuracy: 0.2222 - val_loss: 2.1827 - val_accuracy: 0.2160\n",
      "Epoch 39/40\n",
      "1/1 [==============================] - 0s 24ms/step - loss: 2.2738 - accuracy: 0.2212 - val_loss: 2.1245 - val_accuracy: 0.2160\n",
      "Epoch 40/40\n",
      "1/1 [==============================] - 0s 23ms/step - loss: 2.2118 - accuracy: 0.2212 - val_loss: 2.0730 - val_accuracy: 0.2191\n",
      "Accuracy: model_DNN_0: 0.23136746987951806\n",
      "Model: \"model_DNN_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_12 (InputLayer)        [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 2592 random samples\n",
      "Epoch 1/40\n",
      "2/2 [==============================] - 0s 88ms/step - loss: 13.1240 - accuracy: 0.1975 - val_loss: 12.4124 - val_accuracy: 0.2006\n",
      "Epoch 2/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 12.7142 - accuracy: 0.1975 - val_loss: 12.0174 - val_accuracy: 0.2006\n",
      "Epoch 3/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 12.3069 - accuracy: 0.1975 - val_loss: 11.6248 - val_accuracy: 0.2006\n",
      "Epoch 4/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 11.9016 - accuracy: 0.1975 - val_loss: 11.2346 - val_accuracy: 0.2006\n",
      "Epoch 5/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 11.5008 - accuracy: 0.1975 - val_loss: 10.8466 - val_accuracy: 0.2006\n",
      "Epoch 6/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 11.1031 - accuracy: 0.1975 - val_loss: 10.4613 - val_accuracy: 0.2006\n",
      "Epoch 7/40\n",
      "2/2 [==============================] - 0s 12ms/step - loss: 10.7057 - accuracy: 0.1975 - val_loss: 10.0797 - val_accuracy: 0.2006\n",
      "Epoch 8/40\n",
      "2/2 [==============================] - 0s 12ms/step - loss: 10.3120 - accuracy: 0.1975 - val_loss: 9.7024 - val_accuracy: 0.2006\n",
      "Epoch 9/40\n",
      "2/2 [==============================] - 0s 12ms/step - loss: 9.9259 - accuracy: 0.1975 - val_loss: 9.3340 - val_accuracy: 0.2006\n",
      "Epoch 10/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 9.5509 - accuracy: 0.1975 - val_loss: 8.9756 - val_accuracy: 0.2006\n",
      "Epoch 11/40\n",
      "2/2 [==============================] - 0s 12ms/step - loss: 9.1810 - accuracy: 0.1975 - val_loss: 8.6257 - val_accuracy: 0.2006\n",
      "Epoch 12/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 8.8227 - accuracy: 0.1975 - val_loss: 8.2824 - val_accuracy: 0.2006\n",
      "Epoch 13/40\n",
      "2/2 [==============================] - 0s 14ms/step - loss: 8.4679 - accuracy: 0.1975 - val_loss: 7.9455 - val_accuracy: 0.2006\n",
      "Epoch 14/40\n",
      "2/2 [==============================] - 0s 14ms/step - loss: 8.1195 - accuracy: 0.1975 - val_loss: 7.6150 - val_accuracy: 0.2006\n",
      "Epoch 15/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 7.7835 - accuracy: 0.1975 - val_loss: 7.2904 - val_accuracy: 0.2006\n",
      "Epoch 16/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 7.4475 - accuracy: 0.1980 - val_loss: 6.9743 - val_accuracy: 0.1991\n",
      "Epoch 17/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 7.1246 - accuracy: 0.1991 - val_loss: 6.6667 - val_accuracy: 0.1975\n",
      "Epoch 18/40\n",
      "2/2 [==============================] - 0s 12ms/step - loss: 6.8063 - accuracy: 0.2001 - val_loss: 6.3688 - val_accuracy: 0.2006\n",
      "Epoch 19/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 6.5038 - accuracy: 0.1991 - val_loss: 6.0809 - val_accuracy: 0.2006\n",
      "Epoch 20/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 6.2124 - accuracy: 0.1996 - val_loss: 5.8042 - val_accuracy: 0.1960\n",
      "Epoch 21/40\n",
      "2/2 [==============================] - 0s 12ms/step - loss: 5.9278 - accuracy: 0.2011 - val_loss: 5.5444 - val_accuracy: 0.1898\n",
      "Epoch 22/40\n",
      "2/2 [==============================] - 0s 14ms/step - loss: 5.6670 - accuracy: 0.2047 - val_loss: 5.3010 - val_accuracy: 0.1914\n",
      "Epoch 23/40\n",
      "2/2 [==============================] - 0s 14ms/step - loss: 5.4161 - accuracy: 0.2083 - val_loss: 5.0724 - val_accuracy: 0.1914\n",
      "Epoch 24/40\n",
      "2/2 [==============================] - 0s 16ms/step - loss: 5.1825 - accuracy: 0.2094 - val_loss: 4.8574 - val_accuracy: 0.1960\n",
      "Epoch 25/40\n",
      "2/2 [==============================] - 0s 14ms/step - loss: 4.9663 - accuracy: 0.2109 - val_loss: 4.6544 - val_accuracy: 0.2068\n",
      "Epoch 26/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 4.7580 - accuracy: 0.2114 - val_loss: 4.4610 - val_accuracy: 0.2068\n",
      "Epoch 27/40\n",
      "2/2 [==============================] - 0s 12ms/step - loss: 4.5630 - accuracy: 0.2104 - val_loss: 4.2717 - val_accuracy: 0.2068\n",
      "Epoch 28/40\n",
      "2/2 [==============================] - 0s 12ms/step - loss: 4.3625 - accuracy: 0.2140 - val_loss: 4.0860 - val_accuracy: 0.2052\n",
      "Epoch 29/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 4.1719 - accuracy: 0.2155 - val_loss: 3.9035 - val_accuracy: 0.2052\n",
      "Epoch 30/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 3.9831 - accuracy: 0.2155 - val_loss: 3.7255 - val_accuracy: 0.1975\n",
      "Epoch 31/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 3.8016 - accuracy: 0.2150 - val_loss: 3.5540 - val_accuracy: 0.1975\n",
      "Epoch 32/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 3.6231 - accuracy: 0.2135 - val_loss: 3.3922 - val_accuracy: 0.1929\n",
      "Epoch 33/40\n",
      "2/2 [==============================] - 0s 14ms/step - loss: 3.4592 - accuracy: 0.2104 - val_loss: 3.2429 - val_accuracy: 0.1914\n",
      "Epoch 34/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 3.3068 - accuracy: 0.2068 - val_loss: 3.1101 - val_accuracy: 0.1883\n",
      "Epoch 35/40\n",
      "2/2 [==============================] - 0s 14ms/step - loss: 3.1715 - accuracy: 0.2052 - val_loss: 2.9959 - val_accuracy: 0.1790\n",
      "Epoch 36/40\n",
      "2/2 [==============================] - 0s 12ms/step - loss: 3.0554 - accuracy: 0.2001 - val_loss: 2.8998 - val_accuracy: 0.1744\n",
      "Epoch 37/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 2.9588 - accuracy: 0.1903 - val_loss: 2.8180 - val_accuracy: 0.1836\n",
      "Epoch 38/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 2.8757 - accuracy: 0.1883 - val_loss: 2.7444 - val_accuracy: 0.1867\n",
      "Epoch 39/40\n",
      "2/2 [==============================] - 0s 13ms/step - loss: 2.7986 - accuracy: 0.1898 - val_loss: 2.6733 - val_accuracy: 0.1929\n",
      "Epoch 40/40\n",
      "2/2 [==============================] - 0s 12ms/step - loss: 2.7228 - accuracy: 0.2114 - val_loss: 2.6013 - val_accuracy: 0.2130\n",
      "Accuracy: model_DNN_1: 0.20822289156626506\n",
      "Model: \"model_DNN_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_13 (InputLayer)        [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 5184 random samples\n",
      "Epoch 1/40\n",
      "4/4 [==============================] - 0s 46ms/step - loss: 4.5030 - accuracy: 0.2034 - val_loss: 4.3069 - val_accuracy: 0.1674\n",
      "Epoch 2/40\n",
      "4/4 [==============================] - 0s 8ms/step - loss: 3.9876 - accuracy: 0.2024 - val_loss: 3.8462 - val_accuracy: 0.1798\n",
      "Epoch 3/40\n",
      "4/4 [==============================] - 0s 8ms/step - loss: 3.5845 - accuracy: 0.2065 - val_loss: 3.5042 - val_accuracy: 0.1921\n",
      "Epoch 4/40\n",
      "4/4 [==============================] - 0s 8ms/step - loss: 3.2948 - accuracy: 0.2086 - val_loss: 3.2614 - val_accuracy: 0.1914\n",
      "Epoch 5/40\n",
      "4/4 [==============================] - 0s 7ms/step - loss: 3.0903 - accuracy: 0.2171 - val_loss: 3.0810 - val_accuracy: 0.2168\n",
      "Epoch 6/40\n",
      "4/4 [==============================] - 0s 7ms/step - loss: 2.9415 - accuracy: 0.2263 - val_loss: 2.9346 - val_accuracy: 0.2176\n",
      "Epoch 7/40\n",
      "4/4 [==============================] - 0s 7ms/step - loss: 2.8176 - accuracy: 0.2299 - val_loss: 2.7975 - val_accuracy: 0.2176\n",
      "Epoch 8/40\n",
      "4/4 [==============================] - 0s 7ms/step - loss: 2.6952 - accuracy: 0.2320 - val_loss: 2.6636 - val_accuracy: 0.2253\n",
      "Epoch 9/40\n",
      "4/4 [==============================] - 0s 8ms/step - loss: 2.5777 - accuracy: 0.2508 - val_loss: 2.5438 - val_accuracy: 0.2546\n",
      "Epoch 10/40\n",
      "4/4 [==============================] - 0s 8ms/step - loss: 2.4722 - accuracy: 0.2749 - val_loss: 2.4377 - val_accuracy: 0.2716\n",
      "Epoch 11/40\n",
      "4/4 [==============================] - 0s 7ms/step - loss: 2.3785 - accuracy: 0.2886 - val_loss: 2.3440 - val_accuracy: 0.2793\n",
      "Epoch 12/40\n",
      "4/4 [==============================] - 0s 7ms/step - loss: 2.2944 - accuracy: 0.2953 - val_loss: 2.2631 - val_accuracy: 0.2901\n",
      "Epoch 13/40\n",
      "4/4 [==============================] - 0s 7ms/step - loss: 2.2203 - accuracy: 0.2994 - val_loss: 2.1921 - val_accuracy: 0.2909\n",
      "Epoch 14/40\n",
      "4/4 [==============================] - 0s 7ms/step - loss: 2.1547 - accuracy: 0.3066 - val_loss: 2.1280 - val_accuracy: 0.2948\n",
      "Epoch 15/40\n",
      "4/4 [==============================] - 0s 8ms/step - loss: 2.0924 - accuracy: 0.3120 - val_loss: 2.0693 - val_accuracy: 0.3056\n",
      "Epoch 16/40\n",
      "4/4 [==============================] - 0s 8ms/step - loss: 2.0336 - accuracy: 0.3248 - val_loss: 2.0147 - val_accuracy: 0.3341\n",
      "Epoch 17/40\n",
      "4/4 [==============================] - 0s 7ms/step - loss: 1.9780 - accuracy: 0.3354 - val_loss: 1.9621 - val_accuracy: 0.3387\n",
      "Epoch 18/40\n",
      "4/4 [==============================] - 0s 8ms/step - loss: 1.9240 - accuracy: 0.3408 - val_loss: 1.9115 - val_accuracy: 0.3441\n",
      "Epoch 19/40\n",
      "4/4 [==============================] - 0s 7ms/step - loss: 1.8730 - accuracy: 0.3447 - val_loss: 1.8618 - val_accuracy: 0.3457\n",
      "Epoch 20/40\n",
      "4/4 [==============================] - 0s 8ms/step - loss: 1.8229 - accuracy: 0.3483 - val_loss: 1.8141 - val_accuracy: 0.3526\n",
      "Epoch 21/40\n",
      "4/4 [==============================] - 0s 9ms/step - loss: 1.7749 - accuracy: 0.3503 - val_loss: 1.7683 - val_accuracy: 0.3495\n",
      "Epoch 22/40\n",
      "4/4 [==============================] - 0s 8ms/step - loss: 1.7303 - accuracy: 0.3449 - val_loss: 1.7250 - val_accuracy: 0.3488\n",
      "Epoch 23/40\n",
      "4/4 [==============================] - 0s 7ms/step - loss: 1.6880 - accuracy: 0.3449 - val_loss: 1.6851 - val_accuracy: 0.3488\n",
      "Epoch 24/40\n",
      "4/4 [==============================] - 0s 7ms/step - loss: 1.6493 - accuracy: 0.3447 - val_loss: 1.6485 - val_accuracy: 0.3480\n",
      "Epoch 25/40\n",
      "4/4 [==============================] - 0s 8ms/step - loss: 1.6143 - accuracy: 0.3449 - val_loss: 1.6155 - val_accuracy: 0.3472\n",
      "Epoch 26/40\n",
      "4/4 [==============================] - 0s 7ms/step - loss: 1.5825 - accuracy: 0.3447 - val_loss: 1.5860 - val_accuracy: 0.3472\n",
      "Epoch 27/40\n",
      "4/4 [==============================] - 0s 7ms/step - loss: 1.5550 - accuracy: 0.3449 - val_loss: 1.5599 - val_accuracy: 0.3488\n",
      "Epoch 28/40\n",
      "4/4 [==============================] - 0s 8ms/step - loss: 1.5299 - accuracy: 0.3441 - val_loss: 1.5373 - val_accuracy: 0.3434\n",
      "Epoch 29/40\n",
      "4/4 [==============================] - 0s 8ms/step - loss: 1.5083 - accuracy: 0.3400 - val_loss: 1.5174 - val_accuracy: 0.3395\n",
      "Epoch 30/40\n",
      "4/4 [==============================] - 0s 8ms/step - loss: 1.4895 - accuracy: 0.3390 - val_loss: 1.5001 - val_accuracy: 0.3372\n",
      "Epoch 31/40\n",
      "4/4 [==============================] - 0s 8ms/step - loss: 1.4732 - accuracy: 0.3385 - val_loss: 1.4844 - val_accuracy: 0.3380\n",
      "Epoch 32/40\n",
      "4/4 [==============================] - 0s 8ms/step - loss: 1.4591 - accuracy: 0.3403 - val_loss: 1.4700 - val_accuracy: 0.3426\n",
      "Epoch 33/40\n",
      "4/4 [==============================] - 0s 7ms/step - loss: 1.4470 - accuracy: 0.3459 - val_loss: 1.4570 - val_accuracy: 0.3472\n",
      "Epoch 34/40\n",
      "4/4 [==============================] - 0s 7ms/step - loss: 1.4341 - accuracy: 0.3488 - val_loss: 1.4453 - val_accuracy: 0.3480\n",
      "Epoch 35/40\n",
      "4/4 [==============================] - 0s 7ms/step - loss: 1.4239 - accuracy: 0.3519 - val_loss: 1.4341 - val_accuracy: 0.3480\n",
      "Epoch 36/40\n",
      "4/4 [==============================] - 0s 8ms/step - loss: 1.4134 - accuracy: 0.3537 - val_loss: 1.4237 - val_accuracy: 0.3495\n",
      "Epoch 37/40\n",
      "4/4 [==============================] - 0s 7ms/step - loss: 1.4047 - accuracy: 0.3542 - val_loss: 1.4135 - val_accuracy: 0.3495\n",
      "Epoch 38/40\n",
      "4/4 [==============================] - 0s 8ms/step - loss: 1.3962 - accuracy: 0.3544 - val_loss: 1.4046 - val_accuracy: 0.3488\n",
      "Epoch 39/40\n",
      "4/4 [==============================] - 0s 7ms/step - loss: 1.3887 - accuracy: 0.3537 - val_loss: 1.3970 - val_accuracy: 0.3488\n",
      "Epoch 40/40\n",
      "4/4 [==============================] - 0s 8ms/step - loss: 1.3827 - accuracy: 0.3542 - val_loss: 1.3903 - val_accuracy: 0.3511\n",
      "Accuracy: model_DNN_2: 0.3448855421686747\n",
      "Model: \"model_DNN_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_14 (InputLayer)        [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 10368 random samples\n",
      "Epoch 1/40\n",
      "8/8 [==============================] - 0s 23ms/step - loss: 4.3043 - accuracy: 0.2027 - val_loss: 4.0979 - val_accuracy: 0.1948\n",
      "Epoch 2/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 3.7077 - accuracy: 0.2027 - val_loss: 3.5135 - val_accuracy: 0.1948\n",
      "Epoch 3/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 3.1668 - accuracy: 0.2033 - val_loss: 2.9941 - val_accuracy: 0.2006\n",
      "Epoch 4/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 2.6957 - accuracy: 0.2140 - val_loss: 2.5684 - val_accuracy: 0.2110\n",
      "Epoch 5/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 2.3302 - accuracy: 0.2468 - val_loss: 2.2497 - val_accuracy: 0.2581\n",
      "Epoch 6/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 2.0607 - accuracy: 0.2851 - val_loss: 2.0105 - val_accuracy: 0.2859\n",
      "Epoch 7/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.8517 - accuracy: 0.3030 - val_loss: 1.8137 - val_accuracy: 0.2897\n",
      "Epoch 8/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.6822 - accuracy: 0.3107 - val_loss: 1.6614 - val_accuracy: 0.2986\n",
      "Epoch 9/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.5624 - accuracy: 0.3162 - val_loss: 1.5645 - val_accuracy: 0.3040\n",
      "Epoch 10/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.4918 - accuracy: 0.3239 - val_loss: 1.5054 - val_accuracy: 0.3137\n",
      "Epoch 11/40\n",
      "8/8 [==============================] - 0s 6ms/step - loss: 1.4499 - accuracy: 0.3381 - val_loss: 1.4655 - val_accuracy: 0.3237\n",
      "Epoch 12/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.4234 - accuracy: 0.3423 - val_loss: 1.4431 - val_accuracy: 0.3233\n",
      "Epoch 13/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.4101 - accuracy: 0.3452 - val_loss: 1.4307 - val_accuracy: 0.3237\n",
      "Epoch 14/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.4012 - accuracy: 0.3465 - val_loss: 1.4223 - val_accuracy: 0.3221\n",
      "Epoch 15/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3941 - accuracy: 0.3466 - val_loss: 1.4159 - val_accuracy: 0.3233\n",
      "Epoch 16/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3878 - accuracy: 0.3470 - val_loss: 1.4110 - val_accuracy: 0.3229\n",
      "Epoch 17/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3831 - accuracy: 0.3472 - val_loss: 1.4071 - val_accuracy: 0.3260\n",
      "Epoch 18/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3791 - accuracy: 0.3461 - val_loss: 1.4037 - val_accuracy: 0.3256\n",
      "Epoch 19/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3760 - accuracy: 0.3470 - val_loss: 1.4003 - val_accuracy: 0.3272\n",
      "Epoch 20/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3734 - accuracy: 0.3465 - val_loss: 1.3972 - val_accuracy: 0.3260\n",
      "Epoch 21/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3712 - accuracy: 0.3470 - val_loss: 1.3943 - val_accuracy: 0.3252\n",
      "Epoch 22/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3691 - accuracy: 0.3468 - val_loss: 1.3921 - val_accuracy: 0.3264\n",
      "Epoch 23/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3675 - accuracy: 0.3461 - val_loss: 1.3898 - val_accuracy: 0.3275\n",
      "Epoch 24/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3658 - accuracy: 0.3465 - val_loss: 1.3879 - val_accuracy: 0.3299\n",
      "Epoch 25/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3645 - accuracy: 0.3466 - val_loss: 1.3865 - val_accuracy: 0.3318\n",
      "Epoch 26/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3633 - accuracy: 0.3471 - val_loss: 1.3852 - val_accuracy: 0.3322\n",
      "Epoch 27/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3623 - accuracy: 0.3495 - val_loss: 1.3844 - val_accuracy: 0.3345\n",
      "Epoch 28/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3614 - accuracy: 0.3495 - val_loss: 1.3834 - val_accuracy: 0.3360\n",
      "Epoch 29/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3605 - accuracy: 0.3517 - val_loss: 1.3826 - val_accuracy: 0.3395\n",
      "Epoch 30/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3596 - accuracy: 0.3548 - val_loss: 1.3816 - val_accuracy: 0.3449\n",
      "Epoch 31/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3587 - accuracy: 0.3619 - val_loss: 1.3807 - val_accuracy: 0.3472\n",
      "Epoch 32/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3578 - accuracy: 0.3603 - val_loss: 1.3798 - val_accuracy: 0.3438\n",
      "Epoch 33/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3571 - accuracy: 0.3605 - val_loss: 1.3792 - val_accuracy: 0.3499\n",
      "Epoch 34/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3562 - accuracy: 0.3654 - val_loss: 1.3780 - val_accuracy: 0.3542\n",
      "Epoch 35/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3554 - accuracy: 0.3714 - val_loss: 1.3773 - val_accuracy: 0.3627\n",
      "Epoch 36/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3546 - accuracy: 0.3803 - val_loss: 1.3767 - val_accuracy: 0.3719\n",
      "Epoch 37/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3539 - accuracy: 0.3879 - val_loss: 1.3756 - val_accuracy: 0.3754\n",
      "Epoch 38/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3529 - accuracy: 0.3849 - val_loss: 1.3747 - val_accuracy: 0.3700\n",
      "Epoch 39/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3522 - accuracy: 0.3800 - val_loss: 1.3738 - val_accuracy: 0.3700\n",
      "Epoch 40/40\n",
      "8/8 [==============================] - 0s 5ms/step - loss: 1.3514 - accuracy: 0.3848 - val_loss: 1.3731 - val_accuracy: 0.3758\n",
      "Accuracy: model_DNN_3: 0.3785301204819277\n",
      "Model: \"model_DNN_4\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_15 (InputLayer)        [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 20736 random samples\n",
      "Epoch 1/40\n",
      "16/16 [==============================] - 0s 14ms/step - loss: 4.6780 - accuracy: 0.1141 - val_loss: 3.9171 - val_accuracy: 0.1678\n",
      "Epoch 2/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 3.4147 - accuracy: 0.1674 - val_loss: 2.7499 - val_accuracy: 0.1534\n",
      "Epoch 3/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 2.3250 - accuracy: 0.1478 - val_loss: 1.8609 - val_accuracy: 0.1508\n",
      "Epoch 4/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.7147 - accuracy: 0.2124 - val_loss: 1.6472 - val_accuracy: 0.2589\n",
      "Epoch 5/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.5903 - accuracy: 0.2781 - val_loss: 1.5262 - val_accuracy: 0.2975\n",
      "Epoch 6/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.4943 - accuracy: 0.3072 - val_loss: 1.4536 - val_accuracy: 0.3225\n",
      "Epoch 7/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.4411 - accuracy: 0.3482 - val_loss: 1.4128 - val_accuracy: 0.3704\n",
      "Epoch 8/40\n",
      "16/16 [==============================] - 0s 3ms/step - loss: 1.4128 - accuracy: 0.3625 - val_loss: 1.3918 - val_accuracy: 0.3542\n",
      "Epoch 9/40\n",
      "16/16 [==============================] - 0s 3ms/step - loss: 1.3975 - accuracy: 0.3482 - val_loss: 1.3800 - val_accuracy: 0.3519\n",
      "Epoch 10/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3887 - accuracy: 0.3564 - val_loss: 1.3724 - val_accuracy: 0.3628\n",
      "Epoch 11/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3828 - accuracy: 0.3632 - val_loss: 1.3673 - val_accuracy: 0.3580\n",
      "Epoch 12/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3778 - accuracy: 0.3589 - val_loss: 1.3629 - val_accuracy: 0.3611\n",
      "Epoch 13/40\n",
      "16/16 [==============================] - 0s 3ms/step - loss: 1.3737 - accuracy: 0.3607 - val_loss: 1.3590 - val_accuracy: 0.3596\n",
      "Epoch 14/40\n",
      "16/16 [==============================] - 0s 3ms/step - loss: 1.3703 - accuracy: 0.3600 - val_loss: 1.3560 - val_accuracy: 0.3652\n",
      "Epoch 15/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3675 - accuracy: 0.3639 - val_loss: 1.3531 - val_accuracy: 0.3659\n",
      "Epoch 16/40\n",
      "16/16 [==============================] - 0s 3ms/step - loss: 1.3645 - accuracy: 0.3693 - val_loss: 1.3497 - val_accuracy: 0.3731\n",
      "Epoch 17/40\n",
      "16/16 [==============================] - 0s 3ms/step - loss: 1.3617 - accuracy: 0.3635 - val_loss: 1.3471 - val_accuracy: 0.3706\n",
      "Epoch 18/40\n",
      "16/16 [==============================] - 0s 3ms/step - loss: 1.3588 - accuracy: 0.3855 - val_loss: 1.3452 - val_accuracy: 0.3875\n",
      "Epoch 19/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3564 - accuracy: 0.3863 - val_loss: 1.3419 - val_accuracy: 0.3835\n",
      "Epoch 20/40\n",
      "16/16 [==============================] - 0s 3ms/step - loss: 1.3533 - accuracy: 0.3830 - val_loss: 1.3391 - val_accuracy: 0.3868\n",
      "Epoch 21/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3509 - accuracy: 0.3983 - val_loss: 1.3360 - val_accuracy: 0.3953\n",
      "Epoch 22/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3478 - accuracy: 0.4057 - val_loss: 1.3330 - val_accuracy: 0.4115\n",
      "Epoch 23/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3449 - accuracy: 0.4071 - val_loss: 1.3302 - val_accuracy: 0.3997\n",
      "Epoch 24/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3419 - accuracy: 0.4039 - val_loss: 1.3272 - val_accuracy: 0.4062\n",
      "Epoch 25/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3394 - accuracy: 0.4028 - val_loss: 1.3247 - val_accuracy: 0.4157\n",
      "Epoch 26/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3366 - accuracy: 0.4220 - val_loss: 1.3215 - val_accuracy: 0.4238\n",
      "Epoch 27/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3336 - accuracy: 0.4182 - val_loss: 1.3185 - val_accuracy: 0.4174\n",
      "Epoch 28/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3307 - accuracy: 0.4267 - val_loss: 1.3156 - val_accuracy: 0.4253\n",
      "Epoch 29/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3277 - accuracy: 0.4304 - val_loss: 1.3122 - val_accuracy: 0.4392\n",
      "Epoch 30/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3244 - accuracy: 0.4413 - val_loss: 1.3092 - val_accuracy: 0.4470\n",
      "Epoch 31/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3219 - accuracy: 0.4476 - val_loss: 1.3059 - val_accuracy: 0.4597\n",
      "Epoch 32/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3188 - accuracy: 0.4448 - val_loss: 1.3024 - val_accuracy: 0.4676\n",
      "Epoch 33/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3157 - accuracy: 0.4543 - val_loss: 1.3005 - val_accuracy: 0.4371\n",
      "Epoch 34/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3125 - accuracy: 0.4640 - val_loss: 1.2973 - val_accuracy: 0.4529\n",
      "Epoch 35/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.3096 - accuracy: 0.4689 - val_loss: 1.2929 - val_accuracy: 0.4699\n",
      "Epoch 36/40\n",
      "16/16 [==============================] - 0s 3ms/step - loss: 1.3064 - accuracy: 0.4643 - val_loss: 1.2893 - val_accuracy: 0.4838\n",
      "Epoch 37/40\n",
      "16/16 [==============================] - 0s 3ms/step - loss: 1.3028 - accuracy: 0.4758 - val_loss: 1.2857 - val_accuracy: 0.4921\n",
      "Epoch 38/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.2979 - accuracy: 0.4878 - val_loss: 1.2787 - val_accuracy: 0.4952\n",
      "Epoch 39/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.2925 - accuracy: 0.5001 - val_loss: 1.2747 - val_accuracy: 0.5017\n",
      "Epoch 40/40\n",
      "16/16 [==============================] - 0s 4ms/step - loss: 1.2886 - accuracy: 0.5055 - val_loss: 1.2706 - val_accuracy: 0.5021\n",
      "Accuracy: model_DNN_4: 0.5023253012048193\n",
      "Model: \"model_DNN_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_16 (InputLayer)        [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 41472 random samples\n",
      "Epoch 1/40\n",
      "31/31 [==============================] - 0s 8ms/step - loss: 7.2843 - accuracy: 0.1953 - val_loss: 3.7695 - val_accuracy: 0.2083\n",
      "Epoch 2/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 2.3196 - accuracy: 0.2083 - val_loss: 1.6612 - val_accuracy: 0.2307\n",
      "Epoch 3/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.5548 - accuracy: 0.3218 - val_loss: 1.4809 - val_accuracy: 0.3458\n",
      "Epoch 4/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.4629 - accuracy: 0.3401 - val_loss: 1.4354 - val_accuracy: 0.3201\n",
      "Epoch 5/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.4298 - accuracy: 0.3195 - val_loss: 1.4123 - val_accuracy: 0.3193\n",
      "Epoch 6/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.4115 - accuracy: 0.3222 - val_loss: 1.3980 - val_accuracy: 0.3316\n",
      "Epoch 7/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3981 - accuracy: 0.3266 - val_loss: 1.3858 - val_accuracy: 0.3362\n",
      "Epoch 8/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3870 - accuracy: 0.3446 - val_loss: 1.3760 - val_accuracy: 0.3687\n",
      "Epoch 9/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3770 - accuracy: 0.3749 - val_loss: 1.3669 - val_accuracy: 0.3854\n",
      "Epoch 10/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3684 - accuracy: 0.3955 - val_loss: 1.3590 - val_accuracy: 0.4062\n",
      "Epoch 11/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3603 - accuracy: 0.3999 - val_loss: 1.3510 - val_accuracy: 0.4006\n",
      "Epoch 12/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3523 - accuracy: 0.4081 - val_loss: 1.3428 - val_accuracy: 0.4158\n",
      "Epoch 13/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3439 - accuracy: 0.4171 - val_loss: 1.3362 - val_accuracy: 0.4073\n",
      "Epoch 14/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3363 - accuracy: 0.4207 - val_loss: 1.3287 - val_accuracy: 0.4246\n",
      "Epoch 15/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3294 - accuracy: 0.4327 - val_loss: 1.3220 - val_accuracy: 0.4334\n",
      "Epoch 16/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3223 - accuracy: 0.4406 - val_loss: 1.3144 - val_accuracy: 0.4447\n",
      "Epoch 17/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3151 - accuracy: 0.4421 - val_loss: 1.3086 - val_accuracy: 0.4680\n",
      "Epoch 18/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3088 - accuracy: 0.4541 - val_loss: 1.3007 - val_accuracy: 0.4627\n",
      "Epoch 19/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.3020 - accuracy: 0.4601 - val_loss: 1.2936 - val_accuracy: 0.4622\n",
      "Epoch 20/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2953 - accuracy: 0.4666 - val_loss: 1.2874 - val_accuracy: 0.4633\n",
      "Epoch 21/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2889 - accuracy: 0.4691 - val_loss: 1.2818 - val_accuracy: 0.4526\n",
      "Epoch 22/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2830 - accuracy: 0.4725 - val_loss: 1.2748 - val_accuracy: 0.4809\n",
      "Epoch 23/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2765 - accuracy: 0.4813 - val_loss: 1.2684 - val_accuracy: 0.4759\n",
      "Epoch 24/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2712 - accuracy: 0.4852 - val_loss: 1.2629 - val_accuracy: 0.5179\n",
      "Epoch 25/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2651 - accuracy: 0.4966 - val_loss: 1.2567 - val_accuracy: 0.5107\n",
      "Epoch 26/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2593 - accuracy: 0.4997 - val_loss: 1.2502 - val_accuracy: 0.5016\n",
      "Epoch 27/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2539 - accuracy: 0.5057 - val_loss: 1.2450 - val_accuracy: 0.4976\n",
      "Epoch 28/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2485 - accuracy: 0.5086 - val_loss: 1.2388 - val_accuracy: 0.5205\n",
      "Epoch 29/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2431 - accuracy: 0.5153 - val_loss: 1.2337 - val_accuracy: 0.5231\n",
      "Epoch 30/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2384 - accuracy: 0.5194 - val_loss: 1.2284 - val_accuracy: 0.5416\n",
      "Epoch 31/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2321 - accuracy: 0.5283 - val_loss: 1.2217 - val_accuracy: 0.5358\n",
      "Epoch 32/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2273 - accuracy: 0.5336 - val_loss: 1.2166 - val_accuracy: 0.5312\n",
      "Epoch 33/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2219 - accuracy: 0.5412 - val_loss: 1.2109 - val_accuracy: 0.5281\n",
      "Epoch 34/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2165 - accuracy: 0.5440 - val_loss: 1.2045 - val_accuracy: 0.5681\n",
      "Epoch 35/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2101 - accuracy: 0.5556 - val_loss: 1.2000 - val_accuracy: 0.5662\n",
      "Epoch 36/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2053 - accuracy: 0.5573 - val_loss: 1.1951 - val_accuracy: 0.5444\n",
      "Epoch 37/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.2005 - accuracy: 0.5604 - val_loss: 1.1884 - val_accuracy: 0.5750\n",
      "Epoch 38/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.1962 - accuracy: 0.5632 - val_loss: 1.1838 - val_accuracy: 0.5788\n",
      "Epoch 39/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.1913 - accuracy: 0.5630 - val_loss: 1.1800 - val_accuracy: 0.5837\n",
      "Epoch 40/40\n",
      "31/31 [==============================] - 0s 3ms/step - loss: 1.1864 - accuracy: 0.5685 - val_loss: 1.1740 - val_accuracy: 0.5851\n",
      "Accuracy: model_DNN_5: 0.5774638554216868\n",
      "Model: \"model_DNN_6\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_17 (InputLayer)        [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 82944 random samples\n",
      "Epoch 1/40\n",
      "61/61 [==============================] - 0s 6ms/step - loss: 3.1502 - accuracy: 0.2611 - val_loss: 1.8222 - val_accuracy: 0.2999\n",
      "Epoch 2/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.5200 - accuracy: 0.3333 - val_loss: 1.4352 - val_accuracy: 0.3545\n",
      "Epoch 3/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.4071 - accuracy: 0.3662 - val_loss: 1.3965 - val_accuracy: 0.3700\n",
      "Epoch 4/40\n",
      "61/61 [==============================] - 0s 2ms/step - loss: 1.3785 - accuracy: 0.3792 - val_loss: 1.3750 - val_accuracy: 0.3894\n",
      "Epoch 5/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.3612 - accuracy: 0.3912 - val_loss: 1.3623 - val_accuracy: 0.3926\n",
      "Epoch 6/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.3496 - accuracy: 0.4057 - val_loss: 1.3505 - val_accuracy: 0.4045\n",
      "Epoch 7/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.3394 - accuracy: 0.4168 - val_loss: 1.3404 - val_accuracy: 0.4118\n",
      "Epoch 8/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.3301 - accuracy: 0.4250 - val_loss: 1.3321 - val_accuracy: 0.4338\n",
      "Epoch 9/40\n",
      "61/61 [==============================] - 0s 2ms/step - loss: 1.3226 - accuracy: 0.4316 - val_loss: 1.3248 - val_accuracy: 0.4310\n",
      "Epoch 10/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.3145 - accuracy: 0.4378 - val_loss: 1.3171 - val_accuracy: 0.4307\n",
      "Epoch 11/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.3053 - accuracy: 0.4420 - val_loss: 1.3046 - val_accuracy: 0.4416\n",
      "Epoch 12/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.2940 - accuracy: 0.4440 - val_loss: 1.2933 - val_accuracy: 0.4467\n",
      "Epoch 13/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.2802 - accuracy: 0.4474 - val_loss: 1.2766 - val_accuracy: 0.4538\n",
      "Epoch 14/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.2664 - accuracy: 0.4574 - val_loss: 1.2666 - val_accuracy: 0.4756\n",
      "Epoch 15/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.2561 - accuracy: 0.4700 - val_loss: 1.2553 - val_accuracy: 0.4680\n",
      "Epoch 16/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.2453 - accuracy: 0.4787 - val_loss: 1.2437 - val_accuracy: 0.4852\n",
      "Epoch 17/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.2333 - accuracy: 0.5005 - val_loss: 1.2310 - val_accuracy: 0.5023\n",
      "Epoch 18/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.2206 - accuracy: 0.5237 - val_loss: 1.2200 - val_accuracy: 0.5451\n",
      "Epoch 19/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.2076 - accuracy: 0.5356 - val_loss: 1.2032 - val_accuracy: 0.5417\n",
      "Epoch 20/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.1942 - accuracy: 0.5489 - val_loss: 1.1915 - val_accuracy: 0.5344\n",
      "Epoch 21/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.1813 - accuracy: 0.5595 - val_loss: 1.1784 - val_accuracy: 0.5602\n",
      "Epoch 22/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.1689 - accuracy: 0.5709 - val_loss: 1.1642 - val_accuracy: 0.5788\n",
      "Epoch 23/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.1547 - accuracy: 0.5815 - val_loss: 1.1506 - val_accuracy: 0.5830\n",
      "Epoch 24/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.1418 - accuracy: 0.5851 - val_loss: 1.1373 - val_accuracy: 0.5931\n",
      "Epoch 25/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.1303 - accuracy: 0.5897 - val_loss: 1.1272 - val_accuracy: 0.5938\n",
      "Epoch 26/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.1201 - accuracy: 0.5921 - val_loss: 1.1184 - val_accuracy: 0.5917\n",
      "Epoch 27/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.1093 - accuracy: 0.5973 - val_loss: 1.1061 - val_accuracy: 0.6026\n",
      "Epoch 28/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.1008 - accuracy: 0.6012 - val_loss: 1.0988 - val_accuracy: 0.5980\n",
      "Epoch 29/40\n",
      "61/61 [==============================] - 0s 2ms/step - loss: 1.0927 - accuracy: 0.6021 - val_loss: 1.0898 - val_accuracy: 0.6127\n",
      "Epoch 30/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0864 - accuracy: 0.6059 - val_loss: 1.0845 - val_accuracy: 0.6141\n",
      "Epoch 31/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0788 - accuracy: 0.6125 - val_loss: 1.0767 - val_accuracy: 0.6166\n",
      "Epoch 32/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0726 - accuracy: 0.6125 - val_loss: 1.0707 - val_accuracy: 0.6186\n",
      "Epoch 33/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0669 - accuracy: 0.6151 - val_loss: 1.0643 - val_accuracy: 0.6216\n",
      "Epoch 34/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0622 - accuracy: 0.6172 - val_loss: 1.0594 - val_accuracy: 0.6220\n",
      "Epoch 35/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0578 - accuracy: 0.6189 - val_loss: 1.0563 - val_accuracy: 0.6216\n",
      "Epoch 36/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0530 - accuracy: 0.6226 - val_loss: 1.0514 - val_accuracy: 0.6225\n",
      "Epoch 37/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0495 - accuracy: 0.6214 - val_loss: 1.0477 - val_accuracy: 0.6258\n",
      "Epoch 38/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0456 - accuracy: 0.6230 - val_loss: 1.0440 - val_accuracy: 0.6284\n",
      "Epoch 39/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0425 - accuracy: 0.6262 - val_loss: 1.0402 - val_accuracy: 0.6311\n",
      "Epoch 40/40\n",
      "61/61 [==============================] - 0s 3ms/step - loss: 1.0389 - accuracy: 0.6294 - val_loss: 1.0366 - val_accuracy: 0.6309\n",
      "Accuracy: model_DNN_6: 0.6295301204819277\n",
      "Model: \"model_DNN_7\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_18 (InputLayer)        [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 165888 random samples\n",
      "Epoch 1/40\n",
      "122/122 [==============================] - 0s 4ms/step - loss: 2.9777 - accuracy: 0.2667 - val_loss: 1.4126 - val_accuracy: 0.3269\n",
      "Epoch 2/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.3789 - accuracy: 0.3419 - val_loss: 1.3606 - val_accuracy: 0.3623\n",
      "Epoch 3/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.3536 - accuracy: 0.3807 - val_loss: 1.3451 - val_accuracy: 0.4043\n",
      "Epoch 4/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.3385 - accuracy: 0.4088 - val_loss: 1.3300 - val_accuracy: 0.4210\n",
      "Epoch 5/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.3242 - accuracy: 0.4239 - val_loss: 1.3158 - val_accuracy: 0.4306\n",
      "Epoch 6/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.3098 - accuracy: 0.4358 - val_loss: 1.3009 - val_accuracy: 0.4392\n",
      "Epoch 7/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.2944 - accuracy: 0.4457 - val_loss: 1.2846 - val_accuracy: 0.4492\n",
      "Epoch 8/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.2765 - accuracy: 0.4616 - val_loss: 1.2636 - val_accuracy: 0.4754\n",
      "Epoch 9/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.2524 - accuracy: 0.4973 - val_loss: 1.2401 - val_accuracy: 0.5049\n",
      "Epoch 10/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.2280 - accuracy: 0.5235 - val_loss: 1.2137 - val_accuracy: 0.5357\n",
      "Epoch 11/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.2029 - accuracy: 0.5472 - val_loss: 1.1886 - val_accuracy: 0.5522\n",
      "Epoch 12/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.1785 - accuracy: 0.5703 - val_loss: 1.1660 - val_accuracy: 0.5877\n",
      "Epoch 13/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.1553 - accuracy: 0.5880 - val_loss: 1.1432 - val_accuracy: 0.5967\n",
      "Epoch 14/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.1344 - accuracy: 0.5972 - val_loss: 1.1244 - val_accuracy: 0.6076\n",
      "Epoch 15/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.1161 - accuracy: 0.6066 - val_loss: 1.1079 - val_accuracy: 0.6042\n",
      "Epoch 16/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0997 - accuracy: 0.6172 - val_loss: 1.0911 - val_accuracy: 0.6186\n",
      "Epoch 17/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0850 - accuracy: 0.6206 - val_loss: 1.0777 - val_accuracy: 0.6270\n",
      "Epoch 18/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0727 - accuracy: 0.6223 - val_loss: 1.0646 - val_accuracy: 0.6162\n",
      "Epoch 19/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0620 - accuracy: 0.6257 - val_loss: 1.0541 - val_accuracy: 0.6230\n",
      "Epoch 20/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0519 - accuracy: 0.6309 - val_loss: 1.0456 - val_accuracy: 0.6317\n",
      "Epoch 21/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0437 - accuracy: 0.6340 - val_loss: 1.0392 - val_accuracy: 0.6324\n",
      "Epoch 22/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0366 - accuracy: 0.6347 - val_loss: 1.0315 - val_accuracy: 0.6368\n",
      "Epoch 23/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0298 - accuracy: 0.6398 - val_loss: 1.0276 - val_accuracy: 0.6339\n",
      "Epoch 24/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0237 - accuracy: 0.6402 - val_loss: 1.0193 - val_accuracy: 0.6436\n",
      "Epoch 25/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0186 - accuracy: 0.6430 - val_loss: 1.0168 - val_accuracy: 0.6391\n",
      "Epoch 26/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0143 - accuracy: 0.6425 - val_loss: 1.0088 - val_accuracy: 0.6459\n",
      "Epoch 27/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0089 - accuracy: 0.6456 - val_loss: 1.0054 - val_accuracy: 0.6447\n",
      "Epoch 28/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0043 - accuracy: 0.6482 - val_loss: 1.0002 - val_accuracy: 0.6463\n",
      "Epoch 29/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 1.0005 - accuracy: 0.6494 - val_loss: 0.9956 - val_accuracy: 0.6523\n",
      "Epoch 30/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9964 - accuracy: 0.6510 - val_loss: 0.9959 - val_accuracy: 0.6389\n",
      "Epoch 31/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9925 - accuracy: 0.6525 - val_loss: 0.9885 - val_accuracy: 0.6517\n",
      "Epoch 32/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9888 - accuracy: 0.6536 - val_loss: 0.9856 - val_accuracy: 0.6558\n",
      "Epoch 33/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9864 - accuracy: 0.6551 - val_loss: 0.9824 - val_accuracy: 0.6570\n",
      "Epoch 34/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9822 - accuracy: 0.6564 - val_loss: 0.9792 - val_accuracy: 0.6575\n",
      "Epoch 35/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9794 - accuracy: 0.6571 - val_loss: 0.9774 - val_accuracy: 0.6556\n",
      "Epoch 36/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9772 - accuracy: 0.6581 - val_loss: 0.9750 - val_accuracy: 0.6577\n",
      "Epoch 37/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9747 - accuracy: 0.6585 - val_loss: 0.9726 - val_accuracy: 0.6594\n",
      "Epoch 38/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9719 - accuracy: 0.6589 - val_loss: 0.9690 - val_accuracy: 0.6620\n",
      "Epoch 39/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9700 - accuracy: 0.6595 - val_loss: 0.9680 - val_accuracy: 0.6632\n",
      "Epoch 40/40\n",
      "122/122 [==============================] - 0s 2ms/step - loss: 0.9670 - accuracy: 0.6617 - val_loss: 0.9653 - val_accuracy: 0.6630\n",
      "Accuracy: model_DNN_7: 0.6608373493975903\n",
      "Model: \"model_DNN_8\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_19 (InputLayer)        [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 331776 random samples\n",
      "Epoch 1/40\n",
      "243/243 [==============================] - 1s 3ms/step - loss: 2.1065 - accuracy: 0.3044 - val_loss: 1.3621 - val_accuracy: 0.3913\n",
      "Epoch 2/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.3376 - accuracy: 0.4061 - val_loss: 1.3153 - val_accuracy: 0.4205\n",
      "Epoch 3/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.2868 - accuracy: 0.4580 - val_loss: 1.2598 - val_accuracy: 0.4882\n",
      "Epoch 4/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.2322 - accuracy: 0.5128 - val_loss: 1.2097 - val_accuracy: 0.5369\n",
      "Epoch 5/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.1833 - accuracy: 0.5439 - val_loss: 1.1633 - val_accuracy: 0.5490\n",
      "Epoch 6/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.1396 - accuracy: 0.5625 - val_loss: 1.1268 - val_accuracy: 0.5651\n",
      "Epoch 7/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.1056 - accuracy: 0.5768 - val_loss: 1.0945 - val_accuracy: 0.5811\n",
      "Epoch 8/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.0789 - accuracy: 0.5896 - val_loss: 1.0730 - val_accuracy: 0.5927\n",
      "Epoch 9/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.0581 - accuracy: 0.6051 - val_loss: 1.0534 - val_accuracy: 0.6097\n",
      "Epoch 10/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.0413 - accuracy: 0.6180 - val_loss: 1.0384 - val_accuracy: 0.6226\n",
      "Epoch 11/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.0279 - accuracy: 0.6279 - val_loss: 1.0261 - val_accuracy: 0.6274\n",
      "Epoch 12/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.0169 - accuracy: 0.6352 - val_loss: 1.0186 - val_accuracy: 0.6364\n",
      "Epoch 13/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.0083 - accuracy: 0.6391 - val_loss: 1.0101 - val_accuracy: 0.6396\n",
      "Epoch 14/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 1.0010 - accuracy: 0.6419 - val_loss: 1.0027 - val_accuracy: 0.6426\n",
      "Epoch 15/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9947 - accuracy: 0.6445 - val_loss: 0.9990 - val_accuracy: 0.6386\n",
      "Epoch 16/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9892 - accuracy: 0.6467 - val_loss: 0.9908 - val_accuracy: 0.6430\n",
      "Epoch 17/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9845 - accuracy: 0.6483 - val_loss: 0.9863 - val_accuracy: 0.6488\n",
      "Epoch 18/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9798 - accuracy: 0.6505 - val_loss: 0.9824 - val_accuracy: 0.6481\n",
      "Epoch 19/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9761 - accuracy: 0.6520 - val_loss: 0.9782 - val_accuracy: 0.6490\n",
      "Epoch 20/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9714 - accuracy: 0.6537 - val_loss: 0.9750 - val_accuracy: 0.6469\n",
      "Epoch 21/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9674 - accuracy: 0.6550 - val_loss: 0.9694 - val_accuracy: 0.6538\n",
      "Epoch 22/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9635 - accuracy: 0.6573 - val_loss: 0.9667 - val_accuracy: 0.6525\n",
      "Epoch 23/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9603 - accuracy: 0.6580 - val_loss: 0.9638 - val_accuracy: 0.6495\n",
      "Epoch 24/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9566 - accuracy: 0.6597 - val_loss: 0.9589 - val_accuracy: 0.6575\n",
      "Epoch 25/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9530 - accuracy: 0.6615 - val_loss: 0.9556 - val_accuracy: 0.6583\n",
      "Epoch 26/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9503 - accuracy: 0.6627 - val_loss: 0.9520 - val_accuracy: 0.6599\n",
      "Epoch 27/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9467 - accuracy: 0.6645 - val_loss: 0.9497 - val_accuracy: 0.6613\n",
      "Epoch 28/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9436 - accuracy: 0.6660 - val_loss: 0.9485 - val_accuracy: 0.6568\n",
      "Epoch 29/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9410 - accuracy: 0.6666 - val_loss: 0.9437 - val_accuracy: 0.6612\n",
      "Epoch 30/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9379 - accuracy: 0.6683 - val_loss: 0.9407 - val_accuracy: 0.6646\n",
      "Epoch 31/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9349 - accuracy: 0.6693 - val_loss: 0.9414 - val_accuracy: 0.6648\n",
      "Epoch 32/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9323 - accuracy: 0.6708 - val_loss: 0.9354 - val_accuracy: 0.6672\n",
      "Epoch 33/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9292 - accuracy: 0.6719 - val_loss: 0.9345 - val_accuracy: 0.6626\n",
      "Epoch 34/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9271 - accuracy: 0.6731 - val_loss: 0.9302 - val_accuracy: 0.6692\n",
      "Epoch 35/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9240 - accuracy: 0.6744 - val_loss: 0.9270 - val_accuracy: 0.6698\n",
      "Epoch 36/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9212 - accuracy: 0.6760 - val_loss: 0.9237 - val_accuracy: 0.6715\n",
      "Epoch 37/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9181 - accuracy: 0.6772 - val_loss: 0.9220 - val_accuracy: 0.6726\n",
      "Epoch 38/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9157 - accuracy: 0.6778 - val_loss: 0.9189 - val_accuracy: 0.6732\n",
      "Epoch 39/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9136 - accuracy: 0.6788 - val_loss: 0.9153 - val_accuracy: 0.6761\n",
      "Epoch 40/40\n",
      "243/243 [==============================] - 0s 2ms/step - loss: 0.9105 - accuracy: 0.6802 - val_loss: 0.9134 - val_accuracy: 0.6755\n",
      "Accuracy: model_DNN_8: 0.6768253012048193\n",
      "Model: \"model_DNN_9\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_20 (InputLayer)        [(None, 16)]              0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "fc3 (Dense)                  (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 7,141\n",
      "Trainable params: 7,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "training on 663552 random samples\n",
      "Epoch 1/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 1.8540 - accuracy: 0.3614 - val_loss: 1.3303 - val_accuracy: 0.4336\n",
      "Epoch 2/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 1.3002 - accuracy: 0.4621 - val_loss: 1.2652 - val_accuracy: 0.5013\n",
      "Epoch 3/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 1.2191 - accuracy: 0.5455 - val_loss: 1.1684 - val_accuracy: 0.5861\n",
      "Epoch 4/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 1.1274 - accuracy: 0.5875 - val_loss: 1.0926 - val_accuracy: 0.5951\n",
      "Epoch 5/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 1.0690 - accuracy: 0.6008 - val_loss: 1.0492 - val_accuracy: 0.6102\n",
      "Epoch 6/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 1.0370 - accuracy: 0.6170 - val_loss: 1.0255 - val_accuracy: 0.6265\n",
      "Epoch 7/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 1.0164 - accuracy: 0.6298 - val_loss: 1.0068 - val_accuracy: 0.6349\n",
      "Epoch 8/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 1.0012 - accuracy: 0.6375 - val_loss: 0.9933 - val_accuracy: 0.6413\n",
      "Epoch 9/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9886 - accuracy: 0.6423 - val_loss: 0.9812 - val_accuracy: 0.6463\n",
      "Epoch 10/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9788 - accuracy: 0.6469 - val_loss: 0.9748 - val_accuracy: 0.6479\n",
      "Epoch 11/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9709 - accuracy: 0.6503 - val_loss: 0.9670 - val_accuracy: 0.6540\n",
      "Epoch 12/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9638 - accuracy: 0.6536 - val_loss: 0.9590 - val_accuracy: 0.6570\n",
      "Epoch 13/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9577 - accuracy: 0.6562 - val_loss: 0.9528 - val_accuracy: 0.6588\n",
      "Epoch 14/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9516 - accuracy: 0.6592 - val_loss: 0.9468 - val_accuracy: 0.6625\n",
      "Epoch 15/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9462 - accuracy: 0.6614 - val_loss: 0.9423 - val_accuracy: 0.6647\n",
      "Epoch 16/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9408 - accuracy: 0.6640 - val_loss: 0.9353 - val_accuracy: 0.6676\n",
      "Epoch 17/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9353 - accuracy: 0.6663 - val_loss: 0.9295 - val_accuracy: 0.6705\n",
      "Epoch 18/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9298 - accuracy: 0.6689 - val_loss: 0.9248 - val_accuracy: 0.6721\n",
      "Epoch 19/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9249 - accuracy: 0.6708 - val_loss: 0.9193 - val_accuracy: 0.6742\n",
      "Epoch 20/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9193 - accuracy: 0.6736 - val_loss: 0.9191 - val_accuracy: 0.6697\n",
      "Epoch 21/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9144 - accuracy: 0.6760 - val_loss: 0.9097 - val_accuracy: 0.6797\n",
      "Epoch 22/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9090 - accuracy: 0.6785 - val_loss: 0.9035 - val_accuracy: 0.6818\n",
      "Epoch 23/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.9038 - accuracy: 0.6805 - val_loss: 0.9061 - val_accuracy: 0.6805\n",
      "Epoch 24/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8982 - accuracy: 0.6829 - val_loss: 0.8929 - val_accuracy: 0.6855\n",
      "Epoch 25/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8925 - accuracy: 0.6857 - val_loss: 0.8888 - val_accuracy: 0.6882\n",
      "Epoch 26/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8868 - accuracy: 0.6879 - val_loss: 0.8827 - val_accuracy: 0.6916\n",
      "Epoch 27/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8813 - accuracy: 0.6903 - val_loss: 0.8758 - val_accuracy: 0.6932\n",
      "Epoch 28/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8758 - accuracy: 0.6925 - val_loss: 0.8746 - val_accuracy: 0.6933\n",
      "Epoch 29/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8698 - accuracy: 0.6946 - val_loss: 0.8665 - val_accuracy: 0.6964\n",
      "Epoch 30/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8634 - accuracy: 0.6975 - val_loss: 0.8584 - val_accuracy: 0.6997\n",
      "Epoch 31/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8576 - accuracy: 0.6997 - val_loss: 0.8523 - val_accuracy: 0.7023\n",
      "Epoch 32/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8517 - accuracy: 0.7021 - val_loss: 0.8488 - val_accuracy: 0.7053\n",
      "Epoch 33/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8470 - accuracy: 0.7041 - val_loss: 0.8431 - val_accuracy: 0.7056\n",
      "Epoch 34/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8424 - accuracy: 0.7061 - val_loss: 0.8370 - val_accuracy: 0.7095\n",
      "Epoch 35/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8374 - accuracy: 0.7080 - val_loss: 0.8332 - val_accuracy: 0.7099\n",
      "Epoch 36/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8332 - accuracy: 0.7096 - val_loss: 0.8297 - val_accuracy: 0.7129\n",
      "Epoch 37/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8289 - accuracy: 0.7114 - val_loss: 0.8261 - val_accuracy: 0.7133\n",
      "Epoch 38/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8249 - accuracy: 0.7127 - val_loss: 0.8237 - val_accuracy: 0.7154\n",
      "Epoch 39/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8215 - accuracy: 0.7141 - val_loss: 0.8215 - val_accuracy: 0.7166\n",
      "Epoch 40/40\n",
      "486/486 [==============================] - 1s 2ms/step - loss: 0.8182 - accuracy: 0.7152 - val_loss: 0.8176 - val_accuracy: 0.7166\n",
      "Accuracy: model_DNN_9: 0.7132771084337349\n"
     ]
    }
   ],
   "source": [
    "accuracies=[]\n",
    "histories=[]\n",
    "models=[]\n",
    "for i_step,n_sub_sample in enumerate(n_samples):\n",
    "        models.append( get_model(\"_{}\".format(i_step)) )\n",
    "        print(\"training on {} random samples\".format(n_sub_sample ))\n",
    "        sub_sample = np.random.choice(range(X_train_val.shape[0]), n_sub_sample)\n",
    "        stopping = EarlyStopping(monitor='val_loss', \n",
    "                         patience=10, \n",
    "                         verbose=1, mode='min')\n",
    "        reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, \n",
    "                              mode='min', verbose=1, min_delta=0.001,\n",
    "                              cooldown=4, min_lr=1e-5)\n",
    "        histories.append(models[-1].fit( X_train_val[sub_sample], y_train_val[sub_sample], batch_size=1024,\n",
    "                                        epochs=40, validation_split=0.25, shuffle=True,\n",
    "                                        callbacks = [ stopping,reduce_lr ]) )\n",
    "        _, a = Accuracy(models[-1])\n",
    "        accuracies.append( a )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAghklEQVR4nO3deXxW1Z3H8c8vgQCBhC0BQhJIgLAjW9iqVrGoUCqKC8XquFRl7JROtau2jrZObbVOrXZedqFWi+1YQASkFdSprXVDTNh3iIGQhEASAklICFmeM38QmUgDPIEnuc/yfb9eeclz78m9PzzJ1+O5yzHnHCIiEvqivC5AREQCQ4EuIhImFOgiImFCgS4iEiYU6CIiYUKBLiISJtp5deKEhASXlpbm1elFRELSunXrSp1zic3t8yzQ09LSyM7O9ur0IiIhyczyzrRPUy4iImFCgS4iEiYU6CIiYUKBLiISJhToIiJhQoEuIhImFOgiIm2kwed4c9tBiitrWuX4nt2HLiISKcqqalmclc8fP8yj8OhxHpgxlHsvGxjw8yjQRURayZaCchau2cfKTQeorfcxeUAPHpo5jCuH926V8/kV6GY2HXgGiAaec849ftr+nwNTGz/GAr2cc90CWKeISEg4Ud/A6i0HWbhmHxv2HyU2Jpqbxqdw25Q0hvSJa9VznzPQzSwaeBa4EigAssxspXNu+ydtnHP3N2n/NWBsK9QqIhK0isqP89La/fzpo/2UHqslPaEzj1wznBvGpxDfsX2b1ODPCH0ikOOcywUws0XAtcD2M7S/GXgkMOWJiAQv5xxr95bx4pp9vLHtED7n+NzQXtw2JY1LBiUQFWVtWo8/gZ4M5Df5XABMaq6hmfUH0oG/XXhpIiLBqbq2nuUbCnnxgzx2Haqka6f23H1JOrdO7k9qj1jP6gr0RdG5wFLnXENzO81sHjAPoF+/fgE+tYhI69pbWsUf1uTx8rp8KmvqGZ4Uz09vuIhrRvelU0y01+X5FeiFQGqTzymN25ozF/jqmQ7knFsALADIzMx0ftYoIuIZn8/x9u5iFn6Qxz92l9AuypgxKonbp/RnfP/umLXttMrZ+BPoWUCGmaVzMsjnAl86vZGZDQW6A2sCWqGIiAeOVtfycnYBf/gwj/1l1fSK68D90wZz88RUesV39Lq8Zp0z0J1z9WY2H3iDk7ctPu+c22ZmjwLZzrmVjU3nAouccxp5i0jI2n6gghfX7GPFxkJq6nxMTOvBd6YP4eoRfWgfHdwP1/s1h+6cWwWsOm3bw6d9/kHgyhIRaTt1DT5e33qQF9fsI2vfETq2j2L22GT+ZXIaw/vGe12e3/SkqIhErOKKGl76aD8vrd1PceUJ+vWI5aGZw7hpfCpdY9vm3vFAUqCLSERxzrEu7wgL1+SxeksR9T7H5UMSeWJKGpcNTmzze8cDSYEuIhHheG0DKzcVsvCDPLYXVRDXsR23fyaNWyf3Jz2hs9flBYQCXUTCztHqWrYXVbCjqJIdRRVsP1BBTvExaht8DOkdx49nj+K6sX2JjQmvCAyvv42IRBSfz5FXVs2OoopTwb2jqIID5f//vvGELh0YlhTHpRlpTB3ai0npPYLq3vFAUqCLSEiorq1n58HKT4X3roOVVNWefDA9OsoYkNCZCek9GJYU3/gVR6+44LxnvDUo0EUkqDjnOFRxgu1F5ewoqjw16t57uIpPnnKJ69COYUnx3JSZyrCkOIYlxTO4dxwd23v/+L2XFOgi4pnaeh85xcf+f9Td+M8j1XWn2qT26MSwPvHMGtOXYUnxDE+KJ6V7p7CdNrkQCnQRaRNHqmpPhfYnFyxziiupazg57O7QLoohfeK4ekSfU1MmQ5Pi2uxd4uFAgS4iraK+wce7OaW8uqGQtXvLKGpyoTIxrgPDkuK5bHAiw5LiGNE3nrSenWkX5I/WBzsFuogEjHOObQcqWLa+kJWbDlB67ARdO7XnssGJjOgbf2rknRjXwetSw5ICXUQuWOHR46zYUMjyDYXkFB+jfbRxxdBezB6bwtShiXRoF9kXK9uKAl1EzktFTR2rtxSxfEMhH+aWAZDZvzuPzR7JzFFJdIuN8bjCyKNAFxG/1TX4eGd3Ccs2FPLX7Yc4Ue8jrWcs908bzOyxyfTr6d3ya6JAF5FzcM6xqaCc5esL+PPmIsqqauke254vTkhl9thkxqR20y2EQUKBLiLNyi+rZvmGQlZsKCS3tIqYdlFcOaw3s8cm89nBicS00x0pwUaBLiKnlFfX8dqWIpZvKCBr3xEAJqX3YN5nBzBjVBJdO+me8GCmQBeJcLX1Pv6+q5jl6wv5285iaht8DEzszLevHsK1Y/qS0l3z4qFCgS4SgZxzrN9/hGXrC3ltSxFHq+tI6BLDLZP7cf3YFEYmx2tePAQp0EUiyL7SqpPz4hsLyTtcTcf2UVw1vA+zxyZzSUZC0C+CLGenQBcJc0eqavnL5gMs21DIhv1HMYMpA3oyf+ogpo/sQ5zelRI2FOgiYSprXxkL3snl7V3F1DU4hvSO44EZQ7l2TF+SunbyujxpBQp0kTBTXFnD46t2smxDIQldOnD7lDRmj0tmeJLmxcOdAl0kTNQ3+Pjjh3n87M3d1NQ38NWpA/nq1EFht26mnJl6WiQMrMsr46EV29hRVMGlGQn8cNYIBiR28bosaWMKdJEQVnrsBE+s3snL6wpI6tqRX90yjukj+2hqJUIp0EVCUIPP8dLaPJ58YxfVtQ3ce9lAvnbFIDp30K90JFPvi4SY9fuP8PCrW9laWMFnBvbk0WtHMKhXnNdlSRBQoIuEiLKqWn76+k4WZeXTO74D/33zWL5wUZKmV+QUBbpIkGvwORZl7eenr++i6kQ98z47gH//XAZdNL0ip/HrJ8LMpgPPANHAc865x5tpMwf4AeCATc65LwWwTpGItCn/KP/x6lY2F5QzKb0H/3ndSAb31vSKNO+cgW5m0cCzwJVAAZBlZiudc9ubtMkAHgQuds4dMbNerVWwSCQ4UlXLk2/u4k8f7SehSweemTuGWaP7anpFzsqfEfpEIMc5lwtgZouAa4HtTdrcAzzrnDsC4JwrDnShIpHA53O8vC6fx1fvpKKmni9fnM590zL0vhXxiz+BngzkN/lcAEw6rc1gADN7n5PTMj9wzr0ekApFIsTWwnIeWrGVjflHmZDWnUevHcmwpHivy5IQEqirKu2ADOByIAV4x8xGOeeONm1kZvOAeQD9+vUL0KlFQlt5dR3/9eYu/rg2j56dY3hqzmhmj03W9Iq0mD+BXgikNvmc0ritqQJgrXOuDthrZrs5GfBZTRs55xYACwAyMzPd+RYtEg58Pscr6wt4fPVOjlTXcvuUNO6/crCWeZPz5k+gZwEZZpbOySCfC5x+B8sK4GbgBTNL4OQUTG4A6xQJK9sOlPPwq9tYl3eEcf268eJdExnRt6vXZUmIO2egO+fqzWw+8AYn58efd85tM7NHgWzn3MrGfVeZ2XagAfi2c+5waxYuEooqaup46s3dvLhmH91iY/jpjRdx47gUoqI0vSIXzpzzZuYjMzPTZWdne3JukbbmnGP5hkJ+vGonh6tOcOuk/nzrqiF0jdX0irSMma1zzmU2t0+Pmom0sp0HK3h4xTY+2lfGmNRuvHDHBEalaHpFAk+BLtJKKmvqePqve/j9B/uI79iOx68fxZzMVE2vSKtRoIsEmHOOlZsO8KPXdlB67AQ3T+zHt68aQvfOMV6XJmFOgS4SQLX1Ph5Ytpll6wu5KKUrz92WyejUbl6XJRFCgS4SIOXH6/jKH9fxwceHuW9aBl+7IoNoTa9IG1KgiwRAwZFqvvz7LPaWVvHUnNFcPy7F65IkAinQRS7QloJyvrwwi5q6BhbeOZHPDErwuiSJUAp0kQvw1o5DzH9pAz06x/DS3ZPI0LvKxUMKdJHz9Ic1+3hk5TZG9O3K7+7IpFdcR69LkginQBdpIZ/P8fjrO1nwTi7ThvXiFzePJTZGv0riPf0UirRATV0D31iykVVbDnLblP48cs0I3ckiQUOBLuKnsqpa7nkxm/X7j/DQzGHcdUm63lkuQUWBLuKHvaVV3PnCRxSV1/DLL41jxqgkr0sS+ScKdJFzWJdXxt0LszEzXrpnMuP7d/e6JJFmKdBFzuK1zUXcv2Qjyd068cIdE0hL6Ox1SSJnpEAXaYZzjgXv5PKT1TvJ7N+dBbdl0kMv15Igp0AXOU19g48f/Hkbf/xwPzMvSuJnN42mY/tor8sSOScFukgTVSfq+dqfNvC3ncX862UD+O7VQ/X+cgkZCnSRRsUVNXx5YRbbD1Two+tGcuvk/l6XJNIiCnQRYPehSu58IYsj1bX87vYJTB3ay+uSRFpMgS4R7/2cUu79wzo6xUSz5F+nMDJZ631KaFKgS0Rbuq6AB17ZzMDELjx/5wSSu3XyuiSR86ZAl4jknOPpv+7hmbf2cMmgBH556zjiO7b3uiyRC6JAl4hTW+/jwWVbeGV9ATeOT+HHs0cR0y7K67JELpgCXSJK03U/v3HlYL52xSC9YEvChgJdIobW/ZRwp0CXiKB1PyUSKNAl7GndT4kUCnQJa3/4MI9HXt2qdT8lIijQJSw1Xffzc0NPrvvZuYN+3CW86Sdcwo7W/ZRI5dfNt2Y23cx2mVmOmT3QzP47zKzEzDY2ft0d+FJFzq2sqpZbnlvLqi0H+f7nh/HDWQpziRznHKGbWTTwLHAlUABkmdlK59z205ouds7Nb4UaRfxSeuwEN/7qg5Prft4yjs9r3U+JMP5MuUwEcpxzuQBmtgi4Fjg90EU845zju0s3c6C8hpfunkRmWg+vSxJpc/5MuSQD+U0+FzRuO90NZrbZzJaaWWpzBzKzeWaWbWbZJSUl51GuSPMWZeXz1s5ivjt9qMJcIlagXmDxZyDNOXcR8L/AwuYaOecWOOcynXOZiYmJATq1RLp9pVX851+2c/Ggntz5mTSvyxHxjD+BXgg0HXGnNG47xTl32Dl3ovHjc8D4wJQncnb1DT7uW7yRdlHGf900WsvFSUTzJ9CzgAwzSzezGGAusLJpAzNrevVpFrAjcCWKnNkv3/6YjflH+dHsUSR11bvMJbKd86Koc67ezOYDbwDRwPPOuW1m9iiQ7ZxbCfy7mc0C6oEy4I5WrFkEgE35R3nmrT3MGt2XWaP7el2OiOfMOefJiTMzM112drYn55bQV11bzxd+8R7H6xp4/eufpWusFqeQyGBm65xzmc3t05OiEpJ+smonuaVVvHT3JIW5SCMt0yIh5++7ivnDh3ncdUm6XoMr0oQCXUJKWVUt31m6mSG94/j21UO8LkckqGjKRUKGc44Hl23maHUtC++cSMf20V6XJBJUNEKXkLF0XQFvbDvEN68awvC+8V6XIxJ0FOgSEvLLqvnhn7czMb0H91w6wOtyRIKSAl2CXoPP8Y0lGwF4as5ovQ5X5Aw0hy5B7zfvfEzWviP87KbRpHSP9bockaClEboEta2F5fz8f3fz+VF9uH5ccy/5FJFPKNAlaNXUNXD/4o10j43hsetGYaapFpGz0ZSLBK0nXt/JnuJjLPzyRLp3jvG6HJGgpxG6BKV395Twwvv7uH1Kfy4brHfni/hDgS5B52h1Ld96eRMDEzvzwIxhXpcjEjI05SJBxTnHQyu2cvhYLc/dNoFOMXoaVMRfGqFLUHl14wH+srmI+6ZlMCqlq9fliIQUBboEjcKjx/mPV7cyvn937r1soNfliIQcBboEBZ/P8a0lm/D5HD+fM4Z20frRFGkp/dZIUPjde3tZk3uYh68ZTr+eehpU5Hwo0MVzO4oqePKNXVw1vDdzMlO9LkckZCnQxVMn6k8+DRrfqR0/uV5Pg4pcCN22KJ762Zu72XmwkufvyKRnlw5elyMS0jRCF8+s+fgwv303ly9N6scVQ3t7XY5IyFOgiycqaur45pKN9O8Ry0Mz9TSoSCBoykU88cir2zhUeYKl904hNkY/hiKBoBG6tLm/bD7A8g2FzJ86iLH9untdjkjYUKBLmzpYXsP3l29ldGo35l8xyOtyRMKKAl3ajM/n+PbSTdTW+/j5nNG019OgIgGl3yhpMwvX7OPdPaV8f+YwBiR28bockbCjQJc2sedQJY+v3snUIYncMqmf1+WIhCUFurS62nof9y3eSOcO7Xjixov0NKhIK/Er0M1supntMrMcM3vgLO1uMDNnZpmBK1FC3dN/3c22AxX85PpR9Irr6HU5ImHrnIFuZtHAs8AMYDhws5kNb6ZdHPB1YG2gi5TQlbWvjF//42PmZKZw9Yg+XpcjEtb8GaFPBHKcc7nOuVpgEXBtM+3+E3gCqAlgfRLCKmvq+MaSjSR378TD14zwuhyRsOdPoCcD+U0+FzRuO8XMxgGpzrnXznYgM5tnZtlmll1SUtLiYiW0PPrn7RQeOc7P54yhSwc9DSrS2i74oqiZRQFPAd88V1vn3ALnXKZzLjMxMfFCTy1B7PWtB3l5XQFfuXwgmWk9vC5HJCL4E+iFQNNVB1Iat30iDhgJvG1m+4DJwEpdGI1cxZU1fG/5FkYmx/P1zw32uhyRiOFPoGcBGWaWbmYxwFxg5Sc7nXPlzrkE51yacy4N+BCY5ZzLbpWKJag55/jO0s1Unajn6S+OIaad7owVaSvn/G1zztUD84E3gB3AEufcNjN71MxmtXaBElr+Z+1+3t5VwoMzhjKoV5zX5YhEFL+uVDnnVgGrTtv28BnaXn7hZUkoyi05xmOv7eDSjARum5LmdTkiEUf/PywBUdfg4/7FG4lpF8WTN44mKkpPg4q0Nd1LJhfMOcfTf93NpoJynv3SOPp01dOgIl5QoMt5O1RRw9J1BSzJzifvcDXXj01m5kVJXpclErEU6NIi9Q0+/r6rhMVZ+/n7rhIafI5J6T24b1oGX7ior9fliUQ0Bbr4ZV9pFYuz83llXQHFlSdIjOvAvM8OYE5mKukJnb0uT0RQoMtZ1NQ1sHprEYuz8vkwt4wogyuG9mJOZipTh/bSikMiQUaBLv9ka2E5S7LzWb6hkMqaevr1iOXbVw/hhnEpuuApEsQU6AJA+fE6Vm46wOKs/WwtrCCmXRQzRvbhixNSmZzeU7chioQABXoEc87x0d4yFmfl89qWIk7U+xiWFM8PZ43gujHJdI1t73WJItICCvQIVFxZwyvrClmSnc/e0iriOrTjxvEpzJ3Qj5HJ8VoiTiREKdAjRH2Dj3/sLmFRVj5/21lMg88xMa0H86cO4vOjkugUE+11iSJygRToYW7/4WqWZOfz8rp8DlWcIKFLDHdfms6czFQGJnbxujwRCSAFehiqqWvgjW0HWZyVzwcfHybK4PIhvfjhrFQ+N0y3G4qEKwV6GNlRVMHirJO3G5YfryO1Rye+eeVgbsxMIalrJ6/LE5FWpkAPccdO1PPqxkKWZOWzqaCcmOgorh7Zh7kTUpkyQLcbikQSBXoIq6lr4Lpn3yen+BhD+8TxyDXDuW5MMt07x3hdmoh4QIEewp79ew45xcf49a3juHpEH91uKBLhFOghavehSn719sdcPy6Z6SP1yloR0YpFIcnnczy4bAtxHdvx0MzhXpcjIkFCgR6CXvpoP+vyjvDQzOH00Hy5iDRSoIeYQxU1PLF6JxcP6sn145K9LkdEgogCPcT8YOU2aht8PHbdKF0EFZFPUaCHkDe3HWT11oN8fVoGaVolSEROo0APEZU1dTz86jaG9onjnksHeF2OiAQh3bYYIn725m4OVdbwq1vH6V0sItIsJUMI2Jh/lIVr9nHb5P6M7dfd63JEJEgp0INcXYOPB17ZTO+4jnzr6iFelyMiQUxTLkHuuXf3svNgJQv+ZTxxHbUknIicmUboQSzvcBVP/3U300f04aoRfbwuR0SCnAI9SDnn+P7yrcRER/GDWSO8LkdEQoBfgW5m081sl5nlmNkDzey/18y2mNlGM3vPzPSCkQu0fEMh7+WU8p0ZQ+nTtaPX5YhICDhnoJtZNPAsMAMYDtzcTGC/5Jwb5ZwbA/wUeCrQhUaSsqpafvTaDsb3784tE/t5XY6IhAh/RugTgRznXK5zrhZYBFzbtIFzrqLJx86AC1yJkeex13ZQcbyOH88epRWHRMRv/tzlkgzkN/lcAEw6vZGZfRX4BhADXNHcgcxsHjAPoF8/jTyb835OKa+sL2D+1EEM6RPndTkiEkICdlHUOfesc24g8F3goTO0WeCcy3TOZSYmJgbq1GGjpq6B7y3fQnpCZ+ZfMcjrckQkxPgT6IVAapPPKY3bzmQRcN0F1BSxfvHWHvIOV/PY7JF0bB/tdTkiEmL8CfQsIMPM0s0sBpgLrGzawMwymnycCewJXImRYUdRBQveyeWm8Sl8ZmCC1+WISAg65xy6c67ezOYDbwDRwPPOuW1m9iiQ7ZxbCcw3s2lAHXAEuL01iw43DY1LynXt1J7vfX6Y1+WISIjy69F/59wqYNVp2x5u8uevB7iuiPI/a/PYmH+UZ+aOobuWlBOR86QnRT1WVH6cn76+i0szEpg1uq/X5YhICFOge+yRV7dR79OSciJy4RToHnp960He3H6I+6cNpl/PWK/LEZEQp0D3SEVNHY+s3MrwpHjuuiTd63JEJAzofegeefL1XZRUnmDBv2TSTkvKiUgAKEk8sC7vCH9cm8cdn0lndGo3r8sRkTChQG9jtfU+vrdsC0nxHfnmVYO9LkdEwoimXNrYb9/NZdehSp67LZPOHfSvX0QCRyP0NrS3tIpn3trDzFFJTBve2+tyRCTMKNDbiHOO7y3bQod2UTxyjRZ0EpHAU6C3kaXrCliTe5gHZwyjV7yWlBORwFOgt4HSYyd4bNUOJqR1Z+6E1HN/g4jIeVCgt4Ef/WU7VSfq+cn1WlJORFqPAr2VvbO7hBUbD/CVywcxqJeWlBOR1qNAb0XHaxv4/ootDEjszL9dPtDrckQkzOlG6Fb09Fu7yS87zqJ5k7WknIi0Oo3QW8m2A+U89+5e5k5IZfKAnl6XIyIRQIHeCj5ZUq57bAwPztCSciLSNhTorWDhB/vYXFDOI9cMp2tse6/LEZEIoUAPsMKjx/mvN3cxdUgiX7goyetyRCSChNxF0bd2HGLZ+kIuHpTApRkJpPYInpV+nHM8vGIrzsGj147UknIi0qZCLtDLqmpZl3eE17YUAdC/Z+zJcB+UwJSBPekWG+NZbau3HuStncU8NHNYUP2HRkQigznnPDlxZmamy87OPq/vdc7xcUkV7+0p4b2cUj7MLePYiXrM4KLkrlw8KIFLMhIY3787Hdq1ze2C5cfrmPbUP+gd34EV/3axViESkVZhZuucc5nN7Qu5ETqAmTGoVxcG9erCHRenU9fgY1P+Ud7LKeW9PaX85p1cfvn2x3RsH8XE9J5cOiiBiwclMLRPXKs9ev/E6zs5fOwEL9wxQWEuIp4IyUA/XfvoKDLTepCZ1oP7pg2msqaOtbllJwM+p5THVu0AIKFLDBc3hvulGQkkde0UkPNn7SvjpbX7uefSdEYmdw3IMUVEWiosAv10cR3bM21471OLSBSVH+f9nMONUzSHeXXjAQAGJHY+NXqfPLAn8R1bfovhifoGHly2heRunbj/Si0pJyLeCctAP11S107cOD6FG8en4Jxj16FK3ttzcvS+JLuAhWvyiI4yRqd05ZKMRC7NSGBMajfa+zF18pt/5JJTfIwX7pxAbExE/OsUkSAVkhdFA6m23sf6/UdOBfzmgqP4HHSOiWbygJ6npmcG9eryT7chflxyjBlPv8vVI/vw3zeP9ehvICKRJOwuigZSTLsoJg/oyeQBPfnW1UMor65jTe5h3ssp4f2cw7y1sxiA3vEdTt490/iV0KUDDy7bQsf2UTz8BS0pJyLei/hAP13X2PZMH9mH6SP7AFBwpJr3c0p5d08pb+8qYdn6QgBSunei4MhxnrhhFIlxHbwsWUQE8DPQzWw68AwQDTznnHv8tP3fAO4G6oES4MvOubwA1+qJlO6xfHFCP744oR8+n2N7UcWp2yMz+3dnTqaWlBOR4HDOOXQziwZ2A1cCBUAWcLNzbnuTNlOBtc65ajP7CnC5c+6LZztusMyhi4iEkrPNofvzBMxEIMc5l+ucqwUWAdc2beCc+7tzrrrx44dAyoUULCIiLedPoCcD+U0+FzRuO5O7gNXN7TCzeWaWbWbZJSUl/lcpIiLnFNBn1M3sViATeLK5/c65Bc65TOdcZmJiYiBPLSIS8fy5KFoINL3yl9K47VPMbBrwfeAy59yJwJQnIiL+8meEngVkmFm6mcUAc4GVTRuY2VjgN8As51xx4MsUEZFzOWegO+fqgfnAG8AOYIlzbpuZPWpmsxqbPQl0AV42s41mtvIMhxMRkVbi133ozrlVwKrTtj3c5M/TAlyXiIi0kF7cLSISJjx7OZeZlQAX8jRpV6A8QOVcyPFa8n3+tD1XmzPtb8n2BKD0HHW0hVDrQ/Xfp4Va//nb9mxtzmdfoPuwv3Ou+dsEnXMh+QUsCIbjteT7/Gl7rjZn2t+S7UC21/0Xin2o/gvt/gtEH57Pvrbsw1CecvlzkByvJd/nT9tztTnT/pZuDwah1ofqv08Ltf7zt+3Z2pzPvjbrQ8+mXMQ7ZpbtzvAuCAl+6r/Q11p9GMojdDl/C7wuQC6I+i/0tUofaoQuIhImNEIXEQkTCnQRkTChQBcRCRMKdPkUM7vOzH5rZovN7Cqv65GWMbMBZvY7M1vqdS3iHzPrbGYLG3/vbrmQYynQw4iZPW9mxWa29bTt081sl5nlmNkDZzuGc26Fc+4e4F7grMsISmAFqP9ynXN3tW6lci4t7MvrgaWNv3ez/ulgLaBADy+/B6Y33dC4JuyzwAxgOHCzmQ03s1Fm9pfTvno1+daHGr9P2s7vCVz/ibd+j599yck1Jj5ZFa7hQk7q19sWJTQ4594xs7TTNp9aExbAzBYB1zrnfgJ84fRjmJkBjwOrnXPrW7lkaSIQ/SfBoSV9ycllPVOAjVzgIFsj9PDX0jVhvwZMA240s3tbszDxS4v6z8x6mtmvgbFm9mBrFyctcqa+XAbcYGa/4gJfE6ARunyKc+4XwC+8rkPOj3PuMCevf0iIcM5VAXcG4lgaoYc/v9aElaCl/gsfrd6XCvTwd841YSWoqf/CR6v3pQI9jJjZn4A1wBAzKzCzu9wZ1oT1sk5pnvovfHjVl3o5l4hImNAIXUQkTCjQRUTChAJdRCRMKNBFRMKEAl1EJEwo0EVEwoQCXUQkTCjQRUTChAJdRCRM/B8Q/ujPH8BkxAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fractions,accuracies)\n",
    "plt.xscale('log')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise:\n",
    "reproduce this with error bars (K-folding)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Challenge:\n",
    "Can one predict the asymptotic performance from the performance at smaller training dataset size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}