cnn_final.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import librosa\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.utils import to_categorical\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create labels\n",
    "Input: Folder Path   \n",
    "Output: Tuple (Label, Indices of the labels, one-hot encoded labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = \"./eng_data/\"\n",
    "\n",
    "def get_labels(path=DATA_PATH):\n",
    "    labels = os.listdir(path)\n",
    "    label_indices = np.arange(0, len(labels))\n",
    "    return labels, label_indices, to_categorical(label_indices)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Converting wave to mfcc\n",
    "Input: path of file, maximum pad length(default=11)   \n",
    "Output: list of mfcc vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def wav2mfcc(file_path, max_len=11):\n",
    "    wave, sr = librosa.load(file_path, mono=True, sr=None)\n",
    "    wave = wave[::3]\n",
    "    try:\n",
    "        mfcc = librosa.feature.mfcc(wave, sr=16000)\n",
    "    except Exception as e:\n",
    "        print(file_path)\n",
    "        print(e)\n",
    "        \n",
    "    # If maximum length exceeds mfcc lengths then pad the remaining ones\n",
    "    if (max_len > mfcc.shape[1]):\n",
    "        pad_width = max_len - mfcc.shape[1]\n",
    "        mfcc = np.pad(mfcc, pad_width=((0, 0), (0, pad_width)), mode='constant')\n",
    "\n",
    "    # Else cutoff the remaining parts\n",
    "    else:\n",
    "        mfcc = mfcc[:, :max_len]\n",
    "    \n",
    "    return mfcc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving mfcc vectors to .npy files\n",
    "Input: path to write files   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_data_to_array(path=DATA_PATH, max_len=11):\n",
    "    labels, _, _ = get_labels(path)\n",
    "\n",
    "    for label in labels:\n",
    "        mfcc_vectors = []\n",
    "        wavfiles = [path + label + '/' + wavfile for wavfile in os.listdir(path + '/' + label)]\n",
    "        \n",
    "        for wavfile in tqdm(wavfiles, \"Saving vectors of label - '{}'\".format(label)):\n",
    "            mfcc = wav2mfcc(wavfile, max_len=max_len)\n",
    "            mfcc_vectors.append(mfcc)\n",
    "            \n",
    "        np.save(label + '.npy', mfcc_vectors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dividing data into training set and testing set\n",
    "Input: split ratio (default=0.8)    \n",
    "Output: 2 arrays of training and testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_train_test(split_ratio=0.8):\n",
    "    # Get available labels\n",
    "    labels, indices, _ = get_labels(DATA_PATH)\n",
    "\n",
    "    # Getting first arrays\n",
    "    X = np.load(labels[0] + '.npy')\n",
    "    y = np.zeros(X.shape[0])\n",
    "\n",
    "    # Append all of the dataset into one single array, same goes for y\n",
    "    for i, label in enumerate(labels[1:]):\n",
    "        x = np.load(label + '.npy')\n",
    "        X = np.vstack((X, x))\n",
    "        y = np.append(y, np.full(x.shape[0], fill_value= (i+1)))\n",
    "\n",
    "    assert X.shape[0] == len(y)\n",
    "\n",
    "    return train_test_split(X, y, test_size= (1 - split_ratio), shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialization by calling the necessay functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D, Activation, BatchNormalization\n",
    "from keras.utils import to_categorical\n",
    "\n",
    "# Second dimension of the feature is dim2\n",
    "feature_dim_2 = 11\n",
    "\n",
    "# Save data to array file first (Comment next line when npy files are created to save time)\n",
    "# save_data_to_array(max_len=feature_dim_2)\n",
    "\n",
    "# Loading train set and test set\n",
    "X_train, X_test, y_train, y_test = get_train_test()\n",
    "\n",
    "# Feature dimension\n",
    "feature_dim_1 = 20\n",
    "channel = 1\n",
    "epochs = 20\n",
    "batch_size = 100\n",
    "verbose = 1\n",
    "num_classes = 7\n",
    "\n",
    "# Reshaping to perform 2D convolution\n",
    "X_train = X_train.reshape(X_train.shape[0], feature_dim_1, feature_dim_2, channel)\n",
    "X_test = X_test.reshape(X_test.shape[0], feature_dim_1, feature_dim_2, channel)\n",
    "\n",
    "y_train_hot = to_categorical(y_train)\n",
    "y_test_hot = to_categorical(y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model():\n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(32, kernel_size=(2, 2), activation='relu', input_shape=(feature_dim_1, feature_dim_2, channel)))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Conv2D(48, kernel_size=(2, 2), activation='relu'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Conv2D(120, kernel_size=(2, 2), activation='relu'))\n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(128, activation='relu'))\n",
    "    model.add(Dense(64, activation='relu'))\n",
    "    model.add(Dense(num_classes, activation='softmax'))\n",
    "    \n",
    "    model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "                  optimizer=keras.optimizers.Adadelta(),\n",
    "                  metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predicts a sample\n",
    "def predict(filepath, model):\n",
    "    sample = wav2mfcc(filepath)\n",
    "    sample_reshaped = sample.reshape(1, feature_dim_1, feature_dim_2, channel)\n",
    "    return get_labels()[0][\n",
    "            np.argmax(model.predict(sample_reshaped))\n",
    "    ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Starting the training process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13256, 20, 11, 1)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 13256 samples, validate on 3314 samples\n",
      "Epoch 1/15\n",
      "13256/13256 [==============================] - 41s 3ms/step - loss: 1.1219 - acc: 0.6072 - val_loss: 0.6894 - val_acc: 0.7634\n",
      "Epoch 2/15\n",
      "13256/13256 [==============================] - 40s 3ms/step - loss: 0.4434 - acc: 0.8489 - val_loss: 0.4341 - val_acc: 0.8488\n",
      "Epoch 3/15\n",
      "13256/13256 [==============================] - 38s 3ms/step - loss: 0.2765 - acc: 0.9035 - val_loss: 0.4309 - val_acc: 0.8627\n",
      "Epoch 4/15\n",
      "13256/13256 [==============================] - 32s 2ms/step - loss: 0.1758 - acc: 0.9400 - val_loss: 0.3629 - val_acc: 0.8871\n",
      "Epoch 5/15\n",
      "13256/13256 [==============================] - 38s 3ms/step - loss: 0.1055 - acc: 0.9641 - val_loss: 0.3610 - val_acc: 0.8956\n",
      "Epoch 6/15\n",
      "13256/13256 [==============================] - 39s 3ms/step - loss: 0.0643 - acc: 0.9799 - val_loss: 0.3418 - val_acc: 0.9013\n",
      "Epoch 7/15\n",
      "13256/13256 [==============================] - 39s 3ms/step - loss: 0.0371 - acc: 0.9885 - val_loss: 0.4298 - val_acc: 0.8890\n",
      "Epoch 8/15\n",
      "13256/13256 [==============================] - 38s 3ms/step - loss: 0.0153 - acc: 0.9972 - val_loss: 0.3854 - val_acc: 0.9040\n",
      "Epoch 9/15\n",
      "13256/13256 [==============================] - 38s 3ms/step - loss: 0.0135 - acc: 0.9964 - val_loss: 0.3782 - val_acc: 0.9110\n",
      "Epoch 10/15\n",
      "13256/13256 [==============================] - 37s 3ms/step - loss: 0.0071 - acc: 0.9983 - val_loss: 0.5060 - val_acc: 0.8986\n",
      "Epoch 11/15\n",
      "13256/13256 [==============================] - 41s 3ms/step - loss: 0.0062 - acc: 0.9986 - val_loss: 0.3985 - val_acc: 0.9155\n",
      "Epoch 12/15\n",
      "13256/13256 [==============================] - 40s 3ms/step - loss: 0.0018 - acc: 0.9999 - val_loss: 0.4242 - val_acc: 0.9158\n",
      "Epoch 13/15\n",
      "13256/13256 [==============================] - 38s 3ms/step - loss: 9.4748e-04 - acc: 0.9999 - val_loss: 0.4175 - val_acc: 0.9137\n",
      "Epoch 14/15\n",
      "13256/13256 [==============================] - 32s 2ms/step - loss: 2.4696e-04 - acc: 1.0000 - val_loss: 0.4319 - val_acc: 0.9179\n",
      "Epoch 15/15\n",
      "13256/13256 [==============================] - 33s 2ms/step - loss: 1.1744e-04 - acc: 1.0000 - val_loss: 0.4275 - val_acc: 0.9149\n"
     ]
    }
   ],
   "source": [
    "model = get_model()\n",
    "history = model.fit(X_train, y_train_hot, batch_size=batch_size, epochs=15, verbose=verbose, validation_data=(X_test, y_test_hot))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model visualized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['val_loss', 'val_acc', 'loss', 'acc'])\n"
     ]
    }
   ],
   "source": [
    "print(history.history.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reset css and font defaults in:\r\n",
      "/home/subtleseeker/.jupyter/custom &\r\n",
      "/home/subtleseeker/.local/share/jupyter/nbextensions\r\n"
     ]
    }
   ],
   "source": [
    "!jt -r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/subtleseeker/anaconda3/lib/python3.6/site-packages/matplotlib/figure.py:98: MatplotlibDeprecationWarning: \n",
      "Adding an axes using the same arguments as a previous axes currently reuses the earlier instance.  In a future version, a new instance will always be created and returned.  Meanwhile, this warning can be suppressed, and the future behavior ensured, by passing a unique label to each axes instance.\n",
      "  \"Adding an axes using the same arguments as a previous axes \"\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8XHW5+PHPk8naJE3apE3bpEsKXWmBlrAWJBVKC7IJKFRRQLCioiji7+IVAUGlePUq3nLFXq2gIBVBpGKhrJFVu+9J2rSFkibTJV2StNlm5vn9cU7aaZpl0uZkJsnzfr3Oa+ac8z0zz4Qyz3yX8/2KqmKMMca0Jy7aARhjjIl9liyMMcZ0yJKFMcaYDlmyMMYY0yFLFsYYYzpkycIYY0yHLFkYY4zpkCULY4wxHbJkYYwxpkPx0Q6gq2RnZ+uoUaOiHYYxxvQoK1as2KOqgzoq12uSxahRo1i+fHm0wzDGmB5FRD6KpJw1QxljjOmQJQtjjDEdsmRhjDGmQ72mz6I1TU1NlJeXU19fH+1Quk1ycjJ5eXkkJCREOxRjTC/Sq5NFeXk56enpjBo1ChGJdjieU1WqqqooLy8nPz8/2uEYY3oRz5qhRGSBiOwSkfVtnBcR+ZWIlInIWhGZGnbuJhHZ7G43HW8M9fX1ZGVl9YlEASAiZGVl9amalDGme3jZZ/EEMKud85cCY9xtDvBrABEZCNwPnA2cBdwvIgOON4i+kiia9bXPa4zpHp41Q6nq2yIyqp0iVwF/UGdd13+JSKaIDAUKgddUdS+AiLyGk3Se8SpWY0x0qCqBkNIYCNEQCLmPwWOeH9kP0dAUpDEYoqEpRGMwRMiDpaFbvmT48tPNT/WYfW31fKsv2MWGZKTwubNHePoe0eyzyAU+Dtsvd4+1dfwYIjIHp1ZCTk4ORUVFR53PyMigpqam6yLupKqqKq688koAdu7cic/nIzs7G4C33nqLxMTEDl/jq1/9KnfddRdjxoyJ+H3r6+uP+VuY6KkLKA1BJV4EXxz4xNnixLuaYEiVxiDUB93HgNIQhIagUh9wHhvc8w3ufr17vjHofLcpRz+GANT5Ugy1cl7h8HHc8kfOK0GFphAEQtAUUppC0BRs8aXai3lZ5x+dEcewuq0evkN0k0Vrfztt5/ixB1XnA/MBCgoKtLCw8KjzxcXFpKenn1iUJyA9PZ21a9cC8MADD5CWlsbdd999VBlVRVWJi2u9RfCpp57q9PsmJyczZcqUzgdsTtjBhgAbKqpZW76fdTsOsG7HAbbuPtRm+QSfEB8XR7xPSPDF4YsTEuKEeJ97zD0X74tzjzvl4uMEX1wcDYEgBxsCHGoMuluAgw1B6pqCnYo7JcFHapKPlEQfyfE+fHGCiBAnEOc++lrsiwiCux/nPB45dmTfKQvxcXEkxceRlBBHoi+OpASf89jasYQ4kuJ9JMY755sfnc13+FhifBxxHiXc5pcV9ysp/G3kcBlpsX/08d4kmsmiHBgetp8HVLjHC1scL+q2qLpBWVkZV199Neeffz7//ve/eemll/jhD3/IypUrqaur4/rrr+e+++4D4Pzzz2fevHlMmjSJ7Oxsbr/9dl5++WX69evHiy++yODBg6P8afquusYgGysPsLbcSQrryg9Qtrv2cIvD0IxkJudm8OnTc8lMTSQQDBEIKk0h5zEQDBEIOc0wTe65QChEk3uuKaRh1xx5XhsIuGWVlIQ4+iXGk52WRGpSPCmJPlITffRLjKdfoo9+SfH0cxPB4WOJ8YcTQ2piPCkJPuLiet+Xm+la0UwWi4A7RGQhTmf2AVWtFJElwE/COrUvAb53om/2w79vYGNF9Ym+zFEmDuvP/VecclzXbty4kd///vc8/vjjAMydO5eBAwcSCASYPn061113HRMnTjzqmgMHDnDhhRcyd+5c7rrrLhYsWMA999xzwp/DdKy+KUiJv4Z15fsPJ4dNO2sIuYlhUHoSp+Zm8KlTh3JqXgaTcjMYnJ4c3aCN6UKeJQsReQanhpAtIuU4I5wSAFT1cWAxcBlQBhwCbnHP7RWRh4Bl7ks92NzZ3ZucdNJJnHnmmYf3n3nmGX73u98RCASoqKhg48aNxySLlJQULr30UgDOOOMM3nnnnW6Nua9oCATZ5K9l7Y79rCt3ag6bdtYQcDNDVmoik/MyuGRiDpPzMjk1L4Oc/pYYTO/m5Wio2R2cV+DrbZxbACzoyniOtwbgldTU1MPPN2/ezKOPPsrSpUvJzMzkxhtvbPVeifAOcZ/PRyAQ6JZYe7PGQIhSfw3rdhxgfYXTlFTqr6ExGAIgs18Ck3MzmDNuNKfmZTA5L5NhGcm9sk3amPb06ju4e4rq6mrS09Pp378/lZWVLFmyhFmz2rtFxRyPhkDwSGJwO59L/TU0BZ0aQ3pyPJOGZXDL+aM4NdepMeQNSLHEYAyWLGLC1KlTmThxIpMmTWL06NFMmzYt2iH1ePVNxyaGTTuPJIb+yfFMzsvgS+fnMzk3g8m5GYwY2M8SgzFtEPX4ZpHuUlBQoC0XPyouLmbChAlRiih6+trnPtz5vOMA68M6n5v7GDJSnKakSW5SmJybwfCBVmMwBkBEVqhqQUflrGZhepyK/XW8UbKLdeX7Wbejms1hiaG5j+HL40YfTgzWlGTMibNkYXqE2oYAL6+r5IVVO/hgaxWqMKBfApNyM5juJoZJlhiM8YwlCxOzAsEQ75bt4YVVO1iywU99U4iRWf2486IxXHnaMPKzUy0xGNNNLFmYmLOxopq/riznxTUV7K5pICMlgWun5nHN1FymjhhgCcKYKLBkYWLCzup6/rZqBy+s2kGJv4YEnzB93GCumZrL9PGDSYr3RTtEY/o0SxYmag41Bliywc9fV+7gvbI9hBSmjMjkoatO4fJThzEgteNZeY0x3cOShYeqqqq46KKLAPD7/fh8PgYNGgTA0qVLI5qiHGDBggVcdtllDBkyxLNYu0swpHywpYq/rirnlfV+DjUGyRuQwh3TT+bqKbmMHpQW7RCNMa2wZOGhrKwsVq9eDbQ9RXkkFixYwNSpU3t0sij11/DXVeW8uKoCf3U96cnxXHX6MD49JY+CkQNs1lNjYpwliyh58skneeyxx2hsbOS8885j3rx5hEIhbrnlFlavXo2qMmfOHHJycli9ejXXX389KSkpnaqRRFt9U5Bnlm7nuRXlbKioJj5OKBw3iB9cPpGLJgwmOcH6IYzpKfpOsnj5HvCv69rXHDIZLp3b6cvWr1/PCy+8wPvvv098fDxz5sxh4cKFnHTSSezZs4d165w49+/fT2ZmJv/zP//DvHnzOP3007s2fo+EQsqLa3bwX6+UUnGgnlPzMnjgiolccdowstKSoh2eMeY49J1kEUNef/11li1bRkGBc4d9XV0dw4cPZ+bMmZSWlnLnnXdy2WWXcckll0Q50s7799Yqfry4mLXlB5icm8F/X38654zOinZYxpgT1HeSxXHUALyiqnzpS1/ioYceOubc2rVrefnll/nVr37F888/z/z586MQYedt23OQhxcX8+rGnQzNSOYX15/GVaflWl+EMb1E30kWMeTiiy/muuuu48477yQ7O5uqqioOHjxISkoKycnJfOYznyE/P5/bb78dcNbyrqmpiXLUrdt3sJFH39jMU//6iKT4OL47cxy3np9v/RHG9DKeJgsRmQU8CviA36rq3BbnR+IscjQI2AvcqKrl7rkg0NzJsF1Vr/Qy1u40efJk7r//fi6++GJCoRAJCQk8/vjj+Hw+br31VlQVEeGRRx4B4JZbbuG2226LqQ7uhkCQP7z/Ef/z5mZqGwLccNYIvn3xWAalW5+EMb2RZ1OUi4gP2ATMAMpxlkmdraobw8r8BXhJVZ8UkU8Ct6jqF9xztaoa8aB7m6L8CC8/t6qyeJ2fR14pYfveQxSOG8R/XjaBsTnpnryfMcZbsTBF+VlAmapudQNaCFwFbAwrMxH4tvv8LeBvHsZjTtDK7fv48T+KWfHRPsYPSeePt57FBWMGRTssY0w38DJZ5AIfh+2XA2e3KLMGuBanqerTQLqIZKlqFZAsIsuBADBXVY9JJCIyB5gDkJOTQ1FR0VHnMzIyYrat30v19fXH/C1OxO5DIf6yqZGl/iAZScItkxK5IDdIcMcGinZ02dsYY2KYl8mitWEwLdu87gbmicjNwNvADpzkADBCVStEZDTwpoisU9UtR72Y6nxgPjjNUIWFhUe9eHFxMWlpaX1qllJVJTk5mSlTppzwa1XXN/HYW2X8/v0PiRP45idP5isXnkRqko2LMKav8fL/+nJgeNh+HlARXkBVK4BrAEQkDbhWVQ+EnUNVt4pIETAFOCpZdCQ5OZmqqiqysrL6RMJQVaqqqkhOTj6h12kKhnhm6XZ++fpm9h1q5Jopedw9cyxDM1K6KFJjTE/jZbJYBowRkXycGsMNwOfCC4hINrBXVUPA93BGRiEiA4BDqtrglpkG/LSzAeTl5VFeXs7u3btP7JP0IMnJyeTl5R3XtarKG8W7+MnLxWzdfZBzR2fx/U9NYFJuRhdHaYzpaTxLFqoaEJE7gCU4Q2cXqOoGEXkQWK6qi4BC4GERUZxmqK+7l08AfiMiISAOp89i4zFv0oGEhATy8/O74NP0fhsrqnnopY18sLWK0YNS+e0XC7howuA+USMzxnTMs6Gz3a21obMmMi+u3sF3n1tLWlI837p4DLPPGkGCLy7aYRljukEsDJ01MS4UUn75+iZ+9WYZZ+cP5PEbz7AFh4wxrbJk0UfVNwX5zl/W8I+1lXy2II8fXT2ZxHirTRhjWmfJog/aVVPPl/+wgrXl+/nPy8bz5QtGW9+EMaZdliz6mI0V1dz25DL2HWri8RvPYOYpPXf1PWNM97Fk0Ye8vnEn31y4iv7JCfzl9nNtSKwxJmKWLPoAVeV3727jx4uLmZybwf99sYCc/id2454xpm+xZNHLNQVD3Pfiep5Z+jGXTR7Czz9zOimJttaEMaZzLFn0YvsPNfK1p1fy/pYq7ph+MnfNGGsr1xljjosli15q256D3PrEMsr31fHfnz2Na6Ye3xQgxhgDlix6pQ+2VHH7UyvwxQlPf/lszhw1MNohGdO3NdVBjR8aayE5A1IGQGIa9KAh65Ysepk/L9vO919Yz6jsVBbcdCYjsvpFOyRjjlB1t5CzEfb88KZHP7ZbJgTxyZCY6nz5+rrxK00VGqqhZifU+sMe/VC7M+xxJzQcOPb6uHgnaTRvyZlH7x+zueeTMyCu+/sdLVn0EsGQ8sgrJcx/eysXjMnmsc9PpX9yQrTDMn3dob2w9s+w8o+wayPHLmnTxeKTnaSRmApJ6UeSyOH95udp7vM29hNS4FBVK1/8zY+VULsLmg61HkNaDqQPgUHjYXThkf2kdKivhrp9R2/1+51Es7sY6vY7Sag9zbWT5m3IZJjxoBd/0SMfy9NXN93iYEOAOxeu5vXinXzx3JHcd/lE4m0iQBMtoRB8+A6sfBKK/w7BRhg2FS64C+ISQOKc5hcR97m70WI/onICgXpoPAgNtU4zT2Pt0fv1+6F6x9HnQ4GOPsWxkvof+dLPLXAem/fDH5MzTrx5KdgE9QfCEsr+YxNM+FZdeWLvFwFLFj1cxf46bn1yOaX+an545SncdN6oaIdkvFK3D3Zvgj2lsLsU9m+HoafCyTNgyKkQF+UfCNWVsPppWPVH2Peh86V5xi0w9QvOL99YoeoksGOSS82R50110G8gpA2B9BznMbEbm3R9CZCa7WwxwpJFD7bm4/3c9ofl1DUGWXDzmRSOGxztkMyJUnWaOvaUHp0YdpfCwV1HyvmSoP9QKF4Eb/4IUgfDmBlw8sVw0nSnaaI7BAOw+VVY+QfnUYMw6gKY/n2YcIXTnBNrRCA+ydlSs6IdTY9hyaKH+sfaSu56djWD0pN4+razGZuTHu2QTGeEgrD/Iych7C6BPZuchLBn89GdoUkZMGgsjLnEecwe5zxmjnQ6OWt3Qdkbzhd1yT+cX/big+FnOYljjFvr6OpRN3u3wqqnYNXTTlt76mCY9k2Y8gXIOqlr38vEBE8XPxKRWcCjOCvl/VZV57Y4PxJnKdVBwF7gRlUtd8/dBNzrFv2Rqj7Z3nv1lcWPVJV5b5bx89c2ccbIAfzmC2eQnZYU7bBMe/Zug4pVYQlhk5MUgg1HyqTlQPZYGDTuSEIYNN45HukXfTAAO1Y4iaPsNahc4772EDdxXAyjpzujao5HUz2UvOT0RWx72+kzGHMJTP2i8+izARU9UaSLH3mWLETEB2wCZgDlOGtyzw5fHlVE/gK8pKpPisgngVtU9QsiMhBYDhTgDJ9YAZyhqvvaer++kix++PcN/P69D/n0lFwevmYyyQk2dUdMqq6A9X+F9c85iQIAgcwRThI4XEsYB9ljvGk2qtkJW9xax5Y3nQ5T8cHws53EcfIMpy+ho2S0c6PTzLR2odNvkjkCpnwRpnwe+g/r+rhNt4qFZHEu8ICqznT3vwegqg+HldkAzFTVcnEWVDigqv1FZDZQqKpfccv9BihS1Wfaer++kCyeW1HO3X9Zw83njeL+Kyb2/jUoVJ3hi/s+dLdtsO8j53lDNYyc5vxiHjkNEmJgYsRDe2Hji7D+efjwXUBh6Okw+TrIv9BJCtFqww8GYMdyJ3Fsfg38a53j6UPh5IucxHHSdKdTGpzO3vV/dZLEjuXgS4Txlzu1iPwLo9+ZbrpMLCyrmgt8HLZfDpzdoswa4FqcpqpPA+kiktXGtbnehRr71u84wPdfWMc5owdy76cm9J5EEWhwRvUcTggttsbao8un5cCAUc6X2rLfwb/+FxL6Qf4njrTRDxjVffE31ELpy04NouwNCDVB1hgovAcmXQfZJ3dfLO3xxcOIc5ztovucTvSy153EsfHvTv9DXLxT68jIg+KXoOmgUwua+TCcer11BvdxXiaL1r7NWlZj7gbmicjNwNvADiAQ4bWIyBxgDkBOTg5FRUUnEG7sqmlUHni/jtR4+NzIet595+1ohxQ5VRKaDpBS5ye53k9K3c6jHpMa9iJh/2mDcYnUJ+dQlzKE+kGFzmPzfvJgQr4jNYi4vAYy968jq2oFA7evJGXTKwAc7JfH3oFT2TvwDPZnnoLGdW1buoSaGLh3FYN3vU32nqX4Qg3UJ2WxK/dT7Bp8IbVp+YDA+nKc3zmxKg8G34Jkf5H+1SUM3LuSgVUrSClfxe5B51E5dAbV/cdBg8CyddEO1kSZl8miHBgetp8HVIQXUNUK4BoAEUkDrlXVAyJSDhS2uLao5Ruo6nxgPjjNUIWFhS2L9HiBYIibf7+MmqZ6nr39XE4ffpydk90hGHCGelauObL51x1bO0gf5vz6H3Wa8xi2+dIGkypCasRvOtN5UIWqLbD5VVLLXiP1wyUML18ECalOraO5jX7AyOP7bKEgfPQerHvOaWqq3w8pA2Hq52HSdSSPOJcRcXGMOL5XjwEXHbU31N2MaeZlslgGjBGRfJwaww3A58ILiEg2sFdVQ8D3cEZGASwBfiIizb1+l7jn+5yfvbqJd8v28Mi1k2MrUQQaYFcxVK4+khh2bnDupgXnS3rIZDj9c06zTHNCyBzhTf+CiNPkk30ynPs158aqbe84o4I2vwqbXnbKZY9zmqrGzIAR5zpj7duiChUrYd3zTj9Erd/5XBMud5qYTppuI4BMn+FZslDVgIjcgfPF7wMWqOoGEXkQWK6qi3BqDw+LiOI0Q33dvXaviDyEk3AAHlTVvV7FGqsWr6vk8X9uYfZZI7j+zCj+Zm085CSC8MSwq9hpnwdnGoShp8GZtzkdukNPc8baR2Gys8MSU2HcLGdTdYaqlr3mtNEvnQ8fzHO++Edf6N7MNgMy3Yrw7lKnBrH+Oed+Al+iMzR00rUwdlb33slrTIzw9D6L7tTbRkNt3lnDVY+9x9icdP78lXNIiu+mL976aqfpKLwpaU+pO/snTtPLMDchNG+Zo3rW6JiGWmfuos1u8jiw3Tk+aLxTU/Cvc+4hyP+EU4OYcMXx35tgTIyLhdFQ5jhV1zcx548r6Jfo49c3TvU+UdTshLf/C7a+BVVlR46nDXGSwYQrjiSI/rk9ag7+ViWlwbhLnU3VuUlu82tOzSPQALMegVM+7cwJZIwBLFnEnFBIuevPa9i+9xB/uu1shmZ4OC6/qc5pjnnnF87EamMugVNvcGsMpzqzaPZ2Is6NcYPGwXl3RDsaY2KWJYsYM++tMl4v3sl9l0/k7NEejWtXdTpsX38ADnzs3Gw140Gb08cY0yZLFjHkrZJd/OL1TVx9+jBumTbKmzf5eBks+R6UL3NGK139v07bvDHGtMOSRYz4qOogdy5cxfgh/Xn4mlO7/g7t/dvh9R86I3zScuCqx+C02dEdsWSM6TEsWcSAQ40BvvLHFYgIv7nxDFISu/ALvKEG3v0FfPCYs/+J78K0bzmdvMYYEyFLFlGmqtzz/DpKd9bwxC1nMSKri8bwh4LO2gZv/shZM3jyZ+Ci+4/cS2CMMZ1gySLKfvfuNhatqeC7M8dx4dhBXfOiW/8JS74PO9dB3llww58gr8Nh1MYY0yZLFlH0wZYqHn65hJmn5PC1wi4YiVS1BV69F0oXQ8YIuG4BnHJNz78vwhgTdZYsoqRifx13/Gklo7L68bPPnHZiHdp1++CfP3WmsYhPdqagPudrsbn+sTGmR7JkEQX1TUG++tQKGgIhfvOFAtKTj3MyumATLF8ARQ9D3X6Y+gWYfq/deWyM6XKWLKLggUUbWFN+gMdvPIOTBx/HqCRVZybVV+91pqrIvxBm/ti5b8IYYzxgyaKbPbN0OwuXfczXp5/ErEmdnE4jFISP3od3fgZbiyDrZJi90JkJ1foljDEesmTRjVZt38f9L27gE2MHcdeMcZFdFGiEbf+E4kVQshgO7YHkTJg1FwpuhfhEb4M2xhgsWXSb3TUNfPWpleRkJPGrG07HF9dOTaDxoLOec/HfYdMr0FANiWkwdqYzA+zJM+ymOmNMt7Jk0Q2agiG+/qeV7K9r5Pmvnkdmv1ZqA/UHYNMSpwax+XUI1EHKAJhwJUy80umX8GKFOWOMiYCnyUJEZgGP4qyU91tVndvi/AjgSSDTLXOPqi4WkVFAMVDqFv2Xqt7uZaxeenhxCUu37eUX15/GKcMyjpw4uAdK/uHUILYWOSvPpQ2BKTc6NYiR08Bn+dwYE32efROJiA94DJgBlAPLRGSRqm4MK3Yv8Kyq/lpEJgKLgVHuuS2qerpX8XWXF1fvYMF727j5vFF8ekoeHCiH4pecBLH9fWcFusyRcM7tTi0it6BnrTpnjOkTvPzZehZQpqpbAURkIXAVEJ4sFOjvPs8AKjyMp9ttrKjmP55fy5V5dfwgcwn83zdgxwrn5KAJcMHdThNTziQbzWSMiWleJotc4OOw/XLg7BZlHgBeFZFvAKnAxWHn8kVkFVAN3Kuq73gYa5c7UF3NB7//Ln+Pf58xe7bDm8CwKc5kfhOugOwx0Q7RGGMi5mWyaO2nsrbYnw08oao/F5FzgT+KyCSgEhihqlUicgbwNxE5RVWrj3oDkTnAHICcnByKioq6/EMcr/jVT3Br0wtUpE5k89Db2JN9Dg3JgyAIrN8B7Ih2iMYYEzEvk0U5ED4fdh7HNjPdCswCUNUPRCQZyFbVXUCDe3yFiGwBxgLLwy9W1fnAfICCggItLCz04GMch4N7aHz7FRbrecz6zmLi4gSrRxhjerIOe1JF5A4RGXAcr70MGCMi+SKSCNwALGpRZjtwkfs+E4BkYLeIDHI7yBGR0cAYYOtxxBAd7z1KfKiBl7NuIq69+ymMMaaHiGTYzRCckUzPisgsiXB6VFUNAHcAS3CGwT6rqhtE5EERudIt9h3gyyKyBngGuFlVFfgEsNY9/hxwu6ru7dxHi5LaXeiy3/Iy00gffkq0ozHGmC7RYTOUqt4rIj8ALgFuAeaJyLPA71R1SwfXLsYZDht+7L6w5xuBaa1c9zzwfESfINa89ygE6vlZw9V8aUh6tKMxxpguEdGAfvfXvt/dAsAA4DkR+amHsfU8NX5Y9lv8I69imw5l/ND+HV9jjDE9QCR9Ft8UkRXAT4H3gMmq+lXgDOBaj+PrWd79JQSbeHPwTQCMzbGahTGmd4hkNFQ2cI2qfhR+UFVDInK5N2H1QNWVzkJEp81maXUmuZlKRspxLmpkjDExJpJmqMXA4c5lEUkXkbMBVLXYq8B6nHf/GzQIn7ibksoaxlt/hTGmF4kkWfwaqA3bP+geM80O7IAVT8Dpn6Ox/0i27K5l/FBLFsaY3iOSZCFuBzfgND9hU5sf7Z2fO0udXnA3W3bXEggp44ZY57YxpveIJFlsdTu5E9ztTnrSDXJe2/8xrPyDM634gJGU+J0ZSSZYM5QxpheJJFncDpyHM5lR82SAc7wMqkd552fOjLEXfAeAksoaEn1x5GenRjkwY4zpOpHclLcLZ6oO09K+j2DVU3DGzZDpTINV4q9hTE4a8T5bk8IY03t0mCzcyf1uBU7BmbsJAFX9kodx9Qxv/xeID86/6/ChEn81007OjmJQxhjT9SL5+ftHnPmhZgL/xJk9tsbLoHqEvVth9Z+cWkVGrnPoYCM7qxuYYJ3bxpheJpJkcbKq/gA4qKpPAp8CJnsbVg/w9s/AlwDnf/vwoebObRs2a4zpbSJJFk3u4353YaIMjqyT3TdVbYE1C6HgS9B/6OHDpX6nwjXeahbGmF4mkvsl5rvrWdyLsx5FGvADT6OKdf/8KfgSYdq3jjpcUllDVmoig9KTohSYMcZ4o91kISJxQLWq7gPeBkZ3S1SxbM9mWPcsnPM1SM856lSJv9qaoIwxvVK7zVDu3dp3dFMsPcM/H4H45GNqFcGQUrqzxpqgjDG9UiR9Fq+JyN0iMlxEBjZvkby4u7JeqYiUicg9rZwfISJvicgqEVkrIpeFnfuee12piMzsxGfyzu5SWPccnPVlSBt01KmPqg5S3xSyCQSNMb1SJH0WzfdTfD3smNJBk5S7hvZjwAycO7+Xicgid3W8ZvfiLLf6axGZiDPD7Sj3+Q0493YMA14XkbGqGozkQ3mmaC4kpsJ5dx45Me7rAAAUx0lEQVRzyjq3jTG9WSR3cOcf52ufBZSp6lYAEVkIXAWEJwsFmr9dM4AK9/lVwEJVbQC2iUiZ+3ofHGcsJ27nRtjwgjNUNjXrmNPF/hriBMbkpEUhOGOM8VYkd3B/sbXjqvqHDi7NBT4O22+eVyrcA8CrIvINIBW4OOzaf7W4NrejWD31z7mQmAbnfaPV0yWV1eRnp5Kc4OvmwIwxxnuRNEOdGfY8GbgIWAl0lCyklWPaYn828ISq/lxEzgX+6N7LEcm1iMgc3EkNc3JyKCoq6iCk45Na+yFnbnyRD0d+lg+Xrm21zKpth8jPiPMsBmOMiaZImqGO+iktIhk4U4B0pBwYHrafx5Fmpma3ArPc9/nAnYcqO8JrUdX5wHyAgoICLSwsjCCs47Dw85DUn1E3/JRRKQOOOX2wIcDuV5bwxfNPorBwjDcxGGNMFB3P1KiHgEi+EZcBY0QkX0QScTqsF7Uosx2npoKITMCpuex2y90gIkkiku++39LjiPXEVa6Bkpec+ypaSRQApTvdzu2h1rltjOmdIumz+DtHmoDigInAsx1dp6oBEbkDWAL4gAWqukFEHgSWq+oi4DvA/4nIt933uNldlW+DiDyL0xkeAL4etZFQRXMhOQPO+WqbRUoqm0dC2bBZY0zvFEmfxc/CngeAj1S1PJIXV9XFOMNhw4/dF/Z8IzCtjWt/DPw4kvfxTMUqKF0M078PKZltFivxV5OWFE/egJRuDM4YY7pPJMliO1CpqvUAIpIiIqNU9UNPI4sFbz0MyZlw9u3tFivx1zB+SDoirfXLG2NMzxdJn8VfgFDYftA91ruVr4DNS5yhsslt90WoKiWV1YyzJihjTC8WSbKIV9XG5h33eaJ3IcWIop9AykA4+yvtFqs8UE91fcA6t40xvVokyWK3iFzZvCMiVwF7vAspBny8FMpeh2nfhKT2awzNCx5NsJqFMaYXi6TP4nbgaRGZ5+6XA63e1d1rvPUT6JcNZ365w6Il7pxQYy1ZGGN6sUhuytsCnCMiaYCoau9ef/ujD2DrWzDjIUjqeJ6nksoacjNT6J+c0A3BGWNMdHTYDCUiPxGRTFWtVdUaERkgIj/qjuCiougnkDoYzrwtouIl/mom2IJHxpheLpI+i0tVdX/zjrtq3mXtlO+5PnwXtr0N538LEvt1WLwhEGTL7oM2LbkxpteLJFn4ROTwotIikgL0vkWmVZ2+irQcKPhSx+WBsl21BENqS6kaY3q9SDq4nwLeEJHfu/u3AE96F1KUbHsbPnoPZj0CCZHdiX1kwSNLFsaY3i2SDu6fishanLUmBHgFGOl1YN1KFYoehvShcMbNEV9W4q8hMT6OUVmp3sVmjDExINJZZ/04d3FfizNLbLFnEUXD1rdg+wdwwXcgITniy4orqxmbk0a873gm7zXGmJ6jzZqFiIzFmVZ8NlAF/Bln6Oz0boqte6g6c0D1z4Wpnbt9pMRfw4VjB3kUmDHGxI72mqFKgHeAK1S1DMCdSrx32bsV/Otg5o8hPvJ++6raBnbXNFh/hTGmT2gvWVyLU7N4S0ReARbS+nKnPVvWSXDnmjYXNmrLkc5tGzZrjOn92mxsV9UXVPV6YDxQBHwbyBGRX4vIJd0UX/dIz4H4zs2NWNycLGzYrDGmD+iwZ1ZVD6rq06p6Oc5a2KuBeyJ5cRGZJSKlIlImIsdcIyK/EJHV7rZJRPaHnQuGnWu5HGvUlVRWk52WRHZa77vlxBhjWorkPovDVHUv8Bt3a5eI+IDHgBk4kw8uE5FF7up4za/37bDy3wCmhL1Enaqe3pn4ulPpzhrrrzDG9Blejvk8CyhT1a3uGhgLgavaKT8beMbDeLpMMKSU+i1ZGGP6Di+TRS7wcdh+uXvsGCIyEsgH3gw7nCwiy0XkXyJytXdhdt6HVQdpCIRswSNjTJ/RqWaoTmpt5JS2UfYG4DlVDYYdG6GqFSIyGnhTRNa506UfeQOROcAcgJycHIqKirog7I4t9QcAOLijlKKasm55T2OMiSYvk0U5MDxsPw+oaKPsDcDXww+oaoX7uFVEinD6M7a0KDMfmA9QUFCghYWFXRF3h1a+WoovbgvXX1pIcoKvW97TGGOiyctmqGXAGBHJF5FEnIRwzKgmERkHDAA+CDs2oHmmWxHJBqYBG1teGy3F/hrys1MtURhj+gzPahaqGhCRO4AlgA9YoKobRORBYLmqNieO2cBCVQ1vopoA/EZEQjgJbW74KKpoK/FXc1peZrTDMMaYbuNlMxSquhhY3OLYfS32H2jluveByV7GdrxqGwJ8vLeOG84cEe1QjDGm29h0qZ3UPM3HuBwbNmuM6TssWXRSib8asGk+jDF9iyWLTiqprCE9KZ7czMhW0zPGmN7AkkUnlfirGT80HZHeNwGvMca0xZJFJ6gqJf4am5bcGNPnWLLohIoD9dTUBxhnc0IZY/oYSxadUFLpdG5PsM5tY0wfY8miE0rcYbNjbdisMaaPsWTRCcWV1QwfmEJ6ckK0QzHGmG5lyaITSv01jMuxzm1jTN9jySJC9U1Btu45aP0Vxpg+yZJFhMp21RIMqQ2bNcb0SZYsItTcuW3TfBhj+iJLFhEq9VeTFB/HqKzUaIdijDHdzpJFhEr8NYzNSccXZ9N8GGP6HksWESqurGG83bltjOmjLFlEYE9tA3tqGxg/1Dq3jTF9k6fJQkRmiUipiJSJyD2tnP+FiKx2t00isj/s3E0istndbvIyzo40L3hkNQtjTF/l2bKqIuIDHgNmAOXAMhFZFL6Wtqp+O6z8N4Ap7vOBwP1AAaDACvfafV7F255id04oSxbGmL7Ky5rFWUCZqm5V1UZgIXBVO+VnA8+4z2cCr6nqXjdBvAbM8jDWdpX4axiUnkRWWlK0QjDGmKjyrGYB5AIfh+2XA2e3VlBERgL5wJvtXJvbynVzgDkAOTk5FBUVnXDQrVm2uY6cRPHs9Y0xJtZ5mSxaG2OqbZS9AXhOVYOduVZV5wPzAQoKCrSwsPA4wmxfIBjC//oSLjl3BIWFE7v89Y0xpifwshmqHBgetp8HVLRR9gaONEF19lpPfVh1iIZAyKb5MMb0aV4mi2XAGBHJF5FEnISwqGUhERkHDAA+CDu8BLhERAaIyADgEvdYtyvxu53bNs2HMaYP86wZSlUDInIHzpe8D1igqhtE5EFguao2J47ZwEJV1bBr94rIQzgJB+BBVd3rVaztKamswRcnnDw4LRpvb4wxMcHLPgtUdTGwuMWx+1rsP9DGtQuABZ4FF6ESfzWjs1NJivdFOxRjjIkau4O7AyX+Grtz2xjT51myaEd1fRPl++rsZjxjTJ9nyaIdm9xpPmx1PGNMX2fJoh3FbrIYZ8NmjTF9nCWLdpT6q0lPjmdYRnK0QzHGmKiyZNGOksoaJgzpj4gteGSM6dssWbRBVd2RUNZfYYwxlizaUL6vjtqGgE3zYYwxWLJoU+nhzm2rWRhjjCWLNjTPCWXJwhhjLFm0qdhfw4iB/UhL8nRGFGOM6REsWbShpLLa7tw2xhiXJYtW1DcF2bbnoCULY4xxWbJoRdmuWkKKTSBojDEuSxatKK50FzyymoUxxgCWLFpV4q8hOSGOkVmp0Q7FGGNigqfJQkRmiUipiJSJyD1tlPmsiGwUkQ0i8qew40ERWe1uxyzH6qVSfw1jc9Lxxdk0H8YYAx6ulCciPuAxYAZQDiwTkUWqujGszBjge8A0Vd0nIoPDXqJOVU/3Kr72lPir+eT4wR0XNMaYPsLLmsVZQJmqblXVRmAhcFWLMl8GHlPVfQCqusvDeCKyu6aBPbWNNs2HMcaE8fKOs1zg47D9cuDsFmXGAojIe4APeEBVX3HPJYvIciAAzFXVv7V8AxGZA8wByMnJoaio6ISDXr8nCEDDzq0UFX10wq9njDG9gZfJorUGf23l/ccAhUAe8I6ITFLV/cAIVa0QkdHAmyKyTlW3HPViqvOB+QAFBQVaWFh4wkFvfnsrUMz1sy5gYGriCb+eMcb0Bl42Q5UDw8P284CKVsq8qKpNqroNKMVJHqhqhfu4FSgCpngY62El/hoGpydZojDGmDBeJotlwBgRyReRROAGoOWopr8B0wFEJBunWWqriAwQkaSw49OAjXSDEn+13YxnjDEteJYsVDUA3AEsAYqBZ1V1g4g8KCJXusWWAFUishF4C/iuqlYBE4DlIrLGPT43fBSVVwLBEJt31jLBbsYzxpijeDqlqqouBha3OHZf2HMF7nK38DLvA5O9jK012/YcpDEYsmnJjTGmBbuDO0yJu+CRDZs1xpijWbIIU+KvJj5OOGmwTfNhjDHhLFmEKams4aRBaSTF+6IdijHGxBRLFmFK/DXWX2GMMa2wZOGqrm9ix/46xg+1ZGGMMS1ZsnCVup3bE6xz2xhjjmHJwlXSvOCR1SyMMeYYlixcxf4a+ifHM6R/crRDMcaYmGPJwlXqr2H80P6I2IJHxhjTkiULIBRSSv01Ns2HMca0wZIFsGN/HbUNAZtA0Bhj2mDJAihu7ty2moUxxrTKkgVHhs2OzbFkYYwxrbFkgXPn9sisfqQmeToJrzHG9FiWLIBif7U1QRljTDv6fLKoawzy4Z6DNi25Mca0w9NkISKzRKRURMpE5J42ynxWRDaKyAYR+VPY8ZtEZLO73eRVjLUNAS4/dRhn5Q/06i2MMabHE2exOg9eWMQHbAJmAOU4a3LPDl8eVUTGAM8Cn1TVfSIyWFV3ichAYDlQACiwAjhDVfe19X4FBQW6fPlyTz6LMcb0ViKyQlULOirnZc3iLKBMVbeqaiOwELiqRZkvA481JwFV3eUenwm8pqp73XOvAbM8jNUYY0w7vEwWucDHYfvl7rFwY4GxIvKeiPxLRGZ14lpjjDHdxMuxoq1NstSyzSseGAMUAnnAOyIyKcJrEZE5wByAnJwcioqKTiBcY4wxbfEyWZQDw8P284CKVsr8S1WbgG0iUoqTPMpxEkj4tUUt30BV5wPzwemzKCwsbFnEGGNMF/CyGWoZMEZE8kUkEbgBWNSizN+A6QAiko3TLLUVWAJcIiIDRGQAcIl7zBhjTBR4VrNQ1YCI3IHzJe8DFqjqBhF5EFiuqos4khQ2AkHgu6paBSAiD+EkHIAHVXWvV7EaY4xpn2dDZ7ubDZ01xpjOi4Whs8YYY3qJXlOzEJHdwEcn8BLZwJ4uCsdrPSlW6Fnx9qRYoWfF25NihZ4V74nEOlJVB3VUqNckixMlIssjqYrFgp4UK/SseHtSrNCz4u1JsULPirc7YrVmKGOMMR2yZGGMMaZDliyOmB/tADqhJ8UKPSvenhQr9Kx4e1Ks0LPi9TxW67MwxhjTIatZGGOM6VCfTxaRLNAUK0RkuIi8JSLF7mJRd0Y7po6IiE9EVonIS9GOpSMikikiz4lIifs3PjfaMbVFRL7t/htYLyLPiEhytGMKJyILRGSXiKwPOzZQRF5zFzR7zZ3KJ+raiPW/3H8Ha0XkBRHJjGaM4VqLN+zc3SKi7vRJXapPJwt3gabHgEuBicBsEZkY3ajaFQC+o6oTgHOAr8d4vAB3AsXRDiJCjwKvqOp44DRiNG4RyQW+CRSo6iSc6XRuiG5Ux3iCY9eguQd4Q1XHAG+4+7HgCY6N9TVgkqqeirOI2/e6O6h2PEEr6/uIyHCcxea2e/GmfTpZENkCTTFDVStVdaX7vAbnyyxm1/kQkTzgU8Bvox1LR0SkP/AJ4HcAqtqoqvujG1W74oEUEYkH+nHsjM5RpapvAy3nc7sKeNJ9/iRwdbcG1YbWYlXVV1U14O7+C2fm65jQxt8W4BfA/6OV5Ry6Ql9PFj12kSURGQVMAf4d3Uja9Uucf7yhaAcSgdHAbuD3brPZb0UkNdpBtUZVdwA/w/kFWQkcUNVXoxtVRHJUtRKcHz7A4CjHE6kvAS9HO4j2iMiVwA5VXePVe/T1ZBHRIkuxRkTSgOeBb6lqdbTjaY2IXA7sUtUV0Y4lQvHAVODXqjoFOEjsNJMcxW3rvwrIB4YBqSJyY3Sj6p1E5Ps4zb9PRzuWtohIP+D7wH1evk9fTxaRLNAUU0QkASdRPK2qf412PO2YBlwpIh/iNO99UkSeim5I7SoHylW1uab2HE7yiEUXA9tUdbe7cNhfgfOiHFMkdorIUAD3cVeU42mXiNwEXA58XmP7HoOTcH44rHH/f8sDVorIkK58k76eLCJZoClmiIjgtKkXq+p/Rzue9qjq91Q1T1VH4fxd31TVmP31q6p+4GMRGeceugjYGMWQ2rMdOEdE+rn/Ji4iRjvjW1gE3OQ+vwl4MYqxtEtEZgH/AVypqoeiHU97VHWdqg5W1VHu/2/lwFT333SX6dPJwu3Aal6gqRh4VlU3RDeqdk0DvoDzK321u10W7aB6kW8AT4vIWuB04CdRjqdVbu3nOWAlsA7n/+OYuttYRJ4BPgDGiUi5iNwKzAVmiMhmnFE7c6MZY7M2Yp0HpAOvuf+fPR7VIMO0Ea/37xvbtStjjDGxoE/XLIwxxkTGkoUxxpgOWbIwxhjTIUsWxhhjOmTJwhhjTIcsWRjTCSISDBu2vLorZyoWkVGtzSRqTCyIj3YAxvQwdap6erSDMKa7Wc3CmC4gIh+KyCMistTdTnaPjxSRN9x1Ed4QkRHu8Rx3nYQ17tY8XYdPRP7PXaviVRFJidqHMiaMJQtjOielRTPU9WHnqlX1LJy7f3/pHpsH/MFdF+Fp4Ffu8V8B/1TV03DmoGqeOWAM8JiqngLsB671+PMYExG7g9uYThCRWlVNa+X4h8AnVXWrO9mjX1WzRGQPMFRVm9zjlaqaLSK7gTxVbQh7jVHAa+7iQIjIfwAJqvoj7z+ZMe2zmoUxXUfbeN5WmdY0hD0PYv2KJkZYsjCm61wf9viB+/x9jix5+nngXff5G8BX4fA65f27K0hjjof9ajGmc1JEZHXY/iuq2jx8NklE/o3zI2y2e+ybwAIR+S7OSny3uMfvBOa7M4YGcRJHpefRG3OcrM/CmC7g9lkUqOqeaMdijBesGcoYY0yHrGZhjDGmQ1azMMYY0yFLFsYYYzpkycIYY0yHLFkYY4zpkCULY4wxHbJkYYwxpkP/H1LSo8ic3wwEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VPW9//HXZyYrkIQ9IIussoqIcUGt4laxLliXitbWoi23i9Vba3vt7962Lu2tva1tVagttaDe9oJrW2q1VtG4K7KJyqKAiAEiiyxhyTqf3x9nEoaQDchkZjLv5+Mxj5lz5ntmPoFk3nO2zzF3R0REBCCU6AJERCR5KBRERKSOQkFEROooFEREpI5CQURE6igURESkjkJBRETqKBRERKSOQkFEROpkJLqAg9W9e3cfMGBAossQEUkpCxcu3OLuPZobl3KhMGDAABYsWJDoMkREUoqZfdSScdp8JCIidRQKIiJSR6EgIiJ1Um6fQkOqqqooKSmhvLw80aW0mZycHPr27UtmZmaiSxGRdqRdhEJJSQl5eXkMGDAAM0t0OXHn7mzdupWSkhIGDhyY6HJEpB1pF5uPysvL6datW1oEAoCZ0a1bt7RaMxKRttEuQgFIm0ColW4/r4i0jXYTCs3ZXVHNxh170eVHRUQalzahsLeqhs1lFVRHWj8Utm7dytixYxk7diy9evWiT58+ddOVlZUteo0pU6awcuXKVq9NRORgtIsdzS2RkxkGgnDIDLduFnbr1o0lS5YAcOutt9KpUyduvvnm/ca4O+5OKNTwe8+aNatVaxIRORRps6aQkxH8qOVVNW32nqtWrWL06NF8/etfZ9y4cWzcuJGpU6dSVFTEqFGjuP322+vGnnrqqSxZsoTq6mo6d+7MLbfcwjHHHMP48ePZtGlTm9UsIumt3a0p3Pb391i2YWeDz+2prCEcMrIzDi4LRx6Rz48vHHVI9SxbtoxZs2bxu9/9DoA777yTrl27Ul1dzRlnnMFll13GyJEj91tmx44dnH766dx5553cdNNNzJw5k1tuueWQ3l9E5GCkzZoCQMgg0sY7mgcPHszxxx9fNz179mzGjRvHuHHjWL58OcuWLTtgmdzcXM477zwAjjvuONauXdtW5YpImmt3awpNfaMv3VHO5rIKRh2RTyjUNod0duzYse7xBx98wN133838+fPp3LkzV199dYPnGmRlZdU9DofDVFdXt0mtIiJptaaQkxnCcSqq226/QqydO3eSl5dHfn4+Gzdu5JlnnklIHSIijWl3awpN2XcEUoTcrGYGx8G4ceMYOXIko0ePZtCgQZxyyiltX4SISBMs1U7mKioq8voX2Vm+fDkjRoxodll3570NO+naMYsjOufGq8Q209KfW0TEzBa6e1Fz49Jq85GZkZMZatPDUkVEUklahQIEm5DKq2rU7kJEpAFpGQrVEY9LuwsRkVSXlqEAQbsLERHZX/qFQgLaXYiIpIq0C4WMcIiscIjyykiiSxERSTppFwoQ3dnciiewtUbrbICZM2dSWlraanWJiBystDp5rVZOZpiy8moiEW+VdhctaZ3dEjNnzmTcuHH06tXrsGsSETkUaRoK+9pd5GbF95/gwQcfZPr06VRWVnLyySczbdo0IpEIU6ZMYcmSJbg7U6dOpbCwkCVLlnDFFVeQm5vL/Pnz9+uBJCLSFuL2iWhmM4ELgE3uPrqB5w24G/gcsAf4irsvOuw3fvoWKH2nySH57gyqrCEjMwSNXPRmP72OhvPuPOhS3n33Xf7yl7/w2muvkZGRwdSpU5kzZw6DBw9my5YtvPNOUOf27dvp3Lkz9957L9OmTWPs2LEH/V4iIq0hnvsUHgAmNvH8ecDQ6G0qcF8ca9mPWXCLxPlcheeee4633nqLoqIixo4dy4svvsjq1asZMmQIK1eu5MYbb+SZZ56hoKAgrnWIiLRU3NYU3P0lMxvQxJBJwEMenFr8hpl1NrPe7r7xsN64Bd/oDdi4qYyQGYN6dDqst2uKu3Pttddyxx13HPDc0qVLefrpp7nnnnt4/PHHmTFjRtzqEBFpqUQefdQH+DhmuiQ6r020RbuLs88+m0ceeYQtW7YAwVFK69atY/Pmzbg7l19+ObfddhuLFgVbzfLy8igrK4tbPSIizUnkjuaGDvtp8BPazKYSbGKisLCQ4uLi/Z4vKCg46A9TqwlaXWzfWUZGK15wp6KigszMTMrKyhgwYADf//73OfPMM4lEImRmZvLrX/+acDjM9ddfj7tjZtx2222UlZUxefJkrr32WnJzc3nhhRea3dFcXl5+wL+FiMjhiGvr7Ojmoycb2dH8e6DY3WdHp1cCE5rbfHQ4rbNj7aqoZs3mXQzo3pH8nMyDWjZZqHW2iLRUKrTOngt82QInATsOe3/CQcjJVLsLEZH64nlI6mxgAtDdzEqAHwOZAO7+O+ApgsNRVxEckjolXrU0JCOkdhciIvXF8+ijK5t53oFvteL7EZz60HKt3e6iLel6ECISD+2i91FOTg5bt2496A/KnMwwFVWRuJ+v0Nrcna1bt5KTk5PoUkSknWkXbS769u1LSUkJmzdvPqjl9lbWsHV3JZFt2WRlpFY+5uTk0Ldv30SXISLtTLsIhczMTAYOHHjQy63evItL7nqRX1w2hsuP7heHykREUktqfT1uZQO6dSQnM8TyjTphTEQE0jwUwiFjWGEeK0p3JroUEZGkkNahADCidz7LN+7U0TwiIigUGN4rj217qthUVpHoUkREEi7tQ2FE73wAlm3UJiQRkbQPheHRUFihnc0iIgqFgtxM+nTOZbnWFEREFAoAI3rrCCQREVAoADC8Vz6rN+9Wx1QRSXsKBYKdzTURZ9WmXYkuRUQkoRQKwPDeeQDaryAiaU+hgNpdiIjUUiigdhciIrUUClFqdyEiolCoM6J3vtpdiEjaUyhEDe8V7GxWuwsRSWcKhSi1uxARUSjUUbsLERGFwn7U7kJE0p1CIYbaXYhIulMoxFC7CxFJdwqFGGp3ISLpTqEQQ+0uRCTdxTUUzGyima00s1VmdksDz/c3sxfMbLGZLTWzz8WznuaEQ8awXvna2SwiaStuoWBmYWA6cB4wErjSzEbWG/ZfwCPufiwwGfhtvOppqRG98tTuQkTSVjzXFE4AVrn7GnevBOYAk+qNcSA/+rgA2BDHelpE7S5EJJ3FMxT6AB/HTJdE58W6FbjazEqAp4Bvx7GeFlG7CxFJZxlxfG1rYF79bTJXAg+4+11mNh74XzMb7e6R/V7IbCowFaCwsJDi4uJ41AvA7qqgxKdeXYJtzIrb+4iIJKN4hkIJ0C9mui8Hbh66DpgI4O6vm1kO0B3YFDvI3WcAMwCKiop8woQJcSo58N8Ln6citwsTJhwb1/cREUk28dx89BYw1MwGmlkWwY7kufXGrAPOAjCzEUAOsDmONbWI2l2ISLqKWyi4ezVwPfAMsJzgKKP3zOx2M7soOuy7wNfM7G1gNvAVT4LDftTuQkTSVTw3H+HuTxHsQI6d96OYx8uAU+JZw6GIbXcxuk9BossREWkzOqO5ASPU7kJE0pRCoQFHqt2FiKQphUID1O5CRNKVQqERanchIulIodAItbsQkXSkUGiE2l2ISDpSKDRieO+gT98K7WwWkTSiUGhEQW4mfTrn6rBUEUkrCoUmqN2FiKQbhUIT1O5CRNKNQqEJse0uRETSgUKhCWp3ISLpRqHQBLW7EJF0kz6hsHc7vPfXg1pE7S5EJN2kTyi88Vt49CuweeVBLaZ2FyKSTtInFE74N8jMhVd+fVCLqd2FiKST9AmFjt2g6FpY+gh8+mGLF1O7CxFJJ+kTCgDjr4dQBrz6mxYvonYXIpJO0isU8nvDsVfD4j/DjvUtWkTtLkQknaRXKACcciPg8Nq9LV5E7S5EJF2kXyh0ORLGTIaFD8CuTS1aZERvtbsQkfSQfqEAcOp3oKYCXp/eouHDe6ndhYikh/QMhe5DYNTn4a37Yc+nzQ5XuwsRSRfpGQoAn/kuVO6C+TOaHVrb7mJFqY5AEpH2LX1DoXAUDL8A3rgPypteA6htd6E1BRFp79I3FCBYWyjfDgv+2OxQtbsQkXSQ3qHQZxwMPivY4Vy5p8mhanchIukgrqFgZhPNbKWZrTKzWxoZ8wUzW2Zm75nZ/8Wzngad9j3YvRkWPdTkMLW7EJF00KJQMLPBZpYdfTzBzG4ws87NLBMGpgPnASOBK81sZL0xQ4EfAKe4+yjg3w/hZzg8R46HI0+FV++G6sbXAtTuQkTSQUvXFB4HasxsCPBHYCDQ3Lf6E4BV7r7G3SuBOcCkemO+Bkx3920A7t6ys8la22nfhbINsKTxH0ntLkQkHWS0cFzE3avN7PPAb9z9XjNb3MwyfYCPY6ZLgBPrjTkKwMxeBcLAre7+z/ovZGZTgakAhYWFFBcXt7DsFnJjXN5QMp/7GfN39sdD4QaH9cisZOHqjRQX72jd9xcRSRItDYUqM7sSuAa4MDovs5llrIF59Q/dyQCGAhOAvsDLZjba3bfvt5D7DGAGQFFRkU+YMKGFZR+EI+6A2ZM5vdtmOGZyg0MWVq7kt8WrOemUz5CT2XBwiIikspZuPpoCjAd+6u4fmtlA4E/NLFMC9IuZ7gtsaGDM39y9yt0/BFYShETbO2oiFI6Gl++CSKTBIWp3ISLtXYtCwd2XufsN7j7bzLoAee5+ZzOLvQUMNbOBZpYFTAbm1hvzV+AMADPrTrA5ac1B/QStxSw4b2HL+7C8fpkBtbsQkfaupUcfFZtZvpl1Bd4GZpnZr5paxt2rgeuBZ4DlwCPu/p6Z3W5mF0WHPQNsNbNlwAvA99x966H+MIdt5CToNhRe+iU0cJKa2l2ISHvX0s1HBe6+E7gEmOXuxwFnN7eQuz/l7ke5+2B3/2l03o/cfW70sbv7Te4+0t2Pdvc5h/qDtIpQOFhb+OQdeP+ZA55WuwsRae9aGgoZZtYb+ALwZBzrSbyjL4PO/eGlXzS4tqB2FyLSnrU0FG4n2NSz2t3fMrNBwAfxKyuBwpnB9RbWL4APXzzgabW7EJH2rKU7mh919zHu/o3o9Bp3vzS+pSXQ2C9CXu9g30I9I6JnNqvdhYi0Ry3d0dzXzP5iZpvM7BMze9zM+sa7uITJyIaTb4C1L8O6N/Z7ali0B5LaXYhIe9TSzUezCA4nPYLgTOW/R+e1X8ddAx26H7C2oHYXItKetTQUerj7LHevjt4eAHrEsa7Ey+oI478Fq56FDft39BjRO48VpQoFEWl/WhoKW8zsajMLR29XA4k7n6CtHP9VyCk4YG1hRO98Vm/eTXlVTYIKExGJj5aGwrUEh6OWAhuBywhaX7RvOflw4tdhxZPwybK62Wp3ISLtVUuPPlrn7he5ew937+nuFxOcyNb+nfh1yOoEr+w7gVvtLkSkvTqcK6/d1GpVJLMOXeH46+Ddx2HrakDtLkSk/TqcUGioNXb7NP56CGfVrS2o3YWktIoyeOhimH0llOt3WPZ3OKGQPn0eOvWEcdfA23Ng+zoARvZWuwtJQZV74P8mw4cvwQf/glnnwY71ia5KkkiToWBmZWa2s4FbGcE5C+njlBsAg1fvAYKdzWp3ISmlqhwe/iJ89CpcMgO++Chs+wjuPxtK30l0dZIkmgwFd89z9/wGbnnu3tKrtrUPBX1h7JWw6CEoK1W7C0kt1ZXw6DWw+nmYND1o/Dj4TLg2evXbmefBqucSW6MkhcPZfJR+Tv0ORKrgtXvV7kJSR001PH4dvP9POP9XcOwX9z3XazR8bR50GQB//kLwpUfSmkLhYHQdBEdfDgtmURDZqXYXkvwiNfDXrwdXEzz3Z8GRdPXlHwFTnoJBE2Dut2HeHQ22jZf0oFA4WKfeBFV74M371O5CklskAn+/Ad55FM76EYz/ZuNjc/Lhqodh3Jfh5V/CE1OhWvvL0pFC4WD1HA4jLoQ3f8/YHqZ2F5Kc3OHp78HiP8Hp/xFcUbA54Uy48B4484fwziPwp0th77b41ypJRaFwKE67GSp2cs6uv6vdhSQfd/jXf8Fb9wct4Cf8oOXLmgW/35fcH7SN/+O5wRFKkjYUCoei9zEw9FyGrHmQXMq1X0GSy/M/gdenwQn/BufcHnzQH6wxl8OX/gK7SoNDVtcvav06JSkpFA7VaTcTLt/GNVkvqN2FJI+XfhHsExh3DUy889ACodbAz8B1z0JmDjxwPqx8uvXqlKSlUDhU/U6AgacxNeNJFq7eSE1ER2tIgr12b7CWMGYyXPAbCLXCn3ePYXDdc8H9nKtg/h8O/zUlqaXXCWit7bTv0fXBC5m4eRZPPrGBSccNCC7lGc6EcPb+j8OZ0elsCIUP7xucSH3z/xDsRxh5cXByWmsEQq28QvjKP+Cx6+Cpm2H7R3D27a37HpI0LNV69xQVFfmCBQsSXUbAHZ91Hrbu9YNc0KIBkbXvlpEVDY/o44wcOOrcYLtwZk5cyk9LVeWw+H+DflZDz20f/7aLHgrOLxh2PnzhweALSDxEauCft8D8GTByEnz+95CZG5/3SmbuUL496Bm1owR2fBy9j7nVVAa/Wxm5wX1mh+BvOjO38fumnsuIvkZeYXDhr0NgZgvdvajZcQqFw1RRxu6NK/j+nAV4TSU/nzSMvIxI8EtRe6uuiHlcCTUVjTyOGb/30+AyoAX94awfwujL9M3scG1aDo9/FT55N5jOLoBRk2DMFdD/5NT89136SHBOwZCzYPL/BV824skdXp8erJX0OwEmz4aO3eLzXjXVsHl5sJN7y/vBh2N2XvSWH1znpG46Oi87L/hSdTiqK6FsQ8yHfAMf+pX1jjgMZUJBH8jvG9xn5EB1OVTtrXdfDtV797+v2kOL+4uef1dwRchDoFBoY++u38Elv32Nzwztzv3XFGGtsXloTTH864dQujQ44umzP4GBpx3+66Yb92DzyrM/DD40LpoWfHgufRiW/z34A8/vGxxxM+YK6Dki0RW3zHt/hcemwJGnBM3t2vJb+3t/DcKooA988THoNvjwXs8dPl0TfBFavwjWL4SNbwcfnBCsRddU0qIPz3B2w2Gx361TMN8jB37g7/rkwPfp0D3of1bQFwr6BT933eO+0LHnoX+pcA9+tgMCpIH7I4495H/rpAgFM5sI3A2Egfvd/c5Gxl0GPAoc7+5NfuInaygAzHr1Q277+zJ+eMFIrjt1YOu8aCQSnEg07w7YWRJs8jjn9uAkOmners3wt28GbaKHfjbY3t6p577nK3cHR9UsfRhWzQOvgV5HB+Ew+jLI75242puy8ml4+GroUwRXPx58yLW1dW/C7MnB/rEr5wRrDi1VVhp8+G+IBsCGxftOlMvICb4EHTEO+hwHfcYFLWYg+FZdURa97Yx53NC8XY2PrYk5WzsjJ+YDP+aDPr/PvgBoB5vJEh4KZhYG3gfOAUqAt4Ar3X1ZvXF5wD+ALOD6VA4Fd2fq/y6keOUmHv/GyYzp27n1XrxqL7z5O3j5V8E323FfDk5KyuvVeu/R3nzwLPz1G8GFZD77Ezjha03v4N+1Gd57IgiI9QvBQjDw9CAgRlwQfMNMBqvmBR/GhaPhy38LWlQkytbV8OfLYOeGoB33yEkHjinfsf8awIbFsDN6DQcLQ8+R0OfYIACOGBesqcVrv0it6oogNCC4umIaHPiRDKEwHrjV3c+NTv8AwN1/Vm/cb4DngJuBm1M5FAC276nkc3e/TGZGiCe/fSp5Oa38y717K7z0P8HZquHs4DoP469PzDfFZFVVDs/+COb/HnqOgkvvh8KRB/caW1YFa2hLH4Zta4MdhsPPDwJi8Bnx/9BqzIcvBx/C3YbCNXODD7RE2701CKmSt+Czd0C/k6JrANEQ2PrBvrFdB+2/BtBrDGR1SFztaSQZQuEyYKK7fzU6/SXgRHe/PmbMscB/ufulZlZMI6FgZlOBqQCFhYXHzZkzJy41t5YPttXws/nlFBWG+cYx2a2zf6Ge3D0bGfjhQ/Tc/BoVWV1YO+AqSnudhYfCrf5eqaTjrrWMWP4rOu3+iJI+F7Jm0JeJhA9jx6M7+TtXUvhJMT03vUJmdRmVmQVs6nkqnxROoCxvaJt9y8zfsZxj3r6V8pweLBn7U6qyDu0olHgI1VQwfMVv6Ln5tbp5FVldKMsbys78oZTlDaUsbwjVmUmytpWGzjjjjISHwuXAufVC4QR3/3Z0OgQ8D3zF3dc2FQqxkn1Nodb0F1bxi2dW8rNLjubKE/rH743WvRkcCVIyH3qMgHNuC7adp8Hq8H7c4c3fB2sIOQVw8X0w9OzWfY/qyuBCNEsfDrbp11RA18HB2sOYy/dt946H9YvgoUnQsQdMeTo4NDHZRCLB5reM7GBtIP+I9Ps9TGLJsKbQ5OYjMysAVgO1x3b1Aj4FLmoqGFIlFCIR55pZ85n/4afMvf7UuovyxIV70C//uVuDIzgGfCbYhn7E2Pi9ZzLZtSnYd7DqOThqYnB0Uace8X3P8h2wbG4QEGtfARx6jw12UOYUBEe25OQ3cF+wbzo7L9iB2dwHZ+k78MAFwXJTng7eQ+QgJUMoZBDsaD4LWE+wo/kqd3+vkfHFtKM1BYDNZRWcd/fLdOmQydzrTyU3K86bdqorYeEsKL4zOM9hzBVBG+TO/eL7von0/jPw128GO98/+5PgGO62/na6owTeeSw4wmnPp8GRLuU7obIFPbFCmUE41AVHvUDJzoOFDwTfvqc8FVwhTeQQJDwUokV8DvgNwSGpM939p2Z2O7DA3efWG1tMOwsFgFc+2MKXZr7JF47rx88vG9M2b1q+A175NbxxX7AWcdLXg4sD5bbi0VCJVrU3OIfjrT8ER+Fcen/ynV8Qqdl3OGT5zv3vG5rX2H1Bv6Bjafchif6JJIUlRSjEQ6qFAsAvn1nJtBdWcffksUwa26ft3nj7x0GDtKUPQ24XOP37UHTd4Z/xmWil7wbXHN68Ak76Fpz94/ifyZsokUhwn4pnW0tSUSgkkeqaCFf+4Q2WbdjJkzd8hoHdO7ZtARvfDr5Vf/hi8K2z85FBU75QRsytpdPhhsdkdYRuQ6D7UcHO0HhswolEgnM1nvtxEHIX3xe0dxCRZikUksyG7Xv53D0v07dLLo9/42SyM9r40FH3YEfs/D8EZ/FGqmNuNQc37c1cfjSnIAiH7kftC4ruR0HXgYd+fH/ZJ8HO5NXz4KjzYNI06Nj90F5LJA0pFJLQs8s+4WsPLeArJw/g1otGJbqcQ+d+YFCU7whOUtpSe3s/uN9Vum+5UEawo7T7UdB96P7B0dRJWCv/GbSqqNwN5/402ASmQx1FDkpLQ0HXU2hD54wsZMopA5j16lrGD+7GuaNStEWFGYQzglut3M7Q5UgYUu/cgPIdwdnBW2uDIhoWHzwLkap94zp0rxcWQ4Pj/t/8XXD2duHRcNkfg4u9iEjcKBTa2C3nDWfB2m18/7GljO5TQJ/Oqd9oq0k5BdD3uOAWq6Y6uFjLlpiw2LoKVjwJe7buP3b89XDWj9rvzmSRJKLNRwnw0dbdnH/PKwzrlcecqSeRGdaRJfvZvXXfmkX3YdD/xERXJJLyWrr5SJ9GCXBkt4789yVHs/Cjbfz62fcTXU7y6dgN+p8UdIJVIIi0KYVCglx0zBFMPr4f9724mpc/2JzockREAIVCQv34wlEM7dmJ7zy8hE1l5YkuR0REoZBIuVlhpl01jl0V1Xzn4SXURFJr/46ItD8KhQQ7qjCP2y4axaurtnJf8apElyMiaU6hkAS+UNSPi445gl89+z5vrf000eWISBpTKCQBM+Onnx9Nv64duGH2Yrbtrkx0SSKSphQKSSIvJ5NpV45jy64Kbn70bVLt/BERaR8UCknk6L4F/L/PjWDeik3MfHVtossRkTSkUEgyXzl5AOeMLOTOp5ez8CPtXxCRtqVQSDJmxi8uG0Pvglyu+sOb/G3J+kSXJCJpRKGQhDp3yOKJb57MMX07c+OcJfz8nyt0DoOItAmFQpLq3imbP331RK46sT/3Fa/maw8tYGd5VfMLiogcBoVCEsvKCPHfnz+aOy4ezUvvb+bz01/lwy27E12WiLRjCoUU8KWTjuRPXz2RbXuqmDTtFV58Xw30RCQ+FAop4qRB3fjbt07hiM65TJk1n/tfXqNzGUSk1SkUUki/rh14/Bsnc+6oXvzkH8v57qNvU15Vk+iyRKQdUSikmI7ZGUy/ahzfOfsonli0nitmvMEnO9V2W0Rah0IhBYVCxo1nD+V3Vx/HB5+UceG9r7B43bZElyUi7YBCIYVNHN2LJ755MtmZIa6Y8QaPLyxJdEkikuLiGgpmNtHMVprZKjO7pYHnbzKzZWa21MzmmdmR8aynPRreK5+53zqV4/p34buPvs1PnlxGdU0k0WWJSIqKWyiYWRiYDpwHjASuNLOR9YYtBorcfQzwGPA/8aqnPevSMYuHrjuBa8Yfyf2vfMiUB95ixx6d6CYiBy+eawonAKvcfY27VwJzgEmxA9z9BXffE518A+gbx3ratcxwiNsmjebOS47mjTVbufi3r7JqU1miyxKRFBPPUOgDfBwzXRKd15jrgKfjWE9amHxCf/7vaydRVl7FxdNfY97yTxJdkoikkIw4vrY1MK/Bs63M7GqgCDi9keenAlMBCgsLKS4ubqUS268fHBfmnsXVfPXBBVw6NJPzB2Vi1tB/iYjIPvEMhRKgX8x0X2BD/UFmdjbwn8Dp7l7R0Au5+wxgBkBRUZFPmDCh1Yttjz53Vg3fe+xtHlu6kYoOPfifS8eQmxVOdFkiksTiufnoLWComQ00syxgMjA3doCZHQv8HrjI3TfFsZa0lJsV5t4rj+V75w7jyaUbuPz3r7Fh+95ElyUiSSxuoeDu1cD1wDPAcuARd3/PzG43s4uiw34BdAIeNbMlZja3kZeTQ2RmfOuMIdz/5SLWbtnDRdNeYe7bG9Q3SUQaZKn24VBUVOQLFixIdBkpadWmMm6YvYRlG3dy3JFd+OEFIxnbr3OiyxKRNmBmC929qLlxOqM5jQzpmcffv30qP7/0aD7aupuLp7/KTQ8voXSHeieJSEChkGbCIeOK4/vzws0T+MaEwTy5dCNn/LKYu58/kGMEAAANDElEQVT7gL2V6rgqku4UCmkqLyeT/5g4nHnfPZ0zhvfg18+9z5l3FfO3Jeu1v0EkjSkU0ly/rh347ReP4+GpJ9GtUxY3zlnCJfe9xiJ1XRVJSwoFAeDEQd2Y+61T+cVlYyjZtpdLfvsaN85ZrENYRdKMQkHqhELG5UX9eOHmCVx/xhCefreUM+8q5lfPvs+eyupElycibUChIAfolJ3BzecOY95Np3P2iELumfcBZ/yymCcWlRCJaH+DSHumUJBG9evagWlXjeOxr4+nMD+Hmx55m8//9lUWfvRpoksTkThRKEizigZ05a/fPIW7Lj+G0p3lXHrf63x79mJKtu1pfmERSSkKBWmRUMi49Li+PP/dCdxw5hD+9V4pZ931Inf9ayW7K7S/QaS9UCjIQemYncFNnx3G8zdPYOLoXtz7/CrO+GUxs+evo6xcV3sTSXXqfSSHZeFH27jjyWUs+Xg7WRkhThvag/PH9OLsEYXk5WQmujwRiWpp7yOFghw2d2fhR9v4xzsbefqdUkp3ltcFxAVjenPWiJ4KCJEEUyhIQkQizqJ1BwbE6Uf14PyjFRAiiaJQkISLDYin3tnIJzsr6gIiWIMopFN2PC/+JyK1FAqSVGoD4smlG3n63X0BMeGoHpyvgBCJO4WCJC0FhEjbUyhISohEnIXrtvGPRgLijOE9ydc+CJHDplCQlNNQQJjB0J6dOLZfF8Yd2Zlj+3dhSI9OhEKW6HJFUopCQVJabUC8vnori9ZtY/G67ezYG5wcl5edwdj+nTm2XxASx/bvTOcOWQmuWCS5tTQUtOFWklIoZBw/oCvHD+gKBOdCfLhlN4vWbWdxNCSmvbCK2qatg7p3rAuIY/t3ZlhhHhlhnbAvcrC0piApa3dFNUtLdtStSSxet42tuysB6JAVZkzfAo7t34Vx0bDo3ik7wRWLJI7WFKTd65idwfjB3Rg/uBsQrE18/OleFn8chMSiddv4w0trqI6uTvTrmhsERL/OjOpTwLBeedqJLVKPQkHaDTOjf7cO9O/WgUlj+wBQXlXDu+v3rU28sWYrf1uyoW6ZPp1zGd4rj2HR24je+Qzs3pFMbXqSNKVQkHYtJzNM0YCuFEX3TQBs3LGX5Rt3sqK0jBUby1hZWsaL72+uW6PICocY1KMjI3rn7wuLXvkU5mdjpqOepH1TKEja6V2QS++CXM4cXlg3r7I6wurNu1hZWsby0p2sLC3jjTVb+cvi9XVjCnIzGd4rL7pmkc/w3nkMK8yjo060k3Ykrr/NZjYRuBsIA/e7+531ns8GHgKOA7YCV7j72njWJNKQrIwQI3rnM6J3PhfTp27+jj1VrCjdycpPyqJrFjt5fNF6dlV8VDemX9dchvfKZ0jPTuRmhgmHjHDIyAgZITMywsF02KLzw9H5oRDhEIRDoWBsdJlwzC0jZGSGQ3TKzqBjdgYds8NkZ4QT8U8kaSJuoWBmYWA6cA5QArxlZnPdfVnMsOuAbe4+xMwmAz8HrohXTSIHq6BDJicO6saJg7rVzXN3SrbtZUVpGStLo5uhSst4fsUmaiLxP5ovM2x1IbEvLDLIi4ZG7PxO+40J1013ys4gOyOMhSBkRsiCe7Pa6WCeNpeln3iuKZwArHL3NQBmNgeYBMSGwiTg1ujjx4BpZmaeasfJSloxM/p17UC/rh04Z2Thfs9FIk51xKmJONWRCJEIVEci1EScGneqa7zucU0kmI547TIRamLHR2+V1RF2VVSzu6KaXRXV7KqoYfd+09Xs2FPJ+m172B19bldlNa31VxSKCYp9oRETIqH9QyQrHCInM0ROZjh6C5GTETzOrp2fEY4Zs29eduxyGSGyY5bf//32vb9h+9cYDTqjftDFLKOwa1Q8Q6EP8HHMdAlwYmNj3L3azHYA3YAtcaxLJG5CISOrrgVH4jbzRCLO3qqauuDYXVFDWUXVvtCoqKayOkLEHXeIuBOJ3tcuXzvtMc9FnOh07PO1zzk1kWD/THl1DRVVNZRXRSivqmHH3irKqyLsrayhonrf/Oo2WLNqTP2QqB8iBmAHzrO66X2BZHWvE7NsjPohdEAkWePPxS5741lDufCYIw7nx25WPEOhoSiu/xvQkjGY2VRgKkBhYSHFxcWHXZxIOsoEOkdvDYp7noWIvTR8TcSpjEBVDVRGnMoaqKq7h8qa2ucdB9zZ7z5Sb3r/+cFEpN7zdcvUzq8/HbMcBCEYPO8HvE/t2lhttkWiY5pS//mmhtd/7qMPllG87f2m3+AwxTMUSoB+MdN9gQ2NjCkxswygAPi0/gu5+wxgBgRnNE+YMCEe9YqIpL14nqHzFjDUzAaaWRYwGZhbb8xc4Jro48uA57U/QUQkceK2phDdR3A98AzByuhMd3/PzG4HFrj7XOCPwP+a2SqCNYTJ8apHRESaF9fzFNz9KeCpevN+FPO4HLg8njWIiEjLqcGLiIjUUSiIiEgdhYKIiNRRKIiISB2FgoiI1Em5y3Ga2Wbgo2YHNqw7qdVCI5XqTaVaIbXqTaVaIbXqTaVa4fDqPdLdezQ3KOVC4XCY2YKWXKM0WaRSvalUK6RWvalUK6RWvalUK7RNvdp8JCIidRQKIiJSJ91CYUaiCzhIqVRvKtUKqVVvKtUKqVVvKtUKbVBvWu1TEBGRpqXbmoKIiDQhbULBzCaa2UozW2VmtyS6nsaYWT8ze8HMlpvZe2Z2Y6JragkzC5vZYjN7MtG1NMXMOpvZY2a2IvpvPD7RNTXFzL4T/T1418xmm1lOomuKZWYzzWyTmb0bM6+rmT1rZh9E77skssZajdT6i+jvwlIz+4uZNXr9obbUUK0xz91sZm5m3ePx3mkRCmYWBqYD5wEjgSvNbGRiq2pUNfBddx8BnAR8K4lrjXUjsDzRRbTA3cA/3X04cAxJXLOZ9QFuAIrcfTRBC/pkay//ADCx3rxbgHnuPhSYF51OBg9wYK3PAqPdfQzwPvCDti6qEQ9wYK2YWT/gHGBdvN44LUIBOAFY5e5r3L0SmANMSnBNDXL3je6+KPq4jOBDq09iq2qamfUFzgfuT3QtTTGzfOA0gut44O6V7r49sVU1KwPIjV6ZsAMHXr0wodz9JQ68WuIk4MHo4weBi9u0qEY0VKu7/8vdq6OTbxBcITLhGvl3Bfg18H2avornYUmXUOgDfBwzXUKSf9ACmNkA4FjgzcRW0qzfEPyiRpobmGCDgM3ArOimrvvNrGOii2qMu68HfknwrXAjsMPd/5XYqlqk0N03QvAlB+iZ4Hpa6lrg6UQX0RgzuwhY7+5vx/N90iUUrIF5SX3YlZl1Ah4H/t3ddya6nsaY2QXAJndfmOhaWiADGAfc5+7HArtJnk0bB4hui58EDASOADqa2dWJrap9MrP/JNh0++dE19IQM+sA/Cfwo+bGHq50CYUSoF/MdF+SbDU8lpllEgTCn939iUTX04xTgIvMbC3BZrkzzexPiS2pUSVAibvXrnk9RhASyeps4EN33+zuVcATwMkJrqklPjGz3gDR+00JrqdJZnYNcAHwxSS+Rvxggi8Hb0f/1voCi8ysV2u/UbqEwlvAUDMbaGZZBDvr5ia4pgaZmRFs817u7r9KdD3NcfcfuHtfdx9A8O/6vLsn5bdZdy8FPjazYdFZZwHLElhSc9YBJ5lZh+jvxVkk8Y7xGHOBa6KPrwH+lsBammRmE4H/AC5y9z2Jrqcx7v6Ou/d09wHRv7USYFz0d7pVpUUoRHckXQ88Q/BH9Yi7v5fYqhp1CvAlgm/cS6K3zyW6qHbk28CfzWwpMBb47wTX06joGs1jwCLgHYK/16Q6A9fMZgOvA8PMrMTMrgPuBM4xsw8IjpS5M5E11mqk1mlAHvBs9G/tdwktMqqRWtvmvZN3bUlERNpaWqwpiIhIyygURESkjkJBRETqKBRERKSOQkFEROooFETqMbOamMOBl7RmV10zG9BQ50uRZJGR6AJEktBedx+b6CJEEkFrCiItZGZrzeznZjY/ehsSnX+kmc2L9uSfZ2b9o/MLoz36347ealtUhM3sD9HrJPzLzHIT9kOJ1KNQEDlQbr3NR1fEPLfT3U8gOBP2N9F504CHoj35/wzcE51/D/Ciux9D0GOp9iz6ocB0dx8FbAcujfPPI9JiOqNZpB4z2+XunRqYvxY4093XRJsWlrp7NzPbAvR296ro/I3u3t3MNgN93b0i5jUGAM9GL0CDmf0HkOnuP4n/TybSPK0piBwcb+RxY2MaUhHzuAbt25MkolAQOThXxNy/Hn38Gvsuk/lF4JXo43nAN6DuGtb5bVWkyKHSNxSRA+Wa2ZKY6X+6e+1hqdlm9ibBF6oro/NuAGaa2fcIruw2JTr/RmBGtMNlDUFAbIx79SKHQfsURFoouk+hyN23JLoWkXjR5iMREamjNQUREamjNQUREamjUBARkToKBRERqaNQEBGROgoFERGpo1AQEZE6/x92+qv9Fb4dIwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# # summarize history for accuracy\n",
    "# plt.plot(history.history['acc'])\n",
    "# plt.plot(history.history['val_acc'])\n",
    "# # plt.title('Training')\n",
    "# plt.ylabel('Accuracy')\n",
    "# plt.xlabel('Epoch')\n",
    "# plt.legend(['Train', 'Test'], loc='upper left')\n",
    "# ax = plt.axes()        \n",
    "# ax.yaxis.grid()\n",
    "# plt.show()\n",
    "\n",
    "\n",
    "# # summarize history for loss\n",
    "# plt.plot(history.history['loss'])\n",
    "# plt.plot(history.history['val_loss'])\n",
    "# # plt.title('Loss')\n",
    "# plt.ylabel('Loss')\n",
    "# plt.xlabel('Epoch')\n",
    "# plt.legend(['Train', 'Test'], loc='upper left')\n",
    "# ax = plt.axes()        \n",
    "# ax.yaxis.grid()\n",
    "# plt.show()\n",
    "\n",
    "# summarize history for accuracy\n",
    "plt.plot(h['acc'])\n",
    "plt.plot(h['val_acc'])\n",
    "# plt.title('Training')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Test'], loc='upper left')\n",
    "ax = plt.axes()        \n",
    "ax.yaxis.grid()\n",
    "plt.show()\n",
    "\n",
    "\n",
    "# summarize history for loss\n",
    "plt.plot(h['loss'])\n",
    "plt.plot(h['val_loss'])\n",
    "# plt.title('Loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Test'], loc='upper left')\n",
    "ax = plt.axes()        \n",
    "ax.yaxis.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy\n",
      "1 60.72 76.34\n",
      "2 84.89 84.88\n",
      "3 90.35 86.27\n",
      "4 94.0 88.71\n",
      "5 96.41 89.56\n",
      "6 97.99 90.13\n",
      "7 98.85 88.9\n",
      "8 99.72 90.4\n",
      "9 99.64 91.1\n",
      "10 99.83 89.86\n",
      "11 99.86 91.55\n",
      "12 99.99 91.58\n",
      "13 99.99 91.37\n",
      "14 100.0 91.79\n",
      "15 100.0 91.49\n",
      "\n",
      "Loss\n",
      "1 112.19 68.94\n",
      "2 44.34 43.41\n",
      "3 27.65 43.09\n",
      "4 17.58 36.29\n",
      "5 10.55 36.1\n",
      "6 6.43 34.18\n",
      "7 3.71 42.98\n",
      "8 1.53 38.54\n",
      "9 1.35 37.82\n",
      "10 0.71 50.6\n",
      "11 0.62 39.85\n",
      "12 0.18 42.42\n",
      "13 0.09 41.75\n",
      "14 0.02 43.19\n",
      "15 0.01 42.75\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy\")\n",
    "for i,j in enumerate(zip(history.history['acc'], history.history['val_acc'])):\n",
    "    acc = j[0]*100\n",
    "    val_acc = j[1]*100\n",
    "    print(i+1,acc.round(decimals=2),val_acc.round(decimals=2))\n",
    "    \n",
    "print()\n",
    "\n",
    "print(\"Loss\")\n",
    "for i,j in enumerate(zip(history.history['loss'], history.history['val_loss'])):\n",
    "    acc = j[0]*100\n",
    "    val_acc = j[1]*100\n",
    "    print(i+1,acc.round(decimals=2),val_acc.round(decimals=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_11 (Conv2D)           (None, 19, 10, 32)        160       \n",
      "_________________________________________________________________\n",
      "batch_normalization_10 (Batc (None, 19, 10, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv2d_12 (Conv2D)           (None, 18, 9, 48)         6192      \n",
      "_________________________________________________________________\n",
      "batch_normalization_11 (Batc (None, 18, 9, 48)         192       \n",
      "_________________________________________________________________\n",
      "conv2d_13 (Conv2D)           (None, 17, 8, 120)        23160     \n",
      "_________________________________________________________________\n",
      "batch_normalization_12 (Batc (None, 17, 8, 120)        480       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_9 (MaxPooling2 (None, 8, 4, 120)         0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 3840)              0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 128)               491648    \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 7)                 455       \n",
      "=================================================================\n",
      "Total params: 530,671\n",
      "Trainable params: 530,271\n",
      "Non-trainable params: 400\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'_built': True,\n",
      " '_collected_trainable_weights': [<tf.Variable 'conv2d_11/kernel:0' shape=(2, 2, 1, 32) dtype=float32_ref>,\n",
      "                                  <tf.Variable 'conv2d_11/bias:0' shape=(32,) dtype=float32_ref>,\n",
      "                                  <tf.Variable 'batch_normalization_10/gamma:0' shape=(32,) dtype=float32_ref>,\n",
      "                                  <tf.Variable 'batch_normalization_10/beta:0' shape=(32,) dtype=float32_ref>,\n",
      "                                  <tf.Variable 'conv2d_12/kernel:0' shape=(2, 2, 32, 48) dtype=float32_ref>,\n",
      "                                  <tf.Variable 'conv2d_12/bias:0' shape=(48,) dtype=float32_ref>,\n",
      "                                  <tf.Variable 'batch_normalization_11/gamma:0' shape=(48,) dtype=float32_ref>,\n",
      "                                  <tf.Variable 'batch_normalization_11/beta:0' shape=(48,) dtype=float32_ref>,\n",
      "                                  <tf.Variable 'conv2d_13/kernel:0' shape=(2, 2, 48, 120) dtype=float32_ref>,\n",
      "                                  <tf.Variable 'conv2d_13/bias:0' shape=(120,) dtype=float32_ref>,\n",
      "                                  <tf.Variable 'batch_normalization_12/gamma:0' shape=(120,) dtype=float32_ref>,\n",
      "                                  <tf.Variable 'batch_normalization_12/beta:0' shape=(120,) dtype=float32_ref>,\n",
      "                                  <tf.Variable 'dense_4/kernel:0' shape=(3840, 128) dtype=float32_ref>,\n",
      "                                  <tf.Variable 'dense_4/bias:0' shape=(128,) dtype=float32_ref>,\n",
      "                                  <tf.Variable 'dense_5/kernel:0' shape=(128, 64) dtype=float32_ref>,\n",
      "                                  <tf.Variable 'dense_5/bias:0' shape=(64,) dtype=float32_ref>,\n",
      "                                  <tf.Variable 'dense_6/kernel:0' shape=(64, 7) dtype=float32_ref>,\n",
      "                                  <tf.Variable 'dense_6/bias:0' shape=(7,) dtype=float32_ref>],\n",
      " '_compute_previous_mask': True,\n",
      " '_expects_training_arg': False,\n",
      " '_feed_input_names': ['conv2d_11_input'],\n",
      " '_feed_input_shapes': [(None, 20, 11, 1)],\n",
      " '_feed_inputs': [<tf.Tensor 'conv2d_11_input:0' shape=(?, 20, 11, 1) dtype=float32>],\n",
      " '_feed_loss_fns': [<function categorical_crossentropy at 0x7fc87e10f378>],\n",
      " '_feed_output_names': ['dense_6'],\n",
      " '_feed_output_shapes': [(None, 7)],\n",
      " '_feed_outputs': [<tf.Tensor 'dense_6/Softmax:0' shape=(?, 7) dtype=float32>],\n",
      " '_feed_sample_weight_modes': [None],\n",
      " '_feed_sample_weights': [<tf.Tensor 'dense_6_sample_weights:0' shape=(?,) dtype=float32>],\n",
      " '_feed_targets': [<tf.Tensor 'dense_6_target:0' shape=(?, ?) dtype=float32>],\n",
      " '_function_kwargs': {},\n",
      " '_inbound_nodes': [<keras.engine.base_layer.Node object at 0x7fc86c8422b0>],\n",
      " '_initial_weights': None,\n",
      " '_input_coordinates': [(<keras.engine.input_layer.InputLayer object at 0x7fc86c842080>,\n",
      "                         0,\n",
      "                         0)],\n",
      " '_input_layers': [<keras.engine.input_layer.InputLayer object at 0x7fc86c842080>],\n",
      " '_is_compiled': True,\n",
      " '_is_graph_network': True,\n",
      " '_layers': [<keras.engine.input_layer.InputLayer object at 0x7fc86c842080>,\n",
      "             <keras.layers.convolutional.Conv2D object at 0x7fc86c842eb8>,\n",
      "             <keras.layers.normalization.BatchNormalization object at 0x7fc86c842390>,\n",
      "             <keras.layers.convolutional.Conv2D object at 0x7fc86c884f60>,\n",
      "             <keras.layers.normalization.BatchNormalization object at 0x7fc86c857908>,\n",
      "             <keras.layers.convolutional.Conv2D object at 0x7fc86c80be80>,\n",
      "             <keras.layers.normalization.BatchNormalization object at 0x7fc86da6df98>,\n",
      "             <keras.layers.pooling.MaxPooling2D object at 0x7fc86c8343c8>,\n",
      "             <keras.layers.core.Flatten object at 0x7fc86c8d17f0>,\n",
      "             <keras.layers.core.Dense object at 0x7fc86c931198>,\n",
      "             <keras.layers.core.Dense object at 0x7fc86c9078d0>,\n",
      "             <keras.layers.core.Dense object at 0x7fc86c97ec50>],\n",
      " '_layers_by_depth': {0: [<keras.layers.core.Dense object at 0x7fc86c97ec50>],\n",
      "                      1: [<keras.layers.core.Dense object at 0x7fc86c9078d0>],\n",
      "                      2: [<keras.layers.core.Dense object at 0x7fc86c931198>],\n",
      "                      3: [<keras.layers.core.Flatten object at 0x7fc86c8d17f0>],\n",
      "                      4: [<keras.layers.pooling.MaxPooling2D object at 0x7fc86c8343c8>],\n",
      "                      5: [<keras.layers.normalization.BatchNormalization object at 0x7fc86da6df98>],\n",
      "                      6: [<keras.layers.convolutional.Conv2D object at 0x7fc86c80be80>],\n",
      "                      7: [<keras.layers.normalization.BatchNormalization object at 0x7fc86c857908>],\n",
      "                      8: [<keras.layers.convolutional.Conv2D object at 0x7fc86c884f60>],\n",
      "                      9: [<keras.layers.normalization.BatchNormalization object at 0x7fc86c842390>],\n",
      "                      10: [<keras.layers.convolutional.Conv2D object at 0x7fc86c842eb8>],\n",
      "                      11: [<keras.engine.input_layer.InputLayer object at 0x7fc86c842080>]},\n",
      " '_losses': [],\n",
      " '_network_nodes': {'batch_normalization_10_ib-0',\n",
      "                    'batch_normalization_11_ib-0',\n",
      "                    'batch_normalization_12_ib-0',\n",
      "                    'conv2d_11_ib-0',\n",
      "                    'conv2d_11_input_ib-0',\n",
      "                    'conv2d_12_ib-0',\n",
      "                    'conv2d_13_ib-0',\n",
      "                    'dense_4_ib-0',\n",
      "                    'dense_5_ib-0',\n",
      "                    'dense_6_ib-0',\n",
      "                    'flatten_2_ib-0',\n",
      "                    'max_pooling2d_9_ib-0'},\n",
      " '_nodes_by_depth': {0: [<keras.engine.base_layer.Node object at 0x7fc86c968630>],\n",
      "                     1: [<keras.engine.base_layer.Node object at 0x7fc86c973e48>],\n",
      "                     2: [<keras.engine.base_layer.Node object at 0x7fc86c9074e0>],\n",
      "                     3: [<keras.engine.base_layer.Node object at 0x7fc86c913208>],\n",
      "                     4: [<keras.engine.base_layer.Node object at 0x7fc86c8c7198>],\n",
      "                     5: [<keras.engine.base_layer.Node object at 0x7fc86c8c7be0>],\n",
      "                     6: [<keras.engine.base_layer.Node object at 0x7fc86c8db160>],\n",
      "                     7: [<keras.engine.base_layer.Node object at 0x7fc86c816710>],\n",
      "                     8: [<keras.engine.base_layer.Node object at 0x7fc86c80b438>],\n",
      "                     9: [<keras.engine.base_layer.Node object at 0x7fc86c84dda0>],\n",
      "                     10: [<keras.engine.base_layer.Node object at 0x7fc86c8b8b38>],\n",
      "                     11: [<keras.engine.base_layer.Node object at 0x7fc86c842160>]},\n",
      " '_outbound_nodes': [],\n",
      " '_output_coordinates': [(<keras.layers.core.Dense object at 0x7fc86c97ec50>,\n",
      "                          0,\n",
      "                          0)],\n",
      " '_output_layers': [<keras.layers.core.Dense object at 0x7fc86c97ec50>],\n",
      " '_output_mask_cache': {'140498790786104_94335365704496': None},\n",
      " '_output_shape_cache': {},\n",
      " '_output_tensor_cache': {},\n",
      " '_per_input_losses': {},\n",
      " '_per_input_updates': {},\n",
      " '_updates': [],\n",
      " '_uses_inputs_arg': True,\n",
      " 'history': <keras.callbacks.History object at 0x7fc86c842d68>,\n",
      " 'input_names': ['conv2d_11_input'],\n",
      " 'inputs': [<tf.Tensor 'conv2d_11_input:0' shape=(?, 20, 11, 1) dtype=float32>],\n",
      " 'loss': <function categorical_crossentropy at 0x7fc87e10f378>,\n",
      " 'loss_functions': [<function categorical_crossentropy at 0x7fc87e10f378>],\n",
      " 'loss_weights': None,\n",
      " 'metrics': ['accuracy'],\n",
      " 'metrics_names': ['loss', 'acc'],\n",
      " 'metrics_tensors': [<tf.Tensor 'metrics_1/acc/Mean:0' shape=() dtype=float32>],\n",
      " 'metrics_updates': [],\n",
      " 'name': 'sequential_5',\n",
      " 'optimizer': <keras.optimizers.Adadelta object at 0x7fc86c9731d0>,\n",
      " 'output_names': ['dense_6'],\n",
      " 'outputs': [<tf.Tensor 'dense_6/Softmax:0' shape=(?, 7) dtype=float32>],\n",
      " 'predict_function': None,\n",
      " 'sample_weight_mode': None,\n",
      " 'sample_weight_modes': [None],\n",
      " 'sample_weights': [<tf.Tensor 'dense_6_sample_weights:0' shape=(?,) dtype=float32>],\n",
      " 'stateful_metric_functions': [],\n",
      " 'stateful_metric_names': [],\n",
      " 'stop_training': False,\n",
      " 'supports_masking': False,\n",
      " 'targets': [<tf.Tensor 'dense_6_target:0' shape=(?, ?) dtype=float32>],\n",
      " 'test_function': <keras.backend.tensorflow_backend.Function object at 0x7fc86c973828>,\n",
      " 'total_loss': <tf.Tensor 'loss_1/mul:0' shape=() dtype=float32>,\n",
      " 'train_function': <keras.backend.tensorflow_backend.Function object at 0x7fc86c9683c8>,\n",
      " 'trainable': True,\n",
      " 'weighted_metrics': None}\n"
     ]
    }
   ],
   "source": [
    "# A nice method to check all the attributes of an object\n",
    "from pprint import pprint\n",
    "pprint(vars(model))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "left\n"
     ]
    }
   ],
   "source": [
    "print(predict('./eng_data/left/1cc80e39_nohash_0.wav', model=model))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confidence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def confidence(filepath, model):\n",
    "    sample = wav2mfcc(filepath)\n",
    "    sample_reshaped = sample.reshape(1, feature_dim_1, feature_dim_2, channel)\n",
    "    u = model.predict(sample_reshaped)\n",
    "#     print(u)\n",
    "    return u, get_labels()[0][\n",
    "            np.argmax(model.predict(sample_reshaped))\n",
    "    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "labs = get_labels()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "down      3.5824613e-16\n",
      "go      3.0647503e-12\n",
      "left      1.0\n",
      "on      1.160079e-13\n",
      "right      4.758897e-09\n",
      "up      3.4712777e-10\n",
      "yes      1.6291062e-11\n",
      "left\n"
     ]
    }
   ],
   "source": [
    "confs, word = confidence('./eng_data/left/1cc80e39_nohash_0.wav', model=model)\n",
    "for lab,conf in zip(labs,confs[0]):\n",
    "    print(lab , \"    \" ,conf)\n",
    "print(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def confidence(new_sample, model):\n",
    "\n",
    "    new_sample = new_sample.reshape(-1)\n",
    "    # print(\"Ye: \" + str(new_sample.shape))\n",
    "    sample = array2mfcc(new_sample)\n",
    "    sample_reshaped = sample.reshape(1,20,11,1)\n",
    "    return model.predict(sample_reshaped)\n",
    "\n",
    "\n",
    "def get_conf(inp, model):\n",
    "    labs = get_labels()[0]\n",
    "    inp = np.array(inp)\n",
    "    confs = confidence(inp, model=model)\n",
    "#     print(\"zzz\")\n",
    "#     print(confs[0])\n",
    "#     print(labs)\n",
    "    for lab,conf in zip(labs,confs[0]):\n",
    "        print(lab , \"    \" ,conf)\n",
    "    # print(word)\n",
    "    \n",
    "def get_conf2(sample, model):\n",
    "    for a,b in zip(get_labels()[0], model.predict(sample)[0]):\n",
    "        print(a, \"     \", b)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in model.predict(sample_reshaped)[0]:\n",
    "#     print(i)\n",
    "# model.predict(sample_reshaped)[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing the model in realtime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sounddevice as sd\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Converting the recorded array to mfcc\n",
    "Instead of recording speech into a file and reading from it to create an array, this function would directly convert the recorded array into mfcc coefficients."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def array2mfcc(wave, max_len=11):\n",
    "    sr = 16000\n",
    "    wave = wave[::3]\n",
    "    mfcc = librosa.feature.mfcc(wave, sr=16000)\n",
    "\n",
    "    # If maximum length exceeds mfcc lengths then pad the remaining ones\n",
    "    if (max_len > mfcc.shape[1]):\n",
    "        pad_width = max_len - mfcc.shape[1]\n",
    "        mfcc = np.pad(mfcc, pad_width=((0, 0), (0, pad_width)), mode='constant')\n",
    "\n",
    "    # Else cutoff the remaining parts\n",
    "    else:\n",
    "        mfcc = mfcc[:, :max_len]\n",
    "    \n",
    "    return mfcc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['down', 'go', 'left', 'on', 'right', 'up', 'yes']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## For reference: These are the words trained to the classifier \n",
    "get_labels()[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Recording...\n",
    "Please speak after executing the next cell.   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "duration = 1  # seconds\n",
    "fs = 16000\n",
    "new_sample = sd.rec(int(duration * fs), samplerate=fs, channels=1)\n",
    "time.sleep(duration)\n",
    "sd.play(new_sample, fs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "go\n",
      "down       0.00019802894\n",
      "go       0.91213584\n",
      "left       7.179345e-05\n",
      "on       0.0728288\n",
      "right       0.014749701\n",
      "up       1.5815662e-05\n",
      "yes       1.6174128e-07\n"
     ]
    }
   ],
   "source": [
    "new_sample = new_sample.reshape(-1)\n",
    "sample = array2mfcc(new_sample)\n",
    "sample_reshaped = sample.reshape(1,20,11,1)\n",
    "\n",
    "print(get_labels()[0][np.argmax(model.predict(sample_reshaped))])\n",
    "get_conf2(sample_reshaped, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "..stop.."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_json = model.to_json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"model73.json\", \"w\") as json_file:\n",
    "    json_file.write(model_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_weights(\"model73.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open(\"history73\", \"wb\") as f:\n",
    "    pickle.dump(history.history, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded model from disk\n"
     ]
    }
   ],
   "source": [
    "from keras.models import model_from_json\n",
    "# load json and create model\n",
    "json_file = open('model73.json', 'r')\n",
    "model_json = json_file.read()\n",
    "json_file.close()\n",
    "model = model_from_json(model_json)\n",
    "# load weights into new model\n",
    "model.load_weights(\"model73.h5\")\n",
    "print(\"Loaded model from disk\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open(\"history73\", \"rb\") as f:\n",
    "    h = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}