Capsule Network - SampleDataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "train_filename = \"data_preprocessed/train_data_sample_gray.p\"\n",
    "(train_labels, train_data, train_tensors) = pickle.load(open(train_filename, mode='rb'))\n",
    "\n",
    "valid_filename = \"data_preprocessed/valid_data_sample_gray.p\"\n",
    "(valid_labels, valid_data, valid_tensors) = pickle.load(open(valid_filename, mode='rb'))\n",
    "\n",
    "test_filename = \"data_preprocessed/test_data_sample_gray.p\"\n",
    "(test_labels, test_data, test_tensors) = pickle.load(open(test_filename, mode='rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert label from 1D to 2D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def onhotLabels(label):\n",
    "    from sklearn.preprocessing import OneHotEncoder\n",
    "    enc = OneHotEncoder()\n",
    "    enc.fit(label)\n",
    "    return enc.transform(label).toarray()\n",
    "\n",
    "train_labels = onhotLabels(train_labels)\n",
    "valid_labels = onhotLabels(valid_labels)\n",
    "test_labels = onhotLabels(test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CapsNet model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 64, 64, 1)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv1 (Conv2D)                  (None, 32, 32, 256)  20992       input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "primarycap_conv2d (Conv2D)      (None, 16, 16, 256)  5308672     conv1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "primarycap_reshape (Reshape)    (None, 8192, 8)      0           primarycap_conv2d[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "primarycap_squash (Lambda)      (None, 8192, 8)      0           primarycap_reshape[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "digitcaps (CapsuleLayer)        (None, 2, 16)        2097152     primarycap_squash[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, 2)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "mask_1 (Mask)                   (None, 32)           0           digitcaps[0][0]                  \n",
      "                                                                 input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "capsnet (Length)                (None, 2)            0           digitcaps[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "decoder (Sequential)            (None, 64, 64, 1)    4740608     mask_1[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 12,167,424\n",
      "Trainable params: 12,167,424\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import argparse\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras import callbacks\n",
    "\n",
    "import numpy as np\n",
    "from keras import layers, models, optimizers\n",
    "from keras import backend as K\n",
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image\n",
    "from capsulelayers import CapsuleLayer, PrimaryCap, Length, Mask\n",
    "\n",
    "def CapsNet(input_shape, n_class, routings):\n",
    "    \"\"\"\n",
    "    A Capsule Network on MNIST.\n",
    "    :param input_shape: data shape, 3d, [width, height, channels]\n",
    "    :param n_class: number of classes\n",
    "    :param routings: number of routing iterations\n",
    "    :return: Two Keras Models, the first one used for training, and the second one for evaluation.\n",
    "            `eval_model` can also be used for training.\n",
    "    \"\"\"\n",
    "    x = layers.Input(shape=input_shape)\n",
    "\n",
    "    # Layer 1: Just a conventional Conv2D layer\n",
    "    conv1 = layers.Conv2D(filters=256, kernel_size=9, strides=2, padding='same', activation='relu', name='conv1')(x)\n",
    "\n",
    "    # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]\n",
    "    primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='same')\n",
    "\n",
    "    # Layer 3: Capsule layer. Routing algorithm works here.\n",
    "    digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings,\n",
    "                             name='digitcaps')(primarycaps)\n",
    "\n",
    "    # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.\n",
    "    # If using tensorflow, this will not be necessary. :)\n",
    "    out_caps = Length(name='capsnet')(digitcaps)\n",
    "\n",
    "    # Decoder network.\n",
    "    y = layers.Input(shape=(n_class,))\n",
    "    masked_by_y = Mask()([digitcaps, y])  # The true label is used to mask the output of capsule layer. For training\n",
    "    masked = Mask()(digitcaps)  # Mask using the capsule with maximal length. For prediction\n",
    "\n",
    "    # Shared Decoder model in training and prediction\n",
    "    decoder = models.Sequential(name='decoder')\n",
    "    decoder.add(layers.Dense(512, activation='relu', input_dim=16*n_class))\n",
    "    decoder.add(layers.Dense(1024, activation='relu'))\n",
    "    decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))\n",
    "    decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))\n",
    "\n",
    "    # Models for training and evaluation (prediction)\n",
    "    train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])\n",
    "    eval_model = models.Model(x, [out_caps, decoder(masked)])\n",
    "\n",
    "    # manipulate model\n",
    "    noise = layers.Input(shape=(n_class, 16))\n",
    "    noised_digitcaps = layers.Add()([digitcaps, noise])\n",
    "    masked_noised_y = Mask()([noised_digitcaps, y])\n",
    "    manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))\n",
    "    return train_model, eval_model, manipulate_model\n",
    "\n",
    "model, eval_model, manipulate_model = CapsNet(input_shape=train_tensors.shape[1:],\n",
    "                                                  n_class=len(np.unique(train_labels)),\n",
    "                                                  routings=4)\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "\n",
    "def binary_accuracy(y_true, y_pred):\n",
    "    y_true, y_pred = K.cast(K.argmax(y_true, axis=1), K.floatx()), K.cast(K.argmax(y_pred, axis=1), K.floatx())\n",
    "    return K.mean(K.equal(y_true, K.round(y_pred)))\n",
    "\n",
    "def precision_threshold(threshold = 0.5):\n",
    "    def precision(y_true, y_pred):\n",
    "        y_true, y_pred = K.cast(K.argmax(y_true, axis=1), K.floatx()), K.cast(K.argmax(y_pred, axis=1), K.floatx())\n",
    "        threshold_value = threshold\n",
    "        y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), threshold_value), K.floatx())\n",
    "        true_positives = K.round(K.sum(K.clip(y_true * y_pred, 0, 1)))\n",
    "        predicted_positives = K.sum(y_pred)\n",
    "        precision_ratio = true_positives / (predicted_positives + K.epsilon())\n",
    "        return precision_ratio\n",
    "    return precision\n",
    "\n",
    "def recall_threshold(threshold = 0.5):\n",
    "    def recall(y_true, y_pred):\n",
    "        y_true, y_pred = K.cast(K.argmax(y_true, axis=1), K.floatx()), K.cast(K.argmax(y_pred, axis=1), K.floatx())\n",
    "        threshold_value = threshold\n",
    "        y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), threshold_value), K.floatx())\n",
    "        true_positives = K.round(K.sum(K.clip(y_true * y_pred, 0, 1)))\n",
    "        possible_positives = K.sum(K.clip(y_true, 0, 1))\n",
    "        recall_ratio = true_positives / (possible_positives + K.epsilon())\n",
    "        return recall_ratio\n",
    "    return recall\n",
    "\n",
    "def fbeta_score_threshold(beta = 1, threshold = 0.5):\n",
    "    def fbeta_score(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        beta_value = beta\n",
    "        p = precision_threshold(threshold_value)(y_true, y_pred)\n",
    "        r = recall_threshold(threshold_value)(y_true, y_pred)\n",
    "        bb = beta_value ** 2\n",
    "        fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())\n",
    "        return fbeta_score\n",
    "    return fbeta_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def margin_loss(y_true, y_pred):\n",
    "    \"\"\"\n",
    "    Margin loss for Eq.(4). When y_true[i, :] contains not just one `1`, this loss should work too. Not test it.\n",
    "    :param y_true: [None, n_classes]\n",
    "    :param y_pred: [None, num_capsule]\n",
    "    :return: a scalar loss value.\n",
    "    \"\"\"\n",
    "    L = y_true * K.square(K.maximum(0., 0.9 - y_pred)) + \\\n",
    "        0.5 * (1 - y_true) * K.square(K.maximum(0., y_pred - 0.1))\n",
    "\n",
    "    return K.mean(K.sum(L, 1))\n",
    "\n",
    "def train(model, data, lr, lr_decay, lam_recon, batch_size, shift_fraction, epochs):\n",
    "    \"\"\"\n",
    "    Training a CapsuleNet\n",
    "    :param model: the CapsuleNet model\n",
    "    :param data: a tuple containing training and testing data, like `((x_train, y_train), (x_test, y_test))`\n",
    "    :param args: arguments\n",
    "    :return: The trained model\n",
    "    \"\"\"\n",
    "    # unpacking the data\n",
    "    (x_train, y_train), (x_test, y_test) = data\n",
    "\n",
    "    # callbacks\n",
    "    log = callbacks.CSVLogger('saved_models/CapsNet_log.csv')\n",
    "    tb = callbacks.TensorBoard(log_dir='saved_models/tensorboard-logs',\n",
    "                               batch_size=batch_size, histogram_freq=0)\n",
    "    checkpoint = callbacks.ModelCheckpoint(filepath='saved_models/CapsNet.best.from_scratch.hdf5', \n",
    "                               verbose=1, save_best_only=True)\n",
    "    cb_lr_decay = callbacks.LearningRateScheduler(schedule=lambda epoch: lr * (lr_decay ** epoch))\n",
    "\n",
    "    # compile the model\n",
    "    model.compile(optimizer='sgd', loss='binary_crossentropy', \n",
    "              metrics=[precision_threshold(threshold = 0.5), \n",
    "                      recall_threshold(threshold = 0.5), \n",
    "                      fbeta_score_threshold(beta=0.5, threshold = 0.5),\n",
    "                      binary_accuracy])\n",
    "\n",
    "    # Training without data augmentation:\n",
    "#     model.fit([x_train, y_train], [y_train, x_train], batch_size=batch_size, epochs=epochs,\n",
    "#               validation_data=[[x_test, y_test], [y_test, x_test]], callbacks=[log, tb, checkpoint, cb_lr_decay])\n",
    "\n",
    "    # Begin: Training with data augmentation ---------------------------------------------------------------------#\n",
    "    def train_generator(x, y, batch_size, shift_fraction=0.):\n",
    "        train_datagen = ImageDataGenerator(width_shift_range=shift_fraction,\n",
    "                                           height_shift_range=shift_fraction)  # shift up to 2 pixel for MNIST\n",
    "        generator = train_datagen.flow(x, y, batch_size=batch_size)\n",
    "        while 1:\n",
    "            x_batch, y_batch = generator.next()\n",
    "            yield ([x_batch, y_batch], [y_batch, x_batch])\n",
    "\n",
    "    # Training with data augmentation. If shift_fraction=0., also no augmentation.\n",
    "    model.fit_generator(generator=train_generator(x_train, y_train, batch_size, shift_fraction),\n",
    "                        steps_per_epoch=int(y_train.shape[0] / batch_size),\n",
    "                        epochs=epochs,\n",
    "                        validation_data=[[x_test, y_test], [y_test, x_test]],\n",
    "                        callbacks=[log, tb, checkpoint, cb_lr_decay])\n",
    "    # End: Training with data augmentation -----------------------------------------------------------------------#\n",
    "\n",
    "    from utils import plot_log\n",
    "    plot_log('saved_models/CapsNet_log.csv', show=True)\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.5321 - capsnet_loss: 0.8389 - decoder_loss: 0.6932 - capsnet_precision: 0.0795 - capsnet_recall: 0.0575 - capsnet_fbeta_score: 0.0610 - capsnet_binary_accuracy: 0.5408 - decoder_precision: 0.9648 - decoder_recall: 0.9693 - decoder_fbeta_score: 0.9656 - decoder_binary_accuracy: 0.0118Epoch 00001: val_loss improved from inf to 1.39722, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 37s 345ms/step - loss: 1.5309 - capsnet_loss: 0.8378 - decoder_loss: 0.6932 - capsnet_precision: 0.0787 - capsnet_recall: 0.0569 - capsnet_fbeta_score: 0.0604 - capsnet_binary_accuracy: 0.5404 - decoder_precision: 0.9650 - decoder_recall: 0.9693 - decoder_fbeta_score: 0.9658 - decoder_binary_accuracy: 0.0118 - val_loss: 1.3972 - val_capsnet_loss: 0.7041 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.0000e+00 - val_capsnet_recall: 0.0000e+00 - val_capsnet_fbeta_score: 0.0000e+00 - val_capsnet_binary_accuracy: 0.5400 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9697 - val_decoder_fbeta_score: 0.9615 - val_decoder_binary_accuracy: 0.0097\n",
      "Epoch 2/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3999 - capsnet_loss: 0.7067 - decoder_loss: 0.6932 - capsnet_precision: 0.0000e+00 - capsnet_recall: 0.0000e+00 - capsnet_fbeta_score: 0.0000e+00 - capsnet_binary_accuracy: 0.5393 - decoder_precision: 0.9649 - decoder_recall: 0.9697 - decoder_fbeta_score: 0.9658 - decoder_binary_accuracy: 0.0119Epoch 00002: val_loss improved from 1.39722 to 1.39503, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 34s 321ms/step - loss: 1.3997 - capsnet_loss: 0.7066 - decoder_loss: 0.6932 - capsnet_precision: 0.0000e+00 - capsnet_recall: 0.0000e+00 - capsnet_fbeta_score: 0.0000e+00 - capsnet_binary_accuracy: 0.5401 - decoder_precision: 0.9650 - decoder_recall: 0.9697 - decoder_fbeta_score: 0.9658 - decoder_binary_accuracy: 0.0118 - val_loss: 1.3950 - val_capsnet_loss: 0.7019 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.0000e+00 - val_capsnet_recall: 0.0000e+00 - val_capsnet_fbeta_score: 0.0000e+00 - val_capsnet_binary_accuracy: 0.5400 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9695 - val_decoder_fbeta_score: 0.9615 - val_decoder_binary_accuracy: 0.0097\n",
      "Epoch 3/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3981 - capsnet_loss: 0.7049 - decoder_loss: 0.6932 - capsnet_precision: 0.0286 - capsnet_recall: 0.0027 - capsnet_fbeta_score: 0.0099 - capsnet_binary_accuracy: 0.5375 - decoder_precision: 0.9639 - decoder_recall: 0.9696 - decoder_fbeta_score: 0.9650 - decoder_binary_accuracy: 0.0117Epoch 00003: val_loss improved from 1.39503 to 1.39319, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 34s 322ms/step - loss: 1.3978 - capsnet_loss: 0.7047 - decoder_loss: 0.6932 - capsnet_precision: 0.0283 - capsnet_recall: 0.0027 - capsnet_fbeta_score: 0.0098 - capsnet_binary_accuracy: 0.5389 - decoder_precision: 0.9641 - decoder_recall: 0.9696 - decoder_fbeta_score: 0.9651 - decoder_binary_accuracy: 0.0117 - val_loss: 1.3932 - val_capsnet_loss: 0.7000 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.0000e+00 - val_capsnet_recall: 0.0000e+00 - val_capsnet_fbeta_score: 0.0000e+00 - val_capsnet_binary_accuracy: 0.5400 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9694 - val_decoder_fbeta_score: 0.9614 - val_decoder_binary_accuracy: 0.0097\n",
      "Epoch 4/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3941 - capsnet_loss: 0.7009 - decoder_loss: 0.6932 - capsnet_precision: 0.0000e+00 - capsnet_recall: 0.0000e+00 - capsnet_fbeta_score: 0.0000e+00 - capsnet_binary_accuracy: 0.5440 - decoder_precision: 0.9660 - decoder_recall: 0.9695 - decoder_fbeta_score: 0.9667 - decoder_binary_accuracy: 0.0120Epoch 00004: val_loss improved from 1.39319 to 1.39141, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 34s 324ms/step - loss: 1.3941 - capsnet_loss: 0.7009 - decoder_loss: 0.6932 - capsnet_precision: 0.0000e+00 - capsnet_recall: 0.0000e+00 - capsnet_fbeta_score: 0.0000e+00 - capsnet_binary_accuracy: 0.5439 - decoder_precision: 0.9662 - decoder_recall: 0.9695 - decoder_fbeta_score: 0.9668 - decoder_binary_accuracy: 0.0119 - val_loss: 1.3914 - val_capsnet_loss: 0.6983 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.0291 - val_capsnet_recall: 0.0018 - val_capsnet_fbeta_score: 0.0073 - val_capsnet_binary_accuracy: 0.5400 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9693 - val_decoder_fbeta_score: 0.9614 - val_decoder_binary_accuracy: 0.0097\n",
      "Epoch 5/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3938 - capsnet_loss: 0.7006 - decoder_loss: 0.6932 - capsnet_precision: 0.0952 - capsnet_recall: 0.0066 - capsnet_fbeta_score: 0.0257 - capsnet_binary_accuracy: 0.5399 - decoder_precision: 0.9650 - decoder_recall: 0.9694 - decoder_fbeta_score: 0.9658 - decoder_binary_accuracy: 0.0121Epoch 00005: val_loss improved from 1.39141 to 1.38998, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 34s 323ms/step - loss: 1.3937 - capsnet_loss: 0.7005 - decoder_loss: 0.6932 - capsnet_precision: 0.1038 - capsnet_recall: 0.0073 - capsnet_fbeta_score: 0.0283 - capsnet_binary_accuracy: 0.5407 - decoder_precision: 0.9652 - decoder_recall: 0.9694 - decoder_fbeta_score: 0.9660 - decoder_binary_accuracy: 0.0121 - val_loss: 1.3900 - val_capsnet_loss: 0.6968 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.0582 - val_capsnet_recall: 0.0038 - val_capsnet_fbeta_score: 0.0149 - val_capsnet_binary_accuracy: 0.5391 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9692 - val_decoder_fbeta_score: 0.9614 - val_decoder_binary_accuracy: 0.0096\n",
      "Epoch 6/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3911 - capsnet_loss: 0.6979 - decoder_loss: 0.6932 - capsnet_precision: 0.0444 - capsnet_recall: 0.0037 - capsnet_fbeta_score: 0.0131 - capsnet_binary_accuracy: 0.5470 - decoder_precision: 0.9645 - decoder_recall: 0.9693 - decoder_fbeta_score: 0.9654 - decoder_binary_accuracy: 0.0120Epoch 00006: val_loss improved from 1.38998 to 1.38856, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 34s 323ms/step - loss: 1.3912 - capsnet_loss: 0.6980 - decoder_loss: 0.6932 - capsnet_precision: 0.0440 - capsnet_recall: 0.0037 - capsnet_fbeta_score: 0.0130 - capsnet_binary_accuracy: 0.5463 - decoder_precision: 0.9643 - decoder_recall: 0.9693 - decoder_fbeta_score: 0.9653 - decoder_binary_accuracy: 0.0120 - val_loss: 1.3886 - val_capsnet_loss: 0.6954 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.1455 - val_capsnet_recall: 0.0132 - val_capsnet_fbeta_score: 0.0479 - val_capsnet_binary_accuracy: 0.5409 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9692 - val_decoder_fbeta_score: 0.9614 - val_decoder_binary_accuracy: 0.0096\n",
      "Epoch 7/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3907 - capsnet_loss: 0.6976 - decoder_loss: 0.6932 - capsnet_precision: 0.2381 - capsnet_recall: 0.0208 - capsnet_fbeta_score: 0.0734 - capsnet_binary_accuracy: 0.5437 - decoder_precision: 0.9644 - decoder_recall: 0.9691 - decoder_fbeta_score: 0.9652 - decoder_binary_accuracy: 0.0119Epoch 00007: val_loss improved from 1.38856 to 1.38721, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 34s 323ms/step - loss: 1.3906 - capsnet_loss: 0.6974 - decoder_loss: 0.6932 - capsnet_precision: 0.2452 - capsnet_recall: 0.0213 - capsnet_fbeta_score: 0.0755 - capsnet_binary_accuracy: 0.5445 - decoder_precision: 0.9646 - decoder_recall: 0.9691 - decoder_fbeta_score: 0.9654 - decoder_binary_accuracy: 0.0119 - val_loss: 1.3872 - val_capsnet_loss: 0.6941 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.0873 - val_capsnet_recall: 0.0083 - val_capsnet_fbeta_score: 0.0296 - val_capsnet_binary_accuracy: 0.5400 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9691 - val_decoder_fbeta_score: 0.9614 - val_decoder_binary_accuracy: 0.0096\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3888 - capsnet_loss: 0.6957 - decoder_loss: 0.6932 - capsnet_precision: 0.1333 - capsnet_recall: 0.0116 - capsnet_fbeta_score: 0.0417 - capsnet_binary_accuracy: 0.5402 - decoder_precision: 0.9653 - decoder_recall: 0.9692 - decoder_fbeta_score: 0.9660 - decoder_binary_accuracy: 0.0113Epoch 00008: val_loss improved from 1.38721 to 1.38614, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 34s 322ms/step - loss: 1.3890 - capsnet_loss: 0.6959 - decoder_loss: 0.6932 - capsnet_precision: 0.1321 - capsnet_recall: 0.0115 - capsnet_fbeta_score: 0.0413 - capsnet_binary_accuracy: 0.5395 - decoder_precision: 0.9655 - decoder_recall: 0.9692 - decoder_fbeta_score: 0.9661 - decoder_binary_accuracy: 0.0112 - val_loss: 1.3861 - val_capsnet_loss: 0.6930 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.3345 - val_capsnet_recall: 0.0320 - val_capsnet_fbeta_score: 0.1099 - val_capsnet_binary_accuracy: 0.5473 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9690 - val_decoder_fbeta_score: 0.9614 - val_decoder_binary_accuracy: 0.0096\n",
      "Epoch 9/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3878 - capsnet_loss: 0.6946 - decoder_loss: 0.6932 - capsnet_precision: 0.3270 - capsnet_recall: 0.0273 - capsnet_fbeta_score: 0.0996 - capsnet_binary_accuracy: 0.5488 - decoder_precision: 0.9647 - decoder_recall: 0.9691 - decoder_fbeta_score: 0.9655 - decoder_binary_accuracy: 0.0117Epoch 00009: val_loss improved from 1.38614 to 1.38522, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 34s 323ms/step - loss: 1.3880 - capsnet_loss: 0.6949 - decoder_loss: 0.6932 - capsnet_precision: 0.3334 - capsnet_recall: 0.0275 - capsnet_fbeta_score: 0.1009 - capsnet_binary_accuracy: 0.5481 - decoder_precision: 0.9646 - decoder_recall: 0.9691 - decoder_fbeta_score: 0.9654 - decoder_binary_accuracy: 0.0117 - val_loss: 1.3852 - val_capsnet_loss: 0.6921 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.2182 - val_capsnet_recall: 0.0222 - val_capsnet_fbeta_score: 0.0734 - val_capsnet_binary_accuracy: 0.5436 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9690 - val_decoder_fbeta_score: 0.9613 - val_decoder_binary_accuracy: 0.0095\n",
      "Epoch 10/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3882 - capsnet_loss: 0.6950 - decoder_loss: 0.6932 - capsnet_precision: 0.4125 - capsnet_recall: 0.0374 - capsnet_fbeta_score: 0.1319 - capsnet_binary_accuracy: 0.5440 - decoder_precision: 0.9623 - decoder_recall: 0.9690 - decoder_fbeta_score: 0.9636 - decoder_binary_accuracy: 0.0120Epoch 00010: val_loss improved from 1.38522 to 1.38431, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 34s 322ms/step - loss: 1.3881 - capsnet_loss: 0.6949 - decoder_loss: 0.6932 - capsnet_precision: 0.4087 - capsnet_recall: 0.0371 - capsnet_fbeta_score: 0.1307 - capsnet_binary_accuracy: 0.5439 - decoder_precision: 0.9626 - decoder_recall: 0.9690 - decoder_fbeta_score: 0.9638 - decoder_binary_accuracy: 0.0120 - val_loss: 1.3843 - val_capsnet_loss: 0.6912 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.3345 - val_capsnet_recall: 0.0320 - val_capsnet_fbeta_score: 0.1099 - val_capsnet_binary_accuracy: 0.5473 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9690 - val_decoder_fbeta_score: 0.9613 - val_decoder_binary_accuracy: 0.0095\n",
      "Epoch 11/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3872 - capsnet_loss: 0.6940 - decoder_loss: 0.6932 - capsnet_precision: 0.4667 - capsnet_recall: 0.0518 - capsnet_fbeta_score: 0.1672 - capsnet_binary_accuracy: 0.5467 - decoder_precision: 0.9654 - decoder_recall: 0.9690 - decoder_fbeta_score: 0.9661 - decoder_binary_accuracy: 0.0118Epoch 00011: val_loss improved from 1.38431 to 1.38356, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 34s 323ms/step - loss: 1.3871 - capsnet_loss: 0.6940 - decoder_loss: 0.6932 - capsnet_precision: 0.4670 - capsnet_recall: 0.0522 - capsnet_fbeta_score: 0.1681 - capsnet_binary_accuracy: 0.5478 - decoder_precision: 0.9654 - decoder_recall: 0.9690 - decoder_fbeta_score: 0.9660 - decoder_binary_accuracy: 0.0118 - val_loss: 1.3836 - val_capsnet_loss: 0.6904 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.3927 - val_capsnet_recall: 0.0456 - val_capsnet_fbeta_score: 0.1469 - val_capsnet_binary_accuracy: 0.5518 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9690 - val_decoder_fbeta_score: 0.9613 - val_decoder_binary_accuracy: 0.0095\n",
      "Epoch 12/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3844 - capsnet_loss: 0.6912 - decoder_loss: 0.6932 - capsnet_precision: 0.3294 - capsnet_recall: 0.0373 - capsnet_fbeta_score: 0.1227 - capsnet_binary_accuracy: 0.5497 - decoder_precision: 0.9655 - decoder_recall: 0.9689 - decoder_fbeta_score: 0.9661 - decoder_binary_accuracy: 0.0115Epoch 00012: val_loss improved from 1.38356 to 1.38281, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 34s 324ms/step - loss: 1.3843 - capsnet_loss: 0.6912 - decoder_loss: 0.6932 - capsnet_precision: 0.3282 - capsnet_recall: 0.0377 - capsnet_fbeta_score: 0.1230 - capsnet_binary_accuracy: 0.5495 - decoder_precision: 0.9653 - decoder_recall: 0.9689 - decoder_fbeta_score: 0.9660 - decoder_binary_accuracy: 0.0115 - val_loss: 1.3828 - val_capsnet_loss: 0.6897 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.4267 - val_capsnet_recall: 0.0530 - val_capsnet_fbeta_score: 0.1650 - val_capsnet_binary_accuracy: 0.5555 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9689 - val_decoder_fbeta_score: 0.9613 - val_decoder_binary_accuracy: 0.0095\n",
      "Epoch 13/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3837 - capsnet_loss: 0.6906 - decoder_loss: 0.6932 - capsnet_precision: 0.5105 - capsnet_recall: 0.0677 - capsnet_fbeta_score: 0.2101 - capsnet_binary_accuracy: 0.5580 - decoder_precision: 0.9653 - decoder_recall: 0.9690 - decoder_fbeta_score: 0.9659 - decoder_binary_accuracy: 0.0116Epoch 00013: val_loss improved from 1.38281 to 1.38236, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 34s 324ms/step - loss: 1.3836 - capsnet_loss: 0.6904 - decoder_loss: 0.6932 - capsnet_precision: 0.5151 - capsnet_recall: 0.0685 - capsnet_fbeta_score: 0.2126 - capsnet_binary_accuracy: 0.5589 - decoder_precision: 0.9652 - decoder_recall: 0.9690 - decoder_fbeta_score: 0.9659 - decoder_binary_accuracy: 0.0116 - val_loss: 1.3824 - val_capsnet_loss: 0.6892 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.3345 - val_capsnet_recall: 0.0320 - val_capsnet_fbeta_score: 0.1099 - val_capsnet_binary_accuracy: 0.5473 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9689 - val_decoder_fbeta_score: 0.9613 - val_decoder_binary_accuracy: 0.0095\n",
      "Epoch 14/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3851 - capsnet_loss: 0.6920 - decoder_loss: 0.6932 - capsnet_precision: 0.5337 - capsnet_recall: 0.0678 - capsnet_fbeta_score: 0.2066 - capsnet_binary_accuracy: 0.5515 - decoder_precision: 0.9652 - decoder_recall: 0.9689 - decoder_fbeta_score: 0.9658 - decoder_binary_accuracy: 0.0115Epoch 00014: val_loss improved from 1.38236 to 1.38169, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 34s 324ms/step - loss: 1.3853 - capsnet_loss: 0.6921 - decoder_loss: 0.6932 - capsnet_precision: 0.5334 - capsnet_recall: 0.0678 - capsnet_fbeta_score: 0.2067 - capsnet_binary_accuracy: 0.5513 - decoder_precision: 0.9651 - decoder_recall: 0.9689 - decoder_fbeta_score: 0.9658 - decoder_binary_accuracy: 0.0115 - val_loss: 1.3817 - val_capsnet_loss: 0.6885 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.4461 - val_capsnet_recall: 0.0572 - val_capsnet_fbeta_score: 0.1749 - val_capsnet_binary_accuracy: 0.5564 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9689 - val_decoder_fbeta_score: 0.9613 - val_decoder_binary_accuracy: 0.0095\n",
      "Epoch 15/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3843 - capsnet_loss: 0.6911 - decoder_loss: 0.6932 - capsnet_precision: 0.5532 - capsnet_recall: 0.0791 - capsnet_fbeta_score: 0.2330 - capsnet_binary_accuracy: 0.5568 - decoder_precision: 0.9659 - decoder_recall: 0.9689 - decoder_fbeta_score: 0.9664 - decoder_binary_accuracy: 0.0118Epoch 00015: val_loss improved from 1.38169 to 1.38129, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 34s 324ms/step - loss: 1.3841 - capsnet_loss: 0.6909 - decoder_loss: 0.6932 - capsnet_precision: 0.5542 - capsnet_recall: 0.0800 - capsnet_fbeta_score: 0.2349 - capsnet_binary_accuracy: 0.5581 - decoder_precision: 0.9658 - decoder_recall: 0.9689 - decoder_fbeta_score: 0.9664 - decoder_binary_accuracy: 0.0118 - val_loss: 1.3813 - val_capsnet_loss: 0.6881 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.4558 - val_capsnet_recall: 0.0547 - val_capsnet_fbeta_score: 0.1719 - val_capsnet_binary_accuracy: 0.5564 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9689 - val_decoder_fbeta_score: 0.9613 - val_decoder_binary_accuracy: 0.0095\n",
      "Epoch 16/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3834 - capsnet_loss: 0.6903 - decoder_loss: 0.6932 - capsnet_precision: 0.5132 - capsnet_recall: 0.0835 - capsnet_fbeta_score: 0.2343 - capsnet_binary_accuracy: 0.5515 - decoder_precision: 0.9649 - decoder_recall: 0.9689 - decoder_fbeta_score: 0.9656 - decoder_binary_accuracy: 0.0113Epoch 00016: val_loss improved from 1.38129 to 1.38085, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 34s 324ms/step - loss: 1.3836 - capsnet_loss: 0.6905 - decoder_loss: 0.6932 - capsnet_precision: 0.5178 - capsnet_recall: 0.0831 - capsnet_fbeta_score: 0.2340 - capsnet_binary_accuracy: 0.5498 - decoder_precision: 0.9650 - decoder_recall: 0.9689 - decoder_fbeta_score: 0.9657 - decoder_binary_accuracy: 0.0114 - val_loss: 1.3809 - val_capsnet_loss: 0.6877 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.4461 - val_capsnet_recall: 0.0572 - val_capsnet_fbeta_score: 0.1749 - val_capsnet_binary_accuracy: 0.5564 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9689 - val_decoder_fbeta_score: 0.9613 - val_decoder_binary_accuracy: 0.0095\n",
      "Epoch 17/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3818 - capsnet_loss: 0.6887 - decoder_loss: 0.6932 - capsnet_precision: 0.5757 - capsnet_recall: 0.0837 - capsnet_fbeta_score: 0.2434 - capsnet_binary_accuracy: 0.5625 - decoder_precision: 0.9639 - decoder_recall: 0.9690 - decoder_fbeta_score: 0.9648 - decoder_binary_accuracy: 0.0119Epoch 00017: val_loss improved from 1.38085 to 1.38048, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 34s 324ms/step - loss: 1.3820 - capsnet_loss: 0.6888 - decoder_loss: 0.6932 - capsnet_precision: 0.5765 - capsnet_recall: 0.0839 - capsnet_fbeta_score: 0.2442 - capsnet_binary_accuracy: 0.5616 - decoder_precision: 0.9640 - decoder_recall: 0.9690 - decoder_fbeta_score: 0.9649 - decoder_binary_accuracy: 0.0119 - val_loss: 1.3805 - val_capsnet_loss: 0.6873 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.4461 - val_capsnet_recall: 0.0572 - val_capsnet_fbeta_score: 0.1749 - val_capsnet_binary_accuracy: 0.5564 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9689 - val_decoder_fbeta_score: 0.9613 - val_decoder_binary_accuracy: 0.0095\n",
      "Epoch 18/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3840 - capsnet_loss: 0.6908 - decoder_loss: 0.6932 - capsnet_precision: 0.5576 - capsnet_recall: 0.0947 - capsnet_fbeta_score: 0.2609 - capsnet_binary_accuracy: 0.5521 - decoder_precision: 0.9664 - decoder_recall: 0.9689 - decoder_fbeta_score: 0.9668 - decoder_binary_accuracy: 0.0111Epoch 00018: val_loss improved from 1.38048 to 1.38006, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 34s 324ms/step - loss: 1.3840 - capsnet_loss: 0.6908 - decoder_loss: 0.6932 - capsnet_precision: 0.5523 - capsnet_recall: 0.0938 - capsnet_fbeta_score: 0.2584 - capsnet_binary_accuracy: 0.5513 - decoder_precision: 0.9665 - decoder_recall: 0.9689 - decoder_fbeta_score: 0.9669 - decoder_binary_accuracy: 0.0111 - val_loss: 1.3801 - val_capsnet_loss: 0.6869 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5333 - val_capsnet_recall: 0.0769 - val_capsnet_fbeta_score: 0.2275 - val_capsnet_binary_accuracy: 0.5582 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9689 - val_decoder_fbeta_score: 0.9613 - val_decoder_binary_accuracy: 0.0095\n",
      "Epoch 19/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3828 - capsnet_loss: 0.6896 - decoder_loss: 0.6932 - capsnet_precision: 0.6126 - capsnet_recall: 0.1160 - capsnet_fbeta_score: 0.3079 - capsnet_binary_accuracy: 0.5646 - decoder_precision: 0.9643 - decoder_recall: 0.9688 - decoder_fbeta_score: 0.9651 - decoder_binary_accuracy: 0.0114Epoch 00019: val_loss improved from 1.38006 to 1.37979, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 34s 323ms/step - loss: 1.3825 - capsnet_loss: 0.6893 - decoder_loss: 0.6932 - capsnet_precision: 0.6163 - capsnet_recall: 0.1160 - capsnet_fbeta_score: 0.3086 - capsnet_binary_accuracy: 0.5663 - decoder_precision: 0.9642 - decoder_recall: 0.9688 - decoder_fbeta_score: 0.9650 - decoder_binary_accuracy: 0.0114 - val_loss: 1.3798 - val_capsnet_loss: 0.6866 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5042 - val_capsnet_recall: 0.0671 - val_capsnet_fbeta_score: 0.2035 - val_capsnet_binary_accuracy: 0.5564 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9689 - val_decoder_fbeta_score: 0.9613 - val_decoder_binary_accuracy: 0.0095\n",
      "Epoch 20/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3803 - capsnet_loss: 0.6872 - decoder_loss: 0.6932 - capsnet_precision: 0.5759 - capsnet_recall: 0.1119 - capsnet_fbeta_score: 0.2913 - capsnet_binary_accuracy: 0.5640 - decoder_precision: 0.9651 - decoder_recall: 0.9689 - decoder_fbeta_score: 0.9658 - decoder_binary_accuracy: 0.0115Epoch 00020: val_loss improved from 1.37979 to 1.37952, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 34s 323ms/step - loss: 1.3805 - capsnet_loss: 0.6874 - decoder_loss: 0.6932 - capsnet_precision: 0.5704 - capsnet_recall: 0.1108 - capsnet_fbeta_score: 0.2885 - capsnet_binary_accuracy: 0.5628 - decoder_precision: 0.9652 - decoder_recall: 0.9689 - decoder_fbeta_score: 0.9659 - decoder_binary_accuracy: 0.0115 - val_loss: 1.3795 - val_capsnet_loss: 0.6864 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.4558 - val_capsnet_recall: 0.0626 - val_capsnet_fbeta_score: 0.1880 - val_capsnet_binary_accuracy: 0.5564 - val_decoder_precision: 0.9596 - val_decoder_recall: 0.9689 - val_decoder_fbeta_score: 0.9613 - val_decoder_binary_accuracy: 0.0095\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARwAAAG0CAYAAADgu5hjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3Xd8FHX6wPHPkwIJVZp0BJRDCCSEJIAgXUQsNMWoiAYPPFSKep7yEwsq3ClyFjwVUCliAUEBhVNPDjjgqMELVaQrUUoIEBIgkux+f3/MZNnEbHZDspsEnvfrtWRn5jvfeXZInp36jBhjUEqpQAgq6QCUUpcPTThKqYDRhKOUChhNOEqpgNGEo5QKGE04SqmA0YSjvBKRYBHJEJFGxdn2IuKYICKzirtfFTghJR2AKn4ikuE2WAH4DXDYw38yxnxcmP6MMQ6gUnG3VZcfTTiXIGOM6w9eRA4Cw4wxyzy1F5EQY0x2IGJTlzfdpboM2bsm80TkUxFJB+4VketEZL2InBKRwyIyRURC7fYhImJEpLE9/JE9/WsRSReRdSLSpLBt7el9RGS3iKSJyFsi8l8RSfDxcwwQkR12zMtFpLnbtKdF5FcROS0iu0Skmz2+g4h8b48/KiKvFsMqVT7ShHP5GgB8AlQF5gHZwBigJtAJuAn4UwHz3wM8C1QHfgZeKmxbEbkS+Az4i73cA0A7X4IXkRbAHGAUUAtYBnwpIqEiEmHH3tYYUwXoYy8X4C3gVXv8NcACX5aniocmnMvXGmPMV8YYpzHmnDFmkzFmgzEm2xizH5gOdC1g/gXGmERjTBbwMdDmItreCiQZYxbb014HjvsY/13Al8aY5fa8L2Mlz/ZYyTMMiLB3Fw/YnwkgC2gmIjWMMenGmA0+Lk8VA004l69D7gMicq2ILBWRIyJyGngRa6vDkyNu789S8IFiT23rucdhrDuJk32IPWfen9zmddrz1jfG/Aj8GeszHLN3HevYTYcCLYEfRWSjiNzs4/JUMdCEc/nKWyZgGrAduMbe3XgOED/HcBhokDMgIgLU93HeX4Gr3OYNsvv6BcAY85ExphPQBAgG/maP/9EYcxdwJfB34HMRCSv6R1G+0ISjclQG0oAz9vGRgo7fFJclQFsRuU1EQrCOIdXycd7PgL4i0s0+uP0XIB3YICItRKS7iJQHztkvJ4CIDBGRmvYWURpW4nUW78dSnmjCUTn+DNyP9Uc7DetAsl8ZY44C8cBrQCpwNfA/rOuGvM27Ayved4EUrIPcfe3jOeWBSVjHg44A1YBx9qw3Az/YZ+cmA/HGmPPF+LFUAUQLcKnSQkSCsXaV7jDGrC7peFTx0y0cVaJE5CYRucLe/XkW6yzSxhIOS/mJJhxV0q4H9mPtFvUGBhhjvO5SqbJJd6mUUgGjWzhKqYDRhKOUCphSebd4zZo1TePGjUs6DKWUjzZv3nzcGOP1GqpSmXAaN25MYmJiSYehlPKRiPzkvZXuUimlAkgTjlIqYDThKKUCRhOOUipgNOEopQJGE4679CMwZyDsW1HSkSh1SdKE4+78Gdj3b8g4WtKRKHVJ0oSTL38XulPq8qQJRykVMJpw3OXcOS+6haOUP2jCyZcmHKX8QRNOLlobSCl/0oSTH92lUsovNOG40+qHSvmVJpxcNOEo5U+acPKju1RK+YUmHHeuXSpNOEr5gyacXHSXSil/0oSTH92lUsovNOG4010qpfxKE04uukullD9pwsmP7lIp5ReacNzpLpVSfqUJRykVMJpwctHyFEr5kyYcd7pLpZRfacJRSgVMkRKOiMwQkWMist1LuzgRyRaRO4qyPP/TXSql/KmoWzizgJsKaiAiwcArwL+KuCz/010qpfyqSAnHGLMKOOGl2Sjgc+BYUZallCr7QvzZuYjUBwYA3YG4Yu08KxO+eQqObAMEJMjaFZIg64W4DecdH+T2srdmjBPOnsgJvFhDVUpZ/JpwgDeAp4wxTvHyRywiDwIPAjRq1Mh7z4e3wOZZRY8wP5/eVfD0K1tC7AMQFAwSDEEh9iv4wk8JttoaBzgd4My2dtmMwxofMQBCyvsnfqVKKTFFLKspIo2BJcaYVvlMO8CFAyI1gbPAg8aYRQX1GRsbaxITE70vPHUfnDtpbZ0YY/3E/unMBkc2OM6DMwscWdb7nJ9Oe5rjPBzfC1vnFu6DF4fYP9pbU3LhJ0D2Ofj+Q/8tt1YLuP8rqFTLf8tQlxUR2WyMifXWzq9bOMaYJm4BzcJKTAUmm0KpcXWxdcXAabmHX6xpJSq/Edi5yD5QbdwOWBvITPPjcoGUH2DyNRBeHULDrVdQqFtoOd8R4mEYe1dW+SRuOISGceGLBe/vndlw5viFL1L3F8baanZ9webTJtfLeHjvoU1O/23ugesfK9ZDDEVKOCLyKdANqCkiycDzQCiAMWZqkaMrSc8dL9nlO7KthOfaHXPav2Tuu2gOcDovjDOOC1t22ees41xZZyHrnD18Ds6ftcY5s+zp5+zhbHLdLZ93y9c9IYImnMLY9B6EhONadzlfMgW+dwY2xvz8+wXoOAqCQ7239VGREo4x5u5CtE0oyrIuO8Eh1kvlzxhybx16+WmcuRN0zrd5zpaCcVxol2tczrDJZ5wz97zuXwg500IrwB96F34rITMNtn9hndgICs5zoiPPiZC8J0SCgsl90sTTK6dNcP5tK11ZrMkG/H/QWCn/ECn8H3FZElYVYoeWdBTFTm9tUEoFjCYcpVTAaMJRSgWMJhylVMBowlFKBYwmHKVUwGjCUUoFjCYcpVTAaMJRSgWMJhylVMBowlFKBYwmHKVUwGjCUUoFjCYcpVTAaMJRSgWMJhylVMBowlFKBYwmHKVUwGjCUUoFjCYcpVTAaMJRSgWMJhylVMBowlFKBYwmHKVUwOiD8FSplpWVRXJyMpmZmSUdigLCwsJo0KABoaEX90ROTTiqVEtOTqZy5co0btwYuZSftFkGGGNITU0lOTmZJk2aXFQfukulSrXMzExq1KihyaYUEBFq1KhRpK1NTTiq1NNkU3oU9f+iSAlHRGaIyDER2e5h+mAR2Soi20RkrYhEFWV5SqmyrahbOLOAmwqYfgDoaoxpDbwETC/i8pRSZViREo4xZhVwooDpa40xJ+3B9UCDoixPqcvdypUrWbt2bYFtxo8fz+TJkwMUUeEE8hjOH4GvA7g8pS45viSc0iwgp8VFpDtWwrm+gDYPAg8CNGrUKBBhqTLmha92sPPX08XaZ8t6VXj+tgiv7T788EMmT56MiBAZGcmdd97JhAkTOH/+PDVq1ODjjz+mdu3ajB8/nn379rF3716OHz/Ok08+yfDhwzl8+DDx8fGcPn2a7Oxs3n33XTp37kylSpUYM2YMS5YsITw8nMWLF1O7dm1SUlIYMWIEP//8MwBvvPEG9evXZ+rUqQQHB/PRRx/x1ltv0blz5wLjTkpKYsSIEZw9e5arr76aGTNmUK1aNaZMmcLUqVMJCQmhZcuWzJ07l//85z+MGTMGsA4Or1q1isqVKxd9Jbvxe8IRkUjgfaCPMSbVUztjzHTsYzyxsbHG33Ep5asdO3YwYcIE1q5dS82aNTlx4gQiwvr16xER3n//fSZNmsTf//53ALZu3cr69es5c+YM0dHR3HLLLXz66af07t2bcePG4XA4OHv2LABnzpyhQ4cOTJw4kSeffJL33nuPZ555hjFjxvDYY49x/fXX8/PPP9O7d29++OEHRowYQaVKlXjiiSd8iv2+++7jrbfeomvXrjz33HO88MILvPHGG7z88sscOHCA8uXLc+rUKQAmT57M22+/TadOncjIyCAsLKzY16VfE46INAK+AIYYY3b7c1nq0ufLlog/LF++nEGDBlGzZk0AqlevzrZt24iPj+fw4cOcP38+14Vw/fr1Izw8nPDwcLp3787GjRuJi4vjgQceICsri/79+9OmTRsAypUrx6233gpATEwM3333HQDLli1j586drj5Pnz5NRkZGoeJOS0vj1KlTdO3aFYD777+fQYMGARAZGcngwYPp378//fv3B6BTp048/vjjDB48mIEDB9KgQfEfci3qafFPgXVAcxFJFpE/isgIERlhN3kOqAG8IyJJIpJYxHiVKhVGjRrFyJEj2bZtG9OmTct1MVzea1VEhC5durBq1Srq169PQkICH374IQChoaGu9sHBwWRnZwPgdDpZv349SUlJJCUl8csvv1CpUqVii3/p0qU88sgjfP/998TFxZGdnc3YsWN5//33OXfuHJ06dWLXrl3FtrwcRT1Ldbcxpq4xJtQY08AY84ExZqoxZqo9fZgxppoxpo39ii2esJUKnB49ejB//nxSU60jAidOnCAtLY369esDMHv27FztFy9eTGZmJqmpqaxcuZK4uDh++uknateuzfDhwxk2bBjff/99gcu88cYbeeutt1zDSUlJAFSuXJn09HSf4q5atSrVqlVj9erVAMyZM4euXbvidDo5dOgQ3bt355VXXiEtLY2MjAz27dtH69ateeqpp4iLi/NLwtF7qZTyIiIignHjxtG1a1eCg4OJjo5m/PjxDBo0iGrVqtGjRw8OHDjgah8ZGUn37t05fvw4zz77LPXq1WP27Nm8+uqrhIaGUqlSJdcWjidTpkzhkUceITIykuzsbLp06cLUqVO57bbbuOOOO1i8eLFPB41nz57tOmjctGlTZs6cicPh4N577yUtLQ1jDKNHj+aKK67g2WefZcWKFQQFBREREUGfPn2KZf25E2NK3/HZ2NhYk5ioe18KfvjhB1q0aFHSYfhs/PjxhTqoWxbl938iIpt92YPRe6mUUgGju1RKFaPx48cHbFkTJ05k/vz5ucYNGjSIcePGBSyGwtKEo1QZNW7cuFKdXPKju1RKqYDRhKOUChhNOEqpgNGEo5QXxXmF7+VOE45SKmA04SjlI2MMf/nLX2jVqhWtW7dm3rx5ABw+fJguXbrQpk0bWrVqxerVq3E4HCQkJLjavv766yUcfemgp8VV2fH1WDiyrXj7rNMa+rzsU9MvvviCpKQktmzZwvHjx4mLi6NLly588sknvys9kXPD5fbtVrnvnBIQlzvdwlHKR2vWrOHuu+8mODiY2rVr07VrVzZt2kRcXBwzZ85k/PjxbNu2jcqVK9O0aVP279/PqFGj+Oabb6hSpUpJh18q6BaOKjt83BIJtJzSE0uXLiUhIYHHH3+c++67jy1btvDtt98ydepUPvvsM2bMmFHSoZY43cJRykedO3dm3rx5OBwOUlJSWLVqFe3atcu39MTx48dxOp3cfvvtTJgwwWs5isuFbuEo5aMBAwawbt06oqKiEBEmTZpEnTp18i098csvvzB06FCcTicAf/vb30o4+tJBy1OoUq2slae4HGh5CqVUmaAJRykVMJpwlFIBowlHKRUwmnCUUgGjCUcpFTCacJQqpPHjxzN58uRS3fesWbMYOXJkMURUvDThKFWG5Typs6zQhKOUDyZOnMgf/vAHrr/+en788UcA9u3bx0033URMTAydO3d2Pany6NGjDBgwgKioKKKioli7di0Ar732Gq1ataJVq1a88cYbBfZdUP8JCQmMGDGC9u3b8+STT3qN/eDBg/To0YPIyEh69uzJzz//DMD8+fNp1aoVUVFRdOnSBYAdO3bQrl072rRpQ2RkJHv27CmGtXeB3tqgyoxXNr7CrhPF+/jZa6tfy1PtniqwzebNm5k7dy5JSUlkZ2fTtm1bYmJiePDBB5k6dSrNmjVjw4YNPPzwwyxfvpzRo0fTtWtXFi5ciMPhICMjg82bNzNz5kw2bNiAMYb27du7HrubX9+Ax/4BkpOTWbt2LcHBwV4/46hRo7j//vu5//77mTFjBqNHj2bRokW8+OKLfPvtt9SvX99VPmPq1KmMGTOGwYMHc/78eRwORxHXcG6acJTyYvXq1QwYMIAKFSoA0LdvXzIzM1m7di2DBg1ytfvtt98AWL58uetRvsHBwVStWpU1a9YwYMAAKlasCMDAgQNZvXo1Tqfzd30DZGRkeOwfrOdP+ZJsANatW8cXX3wBwJAhQ1xbRZ06dSIhIYE777yTgQMHAnDdddcxceJEkpOTGThwIM2aNSvk2iqYJhxVZnjbEgkkp9PJFVdcQVJSUon0n5O4imLq1Kls2LCBpUuXEhMTw+bNm7nnnnto3749S5cu5eabb2batGn06NGjyMvKUaRjOCIyQ0SOich2D9NFRKaIyF4R2SoibYuyPKVKQpcuXVi0aBHnzp0jPT2dr776igoVKtCkSRPXky+NMWzZsgWAnj178u677wLgcDhIS0ujc+fOLFq0iLNnz3LmzBkWLlxI586d8+0boEqVKh77L6yOHTsyd+5cAD7++GM6d+4MWMeI2rdvz4svvkitWrU4dOgQ+/fvp2nTpowePZp+/fqxdevWi19x+SjqQeNZwE0FTO8DNLNfDwLvFnF5SgVc27ZtiY+PJyoqij59+hAXFwdYf7wffPABUVFRREREsHjxYgDefPNNVqxYQevWrYmJiWHnzp20bduWhIQE2rVrR/v27Rk2bBjR0dEe+y6o/8J66623mDlzJpGRkcyZM4c333wTgL/85S+0bt2aVq1a0bFjR6Kiovjss89o1aoVbdq0Yfv27dx3331FXHu5Fbk8hYg0BpYYY1rlM20asNIY86k9/CPQzRhzuKA+tTyFyqHlKUqf0lyeoj5wyG042R6nlLoMlZqDxiLyINZuF40aNSrhaJQqG2bOnOnaRcrRqVMn3n777RKKqGD+Tji/AA3dhhvY437HGDMdmA7WLpWf41LqkjB06FCGDh1a0mH4zN+7VF8C99lnqzoAad6O3yiVV2ksg3u5Kur/RZG2cETkU6AbUFNEkoHngVA7sKnAP4Gbgb3AWaDspGJVKoSFhZGamkqNGjUQkZIO57JmjCE1NZWwsLCL7qNICccYc7eX6QZ4pCjLUJe3Bg0akJycTEpKSkmHorC+ABo0aHDR85eag8ZK5Sc0NJQmTZqUdBiqmOjd4kqpgNGEo5QKGE04SqmA0YSjlAoYTThKqYDRhKOUChhNOEqpgNGEo5QKGE04SqmA0YSjlAoYTThKqYDRhKOUChhNOEqpgNGEo5QKGE04SqmA0YSjlAoYTThKqYDRhKOUChhNOEqpgNGEo5QKGE04SqmA0YSjlAoYTThKqYDRhKOUChhNOG6OpWfy0EebWb8/taRDUeqSpAnHzdnfHHy9/Qi/njpX0qEodUnShJMPkZKOQKlLkyYcN6akA1DqElekhCMiN4nIjyKyV0TG5jO9kYisEJH/ichWEbm5KMsLFEE3cZTyh4tOOCISDLwN9AFaAneLSMs8zZ4BPjPGRAN3Ae9c7PICwRhrG0d3qZTyj6Js4bQD9hpj9htjzgNzgX552higiv2+KvBrEZbnd7pLpZR/hRRh3vrAIbfhZKB9njbjgX+JyCigInBDEZanlCrj/H3Q+G5gljGmAXAzMEdE8l2miDwoIokikpiSkuLnsPJn71Ehuk+llF8UJeH8AjR0G25gj3P3R+AzAGPMOiAMqJlfZ8aY6caYWGNMbK1atYoQVlHoTpVS/lSUhLMJaCYiTUSkHNZB4S/ztPkZ6AkgIi2wEk7JbL4Ugm7fKOUfF51wjDHZwEjgW+AHrLNRO0TkRRHpazf7MzBcRLYAnwIJJudUUCl0YZeqZONQ6lJVlIPGGGP+Cfwzz7jn3N7vBDoVZRmBVGozoVKXCL3SOB964Z9S/qEJx43uUinlX5pw3BjdqVLKrzTh5EM3cJTyD004bnSXSin/0oSjlAoYTThuLlwhpJs4SvmDJhw3etBYKf/ShJMPPYajlH9ownHjOmhcsmEodcnShKOUChhNOPnQejhK+UeRbt4sSVkOJ69/t5vdRzMQgSCx7oEKCrIShgBBItZ4EbuN23g71TqchmynweE0nDhzHtBdKqX8pcwmnP/9fIp3Vu7zS9/DPkwscPp1TWtwe0wDQoKE4CBx/Qx2DQdZP4OFIBGMsRKawxiMsY4VdWhanZBg3cBUl5cym3DaNanO2wn1OXTquH0y2+A01pMXjAGnyRlrcDjAYQzZTidZDvuP32ENZzsNySfPsfnX3QSFnLHmcVSAoN8IKpf/I383pTZk07fBRYrfca4Rg9s1BrG2qEQubFllOQzzkr5H7HiKQ7ngIMqFWMmwduUqvNa/D1XCyl10f7tP7mb1L6s5ff40e0/tLbY4A6lBpQZcWeHKQs0TUSOCcsG+r7ew4DBa1sj7MBPvsp3ZpJwr2Vp1DSoXfv14I6WxHlZsbKxJTCx4KyPpWBJDvh4SoIiUujxtvnezTwlWRDYbY2K9tSuzWzgta7Qkvnk8u0/uLnJfDqeDrce3FkNUvns46mHqVKyT77Rsk83riW+QnnXab8tvVf4+jLM8WQ4n5x3OC1uEeb5/Lgwbt3/hl5DZfovtUpN59BaMI6xQ80jQb4RU2eaniHyTfboNmKJtyedVZrdw1KXHGLfdYqxkZzCupOc+bHBr57R2oZ3mwnEyh9Madn/vNAaHM880e7zTaS3b/b23acatz5xpjjztK5YLYWDb+taxPLfPeeH9hSvc3f8U085lMW/TIYIEgoLEOiGSc+JDLpwQCQoSJOfkCG7juHByxJrnwvyuEyl2m2Br5gt92stoUrMidauG+/R/d8lv4ahLj4gQLHDpnyf0/vnCQoMZ3bNZAGIJLD1NopQKGE04SqmA0YSjlAoYTThKqYDRhKOUChhNOEqpgCmV1+GISArwU4AXWxM4HuBlFqQ0xVOaYoHSFU9pigVKLp6rjDG1vDUqlQmnJIhIoi8XLgVKaYqnNMUCpSue0hQLlL548tJdKqVUwGjCUUoFjCacC6aXdAB5lKZ4SlMsULriKU2xQOmLJxc9hqOUChjdwlFKBcwlmXBEZIaIHBOR7R6mdxORNBFJsl/P5ZkeLCL/E5ElbuOaiMgGEdkrIvNExKeyb36KZZaIHHCbp40vsRQ1HhE5KCLb7PGJbuOri8h3IrLH/lmthOMZLyK/uM1zcwBiuUJEFojILhH5QUSuK+F14ymei1o3xcbYdT0upRfQBWgLbPcwvRuwpID5Hwc+cW8DfAbcZb+fCjxUgrHMAu4I9LoBDgI18xk/CRhrvx8LvFLC8YwHngjwupkNDLPflwOuKOF14ymei1o3xfW6JLdwjDGrgBMXM6+INABuAd53GydAD2CBPWo20L8kYimqosRTgH5Y6wQKsW78GM9FudhYRKQqVnL4wO7nvDHmlD054OvGSzwl6pJMOD66TkS2iMjXIhLhNv4N4EnA6TauBnDKGJNtDycD9UsolhwTRWSriLwuIuWLMZaC4jHAv0Rks4g86Da+tjHmsP3+CFC7hOMBGGmvnxmF2Y25yFiaACnATHv3930RqWhPK4l1U1A84L91411JbVr5+wU0xvOmaBWgkv3+ZmCP/f5W4J28m6tYl4vvdZu/oae+/R2LPVwXq2xceaxvzef8vW7s4fr2zyuBLUAXe/hUnj5OlnA8tYFgrC/UicAMP/9fxQLZQHt7+E3gpZJaN17iueh1Uxyvy3ILxxhz2hiTYb//JxAqIjWBTkBfETkIzAV6iMhHQCpwhYjklGRtAPxSQrFgjDlsLL8BM4F2xRGLl3gwxvxi/zwGLHRb7lERqQtg/zxWkvEYY44aYxzGGCfwHsW0fgqIJRlINsZssJsuwDr2AiWzbjzG469146vLMuGISB37uAwi0g5rPaQaY/7PGNPAGNMYuAtYboy511hfDSuAO+wu7gcWl0QsdrucX2DBOiaQ71mM4oxHRCqKSGV7fEXgRrflfom1TqAY183FxpOzfmwDKKb1U8D/1RHgkIg0t5v2BHba7wO+bgqKx1/rxmeB3JwK1Av4FDgMZGFl+z8CI4AR9vSRwA6szfD1QMd8+uhG7t2YpsBGYC8wHyhfgrEsB7Zh/bJ8hL1Z7c947M+/xX7tAMa59VkD+DewB1gGVC/heObY62cr1h98XX//XwFtgER7mYuAaiW1brzEc1HrprheeqWxUipgLstdKqVUydCEo5QKGE04SqmA0YRzEcS6vylDRBoVZ9uSJCLXiEixH9ATkRvsU/s5wz+KSGdf2l7Est4Xkacvdn7lf5fFo35FJMNtsALwG+Cwh/9kjPm4MP0ZYxxApeJuezkwxjT33so7ERkG3GuM6ebW97Di6Fv5z2WRcIwxrj94+xt0mDFmmaf2IhJiLtzGoFSJupR+H3WXChCRCWKVnPhURNKBe0XkOhFZLyKnROSwiEwRkVC7fYiIGBFpbA9/ZE//WkTSRWSdiDQpbFt7eh8R2S1W2YG3ROS/IpLgIW5fYvyTWCU1TorIFLd5g8W6DytVRPYDNxWwfsaJyNw8494Wkdfs98PEKoGQLiL77K0PT30li0g3+30FEZljx7YDiMnT9hkR2W/3u0NE+trjWwP/ADrbu6vH3dbteLf5R9ifPVVEFsmFCyYLXDeFWc858YjIMhE5ISJHRORJt+U8a6+T0yKSKCL18tt9FZE1Of/P9vpcZS/nBPCMiDQTkRX2Mo7b662q2/xX2Z8xxZ7+poiE2TG3cGtXV0TOikgNT5/XrwJ50U9peGGVNLghz7gJwHngNqwkHA7EAe2xtgKbAruBkXb7EKwbBxvbwx9hPZojFggF5gEfXUTbK4F0rDuMQ7FKU2QBCR4+iy8xLgaqYt2TcyLns3PhorEGWBenrbJ+HfJdTlMgA6jo1vcxINYevs1uk3NX/Tkg0p52A3DQra9koJv9fjKwEqgGXIV1Nax72zux7hsLAu6xY6htTxsGrMwT50fAePv9jXaMbYAw4B2sq7W9rptCrueqwFFgDNa9bVWAdva0/8O6KK+Z/RnaANWBa/Kua2BNzv+z/dmygYew7nsKB/6AdcVwOfv35L/AZLfPs91enxXt9p3sadOBiW7L+TOwsMT+/ko6AQT8A3tOOMu9zPcEMD/PL6x7Epnq1rYv9g13hWz7ALDabZpgXWma4ONnyy/GDm7Tv8CuhYKVYIa5Tbs57x9Bnr7XA/fY7/sAPxbQdgnwiP2+oITzs/v/BfCwe9t8+t0O3GK/95ZwZgN/dZsC3n3DAAAgAElEQVRWBeu4XQNv66aQ63kIsMlDu3058eYZ70vC2e8lhjtylgt0xroTPTifdp2AA1woJ5wEDCzuvytfX7pLdcEh9wERuVZEltqbyKeBF7HuGvfkiNv7sxR8oNhT23rucRjrNyTZUyc+xujTsvD+4MFPgLvt9/fYwzlx3CpWNcQTInIKa+uioHWVo25BMYhIglilF07Z/V7rY79gfT5Xf8aY08BJcpcV8en/zMt6boiVWPJT0DRv8v4+1hGRz8Sq1ncaqwibewwHjXWCIhdjzH+xtpauF5FWQCNg6UXGVGSacC7Ie0p4GtY36jXGmCrAc1hbHP50GOsbGHDdnFlQ3Z2ixHgY6xc1h7fT9p8BN4hIfaxdvk/sGMOx7kb+G9buzhXAv3yM44inGESkKfAu1m5FDbvfXW79ejuF/yvWblpOf5Wxdt0u5i7/gtbzIeBqD/N5mnbGjqmC27g6edrk/XyvYJ1dbW3HkJAnhqtEJNhDHB8C92JtjX1mrCoDJUITjmeVgTTgjH3Q7U8BWOYSoK2I3CZWKYwxQEGPTy1KjJ8Bj4pIffsA4lMFNTbWHchrsL5ZfzTG7LEnlcc6rpACOETkVqxjDb7G8LRY9XcbYR1XylEJ648uBSv3DsfawslxFGjgfvA2j0+BP4pIpFgFyv6GtbvqcYuxAAWt5y+BRiIyUkTKi0gVse7cBqtS4wQRuVosbUSkOlaiPYJ1ciJYrOJhV1GwyliJKk1EGmLt1uVYh1VC5a9iHYgPF5FObtPnYO2C3YOVfEqMJhzP/oxVTiAd6xtunr8XaIw5CsQDr2H9Al0N/A/rm624Y3wX6y7mbcAmLpRPLcgnWMdkXLtTxipd+RhWPZoTWL/YS/Kd+/eex9rSOgh8jdsfgzFmK/AW1h36h4HmwAa3eb/DugP7qIi47xrlzP8N1q7PQnv+RsBgH+PKy+N6NsakAb2A27GS4G6gqz35Vaw7tf8NnMY6gBtm7yoPB57GOoFwTZ7Plp/nsWrXpGEluc/dYsjGKtjWAmtr52culFLBGHMQ6//5N2PM2kJ+9mKld4uXYvYm8q9YBdNXl3Q8quwSkQ+xDkSPL8k4LosL/8oSEbkJ64zQOazTqllY3/JKXRT7eFg/oHVJx6K7VKXP9cB+rGMXvYEBJXmQT5VtIvI3rGuB/mqM+bnE49FdKqVUoOgWjlIqYDThKKUCplQeNK5Zs6Zp3LhxSYehlPLR5s2bjxtjCrpmDCilCadx48YkJiZ6b6iUKhVExNutMYDuUimlAkgTjlIqYDThKKUCplQew7kUZGVlkZycTGZmZkmHolSxCQsLo0GDBoSGerpntmCacPwkOTmZypUr07hxY6wqE0qVbcYYUlNTSU5OpkmTJt5nyIcmHD/JzMzUZKNccq7oN4D11mDMhaI3ecc5jSEzy2lVyrOHjatCJzhxe29yz5fTzmnAkGfYHueLCuVCuLpWRdfvsIhQo0YNUlJSLno9aMLxo8s12RhjyHYash1OshyGLIeTbKf1M8thj7enq9Lr7PnfPyiiqL/TmnCKyOE0nMtycPZ8Nmd/c9jvHQRlOUg7l4UxBofT+nZxGuP6BqoafmEf2PXtBq5vqpT03/L9Dy8uYaHBlAu2zhlYy837DYzr2zVXjHZ8WZosfHZFhXKUCwlCsEv0CQhCzt+ugP3eGieA08CZ37IRgSDJGS/2sPWHL7hNE7HG4/bebb7cwyX3ZVhmE07a2SzunLaOH4+ml3Qo+Xqvb12CU894nH48o2RvAM/Mssrf5voDsN9f+IXO2ZTG/imumpYnz573eVmb1q0hNDSUNrHtPbZ59rGH6XJDb3rd0i/X+B1b/sdXn89l7Iuv5BovCKHBQkhwEKHBQmhwECFBeYaDhWCRMrulWb1iuZIOodh5TTgiMgOrmtgxY0yrfKZ3w3rcxgF71BfGmBftaTcBb2I96uJ9Y8zLxRQ3h06eLbXJpiAiQrBA3arhhATbf9Du33b2N9DZ8w6ynYYg+xsuyP7WCgpye2+Pl6AL7wOlYfUK3hvZvtiRSKVKlYjs39tjm2oVy3FVjYpENrgi1/jIBt25+5buFx2nO4fDQXCwp7K/JSc7O5uQkDL73V84PjwSowvQFvtRJvlM7wYsyWd8MFbF+qZYNW+3AC19eZRETEyMKet27txZ0iGY2bNnm9atW5vIyEhz7733mi+//NK0a9fOtGnTxvTs2dMcOXLEGGPM888/b+69917ToUMHc80115jp06cbY4z59ddfTefOnU1UVJSJiIgwq1atMsYYU7FiRfP000+byMhI0759e1c/x44dMwMHDjSxsbEmNjbWrFmzxhw4cMDUrl3b1KtXz0RFRbn6yOv+++83f/rTn0xMTIxp1qyZ+eqrr4wxxqxYscLccsstrjiHDh1qunbtapo0aWLefPNN1/z9+vUzbdu2NS1btjTTpk1zja9YsaJ5/PHHTWRkpHnhhRdMv379XNP+9a9/mf79+3tcfyNGjDAxMTGmZcuW5rnnnnON37hxo7nuuutMZGSkiYuLM6dPnzbZ2dnmz3/+s4mIiDCtW7c2U6ZMMcYYc9VVV5mUlBRjjDGbNm0yXbt2zbXOO3bsaO666y5z4MABc/3115vo6GgTHR1t/vvf/7qW9/LLL5tWrVqZyMhI89RTT5m9e/ea6Oho1/Tdu3fnGva3/H63gUTjw9+217RqjFkl9lMjC6kdsNcYsx9ArCc39sN62Nll5YWvdrDz19PF2mfLelV4/rYIj9N37NjBhAkTWLt2LTVr1uTEiROICOvXr0dEeP/995k0aRJ///vfAdi6dSvr16/nzJkzREdHc8stt/Dpp5/Su3dvxo0bh8Ph4OzZswCcOXOGDh06MHHiRJ588knee+89nnnmGcaMGcNjjz3G9ddfz88//0zv3r354YcfGDFiBJUqVeKJJ57wGC/AwYMH2bhxI/v27aN79+7s3bv3d2127drFihUrSE9Pp3nz5jz00EOEhoYyY8YMqlevzrlz54iLi+P222+nRo0anDlzhvbt2/P3v/8dYwwtWrQgJSWFWrVqMXPmTB544AGP8UycOJHq1avjcDjo2bMnW7du5dprryU+Pp558+YRFxfH6dOnCQ8PZ/r06Rw8eJCkpCRCQkI4ceKE1//DnTt3smbNGsLDwzl79izfffcdYWFh7Nmzh7vvvpvExES+/vprFi9ezIYNG6hQoQInTpygevXqVK1alaSkJNq0acPMmTMZOnSo1+WVBsW1HXediGzBqr/7hDFmB9bjTdyfrZOM9fRCFQDLly9n0KBB1KxpPbqoevXqbNu2jfj4eA4fPsz58+dzXUvRr18/wsPDCQ8Pp3v37mzcuJG4uDgeeOABsrKy6N+/P23atAGgXLly3HrrrQDExMTw3XffAbBs2TJ27rzwfXL69GkyMjJ8jvnOO+8kKCiIZs2a0bRpU3bt2vW7Nrfccgvly5enfPnyXHnllRw9epQGDRowZcoUFi5cCMChQ4fYs2cPNWrUIDg4mNtvvx2wdmeHDBnCRx99xNChQ1m3bh0ffuj5IQafffYZ06dPJzs7m8OHD7Nz505EhLp16xIXFwdAlSpVXJ99xIgRrl2j6tWre/28ffv2JTw8HLAuFB05ciRJSUkEBweze/duV79Dhw6lQoUKufodNmwYM2fO5LXXXmPevHls3Fg2qtAWR8L5HrjKGJMhIjdjValvVthO7EdlPAjQqJG3RySVLQVtiQTSqFGjePzxx+nbty8rV65k/Pjxrml5D6yKCF26dGHVqlUsXbqUhIQEHn/8ce677z5CQ0Nd7YODg8nOts6mOZ1O1q9fT1hY2EXFl18MeZUvX971PmfZK1euZNmyZaxbt44KFSrQrVs31xXeYWFhuY7bDB06lNtuu42wsDAGDRrk8djJgQMHmDx5Mps2baJatWokJCRc1FXjISEhOJ3WGb2881esWNH1/vXXX6d27dps2bIFp9PpdR3efvvtvPDCC/To0YOYmBhq1CiZR4UXVpHvpTLGnDbGZNjv/wmEikhNrAeOuT/krAEFPITMGDPdGBNrjImtVctrWQ3lRY8ePZg/fz6pqakAnDhxgrS0NOrXt56rN3v27FztFy9eTGZmJqmpqaxcuZK4uDh++uknateuzfDhwxk2bBjff/99gcu88cYbeeutt1zDSUlJAFSuXJn0dO8H+OfPn4/T6WTfvn3s37+f5s2b+/RZ09LSqFatGhUqVGDXrl2sX7/eY9t69epRr149JkyYUOBuyOnTp6lYsSJVq1bl6NGjfP311wA0b96cw4cPs2nTJgDS09PJzs6mV69eTJs2zZV8c3apGjduzObNmwH4/PPP81nShc9Qt25dgoKCmDNnDg6HdRaxV69ezJw507U7m9NvWFgYvXv35qGHHiozu1NQDAnHfgSp2O/b2X2mYj3rqJmINBGRcsBdWM/TUQEQERHBuHHj6Nq1K1FRUTz++OOMHz+eQYMGERMT49rVyhEZGUn37t3p0KEDzz77LPXq1WPlypVERUURHR3NvHnzGDNmTIHLnDJlComJiURGRtKyZUumTp0KwG233cbChQtp06YNq1d7ftpNo0aNaNeuHX369GHq1Kk+bynddNNNZGdn06JFC8aOHUuHDh0KbD948GAaNmxIixYtPLbJ+dzXXnst99xzD506Wc+VK1euHPPmzWPUqFFERUXRq1cvMjMzGTZsGI0aNSIyMpKoqCg++cR6dNfzzz/PmDFjiI2NLfAM2cMPP8zs2bOJiopi165drq2fm266ib59+xIbG0ubNm2YPHlyrs8RFBTEjTfe6NN6Kg28FlEXkU+xzkTVxHrQ1/NAKIAxZqqIjMR6HGs21qNNHjf2w7bsXaw3sM5YzTDGTPQlqNjYWFPWC3D98MMPBf5Clybjx4/36aDupWLkyJFER0fzxz/+saRDKZLJkyeTlpbGSy+9FNDl5ve7LSKbjTGx3ub15SzV3V6m/wP4h4dp/wT+6W0ZSgVKTEwMFStWdJ2dK6sGDBjAvn37WL58eUmHUiiXydVGqiDuB4/9beLEicyfPz/XuEGDBjFu3LiALD/neIq79u3b89tvua/8njNnDq1bl/hz4zzKOSNX1mjCUQE1bty4gCUXX23Y4O2x3qq4aMU/pVTAaMJRSgWMJhylVMBowlFKBYwmHAVApUqVfG6bkJDAggUL/BZLcfXvqZ/ExERGjx5d5P5V4elZKlXmFbaeTGxsLLGxXq9R84nW2Ckc3cIJhK/Hwsxbivf19dgCFzl27Fjefvtt1/D48eOZMGECPXv2pG3btrRu3ZrFixf7FL4xhpEjR9K8eXNuuOEGjh075pq2efNmunbtSkxMDL179+bw4cMA7N27lxtuuIGoqCjatm3Lvn37MMbwl7/8hVatWtG6dWvmzZt30f1369aNRx99lNjYWN58802PsS9btozY2Fj+8Ic/sGTJEgBWrlzputt9/PjxPPDAA3Tr1o2mTZsyZcoU17z9+/cnJiaGiIgIpk+f7hpfqVIl/vznPxMVFcXEiRPp37+/a9p3333HgAEDPMbz0EMPERsbS0REBM8//7xr/KZNm+jYsSNRUVG0a9eO9PR0HA4HTzzxBK1atSIyMtJ1n1rjxo05fvw4YG2tdevWzfVZhgwZQqdOnRgyZAgHDx6kc+fOtG3blrZt27J27VrX8l555RVat25NVFQUY8eOZd++fbRt29Y1fc+ePbmGi0vpS4GqWMTHx/Poo4/yyCOPAFaphW+//ZbRo0dTpUoVjh8/TocOHejbt6/XEpwLFy7kxx9/ZOfOnRw9epSWLVu6ylaMGjWKxYsXU6tWLebNm8e4ceOYMWMGgwcPZuzYsQwYMIDMzEycTidffPEFSUlJbNmyhePHjxMXF0eXLl1Yt25dofsHOH/+vNdn0GuNndJVY0cTTiD0KbbKqj6Ljo7m2LFj/Prrr6SkpFCtWjXq1KnDY489xqpVqwgKCuKXX37h6NGj1KlTp8C+Vq1axd13301wcDD16tWjR48eAPz4449s376dXr16AdbuRd26dUlPT+eXX35xfdPn3IS5Zs0aVz+1a9ema9eubNq0qdD954iPj/e6HrTGTumqsaMJ5xI2aNAgFixYwJEjR4iPj+fjjz8mJSWFzZs3ExoaSuPGjYv0ZFBjDBEREaxbty7XeF9KURSl/xzu9WQ80Ro7pavGjh7DuYTFx8czd+5cFixYwKBBg0hLS+PKK68kNDSUFStW8NNPP/nUT5cuXZg3bx4Oh4PDhw+zYsUKwKoNk5KS4koIWVlZ7Nixg8qVK9OgQQMWLVoEwG+//cbZs2fp3Lmzq5+UlBRWrVpFu3btCt1/YWiNndJVY0cTziUsIiKC9PR06tevT926dRk8eDCJiYm0bt2aDz/8kGuvvdanfgYMGECzZs1o2bIl9913H9dddx1g1YZZsGABTz31FFFRUbRp08Z1YHLOnDlMmTKFyMhIOnbsyJEjRxgwYICrXkyPHj2YNGkSderUuaj+faU1dkpXjR2v9XBKgtbDUWXN5VRjx6/1cJRSBdMaO77ThKNctm3bxpAhQ3KNK1++fKkv36A1dopHIGrs+FJi1NuTNwcDT2E9MDIdeMgYs8WedtAe5wCyfdnkAt2lUqo0K8oulS8HjWcBNxUw/QDQ1RjTGngJmJ5nendjTBtfk41S6tJV5Cdv5hRMt63HehyMUkr9TnGfFv8j8LXbsAH+JSKb7QfdKaUuY8V20FhEumMlnOvdRl9vjPlFRK4EvhORXcaYVR7mv2SfvKmUshTLFo6IRALvA/2MMak5440xv9g/jwELgXae+tAnb/rX+PHjc13gVRr79tTPr7/+yh133FHk/lXJK44nbzYCvgCGGGN2u42vKCKVc94DNwLbi7o8VfblXKrvq3r16hVbwa+cy/tLm8Kuk7LK6y6V+5M3RSSZPE/eBJ4DagDv2DfG5Zz+rg0stMeFAJ8YY77xw2co9V7Z+Aq7Tvz+LuWiuLb6tTzV7qkC20ycOJHZs2dz5ZVX0rBhQ2JiYti3bx+PPPIIKSkpVKhQgffee49rr72Wo0ePMmLECPbv3w/Au+++S8eOHXnttddc5SCGDRvGo48+6rFvwGP/CQkJhIWF8b///Y9OnTrx2muv5Rvzli1buO666zh+/DhPPvkkw4cP5+DBg9x6661s376dWbNm8eWXX3L27Fn27dvHgAEDmDRpEmDVmtm0aRPnzp3jjjvu4IUXXgCse4/i4+P57rvvuP322/n8889dz0nfs2cP8fHxHp+b/uKLL/LVV19x7tw5OnbsyLRp0xAR9u7dy4gRI0hJSSE4OJj58+dz9dVX88orr/DRRx8RFBREnz59ePnll+nWrRuTJ08mNjaW48ePExsby8GDB5k1axZffPEFGRkZOBwOli5dSr9+/Th58iRZWVlMmDCBfv36AfDhhx8yefJkRITIyEjeeecdIiMj2b17N6GhoZw+fZqoqCjXcGlVHE/eHAYMy2f8fiDq4kNTRbF582bmzp1LUlIS2dnZtG3blpiYGB588EGmTp1Ks2bN2LBhAw8//DDLly9n9OjRdO3alYULF+JwOMjIyGDz5s3MnDmTDRs2YIyhffv2dO3aFafTmW/fgMf+AZKTk1m7dm2B9/9s3bqV9evXc+bMGaKjo7nlllt+1yYpKYn//e9/lC9fnubNmzNq1CgaNmyYb62ZyMhIAGrUqOFKKsuWLfO57svIkSN57rnnABgyZAhLlizhtttuy7feT351Zrz5/vvv2bp1K9WrVyc7O5uFCxf+rl7Rzp07mTBhAmvXrqVmzZqcOHGCypUr061bN5YuXUr//v2ZO3cuAwcOLNXJBvRK44DwtiXiD6tXr2bAgAGumid9+/YlMzOTtWvXMmjQIFe7nKthly9f7qrjEhwcTNWqVVmzZg0DBgxw3fQ3cOBAVq9ejdPp/F3fABkZGR77B+vqX2/lOPv160d4eDjh4eF0796djRs30qZNm1xtevbsSdWqVQFo2bIlP/30Ew0bNsy31kxOwnGvnVOYui8rVqxg0qRJnD17lhMnThAREUG3bt3yrffjqc5MQXr16uVqZ4zh6aef/l29ouXLlzNo0CBq1qyZq99hw4YxadIk+vfvz8yZM3nvvfe8Lq+kacK5jDidTq644gqSkpJKpH9/1q/xVmvGfdm+1n3JzMzk4YcfJjExkYYNGzJ+/Hi/1q8pbL2iTp06cfDgQVauXInD4aBVq9/dCFDqaHmKS1SXLl1YtGgR586dIz09na+++ooKFSrQpEkT131Hxhi2bNkCWFsN7777LmAdWE1LS6Nz584sWrSIs2fPcubMGRYuXEjnzp3z7RusSnWe+vfV4sWLyczMJDU1lZUrV7qq4HnjqdZMfnyt+5Lzx16zZk0yMjJcB6491fvxVGfGvX5NQQe/PdUr6tGjB/Pnzyc1NTVXvwD33Xcf99xzj9/q1xQ3TTiXqLZt2xIfH09UVBR9+vRx/eF+/PHHfPDBB0RFRREREeEqpP7mm2+yYsUKWrduTUxMDDt37qRt27YkJCTQrl072rdvz7Bhw4iOjvbYd0H9+yoyMpLu3bvToUMHnn32WerVq+fTfJ5qzXjiS92XK664guHDh9OqVSt69+6d63PmV+/HU52ZJ554gnfffZfo6GhX8XNPMeVXrygiIoJx48bRtWtXoqKiePzxx3PNc/LkSe6+u8BDraWG1sPxE715s3Tzpe5LWbBgwQIWL17MnDlzArZMrYejVCEEou5LIIwaNYqvv/6af/7znyUdis804aiAmzlz5u+eJdWpU6dcz9Hyp/zqvgwYMIADBw7kGvfKK6/Qu3fvgMR0MXKeU1WWaMJRATd06NBSd5AzEMWnlB40VkoFkCYcpVTAaMJRSgWMJhylVMBowlEAVKpUqaRD8NmiRYvYuXNngW26detGftdyffnll7z8cuCf9a4smnBUmeNLwvGkb9++jB07tsgxGGNc90eVNqW5to6eFg+AI3/9K7/9ULz1cMq3uJY6Tz/tcfrYsWNp2LAhjzzyCGBV0wsJCWHFihX51lvxJr86L++99x7Tp0/n/PnzXHPNNcyZM4cKFSq4at8kJiZy+vRpXnvtNW699VZ27NjB0KFDOX/+PE6nk88//5zQ0FD69OnD9ddfz9q1a6lfvz6LFy8mPDw839o6J06c4Msvv+Q///kPEyZM4PPPP+fqq6/ON+Y5c+YwbNgwsrOzmTFjBu3atWPWrFkkJibyj3/8g4SEBKpUqUJiYiJHjhxh0qRJ3HHHHWRkZORbl+bgwYP07t2b9u3bs3nzZu68805OnjzJG2+8AcB7773Hzp07ef311/ONp3///hw6dIjMzEzGjBnDgw9aZb6/+eYbnn76aRwOBzVr1uTf//43GRkZjBo1isTERESE559/nttvv51KlSqRkZEBWFcZL1myhFmzZv2u3tBdd93FmDFjyMzMJDw8nJkzZ9K8eXMcDgdPPfUU33zzDUFBQQwfPpyIiAimTJniujfsu+++45133vHLpQKacC5R8fHxPProo66E89lnn/Htt98yevTo39Vbye+ObHee6rwMHDiQ4cOHA/DMM8/wwQcfMGrUKAAOHjzIxo0b2bdvH927d2fv3r1MnTqVMWPGMHjwYM6fP4/D4eDo0aPs2bOHTz/9lPfee48777yTzz//nHvvvddjbZ2+ffty6623ei07evbsWZKSkli1ahUPPPAA27f/vuDk4cOHWbNmDbt27aJv377ccccdhIWF5VuXBqyCXbNnz6ZDhw5kZGQQFRXFq6++SmhoKDNnzmTatGke45kxYwbVq1fn3LlzxMXFcfvtt+N0Ohk+fDirVq2iSZMmrnX70ksvUbVqVbZt2wbAyZMnC/yskLve0OnTp1m9ejUhISEsW7aMp59+ms8//5zp06dz8OBBkpKSCAkJ4cSJE1SrVo2HH36YlJQUatWqxcyZM3nggQe8Lu9iaMIJgIK2RPwlOjqaY8eO8euvv5KSkkK1atWoU6cOjz322O/qrdSpU6fAvjzVedm+fTvPPPMMp06dIiMjI9dVuXfeeSdBQUE0a9aMpk2bsmvXLq677jomTpxIcnIyAwcOpFmzZgA0adLEVfMmJiaGgwcPeq2t44ucGxq7dOnC6dOnOXXq1O/a9O/fn6CgIFq2bMnRo0cBz3VpAK666io6dOgAWMe9evTowZIlS2jRogVZWVkFPllzypQprq2GQ4cOsWfPHlJSUujSpQtNmjTJtW6XLVvG3LlzXfNWq1bN6+d1rzeUlpbG/fffz549exARsrKyXP2OGDGCkJCQXMsbMmQIH330EUOHDmXdunWu2kjFzaeE48PTNwV4E7gZOAskGGO+t6fdDzxjN51gjJldHIEr7wYNGsSCBQs4cuQI8fHxha634k1CQgKLFi0iKiqKWbNmsXLlSte0/Ora3HPPPbRv356lS5dy8803M23aNJo2bfq7+jbnzp0rlto9ha2tk3Mjc0HrKW9Nn2HDhvHXv/6Va6+9tsCrp1euXMmyZctYt24dFSpUoFu3bhe17t0/Q0G1dZ599lm6d+/OwoULOXjwIN26dSuw36FDh3LbbbcRFhbGoEGDXAmpuPl60HgWBT99sw/QzH49CLwLICLVsWogt8d6YsPzIuI9VatiER8fz9y5c1mwYAGDBg3yWG/FG091XtLT06lbty5ZWVl8/PHHueaZP38+TqeTffv2sX//fpo3b87+/ftp2rQpo0ePpl+/fmzdutXjMguqrVO5cmXS09O9xj1v3jwA1qxZQ9WqVV1VAr0pzHpq3749hw4d4pNPPimwRERaWhrVqlWjQoUK7Nq1i/Xr1wPQoUMHVq1a5bqPK2fd9urVK9e9ZTm7VLVr1+aHH37A6XQWeIwlLS2N+vXrAzBr1izX+F69ejFt2jTXgeWc5dWrV4969eoxYcIEv9524iCl3lYAAA/HSURBVFPCsZ8lVVCB1n7Ah8ayHrhCROoCvYHvjDEnjDEnge8oOHGpYhQREUF6ejr169enbt26HuuteOOpzstLL71E+/bt6dSp0+/6atSoEe3ataNPnz5MnTqVsLAwPvvsM1q1akWbNm3Yvn079913X4HL9VRb56677uLVV18lOjqaffv2eZw/LCyM6OhoRowYwQcffODTZwXPdWk8ufPOO+nUqVOBuz033XQT2dnZtGjRgrFjx7p2y2rVqsX06dMZOHAgUVFRrlKozzzzDCdPnqRVq1ZERUWxYsUKAF5++WVuvfVWOnbsSN26dT0u78knn+T//u//iI6OznXWatiwYTRq1IjIyEiioqL45JNPcn3uhg0b+resijHGpxfQGNjuYdoSrIfe5Qz/G4gFngCecRv/LPCEt2XFxMSYsm7nzp0lHUKJuf/++838+fNLOoyAueWWW8yyZctKOowie+SRR8z777/vtV1+v9tAovEhj5Sa63BE5EERSRSRxJSUlJIORymvTp06xR/+8AfCw8Pp2bNnSYdTJDExMWzdupV7773Xr8spriNDvwAN3YYb2ON+wXqmlfv4lfl1YIyZDkwHq+JfMcWlCmHbtm0MGTIk17jy5cuzYcOGQvXjfszA3x555BH++9//5ho3ZsyYgJS/uOKKK9i9e3eucampqfkmn3//+98ei7WXBjk1l/2tuBLOl8BIEZmLdYA4zRhzWES+Bf7qdqD4RuD/immZqpi1bt3ab0908JdAFe3yVY0aNcrcOgwkX0+Le3v65j+xTonvxTotPtSedkJEXgI22V29aIzx/nSwS4QxxutFdUqVJaaINdB9SjjG+9M3DfCIh2kzgBmFD61sCwsLIzU1lRo1amjSUZcEYwypqamuB/9dDL3S2E8aNGhAcnIyegBcXUrCwsJo0KDBRc+vCcdPQkNDXZerK6Uspea0uFLq0qcJRykVMJpwlFIBowlHKRUwmnCUUgGjCUcpFTCacJRSAaMJRykVMJpwlFIBowlHKRUwmnCUUgGjCUcpFTCacJRSAaMJRykVMJpwlFIB41PCEZGbRORHEdkrImPzmf66iCTZr90icsptmsNt2pfFGbxSqmzxWoBLRIKBt4FeQDKwSUS+NMbszGljjHnMrf0oINqti3PGmDbFF7JSqqzyZQunHbDXGLPfGHMemIv1pE1P7v7/9u4/yK6yvuP4+3M3uwnZxWRDYhq0+UW1gqNC3cGxzAgFgQCdRKZaMwwzUOk4YjP9w+nUMHQsg7VF/aM6w3SKjSjiFJTMtLUODg1E2w4aIKgE0DCEEJSEECAEbaLZ3Xu//eM8d/fs3V83u2fPvbv7eU3OnPM858f97rN3v3vuvZvnC9xTRHBmNrc0k3DeAvwy134x9Y0iaQ2wDtiZ616UCtztkvSh8R7EhfDM5r6i3zTeDGyPiGqub01E9AHXAF+SdNZYJ0bEVyKiLyL6VqxYUXBYZtYOmkk441XVHMtmGl5ORcTBtN5PVnXzvNGnmdl80EzCeQx4m6R1krrIksqoT5skvQPoBX6U6+uVtDBtLwcuAH7WeK6ZzQ+TfkoVEYOStgAPAB3AnRHxtKRbgd0RUU8+m4F7Y2RpvrOBOyTVyJLbbflPt8xsftF0S3fOhL6+vti9e3erwzCzJkl6PL1XOyH/pbGZlcYJx8xK44RjZqVxwjGz0jjhmFlpnHDMrDROOGZWGiccMyuNE46ZlcYJx8xK44RjZqVxwjGz0jjhmFlpJp2ewma3iCAGBoj+fuLkyWzp76d2sj/r68/aUauhjgWoowKj1h0oLfXtfRdf0uovbdZYvmULlZ5uJGUd9XXWGN0nUTt+nF8/9NDowxh53Fjnju6b2rlvuvIKln7kI8NxF8AJZxxHv3E3x+67j0pPD5XubiqLFw9vdy9O66xflQqoAhVl35wx20rHaXhfrq3K8HExMMjhW2+l/7nnWj0MVoBXb7+91SFMyYldu1h69dXQ2VnYNWdtwvnNnj0c+NOPtjqMWWvJpk2cvuFy1NVFpasLKhWiWoVqlajWoDpIVGtEdRDq61qNGKxCrcrAwUP8du9eBg4d4uTeva3+cqZs+SdvhEpH1oiAqBERabveF0C2XvSud1NZvHioTcTI4xv6K4sWcdp73jP8gPn5p9L2WHNSRf8Ag4dfGv+4GLUxfO3JHiMYfRyjz+1aswYVmGygyYQjaQPwZbIZ/7ZFxG0N+68HvsjwXMe3R8S2tO864G9S/99FxF0FxE319deLuEzLLLvuOk6/9INErZae2LXsB7qWPVkHX3uV2okTSMp+8AcG0tKf284WBgeJ/oFR/UPL4OCIdueZZ7Lypq10LF3a6mEoTURk4zQwQC29tKz09NDR09Pq0MbVufLNrQ6hcIUUwku+FRFbGs5dBvwt0Ef2Y/V4Onfa2aLnwgs5e+/Pp3sZmyckQWcn6uxMdyjWCjNRCC/vcmBHRBxNSWYHsGFqoZrZbFdkIbw/kbRH0nZJ9bIyTRfRM7O5r6i/w/lPYG1EvJvsLuaU36dx5U2zua+QQngR8VpEnEzNbcB7mz03dw1X3jSb4wophCdpVa65Eai/m/sAcFkqiNcLXJb6zGweKqoQ3l9K2ggMAkeB69O5RyV9lixpAdwaEUdn4Osws1nAhfDMbNpcCM/M2o4TjpmVxgnHzErjhGNmpXHCMbPSOOGYWWmccMysNE44ZlYaJxwzK40TjpmVxgnHzErjhGNmpXHCMbPSOOGYWWmccMysNE44ZlYaJxwzK01TCUfSBknPSNonaesY+z8l6WepTMxDktbk9lUl/TQt32k818zmj6Iqb/4E6IuIE5JuBL4A1At//yYizi04bjObhQqpvBkR34+IE6m5i6wcjJnZCEVW3qy7Afherr0oFbjbJelDU4jRzOaISV9SnQpJ1wJ9wIW57jURcVDSemCnpCcj4rkxzv048HGA1atXFxmWmbWJQipvAkj6IHAzsDFXhZOIOJjW+4EfAOeN9SCuvGk29xVVefM84A6yZHMk198raWHaXg5cAOTfbDazeaSoyptfBHqA+yQB/CIiNgJnA3dIqpElt9saPt0ys3nElTfNbNpcedPM2o4TjpmVxgnHzErjhGNmpXHCMbPSOOGYWWmccMysNE44ZlYaJxwzK40TjpmVxgnHzErjhGNmpXHCMbPSOOGYWWmccMysNE44ZlYaJxwzK01RlTcXSvpW2v+IpLW5fTel/mckXV5c6GY22xRVefMG4PWI+D1Jm4HPAx+VdA7ZpOvvBM4EHpT09oioFhH8D17YyaHjL43oC9KUqbmpU4f6cu3GqVV3/OJB9r/xPACrT1/N8cHjPJ/aja5Yu4HFnd1D142IoWvm+2rURuzP/gWSuOYd17BowaLhmBrjeWEHB351YMKvXwhJRGTXFBqxL20Mbaf5puld2MtV668aaufPa9bDBx/m3mfu5ciJI5MfPIdcuuZSlixcMtSOhudZ4/exp6uHTWdtokMdTV2//vzpr/Wz/9j+AiKeWP05MJZzlp3D+qXri328yeY0lvR+4JaIuDy1bwKIiH/IHfNAOuZHkhYAh4EVwNb8sfnjJnrMZuY0fuype/jY438/8VdnZtPy42sepbPztEmPa3ZO42YK4Y1VefN94x2Tqjy8AZyR+nc1nDtR1c6mvfesK/j0o7fzwvFDo/ZpaB2j+ka0h3YH9/d0c7Qj+y20YnCQVxaMPzQLIuitVkdcU/kl6tsxdEwl9de3b1i0mu7ekb896r9tYuC33H/wv9nf1cmIMCF3RWj8VTGqreG+/Hp5tcb1y86j47Rl1O/JTvUe5+Yj/8uvO+bnW4BnDDZ+7yOtGbEGWFKt8Wedq+hc/vamry/gxMk3eOTlx8Y9pojSB5Nd449Wnk9n7i68CIVW3pyOU628WTltKdd+7OHCHv/ThV2pGJe0OoBJ/LDVAcwDc7EudlGVN4eOSS+plgCvNXku4MqbZvNBIZU3U/u6tP1hYGdkbw59B9icPsVaB7wNeLSY0M1stimq8uZXgbsl7QOOkiUl0nHfJivvOwj8RVGfUJnZ7OPKm2Y2ba68aWZtxwnHzErTli+pJL0CvFDywy4HXi35MSfSTvG0UyzQXvG0UyzQunjWRMSkHy+3ZcJpBUm7m3kNWpZ2iqedYoH2iqedYoH2i6eRX1KZWWmccMysNE44w77S6gAatFM87RQLtFc87RQLtF88I/g9HDMrje9wzKw0czLhSLpT0hFJT42z/yJJb0j6aVo+07C/Q9JPJH0317cuzWa4L81u2NXCWL4u6fncOec2E8t045F0QNKTqX93rn+ZpB2Snk3r3hbHc4ukg7lzriwhlqWStkvaK+nnaR6pVo7NePFMaWwKExFzbgE+APwB8NQ4+y8CvjvB+Z8C/jV/DPBtYHPa/mfgxhbG8nXgw2WPDXAAWD5G/xeArWl7K/D5FsdzC/BXJY/NXcCfp+0uYGmLx2a8eKY0NkUtc/IOJyL+h+w/kZ4ySW8FrgK25foEXAxsT1130eR0JUXHMl3TiWcCm8jGBE5hbGYwnimZaiySlpAlh6+m6/RHxLG0u/SxmSSelpqTCadJ75f0hKTvSXpnrv9LwF9DmpA4cwZwLCIGU7uwmQunEEvd5yTtkfSPkhYWGMtE8QTwX5IeVzZhWt3KiKhPLn0YWNnieAC2pPG581RexkwxlnXAK8DX0svfbZK6075WjM1E8cDMjc3kWnVrNdMLsJbxb0XfBPSk7SuBZ9P2HwP/1Hi7Svbn4vty5//ueNee6VhSexXZTJQLyX5rfmamxya135LWbwaeAD6Q2scarvF6i+NZSTaVSgX4HNmUKjP5veojm37lfan9ZeCzrRqbSeKZ8tgUsczLO5yI+FVE/F/avh/olLQcuADYKOkAcC9wsaRvks1euFTZbIYwwcyFJcRCRLwUmZPA14Dzi4hlkniIiINpfQT4t9zjvixpFUBaF1bKYSrxRMTLEVGNiBrwLxQ0PhPE8iLwYkQ8kg7dTvbeC7RmbMaNZ6bGplnzMuFI+p30vgySzicbh9ci4qaIeGtErCWbRGxnRFwb2a+G75PNZgjZ7Ib/0YpY0nH1J7DI3hMY81OMIuOR1C3p9NTfDVyWe9z8jI+Fjc1U46mPT3I1BY3PBN+rw8AvJf1+OvQSsknnoAVjM1E8MzU2TSvzdqqsBbgHeAkYIMv2NwCfAD6R9m8Bnia7Dd8F/OEY17iIkS9j1pNNj7oPuA9Y2MJYdgJPkj1Zvkm6rZ7JeNLX/0RangZuzl3zDOAh4FngQWBZi+O5O43PHrIf+FUz/b0CzgV2p8f8d6C3VWMzSTxTGpuiFv+lsZmVZl6+pDKz1nDCMbPSOOGYWWmccMysNE44ZlYaJxwzK40TjpmVxgnHzErz/6ysKz1FqldSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f61cc2c0c10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<keras.engine.training.Model at 0x7f61d5643a50>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train(model=model, data=((train_tensors, train_labels), (valid_tensors, valid_labels)), \n",
    "      lr=0.001, lr_decay=0.9, lam_recon=0.392, batch_size=32, shift_fraction=0.1, epochs=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.load_weights('saved_models/CapsNet.best.from_scratch.hdf5')\n",
    "prediction = eval_model.predict(test_tensors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.734694 %\n",
      "Recall: 0.073022 %\n",
      "Fscore: 0.261248 %\n"
     ]
    }
   ],
   "source": [
    "threshold = 0.5\n",
    "beta = 0.5\n",
    "\n",
    "pre = K.eval(precision_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction[0])))\n",
    "rec = K.eval(recall_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction[0])))\n",
    "fsc = K.eval(fbeta_score_threshold(beta = beta, threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction[0])))\n",
    "\n",
    "print (\"Precision: %f %%\\nRecall: %f %%\\nFscore: %f %%\"% (pre, rec, fsc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.57466066"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K.eval(binary_accuracy(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction[0])))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}