Proj3_v1_learning_embedding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "keras 2.0.8\n",
      "tensorflow 1.3.0\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "## To get helper functions from ano`ther folder\n",
    "# sys.path.insert(0, '../') # if _helper_basics_ is in previous folder\n",
    "now_i_am_at = 'home' # home dso test\n",
    "if now_i_am_at=='home':\n",
    "    sys.path.insert(0, 'E:/Leonard HDD/Dropbox/DSO/Tasks/')\n",
    "elif now_i_am_at=='dso':\n",
    "    sys.path.insert(0, 'D:/Dropbox/DSO/Tasks')\n",
    "\n",
    "from _helper_basics_ import *\n",
    "\n",
    "%matplotlib inline\n",
    "plt.rcParams['savefig.dpi'] = 100\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "print('keras', keras.__version__)\n",
    "print('tensorflow', tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_mode = 'reuters'\n",
    "Data_dir = 'Data_'+data_mode+'/'\n",
    "\n",
    "X_train = np.load(Data_dir+'X_train_reu.npy')\n",
    "Y_train = np.load(Data_dir+'Y_train_reu.npy')\n",
    "X_test = np.load(Data_dir+'X_test_reu.npy')\n",
    "Y_test = np.load(Data_dir+'Y_test_reu.npy')\n",
    "\n",
    "print('X_train', X_train.shape)\n",
    "print('Y_train', Y_train.shape)\n",
    "print('X_test ', X_test.shape)\n",
    "print('Y_test ', Y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train (20000, 80)\n",
      "Y_train (20000, 2)\n",
      "X_test  (4000, 80)\n",
      "Y_test  (4000, 2)\n"
     ]
    }
   ],
   "source": [
    "data_mode = 'imdb'\n",
    "Data_dir = 'Data_'+data_mode+'/'\n",
    "\n",
    "## Without splitting into train/test set\n",
    "# X_train = np.load(Data_dir+'X_train_imdb.npy')\n",
    "# Y_train = np.load(Data_dir+'Y_train_imdb.npy')\n",
    "\n",
    "## Split into train/test set\n",
    "X_train = np.load(Data_dir+'X_train_spliced.npy')\n",
    "Y_train = np.load(Data_dir+'Y_train_spliced.npy')\n",
    "X_test = np.load(Data_dir+'X_test_spliced.npy')\n",
    "Y_test = np.load(Data_dir+'Y_test_spliced.npy')\n",
    "\n",
    "print('X_train', X_train.shape)\n",
    "print('Y_train', Y_train.shape)\n",
    "print('X_test ', X_test.shape)\n",
    "print('Y_test ', Y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Archi_dir = \"v1/\"\n",
    "Weights_path = Archi_dir+\"Logs/\"\n",
    "Ckpt_Mod_Weights_fold = Archi_dir+\"Checkpoint_Model_Weights/\"\n",
    "if not os.path.exists(Archi_dir): os.mkdir(Archi_dir)\n",
    "if not os.path.exists(Weights_path): os.mkdir(Weights_path)\n",
    "if not os.path.exists(Ckpt_Mod_Weights_fold): os.mkdir(Ckpt_Mod_Weights_fold)\n",
    "    \n",
    "plot_path_dir = Archi_dir+'Plots/'\n",
    "if not os.path.exists(plot_path_dir): os.mkdir(plot_path_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_words = 20000\n",
    "input_length  = 80\n",
    "num_class  = Y_train.shape[1] # 46\n",
    "input_dim, output_dim = num_words, 128\n",
    "########################### Architecture ##################################\n",
    "def build_model_reg(dropout_rate, reg_l1=0., reg_l2=0.):\n",
    "    Inp = Input(shape=(input_length,), name=\"Input\")\n",
    "    x   = Embedding(input_dim, output_dim, mask_zero=True, input_length=input_length)(Inp)\n",
    "    x   = GRU(128, name=\"RNN_1\", return_sequences=False, kernel_regularizer=l2(reg_l2))(x) \n",
    "    x   = Dropout(dropout_rate)(x)\n",
    "    x   = Dense(num_class, activation='softmax',name = \"Output\" )(x)\n",
    "    model = Model(Inp, x)\n",
    "    ########################### Printing information ########################\n",
    "    model.summary()\n",
    "    # print_model_weights(model)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "learning_rate None\n",
      "decay 1e-09\n",
      "training_epochs 100\n",
      "batch_size 32\n"
     ]
    }
   ],
   "source": [
    "######################## Training Parameters ###############################\n",
    "learning_rate = None;\tprint('learning_rate',learning_rate)\n",
    "decay = 1e-9;          \tprint('decay',decay)\n",
    "training_epochs = 100;\tprint('training_epochs',training_epochs)\n",
    "batch_size = 32;\t\tprint('batch_size',batch_size)\n",
    "######################## Checkpoints ###############################\n",
    "reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=10, min_lr=1e-6, cooldown=5, epsilon=1e-05, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regularization with lr=1e-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Input (InputLayer)           (None, 80)                0         \n",
      "_________________________________________________________________\n",
      "embedding_9 (Embedding)      (None, 80, 128)           2560000   \n",
      "_________________________________________________________________\n",
      "RNN_1 (GRU)                  (None, 128)               98688     \n",
      "_________________________________________________________________\n",
      "dropout_7 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 2)                 258       \n",
      "=================================================================\n",
      "Total params: 2,658,946\n",
      "Trainable params: 2,658,946\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "learning_rate 0.001\n",
      "ckpt_path :  v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-{epoch:04d}_VCA-{val_categorical_accuracy:.2f}.hdf5 \n",
      "\n",
      "Train on 20000 samples, validate on 4000 samples\n",
      "Epoch 1/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.4717 - categorical_accuracy: 0.7804Epoch 00000: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0000_VCA-0.92.hdf5\n",
      "20000/20000 [==============================] - 144s - loss: 0.4714 - categorical_accuracy: 0.7806 - val_loss: 0.2388 - val_categorical_accuracy: 0.9183\n",
      "Epoch 2/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.2614 - categorical_accuracy: 0.9004Epoch 00001: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0001_VCA-0.96.hdf5\n",
      "20000/20000 [==============================] - 137s - loss: 0.2617 - categorical_accuracy: 0.9003 - val_loss: 0.1625 - val_categorical_accuracy: 0.9610\n",
      "Epoch 3/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.1656 - categorical_accuracy: 0.9424Epoch 00002: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0002_VCA-0.98.hdf5\n",
      "20000/20000 [==============================] - 138s - loss: 0.1655 - categorical_accuracy: 0.9424 - val_loss: 0.0736 - val_categorical_accuracy: 0.9832\n",
      "Epoch 4/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0960 - categorical_accuracy: 0.9699Epoch 00003: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0003_VCA-0.98.hdf5\n",
      "20000/20000 [==============================] - 137s - loss: 0.0962 - categorical_accuracy: 0.9698 - val_loss: 0.0734 - val_categorical_accuracy: 0.9842\n",
      "Epoch 5/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0602 - categorical_accuracy: 0.9836Epoch 00004: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0004_VCA-0.99.hdf5\n",
      "20000/20000 [==============================] - 137s - loss: 0.0603 - categorical_accuracy: 0.9836 - val_loss: 0.0331 - val_categorical_accuracy: 0.9928\n",
      "Epoch 6/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0441 - categorical_accuracy: 0.9886Epoch 00005: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0005_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 137s - loss: 0.0441 - categorical_accuracy: 0.9886 - val_loss: 0.0289 - val_categorical_accuracy: 0.9965\n",
      "Epoch 7/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0281 - categorical_accuracy: 0.9939Epoch 00006: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0006_VCA-0.99.hdf5\n",
      "20000/20000 [==============================] - 137s - loss: 0.0281 - categorical_accuracy: 0.9938 - val_loss: 0.0291 - val_categorical_accuracy: 0.9935\n",
      "Epoch 8/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0253 - categorical_accuracy: 0.9946Epoch 00007: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0007_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 137s - loss: 0.0253 - categorical_accuracy: 0.9946 - val_loss: 0.0105 - val_categorical_accuracy: 0.9995\n",
      "Epoch 9/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0225 - categorical_accuracy: 0.9941Epoch 00008: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0008_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 138s - loss: 0.0226 - categorical_accuracy: 0.9941 - val_loss: 0.0121 - val_categorical_accuracy: 0.9982\n",
      "Epoch 10/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0203 - categorical_accuracy: 0.9952Epoch 00009: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0009_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 136s - loss: 0.0203 - categorical_accuracy: 0.9952 - val_loss: 0.0109 - val_categorical_accuracy: 0.9985\n",
      "Epoch 11/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0212 - categorical_accuracy: 0.9951Epoch 00010: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0010_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 137s - loss: 0.0213 - categorical_accuracy: 0.9951 - val_loss: 0.0106 - val_categorical_accuracy: 0.9980\n",
      "Epoch 12/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0165 - categorical_accuracy: 0.9958Epoch 00011: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0011_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 137s - loss: 0.0165 - categorical_accuracy: 0.9959 - val_loss: 0.0102 - val_categorical_accuracy: 0.9990\n",
      "Epoch 13/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0165 - categorical_accuracy: 0.9964Epoch 00012: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0012_VCA-0.99.hdf5\n",
      "20000/20000 [==============================] - 137s - loss: 0.0165 - categorical_accuracy: 0.9964 - val_loss: 0.0289 - val_categorical_accuracy: 0.9922\n",
      "Epoch 14/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0129 - categorical_accuracy: 0.9979 ETA: Epoch 00013: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0013_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 137s - loss: 0.0129 - categorical_accuracy: 0.9979 - val_loss: 0.0080 - val_categorical_accuracy: 0.9992\n",
      "Epoch 15/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0112 - categorical_accuracy: 0.9977Epoch 00014: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0014_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 137s - loss: 0.0111 - categorical_accuracy: 0.9978 - val_loss: 0.0077 - val_categorical_accuracy: 0.9995\n",
      "Epoch 16/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0103 - categorical_accuracy: 0.9981Epoch 00015: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0015_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 137s - loss: 0.0103 - categorical_accuracy: 0.9981 - val_loss: 0.0070 - val_categorical_accuracy: 0.9992\n",
      "Epoch 17/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0094 - categorical_accuracy: 0.9985Epoch 00016: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0016_VCA-0.99.hdf5\n",
      "20000/20000 [==============================] - 136s - loss: 0.0094 - categorical_accuracy: 0.9986 - val_loss: 0.0200 - val_categorical_accuracy: 0.9948\n",
      "Epoch 18/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0147 - categorical_accuracy: 0.9965Epoch 00017: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0017_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 136s - loss: 0.0146 - categorical_accuracy: 0.9965 - val_loss: 0.0097 - val_categorical_accuracy: 0.9990\n",
      "Epoch 19/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0086 - categorical_accuracy: 0.9987Epoch 00018: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0018_VCA-1.00.hdf5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20000/20000 [==============================] - 136s - loss: 0.0088 - categorical_accuracy: 0.9987 - val_loss: 0.0084 - val_categorical_accuracy: 0.9985\n",
      "Epoch 20/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0122 - categorical_accuracy: 0.9976Epoch 00019: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0019_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 136s - loss: 0.0126 - categorical_accuracy: 0.9975 - val_loss: 0.0086 - val_categorical_accuracy: 0.9990\n",
      "Epoch 21/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0098 - categorical_accuracy: 0.9981Epoch 00020: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0020_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 136s - loss: 0.0098 - categorical_accuracy: 0.9981 - val_loss: 0.0050 - val_categorical_accuracy: 0.9998\n",
      "Epoch 22/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0056 - categorical_accuracy: 0.9995Epoch 00021: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0021_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 136s - loss: 0.0056 - categorical_accuracy: 0.9995 - val_loss: 0.0058 - val_categorical_accuracy: 0.9995\n",
      "Epoch 23/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0042 - categorical_accuracy: 0.9996Epoch 00022: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0022_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 136s - loss: 0.0042 - categorical_accuracy: 0.9996 - val_loss: 0.0027 - val_categorical_accuracy: 1.0000\n",
      "Epoch 24/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0048 - categorical_accuracy: 0.9993 ETA: 2s - loss: 0.0047 - cateEpoch 00023: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0023_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 136s - loss: 0.0048 - categorical_accuracy: 0.9994 - val_loss: 0.0060 - val_categorical_accuracy: 0.9992\n",
      "Epoch 25/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0107 - categorical_accuracy: 0.9976 ETAEpoch 00024: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0024_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 136s - loss: 0.0107 - categorical_accuracy: 0.9977 - val_loss: 0.0044 - val_categorical_accuracy: 0.9998\n",
      "Epoch 26/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0075 - categorical_accuracy: 0.9986Epoch 00025: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0025_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 136s - loss: 0.0075 - categorical_accuracy: 0.9986 - val_loss: 0.0067 - val_categorical_accuracy: 0.9992\n",
      "Epoch 27/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0094 - categorical_accuracy: 0.9985Epoch 00026: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0026_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 136s - loss: 0.0094 - categorical_accuracy: 0.9986 - val_loss: 0.0036 - val_categorical_accuracy: 1.0000\n",
      "Epoch 28/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0038 - categorical_accuracy: 0.9998Epoch 00027: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0027_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 136s - loss: 0.0038 - categorical_accuracy: 0.9999 - val_loss: 0.0027 - val_categorical_accuracy: 1.0000\n",
      "Epoch 29/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0118 - categorical_accuracy: 0.9975Epoch 00028: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0028_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 136s - loss: 0.0118 - categorical_accuracy: 0.9976 - val_loss: 0.0052 - val_categorical_accuracy: 1.0000\n",
      "Epoch 30/30\n",
      "19968/20000 [============================>.] - ETA: 0s - loss: 0.0060 - categorical_accuracy: 0.9992 ETA: 4s - lEpoch 00029: saving model to v1/Checkpoint_Model_Weights/1e-3/weights_LR-1.00e-03_Epoch-0029_VCA-1.00.hdf5\n",
      "20000/20000 [==============================] - 136s - loss: 0.0060 - categorical_accuracy: 0.9992 - val_loss: 0.0048 - val_categorical_accuracy: 0.9995\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'history_1' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-31-6829612b36cf>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     16\u001b[0m                                          callbacks=[reduce_lr, csv_log, ckpt]) \n\u001b[0;32m     17\u001b[0m     \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfigsize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m14\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 18\u001b[1;33m     \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msubplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m;\u001b[0m\u001b[0mplot_loss\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhistory_1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmetric_list\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'loss'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'val_loss'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtitle\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'Loss'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlegend_list\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'train'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'val'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m  \u001b[0mplot_path\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mplot_path\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     19\u001b[0m     \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msubplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m;\u001b[0m\u001b[0mplot_loss\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhistory_1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmetric_list\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'categorical_accuracy'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'val_categorical_accuracy'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtitle\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'Accuracy'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlegend_list\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'train'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'val'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mplot_path\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mplot_path\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     20\u001b[0m     \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtight_layout\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'history_1' is not defined"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAD8CAYAAABZ/vJZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADShJREFUeJzt3X+o3fV9x/Hna0mF1XZV6m3pkkizEWsDq0NvrQzZ7Mpm\n4v4IBf9QS2VSCLJa+qcyWPuH/6x/DErxRwgSpP80f6zSpiOtG4zWgbPLDagxinIXmUkUjFo6sDC5\n5L0/7tk4u8v1fvP23HtOkucDLtzv+X7uPe98SO7zfu+55yRVhSRJ5+u3pj2AJOnCZEAkSS0GRJLU\nYkAkSS0GRJLUYkAkSS1rBiTJgSRvJnlhlfNJ8r0ki0meT3L95MeUJM2aIVcgjwO73uf8bmDH6G0v\n8OgHH0uSNOvWDEhVPQW88z5L9gDfr2XPAFck+dSkBpQkzabNE/gcW4CTY8enRre9sXJhkr0sX6Vw\n+eWX33DttddO4O4lSV1Hjx59q6rmOh87iYAMVlX7gf0A8/PztbCwsJF3L0laIcl/dD92Er+FdRrY\nNna8dXSbJOkiNomAHALuHv021k3Ar6vq//34SpJ0cVnzR1hJfgDcAlyV5BTwbeBDAFW1DzgM3AYs\nAr8B7lmvYSVJs2PNgFTVnWucL+DrE5tIknRB8JnokqQWAyJJajEgkqQWAyJJajEgkqQWAyJJajEg\nkqQWAyJJajEgkqQWAyJJajEgkqQWAyJJajEgkqQWAyJJajEgkqQWAyJJajEgkqQWAyJJajEgkqQW\nAyJJajEgkqQWAyJJajEgkqQWAyJJajEgkqQWAyJJajEgkqQWAyJJajEgkqQWAyJJajEgkqQWAyJJ\najEgkqQWAyJJajEgkqSWQQFJsivJy0kWkzxwjvMfS/KTJM8lOZ7knsmPKkmaJWsGJMkm4GFgN7AT\nuDPJzhXLvg68WFXXAbcAf5fksgnPKkmaIUOuQG4EFqvqRFW9BxwE9qxYU8BHkwT4CPAOsDTRSSVJ\nM2VIQLYAJ8eOT41uG/cQ8FngdeAY8M2qOrvyEyXZm2QhycKZM2eaI0uSZsGkHkS/FXgW+F3gD4GH\nkvzOykVVtb+q5qtqfm5ubkJ3LUmahiEBOQ1sGzveOrpt3D3AE7VsEXgVuHYyI0qSZtGQgBwBdiTZ\nPnpg/A7g0Io1rwFfAkjySeAzwIlJDipJmi2b11pQVUtJ7gOeBDYBB6rqeJJ7R+f3AQ8Cjyc5BgS4\nv6reWse5JUlTtmZAAKrqMHB4xW37xt5/HfjzyY4mSZplPhNdktRiQCRJLQZEktRiQCRJLQZEktRi\nQCRJLQZEktRiQCRJLQZEktRiQCRJLQZEktRiQCRJLQZEktRiQCRJLQZEktRiQCRJLQZEktRiQCRJ\nLQZEktRiQCRJLQZEktRiQCRJLQZEktRiQCRJLQZEktRiQCRJLQZEktRiQCRJLQZEktRiQCRJLQZE\nktRiQCRJLQZEktRiQCRJLYMCkmRXkpeTLCZ5YJU1tyR5NsnxJL+Y7JiSpFmzea0FSTYBDwN/BpwC\njiQ5VFUvjq25AngE2FVVryX5xHoNLEmaDUOuQG4EFqvqRFW9BxwE9qxYcxfwRFW9BlBVb052TEnS\nrBkSkC3AybHjU6Pbxl0DXJnk50mOJrn7XJ8oyd4kC0kWzpw505tYkjQTJvUg+mbgBuAvgFuBv0ly\nzcpFVbW/quaran5ubm5Cdy1JmoY1HwMBTgPbxo63jm4bdwp4u6reBd5N8hRwHfDKRKaUJM2cIVcg\nR4AdSbYnuQy4Azi0Ys2PgZuTbE7yYeALwEuTHVWSNEvWvAKpqqUk9wFPApuAA1V1PMm9o/P7quql\nJD8DngfOAo9V1QvrObgkabpSVVO54/n5+VpYWJjKfUuSliU5WlXznY/1meiSpBYDIklqMSCSpBYD\nIklqMSCSpBYDIklqMSCSpBYDIklqMSCSpBYDIklqMSCSpBYDIklqMSCSpBYDIklqMSCSpBYDIklq\nMSCSpBYDIklqMSCSpBYDIklqMSCSpBYDIklqMSCSpBYDIklqMSCSpBYDIklqMSCSpBYDIklqMSCS\npBYDIklqMSCSpBYDIklqMSCSpBYDIklqMSCSpJZBAUmyK8nLSRaTPPA+6z6fZCnJ7ZMbUZI0i9YM\nSJJNwMPAbmAncGeSnaus+w7wj5MeUpI0e4ZcgdwILFbViap6DzgI7DnHum8APwTenOB8kqQZNSQg\nW4CTY8enRrf9ryRbgC8Dj77fJ0qyN8lCkoUzZ86c76ySpBkyqQfRvwvcX1Vn329RVe2vqvmqmp+b\nm5vQXUuSpmHzgDWngW1jx1tHt42bBw4mAbgKuC3JUlX9aCJTSpJmzpCAHAF2JNnOcjjuAO4aX1BV\n2//n/SSPA/9gPCTp4rZmQKpqKcl9wJPAJuBAVR1Pcu/o/L51nlGSNIOGXIFQVYeBwytuO2c4quov\nP/hYkqRZ5zPRJUktBkSS1GJAJEktBkSS1GJAJEktBkSS1GJAJEktBkSS1GJAJEktBkSS1GJAJEkt\nBkSS1GJAJEktBkSS1GJAJEktBkSS1GJAJEktBkSS1GJAJEktBkSS1GJAJEktBkSS1GJAJEktBkSS\n1GJAJEktBkSS1GJAJEktBkSS1GJAJEktBkSS1GJAJEktBkSS1GJAJEktBkSS1DIoIEl2JXk5yWKS\nB85x/itJnk9yLMnTSa6b/KiSpFmyZkCSbAIeBnYDO4E7k+xcsexV4E+q6g+AB4H9kx5UkjRbhlyB\n3AgsVtWJqnoPOAjsGV9QVU9X1a9Gh88AWyc7piRp1gwJyBbg5NjxqdFtq/ka8NNznUiyN8lCkoUz\nZ84Mn1KSNHMm+iB6ki+yHJD7z3W+qvZX1XxVzc/NzU3yriVJG2zzgDWngW1jx1tHt/0fST4HPAbs\nrqq3JzOeJGlWDbkCOQLsSLI9yWXAHcCh8QVJrgaeAL5aVa9MfkxJ0qxZ8wqkqpaS3Ac8CWwCDlTV\n8ST3js7vA74FfBx4JAnAUlXNr9/YkqRpS1VN5Y7n5+drYWFhKvctSVqW5Gj3G36fiS5JajEgkqQW\nAyJJajEgkqQWAyJJajEgkqQWAyJJajEgkqQWAyJJajEgkqQWAyJJajEgkqQWAyJJajEgkqQWAyJJ\najEgkqQWAyJJajEgkqQWAyJJajEgkqQWAyJJajEgkqQWAyJJajEgkqQWAyJJajEgkqQWAyJJajEg\nkqQWAyJJajEgkqQWAyJJajEgkqQWAyJJajEgkqQWAyJJahkUkCS7krycZDHJA+c4nyTfG51/Psn1\nkx9VkjRL1gxIkk3Aw8BuYCdwZ5KdK5btBnaM3vYCj054TknSjBlyBXIjsFhVJ6rqPeAgsGfFmj3A\n92vZM8AVST414VklSTNk84A1W4CTY8engC8MWLMFeGN8UZK9LF+hAPxXkhfOa9pLx1XAW9MeYka5\nN6tzb1bn3qzuM90PHBKQiamq/cB+gCQLVTW/kfd/oXBvVuferM69WZ17s7okC92PHfIjrNPAtrHj\nraPbzneNJOkiMiQgR4AdSbYnuQy4Azi0Ys0h4O7Rb2PdBPy6qt5Y+YkkSRePNX+EVVVLSe4DngQ2\nAQeq6niSe0fn9wGHgduAReA3wD0D7nt/e+qLn3uzOvdmde7N6tyb1bX3JlU1yUEkSZcIn4kuSWox\nIJKklnUPiC+DsroBe/OV0Z4cS/J0kuumMec0rLU3Y+s+n2Qpye0bOd80DdmbJLckeTbJ8SS/2OgZ\np2XAv6mPJflJkudGezPk8doLXpIDSd5c7bl37a/DVbVubyw/6P7vwO8BlwHPATtXrLkN+CkQ4Cbg\nl+s506y8DdybPwKuHL2/270557p/ZvmXOG6f9tyzsjfAFcCLwNWj409Me+4Z2pu/Br4zen8OeAe4\nbNqzb8De/DFwPfDCKudbX4fX+wrEl0FZ3Zp7U1VPV9WvRofPsPz8mkvBkL83AN8Afgi8uZHDTdmQ\nvbkLeKKqXgOoqktlf4bsTQEfTRLgIywHZGljx9x4VfUUy3/W1bS+Dq93QFZ7iZPzXXMxOt8/99dY\n/g7hUrDm3iTZAnyZS++FO4f8vbkGuDLJz5McTXL3hk03XUP25iHgs8DrwDHgm1V1dmPGm2mtr8Mb\n+lIm6knyRZYDcvO0Z5kh3wXur6qzy99Masxm4AbgS8BvA/+a5JmqemW6Y82EW4FngT8Ffh/4pyT/\nUlX/Od2xLkzrHRBfBmV1g/7cST4HPAbsrqq3N2i2aRuyN/PAwVE8rgJuS7JUVT/amBGnZsjenALe\nrqp3gXeTPAVcB1zsARmyN/cAf1vLP/hfTPIqcC3wbxsz4sxqfR1e7x9h+TIoq1tzb5JcDTwBfPUS\n++5xzb2pqu1V9emq+jTw98BfXQLxgGH/pn4M3Jxkc5IPs/zq2S9t8JzTMGRvXmP5yowkn2T5lWhP\nbOiUs6n1dXhdr0Bq/V4G5YI3cG++BXwceGT0nfZSXQKvKDpwby5JQ/amql5K8jPgeeAs8FhVXfT/\ndcLAvzcPAo8nOcbybxzdX1UX/cu8J/kBcAtwVZJTwLeBD8EH+zrsS5lIklp8JrokqcWASJJaDIgk\nqcWASJJaDIgkqcWASJJaDIgkqeW/AQ1Yo2kAGAaSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2696f1e0978>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_model_v1_2 = build_model_reg(dropout_rate=0.2, reg_l2=5/len(Y_train))\n",
    "\n",
    "learning_rate = 1e-3;   print('learning_rate',learning_rate)\n",
    "adam_opt  = Adam(lr=learning_rate, decay=decay)\n",
    "train_model_v1_2.compile(loss='categorical_crossentropy', optimizer=adam_opt, metrics=['categorical_accuracy'])\n",
    "######################## Checkpoints ###############################\n",
    "CkptFold_det = ['{:.2e}'.format(learning_rate), Ckpt_Mod_Weights_fold, '1e-3/'] # [str_int, Ckpt_Mod_Weights_Fold, LR_Fold]\n",
    "ModelCheckpoint_det = ['val_categorical_accuracy', 1, False, True, 1] # [monitor_modes, verbose_ckpt, save_best_only, save_weights_only, period]\n",
    "ckpt = ckpt_saving(CkptFold_det, ModelCheckpoint_det, save_all=True)\n",
    "csv_log = CSVLogger(Weights_path+'v1_2_Trg_log_{:.2e}.txt'.format(learning_rate), '\\t', append=False)\n",
    "plot_path = plot_path_dir+'v1_2_{:.2e}.png'.format(learning_rate)\n",
    "############################### Training #############################\n",
    "try:\n",
    "    history_v1_2 = train_model_v1_2.fit(X_train,Y_train, batch_size=batch_size, epochs=30, verbose=1, \n",
    "                                         validation_split = 0., validation_data=(X_test,Y_test), # None (X_val,Y_val)\n",
    "                                         callbacks=[reduce_lr, csv_log, ckpt]) \n",
    "    plt.figure(figsize=(14,4))\n",
    "    plt.subplot(1,2,1);plot_loss(history_v1_2, metric_list=['loss','val_loss'], title='Loss', legend_list=['train', 'val'],  plot_path=plot_path)\n",
    "    plt.subplot(1,2,2);plot_loss(history_v1_2, metric_list=['categorical_accuracy','val_categorical_accuracy'], title='Accuracy', legend_list=['train', 'val'], plot_path=plot_path)\n",
    "    plt.tight_layout()\n",
    "except KeyboardInterrupt: \n",
    "    print('\\n\\nKeyboardInterrupt !!!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+gAAAEYCAYAAADPrtzUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XecXPV97//XZ9rOVrVdSaveQRISwohminFsbHpzAOOO\nC+Yax9i5Ti5JnBvnZzs3TuIU55oQ4ktiO8a2XGQwCAvTbYolASoIEEhCYlddq12ttszslM/vjzMr\nrYTKlhnN7O77+Xicx5nTvvOZXdDZz3y/5/M1d0dEREREREREiitU7ABERERERERERAm6iIiIiIiI\nSElQgi4iIiIiIiJSApSgi4iIiIiIiJQAJegiIiIiIiIiJUAJuoiIiIiIiEgJUIIuIiIiIiIiUgKU\noIsMU2a2xczeW+w4RERE5HBm9qSZNZtZWbFjEZGTSwm6iIiIiEiJMLNpwIWAA1efxPeNnKz3EpFj\nU4IuIocxs8+Y2UYz22dmD5jZhNx+M7N/MrPdZtZqZuvM7LTcscvN7BUzO2Bm28zsy8X9FCIiIoPW\nx4Dngf8CPt6908zKzexbZrbVzPab2e/MrDx37AIze9bMWsyswcw+kdv/pJl9ukcbnzCz3/XYdjO7\n3czeAN7I7fuXXButZvaCmV3Y4/ywmf25mW3K3fNfMLPJZvYdM/tWzw+R+xviS4X4AYkMZUrQReQg\nM/sD4P8ANwL1wFbgx7nD7wMuAuYAI3LnNOWO/T/gs+5eDZwGPH4SwxYRERlKPgb8MLe838zG5fb/\nA3Am8E5gNPCnQNbMpgIPA/8K1AGLgNV9eL9rgXOAebntlbk2RgP3AT81s3ju2B8DNwOXAzXAJ4EO\n4HvAzWYWAjCzWuC9uetFpA+UoItITx8G7nX3F909CfwZcF5uuF0KqAZOBczdX3X3HbnrUsA8M6tx\n92Z3f7EIsYuIiAxqZnYBMBVY4u4vAJuAD+US308Cd7j7NnfPuPuzuXv1h4BH3f1H7p5y9yZ370uC\n/n/cfZ+7dwK4+3/n2ki7+7eAMuCU3LmfBr7i7hs8sCZ37gpgP/Ce3HkfBJ50910D/JGIDDtK0EWk\npwkEveYAuHsbQS/5RHd/HPi/wHeA3WZ2j5nV5E79AMG36VvN7CkzO+8kxy0iIjIUfBx4xN335rbv\ny+2rBeIECfuRJh9jf2819Nwwsy+b2au5YfQtBKPmanvxXt8DPpJ7/RHgBwOISWTYUoIuIj1tJ/jm\nHgAzqwTGANsA3P3b7n4mwTC4OcCf5PavdPdrgLHAL4ElJzluERGRQS33PPmNwLvMbKeZ7QS+BJxO\n8NhZAph5lEsbjrEfoB2o6LE9/ijneI8YLiQYOn8jMMrdRxL0jFsv3uu/gWvM7HRgLsHfAyLSR0rQ\nRYa3qJnFuxfgR8AtZrYoN7XL3wC/d/ctZnaWmZ1jZlGCG36C4Nm3mJl92MxGuHsKaAWyRftEIiIi\ng9O1QIbgS/BFuWUu8FuC59LvBf7RzCbkirWdl7tX/xB4r5ndaGYRMxtjZotyba4GrjezCjObBXzq\nBDFUA2lgDxAxs/9N8Kx5t+8CXzOz2bnisQvNbAyAuzcSPL/+A+Dn3UPmRaRvlKCLDG/LgM4ey8XA\nXwI/B3YQfEv+wdy5NcB/AM0Ew+CbgL/PHfsosMXMWoHbCJ5lFxERkd77OPCf7v6Wu+/sXggeL/sw\ncCewjiAJ3gd8Ewi5+1sEj5n9z9z+1QS97gD/BHQBuwiGoP/wBDEsB34NvE5wr09w+BD4fyQYJfcI\nwRfy/w8o73H8e8ACNLxdpN/M3U98loiIiIiIyHGY2UUEQ92nupIMkX5RD7qIiIiIiAxI7hG4O4Dv\nKjkX6T8l6CIiIiIi0m9mNhdoIShm989FDkdkUNMQdxEREREREZESoB50ERERERERkRIQKXYA/VFb\nW+vTpk0rdhgiIiIF88ILL+x197pix3Ey6L4uIiJDXW/v64MyQZ82bRqrVq0qdhgiIiIFY2Zbix3D\nyaL7uoiIDHW9va9riLuIiIiIiIhICVCCLiIiIiIiIlIClKCLiIiIiIiIlIBB+Qz60aRSKRobG0kk\nEsUOpaDi8TiTJk0iGo0WOxQREZGC0X1dRESGoyGToDc2NlJdXc20adMws2KHUxDuTlNTE42NjUyf\nPr3Y4YiIiBSM7usiIjIcDZkh7olEgjFjxgzZmziAmTFmzJgh35sgIiLFZWb3mtluM3v5GMfNzL5t\nZhvNbK2ZvaPHsUvNbEPu2J39jUH3dRERGY6GTIIODOmbeLfh8BlFRKTo/gu49DjHLwNm55ZbgX8D\nMLMw8J3c8XnAzWY2r79BDId73nD4jCIi0ntDZoh7f3V2ZehMpRldWVbsUEREREqCuz9tZtOOc8o1\nwPfd3YHnzWykmdUD04CN7r4ZwMx+nDv3lcJGLCKSZ+6QTkBXOyQPgGchFDm0hKMQCh++z0JwrC/d\nshlIdUI6CenudQJSiWCd7sRTCTyVgEwSi5RhZdUQq4RYFfR8Ha2AUD/7Wd0hm+4RSwIyXRCOQSQO\n0XiwDkWO/VlOJJuFVHvuZ9cGXbml+3W696OG0hmnvSvNgUQagDFVMcqj4b7FY2GIlEG0PFhH4oeW\naPzw7Ui8/z/bPBn2CXprIsWu1gQjy2OEQv3/FrulpYX77ruPz33uc3267vLLL+e+++5j5MiR/X5v\nERGRk2wi0NBjuzG372j7zzlaA2Z2K0HvO1OmTClMlAOg+3o/ZDOw+Ql44XvQsALKR0JFLVSOya1r\nj75dMQbCA/iT1D1Y53M0gnuQlCVaoLMFEvsPvU62wrjTYPpF+X3Po8Ww9Zng59nZHCRVR10ykEkd\net3zWCgMoWgugTwimQxHDt/uPh6rhGkXwuz3waipJwwzk3W60lnMIN6XxCnZBlt+B3s3QLjsUOL0\ntoTp7YlVSyrEgWSWUWVOpXVhmWSQbPZMOg8uycMT41Rn8Lvtas8lju1H2c4lkp7p868tTYSMhcl4\niBRhIqSJkSLKiduy3HIiWYwEZXRYOQnK6QyVk7BykqE45SGnMpyiIpSmjBRldBHJdhHOJrHun4ln\nT/wmFoJI+VF+L2V4JE4mVEaGMJbqwFJtWFf7wXUo1d6LT9E7EWBEbjlZ0hYlbTEaai9g9ueWnMR3\nDgz7BL0sEnxDkkxnKI/1/8fR0tLCXXfd9bYbeTqdJhI5drvLli3r93uKiIgMVu5+D3APwOLFi73I\n4byN7ut9sH8bvPTfwbL/LSgfDbMvCRKdjibY9Qp07A2SzGOJjwwS9lhVkBR1J5qZ1NuTzmwGsqlD\n293JRrhnIlF2zOTisP3Z1NsT8ERu+0RJzLjT4NzPwYI/DNrLg0zWaWrtoHPd/Yx86S5G7FtHZ2QE\n++MTyVoED0VwC+OhMG5lkNsmGhzrTrY9FA56DT2DZ9J4Ng2ZHj/DVBpL5n6u3kUo24F5BsumqUi3\nUPvK/QA0RKayKnYWz0cWs9pPIZE1kqksXZksyVSGrkyWVObQ/761VTEmj65gyugKpo6uOPR6TCVj\nq2KE9qyHjY/CxsfgreeDn38/jMwt/ZWNxMlEqshEK0mHy+kKV9IVriAZqSURLSdRVU6HxemgnHaP\n0+ZlHOhyOhJJOhJJkskk5hkiZAiTIUKWMBnKQlmqY1AdM6qiUBlxLBwlEyojHS4jE4oFiW2ojHQ4\nRiYUJxvu3hcnGy4jE46RtTIsmySc6iCcbiecaieSbieS6SCS7iCWCV7HMh3EMp3Esu2UZTupzLSS\nSIXYnQjTlomRpJIkURIeI0mUUDRONF5BWbyS8vIKyisrqa6sorKignRXkq5kB6lEB+muTjJdnWS7\nOsl29/AnE1gmSTiTJJzdTxkpImTopIw2j9PBONp8Ku2U00E8t6/HOvezbCdOktjB79W6mcGI8hij\nKqOMKo8xsjLKqIpYsFRGGVkezHTxVnMnDU0dNDR38FZTB/s7D/03FA4ZE0eWM3l0BZNHlzN5dCVT\nRldQHnaa9rfS3HqA/a2tHGhr40DbATo6OujoaCOcSVJGijhdwZcalmJULEM8OpfZA/jvrL+UoEeC\nb/qS6Szlsf63c+edd7Jp0yYWLVpENBolHo8zatQoXnvtNV5//XWuvfZaGhoaSCQS3HHHHdx6660A\nTJs2jVWrVtHW1sZll13GBRdcwLPPPsvEiRO5//77KS8vz8fHFBERyadtwOQe25Ny+6LH2D/oDKn7\neiYFzVtgxKSgJzJfbb6+HF78XpBweRZmXAyX/DWcesXRE9ZMOkjSO/ZC+94e66ZD28m2owwdPspQ\n4lA4d15u27OHeksPGzbcYzhx8kCP/YmgJzUcDb4cKB8Z9OSPnhm87t4XH9HjdW47VgUblsHzd8H9\nn4NHvwpnfRoWfxKq6g77yO5OVyZLIpUlkcrQ1NbFrgMJdrcm2NWaZFduvftAgpb9+7m44zd8MryM\naaFdvJkdx99lPsnPEheRbBvAH6nHEA4Z0bARDYWIRkLB63CIaDzEDNvBO7OrODu1kqs6lnIdP6M9\nVM2GyrPYMPadvDnynWTioyiLhCiLhIlFQmTdaWzu4K19HbywtZlfrdnOSG/lgtA63hVey4WhdYy1\nFgB2xWeya/KHSE17N5nxi9i+r43tTc3satrPnuYWmvYfINvVSZkFPcDllmJ8JUyohHEVUBeHeATa\nMhEOpCO0psO0dIVo6QqzrytEUzLE3oTRkY0elqAmiJEgRrYXZbjCIaMiGiYeCxOPhhhZHqO2NkZd\ndRm1VWUH192v66rKqCmPlExdifZkmp2tCXa0JNixv5P9+xPs2J9g5/5OduxPsGNH4rDk9khVZRFq\n4hFqyqNU10SoiUepKY8e3FcTj1JRFsYwHD+YcMeBuDujCAaBuDvOoYEuDkRCxujKGGOqYtRWlTGm\nMsbIihjhfoxmbm7vYvPedt7c287mPW28ubedNXva+eWWdrrSCaDpsPMjoRrG1YylfkSccVPi1NfE\nGT8iWOpHxBk/opyx1WVEw8Ub5m5+5NcXg8DixYt91apVh+179dVXmTt3LgB//av1vLK9tdfttSfT\nxCKh4/4i5k2o4a+umn/M41u2bOHKK6/k5Zdf5sknn+SKK67g5ZdfPjhtyr59+xg9ejSdnZ2cddZZ\nPPXUU4wZM+awG/msWbNYtWoVixYt4sYbb+Tqq6/mIx/5yNveq+dnFRGRocnMXnD3xUV8/2nAg+5+\n2lGOXQF8HricYAj7t939bDOLAK8D7yFIzFcCH3L39cd7r3zf13tjSN/X00nY9kIwPHrLM8Fw81R7\n0KNadyrUnw4TFgXrcadBWVXv227aBC/9AFbfB227oLoeFn0Y3vFRGDUtf5+hD9ydrEPI+ld0L5t1\nOlIZOpJp2pJpOroyuXWatuTh+9uTadq7gtfJrgzTD6zkPS0/54zkCrqI8pvou/gRV/BKdjKJVIZE\nKkP2OH9qj66MMasywc32CJe03U9VZj97Rixg27zPYKdeybiRlYypihEJGVkPetiz7qSzHrzOBq+z\nHmwfXDw4FgkHiXcsHDr4OhoO/ubtdTKU2A+bnoA3HgmW9j3B8OdJZ8Oc98Hs98O4+UEXaCYFjSth\n42NkNz6G7ViN4SSiI9lcfRYrw2fwm675rG4ppy2ZPuxtIiFj8ugKpo6pYNqYysPWk0ZVEIv0LWFy\ndw4k07S0p2ju6GJfRxctHV2kMk5FLEx5NFjiPV6Xx3JLNFzUBO1k6U7i9x5IUlnWnYRHqCqLEBnk\nnz+Tdba3dLJ5bzvJVIb6EeWMHxFnTOXAHmseiN7e14d9DzoE/5gf7x/P/jj77LMPm9P029/+NkuX\nLgWgoaGBN954gzFjxhx2zfTp01m0aBEAZ555Jlu2bMlvUCIiIr1gZj8CLgZqzawR+CuC3nHc/W5g\nGUFyvhHoAG7JHUub2eeB5UAYuPdEyflgUdL39VRnkBRteSZIyhtXHirCNHY+nPFhGL8w6EXfsQY2\n/gbW3Je72KB2NtTnEvb606F+YdBTfLD9BLz6q6C3fMtvcQvTNeMS9l/8QfaMv4j2lNG+O017w3Y6\nkkFy25XJEguHiEfDlEWOWEdDb9+XW7tDS2cXze0pWjq7aOlIBcvB1100d6TY3xEkXS2dweuuzKHh\n6GYQMsPIre3Qvu79ZhAKGV3pLB1dvX/OOB4NURmLUB4LE4+G2Rqdz/MjFzKdbVzRcT+XtD3CFf4o\nG6vPYuX4m9le+07isejBzzemMsbYmjjjasqoS22nbOW/wUs/DHr551wG53+BuinnUXeULxrCRr96\nGAcsPgLmXxss2SxsfwneWA6v/xoe+/+CpWYSjJ0LDb8PntG3MKFJZ8G7/xxmvof4hEXMC4WZB3yc\nIHlu7kjx1r4O2hJppoyuYMLIeF6TQjMLEs54lCljKvLW7lBSWRZhZl0VM+v68CXdIBHOfeEzefTg\n+90PyQT9eN+IH82be9tJZ7LMHledtxgqKysPvn7yySd59NFHee6556ioqODiiy8+6pynZWWHhoOF\nw2E6OzvzFo+IiEhvufvNJzjuwO3HOLaMIIHPm77e1wuhpO7rybYgEeruId/2QvAsr4Vg/AJY/CmY\n+s5gqRj99uvd4cBO2LE6SNh3rAnaWneoGFJXzVQ2R2azpSPKOxO/pYY2Gn0sP87cyJL0u9i9fhSs\nB3g+P5+pF7qHGY+siDKyIsrMuipGVUYZUR5UdXaCnnRyPerZ3NDarPvBobYH9+fWsXCIirIIVWVh\nKmJBz2FlWYTKWDhYlwXriliw7/gJ5A3QsQ9e+E9mrfgPZr3xx9A8B879H7DwgxDLJQrbXoDffBte\nfSAY1XD6TXDeH8HYU0/CT3GAQiGYdGawvPvPoXVH8IXP68th7xtw2vUw8z1BAb3yYz8lbhYMcR5d\nmf+h+yKD3ZBM0PuqLBKiPZnG3fv93Eh1dTUHDhw46rH9+/czatQoKioqeO2113j++ZN3MxMREZG+\nK8n7+sZH4el/CHrIs+kguZtwBpz3OZh6Pkw59/Ce72Mxg5r6YDnlsoO7t259k5dWPE3TxhXUN7/O\ngtAa/sBaWFt9AS+OuZodo8+iKh7jltihhLZnElsZy72ORYhFQnSlsyTTWZLpDInU4etkKksit06m\ng+ezk+mgF3xkRZRRFdGDBaO6k/I+VQgvlorRcOH/DBLuV34Jz30HHvwSPPY1WPSh4MuQLb+FshHw\nzi/AObcFv4fBqqYe3vGxYBGRvFCCTpCgZ91JZbLEIv37x3/MmDGcf/75nHbaaZSXlzNu3LiDxy69\n9FLuvvtu5s6dyymnnMK5556br9BFRESkAErqvr5rPTzyFdj0ePCc9/l3BAn55HP69vz4UTQ2d/Dg\n2h38as121m9vxayWs6d9hKvePYGK08YTq4yx2Iz+FEOozE9h88EpEoOFN8KCG+Ct54JE/bnvQM0E\neN83goQ2XlPsKEWkBA3JInF91ZZMs3lPG9NrK6mOR/MRYkGpSJyIyNBX7CJxJ1O+7+uDzTE/a+sO\neOIbsPqHUFYD7/rToFr4AKf02t2a4KF1QVL+4ltBVe1Fk0dy1ekTuGJBPeNHxAfUvhxDxz4oqw4q\nx4vIsKMicX1waC70LPl7Cl1ERESkH7ra4dl/hWf+JaiKfe7ngmHTR3uevJea27t4+OWd/GrNdp5/\nswl3mFtfw59eegpXLpigIlonwwB+fyIyfChBJ5jWIRwykqneV/EUERERyatsJpi+7PGvQ9tOmHct\nvPevYPSMfjW3r72LR9bv5KF1O3huUxPprDOjtpIv/MFsrjq9nllj1S0hIlJqlKATVJIsi4QPFicR\nEREROak2PQ6P/CXsehkmnQU3fh+mnNPnZva2JVm+ficPr9vJc5ubyGSdKaMr+PSFM7jq9Hrm1df0\nuyCuiIgUnhL0nLJIiLZkuthhiIiIyHCS6oS2PfDQdTByKtzwX0HPeR+S6D0Hkvx6/U4eXreD5zc3\nkXWYXlvJbe+awWWn1TN/gpJyEZHBQgl6Tlk0RHNHlkw2Szh0vDkuRURERAbAPZi3/MBO6GiCTDKo\n7H32Z3pdAG53a4Jfr9/JQ2t3sGLLPtxhRl0lt797FpcvqOfU8dVKykVEBiEl6DlluenVkuksFTEl\n6CIiItIL2Qykk+CZYG7ybCb3usfi6R6vc2scMKisg+oYzL/khG+VyTo/WvEWD6zezsqtQVI+a2wV\nf/QHs7liQT1zxlUpKRcRGeSUoOccrOSeylIRK/z7VVVV0dbWVvg3EhERkcLIpGDPa5BNUzX7fNre\neObw46EIWBhCuSUcC/Z1b8dHQCQO2w+c8K0SqQx/vGQ1y9btZM64Ku54T5CUzx6nQm8iIkOJEvSc\nWCSEYSTTquQuIiIiJ+AOLW8FveEjp4KFoO7UQwm5hfr0HPnx7O9Mcev3V/H7N/fxlSvm8ukL+1fV\nXURESp8S9JyQGbFIqN+V3O+8804mT57M7bffDsBXv/pVIpEITzzxBM3NzaRSKb7+9a9zzTXX5DNs\nERERKYAT3teTCb7+5c9wzQ0fPjS/dbQ873Hsak3w8XtXsGlPG//ywUVcs2hi3t9DRERKx9BM0B++\nE3au6/NlU1MZsu4QO8qPZfwCuOxvj3ntTTfdxBe/+MWDN/IlS5awfPlyvvCFL1BTU8PevXs599xz\nufrqq/V8mIiISF/0875+XAO5r1eUsfe15zj3qo9z9cfvoFB39Y272/j4vSto6ejiPz9xNhfMri3Q\nO4mISKkYmgl6P1koV78Fx/p4uz3jjDPYvXs327dvZ8+ePYwaNYrx48fzpS99iaeffppQKMS2bdvY\ntWsX48ePL9AnEBERkXw45n39i1/k6ScfJYSxbedudu3eXZD7+otvNfPJ/1pJJGT85LPncdrEEXl/\nDxERKT1DM0E/zjfix9Pe3kVjcwenjKumLBru8/U33HADP/vZz9i5cyc33XQTP/zhD9mzZw8vvPAC\n0WiUadOmkUgk+hWbiIjIsNXP+/pAHfW+vrORF5b9gGjdTKbNO7Mg9/XHXt3F7fe9yPiaON/75NlM\nHVOZ9/cQEZHSNDQT9H46WMk9ne1Xgn7TTTfxmc98hr179/LUU0+xZMkSxo4dSzQa5YknnmDr1q35\nDllEREQK5G339R/9kLEjyolW1/LE82sKcl9fsrKBP1u6jnn1NfznLWdRW9W7edFFRGRoUILew6EE\nPQNE+3z9/PnzOXDgABMnTqS+vp4Pf/jDXHXVVSxYsIDFixdz6qmn5jliERERKZTD7uvjx/Hhy87n\nqvt+wIJ3Xc3ixWfl9b7u7nzniY38wyOvc+HsWu7+yJlUlunPNBGR4Ub/8vcQCYeIhEIkU/2r5A6w\nbt2hIja1tbU899xzRz1Pc6CLiIiUvoP39dYd1I6I89xvn4LykW87byD3dXf4qwfW8/3ntnLdGRP5\n5gcWEst1GoiIyPCiBP0IZdH+T7UmIiIiQ1BXO7TthPLRR03OByKbdfa1d/H95xq59aIZ3HnpqYRC\nmu1FRGS4UoJ+hLJIiNbOVLHDEBERkVKQzUDzVghFYUR+5yBPZ7NsbeqgM5XhK1fM5dMXzshr+yIi\nMvgMqQTd3Qc8x3hZJEw620U6kyUSLr3hZe5e7BBEREROinzc1wfswA7IJGHMLAjl78+mVDrLm03t\nJFIZRlfGuOR0JeciIgKll4H2Uzwep6mpacAJbFn0UCX3UuPuNDU1EY/Hix2KiIhIQeXrvj4gyQPQ\nvgcq66CsOm/NZrLOpr1tJFMZRoWSjK7RNGoiIhIoeA+6mV0K/AsQBr7r7kedzNTMzgKeAz7o7j/r\n6/tMmjSJxsZG9uzZM6B409ksu/Yn6dobLcnqqfF4nEmTJhU7DBERkYLK13293zwLB3YCBtUx2H4g\nb023dqZoTaSpqy4jXVWh+7qIiBxU0AzUzMLAd4BLgEZgpZk94O6vHOW8bwKP9Pe9otEo06dPH0i4\nQPCt9h/+71/zsfOm8hdXzB1weyIiItJ3+bqv99vS22DtEvjUb2DS/Lw1u2N/J9f9w5NcMm88/3pz\n/toVEZGhodBD3M8GNrr7ZnfvAn4MXHOU8/4I+Dmwu8DxnFA4ZEyvrWTznvZihyIiIiLF8OqvYM2P\n4KIvw6Qz89r03y/fQNbhT99/Sl7bFRGRoaHQCfpEoKHHdmNu30FmNhG4Dvi34zVkZrea2SozW1Xo\n4W4z66rYtEfzlIuIiAw7bXvgV1+E+tPhoj/Ja9Mvb9vP0pe2ccv505g8uiKvbYuIyNBQCkXi/hn4\nX+5+3Kps7n6Puy9298V1dXUFDWhmXSVv7esgmc4U9H1ERESkhLjDr+4IisNddw+Eo3ls2vnGQ68y\nsjzK7e+elbd2RURkaCl0FbRtwOQe25Ny+3paDPw4N41KLXC5maXd/ZcFju2YZo6tIuuwtamDOePy\nV7VVREREStjq+2DDQ/C+b8DYU/Pa9GOv7ua5zU389dXzqYnnL/EXEZGhpdA96CuB2WY23cxiwAeB\nB3qe4O7T3X2au08DfgZ8rpjJOQRD3AE27dYwdxERkWGh5S14+H/B1Avg3M/ltelUJsvfPPwqM+oq\n+dA5U/LatoiIDC0F7UF397SZfR5YTjDN2r3uvt7Mbssdv7uQ799f02uD+Uj1HLqIiMgw8dCXAYdr\n74JQfvsvfrTiLTbvaec/PraYaLgUni4UEZFSVfCJvt19GbDsiH1HTczd/ROFjqc3KssiTBgRZ5Mq\nuYuIiAx9+xvhjeVw8Z/BqKl5bbo1keKfH32Dc2eM5r1zx+a1bRERGXr0Ne4xzBxbxWb1oIuIiAx9\n634WrBfemPem73piE/vau/iLy+eRq7cjIiJyTErQjyGYaq0ddy92KCIiIlJIa5fApLNh9Iy8Ntuw\nr4N7n3mT68+YyIJJI/LatoiIDE1K0I9hRl0lbck0uw8kix2KiIiIFMrOl2H3+oL0nv/98g0Y8OX3\nn5L3tkVEZGhSgn4MquQuIiKDnZmNKXYMJW/dEghFYP51eW12dUMLD6zZzmcunMGEkeV5bVtERIYu\nJejHcDBB13PoIiIyeD1vZj81s8tND0C/XTYbPH8+8z1QWZu3Zt2dbzz0CrVVMW67eGbe2hURkaFP\nCfoxjKukTV9RAAAgAElEQVQpozIWViV3EREZzOYA9wAfBd4ws78xszlFjql0bH0GWrflfXj78vU7\nWbmlmS9dMoeqsoJPmCMiIkOIEvRjMDNmjq1SD7qIiAxaHviNu98MfAb4OLDCzJ4ys/OOdZ2ZXWpm\nG8xso5ndeZTjo8xsqZmtNbMVZnZaj2NbzGydma02s1UF+WD5sm4JxKrglMvz1mRXOsvfPvwas8dW\ncdPiyXlrV0REhgd9rXscM+uqWPHmvmKHISIi0i+5Z9A/QtCDvgv4I+ABYBHwU2D6Ua4JA98BLgEa\ngZVm9oC7v9LjtD8HVrv7dWZ2au789/Q4/m5331uAj5Q/qQSsvx/mXgWxirw1+9/Pb2VLUwf/ectZ\nRMLqBxERkb7RneM4ZtZVsq2lk46udLFDERER6Y/ngBrgWne/wt1/4e5pd18F3H2Ma84GNrr7Znfv\nAn4MXHPEOfOAxwHc/TVgmpmNK8xHKJA3HoHkflhwQ96a3N+R4tuPv8EFs2q5eE5d3toVEZHhQwn6\ncczIFYrbrOfQRURkcDrF3b/m7o1HHnD3bx7jmolAQ4/txty+ntYA1wOY2dnAVGBSd9PAo2b2gpnd\neqzAzOxWM1tlZqv27NnTu0+TT2t/AlXjYPq78tbkvz7+Bvs7U/z55XNRTT4REekPJejHoUruIiIy\nyD1iZiO7N3LPji/PQ7t/C4w0s9UEw+ZfAjK5Yxe4+yLgMuB2M7voaA24+z3uvtjdF9fVneTe5s7m\noAf9tA9AOD9P+21taud7z23hhjMnMW9CTV7aFBGR4UfPoB/H1DEVhAxVchcRkcGqzt1bujfcvdnM\nxp7gmm1Az+pmk3L7DnL3VuAWgNz0bW8Cm3PHtuXWu81sKcGQ+acH+Dny65X7IdOV1+rt3/z1a0RC\nIf7n+07JW5siIjL8qAf9OOLRMJNHV6gHXUREBquMmU3p3jCzqQRD0I9nJTDbzKabWQz4IEFhuYPM\nbGTuGMCngafdvdXMKs2sOndOJfA+4OU8fZb8WftTGDMb6hflpbkXtu5j2bqdfPZdMxhXE89LmyIi\nMjypB/0EZtZVsWm3EnQRERmU/gL4nZk9BRhwIXDM58IB3D1tZp8HlgNh4F53X29mt+WO3w3MBb5n\nZg6sBz6Vu3wcsDT3/HUEuM/df53/jzUALQ2w9Xfw7r+APDwn7u58/aFXGVtdxq0XzchDgCIiMpwp\nQT+BmXWVPLNxL9msEwqp4IuIiAwe7v5rM3sHcG5u1xd7M/2Zuy8Dlh2x7+4er58D5hzlus3A6QMK\nutBe/lmwzlP19gfX7uClt1r4uw8spCKmP6tERGRgNMT9BGbWVZFMZ9nW0lnsUERERPojA+wGWoF5\nxyraNmysXQKTzobRb5sCvs+yWecfHtnAqeOr+cCZk058gYiIyAkoQT+BmWNVyV1ERAYnM/s0QYG2\n5cBf59ZfLWZMRbXzZdj9St6Kw/3+zX1sbergf1w8k7BG2YmISB4oQT+BGbWVgCq5i4jIoHQHcBaw\n1d3fDZwBtBz/kiFs7U8gFIH51+eluaUvNVIZC/O+eePz0p6IiIgS9BMYXRljZEVUPegiIjIYJdw9\nAWBmZe7+GjA85wHLZmHdz2DWe6FyzICbS6QyPLxuJ5eeVk95LJyHAEVERJSgn5CZqZK7iIgMVo1m\nNhL4JfAbM7sf2FrkmIpj6+/gwPa8FYd79NVdHEimue6MiXlpT0REBFTFvVdm1lXy+Gt7ih2GiIhI\nn7j7dbmXXzWzJ4ARQGlNe3ayrF0CsSo45fK8NLf0xW2MqynjvJkD740XERHpph70XphZV8XetiT7\nO1PFDkVERKRXzCxsZq91b7v7U+7+gLt3FTOuokgl4JX7Ye5VEKsYcHNNbUmeen0P1y6aqOJwIiKS\nV0rQe2FmXVDJfbOeQxcRkUHC3TPABjObUuxYiu6N5ZBszVv19l+t2U4661z3Dg1vFxGR/NIQ9144\nNNVaO2dMGVXkaERERHptFLDezFYAB6cjcferixdSEaxdAlXjYPq78tLc0pe2Mbe+hlPH1+SlPRER\nkW5K0Hth8qhyomFTJXcRERls/rLYARRdZzO88Qic9RkIDbza+qY9baxp3M9fXD43D8GJiIgcTgl6\nL0TCIaaOqVQldxERGVTc/alix1B0r9wPmS5YmJ/q7Utf3EbI4JpFE/LSnoiISE9K0HtpZl0lG5Wg\ni4jIIGJmBwDPbcaAKNDu7sNnbPbaJVA7B+oXDbipbNZZ+tI2zp9Vy9iaeB6CExEROZyKxPXSzLoq\n3trXQSqTLXYoIiIiveLu1e5ek0vIy4EPAHcVOayTp6UBtj4DC24EG3i19ZVb9rGtpZPrVRxOREQK\nRAl6L82sqyKVcRr2dRQ7FBERkT7zwC+B9xc7lpNm3U+D9YI/zEtzS1/aRkUszPvnj89LeyIiIkfS\nEPde6lnJfUZu2jUREZFSZmbX99gMAYuBRJHCObncg+Htk8+B0dMH3FwileGhdTu4dP54KmL680lE\nRApDd5hemlFXCQTVWy9hXJGjERER6ZWrerxOA1uAa4oTykm262XY8ypc8a28NPf4a7s5kEhz7Rka\n3i4iIoWjBL2XauJRxlaXqZK7iIgMGu5+S7FjKJq1SyAUgXnX5aW5X7y4jbHVZZw/qzYv7YmIiByN\nnkHvgxl1lZoLXUREBg0z+56ZjeyxPcrM7i1mTCdFNgPrfgazLoHKMQNubl97F09u2M01iyYQDg28\n2JyIiMixFDxBN7NLzWyDmW00szuPcvwaM1trZqvNbJWZXVDomPprZl0Vm/a04+4nPllERKT4Frp7\nS/eGuzcDZxQxnpNj6zNwYHve5j5/cO120lnnujMm5aU9ERGRYylogm5mYeA7wGXAPOBmM5t3xGmP\nAae7+yLgk8B3CxnTQMysq2J/Z4p97V3FDkVERKQ3QmY2qnvDzEYzHB5vW/sTiFXDnMvy0twvXtzG\nqeOrmTdh+EwfLyIixVHoHvSzgY3uvtndu4Afc0RxGndv80Nd0pVAyXZP96zkLiIiMgh8C3jOzL5m\nZl8DngX+rsgxFVYqAa88AHOvgljFgJvbvKeN1Q0tXKficCIichIUOkGfCDT02G7M7TuMmV1nZq8B\nDxH0or+Nmd2aGwK/as+ePQUJ9kRm9qjkLiIiUurc/fvA9cCu3HK9u/+guFEV2Ou/hmQrLLwxL839\n8qVtmME1i5Sgi4hI4ZVEkTh3X+rupwLXAl87xjn3uPtid19cV1d3cgPMmTCinHg0pEruIiIyKJjZ\nuUCDu/9fd/+/QKOZnVPsuApq3U+hajxMv2jATbk7S1dv4/yZtYwfEc9DcCIiIsdX6AR9GzC5x/ak\n3L6jcvengRlmVpJzmIRCxozaKvWgi4jIYPFvQM+bVltu39B1xkfhvV+FUHjATa3a2kzDvk4NbxcR\nkZOm0IViVgKzzWw6QWL+QeBDPU8ws1nAJnd3M3sHUAY0FTiufps5too1DS0nPlFERKT4rEedF9w9\na2ZDu0jcKZfmralfvLiN8miYS08bn7c2RUREjqegPejungY+DywHXgWWuPt6M7vNzG7LnfYB4GUz\nW01Q8f0mL+F5zGbUVtLQ3EEilSl2KCIiIiey2cy+YGbR3HIHsLnYQQ0GiVSGh9Zu5/3zx1FZNrS/\n0xARkdJR8DuOuy8Dlh2x7+4er78JfLPQceTLzLFVuMPWpg5OGV9d7HBERESO5zbg28BXCGZJeQy4\ntagRDRJPvLab1kSaazW8XURETiJ9JdxHPSu5K0EXEZFS5u67CR4vkz5a+tI2aqvKuGBWSZbFERGR\nIUoJeh/NqM3Nha5K7iIiUuLMLA58CpgPHCxD7u5HndJUAs3tXTyxYTcfO28akXBJTHgjIiLDhO46\nfVQeCzNxZLkquYuIyGDwA2A88H7gKYLZVA4UNaJB4MF1O0hlXNXbRUTkpFOC3g8zx1axaU97scMQ\nERE5kVnu/pdAu7t/D7gCGNrzoOfB0hcbmTOuivkTaoodioiIDDNK0PthZl0lm/a0UcLF5kVERABS\nuXWLmZ0GjADGFjGekrdlbzsvvtXCdWdMwsyKHY6IiAwzvUrQzewGM6vOvf6Kmf0iN2f5sDSzroqO\nrgw7WxPFDkVEROR47jGzUQRV3B8AXmEQzZxSDEtf2oYZXHvGhGKHIiIiw1Bve9D/0t0PmNkFwHuB\n/wf8W+HCKm0zcpXcN2uYu4iIlDB3/667N7v70+4+w93Huvu/dx83s48XM75S4+78cvU2zpsxhvoR\n5cUOR0REhqHeJuiZ3PoK4B53fwiIFSak0jerLlfJXYXiRERkcLuj2AGUkhffamZrU4eKw4mISNH0\nNkHfZmb/DtwELDOzsj5cO+TUVZdRXRbRVGsiIjLY6SHrHn7x4jbi0RCXLagvdigiIjJM9TbJvhFY\nDrzf3VuA0cCfFCyqEmdmzFAldxERGfyOWu3UzC41sw1mttHM7jzK8VFmttTM1prZilwBul5dW6qS\n6QwPrt3B++aNp6osUuxwRERkmOptgl4PPOTub5jZxcANwIqCRTUIzKqr4tUdrWSzquQuIiKD1tt6\n0M0sDHwHuAyYB9xsZvOOOO3PgdXuvhD4GPAvfbi2JD3x2h72d6Y0vF1ERIqqtwn6z4GMmc0C7gEm\nA/cVLKpB4KI5tTS1d7Fyy75ihyIiItJfzxxl39nARnff7O5dwI+Ba444Zx7wOIC7vwZMM7Nxvby2\nJP3ypW3UVsW4cHZtsUMREZFhrLdjuLLunjaz64F/dfd/NbOXChlYqXvv3HHEoyEeXLuDc2aMKXY4\nIiIiB5nZHx/vuLv/Y279+aMcngg09NhuBM454pw1wPXAb83sbGAqMKmX13bHeCtwK8CUKVOOF+5J\nsaaxhQtn1xEJD9sSOyIiUgJ6exdKmdnNBMPYHsztixYmpMGhsizCH5w6lodf3kE6ky12OCIiIj1V\nn2AZqL8FRprZauCPgJc4NONLr7j7Pe6+2N0X19XV5SGk/kumM+xsTTB5dEVR4xAREeltD/otwG3A\nN9z9TTObDvygcGENDlcunMCydTv5/Zv7OH+WhsSJiEhpcPe/HsDl2wgeZes2KbevZ/utBH8bYGYG\nvAlsBspPdG0p2t6SwB0mj9Lc5yIiUly9StDd/RUz+zIwJ1epdYO7f7OwoZW+d58ylopYmAfX7lCC\nLiIiJcfM4sCngPlAvHu/u3/yOJetBGbnvozfBnwQ+NAR7Y4EOnLPmX8aeNrdW83shNeWosbmDgD1\noIuISNH1aoh7rnL7GwSVWe8CXjeziwoY16BQHgvz3rnj+PXLO0hpmLuIiJSeHwDjgfcDTxH0aB84\n3gXungY+TzC96qvAEndfb2a3mdltudPmAi+b2QaCiu13HO/avH+qPGvY1wkoQRcRkeLr7RD3bwHv\nc/cNAGY2B/gRcGahAhssrlxYzwNrtvPspibeNae4z9CJiIgcYZa732Bm17j798zsPuC3J7rI3ZcB\ny47Yd3eP188Bc3p7balraO4gEjLG18RPfLKIiEgB9bZIXLQ7OQdw99cZ5kXiul00p47qsggPrtle\n7FBERESOlMqtW3KPqI0AxhYxnpLUsK+DCSPLCYfeNi28iIjISdXbBH2VmX3XzC7OLf8BrCpkYCfN\n7tfg9//e78vj0TCXzBvH8vU76UprmLuIiJSUe8xsFPAV4AHgFeDvihtS6Wls7mTyaBWIExGR4utt\ngv4/CG7qX8gtr+T2DX6v/BIe/lPYu7HfTVx5ej2tiTS/fWNPHgMTEREZGHf/rrs3u/vT7j7D3cf2\nHKougcbmDiaN1PPnIiJSfL1K0N096e7/6O7X55Z/cvdkoYM7Kc68BUJRWPkf/W7igll1jCiP8uDa\nHXkMTEREZGDM7G9yFde7t0eZ2deLGVOp6ehKs7etSz3oIiJSEo6boJvZOjNbe6zlZAVZUNXjYP51\n8NIPIXncwrbHFIuEeP/8cfzmlV0kUpk8BygiItJvl7l7S/eGuzcDlxcxnpKzrVkV3EVEpHScqAf9\nSuCq4yxDwzmfha4DsPpH/W7iyoUTaEumeep1DXMXEZGSETazsu4NMysHyo5z/rDTkJsDfdIo9aCL\niEjxHTdBd/etx1u6zzOz5wofagFNWgwTz4QV/w7Z/hV6e+fMMYyujGmYu4iIlJIfAo+Z2afM7FPA\nb4DvFTmmknJwDvRR6kEXEZHi622RuBMZ/BOHnv1ZaNoImx/v1+WRcIhLTxvPY6/uorNLw9xFRKT4\n3P2bwDeAubnla+6uKu49NDZ3UBYJUVetgQUiIlJ8+UrQPU/tFM/8a6Fy7ICmXLtyYT0dXRkef213\nHgMTERHpP3d/2N2/nFuWFzueUtOwr5OJo8ox0xzoIiJSfPlK0Ae/SBksvgXeeASaNvWriXOmj6G2\nqowH127Pc3AiIiK9Z2a/y60PmFlrj+WAmbUWO75S0tDcoeHtIiJSMvKVoA+Nr50XfxJCEVjRvynX\nwiHj8gXjefy13bQl03kOTkREpHfc/YLcutrda3os1e5eU+z4Skljc6emWBMRkZKRrwT9o3lqp7iq\nx8O8a2F1/6dcu3LhBJLpLI+9uivPwYmIiPSemYXN7LVix1HKWhMp9nem1IMuIiIl40TzoB85NO6o\nQ+Tc/eXCh3qSnHMbJFthzY/7dfniqaMYXxPnV2tUzV1ERIrH3TPABjObUuxYSlXDvu4p1pSgi4hI\naYgc76C7V5+sQErGpMUw4QxYcQ+c9WnoY9GYUMi4fEE9//38VloTKWri0QIFKiIickKjgPVmtgJo\n797p7lcXL6TS0dicm2JNQ9xFRKREHDdBP5KZjaXHlGru/lbeIyo2s6AXfelnYfMTMPMP+tzElafX\nc+8zb/Kb9bv4wJmTChCkiIhIr/xlsQMoZd096BriLiIipaJXz6Cb2dVm9gbwJvAUsAV4uIBxFdf8\n66Cyrt9Trp0xeSQTR5armruIiBSVuz8FvAZU55ZXc/uEoAe9MhZmZIVGu4mISGnobZG4rwHnAq+7\n+3TgPcDzvbnQzC41sw1mttHM7jzK8Q+b2VozW2dmz5rZ6b2OvlAiZXDmLfD6cti3uc+XmxlXLqzn\nt2/spaWjqwABioiInJiZ3QisAG4AbgR+b2Z/WNyoSkfDvg4mj67QHOgiIlIyepugp9y9CQiZWcjd\nnwAWn+giMwsD3wEuA+YBN5vZvCNOexN4l7svIPgi4J5eR19Iiz8JoTCs+G6/Lr9y4QTSWWf5+p15\nDkxERKTX/gI4y90/7u4fA85Gw94PamzuVIE4EREpKb1N0FvMrAr4LfBDM/sXehSbOY6zgY3uvtnd\nu4AfA9f0PMHdn3X35tzm80BpPLRdUw/zroGX/huSbX2+/LSJNUwZXcGDa1XNXUREiibk7rt7bDeR\nvylWBzV3p6G5g0mjVCBORERKR29v0k8AI4A7gF8Dm4CrenHdRKChx3Zjbt+xfIpjPNtuZrea2Soz\nW7Vnz55eBT1gZ38Wkvthbd+nXOse5v7spiaa2pIFCE5EROSEfm1my83sE2b2CeAhYFmRYyoJ+9q7\n6OjKMHm0etBFRKR09DZBjwCPAE8SFJn5SW7Ie96Y2bsJEvT/dbTj7n6Puy9298V1dXX5fOtjm3w2\n1C+C398D7n2+/MqFE8hknYdf1jB3ERE5+dz9TwgeHVuYW+5x96PeZ4ebg1OsqQddRERKSK8SdHf/\na3efD9wO1ANPmdmjvbh0GzC5x/ak3L7DmNlC4LvANflO/AfEDM75LOzdAJuf7PPlc+urmVFXyUMa\n5i4iIkXi7j939z/OLUuLHU+paGjOTbGmHnQRESkhfX0ObTewk+AZtrG9OH8lMNvMpptZDPgg8EDP\nE8xsCvAL4KPu/nof4ym8+ddDRS2s6HvtumCY+wR+/2YTuw8kChCciIjIsZnZATNrPWJpMLOlZjaj\n2PEVU8O+oAddz6CLiEgp6e086J8zsyeBx4AxwGfcfeGJrnP3NPB5YDnwKrDE3deb2W1mdlvutP+d\na/MuM1ttZqv68TkKJxqHMz8BGx6GfW/2+fKrFtaTdXh4nYa5i4jISffPwJ8Q1H+ZBHwZuI+gaOu9\nRYyr6BqbOxhZEaU6rjnQRUSkdPS2B30y8EV3n+/uX3X3V3r7Bu6+zN3nuPtMd/9Gbt/d7n537vWn\n3X2Uuy/KLSecvu2kO+tTYCFY2fcp12aPq+aUcdU8uHZ7AQITERE5rqvd/d/d/YC7t7r7PcD73f0n\nwKhiB1dMDc2dTNYUayIiUmJ6+wz6n7n76kIHU7JqJsC8q+GlH0BXb2aXO9yVC+tZuaWZHfs7CxCc\niIjIMXWY2Y1mFsotNwLdz1z1vfrpENK4T1OsiYhI6dFcqL119mchsR/W/qTPl16xsB5AxeJERORk\n+zDwUYIaMrtyrz9iZuUEj6ANS9ms09jSqQJxIiJScpSg99aUc2H8wn5NuTajrop59TU8qARdRERO\nInff7O5XuXutu9flXm909053/12x4yuWPW1JutJZTbEmIiIlRwl6b3VPubbnVXjz6T5ffuXp9axu\naKFhX0cBghMREXk7M5tjZo+Z2cu57YVm9pVix1Vs3ffiSXoGXURESowS9L447Q+hYky/ply7csEE\nAJatUy+6iIicNP8B/BmQAnD3tQRTng5rh+ZAVw+6iIiUFiXofRGNwzs+DhuWQfPWPl06ZUwFp08a\noWHuIiJyMlW4+4oj9qWLEkkJaTw4B7p60EVEpLQoQe+rsz4FWL+mXLty4QTWbdvPlr19rwQvIiLS\nD3vNbCa5iu1m9ofAsP+muKG5g7rqMuLRcLFDEREROYwS9L4aMQnmXgkvfh+6+vY8+cFq7hrmLiIi\nJ8ftwL8Dp5rZNuCLwG3FDan4GvZ1aoo1EREpSUrQ++Oc2yDRAuuW9OmyCSPLWTx1FD9/oZFsdlhP\nPysiIieHu/t7gTrgVHe/AN37aWzpYLKGt4uISAka9jfpfplyHoxb0K8p1z72zmls3tvOI6/sLFBw\nIiIiB/0cwN3b3f1Abt/PihhP0aUzWba3JFQgTkRESpIS9P4wg8W3wO71sHNtny69/LTxTB1TwV1P\nbsL7mNyLiIj0hpmdamYfAEaY2fU9lk8A8SKHV1Q79ifIZF0F4kREpCQpQe+v+ddBOAZrftynyyLh\nEJ+9aCZrG/fz7KamAgUnIiLD3CnAlcBI4KoeyzuAzxQxrqJrbA4quGuIu4iIlCIl6P1VMRrmXArr\nfgqZVJ8u/cCZExlbXcZdT24sUHAiIjKcufv97n4LcKW739Jj+YK7P3ui683sUjPbYGYbzezOoxwf\nYWa/MrM1ZrbezG7pcWyLma0zs9VmtirPH23ANAe6iIiUMiXoA3H6zdC+BzY93qfLyiJhPnXBdJ7Z\n2MSahpYCBSciIsJLZna7md1lZvd2L8e7wMzCwHeAy4B5wM1mNu+I024HXnH304GLgW+ZWazH8Xe7\n+yJ3X5y/j5Ifjfs6MIP6EUrQRUSk9ChBH4hZ74WK/7+9e4+Pq67zP/76zEwmt8m9aXqnV2gpUsQK\nyqICCgIqKLjY6iq4KKKg7s3Luv52cdHf7vpw96cuKKKioCiyu8riilyXu1AoCBRKoaUtTe9NkyaZ\nXGYmM5/fH2fapk0CbTrJzGTez8fjPM5lzjn5zDen+fZzzvd8v03w7C8P+9CPvOUoaisifP+BV8Yg\nMBEREQB+BkwB3g08CMwAul/zCDgJWOfu6909CdwCnH/QPg7UmJkBMaAdGMhl4GNlc0cfU2sriEb0\nXyARESk8qp2ORCQKx30Q1twBfYf3JDxWHuHiU2Zz1+rtrNsZH6MARUSkxM139/8D9Lj7jcB7gJNf\n55jpQOug9c3ZbYNdAywCtgKrgM+7eyb7mQP3mtlTZnbZSD/EzC4zs5VmtnLXrl2H/o2OUGtHLzMa\n9f65iIgUJiXoR2rJMkgnYPVth33oJafMpjwS4roH9RRdRETGxN5OUvaY2XFAHTA5B+d9N/AMMA04\nAbjGzGqzn53q7icQNJG/wszePtwJ3P16d1/q7kubm5tzENKhaW3vY0aDmreLiEhhUoJ+pKa9ESYd\nc9i9uQM0xcpZ9uZZ3PbHLWzZ0zcGwYmISIm73swagK8CtwOrgW++zjFbgJmD1mdktw32ceDXHlgH\nbAAWArj7lux8J/AbgibzBSExkGZHd796cBcRkYKlBP1ImQVP0Tc9Bu3rD/vwT759LgA/evjwjxUR\nEXkt7v4jd+9w94fcfa67T3b3617nsCeBBWY2J9vx2zKC5H6wTcA7AcyshWBYt/VmVm1mNdnt1cBZ\nwPO5/E5HYuueftxhppq4i4hIgVKCngvHXwQYPHfrYR86vb6S80+Yzi1PtNLek8x9bCIiUrLM7P+a\nWf2g9QYz+/prHePuA8CVwF3Ai8Ct7v6CmV1uZpdnd7saOMXMVgH3AV9y9zagBXjEzJ4FngB+5+53\n5v6bjU5re3aINTVxFxGRAqUEPRfqZsCctwe9ubsf9uGXv2Mufak0P310wxgEJyIiJewcd9/Xi6m7\ndwDnvt5B7n6Hux/t7vPc/RvZbdftffru7lvd/Sx3f4O7H+fuP89uX+/uS7LT4r3HFoq9Y6CrkzgR\nESlUStBzZcly6NgIrSsO+9AFLTWcdWwLNz72KvFEUYxSIyIixSFsZuV7V8ysEih/jf0ntM0dfZSF\njSm1FfkORUREZFhK0HNl0fugrGpUY6IDfOb0+XT2pfjlik05DkxERErYzcB9ZnapmV0K3APcmOeY\n8qa1vZdp9ZWEQ5bvUERERIalBD1XymOw6Dx4/jeQ6j/sw0+YWc8p85r40SPrSQykxyBAEREpNe7+\nL8DXCcYsXwRc7e6v14v7hNXaoSHWRESksClBz6UlyyDRCS//flSHf/q0eezoSvDrpw8ezUZERGR0\n3P1Od/+b7HTX4M/M7LF8xZUPWzp6NcSaiIgUNCXouTTn7VAzbVRjogOcOn8Sb5hexw8efIV05vA7\nm7NVb2sAACAASURBVBMRETlMJfMydm9ygLZ4UkOsiYhIQVOCnkuhcDDk2tp7IL7rsA83Mz5z2jw2\n7u7l989vG4MARUREDlAyd4M3d/QBqIm7iIgUNCXoubZkGXganv/PUR1+1uIpzJ1UzffufwUfxZBt\nIiIiMtTmvUOsqYm7iIgUMCXouTZ5EUw9YdS9uYdDxuXvmMfqbV08tLYtx8GJiIgcoGS6M29tD56g\nz2zUE3QRESlcStDHwpLlsO1Z2LF6VIe//43TmVpXwffuX5fjwERERA7w0XwHMF5a23spj4RojpXs\nMPAiIlIElKCPheMuhFAEnhtdZ3HRSIhPvG0uKza089SrHTkOTkREJjoz6zazrmGmbjPr2rufuz+f\nzzjH0+bsEGtmJdNoQEREipAS9LEQa4b5Z8Jzt0JmdGOaL3vzTOqryvj+A3qKLiIih8fda9y9dpip\nxt1r8x1fPrR29KoHdxERKXhK0MfKkmXQvQ02PDiqw6vLI1xyymzufXEnL23vznFwIiJSSsxsspnN\n2jvlO558aG3XGOgiIlL4xjxBN7OzzewlM1tnZl8e5vOFZvaYmSXM7G/GOp5xc/TZUFE36jHRAS5+\n62yqomGue/CVHAYmIiKlwszOM7O1wAbgQWAj8Pu8BpUHnX0puvoHNMSaiIgUvDFN0M0sDFwLnAMc\nCyw3s2MP2q0d+BzwrbGMZdyVVcDiC+DF30JidE/AG6qjLD9pFrc/u5XW9t4cBygiIiXgauAtwMvu\nPgd4J/B4fkMaf3uHWFMTdxERKXRj/QT9JGCdu6939yRwC3D+4B3cfae7PwmkxjiW8bdkOaR6gyR9\nlD7xtjmEDK5/aH0OAxMRkRKRcvfdQMjMQu5+P7A030GNt31DrKmJu4iIFLixTtCnA62D1jdntx02\nM7vMzFaa2cpdu3blJLgxN/MkaJgz6jHRAabWVXLBG2dw68pWdnUnchiciIiUgD1mFgMeBm42s+8A\nPXmOadztfYKuJu4iIlLoiqaTOHe/3t2XuvvS5ubmfIdzaMyCp+gbHoY9ra+//wg+9Y65JNMZvqce\n3UVE5PDcD9QBnwfuBF4B3pfXiPJgc0cfsfII9VVl+Q5FRETkNY11gr4FmDlofUZ2W+k4/iLAYdWt\noz7F3OYYHz5pFj95dCO/WLEpd7GJiMhEFwHuBh4AaoBfZZu8l5TW9l6NgS4iIkVhrBP0J4EFZjbH\nzKLAMuD2Mf6ZhaVxDsw6JejN3X3Up7nqvMWcfkwzX71tFXc+vy2HAYqIyETl7l9z98XAFcBU4EEz\nuzfPYY271o5eZuj9cxERKQJjmqC7+wBwJXAX8CJwq7u/YGaXm9nlAGY2xcw2A38FfNXMNptZ7VjG\nNe6WLIO2l2Hr06M+RVk4xLUfOZElM+v53C3P8Pj6knsAIiIio7cT2A7sBibnOZZx5e5s7uhjZqPe\nPxcRkcI35u+gu/sd7n60u89z929kt13n7tdll7e7+wx3r3X3+uxy11jHNa4Wvx/C5Uc0JjpAVTTC\nDRe/mVmNVXzyxpWs3jqxiklERHLLzD5jZg8A9wFNwCfd/fj8RjW+2nuS9CbT6sFdRESKQtF0ElfU\nKupg4Xtg1X/CQPKITtVQHeWmPz+JWEWEi3/yBJt2a3x0EREZ0UzgL9x9sbtf5e6r8x3QeGvtyA6x\npjHQRUSkCChBHy9LlkNfO6y754hPNa2+kpv+/CRS6Qwfu2EFbXENvyYiIkO5+9+6+zP5jiOfNMSa\niIgUEyXo42XeGVDdfERjog+2oKWGH1/8ZrZ39fPxnzxJPDGQk/OKiIhMJK3teoIuIiLFQwn6eAlH\n4A0XwUt3Qm97Tk75pqMa+N5HTmT1ti4+9bOVJAbSOTmviIjIRNHa0UtDVRmx8ki+QxEREXldStDH\n05JlkEnBMzdDfBf07A6S9d526NsD/Z2Q6IZEHJI9kOqDVD8MJCCdGvaUZyxs4ZsXHs+j63bz17c+\nSyYz+qHcREREJppgDHQ9PRcRkeKg28njacobYPJiuPurwXS4Fn8ALvwxhMIHbL7wTTNoiyf4p9+v\noak6ylXnLcbMchS0iIhI8drS0cfCqTX5DkNEROSQKEEfT2bwwRtg48PBujvg4JnXX97TCk/9BBrn\nwjv/fsipP/WOebTFE/zw4Q0015Rz5RkLxvGLiYiIFJ5MJhgD/cxjW/IdioiIyCFRgj7eJi8MpsPl\n2YT94X+FluPguAuG7PK35yyiLZ7kW3e/zKRYOctOmpWDgEVERIrTzu4EyXRGPbiLiEjRUIJeLMzg\n3G/BrjXw31fApAVBk/lBQiHjmx88no7eJF/5zSoaqqO8e/GUPAUsIiKSX/uGWFMP7iIiUiTUSVwx\niUThop9BRT388sNBJ3MHKQuH+N5HTuT4GfV89pd/ZMX6ofuIiIiUgtZsgj5TncSJiEiRUIJebGpa\nYNnNEN8B/3HxsL27V0Uj/OSSNzOzoZJP3LSSJzfmZlg3ERGRYrJ3DHQ1cRcRkWKhBL0YTT8Rzvtu\n0NncXX837C4N1VFuuvRkasoj/Ol1j3HxDU/w9KaOcQ5UREQkfzZ39NJcU05FWfj1dxYRESkAStCL\n1ZJl8NYr4YkfwNM/G3aX6fWV3P1X7+CLZx/Dc5v3cMH3/sBHf7yCp17VE3UREZn4Wtv7mKmn5yIi\nUkSUoBezd30N5p4Ov/sraH1i2F1i5RE+c9p8HvnSGXz5nIWs3trFhd9/jD/70Qo1fRcRkQmttaOX\nmeogTkREiogS9GIWjgTjqtdOg1/9GXRtHXHX6vIIl79jHg9/6XS+cu5C1mzv4k+ve4wP//BxdSQn\nIiITzkA6w7bOfr1/LiIiRUUJerGraoTlt0CyJ0jSU/2vvXs0wmVvn8fDXzyDr75nEWt3xvnQ9Y+z\n7PrHeOwVJeoiIgJmdraZvWRm68zsy8N8XmdmvzWzZ83sBTP7+KEeO162dfaTzrh6cBcRkaKiBH0i\nmLwIPvAD2PIU/M9fgvvrHlIZDfOJt83l4S+ezt+/91jW7+ph+Q8f56IfPMYf1rXhh3AOERGZeMws\nDFwLnAMcCyw3s2MP2u0KYLW7LwFOA/7VzKKHeOy42DfEmpq4i4hIEVGCPlEsei+c9rfw7C9gxXWH\nfFhFWZg/P3UOD33xdK5637G8uruHD/9oBRf94DEeXrtLibqISOk5CVjn7uvdPQncApx/0D4O1JiZ\nATGgHRg4xGPHxeYODbEmIiLFRwn6RPL2L8LC9wZDr61/4LAOrSgLc8mfzOHBL5zO1ecvZnNHHx/9\n8RNc8P0/cP+anUrURURKx3SgddD65uy2wa4BFgFbgVXA5909c4jHAmBml5nZSjNbuWvXrlzFvv8H\nt/cSMphWrwRdRESKhxL0iSQUgg9cB5OOhv+4BNo3HPYpKsrCfPSts3ngC6fxjQ8cx86uBB//6ZOc\nd82j3LN6hxJ1EREBeDfwDDANOAG4xsxqD+cE7n69uy9196XNzc05D7C1o4+pdZWUhfVfHRERKR6q\ntSaa8hpY/ovgPfRbPgyJ+OhOEwnzkZOP4oEvnMY3Lzyezr4Un7xpJed+9xF+v2obmYwSdRGRCWoL\nMHPQ+ozstsE+DvzaA+uADcDCQzx2XLS296p5u4iIFB0l6BNR41z405/ArjVw26chkxn1qcrCIS56\n80z+96/fwb9dtIREKs2nb36as7/zELc/u5W0EnURkYnmSWCBmc0xsyiwDLj9oH02Ae8EMLMW4Bhg\n/SEeOy42d/QxQz24i4hIkYnkOwAZI/POgDOvhrv/Dq47FZrmQcNRUH8UNMwO5vWzoKzikE4XCYe4\n4MQZnH/CdH63ahv/ft9aPvfLP/Lte1/ms2fM533HTyOiZoQiIkXP3QfM7ErgLiAM3ODuL5jZ5dnP\nrwOuBn5qZqsAA77k7m0Awx073t8hMZBmR3c/Mxv1BF1ERIqLEvSJ7K1XgGdgw4Ow80V4+S5IJw7c\nJzZlUOKeTdr3Je+VQVN5zwDBPOzOeTMzvPdjM3no5R38/LENXHPry9x2dwXLT5rJO4+dRtnko8Es\nL195QkkPBOUYCuc7EhEpMe5+B3DHQduuG7S8FTjrUI8db1s6+nBHY6CLiEjRUYI+kZnBn3wumCBo\n6h7fAXtehY5XD5xvehye/89sMv76QgQD354GUA70AQ8G0/ryY7l71ufom/wmJsWiNMXKaaoO5pNi\nUeoqyzAl8CPLZILfxb1XQaQCzvsuzD4131GJiBQNDbEmIiLFSgl6KQmFoHZqMM16y9DP0yno3Bwk\n7HtaIZ0EC2UnC+bYMOuGY7ywLc4zL77EOXt+yeVrL+e2NafwL6nlbKPpgB8TCRlNsShN1eU0xaJM\nyibwjbFoMM9uD5ajxMojpZPQtz4Jd34ZtqyEqUugvxN++h448WI48x+hsj7fEYqIFLzWjl4AZjbq\nCbqIiBQXJeiyX7gMGucE02Ey4Lg3wHFnAYmvwKPf5vw//Dvnl/+R3Us+xdoFl7KzP0xbPMnueILd\n8SS7exK0xZNsaOthdzxJXyo97Lmj4RBNsSBZb6yO7nsav3e5vqqM2soy6iuD5brKMqqi4eJK6js3\nw71fg1W3QqwFzv8eLFkOA/3wwD/BY9fAy3fCud+CY8/Ld7QiIgWttb2PsrDRUnto/ayIiIgUCiXo\nknvlMTjjq9iJH4N7r2LSU99m0su/gnf+A5zyoeBJ/jB6kwPsjidp7wmm3T1J2nv2JvP7t21o66G9\nJ0lvcviEHqAsbNRVBsl6fVXQrL6+MpvIV5VRHY2QTGdIDGRIDKRJpDLBeipYTw7s/2z/coaycIg5\nk6qY3VTNnEn7p8bq6OhuCCR74NHvwqPfCV4veNvfwKl/GZQhQLQKzroajrsQbr8Sbv0oLHxvkKjX\nTj38n1co0inY9iw0HxMMDSgikkObO3qZVl9JOFREN2pFRERQgi5jqX4WfPAGOOlTQbPt2y6HJ34A\nZ//zsE3sq6IRqhojh9wksT+VZndPks7eFHv6knT1pdjTm6KzL8WevmDemV3f2d3Pyzu66exL0d0/\ncMB5opEQ5ZEQ5ZFwdh46YFtVNEJDVbCtP5XmxW3d3P3CDgYGDTFXUxFhbjZZnz0ocZ89qZrairKh\nwe99z/yef4DurbD4A/CurwUd9Q1n2gnwyfvhsWuDJ+rXngxnfi1o+j7CDY+C1L4enr4J/ngz9OyE\ncDnMPQ0WvgeOORdizfmOUEQmgNaOPnUQJyIiRcnci28c66VLl/rKlSvzHYYcjkwGVv1H0PHZoSSk\nY2ggnaEvlSYaCRENh0b15DuVzrC5o4+NbT2sb+thY1sPG7LT1s6g9+C9JmXfs9/7cxYOrOFTvT9k\nYfolXg7P5/qqT/JCZPGwPydWHmZKXSXT6iqYUlfB1LpKZtt25j7+FaKtj8JRp8L7vgOT5o+qLMbF\nQBLW/A88fSOsfyDou+Dos+HY9wdP0df8FvZsCrbPfEuQrC96bzAc4DDcvbheXxAZJTN7yt2X5juO\n8ZDrev1NV9/DWYtb+KcLjs/ZOUVERI7EodbrStBlfB3cpPuUK7NNuidOM+f+VJpN7b2s39XDxt09\nrN8Vp6M3RcPATi5s/xEnx+9jT7iJXzdeyuOxM3Eb/gm4O3T3p9jW2c/2zn6S6cE97DvLIw/ylcjN\nlJPkrkmXsHrOxbTUx5haX0ljdZTqaISaigjV5RFi5RGikXF+0t62LkjKn/kF9LZB3Sw48WPwxo9A\n7bQDv+j2VbDmd0Eiv+N5AAaaF7Nj2rt4NvY2VvZNY11bD+t2dLOjO8FRTVUsmlLLwik1LJwazGc0\nVOY0cXd3dvck6e4foLE6Sm1FCXVWKAVBCfro9CYHOPbv7+IL7z6GK04v4JuXIiJSUpSgS2E7uFO0\nt38BaqZAIg7J7JQ4aL5vuQeS3cHyQAIi5VBWFYzbXlY5wvLgbZXB8GX7eqgfabLht4fCYOGgabmF\ns+sHLe/bJ7v+/H8FNyY8A6d89sD3zA/B3mRxe2c/W/f0sb2rn617+unZvZlzN3+bt/Y/zBqfxReT\nn+Q5nzfsOaLhELGKCNXlYWLlZcTKw8TKgwS+piJCdTRCRVk4aFmQbV0QHdTcf/D60M+yx5Gk6pU7\niD77M0KvPhKUwcJz4cRLYN7pQ8Z0d3d2dCVYtzPOup3drN0Zp2vrWua0PcCp6cdZai8TMqfVJ7Oy\n4q1sanknfVPexPq2ftZs72ZTe+++c9WURzhmSg2LptaycGoNC6fUcsyUGmLlw7/JkxhIs21PUJ5b\nstPWPX1s3dO/bzkxsP+mSFnY9o080BQrZ1J1dN9yU3V2NIJB6xVlE2/8+kzGebW9l7U7uqkoCzO5\ntpzJNRU0VBX30InuztqdcVas383jG9rZuqePec2x4AbQlOB6mhQrH/e4lKCPzss7ujnr/z3Ed5ad\nwPknTM/JOUVERI6UEnQpDptXBu+nb35y6GcWgmgsmMpjEK3OLtfs3xapCJL0VB+keg+aD7OcSY3/\nd9xr8QXBe+P1s3J/7jW/w3/31xDfQdviS9k06/30ppx4yulNZuhJOfFkht5Uhu5EhnjSiSfT9CSd\n7mSwrTuRoXfASXiEAcIEffMfmnm2heXh/+XC8MM0WJxXM5P5Vfp0/pvT6Iw0DUnqg1cLYNPuXroT\n+/sEqK2IMH9yjAWTa5g/Ocai2gTHdj1Cw6a7sQ0PBEP/lddBVQOUVZEOV9DrUbrSZXQkI7QlQuzo\nC9GZjtJHlH4vp6IqRkNdLbHqarr7U3T0JunqTRDvTxLCMXzfvLY8TENVhPqKMPVVEeoqIkSj5eyy\nJrZkGtmUquPVRDVtPQO0xZO0xRMHJPGD1VYE/SnMbKhiZmMlsxqrmJFdn9FQWfAJfFd/ipe2d/Pi\nti5e3BbMX9rePexoC2VhY3JNBc015UyuKaeltoLJNeVBAr93uaaCpuooyXSG7v4B4okBuvuDPiGC\nKTV0eyL4LJFKc1RTFUe31HB0Sw3HTKlhck35qG8KZDLOi9u7eGJDOyvWt/PExnbae5IATK2r4Kim\nKtbt7KEtnth3zKRYeTZhr9l3I2j+5NiY/h6VoI/OfS/u4NIbV/Lrz5zCibMacnJOERGRI6UEXYqH\ne/AustmBCXhZZbAtl9KpIFkfSARPs0ecfITt6eB9ek9DJn3QPLvdMwd9loHGuTD9xNx+l4P1dwbv\n+K+8ISen81AEQmV4qAwPRchklzOhCBkrI21h0laGpZPUdb1E2iJsmnw6q6dewIaaN5HMGMmBTDCl\n04OWg3k648xsrGL+5Bjzm2PMb4nRHHuNpCvRDWvvgY2PZFtU9Ax7I8ZTvXiqF0v1YYzB37dQBGqm\nQu00vGYqA9VTiZdPpiPSTFtoEtu9kS0DtWztTtPa0Utrey9bOnrwgSRRBijLTlNjIWbWlTGjJsz0\n2jBTa8JMqQ5TWxMjWjeVaN0UKquqKI+Mrp+EEWXS0NMG3dsgvoNMxtmRiPBKp7O2w3mhLc3zuwZY\nu8dJEySfdZVlLMq2Sjh2ai1HT6khlc6wo6ufnV0JdnYnaOuM09vZRqJ7N8l4O5FkJ/XEqbc4ddZD\nHT3UWi+dXs0ur2cXdcHcg/luaskQvIZRWRampiJCrCJCTUUZ0bCxoa2Htnhy39eorQhaTCxoqeGY\nlhoWtMQ4pqWGpmGedA+kM6ze1sWK9e2s2LCbJza005XtLHJGQyUnz2ni5LmNvGVOEzMb978q0RZP\n8NL2btZs72bNti7WbO/m5R3d+27KhAzmTKoOXrNoCV63eNeiyTn7fSlBH52fPrqBq367mif/7l00\n14x/ywcREZHhFEyCbmZnA98BwsCP3P2fD/rcsp+fC/QCl7j70691TiXoIq9h23NBb+kj3XzYe9Ng\nuO2ZgaCVQTo7DV5OJ4PPD1hOBsfOPQ2WfLiwemF3z7auyCbvA/1DX1vg4NcYbP9872cDiaBjw66t\n0LUlOz9oOdV70A/P3mxKJ4My9OGfsr+eDo+x0+tptwY6Qg10RhrpDjfRE22kN9pMf8UkUpXNUF5H\nOATlyXaqkm3EEm3UpNqoSe2iZmA3ddmpIb2bukwHYQ4tnnQoikVjWHk1Fo0Fw/5Fq4NXRlK90NcB\nfXuCKdn9mudKRGpIhKqpSHcTTfcM+dwthFc2YTUtWKwlePWlJjuvboZIBV2pEK3dGTZ1DrC+I8X6\n9gFe2p1kd7+RJEKSMmJVVcxuaeToKTU0Vpfz/KvbWb+plfJkB/UWZ2FNkuObMhxdm2BWRYJYugt6\nd0NfezDv7Qi+ywGvrYSDGzOhEG5h0oRIuZFKG4mMkUhDf9roDtXyxqtWjOp3PRwl6KPz9f9Zzc9X\nvMqL/3h2Ub96ISIiE8uh1utjOsyamYWBa4Ezgc3Ak2Z2u7uvHrTbOcCC7HQy8P3sXERGY+rxwVTq\nzKCsIpiOVKwZpi4Z/jN36N8zKHHPTokuCEchXHbQfOjygEXY3Qc7e9L09MQJ9ewg3LOLaP9Oyvvb\nmJHYxTHJtdQMtFGWSkH/gSEkKCNMhghDm593Wi3toSY6wk28Wj6bzsgkuiJNdEcm0V3WxKSaCubX\nh5hT60yrzhBN92b7eeghnIxDcu96fN92+jqgrBpqZ0DLcVDZsH+qqB+0nl2uqKM8FGbfs8xkD8R3\nZqcdEN+BxXdi2WXiO2DXmmCeGfQKBLA4Ox1g8K84A2yD5NYIGYwKSwVva+z94Qlga3ba+7pEVRNU\nTYJJRwfL0digFjMHtoqxzACRTJqIp6kc1JomlR6g1/W0thC0dvQyo6FKybmIiBSlsR4H/SRgnbuv\nBzCzW4DzgcEJ+vnATR48yn/czOrNbKq7bxvj2EREjpzZ/oS0Zfjh8l5PBGjJTq/JPXiVIb4T4tuD\nefd2yuPbg2Q/NiXobLFmajCPtVAXiVIHzBlVZGMkWg2Nc4LptWQywc2Pnl1BK4h0MmjRkE4Ew/el\nE8H6QOKgzxKUDSRIDaQh1ghVjUHiXZmdVzUGv69wWc6+UhlQl7OzyZE4dUGz3j0XEZGiNdYJ+nSg\nddD6ZoY+HR9un+nAAQm6mV0GXAYwa9YYdLIlIlLozLJPpeuh+eh8RzP2QqFsct142IcaEM19RFIE\nPvqWo/IdgoiIyKiN88DIo+fu17v7Undf2txcQO+5ioiIiIiIiOTAWCfoW4CZg9ZnZLcd7j4iIiIi\nIiIiE9pYJ+hPAgvMbI6ZRYFlwO0H7XM78DELvAXo1PvnIiIiIiIiUmrG9B10dx8wsyuBuwiGWbvB\n3V8ws8uzn18H3EEwxNo6gmHWPj6WMYmIiIiIiIgUorHuJA53v4MgCR+87bpByw5cMdZxiIiIiIiI\niBSyoukkTkRERERERGQiU4IuIiIiIiIiUgCUoIuIiIiIiIgUAAteAS8uZrYLeDWHp5wEtOXwfBOF\nymUolcnwVC5DqUyGp3IZaqQyOcrdm8c7mHxQvT5uVC5DqUyGUpkMT+UylMpkeEdUrxdlgp5rZrbS\n3ZfmO45Co3IZSmUyPJXLUCqT4alchlKZ5J7KdHgql6FUJkOpTIanchlKZTK8Iy0XNXEXERERERER\nKQBK0EVEREREREQKgBL0wPX5DqBAqVyGUpkMT+UylMpkeCqXoVQmuacyHZ7KZSiVyVAqk+GpXIZS\nmQzviMpF76CLiIiIiIiIFAA9QRcREREREREpAErQRURERERERApAySfoZna2mb1kZuvM7Mv5jqcQ\nmNlGM1tlZs+Y2cp8x5MvZnaDme00s+cHbWs0s3vMbG123pDPGMfbCGVylZltyV4vz5jZufmMcbyZ\n2Uwzu9/MVpvZC2b2+ez2Ur9WRiqXkr1ezKzCzJ4ws2ezZfK17PaSvlZySXX68FSvB1SvD6V6fSjV\n68NTvT7UWNXrJf0OupmFgZeBM4HNwJPAcndfndfA8szMNgJL3b0t37Hkk5m9HYgDN7n7cdlt3wTa\n3f2fs//5a3D3L+UzzvE0QplcBcTd/Vv5jC1fzGwqMNXdnzazGuAp4P3AJZT2tTJSuVxEiV4vZmZA\ntbvHzawMeAT4PHABJXyt5Irq9JGpXg+oXh9K9fpQqteHp3p9qLGq10v9CfpJwDp3X+/uSeAW4Pw8\nxyQFwt0fAtoP2nw+cGN2+UaCP0wlY4QyKWnuvs3dn84udwMvAtPRtTJSuZQsD8Szq2XZySnxayWH\nVKfLa1K9PpTq9aFUrw9P9fpQY1Wvl3qCPh1oHbS+mRK/0LIcuNfMnjKzy/IdTIFpcfdt2eXtQEs+\ngykgnzWz57JN5UqqyddgZjYbeCOwAl0r+xxULlDC14uZhc3sGWAncI+761rJHdXpI1O9PjL9+xte\nyf6dHkz1+vBUr+83FvV6qSfoMrxT3f0E4BzgimzzJzmIB++HlO47Ivt9H5gLnABsA/41v+Hkh5nF\ngP8C/sLduwZ/VsrXyjDlUtLXi7uns39fZwAnmdlxB31esteKjCnV64dA//72Kem/03upXh+e6vUD\njUW9XuoJ+hZg5qD1GdltJc3dt2TnO4HfEDQblMCO7Ds4e9/F2ZnnePLO3Xdk/zhlgB9SgtdL9r2j\n/wJudvdfZzeX/LUyXLnoegm4+x7gfuBsdK3kiur0Eahef03693cQ/Z1WvT4S1esjy2W9XuoJ+pPA\nAjObY2ZRYBlwe55jyiszq852/ICZVQNnAc+/9lEl5Xbg4uzyxcB/5zGWgrD3D1DWByix6yXbQciP\ngRfd/d8GfVTS18pI5VLK14uZNZtZfXa5kqAzszWU+LWSQ6rTh6F6/XXp399BSvnvNKheH4nq9aHG\nql4v6V7cAbJDAXwbCAM3uPs38hxSXpnZXIK76wAR4BelWiZm9kvgNGASsAP4B+A24FZgFvAqcJG7\nl0znKiOUyWkEzZoc2Ah8atB7NxOemZ0KPAysAjLZzV8heC+rlK+VkcplOSV6vZjZ8QSdxYQJbpDf\n6u7/aGZNlPC1kkuq04dSvb6f6vWhVK8PpXp9eKrXhxqrer3kE3QRERERERGRQlDqTdxFREREuNKR\n+wAAAnNJREFURERECoISdBEREREREZECoARdREREREREpAAoQRcREREREREpAErQRURERERERAqA\nEnSREmBmaTN7ZtD05Ryee7aZlcyYlyIiIvmmel1k4orkOwARGRd97n5CvoMQERGRnFC9LjJB6Qm6\nSAkzs41m9k0zW2VmT5jZ/Oz22Wb2v2b2nJndZ2azsttbzOw3ZvZsdjole6qwmf3QzF4ws7vNrDK7\n/+fMbHX2PLfk6WuKiIiUBNXrIsVPCbpIaag8qCnchwZ91unubwCuAb6d3fbvwI3ufjxwM/Dd7Pbv\nAg+6+xLgROCF7PYFwLXuvhjYA1yY3f5l4I3Z81w+Vl9ORESkxKheF5mgzN3zHYOIjDEzi7t7bJjt\nG4Ez3H29mZUB2929yczagKnunspu3+buk8xsFzDD3RODzjEbuMfdF2TXvwSUufvXzexOIA7cBtzm\n7vEx/qoiIiITnup1kYlLT9BFxEdYPhyJQctp9vdv8R7gWoK78k+amfq9EBERGVuq10WKmBJ0EfnQ\noPlj2eU/AMuyyx8BHs4u3wd8GsDMwmZWN9JJzSwEzHT3+4EvAXXAkLv9IiIiklOq10WKmO56iZSG\nSjN7ZtD6ne6+d0iWBjN7juBu+fLsts8CPzGzLwC7gI9nt38euN7MLiW4o/5pYNsIPzMM/Dxb2Rvw\nXXffk7NvJCIiUrpUr4tMUHoHXaSEZd9VW+rubfmORURERI6M6nWR4qcm7iIiIiIiIiIFQE/QRURE\nRERERAqAnqCLiIiIiIiIFAAl6CIiIiIiIiIFQAm6iIiIiIiISAFQgi4iIiIiIiJSAJSgi4iIiIiI\niBSA/w/jKxBBAimDfAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x26a0096cd30>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(14,4))\n",
    "plt.subplot(1,2,1);plot_loss(history_v1_2, metric_list=['loss','val_loss'], title='Loss', legend_list=['train', 'val'],  plot_path=plot_path)\n",
    "plt.subplot(1,2,2);plot_loss(history_v1_2, metric_list=['categorical_accuracy','val_categorical_accuracy'], title='Accuracy', legend_list=['train', 'val'], plot_path=plot_path)\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20000/20000 [==============================] - 50s    \n",
      "\n",
      "result_train:\n",
      " [0.0050589485116302969, 0.99939999999999996]\n",
      "4000/4000 [==============================] - 10s    \n",
      "\n",
      "result_test:\n",
      " [0.004798408726230264, 0.99950000000000006]\n"
     ]
    }
   ],
   "source": [
    "############################## Evaluation #############################\n",
    "result_train = train_model_v1_2.evaluate(X_train, Y_train, verbose=1)\n",
    "print('\\nresult_train:\\n',result_train)\n",
    "result_test = train_model_v1_2.evaluate(X_test, Y_test, verbose=1)\n",
    "print('\\nresult_test:\\n',result_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}