transfer4uspVGG16SoftMaxNonEQU.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/SerG1oAC/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "# Imports\n",
    "import numpy as np\n",
    "from numpy import genfromtxt\n",
    "import pylab as pl\n",
    "from io import StringIO\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import itertools\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.framework import ops\n",
    "import urllib\n",
    "import random\n",
    "import sys\n",
    "from IPython.display import clear_output\n",
    "import re\n",
    "import os, os.path, gc, time, shutil, matplotlib, glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "modelPath = './transfer4uspVGG16NonEQUVerify_Jul_0518'\n",
    "if not os.path.isdir(modelPath):\n",
    "    os.makedirs(modelPath)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    print(cm)\n",
    "    \n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    fmt = '.2f' if normalize else 'd'\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, format(cm[i, j], fmt),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Published Image quantity: 5505\n",
      "'4.62406833671_-74.0571218553_-90.jpg'\n",
      "'4.6461951_-74.0536636_180.jpg'\n",
      "'4.6461951_-74.0536636_180.jpg'\n"
     ]
    }
   ],
   "source": [
    "# Image set directory root\n",
    "imgRoot = '/Users/SerG1oAC/Documents/dbintroUdacity/fullstack/vagrant/ggStreetView/map/localidades/chapinero/imgs/'\n",
    "\n",
    "# Load published images\n",
    "pblImgFile = open('mstrPublishedLst', 'r') \n",
    "pblImgSet =  pblImgFile.readlines() \n",
    "print('Published Image quantity: ' + str(len(pblImgSet)))\n",
    "print(pblImgSet[0][:-2])\n",
    "print(pblImgSet[len(pblImgSet) - 1][:-2])\n",
    "print(pblImgSet[-1][:-2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load image feature vectors extracted with the specified model\n",
    "imageFeatures = genfromtxt('VGG19Chapinero_Ftrs.csv', delimiter=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image featured set shape:  (5505, 512)\n"
     ]
    }
   ],
   "source": [
    "print(\"Image featured set shape: \", imageFeatures.shape)\n",
    "imgFtrStd = np.expand_dims(np.std(imageFeatures, axis=0), axis=0) \n",
    "imgFtrMean = np.expand_dims(np.mean(imageFeatures, axis=0) , axis=0) \n",
    "imgNrmFeatures = ((imageFeatures - imgFtrMean)/(imgFtrStd+0.00000000001))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training/Validation Set description from descriptorIndexerNonEQUX_Jul_0518.txt\n",
      "Vote set shape                    (25538, 3)\n",
      "Training percentage               70 %\n",
      "Training mini batch size          64\n",
      "Total amount of training votes    17876\n",
      "Total amount of test votes        7662\n",
      "Number of training mini batches   280\n",
      "Last batch size                   20\n",
      "Number of test mini batches       120\n",
      "Last test batch size              46\n",
      "Test Zeros 0, Ones 3831, Twos 3831\n",
      "Train Zeros 0, Ones 8938, Twos 8938\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Load vote indexer file\n",
    "# voteSetFile = 'descriptorIndexerNonEQUX_Jun_0418.txt'\n",
    "voteSetFile = 'descriptorIndexerNonEQUX_Jul_0518.txt'\n",
    "# voteSetFile = 'descriptorIndexerNonEQUX_May_2718.txt'\n",
    "# voteSetFile = 'descriptorIndexerNonEQU_May_2718.txt'\n",
    "# voteSetFile = 'descriptorIndexerHBoostX_May_2718.txt'\n",
    "# voteSetFile = 'descriptorIndexerHBoostX_Nov_24.txt'\n",
    "# voteSetFile = 'descriptorIndexer_Nov_24.txt'\n",
    "# voteSetFile = 'descriptorIndexerNrml_Nov_24.txt'\n",
    "# voteSetFile = 'escriptorIndexerBoost_Nov_24.txt'\n",
    "# voteSetFile = 'descriptorIndexerHBoost_Nov_24.txt'\n",
    "# voteSetFile = 'descriptorIndexerNrml_Nov_05.txt'\n",
    "# voteSetFile = 'escriptorIndexerBoost_Nov_05.txt' # %75!!! with DROP-OUT\n",
    "# voteSetFile = 'descriptorIndexerHBoost_Nov_05.txt'\n",
    "print(\"Training/Validation Set description from \" + voteSetFile)\n",
    "\n",
    "votesIndex = genfromtxt(voteSetFile, delimiter=',')\n",
    "print(\"Vote set shape                   \", votesIndex.shape)\n",
    "\n",
    "\n",
    "# training percentage\n",
    "perTrain = 70\n",
    "print('Training percentage              ' , perTrain,'%')\n",
    "# miniBatch Size\n",
    "minibatch_size = 64\n",
    "print('Training mini batch size         ' , minibatch_size)\n",
    "# Total amount of training votes\n",
    "ImgVtTrainSize = np.int(np.floor((votesIndex.shape[0]*perTrain)/100))\n",
    "print(\"Total amount of training votes   \" , ImgVtTrainSize)\n",
    "# Total amount of test votes\n",
    "ImgVtTestSize = votesIndex.shape[0] - ImgVtTrainSize\n",
    "print(\"Total amount of test votes       \" , ImgVtTestSize)\n",
    "num_minibatches = int(ImgVtTrainSize / minibatch_size)\n",
    "LastBatchSize = ImgVtTrainSize - num_minibatches*minibatch_size\n",
    "if LastBatchSize > 0:\n",
    "    num_minibatches = num_minibatches + 1\n",
    "print(\"Number of training mini batches  \" , num_minibatches)\n",
    "print(\"Last batch size                  \", LastBatchSize)\n",
    "#--Test figures\n",
    "iniTestIdx = ImgVtTrainSize\n",
    "num_Testminibatches = int(ImgVtTestSize / minibatch_size)\n",
    "LastTestBatchSize = ImgVtTestSize - num_Testminibatches*minibatch_size\n",
    "if LastTestBatchSize > 0:\n",
    "    num_Testminibatches = num_Testminibatches + 1\n",
    "print(\"Number of test mini batches      \" , num_Testminibatches)\n",
    "print(\"Last test batch size             \", LastTestBatchSize)\n",
    "\n",
    "testZ = 0\n",
    "testO = 0\n",
    "testT = 0 \n",
    "for i in range(votesIndex.shape[0]-ImgVtTestSize, votesIndex.shape[0]):\n",
    "    if votesIndex[i][2] == 0:\n",
    "        testZ += 1\n",
    "    elif votesIndex[i][2] == 1:\n",
    "        testO += 1\n",
    "    else:\n",
    "        testT += 1\n",
    "        \n",
    "print(\"Test Zeros %i, Ones %i, Twos %i\" % (testZ, testO, testT))\n",
    "\n",
    "trainZ = 0\n",
    "trainO = 0\n",
    "trainT = 0 \n",
    "for i in range(0, votesIndex.shape[0]-ImgVtTestSize):\n",
    "    if votesIndex[i][2] == 0:\n",
    "        trainZ += 1\n",
    "    elif votesIndex[i][2] == 1:\n",
    "        trainO += 1\n",
    "    else:\n",
    "        trainT += 1\n",
    "        \n",
    "print(\"Train Zeros %i, Ones %i, Twos %i\" % (trainZ, trainO, trainT))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getMiniBatch(minbIdx, prm = None):\n",
    "    # minbIdx: miniBatch index\n",
    "    # training set permutation array\n",
    "    iniIdx = minbIdx * minibatch_size\n",
    "    endIdx = iniIdx + minibatch_size - 1\n",
    "    if endIdx > (ImgVtTrainSize - 1):\n",
    "        endIdx = ImgVtTrainSize - 1\n",
    "    vsize = (endIdx - iniIdx) + 1  \n",
    "    # build room\n",
    "    x_train_orig = np.zeros((vsize, imageFeatures.shape[1]*2)) \n",
    "    y_train_orig = np.zeros((vsize,2))\n",
    "    idxIte = 0\n",
    "    for i in range(iniIdx,(endIdx+1)):\n",
    "        imgAIdx = 0\n",
    "        imgBIdx = 0\n",
    "        voteCode = 0\n",
    "        if prm == None:\n",
    "            imgAIdx = votesIndex[i][0] - 1\n",
    "            imgBIdx = votesIndex[i][1] - 1\n",
    "            voteCode = votesIndex[i][2]\n",
    "        else:\n",
    "            imgAIdx = votesIndex[prm[i]][0] - 1\n",
    "            imgBIdx = votesIndex[prm[i]][1] - 1\n",
    "            voteCode = votesIndex[prm[i]][2]\n",
    "        \n",
    "        y_train_orig[idxIte] = np.zeros((1,2))\n",
    "        y_train_orig[idxIte][np.int(voteCode-1)] = 1 \n",
    "        x_train_orig[idxIte] = np.expand_dims(np.concatenate((imgNrmFeatures[np.int(imgAIdx)], imgNrmFeatures[np.int(imgBIdx)]), axis=0), axis=0)[0]\n",
    "            \n",
    "        idxIte =  idxIte + 1\n",
    "\n",
    "    return  x_train_orig, y_train_orig     \n",
    "        \n",
    "    \n",
    "\n",
    "def getTestMiniBatch(minTbIdx):\n",
    "    # minbIdx: miniBatch index\n",
    "    # training set permutation array\n",
    "    minbIdx = int(minTbIdx%num_Testminibatches)\n",
    "    \n",
    "    iniIdx = minbIdx * minibatch_size\n",
    "    endIdx = iniIdx + minibatch_size - 1\n",
    "    if endIdx > (ImgVtTestSize - 1):\n",
    "        endIdx = ImgVtTestSize - 1\n",
    "    vsize = (endIdx - iniIdx) + 1  \n",
    "    # build room\n",
    "    x_train_orig = np.zeros((vsize, imageFeatures.shape[1]*2)) \n",
    "    y_train_orig = np.zeros((vsize,2))\n",
    "    idxIte = 0\n",
    "    for i in range(iniIdx,(endIdx+1)):\n",
    "        imgAIdx = votesIndex[i][0] - 1\n",
    "        imgBIdx = votesIndex[i][1] - 1\n",
    "        voteCode = votesIndex[i][2]\n",
    "        \n",
    "        y_train_orig[idxIte] = np.zeros((1,2))\n",
    "        y_train_orig[idxIte][np.int(voteCode-1)] = 1 \n",
    "        x_train_orig[idxIte] = np.expand_dims(np.concatenate((imgNrmFeatures[np.int(imgAIdx)], imgNrmFeatures[np.int(imgBIdx)]), axis=0), axis=0)[0]\n",
    "            \n",
    "        idxIte =  idxIte + 1\n",
    "    \n",
    "    return  x_train_orig, y_train_orig\n",
    "\n",
    "def getTestSet():\n",
    "    # build room\n",
    "    x_train_orig = np.zeros((ImgVtTestSize, imageFeatures.shape[1]*2)) \n",
    "    y_train_orig = np.zeros((ImgVtTestSize,2))\n",
    "    idxIte = 0\n",
    "    for i in range(votesIndex.shape[0]-ImgVtTestSize, votesIndex.shape[0]):\n",
    "        imgAIdx = votesIndex[i][0] - 1\n",
    "        imgBIdx = votesIndex[i][1] - 1\n",
    "        voteCode = votesIndex[i][2]\n",
    "        \n",
    "        y_train_orig[idxIte] = np.zeros((1,2))\n",
    "        y_train_orig[idxIte][np.int(voteCode-1)] = 1 \n",
    "        x_train_orig[idxIte] = np.expand_dims(np.concatenate((imgNrmFeatures[np.int(imgAIdx)], imgNrmFeatures[np.int(imgBIdx)]), axis=0), axis=0)[0]\n",
    "            \n",
    "        idxIte =  idxIte + 1\n",
    "    \n",
    "    return  x_train_orig, y_train_orig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gX_test :  (7662, 1024)\n",
      "gY_test :  (7662, 2)\n"
     ]
    }
   ],
   "source": [
    "gX_test, gY_test = getTestSet()\n",
    "print(\"gX_test : \" , gX_test.shape)\n",
    "print(\"gY_test : \" , gY_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_placeholders(n_x, n_y):\n",
    "    X = tf.placeholder(tf.float32, shape=(None,n_x), name = 'X') \n",
    "    Y = tf.placeholder(tf.float32, shape=(None, n_y), name = 'Y')\n",
    "    DRP_En = tf.placeholder(tf.bool, shape=(), name = 'DRP_En')\n",
    "    \n",
    "    return X,Y,DRP_En"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_propagationBackup(X, DRP_En):\n",
    "    \n",
    "    prtVlmSize = True\n",
    "    # Nov 05\n",
    "    # 0.3, 0.25, 0.2 => 78% over test\n",
    "    # 0.4, 0.3, 0.2 => 78.4% over test\n",
    "    # Nov 25\n",
    "    # 0.44, 0.33, 0.22 => 72% over test\n",
    "    # 0.5, 0.4, 0.3 => 73% over test\n",
    "    DRPOUT_X = tf.layers.dropout(inputs=X, rate=0.5, training=DRP_En)\n",
    "    Z4 = tf.contrib.layers.fully_connected(DRPOUT_X, 1024)\n",
    "    DRPOUT_Z4 = tf.layers.dropout(inputs=Z4, rate=0.45, training=DRP_En)\n",
    "    Z5 = tf.contrib.layers.fully_connected(DRPOUT_Z4, 512)\n",
    "    DRPOUT_Z5 = tf.layers.dropout(inputs=Z5, rate=0.4, training=DRP_En)\n",
    "    Z6p = tf.contrib.layers.fully_connected(DRPOUT_Z5, 256)\n",
    "    DRPOUT_Z6p = tf.layers.dropout(inputs=Z6p, rate=0.35, training=DRP_En)\n",
    "    Z6pp = tf.contrib.layers.fully_connected(DRPOUT_Z6p, 128)\n",
    "    DRPOUT_Z6pp = tf.layers.dropout(inputs=Z6pp, rate=0.3, training=DRP_En)\n",
    "    Z6 = tf.contrib.layers.fully_connected(DRPOUT_Z6pp, 3, activation_fn=None)\n",
    "    \n",
    "    return Z6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_propagationWinner(X, DRP_En):\n",
    "    \"\"\"\n",
    "    This setup works well along with\n",
    "    VGG19_Ftrs.csv features\n",
    "    59.7%\n",
    "    learning rate 0.00001\n",
    "    80 20\n",
    "    minibatch size 64\n",
    "    \"\"\"\n",
    "    \n",
    "    DRPOUT_X = tf.layers.dropout(inputs=X, rate=0.5, training=DRP_En)\n",
    "    Z4 = tf.contrib.layers.fully_connected(DRPOUT_X, 1024)\n",
    "    DRPOUT_Z4 = tf.layers.dropout(inputs=Z4, rate=0.45, training=DRP_En)\n",
    "    Z5 = tf.contrib.layers.fully_connected(DRPOUT_Z4, 512)\n",
    "    DRPOUT_Z5 = tf.layers.dropout(inputs=Z5, rate=0.3, training=DRP_En)\n",
    "    Z6 = tf.contrib.layers.fully_connected(DRPOUT_Z5, 3, activation_fn=None)\n",
    "    \n",
    "    return Z6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_propagation(X, DRP_En):\n",
    "    \n",
    "    \n",
    "    DRPOUT_X = tf.layers.dropout(inputs=X, rate=0.5, training=DRP_En)\n",
    "    Z4 = tf.contrib.layers.fully_connected(DRPOUT_X, 1024)\n",
    "    DRPOUT_Z4 = tf.layers.dropout(inputs=Z4, rate=0.45, training=DRP_En)\n",
    "    Z5 = tf.contrib.layers.fully_connected(DRPOUT_Z4, 512)\n",
    "    DRPOUT_Z5 = tf.layers.dropout(inputs=Z5, rate=0.3, training=DRP_En)\n",
    "    Z6 = tf.contrib.layers.fully_connected(DRPOUT_Z5, 2, activation_fn=None)\n",
    "    \n",
    "    return Z6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_cost(Z6, Y):\n",
    "    # cost = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits = Z6, labels = Y))\n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits = Z6, labels = Y))\n",
    "    \n",
    "    return cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model(learning_rate = 0.00001, num_epochs = 250, print_cost = False, minibatch_size = minibatch_size):\n",
    "    \n",
    "    \n",
    "    ops.reset_default_graph()                         # to be able to rerun the model without overwriting tf variables\n",
    "    tf.set_random_seed(12)                             # to keep results consistent (tensorflow seed)\n",
    "    seed = 22                                          # to keep results consistent (numpy seed)\n",
    "                                        \n",
    "    costs = []                                        # To keep track of the cost\n",
    "    Testcosts = []\n",
    "    EpochIteration = []\n",
    "    ZACCplot = []\n",
    "    OACCplot = []\n",
    "    TACCplot = []\n",
    "    ACCplot = []\n",
    "    \n",
    "    X_val = np.zeros((1, imageFeatures.shape[1]*2)) \n",
    "    Y_val = np.zeros((1,2))\n",
    "    lastBestTestCost = -1.\n",
    "    \n",
    "    # Create Placeholders of the correct shape\n",
    "    X, Y, DRP_En = create_placeholders(imageFeatures.shape[1]*2, 2)\n",
    "     \n",
    "    # Forward propagation: Build the forward propagation in the tensorflow graph\n",
    "    Z6 = forward_propagation(X, DRP_En)\n",
    "    \n",
    "    # Cost function: Add cost function to tensorflow graph\n",
    "    cost = compute_cost(Z6, Y)\n",
    "    \n",
    "    # Backpropagation: Define the tensorflow optimizer. Use an AdamOptimizer that minimizes the cost.\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)\n",
    "    # optimizer = tf.train.RMSPropOptimizer(learning_rate).minimize(cost)\n",
    "   \n",
    "   \n",
    "    # SAVER ops\n",
    "    \n",
    "    PREDICT_op = tf.argmax(Z6, 1, name = 'PREDICT_op')\n",
    "    SOFTMAX_op = tf.nn.softmax(Z6, name = 'SOFTMAX_op')\n",
    "    CORRECT_PREDICTION_op = tf.equal(PREDICT_op, tf.argmax(Y, 1), name = 'CORRECT_PREDICTION_op')\n",
    "    ACCURACY = tf.reduce_mean(tf.cast(CORRECT_PREDICTION_op, \"float\"), name = 'ACCURACY')\n",
    "    # Initialize all the variables globally\n",
    "    init = tf.global_variables_initializer()\n",
    "     \n",
    "    # Add ops to save and restore all the variables.\n",
    "    saver = tf.train.Saver()\n",
    "\n",
    "    # Start the session to compute the tensorflow graph\n",
    "    with tf.Session() as sess:\n",
    "        \n",
    "        # Run the initialization\n",
    "        sess.run(init)\n",
    "        \n",
    "        # Do the training loop\n",
    "        for epoch in range(num_epochs):\n",
    "            EpochIteration.append(epoch)\n",
    "            minibatch_cost = 0.\n",
    "            minibatch_testCost = 0.\n",
    "            seed = seed + 1\n",
    "            np.random.seed(seed)\n",
    "            permutation4trn = list(np.random.permutation(ImgVtTrainSize))\n",
    "            for it in range(0, num_minibatches):\n",
    "                X_train, Y_train = getMiniBatch(it,permutation4trn)\n",
    "                _ , temp_cost = sess.run([optimizer, cost], feed_dict={X: X_train, Y: Y_train, DRP_En: True})\n",
    "                minibatch_cost += temp_cost / num_minibatches\n",
    "                \n",
    "            temp_testcost = cost.eval({X: gX_test, Y: gY_test, DRP_En: False})\n",
    "            minibatch_testCost += temp_testcost/num_Testminibatches\n",
    "            \n",
    "            \n",
    "            # build cost list\n",
    "            costs.append(minibatch_cost)\n",
    "            Testcosts.append(temp_testcost)\n",
    "            \n",
    "            print(\"Epoch %i  train cost %.3f test cost %.3f\" % (epoch,minibatch_cost,temp_testcost))\n",
    "            \n",
    "            \n",
    "            test_accuracy = ACCURACY.eval({X: gX_test, Y: gY_test, DRP_En: False})\n",
    "            if test_accuracy > lastBestTestCost:\n",
    "                lastBestTestCost = test_accuracy\n",
    "                # remove previuous best model\n",
    "                files = glob.glob(modelPath + '/*')\n",
    "                for f in files:\n",
    "                    os.remove(f)\n",
    "                # Save the variables to disk.\n",
    "                print(\"New Test Accuaracy: %.3f !!!\" % (test_accuracy)) \n",
    "                saver.save(sess, modelPath + '/model' + str(int((test_accuracy)*100)))\n",
    "               \n",
    "            \n",
    "                \n",
    "        # plot the cost\n",
    "        plt.plot(np.squeeze(EpochIteration), np.squeeze(costs), 'r-', np.squeeze(EpochIteration), np.squeeze(Testcosts), 'b-')\n",
    "        plt.ylabel('costs(r)-testcosts(b)')\n",
    "        plt.xlabel('epoch iteration')\n",
    "        plt.title(\"Learning rate =\" + str(learning_rate))\n",
    "        plt.savefig('nn4usp', format='pdf')\n",
    "        plt.show()    \n",
    "        \n",
    "   \n",
    "                \n",
    "        return costs,Testcosts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0  train cost 1.013 test cost 0.613\n",
      "New Test Accuaracy: 0.668 !!!\n",
      "Epoch 1  train cost 0.857 test cost 0.543\n",
      "New Test Accuaracy: 0.725 !!!\n",
      "Epoch 2  train cost 0.789 test cost 0.514\n",
      "New Test Accuaracy: 0.745 !!!\n",
      "Epoch 3  train cost 0.730 test cost 0.490\n",
      "New Test Accuaracy: 0.760 !!!\n",
      "Epoch 4  train cost 0.686 test cost 0.478\n",
      "New Test Accuaracy: 0.767 !!!\n",
      "Epoch 5  train cost 0.670 test cost 0.473\n",
      "New Test Accuaracy: 0.769 !!!\n",
      "Epoch 6  train cost 0.647 test cost 0.469\n",
      "New Test Accuaracy: 0.773 !!!\n",
      "Epoch 7  train cost 0.633 test cost 0.461\n",
      "New Test Accuaracy: 0.779 !!!\n",
      "Epoch 8  train cost 0.623 test cost 0.457\n",
      "New Test Accuaracy: 0.781 !!!\n",
      "Epoch 9  train cost 0.614 test cost 0.455\n",
      "New Test Accuaracy: 0.782 !!!\n",
      "Epoch 10  train cost 0.598 test cost 0.453\n",
      "New Test Accuaracy: 0.783 !!!\n",
      "Epoch 11  train cost 0.585 test cost 0.448\n",
      "New Test Accuaracy: 0.786 !!!\n",
      "Epoch 12  train cost 0.582 test cost 0.447\n",
      "New Test Accuaracy: 0.787 !!!\n",
      "Epoch 13  train cost 0.573 test cost 0.446\n",
      "New Test Accuaracy: 0.789 !!!\n",
      "Epoch 14  train cost 0.569 test cost 0.447\n",
      "Epoch 15  train cost 0.559 test cost 0.445\n",
      "New Test Accuaracy: 0.791 !!!\n",
      "Epoch 16  train cost 0.549 test cost 0.443\n",
      "Epoch 17  train cost 0.548 test cost 0.444\n",
      "New Test Accuaracy: 0.792 !!!\n",
      "Epoch 18  train cost 0.541 test cost 0.443\n",
      "New Test Accuaracy: 0.795 !!!\n",
      "Epoch 19  train cost 0.542 test cost 0.441\n",
      "New Test Accuaracy: 0.795 !!!\n",
      "Epoch 20  train cost 0.535 test cost 0.439\n",
      "New Test Accuaracy: 0.797 !!!\n",
      "Epoch 21  train cost 0.529 test cost 0.440\n",
      "Epoch 22  train cost 0.524 test cost 0.441\n",
      "Epoch 23  train cost 0.524 test cost 0.442\n",
      "Epoch 24  train cost 0.521 test cost 0.441\n",
      "Epoch 25  train cost 0.510 test cost 0.440\n",
      "New Test Accuaracy: 0.799 !!!\n",
      "Epoch 26  train cost 0.507 test cost 0.441\n",
      "Epoch 27  train cost 0.508 test cost 0.441\n",
      "New Test Accuaracy: 0.799 !!!\n",
      "Epoch 28  train cost 0.497 test cost 0.440\n",
      "Epoch 29  train cost 0.497 test cost 0.441\n",
      "Epoch 30  train cost 0.495 test cost 0.440\n",
      "Epoch 31  train cost 0.491 test cost 0.439\n",
      "New Test Accuaracy: 0.801 !!!\n",
      "Epoch 32  train cost 0.491 test cost 0.439\n",
      "Epoch 33  train cost 0.488 test cost 0.439\n",
      "Epoch 34  train cost 0.490 test cost 0.439\n",
      "Epoch 35  train cost 0.486 test cost 0.438\n",
      "New Test Accuaracy: 0.803 !!!\n",
      "Epoch 36  train cost 0.482 test cost 0.438\n",
      "Epoch 37  train cost 0.477 test cost 0.439\n",
      "New Test Accuaracy: 0.804 !!!\n",
      "Epoch 38  train cost 0.483 test cost 0.438\n",
      "Epoch 39  train cost 0.478 test cost 0.437\n",
      "Epoch 40  train cost 0.479 test cost 0.439\n",
      "Epoch 41  train cost 0.473 test cost 0.438\n",
      "New Test Accuaracy: 0.804 !!!\n",
      "Epoch 42  train cost 0.476 test cost 0.438\n",
      "Epoch 43  train cost 0.473 test cost 0.438\n",
      "New Test Accuaracy: 0.805 !!!\n",
      "Epoch 44  train cost 0.466 test cost 0.438\n",
      "Epoch 45  train cost 0.467 test cost 0.437\n",
      "New Test Accuaracy: 0.807 !!!\n",
      "Epoch 46  train cost 0.466 test cost 0.437\n",
      "Epoch 47  train cost 0.463 test cost 0.436\n",
      "New Test Accuaracy: 0.807 !!!\n",
      "Epoch 48  train cost 0.463 test cost 0.437\n",
      "New Test Accuaracy: 0.808 !!!\n",
      "Epoch 49  train cost 0.460 test cost 0.436\n",
      "New Test Accuaracy: 0.809 !!!\n",
      "Epoch 50  train cost 0.460 test cost 0.437\n",
      "Epoch 51  train cost 0.461 test cost 0.436\n",
      "Epoch 52  train cost 0.461 test cost 0.436\n",
      "Epoch 53  train cost 0.451 test cost 0.434\n",
      "Epoch 54  train cost 0.454 test cost 0.435\n",
      "Epoch 55  train cost 0.455 test cost 0.434\n",
      "Epoch 56  train cost 0.452 test cost 0.434\n",
      "New Test Accuaracy: 0.809 !!!\n",
      "Epoch 57  train cost 0.452 test cost 0.434\n",
      "New Test Accuaracy: 0.810 !!!\n",
      "Epoch 58  train cost 0.451 test cost 0.434\n",
      "Epoch 59  train cost 0.451 test cost 0.434\n",
      "Epoch 60  train cost 0.448 test cost 0.434\n",
      "Epoch 61  train cost 0.450 test cost 0.433\n",
      "Epoch 62  train cost 0.448 test cost 0.434\n",
      "Epoch 63  train cost 0.450 test cost 0.434\n",
      "New Test Accuaracy: 0.811 !!!\n",
      "Epoch 64  train cost 0.448 test cost 0.433\n",
      "Epoch 65  train cost 0.446 test cost 0.433\n",
      "Epoch 66  train cost 0.445 test cost 0.432\n",
      "New Test Accuaracy: 0.812 !!!\n",
      "Epoch 67  train cost 0.445 test cost 0.432\n",
      "Epoch 68  train cost 0.439 test cost 0.431\n",
      "Epoch 69  train cost 0.443 test cost 0.432\n",
      "New Test Accuaracy: 0.813 !!!\n",
      "Epoch 70  train cost 0.441 test cost 0.430\n",
      "New Test Accuaracy: 0.813 !!!\n",
      "Epoch 71  train cost 0.440 test cost 0.431\n",
      "Epoch 72  train cost 0.442 test cost 0.431\n",
      "Epoch 73  train cost 0.436 test cost 0.430\n",
      "Epoch 74  train cost 0.436 test cost 0.429\n",
      "Epoch 75  train cost 0.435 test cost 0.428\n",
      "Epoch 76  train cost 0.437 test cost 0.428\n",
      "Epoch 77  train cost 0.432 test cost 0.427\n",
      "Epoch 78  train cost 0.434 test cost 0.428\n",
      "Epoch 79  train cost 0.435 test cost 0.428\n",
      "New Test Accuaracy: 0.813 !!!\n",
      "Epoch 80  train cost 0.436 test cost 0.427\n",
      "New Test Accuaracy: 0.815 !!!\n",
      "Epoch 81  train cost 0.434 test cost 0.428\n",
      "New Test Accuaracy: 0.815 !!!\n",
      "Epoch 82  train cost 0.434 test cost 0.428\n",
      "Epoch 83  train cost 0.431 test cost 0.426\n",
      "Epoch 84  train cost 0.432 test cost 0.427\n",
      "Epoch 85  train cost 0.426 test cost 0.426\n",
      "New Test Accuaracy: 0.816 !!!\n",
      "Epoch 86  train cost 0.430 test cost 0.425\n",
      "Epoch 87  train cost 0.424 test cost 0.425\n",
      "Epoch 88  train cost 0.427 test cost 0.425\n",
      "Epoch 89  train cost 0.429 test cost 0.425\n",
      "Epoch 90  train cost 0.425 test cost 0.425\n",
      "Epoch 91  train cost 0.422 test cost 0.425\n",
      "Epoch 92  train cost 0.424 test cost 0.424\n",
      "Epoch 93  train cost 0.425 test cost 0.424\n",
      "Epoch 94  train cost 0.425 test cost 0.423\n",
      "New Test Accuaracy: 0.816 !!!\n",
      "Epoch 95  train cost 0.426 test cost 0.424\n",
      "New Test Accuaracy: 0.816 !!!\n",
      "Epoch 96  train cost 0.427 test cost 0.423\n",
      "New Test Accuaracy: 0.816 !!!\n",
      "Epoch 97  train cost 0.427 test cost 0.423\n",
      "Epoch 98  train cost 0.424 test cost 0.423\n",
      "Epoch 99  train cost 0.422 test cost 0.422\n",
      "Epoch 100  train cost 0.417 test cost 0.421\n",
      "Epoch 101  train cost 0.426 test cost 0.422\n",
      "Epoch 102  train cost 0.420 test cost 0.421\n",
      "New Test Accuaracy: 0.817 !!!\n",
      "Epoch 103  train cost 0.419 test cost 0.420\n",
      "New Test Accuaracy: 0.817 !!!\n",
      "Epoch 104  train cost 0.417 test cost 0.419\n",
      "New Test Accuaracy: 0.818 !!!\n",
      "Epoch 105  train cost 0.416 test cost 0.419\n",
      "New Test Accuaracy: 0.818 !!!\n",
      "Epoch 106  train cost 0.417 test cost 0.419\n",
      "Epoch 107  train cost 0.413 test cost 0.419\n",
      "Epoch 108  train cost 0.420 test cost 0.418\n",
      "New Test Accuaracy: 0.819 !!!\n",
      "Epoch 109  train cost 0.415 test cost 0.418\n",
      "Epoch 110  train cost 0.414 test cost 0.417\n",
      "New Test Accuaracy: 0.819 !!!\n",
      "Epoch 111  train cost 0.411 test cost 0.417\n",
      "New Test Accuaracy: 0.819 !!!\n",
      "Epoch 112  train cost 0.419 test cost 0.418\n",
      "Epoch 113  train cost 0.413 test cost 0.418\n",
      "Epoch 114  train cost 0.416 test cost 0.418\n",
      "New Test Accuaracy: 0.820 !!!\n",
      "Epoch 115  train cost 0.408 test cost 0.417\n",
      "Epoch 116  train cost 0.414 test cost 0.417\n",
      "Epoch 117  train cost 0.412 test cost 0.416\n",
      "Epoch 118  train cost 0.415 test cost 0.416\n",
      "New Test Accuaracy: 0.820 !!!\n",
      "Epoch 119  train cost 0.407 test cost 0.416\n",
      "New Test Accuaracy: 0.821 !!!\n",
      "Epoch 120  train cost 0.408 test cost 0.415\n",
      "Epoch 121  train cost 0.416 test cost 0.416\n",
      "New Test Accuaracy: 0.822 !!!\n",
      "Epoch 122  train cost 0.409 test cost 0.415\n",
      "Epoch 123  train cost 0.406 test cost 0.414\n",
      "Epoch 124  train cost 0.409 test cost 0.414\n",
      "Epoch 125  train cost 0.403 test cost 0.413\n",
      "Epoch 126  train cost 0.411 test cost 0.414\n",
      "New Test Accuaracy: 0.822 !!!\n",
      "Epoch 127  train cost 0.405 test cost 0.413\n",
      "New Test Accuaracy: 0.823 !!!\n",
      "Epoch 128  train cost 0.402 test cost 0.412\n",
      "Epoch 129  train cost 0.407 test cost 0.412\n",
      "Epoch 130  train cost 0.404 test cost 0.412\n",
      "Epoch 131  train cost 0.403 test cost 0.412\n",
      "Epoch 132  train cost 0.408 test cost 0.412\n",
      "New Test Accuaracy: 0.823 !!!\n",
      "Epoch 133  train cost 0.406 test cost 0.412\n",
      "Epoch 134  train cost 0.402 test cost 0.411\n",
      "New Test Accuaracy: 0.824 !!!\n",
      "Epoch 135  train cost 0.405 test cost 0.411\n",
      "Epoch 136  train cost 0.401 test cost 0.411\n",
      "Epoch 137  train cost 0.406 test cost 0.411\n",
      "Epoch 138  train cost 0.400 test cost 0.410\n",
      "Epoch 139  train cost 0.404 test cost 0.410\n",
      "Epoch 140  train cost 0.401 test cost 0.410\n",
      "Epoch 141  train cost 0.399 test cost 0.410\n",
      "Epoch 142  train cost 0.402 test cost 0.410\n",
      "Epoch 143  train cost 0.403 test cost 0.410\n",
      "Epoch 144  train cost 0.398 test cost 0.409\n",
      "Epoch 145  train cost 0.394 test cost 0.408\n",
      "Epoch 146  train cost 0.398 test cost 0.407\n",
      "Epoch 147  train cost 0.398 test cost 0.406\n",
      "New Test Accuaracy: 0.825 !!!\n",
      "Epoch 148  train cost 0.401 test cost 0.407\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 149  train cost 0.399 test cost 0.408\n",
      "Epoch 150  train cost 0.398 test cost 0.407\n",
      "Epoch 151  train cost 0.399 test cost 0.407\n",
      "Epoch 152  train cost 0.396 test cost 0.407\n",
      "Epoch 153  train cost 0.396 test cost 0.406\n",
      "Epoch 154  train cost 0.393 test cost 0.406\n",
      "Epoch 155  train cost 0.390 test cost 0.405\n",
      "Epoch 156  train cost 0.391 test cost 0.405\n",
      "Epoch 157  train cost 0.398 test cost 0.405\n",
      "Epoch 158  train cost 0.391 test cost 0.404\n",
      "Epoch 159  train cost 0.390 test cost 0.403\n",
      "Epoch 160  train cost 0.390 test cost 0.404\n",
      "Epoch 161  train cost 0.393 test cost 0.403\n",
      "Epoch 162  train cost 0.392 test cost 0.403\n",
      "Epoch 163  train cost 0.391 test cost 0.404\n",
      "Epoch 164  train cost 0.394 test cost 0.404\n",
      "Epoch 165  train cost 0.389 test cost 0.403\n",
      "Epoch 166  train cost 0.391 test cost 0.402\n",
      "Epoch 167  train cost 0.388 test cost 0.402\n",
      "Epoch 168  train cost 0.389 test cost 0.402\n",
      "Epoch 169  train cost 0.384 test cost 0.402\n",
      "Epoch 170  train cost 0.384 test cost 0.401\n",
      "Epoch 171  train cost 0.389 test cost 0.402\n",
      "Epoch 172  train cost 0.388 test cost 0.401\n",
      "Epoch 173  train cost 0.383 test cost 0.401\n",
      "Epoch 174  train cost 0.387 test cost 0.401\n",
      "Epoch 175  train cost 0.389 test cost 0.402\n",
      "Epoch 176  train cost 0.389 test cost 0.402\n",
      "Epoch 177  train cost 0.384 test cost 0.401\n",
      "Epoch 178  train cost 0.387 test cost 0.401\n",
      "Epoch 179  train cost 0.386 test cost 0.401\n",
      "Epoch 180  train cost 0.381 test cost 0.399\n",
      "New Test Accuaracy: 0.826 !!!\n",
      "Epoch 181  train cost 0.383 test cost 0.400\n",
      "Epoch 182  train cost 0.384 test cost 0.400\n",
      "Epoch 183  train cost 0.379 test cost 0.399\n",
      "Epoch 184  train cost 0.382 test cost 0.399\n",
      "Epoch 185  train cost 0.385 test cost 0.399\n",
      "Epoch 186  train cost 0.379 test cost 0.399\n",
      "Epoch 187  train cost 0.382 test cost 0.399\n",
      "Epoch 188  train cost 0.381 test cost 0.399\n",
      "Epoch 189  train cost 0.381 test cost 0.398\n",
      "Epoch 190  train cost 0.377 test cost 0.398\n",
      "New Test Accuaracy: 0.826 !!!\n",
      "Epoch 191  train cost 0.384 test cost 0.397\n",
      "New Test Accuaracy: 0.827 !!!\n",
      "Epoch 192  train cost 0.381 test cost 0.398\n",
      "Epoch 193  train cost 0.380 test cost 0.398\n",
      "Epoch 194  train cost 0.377 test cost 0.397\n",
      "Epoch 195  train cost 0.381 test cost 0.397\n",
      "Epoch 196  train cost 0.376 test cost 0.396\n",
      "Epoch 197  train cost 0.376 test cost 0.397\n",
      "Epoch 198  train cost 0.376 test cost 0.397\n",
      "Epoch 199  train cost 0.375 test cost 0.396\n",
      "Epoch 200  train cost 0.374 test cost 0.396\n",
      "Epoch 201  train cost 0.375 test cost 0.396\n",
      "Epoch 202  train cost 0.373 test cost 0.395\n",
      "Epoch 203  train cost 0.375 test cost 0.395\n",
      "Epoch 204  train cost 0.378 test cost 0.395\n",
      "Epoch 205  train cost 0.373 test cost 0.395\n",
      "Epoch 206  train cost 0.373 test cost 0.395\n",
      "Epoch 207  train cost 0.377 test cost 0.395\n",
      "Epoch 208  train cost 0.372 test cost 0.395\n",
      "Epoch 209  train cost 0.372 test cost 0.394\n",
      "Epoch 210  train cost 0.373 test cost 0.394\n",
      "Epoch 211  train cost 0.366 test cost 0.393\n",
      "New Test Accuaracy: 0.827 !!!\n",
      "Epoch 212  train cost 0.371 test cost 0.393\n",
      "Epoch 213  train cost 0.368 test cost 0.394\n",
      "Epoch 214  train cost 0.367 test cost 0.394\n",
      "Epoch 215  train cost 0.369 test cost 0.393\n",
      "Epoch 216  train cost 0.363 test cost 0.393\n",
      "Epoch 217  train cost 0.369 test cost 0.393\n",
      "Epoch 218  train cost 0.365 test cost 0.393\n",
      "Epoch 219  train cost 0.367 test cost 0.392\n",
      "Epoch 220  train cost 0.362 test cost 0.392\n",
      "Epoch 221  train cost 0.360 test cost 0.391\n",
      "Epoch 222  train cost 0.366 test cost 0.391\n",
      "Epoch 223  train cost 0.361 test cost 0.391\n",
      "Epoch 224  train cost 0.362 test cost 0.390\n",
      "Epoch 225  train cost 0.363 test cost 0.391\n",
      "Epoch 226  train cost 0.366 test cost 0.392\n",
      "Epoch 227  train cost 0.361 test cost 0.391\n",
      "Epoch 228  train cost 0.366 test cost 0.392\n",
      "Epoch 229  train cost 0.366 test cost 0.391\n",
      "Epoch 230  train cost 0.363 test cost 0.391\n",
      "Epoch 231  train cost 0.358 test cost 0.390\n",
      "Epoch 232  train cost 0.365 test cost 0.391\n",
      "Epoch 233  train cost 0.361 test cost 0.392\n",
      "Epoch 234  train cost 0.358 test cost 0.391\n",
      "Epoch 235  train cost 0.357 test cost 0.391\n",
      "Epoch 236  train cost 0.360 test cost 0.391\n",
      "Epoch 237  train cost 0.361 test cost 0.391\n",
      "Epoch 238  train cost 0.357 test cost 0.391\n",
      "Epoch 239  train cost 0.355 test cost 0.390\n",
      "Epoch 240  train cost 0.359 test cost 0.391\n",
      "Epoch 241  train cost 0.359 test cost 0.391\n",
      "Epoch 242  train cost 0.357 test cost 0.390\n",
      "Epoch 243  train cost 0.362 test cost 0.390\n",
      "Epoch 244  train cost 0.355 test cost 0.390\n",
      "Epoch 245  train cost 0.357 test cost 0.390\n",
      "Epoch 246  train cost 0.355 test cost 0.389\n",
      "Epoch 247  train cost 0.357 test cost 0.389\n",
      "Epoch 248  train cost 0.359 test cost 0.390\n",
      "Epoch 249  train cost 0.355 test cost 0.390\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmclWX9//HXhxl2hnVYlB0EEowUETfcfrmglisVpuZWZiVlfa3M3NKvZYumpllWLpW5myt93XLNVEABBVwQZRGBQTZZB2Y+vz8+9wzHYZYzw9xzZnk/H4/zOOfc5z73/bnnwP25r+u6r+syd0dERASgVa4DEBGRxkNJQUREyikpiIhIOSUFEREpp6QgIiLllBRERKSckoI0S2b2LzM7LddxiDQ1SgpSr8zsAzM7NNdxuPuR7n57ruMAMLNnzezrDbCfL5vZS2a2wcyerYftfdXMFpjZejN70My6Z3z2rJltMrN1yePtHd2fNA5KCtLkmFl+rmMo05hiAVYC1wJX7eiGzGwU8EfgVKA3sAH4fYXVznX3TsljxI7uUxoHJQVpMGb2BTObYWarkyva0RmfXWBm75nZJ2Y2x8yOz/jsdDP7j5n91sxWApcly140s9+Y2Soze9/Mjsz4TvnVeRbrDjaz55N9P2VmN5rZ36s4hoPNbLGZ/djMlgK3mlk3M3vUzIqS7T9qZv2S9a8EDgBuSK6ob0iWf8bMnjSzlWb2tpl9eUf/vu7+lLvfAyypIvZ9kr/7ajObaWYHV7O5k4FH3P15d18HXAycYGYFOxqnNG5KCtIgzGwMcAvwTaAHcRX6sJm1TVZ5jzh5dgF+BvzdzHbK2MTewHygF3BlxrK3gULgV8BfzMyqCKG6df8BvJrEdRlxdVydPkB3YCBwNvH/6Nbk/QBgI3ADgLv/FHiBbVfV55pZR+DJZL+9gJOA3ydX59sxs98nJ/LKHrNqiLVsG32Bx4D/TWI/H7jfzHpW8ZVRwMyyN+7+HlAMDM9Y5xdmtiJJ2AdnE4c0fkoK0lC+AfzR3V9x95Kkvn8zsA+Au9/r7kvcvdTd7wbeBcZlfH+Ju//O3be6+8Zk2QJ3/5O7lwC3AzsRVR2VqXRdMxsA7AVc4u7F7v4i8HANx1IKXOrum919o7t/7O73u/sGd/+ESFoHVfP9LwAfuPutyfG8BtwPTKxsZXf/trt3reIxurLvVOIUYIq7T0n+xk8C04Cjqli/E7CmwrI1QFlJ4cfAEKAvcDPwiJkNzTIWacSUFKShDAT+J/MqF+gP7AxgZl/LqFpaDexGXNWXWVTJNpeWvXD3DcnLTlXsv6p1dwZWZiyral+Zitx9U9kbM+tgZn9MGmXXAs8DXc0sr4rvDwT2rvC3OJkogaRlIPClCvscD+xkZgdkNBjPTtZfB3SusI3OwCcASXL/JEmMtwP/oeoEI01IY2okk+ZtEXClu19Z8QMzGwj8Cfg88F93LzGzGUBmVVBaw/l+BHQ3sw4ZiaF/Dd+pGMv/ACOAvd19qZntDrzOtvgrrr8IeM7dD8smQDP7A3GlX5kF7l5ptVMl+/ybu3+jis8rJtPZwOcyYhgCtAXeqeL7zqd/L2miVFKQNLQ2s3YZj3zipH+Ome1toaOZHZ00XHYkTipFAGZ2BlFSSJ27LyCqUS4zszZmti/wxVpupoBoR1htcdvmpRU+X0ZUtZR5FBhuZqeaWevksZeZ7VpFjOdk3OVT8VGeEMwsz8zaERd7rZK/fevk478DXzSzI8rWSxrN+1VxTHck6x+QtIFcDjzg7p+YWddkO+3MLN/MTgYOBB6v1V9NGiUlBUnDFOIkWfa4zN2nEe0KNwCrgHnA6QDuPge4GvgvcQL9LFEd0VBOBvYFPiYaYu8m2juydS3QHlgBvAz8X4XPrwMmJncmXZ+0OxwOTCLuFFoK/JK4Et8RpxJ/75uIRvuNRDLG3RcBxwIXEsl3EfBDqjgHuPts4BwiOSwnEt+3k49bE3+nouSYJwPHubv6KjQDpkl2RD7NzO4G3nL3ilf8Is2eSgrS4iVVN0PNrJWZTSCuqB/MdVwiuaCGZpG46+cBop/CYuBb7v56bkMSyQ1VH4mISDlVH4mISLkmV31UWFjogwYNynUYIiJNyvTp01e4e1XDmpRrcklh0KBBTJs2LddhiIg0KWa2IJv1VH0kIiLllBRERKSckoKIiJRTUhARkXJKCiIiUk5JQUREyikpiIhIuZaTFF58ES66CLZuzXUkIiKNVstJCi+/DFdeCRs31ryuiEgLlVpSMLNbzGy5mb1ZxedmZteb2Twzm2VmY9KKBYA2beK5uDjV3YiINGVplhRuAyZU8/mRwLDkcTYxW1R62iaTWm2uzYRaIiItS2pJwd2fB1ZWs8qxwF89vAx0NbOd0opHSUFEpGa5bFPoS8wTW2Zxsmw7Zna2mU0zs2lFRUV125uqj0REapTLpGCVLKt0xh93v9ndx7r72J49axz5tXIqKYiI1CiXSWEx0D/jfT9gSWp7U1IQEalRLpPCw8DXkruQ9gHWuPtHqe1N1UciIjVKbZIdM7sTOBgoNLPFwKVAawB3/wMwBTgKmAdsAM5IKxZAJQURkSyklhTc/aQaPnfgO2ntfztKCiIiNWo5PZpVfSQiUqOWkxRUUhARqZGSgoiIlGs5SUHVRyIiNWo5SUElBRGRGikpiIhIuZaTFFR9JCJSo5aTFFRSEBGpUctJCnl58VBSEBGpUstJChBVSKo+EhGpUstKCm3bqqQgIlINJQURESnXspKCqo9ERKrVspKCSgoiItVSUhARkXItKymo+khEpFotKymopCAiUi0lBRERKdeykoKqj0REqtWykoJKCiIi1VJSEBGRcqkmBTObYGZvm9k8M7ugks8HmtnTZjbLzJ41s35pxqPqIxGR6qWWFMwsD7gROBIYCZxkZiMrrPYb4K/uPhq4HPhFWvEAKimIiNQgzZLCOGCeu89392LgLuDYCuuMBJ5OXj9Tyef1S0lBRKRaaSaFvsCijPeLk2WZZgInJq+PBwrMrEfFDZnZ2WY2zcymFRUV1T0iVR+JiFQrzaRglSzzCu/PBw4ys9eBg4APga3bfcn9Zncf6+5je/bsWfeIVFIQEalWforbXgz0z3jfD1iSuYK7LwFOADCzTsCJ7r4mtYiUFEREqpVmSWEqMMzMBptZG2AS8HDmCmZWaGZlMfwEuCXFeKL6qLQUSkpS3Y2ISFOVWlJw963AucDjwFzgHnefbWaXm9kxyWoHA2+b2TtAb+DKtOIBoqQAKi2IiFQhzeoj3H0KMKXCsksyXt8H3JdmDJ+SmRQ6dGiw3YqINBUtq0dzmzbxrDuQREQq1bKSgqqPRESqpaQgIiLlWlZSUPWRiEi1WlZSUElBRKRaSgoiIlKuZSUFVR+JiFSrZSWF9u3jecOG3MYhItJItayk0LVrPK9Jb3glEZGmrGUlhS5d4nn16tzGISLSSLWspKCSgohItVpWUmjfHvLzVVIQEalCVgPimdlY4ABgZ2Aj8CbwlLuvTDG2+mcWpQWVFEREKlVtScHMTjez14i5DtoDbwPLgfHAk2Z2u5kNSD/MetSli0oKIiJVqKmk0BHY3903Vvahme0ODAMW1ndgqVFJQUSkStUmBXe/sYbPZ9RvOA2gSxclBRGRKmTV0GxmQ8zsETNbYWbLzewhMxuSdnCp6NpV1UciIlXI9u6jfwD3AH2IxuZ7gTvTCipVKimIiFQp26Rg7v43d9+aPP4OeJqBpUYNzSIiVaq2TcHMuicvnzGzC4C7iGTwFeCxlGNLR9eusG4dbN0afRZERKRcTWfF6UQSsOT9NzM+c+CKNIJKVdlQF2vXQvfu1a8rItLC1HT30eAd2biZTQCuA/KAP7v7VRU+HwDcDnRN1rnA3afsyD5rlDnUhZKCiMin1NR5bXwNn3c2s92q+CwPuBE4EhgJnGRmIyusdhFwj7vvAUwCfp9t4HVWVlJQY7OIyHZqqj460cx+BfwfUZVUBLQDdgEOAQYC/1PFd8cB89x9PoCZ3QUcC8zJWMeBzsnrLsCSOhxD7ZSVFNTYLCKynZqqj75vZt2AicCXgJ2IsY/mAn909xer+XpfYFHG+8XA3hXWuQx4wswmE72nD61sQ2Z2NnA2wIABOziqhkoKIiJVqvH2G3dfBfwpedSGVbKs4m2sJwG3ufvVZrYv8Dcz283dSyvEcDNwM8DYsWN37FZYlRRERKqUbY/m7yXtB2Zmfzaz18zs8Bq+thjon/G+H9tXD51FdIrD3f9LVE0VZhd6HamkICJSpWw7r53p7muBw4FewBnAVdV/hanAMDMbbGZtiIbkhyussxD4PICZ7UokhaIsY6obzb4mIlKlrHs0J89HAbe6+0wqrx4q5+5bgXOBx4k2iHvcfbaZXW5mxySr/Q/wDTObSQybcbq7p9tTOj8fCgpg1apUdyMi0hRl26V3upk9AQwGfmJmBUBpDd8h6XMwpcKySzJezwH2zz7cetK9O6xsWvMDiYg0hGyTwlnA7sB8d99gZj2IKqSmSUlBRKRS2VYfPenur7n7agB3/xj4bXphpaxbN1UfiYhUoqYB8doBHYDCpL9CWTtCZ2II7aape3eYPTvXUYiINDo1VR99EziPSADT2ZYU1hJDWDRN3bqp+khEpBI19Wi+DrjOzCa7++8aKKb0de8e1UfuYNXeRCUi0qJk26awNLnjCDO7yMweMLMxKcaVru7dobgYNmzIdSQiIo1KtknhYnf/JBk19QhiuOub0gsrZd26xbOqkEREPiXbpFCSPB8N3OTuDwFt0gmpAZTNo6A7kEREPiXbpPChmf0R+DIwxcza1uK7jY9KCiIilcr2xP5lYriKCUlfhe7AD1OLKm0qKYiIVCqrpODuG4D3gCPM7Fygl7s/kWpkaSpLCiopiIh8StZDZwN3ECOk9gL+nkyM0zSp+khEpFK1Gftob3dfD2BmvwT+CzTNvgudOsVoqao+EhH5lNoMnV2S8b6EGobObtTMNCieiEglsi0p3Aq8Ymb/TN4fB/wlnZAaiIa6EBHZTlZJwd2vMbNngfFECeEMd389zcBSV1gIK1bkOgoRkUYlq6RgZvsAs939teR9gZnt7e6vpBpdmvr0gblzcx2FiEijkm2bwk3Auoz362nKw1wA9O4NS5fmOgoRkUYl64bmzLmT3b2U7NsjGqc+faJNYfPmXEciItJoZJsU5pvZd82sdfL4HjA/zcBS16dPPC9fnts4REQakWyTwjnAfsCHwGJgb+AbaQXVIMqSgqqQRETKZVsFNMzdJ2UuMLP9gaLqvmRmE4DrgDzgz+5+VYXPfwsckrztQAyf0TXLmHaMkoKIyHayTQq/AypOqlPZsnJmlkdM2XkYUbqYamYPu/ucsnXc/fsZ608G9sgynh2npCAisp1qk4KZ7UtUG/U0sx9kfNSZuPqvzjhgnrvPT7Z1F3AsMKeK9U8CLs0m6HrRq1c8KymIiJSrqU2hDdCJSB4FGY+1wMQavtsXWJTxfnGybDtmNhAYDPy7is/PNrNpZjatqKjaGqvstW0bQ10oKYiIlKu2pODuzwHPmdlt7r4AwMxaAZ3cfW0N265sbCSvZBnAJOA+dy+p7EN3vxm4GWDs2LFVbaP2+vRRUhARyZDt3Ue/MLPOZtaRqP5528xqmmRnMdA/430/YEkV604C7swylvqjpCAi8inZJoWRScngOGAKMAA4tYbvTAWGmdlgM2tDnPgfrriSmY0AuhFDcTcsJQURkU/JNim0NrPWRFJ4yN23UHVVEADuvhU4l5jGcy5wj7vPNrPLzeyYjFVPAu7K7DHdYAYMgEWLYMOGBt+1iEhjlG1S+CPwAdAReD5pGK6pTQF3n+Luw919qLtfmSy7xN0fzljnMne/oPah14Px42HLFnj55ZzsXkSkscl2jubr3b2vux/lYQHbOp01XePHQ6tW8NxzuY5ERKRRyHaO5t5m9hcz+1fyfiRwWqqRNYQuXWCPPZQUREQS2VYf3Ua0DeycvH8HOC+NgBrcQQdF9dGmTbmOREQk57JNCoXufg9QCuWNyJX2KWhyxo+P4bNnzMh1JCIiOZdtUlhvZj1I7jhKZmJbk1pUDWn33eN51qzcxiEi0ghkOyDeD4g+BkPN7D9AT+BLqUXVkAYNgoICJQUREbJPCrOBg4ARxPAVb5N9KaNxM4PRo2HmzFxHIiKSc9me2P/r7lvdfba7v5l0Xmv4HshpGT06Sgo56D8nItKYVJsUzKyPme0JtDezPcxsTPI4mJgUp3n43Odg7VpYuDDXkYiI5FRN1UdHAKcTg9ldzbaRT9cCF6YXVgMbPTqeZ8yAgQNzG4uISA7VNHT27cDtZnaiu9/fQDE1vM99LuZXePZZOPbYXEcjIpIz2Q5zUZ4QzOzR9MLJkQ4d4JBDYMqUXEciIpJTdbmDqNLZ05q8I4+Ed96B997LdSQiIjlTl6Twer1H0RgcdVQ8q7QgIi1YrZOCu5+ZRiA5t8suMHIk3HFHriMREcmZGpOCme1rZjea2SwzKzKzhWY2xcy+Y2ZdGiLIBvPNb8Irr8C0abmOREQkJ2rqp/Av4OvECKkTgJ2AkcBFQDvgoQqzqDVtp50GHTvCjTfmOhIRkZyw6mbBNLNCd19R7QayWKc+jR071qeleSX/7W/DLbfAhx9Cjx7p7UdEpAGZ2XR3H1vTetWWFNx9hZnlmdlT1a1TlwAbre98J4bS/stfch2JiEiDq7FNwd1LgA1Nvf1g3jy4//4shjcaNQoOPhhuuglKSxsiNBGRRiPbu482AW8kU3JeX/ZIM7D69sADMHEirFuXxcqnnw4ffABvvJFyVCIijUu2SeEx4GLgeWB6xqNaZjbBzN42s3lmdkEV63zZzOaY2Wwz+0e2gddWl6ScsyabqYH+3/+L52eeSSscEZFGKav5FJIxkGrFzPKAG4HDgMXAVDN72N3nZKwzDPgJsL+7rzKzXrXdT7bKksLatVms3L8/DB0K//43nNc8pqIWEclGTbekPmJmXzSz1pV8NsTMLjezqjqzjQPmuft8dy8G7gIqjjb3DeBGd18F4O7La38I2alVSQFiLKTnn4eS5jEVtYhINmqqPvoGcADwlplNTTqt/dvM3gf+CEx391uq+G5fYFHG+8VsP27ScGC4mf3HzF42swmVbcjMzjazaWY2raioqMaDqkydksKaNfDaa3Xan4hIU1TT0NlLgR8BPzKzQUTntY3AO+6+oYZtWyXLKt77kw8MAw4m5mx4wcx2c/fVFeK4GbgZop9CDfutVK2TwmGHQatW8MgjsNdeddmliEiTk1VDs5l1BBa6+3+BDcChlVUpVbAY6J/xvh+wpJJ1HnL3Le7+PjH387CsIq+lWieFnj1h/Hh48ME0whERaZSyvfvoeaCdmfUFngbOAG6r4TtTgWFmNtjM2gCTgIcrrPMgcAhEz2iiOml+ljHVSq2TAsDxx8dtqRpOW0RaiGyTgiXVRScAv3P344kxkKrk7luBc4lxk+YC97j77KRxumy8pMeBj81sDvAM8EN3/7guB1KTTp2iNqhWSeG44+J58mRYujSNsEREGpWsbkkFzMz2BU4Gzsr2u+4+BZhSYdklGa8d+EHySJUZdO5cy6QwaBBcey1ccAGccAL85z+xIRGRZirbpPA9oj/BP5Or/SHElX2T0qVLlv0UMn3ve5CfD+eeCy+9BPvvn0psIiKNQbbVR73d/Rh3/yWAu88HXkgvrHR06VLLkkKZ00+Hbt3g17+u75BERBqVbJPCT7Jc1qjVOSl07Ajf/z489FAMlCci0kxVW31kZkcCRwF9KwyA1xnYmmZgaejSJaZJqJMLL4xZ2SZPjlFUd921PkMTEWkUaiopLAGmEaOkZg6E9zBwRLqh1b86lxQA8vLg1lvjWTOziUgzVVOP5pnATDP7h7tvATCzbkD/svGKmpJa331UUc+eMGkS3H47/PznsUERkWYk2zaFJ82ss5l1B2YCt5rZNSnGlYqykkKNE+1UZ/LkmJTh9loPHCsi0uhlmxS6uPtaovPare6+J3BoemGlo0sX2LoVNm7cgY2MHQt77w033KCZ2USk2ck2KeSb2U7Al4FHU4wnVXUa6qIykyfDO+/E3UgiIs1ItknhcmJIivfcfWrSee3d9MJKR60m2qnOxIkxEc8JJ8TrHaqPEhFpPLJKCu5+r7uPdvdvJe/nu/uJ6YZW/+qtpNC2LUydCj/4Adx/P9x5JyxYsMPxiYjkWrZDZ/czs3+a2XIzW2Zm95tZv7SDq2/du8fzihX1sLHeveFXv4IxY+Dkk2OcpHvuqYcNi4jkTrbVR7cSfRN2JmZPeyRZ1qQMGRLP8+bV0wbz8uIupHPPhVGj4Pzzd7AVW0Qkt7JNCj3d/VZ335o8bgN6phhXKnr1goKCekwKALvtBr/7XXRoW7QIrriiHjcuItKwsk0KK8zsFDPLSx6nAKnMe5AmMxg2DN5No4n8oIPgzDPhqqvg2WdT2IGISPqyTQpnErejLgU+AiYSs681ObvsklJSALj++sg6p5wCHze5nCkiknVSuAI4zd17unsvIklcllpUKRo2DD74ALZsSWHjHTvGnUjLl8Phh8MPfwiLF6ewIxGRdGSbFEZnjnXk7iuBPdIJKV3DhkFJCbz/fko7GDMGbr45OkNcfz185jPw6qsp7UxEpH5lmxRaJQPhAZCMgZTtrG2NyrBh8ZxaFRLEpDzvvgtz50Lr1pEcRESagGxP7FcDL5nZfYAT7QtXphZVihokKZQZMgS+/GX4+99jEL1OnRpgpyIidZdtj+a/AicCy4Ai4AR3/1uagaWlsDA6sc2d20A7POUU2LAB7r23gXYoIlJ32VYf4e5z3P0Gd/+du8/J5jtmNsHM3jazeWZ2QSWfn25mRWY2I3l8vTbB14VZ9DN7882095TYf38YORK+8Q0YMQL69q2HwZdERNKRdVKoLTPLA24EjgRGAieZ2chKVr3b3XdPHn9OK55Mu+0Gs2c30Dh2rVrBiy/Ct74VQ2MsWRLjJYmINEKpJQVgHDAvGTyvGLgLODbF/WVt1KgYFK/O8zXXVrdu0ev5ueeio8Rf/9pAOxYRqZ00k0JfYFHG+8XJsopONLNZZnafmfWvbENmdraZTTOzaUVFRTsc2G67xfPs2Tu8qdoxg699LXo8n3QSPPight0WkUYlzaRglSyreAZ8BBjk7qOBp4BK57h095vdfay7j+3Zc8eHXBo1Kp4brF0h05lnwr77Rqnh+ONjPoZ163IQiIjI9tJMCouBzCv/fsCSzBXc/WN335y8/ROwZ4rxlCssjOr9nCSFvn3hpZdg4UL45S+jtDBkCBx8MLzxBjz1VIo960REqpdmUpgKDDOzwWbWBphEDL9dLpnis8wxQEPdKMq4cfDvf+ew9iY/H370I3jiCZgwAd5+Gz73OTjsMNh9d3jssRwFJiItWWpJwd23AucS03jOBe5x99lmdrmZHZOs9l0zm21mM4HvAqenFU9FEyfGxXrOR6D4/Oej4Xnq1Jis54YbojH6uOO2zQG9YAEUF+c2ThFpEcybWEPn2LFjfdq0aTu8ndWrowpp8mT4zW/qIbD6tHZtlBhmzIA//AHOPjve33cfbN4cdzOJiNSCmU1397E1rZdm9VGj1rUrHHEE3HFHPU3PWZ86d47qox49omG6TRv4178iGQwdGm0Os2alNNSriLRkLTYpAFx8MaxaFVVJja52prAwqpUKC6OEcNllUb1UWgp77BHtDwcd1ICdLUSkJWjRSWGvveAvf4m7QydPboRdBg49FJYtgyOPhEsvhVtugdtugy5d4Hvfi7uVJk6MRCEiUg+a5PDX9enkk+PW1KuuggMOiPHrGpVWFfL2ccfFA2LuhtNOi4w2bBjssw/svXd0khMRqYMW29CcqbQ0zqcffgjvvBMTqDUJ7lGa+Pe/ty374hcju+21V/R9EBFBDc210qoVXHNNjFV34YWNsBqpKmbRIL1oUQT/61/Dk09G/4dDDoHvfCeqnfbYAw48EJ5+OtcRi0gjp5JChsmTo5vA+efDJZdAQUEqu0nXJ5/Apk3w4x9H+4M7jB8fdyy1bx+d5CpWSYlIs6eSQh1cdx2cdVb0Wxg0CH7xiyZ412dBAfTsGY3SmzbBypXwwgtxUPPmxUGdfHK0R9x+e5Qy1q/PddQi0kiopFCJV16BK66ImpnzzoPf/jbV3TWMLVsi0y1ZEre59umzbfCnTp1iMqD8/EgaM2bEeEzDh8dQsvvtF30lRKTJyrak0OLvPqrM3nvDo4/GXZ/XXhu9nydMgHbt4nx52mlxfm1SWreGP/0JZs6MerL27eGee6K66eWX48AWLoTRo7d9p23b6EHdqxf87GdwxhmxTESaLZUUqlFcDOecAw88EJPylGnTBs49N7oLbNkCkyZtG467SVu0CH7608iA774Ly5dHA/VNN0VnjjZtIpnsskuM5jptGowdG93DRaRRy7akoKSQha1b4/y3dm10B7jiimjDhWizLSmJKvpJk+K8+frrUQX1wQdxTu3bNy7Ily2LRDN8eCSZPn2iY/JnPhM1OgUFcbHep0+ss3kzzJ8fr/OTMl1p6bZ24rKfLvVuCe7wf/8XkwOtWQN//vO2qUULC6MD3Zw5kUheeqkJFqNEmj8lhZQtXLitJuWuu+Bvf4Pp0+N9z57R76Ffv2jjXb06+j706RMn9HffjVLGhx/GZ5Xp0CFKIVu2RO3NyJGRZBYtgu7dYePGaB8uKIDBg+M8PHgwDBwY+8jLi+quzZtjKKVu3eKivnfvOMfvUCK56ir4yU/gBz+INoepU2MHy5fDZz8LX/96DP/97ruwdGlUV5nFjhctij9c7947EICI1JaSQg6Une969szupOsOixfHTUGrVsVF+M47x92j774bpY6hQ6Nv2sKFcR4dPjxuKOrQIRLCqlWRLN5/Px4bNlS9v/z8SEwffxzzSfTuHdsoKIhYVq2Kz/v2jbg+/jhqhrp1i8Sy555RqunQAdqvXUZpz96fvrv1jjsq7xI+eTLMnRvJY82aCOSww2JDl1wS2Sw/P6qmRCQVSgotkHskDIgqr82bI0mtXQtFRTHJ25IlcZKfOjWSwLp82/bwAAAUG0lEQVR1UbUFsXzZsvhe69aR3Fav3j7R5OdHspo3LxJF2XodOsDoQWsZ85kN7LflOQYNzYM774xGmf794QtfiJLE229HJ7ulS2NnJSUxcfbDD8Pjj0diyc+PwLZu1VDhIvVASUHqxD2GEi8oiOoniCqssu4OS5bEY/bsaG/+z3+inaRHjygEvPlmnOch7nTt17eUYQXLWNWmN1tLW9GlSyScjz6CtSu3cLQ/Sp+O6+g9+2n2az+Dgo3L6HPx2dimjXDTTZS0aY8/8E/yD9p/W5DFxVFV9d57MRHR+vXRrnHYYdvWmTkzbqttkj0QReqfkoLkxJYt0eb8wgtxzn7//Xju0SOqw1atiov/nXaKto/HH4/3mTqwnjxK2NSqA1tKo4W9V8EG+rX/mL4b59Fv3VsUehF5lJCXb7TKa0Xf4vf57BljGXne4bRrbzBiRAz18cQT6sEtgpKCNBFr18Y5e967zqzXtvDJimLm/eEp2HVX2u8+gnalGyj9x10sWbSVD60/izuNYPHWPqzc2KHS7bWmmF07LmT5+o60YxOjBq3nkK/0ZrdDetK7d9wsVVgI7datgGeeiRJGZgPQDrfCizROSgrSfGzaBPfeG9VDffoAce52j+aIrVtjGus3nlnB9Iv+yayVfekzrIDiLa145YNezGPYdpvsmLeRHiXL6dmvHbvs35tOneDNJz7kvSXt6TqgMz26lrD7Xm0YOMjo0yfuJFuxIu7yGjUKPn5/LYumLWPkscMoLGzgv4dIHSgpSMv09tsxGOA110SbwuLFLP/lrbx1y0us2NCeFa16s6K0Gx/TgxUdBrJsQwHz2o1iY9uuDFg7m8/566xt34dlGwuY3m5/1mxqV+Mue/eO9vMRI6LhfvHimPJiwoS4k6tNm7hFWCSXlBREMpWURONGnz5w6qlxv+0jj8CVV8b813PmRAv4T38a/TB698bXb2BTSWsWr+rAh/SlkI95f+TRvHngtyn4w6/YhXnMGX48b+x3Nm+8tpV5C/JpbSUU2greWtXnU7tv1SqSQ+YjPz/u2OrfPzol9u69rSPjsmVRMunfP2qzOnaMJDN0aI7+ftLkNYqkYGYTgOuAPODP7n5VFetNBO4F9nL3as/4SgpSLzLbDjZvjv4S/fvH+CUlJTEe1PjxcQvWfffFuvPnR5+LHj3i/fe/H0nkwAOjC/u++8ZttNOnM/fs3/LCnuexcup7FG92ivsPpbhoDVseeITiQcMoHr0XxSV55SWLPLayZFkeCxZsa8/o0uXTw6tAlEbGjIkqrPXr4a234masAQOiT0m7dvE8cOC2R9euaiaRRjAgnpnlATcChwGLgalm9rC7z6mwXgHwXeCVtGIR2U7mWbJtW/jlL7e9z8uLUWMvuijqhY4+OpYXF0eponXrGD73wAPj9YUXxnrPPhvrjRjBrn85n13bvQ+33hhJ5pBDorPIundh+mbIGxdJ5OV/xbbOPx9OOYUNhx/H6ikv0fOK79J6556sXryOVs89Q5EX8tjKfXnyyRhJ5M47o6QxfHiUOl56KaquKhvqvaAgSh89ekSYI0du61fSu3ckl8WL47AHDIjPhwyJDoudOn266kvt8M1faiUFM9sXuMzdj0je/wTA3X9RYb1rgaeA84HzVVKQJmf9+jjDfu1r0eL94INw0klxO+yee8br3/0uurw/+GCcuU89NXoFdu8enUDat4+xS/LyIokUFkaiuOaa6JPRujW8+GJ0RQc+WVNK/v9eRvsXn4wBC0ePhnnzKOnTl483dWThwgil7LFyJSxd6sycaRQVRXVWaem2Q2jbdlujfUVdukQ4mzZFXhsxItpK1q2LPiurVsV63btH4mnXLpLLxImRRNavj9JKr17betB36RKFqvz8+J7aXNKX8+qjpEpogrt/PXl/KrC3u5+bsc4ewEXufqKZPUsVScHMzgbOBhgwYMCeCxYsSCVmkR1WNmKhe1QpjRwZl9ylpdGO0bNnrDd3bjQa7LJLjM9+zjkx+VFpKVx9dYwr9eqrMQLjtdfCt78d33v66Zjz4pVXoqdgp05xds7PjzP6gAHRJjJ8eCSkMr/+Ndx9N/7c86zc1IHOnaPWbOXKOImXhbV4cYS2YEGctNeujXVWrIhd9OwZbflLl8aud945EkHZMCkrVsR2Z8zYvuqrKmZRYtl11+i8vm5d5MHi4rjba8iQGAll48bYZocO0cbSoUM8Bg2KXFk2t3rmoJGyTWNICl8CjqiQFMa5++TkfSvg38Dp7v5BdUkhk0oK0mxt3RqXzGZx2f7YY1FF1bVrDNN78MFxxi0thc9/Ho45Jobmve22OIsOGBAz7JVdNH3nO5Fgli6Ny/vNm6Nh/cIL4/O1a6N34eDBsc+f/Sy2eeCBO3wo69dHYig7cZcljLJEs2ZNlBi2bo3lixZFslm9OpJN797xp/jgg2jKWbs2ttuuXZRYKtOmTfzZSkqiJNK2bSS07t239U8ZPDiq0CCq3EpKot1lwID487hHoatv3/h+jx7ld0E3eY0hKVRbfWRmXYD3gHXJV/oAK4FjqksMSgrSYj31FJx9dpQUvvKVytdZvz4u9e+8M6qexoyJ5XPmxOX09OnwpS/FbUw33BC3ObVqFWe/oqIo1Zx6apy9f/GLWJ45sVLFy/DS0jjjd+qU2gRMZaWQtm2jNFBaGqWGDRuiVDF7NsyatW2sxfz8KJQVF0dCWL06Ek9RUTTML18e2x0yJMKeN6/6gSQLC2O91q2j4NavX/w5p0+PktKWLdtKOx06RFJ54414fPJJ/Em7do1EVfYoKIiENH9+fN6/f+yjuDiSVLt28bzzzvGTllUDHnXUpwuAtdEYkkI+8A7weeBDYCrwVXefXcX6z6KSgkj9eeihGMa8c2e47DI44IAoPUyfHslg1Ci44II4U06fHtVXP/xhnD1bt46znXuULn7842h4v/rq+N7tt0f11cUXx1m1Y0e4//4oeWzeHA0KfftWHlcOW6vdI3kUF0cbB8TJed26OBGXlMBrr0UJY/PmbeN8bdoU72fNij/PsGExFP3y5du+V1QU23799agKGzcuksGaNds/1q2LP8GgQZE4Fi3a1qRUXbXb738P3/pW3Y4950khCeIo4FriltRb3P1KM7scmObuD1dY91mUFETqV1UzMa1cGZesFVt4ly6NM9zatTGZ0ty5ccfV/vvH6IdHHx1VWWaRWPbYIxrSy5JEme7dY2KmGTOigfzMMyMp/fzncOONUTVWVoops3UrnHVW3JX1zW822ducdjTnrV0bCWjevCjhlJUaBgzYNkhlXTSKpJAGJQWRBrR5c5zQ33oLDj88Sg0vvRS32O67b4xo2L59tFCfdVa0cey5Jxx6aFw6Q9TndO0K++0Xw6O3aROV+4WFUTeyyy5xydyvX9yeC9FT75RTYp/XXBPzgw8fnrM/Q3OgpCAi6Zk/Pyq8q7p0fe01uPVWOO20qCwfOzbqR37zm6hbOfLIuPQdMSLqTt55JxoL9tkn5t24+eaYWapNm6jrGTo0buvdsiWqr4YM2f5yfOpUOP30qNKaNCnqdl59NUo3TbTUUZ+UFESk8Xj11XhO+lkwZ06UFspm23vwweiX8be/RQnEPWby+8c/4Pjjo6d5cfG27fXpE+stXRqttsOHx11Y69bFNi+9FK6/PhoFfvrTmD728cejnWPvvRv00BsLJQURaT7mzo0TfKdO0ZL74osxFEmfPjFj04cfRpXTpZfGaIQLF0Zr8JgxcPfd27ZTWBglmJ//PPp/jBsXyWbLlmgsLy6O0kkzpKQgIi2DezRSt24d7zdtitbaHj3i/QMPRFLp3j3GrirTqVM0kj/2WHz3f/83bvn9+c+jXaRLl7i/FKK6bODAbQ3zW7ZEj/XDDotqr1mzooRS1gmiEco2KeDuTeqx5557uohInZxxhnvHju5PPOF+9NHunTu7jxvn3rp1TNHRtm3ZVB3uBQXus2a5P/usu5n7pEnuJSXub77pPn58rHPGGe6jRm37zrnnum/e7H7ssfF669ZcH3E54q7PGs+xKimISMtRUhI94SrOjHTTTTEo4pQp0Tax884xNMiWLdEjbcWKuFOqbKyqDh3iFtvHH49Sxs03x+23110XjeUvvxzbnTgxtt21azSUv/BC3MZ74YVR6igtje127BidEK67LgZWrNjHY9OmHbsfFVUfiYjUTsUOBnPnxu24r7wSyeKtt2LZyJHw1a9Gp8BzzoEvfhFOOCG+c+aZ0WYxblwkhAsvjBP+gAHRxXnQoBi74+ijowH9Rz+KJDN4cHRZLi2FY4+FI46A3XeP9o2LL4Ynn4x+I6edVufDU1IQEdlR7tGQvfPO2a2/bl30/v7Wt2C33SIR/OY30eHv/POjv8UNN0Qy2Lgx2i5OPDEmfMrLizk8rqow7Uz37pFMXnstOgl+7Wt1OhQlBRGRxuqjj6LH90knfbpaqLg4SgT77BNjXblHgmnTJp4vuCD6dtSBkoKIiJTLNilo1HERESmnpCAiIuWUFEREpJySgoiIlFNSEBGRckoKIiJSTklBRETKKSmIiEi5Jtd5zcyKgAV1/HohsKIew2kKWuIxQ8s8bh1zy1DXYx7o7j1rWqnJJYUdYWbTsunR15y0xGOGlnncOuaWIe1jVvWRiIiUU1IQEZFyLS0p3JzrAHKgJR4ztMzj1jG3DKkec4tqUxARkeq1tJKCiIhUQ0lBRETKtZikYGYTzOxtM5tnZhfkOp60mNkHZvaGmc0ws2nJsu5m9qSZvZs8d8t1nDvCzG4xs+Vm9mbGskqP0cL1ye8+y8zG5C7yuqvimC8zsw+T33qGmR2V8dlPkmN+28yOyE3UO8bM+pvZM2Y218xmm9n3kuXN9reu5pgb7rd292b/APKA94AhQBtgJjAy13GldKwfAIUVlv0KuCB5fQHwy1zHuYPHeCAwBnizpmMEjgL+BRiwD/BKruOvx2O+DDi/knVHJv/G2wKDk3/7ebk+hjoc807AmOR1AfBOcmzN9reu5pgb7LduKSWFccA8d5/v7sXAXcCxOY6pIR0L3J68vh04Loex7DB3fx5YWWFxVcd4LPBXDy8DXc1sp4aJtP5UccxVORa4y903u/v7wDzi/0CT4u4fuftryetPgLlAX5rxb13NMVel3n/rlpIU+gKLMt4vpvo/dFPmwBNmNt3Mzk6W9Xb3jyD+0QG9chZdeqo6xub+25+bVJXcklEt2OyO2cwGAXsAr9BCfusKxwwN9Fu3lKRglSxrrvfi7u/uY4Ajge+Y2YG5DijHmvNvfxMwFNgd+Ai4OlnerI7ZzDoB9wPnufva6latZFmTPO5KjrnBfuuWkhQWA/0z3vcDluQollS5+5LkeTnwT6IouaysGJ08L89dhKmp6hib7W/v7svcvcTdS4E/sa3aoNkcs5m1Jk6Od7j7A8niZv1bV3bMDflbt5SkMBUYZmaDzawNMAl4OMcx1Tsz62hmBWWvgcOBN4ljPS1Z7TTgodxEmKqqjvFh4GvJnSn7AGvKqh6augr15ccTvzXEMU8ys7ZmNhgYBrza0PHtKDMz4C/AXHe/JuOjZvtbV3XMDfpb57q1vQFb9Y8iWvLfA36a63hSOsYhxJ0IM4HZZccJ9ACeBt5NnrvnOtYdPM47iSL0FuJK6ayqjpEoXt+Y/O5vAGNzHX89HvPfkmOalZwcdspY/6fJMb8NHJnr+Ot4zOOJqpBZwIzkcVRz/q2rOeYG+601zIWIiJRrKdVHIiKSBSUFEREpp6QgIiLllBRERKSckoKIiJRTUhCphJkdbGaPZrHe5WZ2aPL6PDPrUI8xHGdmIyvbl0hadEuqSCXM7GBiVMov1OI7HxD3xq+oxXfy3L2kis9uAx519/uy3Z7IjlJJQZosMzvFzF5Nxpf/o5nlJcvXmdnVZvaamT1tZj2T5bub2cvJoGL/zBiHfxcze8rMZibfGZrsopOZ3Wdmb5nZHUlv04ox3GZmE83su8DOwDNm9kzy2eFm9t9km/cm49mUzXlxiZm9CHzJzL5hZlOT/d9vZh3MbD/gGODXyfENLdtXso3Pm9nrFnNn3GJmbTO2/bNkn2+Y2WfS/A2k+VFSkCbJzHYFvkIMALg7UAKcnHzcEXjNY2DA54BLk+V/BX7s7qOJ3qFly+8AbnT3zwH7ET2HIUaoPI8Ys34IsH9V8bj79cSYM4e4+yFmVghcBByaxDEN+EHGVza5+3h3vwt4wN33SvY/FzjL3V8ieq7+0N13d/f3Mo69HXAb8BV3/yyQD3wrY9srkn3eBJxf7R9SpAIlBWmqPg/sCUw1sxnJ+yHJZ6XA3cnrvwPjzawL0NXdn0uW3w4cmIwV1dfd/wng7pvcfUOyzqvuvthjELIZwKBaxLcPkUz+k8R3GjAw4/O7M17vZmYvmNkbRGIbVcO2RwDvu/s7mceS8XnZwHHTaxmzCPm5DkCkjgy43d1/ksW61TWcVTb0cJnNGa9LqN3/FwOedPeTqvh8fcbr24Dj3H2mmZ0OHJzFtqtTFndtYxZRSUGarKeBiWbWC8rn7S27Em8FTExefxV40d3XAKvM7IBk+anAcx5j1S82s+OS7bTdgTuIPiGmUAR4GdjfzHZJttvBzIZX8b0C4KNkyOSTM5Znbi/TW8Cgsm2XHUsdYxb5FCUFaZLcfQ5RZ/+Emc0CniTmt4W4Ch9lZtOB/wdcniw/jWi4nUVMVlK2/FTgu8nyl4A+dQzrZuBfZvaMuxcBpwN3Jtt9Gaiq0fdiYnatJ4kTfpm7gB8mDcpljd+4+ybgDODepMqpFPhDHWMW+RTdkirNjpmtc/dOuY5DpClSSUFERMqppCAiIuVUUhARkXJKCiIiUk5JQUREyikpiIhIOSUFEREp9/8BuZxIDPqfNmAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "costs,Testcosts = model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./transfer4uspVGG16NonEQUVerify_Jul_0518/model82\n",
      "Confusion matrix, without normalization\n",
      "[[2140  414]\n",
      " [ 360 2195]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVoAAAEmCAYAAAAjsVjMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XucXdP9//HXeyZECEnaJBqJoO5CJe6NIlUUde2v1P1OKb6lqtXSr1u1vi2qLnUrRVtB664I1bonqYQgKiEhyIUkVNwicvn8/lhr4mTMnHMmmTNzzuT9zGM/cvbaa++9zuwzn7Nm7bXXUkRgZmaVU9feBTAz6+gcaM3MKsyB1syswhxozcwqzIHWzKzCHGjNzCrMgbYFJHWRdI+kWZL+ugTHOVDSg61ZtvYiaRtJ46vlfJJWlxSSOrVVmWqFpEmSdsivfybpDxU4x5WSft7ax6116oj9aCUdAPwQWA/4ABgDnBcRTyzhcQ8GTgQGR8S8JS5olZMUwNoRMaG9y9IcSZOAoyLiH3l9deA1YJnWvkaSrgcmR8QZrXncttL4Z9UKxzssH+9rrXG8jqzD1Wgl/RC4GPglsDLQH/g9sGcrHH414OWlIciWw7XGyvHPtoOJiA6zAN2AD4F9iuTpTArEU/NyMdA5bxsCTAZOAaYD04DD87azgU+BufkcRwJnAX8uOPbqQACd8vphwKukWvVrwIEF6U8U7DcYeBqYlf8fXLDtEeBc4Ml8nAeBns28t4by/7ig/HsBuwIvA+8CPyvIvwUwHHgv570MWDZveyy/l4/y+/1uwfF/ArwF/KkhLe+zZj7HJnl9FWAmMKSMa3cDcEp+3Tef+/t5fa18XDU635+ABcDsXMYfF1yDQ4E38vlPL/P6L3Jdclrk8x+Tr/2n+Vz3NPM+AjgWeAX4L3A5n/3lWAecAbyer8+NQLdGn50jc7kfK0g7HHgzH+9YYHPg+XzdLis495rAP4F38vv+C9C9YPskYIf8+izyZzdf9w8LlnnAWXnbacBE0mfvP8DeOX194BNgft7nvZx+PfCLgnMeDUzI1+9uYJVyflYdbWn3ArTqm4Gd84ekU5E85wAjgN5AL+Ap4Ny8bUje/xxgGVKA+hjo0fjD2cx6wy9GJ2AF4H1g3bytDzAgvz6M/AsNfCF/yA7O++2f17+Ytz+SP+jrAF3y+vnNvLeG8v9vLv/RwAzgJmBFYED+5fhyzr8psFU+7+rAS8BJjX4R1mri+P9HClhdKAh8Oc/R+TjLA8OAC8q8dkeQgxdwQH7PtxRsu6ugDIXnm0QOHo2uwTW5fBsDc4D1y7j+C69LUz8DGgWRZt5HAPcC3Ul/Tc0Adi54HxOALwNdgduBPzUq942kz06XgrQrgeWAnfL1uzOXvy8pYG+Xj7EWsGO+Nr1Iwfripn5WNPrsFuQZmMs8KK/vQ/rCrCN92X4E9Cny81r4MwK2JwX8TXKZLgUeK+dn1dGWjtZ08EVgZhT/0/5A4JyImB4RM0g11YMLts/N2+dGxH2kb+t1F7M8C4ANJXWJiGkR8WITeb4FvBIRf4qIeRExFBgH7F6Q548R8XJEzAZuJf0yNGcuqT16LnAz0BP4XUR8kM//IvAVgIgYHREj8nknAVcB25Xxns6MiDm5PIuIiGtINZSRpC+X00scr8GjwDaS6oBtgV8DW+dt2+XtLXF2RMyOiOeA50gBF0pf/9ZwfkS8FxFvAP/is+t1IHBRRLwaER8CPwX2a9RMcFZEfNToZ3tuRHwSEQ+SAt3QXP4pwOPAIICImBARD+VrMwO4iNLXcyFJvUhB/MSIeDYf868RMTUiFkTELaRru0WZhzwQuC4inomIOfn9fjW3ozdo7mfVoXS0QPsO0LNE+9YqpD/dGrye0xYeo1Gg/phU+2iRiPiIVAM4Fpgm6e+S1iujPA1l6luw/lYLyvNORMzPrxt+Wd8u2D67YX9J60i6V9Jbkt4ntWv3LHJsgBkR8UmJPNcAGwKX5l+wkiJiIulLbSCwDammM1XSuixeoG3uZ1bq+reGlpy7E+leQoM3mzhe4+vX3PXsLelmSVPy9fwzpa8ned9lgL8BN0XEzQXph0gaI+k9Se+RrmtZx6TR+81fLu+w+J/tmtXRAu1w0p9WexXJM5V0U6tB/5y2OD4i/Ync4EuFGyNiWETsSKrZjSMFoFLlaSjTlMUsU0tcQSrX2hGxEvAzUjtoMUW7qUjqSmr3vBY4S9IXWlCeR4HvkNqJp+T1Q4AepJ4jLS5PE4pd/0Wup6RFrudinKucc89j0cC5JOf4Vd7/K/l6HkTp69ngUlI77MIeFZJWI31mTyA1ZXUHxhYcs1RZF3m/klYg/dXZFp/tqtKhAm1EzCK1T14uaS9Jy0taRtIukn6dsw0FzpDUS1LPnP/Pi3nKMcC2kvpL6kb60wgASStL2iN/uOaQamvzmzjGfcA6kg6Q1EnSd4ENSDW6SluR1I78Ya5tH9do+9uk9sSW+B0wOiKOAv5Oal8EQNJZkh4psu+jpF/qx/L6I6TudE8U1NIba2kZi13/54ABkgZKWo7Ujrkk52rq3CdLWiN/If2S1A7dWr1YViTfmJLUFzi1nJ0kfY/0V8MBEbGgYNMKpGA6I+c7nFSjbfA20E/Sss0c+ibg8Pzz7Ex6vyNzM9VSpUMFWoCIuIjUh/YM0gfkTdIv7505yy+AUaS7ti8Az+S0xTnXQ8At+VijWTQ41pF6L0wl3XHdDvh+E8d4B9gt532HdOd8t4iYuThlaqEfkW48fUCqudzSaPtZwA35z8Z9Sx1M0p6kG5LH5qQfAptIOjCvr0rqPdGcR0nBoiHQPkGqYT7W7B6pFndGLuOPSpWRItc/Il4m3Sz7B6ktsnG/62uBDfK57qTlriP1lHiM1AvlE9IXSWs5m3TjaRbpS+72Mvfbn/QFMlXSh3n5WUT8B7iQ9Jfi28BGLHr9/klq839L0uc+rxHxMPBz4DZSr5Y1gf0W543Vug75wIJVJ0ljgG/kLxezpYYDrZlZhXW4pgMzs2rjQGtmVmEOtGZmFeaBKxrRMl1Cnbu1dzGsGRuv26+9i2BFvPH6JN6ZObPcvrtlqV9ptYh5n3sI8XNi9oxhEbFza567tTjQNqLO3ei84SHtXQxrxqOPnN/eRbAittu63KdzyxfzZtN53ZK9C/lkzOXlPrHW5hxozazKCVTbrZwOtGZW3QTU1bd3KZaIA62ZVT+1arNvm3OgNbMq56YDM7PKc43WzKyCJLfRmplVnJsOzMwqzE0HZmaV5JthZmaV5X60ZmaV5hqtmVnl1bmN1syscoRrtGZmFedeB2ZmleQHFszMKs9NB2ZmFSTVfNNBbX9NmNnSQXWll1KHkFaV9C9JL0l6UdIPcvoXJD0k6ZX8f4+cLkmXSJog6XlJmxQc69Cc/xVJh5Y6twOtmVW53EZbailtHnBKRKwPbAUcL2kD4DTg4YhYG3g4rwPsAqydl2OAKyAFZuBMYEtgC+DMhuDcHAdaM6t+Dc0HxZYSImJaRDyTX38AvAT0BfYEbsjZbgD2yq/3BG6MZATQXVIf4JvAQxHxbkT8F3gIKDoppNtozay6VaAfraTVgUHASGDliJgGKRhL6p2z9QXeLNhtck5rLr1ZDrRmVuXKfgS3p6RRBetXR8TVnzua1BW4DTgpIt5X87XhpjZEkfRmOdCaWfUrr9fBzIjYrPhhtAwpyP4lIm7PyW9L6pNrs32A6Tl9MrBqwe79gKk5fUij9EeKnddttGZW/VrhZphS1fVa4KWIuKhg091AQ8+BQ4G7CtIPyb0PtgJm5SaGYcBOknrkm2A75bRmuUZrZtVNrTZ619bAwcALksbktJ8B5wO3SjoSeAPYJ2+7D9gVmAB8DBwOEBHvSjoXeDrnOyci3i12YgdaM6t+rfDAQkQ8QdPtqwDfaCJ/AMc3c6zrgOvKPbcDrZlVvSI3rGqCA62ZVbXUcuBAa2ZWQXKN1sys0hxozcwqzIHWzKyS3EZrZlZZchutmVnlOdCamVWYA62ZWYU50JqZVZJvhpmZVZZvhpmZtQEHWjOzSqvtOOtAa2ZVTlBXV9tzFDjQmlnVc9OBmVkFdYSbYbVdH18K9evdjQcu/x7P3nwKo2/6IcfvuzUA395+I0bf9EM+eup8Nlmv3+f2W3Xl7sz457mcdMC2C9N23GodnrvlVMb+9cf86OAhbfUWljrz58/na1ttyr7f3h2Aq6+4nIED1qFbl3remTnzc/lHj3qaHissw523/62ti1q9VMZSxRxoa8y8+Qs47ZJ7GbTfhWx31OV87zuDWW/13rz46tvsd9qfeGLMa03u9+uTdufB4eMXrtfViYt/tDd7nnwtg/a/kH12Gsh6q/ducl9bMldcdgnrrrvewvUtvzqYu+57kP79V/tc3vnz53PmGT/lGzvu1JZFrG5KTQellmrmQFtj3nrnA8aMnwLAhx/PYdyk6azSuxvjJ03nlTdmNLnP7tsO4LUp7/Kf195emLb5BqsycfJMJk19l7nz5vPXh55jt20HtMl7WJpMmTyZYQ/cxyGHH7kwbeOBg1httdWbzH/V7y9jz72+Ta9e/tIrVFdXV3KpZtVdOiuqf58eDFxnFZ4e+0azeZZfbhlOOXgI51370CLpq/TqxuTpsxauT5k+i769VqpYWZdWp516Muecd35ZgWDqlCnce/edHHH099qgZDWmFZoOJF0nabqksQVpt0gak5dJDbPjSlpd0uyCbVcW7LOppBckTZB0icqoTlcs0Er6raSTCtaHSfpDwfqFkn4mqWRDlKRukm6UNDEvN0rqVqmy14IVuizL0F8dzKkX38MHH89pNt/Pj96JS29+nI9mf7pIelMfjWjtQi7lHrjvXnr17s2gTTYtK/9pp57M2b/4FfX19RUuWe1ppaaD64GdCxMi4rsRMTAiBgK3AbcXbJ7YsC0iji1IvwI4Blg7L4scsymV7HXwFGl+9Isl1QE9gcIq02DgpIj4ZRnHuhYYGxGHAEg6G/gDn82/vlTpVF/H0F8dzC3DnuWuR8YWzbv5gP7svf1GnHfCrnTr2oUFC4JPPp3Hs+Mm06/3Z99VfXt3Y+qM9ytd9KXKiOFPcf+99/DQA/fzyZxP+OD99zn68IO55o9/ajL/s8+M5ohDDgDgnXdm8uCw++nUqRO77bFXWxa76rRWG2xEPCZp9WbOIWBfYPsSZekDrBQRw/P6jcBewP3F9qtkoH0S+G1+PQAYC/SR1AP4GFgf+K+ksRGxoaTDgD2A5YE1gTsi4seS1gI2Bb5bcOxzgAmS1gRWBc4CZgIbAqOBgyIiJG0KXAR0zdsPi4hpFXzPbeLK0/dh/KTpXDL08ZJ5dzj2ioWvTz9qRz76eA5X/u0p6uvrWGvVnqzWpwdTZ7zPPjtuzGH/O7SSxV7qnHXuLznr3FSPePyxR7j04gubDbIAL4ybuPD1cUcfzjd3+dZSH2QblNkG21PSqIL1qyPi6jJPsQ3wdkS8UpC2hqRngfeBMyLicaAvMLkgz+ScVlTFAm1ETJU0T1J/Uu11eC7QV4FZwPPAp412GwgMAuYA4yVdCmwAjImI+QXHnp/bUgaQfgiD8uuppAC/taSRwKXAnhExQ9J3gfOAIyr1ntvC4I1X58BdN+WFCdMYcWNqmTnzigfovGw9F52yJz27d+X2iw7n+ZenssdJ1zZ7nPnzF3DyBXdxz++Oor6ujhvufZqXCm6WWeVcefml/O6i3/D2228xePOB7LjzLlx2xTXtXazqVl6FdmZEbLaYZ9gfKKxpTAP6R8Q7ucJ2p6QBzZSkZKtbpR9YeJIUZAeTapZ98+tZpKaFxh6OiFkAkv4DrEZ6Y029kcL0f0fE5LzfGGB14D1SDfeh/GdHPemH9/kDSceQ2lxg2eq+IfTUc5PostWPm9x296MvFt33vD8sekNs2PBxDBs+rtXKZs3bZtshbLPtEACOPf5Ejj3+xKL5r7jmj21QqtpRye5bkjoB3yb95QxARMwhVfiIiNGSJgLrkGqwhR3V+5EqeEVVOtA+RQqsG5GaDt4ETiHVQq9rIn/hXZ35uXwvAoMk1UXEAoDc5rsx8BLpjTa1n4AXI+KrpQqZ/7y4GqCu65d8T8ismqjij+DuAIxrqKwBSOoFvJv/ev4y6abXqxHxrqQPJG0FjAQOIf3lXFSlu3c9CexGLnBEvAt0JzUfDC/nABExAXgWOKMg+QzgmbytOeOBXpK+CiBpmVz1N7MaIkRdXeml5HGkoaS4s66kyZIaOjfvx6LNBgDbAs9Leg74G3Bsjl8Ax5Fuxk8AJlLiRhhUvkb7Aqm3wU2N0rpGxExJXcs8zpHApZImkGqqw3NasyLiU0nfAS7JXcE6AReTashmVkNao0IbEfs3k35YE2m3kbp7NZV/FKlZsmwVDbT5BtZKjdIOK3g9iVzgiLie1M+tYdtuBa//CxzUzDkeAR4pWD+h4PUY0jeTmdWwan/EthSP3mVm1U2tU6NtTw60ZlbVBGW1wVYzB1ozq3oOtGZmleSmAzOzyhK+GWZmVmHVP7B3KQ60Zlb13EZrZlZJbqM1M6sst9GambWBGo+zDrRmVv3cRmtmVkmVHyax4hxozayqpTba9i7FknGgNbMq5360ZmYVV+Nx1oHWzKqcfDPMzKyi3I/WzKwN1HqgrfTkjGZmS0wqvZQ+hq6TNF3S2IK0syRNkTQmL7sWbPuppAmSxkv6ZkH6zjltgqTTyim/A62ZVbfcRruks+CS5iTcuYn030bEwLzcByBpA9LsuAPyPr+XVC+pHrgc2AXYANg/5y3KTQdmVtXUSt27IuIxSauXmX1P4OaImAO8lmfg3iJvmxARrwJIujnn/U+xg7lGa2ZVr8ymg56SRhUsx5R5+BMkPZ+bFnrktL7AmwV5Jue05tKLco3WzKpeXXk12pkRsVkLD30FcC4Q+f8LgSNInR0aC5qunEapkzQbaCWtVGzHiHi/1MHNzFpDpTodRMTbn51D1wD35tXJwKoFWfsBU/Pr5tKbVaxG+yIpUhe+xYb1APqXOriZ2ZKSoL5CDyxI6hMR0/Lq3kBDj4S7gZskXQSsAqwN/JsU/9aWtAYwhXTD7IBS52k20EbEqs1tMzNrS61xM0zSUGAIqS13MnAmMETSQFLlcRLwPYCIeFHSraSbXPOA4yNifj7OCcAwoB64LiJeLHXustpoJe0HfDkifimpH7ByRIxu0bs0M1tMrdF0EBH7N5F8bZH85wHnNZF+H3BfS85dsteBpMuArwMH56SPgStbchIzs8UlchevEv+qWTk12sERsYmkZwEi4l1Jy1a4XGZmiVSxNtq2Uk6gnSupjtyFQdIXgQUVLZWZWYEaH+qgrAcWLgduA3pJOht4Avi/ipbKzCwTqR9tqaWalazRRsSNkkYDO+SkfSJibLF9zMxaU5XH0ZLKfTKsHphL809GmJlVhDrAwN/l9Do4HRhK6rTbj9SJ96eVLpiZWYMO33QAHARsGhEfA0g6DxgN/KqSBTMza1DdYbS0cgLt643ydQJerUxxzMw+r9ZnWCg2qMxvSW2yHwMvShqW13ci9TwwM6u41OugvUuxZIrVaBt6FrwI/L0gfUTlimNm1ojKnkGhahUbVKbZZ4DNzNpSh206aCBpTdLAChsAyzWkR8Q6FSyXmRnQMZoOyukTez3wR9L73QW4Fbi5gmUyM1uEpJJLNSsn0C4fEcMAImJiRJxBGs3LzKziJKiXSi7VrJzuXXOUvi4mSjqWNKp478oWy8zsM1UeR0sqJ9CeDHQF/ofUVtuNNHmZmVmbqPamgVLKGVRmZH75AZ8N/m1m1mZqPM4WfWDhDopMoxsR365IiczMCojqH8uglGI12svarBRVZNC6/XjyiV+3dzGsGT02P6G9i2BFzBn/RusftAOM3lXsgYWH27IgZmbNaY2xWSVdB+wGTI+IDXPab4DdgU+BicDhEfGepNWBl4DxefcREXFs3mdTUrfXLqRJGn8QEc3+9d9a5TczqxjRav1orwd2bpT2ELBhRHwFeBkoHAJ2YkQMzMuxBelXAMcAa+el8TE/x4HWzKpenUovpUTEY8C7jdIejIh5eXUEacztZknqA6wUEcNzLfZGYK+S5S9dvIUn6FxuXjOz1iJBfZ1KLkBPSaMKlmNaeKojgPsL1teQ9KykRyVtk9P6ApML8kzOaUWVM9bBFsC1pP6z/SVtDBwVESeWW3ozsyVR5r2wmRGx2eIcP88kMw/4S06aBvSPiHdym+ydkgbQ9BjkRdtnobwa7SWkBuR3ACLiOfwIrpm1Ian0svjH1qGkGHdgw02tiJgTEQ0xbzTpRtk6pBpsYfNCP2BqqXOUE2jrIuL1Rmnzy9jPzGyJVXK6cUk7Az8B9miYriun95JUn19/mXTT69WImAZ8IGmrPDTBIcBdpc5TziO4b+bmg8gnPpF0d87MrE3Ut0I3WklDgSGkttzJwJmkXgadgYdyz4WGblzbAudImkeqWB4bEQ030o7js+5d97Nou26Tygm0x5GaD/oDbwP/yGlmZhWnVprlNiL2byK5yQkOIuI24LZmto0CNmzJucsZ62A6sF9LDmpm1ppq/AncsnodXEMTd9UioqVdJ8zMFkuNP4FbVtPBPwpeLwfsDbxZmeKYmS2q4WZYLSun6eCWwnVJfyI9tmZmVnmC+hp/hrWcGm1jawCrtXZBzMyaoyafE6gd5bTR/pfP2mjrSM8Kn1bJQpmZNegIs+AWDbS5Q+7GpHnCABaUGg7MzKy1dehAGxEh6Y6I2LStCmRmVkjQMGhMzSqnifnfkjapeEnMzJpSxjgH1d4podicYZ3yOI1fA46WNBH4iPQFExHh4GtmbaIjd+/6N7AJZQxqa2ZWKR39ZpgAImJiG5XFzKxJNV6hLRpoe0n6YXMbI+KiCpTHzGwRQtTXeKQtFmjrga40PaK4mVnbKHNOsGpWLNBOi4hz2qwkZmbN6Mg3w2r7nZlZh5CmG2/vUiyZYoH2G21WCjOzImr9gYVmA23BtA1mZu1GlPdkVTVbnNG7zMzajtJ0NrWs1r8ozGwpoDKWkseQrpM0XdLYgrQvSHpI0iv5/x45XZIukTRB0vOFwxBIOjTnfyVPVV6SA62ZVTUB9VLJpQzXAzs3SjsNeDgi1gYe5rMhYHchTTG+NnAMcAWkwEyaPXdLYAvgzIbgXIwDrZlVvdYYVCYiHiONp11oT+CG/PoGPhtyYE/gxkhGAN0l9QG+CTwUEe9GxH9Js800Dt6f4zZaM6tyKreNtqekUQXrV0fE1SX2WTkipgFExDRJvXN6XxadG3FyTmsuvSgHWjOrai3odTAzIjZrxdM2FkXSi3LTgZlVPUkll8X0dm4SIP8/PadPBlYtyNcPmFokvSgHWjOrbkqP4JZaFtPdQEPPgUOBuwrSD8m9D7YCZuUmhmHATpJ65JtgO+W0otx0YGZVrbUeWJA0FBhCasudTOo9cD5wq6QjgTeAfXL2+4BdgQnAx8DhkB7kknQu8HTOd045D3c50JpZ1WuNBxYiYv9mNn1uuIE8Ce3xzRznOuC6lpzbgdbMql5tPxfmQGtmVa7hgYVa5kBrZlWvxuOsA62ZVTuhGm88cKA1s6rnGq2ZWQVJbqM1M6u4Go+zDrRmVv1qvY3Wj+DWsE8++YSvfXULtthkYzbZeADnnn0mABHBmT8/nY02WIeBG63P5ZdesjD9hyf9DwPWW4vNB32FZ595pj2L3yH1W7k7D1z9Pzx72xmM/tvpHL//EAC+vcMgRv/tdD4afQmbbNB/Yf5lOtVz1VkH8fStP2PkLaexzaZrL9w27Jof8NwdP2fEzacx4ubT6NWja1u/naog0nTjpZZq5hptDevcuTMPPPRPunbtyty5c9l+u6+x0zd3Yfy4l5j85ps8N3YcdXV1TJ+exskY9sD9TJzwCmNfeoV/jxzJ/5xwHI8/NbKd30XHMm/+Ak676HbGjJtM1+U789RNP+HhkeN4ceJU9jvlGi47Y9GHk4749tYAbL7vL+nVoyt3XvZ9vnbQb0gPJsHhp9/AM/95o83fR7Wp9RqtA20Nk0TXrqmWM3fuXObNnYskrr7qCm74003U1aU/WHr3TkNs3nv3XRxw0CFIYsuttmLWrPeYNm0affr0abf30NG8NfN93pr5PgAffjyHca+9xSq9uvPPkeOazL/el7/Ev/49HoAZ//2QWR/MZtMN+jPqxdfbrMy1YAkGjakKbjqocfPnz2fLTQfSf5XebL/Djmyx5Za89upE/vbXW9h6y83Yc7ddmPDKKwBMnTqFfv0+G+Gtb99+TJ0ypb2K3uH17/MFBq7bj6fHTmo2zwsvT2H3IRtRX1/Haqt8kUEbrEq/L302M8pVZx3EiJtP47SjSw7i32G56aAISb8FXo+Ii/P6MODNiDgqr18IvANsEhHfKXKckUBn4AtAF6AhMuwVEZMqVf5aUV9fz8jRY3jvvff47nf25sWxY5kzZw6dl1uOJ0eO4s47bud7Rx/Bw488vvDP0UK1PrtotVqhy7IMveAoTr3gNj746JNm891w13DWW2NlnvzLj3lj2ruMeO415s2fD8DhP7ueqTNm0XX5zgy94CgO2G0Lbrr33231FqpI7T+wUMka7VPAYABJdUBPYEDB9sGkSdGaDbIAEbFlRAwE/he4JSIG5mVSZYpdm7p378622w3hwQcfoG+/fuy99/8DYM+99mbsC88DqQY7efJns3BMmTKZPqus0i7l7cg6dapj6AVHc8v9o7jrn88VzTt//gJ+fOHtbLXf+ex78tV0X7ELE96YAcDUGbOA1ARxy/2j2HzAahUve1UqY76waq8vVDLQPkkOtKQAOxb4IA+Y2xlYH/hvw9S/kg6TdLukB/I0vr8udnBJBzTkkXSKpJfz63UlPZJf7yhpjKQXJF0jadmKvNN2MmPGDN577z0AZs+ezT8f/gfrrrseu++xF4/8658APP7Yo6y19joAfGv3PbjpzzcSEYwcMYKVVurm9tkKuPLMAxn/2ltc8ud/lszbZbllWH659LHcfsv1mDd/AeNefYv6+jq+2H0FIAXuXbfdkBcnTqtouatVK86C224q1nQQEVMlzZPUnxRwh5MmMfsqMAt4Hvi00W76iX/yAAAPJklEQVQDgUHAHGC8pEsj4k2a9hhwYn69DTBL0peArwGPS1qeNGbkkIiYKOkvpGmDL2t8IEnH5G2s2r9/481V661p0zj6iEOZP38+C2IB/+87+7Lrt3Zj8NZf4/BDDuTS3/2WFbp25Yqr/gDAzrvsyrD772PAemuxfJflueoPf2znd9DxDB74ZQ7cbUteeHkKI25OM1efednddF6mExf9ZB969ujK7Zccy/Pjp7DH8ZfTq8eK3PP741mwIJg64z2OPCNNyNp5mU7cffnxLNOpnvr6Ov41chzX3f5ke761dlXdYbQ0NdVu12oHT8HtHtIc6ReRAu1gUqD9InAlcG9EbCjpMGDriDg673s/cF5EPJHXDwM2i4gTCo7/MikwPwzcBkwCvgXcRGr//U1EbJ/zfhM4MiL2LVbmTTfdLJ4cOapYFmtHPTY/oXQmazdzxt/Kgo+nt2pcXH+jQfHHO/9VMt9X1+oxuhUnZ2xVle510NBOuxGp6WAEqUY7mNS00NicgtfzKV3jHgEcCfwHeJxUs90qn7fWvwTNLFMZ/6pZpQPtk8BuwLsRMT/PrdOdFGyHt8LxHwN+lP8fDXwT+CAiPiQF37UlfTnnPQh4tBXOaWZtzDfDinuB1NtgRKO0WRExsxWO/zhp6t/HImIuqevX4wAR8TGptnu7pBdIteVrWuGcZtbGWiPQ5hvlYwqW9yWdJOksSVMK0nct2OenkiZIGp+bHxdLRZ8Mi4j5wEqN0g4reD0J2DC/vh64vmDbbo32W2R7ThtPQRNBQ3tswfqDwIOL/w7MrL2J1nkEN8eLgQCS6kkVsztIM9z+NiIuWOS80gbAfqReU6sA/5C0To5rLeInw8ysulWmH+03gIkRUexZ5z2BmyNiTkS8Rpp6fIvFeQsOtGZW9VTG0kL7AUML1k+Q9Lyk6yQ1PAPdFyjsXjo5p7WYA62ZVTkhlV6AnpJGFSzHNHm09ODSHsBfc9IVwJqkZoVpwIULT/x5i9Uf1qN3mVnVK7NpYGaZ/Wh3AZ6JiLcBGv5P59E1wL15dTLpZnuDfsDUskrSiGu0ZlbVymk2aGHTwf4UNBtIKnwOfW9Sn3+Au4H9JHWWtAawNrBYo/q4Rmtm1a+V+snmR/N3BL5XkPxrSQNJzQKTGrZFxIuSbiX1yZ8HHL84PQ7AgdbMakBrDfyd+9d/sVHawUXynwect6TndaA1s6pX5Q9+leRAa2bVbTH7b1UTB1ozq3rVPmhMKQ60ZlbVRPUPGlOKA62ZVT0HWjOzCnPTgZlZhblGa2ZWYTUeZx1ozay6pZthtR1qHWjNrLrVwFQ1pTjQmlnVq/E460BrZjWgxiOtA62ZVbnqn068FAdaM6tqAupqO8460JpZDXCgNTOrLDcdmJlVmLt3mZlVktxGa2bWBmo70jrQmllV6wjj0Xq6cTOreq013bikSZJekDRG0qic9gVJD0l6Jf/fI6dL0iWSJkh6XtImi1t+B1ozq3p1UsmlBb4eEQMjYrO8fhrwcESsDTyc1wF2AdbOyzHAFYtd/sXd0cyszbRWlbZpewI35Nc3AHsVpN8YyQigu6Q+i3MCB1ozq3plxtmekkYVLMc0cagAHpQ0umD7yhExDSD/3zun9wXeLNh3ck5rMd8MM7OqpvKHSZxZ0BzQnK0jYqqk3sBDksYVO3UTaVFWSRpxjdbMqp7K+FeOiJia/58O3AFsAbzd0CSQ/5+es08GVi3YvR8wdXHK70BrZlWvoVZbbCl9DK0gacWG18BOwFjgbuDQnO1Q4K78+m7gkNz7YCtgVkMTQ0u56cDMql4r9aNdGbgjT4vTCbgpIh6Q9DRwq6QjgTeAfXL++4BdgQnAx8Dhi3tiB1ozq3KtMx5tRLwKbNxE+jvAN5pID+D4JT4xDrRmVuU6wpNhDrRmVvUcaM3MKszj0ZqZVZKnGzczq6wlf8K2/TnQmlnVU41XaR1ozazq1XicdaA1s+pX43HWgdbMakCNR1oHWjOrerXevUvpKTNrIGkG8Hp7l6MV9QRmtnchrFkd7fqsFhG9WvOAkh4g/ZxKmRkRO7fmuVuLA20HJ2lUGWN0Wjvx9Vk6eJhEM7MKc6A1M6swB9qO7+r2LoAV5euzFHAbrZlZhblGa2ZWYQ60ZmYV5kBrZlZhDrRmNUK1PoTVUsyBdinV3C+tf5mrR+NrEb5zXbPc62ApJ+lYYEPSdMp3RcST7Vwka0TSvsAGwERgdET8p52LZC3kGu1STNL3ge8AfwS2yK+tikg6Efgx8AkwCPilpM3bt1TWUh69a+nWA9gLOByYDZwqqTOwYkR0pIFOaoYkNWoi2BA4OCJektQb2Bf4f5JeAOa4OaE2uEa7lGim7XUVYBTw9YjYJSLmAYcCe0jyZ6MdNAROSdvnpO7AcXnbdGAs0B/41EG2dviXaSlQWEuS9E1JO+TAez4wjfTLi6TDgZOAJyJiQbsVeCnU8MWmpBNwvKSDgNOBlSX9JGftSQq+K7ZPSW1x+GbYUkTSkcAPgVnAU8AdwEfApcDbQB/gaN9saT+SBkbEGEm7AQNJ7ef9gStIN8PWBfaPiBfasZjWQm6j7aAat/XlX9w9ImKApJVINaVvATdFxDaSlgM6R8SsdiryUk/SIOARSWcCo4HNgQkRcXO+AdYH+Cgi3mnPclrLuemg4+raaH0zYIikDSPifVItth74vqTNI+ITB9m2JWldScvm1ysArwLDgR2BlYC5wPmShkTE3Ih4w0G2NjnQdkCSdgWGSrpR0j2SNgZuBH4HnC1pnYiYDFxOmkalI03dUxMk7QLcA3xJ0ibAxcDawMlAN2AeqWlnFWB3Scu0V1ltybmNtoOR9E3SGKffByYBBwPrAzcBzwDfBb4C/G9EjJNUHxHz26m4S6V8jX5Pamv9t6QBpGtyJqnP7Eak+a+ukrQ/8FRE+MuwhjnQdhC5F0En0s2Tv0fE0IJtRwF7A6cCbwI/B1YGjgLmuZtQ25G0E/AHYDxwQUQMK9i2BekLshcwANgwIj5sl4Jaq/LNsA4iB8u5kmYDbwFIWi63vf5B0vrAWRGxr6TfkTq7z23PMi9tJG1Haq45HhDwHUlfjIibAHLt9jVS7fZcUlcuB9oOwDXaDkbS5UCfiPh2Xl8+Ij6WtCVwZEQc074lXHpJ6geskgNqb2BXYDDwSEOwLchb577MHYdvhtW4Jp74OhuYLel/ASLi45y+CdBDUmeP0NW2JNXlwDk5Iv4NC5/yuhd4ktQbZL/CfRxkOxY3HdSwRk98bQ+sADwA3AAcLOle4BpgHeBYYM+ImNNe5V0a5Wu0IL/+OumJrnsimSnpPmABsKekeRHxt/Ysr1WGmw5qUEONtCDIfo/U7jeXNHbBZaQnvU4BlgWWB37nJ77aTpFr9CnwAvAbYFxELMjNCN8gNSFMa6ciWwU50NYgSV0iYnZ+/XVSb4JvRURIuooUcC+MiNdyHrf3tbES1+gKYD7pxtj4HGx9jTowt9HWGEmbAr+XVJ+fJtqB1O9y25zlB6Qnvs6RtG5O87dpGyrjGp1MukankR5ScJtsB+dAW0MkrQnsDvwW2IbUF/YXpKe+9pW0RUR8Qho4ZiZp8BhPgdKGWniN3iVfI+vYHGhrSydSd6DNgPWAu4DewAWk4Q4PkTQ4ImZHxMkR8Vb7FXWp5Wtkn+NAWwMkrSGpW0SMB35NupEynjQwzF9Jd7IvBz4gjb6/nLtwtS1fIyvGN8OqnKR1gFtIv7QnRcRbeVzZbYCzSPN87Q0cCLwHdPI0NG3L18hKcY22+r0KvAhsDfxfHpnrA9KYBRtHxAXAE6Tn5z/0L3C78DWyolyjrVKS+gPLRcTLknoCJ5K+GCeT7lRvQxqd69CI+ERST/8Cty1fIyuXnwyrQrlL0M+BZSXdERF3SnqVNOX0g6QxStcH9gH+CxzrX+C25WtkLeFAW4Ui4iNJPyc9LXS5pD7ABNJjtBMiYrSkI4BjgKFFDmUV4mtkLeGmgyqXR9//P+AhUhtgN9IoXBP9NFF18DWyUnwzrMpFxDPA4aTO7ZNJTxftoTQ9tb8lq4CvkZXiGm2NyHNGLQdcSBrHYHw7F8ka8TWy5jjQmplVmJsOzMwqzIHWzKzCHGjNzCrMgdbMrMIcaM3MKsyB1syswhxorVmS5ksaI2mspL9KWn4JjjUkz8qLpD0knVYkb3dJ31+Mc5wl6UflpjfKc72k77TgXKtLGtvSMtrSyYHWipkdEQMjYkPS7K3HFm5U0uLPUETcHRHnF8nSHWhxoDWrVg60Vq7HgbVyTe4lSb8HngFWlbSTpOGSnsk1364AknaWNE7SE8C3Gw4k6TBJl+XXK0u6Q9JzeRkMnA+smWvTv8n5TpX0tKTnJZ1dcKzTJY2X9A9gXUqQdHQ+znOSbmtUS99B0uOSXpa0W85fL+k3Bef+3pL+IG3p40BrJUnqBOwCvJCT1gVujIhBwEfAGcAOEbEJMAr4oaTlgGtIExVuA3ypmcNfAjwaERsDm5AG0D4NmJhr06dK2ok0vusWwEBgU0nb5tlm9wMGkQL55mW8ndsjYvN8vpeAIwu2rQ5sB3wLuDK/hyOBWRGxeT7+0ZLWKOM8Zgt5mEQrpoukMfn148C1pHFWX4+IETl9K2AD4Mk8BdaywHDSxISvRcQrAJL+TBoysLHtgUMAImI+MEtSj0Z5dsrLs3m9KynwrgjcEREf53PcXcZ72lDSL0jNE12BYQXbbs0jbb2Sx5ZdL5/3KwXtt93yuV8u41xmgAOtFTc7IgYWJuRg+lFhEvBQROzfKN9AWm/kKgG/ioirGp3jpMU4x/XAXhHxnKTDgCEF2xofK/K5T4yIwoCMpNVbeF5birnpwJbUCGBrSWsBSFo+T1Y4DlhD0po53/7N7P8wcFzet17SSqT5tlYsyDMMOKKg7bevpN7AY8DekrpIWpHUTFHKisC0PNLWgY227SOpLpf5y6TJFocBx+X8SFonz65gVjbXaG2JRMSMXDMcKqlzTj4jz6N1DPB3STNJkxNu2MQhfgBcrTRr7HzguIgYLunJ3H3q/txOuz4wPNeoPwQOiohnJN0CjAFeJzVvlPJzYGTO/wKLBvTxwKPAyqSpZz6R9AdS2+0zSiefAexV3k/HLPEwiWZmFeamAzOzCnOgNTOrMAdaM7MKc6A1M6swB1ozswpzoDUzqzAHWjOzCvv/waBdE1An1lwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ONE_ACC :  0.838\n",
      "TWO_ACC :  0.859\n",
      "GACC :     0.849\n"
     ]
    }
   ],
   "source": [
    "saver4Pre = tf.train.import_meta_graph('./transfer4uspVGG16NonEQUVerify_Jul_0518/model82.meta')\n",
    "with tf.Session() as sess:\n",
    "    # Step-2: Now let's load the weights saved using the restore method.\n",
    "    saver4Pre.restore(sess, tf.train.latest_checkpoint('./transfer4uspVGG16NonEQUVerify_Jul_0518'))\n",
    "    # Accessing the default graph which we have restored\n",
    "    graph = tf.get_default_graph()\n",
    "    # Now, let's get hold of the op that we can be processed to get the output.\n",
    "    # In the original network y_pred is the tensor that is the prediction of the network\n",
    "    PREDICT_op = graph.get_tensor_by_name(\"PREDICT_op:0\")\n",
    "    SOFTMAX_op = graph.get_tensor_by_name('SOFTMAX_op:0')\n",
    "    X = graph.get_tensor_by_name(\"X:0\") \n",
    "    \n",
    "    Y = graph.get_tensor_by_name(\"Y:0\")\n",
    "    DRP_En = graph.get_tensor_by_name(\"DRP_En:0\")\n",
    "    # for i in range(0, votesIndex.shape[0]):\n",
    "    X_val = np.zeros((1, imageFeatures.shape[1]*2)) \n",
    "    Y_val = np.zeros((1,2))\n",
    "    # for i in range(0, votesIndex.shape[0]):\n",
    "    prediction = []\n",
    "    actulLabel = []\n",
    "   \n",
    "    # for i in range(22338, 24092):\n",
    "    # for i in range(0, votesIndex.shape[0]-ImgVtTestSize):\n",
    "    # for i in range(votesIndex.shape[0]-ImgVtTestSize, votesIndex.shape[0]):\n",
    "    # for i in range(votesIndex.shape[0]-ImgVtTestSize, (votesIndex.shape[0]-ImgVtTestSize + 2553)):\n",
    "    for i in range((votesIndex.shape[0]-ImgVtTestSize + 2553), votesIndex.shape[0]):\n",
    "        # minus 1 because originall indexing list was done in MATLAB.\n",
    "        # MATLAB index stars at 1\n",
    "        imgAIdx = votesIndex[i][0] - 1\n",
    "        imgBIdx = votesIndex[i][1] - 1\n",
    "        voteCode = votesIndex[i][2]\n",
    "        \n",
    "        Y_val = np.zeros((1,2))\n",
    "        Y_val[0][np.int(voteCode-1)] = 1\n",
    "        X_val[0] = np.expand_dims(np.concatenate((imgNrmFeatures[np.int(imgAIdx)], imgNrmFeatures[np.int(imgBIdx)]), axis=0), axis=0)[0]\n",
    "        \n",
    "        result = sess.run(PREDICT_op, feed_dict={X: X_val, Y: Y_val, DRP_En: False})\n",
    "        prediction.append(np.int(result[0]))\n",
    "        actulLabel.append(np.int(voteCode-1))\n",
    "        \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    # pout = np.concatenate((np.transpose(actulLabelArr), np.transpose(predictionArr)), axis=1)\n",
    "    # print(pout)\n",
    "    # Compute confusion matrix\n",
    "    cnf_matrix = confusion_matrix(np.asarray(actulLabel), np.asarray(prediction))\n",
    "    np.set_printoptions(precision=2)\n",
    "    # Plot non-normalized confusion matrix\n",
    "    plt.figure()\n",
    "    class_names = ['WinOne', 'WinTwo']\n",
    "    plot_confusion_matrix(cnf_matrix, classes=class_names,\n",
    "                      title='Confusion matrix, without normalization')\n",
    "    plt.show()\n",
    "    ONE_ACC = cnf_matrix[0][0]/(cnf_matrix[0][0] + cnf_matrix[0][1])\n",
    "    TWO_ACC = cnf_matrix[1][1]/(cnf_matrix[1][0] + cnf_matrix[1][1])\n",
    "    GACC = (cnf_matrix[0][0] + cnf_matrix[1][1] )/(np.sum(cnf_matrix))\n",
    "                                   \n",
    "    print('ONE_ACC :  ' + ('%.3f' % ONE_ACC))\n",
    "    print('TWO_ACC :  ' + ('%.3f' % TWO_ACC))\n",
    "    print('GACC :     ' + ('%.3f' % GACC))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}