find_oodl.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.keras.models import Model\n",
    "from utility import calculate_input_gradients, perturb_inputs, preprocess_images, \\\n",
    "                    extract_features, postprocess_features, save_data_hdf5, get_dataset_hdf5, \\\n",
    "                    build_one_class_svm, combine_inliners_outliers, extract_all_features,\\\n",
    "                    extract_layer_features\n",
    "from utility_db_outliers import load_dataset\n",
    "from models_lib import load_custom_model_for_ds\n",
    "import h5py\n",
    "from general_setting import *\n",
    "from metrics import *\n",
    "from matplotlib.ticker import FuncFormatter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 Physical GPUs, 1 Logical GPUs\n"
     ]
    }
   ],
   "source": [
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "if gpus:\n",
    "    try:\n",
    "        # Currently, memory growth needs to be the same across GPUs\n",
    "        for gpu in gpus:\n",
    "            tf.config.experimental.set_memory_growth(gpu, True)\n",
    "        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
    "        print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n",
    "    except RuntimeError as e:\n",
    "        # Memory growth must be set before GPUs have been initialized\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MNIST -> CUSTOM-MNIST\n"
     ]
    }
   ],
   "source": [
    "\n",
    "FEA_LOC=\"./features/\"     #The folder in which extracted features are saved.\n",
    "if not os.path.isdir(FEA_LOC): os.makedirs(FEA_LOC)\n",
    "BATCH_SIZE = 1000\n",
    "id_name=ID_DS_LIST[0]     # selects the ID dataset.\n",
    "id_model=ID_MODEL_LIST[0] # select the deep model used for training ID dataset.\n",
    "print(id_name,\"->\", id_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We specified the index of layers for several architectures we explored in our experiement.\n",
    "# you can make it to be done automatically or chane the index to suit your architecture.\n",
    "\n",
    "if id_name==\"MNIST\" and id_model==\"CUSTOM-MNIST\":\n",
    "    OOD_DS_LIST       = OOD_DS_LIST_MNIST\n",
    "    slc_layers     = (0,1,2,5)\n",
    "    #**********************************************************\n",
    "elif id_name==\"CIFAR10\":\n",
    "    OOD_DS_LIST       = OOD_DS_LIST_CIFAR10\n",
    "    if id_model==\"VGG16\":\n",
    "        slc_layers = (0,3,5,6,9,11,12,15,18,20,21,24,27,30,33,36,38)\n",
    "    elif id_model==\"ResNet-V1-44\":\n",
    "        slc_layers = (3,6,10,13,17,20,24,27,31,34,38,41,45,48,52,55,60,63,67,70,74,77,81,84,88,91,95,98,102,105,110,113,117,120,124,127,131,134,138,141,145,148,152)        \n",
    "    #**********************************************************\n",
    "elif id_name==\"CIFAR100\":\n",
    "    OOD_DS_LIST      = OOD_DS_LIST_CIFAR100\n",
    "    if  id_model==\"VGG16\":\n",
    "        slc_layers = (0,3,6,9,12,15,18,21,24,27,30,33,36,42)\n",
    "    elif id_model==\"ResNet-V1-44\":\n",
    "        slc_layers = (3,6,10,13,17,20,24,27,31,34,38,41,45,48,52,55,60,63,67,70,74,77,81,84,88,91,95,98,102,105,110,113,117,120,124,127,131,134,138,141,145,148,152)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The weights of CUSTOM-MNIST model was loaded.\n"
     ]
    }
   ],
   "source": [
    "model = load_custom_model_for_ds(id_name, id_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing is not needed.\n",
      "Preprocessing is not needed.\n"
     ]
    }
   ],
   "source": [
    "(train_images,_),(test_images, _) = load_dataset(id_name)\n",
    "train_img_processed=preprocess_images(id_name, train_images, id_model, verbose=True)\n",
    "test_img_processed=preprocess_images(id_name, test_images, id_model, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting features for the training set\n",
      "The featurs of layer 0 was saved. The shape of saved features is (60000, 32)\n",
      "The featurs of layer 1 was saved. The shape of saved features is (60000, 64)\n",
      "The featurs of layer 2 was saved. The shape of saved features is (60000, 64)\n",
      "The featurs of layer 5 was saved. The shape of saved features is (60000, 128)\n",
      "Extracting features for the test set\n",
      "The featurs of layer 0 was saved. The shape of saved features is (10000, 32)\n",
      "The featurs of layer 1 was saved. The shape of saved features is (10000, 64)\n",
      "The featurs of layer 2 was saved. The shape of saved features is (10000, 64)\n",
      "The featurs of layer 5 was saved. The shape of saved features is (10000, 128)\n"
     ]
    }
   ],
   "source": [
    "base_address=FEA_LOC+\"/\"+id_name+\"-\"+id_model+\"/\"\n",
    "if not os.path.isdir(base_address): os.makedirs(base_address)\n",
    "\n",
    "file_address_train =base_address+\"TRAIN\"\n",
    "if not os.path.isfile(file_address_train+\".hdf5\"):\n",
    "    print(\"Extracting features for the training set\")\n",
    "    extract_all_features(model, train_img_processed, slc_layers, BATCH_SIZE, file_address_train)\n",
    "     \n",
    "else:\n",
    "    print(\"The train set features are available.\") \n",
    "    \n",
    "file_address_test =base_address+\"TEST\"\n",
    "if not os.path.isfile(file_address_test+\".hdf5\"):\n",
    "    print(\"Extracting features for the test set\")\n",
    "    extract_all_features(model, test_img_processed, slc_layers, BATCH_SIZE, file_address_test)\n",
    "else:\n",
    "    print(\"The test set features are available.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The name of OOD dataset is FASHION_MNIST\n"
     ]
    }
   ],
   "source": [
    "ood_ds_name = OOD_DS_LIST[0] # selects the OOD dataset used for finding the OODL.\n",
    "print(\"The name of OOD dataset is \"+ ood_ds_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing is not needed.\n",
      "Extracting features for the OOD datasets\n",
      "The featurs of layer 0 was saved. The shape of saved features is (10000, 32)\n",
      "The featurs of layer 1 was saved. The shape of saved features is (10000, 64)\n",
      "The featurs of layer 2 was saved. The shape of saved features is (10000, 64)\n",
      "The featurs of layer 5 was saved. The shape of saved features is (10000, 128)\n"
     ]
    }
   ],
   "source": [
    "(_,_),(outlier_data, _) = load_dataset(ood_ds_name)\n",
    "outlier_data_processed = preprocess_images(id_name, outlier_data, id_model, verbose=True)\n",
    "file_address_ood =base_address+ood_ds_name\n",
    "if not os.path.isfile(file_address_ood+\".hdf5\"):\n",
    "    print(\"Extracting features for the OOD datasets\")\n",
    "    extract_all_features(model, outlier_data_processed, slc_layers, BATCH_SIZE, file_address_ood)\n",
    "else:\n",
    "    print(\"The OOD set features are available.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 / (0, 1, 2, 5)\n",
      "The number of errors for the ID dataset 14\n",
      "The number of errors for the OOD dataset 872\n",
      "1 / (0, 1, 2, 5)\n",
      "The number of errors for the ID dataset 16\n",
      "The number of errors for the OOD dataset 541\n",
      "2 / (0, 1, 2, 5)\n",
      "The number of errors for the ID dataset 12\n",
      "The number of errors for the OOD dataset 657\n",
      "5 / (0, 1, 2, 5)\n",
      "The number of errors for the ID dataset 26\n",
      "The number of errors for the OOD dataset 9992\n"
     ]
    }
   ],
   "source": [
    "aupr_in=[]\n",
    "aupr_out=[]\n",
    "auroc=[]\n",
    "detection_error=[]\n",
    "fpr_at_95_tpr=[]\n",
    "\n",
    "for l in slc_layers:\n",
    "    print(l,\"/\",slc_layers)\n",
    "    training_images = get_dataset_hdf5(\"l_\"+str(l), file_address_train)\n",
    "\n",
    "    test_images     = get_dataset_hdf5(\"l_\"+str(l), file_address_test)\n",
    "    ood_images      = get_dataset_hdf5(\"l_\"+str(l), file_address_ood)\n",
    "    clf_mod, ss_mod = build_one_class_svm(training_images,test_images,ood_images,show_eval=False )\n",
    "    #--------------------------------------------------------------------------------------\n",
    "    test_normalized  = ss_mod.transform(test_images)\n",
    "    pred_test = clf_mod.predict(test_normalized)\n",
    "    base_test_n_errors  = pred_test[pred_test == -1].size\n",
    "    print(\"The number of errors for the ID dataset\", base_test_n_errors)\n",
    "    ood_normalized  = ss_mod.transform(ood_images)\n",
    "    pred_ood = clf_mod.predict(ood_normalized)\n",
    "    base_ood_n_errors  = pred_ood[pred_ood == 1].size\n",
    "    print(\"The number of errors for the OOD dataset\", base_ood_n_errors)\n",
    "    #---------------------------------------------------------------------------------------\n",
    "    mix_data, mix_labels = combine_inliners_outliers(test_normalized,ood_normalized)\n",
    "    scores = - clf_mod.score_samples(mix_data)\n",
    "    results = get_summary_statistics(scores, mix_labels)\n",
    "    aupr_in.append(results[\"aupr_in\"])\n",
    "    aupr_out.append(results[\"aupr_out\"])\n",
    "    auroc.append(results[\"auroc\"])\n",
    "    detection_error.append(results[\"detection_error\"])\n",
    "    fpr_at_95_tpr.append(results[\"fpr_at_95_tpr\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAJ3CAYAAAA9CywoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzde5wedX3+/9cFolGIAgZiOBlEikVbpS54QgQ8VKkYsIqgQgRsPFasrRpPBay2qJUWqz9rWtFgAaECEgVB5CtSKqcEopwVadCEhISThJMIXL8/ZhZu1t29Z/eee+9J5no+Hvdj75l77pn37r373vmcZZuIiCbYYNABREQMS0KKiMZIQoqIxkhCiojGSEKKiMZIQoqIxkhCCgAkWdKzx3n9Gkl7VjzXMkmvqi24PpL0Nkk/HHQcUUhCWseVf/wPSpoxYv+VZZKZPYlzflPSZzr32X6u7Qt6CnYKSZpdfv9PGO842yfafs1UxRXjS0JaP/wfcNDwhqQ/AZ4yuHDWDd2SVUy9JKT1w7eAQzq25wIndB4g6QJJ7+zYfoeki0aeSNI84G3ARyTdI+l75f5Hi2GSjpL0HUmnSFor6QpJzx8tMEkbSJov6VeSbpd0qqTNxzh2T0nLJX1E0mpJKyXtJ2kfSb+QdIekj1c894Xl17vK7+Ml5ff8v5L+RdLtwFEjfw6SnivpvPJatw5fT9JukhZLurvcf+zoH0X0Iglp/XAJ8FRJfyxpQ+BA4L8mcyLbC4ATgc/b3sT2vmMcOgf4b2Bz4CTgu5I2GuW4vwb2A14BbAXcCXxlnBCeAUwDtgb+HvgP4O3AC4GXA5+StH2Fc+9Rft20/D4uLrdfBNwEzAQ+23lhSdOBHwHnlOd7NnB++fJxwHG2nwrsAJw6zvcQk5SEtP4Yvkt6NXAdsKLP11ti+zu2fw8cS5FEXjzKce8GPmF7ue3fAUcBbxqnuPR74LPleb8NzKBIBGttXwNcCwzfjU303AC32P432w/Zvn/Ea68HVtn+ou0Hymte2hHXsyXNsH2P7UvGuUZMUsrQ649vURRTtmdEca1PfjP8xPYjkpZT3FWM9EzgDEmPdOx7mOIOZbSkebvth8vnwwnj1o7X7wc2qXDurnGPYlvgV2O8djjwaeB6Sf8HHG37++OcKyYhd0jrCds3U1Ru7wOcPsoh9/L4iu5njHe6CpfcdviJpA2AbYBbRjnuN8DrbG/a8Zhmu447uPHOPdb3MN739hvgWaO+yf6l7YOALYHPAd+RtHEvwccfSkJavxwO7G373lFeWwq8UdJTyv5Gh49znlsZ4w+zwwslvbEsHn0Q+B1FXdZI/w58VtIzASRtIWlOt2+kovHOvQZ4hO7fR6fvA7MkfVDSkyRNl/Si8txvl7SF7UeAu8rjHxnzTDEpSUjrEdu/sr14jJf/BXiQItkspKi4HsvXgZ0l3SXpu2MccybwFoqK5IOBN5b1PiMdBywCfihpLUXSelHXb6aaMc9t+z6KSuv/Lb+P0eq3Hsf2Woo6uH2BVcAvgb3Kl18LXCPpnvK6B45SBxU9UiZoi4mSdBTwbNtvH3QssX7JHVJENEYSUkQ0RopsEdEYuUOKiMZIQoqIxkhCmqByWotlg44jxicpdRHroEYlpKpz+5Tz9VjSbh3HPLvzl3CU0e0fl/R/5cjv5ZJOKfdfU+67R9LDkh7o2P44k1CODD+77P9yh6TLJB1avjbWKPvO0fTbSDpN0m2Sfivp6vJ9L++I7d7yZ3BPx2O78v2vL695bzkK/kRJ23Rc6x3le/9lRAxzyv3fHOd7u6A85vkj9p9R7t+z3D6q3D6g45gnjPI5fqbj9cMlXa9iBoFby5/hdEk/6Pgef1/+jgxv/3v1T+bR6xxVnqfzZ/eREcf8V3nMzBH7NyvjXqVi5P8Nkj482vfX8Z7PDP9MRztGxQwD3y8/67WSzlfZIbN8/dnlexaNOO+3JX2yy/f6qvK9/z1i/wvL/T8aEddSSeo47hhJ/9kZR8dr25Wf+/Dv6VWSDlYxa8N4v6ejDTECGpaQSlXn9rkD+Mwo+/+ApLkUnfdeZXsTYIhyFHc58dgm5f7/Ad4/vG37HycavKSXAP8P+AnFaPGnA+8BXjeB03yLYhjDM8v3Hwzcavt/OmJ9bnnsph3x/lrSmyhG3/8rxcDU51L0or5I0mYd1/gVcIAePxB1LvCLCvH9go7pTiQ9HXgJRe/oTncAR6uYgWBckl4B/CNwkO3pwB8DpwDYfl3H9905E8Emtt9dId7RnNJxjk1sf74jlunA/sDdwFtHvO9LwBOB5wCbUsw2MNb4t64k7Qj8L3AFMJtiloPvAeer4x9u6aWj7KviVmAPSZt27Bvrs94WeHPF855IMXPCdhS/p3OB1bYv6Pi8ng8w4mc92hAjoJkJqevcPqWFwJ+Wv8jd7Aqca/tXALZXldNs9MMXgIW2P2f7NheW2D6g6zsfsyvwTdv3lqPSr7T9g25vKv+zfRH4jO2TbN9vexXwTuAe4G86Dl8FXAX8efnezYGXUvR87uZE4C0dieYg4AyKnuCdzin3VelAuStwse0rAWzfYXth2Xt6qr0ZWE2RIOeOeG1X4CTbd9l+xPZ1tkcbO1jVp4ELbf+97Ttt3237X4GTgWNGHPsFRkyZUtEDFEnuLQAqpokZ/sc10uep+E+E4mfxDdv3lb+nV9g+dxLxPaqJCanq3D73UfzCVPmALgEOkfRhSUMVf9gTJukpFHcK3+nxVJcAX5F0oMpiWEU7Ufy3etzteTn+6jSKYRGdTuCx5H8gxXCQ31W4zi0U04AMT/16CKP/0zDwKeBIjT5XUqdLgT+XdLSkl0l6UoU4+mUuRUI4GfiTEcXTS4B/UlHs3bGGa72aEZ9X6VTg5SN+Dv9WxrPnJK7T+Vm/DriSx8+i0Hnd31HclXdzCfBVSW+RtG3XoytoYkKC6nP7fA3YTtK4xSHb/0UxmdefUxSlVkv6aH3hPmozip/pyh7P82aK4uOngP8ry/W7VnjfcN3baNdf2fH6sDOAPSU9jbGTylhOoEjyz6EoNl482kG2F1EU5d452usdx/0P8Ebgz4CzgNslHduvfx4UxdW7Oh5bAaiY/O3lFHdBtwAX8Pg79vdSFCU/AFwn6ZeSRs7J/fPOcwN/N1oA5R3t0xn783oCRbEQeHR83j9SsapihP+hGDi8A+N/1qaYGK/KP5E3AhcDRwI3q5g59IWTiO1RTU5IbwXewTh/JOWkXP9QPsZVTub+KooP+N3AP0j681qifcydFCPAZ41zzEPAaB/0RhSTgFHeus+3/VyKuX2WUszIqFHe1+m28uto15/V8Trlde6n+OP/JPB02//b+bqkf9fYFfynA3sD76f4vMbzSeATFJO4jcn2D1zMULk5xYyU76BLIuvBqSOmLRmu1zgEuMr21eX2icDbhuvayuLJZ2z/GUUyOR04rUzqw/6089zAP48WgIteybcz9uf1MI/NLDCs0j/hMa71X8ARFAn3zHGOXURRZO32T+QO2x+xvTPF7+k1FP/kJq2RCanC3D6dvkGRZN5Y8dy/t/3fwM+B5/US5yjnvo/iP8ZfjnPYryl+oTpbMp5CMc/OzaOc8zaKX+itKP5Qx3MDsJwRlZIq5iv6Sx6bjrXTCcDfMkqx2Pa7x6rgL7/XH1BU2I+bkGyfB9xIcXfRVVk3cz5F40Ctn9F4ys/kEOCPVLSiraKoU5lJWdc2Is7fAv9EMWHc7Ele9keMXol8AHBR+U+385q/o6h3+gzQ7R/USCcA7wMW2X6gy7GfoPhHMu4/kY641lDUX247IjlPSCMTUmm8uX0eZfshilvGMYtgZXn/L1Q0IW9Q/nd5LkW9Rd0+AryjrK96enn950v6dvn6pRSVjPMlTVMxydcxwGLKhCTpc5KeVzbFTqf4o7/R9u3jXbj8L/h3wCclvbU8/zOA/wSeSjEFyUg/oSga/9skvtePA6+wvazCsZ+g+NmMSkWXgwNVNKurbE16BaPPsdQvu1O0Mg0BLygfz6OoVzmkjPPIsh7yiZKmURTd7qCYqmQyjgJeIenT5fc+XdIHKUoI88d4zzcpPs8JrX1n+0ZgT4oiWbdjf0TRCjdmXZKkz6vosrChpKdS/J5eXybqSWlsQvL4c/uMdDLj19vcTfHH82uKW+DPA++x/Qf9gXpl+6cURZm9gZsk3QEsAM4uX/8d8BcUvxjLKZpNtwIOKBMKFN0czihjvYmi+f8NFa9/CsUv0d9QFAeuBZ4MvGy0hFa2Ap5v+45JfK+3VP0ZlsXBy8Y55E7gryj+sO+muGP7gu3x5m2q21zgDNvXlC2xq8pWyuOAN3Q0my+k+NneQvE5/kV5xzhhtq+nKEINUfxDWklRXH21x5i3u+OfcLc75tHe+z+2q9ZxfqLLNTahKPr9lqLrw1YU3SAmLYNrJ0hFh7YLbM8ebCQxHkm2PdEiTQxYY++QIqJ9kpAm7i6KXtDRbEcPOoCpIulTevzQjOHH9wYd20SlyBYRjZE7pIhojL4uFCnpeIrVQFfbfl65b3OKnq6zgWUUrUt3ln1AjqPoe3Qf8A7bV3S7xowZMzx79uy+xB8RY1uyZMlttreo85z9Xrn2m8CXeXxv6/nA+baPkTS/3P4oxfiaHcvHi4CvUmG5nNmzZ7N4cdXeARFRF0l/0JG3V30tstm+kKLTWKc5FP04KL/u17H/hLJfzCXAppLGG4IREeuZQdQhzezomLWKx9Zh35rHr7u+vNz3ByTNk7RY0uI1a0ZOwRMR66p+F9nGZduaxFSjLuYyWgAwNDS07jYTnlRTv7231vgjaGJM0RqDSEi3Spple2VZJFtd7l9BMY5o2DaMPe3IxNX1hwb5Y4vok0EU2Rbx2Cx8c3lsGoRFFPPrSMU67L+dwJibiFgP9LvZ/2SKwYczJC2nGBB4DHCqpMMpBhMOT+16NkWT/40Uzf6H9jO2iGieviYk2weN8dIrRznWFHO1RERLpad2RDRGElJENEYSUkQ0RhJSRDRGElJENEYSUkQ0RhJSRDRGElJENEYSUkQ0RhJSRDRGElJENEalsWyShihW19wKuB+4GjjP9p19jC0iWmbcOyRJh0q6AvgYxXLMN1DMX7Q78CNJCyVt1/8wI6INut0hPYViTfj7R3tR0gsoJuX/dd2BRUT7jJuQbH+ly+tL6w0nItpsQpXakvaVdIGkSyS9t19BRUQ7datDesGIXQcDewEvBd7Tr6Aiop261SG9R9IGwKdsr6JYpuiTwCPALf0OLiLapVsd0rskPR/4mqQlwN8DL6Go7P7nKYgvIlqkax2S7Z/ZngNcSbFCyFa2F9n+Xd+ji4hW6VaH9G5JP5X0U2Bj4LUUS1yfK2mPKYkwIlqj2x3Se22/lKIi+8O2H7L9JeBAYL++RxcRrdKtUnuFpI9T1BldP7yzHDLyoX4GFhHt0+0OaQ5wFXARcEj/w4mINut2h7SV7e+N9aIkAVvbXl5vWBHRRt0S0hfKfkhnAkuANcA04NkU9UqvpFgeOwkpInrWrR/SmyXtDLwNOAyYBdwHXAecDXzW9gN9jzIiWqHrfEi2rwU+MQWxRETLZcbIiGiMJKSIaIwkpIhojEpzagNI2hp4Zud7bF/Yj6Aiop2qTvL/OeAtwLXAw+VuA0lIEVGbqndI+wE7ZYR/RPRT1Tqkm4CN+hlIRETVO6T7gKWSzgcevUuy/YG+RBURrVQ1IS0qH7WRtAxYS1En9ZDtIUmbA6cAs4FlwAFZjDKiPSolJNsL+3T9vWzf1rE9Hzjf9jGS5pfbH+3TtSOiYcZNSJJOtX2ApKsoWtUex/af1hzPHGDP8vlC4AKSkCJao9sd0hHl19f34doGfijJwNdsLwBm2l5Zvr4KmDnaGyXNA+YBbLddVvKOWF90G+2/svx683jHSbrY9ksmeO3dba+QtCVwnqTrO1+07TJZjRbXAmABwNDQ0KjHRMS6p66hI9Mm+gbbK8qvq4EzgN2AWyXNAii/rq4pvohYB9SVkCZ0lyJpY0nTh58DrwGupmjJm1seNpdiYriIaInKY9lqNhM4o5gBlycAJ9k+R9LlwKmSDgduBg4YUHwRMQB1JSRN5GDbNwHPH2X/7RTT4kZEC3UtsknaUNKPuxx2cE3xRESLVVlK+2HgEUlPG+eYq2uNKiJaqWqR7R7gKknnAfcO78xYtoioU9WEdHr5iIjom8pj2SQ9EXgORRP/DbYf7GtkEdE6VWeM3Af4GvAriha17SW9y/YP+hlcRLRL1SLbsRQj828EkLQDcBaQhBQRtanaU3vtcDIq3UQxl1FERG2q3iEtlnQ2cCpFHdKbgcslvRHAdiq8I6JnVRPSNOBW4BXl9hrgycC+FAkqCSkiela1le3QfgcSET06aUIjuMb21sHN6FO1le0bjD5j5GG1RxQRrVW1yPb9jufTgP2BW+oPJyLarGqR7bTObUknAxf1JaKIaK3JTtC2I7BlnYFERFStQ1pLUYek8usqshpIRNSsapFter8DiYioPGOkpM0oimqPTuhv+8J+BBUR7VS1yPZOijXatgGWAi8GLgb27l9oEdE2VSu1jwB2BW62vRewC3BX36KKiFaqmpAesP0AgKQn2b4e2Kl/YUVEG1WtQ1ouaVPguxSrzN5JsUxRRERtqray7V8+PapcgeRpwDl9iyoiWmnC67LZ/kk/AomIqGsp7YiIniUhRURjJCFFRGMkIUVEYyQhRURjJCFFRGMkIUVEYyQhRURjJCFFRGMkIUVEYyQhRURjJCFFRGM0LiFJeq2kGyTdKGn+oOOJiKnTqIQkaUPgK8DrgJ2BgyTtPNioImKqNCohAbsBN9q+yfaDwLeBOQOOKSKmyITnQ+qzrYHfdGwvB1408iBJ84B55eY9km6o6fozgNu6HvU21XS5SrrHNLXxQDNjimrq/Oye2WswIzUtIVViewGwoO7zSlpse6ju8/YiMUWdmv7ZNa3ItgLYtmN7m3JfRLRA0xLS5cCOkraX9ETgQGDRgGOKiCnSqCKb7YckvR84F9gQON72NVMYQu3FwBokpqhToz872R50DBERQPOKbBHRYklIEdEYSUilpg1ZkXS8pNWSrh50LMMkbSvpx5KulXSNpCMGHVNUI2mZpKskLZW0eNDxjCV1SDw6ZOUXwKspOmNeDhxk+9oBxrQHcA9wgu3nDSqOTpJmAbNsXyFpOrAE2G+QP6eoRtIyYMh2946/A5Q7pELjhqzYvhC4Y5AxjGR7pe0ryudrgesoetdH1CIJqTDakJX8oY1D0mxgF+DSwUYSFRn4oaQl5dCrRmpUP6RYN0jaBDgN+KDtuwcdT1Syu+0VkrYEzpN0fXkX3ii5QypkyEpFkjaiSEYn2j590PFENbZXlF9XA2dQVFM0ThJSIUNWKpAk4OvAdbaPHXQ8UY2kjctGCCRtDLwGaEzrbackJIohK8DwkJXrgFOneMjKH5B0MnAxsJOk5ZIOH2Q8pZcBBwN7l83HSyXtM+igoquZwEWSfgZcBpxl+5wBxzSqNPtHRGPkDikiGiMJKSIao6/N/pKOB14PrB7ubSxpc+AUYDawDDjA9p1lhelxwD7AfcA7hjvhjWfGjBmePXt2X+KPiLEtWbLkNttb1HnOfvdD+ibwZeCEjn3zgfNtH1OOGZsPfJRipZEdy8eLgK8yynzaI82ePZvFixs7NCdivSXp5rrP2dci2xjDH+YAC8vnC4H9Ovaf4MIlwKbl2KmIaIlB1CHNtL2yfL6KokkSJjB8Q9I8SYslLV6zZk3/Io2IKTXQSm0XfQ4m3O/A9gLbQ7aHttii1iJsRAzQIMay3Spplu2VZZFsdbm/dcM3VNPSZXV2JWtiTNEeg7hDWgTMLZ/PBc7s2H+ICi8GfttRtOuZVN8jIvqj383+JwN7AjMkLQeOBI4BTi2HQtwMHFAefjZFk/+NFM3+h/Yztohonr4mJNsHjfHSK0c51sD7+hlPRDRbempHRGMkIUVEYyQhRURjJCFFRGMkIUVEYyQhRURjJCFFRGMkIUVEY1TqGClpCHg5sBVwP8WKBefZvrOPsUVEy4x7hyTpUElXAB8DngzcQDEYdnfgR5IWStqu/2FGRBt0u0N6CvAy2/eP9qKkF1DM8PjrugOLiPYZNyHZ/kqX15fWG05EtNmEKrUl7SvpAkmXSHpvv4KKiHbqVof0ghG7Dgb2Al4KvKdfQUVEO3WrQ3qPpA2AT9leRTHn9SeBR4Bb+h1cRLRLtzqkd0l6PvA1SUuAvwdeQlHZ/c9TEF9EtEjXOiTbP7M9B7iSYrrZrWwvsv27vkcXEa3SrQ7p3ZJ+KumnwMbAaynWSztX0h5TEmFEtEa3O6T32n4pRUX2h20/ZPtLwIE8tsBjREQtulVqr5D0cYo6o+uHd5ZDRj7Uz8Aion263SHNAa4CLgIO6X84EdFm3e6QtrL9vbFelCRga9vL6w0rItqoW0L6QtkP6UxgCbAGmAY8m6Je6ZUUa60lIUVEz7r1Q3qzpJ2BtwGHAbMoFnG8jmJhx8/afqDvUUZEK3SdD8n2tcAnpiCWiGi5zBgZEY2RhBQRjZGEFBGNUSkhSTq/yr6IiF6MW6ktaRpFL+0ZkjYDVL70VGDrPscWES3TrZXtXcAHKVYbuaJj/93Al/sVVES0U7d+SMcBx0n6a9v/NkUxRURLVVqXDfitpD8Yy2b7hJrjiYgWq5qQdu14Po1iyMgVQBJSRNSmUkKy/ded25I2Bb7dl4giorUm2w/pXmD7OgOJiKh0hyTpe4DLzQ2BPwZO7eXCkpYBa4GHgYdsD0naHDgFmA0sAw4oJ4OLiBaoWofUucLIQ8DNNc2BtJft2zq25wPn2z5G0vxy+6M1XCci1gGVimy2f0Ixhe10YDPgwT7FMwdYWD5fSObtjmiVqkNHDgAuA94MHABcKulNPV7bwA8lLZE0r9w30/bK8vkqYGaP14iIdUjVItsngF1trwaQtAXwI+A7PVx7d9srJG0JnCfp+s4XbVuSR3tjmcDmAWy33XY9hBARTVK1lW2D4WRUun0C7x2V7RXl19XAGcBuwK2SZgGUX1eP8d4FtodsD22xxRa9hBERDVI1qZxTLg75DknvAM6imMJ2UiRtLGn68HPgNcDVwCJgbnnYXIq5vCOiJap2jPywpDcCu5e7Ftg+o4frzgTOKBYt4QnASbbPkXQ5cKqkw4GbKeqrIqIlqtYhYft04PTRXpN0se2XTOBcNwHPH2X/7RTDUiKiheqaMXJaTeeJiBarKyGN2hoWETERmVM7IhqjroSk7odERIyva0KStKGkH3c57OCa4omIFuuakGw/DDwi6WnjHHN1rVFFRCtVbfa/B7hK0nkUcyEBYPsDfYkqIlqpakIasw9SRERdqvbUXijpicBzKJr4b7DdrylIIqKlqs4YuQ/wNeBXFC1q20t6l+0f9DO4iGiXqkW2Yylmd7wRQNIOFANsk5AiGkI1db7xALs5V+2HtHY4GZVuopgPOyKiNlXvkBZLOptiYn9TzBx5eTkDwPDA24iInlRNSNOAW4FXlNtrgCcD+1IkqCSkiOhZ1Va2Q/sdSERE1Va2bzDKiH7bh9UeUUS0VtUi2/c7nk8D9gduqT+ciGizqkW20zq3JZ0MXNSXiCKitSY7/ciOwJZ1BhIRUbUOaS1FHZLKr6vIEtcRUbOqRbbp/Q4kIqLyqiOSNqMoqj06ob/tC/sRVES0U9Ui2zuBI4BtgKXAi4GLgb37F1pEtE3VSu0jgF2Bm23vBewC3NW3qCKilaompAdsPwAg6Um2rwd26l9YEdFGVeuQlkvaFPgucJ6kOymWuo6IqE3VVrb9y6dHlSuQPA04p29RRUQrVW5lG2b7J/0IJCIiK9dGRGMkIUVEYyQhRURjJCFFRGMkIUVEYyQhRURjJCFFRGMkIUVEYzQuIUl6raQbJN0oaf6g44mIqdOohCRpQ+ArwOuAnYGDJO082KgiYqo0KiEBuwE32r7J9oPAt4E5A44pIqZI0xLS1sBvOraXl/siogUmPLi2CSTNA+aVm/dIuqGmU88Abut+/ZquVk3XmKY4HmhmTFFNnZ/dM3sNZqSmJaQVwLYd29uU+x7H9gJgQd0Xl7TY9lDd5+1FYoo6Nf2za1qR7XJgR0nbS3oicCCwaMAxRcQUadQdku2HJL0fOBfYEDje9jUDDisipkijEhKA7bOBswd0+dqLgTVITFGnRn92sj3oGCIigObVIUVEiyUhlZo2ZEXS8ZJWS7p60LEMk7StpB9LulbSNZKOGHRMUY2kZZKukrRU0uJBxzOWFNl4dMjKL4BXU3TGvBw4yPa1A4xpD+Ae4ATbzxtUHJ0kzQJm2b5C0nRgCbDfIH9OUY2kZcCQ7a797AYpd0iFxg1ZsX0hcMcgYxjJ9krbV5TP1wLXkZ70UaMkpEKGrEyQpNkUS6pfOthIoiIDP5S0pBzp0EiNa/aP5pO0CXAa8EHbdw86nqhkd9srJG1Jsfr09eVdeKPkDqlQachKgKSNKJLRibZPH3Q8UY3tFeXX1cAZFNUUjZOEVMiQlQokCfg6cJ3tYwcdT1QjaeOyEQJJGwOvARrTetspCYliyAowPGTlOuDUQQ9ZkXQycDGwk6Tlkg4fZDyllwEHA3uXzcdLJe0z6KCiq5nARZJ+BlwGnGX7nAHHNKo0+0dEY+QOKSIaIwkpIhojCSkiGqOv/ZAkHQ+8Hlg9PPxB0ubAKcBsYBlwgO07yxac44B9gPuAdwz3Ch7PjBkzPHv27L7EHxFjW7JkyW22t6jznP3uGPlN4MvACR375gPn2z6mHMQ6H/goxdJHO5aPFwFfLb+Oa/bs2Sxe3NixghHrLUk3133OvhbZxhiPNQdYWD5fCOzXsf8EFy4BNi0Hc0ZESwyiDmmm7ZXl81UUfSRgAuPJJM2TtFjS4jVr1vQv0oiYUgMdy2bbkibcEapz1ZGhoaF1tiPV0UcfXct5jjzyyFrOA82MKdpjEAnpVkmzbK8si2Sry/19HU9W1x8a5I8tol8GUWRbBMwtn88FzuzYf4gKLwZ+21G0i4gW6Hez/8nAnsAMScuBI4FjgFPLsVk3AweUh59N0eR/I0Wz/6H9jC0imqevCcn2QWO89MpRjjXwvnXbADAAACAASURBVH7GExHNlp7aEdEYSUgR0RhJSBHRGElIEdEYSUgR0RhJSBHRGElIEdEYSUgR0RhJSBHRGElIEdEYSUgR0RhJSBHRGElIEdEYSUgR0RhJSBHRGJXnQ5I0BLwc2Aq4H7gaOM/2nX2KLSJapusdkqRDJV0BfAx4MnADxTzYuwM/krRQ0nb9DTMi2qDKHdJTgJfZvn+0FyW9gGJxx1/XGVhEtE/XhGT7K11eX1pfOBHRZhOu1Ja0r6QLJF0i6b39CCoi2qlKHdILRuw6GNgLeCnwnn4EFRHtVKUO6T2SNgA+ZXsVxXLXnwQeAW7pZ3AR0S5V6pDeJen5wNckLQH+HngJRWX3P/c5vohokUp1SLZ/ZnsOcCXFSrNb2V5k+3d9jS4iWqVKHdK7Jf1U0k+BjYHXAptKOlfSHn2PMCJao8od0nttv5SiIvvDth+y/SXgQGC/vkYXEa1SpVJ7haSPU9QZXT+8sxwy8qF+BRYR7VPlDmkOcBVwEXBIf8OJiDarcoe0le3vjfWiJAFb215eX1gR0UZVEtIXyn5IZwJLgDXANODZFPVKrwSOBJKQIqInVfohvVnSzsDbgMOAWcB9wHXA2cBnbT/Q1ygjohUqzYdk+1rgE32OJSJaLjNGRkRjJCFFRGMkIUVEY1ROSJL2l/S0ju1NJaWndkTUZiJ3SEfa/u3whu27KJr7J0XSMklXSVoqaXG5b3NJ50n6Zfl1s8mePyLWPRNJSKMdW3nVkjHsZfsFtofK7fnA+bZ3BM4vtyOiJSaSkBZLOlbSDuXjWIqOknWaAywsny8kg3cjWmUiCemvgQeBU8rH74D39XBtAz+UtETSvHLfTNsry+ergJmjvVHSPEmLJS1es2ZNDyFERJNULnLZvheYL2l6sel7erz27rZXSNoSOE/S9Z0v2rYkjxHLAmABwNDQ0KjHRMS6ZyKtbH8i6UqKFWuvKe9snjfZC9teUX5dDZwB7AbcKmlWeb1ZFAtSRkRLTKTI9jXgQ7afafuZwN9S3qVMlKSNyzstJG0MvIYi0S0C5paHzaUY0BsRLTGRVrKNbf94eMP2BWUymYyZwBnFzCU8ATjJ9jmSLgdOlXQ4cDNwwCTPHxHroIkkpJskfQr4Vrn9duCmyVzU9k3A80fZfzvFdCYR0UITKbIdBmwBnA6cBswo90VE1KLSHZKkDYFP2P5An+OJiBarui7bw8DufY4lIlpuInVIV0paBPw3cO/wTtun1x5VRLTSRBLSNOB2YO+OfaaoU4qI6NlEemof2s9AIiIqJyRJ36C4I3oc22lpi4haTKTI9v2O59OA/YFb6g0nItpsIkW20zq3JZ1MsZptREQteplTe0dgy7oCiYiYSB3SWoo6JJVfVwEf7VNcEdFCEymyTe9nIBERE5oTu5x0f0eKSm0AbF9Yd1AR0U4TKbK9EzgC2AZYCrwYuJjHd5SMiJi0iVRqHwHsCtxsey9gF+CuvkQVEa00kYT0gO0HACQ9yfb1wE79CSsi2mgidUjLJW0KfJdiUv47KWZ1jIioxURa2fYvnx4l6cfA04Bz+hJVRLTSpFaetf2TugOJiOilp3ZERK2SkCKiMZKQIqIxkpAiojGSkCKiMZKQIqIxkpAiojGSkCKiMZKQIqIxJtVTOyKa5+ijj67lPEceeWQt55mM3CFFRGMkIUVEYyQhRURjJCFFRGMkIUVEYyQhRURjJCFFRGM0LiFJeq2kGyTdKGn+oOOJiKnTqIQkaUPgK8DrgJ2BgyTtPNioImKqNCohAbsBN9q+yfaDwLeBOQOOKSKmiGwPOoZHSXoT8Frb7yy3DwZeZPv9I46bB8wrN3cCbqgphBnAbTWdqy6JKepU52f3TNtb1HQuYB0dy2Z7AbCg7vNKWmx7qO7z9iIxRZ2a/tk1rci2Ati2Y3ubcl9EtEDTEtLlwI6Stpf0ROBAYNGAY4qIKdKoIpvthyS9HzgX2BA43vY1UxhC7cXAGiSmqFOjP7tGVWpHRLs1rcgWES2WhBQRjZGEVGrakBVJx0taLenqQccyTNK2kn4s6VpJ10g6YtAxRTWSlkm6StJSSYsHHc9YUofEo0NWfgG8GlhO0dp3kO1rBxjTHsA9wAm2nzeoODpJmgXMsn2FpOnAEmC/Qf6cohpJy4Ah243u0Jo7pELjhqzYvhC4Y5AxjGR7pe0ryudrgeuArQcbVaxPkpAKWwO/6dheTv7QxiVpNrALcOlgI4mKDPxQ0pJy6FUjNaofUqwbJG0CnAZ80Pbdg44nKtnd9gpJWwLnSbq+vAtvlNwhFTJkpSJJG1EkoxNtnz7oeKIa2yvKr6uBMyiqKRonCamQISsVSBLwdeA628cOOp6oRtLGZSMEkjYGXgM0pvW2UxISxZAVYHjIynXAqVM8ZOUPSDoZuBjYSdJySYcPMp7Sy4CDgb3L5uOlkvYZdFDR1UzgIkk/Ay4DzrJ9zoBjGlWa/SOiMXKHFBGNkYQUEY1RS7N/WVH2gO2H6zjfRMyYMcOzZ8+e6stGtN6SJUtua8QUtpI2oGiJehuwK/A74EmSbgPOAr5m+8baohzH7NmzWby4sUNzItZbkm6u+5yTLbL9GNgB+BjwDNvb2t4S2B24BPicpLfXFGNEtMRki2yvsv37kTtt30HRae60sgNdRERlk0pII5ORpGnA24EnAyfZvn20hBURfSTVc54BdgWqq5XtOOBB4E7guzWdMyJaZlIJSdLJknbo2LU58N8UxbXN6ggsItpnsnVInwA+I2kl8A/AP1MM2JsGHFVPaBHRNpOtQ7oJeKuk3YFTKJr6/2IQ/ZAiYv0x2SLbZpLeB+wMvJmi7uhcSfvWGVxEtMtkK7W/C9xFMQvdt2x/C9gX2EXS9+oKLiLaZbJ1SE8HvkPRzP8uANv3A58uJ4KPiJiwySakI4FzgIeBxy0ZZHtlr0FFRDtNtlL7NIom/oiI2ky2Uvs/JI26Vlg5XeZhkt7WW2gR0TaTLbJ9Bfh7SX9CMTfvGoo+SDsCTwWOB06sJcKIaI3JFtmWAgeUy+EMAbOA+ykmf7+hxvgiokV6mqDN9j3ABfWEEhFtlylsI6IxBpqQJG0o6UpJ3y+3t5d0qaQbJZ1SrpEWES0x6YRUJpN/7vH6R1Csgzbsc8C/2H42xXCUJqxFFhFTZNIJqRxIu/tk3y9pG+AvgP8stwXsTdEDHGAhsN9kzx8R655eVx25UtIiirmQ7h3eWXHN938FPgJML7efDtxVriILsBzYerQ3SpoHzAPYbrvtJhd5RDROr3VI04DbKe5s9i0fr+/2JkmvB1bbXjKZi9peYHvI9tAWW9S6CktEDFCvzf6HTvKtLwPeUK4LP42iM+VxwKaSnlDeJW0DrOglvohYt/R0hyRpG0lnSFpdPk4r64bGZftjtrexPZtifbf/Z/ttFMsrvak8bC5wZi/xRcS6pdci2zeARcBW5eN75b7J+ijwIUk3UtQpfb3H+CJiHdJrpfYWtjsT0DclfXAiJ7B9AWVv73Jq3N16jCki1lG93iHdLuntZZ+kDcvVam+vI7CIaJ9eE9JhwAHAKmAlRf3PZCu6I6LlJl1kk7Qh8Ebbb6gxnohosV57ah9UYywR0XK9Vmr/r6QvU6zN1tlT+4oezxsRLdRrQnpB+fXTHftM0XM7ImJCeqlD2gD4qu1Ta4wnIlqslzqkRygGx0ZE1KLXZv8fSfo7SdtK2nz4UUtkEdE6vdYhvaX8+r6OfQae1eN5I6KFeh3tv31dgURE9Dra/ymSPilpQbm9YznXUUTEhNUx2v9B4KXl9grgMz2eMyJaqteEtIPtzwO/B7B9H6Ceo4qIVuo1IT0o6ckUFdlI2gH4Xc9RRUQr9drKdiRwDrCtpBMppqZ9R69BRUQ79drKdp6kK4AXUxTVjrB92/Drkp5r+5oeY4yIluj1DgnbtwNnjfHyt4A/6/UaEdEO/V5KOxXcEVFZvxOS+3z+iFiP9DshRURU1u+E9GCfzx8R65GeK7UlbQ08s/Ncti8sv7641/NHRHv0lJAkfY5ixP+1wMPlbgMX9hhXRLRQr3dI+wE72U7v7IjoWa91SDcBG9URSEREr3dI9wFLJZ1Pxxg22x/o8bwR0UK9JqRF5WNCJG0LnADMpKhzWmD7uHL621OA2cAy4ADbd/YYY0SsI3ody7ZQ0hOBPyp33WD79xXe+hDwt7avkDQdWCLpPIqBuefbPkbSfGA+8NFeYoyIdUevM0buCfwS+Arw/wG/kLRHt/fZXjm8mKTttcB1wNbAHGBhedhCikrziGiJXotsXwReY/sGAEl/BJwMvLDqCSTNBnYBLgVm2l5ZvrSKokg32nvmAfMAtttuu0mGHhFN02sr20bDyQjA9i+YQKubpE2A04AP2r678zXbZoyxcLYX2B6yPbTFFltMLvKIaJxe75AWS/pP4L/K7bcBi6u8UdJGFMnoRNunl7tvlTTL9kpJs4DVPcYXEeuQXu+Q3kPRS/sD5ePact+4JAn4OnCd7WM7XloEzC2fzwXO7DG+iFiHqCgZTfFFpd2B/wGuAh4pd3+coh7pVGA74GaKZv87xjvX0NCQFy+udFMWsX5TTdOPVcwJkpbYHqrnooVJFdkknWr7AElXMUo9j+0/He/9ti9i7MnbXjmZmCJi3TfZOqQjyq9ZFDIiajOpOqSOpvn32r658wG8t77wIqJNeq3UfvUo+17X4zkjoqUmW4f0Hoo7oR0k/bzjpenAT+sILCLaZ7J1SCcBPwD+iWK82bC13VrFIiLGMtk6pN/aXgYcB9zRUX/0kKQX1RlgRLRHr3VIXwXu6di+p9wXETFhvSYkuaNnpe1HqGHhgIhop56nsJX0AUkblY8jKKa1jYiYsF4T0ruBlwIrgOXAiyinBYmImKheZ4xcDRxYUywR0XK9zhj5R5LOl3R1uf2nkj5ZT2gR0Ta9Ftn+A/gY8HsA2z8nd0wRMUm9JqSn2L5sxL6HejxnRLRUrwnpNkk7UE5BIulNwMrx3xIRMbpe+wy9D1gAPEfSCuD/KKaxjYiYsF4Tkm2/StLGwAa210ravo7AIqJ9ei2ynQZg+95yfTWA7/R4zohoqclOP/Ic4LnA0yS9seOlpwLT6gisdnXNNwyV5xyOiImZbJFtJ4rpazcF9u3Yvxb4q16Dioh2mlRCsn0mcKakl9i+uOaYIqKleq1Duj09tSOiLumpHRGNkZ7aEdEYvfZDSk/t9c0Ur34a0akfPbXf3nNUbZE//ojH6anIZvsm268CtgCeY3v3cvL/SZP0Wkk3SLpR0vzu74iI9cVkO0Z+aIz9ANg+dpLn3RD4CsUClMuByyUtsn3tZM4XEeuWyRbZppdfdwJ2BRaV2/sCIyu5J2I34EbbNwFI+jYwB0hCimiByXaMPBpA0oXAnw2PY5N0FHBWD/FsDfymY3t4nu6IaIFeK7VnAg92bD9Y7usrSfN4bDGBeyTdUNOpZwC3VQigpstV0j2mqY0HmhlTVFPnZ/fMXoMZqdeEdAJwmaQzyu39gG/2cL4VwLYd29uU+x7H9gKK1r1aSVpse6ju8/YiMUWdmv7Z9brqyGcl/QB4ebnrUNtX9nDKy4EdyzmVVlD0+n5rLzFGxLqj51VmbV8BXFFDLNh+SNL7gXOBDYHjbV9Tx7kjovkat+y17bOBswd0+dqLgTVITFGnRn92cnr5RkRD9Dq4NiKiNklIpaYNWZF0vKTVw3NNNYGkbSX9WNK1kq6RdMSgY4pqJC2TdJWkpZIWDzqesaTIxqNDVn5Bx5AV4KBBDlmRtAdwD3CC7ecNKo5OkmYBs2xfIWk6sATYL0N7mk/SMmDIdvd+dgOUO6TCo0NWbD8IDA9ZGRjbFwJ3DDKGkWyvLFtVKXvnX0fRuz6iFklIhdGGrOQPbRySZgO7AJcONpKoyMAPJS0pRzo0UuOa/aP5JG1CsSbfB23fPeh4opLdba+QtCVwnqTry7vwRskdUqHSkJUASRtRJKMTbZ8+6HiiGtsryq+rgTMoqikaJwmp8OiQFUlPpBiysqjLe1pHxYRXXweum+ycVzH1JG1cNkJQLnv/GqAxrbedkpAohqwAw0NWrgNOHfSQFUknAxcDO0laLunwQcZTehlwMLB32Xy8VNI+gw4qupoJXCTpZxTzlZ1l+5wBxzSqNPtHRGPkDikiGiMJKSIao6/N/pKOB14PrB7ubSxpc+AUYDawDDjA9p1lhelxwD7AfcA7hjvhjWfGjBmePXt2X+KPiLEtWbLkNttb1HnOfvdD+ibwZYqZJYfNB863fUw5Zmw+8FHgdcCO5eNFwFepMJ/27NmzWby4sUNzItZbkm6u+5x9LbKNMfxhDrCwfL6QYtrb4f0nuHAJsGk5dioiWmIQdUgzbQ8vt72KxxYFyPCNiJYbaKW2iz4HE+53IGmepMWSFq9Zs6YPkUXEIAxiLNutkmbZXlkWyVaX+ysP3+hcdWRoaCgdqSIAHV3P0lM+cnB/UpXukCRtKWl/Se+TdJik3SRN9u5qETC3fD4XOLNj/yEqvBj4bUfRLiJaYNw7JEl7UbSCbQ5cSXE3M42iInoHSd8BvjjWiO9y+MOewAxJy4EjgWOAU8uhEDcDB5SHn03R5H8jRbP/oT19ZxGxzulWZNsH+Cvbvx75gqQnUPQxejXF6O8/YPugMc77ylGONfC+LvFExHps3IRk+8PjvPYQ8N3aI4qI1ppQPZCkF0s6R9IFkvbvV1AR0U7d6pCeYXtVx64PAfsDopi69Iw+xhYRLdOtDunfJV0BfN72A8BdwJuAR4BMXRoRtRq3yGZ7P4rWte9LOgT4IPAk4Ok8NuQjIqIWXeuQbH8P+HPgaRRFtF/Y/pLtdJGOiFqNm5AkvUHSj4FzKObgfQswR9K3Je0wFQFGRHt0q0P6DMXqBE8GzrW9G/C3knYEPksxGX5ERC26JaTfAm8EnsJjY86w/UuSjCKiZt3qkPanqMB+AvDW/ocTEW3W7Q7pAdv/Nt4BkjaxfU+NMUVES3W7QzpT0hcl7VEuMAeApGdJOlzSucBr+xtiRLRFt7FsrywXAnwX8DJJmwEPATcAZwFzR/TkjoiYtK4TtNk+m2JqkIiIvsq6bBHRGElIEdEYSUgR0RhdE5KkDSVdPxXBRES7VRlc+zBwg6TtpiCeiGixqssgbQZcI+ky4N7hnbbf0JeoIqKVqiakT/U1iogIKiYk2z+R9AyKkf8GLk+HyIioW9WFIt8JXEYx8v9NwCWSDutnYBHRPlWLbB8GdrF9O4CkpwM/BY7vV2AR0T5V+yHdDqzt2F5b7ouIqE3VO6QbgUslnUlRhzQH+LmkDwHYPrZP8UVEi1RNSL8qH8POLL9OrzeciGizqgnpWtv/3blD0ptH7ouI6EXVOqSPVdwXETFp3ZbSfh2wD7C1pC91vPRUionaIiJq063IdguwGHgDsKRj/1rgb/oVVES0U7cpbH8G/EzSSbZ/P9Zxkk6z/Ze1RxcRrVKpDmm8ZFR6Vg2xRETL1TVBm2s6T0S0WGaMjIjGqCshqabzRESL1ZWQPlrTeSKixcZNSJJe2/H8aZK+Lunnkk6SNHP4Nds/nOiFJS2TdJWkpZIWl/s2l3SepF+WXzeb6HkjYt3V7Q7pHzuefxFYCewLXA58rYbr72X7BbaHyu35wPm2dwTOL7cjoiUmUmQbsv1J2zfb/hdgdh/imQMsLJ8vBPbrwzUioqG69dTespxiRMBTJcn2cBN/r/VPBn4oycDXbC8AZtpeWb6+Cpg52hslzQPmAWy3XRZDiVhfdEtI/8FjU4wsBGYAa8r5tZf2eO3dba+QtCVw3si132y7TFZ/oExeCwCGhobSBypiPdFt6MjRY+xfBRzSy4Vtryi/rpZ0BsUCArdKmmV7paRZwOperhER65au8yFJehbF5P7bAg8DvwBOsn33ZC8qaWNgA9try+evAT4NLALmAseUX88c+ywRsb7p1ux/BEVr2jRgV+BJFInpEkl79nDdmcBFkn5GsZrJWbbPoUhEr5b0S+BV5XZEtES3O6R3Ai+w/bCkY4Gzbe8p6WsUdy+7TOaitm8Cnj/K/tuBV07mnBGx7qvSUjactJ4EbAJg+9fARv0KKiLaqdsd0n8Cl0u6FHg58DkASVsAd/Q5tohomW6tbMdJ+hHwx8AXbV9f7l8D7DF8nKTNbN/Z10gjYr3XtZXN9jXANV0OOx/4s1oiiojWyvQjEdEYmTEyIhojM0ZGRGOkyBYRjVEpIUn6Vpd96cwYET2reof03M4NSRsCLxzetp0+SRHRs25j2T4maS3wp5LuLh9rKUbhZ+BrRNRq3IRk+59sTwe+YPup5WO67afb/tgUxRgRLdG1YySA7Y+VE+7vSDHyf3j/hf0KLCLap1JCkvRO4AhgG4qZIl8MXAzs3b/QIqJtqlZqH0ExH9LNtveimHbkrr5FFRGtVDUhPWD7AQBJTyoH2e7Uv7Aioo0qFdmA5ZI2Bb5LMSH/ncDN/QsrItqoaqX2/uXToyT9GHgacM7w65l+JCLqUPUO6VG2fzLK7kw/EhE9y1i2iGiMTD8SEY2R6UciojFSZIuIxuiakCRtKOn6Lodl+pGI6FnXhGT7YeAGSduNc0ymH4mInlVt9t8MuEbSZcC9wzttv6EvUUVEK1VNSJ/qaxQREVTvqf0TSc8AdqNo4r/c9qq+RhYRrVN1Tu13ApcBbwTeBFwi6bB+BhYR7VO1yPZhYBfbtwNIejrwU+D4fgUWEe1TtR/S7cDaju215b6IiNpUvUO6EbhU0pkUdUhzgJ9L+hCA7WP7FF9tdHR9fTd9ZEbKRPRD1YT0q/IxbHjFken1hhMRbTZuQpL0LdsHA3fZPm6KYoqIlupWh/RCSVsBh0naTNLmnY+pCDAi2qNbke3fKSZfexawhMcPonW5PyKiFt0WivyS7T8Gjrf9LNvbdzz6kowkvVbSDZJulDS/H9eIiGaq1Oxv+z39DgSKmQWArwCvA3YGDpK081RcOyIGb8JzavfZbsCNtm8CkPRtii4G1w40qhapq3tEukbEZDQtIW0N/KZjeznwopEHSZoHzCs375F0Q03XnwHc1u0gHTWl89FVimmKdY1pin9GUV2dn90ze45mhKYlpEpsLwAW1H1eSYttD9V93l4kpqhT0z+7ps2pvQLYtmN7m3JfRLRA0xLS5cCOkraX9ETgQGDRgGOKiCnSqCKb7YckvR84F9iQorvBNVMYQu3FwBokpqhToz872WkNiYhmaFqRLSJaLAkpIhojCanUtCErko6XtFrS1YOOZZikbSX9WNK1kq6RdMSgY4pqJC2TdJWkpZIWDzqesaQOiUeHrPwCeDVFZ8zLgYNsD6yHuKQ9gHuAE2w/b1BxdJI0C5hl+wpJ0ykGXO83yJ9TVCNpGTBku2mdbB8nd0iFR4es2H4QGB6yMjC2LwQatQCn7ZW2ryifrwWuo+hdH1GLJKTCaENW8oc2DkmzgV2ASwcbSVRk4IeSlpRDrxqpUf2QYt0gaRPgNOCDtu8edDxRye62V0jaEjhP0vXlXXij5A6pkCErFUnaiCIZnWj79EHHE9XYXlF+XQ2cQVFN0ThJSIUMWalAkoCvA9etCyvNREHSxmUjBJI2Bl4DNKb1tlMSEsWQFWB4yMp1wKlTPGTlD0g6GbgY2EnSckmHDzKe0suAg4G9y+bjpZL2GXRQ0dVM4CJJP6NYgfos2+cMOKZRpdk/Ihojd0gR0RhJSBHRGElIEdEYSUgR0RhJSBHRGElI0RNJ9ww6hlh/JCHFOkFShjm1QBJS1E7SvpIulXSlpB9JmilpA0m/lLRFecwG5dxTW5SP0yRdXj5eVh5zlKRvSfpf4FuSnivpsrJD5s8l7TjQbzRql4QU/XAR8GLbu1BM5fIR248A/wW8rTzmVcDPbK8BjgP+xfauwF8C/9lxrp2BV9k+CHg3cJztFwBDFLMyxHokt8HRD9sAp5QTuj0R+L9y//HAmcC/AocB3yj3vwrYuRgqB8BTyxkFABbZvr98fjHwCUnbAKfb/mV/v42YarlDin74N+DLtv8EeBcwDcD2b4Bbpf+/vfOPsauo4vjn2xVoqbQV2xhrhEqBNmBgdSuklWpplJggULX+IBi6aozRpLSpNJpUsQkiUIxBbEgjpK4aftSq1Fq1sNndslvL2rp0S7coarpComiU2Co/ClqOf5zz2Luv97739ke7b5v5JC879975cWbu7Lkzc+fco8W4tfmvIv4EfETVGL+3mFlpsfyFUqZmdj9wNfAS8MvIJ3ESkRRS4ngwlYHPtywru3YvPnXbbGZH49wjwPJSBEmNeZlKOgc4aGZ34SOti0ZT6MTYkxRSYqScHl8jKP1WAWuBzZJ6gPJvOG8FXs/AdA3gBmBeLFQ/ia8V5fExoE9SL/B24AejWZHE2JOs/RMnFEnz8AXshWMtS6L+SIvaiRNGuJf6PANv2hKJQaQRUiKRqBvqcg1J0hszXyT8m6S/RPhQrDEMN99mSetHU9Yq5U2T9IUa466VdGOVOEskXTAaskiaKenHw8lrLJG0SNK2IabZEVPFE4qklZJOH0makZjmDKetRspIHZzWpUIys+dKr4CBDfiaQyPQCLw6ttINiWlATQqpRpbgGwVHLIuZ/dXMlo6KVDVQbvpRqynIODcZWQkMSSENM82YUHBvWoAPDDfPulRIVWiQdE+4cn5E0iQASbMlbQ+/U12S5lbKRNIsSe3xZqdN0lmSGiT1y5km6ajcgyySOiWdFx9M3xgmDHslXRPX88wabgNmx7k7cmRYI+kPknYCczLnj6mLpAX4Hpw7Ir/ZRXWWm2o8JGlf/BaUyxL174v4EyV9T+5qea+ky+N8s6SfRhl/lLSuoC2bJD0acjws3xBZGpncKXfdvEJSi6QNkn4DrJN0pqQt0V7dki6KdINMRnKKnCLpF3LX5xskTYh0V0h6TNLjkjZrYHNlVtZro559km7PnH9e0i3ReN+k9gAABUtJREFUXt2S3pS5F92R5uulEYukN0ef6I28FpaVcwMwE+iQ1FGp7Epp4nyeXLnmNkVIuiTaZq+kXZLmxPlOZbZZSNop6eIK/bxZ0lZJ7UBbeTkjdnBqZnX9w18h3xjhWcD/gMY4/hHwyQi3AedF+FKgPSevZnzDHsDPgWUR/jSwJcLbgQuBD+LeSNYApwH9cf0bmTKn4S64J+ObAa+L86cCk0LevoJ6NQH78afhFOBPmXrm1gV/+izN5FEUbxPuMw2gAd8XNEiW7DHwRWBjhOcCz+CbGZuBg5F+IvA08NayepwC7AJmxPHHM3ntAO7OxG0BtgENcfwd4GsRXgz0Zu55DzApp90WAUeAc6JurcBSYDrQCUyOeF8CbsrIMQ//Z38GmIG/0GnHXYGDO1K8KsLrgK9EeBvuVh18O8LzmTZbk2njM3Jk/TMwPcKFZRelqSLX/bivNYCzcE8weW21LcJTgNdF+H3ATyK8DLgzwucDv63Sz5txk50zK/zPzqKg31f7jcfhcL+Z9Ua4B5gVT8IF+N6XUrzTquQzH/hwhH+I32yALuA9wNuAW4HPAo/iygnchczVGljvmYh3iGPMGjKy5LEQeMjMXgSQtDX+1lSXKvEWA9cDmG8+PCzpDRVkuQxXDpjZ7yU9jXdOgDYzOxxlPgmczWAvv3PwPUGtIUcD8Gzm+qaysrIbIi/Dbdcws3b52uGUuJY1GSlnt5kdDJkeiHyO4NPZX4ccp+L3JMu7gB3m9nNIug+/11uAV3DlA96v3h/h+fhUGVwJfDPCe4CNcj91WzJ9sohKZVeiSK5ccxsb2OFezlTg+/KRu+EPEoDNwFclrcYfzC1xvqifA7Sa2XFx8z4eFdLLmfBRfCQyAThkvs40UjrxV9MzgZuA1fiTpiuuC/iImT1Vlu53MRW5Ejdr+Bw+uhgqtdZlNOtcifL2Lu8zAg6Y2fyC9C9UOS6iUrzyV8MWcrSaG+EOh/9aPN7Jr+fgAs065dP5K4EWSd8ys+OxUbNIrpK5zZEa87kZ6DCzD8ndoO8AMLMXJbUC1+AbT5sifm4/l3Qptd/DITMe15COwdydc7+kj4I7NJR0cZVku3CHkOD7YkoKZzc+8ng1bnYvbo9Vcjv8MLBc8WiS9I74m2fW8B/gjILyO4ElkibJnfhdVUNdXsuvSrw2XKkiXxebWkWWrmgDJJ2PPwnLFW4RTwEzJM2P9KdIurDGtNlyFwH/tNpcc18id+o5AZ8i7gS6gXdLOjfymxx1ybIbeK+k6ZIagGvx0W8luolRHAP9BUlnA383s3twc5h35qTNtnmtZVe6T1lqMrfJkDXnaS67di9wF7DHzP4V53L7+fHmpFBIwXXAZ+TO8A7gGr8Sy4FPSXoCd364AsDMXsanJN0RrwvvIPvj+GZ8uPuEpANxDDlmDWb2HD6F6FPZoraZPY5PZ/bhRqZ7MpeL6vIgsDoWGWdXiLcCuFzSfnyYf0ElWYC7gQkRfxPQHO1QFTN7BV/DuT3k6MUVei2sBZriHtzGsXZvRewB1uNOPfvxqe8/8H+0ByK/x/D1sKyszwJfBjrwdu8xs59VKWslsCryPBc4HOcXAfsk7cWV4rdz0n4X2C6pYwhlv5amily1mtuUWAfcGvIOGv2ZWQ/wbwab8xT184pohA5O08bIRKIC8j1BL5mZSfoEvsBd7WE3rpA0E5/CzTX/btWYMR7XkBKJE0kTsD6mLofwhd+TBknXA7cAq8ZaGUEaISUSiTriZFpDSiQS45ykkBKJRN2QFFIikagbkkJKJBJ1Q1JIiUSibkgKKZFI1A3/BwcrYljO+onRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x720 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The lowest detection error belongs to the layer  1\n"
     ]
    }
   ],
   "source": [
    "x = slc_layers\n",
    "bar_size=0.5\n",
    "row_size = 5\n",
    "figsize=(len(slc_layers),10)\n",
    "fig = plt.figure(1, figsize=figsize)\n",
    "\n",
    "plt.subplot(row_size, 1, 1)\n",
    "plt.bar(x, aupr_in, width=bar_size, color=\"orange\")\n",
    "plt.title(\"Multiple metrics\\n\"+id_name+\" | \"+id_model+\" | \"+ood_ds_name)\n",
    "plt.ylabel('aupr_in (%)')\n",
    "plt.xticks(x)\n",
    "\n",
    "plt.subplot(row_size, 1, 2)\n",
    "plt.bar(x, aupr_out, width=bar_size, color=\"blue\")\n",
    "plt.ylabel('aupr_out (%)')\n",
    "plt.xticks(x)\n",
    "\n",
    "plt.subplot(row_size, 1, 3)\n",
    "plt.bar(x, auroc, width=bar_size, color=\"gray\")\n",
    "plt.ylabel('auroc (%)')\n",
    "plt.xticks(x)\n",
    "\n",
    "plt.subplot(row_size, 1, 4)\n",
    "plt.bar(x, detection_error, width=bar_size, color=\"red\")\n",
    "plt.ylabel('detection_error (%)')\n",
    "plt.xticks(x)\n",
    "\n",
    "plt.subplot(row_size, 1, 5)\n",
    "plt.bar(x, fpr_at_95_tpr, width=bar_size, color=\"green\")\n",
    "plt.ylabel('fpr_at_95_tpr (%)')\n",
    "plt.xticks(x)\n",
    "\n",
    "plt.xlabel('Layers\\n'+\"The lowest detection error belongs to the layer \"+\\\n",
    "                           str(slc_layers[np.argmin(detection_error)]))\n",
    "plt.show()\n",
    "print(\"The lowest detection error belongs to the layer \", str(slc_layers[np.argmin(detection_error)]))\n",
    "fig.savefig(base_address+id_name+\"-\"+id_model+\"-\"+ood_ds_name+\".png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}