LDU_demo_notebook.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "LDU_demo_notebook.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "sGNuZWLd9ODb",
        "ciPGnuuR9oZz",
        "2peEZI_79yWp",
        "I2q_6p-j98lC"
      ],
      "machine_shape": "hm",
      "authorship_tag": "ABX9TyO5TXguyydf8+rz3B58R+Bp",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/liu-res/Learning-to-Defer-with-Uncertainty/blob/main/LDU_demo_notebook.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dYKpT-c6s91F"
      },
      "source": [
        "The LDU_demo_notebook.ipynb can be used to apply the learning to defer with uncertainty (LDU) algorithm on variouse deep learning classification tasks, to reduce the risk or uncertainty in predictions. The LDU algorithm is described in publication at http://arxiv.org/abs/2108.07392"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sGNuZWLd9ODb"
      },
      "source": [
        "#### Example base model where the LDU algorithm will apply:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JdmlFwOy75Q_",
        "outputId": "86a49717-f2c2-439a-fc4b-4555757fae8a"
      },
      "source": [
        "import numpy as np\n",
        "import torch\n",
        "import torchvision\n",
        "import matplotlib.pyplot as plt\n",
        "from time import time\n",
        "from torchvision import datasets, transforms\n",
        "from torch import nn, optim\n",
        "import torch.nn.functional as F\n",
        "from sklearn.metrics import classification_report\n",
        "import os\n",
        "from google.colab import drive\n",
        "drive.mount('drive')\n",
        "\n",
        "class base_model_class(nn.Module):\n",
        "    # predict if the MNIST digit is even or odd number\n",
        "    def __init__(self):\n",
        "        super(base_model_class, self).__init__()\n",
        "        self.input_size = 784\n",
        "        self.hidden_sizes = [128, 64]\n",
        "        self.output_size = 2 #10, \n",
        "        self.l1 = nn.Linear(self.input_size, self.hidden_sizes[0])\n",
        "        self.relu = nn.ReLU()\n",
        "        self.l2 = nn.Linear(self.hidden_sizes[0], self.hidden_sizes[1])\n",
        "        self.l3 = nn.Linear(self.hidden_sizes[1], self.output_size)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.l1(x)\n",
        "        x = self.relu(x)\n",
        "        x = self.l2(x)\n",
        "        x = self.relu(x)\n",
        "        x = self.l3(x)\n",
        "        return F.softmax(x, dim=1)\n",
        "\n",
        "\n",
        "def trainer(model):\n",
        "    transform = transforms.Compose([transforms.ToTensor(),\n",
        "                                    transforms.Normalize((0.5,), (0.5,)),\n",
        "                                    ])\n",
        "    path_to_trainset = '/content/drive/MyDrive/demo_data/'\n",
        "    if not os.path.exists(path_to_trainset):\n",
        "        os.makedirs(path_to_trainset)\n",
        "\n",
        "    dataset = datasets.MNIST(path_to_trainset, download=True, train=True, transform=transform)\n",
        "    trainloader = torch.utils.data.DataLoader(dataset, batch_size=64, shuffle=True)\n",
        "\n",
        "    device = 'cpu'\n",
        "    epochs = 15\n",
        "    loss_function = nn.NLLLoss().to(device)\n",
        "    optimizer = optim.SGD(model.parameters(), lr=0.003, momentum=0.9)\n",
        "    time0 = time()\n",
        "    epochs = epochs\n",
        "    for e in range(epochs):\n",
        "        running_loss = 0\n",
        "        for images, labels in trainloader:\n",
        "            # Flatten MNIST images into a 784 long vector\n",
        "            images = images.view(images.shape[0], -1)\n",
        "            # Training pass\n",
        "            optimizer.zero_grad()\n",
        "            output = model(images)\n",
        "            # Convert MNIST labels to binary label indicating even or non-even(odd) digit\n",
        "            labels = (labels%2==0).long()\n",
        "            loss = loss_function(output, labels)\n",
        "            # This is where the model learns by backpropagating\n",
        "            loss.backward()\n",
        "            # And optimizes its weights here\n",
        "            optimizer.step()\n",
        "            running_loss += loss.item()\n",
        "        print(\"Epoch {} - Training loss: {}\".format(e, running_loss / len(trainloader)))\n",
        "    print(\"\\nTraining Time (in minutes) =\", (time() - time0) / 60)\n",
        "    return model\n",
        "\n",
        "\n",
        "def validator(model, train=False, shuffle=False):\n",
        "    transform = transforms.Compose([transforms.ToTensor(),\n",
        "                                    transforms.Normalize((0.5,), (0.5,)),\n",
        "                                    ])\n",
        "    path_to_testset = '../demo_data/'\n",
        "    if not os.path.exists(path_to_testset):\n",
        "        os.makedirs(path_to_testset)\n",
        "\n",
        "    dataset = datasets.MNIST(path_to_testset, download=True, train=train, transform=transform)\n",
        "    dataloader = torch.utils.data.DataLoader(dataset, batch_size=64, shuffle=shuffle)\n",
        "\n",
        "    with torch.set_grad_enabled(False):\n",
        "        label = []\n",
        "        pred = []\n",
        "        prob_NP = np.empty((0, model.output_size)).astype(float)  # 2-D array\n",
        "        for images,labels in dataloader:\n",
        "            # Flatten MNIST images into a 784 long vector\n",
        "            images = images.view(images.shape[0], -1)\n",
        "            out_mini = model(images)\n",
        "            label.extend((labels%2==0).long().tolist())\n",
        "            pred.extend(torch.max(out_mini.cpu().data, 1)[1].numpy().astype(int).tolist())\n",
        "            prob_NP = np.concatenate((prob_NP, out_mini.cpu().data.numpy().astype(float)), axis=0)\n",
        "            del images\n",
        "            del out_mini\n",
        "        print('\\nClassification Report\\n')\n",
        "        print(classification_report(label, pred, target_names=['cls{}'.format(i) for i in range(model.output_size)]))\n",
        "    return label, pred, prob_NP\n",
        "\n",
        "\n"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at drive; to attempt to forcibly remount, call drive.mount(\"drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ciPGnuuR9oZz"
      },
      "source": [
        "#### Generate deep ensembles:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j-0zZZwf9uZS",
        "outputId": "888f2b52-4336-44e6-b2f8-78525bced636"
      },
      "source": [
        "#!~/.venvs/pt/bin/python\n",
        "# -*- coding: utf-8 -*-\n",
        "from sklearn.metrics import f1_score, confusion_matrix\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "from torch.autograd import Variable\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import os\n",
        "import math\n",
        "from torch.nn.utils import clip_grad_norm_\n",
        "import logging\n",
        "import random\n",
        "from scipy.stats import entropy\n",
        "from google.colab import drive\n",
        "drive.mount('drive')\n",
        "\n",
        "class deep_ensemble_class:\n",
        "    def __init__(self, n_ensembles, device):\n",
        "        self.n_ensembles = n_ensembles\n",
        "        self.device = device\n",
        "        random.seed(47)\n",
        "        self.list_seeds = random.sample(range(100), self.n_ensembles)\n",
        "        self.path_ensemble = '/content/drive/MyDrive/ensemble_data/'\n",
        "\n",
        "    def calculate_binned_entropy(self, i_preds, n_classes):\n",
        "        # i_preds: predictions from deep ensembles for ith data point\n",
        "        pks = []\n",
        "        for i in range(n_classes):\n",
        "            pks.append(float(len(i_preds[i_preds == i]) / self.n_ensembles))\n",
        "        pks = np.array(pks)\n",
        "        binned_entropy = entropy(pks)\n",
        "        return binned_entropy\n",
        "\n",
        "    def add_entropies(self, df_ens, pred_NP, n_classes, filter_zeros):\n",
        "        logging.info('calculating uncertainty-related features...')\n",
        "        if n_classes == 2:  # binary classification\n",
        "            binned_entropy = []\n",
        "            continuous_entropy = []\n",
        "            col = ['ens{}cls1'.format(i) for i in range(self.n_ensembles)]\n",
        "            df_probs = df_ens[col]\n",
        "            for i in range(df_ens.shape[0]):\n",
        "                preds = pred_NP[:, i]\n",
        "                binned_entropy.append(self.calculate_binned_entropy(preds, n_classes))\n",
        "                continuous_entropy.append(entropy(df_probs.loc[i]))\n",
        "            if filter_zeros:\n",
        "                if not all(v == 0 for v in binned_entropy):\n",
        "                    df_ens['binned_entropy'] = binned_entropy\n",
        "                else:\n",
        "                    logging.info('binned entropy are all zero, will not included in LDU')\n",
        "\n",
        "                if not all(v == 0 for v in continuous_entropy):\n",
        "                    df_ens['continuous_entropy'] = continuous_entropy\n",
        "                else:\n",
        "                    logging.info('continuous entropy for binary class are all zero, will not included in LDU')\n",
        "            else:\n",
        "                df_ens['binned_entropy'] = binned_entropy\n",
        "                df_ens['continuous_entropy'] = continuous_entropy\n",
        "\n",
        "        else:  # multi-classificaton\n",
        "            binned_entropy = []\n",
        "            for i in range(df_ens.shape[0]):\n",
        "                preds = pred_NP[:, i]\n",
        "                binned_entropy.append(self.calculate_binned_entropy(preds, n_classes))\n",
        "\n",
        "            if filter_zeros:\n",
        "                if not all(v == 0 for v in binned_entropy):\n",
        "                    df_ens['binned_entropy'] = binned_entropy\n",
        "                else:\n",
        "                    logging.info('binned entropy are all zero, will not included in LDU')\n",
        "            else:\n",
        "                df_ens['binned_entropy'] = binned_entropy\n",
        "\n",
        "            for c in range(n_classes):\n",
        "                continuous_entropy = []\n",
        "                col = ['ens{}cls{}'.format(i, c) for i in range(self.n_ensembles)]\n",
        "                df_probs = df_ens[col]\n",
        "                for i in range(df_ens.shape[0]):\n",
        "                    continuous_entropy.append(entropy(df_probs.loc[i]))\n",
        "                if filter_zeros:\n",
        "                    if not all(v == 0 for v in continuous_entropy):\n",
        "                        df_ens['continuous_entropy_cls{}'.format(c)] = continuous_entropy\n",
        "                    else:\n",
        "                        logging.info('continuous entropy for class {} are all zero, will not included in LDU'.format(c))\n",
        "                else:\n",
        "                    df_ens['continuous_entropy_cls{}'.format(c)] = continuous_entropy\n",
        "        return df_ens\n",
        "\n",
        "    def ens_to_dataset(self, df_ens):\n",
        "        x_NP = df_ens.drop(['label'], axis=1).to_numpy()\n",
        "        labelNP = df_ens.label.to_numpy()\n",
        "        x = torch.FloatTensor(x_NP.tolist())\n",
        "        label = torch.LongTensor(labelNP.tolist())\n",
        "        dataset = torch.utils.data.TensorDataset(x, label)\n",
        "        return dataset\n",
        "\n",
        "    def gether_ensembles(self, base_model_class, trainer, validator):\n",
        "        df_train_ens = pd.DataFrame()\n",
        "        df_test_ens = pd.DataFrame()\n",
        "        train_pred = []\n",
        "        test_pred = []\n",
        "        compare_train_order = []\n",
        "        compare_test_order = []\n",
        "\n",
        "        for i in range(self.n_ensembles):\n",
        "            # seed: control initialization state of the model\n",
        "            seed = self.list_seeds[i]\n",
        "            logging.info('ensemble {}, seed={} :'.format(i, seed))\n",
        "            torch.cuda.empty_cache()\n",
        "            torch.manual_seed(seed)\n",
        "            model = base_model_class().to(self.device)\n",
        "            model = trainer(model)\n",
        "            train_label, train_pred_list, train_prob_NP = validator(model, train=True, shuffle=False)\n",
        "            test_label, test_pred_list, test_prob_NP = validator(model, train=False, shuffle=False)\n",
        "            if i != 0:\n",
        "                if compare_train_order != train_label or compare_test_order != test_label:\n",
        "                    logging.error('validation orders changed, need to turn off shuffle in validator')\n",
        "                    return None, None, None, None\n",
        "\n",
        "            compare_train_order = train_label\n",
        "            compare_test_order = test_label\n",
        "\n",
        "            train_pred.append(train_pred_list)\n",
        "            test_pred.append(test_pred_list)\n",
        "            for j in range(train_prob_NP.shape[1]):\n",
        "                df_train_ens['ens{}cls{}'.format(i, j)] = train_prob_NP[:, j].tolist()\n",
        "                df_test_ens['ens{}cls{}'.format(i, j)] = test_prob_NP[:, j].tolist()\n",
        "\n",
        "        n_classes = train_prob_NP.shape[1]\n",
        "        df_train_ens['label'] = train_label\n",
        "        df_test_ens['label'] = test_label\n",
        "        train_pred = np.array(train_pred)\n",
        "        test_pred = np.array(test_pred)\n",
        "\n",
        "        df_train_ens = self.add_entropies(df_train_ens, train_pred, n_classes, filter_zeros=True)\n",
        "        df_test_ens = self.add_entropies(df_test_ens, test_pred, n_classes, filter_zeros=False)\n",
        "        df_test_ens = df_test_ens[df_train_ens.columns.tolist()]\n",
        "\n",
        "        if not os.path.exists(self.path_ensemble):\n",
        "            os.makedirs(self.path_ensemble)\n",
        "        df_train_ens.to_csv(os.path.join(self.path_ensemble, 'train_ensembles.csv'), index=False)\n",
        "        df_test_ens.to_csv(os.path.join(self.path_ensemble, 'test_ensembles.csv'), index=False)\n",
        "        pd.DataFrame([n_classes]).to_csv(os.path.join(self.path_ensemble, 'n_classes.csv'), index=False)\n",
        "\n",
        "        return n_classes, df_train_ens.shape[1]-1, self.ens_to_dataset(df_train_ens), self.ens_to_dataset(df_test_ens)\n",
        "\n",
        "    def load_ensemble(self):\n",
        "        df_train_ens = pd.read_csv(os.path.join(self.path_ensemble, 'train_ensembles.csv'), sep=',')\n",
        "        df_test_ens = pd.read_csv(os.path.join(self.path_ensemble, 'test_ensembles.csv'), sep=',')\n",
        "        n_classes = pd.read_csv(os.path.join(self.path_ensemble, 'n_classes.csv'), sep=',')\n",
        "        return int(n_classes.loc[0]), df_train_ens.shape[1] - 1, self.ens_to_dataset(df_train_ens), self.ens_to_dataset(df_test_ens)\n",
        "\n",
        "\n"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at drive; to attempt to forcibly remount, call drive.mount(\"drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2peEZI_79yWp"
      },
      "source": [
        "#### LDU network:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O2Hwpx_093gZ",
        "outputId": "afd09390-62ba-4b6e-aae4-2670d3e9312d"
      },
      "source": [
        "#!~/.venvs/pt/bin/python\n",
        "# -*- coding: utf-8 -*-\n",
        "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torch.autograd import Variable\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import os\n",
        "import math\n",
        "from torch.nn.utils import clip_grad_norm_\n",
        "import logging\n",
        "from tabulate import tabulate\n",
        "from google.colab import drive\n",
        "drive.mount('drive')\n",
        "\n",
        "\n",
        "class ldu_model(nn.Module):\n",
        "\n",
        "    def __init__(self, n_classes, input_size):\n",
        "        super(ldu_model, self).__init__()\n",
        "        self.n_classes = n_classes\n",
        "        self.input_size = input_size\n",
        "        self.hidden_size = 200\n",
        "        self.output_size = n_classes + 1\n",
        "        self.l1 = nn.Linear(self.input_size, self.hidden_size)\n",
        "        self.sigmoid = nn.Sigmoid()\n",
        "        self.l3 = nn.Linear(self.hidden_size, self.output_size)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.l1(x)\n",
        "        x = self.sigmoid(x)\n",
        "        x = self.l3(x)\n",
        "        x = F.softmax(x, dim=1)\n",
        "        return x\n",
        "\n",
        "\n",
        "class learning_ldu_class:\n",
        "\n",
        "    def __init__(self, alpha_list, device='cpu', learning_rate=0.0009, epochs=20):\n",
        "        self.device = device\n",
        "        self.learning_rate = learning_rate\n",
        "        self.epochs = epochs\n",
        "        self.alpha_list = alpha_list\n",
        "        self.n_classes = None\n",
        "        self.input_size = None\n",
        "        self.seed = 47\n",
        "        self.batch_size = 64\n",
        "        self.defer_loss = nn.CrossEntropyLoss().to(self.device)\n",
        "        self.pred_loss = nn.CrossEntropyLoss().to(self.device)\n",
        "\n",
        "    def loss_function(self, outputs, labels, alpha):\n",
        "        # Assume expert's predictions always correct and the expert cost is a constant\n",
        "        batch_size = outputs.size()[0]\n",
        "        rc = torch.tensor([self.n_classes] * batch_size)\n",
        "        loss = self.defer_loss(outputs[range(batch_size)], rc) + alpha * self.pred_loss(outputs[range(batch_size)], labels)\n",
        "        return loss\n",
        "\n",
        "    def start_training(self, model, train_set, alpha):\n",
        "        # model: the LDU model\n",
        "        # alpha: the weight on defer option\n",
        "        train_loader = torch.utils.data.DataLoader(dataset=train_set, batch_size=self.batch_size, shuffle=True)\n",
        "        model = model.to(self.device)\n",
        "        # setup optimizer\n",
        "        optimizer = optim.Adam(model.parameters(), lr=self.learning_rate)\n",
        "        # track training loss\n",
        "        loss_epoch = np.zeros(self.epochs)\n",
        "        # Training loop\n",
        "        for e in range(self.epochs):\n",
        "            running_loss = 0\n",
        "            for x_var, y_var in train_loader:\n",
        "                x_var = Variable(x_var)\n",
        "                y_var = Variable(y_var)\n",
        "                optimizer.zero_grad()\n",
        "                out_mini = model(x_var)\n",
        "                loss = self.loss_function(out_mini, y_var, alpha)\n",
        "                # compute the gradient based on the loss\n",
        "                loss.backward()\n",
        "                # clip gradient to prevent underflow ( check by: out_mini.min())\n",
        "                # clip_grad_norm_(parameters=model.parameters(), max_norm=1.0)\n",
        "                optimizer.step()\n",
        "                running_loss += loss.item()\n",
        "\n",
        "            logging.info('epoch: {}, loss = {}'.format(e, running_loss/len(train_loader)))\n",
        "            loss_epoch[e] = running_loss/len(train_loader)\n",
        "\n",
        "        return model\n",
        "\n",
        "    def predict(self, model, test_set):\n",
        "        test_loader = torch.utils.data.DataLoader(dataset=test_set, batch_size=self.batch_size, shuffle=False)\n",
        "        with torch.set_grad_enabled(False):\n",
        "            pred = []\n",
        "            label = []\n",
        "            for x_var, y_var in test_loader:\n",
        "                out_mini = model(x_var)\n",
        "                pred.extend(torch.max(out_mini.cpu().data, 1)[1].numpy().astype(int).tolist())\n",
        "                label.extend(y_var.cpu().data.numpy().astype(float).tolist())\n",
        "\n",
        "                del x_var\n",
        "                del out_mini\n",
        "            return pred, label\n",
        "\n",
        "    def get_scores(self, test_y, pred_y):\n",
        "        # test_y: list of true testing labels\n",
        "        # pred_y: list of predicted labels\n",
        "        # return f1, recall, precision\n",
        "        if len(test_y) < 1:\n",
        "            return [None, None, None, None]\n",
        "        else:\n",
        "            f1 = f1_score(test_y, pred_y, average='micro', zero_division=0)\n",
        "            recall = recall_score(test_y, pred_y, average='micro', zero_division=0)\n",
        "            precision = precision_score(test_y, pred_y, average='micro', zero_division=0)\n",
        "            return [f1, recall, precision]\n",
        "\n",
        "    def triage_scores(self, pred_y, test_y, n_classes):\n",
        "        df = pd.DataFrame(\n",
        "            {'true_y': test_y,\n",
        "             'pred_y': pred_y\n",
        "             })\n",
        "        df_1 = df[df.pred_y != n_classes]\n",
        "        df_2 = df[df.pred_y == n_classes]\n",
        "\n",
        "        if not df_2.empty:\n",
        "            df_expert = df_2.copy()\n",
        "            df_expert['pred_y'] = df_expert['true_y']\n",
        "            df_overall = df_1.append(df_expert)\n",
        "            deferred_size = df_2.shape[0]\n",
        "        else:\n",
        "            df_overall = df_1.copy()\n",
        "            deferred_size = 0\n",
        "\n",
        "        deferred_ratio = deferred_size / float(df.shape[0])\n",
        "        logging.info('deferred ratio = {}'.format(deferred_ratio))\n",
        "\n",
        "        if not df_1.empty:\n",
        "            size_for_automation = df_1.shape[0]\n",
        "            model_scores = self.get_scores(df_1.true_y.tolist(), df_1.pred_y.tolist())\n",
        "            logging.info(\n",
        "                'Automated F1={}, recall={}, precision={}'.format(model_scores[0], model_scores[1], model_scores[2]))\n",
        "        else:\n",
        "            size_for_automation = 0\n",
        "            model_scores = [None, None, None]\n",
        "\n",
        "        overall_scores = self.get_scores(df_overall.true_y.tolist(), df_overall.pred_y.tolist())\n",
        "\n",
        "        result = [deferred_ratio, deferred_size, size_for_automation]\n",
        "        result.extend(overall_scores)\n",
        "        result.extend(model_scores)\n",
        "\n",
        "        return result\n",
        "\n",
        "    def tune_ldu(self, ensemble_funcion):\n",
        "        self.n_classes, self.input_size, train_set, test_set = ensemble_funcion()\n",
        "        logging.info('num classes : {},  LDU input size : {}'.format(self.n_classes, self.input_size))\n",
        "\n",
        "        df_score = pd.DataFrame(columns=['Alpha', 'Deferred_Ratio', 'Deferred_Size', 'Auto_Size', \\\n",
        "                                         'Overall_F1', 'Overall_Recall', 'Overall_Precision', \\\n",
        "                                         'Auto_F1 (micro)', 'Auto_Recall (micro)', 'Auto_Precision (micro)'],\n",
        "                                dtype=object)\n",
        "        i = 0\n",
        "        for alpha in self.alpha_list:\n",
        "            logging.info('LDU : alpha={}'.format(alpha))\n",
        "            torch.cuda.empty_cache()\n",
        "            torch.manual_seed(self.seed)\n",
        "            model = ldu_model(n_classes=self.n_classes, input_size=self.input_size)\n",
        "\n",
        "            model = self.start_training(model, train_set, alpha)\n",
        "            pred, label = self.predict(model, test_set)\n",
        "            result = self.triage_scores(pred, label, self.n_classes)\n",
        "            df_score.loc[i] = [alpha] + result\n",
        "            i += 1\n",
        "\n",
        "        print(tabulate(df_score, headers='keys', tablefmt='psql'))\n",
        "        return df_score\n"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at drive; to attempt to forcibly remount, call drive.mount(\"drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I2q_6p-j98lC"
      },
      "source": [
        "#### Run LDU algorithm demo:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_unCqxVd-Fpk",
        "outputId": "257c92ce-eb53-4fdc-b719-ef1da9898737"
      },
      "source": [
        "import logging\n",
        "import numpy as np\n",
        "from google.colab import drive\n",
        "drive.mount('drive')\n",
        "\"\"\"\n",
        "Plugin own base_model_class, trainer, validator in example base model\n",
        "import base_model_class, trainer, validator\n",
        "\"\"\"\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    import os\n",
        "    import sys\n",
        "    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'\n",
        "    sys.path.append('libs')\n",
        "    logging.getLogger().setLevel(logging.INFO)\n",
        "    s1 = deep_ensemble_class(n_ensembles=10, device='cpu')\n",
        "    s2 = learning_ldu_class(alpha_list=np.arange(1.9, 3.9, 0.2).tolist(), device='cpu', learning_rate=0.0008, epochs=5)\n",
        "\n",
        "    if not os.path.exists(s1.path_ensemble): # from training base model\n",
        "        ldu_scores = s2.tune_ldu(lambda: s1.gether_ensembles(base_model_class, trainer, validator))\n",
        "    else: # from ensemble data\n",
        "        ldu_scores = s2.tune_ldu(lambda: s1.load_ensemble())\n",
        "\n",
        "    if not os.path.exists('/content/drive/MyDrive/ldu_result'):\n",
        "        os.makedirs('/content/drive/MyDrive/ldu_result')\n",
        "    ldu_scores.to_csv(os.path.join('/content/drive/MyDrive/ldu_result', 'ldu_scores.csv'), index=False)\n"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:root:ensemble 0, seed=45 :\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at drive; to attempt to forcibly remount, call drive.mount(\"drive\", force_remount=True).\n",
            "Epoch 0 - Training loss: -0.813834425355834\n",
            "Epoch 1 - Training loss: -0.8852618528581632\n",
            "Epoch 2 - Training loss: -0.9144706295878648\n",
            "Epoch 3 - Training loss: -0.9402636156153323\n",
            "Epoch 4 - Training loss: -0.9535922433521702\n",
            "Epoch 5 - Training loss: -0.9607571510872098\n",
            "Epoch 6 - Training loss: -0.9644989745576245\n",
            "Epoch 7 - Training loss: -0.9680954984256199\n",
            "Epoch 8 - Training loss: -0.9707762233611109\n",
            "Epoch 9 - Training loss: -0.9727823816891163\n",
            "Epoch 10 - Training loss: -0.9741469859314371\n",
            "Epoch 11 - Training loss: -0.9758333202872449\n",
            "Epoch 12 - Training loss: -0.9770013918119199\n",
            "Epoch 13 - Training loss: -0.9781477789380657\n",
            "Epoch 14 - Training loss: -0.9792244666929184\n",
            "\n",
            "Training Time (in minutes) = 2.1195475578308107\n",
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.99      0.98      0.98     30508\n",
            "        cls1       0.98      0.99      0.98     29492\n",
            "\n",
            "    accuracy                           0.98     60000\n",
            "   macro avg       0.98      0.98      0.98     60000\n",
            "weighted avg       0.98      0.98      0.98     60000\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:root:ensemble 1, seed=8 :\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.98      0.98      0.98      5074\n",
            "        cls1       0.98      0.98      0.98      4926\n",
            "\n",
            "    accuracy                           0.98     10000\n",
            "   macro avg       0.98      0.98      0.98     10000\n",
            "weighted avg       0.98      0.98      0.98     10000\n",
            "\n",
            "Epoch 0 - Training loss: -0.8139152022312953\n",
            "Epoch 1 - Training loss: -0.8812870760716355\n",
            "Epoch 2 - Training loss: -0.9100940018447478\n",
            "Epoch 3 - Training loss: -0.937942372075022\n",
            "Epoch 4 - Training loss: -0.9523910366014631\n",
            "Epoch 5 - Training loss: -0.9596892055799203\n",
            "Epoch 6 - Training loss: -0.9643944275023332\n",
            "Epoch 7 - Training loss: -0.968103573901821\n",
            "Epoch 8 - Training loss: -0.9711289788995471\n",
            "Epoch 9 - Training loss: -0.9730748116715885\n",
            "Epoch 10 - Training loss: -0.9750166649121974\n",
            "Epoch 11 - Training loss: -0.976245145871441\n",
            "Epoch 12 - Training loss: -0.9775875089392225\n",
            "Epoch 13 - Training loss: -0.9785807196900789\n",
            "Epoch 14 - Training loss: -0.9801688581260283\n",
            "\n",
            "Training Time (in minutes) = 2.1006414969762166\n",
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.95      0.99      0.97     30508\n",
            "        cls1       0.99      0.94      0.97     29492\n",
            "\n",
            "    accuracy                           0.97     60000\n",
            "   macro avg       0.97      0.97      0.97     60000\n",
            "weighted avg       0.97      0.97      0.97     60000\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:root:ensemble 2, seed=55 :\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.94      0.99      0.97      5074\n",
            "        cls1       0.99      0.94      0.96      4926\n",
            "\n",
            "    accuracy                           0.97     10000\n",
            "   macro avg       0.97      0.97      0.97     10000\n",
            "weighted avg       0.97      0.97      0.97     10000\n",
            "\n",
            "Epoch 0 - Training loss: -0.814471991553998\n",
            "Epoch 1 - Training loss: -0.8818853634761087\n",
            "Epoch 2 - Training loss: -0.9082121229502184\n",
            "Epoch 3 - Training loss: -0.9345049652844858\n",
            "Epoch 4 - Training loss: -0.9497338781860083\n",
            "Epoch 5 - Training loss: -0.9574895023282911\n",
            "Epoch 6 - Training loss: -0.9626767493641453\n",
            "Epoch 7 - Training loss: -0.966269744230486\n",
            "Epoch 8 - Training loss: -0.9694538184447583\n",
            "Epoch 9 - Training loss: -0.97188892001028\n",
            "Epoch 10 - Training loss: -0.9724817417069539\n",
            "Epoch 11 - Training loss: -0.97517495648439\n",
            "Epoch 12 - Training loss: -0.9763594889310377\n",
            "Epoch 13 - Training loss: -0.9782026243616523\n",
            "Epoch 14 - Training loss: -0.9787701652375366\n",
            "\n",
            "Training Time (in minutes) = 2.088308274745941\n",
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.98      0.98      0.98     30508\n",
            "        cls1       0.98      0.98      0.98     29492\n",
            "\n",
            "    accuracy                           0.98     60000\n",
            "   macro avg       0.98      0.98      0.98     60000\n",
            "weighted avg       0.98      0.98      0.98     60000\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:root:ensemble 3, seed=70 :\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.98      0.98      0.98      5074\n",
            "        cls1       0.98      0.98      0.98      4926\n",
            "\n",
            "    accuracy                           0.98     10000\n",
            "   macro avg       0.98      0.98      0.98     10000\n",
            "weighted avg       0.98      0.98      0.98     10000\n",
            "\n",
            "Epoch 0 - Training loss: -0.8117752036750953\n",
            "Epoch 1 - Training loss: -0.8871775659035518\n",
            "Epoch 2 - Training loss: -0.9166598073455062\n",
            "Epoch 3 - Training loss: -0.9396271622384281\n",
            "Epoch 4 - Training loss: -0.9525436061913016\n",
            "Epoch 5 - Training loss: -0.9588024899649468\n",
            "Epoch 6 - Training loss: -0.9634891537460946\n",
            "Epoch 7 - Training loss: -0.9663453313079216\n",
            "Epoch 8 - Training loss: -0.9685208358998492\n",
            "Epoch 9 - Training loss: -0.9702789106729951\n",
            "Epoch 10 - Training loss: -0.9724688025425746\n",
            "Epoch 11 - Training loss: -0.9743972650087719\n",
            "Epoch 12 - Training loss: -0.9752878280463757\n",
            "Epoch 13 - Training loss: -0.9770063516427713\n",
            "Epoch 14 - Training loss: -0.9770051767068632\n",
            "\n",
            "Training Time (in minutes) = 2.090570938587189\n",
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.98      0.99      0.98     30508\n",
            "        cls1       0.99      0.98      0.98     29492\n",
            "\n",
            "    accuracy                           0.98     60000\n",
            "   macro avg       0.98      0.98      0.98     60000\n",
            "weighted avg       0.98      0.98      0.98     60000\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:root:ensemble 4, seed=58 :\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.97      0.98      0.98      5074\n",
            "        cls1       0.98      0.97      0.98      4926\n",
            "\n",
            "    accuracy                           0.98     10000\n",
            "   macro avg       0.98      0.98      0.98     10000\n",
            "weighted avg       0.98      0.98      0.98     10000\n",
            "\n",
            "Epoch 0 - Training loss: -0.8171965680968787\n",
            "Epoch 1 - Training loss: -0.888162299196349\n",
            "Epoch 2 - Training loss: -0.9188540111472612\n",
            "Epoch 3 - Training loss: -0.9419950198517171\n",
            "Epoch 4 - Training loss: -0.9546007303667984\n",
            "Epoch 5 - Training loss: -0.9606150375373328\n",
            "Epoch 6 - Training loss: -0.9650471378236946\n",
            "Epoch 7 - Training loss: -0.9684594206210139\n",
            "Epoch 8 - Training loss: -0.9711612693028155\n",
            "Epoch 9 - Training loss: -0.9723775500554774\n",
            "Epoch 10 - Training loss: -0.9746142246448664\n",
            "Epoch 11 - Training loss: -0.9749468747359603\n",
            "Epoch 12 - Training loss: -0.9766332911275851\n",
            "Epoch 13 - Training loss: -0.9780748304146439\n",
            "Epoch 14 - Training loss: -0.9788450144374294\n",
            "\n",
            "Training Time (in minutes) = 2.0870907584826153\n",
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.98      0.98      0.98     30508\n",
            "        cls1       0.98      0.98      0.98     29492\n",
            "\n",
            "    accuracy                           0.98     60000\n",
            "   macro avg       0.98      0.98      0.98     60000\n",
            "weighted avg       0.98      0.98      0.98     60000\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:root:ensemble 5, seed=73 :\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.98      0.98      0.98      5074\n",
            "        cls1       0.98      0.98      0.98      4926\n",
            "\n",
            "    accuracy                           0.98     10000\n",
            "   macro avg       0.98      0.98      0.98     10000\n",
            "weighted avg       0.98      0.98      0.98     10000\n",
            "\n",
            "Epoch 0 - Training loss: -0.8180063467607823\n",
            "Epoch 1 - Training loss: -0.8868038079250596\n",
            "Epoch 2 - Training loss: -0.9188629947999901\n",
            "Epoch 3 - Training loss: -0.9434269017883455\n",
            "Epoch 4 - Training loss: -0.9550023201559144\n",
            "Epoch 5 - Training loss: -0.9610094577391773\n",
            "Epoch 6 - Training loss: -0.9652571962840522\n",
            "Epoch 7 - Training loss: -0.9696778359570737\n",
            "Epoch 8 - Training loss: -0.9717000461717659\n",
            "Epoch 9 - Training loss: -0.9741397981069235\n",
            "Epoch 10 - Training loss: -0.97585970462004\n",
            "Epoch 11 - Training loss: -0.976851071884383\n",
            "Epoch 12 - Training loss: -0.978224474992325\n",
            "Epoch 13 - Training loss: -0.979139377631104\n",
            "Epoch 14 - Training loss: -0.9805944488246812\n",
            "\n",
            "Training Time (in minutes) = 2.072359510262807\n",
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.98      0.99      0.98     30508\n",
            "        cls1       0.99      0.98      0.98     29492\n",
            "\n",
            "    accuracy                           0.98     60000\n",
            "   macro avg       0.98      0.98      0.98     60000\n",
            "weighted avg       0.98      0.98      0.98     60000\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:root:ensemble 6, seed=43 :\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.98      0.98      0.98      5074\n",
            "        cls1       0.98      0.98      0.98      4926\n",
            "\n",
            "    accuracy                           0.98     10000\n",
            "   macro avg       0.98      0.98      0.98     10000\n",
            "weighted avg       0.98      0.98      0.98     10000\n",
            "\n",
            "Epoch 0 - Training loss: -0.8216982615718456\n",
            "Epoch 1 - Training loss: -0.8885205263522133\n",
            "Epoch 2 - Training loss: -0.9198986575928833\n",
            "Epoch 3 - Training loss: -0.9438573534427676\n",
            "Epoch 4 - Training loss: -0.9547698129214712\n",
            "Epoch 5 - Training loss: -0.9616294173416553\n",
            "Epoch 6 - Training loss: -0.9661301233366862\n",
            "Epoch 7 - Training loss: -0.969175829117232\n",
            "Epoch 8 - Training loss: -0.9714718905847464\n",
            "Epoch 9 - Training loss: -0.9732371802523192\n",
            "Epoch 10 - Training loss: -0.9748729627523849\n",
            "Epoch 11 - Training loss: -0.9766441356779924\n",
            "Epoch 12 - Training loss: -0.9779695379200266\n",
            "Epoch 13 - Training loss: -0.9783594548575151\n",
            "Epoch 14 - Training loss: -0.9798257985094717\n",
            "\n",
            "Training Time (in minutes) = 2.081741225719452\n",
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.98      0.99      0.98     30508\n",
            "        cls1       0.99      0.98      0.98     29492\n",
            "\n",
            "    accuracy                           0.98     60000\n",
            "   macro avg       0.98      0.98      0.98     60000\n",
            "weighted avg       0.98      0.98      0.98     60000\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:root:ensemble 7, seed=32 :\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.98      0.98      0.98      5074\n",
            "        cls1       0.98      0.98      0.98      4926\n",
            "\n",
            "    accuracy                           0.98     10000\n",
            "   macro avg       0.98      0.98      0.98     10000\n",
            "weighted avg       0.98      0.98      0.98     10000\n",
            "\n",
            "Epoch 0 - Training loss: -0.8142846571102834\n",
            "Epoch 1 - Training loss: -0.8814949985506184\n",
            "Epoch 2 - Training loss: -0.9054496533580935\n",
            "Epoch 3 - Training loss: -0.9334075919219426\n",
            "Epoch 4 - Training loss: -0.9494493631030451\n",
            "Epoch 5 - Training loss: -0.9584889318516006\n",
            "Epoch 6 - Training loss: -0.9633073068376797\n",
            "Epoch 7 - Training loss: -0.9668304797555847\n",
            "Epoch 8 - Training loss: -0.9700068819370351\n",
            "Epoch 9 - Training loss: -0.9719199610036066\n",
            "Epoch 10 - Training loss: -0.9742566320433546\n",
            "Epoch 11 - Training loss: -0.9756570671921345\n",
            "Epoch 12 - Training loss: -0.976974289745156\n",
            "Epoch 13 - Training loss: -0.9778569314017225\n",
            "Epoch 14 - Training loss: -0.9790865160000604\n",
            "\n",
            "Training Time (in minutes) = 2.0883962472279864\n",
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.98      0.98      0.98     30508\n",
            "        cls1       0.98      0.98      0.98     29492\n",
            "\n",
            "    accuracy                           0.98     60000\n",
            "   macro avg       0.98      0.98      0.98     60000\n",
            "weighted avg       0.98      0.98      0.98     60000\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:root:ensemble 8, seed=65 :\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.98      0.98      0.98      5074\n",
            "        cls1       0.98      0.98      0.98      4926\n",
            "\n",
            "    accuracy                           0.98     10000\n",
            "   macro avg       0.98      0.98      0.98     10000\n",
            "weighted avg       0.98      0.98      0.98     10000\n",
            "\n",
            "Epoch 0 - Training loss: -0.8134198209433667\n",
            "Epoch 1 - Training loss: -0.8870780492133932\n",
            "Epoch 2 - Training loss: -0.9164371625192638\n",
            "Epoch 3 - Training loss: -0.9403774158786863\n",
            "Epoch 4 - Training loss: -0.9525832316514525\n",
            "Epoch 5 - Training loss: -0.9595877030637981\n",
            "Epoch 6 - Training loss: -0.9640273342508752\n",
            "Epoch 7 - Training loss: -0.9676635185919845\n",
            "Epoch 8 - Training loss: -0.9707284092521871\n",
            "Epoch 9 - Training loss: -0.973141842623001\n",
            "Epoch 10 - Training loss: -0.974128914031901\n",
            "Epoch 11 - Training loss: -0.9761451166957172\n",
            "Epoch 12 - Training loss: -0.977706017206981\n",
            "Epoch 13 - Training loss: -0.9785503787018343\n",
            "Epoch 14 - Training loss: -0.9793854721192358\n",
            "\n",
            "Training Time (in minutes) = 2.0901648879051207\n",
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.99      0.98      0.98     30508\n",
            "        cls1       0.98      0.99      0.98     29492\n",
            "\n",
            "    accuracy                           0.98     60000\n",
            "   macro avg       0.98      0.98      0.98     60000\n",
            "weighted avg       0.98      0.98      0.98     60000\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:root:ensemble 9, seed=49 :\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.99      0.97      0.98      5074\n",
            "        cls1       0.97      0.99      0.98      4926\n",
            "\n",
            "    accuracy                           0.98     10000\n",
            "   macro avg       0.98      0.98      0.98     10000\n",
            "weighted avg       0.98      0.98      0.98     10000\n",
            "\n",
            "Epoch 0 - Training loss: -0.8163834288875177\n",
            "Epoch 1 - Training loss: -0.8858002628217628\n",
            "Epoch 2 - Training loss: -0.9155481884728617\n",
            "Epoch 3 - Training loss: -0.9399554772036416\n",
            "Epoch 4 - Training loss: -0.9525233998354564\n",
            "Epoch 5 - Training loss: -0.9587709638736904\n",
            "Epoch 6 - Training loss: -0.9636608763798468\n",
            "Epoch 7 - Training loss: -0.9666691388148488\n",
            "Epoch 8 - Training loss: -0.9694873797359751\n",
            "Epoch 9 - Training loss: -0.9715578941775284\n",
            "Epoch 10 - Training loss: -0.9732696046707219\n",
            "Epoch 11 - Training loss: -0.9747171492210583\n",
            "Epoch 12 - Training loss: -0.9761110575341466\n",
            "Epoch 13 - Training loss: -0.977518026635591\n",
            "Epoch 14 - Training loss: -0.9776245208183078\n",
            "\n",
            "Training Time (in minutes) = 2.0721216519673664\n",
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.98      0.98      0.98     30508\n",
            "        cls1       0.98      0.98      0.98     29492\n",
            "\n",
            "    accuracy                           0.98     60000\n",
            "   macro avg       0.98      0.98      0.98     60000\n",
            "weighted avg       0.98      0.98      0.98     60000\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:root:calculating uncertainty-related features...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Classification Report\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        cls0       0.98      0.98      0.98      5074\n",
            "        cls1       0.98      0.98      0.98      4926\n",
            "\n",
            "    accuracy                           0.98     10000\n",
            "   macro avg       0.98      0.98      0.98     10000\n",
            "weighted avg       0.98      0.98      0.98     10000\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:root:calculating uncertainty-related features...\n",
            "INFO:root:num classes : 2,  LDU input size : 22\n",
            "INFO:root:LDU : alpha=1.9\n",
            "INFO:root:epoch: 0, loss = 2.6423375024470186\n",
            "INFO:root:epoch: 1, loss = 2.617450522715603\n",
            "INFO:root:epoch: 2, loss = 2.61717008019307\n",
            "INFO:root:epoch: 3, loss = 2.616944950018356\n",
            "INFO:root:epoch: 4, loss = 2.6167791423512927\n",
            "INFO:root:deferred ratio = 0.0063\n",
            "INFO:root:Automated F1=0.9838985609338835, recall=0.9838985609338835, precision=0.9838985609338835\n",
            "INFO:root:LDU : alpha=2.1\n",
            "INFO:root:epoch: 0, loss = 2.767903873915357\n",
            "INFO:root:epoch: 1, loss = 2.738152930223103\n",
            "INFO:root:epoch: 2, loss = 2.737998733134158\n",
            "INFO:root:epoch: 3, loss = 2.737822722778646\n",
            "INFO:root:epoch: 4, loss = 2.7376617823582468\n",
            "INFO:root:deferred ratio = 0.001\n",
            "INFO:root:Automated F1=0.9813813813813814, recall=0.9813813813813814, precision=0.9813813813813814\n",
            "INFO:root:LDU : alpha=2.3000000000000003\n",
            "INFO:root:epoch: 0, loss = 2.887781199870079\n",
            "INFO:root:epoch: 1, loss = 2.853531620649895\n",
            "INFO:root:epoch: 2, loss = 2.852968829018729\n",
            "INFO:root:epoch: 3, loss = 2.8525480479319723\n",
            "INFO:root:epoch: 4, loss = 2.852381798504258\n",
            "INFO:root:deferred ratio = 0.0001\n",
            "INFO:root:Automated F1=0.980998099809981, recall=0.980998099809981, precision=0.980998099809981\n",
            "INFO:root:LDU : alpha=2.5000000000000004\n",
            "INFO:root:epoch: 0, loss = 3.0060775089365586\n",
            "INFO:root:epoch: 1, loss = 2.966911502738497\n",
            "INFO:root:epoch: 2, loss = 2.9666582953446965\n",
            "INFO:root:epoch: 3, loss = 2.966385770708259\n",
            "INFO:root:epoch: 4, loss = 2.966258156528351\n",
            "INFO:root:deferred ratio = 0.0\n",
            "INFO:root:Automated F1=0.9811, recall=0.9811, precision=0.9811\n",
            "INFO:root:LDU : alpha=2.7000000000000006\n",
            "INFO:root:epoch: 0, loss = 3.1243096490912854\n",
            "INFO:root:epoch: 1, loss = 3.080206476294918\n",
            "INFO:root:epoch: 2, loss = 3.079924025515249\n",
            "INFO:root:epoch: 3, loss = 3.0796334161433077\n",
            "INFO:root:epoch: 4, loss = 3.0795002852929936\n",
            "INFO:root:deferred ratio = 0.0\n",
            "INFO:root:Automated F1=0.9810000000000001, recall=0.981, precision=0.981\n",
            "INFO:root:LDU : alpha=2.900000000000001\n",
            "INFO:root:epoch: 0, loss = 3.2425132857711074\n",
            "INFO:root:epoch: 1, loss = 3.19349858247395\n",
            "INFO:root:epoch: 2, loss = 3.1931863918995806\n",
            "INFO:root:epoch: 3, loss = 3.1928782061473138\n",
            "INFO:root:epoch: 4, loss = 3.192737319830384\n",
            "INFO:root:deferred ratio = 0.0\n",
            "INFO:root:Automated F1=0.9809, recall=0.9809, precision=0.9809\n",
            "INFO:root:LDU : alpha=3.100000000000001\n",
            "INFO:root:epoch: 0, loss = 3.360702715194556\n",
            "INFO:root:epoch: 1, loss = 3.3067883413229415\n",
            "INFO:root:epoch: 2, loss = 3.306446624971402\n",
            "INFO:root:epoch: 3, loss = 3.3061206117113517\n",
            "INFO:root:epoch: 4, loss = 3.305972448798385\n",
            "INFO:root:deferred ratio = 0.0\n",
            "INFO:root:Automated F1=0.9808, recall=0.9808, precision=0.9808\n",
            "INFO:root:LDU : alpha=3.300000000000001\n",
            "INFO:root:epoch: 0, loss = 3.478855197109393\n",
            "INFO:root:epoch: 1, loss = 3.4200760954732834\n",
            "INFO:root:epoch: 2, loss = 3.4197052666373344\n",
            "INFO:root:epoch: 3, loss = 3.419361721478037\n",
            "INFO:root:epoch: 4, loss = 3.41920627078522\n",
            "INFO:root:deferred ratio = 0.0\n",
            "INFO:root:Automated F1=0.9808, recall=0.9808, precision=0.9808\n",
            "INFO:root:LDU : alpha=3.5000000000000013\n",
            "INFO:root:epoch: 0, loss = 3.596997777790403\n",
            "INFO:root:epoch: 1, loss = 3.5333645747922886\n",
            "INFO:root:epoch: 2, loss = 3.5329643091413256\n",
            "INFO:root:epoch: 3, loss = 3.532603025690579\n",
            "INFO:root:epoch: 4, loss = 3.5324398947677125\n",
            "INFO:root:deferred ratio = 0.0\n",
            "INFO:root:Automated F1=0.9808, recall=0.9808, precision=0.9808\n",
            "INFO:root:LDU : alpha=3.7000000000000015\n",
            "INFO:root:epoch: 0, loss = 3.715115878373575\n",
            "INFO:root:epoch: 1, loss = 3.6466528535651754\n",
            "INFO:root:epoch: 2, loss = 3.6462231866840615\n",
            "INFO:root:epoch: 3, loss = 3.6458440932637846\n",
            "INFO:root:epoch: 4, loss = 3.6456731852692075\n",
            "INFO:root:deferred ratio = 0.0\n",
            "INFO:root:Automated F1=0.9808, recall=0.9808, precision=0.9808\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----+---------+------------------+-----------------+-------------+--------------+------------------+---------------------+-------------------+-----------------------+--------------------------+\n",
            "|    |   Alpha |   Deferred_Ratio |   Deferred_Size |   Auto_Size |   Overall_F1 |   Overall_Recall |   Overall_Precision |   Auto_F1 (micro) |   Auto_Recall (micro) |   Auto_Precision (micro) |\n",
            "|----+---------+------------------+-----------------+-------------+--------------+------------------+---------------------+-------------------+-----------------------+--------------------------|\n",
            "|  0 |     1.9 |           0.0063 |              63 |        9937 |       0.984  |           0.984  |              0.984  |          0.983899 |              0.983899 |                 0.983899 |\n",
            "|  1 |     2.1 |           0.001  |              10 |        9990 |       0.9814 |           0.9814 |              0.9814 |          0.981381 |              0.981381 |                 0.981381 |\n",
            "|  2 |     2.3 |           0.0001 |               1 |        9999 |       0.981  |           0.981  |              0.981  |          0.980998 |              0.980998 |                 0.980998 |\n",
            "|  3 |     2.5 |           0      |               0 |       10000 |       0.9811 |           0.9811 |              0.9811 |          0.9811   |              0.9811   |                 0.9811   |\n",
            "|  4 |     2.7 |           0      |               0 |       10000 |       0.981  |           0.981  |              0.981  |          0.981    |              0.981    |                 0.981    |\n",
            "|  5 |     2.9 |           0      |               0 |       10000 |       0.9809 |           0.9809 |              0.9809 |          0.9809   |              0.9809   |                 0.9809   |\n",
            "|  6 |     3.1 |           0      |               0 |       10000 |       0.9808 |           0.9808 |              0.9808 |          0.9808   |              0.9808   |                 0.9808   |\n",
            "|  7 |     3.3 |           0      |               0 |       10000 |       0.9808 |           0.9808 |              0.9808 |          0.9808   |              0.9808   |                 0.9808   |\n",
            "|  8 |     3.5 |           0      |               0 |       10000 |       0.9808 |           0.9808 |              0.9808 |          0.9808   |              0.9808   |                 0.9808   |\n",
            "|  9 |     3.7 |           0      |               0 |       10000 |       0.9808 |           0.9808 |              0.9808 |          0.9808   |              0.9808   |                 0.9808   |\n",
            "+----+---------+------------------+-----------------+-------------+--------------+------------------+---------------------+-------------------+-----------------------+--------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dn-FSes2m74V"
      },
      "source": [
        "#### Adjusting the range of alpha:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qpz8lUXcm6Rz",
        "outputId": "c1681c45-6878-4f2a-af3e-40c0dd6245ed"
      },
      "source": [
        "s2 = learning_ldu_class(alpha_list=np.arange(1.1, 2.3, 0.1).tolist(), device='cpu', learning_rate=0.0008, epochs=5)\n",
        "\n",
        "if not os.path.exists(s1.path_ensemble): # from training base model\n",
        "    ldu_scores = s2.tune_ldu(lambda: s1.gether_ensembles(base_model_class, trainer, validator))\n",
        "else: # from ensemble data\n",
        "    ldu_scores = s2.tune_ldu(lambda: s1.load_ensemble())\n",
        "\n",
        "if not os.path.exists('/content/drive/MyDrive/ldu_result'):\n",
        "    os.makedirs('/content/drive/MyDrive/ldu_result')\n",
        "ldu_scores.to_csv(os.path.join('/content/drive/MyDrive/ldu_result', 'ldu_scores.csv'), index=False)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:root:num classes : 2,  LDU input size : 22\n",
            "INFO:root:LDU : alpha=1.1\n",
            "INFO:root:epoch: 0, loss = 2.0280751180547134\n",
            "INFO:root:epoch: 1, loss = 2.0175374959831807\n",
            "INFO:root:epoch: 2, loss = 2.017370380318241\n",
            "INFO:root:epoch: 3, loss = 2.0172639132054377\n",
            "INFO:root:epoch: 4, loss = 2.017209384995483\n",
            "INFO:root:deferred ratio = 0.0423\n",
            "INFO:root:Automated F1=0.9936305732484076, recall=0.9936305732484076, precision=0.9936305732484076\n",
            "INFO:root:LDU : alpha=1.2000000000000002\n",
            "INFO:root:epoch: 0, loss = 2.1183067049298967\n",
            "INFO:root:epoch: 1, loss = 2.1063731141181896\n",
            "INFO:root:epoch: 2, loss = 2.106152995563011\n",
            "INFO:root:epoch: 3, loss = 2.106043584311186\n",
            "INFO:root:epoch: 4, loss = 2.1059726758806434\n",
            "INFO:root:deferred ratio = 0.0273\n",
            "INFO:root:Automated F1=0.9908502107535725, recall=0.9908502107535725, precision=0.9908502107535725\n",
            "INFO:root:LDU : alpha=1.3000000000000003\n",
            "INFO:root:epoch: 0, loss = 2.203817854303795\n",
            "INFO:root:epoch: 1, loss = 2.190314061606108\n",
            "INFO:root:epoch: 2, loss = 2.1900446318360025\n",
            "INFO:root:epoch: 3, loss = 2.1899309374376146\n",
            "INFO:root:epoch: 4, loss = 2.189847587776591\n",
            "INFO:root:deferred ratio = 0.0213\n",
            "INFO:root:Automated F1=0.9895780116481047, recall=0.9895780116481047, precision=0.9895780116481047\n",
            "INFO:root:LDU : alpha=1.4000000000000004\n",
            "INFO:root:epoch: 0, loss = 2.2850345589204637\n",
            "INFO:root:epoch: 1, loss = 2.2699400328878148\n",
            "INFO:root:epoch: 2, loss = 2.269629716873169\n",
            "INFO:root:epoch: 3, loss = 2.2694908108538403\n",
            "INFO:root:epoch: 4, loss = 2.2694054620860737\n",
            "INFO:root:deferred ratio = 0.0168\n",
            "INFO:root:Automated F1=0.9877949552481693, recall=0.9877949552481693, precision=0.9877949552481693\n",
            "INFO:root:LDU : alpha=1.5000000000000004\n",
            "INFO:root:epoch: 0, loss = 2.362595340082132\n",
            "INFO:root:epoch: 1, loss = 2.345725673348156\n",
            "INFO:root:epoch: 2, loss = 2.3453814218293374\n",
            "INFO:root:epoch: 3, loss = 2.345210873750227\n",
            "INFO:root:epoch: 4, loss = 2.345126857127208\n",
            "INFO:root:deferred ratio = 0.0134\n",
            "INFO:root:Automated F1=0.9867220758159335, recall=0.9867220758159335, precision=0.9867220758159335\n",
            "INFO:root:LDU : alpha=1.6000000000000005\n",
            "INFO:root:epoch: 0, loss = 2.436861701611517\n",
            "INFO:root:epoch: 1, loss = 2.418050403025613\n",
            "INFO:root:epoch: 2, loss = 2.4176892883487855\n",
            "INFO:root:epoch: 3, loss = 2.417495128188306\n",
            "INFO:root:epoch: 4, loss = 2.4174028716361855\n",
            "INFO:root:deferred ratio = 0.0117\n",
            "INFO:root:Automated F1=0.9860366285540828, recall=0.9860366285540828, precision=0.9860366285540828\n",
            "INFO:root:LDU : alpha=1.7000000000000006\n",
            "INFO:root:epoch: 0, loss = 2.507960136765356\n",
            "INFO:root:epoch: 1, loss = 2.48727121459904\n",
            "INFO:root:epoch: 2, loss = 2.4868976937682388\n",
            "INFO:root:epoch: 3, loss = 2.4867044008616954\n",
            "INFO:root:epoch: 4, loss = 2.486576563768041\n",
            "INFO:root:deferred ratio = 0.0094\n",
            "INFO:root:Automated F1=0.9847567131031698, recall=0.9847567131031698, precision=0.9847567131031698\n",
            "INFO:root:LDU : alpha=1.8000000000000007\n",
            "INFO:root:epoch: 0, loss = 2.5764019400326172\n",
            "INFO:root:epoch: 1, loss = 2.553660855110266\n",
            "INFO:root:epoch: 2, loss = 2.553320114546493\n",
            "INFO:root:epoch: 3, loss = 2.553124194460383\n",
            "INFO:root:epoch: 4, loss = 2.552963161773519\n",
            "INFO:root:deferred ratio = 0.0075\n",
            "INFO:root:Automated F1=0.9842821158690176, recall=0.9842821158690176, precision=0.9842821158690176\n",
            "INFO:root:LDU : alpha=1.9000000000000008\n",
            "INFO:root:epoch: 0, loss = 2.6423375024470186\n",
            "INFO:root:epoch: 1, loss = 2.617450522715603\n",
            "INFO:root:epoch: 2, loss = 2.61717008019307\n",
            "INFO:root:epoch: 3, loss = 2.616944950018356\n",
            "INFO:root:epoch: 4, loss = 2.6167791423512927\n",
            "INFO:root:deferred ratio = 0.0063\n",
            "INFO:root:Automated F1=0.9838985609338835, recall=0.9838985609338835, precision=0.9838985609338835\n",
            "INFO:root:LDU : alpha=2.000000000000001\n",
            "INFO:root:epoch: 0, loss = 2.7060684341865815\n",
            "INFO:root:epoch: 1, loss = 2.6788589745950597\n",
            "INFO:root:epoch: 2, loss = 2.6786537907525165\n",
            "INFO:root:epoch: 3, loss = 2.6784470188083933\n",
            "INFO:root:epoch: 4, loss = 2.6782559746109853\n",
            "INFO:root:deferred ratio = 0.0041\n",
            "INFO:root:Automated F1=0.9830304247414399, recall=0.9830304247414399, precision=0.9830304247414399\n",
            "INFO:root:LDU : alpha=2.100000000000001\n",
            "INFO:root:epoch: 0, loss = 2.767903873915357\n",
            "INFO:root:epoch: 1, loss = 2.738152930223103\n",
            "INFO:root:epoch: 2, loss = 2.737998733134158\n",
            "INFO:root:epoch: 3, loss = 2.737822722778646\n",
            "INFO:root:epoch: 4, loss = 2.7376617823582468\n",
            "INFO:root:deferred ratio = 0.001\n",
            "INFO:root:Automated F1=0.9813813813813814, recall=0.9813813813813814, precision=0.9813813813813814\n",
            "INFO:root:LDU : alpha=2.200000000000001\n",
            "INFO:root:epoch: 0, loss = 2.828406440169572\n",
            "INFO:root:epoch: 1, loss = 2.7960475451910676\n",
            "INFO:root:epoch: 2, loss = 2.795712598097096\n",
            "INFO:root:epoch: 3, loss = 2.795481946676779\n",
            "INFO:root:epoch: 4, loss = 2.795355450878265\n",
            "INFO:root:deferred ratio = 0.0009\n",
            "INFO:root:Automated F1=0.9811830647582824, recall=0.9811830647582824, precision=0.9811830647582824\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----+---------+------------------+-----------------+-------------+--------------+------------------+---------------------+-------------------+-----------------------+--------------------------+\n",
            "|    |   Alpha |   Deferred_Ratio |   Deferred_Size |   Auto_Size |   Overall_F1 |   Overall_Recall |   Overall_Precision |   Auto_F1 (micro) |   Auto_Recall (micro) |   Auto_Precision (micro) |\n",
            "|----+---------+------------------+-----------------+-------------+--------------+------------------+---------------------+-------------------+-----------------------+--------------------------|\n",
            "|  0 |     1.1 |           0.0423 |             423 |        9577 |       0.9939 |           0.9939 |              0.9939 |          0.993631 |              0.993631 |                 0.993631 |\n",
            "|  1 |     1.2 |           0.0273 |             273 |        9727 |       0.9911 |           0.9911 |              0.9911 |          0.99085  |              0.99085  |                 0.99085  |\n",
            "|  2 |     1.3 |           0.0213 |             213 |        9787 |       0.9898 |           0.9898 |              0.9898 |          0.989578 |              0.989578 |                 0.989578 |\n",
            "|  3 |     1.4 |           0.0168 |             168 |        9832 |       0.988  |           0.988  |              0.988  |          0.987795 |              0.987795 |                 0.987795 |\n",
            "|  4 |     1.5 |           0.0134 |             134 |        9866 |       0.9869 |           0.9869 |              0.9869 |          0.986722 |              0.986722 |                 0.986722 |\n",
            "|  5 |     1.6 |           0.0117 |             117 |        9883 |       0.9862 |           0.9862 |              0.9862 |          0.986037 |              0.986037 |                 0.986037 |\n",
            "|  6 |     1.7 |           0.0094 |              94 |        9906 |       0.9849 |           0.9849 |              0.9849 |          0.984757 |              0.984757 |                 0.984757 |\n",
            "|  7 |     1.8 |           0.0075 |              75 |        9925 |       0.9844 |           0.9844 |              0.9844 |          0.984282 |              0.984282 |                 0.984282 |\n",
            "|  8 |     1.9 |           0.0063 |              63 |        9937 |       0.984  |           0.984  |              0.984  |          0.983899 |              0.983899 |                 0.983899 |\n",
            "|  9 |     2   |           0.0041 |              41 |        9959 |       0.9831 |           0.9831 |              0.9831 |          0.98303  |              0.98303  |                 0.98303  |\n",
            "| 10 |     2.1 |           0.001  |              10 |        9990 |       0.9814 |           0.9814 |              0.9814 |          0.981381 |              0.981381 |                 0.981381 |\n",
            "| 11 |     2.2 |           0.0009 |               9 |        9991 |       0.9812 |           0.9812 |              0.9812 |          0.981183 |              0.981183 |                 0.981183 |\n",
            "+----+---------+------------------+-----------------+-------------+--------------+------------------+---------------------+-------------------+-----------------------+--------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DE4-fSW0_iGj"
      },
      "source": [
        "#### Display the trend of F1 scores and defer rates against alpha value (weight of defer loss) during training: "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 304
        },
        "id": "8_zvW-RhASy3",
        "outputId": "ce51b427-bc4a-44bd-ab41-d35503fd99a5"
      },
      "source": [
        "from mpl_toolkits.axes_grid1 import host_subplot\n",
        "import mpl_toolkits.axisartist as AA\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "def plot_alpha_acc(x, y1, y2, title):\n",
        "    fig = plt.figure()\n",
        "    fig.suptitle(title, fontsize=12)\n",
        "    host = host_subplot(111, axes_class=AA.Axes)\n",
        "    plt.subplots_adjust(right=0.75)\n",
        "\n",
        "    par1 = host.twinx()\n",
        "\n",
        "    offset = 0\n",
        "    new_fixed_axis = par1.get_grid_helper().new_fixed_axis\n",
        "    par1.axis[\"right\"] = new_fixed_axis(loc=\"right\", axes=par1,\n",
        "                                            offset=(offset, 0))\n",
        "\n",
        "    par1.axis[\"right\"].toggle(all=True)\n",
        "\n",
        "\n",
        "    host.set_xlim(min(x), max(x))\n",
        "    host.set_ylim(0.9, 1.0)\n",
        "\n",
        "    host.set_xlabel(\"Alpha\")\n",
        "    host.set_ylabel(\"F1 score\")\n",
        "    par1.set_ylabel(\"Defer Rate\")\n",
        "\n",
        "    p1, = host.plot(x, y1, label=\"automated F1 score\")\n",
        "    p2, = par1.plot(x, y2, label=\"defer rate\")\n",
        "    # p2, = host.plot(x, y2, label=\"Overall F1 score\")\n",
        "    \n",
        "    par1.set_ylim(0, 0.05)\n",
        "\n",
        "    host.legend(loc=\"lower left\")\n",
        "\n",
        "    host.axis[\"left\"].label.set_color(p1.get_color())\n",
        "    par1.axis[\"right\"].label.set_color(p2.get_color())\n",
        "\n",
        "    plt.draw()\n",
        "    plt.show()\n",
        "\n",
        "    #plt.savefig(\"Test\")\n",
        "\n",
        "'Alpha', 'Deferred_Ratio', 'Deferred_Size', 'Auto_Size', \\\n",
        "                                         'Overall_F1', 'Overall_Recall', 'Overall_Precision', \\\n",
        "                                         'Auto_F1 (micro)', 'Auto_Recall (micro)', 'Auto_Precision (micro)'\n",
        "x = ldu_scores['Alpha'].tolist()\n",
        "y1 = ldu_scores['Auto_F1 (micro)'].tolist()\n",
        "y2 = ldu_scores['Deferred_Ratio'].tolist()\n",
        "# y2 = ldu_scores['Overall_F1'].tolist()\n",
        "\n",
        "\n",
        "\n",
        "plot_alpha_acc(x, y1, y2, 'F1 scores and defer rates vs. alpha')\n"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW0AAAEfCAYAAACK65foAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXhV1dX48e/KnJABCGCZJIATc5hBZRCrYrEiOCBOoL6iFVt/bdViW/VqteKrfds6tErrhFqroFgVB5SKQxUEFAdAkSHKZAkEEhIyZ/3+2CfhJoTkJuTm5t6sz/PcJ+eevc8569wkKyf77LO3qCrGGGPCQ1SoAzDGGBM4S9rGGBNGLGkbY0wYsaRtjDFhxJK2McaEEUvaxhgTRixpm4gnIstE5H8CrDteRLYFWFdE5HER2SsiHx9ZlEZEfCLydFPXjTSWtMOAiGSJSKGI5Pu9unhl80TkaxGpEJGZIQ61tTkZOA3opqojQhFAa05erZUl7fDxY1VN9nvt8NZ/BlwLfBLC2AAQkZhQx9DMegBZqlrQ0A0D+axa4edpAmBJO8yp6kOquhQoqq+uiPxIRNaJyH4R2S4iN/iVTRaRNSKSJyKbRGSit76LiLwsIjkislFErvLbxiciC0XkaRHJA2aKSJqIPCoiO71j3Cki0V79Y0TkXRHJFZHdIvJcHbEuEJHvvbrviUg/v7InROQhEVnsncsKEentV36aiHzlbfsgIHUcJ9Hb314RWQcMr1HeRUReEJFsEdkiIj/z1l8J/B0Y7f3nc7u3/izvc9wnIh+KyEC/fWWJyK9E5HOgoLakLCIqIrNF5BvgG2/dn0Vkq/e9WS0iY7z1E4FfA9O8GD7z1h/x90BEXheR62qs+0xEpnrNQn8UkV1eTF+ISP/DfcY19lHrudRSL8P7LGaJyA7vXG6oUS1OROZ7PwNrRWSY3/ZzvJ/j/d7P/JRA4gsLqmqvFv4CsoAf1lPnA2BmPXV2AmO85XbAEG95BJCL+1c/CugKnOCVvQf8BUgAMoFsYIJX5gNKgXO87RKBRcAjQBugE/AxcLVX/1ngN17dBODkOmK9AkgB4oE/AWv8yp4A9nhxxwDPAP/0yjoA+4HzgFjg50AZ8D+HOc5c4H2gPdAd+BLY5pVFAauBW4E4oBewGTjDK58JfOC3r8HALmAkEA3M8L538X7fxzXecRIPE48Cb3nxJHrrLgHSvXP9JfA9kOD3PXi6xj6O+HsAXAb8x+99X2Cf9/04w/tc2uL+IPYBOgf4sxzQuQAZ3mfxrHceA3A/ez/0q1sE/Mj7rO8Glvsd53ygi3ee04CCQGNs6a+QB2CvAL5J7pc93/ul2Qe8VEudQJL2d8DVQGqN9Y8Af6ylfnegHEjxW3c38IS37APe8ys7Cij2T0jAdOAdb3k+MA/XBtyQ82/r/QKnee+fAP7uV/4j4Ctv+bIav7wCbOPwSXszMNHv/SwOJu2RwHc16t8MPO4tz6R60v4r8Lsa9b8Gxvl9H6+o51wV749iHXX2AoP8vgdP+5U1yfcA9wezAOjhvb8LeMxbngBsAEYBUUf4s13ruXAwaZ/gV/d/gUf96r7tV9YXKKzjOGuAyUcSa0t5WfNI+DhHVdt6r3MauY9zcQnuW+9f5NHe+u7AplrqdwFyVHW/37pvcVfilbb6LffAXd3u9JoH9uH+IHTyym/CJdGPvX9nr6gtSBGJFpG53r+3ebhkB+4qutL3fssHgGS/mKtiUvcb6x9jbefoX/5tjfPpUnku3vn8GpcYa9MD+GWN+t29Y1SqK5Za64jIDSKy3mvS2AekUf2zqBnDEX8PvO/5YuBCb9V03H80qOq/gQeBh4Bd4m6GpwZwXg09Fzj0e+P/Wdb8GUiobHISkcv8mqn2Af3rOU7YsBsdrYiqrgQmi0gscB3wPC6pbAV617LJDqC9iKT4Je6jge3+u/Vb3oq7yuugqmW1HP974CoAETkZeFtE3lPVjTWqXgRMBn6IS9hpuCuyw7ZN+9npnRPeccT/fR3113rvj65xPltU9dgAjltZ/y5VvauOOoEMq1lVx2vzvQk4FVirqhUi4v9Z1NxfU30PwDVN3CYi7+GaUt7x28/9wP0i0gn3c3QjcEtdJxXAudSmO/CVt3w07meyTiLSA/ibd5yPVLVcRNbUc5ywYVfaYU5E4kQkAfcDGSsiCSJyyPfVq3exiKSpaimQB1R4xY8Cl4vIqSISJSJdReQEVd0KfAjc7e13IHAlUGsXM1XdCSwB/iAiqd6+eovIOC+G80Wkm1d9Ly7hVNSyqxRc4tkDJAG/b8BHshjo590wiwF+BvygjvrPAzeLSDsvtp/6lX0M7Bd38zDR+w+gv4gMr31X/A24RkRGejfr2ojIJBFJaUD8NaXg2uSzgRgRuRXwv6r9L5BR+T1vwu8BwGu4K/c7gOdUtcLbx3DvHGNxTShFdeyjIedSm1tEJEncjejLgcPevPbTBnde2V68l+OutCOCJe3wtwQoBE7EtVUWAmMPU/dSIMtrcrgGuBhAVT/G/UL8EXdD8l3cLyu4f4szcFc4i4DbVPXtOuK5DHfTbh0uKSwEOntlw4EVIpIPvAxcr6qba9nHfNy/wtu9/Syv43jVqOpu3E2oubikfyzwnzo2ud071hbcZ/mU377KgbNwN2C3ALtxPUbSDnPsVbir2Adx574R1+59JN4E3sC1IX+LS5D+TQYLvK97RKSy22dTfA9Q1WLgRdx/PP/wK0rF/YHa68W0B7gXQER+LSKvN/JcavMu7nNcCtynqkvqqY+qrgP+AHyE+6M2gLp/BsKKeI30xhjTYohIBu4PZWxtzTytmV1pG2NMGLGkbYwxYcSaR4wxJozYlbYxxoQRS9rGGBNGLGkbY0wYsaRtjDFhxJK2McaEEUvaxhgTRixpG2NMGLGkbYwxYcSStjHGhBFL2sYYE0aCmrRF5DFxk39+eZhyEZH7xU0Y+7mIDPErmyEi33ivGcGM0xhjRGSiiHzt5aM5tZTHi8hzXvkKbyTCykmIC72ZctaIyMPBjDPYV9pPABPrKD8TN97xsbi5+f4KICLtgdtwc/SNwM2e0S6okRpjWi1xs9U/hMtJfYHpItK3RrUrgb2qegxu7Pl7/Mo2qWqm97ommLEGNWmr6ntATh1VJgPz1VkOtBWRzrjZnt9S1RxV3Yubnbqu5G+MMUdiBLBRVTeragnwT1x+8jcZeNJbXgic6k1n16xC3abdleozV2zz1h1uvTHGBEMgOaeqjjcxQy6Q7pX1FJFPxU2YPSaYgdrEvsYYc2R2Aker6h4RGQq8JCL9VDUvGAcLddLeTvWZsrt567YD42usX1bbDpKTk9V/TPAOHTrQsWPHJgswOzu7SfcXKpFwHnYOLUO4nkN2dja7d+8G4MCBA8WqmuBXfLhcRC11tnmTRqcBe7wEVAygqqtFZBNwHLAqKCeiqkF94SaF/fIwZZOA13EziY8CPvbWt8fND9fOe20B2te2j6FDh2owBXv/zSUSzsPOoWWIhHMACrR6LooBNgM9cZMifwb0q1FnNvCwt3wh8Ly33BGI9pZ74ZJ7rfmqKV5BvdIWkWdxV8wdRGQbrkdILICqPgy8BvwIN9vyAdyM4Khqjoj8Dljp7eoOVa3rhqYxxjSaqpaJyHW4GeOjgcdUda2I3AGsUtWXgUeBp0RkI66DxYXe5mOBO0SkFKgArglmvgpq0lbV6fWUK+6vV21ljwGPBSMuY4ypSVVfw11I+q+71W+5CDi/lu1eAF4IeoCeUPceafFmzZoV6hCaRCSch51DyxAJ5wBkhzqAxgr7iX2HDRumq1YFp73fGBOZRGS1qg4LdRyNYVfaxhgTRixpG2NMGLGkbYwxYcSStjHGhBFL2sYYE0YsaRtjTBixpG2MMWEk7JP2up15zJq/ikc/2MKX23MprwjvfufGGFOXUI/yd8RSE2JZ/30eS9b9F4CUhBhGZLRnZK/2jOyZTr8uqcREh/3fJmOMASIgaXdrl8j7N01gx75CVmzZw4rNOazYksPSr3YBkBwfw9Ae7aqS+MBuacRaEjfGhKmIfYz9v3lFrNiSw4rNe1ixJYeNu/IBSIyNdkm8Z3tG9kpnUPc04mOimztsY0wIhfNj7BGbtGvanV/Mx35J/Kvv9wMQHxPFkKMPXokPProtCbGWxI2JZJa0Q6ixA0btLSjh46wcrzllD+t25qEKcdFRZHZvW5XEh/RoS1Jc2LciGWP8WNIOoaYa5S/3QCkrs1wCX7Elhy+351KhECXQKSWBzm0T6JKWSOe0BDq39b6mJdClbSIdk+OJimr2SZmNMY0UzknbLiE9aUmx/LDvUfyw71EA7C8qZdW3e/n0u33s2FfIztxC1u/MY+lX/6WotKLatjFRwlGpCXRpm0DntIMJvXPbRJfo2yaQ3iYOEUvsxpgjY0n7MFISYjnl+E6ccnynautVlX0HStmRW8jOfUXszCti575CduYWsWNfIWu27uONL4soKa+e2OOio/iB39X5waSeQPf2SfRIT7IbosaYelnSbiARoV2bONq1iaNfl7Ra61RUKHsKSvg+t8hL7l5Sz3UJ/uMtOXyfV1TtQaDoKOHo9kn07tiG3p2S6d3RvY7plExaYmxznZ4xpoWzpB0EUVFCx5R4OqbEM6Bb7Ym9vELZnV/Mjn2FfJdzgI278tmUnc/GXfm8t2F3tSv1DsnxHNOpTVUS790xmd6dkumSlmBNLsa0Mpa0QyTaawc/KjWBwUe3q1ZWVl7B1r2FbNqVz8bsfDZ5Cf2Vz3aQV1RWVS8pLppeHdtwjN9Vee9OydbUYkwEs6TdAsVER9GzQxt6dmjDDzmqar2qsju/pNpV+absfD7eksNLa3ZU1TukqaVDMj9IS6BTajwdk+NplxRnvV2MCVOWtMOIyMFml9G906uVFRSXsWV3wSEJ/d0N2ZSWV+/WGRMldEiOr9pXpxpf3XICHVPi7UEjY1oYS9oRok18DP27ptG/a/U29LLyCrbtLWTX/mJ27S8ie38x2fuL2eV9/T63iM+35bKnoJjauuynxMfQ0btC90/mNZN8e7t6N6ZZWNKOcDHRUWR0aENGhzZ11isrryDnQAm78orJzi+uSu7+ry+355K9fxcFJeWHbC/iBudKTYglJcF9TU2MISUhltQE72vVe69O4sG6KQkxdlVvTAAsaRvAJfdOKQl0Skmot25BcZlL5F5y35VXRE5BCXlFZewvKiOvqJT9RaXs2FdEXtF+9heVsb+olPqGOo+LiSLVL4n7J/XUxFg6pcS7h5e8p1M7psQTbVf3ppWxpG0arE18DG3iY+q9evenqhSUlJNXWFotsecVuoSe563zf+8SfyH7i8rILSyluKz2J1GrHljyG2qg8gGm9vYkqokwlrRNsxARkuNjSI5v3I+cqpJb6K7ed+YWVj2oVN+TqPExUd6wAn5J3e9r57REUhNiLLGbsBH+Sbs4L9QRmGYgIrRNiqNtUhx9u6TWWqfySdSduYVVyb0yqe/MLWL5pj18n1d0SDNNm7hoOrdNpFNKPElx0STGxZAUG01inHtVLifFxZAYF0VibAxJcdEkxUWTEBvtLcd4daJtkg0TVOGftPN21F/HtAr+T6IO7FZ7nbLyCnbtL66W2Cu/7skvYd+BUgpLyzlQUkZhSTmFpeWHdJmsT0yUVCXwpLgYv8QeTXxMNAmxUSTERhMf474mxEbVuj4+Jpr42CgSYg5fJyE22tr1W5nwT9qlhbDzc+g8MNSRmDAQEx1Fl7aJdGmbyNAegW1TWl7BgZJyikrLOVBSPaEfKCmnsMT7WlpOYUmZV8e/fjmFpWXkF5exO7+E4rJyiksrKCotp7jMfS07ggmpY6KkKvknxkXTLinOe8XSrk0c7ZPiaOt9bdcmlnZJcbRvE0fbpFh7cjYMhX/Slij47FlL2iZoYqOjSEuMCurAXWXlFRSVVVBcWk6Rl8iLSysoKiuvWi4uK6fIS/YHE76rU1n3QHEZew+Usu9ACZt357O3oJT84rLDHrdNXLRL7G0s0YeL8E/aCanw+XPww9shJi7U0RjTKDHRUSRHRzX6Rm1dSsoq2HeghJwDJeQUuCagnIIS9haUsPdAKXur1geW6Dskx5GR7oZZyOjQhl7e14z0NiTGWUIPtvBP2onpcOC/8M0S6HNWqKMxpsWJi4miU2oCnVLr74NfyT/R7y04mNj3FpSwbW8hW/YUsGxDNtmrt1XbrnNagkvoHdvQM90l854d2nB0+yTiYuwGbVMIetIWkYnAn4Fo4O+qOrdGeQ/gMaAjkANcoqrbvLL/BSYBUcBbwPVac360hBRIBtb8w5K2MU0k0ESfX1xG1u4CtuwuqPq6ZU8Br32xk30HSqvqRQl0a5fkknh6kt9VejJd2yW2iJupAeSqeGA+MBTYA0xT1Sy/8qOBdYBPVe8LVpxBTdoiEg08BJwGbANWisjLqrrOr9p9wHxVfVJEJgB3A5eKyInASUBlY/UHwDhgWY2jwMBpsPwvkJ8NyR2DeUrGGD/JhxnzBmDfgRKXxCsT+p4DbNmdzyff7q3W/BIbLXRvn+SaWdLb0L19EvExUURHCbHRlV+FmKgooqOF2KiD62rWiY6KIiZKiPHqV1uOFmKipNY++QHmqiuBvap6jIhcCNwDTPMr/z/g9Sb4WOsU7CvtEcBGVd0MICL/BCbj/hpV6gv8wlt+B3jJW1YgAYgDBIgF/lvrUTIvhg/vhy+eh9Gzm/ocjDGN0DYpjsFHxx0yXryqkp1fTNbuA2TtLmCz31X6+9/sPuTJ16Z2mKv6QHLVZMDnLS8EHhQRUVUVkXOALUBBsOKuFOyk3RXY6vd+GzCyRp3PgKm4f0umACkikq6qH4nIO8BOXNJ+UFXX13qUTidA16Hw6TMw6lo3epExpkUSkapxbkb0bF+trPIBqdLyCsor1O+ruq8V1deXlStlFUpZeYX7WlFxcJ23/uD2FZSWuzo33X1IWIHkqqo6qlomIrlAuogUAb/CXaXf0GQf1GG0hBuRN+D+Ys0E3gO2A+UicgzQB6h8TOItERmjqu/XupfMi2DxL2HnZ9AlsxnCNsY0tcoHpILtpqbdnQ/4o6rmN8dwCMFO2tuB7n7vu3nrqqjqDtyVNiKSDJyrqvtE5Cpguarme2WvA6OBakk7OzubYcOGQUU5/PcAs/b8ilkPvBm8MzLGhKV58+Yxb968yrcdahTXm6v86mwTkRggDXdDciRwntdxoi1QISJFqvpgE58CAFKzM0aT7tyd2AbgVNwJrwQuUtW1fnU6ADmqWiEidwHlqnqriEwDrgIm4ppH3gD+pKqv+B9j2LBhumrVKvdmweWw+R345dcQE/y/1saY8CQiq1V1mN/7QHLVbGCAql7j3YicqqoX1NivD8gPZu+RoHacVNUy4DrgTWA98LyqrhWRO0TkbK/aeOBrEdkAHAXc5a1fCGwCvsC1e39WM2EfIvNiKNwLG95o8nMxxkSuAHPVo7g27I24zhNzQhFrUK+0m0O1K+2KcvhjP+g8CC56LrSBGWNarJpX2uEksh5RioqGQRfCN2/B/tp7BxpjTDiLrKQNMOgi0HI3HokxxkSYyEvaHY+DbsPdY+1h3vRjjDE1RV7SBndDMns97Pg01JEYY0yTisyk3W8KxCTAmmdCHYkxxjSpyEzaiW3hhLPgi4VQWhTqaIwxpslEZtIGGHwxFO2DDUEfdMsYY5pN5CbtnuMgtasbRMoYYyJE5Cbtyj7bm5ZC3s5QR2OMMU0icpM2uF4kWmF9to0xESOyk3Z6b+g+yvUisT7bxpgIENlJG9w427s3wPbVoY7EGGOOWOQn7X5TICbR+mwbYyJC5CfthFToezZ88QKUFoY6GmOMOSKRn7TBNZEU58JXi0MdiTHGHJHWkbQzxkJadzeIlDHGhLHWkbSjomDQdNj0b8itOe2bMcaEj9aRtAEypwMKn/8z1JEYY0yjtZ6k3b4X9DjJxtk2xoS11pO0wd2Q3LMRtn4c6kiMMaZRWlfS7jsZYpOsz7YxJmy1rqQdnwJ9z4G1i6DkQKijMcaYBmtdSRu8Ptt58NWroY7EGGMarPUl7R4nQdujrYnEGBOWWl/SjoqCQRfB5ndh39ZQR2OMMQ3S+pI2HOyz/Zn12TbGhJfWmbTbZUDGGBtn2xgTdlpn0gZ3Q3LvFvhueagjMcaYgLXepN13MsQlw5qnQx2JMcYErPUm7bg2Xp/tl6CkINTRGGNMQFpv0gbXRFKSD+tfCXUkxhgTkNadtHuc6G5KfmpNJMaY8NC6k7YIZF4MWe/D3m9DHY0xxtSrdSdtgEEXAmJ9to0xYcGSdtujoedY12e7oiLU0RhjTJ2CnrRFZKKIfC0iG0VkTi3lPURkqYh8LiLLRKSbX9nRIrJERNaLyDoRyQhKkJkXw75v4bsPg7J7Y0zLF0CuiheR57zyFZX5SERGiMga7/WZiEwJZpxBTdoiEg08BJwJ9AWmi0jfGtXuA+ar6kDgDuBuv7L5wL2q2gcYAewKSqB9fgxxKTbxrzGtVIC56kpgr6oeA/wRuMdb/yUwTFUzgYnAIyISE9CBfWlJDY012FfaI4CNqrpZVUuAfwKTa9TpC/zbW36nstz7wGJU9S0AVc1X1eAMgh2XBP2nuD7bxflBOYQxpkULJFdNBp70lhcCp4qIqOoBVS3z1icA9Y+N4Us7EV/aOuAr7/0gfGl/CSTQYCftroD/UHrbvHX+PgOmestTgBQRSQeOA/aJyIsi8qmI3Ov9NQyOzIuhtADW/StohzDGtFiB5KqqOl6SzgXSAURkpIisBb4ArvFL4ofzR+AMYA8AvtzPgLGBBNoSbkTeAIwTkU+BccB2oByIAcZ45cOBXsDMoEXRfSS0721NJMaYBlPVFaraD5erbhaRhHo38uXWHBu6PJBjBTtpbwe6+73v5q2roqo7VHWqqg4GfuOt24f7S7fG+3elDHgJGFLzANnZ2QwbNqzqNW/evMZFKuKekPz2A8jZ0rh9GGNarHnz5lXlCaBDjeJ6c5V/Ha/NOo3KK2WPqq4H8oH+9YSzFV/aiYDiS4vFl3YDsD6Q8wissbzxVgLHikhP3AlfCFzkX0FEOgA5qloB3Aw85rdtWxHpqKrZwARgVc0DdOzYkVWrDlndOIMuhH/fCZ89C6f8umn2aYxpEWbNmsWsWbMAEJHdNYrrzVXAy8AM4CPgPODfqqreNltVtUxEegAnAFn1hHMN8Gdck8t2YAlwbSDnEVDSzpiz+GTg2Ky5kx7PmLO4I5CcNXdSvZej3klcB7wJRAOPqepaEbkDWKWqLwPjgbtFRIH3gNnetuUicgOwVEQEWA38LZB4Gy2tG/QaD2uehXFz3Cw3xpiIF2CuehR4SkQ2Ajm4xA5wMjBHREqBCuBaVa35R6Gm4/HlXlxtjS/tJOA/9cVab9LOmLP4NmAYcDzwOBALPA2cVN+2AKr6GvBajXW3+i0vxN2JrW3bt4CBgRynyQy+BF640j3a3mtcsx7aGBM6AeSqIuD8WrZ7CniqgYd7gEObe2tbd4hArrSnAIOBTwCy5k7akTFncUoDAwwfJ0yC+FR3Q9KStjGmKfnSRgMnAh3xpf3CryQVd4Vfr0D+/y/JmjtJ8foeZsxZ3KahcYaV2EToP9V1/SvKC3U0xpjIEgck4y6YU/xeebh28noFcqX9fMacxY8AbTPmLL4KuIJgty2HWuYlsPoJWPcSDLks1NEYYyKFL/dd4F18aU/gy23U0KJ1Ju2MOYsFeA53NzQP1659a9bcSW815mBho9swSD/WNZFY0jbGNL0D+NLuBfrhnqJ0fLkT6tuwzqSdNXeSZsxZ/FrW3EkDgMhO1P4q+2wvvR32bIL03qGOyBgTWZ7BXRCfhev+NwPIDmTDQNq0P8mYs3h442MLU4MuBImyJySNMcGQji/3UaAUX+67+HKvwD2LUq9A2rRHAhdnzFn8LVAACKBZcyc1b1e85pbaBXpPOPigTVTwhj0xxrQ6pd7XnfjSJgE7gPaBbBhI0j6jsVGFvcyLYeHl8OEDcPL/C3U0xpjIcSe+tDTgl7j+2alAQEmm3qSdNXfStxlzFg/CDd4E8H7W3EmfNTbSsNJ3MvSbCm/fBglpMOzyUEdkjIkEvtxXvaVc4BS3Li2gBxYDeSLyeuAq4EVv1dMZcxbPy5o76YGGRxpmoqJhyiNQkg+v/hziU2BAQF0pjTHmUL60aOAC3Jgjb+DL/RJf2lnAr4FE3IOMdQqkeeRKYGTW3EkFABlzFt+DGzAl8pM2QEwcXDAfnj4XFl0Ncclw/MRQR2WMCU+P4kYK/Bi4H1/aDtwwIXPw5b4UyA4C6T0iVB/ntdxb13rEJsL0f8IPBsCCGbDl/VBHZIwJT8OA0/Dl3gz8CNfl76RAEzYElrQfB1ZkzFnsy5iz2Acsx/21aF0SUuGSF6FdBjx7IWxbHeqIjDHhpwRfbgUAvtwiYDO+3D11b1KdqNY/nVnGnMVDcMMPgrsR+WkDAw2aYcOGaZONpx2IvJ3w+EQo3AeXvw5H1Zz70xjT0onIalUd1uwH9qUdADZWhgH09t4LoPhy6+1KHciNyFHA2qy5kz7x3qdmzFk8MmvupBWNDjycpXaGy/4Fj02Ep85xiduemDTGBKbPke4gkBuRf6X6GK/5taxrXdplwKUvweNnwvxz4Io3IK3mHKDGGFNDIweJ8hfQjUhvaFYAsuZOqiD405S1fJ1OgEtfhMK97oq7oL6JKowx5sgFknw3Z8xZ/DPc1TW4ecw2By+kMNJlMFz0HDw9FZ6aAjNfdQ/hGGNMkARypX0NbqaF7bgZ0kcCs4IZVFjJOAmmPQ271sM/pkHJgVBHZIxpyXxp0fjSnmns5gH1HmnJmr33yOGsXQQLr4Bep8D0ZyEmPtQRGWMOI2S9Ryr50j4AJuDLLWnopoH0Hvlf4E6gEHgDN9Huz7PmTnq6oQeLaP2mQHE+vHwdvPA/cN7jEG1N/8aYWm0G/oMv7WXc6KmOL/f/6tswkOaR07PmTsrDPbmTBRwD3NioMCPdkEvhjLth/cvwys+goiLUERljWqZNwKu4HOw/V2S9ArkUrKwzCViQNXdSbsacxY0JsnUYfS0U5cK7c0T9A84AACAASURBVN2s7hPvdjPhGGNMJV/u7e5rWhK+3AbdCAskab+aMWfxV7jmkZ9kzFncEShqcJCtyfg5UJwHy//iHn8/5dehjsgY05L40kbjhgNJBo7GlzYIuBpf7rX1bVpv80jW3ElzcL1HhmXNnVQKHAAmH1nEEU4Ezvg9DL4E3r0HPnww1BEZY1qWP+EmmHHjjvhyPwPGBrJhQHfKsuZOyvFbLsC/4dzUTgR+fL+7ObnkN24s7qEzQh2VMaal8OVuxVftuY7yw1X1Z90bgikqGqb+DUoK4JXrIT4Z+p8b6qiMMaG3FV/aiYDiS4sFrgfWB7JhIL1HzJGonETh6NHw4izYsCTUERljQu8aYDZuBpvtQKb3vl6NStoZcxaf0JjtWq24JLjon3BUP3j+Usj6INQRGWNCwZd2j7d0Cr7ci/HlHoUvtxO+3EsCHVe7sVfadrnYUAlpcMkiaNsD/nEhbP8k1BEZY5rfj/ClCXBzY3dw2DbtjDmL7z9MkQBtG3vAVq1NOlz2khuL++mpbizuTkc8vK4xJny8AewFkvGl5VE5+cHBSRBS69tBXTciLwd+CRTXUja94bEaAFK7HJxEoXIs7vY9Qx2VMaY5+HJvBG7El/YvfLmN6jpdV9JeCXyZNXfShzULvLkiTWO17+muuB8/E+ZPdok7tUuoozLGNBdf7mR8aT2AY/Hlvo0vLRGIwZe7v75N62rTPg9YU1tB1txJdml4pDr1gUtegAM5Lnn/d22oIzLGNBdf2lXAQuARb003IKAZ2etK2slZcycd8eDQIjJRRL4WkY0iMqeW8h4islREPheRZSLSrUZ5qohsE5HIe6yw61C4dBGUFsLfT3PDuxpjQiKAXBUvIs955StEJMNbf5qIrBaRL7yvEwI43GzgJCAPAF/uN0CnQOKsK2lXZf2MOYtfCGRnNYlINPAQcCbQF5guIjWnL78PmK+qA4E7gLtrlP8OeK8xxw8L3YfDrHddd8AFM+Gt26AioAejjDFNJMBcdSWwV1WPAf4IVHbf2w38WFUHADOApwI4ZHG1sbR9aTG4G5L1qitp+w9N1yuQndViBLBRVTeragnwTw4dt6Qv8G9v+R3/chEZChxFpHcxTO0MMxfDsCvgP3+CZ85zzSbGmOYSSK6aDDzpLS8EThURUdVPVXWHt34tkCgi9c2C8i6+tF8DifjSTgMWAK8EEmhdSVsPs9wQXYGtfu+3eev8fQZM9ZanACkiki4iUcAfgBsaeezwEhMHZ/3RjVeS9QHMGw/ffxHqqIxpLQLJVVV1VLUMyAXSa9Q5F/hEVWvrdedvDpANfAFcDbwG/DaQQOvqPTIoY87iyn6Eid4y3nvNmjup3v6EAboBeFBEZuKaQbbjBk65FnhNVbdJaxqPeugM11Ty3CWunXvygzDgvFBHZYyph4j0wzWZnF5vZV9uBb60l4CX8OVmN+Q4h03aWXMnRTdkR4exHeju976bt66K92/FVAARSQbOVdV9IjIaGCMi1+LGnI0TkXxVrXaDIDs7m2HDDk71NmvWLGbNCvN5h7sNc+3cC2bAC1fCzjVwqs+mLzPmCMybN4958+ZVvu1Qo7jeXOVXZ5uIxABpeEOreh0oFgGXqeqmwwbhnoa8DbiOypYOX1o58AC+3DsCOY+gTuzrndgG4FTcCa8ELlLVtX51OgA5qlohIncB5ap6a439zASGqep1NY/RYib2DYayEnjz17Dyb9BznJt3sk3N/8aMMQ1Vc2LfAHPVbGCAql4jIhcCU1X1AhFpC7wL3K6qL9Z5YF/aL3A3O2fhy93iresF/BV4A1/uH+uLPaij/HntPtcBb+KGHXxeVdeKyB0icrZXbTzwtYhswN10vCuYMYWVmDiYdB9Mfgi+W+7auXd+FuqojIk4AeaqR4F0EdkI/ALXLo233THArSKyxnsdrvvepcD0qoQN4MvdDFwCXBZIrEG90m4OEX2l7W/7anjuUjiwB85+AAZeEOqIjAlbNa+0m40v7Ut8uf0bXObHxtMOF12HunburkPhxavgjZuhvCzUURljGqakkWVV7M5WOEnu6AabWvJbN2nw91/A+U9Am5r3VIwxLdQgb3S/mgRICGQHlrTDTXQsnHkPdM6EV/8fPDIOLnwaugwOdWTGmPr4co+4V541j4SrzOludEARePQMWPOPUEdkjGkGlrTDWZfBMGsZdB8BL/0EXrsRyktDHZUxJogsaYe7Nh3g0pdg1Gz4eB48eTbk7wp1VMaYILGkHQmiY2Di72Hq32DHp66de9vqUEdljAkCS9qRZOAFcOUSl8QfnwifBDJCpDEmnFjSjjSdB7r+3D1OhJevg1d/4R6HN8ZEBEvakSipPVz8Apz4M1j1KDz5Y9j7baijMsY0AUvakSo6Bk7/HZz3mHsI56GR8N69UFbfML/GmJbMknak638uXPcxHHc6/PtO+Mso2Ph2qKMyxjSSJe3WIK0bXDAfLnkRJAqePtdNsrBva/3bGmNaFEvarckxp8JPPoRTb4Vv3oaHRsD7/2c3Ko0JI5a0W5uYeBjzS9dk0nsCLL0d/noibHon1JEZYwJgSbu1ans0XPgMXLwQKsrgqXPg+RmQW3OGJWNMS2JJu7U79jS4djmc8lvY8AY8OBz+82drMjGmhbKkbSA2AcbdCLNXQK9x8Nat8PDJsPndUEdmjKnBkrY5qF0GTH8Wpj8HZUUw/2xYeAXk7Qh1ZMYYjyVtc6jjJ7qr7vE3w/pXXZPJhw/asK/GtACWtE3tYhNh/ByYvRx6nARLfgMPj4GsD0IdmTGtmiVtU7f2veDi52H6P6G0AJ6YBC9cBfu/D3VkxrRKlrRNYI4/E65dAWNvgnUvuSaT5X+1GeGNaWaWtE3g4pJgwm9cF8HuI+CNOfDIWPj2o1BHZkyrYUnbNFx6b/dQzrRnoDjPTbjw3CXwxUIo3Bvq6IyJaDGhDsCEKRHoc5Z7FP79P8DqJ2D9KyDRbgKG4ya6JpX03qGO1JiIIqoa6hiOyLBhw3TVqlWhDsNUlMP21fD1a/D1G5C93q3vcNzBBN5thBvn25gQE5HVqjos1HE0hiVtExx7s1zy3vA6ZP0HKkohsR0ce7pL4secCglpoY7StFKWtEPIknYYKMqDTUtdEv9mCRTmQFSM6/99/JkuibfvGeooTStiSTuELGmHmYpy2Pqxa0bZ8Abs3uDWdzzBrxllOERFhzZOE9EsaYeQJe0wt2eTS95fvw7ffghaDknprhnl+DPdjc74lFBHaSKMJe0QsqQdQQr3ufkrN3jNKEW5EB0HGSfDcWdCvymQ3DHUUZoIYEk7hCxpR6jyMti63F2Bf/065GyCqFjoezYMu8K1h4uEOkoTpixph5Al7VZi13rXF3zNs1CcCx2Od8l70IWQ2DbU0ZkwY0k7hCxptzIlB+DLF2DVY7DjE4hJhAHnugTedWioozNhIpyTdtAfYxeRiSLytYhsFJE5tZT3EJGlIvK5iCwTkW7e+kwR+UhE1npl04IdqwkDcUkw5FKY9Q7MWgYDL4AvX4S/TYBHxsHqJ6GkINRRmjAUQK6KF5HnvPIVIpLhrU8XkXdEJF9EHgx6nMG80haRaGADcBqwDVgJTFfVdX51FgCvquqTIjIBuFxVLxWR4wBV1W9EpAuwGuijqvv8j2FX2oaiXPj8eVj5qHsSMz7VNZsMvRyO6hvq6EwLVPNKO8BcdS0wUFWvEZELgSmqOk1E2gCDgf5Af1W9LpixB/tKewSwUVU3q2oJ8E9gco06fYF/e8vvVJar6gZV/cZb3gHsAqzrgDlUQhqMuAqu/Qguf8P19179BPx1NDw2ET5fAGXFoY7StGyB5KrJwJPe8kLgVBERVS1Q1Q+AouYINNhJuyuw1e/9Nm+dv8+Aqd7yFCBFRNL9K4jICCAO2BSkOE0kEIEeo+Hcv8EvvoLTfucma3jxf+D/+sCSWyBnc6ijNC1TILmqqo6qlgG5QDrNrCUMzXoDME5EPgXGAduB8spCEekMPIVrNqkITYgm7LRJh5N+Bj/9BC5d5EYe/OghuH8wPDXFjUhoEziYMBTsIde2A9393nfz1lXxmj6mAohIMnBuZbu1iKQCi4HfqOry2g6QnZ3NsGEHbwLPmjWLWbNmNeU5mHAWFeWequw9wc0q/8lTrunkuUsgpTMMmQFDLoO0mhdVJtLMmzePefPmVb7tUKO43lzlV2ebiMQAacCeIIRap2DfiIzBNe6fijvhlcBFqrrWr04HIEdVK0TkLqBcVW8VkTjgdeAVVf3T4Y5hNyJNg5WXwTdvum6DG5eCRLlH5ofMgN6nQHRsqCM0QVbLjchActVsYIDfjcipqnqBX/lMYFiwb0QG9UpbVctE5DrgTSAaeExV14rIHcAqVX0ZGA/cLSIKvAfM9ja/ABgLpHsfBsBMVV0TzJhNKxAdAydMcq+cLfDJk+4K/KtX3fCxfX7sHpnPGGvjf7cSAeaqR4GnRGQjkANcWLm9iGQBqUCciJwDnO7f86Qp2cM1xoDrXbJxKaxd5EYgLMl3A1f1OdtL4CfbyIMRJJwfrrHLCGMAYuLhhB+5V2mhG7hq7SLX/3v149CmI/Sd7BL40aMtgZuQicikXVpayrZt2ygqapZukyaMJCQk0K1bN2Jj62i3jk10TSR9fuwem9/4lkvga/4BK/8OyUcdTODdR7mbncY0k4hsHtmyZQspKSmkp6cjNhKc8agqe/bsYf/+/fTs2YiZckoKYMObLoF/swTKilwPlL7nuATebbgl8DBhzSMtTFFRERkZGZawTTUiQnp6OtnZ2Y3bQVwb6D/VvYrz3bjfaxe5Xigr/gqpXf0S+DAbOtYERUQmbcAStqlVk/1cxCfDgPPcqyjvYAJf+TdY/hCkdYd+XgLvMsQSuGky9r9cC/HSSy+xbl1QeghV8/vf/77B2zzxxBNcd92hXU+feOIJOnbsSGZmJpmZmVx22WUALFiwgH79+hEVFUWr6NmTkOpGG5z+LNy4EaY8Ap36wvKH3eiDfx4Ib90KOz6FMG+ONKFnSbuFaMlJuy7Tpk1jzZo1rFmzhvnz5wPQv39/XnzxRcaOHdukx6pLWVkLeSQ9Ic2NMHjx83DjNzD5L27Cho8egnnj4X97wtPnwju/hw1LoGB3qCM2YcaSdpCcc845DB06lH79+vk/OktycnLV8sKFC5k5cyYffvghL7/8MjfeeCOZmZls2rSJNWvWMGrUKAYOHMiUKVPYu3cvAOPHj+fnP/85w4YNo0+fPqxcuZKpU6dy7LHH8tvf/rbO48+ZM4fCwkIyMzO5+OKLAXj66acZMWIEmZmZXH311ZSXu2FfHn/8cY477jhGjBjBf/7znwade58+fTj++OPrrLNz507Gjh1LZmYm/fv35/333wfgjTfeYMiQIQwaNIhTTz0VgJycHM455xwGDhzIqFGj+PzzzwHw+XxceumlnHTSSVx66aVkZ2dz7rnnMnz4cIYPH97guJtcYjsYfDFcshBu8BJ4nx9D3k547174x/lwb2/48yBYeIVL7N+tcF0OjTkcVQ3r19ChQ7WmdevWHbKuue3Zs0dVVQ8cOKD9+vXT3bt3q6pqmzZtquosWLBAZ8yYoaqqM2bM0AULFlSVDRgwQJctW6aqqrfccotef/31qqo6btw4vemmm1RV9U9/+pN27txZd+zYoUVFRdq1a9eq4wRy/HXr1ulZZ52lJSUlqqr6k5/8RJ988kndsWOHdu/eXXft2qXFxcV64okn6uzZsw85x8cff1w7dOiggwYN0kGDBuljjz1WrXzcuHG6cuXKWj+f++67T++8805VVS0rK9O8vDzdtWuXduvWTTdv3lztHK677jr1+Xyqqrp06VIdNGiQqqredtttOmTIED1w4ICqqk6fPl3ff/99VVX99ttv9YQTTqj12C3h50OL9qtu+UD1gz+pPnep6h/6qt6W6l63t1d9eIzqK/9P9ZOnVP+7XrW8PNQRRxTcU44hz1+NeUXsjchKt7+ylnU78pp0n327pHLbj/vVWef+++9n0aJFAGzdupVvvvmG9PTARnHMzc1l3759jBs3DoAZM2Zw/vnnV5WfffbZAAwYMIB+/frRuXNnAHr16sXWrVtJT08P6PhLly5l9erVDB8+HIDCwkI6derEihUrGD9+PB07uuHLp02bxoYNG2qNddq0aTz4YMMn6xg+fDhXXHEFpaWlnHPOOWRmZrJs2TLGjh1b1R2vffv2AHzwwQe88MILAEyYMIE9e/aQl5dX9VkkJiYC8Pbbb1drYsrLyyM/P7/afzctRnwyZJzkXpX2fw/bVx98fbHQ9UwBiEuBLpmuV0rXodB1GKR2Dk3sJqQiPmmHwrJly3j77bf56KOPSEpKYvz48VUP+vj3Xmjswz/x8fEAREVFVS1Xvi8rK6vz+P5UlRkzZnD33XdXW//SSy81Kq6GGDt2LO+99x6LFy9m5syZ/OIXv6Bdu3YN3k+bNm2qlisqKli+fDkJCQlNGWrzSfnBwTFRACoqYM9G2L7KJfFtq+DDB6DCa79P6QLdhnpJfCh0GQzxKaGL3zSLiE/a9V0RB0Nubi7t2rUjKSmJr776iuXLD44qe9RRR7F+/XqOP/54Fi1aREqK+yVLSUlh//79AKSlpdGuXTvef/99xowZw1NPPVV11X2kx4+NjaW0tJTY2FhOPfVUJk+ezM9//nM6depETk4O+/fvZ+TIkVx//fXs2bOH1NRUFixYwKBBg5ro03G+/fZbunXrxlVXXUVxcTGffPIJv/nNb7j22mvZsmULPXv2JCcnh/bt2zNmzBieeeYZbrnlFpYtW0aHDh1ITU09ZJ+nn346DzzwADfeeCMAa9asITMzs0njblZRUdDxOPfKvMitKy2C7z8/mMS3r3ZjgwMg8IP+MGg6DLzQjSluIk7EJ+1QmDhxIg8//HDVDblRo0ZVlc2dO5ezzjqLjh07MmzYMPLz8wG48MILueqqq7j//vtZuHAhTz75JNdccw0HDhygV69ePP74401y/FmzZjFw4ECGDBnCM888w5133snpp59ORUUFsbGxPPTQQ4waNQqfz8fo0aNp27ZtgxPfokWL+OlPf0p2djaTJk0iMzOTN998s1qdZcuWce+99xIbG0tycjLz58+nY8eOzJs3j6lTp1JRUUGnTp1466238Pl8XHHFFQwcOJCkpCSefPLJWo97//33M3v2bAYOHEhZWRljx47l4YcfblDsLV5sAnQf4V6VDuQcbFL5Zgm8+Wt46zZ3xT7kMuh1ij2pGUEi8jH29evX06dPnxBFZFq6iP/5+O86+PQp+OxZKNwLaUfD4EtcT5a0bqGOrkUI58fY7c+vMZHmqL4w8W745ddw3mOQ3guW/R7+2N/1EV/3LygrCXWUppGsecSYSBUTD/3Pda+9WfDpM/Dp0/D8ZZDUATKnw+DLXJu5CRt2pW1Ma9AuAyb8Bn7+JVy0AI4eBcv/Cg8Nh0fPcAm9pCDUUZoA2JW2Ma1JVDQcd7p75e9y7d6fzId/XQuv/woGnOtuXtogVy2WJW1jWqvkTnDS9XDiz+C7j1zy/uw5N1v9Uf1d8h5wPiS1D3Wkxo81jxjT2olAjxNhysNww9cw6f8gKgZevwn+cAIsvBI2v+se9jEhZ0m7Gfh8Pu67774662RnZzNy5EgGDx5cNXhSsKxZs4bXXnstqMcwYSohDYZfCVe/C1e/D0NnuOnW5p8NDwx2A13lN3ISCdMkLGm3EEuXLmXAgAF8+umnjBkzJqBtKkfkq01dQ5Va0jYB6TwQfnSv6zo49W9uYod/3wn3Z8K799qNyxCxpB0kd911F8cddxwnn3wyX3/9ddX6TZs2MXHiRIYOHcqYMWP46quvWLNmDTfddBP/+te/yMzMpLCwkCVLljB69GiGDBnC+eefX/XkZEZGBr/61a8YMmQICxYsqHbMmTNncs011zBy5EhuuukmPv74Y0aPHs3gwYM58cQT+frrrykpKeHWW2/lueeeIzMzk+eee46CggKuuOIKRowYweDBg/nXv/7VrJ+VaeFiE90kDzNfhdkfQ+9T4J074YGhrh284vAXDyYIQj3M4JG+WuLQrKtWrdL+/ftrQUGB5ubmau/evfXee+9VVdUJEybohg0bVFV1+fLlesopp6iqG+a0cvjT7OxsHTNmjObn56uq6ty5c/X2229XVdUePXroPffcU+txZ8yYoZMmTdKysjJVVc3NzdXS0lJVVX3rrbd06tSphxxLVfXmm2/Wp556SlVV9+7dq8cee2zVsSNRqH8+IsK3H6n+7YduKNkHR6p+/aZqRUWoowoYNjRrC/b6HPj+i6bd5w8GwJlzD1v8/vvvM2XKFJKSkoCDQ6nm5+fz4YcfVhtmtbi4+JDtly9fzrp16zjpJDdsZ0lJCaNHj64qnzZt2mGPff755xMdHQ24gaNmzJjBN998g4hQWlpa6zZLlizh5Zdfrmp3Lyoq4rvvvovsR73NkTl6FFy5BNa/DG/73IQOPcfCab9zQ8iaoIn8pN2CVFRU0LZtW9asWVNnPVXltNNO49lnn6213H840rrKbrnlFk455RQWLVpEVlYW48ePP+zxXnjhhXpnmzGmGhHoOxmOOxNWPw7v3gPzxsGAC2DCb6Fdj1BHGJEiP2nXcUUcLGPHjmXmzJncfPPNlJWV8corr3D11VeTmppKz549WbBgAeeffz6qyueff37IsKejRo1i9uzZbNy4kWOOOYaCggK2b9/Occc17HHj3NxcunbtCrhJeCv5DwMLcMYZZ/DAAw/wwAMPICJ8+umnDB48uPEfgGldYuJg5NVubswP/gTL/wLrXnLrxvzSTbtmmozdiAyCIUOGMG3aNAYNGsSZZ55ZNTMMwDPPPMOjjz7KoEGD6NevX603/Tp27MgTTzzB9OnTGThwIKNHj+arr75qcBw33XQTN998M4MHD67Wm+SUU05h3bp1VTcib7nlFkpLSxk4cCD9+vXjlltuadyJm9YtIQ1+eBv89BP3UM6HD8KfM93XskObAU3j2NCsptWxn49m8v0XblzvTUuh7dFw6m3Qb2qLGNvbhmY1xpiafjAALn0RLl0E8WnwwpXw9wmQ9UGoIwtrlrSNMcHVe4J7wvKch90gVU9Mgn9Mg10Nb/IzlrSNMc0hKtqN3/3T1a6Z5NsP4a+j4eWfuVnoTcAiNmmHe1u9CQ77uQix2EQY8wv42RoYMQvWPAP3D4Z3fg/F+aGOLixEZNJOSEhgz5499gtqqlFV9uzZQ0JCQqhDMW3S4cx73GPxx57u+njfPxhWPgrlhx83xzRD7xERmQj8GYgG/q6qc2uU9wAeAzoCOcAlqrrNK5sB/NareqeqHjINd229R0pLS9m2bRtFRUVNfTomzCUkJNCtWzdiY2NDHYrxt3UlLPktbF0Oqd0g5SiIinVDxEbH1Fiu7339dWXIJYf0HgkgV8UD84GhwB5gmqpmeWU3A1cC5cDPVPXNYH1UQU3aIhINbABOA7YBK4HpqrrOr84C4FVVfVJEJgCXq+qlItIeWAUMAxRYDQxV1b3+x6gtaRtjwpAqfP0arPkHlBZCRdnBV3kpVJS6wanKS2spq7GsdQ9iJbfnVUvaAeaqa4GBqnqNiFwITFHVaSLSF3gWGAF0Ad4GjlOtJ4jGCubAJsBo4E2/9zcDN9eosxbo7i0LkOctTwce8av3iPch1jtgVFN65JFHgrr/5hIJ52Hn0DKExTmUl6uWlagWF6gW5qoW7FHd/1/VfdtUc7YokKUNz1VvAqO95Rhgt5ezqtX1rxeMV7DbtLsCW/3eb/PW+fsMmOotTwFSRCQ9wG2Dbt68ec19yKCIhPOwc2gZwuIcoqIgOhbikiAh1U2ZltwJ0rq6SY5dc6y/QPJNVR1VLQNygWbPVS3hRuQNwDgR+RQYB2zHtQsZY4ypIdgDRm0Huvu97+atq6KqO/CutEUkGThXVfeJyHZgfI1tl9U8wOrVq/NFxP+PTzbu35am0kFEmnJ/oRIJ52Hn0DKE6zl04OAVdnSNsnpzlV+dbSISA6ThbkgGsm2TCfaNyBhc4/6puJNYCVykqmv96nQAclS1QkTuAspV9VbvRuRqYIhX9RPcjcicoAVsjGmVAsxVs4EBevBG5FRVvUBE+gH/4OCNyKXAsRqkG5FBvdJW1TIRuQ7XMB8NPKaqa0XkDtzMES/jrqbvFhEF3gNme9vmiMjvcB8ewB2WsI0xwRBgrnoUeEpENuK6J1/obbtWRJ4H1gFlwOxgJWyIgFH+jDGmNWkJNyJbBBF5TER2iciXhym/WEQ+F5EvRORDERlUW71Qqu8c/OoNF5EyETmvuWILVCDnICLjRWSNiKwVkXebM75ABPCzlCYir4jIZ945XN7cMdZHRLqLyDsiss6L8fpa6oiI3C8iG73fjSG17StUAjyHFv97fYhg9SUMtxcwFtd+/uVhyk8E2nnLZwIrQh1zQ8/BqxMN/Bt4DTgv1DE34vvQFvdv6NHe+06hjrkR5/Br4B5vufJJ4LhQx10jxs7AEG85Bdfe27dGnR8Br+P6Ko9qab8TAZ5Di/+9rvmyK22Pqr6H++U5XPmHevBpzOW4O8QtSn3n4Pkp8AKwK/gRNVwA53AR8KKqfufVb3HnEcA5KO55BAGSvbotasANVd2pqp94y/uB9Rza93gyMF+d5UBbEenczKEeViDnEA6/1zVZ0m6cK3FXGGFFRLriHmD6a6hjOQLHAe1EZJmIrBaRy0IdUCM8CPQBdgBfANerakVoQzo8EckABgMrahS1iAfgAlHHOfgLi9/ryJ/Yt4mJyCm4b+7JoY6lEf4E/Epd98pQx9JYMbgBe04FEoGPRGS5qm4IbVgNcgawBpgA/P/27ifEyiqM4/j3h7ooaBNCq2wkogFLF1kGuRJa5KYBExrEMKJNtHAXBeXCTat2RpCCINEsHP8tZMid5sYKShcGSQMhqRGFPQAAAu9JREFUuBrIwCKwfi3OGdHb2H2VuXPfY7/P5t773nMvz+HyPpx7eJ/nfRI4I+mc7d/GG9a/1dqJWWBvH+ProsscWjqvk7TvgaSNwEHgFdsL447nPmwGZmrCXgtsl3TT9onxhnVPrgILtm8ANySdBTZR9itb8SbwsctG6hVJ88AkcGG8Yd1J0hpKsvvC9rElhqxoUcn96DCH5s7rbI90JGkdcAzY3diq7hbb621P2J4AjgLvNJawAU4CWyWtlvQwsIWyV9mSXyj/FJD0GPA08PNYIxpQ99sPAZdtf3KXYaeAN+pVJC8C121fW7Egh+gyhxbP66y0K0lfUgp91kq6CuwD1gDY/gz4iNIc5tO6Ur3pnt3NucMcem/YHGxfljQHXAT+pvQ9/s9LHFdah99hP3BY0iXKlRfv2e5bWfhLwG7gkqTv67EPgHVwax6nKVeQXAF+p/yD6JMuc+j9eT0oxTUREQ3J9khEREOStCMiGpKkHRHRkCTtiIiGJGlHMyRNSbKkyfp6okNzrKFjIlqSpB0tmQa+ro8R/0tJ2tGEWoq8lVJq/PoS7++RdLL2JPlJ0r7b3l4l6fPanvMrSQ/Vz7wt6ZvaInW2FutE9FqSdrTiVWCuVq0tSHpuiTEvADuAjcBOSYtFEk8BB2xvAH6tY6B0C3ze9iZKVeVbI51BxDJI0o5WTAMz9fkMS2+RnLG9YPsPSmnyYvOfeduLFXHfARP1+TOSztXKxF3AhpFEHrGMUsYevadyk+dtwLMq9xJdRelJfWBg6GB57+LrP2879helOyDAYWDK9g+S9lBKzyN6LSvtaMFrwBHbT9SGV48D89zZYQ7gZUmP1j3rKeD8kO99BLhWO8HtWvaoI0YgSTtaMA0cHzg2C7w/cOxCPX4RmLX97ZDv/ZDSFP888OMyxBkxcmkYFQ+Eur2x2fa7444lYpSy0o6IaEhW2hERDclKOyKiIUnaERENSdKOiGhIknZEREOStCMiGpKkHRHRkH8AE8TPECj36XkAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}