03_ab_testing_course_covid.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNA4zvdn5igX/dXizw2Ovgu",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/glebmikha/ab-testing-course/blob/main/03_ab_testing_course_covid.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Мой Telegram-канал: https://t.me/mikhaylovgleb"
      ],
      "metadata": {
        "id": "jetpmr3F7_3N"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Data upload"
      ],
      "metadata": {
        "id": "YJGXzXHjsXzY"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "96rXJWQLstbZ"
      },
      "outputs": [],
      "source": [
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "https://drive.google.com/file/d/1QqWhauAgV8PQV-bLNyHnDjVV2sLFq9UA/view?usp=sharing"
      ],
      "metadata": {
        "id": "r6-uCoDZs3Im"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install gdown==v4.6.3"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qZ_MO_UOzn5e",
        "outputId": "7faed797-1f18-4b3d-aea3-bd356b10707e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting gdown==v4.6.3\n",
            "  Downloading gdown-4.6.3-py3-none-any.whl (14 kB)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from gdown==v4.6.3) (3.13.1)\n",
            "Requirement already satisfied: requests[socks] in /usr/local/lib/python3.10/dist-packages (from gdown==v4.6.3) (2.31.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from gdown==v4.6.3) (1.16.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from gdown==v4.6.3) (4.66.1)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (from gdown==v4.6.3) (4.11.2)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4->gdown==v4.6.3) (2.5)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests[socks]->gdown==v4.6.3) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests[socks]->gdown==v4.6.3) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests[socks]->gdown==v4.6.3) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests[socks]->gdown==v4.6.3) (2023.11.17)\n",
            "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /usr/local/lib/python3.10/dist-packages (from requests[socks]->gdown==v4.6.3) (1.7.1)\n",
            "Installing collected packages: gdown\n",
            "  Attempting uninstall: gdown\n",
            "    Found existing installation: gdown 4.6.6\n",
            "    Uninstalling gdown-4.6.6:\n",
            "      Successfully uninstalled gdown-4.6.6\n",
            "Successfully installed gdown-4.6.3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! gdown 1QqWhauAgV8PQV-bLNyHnDjVV2sLFq9UA"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BWhNMvtGs4_b",
        "outputId": "7f237621-3929-4f1e-e32d-22a10f79592d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1QqWhauAgV8PQV-bLNyHnDjVV2sLFq9UA\n",
            "To: /content/covid.csv\n",
            "\r  0% 0.00/60.0k [00:00<?, ?B/s]\r100% 60.0k/60.0k [00:00<00:00, 79.4MB/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv('/content/covid.csv')"
      ],
      "metadata": {
        "id": "5iepRWPjs6EV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5mzHvP08s-Ek",
        "outputId": "ffc10a3a-707b-489d-a7d6-3fdbee06f834"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10000"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "bs_t3nE6s_k3",
        "outputId": "99af6acd-517e-4522-e6c9-5f278f1e56c8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   y_true  y_pred1  y_pred2\n",
              "0       0        0        0\n",
              "1       1        1        0\n",
              "2       1        0        1\n",
              "3       0        1        0\n",
              "4       1        0        1"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-645c098a-35c1-48a3-8bf4-697b0e06cfa5\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>y_true</th>\n",
              "      <th>y_pred1</th>\n",
              "      <th>y_pred2</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-645c098a-35c1-48a3-8bf4-697b0e06cfa5')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-645c098a-35c1-48a3-8bf4-697b0e06cfa5 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-645c098a-35c1-48a3-8bf4-697b0e06cfa5');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-6caa0050-2848-44b9-8acf-11f955e3cb36\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-6caa0050-2848-44b9-8acf-11f955e3cb36')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-6caa0050-2848-44b9-8acf-11f955e3cb36 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Error rate"
      ],
      "metadata": {
        "id": "R3VLA0-LsdW9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "len(df[df['y_true'] != df['y_pred1']])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cz0nhuoHoQGz",
        "outputId": "eff6329e-d481-4d1b-bd7a-75e8cb51708b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1800"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "czPYK_yvoxKy",
        "outputId": "d82259b1-32d8-4dc2-89ae-6864c9784e62"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10000"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(df[df['y_true'] != df['y_pred1']]) / len(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K1NzG3leo27W",
        "outputId": "81b8bd60-ddbd-47ad-ede7-3c4b725e405e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.18"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "(df['y_true'] != df['y_pred1']).mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cgfz7Rl2o5W-",
        "outputId": "d2931e6d-afa4-4cbc-8699-fe831a2ebfe5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.18"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "(df['y_true'] != df['y_pred1']).sum()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "58dd5jQNo9xJ",
        "outputId": "04ae1e21-90d6-429d-d79a-b98d96f16e3d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1800"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "(df['y_true'] != df['y_pred1']).count()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gkq-W7WvpIil",
        "outputId": "3cae83c7-332a-4ea2-e768-05806c9387bf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10000"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "(df['y_true'] != df['y_pred1']).sum() / (df['y_true'] != df['y_pred1']).count()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h-QMOPZxpPU_",
        "outputId": "2f8637d4-48e7-44ac-ebbc-da14319c560d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.18"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "(df['y_true'] != df['y_pred1']).mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-K_4Qt02pRzJ",
        "outputId": "cd016ad7-9029-4550-d05c-4740ced7aa2a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.18"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "(df['y_true'] != df['y_pred2']).mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8L7MXKIApVWo",
        "outputId": "10100e0d-9877-4aa9-980c-2fa3731cc8e3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.18"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Accuracy"
      ],
      "metadata": {
        "id": "vbnnXxt5sgT4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import accuracy_score"
      ],
      "metadata": {
        "id": "s2nmYsfZqew9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_score(df['y_true'],df['y_pred1'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kTGreJLYqlFD",
        "outputId": "bc966d35-6793-4d20-c642-c4af4e66211d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.82"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_score(df['y_true'],df['y_pred2'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tLgHKAEEqsY1",
        "outputId": "231abc52-77ec-43d3-a0e9-d7e218e7421a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.82"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Positive and Negative"
      ],
      "metadata": {
        "id": "IcJRsntyr8xq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "positives = df[df['y_true'] == 1].copy()"
      ],
      "metadata": {
        "id": "vMvHKTm0s1n_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "negatives = df[df['y_true'] == 0].copy()"
      ],
      "metadata": {
        "id": "WOxved8xtCHI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## FNR"
      ],
      "metadata": {
        "id": "G1zd1YontZDl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "(positives['y_true'] != positives['y_pred1']).mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0ZjvtdxitIfa",
        "outputId": "266e84ff-c5c7-4e4c-8227-9cd22a53924b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.2949852507374631"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "(positives['y_true'] != positives['y_pred2']).mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1k1zJk9ktNjW",
        "outputId": "9e60a7fb-4875-4cc5-a57f-c51cf6052d71"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.344149459193707"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## FPR"
      ],
      "metadata": {
        "id": "AVMK_2PPtQup"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "(negatives['y_true'] != negatives['y_pred1']).mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e0c71922-e51a-4b2d-bfab-774b797d3094",
        "id": "UQOkyn6ithS-"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.061037639877924724"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "(negatives['y_true'] != negatives['y_pred2']).mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "99192dc6-6f65-449e-ab33-25bbdbe90200",
        "id": "vBrtuKohthTK"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.01017293997965412"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Sensitivity"
      ],
      "metadata": {
        "id": "MtahVx5Fvp2T"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_score(positives['y_true'],positives['y_pred1'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_6DcZYaqv0a9",
        "outputId": "84095d69-121a-4321-ccb7-3ad2ab98239f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7050147492625368"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_score(positives['y_true'],positives['y_pred2'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "osFAK4lAv4aE",
        "outputId": "3ad24cb6-63eb-4086-a6cd-ff529788824a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.655850540806293"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Specificity"
      ],
      "metadata": {
        "id": "oOZpKtCjv6Lt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_score(negatives['y_true'],negatives['y_pred1'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-yN1Apedv_HL",
        "outputId": "d24e983e-a883-4a46-b795-02f0334cbbe5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9389623601220752"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_score(negatives['y_true'],negatives['y_pred2'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0YSc6g7RwCeJ",
        "outputId": "c1d58112-fc3d-47bf-9fad-0569d332e180"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9898270600203459"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Sensitivity and Specificity as functions"
      ],
      "metadata": {
        "id": "evQ0tjnQwGXm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import recall_score"
      ],
      "metadata": {
        "id": "LrjXHGey0kWr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "recall_score(df['y_true'],df['y_pred1'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZoSEYpXt0nmV",
        "outputId": "6fb04a99-7223-406d-d12e-af98f7b6a94c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7050147492625368"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "recall_score(df['y_true'],df['y_pred2'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QWoybpy70xy9",
        "outputId": "85aaf820-b45e-40cb-b05d-79ff478688a6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.655850540806293"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "recall_score(df['y_true'],df['y_pred1'],pos_label=0)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "o4PcEnfd015Z",
        "outputId": "baf93635-d26d-4848-e7c0-419a3eef5a36"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9389623601220752"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "recall_score(df['y_true'],df['y_pred2'],pos_label=0)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kLM2g3Xq0-Ph",
        "outputId": "c46f41ed-7329-4905-863d-a3f6c6bcd219"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9898270600203459"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Confusion matrix"
      ],
      "metadata": {
        "id": "nNcw0SFP1ApL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay"
      ],
      "metadata": {
        "id": "QlTBqkPs3gkl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cm = confusion_matrix(df['y_true'], df['y_pred1'])\n",
        "disp = ConfusionMatrixDisplay(confusion_matrix=cm)\n",
        "disp.plot()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 484
        },
        "id": "H_eYKS703hf9",
        "outputId": "6a03130e-bc6b-449f-b8e4-f0d8bad176bc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7dafcfa4f4c0>"
            ]
          },
          "metadata": {},
          "execution_count": 35
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 2 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAGwCAYAAAA0bWYRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABKCElEQVR4nO3de1xUdfoH8M/MwAwgzCAoMxBIKKWgqCuWzbaaJoHKlq72K8uU8tJq4CbmdTNFLWl1zTRNKytsV1O76CaUihioiaYoaV4oiAJDwDIYQWFg5vz+II5OOsoww0XO572v81rnnO8584zrMg/P8z3fIxMEQQARERFJlrylAyAiIqKWxWSAiIhI4pgMEBERSRyTASIiIoljMkBERCRxTAaIiIgkjskAERGRxDm1dAD2MJvNKCoqgoeHB2QyWUuHQ0RENhIEAZcuXYKfnx/k8qb7/bSqqgpGo9Hu6yiVSri4uDggotbltk4GioqKEBAQ0NJhEBGRnQoLC+Hv798k166qqkJQoDuKS012X0un0yE/P7/NJQS3dTLg4eEBAPjp2J1Qu7PjQW3T3+4Oa+kQiJpMLWpwAJ+LP8+bgtFoRHGpCT9l3Qm1R+O/KwyXzAgM/xFGo5HJQGtS3xpQu8vt+h+YqDVzkjm3dAhETef3BfGbo9Xr7iGDu0fj38eMttuOvq2TASIiooYyCWaY7Hgaj0kwOy6YVobJABERSYIZAsxofDZgz7mtHWvrREREEsdkgIiIJMHsgP801quvvgqZTIZp06aJ+wYOHAiZTGaxTZ482eK8goICREdHw83NDT4+Ppg5cyZqa2stxqSnp6NPnz5QqVQIDg5GUlKSzfGxTUBERJJgEgSYhMaX+ht77pEjR/DWW2+hZ8+e1x2bNGkSFi1aJL52c3O7+n4mE6Kjo6HT6XDw4EGcP38e48aNg7OzM5YsWQIAyM/PR3R0NCZPnoyNGzciLS0NEydOhK+vL6KiohocIysDRERETaSiogJjxozBO++8g/bt21933M3NDTqdTtzUarV4bPfu3Th9+jT++9//onfv3hg6dCgWL16MNWvWiAsorVu3DkFBQVi+fDlCQkIQFxeHRx99FCtWrLApTiYDREQkCfUTCO3ZAMBgMFhs1dXVVt8zNjYW0dHRiIiIuOHxjRs3okOHDujRowfmzp2Ly5cvi8cyMzMRFhYGrVYr7ouKioLBYMCpU6fEMX+8dlRUFDIzM236u2GbgIiIJMEMASYH3E3wx5VvFyxYgISEhOvGb968GceOHcORI0dueL0nn3wSgYGB8PPzw4kTJzB79mzk5OTg008/BQAUFxdbJAIAxNfFxcU3HWMwGHDlyhW4uro26LMxGSAiIrJBYWGhRTlfpVLdcMzzzz+P1NRUq6sVPvvss+Kfw8LC4Ovri8GDByMvLw9dunRxfOA3wTYBERFJgqPaBGq12mK7UTKQlZWF0tJS9OnTB05OTnByckJGRgZWrVoFJycnmEzXPyehX79+AIDc3FwAdc9BKCkpsRhT/1qn0910jFqtbnBVAGAyQEREElF/N4E9W0MNHjwYJ0+eRHZ2trj17dsXY8aMQXZ2NhQKxXXnZGdnAwB8fX0BAHq9HidPnkRpaak4JjU1FWq1GqGhoeKYtLQ0i+ukpqZCr9fb9HfDNgEREZGDeXh4oEePHhb72rVrB29vb/To0QN5eXnYtGkThg0bBm9vb5w4cQLx8fEYMGCAeAtiZGQkQkNDMXbsWCxduhTFxcWYN28eYmNjxWrE5MmTsXr1asyaNQvjx4/H3r17sXXrVqSkpNgULysDREQkCWYHbI6iVCqxZ88eREZGolu3bnjhhRcwatQo7NixQxyjUCiQnJwMhUIBvV6Pp556CuPGjbNYlyAoKAgpKSlITU1Fr169sHz5cqxfv96mNQYAQCYIdqzA0MIMBgM0Gg1++64zn1pIbVaUX++WDoGoydQKNUjH/1BeXm4xKc+R6r8rTp3xgYcd3xWXLpnRPaS0SWNtKWwTEBGRJJgE2PnUQsfF0trw12kiIiKJY2WAiIgkwd6+vyPnDLQ2TAaIiEgSzJDBBJld57dVbBMQERFJHCsDREQkCWahbrPn/LaKyQAREUmCyc42gT3ntnZsExAREUkcKwNERCQJrAxYx2SAiIgkwSzIYBbsuJvAjnNbO7YJiIiIJI6VASIikgS2CaxjMkBERJJgghwmOwriJgfG0towGSAiIkkQ7JwzIHDOABEREbVVrAwQEZEkcM6AdUwGiIhIEkyCHCbBjjkDbXg5YrYJiIiIJI6VASIikgQzZDDb8TuwGW23NMBkgIiIJIFzBqxjm4CIiEjiWBkgIiJJsH8CIdsEREREt7W6OQN2PKiIbQIiIiJqq1gZICIiSTDb+WwC3k1ARER0m+OcAeuYDBARkSSYIec6A1ZwzgAREZHEsTJARESSYBJkMNnxGGJ7zm3tmAwQEZEkmOycQGhim4CIiIjaKlYGiIhIEsyCHGY77iYwt+G7CVgZICIiSahvE9izNdarr74KmUyGadOmifuqqqoQGxsLb29vuLu7Y9SoUSgpKbE4r6CgANHR0XBzc4OPjw9mzpyJ2tpaizHp6eno06cPVCoVgoODkZSUZHN8TAaIiIia0JEjR/DWW2+hZ8+eFvvj4+OxY8cOfPTRR8jIyEBRURFGjhwpHjeZTIiOjobRaMTBgwexYcMGJCUlYf78+eKY/Px8REdHY9CgQcjOzsa0adMwceJE7Nq1y6YYmQwQEZEkmHH1joLGbOZGvGdFRQXGjBmDd955B+3btxf3l5eX491338Vrr72GBx98EOHh4Xj//fdx8OBBHDp0CACwe/dunD59Gv/973/Ru3dvDB06FIsXL8aaNWtgNBoBAOvWrUNQUBCWL1+OkJAQxMXF4dFHH8WKFStsipPJABERSUL9okP2bABgMBgsturqaqvvGRsbi+joaERERFjsz8rKQk1NjcX+bt26oVOnTsjMzAQAZGZmIiwsDFqtVhwTFRUFg8GAU6dOiWP+eO2oqCjxGg3FZICIiMgGAQEB0Gg04paYmHjDcZs3b8axY8dueLy4uBhKpRKenp4W+7VaLYqLi8Ux1yYC9cfrj91sjMFgwJUrVxr8mXg3ARERSYL9zyaoO7ewsBBqtVrcr1KprhtbWFiI559/HqmpqXBxcWn0ezYXVgaIiEgSzJDZvQGAWq222G6UDGRlZaG0tBR9+vSBk5MTnJyckJGRgVWrVsHJyQlarRZGoxFlZWUW55WUlECn0wEAdDrddXcX1L++1Ri1Wg1XV9cG/90wGSAiIkmorwzYszXU4MGDcfLkSWRnZ4tb3759MWbMGPHPzs7OSEtLE8/JyclBQUEB9Ho9AECv1+PkyZMoLS0Vx6SmpkKtViM0NFQcc+016sfUX6Oh2CYgIiJyMA8PD/To0cNiX7t27eDt7S3unzBhAqZPnw4vLy+o1WpMnToVer0e9913HwAgMjISoaGhGDt2LJYuXYri4mLMmzcPsbGxYjVi8uTJWL16NWbNmoXx48dj79692Lp1K1JSUmyKl8kAERFJgv3PJnBsMX3FihWQy+UYNWoUqqurERUVhTfffFM8rlAokJycjClTpkCv16Ndu3aIiYnBokWLxDFBQUFISUlBfHw8Vq5cCX9/f6xfvx5RUVE2xSIThNt3fUWDwQCNRoPfvusMtQc7HtQ2Rfn1bukQiJpMrVCDdPwP5eXlFpPyHKn+u2Lpkf5wdW/878BXKmox6579TRprS+E3KBERkcSxTUBERJJgtrNNYG7Dvz8zGSAiIkmw/6mFbTcZaLufjIiIiBqElQEiIpIEE2Qw/b5wUGPPb6uYDBARkSSwTWBd2/1kRERE1CCsDBARkSSYYF+p3+S4UFodJgNERCQJbBNYx2SAiIgkwVGPMG6L2u4nIyIiogZhZYCIiCRBgAxmO+YMCLy1kIiI6PbGNoF1bfeTERERUYOwMkBERJJgFmQwC40v9dtzbmvHZICIiCTBZOdTC+05t7Vru5+MiIiIGoSVASIikgS2CaxjMkBERJJghhxmOwri9pzb2rXdT0ZEREQNwsoAERFJgkmQwWRHqd+ec1s7JgNERCQJnDNgHZMBIiKSBMHOpxYKXIGQiIiI2ipWBoiISBJMkMFkx8OG7Dm3tWMyQEREkmAW7Ov7mwUHBtPKsE1AREQkcawMSNyWN3zwXqIfRky8gCmLfhb3nz7qhqR/+eLsMTcoFEDn7lewZFMeVK51qfGmlVp8vUeNH065wkkp4NOzJ6+7dpRf7+v2zX3zRwwcUdZUH4foOn8d9wuix/0KbYARAPBTjgs2rtDi6JdqAICzyoxnFxRh4CNlcFYJyEr3wBtz70DZL87iNTreYcTUxHPodX8FqioVSP2oPd5b4guzqe2Wjdsis50TCO05t7VjMiBhOdmuSPmvN4JCr1jsP33UDS+O6YLRcSV47uWfoVAI+OG0K2TX/P+g1ijDgIfLENK3Ers+9Lb6Hi+sKEDfQQbxtbva5PDPQXQzF847470lvvg5XwWZDHjo/y4i4f0fERt5N376zgWTE4pwb4QBL/89EJUGBWJf+Rnz3/0R04ffBQCQywUs/iAfv11wQvwjd8HLpwYzVxXAVCPD+6/6tvCnI1uYIYPZjr6/Pee2dq0izVmzZg3uvPNOuLi4oF+/fvj6669bOqQ270qlHP+KC8S0ZYXw0Fh+Qb+VcAdGTLiAx6eW4s6uVQgIrsYDj5RBqbraMBs3sxgjn72AoG5VN30fd7UJXj614qZ0acNNN2qVDqdqcGSvGkX5Kvz8gwpJ//JFVaUc3cIr4eZhQtQTF/FWgh+++coDuSfd8Nr0AHS/5zK69akEAPR54BI63V2Ff8V1wg+nXHH0SzU+WKrDw0//Aidncwt/OiLHaPFkYMuWLZg+fToWLFiAY8eOoVevXoiKikJpaWlLh9amrf6nP+4dbECfARUW+8t+ccLZY+3g6V2LaQ/fhcd7dseMkcH49nC7xr3Pi3fg/7r3wNRhd2HXh14QmAtQC5LLBTww/Deo3Mw4c7Qd7up5Gc5KAcf3e4hjCnNdUHLOGSHhlwEAoX0v48ezLhZtg6PpHminNiOw682TYWpd6lcgtGdrq1q8TfDaa69h0qRJeOaZZwAA69atQ0pKCt577z3MmTOnhaNrm9K3eyL3pCve+Py7646d/0kJAPjPazpMeqkIXbpfwZ6P22PO413w1t6zuKOzscHvM27mefS+vwIqVzOyMjzwxj/9caVSjhETf3HYZyFqiDu7XcHrO3KhVJlxpVKORRPuRMH3LujS4wqM1TJUGhQW48suOMHLpwYA0L5jDX67YPmjsj4xaN+xtnk+ADkE5wxY16KfzGg0IisrCxEREeI+uVyOiIgIZGZmXje+uroaBoPBYiPblP7sjLXz78Ds1T/dsGRv/r3qOeypXxE1+iKCw65g8sIi+Hepxq7N1ucG3MiY+BJ0v7cSwWFX8HhcKf5vSik+WuvjiI9BZJNzeSo899Dd+Ef0XUj+oANmrCxAp7v4Wz01rbVr16Jnz55Qq9VQq9XQ6/X44osvxOMDBw6ETCaz2CZPnmxxjYKCAkRHR8PNzQ0+Pj6YOXMmamstk9D09HT06dMHKpUKwcHBSEpKsjnWFq0M/PLLLzCZTNBqtRb7tVotzp49e934xMRELFy4sLnCa5NyT7ih7BdnxEZ1FfeZTTKcPNQOn73fAe/uPwMACLzb8gdlQHAVSn92hj269bmMTa/rYKyWWcw/IGpqtTVyFP2oAgDknnRD196XMWLiBWR85gmlSkA7tcmiOuDZsRYXS+v+vf92wRld/3TZ4nqeHWp+P9bixVWygRl2PpvAxgmE/v7+ePXVV3HXXXdBEARs2LABw4cPx/Hjx9G9e3cAwKRJk7Bo0SLxHDc3N/HPJpMJ0dHR0Ol0OHjwIM6fP49x48bB2dkZS5YsAQDk5+cjOjoakydPxsaNG5GWloaJEyfC19cXUVFRDY71tvqXPHfuXEyfPl18bTAYEBAQ0IIR3X5697+Et/ZaJlrL4zshILgKj8WWwjfQCG+dEefyVBZjfv5Bhb4PXrLrvfNOucLds5aJALU4mQxwVgr4/oQbaowy/Okvl3Dgc08AgH+XKmj9a3Amq+6H8umjbhj9jxJovGtQ/mtdgtBnQAUqDXIUfOfSUh+BGkGw824CwcZzH374YYvXr7zyCtauXYtDhw6JyYCbmxt0Ot0Nz9+9ezdOnz6NPXv2QKvVonfv3li8eDFmz56NhIQEKJVKrFu3DkFBQVi+fDkAICQkBAcOHMCKFStsSgZatE3QoUMHKBQKlJSUWOwvKSm54V+OSqUSyy31G9nGzd2MO7tVWWwubmZ4tDfhzm5VkMmAR6dcwPZ3O2J/sgY/5yuxYakOhXkuGPLEr+J1Ss85I+9bV5T+7AyzCcj71hV537riSmXdP6lDu9X4YqMXfjzrgp/zldixwRubV/lg+DOcL0DN65m559GjXwW0/kbc2e0Knpl7Hj3/XIEvt7XH5UsK7PrQC88mFKHXnysQHHYZL6woxOmjbjh7rG7S7LEMDxR854JZbxSgc+gVhD9gwNOzi7EjqQNqjG23h9wW1T+10J4NwHXt6urq6lu+t8lkwubNm1FZWQm9Xi/u37hxIzp06IAePXpg7ty5uHz5ahUqMzMTYWFhFtXzqKgoGAwGnDp1Shxzbau9fsyNWu0306KVAaVSifDwcKSlpWHEiBEAALPZjLS0NMTFxbVkaJI2ctIF1FTJsG7BHbhUpkDn0CokfpgHvzuvTh784N++SN3qJb5+LrKu7bD041z0+nMFFM4CdiR1wFsJKggC4HenEX9PKMLQMb9e935ETcmzQy1mriqAl08tLl9SIP+MC158sjOO7au7g2Bdgh/MAvDSOz/CWSXgaLoHVs+9QzzfbJZh/rggTH31HFbs+B5Vl+XY85EXNiy78W9z1Pb9sSK9YMECJCQk3HDsyZMnodfrUVVVBXd3d2zbtg2hoaEAgCeffBKBgYHw8/PDiRMnMHv2bOTk5ODTTz8FABQXF9+wjV5/7GZjDAYDrly5AldX1wZ9phZvE0yfPh0xMTHo27cv7r33Xrz++uuorKwU7y6gprfsk9zr9j0+tRSPT7V+e+eM1wsw4/UCq8fvGXQJ9wyyr61A5AgrXrh5K7GmWo41//THmn/6Wx1T+rMSL43t7OjQqJk56m6CwsJCi8q0SqWydgq6du2K7OxslJeX4+OPP0ZMTAwyMjIQGhqKZ599VhwXFhYGX19fDB48GHl5eejSpUuj42yMFk8GHn/8cVy4cAHz589HcXExevfujZ07d16X6RAREdnj2lJ/Y88HYFObWqlUIjg4GAAQHh6OI0eOYOXKlXjrrbeuG9uvXz8AQG5uLrp06QKdTnfdInz1bfX6VrpOp7thq12tVje4KgC0gkWHACAuLg4//fQTqqurcfjwYfEvhIiIqC0xm81W5xhkZ2cDAHx965a51uv1OHnypMUifKmpqVCr1WKrQa/XIy0tzeI6qampFvMSGqLFKwNERETNobmfTTB37lwMHToUnTp1wqVLl7Bp0yakp6dj165dyMvLw6ZNmzBs2DB4e3vjxIkTiI+Px4ABA9CzZ08AQGRkJEJDQzF27FgsXboUxcXFmDdvHmJjY8XWxOTJk7F69WrMmjUL48ePx969e7F161akpKTYFCuTASIikgRHtQkaqrS0FOPGjcP58+eh0WjQs2dP7Nq1Cw899BAKCwuxZ88ecZ5cQEAARo0ahXnz5onnKxQKJCcnY8qUKdDr9WjXrh1iYmIs1iUICgpCSkoK4uPjsXLlSvj7+2P9+vU23VYIMBkgIiJqEu+++67VYwEBAcjIyLjlNQIDA/H555/fdMzAgQNx/Phxm+O7FpMBIiKShOauDNxOmAwQEZEkMBmwrlXcTUBEREQth5UBIiKSBFYGrGMyQEREkiDA9tsD/3h+W8VkgIiIJIGVAes4Z4CIiEjiWBkgIiJJYGXAOiYDREQkCUwGrGObgIiISOJYGSAiIklgZcA6JgNERCQJgiCDYMcXuj3ntnZsExAREUkcKwNERCQJZsjsWnTInnNbOyYDREQkCZwzYB3bBERERBLHygAREUkCJxBax2SAiIgkgW0C65gMEBGRJLAyYB3nDBAREUkcKwNERCQJgp1tgrZcGWAyQEREkiAAEAT7zm+r2CYgIiKSOFYGiIhIEsyQQcYVCG+IyQAREUkC7yawjm0CIiIiiWNlgIiIJMEsyCDjokM3xGSAiIgkQRDsvJugDd9OwDYBERGRxLEyQEREksAJhNYxGSAiIklgMmAdkwEiIpIETiC0jnMGiIiImsDatWvRs2dPqNVqqNVq6PV6fPHFF+LxqqoqxMbGwtvbG+7u7hg1ahRKSkosrlFQUIDo6Gi4ubnBx8cHM2fORG1trcWY9PR09OnTByqVCsHBwUhKSrI5ViYDREQkCfV3E9iz2cLf3x+vvvoqsrKycPToUTz44IMYPnw4Tp06BQCIj4/Hjh078NFHHyEjIwNFRUUYOXKkeL7JZEJ0dDSMRiMOHjyIDRs2ICkpCfPnzxfH5OfnIzo6GoMGDUJ2djamTZuGiRMnYteuXTbFKhOE2/dmCYPBAI1Gg9++6wy1B/Maapui/Hq3dAhETaZWqEE6/ofy8nKo1eomeY/674q7/jsHCjeXRl/HdLkK3z/1KgoLCy1iValUUKlUDbqGl5cXli1bhkcffRQdO3bEpk2b8OijjwIAzp49i5CQEGRmZuK+++7DF198gb/+9a8oKiqCVqsFAKxbtw6zZ8/GhQsXoFQqMXv2bKSkpODbb78V32P06NEoKyvDzp07G/zZ+A1KRERkg4CAAGg0GnFLTEy85TkmkwmbN29GZWUl9Ho9srKyUFNTg4iICHFMt27d0KlTJ2RmZgIAMjMzERYWJiYCABAVFQWDwSBWFzIzMy2uUT+m/hoNxQmEREQkCY66m+BGlQFrTp48Cb1ej6qqKri7u2Pbtm0IDQ1FdnY2lEolPD09LcZrtVoUFxcDAIqLiy0Sgfrj9cduNsZgMODKlStwdXVt0GdjMkBERJIg/L7Zcz4AcUJgQ3Tt2hXZ2dkoLy/Hxx9/jJiYGGRkZNgRRdNgMkBERNRElEolgoODAQDh4eE4cuQIVq5ciccffxxGoxFlZWUW1YGSkhLodDoAgE6nw9dff21xvfq7Da4d88c7EEpKSqBWqxtcFQA4Z4CIiCSivk1gz2Yvs9mM6upqhIeHw9nZGWlpaeKxnJwcFBQUQK/XAwD0ej1OnjyJ0tJScUxqairUajVCQ0PFMddeo35M/TUaipUBIiKSBkf1CRpo7ty5GDp0KDp16oRLly5h06ZNSE9Px65du6DRaDBhwgRMnz4dXl5eUKvVmDp1KvR6Pe677z4AQGRkJEJDQzF27FgsXboUxcXFmDdvHmJjY8V5CpMnT8bq1asxa9YsjB8/Hnv37sXWrVuRkpJiU6xMBoiISBrs/e3exnNLS0sxbtw4nD9/HhqNBj179sSuXbvw0EMPAQBWrFgBuVyOUaNGobq6GlFRUXjzzTfF8xUKBZKTkzFlyhTo9Xq0a9cOMTExWLRokTgmKCgIKSkpiI+Px8qVK+Hv74/169cjKirKpli5zgBRK8d1Bqgta851BjonvQi5HesMmC9X4YenX2nSWFsKKwNERCQJjVlF8I/nt1VMBoiISBL41ELrWFsnIiKSOFYGiIhIGgSZzZMArzu/jWIyQEREksA5A9axTUBERCRxrAwQEZE0NPOiQ7cTJgNERCQJvJvAugYlA5999lmDL/jII480OhgiIiJqfg1KBkaMGNGgi8lkMphMJnviISIiajptuNRvjwYlA2azuanjICIialJsE1hn190EVVVVjoqDiIioaQkO2Noom5MBk8mExYsX44477oC7uzt++OEHAMBLL72Ed9991+EBEhERUdOyORl45ZVXkJSUhKVLl0KpVIr7e/TogfXr1zs0OCIiIseROWBrm2xOBj744AO8/fbbGDNmDBQKhbi/V69eOHv2rEODIyIichi2CayyORn4+eefERwcfN1+s9mMmpoahwRFREREzcfmZCA0NBT79++/bv/HH3+MP/3pTw4JioiIyOFYGbDK5hUI58+fj5iYGPz8888wm8349NNPkZOTgw8++ADJyclNESMREZH9+NRCq2yuDAwfPhw7duzAnj170K5dO8yfPx9nzpzBjh078NBDDzVFjERERNSEGvVsgv79+yM1NdXRsRARETUZPsLYukY/qOjo0aM4c+YMgLp5BOHh4Q4LioiIyOH41EKrbE4Gzp07hyeeeAJfffUVPD09AQBlZWX485//jM2bN8Pf39/RMRIREVETsnnOwMSJE1FTU4MzZ87g4sWLuHjxIs6cOQOz2YyJEyc2RYxERET2q59AaM/WRtlcGcjIyMDBgwfRtWtXcV/Xrl3xxhtvoH///g4NjoiIyFFkQt1mz/ltlc3JQEBAwA0XFzKZTPDz83NIUERERA7HOQNW2dwmWLZsGaZOnYqjR4+K+44ePYrnn38e//73vx0aHBERETW9BlUG2rdvD5nsaq+ksrIS/fr1g5NT3em1tbVwcnLC+PHjMWLEiCYJlIiIyC5cdMiqBiUDr7/+ehOHQURE1MTYJrCqQclATExMU8dBRERELaTRiw4BQFVVFYxGo8U+tVptV0BERERNgpUBq2yeQFhZWYm4uDj4+PigXbt2aN++vcVGRETUKvGphVbZnAzMmjULe/fuxdq1a6FSqbB+/XosXLgQfn5++OCDD5oiRiIiImpCNicDO3bswJtvvolRo0bByckJ/fv3x7x587BkyRJs3LixKWIkIiKyXzOvQJiYmIh77rkHHh4e8PHxwYgRI5CTk2MxZuDAgZDJZBbb5MmTLcYUFBQgOjoabm5u8PHxwcyZM1FbW2sxJj09HX369IFKpUJwcDCSkpJsitXmZODixYvo3LkzgLr5ARcvXgQA/OUvf8G+fftsvRwREVGzqF+B0J7NFhkZGYiNjcWhQ4eQmpqKmpoaREZGorKy0mLcpEmTcP78eXFbunSpeMxkMiE6OhpGoxEHDx7Ehg0bkJSUhPnz54tj8vPzER0djUGDBiE7OxvTpk3DxIkTsWvXrgbHavMEws6dOyM/Px+dOnVCt27dsHXrVtx7773YsWOH+OAiIiKitspgMFi8VqlUUKlU143buXOnxeukpCT4+PggKysLAwYMEPe7ublBp9Pd8L12796N06dPY8+ePdBqtejduzcWL16M2bNnIyEhAUqlEuvWrUNQUBCWL18OAAgJCcGBAwewYsUKREVFNegz2VwZeOaZZ/DNN98AAObMmYM1a9bAxcUF8fHxmDlzpq2XIyIiah4OmkAYEBAAjUYjbomJiQ16+/LycgCAl5eXxf6NGzeiQ4cO6NGjB+bOnYvLly+LxzIzMxEWFgatVivui4qKgsFgwKlTp8QxERERFteMiopCZmZmg+ICGlEZiI+PF/8cERGBs2fPIisrC8HBwejZs6etlyMiIrqtFBYWWtxGf6OqwB+ZzWZMmzYN999/P3r06CHuf/LJJxEYGAg/Pz+cOHECs2fPRk5ODj799FMAQHFxsUUiAEB8XVxcfNMxBoMBV65cgaur6y3js2udAQAIDAxEYGCgvZchIiJqUjLY+dTC3/9brVbbvKZObGwsvv32Wxw4cMBi/7PPPiv+OSwsDL6+vhg8eDDy8vLQpUuXxgdrowYlA6tWrWrwBf/xj380OhgiIqK2Ji4uDsnJydi3bx/8/f1vOrZfv34AgNzcXHTp0gU6nQ5ff/21xZiSkhIAEOcZ6HQ6cd+1Y9RqdYOqAkADk4EVK1Y06GIymaxFkoGIf46Hk7NLs78vUXPofvhkS4dA1GSMFUbgwWZ6s2Z+UJEgCJg6dSq2bduG9PR0BAUF3fKc7OxsAICvry8AQK/X45VXXkFpaSl8fHwAAKmpqVCr1QgNDRXHfP755xbXSU1NhV6vb3CsDUoG8vPzG3xBIiKiVqmZlyOOjY3Fpk2b8L///Q8eHh5ij1+j0cDV1RV5eXnYtGkThg0bBm9vb5w4cQLx8fEYMGCAOAcvMjISoaGhGDt2LJYuXYri4mLMmzcPsbGx4lyFyZMnY/Xq1Zg1axbGjx+PvXv3YuvWrUhJSWlwrDbfTUBERES3tnbtWpSXl2PgwIHw9fUVty1btgAAlEol9uzZg8jISHTr1g0vvPACRo0ahR07dojXUCgUSE5OhkKhgF6vx1NPPYVx48Zh0aJF4pigoCCkpKQgNTUVvXr1wvLly7F+/foG31YIOGACIRER0W2hmSsDgnDzEwICApCRkXHL6wQGBl7XBvijgQMH4vjx4zbFdy0mA0REJAmNWUXwj+e3VWwTEBERSRwrA0REJA3N3Ca4nTSqMrB//3489dRT0Ov1+PnnnwEA//nPf65bTIGIiKjVcNByxG2RzcnAJ598gqioKLi6uuL48eOorq4GULfm8pIlSxweIBERETUtm5OBl19+GevWrcM777wDZ2dncf/999+PY8eOOTQ4IiIiR2nuRxjfTmyeM5CTk2Px6MV6Go0GZWVljoiJiIjI8Zp5BcLbic2VAZ1Oh9zc3Ov2HzhwAJ07d3ZIUERERA7HOQNW2ZwMTJo0Cc8//zwOHz4MmUyGoqIibNy4ETNmzMCUKVOaIkYiIiJqQja3CebMmQOz2YzBgwfj8uXLGDBgAFQqFWbMmIGpU6c2RYxERER246JD1tmcDMhkMrz44ouYOXMmcnNzUVFRgdDQULi7uzdFfERERI7BdQasavSiQ0qlUnx8IhEREd2+bE4GBg0aBJnM+ozKvXv32hUQERFRk7D39kBWBq7q3bu3xeuamhpkZ2fj22+/RUxMjKPiIiIiciy2CayyORlYsWLFDfcnJCSgoqLC7oCIiIioeTnsqYVPPfUU3nvvPUddjoiIyLG4zoBVDntqYWZmJlxcXBx1OSIiIofirYXW2ZwMjBw50uK1IAg4f/48jh49ipdeeslhgREREVHzsDkZ0Gg0Fq/lcjm6du2KRYsWITIy0mGBERERUfOwKRkwmUx45plnEBYWhvbt2zdVTERERI7HuwmssmkCoUKhQGRkJJ9OSEREtx0+wtg6m+8m6NGjB3744YemiIWIiIhagM3JwMsvv4wZM2YgOTkZ58+fh8FgsNiIiIhaLd5WeEMNnjOwaNEivPDCCxg2bBgA4JFHHrFYllgQBMhkMphMJsdHSUREZC/OGbCqwcnAwoULMXnyZHz55ZdNGQ8RERE1swYnA4JQlxI98MADTRYMERFRU+GiQ9bZdGvhzZ5WSERE1KqxTWCVTcnA3XfffcuE4OLFi3YFRERERM3LpmRg4cKF161ASEREdDtgm8A6m5KB0aNHw8fHp6liISIiajpsE1jV4HUGOF+AiIiobWpwMlB/NwEREdFtyZ4FhxpRVUhMTMQ999wDDw8P+Pj4YMSIEcjJybEYU1VVhdjYWHh7e8Pd3R2jRo1CSUmJxZiCggJER0fDzc0NPj4+mDlzJmpray3GpKeno0+fPlCpVAgODkZSUpJNsTY4GTCbzWwREBHRbau5n02QkZGB2NhYHDp0CKmpqaipqUFkZCQqKyvFMfHx8dixYwc++ugjZGRkoKioCCNHjhSPm0wmREdHw2g04uDBg9iwYQOSkpIwf/58cUx+fj6io6MxaNAgZGdnY9q0aZg4cSJ27drV4FhtfoQxERHRbamZ5wzs3LnT4nVSUhJ8fHyQlZWFAQMGoLy8HO+++y42bdqEBx98EADw/vvvIyQkBIcOHcJ9992H3bt34/Tp09izZw+0Wi169+6NxYsXY/bs2UhISIBSqcS6desQFBSE5cuXAwBCQkJw4MABrFixAlFRUQ2K1eZnExAREUnZH5/JU11d3aDzysvLAQBeXl4AgKysLNTU1CAiIkIc061bN3Tq1AmZmZkAgMzMTISFhUGr1YpjoqKiYDAYcOrUKXHMtdeoH1N/jYZgMkBERNLgoDkDAQEB0Gg04paYmHjLtzabzZg2bRruv/9+9OjRAwBQXFwMpVIJT09Pi7FarRbFxcXimGsTgfrj9cduNsZgMODKlSu3jA1gm4CIiCTCUesMFBYWQq1Wi/tVKtUtz42NjcW3336LAwcOND6AJsTKABERkQ3UarXFdqtkIC4uDsnJyfjyyy/h7+8v7tfpdDAajSgrK7MYX1JSAp1OJ475490F9a9vNUatVsPV1bVBn4nJABERSUMz31ooCALi4uKwbds27N27F0FBQRbHw8PD4ezsjLS0NHFfTk4OCgoKoNfrAQB6vR4nT55EaWmpOCY1NRVqtRqhoaHimGuvUT+m/hoNwTYBERFJQnMvRxwbG4tNmzbhf//7Hzw8PMQev0ajgaurKzQaDSZMmIDp06fDy8sLarUaU6dOhV6vx3333QcAiIyMRGhoKMaOHYulS5eiuLgY8+bNQ2xsrFiRmDx5MlavXo1Zs2Zh/Pjx2Lt3L7Zu3YqUlJQGx8rKABERURNYu3YtysvLMXDgQPj6+orbli1bxDErVqzAX//6V4waNQoDBgyATqfDp59+Kh5XKBRITk6GQqGAXq/HU089hXHjxmHRokXimKCgIKSkpCA1NRW9evXC8uXLsX79+gbfVgiwMkBERFLRzOsMNGTlXhcXF6xZswZr1qyxOiYwMBCff/75Ta8zcOBAHD9+3LYAr8FkgIiIpIEPKrKKbQIiIiKJY2WAiIgkQfb7Zs/5bRWTASIikga2CaxiMkBERJLQ3LcW3k44Z4CIiEjiWBkgIiJpYJvAKiYDREQkHW34C90ebBMQERFJHCsDREQkCZxAaB2TASIikgbOGbCKbQIiIiKJY2WAiIgkgW0C65gMEBGRNLBNYBXbBERERBLHygAREUkC2wTWMRkgIiJpYJvAKiYDREQkDUwGrOKcASIiIoljZYCIiCSBcwasYzJARETSwDaBVWwTEBERSRwrA0REJAkyQYBMaPyv9/ac29oxGSAiImlgm8AqtgmIiIgkjpUBIiKSBN5NYB2TASIikga2Caxim4CIiEjiWBkgIiJJYJvAOiYDREQkDWwTWMVkgIiIJIGVAes4Z4CIiEjimAwQEZE0CA7YbLBv3z48/PDD8PPzg0wmw/bt2y2OP/3005DJZBbbkCFDLMZcvHgRY8aMgVqthqenJyZMmICKigqLMSdOnED//v3h4uKCgIAALF261LZAwWSAiIgkpL5V0JjNVpWVlejVqxfWrFljdcyQIUNw/vx5cfvwww8tjo8ZMwanTp1CamoqkpOTsW/fPjz77LPicYPBgMjISAQGBiIrKwvLli1DQkIC3n77bZti5ZwBIiKiJjB06FAMHTr0pmNUKhV0Ot0Nj505cwY7d+7EkSNH0LdvXwDAG2+8gWHDhuHf//43/Pz8sHHjRhiNRrz33ntQKpXo3r07srOz8dprr1kkDbfCygAREUmDINi/oe638Wu36urqRoeUnp4OHx8fdO3aFVOmTMGvv/4qHsvMzISnp6eYCABAREQE5HI5Dh8+LI4ZMGAAlEqlOCYqKgo5OTn47bffGhwHkwEiIpIEe1oE17YKAgICoNFoxC0xMbFR8QwZMgQffPAB0tLS8K9//QsZGRkYOnQoTCYTAKC4uBg+Pj4W5zg5OcHLywvFxcXiGK1WazGm/nX9mIZgm4CIiMgGhYWFUKvV4muVStWo64wePVr8c1hYGHr27IkuXbogPT0dgwcPtjtOW7AyQERE0uCguwnUarXF1thk4I86d+6MDh06IDc3FwCg0+lQWlpqMaa2thYXL14U5xnodDqUlJRYjKl/bW0uwo0wGSAiIkmQme3fmtK5c+fw66+/wtfXFwCg1+tRVlaGrKwscczevXthNpvRr18/ccy+fftQU1MjjklNTUXXrl3Rvn37Br83kwEiIqImUFFRgezsbGRnZwMA8vPzkZ2djYKCAlRUVGDmzJk4dOgQfvzxR6SlpWH48OEIDg5GVFQUACAkJARDhgzBpEmT8PXXX+Orr75CXFwcRo8eDT8/PwDAk08+CaVSiQkTJuDUqVPYsmULVq5cienTp9sUK+cMSFDvzkV4ctA36Or/CzpqLmPOe5HY922QePzF0V8i+t7vLM45dNYf09+OFl97uFVh+t++wl+6/wSzIEP6iSC8vu1+XDE6i2O6+P6KF0YdQEjABZRVuODjAz2w8cveTf75SNoufVKDik9rUFtUV9N17iyHZoIzXP9c9+OuZMoVVB+z/BXP/W9O8JpztdRbfdqEsjVGGM+aARmgClXAM84ZyrsVAIDaIjOK/nbluvfWrneBKkzRVB+N7NXMzyY4evQoBg0aJL6u/4KOiYnB2rVrceLECWzYsAFlZWXw8/NDZGQkFi9ebNF22LhxI+Li4jB48GDI5XKMGjUKq1atEo9rNBrs3r0bsbGxCA8PR4cOHTB//nybbisEmAxIkouyFrlF3kj+uhtefWb3DcdkngnAK5sHiq9rai1/wCWM2Qtv9WU8vy4aTgozXhydjtmP7UPCf+smvbipjHj97yk4+v0dWPZRf3TxvYh/js5AxRUl/ncotMk+G5HCRwbP55RwCqgrfFam1OLCzGro/iOHsnPdvnbDneD596uJq0wlE/9svizgwvNVcO3vBK9ZKggmoPxtI0qfr8YdO1whc7o61me1C5w7X30t11z9M7U+zf1sgoEDB0IQrJ+0a9euW17Dy8sLmzZtuumYnj17Yv/+/bYF9wct2ia41VKN1DQOne2Et7+4F/tOBlkdU1OrwMVLbuJ26crVTDXQ5zfoQwrx6pYHcLpAixP5vnht2/2I6J2LDupKAEBU+PdwdjLjlc0DkV/ihT3Zwfhofw+MfuBkk38+kja3/k5wvd8Jzp3kcO4kh+cUJeRugPFbkzhG7gIovOXiJne/+iVe85MZZgOg+bsznAPrEgjNRCXMFwXUnrf8wS7XWF7n2kSBWiEHrTPQFrVoMtCQpRqpZfwpuAgpCzfgwzmbMWPUfqjdqsRjPe4sgeGyEmfPdRT3Hf3OH2ZBhtDAupmvPQJLkJ3ni1rT1YrC4Rx/BGrL4OHa+AU6iGwhmARU7q6F+Qqg6nH132Llrlqci6zE+Scuo2yNEeaqqz/knTvJIdcAFZ/VQqgRYK4SUPlZDZzulMHJ1/LL/sKMapwbUomSSVdweV9ts30uIkdr0TZBQ5ZqvFZ1dbXFSk8Gg6EpwpK8w2cDkHEyCEUXPeDvbcDfh32N1579HM+uHAGzIIe3x2X8VuFqcY7JLMelyyp4e1wGAHirr6DooofFmIuX3AAAXh6XLSoNRI5mzDWjZOIVCEZA5gp0/JcKzvUtgkgnKHxlUHSQoybXjLLVRtQUmNHxXy4AAHk7GXzWuuKXWVUwvFc3Q9spQAaflS7ib/4yNxk8n1dC1VMOyIErX5rwy6xqdFgKuA1g97W14iOMrbut/tUmJiZi4cKFLR1Gm7cnO1j88w/nvZFb5I2P532IPwUXIet7/xaMjKhhnANl0P3HFUKFgMt7Tfh1UTW0a+Vw7iyH+9+uzhVQBsuh6CBDaWwVas6Z4ewvh7lKwMVXqqHqqYD3YifABBg21uDC9Cpo33eF3EUGhacM6ievXkcVqkDtBQGX/lvDZKA1a+YJhLeT2+rWwrlz56K8vFzcCgsLWzokSSi6qMZvFS7w71BXifn1khvau1vOpFbIzfBwq8avv//2/6vBFV5/GOP1e9WgvkJA1FRkzjI4B8ihDFHAM1YJ57vkuLSl5oZjld3rfgzWnqu7w+Dy7lrUFpnh9ZISqlAFVGEKdFisQm2RgCv7TDe8BgCoustRc64Nf1tQm3ZbJQMqleq6lZ+o6XXUVEDjVoVfDXVf4t/+qIXazYiu/hfEMeHBP0MuE3D6p7p1tL/9SYveXc5DIb/6w/Oeu8/hpxJPtgio+ZkB4ca5AGq+q0sCFN51Pw6FKgByGXDt9ADZ79tNvuuN35uh6MAJhK2Zo55N0BbdVskAOYarsgZ3+f2Cu/x+AQD4el3CXX6/QOt5Ca7KGsQ+nInugSXQtb+E8LvO4V/jd+HcLxocPhsAAPiptD0yzwRgzmP7ENKpFGF3FmP6yK+wJzsYvxjaAQB2HwtGTa0c/3w8A0HaixjcOxeP9f8WmzPCWuxzkzSUrTGi6rgJtUVmGHPNKFtjRPUxM9yinFBzzozyd40wnqk7fnlfLX5dWA3Vn+RQ3lX349DlXgXMlwT8tsyImnwzjD+Y8eviakABuITXjalIqUHlrlrU/GhGzY9mlCcZUbmjFh7/xxZBq8a7Caziv1wJ6hZwAWtid4ivnx+RCQBI+fpuLPukP4J9L2JY3+/g7mrELwY3fJ3jj7e/uAc119wZkLDxQbww8iusmpwM4fdFh1Zsu188XlmlwrS3ovHCqAN4b/qnKK90wfup4VxjgJqc6TcBvy6shukXAXJ3wDlYjo4rXeDaT4HaEjOqjphwaXMNzFWAk48MroOcoHnmav/f+U45Ov7bBYb1RhRPvAKZHHC+Ww6f112g6HD196fy94wwFQuAou6cDi+r4DaYP1Lp9tSi/3IrKirEBzIAV5dq9PLyQqdOnVowsrbteJ4f/jz971aPx1+z0qA1ly67iAsMWZN33hvPrR5uc3xE9vCeZ70N5aSVQ7vO1erxeq79FHDtZ32ce7Qz3KOdrR6n1ol3E1jXosnAzZZqTEpKaqGoiIioTeLdBFa1aDJwq6UaiYiIqOmxwUVERJLANoF1TAaIiEgazELdZs/5bRSTASIikgbOGbCK6wwQERFJHCsDREQkCTLYOWfAYZG0PkwGiIhIGuxdRbAN3/3GNgEREZHEsTJARESSwFsLrWMyQERE0sC7Caxim4CIiEjiWBkgIiJJkAkCZHZMArTn3NaOyQAREUmD+ffNnvPbKLYJiIiIJI6VASIikgS2CaxjMkBERNLAuwmsYjJARETSwBUIreKcASIiIoljZYCIiCSBKxBax2SAiIikgW0Cq9gmICIikjhWBoiISBJk5rrNnvPbKlYGiIhIGurbBPZsNti3bx8efvhh+Pn5QSaTYfv27X8IR8D8+fPh6+sLV1dXRERE4Pvvv7cYc/HiRYwZMwZqtRqenp6YMGECKioqLMacOHEC/fv3h4uLCwICArB06VKb/2qYDBARETWByspK9OrVC2vWrLnh8aVLl2LVqlVYt24dDh8+jHbt2iEqKgpVVVXimDFjxuDUqVNITU1FcnIy9u3bh2effVY8bjAYEBkZicDAQGRlZWHZsmVISEjA22+/bVOsbBMQEZE0NPOiQ0OHDsXQoUNvfClBwOuvv4558+Zh+PDhAIAPPvgAWq0W27dvx+jRo3HmzBns3LkTR44cQd++fQEAb7zxBoYNG4Z///vf8PPzw8aNG2E0GvHee+9BqVSie/fuyM7OxmuvvWaRNNwKKwNERCQJ9csR27MBdb+NX7tVV1fbHEt+fj6Ki4sREREh7tNoNOjXrx8yMzMBAJmZmfD09BQTAQCIiIiAXC7H4cOHxTEDBgyAUqkUx0RFRSEnJwe//fZbg+NhMkBERGSDgIAAaDQacUtMTLT5GsXFxQAArVZrsV+r1YrHiouL4ePjY3HcyckJXl5eFmNudI1r36Mh2CYgIiJpcNA6A4WFhVCr1eJulUplb2QtjpUBIiKSBgGA2Y7t9zxCrVZbbI1JBnQ6HQCgpKTEYn9JSYl4TKfTobS01OJ4bW0tLl68aDHmRte49j0agskAERFJgqPmDDhCUFAQdDod0tLSxH0GgwGHDx+GXq8HAOj1epSVlSErK0scs3fvXpjNZvTr108cs2/fPtTU1IhjUlNT0bVrV7Rv377B8TAZICIiagIVFRXIzs5GdnY2gLpJg9nZ2SgoKIBMJsO0adPw8ssv47PPPsPJkycxbtw4+Pn5YcSIEQCAkJAQDBkyBJMmTcLXX3+Nr776CnFxcRg9ejT8/PwAAE8++SSUSiUmTJiAU6dOYcuWLVi5ciWmT59uU6ycM0BERNIgwM45A7YNP3r0KAYNGiS+rv+CjomJQVJSEmbNmoXKyko8++yzKCsrw1/+8hfs3LkTLi4u4jkbN25EXFwcBg8eDLlcjlGjRmHVqlXicY1Gg927dyM2Nhbh4eHo0KED5s+fb9NthQAgE4Tb98kLBoMBGo0G4Y++DCdnl1ufQHQb6h5/sqVDIGoyxgoj/vvghygvL7eYlOdI9d8VD/aaDSdF4yf71ZqqsfebfzVprC2FbQIiIiKJY5uAiIikwQxAZuf5bRSTASIikgR77whw5N0ErQ3bBERERBLHygAREUmDg1YgbIuYDBARkTQwGbCKbQIiIiKJY2WAiIikgZUBq5gMEBGRNPDWQquYDBARkSTw1kLrOGeAiIhI4lgZICIiaeCcAauYDBARkTSYBUBmxxe6ue0mA2wTEBERSRwrA0REJA1sE1jFZICIiCTCzmQAbTcZYJuAiIhI4lgZICIiaWCbwComA0REJA1mAXaV+nk3AREREbVVrAwQEZE0COa6zZ7z2ygmA0REJA2cM2AVkwEiIpIGzhmwinMGiIiIJI6VASIikga2CaxiMkBERNIgwM5kwGGRtDpsExAREUkcKwNERCQNbBNYxWSAiIikwWwGYMdaAea2u84A2wREREQSx8oAERFJA9sEVjEZICIiaWAyYBXbBERERE0gISEBMpnMYuvWrZt4vKqqCrGxsfD29oa7uztGjRqFkpISi2sUFBQgOjoabm5u8PHxwcyZM1FbW+vwWFkZICIiaWiB5Yi7d++OPXv2iK+dnK5+7cbHxyMlJQUfffQRNBoN4uLiMHLkSHz11VcAAJPJhOjoaOh0Ohw8eBDnz5/HuHHj4OzsjCVLljT+c9wAkwEiIpIEQTBDsOPJg40518nJCTqd7rr95eXlePfdd7Fp0yY8+OCDAID3338fISEhOHToEO677z7s3r0bp0+fxp49e6DVatG7d28sXrwYs2fPRkJCApRKZaM/yx+xTUBERNIgCHW/3Td2+33OgMFgsNiqq6utvuX3338PPz8/dO7cGWPGjEFBQQEAICsrCzU1NYiIiBDHduvWDZ06dUJmZiYAIDMzE2FhYdBqteKYqKgoGAwGnDp1yqF/NUwGiIiIbBAQEACNRiNuiYmJNxzXr18/JCUlYefOnVi7di3y8/PRv39/XLp0CcXFxVAqlfD09LQ4R6vVori4GABQXFxskQjUH68/5khsExARkTQIds4Z+L0yUFhYCLVaLe5WqVQ3HD506FDxzz179kS/fv0QGBiIrVu3wtXVtfFxNAFWBoiISBrMZvs3AGq12mKzlgz8kaenJ+6++27k5uZCp9PBaDSirKzMYkxJSYk4x0Cn0113d0H96xvNQ7AHkwEiIqJmUFFRgby8PPj6+iI8PBzOzs5IS0sTj+fk5KCgoAB6vR4AoNfrcfLkSZSWlopjUlNToVarERoa6tDY2CYgIiJpcFCboKFmzJiBhx9+GIGBgSgqKsKCBQugUCjwxBNPQKPRYMKECZg+fTq8vLygVqsxdepU6PV63HfffQCAyMhIhIaGYuzYsVi6dCmKi4sxb948xMbGNrga0VBMBoiISBIEsxmCrPluLTx37hyeeOIJ/Prrr+jYsSP+8pe/4NChQ+jYsSMAYMWKFZDL5Rg1ahSqq6sRFRWFN998UzxfoVAgOTkZU6ZMgV6vR7t27RATE4NFixY1+jNYw2SAiIioCWzevPmmx11cXLBmzRqsWbPG6pjAwEB8/vnnjg7tOkwGiIhIGpq5TXA7YTJARETSYBYAGZOBG+HdBERERBLHygAREUmDIABo/ATCtlwZYDJARESSIJgFCHa0CQQmA0RERLc5wQz7KgN2nNvKcc4AERGRxLEyQEREksA2gXVMBoiISBrYJrDqtk4G6rM0U01VC0dC1HSMFcaWDoGoyRgrawA0z2/dtaixa82hWtQ4LphWRibcxnWPc+fOISAgoKXDICIiOxUWFsLf379Jrl1VVYWgoCAUFxfbfS2dTof8/Hy4uLg4ILLW47ZOBsxmM4qKiuDh4QGZTNbS4UiCwWBAQEAACgsLoVarWzocIofiv+/mJwgCLl26BD8/P8jlTTenvaqqCkaj/VU2pVLZ5hIB4DZvE8jl8ibLJOnm1Go1f1hSm8V/381Lo9E0+Xu4uLi0yS9xR+GthURERBLHZICIiEjimAyQTVQqFRYsWACVStXSoRA5HP99k1Td1hMIiYiIyH6sDBAREUkckwEiIiKJYzJAREQkcUwGiIiIJI7JADXYmjVrcOedd8LFxQX9+vXD119/3dIhETnEvn378PDDD8PPzw8ymQzbt29v6ZCImhWTAWqQLVu2YPr06ViwYAGOHTuGXr16ISoqCqWlpS0dGpHdKisr0atXL6xZs6alQyFqEby1kBqkX79+uOeee7B69WoAdc+FCAgIwNSpUzFnzpwWjo7IcWQyGbZt24YRI0a0dChEzYaVAbolo9GIrKwsREREiPvkcjkiIiKQmZnZgpEREZEjMBmgW/rll19gMpmg1Wot9mu1Woc8EpSIiFoWkwEiIiKJYzJAt9ShQwcoFAqUlJRY7C8pKYFOp2uhqIiIyFGYDNAtKZVKhIeHIy0tTdxnNpuRlpYGvV7fgpEREZEjOLV0AHR7mD59OmJiYtC3b1/ce++9eP3111FZWYlnnnmmpUMjsltFRQVyc3PF1/n5+cjOzoaXlxc6derUgpERNQ/eWkgNtnr1aixbtgzFxcXo3bs3Vq1ahX79+rV0WER2S09Px6BBg67bHxMTg6SkpOYPiKiZMRkgIiKSOM4ZICIikjgmA0RERBLHZICIiEjimAwQERFJHJMBIiIiiWMyQEREJHFMBoiIiCSOyQAREZHEMRkgstPTTz+NESNGiK8HDhyIadOmNXsc6enpkMlkKCsrszpGJpNh+/btDb5mQkICevfubVdcP/74I2QyGbKzs+26DhE1HSYD1CY9/fTTkMlkkMlkUCqVCA4OxqJFi1BbW9vk7/3pp59i8eLFDRrbkC9wIqKmxgcVUZs1ZMgQvP/++6iursbnn3+O2NhYODs7Y+7cudeNNRqNUCqVDnlfLy8vh1yHiKi5sDJAbZZKpYJOp0NgYCCmTJmCiIgIfPbZZwCulvZfeeUV+Pn5oWvXrgCAwsJCPPbYY/D09ISXlxeGDx+OH3/8UbymyWTC9OnT4enpCW9vb8yaNQt/fLzHH9sE1dXVmD17NgICAqBSqRAcHIx3330XP/74o/hwnPbt20Mmk+Hpp58GUPeI6MTERAQFBcHV1RW9evXCxx9/bPE+n3/+Oe6++264urpi0KBBFnE21OzZs3H33XfDzc0NnTt3xksvvYSamprrxr311lsICAiAm5sbHnvsMZSXl1scX79+PUJCQuDi4oJu3brhzTfftDkWImo5TAZIMlxdXWE0GsXXaWlpyMnJQWpqKpKTk1FTU4OoqCh4eHhg//79+Oqrr+Du7o4hQ4aI5y1fvhxJSUl47733cODAAVy8eBHbtm276fuOGzcOH374IVatWoUzZ87grbfegru7OwICAvDJJ58AAHJycnD+/HmsXLkSAJCYmIgPPvgA69atw6lTpxAfH4+nnnoKGRkZAOqSlpEjR+Lhhx9GdnY2Jk6ciDlz5tj8d+Lh4YGkpCScPn0aK1euxDvvvIMVK1ZYjMnNzcXWrVuxY8cO7Ny5E8ePH8dzzz0nHt+4cSPmz5+PV155BWfOnMGSJUvw0ksvYcOGDTbHQ0QtRCBqg2JiYoThw4cLgiAIZrNZSE1NFVQqlTBjxgzxuFarFaqrq8Vz/vOf/whdu3YVzGazuK+6ulpwdXUVdu3aJQiCIPj6+gpLly4Vj9fU1Aj+/v7iewmCIDzwwAPC888/LwiCIOTk5AgAhNTU1BvG+eWXXwoAhN9++03cV1VVJbi5uQkHDx60GDthwgThiSeeEARBEObOnSuEhoZaHJ89e/Z11/ojAMK2bdusHl+2bJkQHh4uvl6wYIGgUCiEc+fOifu++OILQS6XC+fPnxcEQRC6dOkibNq0yeI6ixcvFvR6vSAIgpCfny8AEI4fP271fYmoZXHOALVZycnJcHd3R01NDcxmM5588kkkJCSIx8PCwizmCXzzzTfIzc2Fh4eHxXWqqqqQl5eH8vJynD9/Hv369ROPOTk5oW/fvte1CuplZ2dDoVDggQceaHDcubm5uHz5Mh566CGL/UajEX/6058AAGfOnLGIAwD0en2D36Peli1bsGrVKuTl5aGiogK1tbVQq9UWYzp16oQ77rjD4n3MZjNycnLg4eGBvLw8TJgwAZMmTRLH1NbWQqPR2BwPEbUMJgPUZg0aNAhr166FUqmEn58fnJws/7m3a9fO4nVFRQXCw8OxcePG667VsWPHRsXg6upq8zkVFRUAgJSUFIsvYaBuHoSjZGZmYsyYMVi4cCGioqKg0WiwefNmLF++3OZY33nnneuSE4VC4bBYiahpMRmgNqtdu3YIDg5u8Pg+ffpgy5Yt8PHxue6343q+vr44fPgwBgwYAKDuN+CsrCz06dPnhuPDwsJgNpuRkZGBiIiI647XVyZMJpO4LzQ0FCqVCgUFBVYrCiEhIeJkyHqHDh269Ye8xsGDBxEYGIgXX3xR3PfTTz9dN66goABFRUXw8/MT30cul6Nr167QarXw8/PDDz/8gDFjxtj0/kTUenACIdHvxowZgw4dOmD48OHYv38/8vPzkZ6ejn/84x84d+4cAOD555/Hq6++iu3bt+Ps2bN47rnnbrpGwJ133omYmBiMHz8e27dvF6+5detWAEBgYCBkMhmSk5Nx4cIFVFRUwMPDAzNmzEB8fDw2bNiAvLw8HDt2DG+88YY4KW/y5Mn4/vvvMXPmTOTk5GDTpk1ISkqy6fPeddddKCgowObNm5GXl4dVq1bdcDKki4sLYmJi8M0332D//v34xz/+gcceeww6nQ4AsHDhQiQmJmLVqlX47rvvcPLkSbz//vt47bXXbIqHiFoOkwGi37m5uWHfvn3o1KkTRo4ciZCQEEyYMAFVVVVipeCFF17A2LFjERMTA71eDw8PD/ztb3+76XXXrl2LRx99FM899xy6deuGSZMmobKyEgBwxx13YOHChZgzZw60Wi3i4uIAAIsXL8ZLL72ExMREhISEYMiQIUhJSUFQUBCAuj7+J598gu3bt6NXr15Yt24dlixZYtPnfeSRRxAfH4+4uDj07t0bBw8exEsvvXTduODgYIwcORLDhg1DZGQkevbsaXHr4MSJE7F+/Xq8//77CAsLwwMPPICkpCQxViJq/WSCtZlPREREJAmsDBAREUkckwEiIiKJYzJAREQkcUwGiIiIJI7JABERkcQxGSAiIpI4JgNEREQSx2SAiIhI4pgMEBERSRyTASIiIoljMkBERCRx/w/jLbvXiQmGywAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cm = confusion_matrix(df['y_true'], df['y_pred2'])\n",
        "disp = ConfusionMatrixDisplay(confusion_matrix=cm)\n",
        "disp.plot()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 484
        },
        "id": "JdN4Z8o93zQM",
        "outputId": "13ad45fc-7061-4ab1-99f0-5674d36c1b02"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7dafa4d173a0>"
            ]
          },
          "metadata": {},
          "execution_count": 36
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 2 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAGwCAYAAAA0bWYRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA5iUlEQVR4nO3de3RTZdr38V96SHpMoCCthYJ1UKDKQVCh4wkUqYgKA74zOqhVQR+0oIByGgUBD/iAiqAIjqgVB0bQGRgBxeEBAZWKAtZBBBREQUsLim1psU2b7PeP2mimRBqSkpL9/ay11zJ733vnCnY1V6/rvve2GIZhCAAAmFZEqAMAAAChRTIAAIDJkQwAAGByJAMAAJgcyQAAACZHMgAAgMmRDAAAYHJRoQ4gEG63WwUFBUpMTJTFYgl1OAAAPxmGoSNHjig1NVUREQ3392lFRYWcTmfA17FarYqJiQlCRI3LKZ0MFBQUKC0tLdRhAAACtH//frVq1apBrl1RUaH0NgkqPOgK+FopKSnau3dv2CUEp3QykJiYKEn6ZusZsifQ8UB4+sPZHUMdAtBgqlWl9/WW5/d5Q3A6nSo86NI3W86QPfHEvytKj7jVptvXcjqdJAONSW1rwJ4QEdD/YKAxi7JEhzoEoOH8fEP8k9HqTUi0KCHxxN/HrfBtR5/SyQAAAPXlMtxyBfA0HpfhDl4wjQzJAADAFNwy5NaJZwOBnNvYUVsHAMDkqAwAAEzBLbcCKfQHdnbjRjIAADAFl2HIZZx4qT+Qcxs72gQAAJgclQEAgCkwgdA3kgEAgCm4ZchFMnBMtAkAADA5KgMAAFOgTeAbyQAAwBRYTeAbbQIAAEyOygAAwBTcP2+BnB+uSAYAAKbgCnA1QSDnNnYkAwAAU3AZCvCphcGLpbFhzgAAACZHZQAAYArMGfCNZAAAYApuWeSSJaDzwxVtAgAATI7KAADAFNxGzRbI+eGKZAAAYAquANsEgZzb2NEmAADA5KgMAABMgcqAbyQDAABTcBsWuY0AVhMEcG5jR5sAAACTozIAADAF2gS+kQwAAEzBpQi5AiiIu4IYS2NDMgAAMAUjwDkDBnMGAABAuKIyAAAwBeYM+EYyAAAwBZcRIZcRwJyBML4dMW0CAABMjsoAAMAU3LLIHcDfwG6Fb2mAZAAAYArMGfCNNgEAACZHZQAAYAqBTyCkTQAAwCmtZs5AAA8qok0AAADCFZUBAIApuAN8NgGrCQAAOMUxZ8A3kgEAgCm4FcF9BnxgzgAAACZHZQAAYAouwyJXAI8hDuTcxo5kAABgCq4AJxC6aBMAAIBwRWUAAGAKbiNC7gBWE7hZTQAAwKmNNoFvtAkAADA5KgMAAFNwK7AVAe7ghdLokAwAAEwh8JsOhW8xPXw/GQAAqBcqAwAAUwj82QTh+/czyQAAwBTcssitQOYMcAdCAABOaVQGfAvfTwYAAOqFygAAwBQCv+lQ+P79TDIAADAFt2GRO5D7DITxUwvDN80BAAD1QmUAAGAK7gDbBOF80yGSAQCAKQT+1MLwTQbC95MBANBIPP7447JYLBo5cqRnX0VFhXJyctSsWTMlJCRo0KBBKioq8jpv37596tevn+Li4tSiRQuNGTNG1dXVXmPWrVunrl27ymazqW3btsrNzfU7PpIBAIApuGQJeDsRH3/8sZ5//nl16tTJa/+oUaO0fPlyvf7661q/fr0KCgo0cODAX+J1udSvXz85nU5t3LhRr7zyinJzczVp0iTPmL1796pfv37q1auX8vPzNXLkSA0dOlTvvPOOXzGSDAAATKG2TRDI5q+ysjINHjxYL7zwgpo2berZX1JSohdffFFPPfWULr/8cnXr1k0vv/yyNm7cqA8//FCS9O9//1uff/65/va3v6lLly7q27evHn74Yc2ZM0dOp1OSNG/ePKWnp+vJJ59Uhw4dNHz4cF1//fWaOXOmX3GSDAAA4IfS0lKvrbKy0ufYnJwc9evXT7179/bav2XLFlVVVXntb9++vVq3bq28vDxJUl5enjp27Kjk5GTPmKysLJWWlmr79u2eMf997aysLM816otkAABgCi4F2iqokZaWJofD4dmmTZt2zPd77bXXtHXr1mMeLywslNVqVZMmTbz2Jycnq7Cw0DPm14lA7fHaY781prS0VD/99FO9/21YTQAAMIVgrSbYv3+/7Ha7Z7/NZqszdv/+/br33nu1evVqxcTEnPB7nixUBgAAplD7oKJANkmy2+1e27GSgS1btujgwYPq2rWroqKiFBUVpfXr12v27NmKiopScnKynE6niouLvc4rKipSSkqKJCklJaXO6oLa18cbY7fbFRsbW+9/G5IBAACC7IorrtC2bduUn5/v2c4//3wNHjzY89/R0dFas2aN55xdu3Zp3759yszMlCRlZmZq27ZtOnjwoGfM6tWrZbfblZGR4Rnz62vUjqm9Rn3RJgAAmIIhi9wnuDyw9vz6SkxM1Lnnnuu1Lz4+Xs2aNfPsHzJkiEaPHq2kpCTZ7XaNGDFCmZmZ6tGjhySpT58+ysjI0M0336zp06ersLBQDz74oHJycjzViGHDhunZZ5/V2LFjdfvtt2vt2rVasmSJVq5c6ddnIxkAAJjCr0v9J3p+MM2cOVMREREaNGiQKisrlZWVpeeee85zPDIyUitWrNBdd92lzMxMxcfHKzs7W1OnTvWMSU9P18qVKzVq1CjNmjVLrVq10vz585WVleVXLBbDMIygfbKTrLS0VA6HQz9+cabsiXQ8EJ6yUruEOgSgwVQbVVqnf6mkpMRrUl4w1X5XjNnYT7aE6BO+TmVZlWb8fmWDxhoqVAYAAKbAI4x9IxkAAJiCK8CnFgZybmMXvp8MAADUC5UBAIAp0CbwjWQAAGAKbkXIHUBBPJBzG7vw/WQAAKBeqAwAAEzBZVjkCqDUH8i5jR3JAADAFJgz4BvJAADAFIwAn1poBPkOhI1J+H4yAABQL1QGAACm4JJFrgAeVBTIuY0dyQAAwBTcRmB9f/cp+ySf46NNAACAyVEZMLnFz7TQS9NSNWDoId019TtJ0uGDUZr/cKq2bkjU0bIIpf2uUjfcW6RL+pV4nbvp/+xaODNZe3fEympzq2OPck1+ea/n+LGetjfhua/Vc0BxQ34k4Lhuuq9QN99X5LVv/26bhl7aXpIUbXPrzocK1PO6YkXbDG1Zl6hnJrRU8fcn/sQ7hJ47wAmEgZzb2JEMmNiu/Fit/FszpWf85LV/xj2tVVYaqcm5e+VIqta7S5vqsf85Q8+8/YXadqwZ+95Kh54ek6bbxh9Ql4vK5HJJX++MrfMe983cp/N7lXpeJ9hdDfuhgHr6emeMxv/pTM9rl+uX8vGwyQW6sHepHvmfNiovjVTOo99p0otfa3T/s0IRKoLELYvcAfT9Azm3sWsUac6cOXN0xhlnKCYmRt27d9dHH30U6pDC3k/lEfrf4W00csZ+JTq8v6A/3xyv/rd/r/bnHdXpbZz688gixTtc+vI/NV/2rmpp3qSWuuPBAl1zyw9q9btKtTm7UpddV1znfRLsLiW1qPZs1pgwbrrhlOJyST8eivZspYdr/jaKS3Qp68bDen5yqj79IFG7t8XpqdFpOueCo2rftTzEUQMNI+TJwOLFizV69Gg99NBD2rp1qzp37qysrCwdPHgw1KGFtWf/0koXXlGqrpeW1TmWcX651r/ZRKU/RsrtltYtayJnhUWdfl8z9sttcfr+gFWWCOnuK8/WjV3O0QODz9TXO2Pqvs8DLfX/zjlXI64+S+/8PUkGuQAaiZbpTi3aul25eTs07tlvdFpLpyTprE5HFW019Ml7iZ6x+3fHqOjbaHXodjRU4SIIau9AGMgWrkKeDDz11FO64447dNtttykjI0Pz5s1TXFycXnrppVCHFrbWLWui3dtidfuEA8c8/sDz38hVZdH/O6ejrjmjs2aNS9NDL36tluk1vywLv7FKkv72ZIpuHFmkqQu+UoLDpTGD2qr0x0jPdW4Zc0APzPtG017bo4uvLtEzf2mlf73YvOE/IHAcO7fG6YmRaXpg8Jl6ZnxLpbR26smluxUbX1PJclZaVF4a6XVO8aEoJbWoClHECIbaOQOBbOEqpHMGnE6ntmzZogkTJnj2RUREqHfv3srLy6szvrKyUpWVlZ7XpaWldcbgtx38LlpzJ7XUtNf2+CzZvzI9RWWlkXp88W7Zk6qVt8qhR4edoSeXfqn0DhVyu2vG3firSYX3zdynm7qdo/dWNFG/m3+QJA0e9csErbYdf1LF0Qi9PreFBgz9vmE/JHAcm9+1e/57745Y7fwkXq9+9Lkuva5Yzorw/YUP+BLSn/rvv/9eLpdLycnJXvuTk5NVWFhYZ/y0adPkcDg8W1pa2skKNWzs/k+cir+PVk5WO/VN66y+aZ31n7wE/evF5uqb1lkFX1v15sunafRT+3XeJWX63TkVuum+Ip3V6ajezK35qz4puVqS1PqsCs91rTZDKW0qdfA737Ot23c9qu8PWOWsDN9SG05N5aWR+vYrm1LPcOrwwShZbYbi/2uya5PTqnX4IKsJTmVuWTzPJzihjQmEjcOECRNUUlLi2fbv3x/qkE45XS45oufX7tTc1bs829mdj+rygT9q7updqvyp5kciIsK7ahAZacj4uSJwVqejira59e0em+d4dZVUtN+q5Fa+y6h7tscqoUm1rDYmDqBxiYlzKbVNTSLw5X/iVOW06LyLj3iOt/pdhZJbVWnHlrgQRolAGT+vJjjRzQjjZCCkbYLmzZsrMjJSRUXe632LioqUkpJSZ7zNZpPNZquzH/UXl+DWGe0rvPbFxLmV2NSlM9pXqLpKSk2v1KyxabpjUoHsTau1cZVDWzckauqCryRJ8Ylu9bv5B736ZIpOS61Si1ZOvTG3hSTpkmuKJUkf/tuuHw9FqUO3msRh64ZEvTa7ha4fduikfl7gWO6YVKAP/23XwW+tapZSpZvvL5TLLa1b2lRHj0Tqnb8n6c7JBTpSHKXyIxHKefQ7fb45Tju3xoc6dASApxb6FtJkwGq1qlu3blqzZo0GDBggSXK73VqzZo2GDx8eytBMKypaeuTVPXrxsVQ9lJ2un8ojlJru1P2z9unCK375S+mOid8pMtLQ9Htay1kRoXbnHdX/vr5HiU1qSquR0YaW5zbX85NtMgwp9Qyn/mdygfoO/iFUHw3waH56lSY8940Sm7pU8kOUtn8cr5HXnKWSn5cXzpucKrchTXzha0XbDG1el6hnJ7QMcdRAw7EYRmgXey1evFjZ2dl6/vnndeGFF+rpp5/WkiVLtHPnzjpzCf5baWmpHA6HfvziTNkTT6mOB1Bvx7qTIxAuqo0qrdO/VFJSIrvdfvwTTkDtd8UfVt+m6HjrCV+nqtyppVe+3KCxhkrI70D4pz/9SYcOHdKkSZNUWFioLl26aNWqVcdNBAAA8AdtAt9CngxI0vDhw2kLAAAQIo0iGQAAoKHxbALfSAYAAKZAm8A3Zt0BAGByVAYAAKZAZcA3kgEAgCmQDPhGmwAAAJOjMgAAMAUqA76RDAAATMFQYMsDw/kRayQDAABToDLgG3MGAAAwOSoDAABToDLgG8kAAMAUSAZ8o00AAIDJURkAAJgClQHfSAYAAKZgGBYZAXyhB3JuY0ebAAAAk6MyAAAwBbcsAd10KJBzGzuSAQCAKTBnwDfaBAAAmByVAQCAKTCB0DeSAQCAKdAm8I1kAABgClQGfGPOAAAAJkdlAABgCkaAbYJwrgyQDAAATMGQZBiBnR+uaBMAAGByVAYAAKbglkUW7kB4TCQDAABTYDWBb7QJAAAwOSoDAABTcBsWWbjp0DGRDAAATMEwAlxNEMbLCWgTAABgclQGAACmwARC30gGAACmQDLgG8kAAMAUmEDoG3MGAAAwOSoDAABTYDWBbyQDAABTqEkGApkzEMRgGhnaBAAAmByVAQCAKbCawDcqAwAAUzCCsPlj7ty56tSpk+x2u+x2uzIzM/X22297jldUVCgnJ0fNmjVTQkKCBg0apKKiIq9r7Nu3T/369VNcXJxatGihMWPGqLq62mvMunXr1LVrV9lsNrVt21a5ubl+RkoyAABAg2jVqpUef/xxbdmyRZs3b9bll1+u/v37a/v27ZKkUaNGafny5Xr99de1fv16FRQUaODAgZ7zXS6X+vXrJ6fTqY0bN+qVV15Rbm6uJk2a5Bmzd+9e9evXT7169VJ+fr5GjhypoUOH6p133vErVothnLpTIkpLS+VwOPTjF2fKnkheg/CUldol1CEADabaqNI6/UslJSWy2+0N8h613xVnLviLIuNiTvg6rqMV+uqWx7R//36vWG02m2w2W72ukZSUpBkzZuj666/XaaedpkWLFun666+XJO3cuVMdOnRQXl6eevToobffflvXXHONCgoKlJycLEmaN2+exo0bp0OHDslqtWrcuHFauXKlPvvsM8973HDDDSouLtaqVavq/dn4BgUAmEOQ+gRpaWlyOByebdq0acd9a5fLpddee03l5eXKzMzUli1bVFVVpd69e3vGtG/fXq1bt1ZeXp4kKS8vTx07dvQkApKUlZWl0tJST3UhLy/P6xq1Y2qvUV9MIAQAmEOAEwj187nHqgz4sm3bNmVmZqqiokIJCQlaunSpMjIylJ+fL6vVqiZNmniNT05OVmFhoSSpsLDQKxGoPV577LfGlJaW6qefflJsbGy9PhrJAAAAfqidEFgf7dq1U35+vkpKSvTGG28oOztb69evb+AI/UcyAAAwhVDcgdBqtapt27aSpG7duunjjz/WrFmz9Kc//UlOp1PFxcVe1YGioiKlpKRIklJSUvTRRx95Xa92tcGvx/z3CoSioiLZ7fZ6VwUk5gwAAEyi9j4DgWyBcrvdqqysVLdu3RQdHa01a9Z4ju3atUv79u1TZmamJCkzM1Pbtm3TwYMHPWNWr14tu92ujIwMz5hfX6N2TO016ovKAAAADWDChAnq27evWrdurSNHjmjRokVat26d3nnnHTkcDg0ZMkSjR49WUlKS7Ha7RowYoczMTPXo0UOS1KdPH2VkZOjmm2/W9OnTVVhYqAcffFA5OTmeeQrDhg3Ts88+q7Fjx+r222/X2rVrtWTJEq1cudKvWEkGAADmYFg8kwBP+Hw/HDx4ULfccosOHDggh8OhTp066Z133tGVV14pSZo5c6YiIiI0aNAgVVZWKisrS88995zn/MjISK1YsUJ33XWXMjMzFR8fr+zsbE2dOtUzJj09XStXrtSoUaM0a9YstWrVSvPnz1dWVpZfsXKfAaCR4z4DCGcn8z4DbeZPVEQA9xlwH63QN0MfbtBYQ4VvUAAATI42AQDAHE7kAQP/fX6YIhkAAJgCTy30rV7JwJtvvlnvC1533XUnHAwAADj56pUMDBgwoF4Xs1gscrlcgcQDAEDDCeNSfyDqlQy43e6GjgMAgAZFm8C3gFYTVFRUBCsOAAAaVpCeWhiO/E4GXC6XHn74YbVs2VIJCQn66quvJEkTJ07Uiy++GPQAAQBAw/I7GXj00UeVm5ur6dOny2q1evafe+65mj9/flCDAwAgeCxB2MKT38nAggUL9Ne//lWDBw9WZGSkZ3/nzp21c+fOoAYHAEDQ0Cbwye9k4LvvvvM8jvHX3G63qqqqghIUAAA4efxOBjIyMvTee+/V2f/GG2/ovPPOC0pQAAAEHZUBn/y+A+GkSZOUnZ2t7777Tm63W//85z+1a9cuLViwQCtWrGiIGAEACNxJfmrhqcTvykD//v21fPly/d///Z/i4+M1adIk7dixQ8uXL/c8lhEAAJw6TujZBJdccolWr14d7FgAAGgwhlGzBXJ+uDrhBxVt3rxZO3bskFQzj6Bbt25BCwoAgKDjqYU++Z0MfPvtt7rxxhv1wQcfqEmTJpKk4uJi/f73v9drr72mVq1aBTtGAADQgPyeMzB06FBVVVVpx44dOnz4sA4fPqwdO3bI7XZr6NChDREjAACBq51AGMgWpvyuDKxfv14bN25Uu3btPPvatWunZ555RpdccklQgwMAIFgsRs0WyPnhyu9kIC0t7Zg3F3K5XEpNTQ1KUAAABB1zBnzyu00wY8YMjRgxQps3b/bs27x5s+6991498cQTQQ0OAAA0vHpVBpo2bSqL5ZdeSXl5ubp3766oqJrTq6urFRUVpdtvv10DBgxokEABAAgINx3yqV7JwNNPP93AYQAA0MBoE/hUr2QgOzu7oeMAAAAhcsI3HZKkiooKOZ1Or312uz2ggAAAaBBUBnzyewJheXm5hg8frhYtWig+Pl5Nmzb12gAAaJR4aqFPficDY8eO1dq1azV37lzZbDbNnz9fU6ZMUWpqqhYsWNAQMQIAgAbkd5tg+fLlWrBggXr27KnbbrtNl1xyidq2bas2bdpo4cKFGjx4cEPECQBAYFhN4JPflYHDhw/rzDPPlFQzP+Dw4cOSpIsvvlgbNmwIbnQAAARJ7R0IA9nCld/JwJlnnqm9e/dKktq3b68lS5ZIqqkY1D64CAAAnDr8TgZuu+02ffrpp5Kk8ePHa86cOYqJidGoUaM0ZsyYoAcIAEBQMIHQJ7/nDIwaNcrz371799bOnTu1ZcsWtW3bVp06dQpqcAAAoOEFdJ8BSWrTpo3atGkTjFgAAGgwFgX41MKgRdL41CsZmD17dr0veM8995xwMAAA4OSrVzIwc+bMel3MYrGEJBm4+ImhirTGnPT3BU6GxHcOhDoEoMFUl1dKfzhJb8bSQp/qlQzUrh4AAOCUxe2IffJ7NQEAAAgvAU8gBADglEBlwCeSAQCAKQR6F0HuQAgAAMIWlQEAgDnQJvDphCoD7733nm666SZlZmbqu+++kyS9+uqrev/994MaHAAAQcPtiH3yOxn4xz/+oaysLMXGxuqTTz5RZWWlJKmkpESPPfZY0AMEAAANy+9k4JFHHtG8efP0wgsvKDo62rP/oosu0tatW4MaHAAAwcIjjH3ze87Arl27dOmll9bZ73A4VFxcHIyYAAAIPu5A6JPflYGUlBTt3r27zv73339fZ555ZlCCAgAg6Jgz4JPfycAdd9yhe++9V5s2bZLFYlFBQYEWLlyo+++/X3fddVdDxAgAABqQ322C8ePHy+1264orrtDRo0d16aWXymaz6f7779eIESMaIkYAAALGTYd88zsZsFgseuCBBzRmzBjt3r1bZWVlysjIUEJCQkPEBwBAcHCfAZ9O+KZDVqtVGRkZwYwFAACEgN/JQK9evWSx+J5RuXbt2oACAgCgQQS6PJDKwC+6dOni9bqqqkr5+fn67LPPlJ2dHay4AAAILtoEPvmdDMycOfOY+ydPnqyysrKAAwIAACdX0J5aeNNNN+mll14K1uUAAAgu7jPgU9CeWpiXl6eYmJhgXQ4AgKBiaaFvficDAwcO9HptGIYOHDigzZs3a+LEiUELDAAAnBx+JwMOh8PrdUREhNq1a6epU6eqT58+QQsMAACcHH4lAy6XS7fddps6duyopk2bNlRMAAAEH6sJfPJrAmFkZKT69OnD0wkBAKccHmHsm9+rCc4991x99dVXDRELAAAIAb+TgUceeUT333+/VqxYoQMHDqi0tNRrAwCg0WJZ4THVe87A1KlTdd999+nqq6+WJF133XVetyU2DEMWi0Uulyv4UQIAECjmDPhU72RgypQpGjZsmN59992GjAcAAJxk9U4GDKMmJbrssssaLBgAABoKNx3yza+lhb/1tEIAABo12gQ++TWB8Oyzz1ZSUtJvbgAAQJo2bZouuOACJSYmqkWLFhowYIB27drlNaaiokI5OTlq1qyZEhISNGjQIBUVFXmN2bdvn/r166e4uDi1aNFCY8aMUXV1tdeYdevWqWvXrrLZbGrbtq1yc3P9itWvysCUKVPq3IEQAIBTwcluE6xfv145OTm64IILVF1drb/85S/q06ePPv/8c8XHx0uSRo0apZUrV+r111+Xw+HQ8OHDNXDgQH3wwQeSam72169fP6WkpGjjxo06cOCAbrnlFkVHR+uxxx6TJO3du1f9+vXTsGHDtHDhQq1Zs0ZDhw7V6aefrqysrHp+ttrJAMcRERGhwsJCtWjRwr9/jQZUWloqh8Ohc+58TJFWHpKE8JR43YFQhwA0mOrySm36w2yVlJTIbrc3yHvUflecfd9jirSd+HeFq7JCXzz5lxOO9dChQ2rRooXWr1+vSy+9VCUlJTrttNO0aNEiXX/99ZKknTt3qkOHDsrLy1OPHj309ttv65prrlFBQYGSk5MlSfPmzdO4ceN06NAhWa1WjRs3TitXrtRnn33mea8bbrhBxcXFWrVqVb1iq3ebgPkCAACozv11Kisr63VeSUmJJHla6lu2bFFVVZV69+7tGdO+fXu1bt1aeXl5kmqeCNyxY0dPIiBJWVlZKi0t1fbt2z1jfn2N2jG116iPeicD9SwgAADQOAVyw6FfTT5MS0uTw+HwbNOmTTvuW7vdbo0cOVIXXXSRzj33XElSYWGhrFarmjRp4jU2OTlZhYWFnjG/TgRqj9ce+60xpaWl+umnn44bm+THnAG3213foQAANDrBmjOwf/9+rzaBzWY77rk5OTn67LPP9P777594AA3I79sRAwBwSgpSZcBut3ttx0sGhg8frhUrVujdd99Vq1atPPtTUlLkdDrrPPyvqKhIKSkpnjH/vbqg9vXxxtjtdsXGxh73n0UiGQAAoEEYhqHhw4dr6dKlWrt2rdLT072Od+vWTdHR0VqzZo1n365du7Rv3z5lZmZKkjIzM7Vt2zYdPHjQM2b16tWy2+3KyMjwjPn1NWrH1F6jPvxaWggAwCnrJN90KCcnR4sWLdK//vUvJSYmenr8DodDsbGxcjgcGjJkiEaPHq2kpCTZ7XaNGDFCmZmZ6tGjhySpT58+ysjI0M0336zp06ersLBQDz74oHJycjwViWHDhunZZ5/V2LFjdfvtt2vt2rVasmSJVq5cWe9YSQYAAKZwsu8zMHfuXElSz549vfa//PLLuvXWWyVJM2fOVEREhAYNGqTKykplZWXpueee84yNjIzUihUrdNdddykzM1Px8fHKzs7W1KlTPWPS09O1cuVKjRo1SrNmzVKrVq00f/78et9jQCIZAACgQdRnFV5MTIzmzJmjOXPm+BzTpk0bvfXWW795nZ49e+qTTz7xO8ZaJAMAAHPg2QQ+kQwAAEyBpxb6xmoCAABMjsoAAMAcaBP4RDIAADAHkgGfaBMAAGByVAYAAKZg+XkL5PxwRTIAADAH2gQ+kQwAAEyBpYW+MWcAAACTozIAADAH2gQ+kQwAAMwjjL/QA0GbAAAAk6MyAAAwBSYQ+kYyAAAwB+YM+ESbAAAAk6MyAAAwBdoEvpEMAADMgTaBT7QJAAAwOSoDAABToE3gG8kAAMAcaBP4RDIAADAHkgGfmDMAAIDJURkAAJgCcwZ8IxkAAJgDbQKfaBMAAGByVAYAAKZgMQxZjBP/8z6Qcxs7kgEAgDnQJvCJNgEAACZHZQAAYAqsJvCNZAAAYA60CXyiTQAAgMlRGQAAmAJtAt9IBgAA5kCbwCeSAQCAKVAZ8I05AwAAmByVAQCAOdAm8IlkAABgGuFc6g8EbQIAAEyOygAAwBwMo2YL5PwwRTIAADAFVhP4RpsAAACTozIAADAHVhP4RDIAADAFi7tmC+T8cEWbAAAAk6MyYEJd0wp0S2a+MlIO6bTEoxr1+lVa90W65/gnD8w95nkz1/TQgg/PkyStzPmbUpsc8To+e213vZzX1fP6rBY/aHzWBp2Tekg/Ho3Rax931Cs/nw80lMjlpYpaWSpLUbUkyWhjVdXgJnJfECdJip71vSI++UmWH1xSrEXuDjGqGtJURmtrzQVKXbI+fkgRe53SEZcMR6TcmXGqui1Jiq/5+yni059kG1tY571/+nualMSv1UaLNoFP/NSaUKy1Sl8UNdO/Pm2vp65/p87x3k9ne72+6Hf79NA172rNzt957X9u/QX65ycZntflzmjPf8dbnXruxuXatLeVHn37Mp3V4gc9dM06Ham0eZ0DBJtxWpSqbk+S0TJaMqTI1UdknVykyjktZZxhlfssq1yXx8s4LUo64lb034pl/UuhKl9JkyItkkVyZcap6tamkiNCloJqRT/7g6KPfK+qCS283qvixVYy4iy/7GgSeZI/LfzBagLfQpoMbNiwQTNmzNCWLVt04MABLV26VAMGDAhlSKbwwZ42+mBPG5/HfyiP83rd8+y9+vjrlvqu2O61v7wyus7YWlef+4WiI92avKKXqt2R+ur7JLVL/kE3XfgpyQAalLuH989k9W1JilpxRBE7K+U6wyrX1b/6OU6RqrKbKuau72QpqpaRGi0lRsp17S9jjORoVV+bqKjXS+q8l9EkQkogAThlcJ8Bn0I6Z6C8vFydO3fWnDlzQhkGfkNS/FFd3Hafln3avs6x237/id4d9ZL+PuR13dLjE0X+anZNp5ZF2rrvdFW7f/lFufGrNKU3L1ZiTOVJiR2Qy1DkujKp0i13B1vd4xVuRf37iNwpUTWVgmP5oVqRHxyVu1NMnUO2uwsUc+M+WccfUMT2iuDGDpxEIa0M9O3bV3379q33+MrKSlVW/vJFUlpa2hBh4Veu7bhLR53RWrvzTK/9f/+4o3YUNldpRYw6tyrUiJ4f6rSEo3ry/y6SJDVLOFqnknC4PFaS1Dz+qI5UHOMXMxAklr1O2UYWSE5Dio2Qc1KyjDZWz/HI5aWKnn9YlgpD7lbRck5LkaItXteInnZQkXlHZak05OoRp6pRzT3HjKRIOe9pJvfZNlmchiJXHZF1zAFVzkqVcRY/240VbQLfTqk5A9OmTdOUKVNCHYap9O+8U29/dpacLu8flb991Nnz318ebKYqV4Qe6LtBs9/toSoXZVOEltEqWpXPtZSOuhX5XrmsTxxS5YzTPQmB6/IEubvGynK4WlFvlMr66EFVzjxdsv5SLK36nyRVD24iy3dVin7pR0U/f1hVI2oSAiPNKldazbUMSe5zYmQ5UK2opSWqGtuiTjxoJJhA6NMptbRwwoQJKikp8Wz79+8PdUhh7by0AqU3L9bS/A7HHbvtu2RFR7qV6qip1vxQFqdm8Ue9xiTF/yRJ+t7HPAMgaKItMlpGyzjLpurbk+ROtypq2a8qifERMlpGy90xVs4HW8iyv0qRH3j/vCopSkZrq9yZ8aq6t7miVhyRfqj2+ZbudjZFFPg+DjRmp1RlwGazyWajBHeyDOi8U58fOE1fHGx+3LHtkr+Xy23R4aM1X/T/+S5ZOT0/UlSEyzNvoEf6fu39vgktApx8hqQqH3/W1e72dVzyTByzVBk+/ziM2FMpI4mqWGNGm8C3UyoZQHDERlcpLemXmdEtm5Tq7OTvVfqTTYWliZJqlgZe2WGPnlrz+zrnd2pZqHNTi7T5m5Yqd1rVqWWh7r/yA7312VmeL/q3t5+lOy/ZrIf6rdPLeeep7WmH9ecLtumJn+cUAA0l6qXDcl8QWzMh8CdDke+WKeI/FXI+miLLgSpFri+Xq1us5IiU5VC1opaUSFaLXBfWJLIRHx2V5UeX3O1sUoxFlm+qFD3/sFzn2GSk1CyfjfxniYyUqJq2Q5WhqLePKOLTCjkfSwnlR8fxsJrAJ5IBE8o4/aDm3/ym5/X9V26UJL35aTs9tOJySVLWObsli7Rqe9s65ztdkco6Z7eGXbpZ0ZEuFRTbtfCjznp10y/zCMoqbbr779dqfNYGLRryhoqPxuiv75/PskI0OEuxS9EzvpflcLUUFyF3ulXOR1Pk7hYr/VCtiM8qFLW0RCpzy2gSKXfHmJr5ArX3CLBaFPn2EUU/f1iqMmScFinXRfGq/pPjl/eoNhT118M1Ny6yWWreY1qK3F1iQ/SpgcBYDCN0qU5ZWZl2794tSTrvvPP01FNPqVevXkpKSlLr1q2Pe35paakcDofOufMxRVrrLvsBwkHidQdCHQLQYKrLK7XpD7NVUlIiu91+/BNOQO13RWbfqYqKPvHviuqqCuW9PalBYw2VkFYGNm/erF69enlejx49WpKUnZ2t3NzcEEUFAAhLrCbwKaTJQM+ePRXCwgQAABBzBgAAJsFqAt9IBgAA5uA2arZAzg9TJAMAAHNgzoBPp9QdCAEAQPBRGQAAmIJFAc4ZCFokjQ/JAADAHLgDoU+0CQAAMDkqAwAAU2BpoW9UBgAA5mAEYfPDhg0bdO211yo1NVUWi0XLli3zDscwNGnSJJ1++umKjY1V79699eWXX3qNOXz4sAYPHiy73a4mTZpoyJAhKisr8xrzn//8R5dccoliYmKUlpam6dOn+xeoSAYAAGgQ5eXl6ty5s+bMmXPM49OnT9fs2bM1b948bdq0SfHx8crKylJFRYVnzODBg7V9+3atXr1aK1as0IYNG3TnnXd6jpeWlqpPnz5q06aNtmzZohkzZmjy5Mn661//6lestAkAAKZgMQxZApgE6O+5ffv2Vd++fY95zDAMPf3003rwwQfVv39/SdKCBQuUnJysZcuW6YYbbtCOHTu0atUqffzxxzr//PMlSc8884yuvvpqPfHEE0pNTdXChQvldDr10ksvyWq16pxzzlF+fr6eeuopr6TheKgMAADMwR2ETTV/jf96q6ys9DuUvXv3qrCwUL179/bsczgc6t69u/Ly8iRJeXl5atKkiScRkKTevXsrIiJCmzZt8oy59NJLZbVaPWOysrK0a9cu/fjjj/WOh2QAAAA/pKWlyeFweLZp06b5fY3CwkJJUnJystf+5ORkz7HCwkK1aNHC63hUVJSSkpK8xhzrGr9+j/qgTQAAMIVgtQn2798vu93u2W+z2QKOLdSoDAAAzCFIqwnsdrvXdiLJQEpKiiSpqKjIa39RUZHnWEpKig4ePOh1vLq6WocPH/Yac6xr/Po96oNkAABgDrV3IAxkC5L09HSlpKRozZo1nn2lpaXatGmTMjMzJUmZmZkqLi7Wli1bPGPWrl0rt9ut7t27e8Zs2LBBVVVVnjGrV69Wu3bt1LRp03rHQzIAAEADKCsrU35+vvLz8yXVTBrMz8/Xvn37ZLFYNHLkSD3yyCN68803tW3bNt1yyy1KTU3VgAEDJEkdOnTQVVddpTvuuEMfffSRPvjgAw0fPlw33HCDUlNTJUl//vOfZbVaNWTIEG3fvl2LFy/WrFmzNHr0aL9iZc4AAMAUTvYdCDdv3qxevXp5Xtd+QWdnZys3N1djx45VeXm57rzzThUXF+viiy/WqlWrFBMT4zln4cKFGj58uK644gpFRERo0KBBmj17tue4w+HQv//9b+Xk5Khbt25q3ry5Jk2a5NeywprPZpy6T14oLS2Vw+HQOXc+pkhrzPFPAE5BidcdCHUIQIOpLq/Upj/MVklJidekvGCq/a64LPNBRUWd+HdFdXWF1uc90qCxhgptAgAATI42AQDAFCzumi2Q88MVyQAAwBwCXRFw6nbVj4s2AQAAJkdlAABgDifwGOI654cpkgEAgCmc7KcWnkpoEwAAYHJUBgAA5sAEQp9IBgAA5mBICmR5YPjmAiQDAABzYM6Ab8wZAADA5KgMAADMwVCAcwaCFkmjQzIAADAHJhD6RJsAAACTozIAADAHtyRLgOeHKZIBAIApsJrAN9oEAACYHJUBAIA5MIHQJ5IBAIA5kAz4RJsAAACTozIAADAHKgM+kQwAAMyBpYU+kQwAAEyBpYW+MWcAAACTozIAADAH5gz4RDIAADAHtyFZAvhCd4dvMkCbAAAAk6MyAAAwB9oEPpEMAABMIsBkQOGbDNAmAADA5KgMAADMgTaBTyQDAABzcBsKqNTPagIAABCuqAwAAMzBcNdsgZwfpkgGAADmwJwBn0gGAADmwJwBn5gzAACAyVEZAACYA20Cn0gGAADmYCjAZCBokTQ6tAkAADA5KgMAAHOgTeATyQAAwBzcbkkB3CvAHb73GaBNAACAyVEZAACYA20Cn0gGAADmQDLgE20CAABMjsoAAMAcuB2xTyQDAABTMAy3jACePBjIuY0dyQAAwBwMI7C/7pkzAAAAwhWVAQCAORgBzhkI48oAyQAAwBzcbskSQN8/jOcM0CYAAMDkqAwAAMyBNoFPJAMAAFMw3G4ZAbQJwnlpIW0CAABMjsoAAMAcaBP4RDIAADAHtyFZSAaOhTYBAAAmR2UAAGAOhiEpkPsMhG9lgGQAAGAKhtuQEUCbwCAZAADgFGe4FVhlgKWFAAAgTFEZAACYAm0C30gGAADmQJvAp1M6GajN0lzOihBHAjSc6vLKUIcANJjqozU/3yfjr+5qVQV0z6FqVQUvmEbGYpzCdY9vv/1WaWlpoQ4DABCg/fv3q1WrVg1y7YqKCqWnp6uwsDDga6WkpGjv3r2KiYkJQmSNxymdDLjdbhUUFCgxMVEWiyXU4ZhCaWmp0tLStH//ftnt9lCHAwQVP98nn2EYOnLkiFJTUxUR0XBz2isqKuR0OgO+jtVqDbtEQDrF2wQRERENlknit9ntdn5ZImzx831yORyOBn+PmJiYsPwSDxaWFgIAYHIkAwAAmBzJAPxis9n00EMPyWazhToUIOj4+YZZndITCAEAQOCoDAAAYHIkAwAAmBzJAAAAJkcyAACAyZEMoN7mzJmjM844QzExMerevbs++uijUIcEBMWGDRt07bXXKjU1VRaLRcuWLQt1SMBJRTKAelm8eLFGjx6thx56SFu3blXnzp2VlZWlgwcPhjo0IGDl5eXq3Lmz5syZE+pQgJBgaSHqpXv37rrgggv07LPPSqp5LkRaWppGjBih8ePHhzg6IHgsFouWLl2qAQMGhDoU4KShMoDjcjqd2rJli3r37u3ZFxERod69eysvLy+EkQEAgoFkAMf1/fffy+VyKTk52Wt/cnJyUB4JCgAILZIBAABMjmQAx9W8eXNFRkaqqKjIa39RUZFSUlJCFBUAIFhIBnBcVqtV3bp105o1azz73G631qxZo8zMzBBGBgAIhqhQB4BTw+jRo5Wdna3zzz9fF154oZ5++mmVl5frtttuC3VoQMDKysq0e/duz+u9e/cqPz9fSUlJat26dQgjA04Olhai3p599lnNmDFDhYWF6tKli2bPnq3u3buHOiwgYOvWrVOvXr3q7M/OzlZubu7JDwg4yUgGAAAwOeYMAABgciQDAACYHMkAAAAmRzIAAIDJkQwAAGByJAMAAJgcyQAAACZHMgAAgMmRDAABuvXWWzVgwADP6549e2rkyJEnPY5169bJYrGouLjY5xiLxaJly5bV+5qTJ09Wly5dAorr66+/lsViUX5+fkDXAdBwSAYQlm699VZZLBZZLBZZrVa1bdtWU6dOVXV1dYO/9z//+U89/PDD9Rpbny9wAGhoPKgIYeuqq67Syy+/rMrKSr311lvKyclRdHS0JkyYUGes0+mU1WoNyvsmJSUF5ToAcLJQGUDYstlsSklJUZs2bXTXXXepd+/eevPNNyX9Utp/9NFHlZqaqnbt2kmS9u/frz/+8Y9q0qSJkpKS1L9/f3399deea7pcLo0ePVpNmjRRs2bNNHbsWP334z3+u01QWVmpcePGKS0tTTabTW3bttWLL76or7/+2vNwnKZNm8pisejWW2+VVPOI6GnTpik9PV2xsbHq3Lmz3njjDa/3eeutt3T22WcrNjZWvXr18oqzvsaNG6ezzz5bcXFxOvPMMzVx4kRVVVXVGff8888rLS1NcXFx+uMf/6iSkhKv4/Pnz1eHDh0UExOj9u3b67nnnvM7FgChQzIA04iNjZXT6fS8XrNmjXbt2qXVq1drxYoVqqqqUlZWlhITE/Xee+/pgw8+UEJCgq666irPeU8++aRyc3P10ksv6f3339fhw4e1dOnS33zfW265RX//+981e/Zs7dixQ88//7wSEhKUlpamf/zjH5KkXbt26cCBA5o1a5Ykadq0aVqwYIHmzZun7du3a9SoUbrpppu0fv16STVJy8CBA3XttdcqPz9fQ4cO1fjx4/3+N0lMTFRubq4+//xzzZo1Sy+88IJmzpzpNWb37t1asmSJli9frlWrVumTTz7R3Xff7Tm+cOFCTZo0SY8++qh27Nihxx57TBMnTtQrr7zidzwAQsQAwlB2drbRv39/wzAMw+12G6tXrzZsNptx//33e44nJycblZWVnnNeffVVo127dobb7fbsq6ysNGJjY4133nnHMAzDOP30043p06d7jldVVRmtWrXyvJdhGMZll11m3HvvvYZhGMauXbsMScbq1auPGee7775rSDJ+/PFHz76KigojLi7O2Lhxo9fYIUOGGDfeeKNhGIYxYcIEIyMjw+v4uHHj6lzrv0kyli5d6vP4jBkzjG7dunleP/TQQ0ZkZKTx7bffeva9/fbbRkREhHHgwAHDMAzjd7/7nbFo0SKv6zz88MNGZmamYRiGsXfvXkOS8cknn/h8XwChxZwBhK0VK1YoISFBVVVVcrvd+vOf/6zJkyd7jnfs2NFrnsCnn36q3bt3KzEx0es6FRUV2rNnj0pKSnTgwAF1797dcywqKkrnn39+nVZBrfz8fEVGRuqyyy6rd9y7d+/W0aNHdeWVV3rtdzqdOu+88yRJO3bs8IpDkjIzM+v9HrUWL16s2bNna8+ePSorK1N1dbXsdrvXmNatW6tly5Ze7+N2u7Vr1y4lJiZqz549GjJkiO644w7PmOrqajkcDr/jARAaJAMIW7169dLcuXNltVqVmpqqqCjvH/f4+Hiv12VlZerWrZsWLlxY51qnnXbaCcUQGxvr9zllZWWSpJUrV3p9CUs18yCCJS8vT4MHD9aUKVOUlZUlh8Oh1157TU8++aTfsb7wwgt1kpPIyMigxQqgYZEMIGzFx8erbdu29R7ftWtXLV68WC1atKjz13Gt008/XZs2bdKll14qqeYv4C1btqhr167HHN+xY0e53W6tX79evXv3rnO8tjLhcrk8+zIyMmSz2bRv3z6fFYUOHTp4JkPW+vDDD4//IX9l48aNatOmjR544AHPvm+++abOuH379qmgoECpqame94mIiFC7du2UnJys1NRUffXVVxo8eLBf7w+g8WACIfCzwYMHq3nz5urfv7/ee+897d27V+vWrdM999yjb7/9VpJ077336vHHH9eyZcu0c+dO3X333b95j4AzzjhD2dnZuv3227Vs2TLPNZcsWSJJatOmjSwWi1asWKFDhw6prKxMiYmJuv/++zVq1Ci98sor2rNnj7Zu3apnnnnGMylv2LBh+vLLLzVmzBjt2rVLixYtUm5url+f96yzztK+ffv02muvac+ePZo9e/YxJ0PGxMQoOztbn376qd577z3dc889+uMf/6iUlBRJ0pQpUzRt2jTNnj1bX3zxhbZt26aXX35ZTz31lF/xAAgdkgHgZ3FxcdqwYYNat26tgQMHqkOHDhoyZIgqKio8lYL77rtPN998s7Kzs5WZmanExET94Q9/+M3rzp07V9dff73uvvtutW/fXnfccYfKy8slSS1bttSUKVM0fvx4JScna/jw4ZKkhx9+WBMnTtS0adPUoUMHXXXVVVq5cqXS09Ml1fTx//GPf2jZsmXq3Lmz5s2bp8cee8yvz3vddddp1KhRGj58uLp06aKNGzdq4sSJdca1bdtWAwcO1NVXX60+ffqoU6dOXksHhw4dqvnz5+vll19Wx44dddlllyk3N9cTK4DGz2L4mvkEAABMgcoAAAAmRzIAAIDJkQwAAGByJAMAAJgcyQAAACZHMgAAgMmRDAAAYHIkAwAAmBzJAAAAJkcyAACAyZEMAABgcv8f7mIvL+QOVWAAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "confusion_matrix(df['y_true'], df['y_pred2'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yYOJLkgz4YCN",
        "outputId": "cb67d738-10b8-4e9f-e328-f3c487c1e9d4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[4865,   50],\n",
              "       [1750, 3335]])"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tn, fp, fn, tp = confusion_matrix(df['y_true'], df['y_pred2']).ravel()"
      ],
      "metadata": {
        "id": "RSjwhInO4rkY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tn, fp, fn, tp"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MpaTHD6743BO",
        "outputId": "14822016-069e-43bb-9d17-5ed884e91b50"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(4865, 50, 1750, 3335)"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tp / (tp + fn)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "recu1iQM44bv",
        "outputId": "45ebfdc1-f6f2-494b-93ff-97f00d488664"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.655850540806293"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Мой Telegram-канал: https://t.me/mikhaylovgleb"
      ],
      "metadata": {
        "id": "EUIpMlMe4_vR"
      }
    }
  ]
}