Hyperparameter_backtracking_tree_search.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Hyperparameter backtracking tree search.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMTPzOA/fI70gAFnTo4NqBO",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sherna90/analisis_algoritmos/blob/master/Hyperparameter_backtracking_tree_search.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xU7YHcrHfm2M"
      },
      "source": [
        "# Hyperaparameter Backtracking Tree Search"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rT-aJyMrQvH6"
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "from time import time\n",
        "import scipy.stats as stats\n",
        "from sklearn.utils.fixes import loguniform\n",
        "\n",
        "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
        "from sklearn.datasets import load_digits\n",
        "from sklearn.linear_model import SGDClassifier\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# get some data\n",
        "X, y = load_digits(return_X_y=True)\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)\n",
        "# build a classifier\n",
        "clf = SGDClassifier(loss='log',learning_rate='constant',eta0=1e-3,fit_intercept=True,shuffle=False)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BtiHV79fTPF1",
        "outputId": "6f29683b-6b3c-4400-87b7-4a4c50e842c3"
      },
      "source": [
        "clf.get_params()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'alpha': 0.0001,\n",
              " 'average': False,\n",
              " 'class_weight': None,\n",
              " 'early_stopping': False,\n",
              " 'epsilon': 0.1,\n",
              " 'eta0': 0.001,\n",
              " 'fit_intercept': True,\n",
              " 'l1_ratio': 0.15,\n",
              " 'learning_rate': 'constant',\n",
              " 'loss': 'log',\n",
              " 'max_iter': 1000,\n",
              " 'n_iter_no_change': 5,\n",
              " 'n_jobs': None,\n",
              " 'penalty': 'l2',\n",
              " 'power_t': 0.5,\n",
              " 'random_state': None,\n",
              " 'shuffle': False,\n",
              " 'tol': 0.001,\n",
              " 'validation_fraction': 0.1,\n",
              " 'verbose': 0,\n",
              " 'warm_start': False}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P7VyGARmXiuX",
        "outputId": "88311693-dfdf-4cff-b374-7b91ae6c29ef"
      },
      "source": [
        "clf.fit(X_train,y_train)\n",
        "clf.score(X_test,y_test)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9595959595959596"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wTZ7FezKRIqO",
        "outputId": "e0dbb975-58e7-4e48-8ddc-3da07a255927"
      },
      "source": [
        "param_grid = {'penalty': ['l2','l1'],\n",
        "              'eta0': [1e-4,1e-2,1e-1],\n",
        "              'alpha':[1e-3,1e-2,1e-1,1.0,10.]}\n",
        "\n",
        "# run randomized search\n",
        "grid_search = GridSearchCV(clf, param_grid=param_grid)\n",
        "\n",
        "start = time()\n",
        "grid_search.fit(X_train, y_train)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GridSearchCV(cv=None, error_score=nan,\n",
              "             estimator=SGDClassifier(alpha=0.0001, average=False,\n",
              "                                     class_weight=None, early_stopping=False,\n",
              "                                     epsilon=0.1, eta0=0.001,\n",
              "                                     fit_intercept=True, l1_ratio=0.15,\n",
              "                                     learning_rate='constant', loss='log',\n",
              "                                     max_iter=1000, n_iter_no_change=5,\n",
              "                                     n_jobs=None, penalty='l2', power_t=0.5,\n",
              "                                     random_state=None, shuffle=False,\n",
              "                                     tol=0.001, validation_fraction=0.1,\n",
              "                                     verbose=0, warm_start=False),\n",
              "             iid='deprecated', n_jobs=None,\n",
              "             param_grid={'alpha': [0.001, 0.01, 0.1, 1.0, 10.0],\n",
              "                         'eta0': [0.0001, 0.01, 0.1], 'penalty': ['l2', 'l1']},\n",
              "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
              "             scoring=None, verbose=0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uqC6eeuWWAU2",
        "outputId": "48c7abed-cdfe-47bc-b350-eb9f8042055c"
      },
      "source": [
        "grid_search.best_params_"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'alpha': 0.01, 'eta0': 0.0001, 'penalty': 'l1'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9MAtGyM7WZsT",
        "outputId": "18adce87-5baf-4843-ffe5-8adecfc56a33"
      },
      "source": [
        "clf.set_params(**grid_search.best_params_)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SGDClassifier(alpha=0.01, average=False, class_weight=None,\n",
              "              early_stopping=False, epsilon=0.1, eta0=0.0001,\n",
              "              fit_intercept=True, l1_ratio=0.15, learning_rate='constant',\n",
              "              loss='log', max_iter=1000, n_iter_no_change=5, n_jobs=None,\n",
              "              penalty='l1', power_t=0.5, random_state=None, shuffle=False,\n",
              "              tol=0.001, validation_fraction=0.1, verbose=0, warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pu06O6h1XO8R",
        "outputId": "2799c063-30ce-42b7-e097-f5ac71dfbe15"
      },
      "source": [
        "clf.fit(X_train,y_train)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SGDClassifier(alpha=0.01, average=False, class_weight=None,\n",
              "              early_stopping=False, epsilon=0.1, eta0=0.0001,\n",
              "              fit_intercept=True, l1_ratio=0.15, learning_rate='constant',\n",
              "              loss='log', max_iter=1000, n_iter_no_change=5, n_jobs=None,\n",
              "              penalty='l1', power_t=0.5, random_state=None, shuffle=False,\n",
              "              tol=0.001, validation_fraction=0.1, verbose=0, warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MwXzbuXyXRIb",
        "outputId": "5b30c95d-7dd5-4484-fcaa-32d70bef27b0"
      },
      "source": [
        "clf.score(X_test,y_test)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9579124579124579"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GJh0dMIg_aZZ"
      },
      "source": [
        "# Backtracking Tree Search"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NuD3aMAuOReK",
        "outputId": "fcb17c09-1f8b-40ef-8975-0a9c9e715b72"
      },
      "source": [
        "import itertools\n",
        "\n",
        "def score(clf,X_train,y_train,X_test,y_test,par):\n",
        "  clf.set_params(**par)\n",
        "  clf.fit(X_train,y_train)\n",
        "  return clf.score(X_test,y_test)\n",
        "\n",
        "param_grid = {'penalty': ['l2','l1'],\n",
        "              'eta0': np.linspace(0.0001,1.0,10),\n",
        "              'alpha':np.linspace(0.001,1.0,10)}\n",
        "\n",
        "params=[ [{u:v} for u,vals in param_grid.items() for v in vals if u==node] for node in param_grid.keys()]\n",
        "\n",
        "i=1\n",
        "max_val=-np.infty\n",
        "min_val=np.infty\n",
        "for element in itertools.product(*params):\n",
        "  par=dict()\n",
        "  for v in element:\n",
        "    par.update(v)\n",
        "  current_val=score(clf,X_train,y_train,X_test,y_test,par)\n",
        "  print(i,par,current_val)\n",
        "  min_val=min(current_val,min_val)\n",
        "  max_val=max(current_val,max_val)\n",
        "  i+=1\n",
        "print('min val {0}, max_val : {1}'.format(min_val,max_val))"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1 {'penalty': 'l2', 'eta0': 0.0001, 'alpha': 0.001} 0.9612794612794613\n",
            "2 {'penalty': 'l2', 'eta0': 0.0001, 'alpha': 0.112} 0.9629629629629629\n",
            "3 {'penalty': 'l2', 'eta0': 0.0001, 'alpha': 0.223} 0.9629629629629629\n",
            "4 {'penalty': 'l2', 'eta0': 0.0001, 'alpha': 0.334} 0.9612794612794613\n",
            "5 {'penalty': 'l2', 'eta0': 0.0001, 'alpha': 0.445} 0.9579124579124579\n",
            "6 {'penalty': 'l2', 'eta0': 0.0001, 'alpha': 0.556} 0.9562289562289562\n",
            "7 {'penalty': 'l2', 'eta0': 0.0001, 'alpha': 0.667} 0.9545454545454546\n",
            "8 {'penalty': 'l2', 'eta0': 0.0001, 'alpha': 0.778} 0.9545454545454546\n",
            "9 {'penalty': 'l2', 'eta0': 0.0001, 'alpha': 0.889} 0.9528619528619529\n",
            "10 {'penalty': 'l2', 'eta0': 0.0001, 'alpha': 1.0} 0.9511784511784511\n",
            "11 {'penalty': 'l2', 'eta0': 0.11120000000000001, 'alpha': 0.001} 0.8956228956228957\n",
            "12 {'penalty': 'l2', 'eta0': 0.11120000000000001, 'alpha': 0.112} 0.5185185185185185\n",
            "13 {'penalty': 'l2', 'eta0': 0.11120000000000001, 'alpha': 0.223} 0.15488215488215487\n",
            "14 {'penalty': 'l2', 'eta0': 0.11120000000000001, 'alpha': 0.334} 0.09427609427609428\n",
            "15 {'penalty': 'l2', 'eta0': 0.11120000000000001, 'alpha': 0.445} 0.09259259259259259\n",
            "16 {'penalty': 'l2', 'eta0': 0.11120000000000001, 'alpha': 0.556} 0.09259259259259259\n",
            "17 {'penalty': 'l2', 'eta0': 0.11120000000000001, 'alpha': 0.667} 0.09259259259259259\n",
            "18 {'penalty': 'l2', 'eta0': 0.11120000000000001, 'alpha': 0.778} 0.09259259259259259\n",
            "19 {'penalty': 'l2', 'eta0': 0.11120000000000001, 'alpha': 0.889} 0.09259259259259259\n",
            "20 {'penalty': 'l2', 'eta0': 0.11120000000000001, 'alpha': 1.0} 0.09259259259259259\n",
            "21 {'penalty': 'l2', 'eta0': 0.2223, 'alpha': 0.001} 0.8956228956228957\n",
            "22 {'penalty': 'l2', 'eta0': 0.2223, 'alpha': 0.112} 0.26262626262626265\n",
            "23 {'penalty': 'l2', 'eta0': 0.2223, 'alpha': 0.223} 0.09932659932659933\n",
            "24 {'penalty': 'l2', 'eta0': 0.2223, 'alpha': 0.334} 0.09259259259259259\n",
            "25 {'penalty': 'l2', 'eta0': 0.2223, 'alpha': 0.445} 0.09259259259259259\n",
            "26 {'penalty': 'l2', 'eta0': 0.2223, 'alpha': 0.556} 0.09259259259259259\n",
            "27 {'penalty': 'l2', 'eta0': 0.2223, 'alpha': 0.667} 0.09259259259259259\n",
            "28 {'penalty': 'l2', 'eta0': 0.2223, 'alpha': 0.778} 0.09259259259259259\n",
            "29 {'penalty': 'l2', 'eta0': 0.2223, 'alpha': 0.889} 0.09259259259259259\n",
            "30 {'penalty': 'l2', 'eta0': 0.2223, 'alpha': 1.0} 0.09259259259259259\n",
            "31 {'penalty': 'l2', 'eta0': 0.33340000000000003, 'alpha': 0.001} 0.8400673400673401\n",
            "32 {'penalty': 'l2', 'eta0': 0.33340000000000003, 'alpha': 0.112} 0.09595959595959595\n",
            "33 {'penalty': 'l2', 'eta0': 0.33340000000000003, 'alpha': 0.223} 0.09259259259259259\n",
            "34 {'penalty': 'l2', 'eta0': 0.33340000000000003, 'alpha': 0.334} 0.09259259259259259\n",
            "35 {'penalty': 'l2', 'eta0': 0.33340000000000003, 'alpha': 0.445} 0.09259259259259259\n",
            "36 {'penalty': 'l2', 'eta0': 0.33340000000000003, 'alpha': 0.556} 0.09259259259259259\n",
            "37 {'penalty': 'l2', 'eta0': 0.33340000000000003, 'alpha': 0.667} 0.09259259259259259\n",
            "38 {'penalty': 'l2', 'eta0': 0.33340000000000003, 'alpha': 0.778} 0.09259259259259259\n",
            "39 {'penalty': 'l2', 'eta0': 0.33340000000000003, 'alpha': 0.889} 0.09259259259259259\n",
            "40 {'penalty': 'l2', 'eta0': 0.33340000000000003, 'alpha': 1.0} 0.09259259259259259\n",
            "41 {'penalty': 'l2', 'eta0': 0.4445, 'alpha': 0.001} 0.9276094276094277\n",
            "42 {'penalty': 'l2', 'eta0': 0.4445, 'alpha': 0.112} 0.10101010101010101\n",
            "43 {'penalty': 'l2', 'eta0': 0.4445, 'alpha': 0.223} 0.09259259259259259\n",
            "44 {'penalty': 'l2', 'eta0': 0.4445, 'alpha': 0.334} 0.09259259259259259\n",
            "45 {'penalty': 'l2', 'eta0': 0.4445, 'alpha': 0.445} 0.09259259259259259\n",
            "46 {'penalty': 'l2', 'eta0': 0.4445, 'alpha': 0.556} 0.09259259259259259\n",
            "47 {'penalty': 'l2', 'eta0': 0.4445, 'alpha': 0.667} 0.09259259259259259\n",
            "48 {'penalty': 'l2', 'eta0': 0.4445, 'alpha': 0.778} 0.09259259259259259\n",
            "49 {'penalty': 'l2', 'eta0': 0.4445, 'alpha': 0.889} 0.09259259259259259\n",
            "50 {'penalty': 'l2', 'eta0': 0.4445, 'alpha': 1.0} 0.09259259259259259\n",
            "51 {'penalty': 'l2', 'eta0': 0.5556, 'alpha': 0.001} 0.8518518518518519\n",
            "52 {'penalty': 'l2', 'eta0': 0.5556, 'alpha': 0.112} 0.09259259259259259\n",
            "53 {'penalty': 'l2', 'eta0': 0.5556, 'alpha': 0.223} 0.09259259259259259\n",
            "54 {'penalty': 'l2', 'eta0': 0.5556, 'alpha': 0.334} 0.09259259259259259\n",
            "55 {'penalty': 'l2', 'eta0': 0.5556, 'alpha': 0.445} 0.09259259259259259\n",
            "56 {'penalty': 'l2', 'eta0': 0.5556, 'alpha': 0.556} 0.09259259259259259\n",
            "57 {'penalty': 'l2', 'eta0': 0.5556, 'alpha': 0.667} 0.09259259259259259\n",
            "58 {'penalty': 'l2', 'eta0': 0.5556, 'alpha': 0.778} 0.09259259259259259\n",
            "59 {'penalty': 'l2', 'eta0': 0.5556, 'alpha': 0.889} 0.09259259259259259\n",
            "60 {'penalty': 'l2', 'eta0': 0.5556, 'alpha': 1.0} 0.09259259259259259\n",
            "61 {'penalty': 'l2', 'eta0': 0.6667000000000001, 'alpha': 0.001} 0.7727272727272727\n",
            "62 {'penalty': 'l2', 'eta0': 0.6667000000000001, 'alpha': 0.112} 0.09259259259259259\n",
            "63 {'penalty': 'l2', 'eta0': 0.6667000000000001, 'alpha': 0.223} 0.09259259259259259\n",
            "64 {'penalty': 'l2', 'eta0': 0.6667000000000001, 'alpha': 0.334} 0.09259259259259259\n",
            "65 {'penalty': 'l2', 'eta0': 0.6667000000000001, 'alpha': 0.445} 0.09259259259259259\n",
            "66 {'penalty': 'l2', 'eta0': 0.6667000000000001, 'alpha': 0.556} 0.09259259259259259\n",
            "67 {'penalty': 'l2', 'eta0': 0.6667000000000001, 'alpha': 0.667} 0.09259259259259259\n",
            "68 {'penalty': 'l2', 'eta0': 0.6667000000000001, 'alpha': 0.778} 0.09259259259259259\n",
            "69 {'penalty': 'l2', 'eta0': 0.6667000000000001, 'alpha': 0.889} 0.09259259259259259\n",
            "70 {'penalty': 'l2', 'eta0': 0.6667000000000001, 'alpha': 1.0} 0.09259259259259259\n",
            "71 {'penalty': 'l2', 'eta0': 0.7778, 'alpha': 0.001} 0.8063973063973064\n",
            "72 {'penalty': 'l2', 'eta0': 0.7778, 'alpha': 0.112} 0.09595959595959595\n",
            "73 {'penalty': 'l2', 'eta0': 0.7778, 'alpha': 0.223} 0.09259259259259259\n",
            "74 {'penalty': 'l2', 'eta0': 0.7778, 'alpha': 0.334} 0.09259259259259259\n",
            "75 {'penalty': 'l2', 'eta0': 0.7778, 'alpha': 0.445} 0.09259259259259259\n",
            "76 {'penalty': 'l2', 'eta0': 0.7778, 'alpha': 0.556} 0.09259259259259259\n",
            "77 {'penalty': 'l2', 'eta0': 0.7778, 'alpha': 0.667} 0.09259259259259259\n",
            "78 {'penalty': 'l2', 'eta0': 0.7778, 'alpha': 0.778} 0.09259259259259259\n",
            "79 {'penalty': 'l2', 'eta0': 0.7778, 'alpha': 0.889} 0.09259259259259259\n",
            "80 {'penalty': 'l2', 'eta0': 0.7778, 'alpha': 1.0} 0.09259259259259259\n",
            "81 {'penalty': 'l2', 'eta0': 0.8889, 'alpha': 0.001} 0.8905723905723906\n",
            "82 {'penalty': 'l2', 'eta0': 0.8889, 'alpha': 0.112} 0.09259259259259259\n",
            "83 {'penalty': 'l2', 'eta0': 0.8889, 'alpha': 0.223} 0.09259259259259259\n",
            "84 {'penalty': 'l2', 'eta0': 0.8889, 'alpha': 0.334} 0.09259259259259259\n",
            "85 {'penalty': 'l2', 'eta0': 0.8889, 'alpha': 0.445} 0.09259259259259259\n",
            "86 {'penalty': 'l2', 'eta0': 0.8889, 'alpha': 0.556} 0.09259259259259259\n",
            "87 {'penalty': 'l2', 'eta0': 0.8889, 'alpha': 0.667} 0.09259259259259259\n",
            "88 {'penalty': 'l2', 'eta0': 0.8889, 'alpha': 0.778} 0.09259259259259259\n",
            "89 {'penalty': 'l2', 'eta0': 0.8889, 'alpha': 0.889} 0.09259259259259259\n",
            "90 {'penalty': 'l2', 'eta0': 0.8889, 'alpha': 1.0} 0.09259259259259259\n",
            "91 {'penalty': 'l2', 'eta0': 1.0, 'alpha': 0.001} 0.8737373737373737\n",
            "92 {'penalty': 'l2', 'eta0': 1.0, 'alpha': 0.112} 0.09259259259259259\n",
            "93 {'penalty': 'l2', 'eta0': 1.0, 'alpha': 0.223} 0.09259259259259259\n",
            "94 {'penalty': 'l2', 'eta0': 1.0, 'alpha': 0.334} 0.09259259259259259\n",
            "95 {'penalty': 'l2', 'eta0': 1.0, 'alpha': 0.445} 0.09259259259259259\n",
            "96 {'penalty': 'l2', 'eta0': 1.0, 'alpha': 0.556} 0.09259259259259259\n",
            "97 {'penalty': 'l2', 'eta0': 1.0, 'alpha': 0.667} 0.09259259259259259\n",
            "98 {'penalty': 'l2', 'eta0': 1.0, 'alpha': 0.778} 0.09259259259259259\n",
            "99 {'penalty': 'l2', 'eta0': 1.0, 'alpha': 0.889} 0.09259259259259259\n",
            "100 {'penalty': 'l2', 'eta0': 1.0, 'alpha': 1.0} 0.09259259259259259\n",
            "101 {'penalty': 'l1', 'eta0': 0.0001, 'alpha': 0.001} 0.9612794612794613\n",
            "102 {'penalty': 'l1', 'eta0': 0.0001, 'alpha': 0.112} 0.9242424242424242\n",
            "103 {'penalty': 'l1', 'eta0': 0.0001, 'alpha': 0.223} 0.8703703703703703\n",
            "104 {'penalty': 'l1', 'eta0': 0.0001, 'alpha': 0.334} 0.835016835016835\n",
            "105 {'penalty': 'l1', 'eta0': 0.0001, 'alpha': 0.445} 0.8215488215488216\n",
            "106 {'penalty': 'l1', 'eta0': 0.0001, 'alpha': 0.556} 0.8164983164983165\n",
            "107 {'penalty': 'l1', 'eta0': 0.0001, 'alpha': 0.667} 0.803030303030303\n",
            "108 {'penalty': 'l1', 'eta0': 0.0001, 'alpha': 0.778} 0.7878787878787878\n",
            "109 {'penalty': 'l1', 'eta0': 0.0001, 'alpha': 0.889} 0.7777777777777778\n",
            "110 {'penalty': 'l1', 'eta0': 0.0001, 'alpha': 1.0} 0.7441077441077442\n",
            "111 {'penalty': 'l1', 'eta0': 0.11120000000000001, 'alpha': 0.001} 0.9377104377104377\n",
            "112 {'penalty': 'l1', 'eta0': 0.11120000000000001, 'alpha': 0.112} 0.7356902356902357\n",
            "113 {'penalty': 'l1', 'eta0': 0.11120000000000001, 'alpha': 0.223} 0.3888888888888889\n",
            "114 {'penalty': 'l1', 'eta0': 0.11120000000000001, 'alpha': 0.334} 0.25252525252525254\n",
            "115 {'penalty': 'l1', 'eta0': 0.11120000000000001, 'alpha': 0.445} 0.2558922558922559\n",
            "116 {'penalty': 'l1', 'eta0': 0.11120000000000001, 'alpha': 0.556} 0.19528619528619529\n",
            "117 {'penalty': 'l1', 'eta0': 0.11120000000000001, 'alpha': 0.667} 0.19696969696969696\n",
            "118 {'penalty': 'l1', 'eta0': 0.11120000000000001, 'alpha': 0.778} 0.19696969696969696\n",
            "119 {'penalty': 'l1', 'eta0': 0.11120000000000001, 'alpha': 0.889} 0.1919191919191919\n",
            "120 {'penalty': 'l1', 'eta0': 0.11120000000000001, 'alpha': 1.0} 0.1835016835016835\n",
            "121 {'penalty': 'l1', 'eta0': 0.2223, 'alpha': 0.001} 0.9343434343434344\n",
            "122 {'penalty': 'l1', 'eta0': 0.2223, 'alpha': 0.112} 0.6060606060606061\n",
            "123 {'penalty': 'l1', 'eta0': 0.2223, 'alpha': 0.223} 0.4175084175084175\n",
            "124 {'penalty': 'l1', 'eta0': 0.2223, 'alpha': 0.334} 0.21043771043771045\n",
            "125 {'penalty': 'l1', 'eta0': 0.2223, 'alpha': 0.445} 0.2053872053872054\n",
            "126 {'penalty': 'l1', 'eta0': 0.2223, 'alpha': 0.556} 0.24074074074074073\n",
            "127 {'penalty': 'l1', 'eta0': 0.2223, 'alpha': 0.667} 0.19696969696969696\n",
            "128 {'penalty': 'l1', 'eta0': 0.2223, 'alpha': 0.778} 0.19696969696969696\n",
            "129 {'penalty': 'l1', 'eta0': 0.2223, 'alpha': 0.889} 0.19528619528619529\n",
            "130 {'penalty': 'l1', 'eta0': 0.2223, 'alpha': 1.0} 0.19528619528619529\n",
            "131 {'penalty': 'l1', 'eta0': 0.33340000000000003, 'alpha': 0.001} 0.9562289562289562\n",
            "132 {'penalty': 'l1', 'eta0': 0.33340000000000003, 'alpha': 0.112} 0.7508417508417509\n",
            "133 {'penalty': 'l1', 'eta0': 0.33340000000000003, 'alpha': 0.223} 0.3771043771043771\n",
            "134 {'penalty': 'l1', 'eta0': 0.33340000000000003, 'alpha': 0.334} 0.2878787878787879\n",
            "135 {'penalty': 'l1', 'eta0': 0.33340000000000003, 'alpha': 0.445} 0.1919191919191919\n",
            "136 {'penalty': 'l1', 'eta0': 0.33340000000000003, 'alpha': 0.556} 0.17003367003367004\n",
            "137 {'penalty': 'l1', 'eta0': 0.33340000000000003, 'alpha': 0.667} 0.1531986531986532\n",
            "138 {'penalty': 'l1', 'eta0': 0.33340000000000003, 'alpha': 0.778} 0.19696969696969696\n",
            "139 {'penalty': 'l1', 'eta0': 0.33340000000000003, 'alpha': 0.889} 0.19528619528619529\n",
            "140 {'penalty': 'l1', 'eta0': 0.33340000000000003, 'alpha': 1.0} 0.19528619528619529\n",
            "141 {'penalty': 'l1', 'eta0': 0.4445, 'alpha': 0.001} 0.9427609427609428\n",
            "142 {'penalty': 'l1', 'eta0': 0.4445, 'alpha': 0.112} 0.6279461279461279\n",
            "143 {'penalty': 'l1', 'eta0': 0.4445, 'alpha': 0.223} 0.5117845117845118\n",
            "144 {'penalty': 'l1', 'eta0': 0.4445, 'alpha': 0.334} 0.5117845117845118\n",
            "145 {'penalty': 'l1', 'eta0': 0.4445, 'alpha': 0.445} 0.1919191919191919\n",
            "146 {'penalty': 'l1', 'eta0': 0.4445, 'alpha': 0.556} 0.20875420875420875\n",
            "147 {'penalty': 'l1', 'eta0': 0.4445, 'alpha': 0.667} 0.13131313131313133\n",
            "148 {'penalty': 'l1', 'eta0': 0.4445, 'alpha': 0.778} 0.19528619528619529\n",
            "149 {'penalty': 'l1', 'eta0': 0.4445, 'alpha': 0.889} 0.19528619528619529\n",
            "150 {'penalty': 'l1', 'eta0': 0.4445, 'alpha': 1.0} 0.19528619528619529\n",
            "151 {'penalty': 'l1', 'eta0': 0.5556, 'alpha': 0.001} 0.9393939393939394\n",
            "152 {'penalty': 'l1', 'eta0': 0.5556, 'alpha': 0.112} 0.7727272727272727\n",
            "153 {'penalty': 'l1', 'eta0': 0.5556, 'alpha': 0.223} 0.5488215488215489\n",
            "154 {'penalty': 'l1', 'eta0': 0.5556, 'alpha': 0.334} 0.2996632996632997\n",
            "155 {'penalty': 'l1', 'eta0': 0.5556, 'alpha': 0.445} 0.25084175084175087\n",
            "156 {'penalty': 'l1', 'eta0': 0.5556, 'alpha': 0.556} 0.20707070707070707\n",
            "157 {'penalty': 'l1', 'eta0': 0.5556, 'alpha': 0.667} 0.19696969696969696\n",
            "158 {'penalty': 'l1', 'eta0': 0.5556, 'alpha': 0.778} 0.09259259259259259\n",
            "159 {'penalty': 'l1', 'eta0': 0.5556, 'alpha': 0.889} 0.19528619528619529\n",
            "160 {'penalty': 'l1', 'eta0': 0.5556, 'alpha': 1.0} 0.19528619528619529\n",
            "161 {'penalty': 'l1', 'eta0': 0.6667000000000001, 'alpha': 0.001} 0.9427609427609428\n",
            "162 {'penalty': 'l1', 'eta0': 0.6667000000000001, 'alpha': 0.112} 0.5875420875420876\n",
            "163 {'penalty': 'l1', 'eta0': 0.6667000000000001, 'alpha': 0.223} 0.3771043771043771\n",
            "164 {'penalty': 'l1', 'eta0': 0.6667000000000001, 'alpha': 0.334} 0.2558922558922559\n",
            "165 {'penalty': 'l1', 'eta0': 0.6667000000000001, 'alpha': 0.445} 0.19528619528619529\n",
            "166 {'penalty': 'l1', 'eta0': 0.6667000000000001, 'alpha': 0.556} 0.2239057239057239\n",
            "167 {'penalty': 'l1', 'eta0': 0.6667000000000001, 'alpha': 0.667} 0.12962962962962962\n",
            "168 {'penalty': 'l1', 'eta0': 0.6667000000000001, 'alpha': 0.778} 0.19528619528619529\n",
            "169 {'penalty': 'l1', 'eta0': 0.6667000000000001, 'alpha': 0.889} 0.19528619528619529\n",
            "170 {'penalty': 'l1', 'eta0': 0.6667000000000001, 'alpha': 1.0} 0.19528619528619529\n",
            "171 {'penalty': 'l1', 'eta0': 0.7778, 'alpha': 0.001} 0.9393939393939394\n",
            "172 {'penalty': 'l1', 'eta0': 0.7778, 'alpha': 0.112} 0.7744107744107744\n",
            "173 {'penalty': 'l1', 'eta0': 0.7778, 'alpha': 0.223} 0.3501683501683502\n",
            "174 {'penalty': 'l1', 'eta0': 0.7778, 'alpha': 0.334} 0.3468013468013468\n",
            "175 {'penalty': 'l1', 'eta0': 0.7778, 'alpha': 0.445} 0.1936026936026936\n",
            "176 {'penalty': 'l1', 'eta0': 0.7778, 'alpha': 0.556} 0.14983164983164984\n",
            "177 {'penalty': 'l1', 'eta0': 0.7778, 'alpha': 0.667} 0.20202020202020202\n",
            "178 {'penalty': 'l1', 'eta0': 0.7778, 'alpha': 0.778} 0.18686868686868688\n",
            "179 {'penalty': 'l1', 'eta0': 0.7778, 'alpha': 0.889} 0.1936026936026936\n",
            "180 {'penalty': 'l1', 'eta0': 0.7778, 'alpha': 1.0} 0.19528619528619529\n",
            "181 {'penalty': 'l1', 'eta0': 0.8889, 'alpha': 0.001} 0.9511784511784511\n",
            "182 {'penalty': 'l1', 'eta0': 0.8889, 'alpha': 0.112} 0.531986531986532\n",
            "183 {'penalty': 'l1', 'eta0': 0.8889, 'alpha': 0.223} 0.4663299663299663\n",
            "184 {'penalty': 'l1', 'eta0': 0.8889, 'alpha': 0.334} 0.20707070707070707\n",
            "185 {'penalty': 'l1', 'eta0': 0.8889, 'alpha': 0.445} 0.5218855218855218\n",
            "186 {'penalty': 'l1', 'eta0': 0.8889, 'alpha': 0.556} 0.22727272727272727\n",
            "187 {'penalty': 'l1', 'eta0': 0.8889, 'alpha': 0.667} 0.13131313131313133\n",
            "188 {'penalty': 'l1', 'eta0': 0.8889, 'alpha': 0.778} 0.18855218855218855\n",
            "189 {'penalty': 'l1', 'eta0': 0.8889, 'alpha': 0.889} 0.19528619528619529\n",
            "190 {'penalty': 'l1', 'eta0': 0.8889, 'alpha': 1.0} 0.09259259259259259\n",
            "191 {'penalty': 'l1', 'eta0': 1.0, 'alpha': 0.001} 0.9444444444444444\n",
            "192 {'penalty': 'l1', 'eta0': 1.0, 'alpha': 0.112} 0.5521885521885522\n",
            "193 {'penalty': 'l1', 'eta0': 1.0, 'alpha': 0.223} 0.6885521885521886\n",
            "194 {'penalty': 'l1', 'eta0': 1.0, 'alpha': 0.334} 0.25084175084175087\n",
            "195 {'penalty': 'l1', 'eta0': 1.0, 'alpha': 0.445} 0.17845117845117844\n",
            "196 {'penalty': 'l1', 'eta0': 1.0, 'alpha': 0.556} 0.16835016835016836\n",
            "197 {'penalty': 'l1', 'eta0': 1.0, 'alpha': 0.667} 0.16835016835016836\n",
            "198 {'penalty': 'l1', 'eta0': 1.0, 'alpha': 0.778} 0.19023569023569023\n",
            "199 {'penalty': 'l1', 'eta0': 1.0, 'alpha': 0.889} 0.19023569023569023\n",
            "200 {'penalty': 'l1', 'eta0': 1.0, 'alpha': 1.0} 0.1936026936026936\n",
            "min val 0.09259259259259259, max_val : 0.9629629629629629\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d2-kcDlmau1C"
      },
      "source": [
        "# Hyperactive Simulated Annealing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 475
        },
        "id": "ErZ5-n62aj5z",
        "outputId": "5ab2c1e7-aec9-493e-afa8-174b73db5c8e"
      },
      "source": [
        "!pip install hyperactive"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting hyperactive\n",
            "  Downloading https://files.pythonhosted.org/packages/41/a1/4f5aa9c62a86d982bc114e2df374f9966bf6bab967d26bba585aea451b5d/hyperactive-3.1.0-py3-none-any.whl\n",
            "Collecting tqdm>=4.48.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b4/20/9f1e974bb4761128fc0d0a32813eaa92827309b1756c4b892d28adfb4415/tqdm-4.61.1-py2.py3-none-any.whl (75kB)\n",
            "\u001b[K     |████████████████████████████████| 81kB 8.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.18.1 in /usr/local/lib/python3.7/dist-packages (from hyperactive) (1.19.5)\n",
            "Collecting gradient-free-optimizers<1.0.0,>=0.2.6\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4f/6a/55fa1cc396c899110982f6a59843849eadecb74807b8d1a81d5b887274a3/gradient_free_optimizers-0.3.1-py3-none-any.whl (66kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 7.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from hyperactive) (1.1.5)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from gradient-free-optimizers<1.0.0,>=0.2.6->hyperactive) (1.4.1)\n",
            "Requirement already satisfied: scikit-learn!=0.23.*,>=0.21 in /usr/local/lib/python3.7/dist-packages (from gradient-free-optimizers<1.0.0,>=0.2.6->hyperactive) (0.22.2.post1)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->hyperactive) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->hyperactive) (2018.9)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn!=0.23.*,>=0.21->gradient-free-optimizers<1.0.0,>=0.2.6->hyperactive) (1.0.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->hyperactive) (1.15.0)\n",
            "Installing collected packages: tqdm, gradient-free-optimizers, hyperactive\n",
            "  Found existing installation: tqdm 4.41.1\n",
            "    Uninstalling tqdm-4.41.1:\n",
            "      Successfully uninstalled tqdm-4.41.1\n",
            "Successfully installed gradient-free-optimizers-0.3.1 hyperactive-3.1.0 tqdm-4.61.1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "tqdm"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rAs2Cqsta7l8",
        "outputId": "12bfb8d8-d6d3-4554-9792-1da277c02504"
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "from time import time\n",
        "import scipy.stats as stats\n",
        "from sklearn.utils.fixes import loguniform\n",
        "from scipy.stats import gamma\n",
        "\n",
        "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
        "from sklearn.datasets import load_digits\n",
        "from sklearn.linear_model import SGDClassifier\n",
        "from sklearn.model_selection import train_test_split\n",
        "import scipy\n",
        "\n",
        "from hyperactive import Hyperactive, SimulatedAnnealingOptimizer\n",
        "\n",
        "# get some data\n",
        "X, y = load_digits(return_X_y=True)\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)\n",
        "# build a classifier\n",
        "clf = SGDClassifier(loss='log',learning_rate='constant',eta0=1e-3,fit_intercept=True,shuffle=False)\n",
        "\n",
        "\n",
        "\n",
        "def model(par):\n",
        "    clf.set_params(**par)\n",
        "    clf.fit(X_train,y_train)\n",
        "    return clf.score(X_test,y_test)\n",
        "\n",
        "\n",
        "param_grid = {'penalty': ['l2','l1'],\n",
        "              'eta0': loguniform(1e-3,1e0).rvs(10000),\n",
        "              'alpha':gamma(.5,1e-3).rvs(10000)}\n",
        "\n",
        "\n",
        "\n",
        "optimizer = SimulatedAnnealingOptimizer(\n",
        "    epsilon=0.1,\n",
        "    distribution=\"laplace\",\n",
        "    n_neighbours=4,\n",
        "    rand_rest_p=0.1,\n",
        "    p_accept=0.15,\n",
        "    norm_factor=\"adaptive\",\n",
        "    annealing_rate=0.999,\n",
        "    start_temp=0.8,\n",
        ")\n",
        "\n",
        "hyper = Hyperactive()\n",
        "hyper.add_search(model, param_grid, optimizer=optimizer, n_iter=100)\n",
        "hyper.run()"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "                                                                                                                               "
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Results: 'model'  \n",
            "   Best score: 0.9545454545454546  \n",
            "   Best parameter set:\n",
            "      'penalty' : l2  \n",
            "      'eta0'    : 0.0012194055429015237  \n",
            "      'alpha'   : 0.09595628954124737  \n",
            "   Best iteration: 67  \n",
            " \n",
            "   Evaluation time   : 7.650762319564819 sec    [97.99 %]\n",
            "   Optimization time : 0.15677332878112793 sec    [2.01 %]\n",
            "   Iteration time    : 7.807535648345947 sec    [12.81 iter/sec]\n",
            " \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OPxO_cB924Y8"
      },
      "source": [
        "# Random Search\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OIz9bjdZ23uk",
        "outputId": "08cced6b-af1c-408f-cef3-bcee6ca9b850"
      },
      "source": [
        "from sklearn.model_selection import ParameterSampler\n",
        "\n",
        "def model(clf,X_train,y_train,X_test,y_test,par):\n",
        "  clf.set_params(**par)\n",
        "  clf.fit(X_train,y_train)\n",
        "  return clf.score(X_test,y_test)\n",
        "\n",
        "param_dist = {'penalty': ['l2','l1'],\n",
        "              'eta0': loguniform(1e-3,1e0),\n",
        "              'alpha':gamma(.5,1e-3)}\n",
        "\n",
        "param_list = list(ParameterSampler(param_dist, n_iter=100))\n",
        "\n",
        "max_val=-np.infty\n",
        "min_val=np.infty\n",
        "for i,par in enumerate(param_list):\n",
        "  current_val=model(clf,X_train,y_train,X_test,y_test,par)\n",
        "  min_val=min(current_val,min_val)\n",
        "  max_val=max(current_val,max_val)\n",
        "  print('Parametro {0}, score : {1}'.format(i,current_val))\n",
        "\n",
        "print('min val {0}, max_val : {1}'.format(min_val,max_val))"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Parametro 0, score : 0.27104377104377103\n",
            "Parametro 1, score : 0.09259259259259259\n",
            "Parametro 2, score : 0.09259259259259259\n",
            "Parametro 3, score : 0.877104377104377\n",
            "Parametro 4, score : 0.14309764309764308\n",
            "Parametro 5, score : 0.09764309764309764\n",
            "Parametro 6, score : 0.8737373737373737\n",
            "Parametro 7, score : 0.5168350168350169\n",
            "Parametro 8, score : 0.6060606060606061\n",
            "Parametro 9, score : 0.9057239057239057\n",
            "Parametro 10, score : 0.9074074074074074\n",
            "Parametro 11, score : 0.8703703703703703\n",
            "Parametro 12, score : 0.7070707070707071\n",
            "Parametro 13, score : 0.9562289562289562\n",
            "Parametro 14, score : 0.7138047138047138\n",
            "Parametro 15, score : 0.19696969696969696\n",
            "Parametro 16, score : 0.9057239057239057\n",
            "Parametro 17, score : 0.8552188552188552\n",
            "Parametro 18, score : 0.8063973063973064\n",
            "Parametro 19, score : 0.9562289562289562\n",
            "Parametro 20, score : 0.5185185185185185\n",
            "Parametro 21, score : 0.1936026936026936\n",
            "Parametro 22, score : 0.8417508417508418\n",
            "Parametro 23, score : 0.10774410774410774\n",
            "Parametro 24, score : 0.5\n",
            "Parametro 25, score : 0.8956228956228957\n",
            "Parametro 26, score : 0.1936026936026936\n",
            "Parametro 27, score : 0.6801346801346801\n",
            "Parametro 28, score : 0.7912457912457912\n",
            "Parametro 29, score : 0.09259259259259259\n",
            "Parametro 30, score : 0.7272727272727273\n",
            "Parametro 31, score : 0.6616161616161617\n",
            "Parametro 32, score : 0.21885521885521886\n",
            "Parametro 33, score : 0.8316498316498316\n",
            "Parametro 34, score : 0.8653198653198653\n",
            "Parametro 35, score : 0.9478114478114478\n",
            "Parametro 36, score : 0.7407407407407407\n",
            "Parametro 37, score : 0.17003367003367004\n",
            "Parametro 38, score : 0.9006734006734006\n",
            "Parametro 39, score : 0.38215488215488214\n",
            "Parametro 40, score : 0.1531986531986532\n",
            "Parametro 41, score : 0.09259259259259259\n",
            "Parametro 42, score : 0.10774410774410774\n",
            "Parametro 43, score : 0.4208754208754209\n",
            "Parametro 44, score : 0.6835016835016835\n",
            "Parametro 45, score : 0.09259259259259259\n",
            "Parametro 46, score : 0.9023569023569024\n",
            "Parametro 47, score : 0.6599326599326599\n",
            "Parametro 48, score : 0.5454545454545454\n",
            "Parametro 49, score : 0.17676767676767677\n",
            "Parametro 50, score : 0.09259259259259259\n",
            "Parametro 51, score : 0.9191919191919192\n",
            "Parametro 52, score : 0.34175084175084175\n",
            "Parametro 53, score : 0.8872053872053872\n",
            "Parametro 54, score : 0.8973063973063973\n",
            "Parametro 55, score : 0.9393939393939394\n",
            "Parametro 56, score : 0.7003367003367004\n",
            "Parametro 57, score : 0.39225589225589225\n",
            "Parametro 58, score : 0.8451178451178452\n",
            "Parametro 59, score : 0.09259259259259259\n",
            "Parametro 60, score : 0.6666666666666666\n",
            "Parametro 61, score : 0.1919191919191919\n",
            "Parametro 62, score : 0.7710437710437711\n",
            "Parametro 63, score : 0.6616161616161617\n",
            "Parametro 64, score : 0.09259259259259259\n",
            "Parametro 65, score : 0.09259259259259259\n",
            "Parametro 66, score : 0.9410774410774411\n",
            "Parametro 67, score : 0.8737373737373737\n",
            "Parametro 68, score : 0.12626262626262627\n",
            "Parametro 69, score : 0.3956228956228956\n",
            "Parametro 70, score : 0.45454545454545453\n",
            "Parametro 71, score : 0.5673400673400674\n",
            "Parametro 72, score : 0.09427609427609428\n",
            "Parametro 73, score : 0.6919191919191919\n",
            "Parametro 74, score : 0.9461279461279462\n",
            "Parametro 75, score : 0.19696969696969696\n",
            "Parametro 76, score : 0.8013468013468014\n",
            "Parametro 77, score : 0.265993265993266\n",
            "Parametro 78, score : 0.7760942760942761\n",
            "Parametro 79, score : 0.37542087542087543\n",
            "Parametro 80, score : 0.2356902356902357\n",
            "Parametro 81, score : 0.8249158249158249\n",
            "Parametro 82, score : 0.09259259259259259\n",
            "Parametro 83, score : 0.36363636363636365\n",
            "Parametro 84, score : 0.7104377104377104\n",
            "Parametro 85, score : 0.10774410774410774\n",
            "Parametro 86, score : 0.8737373737373737\n",
            "Parametro 87, score : 0.8181818181818182\n",
            "Parametro 88, score : 0.9292929292929293\n",
            "Parametro 89, score : 0.42424242424242425\n",
            "Parametro 90, score : 0.930976430976431\n",
            "Parametro 91, score : 0.5437710437710438\n",
            "Parametro 92, score : 0.7996632996632996\n",
            "Parametro 93, score : 0.7188552188552189\n",
            "Parametro 94, score : 0.4882154882154882\n",
            "Parametro 95, score : 0.601010101010101\n",
            "Parametro 96, score : 0.835016835016835\n",
            "Parametro 97, score : 0.6616161616161617\n",
            "Parametro 98, score : 0.2474747474747475\n",
            "Parametro 99, score : 0.9175084175084175\n",
            "min val 0.09259259259259259, max_val : 0.9562289562289562\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5AnJ_vCi3PG6"
      },
      "source": [
        "# Simulated Annealing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EmEjj1wS3Rr0"
      },
      "source": [
        "import math\n",
        "\n",
        "class temperature_scheduling:\n",
        "    \n",
        "    def __init__(self,initial_temperature,min_temperature,gamma):\n",
        "        self.initial_temperature=initial_temperature\n",
        "        self.min_temperature=min_temperature\n",
        "        self.temperature=initial_temperature\n",
        "        self.gamma=gamma\n",
        "        \n",
        "    def exponential_decay(self):\n",
        "        self.temperature=self.gamma*self.temperature\n",
        "        return max(self.temperature,self.min_temperature)\n",
        "    \n",
        "    def fast_decay(self,k):\n",
        "        self.temperature=self.initial_temperature/k\n",
        "        return max(self.temperature,self.min_temperature)\n",
        "    \n",
        "    def logarithmic_decay(self,k):\n",
        "        self.temperature=self.initial_temperature*math.log(2,2)/math.log(k+1,2)\n",
        "        return max(self.temperature,self.min_temperature)"
      ],
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bc-HNwe23lPk"
      },
      "source": [
        "from sklearn.model_selection import ParameterSampler\n",
        "import copy \n",
        "\n",
        "def mutation(param_dist,par,prior=False):\n",
        "  new_par=copy.deepcopy(par) \n",
        "  param_to_update = np.random.choice(list(param_dist.keys()))\n",
        "  param_vals = param_dist[param_to_update]\n",
        "  if isinstance(param_vals,list):\n",
        "    new_par[param_to_update]=np.random.choice(param_vals)\n",
        "  else:\n",
        "    if prior:\n",
        "      new_par[param_to_update]=param_vals.rvs()\n",
        "    else:\n",
        "      new_par[param_to_update]=np.abs(par[param_to_update]+np.random.normal(0,1e-1))\n",
        "  return new_par\n",
        "\n",
        "param_list = list(ParameterSampler(param_dist, n_iter=1))"
      ],
      "execution_count": 56,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hoy1gXZS3dPE"
      },
      "source": [
        "k=0.85\n",
        "initial_temperature=1.2*(max_val)\n",
        "min_temp=1e-4\n",
        "\n",
        "#par=param_list[0]\n",
        "temp=temperature_scheduling(initial_temperature,min_temp,k)\n",
        "current_val=score(clf,X_train,y_train,X_test,y_test,par)\n",
        "epochs=1000\n",
        "loss=[]\n",
        "accept_prob=[]\n",
        "\n",
        "for i in range(1,epochs):\n",
        "    new_par=mutation(param_dist,par,prior=True)\n",
        "    next_value=score(clf,X_train,y_train,X_test,y_test,new_par)\n",
        "    delta = (next_value-current_val)\n",
        "    T=temp.logarithmic_decay(i)\n",
        "    if delta>0:\n",
        "        par=copy.deepcopy(new_par)\n",
        "        current_val=next_value\n",
        "    else:\n",
        "        alpha=min(1,np.exp(delta/T))\n",
        "        if alpha>np.random.random():\n",
        "            par=copy.deepcopy(new_par)\n",
        "            current_val=next_value\n",
        "        accept_prob.append(alpha)\n",
        "    loss.append(current_val)"
      ],
      "execution_count": 67,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 285
        },
        "id": "oJD3LVXz6YUM",
        "outputId": "ab1b7f86-f5e3-45b3-c425-c78b06d39179"
      },
      "source": [
        "plt.plot(loss)"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f36d4f8d450>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 68
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD7CAYAAAB68m/qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2dd5zcxPn/P492r/ju3O/cbVwxNqYYTG+mG5NACCmU/EgICcmXUFJIAoEQIIGQHpKQEEKAfBPKl1ASikMxJXSDaTa2Ma64925f2V3N7w9ppNFI2pX2du9udc/79bJvV6sy0sw8euYzz8yQEAIMwzBM5WN0dgIYhmGY0sAGnWEYJiGwQWcYhkkIbNAZhmESAht0hmGYhMAGnWEYJiEUNOhEdBcRbSCiD0J+JyL6HREtJqI5RHRQ6ZPJMAzDFCKKh34PgGl5fj8NwDj738UA/tT+ZDEMwzBxSRfaQQjxEhGNzLPLmQD+V1gjlN4goj5ENFgIsTbfeRsbG8XIkflOyzAMw+i8/fbbm4QQTUG/FTToERgKYKXyfZW9La9BHzlyJGbPnl2CyzMMw3QfiOjjsN86tFOUiC4motlENHvjxo0deWmGYZjEUwqDvhrAcOX7MHubDyHEHUKIKUKIKU1NgS0GhmEYpkhKYdAfA3CBHe1yOIDthfRzhmEYpvQU1NCJ6H4AUwE0EtEqAD8CUAUAQojbAcwAMB3AYgB7AFxYrsQyDMMw4USJcjm3wO8CwDdKliKGYRimKHikKMMwTEJgg84wDJMQupVBz+ZM3PnyUizesLOzk8IwDFNyupVBn792B37y5AJ896E5efe7d9bHeHC2O1Zq8YaduOCuN/HTGQvKncSS8MqiTfjK32Zj9bZmfLB6O56Ztw7z1mzv7GShuS2H91duAwCs2daM5Zt2d2p6lm3ajYfeXoXWbK7d59q+J4NvP/genp2/HjtbMiVIXTxypsCPn5iPlz7i8R1BCCHwxtLNiLrk5vJNu/HV/52NS+97B3e+vBQX/+9sbN/T8fkal1KMFO0UTFPg1N++hK17Mnjxu1PRUJP/Vna2ZLB0o2VAFq3fFbpfSyaHax615iH73BQrvP6kX78EAHjpo4248tTxqEr534NCCGzc2YoBvWoLpn3t9mY8/PYqXDJ1LAyDQvfbursNPapTqK1KFTynyv1vrsDMBesxc8F632/3fuUwHDmmP4is6576m5dw7N6NuOb0iQCs53rKb1/C4g3WM+pbV4WrT5uAwX1qccw4/9gB0xT46yvL8OmDhqJ/Qw0AYMvuNmRzJhau34mL//dtEAHfn7YPZsxdi1nLtniOX3LzdKTyPINCmKbArc8twsebd+Ow0f1x7qEjIh97xh9ewc6WLK785/s4acIA7DukN+prUrj42DGB+98762Nc8+gH+NnZ+2H11mYcMaYRR4zpDwB4dckmPPLOajzyjjUE46KjR+GHn5gY+15uf2kJzj90L/Suq3K2N7flMOG6p3DrOQfiuL2b0KM6hZq0t0y8/fFW/PWVZZj98VYcu3dlj/HY3pzBrTMXYcrIvnj8/TWor0njl589INKxby7bgu8/PAfHjmvEDWdOcrY/PW8dvv6Pd3DTWZNw/mF75T3H0o27cMKv/ut8f2KOFYV99PurccERI0OPO/13L+PA4X1w01n7RUprOahYg54xTSyyjc6tMz9yDJLkvZXb8IfnF+GP5x+M6rSBC+56E++usLzD3W3Z0PNKoy/ZsrvN8/2dj7fisNH9fcc9+u5qfPvB9/HoJUdi8oi+edN+zh1v4OPNe3DKvoOw98Cevt9//9wizP54K/770UaMG9CAZ799XN7z6fTqURX62/l3zsI9Fx6CqeMHAAAWrt+Jhet34owDhuLRd1dj4pBejjEHgK17Mvjew1aLZvktp3vOlc2ZmHT902jJmFi6aRe2N2ewbNMeLFi7A1UpwrdO3hvNGcv7/dFj8wLTs2FnCwb37gHA8jJzpkB1OnrDcfHGXbj1uUUAgH+9tyaWQW/LmgCACYN7YeaCDZi5YAMA4LRJgzG8X51vf9m6+M8H6/Diwo24+7XlmHv9qQAA3fH76yvL8IPpE2K9rOau3o6fP7UQP39qIRbddJrjOKzZ3gwAuHXmIlzxwHs4fHQ/PHDxEb5jAxPSBdnZkoFpAmQAvWr9ZfW5Betx16vL8K/3Vjv1r1dtFa77ZPALcu6q7fjhvz/AjWfui3teW4Zlm3Zj2abduOb0iU5Z2rCzFQDw2pLNjkFvy5pIGeTLoxVb9gReZ5N9jjDmrdmBeWt2sEEvhpzpFty/vLwMV5463uO1fOq2VwEA1zw6Fz//zP6OMQfCy/y7K7birD++5tm2bY/XoM9csB6LNuzC7OVbcN0n90W/+moA7otg5oL1BQ362u0tACwPXKcta+JXz37kfF+0wduaWLpxF26e8SE+XLcDT152jMeTk5imQF11Crd/4WCM6FeH1qyJdIrw7optuPKf72PTLv91r/33B47Bisq3HnwfLRnLKLZmTMyYu875LZMT2LYng5q0gf9ccQxueHw+/hsgB2RzAi8s3IDL7nsXu1qzaGyoxuxrT46chsXa89ndmkW91loTQmDemh3Yd0gvp2UCAHXVKXx2yjCkiLBg7Q5n+4K1OwIN+o5myxHYZje9d7a4jsHuVr+TcNqtL+Hpbx7ruWY+1BfZeyu34ZCR/QBYzwgA0inrPG8s9bZynpizBj9+Yj4A62V+36wV+N1zi/DE5UejoSaNN5ZuxqShvdFot6DKxfY9GXy4bgcOHdUPRIS7X12Gp+etw4rNe7BmewuW3jwdMz5Yi0vve9c55udn74/PHTIca7c34+YZH6I1k8MyW4pT7exdry4LNegzF6zHeyu34bkFG7CnzZXP9r72P/j05KE4bnwTrvu35VA8OWctjh67AqdMHIiDfzITh4zsi39+/UjP+cwQA5EL2f7ChxuwaVd+Y99RVKyGLg36sL6Wd7ctRN/659ursDHgYW8MeNte+c/3fdvUAgIApgCu/dcH+Nd7azwGUKYj6LwA8KN/f4DL77cKcq1dcbcGpHnzbv/xF979pqP9PTVvHWYuWI9VW5vxu+cXBV4rY5roW1eNY/duwsjGeowf1BNjmhpw7LhGAJasBADXK16zbsxPmzQIp00a5Nl258tLnc9f//vbePz9Nc73R971z/awbU8b+tZVY3RTA/YZ5G+JAEDWFPjOg+9jl20QN+1qQyZnBu4bhP6897/hGZ+Gfdery/GJ37+C15dudraZpsD25gz61lX7JK2wCr3DPu+OZvf8//OPtwEAOwMM+kfrd+HyB96LfC/qddVylzWt55Eygqur2qpszZj4waNzsW5HCx5/fw0eeWc1vnT3W/jBI3OdfVZva8ZRtzyPY37+PA768bM46MfPYuRVT2L/65/GLf/5MHJ6VZrbcjjgxmfw+TvewC+eXoijbnkeNzw+H28s3YI1tgNjCuGkVcpRq7dZrY/Xl2zG4++vwZKNuxyPWW8dhyHrhkGEZq2+PvLualyh5cHVj8zFwT+ZCQB4a/lW3PXKMvzpxSVOi00vfvNuOBVVKYIZ4gheeM9bBfvlvv/QHEz77Uv4aH15AzIq0qC/tmQTfmoXvCF2c/07D/qNsWTV1mbftqBmlVqADLIKyi6toqotAwH3s3TCsiG5/rfXP8ZjtgGssQ3IzAXrMUsxMnoaJC8s3OgU/LXbWpztQVq+TGNVyu8VyutKg37Pa8s9v3/qwCHO52+dvDe+dORIz+8/edLtFJ79sddLDGLbngz62C2IMBkomzN996y/RPMhJZ0/nW+tq5IzBdbv8Br5Z+evc9IjuXfWxzAF0NhQ4zwXSdj7ZLttyLcrBv0/H1jn1l8iN51l6bdxOqNN5bqZrPvF8dBD5JvdbVnUpA2cuM8Aj5zY2FCDrXYL85n563H2n17D5//8Oh59ZxVWb2vGyi3NSBuEAT0tz31HSxYvLtwQOb0qP3vKfRH88cUlTnlVEUpaLzp6lFPHADcf7/vq4Xjqm8fia8eN9hlQue8PHp2LT/z+ZYy86kmc/afX8LvnFwOw6qw8j+Rrx43G45cejae+eUxo2m98Yj5+9tSHePidVQD8L/T6mjQMIphhFj2AnS0ZXHj3m3jKLh/PL9yAD9ftxD/eCJ0osSRUpEE/7y+zcN+sFQCAHtVWZXxl8SbPPmor9/kF/kK6fkcLJl73FO6btQLrd7SgLWsimxMY2qcHBvaqgSmAlozpa0qrBt1TAe1KJyufZN6a7fjeQ96XjezAfejtVfjyPW95flu5xV8RAFfvlXINAPSsDVbMsqYI1G57aAZdp199DR6/9Gjcc+EhGDegwXeOo8c2Op/3tOXwlaNHeX6/+rR9cJzSIbenLed4v71tg37UWG//QybnryTNbTks3rAL763cVrASSY+sry19Wef0WuSetk4rjW5zWw4/e2ohAOBTk4eitspbDfSmdc4U+MPzizBvjSXLbGv2Gu8H31qJ385chGrlBdu7RxXOOGCIJ/2Pv78GI6960vH0dVRDot6DzC81O9SX4O7WLBpq0qivSTtplOdTPda125oxa9kW/PIZV9IbP6gnjhvv5llYCzMfu1qzPucgCCHctFr343q9Mp2yvGzaGSBH5ky0ZHK4b9YKfLDauk/VMTMMr4f+yCVH4urTJmC/Yb2xz6BeBdMn60dQmUsZ5Kn7Kr0DnJU5q7bjhYUb8Tu7f0fec1s2euuzGCrSoKscOqpf4Ha1cv3hhcW+3y+59x3sacvhB4/OxWE3P4erH5mL1pyJTx4wBF89ZjQAq6mr6qTWNsWgKxVQZrae6f96dzUenL3Ksy1nCpw0YSC+dORI7G7LYeovXnA8I+klSM60vWZp+PYoHliQQZ+9fAuenLMWbQFuZlWKYJD1ogoy6lUpwn7DemPq+AEgIp/2K7+apsCetpxPqx7cpwfu/OIUx7Nvy5nOS6HJ9gL1F15QJVm7vRkn/fq/+NRtr+L5D/N7jC3ZHGrSBuqqXS9bv4bseFuycTc27GzBr55ZiF2tWVx+wlj07lHlixjRK/StMz9yjODYAQ2+NMtO49+de6CzzSBCOkWe8vLHF5cAAFZsDu50U8uTmn/S61Svulppde5uzaGuJuV54QKWkVTDb88/3B/dUZM2PPe/eXc8yQsANtuS5j6DeuKFK6dizvWnoL/ygpWYQjhpBaRBt+6q1TZ08uW6xZYeT99/sOd+NmutuZMnDvR8Vz30QpFvOve9uQK/fHphoFaeIgrV0Af39ke2zbdfrPX2vQrtPstFxRv0vnXV+OIRe/nekjUxIiUAS/5oy5qoSRuOITMFfE2knOKWq/VaVtys6c2wIAmmOZNDU89q9LIN8vLNe/CO3Wmrp/sou5LKSqYaK1X7FULgxsfn4zO3vw4g2NMnItRWpdCSyeHJOe6EmPW2MdRfELqTL6+9x640srBKhvSuRVXKwIBeNU6aU/azHN7X6mTc2ZL1yEEZ01/A1Y7glVuDjZ+kxW4FqM9Cf5lJY3/HS0tx6E3P4c5XlgEALjl+LAD4PXQtz15dYslivXtU4ZrTJzjbv3iEayA/P2U4pk1yjY9BhHQery4IdVfVk5Mdz6o9UVtPu1qzqK9O43OHDMe5h7ozWd88Y4ET3QEAoxvrfdesSad8ZW57c/R469v/uwRvLd8KAPifqWMwqrEevWqrQgMDZFoBy0FQPXSDXEfsPDsS5agx3lbh60u8EqVutFUPvTpEkgSAOdefghevnIrlt5yO/1xhyTFvLtuCP7yw2HneKkThwRS6zPPUB+twkz1mZUBPy9jLQx99d3VZx4RUnEH/q10ZJSnDMlT6gIHqdLzYbel9VqcNx5AJIXxeqGqghcdDlx0q+T3QXa1ZbNzZitqqlCcGXZ5LvY1JQ3s5+qY06BnTxNgBDQCspviEHz6Fx99fg9asibte9T6bIHpUpdCcyXk03zMnD8VvPn8ALtA0c0Pz0OXLauE6q2MnyEMH4BjxTM6E7McbP6gnPnvwMPz8M/t7jK/uTQNW2Jpkc0BEjkpzJoceVSnUptVzBntBFx87Gnv1t14sA3vVOOmo1cqK7onJF1DKIExVJCV1zMEtZ3tD1axwOCNQUgpqPQHe8tSaNXHbC4tx/WPzcP+bK3z7Ss82mzPx7Pz1zr1IYwlYmrjkt58/ECdMGOA7T02V4TPoO5oz2Liz1dd/FMQt//nQCSZIK522MkhAT/OeNq/komroPapSjjN18sSBWH7L6Z7zXPfvebhZG9yn3u99s1Z4PPi+Aa0ESa/aKoy0X3B6f9PmgCCKfJKL3uej9h/Ic6tF6vTfvYLZywv3QRVDxRn0CYO90RK60ZHE9dDV4+QZTeHvIFE9ADV/ZcXVK7BeCGbYnvHopgbH8FnnEp6/EtnxqWr08t7WbW9BcyaHn85YELkj0fLQvQalNp3CWZOH+WKC9Wcr03Ddv62BV7ocNdB++UjvUcb5ym2/+OwBmDS0txPqad2P37g9Pc816H94YTGe/9A/QErSkjGtwVfVbn7reSAg0K++Gj+YPgEj+9f77q2PFvqpSy7SOTAIHhlKykgAfPKUQbA9dP/97WkNziv1sh9v3o1fPL0QD7y1Au+u2OpLpywmu+1zDbVfpmGD0D41eahPWgIsD10/ZltzBofcNBOf+dNrvv3zkVYMY1C9FHa/lLyeQW55b8nknP4wldFN9U4L8uPNVoTMcXs3Oc++QWlVSkM689vHYu71pwRq24Hp1qKHglrVKSNccmnR6p6UM2VfHOANoAD8411KRcUZ9CPHNOKKE8c532Uh0h91nMEpgFtBahTPWQjhM8i7PQY9WENfu73Z8W704+ev3YGGmjS+cNgIj4cud1PPSSDFoJv470cbMXf1difm1T3Gq63nozptoC1nep6XXtgkepScvJfJI/oAAD590FDntycuOxppO63SiGdyIrBiD+zperZhUUEAcK0tb7z00abA319bsgmPvb8GNWnDY5Quu/8dXHb/u/ib3VEnBJyXtJSV1FQdp42s1CtutV3G9HsZ0DM8rtsw/Bq6JCyv1LyX5eyWT++POdefikG9aj0vDZln8phDRloSx6ShhTv/ANfhsTR067OUKOQAmg/XxQuxUz3dILVj6542vLdym7Ofp1M0kwt84QzrW4d5N07D56YMczqTh/dzvfYGTfYb0LMGYwf0dDrCo6B3/ge1GvNFueiSyxt25FpNOuXkj1qkXr/6BHzukOEoBxVn0AFvxTKILLlEe9Z6p8ybPzgx0rlrUl4NXTfIe5RmqFoBZcV9ZfEmHPHT552Yc71CZ01Xp08ZQR66Nz2y8LflTDz8ttVhKsPyZHM1p0Uz5CNtELI501PAwrRBv4fuepv966sdfRAAJg3t7bkG4PXQVZp6uYZwnRK1A7iRBgDwlWNGo399dWgn3UP28zh54kCPbLJ1TwYvfLgBf7I7IQVcSU1WdNU4plMGrjptH4yzpSzTFPi/t1Y4zWL5UtWfR/966z4+ecAQ6KQ0DX3zrlbHc9MNgMQTh26XM+mwpFPk8QRlOZEvH+kcTJs0GC9eOdVz3m+fvLfvWvLFVluVcsI2e/Wwtv3fWyt9+0dB9XSDprT4xr3vIGcKbLHDR8n20Jdt2o1H3lntcyBU+tRVY3tzBjlTeFq2uuwXFtqZDz38V0qL4wY04MpTrGenduA+OWct7nplGYQQyORMXx2XzlxVipy6pdaxfNp+e6lIg64+j5RBIJDPx0wZhCFK73PQiEoVaRxrqrwaui6BqB66EMBrizdhxty1vqgRGU6lvxCEcI2JWjDdjA+WXLI54WnSAu47zDSFJ135SBmW1xilq86vodudokqkQuBxhvsSSgV46JOGuMZfRohIfvaZ/T3f0ykK9JgA69nu1b8O3zllvE8Hnb7fIMeLtR6p9bvjoWvJ+vpxY/B/X7OG0+9oyeL7D8/FuX95A4Bq0L3HVKcNvHXNSfhVwDwjBlkaejYnsGTjLky5aaYzAnJ3iOSiZr2U0KSBqkoZaMmqZc92AOw8MUKM3P9MHYPLlRatpM7WnlUPfXST9UILGiQVBbV8BhnW91dZnYHNdgvFMCyDd8Pj1gC3sJBdwOqQbsmYaM7kPC8Ln0GPYCzvvvCQ0HQDbh/HvV85DJeeYD07S0O3fv/Gfe/gxifmY/GGXYFSZ0vGRHXKCNXd46oHcahIg656VynbQ9cNoYDXSwhqzun7A9bb08jjobcplWp7cwbn3TkLl9z7Dp6dv96TUQcOt2QJ/e29eluzY0w8kovp99CJ4JFc9EriHiMiSy5VKcP20IM7d1X8US5Wid7dlvV0RunIdG7c2RoYMXHCPv7OOQBobKhGo9ayShtGYCQMYD0r+cIgIs/IVqvDTX4TzjOXo3SDpCB5Lvlyllq8bGXoOnnKIDT1rAmsoIZht4ZME5t3tUEI4DMHDwPgLUPe+3Hz4WM7tFF6vSktxlqV29Q0Wvfmvycd2dk4sFctBtmdu4eP7o+9BzagNaQFUQjV0w0b1aqm2SDL4OmttCBk/0Fb1usk9IzhoR9kS4XHj/eWP/0Y6UCo9dMw/P1blz/wHq55dC50WjM5O0SYFMnFPbacBr0i53JRK6PlofsRInhwTRhyz6qU66GbQkB3DtUON/XtvHVPG6oMguxjNzUPSvLiwo2O9qo6BsEauiu5ZHKmr5JklZdAZMklQNcN89ZVA1adMtCSMbF9TwbbmzOeuG8dNX82BAxU6d8QHn2gN9Wr8njopik8nvbVp01wRm6q21UNnZwXQNC1rb/64A9ZGfXyFDQaV5Ky49BVp+DkiQPx0NurAiNfAO/LfKE9RNyRXAzSok5cuQ3QjLjHEAWn8YYz9kVtVQrD+vYAEeGta05CY0M1npm3ruhYadUwhr1IALeMy07RKPV0kBJRlFKkjDrNsch3rge/dkTg8H3dq5eOi6E5jrpzt2DtDs8cQJLmTA5VacPTR6AeyZKLRlDh1fNJCL8XdsTo/hio6LcXHjXS+Sx1xFTKknAAq7DpBlnVc9UIht2tWaRTBt78wYnoWZt2Cpwel66mNUhD9zgBRJ4oF92AyHObQjhGSMbUhlFlGJEHjqjPuTptYN2OFhxw4zN4Y+mWvJ1O6n0FiTt964INuhD+CplOGYHPELAMpbp/VVo91pXhLJlL3lMeI6xIRSquRxm8fxCGQY6Bk+mXHbdhYYuyDKhTxUoPvSpleGQ9xwEIkFz0FmwQtVUpDO9X5+zb1LPG+Vy8QXfNyYl2iOTEwf5OWlnGyTZ4+fJEImfkBLz3pEfG5JNc0ikj0DvWPfSn5q3zXcewo1zCOkbVvp+WTM5yDI3gcOSok7UVQ8V76IYhJRfvPgL+CnjvVw6DKQTe/ngrxg3siV61aUwY1Avfe3gORvavw8adrUjb5wOsc+rNLNV7Uz2tTE5Y82L0qkX/+mrnuHwDS4Li0MPDFk1PhUkZrucqhOupFfJ20ilCWzZap6h6ruq0AbQCx4xrxPHjBzgDnp77znG+e/S8qAJsQ8ogvHjlVEz95Yu+3/TKnTYo1KPNCW8Ujfp81DIhIJyXdL66JM+le+hjbG35V5870LM9bC4deS7ZopL55HRwhxhMWQbGDWhA7x5V2N6c8XSKytkerX2tv/LZq8/c+zk4fbpurKY7bGqIQqjnnDS0N5bfcjqu/ddczLe9WKkpqx66EMJXT4NQQ12l1LVpV6vHkALFdYrqx8igA7VBLGPmw6Ky6qpTTmd3S8ZEv+q0V3KJ1GvVfirSQ1crZcoeoq4/sKCmnBVKZuCw0f3Rr74a6ZSBzx0yHNVpdwBIisip2CJQQ1c99GBDpja18oXleePQ4VzTuU94jUA65a20Ms2mEl5ZyNtJpwxkTOF5XqFhi8q55GRRx4xrxJePHoXx9uyJY5oafHO6pwJeVDo1Vf6iRxQkaxihA4V0Wc0vgbgvPNdDDzyVJ926wZVGcT8lkkfdPwgZhw64HnmKCFUpCvXQ5WaDyEmnvKf9h/b2lHt93II38ktJR0gaw15GhuH10ONMSBUkQXlfuG7LV/5mChHJY/X2ERDu/tIhuPms/TCody1+pEyrG/aiinrusO1Scglz0NSWQkvW0tBJsQNhTlOpqUiDHqSh+zz0iE0563zqFKXkvJnNgCgXtTLqnXWywMpwLCC6hx42sEh6nfqEW7LDTV7D1Dz0Y8Z55/WQVMUIW1Qfnxzu36dHuP6tps05d8g+g3v3wOOXHu1MfyAJkjXCXoo50+uh11WnYZB17+ppBPwaetA9yxesbnDlvjJtslM331whKTsOHXBlOiJCdcoI9dAdQ2e4+Sjz/4YzJ2HhT07D/V893JMm9xivwZOEeayhBl3z0MNePkHoA3TC0mIqLy5Lcil87irNmRnUuxbnHWYtZnLhUaMcjb0YDz3shaIrATnTK6GeoYSrDlEkISEsx8lQ7EAH2fPKlFx8zcuA/LAkl2iZSyDPIgJulIv/jezR0DUpIKUOmAiYrEs2o539Azx03aDLc5qmQJV236rkok6xuvTm6aHSQlAoVVhh01+cOVP4RlUWOi6fZ7LfsN6Yvt9gPKDEPet5VpWiUM0/J7yGrDptYM71p6ImbeCGx+d5YoDzdYZK5G+6wVWjMgDgjv93MFqyZt6lAeVcLoCbN0RAVTq8D0Od19sJbdVbmeTdV57Ko/eGfFYJ69AlIo+H3pbLf58qQd5xkLYv0y4dnyj1NExS0rcFvVSKRZeuVHtw7ekTcPLEgc6U2MP71eFNZTh/lR0tp5bBI8f0DwwhLSUV6qG7n8M6RREjysUg11Crlcka+u/dN2hCLokM5UsZrnam7iMruMzkIGlC9VhIabZnTeExXpbk4naKqp6a1a8Q7pll9JGiIUZXrWhXnDgO5x82AoeN8i+/p6NW7LDFIiS3nL0/Xvn+8U46fJ2idix3EEII6DakoSaNqpThGZugSkr5jAfZUoffoLsGCLC8r0Iz+akaeptTtlDAQ3ePdSUXbxVVy6aatpRH73U/68/ztvMOwuUnjvNFh6jHqi/8ONO9Bnn9esc6AEwc0sv+zTJ4+dbVlXgGLQXkofy5GMklDP0laSoaek3a8KRDbxFX2zObus9SYEejDjkAACAASURBVGRjPQ4PWL6ylFSkQSftQRPIZ9HNiAVFnk9mVNrwDizKJ5nov8lFHFTtTN1HT0+Q5CKE8HV+WucxnQpz9kHDbMlF1dDt/Qtq6PbAIiXpYfKMmtx9h/TCTWftV3CAlnVcYcnFk6aA+1XTm4koueg4+n1EDV1e3y+5WOGRcaITUkqUy0pnzm4raqlQlItB7jPUDZRTNmXYoilfNsEerP48T99/cODIUff83v2DlkkMI8igq9eXg7p+e85k+1rSQy98bm//kf93We6LkVwA4GvHjvbMngn4ZSxVQ08ZhidNn5o8FK9ffYLzXXroavRa6V414VSkQfdp6OTv2BPwe29hENxmsRx5CrgDi8LqsR5OJycDkr331j5uunRjGyy5uN4GKfuoGvoNZ+6reejqEPD89yo9Xvm83rrmJEzfb3DgvmG6bCFUAx2lM8i5X/JfJ1+naE5rtaiopxFQDbps0YV3BOthe1FD61RShjt45x17ci2DLM+ukIdOSse8bqBkMnweeoC0Ie8nDvrjfHGhfx3YIK49fYInEiUoLTlT4NBR/XwLXESSXArckzsJXHEm7erpE3DChIGhv8tWt+v4ka8+q+XeJ7kgv9xXKirUoLuf04Z3YNGH63bgb68tj9UpSorkkjLc5q6UMsK8Xl0KcA2666GrUQK6t6R6GqqHrt6RDMs0Fe8gbRDShqGtnuSv2EFITVoWtLBVj+R9hKU9H2qdCotyUVHv1y+5hE9bmi9vCG7rQH+m+UgZFCi5xHX8iAj726OF5XufyPLQl4TMtCdUD915qesGnTz7Bo0UVYnrscrzy7mQXl4cPDGaJGUQLj1+LL5iLwqjo17ealGp14o+sEiXG4PSARTvoetp9Z3f9tCzip3w12f3e1Xa8IwujVMG20NFdooGGRppN6b99mUA1oCGYiSXlKI/y7BFw2ob+o6Tx4xuqsfSjbudlUvU3u1sHoMe1HloCv+bXMor1Uoki9opql6nUKeQPlI0n/3XK19UxjY1oGdtGjtbsrF79/Usq0pZA5pEQHibGaC5u+mlwGda6CWfoiCDHn8wSErRwWXriWBN3LR6WzPMgNaFGs4n06nP/CiP0OPQw5IXtQ44+9u79+5R5ZnvXeX8O9/Atj0ZPPw/RxZ8Yav5o0tk6nzocQjK8zCJKg75+1escuDUs1R+g16dIrQKNQ69Yzz0ijTo6oMZO6DBlly85PPedNRO0bTPQ3e93rrqlLYau3XV75w8HkP61GKCPSrOq6G7xkH3HtQCJCumUHr91UgHS79zj0sb5AmbdEYMRpBctuxuwx8DluXTKbbpPqBXLWZcfgyO+fkLsSpsUKuqtiqFbXsy+OfsVb4pR/PJYdb53Mokz1vIvhkhGnpcx89q6dlymdLhPm3SIPz1lWXImCZ++MgHngmp5PSwBhEOGNYHH2/e45tiQZeM9HBVXzpiSy72czIIPaoMX/6ZpsCri63pYeViEvkuoZYhU3jzy4oGK0YWKpeHHn6sbLnJBVfShpHfQ08ZaMsJTxx6B9jzypRc5IM7bdIgN6IhwHBElQmIyKOhqwOL1Cahf95kq6L2qDYweURfz8T9Qgj88F8f4KP17nJqureknu+x99dg3+uewvurtrsx0/bftGEgawrPvB26hx51pOhJtk4oZ2fM1wzMFy1RCOkpRRmX0thQjS8dORJ/v+gw33Xk3PcbA1aRKdRc90ouFvni0AFv9JB6nbhGR+0PUL1oOY/N9uYMHpy9Cqu27UHOFFiwboezwDMR8PPP7I9HLjkSAzUv2dHQTfk3v9QWN9/UF5/sCJS0ZU1s2u3mg2kWHv+opkv30KXkEpfAKZnt+ZGa8sxRX4i8kothDf3/3J9fd777DLpyb3JOKHfof7QBVO2lIj101YsAEOihq6MDC0EIH1ik6n7+eZPdJrKevpcXbcLLiyz9ceyABizesEvxHqQ37T3OMbL2ZjUyI2da80gYBGcude+8MsFp0Tl6XCPOmjwUj7672nONIMJGH0bBlcIiaOhEuP6MfQEA63d4Z94b3McyaEEjFvW5sb3ndD8LwHk7FvTQiZxRsRKzCO9KrfCqbOIMXrKv8eWjRuHCo0bh639/25lDxDCstV8PCliX0/XQ4Tl36OCY2Bq6ex3DII/Bvez+dzyrSQlR2PNUL581ha/VZ4r4Rj0oz2//wsFYt70Fw/vVxTqXJ615npU6tgQIHtWse+hhk3OVk4r00HU9NCgbcjHi0IncYfRpQ41Dt4zoISP7AQA+a09/KgmalQ3w6ubHjGvEvkrcrUq4MfJuT6esDlD1ntIpCu4UjXDPFPJZRz1XXO9CavlxHTD9GXlfgV7yadueUFbF6ES5jzXKdK6mPQo3dpSLoqE707GSa+SlQZcvee/Q9sLnd2fzhO94lWI7RWUsvPoeVY05oLyo8jwbfYpoNTly8ip5L18+alSkNAYZ3tqqFEY21sdukXjOq9yHXOxEkjIISze5ndktmZw/ak3V0NNaHHoMB7M9VKRBd2JBVS9M+PeJqh96dD0lakaGA45qrMfyW073zeOdDTGiO5QFmNXQJV9MccjTdzx0uJU9qxmWlC3D+NIS5Z6VXfIZOPWnuFqsK7nE9L5C+hmCTmOaIjAmWeLac+ExVPn45AHeEM6cELFaexIisv95JRd5fzI0UobZ6XOVhKE/j6CBRUH7R0UmQ0qP+VpYUfJWvX42oFNUtoIPHdkP1ylzsuSjXLPPynsf01Tvm7X0hH0GOOu2AtYiJWGjeIGAOHTklzdLRUUbdOdNHVBo88Uo66h7pT0auvBM0aobZHVIt8oOdXh/nnCrMCOpV0K52LCpVIi0JrnIDqco96wWrPzN5WhGJt+x7Y1ycTTjAOORrxVGZOWfEAKmqXro+a//o0/u672GaZ0jalm66OhRGNqnh7OcmzqPtmrQdQ89HdWg2zVWXX7QOnfwMaXU0HVkqzCq5KKHf8rgAdMs3JkflMZS49YtwzcN7xcO3wtPffNYpxWxpy3re+ZypLE8h3fof/4O/FJRkQZdPiRdclG9CT3mNR/6JDzuXC52ITTcjFbJhXjF6nwtno5Fbb+wlUv0ZBtkeeg5Ux1AoXWKFtkiyYfegRUHaaD26l8f6zjdcJLyYjj4x8/ipzMWOL/lk0Lk1ssfeA9PzVvnvFjitsitllF0I/LDT0zEq1ed4KyQZRjk6WuR52lT4pnlfpJ8aVQHvQFROkUjJdt3bTmFRL5O7e8+ZC0fmO/R6GGLXg1dLiITbzGa9sgq7T3vyROtoIKD9vL3bwDK/PVp8sahg6NcQlGHSAPBBcrUhtDnQz1eDVt0PHT55tY8dBk2qBshdf5u0l4WKpOG9saPPjnRt4zW/sN6Y0xTPa6avo9zXVfLddOpjlTNxfAi1b3yd4q6n+NWotqqFO68YAr+ftGhsY4LMkxElke7eXcb/vzSUme7aeYx6GRVosftyZOyyoyHQHRtXw4uK9aGqDoqIcBDV1Ykco/JJ7lYf92wRet7WP7Ej86RHjphR3MG763chsUbdgbu+97KbZHPJ9OqpkZ6sHH7KOLKf1FxpM48pz9iTH8suXl6YIc14LY0qu3F5j1D/7uKh05E04hoIREtJqKrAn4fQUQvENG7RDSHiKaXPqkuYSF6aiXVJ7PKh38qAXKuYw3Fz++hF6qA8me9g6oqZeDCo0ZhdJPXi22orcJz35nqFBqpoavyjxXl4u0ULcZDz6+hFy+5AMBJEweisSFeGFnYoJHb/7vE/uxuL+TZqeVBfox7F8IsbmCRJEXui5fIH+US7KHnyxPrrzvOIVj2c64fW3Jx0y0N9m0vLMl7TNTQV+u730PP92IOPGeZPPQ4o4nDkPnhH/rfMWGLBQ06EaUA3AbgNAATAZxLRHrvxbUAHhRCTAZwDoA/ljqhKvqyWzIjhLZPMfmuRic4EQRhHnrO21IIPJ/hZqo+qlWiZ7R+Pjn8XY27ThvkmeNEny89H8V0zpSpDgVcJ8BDVz4fMcadrS7foBR90RNdpouKaevwxXvoqoZOjjFqzaOhU55a6R/6nz+6qWgNXUlDe7Le13Gonld2ikaUXL43bTwOH90Pk7SFRkpFKVYVOu/QEThoRB8cOaa/88ICOm5gUZQ49EMBLBZCLAUAInoAwJkA5iv7CABy8cDeANaUMpE6ejNT1lGPhq4MJimEJ5rDcCtdxolNt36r8nnoXi8r+Nxe7z+IfF6MdZzhRLm4o0cNzyyEZhyDHqFp6UtjB1n08GHd1r2qCwnkk0L0zcKJ/Y+XHldyKe7+VQ3dklys7a6Gbm2I2gEtf9GH/uuts4aaNHa1Zp0pnSOnN2I0kCdNeXbVp+n1a+j5X8wql0wdi0umjo2crri4UZjFl/UbzpzkfL531gpvHHoHVKEouT0UwErl+yoAh2n7XA/gGSK6DEA9gJOCTkREFwO4GABGjBgRN60OutThFHJ1n1z0Sij3G9NUb4Wa2dud+OECUS5RwsyA6FEt+m4pw52cS50mVPXQ48XdR9rNQzlXKlcJvAVlmyeP873ESJNcivTQBYqbbVGSMtyBSuosivmjXMLPFzb0XzdC/770KKzd1uJbNq8Q8jSe51rg1vP9/MkDBuO9lVvxjzdWAPC/uHKm2a4+inJQqqQQaXHoFTQ517kA7hFC/IqIjgDwdyKaJITwDLkTQtwB4A4AmDJlStHtG30dxaC6lo0xvE8eL9fGlJXDmVmN/JXOuQYKGXT1zR+2j2bQtYSnDAM7W7OoTrvzR6RS3giEXKw5MbwvwnzcecEUbNnd5kwFW26ICMePb8J5h+3lbFMfu2qk1QikINQCVsQIc+ca+hwkcTBILSf+TtGgaSUiaeh2zXrqA2t0qV42xzQ1OItbx0uvrFOKBNQOQ1STTuHECQMVg+69likKvJi7OA99/QgM6h08iVmK3Dh+S0Mvf3qiGPTVANRZkYbZ21QuAjANAIQQrxNRLYBGABtKkUidsMEUaqW1JJd4Hro+gZNusPXYVDcePv+5zQJdcnpG62W7riqF15daEyKNsTtQ9Qq8dOOuyCFqruRS+PmcNDF8juhycfeF3sgYNR9VnVOdOE0naNEToAgPXbQvQsGjoYOUTlFrmgeZj+oc7PmMmx7fL18M7Rny7j2/nYYY91vo2ag/B83lUujF3FFEGPjqY4o9ijwIz9D/LqShvwVgHBGNgmXIzwFwnrbPCgAnAriHiCYAqAUQbWb8IpAPSfcmPJW9wEx8KnI3WaicWfK0sESf5GJ6Pfgg1NFiYWU2aC4YlR9/ahLetRdJkDM66pV+1rItmDS0F6JA2t+ujudxeFoleTR0bbsQhVtTktu/cBB+8+wiLFy/U+kULe5pGVqUi94pKvNxZ0sWAHDW5KGhCziryDK1uy2LQ0f2K5mHG6Shv7dyK8Zf+592nxPwG3drrEf5BgsVQ6mS4otD7woeuhAiS0SXAngaQArAXUKIeUR0I4DZQojHAHwHwF+I6Fuw0v4lUcxExxEJWnbLSquyTxxdTvNKZOGSUSxye311GgcM7wPTFFiycZdPYw9ClVzCCq3u4evfxw5owFhtbgnVQ7/29Ak4acJADOgVLUSwmE7RziTsubVkcqELGBOCoxaCOtB1pk0ajB3NWXzv4TntNjgpZXAOkZtvehy67GAPWw5QIsva9x6ag7+8tBSLNuyKrZPnw4lDV8pX2IIc+jHhvwfvaxCwYUeLPS9KEYktMaWIclFRF9zuUgtcCCFmAJihbbtO+TwfwFGlTVo4x+3dhF88vRCn7GvJAUHlScTohAiavRHwLksn//77G9ZtHnjjMxE1dPc3J2wxzz7W9QunW11qa1jfHhjZGH1EZkcUrFIS5KDnTIHWbP4V6YPi0OO+5OWArqIlFy1MT5YxPcrFXTghv3eupr+Pvb7rsk35DW4c5H2WUgEJm7VzRL86vGAvcTeiRJJRe5BL440tou8hiGzOxPbmDFZs3tN1PPSuyKShvbH8ltOd7565SSi+FiZ3k9JJmOSiYhCh1dZB80ouav0MlQd0yaVwmr0aeryS4njoFWLY1ccjPeuWjPXs9QUg1GOCOkWjhqSpRki0w0PXjZk+sEjmY1aZ7TMfap5dPX0CPv3H19CcyeU5Im56rb+l7KQM09CvP2NffHeaNRq6oabzTdHopgbcc+EhOHRUuC4eh4P36osHZ6/Chp0tXUpDrxj8A3aiHefOYBjioQecqKEmjS0RV2xxDWjY9YPTk48eiiGLa2vI96FroxphmcVy5ahQg67dnBOHHtmgW3/bO/Tfs3gz3OlzdQ1dOg+FDLr68wB7MYd8E2jFpdRx6NbvyktNa7F0BUOuMnX8gMI7RWRYX6vVkTNjepjtoCLnctFxdFF4BxNF9cacZmaBTlEVtQMybJItwFuhQzX0Ap2iQfRQpIa4xUQ+lwqx54Fhi822Qe8RMnDGaqn5R4pGvWffBG3FeujqCFByz9uW8xpwZz7+QmKy8nPPGktyueCIvUJ2jk/UaYbjndN//u6AfIbqmrLlpmu9HotEPijZNDaLfICu5GJ9zzfH+LdP3hv7DOqF/g3VjqcUhGd8RkiCimne1nk89OKKSqXUrUAPPWNFhfTIp6EHfI46UlReUq4rW+wz1vPf8dAzslPUSpD04gpNKGdo3u6Sm6eXWO+2/sYpk4WkuzANPek4K1YVmG+nlCTDoIcYzbiSiyxs7sCi8Hkyxg7oictP7BkhbVQwykUfhRlXcolLpRhyiddDtx5mc0HJJXikqDodb/5rWvvNX7MDC9ftRM/a4qqKN2SPnLECW/dYcl3vHpaX7SxSXsBD1/XoUg/IUQfryaH5hSgsuSifK6Zd2H5kXmeVcQjlJhGSi0TA+9CiNht1yUVWkl8/+5FnezFEmVNcl2yiXK5dkoszUrRSKpc/nVKiCI3Z9j3seK02efhl97+LFVv2FG049fyX3zfZi15Lg67O0hf1fOWI3XYHFlHJzu8dHVqSU1YEzlKWHeihJ8KgO7Mtar2icSuvlFb0Ye5xC+F3Tx3vfE4ZaihYcDp1D72mqnC2qJMexe4UJe/fSkI+uUKDtXzmPOZcLvp+xXbe6UP65fdNO1vRqzbtfP/mSXujZ00a+wzK3+orpgUaB1VDjxslVuic+uekI+2JOjlbuUmGQbeflD6jWdwQNVm5qlIGXvru8c7vcSfU/8bxYzF5RB/PubWkedA9dH3t0iAaatth0Aukp6shAgLKnU0Rb8KNQ49aJrzf9VkDo+LpFAWclYzWbG/xzBV/9LhGzL3hVPSsrcp7vvbOUV8IdUyGeq18Hf8FJZeA83cHWENvJ8XGespj1Mo3or870KEYyUW+BKJUQL/kUvh6h492Y2XjSieuh14ZlUtdT1SGHxYayq9vFppHX2gcczFjA4LQO0XHDWjAzWfth+3NGeelHwc1WeWQL+Q5DfLWpc8ePAz3zlpR5DnVOtCOxFUYhu6hd0B9S4RB9zbp3O3RNXTbQw/Zv5glr9SOVqe5H+Lk6LppFL1WenoAYr/FKsWQS8yAzs1CDro/Dr3AARqFpjSOildDt0aKnndY8VNHl1tD37DT0vatlbvc7Xk99AIP1VMnu5FFdz10s8CepSMRkouDNtw/sgYovZKQwlZMh5g03nqFDkKvLHGvFzd1lSa5mAHx5O5LMqqHbu9fpOTSp0d1pON0PjV5aGiaisEos3Hcd4g1vuKEfQZ46lI+g16IKLJjEpHZI5ct4E7RiMjnpM85HPX5OUvEhTzxYiqOV4vMn56a9hr0hIvoXgnd+hJ3rEGc2RYBb+X71kl747ITi1sp5/8d7g76KcXjLndk0sXHjsb8G0/F1PEDPJNV1bRjgRM1tLTYvohKxNA89C4zOVdXx+kULXLof6GJm9oruejbdPQol7jXi++hU1HHdRaBHrr9PexR6ZsL7e873t6xZ20aV5w0LtpBBSiFRFJuL4+IHKOr1qeavAO48ndIjG5qwMP/cyR2tWZxyMi+JUlnJeCLcuFO0Wi4HnqRGqP03kIsetx1KD3HBnQI6cXf1yka20OPl6ZK7hSVSI877B7CJZdo15T5FndNznyU4nF3Vpa1dwnCg/fqPoZc4oty6YBrJkNyCdPo7C+1BeK69UWndYrx0CXeKIfg8+gxzvE99PJ69J2Np1NU/i0wN0u4oSf7PPm9yvLMOtj+c3Vk2J/6hKKMjWC8yKySI87ZQ4+J0OIWCYQXr5xacNi2rNyhUS7tGSmqTc4UxOn7D0Z9TQpf/8c7RV0v6QOLgibZknkWtZURV0N3+0CiprIwpXje7XEuYqNY9I5aJDxJyHrs9veUP+8SkUvqwCLSto9srEf/hvwr+cioovCIifZ46IUloNqqFKZNGuzuV/Yol0rT0NVvMg7d+hb1UclTyNbQIXnWggT8o3tLQUmiXDow7O/35012PrcnyqW7whp6kTgaujYrXtSy7yw6HdYpWkQlcg2Css3+W5vOP7FWoTmx/ReLuXsFa+jyozPFdMjNh2noY5rq8cRlR2NUgRWe5HlLatAr5hVqceq+g5zPbNDjI1++2ZxcU5ajXKIRppfGrECl1NAH9Kx10tC3zophnjikF47duwnnHDI877FxjUixGnqlmJegpeTcTtHgY8IWuCAiTIqwBqcR8EJuL5U8pkaVXPrUVeHwUf3x0qKNzkIjjJ/OmMslGQbdxheHHi/IpaRRLsP7WRN8NWdy+P60fTC6qR5nTR6KTx80rOCx5dbQ5QEV4qB7kIY8bhhi3CXL9Zk3S0GltIiCUD30tGHg9v93MG56cj7+8vKy2M+2u+AscMGSSzyc56THoUc8vr7GkkDCpi4tplJ/evIwLNm4G9P3G4Qe1SlccMTIyMeWPw698hEFOpp8kkvM87seeumeViV76OrCG7KaVPILqiOQj6wjwxaTYdBD4rujFrgff2oS3li6xaMZqhQjuYzoX4ffnzu58I4BlDKyIohKroe+sMVQySXkwIg4y/SVtFO0ch980AA5fRpoxot/tkXW0CPhzofu3R41dnbfIb2x75BwXbWjJxQq99D/SuucU9FHikaebTGmRQ8a6dudUetA2CpRjBf/bIvlv2YyDLryoFSjfv6he/l3LoIOjf1FEZ2iRUa5VCL6Ahfl0tAl3Wn+7iCG9+uBXS1Zjwtw7ekTAVR2i6Mj6IyRookw6BLdC+tdl3+xgKiUet3GQsQNWyxWQ6/EFrPTKVpopGjY9LkRcVZE6uYu+tPfPBamsNZWBYBDRvbF8REWYGFUD73jpltMhEEvt4Hq+pJLvPMnwbFyo1yixqHHKxyiwNiE7oI7UVf486tAv6BD8E2f2xHX7IBrlJ2wTtFS0dFOWvymbHk1966EOx96TMkl5nXkwKXuLrlI3AVF+HlEhYhgELDRXjSE50OPSNgi0aWiozX0uHTx5JUMVYoqJLnoxC0acqV2NugWcddwZSzqqtOYuWA9ACCTLf/KRYkw6OUuZF1dRy1yXFHFkTLIt8BFeJRL+27SmVqgBM/q81PyjwyuBJyRtsq2Ci1GHcq9XzkMx4xrBADsyZR/VG0yDLqNEOXx0ru+h15c2GKlaZ9pg3xL0EWOQ49JoRdGHG45ez8svXl6u8/TFejiVaHLccDwPpg83FoMPJMtf41LVKcoUB4jxR561yCdMoqOQ4+LDDUrRYQTEVXsM3eotLd/F0KOQM/kyi+5JMOgOyPXynP+jgpbvOfCQ/Dq4k2xj4sd5RL7Cl2DdIDkUi44bNELd4oWT1WaDXosyl3EOkpymTp+AKaOjx/jG3u2xQqtk+mUK7mgzJLLgcP7oLbKwCVTx7TzTMkgn8RVieMZOhLpobexQY9H3OHdUSn33CrtJb6HXpkWPW0Y7lwuKG+naJ+6anz449PadY4k4U4/rGyszGLU4VTbgxk6wkPv4qYqGk4cukBZtL6u3ikaF/d5VZZrpUpfhaJQEpZlnc6wvnUAgKPHNrkbK6v4dBqOhs6dotEo98Cijh76H5fuYrzSBjmZ7MahR7t5XhOzfYxqrMcbV5+Igb3yL+fI+OnITtFIpZyIphHRQiJaTERXhezzOSKaT0TziOi+0iazQPrKPLCoq4+sjK+hd+37CUONQ3cll+B91c1Pf/NYvHLV8WVOXfIZ1LvWW3Yqsxh1OLJTtEto6ESUAnAbgJMBrALwFhE9JoSYr+wzDsDVAI4SQmwlog6dvadC7VOnUWmP64oTx2HmgvWor0470SemG3YRjFIoxg1o4GiVMlKuvquk0NU09EMBLBZCLBVCtAF4AMCZ2j5fBXCbEGIrAAghNpQ2mdFoyZjY2ZrtjEt3KkmfnOtbJ++NJy8/BlVpck1HoRWL1M8Vdr+FePeHJ+Ota07q7GQwEWnqaclUQ/vUlf1aUTT0oQBWKt9XAThM22dvACCiVwGkAFwvhHiqJCmMwd2vLuvoS3YJio1DrzS/ikC+NUWjON6VKjGF0be+urOT4KFSo6Y6ioP36oe7vjQFR41tLPu1StUpmgYwDsBUAMMAvERE+wkhtqk7EdHFAC4GgBEjRpTo0m6F3birtWTnrCS6i4ZObp+oM3lW1OlzmfLBkkthTthnYIdcJ4rkshqAOrvQMHubyioAjwkhMkKIZQA+gmXgPQgh7hBCTBFCTGlqatJ/LhpZd7trJEPSJZcgCkro7DWWHX7GXY8oFvAtAOOIaBQRVQM4B8Bj2j7/guWdg4gaYUkwS0uYzrxIA1Wd7qYGvcz7dyX0ybnKNZcLUxj2zLseBS2gECIL4FIATwNYAOBBIcQ8IrqRiM6wd3sawGYimg/gBQDfFUJsLleiw+i2Br2bzM5FZHWKfrhuB5rlVKSVeSsMUxYiaehCiBkAZmjbrlM+CwDftv91ONz0izt9rkWFDRQFAdiyuxXTfvuyu61Mc7kwheF61/VIhEtLlRq2USK6i4ZOBOxp9S4SwJJL51NpjkGSSYZBt/+We0rVrkp8FNEGBgAADblJREFUDb0yrR3Bb6i5U5RhXBIxl4uk1Ob85e8djw07u34oZOwViyrU1gXdJ6/52flwFnQdEmHQPbMtlpDh/eowvF/5R3e1l+4S5ULK/862cBed6SC6acO4S5IIyUXW3u4muQzt0wNA/NkgK9WjipPuCr3FiqJSy1GSYQ+9grn9CwfjreVbMKxvj1jHlXt2ynKip7lcC1wwTCWSDINu/+1uAx32G9Yb+w3rHf/AirV1hJxm0NluM4xLQiQXiwp0ODuFSrWBREDO1Ax62L7lTw7DdDkSYdBl87q7aejFUqlyBMGdlEvCcegM45IMg27/ZXsejUq1dUTKwhbKtrB9Gaa7kQyDbldevbIzwZR7DdZyQYEaOltuhpEkwqC7VJqJ6hwq1QYSeaNc8t0HjxQtP/yEux6JMOjdNWyxWCrZ2KmdovnuolJfWgzTHpJh0LvpwKJiqVRjp2voLLcwjJdEGHSwht4t0FsWMQfIMkziSYRB7+az58aGKrdXVPsabtHZe2e6I4kw6JKXPtrY2UmoCCr1Begz0Xk7RZmOohKnkEgqiTDojQ01nZ2EiqJ/QzUAYNLQXp2cknjoXjd3ijKMl0QY9ElDe+MH0/fp7GRUDEeOacTrV5+Av3350M5OSix0G92zNhFTETFMyUhMjRjYq7azk1BRDO4db4bGrkifuurQ3yo5NLNS4FZQ1yMRHjrTPdANSJ8eVZH3ZZjuQGIMOkc1JB89h/vU5THo5U0Kw3RJEmPQmeSjv7TPPmhYJ6WEYbomidHQ2SNLPmoe//2iQ3HMuKbwfblAdBgctdh1SIyHzhW4G6Dkcdg86IE7M2WBO567Hskx6Fy4Eo+ax4XsOb/gy093W/KxEkiOQecK3K0o7KEzTPcjOQa9sxPAlB2KIbkMssclHDmmfzmT1K3hVnHXIzGdokzyUc1HIQf9gOF9MPf6U9CjKlXWNDGVNydQkkmMQecWePLxeuiF9+9ZGx6nzjBJJDGSC4suycfbKcr53VXgnOg6JMagc/1OPnE0dKbjYMml65Acg97ZCWDKTlzJhSkv/E7teiTHoHPp6lZwhAXD+EmOQe/sBDAdQPSBRQzTHUmMQWeSD2voDJOfSAadiKYR0UIiWkxEV+XZ72wiEkQ0pXRJjAbX7+SjZrHBrkiXgSfn6joUrBZElAJwG4DTAEwEcC4RTQzYryeAKwDMKnUio8AGPfmklJ5Q9tAZxk8UP+dQAIuFEEuFEG0AHgBwZsB+PwbwMwAtJUxfZLiTLPmoRpxzu+vA79auQxSDPhTASuX7KnubAxEdBGC4EOLJEqYtHlyoEo/HoLMV6TKw5NJ1aLcSSUQGgF8D+E6EfS8motlENHvjxo3tvbT33CU9G9MVMTgOvUvBWdD1iGLQVwMYrnwfZm+T9AQwCcCLRLQcwOEAHgvqGBVC3CGEmCKEmNLUFL7aDMMEwRo6w+QnikF/C8A4IhpFRNUAzgHwmPxRCLFdCNEohBgphBgJ4A0AZwghZpclxSFwEzz5EHEcOsPko6BBF0JkAVwK4GkACwA8KISYR0Q3EtEZ5U5gVLh+J5+UUlqrUhy3yDA6kabPFULMADBD23ZdyL5T25+s+LDHlnxSSibXVfM8510FXoqu65AYN4fDFpOPKrn0YIPOMD6SY9DZnicetVO0miUXhvHBtYKpGKQ9r0kb3AneFeA86HIkxqBz0Uo+hm3Ra9KJKbYVTWNDNQCgf311J6eEkSRmTVG26MlHxp7X8MLPXYLzD9sLDTVpnHng0MI7Mx1CYgw6d4omHxnlwvp51yBlED590LDOTgajkJiawXJe8pF5zFPnMkwwiakabM+Tj4xy4WH/DBNMcgw6V/LEwwadYfKTGIPOJB/50mZ7zjDBJMagcyVPPrJTlD10hgkmOQa9sxPAlB05sCjFBp1hAkmOQec6nnjkwCLOa4YJJjEGXffR63nypsRhsOTCMHlJzsAipY4/esmRGNKnR+clhikLcjwRx6EzTDCJMegqk0f07ewkMGWAPXSGyU9ifB2u4snHcMIWObcZJojkGHSu5InH9dA7OSEM00VJjkHv7AQwZcfR0PnlzTCBJMegcx1PPOyhM0x+kmPQ2UdPPKyhM0x+EmPQmeQjJ+dKs4vOMIEkxqCz05Z8xg1swP7DeuMrx4zq7KQwTJckkXHoTDIZ1rcOj116dGcng2G6LOyhMwzDJITkGHTuFGUYppuTHIPO9pxhmG4OG3SGYZiEkBiDzjAM091JjEFnDZ1hmO5Ocgw623OGYbo5yTHonZ0AhmGYTiY5Bp0tOsMw3ZzEGHT20RmG6e4kyKAzDMN0bxJj0FlyYRimu5Mcg97ZCWAYhulkIhl0IppGRAuJaDERXRXw+7eJaD4RzSGi54hor9IntWAaO/qSDMMwXYqCBp2IUgBuA3AagIkAziWiidpu7wKYIoTYH8BDAH5e6oQWgs05wzDdnSge+qEAFgshlgoh2gA8AOBMdQchxAtCiD321zcADCttMgvDDjrDMN2dKAZ9KICVyvdV9rYwLgLwn/YkimEYholPSVcsIqIvAJgC4LiQ3y8GcDEAjBgxopSX5rlcGIbp9kTx0FcDGK58H2Zv80BEJwG4BsAZQojWoBMJIe4QQkwRQkxpamoqJr2hsOTCMEx3J4pBfwvAOCIaRUTVAM4B8Ji6AxFNBvBnWMZ8Q+mTyTAMwxSioEEXQmQBXArgaQALADwohJhHRDcS0Rn2br8A0ADgn0T0HhE9FnK6ssEeOsMw3Z1IGroQYgaAGdq265TPJ5U4XbHhOHSGYbo7PFKUYRgmISTGoDMMw3R3EmPQWXFhGKa7kxyDzqILwzDdnOQYdLbnDMN0c5Jj0Ds7AQzDMJ1MYgw6W3SGYbo7yTHoDMMw3ZzEGHTuFGUYpruTHIPO9pxhmG5Ocgx6ZyeAYRimk0mOQWcXnWGYbk5yDHpnJ4BhGKaTSY5BZ4vOMEw3JzEGnWEYpruTGIPOYYsMw3R3EmPQ2Z4zDNPdSYxBZw2dYZjuTnIMemcngGEYppOJtKZoJdBQk8bXjh2NSUN7d3ZSGIZhOoXEGHQiwtXTJ3R2MhiGYTqNxEguDMMw3R026AzDMAmBDTrDMExCYIPOMAyTENigMwzDJAQ26AzDMAmBDTrDMExCYIPOMAyTEEgI0TkXJtoI4OMiD28EsKmEyakE+J67B3zP3YP23PNeQoimoB86zaC3ByKaLYSY0tnp6Ej4nrsHfM/dg3LdM0suDMMwCYENOsMwTEKoVIN+R2cnoBPge+4e8D13D8pyzxWpoTMMwzB+KtVDZxiGYTQqzqAT0TQiWkhEi4noqs5OT6kgouFE9AIRzSeieUR0hb29HxE9S0SL7L997e1ERL+zn8McIjqoc++gOIgoRUTvEtET9vdRRDTLvq//I6Jqe3uN/X2x/fvIzkx3eyCiPkT0EBF9SEQLiOiIbpDP37LL9QdEdD8R1SYtr4noLiLaQEQfKNti5ysRfdHefxERfTFOGirKoBNRCsBtAE4DMBHAuUQ0sXNTVTKyAL4jhJgI4HAA37Dv7SoAzwkhxgF4zv4OWM9gnP3vYgB/6vgkl4QrACxQvv8MwG+EEGMBbAVwkb39IgBb7e2/sferVG4F8JQQYh8AB8C6/8TmMxENBXA5gClCiEkAUgDOQfLy+h4A07RtsfKViPoB+BGAwwAcCuBH8iUQCSFExfwDcASAp5XvVwO4urPTVaZ7/TeAkwEsBDDY3jYYwEL7858BnKvs7+xXKf8ADLML+QkAnoC1NOwmAGk9vwE8DeAI+3Pa3o86+x6KuOfeAJbpaU94Pg8FsBJAPzvvngBwahLzGsBIAB8Um68AzgXwZ2W7Z79C/yrKQ4dbMCSr7G2Jwm5iTgYwC8BAIcRa+6d1AAban5PwLH4L4HsATPt7fwDbhBBZ+7t6T8792r9vt/evNEYB2AjgbltqupOI6pHgfBZCrAbwSwArAKyFlXdvI/l5DcTP13bld6UZ9MRDRA0AHgbwTSHEDvU3Yb2yExGWRESfALBBCPF2Z6elg0kDOAjAn4QQkwHshtsMB5CsfAYAWzI4E9bLbAiAevilicTTEflaaQZ9NYDhyvdh9rZEQERVsIz5vUKIR+zN64losP37YAAb7O2V/iyOAnAGES0H8AAs2eVWAH2ISC5ert6Tc7/2770BbO7IBJeIVQBWCSFm2d8fgmXgk5rPAHASgGVCiI1CiAyAR2Dlf9LzGoifr+3K70oz6G8BGGf3jlfD6lh5rJPTVBKIiAD8FcACIcSvlZ8eAyB7ur8IS1uX2y+we8sPB7Bdadp1eYQQVwshhgkhRsLKx+eFEOcDeAHAZ+zd9PuVz+Ez9v4V58UKIdYBWElE4+1NJwKYj4Tms80KAIcTUZ1dzuU9JzqvbeLm69MATiGivnbL5hR7WzQ6uxOhiE6H6QA+ArAEwDWdnZ4S3tfRsJpjcwC8Z/+bDks7fA7AIgAzAfSz9ydYET9LAMyFFUHQ6fdR5L1PBfCE/Xk0gDcBLAbwTwA19vZa+/ti+/fRnZ3udtzvgQBm23n9LwB9k57PAG4A8CGADwD8HUBN0vIawP2w+ggysFpiFxWTrwC+bN/7YgAXxkkDjxRlGIZJCJUmuTAMwzAhsEFnGIZJCGzQGYZhEgIbdIZhmITABp1hGCYhsEFnGIZJCGzQGYZhEgIbdIZhmITw/wE56Dwg7hv2WAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DGZbkNku6dQf",
        "outputId": "61a8e401-54b3-4e0f-9cec-7e0e102ad505"
      },
      "source": [
        "max(loss)"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9629629629629629"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JJDsUtIOAO4G",
        "outputId": "1993e0c9-f3cb-43e6-f383-7044dba8e48f"
      },
      "source": [
        "min(loss)"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.09259259259259259"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3wrIG70vAX7O",
        "outputId": "6d1c175b-c71d-4471-e5c6-58869fd33dd1"
      },
      "source": [
        "loss[-1]"
      ],
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9191919191919192"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 71
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CtqvAdy6ApWy",
        "outputId": "66652486-0661-4daa-8dff-0e06a9e3fd30"
      },
      "source": [
        "par"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'alpha': 0.2612108431069134, 'eta0': 0.003425562625614941, 'penalty': 'l2'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 72
        }
      ]
    }
  ]
}