L10/code/10_06_kfold-sele.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "STAT 451: Machine Learning (Fall 2020)  \n",
    "Instructor: Sebastian Raschka (sraschka@wisc.edu)  \n",
    "\n",
    "Course website: http://pages.stat.wisc.edu/~sraschka/teaching/stat451-fs2020/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# L10: Model Evaluation 3 -- Cross-Validation and Model Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sebastian Raschka \n",
      "last updated: 2020-11-20 \n",
      "\n",
      "CPython 3.8.2\n",
      "IPython 7.18.1\n",
      "\n",
      "numpy 1.18.5\n",
      "mlxtend 0.18.0.dev0\n",
      "matplotlib 3.3.1\n",
      "sklearn 0.23.2\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -a 'Sebastian Raschka' -u -d -v -p numpy,mlxtend,matplotlib,sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"margin-bottom:5cm;\"></p>\n",
    "\n",
    "\n",
    "## GridSearch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- For hyperparameter tuning and model selection, `GridSearchCV` is really useful"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from mlxtend.data import iris_data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "X, y = iris_data()\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=123, test_size=0.15, \n",
    "                                                    shuffle=True, stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Accuracy: 96.09%\n",
      "Best Params: {'criterion': 'gini', 'max_depth': 3}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "\n",
    "param_grid = [{'max_depth': [1, 2, 3, 4, 5, 6, None],\n",
    "               'criterion': ['gini', 'entropy']}]\n",
    "\n",
    "gs = GridSearchCV(estimator=DecisionTreeClassifier(random_state=123),\n",
    "                  param_grid=param_grid,\n",
    "                  refit=True,\n",
    "                  cv=10,\n",
    "                  n_jobs=None)\n",
    "\n",
    "gs.fit(X_train, y_train)\n",
    "\n",
    "print('Best Accuracy: %.2f%%' % (gs.best_score_*100))\n",
    "print('Best Params:', gs.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.653846153846154 {'criterion': 'gini', 'max_depth': 1}\n",
      "0.9121794871794873 {'criterion': 'gini', 'max_depth': 2}\n",
      "0.9608974358974359 {'criterion': 'gini', 'max_depth': 3}\n",
      "0.951923076923077 {'criterion': 'gini', 'max_depth': 4}\n",
      "0.9602564102564102 {'criterion': 'gini', 'max_depth': 5}\n",
      "0.9602564102564102 {'criterion': 'gini', 'max_depth': 6}\n",
      "0.9602564102564102 {'criterion': 'gini', 'max_depth': None}\n",
      "0.653846153846154 {'criterion': 'entropy', 'max_depth': 1}\n",
      "0.9121794871794873 {'criterion': 'entropy', 'max_depth': 2}\n",
      "0.9608974358974359 {'criterion': 'entropy', 'max_depth': 3}\n",
      "0.9365384615384617 {'criterion': 'entropy', 'max_depth': 4}\n",
      "0.9448717948717948 {'criterion': 'entropy', 'max_depth': 5}\n",
      "0.9525641025641025 {'criterion': 'entropy', 'max_depth': 6}\n",
      "0.9525641025641025 {'criterion': 'entropy', 'max_depth': None}\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(gs.cv_results_['mean_test_score'])):\n",
    "    print(gs.cv_results_['mean_test_score'][i], gs.cv_results_['params'][i]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- We an summarize the results in a heatmap, but note that this only works if you have 1 or 2 hyperparameters to consider"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAADnCAYAAAAJiLFaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAuUElEQVR4nO3deXwV1f3/8dc7KyGERdawCZVFEHdc6lJFBbWWuq+tVb4utVW/2sXqz7bWqrVqbdVWv7UuuNZqtfWr9osL7tYNQXCjIggCYQsIhCVAkns/vz9mAjchJHPhJnMvfp6PxzySO3Nm5pOb5H7mnDPnjMwM55xzLqq8uANwzjmXWzxxOOecS4snDuecc2nxxOGccy4tnjicc86lpSDuAJxz7qvoyFGl9uXyRKSyUz7c8LyZHdXKIUXmicM552Lw5fIEk57vH6lsfvnMbq0cTlo8cTjnXAwMSJKMO4yt4onDOediYBi1Fq2pKtt44nDOuZjkao3D76pyzrkYGEbCoi1RSDpK0gxJsyRd0cT2LpKelPShpEmSRqRs+0LSR5KmSZrc0rm8xuGcczFJkpm5AiXlA3cAo4EK4D1JT5vZ9JRiVwLTzOx4STuH5Q9P2T7KzJZFOZ/XOJxzLgYGJLBISwT7ArPMbLaZ1QCPAsc2KjMceAnAzD4FBkjquTWxe+JwzrmYJLFIC9BN0uSU5fxGh+oDzE95XRGuS/UBcAKApH2BHYG+4TYDXpA0pYljb8abqpxzLgYG1EZ/rMUyMxvZzHZt4RSpbgBukzQN+AiYCtSF2w40s4WSegATJX1qZq9v6WSeOJxzLgYWvRkqigqgX8rrvsDCBuczWwWMA5AkYE64YGYLw6+Vkp4kaPraYuLwpirnnIuDQSLiEsF7wGBJAyUVAacBT6cWkNQ53AZwLvC6ma2SVCqpLCxTCowBPm7uZF7jcM65GAQjxzN0LLM6SRcBzwP5wHgz+0TSBeH2O4FhwIOSEsB04Jxw957Ak0ElhALgETN7rrnzeeJwzrlYiESTXRNbx8wmABMarbsz5fu3gcFN7Dcb2D2dc3nicM65GASd45lLHG3JE4dzzsUgGMfhicM551wakl7jcM45F5XXOJxzzqXFEIkcHRHhicM552LiTVXOOeciM0SN5ccdxlbxxOGcczEIBgB6U5Vzzrk0eOe4c865yMxEwrzG4ZxzLg1Jr3E455yLKugcz82P4NyM2jnncpx3jjvnnEtbwsdxOOeci8pHjjvnnEtb0u+qcs45F1UwyaEnDueccxEZotanHHHOOReVGT4A0DnnXDrkAwCdc85FZ3iNwznnXJq8c9w551xkhvxBTs4556IzoNbnqnLOORed/HkczjnnojNyd+R4bkbtnHPbgURY62hpiULSUZJmSJol6YomtneR9KSkDyVNkjQi6r6NeeJwzrkYmImk5UVaWiIpH7gDOBoYDpwuaXijYlcC08xsN+B7wG1p7NuAJw7nnItB0DmeH2mJYF9glpnNNrMa4FHg2EZlhgMvAZjZp8AAST0j7tuAJw7nnItF8MzxKAvQTdLklOX8RgfrA8xPeV0Rrkv1AXACgKR9gR2BvhH3bcA7x51zLgZB53jku6qWmdnIZrY3dSBr9PoG4DZJ04CPgKlAXcR9G/DE4ZxzMcngyPEKoF/K677AwtQCZrYKGAcgScCccGnf0r6NeVOVc87FoH7keJQlgveAwZIGSioCTgOeTi0gqXO4DeBc4PUwmbS4b2Ne43DOuZgkM3TtbmZ1ki4CngfygfFm9omkC8LtdwLDgAclJYDpwDnN7dvc+TxxOOdcDMygNpm5Rh8zmwBMaLTuzpTv3wYGR923OZ44nHMuBkFTVW72FnjicM65mPhcVc455yJL83bcrOKJwznnYuFNVc4559Lkzxx3zjkXWXBXVaR5qLKOJw7nnIuBPzrWOedc2rypyjnnXGR+V5Vzzrm0+V1VzjnnIjMTdZ44nHPOpcObqpxzzkXmfRzOOefS5onDOedcZD6OwznnXNp8HIdzzrnIzKAugw9yakueOJxzLibeVOWccy4y7+NwzjmXNvPE4ZxzLh3eOe6ccy4yM+/j2C4UqdjaURp3GJvpsktt3CFsUff87IytjmTcITSpgNy8i8Y19MX8WpYtT2zjp75I+F1Vua8dpeynw+MOYzPHP7E07hC26ILOC+IOoUkrEtVxh9CkLvnt4w7BZcC+R87PyHG8j8M551xkPleVc8659FjQz5GLPHE451xMcvWuqtzsmXHOuRxnYed4lCUKSUdJmiFplqQrmtjeSdIzkj6Q9ImkcSnbvpD0kaRpkia3dC6vcTjnXEwy1VQlKR+4AxgNVADvSXrazKanFLsQmG5mYyV1B2ZI+quZ1YTbR5nZsijn8xqHc87FxEyRlgj2BWaZ2ewwETwKHNv4dECZJAEdgOVA3dbE7YnDOediYJZW4ugmaXLKcn6jw/UBUu8RrgjXpbodGAYsBD4CLjGz+gFPBrwgaUoTx96MN1U551xM0rgdd5mZjWxme1MHatwQdiQwDTgM2AmYKOkNM1sFHGhmCyX1CNd/amavb+lkXuNwzrmYmEVbIqgA+qW87ktQs0g1DvinBWYBc4CdgzhsYfi1EniSoOlrizxxOOdcDAyRTOZFWiJ4DxgsaaCkIuA04OlGZeYBhwNI6gkMBWZLKpVUFq4vBcYAHzd3Mm+qcs65mGRq/J+Z1Um6CHgeyAfGm9knki4It98JXAvcL+kjgqaty81smaSvAU8GfeYUAI+Y2XPNnc8Th3POxcEyO1eVmU0AJjRad2fK9wsJahON95sN7J7OuTxxOOdcXHJ0yhHv40jTMlvMW/Ycb9qzfGGfNllmuVXyjk3kbXuByfbqxvX/tgm8bS/wjk3kXXspo3F98sZyfn30JH515Lu8cPe8zbZXV9Vy10Uf85tjJ3PTKe+z8LO1G7c99PMZXH7gW1w39r2MxlTvuZfXMuyguQz5+lxu/NOKzbavWJnghHGL2OOweex/9Hw+/nTDxm0rqxKcfO4ihh80l10Onsvbk9dlLK6XXlnPft9Ywj4HLuG221dvtn3lyiTfO2c53ziiktHHLOU/n26aQn7P/Zdw8OGVHDqmksO/mdnZi7P1/crWuLI9tuZkcBxHm8r5Goeka4DXzezFZsp8GxhuZjdsy7nMjBlMZU8Oph3tmcRLdLPedFDHjWVqrWZTGbWnxtY3OMbeHEKRircljM0kE8bfr53JxffuRueexdx0yvvsOqor5YM2PVvkubvm0WdYB86/fQSLZ1fz2LUzueS+oHa6/3E9OeSM3jx4RdOJcFskEsbFVy7l+cf60Le8gP2Ons/YMaUMH1q0scxv/7iCPUYU88/7yvl0Zg0XX7mUiY8Ht6Bf+stlHDmqPY/fU05NjVG9LjPP2UgkjMt/UcUTj3Sld3k+o49ZylFj2jF0SOHGMrf8aTUjdinkwXt3YOasWn728yqefKzbxu3/+3hXuu6Qn5F4UuPK1vcrG+PK9tiaY0AymX1JIYqcr3GY2VXNJY2wzNPbmjQAqlhOCR1orw7kKY+e9GNpozveFjOf7vShnYLnLhSp3baetkVffLiK7v1L6NavhIKiPPb+Zg8+fPnLhnHNqmbo/l0A6PW19ixfsJ5Vy4KZBgbv05nSzoWbHTcTJk1dz04DCvnajoUUFYlTj+3A08+vaVBm+mc1HHZQCQA7Dy7ii/m1LFlax6rVSd54Zx3nnBEk5qIi0blTZj6o359Wy8ABBQzYsYCiInH8sSU8+0LDJD9jZh3fOCj48Bk8qJD5FQkqlyYycv4tydb3K1vjyvbYmmWAKdqSZXIqcUj6paRPJU2U9DdJP5V0v6STwu1fSPq1pPfDCbt2DtefLen2bT3/BtbRjpKNr9tRwgYaVmurWU0dtUy2V3nXXmShzW2wfSpv8K69SIXN3tZwNlpZWUOXXptqMZ17FrNyyYYGZfrsXMoHE4MmlS8+XMXyhes3K9MaFixO0K/PpqTUp7yABYsbfvjuPryYJycE/+iTpq5nbkUdFQvrmD23lu5d8/mvSyvZe/Q8zvtJJWurM3M1uGhRgt7lmz4gevfKZ9GihnGNGF7Iv54Nksn7U2uYX5FgYVhGgpPOWM5hRy/lgYfXkinZ+n5la1zZHltLMjiOo03lTOKQNBI4EdgTOAHY0ijKZWa2F/Bn4KcRjnt+/TD+Wrb9g9QwVrGCPTmIPTmYOfyHtRa0n+/DKPbTEezJQVTwOSssQ23jTfxhqdFFypjz+lO9qo7rj5/Mqw8voO+wMvLyW/9Kpqk/+saxXX5xF1ZUJdnriHncfm8Ve44opqBA1NUZ73+0gQvO6sSUif0pLVGT7ddbFVcT69QosEsu7EBVVZJDx1Ry931r2XVEIQUFQZn/e7IbrzzXncce2oHxD6zlrXcyk4Sz9v3K0riyPbYWWcQly0Tu45B0ADAgdR8ze7AVYtqSg4CnzGxdGM8zWyj3z/DrFIIE0ywzuwu4C6Cjdmj2V1RMCetTahjrWUdxSg0EglpIIUXkq4B8Cuhi3VhDFaWUUaygbJHa0d16s4rldKF7SyG2qHPPIlYsTunsW7KBTj0a9qOUdCjgzOt3BoK+mquOeJeufVu/Ga1veT7zF2zqVF6wqI7ePRs2BXQsy2P8rT03xrbTvnMZ2L+Q6nXJoM16ryDOE7/VgRtvz8w/de/y/I21B4CFixP06tXwOqqsLI8//aHLxrj2+nolO/YLYi/vFXzt3i2fbx7Vjven1XLA/tved5Wt71e2xpXtsTUvOzu+o4hU45D0EHAzwYf3PuHS3LwprSHqO1z/CZogw53/HenCOtawztaStCRLmE93yhuU6U5vVrKMpCVJWB1VLKeUMhJWR50Ff9wJq2M5SyilU0bi2nHXjlTOXceyinXU1SSZMqGSXUd1bVCmelUddTVBFfytxxczaGRnSjq0/r0R++zRjllzapkzr5aaGuOxp9Yw9sjSBmVWViWoqQly9j1/XcXB+5fQsSyPXj0K6Ne7gBmzgr6Yl/9dzfAhRZudY2vsuXshs+fUMXdeHTU1xpNPreOo0Q0TaVVVcmNcDz1Szdf3K6KsLI+11UlWrwney7XVSV59fQPDhmbmvczW9ytb48r22Fq0ndc4RhLclRTnj/Bv4C+SfksQ9zHA3W0ZQJ7yGGp7MJU3MIzeDKCDOlFhnwPQVztRqo50tV68y0RA9GEgHdSJalvDh7wdDPrB6EU/uqlXRuLKLxCn/GIQd5z7Ecmk8fUTetF7cClvPBp03B98Wm8Wf76WB6+YQV4+9NqplO9eN2Tj/uN/Mp2Zk6pYs7KWnx/6NsdcNIADTirf0unSUlAg/nh9d44+fSGJhDHutI7sMrSYOx+oAuCCszrxn5k1nP3fleTnwbAhRdzzhx4b97/tN90588Il1NQaA/sXMv7WHls6Vdpx3XBtJ07+zpckk3DGqe3ZeWgh9z0U9FeMO7OUz2bV8cNLVpCfL4YOLuC2mzsDsHRpkrPOXQ5AXQJOPK6Ew0dlpvaWze9XNsaV7bE1y8By9K4qRckFkh4H/tvMFrV+SM3GcTVwOjAXWAq8ChwI/MvMnpD0BTAyHEY/ErjZzA6VdHa4/qLmjt9RO9h+OrwVf4Ktc/z0zI4TyKQLOi+IO4QmrUhUxx1Ck7rkt487BJcB+x45n8kfrN+mT/3igX2t/NcXRyo796wrprQwO26bilrj6AZMlzSJTU1BmNm3WyWqLbvZzK6W1B54Hfi9mW2sdZjZgJTvJwOHht/fD9zfloE651yLsrAZKoqoiePq1gwiDXdJGg60Ax4ws/fjDsg557ba9pw4zOy1cBrefcJVk8J529uUmZ3R1ud0zrlWUT8AMAdFvavqFGAScDJwCvBu/aA755xzWydXBwBGbar6ObBPfS1DUnfgReCJ1grMOee2ezl6V1XUxJHXqGnqS3Jo1LlzzmUjZWFtIoqoieM5Sc8Dfwtfn0qjB4Y455xLQ5YO7osiauf4ZZJOJBgzIeAuM3uyVSNzzrntWnbOfBtF5HkSzOwfwD9aMRbnnPtq2R5rHJL+bWYHSVpNwx9RgJlZxy3s6pxzriVtN4N7RjWbOMzsoPBrWduE45xzXxHb8zgOSXmSPm6LYJxz7qtEFm3JNi0mDjNLAh9I6t8G8Tjn3FfHdj6tejnwSTjJ4cbnZMYwyaFzkcyua/1njWyNHZJrWi4Ugx3y2+g529uJRK52TmRI1P+uX7dqFM459xWUjc1QUaQzyeGOwGAzezGc1twvUZxzbmsZOTvlSNRJDs8jmJfqL+GqPsD/tlJMzjn31ZDBPg5JR0maIWmWpCua2N5J0jOSPpD0iaRxUfdtLOp8UxcSjBpfBWBmM4E2er6ic85tnzJ1V5WkfOAO4GhgOHB6+OyiVBcC081sd4KH3P1eUlHEfRuImjg2mFlNSpAFZGVfv3PO5ZDM1Tj2BWaZ2ezws/pR4NgmzlYmSUAHYDlQF3HfBqImjtckXQmUSBoNPA48E3Ff55xzTYmeOLpJmpyynN/oSH2A+SmvK8J1qW4HhgELgY+AS8LhFlH2bSDqXVVXAOeEJ/s+MCH1Wd/OOefSk+bgvmVmNrK5wzWxrvHRjwSmAYcBOwETJb0Rcd8GoiaOi83sNmBjspB0SbjOOefc1sjcXVUVQL+U130JahapxgE3mJkBsyTNAXaOuG8DUZuqzmpi3dkR93XOOdeEDE458h4wWNJASUXAacDTjcrMAw4HkNQTGArMjrhvAy3Njns6cAYwUFLqgcoIngLonHNua2XoFiMzq5N0EfA8wRi78Wb2iaQLwu13AtcC90v6iKB56nIzWwbQ1L7Nna+lpqq3gEVAN+D3KetXAx+m+8M555wLZXgCQzObQKMns4YJo/77hcCYqPs2p6Vp1ecCc4GvRz2gc865iHJ0UIM/yMk552KiHJ0r0R/k5JxzLi3+ICfnnIvL9vo8DjNLhpNi9TezeW0RVDZbZov5jGkYRh8GMkA7b1ZmuVXyGR9gGIUUMVKHAvBvm0A+BQgh8thPh2csrk/eWM4T188imTQOPKmcMec1fO5WdVUtD/98Bkvnr6ewOI/vXjeU3kNKAXjo5zP4+NUvKduhkF88s0/GYqr33Mtr+dFVy0gk4JwzOnL5xV0abF+xMsE5P6pk9txa2hWLe27pwYidiwFYWZXgvJ9U8smnNUhwzy09+PrIkozE9far6/j9NStIJuDYU0s564edGmxfVZXk2su+ZMG8OoqKxS9v2oGdhhZt3J5IGGeNXUz3XvncMj5zU7e99uoGrrt6FYkEnHJaCRdc2KHB9qqVSa64rIp5cxMUF4sbbu7IkKGFDeI67ltf0qtnPnff36Xx4bfai6+s58qrqkgkjTNPL+XSixo2RKxcmeTin6xgztw62hWLP/6+C8N3DuLafb/FdOgg8vNEQQG8/Gxmp7rL5ti2KEuf7hdF1jzISdJxwGdmNj1Tx8w0M2MGU9mTg2lHeybxEt2sNx20qaun1mo2lVF7amx9g2PszSEUqTijcSUTxt+vncnF9+5G557F3HTK++w6qivlg0o3lnnurnn0GdaB828fweLZ1Tx27UwuuW93APY/rieHnNGbB6/4NKNxQfAhdvGVS3n+sT70LS9gv6PnM3ZMKcNTPoB/+8cV7DGimH/eV86nM2u4+MqlTHw8mPHg0l8u48hR7Xn8nnJqaozqdZlpFE4kjJuuWsHtD/egR698zvr2Yg4e3Z6vDd70AXz/HVUMGV7E7+7qzhezarnpquX8zyM9N25/9L7VDBhUyNo1mWuoTiSMq3+xigf+2oVe5fmcMPZLDh/djsFDNv2r/vmONQwbXsif7+7C57PquPoXq3jo0R02xT2+mkGDClizOnOfSomE8bOfr+Sff+tG7/J8Dv9mJUeNacfOQza9X3/402pG7FLIQ/d25bNZtfzsyir+9+/dNm5/+vFudN0h809jyObYWpSjiaPZpipJgyQdSPAgp28B1xDcljsZeCrDsRxHMDNjU3FkxePcqlhOCR1orw7kKY+e9GNpowGWi5lPd/rQTu0BKFK7Vo/riw9X0b1/Cd36lVBQlMfe3+zBhy83HGazeFY1Q/cPrj57fa09yxesZ9WyYN7Kwft0prRz4WbHzYRJU9ez04BCvrZjIUVF4tRjO/D08w2fgjf9sxoOOyioRew8uIgv5teyZGkdq1YneeOddZxzRpCYi4pE506Z+ef+ZFoNfXcsoE//AgqLxJix7Xn9heoGZebMrGWfA4MkP2BQIYsqEny5NAHAkkV1vPnyOo49rcNmx94WH0yrZccB+fTfsYCiInHM2Ha8+ELDi49ZMxMccGCQeHcaVEBFRYJlYVyLFiV49aUNnHJaZmpl9aZMrWHggAIGhHGdcGx7nn2+YVwzPqvlkIOC92vIoELmVdRRGcbVmrI5thblaFNVS30ctwKrzey11IXgft/jWjq4pO9KmiRpmqS/SMqXtEbSb8Lmr3ck9ZR0APBt4Hdh2Z0kvSrpekmvAZdIOlzSVEkfSRovBZftkr6QdGN4nklhsiuTNEdSYVimY1humz4dN7COdmz6h2xHCRtY16BMNaupo5bJ9irv2osstLkNtk/lDd61F6mw2dsSSgMrK2vo0mtTLaZzz2JWLtnQoEyfnUv5YOJSIEg0yxeu36xMa1iwOEG/Ppve9j7lBSxY3PAfdvfhxTw5IUgmk6auZ25FHRUL65g9t5buXfP5r0sr2Xv0PM77SSVrqzNzdb90SYKevTcloR7lBSxd0jCuwcOKeOW54Pf7ybQNLF5QR2UY+y3XrODi/9eFvAw/h2fJ4iTlKXH1Ks9nyZKGP/POwwp4/rngg/GDaTUsXJBg8aKgzHVXr+LyK8tQ1DkhIlq0OEmflLh6l+ezqNHvccTwQp6ZELxfU6bWML8iwcJFQRkJTjz9S0YdVcn9D68lk7I5tuaI4K6qKEu2aenPa4CZbTbQz8wmAwOa21HSMOBU4EAz2wNIAN8BSoF3wjnhXwfOM7O3CIa4X2Zme5jZ5+FhOpvZIQRzxd8PnGpmuxI0sf0g5XSrzGxfgtkfbzWz1cCrwDHh9tOAf5hZbRNxnl8/42Qt2/5BahirWMGeHMSeHMwc/sNaWw3APoxiPx3BnhxEBZ+zwpZu8/nCk25GjT7QxpzXn+pVdVx//GRefXgBfYeVkZff+k8fswixXX5xF1ZUJdnriHncfm8Ve44opqBA1NUZ73+0gQvO6sSUif0pLRE3/mlFq8XVeKq37/2gI6urknzn6EX8/YHVDNmliPx8eOOldXTpms+wXYuaOEjm42r8fn3/h6WsqjLGHrWMB++rZvguBeQXwMsvrqdrtzxG7Jb52mOUuC65qIyVVcY3Rldy9/g17DaikILwb+zZ/+3Oq8/34O8Pd+Xe+9fw1juZu2jJ5tiaFXG6kWzsB2mpCai5dpaW6sKHA3sD7wXTv1MCVAI1wL/CMlOA0c0c47Hw61Bgjpl9Fr5+gOChJLeGr/+W8vWW8Pt7gJ8RPKlwHHBeUycws7uAuwA6aodmf0XFlLA+pYaxnnUUN3ob2lFCIUXkq4B8Cuhi3VhDFaWUUaygbJHa0d16s4rldKF7c6eMpHPPIlYs3vTHvnLJBjr1aNiPUtKhgDOvDzryzYyrjniXrn1bvxmtb3k+8xdsytcLFtXRu2fD5qaOZXmMv7Xnxth22ncuA/sXUr0uGfSL7BXEeeK3OnDj7ZlJHD165bNk4aar0spFdXTv0TCuDmV5XHVz141xHXfQQnr3K2DiM9W88eI63nplARs2GGvXGFdduoxrbu3GtupVnseilLgWL0rQo0fD67uysjxu/H2njXEdeuBS+vbL519Pr+eliRt47ZVKNmyANauT/PiSlfzhts7bHFfv8jwWpMS1cFGCXk38Hu+4pcvGuPbYfwn9+wdlynsFX7t3y+eYo0uYMq2GA/bPTF9fNsfWoixMClG0VON4L3xsbAOSziH40G+OgAfCGsQeZjbUzK4GasPZGSGohTSXvOrrjS1dGlvj783sTWCApEOAfDPb5luKO9KFdaxhna0laUmWMJ/ulDco053erGQZSUuSsDqqWE4pZSSsjrqwwpOwOpazhFI6NXWatO24a0cq565jWcU66mqSTJlQya6jujYoU72qjrqaoM771uOLGTSyMyUdWr/raJ892jFrTi1z5tVSU2M89tQaxh5Z2qDMyqoENTXBr/Cev67i4P1L6FiWR68eBfTrXcCMWUFfzMv/rmb4kMxc5Q/fvYj5X9SyYH4dtTXGC89Uc/DohhcBq6uS1IZxPfXoWvbYr5gOZXlceHln/vVOH556sw+/+VM3Rh5QnJGkAbDb7oXMnZNg/rw6amqM/3tmPYePbvghtqoqufH9euxv69hn3yLKyvK47Ioy3pzUg9fe6sGtt3fi6wcUZyRpAOy1RxGz59QxN4zrn09Vc9SYhhceVSlxPfhINQfsV0THsjzWVidZHd5AsLY6ySuvbWDY0MzVirI5thblaB9HS58clwJPSvoOmxLFSKAIOL6FfV8CnpJ0i5lVStqBYHLELVndzPZPCZLAIDObBZwJvJay/VTghvDr2ynrHySohVzbQqyR5CmPobYHU3kDw+jNADqoExVhy1pf7USpOtLVevEuEwHRh4F0UCeqbQ0f8jZY0JzVi350U69MhEV+gTjlF4O449yPSCaNr5/Qi96DS3nj0aDj/uDTerP487U8eMUM8vKh106lfPe6IRv3H/+T6cycVMWalbX8/NC3OeaiARxwUvmWTpeWggLxx+u7c/TpC0kkjHGndWSXocXc+UAVABec1Yn/zKzh7P+uJD8Phg0p4p4/bLod8rbfdOfMC5dQU2sM7F/I+Fszc6tkQYG47Jod+O/vVZJMwNhTStlpSBH/eDhoVjzxu2XMmVXLr3/yJXl5MHBwIb+4qWsLR81MXL+6tiPjzlxBIgEnn1rCkKGFPPJQ0HF/xpntmTWrjst+VEV+PgwaXMBvb8rMBUhLcd10XWdOOmMZiSR859RShg0t5L4Hg2u7cd8rZcbMWn54yQry88XQIQX88ebgCn/p0iRnnhPcrFGXgJOOa88RozJX283m2FqSjc1QUciabOxtVEgaBYwIX35iZi9HOrh0KvD/CGo2tQTNSy+aWYdw+0nAt8zs7PDurbuBDcBJwL3AT8P+FCQdDtxMkOzeA35gZhskfQHcB3wzPM/pYXJBUi9gDlBuZitbirejdrBMjq3IlOOnZ6gvpBVc0HlB3CE0acqGmpYLxWCHvCyNKz+GW1Fz2GFHVzL1g5pt6iQsKe9nXzvrx5HKTr/xx1NaeJBTm4rUVmFmrwCvpHtwM3uMTf0U9TqkbH8CeCL8/k0a3o57aKNjvQTsuYVT3WFmv25i/UHAE1GShnPOtSnLzjumosiK8RGtQdKfgKMJaiLOOZd9crSpKucTh5kN2ML6i9s4FOecS0uu9nHkfOJwzrmc5YnDOedcZFl6q20Unjiccy4GwpuqnHPOpckTh3POufR44nDOOZcWTxzOOeciy9KZb6PwxOGcc3HxxOGccy4dPuWIc865tHhTlXPOueh8AOD2YTUrlr1oT8xtuWQk3YBlmTjQi8MycZSNMhYXBPPkZ0hG48qgbI0Lsje2r0JcO2bkKBlMHJKOAm4D8oF7zOyGRtsvI3h8NwSf/cOA7ma2PHw8xWqCh+vVtTSFuyeOFGa27c9xDUmanE3z59fzuNKTrXFB9sbmcUWTyZHjkvKBOwgexV1B8PTWp81sen0ZM/sd8Luw/FjgR2a2POUwo8wsUmJt6dGxzjnnWomSFmmJYF9glpnNNrMa4FHg2GbKn07wdNSt4onDOefiEPV540He6CZpcspyfqOj9QHmp7yuCNdtRlJ74CjgH42ieUHSlCaOvRlvqmo9d8UdwBZ4XOnJ1rgge2PzuCJKo6lqWQvNbE09xnZLRx8LvNmomepAM1soqQcwUdKnZvb6lk7mNY5WYmZZ90cKHle6sjUuyN7YPK40RK9xtKQC6Jfyui+wcAtlT6NRM5WZLQy/VgJPEjR9bZEnDueci4ks2hLBe8BgSQMlFREkh6c3O5/UCTgEeCplXamksvrvgTHAx82dzJuqnHMuLhm6q8rM6iRdBDxPcDvueDP7RNIF4fY7w6LHAy+Y2dqU3XsCT0qCICc8YmbPNXc+TxzONUOSzCxHh2m1LUmljT6QXHMss1OOmNkEYEKjdXc2en0/cH+jdbOB3dM5lzdVZVh4P3VWkTRI0khJxXHHkkrSLpIOkdQ17lhSSTpI0pkAZmYKL8XiJmmspEvijqMpko4Fbgw7V10E9eM4MtRU1aa8xpEhkoaY2WdmlpCUb2aJuGMCkPQt4HrgS2CxpF+Z2Wcxh4Wko4EbgdlAoaRzzGxxzDHlAe2BvwQvVWpmd4bJI8/MYpuSTtIY4Frgsrhi2BJJhxD8Li8OO1ezXtbUJLMghK3hNY4MCD+cp0l6BKA+ecQcFpIOAG4GzjKzUcAK4Ip4owJJhxJMjXCumR0H1AAjYgwJADNLmtka4AHgXuAAST+q3xZXXOHv8SHgfDObKKmTpB3D+/Gzwd4EU1xMlNRb0mhJ+4UdsVknNWlIOljSgZJ6xhKL1zi+msK7EC4CLiX4oHnYzL6bRTWPG8xsavj9r4C7JRWb2YYYY1oCfN/MJknqBexH8P98MjAR+EfMV4N1QH+CBHKupD8AG4ArAcWQRL4EaoHysFnvCWAdsEbS38mO96so/P4JYG64TpIuNrMVsUXWhJSkcRnwTYJ4O0u6xszeb7tAyNlJDr3GsY3CzsD/Ah4Bfgq0k/RwuC3upPEu8E/Y2PdSTDA5W8dwXSx9C2b2HzN7JXx5DvA/Yc3jHeBkgsno4vQUsNjMXgImAxcAHS3Q5jUPM5sBHAPcAnxA8Lf2LeA54ESgS1vH1MjLwHmSHgXuNrPTCS5S1tDCeIC4SNqdYNDbKIIR1/kErQaFbRpHMtqSbTxxZICZLTSzNeEEYd8HSuqTh6S9JO0cU1wJM1sVvhSwElhuZkslfQe4TlJJHLHVM7PfmNl14ff3AWU0HMgUh3XAUEnnESSNG4D+kr4fV0Bm9gFBsvitmd0dNquNJ0ga/eOKK4ztY4KLpv2AgeG62QQfxhmbOHRbNHGDw1pgrqQ/EjS1nRheFBzRlv8TuZo4vKkqw8zsy/AD5neSPiX45xkVc1iYWR1B08Z8Sb8lGORztpmtiyumxh2Ukk4kuKd8SyNe20Q49cJ84JfAhWb2jKRRwKyY45oObJztNHy/ugOLYgtqk2cJahlXS6p/NMGeBEk3Vo36NDoQ9Kl9DnQGvgacYmY1ks4FfgAcSXDx0LqMnO0c98TRCsxsmaQPgaOB0WZWEXdM4RVXIXBw+PVwM5sZZ0wp/8zFwHeBHwOnxn13Vehu4CkzmxK+fi3ODvJU4e9yHMFV/slmtiTmkOovTB6U9DFwEkGz6Dgz+zzOuBoljZ8CBxH0x5xJMHfVOOAPkuYR1OhOiTq1eEbiy8284YmjNUjqQtDpNsbMPoo7Htj4IV0j6VrgvbiTRiNJgqvmE8L2/NiZ2Xxgfv0HT7YkjRSzCd6vT+MOJFXYudx2HcwtSEkahxEkhguA8wn6/0YS3PAwmiDR3W1mbVur9MTh6pnZCkljzWx93LE04YGsuH89hZnV0mjEa7bItvcKNsb0atxx5Irw9u+LgJfCRPtjSUlgEnComf01lrjI3RqHd463kixNGln5QehcJjXRET4HWAoMC++mwsx+SnDr9wuS8sPBn23Loj3EKeKDnNqU1zicc9uNRn0aYwnGk6wELgZuBU6WhJl9YGYXSuoR623z2ZcTIvEah3NuuyPph8A1BJ3h4wkG6P6I4E6q70mqn6lgaRzx1fOR4845FxNJ/YEvzWxtONHiycAZZvYfSTcTDORcAPwGuBxYDDE33RqQhc1QUXiNwzmX08J5pn4C/EBSh3CixWUE4zUIpzz5EbCbmS0CLmvLW26blbknALYpTxzOuVy3lOAJeL2BcWHn+GzgUUn1rSo7An3DqXfq4glzc95U5ZxzbUjSYCDPzGZI+itQRTDo9jwzu1zSn4HXw8G4+wHfyYL54xrIxjumovDE4ZzLOeEEnTOAZZJ+DSQIRoJ3AgZJ+r6Z/UDSfkAJcKOZzYkv4iZkaTNUFJ44nHM5J5wT7gjgRYIm992Bxwhm5K0Bdg2brO6L+RECWxQMAMzNzOGJwzmXk8zsZUlHAn8kSBw9gcOA0wimcx8K/I3gWSrZKdsmsonIE4dzLmeFTx38KfAxsL+ZPSDpaYKJPNubWVW8ETbPaxzOORcDM/u/cO6pdyR93cy+jDumSLyPwznn4mNmz0oqAl6UtHcWzmbchOychyoKTxzOue2CmT0l6aXcSBohb6pyzrl4mdmauGOIzLLzsbBReOJwzrm45GiNw6cccc65uGRwripJR0maIWmWpCua2H6ZpGnh8rGkhKQdouzbmCcO55yLiZLJSEuLxwnm4LqDYMqV4cDpkoanljGz35nZHma2B/D/gNfMbHmUfRvzxOGcc3EwggGAUZaW7QvMMrPZZlYDPAoc20z50wkGR27Nvp44XO6SZJIeSnldIGmppH+1wrlelTRyK/c9LvUKbluO5bYfwpBFW4BukianLOc3OlwfYH7K64pw3ebnldoDRwH/SHffet457nLZWmCEpBIzWweMJnhYT7Y5DvgXMD3mOFy2id45vszMmrvYaPycddhy78hY4E0zW74V+wJe43C571ngmPD71Oo3kvaV9JakqeHXoeH6H0saH36/a9hR2D71oJJKJD0q6UNJjxHMsFq/bYyktyW9L+lxSR3C9V9IulHSpHAZJOkA4NvA78JOyZ3Cw5wclvlM0sGt9N64bGcWbWlZBdAv5XVfYOEWyp5Gyv9JmvsCnjhc7nsUOE1SO2A34N2UbZ8C3zCzPYGrgOvD9bcSTL19PHAf8H0zq2503B8A1Wa2G8HjRvcGkNQN+AVwhJntRfBI0h+n7LfKzPYFbgduNbO3gKcJnjq3h5l9HpYrCMtdCvxqG98Dl4sy28fxHjBY0sBwBP1pBH93DUjqBBwCPJXuvqm8qcrlNDP7UNIAgtrGhEabOwEPhA/8MYKJ7zCzpKSzgQ+Bv5jZm00c+hsEs67Wn+PDcP3+BHeevBnM2k0R8HbKfn9L+XpLM6H/M/w6BRjQ7A/ptltR7piKwszqJF0EPA/kA+PN7BNJF4Tb7wyLHg+8YGZrW9q3ufN54nDbg6eBm4FDga4p668FXjGz48Pk8mrKtsEEz27o3cxxm2ojEDDRzE6PsE9zbQz1U30n8P/Dr6jIzVDRjmY2gUYXTykJo/71/cD9UfZtjjdVue3BeOAaM/uo0fpObOosP7t+ZVhdv42gVtFV0klNHPN14Dth+REEzWAA7wAHShoUbmsvaUjKfqemfK2viawGytL/sdx2zchkH0eb8sThcp6ZVZjZbU1sugn4raQ3Carg9W4B/sfMPgPOAW6Q1KPRvn8GOoRNVD8DJoXnWkqQhP4WbnsH2Dllv2JJ7wKXAD8K1z0KXBZ20u+Ec/Uy18fRpmRZmM2cy0WSvgBGmtmyuGNx2a9TSbkdMHBcpLLP/ee3U1q4HbdNeduqc87FJUcv3D1xOJchZjYg7hhcDjGDRBa2Q0XgicM55+LiNQ7nnHNp8cThnHMuMgP8mePOOeeiM8ihx6On8sThnHNxMLxz3DnnXJq8j8M551xaPHE455yLLjvnoYrCE4dzzsXBgAxNq97WPHE451xcvMbhnHMuOp9yxDnnXDoMzMdxOOecS4uPHHfOOZcW7+NwzjkXmZnfVeWccy5NXuNwzjkXnWGJRNxBbBVPHM45FwefVt0551za/HZc55xzURlgXuNwzjkXmfmDnJxzzqUpVzvHZTl6O5hzzuUySc8B3SIWX2ZmR7VmPOnwxOGccy4teXEH4JxzLrd44nDOOZcWTxzOOefS4onDOedcWjxxOOecS8v/B+VM4H3kZ3JIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from mlxtend.plotting import heatmap\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "grid = np.zeros((2, 7))\n",
    "grid[0, :] = gs.cv_results_['mean_test_score'][:7]\n",
    "grid[1, :] = gs.cv_results_['mean_test_score'][7:]\n",
    "\n",
    "\n",
    "heatmap(grid, \n",
    "        column_names=[1, 2, 3, 4, 5, 6, \"None\"], \n",
    "        row_names=['gini', 'entropy'])\n",
    "plt.ylabel('Criterion')\n",
    "plt.xlabel('Max depth')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- After running `GridSearchCV`, we would fit a model with the best settings (hyperparameters on the whole training set\n",
    "- By default, `GridSearchCV` already does this for us (unless, we set `refit=False`), so that we can evaluate the performance of the model on the test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 95.65%\n"
     ]
    }
   ],
   "source": [
    "# gs.best_estimator_.fit(X_train, y_train)\n",
    "print('Accuracy: %.2f%%' % (gs.best_estimator_.score(X_test, y_test)*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"margin-bottom:5cm;\"></p>\n",
    "\n",
    "- `GridSearchCV` is also compatible with the .632-Bootstrap iterator we defined earlier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Accuracy: 95.12%\n",
      "Best Params: {'criterion': 'entropy', 'max_depth': 3}\n",
      "Accuracy: 95.65%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from mlxtend.evaluate import BootstrapOutOfBag\n",
    "\n",
    "\n",
    "param_grid = [{'max_depth': [1, 2, 3, 4, 5, 6, None],\n",
    "               'criterion': ['gini', 'entropy']}]\n",
    "\n",
    "gs = GridSearchCV(estimator=DecisionTreeClassifier(random_state=123),\n",
    "                  param_grid=param_grid,\n",
    "                  cv=BootstrapOutOfBag(n_splits=200, random_seed=123))\n",
    "\n",
    "gs.fit(X_train, y_train)\n",
    "\n",
    "print('Best Accuracy: %.2f%%' % (gs.best_score_*100))\n",
    "print('Best Params:', gs.best_params_)\n",
    "print('Accuracy: %.2f%%' % (gs.best_estimator_.score(X_test, y_test)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95% CI: [ 89.12, 100.00 ]\n"
     ]
    }
   ],
   "source": [
    "best_model_idx = np.argmin(gs.cv_results_['rank_test_score'])\n",
    "scores = [gs.cv_results_['split%d_test_score' % i][best_model_idx] for i in range(200)]\n",
    "\n",
    "bootstrap_percentile_lower = np.percentile(scores, 2.5)\n",
    "bootstrap_percentile_upper = np.percentile(scores, 97.5)\n",
    "print('95%% CI: [ %.2f, %.2f ]' \n",
    "      % (100*round(bootstrap_percentile_lower, 4), \n",
    "         100*round(bootstrap_percentile_upper, 4)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"margin-bottom:5cm;\"></p>\n",
    "\n",
    "- Note that feature scaling is unnecessary when we are using decsion trees, however it DOES matter in most other learning algorithms, including k-Nearest-Neighbors\n",
    "- To avoid introducing bias, we have to compute the parameters for scaling (e.g., the mean and standard deviation in context of z-score normalization) on the training fold to scale the training AND test fold in a given iteration\n",
    "- To make this more convenient, this is where scikit-learn's `Pipeline` class (or `make_pipeline` function) comes in handy as the next cell demonstrates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Accuracy: 94.78%\n",
      "Best Params: {'kneighborsclassifier__n_neighbors': 12, 'kneighborsclassifier__p': 1}\n",
      "Accuracy: 91.30%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "\n",
    "pipe = make_pipeline(StandardScaler(), \n",
    "                     KNeighborsClassifier())\n",
    "\n",
    "param_grid = [{'kneighborsclassifier__n_neighbors': list(range(1, 16)),\n",
    "               'kneighborsclassifier__p': [1, 2]}]\n",
    "\n",
    "\n",
    "gs = GridSearchCV(estimator=pipe,\n",
    "                  param_grid=param_grid,\n",
    "                  n_jobs=-1,\n",
    "                  cv=BootstrapOutOfBag(n_splits=200, random_seed=123))\n",
    "\n",
    "gs.fit(X_train, y_train)\n",
    "\n",
    "print('Best Accuracy: %.2f%%' % (gs.best_score_*100))\n",
    "print('Best Params:', gs.best_params_)\n",
    "print('Accuracy: %.2f%%' % (gs.best_estimator_.score(X_test, y_test)*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "<br>\n",
    "<br>\n",
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Randomized Search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Here, each setting is sampled from a distribution (discrete or continuous) over possible parameter values\n",
    "- Instead of running all combinations exhaustively like in GridSearch, a budget is specified (`n_iter` specifies the number of candidates to evaluate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ParamSampler\n",
    "\n",
    "- To explain these concepts, let's take a look at the `ParamSampler` first, which is used under the hood of the `RandomizedSearchCV`; it will help us building some intuition."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see in the example below, lists are also supported; they will behave like gridsearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sebastian/miniconda3/lib/python3.8/site-packages/sklearn/model_selection/_search.py:278: UserWarning: The total space of parameters 6 is smaller than n_iter=10. Running 6 iterations. For exhaustive searches, use GridSearchCV.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'parameter 2': 'a', 'parameter 1': 1},\n",
       " {'parameter 2': 'b', 'parameter 1': 1},\n",
       " {'parameter 2': 'a', 'parameter 1': 2},\n",
       " {'parameter 2': 'b', 'parameter 1': 2},\n",
       " {'parameter 2': 'a', 'parameter 1': 3},\n",
       " {'parameter 2': 'b', 'parameter 1': 3}]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import ParameterSampler\n",
    "\n",
    "\n",
    "d = {'parameter 1': [1, 2, 3], \n",
    "     'parameter 2': ['a', 'b']}\n",
    "\n",
    "ps = ParameterSampler(param_distributions=d,\n",
    "                      n_iter=10, \n",
    "                      random_state=123)\n",
    "\n",
    "list(ps)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- We can define probability distributions using SciPy's selection of distributions (https://docs.scipy.org/doc/scipy/reference/stats.html).\n",
    "\n",
    "- Consider a discrete random uniform distribution for parameter 1 (note that in this case, there is a sampling with replacement from the distribution)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'parameter 1': 3, 'parameter 2': 'b'},\n",
       " {'parameter 1': 3, 'parameter 2': 'a'},\n",
       " {'parameter 1': 1, 'parameter 2': 'a'},\n",
       " {'parameter 1': 3, 'parameter 2': 'b'},\n",
       " {'parameter 1': 3, 'parameter 2': 'b'},\n",
       " {'parameter 1': 2, 'parameter 2': 'a'},\n",
       " {'parameter 1': 2, 'parameter 2': 'a'},\n",
       " {'parameter 1': 2, 'parameter 2': 'a'},\n",
       " {'parameter 1': 2, 'parameter 2': 'a'},\n",
       " {'parameter 1': 3, 'parameter 2': 'a'}]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import scipy.stats\n",
    "\n",
    "d = {'parameter 1': scipy.stats.randint(low=1, high=4), # integer between 1-3\n",
    "     'parameter 2': ['a', 'b']}\n",
    "\n",
    "ps = ParameterSampler(param_distributions=d,\n",
    "                      n_iter=10, \n",
    "                      random_state=123)\n",
    "\n",
    "list(ps)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- below is an example for sampling from a uniform continuous distribution:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAPAUlEQVR4nO3db4xc113G8e9TO0nThiqJvAmubepUMn+cStCwMi6VqkqpFJMinBdEcqW2VhXJogqQIiSw+4KIF5aChCookCKrLbiiJLLSQqzSAMG0qpBowiZNaRzXZGkgXmLibav+A5Ri8+PF3EqjzWx2dma9u7Pn+5FW995zz91zzp7k2btnZq5TVUiS2vCqte6AJGn1GPqS1BBDX5IaYuhLUkMMfUlqyOa17sBStmzZUjt37lzrbkjSRHniiSe+XlVTC8vXfejv3LmTmZmZte6GJE2UJP8+qNzlHUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1Jasi6/0TuJNp5+K/WrO1/u++da9b2Wlmrn3eLP2tNPu/0JakhS4Z+ko8nuZDk6b6y65M8muTZbntd37kjSWaTnE1yW1/5Tyf5Snfuw0my8sORJL2SYZZ3/hT4Q+ATfWWHgVNVdV+Sw93xbybZDRwAbgZeD/xdkh+tqkvAR4BDwBeBzwL7gEdWaiBSS1pb0nLJdOUseadfVV8AvrmgeD9wvNs/DtzRV/5gVb1UVc8Bs8CeJFuB11XVP1bvX2L/RN81kqRVMuqa/o1VdR6g297QlW8DzvXVm+vKtnX7C8sHSnIoyUySmfn5+RG7KElaaKXfvTNonb5eoXygqjoGHAOYnp5etJ4kXW4bbSlt1NB/McnWqjrfLd1c6MrngB199bYDL3Tl2weUa4NYyzVXScMbNfRPAgeB+7rtw33lf57kQ/ReyN0FPF5Vl5J8N8le4DHgvcAfjNXzIbQYRC2Oea34s9YkWjL0kzwAvB3YkmQOuJde2J9IchfwPHAnQFWdTnICeAa4CNzdvXMH4P303gl0Nb137fjOHUlaZUuGflW9a5FTty5S/yhwdED5DPCmZfVO0rriXzeTz0/kSlJDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUkLFCP8mvJTmd5OkkDyR5dZLrkzya5Nlue11f/SNJZpOcTXLb+N2XJC3HyKGfZBvwq8B0Vb0J2AQcAA4Dp6pqF3CqOybJ7u78zcA+4P4km8brviRpOcZd3tkMXJ1kM/Aa4AVgP3C8O38cuKPb3w88WFUvVdVzwCywZ8z2JUnLMHLoV9V/AL8LPA+cB75dVX8L3FhV57s654Ebuku2Aef6vsVcV/YySQ4lmUkyMz8/P2oXJUkLjLO8cx29u/ebgNcDr03y7le6ZEBZDapYVceqarqqpqempkbtoiRpgXGWd94BPFdV81X1v8CngZ8FXkyyFaDbXujqzwE7+q7fTm85SJK0SsYJ/eeBvUlekyTArcAZ4CRwsKtzEHi42z8JHEhyVZKbgF3A42O0L0laps2jXlhVjyV5CHgSuAh8CTgGXAOcSHIXvV8Md3b1Tyc5ATzT1b+7qi6N2X9J0jKMHPoAVXUvcO+C4pfo3fUPqn8UODpOm5Kk0fmJXElqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDxgr9JNcmeSjJV5OcSfKWJNcneTTJs932ur76R5LMJjmb5Lbxuy9JWo5x7/R/H/jrqvpx4CeBM8Bh4FRV7QJOdcck2Q0cAG4G9gH3J9k0ZvuSpGUYOfSTvA54G/AxgKr6flV9C9gPHO+qHQfu6Pb3Aw9W1UtV9RwwC+wZtX1J0vKNc6f/RmAe+JMkX0ry0SSvBW6sqvMA3faGrv424Fzf9XNd2cskOZRkJsnM/Pz8GF2UJPUbJ/Q3A7cAH6mqNwP/RbeUs4gMKKtBFavqWFVNV9X01NTUGF2UJPUbJ/TngLmqeqw7fojeL4EXk2wF6LYX+urv6Lt+O/DCGO1LkpZp5NCvqv8EziX5sa7oVuAZ4CRwsCs7CDzc7Z8EDiS5KslNwC7g8VHblyQt3+Yxr/8V4JNJrgS+BryP3i+SE0nuAp4H7gSoqtNJTtD7xXARuLuqLo3ZviRpGcYK/ap6CpgecOrWReofBY6O06YkaXR+IleSGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIaMHfpJNiX5UpLPdMfXJ3k0ybPd9rq+ukeSzCY5m+S2cduWJC3PStzp3wOc6Ts+DJyqql3Aqe6YJLuBA8DNwD7g/iSbVqB9SdKQxgr9JNuBdwIf7SveDxzv9o8Dd/SVP1hVL1XVc8AssGec9iVJyzPunf7vAb8B/F9f2Y1VdR6g297QlW8DzvXVm+vKXibJoSQzSWbm5+fH7KIk6QdGDv0kPw9cqKonhr1kQFkNqlhVx6pquqqmp6amRu2iJGmBzWNc+1bgF5LcDrwaeF2SPwNeTLK1qs4n2Qpc6OrPATv6rt8OvDBG+5KkZRr5Tr+qjlTV9qraSe8F2r+vqncDJ4GDXbWDwMPd/kngQJKrktwE7AIeH7nnkqRlG+dOfzH3ASeS3AU8D9wJUFWnk5wAngEuAndX1aXL0L4kaRErEvpV9Xng893+N4BbF6l3FDi6Em1KkpbPT+RKUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktSQkUM/yY4kn0tyJsnpJPd05dcneTTJs932ur5rjiSZTXI2yW0rMQBJ0vDGudO/CPx6Vf0EsBe4O8lu4DBwqqp2Aae6Y7pzB4CbgX3A/Uk2jdN5SdLyjBz6VXW+qp7s9r8LnAG2AfuB412148Ad3f5+4MGqeqmqngNmgT2jti9JWr4VWdNPshN4M/AYcGNVnYfeLwbghq7aNuBc32VzXdmg73coyUySmfn5+ZXooiSJFQj9JNcAnwI+UFXfeaWqA8pqUMWqOlZV01U1PTU1NW4XJUmdsUI/yRX0Av+TVfXprvjFJFu781uBC135HLCj7/LtwAvjtC9JWp5x3r0T4GPAmar6UN+pk8DBbv8g8HBf+YEkVyW5CdgFPD5q+5Kk5ds8xrVvBd4DfCXJU13ZB4H7gBNJ7gKeB+4EqKrTSU4Az9B758/dVXVpjPYlScs0cuhX1T8weJ0e4NZFrjkKHB21TUnSePxEriQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhqx76SfYlOZtkNsnh1W5fklq2qqGfZBPwR8DPAbuBdyXZvZp9kKSWrfad/h5gtqq+VlXfBx4E9q9yHySpWZtXub1twLm+4zngZxZWSnIIONQdfi/J2RHb2wJ8fcRr15uNMpaNMg5wLOvVhhhLfmfscbxhUOFqh34GlNXLCqqOAcfGbiyZqarpcb/PerBRxrJRxgGOZb3aKGO5XONY7eWdOWBH3/F24IVV7oMkNWu1Q/+fgF1JbkpyJXAAOLnKfZCkZq3q8k5VXUzyy8DfAJuAj1fV6cvY5NhLROvIRhnLRhkHOJb1aqOM5bKMI1UvW1KXJG1QfiJXkhpi6EtSQzZE6C/1aIf0fLg7/89JblmLfi5liHG8Pcm3kzzVff3WWvRzKUk+nuRCkqcXOT8R8wFDjWUi5gQgyY4kn0tyJsnpJPcMqLPu52bIcUzEvCR5dZLHk3y5G8tvD6izsnNSVRP9Re8F4X8F3ghcCXwZ2L2gzu3AI/Q+J7AXeGyt+z3iON4OfGat+zrEWN4G3AI8vcj5dT8fyxjLRMxJ19etwC3d/g8B/zKh/68MM46JmJfu53xNt38F8Biw93LOyUa40x/m0Q77gU9UzxeBa5NsXe2OLmHDPKKiqr4AfPMVqkzCfABDjWViVNX5qnqy2/8ucIbep+T7rfu5GXIcE6H7OX+vO7yi+1r47poVnZONEPqDHu2w8D+AYeqstWH7+JbuT8FHkty8Ol1bcZMwH8sxcXOSZCfwZnp3lv0mam5eYRwwIfOSZFOSp4ALwKNVdVnnZLUfw3A5DPNoh6Ee/7DGhunjk8Abqup7SW4H/hLYdbk7dhlMwnwMa+LmJMk1wKeAD1TVdxaeHnDJupybJcYxMfNSVZeAn0pyLfAXSd5UVf2vIa3onGyEO/1hHu0wCY9/WLKPVfWdH/wpWFWfBa5IsmX1urhiJmE+hjJpc5LkCnpB+cmq+vSAKhMxN0uNY9LmBaCqvgV8Hti34NSKzslGCP1hHu1wEnhv9yr4XuDbVXV+tTu6hCXHkeSHk6Tb30Nv/r6x6j0d3yTMx1AmaU66fn4MOFNVH1qk2rqfm2HGMSnzkmSqu8MnydXAO4CvLqi2onMy8cs7tcijHZL8Unf+j4HP0nsFfBb4b+B9a9XfxQw5jl8E3p/kIvA/wIHqXt5fT5I8QO/dE1uSzAH30nuBamLm4weGGMtEzEnnrcB7gK90a8gAHwR+BCZqboYZx6TMy1bgeHr/wNSrgBNV9ZnLmV8+hkGSGrIRlnckSUMy9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JD/h9uUUWzb86oNQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "\n",
    "plt.hist(scipy.stats.uniform(loc=0, scale=3).rvs(10000))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'parameter 1': 2.089407556793585, 'parameter 2': 'a'},\n",
       " {'parameter 1': 1.285412778560477, 'parameter 2': 'a'},\n",
       " {'parameter 1': 1.6539443072486737, 'parameter 2': 'b'},\n",
       " {'parameter 1': 1.473356800297792, 'parameter 2': 'b'},\n",
       " {'parameter 1': 2.9422925951538463, 'parameter 2': 'a'},\n",
       " {'parameter 1': 1.7390828910678304, 'parameter 2': 'b'},\n",
       " {'parameter 1': 1.1763525545824516, 'parameter 2': 'a'},\n",
       " {'parameter 1': 1.8819510280128502, 'parameter 2': 'b'},\n",
       " {'parameter 1': 1.3157167340388733, 'parameter 2': 'b'},\n",
       " {'parameter 1': 2.084265531555807, 'parameter 2': 'a'}]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d = {'parameter 1': scipy.stats.uniform(loc=0, scale=3), # float between 1-3\n",
    "     'parameter 2': ['a', 'b']}\n",
    "\n",
    "ps = ParameterSampler(param_distributions=d,\n",
    "                      n_iter=10, \n",
    "                      random_state=123)\n",
    "\n",
    "list(ps)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "<br>\n",
    "<br>\n",
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RandomizedSeachCV example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAR0klEQVR4nO3df6jd9X3H8eer0VppJ1W8upCbNRbCNhVsa8gyhNHNbYY5Gv+YkMFmGEKYuNGyQYn9Y6N/BNw/Y3NMR2g7I+sqYb8Mdm4LacsYOO21tbUxOrPq9JLM3LZ0tduwaN/7434Gh5tz7/3emJxzzef5gMP5nvf38z3f9/l47ytfv+d7zk1VIUnqwzum3YAkaXIMfUnqiKEvSR0x9CWpI4a+JHXkomk3sJorr7yytmzZMu02JOlt5amnnvpWVc0sra/70N+yZQtzc3PTbkOS3laS/Me4uqd3JKkjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI+v+E7lvxZZ9n5/Kfl+699ap7FeSVuORviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwaFfpL3JvmrJM8lOZ7kp5NckeRIkhfa/eUj4+9JciLJ80luGanfmOSZtu6+JDkfL0qSNN7QI/0/Bv6hqn4CuAE4DuwDjlbVVuBoe0ySa4HdwHXATuD+JBva8zwA7AW2ttvOc/Q6JEkDrBr6SS4Dfgb4NEBV/aCqvgvsAg62YQeB29ryLuDhqnq9ql4ETgDbk2wELquqx6uqgIdGtpEkTcCQI/33AwvAnyf5apJPJXk3cHVVnQJo91e18ZuAV0a2n2+1TW15aV2SNCFDQv8i4EPAA1X1QeC/aadyljHuPH2tUD/zCZK9SeaSzC0sLAxoUZI0xJDQnwfmq+qJ9vivWPxH4NV2yoZ2f3pk/OaR7WeBk60+O6Z+hqo6UFXbqmrbzMwZf8xdknSWVg39qvpP4JUkP95KNwPPAoeBPa22B3ikLR8Gdie5JMk1LL5h+2Q7BfRakh3tqp07RraRJE3A0C9c+23gs0neCXwT+A0W/8E4lORO4GXgdoCqOpbkEIv/MLwB3F1Vb7bnuQt4ELgUeKzdJEkTMij0q+ppYNuYVTcvM34/sH9MfQ64fg39SZLOIT+RK0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6sig0E/yUpJnkjydZK7VrkhyJMkL7f7ykfH3JDmR5Pkkt4zUb2zPcyLJfUly7l+SJGk5aznS/9mq+kBVbWuP9wFHq2orcLQ9Jsm1wG7gOmAncH+SDW2bB4C9wNZ22/nWX4Ikaai3cnpnF3CwLR8EbhupP1xVr1fVi8AJYHuSjcBlVfV4VRXw0Mg2kqQJGBr6BfxTkqeS7G21q6vqFEC7v6rVNwGvjGw732qb2vLS+hmS7E0yl2RuYWFhYIuSpNVcNHDcTVV1MslVwJEkz60wdtx5+lqhfmax6gBwAGDbtm1jx0iS1m7QkX5VnWz3p4G/BbYDr7ZTNrT70234PLB5ZPNZ4GSrz46pS5ImZNXQT/LuJD/y/8vALwLfAA4De9qwPcAjbfkwsDvJJUmuYfEN2yfbKaDXkuxoV+3cMbKNJGkChpzeuRr423Z15UXAX1bVPyT5MnAoyZ3Ay8DtAFV1LMkh4FngDeDuqnqzPdddwIPApcBj7SZJmpBVQ7+qvgncMKb+beDmZbbZD+wfU58Drl97m5Kkc8FP5EpSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjoyOPSTbEjy1SSPtsdXJDmS5IV2f/nI2HuSnEjyfJJbRuo3JnmmrbsvSc7ty5EkrWQtR/ofBY6PPN4HHK2qrcDR9pgk1wK7geuAncD9STa0bR4A9gJb223nW+pekrQmg0I/ySxwK/CpkfIu4GBbPgjcNlJ/uKper6oXgRPA9iQbgcuq6vGqKuChkW0kSRMw9Ej/j4CPAz8cqV1dVacA2v1Vrb4JeGVk3HyrbWrLS+tnSLI3yVySuYWFhYEtSpJWs2roJ/ll4HRVPTXwOcedp68V6mcWqw5U1baq2jYzMzNwt5Kk1Vw0YMxNwEeS/BLwLuCyJH8BvJpkY1WdaqduTrfx88Dmke1ngZOtPjumLkmakFWP9KvqnqqaraotLL5B+4Wq+jXgMLCnDdsDPNKWDwO7k1yS5BoW37B9sp0Cei3JjnbVzh0j20iSJmDIkf5y7gUOJbkTeBm4HaCqjiU5BDwLvAHcXVVvtm3uAh4ELgUeazdJ0oSsKfSr6kvAl9ryt4Gblxm3H9g/pj4HXL/WJiVJ54afyJWkjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHVk1dBP8q4kTyb5WpJjST7Z6lckOZLkhXZ/+cg29yQ5keT5JLeM1G9M8kxbd1+SnJ+XJUkaZ8iR/uvAz1XVDcAHgJ1JdgD7gKNVtRU42h6T5FpgN3AdsBO4P8mG9lwPAHuBre2289y9FEnSalYN/Vr0/fbw4nYrYBdwsNUPAre15V3Aw1X1elW9CJwAtifZCFxWVY9XVQEPjWwjSZqAQef0k2xI8jRwGjhSVU8AV1fVKYB2f1Ubvgl4ZWTz+Vbb1JaX1sftb2+SuSRzCwsLa3g5kqSVDAr9qnqzqj4AzLJ41H79CsPHnaevFerj9negqrZV1baZmZkhLUqSBljT1TtV9V3gSyyei3+1nbKh3Z9uw+aBzSObzQInW312TF2SNCFDrt6ZSfLetnwp8PPAc8BhYE8btgd4pC0fBnYnuSTJNSy+YftkOwX0WpId7aqdO0a2kSRNwEUDxmwEDrYrcN4BHKqqR5M8DhxKcifwMnA7QFUdS3IIeBZ4A7i7qt5sz3UX8CBwKfBYu0mSJmTV0K+qrwMfHFP/NnDzMtvsB/aPqc8BK70fIEk6j/xEriR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI0P+cpbWaMu+z09t3y/de+vU9i1p/fNIX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjqwa+kk2J/likuNJjiX5aKtfkeRIkhfa/eUj29yT5ESS55PcMlK/Mckzbd19SXJ+XpYkaZwhR/pvAL9bVT8J7ADuTnItsA84WlVbgaPtMW3dbuA6YCdwf5IN7bkeAPYCW9tt5zl8LZKkVawa+lV1qqq+0pZfA44Dm4BdwME27CBwW1veBTxcVa9X1YvACWB7ko3AZVX1eFUV8NDINpKkCVjTOf0kW4APAk8AV1fVKVj8hwG4qg3bBLwystl8q21qy0vr4/azN8lckrmFhYW1tChJWsHg0E/yHuCvgY9V1fdWGjqmVivUzyxWHaiqbVW1bWZmZmiLkqRVDAr9JBezGPifraq/aeVX2ykb2v3pVp8HNo9sPgucbPXZMXVJ0oQMuXonwKeB41X1hyOrDgN72vIe4JGR+u4klyS5hsU3bJ9sp4BeS7KjPecdI9tIkiZgyFcr3wT8OvBMkqdb7RPAvcChJHcCLwO3A1TVsSSHgGdZvPLn7qp6s213F/AgcCnwWLtJkiZk1dCvqn9h/Pl4gJuX2WY/sH9MfQ64fi0NSpLOHT+RK0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6siqoZ/kM0lOJ/nGSO2KJEeSvNDuLx9Zd0+SE0meT3LLSP3GJM+0dfclybl/OZKklQw50n8Q2Lmktg84WlVbgaPtMUmuBXYD17Vt7k+yoW3zALAX2NpuS59TknSerRr6VfXPwHeWlHcBB9vyQeC2kfrDVfV6Vb0InAC2J9kIXFZVj1dVAQ+NbCNJmpCzPad/dVWdAmj3V7X6JuCVkXHzrbapLS+tj5Vkb5K5JHMLCwtn2aIkaalz/UbuuPP0tUJ9rKo6UFXbqmrbzMzMOWtOknp3tqH/ajtlQ7s/3erzwOaRcbPAyVafHVOXJE3Q2Yb+YWBPW94DPDJS353kkiTXsPiG7ZPtFNBrSXa0q3buGNlGkjQhF602IMnngA8DVyaZB34fuBc4lORO4GXgdoCqOpbkEPAs8AZwd1W92Z7qLhavBLoUeKzddI5t2ff5qez3pXtvncp+Ja3NqqFfVb+6zKqblxm/H9g/pj4HXL+m7iRJ55SfyJWkjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOrPqXs6QhpvVnGsE/1SithUf6ktQRQ1+SOmLoS1JHDH1J6oihL0kd8eodve1N68ohrxrS29HEj/ST7EzyfJITSfZNev+S1LOJHukn2QD8KfALwDzw5SSHq+rZSfYhnQt+NkFvR5M+0t8OnKiqb1bVD4CHgV0T7kGSujXpc/qbgFdGHs8DP7V0UJK9wN728PtJnj/L/V0JfOsstz3f7O3srNfeJtpX/mBNw9frnIG9na0hvb1vXHHSoZ8xtTqjUHUAOPCWd5bMVdW2t/o854O9nZ312tt67Qvs7WxdqL1N+vTOPLB55PEscHLCPUhStyYd+l8Gtia5Jsk7gd3A4Qn3IEndmujpnap6I8lvAf8IbAA+U1XHzuMu3/IpovPI3s7Oeu1tvfYF9na2LsjeUnXGKXVJ0gXKr2GQpI4Y+pLUkQsi9Ff7aocsuq+t/3qSD62Tvj6c5L+SPN1uvzeJvtq+P5PkdJJvLLN+KnM2sLepzFuSzUm+mOR4kmNJPjpmzLR+1ob0Nq15e1eSJ5N8rfX2yTFjpjVvQ3qb2u9p2/+GJF9N8uiYdWuft6p6W99YfEP434H3A+8EvgZcu2TMLwGPsfg5gR3AE+ukrw8Dj05p3n4G+BDwjWXWT3zO1tDbVOYN2Ah8qC3/CPBv6+FnbQ29TWveArynLV8MPAHsWCfzNqS3qf2etv3/DvCX43o4m3m7EI70h3y1wy7goVr0r8B7k2xcB31NTVX9M/CdFYZMY86G9jYVVXWqqr7Sll8DjrP4KfNRU5m3gb1NRZuL77eHF7fb0itIpjVvQ3qbmiSzwK3Ap5YZsuZ5uxBCf9xXOyz9YR8yZhp9Afx0+1/Lx5Jcd557WotpzNlaTHXekmwBPsjikeGoqc/bCr3BlOatnaJ4GjgNHKmqdTNvA3qD6f28/RHwceCHy6xf87xdCKE/5KsdBn39wzk2ZJ9fAd5XVTcAfwL83XnuaS2mMWdDTXXekrwH+GvgY1X1vaWrx2wysXlbpbepzVtVvVlVH2DxU/jbk1y/ZMjU5m1Ab1OZtyS/DJyuqqdWGjamtuK8XQihP+SrHabx9Q+r7rOqvvf//2tZVX8PXJzkyvPc11Dr9iszpjlvSS5mMVQ/W1V/M2bI1OZttd7Ww89bVX0X+BKwc8mqqf+8LdfbFOftJuAjSV5i8fTwzyX5iyVj1jxvF0LoD/lqh8PAHe2d7h3Af1XVqWn3leRHk6Qtb2fxv8e3z3NfQ01jzgaZ1ry1fX4aOF5Vf7jMsKnM25DepjhvM0ne25YvBX4eeG7JsGnN26q9TWvequqeqpqtqi0s5scXqurXlgxb87y97f9cYi3z1Q5JfrOt/zPg71l8l/sE8D/Ab6yTvn4FuCvJG8D/ArurvSV/viX5HItXJVyZZB74fRbfxJranK2ht2nN203ArwPPtHPAAJ8Afmykt2nN25DepjVvG4GDWfwjSu8ADlXVo9P+HV1Db1P7PR3nrc6bX8MgSR25EE7vSJIGMvQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSR/4PZV+wqs4SmiIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(scipy.stats.expon(scale=0.4).rvs(10000))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.experimental import enable_hist_gradient_boosting\n",
    "from sklearn.ensemble import HistGradientBoostingClassifier\n",
    "from sklearn.model_selection import RandomizedSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Accuracy: 96.73%\n",
      "Best Params: {'l2_regularization': 0.7770044388897804, 'learning_rate': 0.011352363969357031, 'max_depth': 5}\n"
     ]
    }
   ],
   "source": [
    "gbm = HistGradientBoostingClassifier(random_state=123)\n",
    "\n",
    "d = {'learning_rate': scipy.stats.expon(scale=0.4),\n",
    "     'max_depth': scipy.stats.randint(low=3, high=10),\n",
    "     'l2_regularization': scipy.stats.uniform(loc=0, scale=1)}\n",
    "\n",
    "rcv = RandomizedSearchCV(estimator=gbm,\n",
    "                         param_distributions=d, \n",
    "                         n_iter=200,\n",
    "                         cv=10,\n",
    "                         refit=True,\n",
    "                         random_state=123,\n",
    "                         n_jobs=-1)\n",
    "\n",
    "rcv.fit(X_train, y_train)\n",
    "\n",
    "print('Best Accuracy: %.2f%%' % (rcv.best_score_*100))\n",
    "print('Best Params:', rcv.best_params_)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}