examples/demo_imbalanced_learn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://kakate%40us.ibm.com:****@na.artifactory.swg-devops.com/artifactory/api/pypi/wcp-nlp-pypi-virtual/simple\r\n",
      "Requirement already satisfied: liac-arff>=2.4.0 in /Users/kakateus.ibm.com/venv/lale39/lib/python3.9/site-packages (2.5.0)\r\n"
     ]
    }
   ],
   "source": [
    "#Required for accessing openml datasets from Lale\n",
    "!pip install 'liac-arff>=2.4.0'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset with class imbalance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lale.datasets.openml\n",
    "import pandas as pd\n",
    "(train_X, train_y), (test_X, test_y) = lale.datasets.openml.fetch(\n",
    "    'breast-cancer', 'classification', preprocess=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([140,  51])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.bincount(train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.kernel_approximation import Nystroem as Nys\n",
    "from lale.lib.lale import NoOp\n",
    "from sklearn.linear_model import LogisticRegression as LR\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from lale.lib.lale import Hyperopt\n",
    "from lale.lib.imblearn import SMOTE, CondensedNearestNeighbour, SMOTEENN\n",
    "from sklearn.metrics import accuracy_score\n",
    "import lale\n",
    "lale.wrap_imported_operators()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### A pipeline without any imbalance correction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: cluster:(root) Pages: 1 -->\n",
       "<svg width=\"258pt\" height=\"60pt\"\n",
       " viewBox=\"0.00 0.00 258.08 60.33\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 56.3259)\">\n",
       "<title>cluster:(root)</title>\n",
       "<g id=\"a_graph0\"><a xlink:title=\"(root) = ...\">\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-56.3259 254.083,-56.3259 254.083,4 -4,4\"/>\n",
       "</a>\n",
       "</g>\n",
       "<!-- min_max_scaler -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>min_max_scaler</title>\n",
       "<g id=\"a_node1\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.sklearn.min_max_scaler.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"min_max_scaler = MinMaxScaler()\">\n",
       "<ellipse fill=\"#b0e2ff\" stroke=\"#000000\" cx=\"27\" cy=\"-26.163\" rx=\"27\" ry=\"26.3269\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-33.863\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Min&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-22.863\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Max&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-11.863\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Scaler</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- pca -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>pca</title>\n",
       "<g id=\"a_node2\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.sklearn.pca.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"pca = PCA()\">\n",
       "<ellipse fill=\"#b0e2ff\" stroke=\"#000000\" cx=\"117\" cy=\"-26.163\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-22.863\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">PCA</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- min_max_scaler&#45;&gt;pca -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>min_max_scaler&#45;&gt;pca</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M54.003,-26.163C62.0277,-26.163 70.9665,-26.163 79.5309,-26.163\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"79.7051,-29.663 89.705,-26.163 79.705,-22.663 79.7051,-29.663\"/>\n",
       "</g>\n",
       "<!-- random_forest_classifier -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>random_forest_classifier</title>\n",
       "<g id=\"a_node3\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.sklearn.random_forest_classifier.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"random_forest_classifier = RandomForestClassifier()\">\n",
       "<ellipse fill=\"#b0e2ff\" stroke=\"#000000\" cx=\"215.0415\" cy=\"-26.163\" rx=\"35.083\" ry=\"26.3269\"/>\n",
       "<text text-anchor=\"middle\" x=\"215.0415\" y=\"-33.863\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Random&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"215.0415\" y=\"-22.863\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Forest&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"215.0415\" y=\"-11.863\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Classifier</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- pca&#45;&gt;random_forest_classifier -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>pca&#45;&gt;random_forest_classifier</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M144.0498,-26.163C152.0345,-26.163 160.9997,-26.163 169.8014,-26.163\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"169.9902,-29.663 179.9902,-26.163 169.9902,-22.663 169.9902,-29.663\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x143d710d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pipeline_without_correction =  MinMaxScaler() >> PCA() >> RandomForestClassifier()\n",
    "pipeline_without_correction.visualize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████| 10/10 [00:14<00:00,  1.41s/trial, best loss: -0.6583766233766233]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tid</th>\n",
       "      <th>loss</th>\n",
       "      <th>time</th>\n",
       "      <th>log_loss</th>\n",
       "      <th>status</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>p0</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.623864</td>\n",
       "      <td>0.203896</td>\n",
       "      <td>0.567272</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p1</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.631656</td>\n",
       "      <td>0.237808</td>\n",
       "      <td>0.566987</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p2</th>\n",
       "      <td>2</td>\n",
       "      <td>-0.658377</td>\n",
       "      <td>0.229631</td>\n",
       "      <td>0.558715</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p3</th>\n",
       "      <td>3</td>\n",
       "      <td>-0.579383</td>\n",
       "      <td>0.212269</td>\n",
       "      <td>0.628262</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p4</th>\n",
       "      <td>4</td>\n",
       "      <td>-0.616786</td>\n",
       "      <td>0.252034</td>\n",
       "      <td>0.567862</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p5</th>\n",
       "      <td>5</td>\n",
       "      <td>-0.618864</td>\n",
       "      <td>0.281458</td>\n",
       "      <td>0.585125</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p6</th>\n",
       "      <td>6</td>\n",
       "      <td>-0.650032</td>\n",
       "      <td>0.342633</td>\n",
       "      <td>0.556056</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p7</th>\n",
       "      <td>7</td>\n",
       "      <td>-0.652825</td>\n",
       "      <td>0.270016</td>\n",
       "      <td>0.558804</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p8</th>\n",
       "      <td>8</td>\n",
       "      <td>-0.633604</td>\n",
       "      <td>0.214408</td>\n",
       "      <td>0.578119</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p9</th>\n",
       "      <td>9</td>\n",
       "      <td>-0.632500</td>\n",
       "      <td>0.278336</td>\n",
       "      <td>0.576655</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      tid      loss      time  log_loss status\n",
       "name                                          \n",
       "p0      0 -0.623864  0.203896  0.567272     ok\n",
       "p1      1 -0.631656  0.237808  0.566987     ok\n",
       "p2      2 -0.658377  0.229631  0.558715     ok\n",
       "p3      3 -0.579383  0.212269  0.628262     ok\n",
       "p4      4 -0.616786  0.252034  0.567862     ok\n",
       "p5      5 -0.618864  0.281458  0.585125     ok\n",
       "p6      6 -0.650032  0.342633  0.556056     ok\n",
       "p7      7 -0.652825  0.270016  0.558804     ok\n",
       "p8      8 -0.633604  0.214408  0.578119     ok\n",
       "p9      9 -0.632500  0.278336  0.576655     ok"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimizer = Hyperopt(estimator=pipeline_without_correction, max_evals = 10, scoring='roc_auc')\n",
    "trained_optimizer = optimizer.fit(np.array(train_X), np.array(train_y))\n",
    "predictions = trained_optimizer.predict(test_X)\n",
    "trained_optimizer.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6421052631578947\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: cluster:(root) Pages: 1 -->\n",
       "<svg width=\"258pt\" height=\"60pt\"\n",
       " viewBox=\"0.00 0.00 258.08 60.33\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 56.3259)\">\n",
       "<title>cluster:(root)</title>\n",
       "<g id=\"a_graph0\"><a xlink:title=\"(root) = ...\">\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-56.3259 254.083,-56.3259 254.083,4 -4,4\"/>\n",
       "</a>\n",
       "</g>\n",
       "<!-- min_max_scaler -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>min_max_scaler</title>\n",
       "<g id=\"a_node1\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.sklearn.min_max_scaler.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"min_max_scaler = MinMaxScaler\">\n",
       "<ellipse fill=\"#ffffff\" stroke=\"#000000\" cx=\"27\" cy=\"-26.163\" rx=\"27\" ry=\"26.3269\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-33.863\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Min&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-22.863\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Max&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-11.863\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Scaler</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- pca -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>pca</title>\n",
       "<g id=\"a_node2\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.sklearn.pca.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"pca = PCA(n_components=0.34508967502645044, svd_solver=&#39;full&#39;)\">\n",
       "<ellipse fill=\"#ffffff\" stroke=\"#000000\" cx=\"117\" cy=\"-26.163\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-22.863\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">PCA</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- min_max_scaler&#45;&gt;pca -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>min_max_scaler&#45;&gt;pca</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M54.003,-26.163C62.0277,-26.163 70.9665,-26.163 79.5309,-26.163\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"79.7051,-29.663 89.705,-26.163 79.705,-22.663 79.7051,-29.663\"/>\n",
       "</g>\n",
       "<!-- random_forest_classifier -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>random_forest_classifier</title>\n",
       "<g id=\"a_node3\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.sklearn.random_forest_classifier.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"random_forest_classifier = RandomForestClassifier(criterion=&#39;entropy&#39;, max_features=None, min_samples_leaf=0.30443331216349984, min_samples_split=0.3224970555951517, n_estimators=58)\">\n",
       "<ellipse fill=\"#ffffff\" stroke=\"#000000\" cx=\"215.0415\" cy=\"-26.163\" rx=\"35.083\" ry=\"26.3269\"/>\n",
       "<text text-anchor=\"middle\" x=\"215.0415\" y=\"-33.863\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Random&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"215.0415\" y=\"-22.863\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Forest&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"215.0415\" y=\"-11.863\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Classifier</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- pca&#45;&gt;random_forest_classifier -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>pca&#45;&gt;random_forest_classifier</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M144.0498,-26.163C152.0345,-26.163 160.9997,-26.163 169.8014,-26.163\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"169.9902,-29.663 179.9902,-26.163 169.9902,-22.663 169.9902,-29.663\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x14445f070>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(accuracy_score(test_y, predictions))\n",
    "best_estimator = trained_optimizer.get_pipeline()\n",
    "best_estimator.visualize()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Over-sampling from imbalanced-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████| 10/10 [00:14<00:00,  1.40s/trial, best loss: -0.6835011143534503]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tid</th>\n",
       "      <th>loss</th>\n",
       "      <th>time</th>\n",
       "      <th>log_loss</th>\n",
       "      <th>status</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>p0</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.624954</td>\n",
       "      <td>0.190958</td>\n",
       "      <td>0.616522</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p1</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.650157</td>\n",
       "      <td>0.242815</td>\n",
       "      <td>0.655870</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p2</th>\n",
       "      <td>2</td>\n",
       "      <td>-0.683501</td>\n",
       "      <td>0.211413</td>\n",
       "      <td>0.666853</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p3</th>\n",
       "      <td>3</td>\n",
       "      <td>-0.604101</td>\n",
       "      <td>0.185956</td>\n",
       "      <td>0.599714</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p4</th>\n",
       "      <td>4</td>\n",
       "      <td>-0.517529</td>\n",
       "      <td>0.265853</td>\n",
       "      <td>0.655569</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p5</th>\n",
       "      <td>5</td>\n",
       "      <td>-0.556187</td>\n",
       "      <td>0.363249</td>\n",
       "      <td>0.635348</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p6</th>\n",
       "      <td>6</td>\n",
       "      <td>-0.568477</td>\n",
       "      <td>0.298747</td>\n",
       "      <td>0.648213</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p7</th>\n",
       "      <td>7</td>\n",
       "      <td>-0.579616</td>\n",
       "      <td>0.228733</td>\n",
       "      <td>0.658655</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p8</th>\n",
       "      <td>8</td>\n",
       "      <td>-0.630238</td>\n",
       "      <td>0.222252</td>\n",
       "      <td>0.617986</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p9</th>\n",
       "      <td>9</td>\n",
       "      <td>-0.590678</td>\n",
       "      <td>0.328617</td>\n",
       "      <td>0.641440</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      tid      loss      time  log_loss status\n",
       "name                                          \n",
       "p0      0 -0.624954  0.190958  0.616522     ok\n",
       "p1      1 -0.650157  0.242815  0.655870     ok\n",
       "p2      2 -0.683501  0.211413  0.666853     ok\n",
       "p3      3 -0.604101  0.185956  0.599714     ok\n",
       "p4      4 -0.517529  0.265853  0.655569     ok\n",
       "p5      5 -0.556187  0.363249  0.635348     ok\n",
       "p6      6 -0.568477  0.298747  0.648213     ok\n",
       "p7      7 -0.579616  0.228733  0.658655     ok\n",
       "p8      8 -0.630238  0.222252  0.617986     ok\n",
       "p9      9 -0.590678  0.328617  0.641440     ok"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline_with_correction =  SMOTE(operator = MinMaxScaler() >> PCA() >> RandomForestClassifier())\n",
    "optimizer = Hyperopt(estimator=pipeline_with_correction, max_evals = 10, scoring='roc_auc')\n",
    "trained_optimizer = optimizer.fit(np.array(train_X), np.array(train_y))\n",
    "predictions = trained_optimizer.predict(test_X)\n",
    "trained_optimizer.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.631578947368421\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: cluster:(root) Pages: 1 -->\n",
       "<svg width=\"306pt\" height=\"130pt\"\n",
       " viewBox=\"0.00 0.00 306.08 130.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 126)\">\n",
       "<title>cluster:(root)</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-126 302.083,-126 302.083,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\">\n",
       "<title>cluster:(root)</title>\n",
       "<g id=\"a_clust1\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.imblearn.smote.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"(root) = SMOTE(operator=pipeline)\">\n",
       "<polygon fill=\"#ffffff\" stroke=\"#000000\" points=\"8,-8 8,-114 290.083,-114 290.083,-8 8,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"149.0415\" y=\"-98.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">SMOTE</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\">\n",
       "<title>cluster:pipeline</title>\n",
       "<g id=\"a_clust2\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.imblearn.smote.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"pipeline = ...\">\n",
       "<path fill=\"#b0e2ff\" stroke=\"#000000\" d=\"M28,-16C28,-16 270.083,-16 270.083,-16 276.083,-16 282.083,-22 282.083,-28 282.083,-28 282.083,-72 282.083,-72 282.083,-78 276.083,-84 270.083,-84 270.083,-84 28,-84 28,-84 22,-84 16,-78 16,-72 16,-72 16,-28 16,-28 16,-22 22,-16 28,-16\"/>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- min_max_scaler -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>min_max_scaler</title>\n",
       "<g id=\"a_node1\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.sklearn.min_max_scaler.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"min_max_scaler = MinMaxScaler()\">\n",
       "<ellipse fill=\"#b0e2ff\" stroke=\"#000000\" cx=\"51\" cy=\"-50\" rx=\"27\" ry=\"26.3269\"/>\n",
       "<text text-anchor=\"middle\" x=\"51\" y=\"-57.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Min&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"51\" y=\"-46.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Max&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"51\" y=\"-35.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Scaler</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- pca -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>pca</title>\n",
       "<g id=\"a_node2\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.sklearn.pca.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"pca = PCA(n_components=0.34508967502645044, svd_solver=&#39;full&#39;)\">\n",
       "<ellipse fill=\"#b0e2ff\" stroke=\"#000000\" cx=\"141\" cy=\"-50\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"141\" y=\"-46.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">PCA</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- min_max_scaler&#45;&gt;pca -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>min_max_scaler&#45;&gt;pca</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M78.003,-50C86.0277,-50 94.9665,-50 103.5309,-50\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"103.7051,-53.5001 113.705,-50 103.705,-46.5001 103.7051,-53.5001\"/>\n",
       "</g>\n",
       "<!-- random_forest_classifier -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>random_forest_classifier</title>\n",
       "<g id=\"a_node3\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.sklearn.random_forest_classifier.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"random_forest_classifier = RandomForestClassifier(criterion=&#39;entropy&#39;, max_features=None, min_samples_leaf=0.30443331216349984, min_samples_split=0.3224970555951517, n_estimators=58)\">\n",
       "<ellipse fill=\"#b0e2ff\" stroke=\"#000000\" cx=\"239.0415\" cy=\"-50\" rx=\"35.083\" ry=\"26.3269\"/>\n",
       "<text text-anchor=\"middle\" x=\"239.0415\" y=\"-57.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Random&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"239.0415\" y=\"-46.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Forest&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"239.0415\" y=\"-35.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Classifier</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- pca&#45;&gt;random_forest_classifier -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>pca&#45;&gt;random_forest_classifier</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M168.0498,-50C176.0345,-50 184.9997,-50 193.8014,-50\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"193.9902,-53.5001 203.9902,-50 193.9902,-46.5001 193.9902,-53.5001\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x1424103d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(accuracy_score(test_y, predictions))\n",
    "best_estimator = trained_optimizer.get_pipeline()\n",
    "best_estimator.visualize()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Under-sampling from imbalanced-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████| 10/10 [00:22<00:00,  2.26s/trial, best loss: -0.6813611664876958]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tid</th>\n",
       "      <th>loss</th>\n",
       "      <th>time</th>\n",
       "      <th>log_loss</th>\n",
       "      <th>status</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>p0</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.642447</td>\n",
       "      <td>0.353958</td>\n",
       "      <td>0.604473</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p1</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.598246</td>\n",
       "      <td>0.424864</td>\n",
       "      <td>0.626900</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p2</th>\n",
       "      <td>2</td>\n",
       "      <td>-0.679520</td>\n",
       "      <td>0.452510</td>\n",
       "      <td>0.607253</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p3</th>\n",
       "      <td>3</td>\n",
       "      <td>-0.581329</td>\n",
       "      <td>0.354175</td>\n",
       "      <td>0.648877</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p4</th>\n",
       "      <td>4</td>\n",
       "      <td>-0.626233</td>\n",
       "      <td>0.468622</td>\n",
       "      <td>0.619565</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p5</th>\n",
       "      <td>5</td>\n",
       "      <td>-0.644204</td>\n",
       "      <td>0.416804</td>\n",
       "      <td>0.613639</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p6</th>\n",
       "      <td>6</td>\n",
       "      <td>-0.649106</td>\n",
       "      <td>0.445763</td>\n",
       "      <td>0.618898</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p7</th>\n",
       "      <td>7</td>\n",
       "      <td>-0.599145</td>\n",
       "      <td>0.505183</td>\n",
       "      <td>0.624448</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p8</th>\n",
       "      <td>8</td>\n",
       "      <td>-0.584214</td>\n",
       "      <td>0.368897</td>\n",
       "      <td>0.622633</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p9</th>\n",
       "      <td>9</td>\n",
       "      <td>-0.681361</td>\n",
       "      <td>0.444388</td>\n",
       "      <td>0.582320</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      tid      loss      time  log_loss status\n",
       "name                                          \n",
       "p0      0 -0.642447  0.353958  0.604473     ok\n",
       "p1      1 -0.598246  0.424864  0.626900     ok\n",
       "p2      2 -0.679520  0.452510  0.607253     ok\n",
       "p3      3 -0.581329  0.354175  0.648877     ok\n",
       "p4      4 -0.626233  0.468622  0.619565     ok\n",
       "p5      5 -0.644204  0.416804  0.613639     ok\n",
       "p6      6 -0.649106  0.445763  0.618898     ok\n",
       "p7      7 -0.599145  0.505183  0.624448     ok\n",
       "p8      8 -0.584214  0.368897  0.622633     ok\n",
       "p9      9 -0.681361  0.444388  0.582320     ok"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline_with_correction =  CondensedNearestNeighbour(operator = MinMaxScaler() >> PCA() >> RandomForestClassifier())\n",
    "optimizer = Hyperopt(estimator=pipeline_with_correction, max_evals = 10, scoring='roc_auc')\n",
    "trained_optimizer = optimizer.fit(np.array(train_X), np.array(train_y))\n",
    "predictions = trained_optimizer.predict(test_X)\n",
    "trained_optimizer.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6947368421052632\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: cluster:(root) Pages: 1 -->\n",
       "<svg width=\"306pt\" height=\"130pt\"\n",
       " viewBox=\"0.00 0.00 306.08 130.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 126)\">\n",
       "<title>cluster:(root)</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-126 302.083,-126 302.083,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\">\n",
       "<title>cluster:(root)</title>\n",
       "<g id=\"a_clust1\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.imblearn.condensed_nearest_neighbour.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"(root) = CondensedNearestNeighbour(operator=pipeline)\">\n",
       "<polygon fill=\"#ffffff\" stroke=\"#000000\" points=\"8,-8 8,-114 290.083,-114 290.083,-8 8,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"149.0415\" y=\"-98.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">CondensedNearestNeighbour</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\">\n",
       "<title>cluster:pipeline</title>\n",
       "<g id=\"a_clust2\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.imblearn.condensed_nearest_neighbour.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"pipeline = ...\">\n",
       "<path fill=\"#b0e2ff\" stroke=\"#000000\" d=\"M28,-16C28,-16 270.083,-16 270.083,-16 276.083,-16 282.083,-22 282.083,-28 282.083,-28 282.083,-72 282.083,-72 282.083,-78 276.083,-84 270.083,-84 270.083,-84 28,-84 28,-84 22,-84 16,-78 16,-72 16,-72 16,-28 16,-28 16,-22 22,-16 28,-16\"/>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- min_max_scaler -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>min_max_scaler</title>\n",
       "<g id=\"a_node1\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.sklearn.min_max_scaler.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"min_max_scaler = MinMaxScaler()\">\n",
       "<ellipse fill=\"#b0e2ff\" stroke=\"#000000\" cx=\"51\" cy=\"-50\" rx=\"27\" ry=\"26.3269\"/>\n",
       "<text text-anchor=\"middle\" x=\"51\" y=\"-57.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Min&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"51\" y=\"-46.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Max&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"51\" y=\"-35.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Scaler</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- pca -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>pca</title>\n",
       "<g id=\"a_node2\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.sklearn.pca.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"pca = PCA(svd_solver=&#39;full&#39;, whiten=True)\">\n",
       "<ellipse fill=\"#b0e2ff\" stroke=\"#000000\" cx=\"141\" cy=\"-50\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"141\" y=\"-46.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">PCA</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- min_max_scaler&#45;&gt;pca -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>min_max_scaler&#45;&gt;pca</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M78.003,-50C86.0277,-50 94.9665,-50 103.5309,-50\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"103.7051,-53.5001 113.705,-50 103.705,-46.5001 103.7051,-53.5001\"/>\n",
       "</g>\n",
       "<!-- random_forest_classifier -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>random_forest_classifier</title>\n",
       "<g id=\"a_node3\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.sklearn.random_forest_classifier.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"random_forest_classifier = RandomForestClassifier(criterion=&#39;entropy&#39;, max_depth=4, max_features=None, min_samples_leaf=3, min_samples_split=3, n_estimators=53)\">\n",
       "<ellipse fill=\"#b0e2ff\" stroke=\"#000000\" cx=\"239.0415\" cy=\"-50\" rx=\"35.083\" ry=\"26.3269\"/>\n",
       "<text text-anchor=\"middle\" x=\"239.0415\" y=\"-57.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Random&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"239.0415\" y=\"-46.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Forest&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"239.0415\" y=\"-35.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Classifier</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- pca&#45;&gt;random_forest_classifier -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>pca&#45;&gt;random_forest_classifier</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M168.0498,-50C176.0345,-50 184.9997,-50 193.8014,-50\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"193.9902,-53.5001 203.9902,-50 193.9902,-46.5001 193.9902,-53.5001\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x1469763a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(accuracy_score(test_y, predictions))\n",
    "best_estimator = trained_optimizer.get_pipeline()\n",
    "best_estimator.visualize()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combined over and under sampling from imbalanced-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████| 10/10 [00:12<00:00,  1.26s/trial, best loss: -0.664063068312234]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tid</th>\n",
       "      <th>loss</th>\n",
       "      <th>time</th>\n",
       "      <th>log_loss</th>\n",
       "      <th>status</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>p0</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.574195</td>\n",
       "      <td>0.197406</td>\n",
       "      <td>0.874854</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p1</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.581562</td>\n",
       "      <td>0.211810</td>\n",
       "      <td>0.968561</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p2</th>\n",
       "      <td>2</td>\n",
       "      <td>-0.664063</td>\n",
       "      <td>0.200366</td>\n",
       "      <td>1.025431</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p3</th>\n",
       "      <td>3</td>\n",
       "      <td>-0.661621</td>\n",
       "      <td>0.183795</td>\n",
       "      <td>0.721990</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p4</th>\n",
       "      <td>4</td>\n",
       "      <td>-0.554206</td>\n",
       "      <td>0.293048</td>\n",
       "      <td>0.857730</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p5</th>\n",
       "      <td>5</td>\n",
       "      <td>-0.593452</td>\n",
       "      <td>0.240670</td>\n",
       "      <td>0.826638</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p6</th>\n",
       "      <td>6</td>\n",
       "      <td>-0.636524</td>\n",
       "      <td>0.264110</td>\n",
       "      <td>0.809327</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p7</th>\n",
       "      <td>7</td>\n",
       "      <td>-0.637288</td>\n",
       "      <td>0.219459</td>\n",
       "      <td>0.894342</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p8</th>\n",
       "      <td>8</td>\n",
       "      <td>-0.657068</td>\n",
       "      <td>0.185566</td>\n",
       "      <td>1.327299</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p9</th>\n",
       "      <td>9</td>\n",
       "      <td>-0.626770</td>\n",
       "      <td>0.241434</td>\n",
       "      <td>1.025155</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      tid      loss      time  log_loss status\n",
       "name                                          \n",
       "p0      0 -0.574195  0.197406  0.874854     ok\n",
       "p1      1 -0.581562  0.211810  0.968561     ok\n",
       "p2      2 -0.664063  0.200366  1.025431     ok\n",
       "p3      3 -0.661621  0.183795  0.721990     ok\n",
       "p4      4 -0.554206  0.293048  0.857730     ok\n",
       "p5      5 -0.593452  0.240670  0.826638     ok\n",
       "p6      6 -0.636524  0.264110  0.809327     ok\n",
       "p7      7 -0.637288  0.219459  0.894342     ok\n",
       "p8      8 -0.657068  0.185566  1.327299     ok\n",
       "p9      9 -0.626770  0.241434  1.025155     ok"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline_with_correction =  SMOTEENN(operator = MinMaxScaler() >> PCA() >> RandomForestClassifier())\n",
    "optimizer = Hyperopt(estimator=pipeline_with_correction, max_evals = 10, scoring='roc_auc')\n",
    "trained_optimizer = optimizer.fit(np.array(train_X), np.array(train_y))\n",
    "predictions = trained_optimizer.predict(test_X)\n",
    "trained_optimizer.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35789473684210527\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: cluster:(root) Pages: 1 -->\n",
       "<svg width=\"306pt\" height=\"130pt\"\n",
       " viewBox=\"0.00 0.00 306.08 130.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 126)\">\n",
       "<title>cluster:(root)</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-126 302.083,-126 302.083,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\">\n",
       "<title>cluster:(root)</title>\n",
       "<g id=\"a_clust1\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.imblearn.smoteenn.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"(root) = SMOTEENN(operator=pipeline)\">\n",
       "<polygon fill=\"#ffffff\" stroke=\"#000000\" points=\"8,-8 8,-114 290.083,-114 290.083,-8 8,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"149.0415\" y=\"-98.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">SMOTEENN</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\">\n",
       "<title>cluster:pipeline</title>\n",
       "<g id=\"a_clust2\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.imblearn.smoteenn.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"pipeline = ...\">\n",
       "<path fill=\"#b0e2ff\" stroke=\"#000000\" d=\"M28,-16C28,-16 270.083,-16 270.083,-16 276.083,-16 282.083,-22 282.083,-28 282.083,-28 282.083,-72 282.083,-72 282.083,-78 276.083,-84 270.083,-84 270.083,-84 28,-84 28,-84 22,-84 16,-78 16,-72 16,-72 16,-28 16,-28 16,-22 22,-16 28,-16\"/>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- min_max_scaler -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>min_max_scaler</title>\n",
       "<g id=\"a_node1\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.sklearn.min_max_scaler.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"min_max_scaler = MinMaxScaler()\">\n",
       "<ellipse fill=\"#b0e2ff\" stroke=\"#000000\" cx=\"51\" cy=\"-50\" rx=\"27\" ry=\"26.3269\"/>\n",
       "<text text-anchor=\"middle\" x=\"51\" y=\"-57.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Min&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"51\" y=\"-46.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Max&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"51\" y=\"-35.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Scaler</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- pca -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>pca</title>\n",
       "<g id=\"a_node2\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.sklearn.pca.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"pca = PCA(n_components=0.34508967502645044, svd_solver=&#39;full&#39;)\">\n",
       "<ellipse fill=\"#b0e2ff\" stroke=\"#000000\" cx=\"141\" cy=\"-50\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"141\" y=\"-46.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">PCA</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- min_max_scaler&#45;&gt;pca -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>min_max_scaler&#45;&gt;pca</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M78.003,-50C86.0277,-50 94.9665,-50 103.5309,-50\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"103.7051,-53.5001 113.705,-50 103.705,-46.5001 103.7051,-53.5001\"/>\n",
       "</g>\n",
       "<!-- random_forest_classifier -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>random_forest_classifier</title>\n",
       "<g id=\"a_node3\"><a xlink:href=\"https://lale.readthedocs.io/en/latest/modules/lale.lib.sklearn.random_forest_classifier.html\" target=\"_blank\" rel=\"noopener noreferrer\" xlink:title=\"random_forest_classifier = RandomForestClassifier(criterion=&#39;entropy&#39;, max_features=None, min_samples_leaf=0.30443331216349984, min_samples_split=0.3224970555951517, n_estimators=58)\">\n",
       "<ellipse fill=\"#b0e2ff\" stroke=\"#000000\" cx=\"239.0415\" cy=\"-50\" rx=\"35.083\" ry=\"26.3269\"/>\n",
       "<text text-anchor=\"middle\" x=\"239.0415\" y=\"-57.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Random&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"239.0415\" y=\"-46.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Forest&#45;</text>\n",
       "<text text-anchor=\"middle\" x=\"239.0415\" y=\"-35.7\" font-family=\"Times,serif\" font-size=\"11.00\" fill=\"#000000\">Classifier</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- pca&#45;&gt;random_forest_classifier -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>pca&#45;&gt;random_forest_classifier</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M168.0498,-50C176.0345,-50 184.9997,-50 193.8014,-50\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"193.9902,-53.5001 203.9902,-50 193.9902,-46.5001 193.9902,-53.5001\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x13fc21e80>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(accuracy_score(test_y, predictions))\n",
    "best_estimator = trained_optimizer.get_pipeline()\n",
    "best_estimator.visualize()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}