model_tuning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Author: Kyle Nickerson"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Intro  \n",
    "This notebook has code to try and fit a variety of ml classifiers to pmlb datasets. You can change which classifiers are used, and the criteria for which datasets are included"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from pmlb import fetch_data, classification_dataset_names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Overview of datasets in pmlb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df = pd.read_csv(\"data_df.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>n_instances</th>\n",
       "      <th>n_features</th>\n",
       "      <th>n_binary_features</th>\n",
       "      <th>n_categorical_features</th>\n",
       "      <th>n_continuous_features</th>\n",
       "      <th>endpoint_type</th>\n",
       "      <th>n_classes</th>\n",
       "      <th>imbalance</th>\n",
       "      <th>task</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1027_ESL</td>\n",
       "      <td>488</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>continuous</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.099363</td>\n",
       "      <td>regression</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1028_SWD</td>\n",
       "      <td>1000</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>continuous</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.108291</td>\n",
       "      <td>regression</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1029_LEV</td>\n",
       "      <td>1000</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>continuous</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.111245</td>\n",
       "      <td>regression</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1030_ERA</td>\n",
       "      <td>1000</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>continuous</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.031251</td>\n",
       "      <td>regression</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1089_USCrime</td>\n",
       "      <td>47</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>continuous</td>\n",
       "      <td>42.0</td>\n",
       "      <td>0.002970</td>\n",
       "      <td>regression</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>279</th>\n",
       "      <td>wine_quality_red</td>\n",
       "      <td>1599</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>categorical</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.228804</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>280</th>\n",
       "      <td>wine_quality_white</td>\n",
       "      <td>4898</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>categorical</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.211974</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>wine_recognition</td>\n",
       "      <td>178</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>categorical</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.012530</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282</th>\n",
       "      <td>xd6</td>\n",
       "      <td>973</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>categorical</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.114332</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>283</th>\n",
       "      <td>yeast</td>\n",
       "      <td>1479</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>categorical</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.127818</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>284 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                dataset  n_instances  n_features  n_binary_features  \\\n",
       "0              1027_ESL          488           4                  0   \n",
       "1              1028_SWD         1000          10                  0   \n",
       "2              1029_LEV         1000           4                  0   \n",
       "3              1030_ERA         1000           4                  0   \n",
       "4          1089_USCrime           47          13                  0   \n",
       "..                  ...          ...         ...                ...   \n",
       "279    wine_quality_red         1599          11                  0   \n",
       "280  wine_quality_white         4898          11                  0   \n",
       "281    wine_recognition          178          13                  0   \n",
       "282                 xd6          973           9                  9   \n",
       "283               yeast         1479           8                  0   \n",
       "\n",
       "     n_categorical_features  n_continuous_features endpoint_type  n_classes  \\\n",
       "0                         0                      4    continuous        9.0   \n",
       "1                         0                     10    continuous        4.0   \n",
       "2                         0                      4    continuous        5.0   \n",
       "3                         0                      4    continuous        9.0   \n",
       "4                         0                     13    continuous       42.0   \n",
       "..                      ...                    ...           ...        ...   \n",
       "279                       0                     11   categorical        6.0   \n",
       "280                       0                     11   categorical        7.0   \n",
       "281                       2                     11   categorical        3.0   \n",
       "282                       0                      0   categorical        2.0   \n",
       "283                       0                      8   categorical        9.0   \n",
       "\n",
       "     imbalance            task  \n",
       "0     0.099363      regression  \n",
       "1     0.108291      regression  \n",
       "2     0.111245      regression  \n",
       "3     0.031251      regression  \n",
       "4     0.002970      regression  \n",
       "..         ...             ...  \n",
       "279   0.228804  classification  \n",
       "280   0.211974  classification  \n",
       "281   0.012530  classification  \n",
       "282   0.114332  classification  \n",
       "283   0.127818  classification  \n",
       "\n",
       "[284 rows x 10 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set classifiers & parameters to try"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(RandomForestClassifier(),\n",
       "  {'n_estimators': [100, 500],\n",
       "   'min_samples_split': [2, 4, 8],\n",
       "   'max_features': ['auto', 'sqrt', 'log2']}),\n",
       " (KNeighborsClassifier(),\n",
       "  {'n_neighbors': [3, 5, 9, 15], 'weights': ['uniform', 'distance']}),\n",
       " (LogisticRegression(max_iter=1000),\n",
       "  {'penalty': ['l1', 'l2', 'elasticnet', 'none'],\n",
       "   'C': [0.5, 0.75, 1.0, 1.5, 2.0]}),\n",
       " (GaussianNB(), {'var_smoothing': [1e-05, 1e-09, 1e-13]}),\n",
       " (SVC(cache_size=800, max_iter=10000),\n",
       "  {'C': [0.5, 0.75, 1.0, 1.5, 2.0],\n",
       "   'kernel': ['linear', 'poly', 'rbf', 'sigmoid']})]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_clfs = [(RandomForestClassifier(), {\"n_estimators\": [100, 500], \"min_samples_split\": [2, 4, 8], \"max_features\": [\"auto\", \"sqrt\", \"log2\"]}),\n",
    "# (GradientBoostingClassifier(), {\"learning_rate\": [0.05, 0.1, 0.2], \"min_samples_split\": [2, 4, 8], \"max_features\": [\"auto\", \"sqrt\", \"log2\"]}),\n",
    " (KNeighborsClassifier(), {\"n_neighbors\": [3, 5, 9, 15], \"weights\": [\"uniform\", \"distance\"]}),\n",
    "  (LogisticRegression(max_iter=1000), { \"penalty\": [\"l1\", \"l2\", \"elasticnet\", \"none\"], \"C\": [0.5, 0.75, 1.0, 1.5, 2.0]}),\n",
    " (GaussianNB(), { \"var_smoothing\": [1e-5, 1e-9, 1e-13]}),\n",
    " (SVC(cache_size=800, max_iter=10_000), {\"C\": [0.5, 0.75, 1.0, 1.5, 2.0], \"kernel\": [\"linear\", \"poly\", \"rbf\", \"sigmoid\"]})\n",
    "]\n",
    "all_clfs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set up dataset filters\n",
    "Settings here can be used to select a subset of the datasets which meet specific criteria"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_samples = 100\n",
    "max_samples= 5_000\n",
    "\n",
    "min_features = 3\n",
    "max_features = 1000\n",
    "\n",
    "bin_feats_ok = True  # use datasets with binary features ?\n",
    "cat_feats_ok = True  # use datasets with categorical features ?\n",
    "cont_feats_ok = True # use datasets with continous features ?\n",
    "\n",
    "use_reg = False  # use datasets with regression problems?\n",
    "use_clas = True  # use datasets with classification problems?\n",
    "\n",
    "min_classes = 2 # number of classes in dataset (for classification)\n",
    "max_classes = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "69 selected out of 284\n"
     ]
    }
   ],
   "source": [
    "conds = data_df[\"n_instances\"] >= min_samples\n",
    "conds = conds & (data_df[\"n_instances\"] <= max_samples)\n",
    "\n",
    "conds = conds & (data_df[\"n_features\"] >= min_features)\n",
    "conds = conds & (data_df[\"n_features\"] <= max_features)\n",
    "\n",
    "feat_type_info = [\n",
    "    (\"n_binary_features\", bin_feats_ok),\n",
    "    (\"n_categorical_features\", cat_feats_ok),\n",
    "    (\"n_continuous_features\", cont_feats_ok),\n",
    "]\n",
    "\n",
    "for col_name, is_ok in feat_type_info:\n",
    "    if not is_ok:\n",
    "        conds = conds & (data_df[col_name] == 0)\n",
    "\n",
    "task_info = [\n",
    "    (\"regression\", use_reg),\n",
    "    (\"classification\", use_clas),    \n",
    "]\n",
    "\n",
    "for task, use in task_info:\n",
    "    if not use:\n",
    "        conds = conds & (data_df[\"task\"] != task)    \n",
    "        \n",
    "        \n",
    "conds = conds & (data_df[\"n_classes\"] >= min_classes)\n",
    "conds = conds & (data_df[\"n_classes\"] <= max_classes)  \n",
    "    \n",
    "print(f\"{sum(conds)} selected out of {len(conds)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>n_instances</th>\n",
       "      <th>n_features</th>\n",
       "      <th>n_binary_features</th>\n",
       "      <th>n_categorical_features</th>\n",
       "      <th>n_continuous_features</th>\n",
       "      <th>endpoint_type</th>\n",
       "      <th>n_classes</th>\n",
       "      <th>imbalance</th>\n",
       "      <th>task</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>GAMETES_Epistasis_2_Way_1000atts_0.4H_EDM_1_ED...</td>\n",
       "      <td>1600</td>\n",
       "      <td>1000</td>\n",
       "      <td>31</td>\n",
       "      <td>969</td>\n",
       "      <td>0</td>\n",
       "      <td>categorical</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>GAMETES_Epistasis_2_Way_20atts_0.1H_EDM_1_1</td>\n",
       "      <td>1600</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>categorical</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>GAMETES_Epistasis_2_Way_20atts_0.4H_EDM_1_1</td>\n",
       "      <td>1600</td>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>categorical</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>GAMETES_Epistasis_3_Way_20atts_0.2H_EDM_1_1</td>\n",
       "      <td>1600</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>categorical</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>GAMETES_Heterogeneity_20atts_1600_Het_0.4_0.2_...</td>\n",
       "      <td>1600</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>categorical</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269</th>\n",
       "      <td>tic_tac_toe</td>\n",
       "      <td>958</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>categorical</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.094181</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>271</th>\n",
       "      <td>tokyo1</td>\n",
       "      <td>959</td>\n",
       "      <td>44</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>44</td>\n",
       "      <td>categorical</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.077515</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>274</th>\n",
       "      <td>vote</td>\n",
       "      <td>435</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>categorical</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.051795</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>278</th>\n",
       "      <td>wdbc</td>\n",
       "      <td>569</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>categorical</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.064940</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282</th>\n",
       "      <td>xd6</td>\n",
       "      <td>973</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>categorical</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.114332</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>69 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               dataset  n_instances  \\\n",
       "120  GAMETES_Epistasis_2_Way_1000atts_0.4H_EDM_1_ED...         1600   \n",
       "121        GAMETES_Epistasis_2_Way_20atts_0.1H_EDM_1_1         1600   \n",
       "122        GAMETES_Epistasis_2_Way_20atts_0.4H_EDM_1_1         1600   \n",
       "123        GAMETES_Epistasis_3_Way_20atts_0.2H_EDM_1_1         1600   \n",
       "124  GAMETES_Heterogeneity_20atts_1600_Het_0.4_0.2_...         1600   \n",
       "..                                                 ...          ...   \n",
       "269                                        tic_tac_toe          958   \n",
       "271                                             tokyo1          959   \n",
       "274                                               vote          435   \n",
       "278                                               wdbc          569   \n",
       "282                                                xd6          973   \n",
       "\n",
       "     n_features  n_binary_features  n_categorical_features  \\\n",
       "120        1000                 31                     969   \n",
       "121          20                  0                      20   \n",
       "122          20                  2                      18   \n",
       "123          20                  0                      20   \n",
       "124          20                  1                      19   \n",
       "..          ...                ...                     ...   \n",
       "269           9                  0                       9   \n",
       "271          44                  0                       0   \n",
       "274          16                  0                      16   \n",
       "278          30                  0                       0   \n",
       "282           9                  9                       0   \n",
       "\n",
       "     n_continuous_features endpoint_type  n_classes  imbalance            task  \n",
       "120                      0   categorical        2.0   0.000000  classification  \n",
       "121                      0   categorical        2.0   0.000000  classification  \n",
       "122                      0   categorical        2.0   0.000000  classification  \n",
       "123                      0   categorical        2.0   0.000000  classification  \n",
       "124                      0   categorical        2.0   0.000000  classification  \n",
       "..                     ...           ...        ...        ...             ...  \n",
       "269                      0   categorical        2.0   0.094181  classification  \n",
       "271                     44   categorical        2.0   0.077515  classification  \n",
       "274                      0   categorical        2.0   0.051795  classification  \n",
       "278                     30   categorical        2.0   0.064940  classification  \n",
       "282                      0   categorical        2.0   0.114332  classification  \n",
       "\n",
       "[69 rows x 10 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = data_df[conds]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'breast': ('Breast', 0),\n",
       " 'GAMETES_Epistasis_2_Way_1000atts_0.4H_EDM_1_EDM_1_1': ('GAMETES_Epistasis',\n",
       "  4),\n",
       " 'Hill_Valley_without_noise': ('HV_without_noise', 2),\n",
       " 'Hill_Valley_with_noise': ('HV_with_noise', 3),\n",
       " 'monk2': ('Monk2', 1),\n",
       " 'parity5+5': ('Parity5+5', 5)}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ds_name: (short_name, index)\n",
    "ds_order = { 'breast': (\"Breast\", 0),\n",
    "             'GAMETES_Epistasis_2_Way_1000atts_0.4H_EDM_1_EDM_1_1': (\"GAMETES_Epistasis\", 4),\n",
    "             'Hill_Valley_without_noise': (\"HV_without_noise\", 2),\n",
    "             'Hill_Valley_with_noise': (\"HV_with_noise\", 3),\n",
    "             'monk2': (\"Monk2\", 1),\n",
    "             'parity5+5': (\"Parity5+5\", 5), \n",
    "           }\n",
    "\n",
    "ds_order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ds_order)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['breast',\n",
       " 'monk2',\n",
       " 'Hill_Valley_without_noise',\n",
       " 'Hill_Valley_with_noise',\n",
       " 'GAMETES_Epistasis_2_Way_1000atts_0.4H_EDM_1_EDM_1_1',\n",
       " 'parity5+5']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_ordered_names(dso, full=False):\n",
    "    x = [0]*len(dso)\n",
    "    for fname, (sname, i) in dso.items():\n",
    "        if full:\n",
    "            x[i] = fname\n",
    "        else:\n",
    "            x[i] = sname\n",
    "            \n",
    "    return x\n",
    "\n",
    "datasets = get_ordered_names(ds_order, full=True)\n",
    "datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Try fitting a variety of classifiers to the filtered datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running with dataset = GAMETES_Epistasis_2_Way_1000atts_0.4H_EDM_1_EDM_1_1 (0 of 69)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>n_instances</th>\n",
       "      <th>n_features</th>\n",
       "      <th>n_binary_features</th>\n",
       "      <th>n_categorical_features</th>\n",
       "      <th>n_continuous_features</th>\n",
       "      <th>endpoint_type</th>\n",
       "      <th>n_classes</th>\n",
       "      <th>imbalance</th>\n",
       "      <th>task</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>GAMETES_Epistasis_2_Way_1000atts_0.4H_EDM_1_ED...</td>\n",
       "      <td>1600</td>\n",
       "      <td>1000</td>\n",
       "      <td>31</td>\n",
       "      <td>969</td>\n",
       "      <td>0</td>\n",
       "      <td>categorical</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               dataset  n_instances  \\\n",
       "120  GAMETES_Epistasis_2_Way_1000atts_0.4H_EDM_1_ED...         1600   \n",
       "\n",
       "     n_features  n_binary_features  n_categorical_features  \\\n",
       "120        1000                 31                     969   \n",
       "\n",
       "     n_continuous_features endpoint_type  n_classes  imbalance            task  \n",
       "120                      0   categorical        2.0        0.0  classification  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Fitting RandomForestClassifier\tTook: 32.92 secs\tTraining acc = 1.00 \tCV acc = 0.50 \tTest acc = 0.53\n",
      "\n",
      "\n",
      "Fitting KNeighborsClassifier\tTook: 5.11 secs\tTraining acc = 0.69 \tCV acc = 0.49 \tTest acc = 0.50\n",
      "\n",
      "\n",
      "Fitting LogisticRegression(max_iter=1000\tTook: 4.47 secs\tTraining acc = 1.00 \tCV acc = nan \tTest acc = 0.49\n",
      "\n",
      "\n",
      "Fitting GaussianNB\tTook: 0.19 secs\tTraining acc = 0.75 \tCV acc = 0.51 \tTest acc = 0.48\n",
      "\n",
      "\n",
      "Fitting SVC(cache_size=800, max_iter=10000\tTook: 34.65 secs\tTraining acc = 1.00 \tCV acc = 0.51 \tTest acc = 0.47\n",
      "\n",
      "\n",
      "Running with dataset = Hill_Valley_with_noise (6 of 69)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>n_instances</th>\n",
       "      <th>n_features</th>\n",
       "      <th>n_binary_features</th>\n",
       "      <th>n_categorical_features</th>\n",
       "      <th>n_continuous_features</th>\n",
       "      <th>endpoint_type</th>\n",
       "      <th>n_classes</th>\n",
       "      <th>imbalance</th>\n",
       "      <th>task</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>Hill_Valley_with_noise</td>\n",
       "      <td>1212</td>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>100</td>\n",
       "      <td>categorical</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    dataset  n_instances  n_features  n_binary_features  \\\n",
       "126  Hill_Valley_with_noise         1212         100                  0   \n",
       "\n",
       "     n_categorical_features  n_continuous_features endpoint_type  n_classes  \\\n",
       "126                       0                    100   categorical        2.0   \n",
       "\n",
       "     imbalance            task  \n",
       "126        0.0  classification  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Fitting RandomForestClassifier\tTook: 36.44 secs\tTraining acc = 1.00 \tCV acc = 0.53 \tTest acc = 0.59\n",
      "\n",
      "\n",
      "Fitting KNeighborsClassifier\tTook: 0.20 secs\tTraining acc = 1.00 \tCV acc = 0.50 \tTest acc = 0.49\n",
      "\n",
      "\n",
      "Fitting LogisticRegression(max_iter=1000\tTook: 13.05 secs\tTraining acc = 0.99 \tCV acc = nan \tTest acc = 0.96\n",
      "\n",
      "\n",
      "Fitting GaussianNB\tTook: 0.02 secs\tTraining acc = 0.50 \tCV acc = 0.50 \tTest acc = 0.48\n",
      "\n",
      "\n",
      "Fitting SVC(cache_size=800, max_iter=10000\tTook: 1.44 secs\tTraining acc = 0.85 \tCV acc = 0.56 \tTest acc = 0.88\n",
      "\n",
      "\n",
      "Running with dataset = Hill_Valley_without_noise (7 of 69)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>n_instances</th>\n",
       "      <th>n_features</th>\n",
       "      <th>n_binary_features</th>\n",
       "      <th>n_categorical_features</th>\n",
       "      <th>n_continuous_features</th>\n",
       "      <th>endpoint_type</th>\n",
       "      <th>n_classes</th>\n",
       "      <th>imbalance</th>\n",
       "      <th>task</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>Hill_Valley_without_noise</td>\n",
       "      <td>1212</td>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>100</td>\n",
       "      <td>categorical</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.000098</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       dataset  n_instances  n_features  n_binary_features  \\\n",
       "127  Hill_Valley_without_noise         1212         100                  0   \n",
       "\n",
       "     n_categorical_features  n_continuous_features endpoint_type  n_classes  \\\n",
       "127                       0                    100   categorical        2.0   \n",
       "\n",
       "     imbalance            task  \n",
       "127   0.000098  classification  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Fitting RandomForestClassifier\tTook: 38.15 secs\tTraining acc = 1.00 \tCV acc = 0.57 \tTest acc = 0.57\n",
      "\n",
      "\n",
      "Fitting KNeighborsClassifier\tTook: 0.20 secs\tTraining acc = 1.00 \tCV acc = 0.56 \tTest acc = 0.57\n",
      "\n",
      "\n",
      "Fitting LogisticRegression(max_iter=1000\tTook: 1.55 secs\tTraining acc = 1.00 \tCV acc = nan \tTest acc = 1.00\n",
      "\n",
      "\n",
      "Fitting GaussianNB\tTook: 0.01 secs\tTraining acc = 0.51 \tCV acc = 0.51 \tTest acc = 0.50\n",
      "\n",
      "\n",
      "Fitting SVC(cache_size=800, max_iter=10000\tTook: 1.39 secs\tTraining acc = 0.83 \tCV acc = 0.59 \tTest acc = 0.84\n",
      "\n",
      "\n",
      "Running with dataset = breast (16 of 69)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>n_instances</th>\n",
       "      <th>n_features</th>\n",
       "      <th>n_binary_features</th>\n",
       "      <th>n_categorical_features</th>\n",
       "      <th>n_continuous_features</th>\n",
       "      <th>endpoint_type</th>\n",
       "      <th>n_classes</th>\n",
       "      <th>imbalance</th>\n",
       "      <th>task</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <td>breast</td>\n",
       "      <td>699</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>categorical</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.096375</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    dataset  n_instances  n_features  n_binary_features  \\\n",
       "157  breast          699          10                  0   \n",
       "\n",
       "     n_categorical_features  n_continuous_features endpoint_type  n_classes  \\\n",
       "157                       1                      9   categorical        2.0   \n",
       "\n",
       "     imbalance            task  \n",
       "157   0.096375  classification  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Fitting RandomForestClassifier\tTook: 11.04 secs\tTraining acc = 0.99 \tCV acc = 0.96 \tTest acc = 0.99\n",
      "\n",
      "\n",
      "Fitting KNeighborsClassifier\tTook: 0.05 secs\tTraining acc = 1.00 \tCV acc = 0.57 \tTest acc = 0.52\n",
      "\n",
      "\n",
      "Fitting LogisticRegression(max_iter=1000\tTook: 0.06 secs\tTraining acc = 0.50 \tCV acc = nan \tTest acc = 0.50\n",
      "\n",
      "\n",
      "Fitting GaussianNB\tTook: 0.01 secs\tTraining acc = 0.96 \tCV acc = 0.70 \tTest acc = 0.96\n",
      "\n",
      "\n",
      "Fitting SVC(cache_size=800, max_iter=10000\tTook: 0.30 secs\tTraining acc = 0.50 \tCV acc = 0.50 \tTest acc = 0.50\n",
      "\n",
      "\n",
      "Running with dataset = monk2 (51 of 69)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>n_instances</th>\n",
       "      <th>n_features</th>\n",
       "      <th>n_binary_features</th>\n",
       "      <th>n_categorical_features</th>\n",
       "      <th>n_continuous_features</th>\n",
       "      <th>endpoint_type</th>\n",
       "      <th>n_classes</th>\n",
       "      <th>imbalance</th>\n",
       "      <th>task</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>230</th>\n",
       "      <td>monk2</td>\n",
       "      <td>601</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>categorical</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.098895</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    dataset  n_instances  n_features  n_binary_features  \\\n",
       "230   monk2          601           6                  2   \n",
       "\n",
       "     n_categorical_features  n_continuous_features endpoint_type  n_classes  \\\n",
       "230                       4                      0   categorical        2.0   \n",
       "\n",
       "     imbalance            task  \n",
       "230   0.098895  classification  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Fitting RandomForestClassifier\tTook: 11.27 secs\tTraining acc = 1.00 \tCV acc = 0.77 \tTest acc = 0.84\n",
      "\n",
      "\n",
      "Fitting KNeighborsClassifier\tTook: 0.04 secs\tTraining acc = 1.00 \tCV acc = 0.61 \tTest acc = 0.72\n",
      "\n",
      "\n",
      "Fitting LogisticRegression(max_iter=1000\tTook: 0.11 secs\tTraining acc = 0.50 \tCV acc = nan \tTest acc = 0.50\n",
      "\n",
      "\n",
      "Fitting GaussianNB\tTook: 0.01 secs\tTraining acc = 0.50 \tCV acc = 0.50 \tTest acc = 0.50\n",
      "\n",
      "\n",
      "Fitting SVC(cache_size=800, max_iter=10000\tTook: 0.23 secs\tTraining acc = 0.69 \tCV acc = 0.51 \tTest acc = 0.59\n",
      "\n",
      "\n",
      "Running with dataset = parity5+5 (54 of 69)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>n_instances</th>\n",
       "      <th>n_features</th>\n",
       "      <th>n_binary_features</th>\n",
       "      <th>n_categorical_features</th>\n",
       "      <th>n_continuous_features</th>\n",
       "      <th>endpoint_type</th>\n",
       "      <th>n_classes</th>\n",
       "      <th>imbalance</th>\n",
       "      <th>task</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>parity5+5</td>\n",
       "      <td>1124</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>categorical</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.000079</td>\n",
       "      <td>classification</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       dataset  n_instances  n_features  n_binary_features  \\\n",
       "240  parity5+5         1124          10                 10   \n",
       "\n",
       "     n_categorical_features  n_continuous_features endpoint_type  n_classes  \\\n",
       "240                       0                      0   categorical        2.0   \n",
       "\n",
       "     imbalance            task  \n",
       "240   0.000079  classification  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Fitting RandomForestClassifier\tTook: 12.60 secs\tTraining acc = 1.00 \tCV acc = 0.49 \tTest acc = 0.60\n",
      "\n",
      "\n",
      "Fitting KNeighborsClassifier\tTook: 0.08 secs\tTraining acc = 1.00 \tCV acc = 0.49 \tTest acc = 0.56\n",
      "\n",
      "\n",
      "Fitting LogisticRegression(max_iter=1000\tTook: 0.06 secs\tTraining acc = 0.52 \tCV acc = nan \tTest acc = 0.45\n",
      "\n",
      "\n",
      "Fitting GaussianNB\tTook: 0.01 secs\tTraining acc = 0.52 \tCV acc = 0.43 \tTest acc = 0.45\n",
      "\n",
      "\n",
      "Fitting SVC(cache_size=800, max_iter=10000\tTook: 0.47 secs\tTraining acc = 0.48 \tCV acc = 0.43 \tTest acc = 0.48\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import balanced_accuracy_score, make_scorer\n",
    "\n",
    "use_p = 1   # if less than 1, each dataset is included with this probability (this lets you randomly select subset of the datasets)\n",
    "n_jobs = 5\n",
    "metric = balanced_accuracy_score\n",
    "\n",
    "cv_scores = {}\n",
    "test_scores = {}\n",
    "training_scores = {}\n",
    "ds_names = []\n",
    "\n",
    "def score_grid_clf(clf, x, y):\n",
    "    return metric(y, clf.best_estimator_.predict(x))\n",
    "\n",
    "import warnings\n",
    "\n",
    "with warnings.catch_warnings():\n",
    "    warnings.simplefilter(\"ignore\")\n",
    "\n",
    "    for i, ds in enumerate(df.dataset):\n",
    "        if ds in datasets:   # only run on specific subset of datasets\n",
    "#         if np.random.random() < use_p:\n",
    "\n",
    "            print(\"Running with dataset =\", ds, \"({} of {})\".format(i, len(df.dataset)))\n",
    "            ds_names.append(ds)\n",
    "            display(df[df[\"dataset\"]==ds])\n",
    "            print()\n",
    "            X, y = fetch_data(ds, return_X_y=True)\n",
    "            train_X, test_X, train_y, test_y = train_test_split(X, y)\n",
    "\n",
    "\n",
    "            for clf_info in all_clfs:\n",
    "                clf = GridSearchCV(*clf_info, scoring=make_scorer(metric), n_jobs=n_jobs)\n",
    "                clf_name = str(clf.get_params()['estimator']).strip(\"()\")\n",
    "                print(\"Fitting\", clf_name, end=\"\\t\")\n",
    "                clf.fit(train_X, train_y)     \n",
    "\n",
    "                resdf = pd.DataFrame(clf.cv_results_) \n",
    "\n",
    "                took = (sum(clf.cv_results_[\"mean_fit_time\"])+ sum(clf.cv_results_[\"mean_score_time\"]))*5/n_jobs\n",
    "                print(\"Took: {:.2f} secs\".format(took), end=\"\\t\")\n",
    "                \n",
    "\n",
    "                if clf_name not in cv_scores.keys():\n",
    "                    cv_scores[clf_name] = []\n",
    "                \n",
    "                if clf_name not in test_scores.keys():\n",
    "                    test_scores[clf_name] = []\n",
    "                    \n",
    "                if clf_name not in training_scores.keys():   \n",
    "                    training_scores[clf_name] = []\n",
    "                    \n",
    "                cv_acc = np.mean(clf.cv_results_[\"mean_test_score\"])\n",
    "#                 test_acc = clf.best_estimator_.score(test_X, test_y)\n",
    "#                 tr_acc =  clf.best_estimator_.score(train_X, train_y)\n",
    "                test_acc = score_grid_clf(clf, test_X, test_y)\n",
    "                tr_acc =  score_grid_clf(clf, train_X, train_y)\n",
    "                \n",
    "                                \n",
    "                cv_scores[clf_name].append(cv_acc)\n",
    "                test_scores[clf_name].append(test_acc)\n",
    "                training_scores[clf_name].append(tr_acc)\n",
    "                \n",
    "                \n",
    "                print(\"Training acc = {:.2f}\".format(tr_acc), \"\\tCV acc = {:.2f}\".format(cv_acc), \"\\tTest acc = {:.2f}\".format(test_acc))\n",
    "                \n",
    "                print(\"\\n\")\n",
    "        \n",
    "     "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### View results\n",
    "Note: This might only shows the results for the datasets I was working with, but the results for all should be in 'test_scores'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <th>KNeighborsClassifier</th>\n",
       "      <th>LogisticRegression(max_iter=1000</th>\n",
       "      <th>GaussianNB</th>\n",
       "      <th>SVC(cache_size=800, max_iter=10000</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>breast</th>\n",
       "      <td>0.525303</td>\n",
       "      <td>0.497975</td>\n",
       "      <td>0.490199</td>\n",
       "      <td>0.477823</td>\n",
       "      <td>0.472422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>monk2</th>\n",
       "      <td>0.593164</td>\n",
       "      <td>0.494263</td>\n",
       "      <td>0.963768</td>\n",
       "      <td>0.484011</td>\n",
       "      <td>0.884042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hill_Valley_without_noise</th>\n",
       "      <td>0.570949</td>\n",
       "      <td>0.574414</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.501177</td>\n",
       "      <td>0.839144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hill_Valley_with_noise</th>\n",
       "      <td>0.986609</td>\n",
       "      <td>0.522345</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.964953</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GAMETES_Epistasis_2_Way_1000atts_0.4H_EDM_1_EDM_1_1</th>\n",
       "      <td>0.836786</td>\n",
       "      <td>0.724337</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.594625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>parity5+5</th>\n",
       "      <td>0.596794</td>\n",
       "      <td>0.559593</td>\n",
       "      <td>0.452926</td>\n",
       "      <td>0.452926</td>\n",
       "      <td>0.475242</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    RandomForestClassifier  \\\n",
       "dataset                                                                      \n",
       "breast                                                            0.525303   \n",
       "monk2                                                             0.593164   \n",
       "Hill_Valley_without_noise                                         0.570949   \n",
       "Hill_Valley_with_noise                                            0.986609   \n",
       "GAMETES_Epistasis_2_Way_1000atts_0.4H_EDM_1_EDM...                0.836786   \n",
       "parity5+5                                                         0.596794   \n",
       "\n",
       "                                                    KNeighborsClassifier  \\\n",
       "dataset                                                                    \n",
       "breast                                                          0.497975   \n",
       "monk2                                                           0.494263   \n",
       "Hill_Valley_without_noise                                       0.574414   \n",
       "Hill_Valley_with_noise                                          0.522345   \n",
       "GAMETES_Epistasis_2_Way_1000atts_0.4H_EDM_1_EDM...              0.724337   \n",
       "parity5+5                                                       0.559593   \n",
       "\n",
       "                                                    LogisticRegression(max_iter=1000  \\\n",
       "dataset                                                                                \n",
       "breast                                                                      0.490199   \n",
       "monk2                                                                       0.963768   \n",
       "Hill_Valley_without_noise                                                   1.000000   \n",
       "Hill_Valley_with_noise                                                      0.500000   \n",
       "GAMETES_Epistasis_2_Way_1000atts_0.4H_EDM_1_EDM...                          0.500000   \n",
       "parity5+5                                                                   0.452926   \n",
       "\n",
       "                                                    GaussianNB  \\\n",
       "dataset                                                          \n",
       "breast                                                0.477823   \n",
       "monk2                                                 0.484011   \n",
       "Hill_Valley_without_noise                             0.501177   \n",
       "Hill_Valley_with_noise                                0.964953   \n",
       "GAMETES_Epistasis_2_Way_1000atts_0.4H_EDM_1_EDM...    0.500000   \n",
       "parity5+5                                             0.452926   \n",
       "\n",
       "                                                    SVC(cache_size=800, max_iter=10000  \n",
       "dataset                                                                                 \n",
       "breast                                                                        0.472422  \n",
       "monk2                                                                         0.884042  \n",
       "Hill_Valley_without_noise                                                     0.839144  \n",
       "Hill_Valley_with_noise                                                        0.500000  \n",
       "GAMETES_Epistasis_2_Way_1000atts_0.4H_EDM_1_EDM...                            0.594625  \n",
       "parity5+5                                                                     0.475242  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = pd.DataFrame.from_dict(test_scores, ) # df.dataset\n",
    "result[\"dataset\"] = datasets\n",
    "result.set_index(\"dataset\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}