python/H2O-GBM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import h2o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"4\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU:  72\n",
      "Memory:  svmem(total=1622764929024, available=1422414950400, percent=12.3, used=194895474688, free=556070727680, active=910295629824, inactive=111481761792, buffers=0, cached=871798726656, shared=2571857920, slab=15486603264)\n"
     ]
    }
   ],
   "source": [
    "import multiprocessing\n",
    "print(\"CPU: \",multiprocessing.cpu_count())\n",
    "\n",
    "import psutil\n",
    "print(\"Memory: \",psutil.virtual_memory())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking whether there is an H2O instance running at http://127.0.0.1:54321 . connected.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O cluster uptime:</td>\n",
       "<td>12 mins 05 secs</td></tr>\n",
       "<tr><td>H2O cluster timezone:</td>\n",
       "<td>Europe/Istanbul</td></tr>\n",
       "<tr><td>H2O data parsing timezone:</td>\n",
       "<td>UTC</td></tr>\n",
       "<tr><td>H2O cluster version:</td>\n",
       "<td>3.26.0.3</td></tr>\n",
       "<tr><td>H2O cluster version age:</td>\n",
       "<td>25 days </td></tr>\n",
       "<tr><td>H2O cluster name:</td>\n",
       "<td>H2O_from_python_96273_ugsf64</td></tr>\n",
       "<tr><td>H2O cluster total nodes:</td>\n",
       "<td>1</td></tr>\n",
       "<tr><td>H2O cluster free memory:</td>\n",
       "<td>88.3 Gb</td></tr>\n",
       "<tr><td>H2O cluster total cores:</td>\n",
       "<td>72</td></tr>\n",
       "<tr><td>H2O cluster allowed cores:</td>\n",
       "<td>5</td></tr>\n",
       "<tr><td>H2O cluster status:</td>\n",
       "<td>locked, healthy</td></tr>\n",
       "<tr><td>H2O connection url:</td>\n",
       "<td>http://127.0.0.1:54321</td></tr>\n",
       "<tr><td>H2O connection proxy:</td>\n",
       "<td>None</td></tr>\n",
       "<tr><td>H2O internal security:</td>\n",
       "<td>False</td></tr>\n",
       "<tr><td>H2O API Extensions:</td>\n",
       "<td>Amazon S3, XGBoost, Algos, AutoML, Core V3, Core V4</td></tr>\n",
       "<tr><td>Python version:</td>\n",
       "<td>3.6.3 final</td></tr></table></div>"
      ],
      "text/plain": [
       "--------------------------  ---------------------------------------------------\n",
       "H2O cluster uptime:         12 mins 05 secs\n",
       "H2O cluster timezone:       Europe/Istanbul\n",
       "H2O data parsing timezone:  UTC\n",
       "H2O cluster version:        3.26.0.3\n",
       "H2O cluster version age:    25 days\n",
       "H2O cluster name:           H2O_from_python_96273_ugsf64\n",
       "H2O cluster total nodes:    1\n",
       "H2O cluster free memory:    88.3 Gb\n",
       "H2O cluster total cores:    72\n",
       "H2O cluster allowed cores:  5\n",
       "H2O cluster status:         locked, healthy\n",
       "H2O connection url:         http://127.0.0.1:54321\n",
       "H2O connection proxy:\n",
       "H2O internal security:      False\n",
       "H2O API Extensions:         Amazon S3, XGBoost, Algos, AutoML, Core V3, Core V4\n",
       "Python version:             3.6.3 final\n",
       "--------------------------  ---------------------------------------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#h2o.init()\n",
    "h2o.init(ip=\"127.0.0.1\", max_mem_size_GB = 100, nthreads = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n",
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "hf_positive = h2o.import_file('dataset/train_true_positive_features.csv')\n",
    "hf_negative = h2o.import_file('dataset/train_true_negative_features.csv')\n",
    "hf = hf_positive.rbind(hf_negative)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "positive instances:  (165179, 34)\n",
      "negative instances:  (283367, 34)\n",
      "total instances:  (448546, 34)\n"
     ]
    }
   ],
   "source": [
    "print(\"positive instances: \",hf_positive.shape)\n",
    "print(\"negative instances: \",hf_negative.shape)\n",
    "print(\"total instances: \",hf.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "hf = hf[['vgg_cosine', 'vgg_euclidean_l2'\n",
    "         , 'facenet_cosine', 'facenet_euclidean_l2'\n",
    "         , 'openface_cosine', 'openface_euclidean_l2'\n",
    "         , 'is_related']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th style=\"text-align: right;\">  vgg_cosine</th><th style=\"text-align: right;\">  vgg_euclidean_l2</th><th style=\"text-align: right;\">  facenet_cosine</th><th style=\"text-align: right;\">  facenet_euclidean_l2</th><th style=\"text-align: right;\">  openface_cosine</th><th style=\"text-align: right;\">  openface_euclidean_l2</th><th style=\"text-align: right;\">  is_related</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\">    0.618396</td><td style=\"text-align: right;\">           1.11211</td><td style=\"text-align: right;\">         1.25131</td><td style=\"text-align: right;\">               1.58197</td><td style=\"text-align: right;\">         1.12544 </td><td style=\"text-align: right;\">                1.50029</td><td style=\"text-align: right;\">           1</td></tr>\n",
       "<tr><td style=\"text-align: right;\">    0.601191</td><td style=\"text-align: right;\">           1.09653</td><td style=\"text-align: right;\">         1.14205</td><td style=\"text-align: right;\">               1.51133</td><td style=\"text-align: right;\">         1.08315 </td><td style=\"text-align: right;\">                1.47183</td><td style=\"text-align: right;\">           1</td></tr>\n",
       "<tr><td style=\"text-align: right;\">    0.543063</td><td style=\"text-align: right;\">           1.04217</td><td style=\"text-align: right;\">         1.10449</td><td style=\"text-align: right;\">               1.48627</td><td style=\"text-align: right;\">         1.14981 </td><td style=\"text-align: right;\">                1.51645</td><td style=\"text-align: right;\">           1</td></tr>\n",
       "<tr><td style=\"text-align: right;\">    0.618544</td><td style=\"text-align: right;\">           1.11224</td><td style=\"text-align: right;\">         1.24833</td><td style=\"text-align: right;\">               1.58008</td><td style=\"text-align: right;\">         1.09367 </td><td style=\"text-align: right;\">                1.47897</td><td style=\"text-align: right;\">           1</td></tr>\n",
       "<tr><td style=\"text-align: right;\">    0.60665 </td><td style=\"text-align: right;\">           1.1015 </td><td style=\"text-align: right;\">         1.15115</td><td style=\"text-align: right;\">               1.51733</td><td style=\"text-align: right;\">         1.11618 </td><td style=\"text-align: right;\">                1.49411</td><td style=\"text-align: right;\">           1</td></tr>\n",
       "<tr><td style=\"text-align: right;\">    0.630702</td><td style=\"text-align: right;\">           1.12312</td><td style=\"text-align: right;\">         1.22153</td><td style=\"text-align: right;\">               1.56303</td><td style=\"text-align: right;\">         1.20384 </td><td style=\"text-align: right;\">                1.55167</td><td style=\"text-align: right;\">           1</td></tr>\n",
       "<tr><td style=\"text-align: right;\">    0.742856</td><td style=\"text-align: right;\">           1.2189 </td><td style=\"text-align: right;\">         1.09549</td><td style=\"text-align: right;\">               1.48019</td><td style=\"text-align: right;\">         1.07032 </td><td style=\"text-align: right;\">                1.4631 </td><td style=\"text-align: right;\">           1</td></tr>\n",
       "<tr><td style=\"text-align: right;\">    0.558462</td><td style=\"text-align: right;\">           1.05685</td><td style=\"text-align: right;\">         1.11984</td><td style=\"text-align: right;\">               1.49655</td><td style=\"text-align: right;\">         1.24634 </td><td style=\"text-align: right;\">                1.57882</td><td style=\"text-align: right;\">           1</td></tr>\n",
       "<tr><td style=\"text-align: right;\">    0.648851</td><td style=\"text-align: right;\">           1.13917</td><td style=\"text-align: right;\">         1.10773</td><td style=\"text-align: right;\">               1.48844</td><td style=\"text-align: right;\">         0.612283</td><td style=\"text-align: right;\">                1.1066 </td><td style=\"text-align: right;\">           1</td></tr>\n",
       "<tr><td style=\"text-align: right;\">    0.677681</td><td style=\"text-align: right;\">           1.1642 </td><td style=\"text-align: right;\">         1.03272</td><td style=\"text-align: right;\">               1.43716</td><td style=\"text-align: right;\">         0.571711</td><td style=\"text-align: right;\">                1.06931</td><td style=\"text-align: right;\">           1</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#convert target label to factor because this is a binary classification\n",
    "#otherwise, there would be a regression problem\n",
    "hf['is_related'] = hf['is_related'].asfactor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#70% train, 15% test, 15% validation\n",
    "train, test, validation = hf.split_frame(ratios=[0.70, 0.15], seed=17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train set size:  (314122, 7)\n",
      "test set size:  (67160, 7)\n",
      "validation set size:  (67264, 7)\n"
     ]
    }
   ],
   "source": [
    "print(\"train set size: \",train.shape)\n",
    "print(\"test set size: \", test.shape)\n",
    "print(\"validation set size: \", validation.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from h2o.estimators.gbm import H2OGradientBoostingEstimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = H2OGradientBoostingEstimator(\n",
    "    ntrees = 1000\n",
    "    , learn_rate = 0.01\n",
    "    , stopping_rounds = 50\n",
    "    , stopping_metric = \"AUC\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gbm Model Build progress: |███████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "model.train(x = hf.names[0:-1], y = hf.names[-1]\n",
    "    , training_frame = train\n",
    "    , validation_frame = test\n",
    "    #, verbose = True\n",
    "    , model_id = \"GBM_Kinship\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Details\n",
      "=============\n",
      "H2OGradientBoostingEstimator :  Gradient Boosting Machine\n",
      "Model Key:  GBM_Kinship\n",
      "\n",
      "\n",
      "Model Summary: "
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>number_of_trees</th>\n",
       "      <th>number_of_internal_trees</th>\n",
       "      <th>model_size_in_bytes</th>\n",
       "      <th>min_depth</th>\n",
       "      <th>max_depth</th>\n",
       "      <th>mean_depth</th>\n",
       "      <th>min_leaves</th>\n",
       "      <th>max_leaves</th>\n",
       "      <th>mean_leaves</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>268761.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.695</td>\n",
       "      <td>1.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>16.503</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     number_of_trees  number_of_internal_trees  model_size_in_bytes  \\\n",
       "0             1000.0                    1000.0             268761.0   \n",
       "\n",
       "   min_depth  max_depth  mean_depth  min_leaves  max_leaves  mean_leaves  \n",
       "0        0.0        5.0       2.695         1.0        32.0       16.503  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "ModelMetricsBinomial: gbm\n",
      "** Reported on train data. **\n",
      "\n",
      "MSE: 0.19740943929831983\n",
      "RMSE: 0.44430782043344663\n",
      "LogLoss: 0.5789539838842188\n",
      "Mean Per-Class Error: 0.3333046491681597\n",
      "AUC: 0.7276208150873205\n",
      "pr_auc: 0.6123781904410922\n",
      "Gini: 0.4552416301746409\n",
      "\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.3082383667488498: "
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>110415.0</td>\n",
       "      <td>87952.0</td>\n",
       "      <td>0.4434</td>\n",
       "      <td>(87952.0/198367.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>27576.0</td>\n",
       "      <td>88179.0</td>\n",
       "      <td>0.2382</td>\n",
       "      <td>(27576.0/115755.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Total</td>\n",
       "      <td>137991.0</td>\n",
       "      <td>176131.0</td>\n",
       "      <td>0.3678</td>\n",
       "      <td>(115528.0/314122.0)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0         1   Error                  Rate\n",
       "0      0  110415.0   87952.0  0.4434    (87952.0/198367.0)\n",
       "1      1   27576.0   88179.0  0.2382    (27576.0/115755.0)\n",
       "2  Total  137991.0  176131.0  0.3678   (115528.0/314122.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Maximum Metrics: Maximum metrics at their respective thresholds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>threshold</th>\n",
       "      <th>value</th>\n",
       "      <th>idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>max f1</td>\n",
       "      <td>0.308238</td>\n",
       "      <td>0.604202</td>\n",
       "      <td>254.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>max f2</td>\n",
       "      <td>0.152925</td>\n",
       "      <td>0.753347</td>\n",
       "      <td>352.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>max f0point5</td>\n",
       "      <td>0.459942</td>\n",
       "      <td>0.578745</td>\n",
       "      <td>170.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>max accuracy</td>\n",
       "      <td>0.501562</td>\n",
       "      <td>0.697356</td>\n",
       "      <td>148.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>max precision</td>\n",
       "      <td>0.928695</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>max recall</td>\n",
       "      <td>0.055597</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>398.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>max specificity</td>\n",
       "      <td>0.928695</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>max absolute_mcc</td>\n",
       "      <td>0.391396</td>\n",
       "      <td>0.325785</td>\n",
       "      <td>208.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>max min_per_class_accuracy</td>\n",
       "      <td>0.365508</td>\n",
       "      <td>0.663764</td>\n",
       "      <td>222.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>max mean_per_class_accuracy</td>\n",
       "      <td>0.369733</td>\n",
       "      <td>0.666695</td>\n",
       "      <td>220.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        metric  threshold     value    idx\n",
       "0                       max f1   0.308238  0.604202  254.0\n",
       "1                       max f2   0.152925  0.753347  352.0\n",
       "2                 max f0point5   0.459942  0.578745  170.0\n",
       "3                 max accuracy   0.501562  0.697356  148.0\n",
       "4                max precision   0.928695  1.000000    0.0\n",
       "5                   max recall   0.055597  1.000000  398.0\n",
       "6              max specificity   0.928695  1.000000    0.0\n",
       "7             max absolute_mcc   0.391396  0.325785  208.0\n",
       "8   max min_per_class_accuracy   0.365508  0.663764  222.0\n",
       "9  max mean_per_class_accuracy   0.369733  0.666695  220.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Gains/Lift Table: Avg response rate: 36.85 %, avg score: 36.85 %\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>cumulative_data_fraction</th>\n",
       "      <th>lower_threshold</th>\n",
       "      <th>lift</th>\n",
       "      <th>cumulative_lift</th>\n",
       "      <th>response_rate</th>\n",
       "      <th>score</th>\n",
       "      <th>cumulative_response_rate</th>\n",
       "      <th>cumulative_score</th>\n",
       "      <th>capture_rate</th>\n",
       "      <th>cumulative_capture_rate</th>\n",
       "      <th>gain</th>\n",
       "      <th>cumulative_gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0.010002</td>\n",
       "      <td>0.828712</td>\n",
       "      <td>2.402755</td>\n",
       "      <td>2.402755</td>\n",
       "      <td>0.885423</td>\n",
       "      <td>0.864628</td>\n",
       "      <td>0.885423</td>\n",
       "      <td>0.864628</td>\n",
       "      <td>0.024034</td>\n",
       "      <td>0.024034</td>\n",
       "      <td>140.275528</td>\n",
       "      <td>140.275528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>0.020002</td>\n",
       "      <td>0.786386</td>\n",
       "      <td>2.216042</td>\n",
       "      <td>2.309414</td>\n",
       "      <td>0.816619</td>\n",
       "      <td>0.805583</td>\n",
       "      <td>0.851027</td>\n",
       "      <td>0.835110</td>\n",
       "      <td>0.022159</td>\n",
       "      <td>0.046192</td>\n",
       "      <td>121.604221</td>\n",
       "      <td>130.941360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>0.030007</td>\n",
       "      <td>0.763294</td>\n",
       "      <td>2.107570</td>\n",
       "      <td>2.242111</td>\n",
       "      <td>0.776647</td>\n",
       "      <td>0.773547</td>\n",
       "      <td>0.826225</td>\n",
       "      <td>0.814583</td>\n",
       "      <td>0.021088</td>\n",
       "      <td>0.067280</td>\n",
       "      <td>110.756993</td>\n",
       "      <td>124.211096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>4</td>\n",
       "      <td>0.040000</td>\n",
       "      <td>0.742632</td>\n",
       "      <td>2.086914</td>\n",
       "      <td>2.203340</td>\n",
       "      <td>0.769035</td>\n",
       "      <td>0.753464</td>\n",
       "      <td>0.811938</td>\n",
       "      <td>0.799314</td>\n",
       "      <td>0.020854</td>\n",
       "      <td>0.088134</td>\n",
       "      <td>108.691396</td>\n",
       "      <td>120.333950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>0.050003</td>\n",
       "      <td>0.719768</td>\n",
       "      <td>2.007190</td>\n",
       "      <td>2.164102</td>\n",
       "      <td>0.739656</td>\n",
       "      <td>0.731279</td>\n",
       "      <td>0.797479</td>\n",
       "      <td>0.785704</td>\n",
       "      <td>0.020077</td>\n",
       "      <td>0.108211</td>\n",
       "      <td>100.719025</td>\n",
       "      <td>116.410216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>6</td>\n",
       "      <td>0.100012</td>\n",
       "      <td>0.640659</td>\n",
       "      <td>1.847527</td>\n",
       "      <td>2.005805</td>\n",
       "      <td>0.680820</td>\n",
       "      <td>0.675509</td>\n",
       "      <td>0.739146</td>\n",
       "      <td>0.730603</td>\n",
       "      <td>0.092393</td>\n",
       "      <td>0.200605</td>\n",
       "      <td>84.752721</td>\n",
       "      <td>100.580461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>7</td>\n",
       "      <td>0.150002</td>\n",
       "      <td>0.578208</td>\n",
       "      <td>1.665916</td>\n",
       "      <td>1.892532</td>\n",
       "      <td>0.613895</td>\n",
       "      <td>0.608137</td>\n",
       "      <td>0.697404</td>\n",
       "      <td>0.689790</td>\n",
       "      <td>0.083279</td>\n",
       "      <td>0.283884</td>\n",
       "      <td>66.591561</td>\n",
       "      <td>89.253232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>8</td>\n",
       "      <td>0.200002</td>\n",
       "      <td>0.527343</td>\n",
       "      <td>1.504394</td>\n",
       "      <td>1.795499</td>\n",
       "      <td>0.554374</td>\n",
       "      <td>0.554116</td>\n",
       "      <td>0.661647</td>\n",
       "      <td>0.655872</td>\n",
       "      <td>0.075219</td>\n",
       "      <td>0.359103</td>\n",
       "      <td>50.439384</td>\n",
       "      <td>79.549924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>9</td>\n",
       "      <td>0.300024</td>\n",
       "      <td>0.455729</td>\n",
       "      <td>1.332786</td>\n",
       "      <td>1.641240</td>\n",
       "      <td>0.491136</td>\n",
       "      <td>0.489654</td>\n",
       "      <td>0.604802</td>\n",
       "      <td>0.600458</td>\n",
       "      <td>0.133307</td>\n",
       "      <td>0.492411</td>\n",
       "      <td>33.278565</td>\n",
       "      <td>64.124010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>10</td>\n",
       "      <td>0.400001</td>\n",
       "      <td>0.396810</td>\n",
       "      <td>1.156587</td>\n",
       "      <td>1.520105</td>\n",
       "      <td>0.426206</td>\n",
       "      <td>0.426881</td>\n",
       "      <td>0.560164</td>\n",
       "      <td>0.557074</td>\n",
       "      <td>0.115632</td>\n",
       "      <td>0.608043</td>\n",
       "      <td>15.658664</td>\n",
       "      <td>52.010470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>11</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.341210</td>\n",
       "      <td>0.996076</td>\n",
       "      <td>1.415300</td>\n",
       "      <td>0.367057</td>\n",
       "      <td>0.368175</td>\n",
       "      <td>0.521543</td>\n",
       "      <td>0.519294</td>\n",
       "      <td>0.099607</td>\n",
       "      <td>0.707650</td>\n",
       "      <td>-0.392437</td>\n",
       "      <td>41.529956</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>12</td>\n",
       "      <td>0.600003</td>\n",
       "      <td>0.286003</td>\n",
       "      <td>0.847804</td>\n",
       "      <td>1.320715</td>\n",
       "      <td>0.312418</td>\n",
       "      <td>0.313215</td>\n",
       "      <td>0.486688</td>\n",
       "      <td>0.484947</td>\n",
       "      <td>0.084783</td>\n",
       "      <td>0.792432</td>\n",
       "      <td>-15.219644</td>\n",
       "      <td>32.071488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.699999</td>\n",
       "      <td>0.240192</td>\n",
       "      <td>0.723107</td>\n",
       "      <td>1.235345</td>\n",
       "      <td>0.266467</td>\n",
       "      <td>0.262708</td>\n",
       "      <td>0.455229</td>\n",
       "      <td>0.453200</td>\n",
       "      <td>0.072308</td>\n",
       "      <td>0.864740</td>\n",
       "      <td>-27.689346</td>\n",
       "      <td>23.534537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>14</td>\n",
       "      <td>0.799998</td>\n",
       "      <td>0.196423</td>\n",
       "      <td>0.587970</td>\n",
       "      <td>1.154424</td>\n",
       "      <td>0.216669</td>\n",
       "      <td>0.217394</td>\n",
       "      <td>0.425409</td>\n",
       "      <td>0.423724</td>\n",
       "      <td>0.058797</td>\n",
       "      <td>0.923537</td>\n",
       "      <td>-41.203029</td>\n",
       "      <td>15.442373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>15</td>\n",
       "      <td>0.899997</td>\n",
       "      <td>0.153484</td>\n",
       "      <td>0.462618</td>\n",
       "      <td>1.077557</td>\n",
       "      <td>0.170476</td>\n",
       "      <td>0.176456</td>\n",
       "      <td>0.397083</td>\n",
       "      <td>0.396250</td>\n",
       "      <td>0.046262</td>\n",
       "      <td>0.969798</td>\n",
       "      <td>-53.738205</td>\n",
       "      <td>7.755669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.030628</td>\n",
       "      <td>0.302009</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.111292</td>\n",
       "      <td>0.118801</td>\n",
       "      <td>0.368503</td>\n",
       "      <td>0.368504</td>\n",
       "      <td>0.030202</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-69.799050</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      group  cumulative_data_fraction  lower_threshold      lift  \\\n",
       "0         1                  0.010002         0.828712  2.402755   \n",
       "1         2                  0.020002         0.786386  2.216042   \n",
       "2         3                  0.030007         0.763294  2.107570   \n",
       "3         4                  0.040000         0.742632  2.086914   \n",
       "4         5                  0.050003         0.719768  2.007190   \n",
       "5         6                  0.100012         0.640659  1.847527   \n",
       "6         7                  0.150002         0.578208  1.665916   \n",
       "7         8                  0.200002         0.527343  1.504394   \n",
       "8         9                  0.300024         0.455729  1.332786   \n",
       "9        10                  0.400001         0.396810  1.156587   \n",
       "10       11                  0.500000         0.341210  0.996076   \n",
       "11       12                  0.600003         0.286003  0.847804   \n",
       "12       13                  0.699999         0.240192  0.723107   \n",
       "13       14                  0.799998         0.196423  0.587970   \n",
       "14       15                  0.899997         0.153484  0.462618   \n",
       "15       16                  1.000000         0.030628  0.302009   \n",
       "\n",
       "    cumulative_lift  response_rate     score  cumulative_response_rate  \\\n",
       "0          2.402755       0.885423  0.864628                  0.885423   \n",
       "1          2.309414       0.816619  0.805583                  0.851027   \n",
       "2          2.242111       0.776647  0.773547                  0.826225   \n",
       "3          2.203340       0.769035  0.753464                  0.811938   \n",
       "4          2.164102       0.739656  0.731279                  0.797479   \n",
       "5          2.005805       0.680820  0.675509                  0.739146   \n",
       "6          1.892532       0.613895  0.608137                  0.697404   \n",
       "7          1.795499       0.554374  0.554116                  0.661647   \n",
       "8          1.641240       0.491136  0.489654                  0.604802   \n",
       "9          1.520105       0.426206  0.426881                  0.560164   \n",
       "10         1.415300       0.367057  0.368175                  0.521543   \n",
       "11         1.320715       0.312418  0.313215                  0.486688   \n",
       "12         1.235345       0.266467  0.262708                  0.455229   \n",
       "13         1.154424       0.216669  0.217394                  0.425409   \n",
       "14         1.077557       0.170476  0.176456                  0.397083   \n",
       "15         1.000000       0.111292  0.118801                  0.368503   \n",
       "\n",
       "    cumulative_score  capture_rate  cumulative_capture_rate        gain  \\\n",
       "0           0.864628      0.024034                 0.024034  140.275528   \n",
       "1           0.835110      0.022159                 0.046192  121.604221   \n",
       "2           0.814583      0.021088                 0.067280  110.756993   \n",
       "3           0.799314      0.020854                 0.088134  108.691396   \n",
       "4           0.785704      0.020077                 0.108211  100.719025   \n",
       "5           0.730603      0.092393                 0.200605   84.752721   \n",
       "6           0.689790      0.083279                 0.283884   66.591561   \n",
       "7           0.655872      0.075219                 0.359103   50.439384   \n",
       "8           0.600458      0.133307                 0.492411   33.278565   \n",
       "9           0.557074      0.115632                 0.608043   15.658664   \n",
       "10          0.519294      0.099607                 0.707650   -0.392437   \n",
       "11          0.484947      0.084783                 0.792432  -15.219644   \n",
       "12          0.453200      0.072308                 0.864740  -27.689346   \n",
       "13          0.423724      0.058797                 0.923537  -41.203029   \n",
       "14          0.396250      0.046262                 0.969798  -53.738205   \n",
       "15          0.368504      0.030202                 1.000000  -69.799050   \n",
       "\n",
       "    cumulative_gain  \n",
       "0        140.275528  \n",
       "1        130.941360  \n",
       "2        124.211096  \n",
       "3        120.333950  \n",
       "4        116.410216  \n",
       "5        100.580461  \n",
       "6         89.253232  \n",
       "7         79.549924  \n",
       "8         64.124010  \n",
       "9         52.010470  \n",
       "10        41.529956  \n",
       "11        32.071488  \n",
       "12        23.534537  \n",
       "13        15.442373  \n",
       "14         7.755669  \n",
       "15         0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "ModelMetricsBinomial: gbm\n",
      "** Reported on validation data. **\n",
      "\n",
      "MSE: 0.19726966691087394\n",
      "RMSE: 0.444150500293396\n",
      "LogLoss: 0.5786944223627535\n",
      "Mean Per-Class Error: 0.33191619190721755\n",
      "AUC: 0.727593424555288\n",
      "pr_auc: 0.610124995970803\n",
      "Gini: 0.45518684911057594\n",
      "\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.3059424318662228: "
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>23540.0</td>\n",
       "      <td>18949.0</td>\n",
       "      <td>0.446</td>\n",
       "      <td>(18949.0/42489.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5745.0</td>\n",
       "      <td>18926.0</td>\n",
       "      <td>0.2329</td>\n",
       "      <td>(5745.0/24671.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Total</td>\n",
       "      <td>29285.0</td>\n",
       "      <td>37875.0</td>\n",
       "      <td>0.3677</td>\n",
       "      <td>(24694.0/67160.0)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                0        1   Error                Rate\n",
       "0      0  23540.0  18949.0   0.446   (18949.0/42489.0)\n",
       "1      1   5745.0  18926.0  0.2329    (5745.0/24671.0)\n",
       "2  Total  29285.0  37875.0  0.3677   (24694.0/67160.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Maximum Metrics: Maximum metrics at their respective thresholds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>threshold</th>\n",
       "      <th>value</th>\n",
       "      <th>idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>max f1</td>\n",
       "      <td>0.305942</td>\n",
       "      <td>0.605187</td>\n",
       "      <td>253.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>max f2</td>\n",
       "      <td>0.152247</td>\n",
       "      <td>0.753210</td>\n",
       "      <td>353.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>max f0point5</td>\n",
       "      <td>0.460136</td>\n",
       "      <td>0.578742</td>\n",
       "      <td>166.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>max accuracy</td>\n",
       "      <td>0.505080</td>\n",
       "      <td>0.698392</td>\n",
       "      <td>142.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>max precision</td>\n",
       "      <td>0.908542</td>\n",
       "      <td>0.984127</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>max recall</td>\n",
       "      <td>0.060960</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>397.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>max specificity</td>\n",
       "      <td>0.928602</td>\n",
       "      <td>0.999976</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>max absolute_mcc</td>\n",
       "      <td>0.375693</td>\n",
       "      <td>0.326135</td>\n",
       "      <td>215.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>max min_per_class_accuracy</td>\n",
       "      <td>0.362964</td>\n",
       "      <td>0.666996</td>\n",
       "      <td>222.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>max mean_per_class_accuracy</td>\n",
       "      <td>0.364870</td>\n",
       "      <td>0.668084</td>\n",
       "      <td>221.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        metric  threshold     value    idx\n",
       "0                       max f1   0.305942  0.605187  253.0\n",
       "1                       max f2   0.152247  0.753210  353.0\n",
       "2                 max f0point5   0.460136  0.578742  166.0\n",
       "3                 max accuracy   0.505080  0.698392  142.0\n",
       "4                max precision   0.908542  0.984127    3.0\n",
       "5                   max recall   0.060960  1.000000  397.0\n",
       "6              max specificity   0.928602  0.999976    0.0\n",
       "7             max absolute_mcc   0.375693  0.326135  215.0\n",
       "8   max min_per_class_accuracy   0.362964  0.666996  222.0\n",
       "9  max mean_per_class_accuracy   0.364870  0.668084  221.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Gains/Lift Table: Avg response rate: 36.73 %, avg score: 36.78 %\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>cumulative_data_fraction</th>\n",
       "      <th>lower_threshold</th>\n",
       "      <th>lift</th>\n",
       "      <th>cumulative_lift</th>\n",
       "      <th>response_rate</th>\n",
       "      <th>score</th>\n",
       "      <th>cumulative_response_rate</th>\n",
       "      <th>cumulative_score</th>\n",
       "      <th>capture_rate</th>\n",
       "      <th>cumulative_capture_rate</th>\n",
       "      <th>gain</th>\n",
       "      <th>cumulative_gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0.010006</td>\n",
       "      <td>0.828643</td>\n",
       "      <td>2.369794</td>\n",
       "      <td>2.369794</td>\n",
       "      <td>0.870536</td>\n",
       "      <td>0.863653</td>\n",
       "      <td>0.870536</td>\n",
       "      <td>0.863653</td>\n",
       "      <td>0.023712</td>\n",
       "      <td>0.023712</td>\n",
       "      <td>136.979363</td>\n",
       "      <td>136.979363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>0.020116</td>\n",
       "      <td>0.785004</td>\n",
       "      <td>2.241124</td>\n",
       "      <td>2.305126</td>\n",
       "      <td>0.823270</td>\n",
       "      <td>0.804792</td>\n",
       "      <td>0.846780</td>\n",
       "      <td>0.834070</td>\n",
       "      <td>0.022658</td>\n",
       "      <td>0.046370</td>\n",
       "      <td>124.112442</td>\n",
       "      <td>130.512568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>0.030003</td>\n",
       "      <td>0.762510</td>\n",
       "      <td>2.086765</td>\n",
       "      <td>2.233170</td>\n",
       "      <td>0.766566</td>\n",
       "      <td>0.772705</td>\n",
       "      <td>0.820347</td>\n",
       "      <td>0.813848</td>\n",
       "      <td>0.020632</td>\n",
       "      <td>0.067002</td>\n",
       "      <td>108.676545</td>\n",
       "      <td>123.316975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>4</td>\n",
       "      <td>0.040009</td>\n",
       "      <td>0.741507</td>\n",
       "      <td>2.057872</td>\n",
       "      <td>2.189329</td>\n",
       "      <td>0.755952</td>\n",
       "      <td>0.752465</td>\n",
       "      <td>0.804243</td>\n",
       "      <td>0.798497</td>\n",
       "      <td>0.020591</td>\n",
       "      <td>0.087593</td>\n",
       "      <td>105.787207</td>\n",
       "      <td>118.932902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.717152</td>\n",
       "      <td>1.983857</td>\n",
       "      <td>2.148271</td>\n",
       "      <td>0.728763</td>\n",
       "      <td>0.729365</td>\n",
       "      <td>0.789160</td>\n",
       "      <td>0.784683</td>\n",
       "      <td>0.019821</td>\n",
       "      <td>0.107414</td>\n",
       "      <td>98.385658</td>\n",
       "      <td>114.827125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>6</td>\n",
       "      <td>0.100045</td>\n",
       "      <td>0.637059</td>\n",
       "      <td>1.882312</td>\n",
       "      <td>2.015232</td>\n",
       "      <td>0.691461</td>\n",
       "      <td>0.673227</td>\n",
       "      <td>0.740289</td>\n",
       "      <td>0.728930</td>\n",
       "      <td>0.094200</td>\n",
       "      <td>0.201613</td>\n",
       "      <td>88.231172</td>\n",
       "      <td>101.523211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>7</td>\n",
       "      <td>0.150030</td>\n",
       "      <td>0.576325</td>\n",
       "      <td>1.650202</td>\n",
       "      <td>1.893616</td>\n",
       "      <td>0.606196</td>\n",
       "      <td>0.605305</td>\n",
       "      <td>0.695613</td>\n",
       "      <td>0.687742</td>\n",
       "      <td>0.082486</td>\n",
       "      <td>0.284099</td>\n",
       "      <td>65.020161</td>\n",
       "      <td>89.361565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>8</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.526255</td>\n",
       "      <td>1.507120</td>\n",
       "      <td>1.797049</td>\n",
       "      <td>0.553635</td>\n",
       "      <td>0.552535</td>\n",
       "      <td>0.660140</td>\n",
       "      <td>0.653960</td>\n",
       "      <td>0.075311</td>\n",
       "      <td>0.359410</td>\n",
       "      <td>50.711951</td>\n",
       "      <td>79.704917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>9</td>\n",
       "      <td>0.300045</td>\n",
       "      <td>0.454845</td>\n",
       "      <td>1.333359</td>\n",
       "      <td>1.642440</td>\n",
       "      <td>0.489805</td>\n",
       "      <td>0.488514</td>\n",
       "      <td>0.603345</td>\n",
       "      <td>0.598795</td>\n",
       "      <td>0.133395</td>\n",
       "      <td>0.492805</td>\n",
       "      <td>33.335924</td>\n",
       "      <td>64.243984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>10</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.395776</td>\n",
       "      <td>1.150853</td>\n",
       "      <td>1.519598</td>\n",
       "      <td>0.422762</td>\n",
       "      <td>0.426238</td>\n",
       "      <td>0.558219</td>\n",
       "      <td>0.555675</td>\n",
       "      <td>0.115034</td>\n",
       "      <td>0.607839</td>\n",
       "      <td>15.085253</td>\n",
       "      <td>51.959791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>11</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.339761</td>\n",
       "      <td>1.011714</td>\n",
       "      <td>1.418021</td>\n",
       "      <td>0.371650</td>\n",
       "      <td>0.366786</td>\n",
       "      <td>0.520905</td>\n",
       "      <td>0.517897</td>\n",
       "      <td>0.101171</td>\n",
       "      <td>0.709011</td>\n",
       "      <td>1.171416</td>\n",
       "      <td>41.802116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>12</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.285901</td>\n",
       "      <td>0.830935</td>\n",
       "      <td>1.320173</td>\n",
       "      <td>0.305241</td>\n",
       "      <td>0.312678</td>\n",
       "      <td>0.484961</td>\n",
       "      <td>0.483694</td>\n",
       "      <td>0.083094</td>\n",
       "      <td>0.792104</td>\n",
       "      <td>-16.906489</td>\n",
       "      <td>32.017348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.240300</td>\n",
       "      <td>0.721090</td>\n",
       "      <td>1.234590</td>\n",
       "      <td>0.264890</td>\n",
       "      <td>0.262339</td>\n",
       "      <td>0.453523</td>\n",
       "      <td>0.452072</td>\n",
       "      <td>0.072109</td>\n",
       "      <td>0.864213</td>\n",
       "      <td>-27.891046</td>\n",
       "      <td>23.459006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>14</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.196655</td>\n",
       "      <td>0.588545</td>\n",
       "      <td>1.153834</td>\n",
       "      <td>0.216200</td>\n",
       "      <td>0.217502</td>\n",
       "      <td>0.423857</td>\n",
       "      <td>0.422751</td>\n",
       "      <td>0.058855</td>\n",
       "      <td>0.923068</td>\n",
       "      <td>-41.145474</td>\n",
       "      <td>15.383446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>15</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.154382</td>\n",
       "      <td>0.476268</td>\n",
       "      <td>1.078549</td>\n",
       "      <td>0.174955</td>\n",
       "      <td>0.176781</td>\n",
       "      <td>0.396201</td>\n",
       "      <td>0.395421</td>\n",
       "      <td>0.047627</td>\n",
       "      <td>0.970694</td>\n",
       "      <td>-52.373232</td>\n",
       "      <td>7.854926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.040826</td>\n",
       "      <td>0.293057</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.107653</td>\n",
       "      <td>0.118941</td>\n",
       "      <td>0.367347</td>\n",
       "      <td>0.367773</td>\n",
       "      <td>0.029306</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-70.694337</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      group  cumulative_data_fraction  lower_threshold      lift  \\\n",
       "0         1                  0.010006         0.828643  2.369794   \n",
       "1         2                  0.020116         0.785004  2.241124   \n",
       "2         3                  0.030003         0.762510  2.086765   \n",
       "3         4                  0.040009         0.741507  2.057872   \n",
       "4         5                  0.050000         0.717152  1.983857   \n",
       "5         6                  0.100045         0.637059  1.882312   \n",
       "6         7                  0.150030         0.576325  1.650202   \n",
       "7         8                  0.200000         0.526255  1.507120   \n",
       "8         9                  0.300045         0.454845  1.333359   \n",
       "9        10                  0.400000         0.395776  1.150853   \n",
       "10       11                  0.500000         0.339761  1.011714   \n",
       "11       12                  0.600000         0.285901  0.830935   \n",
       "12       13                  0.700000         0.240300  0.721090   \n",
       "13       14                  0.800000         0.196655  0.588545   \n",
       "14       15                  0.900000         0.154382  0.476268   \n",
       "15       16                  1.000000         0.040826  0.293057   \n",
       "\n",
       "    cumulative_lift  response_rate     score  cumulative_response_rate  \\\n",
       "0          2.369794       0.870536  0.863653                  0.870536   \n",
       "1          2.305126       0.823270  0.804792                  0.846780   \n",
       "2          2.233170       0.766566  0.772705                  0.820347   \n",
       "3          2.189329       0.755952  0.752465                  0.804243   \n",
       "4          2.148271       0.728763  0.729365                  0.789160   \n",
       "5          2.015232       0.691461  0.673227                  0.740289   \n",
       "6          1.893616       0.606196  0.605305                  0.695613   \n",
       "7          1.797049       0.553635  0.552535                  0.660140   \n",
       "8          1.642440       0.489805  0.488514                  0.603345   \n",
       "9          1.519598       0.422762  0.426238                  0.558219   \n",
       "10         1.418021       0.371650  0.366786                  0.520905   \n",
       "11         1.320173       0.305241  0.312678                  0.484961   \n",
       "12         1.234590       0.264890  0.262339                  0.453523   \n",
       "13         1.153834       0.216200  0.217502                  0.423857   \n",
       "14         1.078549       0.174955  0.176781                  0.396201   \n",
       "15         1.000000       0.107653  0.118941                  0.367347   \n",
       "\n",
       "    cumulative_score  capture_rate  cumulative_capture_rate        gain  \\\n",
       "0           0.863653      0.023712                 0.023712  136.979363   \n",
       "1           0.834070      0.022658                 0.046370  124.112442   \n",
       "2           0.813848      0.020632                 0.067002  108.676545   \n",
       "3           0.798497      0.020591                 0.087593  105.787207   \n",
       "4           0.784683      0.019821                 0.107414   98.385658   \n",
       "5           0.728930      0.094200                 0.201613   88.231172   \n",
       "6           0.687742      0.082486                 0.284099   65.020161   \n",
       "7           0.653960      0.075311                 0.359410   50.711951   \n",
       "8           0.598795      0.133395                 0.492805   33.335924   \n",
       "9           0.555675      0.115034                 0.607839   15.085253   \n",
       "10          0.517897      0.101171                 0.709011    1.171416   \n",
       "11          0.483694      0.083094                 0.792104  -16.906489   \n",
       "12          0.452072      0.072109                 0.864213  -27.891046   \n",
       "13          0.422751      0.058855                 0.923068  -41.145474   \n",
       "14          0.395421      0.047627                 0.970694  -52.373232   \n",
       "15          0.367773      0.029306                 1.000000  -70.694337   \n",
       "\n",
       "    cumulative_gain  \n",
       "0        136.979363  \n",
       "1        130.512568  \n",
       "2        123.316975  \n",
       "3        118.932902  \n",
       "4        114.827125  \n",
       "5        101.523211  \n",
       "6         89.361565  \n",
       "7         79.704917  \n",
       "8         64.243984  \n",
       "9         51.959791  \n",
       "10        41.802116  \n",
       "11        32.017348  \n",
       "12        23.459006  \n",
       "13        15.383446  \n",
       "14         7.854926  \n",
       "15         0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Scoring History: "
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>duration</th>\n",
       "      <th>number_of_trees</th>\n",
       "      <th>training_rmse</th>\n",
       "      <th>training_logloss</th>\n",
       "      <th>training_auc</th>\n",
       "      <th>training_pr_auc</th>\n",
       "      <th>training_lift</th>\n",
       "      <th>training_classification_error</th>\n",
       "      <th>validation_rmse</th>\n",
       "      <th>validation_logloss</th>\n",
       "      <th>validation_auc</th>\n",
       "      <th>validation_pr_auc</th>\n",
       "      <th>validation_lift</th>\n",
       "      <th>validation_classification_error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:19:12</td>\n",
       "      <td>0.012 sec</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.482399</td>\n",
       "      <td>0.658154</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.631497</td>\n",
       "      <td>0.482083</td>\n",
       "      <td>0.657531</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.632653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:19:13</td>\n",
       "      <td>1.333 sec</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.481705</td>\n",
       "      <td>0.656718</td>\n",
       "      <td>0.721281</td>\n",
       "      <td>0.569402</td>\n",
       "      <td>2.289071</td>\n",
       "      <td>0.359561</td>\n",
       "      <td>0.481386</td>\n",
       "      <td>0.656089</td>\n",
       "      <td>0.723266</td>\n",
       "      <td>0.570940</td>\n",
       "      <td>2.291923</td>\n",
       "      <td>0.357370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:19:13</td>\n",
       "      <td>1.639 sec</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.481025</td>\n",
       "      <td>0.655313</td>\n",
       "      <td>0.721487</td>\n",
       "      <td>0.569547</td>\n",
       "      <td>2.289071</td>\n",
       "      <td>0.359561</td>\n",
       "      <td>0.480702</td>\n",
       "      <td>0.654678</td>\n",
       "      <td>0.723424</td>\n",
       "      <td>0.571011</td>\n",
       "      <td>2.291923</td>\n",
       "      <td>0.357370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:19:14</td>\n",
       "      <td>1.841 sec</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.480356</td>\n",
       "      <td>0.653936</td>\n",
       "      <td>0.721588</td>\n",
       "      <td>0.589096</td>\n",
       "      <td>2.289071</td>\n",
       "      <td>0.359561</td>\n",
       "      <td>0.480030</td>\n",
       "      <td>0.653294</td>\n",
       "      <td>0.723559</td>\n",
       "      <td>0.589766</td>\n",
       "      <td>2.291923</td>\n",
       "      <td>0.357370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:19:14</td>\n",
       "      <td>2.027 sec</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.479701</td>\n",
       "      <td>0.652590</td>\n",
       "      <td>0.722411</td>\n",
       "      <td>0.590812</td>\n",
       "      <td>2.288996</td>\n",
       "      <td>0.376733</td>\n",
       "      <td>0.479373</td>\n",
       "      <td>0.651944</td>\n",
       "      <td>0.724206</td>\n",
       "      <td>0.591456</td>\n",
       "      <td>2.293831</td>\n",
       "      <td>0.356358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:19:14</td>\n",
       "      <td>2.209 sec</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.479057</td>\n",
       "      <td>0.651270</td>\n",
       "      <td>0.722372</td>\n",
       "      <td>0.590803</td>\n",
       "      <td>2.288996</td>\n",
       "      <td>0.371531</td>\n",
       "      <td>0.478725</td>\n",
       "      <td>0.650617</td>\n",
       "      <td>0.724238</td>\n",
       "      <td>0.591521</td>\n",
       "      <td>2.293831</td>\n",
       "      <td>0.356626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:19:14</td>\n",
       "      <td>2.410 sec</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.478426</td>\n",
       "      <td>0.649978</td>\n",
       "      <td>0.722452</td>\n",
       "      <td>0.590815</td>\n",
       "      <td>2.288996</td>\n",
       "      <td>0.371149</td>\n",
       "      <td>0.478091</td>\n",
       "      <td>0.649321</td>\n",
       "      <td>0.724261</td>\n",
       "      <td>0.591450</td>\n",
       "      <td>2.293831</td>\n",
       "      <td>0.356358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:19:14</td>\n",
       "      <td>2.603 sec</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.477805</td>\n",
       "      <td>0.648711</td>\n",
       "      <td>0.722552</td>\n",
       "      <td>0.590854</td>\n",
       "      <td>2.292176</td>\n",
       "      <td>0.376284</td>\n",
       "      <td>0.477467</td>\n",
       "      <td>0.648047</td>\n",
       "      <td>0.724298</td>\n",
       "      <td>0.591508</td>\n",
       "      <td>2.304677</td>\n",
       "      <td>0.356313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:19:15</td>\n",
       "      <td>2.801 sec</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.477197</td>\n",
       "      <td>0.647471</td>\n",
       "      <td>0.722574</td>\n",
       "      <td>0.590964</td>\n",
       "      <td>2.292660</td>\n",
       "      <td>0.375908</td>\n",
       "      <td>0.476856</td>\n",
       "      <td>0.646803</td>\n",
       "      <td>0.724324</td>\n",
       "      <td>0.591713</td>\n",
       "      <td>2.304677</td>\n",
       "      <td>0.356313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:19:15</td>\n",
       "      <td>2.995 sec</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.476599</td>\n",
       "      <td>0.646254</td>\n",
       "      <td>0.722710</td>\n",
       "      <td>0.591004</td>\n",
       "      <td>2.292660</td>\n",
       "      <td>0.376370</td>\n",
       "      <td>0.476255</td>\n",
       "      <td>0.645579</td>\n",
       "      <td>0.724557</td>\n",
       "      <td>0.591808</td>\n",
       "      <td>2.304677</td>\n",
       "      <td>0.356313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:19:15</td>\n",
       "      <td>3.214 sec</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.476012</td>\n",
       "      <td>0.645063</td>\n",
       "      <td>0.723008</td>\n",
       "      <td>0.597556</td>\n",
       "      <td>2.354895</td>\n",
       "      <td>0.369474</td>\n",
       "      <td>0.475665</td>\n",
       "      <td>0.644383</td>\n",
       "      <td>0.724929</td>\n",
       "      <td>0.598834</td>\n",
       "      <td>2.341662</td>\n",
       "      <td>0.357088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:19:15</td>\n",
       "      <td>3.437 sec</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.475437</td>\n",
       "      <td>0.643896</td>\n",
       "      <td>0.723054</td>\n",
       "      <td>0.597600</td>\n",
       "      <td>2.354895</td>\n",
       "      <td>0.363569</td>\n",
       "      <td>0.475087</td>\n",
       "      <td>0.643211</td>\n",
       "      <td>0.724965</td>\n",
       "      <td>0.598863</td>\n",
       "      <td>2.341662</td>\n",
       "      <td>0.361510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:19:16</td>\n",
       "      <td>3.895 sec</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.474871</td>\n",
       "      <td>0.642751</td>\n",
       "      <td>0.723130</td>\n",
       "      <td>0.597548</td>\n",
       "      <td>2.359000</td>\n",
       "      <td>0.362884</td>\n",
       "      <td>0.474520</td>\n",
       "      <td>0.642063</td>\n",
       "      <td>0.725024</td>\n",
       "      <td>0.598955</td>\n",
       "      <td>2.343449</td>\n",
       "      <td>0.361286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:19:16</td>\n",
       "      <td>4.114 sec</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.474316</td>\n",
       "      <td>0.641629</td>\n",
       "      <td>0.723152</td>\n",
       "      <td>0.597535</td>\n",
       "      <td>2.359000</td>\n",
       "      <td>0.363209</td>\n",
       "      <td>0.473962</td>\n",
       "      <td>0.640935</td>\n",
       "      <td>0.725071</td>\n",
       "      <td>0.598807</td>\n",
       "      <td>2.343449</td>\n",
       "      <td>0.361629</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:19:20</td>\n",
       "      <td>8.169 sec</td>\n",
       "      <td>59.0</td>\n",
       "      <td>0.457322</td>\n",
       "      <td>0.607547</td>\n",
       "      <td>0.724917</td>\n",
       "      <td>0.603430</td>\n",
       "      <td>2.378956</td>\n",
       "      <td>0.387066</td>\n",
       "      <td>0.456853</td>\n",
       "      <td>0.606634</td>\n",
       "      <td>0.726611</td>\n",
       "      <td>0.604619</td>\n",
       "      <td>2.352443</td>\n",
       "      <td>0.367764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:19:27</td>\n",
       "      <td>14.730 sec</td>\n",
       "      <td>124.0</td>\n",
       "      <td>0.448565</td>\n",
       "      <td>0.589201</td>\n",
       "      <td>0.725773</td>\n",
       "      <td>0.608407</td>\n",
       "      <td>2.374672</td>\n",
       "      <td>0.378242</td>\n",
       "      <td>0.448037</td>\n",
       "      <td>0.588163</td>\n",
       "      <td>0.727045</td>\n",
       "      <td>0.608089</td>\n",
       "      <td>2.351360</td>\n",
       "      <td>0.360721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:19:35</td>\n",
       "      <td>23.317 sec</td>\n",
       "      <td>210.0</td>\n",
       "      <td>0.445549</td>\n",
       "      <td>0.582153</td>\n",
       "      <td>0.726333</td>\n",
       "      <td>0.609730</td>\n",
       "      <td>2.382533</td>\n",
       "      <td>0.363792</td>\n",
       "      <td>0.445033</td>\n",
       "      <td>0.581121</td>\n",
       "      <td>0.727299</td>\n",
       "      <td>0.609588</td>\n",
       "      <td>2.353590</td>\n",
       "      <td>0.365873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:19:44</td>\n",
       "      <td>32.124 sec</td>\n",
       "      <td>301.0</td>\n",
       "      <td>0.444802</td>\n",
       "      <td>0.580213</td>\n",
       "      <td>0.726761</td>\n",
       "      <td>0.611164</td>\n",
       "      <td>2.395489</td>\n",
       "      <td>0.368341</td>\n",
       "      <td>0.444352</td>\n",
       "      <td>0.579316</td>\n",
       "      <td>0.727370</td>\n",
       "      <td>0.610038</td>\n",
       "      <td>2.361692</td>\n",
       "      <td>0.365173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:19:52</td>\n",
       "      <td>40.025 sec</td>\n",
       "      <td>388.0</td>\n",
       "      <td>0.444547</td>\n",
       "      <td>0.579539</td>\n",
       "      <td>0.727076</td>\n",
       "      <td>0.611486</td>\n",
       "      <td>2.394118</td>\n",
       "      <td>0.358927</td>\n",
       "      <td>0.444196</td>\n",
       "      <td>0.578859</td>\n",
       "      <td>0.727526</td>\n",
       "      <td>0.610044</td>\n",
       "      <td>2.361692</td>\n",
       "      <td>0.361138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td></td>\n",
       "      <td>2019-09-18 12:20:01</td>\n",
       "      <td>49.604 sec</td>\n",
       "      <td>493.0</td>\n",
       "      <td>0.444366</td>\n",
       "      <td>0.579090</td>\n",
       "      <td>0.727467</td>\n",
       "      <td>0.611809</td>\n",
       "      <td>2.401028</td>\n",
       "      <td>0.373890</td>\n",
       "      <td>0.444157</td>\n",
       "      <td>0.578717</td>\n",
       "      <td>0.727464</td>\n",
       "      <td>0.610060</td>\n",
       "      <td>2.369794</td>\n",
       "      <td>0.365232</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                timestamp    duration  number_of_trees  training_rmse  \\\n",
       "0     2019-09-18 12:19:12   0.012 sec              0.0       0.482399   \n",
       "1     2019-09-18 12:19:13   1.333 sec              1.0       0.481705   \n",
       "2     2019-09-18 12:19:13   1.639 sec              2.0       0.481025   \n",
       "3     2019-09-18 12:19:14   1.841 sec              3.0       0.480356   \n",
       "4     2019-09-18 12:19:14   2.027 sec              4.0       0.479701   \n",
       "5     2019-09-18 12:19:14   2.209 sec              5.0       0.479057   \n",
       "6     2019-09-18 12:19:14   2.410 sec              6.0       0.478426   \n",
       "7     2019-09-18 12:19:14   2.603 sec              7.0       0.477805   \n",
       "8     2019-09-18 12:19:15   2.801 sec              8.0       0.477197   \n",
       "9     2019-09-18 12:19:15   2.995 sec              9.0       0.476599   \n",
       "10    2019-09-18 12:19:15   3.214 sec             10.0       0.476012   \n",
       "11    2019-09-18 12:19:15   3.437 sec             11.0       0.475437   \n",
       "12    2019-09-18 12:19:16   3.895 sec             12.0       0.474871   \n",
       "13    2019-09-18 12:19:16   4.114 sec             13.0       0.474316   \n",
       "14    2019-09-18 12:19:20   8.169 sec             59.0       0.457322   \n",
       "15    2019-09-18 12:19:27  14.730 sec            124.0       0.448565   \n",
       "16    2019-09-18 12:19:35  23.317 sec            210.0       0.445549   \n",
       "17    2019-09-18 12:19:44  32.124 sec            301.0       0.444802   \n",
       "18    2019-09-18 12:19:52  40.025 sec            388.0       0.444547   \n",
       "19    2019-09-18 12:20:01  49.604 sec            493.0       0.444366   \n",
       "\n",
       "    training_logloss  training_auc  training_pr_auc  training_lift  \\\n",
       "0           0.658154      0.500000         0.000000       1.000000   \n",
       "1           0.656718      0.721281         0.569402       2.289071   \n",
       "2           0.655313      0.721487         0.569547       2.289071   \n",
       "3           0.653936      0.721588         0.589096       2.289071   \n",
       "4           0.652590      0.722411         0.590812       2.288996   \n",
       "5           0.651270      0.722372         0.590803       2.288996   \n",
       "6           0.649978      0.722452         0.590815       2.288996   \n",
       "7           0.648711      0.722552         0.590854       2.292176   \n",
       "8           0.647471      0.722574         0.590964       2.292660   \n",
       "9           0.646254      0.722710         0.591004       2.292660   \n",
       "10          0.645063      0.723008         0.597556       2.354895   \n",
       "11          0.643896      0.723054         0.597600       2.354895   \n",
       "12          0.642751      0.723130         0.597548       2.359000   \n",
       "13          0.641629      0.723152         0.597535       2.359000   \n",
       "14          0.607547      0.724917         0.603430       2.378956   \n",
       "15          0.589201      0.725773         0.608407       2.374672   \n",
       "16          0.582153      0.726333         0.609730       2.382533   \n",
       "17          0.580213      0.726761         0.611164       2.395489   \n",
       "18          0.579539      0.727076         0.611486       2.394118   \n",
       "19          0.579090      0.727467         0.611809       2.401028   \n",
       "\n",
       "    training_classification_error  validation_rmse  validation_logloss  \\\n",
       "0                        0.631497         0.482083            0.657531   \n",
       "1                        0.359561         0.481386            0.656089   \n",
       "2                        0.359561         0.480702            0.654678   \n",
       "3                        0.359561         0.480030            0.653294   \n",
       "4                        0.376733         0.479373            0.651944   \n",
       "5                        0.371531         0.478725            0.650617   \n",
       "6                        0.371149         0.478091            0.649321   \n",
       "7                        0.376284         0.477467            0.648047   \n",
       "8                        0.375908         0.476856            0.646803   \n",
       "9                        0.376370         0.476255            0.645579   \n",
       "10                       0.369474         0.475665            0.644383   \n",
       "11                       0.363569         0.475087            0.643211   \n",
       "12                       0.362884         0.474520            0.642063   \n",
       "13                       0.363209         0.473962            0.640935   \n",
       "14                       0.387066         0.456853            0.606634   \n",
       "15                       0.378242         0.448037            0.588163   \n",
       "16                       0.363792         0.445033            0.581121   \n",
       "17                       0.368341         0.444352            0.579316   \n",
       "18                       0.358927         0.444196            0.578859   \n",
       "19                       0.373890         0.444157            0.578717   \n",
       "\n",
       "    validation_auc  validation_pr_auc  validation_lift  \\\n",
       "0         0.500000           0.000000         1.000000   \n",
       "1         0.723266           0.570940         2.291923   \n",
       "2         0.723424           0.571011         2.291923   \n",
       "3         0.723559           0.589766         2.291923   \n",
       "4         0.724206           0.591456         2.293831   \n",
       "5         0.724238           0.591521         2.293831   \n",
       "6         0.724261           0.591450         2.293831   \n",
       "7         0.724298           0.591508         2.304677   \n",
       "8         0.724324           0.591713         2.304677   \n",
       "9         0.724557           0.591808         2.304677   \n",
       "10        0.724929           0.598834         2.341662   \n",
       "11        0.724965           0.598863         2.341662   \n",
       "12        0.725024           0.598955         2.343449   \n",
       "13        0.725071           0.598807         2.343449   \n",
       "14        0.726611           0.604619         2.352443   \n",
       "15        0.727045           0.608089         2.351360   \n",
       "16        0.727299           0.609588         2.353590   \n",
       "17        0.727370           0.610038         2.361692   \n",
       "18        0.727526           0.610044         2.361692   \n",
       "19        0.727464           0.610060         2.369794   \n",
       "\n",
       "    validation_classification_error  \n",
       "0                          0.632653  \n",
       "1                          0.357370  \n",
       "2                          0.357370  \n",
       "3                          0.357370  \n",
       "4                          0.356358  \n",
       "5                          0.356626  \n",
       "6                          0.356358  \n",
       "7                          0.356313  \n",
       "8                          0.356313  \n",
       "9                          0.356313  \n",
       "10                         0.357088  \n",
       "11                         0.361510  \n",
       "12                         0.361286  \n",
       "13                         0.361629  \n",
       "14                         0.367764  \n",
       "15                         0.360721  \n",
       "16                         0.365873  \n",
       "17                         0.365173  \n",
       "18                         0.361138  \n",
       "19                         0.365232  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "See the whole table with table.as_data_frame()\n",
      "\n",
      "Variable Importances: "
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>variable</th>\n",
       "      <th>relative_importance</th>\n",
       "      <th>scaled_importance</th>\n",
       "      <th>percentage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>facenet_euclidean_l2</td>\n",
       "      <td>261705.156250</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.468295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>facenet_cosine</td>\n",
       "      <td>162436.656250</td>\n",
       "      <td>0.620686</td>\n",
       "      <td>0.290664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>vgg_euclidean_l2</td>\n",
       "      <td>73451.710938</td>\n",
       "      <td>0.280666</td>\n",
       "      <td>0.131434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>vgg_cosine</td>\n",
       "      <td>34810.042969</td>\n",
       "      <td>0.133012</td>\n",
       "      <td>0.062289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>openface_cosine</td>\n",
       "      <td>15869.265625</td>\n",
       "      <td>0.060638</td>\n",
       "      <td>0.028396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>openface_euclidean_l2</td>\n",
       "      <td>10574.509766</td>\n",
       "      <td>0.040406</td>\n",
       "      <td>0.018922</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                variable  relative_importance  scaled_importance  percentage\n",
       "0   facenet_euclidean_l2        261705.156250           1.000000    0.468295\n",
       "1         facenet_cosine        162436.656250           0.620686    0.290664\n",
       "2       vgg_euclidean_l2         73451.710938           0.280666    0.131434\n",
       "3             vgg_cosine         34810.042969           0.133012    0.062289\n",
       "4        openface_cosine         15869.265625           0.060638    0.028396\n",
       "5  openface_euclidean_l2         10574.509766           0.040406    0.018922"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_perf = model.model_performance(validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7225285769297011"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_perf.auc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6940562559467174"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_perf.accuracy()[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_perf = model.model_performance(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.727593424555288"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_perf.auc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6983918999404407"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_perf.accuracy()[0][1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gbm prediction progress: |████████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "predictions = model.predict(test_data = validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th style=\"text-align: right;\">  predict</th><th style=\"text-align: right;\">      p0</th><th style=\"text-align: right;\">       p1</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\">        1</td><td style=\"text-align: right;\">0.462883</td><td style=\"text-align: right;\">0.537117 </td></tr>\n",
       "<tr><td style=\"text-align: right;\">        0</td><td style=\"text-align: right;\">0.815415</td><td style=\"text-align: right;\">0.184585 </td></tr>\n",
       "<tr><td style=\"text-align: right;\">        1</td><td style=\"text-align: right;\">0.48551 </td><td style=\"text-align: right;\">0.51449  </td></tr>\n",
       "<tr><td style=\"text-align: right;\">        1</td><td style=\"text-align: right;\">0.322137</td><td style=\"text-align: right;\">0.677863 </td></tr>\n",
       "<tr><td style=\"text-align: right;\">        1</td><td style=\"text-align: right;\">0.4286  </td><td style=\"text-align: right;\">0.5714   </td></tr>\n",
       "<tr><td style=\"text-align: right;\">        0</td><td style=\"text-align: right;\">0.85153 </td><td style=\"text-align: right;\">0.14847  </td></tr>\n",
       "<tr><td style=\"text-align: right;\">        0</td><td style=\"text-align: right;\">0.909476</td><td style=\"text-align: right;\">0.0905237</td></tr>\n",
       "<tr><td style=\"text-align: right;\">        1</td><td style=\"text-align: right;\">0.336492</td><td style=\"text-align: right;\">0.663508 </td></tr>\n",
       "<tr><td style=\"text-align: right;\">        1</td><td style=\"text-align: right;\">0.631113</td><td style=\"text-align: right;\">0.368887 </td></tr>\n",
       "<tr><td style=\"text-align: right;\">        0</td><td style=\"text-align: right;\">0.857651</td><td style=\"text-align: right;\">0.142349 </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions_pd = predictions.as_data_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "predict    0.000000\n",
       "p0         0.886282\n",
       "p1         0.113718\n",
       "Name: 0, dtype: float64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions_pd.iloc[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Store model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "saved_model = h2o.save_model(model, path = \"\", force=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/outputs/sefik/kinship/GBM_Kinship'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saved_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load the best model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#you can directly load the best model by running the load_model command\n",
    "restored_model = h2o.load_model(saved_model)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}