experiments/02_BCI.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Experiment 02: BCI\n",
    "\n",
    "This experiment uses a Brain Computer Interface dataset. The purpose is to try and predict when the participant is paying attention. The dataset consists of recordings from a number of electrodes placed over the scalp.\n",
    "\n",
    "The details of the machine we used and the version of the libraries can be found in [experiment 01](01_airline.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "System version: 3.5.2 |Anaconda custom (64-bit)| (default, Jul  2 2016, 17:53:06) \n",
      "[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)]\n",
      "XGBoost version: 0.6\n",
      "LightGBM version: 0.2\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import sys\n",
    "import warnings\n",
    "from copy import deepcopy\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pkg_resources\n",
    "from IPython.display import SVG, display\n",
    "from bokeh.io import export_svgs\n",
    "from bokeh.io import show, output_notebook\n",
    "from bokeh.models import HoverTool\n",
    "from bokeh.models.sources import ColumnDataSource\n",
    "from bokeh.plotting import figure\n",
    "from libs.loaders import load_bci\n",
    "from libs.timer import Timer\n",
    "from libs.utils import get_number_processors\n",
    "from lightgbm import LGBMClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score, f1_score\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from tqdm import tqdm_notebook\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "print(\"System version: {}\".format(sys.version))\n",
    "print(\"XGBoost version: {}\".format(pkg_resources.get_distribution('xgboost').version))\n",
    "print(\"LightGBM version: {}\".format(pkg_resources.get_distribution('lightgbm').version))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"http://bokeh.pydata.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"0e4daf9b-11b6-4620-b356-b021b9e5b7a4\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(global) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof (window._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n",
       "    window._bokeh_onload_callbacks = [];\n",
       "    window._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "\n",
       "  \n",
       "  if (typeof (window._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    window._bokeh_timeout = Date.now() + 5000;\n",
       "    window._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    if (window.Bokeh !== undefined) {\n",
       "      var el = document.getElementById(\"0e4daf9b-11b6-4620-b356-b021b9e5b7a4\");\n",
       "      el.textContent = \"BokehJS \" + Bokeh.version + \" successfully loaded.\";\n",
       "    } else if (Date.now() < window._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      window._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n",
       "    }\n",
       "    finally {\n",
       "      delete window._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.info(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(js_urls, callback) {\n",
       "    window._bokeh_onload_callbacks.push(callback);\n",
       "    if (window._bokeh_is_loading > 0) {\n",
       "      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    window._bokeh_is_loading = js_urls.length;\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var s = document.createElement('script');\n",
       "      s.src = url;\n",
       "      s.async = false;\n",
       "      s.onreadystatechange = s.onload = function() {\n",
       "        window._bokeh_is_loading--;\n",
       "        if (window._bokeh_is_loading === 0) {\n",
       "          console.log(\"Bokeh: all BokehJS libraries loaded\");\n",
       "          run_callbacks()\n",
       "        }\n",
       "      };\n",
       "      s.onerror = function() {\n",
       "        console.warn(\"failed to load library \" + url);\n",
       "      };\n",
       "      console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "    }\n",
       "  };var element = document.getElementById(\"0e4daf9b-11b6-4620-b356-b021b9e5b7a4\");\n",
       "  if (element == null) {\n",
       "    console.log(\"Bokeh: ERROR: autoload.js configured with elementid '0e4daf9b-11b6-4620-b356-b021b9e5b7a4' but no matching script tag was found. \")\n",
       "    return false;\n",
       "  }\n",
       "\n",
       "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.6.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.6.min.js\"];\n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    \n",
       "    function(Bokeh) {\n",
       "      \n",
       "    },\n",
       "    \n",
       "    function(Bokeh) {\n",
       "      \n",
       "      document.getElementById(\"0e4daf9b-11b6-4620-b356-b021b9e5b7a4\").textContent = \"BokehJS is loading...\";\n",
       "    },\n",
       "    function(Bokeh) {\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.6.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.6.min.css\");\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.6.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.6.min.css\");\n",
       "    }\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if ((window.Bokeh !== undefined) || (force === true)) {\n",
       "      for (var i = 0; i < inline_js.length; i++) {\n",
       "        inline_js[i](window.Bokeh);\n",
       "      }if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < window._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!window._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      window._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"0e4daf9b-11b6-4620-b356-b021b9e5b7a4\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (window._bokeh_is_loading === 0) {\n",
       "    console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(js_urls, function() {\n",
       "      console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(this));"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "output_notebook()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## XGBoost vs LightGBM benchmark\n",
    "\n",
    "We will compare both libraries on speed and preformance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24\n"
     ]
    }
   ],
   "source": [
    "number_processors = get_number_processors()\n",
    "print(number_processors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Below we define the modeling pipelines for our two classifiers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "pipeline_steps = [('scale', StandardScaler())]\n",
    "continuous_pipeline = Pipeline(steps=pipeline_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "featurisers = [('continuous', continuous_pipeline)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "xgb_clf_pipeline = Pipeline(steps=[('features', FeatureUnion(featurisers)),\n",
    "                                          ('clf', XGBClassifier(max_depth=3, \n",
    "                                                                learning_rate=0.1, \n",
    "                                                                scale_pos_weight=2,\n",
    "                                                                n_estimators=100,\n",
    "                                                                gamma=0.1,\n",
    "                                                                min_child_weight=1,\n",
    "                                                                reg_lambda=1,\n",
    "                                                                subsample=1,\n",
    "                                                                nthread=number_processors\n",
    "                                                                ))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "xgb_hist_clf_pipeline = Pipeline(steps=[('features', FeatureUnion(featurisers)),\n",
    "                                          ('clf', XGBClassifier(max_depth=0, \n",
    "                                                                learning_rate=0.1, \n",
    "                                                                scale_pos_weight=2,\n",
    "                                                                n_estimators=100,\n",
    "                                                                gamma=0.1,\n",
    "                                                                min_child_weight=1,\n",
    "                                                                reg_lambda=1,\n",
    "                                                                subsample=1,\n",
    "                                                                max_leaves=2**3,\n",
    "                                                                grow_policy='lossguide',\n",
    "                                                                tree_method='hist',\n",
    "                                                                nthread=number_processors\n",
    "                                                                ))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "lgbm_clf_pipeline = Pipeline(steps=[('features', FeatureUnion(featurisers)),\n",
    "                                          ('clf', LGBMClassifier(num_leaves=2**3, \n",
    "                                                                learning_rate=0.1, \n",
    "                                                                scale_pos_weight=2,\n",
    "                                                                n_estimators=100,\n",
    "                                                                min_split_gain=0.1,\n",
    "                                                                min_child_weight=1,\n",
    "                                                                reg_lambda=1,\n",
    "                                                                subsample=1,\n",
    "                                                                nthread=number_processors\n",
    "                                                                ))])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "The dataset has been preprepared by extracting 800ms epochs from each channel. The data was then lowpass filtered at 18Hz and downsampled by a factor of 6. This results is a feature vector of\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:libs.loaders:MOUNT_POINT not found in environment. Defaulting to /fileshare\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.19 s, sys: 464 ms, total: 2.66 s\n",
      "Wall time: 19.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "X, y, X_test, y_test = load_bci()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "metrics_dict = {\n",
    "    'Accuracy': accuracy_score,\n",
    "    'Precision': precision_score,\n",
    "    'Recall': recall_score,\n",
    "    'AUC': roc_auc_score,\n",
    "    'F1': f1_score,\n",
    "}\n",
    "\n",
    "def classification_metrics(metrics, y_true, y_pred):\n",
    "    return {metric_name:metric(y_true, y_pred) for metric_name, metric in metrics.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict = dict()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer() as t_train:\n",
    "    xgb_clf_pipeline.fit(np.concatenate(X),np.concatenate(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer() as t_test:\n",
    "    y_pred = xgb_clf_pipeline.predict(np.concatenate(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['xgb']={\n",
    "    'train_time': t_train.interval,\n",
    "    'test_time': t_test.interval,\n",
    "    'performance': classification_metrics(metrics_dict, \n",
    "                                          np.concatenate(y_test), \n",
    "                                          y_pred) \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer() as t_train:\n",
    "    xgb_hist_clf_pipeline.fit(np.concatenate(X),np.concatenate(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer() as t_test:\n",
    "    y_pred = xgb_hist_clf_pipeline.predict(np.concatenate(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['xgb_hist']={\n",
    "    'train_time': t_train.interval,\n",
    "    'test_time': t_test.interval,\n",
    "    'performance': classification_metrics(metrics_dict, \n",
    "                                          np.concatenate(y_test), \n",
    "                                          y_pred) \n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer() as t_train:\n",
    "    lgbm_clf_pipeline.fit(np.concatenate(X),np.concatenate(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer() as t_test:\n",
    "    y_pred = lgbm_clf_pipeline.predict(np.concatenate(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['lgbm']={\n",
    "    'train_time': t_train.interval,\n",
    "    'test_time': t_test.interval,\n",
    "    'performance': classification_metrics(metrics_dict, \n",
    "                                          np.concatenate(y_test), \n",
    "                                          y_pred) \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"lgbm\": {\n",
      "        \"performance\": {\n",
      "            \"AUC\": 0.5348626373626374,\n",
      "            \"Accuracy\": 0.8802275008364001,\n",
      "            \"F1\": 0.13734939759036147,\n",
      "            \"Precision\": 0.5588235294117647,\n",
      "            \"Recall\": 0.0782967032967033\n",
      "        },\n",
      "        \"test_time\": 0.19060174500009452,\n",
      "        \"train_time\": 7.311910866001199\n",
      "    },\n",
      "    \"xgb\": {\n",
      "        \"performance\": {\n",
      "            \"AUC\": 0.5277435897435897,\n",
      "            \"Accuracy\": 0.8812311809969889,\n",
      "            \"F1\": 0.11027568922305765,\n",
      "            \"Precision\": 0.6285714285714286,\n",
      "            \"Recall\": 0.06043956043956044\n",
      "        },\n",
      "        \"test_time\": 0.2988008609991084,\n",
      "        \"train_time\": 11.515288856999177\n",
      "    },\n",
      "    \"xgb_hist\": {\n",
      "        \"performance\": {\n",
      "            \"AUC\": 0.536521978021978,\n",
      "            \"Accuracy\": 0.8810639009702241,\n",
      "            \"F1\": 0.14234016887816647,\n",
      "            \"Precision\": 0.5841584158415841,\n",
      "            \"Recall\": 0.08104395604395605\n",
      "        },\n",
      "        \"test_time\": 0.4702643149994401,\n",
      "        \"train_time\": 41.84106117400006\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Results\n",
    "print(json.dumps(results_dict, indent=4, sort_keys=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Concept Drift\n",
    "Concept drift can occur across many domains and can be caused by many factors such as sensor drift, seasonality, regime change, etc. One of the ways to combat this is by retraining. If the cause for the drift is observable then it can be incorporated into the model. In this specific instance the cause for the drift is an increase in the impedance in the electrodes, fatigue as well as a number of other factors. These are not observable in the dataset so we will have to retrain the model at regular intervals and observe the difference in performance. This constant retraining can be costly especially with large datasets. Below we will compare XGBoost and LightGBM in a naive retraining scenario."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def _predict(model, X_test, y_test, auc_list, interval_list):\n",
    "    with Timer() as t:\n",
    "        y_pred = model.predict_proba(X_test)\n",
    "        auc_list.append(roc_auc_score(y_test, y_pred[:, 1]))\n",
    "    interval_list.append(t.interval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def run_train_retrain_comparison(model, X, y, X_test, y_test):\n",
    "    retrain_model = deepcopy(model)\n",
    "    historical_X = deepcopy(X.tolist())\n",
    "    historical_y = deepcopy(y.tolist())\n",
    "    train_auc = list()\n",
    "    retrain_auc = list()\n",
    "    test_interval = list()\n",
    "    retrain_test_interval = list()\n",
    "    retrain_interval = list()\n",
    "    \n",
    "    for X_test_run, y_test_run in  tqdm_notebook(zip(X_test, y_test), total=len(y_test)):\n",
    "        prev_model = deepcopy(retrain_model)\n",
    "        \n",
    "        _predict(model, X_test_run, y_test_run, train_auc, test_interval)\n",
    "        _predict(retrain_model, X_test_run, y_test_run, retrain_auc, retrain_test_interval)\n",
    "        \n",
    "        historical_X.append(X_test_run)\n",
    "        historical_y.append(y_test_run)\n",
    "\n",
    "        new_train_X  = np.concatenate(historical_X)\n",
    "        with Timer() as t:\n",
    "            retrain_model.fit(new_train_X, np.concatenate(historical_y))\n",
    "        retrain_interval.append(t.interval)    \n",
    "        \n",
    "    return pd.DataFrame({\n",
    "        'Test interval':test_interval,\n",
    "        'Retrain Test interval':retrain_test_interval,\n",
    "        'Retrain interval':retrain_interval,\n",
    "        'Train AUC':train_auc,\n",
    "        'Retrain AUC':retrain_auc\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "df = run_train_retrain_comparison(xgb_clf_pipeline, X, y, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "retrain_results_dict = dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "retrain_results_dict['xgb'] = df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "df = run_train_retrain_comparison(lgbm_clf_pipeline, X, y, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "retrain_results_dict['lgbm'] = df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "data_cds = (retrain_results_dict['xgb'][['Train AUC', 'Retrain AUC']].rolling(window=10)\n",
    "                                                                     .mean()\n",
    "                                                                     .dropna()\n",
    "                                                                     .pipe(ColumnDataSource))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <div class=\"bk-plotdiv\" id=\"93587c6c-672a-4437-9a8e-7cb26784400e\"></div>\n",
       "    </div>\n",
       "<script type=\"text/javascript\">\n",
       "  \n",
       "  (function(global) {\n",
       "    function now() {\n",
       "      return new Date();\n",
       "    }\n",
       "  \n",
       "    var force = false;\n",
       "  \n",
       "    if (typeof (window._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n",
       "      window._bokeh_onload_callbacks = [];\n",
       "      window._bokeh_is_loading = undefined;\n",
       "    }\n",
       "  \n",
       "  \n",
       "    \n",
       "    if (typeof (window._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "      window._bokeh_timeout = Date.now() + 0;\n",
       "      window._bokeh_failed_load = false;\n",
       "    }\n",
       "  \n",
       "    var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "       \"<div style='background-color: #fdd'>\\n\"+\n",
       "       \"<p>\\n\"+\n",
       "       \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "       \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "       \"</p>\\n\"+\n",
       "       \"<ul>\\n\"+\n",
       "       \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "       \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "       \"</ul>\\n\"+\n",
       "       \"<code>\\n\"+\n",
       "       \"from bokeh.resources import INLINE\\n\"+\n",
       "       \"output_notebook(resources=INLINE)\\n\"+\n",
       "       \"</code>\\n\"+\n",
       "       \"</div>\"}};\n",
       "  \n",
       "    function display_loaded() {\n",
       "      if (window.Bokeh !== undefined) {\n",
       "        var el = document.getElementById(\"93587c6c-672a-4437-9a8e-7cb26784400e\");\n",
       "        el.textContent = \"BokehJS \" + Bokeh.version + \" successfully loaded.\";\n",
       "      } else if (Date.now() < window._bokeh_timeout) {\n",
       "        setTimeout(display_loaded, 100)\n",
       "      }\n",
       "    }\n",
       "  \n",
       "    function run_callbacks() {\n",
       "      try {\n",
       "        window._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n",
       "      }\n",
       "      finally {\n",
       "        delete window._bokeh_onload_callbacks\n",
       "      }\n",
       "      console.info(\"Bokeh: all callbacks have finished\");\n",
       "    }\n",
       "  \n",
       "    function load_libs(js_urls, callback) {\n",
       "      window._bokeh_onload_callbacks.push(callback);\n",
       "      if (window._bokeh_is_loading > 0) {\n",
       "        console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "        return null;\n",
       "      }\n",
       "      if (js_urls == null || js_urls.length === 0) {\n",
       "        run_callbacks();\n",
       "        return null;\n",
       "      }\n",
       "      console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "      window._bokeh_is_loading = js_urls.length;\n",
       "      for (var i = 0; i < js_urls.length; i++) {\n",
       "        var url = js_urls[i];\n",
       "        var s = document.createElement('script');\n",
       "        s.src = url;\n",
       "        s.async = false;\n",
       "        s.onreadystatechange = s.onload = function() {\n",
       "          window._bokeh_is_loading--;\n",
       "          if (window._bokeh_is_loading === 0) {\n",
       "            console.log(\"Bokeh: all BokehJS libraries loaded\");\n",
       "            run_callbacks()\n",
       "          }\n",
       "        };\n",
       "        s.onerror = function() {\n",
       "          console.warn(\"failed to load library \" + url);\n",
       "        };\n",
       "        console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "        document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "      }\n",
       "    };var element = document.getElementById(\"93587c6c-672a-4437-9a8e-7cb26784400e\");\n",
       "    if (element == null) {\n",
       "      console.log(\"Bokeh: ERROR: autoload.js configured with elementid '93587c6c-672a-4437-9a8e-7cb26784400e' but no matching script tag was found. \")\n",
       "      return false;\n",
       "    }\n",
       "  \n",
       "    var js_urls = [];\n",
       "  \n",
       "    var inline_js = [\n",
       "      function(Bokeh) {\n",
       "        (function() {\n",
       "          var fn = function() {\n",
       "            var docs_json = {\"ac6fbfa6-e70c-428f-82c8-ffeb8a67f01b\":{\"roots\":{\"references\":[{\"attributes\":{\"callback\":null},\"id\":\"932895b5-bed4-4ccf-9230-81f86a62481d\",\"type\":\"DataRange1d\"},{\"attributes\":{\"below\":[{\"id\":\"a4e5c59f-749e-4936-bb6c-4b8247211cff\",\"type\":\"LinearAxis\"}],\"left\":[{\"id\":\"a35ac959-9b70-4ab3-93f9-ace876fa7ba0\",\"type\":\"LinearAxis\"}],\"plot_height\":350,\"plot_width\":700,\"renderers\":[{\"id\":\"a4e5c59f-749e-4936-bb6c-4b8247211cff\",\"type\":\"LinearAxis\"},{\"id\":\"dd1cb37a-4339-4e9b-96e5-ff753d67251e\",\"type\":\"Grid\"},{\"id\":\"a35ac959-9b70-4ab3-93f9-ace876fa7ba0\",\"type\":\"LinearAxis\"},{\"id\":\"54339c6a-b2e5-4f3a-9b2d-ff6dd416f250\",\"type\":\"Grid\"},{\"id\":\"74581f59-1b44-41b5-9832-f9e7419ab919\",\"type\":\"BoxAnnotation\"},{\"id\":\"6d920ba9-1ba3-43af-95df-890174e67f5a\",\"type\":\"Legend\"},{\"id\":\"97b33cbe-be1d-461f-9326-6277f2bf49cc\",\"type\":\"GlyphRenderer\"},{\"id\":\"f96bb0f0-f093-4419-9a1b-78730562b6f0\",\"type\":\"GlyphRenderer\"},{\"id\":\"1c0cc4bc-5248-413b-98e1-b8a8bd37c0cf\",\"type\":\"GlyphRenderer\"},{\"id\":\"b8d6411b-1c15-4687-82ad-429d675247d8\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"d2301da3-f64d-45d9-b863-74cbfcfddc18\",\"type\":\"Title\"},\"tool_events\":{\"id\":\"48c7ae15-d8c2-44ea-855b-8fea7a4a0293\",\"type\":\"ToolEvents\"},\"toolbar\":{\"id\":\"77f8efc7-1fbe-4604-b0b4-763bb29a7d7f\",\"type\":\"Toolbar\"},\"x_range\":{\"id\":\"d6faaab5-b827-4756-af46-e94c2ef044f6\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"99588932-5ffc-44ae-a12c-f32e8b47941a\",\"type\":\"LinearScale\"},\"y_range\":{\"id\":\"932895b5-bed4-4ccf-9230-81f86a62481d\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"ae0b8d3b-502d-4a72-99d5-64be446b5b54\",\"type\":\"LinearScale\"}},\"id\":\"9ff48279-4361-4e7a-a6b3-2195e6af8191\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"plot\":{\"id\":\"9ff48279-4361-4e7a-a6b3-2195e6af8191\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"b6ce163c-9b72-4de3-ba38-4cb1d42fcf73\",\"type\":\"ResetTool\"},{\"attributes\":{\"plot\":{\"id\":\"9ff48279-4361-4e7a-a6b3-2195e6af8191\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"c2eb8bcf-c9f1-4227-bdfe-fffd829722c2\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"fill_color\":{\"value\":\"white\"},\"line_color\":{\"value\":\"#5975a4\"},\"line_width\":{\"value\":1.5},\"size\":{\"units\":\"screen\",\"value\":6},\"x\":{\"field\":\"index\"},\"y\":{\"field\":\"Train AUC\"}},\"id\":\"92a49f6a-7eb7-4c05-8784-aec4334f2272\",\"type\":\"Circle\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"lightgrey\"},\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":{\"value\":1.0},\"line_color\":{\"value\":\"black\"},\"line_dash\":[4,4],\"line_width\":{\"value\":2},\"plot\":null,\"render_mode\":\"css\",\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"74581f59-1b44-41b5-9832-f9e7419ab919\",\"type\":\"BoxAnnotation\"},{\"attributes\":{},\"id\":\"ebb849c0-3cea-4fc2-9652-a85fe2f616dd\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"8f8d877f-3930-418c-8e11-ce484f2e925b\",\"type\":\"BasicTicker\"},{\"attributes\":{\"formatter\":{\"id\":\"b9ed1217-a568-4f9d-84a8-289f2b7d7aa1\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"9ff48279-4361-4e7a-a6b3-2195e6af8191\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"ebb849c0-3cea-4fc2-9652-a85fe2f616dd\",\"type\":\"BasicTicker\"}},\"id\":\"a4e5c59f-749e-4936-bb6c-4b8247211cff\",\"type\":\"LinearAxis\"},{\"attributes\":{\"callback\":null,\"mode\":\"vline\",\"plot\":{\"id\":\"9ff48279-4361-4e7a-a6b3-2195e6af8191\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"renderers\":[{\"id\":\"97b33cbe-be1d-461f-9326-6277f2bf49cc\",\"type\":\"GlyphRenderer\"}],\"tooltips\":[[\"Train AUC\",\"@{Train AUC}{0.4f}\"],[\"Retrain AUC\",\"@{Retrain AUC}{0.4f}\"]]},\"id\":\"bee0291d-16eb-4cdd-8577-61ac084e526d\",\"type\":\"HoverTool\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"69a699dd-01e8-49ca-9aa3-aea9de7d2674\",\"type\":\"PanTool\"},{\"id\":\"c2eb8bcf-c9f1-4227-bdfe-fffd829722c2\",\"type\":\"WheelZoomTool\"},{\"id\":\"f825beb7-cebe-4796-b4c3-57a6c807adb2\",\"type\":\"BoxZoomTool\"},{\"id\":\"b6ce163c-9b72-4de3-ba38-4cb1d42fcf73\",\"type\":\"ResetTool\"},{\"id\":\"bee0291d-16eb-4cdd-8577-61ac084e526d\",\"type\":\"HoverTool\"}]},\"id\":\"77f8efc7-1fbe-4604-b0b4-763bb29a7d7f\",\"type\":\"Toolbar\"},{\"attributes\":{\"callback\":null},\"id\":\"d6faaab5-b827-4756-af46-e94c2ef044f6\",\"type\":\"DataRange1d\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"line_width\":{\"value\":1.5},\"size\":{\"units\":\"screen\",\"value\":6},\"x\":{\"field\":\"index\"},\"y\":{\"field\":\"Train AUC\"}},\"id\":\"63b15c9c-cb4c-4fb3-9da3-c7c33ba8ac87\",\"type\":\"Circle\"},{\"attributes\":{\"line_cap\":\"round\",\"line_color\":{\"value\":\"#5975a4\"},\"line_width\":{\"value\":6},\"x\":{\"field\":\"index\"},\"y\":{\"field\":\"Train AUC\"}},\"id\":\"6df473d1-dee3-47df-b691-5c5031384e21\",\"type\":\"Line\"},{\"attributes\":{},\"id\":\"e7e31a92-99a0-4389-9927-7d9d038350dc\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"ae0b8d3b-502d-4a72-99d5-64be446b5b54\",\"type\":\"LinearScale\"},{\"attributes\":{\"data_source\":{\"id\":\"73390f6e-1847-4d94-b4d7-7c7a5ae052bd\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"0c7f1bd1-a8e4-497f-9ea7-6c08cb869e21\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"840d1c9a-aea8-4300-90bb-c7e2da41e42e\",\"type\":\"Circle\"},\"selection_glyph\":null},\"id\":\"b8d6411b-1c15-4687-82ad-429d675247d8\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"plot\":null,\"text\":\"\"},\"id\":\"d2301da3-f64d-45d9-b863-74cbfcfddc18\",\"type\":\"Title\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"9ff48279-4361-4e7a-a6b3-2195e6af8191\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"8f8d877f-3930-418c-8e11-ce484f2e925b\",\"type\":\"BasicTicker\"}},\"id\":\"54339c6a-b2e5-4f3a-9b2d-ff6dd416f250\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"b9ed1217-a568-4f9d-84a8-289f2b7d7aa1\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"label\":{\"value\":\" Train AUC\"},\"renderers\":[{\"id\":\"97b33cbe-be1d-461f-9326-6277f2bf49cc\",\"type\":\"GlyphRenderer\"}]},\"id\":\"f0f4bd4f-c017-4626-adbe-17579c2fbde3\",\"type\":\"LegendItem\"},{\"attributes\":{\"fill_color\":{\"value\":\"white\"},\"line_color\":{\"value\":\"#a1bae3\"},\"line_width\":{\"value\":1.5},\"size\":{\"units\":\"screen\",\"value\":6},\"x\":{\"field\":\"index\"},\"y\":{\"field\":\"Retrain AUC\"}},\"id\":\"0c7f1bd1-a8e4-497f-9ea7-6c08cb869e21\",\"type\":\"Circle\"},{\"attributes\":{\"items\":[{\"id\":\"f0f4bd4f-c017-4626-adbe-17579c2fbde3\",\"type\":\"LegendItem\"},{\"id\":\"163a2504-8f64-4305-b345-41b2e92a896b\",\"type\":\"LegendItem\"}],\"plot\":{\"id\":\"9ff48279-4361-4e7a-a6b3-2195e6af8191\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"6d920ba9-1ba3-43af-95df-890174e67f5a\",\"type\":\"Legend\"},{\"attributes\":{\"overlay\":{\"id\":\"74581f59-1b44-41b5-9832-f9e7419ab919\",\"type\":\"BoxAnnotation\"},\"plot\":{\"id\":\"9ff48279-4361-4e7a-a6b3-2195e6af8191\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"f825beb7-cebe-4796-b4c3-57a6c807adb2\",\"type\":\"BoxZoomTool\"},{\"attributes\":{\"line_alpha\":{\"value\":0.1},\"line_cap\":\"round\",\"line_color\":{\"value\":\"#1f77b4\"},\"line_width\":{\"value\":6},\"x\":{\"field\":\"index\"},\"y\":{\"field\":\"Retrain AUC\"}},\"id\":\"268860d5-0d8d-471c-a12a-55b0fb1e6bbf\",\"type\":\"Line\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"line_width\":{\"value\":1.5},\"size\":{\"units\":\"screen\",\"value\":6},\"x\":{\"field\":\"index\"},\"y\":{\"field\":\"Retrain AUC\"}},\"id\":\"840d1c9a-aea8-4300-90bb-c7e2da41e42e\",\"type\":\"Circle\"},{\"attributes\":{\"label\":{\"value\":\" Retrain AUC\"},\"renderers\":[{\"id\":\"1c0cc4bc-5248-413b-98e1-b8a8bd37c0cf\",\"type\":\"GlyphRenderer\"}]},\"id\":\"163a2504-8f64-4305-b345-41b2e92a896b\",\"type\":\"LegendItem\"},{\"attributes\":{\"data_source\":{\"id\":\"73390f6e-1847-4d94-b4d7-7c7a5ae052bd\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"6df473d1-dee3-47df-b691-5c5031384e21\",\"type\":\"Line\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"386e29e9-4903-4e7e-91d2-90c6594969af\",\"type\":\"Line\"},\"selection_glyph\":null},\"id\":\"97b33cbe-be1d-461f-9326-6277f2bf49cc\",\"type\":\"GlyphRenderer\"},{\"attributes\":{},\"id\":\"99588932-5ffc-44ae-a12c-f32e8b47941a\",\"type\":\"LinearScale\"},{\"attributes\":{\"data_source\":{\"id\":\"73390f6e-1847-4d94-b4d7-7c7a5ae052bd\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"92a49f6a-7eb7-4c05-8784-aec4334f2272\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"63b15c9c-cb4c-4fb3-9da3-c7c33ba8ac87\",\"type\":\"Circle\"},\"selection_glyph\":null},\"id\":\"f96bb0f0-f093-4419-9a1b-78730562b6f0\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"plot\":{\"id\":\"9ff48279-4361-4e7a-a6b3-2195e6af8191\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"ebb849c0-3cea-4fc2-9652-a85fe2f616dd\",\"type\":\"BasicTicker\"}},\"id\":\"dd1cb37a-4339-4e9b-96e5-ff753d67251e\",\"type\":\"Grid\"},{\"attributes\":{\"data_source\":{\"id\":\"73390f6e-1847-4d94-b4d7-7c7a5ae052bd\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"3da3e408-75ee-423a-8fb8-61180b74b8b4\",\"type\":\"Line\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"268860d5-0d8d-471c-a12a-55b0fb1e6bbf\",\"type\":\"Line\"},\"selection_glyph\":null},\"id\":\"1c0cc4bc-5248-413b-98e1-b8a8bd37c0cf\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"plot\":{\"id\":\"9ff48279-4361-4e7a-a6b3-2195e6af8191\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"69a699dd-01e8-49ca-9aa3-aea9de7d2674\",\"type\":\"PanTool\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"Train AUC\",\"index\",\"Retrain AUC\"],\"data\":{\"Retrain AUC\":{\"__ndarray__\":\"VvppraUp6j+m9A79n87pP6qzQsR/suk/Anxk2y9r6T/dBmJBJ1DpPzp4PlSMKuk/Zg4kVIEl6T9y7C8v5CXpPz8bPY6J3+g/43Vw+CLg6D/iebbZzMHoPz8+CoEhveg/HTXqxcR76D+kFF3Hd4roPyWqoFLkceg/hPfcff566D9irGi+G2DoP/Z23AZNMeg/JteNJ45M6D8YWd83tFLoP28bX/lwdOg/vBESTY+e6D83NwbYBIHoPwgzo19cbOg/jwJUi5xa6D82OLXY3y/oPw==\",\"dtype\":\"float64\",\"shape\":[26]},\"Train AUC\":{\"__ndarray__\":\"dqghcxgp6j9A2Ldve/LpPzh28W2awek/CsBg8eaR6T8bmzB0/2vpP4DO4ZsoRuk/0c9AsmQp6T/qzXd3DQvpP8UT5eTQ7eg/gbeX2jXT6D/u2LxZ2rjoP6iYMPtymug/Ll1ovgN86D9QHMMVO2XoP3SA3pqEUeg/h2BQ9kU+6D+vaocMuyvoP5aX113rGOg/qoXnsjUG6D/6sBjC1vLnPw6Rih2Y3+c/qtP+8YbR5z82JfU9H8TnPxJGshoDuOc/+kSKwaGq5z/+p2FFU53nPw==\",\"dtype\":\"float64\",\"shape\":[26]},\"index\":[9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34]}},\"id\":\"73390f6e-1847-4d94-b4d7-7c7a5ae052bd\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"line_alpha\":{\"value\":0.1},\"line_cap\":\"round\",\"line_color\":{\"value\":\"#1f77b4\"},\"line_width\":{\"value\":6},\"x\":{\"field\":\"index\"},\"y\":{\"field\":\"Train AUC\"}},\"id\":\"386e29e9-4903-4e7e-91d2-90c6594969af\",\"type\":\"Line\"},{\"attributes\":{\"axis_label\":\"AUC\",\"formatter\":{\"id\":\"e7e31a92-99a0-4389-9927-7d9d038350dc\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"9ff48279-4361-4e7a-a6b3-2195e6af8191\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"8f8d877f-3930-418c-8e11-ce484f2e925b\",\"type\":\"BasicTicker\"}},\"id\":\"a35ac959-9b70-4ab3-93f9-ace876fa7ba0\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"48c7ae15-d8c2-44ea-855b-8fea7a4a0293\",\"type\":\"ToolEvents\"},{\"attributes\":{\"line_cap\":\"round\",\"line_color\":{\"value\":\"#a1bae3\"},\"line_width\":{\"value\":6},\"x\":{\"field\":\"index\"},\"y\":{\"field\":\"Retrain AUC\"}},\"id\":\"3da3e408-75ee-423a-8fb8-61180b74b8b4\",\"type\":\"Line\"}],\"root_ids\":[\"9ff48279-4361-4e7a-a6b3-2195e6af8191\"]},\"title\":\"Bokeh Application\",\"version\":\"0.12.6\"}};\n",
       "            var render_items = [{\"docid\":\"ac6fbfa6-e70c-428f-82c8-ffeb8a67f01b\",\"elementid\":\"93587c6c-672a-4437-9a8e-7cb26784400e\",\"modelid\":\"9ff48279-4361-4e7a-a6b3-2195e6af8191\"}];\n",
       "            \n",
       "            Bokeh.embed.embed_items(docs_json, render_items);\n",
       "          };\n",
       "          if (document.readyState != \"loading\") fn();\n",
       "          else document.addEventListener(\"DOMContentLoaded\", fn);\n",
       "        })();\n",
       "      },\n",
       "      function(Bokeh) {\n",
       "      }\n",
       "    ];\n",
       "  \n",
       "    function run_inline_js() {\n",
       "      \n",
       "      if ((window.Bokeh !== undefined) || (force === true)) {\n",
       "        for (var i = 0; i < inline_js.length; i++) {\n",
       "          inline_js[i](window.Bokeh);\n",
       "        }if (force === true) {\n",
       "          display_loaded();\n",
       "        }} else if (Date.now() < window._bokeh_timeout) {\n",
       "        setTimeout(run_inline_js, 100);\n",
       "      } else if (!window._bokeh_failed_load) {\n",
       "        console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "        window._bokeh_failed_load = true;\n",
       "      } else if (force !== true) {\n",
       "        var cell = $(document.getElementById(\"93587c6c-672a-4437-9a8e-7cb26784400e\")).parents('.cell').data().cell;\n",
       "        cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "      }\n",
       "  \n",
       "    }\n",
       "  \n",
       "    if (window._bokeh_is_loading === 0) {\n",
       "      console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "      run_inline_js();\n",
       "    } else {\n",
       "      load_libs(js_urls, function() {\n",
       "        console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "        run_inline_js();\n",
       "      });\n",
       "    }\n",
       "  }(this));\n",
       "</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# XGB Retrain Results\n",
    "p = figure(y_axis_label='AUC', plot_width=700, plot_height=350, tools=\"pan,wheel_zoom,box_zoom,reset\")\n",
    "l1 = p.line('index', 'Train AUC', legend=' Train AUC', line_color=\"#5975a4\", source=data_cds, line_width=6, line_cap=\"round\")\n",
    "p.circle('index', 'Train AUC', fill_color=\"white\", size=6, line_width=1.5, line_color=\"#5975a4\", source=data_cds)\n",
    "p.line('index', 'Retrain AUC', legend=' Retrain AUC', line_color=\"#a1bae3\", source=data_cds, line_width=6, line_cap=\"round\")\n",
    "p.circle('index', 'Retrain AUC',  fill_color=\"white\", size=6, line_width=1.5, line_color=\"#a1bae3\", source=data_cds)\n",
    "l1_hover = HoverTool(renderers=[l1], tooltips=[( 'Train AUC',  '@{Train AUC}{0.4f}' ), ( 'Retrain AUC',  '@{Retrain AUC}{0.4f}' )], mode='vline')\n",
    "p.add_tools(l1_hover)\n",
    "show(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"350\" style=\"width: 700px; height: 350px;\" version=\"1.1\" width=\"700\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs/><g><g/><g transform=\"scale(1,1) translate(0.5,0.5)\"><rect fill=\"#FFFFFF\" height=\"350\" stroke=\"none\" width=\"700\" x=\"0\" y=\"0\"/><rect fill=\"#ffffff\" height=\"350\" stroke=\"none\" width=\"700\" x=\"0\" y=\"0\"/><rect fill=\"#FFFFFF\" height=\"318.9925\" stroke=\"none\" width=\"609.38875\" x=\"60.61125000000001\" y=\"4\"/><rect fill=\"#ffffff\" height=\"318.9925\" stroke=\"none\" width=\"609.38875\" x=\"60.61125000000001\" y=\"4\"/><g><rect fill=\"none\" height=\"318.9925\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\" width=\"609.38875\" x=\"60.61125000000001\" y=\"4\"/></g><g><g><path d=\" M 110 323 L 110 4\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 221 323 L 221 4\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 332 323 L 332 4\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 443 323 L 443 4\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 554 323 L 554 4\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 664 323 L 664 4\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/></g><g><path d=\" M 61 301 L 670 301\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 61 228 L 670 228\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 61 155 L 670 155\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 61 83 L 670 83\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 61 10 L 670 10\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/></g><g><path fill=\"none\" stroke=\"none\"/><path d=\" M 88.31073863636371 18.74490718703646 L 110.4703295454546 43.02355597257247 L 132.6299204545455 64.75294663562727 L 154.78951136363642 85.95873385808136 L 176.94910227272734 102.80927224137668 L 199.1086931818182 119.63090747135539 L 221.26828409090913 132.4186678254764 L 243.42787500000006 145.90680603966393 L 265.58746590909095 158.90409772034172 L 287.74705681818193 170.73177072083172 L 309.9066477272728 182.44910907518442 L 332.06623863636366 195.96527458955688 L 354.22582954545464 209.4951034127689 L 376.3854204545455 219.62377366180726 L 398.5450113636364 228.38720974725356 L 420.70460227272736 236.94254312882367 L 442.8641931818182 245.18576815126426 L 465.0237840909092 253.54841384406018 L 487.18337500000007 261.86588395432 L 509.34296590909094 270.47730400410956 L 531.5025568181819 279.03263738567966 L 553.6621477272728 285.2862287395301 L 575.8217386363638 291.2455334414344 L 597.9813295454546 296.62905991127036 L 620.1409204545455 302.57732886372696 L 642.3005113636365 308.4928409090908\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-linecap=\"round\" stroke-miterlimit=\"10\" stroke-width=\"6\"/></g><g><path fill=\"none\" stroke=\"none\"/><path d=\" M 468.0237840909092 253.54841384406018 A 3 3 0 1 1 468.0237825909093 253.5454138445602\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 445.8641931818182 245.18576815126426 A 3 3 0 1 1 445.86419168181834 245.18276815176426\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 423.70460227272736 236.94254312882367 A 3 3 0 1 1 423.7046007727275 236.93954312932368\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 401.5450113636364 228.38720974725356 A 3 3 0 1 1 401.5450098636365 228.38420974775357\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 379.3854204545455 219.62377366180726 A 3 3 0 1 1 379.3854189545456 219.62077366230727\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 357.22582954545464 209.4951034127689 A 3 3 0 1 1 357.22582804545476 209.49210341326892\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 335.06623863636366 195.96527458955688 A 3 3 0 1 1 335.0662371363638 195.9622745900569\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 312.9066477272728 182.44910907518442 A 3 3 0 1 1 312.9066462272729 182.44610907568443\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 290.74705681818193 170.73177072083172 A 3 3 0 1 1 290.74705531818205 170.72877072133173\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 268.58746590909095 158.90409772034172 A 3 3 0 1 1 268.58746440909107 158.90109772084173\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 246.42787500000006 145.90680603966393 A 3 3 0 1 1 246.42787350000017 145.90380604016394\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 224.26828409090913 132.4186678254764 A 3 3 0 1 1 224.26828259090925 132.4156678259764\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 202.1086931818182 119.63090747135539 A 3 3 0 1 1 202.10869168181833 119.62790747185538\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 179.94910227272734 102.80927224137668 A 3 3 0 1 1 179.94910077272746 102.80627224187667\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 157.78951136363642 85.95873385808136 A 3 3 0 1 1 157.78950986363654 85.95573385858135\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 135.6299204545455 64.75294663562727 A 3 3 0 1 1 135.62991895454562 64.74994663612726\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 113.4703295454546 43.02355597257247 A 3 3 0 1 1 113.47032804545474 43.02055597307247\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 91.31073863636371 18.74490718703646 A 3 3 0 1 1 91.31073713636384 18.74190718753646\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 490.18337500000007 261.86588395432 A 3 3 0 1 1 490.1833735000002 261.86288395482\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 512.3429659090909 270.47730400410956 A 3 3 0 1 1 512.3429644090911 270.47430400460956\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 534.5025568181819 279.03263738567966 A 3 3 0 1 1 534.5025553181821 279.0296373861797\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 556.6621477272728 285.2862287395301 A 3 3 0 1 1 556.662146227273 285.2832287400301\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 578.8217386363638 291.2455334414344 A 3 3 0 1 1 578.8217371363639 291.2425334419344\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 600.9813295454546 296.62905991127036 A 3 3 0 1 1 600.9813280454548 296.62605991177037\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 623.1409204545455 302.57732886372696 A 3 3 0 1 1 623.1409189545457 302.57432886422697\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 645.3005113636365 308.4928409090908 A 3 3 0 1 1 645.3005098636366 308.4898409095908\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/></g><g><path fill=\"none\" stroke=\"none\"/><path d=\" M 88.31073863636371 18.499659090909063 L 110.4703295454546 58.96407373346938 L 132.6299204545455 71.46757785801992 L 154.78951136363642 103.1697733900105 L 176.94910227272734 115.187704538851 L 199.1086931818182 131.90528841689138 L 221.26828409090913 134.14716778714956 L 243.42787500000006 133.97550057351418 L 265.58746590909095 165.25175489874164 L 287.74705681818193 164.98534322469732 L 309.9066477272728 178.47155456199562 L 332.06623863636366 180.54732648197114 L 354.22582954545464 209.6044547784786 L 376.3854204545455 203.06982881135718 L 398.5450113636364 213.99522076484845 L 420.70460227272736 209.94877930059192 L 442.8641931818182 221.90097576793823 L 465.0237840909092 242.70949444927828 L 487.18337500000007 230.59340321324316 L 509.34296590909094 227.8599718167934 L 531.5025568181819 212.86186280512266 L 553.6621477272728 194.13787457506487 L 575.8217386363638 207.27041641815094 L 597.9813295454546 216.45407921963624 L 620.1409204545455 224.3446400749358 L 642.3005113636365 243.34364618806148\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-linecap=\"round\" stroke-miterlimit=\"10\" stroke-width=\"6\"/></g><g><path fill=\"none\" stroke=\"none\"/><path d=\" M 468.0237840909092 242.70949444927828 A 3 3 0 1 1 468.0237825909093 242.7064944497783\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 445.8641931818182 221.90097576793823 A 3 3 0 1 1 445.86419168181834 221.89797576843824\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 423.70460227272736 209.94877930059192 A 3 3 0 1 1 423.7046007727275 209.94577930109193\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 401.5450113636364 213.99522076484845 A 3 3 0 1 1 401.5450098636365 213.99222076534846\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 379.3854204545455 203.06982881135718 A 3 3 0 1 1 379.3854189545456 203.0668288118572\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 357.22582954545464 209.6044547784786 A 3 3 0 1 1 357.22582804545476 209.60145477897862\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 335.06623863636366 180.54732648197114 A 3 3 0 1 1 335.0662371363638 180.54432648247115\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 312.9066477272728 178.47155456199562 A 3 3 0 1 1 312.9066462272729 178.46855456249563\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 268.58746590909095 165.25175489874164 A 3 3 0 1 1 268.58746440909107 165.24875489924165\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 290.74705681818193 164.98534322469732 A 3 3 0 1 1 290.74705531818205 164.98234322519733\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 246.42787500000006 133.97550057351418 A 3 3 0 1 1 246.42787350000017 133.9725005740142\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 224.26828409090913 134.14716778714956 A 3 3 0 1 1 224.26828259090925 134.14416778764956\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 202.1086931818182 131.90528841689138 A 3 3 0 1 1 202.10869168181833 131.9022884173914\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 179.94910227272734 115.187704538851 A 3 3 0 1 1 179.94910077272746 115.18470453935099\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 157.78951136363642 103.1697733900105 A 3 3 0 1 1 157.78950986363654 103.1667733905105\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 135.6299204545455 71.46757785801992 A 3 3 0 1 1 135.62991895454562 71.46457785851992\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 113.4703295454546 58.96407373346938 A 3 3 0 1 1 113.47032804545474 58.96107373396938\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 91.31073863636371 18.499659090909063 A 3 3 0 1 1 91.31073713636384 18.49665909140906\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 490.18337500000007 230.59340321324316 A 3 3 0 1 1 490.1833735000002 230.59040321374317\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 512.3429659090909 227.8599718167934 A 3 3 0 1 1 512.3429644090911 227.8569718172934\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 534.5025568181819 212.86186280512266 A 3 3 0 1 1 534.5025553181821 212.85886280562266\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 556.6621477272728 194.13787457506487 A 3 3 0 1 1 556.662146227273 194.13487457556488\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 578.8217386363638 207.27041641815094 A 3 3 0 1 1 578.8217371363639 207.26741641865095\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 600.9813295454546 216.45407921963624 A 3 3 0 1 1 600.9813280454548 216.45107922013625\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 623.1409204545455 224.3446400749358 A 3 3 0 1 1 623.1409189545457 224.3416400754358\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 645.3005113636365 243.34364618806148 A 3 3 0 1 1 645.3005098636366 243.3406461885615\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/></g></g><g><g/><g><path d=\" M 527.328125 14 L 660 14 L 660 77 L 527.328125 77 L 527.328125 14 Z\" fill=\"#ffffff\" fill-opacity=\"0.95\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\" stroke-opacity=\"0.5\"/><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"10pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"start\" text-decoration=\"normal\" x=\"562.328125\" y=\"34\"> Train AUC</text><g><path d=\" M 537.328125 34 L 557.328125 34\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-linecap=\"round\" stroke-miterlimit=\"10\" stroke-width=\"6\"/></g><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"10pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"start\" text-decoration=\"normal\" x=\"562.328125\" y=\"57\"> Retrain AUC</text><g><path d=\" M 537.328125 57 L 557.328125 57\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-linecap=\"round\" stroke-miterlimit=\"10\" stroke-width=\"6\"/></g></g></g><g><g><path d=\" M 61 323 L 670 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 110 329 L 110 321\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 221 329 L 221 321\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 332 329 L 332 321\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 443 329 L 443 321\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 554 329 L 554 321\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 664 329 L 664 321\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 88 327 L 88 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 66 327 L 66 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 110 327 L 110 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 133 327 L 133 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 155 327 L 155 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 177 327 L 177 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 199 327 L 199 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 221 327 L 221 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 243 327 L 243 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 266 327 L 266 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 288 327 L 288 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 310 327 L 310 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 332 327 L 332 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 354 327 L 354 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 376 327 L 376 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 399 327 L 399 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 421 327 L 421 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 443 327 L 443 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 465 327 L 465 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 487 327 L 487 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 509 327 L 509 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 532 327 L 532 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 554 327 L 554 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 576 327 L 576 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 598 327 L 598 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 620 327 L 620 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 642 327 L 642 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 664 327 L 664 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><text dominant-baseline=\"hanging\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"110\" y=\"334\">10</text><text dominant-baseline=\"hanging\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"221\" y=\"334\">15</text><text dominant-baseline=\"hanging\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"332\" y=\"334\">20</text><text dominant-baseline=\"hanging\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"443\" y=\"334\">25</text><text dominant-baseline=\"hanging\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"554\" y=\"334\">30</text><text dominant-baseline=\"hanging\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"664\" y=\"334\">35</text><text dominant-baseline=\"hanging\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"10pt\" font-style=\"italic\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"365.3056250000001\" y=\"353.99999999999983\"/></g><g><path d=\" M 61 323 L 61 4\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 55 301 L 63 301\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 55 228 L 63 228\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 55 155 L 63 155\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 55 83 L 63 83\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 55 10 L 63 10\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 316 L 61 316\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 301 L 61 301\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 286 L 61 286\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 272 L 61 272\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 257 L 61 257\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 243 L 61 243\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 228 L 61 228\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 214 L 61 214\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 199 L 61 199\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 185 L 61 185\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 170 L 61 170\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 155 L 61 155\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 141 L 61 141\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 126 L 61 126\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 112 L 61 112\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 97 L 61 97\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 83 L 61 83\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 68 L 61 68\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 53 L 61 53\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 39 L 61 39\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 24 L 61 24\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 10 L 61 10\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"50\" y=\"301\">0.74</text><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"50\" y=\"228\">0.76</text><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"50\" y=\"155\">0.78</text><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"50\" y=\"83\">0.8</text><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"50\" y=\"10\">0.82</text><g transform=\"translate(17.73000000000001,163.49624999999992) rotate(-90,0,0)\"><text dominant-baseline=\"alphabetic\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"10pt\" font-style=\"italic\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"0\" y=\"0\">AUC</text><g transform=\"rotate(90,0,0) translate(-17.73000000000001,-163.49624999999992)\"/></g></g></g></g></g></svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# SVG: XGB Retrain Results \n",
    "# Save the plot and display it so that is is visible on github\n",
    "p.output_backend = \"svg\"\n",
    "export_svgs(p, filename=\"xgb_bci_train_retrain.svg\")\n",
    "display(SVG('xgb_bci_train_retrain.svg'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Overall we can see that retraining does seem to help a bit. Of course the retraining done here is naive with no consideration that later samples may also contain more noise. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "data_cds = (retrain_results_dict['lgbm'][['Train AUC', 'Retrain AUC']].rolling(window=10)\n",
    "                                                                      .mean()\n",
    "                                                                      .dropna()\n",
    "                                                                      .pipe(ColumnDataSource))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <div class=\"bk-plotdiv\" id=\"cc98270a-d7e4-48a7-a37c-fbaf0adc2d59\"></div>\n",
       "    </div>\n",
       "<script type=\"text/javascript\">\n",
       "  \n",
       "  (function(global) {\n",
       "    function now() {\n",
       "      return new Date();\n",
       "    }\n",
       "  \n",
       "    var force = false;\n",
       "  \n",
       "    if (typeof (window._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n",
       "      window._bokeh_onload_callbacks = [];\n",
       "      window._bokeh_is_loading = undefined;\n",
       "    }\n",
       "  \n",
       "  \n",
       "    \n",
       "    if (typeof (window._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "      window._bokeh_timeout = Date.now() + 0;\n",
       "      window._bokeh_failed_load = false;\n",
       "    }\n",
       "  \n",
       "    var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "       \"<div style='background-color: #fdd'>\\n\"+\n",
       "       \"<p>\\n\"+\n",
       "       \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "       \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "       \"</p>\\n\"+\n",
       "       \"<ul>\\n\"+\n",
       "       \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "       \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "       \"</ul>\\n\"+\n",
       "       \"<code>\\n\"+\n",
       "       \"from bokeh.resources import INLINE\\n\"+\n",
       "       \"output_notebook(resources=INLINE)\\n\"+\n",
       "       \"</code>\\n\"+\n",
       "       \"</div>\"}};\n",
       "  \n",
       "    function display_loaded() {\n",
       "      if (window.Bokeh !== undefined) {\n",
       "        var el = document.getElementById(\"cc98270a-d7e4-48a7-a37c-fbaf0adc2d59\");\n",
       "        el.textContent = \"BokehJS \" + Bokeh.version + \" successfully loaded.\";\n",
       "      } else if (Date.now() < window._bokeh_timeout) {\n",
       "        setTimeout(display_loaded, 100)\n",
       "      }\n",
       "    }\n",
       "  \n",
       "    function run_callbacks() {\n",
       "      try {\n",
       "        window._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n",
       "      }\n",
       "      finally {\n",
       "        delete window._bokeh_onload_callbacks\n",
       "      }\n",
       "      console.info(\"Bokeh: all callbacks have finished\");\n",
       "    }\n",
       "  \n",
       "    function load_libs(js_urls, callback) {\n",
       "      window._bokeh_onload_callbacks.push(callback);\n",
       "      if (window._bokeh_is_loading > 0) {\n",
       "        console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "        return null;\n",
       "      }\n",
       "      if (js_urls == null || js_urls.length === 0) {\n",
       "        run_callbacks();\n",
       "        return null;\n",
       "      }\n",
       "      console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "      window._bokeh_is_loading = js_urls.length;\n",
       "      for (var i = 0; i < js_urls.length; i++) {\n",
       "        var url = js_urls[i];\n",
       "        var s = document.createElement('script');\n",
       "        s.src = url;\n",
       "        s.async = false;\n",
       "        s.onreadystatechange = s.onload = function() {\n",
       "          window._bokeh_is_loading--;\n",
       "          if (window._bokeh_is_loading === 0) {\n",
       "            console.log(\"Bokeh: all BokehJS libraries loaded\");\n",
       "            run_callbacks()\n",
       "          }\n",
       "        };\n",
       "        s.onerror = function() {\n",
       "          console.warn(\"failed to load library \" + url);\n",
       "        };\n",
       "        console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "        document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "      }\n",
       "    };var element = document.getElementById(\"cc98270a-d7e4-48a7-a37c-fbaf0adc2d59\");\n",
       "    if (element == null) {\n",
       "      console.log(\"Bokeh: ERROR: autoload.js configured with elementid 'cc98270a-d7e4-48a7-a37c-fbaf0adc2d59' but no matching script tag was found. \")\n",
       "      return false;\n",
       "    }\n",
       "  \n",
       "    var js_urls = [];\n",
       "  \n",
       "    var inline_js = [\n",
       "      function(Bokeh) {\n",
       "        (function() {\n",
       "          var fn = function() {\n",
       "            var docs_json = {\"0be2b436-a033-483a-bd8d-c22215438d8d\":{\"roots\":{\"references\":[{\"attributes\":{\"data_source\":{\"id\":\"30e24beb-cf6a-4bb9-a50e-5c6c06b588c6\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"16f79598-883b-45d2-95aa-ac2990fc38bb\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"c8419b62-990e-4f42-aa64-e5390f46685b\",\"type\":\"Circle\"},\"selection_glyph\":null},\"id\":\"24ada69b-7a20-4d2a-81e7-f7062228a8ad\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"formatter\":{\"id\":\"afc6d79f-095a-4a54-848c-c903bd8c8bc5\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"4ea59fbb-22fe-4f19-8fd3-861e0e77cc62\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"24bb6b22-fccc-4751-a279-7adeb7c9521e\",\"type\":\"BasicTicker\"}},\"id\":\"b5fe49cf-f0c8-4080-a419-c8f2d2e64271\",\"type\":\"LinearAxis\"},{\"attributes\":{\"line_alpha\":{\"value\":0.1},\"line_cap\":\"round\",\"line_color\":{\"value\":\"#1f77b4\"},\"line_width\":{\"value\":6},\"x\":{\"field\":\"index\"},\"y\":{\"field\":\"Train AUC\"}},\"id\":\"696322e0-a7ff-486e-b04d-8fe889d494d5\",\"type\":\"Line\"},{\"attributes\":{\"items\":[{\"id\":\"a0e37cae-fbbc-4552-8257-8f30d24c1e2c\",\"type\":\"LegendItem\"},{\"id\":\"0dc96401-cda8-48c2-a6cd-a3a679811a84\",\"type\":\"LegendItem\"}],\"plot\":{\"id\":\"4ea59fbb-22fe-4f19-8fd3-861e0e77cc62\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"09e0c9c1-ec3b-4676-8d70-da8445ecf697\",\"type\":\"Legend\"},{\"attributes\":{\"data_source\":{\"id\":\"30e24beb-cf6a-4bb9-a50e-5c6c06b588c6\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"4c2c3540-0654-4a18-809a-57461919f1e1\",\"type\":\"Line\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"696322e0-a7ff-486e-b04d-8fe889d494d5\",\"type\":\"Line\"},\"selection_glyph\":null},\"id\":\"76562eac-37c6-445b-b3cc-dbc0140abf24\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"fill_color\":{\"value\":\"white\"},\"line_color\":{\"value\":\"#a1bae3\"},\"line_width\":{\"value\":1.5},\"size\":{\"units\":\"screen\",\"value\":6},\"x\":{\"field\":\"index\"},\"y\":{\"field\":\"Retrain AUC\"}},\"id\":\"16f79598-883b-45d2-95aa-ac2990fc38bb\",\"type\":\"Circle\"},{\"attributes\":{\"fill_color\":{\"value\":\"white\"},\"line_color\":{\"value\":\"#5975a4\"},\"line_width\":{\"value\":1.5},\"size\":{\"units\":\"screen\",\"value\":6},\"x\":{\"field\":\"index\"},\"y\":{\"field\":\"Train AUC\"}},\"id\":\"f749c8a1-ae0c-4fd4-9097-c61b4a06f4c2\",\"type\":\"Circle\"},{\"attributes\":{\"below\":[{\"id\":\"b5fe49cf-f0c8-4080-a419-c8f2d2e64271\",\"type\":\"LinearAxis\"}],\"left\":[{\"id\":\"ab26f9c9-8305-4460-ad7e-71242189cf24\",\"type\":\"LinearAxis\"}],\"plot_height\":350,\"plot_width\":700,\"renderers\":[{\"id\":\"b5fe49cf-f0c8-4080-a419-c8f2d2e64271\",\"type\":\"LinearAxis\"},{\"id\":\"12d1048b-e56e-4d2c-b8f0-c85186624360\",\"type\":\"Grid\"},{\"id\":\"ab26f9c9-8305-4460-ad7e-71242189cf24\",\"type\":\"LinearAxis\"},{\"id\":\"265fc3d4-e318-4700-a931-4983c9d5b253\",\"type\":\"Grid\"},{\"id\":\"f5f69278-1876-4bbe-b905-f033ae1152bd\",\"type\":\"BoxAnnotation\"},{\"id\":\"09e0c9c1-ec3b-4676-8d70-da8445ecf697\",\"type\":\"Legend\"},{\"id\":\"76562eac-37c6-445b-b3cc-dbc0140abf24\",\"type\":\"GlyphRenderer\"},{\"id\":\"9312cc0a-9a36-4582-96cf-45535c4a3515\",\"type\":\"GlyphRenderer\"},{\"id\":\"f86c3dc5-09f5-42c2-9f8d-6b1058505fad\",\"type\":\"GlyphRenderer\"},{\"id\":\"24ada69b-7a20-4d2a-81e7-f7062228a8ad\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"667a91e4-bdc0-4c41-9cc9-21b8b007dfdb\",\"type\":\"Title\"},\"tool_events\":{\"id\":\"4bcce06f-0135-4ec8-b55d-4acb61b31b10\",\"type\":\"ToolEvents\"},\"toolbar\":{\"id\":\"6af12aec-d57a-40f9-8b72-62799e7c18c9\",\"type\":\"Toolbar\"},\"x_range\":{\"id\":\"440efc3c-15de-4f3d-90bf-b05730720178\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"f3b86823-6232-4fad-94bf-0b9be54540d0\",\"type\":\"LinearScale\"},\"y_range\":{\"id\":\"72ea81cd-a3ce-458c-9456-749b71ccf85e\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"33b8aac6-8176-4d2b-9eaa-f1bbd478fb31\",\"type\":\"LinearScale\"}},\"id\":\"4ea59fbb-22fe-4f19-8fd3-861e0e77cc62\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"afc6d79f-095a-4a54-848c-c903bd8c8bc5\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"line_width\":{\"value\":1.5},\"size\":{\"units\":\"screen\",\"value\":6},\"x\":{\"field\":\"index\"},\"y\":{\"field\":\"Retrain AUC\"}},\"id\":\"c8419b62-990e-4f42-aa64-e5390f46685b\",\"type\":\"Circle\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"Train AUC\",\"index\",\"Retrain AUC\"],\"data\":{\"Retrain AUC\":{\"__ndarray__\":\"9jytkBk56j/47VbF/RXqP8uqOtoa3+k/kMi54gR+6T9Vd4vO5FfpP1M4wCCJMuk/vYA5F80h6T99mFW1sCXpP766Sjxauug/OEWmif7P6D+IYRkDd9LoP22Jfi6bo+g/Nk3B2wSC6D/ZcCjzrpfoP1vI/Me+iug/1q6XqQuM6D8+rhGDKoHoP2olKzoGN+g/FJcy+ZpS6D+Jz9lt9ULoP1O/B7aNWug//ePR5YKF6D8WY9I98WvoP0L3UpIijug/HtUwcABs6D9zMTDT0EXoPw==\",\"dtype\":\"float64\",\"shape\":[26]},\"Train AUC\":{\"__ndarray__\":\"I7IwwGIp6j84hl3fDfrpP7bIwPKT6Ok/X75z9hGV6T+3Gt9H7WrpP+JW5TZKIek/WaO6ar0N6T+ivENJ7uroP8LcK1t8r+g/AUyuwsWc6D81jLH2BaDoP4pHs2wrnOg/WEzgcRp06D8DN97LD4roP0YIucTUYug/Yic6mRJp6D8Ja0RFdljoP/JPvBYiSOg/LJA6c4dA6D8jUQQKeSPoP8H1rrywJOg/Arqm3goi6D8/mEaP1fjnP8tcCU5Lyuc/1Q6CEkfP5z+nceJLEJbnPw==\",\"dtype\":\"float64\",\"shape\":[26]},\"index\":[9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34]}},\"id\":\"30e24beb-cf6a-4bb9-a50e-5c6c06b588c6\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"callback\":null},\"id\":\"72ea81cd-a3ce-458c-9456-749b71ccf85e\",\"type\":\"DataRange1d\"},{\"attributes\":{\"plot\":{\"id\":\"4ea59fbb-22fe-4f19-8fd3-861e0e77cc62\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"24bb6b22-fccc-4751-a279-7adeb7c9521e\",\"type\":\"BasicTicker\"}},\"id\":\"12d1048b-e56e-4d2c-b8f0-c85186624360\",\"type\":\"Grid\"},{\"attributes\":{\"axis_label\":\"AUC\",\"formatter\":{\"id\":\"26f3b1e1-02cc-4794-8a33-11f85afeb2c2\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"4ea59fbb-22fe-4f19-8fd3-861e0e77cc62\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"1d7f1568-c054-4b70-9ee7-3cdccafb3715\",\"type\":\"BasicTicker\"}},\"id\":\"ab26f9c9-8305-4460-ad7e-71242189cf24\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"33b8aac6-8176-4d2b-9eaa-f1bbd478fb31\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1d7f1568-c054-4b70-9ee7-3cdccafb3715\",\"type\":\"BasicTicker\"},{\"attributes\":{\"plot\":null,\"text\":\"\"},\"id\":\"667a91e4-bdc0-4c41-9cc9-21b8b007dfdb\",\"type\":\"Title\"},{\"attributes\":{\"label\":{\"value\":\" Retrain AUC\"},\"renderers\":[{\"id\":\"f86c3dc5-09f5-42c2-9f8d-6b1058505fad\",\"type\":\"GlyphRenderer\"}]},\"id\":\"0dc96401-cda8-48c2-a6cd-a3a679811a84\",\"type\":\"LegendItem\"},{\"attributes\":{\"line_alpha\":{\"value\":0.1},\"line_cap\":\"round\",\"line_color\":{\"value\":\"#1f77b4\"},\"line_width\":{\"value\":6},\"x\":{\"field\":\"index\"},\"y\":{\"field\":\"Retrain AUC\"}},\"id\":\"c6c65893-7d9e-4ac6-abe1-dfa664e1e1d1\",\"type\":\"Line\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"line_width\":{\"value\":1.5},\"size\":{\"units\":\"screen\",\"value\":6},\"x\":{\"field\":\"index\"},\"y\":{\"field\":\"Train AUC\"}},\"id\":\"e9f402ce-d196-47fa-a6b7-4461ef379245\",\"type\":\"Circle\"},{\"attributes\":{\"data_source\":{\"id\":\"30e24beb-cf6a-4bb9-a50e-5c6c06b588c6\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"f6fe8afb-f730-4045-b39f-5ba05bc8be9e\",\"type\":\"Line\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"c6c65893-7d9e-4ac6-abe1-dfa664e1e1d1\",\"type\":\"Line\"},\"selection_glyph\":null},\"id\":\"f86c3dc5-09f5-42c2-9f8d-6b1058505fad\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"line_cap\":\"round\",\"line_color\":{\"value\":\"#a2cfae\"},\"line_width\":{\"value\":6},\"x\":{\"field\":\"index\"},\"y\":{\"field\":\"Retrain AUC\"}},\"id\":\"f6fe8afb-f730-4045-b39f-5ba05bc8be9e\",\"type\":\"Line\"},{\"attributes\":{\"plot\":{\"id\":\"4ea59fbb-22fe-4f19-8fd3-861e0e77cc62\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"b11b330c-9969-4cc8-a89a-7663de5a0766\",\"type\":\"PanTool\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"4ea59fbb-22fe-4f19-8fd3-861e0e77cc62\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"1d7f1568-c054-4b70-9ee7-3cdccafb3715\",\"type\":\"BasicTicker\"}},\"id\":\"265fc3d4-e318-4700-a931-4983c9d5b253\",\"type\":\"Grid\"},{\"attributes\":{\"plot\":{\"id\":\"4ea59fbb-22fe-4f19-8fd3-861e0e77cc62\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"8b7da891-315d-4aa2-a9ca-5c41fc0f5c8d\",\"type\":\"WheelZoomTool\"},{\"attributes\":{},\"id\":\"26f3b1e1-02cc-4794-8a33-11f85afeb2c2\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"callback\":null},\"id\":\"440efc3c-15de-4f3d-90bf-b05730720178\",\"type\":\"DataRange1d\"},{\"attributes\":{\"overlay\":{\"id\":\"f5f69278-1876-4bbe-b905-f033ae1152bd\",\"type\":\"BoxAnnotation\"},\"plot\":{\"id\":\"4ea59fbb-22fe-4f19-8fd3-861e0e77cc62\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"8369685b-fa6c-440d-9450-198dd73a9a07\",\"type\":\"BoxZoomTool\"},{\"attributes\":{\"line_cap\":\"round\",\"line_color\":{\"value\":\"#5f9e6f\"},\"line_width\":{\"value\":6},\"x\":{\"field\":\"index\"},\"y\":{\"field\":\"Train AUC\"}},\"id\":\"4c2c3540-0654-4a18-809a-57461919f1e1\",\"type\":\"Line\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"b11b330c-9969-4cc8-a89a-7663de5a0766\",\"type\":\"PanTool\"},{\"id\":\"8b7da891-315d-4aa2-a9ca-5c41fc0f5c8d\",\"type\":\"WheelZoomTool\"},{\"id\":\"8369685b-fa6c-440d-9450-198dd73a9a07\",\"type\":\"BoxZoomTool\"},{\"id\":\"cfa1db3f-aff6-4540-8a33-3056096ea467\",\"type\":\"ResetTool\"},{\"id\":\"90a6c38e-2fcc-4c56-8087-f839a1c83803\",\"type\":\"HoverTool\"}]},\"id\":\"6af12aec-d57a-40f9-8b72-62799e7c18c9\",\"type\":\"Toolbar\"},{\"attributes\":{\"callback\":null,\"mode\":\"vline\",\"plot\":{\"id\":\"4ea59fbb-22fe-4f19-8fd3-861e0e77cc62\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"renderers\":[{\"id\":\"76562eac-37c6-445b-b3cc-dbc0140abf24\",\"type\":\"GlyphRenderer\"}],\"tooltips\":[[\"Train AUC\",\"@{Train AUC}{0.4f}\"],[\"Retrain AUC\",\"@{Retrain AUC}{0.4f}\"]]},\"id\":\"90a6c38e-2fcc-4c56-8087-f839a1c83803\",\"type\":\"HoverTool\"},{\"attributes\":{\"label\":{\"value\":\" Train AUC\"},\"renderers\":[{\"id\":\"76562eac-37c6-445b-b3cc-dbc0140abf24\",\"type\":\"GlyphRenderer\"}]},\"id\":\"a0e37cae-fbbc-4552-8257-8f30d24c1e2c\",\"type\":\"LegendItem\"},{\"attributes\":{\"plot\":{\"id\":\"4ea59fbb-22fe-4f19-8fd3-861e0e77cc62\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"cfa1db3f-aff6-4540-8a33-3056096ea467\",\"type\":\"ResetTool\"},{\"attributes\":{},\"id\":\"f3b86823-6232-4fad-94bf-0b9be54540d0\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"4bcce06f-0135-4ec8-b55d-4acb61b31b10\",\"type\":\"ToolEvents\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"lightgrey\"},\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":{\"value\":1.0},\"line_color\":{\"value\":\"black\"},\"line_dash\":[4,4],\"line_width\":{\"value\":2},\"plot\":null,\"render_mode\":\"css\",\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"f5f69278-1876-4bbe-b905-f033ae1152bd\",\"type\":\"BoxAnnotation\"},{\"attributes\":{},\"id\":\"24bb6b22-fccc-4751-a279-7adeb7c9521e\",\"type\":\"BasicTicker\"},{\"attributes\":{\"data_source\":{\"id\":\"30e24beb-cf6a-4bb9-a50e-5c6c06b588c6\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"f749c8a1-ae0c-4fd4-9097-c61b4a06f4c2\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"e9f402ce-d196-47fa-a6b7-4461ef379245\",\"type\":\"Circle\"},\"selection_glyph\":null},\"id\":\"9312cc0a-9a36-4582-96cf-45535c4a3515\",\"type\":\"GlyphRenderer\"}],\"root_ids\":[\"4ea59fbb-22fe-4f19-8fd3-861e0e77cc62\"]},\"title\":\"Bokeh Application\",\"version\":\"0.12.6\"}};\n",
       "            var render_items = [{\"docid\":\"0be2b436-a033-483a-bd8d-c22215438d8d\",\"elementid\":\"cc98270a-d7e4-48a7-a37c-fbaf0adc2d59\",\"modelid\":\"4ea59fbb-22fe-4f19-8fd3-861e0e77cc62\"}];\n",
       "            \n",
       "            Bokeh.embed.embed_items(docs_json, render_items);\n",
       "          };\n",
       "          if (document.readyState != \"loading\") fn();\n",
       "          else document.addEventListener(\"DOMContentLoaded\", fn);\n",
       "        })();\n",
       "      },\n",
       "      function(Bokeh) {\n",
       "      }\n",
       "    ];\n",
       "  \n",
       "    function run_inline_js() {\n",
       "      \n",
       "      if ((window.Bokeh !== undefined) || (force === true)) {\n",
       "        for (var i = 0; i < inline_js.length; i++) {\n",
       "          inline_js[i](window.Bokeh);\n",
       "        }if (force === true) {\n",
       "          display_loaded();\n",
       "        }} else if (Date.now() < window._bokeh_timeout) {\n",
       "        setTimeout(run_inline_js, 100);\n",
       "      } else if (!window._bokeh_failed_load) {\n",
       "        console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "        window._bokeh_failed_load = true;\n",
       "      } else if (force !== true) {\n",
       "        var cell = $(document.getElementById(\"cc98270a-d7e4-48a7-a37c-fbaf0adc2d59\")).parents('.cell').data().cell;\n",
       "        cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "      }\n",
       "  \n",
       "    }\n",
       "  \n",
       "    if (window._bokeh_is_loading === 0) {\n",
       "      console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "      run_inline_js();\n",
       "    } else {\n",
       "      load_libs(js_urls, function() {\n",
       "        console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "        run_inline_js();\n",
       "      });\n",
       "    }\n",
       "  }(this));\n",
       "</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# LGBM Retrain Results\n",
    "p = figure(y_axis_label='AUC', plot_width=700, plot_height=350, tools=\"pan,wheel_zoom,box_zoom,reset\")\n",
    "l1 = p.line('index', 'Train AUC', legend=' Train AUC', line_color='#5f9e6f', source=data_cds, line_width=6, line_cap=\"round\")\n",
    "p.circle('index', 'Train AUC', fill_color=\"white\", size=6, line_width=1.5, line_color=\"#5975a4\", source=data_cds)\n",
    "p.line('index', 'Retrain AUC', legend=' Retrain AUC', line_color='#a2cfae', source=data_cds, line_width=6, line_cap=\"round\")\n",
    "p.circle('index', 'Retrain AUC',  fill_color=\"white\", size=6, line_width=1.5, line_color=\"#a1bae3\", source=data_cds)\n",
    "l1_hover = HoverTool(renderers=[l1], tooltips=[( 'Train AUC',  '@{Train AUC}{0.4f}' ), ( 'Retrain AUC',  '@{Retrain AUC}{0.4f}' )], mode='vline')\n",
    "p.add_tools(l1_hover)\n",
    "show(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:bokeh.io:The webdriver raised a TimeoutException while waiting for                      a 'bokeh:idle' event to signify that the layout has rendered.                      Something may have gone wrong.\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"350\" style=\"width: 700px; height: 350px;\" version=\"1.1\" width=\"700\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs/><g><g/><g transform=\"scale(1,1) translate(0.5,0.5)\"><rect fill=\"#FFFFFF\" height=\"350\" stroke=\"none\" width=\"700\" x=\"0\" y=\"0\"/><rect fill=\"#ffffff\" height=\"350\" stroke=\"none\" width=\"700\" x=\"0\" y=\"0\"/><rect fill=\"#FFFFFF\" height=\"318.9925\" stroke=\"none\" width=\"609.38875\" x=\"60.61125000000001\" y=\"4\"/><rect fill=\"#ffffff\" height=\"318.9925\" stroke=\"none\" width=\"609.38875\" x=\"60.61125000000001\" y=\"4\"/><g><rect fill=\"none\" height=\"318.9925\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\" width=\"609.38875\" x=\"60.61125000000001\" y=\"4\"/></g><g><g><path d=\" M 110 323 L 110 4\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 221 323 L 221 4\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 332 323 L 332 4\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 443 323 L 443 4\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 554 323 L 554 4\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 664 323 L 664 4\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/></g><g><path d=\" M 61 298 L 670 298\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 61 228 L 670 228\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 61 157 L 670 157\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 61 87 L 670 87\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 61 17 L 670 17\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/></g><g><path fill=\"none\" stroke=\"none\"/><path d=\" M 88.31073863636371 25.250389283049117 L 110.4703295454546 45.5838592144064 L 132.6299204545455 53.09160195829236 L 154.78951136363642 88.96624049247748 L 176.94910227272734 107.07084929674329 L 199.1086931818182 138.70503617072063 L 221.26828409090913 147.1036433162817 L 243.42787500000006 162.05751222850358 L 265.58746590909095 187.59489219145507 L 287.74705681818193 195.63404478017856 L 309.9066477272728 194.2375152519262 L 332.06623863636366 195.89303752905516 L 354.22582954545464 213.1053909219977 L 376.3854204545455 203.67213755202692 L 398.5450113636364 220.5254558989791 L 420.70460227272736 217.84411920473394 L 442.8641931818182 224.97999870253625 L 465.0237840909092 231.99480884202603 L 487.18337500000007 235.26148518543278 L 509.34296590909094 247.74396650661038 L 531.5025568181819 247.22090271966454 L 553.6621477272728 248.35843949904165 L 575.8217386363638 266.0613556280905 L 597.9813295454546 286.0548009786562 L 620.1409204545455 283.9139212118448 L 642.3005113636365 308.4928409090908\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#5f9e6f\" stroke-linecap=\"round\" stroke-miterlimit=\"10\" stroke-width=\"6\"/></g><g><path fill=\"none\" stroke=\"none\"/><path d=\" M 468.0237840909092 231.99480884202603 A 3 3 0 1 1 468.0237825909093 231.99180884252604\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 445.8641931818182 224.97999870253625 A 3 3 0 1 1 445.86419168181834 224.97699870303626\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 423.70460227272736 217.84411920473394 A 3 3 0 1 1 423.7046007727275 217.84111920523395\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 401.5450113636364 220.5254558989791 A 3 3 0 1 1 401.5450098636365 220.5224558994791\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 379.3854204545455 203.67213755202692 A 3 3 0 1 1 379.3854189545456 203.66913755252693\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 357.22582954545464 213.1053909219977 A 3 3 0 1 1 357.22582804545476 213.1023909224977\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 335.06623863636366 195.89303752905516 A 3 3 0 1 1 335.0662371363638 195.89003752955517\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 290.74705681818193 195.63404478017856 A 3 3 0 1 1 290.74705531818205 195.63104478067856\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 312.9066477272728 194.2375152519262 A 3 3 0 1 1 312.9066462272729 194.2345152524262\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 268.58746590909095 187.59489219145507 A 3 3 0 1 1 268.58746440909107 187.59189219195508\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 246.42787500000006 162.05751222850358 A 3 3 0 1 1 246.42787350000017 162.0545122290036\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 224.26828409090913 147.1036433162817 A 3 3 0 1 1 224.26828259090925 147.1006433167817\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 202.1086931818182 138.70503617072063 A 3 3 0 1 1 202.10869168181833 138.70203617122064\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 179.94910227272734 107.07084929674329 A 3 3 0 1 1 179.94910077272746 107.06784929724328\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 157.78951136363642 88.96624049247748 A 3 3 0 1 1 157.78950986363654 88.96324049297748\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 135.6299204545455 53.09160195829236 A 3 3 0 1 1 135.62991895454562 53.08860195879236\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 113.4703295454546 45.5838592144064 A 3 3 0 1 1 113.47032804545474 45.580859214906404\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 91.31073863636371 25.250389283049117 A 3 3 0 1 1 91.31073713636384 25.247389283549115\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 490.18337500000007 235.26148518543278 A 3 3 0 1 1 490.1833735000002 235.25848518593278\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 512.3429659090909 247.74396650661038 A 3 3 0 1 1 512.3429644090911 247.7409665071104\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 534.5025568181819 247.22090271966454 A 3 3 0 1 1 534.5025553181821 247.21790272016455\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 556.6621477272728 248.35843949904165 A 3 3 0 1 1 556.662146227273 248.35543949954166\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 578.8217386363638 266.0613556280905 A 3 3 0 1 1 578.8217371363639 266.0583556285905\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 600.9813295454546 286.0548009786562 A 3 3 0 1 1 600.9813280454548 286.0518009791562\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 623.1409204545455 283.9139212118448 A 3 3 0 1 1 623.1409189545457 283.9109212123448\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 645.3005113636365 308.4928409090908 A 3 3 0 1 1 645.3005098636366 308.4898409095908\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/></g><g><path fill=\"none\" stroke=\"none\"/><path d=\" M 88.31073863636371 18.499659090908608 L 110.4703295454546 33.582177996036535 L 132.6299204545455 57.161182551053116 L 154.78951136363642 98.86890442008689 L 176.94910227272734 115.24740272743293 L 199.1086931818182 131.29632006611155 L 221.26828409090913 138.48541120285518 L 243.42787500000006 136.8146540581456 L 265.58746590909095 182.92658904996324 L 287.74705681818193 173.62934551230956 L 309.9066477272728 172.56798307083818 L 332.06623863636366 192.69832143444955 L 354.22582954545464 207.1272645203553 L 376.3854204545455 197.82030599467907 L 398.5450113636364 203.37849351712475 L 420.70460227272736 202.81988170582372 L 442.8641931818182 207.49367412543597 L 465.0237840909092 239.34470394798245 L 487.18337500000007 227.49598614569277 L 509.34296590909094 234.21773522703188 L 531.5025568181819 224.0814699965149 L 553.6621477272728 205.62696706680617 L 575.8217386363638 216.611306592662 L 597.9813295454546 201.92224470245037 L 620.1409204545455 216.58580474910286 L 642.3005113636365 232.99037160764328\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#a2cfae\" stroke-linecap=\"round\" stroke-miterlimit=\"10\" stroke-width=\"6\"/></g><g><path fill=\"none\" stroke=\"none\"/><path d=\" M 468.0237840909092 239.34470394798245 A 3 3 0 1 1 468.0237825909093 239.34170394848246\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 445.8641931818182 207.49367412543597 A 3 3 0 1 1 445.86419168181834 207.49067412593598\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 423.70460227272736 202.81988170582372 A 3 3 0 1 1 423.7046007727275 202.81688170632373\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 401.5450113636364 203.37849351712475 A 3 3 0 1 1 401.5450098636365 203.37549351762476\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 379.3854204545455 197.82030599467907 A 3 3 0 1 1 379.3854189545456 197.81730599517908\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 357.22582954545464 207.1272645203553 A 3 3 0 1 1 357.22582804545476 207.12426452085532\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 335.06623863636366 192.69832143444955 A 3 3 0 1 1 335.0662371363638 192.69532143494956\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 268.58746590909095 182.92658904996324 A 3 3 0 1 1 268.58746440909107 182.92358905046325\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 290.74705681818193 173.62934551230956 A 3 3 0 1 1 290.74705531818205 173.62634551280956\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 312.9066477272728 172.56798307083818 A 3 3 0 1 1 312.9066462272729 172.5649830713382\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 224.26828409090913 138.48541120285518 A 3 3 0 1 1 224.26828259090925 138.48241120335518\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 246.42787500000006 136.8146540581456 A 3 3 0 1 1 246.42787350000017 136.81165405864562\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 202.1086931818182 131.29632006611155 A 3 3 0 1 1 202.10869168181833 131.29332006661156\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 179.94910227272734 115.24740272743293 A 3 3 0 1 1 179.94910077272746 115.24440272793292\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 157.78951136363642 98.86890442008689 A 3 3 0 1 1 157.78950986363654 98.86590442058689\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 135.6299204545455 57.161182551053116 A 3 3 0 1 1 135.62991895454562 57.15818255155312\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 113.4703295454546 33.582177996036535 A 3 3 0 1 1 113.47032804545474 33.57917799653654\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 91.31073863636371 18.499659090908608 A 3 3 0 1 1 91.31073713636384 18.496659091408606\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 490.18337500000007 227.49598614569277 A 3 3 0 1 1 490.1833735000002 227.49298614619278\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 512.3429659090909 234.21773522703188 A 3 3 0 1 1 512.3429644090911 234.2147352275319\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 534.5025568181819 224.0814699965149 A 3 3 0 1 1 534.5025553181821 224.0784699970149\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 556.6621477272728 205.62696706680617 A 3 3 0 1 1 556.662146227273 205.62396706730618\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 578.8217386363638 216.611306592662 A 3 3 0 1 1 578.8217371363639 216.608306593162\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 600.9813295454546 201.92224470245037 A 3 3 0 1 1 600.9813280454548 201.91924470295038\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 623.1409204545455 216.58580474910286 A 3 3 0 1 1 623.1409189545457 216.58280474960287\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/><path d=\" M 645.3005113636365 232.99037160764328 A 3 3 0 1 1 645.3005098636366 232.9873716081433\" fill=\"white\" paint-order=\"fill stroke markers\" stroke=\"#a1bae3\" stroke-miterlimit=\"10\" stroke-width=\"1.5\"/></g></g><g><g/><g><path d=\" M 527.328125 14 L 660 14 L 660 77 L 527.328125 77 L 527.328125 14 Z\" fill=\"#ffffff\" fill-opacity=\"0.95\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\" stroke-opacity=\"0.5\"/><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"10pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"start\" text-decoration=\"normal\" x=\"562.328125\" y=\"34\"> Train AUC</text><g><path d=\" M 537.328125 34 L 557.328125 34\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#5f9e6f\" stroke-linecap=\"round\" stroke-miterlimit=\"10\" stroke-width=\"6\"/></g><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"10pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"start\" text-decoration=\"normal\" x=\"562.328125\" y=\"57\"> Retrain AUC</text><g><path d=\" M 537.328125 57 L 557.328125 57\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#a2cfae\" stroke-linecap=\"round\" stroke-miterlimit=\"10\" stroke-width=\"6\"/></g></g></g><g><g><path d=\" M 61 323 L 670 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 110 329 L 110 321\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 221 329 L 221 321\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 332 329 L 332 321\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 443 329 L 443 321\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 554 329 L 554 321\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 664 329 L 664 321\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 88 327 L 88 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 66 327 L 66 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 110 327 L 110 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 133 327 L 133 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 155 327 L 155 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 177 327 L 177 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 199 327 L 199 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 221 327 L 221 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 243 327 L 243 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 266 327 L 266 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 288 327 L 288 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 310 327 L 310 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 332 327 L 332 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 354 327 L 354 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 376 327 L 376 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 399 327 L 399 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 421 327 L 421 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 443 327 L 443 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 465 327 L 465 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 487 327 L 487 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 509 327 L 509 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 532 327 L 532 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 554 327 L 554 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 576 327 L 576 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 598 327 L 598 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 620 327 L 620 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 642 327 L 642 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 664 327 L 664 323\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><text dominant-baseline=\"hanging\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"110\" y=\"334\">10</text><text dominant-baseline=\"hanging\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"221\" y=\"334\">15</text><text dominant-baseline=\"hanging\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"332\" y=\"334\">20</text><text dominant-baseline=\"hanging\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"443\" y=\"334\">25</text><text dominant-baseline=\"hanging\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"554\" y=\"334\">30</text><text dominant-baseline=\"hanging\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"664\" y=\"334\">35</text><text dominant-baseline=\"hanging\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"10pt\" font-style=\"italic\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"365.3056250000001\" y=\"353.99999999999983\"/></g><g><path d=\" M 61 323 L 61 4\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 55 298 L 63 298\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 55 228 L 63 228\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 55 157 L 63 157\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 55 87 L 63 87\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 55 17 L 63 17\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 312 L 61 312\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 298 L 61 298\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 284 L 61 284\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 270 L 61 270\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 256 L 61 256\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 242 L 61 242\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 228 L 61 228\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 214 L 61 214\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 200 L 61 200\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 186 L 61 186\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 171 L 61 171\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 157 L 61 157\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 143 L 61 143\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 129 L 61 129\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 115 L 61 115\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 101 L 61 101\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 87 L 61 87\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 73 L 61 73\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 59 L 61 59\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 45 L 61 45\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 31 L 61 31\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 57 17 L 61 17\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"50\" y=\"298\">0.74</text><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"50\" y=\"228\">0.76</text><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"50\" y=\"157\">0.78</text><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"50\" y=\"87\">0.8</text><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"50\" y=\"17\">0.82</text><g transform=\"translate(17.73000000000001,163.49624999999992) rotate(-90,0,0)\"><text dominant-baseline=\"alphabetic\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"10pt\" font-style=\"italic\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"0\" y=\"0\">AUC</text><g transform=\"rotate(90,0,0) translate(-17.73000000000001,-163.49624999999992)\"/></g></g></g></g></g></svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# SVG: LGBM Retrain Results\n",
    "#Save the plot and display it so that is is visible on github\n",
    "p.output_backend = \"svg\"\n",
    "export_svgs(p, filename=\"lgbm_bci_train_retrain.svg\")\n",
    "display(SVG('lgbm_bci_train_retrain.svg'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Retrainig does also seem to improve the results for the LightGBM model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "xgb_series = retrain_results_dict['xgb'][['Retrain Test interval', 'Retrain interval', 'Test interval']].sum()\n",
    "lgbm_series = retrain_results_dict['lgbm'][['Retrain Test interval', 'Retrain interval', 'Test interval']].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "t_res = pd.DataFrame([['XGBoost', xgb_series['Retrain interval']/60],\n",
    "                      ['LightGBM', lgbm_series['Retrain interval']/60]], \n",
    "                     columns=['Model', 'Time(m)']).assign(color=[\"#5975a4\", \"#5f9e6f\"])\n",
    "t_cds = ColumnDataSource(t_res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <div class=\"bk-plotdiv\" id=\"8dd7c724-0fd7-40bb-889e-c042904192b6\"></div>\n",
       "    </div>\n",
       "<script type=\"text/javascript\">\n",
       "  \n",
       "  (function(global) {\n",
       "    function now() {\n",
       "      return new Date();\n",
       "    }\n",
       "  \n",
       "    var force = false;\n",
       "  \n",
       "    if (typeof (window._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n",
       "      window._bokeh_onload_callbacks = [];\n",
       "      window._bokeh_is_loading = undefined;\n",
       "    }\n",
       "  \n",
       "  \n",
       "    \n",
       "    if (typeof (window._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "      window._bokeh_timeout = Date.now() + 0;\n",
       "      window._bokeh_failed_load = false;\n",
       "    }\n",
       "  \n",
       "    var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "       \"<div style='background-color: #fdd'>\\n\"+\n",
       "       \"<p>\\n\"+\n",
       "       \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "       \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "       \"</p>\\n\"+\n",
       "       \"<ul>\\n\"+\n",
       "       \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "       \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "       \"</ul>\\n\"+\n",
       "       \"<code>\\n\"+\n",
       "       \"from bokeh.resources import INLINE\\n\"+\n",
       "       \"output_notebook(resources=INLINE)\\n\"+\n",
       "       \"</code>\\n\"+\n",
       "       \"</div>\"}};\n",
       "  \n",
       "    function display_loaded() {\n",
       "      if (window.Bokeh !== undefined) {\n",
       "        var el = document.getElementById(\"8dd7c724-0fd7-40bb-889e-c042904192b6\");\n",
       "        el.textContent = \"BokehJS \" + Bokeh.version + \" successfully loaded.\";\n",
       "      } else if (Date.now() < window._bokeh_timeout) {\n",
       "        setTimeout(display_loaded, 100)\n",
       "      }\n",
       "    }\n",
       "  \n",
       "    function run_callbacks() {\n",
       "      try {\n",
       "        window._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n",
       "      }\n",
       "      finally {\n",
       "        delete window._bokeh_onload_callbacks\n",
       "      }\n",
       "      console.info(\"Bokeh: all callbacks have finished\");\n",
       "    }\n",
       "  \n",
       "    function load_libs(js_urls, callback) {\n",
       "      window._bokeh_onload_callbacks.push(callback);\n",
       "      if (window._bokeh_is_loading > 0) {\n",
       "        console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "        return null;\n",
       "      }\n",
       "      if (js_urls == null || js_urls.length === 0) {\n",
       "        run_callbacks();\n",
       "        return null;\n",
       "      }\n",
       "      console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "      window._bokeh_is_loading = js_urls.length;\n",
       "      for (var i = 0; i < js_urls.length; i++) {\n",
       "        var url = js_urls[i];\n",
       "        var s = document.createElement('script');\n",
       "        s.src = url;\n",
       "        s.async = false;\n",
       "        s.onreadystatechange = s.onload = function() {\n",
       "          window._bokeh_is_loading--;\n",
       "          if (window._bokeh_is_loading === 0) {\n",
       "            console.log(\"Bokeh: all BokehJS libraries loaded\");\n",
       "            run_callbacks()\n",
       "          }\n",
       "        };\n",
       "        s.onerror = function() {\n",
       "          console.warn(\"failed to load library \" + url);\n",
       "        };\n",
       "        console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "        document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "      }\n",
       "    };var element = document.getElementById(\"8dd7c724-0fd7-40bb-889e-c042904192b6\");\n",
       "    if (element == null) {\n",
       "      console.log(\"Bokeh: ERROR: autoload.js configured with elementid '8dd7c724-0fd7-40bb-889e-c042904192b6' but no matching script tag was found. \")\n",
       "      return false;\n",
       "    }\n",
       "  \n",
       "    var js_urls = [];\n",
       "  \n",
       "    var inline_js = [\n",
       "      function(Bokeh) {\n",
       "        (function() {\n",
       "          var fn = function() {\n",
       "            var docs_json = {\"e02df046-3df5-408c-b8b7-ee6312fd1db3\":{\"roots\":{\"references\":[{\"attributes\":{\"fill_color\":{\"field\":\"color\"},\"line_color\":{\"field\":\"color\"},\"top\":{\"field\":\"Time(m)\"},\"width\":{\"value\":0.8},\"x\":{\"field\":\"Model\"}},\"id\":\"efbf5e6a-5c37-4ba6-a4fb-8565656b5a5a\",\"type\":\"VBar\"},{\"attributes\":{\"plot\":null,\"text\":\"\"},\"id\":\"f4c43c67-a369-43f7-8dd0-ae515bf292b7\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"e3d93373-5775-463e-be60-c476717524bb\",\"type\":\"CategoricalTicker\"},{\"attributes\":{},\"id\":\"c26f924a-e2bb-423e-987a-8ca5059cec2a\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"plot\":{\"id\":\"e5037fe3-f0e4-4fbc-83e9-db78baecc39e\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"509a3724-278e-44b4-9040-ed3271dd9e29\",\"type\":\"WheelZoomTool\"},{\"attributes\":{},\"id\":\"70cfae35-b4c7-43cd-96fd-b2827d969974\",\"type\":\"BasicTicker\"},{\"attributes\":{\"below\":[{\"id\":\"eab7fb58-46b6-4f59-8323-69453c9d041f\",\"type\":\"CategoricalAxis\"}],\"left\":[{\"id\":\"3f3359c6-3629-4381-8c84-b163d009cc59\",\"type\":\"LinearAxis\"}],\"plot_height\":500,\"plot_width\":300,\"renderers\":[{\"id\":\"eab7fb58-46b6-4f59-8323-69453c9d041f\",\"type\":\"CategoricalAxis\"},{\"id\":\"0b8750f0-3742-4c14-8b3f-f589a1a29cf5\",\"type\":\"Grid\"},{\"id\":\"3f3359c6-3629-4381-8c84-b163d009cc59\",\"type\":\"LinearAxis\"},{\"id\":\"fbac0131-7581-4eed-874a-7b4a1cb10e68\",\"type\":\"Grid\"},{\"id\":\"b71f074d-5b66-4941-9b08-fc20673dbb09\",\"type\":\"BoxAnnotation\"},{\"id\":\"d5a83e91-00af-4e55-9507-6058e0a965c4\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"f4c43c67-a369-43f7-8dd0-ae515bf292b7\",\"type\":\"Title\"},\"tool_events\":{\"id\":\"995a4608-cb19-434c-a783-ac95d8488f44\",\"type\":\"ToolEvents\"},\"toolbar\":{\"id\":\"5c6e7477-201e-4b01-9a50-271946d61a38\",\"type\":\"Toolbar\"},\"x_range\":{\"id\":\"ff84dc32-11e1-4290-8168-dc9bdd38449e\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"080f1435-27be-45ec-bc28-6ddfe55c8005\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1629caae-8201-4a2e-a540-61663979720d\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"2dbf5d34-d9ca-458e-bc98-b53ca48b5059\",\"type\":\"LinearScale\"}},\"id\":\"e5037fe3-f0e4-4fbc-83e9-db78baecc39e\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"index\",\"Model\",\"color\",\"Time(m)\"],\"data\":{\"Model\":[\"XGBoost\",\"LightGBM\"],\"Time(m)\":{\"__ndarray__\":\"78o+NcNQH0CrOrzyoL8PQA==\",\"dtype\":\"float64\",\"shape\":[2]},\"color\":[\"#5975a4\",\"#5f9e6f\"],\"index\":[0,1]}},\"id\":\"9e17f3b3-6ccf-4b75-8312-bcb742b8f5d3\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"plot\":{\"id\":\"e5037fe3-f0e4-4fbc-83e9-db78baecc39e\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"e4c85e51-02db-431e-8393-195b04720a8a\",\"type\":\"SaveTool\"},{\"attributes\":{},\"id\":\"995a4608-cb19-434c-a783-ac95d8488f44\",\"type\":\"ToolEvents\"},{\"attributes\":{\"axis_label\":\"Time (minutes)\",\"formatter\":{\"id\":\"b5134912-da51-469f-92c7-348eb93f2fb8\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"e5037fe3-f0e4-4fbc-83e9-db78baecc39e\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"70cfae35-b4c7-43cd-96fd-b2827d969974\",\"type\":\"BasicTicker\"}},\"id\":\"3f3359c6-3629-4381-8c84-b163d009cc59\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"080f1435-27be-45ec-bc28-6ddfe55c8005\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"data_source\":{\"id\":\"9e17f3b3-6ccf-4b75-8312-bcb742b8f5d3\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"efbf5e6a-5c37-4ba6-a4fb-8565656b5a5a\",\"type\":\"VBar\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"60f26269-7293-4fff-8c60-b2bf8ff35ee8\",\"type\":\"VBar\"},\"selection_glyph\":null},\"id\":\"d5a83e91-00af-4e55-9507-6058e0a965c4\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"28f455ae-5bb0-41f6-95e1-0ba90fc04228\",\"type\":\"PanTool\"},{\"id\":\"509a3724-278e-44b4-9040-ed3271dd9e29\",\"type\":\"WheelZoomTool\"},{\"id\":\"0433bae9-e19b-4dd9-9a2d-d9a3b2d3b772\",\"type\":\"BoxZoomTool\"},{\"id\":\"e4c85e51-02db-431e-8393-195b04720a8a\",\"type\":\"SaveTool\"},{\"id\":\"dd6ec673-5dfb-4e0c-98ff-b2fa1de7ffca\",\"type\":\"ResetTool\"},{\"id\":\"acb85ce9-8b52-4dd8-b1ce-5d50f26a197c\",\"type\":\"HelpTool\"}]},\"id\":\"5c6e7477-201e-4b01-9a50-271946d61a38\",\"type\":\"Toolbar\"},{\"attributes\":{\"plot\":{\"id\":\"e5037fe3-f0e4-4fbc-83e9-db78baecc39e\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"dd6ec673-5dfb-4e0c-98ff-b2fa1de7ffca\",\"type\":\"ResetTool\"},{\"attributes\":{\"plot\":{\"id\":\"e5037fe3-f0e4-4fbc-83e9-db78baecc39e\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"28f455ae-5bb0-41f6-95e1-0ba90fc04228\",\"type\":\"PanTool\"},{\"attributes\":{\"plot\":{\"id\":\"e5037fe3-f0e4-4fbc-83e9-db78baecc39e\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"acb85ce9-8b52-4dd8-b1ce-5d50f26a197c\",\"type\":\"HelpTool\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"lightgrey\"},\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":{\"value\":1.0},\"line_color\":{\"value\":\"black\"},\"line_dash\":[4,4],\"line_width\":{\"value\":2},\"plot\":null,\"render_mode\":\"css\",\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"b71f074d-5b66-4941-9b08-fc20673dbb09\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"plot\":{\"id\":\"e5037fe3-f0e4-4fbc-83e9-db78baecc39e\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"e3d93373-5775-463e-be60-c476717524bb\",\"type\":\"CategoricalTicker\"}},\"id\":\"0b8750f0-3742-4c14-8b3f-f589a1a29cf5\",\"type\":\"Grid\"},{\"attributes\":{\"callback\":null},\"id\":\"1629caae-8201-4a2e-a540-61663979720d\",\"type\":\"DataRange1d\"},{\"attributes\":{\"formatter\":{\"id\":\"c26f924a-e2bb-423e-987a-8ca5059cec2a\",\"type\":\"CategoricalTickFormatter\"},\"plot\":{\"id\":\"e5037fe3-f0e4-4fbc-83e9-db78baecc39e\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"e3d93373-5775-463e-be60-c476717524bb\",\"type\":\"CategoricalTicker\"}},\"id\":\"eab7fb58-46b6-4f59-8323-69453c9d041f\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"b5134912-da51-469f-92c7-348eb93f2fb8\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"e5037fe3-f0e4-4fbc-83e9-db78baecc39e\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"70cfae35-b4c7-43cd-96fd-b2827d969974\",\"type\":\"BasicTicker\"}},\"id\":\"fbac0131-7581-4eed-874a-7b4a1cb10e68\",\"type\":\"Grid\"},{\"attributes\":{\"overlay\":{\"id\":\"b71f074d-5b66-4941-9b08-fc20673dbb09\",\"type\":\"BoxAnnotation\"},\"plot\":{\"id\":\"e5037fe3-f0e4-4fbc-83e9-db78baecc39e\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"0433bae9-e19b-4dd9-9a2d-d9a3b2d3b772\",\"type\":\"BoxZoomTool\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"top\":{\"field\":\"Time(m)\"},\"width\":{\"value\":0.8},\"x\":{\"field\":\"Model\"}},\"id\":\"60f26269-7293-4fff-8c60-b2bf8ff35ee8\",\"type\":\"VBar\"},{\"attributes\":{},\"id\":\"2dbf5d34-d9ca-458e-bc98-b53ca48b5059\",\"type\":\"LinearScale\"},{\"attributes\":{\"callback\":null,\"factors\":[\"XGBoost\",\"LightGBM\"]},\"id\":\"ff84dc32-11e1-4290-8168-dc9bdd38449e\",\"type\":\"FactorRange\"}],\"root_ids\":[\"e5037fe3-f0e4-4fbc-83e9-db78baecc39e\"]},\"title\":\"Bokeh Application\",\"version\":\"0.12.6\"}};\n",
       "            var render_items = [{\"docid\":\"e02df046-3df5-408c-b8b7-ee6312fd1db3\",\"elementid\":\"8dd7c724-0fd7-40bb-889e-c042904192b6\",\"modelid\":\"e5037fe3-f0e4-4fbc-83e9-db78baecc39e\"}];\n",
       "            \n",
       "            Bokeh.embed.embed_items(docs_json, render_items);\n",
       "          };\n",
       "          if (document.readyState != \"loading\") fn();\n",
       "          else document.addEventListener(\"DOMContentLoaded\", fn);\n",
       "        })();\n",
       "      },\n",
       "      function(Bokeh) {\n",
       "      }\n",
       "    ];\n",
       "  \n",
       "    function run_inline_js() {\n",
       "      \n",
       "      if ((window.Bokeh !== undefined) || (force === true)) {\n",
       "        for (var i = 0; i < inline_js.length; i++) {\n",
       "          inline_js[i](window.Bokeh);\n",
       "        }if (force === true) {\n",
       "          display_loaded();\n",
       "        }} else if (Date.now() < window._bokeh_timeout) {\n",
       "        setTimeout(run_inline_js, 100);\n",
       "      } else if (!window._bokeh_failed_load) {\n",
       "        console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "        window._bokeh_failed_load = true;\n",
       "      } else if (force !== true) {\n",
       "        var cell = $(document.getElementById(\"8dd7c724-0fd7-40bb-889e-c042904192b6\")).parents('.cell').data().cell;\n",
       "        cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "      }\n",
       "  \n",
       "    }\n",
       "  \n",
       "    if (window._bokeh_is_loading === 0) {\n",
       "      console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "      run_inline_js();\n",
       "    } else {\n",
       "      load_libs(js_urls, function() {\n",
       "        console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "        run_inline_js();\n",
       "      });\n",
       "    }\n",
       "  }(this));\n",
       "</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# XGB vs LGBM Retrain Time\n",
    "p = figure(plot_width=300, plot_height=500, x_range=t_res['Model'].tolist(), y_axis_label='Time (minutes)')\n",
    "p.vbar(x='Model', width=0.8, bottom=0, top='Time(m)', color=\"color\", source=t_cds)\n",
    "show(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:bokeh.io:The webdriver raised a TimeoutException while waiting for                      a 'bokeh:idle' event to signify that the layout has rendered.                      Something may have gone wrong.\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"500\" style=\"width: 300px; height: 500px;\" version=\"1.1\" width=\"300\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs/><g><g/><g transform=\"scale(1,1) translate(0.5,0.5)\"><rect fill=\"#FFFFFF\" height=\"500\" stroke=\"none\" width=\"300\" x=\"0\" y=\"0\"/><rect fill=\"#ffffff\" height=\"500\" stroke=\"none\" width=\"300\" x=\"0\" y=\"0\"/><rect fill=\"#FFFFFF\" height=\"468.9925\" stroke=\"none\" width=\"228.58718749999997\" x=\"41.412812500000015\" y=\"3.999999999999943\"/><rect fill=\"#ffffff\" height=\"468.9925\" stroke=\"none\" width=\"228.58718749999997\" x=\"41.412812500000015\" y=\"3.999999999999943\"/><g><rect fill=\"none\" height=\"468.9925\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\" width=\"228.58718749999997\" x=\"41.412812500000015\" y=\"3.999999999999943\"/></g><g><g><path d=\" M 99 473 L 99 4\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 213 473 L 213 4\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/></g><g><path d=\" M 41 452 L 270 452\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 41 343 L 270 343\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 41 234 L 270 234\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 41 125 L 270 125\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 41 16 L 270 16\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/></g><g><path fill=\"none\" stroke=\"none\"/><rect fill=\"#5975a4\" height=\"426.3568181818182\" stroke=\"none\" width=\"91.43487500000083\" x=\"52.84217187499959\" y=\"25.317840909090876\"/><path d=\" M 52.84217187499959 25.317840909090876 L 144.27704687500042 25.317840909090876 L 144.27704687500042 451.6746590909091 L 52.84217187499959 451.6746590909091 L 52.84217187499959 25.317840909090876 Z\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#5975a4\" stroke-miterlimit=\"10\"/><rect fill=\"#5f9e6f\" height=\"216.1265161619247\" stroke=\"none\" width=\"91.43487499999901\" x=\"167.1357656250005\" y=\"235.54814292898436\"/><path d=\" M 167.1357656250005 235.54814292898436 L 258.5706406249995 235.54814292898436 L 258.5706406249995 451.6746590909091 L 167.1357656250005 451.6746590909091 L 167.1357656250005 235.54814292898436 Z\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#5f9e6f\" stroke-miterlimit=\"10\"/></g></g><g/><g><g><path d=\" M 41 473 L 270 473\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 99 479 L 99 471\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 213 479 L 213 471\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><text dominant-baseline=\"hanging\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"99\" y=\"484\">XGBoost</text><text dominant-baseline=\"hanging\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"213\" y=\"484\">LightGBM</text><text dominant-baseline=\"hanging\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"10pt\" font-style=\"italic\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"155.70640625000001\" y=\"504\"/></g><g><path d=\" M 41 473 L 41 4\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 35 452 L 43 452\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 35 343 L 43 343\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 35 234 L 43 234\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 35 125 L 43 125\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 35 16 L 43 16\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 452 L 41 452\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 430 L 41 430\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 408 L 41 408\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 386 L 41 386\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 365 L 41 365\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 343 L 41 343\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 321 L 41 321\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 299 L 41 299\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 277 L 41 277\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 256 L 41 256\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 234 L 41 234\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 212 L 41 212\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 190 L 41 190\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 168 L 41 168\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 147 L 41 147\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 125 L 41 125\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 103 L 41 103\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 81 L 41 81\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 60 L 41 60\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 38 L 41 38\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 37 16 L 41 16\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"30\" y=\"452\">0</text><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"30\" y=\"343\">2</text><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"30\" y=\"234\">4</text><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"30\" y=\"125\">6</text><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"30\" y=\"16\">8</text><g transform=\"translate(17.730000000000015,238.49624999999997) rotate(-90,0,0)\"><text dominant-baseline=\"alphabetic\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"10pt\" font-style=\"italic\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"0\" y=\"0\">Time (minutes)</text><g transform=\"rotate(90,0,0) translate(-17.730000000000015,-238.49624999999997)\"/></g></g></g></g></g></svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# SVG: XGB vs LGBM Retrain Time\n",
    "# Save the plot and display it so that is is visible on github\n",
    "p.output_backend = \"svg\"\n",
    "export_svgs(p, filename=\"xgb_lgbm_bar.svg\")\n",
    "display(SVG('xgb_lgbm_bar.svg'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "LightGBM seems to be almost twice as fast as XGBoost and offer similar performance. In general, leaf-wise algorithms are more efficient and converge faster than depth-wise. However, it can easily over-fit when the data is small or the number of leaves is too high."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.5",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}