experiments/03_football.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Football match prediction\n",
    "\n",
    "In this experiment we are going to use the [Kaggle football dataset](https://www.kaggle.com/hugomathien/soccer). The dataset has information from +25,000 matches, +10,000 players from 11 European Countries with their lead championship during seasons 2008 to 2016. It also contains players attributes sourced from EA Sports' FIFA video game series. The problem we address is to try to predict if a match is going to end as win, draw or defeat. \n",
    "\n",
    "Part of the code use in this notebook is this [kaggle kernel](https://www.kaggle.com/airback/match-outcome-prediction-in-football).\n",
    "\n",
    "The details of the machine we used and the version of the libraries can be found in [experiment 01](01_airline.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "System version: 3.5.2 |Anaconda custom (64-bit)| (default, Jul  2 2016, 17:53:06) \n",
      "[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)]\n",
      "XGBoost version: 0.6\n",
      "LightGBM version: 0.2\n"
     ]
    }
   ],
   "source": [
    "import os,sys\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import itertools\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import accuracy_score, roc_auc_score, f1_score, log_loss, precision_score, recall_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from xgboost import XGBClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from libs.loaders import load_football\n",
    "from libs.football import get_fifa_data, create_feables\n",
    "from libs.timer import Timer\n",
    "from libs.utils import get_number_processors\n",
    "from libs.metrics import classification_metrics_multilabel\n",
    "import pickle\n",
    "import pkg_resources\n",
    "import matplotlib.pylab as plt\n",
    "import json\n",
    "\n",
    "from bokeh.io import show, output_notebook\n",
    "from bokeh.charts import Bar\n",
    "from bokeh.models.renderers import GlyphRenderer\n",
    "from bokeh.models.glyphs import Rect\n",
    "from bokeh.models import Legend\n",
    "from toolz import curry\n",
    "from bokeh.io import export_svgs\n",
    "from IPython.display import SVG, display\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "print(\"System version: {}\".format(sys.version))\n",
    "print(\"XGBoost version: {}\".format(pkg_resources.get_distribution('xgboost').version))\n",
    "print(\"LightGBM version: {}\".format(pkg_resources.get_distribution('lightgbm').version))\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"http://bokeh.pydata.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"ac8c372f-da5a-4e9d-a6c5-74acdd4a4967\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(global) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof (window._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n",
       "    window._bokeh_onload_callbacks = [];\n",
       "    window._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "\n",
       "  \n",
       "  if (typeof (window._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    window._bokeh_timeout = Date.now() + 5000;\n",
       "    window._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    if (window.Bokeh !== undefined) {\n",
       "      var el = document.getElementById(\"ac8c372f-da5a-4e9d-a6c5-74acdd4a4967\");\n",
       "      el.textContent = \"BokehJS \" + Bokeh.version + \" successfully loaded.\";\n",
       "    } else if (Date.now() < window._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      window._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n",
       "    }\n",
       "    finally {\n",
       "      delete window._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.info(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(js_urls, callback) {\n",
       "    window._bokeh_onload_callbacks.push(callback);\n",
       "    if (window._bokeh_is_loading > 0) {\n",
       "      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    window._bokeh_is_loading = js_urls.length;\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var s = document.createElement('script');\n",
       "      s.src = url;\n",
       "      s.async = false;\n",
       "      s.onreadystatechange = s.onload = function() {\n",
       "        window._bokeh_is_loading--;\n",
       "        if (window._bokeh_is_loading === 0) {\n",
       "          console.log(\"Bokeh: all BokehJS libraries loaded\");\n",
       "          run_callbacks()\n",
       "        }\n",
       "      };\n",
       "      s.onerror = function() {\n",
       "        console.warn(\"failed to load library \" + url);\n",
       "      };\n",
       "      console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "    }\n",
       "  };var element = document.getElementById(\"ac8c372f-da5a-4e9d-a6c5-74acdd4a4967\");\n",
       "  if (element == null) {\n",
       "    console.log(\"Bokeh: ERROR: autoload.js configured with elementid 'ac8c372f-da5a-4e9d-a6c5-74acdd4a4967' but no matching script tag was found. \")\n",
       "    return false;\n",
       "  }\n",
       "\n",
       "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.6.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.6.min.js\"];\n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    \n",
       "    function(Bokeh) {\n",
       "      \n",
       "    },\n",
       "    \n",
       "    function(Bokeh) {\n",
       "      \n",
       "      document.getElementById(\"ac8c372f-da5a-4e9d-a6c5-74acdd4a4967\").textContent = \"BokehJS is loading...\";\n",
       "    },\n",
       "    function(Bokeh) {\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.6.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.6.min.css\");\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.6.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.6.min.css\");\n",
       "    }\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if ((window.Bokeh !== undefined) || (force === true)) {\n",
       "      for (var i = 0; i < inline_js.length; i++) {\n",
       "        inline_js[i](window.Bokeh);\n",
       "      }if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < window._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!window._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      window._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"ac8c372f-da5a-4e9d-a6c5-74acdd4a4967\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (window._bokeh_is_loading === 0) {\n",
       "    console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(js_urls, function() {\n",
       "      console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(this));"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "output_notebook()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## 1) XGBoost vs LightGBM benchmark\n",
    "\n",
    "In the next section we compare both libraries speed, accuracy and other metrics for the football dataset.\n",
    "\n",
    "\n",
    "### Data loading and management\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:libs.loaders:MOUNT_POINT not found in environment. Defaulting to /fileshare\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11, 2)\n",
      "(25979, 115)\n",
      "(11, 3)\n",
      "(299, 5)\n",
      "(183978, 42)\n",
      "CPU times: user 3.93 s, sys: 552 ms, total: 4.48 s\n",
      "Wall time: 19.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "countries, matches, leagues, teams, players = load_football()\n",
    "print(countries.shape)\n",
    "print(matches.shape)\n",
    "print(leagues.shape)\n",
    "print(teams.shape)\n",
    "print(players.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>country_id</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Belgium Jupiler League</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1729</td>\n",
       "      <td>1729</td>\n",
       "      <td>England Premier League</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4769</td>\n",
       "      <td>4769</td>\n",
       "      <td>France Ligue 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7809</td>\n",
       "      <td>7809</td>\n",
       "      <td>Germany 1. Bundesliga</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10257</td>\n",
       "      <td>10257</td>\n",
       "      <td>Italy Serie A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>13274</td>\n",
       "      <td>13274</td>\n",
       "      <td>Netherlands Eredivisie</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>15722</td>\n",
       "      <td>15722</td>\n",
       "      <td>Poland Ekstraklasa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>17642</td>\n",
       "      <td>17642</td>\n",
       "      <td>Portugal Liga ZON Sagres</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>19694</td>\n",
       "      <td>19694</td>\n",
       "      <td>Scotland Premier League</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>21518</td>\n",
       "      <td>21518</td>\n",
       "      <td>Spain LIGA BBVA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>24558</td>\n",
       "      <td>24558</td>\n",
       "      <td>Switzerland Super League</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  country_id                      name\n",
       "0       1           1    Belgium Jupiler League\n",
       "1    1729        1729    England Premier League\n",
       "2    4769        4769            France Ligue 1\n",
       "3    7809        7809     Germany 1. Bundesliga\n",
       "4   10257       10257             Italy Serie A\n",
       "5   13274       13274    Netherlands Eredivisie\n",
       "6   15722       15722        Poland Ekstraklasa\n",
       "7   17642       17642  Portugal Liga ZON Sagres\n",
       "8   19694       19694   Scotland Premier League\n",
       "9   21518       21518           Spain LIGA BBVA\n",
       "10  24558       24558  Switzerland Super League"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "leagues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>country_id</th>\n",
       "      <th>league_id</th>\n",
       "      <th>season</th>\n",
       "      <th>stage</th>\n",
       "      <th>date</th>\n",
       "      <th>match_api_id</th>\n",
       "      <th>home_team_api_id</th>\n",
       "      <th>away_team_api_id</th>\n",
       "      <th>home_team_goal</th>\n",
       "      <th>...</th>\n",
       "      <th>SJA</th>\n",
       "      <th>VCH</th>\n",
       "      <th>VCD</th>\n",
       "      <th>VCA</th>\n",
       "      <th>GBH</th>\n",
       "      <th>GBD</th>\n",
       "      <th>GBA</th>\n",
       "      <th>BSH</th>\n",
       "      <th>BSD</th>\n",
       "      <th>BSA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2008/2009</td>\n",
       "      <td>1</td>\n",
       "      <td>2008-08-17 00:00:00</td>\n",
       "      <td>492473</td>\n",
       "      <td>9987</td>\n",
       "      <td>9993</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>4.00</td>\n",
       "      <td>1.65</td>\n",
       "      <td>3.40</td>\n",
       "      <td>4.50</td>\n",
       "      <td>1.78</td>\n",
       "      <td>3.25</td>\n",
       "      <td>4.00</td>\n",
       "      <td>1.73</td>\n",
       "      <td>3.40</td>\n",
       "      <td>4.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2008/2009</td>\n",
       "      <td>1</td>\n",
       "      <td>2008-08-16 00:00:00</td>\n",
       "      <td>492474</td>\n",
       "      <td>10000</td>\n",
       "      <td>9994</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>3.80</td>\n",
       "      <td>2.00</td>\n",
       "      <td>3.25</td>\n",
       "      <td>3.25</td>\n",
       "      <td>1.85</td>\n",
       "      <td>3.25</td>\n",
       "      <td>3.75</td>\n",
       "      <td>1.91</td>\n",
       "      <td>3.25</td>\n",
       "      <td>3.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2008/2009</td>\n",
       "      <td>1</td>\n",
       "      <td>2008-08-16 00:00:00</td>\n",
       "      <td>492475</td>\n",
       "      <td>9984</td>\n",
       "      <td>8635</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.50</td>\n",
       "      <td>2.35</td>\n",
       "      <td>3.25</td>\n",
       "      <td>2.65</td>\n",
       "      <td>2.50</td>\n",
       "      <td>3.20</td>\n",
       "      <td>2.50</td>\n",
       "      <td>2.30</td>\n",
       "      <td>3.20</td>\n",
       "      <td>2.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2008/2009</td>\n",
       "      <td>1</td>\n",
       "      <td>2008-08-17 00:00:00</td>\n",
       "      <td>492476</td>\n",
       "      <td>9991</td>\n",
       "      <td>9998</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>7.50</td>\n",
       "      <td>1.45</td>\n",
       "      <td>3.75</td>\n",
       "      <td>6.50</td>\n",
       "      <td>1.50</td>\n",
       "      <td>3.75</td>\n",
       "      <td>5.50</td>\n",
       "      <td>1.44</td>\n",
       "      <td>3.75</td>\n",
       "      <td>6.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2008/2009</td>\n",
       "      <td>1</td>\n",
       "      <td>2008-08-16 00:00:00</td>\n",
       "      <td>492477</td>\n",
       "      <td>7947</td>\n",
       "      <td>9985</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1.73</td>\n",
       "      <td>4.50</td>\n",
       "      <td>3.40</td>\n",
       "      <td>1.65</td>\n",
       "      <td>4.50</td>\n",
       "      <td>3.50</td>\n",
       "      <td>1.65</td>\n",
       "      <td>4.75</td>\n",
       "      <td>3.30</td>\n",
       "      <td>1.67</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 115 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  country_id  league_id     season  stage                 date  \\\n",
       "0   1           1          1  2008/2009      1  2008-08-17 00:00:00   \n",
       "1   2           1          1  2008/2009      1  2008-08-16 00:00:00   \n",
       "2   3           1          1  2008/2009      1  2008-08-16 00:00:00   \n",
       "3   4           1          1  2008/2009      1  2008-08-17 00:00:00   \n",
       "4   5           1          1  2008/2009      1  2008-08-16 00:00:00   \n",
       "\n",
       "   match_api_id  home_team_api_id  away_team_api_id  home_team_goal  ...   \\\n",
       "0        492473              9987              9993               1  ...    \n",
       "1        492474             10000              9994               0  ...    \n",
       "2        492475              9984              8635               0  ...    \n",
       "3        492476              9991              9998               5  ...    \n",
       "4        492477              7947              9985               1  ...    \n",
       "\n",
       "    SJA   VCH   VCD   VCA   GBH   GBD   GBA   BSH   BSD   BSA  \n",
       "0  4.00  1.65  3.40  4.50  1.78  3.25  4.00  1.73  3.40  4.20  \n",
       "1  3.80  2.00  3.25  3.25  1.85  3.25  3.75  1.91  3.25  3.60  \n",
       "2  2.50  2.35  3.25  2.65  2.50  3.20  2.50  2.30  3.20  2.75  \n",
       "3  7.50  1.45  3.75  6.50  1.50  3.75  5.50  1.44  3.75  6.50  \n",
       "4  1.73  4.50  3.40  1.65  4.50  3.50  1.65  4.75  3.30  1.67  \n",
       "\n",
       "[5 rows x 115 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matches.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(21374, 115)\n"
     ]
    }
   ],
   "source": [
    "#Reduce match data to fulfill run time requirements\n",
    "cols = [\"country_id\", \"league_id\", \"season\", \"stage\", \"date\", \"match_api_id\", \"home_team_api_id\", \n",
    "        \"away_team_api_id\", \"home_team_goal\", \"away_team_goal\", \"home_player_1\", \"home_player_2\",\n",
    "        \"home_player_3\", \"home_player_4\", \"home_player_5\", \"home_player_6\", \"home_player_7\", \n",
    "        \"home_player_8\", \"home_player_9\", \"home_player_10\", \"home_player_11\", \"away_player_1\",\n",
    "        \"away_player_2\", \"away_player_3\", \"away_player_4\", \"away_player_5\", \"away_player_6\",\n",
    "        \"away_player_7\", \"away_player_8\", \"away_player_9\", \"away_player_10\", \"away_player_11\"]\n",
    "match_data = matches.dropna(subset = cols)\n",
    "print(match_data.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now, using the information from the matches and players, we are going to create features based on the FIFA attributes. This computation is heavy, so we are going to save it the first time we create it.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(21374, 23)\n",
      "CPU times: user 0 ns, sys: 0 ns, total: 0 ns\n",
      "Wall time: 2.84 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "fifa_data_filename = 'fifa_data.pk'\n",
    "if os.path.isfile(fifa_data_filename):\n",
    "    fifa_data = pd.read_pickle(fifa_data_filename)\n",
    "else:\n",
    "    fifa_data = get_fifa_data(match_data, players)\n",
    "    fifa_data.to_pickle(fifa_data_filename)\n",
    "print(fifa_data.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Finally, we are going to compute the features and labels. The labels are related to the result of the team playing at home, they are: `Win`, `Draw`, `Defeat`. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating match features...\n",
      "Generating match labels...\n",
      "Generating bookkeeper data...\n",
      "(19673, 48)\n",
      "CPU times: user 11min 41s, sys: 2.46 s, total: 11min 43s\n",
      "Wall time: 11min 51s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "bk_cols = ['B365', 'BW', 'IW', 'LB', 'PS', 'WH', 'SJ', 'VC', 'GB', 'BS']\n",
    "bk_cols_selected = ['B365', 'BW']      \n",
    "feables = create_feables(match_data, fifa_data, bk_cols_selected, get_overall = True)\n",
    "print(feables.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>match_api_id</th>\n",
       "      <th>home_team_goals_difference</th>\n",
       "      <th>away_team_goals_difference</th>\n",
       "      <th>games_won_home_team</th>\n",
       "      <th>games_won_away_team</th>\n",
       "      <th>games_against_won</th>\n",
       "      <th>games_against_lost</th>\n",
       "      <th>season</th>\n",
       "      <th>League_1.0</th>\n",
       "      <th>League_1729.0</th>\n",
       "      <th>...</th>\n",
       "      <th>away_player_9_overall_rating</th>\n",
       "      <th>away_player_10_overall_rating</th>\n",
       "      <th>away_player_11_overall_rating</th>\n",
       "      <th>B365_Win</th>\n",
       "      <th>B365_Draw</th>\n",
       "      <th>B365_Defeat</th>\n",
       "      <th>BW_Win</th>\n",
       "      <th>BW_Draw</th>\n",
       "      <th>BW_Defeat</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>493017.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>70.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>63.0</td>\n",
       "      <td>0.313804</td>\n",
       "      <td>0.276886</td>\n",
       "      <td>0.409310</td>\n",
       "      <td>0.307825</td>\n",
       "      <td>0.279410</td>\n",
       "      <td>0.412765</td>\n",
       "      <td>Win</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>493025.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>67.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>0.327179</td>\n",
       "      <td>0.286281</td>\n",
       "      <td>0.386540</td>\n",
       "      <td>0.290493</td>\n",
       "      <td>0.300176</td>\n",
       "      <td>0.409331</td>\n",
       "      <td>Defeat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>493027.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>55.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>0.672897</td>\n",
       "      <td>0.209346</td>\n",
       "      <td>0.117757</td>\n",
       "      <td>0.672269</td>\n",
       "      <td>0.226891</td>\n",
       "      <td>0.100840</td>\n",
       "      <td>Win</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>493034.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>74.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>69.0</td>\n",
       "      <td>0.207407</td>\n",
       "      <td>0.259259</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.192717</td>\n",
       "      <td>0.274476</td>\n",
       "      <td>0.532807</td>\n",
       "      <td>Win</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>493040.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>60.0</td>\n",
       "      <td>63.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>0.535211</td>\n",
       "      <td>0.267606</td>\n",
       "      <td>0.197183</td>\n",
       "      <td>0.565759</td>\n",
       "      <td>0.254990</td>\n",
       "      <td>0.179250</td>\n",
       "      <td>Draw</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 48 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   match_api_id  home_team_goals_difference  away_team_goals_difference  \\\n",
       "0      493017.0                         0.0                         0.0   \n",
       "1      493025.0                         0.0                         0.0   \n",
       "2      493027.0                         0.0                         0.0   \n",
       "3      493034.0                         1.0                         2.0   \n",
       "4      493040.0                        -2.0                         0.0   \n",
       "\n",
       "   games_won_home_team  games_won_away_team  games_against_won  \\\n",
       "0                  0.0                  0.0                0.0   \n",
       "1                  0.0                  0.0                0.0   \n",
       "2                  0.0                  0.0                0.0   \n",
       "3                  1.0                  1.0                0.0   \n",
       "4                  0.0                  0.0                0.0   \n",
       "\n",
       "   games_against_lost  season  League_1.0  League_1729.0   ...    \\\n",
       "0                 0.0  2008.0           1              0   ...     \n",
       "1                 0.0  2008.0           1              0   ...     \n",
       "2                 0.0  2008.0           1              0   ...     \n",
       "3                 0.0  2008.0           1              0   ...     \n",
       "4                 0.0  2008.0           1              0   ...     \n",
       "\n",
       "   away_player_9_overall_rating  away_player_10_overall_rating  \\\n",
       "0                          70.0                           68.0   \n",
       "1                          67.0                           73.0   \n",
       "2                          55.0                           58.0   \n",
       "3                          74.0                           70.0   \n",
       "4                          60.0                           63.0   \n",
       "\n",
       "   away_player_11_overall_rating  B365_Win  B365_Draw  B365_Defeat    BW_Win  \\\n",
       "0                           63.0  0.313804   0.276886     0.409310  0.307825   \n",
       "1                           68.0  0.327179   0.286281     0.386540  0.290493   \n",
       "2                           64.0  0.672897   0.209346     0.117757  0.672269   \n",
       "3                           69.0  0.207407   0.259259     0.533333  0.192717   \n",
       "4                           65.0  0.535211   0.267606     0.197183  0.565759   \n",
       "\n",
       "    BW_Draw  BW_Defeat   label  \n",
       "0  0.279410   0.412765     Win  \n",
       "1  0.300176   0.409331  Defeat  \n",
       "2  0.226891   0.100840     Win  \n",
       "3  0.274476   0.532807     Win  \n",
       "4  0.254990   0.179250    Draw  \n",
       "\n",
       "[5 rows x 48 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feables.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's now split features and labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(19673, 46)\n",
      "(19673,)\n"
     ]
    }
   ],
   "source": [
    "features = feables[feables.columns.difference(['match_api_id', 'label'])]\n",
    "labs = feables['label']\n",
    "print(features.shape)\n",
    "print(labs.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Once we have the features and labels defined, let's create the train and test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(features, labs, test_size=0.2, random_state=42, stratify=labs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### XGBoost analysis\n",
    "Once we have done the feature engineering step, we can start to train with each of the libraries. We will start with XGBoost. \n",
    "\n",
    "We are going to save the training and test time, as well as some metrics. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of processors:  24\n"
     ]
    }
   ],
   "source": [
    "number_processors = get_number_processors()\n",
    "print(\"Number of processors: \", number_processors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict = dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "clf = XGBClassifier(max_depth=3,\n",
    "                    n_estimators=300,\n",
    "                    min_child_weight=5,\n",
    "                    learning_rate=0.1,\n",
    "                    colsample_bytree=0.80,\n",
    "                    scale_pos_weight=2,\n",
    "                    gamma=0.1,\n",
    "                    reg_lambda=1,\n",
    "                    subsample=1,\n",
    "                    n_jobs=number_processors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer() as t:\n",
    "    clf.fit(X_train, y_train, verbose=False, eval_metric='merror')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['xgb']={\n",
    "    'train_time': t.interval\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer()as t:\n",
    "    y_pred = clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['xgb']['test_time'] = t.interval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "labels = [\"Win\", \"Draw\", \"Defeat\"]\n",
    "report = classification_metrics_multilabel(y_test, y_pred, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['xgb']['performance'] = report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "clf = XGBClassifier(max_depth=0,\n",
    "                    n_estimators=300,\n",
    "                    min_child_weight=5,\n",
    "                    learning_rate=0.1,\n",
    "                    colsample_bytree=0.80,\n",
    "                    scale_pos_weight=2,\n",
    "                    gamma=0.1,\n",
    "                    reg_lambda=1,\n",
    "                    subsample=1,\n",
    "                    max_leaves=2**3,\n",
    "                    grow_policy='lossguide',\n",
    "                    tree_method='hist',\n",
    "                    n_jobs=number_processors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer() as t:\n",
    "    clf.fit(X_train, y_train, verbose=False, eval_metric='merror')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['xgb_hist']={\n",
    "    'train_time': t.interval\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer()as t:\n",
    "    y_pred = clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['xgb_hist']['test_time'] = t.interval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "report = classification_metrics_multilabel(y_test, y_pred, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['xgb_hist']['performance'] = report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### LightGBM analysis\n",
    "\n",
    "Now let's compare with LightGBM."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "clf = LGBMClassifier(num_leaves=2**3,\n",
    "                      n_estimators=300,\n",
    "                      min_child_weight=5,\n",
    "                      learning_rate=0.1,\n",
    "                      colsample_bytree=0.80,\n",
    "                      scale_pos_weight=2,\n",
    "                      min_split_gain=0.1,\n",
    "                      reg_lambda=1,\n",
    "                      subsample=1,\n",
    "                      nthread=number_processors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer()as t:\n",
    "    clf.fit(X_train, y_train, verbose=False, eval_metric='multi_error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['lgbm']={\n",
    "    'train_time': t.interval\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer() as t:\n",
    "    y_pred = clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['lgbm']['test_time'] = t.interval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "report = classification_metrics_multilabel(y_test, y_pred, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['lgbm']['performance'] = report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Finally, the results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"lgbm\": {\n",
      "        \"performance\": {\n",
      "            \"Accuracy\": 0.5273189326556544,\n",
      "            \"F1\": 0.4594968503939465,\n",
      "            \"Precision\": 0.4627647374365573,\n",
      "            \"Recall\": 0.5273189326556544\n",
      "        },\n",
      "        \"test_time\": 0.01769829506520182,\n",
      "        \"train_time\": 0.5824753509368747\n",
      "    },\n",
      "    \"xgb\": {\n",
      "        \"performance\": {\n",
      "            \"Accuracy\": 0.5285895806861499,\n",
      "            \"F1\": 0.4581501758019101,\n",
      "            \"Precision\": 0.4728104942149788,\n",
      "            \"Recall\": 0.5285895806861499\n",
      "        },\n",
      "        \"test_time\": 0.01739849301520735,\n",
      "        \"train_time\": 2.277035239036195\n",
      "    },\n",
      "    \"xgb_hist\": {\n",
      "        \"performance\": {\n",
      "            \"Accuracy\": 0.5285895806861499,\n",
      "            \"F1\": 0.46087114582653593,\n",
      "            \"Precision\": 0.4747655220586142,\n",
      "            \"Recall\": 0.5285895806861499\n",
      "        },\n",
      "        \"test_time\": 0.013859358034096658,\n",
      "        \"train_time\": 2.4757352949818596\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Results\n",
    "print(json.dumps(results_dict, indent=4, sort_keys=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "As it can be seen, in the case of multilabel LightGBM is faster than XGBoost in both versions. The performance metrics are really poor, so we wouldn't recommend to bet based on this algorithm :-)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "## 2) Concept drift\n",
    "\n",
    "In this section we are trying to find concept drift in the dataset to check if retraining is valuable.\n",
    "\n",
    "### Data management\n",
    "\n",
    "For this experiment we are going to focus in just one league. We selected the Spanish league _La Liga_, which is the one that shows the highest concept drift. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3040, 115)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>country_id</th>\n",
       "      <th>league_id</th>\n",
       "      <th>season</th>\n",
       "      <th>stage</th>\n",
       "      <th>date</th>\n",
       "      <th>match_api_id</th>\n",
       "      <th>home_team_api_id</th>\n",
       "      <th>away_team_api_id</th>\n",
       "      <th>home_team_goal</th>\n",
       "      <th>...</th>\n",
       "      <th>SJA</th>\n",
       "      <th>VCH</th>\n",
       "      <th>VCD</th>\n",
       "      <th>VCA</th>\n",
       "      <th>GBH</th>\n",
       "      <th>GBD</th>\n",
       "      <th>GBA</th>\n",
       "      <th>BSH</th>\n",
       "      <th>BSD</th>\n",
       "      <th>BSA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>21517</th>\n",
       "      <td>21518</td>\n",
       "      <td>21518</td>\n",
       "      <td>21518</td>\n",
       "      <td>2008/2009</td>\n",
       "      <td>1</td>\n",
       "      <td>2008-08-30 00:00:00</td>\n",
       "      <td>530023</td>\n",
       "      <td>10267</td>\n",
       "      <td>8661</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>5.00</td>\n",
       "      <td>1.65</td>\n",
       "      <td>3.40</td>\n",
       "      <td>4.50</td>\n",
       "      <td>1.75</td>\n",
       "      <td>3.30</td>\n",
       "      <td>5.00</td>\n",
       "      <td>1.67</td>\n",
       "      <td>3.40</td>\n",
       "      <td>5.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21518</th>\n",
       "      <td>21519</td>\n",
       "      <td>21518</td>\n",
       "      <td>21518</td>\n",
       "      <td>2008/2009</td>\n",
       "      <td>1</td>\n",
       "      <td>2008-08-31 00:00:00</td>\n",
       "      <td>530084</td>\n",
       "      <td>8371</td>\n",
       "      <td>10205</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2.60</td>\n",
       "      <td>2.90</td>\n",
       "      <td>3.20</td>\n",
       "      <td>2.25</td>\n",
       "      <td>2.75</td>\n",
       "      <td>3.20</td>\n",
       "      <td>2.50</td>\n",
       "      <td>2.80</td>\n",
       "      <td>3.25</td>\n",
       "      <td>2.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21519</th>\n",
       "      <td>21520</td>\n",
       "      <td>21518</td>\n",
       "      <td>21518</td>\n",
       "      <td>2008/2009</td>\n",
       "      <td>1</td>\n",
       "      <td>2008-08-31 00:00:00</td>\n",
       "      <td>530085</td>\n",
       "      <td>9783</td>\n",
       "      <td>8633</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>1.90</td>\n",
       "      <td>3.25</td>\n",
       "      <td>3.25</td>\n",
       "      <td>2.00</td>\n",
       "      <td>3.50</td>\n",
       "      <td>3.20</td>\n",
       "      <td>2.10</td>\n",
       "      <td>3.25</td>\n",
       "      <td>3.25</td>\n",
       "      <td>2.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21520</th>\n",
       "      <td>21521</td>\n",
       "      <td>21518</td>\n",
       "      <td>21518</td>\n",
       "      <td>2008/2009</td>\n",
       "      <td>1</td>\n",
       "      <td>2008-08-31 00:00:00</td>\n",
       "      <td>530086</td>\n",
       "      <td>8388</td>\n",
       "      <td>8634</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1.57</td>\n",
       "      <td>7.00</td>\n",
       "      <td>3.75</td>\n",
       "      <td>1.45</td>\n",
       "      <td>6.50</td>\n",
       "      <td>3.75</td>\n",
       "      <td>1.53</td>\n",
       "      <td>5.50</td>\n",
       "      <td>3.75</td>\n",
       "      <td>1.53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21521</th>\n",
       "      <td>21522</td>\n",
       "      <td>21518</td>\n",
       "      <td>21518</td>\n",
       "      <td>2008/2009</td>\n",
       "      <td>1</td>\n",
       "      <td>2008-08-31 00:00:00</td>\n",
       "      <td>530087</td>\n",
       "      <td>8696</td>\n",
       "      <td>8302</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2.30</td>\n",
       "      <td>2.70</td>\n",
       "      <td>3.10</td>\n",
       "      <td>2.40</td>\n",
       "      <td>2.75</td>\n",
       "      <td>3.25</td>\n",
       "      <td>2.45</td>\n",
       "      <td>2.70</td>\n",
       "      <td>3.25</td>\n",
       "      <td>2.38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 115 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  country_id  league_id     season  stage                 date  \\\n",
       "21517  21518       21518      21518  2008/2009      1  2008-08-30 00:00:00   \n",
       "21518  21519       21518      21518  2008/2009      1  2008-08-31 00:00:00   \n",
       "21519  21520       21518      21518  2008/2009      1  2008-08-31 00:00:00   \n",
       "21520  21521       21518      21518  2008/2009      1  2008-08-31 00:00:00   \n",
       "21521  21522       21518      21518  2008/2009      1  2008-08-31 00:00:00   \n",
       "\n",
       "       match_api_id  home_team_api_id  away_team_api_id  home_team_goal  ...   \\\n",
       "21517        530023             10267              8661               3  ...    \n",
       "21518        530084              8371             10205               1  ...    \n",
       "21519        530085              9783              8633               2  ...    \n",
       "21520        530086              8388              8634               1  ...    \n",
       "21521        530087              8696              8302               1  ...    \n",
       "\n",
       "        SJA   VCH   VCD   VCA   GBH   GBD   GBA   BSH   BSD   BSA  \n",
       "21517  5.00  1.65  3.40  4.50  1.75  3.30  5.00  1.67  3.40  5.00  \n",
       "21518  2.60  2.90  3.20  2.25  2.75  3.20  2.50  2.80  3.25  2.30  \n",
       "21519  1.90  3.25  3.25  2.00  3.50  3.20  2.10  3.25  3.25  2.10  \n",
       "21520  1.57  7.00  3.75  1.45  6.50  3.75  1.53  5.50  3.75  1.53  \n",
       "21521  2.30  2.70  3.10  2.40  2.75  3.25  2.45  2.70  3.25  2.38  \n",
       "\n",
       "[5 rows x 115 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "la_liga_id = 21518\n",
    "matches_target = matches[matches['league_id'] == la_liga_id]\n",
    "print(matches_target.shape)\n",
    "matches_target.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2707, 115)\n"
     ]
    }
   ],
   "source": [
    "cols = [\"country_id\", \"league_id\", \"season\", \"stage\", \"date\", \"match_api_id\", \"home_team_api_id\", \n",
    "        \"away_team_api_id\", \"home_team_goal\", \"away_team_goal\", \"home_player_1\", \"home_player_2\",\n",
    "        \"home_player_3\", \"home_player_4\", \"home_player_5\", \"home_player_6\", \"home_player_7\", \n",
    "        \"home_player_8\", \"home_player_9\", \"home_player_10\", \"home_player_11\", \"away_player_1\",\n",
    "        \"away_player_2\", \"away_player_3\", \"away_player_4\", \"away_player_5\", \"away_player_6\",\n",
    "        \"away_player_7\", \"away_player_8\", \"away_player_9\", \"away_player_10\", \"away_player_11\"]\n",
    "match_data = matches_target.dropna(subset = cols)\n",
    "print(match_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3min 44s, sys: 764 ms, total: 3min 44s\n",
      "Wall time: 3min 47s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "fifa_data = get_fifa_data(match_data, players)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns', None) #show all columns in pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2707, 23)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>home_player_1_overall_rating</th>\n",
       "      <th>home_player_2_overall_rating</th>\n",
       "      <th>home_player_3_overall_rating</th>\n",
       "      <th>home_player_4_overall_rating</th>\n",
       "      <th>home_player_5_overall_rating</th>\n",
       "      <th>home_player_6_overall_rating</th>\n",
       "      <th>home_player_7_overall_rating</th>\n",
       "      <th>home_player_8_overall_rating</th>\n",
       "      <th>home_player_9_overall_rating</th>\n",
       "      <th>home_player_10_overall_rating</th>\n",
       "      <th>home_player_11_overall_rating</th>\n",
       "      <th>away_player_1_overall_rating</th>\n",
       "      <th>away_player_2_overall_rating</th>\n",
       "      <th>away_player_3_overall_rating</th>\n",
       "      <th>away_player_4_overall_rating</th>\n",
       "      <th>away_player_5_overall_rating</th>\n",
       "      <th>away_player_6_overall_rating</th>\n",
       "      <th>away_player_7_overall_rating</th>\n",
       "      <th>away_player_8_overall_rating</th>\n",
       "      <th>away_player_9_overall_rating</th>\n",
       "      <th>away_player_10_overall_rating</th>\n",
       "      <th>away_player_11_overall_rating</th>\n",
       "      <th>match_api_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>21517</th>\n",
       "      <td>65.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>85.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>72.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>530023.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21518</th>\n",
       "      <td>75.0</td>\n",
       "      <td>63.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>72.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>75.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>530084.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21519</th>\n",
       "      <td>74.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>69.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>75.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>85.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>75.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>83.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>86.0</td>\n",
       "      <td>530085.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21521</th>\n",
       "      <td>79.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>72.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>83.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>83.0</td>\n",
       "      <td>86.0</td>\n",
       "      <td>530087.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21523</th>\n",
       "      <td>64.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>63.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>69.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>75.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>72.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>72.0</td>\n",
       "      <td>72.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>530089.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       home_player_1_overall_rating  home_player_2_overall_rating  \\\n",
       "21517                          65.0                          80.0   \n",
       "21518                          75.0                          63.0   \n",
       "21519                          74.0                          65.0   \n",
       "21521                          79.0                          74.0   \n",
       "21523                          64.0                          79.0   \n",
       "\n",
       "       home_player_3_overall_rating  home_player_4_overall_rating  \\\n",
       "21517                          82.0                          79.0   \n",
       "21518                          71.0                          65.0   \n",
       "21519                          76.0                          78.0   \n",
       "21521                          82.0                          56.0   \n",
       "21523                          68.0                          67.0   \n",
       "\n",
       "       home_player_5_overall_rating  home_player_6_overall_rating  \\\n",
       "21517                          76.0                          80.0   \n",
       "21518                          68.0                          73.0   \n",
       "21519                          74.0                          69.0   \n",
       "21521                          57.0                          79.0   \n",
       "21523                          65.0                          77.0   \n",
       "\n",
       "       home_player_7_overall_rating  home_player_8_overall_rating  \\\n",
       "21517                          79.0                          67.0   \n",
       "21518                          74.0                          72.0   \n",
       "21519                          74.0                          77.0   \n",
       "21521                          79.0                          68.0   \n",
       "21523                          63.0                          79.0   \n",
       "\n",
       "       home_player_9_overall_rating  home_player_10_overall_rating  \\\n",
       "21517                          85.0                           68.0   \n",
       "21518                          70.0                           71.0   \n",
       "21519                          79.0                           71.0   \n",
       "21521                          74.0                           82.0   \n",
       "21523                          73.0                           69.0   \n",
       "\n",
       "       home_player_11_overall_rating  away_player_1_overall_rating  \\\n",
       "21517                           89.0                          67.0   \n",
       "21518                           68.0                          78.0   \n",
       "21519                           75.0                          91.0   \n",
       "21521                           72.0                          82.0   \n",
       "21523                           76.0                          71.0   \n",
       "\n",
       "       away_player_2_overall_rating  away_player_3_overall_rating  \\\n",
       "21517                          74.0                          71.0   \n",
       "21518                          73.0                          73.0   \n",
       "21519                          85.0                          84.0   \n",
       "21521                          76.0                          81.0   \n",
       "21523                          74.0                          75.0   \n",
       "\n",
       "       away_player_4_overall_rating  away_player_5_overall_rating  \\\n",
       "21517                          76.0                          72.0   \n",
       "21518                          77.0                          80.0   \n",
       "21519                          80.0                          75.0   \n",
       "21521                          68.0                          77.0   \n",
       "21523                          73.0                          71.0   \n",
       "\n",
       "       away_player_6_overall_rating  away_player_7_overall_rating  \\\n",
       "21517                          71.0                          76.0   \n",
       "21518                          80.0                          75.0   \n",
       "21519                          80.0                          83.0   \n",
       "21521                          83.0                          74.0   \n",
       "21523                          74.0                          72.0   \n",
       "\n",
       "       away_player_8_overall_rating  away_player_9_overall_rating  \\\n",
       "21517                          76.0                          78.0   \n",
       "21518                          84.0                          79.0   \n",
       "21519                          77.0                          82.0   \n",
       "21521                          80.0                          78.0   \n",
       "21523                          76.0                          72.0   \n",
       "\n",
       "       away_player_10_overall_rating  away_player_11_overall_rating  \\\n",
       "21517                           74.0                           77.0   \n",
       "21518                           77.0                           78.0   \n",
       "21519                           84.0                           86.0   \n",
       "21521                           83.0                           86.0   \n",
       "21523                           72.0                           70.0   \n",
       "\n",
       "       match_api_id  \n",
       "21517      530023.0  \n",
       "21518      530084.0  \n",
       "21519      530085.0  \n",
       "21521      530087.0  \n",
       "21523      530089.0  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(fifa_data.shape)\n",
    "fifa_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating match features...\n",
      "Generating match labels...\n",
      "Generating bookkeeper data...\n",
      "(2706, 38)\n",
      "CPU times: user 1min 30s, sys: 396 ms, total: 1min 30s\n",
      "Wall time: 1min 32s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "bk_cols = ['B365', 'BW', 'IW', 'LB', 'PS', 'WH', 'SJ', 'VC', 'GB', 'BS']\n",
    "bk_cols_selected = ['B365', 'BW']      \n",
    "feables = create_feables(match_data, fifa_data, bk_cols_selected, get_overall = True, all_leagues = False)\n",
    "print(feables.shape)\n",
    "feables.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "We are going to split the dataset into three groups: first seasons from 2008 until 2014. This will be our base dataset. Then we will create two more, one for season 2014/2015 and another for 2014/2015."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1991, 38)\n",
      "(364, 38)\n",
      "(351, 38)\n"
     ]
    }
   ],
   "source": [
    "feables_rest = feables[(feables['season'] != 2014) & (feables['season'] != 2015)]\n",
    "print(feables_rest.shape)\n",
    "feables_2014_2015 = feables[feables['season'] == 2014]\n",
    "print(feables_2014_2015.shape)\n",
    "feables_2015_2016 = feables[feables['season'] == 2015]\n",
    "print(feables_2015_2016.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1991, 35)\n",
      "(364, 35)\n",
      "(351, 35)\n"
     ]
    }
   ],
   "source": [
    "X_train = feables_rest[feables_rest.columns.difference(['match_api_id', 'label', 'season'])]\n",
    "y_train = feables_rest['label']\n",
    "X_test1 = feables_2014_2015[feables_rest.columns.difference(['match_api_id', 'label', 'season'])]\n",
    "y_test1 = feables_2014_2015['label']\n",
    "X_test2 = feables_2015_2016[feables_rest.columns.difference(['match_api_id', 'label', 'season'])]\n",
    "y_test2 = feables_2015_2016['label']\n",
    "\n",
    "print(X_train.shape)\n",
    "print(X_test1.shape)\n",
    "print(X_test2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2355, 38)\n"
     ]
    }
   ],
   "source": [
    "feables_up_to_2014_2015 = feables[feables['season'] != 2015]\n",
    "print(feables_up_to_2014_2015.shape)\n",
    "X_train2 = feables_up_to_2014_2015[feables_up_to_2014_2015.columns.difference(['match_api_id', 'label', 'season'])]\n",
    "y_train2 = feables_up_to_2014_2015['label']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Analysis with XGBoost\n",
    "\n",
    "To identify concept drift we need to find a difference in performance over time. First, we are going to train a classifier on the base dataset and predict on the seasons 2014/2015 and 2014/2015."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "clf = XGBClassifier(max_depth=8, \n",
    "                    learning_rate=0.1, \n",
    "                    scale_pos_weight=2,\n",
    "                    min_child_weight=5,\n",
    "                    n_estimators=100,\n",
    "                    subsample=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.05 s, sys: 0 ns, total: 2.05 s\n",
      "Wall time: 2.05 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=8, min_child_weight=5, missing=None, n_estimators=100,\n",
       "       n_jobs=1, nthread=1, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=2, seed=0, silent=True,\n",
       "       subsample=1)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "clf.fit(X_train, y_train, verbose=True, eval_metric='merror')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score of XGBClassifier for training set: 0.9965.\n",
      "Score of XGBClassifier for test set: 0.5220.\n"
     ]
    }
   ],
   "source": [
    "y_pred1 = clf.predict(X_test1)\n",
    "acc_xgb1 = accuracy_score(y_test1, y_pred1)\n",
    "print(\"Score of {} for training set: {:.4f}.\".format(clf.__class__.__name__, accuracy_score(y_train, clf.predict(X_train))))\n",
    "print(\"Score of {} for test set: {:.4f}.\".format(clf.__class__.__name__, acc_xgb1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score of XGBClassifier for training set: 0.9965.\n",
      "Score of XGBClassifier for test set: 0.4872.\n"
     ]
    }
   ],
   "source": [
    "y_pred2 = clf.predict(X_test2)\n",
    "acc_xgb2 = accuracy_score(y_test2, y_pred2)\n",
    "print(\"Score of {} for training set: {:.4f}.\".format(clf.__class__.__name__, accuracy_score(y_train, clf.predict(X_train))))\n",
    "print(\"Score of {} for test set: {:.4f}.\".format(clf.__class__.__name__, acc_xgb2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "We observe that there is a reduction of the accuracy across the seasons. Now let's train on a dataset containing all seasons except 2015/2016."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.41 s, sys: 0 ns, total: 2.41 s\n",
      "Wall time: 2.47 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=8, min_child_weight=5, missing=None, n_estimators=100,\n",
       "       n_jobs=1, nthread=1, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=2, seed=0, silent=True,\n",
       "       subsample=1)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "clf.fit(X_train2, y_train2, verbose=True, eval_metric='merror')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score of XGBClassifier for training set: 0.9919.\n",
      "Score of XGBClassifier for test set: 0.5242.\n"
     ]
    }
   ],
   "source": [
    "y_pred2 = clf.predict(X_test2)\n",
    "acc_xgb2_retrain = accuracy_score(y_test2, y_pred2)\n",
    "print(\"Score of {} for training set: {:.4f}.\".format(clf.__class__.__name__, accuracy_score(y_train2, clf.predict(X_train2))))\n",
    "print(\"Score of {} for test set: {:.4f}.\".format(clf.__class__.__name__, acc_xgb2_retrain))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "We observe that we performance has improved. \n",
    "\n",
    "It is important to note that this dataset is very noisy and the metrics are low. Therefore, there is a possibility for the classifier to learn the noise rather than the real data. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Analysis with LightGBM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now let's try with LightGBM. We perform that exact same steps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "clf = LGBMClassifier(num_leaves=255,\n",
    "                    learning_rate=0.1, \n",
    "                    scale_pos_weight=2,\n",
    "                    min_child_weight=5,\n",
    "                    n_estimators=100,\n",
    "                    subsample=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 944 ms, sys: 8 ms, total: 952 ms\n",
      "Wall time: 951 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(boosting_type='gbdt', colsample_bytree=1, learning_rate=0.1,\n",
       "        max_bin=255, max_depth=-1, min_child_samples=10,\n",
       "        min_child_weight=5, min_split_gain=0, n_estimators=100, nthread=-1,\n",
       "        num_leaves=255, objective='multiclass', reg_alpha=0, reg_lambda=0,\n",
       "        scale_pos_weight=2, seed=0, silent=True, subsample=1,\n",
       "        subsample_for_bin=50000, subsample_freq=1)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "clf.fit(X_train, y_train, verbose=True, eval_metric='multi_error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score of LGBMClassifier for training set: 1.0000.\n",
      "Score of LGBMClassifier for test set: 0.5165.\n"
     ]
    }
   ],
   "source": [
    "y_pred1 = clf.predict(X_test1)\n",
    "acc_lgbm1 = accuracy_score(y_test1, y_pred1)\n",
    "print(\"Score of {} for training set: {:.4f}.\".format(clf.__class__.__name__, accuracy_score(y_train, clf.predict(X_train))))\n",
    "print(\"Score of {} for test set: {:.4f}.\".format(clf.__class__.__name__, acc_lgbm1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score of LGBMClassifier for training set: 1.0000.\n",
      "Score of LGBMClassifier for test set: 0.5128.\n"
     ]
    }
   ],
   "source": [
    "y_pred2 = clf.predict(X_test2)\n",
    "acc_lgbm2 = accuracy_score(y_test2, y_pred2)\n",
    "print(\"Score of {} for training set: {:.4f}.\".format(clf.__class__.__name__, accuracy_score(y_train, clf.predict(X_train))))\n",
    "print(\"Score of {} for test set: {:.4f}.\".format(clf.__class__.__name__, acc_lgbm2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "And now we retrain in the dataset of seasons until 2015. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.11 s, sys: 0 ns, total: 1.11 s\n",
      "Wall time: 1.11 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(boosting_type='gbdt', colsample_bytree=1, learning_rate=0.1,\n",
       "        max_bin=255, max_depth=-1, min_child_samples=10,\n",
       "        min_child_weight=5, min_split_gain=0, n_estimators=100, nthread=-1,\n",
       "        num_leaves=255, objective='multiclass', reg_alpha=0, reg_lambda=0,\n",
       "        scale_pos_weight=2, seed=0, silent=True, subsample=1,\n",
       "        subsample_for_bin=50000, subsample_freq=1)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "clf.fit(X_train2, y_train2, verbose=True, eval_metric='multi_error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score of LGBMClassifier for training set: 1.0000.\n",
      "Score of LGBMClassifier for test set: 0.5556.\n"
     ]
    }
   ],
   "source": [
    "y_pred2 = clf.predict(X_test2)\n",
    "acc_lgbm2_retrain = accuracy_score(y_test2, y_pred2)\n",
    "print(\"Score of {} for training set: {:.4f}.\".format(clf.__class__.__name__, accuracy_score(y_train2, clf.predict(X_train2))))\n",
    "print(\"Score of {} for test set: {:.4f}.\".format(clf.__class__.__name__, accuracy_score(y_test2, y_pred2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Again we observe an improvement in the accuracy."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Plot\n",
    "Finally, let's plot the results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "@curry\n",
    "def change_width(width, glyph_rend):\n",
    "    glyph_rend.data_source.data['width']=[0.3]\n",
    "\n",
    "def is_bar(glyph_rend):\n",
    "    if isinstance(glyph_rend, GlyphRenderer) and isinstance(glyph_rend.glyph, Rect):\n",
    "        return True\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "xgb_metrics = [acc_xgb1, acc_xgb2, acc_xgb2_retrain]\n",
    "lgbm_metrics =[acc_lgbm1, acc_lgbm2, acc_lgbm2_retrain]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "data = {\n",
    "    'Accuracy': xgb_metrics+lgbm_metrics,\n",
    "    'classifier': ['XGBoost']*3+['LightGBM']*3,\n",
    "    'label': ['2014/2015','2015/2016','2015/2016 retrained']*2\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <div class=\"bk-plotdiv\" id=\"43c57489-e7f8-4f86-844a-a159a0f153bc\"></div>\n",
       "    </div>\n",
       "<script type=\"text/javascript\">\n",
       "  \n",
       "  (function(global) {\n",
       "    function now() {\n",
       "      return new Date();\n",
       "    }\n",
       "  \n",
       "    var force = false;\n",
       "  \n",
       "    if (typeof (window._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n",
       "      window._bokeh_onload_callbacks = [];\n",
       "      window._bokeh_is_loading = undefined;\n",
       "    }\n",
       "  \n",
       "  \n",
       "    \n",
       "    if (typeof (window._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "      window._bokeh_timeout = Date.now() + 0;\n",
       "      window._bokeh_failed_load = false;\n",
       "    }\n",
       "  \n",
       "    var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "       \"<div style='background-color: #fdd'>\\n\"+\n",
       "       \"<p>\\n\"+\n",
       "       \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "       \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "       \"</p>\\n\"+\n",
       "       \"<ul>\\n\"+\n",
       "       \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "       \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "       \"</ul>\\n\"+\n",
       "       \"<code>\\n\"+\n",
       "       \"from bokeh.resources import INLINE\\n\"+\n",
       "       \"output_notebook(resources=INLINE)\\n\"+\n",
       "       \"</code>\\n\"+\n",
       "       \"</div>\"}};\n",
       "  \n",
       "    function display_loaded() {\n",
       "      if (window.Bokeh !== undefined) {\n",
       "        var el = document.getElementById(\"43c57489-e7f8-4f86-844a-a159a0f153bc\");\n",
       "        el.textContent = \"BokehJS \" + Bokeh.version + \" successfully loaded.\";\n",
       "      } else if (Date.now() < window._bokeh_timeout) {\n",
       "        setTimeout(display_loaded, 100)\n",
       "      }\n",
       "    }\n",
       "  \n",
       "    function run_callbacks() {\n",
       "      try {\n",
       "        window._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n",
       "      }\n",
       "      finally {\n",
       "        delete window._bokeh_onload_callbacks\n",
       "      }\n",
       "      console.info(\"Bokeh: all callbacks have finished\");\n",
       "    }\n",
       "  \n",
       "    function load_libs(js_urls, callback) {\n",
       "      window._bokeh_onload_callbacks.push(callback);\n",
       "      if (window._bokeh_is_loading > 0) {\n",
       "        console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "        return null;\n",
       "      }\n",
       "      if (js_urls == null || js_urls.length === 0) {\n",
       "        run_callbacks();\n",
       "        return null;\n",
       "      }\n",
       "      console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "      window._bokeh_is_loading = js_urls.length;\n",
       "      for (var i = 0; i < js_urls.length; i++) {\n",
       "        var url = js_urls[i];\n",
       "        var s = document.createElement('script');\n",
       "        s.src = url;\n",
       "        s.async = false;\n",
       "        s.onreadystatechange = s.onload = function() {\n",
       "          window._bokeh_is_loading--;\n",
       "          if (window._bokeh_is_loading === 0) {\n",
       "            console.log(\"Bokeh: all BokehJS libraries loaded\");\n",
       "            run_callbacks()\n",
       "          }\n",
       "        };\n",
       "        s.onerror = function() {\n",
       "          console.warn(\"failed to load library \" + url);\n",
       "        };\n",
       "        console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "        document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "      }\n",
       "    };var element = document.getElementById(\"43c57489-e7f8-4f86-844a-a159a0f153bc\");\n",
       "    if (element == null) {\n",
       "      console.log(\"Bokeh: ERROR: autoload.js configured with elementid '43c57489-e7f8-4f86-844a-a159a0f153bc' but no matching script tag was found. \")\n",
       "      return false;\n",
       "    }\n",
       "  \n",
       "    var js_urls = [];\n",
       "  \n",
       "    var inline_js = [\n",
       "      function(Bokeh) {\n",
       "        (function() {\n",
       "          var fn = function() {\n",
       "            var docs_json = {\"7b97c6a2-00f5-4724-bbb6-035b15a35e94\":{\"roots\":{\"references\":[{\"attributes\":{\"callback\":null,\"column_names\":[\"y\",\"line_color\",\"x\",\"fill_alpha\",\"width\",\"color\",\"height\",\"line_alpha\",\"label\"],\"data\":{\"chart_index\":[{\"classifier\":\"LightGBM\",\"label\":\"2015/2016 retrained\"}],\"classifier\":[\"LightGBM\"],\"color\":[\"#5f9e6f\"],\"fill_alpha\":[0.8],\"height\":[0.5555555555555556],\"label\":[\"2015/2016 retrained\"],\"line_alpha\":[1.0],\"line_color\":[\"white\"],\"width\":[0.3],\"x\":[\"2015/2016 retrained:0.666666666667\"],\"y\":[0.2777777777777778]}},\"id\":\"116d9f58-e823-4dd2-bbf8-9443348e16e8\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"label\":{\"value\":\"LightGBM\"},\"renderers\":[{\"id\":\"d9796aaa-6ea5-4f80-93a0-a485df92d9c9\",\"type\":\"GlyphRenderer\"}]},\"id\":\"09baff80-51bb-4e08-bf25-4cd2aaa788e5\",\"type\":\"LegendItem\"},{\"attributes\":{\"data_source\":{\"id\":\"c6af2b99-48d3-4e38-998f-8c4314b9347c\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"924144dd-7f75-47cd-a1f7-04779b0aada8\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null},\"id\":\"8a3d9208-e916-46e7-9efa-2bc91510b584\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"plot\":{\"id\":\"8f913c58-a7b9-4b92-9891-a733e413bceb\",\"subtype\":\"Chart\",\"type\":\"Plot\"}},\"id\":\"3bdcce41-5023-42b6-827e-054939cf5a96\",\"type\":\"HelpTool\"},{\"attributes\":{\"data_source\":{\"id\":\"116d9f58-e823-4dd2-bbf8-9443348e16e8\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"13a17395-9e1b-446e-b6a5-60c28d95a765\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null},\"id\":\"ccf03fdf-77b0-4d0b-bffd-a6cbc5944b62\",\"type\":\"GlyphRenderer\"},{\"attributes\":{},\"id\":\"6e7664a4-1401-42da-855d-2ced17a854a7\",\"type\":\"LinearScale\"},{\"attributes\":{\"data_source\":{\"id\":\"eb189251-80e5-48dd-b73d-149c72ccedec\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"16ca7de9-6f14-49e6-a533-059044ab270a\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null},\"id\":\"f4ca8500-1fe1-4260-bab1-a0b0082d82aa\",\"type\":\"GlyphRenderer\"},{\"attributes\":{},\"id\":\"35af9654-cb20-497f-b694-591d392c8a95\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"callback\":null,\"factors\":[\"2014/2015\",\"2015/2016\",\"2015/2016 retrained\"]},\"id\":\"844e4aa9-fe3f-41a1-b784-101a4969430c\",\"type\":\"FactorRange\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"lightgrey\"},\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":{\"value\":1.0},\"line_color\":{\"value\":\"black\"},\"line_dash\":[4,4],\"line_width\":{\"value\":2},\"plot\":null,\"render_mode\":\"css\",\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"0bf8101a-a971-411d-b00a-d92d56e04813\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"y\",\"line_color\",\"x\",\"fill_alpha\",\"width\",\"color\",\"height\",\"line_alpha\",\"label\"],\"data\":{\"chart_index\":[{\"classifier\":\"LightGBM\",\"label\":\"2014/2015\"}],\"classifier\":[\"LightGBM\"],\"color\":[\"#5f9e6f\"],\"fill_alpha\":[0.8],\"height\":[0.5164835164835165],\"label\":[\"2014/2015\"],\"line_alpha\":[1.0],\"line_color\":[\"white\"],\"width\":[0.3],\"x\":[\"2014/2015:0.666666666667\"],\"y\":[0.25824175824175827]}},\"id\":\"578099e2-86b1-4059-bf84-3331f28b43bf\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1c995147-ccbb-4376-b068-4cc9d9dd273b\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"callback\":null,\"end\":0.5833333333333334},\"id\":\"caf653c3-fd72-40eb-bfcc-c0baa1662a05\",\"type\":\"Range1d\"},{\"attributes\":{},\"id\":\"d31f4708-9fed-4b7c-90fe-81d3ce3b7a7b\",\"type\":\"BasicTicker\"},{\"attributes\":{\"data_source\":{\"id\":\"578099e2-86b1-4059-bf84-3331f28b43bf\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"c56ce2ad-fdaa-403b-9fbe-bbd4d8ffe303\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null},\"id\":\"d9796aaa-6ea5-4f80-93a0-a485df92d9c9\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"items\":[{\"id\":\"8f6b809f-def1-4512-af80-4398269b88c3\",\"type\":\"LegendItem\"},{\"id\":\"09baff80-51bb-4e08-bf25-4cd2aaa788e5\",\"type\":\"LegendItem\"}],\"location\":[0,-30],\"plot\":{\"id\":\"8f913c58-a7b9-4b92-9891-a733e413bceb\",\"subtype\":\"Chart\",\"type\":\"Plot\"}},\"id\":\"05f18bff-85b6-4a4d-9df2-c02d27825b47\",\"type\":\"Legend\"},{\"attributes\":{},\"id\":\"d2ce2516-3cda-4365-93bc-9e09bbcd4097\",\"type\":\"ToolEvents\"},{\"attributes\":{\"overlay\":{\"id\":\"0bf8101a-a971-411d-b00a-d92d56e04813\",\"type\":\"BoxAnnotation\"},\"plot\":{\"id\":\"8f913c58-a7b9-4b92-9891-a733e413bceb\",\"subtype\":\"Chart\",\"type\":\"Plot\"}},\"id\":\"8589029a-8b7b-4200-b7a5-15c6961698a5\",\"type\":\"BoxZoomTool\"},{\"attributes\":{\"fill_alpha\":{\"field\":\"fill_alpha\"},\"fill_color\":{\"field\":\"color\"},\"height\":{\"field\":\"height\",\"units\":\"data\"},\"line_color\":{\"field\":\"line_color\"},\"width\":{\"field\":\"width\",\"units\":\"data\"},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"c56ce2ad-fdaa-403b-9fbe-bbd4d8ffe303\",\"type\":\"Rect\"},{\"attributes\":{\"plot\":{\"id\":\"8f913c58-a7b9-4b92-9891-a733e413bceb\",\"subtype\":\"Chart\",\"type\":\"Plot\"}},\"id\":\"c02ba394-87cd-488a-b639-d4663329ac6a\",\"type\":\"ResetTool\"},{\"attributes\":{\"items\":[{\"id\":\"8f6b809f-def1-4512-af80-4398269b88c3\",\"type\":\"LegendItem\"},{\"id\":\"09baff80-51bb-4e08-bf25-4cd2aaa788e5\",\"type\":\"LegendItem\"}],\"location\":\"top_left\",\"plot\":{\"id\":\"8f913c58-a7b9-4b92-9891-a733e413bceb\",\"subtype\":\"Chart\",\"type\":\"Plot\"},\"visible\":false},\"id\":\"d67887de-6f7f-4497-bbde-5af5b4439076\",\"type\":\"Legend\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"8f913c58-a7b9-4b92-9891-a733e413bceb\",\"subtype\":\"Chart\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"d31f4708-9fed-4b7c-90fe-81d3ce3b7a7b\",\"type\":\"BasicTicker\"}},\"id\":\"fa67b6c9-ce26-4de4-8252-699a5d9fe50d\",\"type\":\"Grid\"},{\"attributes\":{\"fill_alpha\":{\"field\":\"fill_alpha\"},\"fill_color\":{\"field\":\"color\"},\"height\":{\"field\":\"height\",\"units\":\"data\"},\"line_color\":{\"field\":\"line_color\"},\"width\":{\"field\":\"width\",\"units\":\"data\"},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"16ca7de9-6f14-49e6-a533-059044ab270a\",\"type\":\"Rect\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"y\",\"line_color\",\"x\",\"fill_alpha\",\"width\",\"color\",\"height\",\"line_alpha\",\"label\"],\"data\":{\"chart_index\":[{\"classifier\":\"XGBoost\",\"label\":\"2014/2015\"}],\"classifier\":[\"XGBoost\"],\"color\":[\"#5975a4\"],\"fill_alpha\":[0.8],\"height\":[0.521978021978022],\"label\":[\"2014/2015\"],\"line_alpha\":[1.0],\"line_color\":[\"white\"],\"width\":[0.3],\"x\":[\"2014/2015:0.333333333333\"],\"y\":[0.260989010989011]}},\"id\":\"7610c37a-87de-4cdc-a35b-a62f3c0bd210\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"3fa04ab6-2cb0-4395-b6ac-22e3a9a04a90\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"axis_label\":\"\",\"formatter\":{\"id\":\"a3e46255-4bb5-4cf2-ac3f-d28c9d7de9d1\",\"type\":\"CategoricalTickFormatter\"},\"major_label_orientation\":0.7853981633974483,\"plot\":{\"id\":\"8f913c58-a7b9-4b92-9891-a733e413bceb\",\"subtype\":\"Chart\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"3fa04ab6-2cb0-4395-b6ac-22e3a9a04a90\",\"type\":\"CategoricalTicker\"}},\"id\":\"96511dff-0efd-49c7-915b-618a3e81d15e\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"data_source\":{\"id\":\"fbf38c40-9721-492f-8852-5f213a3e1c86\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"90eb29b6-31f2-445c-9843-9a01708aa702\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null},\"id\":\"64c1d1e8-5419-4464-91b3-813f5fc319be\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"below\":[{\"id\":\"96511dff-0efd-49c7-915b-618a3e81d15e\",\"type\":\"CategoricalAxis\"}],\"css_classes\":null,\"left\":[{\"id\":\"c2fc4764-640b-4262-9d7f-2c0be1dd37f0\",\"type\":\"LinearAxis\"}],\"plot_width\":500,\"renderers\":[{\"id\":\"0bf8101a-a971-411d-b00a-d92d56e04813\",\"type\":\"BoxAnnotation\"},{\"id\":\"b6a95f6d-de9d-451c-996b-771c5b0f98fa\",\"type\":\"GlyphRenderer\"},{\"id\":\"f4ca8500-1fe1-4260-bab1-a0b0082d82aa\",\"type\":\"GlyphRenderer\"},{\"id\":\"8a3d9208-e916-46e7-9efa-2bc91510b584\",\"type\":\"GlyphRenderer\"},{\"id\":\"d9796aaa-6ea5-4f80-93a0-a485df92d9c9\",\"type\":\"GlyphRenderer\"},{\"id\":\"64c1d1e8-5419-4464-91b3-813f5fc319be\",\"type\":\"GlyphRenderer\"},{\"id\":\"ccf03fdf-77b0-4d0b-bffd-a6cbc5944b62\",\"type\":\"GlyphRenderer\"},{\"id\":\"d67887de-6f7f-4497-bbde-5af5b4439076\",\"type\":\"Legend\"},{\"id\":\"96511dff-0efd-49c7-915b-618a3e81d15e\",\"type\":\"CategoricalAxis\"},{\"id\":\"c2fc4764-640b-4262-9d7f-2c0be1dd37f0\",\"type\":\"LinearAxis\"},{\"id\":\"fa67b6c9-ce26-4de4-8252-699a5d9fe50d\",\"type\":\"Grid\"},{\"id\":\"05f18bff-85b6-4a4d-9df2-c02d27825b47\",\"type\":\"Legend\"}],\"right\":[{\"id\":\"05f18bff-85b6-4a4d-9df2-c02d27825b47\",\"type\":\"Legend\"}],\"title\":{\"id\":\"d2063746-bd8a-4a5e-93ad-40a8cb49d246\",\"type\":\"Title\"},\"tool_events\":{\"id\":\"d2ce2516-3cda-4365-93bc-9e09bbcd4097\",\"type\":\"ToolEvents\"},\"toolbar\":{\"id\":\"d802cf31-0924-4d24-a272-394172abb275\",\"type\":\"Toolbar\"},\"toolbar_location\":\"above\",\"x_range\":{\"id\":\"844e4aa9-fe3f-41a1-b784-101a4969430c\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1c995147-ccbb-4376-b068-4cc9d9dd273b\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"caf653c3-fd72-40eb-bfcc-c0baa1662a05\",\"type\":\"Range1d\"},\"y_scale\":{\"id\":\"6e7664a4-1401-42da-855d-2ced17a854a7\",\"type\":\"LinearScale\"}},\"id\":\"8f913c58-a7b9-4b92-9891-a733e413bceb\",\"subtype\":\"Chart\",\"type\":\"Plot\"},{\"attributes\":{\"fill_alpha\":{\"field\":\"fill_alpha\"},\"fill_color\":{\"field\":\"color\"},\"height\":{\"field\":\"height\",\"units\":\"data\"},\"line_color\":{\"field\":\"line_color\"},\"width\":{\"field\":\"width\",\"units\":\"data\"},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"9c1cb582-e3f7-4d71-aed5-09f5b38e8b48\",\"type\":\"Rect\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"y\",\"line_color\",\"x\",\"fill_alpha\",\"width\",\"color\",\"height\",\"line_alpha\",\"label\"],\"data\":{\"chart_index\":[{\"classifier\":\"XGBoost\",\"label\":\"2015/2016 retrained\"}],\"classifier\":[\"XGBoost\"],\"color\":[\"#5975a4\"],\"fill_alpha\":[0.8],\"height\":[0.5242165242165242],\"label\":[\"2015/2016 retrained\"],\"line_alpha\":[1.0],\"line_color\":[\"white\"],\"width\":[0.3],\"x\":[\"2015/2016 retrained:0.333333333333\"],\"y\":[0.2621082621082621]}},\"id\":\"c6af2b99-48d3-4e38-998f-8c4314b9347c\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"plot\":null,\"text\":null},\"id\":\"d2063746-bd8a-4a5e-93ad-40a8cb49d246\",\"type\":\"Title\"},{\"attributes\":{\"fill_alpha\":{\"field\":\"fill_alpha\"},\"fill_color\":{\"field\":\"color\"},\"height\":{\"field\":\"height\",\"units\":\"data\"},\"line_color\":{\"field\":\"line_color\"},\"width\":{\"field\":\"width\",\"units\":\"data\"},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"924144dd-7f75-47cd-a1f7-04779b0aada8\",\"type\":\"Rect\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"ce7c1b45-580c-4caf-a812-6e4975a392b3\",\"type\":\"PanTool\"},{\"id\":\"c4aa33c5-59c6-43b1-81f7-9ad2271c716d\",\"type\":\"WheelZoomTool\"},{\"id\":\"8589029a-8b7b-4200-b7a5-15c6961698a5\",\"type\":\"BoxZoomTool\"},{\"id\":\"dacc2b67-106a-406a-a9f8-ed4190a421be\",\"type\":\"SaveTool\"},{\"id\":\"c02ba394-87cd-488a-b639-d4663329ac6a\",\"type\":\"ResetTool\"},{\"id\":\"3bdcce41-5023-42b6-827e-054939cf5a96\",\"type\":\"HelpTool\"}]},\"id\":\"d802cf31-0924-4d24-a272-394172abb275\",\"type\":\"Toolbar\"},{\"attributes\":{\"plot\":{\"id\":\"8f913c58-a7b9-4b92-9891-a733e413bceb\",\"subtype\":\"Chart\",\"type\":\"Plot\"}},\"id\":\"dacc2b67-106a-406a-a9f8-ed4190a421be\",\"type\":\"SaveTool\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"y\",\"line_color\",\"x\",\"fill_alpha\",\"width\",\"color\",\"height\",\"line_alpha\",\"label\"],\"data\":{\"chart_index\":[{\"classifier\":\"XGBoost\",\"label\":\"2015/2016\"}],\"classifier\":[\"XGBoost\"],\"color\":[\"#5975a4\"],\"fill_alpha\":[0.8],\"height\":[0.48717948717948717],\"label\":[\"2015/2016\"],\"line_alpha\":[1.0],\"line_color\":[\"white\"],\"width\":[0.3],\"x\":[\"2015/2016:0.333333333333\"],\"y\":[0.24358974358974358]}},\"id\":\"eb189251-80e5-48dd-b73d-149c72ccedec\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"plot\":{\"id\":\"8f913c58-a7b9-4b92-9891-a733e413bceb\",\"subtype\":\"Chart\",\"type\":\"Plot\"}},\"id\":\"c4aa33c5-59c6-43b1-81f7-9ad2271c716d\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"fill_alpha\":{\"field\":\"fill_alpha\"},\"fill_color\":{\"field\":\"color\"},\"height\":{\"field\":\"height\",\"units\":\"data\"},\"line_color\":{\"field\":\"line_color\"},\"width\":{\"field\":\"width\",\"units\":\"data\"},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"13a17395-9e1b-446e-b6a5-60c28d95a765\",\"type\":\"Rect\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"y\",\"line_color\",\"x\",\"fill_alpha\",\"width\",\"color\",\"height\",\"line_alpha\",\"label\"],\"data\":{\"chart_index\":[{\"classifier\":\"LightGBM\",\"label\":\"2015/2016\"}],\"classifier\":[\"LightGBM\"],\"color\":[\"#5f9e6f\"],\"fill_alpha\":[0.8],\"height\":[0.5128205128205128],\"label\":[\"2015/2016\"],\"line_alpha\":[1.0],\"line_color\":[\"white\"],\"width\":[0.3],\"x\":[\"2015/2016:0.666666666667\"],\"y\":[0.2564102564102564]}},\"id\":\"fbf38c40-9721-492f-8852-5f213a3e1c86\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"data_source\":{\"id\":\"7610c37a-87de-4cdc-a35b-a62f3c0bd210\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"9c1cb582-e3f7-4d71-aed5-09f5b38e8b48\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null},\"id\":\"b6a95f6d-de9d-451c-996b-771c5b0f98fa\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"plot\":{\"id\":\"8f913c58-a7b9-4b92-9891-a733e413bceb\",\"subtype\":\"Chart\",\"type\":\"Plot\"}},\"id\":\"ce7c1b45-580c-4caf-a812-6e4975a392b3\",\"type\":\"PanTool\"},{\"attributes\":{\"label\":{\"value\":\"XGBoost\"},\"renderers\":[{\"id\":\"b6a95f6d-de9d-451c-996b-771c5b0f98fa\",\"type\":\"GlyphRenderer\"}]},\"id\":\"8f6b809f-def1-4512-af80-4398269b88c3\",\"type\":\"LegendItem\"},{\"attributes\":{},\"id\":\"a3e46255-4bb5-4cf2-ac3f-d28c9d7de9d1\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"axis_label\":\"Accuracy\",\"formatter\":{\"id\":\"35af9654-cb20-497f-b694-591d392c8a95\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"8f913c58-a7b9-4b92-9891-a733e413bceb\",\"subtype\":\"Chart\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"d31f4708-9fed-4b7c-90fe-81d3ce3b7a7b\",\"type\":\"BasicTicker\"}},\"id\":\"c2fc4764-640b-4262-9d7f-2c0be1dd37f0\",\"type\":\"LinearAxis\"},{\"attributes\":{\"fill_alpha\":{\"field\":\"fill_alpha\"},\"fill_color\":{\"field\":\"color\"},\"height\":{\"field\":\"height\",\"units\":\"data\"},\"line_color\":{\"field\":\"line_color\"},\"width\":{\"field\":\"width\",\"units\":\"data\"},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"90eb29b6-31f2-445c-9843-9a01708aa702\",\"type\":\"Rect\"}],\"root_ids\":[\"8f913c58-a7b9-4b92-9891-a733e413bceb\"]},\"title\":\"Bokeh Application\",\"version\":\"0.12.6\"}};\n",
       "            var render_items = [{\"docid\":\"7b97c6a2-00f5-4724-bbb6-035b15a35e94\",\"elementid\":\"43c57489-e7f8-4f86-844a-a159a0f153bc\",\"modelid\":\"8f913c58-a7b9-4b92-9891-a733e413bceb\"}];\n",
       "            \n",
       "            Bokeh.embed.embed_items(docs_json, render_items);\n",
       "          };\n",
       "          if (document.readyState != \"loading\") fn();\n",
       "          else document.addEventListener(\"DOMContentLoaded\", fn);\n",
       "        })();\n",
       "      },\n",
       "      function(Bokeh) {\n",
       "      }\n",
       "    ];\n",
       "  \n",
       "    function run_inline_js() {\n",
       "      \n",
       "      if ((window.Bokeh !== undefined) || (force === true)) {\n",
       "        for (var i = 0; i < inline_js.length; i++) {\n",
       "          inline_js[i](window.Bokeh);\n",
       "        }if (force === true) {\n",
       "          display_loaded();\n",
       "        }} else if (Date.now() < window._bokeh_timeout) {\n",
       "        setTimeout(run_inline_js, 100);\n",
       "      } else if (!window._bokeh_failed_load) {\n",
       "        console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "        window._bokeh_failed_load = true;\n",
       "      } else if (force !== true) {\n",
       "        var cell = $(document.getElementById(\"43c57489-e7f8-4f86-844a-a159a0f153bc\")).parents('.cell').data().cell;\n",
       "        cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "      }\n",
       "  \n",
       "    }\n",
       "  \n",
       "    if (window._bokeh_is_loading === 0) {\n",
       "      console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "      run_inline_js();\n",
       "    } else {\n",
       "      load_libs(js_urls, function() {\n",
       "        console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "        run_inline_js();\n",
       "      });\n",
       "    }\n",
       "  }(this));\n",
       "</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Football Retrain Results\n",
    "bar = Bar(data, values='Accuracy', label=['label'], agg='mean', group=['classifier'], \n",
    "          plot_width=500, color=['#5f9e6f', '#5975a4'], bar_width=2)\n",
    "bar.axis[0].axis_label=''\n",
    "bar.axis[1].axis_label='Accuracy'\n",
    "list(map(change_width(0.4), filter(is_bar, bar.renderers)))\n",
    "bar.toolbar_location='above'\n",
    "bar.add_layout(Legend(items=bar.legend[0].items, location=(0, -30)), 'right')\n",
    "bar.legend[0].visible=False\n",
    "show(bar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"600\" style=\"width: 500px; height: 600px;\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs/><g><g/><g/><g/><g/><g transform=\"scale(1,1) translate(0.5,0.5)\"><rect fill=\"#FFFFFF\" height=\"600\" stroke=\"none\" width=\"500\" x=\"0\" y=\"0\"/><rect fill=\"#ffffff\" height=\"600\" stroke=\"none\" width=\"500\" x=\"0\" y=\"0\"/><rect fill=\"#FFFFFF\" height=\"464.1356590903584\" stroke=\"none\" width=\"334.1809375\" x=\"52.92843750000001\" y=\"29\"/><rect fill=\"#ffffff\" height=\"464.1356590903584\" stroke=\"none\" width=\"334.1809375\" x=\"52.92843750000001\" y=\"29\"/><g><rect fill=\"none\" height=\"464.1356590903584\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\" width=\"334.1809375\" x=\"52.92843750000001\" y=\"29\"/></g><g><g><path d=\" M 53 414 L 387 414\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 53 334 L 387 334\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 53 254 L 387 254\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 53 175 L 387 175\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/><path d=\" M 53 95 L 387 95\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\"/></g><g><path fill=\"none\" stroke=\"none\"/><rect fill=\"#5975a4\" fill-opacity=\"0.8\" height=\"415.3176227339157\" stroke=\"none\" width=\"33.41809375000048\" x=\"73.35060590274065\" y=\"77.81803635644269\"/><path d=\" M 73.35060590274065 77.81803635644269 L 106.76869965274113 77.81803635644269 L 106.76869965274113 493.1356590903584 L 73.35060590274065 493.1356590903584 L 73.35060590274065 77.81803635644269 Z\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"white\" stroke-miterlimit=\"10\"/><path d=\"\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"white\" stroke-miterlimit=\"10\"/></g><g><path fill=\"none\" stroke=\"none\"/><rect fill=\"#5975a4\" fill-opacity=\"0.8\" height=\"387.6297812183212\" stroke=\"none\" width=\"33.4180937500023\" x=\"184.744251736074\" y=\"105.50587787203716\"/><path d=\" M 184.744251736074 105.50587787203716 L 218.1623454860763 105.50587787203716 L 218.1623454860763 493.1356590903584 L 184.744251736074 493.1356590903584 L 184.744251736074 105.50587787203716 Z\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"white\" stroke-miterlimit=\"10\"/><path d=\"\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"white\" stroke-miterlimit=\"10\"/></g><g><path fill=\"none\" stroke=\"none\"/><rect fill=\"#5975a4\" fill-opacity=\"0.8\" height=\"417.098711954217\" stroke=\"none\" width=\"33.41809374999866\" x=\"296.13789756940736\" y=\"76.0369471361414\"/><path d=\" M 296.13789756940736 76.0369471361414 L 329.555991319406 76.0369471361414 L 329.555991319406 493.1356590903584 L 296.13789756940736 493.1356590903584 L 296.13789756940736 76.0369471361414 Z\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"white\" stroke-miterlimit=\"10\"/><path d=\"\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"white\" stroke-miterlimit=\"10\"/></g><g><path fill=\"none\" stroke=\"none\"/><rect fill=\"#5f9e6f\" fill-opacity=\"0.8\" height=\"410.945858284085\" stroke=\"none\" width=\"33.4180937500023\" x=\"110.4818211805927\" y=\"82.18980080627341\"/><path d=\" M 110.4818211805927 82.18980080627341 L 143.899914930595 82.18980080627341 L 143.899914930595 493.1356590903584 L 110.4818211805927 493.1356590903584 L 110.4818211805927 82.18980080627341 Z\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"white\" stroke-miterlimit=\"10\"/><path d=\"\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"white\" stroke-miterlimit=\"10\"/></g><g><path fill=\"none\" stroke=\"none\"/><rect fill=\"#5f9e6f\" fill-opacity=\"0.8\" height=\"408.03134865086446\" stroke=\"none\" width=\"33.41809374999866\" x=\"221.87546701392606\" y=\"85.10431043949393\"/><path d=\" M 221.87546701392606 85.10431043949393 L 255.29356076392472 85.10431043949393 L 255.29356076392472 493.1356590903584 L 221.87546701392606 493.1356590903584 L 221.87546701392606 85.10431043949393 Z\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"white\" stroke-miterlimit=\"10\"/><path d=\"\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"white\" stroke-miterlimit=\"10\"/></g><g><path fill=\"none\" stroke=\"none\"/><rect fill=\"#5f9e6f\" fill-opacity=\"0.8\" height=\"442.03396103843653\" stroke=\"none\" width=\"33.418093749991385\" x=\"333.2691128472594\" y=\"51.10169805192186\"/><path d=\" M 333.2691128472594 51.10169805192186 L 366.6872065972508 51.10169805192186 L 366.6872065972508 493.1356590903584 L 333.2691128472594 493.1356590903584 L 333.2691128472594 51.10169805192186 Z\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"white\" stroke-miterlimit=\"10\"/><path d=\"\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"white\" stroke-miterlimit=\"10\"/></g></g><g/><g><g fill=\"#444444\"><path d=\" M 53 493 L 387 493\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 109 499 L 109 491\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 220 499 L 220 491\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 331 499 L 331 491\" fill=\"\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><g transform=\"translate(108.62526041666668,504.1356590903584) rotate(-45,0,0)\"><text dominant-baseline=\"central\" fill=\"\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"0\" y=\"0\">2014/2015</text><g transform=\"rotate(45,0,0) translate(-108.62526041666668,-504.1356590903584) translate(220.01890625000001,504.1356590903584) rotate(-45,0,0)\"><text dominant-baseline=\"central\" fill=\"\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"0\" y=\"0\">2015/2016</text><g transform=\"rotate(45,0,0) translate(-220.01890625000001,-504.1356590903584) translate(331.41255208333337,504.1356590903584) rotate(-45,0,0)\"><text dominant-baseline=\"central\" fill=\"\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"0\" y=\"0\">2015/2016 retrained</text><g transform=\"rotate(45,0,0) translate(-331.41255208333337,-504.1356590903584)\"><text dominant-baseline=\"hanging\" fill=\"\" font-family=\"helvetica\" font-size=\"10pt\" font-style=\"italic\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"220.01890625000001\" y=\"604\"/></g></g></g></g></g><g><path d=\" M 53 493 L 53 29\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 47 493 L 55 493\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 47 414 L 55 414\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 47 334 L 55 334\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 47 254 L 55 254\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 47 175 L 55 175\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 47 95 L 55 95\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 493 L 53 493\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 477 L 53 477\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 461 L 53 461\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 445 L 53 445\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 429 L 53 429\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 414 L 53 414\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 398 L 53 398\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 382 L 53 382\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 366 L 53 366\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 350 L 53 350\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 334 L 53 334\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 318 L 53 318\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 302 L 53 302\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 286 L 53 286\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 270 L 53 270\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 254 L 53 254\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 239 L 53 239\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 223 L 53 223\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 207 L 53 207\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 191 L 53 191\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 175 L 53 175\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 159 L 53 159\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 143 L 53 143\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 127 L 53 127\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 111 L 53 111\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 95 L 53 95\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 79 L 53 79\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 63 L 53 63\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 48 L 53 48\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><path d=\" M 49 32 L 53 32\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"black\" stroke-miterlimit=\"10\"/><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"42\" y=\"493\">0</text><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"42\" y=\"414\">0.1</text><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"42\" y=\"334\">0.2</text><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"42\" y=\"254\">0.3</text><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"42\" y=\"175\">0.4</text><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"8pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"end\" text-decoration=\"normal\" x=\"42\" y=\"95\">0.5</text><g transform=\"translate(17.73000000000001,261.06782954517917) rotate(-90,0,0)\"><text dominant-baseline=\"alphabetic\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"10pt\" font-style=\"italic\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"middle\" text-decoration=\"normal\" x=\"0\" y=\"0\">Accuracy</text><g transform=\"rotate(90,0,0) translate(-17.73000000000001,-261.06782954517917)\"/></g></g><g/><g><path d=\" M 387.109375 29 L 500 29 L 500 92 L 387.109375 92 L 387.109375 29 Z\" fill=\"#ffffff\" fill-opacity=\"0.95\" paint-order=\"fill stroke markers\" stroke=\"#e5e5e5\" stroke-miterlimit=\"10\" stroke-opacity=\"0.5\"/><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"10pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"start\" text-decoration=\"normal\" x=\"422.109375\" y=\"49\">XGBoost</text><rect fill=\"#5975a4\" fill-opacity=\"0.8\" height=\"16\" stroke=\"none\" width=\"16\" x=\"399.109375\" y=\"41\"/><path d=\" M 399.109375 41 L 415.109375 41 L 415.109375 57 L 399.109375 57 L 399.109375 41 Z\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"white\" stroke-miterlimit=\"10\"/><text dominant-baseline=\"central\" fill=\"#444444\" font-family=\"helvetica\" font-size=\"10pt\" font-style=\"normal\" font-weight=\"normal\" stroke=\"none\" text-anchor=\"start\" text-decoration=\"normal\" x=\"422.109375\" y=\"72\">LightGBM</text><rect fill=\"#5f9e6f\" fill-opacity=\"0.8\" height=\"16\" stroke=\"none\" width=\"16\" x=\"399.109375\" y=\"64\"/><path d=\" M 399.109375 64 L 415.109375 64 L 415.109375 80 L 399.109375 80 L 399.109375 64 Z\" fill=\"none\" paint-order=\"fill stroke markers\" stroke=\"white\" stroke-miterlimit=\"10\"/></g></g></g></g></svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# SVG: Football Retrain Results\n",
    "bar.output_backend = \"svg\"\n",
    "export_svgs(bar, filename=\"football_bar.svg\")\n",
    "display(SVG('football_bar.svg'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.5",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}