gtc/Multi-GPU-H2O-GBM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# H2O.ai GPU Edition Machine Learning $-$ Multi-GPU GBM Demo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### In this demo, we will train 16 gradient boosting models (aka GBMs) on the Higgs boson dataset, with the goal to predict whether a given event in the particle detector stems from an actual Higgs boson.\n",
    "\n",
    "### The dataset is about 500MB in memory (2M rows, 29 cols, double precision floating-point values), so it fits onto the GPU memory.\n",
    "\n",
    "### By using multiple GPUs, we are able to speed up this process significantly, and can train all 16 models in less than 2 minutes (DGX-1 with 8 GPUs) vs 30 minutes on a dual-Xeon server."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import dependencies (This requires Linux Ubuntu 16.04 with CUDA 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## First time only: Install H2O\n",
    "#!pip install http://s3.amazonaws.com/h2o-deepwater/public/nightly/deepwater-h2o-230/h2o-3.11.0.230-py2.py3-none-any.whl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import sys\n",
    "import h2o\n",
    "import subprocess\n",
    "from h2o.estimators import H2OXGBoostEstimator\n",
    "from concurrent.futures import ProcessPoolExecutor\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.patches as mpatches\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as image\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "sns.set_style(\"whitegrid\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Start H2O and import the dataset (Here: Higgs Boson dataset: 2M rows, 29 cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#path = \"https://s3.amazonaws.com/h2o-public-test-data/bigdata/laptop/higgs_head_2M.csv\" ### this works too\n",
    "path = \"/opt/higgs_head_2M.csv\"\n",
    "target = 0\n",
    "drop_cols=[]\n",
    "trainrows=1000000\n",
    "\n",
    "h2o.init(max_mem_size=\"16G\")\n",
    "t0 = time.time()\n",
    "df_hex = h2o.import_file(path)\n",
    "t1 = time.time()\n",
    "print(\"Parsed the dataset in %r seconds\" % (t1-t0))\n",
    "for c in drop_cols:\n",
    "    df_hex = df_hex.drop(c, axis=1)\n",
    "df_hex.refresh()\n",
    "\n",
    "print(df_hex.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define a method to train an H2O XGBoost Gradient Boosting Model in a separate process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def runH2OXGBoost(arg):\n",
    "    h2o.connect(ip='localhost',port=54321,verbose=False)\n",
    "    depth, sample_rate, frame_id, trainrows, drop_cols, target, NGPUS, gpu = arg\n",
    "    try:\n",
    "        df_hex = h2o.get_frame(frame_id)\n",
    "        import os\n",
    "        trainhex = df_hex[:trainrows,:]\n",
    "        trainhex[target] = trainhex[target].asfactor()\n",
    "        learn_rate=0.01\n",
    "        num_round =100\n",
    "        param = {\n",
    "              \"ntrees\":num_round\n",
    "            , \"max_depth\":depth\n",
    "            , \"learn_rate\":learn_rate\n",
    "            , \"sample_rate\":sample_rate\n",
    "            , \"col_sample_rate_per_tree\":sample_rate\n",
    "            , \"min_rows\":5\n",
    "            , \"seed\":12345\n",
    "            , \"tree_method\":\"exact\"\n",
    "            , \"dmatrix_type\":\"dense\"\n",
    "            , \"backend\":(\"gpu\" if NGPUS > 0 else \"cpu\")         \n",
    "            , \"gpu_id\":gpu\n",
    "            , \"score_tree_interval\":num_round\n",
    "        }\n",
    "        model = H2OXGBoostEstimator(**param)\n",
    "        print(\"Training H2O XGBoost model with depth=\", depth, \"sample_rate=\", sample_rate, (\"on GPU\" if NGPUS>0 else \"on CPU\"), gpu)\n",
    "        model.train(x = list(trainhex.columns), y = target, training_frame = trainhex)\n",
    "    except:\n",
    "        print(\"Unexpected error:\", sys.exc_info()[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train 16 H2O XGBoost models each on 8, 4, 2, 1 GPUs and on 0 GPUs (i.e, CPUs only)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "times = []\n",
    "GPU_list = [8,4,2,1,0]\n",
    "for NGPUS in GPU_list:\n",
    "    \n",
    "    Executor = ProcessPoolExecutor(max_workers=max(1,NGPUS))\n",
    "    h2o.no_progress()\n",
    "\n",
    "    t0 = time.time()\n",
    "    print(\"Starting to train H2O XGBoost on %d GPUs\" % NGPUS)\n",
    "    gpu=0\n",
    "    arg = []\n",
    "    futures = []\n",
    "    for depth in range(6,13,2): ## depth 6,8,10,12\n",
    "        for sample_rate in [x/10. for x in range(7,11)]: ## sample_rate 0.7,0.8,0.9,1.0\n",
    "            work = (depth,sample_rate,df_hex.frame_id,trainrows, drop_cols, target, NGPUS, gpu)\n",
    "            futures.append(Executor.submit(runH2OXGBoost, work))\n",
    "            if (NGPUS>0):\n",
    "                gpu=(gpu+1) % NGPUS\n",
    "\n",
    "    import concurrent.futures\n",
    "    concurrent.futures.wait(futures)\n",
    "\n",
    "    t1 = time.time()\n",
    "    print(\"Time to train H2O on %d GPUs: %r\\n\\n\" % (NGPUS, (t1-t0)))\n",
    "    times.append(t1-t0)\n",
    "print(times)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute test set error for all 16 H2O XGBoost models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "testhex         = df_hex[trainrows:trainrows+1000000,:]\n",
    "testhex[target] = testhex[target].asfactor()\n",
    "print(testhex.shape)\n",
    "\n",
    "models = [x[0] for x in h2o.ls().values if \"XGBoost_model\" in x[0]]\n",
    "for m in models:\n",
    "    try:\n",
    "        model = h2o.get_model(m)\n",
    "        perf = h2o.make_metrics(model.predict(testhex)[:,2], testhex[target])\n",
    "        print(m, \"sample rate:\", model.params['sample_rate']['actual'], \n",
    "              \"max_depth:\", model.params['max_depth']['actual'], \n",
    "              \"Test AUC:\", perf.auc())\n",
    "    except:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Performance plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'nGPUs':GPU_list, 't_train':times})\n",
    "label_dic = {'0':'2 CPUs**', '1':'1 GPU*', '2':'2 GPUs*', '4':'4 GPUs*', '8':'8 GPUs*'}\n",
    "colors = {'gpu':'limegreen', 'cpu':'dodgerblue'}\n",
    "\n",
    "df['hardware'] = df['nGPUs'].apply(lambda x: str(x))\n",
    "df = df.replace({'hardware':label_dic})\n",
    "\n",
    "\n",
    "\n",
    "fig = plt.figure(figsize=(12,9))\n",
    "ax =sns.barplot(x=\"t_train\", y=\"hardware\", data=df, orient = 'h', \n",
    "                 palette = list(df[\"nGPUs\"].apply(lambda x:colors['gpu' if x>0 else 'cpu'])))\n",
    "\n",
    "title_text = 'H2O.ai Machine Learning $-$ Gradient Boosting\\nTime to Train 16 H2O XGBoost Models'\n",
    "plt.title(title_text, fontsize=20, y=1.02)\n",
    "plt.xlabel('Time [sec]', fontsize=20, labelpad = 12, fontweight = 'bold')\n",
    "plt.ylabel('')\n",
    "\n",
    "ax.yaxis.set_label_position('right')\n",
    "ax.set_xlim([0,df.t_train.max()+100])\n",
    "ax.tick_params(labelsize = 18)\n",
    "\n",
    "\n",
    "for p in ax.patches:\n",
    "    #print(p)\n",
    "    x=p.get_bbox().get_points()[:,0]\n",
    "    y=p.get_bbox().get_points()[:,1]\n",
    "    ax.annotate('{:.0f}'.format(x[1]), (x[1]-0.035*df.t_train.max(), y.mean()), \n",
    "            ha='center', va='center', fontsize = 20, color = 'w', fontweight = 'bold')\n",
    "\n",
    "footnote_text=\\\n",
    "\"*NVIDIA Tesla P100 (DGX-1), **Dual Intel Xeon E5-2698 v4\"\\\n",
    "\"\\nHiggs dataset (binary classification): 1M rows, 29 cols; max_depth: {6,8,10,12}, sample_rate: {0.7,0.8,0.9,1.0}\"\n",
    "plt.annotate(footnote_text, (0,0), (df.t_train.max()/2, -80), fontsize = 16, xycoords='axes fraction',\n",
    "             textcoords='offset points', va='top', ha='right')\n",
    "\n",
    "ax.invert_yaxis()\n",
    "plt.tight_layout()\n",
    "\n",
    "green_patch = mpatches.Patch(color='limegreen', label='GPU')\n",
    "blue_patch = mpatches.Patch(color='dodgerblue', label='CPU')\n",
    "ax.legend(handles = [blue_patch, green_patch], markerscale = 10, fontsize = 20, loc = 4)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}