h2o-py/demos/airlines_demo_small.ipynb

{
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "name": "",
  "signature": "sha256:e10cd6ed47cd37bd3893052375dcd5eed3d2b7027905b62496882ebcf8004903"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import h2o\n",
      "import pandas\n",
      "import pprint\n",
      "import operator\n",
      "import matplotlib\n",
      "from tabulate import tabulate"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Connect to a cluster\n",
      "h2o.init()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O cluster uptime: </td>\n",
        "<td>6 minutes 1 seconds 983 milliseconds </td></tr>\n",
        "<tr><td>H2O cluster version: </td>\n",
        "<td>3.1.0.99999</td></tr>\n",
        "<tr><td>H2O cluster name: </td>\n",
        "<td>ece</td></tr>\n",
        "<tr><td>H2O cluster total nodes: </td>\n",
        "<td>1</td></tr>\n",
        "<tr><td>H2O cluster total memory: </td>\n",
        "<td>3.56 GB</td></tr>\n",
        "<tr><td>H2O cluster total cores: </td>\n",
        "<td>8</td></tr>\n",
        "<tr><td>H2O cluster allowed cores: </td>\n",
        "<td>8</td></tr>\n",
        "<tr><td>H2O cluster healthy: </td>\n",
        "<td>True</td></tr>\n",
        "<tr><td>H2O Connection ip: </td>\n",
        "<td>127.0.0.1</td></tr>\n",
        "<tr><td>H2O Connection port: </td>\n",
        "<td>54321</td></tr></table></div>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "--------------------------  ------------------------------------\n",
        "H2O cluster uptime:         6 minutes 1 seconds 983 milliseconds\n",
        "H2O cluster version:        3.1.0.99999\n",
        "H2O cluster name:           ece\n",
        "H2O cluster total nodes:    1\n",
        "H2O cluster total memory:   3.56 GB\n",
        "H2O cluster total cores:    8\n",
        "H2O cluster allowed cores:  8\n",
        "H2O cluster healthy:        True\n",
        "H2O Connection ip:          127.0.0.1\n",
        "H2O Connection port:        54321\n",
        "--------------------------  ------------------------------------"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# set this to True if interactive (matplotlib) plots are desired\n",
      "interactive = False\n",
      "if not interactive: matplotlib.use('Agg', warn=False)\n",
      "import matplotlib.pyplot as plt"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 17
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# air_path = [h2o.locate(\"bigdata/laptop/airlines_all.05p.csv\")]\n",
      "# air_path = [h2o.locate(\"bigdata/laptop/flights-nyc/flights14.csv.zip\")]\n",
      "air_path = [h2o.locate(\"smalldata/airlines/allyears2k_headers.zip\")]\n",
      "\n",
      "# ----------\n",
      "\n",
      "# 1- Load data - 1 row per flight.  Has columns showing the origin,\n",
      "# destination, departure and arrival time, carrier information, and\n",
      "# whether the flight was delayed.\n",
      "print \"Import and Parse airlines data\"\n",
      "data = h2o.import_frame(path=air_path)\n",
      "data.describe()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Import and Parse airlines data\n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "Parse Progress: [                                                  ] 00%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "Parse Progress: [##################################################] 100%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "\n",
        "Parsed 43,978 rows and 31 cols:\n",
        "\n"
       ]
      },
      {
       "html": [
        "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>File1</td>\n",
        "<td>/Users/ece/0xdata/h2o-dev/smalldata/airlines/allyears2k_headers.zip</td></tr></table></div>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "-----  -------------------------------------------------------------------\n",
        "File1  /Users/ece/0xdata/h2o-dev/smalldata/airlines/allyears2k_headers.zip\n",
        "-----  -------------------------------------------------------------------"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Rows: 43,978 Cols: 31\n",
        "\n",
        "Chunk compression summary:\n",
        "\n"
       ]
      },
      {
       "html": [
        "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>chunk_type</td>\n",
        "<td>chunk_name</td>\n",
        "<td>count</td>\n",
        "<td>count_percentage</td>\n",
        "<td>size</td>\n",
        "<td>size_percentage</td></tr>\n",
        "<tr><td>C0L</td>\n",
        "<td>Constant Integers</td>\n",
        "<td>10</td>\n",
        "<td>5.376344</td>\n",
        "<td>    800  B</td>\n",
        "<td>0.05040237</td></tr>\n",
        "<tr><td>C0D</td>\n",
        "<td>Constant Reals</td>\n",
        "<td>23</td>\n",
        "<td>12.365591</td>\n",
        "<td>    1.8 KB</td>\n",
        "<td>0.115925446</td></tr>\n",
        "<tr><td>CBS</td>\n",
        "<td>Bits</td>\n",
        "<td>2</td>\n",
        "<td>1.0752689</td>\n",
        "<td>    2.0 KB</td>\n",
        "<td>0.12720299</td></tr>\n",
        "<tr><td>CX0</td>\n",
        "<td>Sparse Bits</td>\n",
        "<td>10</td>\n",
        "<td>5.376344</td>\n",
        "<td>    1.9 KB</td>\n",
        "<td>0.12474586</td></tr>\n",
        "<tr><td>C1</td>\n",
        "<td>1-Byte Integers</td>\n",
        "<td>40</td>\n",
        "<td>21.505377</td>\n",
        "<td>  287.8 KB</td>\n",
        "<td>18.564957</td></tr>\n",
        "<tr><td>C1N</td>\n",
        "<td>1-Byte Integers (w/o NAs)</td>\n",
        "<td>19</td>\n",
        "<td>10.215054</td>\n",
        "<td>  133.1 KB</td>\n",
        "<td>8.58617</td></tr>\n",
        "<tr><td>C1S</td>\n",
        "<td>1-Byte Fractions</td>\n",
        "<td>6</td>\n",
        "<td>3.2258065</td>\n",
        "<td>   43.4 KB</td>\n",
        "<td>2.8024976</td></tr>\n",
        "<tr><td>C2</td>\n",
        "<td>2-Byte Integers</td>\n",
        "<td>76</td>\n",
        "<td>40.860214</td>\n",
        "<td>    1.1 MB</td>\n",
        "<td>69.628105</td></tr></table></div>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "chunk_type    chunk_name                 count    count_percentage    size      size_percentage\n",
        "------------  -------------------------  -------  ------------------  --------  -----------------\n",
        "C0L           Constant Integers          10       5.37634             800  B    0.0504024\n",
        "C0D           Constant Reals             23       12.3656             1.8 KB    0.115925\n",
        "CBS           Bits                       2        1.07527             2.0 KB    0.127203\n",
        "CX0           Sparse Bits                10       5.37634             1.9 KB    0.124746\n",
        "C1            1-Byte Integers            40       21.5054             287.8 KB  18.565\n",
        "C1N           1-Byte Integers (w/o NAs)  19       10.2151             133.1 KB  8.58617\n",
        "C1S           1-Byte Fractions           6        3.22581             43.4 KB   2.8025\n",
        "C2            2-Byte Integers            76       40.8602             1.1 MB    69.6281"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Frame distribution summary:\n",
        "\n"
       ]
      },
      {
       "html": [
        "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td></td>\n",
        "<td>size</td>\n",
        "<td>number_of_rows</td>\n",
        "<td>number_of_chunks_per_column</td>\n",
        "<td>number_of_chunks</td></tr>\n",
        "<tr><td>172.16.2.17:54321</td>\n",
        "<td>    1.5 MB</td>\n",
        "<td>43978.0</td>\n",
        "<td>6.0</td>\n",
        "<td>186.0</td></tr>\n",
        "<tr><td>mean</td>\n",
        "<td>    1.5 MB</td>\n",
        "<td>43978.0</td>\n",
        "<td>6.0</td>\n",
        "<td>186.0</td></tr>\n",
        "<tr><td>min</td>\n",
        "<td>    1.5 MB</td>\n",
        "<td>43978.0</td>\n",
        "<td>6.0</td>\n",
        "<td>186.0</td></tr>\n",
        "<tr><td>max</td>\n",
        "<td>    1.5 MB</td>\n",
        "<td>43978.0</td>\n",
        "<td>6.0</td>\n",
        "<td>186.0</td></tr>\n",
        "<tr><td>stddev</td>\n",
        "<td>      0  B</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td></tr>\n",
        "<tr><td>total</td>\n",
        "<td>    1.5 MB</td>\n",
        "<td>43978.0</td>\n",
        "<td>6.0</td>\n",
        "<td>186.0</td></tr></table></div>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "                   size    number_of_rows    number_of_chunks_per_column    number_of_chunks\n",
        "-----------------  ------  ----------------  -----------------------------  ------------------\n",
        "172.16.2.17:54321  1.5 MB  43978             6                              186\n",
        "mean               1.5 MB  43978             6                              186\n",
        "min                1.5 MB  43978             6                              186\n",
        "max                1.5 MB  43978             6                              186\n",
        "stddev             0  B    0                 0                              0\n",
        "total              1.5 MB  43978             6                              186"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Column-by-Column Summary:\n",
        "\n"
       ]
      },
      {
       "html": [
        "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td></td>\n",
        "<td>Year</td>\n",
        "<td>Month</td>\n",
        "<td>DayofMonth</td>\n",
        "<td>DayOfWeek</td>\n",
        "<td>DepTime</td>\n",
        "<td>CRSDepTime</td>\n",
        "<td>ArrTime</td>\n",
        "<td>CRSArrTime</td>\n",
        "<td>UniqueCarrier</td>\n",
        "<td>FlightNum</td>\n",
        "<td>TailNum</td>\n",
        "<td>ActualElapsedTime</td>\n",
        "<td>CRSElapsedTime</td>\n",
        "<td>AirTime</td>\n",
        "<td>ArrDelay</td>\n",
        "<td>DepDelay</td>\n",
        "<td>Origin</td>\n",
        "<td>Dest</td>\n",
        "<td>Distance</td>\n",
        "<td>TaxiIn</td>\n",
        "<td>TaxiOut</td>\n",
        "<td>Cancelled</td>\n",
        "<td>CancellationCode</td>\n",
        "<td>Diverted</td>\n",
        "<td>CarrierDelay</td>\n",
        "<td>WeatherDelay</td>\n",
        "<td>NASDelay</td>\n",
        "<td>SecurityDelay</td>\n",
        "<td>LateAircraftDelay</td>\n",
        "<td>IsArrDelayed</td>\n",
        "<td>IsDepDelayed</td></tr>\n",
        "<tr><td>type</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>enum</td>\n",
        "<td>int</td>\n",
        "<td>enum</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>enum</td>\n",
        "<td>enum</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>enum</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>enum</td>\n",
        "<td>enum</td></tr>\n",
        "<tr><td>mins</td>\n",
        "<td>1987.0</td>\n",
        "<td>1.0</td>\n",
        "<td>1.0</td>\n",
        "<td>1.0</td>\n",
        "<td>1.0</td>\n",
        "<td>0.0</td>\n",
        "<td>1.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>1.0</td>\n",
        "<td>0.0</td>\n",
        "<td>16.0</td>\n",
        "<td>17.0</td>\n",
        "<td>14.0</td>\n",
        "<td>-63.0</td>\n",
        "<td>-16.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>11.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td></tr>\n",
        "<tr><td>maxs</td>\n",
        "<td>2008.0</td>\n",
        "<td>10.0</td>\n",
        "<td>31.0</td>\n",
        "<td>7.0</td>\n",
        "<td>2400.0</td>\n",
        "<td>2359.0</td>\n",
        "<td>2400.0</td>\n",
        "<td>2359.0</td>\n",
        "<td>9.0</td>\n",
        "<td>3949.0</td>\n",
        "<td>3500.0</td>\n",
        "<td>475.0</td>\n",
        "<td>437.0</td>\n",
        "<td>402.0</td>\n",
        "<td>475.0</td>\n",
        "<td>473.0</td>\n",
        "<td>131.0</td>\n",
        "<td>133.0</td>\n",
        "<td>3365.0</td>\n",
        "<td>128.0</td>\n",
        "<td>254.0</td>\n",
        "<td>1.0</td>\n",
        "<td>3.0</td>\n",
        "<td>1.0</td>\n",
        "<td>369.0</td>\n",
        "<td>201.0</td>\n",
        "<td>323.0</td>\n",
        "<td>14.0</td>\n",
        "<td>373.0</td>\n",
        "<td>1.0</td>\n",
        "<td>1.0</td></tr>\n",
        "<tr><td>sigma</td>\n",
        "<td>6.34436090171</td>\n",
        "<td>1.87471137134</td>\n",
        "<td>9.17579042586</td>\n",
        "<td>1.90501311913</td>\n",
        "<td>465.340899124</td>\n",
        "<td>476.251139993</td>\n",
        "<td>484.347487904</td>\n",
        "<td>492.750434123</td>\n",
        "<td>2.05121227084</td>\n",
        "<td>777.404369164</td>\n",
        "<td>1168.75931155</td>\n",
        "<td>73.9744416606</td>\n",
        "<td>73.40159463</td>\n",
        "<td>69.6363295151</td>\n",
        "<td>29.8402219624</td>\n",
        "<td>26.4388090429</td>\n",
        "<td>37.64411521</td>\n",
        "<td>33.930070329</td>\n",
        "<td>578.43800823</td>\n",
        "<td>4.20197993986</td>\n",
        "<td>9.9050857472</td>\n",
        "<td>0.155193141358</td>\n",
        "<td>0.182676305421</td>\n",
        "<td>0.0497234872189</td>\n",
        "<td>16.2057299045</td>\n",
        "<td>4.41677989873</td>\n",
        "<td>18.6197762215</td>\n",
        "<td>0.403940182102</td>\n",
        "<td>23.4875658741</td>\n",
        "<td>0.496887288343</td>\n",
        "<td>0.499377380318</td></tr>\n",
        "<tr><td>zero_count</td>\n",
        "<td>0</td>\n",
        "<td>0</td>\n",
        "<td>0</td>\n",
        "<td>0</td>\n",
        "<td>0</td>\n",
        "<td>569</td>\n",
        "<td>0</td>\n",
        "<td>569</td>\n",
        "<td>724</td>\n",
        "<td>0</td>\n",
        "<td>2</td>\n",
        "<td>0</td>\n",
        "<td>0</td>\n",
        "<td>0</td>\n",
        "<td>1514</td>\n",
        "<td>6393</td>\n",
        "<td>59</td>\n",
        "<td>172</td>\n",
        "<td>0</td>\n",
        "<td>623</td>\n",
        "<td>557</td>\n",
        "<td>42892</td>\n",
        "<td>81</td>\n",
        "<td>43869</td>\n",
        "<td>7344</td>\n",
        "<td>8840</td>\n",
        "<td>7388</td>\n",
        "<td>8914</td>\n",
        "<td>7140</td>\n",
        "<td>19537</td>\n",
        "<td>20887</td></tr>\n",
        "<tr><td>missing_count</td>\n",
        "<td>0</td>\n",
        "<td>0</td>\n",
        "<td>0</td>\n",
        "<td>0</td>\n",
        "<td>1086</td>\n",
        "<td>0</td>\n",
        "<td>1195</td>\n",
        "<td>0</td>\n",
        "<td>0</td>\n",
        "<td>0</td>\n",
        "<td>32</td>\n",
        "<td>1195</td>\n",
        "<td>13</td>\n",
        "<td>16649</td>\n",
        "<td>1195</td>\n",
        "<td>1086</td>\n",
        "<td>0</td>\n",
        "<td>0</td>\n",
        "<td>35</td>\n",
        "<td>16026</td>\n",
        "<td>16024</td>\n",
        "<td>0</td>\n",
        "<td>9774</td>\n",
        "<td>0</td>\n",
        "<td>35045</td>\n",
        "<td>35045</td>\n",
        "<td>35045</td>\n",
        "<td>35045</td>\n",
        "<td>35045</td>\n",
        "<td>0</td>\n",
        "<td>0</td></tr></table></div>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "               Year           Month          DayofMonth     DayOfWeek      DepTime        CRSDepTime     ArrTime        CRSArrTime     UniqueCarrier    FlightNum      TailNum        ActualElapsedTime    CRSElapsedTime    AirTime        ArrDelay       DepDelay       Origin       Dest          Distance      TaxiIn         TaxiOut       Cancelled       CancellationCode    Diverted         CarrierDelay    WeatherDelay    NASDelay       SecurityDelay    LateAircraftDelay    IsArrDelayed    IsDepDelayed\n",
        "-------------  -------------  -------------  -------------  -------------  -------------  -------------  -------------  -------------  ---------------  -------------  -------------  -------------------  ----------------  -------------  -------------  -------------  -----------  ------------  ------------  -------------  ------------  --------------  ------------------  ---------------  --------------  --------------  -------------  ---------------  -------------------  --------------  --------------\n",
        "type           int            int            int            int            int            int            int            int            enum             int            enum           int                  int               int            int            int            enum         enum          int           int            int           int             enum                int              int             int             int            int              int                  enum            enum\n",
        "mins           1987.0         1.0            1.0            1.0            1.0            0.0            1.0            0.0            0.0              1.0            0.0            16.0                 17.0              14.0           -63.0          -16.0          0.0          0.0           11.0          0.0            0.0           0.0             0.0                 0.0              0.0             0.0             0.0            0.0              0.0                  0.0             0.0\n",
        "maxs           2008.0         10.0           31.0           7.0            2400.0         2359.0         2400.0         2359.0         9.0              3949.0         3500.0         475.0                437.0             402.0          475.0          473.0          131.0        133.0         3365.0        128.0          254.0         1.0             3.0                 1.0              369.0           201.0           323.0          14.0             373.0                1.0             1.0\n",
        "sigma          6.34436090171  1.87471137134  9.17579042586  1.90501311913  465.340899124  476.251139993  484.347487904  492.750434123  2.05121227084    777.404369164  1168.75931155  73.9744416606        73.40159463       69.6363295151  29.8402219624  26.4388090429  37.64411521  33.930070329  578.43800823  4.20197993986  9.9050857472  0.155193141358  0.182676305421      0.0497234872189  16.2057299045   4.41677989873   18.6197762215  0.403940182102   23.4875658741        0.496887288343  0.499377380318\n",
        "zero_count     0              0              0              0              0              569            0              569            724              0              2              0                    0                 0              1514           6393           59           172           0             623            557           42892           81                  43869            7344            8840            7388           8914             7140                 19537           20887\n",
        "missing_count  0              0              0              0              1086           0              1195           0              0                0              32             1195                 13                16649          1195           1086           0            0             35            16026          16024         0               9774                0                35045           35045           35045          35045            35045                0               0"
       ]
      }
     ],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# ----------\n",
      "\n",
      "# 2- Data exploration and munging. Generate scatter plots \n",
      "# of various columns and plot fitted GLM model.\n",
      "\n",
      "# Function to fit a GLM model and plot the fitted (x,y) values\n",
      "def scatter_plot(data, x, y, max_points = 1000, fit = True):\n",
      "    if(fit):\n",
      "        lr = h2o.glm(x = data[[x]], y = data[y], family = \"gaussian\")\n",
      "        coeff = lr.coef()\n",
      "    df = data[[x,y]]\n",
      "    runif = df[y].runif()\n",
      "    df_subset = df[runif < float(max_points)/data.nrow()]\n",
      "    df_py = h2o.as_list(df_subset)\n",
      "    \n",
      "    if(fit): h2o.remove(lr._id)\n",
      "\n",
      "    # If x variable is string, generate box-and-whisker plot\n",
      "    if(df_py[x].dtype == \"object\"):\n",
      "        if interactive: df_py.boxplot(column = y, by = x)\n",
      "    # Otherwise, generate a scatter plot\n",
      "    else:\n",
      "        if interactive: df_py.plot(x = x, y = y, kind = \"scatter\")\n",
      "    \n",
      "    if(fit):\n",
      "        x_min = min(df_py[x])\n",
      "        x_max = max(df_py[x])\n",
      "        y_min = coeff[\"Intercept\"] + coeff[x]*x_min\n",
      "        y_max = coeff[\"Intercept\"] + coeff[x]*x_max\n",
      "        plt.plot([x_min, x_max], [y_min, y_max], \"k-\")\n",
      "    if interactive: plt.show()\n",
      "\n",
      "scatter_plot(data, \"Distance\", \"AirTime\", fit = True)\n",
      "scatter_plot(data, \"UniqueCarrier\", \"ArrDelay\", max_points = 5000, fit = False)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "\r",
        "glm Model Build Progress: [                                                  ] 00%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "glm Model Build Progress: [##################################################] 100%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Group flights by month\n",
      "aggregates = {\"Month\": [\"nrow\", 0, \"all\"], \"Cancelled\": [\"sum\", 1, \"all\"]}\n",
      "bpd = data.group_by(cols=[\"Month\"], aggregates=aggregates)\n",
      "bpd.show()\n",
      "bpd.describe()\n",
      "bpd.dim()\n",
      "\n",
      "# Convert columns to factors\n",
      "data[\"Year\"]      = data[\"Year\"]     .asfactor()\n",
      "data[\"Month\"]     = data[\"Month\"]    .asfactor()\n",
      "data[\"DayOfWeek\"] = data[\"DayOfWeek\"].asfactor()\n",
      "data[\"Cancelled\"] = data[\"Cancelled\"].asfactor()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "First 2 rows and first 3 columns: \n"
       ]
      },
      {
       "html": [
        "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>Month</td>\n",
        "<td>Cancelled</td>\n",
        "<td>Month</td></tr>\n",
        "<tr><td>10</td>\n",
        "<td>19990</td>\n",
        "<td>1999</td></tr>\n",
        "<tr><td>1</td>\n",
        "<td>41979</td>\n",
        "<td>41979</td></tr></table></div>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "  Month    Cancelled    Month\n",
        "-------  -----------  -------\n",
        "     10        19990     1999\n",
        "      1        41979    41979"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Rows: 2 Cols: 3\n",
        "\n",
        "Chunk compression summary:\n",
        "\n"
       ]
      },
      {
       "html": [
        "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>chunk_type</td>\n",
        "<td>chunk_name</td>\n",
        "<td>count</td>\n",
        "<td>count_percentage</td>\n",
        "<td>size</td>\n",
        "<td>size_percentage</td></tr>\n",
        "<tr><td>C1N</td>\n",
        "<td>1-Byte Integers (w/o NAs)</td>\n",
        "<td>1</td>\n",
        "<td>33.333336</td>\n",
        "<td>     70  B</td>\n",
        "<td>28.455284</td></tr>\n",
        "<tr><td>C2S</td>\n",
        "<td>2-Byte Fractions</td>\n",
        "<td>2</td>\n",
        "<td>66.66667</td>\n",
        "<td>    176  B</td>\n",
        "<td>71.544716</td></tr></table></div>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "chunk_type    chunk_name                 count    count_percentage    size    size_percentage\n",
        "------------  -------------------------  -------  ------------------  ------  -----------------\n",
        "C1N           1-Byte Integers (w/o NAs)  1        33.3333             70  B   28.4553\n",
        "C2S           2-Byte Fractions           2        66.6667             176  B  71.5447"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Frame distribution summary:\n",
        "\n"
       ]
      },
      {
       "html": [
        "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td></td>\n",
        "<td>size</td>\n",
        "<td>number_of_rows</td>\n",
        "<td>number_of_chunks_per_column</td>\n",
        "<td>number_of_chunks</td></tr>\n",
        "<tr><td>172.16.2.17:54321</td>\n",
        "<td>    246  B</td>\n",
        "<td>2.0</td>\n",
        "<td>1.0</td>\n",
        "<td>3.0</td></tr>\n",
        "<tr><td>mean</td>\n",
        "<td>    246  B</td>\n",
        "<td>2.0</td>\n",
        "<td>1.0</td>\n",
        "<td>3.0</td></tr>\n",
        "<tr><td>min</td>\n",
        "<td>    246  B</td>\n",
        "<td>2.0</td>\n",
        "<td>1.0</td>\n",
        "<td>3.0</td></tr>\n",
        "<tr><td>max</td>\n",
        "<td>    246  B</td>\n",
        "<td>2.0</td>\n",
        "<td>1.0</td>\n",
        "<td>3.0</td></tr>\n",
        "<tr><td>stddev</td>\n",
        "<td>      0  B</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td></tr>\n",
        "<tr><td>total</td>\n",
        "<td>    246  B</td>\n",
        "<td>2.0</td>\n",
        "<td>1.0</td>\n",
        "<td>3.0</td></tr></table></div>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "                   size    number_of_rows    number_of_chunks_per_column    number_of_chunks\n",
        "-----------------  ------  ----------------  -----------------------------  ------------------\n",
        "172.16.2.17:54321  246  B  2                 1                              3\n",
        "mean               246  B  2                 1                              3\n",
        "min                246  B  2                 1                              3\n",
        "max                246  B  2                 1                              3\n",
        "stddev             0  B    0                 0                              0\n",
        "total              246  B  2                 1                              3"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Column-by-Column Summary:\n",
        "\n"
       ]
      },
      {
       "html": [
        "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td></td>\n",
        "<td>Month</td>\n",
        "<td>Cancelled</td>\n",
        "<td>Month</td></tr>\n",
        "<tr><td>type</td>\n",
        "<td>int</td>\n",
        "<td>int</td>\n",
        "<td>int</td></tr>\n",
        "<tr><td>mins</td>\n",
        "<td>1.0</td>\n",
        "<td>19990.0</td>\n",
        "<td>1999.0</td></tr>\n",
        "<tr><td>maxs</td>\n",
        "<td>10.0</td>\n",
        "<td>41979.0</td>\n",
        "<td>41979.0</td></tr>\n",
        "<tr><td>sigma</td>\n",
        "<td>6.36396103068</td>\n",
        "<td>15548.5710115</td>\n",
        "<td>28270.1291118</td></tr>\n",
        "<tr><td>zero_count</td>\n",
        "<td>0</td>\n",
        "<td>0</td>\n",
        "<td>0</td></tr>\n",
        "<tr><td>missing_count</td>\n",
        "<td>0</td>\n",
        "<td>0</td>\n",
        "<td>0</td></tr></table></div>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "               Month          Cancelled      Month\n",
        "-------------  -------------  -------------  -------------\n",
        "type           int            int            int\n",
        "mins           1.0            19990.0        1999.0\n",
        "maxs           10.0           41979.0        41979.0\n",
        "sigma          6.36396103068  15548.5710115  28270.1291118\n",
        "zero_count     0              0              0\n",
        "missing_count  0              0              0"
       ]
      }
     ],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Calculate and plot travel time\n",
      "hour1 = data[\"CRSArrTime\"] / 100\n",
      "mins1 = data[\"CRSArrTime\"] % 100\n",
      "arrTime = hour1*60 + mins1\n",
      "\n",
      "hour2 = data[\"CRSDepTime\"] / 100\n",
      "mins2 = data[\"CRSDepTime\"] % 100\n",
      "depTime = hour2*60 + mins2\n",
      "\n",
      "# TODO: Replace this once list comprehension is supported. See PUBDEV-1286.\n",
      "# data[\"TravelTime\"] = [x if x > 0 else None for x in (arrTime - depTime)]\n",
      "data[\"TravelTime\"] = h2o.ifelse((arrTime-depTime) > 0, (arrTime-depTime), h2o.H2OFrame(python_obj=[[None] for r in range(data.nrow())]))[0]\n",
      "scatter_plot(data, \"Distance\", \"TravelTime\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "\r",
        "Parse Progress: [                                                  ] 00%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "Parse Progress: [##################################################] 100%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Uploaded py6c3155a4-db83-48e2-a484-689af0060304 into cluster with 43,978 rows and 1 cols\n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "glm Model Build Progress: [                                                  ] 00%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "glm Model Build Progress: [##################################################] 100%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 21
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Impute missing travel times and re-plot\n",
      "data.impute(column = \"Distance\", by = [\"Origin\", \"Dest\"])\n",
      "scatter_plot(data, \"Distance\", \"TravelTime\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "\r",
        "glm Model Build Progress: [                                                  ] 00%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "glm Model Build Progress: [##################################################] 100%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 22
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# ----------\n",
      "# 3- Fit a model on train; using test as validation\n",
      "\n",
      "# Create test/train split\n",
      "s = data[\"Year\"].runif()\n",
      "train = data[s <= 0.75]\n",
      "test  = data[s > 0.75]\n",
      "\n",
      "# Set predictor and response variables\n",
      "myY = \"IsDepDelayed\"\n",
      "myX = [\"Origin\", \"Dest\", \"Year\", \"UniqueCarrier\", \"DayOfWeek\", \"Month\", \"Distance\", \"FlightNum\"]\n",
      "\n",
      "# Simple GLM - Predict Delays\n",
      "data_glm = h2o.glm(x           =train[myX],\n",
      "                   y           =train[myY],\n",
      "                   validation_x=test [myX],\n",
      "                   validation_y=test [myY],\n",
      "                   family      =\"binomial\",\n",
      "                   standardize =True)\n",
      "\n",
      "# Simple GBM\n",
      "data_gbm = h2o.gbm(x              =train[myX],\n",
      "                   y              =train[myY],\n",
      "                   validation_x   =test [myX],\n",
      "                   validation_y   =test [myY],\n",
      "                   balance_classes=True,\n",
      "                   ntrees         =3,\n",
      "                   max_depth      =1,\n",
      "                   distribution   =\"bernoulli\",\n",
      "                   learn_rate     =0.1,\n",
      "                   min_rows       =2)\n",
      "\n",
      "# Complex GBM\n",
      "data_gbm2 = h2o.gbm(x              =train[myX],\n",
      "                    y              =train[myY],\n",
      "                    validation_x   =test [myX],\n",
      "                    validation_y   =test [myY],\n",
      "                    balance_classes=True,\n",
      "                    ntrees         =50,\n",
      "                    max_depth      =5,\n",
      "                    distribution   =\"bernoulli\",\n",
      "                    learn_rate     =0.1,\n",
      "                    min_rows       =2)\n",
      "\n",
      "# Simple Random Forest\n",
      "data_rf = h2o.random_forest(x              =train[myX],\n",
      "                            y              =train[myY],\n",
      "                            validation_x   =test [myX],\n",
      "                            validation_y   =test [myY],\n",
      "                            ntrees         =5,\n",
      "                            max_depth      =2,\n",
      "                            balance_classes=True)\n",
      "\n",
      "# Complex Random Forest\n",
      "data_rf2 = h2o.random_forest(x              =train[myX], \n",
      "                             y              =train[myY],\n",
      "                             validation_x   =test [myX],\n",
      "                             validation_y   =test [myY],\n",
      "                             ntrees         =10,\n",
      "                             max_depth      =5,\n",
      "                             balance_classes=True)\n",
      "\n",
      "# Deep Learning with 5 epochs\n",
      "data_dl = h2o.deeplearning(x                   =train[myX],\n",
      "                           y                   =train[myY],\n",
      "                           validation_x        =test [myX],\n",
      "                           validation_y        =test [myY],\n",
      "                           hidden              =[10,10],\n",
      "                           epochs              =5,\n",
      "                           variable_importances=True,\n",
      "                           balance_classes     =True,\n",
      "                           loss                =\"Automatic\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "\r",
        "glm Model Build Progress: [                                                  ] 00%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "glm Model Build Progress: [####                                              ] 08%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "glm Model Build Progress: [##################################################] 100%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "gbm Model Build Progress: [                                                  ] 00%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "gbm Model Build Progress: [##################################################] 100%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "gbm Model Build Progress: [                                                  ] 00%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "gbm Model Build Progress: [###                                               ] 06%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "gbm Model Build Progress: [##########                                        ] 20%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "gbm Model Build Progress: [################                                  ] 32%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "gbm Model Build Progress: [#######################                           ] 46%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "gbm Model Build Progress: [###############################                   ] 62%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "gbm Model Build Progress: [#######################################           ] 78%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "gbm Model Build Progress: [###############################################   ] 94%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "gbm Model Build Progress: [##################################################] 100%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "drf Model Build Progress: [                                                  ] 00%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "drf Model Build Progress: [##################################################] 100%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "drf Model Build Progress: [                                                  ] 00%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "drf Model Build Progress: [####################                              ] 40%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "drf Model Build Progress: [#############################################     ] 90%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "drf Model Build Progress: [##################################################] 100%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "deeplearning Model Build Progress: [                                                  ] 00%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "deeplearning Model Build Progress: [#####                                             ] 10%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "deeplearning Model Build Progress: [###############################                   ] 62%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\r",
        "deeplearning Model Build Progress: [##################################################] 100%"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Variable importances from each algorithm\n",
      "# Calculate magnitude of normalized GLM coefficients\n",
      "glm_varimp = data_glm.coef_norm()\n",
      "for k,v in glm_varimp.iteritems():\n",
      "    glm_varimp[k] = abs(glm_varimp[k])\n",
      "    \n",
      "# Sort in descending order by magnitude\n",
      "glm_sorted = sorted(glm_varimp.items(), key = operator.itemgetter(1), reverse = True)\n",
      "table = tabulate(glm_sorted, headers = [\"Predictor\", \"Normalized Coefficient\"], tablefmt = \"orgtbl\")\n",
      "print \"Variable Importances:\\n\\n\" + table\n",
      "\n",
      "data_gbm.varimp()\n",
      "data_rf.varimp()"
     ],
     "language": "python",
     "metadata": {
      "scrolled": true
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Variable Importances:\n",
        "\n",
        "| Predictor        |   Normalized Coefficient |\n",
        "|------------------+--------------------------|\n",
        "| Year.2008        |               2.28585    |\n",
        "| Origin.HPN       |               1.69543    |\n",
        "| Origin.LIH       |               1.638      |\n",
        "| Year.2003        |               1.60594    |\n",
        "| Origin.MDW       |               1.47263    |\n",
        "| Year.2007        |               1.47175    |\n",
        "| Dest.LYH         |               1.46167    |\n",
        "| Dest.HTS         |               1.28846    |\n",
        "| Origin.ERI       |               1.278      |\n",
        "| UniqueCarrier.HP |               1.14024    |\n",
        "| Origin.TLH       |               1.09897    |\n",
        "| Origin.HNL       |               1.08373    |\n",
        "| Origin.TRI       |               1.06732    |\n",
        "| Origin.LEX       |               0.997993   |\n",
        "| Origin.CHO       |               0.983148   |\n",
        "| Year.2001        |               0.970752   |\n",
        "| Year.2002        |               0.933122   |\n",
        "| Origin.CAE       |               0.902027   |\n",
        "| Origin.ATL       |               0.894547   |\n",
        "| Origin.BTV       |               0.893457   |\n",
        "| UniqueCarrier.TW |               0.889156   |\n",
        "| Year.2004        |               0.886399   |\n",
        "| Origin.GRR       |               0.871216   |\n",
        "| Dest.HRL         |               0.867962   |\n",
        "| Dest.PSP         |               0.840745   |\n",
        "| Origin.CRP       |               0.822434   |\n",
        "| Origin.LBB       |               0.817352   |\n",
        "| Origin.IAH       |               0.808256   |\n",
        "| Origin.MYR       |               0.802024   |\n",
        "| Year.2006        |               0.798849   |\n",
        "| Origin.ORD       |               0.79383    |\n",
        "| Origin.PBI       |               0.788992   |\n",
        "| Origin.ALB       |               0.784671   |\n",
        "| Year.1994        |               0.763265   |\n",
        "| Dest.CRP         |               0.762099   |\n",
        "| Origin.MIA       |               0.752895   |\n",
        "| Dest.FLL         |               0.749217   |\n",
        "| Origin.MAF       |               0.728962   |\n",
        "| Origin.MLB       |               0.713006   |\n",
        "| Origin.SAV       |               0.71218    |\n",
        "| Origin.JAX       |               0.692974   |\n",
        "| Origin.SRQ       |               0.687874   |\n",
        "| Origin.LYH       |               0.684658   |\n",
        "| Dest.CHO         |               0.684658   |\n",
        "| Origin.TYS       |               0.683076   |\n",
        "| Dest.OGG         |               0.676616   |\n",
        "| Origin.CMH       |               0.673951   |\n",
        "| Origin.ROA       |               0.666507   |\n",
        "| Dest.LBB         |               0.662483   |\n",
        "| Origin.SDF       |               0.641484   |\n",
        "| Origin.AUS       |               0.63652    |\n",
        "| Origin.MRY       |               0.625337   |\n",
        "| Year.1990        |               0.624939   |\n",
        "| Year.1996        |               0.618682   |\n",
        "| Dest.ICT         |               0.616668   |\n",
        "| Dest.ISP         |               0.611902   |\n",
        "| Dest.ELM         |               0.609561   |\n",
        "| Dest.EUG         |               0.591384   |\n",
        "| Origin.OGG       |               0.586688   |\n",
        "| Dest.EYW         |               0.5865     |\n",
        "| Origin.SYR       |               0.582702   |\n",
        "| Dest.TOL         |               0.581877   |\n",
        "| Dest.GEG         |               0.565753   |\n",
        "| Year.1997        |               0.560108   |\n",
        "| Origin.LAX       |               0.559894   |\n",
        "| Dest.PWM         |               0.559299   |\n",
        "| Origin.STL       |               0.546315   |\n",
        "| Origin.PIT       |               0.543209   |\n",
        "| Dest.IAH         |               0.540819   |\n",
        "| Origin.OMA       |               0.538106   |\n",
        "| Origin.PWM       |               0.537704   |\n",
        "| Dest.PBI         |               0.534746   |\n",
        "| Year.2005        |               0.534319   |\n",
        "| Origin.GSO       |               0.525335   |\n",
        "| Dest.IND         |               0.52243    |\n",
        "| Dest.SFO         |               0.515403   |\n",
        "| Origin.PSP       |               0.511714   |\n",
        "| Dest.DAY         |               0.511048   |\n",
        "| Origin.ACY       |               0.510181   |\n",
        "| Dest.LIH         |               0.505466   |\n",
        "| Dest.CAK         |               0.505063   |\n",
        "| UniqueCarrier.WN |               0.498225   |\n",
        "| Origin.IND       |               0.482325   |\n",
        "| Origin.MSY       |               0.480748   |\n",
        "| Origin.DFW       |               0.469985   |\n",
        "| Origin.PHL       |               0.464258   |\n",
        "| Dest.CLE         |               0.462228   |\n",
        "| Origin.LGA       |               0.455195   |\n",
        "| Origin.TUL       |               0.453434   |\n",
        "| Origin.BOI       |               0.447635   |\n",
        "| Dest.SAT         |               0.447196   |\n",
        "| Origin.FLL       |               0.445939   |\n",
        "| Origin.BDL       |               0.439833   |\n",
        "| Origin.SWF       |               0.437605   |\n",
        "| UniqueCarrier.PI |               0.433718   |\n",
        "| Origin.SLC       |               0.426539   |\n",
        "| Origin.MCI       |               0.424545   |\n",
        "| Origin.TUS       |               0.417783   |\n",
        "| Dest.HPN         |               0.411631   |\n",
        "| Origin.DSM       |               0.411606   |\n",
        "| Origin.RNO       |               0.411522   |\n",
        "| Dest.JAX         |               0.408934   |\n",
        "| Dest.GSO         |               0.408902   |\n",
        "| Dest.BDL         |               0.404167   |\n",
        "| Dest.OMA         |               0.394134   |\n",
        "| Origin.JFK       |               0.390293   |\n",
        "| Dest.BTV         |               0.380099   |\n",
        "| Dest.RSW         |               0.379812   |\n",
        "| Dest.FAY         |               0.377839   |\n",
        "| Dest.CAE         |               0.376909   |\n",
        "| Year.1995        |               0.367422   |\n",
        "| Origin.MCO       |               0.361513   |\n",
        "| Dest.CMH         |               0.354512   |\n",
        "| Dest.ROC         |               0.349751   |\n",
        "| Dest.ABQ         |               0.349527   |\n",
        "| Origin.SBN       |               0.347592   |\n",
        "| Dest.SBN         |               0.347142   |\n",
        "| Dest.SEA         |               0.344818   |\n",
        "| Dest.BUR         |               0.34096    |\n",
        "| Origin.DAY       |               0.338753   |\n",
        "| Origin.ABE       |               0.336332   |\n",
        "| Origin.EYW       |               0.336039   |\n",
        "| Dest.CHS         |               0.334728   |\n",
        "| UniqueCarrier.CO |               0.334387   |\n",
        "| Dest.TPA         |               0.331029   |\n",
        "| Origin.ROC       |               0.325753   |\n",
        "| Origin.EGE       |               0.319721   |\n",
        "| Dest.ORF         |               0.317641   |\n",
        "| Origin.OAK       |               0.308584   |\n",
        "| Dest.SMF         |               0.305322   |\n",
        "| Dest.COS         |               0.30247    |\n",
        "| Origin.BUF       |               0.299938   |\n",
        "| Dest.MIA         |               0.298249   |\n",
        "| Dest.MCO         |               0.297899   |\n",
        "| Dest.BUF         |               0.295137   |\n",
        "| Origin.BWI       |               0.294905   |\n",
        "| Dest.ONT         |               0.292885   |\n",
        "| Dest.FNT         |               0.28822    |\n",
        "| Origin.BUR       |               0.281797   |\n",
        "| Year.1992        |               0.280546   |\n",
        "| Origin.IAD       |               0.27936    |\n",
        "| Dest.FAT         |               0.273401   |\n",
        "| Dest.JFK         |               0.269927   |\n",
        "| Dest.STT         |               0.268764   |\n",
        "| UniqueCarrier.US |               0.266601   |\n",
        "| Origin.STX       |               0.263423   |\n",
        "| UniqueCarrier.AA |               0.262845   |\n",
        "| Origin.SFO       |               0.250746   |\n",
        "| Origin.BOS       |               0.248806   |\n",
        "| Year.1991        |               0.248019   |\n",
        "| Origin.MDT       |               0.247859   |\n",
        "| Origin.SAN       |               0.242567   |\n",
        "| Origin.CRW       |               0.242378   |\n",
        "| Origin.CLE       |               0.241045   |\n",
        "| Origin.TPA       |               0.240183   |\n",
        "| DayOfWeek.5      |               0.240084   |\n",
        "| Dest.GRR         |               0.238805   |\n",
        "| Origin.CLT       |               0.234494   |\n",
        "| Dest.PVD         |               0.23327    |\n",
        "| Origin.ISP       |               0.227559   |\n",
        "| Origin.BHM       |               0.223795   |\n",
        "| Dest.LAX         |               0.223084   |\n",
        "| Dest.AUS         |               0.221357   |\n",
        "| Dest.STL         |               0.216283   |\n",
        "| Month.10         |               0.215756   |\n",
        "| Year.1987        |               0.215756   |\n",
        "| Origin.DEN       |               0.207292   |\n",
        "| Dest.HOU         |               0.197121   |\n",
        "| DayOfWeek.2      |               0.196145   |\n",
        "| Origin.COS       |               0.19495    |\n",
        "| Origin.PHF       |               0.192259   |\n",
        "| Year.1999        |               0.184897   |\n",
        "| Dest.SLC         |               0.183123   |\n",
        "| Dest.IAD         |               0.18098    |\n",
        "| Year.1993        |               0.179029   |\n",
        "| Dest.CLT         |               0.17834    |\n",
        "| Dest.MKE         |               0.175025   |\n",
        "| Year.1989        |               0.170732   |\n",
        "| Year.2000        |               0.168468   |\n",
        "| Origin.OKC       |               0.168022   |\n",
        "| Dest.PNS         |               0.167044   |\n",
        "| Origin.SJC       |               0.166982   |\n",
        "| Dest.ABE         |               0.161995   |\n",
        "| Dest.ALB         |               0.155992   |\n",
        "| Dest.KOA         |               0.153826   |\n",
        "| Origin.MKE       |               0.152076   |\n",
        "| Origin.MEM       |               0.150336   |\n",
        "| Dest.ROA         |               0.149467   |\n",
        "| Dest.MSY         |               0.149433   |\n",
        "| Dest.OAJ         |               0.14703    |\n",
        "| Dest.UCA         |               0.146961   |\n",
        "| Origin.ORF       |               0.146924   |\n",
        "| DayOfWeek.6      |               0.144938   |\n",
        "| Dest.OAK         |               0.143941   |\n",
        "| Dest.DFW         |               0.143417   |\n",
        "| Origin.LIT       |               0.142983   |\n",
        "| Origin.ICT       |               0.142473   |\n",
        "| Dest.MSP         |               0.133873   |\n",
        "| Distance         |               0.131822   |\n",
        "| Dest.EWR         |               0.130674   |\n",
        "| Dest.ORD         |               0.127552   |\n",
        "| Dest.DTW         |               0.125984   |\n",
        "| UniqueCarrier.UA |               0.124704   |\n",
        "| Dest.SAN         |               0.123502   |\n",
        "| Dest.PHX         |               0.121518   |\n",
        "| Origin.HRL       |               0.11843    |\n",
        "| Dest.BGM         |               0.115701   |\n",
        "| DayOfWeek.4      |               0.114728   |\n",
        "| Dest.SNA         |               0.110722   |\n",
        "| Dest.LIT         |               0.106559   |\n",
        "| Dest.AVL         |               0.105538   |\n",
        "| Dest.PDX         |               0.100889   |\n",
        "| Origin.AVP       |               0.0999959  |\n",
        "| Origin.DTW       |               0.0987268  |\n",
        "| Month.1          |               0.0971013  |\n",
        "| Dest.BOI         |               0.0957132  |\n",
        "| Dest.PHL         |               0.0861169  |\n",
        "| Origin.KOA       |               0.0846479  |\n",
        "| Dest.TUS         |               0.0833103  |\n",
        "| Origin.ELP       |               0.0815918  |\n",
        "| UniqueCarrier.DL |               0.0797336  |\n",
        "| Dest.MDW         |               0.0783464  |\n",
        "| Dest.MCI         |               0.0773476  |\n",
        "| Dest.SYR         |               0.0771775  |\n",
        "| Origin.BGM       |               0.0750723  |\n",
        "| Dest.SDF         |               0.0728507  |\n",
        "| Origin.PVD       |               0.072424   |\n",
        "| Origin.RDU       |               0.071411   |\n",
        "| Dest.BWI         |               0.0706066  |\n",
        "| Origin.PHX       |               0.0701844  |\n",
        "| Dest.BHM         |               0.0701488  |\n",
        "| Dest.RIC         |               0.0692202  |\n",
        "| Origin.ABQ       |               0.055422   |\n",
        "| Dest.ILM         |               0.0543805  |\n",
        "| Intercept        |               0.0519814  |\n",
        "| Dest.LAS         |               0.0487192  |\n",
        "| Origin.SAT       |               0.0480328  |\n",
        "| DayOfWeek.7      |               0.0460972  |\n",
        "| Dest.DEN         |               0.0459439  |\n",
        "| Dest.AVP         |               0.0450818  |\n",
        "| Dest.ELP         |               0.0445682  |\n",
        "| Dest.BNA         |               0.0420472  |\n",
        "| Dest.DSM         |               0.0419726  |\n",
        "| Origin.HOU       |               0.0393885  |\n",
        "| Origin.PDX       |               0.0363553  |\n",
        "| Origin.SNA       |               0.0354122  |\n",
        "| Origin.SEA       |               0.0344875  |\n",
        "| Dest.ACY         |               0.0340972  |\n",
        "| Origin.CVG       |               0.0328146  |\n",
        "| Year.1998        |               0.0318468  |\n",
        "| Origin.SCK       |               0.031524   |\n",
        "| Origin.GEG       |               0.0312209  |\n",
        "| Dest.RDU         |               0.0310516  |\n",
        "| Origin.LAS       |               0.03026    |\n",
        "| Dest.MYR         |               0.0302372  |\n",
        "| Dest.RNO         |               0.0291863  |\n",
        "| Dest.DCA         |               0.0284603  |\n",
        "| Origin.BNA       |               0.0282776  |\n",
        "| Origin.SJU       |               0.027342   |\n",
        "| Origin.AMA       |               0.0262528  |\n",
        "| DayOfWeek.3      |               0.0247432  |\n",
        "| Dest.SJC         |               0.023634   |\n",
        "| Dest.CVG         |               0.0235706  |\n",
        "| FlightNum        |               0.0230809  |\n",
        "| Origin.RSW       |               0.0229344  |\n",
        "| Origin.SMF       |               0.0190434  |\n",
        "| Dest.PIT         |               0.0188431  |\n",
        "| Dest.TUL         |               0.0187144  |\n",
        "| Origin.EWR       |               0.018464   |\n",
        "| Dest.MHT         |               0.01552    |\n",
        "| Dest.LGA         |               0.0145985  |\n",
        "| Origin.MFR       |               0.0138638  |\n",
        "| Origin.DAL       |               0.0127005  |\n",
        "| Origin.DCA       |               0.00678773 |\n",
        "| Dest.BOS         |               0.00302899 |\n",
        "| Dest.ATL         |               0.00172438 |\n",
        "| Dest.OKC         |               0          |\n",
        "| Dest.ORH         |               0          |\n",
        "| Origin.ANC       |               0          |\n",
        "| Dest.SRQ         |               0          |\n",
        "| Dest.MRY         |               0          |\n",
        "| Dest.SJU         |               0          |\n",
        "| Year.1988        |               0          |\n",
        "| Dest.HNL         |               0          |\n",
        "| Origin.RIC       |               0          |\n",
        "| Origin.LAN       |               0          |\n",
        "| Dest.SWF         |               0          |\n",
        "| Dest.DAL         |               0          |\n",
        "| Origin.UCA       |               0          |\n",
        "| Dest.ERI         |               0          |\n",
        "| Dest.GSP         |               0          |\n",
        "| Origin.CHS       |               0          |\n",
        "| Origin.ONT       |               0          |\n",
        "| Origin.MHT       |               0          |\n",
        "| Dest.JAN         |               0          |\n",
        "| Origin.STT       |               0          |\n",
        "| Dest.CHA         |               0          |\n",
        "| Dest.MDT         |               0          |\n",
        "| Origin.GNV       |               0          |\n",
        "| Dest.AMA         |               0          |\n",
        "| Dest.LEX         |               0          |\n",
        "| Origin.JAN       |               0          |\n",
        "| UniqueCarrier.PS |               0          |\n",
        "| Dest.MAF         |               0          |\n",
        "| Dest.PHF         |               0          |\n",
        "| Dest.ANC         |               0          |\n",
        "| DayOfWeek.1      |               0          |\n",
        "| Dest.SCK         |               0          |\n",
        "| Origin.BIL       |               0          |\n",
        "| Origin.MSP       |               0          |\n",
        "\n",
        "Variable Importances:\n",
        "\n"
       ]
      },
      {
       "html": [
        "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>variable</td>\n",
        "<td>relative_importance</td>\n",
        "<td>scaled_importance</td>\n",
        "<td>percentage</td></tr>\n",
        "<tr><td>Year</td>\n",
        "<td>1232.10693359</td>\n",
        "<td>1.0</td>\n",
        "<td>1.0</td></tr>\n",
        "<tr><td>Month</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td></tr>\n",
        "<tr><td>DayOfWeek</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td></tr>\n",
        "<tr><td>UniqueCarrier</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td></tr>\n",
        "<tr><td>FlightNum</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td></tr>\n",
        "<tr><td>Origin</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td></tr>\n",
        "<tr><td>Dest</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td></tr>\n",
        "<tr><td>Distance</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td></tr></table></div>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "variable       relative_importance    scaled_importance    percentage\n",
        "-------------  ---------------------  -------------------  ------------\n",
        "Year           1232.11                1                    1\n",
        "Month          0                      0                    0\n",
        "DayOfWeek      0                      0                    0\n",
        "UniqueCarrier  0                      0                    0\n",
        "FlightNum      0                      0                    0\n",
        "Origin         0                      0                    0\n",
        "Dest           0                      0                    0\n",
        "Distance       0                      0                    0"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Variable Importances:\n",
        "\n"
       ]
      },
      {
       "html": [
        "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>variable</td>\n",
        "<td>relative_importance</td>\n",
        "<td>scaled_importance</td>\n",
        "<td>percentage</td></tr>\n",
        "<tr><td>Year</td>\n",
        "<td>958.766235352</td>\n",
        "<td>1.0</td>\n",
        "<td>0.464951857621</td></tr>\n",
        "<tr><td>UniqueCarrier</td>\n",
        "<td>421.378601074</td>\n",
        "<td>0.439500876791</td>\n",
        "<td>0.20434674909</td></tr>\n",
        "<tr><td>Origin</td>\n",
        "<td>326.425598145</td>\n",
        "<td>0.340464219649</td>\n",
        "<td>0.158299471379</td></tr>\n",
        "<tr><td>Dest</td>\n",
        "<td>215.331390381</td>\n",
        "<td>0.224592171106</td>\n",
        "<td>0.104424547163</td></tr>\n",
        "<tr><td>DayOfWeek</td>\n",
        "<td>63.3812866211</td>\n",
        "<td>0.0661071325669</td>\n",
        "<td>0.030736634089</td></tr>\n",
        "<tr><td>Distance</td>\n",
        "<td>49.1944389343</td>\n",
        "<td>0.0513101495656</td>\n",
        "<td>0.0238567493553</td></tr>\n",
        "<tr><td>FlightNum</td>\n",
        "<td>27.5988121033</td>\n",
        "<td>0.0287857572426</td>\n",
        "<td>0.013383991303</td></tr>\n",
        "<tr><td>Month</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td>\n",
        "<td>0.0</td></tr></table></div>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "variable       relative_importance    scaled_importance    percentage\n",
        "-------------  ---------------------  -------------------  ------------\n",
        "Year           958.766                1                    0.464952\n",
        "UniqueCarrier  421.379                0.439501             0.204347\n",
        "Origin         326.426                0.340464             0.158299\n",
        "Dest           215.331                0.224592             0.104425\n",
        "DayOfWeek      63.3813                0.0661071            0.0307366\n",
        "Distance       49.1944                0.0513101            0.0238567\n",
        "FlightNum      27.5988                0.0287858            0.013384\n",
        "Month          0                      0                    0"
       ]
      }
     ],
     "prompt_number": 24
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Model performance of GBM model on test data\n",
      "data_gbm2.model_performance(test)"
     ],
     "language": "python",
     "metadata": {
      "scrolled": false
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "ModelMetricsBinomial: gbm\n",
        "** Reported on test data. **\n",
        "\n",
        "MSE: 0.206699881626\n",
        "R^2: 0.170097556169\n",
        "LogLoss: 0.600060926768\n",
        "AUC: 0.738659969845\n",
        "Gini: 0.477319939689\n",
        "\n",
        "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.363358307534:\n",
        "\n"
       ]
      },
      {
       "html": [
        "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td></td>\n",
        "<td>NO</td>\n",
        "<td>YES</td>\n",
        "<td>Error</td>\n",
        "<td>Rate</td></tr>\n",
        "<tr><td>NO</td>\n",
        "<td>2089.0</td>\n",
        "<td>3070.0</td>\n",
        "<td>0.5951</td>\n",
        "<td> (3070.0/5159.0)</td></tr>\n",
        "<tr><td>YES</td>\n",
        "<td>722.0</td>\n",
        "<td>5109.0</td>\n",
        "<td>0.1238</td>\n",
        "<td> (722.0/5831.0)</td></tr>\n",
        "<tr><td>Total</td>\n",
        "<td>2811.0</td>\n",
        "<td>8179.0</td>\n",
        "<td>0.345</td>\n",
        "<td> (3792.0/10990.0)</td></tr></table></div>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "       NO    YES    Error    Rate\n",
        "-----  ----  -----  -------  ----------------\n",
        "NO     2089  3070   0.5951   (3070.0/5159.0)\n",
        "YES    722   5109   0.1238   (722.0/5831.0)\n",
        "Total  2811  8179   0.345    (3792.0/10990.0)"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Maximum Metrics:\n",
        "\n"
       ]
      },
      {
       "html": [
        "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>metric</td>\n",
        "<td>threshold</td>\n",
        "<td>value</td>\n",
        "<td>idx</td></tr>\n",
        "<tr><td>max f1</td>\n",
        "<td>0.363358307534</td>\n",
        "<td>0.729336188437</td>\n",
        "<td>286.0</td></tr>\n",
        "<tr><td>max f2</td>\n",
        "<td>0.203942289596</td>\n",
        "<td>0.85126451689</td>\n",
        "<td>366.0</td></tr>\n",
        "<tr><td>max f0point5</td>\n",
        "<td>0.536019190093</td>\n",
        "<td>0.698500394633</td>\n",
        "<td>193.0</td></tr>\n",
        "<tr><td>max accuracy</td>\n",
        "<td>0.469497349455</td>\n",
        "<td>0.677434030937</td>\n",
        "<td>226.0</td></tr>\n",
        "<tr><td>max precision</td>\n",
        "<td>0.957183699314</td>\n",
        "<td>1.0</td>\n",
        "<td>0.0</td></tr>\n",
        "<tr><td>max absolute_MCC</td>\n",
        "<td>0.500239647442</td>\n",
        "<td>0.351861408571</td>\n",
        "<td>211.0</td></tr>\n",
        "<tr><td>max min_per_class_accuracy</td>\n",
        "<td>0.507579143879</td>\n",
        "<td>0.675712347354</td>\n",
        "<td>208.0</td></tr></table></div>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "metric                      threshold    value     idx\n",
        "--------------------------  -----------  --------  -----\n",
        "max f1                      0.363358     0.729336  286\n",
        "max f2                      0.203942     0.851265  366\n",
        "max f0point5                0.536019     0.6985    193\n",
        "max accuracy                0.469497     0.677434  226\n",
        "max precision               0.957184     1         0\n",
        "max absolute_MCC            0.50024      0.351861  211\n",
        "max min_per_class_accuracy  0.507579     0.675712  208"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 25,
       "text": []
      }
     ],
     "prompt_number": 25
    }
   ],
   "metadata": {}
  }
 ]
}