02Statistical_modeling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "hbVPykaPlop_"
   },
   "source": [
    "# 02 Statistical modeling\n",
    "\n",
    "*Author: Miao Cai* [miao.cai@slu.edu](miao.cai@slu.edu)\n",
    "\n",
    "## Statistical modeling\n",
    "\n",
    "We then use four different models to model the risk during the trip:\n",
    "\n",
    "- Logistic regression\n",
    "- Poisson regression\n",
    "- XGBoost\n",
    "- Deep learning (Neural networks)\n",
    "\n",
    "## import packages and read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 610
    },
    "colab_type": "code",
    "id": "8vsY6y1blCDV",
    "outputId": "0a824137-f613-4120-820b-48f87cfb1aa4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking whether there is an H2O instance running at http://localhost:54321..... not found.\n",
      "Attempting to start a local H2O server...\n",
      "  Java Version: openjdk version \"11.0.2\" 2019-01-15; OpenJDK Runtime Environment 18.9 (build 11.0.2+9); OpenJDK 64-Bit Server VM 18.9 (build 11.0.2+9, mixed mode)\n",
      "  Starting server from /Users/miaocai/anaconda3/envs/py36/lib/python3.6/site-packages/h2o/backend/bin/h2o.jar\n",
      "  Ice root: /var/folders/ng/t3l4gg0s0l398768705lv8vm0000gn/T/tmpunnpzfy6\n",
      "  JVM stdout: /var/folders/ng/t3l4gg0s0l398768705lv8vm0000gn/T/tmpunnpzfy6/h2o_miaocai_started_from_python.out\n",
      "  JVM stderr: /var/folders/ng/t3l4gg0s0l398768705lv8vm0000gn/T/tmpunnpzfy6/h2o_miaocai_started_from_python.err\n",
      "  Server is running at http://127.0.0.1:54321\n",
      "Connecting to H2O server at http://127.0.0.1:54321... successful.\n",
      "Warning: Your H2O cluster version is too old (10 months and 24 days)! Please download and install the latest version from http://h2o.ai/download/\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O cluster uptime:</td>\n",
       "<td>02 secs</td></tr>\n",
       "<tr><td>H2O cluster timezone:</td>\n",
       "<td>America/Chicago</td></tr>\n",
       "<tr><td>H2O data parsing timezone:</td>\n",
       "<td>UTC</td></tr>\n",
       "<tr><td>H2O cluster version:</td>\n",
       "<td>3.22.1.3</td></tr>\n",
       "<tr><td>H2O cluster version age:</td>\n",
       "<td>10 months and 24 days !!!</td></tr>\n",
       "<tr><td>H2O cluster name:</td>\n",
       "<td>H2O_from_python_miaocai_fiaorf</td></tr>\n",
       "<tr><td>H2O cluster total nodes:</td>\n",
       "<td>1</td></tr>\n",
       "<tr><td>H2O cluster free memory:</td>\n",
       "<td>2 Gb</td></tr>\n",
       "<tr><td>H2O cluster total cores:</td>\n",
       "<td>8</td></tr>\n",
       "<tr><td>H2O cluster allowed cores:</td>\n",
       "<td>8</td></tr>\n",
       "<tr><td>H2O cluster status:</td>\n",
       "<td>accepting new members, healthy</td></tr>\n",
       "<tr><td>H2O connection url:</td>\n",
       "<td>http://127.0.0.1:54321</td></tr>\n",
       "<tr><td>H2O connection proxy:</td>\n",
       "<td>None</td></tr>\n",
       "<tr><td>H2O internal security:</td>\n",
       "<td>False</td></tr>\n",
       "<tr><td>H2O API Extensions:</td>\n",
       "<td>XGBoost, Algos, AutoML, Core V3, Core V4</td></tr>\n",
       "<tr><td>Python version:</td>\n",
       "<td>3.6.7 final</td></tr></table></div>"
      ],
      "text/plain": [
       "--------------------------  ----------------------------------------\n",
       "H2O cluster uptime:         02 secs\n",
       "H2O cluster timezone:       America/Chicago\n",
       "H2O data parsing timezone:  UTC\n",
       "H2O cluster version:        3.22.1.3\n",
       "H2O cluster version age:    10 months and 24 days !!!\n",
       "H2O cluster name:           H2O_from_python_miaocai_fiaorf\n",
       "H2O cluster total nodes:    1\n",
       "H2O cluster free memory:    2 Gb\n",
       "H2O cluster total cores:    8\n",
       "H2O cluster allowed cores:  8\n",
       "H2O cluster status:         accepting new members, healthy\n",
       "H2O connection url:         http://127.0.0.1:54321\n",
       "H2O connection proxy:\n",
       "H2O internal security:      False\n",
       "H2O API Extensions:         XGBoost, Algos, AutoML, Core V3, Core V4\n",
       "Python version:             3.6.7 final\n",
       "--------------------------  ----------------------------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "numpy version: 1.15.4\n",
      "h2o version: 3.22.1.3\n"
     ]
    }
   ],
   "source": [
    "# !pip install h2o\n",
    "import numpy as np\n",
    "import h2o\n",
    "from h2o.estimators.glm import H2OGeneralizedLinearEstimator\n",
    "h2o.init()\n",
    "\n",
    "print(\"numpy version:\", np.__version__)\n",
    "print(\"h2o version:\", h2o.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 163
    },
    "colab_type": "code",
    "id": "EsdWiFs6pXTi",
    "outputId": "1c3626d9-69d5-43ea-ad06-b7363ce9776b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th style=\"text-align: right;\">  C1</th><th style=\"text-align: right;\">  y</th><th style=\"text-align: right;\">  Distance</th><th style=\"text-align: right;\">  Precipitation</th><th style=\"text-align: right;\">  Traffic</th><th style=\"text-align: right;\">  y_binary</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\">   0</td><td style=\"text-align: right;\">  0</td><td style=\"text-align: right;\">      1018</td><td style=\"text-align: right;\">              0</td><td style=\"text-align: right;\"> 0.299886</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">   1</td><td style=\"text-align: right;\">  0</td><td style=\"text-align: right;\">       973</td><td style=\"text-align: right;\">              0</td><td style=\"text-align: right;\"> 0.565617</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">   2</td><td style=\"text-align: right;\">  0</td><td style=\"text-align: right;\">      1021</td><td style=\"text-align: right;\">              0</td><td style=\"text-align: right;\"> 0.414564</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">   3</td><td style=\"text-align: right;\">  0</td><td style=\"text-align: right;\">       998</td><td style=\"text-align: right;\">              0</td><td style=\"text-align: right;\"> 0.559767</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">   4</td><td style=\"text-align: right;\">  0</td><td style=\"text-align: right;\">       985</td><td style=\"text-align: right;\">              0</td><td style=\"text-align: right;\"> 0.777217</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = h2o.import_file('https://raw.githubusercontent.com/caimiao0714/optimization_stats_case_study/master/data/simulated_data.csv')\n",
    "df[df['y']  > 0,'y_binary'] = 1\n",
    "df[df['y'] == 0,'y_binary'] = 0\n",
    "df['y_binary'] = df['y_binary'].asfactor()\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 163
    },
    "colab_type": "code",
    "id": "LZf7azQWqAJK",
    "outputId": "f8bb68cf-f007-4460-dca5-961ce5041198"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th style=\"text-align: right;\">  C1</th><th>Node A  </th><th>Node Z  </th><th style=\"text-align: right;\">  Distance</th><th style=\"text-align: right;\">  Precipitation</th><th style=\"text-align: right;\">  Traffic</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\">   0</td><td>node 1  </td><td>node 7  </td><td style=\"text-align: right;\">       800</td><td style=\"text-align: right;\">              0</td><td style=\"text-align: right;\"> 0.254345</td></tr>\n",
       "<tr><td style=\"text-align: right;\">   1</td><td>node 1  </td><td>node 11 </td><td style=\"text-align: right;\">       800</td><td style=\"text-align: right;\">              0</td><td style=\"text-align: right;\"> 0.243435</td></tr>\n",
       "<tr><td style=\"text-align: right;\">   2</td><td>node 1  </td><td>node 12 </td><td style=\"text-align: right;\">      2400</td><td style=\"text-align: right;\">              0</td><td style=\"text-align: right;\"> 0.254188</td></tr>\n",
       "<tr><td style=\"text-align: right;\">   3</td><td>node 2  </td><td>node 6  </td><td style=\"text-align: right;\">      1200</td><td style=\"text-align: right;\">              0</td><td style=\"text-align: right;\"> 0.424037</td></tr>\n",
       "<tr><td style=\"text-align: right;\">   4</td><td>node 2  </td><td>node 10 </td><td style=\"text-align: right;\">       900</td><td style=\"text-align: right;\">              0</td><td style=\"text-align: right;\"> 0.573477</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lk = h2o.import_file('https://raw.githubusercontent.com/caimiao0714/optimization_stats_case_study/master/data/links_traffic_precipitation.csv')\n",
    "lk.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "CjwDYXqmz3gD"
   },
   "source": [
    "### Split into train and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 72
    },
    "colab_type": "code",
    "id": "3DtRIAg3R_pz",
    "outputId": "1c02d579-0ae1-457d-e108-31266566ee45"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7021 rows in training set;\n",
      "1482 rows in test set;\n",
      "1497 rows in validation set.\n"
     ]
    }
   ],
   "source": [
    "df_splits = df.split_frame(ratios = [0.7, 0.15], seed = 123)\n",
    "\n",
    "df_train = df_splits[0]\n",
    "df_test  = df_splits[1]\n",
    "df_valid = df_splits[2]\n",
    "\n",
    "print(str(df_train.nrow) + \" rows in training set;\\n\" + \n",
    "      str(df_test.nrow) + \" rows in test set;\\n\" + \n",
    "      str(df_valid.nrow) + \" rows in validation set.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "eRXKKMQpppC5"
   },
   "source": [
    "## Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 223
    },
    "colab_type": "code",
    "id": "HxoJpwEyvBLZ",
    "outputId": "8ccd7a1d-ea04-4088-f8e3-bb7a3d1d9824"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "glm Model Build progress: |███████████████████████████████████████████████| 100%\n",
      "Coefficients: glm coefficients\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>names</b></td>\n",
       "<td><b>coefficients</b></td>\n",
       "<td><b>standardized_coefficients</b></td></tr>\n",
       "<tr><td>Intercept</td>\n",
       "<td>-3.6044384</td>\n",
       "<td>-2.1440719</td></tr>\n",
       "<tr><td>Distance</td>\n",
       "<td>0.0010077</td>\n",
       "<td>0.0322657</td></tr>\n",
       "<tr><td>Precipitation</td>\n",
       "<td>0.2563795</td>\n",
       "<td>0.0920454</td></tr>\n",
       "<tr><td>Traffic</td>\n",
       "<td>0.8305431</td>\n",
       "<td>0.1876087</td></tr></table></div>"
      ],
      "text/plain": [
       "names          coefficients    standardized_coefficients\n",
       "-------------  --------------  ---------------------------\n",
       "Intercept      -3.60444        -2.14407\n",
       "Distance       0.00100767      0.0322657\n",
       "Precipitation  0.25638         0.0920454\n",
       "Traffic        0.830543        0.187609"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit_logit = H2OGeneralizedLinearEstimator(family='binomial', \n",
    "                                          model_id='fit_logit')\n",
    "fit_logit.train(x = ['Precipitation', 'Traffic', 'Distance'], \n",
    "                y = 'y_binary', \n",
    "                training_frame = df_train)\n",
    "logit_test_fit = fit_logit.model_performance(df_test)\n",
    "fit_logit._model_json['output']['coefficients_table']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 235
    },
    "colab_type": "code",
    "id": "oexziW0Du_Jw",
    "outputId": "da320ea5-fa19-4c8e-e701-46916ad51b0a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic regression model evaluation:\n",
      "train AUC: 0.5596341292919064\n",
      "test  AUC: 0.5638801871833545\n",
      "---\n",
      "train Accuracy[[0.18502530292639058, 0.8936048995869534]]\n",
      "test  Accuracy[[0.17768208465318305, 0.8940620782726046]]\n",
      "---\n",
      "train MSE0.09478002969662627\n",
      "test  MSE0.09376565320196198\n",
      "---\n",
      "train R-square: 0.004278462347631851\n",
      "test  R-square: 0.004429388061521045\n"
     ]
    }
   ],
   "source": [
    "print(\"Logistic regression model evaluation:\")\n",
    "print(\"train AUC: \" + str(fit_logit.auc()))\n",
    "print(\"test  AUC: \" + str(logit_test_fit.auc()))\n",
    "print(\"---\")\n",
    "print(\"train Accuracy\" + str(fit_logit.accuracy()))\n",
    "print(\"test  Accuracy\" + str(logit_test_fit.accuracy()))\n",
    "print(\"---\")\n",
    "print(\"train MSE\" + str(fit_logit.mse()))\n",
    "print(\"test  MSE\" + str(logit_test_fit.mse()))\n",
    "print(\"---\")\n",
    "print(\"train R-square: \" + str(fit_logit.r2()))\n",
    "print(\"test  R-square: \" + str(logit_test_fit.r2()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "b2p8ndTzptqB"
   },
   "source": [
    "## Poisson regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 223
    },
    "colab_type": "code",
    "id": "3j98U9y8n74X",
    "outputId": "5c6ad79d-0a22-4099-f98c-d3efb06329a0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "glm Model Build progress: |███████████████████████████████████████████████| 100%\n",
      "Coefficients: glm coefficients\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>names</b></td>\n",
       "<td><b>coefficients</b></td>\n",
       "<td><b>standardized_coefficients</b></td></tr>\n",
       "<tr><td>Intercept</td>\n",
       "<td>-4.3718523</td>\n",
       "<td>-2.1020510</td></tr>\n",
       "<tr><td>Distance</td>\n",
       "<td>0.0017469</td>\n",
       "<td>0.0559369</td></tr>\n",
       "<tr><td>Precipitation</td>\n",
       "<td>0.3342644</td>\n",
       "<td>0.1200076</td></tr>\n",
       "<tr><td>Traffic</td>\n",
       "<td>0.9473540</td>\n",
       "<td>0.2139947</td></tr></table></div>"
      ],
      "text/plain": [
       "names          coefficients    standardized_coefficients\n",
       "-------------  --------------  ---------------------------\n",
       "Intercept      -4.37185        -2.10205\n",
       "Distance       0.00174692      0.0559369\n",
       "Precipitation  0.334264        0.120008\n",
       "Traffic        0.947354        0.213995"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit_poisson = H2OGeneralizedLinearEstimator(family='Poisson', \n",
    "                                            model_id='fit_poisson')\n",
    "fit_poisson.train(x = ['Precipitation', 'Traffic', 'Distance'], \n",
    "                  #offset_column = 'Distance',\n",
    "                  y = 'y', \n",
    "                  training_frame = df_train)\n",
    "poisson_test_fit = fit_poisson.model_performance(df_test)\n",
    "fit_poisson._model_json['output']['coefficients_table']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 126
    },
    "colab_type": "code",
    "id": "5GxAk4xMnvIu",
    "outputId": "06a49dd0-7ff9-4957-8f01-c6b00e26cd2b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Poisson regression model evaluation:\n",
      "train MSE: 0.16471763615686122\n",
      "test  MSE: 0.17174430698927012\n",
      "---\n",
      "train R-square: 0.006623137684569902\n",
      "test  R-square: 0.0043653505149615635\n"
     ]
    }
   ],
   "source": [
    "print(\"Poisson regression model evaluation:\")\n",
    "print(\"train MSE: \" + str(fit_poisson.mse()))\n",
    "print(\"test  MSE: \" + str(poisson_test_fit.mse()))\n",
    "print(\"---\")\n",
    "print(\"train R-square: \" + str(fit_poisson.r2()))\n",
    "print(\"test  R-square: \" + str(poisson_test_fit.r2()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "VxHYv6lM9LM0"
   },
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "bfBV2DN_9gdd",
    "outputId": "1b0f510c-7e42-4ad9-d4e3-a16463eef505"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "xgboost Model Build progress: |███████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "from h2o.estimators import H2OXGBoostEstimator\n",
    "xgboost_params = {\n",
    "      \"ntrees\" : 50, \n",
    "      \"max_depth\" : 5,\n",
    "      \"learn_rate\" : 0.001,\n",
    "      \"sample_rate\" : 0.7,\n",
    "      \"col_sample_rate_per_tree\" : 0.9,\n",
    "      \"min_rows\" : 5,\n",
    "      \"seed\": 4241,\n",
    "      \"score_tree_interval\": 10\n",
    "}\n",
    "fit_xgboost = H2OXGBoostEstimator(**xgboost_params)\n",
    "fit_xgboost.train(x = ['Precipitation', 'Traffic', 'Distance'], \n",
    "                  y = 'y_binary', \n",
    "                  training_frame = df_train, \n",
    "                  validation_frame = df_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 235
    },
    "colab_type": "code",
    "id": "BLIPHASz-i_1",
    "outputId": "8eb78f48-7f08-46cc-8a11-4ac62de12f9f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBoost regression model evaluation:\n",
      "train AUC: 0.6024401326114551\n",
      "test  AUC: 0.5456138569826353\n",
      "---\n",
      "train Accuracy[[0.4882012605667114, 0.8933200398803589]]\n",
      "test  Accuracy[[0.48741114139556885, 0.8940620782726046]]\n",
      "---\n",
      "train MSE0.2352502407526602\n",
      "test  MSE0.23521980545603724\n",
      "---\n",
      "train R-square: -1.4714460652217607\n",
      "test  R-square: -1.4974808755773372\n"
     ]
    }
   ],
   "source": [
    "xgboost_test_fit = fit_xgboost.model_performance(df_test)\n",
    "print(\"XGBoost regression model evaluation:\")\n",
    "print(\"train AUC: \" + str(fit_xgboost.auc()))\n",
    "print(\"test  AUC: \" + str(xgboost_test_fit.auc()))\n",
    "print(\"---\")\n",
    "print(\"train Accuracy\" + str(fit_xgboost.accuracy()))\n",
    "print(\"test  Accuracy\" + str(xgboost_test_fit.accuracy()))\n",
    "print(\"---\")\n",
    "print(\"train MSE\" + str(fit_xgboost.mse()))\n",
    "print(\"test  MSE\" + str(xgboost_test_fit.mse()))\n",
    "print(\"---\")\n",
    "print(\"train R-square: \" + str(fit_xgboost.r2()))\n",
    "print(\"test  R-square: \" + str(xgboost_test_fit.r2()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "yR3lN6TWAeOf"
   },
   "source": [
    "## Neural networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QFmp0Uv6BBgt"
   },
   "outputs": [],
   "source": [
    "from h2o.estimators.deeplearning import H2OAutoEncoderEstimator, H2ODeepLearningEstimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "WsQAW5mBBRLQ",
    "outputId": "ee38977e-afa8-42c3-fe85-8ececa636be0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "deeplearning Model Build progress: |██████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "fit_DL = H2ODeepLearningEstimator(epochs = 1000, \n",
    "                                  # hidden = [10, 10],\n",
    "                                  model_id = 'Deep learning', \n",
    "                                  seed = 1)\n",
    "fit_DL.train(x = ['Precipitation', 'Traffic', 'Distance'], \n",
    "             y = 'y_binary', \n",
    "             training_frame = df_train, \n",
    "             validation_frame = df_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 235
    },
    "colab_type": "code",
    "id": "MttXXE4PCjRG",
    "outputId": "4882fa7a-d7e2-425a-ff65-c2181bffc257"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deep learning model evaluation:\n",
      "train AUC: 0.5743293556716631\n",
      "test  AUC: 0.5327087442472058\n",
      "---\n",
      "train Accuracy[[0.5431840674778821, 0.8936048995869534]]\n",
      "test  Accuracy[[0.3826870843035134, 0.8940620782726046]]\n",
      "---\n",
      "train MSE0.09486584281605369\n",
      "test  MSE0.09535136906060053\n",
      "---\n",
      "train R-square: 0.003376943625802875\n",
      "test  R-square: -0.012407183261082366\n"
     ]
    }
   ],
   "source": [
    "DL_test_fit = fit_DL.model_performance(df_test)\n",
    "print(\"Deep learning model evaluation:\")\n",
    "print(\"train AUC: \" + str(fit_DL.auc()))\n",
    "print(\"test  AUC: \" + str(DL_test_fit.auc()))\n",
    "print(\"---\")\n",
    "print(\"train Accuracy\" + str(fit_DL.accuracy()))\n",
    "print(\"test  Accuracy\" + str(DL_test_fit.accuracy()))\n",
    "print(\"---\")\n",
    "print(\"train MSE\" + str(fit_DL.mse()))\n",
    "print(\"test  MSE\" + str(DL_test_fit.mse()))\n",
    "print(\"---\")\n",
    "print(\"train R-square: \" + str(fit_DL.r2()))\n",
    "print(\"test  R-square: \" + str(DL_test_fit.r2()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qxbgJh3WEh-C"
   },
   "source": [
    "## Prediction for links data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 290
    },
    "colab_type": "code",
    "id": "POyN7U_iEkJZ",
    "outputId": "5ecedbc3-846c-4f65-ca69-0f3b8a086a1a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "glm prediction progress: |████████████████████████████████████████████████| 100%\n",
      "glm prediction progress: |████████████████████████████████████████████████| 100%\n",
      "xgboost prediction progress: |████████████████████████████████████████████| 100%\n",
      "deeplearning prediction progress: |███████████████████████████████████████| 100%\n",
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n",
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n",
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n",
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th style=\"text-align: right;\">  C1</th><th>Node A  </th><th>Node Z  </th><th style=\"text-align: right;\">  Distance</th><th style=\"text-align: right;\">  Precipitation</th><th style=\"text-align: right;\">  Traffic</th><th style=\"text-align: right;\">  risk_logit</th><th style=\"text-align: right;\">  risk_poisson</th><th style=\"text-align: right;\">  risk_xgboost</th><th style=\"text-align: right;\">    risk_DL</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\">   0</td><td>node 1  </td><td>node 7  </td><td style=\"text-align: right;\">       800</td><td style=\"text-align: right;\">              0</td><td style=\"text-align: right;\"> 0.254345</td><td style=\"text-align: right;\">   0.0699758</td><td style=\"text-align: right;\">     0.0650006</td><td style=\"text-align: right;\">      0.480225</td><td style=\"text-align: right;\">0.000455431</td></tr>\n",
       "<tr><td style=\"text-align: right;\">   1</td><td>node 1  </td><td>node 11 </td><td style=\"text-align: right;\">       800</td><td style=\"text-align: right;\">              0</td><td style=\"text-align: right;\"> 0.243435</td><td style=\"text-align: right;\">   0.0693884</td><td style=\"text-align: right;\">     0.0643322</td><td style=\"text-align: right;\">      0.480225</td><td style=\"text-align: right;\">0.000441614</td></tr>\n",
       "<tr><td style=\"text-align: right;\">   2</td><td>node 1  </td><td>node 12 </td><td style=\"text-align: right;\">      2400</td><td style=\"text-align: right;\">              0</td><td style=\"text-align: right;\"> 0.254188</td><td style=\"text-align: right;\">   0.2739   </td><td style=\"text-align: right;\">     1.0635   </td><td style=\"text-align: right;\">      0.482557</td><td style=\"text-align: right;\">0.999274   </td></tr>\n",
       "<tr><td style=\"text-align: right;\">   3</td><td>node 2  </td><td>node 6  </td><td style=\"text-align: right;\">      1200</td><td style=\"text-align: right;\">              0</td><td style=\"text-align: right;\"> 0.424037</td><td style=\"text-align: right;\">   0.114756 </td><td style=\"text-align: right;\">     0.153534 </td><td style=\"text-align: right;\">      0.482797</td><td style=\"text-align: right;\">0.288758   </td></tr>\n",
       "<tr><td style=\"text-align: right;\">   4</td><td>node 2  </td><td>node 10 </td><td style=\"text-align: right;\">       900</td><td style=\"text-align: right;\">              0</td><td style=\"text-align: right;\"> 0.573477</td><td style=\"text-align: right;\">   0.0978591</td><td style=\"text-align: right;\">     0.104734 </td><td style=\"text-align: right;\">      0.480964</td><td style=\"text-align: right;\">0.021318   </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "risk_logit = fit_logit.predict(lk).as_data_frame(True).p1.tolist()\n",
    "risk_poisson = fit_poisson.predict(lk).as_data_frame(True).predict.tolist()\n",
    "risk_xgboost = fit_xgboost.predict(lk).as_data_frame(True).p1.tolist()\n",
    "risk_DL = fit_DL.predict(lk).as_data_frame(True).p1.tolist()\n",
    "\n",
    "lk_risks = lk.cbind(h2o.H2OFrame(risk_logit).set_names(['risk_logit'])).\\\n",
    "              cbind(h2o.H2OFrame(risk_poisson).set_names(['risk_poisson'])).\\\n",
    "              cbind(h2o.H2OFrame(risk_xgboost).set_names(['risk_xgboost'])).\\\n",
    "              cbind(h2o.H2OFrame(risk_DL).set_names(['risk_DL']))\n",
    "lk_risks.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "UNsjJwTcTRdY",
    "outputId": "7288983c-4cca-4daf-baf3-261b158106bf"
   },
   "outputs": [],
   "source": [
    "lk_risks.as_data_frame().to_csv('data/lk_risks.csv')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "02Statistical modeling.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "py36",
   "language": "python",
   "name": "py36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}