The_Zimnat_Insurance_Assurance_Challenge_by_ZindiWeekendz_WinningSolution.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "The Zimnat Insurance Assurance Challenge by #ZindiWeekendz Approach3_Winning Solution.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "gpwwTJMC1CBu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "fe19a5dc-dbe9-476e-b158-c88e14f51c84"
      },
      "source": [
        "!pip install --upgrade git+https://github.com/stanfordmlgroup/ngboost.git\n",
        "!pip install catboost\n",
        "!pip install lightgbm\n",
        "!pip install xgboost\n"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting git+https://github.com/stanfordmlgroup/ngboost.git\n",
            "  Cloning https://github.com/stanfordmlgroup/ngboost.git to /tmp/pip-req-build-n4j52ajj\n",
            "  Running command git clone -q https://github.com/stanfordmlgroup/ngboost.git /tmp/pip-req-build-n4j52ajj\n",
            "Requirement already satisfied, skipping upgrade: numpy>=1.17.2 in /usr/local/lib/python3.6/dist-packages (from ngboost==0.2.1.dev0) (1.18.4)\n",
            "Requirement already satisfied, skipping upgrade: scipy>=1.3.1 in /usr/local/lib/python3.6/dist-packages (from ngboost==0.2.1.dev0) (1.4.1)\n",
            "Requirement already satisfied, skipping upgrade: scikit-learn>=0.21.3 in /usr/local/lib/python3.6/dist-packages (from ngboost==0.2.1.dev0) (0.22.2.post1)\n",
            "Requirement already satisfied, skipping upgrade: tqdm>=4.36.1 in /usr/local/lib/python3.6/dist-packages (from ngboost==0.2.1.dev0) (4.41.1)\n",
            "Collecting lifelines>=0.22.8\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/44/d6/3b0c9c2775ce299275b36246850e51c80f1ba3c1af9eef2a34b7bdbdb2d4/lifelines-0.24.8-py3-none-any.whl (332kB)\n",
            "\u001b[K     |████████████████████████████████| 337kB 2.7MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.21.3->ngboost==0.2.1.dev0) (0.15.1)\n",
            "Collecting autograd-gamma>=0.3\n",
            "  Downloading https://files.pythonhosted.org/packages/0a/07/d99339c9420b58b723a9189d1373e5c3889758b2202a1a7fe4a3b7a10c5a/autograd_gamma-0.4.2-py2.py3-none-any.whl\n",
            "Requirement already satisfied, skipping upgrade: matplotlib>=3.0 in /usr/local/lib/python3.6/dist-packages (from lifelines>=0.22.8->ngboost==0.2.1.dev0) (3.2.1)\n",
            "Requirement already satisfied, skipping upgrade: pandas>=0.23.0 in /usr/local/lib/python3.6/dist-packages (from lifelines>=0.22.8->ngboost==0.2.1.dev0) (1.0.3)\n",
            "Requirement already satisfied, skipping upgrade: autograd>=1.3 in /usr/local/lib/python3.6/dist-packages (from lifelines>=0.22.8->ngboost==0.2.1.dev0) (1.3)\n",
            "Requirement already satisfied, skipping upgrade: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->lifelines>=0.22.8->ngboost==0.2.1.dev0) (2.4.7)\n",
            "Requirement already satisfied, skipping upgrade: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->lifelines>=0.22.8->ngboost==0.2.1.dev0) (1.2.0)\n",
            "Requirement already satisfied, skipping upgrade: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->lifelines>=0.22.8->ngboost==0.2.1.dev0) (2.8.1)\n",
            "Requirement already satisfied, skipping upgrade: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->lifelines>=0.22.8->ngboost==0.2.1.dev0) (0.10.0)\n",
            "Requirement already satisfied, skipping upgrade: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.23.0->lifelines>=0.22.8->ngboost==0.2.1.dev0) (2018.9)\n",
            "Requirement already satisfied, skipping upgrade: future>=0.15.2 in /usr/local/lib/python3.6/dist-packages (from autograd>=1.3->lifelines>=0.22.8->ngboost==0.2.1.dev0) (0.16.0)\n",
            "Requirement already satisfied, skipping upgrade: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.1->matplotlib>=3.0->lifelines>=0.22.8->ngboost==0.2.1.dev0) (1.12.0)\n",
            "Building wheels for collected packages: ngboost\n",
            "  Building wheel for ngboost (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ngboost: filename=ngboost-0.2.1.dev0-cp36-none-any.whl size=34441 sha256=c7767536e6f82e8e705bf82c4b4947e54c6fb5c3fb143098a2bcb1fde211ab06\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-b19qeg76/wheels/6d/38/b0/7dc3b6546bf28b1cc407b44f75cbecf072fc2bae32e8f6a835\n",
            "Successfully built ngboost\n",
            "Installing collected packages: autograd-gamma, lifelines, ngboost\n",
            "Successfully installed autograd-gamma-0.4.2 lifelines-0.24.8 ngboost-0.2.1.dev0\n",
            "Collecting catboost\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b2/aa/e61819d04ef2bbee778bf4b3a748db1f3ad23512377e43ecfdc3211437a0/catboost-0.23.2-cp36-none-manylinux1_x86_64.whl (64.8MB)\n",
            "\u001b[K     |████████████████████████████████| 64.8MB 59kB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas>=0.24.0 in /usr/local/lib/python3.6/dist-packages (from catboost) (1.0.3)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from catboost) (3.2.1)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from catboost) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from catboost) (1.18.4)\n",
            "Requirement already satisfied: plotly in /usr/local/lib/python3.6/dist-packages (from catboost) (4.4.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from catboost) (1.12.0)\n",
            "Requirement already satisfied: graphviz in /usr/local/lib/python3.6/dist-packages (from catboost) (0.10.1)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.24.0->catboost) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.24.0->catboost) (2018.9)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (1.2.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (2.4.7)\n",
            "Requirement already satisfied: retrying>=1.3.3 in /usr/local/lib/python3.6/dist-packages (from plotly->catboost) (1.3.3)\n",
            "Installing collected packages: catboost\n",
            "Successfully installed catboost-0.23.2\n",
            "Requirement already satisfied: lightgbm in /usr/local/lib/python3.6/dist-packages (2.2.3)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from lightgbm) (1.4.1)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from lightgbm) (0.22.2.post1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from lightgbm) (1.18.4)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->lightgbm) (0.15.1)\n",
            "Requirement already satisfied: xgboost in /usr/local/lib/python3.6/dist-packages (0.90)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from xgboost) (1.18.4)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from xgboost) (1.4.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qnkerdRenU8A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd \n",
        "import numpy as np \n",
        "import math\n",
        "from datetime import datetime\n",
        "import random\n",
        "\n",
        "import sklearn \n",
        "from sklearn.metrics import mean_squared_error\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.metrics import log_loss\n",
        "\n",
        "from ngboost import NGBClassifier \n",
        "from ngboost.distns import Bernoulli\n",
        "from ngboost.learners import default_tree_learner\n",
        "\n",
        "from xgboost import XGBClassifier\n",
        "from catboost import CatBoostClassifier\n",
        "import lightgbm as lgb"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CKP9R28WoumQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "SEED_VAL = 1000\n",
        "random.seed(SEED_VAL)\n",
        "np.random.seed(SEED_VAL)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K0s1hy47oveW",
        "colab_type": "text"
      },
      "source": [
        "# Preapring Data\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mDngqEjvoyIn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "targets = pd.read_csv('drive/My Drive/The Zimnat Insurance Assurance Challenge by #ZindiWeekendz/train.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-C1C2rqI4FFy",
        "colab_type": "text"
      },
      "source": [
        "## Policy Data "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KzHsnOvD2krv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "policy_data = pd.read_csv('drive/My Drive/The Zimnat Insurance Assurance Challenge by #ZindiWeekendz/policy_data.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hvanf3Z_fPLk",
        "colab_type": "code",
        "outputId": "b13c58f3-5014-42c9-d89d-21fe22dcb38c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "policy_data=policy_data.drop_duplicates()\n",
        "print(len(policy_data['Policy ID'].unique()),len(policy_data))"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "51685 278988\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vHRUSxEqqwLf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "policy_data['PPR_PRODCD'] = pd.Categorical(policy_data['PPR_PRODCD'])\n",
        "policy_data['NLO_TYPE'] = pd.Categorical(policy_data['NLO_TYPE'])\n",
        "\n",
        "dfDummiesProd = pd.get_dummies(policy_data['PPR_PRODCD'], prefix = 'category')\n",
        "dfDummiesNLO = pd.get_dummies(policy_data['NLO_TYPE'], prefix = 'category')\n",
        "\n",
        "dfDummiesNLO['Policy ID'] = policy_data['Policy ID']\n",
        "dfDummiesProd['Policy ID'] = policy_data['Policy ID']\n",
        "\n",
        "dfDummiesNLO=dfDummiesNLO.groupby(by='Policy ID').sum()\n",
        "dfDummiesProd=dfDummiesProd.groupby(by='Policy ID').sum()\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-DCeceXhmCzS",
        "colab_type": "code",
        "outputId": "c0dbdc8f-6d6e-4905-b011-4d81244436cd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 226
        }
      },
      "source": [
        "#too null values\n",
        "policy_data.drop(['NPR_SUMASSURED'],axis=1,inplace=True)\n",
        "\n",
        "#NLO_AMOUNT - amount if there’s an extra charge \n",
        "policy_data['NLO_AMOUNT'] = policy_data['NLO_AMOUNT'].fillna(0)\n",
        "policy_data['NLO_AMOUNT'] = policy_data['NLO_AMOUNT'].apply(lambda x : 1 if x>0 else 0)\n",
        "\n",
        "#change date to datetime type \n",
        "policy_data['NP2_EFFECTDATE']= policy_data['NP2_EFFECTDATE'].apply(lambda a:\n",
        "datetime(year=int(a.split('/')[2]), month=int(a.split('/')[1]), day=int(a.split('/')[0]))\n",
        ")\n",
        "policy_data.head()"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Policy ID</th>\n",
              "      <th>NP2_EFFECTDATE</th>\n",
              "      <th>PPR_PRODCD</th>\n",
              "      <th>NPR_PREMIUM</th>\n",
              "      <th>NPH_LASTNAME</th>\n",
              "      <th>CLF_LIFECD</th>\n",
              "      <th>NSP_SUBPROPOSAL</th>\n",
              "      <th>NLO_TYPE</th>\n",
              "      <th>NLO_AMOUNT</th>\n",
              "      <th>AAG_AGCODE</th>\n",
              "      <th>PCL_LOCATCODE</th>\n",
              "      <th>OCCUPATION</th>\n",
              "      <th>CATEGORY</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>PID_EPZDSP8</td>\n",
              "      <td>2019-09-01</td>\n",
              "      <td>PPR_PRODCD_B2KVCE7</td>\n",
              "      <td>265.724174</td>\n",
              "      <td>NPH_LASTNAME_BPN2LEB</td>\n",
              "      <td>2</td>\n",
              "      <td>222</td>\n",
              "      <td>NLO_TYPE_DPBHSAH</td>\n",
              "      <td>0</td>\n",
              "      <td>AAG_AGCODE_APWOOPE</td>\n",
              "      <td>PCL_LOCATCODE_7SHK7I9</td>\n",
              "      <td>OCCUPATION_NNHJ7XV</td>\n",
              "      <td>CATEGORY_GWW4FYB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>PID_6M6G9IB</td>\n",
              "      <td>2018-08-01</td>\n",
              "      <td>PPR_PRODCD_64QNIHM</td>\n",
              "      <td>2795.069380</td>\n",
              "      <td>NPH_LASTNAME_U2H3GC6</td>\n",
              "      <td>1</td>\n",
              "      <td>111</td>\n",
              "      <td>NLO_TYPE_XTHV3A3</td>\n",
              "      <td>1</td>\n",
              "      <td>AAG_AGCODE_9Z3FBGA</td>\n",
              "      <td>PCL_LOCATCODE_7VFS3EQ</td>\n",
              "      <td>OCCUPATION_IKCIDKW</td>\n",
              "      <td>CATEGORY_R821UZV</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>PID_UL0F7LH</td>\n",
              "      <td>2017-08-01</td>\n",
              "      <td>PPR_PRODCD_KOFUYNN</td>\n",
              "      <td>2492.759107</td>\n",
              "      <td>NPH_LASTNAME_B68RERV</td>\n",
              "      <td>1</td>\n",
              "      <td>111</td>\n",
              "      <td>NLO_TYPE_XAJI0Y6</td>\n",
              "      <td>1</td>\n",
              "      <td>AAG_AGCODE_Y0LKFF0</td>\n",
              "      <td>PCL_LOCATCODE_SKPRCR4</td>\n",
              "      <td>OCCUPATION_NUJZA7T</td>\n",
              "      <td>CATEGORY_8DALFYO</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>PID_TRGUBTU</td>\n",
              "      <td>2018-04-01</td>\n",
              "      <td>PPR_PRODCD_KOFUYNN</td>\n",
              "      <td>3982.538095</td>\n",
              "      <td>NPH_LASTNAME_NPN3VGI</td>\n",
              "      <td>1</td>\n",
              "      <td>111</td>\n",
              "      <td>NLO_TYPE_XAJI0Y6</td>\n",
              "      <td>1</td>\n",
              "      <td>AAG_AGCODE_1OCF2N0</td>\n",
              "      <td>PCL_LOCATCODE_SPQHMX5</td>\n",
              "      <td>OCCUPATION_W9XA3KX</td>\n",
              "      <td>CATEGORY_LXSLG6M</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>PID_TODLPIB</td>\n",
              "      <td>2019-12-01</td>\n",
              "      <td>PPR_PRODCD_KOFUYNN</td>\n",
              "      <td>1143.953733</td>\n",
              "      <td>NPH_LASTNAME_9VSNH0E</td>\n",
              "      <td>3</td>\n",
              "      <td>555</td>\n",
              "      <td>NLO_TYPE_DPBHSAH</td>\n",
              "      <td>0</td>\n",
              "      <td>AAG_AGCODE_E31VV8B</td>\n",
              "      <td>PCL_LOCATCODE_0T6GYGX</td>\n",
              "      <td>OCCUPATION_NNHJ7XV</td>\n",
              "      <td>CATEGORY_GWW4FYB</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     Policy ID NP2_EFFECTDATE  ...          OCCUPATION          CATEGORY\n",
              "0  PID_EPZDSP8     2019-09-01  ...  OCCUPATION_NNHJ7XV  CATEGORY_GWW4FYB\n",
              "1  PID_6M6G9IB     2018-08-01  ...  OCCUPATION_IKCIDKW  CATEGORY_R821UZV\n",
              "2  PID_UL0F7LH     2017-08-01  ...  OCCUPATION_NUJZA7T  CATEGORY_8DALFYO\n",
              "3  PID_TRGUBTU     2018-04-01  ...  OCCUPATION_W9XA3KX  CATEGORY_LXSLG6M\n",
              "4  PID_TODLPIB     2019-12-01  ...  OCCUPATION_NNHJ7XV  CATEGORY_GWW4FYB\n",
              "\n",
              "[5 rows x 13 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jnty4al39ceS",
        "colab_type": "code",
        "outputId": "a9352bbf-e7cb-4ac2-d5cb-8e7d065f905d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 226
        }
      },
      "source": [
        "stats_pdata = policy_data.groupby(by=\"Policy ID\").agg({\n",
        "    'NP2_EFFECTDATE':['min','max'],\n",
        "})\n",
        "stats_pdata.columns = [\"_\".join(x) for x in stats_pdata.columns.ravel()]\n",
        "policy_data = pd.merge(policy_data,stats_pdata,on=\"Policy ID\",how=\"left\")\n",
        "#Add counts \n",
        "policy_data['count'] = policy_data.groupby(by='Policy ID').transform('count')['NP2_EFFECTDATE']\n",
        "policy_data.head()"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Policy ID</th>\n",
              "      <th>NP2_EFFECTDATE</th>\n",
              "      <th>PPR_PRODCD</th>\n",
              "      <th>NPR_PREMIUM</th>\n",
              "      <th>NPH_LASTNAME</th>\n",
              "      <th>CLF_LIFECD</th>\n",
              "      <th>NSP_SUBPROPOSAL</th>\n",
              "      <th>NLO_TYPE</th>\n",
              "      <th>NLO_AMOUNT</th>\n",
              "      <th>AAG_AGCODE</th>\n",
              "      <th>PCL_LOCATCODE</th>\n",
              "      <th>OCCUPATION</th>\n",
              "      <th>CATEGORY</th>\n",
              "      <th>NP2_EFFECTDATE_min</th>\n",
              "      <th>NP2_EFFECTDATE_max</th>\n",
              "      <th>count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>PID_EPZDSP8</td>\n",
              "      <td>2019-09-01</td>\n",
              "      <td>PPR_PRODCD_B2KVCE7</td>\n",
              "      <td>265.724174</td>\n",
              "      <td>NPH_LASTNAME_BPN2LEB</td>\n",
              "      <td>2</td>\n",
              "      <td>222</td>\n",
              "      <td>NLO_TYPE_DPBHSAH</td>\n",
              "      <td>0</td>\n",
              "      <td>AAG_AGCODE_APWOOPE</td>\n",
              "      <td>PCL_LOCATCODE_7SHK7I9</td>\n",
              "      <td>OCCUPATION_NNHJ7XV</td>\n",
              "      <td>CATEGORY_GWW4FYB</td>\n",
              "      <td>2019-09-01</td>\n",
              "      <td>2019-09-01</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>PID_6M6G9IB</td>\n",
              "      <td>2018-08-01</td>\n",
              "      <td>PPR_PRODCD_64QNIHM</td>\n",
              "      <td>2795.069380</td>\n",
              "      <td>NPH_LASTNAME_U2H3GC6</td>\n",
              "      <td>1</td>\n",
              "      <td>111</td>\n",
              "      <td>NLO_TYPE_XTHV3A3</td>\n",
              "      <td>1</td>\n",
              "      <td>AAG_AGCODE_9Z3FBGA</td>\n",
              "      <td>PCL_LOCATCODE_7VFS3EQ</td>\n",
              "      <td>OCCUPATION_IKCIDKW</td>\n",
              "      <td>CATEGORY_R821UZV</td>\n",
              "      <td>2018-08-01</td>\n",
              "      <td>2018-08-01</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>PID_UL0F7LH</td>\n",
              "      <td>2017-08-01</td>\n",
              "      <td>PPR_PRODCD_KOFUYNN</td>\n",
              "      <td>2492.759107</td>\n",
              "      <td>NPH_LASTNAME_B68RERV</td>\n",
              "      <td>1</td>\n",
              "      <td>111</td>\n",
              "      <td>NLO_TYPE_XAJI0Y6</td>\n",
              "      <td>1</td>\n",
              "      <td>AAG_AGCODE_Y0LKFF0</td>\n",
              "      <td>PCL_LOCATCODE_SKPRCR4</td>\n",
              "      <td>OCCUPATION_NUJZA7T</td>\n",
              "      <td>CATEGORY_8DALFYO</td>\n",
              "      <td>2017-08-01</td>\n",
              "      <td>2018-05-01</td>\n",
              "      <td>34</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>PID_TRGUBTU</td>\n",
              "      <td>2018-04-01</td>\n",
              "      <td>PPR_PRODCD_KOFUYNN</td>\n",
              "      <td>3982.538095</td>\n",
              "      <td>NPH_LASTNAME_NPN3VGI</td>\n",
              "      <td>1</td>\n",
              "      <td>111</td>\n",
              "      <td>NLO_TYPE_XAJI0Y6</td>\n",
              "      <td>1</td>\n",
              "      <td>AAG_AGCODE_1OCF2N0</td>\n",
              "      <td>PCL_LOCATCODE_SPQHMX5</td>\n",
              "      <td>OCCUPATION_W9XA3KX</td>\n",
              "      <td>CATEGORY_LXSLG6M</td>\n",
              "      <td>2018-04-01</td>\n",
              "      <td>2018-04-01</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>PID_TODLPIB</td>\n",
              "      <td>2019-12-01</td>\n",
              "      <td>PPR_PRODCD_KOFUYNN</td>\n",
              "      <td>1143.953733</td>\n",
              "      <td>NPH_LASTNAME_9VSNH0E</td>\n",
              "      <td>3</td>\n",
              "      <td>555</td>\n",
              "      <td>NLO_TYPE_DPBHSAH</td>\n",
              "      <td>0</td>\n",
              "      <td>AAG_AGCODE_E31VV8B</td>\n",
              "      <td>PCL_LOCATCODE_0T6GYGX</td>\n",
              "      <td>OCCUPATION_NNHJ7XV</td>\n",
              "      <td>CATEGORY_GWW4FYB</td>\n",
              "      <td>2019-12-01</td>\n",
              "      <td>2019-12-01</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     Policy ID NP2_EFFECTDATE  ... NP2_EFFECTDATE_max  count\n",
              "0  PID_EPZDSP8     2019-09-01  ...         2019-09-01     10\n",
              "1  PID_6M6G9IB     2018-08-01  ...         2018-08-01      2\n",
              "2  PID_UL0F7LH     2017-08-01  ...         2018-05-01     34\n",
              "3  PID_TRGUBTU     2018-04-01  ...         2018-04-01     10\n",
              "4  PID_TODLPIB     2019-12-01  ...         2019-12-01     12\n",
              "\n",
              "[5 rows x 16 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lky8huGvPBFo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "policy_data.drop(['NLO_TYPE','PPR_PRODCD'],axis=1,inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4uhScVOMxStV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#last name ~unique identfiefr , NLO AMount replaced by Nlo_amount_sum \n",
        "policy_data.drop(['NPH_LASTNAME','NLO_AMOUNT'],axis=1,inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6zMa0sV96uJ9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "policy_data['monthOfPolicy'] = policy_data['NP2_EFFECTDATE'].apply(lambda x:x.month)\n",
        "policy_data['diffMaxMinDate'] = (policy_data['NP2_EFFECTDATE_max']-policy_data['NP2_EFFECTDATE_min'])\n",
        "policy_data['diffMaxMinDate'] =policy_data['diffMaxMinDate'].apply(lambda x : x.days)\n",
        "policy_data['BOOLdiffMaxMinDate'] = policy_data['diffMaxMinDate'].apply(lambda x: 1 if x>0 else 0)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "btHbY_CHelk_",
        "colab_type": "code",
        "outputId": "59909f22-3f76-4179-a8e7-c2fb61337b7e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 226
        }
      },
      "source": [
        "policy_data = pd.merge(policy_data,dfDummiesNLO,left_on=\"Policy ID\",right_index=True,how=\"left\")\n",
        "policy_data = pd.merge(policy_data,dfDummiesProd,left_on=\"Policy ID\",right_index=True,how=\"left\")\n",
        "policy_data.head()"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Policy ID</th>\n",
              "      <th>NP2_EFFECTDATE</th>\n",
              "      <th>NPR_PREMIUM</th>\n",
              "      <th>CLF_LIFECD</th>\n",
              "      <th>NSP_SUBPROPOSAL</th>\n",
              "      <th>AAG_AGCODE</th>\n",
              "      <th>PCL_LOCATCODE</th>\n",
              "      <th>OCCUPATION</th>\n",
              "      <th>CATEGORY</th>\n",
              "      <th>NP2_EFFECTDATE_min</th>\n",
              "      <th>NP2_EFFECTDATE_max</th>\n",
              "      <th>count</th>\n",
              "      <th>monthOfPolicy</th>\n",
              "      <th>diffMaxMinDate</th>\n",
              "      <th>BOOLdiffMaxMinDate</th>\n",
              "      <th>category_NLO_TYPE_4V30T9N</th>\n",
              "      <th>category_NLO_TYPE_DPBHSAH</th>\n",
              "      <th>category_NLO_TYPE_T3W5UZB</th>\n",
              "      <th>category_NLO_TYPE_XAJI0Y6</th>\n",
              "      <th>category_NLO_TYPE_XTHV3A3</th>\n",
              "      <th>category_NLO_TYPE_ZMF8MDD</th>\n",
              "      <th>category_PPR_PRODCD_165U4LY</th>\n",
              "      <th>category_PPR_PRODCD_64QNIHM</th>\n",
              "      <th>category_PPR_PRODCD_6J9HUC7</th>\n",
              "      <th>category_PPR_PRODCD_8ZHO2CE</th>\n",
              "      <th>category_PPR_PRODCD_APTRA9E</th>\n",
              "      <th>category_PPR_PRODCD_B2KVCE7</th>\n",
              "      <th>category_PPR_PRODCD_EJ7YKFV</th>\n",
              "      <th>category_PPR_PRODCD_GLE2MHV</th>\n",
              "      <th>category_PPR_PRODCD_H6S21FA</th>\n",
              "      <th>category_PPR_PRODCD_ID7TAK9</th>\n",
              "      <th>category_PPR_PRODCD_KOFUYNN</th>\n",
              "      <th>category_PPR_PRODCD_KYXNF1V</th>\n",
              "      <th>category_PPR_PRODCD_OK3TM96</th>\n",
              "      <th>category_PPR_PRODCD_T8ONK55</th>\n",
              "      <th>category_PPR_PRODCD_V6HBYGK</th>\n",
              "      <th>category_PPR_PRODCD_W0F6GK1</th>\n",
              "      <th>category_PPR_PRODCD_XRWHUER</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>PID_EPZDSP8</td>\n",
              "      <td>2019-09-01</td>\n",
              "      <td>265.724174</td>\n",
              "      <td>2</td>\n",
              "      <td>222</td>\n",
              "      <td>AAG_AGCODE_APWOOPE</td>\n",
              "      <td>PCL_LOCATCODE_7SHK7I9</td>\n",
              "      <td>OCCUPATION_NNHJ7XV</td>\n",
              "      <td>CATEGORY_GWW4FYB</td>\n",
              "      <td>2019-09-01</td>\n",
              "      <td>2019-09-01</td>\n",
              "      <td>10</td>\n",
              "      <td>9</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>PID_6M6G9IB</td>\n",
              "      <td>2018-08-01</td>\n",
              "      <td>2795.069380</td>\n",
              "      <td>1</td>\n",
              "      <td>111</td>\n",
              "      <td>AAG_AGCODE_9Z3FBGA</td>\n",
              "      <td>PCL_LOCATCODE_7VFS3EQ</td>\n",
              "      <td>OCCUPATION_IKCIDKW</td>\n",
              "      <td>CATEGORY_R821UZV</td>\n",
              "      <td>2018-08-01</td>\n",
              "      <td>2018-08-01</td>\n",
              "      <td>2</td>\n",
              "      <td>8</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>PID_UL0F7LH</td>\n",
              "      <td>2017-08-01</td>\n",
              "      <td>2492.759107</td>\n",
              "      <td>1</td>\n",
              "      <td>111</td>\n",
              "      <td>AAG_AGCODE_Y0LKFF0</td>\n",
              "      <td>PCL_LOCATCODE_SKPRCR4</td>\n",
              "      <td>OCCUPATION_NUJZA7T</td>\n",
              "      <td>CATEGORY_8DALFYO</td>\n",
              "      <td>2017-08-01</td>\n",
              "      <td>2018-05-01</td>\n",
              "      <td>34</td>\n",
              "      <td>8</td>\n",
              "      <td>273</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>31</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>17</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>17</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>PID_TRGUBTU</td>\n",
              "      <td>2018-04-01</td>\n",
              "      <td>3982.538095</td>\n",
              "      <td>1</td>\n",
              "      <td>111</td>\n",
              "      <td>AAG_AGCODE_1OCF2N0</td>\n",
              "      <td>PCL_LOCATCODE_SPQHMX5</td>\n",
              "      <td>OCCUPATION_W9XA3KX</td>\n",
              "      <td>CATEGORY_LXSLG6M</td>\n",
              "      <td>2018-04-01</td>\n",
              "      <td>2018-04-01</td>\n",
              "      <td>10</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>PID_TODLPIB</td>\n",
              "      <td>2019-12-01</td>\n",
              "      <td>1143.953733</td>\n",
              "      <td>3</td>\n",
              "      <td>555</td>\n",
              "      <td>AAG_AGCODE_E31VV8B</td>\n",
              "      <td>PCL_LOCATCODE_0T6GYGX</td>\n",
              "      <td>OCCUPATION_NNHJ7XV</td>\n",
              "      <td>CATEGORY_GWW4FYB</td>\n",
              "      <td>2019-12-01</td>\n",
              "      <td>2019-12-01</td>\n",
              "      <td>12</td>\n",
              "      <td>12</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>11</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     Policy ID  ... category_PPR_PRODCD_XRWHUER\n",
              "0  PID_EPZDSP8  ...                           0\n",
              "1  PID_6M6G9IB  ...                           0\n",
              "2  PID_UL0F7LH  ...                           0\n",
              "3  PID_TRGUBTU  ...                           0\n",
              "4  PID_TODLPIB  ...                           0\n",
              "\n",
              "[5 rows x 38 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DKeICoPi8H4a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "policy_data.drop(['NP2_EFFECTDATE','NPR_PREMIUM','CLF_LIFECD',\n",
        "                  'NSP_SUBPROPOSAL','NP2_EFFECTDATE_min',\n",
        "                  'NP2_EFFECTDATE_max'],axis=1,inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QXd7jpvQwPDT",
        "colab_type": "code",
        "outputId": "afc97834-b895-4708-f0a8-eab9c0ea4192",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 226
        }
      },
      "source": [
        "policy_data = policy_data.drop_duplicates(subset=['Policy ID'])\n",
        "policy_data.head()"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Policy ID</th>\n",
              "      <th>AAG_AGCODE</th>\n",
              "      <th>PCL_LOCATCODE</th>\n",
              "      <th>OCCUPATION</th>\n",
              "      <th>CATEGORY</th>\n",
              "      <th>count</th>\n",
              "      <th>monthOfPolicy</th>\n",
              "      <th>diffMaxMinDate</th>\n",
              "      <th>BOOLdiffMaxMinDate</th>\n",
              "      <th>category_NLO_TYPE_4V30T9N</th>\n",
              "      <th>category_NLO_TYPE_DPBHSAH</th>\n",
              "      <th>category_NLO_TYPE_T3W5UZB</th>\n",
              "      <th>category_NLO_TYPE_XAJI0Y6</th>\n",
              "      <th>category_NLO_TYPE_XTHV3A3</th>\n",
              "      <th>category_NLO_TYPE_ZMF8MDD</th>\n",
              "      <th>category_PPR_PRODCD_165U4LY</th>\n",
              "      <th>category_PPR_PRODCD_64QNIHM</th>\n",
              "      <th>category_PPR_PRODCD_6J9HUC7</th>\n",
              "      <th>category_PPR_PRODCD_8ZHO2CE</th>\n",
              "      <th>category_PPR_PRODCD_APTRA9E</th>\n",
              "      <th>category_PPR_PRODCD_B2KVCE7</th>\n",
              "      <th>category_PPR_PRODCD_EJ7YKFV</th>\n",
              "      <th>category_PPR_PRODCD_GLE2MHV</th>\n",
              "      <th>category_PPR_PRODCD_H6S21FA</th>\n",
              "      <th>category_PPR_PRODCD_ID7TAK9</th>\n",
              "      <th>category_PPR_PRODCD_KOFUYNN</th>\n",
              "      <th>category_PPR_PRODCD_KYXNF1V</th>\n",
              "      <th>category_PPR_PRODCD_OK3TM96</th>\n",
              "      <th>category_PPR_PRODCD_T8ONK55</th>\n",
              "      <th>category_PPR_PRODCD_V6HBYGK</th>\n",
              "      <th>category_PPR_PRODCD_W0F6GK1</th>\n",
              "      <th>category_PPR_PRODCD_XRWHUER</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>PID_EPZDSP8</td>\n",
              "      <td>AAG_AGCODE_APWOOPE</td>\n",
              "      <td>PCL_LOCATCODE_7SHK7I9</td>\n",
              "      <td>OCCUPATION_NNHJ7XV</td>\n",
              "      <td>CATEGORY_GWW4FYB</td>\n",
              "      <td>10</td>\n",
              "      <td>9</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>PID_6M6G9IB</td>\n",
              "      <td>AAG_AGCODE_9Z3FBGA</td>\n",
              "      <td>PCL_LOCATCODE_7VFS3EQ</td>\n",
              "      <td>OCCUPATION_IKCIDKW</td>\n",
              "      <td>CATEGORY_R821UZV</td>\n",
              "      <td>2</td>\n",
              "      <td>8</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>PID_UL0F7LH</td>\n",
              "      <td>AAG_AGCODE_Y0LKFF0</td>\n",
              "      <td>PCL_LOCATCODE_SKPRCR4</td>\n",
              "      <td>OCCUPATION_NUJZA7T</td>\n",
              "      <td>CATEGORY_8DALFYO</td>\n",
              "      <td>34</td>\n",
              "      <td>8</td>\n",
              "      <td>273</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>31</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>17</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>17</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>PID_TRGUBTU</td>\n",
              "      <td>AAG_AGCODE_1OCF2N0</td>\n",
              "      <td>PCL_LOCATCODE_SPQHMX5</td>\n",
              "      <td>OCCUPATION_W9XA3KX</td>\n",
              "      <td>CATEGORY_LXSLG6M</td>\n",
              "      <td>10</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>PID_TODLPIB</td>\n",
              "      <td>AAG_AGCODE_E31VV8B</td>\n",
              "      <td>PCL_LOCATCODE_0T6GYGX</td>\n",
              "      <td>OCCUPATION_NNHJ7XV</td>\n",
              "      <td>CATEGORY_GWW4FYB</td>\n",
              "      <td>12</td>\n",
              "      <td>12</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>11</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     Policy ID  ... category_PPR_PRODCD_XRWHUER\n",
              "0  PID_EPZDSP8  ...                           0\n",
              "1  PID_6M6G9IB  ...                           0\n",
              "2  PID_UL0F7LH  ...                           0\n",
              "3  PID_TRGUBTU  ...                           0\n",
              "4  PID_TODLPIB  ...                           0\n",
              "\n",
              "[5 rows x 32 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y7whALwt9Usw",
        "colab_type": "text"
      },
      "source": [
        "## Create train and test df"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LYTG0TzN5u9O",
        "colab_type": "code",
        "outputId": "6b9af33c-5899-4ebf-c3bd-aa684b677cf4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81
        }
      },
      "source": [
        "ss = pd.read_csv('/content/drive/My Drive/The Zimnat Insurance Assurance Challenge by #ZindiWeekendz/sample_sub.csv')\n",
        "ss.head(1)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Policy ID</th>\n",
              "      <th>Lapse</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>PID_4928TWH</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     Policy ID  Lapse\n",
              "0  PID_4928TWH      0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fW50HQcjCqeh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train = pd.merge(policy_data,targets[targets.Lapse=='1'],on='Policy ID',how=\"left\")\n",
        "train['Lapse'] = train.Lapse.fillna(0)\n",
        "train.drop('Lapse Year',axis=1,inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ExrD3sx53i2S",
        "colab_type": "code",
        "outputId": "065c25b5-8a1d-46b9-b154-9cc59040263f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 226
        }
      },
      "source": [
        "test = pd.DataFrame()\n",
        "test['Policy ID']= ss['Policy ID']\n",
        "test = pd.merge(test,policy_data,how='left',on='Policy ID')\n",
        "test.head()"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Policy ID</th>\n",
              "      <th>AAG_AGCODE</th>\n",
              "      <th>PCL_LOCATCODE</th>\n",
              "      <th>OCCUPATION</th>\n",
              "      <th>CATEGORY</th>\n",
              "      <th>count</th>\n",
              "      <th>monthOfPolicy</th>\n",
              "      <th>diffMaxMinDate</th>\n",
              "      <th>BOOLdiffMaxMinDate</th>\n",
              "      <th>category_NLO_TYPE_4V30T9N</th>\n",
              "      <th>category_NLO_TYPE_DPBHSAH</th>\n",
              "      <th>category_NLO_TYPE_T3W5UZB</th>\n",
              "      <th>category_NLO_TYPE_XAJI0Y6</th>\n",
              "      <th>category_NLO_TYPE_XTHV3A3</th>\n",
              "      <th>category_NLO_TYPE_ZMF8MDD</th>\n",
              "      <th>category_PPR_PRODCD_165U4LY</th>\n",
              "      <th>category_PPR_PRODCD_64QNIHM</th>\n",
              "      <th>category_PPR_PRODCD_6J9HUC7</th>\n",
              "      <th>category_PPR_PRODCD_8ZHO2CE</th>\n",
              "      <th>category_PPR_PRODCD_APTRA9E</th>\n",
              "      <th>category_PPR_PRODCD_B2KVCE7</th>\n",
              "      <th>category_PPR_PRODCD_EJ7YKFV</th>\n",
              "      <th>category_PPR_PRODCD_GLE2MHV</th>\n",
              "      <th>category_PPR_PRODCD_H6S21FA</th>\n",
              "      <th>category_PPR_PRODCD_ID7TAK9</th>\n",
              "      <th>category_PPR_PRODCD_KOFUYNN</th>\n",
              "      <th>category_PPR_PRODCD_KYXNF1V</th>\n",
              "      <th>category_PPR_PRODCD_OK3TM96</th>\n",
              "      <th>category_PPR_PRODCD_T8ONK55</th>\n",
              "      <th>category_PPR_PRODCD_V6HBYGK</th>\n",
              "      <th>category_PPR_PRODCD_W0F6GK1</th>\n",
              "      <th>category_PPR_PRODCD_XRWHUER</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>PID_4928TWH</td>\n",
              "      <td>AAG_AGCODE_KZITWY3</td>\n",
              "      <td>PCL_LOCATCODE_0T6GYGX</td>\n",
              "      <td>OCCUPATION_9DOM5IG</td>\n",
              "      <td>CATEGORY_8DALFYO</td>\n",
              "      <td>1</td>\n",
              "      <td>8</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>PID_KBLLEGK</td>\n",
              "      <td>AAG_AGCODE_HG2GBMQ</td>\n",
              "      <td>PCL_LOCATCODE_PI2W0SA</td>\n",
              "      <td>OCCUPATION_8DRC11E</td>\n",
              "      <td>CATEGORY_LXSLG6M</td>\n",
              "      <td>2</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>PID_90F0QA3</td>\n",
              "      <td>AAG_AGCODE_C8COEA8</td>\n",
              "      <td>PCL_LOCATCODE_O6OBSFL</td>\n",
              "      <td>OCCUPATION_NNHJ7XV</td>\n",
              "      <td>CATEGORY_GWW4FYB</td>\n",
              "      <td>2</td>\n",
              "      <td>9</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>PID_18F3NHF</td>\n",
              "      <td>AAG_AGCODE_2ZOT2W3</td>\n",
              "      <td>PCL_LOCATCODE_295LOO6</td>\n",
              "      <td>OCCUPATION_NNHJ7XV</td>\n",
              "      <td>CATEGORY_GWW4FYB</td>\n",
              "      <td>1</td>\n",
              "      <td>12</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>PID_SX4QUVO</td>\n",
              "      <td>AAG_AGCODE_L1YVKZO</td>\n",
              "      <td>PCL_LOCATCODE_PEU5TF2</td>\n",
              "      <td>OCCUPATION_NNHJ7XV</td>\n",
              "      <td>CATEGORY_GWW4FYB</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     Policy ID  ... category_PPR_PRODCD_XRWHUER\n",
              "0  PID_4928TWH  ...                           1\n",
              "1  PID_KBLLEGK  ...                           0\n",
              "2  PID_90F0QA3  ...                           0\n",
              "3  PID_18F3NHF  ...                           0\n",
              "4  PID_SX4QUVO  ...                           0\n",
              "\n",
              "[5 rows x 32 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DN_Eio47-3Ef",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "le = LabelEncoder()\n",
        "for i in ['PCL_LOCATCODE','OCCUPATION','CATEGORY','AAG_AGCODE']:\n",
        "  le.fit(policy_data[i])\n",
        "  train[i] = le.transform(train[i])\n",
        "  test[i] = le.transform(test[i])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6UKR8HCf_w5J",
        "colab_type": "code",
        "outputId": "8fc55c7b-ca04-43d8-d9a9-b899bd80643f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 226
        }
      },
      "source": [
        "train.head()"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Policy ID</th>\n",
              "      <th>AAG_AGCODE</th>\n",
              "      <th>PCL_LOCATCODE</th>\n",
              "      <th>OCCUPATION</th>\n",
              "      <th>CATEGORY</th>\n",
              "      <th>count</th>\n",
              "      <th>monthOfPolicy</th>\n",
              "      <th>diffMaxMinDate</th>\n",
              "      <th>BOOLdiffMaxMinDate</th>\n",
              "      <th>category_NLO_TYPE_4V30T9N</th>\n",
              "      <th>category_NLO_TYPE_DPBHSAH</th>\n",
              "      <th>category_NLO_TYPE_T3W5UZB</th>\n",
              "      <th>category_NLO_TYPE_XAJI0Y6</th>\n",
              "      <th>category_NLO_TYPE_XTHV3A3</th>\n",
              "      <th>category_NLO_TYPE_ZMF8MDD</th>\n",
              "      <th>category_PPR_PRODCD_165U4LY</th>\n",
              "      <th>category_PPR_PRODCD_64QNIHM</th>\n",
              "      <th>category_PPR_PRODCD_6J9HUC7</th>\n",
              "      <th>category_PPR_PRODCD_8ZHO2CE</th>\n",
              "      <th>category_PPR_PRODCD_APTRA9E</th>\n",
              "      <th>category_PPR_PRODCD_B2KVCE7</th>\n",
              "      <th>category_PPR_PRODCD_EJ7YKFV</th>\n",
              "      <th>category_PPR_PRODCD_GLE2MHV</th>\n",
              "      <th>category_PPR_PRODCD_H6S21FA</th>\n",
              "      <th>category_PPR_PRODCD_ID7TAK9</th>\n",
              "      <th>category_PPR_PRODCD_KOFUYNN</th>\n",
              "      <th>category_PPR_PRODCD_KYXNF1V</th>\n",
              "      <th>category_PPR_PRODCD_OK3TM96</th>\n",
              "      <th>category_PPR_PRODCD_T8ONK55</th>\n",
              "      <th>category_PPR_PRODCD_V6HBYGK</th>\n",
              "      <th>category_PPR_PRODCD_W0F6GK1</th>\n",
              "      <th>category_PPR_PRODCD_XRWHUER</th>\n",
              "      <th>Lapse</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>PID_EPZDSP8</td>\n",
              "      <td>181</td>\n",
              "      <td>5</td>\n",
              "      <td>144</td>\n",
              "      <td>2</td>\n",
              "      <td>10</td>\n",
              "      <td>9</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>PID_6M6G9IB</td>\n",
              "      <td>170</td>\n",
              "      <td>6</td>\n",
              "      <td>110</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "      <td>8</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>PID_UL0F7LH</td>\n",
              "      <td>550</td>\n",
              "      <td>12</td>\n",
              "      <td>145</td>\n",
              "      <td>1</td>\n",
              "      <td>34</td>\n",
              "      <td>8</td>\n",
              "      <td>273</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>31</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>17</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>17</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>PID_TRGUBTU</td>\n",
              "      <td>34</td>\n",
              "      <td>13</td>\n",
              "      <td>192</td>\n",
              "      <td>3</td>\n",
              "      <td>10</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>PID_TODLPIB</td>\n",
              "      <td>225</td>\n",
              "      <td>1</td>\n",
              "      <td>144</td>\n",
              "      <td>2</td>\n",
              "      <td>12</td>\n",
              "      <td>12</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>11</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     Policy ID  AAG_AGCODE  ...  category_PPR_PRODCD_XRWHUER  Lapse\n",
              "0  PID_EPZDSP8         181  ...                            0      0\n",
              "1  PID_6M6G9IB         170  ...                            0      0\n",
              "2  PID_UL0F7LH         550  ...                            0      0\n",
              "3  PID_TRGUBTU          34  ...                            0      0\n",
              "4  PID_TODLPIB         225  ...                            0      0\n",
              "\n",
              "[5 rows x 33 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rEhDg5P3BdeM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train.rename(columns={'Lapse':'target'},inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z51tB99tAI6H",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train['target']=train['target'].astype('int')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CiuZxoQYDGfC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train.drop('Policy ID',axis=1,inplace=True)\n",
        "test.drop('Policy ID',axis=1,inplace=True)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-fX3NuPjBnTi",
        "colab_type": "code",
        "outputId": "b7c1d3eb-0a06-4e58-8f08-88efba8fe2ad",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(train),len(test)"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(51685, 43707)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aVeI5rBa-05T",
        "colab_type": "text"
      },
      "source": [
        "# Modeling"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4AoVu7Zm_a9f",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = train.drop(['target'],axis=1)\n",
        "y = train['target']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tUp2mVV9A6NO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train,X_test,y_train,y_test = train_test_split(X,y,stratify=y,random_state = SEED_VAL)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MV01GJiEt_uy",
        "colab_type": "text"
      },
      "source": [
        "## Random Forest"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uS1HbsidchnP",
        "colab_type": "code",
        "outputId": "7aec64b9-7f6a-482f-c20a-4f6f8d189484",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "rf = RandomForestClassifier(n_estimators=400,\n",
        "                      \n",
        "                      max_depth = 13,\n",
        "                      random_state=SEED_VAL)\n",
        "rf.fit(X_train,y_train)\n",
        "pred = rf.predict_proba(X_test)[:,1]\n",
        "print(log_loss(y_test,pred))"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.32432570921819254\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5tKagpUryBYL",
        "colab_type": "code",
        "outputId": "793b8ee5-fb5e-41d1-89a1-c4b901a53e16",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        }
      },
      "source": [
        "rf = RandomForestClassifier(n_estimators=400,\n",
        "                      \n",
        "                      max_depth = 13,\n",
        "                      random_state=SEED_VAL)\n",
        "rf.fit(X,y)\n"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n",
              "                       criterion='gini', max_depth=13, max_features='auto',\n",
              "                       max_leaf_nodes=None, max_samples=None,\n",
              "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
              "                       min_samples_leaf=1, min_samples_split=2,\n",
              "                       min_weight_fraction_leaf=0.0, n_estimators=400,\n",
              "                       n_jobs=None, oob_score=False, random_state=1000,\n",
              "                       verbose=0, warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_e-C4wU1yksT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "preds_rf = rf.predict_proba(test)[:,1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p8TWesGwqECz",
        "colab_type": "text"
      },
      "source": [
        "## XGB"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sPOIKzu9qGfe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "xgb_model  = XGBClassifier(\n",
        "                       n_estimators = 700,\n",
        "                       learning_rate=0.075,\n",
        "                       max_depth=3,\n",
        "                       min_child_weight=12,\n",
        "                       colsample_by_tree=0.7,\n",
        "                       seed=SEED_VAL,\n",
        "                       subsample=1,\n",
        "                       \n",
        "                      )\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o2Y4jV1BqP33",
        "colab_type": "code",
        "outputId": "484eae54-abed-4c13-e38e-8c06c1e81a04",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "xgb_model.fit(X_train,y_train)"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "XGBClassifier(base_score=0.5, booster='gbtree', colsample_by_tree=0.7,\n",
              "              colsample_bylevel=1, colsample_bynode=1, colsample_bytree=1,\n",
              "              gamma=0, learning_rate=0.075, max_delta_step=0, max_depth=3,\n",
              "              min_child_weight=12, missing=None, n_estimators=700, n_jobs=1,\n",
              "              nthread=None, objective='binary:logistic', random_state=0,\n",
              "              reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=1000,\n",
              "              silent=None, subsample=1, verbosity=1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QpUUXJ3CqP71",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pred_xgb = np.array(xgb_model.predict_proba(X_test))[:,1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eSQ_kEs5qP6g",
        "colab_type": "code",
        "outputId": "73fc8637-748f-43d9-b504-bb19f26ae66b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "log_loss(y_test,pred_xgb)"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.32026504531254707"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Pcfvi6MtVUd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "xgb_model.fit(X,y)\n",
        "preds_xgb = xgb_model.predict_proba(test)[:,1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d2yr6y7-pDW0",
        "colab_type": "text"
      },
      "source": [
        "## Catboost\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OthHIxy_pGfc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = CatBoostClassifier(iterations=900, \n",
        "                           learning_rate=0.1,\n",
        "                           loss_function='Logloss',\n",
        "                           random_seed = SEED_VAL,\n",
        "                           verbose=100) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3qCa2VYvpGhm",
        "colab_type": "code",
        "outputId": "bc256ff3-6016-43eb-851e-ac0bbf9f0813",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 208
        }
      },
      "source": [
        "model.fit(X_train,y_train)"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0:\tlearn: 0.5479885\ttotal: 70.4ms\tremaining: 1m 3s\n",
            "100:\tlearn: 0.3216773\ttotal: 2.13s\tremaining: 16.8s\n",
            "200:\tlearn: 0.3098372\ttotal: 4.17s\tremaining: 14.5s\n",
            "300:\tlearn: 0.3013640\ttotal: 6.21s\tremaining: 12.4s\n",
            "400:\tlearn: 0.2945252\ttotal: 8.26s\tremaining: 10.3s\n",
            "500:\tlearn: 0.2883160\ttotal: 10.3s\tremaining: 8.22s\n",
            "600:\tlearn: 0.2837161\ttotal: 12.4s\tremaining: 6.18s\n",
            "700:\tlearn: 0.2792504\ttotal: 14.5s\tremaining: 4.12s\n",
            "800:\tlearn: 0.2749949\ttotal: 16.6s\tremaining: 2.05s\n",
            "899:\tlearn: 0.2709167\ttotal: 18.6s\tremaining: 0us\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<catboost.core.CatBoostClassifier at 0x7f6be870dc18>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vggiVUJtpGkf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pred = np.array(model.predict_proba(X_test))[:,1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gm7HW0TgpGmZ",
        "colab_type": "code",
        "outputId": "088580df-48f3-474c-e0d2-d8b0fceeb36a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "log_loss(y_test,pred)"
      ],
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.3166625310104639"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mq-40lmHs4ot",
        "colab_type": "code",
        "outputId": "1fdce30a-68d3-45c4-c0fa-4459345277d7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 208
        }
      },
      "source": [
        "model.fit(X,y)"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0:\tlearn: 0.5437310\ttotal: 25.6ms\tremaining: 23s\n",
            "100:\tlearn: 0.3218548\ttotal: 2.64s\tremaining: 20.9s\n",
            "200:\tlearn: 0.3124369\ttotal: 5.17s\tremaining: 18s\n",
            "300:\tlearn: 0.3044347\ttotal: 7.74s\tremaining: 15.4s\n",
            "400:\tlearn: 0.2988292\ttotal: 10.3s\tremaining: 12.8s\n",
            "500:\tlearn: 0.2938678\ttotal: 12.9s\tremaining: 10.3s\n",
            "600:\tlearn: 0.2897955\ttotal: 15.4s\tremaining: 7.68s\n",
            "700:\tlearn: 0.2861176\ttotal: 18s\tremaining: 5.11s\n",
            "800:\tlearn: 0.2823689\ttotal: 20.6s\tremaining: 2.54s\n",
            "899:\tlearn: 0.2791929\ttotal: 23.1s\tremaining: 0us\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<catboost.core.CatBoostClassifier at 0x7f6be870dc18>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WdchbTjts8oE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "preds_cat = model.predict_proba(test)[:,1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s3Wul3mbpGum",
        "colab_type": "text"
      },
      "source": [
        "## LGB"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tvPpWOe6pIzp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "d_train = lgb.Dataset(X_train, label=y_train)\n",
        "# parameters for LightGBMClassifier\n",
        "params = {\n",
        "    'objective' :'binary',\n",
        "    'learning_rate' : 0.06,\n",
        "    'num_leaves' : 50,\n",
        "    'feature_fraction': 0.5, \n",
        "    'bagging_fraction': 0.85, \n",
        "    'bagging_freq':1,\n",
        "    'boosting_type' : 'gbdt',\n",
        "    'metric': 'binary_logloss',\n",
        "    'max_depth':260,\n",
        "    'seed':SEED_VAL\n",
        "}\n",
        "n_estimators = 200\n",
        "model_lgb = lgb.train(params, d_train, n_estimators, verbose_eval=1)\n",
        "d_train = 0\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W3YpCV7YpI2W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pred_prob_lgbm = model_lgb.predict(X_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cROUBcefpI4X",
        "colab_type": "code",
        "outputId": "1b0c096c-3d54-4a6c-c04e-b69e52d98cb6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "log_loss(y_test,pred_prob_lgbm)"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.31519373350099567"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NlkNQolipI7G",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "d_train = lgb.Dataset(X, label=y)\n",
        "model_lgb = lgb.train(params, d_train, n_estimators, verbose_eval=1)\n",
        "preds_lgb = model_lgb.predict(test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cjz36SQlpJEo",
        "colab_type": "text"
      },
      "source": [
        "## NGboost"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RphBmgD_pK3q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ngb_model  = NGBClassifier(Dist=Bernoulli,\n",
        "                           random_state=SEED_VAL,\n",
        "                           n_estimators=700,\n",
        "                           learning_rate=0.01)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J1ZM57f9rIFD",
        "colab_type": "code",
        "outputId": "9794d7ba-e4aa-4a4e-dd5c-9b2f0644dee2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 416
        }
      },
      "source": [
        "ngb_model.fit(np.array(X_train),np.array(y_train))"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[iter 0] loss=0.4302 val_loss=0.0000 scale=2.0000 norm=4.0000\n",
            "[iter 100] loss=0.3457 val_loss=0.0000 scale=2.0000 norm=3.2476\n",
            "[iter 200] loss=0.3381 val_loss=0.0000 scale=2.0000 norm=3.2088\n",
            "[iter 300] loss=0.3370 val_loss=0.0000 scale=2.0000 norm=3.2087\n",
            "[iter 400] loss=0.3365 val_loss=0.0000 scale=8.0000 norm=12.8339\n",
            "[iter 500] loss=0.3365 val_loss=0.0000 scale=8.0000 norm=12.8337\n",
            "[iter 600] loss=0.3365 val_loss=0.0000 scale=8.0000 norm=12.8337\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "NGBClassifier(Base=DecisionTreeRegressor(ccp_alpha=0.0,\n",
              "                                         criterion='friedman_mse', max_depth=3,\n",
              "                                         max_features=None, max_leaf_nodes=None,\n",
              "                                         min_impurity_decrease=0.0,\n",
              "                                         min_impurity_split=None,\n",
              "                                         min_samples_leaf=1,\n",
              "                                         min_samples_split=2,\n",
              "                                         min_weight_fraction_leaf=0.0,\n",
              "                                         presort='deprecated',\n",
              "                                         random_state=None, splitter='best'),\n",
              "              Dist=<class 'ngboost.distns.categorical.k_categorical.<locals>.Categorical'>,\n",
              "              Score=<class 'ngboost.scores.LogScore'>, col_sample=1.0,\n",
              "              learning_rate=0.01, minibatch_frac=1.0, n_estimators=700,\n",
              "              natural_gradient=True,\n",
              "              random_state=RandomState(MT19937) at 0x7F6BFD2E8DB0, tol=0.0001,\n",
              "              verbose=True, verbose_eval=100)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sM0x9_21pK5s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pred = ngb_model.pred_dist(np.array(X_test))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "14MSmJdArLXc",
        "colab_type": "code",
        "outputId": "2f9df5a9-f43e-4ca7-f69d-9d64f1908639",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "log_loss(y_test,pred.probs[1])"
      ],
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.3358886732837496"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LST3nprspK8N",
        "colab_type": "code",
        "outputId": "bed18713-8ecd-4596-a93a-dead28b20d84",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 416
        }
      },
      "source": [
        "ngb_model.fit(np.array(X),np.array(y))"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[iter 0] loss=0.3364 val_loss=0.0000 scale=2.0000 norm=3.2100\n",
            "[iter 100] loss=0.3362 val_loss=0.0000 scale=8.0000 norm=12.8273\n",
            "[iter 200] loss=0.3362 val_loss=0.0000 scale=8.0000 norm=12.8273\n",
            "[iter 300] loss=0.3362 val_loss=0.0000 scale=8.0000 norm=12.8273\n",
            "[iter 400] loss=0.3362 val_loss=0.0000 scale=0.0002 norm=0.0004\n",
            "[iter 500] loss=0.3362 val_loss=0.0000 scale=0.0002 norm=0.0004\n",
            "[iter 600] loss=0.3362 val_loss=0.0000 scale=0.0002 norm=0.0004\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "NGBClassifier(Base=DecisionTreeRegressor(ccp_alpha=0.0,\n",
              "                                         criterion='friedman_mse', max_depth=3,\n",
              "                                         max_features=None, max_leaf_nodes=None,\n",
              "                                         min_impurity_decrease=0.0,\n",
              "                                         min_impurity_split=None,\n",
              "                                         min_samples_leaf=1,\n",
              "                                         min_samples_split=2,\n",
              "                                         min_weight_fraction_leaf=0.0,\n",
              "                                         presort='deprecated',\n",
              "                                         random_state=None, splitter='best'),\n",
              "              Dist=<class 'ngboost.distns.categorical.k_categorical.<locals>.Categorical'>,\n",
              "              Score=<class 'ngboost.scores.LogScore'>, col_sample=1.0,\n",
              "              learning_rate=0.01, minibatch_frac=1.0, n_estimators=700,\n",
              "              natural_gradient=True,\n",
              "              random_state=RandomState(MT19937) at 0x7F6BFD2E8DB0, tol=0.0001,\n",
              "              verbose=True, verbose_eval=100)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WIVnMW7-rSjE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pred_ngb = ngb_model.pred_dist(np.array(test)).probs[1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8eEGOt9UDV2X",
        "colab_type": "text"
      },
      "source": [
        "# Sub"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_t_FcZuJDXiR",
        "colab_type": "code",
        "outputId": "ce897167-b0cc-4c99-98fd-a3485e22db04",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "source": [
        "sub=pd.DataFrame()\n",
        "sub['Policy ID'] = ss['Policy ID']\n",
        "sub['Lapse'] =  (((preds_xgb+preds_cat+preds_lgb)/3)*0.2+preds_rf*0.1 + pred_ngb*0.7)*0.75\n",
        "sub.head()"
      ],
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Policy ID</th>\n",
              "      <th>Lapse</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>PID_4928TWH</td>\n",
              "      <td>0.244966</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>PID_KBLLEGK</td>\n",
              "      <td>0.000062</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>PID_90F0QA3</td>\n",
              "      <td>0.000057</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>PID_18F3NHF</td>\n",
              "      <td>0.067835</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>PID_SX4QUVO</td>\n",
              "      <td>0.204525</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     Policy ID     Lapse\n",
              "0  PID_4928TWH  0.244966\n",
              "1  PID_KBLLEGK  0.000062\n",
              "2  PID_90F0QA3  0.000057\n",
              "3  PID_18F3NHF  0.067835\n",
              "4  PID_SX4QUVO  0.204525"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HrSfMJqLD6Rk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sub.to_csv('FinalCheck.csv',index=False)\n",
        "#0.2442192983426"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ErSxy7cy40f4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}