blog_notebooks/santander/cudf_tf_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CuDF TF Demo\n",
    "This notebook is for **Archive Only.** Please **do not** expect it to run in the latest releases.  Cell output is saved and shown.  \n",
    "\n",
    "Placed 17/8808. [Blog](https://medium.com/rapids-ai/financial-data-modeling-with-rapids-5bca466f348) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cudf version: 0.6.1+0.gbeb4ef3.dirty\n",
      "tensorflow version: 1.13.1\n"
     ]
    }
   ],
   "source": [
    "import cudf as gd\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from collections import OrderedDict\n",
    "import time\n",
    "from tqdm import tqdm\n",
    "import pickle\n",
    "import random\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "print('cudf version:',gd.__version__)\n",
    "print('tensorflow version:',tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Please download data from https://www.kaggle.com/c/santander-customer-transaction-prediction/data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = '../input'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ETL"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 460 ms, sys: 92.9 ms, total: 553 ms\n",
      "Wall time: 552 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "cols = ['ID_code', 'target'] + ['var_%d'%i for i in range(200)]\n",
    "dtypes = ['int32', 'int32'] + ['float32' for i in range(200)]\n",
    "train_gd = gd.read_csv('%s/train.csv'%PATH,names=cols,dtype=dtypes,skiprows=1)\n",
    "\n",
    "cols = ['ID_code', 'target'] + ['var_%d'%i for i in range(200)]\n",
    "dtypes = ['int32', 'int32'] + ['float32' for i in range(200)]\n",
    "test_gd = gd.read_csv('%s/test.csv'%PATH,names=cols,dtype=dtypes,skiprows=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_code</th>\n",
       "      <th>target</th>\n",
       "      <th>var_0</th>\n",
       "      <th>var_1</th>\n",
       "      <th>var_2</th>\n",
       "      <th>var_3</th>\n",
       "      <th>var_4</th>\n",
       "      <th>var_5</th>\n",
       "      <th>var_6</th>\n",
       "      <th>var_7</th>\n",
       "      <th>...</th>\n",
       "      <th>var_190</th>\n",
       "      <th>var_191</th>\n",
       "      <th>var_192</th>\n",
       "      <th>var_193</th>\n",
       "      <th>var_194</th>\n",
       "      <th>var_195</th>\n",
       "      <th>var_196</th>\n",
       "      <th>var_197</th>\n",
       "      <th>var_198</th>\n",
       "      <th>var_199</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>75153670</td>\n",
       "      <td>0</td>\n",
       "      <td>8.925500</td>\n",
       "      <td>-6.7863</td>\n",
       "      <td>11.908100</td>\n",
       "      <td>5.0930</td>\n",
       "      <td>11.460700</td>\n",
       "      <td>-9.2834</td>\n",
       "      <td>5.1187</td>\n",
       "      <td>18.626602</td>\n",
       "      <td>...</td>\n",
       "      <td>4.4354</td>\n",
       "      <td>3.964200</td>\n",
       "      <td>3.1364</td>\n",
       "      <td>1.691000</td>\n",
       "      <td>18.522701</td>\n",
       "      <td>-2.3978</td>\n",
       "      <td>7.8784</td>\n",
       "      <td>8.5635</td>\n",
       "      <td>12.780300</td>\n",
       "      <td>-1.091400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>75153671</td>\n",
       "      <td>0</td>\n",
       "      <td>11.500600</td>\n",
       "      <td>-4.1473</td>\n",
       "      <td>13.858801</td>\n",
       "      <td>5.3890</td>\n",
       "      <td>12.362201</td>\n",
       "      <td>7.0433</td>\n",
       "      <td>5.6208</td>\n",
       "      <td>16.533800</td>\n",
       "      <td>...</td>\n",
       "      <td>7.6421</td>\n",
       "      <td>7.721400</td>\n",
       "      <td>2.5837</td>\n",
       "      <td>10.951600</td>\n",
       "      <td>15.430499</td>\n",
       "      <td>2.0339</td>\n",
       "      <td>8.1267</td>\n",
       "      <td>8.7889</td>\n",
       "      <td>18.355999</td>\n",
       "      <td>1.951800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>75153672</td>\n",
       "      <td>0</td>\n",
       "      <td>8.609301</td>\n",
       "      <td>-2.7457</td>\n",
       "      <td>12.080500</td>\n",
       "      <td>7.8928</td>\n",
       "      <td>10.582500</td>\n",
       "      <td>-9.0837</td>\n",
       "      <td>6.9427</td>\n",
       "      <td>14.615500</td>\n",
       "      <td>...</td>\n",
       "      <td>2.9057</td>\n",
       "      <td>9.790500</td>\n",
       "      <td>1.6704</td>\n",
       "      <td>1.685800</td>\n",
       "      <td>21.604200</td>\n",
       "      <td>3.1417</td>\n",
       "      <td>-6.5213</td>\n",
       "      <td>8.2675</td>\n",
       "      <td>14.722200</td>\n",
       "      <td>0.396500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>75153673</td>\n",
       "      <td>0</td>\n",
       "      <td>11.060400</td>\n",
       "      <td>-2.1518</td>\n",
       "      <td>8.952200</td>\n",
       "      <td>7.1957</td>\n",
       "      <td>12.584599</td>\n",
       "      <td>-1.8361</td>\n",
       "      <td>5.8428</td>\n",
       "      <td>14.925000</td>\n",
       "      <td>...</td>\n",
       "      <td>4.4666</td>\n",
       "      <td>4.743299</td>\n",
       "      <td>0.7178</td>\n",
       "      <td>1.421400</td>\n",
       "      <td>23.034700</td>\n",
       "      <td>-1.2706</td>\n",
       "      <td>-2.9275</td>\n",
       "      <td>10.2922</td>\n",
       "      <td>17.969700</td>\n",
       "      <td>-8.999599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>75153674</td>\n",
       "      <td>0</td>\n",
       "      <td>9.836900</td>\n",
       "      <td>-1.4834</td>\n",
       "      <td>12.874599</td>\n",
       "      <td>6.6375</td>\n",
       "      <td>12.277200</td>\n",
       "      <td>2.4486</td>\n",
       "      <td>5.9405</td>\n",
       "      <td>19.251400</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.4905</td>\n",
       "      <td>9.521400</td>\n",
       "      <td>-0.1508</td>\n",
       "      <td>9.194201</td>\n",
       "      <td>13.287600</td>\n",
       "      <td>-1.5121</td>\n",
       "      <td>3.9267</td>\n",
       "      <td>9.5031</td>\n",
       "      <td>17.997400</td>\n",
       "      <td>-8.810400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 202 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    ID_code  target      var_0   var_1      var_2   var_3      var_4   var_5  \\\n",
       "0  75153670       0   8.925500 -6.7863  11.908100  5.0930  11.460700 -9.2834   \n",
       "1  75153671       0  11.500600 -4.1473  13.858801  5.3890  12.362201  7.0433   \n",
       "2  75153672       0   8.609301 -2.7457  12.080500  7.8928  10.582500 -9.0837   \n",
       "3  75153673       0  11.060400 -2.1518   8.952200  7.1957  12.584599 -1.8361   \n",
       "4  75153674       0   9.836900 -1.4834  12.874599  6.6375  12.277200  2.4486   \n",
       "\n",
       "    var_6      var_7  ...  var_190   var_191  var_192    var_193    var_194  \\\n",
       "0  5.1187  18.626602  ...   4.4354  3.964200   3.1364   1.691000  18.522701   \n",
       "1  5.6208  16.533800  ...   7.6421  7.721400   2.5837  10.951600  15.430499   \n",
       "2  6.9427  14.615500  ...   2.9057  9.790500   1.6704   1.685800  21.604200   \n",
       "3  5.8428  14.925000  ...   4.4666  4.743299   0.7178   1.421400  23.034700   \n",
       "4  5.9405  19.251400  ...  -1.4905  9.521400  -0.1508   9.194201  13.287600   \n",
       "\n",
       "   var_195  var_196  var_197    var_198   var_199  \n",
       "0  -2.3978   7.8784   8.5635  12.780300 -1.091400  \n",
       "1   2.0339   8.1267   8.7889  18.355999  1.951800  \n",
       "2   3.1417  -6.5213   8.2675  14.722200  0.396500  \n",
       "3  -1.2706  -2.9275  10.2922  17.969700 -8.999599  \n",
       "4  -1.5121   3.9267   9.5031  17.997400 -8.810400  \n",
       "\n",
       "[5 rows x 202 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_gd.head().to_pandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### create new features & normalize\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [01:19<00:00,  1.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 56.4 s, sys: 23.2 s, total: 1min 19s\n",
      "Wall time: 1min 19s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for i in tqdm(range(200)):\n",
    "    col = 'var_%d'%i\n",
    "    new_col = 'new_%s'%col\n",
    "    count_col = 'count_%s'%col\n",
    "    \n",
    "    df = train_gd.groupby(col).agg({col:'count'})\n",
    "    df = df.reset_index()\n",
    "    train_gd = train_gd.merge(df,on=col,how='left')\n",
    "    test_gd = test_gd.merge(df,on=col,how='left')\n",
    "    \n",
    "    # feature values with count==1 have a lot of noise\n",
    "    # we can replace these values with mean value of the column\n",
    "    train_gd[new_col] = train_gd[col] * (train_gd[count_col]>1)\n",
    "    mean = train_gd[new_col].mean()\n",
    "    std = train_gd[new_col].std()\n",
    "    train_gd['mean'] = mean\n",
    "    train_gd[new_col] = train_gd[new_col] + train_gd['mean']*(train_gd[count_col]==1)\n",
    "    train_gd[new_col] = (train_gd[new_col]-mean)/std\n",
    "    train_gd[col] = (train_gd[col]-mean)/std\n",
    "    \n",
    "    test_gd[new_col] = test_gd[col] * (test_gd[count_col]>1)\n",
    "    test_gd['mean'] = mean\n",
    "    test_gd[new_col] = test_gd[new_col] + test_gd['mean']*(test_gd[count_col]==1)\n",
    "    test_gd[new_col] = (test_gd[new_col]-mean)/std\n",
    "    test_gd[col] = (test_gd[col]-mean)/std\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>var_0</th>\n",
       "      <th>new_var_0</th>\n",
       "      <th>var_1</th>\n",
       "      <th>new_var_1</th>\n",
       "      <th>var_2</th>\n",
       "      <th>new_var_2</th>\n",
       "      <th>var_3</th>\n",
       "      <th>new_var_3</th>\n",
       "      <th>var_4</th>\n",
       "      <th>new_var_4</th>\n",
       "      <th>...</th>\n",
       "      <th>var_195</th>\n",
       "      <th>new_var_195</th>\n",
       "      <th>var_196</th>\n",
       "      <th>new_var_196</th>\n",
       "      <th>var_197</th>\n",
       "      <th>new_var_197</th>\n",
       "      <th>var_198</th>\n",
       "      <th>new_var_198</th>\n",
       "      <th>var_199</th>\n",
       "      <th>new_var_199</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.216244</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.920815</td>\n",
       "      <td>0.920815</td>\n",
       "      <td>0.630195</td>\n",
       "      <td>0.630195</td>\n",
       "      <td>1.346586</td>\n",
       "      <td>1.346586</td>\n",
       "      <td>1.246226</td>\n",
       "      <td>1.246226</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.697538</td>\n",
       "      <td>-0.697538</td>\n",
       "      <td>1.280454</td>\n",
       "      <td>1.280454</td>\n",
       "      <td>0.679912</td>\n",
       "      <td>0.679911</td>\n",
       "      <td>0.245115</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.263861</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.030498</td>\n",
       "      <td>1.030498</td>\n",
       "      <td>-1.627658</td>\n",
       "      <td>-1.627658</td>\n",
       "      <td>0.311086</td>\n",
       "      <td>0.311086</td>\n",
       "      <td>0.516251</td>\n",
       "      <td>0.516251</td>\n",
       "      <td>1.112466</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.434332</td>\n",
       "      <td>-1.434332</td>\n",
       "      <td>1.037152</td>\n",
       "      <td>1.037152</td>\n",
       "      <td>0.824626</td>\n",
       "      <td>0.824626</td>\n",
       "      <td>0.597224</td>\n",
       "      <td>0.597224</td>\n",
       "      <td>0.842878</td>\n",
       "      <td>0.842878</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.241152</td>\n",
       "      <td>0.241152</td>\n",
       "      <td>-0.583870</td>\n",
       "      <td>-0.583870</td>\n",
       "      <td>0.417921</td>\n",
       "      <td>0.417921</td>\n",
       "      <td>-0.281280</td>\n",
       "      <td>-0.281280</td>\n",
       "      <td>-0.507862</td>\n",
       "      <td>-0.507862</td>\n",
       "      <td>...</td>\n",
       "      <td>1.771620</td>\n",
       "      <td>1.771620</td>\n",
       "      <td>0.076485</td>\n",
       "      <td>0.076485</td>\n",
       "      <td>-0.344700</td>\n",
       "      <td>-0.344700</td>\n",
       "      <td>0.852539</td>\n",
       "      <td>0.852539</td>\n",
       "      <td>-3.854148</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.427054</td>\n",
       "      <td>-0.427054</td>\n",
       "      <td>0.646435</td>\n",
       "      <td>0.646435</td>\n",
       "      <td>-0.062363</td>\n",
       "      <td>-0.062363</td>\n",
       "      <td>-0.286137</td>\n",
       "      <td>-0.286137</td>\n",
       "      <td>0.835080</td>\n",
       "      <td>0.835080</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.028680</td>\n",
       "      <td>-0.028680</td>\n",
       "      <td>-0.758242</td>\n",
       "      <td>-0.758242</td>\n",
       "      <td>1.004489</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.478298</td>\n",
       "      <td>0.478297</td>\n",
       "      <td>-1.517619</td>\n",
       "      <td>-1.517619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.320872</td>\n",
       "      <td>1.320872</td>\n",
       "      <td>-1.712520</td>\n",
       "      <td>-1.712520</td>\n",
       "      <td>-0.481917</td>\n",
       "      <td>-0.481917</td>\n",
       "      <td>-0.147458</td>\n",
       "      <td>-0.147458</td>\n",
       "      <td>-0.806044</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.895614</td>\n",
       "      <td>0.895614</td>\n",
       "      <td>1.719011</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.057048</td>\n",
       "      <td>0.057048</td>\n",
       "      <td>0.502301</td>\n",
       "      <td>0.502301</td>\n",
       "      <td>0.837644</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 400 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      var_0  new_var_0     var_1  new_var_1     var_2  new_var_2     var_3  \\\n",
       "0  0.216244   0.000000  0.920815   0.920815  0.630195   0.630195  1.346586   \n",
       "1  1.030498   1.030498 -1.627658  -1.627658  0.311086   0.311086  0.516251   \n",
       "2  0.241152   0.241152 -0.583870  -0.583870  0.417921   0.417921 -0.281280   \n",
       "3 -0.427054  -0.427054  0.646435   0.646435 -0.062363  -0.062363 -0.286137   \n",
       "4  1.320872   1.320872 -1.712520  -1.712520 -0.481917  -0.481917 -0.147458   \n",
       "\n",
       "   new_var_3     var_4  new_var_4  ...   var_195  new_var_195   var_196  \\\n",
       "0   1.346586  1.246226   1.246226  ... -0.697538    -0.697538  1.280454   \n",
       "1   0.516251  1.112466   0.000000  ... -1.434332    -1.434332  1.037152   \n",
       "2  -0.281280 -0.507862  -0.507862  ...  1.771620     1.771620  0.076485   \n",
       "3  -0.286137  0.835080   0.835080  ... -0.028680    -0.028680 -0.758242   \n",
       "4  -0.147458 -0.806044   0.000000  ...  0.895614     0.895614  1.719011   \n",
       "\n",
       "   new_var_196   var_197  new_var_197   var_198  new_var_198   var_199  \\\n",
       "0     1.280454  0.679912     0.679911  0.245115     0.000000  1.263861   \n",
       "1     1.037152  0.824626     0.824626  0.597224     0.597224  0.842878   \n",
       "2     0.076485 -0.344700    -0.344700  0.852539     0.852539 -3.854148   \n",
       "3    -0.758242  1.004489     0.000000  0.478298     0.478297 -1.517619   \n",
       "4     0.000000  0.057048     0.057048  0.502301     0.502301  0.837644   \n",
       "\n",
       "   new_var_199  \n",
       "0     0.000000  \n",
       "1     0.842878  \n",
       "2     0.000000  \n",
       "3    -1.517619  \n",
       "4     0.000000  \n",
       "\n",
       "[5 rows x 400 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feas = []\n",
    "for i in range(200):\n",
    "    feas.append('var_%d'%i)\n",
    "    feas.append('new_var_%d'%i)\n",
    "X = train_gd[feas].to_pandas()\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(200000, 400) (200000,) (200000, 400)\n"
     ]
    }
   ],
   "source": [
    "X = X.values\n",
    "y = train_gd['target'].to_pandas().values\n",
    "Xt = test_gd[feas].to_pandas().values\n",
    "print(X.shape,y.shape,Xt.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(200000, 400) (200000,) (200000, 400)\n",
      "(200000, 200, 2) (200000,) (200000, 200, 2)\n"
     ]
    }
   ],
   "source": [
    "print(X.shape,y.shape,Xt.shape)\n",
    "B = X.shape[0]\n",
    "X = np.reshape(X,[B,200,2])\n",
    "Xt = np.reshape(Xt,[B,200,2])\n",
    "print(X.shape,y.shape,Xt.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_valid, y_train, y_valid = train_test_split(\n",
    "    X, y, test_size=0.2, random_state=42, stratify=y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define TF model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class groupNN:\n",
    "    def __init__(self,**params):\n",
    "        self.params = params\n",
    "        \n",
    "    def fit(self,X,y):\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        tf.reset_default_graph()\n",
    "        # build a tf computing graph\n",
    "        logit = self._build()\n",
    "        label = tf.placeholder(dtype=tf.int32,shape=[None]) # B,classes\n",
    "        losst = self.get_loss(logit,label)\n",
    "        opt_op = self.get_opt(losst)\n",
    "        \n",
    "        with tf.Session() as sess:\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            sess.run(tf.local_variables_initializer())\n",
    "            #self.load(sess)\n",
    "            i = 1\n",
    "            ave_loss = []\n",
    "            for c,(Xb,yb,end_epoch) in enumerate(self._batch_gen(shuffle=True)):\n",
    "                loss,_ = sess.run([losst,opt_op],feed_dict={self.inputs:Xb, label:yb})\n",
    "                ave_loss.append(loss)\n",
    "                if end_epoch:\n",
    "                    print(\"Epoch %d train loss %.4f\"%(i,np.mean(ave_loss)))\n",
    "                    i += 1\n",
    "                    ave_loss = []\n",
    "            self.save(sess)\n",
    "    \n",
    "    def _build(self):\n",
    "        # build the computing graph      \n",
    "        netname = 'groupNN'\n",
    "        self.inputs = tf.placeholder(dtype=tf.float32,shape=[None,200,2])\n",
    "        B = tf.shape(self.inputs)[0]\n",
    "        H = self.params.get('hidden_units', 16)\n",
    "        with tf.variable_scope(netname):\n",
    "            net = self.inputs\n",
    "            net = tf.contrib.layers.fully_connected(self.inputs,H)\n",
    "            net = tf.reshape(net,[B,200*H])\n",
    "            net = tf.contrib.layers.fully_connected(net,1,activation_fn=None)\n",
    "            return tf.squeeze(net)\n",
    "        \n",
    "    def predict(self,X):\n",
    "        print('predict')\n",
    "        self.X = X \n",
    "        self.y = None\n",
    "        tf.reset_default_graph()\n",
    "        self.params['epochs'] = 1\n",
    "        # build a tf computing graph\n",
    "        logit = self._build()\n",
    "        logit = tf.nn.sigmoid(logit)\n",
    "        preds = []\n",
    "        #print('here')\n",
    "        with tf.Session() as sess:\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            sess.run(tf.local_variables_initializer())\n",
    "            self.load(sess)\n",
    "            for c,(Xb,_,end_epoch) in enumerate(self._batch_gen(shuffle=False)):\n",
    "                pred = sess.run(logit,feed_dict={self.inputs:Xb})\n",
    "                preds.append(pred)\n",
    "        preds = np.concatenate(preds)\n",
    "        return preds\n",
    "    \n",
    "    def get_opt(self,loss):\n",
    "        learning_rate = self.params.get('learning_rate', 0.001)\n",
    "        opt = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "        return opt.minimize(loss)\n",
    "        \n",
    "    def get_loss(self, logit, label):\n",
    "        # build the loss tensor\n",
    "        label = tf.cast(label,tf.float32)\n",
    "        loss = tf.nn.sigmoid_cross_entropy_with_logits(logits=logit,labels=label)\n",
    "        return tf.reduce_mean(loss)\n",
    "    \n",
    "    def save(self, sess):\n",
    "        varss = tf.trainable_variables()\n",
    "        weights = {} # var.name => var.value: a numpy array\n",
    "        for var in varss:\n",
    "            val = sess.run(var)\n",
    "            weights[var.name] = val\n",
    "        pickle.dump(weights,open('weight.p','wb'))\n",
    "\n",
    "    def load(self, sess, path = 'weight.p'):\n",
    "        weights = pickle.load(open(path,'rb'))\n",
    "        varss = tf.trainable_variables()\n",
    "        for var in varss:\n",
    "            value = weights[var.name]\n",
    "            assign_op = var.assign(value)\n",
    "            sess.run(assign_op)\n",
    "    \n",
    "    def _batch_gen(self, shuffle=True):\n",
    "        X,y = self.X, self.y\n",
    "        B = self.params.get('batch_size', 1024)\n",
    "        epochs = self.params.get('epochs', 10)\n",
    "        ids = [i for i in range(len(X))]\n",
    "        batches = len(X)//B + 1\n",
    "        #print(epochs,batches)\n",
    "        for epoch in range(epochs):\n",
    "            if shuffle:\n",
    "                random.shuffle(ids)\n",
    "            for i in range(batches):                \n",
    "                idx = ids[i*B:(i+1)*B]\n",
    "                if y is not None:\n",
    "                    yield X[idx],y[idx],i==batches-1\n",
    "                else:\n",
    "                    yield X[idx],None,i==batches-1\n",
    "            if (i+1)*B < len(X):\n",
    "                idx = ids[(i+1)*B:len(X)]\n",
    "                yield X[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn = groupNN(hidden_units=16,learning_rate=0.01,epochs=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:From /home/jiwei/anaconda3/envs/cudf0.6/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:From /home/jiwei/anaconda3/envs/cudf0.6/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "Epoch 1 train loss 0.2783\n",
      "Epoch 2 train loss 0.2145\n",
      "Epoch 3 train loss 0.2076\n",
      "Epoch 4 train loss 0.2038\n",
      "Epoch 5 train loss 0.1985\n",
      "Epoch 6 train loss 0.1955\n",
      "Epoch 7 train loss 0.1941\n",
      "Epoch 8 train loss 0.1925\n",
      "Epoch 9 train loss 0.1920\n",
      "Epoch 10 train loss 0.1918\n",
      "Epoch 11 train loss 0.1932\n",
      "Epoch 12 train loss 0.1893\n",
      "Epoch 13 train loss 0.1882\n",
      "Epoch 14 train loss 0.1885\n",
      "Epoch 15 train loss 0.1886\n",
      "Epoch 16 train loss 0.1896\n",
      "Epoch 17 train loss 0.1890\n",
      "Epoch 18 train loss 0.1882\n",
      "Epoch 19 train loss 0.1866\n",
      "Epoch 20 train loss 0.1886\n"
     ]
    }
   ],
   "source": [
    "nn.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "predict\n",
      "Validation AUC 0.9072589318554531\n"
     ]
    }
   ],
   "source": [
    "yp = nn.predict(X_valid)\n",
    "print('Validation AUC',roc_auc_score(y_valid,yp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "predict\n"
     ]
    }
   ],
   "source": [
    "yp = nn.predict(Xt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({'ID_code':test_gd['ID_code'].to_pandas().values,\n",
    "                   'target':yp})\n",
    "submission.to_csv('submission.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}