notebooks/DiCE_with_Titanic_train.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DiCE test: How could Mr survive from the sinking of the Titanic?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DiCE_getting_started.ipynb     DiCE_with_advanced_options.ipynb  train.csv\r\n",
      "DiCE_with_Titanic_train.ipynb  DiCE_with_private_data.ipynb\r\n"
     ]
    }
   ],
   "source": [
    "# import libraries\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pyplot import rcParams\n",
    "from sklearn.model_selection import GridSearchCV, cross_val_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\n",
    "%ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import DiCE\n",
    "import dice_ml\n",
    "from dice_ml.utils import helpers # helper functions\n",
    "\n",
    "# supress deprecation warnings from TF\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "DiCE requires two inputs: a training dataset and a pre-trained ML model. Trained-model wil be generated with tensorflow/keras in this sample. It can also work without access to the full dataset (see this notebook for advanced examples)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use the \"titanic\" dataset from Kaggle (https://www.kaggle.com/c/titanic/overview). train is given as a csv file with survived column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('train.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This dataset has 10 features in first place. The outcome is survived which is binarized to 0 (non-suvived) or 1 (survived)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>survived</th>\n",
       "      <th>pclass</th>\n",
       "      <th>name</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>ticket</th>\n",
       "      <th>fare</th>\n",
       "      <th>cabin</th>\n",
       "      <th>embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   survived  pclass                                               name  \\\n",
       "0         0       3                            Braund, Mr. Owen Harris   \n",
       "1         1       1  Cumings, Mrs. John Bradley (Florence Briggs Th...   \n",
       "2         1       3                             Heikkinen, Miss. Laina   \n",
       "3         1       1       Futrelle, Mrs. Jacques Heath (Lily May Peel)   \n",
       "4         0       3                           Allen, Mr. William Henry   \n",
       "\n",
       "      sex   age  sibsp  parch            ticket     fare cabin embarked  \n",
       "0    male  22.0      1      0         A/5 21171   7.2500   NaN        S  \n",
       "1  female  38.0      1      0          PC 17599  71.2833   C85        C  \n",
       "2  female  26.0      0      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3  female  35.0      1      0            113803  53.1000  C123        S  \n",
       "4    male  35.0      0      0            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    549\n",
      "1    342\n",
      "Name: survived, dtype: int64\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 891 entries, 0 to 890\n",
      "Data columns (total 11 columns):\n",
      "survived    891 non-null int64\n",
      "pclass      891 non-null int64\n",
      "name        891 non-null object\n",
      "sex         891 non-null object\n",
      "age         714 non-null float64\n",
      "sibsp       891 non-null int64\n",
      "parch       891 non-null int64\n",
      "ticket      891 non-null object\n",
      "fare        891 non-null float64\n",
      "cabin       204 non-null object\n",
      "embarked    889 non-null object\n",
      "dtypes: float64(2), int64(4), object(5)\n",
      "memory usage: 76.6+ KB\n"
     ]
    }
   ],
   "source": [
    "# show ratio of survived and non-survived\n",
    "print(df['survived'].value_counts())\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are some missing values and categorical values so that we need to do feature engineering tasks. I implemented the code from (https://medium.com/@simonli_18826/basic-implementation-of-tensorflow-classification-of-titanic-survival-60a9fbcc7a35) as reference."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Mr        525\n",
       "Miss      185\n",
       "Mrs       128\n",
       "Master     40\n",
       "Dr          7\n",
       "Rev         6\n",
       "Name: title, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# feature engineering for name column\n",
    "df['title'] = df['name'].str.extract('([A-Za-z]+)\\.', expand=True)\n",
    "\n",
    "# replacing some titles with more common ones\n",
    "mapping = {'Mlle': 'Miss', 'Major': 'Mr', 'Col': 'Mr', 'Sir': 'Mr',\n",
    "           'Don': 'Mr', 'Mme': 'Mrs', 'Jonkheer': 'Mr', 'Lady': 'Mrs',\n",
    "           'Capt': 'Mr', 'Countess': 'Mrs', 'Ms': 'Miss', 'Dona': 'Mrs'}\n",
    "df.replace({'title': mapping}, inplace=True)\n",
    "df['title'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dict for median age for titles: {'Dr': 46.5, 'Master': 3.5, 'Miss': 21.0, 'Mr': 30.0, 'Mrs': 35.0, 'Rev': 46.5}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEcCAYAAAAoSqjDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAbiklEQVR4nO3deZRcdZ338feXJBD2QBKEpMMkDBAWkYghB8WFiQgYMXA0K/vioLJMGIUn4oYbiory4DqiQUHRJILzBEYGBQTHARfSCgThMARQaBIlBmSVJeH7/HFvX3uaJqkkXX27Ou/XOX1S99at6u+li/rUb7m/isxEkiSATeouQJLUfxgKkqSKoSBJqhgKkqSKoSBJqhgKkqTK4LoL2BAjRozIsWPH1l2GJLWU9vb2v2TmyJ7ua+lQGDt2LIsXL667DElqKRHxx5e7z+4jSVLFUJAkVQwFSVKlpccUJKkZXnjhBTo6Onj22WfrLmWDDB06lLa2NoYMGdLwYwwFSeqmo6ODrbfemrFjxxIRdZezXjKTlStX0tHRwbhx4xp+nN1HktTNs88+y/Dhw1s2EAAiguHDh69za8dQkKQetHIgdFqfczAUJKkmV111Feeff36vPNdWW23VK8/jmIIkNdGqVasYPLjnt9qpU6cyderUPq5ozQZkKLzm7Mua8rztnz+uKc8rqf97+umnmTFjBh0dHaxevZqPfOQjzJ07l8WLFzNixAgWL17MWWedxU033cTHPvYxli1bxh/+8AdGjBjBfffdxyWXXMLee+8NwEEHHcQXvvAFlixZwuLFiznvvPPYd999uf/++9lkk0145plnGD9+PPfffz8PPvggp512GitWrGCLLbbgm9/8JnvssQcPPPAARx11FKtWreKwww7rtfO0+0iSGnDttdcyatQobr/9du688861vhG3t7ezaNEivv/97zNr1iwWLlwIwPLly1m2bBmvec1rqmO33XZb9t13X37+858DcPXVV3PooYcyZMgQTjnlFL785S/T3t7OBRdcwKmnngrAnDlzeO9738utt97Kjjvu2GvnaShIUgP22Wcfrr/+eubOncsvfvELtt122zUeP3XqVDbffHMAZsyYwQ9/+EMAFi5cyPTp019y/MyZM1mwYAEA8+fPZ+bMmTz11FPccsstTJ8+nQkTJvDud7+b5cuXA3DzzTcze/ZsAI499theO88B2X0kSb1t9913p729nWuuuYZzzjmHQw45hMGDB/Piiy8CvGTq55ZbblndHj16NMOHD+eOO+5gwYIFfOMb33jJ80+dOpVzzjmHRx99lPb2diZPnszTTz/NsGHDuO2223qsqRkzpGwpSFIDli1bxhZbbMExxxzDWWedxW9/+1vGjh1Le3s7AFdeeeUaHz9r1iw+97nP8fjjj7PPPvu85P6tttqKSZMmMWfOHA4//HAGDRrENttsw7hx46pWRmZy++23A3DggQcyf/58AC6//PJeO09DQZIasGTJEiZNmsSECRM477zz+PCHP8y5557LnDlzeMMb3sCgQYPW+Php06Yxf/58ZsyY8bLHzJw5k+9973vMnDmz2nf55Zczb9489t13X/bee28WLVoEwEUXXcRXv/pV9t9/fx5//PHeOUkgMrPXnqyvTZw4MXv6PgVnH0naEHfffTd77rln3WX0ip7OJSLaM3NiT8fbUpAkVQwFSVLFUJAkVQwFSVLFUJAkVQwFSVLFUJCkFnHttdcyfvx4dt11115bcrs7l7mQpHXU29dCNXIN1OrVqznttNO47rrraGtrY//992fq1KnstddevVpL01sKETEoIn4XEf9Rbo+LiF9HxL0RsSAiNi33b1ZuLy3vH9vs2iSpVfzmN79h1113ZZdddmHTTTdl1qxZ1dXNvakvuo/mAHd32f4scGFm7gY8Bpxc7j8ZeCwzdwUuLI+TJAEPP/wwY8aMqbbb2tp4+OGHe/33NDUUIqINeBvwrXI7gMnAFeUhlwJHlrePKLcp739zDIQvSZWkXtDTkkStuErq/wX+D/BiuT0c+Gtmriq3O4DR5e3RwEMA5f2Pl8dL0kavra2Nhx56qNru6Ohg1KhRvf57mhYKEXE48Ehmtnfd3cOh2cB9XZ/3lIhYHBGLV6xY0QuVSlL/t//++3PvvffywAMP8PzzzzN//vymfL9zM2cfHQhMjYgpwFBgG4qWw7CIGFy2BtqAZeXxHcAYoCMiBgPbAo92f9LMvBi4GIpVUptYvyT1G4MHD+YrX/kKhx56KKtXr+akk06qvvO5V39Prz9jKTPPAc4BiIiDgLMy8+iI+CEwDZgPHA90Dp9fVW7/srz/Z9nK63pLGrDqWkZ/ypQpTJkypam/o46L1+YC74uIpRRjBvPK/fOA4eX+9wEfqKE2Sdqo9cnFa5l5E3BTeft+YFIPxzwLvPTbrCVJfcZlLiRJFUNBklQxFCRJFUNBklQxFCSpBZx00knssMMOvPKVr2zq73HpbElaRw9+Yp9efb6dP7pkrceccMIJnH766Rx3XHOvkbClIEkt4I1vfCPbb79903+PoSBJqhgKkqSKoSBJqhgKkqSKoSBJLWD27Nm89rWv5Z577qGtrY158+at/UHrwSmpkrSOGplC2tt+8IMf9MnvsaUgSaoYCpKkiqEgSaoYCpLUg4HwbcDrcw6GgiR1M3ToUFauXNnSwZCZrFy5kqFDh67T45x9JEndtLW10dHRwYoVK+ouZYMMHTqUtra2dXqMoSBJ3QwZMoRx48bVXUYt7D6SJFUMBUlSxVCQJFUMBUlSxVCQJFUMBUlSxVCQJFUMBUlSxVCQJFUMBUlSxVCQJFUMBUlSxVCQJFUMBUlSxVCQJFUMBUlSxVCQJFUMBUlSpWmhEBFDI+I3EXF7RPw+Ij5e7h8XEb+OiHsjYkFEbFru36zcXlreP7ZZtUmSetbMlsJzwOTM3BeYABwWEQcAnwUuzMzdgMeAk8vjTwYey8xdgQvL4yRJfahpoZCFp8rNIeVPApOBK8r9lwJHlrePKLcp739zRESz6pMkvVRTxxQiYlBE3AY8AlwH3Af8NTNXlYd0AKPL26OBhwDK+x8HhjezPknS/9bUUMjM1Zk5AWgDJgF79nRY+W9PrYLsviMiTomIxRGxeMWKFb1XrCSpb2YfZeZfgZuAA4BhETG4vKsNWFbe7gDGAJT3bws82sNzXZyZEzNz4siRI5tduiRtVJo5+2hkRAwrb28OHAzcDdwITCsPOx5YVN6+qtymvP9nmfmSloIkqXkGr/2Q9bYTcGlEDKIIn4WZ+R8RcRcwPyI+BfwOmFcePw/4bkQspWghzGpibZKkHjQtFDLzDuDVPey/n2J8ofv+Z4HpzapHkrR2XtEsSaoYCpKkiqEgSaoYCpKkiqEgSaoYCpKkiqEgSaoYCpKkiqEgSaoYCpKkiqEgSaoYCpKkiqEgSaoYCpKkiqEgSao0FAoRcUMj+yRJrW2NX7ITEUOBLYAREbEdEOVd2wCjmlybJKmPre2b194NnEkRAO38PRSeAL7axLokSTVYYyhk5kXARRFxRmZ+uY9qkiTVpKHvaM7ML0fE64CxXR+TmZc1qS5JUg0aCoWI+C7wj8BtwOpydwKGgiQNIA2FAjAR2Cszs5nFSJLq1eh1CncCOzazEElS/RptKYwA7oqI3wDPde7MzKlNqUqSVItGQ+FjzSxCktQ/NDr76OfNLkSSVL9GZx89STHbCGBTYAjwdGZu06zCJEl9r9GWwtZdtyPiSGBSUyqSJNVmvVZJzcz/B0zu5VokSTVrtPvoHV02N6G4bsFrFiRpgGl09tHbu9xeBfwBOKLXq5Ek1arRMYUTm12IJKl+jX7JTltE/HtEPBIRf46IKyOirdnFSZL6VqMDzd8GrqL4XoXRwNXlPknSANJoKIzMzG9n5qry5zvAyCbWJUmqQaOh8JeIOCYiBpU/xwArm1mYJKnvNRoKJwEzgD8By4FpgIPPkjTANDol9ZPA8Zn5GEBEbA9cQBEWkqQBotGWwqs6AwEgMx8FXr2mB0TEmIi4MSLujojfR8Sccv/2EXFdRNxb/rtduT8i4ksRsTQi7oiI/db3pCRJ66fRUNik880bqpbC2loZq4D3Z+aewAHAaRGxF/AB4IbM3A24odwGeCuwW/lzCvD1hs9CktQrGu0++gJwS0RcQbG8xQzgvDU9IDOXU4w/kJlPRsTdFNNZjwAOKg+7FLgJmFvuv6z8ys9fRcSwiNipfB5JUh9o9IrmyyJiMcUieAG8IzPvavSXRMRYiu6mXwOv6Hyjz8zlEbFDedho4KEuD+so9xkKktRHGm0pUIZAw0HQKSK2Aq4EzszMJyLiZQ/t6df28HynUHQvsfPOO69rOZKkNVivpbMbFRFDKALh8sz8Ubn7zxGxU3n/TsAj5f4OYEyXh7cBy7o/Z2ZenJkTM3PiyJFePydJvalpoRBFk2AecHdmfrHLXVcBx5e3jwcWddl/XDkL6QDgcccTJKlvNdx9tB4OBI4FlkTEbeW+DwLnAwsj4mTgQWB6ed81wBRgKfAMXhwnSX2uaaGQmf9Nz+MEAG/u4fgETmtWPZKktWvqmIIkqbUYCpKkiqEgSaoYCpKkiqEgSaoYCpKkiqEgSaoYCpKkiqEgSaoYCpKkiqEgSaoYCpKkiqEgSaoYCpKkiqEgSaoYCpKkiqEgSaoYCpKkiqEgSaoYCpKkiqEgSaoYCpKkiqEgSaoYCpKkiqEgSaoYCpKkiqEgSaoYCpKkyuC6C5D6yoOf2Kcpz7vzR5c05XmlOthSkCRVDAVJUsVQkCRVDAVJUsVQkCRVDAVJUsVQkCRVDAVJUsVQkCRVmhYKEXFJRDwSEXd22bd9RFwXEfeW/25X7o+I+FJELI2IOyJiv2bVJUl6ec1sKXwHOKzbvg8AN2TmbsAN5TbAW4Hdyp9TgK83sS5J0stoWihk5n8Bj3bbfQRwaXn7UuDILvsvy8KvgGERsVOzapMk9ayvF8R7RWYuB8jM5RGxQ7l/NPBQl+M6yn3L+7i+jZoLxknqLwPN0cO+7PHAiFMiYnFELF6xYkWTy5KkjUtfh8KfO7uFyn8fKfd3AGO6HNcGLOvpCTLz4sycmJkTR44c2dRiJWlj09ehcBVwfHn7eGBRl/3HlbOQDgAe7+xmkiT1naaNKUTED4CDgBER0QGcC5wPLIyIk4EHgenl4dcAU4ClwDPAic2qS5L08poWCpk5+2XuenMPxyZwWrNqkSQ1pr8MNEuS+gFDQZJU6evrFFqa8/glDXS2FCRJFUNBklQxFCRJFUNBklQxFCRJFUNBklQxFCRJFUNBklQxFCRJFUNBklQxFCRJFUNBklQxFCRJFVdJVb/ymrMva9pz//vWTXtqacCwpSBJqhgKkqSKoSBJqhgKkqSKoSBJqhgKkqSKoSBJqhgKkqSKoSBJqhgKkqSKoSBJqhgKkqSKC+K1GBeMk9RMthQkSRVbCtIA8OAn9mnac+/80SVNe271P7YUJEkVQ0GSVLH7SOpDzZoo4CQB9RZbCpKkiqEgSaoYCpKkSr8KhYg4LCLuiYilEfGBuuuRpI1NvxlojohBwFeBtwAdwK0RcVVm3lVvZZIa1ayB9PbPH9eU59VL9ZtQACYBSzPzfoCImA8cARgKkvqF5s0e+3xTnnd9LjzsT91Ho4GHumx3lPskSX0kMrPuGgCIiOnAoZn5rnL7WGBSZp7R7bhTgFPKzfHAPX1Y5gjgL334+/qa59e6BvK5gefX2/4hM0f2dEd/6j7qAMZ02W4DlnU/KDMvBi7uq6K6iojFmTmxjt/dFzy/1jWQzw08v77Un7qPbgV2i4hxEbEpMAu4quaaJGmj0m9aCpm5KiJOB34CDAIuyczf11yWJG1U+k0oAGTmNcA1ddexBrV0W/Uhz691DeRzA8+vz/SbgWZJUv3605iCJKlmhsI6iIiouwZpXfm61bowFBoUEZFlX1tEjK+7nmbZGN5ANoZz7NTtdbtH3fXopfrb69FQaFCX/7FOAD4dEVvVW1Hv6/YGsmt/e7H2hm7nOKzuepqty7m+E/hmRGxec0m9IiJeFRGb1V3Hhur2ejw6IkbVXZOhsBYRsV2X268D3g6clplPRcSA+u/X5cV5JnAJxVWWA0qXc3w/cGFEDPjvLIuIdwDvAd6VmX8rF59sWRGxC3AGsE253bIfXrq8Ho8ETqi3msKAelPrbRFxCHBdRBxa7nobsBdwKEBmvtjKL8hOXc8hIo4GjgbekZkrImKniNimvup6X0S8BzgS+FBmPhkRm0dEv5qevSF6eE0+B7wJOBwgM1e30uu2h1ofAnYC3g9/f2NtVRExkSLkfpiZy+p+LRoKazYeeCVwVkS8GTgX+C6wT0RMhuIF2Ur/g3XXrfm6C/A4MA84OCLOAW4Ezo2IcTWWuUF6+PuMAb4I7BwRc4AfA6dGxDat/LeEl/w9946IUZn5Y4pAOCkiZkNrvW67juVFxPjMfAE4Fdg9Ig6ot7p1FxG7RcQBETE5IrYFHgbuA46KiL0yc1Wd9Q2YT0dN8gNgF4pPJmcAmwIXAmcBh0XEkMz8SSt/Uuk2VvI24Epgf4q1p74CvBc4HtiiphI3SLc3yYMy8ybgT8Bs4BXAZRTLqewFrGrlvyX8r7/n2cDBwBMRsZTidft+4FMRMTQzv93fz7Xb366Nov4xEbEAWAT8Htix+7H9WUS8Dfgk8Edga2BXisD+BPDPwHsj4muZeXdtRWamP11+gFcBrypvbwJ8FvgWcAhwNcWXAG0OfIbij7tF3TWv53lu1+X264AfASPL7RHAkPL2FIp1qXauu+YNPN/TgTuAkRTLqIwFhpX3vQ34Vef5t/oP8E/AT8rb36f4cNN5oerbgf+m6I+PumtdwzlEl9t7AFtSfFCZBPwC+CDwS+DeVnltAoeVr7M3ddl3LvAHYE9gh3L7O8DutdVZ93+o/vQDDAdeBB4EplF8Yh5M8Y1wh/L3RfqmlMEwou6a1/M8DwEWUyxVDnAecDdwYpdjNqcYW2gH9qm75vU4x1E9nG9n6I0HhlK0/E6k+MT5yrpr3oBz3azb9iTgI8Bc4D877wf2K//dsu6a1+HczqZYD20B8CkggFHAgRRLQzwAjK67zgbOY/vyveXwcntol/s+Xobb5hQfSs8GdqyrVruPusjMlRFxMHA9xR9nT+BfKfr8Rmbm98opfScAN2Vmq67v3nWsZBXFp5Ongb0jYnJm/iyLWSr/AxyRmR11FruuImI0cHJEfDEzn6II9puAN5SDetOA5RSh9xdgambeV1e9GyIitgROiIifUvxN/wG4DngjsBnwlsx8LiLOAKZExLTMfLq+il9e9y6giPgnivoPiYiFFF25m2TmMopl9W+OiB0y85GaSm5YZj4aEW8Hzo+IX5bvNZtl5nOZeW5EvAnYIzN/FxH3ZOZzddXq2kc9KAeVLwH2o3gDOYri+x5OpPgfLTLzifoq3DARMQL4EMVYyRuBb1C8aZ5F0c95Y2b+Z20FboCI2CYznyivIxlPMVZwBfA94BngcopPnvOBb2fmtbUV20vKfurLgBXAnpmZEXEuMAx4giL4/hk4KjPvrK/SNet8k+yyfRjwamAVMBk4sgy4CcCSzFxdU6nrLSLeSjFWNzEzHyvHJV+IiEXARzLzjppLdPZRTzLzBooB1puAyzPzTcCHM/P5zHyyFQOhvNjnVeXmo8DzFG+YX6OYyfF64AJgNfC6iGi5geVy6vCNEXFw2UJ4NfBWij7cd2bmsWUIHE7RCqxvMG8DdbtG5o/AUiApWrgAF1G0GF6kGFCf1c8D4RBgfkScW15oB0Vf+8EU4yBTu7R4Pknx4azllB+2TgcWR8R2ZSAcRzFg/qd6qyvYffQyMvOacsberRFxYGY+AK0zy6GriBgO3AZ0RMT7KN5EPkTxxhEUn57PAIZQzILYMjOfqancDbE7sDcwNyIGZea3IuJvwIyym+GyiJgGnAMcnZl/rLXa9VS+Bl8sbx9FMYB+MMW413cj4l8z84aIWAF8OospnP1W2SL4OEVrZweKmX2/pWjJ3kzRep0bEY8C76L427Xi6xMogiGK7475r4j4GnAscHJ/6QYzFNagDIYhwPVlX3S2WiDARjVW0nUK8XsiYtPMvLz8VP36iFhN0W10S9kv3ZI6X4PlG8t7KMZ9ngSuKP+O34yIH1EMOE8H/lxbsWsREdtTfIfKEZl5dTn19DyKgdYHIuKzwBsoZlRtBczOzLvqq7h3lMEwiGLW36uzH32hmGMKDYiIrcruiJY2EMdKOrvEMvOO8s3/MxSzyBZStH6+Xob7uyi6k+YOkL/lrhQtvBmZ+cdyXGFT4BaKAefpwEVZ53z3BpW1fw54bTkedA3FdTF3APcACzLzL63YSl+biNiiv7V6DIWNTERMobj24rVZrN80rrNrrNWU3WIrKIKts1vsdxTdYlcB21EE37zMXNQ5CF1XvRuih5k5IylmjXWO/exEMSB7ZWZ+JyIGZ81Xxq6LcgD2S8C1FC3Zb1BM43wXsAQ4s1X/dq3GUNgIlcHwBeDAzHy03NeSn8KiWG7keoo57C9QvKE8DNxedoudSHHR0En9dSrm2nS7sndfiokCHRRdKvsB12TmXVEsS9KWmae14t+z7OL8KbBTZv653LcJsH0Ld2m2HENhIxURR1B80mzZsZJOA7FbrCflzJujKQZfX0ExOPlced8xFFOKZ7dCl9HLKVsMFwCTO4NBfctQ2IgNlLESGFjdYp3KKYuPlbenAWdSXE3/UeAdFBdwvYVixs4ngc/152mnjer6gaVzlpX6jqGgAWOAdYsdAnwa+GBm/jQi9qO4CnsqRSBMpZi1sxlFN9LgzPxbXfX2toH0gaXVOCVVA8ZAmUJc6roUyWbldM1NKLrIPlNeyHUzxfpcO2Xmg3UW29sMhPrYUtCAMxA+ZXZbiuT1FNMyF0TExRQD6c9RzN8/ITNX1FepBhqXudCA06qBsIalSL4OHF0uEPdpiivP96PoWjIQ1KtsKUj9QIPXXBwN/Ftm/rhcxqPlFoRT/2dLQeoHMnMlxfpFbRRLkRxGsRbQMxRLkcynWBLhuIjYmmKhO6nX2VKQ+pEGrrmgXOdIagpDQepnBuI1F2odTkmV+pmBtGy7Wo+hIPVDA+yaC7UQu4+kfmwgXHOh1mIoSJIqTkmVJFUMBUlSxVCQJFUMBUlSxVCQ1lFEDIuIU8vboyLiivL2hPLCs87jToiIr9RVp7Q+DAVp3Q0DTgXIzGWZOa3cP4Him9GkluXFa9K6Ox/4x4i4DbgX2JNiraJPAJtHxOuBz3R9QESMBP4N2LncdWZm3tx3JUuNsaUgrbsPAPdl5gTgbIDMfJ7iu5MXZOaEzFzQ7TEXARdm5v7AO4Fv9WXBUqNsKUh942Bgr3JNI4BtImJrVzxVf2MoSH1jE4pVT/9WdyHSmth9JK27J4Gt12E/wE+B0zs3ImJCE+qSNpihIK2j8lvSbo6IO4HPd7nrRoouotsiYma3h/0LMDEi7oiIu4D39FG50jpxQTxJUsWWgiSpYihIkiqGgiSpYihIkiqGgiSpYihIkiqGgiSpYihIkir/HwIyN/RRbhKKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# impute missing age values using median of title groups\n",
    "title_ages = dict(df.groupby('title')['age'].median())\n",
    "print(\"Dict for median age for titles: {}\".format(title_ages))\n",
    "\n",
    "# create a column of the average ages\n",
    "df['age_med'] = df['title'].apply(lambda x: title_ages[x])\n",
    "\n",
    "# replace all missing ages with the value in this column\n",
    "df['age'].fillna(df['age_med'], inplace=True, )\n",
    "del df['age_med']\n",
    "\n",
    "# Visualize survival rates of titles\n",
    "sns.countplot(x='title', data=df, hue='survived')\n",
    "plt.xticks(rotation=45)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dict for median fare for pclasses: {1: 60.287499999999994, 2: 14.25, 3: 8.05}\n"
     ]
    }
   ],
   "source": [
    "# impute missing fare values using median of pclass groups\n",
    "class_fares = dict(df.groupby('pclass')['fare'].median())\n",
    "print(\"Dict for median fare for pclasses: {}\".format(class_fares))\n",
    "\n",
    "# create a column of the average fares\n",
    "df['fare_med'] = df['pclass'].apply(lambda x: class_fares[x])\n",
    "\n",
    "# replace all missing fares with the value in this column\n",
    "df['fare'].fillna(df['fare_med'], inplace=True, )\n",
    "del df['fare_med']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# feature engineering for embarked and family_zize, dropped cabin, name, and ticket\n",
    "df['embarked'].fillna(method='backfill', inplace=True)\n",
    "df['family_size'] = df['parch'] + df['sibsp']\n",
    "df.drop(['cabin', 'name', 'ticket'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>survived</th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>fare</th>\n",
       "      <th>embarked</th>\n",
       "      <th>title</th>\n",
       "      <th>family_size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>S</td>\n",
       "      <td>Mr</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C</td>\n",
       "      <td>Mrs</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>S</td>\n",
       "      <td>Miss</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>S</td>\n",
       "      <td>Mrs</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>S</td>\n",
       "      <td>Mr</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   survived  pclass     sex   age  sibsp  parch     fare embarked title  \\\n",
       "0         0       3    male  22.0      1      0   7.2500        S    Mr   \n",
       "1         1       1  female  38.0      1      0  71.2833        C   Mrs   \n",
       "2         1       3  female  26.0      0      0   7.9250        S  Miss   \n",
       "3         1       1  female  35.0      1      0  53.1000        S   Mrs   \n",
       "4         0       3    male  35.0      0      0   8.0500        S    Mr   \n",
       "\n",
       "   family_size  \n",
       "0            1  \n",
       "1            1  \n",
       "2            0  \n",
       "3            1  \n",
       "4            0  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training a custom ML model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below, we build an Artificial Neural Network based on Keras Tensorflow framework. The models differ slightly based on the version of TensorFlow you use. So the results you get in subsequent sections might be different from what is shown in this notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# seeding random numbers for reproducability\n",
    "from numpy.random import seed\n",
    "seed(1)\n",
    "tf.random.set_seed(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>fare</th>\n",
       "      <th>pclass_1</th>\n",
       "      <th>pclass_2</th>\n",
       "      <th>pclass_3</th>\n",
       "      <th>sex_female</th>\n",
       "      <th>sex_male</th>\n",
       "      <th>sibsp_0</th>\n",
       "      <th>sibsp_1</th>\n",
       "      <th>sibsp_2</th>\n",
       "      <th>...</th>\n",
       "      <th>title_Rev</th>\n",
       "      <th>family_size_0</th>\n",
       "      <th>family_size_1</th>\n",
       "      <th>family_size_2</th>\n",
       "      <th>family_size_3</th>\n",
       "      <th>family_size_4</th>\n",
       "      <th>family_size_5</th>\n",
       "      <th>family_size_6</th>\n",
       "      <th>family_size_7</th>\n",
       "      <th>family_size_10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>873</th>\n",
       "      <td>0.584016</td>\n",
       "      <td>0.017567</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>828</th>\n",
       "      <td>0.369721</td>\n",
       "      <td>0.015127</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>0.420144</td>\n",
       "      <td>0.050749</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>523</th>\n",
       "      <td>0.546199</td>\n",
       "      <td>0.113168</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>0.584016</td>\n",
       "      <td>0.028302</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 39 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          age      fare  pclass_1  pclass_2  pclass_3  sex_female  sex_male  \\\n",
       "873  0.584016  0.017567         0         0         1           0         1   \n",
       "828  0.369721  0.015127         0         0         1           0         1   \n",
       "99   0.420144  0.050749         0         1         0           0         1   \n",
       "523  0.546199  0.113168         1         0         0           1         0   \n",
       "132  0.584016  0.028302         0         0         1           1         0   \n",
       "\n",
       "     sibsp_0  sibsp_1  sibsp_2  ...  title_Rev  family_size_0  family_size_1  \\\n",
       "873        1        0        0  ...          0              1              0   \n",
       "828        1        0        0  ...          0              1              0   \n",
       "99         0        1        0  ...          0              0              1   \n",
       "523        1        0        0  ...          0              0              1   \n",
       "132        0        1        0  ...          0              0              1   \n",
       "\n",
       "     family_size_2  family_size_3  family_size_4  family_size_5  \\\n",
       "873              0              0              0              0   \n",
       "828              0              0              0              0   \n",
       "99               0              0              0              0   \n",
       "523              0              0              0              0   \n",
       "132              0              0              0              0   \n",
       "\n",
       "     family_size_6  family_size_7  family_size_10  \n",
       "873              0              0               0  \n",
       "828              0              0               0  \n",
       "99               0              0               0  \n",
       "523              0              0               0  \n",
       "132              0              0               0  \n",
       "\n",
       "[5 rows x 39 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess = tf.compat.v1.InteractiveSession()\n",
    "d = dice_ml.Data(dataframe=df, continuous_features=['age', 'fare'], outcome_name='survived')\n",
    "\n",
    "train, _ = d.split_data(d.normalize_data(d.one_hot_encoded_data))\n",
    "X_train = train.loc[:, train.columns != 'survived']\n",
    "y_train = train.loc[:, train.columns == 'survived']\n",
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fef2077ca90>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ann_model = keras.Sequential()\n",
    "ann_model.add(keras.layers.Dense(20, input_shape=(X_train.shape[1],), \n",
    "                                 kernel_regularizer=keras.regularizers.l1(0.001), \n",
    "                                 activation=tf.nn.relu))\n",
    "ann_model.add(keras.layers.Dense(1, activation=tf.nn.sigmoid))\n",
    "\n",
    "ann_model.compile(loss='binary_crossentropy', \n",
    "                  optimizer=tf.keras.optimizers.Adam(0.01), \n",
    "                  metrics=['accuracy'])\n",
    "ann_model.fit(X_train, y_train, validation_split=0.20, epochs=100, verbose=0, class_weight={0:1,1:2})\n",
    "# the training will take some time for 100 epochs.\n",
    "# you can wait or set verbose=1 to see the progress of training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# provide the trained ML model to DiCE's model object\n",
    "backend = 'TF'+tf.__version__[0]\n",
    "m = dice_ml.Model(model=ann_model, backend=backend)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate diverse counterfactuals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Based on the data object d and the model object m, we can now instantiate the DiCE class for generating explanations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initiate DiCE\n",
    "exp = dice_ml.Dice(d, m)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "DiCE provides local explanation for the model m and requires an query input whose outcome needs to be explained. Below we provide a sample input whose outcome is 0 (non-survival) as per the ML model object m. I took this sample from train.cav dataset row 1. His name was Braund, Mr. Owen Harris, 22 years-old man."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counterfactual sample: {'pclass': 3, 'sex': 'male', 'age': 22.0, 'sibsp': 1, 'parch': 0, 'fare': 7.25, 'embarked': 'S', 'title': 'Mr', 'family_size': 1}\n"
     ]
    }
   ],
   "source": [
    "# sample taken from raw data at the first row guy\n",
    "factual_sample = df.iloc[0, 1:].to_dict()\n",
    "print(\"Counterfactual sample: {}\".format(factual_sample))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Given the query input, we can now generate counterfactual explanations to show perturbed inputs from the original input where the ML model outputs class 1 (survival)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Diverse Counterfactuals found! total time taken: 00 min 25 sec\n"
     ]
    }
   ],
   "source": [
    "# generate counterfactuals\n",
    "dice_exp = exp.generate_counterfactuals(factual_sample, total_CFs=4, desired_class=\"opposite\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Query instance (original outcome : 0)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>fare</th>\n",
       "      <th>embarked</th>\n",
       "      <th>title</th>\n",
       "      <th>family_size</th>\n",
       "      <th>survived</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7.25</td>\n",
       "      <td>S</td>\n",
       "      <td>Mr</td>\n",
       "      <td>1</td>\n",
       "      <td>0.136179</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  pclass   sex   age sibsp parch  fare embarked title family_size  survived\n",
       "0      3  male  22.0     1     0  7.25        S    Mr           1  0.136179"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Diverse Counterfactual set (new outcome : 1)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>fare</th>\n",
       "      <th>embarked</th>\n",
       "      <th>title</th>\n",
       "      <th>family_size</th>\n",
       "      <th>survived</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>11.89</td>\n",
       "      <td>C</td>\n",
       "      <td>Mr</td>\n",
       "      <td>1</td>\n",
       "      <td>0.662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>S</td>\n",
       "      <td>Master</td>\n",
       "      <td>1</td>\n",
       "      <td>0.968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>35.7</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>25.43</td>\n",
       "      <td>S</td>\n",
       "      <td>Mr</td>\n",
       "      <td>1</td>\n",
       "      <td>0.871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>16.6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2.54</td>\n",
       "      <td>S</td>\n",
       "      <td>Mrs</td>\n",
       "      <td>1</td>\n",
       "      <td>0.996</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  pclass     sex   age sibsp parch   fare embarked   title family_size  \\\n",
       "0      1    male  22.0     8     0  11.89        C      Mr           1   \n",
       "1      3    male  22.0     1     0   0.00        S  Master           1   \n",
       "2      1  female  35.7     2     0  25.43        S      Mr           1   \n",
       "3      1  female  16.6     1     0   2.54        S     Mrs           1   \n",
       "\n",
       "   survived  \n",
       "0     0.662  \n",
       "1     0.968  \n",
       "2     0.871  \n",
       "3     0.996  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# visualize the resutls\n",
    "dice_exp.visualize_as_dataframe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In case, if you would like to visualize only the changes made to the query instance, you can set the parameter show_only_changes=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Query instance (original outcome : 0)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>fare</th>\n",
       "      <th>embarked</th>\n",
       "      <th>title</th>\n",
       "      <th>family_size</th>\n",
       "      <th>survived</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7.25</td>\n",
       "      <td>S</td>\n",
       "      <td>Mr</td>\n",
       "      <td>1</td>\n",
       "      <td>0.136179</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  pclass   sex   age sibsp parch  fare embarked title family_size  survived\n",
       "0      3  male  22.0     1     0  7.25        S    Mr           1  0.136179"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Diverse Counterfactual set (new outcome : 1)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>fare</th>\n",
       "      <th>embarked</th>\n",
       "      <th>title</th>\n",
       "      <th>family_size</th>\n",
       "      <th>survived</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>8</td>\n",
       "      <td>-</td>\n",
       "      <td>11.89</td>\n",
       "      <td>C</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>0.662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-</td>\n",
       "      <td>Master</td>\n",
       "      <td>-</td>\n",
       "      <td>0.968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>35.7</td>\n",
       "      <td>2</td>\n",
       "      <td>-</td>\n",
       "      <td>25.43</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>0.871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>16.6</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>2.54</td>\n",
       "      <td>-</td>\n",
       "      <td>Mrs</td>\n",
       "      <td>-</td>\n",
       "      <td>0.996</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  pclass     sex   age sibsp parch   fare embarked   title family_size  \\\n",
       "0      1       -     -     8     -  11.89        C       -           -   \n",
       "1      -       -     -     -     -    0.0        -  Master           -   \n",
       "2      1  female  35.7     2     -  25.43        -       -           -   \n",
       "3      1  female  16.6     -     -   2.54        -     Mrs           -   \n",
       "\n",
       "  survived  \n",
       "0    0.662  \n",
       "1    0.968  \n",
       "2    0.871  \n",
       "3    0.996  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# highlight only the changes\n",
    "dice_exp.visualize_as_dataframe(show_only_changes=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Bring the proximity and the features to vary arguments for generating counterfactuals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Diverse Counterfactuals found! total time taken: 01 min 01 sec\n"
     ]
    }
   ],
   "source": [
    "# generate counterfactuals\n",
    "dice_exp = exp.generate_counterfactuals(factual_sample, total_CFs=4, desired_class=\"opposite\", proximity_weight=1.5, diversity_weight=1.0,\n",
    "                                        features_to_vary=['pclass','age','fare','embarked'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Query instance (original outcome : 0)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>fare</th>\n",
       "      <th>embarked</th>\n",
       "      <th>title</th>\n",
       "      <th>family_size</th>\n",
       "      <th>survived</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7.25</td>\n",
       "      <td>S</td>\n",
       "      <td>Mr</td>\n",
       "      <td>1</td>\n",
       "      <td>0.136179</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  pclass   sex   age sibsp parch  fare embarked title family_size  survived\n",
       "0      3  male  22.0     1     0  7.25        S    Mr           1  0.136179"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Diverse Counterfactual set (new outcome : 1)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>fare</th>\n",
       "      <th>embarked</th>\n",
       "      <th>title</th>\n",
       "      <th>family_size</th>\n",
       "      <th>survived</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0.7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6.74</td>\n",
       "      <td>S</td>\n",
       "      <td>Mr</td>\n",
       "      <td>1</td>\n",
       "      <td>0.615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>9.5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>Q</td>\n",
       "      <td>Mr</td>\n",
       "      <td>1</td>\n",
       "      <td>0.733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>14.01</td>\n",
       "      <td>C</td>\n",
       "      <td>Mr</td>\n",
       "      <td>1</td>\n",
       "      <td>0.714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>8.3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>37.11</td>\n",
       "      <td>Q</td>\n",
       "      <td>Mr</td>\n",
       "      <td>1</td>\n",
       "      <td>0.738</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  pclass   sex   age sibsp parch   fare embarked title family_size  survived\n",
       "0      1  male   0.7     1     0   6.74        S    Mr           1     0.615\n",
       "1      1  male   9.5     1     0   0.00        Q    Mr           1     0.733\n",
       "2      1  male  22.0     1     0  14.01        C    Mr           1     0.714\n",
       "3      1  male   8.3     1     0  37.11        Q    Mr           1     0.738"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# visualize the resutls\n",
    "dice_exp.visualize_as_dataframe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}