test_timing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "vTbNLZGY9Zjr",
    "outputId": "ad98f449-8962-40fd-bea2-8e585067a421"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "\n",
    "#Use if working on Colab\n",
    "#from google.colab import drive\n",
    "#drive.mount('/content/drive')\n",
    "#PATH = '/content/drive/My Drive/PPM_Stability/'\n",
    "\n",
    "#If working locally\n",
    "PATH = os.getcwd()\n",
    "sys.path.append(PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "NOTE47wJ9v8E",
    "outputId": "0e69dc67-1419-4609-e176-e3e8fd1dc9c6"
   },
   "outputs": [],
   "source": [
    "#!pip install lime==0.2.0.1\n",
    "#!pip install shap==0.35.0\n",
    "#!pip install pandas==0.19.2\n",
    "#!pip install xgboost==1.0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "-wVsWqvt9zzb",
    "outputId": "c5dc50ef-5eba-4b9d-ede2-c4c23600c064"
   },
   "outputs": [],
   "source": [
    "import EncoderFactory\n",
    "#from DatasetManager_for_colab import DatasetManager\n",
    "from DatasetManager import DatasetManager\n",
    "import BucketFactory\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.pipeline import FeatureUnion, Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "import time\n",
    "import os\n",
    "import sys\n",
    "from sys import argv\n",
    "import pickle\n",
    "from collections import defaultdict\n",
    "import random\n",
    "import joblib\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import xgboost as xgb\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "from keras.models import Sequential, Model, load_model\n",
    "from keras.layers.core import Dense, Activation, Dropout\n",
    "from keras.preprocessing import sequence\n",
    "from keras.layers import Dense, Embedding, Flatten, Input, LSTM\n",
    "from keras.optimizers import Nadam, RMSprop\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "\n",
    "from tensorflow.keras.backend import print_tensor\n",
    "from tensorflow.keras.utils import plot_model\n",
    "from tensorflow.compat.v1 import disable_v2_behavior#, ConfigProto, Session\n",
    "from tensorflow.compat.v1.keras.backend import get_session\n",
    "disable_v2_behavior()\n",
    "\n",
    "import lime\n",
    "import lime.lime_tabular\n",
    "\n",
    "import shap\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "NgQh__fq9_xK"
   },
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "lime_pal = sns.diverging_palette(100, 200, s=150, as_cmap=True)\n",
    "shap_pal = sns.diverging_palette(0, 240, s=150, as_cmap=True)\n",
    "\n",
    "sns.set_style('whitegrid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "id": "SynFz2rV-arK",
    "outputId": "26de37b4-9690-48e8-90a1-bc6889559dc4"
   },
   "outputs": [],
   "source": [
    "dataset_ref = \"production\"\n",
    "params_dir = PATH + \"params\"\n",
    "results_dir = \"results\"\n",
    "bucket_method = \"single\"\n",
    "cls_encoding = \"agg\"\n",
    "cls_method = \"xgboost\"\n",
    "\n",
    "gap = 1\n",
    "n_iter = 1\n",
    "\n",
    "method_name = \"%s_%s\"%(bucket_method, cls_encoding)\n",
    "\n",
    "generate_lime = True\n",
    "generate_model_shap = True\n",
    "\n",
    "sample_size = 0.25\n",
    "exp_iter = 1\n",
    "max_feat = 10\n",
    "\n",
    "dataset_ref_to_datasets = {\n",
    "    #\"bpic2011\": [\"bpic2011_f%s\"%formula for formula in range(1,5)],\n",
    "    \"bpic2015\": [\"bpic2015_%s_f2\"%(municipality) for municipality in range(5,6)],\n",
    "    \"bpic2017\" : [\"bpic2017_accepted\"],\n",
    "    \"bpic2012\" : [\"bpic2012_accepted\"],\n",
    "    #\"insurance\": [\"insurance_activity\", \"insurance_followup\"],\n",
    "    \"sepsis_cases\": [\"sepsis_cases_1\"],# \"sepsis_cases_2\", \"sepsis_cases_4\"]\n",
    "    \"production\" : [\"production\"]\n",
    "}\n",
    "\n",
    "datasets = [dataset_ref] if dataset_ref not in dataset_ref_to_datasets else dataset_ref_to_datasets[dataset_ref]\n",
    "\n",
    "datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9BPaNPbQo39O",
    "outputId": "c49d5458-664e-4912-f5bb-cc1a065d403a",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "  for dataset_name in datasets:\n",
    "      \n",
    "      dataset_manager = DatasetManager(dataset_name)\n",
    "      \n",
    "      for ii in range(n_iter):\n",
    "            num_buckets = len([name for name in os.listdir(os.path.join(PATH,'%s/%s_%s/models'% (dataset_ref, cls_method, method_name)))])\n",
    "\n",
    "            all_pref_len = []\n",
    "            all_feat_len = []\n",
    "            all_shap_times = []\n",
    "            all_lime_times = []\n",
    "            sep_pref_len = []\n",
    "            sep_feat_len = []\n",
    "            all_times = []\n",
    "            all_types = []\n",
    "\n",
    "            sep_data_dict = {'Prefix Length': sep_pref_len, 'Feature Vector Length': sep_feat_len, 'SHAP Running Time': all_shap_times, 'LIME Running Time': all_lime_times}\n",
    "            all_data_dict = {'Prefix Length': all_pref_len, 'Feature Vector Length': all_feat_len, 'Explainer': all_types, 'Running Time': all_times}\n",
    "            sep_timing_path = os.path.join(PATH, \"%s/%s_%s/sep_timing.csv\" % (dataset_ref, cls_method, method_name))\n",
    "            all_timing_path = os.path.join(PATH, \"%s/%s_%s/all_timing.csv\" % (dataset_ref, cls_method, method_name))\n",
    "\n",
    "\n",
    "            for bucket in range(num_buckets):\n",
    "                bucketID = bucket+1\n",
    "                print ('Bucket', bucketID)\n",
    "\n",
    "                pipeline_path = os.path.join(PATH, \"%s/%s_%s/pipelines/pipeline_bucket_%s.joblib\" % (dataset_ref, cls_method, method_name, bucketID))\n",
    "                feat_comb_path = os.path.join(PATH, \"%s/%s_%s/bucketers_and_encoders/feature_combiner_bucket_%s.joblib\" % (dataset_ref, cls_method, method_name, bucketID))\n",
    "                bucketer_path = os.path.join(PATH, \"%s/%s_%s/bucketers_and_encoders/bucketer_bucket_%s.joblib\" % (dataset_ref, cls_method, method_name, bucketID))\n",
    "                cls_path = os.path.join(PATH, \"%s/%s_%s/models/cls_bucket_%s.joblib\" % (dataset_ref, cls_method, method_name, bucketID))\n",
    "\n",
    "                predictor = joblib.load(pipeline_path)\n",
    "                cls = joblib.load(cls_path)\n",
    "                feature_combiner = joblib.load(feat_comb_path)\n",
    "                bucketer = joblib.load(bucketer_path)\n",
    "\n",
    "                #import data for bucket\n",
    "                X_train_path = os.path.join(PATH, \"%s/%s_%s/train_data/bucket_%s_prefixes.pickle\" % (dataset_ref, cls_method, method_name, bucketID))                  \n",
    "                with open (X_train_path, 'rb') as f:\n",
    "                    dt_train_bucket = pickle.load(f)\n",
    "\n",
    "                X_test_path = os.path.join(PATH, \"%s/%s_%s/test_data/bucket_%s_prefixes.pickle\" % (dataset_ref, cls_method, method_name, bucketID))                  \n",
    "                with open (X_test_path, 'rb') as f:\n",
    "                    dt_test_bucket = pickle.load(f)\n",
    "\n",
    "                dt_test_bucket = dt_test_bucket.dropna()\n",
    "                dt_test_sample = dt_test_bucket.sample(frac = sample_size)\n",
    "\n",
    "                dt_testing_sample = feature_combiner.fit_transform(dt_test_sample)\n",
    "                lens = dataset_manager.get_prefix_lengths(dt_test_sample)\n",
    "\n",
    "                feat_len = [len(vec) for vec in dt_testing_sample]\n",
    "\n",
    "                #Get a list of feature names\n",
    "                feat_list = feature_combiner.get_feature_names()\n",
    "\n",
    "                #create explainers now that can be passed later\n",
    "                start_time = time.time()\n",
    "                shap_explainer = shap.TreeExplainer(cls)\n",
    "                duration = time.time() - start_time\n",
    "                print(\"Time taken to create SHAP explainer:\", duration, \"seconds.\")\n",
    "\n",
    "                start_time = time.time()\n",
    "                class_names=['regular','deviant']# regular is 0, deviant is 1, 0 is left, 1 is right\n",
    "                trainingdata = feature_combiner.fit_transform(dt_train_bucket)\n",
    "                lime_explainer = lime.lime_tabular.LimeTabularExplainer(trainingdata,\n",
    "                                feature_names = feat_list,\n",
    "                                class_names=class_names, discretize_continuous=True)\n",
    "                duration = time.time() - start_time\n",
    "                print(\"Time taken to create LIME explainer:\", duration, \"seconds.\")\n",
    "\n",
    "\n",
    "                shap_times = []\n",
    "                lime_times = []\n",
    "\n",
    "                for instance in dt_testing_sample:\n",
    "                \n",
    "                    #generate data for SHAP\n",
    "                    start_time = time.time()\n",
    "                    shap_explainer.shap_values(np.array([instance,]))\n",
    "                    duration = time.time() - start_time\n",
    "                    shap_times.append(duration)\n",
    "\n",
    "                    #generate data for LIME\n",
    "                    start_time = time.time()\n",
    "                    lime_explainer.explain_instance(instance, cls.predict_proba)\n",
    "                    duration = time.time() - start_time\n",
    "                    lime_times.append(duration)\n",
    "\n",
    "                #Update and save times\n",
    "                sep_pref_len.extend(list(lens))\n",
    "                sep_feat_len.extend(list(feat_len))\n",
    "                all_shap_times.extend(shap_times)\n",
    "                all_lime_times.extend(lime_times)\n",
    "\n",
    "                for i in range(2):\n",
    "                    all_pref_len.extend(sep_pref_len)\n",
    "                    all_feat_len.extend(sep_feat_len)\n",
    "                    \n",
    "                all_times.extend(all_shap_times)\n",
    "                all_times.extend(all_lime_times)\n",
    "                all_types.extend([\"SHAP\"]*len(all_shap_times))\n",
    "                all_types.extend([\"LIME\"]*len(all_lime_times))\n",
    "\n",
    "            sep_data = pd.DataFrame(data = sep_data_dict)\n",
    "            sep_data.to_csv(sep_timing_path, index = False)\n",
    "\n",
    "            all_data = pd.DataFrame(data = all_data_dict)\n",
    "            all_data.to_csv(all_timing_path, index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "tuJCUX69yiAO"
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots()\n",
    "ax.plot(sep_data[\"Prefix Length\"], sep_data[\"SHAP Running Time\"], 'ro')#, y = ['SHAP Running Time', 'LIME Running Time'], x = ['Prefix Length'] )#, hue = data['Feature Vector Length'])\n",
    "ax.plot(sep_data[\"Prefix Length\"], sep_data[\"LIME Running Time\"], 'bo')#, y = ['SHAP Running Time', 'LIME Running Time'], x = ['Prefix Length'] )#, hue = data['Feature Vector Length'])\n",
    "#grid.map(sns.scatterplot, color=\".3\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "UORiSY7W90Ow"
   },
   "outputs": [],
   "source": [
    "sns.scatterplot(x = all_data['Prefix Length'], y = all_data['Running Time'], hue = all_data['Explainer'], size = all_data['Feature Vector Length'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "timing.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}