examples/notebooks/spark/pyspark_glm_explain.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark \n",
    "findspark.init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession.builder.appName(\"pyspark_glm_explain\").config(\"spark.sql.execution.arrow.enabled\", \"true\").enableHiveSupport().getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from typing import List\n",
    "from itertools import chain\n",
    "from pyspark.ml import Model\n",
    "from pyspark.ml import Pipeline, PipelineModel\n",
    "from pyspark.sql import SparkSession, DataFrame\n",
    "from pyspark.ml.regression import GeneralizedLinearRegression\n",
    "from pyspark.ml.feature import StringIndexer, VectorAssembler, SQLTransformer, OneHotEncoder\n",
    "\n",
    "from transparency.spark.ohe.decoder import OneHotDecoder\n",
    "from transparency.spark.prediction.explainer.glm import GLMExplainTransformer\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_glm_pipeline_stages(categorical_columns, continuous_columns, label_column, family=\"tweedie\", link=\"identity\",\n",
    "                            variance_power=0.0, link_power=1.0) -> List:\n",
    "    encoders = []\n",
    "    for c in categorical_columns:\n",
    "        indexer = StringIndexer(inputCol=c, outputCol=f\"{c}_IDX\")\n",
    "        encoders.append(indexer)\n",
    "        encoder = OneHotEncoder(inputCol=indexer.getOutputCol(), outputCol=f\"{c}_OHE\", dropLast=False)\n",
    "        encoders.append(encoder)\n",
    "\n",
    "    features_column = f\"features_{label_column}\"\n",
    "    prediction_column = f\"prediction_{label_column}\"\n",
    "\n",
    "    assembler = VectorAssembler(inputCols=[f\"{c}_OHE\" for c in categorical_columns] + continuous_columns,\n",
    "                                outputCol=features_column)\n",
    "\n",
    "    if family == 'tweedie':\n",
    "        _model = GeneralizedLinearRegression(labelCol=label_column, featuresCol=features_column,\n",
    "                                             predictionCol=prediction_column, family=family, linkPower=link_power,\n",
    "                                             variancePower=variance_power)\n",
    "    else:\n",
    "        _model = GeneralizedLinearRegression(labelCol=label_column, featuresCol=features_column,\n",
    "                                             predictionCol=prediction_column, family=family,\n",
    "                                             link=link)\n",
    "\n",
    "    stages: List = encoders + [assembler, _model]\n",
    "\n",
    "    return stages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_feature_coefficients(spark_session: SparkSession, model: Model, prediction_df: DataFrame,\n",
    "                             feature_column: str, ) -> DataFrame:\n",
    "    ml_attributes = prediction_df.schema[feature_column].metadata[\"ml_attr\"][\"attrs\"].values()\n",
    "    attrs = sorted((attr[\"idx\"], attr[\"name\"]) for attr in (chain(*ml_attributes)))\n",
    "    coefficients = [\n",
    "        (idx, re.sub('[^0-9a-zA-Z]+', '_', name), name, float(model.coefficients[idx]))\n",
    "        for\n",
    "        idx, name\n",
    "        in attrs]\n",
    "    coefficients.append((-1, \"Intercept\", \"Intercept\", model.intercept))\n",
    "    coefficients.sort(key=lambda x: x[3], reverse=True)\n",
    "    coefficients_df: DataFrame = spark_session.createDataFrame(coefficients).toDF(\"Feature_Index\",\n",
    "                                                                                  \"Feature\",\n",
    "                                                                                  \"Original_Feature\",\n",
    "                                                                                  \"Coefficient\")\n",
    "\n",
    "    return coefficients_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resolve_link_function(family: str, link: str, link_power=1.0):\n",
    "    _link_types = {\"log\": \"logLink\", \"sqrt\": \"powerHalfLink\", \"identity\": \"identityLink\",\n",
    "                   \"logit\": \"logitLink\", \"inverse\": \"inverseLink\", \"other\": \"otherPowerLink\"}\n",
    "    if 'tweedie' == family:\n",
    "        return _link_types[{0.0: \"log\", 0.5: \"sqrt\", 1.0: \"identity\", -1.0: \"inverse\"}.get(link_power, \"other\")]\n",
    "    else:\n",
    "        return _link_types[link]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_glm_explain_stages(predictions_view: str, coefficients_view: str, label_column: str,\n",
    "                           family: str = 'tweedie', link: str = 'identity',\n",
    "                           variance_power: float = 0.0, link_power: float = 1.0) -> List:\n",
    "    link_function_type = resolve_link_function(family, link, link_power)\n",
    "    stages = [\n",
    "        OneHotDecoder(oheSuffix=\"_OHE\", idxSuffix=\"_IDX\", unknownSuffix=\"Unknown\"),\n",
    "        SQLTransformer(statement=f\"CREATE OR REPLACE TEMPORARY VIEW {predictions_view} AS SELECT * from __THIS__\"),\n",
    "        GLMExplainTransformer(predictionView=predictions_view, coefficientView=coefficients_view,\n",
    "                              linkFunctionType=link_function_type, label=label_column, nested=True,\n",
    "                              calculateSum=True, family=family, variancePower=variance_power, linkPower=link_power\n",
    "                              )\n",
    "    ]\n",
    "    return stages"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GeneralizedLinearRegression\n",
    "\n",
    "- The example below is using **boston dataset** for preciting the **median home prices** for occupied homes.\n",
    "- The example is stricitly used for **demonstrating** the uagae of `GLMExplainTransformer`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "boston_csv = '/work/tests/data/regression/dataset_boston.csv'\n",
    "boston_df: DataFrame = spark.read.option(\"header\", True).option(\"inferSchema\", True).csv(boston_csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_column = 'medv'\n",
    "features_column = f\"features_{label_column}\"\n",
    "prediction_column = f\"prediction_{label_column}\"\n",
    "\n",
    "contrib_column = f\"prediction_{label_column}_contrib\"\n",
    "contrib_column_sum = f\"{contrib_column}_sum\"\n",
    "contrib_column_intercept = f\"{contrib_column}_intercept\"\n",
    "\n",
    "features_coefficient_view = f\"features_coefficient_{label_column}_view\"\n",
    "predictions_view = f\"predictions_{label_column}_view\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_columns=[]\n",
    "continuous_columns = [x for x in boston_df.columns if x not in ['id', label_column]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "stages = get_glm_pipeline_stages([], continuous_columns + categorical_columns, label_column,family=\"gaussian\", link=\"log\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "import uuid\n",
    "pipeline = Pipeline(stages=stages)\n",
    "pipeline_model: PipelineModel = pipeline.fit(boston_df)\n",
    "glm_model = pipeline_model.stages[-1]\n",
    "string_id = uuid.uuid4()\n",
    "glm_model_path: str = f\"/tmp/{string_id}\"\n",
    "glm_model.write().save(glm_model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_df = pipeline_model.transform(boston_df)\n",
    "prediction_df.createOrReplaceTempView(predictions_view)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_coefficients_df = get_feature_coefficients(spark, glm_model, prediction_df, features_column)\n",
    "features_coefficients_df.createOrReplaceTempView(features_coefficient_view)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+---------+----------------+--------------------+\n",
      "|Feature_Index|  Feature|Original_Feature|         Coefficient|\n",
      "+-------------+---------+----------------+--------------------+\n",
      "|           -1|Intercept|       Intercept|  3.5934179294150237|\n",
      "|            5|       rm|              rm| 0.14297146250097179|\n",
      "|            3|     chas|            chas| 0.06606567346006308|\n",
      "|            8|      rad|             rad| 0.01568489582001018|\n",
      "|            2|    indus|           indus| 0.00312300114663444|\n",
      "|            1|       zn|              zn|9.853163534174047E-4|\n",
      "|           11|    black|           black|5.189852722299955E-4|\n",
      "|            6|      age|             age|4.438128429448265...|\n",
      "|            9|      tax|             tax|-5.04566030374328...|\n",
      "|            0|     crim|            crim|-0.00918445383815...|\n",
      "|           12|    lstat|           lstat|-0.03466252972714503|\n",
      "|           10|  ptratio|         ptratio|-0.03482239880119...|\n",
      "|            7|      dis|             dis|-0.04771866722146...|\n",
      "|            4|      nox|             nox| -0.6597313771525636|\n",
      "+-------------+---------+----------------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "features_coefficients_df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "explain_stages = get_glm_explain_stages(predictions_view, features_coefficient_view, label_column, family=\"gaussian\", link=\"log\")\n",
    "explain_pipeline = Pipeline(stages=explain_stages)\n",
    "explain_df = explain_pipeline.fit(prediction_df).transform(prediction_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_index_meta = features_coefficients_df.select(\"Feature_Index\", \"Feature\").filter(\"Feature_Index!=-1\").orderBy(\"Feature_Index\").rdd.collectAsMap()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "expressions = []\n",
    "for key,value in feature_index_meta.items():\n",
    "    expressions.append(f\"AVG({contrib_column}[{key}]) as {value}\")\n",
    "\n",
    "contributions = explain_df.selectExpr(expressions).collect()[0].asDict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Contributions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>crim</th>\n",
       "      <td>-1.190116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zn</th>\n",
       "      <td>0.513555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>indus</th>\n",
       "      <td>1.528219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chas</th>\n",
       "      <td>0.218917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nox</th>\n",
       "      <td>-14.390360</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Contributions\n",
       "crim       -1.190116\n",
       "zn          0.513555\n",
       "indus       1.528219\n",
       "chas        0.218917\n",
       "nox       -14.390360"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plot_contrib_df = pd.DataFrame.from_dict(contributions,orient='index',columns=['Contributions'])\n",
    "plot_contrib_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbsAAAGrCAYAAABZvzmFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAAl30lEQVR4nO3de5RedX3v8feHmwESQMOIIsYBqyLhkupAA3IJYCtK66VaKeIlWs3ytJai4jFWa7HLC5S20tpjOdEeoxUrR7wUjUetCiIoQoBwB0UMgiCGoMhFEJLv+WN28GGcJEMyM0/ml/drrVmzL7+99/f3JOTDb1+enapCkqSWbdHvAiRJmmiGnSSpeYadJKl5hp0kqXmGnSSpeYadJKl5hp2khyX56yQf7aYHk1SSrcZp37OS3JNky/HYn/RoGHZqQpLlSX7V/WO65mfXcdjnc8erxjEe8+lJPpPkjiR3JbkiyVs2NiCSzEtyy/raVdX7q+r1G3OsnmM+4vOrqh9X1fSqWjUe+5ceDcNOLfmj7h/TNT+39rOYRzsiSvJU4HvAzcA+VbUj8CfAEDBj/Cv8reOPywhO2hQZdmpakh2T/HuS25L8JMl714ySkjw1yTeTrOxGUmck2alb9x/ALOCL3Sjxf442OuodvSQ5KclZST6Z5JfA/HUdfxTvAb5TVW+pqtsAqur6qnpFVf2iO8YLk1yd5BdJzk3yzBG1nNiNBu9KcmaSaUm2B/4fsGvvqHct9Z6U5JMj6npdklu7PpzYc7zFSd7bM//w57OWz+8Rp0W7Gs5OcmeSG5K8oWdfJyX5v0k+keTurs9DPevf3n2edye5PsmR6/mroM2cYafWLQYeAn4H+F3gD4A1p+kCfADYFXgm8GTgJICqehXwY34zWvz7MR7vRcBZwE7AGes5/kjP7bYdVZKnA/8JnAAMAF9mOEy26Wn2cuAoYHdgX2B+Vd0LPB+4dZRR78h6R3M48LSu9reP5dTuGD+/TwO3MPz5vwx4f5Ijeta/sGuzE3A28K/d5/AM4E3A/lU1A3gesHx9NWnzZtipJV/oRjy/SPKFJLsALwBOqKp7q+pnwAeBPwWoqhuq6r+r6oGqWgH8E3DYRtbw3ar6QlWtBnZY1/FHMRO4bR37PgZY0tX8IPAPwLbAQT1t/qWqbq2qO4EvAnPGWm9V/Wotbd7T1X8l8DHg2PXsc72SPBl4DvD2qrq/qpYBHwVe3dPs/Kr6cneN7z+A/brlq4DHAHsl2bqqllfVDze2JrXNc/RqyYur6utrZpIcAGwN3JZkzeItGL4mRheG/wwcwvA1sS2An29kDTf3TD9lXccfxUrgievY967ATWtmqmp1kpuBJ/W0+WnP9H3dNmOtdyxtbgL2GcM267MrcGdV3T1i30M98yP7Mi3JVlV1Q5ITGB6Fz07yVeAt/b5Gq02bIzu17GbgAWDnqtqp+9mhqmZ3698PFMM3g+wAvJLhU5trjHwlyL3AdmtmumtvAyPa9G6zvuOP9HXgpevoz60MB+ia44fhU68/Wcc2o9U1luW9ntwzPaurA0Z8HsATHsW+bwUel6T3xptZjK0vVNWnqupghj+PAk4Zy3bafBl2alZ3k8fXgH9MskOSLbqbUtacqpwB3APcleRJwNtG7OJ2YI+e+e8zPLo4OsnWwLsYPp22occf6W+Bg5KcmuQJAEl+p7uBZCfg/wJHJzmyO/5bGQ7T74zh47gdmJlkxzG0HelvkmyXZDbwWuDMbvky4AVJHtfVe8Iox9yDUVTVzV3dH+huotkX+DNg5M0xvyXJM5IckeQxwP3Ar4DVj75b2pwYdmrdq4FtgGsYPkV5Fr85Vfge4FnAXcAS4HMjtv0A8K7uGuCJVXUX8OcMX1v6CcMjm/U9u7au4z9Cd93pQGAQuDrJXcBngaXA3VV1PcOjzw8BdwB/xPANIL9e34dQVdcxfHPLjV1/Hs0ziN8CbgC+AfxDVX2tW/4fwOUM3xzyNX4Tgms84vMbZb/Hdn29Ffg88Le9p6HX4THAyQx/Bj8FHg+841H0R5uh+PJWSVLrHNlJkppn2EmSmmfYSZKaZ9hJkpo3pR8q33nnnWtwcLDfZahxK1euBGDmzJl9rkTSulxyySV3VNXIZ1+BKR52g4ODLF26tN9lqHGLFy8GYP78+X2tQ9K6Jblpbes8jSlJap5hJ0lqnmEnSWrelL5mN5oHH3yQW265hfvvv7/fpajHtGnT2G233dh66637XYqkzVBzYXfLLbcwY8YMBgcH6Xmtivqoqli5ciW33HILu+++e7/LkbQZau405v3338/MmTMNuk1IEmbOnOloW1LfNDeyAwy6TdBU/zO58MaVnLRwSb/LkJq0/OSjJ/wYzY3sJEkaqcmRXa/Bcf6/8bH+H8hPf/pTTjjhBC6++GJ22mkndtllF0477TSe/vSnP6rjnXbaaSxYsIDttttu1PWvf/3rectb3sJee+3F9OnTueeee8a872XLlnHrrbfyghe8AICzzz6ba665hoULFz6qGiVpU+fIbgJUFS95yUuYN28eP/zhD7nkkkv4wAc+wO233/6o93Xaaadx3333jbpu1apVfPSjH2WvvfbaoDqXLVvGl7/85YfnX/jCFxp0kppk2E2Ac845h6233po3vvGNDy/bb7/9OPjgg3nb297G3nvvzT777MOZZw6/2Pncc89l3rx5vOxlL2PPPffkuOOOo6r4l3/5F2699VYOP/xwDj/8cACmT5/OW9/6Vvbbbz+++93vMm/evEd8Zdqb3/xmZs+ezZFHHsmKFSsAHtHmjjvuYHBwkF//+te8+93v5swzz2TOnDmceeaZLF68mDe96U0ALF++nCOOOIJ9992XI488kh//+MfA8FdmHX/88Rx00EHssccenHXWWQDcdtttHHroocyZM4e9996bb3/72xP8KUvS2PU17DKsucC96qqrePazn/1byz/3uc+xbNkyLr/8cr7+9a/ztre9jdtuuw2Ayy67jNNOO41rrrmGG2+8kQsuuIDjjz+eXXfdlXPOOYdzzjkHgHvvvZff+73f4/LLL+fggw9+xP7vvfdehoaGuPrqqznssMN4z3ves9Yat9lmG/7u7/6OY445hmXLlnHMMcc8Yv1f/uVf8prXvIYrrriC4447juOPP/7hdbfddhvnn38+X/rSlx4eCX7qU5/iec973sP9mzNnzgZ9dpI0ESY9aJIMJrk+ySeAe4AfJlmc5PtJzkjy3CQXJPlBkgMmu76JdP7553Psscey5ZZbsssuu3DYYYdx8cUXA3DAAQew2267scUWWzBnzhyWL18+6j623HJLXvrSl466bosttng4tF75yldy/vnnb3Ct3/3ud3nFK14BwKte9apH7OvFL34xW2yxBXvttdfDp2b3339/Pvaxj3HSSSdx5ZVXMmPGjA0+tiSNt36Nqp4GfBiYDTwZ+Edgz+7nFcDBwInAX4/cMMmCJEuTLF1zmm5TM3v2bC655JJHtc1jHvOYh6e33HJLHnrooVHbTZs2jS233HJM+1xzu/9WW23F6tWrAcblWbfeWqsKgEMPPZTzzjuPJz3pScyfP59PfOITG30cSRov/Qq7m6rqwm76R1V1ZVWtBq4GvlHD/4JeCQyO3LCqFlXVUFUNDQyM+tqivjviiCN44IEHWLRo0cPLrrjiCnbaaSfOPPNMVq1axYoVKzjvvPM44IB1D15nzJjB3XffPabjrl69+uFraJ/61KcePs05ODj4cPiuWb++fR900EF8+tOfBuCMM87gkEMOWeexb7rpJnbZZRfe8IY38PrXv55LL710TDVL0mTo16MH9/ZMP9AzvbpnfjXjUN9kPKw4UhI+//nPc8IJJ3DKKacwbdo0BgcHOe2007jnnnvYb7/9SMLf//3f84QnPIHrrrturftasGABRx111MPX7tZl++2356KLLuK9730vj3/84x++AebEE0/k5S9/OYsWLeLoo3/zeRx++OGcfPLJzJkzh3e84x2P2NeHPvQhXvva13LqqacyMDDAxz72sXUe+9xzz+XUU09l6623Zvr06Y7sJG1SsuY01KQdMBkEvlRVe/dOd+sWd/NnjVw3mqGhoRr58tZrr72WZz7zmRNUvTbGVP2zWbx4MRfeuJKv/HrPfpciNWm8BiVJLqmqodHWNXcnpCRJI036acyqWg7sPXK6m58/WjtJkjZGkyO7yT41q/Xzz0RSPzX33ZjTpk1j5cqVvuZnE7LmfXbTpk3rdykbbO4eMzl9/uTf7CRpfDQXdrvtthu33HILm+ozeJurNW8ql6R+aC7stt56a9+GLUl6hCav2UmS1MuwkyQ1z7CTJDXPsJMkNc+wkyQ1z7CTJDXPsJMkNc+wkyQ1z7CTJDXPsJMkNc+wkyQ1z7CTJDWvuS+C3hCDC5f0uwRtwo7aZiVz95jZ7zIkbQRHdpKk5hl2kqTmGXaSpOZtcNglOSHJdhuw3fwku/bMfzTJXhtahyRJ67MxI7sTgFHDLsmW69huPvBw2FXV66vqmo2oQ5KkdVpv2CUZTHJdkjOSXJvkrCTHMxxY5yQ5p2t3T5J/THI5cGCSdye5OMlVSRZl2MuAIeCMJMuSbJvk3CRD3T6OTXJlt80pE9hvSdJmZKwju2cAH66qZwK/BLYBbgUOr6rDuzbbA9+rqv2q6nzgX6tq/6raG9gW+MOqOgtYChxXVXOq6ldrDtCd2jwFOAKYA+yf5MUjC0myIMnSJEtXrFixAV2WJG1uxhp2N1fVBd30J4GDR2mzCvhsz/zhSb6X5EqGA2z2eo6xP3BuVa2oqoeAM4BDRzaqqkVVNVRVQwMDA2MsX5K0ORvrQ+W1nnmA+6tqFUCSacCHgaGqujnJScC0Da5SkqSNMNaR3awkB3bTrwDOB+4GZqyl/ZpguyPJdOBlPevWtt1FwGFJdu5ucDkW+NYY65Mkaa3GOrK7HviLJP8HuAb4N+DXwFeS3Npz3Q6AqvpFko8AVwE/BS7uWb0YOD3Jr4ADe7a5LclC4BwgwJKq+q8N65YkSb8x1rB7qKpeOWLZh7ofAKpqeu/KqnoX8K6RO6qqz/LIa3vzetb9J/CfY6xJkqQx8RtUJEnNW+/IrqqWA3tPfCmSJE0MX/EDLD/56H6XoE3Y4sU+zylNdZ7GlCQ1z7CTJDXPsJMkNc+wkyQ1z7CTJDXPsJMkNc+wkyQ1z7CTJDXPsJMkNc+wkyQ1z7CTJDXPsJMkNc+wkyQ1z7ceNGJw4ZJ+l9Cso7ZZydw9Zva7DEkbwZGdJKl5hp0kqXmGnSSpeeMadknuWc/6vx7jfsbUTpKksZjskd1YQ8ywkySNmwkJuyRPTHJekmVJrkpySJKTgW27ZWd07b6Q5JIkVydZ0C37rXaSJG2MiXr04BXAV6vqfUm2BLarqm8neVNVzelp97qqujPJtsDFST5bVQtHafewLhQXAMyaNWuCypcktWSiTmNeDLw2yUnAPlV191raHZ/kcuBC4MnA09a346paVFVDVTU0MDAwbgVLkto1IWFXVecBhwI/ARYnefXINknmAc8FDqyq/YDLgGkTUY8kafM2UdfsngLcXlUfAT4KPKtb9WCSrbvpHYGfV9V9SfYE5vbsoredJEkbZaKu2c0D3pbkQeAeYM3IbhFwRZJLgdcBb0xyLXA9w6cyGdmuqo6boBolSZuJcQ27qpre/f448PFR1r8deHvPouevZT8j20mStMH8BhVJUvMMO0lS83zFTyOWn3x0v0to1uLFK/pdgqSN5MhOktQ8w06S1DzDTpLUPMNOktQ8w06S1DzDTpLUPMNOktQ8w06S1DzDTpLUPMNOktQ8w06S1DzDTpLUPMNOktQ833qwmRlcuKTfJUw5R22zkrl7zOx3GZI2giM7SVLzDDtJUvMMO0lS8ww7SVLzDDtJUvMMO0lS8yY07JIMJrk2yUeSXJ3ka0m2TTInyYVJrkjy+SSPTfKUJD9IsnOSLZJ8O8kfTGR9kqTNw2SM7J4G/K+qmg38Angp8Ang7VW1L3Al8LdVdRNwCvBvwFuBa6rqayN3lmRBkqVJlq5YsWISypckTXWTEXY/qqpl3fQlwFOBnarqW92yjwOHAlTVR4EdgDcCJ462s6paVFVDVTU0MDAwoYVLktowGWH3QM/0KmCntTVMsh2wWzc7fQJrkiRtRvpxg8pdwM+THNLNvwpYM8o7BTgDeDfwkT7UJklqUL++G/M1wOndSO5G4LVJDgP2B55TVauSvDTJa6vqY32qUZLUiAkNu6paDuzdM/8PPavnjmj+rd5lVfXHE1mbJGnz4XN2kqTm+Yqfzczyk4/udwlTzuLFPuIiTXWO7CRJzTPsJEnNM+wkSc0z7CRJzTPsJEnNM+wkSc0z7CRJzTPsJEnNM+wkSc0z7CRJzTPsJEnNM+wkSc0z7CRJzTPsJEnN8xU/0hhceONKTlq4pN9ljJmvcpIeyZGdJKl5hp0kqXmGnSSpeRsVdkkGk1w1yvJzkwxtwP7mJ/nXjalJkqSRHNlJkpo3HmG3VZIzklyb5Kwk2/WuTPJvSZYmuTrJe3qW75/kO0kuT3JRkhkjtjs6yXeT7DwONUqSNmPjEXbPAD5cVc8Efgn8+Yj176yqIWBf4LAk+ybZBjgT+Kuq2g94LvCrNRskeQmwEHhBVd3Ru7MkC7rwXLpixYpxKF+S1LrxCLubq+qCbvqTwMEj1r88yaXAZcBsYC+GA/K2qroYoKp+WVUPde2PAN4OHF1VPx95sKpaVFVDVTU0MDAwDuVLklo3HmFXa5tPsjtwInBkVe0LLAGmrWd/PwRmAE8fh9okSRqXsJuV5MBu+hXA+T3rdgDuBe5Ksgvw/G759cATk+wPkGRGkjXf5nIT8FLgE0lmj0N9kqTN3HiE3fXAXyS5Fngs8G9rVlTV5QyfvrwO+BRwQbf818AxwIeSXA78Nz0jvqq6DjgO+EySp45DjZKkzdhGfTdmVS0H9hxl1byeNvPXsu3FwNwRixd3P1TVZQxf35MkaaP4nJ0kqXm+9UAag7l7zOT0+b5JQJqqHNlJkppn2EmSmmfYSZKaZ9hJkppn2EmSmmfYSZKaZ9hJkppn2EmSmmfYSZKaZ9hJkppn2EmSmmfYSZKaZ9hJkppn2EmSmucrftQXgwuX9LuEMTtqm5XM3WNmv8uQtBEc2UmSmmfYSZKaZ9hJkpo3KWGXZKckfz4Zx5IkaaTJGtntBBh2kqS+mKywOxl4apJlST6Y5BtJLk1yZZIXASTZP8kVSaYl2T7J1Un2nqT6JEkNm6xHDxYCe1fVnCRbAdtV1S+T7AxcmOTsqro4ydnAe4FtgU9W1VUjd5RkAbAAYNasWZNUviRpKuvHc3YB3p/kUGA18CRgF+CnwN8BFwP3A8ePtnFVLQIWAQwNDdVkFCxJmtr6EXbHAQPAs6vqwSTLgWndupnAdGDrbtm9fahPktSYybpmdzcwo5veEfhZF3SHA0/pafe/gb8BzgBOmaTaJEmNm5SRXVWtTHJBkqsYPk25Z5IrgaXAdQBJXg08WFWfSrIl8J0kR1TVNyejRklSuybtNGZVvWI9TZYDn+jargJ+b6JrkiRtHvwGFUlS8ww7SVLzfMWP+mL5yUf3u4QxW7x4Rb9LkLSRHNlJkppn2EmSmmfYSZKaZ9hJkppn2EmSmmfYSZKaZ9hJkppn2EmSmmfYSZKaZ9hJkppn2EmSmmfYSZKaZ9hJkprnWw+0yRhcuKTfJYzqqG1WMnePmf0uQ9JGcGQnSWqeYSdJap5hJ0lqXt+u2SU5CbgH2AE4r6q+3q9aJElt6/sNKlX17n7XIElq26SexkzyziTfT3I+8Ixu2eIkL+umT05yTZIrkvzDZNYmSWrXpI3skjwb+FNgTnfcS4FLetbPBF4C7FlVlWSnyapNktS2yRzZHQJ8vqruq6pfAmePWH8XcD/w70n+GLhvtJ0kWZBkaZKlK1asmNiKJUlN2GTuxqyqh4ADgLOAPwS+spZ2i6pqqKqGBgYGJrNESdIUNZlhdx7w4iTbJpkB/FHvyiTTgR2r6svAm4H9JrE2SVLDJu2aXVVdmuRM4HLgZ8DFI5rMAP4ryTQgwFsmqzZJUtsm9dGDqnof8L51NDlgsmqRJG0+NplrdpIkTRTDTpLUvL5/g4q0xvKTj+53CaNavNhHXKSpzpGdJKl5hp0kqXmGnSSpeYadJKl5hp0kqXmGnSSpeYadJKl5hp0kqXmGnSSpeYadJKl5hp0kqXmGnSSpeYadJKl5vvVAGoMLb1zJSQuX9LuMR9hU3xIhbYoc2UmSmmfYSZKaZ9hJkppn2EmSmrfJhl2SeUm+1O86JElT36SHXYZtsiErSWrPpDx6kGQQ+CrwPeDZwEVJ9gG2Bc6qqr/t2h0FnAbcB5w/GbVJkto3mc/ZPQ14TVVdmORxVXVnki2BbyTZF/g+8BHgCOAG4MzRdpJkAbAAYNasWZNTuSRpSpvM04k3VdWF3fTLk1wKXAbMBvYC9gR+VFU/qKoCPjnaTqpqUVUNVdXQwMDApBQuSZraJnNkdy9Akt2BE4H9q+rnSRYD0yaxDknSZqYfN4rswHDw3ZVkF+D53fLrgMEkT+3mj+1DbZKkBk36d2NW1eVJLmM43G4GLuiW399dj1uS5D7g28CMya5PktSeSQm7qloO7N0zP38t7b7C8LU7SZLGjc+7SZKa5yt+pDGYu8dMTp/vK3WkqcqRnSSpeYadJKl5hp0kqXmGnSSpeYadJKl5hp0kqXmGnSSpeYadJKl5hp0kqXmGnSSpeYadJKl5hp0kqXmGnSSpeYadJKl5vuJHGoMLb1zJSQuXrHX98pN9/Y+0KXNkJ0lqnmEnSWqeYSdJat4Gh12S7zzK9vOSfGlDjydJ0oba4LCrqoPGsxBJkibKxozs7ul+z0tybpKzklyX5Iwk6dYd1S27FPjjnm1PSnJiz/xVSQaTbJ9kSZLLu2XHbETfJEkCxu/Rg98FZgO3AhcAz0myFPgIcARwA3DmGPZzFHBrVR0NkGTHkQ2SLAAWAMyaNWtcipcktW28blC5qKpuqarVwDJgENgT+FFV/aCqCvjkGPZzJfD7SU5JckhV3TWyQVUtqqqhqhoaGBgYp/IlSS0br7B7oGd6FesfMT404tjTAKrq+8CzGA699yZ59zjVJ0najE3kowfXAYNJntrNH9uzbjnDoUaSZwG7d9O7AvdV1SeBU9e0kSRpY0zY14VV1f3d9bUlSe4Dvg3M6FZ/Fnh1kquB7wHf75bvA5yaZDXwIPA/Jqo+SdLmY4PDrqqmd7/PBc7tWf6mnumvMHztbuS2vwL+YJTdLge+uqE1SZI0Gr9BRZLUPN96II3B3D1mcvp832wgTVWO7CRJzTPsJEnNM+wkSc0z7CRJzTPsJEnNM+wkSc0z7CRJzTPsJEnNM+wkSc0z7CRJzTPsJEnNM+wkSc0z7CRJzTPsJEnN8xU/0hhceONKTlq4ZNR1y0/21T/Sps6RnSSpeYadJKl5hp0kqXmGnSSpeYadJKl5Exp2Sb6Q5JIkVydZ0C37syTfT3JRko8k+ddu+UCSzya5uPt5zkTWJknafEz0owevq6o7k2wLXJxkCfA3wLOAu4FvApd3bf8Z+GBVnZ9kFvBV4Jkjd9iF5gKAWbNmTXD5kqQWTHTYHZ/kJd30k4FXAd+qqjsBknwGeHq3/rnAXknWbLtDkulVdU/vDqtqEbAIYGhoqCa4fklSAyYs7JLMYzjADqyq+5KcC1zHKKO1zhbA3Kq6f6JqkiRtnibymt2OwM+7oNsTmAtsDxyW5LFJtgJe2tP+a8BfrplJMmcCa5MkbUYmMuy+AmyV5FrgZOBC4CfA+4GLgAuA5cBdXfvjgaEkVyS5BnjjBNYmSdqMTNhpzKp6AHj+yOVJllbVom5k93ngC137O4BjJqoeSdLmqx/P2Z2UZBlwFfAjurCTJGmiTPpbD6rqxMk+piRp8+YrfqQxmLvHTE6f76t8pKnKrwuTJDXPsJMkNc+wkyQ1z7CTJDXPsJMkNc+wkyQ1z7CTJDXPsJMkNc+wkyQ1z7CTJDXPsJMkNc+wkyQ1z7CTJDXPtx5IPQYXLvmtZUdts5K5e8zsQzWSxosjO0lS8ww7SVLzDDtJUvMMO0lS8yY97JK8MMnCyT6uJGnzNal3YybZqqrOBs6ezONKkjZv4x52SV4NnAgUcAWwCrgf+F3ggiRXAENV9aYki4FfdeseD7wOeDVwIPC9qpo/3vVJkjY/43oaM8ls4F3AEVW1H/BX3ardgIOq6i2jbPZYhsPtzQyP+D4IzAb2STJnlGMsSLI0ydIVK1aMZ/mSpEaN9zW7I4DPVNUdAFV1Z7f8M1W1ai3bfLGqCrgSuL2qrqyq1cDVwODIxlW1qKqGqmpoYGBgnMuXJLVosm5QuXcd6x7ofq/umV4z7ze8SJI22niH3TeBP0kyEyDJ48Z5/5IkPWrjOnKqqquTvA/4VpJVwGXjuX9JkjbEuJ8mrKqPAx9fx/rFwOJuen7P8uXA3j3z85EkaRz4DSqSpOZ5A4jUY/nJR//WssWLfcRFmuoc2UmSmmfYSZKaZ9hJkppn2EmSmmfYSZKaZ9hJkppn2EmSmmfYSZKaZ9hJkppn2EmSmmfYSZKaZ9hJkppn2EmSmmfYSZKaZ9hJY3DhjSsZXLiEwYVL+l2KpA1g2EmSmmfYSZKaZ9hJkppn2EmSmmfYSZKa15ewS/LGJMu6nx8lOSfJPUnel+TyJBcm2aUftUmS2tOXsKuq06tqDrA/cAvwT8D2wIVVtR9wHvCG0bZNsiDJ0iRLV6xYMVklS5KmsH6fxvxn4JtV9UXg18CXuuWXAIOjbVBVi6pqqKqGBgYGJqdKSdKUtlW/DpxkPvAU4E3dogerqrrpVfSxNklSW/oSKEmeDZwIHFJVq/tRgyRp89Gv0dObgMcB5yQBWNqnOiRJm4G+hF1VvXaUxa/vWX8WcNbkVSRJalm/b1CRJGnCeROINAZz95jJ6fOP7ncZkjaQIztJUvMMO0lS8ww7SVLzDDtJUvMMO0lS8ww7SVLzDDtJUvMMO0lS8ww7SVLzDDtJUvMMO0lS8ww7SVLzDDtJUvMMO0lS8ww7aQwuvHFlv0uQtBEMO0lS8ww7SVLzDDtJUvMmNOySLE7ysok8hiRJ6+PITpLUvHENuySvTnJFksuT/Ee3+NAk30ly45pRXpLpSb6R5NIkVyZ5Ubd8+yRLuu2vSnLMeNYnSdo8bTVeO0oyG3gXcFBV3ZHkccA/AU8EDgb2BM4GzgLuB15SVb9MsjNwYZKzgaOAW6vq6G6fO45ynAXAAoBZs2aNV/mSpIaN58juCOAzVXUHQFXd2S3/QlWtrqprgF26ZQHen+QK4OvAk7p1VwK/n+SUJIdU1V0jD1JVi6pqqKqGBgYGxrF8SVKrJuOa3QM90+l+HwcMAM+uqjnA7cC0qvo+8CyGQ++9Sd49CfVJkho3nmH3TeBPkswE6E5jrs2OwM+q6sEkhwNP6bbZFbivqj4JnMpw8EmStFHG7ZpdVV2d5H3At5KsAi5bR/MzgC8muRJYClzXLd8HODXJauBB4H+MV32SpM3XuIUdQFV9HPj4OtZP737fARw4SpPlwFfHsyZJknzOTpLUPMNOGoO5e8zsdwmSNoJhJ0lqnmEnSWqeYSdJap5hJ0lqnmEnSWqeYSdJap5hJ0lqXqqq3zVssCQrgJv6XccG2hm4o99FTAD7NbXYr6mn1b6NR7+eUlWjvg5nSofdVJZkaVUN9buO8Wa/phb7NfW02reJ7penMSVJzTPsJEnNM+z6Z1G/C5gg9mtqsV9TT6t9m9B+ec1OktQ8R3aSpOYZdpKk5hl2kyjJqUmuS3JFks8n2aln3TuS3JDk+iTP62OZj1qSP0lydZLVSYZGrJuy/VojyVFd/TckWdjvejZUkv+T5GdJrupZ9rgk/53kB93vx/azxg2R5MlJzklyTff38K+65VO6b0mmJbkoyeVdv97TLd89yfe6v49nJtmm37VuiCRbJrksyZe6+Qntl2E3uf4b2Luq9gW+D7wDIMlewJ8Cs4GjgA8n2bJvVT56VwF/DJzXu7CBftHV+7+A5wN7Acd2/ZqKFjP859BrIfCNqnoa8I1ufqp5CHhrVe0FzAX+ovszmup9ewA4oqr2A+YARyWZC5wCfLCqfgf4OfBn/Stxo/wVcG3P/IT2y7CbRFX1tap6qJu9ENitm34R8OmqeqCqfgTcABzQjxo3RFVdW1XXj7JqSvercwBwQ1XdWFW/Bj7NcL+mnKo6D7hzxOIXAR/vpj8OvHgyaxoPVXVbVV3aTd/N8D+gT2KK962G3dPNbt39FHAEcFa3fMr1CyDJbsDRwEe7+TDB/TLs+ud1wP/rpp8E3Nyz7pZu2VTXQr9a6MO67FJVt3XTPwV26WcxGyvJIPC7wPdooG/dqb5lwM8YPjP0Q+AXPf/TPFX/Pp4G/E9gdTc/kwnu11bjuTNBkq8DTxhl1Tur6r+6Nu9k+NTLGZNZ28YYS780tVVVJZmyzyIlmQ58Fjihqn45PFgYNlX7VlWrgDnd9f3PA3v2t6KNl+QPgZ9V1SVJ5k3WcQ27cVZVz13X+iTzgT8EjqzfPOT4E+DJPc1265ZtMtbXr7XY5Ps1Bi30YV1uT/LEqrotyRMZHkFMOUm2Zjjozqiqz3WLm+gbQFX9Isk5wIHATkm26kZBU/Hv43OAFyZ5ATAN2AH4Zya4X57GnERJjmJ46P7CqrqvZ9XZwJ8meUyS3YGnARf1o8Zx1kK/Lgae1t0ptg3DN9yc3eeaxtPZwGu66dcAU26U3l3v+Xfg2qr6p55VU7pvSQbW3LGdZFvg9xm+HnkO8LKu2ZTrV1W9o6p2q6pBhv97+mZVHcdE96uq/JmkH4Zv0LgZWNb9nN6z7p0Mn4+/Hnh+v2t9lP16CcPn2B8Abge+2kK/evrwAobvnv0hw6dt+17TBvbjP4HbgAe7P68/Y/hayTeAHwBfBx7X7zo3oF8HM3zjxhU9/229YKr3DdgXuKzr11XAu7vlezD8P403AJ8BHtPvWjeij/OAL01Gv/y6MElS8zyNKUlqnmEnSWqeYSdJap5hJ0lqnmEnSWqeYSdJap5hJ0lq3v8HJ0RtUy/A9pMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 648x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "final_plot_contrib_df = plot_contrib_df.iloc[plot_contrib_df['Contributions'].abs().argsort()]\n",
    "final_plot_contrib_df.plot(kind='barh', figsize=(9, 7))\n",
    "plt.title('Feature Contributions')\n",
    "plt.axvline(x=0, color='.5')\n",
    "plt.subplots_adjust(left=.3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " # GeneralizedLinearRegression : binomial/logit : classification\n",
    "- The example below is using **prima india diabetes dataset** for preciting the **diabetes outcome** for patient.\n",
    "- The example is stricitly used for **demonstrating** the uagae of `GLMExplainTransformer`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "prima_indian_diabetes_csv = '/work/tests/data/classification/dataset_prima_indian_diabetes.csv'\n",
    "prima_indian_diabetes_df = spark.read.option(\"header\", True).option(\"inferSchema\", True).csv(\n",
    "    prima_indian_diabetes_csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_column = 'outcome'\n",
    "features_column = f\"features_{label_column}\"\n",
    "prediction_column = f\"prediction_{label_column}\"\n",
    "\n",
    "contrib_column = f\"prediction_{label_column}_contrib\"\n",
    "contrib_column_sum = f\"{contrib_column}_sum\"\n",
    "contrib_column_intercept = f\"{contrib_column}_intercept\"\n",
    "\n",
    "features_coefficient_view = f\"features_importance_{label_column}_view\"\n",
    "predictions_view = f\"predictions_{label_column}_view\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_columns=[]\n",
    "continuous_columns = [x for x in prima_indian_diabetes_df.columns if x not in ['id', label_column]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "stages = get_glm_pipeline_stages([], continuous_columns + categorical_columns, label_column,family=\"binomial\", link=\"logit\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "import uuid\n",
    "pipeline = Pipeline(stages=stages)\n",
    "pipeline_model: PipelineModel = pipeline.fit(prima_indian_diabetes_df)\n",
    "glm_model = pipeline_model.stages[-1]\n",
    "string_id = uuid.uuid4()\n",
    "glm_model_path: str = f\"/tmp/{string_id}\"\n",
    "glm_model.write().save(glm_model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_df = pipeline_model.transform(prima_indian_diabetes_df)\n",
    "prediction_df.createOrReplaceTempView(predictions_view)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_coefficients_df = get_feature_coefficients(spark, glm_model, prediction_df, features_column)\n",
    "features_coefficients_df.createOrReplaceTempView(features_coefficient_view)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+--------------------+--------------------+--------------------+\n",
      "|Feature_Index|             Feature|    Original_Feature|         Coefficient|\n",
      "+-------------+--------------------+--------------------+--------------------+\n",
      "|            6|diabetes_pedigree...|diabetes_pedigree...|  0.9451797406211632|\n",
      "|            0|         pregnancies|         pregnancies| 0.12318229835242865|\n",
      "|            5|                 bmi|                 bmi| 0.08970097003094403|\n",
      "|            1|             glucose|             glucose|0.035163714606853205|\n",
      "|            7|                 age|                 age| 0.01486900474447571|\n",
      "|            3|      skin_thickness|      skin_thickness|6.189643648752055E-4|\n",
      "|            4|             insulin|             insulin|-0.00119169898416...|\n",
      "|            2|      blood_pressure|      blood_pressure|-0.01329554690430...|\n",
      "|           -1|           Intercept|           Intercept|  -8.404696366913944|\n",
      "+-------------+--------------------+--------------------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "features_coefficients_df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "explain_stages = get_glm_explain_stages(predictions_view, features_coefficient_view, label_column, family=\"binomial\", link=\"logit\")\n",
    "explain_pipeline = Pipeline(stages=explain_stages)\n",
    "explain_df = explain_pipeline.fit(prediction_df).transform(prediction_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_index_meta = features_coefficients_df.select(\"Feature_Index\", \"Feature\").filter(\"Feature_Index!=-1\").orderBy(\"Feature_Index\").rdd.collectAsMap()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "expressions = []\n",
    "for key,value in feature_index_meta.items():\n",
    "    expressions.append(f\"AVG({contrib_column}[{key}]) as {value}\")\n",
    "\n",
    "contributions = explain_df.selectExpr(expressions).collect()[0].asDict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Contributions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>pregnancies</th>\n",
       "      <td>0.025666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glucose</th>\n",
       "      <td>0.217186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>blood_pressure</th>\n",
       "      <td>-0.075511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>skin_thickness</th>\n",
       "      <td>0.000622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>insulin</th>\n",
       "      <td>-0.008005</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Contributions\n",
       "pregnancies          0.025666\n",
       "glucose              0.217186\n",
       "blood_pressure      -0.075511\n",
       "skin_thickness       0.000622\n",
       "insulin             -0.008005"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plot_contrib_df = pd.DataFrame.from_dict(contributions,orient='index',columns=['Contributions'])\n",
    "plot_contrib_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiQAAAGrCAYAAADw/YzgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAAskElEQVR4nO3deZheVZ2v/ftLGCKDgCGioCFii0xKhAIRJ7BtRHF8RdNObRygaT3StEePeKQ1eLAF8bzijGlbooJKixOCAqJEGhQhgRBmhxgaBDVERWYl/M4fzy54UlSSSlKpFVL357rqqv2svfbav/VUtL619qoiVYUkSVJLG7QuQJIkyUAiSZKaM5BIkqTmDCSSJKk5A4kkSWrOQCJJkpozkEjSw0iS/53k893x1CSVZMNRGntKkjuSTBiN8aRVYSCRNCaSLEpyd/cNb/Bju1EY8/mjVeMI77lTkq8nuTXJbUkWJHnnmn4TT7J/kptW1q+q/q2q3rom9+q75zLvX1X9d1VtXlVLR2N8aVUYSCSNpZd03/AGP25uWcyqriwkeSLwM+BG4ClVtSXwKmAA2GL0K3zI/UdlJURaFxlIJDWVZMsk/5HkliS/SXLs4GpDkicm+VGSJd2KxKlJturOfRmYAny3W235X8OtMvSvAiSZmeT0JKck+TMwY0X3H8YxwE+q6p1VdQtAVV1fVa+tqj9193hpkquT/CnJnCS7DKnlXd2qym1JTksyMclmwPeB7fpXj5ZT78wkpwyp681Jbu7m8K6++81Ocmzf6wfen+W8f8s8AupqOCPJH5L8MsmhfWPNTPKfSb6U5PZuzgN959/TvZ+3J7k+yd+u5J+CxjkDiaTWZgP3AX8DPA04EBh8JBHgw8B2wC7A44GZAFX1BuC/eXDV5SMjvN/LgNOBrYBTV3L/oZ7fXTusJDsBXwWOBCYD36P3DX/jvm6vBg4CngA8FZhRVXcCLwRuHmb1aGi9wzkAeFJX+3tG8hhrhO/f14Cb6L3/hwD/luR5fedf2vXZCjgD+FT3PjwZ+B/A3lW1BfACYNHKatL4ZiCRNJa+3a0c/CnJt5NsC7wIOLKq7qyq3wMfA/4eoKp+WVU/qKp7q2ox8P8Dz13DGn5aVd+uqvuBR67o/sOYBNyygrGnA2d1Nf8V+CjwCGC/vj6fqKqbq+oPwHeBaSOtt6ruXk6fY7r6rwROBl6zkjFXKsnjgWcC76mqe6pqPvB54B/6ul1YVd/r9px8Gdija18KbALsmmSjqlpUVb9a05q0fvN5pKSx9PKqOm/wRZJ9gI2AW5IMNm9Ab48GXWD5OPBsens0NgD+uIY13Nh3vMOK7j+MJcBjVzD2dsANgy+q6v4kNwLb9/X5bd/xXd01I613JH1uAJ4ygmtWZjvgD1V1+5CxB/peD53LxCQbVtUvkxxJbzVrtyTnAO9svWdI6zZXSCS1dCNwL7BNVW3VfTyyqnbrzv8bUPQ2kD4SeD29xziDhv7nyu8ENh180e0FmTykT/81K7v/UOcBr1zBfG6mF3IG7x96j5l+s4JrhqtrJO39Ht93PKWrA4a8H8BjVmHsm4FHJenfrDuFkc2FqvpKVT2L3vtRwPEjuU7jl4FEUjPdxtBzgf+b5JFJNug2sg4+ltkCuAO4Lcn2wLuHDPE7YMe+1z+n91P6wUk2Ao6m9+hgde8/1AeA/ZKckOQxAEn+ptt0uhXwn8DBSf62u///pBd4fjKCt+N3wKQkW46g71D/mmTTJLsBbwJO69rnAy9K8qiu3iOHueeODKOqbuzq/nC38fapwFuAoRtqHyLJk5M8L8kmwD3A3cD9qz4tjScGEkmt/QOwMXANvccxp/PgY5FjgD2B24CzgG8OufbDwNHdnpR3VdVtwNvo7XX4Db0VgpX9bY8V3X8Z3T6IZwBTgauT3AZ8A5gL3F5V19NbxfkkcCvwEnqbRv+ysjehqq6jtyF2YTefVfkbLT8Gfgn8EPhoVZ3btX8ZuILehtJzeTCoDFrm/Rtm3Nd0c70Z+Bbwgf5HbiuwCXAcvffgt8Cjgfeuwnw0DqVqJKuBkiRJa48rJJIkqTkDiSRJas5AIkmSmjOQSJKk5vzDaFqrttlmm5o6dWrrMjRCS5YsAWDSpEmNK5G0Ppo3b96tVTX0bwMBBhKtZVOnTmXu3Lmty9AIzZ49G4AZM2Y0rUPS+inJDcs75yMbSZLUnIFEkiQ1ZyCRJEnNuYdEY+6vf/0rN910E/fcc0/rUtRn4sSJTJgwgaVLl7YuRdI4ZCDRmLvpppvYYostmDp1Kn3/yXc1VFUsWbKEHXbYgYULF7YuR9I45CMbjbl77rmHSZMmGUbWIUmYNGkSj3jEI1qXImmcMpCoCcPIuseviaSWDCSSJKk595CoualHnTWq4y067uAR9fvtb3/LkUceyaWXXspWW23Ftttuy4knnshOO+20Svc78cQTOeyww9h0002HPf/Wt76Vd77zney6665svvnm3HHHHSMee/78+dx888286EUvAuCMM87gmmuu4aijjlqlGiVpXecKicalquIVr3gF+++/P7/61a+YN28eH/7wh/nd7363ymOdeOKJ3HXXXcOeW7p0KZ///OfZddddV6vO+fPn873vfe+B1y996UsNI5LWS66QaFw6//zz2WijjTj88MMfaNtjjz2oKt797nfz/e9/nyQcffTRTJ8+nTlz5jBz5ky22WYbrrrqKvbaay9OOeUUPvnJT3LzzTdzwAEHsM0223D++eez+eab84//+I+cd955fPrTn+boo4/mox/9KAMDAwD8y7/8C+eeey6Pecxj+NrXvsbkyZPZf//9H+hz6623MjAwwM9//nPe//73c/fdd3PhhRfy3ve+l7vvvpu5c+fyqU99ikWLFvHmN7+ZW2+9lcmTJ3PyySczZcoUZsyYwSMf+Ujmzp3Lb3/7Wz7ykY9wyCGHcMsttzB9+nT+/Oc/c9999/HZz36WZz/72Q95by5euISZo7xqJenhbaQrz2vCFRKNS4OhYqhvfvObzJ8/nyuuuILzzjuPd7/73dxyyy0AXH755Zx44olcc801LFy4kIsuuogjjjiC7bbbjvPPP5/zzz8fgDvvvJOnP/3pXHHFFTzrWc9aZvw777yTgYEBrr76ap773OdyzDHHLLfGjTfemA9+8INMnz6d+fPnM3369GXOv+Md7+CNb3wjCxYs4HWvex1HHHHEA+duueUWLrzwQs4888wHVlS+8pWv8IIXvOCB+U2bNm213jtJWhsMJFKfCy+8kNe85jVMmDCBbbfdluc+97lceumlAOyzzz487nGPY4MNNmDatGksWrRo2DEmTJjAK1/5ymHPbbDBBg8Ei9e//vVceOGFq13rT3/6U1772tcC8IY3vGGZsV7+8pezwQYbsOuuuz7wGGrvvffm5JNPZubMmVx55ZVsscUWq31vSRptBhKNS7vtthvz5s1bpWs22WSTB44nTJjAfffdN2y/wb94OhKDv2q74YYbcv/99wOMyl+w7a+1qgB4znOewwUXXMD222/PjBkz+NKXvrTG95Gk0WIg0bj0vOc9j3vvvZdZs2Y90LZgwQK22morTjvtNJYuXcrixYu54IIL2GeffVY41hZbbMHtt98+ovvef//9nH766UDvEcrgI52pU6c+EJAGz69s7P3224+vfe1rAJx66qnD7gfpd8MNN7Dtttty6KGH8ta3vpXLLrtsRDVL0lhwU6uaG4vNUkMl4Vvf+hZHHnkkxx9/PBMnTmTq1KmceOKJ3HHHHeyxxx4k4SMf+QiPecxjuO6665Y71mGHHcZBBx30wF6SFdlss8245JJLOPbYY3n0ox/NaaedBsC73vUuXv3qVzNr1iwOPvjB9+OAAw7guOOOY9q0abz3ve9dZqxPfvKTvOlNb+KEE054YFPrisyZM4cTTjiBjTbaiM0339wVEknrlAwu50prw8DAQM2dO3eZtmuvvZZddtmlUUVakUsvvZT/+O4FnP2XnVuXImkdMlo/OCaZV1UDw53zkY0kSWrOQCJJkpozkKwHksxOckjrOlaFjwrXPX5NJLVkINGYmzhxIkuWLPEb4DqkqliyZAl3331361IkjVP+ls3DTJJ/BV4PLAZuBOYNOb8IGKiqW5MMAB+tqv2TbA58EhgACjimqr6R5DXA/wYCnFVV70kyAfiPvr5fqKqPJXki8GlgMnAXcGhVLf/XT5bjcY97HDfddBOLFy9enbdAa8nEiRO54YYbWpchaZwykDyMJNkbeCWwB7ARcBlDAskK/CtwW1U9pRtr6yTbAccDewF/BM5N8nJ6QWf7qtq967tVN8Ys4PCq+kWSpwOfAZ43TJ2HAYcBTJky5SGFbLTRRjzhCU8YYdkaSz/+8Y9blyBpnPKRzcPLM4HvVNU9VXU78N1VuPb59FY3AKiqPwJ7A3OqanFV3QecCjwHWAjsmOSTSQ4C/tytsOwHfD3JfOBzwGOHu1FVzaqqgaoamDx58qrPUpI07rhCsv65jweD5sTVGaCq/phkD+AFwOHAq4EjgT9V1bRRqFGSpGW4QvLwchHwkiQTuxWLFw/TZxG9RzDQe7wz6AfA2wdfJNkauAR4bpJtun0jrwF+nGQbYIOq+gZwNLBnVf0Z+HWSV3XXpwstkiStMQPJw0hVXQqcASwAvg9cCdw2pNsxwMeTzAWW9rUfC2yd5KokVwAHVNUtwFHA+cAVwLyq+g6wPTCnezRzCjD4N8tfB7ylu/5q4GWjP0tJ0njkI5uHn49W1cwkmwIX0AsR/z54sqr+C9hp6EVVdQfwxmHavwp8dUjbFcCew/T9NXDQGs9AkqQhDCQPP7OS7Epvf8gXq8r/ZKsk6WHPQPIwU1WvbV2DJEmjzT0kkiSpOQOJJElqzkAiSZKaM5BIkqTm3NQqaRn77jiJk2Yc3LoMSeOMKySSJKk5A4kkSWrOQCJJkpozkEiSpOYMJJIkqTkDiSRJas5AIkmSmjOQSJKk5gwkkiSpOQOJJElqzkAiSZKaM5BIkqTmDCSSJKk5A4kkSWrOQCJJkpozkEiSpOYMJJIkqTkDiSRJas5AIkmSmjOQSJKk5gwkkiSpOQOJJElqzkAiSZKa27B1AZLWLRcvXMLMo85qXYa0UouOO7h1CRpFrpBIkqTmDCSSJKk5A4kkSWrOQCJJkpozkEiSpOYMJJIkqTkDiSRJas5AIkmSmjOQjGNJpia5ajWv3S7J6aNdkyRpfPIvtWq1VNXNwCGt65AkrR9cIdGGSU5Ncm2S05NsmmRRkg8nmZ9kbpI9k5yT5FdJDoc1W12RJGkoA4meDHymqnYB/gy8rWv/76qaBvwXMJveasi+wDErGzDJYV2Qmbt48eK1UrQkaf1iINGNVXVRd3wK8Kzu+Izu85XAz6rq9qpaDNybZKsVDVhVs6pqoKoGJk+evFaKliStXwwkquW8vrf7fH/f8eBr9x5JkkaVgURTkjyjO34tcGHLYiRJ45OBRNcDb09yLbA18NnG9UiSxiGX3sexqloE7DzMqal9fWbT29Q6+Hrw3K3A7murNknS+OIKiSRJas5AIkmSmjOQSJKk5gwkkiSpOQOJJElqzkAiSZKa89d+JS1j3x0ncdKMg1uXIWmccYVEkiQ1ZyCRJEnNGUgkSVJzBhJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBRJIkNWcgkSRJzRlIJElScwYSSZLUnIFEkiQ1ZyCRJEnNGUgkSVJzBhJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBRJIkNWcgkSRJzW3YugBpJKYedVbrEsaFgzZewr47TmpdhqRxyBUSSZLUnIFEkiQ1ZyCRJEnNGUgkSVJzBhJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBZAWSTE1y1TDtc5IMjML4M5J8ak3HkSTp4c5A8jCXZMz+/P9Y3kuSNL4YSFZuwySnJrk2yelJNu0/meQ1Sa5MclWS40fQ/qYkP09yCfDMFd04yewkJyWZ213z4q59RpIzkvwI+GGSzZJ8IcklSS5P8rKu325d2/wkC5I8qet7VpIrutqmd30XJdmmOx5IMqc7npnky0kuAr6cZHKSbyS5tPt4yBySHNbVPHfx4sWr965LksYVf+JduScDb6mqi5J8AXjb4Ikk2wHHA3sBfwTOTfJy4JLltP8MOKZrvw04H7h8JfefCuwDPBE4P8nfdO17Ak+tqj8k+TfgR1X15iRbAZckOQ84HPh4VZ2aZGNgAvAi4OaqOribw5YjeA92BZ5VVXcn+Qrwsaq6MMkU4Bxgl/7OVTULmAUwMDBQIxhfkjTOGUhW7saquqg7PgU4ou/c3sCcqloMkORU4DlALaedIe2nATut5P7/WVX3A79IshDYuWv/QVX9oTs+EHhpknd1rycCU4CfAu9L8jjgm1X1iyRXAv+3W7U5s6r+awTvwRlVdXd3/Hxg1ySD5x6ZZPOqumME40iSNCwDycoN/Ql/rH/iX9797+xrC/DKqrp+SN9rk/wMOBj4XpJ/rKofJdmT3krJsUl+WFUfBO7jwUd4E4eM03+vDYB9q+qe1ZyPJEkP4R6SlZuS5Bnd8WuBC/vOXQI8N8k2SSYArwF+vIL2n3Xtk5JsBLxqBPd/VZINkjwR2BEYGjqg99jkHemWLZI8rfu8I7Cwqj4BfAd4aveY6a6qOgU4gd6jH4BF9B4lAbxyBfWcC7xj8EWSaSOYgyRJK2QgWbnrgbcnuRbYGvjs4ImqugU4it5ekCuAeVX1nZW0z6T3KOUi4NoR3P+/6QWc7wOHL2dl4v8AGwELklzdvQZ4NXBVkvnA7sCXgKfQ22MyH/gAcGzX9xjg40nmAktXUM8RwEC3SfYaevtUJElaI6lyz+G6Kslsevs8Tm9dy+oaGBiouXPnrvE4U486axSq0coctPF17LvjJGbMmNG6FEnroSTzqmrYv+PlCokkSWrOTa3rgCTv46H7Sb5eVTMalCNJ0pgzkKwDqupDwIda1yFJUis+spEkSc0ZSCRJUnM+stHDwqLjDm5dwrgwe7b/7SFJbbhCIkmSmjOQSJKk5gwkkiSpOQOJJElqzkAiSZKaM5BIkqTmDCSSJKk5A4kkSWrOQCJJkpozkEiSpOYMJJIkqTkDiSRJas5AIkmSmjOQSJKk5gwkkiSpOQOJJElqzkAiSZKaM5BIkqTmDCSSJKk5A4kkSWrOQCJJkpozkEiSpOYMJJIkqbkNWxcgad1y8cIlzDzqrCb3XnTcwU3uK6k9V0gkSVJzBhJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBRJIkNWcgkSRJzRlItIwkP2ldgyRp/DGQNJJkQusahlNV+7WuQZI0/hhI1oIkU5Ncl+TUJNcmOT3JpkkWJTk+yWXAq5IcmOSnSS5L8vUkm3fXv6i7fl6STyQ5s2ufmeQLSeYkWZjkiL57frvrf3WSw/ra70jyoSRXJLk4ybZd+7ZJvtW1X5Fkv8H+fde+O8mlSRYkOaZr2yzJWd01VyWZPiZvqiRpvWYgWXueDHymqnYB/gy8rWtfUlV7AucBRwPP717PBd6ZZCLwOeCFVbUXMHnIuDsDLwD2AT6QZKOu/c1d/wHgiCSTuvbNgIurag/gAuDQrv0TwI+79j2Bq/tvkuRA4EndfaYBeyV5DnAQcHNV7VFVuwNnD514ksOSzE0yd/HixavwlkmSxisDydpzY1Vd1B2fAjyrOz6t+7wvsCtwUZL5wBuBHegFjoVV9euu31eHjHtWVd1bVbcCvwe27dqPSHIFcDHweHphAuAvwJnd8Txganf8POCzAFW1tKpuG3KfA7uPy4HLurqeBFwJ/F230vPsYa6jqmZV1UBVDUyePDRPSZL0UP7XfteeWs7rO7vPAX5QVa/p75Rk2krGvbfveCmwYZL9gecDz6iqu5LMASZ2ff5aVdXff4T1B/hwVX3uISeSPYEXAccm+WFVfXCEY0qSNCxXSNaeKUme0R2/FrhwyPmLgWcm+Rt4YG/GTsD1wI5Jpnb9RrJHY0vgj10Y2Zne6svK/BD4p+7eE5JsOeT8OcCb+/a1bJ/k0Um2A+6qqlOAE+g97pEkaY0YSNae64G3J7kW2Jru8cigqloMzAC+mmQB8FNg56q6m95+k7OTzANuBx7yWGSIs+mtlFwLHEcv7KzMPwMHJLmS3qOcXYfUdy7wFeCnXZ/TgS2ApwCXdI+ZPgAcO4J7SZK0Qj6yWXvuq6rXD2mb2v+iqn4E7D3MtedX1c5JAnya3oZXqmrmkOt373v5wuGKqKrN+45PpxcsqKrfAS9bSf+PAx8f0uVX9FZPJEkaNa6QrJsO7VYgrqb3OOYh+zgkSVqfuEKyFlTVImD3lfVbwfUfAz42agVJkrSOc4VEkiQ1ZyCRJEnNGUgkSVJzBhJJktScm1olLWPfHSdx0oyDW5chaZxxhUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBRJIkNWcgkSRJzRlIJElScwYSSZLUnIFEkiQ1ZyCRJEnNGUgkSVJzBhJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBRJIkNWcgkSRJzRlIJElScwYSSZLUnIFEkiQ1ZyCRJEnNbdi6AEnrlosXLmHmUWeN+X0XHXfwmN9T0rrDFRJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBRJIkNWcgkSRJzRlIJElScwYSSZLUnIFEkiQ1ZyAZ55J8O8m8JFcnOaxre0uSnye5JMm/J/lU1z45yTeSXNp9PLNt9ZKk9YX/LRu9uar+kOQRwKVJzgL+FdgTuB34EXBF1/fjwMeq6sIkU4BzgF2GDtgFm8MApkyZMgZTkCQ93BlIdESSV3THjwfeAPy4qv4AkOTrwE7d+ecDuyYZvPaRSTavqjv6B6yqWcAsgIGBgVrL9UuS1gMGknEsyf70QsYzququJHOA6xhm1aOzAbBvVd0zJgVKksYN95CMb1sCf+zCyM7AvsBmwHOTbJ1kQ+CVff3PBd4x+CLJtLEsVpK0/jKQjG9nAxsmuRY4DrgY+A3wb8AlwEXAIuC2rv8RwECSBUmuAQ4f84olSeslH9mMY1V1L/DCoe1J5lbVrG6F5FvAt7v+twLTx7RISdK44AqJhjMzyXzgKuDXdIFEkqS1xRUSPURVvat1DZKk8cUVEkmS1JyBRJIkNWcgkSRJzRlIJElSc25qlbSMfXecxEkzDm5dhqRxxhUSSZLUnIFEkiQ1ZyCRJEnNGUgkSVJzBhJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBRJIkNWcgkSRJzRlIJElScwYSSZLUnIFEkiQ1ZyCRJEnNGUgkSVJzBhJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBRJIkNWcgkSRJzW3YugBJ65aLFy5h5lFnjcm9Fh138JjcR9K6zxUSSZLUnIFEkiQ1ZyCRJEnNGUgkSVJzBhJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1NwqB5IkM5O8K8kHkzx/JX3nJBlYhbGnJXnRqta0tiSZmuSq7nggySca1fGqJNcmOX8Ux9wqydv6Xm+X5PTRGl+SpFWx2iskVfX+qjpvNIsBpgHrTCDpV1Vzq+qIkfZPz2itQL0FOLSqDhil8QC2Ah4IJFV1c1UdMorjS5I0YiP6hpnkfUl+nuRC4Mld2+wkh3TH709yaZKrksxKkr7L35Bkfndun67/Zkm+kOSSJJcneVmSjYEPAtO7/tOH69ddv1vXNj/JgiRPWk7dU5Ncl+TUboXh9CSbduf2SvLjJPOSnJPksX3tVyS5Anh731j7JzmzO56c5AdJrk7y+SQ3JNmmu9/1Sb4EXAU8Psm7u/dmQZJj+sZ7fd8cPpdkwnLm8H7gWcB/JDkhyYwkn+o7f2aS/bvjO5J8qKv/4iTbdu3bJvnW4LyS7AccBzyxu/8JQ1aDJiY5OcmV3ft+QNc+I8k3k5yd5BdJPrKcmg9LMjfJ3MWLFw/XRZKkZaw0kCTZC/h7Hly92HuYbp+qqr2ranfgEcCL+85tWlXT6P00/oWu7X3Aj6pqH+AA4ARgI+D9wGlVNa2qThuuX5LNgMOBj3fjDgA3rWAKTwY+U1W7AH8G3pZkI+CTwCFVtVdX14e6/icD76iqPVYw5ge6unYDTgem9J17Une/3bp7PwnYh977t1eS5yTZBZgOPLObw1LgdcPdqKo+CMwFXldV715BTQCbARd3tV8AHNq1fwL4cde+J3A1cBTwq+69Hjru23u3rqcArwG+mGRid25aV/tT6IXHxw9T86yqGqiqgcmTJ6+kZEmSRvZf+3028K2qugsgyRnD9Dkgyf8CNgUeRe8b3ne7c18FqKoLkjwyyVbAgcBLk7yr6zORZb+pD1pev58C70vyOOCbVfWLFdR/Y1Vd1B2fAhwBnA3sDvygW8yZANzS1bZVVV3Q9f8y8MJhxnwW8IpuXmcn+WPfuRuq6uK++g8ELu9eb04voDwV2Au4tLv/I4Dfr2AOI/UX4MzueB7wd93x84B/6OpdCtyWZOsVjPMseoGNqrouyQ3ATt25H1bVbQBJrgF2AG4chdolSePYSALJCnU/OX8GGKiqG5PMpBccBtWQSwoI8Mqqun7IWE8fOvxw/YBrk/wMOBj4XpJ/rKofLafE5d3/6qp6xpD7b7WcMVbFnf1DAh+uqs8Nuc87gC9W1XtXY/z7WHZlq/+9/mtVDc53KaPw9R3GvX3Ha+sekqRxZiR7SC4AXp7kEUm2AF4y5PzgN8Rbk2wODN0YOR0gybOA27qfrs8B3pFueSDJ07q+twNb9F07bL8kOwILq+oTwHforTgsz5Qkg8HjtcCFwPXA5MH2JBsl2a2q/gT8qasVlvMYBbgIeHV37YHA8lYbzgHe3L0vJNk+yaOBHwKHdMckeVSSHVYwh36LgGlJNugel+wzgmt+CPxTd68JSbbkoe91v/+im3uSneitSg0NhZIkjZqVBpKqugw4DbgC+D5w6ZDzfwL+nd4mznOGngfuSXI5cBK93xYB+D/09owsSHJ19xrgfGDXbqPl9BX0ezVwVZL59B69fGkFU7geeHuSa+kFh89W1V/oBafju82r84H9uv5vAj7djZ2HDgfAMcCB3SbQVwG/pfcNfhlVdS7wFeCnSa6kt99ki6q6BjgaODfJAuAHwGNXMId+FwG/Bq6htzfkshFc88/0HqtdSe9Rzq5VtQS4KL3NxicM6f8ZYIOu/2nAjKq6F0mS1pI8uMK//kkyFTiz22w7muNuAiytqvu6VZbPdptTNcTAwEDNnTu3dRkaodmzZ3PxwiWc/Zedx+R+i447eEzuI2ndkGReVQ3798l8/r96pgD/md7fGfkLD/42iyRJWg3rRSBJMonePomh/na0V0cAut/qedpKO66ibqPuJkOa31BVV472vSRJWpesF4Gk2w8xrXUda6qqhv6WkSRJ44L/cT1JktScgUSSJDW3XjyykTR69t1xEifN8LdfJI0tV0gkSVJzBhJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBRJIkNWcgkSRJzRlIJElScwYSSZLUnIFEkiQ1ZyCRJEnNGUgkSVJzBhJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBRJIkNWcgkSRJzRlIJElScwYSSZLUnIFEkiQ1t2HrAiSNnalHnbXC8wdtvIR9d5w0RtVI0oNcIZEkSc0ZSCRJUnMGEkmS1JyBRJIkNWcgkSRJzRlIJElScwYSSZLUnIFkPZDkJ6M83tQkV3XHA0k+MZrjS5I0lH8YbT1QVfutxbHnAnPX1viSJIErJOuFJHd0n/dPMifJ6UmuS3JqknTnjktyTZIFST7atc1OcsjQcYaMvX+SM7vjmUm+0N1jYZIjxmaGkqT1nSsk65+nAbsBNwMXAc9Mci3wCmDnqqokW63B+DsDBwBbANcn+WxV/bW/Q5LDgMMApkyZsga3kiSNF66QrH8uqaqbqup+YD4wFbgNuAf4jyT/H3DXGox/VlXdW1W3Ar8Hth3aoapmVdVAVQ1Mnjx5DW4lSRovDCTrn3v7jpcCG1bVfcA+wOnAi4Gzu/P30f0bSLIBsPHqjL+mBUuSZCAZB5JsDmxZVd8D/gXYozu1CNirO34psNHYVydJkj/djhdbAN9JMhEI8M6u/d+79ivorZrc2ag+SdI4ZyBZD1TV5t3nOcCcvvb/0ddtn2Gu+x2wb1/Te7r2RcDuQ8esqplDrt99TWuXJAl8ZCNJktYBBhJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1Jy/9iuNI4uOO3iF52fPXjxGlUjSslwhkSRJzRlIJElScwYSSZLUnIFEkiQ1ZyCRJEnNGUgkSVJzBhJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBRJIkNWcgkSRJzRlIJElScwYSSZLUnIFEkiQ1ZyCRJEnNGUgkSVJzBhJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBRNIyLl64pHUJksYhA4kkSWrOQCJJkpozkEiSpOYMJJIkqTkDiSRJas5AIkmSmjOQSJKk5gwkkiSpOQOJJElqzkCympIsSrLNMO0/Wc3xjkyyad/rO5bT7/Ak/7CCcfZPcubq1CBJUisbti5gfVNV+63mpUcCpwB3rWT8k1ZzfEmS1lmukIxAks2SnJXkiiRXJZned+4RSb6f5NDu9R3d5/2TzElyepLrkpyaJMsZ/whgO+D8JOf3tX+ou+fFSbbt2mYmeVd3/DdJzuv6XJbkiUPG3TvJ5Ume2F33ha6mhd09B/u9PsklSeYn+VySCd3H7G6+Vyb5l8Fak1yTZEGSry1nPoclmZtk7uLFi1fvTZckjSsGkpE5CLi5qvaoqt2Bs7v2zYHvAl+tqn8f5rqn0Vv52BXYEXjmcINX1SeAm4EDquqArnkz4OKq2gO4ADh0mEtPBT7d9dkPuGXwRJL9gJOAl1XVr7rmnYEXAPsAH0iyUZJdgOnAM6tqGrAUeB0wDdi+qnavqqcAJ3djHAU8raqeChy+nPnMqqqBqhqYPHnycF0kSVqGgWRkrgT+LsnxSZ5dVbd17d8BTq6qLy3nukuq6qaquh+YD0xdhXv+BRjcCzJv6LVJtqAXGL4FUFX3VNXg455dgFnAS6rqv/suO6uq7q2qW4HfA9sCfwvsBVyaZH73ekdgIbBjkk8mOQj4czfGAuDUJK8H7luF+UiStFwGkhGoqp8De9ILJscmeX936iLgoOU9igHu7Tteyqrt2flrVdVqXnsLcA+9FZqV1RPgi1U1rft4clXNrKo/AnsAc+ithHy+u+5g4NP03o9Lk7gPSZK0xgwkI5BkO+CuqjoFOIHeN2OA9wN/pPcNek3dDmwx0s5VdTtwU5KXdzVu0vdbOn+iFxw+nGT/lQz1Q+CQJI/uxnlUkh263yDaoKq+ARwN7JlkA+DxVXU+8B5gS3qPrSRJWiMGkpF5CnBJ90jjA8Cxfef+GXhEko+s4T1mAWf3b2odgTcARyRZAPwEeMzgiar6HfBi4NNJnr68AarqGnqB49xunB8AjwW2B+Z0cz4FeC8wATglyZXA5cAnqupPq1CvJEnDyoNPBaTRNzAwUHPnzm1dhkZo9uzZXLxwCSd98H+2LkXSeijJvKoaGO6cKySSJKk5NySOsSTfAp4wpPk9VXVOi3okSVoXGEjGWFW9onUNkiSta3xkI0mSmjOQSJKk5gwkkpax746TWpcgaRwykEiSpOYMJJIkqTkDiSRJas5AIkmSmjOQSJKk5gwkkiSpOQOJJElqzkAiSZKaM5BIkqTmDCSSJKk5A4kkSWouVdW6Bq3HkiwGbmhdxxraBri1dRFjaLzNF8bfnJ3v+m9dnfMOVTV5uBMGEmklksytqoHWdYyV8TZfGH9zdr7rv4fjnH1kI0mSmjOQSJKk5gwk0srNal3AGBtv84XxN2fnu/572M3ZPSSSJKk5V0gkSVJzBhJJktScgUQCkjwqyQ+S/KL7vPVy+r2x6/OLJG/sa5+T5Pok87uPR49d9SOX5KCuzl8mOWqY85skOa07/7MkU/vOvbdrvz7JC8a08NW0uvNNMjXJ3X1fz5PGvPjVNII5PyfJZUnuS3LIkHPD/vtel63hfJf2fY3PGLuqV98I5vvOJNckWZDkh0l26Du3bn99q8oPP8b9B/AR4Kju+Cjg+GH6PApY2H3eujveujs3BxhoPY+VzHEC8CtgR2Bj4Apg1yF93gac1B3/PXBad7xr138T4AndOBNaz2ktzncqcFXrOaylOU8Fngp8CTikr325/77X1Y81mW937o7Wc1gL8z0A2LQ7/qe+f9Pr/NfXFRKp52XAF7vjLwIvH6bPC4AfVNUfquqPwA+Ag8amvFGxD/DLqlpYVX8BvkZv3v3634fTgb9Nkq79a1V1b1X9GvhlN966bE3m+3C10jlX1aKqWgDcP+Tah+O/7zWZ78PRSOZ7flXd1b28GHhcd7zOf30NJFLPtlV1S3f8W2DbYfpsD9zY9/qmrm3Qyd3S77+uo9/UVlb/Mn2q6j7gNmDSCK9d16zJfAGekOTyJD9O8uy1XewoWZOv0/r6NV6RiUnmJrk4yctHtbK1Y1Xn+xbg+6t57ZjbsHUB0lhJch7wmGFOva//RVVVklX9ffjXVdVvkmwBfAN4A70lYj083QJMqaolSfYCvp1kt6r6c+vCNKp26P53uyPwoyRXVtWvWhc1GpK8HhgAntu6lpFyhUTjRlU9v6p2H+bjO8DvkjwWoPv8+2GG+A3w+L7Xj+vaqKrBz7cDX2HdfJyx3PqH65NkQ2BLYMkIr13XrPZ8u0dTSwCqah695/Y7rfWK19yafJ3W16/xcvX973YhvX1gTxvN4taCEc03yfPp/aD10qq6d1WubclAIvWcAQzuOn8j8J1h+pwDHJhk6+63cA4EzkmyYZJtAJJsBLwYuGoMal5VlwJPSvKEJBvT28Q59DcL+t+HQ4AfVW9H3BnA33e/lfIE4EnAJWNU9+pa7fkmmZxkAkD30/OT6G0CXNeNZM7LM+y/77VU52hZ7fl289ykO94GeCZwzVqrdHSsdL5JngZ8jl4Y6f/Bat3/+rbeVeuHH+vCB719Az8EfgGcBzyqax8APt/X7830NnT+EnhT17YZMA9YAFwNfJx19DdQgBcBP6f3E//7urYP0vs/L4CJwNe7+V0C7Nh37fu6664HXth6LmtzvsAru6/lfOAy4CWt5zKKc96b3v6BO+mtfl3dd+1D/n2v6x+rO19gP+BKer+pciXwltZzGaX5ngf8rvu3Ox844+Hy9fVPx0uSpOZ8ZCNJkpozkEiSpOYMJJIkqTkDiSRJas5AIkmSmjOQSJKk5gwkkiSpuf8HWT31Y5PbAigAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 648x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "final_plot_contrib_df = plot_contrib_df.iloc[plot_contrib_df['Contributions'].abs().argsort()]\n",
    "final_plot_contrib_df.plot(kind='barh', figsize=(9, 7))\n",
    "plt.title('Feature Contributions')\n",
    "plt.axvline(x=0, color='.5')\n",
    "plt.subplots_adjust(left=.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}