baselines.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lime\n",
    "import sklearn\n",
    "import numpy as np\n",
    "import sklearn.ensemble\n",
    "import sklearn.metrics\n",
    "import sklearn.model_selection\n",
    "from lime.lime_tabular import LimeTabularExplainer\n",
    "import pickle\n",
    "import os\n",
    "import pandas as pd\n",
    "import time\n",
    "from sklearn.metrics.pairwise import rbf_kernel\n",
    "import shap\n",
    "\n",
    "\n",
    "from MMD_critic import mmd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_directory = os.path.abspath(os.path.join(os.curdir, os.pardir, \"ADHF\"))\n",
    "data_dir = os.path.join(base_directory, \"data\")\n",
    "\n",
    "df_patients = pd.read_csv(os.path.join(data_dir, \"CHF_data_2015_normalized_z_score_bun_temp_k_na_hb.csv\"),\n",
    "                          header=0, thousands=',',low_memory=False, index_col=0)\n",
    "\n",
    "y = pd.read_csv(os.path.join(data_dir, \"mesurements_for_pred_with_index_2015_z_score.csv\"), index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_label = '1y_mort'\n",
    "# % matplotlib inline\n",
    "# fig = plt.figure()\n",
    "x_train = df_patients.loc[~df_patients['year'].isin([10,11])]\n",
    "y_train = y.loc[x_train.index,y_label]\n",
    "\n",
    "x_test = df_patients.loc[df_patients['year'].isin([10,11])]\n",
    "y_test = y.loc[x_test.index,y_label]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def x_train_test(cols):\n",
    "# % matplotlib inline\n",
    "# fig = plt.figure()\n",
    "#     cols = cols.unique()\n",
    "    x_train = df_patients.loc[~df_patients['year'].isin([10,11]),cols]\n",
    "    x_test = df_patients.loc[df_patients['year'].isin([10,11]),cols]\n",
    "    return x_train, x_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_colls_name = 'indices_disease_drug_admin_personal'\n",
    "four_best_colls_name = 'indices_disease_admin_personal'\n",
    "three_best_colls_name = 'indices_admin_personal'\n",
    "\n",
    "# type_list = [full_colls_name, four_best_colls_name, three_best_colls_name]\n",
    "type_list = [full_colls_name]\n",
    "models_dir = os.path.join(base_directory, \"models\")\n",
    "norm_type = 'z_score'\n",
    "norm_dir = os.path.join(models_dir, norm_type)\n",
    "\n",
    "fpr_tpr_list = []\n",
    "predictors = []\n",
    "prob_pos_list = []\n",
    "y_pred_list = []\n",
    "for type_ in type_list:\n",
    "    df_colls = pd.read_csv(os.path.join(norm_dir, \"coef_1_{}.csv\".format(type_)), index_col=0)\n",
    "    x_train, x_test = x_train_test(df_colls['0'])\n",
    "    logit_reg = pickle.load(open(os.path.join(norm_dir, \"1_{}.pkl\".format(type_)),'rb'))\n",
    "    \n",
    "    # for callibration\n",
    "    prob_pos = logit_reg.predict_proba(x_test)[:, 1]\n",
    "    y_pred = logit_reg.predict(x_test)\n",
    "    prob_pos_list.append(prob_pos)\n",
    "    y_pred_list.append(y_pred)\n",
    "    \n",
    "    # for auc\n",
    "#     fpr, tpr, _ = roc_curve(y_test, prob_pos)\n",
    "#     roc_auc = auc(fpr, tpr)\n",
    "    # save for auc\n",
    "    predictors.append(logit_reg)\n",
    "#     fpr_tpr_list.append((fpr, tpr, roc_auc))    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Todo:\n",
    "* SP-LIME + LIME\n",
    "* SHAP + SP-LIME\n",
    "* LIME + MMD-critic\n",
    "* SHAP + MMD-critic\n",
    "* LIME + $\\alpha, \\beta$- mistakes\n",
    "* All of the above in intervals of condifdence : \n",
    "    * 0-0.3\n",
    "    * 0.3-0.5\n",
    "    * 0.5- 0.8\n",
    "    * 0.8 - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logit_reg = predictors[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_coef = pd.read_csv(os.path.join(models_dir, \"coef_1_indices_disease_admin_personal.csv\"), index_col=0)\n",
    "\n",
    "four_best_cols = model_coef['0']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x_train, x_test = x_train_test(full_colls_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def interpret_data(X, y_, func, indexs, feature_names=None):\n",
    "    if not feature_names:\n",
    "        feature_names=X.columns.tolist()\n",
    "    explainer = LimeTabularExplainer(X, discretize_continuous=False, feature_names=feature_names, class_names=['alive', 'dead'], random_state=24)\n",
    "    explanations = []\n",
    "    for i in indexs:\n",
    "        explanations.append(explainer.explain_instance(X.values[i, :], func, num_features=10))\n",
    "    return explanations, explainer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wrong_predicted = np.where(logit_reg.predict(x_test) != y_test)[0]\n",
    "\n",
    "# y_test.head(16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# explanation = explainer.explain_instance(X[r_idx, :], func)\n",
    "\n",
    "i = [490]\n",
    "print(\"the patient is: {}\".format(\"alive\" if y_test.values[i[0]] == 0 else \"dead\"))\n",
    "\n",
    "y_hat = logit_reg.predict_proba(x_test)\n",
    "\n",
    "exp, explainer = interpret_data(x_test, y_hat, logit_reg.predict_proba, i, )\n",
    "exp[2].show_in_notebook(show_table=True,)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kernal = rbf_kernel(x_train, gamma=0.5)\n",
    "m = 200\n",
    "k = 40\n",
    "\n",
    "selected = mmd.greedy_select_protos(kernal, np.array(range(np.shape(kernal)[0])), m)\n",
    "\n",
    "# select_crit = mmd.select_criticism_regularized(kernal, selected, k, is_K_sparse=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "times, scores = interpret_data(x_test, y_hat, logit_reg.predict_proba)\n",
    "print('%9.4fs %9.4fs %9.4fs' % (min(times), sum(times) / len(times), max(times)))\n",
    "print('%9.4f %9.4f% 9.4f' % (min(scores), sum(scores) / len(scores), max(scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x_train.iloc[selected]\n",
    "\n",
    "# selected\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "intersect = set(select_crit).intersection(set(wrong_predicted))\n",
    "print(intersect)\n",
    "\n",
    "print(\"selected:{}, wrong_predicted:{}, diff:{}\".format(len(select_crit), len(wrong_predicted), len(intersect)))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "shap.initjs()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "explainer = shap.KernelExplainer(logit_reg.predict_proba, x_train.iloc[selected], link='logit')\n",
    "shap_values = explainer.shap_values(x_test[:2])\n",
    "shap.summary_plot(shap_values, x_test.iloc[:0,:], class_names=['Dead', 'Alive'], show=False)\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"test.png\", bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logit_reg.predict_proba(x_test.values)[:, 1][:2]\n",
    "y_test[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "from lime import submodular_pick\n",
    "sp_obj = submodular_pick.SubmodularPick(explainer, x_train.values, logit_reg.predict_proba, sample_size=100, num_features=14,num_exps_desired=6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# todo2:\n",
    "* 2 proto, 2 critic (per interval)\n",
    "* 4 sp-lime (per interval)\n",
    "* 1 Type I & II (per interval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranges = []\n",
    "\n",
    "# 1 - 0 - 0.3\n",
    "ranges.append(np.where(logit_reg.predict_proba(x_test)[:, 1] < 0.3)[0])\n",
    "\n",
    "# 2 - 0.3 - 0.5\n",
    "ranges.append(np.where((logit_reg.predict_proba(x_test)[:, 1] < 0.5) & (logit_reg.predict_proba(x_test)[:, 1] >= 0.3))[0])\n",
    "\n",
    "# 3 - 0.5 - 0.8\n",
    "ranges.append(np.where((logit_reg.predict_proba(x_test)[:, 1] < 0.8) & (logit_reg.predict_proba(x_test)[:, 1] >= 0.5))[0])\n",
    "\n",
    "# 4 - 0.8 - 1\n",
    "ranges.append(np.where((logit_reg.predict_proba(x_test)[:, 1] <= 1) & (logit_reg.predict_proba(x_test)[:, 1] >= 0.8))[0])\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = 2\n",
    "k = 2\n",
    "selected_list = []\n",
    "selected_crit = []\n",
    "for range_ in ranges:\n",
    "    kernal = rbf_kernel(x_test.values[range_], gamma=0.1)\n",
    "    selected = mmd.greedy_select_protos(kernal, np.array(range(np.shape(kernal)[0])), m)\n",
    "    selected_list.append(selected)\n",
    "    selected_crit.append(mmd.select_criticism_regularized(kernal, selected, k, is_K_sparse=False))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "range_selection = zip(selected_list, selected_crit)\n",
    "y_hat = logit_reg.predict_proba(x_test)\n",
    "types = {0: 'prototype',1: 'prototype', 2: 'critic', 3: 'critic'}\n",
    "ranges_type = {0: '[0, 0.3)', 1: '[0.3, 0.5)', 2: '[0.5, 0.8)', 3: '[0.8, 1]'}\n",
    "\n",
    "for idx, selection in enumerate(range_selection):\n",
    "    print(\"for the range of {}:\".format(ranges_type[idx]))\n",
    "    selected = np.hstack(selection)\n",
    "    selected = [ranges[idx][i] for i in selected]\n",
    "    explenation, explainer = interpret_data(x_test, y_hat, logit_reg.predict_proba, selected)\n",
    "    for i in range(4):\n",
    "        print(\"the {} #{} patient is: {}\".format(types[i], selected[i]\n",
    "                                                 , \"alive\" if y_test.values[selected[idx]] == 0 else \"dead\"))\n",
    "        explenation[i].show_in_notebook(show_table=True,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_wine\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "wine = load_wine()\n",
    "\n",
    "data_locs = np.where(np.array(wine['target']) != 2)\n",
    "\n",
    "x = wine['data'][data_locs]\n",
    "y = wine['target'][data_locs]\n",
    "\n",
    "wine_logit_reg = LogisticRegression()\n",
    "\n",
    "wine_logit_reg.fit(x,y)\n",
    "wine_logit_reg.predict(x)\n",
    "\n",
    "wine_logit_reg.score(x,y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def interpret_data2(X, y_, func, indexs, feature_names=None):\n",
    "    if not feature_names:\n",
    "        feature_names=X.columns.tolist()\n",
    "    explainer = LimeTabularExplainer(X, discretize_continuous=False, feature_names=feature_names, class_names=['Class 1', 'Class 2'], random_state=24)\n",
    "    explanations = []\n",
    "    explanations.append(explainer.explain_instance(X[indexs], func, num_features=10))\n",
    "    return explanations, explainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "y_hat = wine_logit_reg.predict_proba(x)\n",
    "\n",
    "explenation, explainer = interpret_data2(x, y_hat, wine_logit_reg.predict_proba, 4, wine['feature_names'])\n",
    "print(\"the #{} wine is in class {}\".format(4 , y[4]+1))\n",
    "explenation[0].show_in_notebook(show_table=True,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_hat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # [np.hstack(i) for i in zip(selected_list, selected_crit)]\n",
    "# range_selection = zip(selected_list, selected_crit)\n",
    "\n",
    "# for idx, selection in enumerate(range_selection):\n",
    "#     print(\"for the range of {}:\".format(ranges_type[idx]))\n",
    "#     selected = np.hstack(selection)\n",
    "#     print([logit_reg.predict_proba(x_test.values[ranges[idx][x]].reshape(1,-1))[0][1] for x in selected])\n",
    "\n",
    "\n",
    "# logit_reg.predict_proba(x_test.values[ranges[1]])\n",
    "# # # np.vstack(selected_list, selected_crit)\n",
    "\n",
    "explainer = LimeTabularExplainer(x_test, discretize_continuous=False, feature_names=x_test.columns.tolist(), class_names=['alive', 'dead'], random_state=24)\n",
    "#     times, scores = [], []\n",
    "explanation = explainer.explain_instance(x_test.values[1847, :], logit_reg.predict_proba, num_features=20)# range\n",
    "\n",
    "with open('bla.html','w') as f:\n",
    "    f.write(explanation.as_html(show_predicted_value=True))\n",
    "# explanation.as_list()\n",
    "# logit_reg.predict_proba(x_test.values[0].reshape(1,-1))[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explainer = LimeTabularExplainer(x_test, discretize_continuous=False, feature_names=x_test.columns.tolist(), class_names=['alive', 'dead'], random_state=24)\n",
    "\n",
    "\n",
    "for idx, range_ in enumerate(ranges):\n",
    "    print(\"for the range of {}:\".format(ranges_type[idx]))\n",
    "#     selected = np.hstack(selection)\n",
    "#     selected = [ranges[idx][i] for i in selected]\n",
    "#     explainer = LimeTabularExplainer(x_test, discretize_continuous=False, feature_names=x_test.columns.tolist(), class_names=['alive', 'dead'], random_state=24)\n",
    "\n",
    "#     explenation, explainer = interpret_data(x_test, y_hat, logit_reg.predict_proba, selected)\n",
    "    sp_obj = submodular_pick.SubmodularPick(explainer, x_train.values[range_], logit_reg.predict_proba, sample_size=20, num_features=5,num_exps_desired=4)\n",
    "\n",
    "    for i in range(4):\n",
    "        print(\"the #{} patient is: {}\".format(selected[i]\n",
    "                                                 , \"alive\" if y_test.values[selected[idx]] == 0 else \"dead\"))\n",
    "        sp_obj.sp_explanations[i].show_in_notebook(show_table=True,)\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logit_reg.predict_proba(x_test)[:, 1][491]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# wrong_predicted = np.where(logit_reg.predict(x_test)[ranges[0]] != y_test.values[ranges[0]])[0]\n",
    "figs_dir = os.path.abspath(os.path.join(os.curdir, \"figs\"))\n",
    "\n",
    "type_1_erros = []\n",
    "type_2_erros = []\n",
    "\n",
    "type_1_erros = np.where(logit_reg.predict(x_test) > y_test.values)[0]\n",
    "type_2_erros = np.where(logit_reg.predict(x_test) < y_test.values)[0]\n",
    "correctly_predicted = np.where(logit_reg.predict(x_test) == y_test.values)[0]\n",
    "    \n",
    "kobe_numbers = 2408\n",
    "np.random.seed(kobe_numbers)\n",
    "    \n",
    "num_of_samples = 5\n",
    "y_hat = logit_reg.predict_proba(x_test)\n",
    "\n",
    "types = {0: 'type-I', 1: 'type-II', 2: 'Correctelly predicted'}\n",
    "\n",
    "selected_type_1 = np.random.choice(type_1_erros, num_of_samples, replace=False)\n",
    "selected_type_2 = np.random.choice(type_2_erros, num_of_samples, replace=False)\n",
    "selected_correct = np.random.choice(correctly_predicted, num_of_samples, replace=False)\n",
    "selected = np.hstack([selected_type_1, selected_type_2, selected_correct])\n",
    "\n",
    "explenation, explainer = interpret_data(x_test, y_hat, logit_reg.predict_proba, selected)\n",
    "\n",
    "for i in range(15):\n",
    "    if i % num_of_samples == 0:\n",
    "        print(\"for the type of of {}:\".format(types[i // num_of_samples]))\n",
    "    print(\"the #{} patient is: {}\".format(selected[i] \n",
    "                                          , \"alive\" if y_test.values[selected[i]] == 0 else \"dead\"))\n",
    "    explenation[i].show_in_notebook(show_table=True,)\n",
    "#     explenation[i].save_to_file(file_path=os.path.join(figs_dir, \"\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figs_dir = os.path.abspath(os.path.join(os.curdir, \"figs\"))\n",
    "\n",
    "type_1_erros = np.where(logit_reg.predict(x_test) > y_test.values)[0]\n",
    "type_2_erros = np.where(logit_reg.predict(x_test) < y_test.values)[0]\n",
    "correctly_predicted = np.where(logit_reg.predict(x_test) == y_test.values)[0]\n",
    "    \n",
    "kobe_numbers = 2408\n",
    "np.random.seed(kobe_numbers)\n",
    "    \n",
    "num_of_samples = 2\n",
    "y_hat = logit_reg.predict_proba(x_test)\n",
    "\n",
    "types = {0: 'type-I', 1: 'type-II', 2: 'Correctelly predicted'}\n",
    "\n",
    "selected_type_1 = np.random.choice(type_1_erros, num_of_samples, replace=False)\n",
    "selected_type_2 = np.random.choice(type_2_erros, num_of_samples, replace=False)\n",
    "selected_correct = np.random.choice(correctly_predicted, num_of_samples, replace=False)\n",
    "# explainer_orig = shap.KernelExplainer(logit_reg.predict_proba, x_train.iloc[selected], link='logit')\n",
    "\n",
    "for idx, selected in enumerate([selected_type_1, selected_type_2, selected_correct]):\n",
    "    for s in selected:\n",
    "        shap_values = explainer.shap_values(x_test.iloc[[s]])\n",
    "        shap.summary_plot(shap_values, x_test.iloc[:0,:], class_names=['Dead', 'Alive'], show=False)\n",
    "        plt.title(\"Shap for {}_{}\".format(types[idx], s))\n",
    "        plt.tight_layout()\n",
    "        plt.savefig(\"Shap for {}_{}\".format(types[idx], s), bbox_inches='tight')\n",
    "        plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "range_selection = zip(selected_list, selected_crit)\n",
    "y_hat = logit_reg.predict_proba(x_test)\n",
    "types = {0: 'prototype',1: 'prototype', 2: 'critic', 3: 'critic'}\n",
    "ranges_type = {0: '[0, 0.3)', 1: '[0.3, 0.5)', 2: '[0.5, 0.8)', 3: '[0.8, 1]'}\n",
    "\n",
    "for idx, selection in enumerate(range_selection):\n",
    "    print(\"for the range of {}:\".format(ranges_type[idx]))\n",
    "    selected = np.hstack(selection)\n",
    "    selected = [ranges[idx][i] for i in selected]\n",
    "#     explenation, explainer = interpret_data(x_test, y_hat, logit_reg.predict_proba, selected)\n",
    "    for i in range(4):\n",
    "#         print(\"the {} #{} patient is: {}\".format(types[i], selected[i]\n",
    "#                                                  , \"alive\" if y_test.values[selected[idx]] == 0 else \"dead\"))\n",
    "#         explenation[i].show_in_notebook(show_table=True,)\n",
    "        shap_values = explainer.shap_values(x_test.iloc[[selected[i]]])\n",
    "        shap.summary_plot(shap_values, x_test.iloc[:0,:], class_names=['Dead', 'Alive'], show=False)\n",
    "        plt.title(\"Shap for the range of {}_{}\".format(ranges_type[idx], selected[i]))\n",
    "        plt.tight_layout()\n",
    "        file_name = os.path.join(figs_dir, \"Shap for the range of {}_{}\".format(types[idx], selected[i]))\n",
    "        plt.savefig(file_name, bbox_inches='tight')\n",
    "        plt.close()\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}