notebooks/extra - imbalanced datasets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Working with highly imbalanced data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "% matplotlib inline\n",
    "plt.rcParams[\"figure.dpi\"] = 200\n",
    "np.set_printoptions(precision=3, suppress=True)\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.preprocessing import scale, StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rng = np.random.RandomState(0)\n",
    "n_samples_1 = 1000\n",
    "n_samples_2 = 100\n",
    "X_syn = np.r_[1.5 * rng.randn(n_samples_1, 2),\n",
    "        0.5 * rng.randn(n_samples_2, 2) + [2, 2]]\n",
    "y_syn = np.array([0] * (n_samples_1) + [1] * (n_samples_2))\n",
    "X_syn_train, X_syn_test, y_syn_train, y_syn_test = train_test_split(X_syn, y_syn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mammography dataset https://www.openml.org/d/310\n",
    "data = pd.read_csv(\"https://www.openml.org/data/v1/get_csv/52214\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = data['class']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = (target == \"'-1'\").astype(np.int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = data.iloc[:, :-1]\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.hist(bins='auto')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.plotting.scatter_matrix(X, c=y, alpha=.2);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X.values, y.values, stratify=y, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "pca = PCA()\n",
    "X_train_pca = pca.fit_transform(X_train)\n",
    "plt.plot(pca.explained_variance_ratio_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorting = np.argsort(y_train)\n",
    "fig, axes = plt.subplots(1, 2)\n",
    "axes[0].scatter(X_train_pca[:, 0], X_train_pca[:, 1], c=y_train[:], alpha=.3)\n",
    "axes[1].scatter(X_train_pca[sorting, 0], X_train_pca[sorting, 1], c=y_train[sorting], alpha=.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import RobustScaler\n",
    "rs = RobustScaler().fit(X_train)\n",
    "X_train_scaled = rs.transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_scaled = PCA()\n",
    "X_train_pca_scaled = pca_scaled.fit_transform(X_train_scaled)\n",
    "plt.plot(pca_scaled.explained_variance_ratio_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 2)\n",
    "axes[0].scatter(X_train_pca_scaled[:, 0], X_train_pca_scaled[:, 1], c=y_train[:], alpha=.3)\n",
    "axes[1].scatter(X_train_pca_scaled[sorting, 0], X_train_pca_scaled[sorting, 1], c=y_train[sorting], alpha=.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorting = np.argsort(y_train)\n",
    "fig, axes = plt.subplots(1, 2, figsize=(8, 4))\n",
    "axes[0].scatter(X_train[:, 3], X_train[:, 4], c=y_train[:], alpha=.3)\n",
    "axes[1].scatter(X_train[sorting, 3], X_train[sorting, 4], c=y_train[sorting], alpha=.3)\n",
    "axes[0].set_title(\"Feature 3 vs 4 random order\")\n",
    "axes[1].set_title(\"Feature 3 vs 4 sorted\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "scores = cross_val_score(LogisticRegression(),\n",
    "                         X_train, y_train, cv=10, scoring='roc_auc')\n",
    "print(scores.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "scores = cross_val_score(LogisticRegressionCV(scoring='roc_auc'), X_train, y_train, cv=10, scoring='roc_auc')\n",
    "print(scores.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "\n",
    "rus = RandomUnderSampler(replacement=False)\n",
    "X_train_subsample, y_train_subsample = rus.fit_sample(X_train, y_train)\n",
    "print(X_train.shape)\n",
    "print(X_train_subsample.shape)\n",
    "print(np.bincount(y_train_subsample))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.pipeline import make_pipeline as make_imb_pipeline\n",
    "\n",
    "undersample_pipe = make_imb_pipeline(RandomUnderSampler(), LogisticRegressionCV())\n",
    "scores = cross_val_score(undersample_pipe, X_train, y_train, cv=10, scoring='roc_auc')\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import RandomOverSampler\n",
    "ros = RandomOverSampler()\n",
    "X_train_oversample, y_train_oversample = ros.fit_sample(X_train, y_train)\n",
    "print(X_train.shape)\n",
    "print(X_train_oversample.shape)\n",
    "print(np.bincount(y_train_oversample))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "oversample_pipe = make_imb_pipeline(RandomOverSampler(), LogisticRegression())\n",
    "scores = cross_val_score(oversample_pipe, X_train, y_train, cv=10, scoring='roc_auc')\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = cross_val_score(LogisticRegression(class_weight='balanced'),\n",
    "                                            X_train, y_train, cv=10, scoring='roc_auc')\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "oversample_pipe.fit(X_train, y_train)\n",
    "props_oversample = oversample_pipe.predict_proba(X_test)[:, 1]\n",
    "fpr_over, tpr_over, _ = roc_curve(y_test, props_oversample)\n",
    "\n",
    "undersample_pipe.fit(X_train, y_train)\n",
    "props_undersample = undersample_pipe.predict_proba(X_test)[:, 1]\n",
    "fpr_under, tpr_under, _ = roc_curve(y_test, props_undersample)\n",
    "\n",
    "lr = LogisticRegression().fit(X_train, y_train)\n",
    "props_original = lr.predict_proba(X_test)[:, 1]\n",
    "fpr_org, tpr_org, _ = roc_curve(y_test, props_original)\n",
    "\n",
    "plt.plot(fpr_org, tpr_org, label=\"original\")\n",
    "plt.plot(fpr_over, tpr_over, label=\"oversample\")\n",
    "plt.plot(fpr_under, tpr_under, label=\"undersample\")\n",
    "plt.legend()\n",
    "plt.xlabel(\"FPR\")\n",
    "plt.ylabel(\"TPR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "scores = cross_val_score(RandomForestClassifier(n_estimators=100),\n",
    "                         X_train, y_train, cv=10, scoring='roc_auc')\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "undersample_pipe_rf = make_imb_pipeline(RandomUnderSampler(), RandomForestClassifier())\n",
    "scores = cross_val_score(undersample_pipe_rf, X_train, y_train, cv=10, scoring='roc_auc')\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "oversample_pipe_rf = make_imb_pipeline(RandomOverSampler(), RandomForestClassifier())\n",
    "scores = cross_val_score(oversample_pipe_rf, X_train, y_train, cv=10, scoring='roc_auc')\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "oversample_pipe_rf.fit(X_train, y_train)\n",
    "props_oversample = oversample_pipe_rf.predict_proba(X_test)[:, 1]\n",
    "fpr_over, tpr_over, _ = roc_curve(y_test, props_oversample)\n",
    "\n",
    "undersample_pipe_rf.fit(X_train, y_train)\n",
    "props_undersample = undersample_pipe_rf.predict_proba(X_test)[:, 1]\n",
    "fpr_under, tpr_under, _ = roc_curve(y_test, props_undersample)\n",
    "\n",
    "rf = RandomForestClassifier(n_estimators=100).fit(X_train, y_train)\n",
    "props_original = rf.predict_proba(X_test)[:, 1]\n",
    "fpr_org, tpr_org, _ = roc_curve(y_test, props_original)\n",
    "\n",
    "plt.plot(fpr_org, tpr_org, label=\"original\")\n",
    "plt.plot(fpr_over, tpr_over, label=\"oversample\")\n",
    "plt.plot(fpr_under, tpr_under, label=\"undersample\")\n",
    "plt.legend()\n",
    "plt.xlabel(\"FPR\")\n",
    "plt.ylabel(\"TPR\")\n",
    "plt.title(\"RF comparison\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Class Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "scores = cross_val_score(LogisticRegression(class_weight='balanced'),\n",
    "                         X_train, y_train, cv=10, scoring='roc_auc')\n",
    "print(scores.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "scores = cross_val_score(RandomForestClassifier(n_estimators=100, class_weight='balanced'),\n",
    "                         X_train, y_train, cv=10, scoring='roc_auc')\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Resampled Ensembles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "def make_resample_tree(random_state=0):\n",
    "    tree = make_imb_pipeline(RandomUnderSampler(random_state=random_state, replacement=True),\n",
    "                             DecisionTreeClassifier(max_features='auto', random_state=random_state))\n",
    "    return \"tree_{}\".format(random_state), tree\n",
    "classifiers = [make_resample_tree(i) for i in range(100)]\n",
    "resampled_rf = VotingClassifier(classifiers, voting='soft')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = cross_val_score(resampled_rf, X_train, y_train, cv=10, scoring='roc_auc')\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import clone\n",
    "\n",
    "def make_resampled_ensemble(estimator, n_estimators=100):\n",
    "    estimators = []\n",
    "    for i in range(n_estimators):\n",
    "        est = clone(estimator)\n",
    "        if hasattr(est, \"random_state\"):\n",
    "            est.random_state = i\n",
    "        pipe = make_imb_pipeline(RandomUnderSampler(random_state=i, replacement=True),\n",
    "                                 est)\n",
    "        estimators.append((\"est_{}\".format(i), pipe))\n",
    "    return VotingClassifier(estimators, voting=\"soft\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "resampled_tree_test = make_resampled_ensemble(DecisionTreeClassifier(max_features='auto'))\n",
    "\n",
    "scores = cross_val_score(resampled_tree_test, X_train, y_train, cv=10, scoring='roc_auc')\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "resampled_lr = make_resampled_ensemble(LogisticRegression())\n",
    "\n",
    "scores = cross_val_score(resampled_lr, X_train, y_train, cv=10, scoring='roc_auc')\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SMOTE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import SMOTE\n",
    "smote = SMOTE()\n",
    "X_train_smote, y_train_smote = smote.fit_sample(X_train, y_train)\n",
    "print(X_train_smote.shape)\n",
    "print(np.bincount(y_train_smote))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_resampled, y_resampled = SMOTE().fit_sample(X_syn, y_syn)\n",
    "fig, axes = plt.subplots(1, 2)\n",
    "axes[0].scatter(X_syn[:, 0], X_syn[:, 1], c=plt.cm.Vega10(y_syn), alpha=.3)\n",
    "axes[1].scatter(X_resampled[:, 0], X_resampled[:, 1], c=plt.cm.Vega10(y_resampled), alpha=.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 2, figsize=(8, 4))\n",
    "sorting = np.argsort(y_train)\n",
    "axes[0].scatter(X_train[sorting, 3], X_train[sorting, 4], c=y_train[sorting], alpha=.3)\n",
    "axes[1].scatter(X_train_smote[:, 3], X_train_smote[:, 4], c=y_train_smote, alpha=.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "fig, axes = plt.subplots(1, 2, figsize=(8, 4))\n",
    "X_smote_sh, y_smote_sh = shuffle(X_train_smote, y_train_smote)\n",
    "axes[0].scatter(X_train[:, 3], X_train[:, 4], c=y_train, alpha=.3)\n",
    "axes[1].scatter(X_smote_sh[:, 3], X_smote_sh[:, 4], c=y_smote_sh, alpha=.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "smote_pipe = make_imb_pipeline(SMOTE(), LogisticRegression())\n",
    "scores = cross_val_score(smote_pipe, X_train, y_train, cv=10, scoring='roc_auc')\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "smote_pipe_rf = make_imb_pipeline(SMOTE(), RandomForestClassifier(n_estimators=100))\n",
    "scores = cross_val_score(smote_pipe_rf, X_train, y_train, cv=10, scoring='roc_auc')\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "param_grid = {'smote__k_neighbors': [3, 5, 7, 9, 11, 15, 31]}\n",
    "search = GridSearchCV(smote_pipe_rf, param_grid, cv=10, scoring=\"roc_auc\")\n",
    "search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "search.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(search.cv_results_)\n",
    "results.plot(\"param_smote__k_neighbors\", [\"mean_test_score\", \"mean_train_score\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "smote_pipe_rf = make_imb_pipeline(SMOTE(k_neighbors=11), RandomForestClassifier(n_estimators=100))\n",
    "scores = cross_val_score(smote_pipe_rf, X_train, y_train, cv=10, scoring='roc_auc')\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import SMOTE\n",
    "smote = SMOTE(k_neighbors=11)\n",
    "X_train_smote11, y_train_smote11 = smote.fit_sample(X_train, y_train)\n",
    "\n",
    "fig, axes = plt.subplots(1, 2, figsize=(8, 4))\n",
    "X_smote_sh11, y_smote_sh11 = shuffle(X_train_smote11, y_train_smote11)\n",
    "axes[0].scatter(X_smote_sh[:, 3], X_smote_sh[:, 4], c=y_smote_sh, alpha=.3)\n",
    "axes[1].scatter(X_smote_sh11[:, 3], X_smote_sh11[:, 4], c=y_smote_sh11, alpha=.3)\n",
    "axes[0].set_title(\"SMOTE k_neighbors=5\")\n",
    "axes[1].set_title(\"SMOTE k_neighbors=11\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.combine import SMOTEENN, SMOTETomek\n",
    "smoteenn_pipe_rf = make_imb_pipeline(SMOTEENN(smote=SMOTE(k_neighbors=11)), RandomForestClassifier(n_estimators=100))\n",
    "scores = cross_val_score(smoteenn_pipe_rf, X_train, y_train, cv=10, scoring='roc_auc')\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "smoteenn = SMOTEENN(smote=SMOTE(k_neighbors=11))\n",
    "\n",
    "X_train_smoteenn, y_train_smoteenn = smoteenn.fit_sample(X_train, y_train)\n",
    "\n",
    "fig, axes = plt.subplots(1, 2, figsize=(8, 4))\n",
    "X_smote_shenn, y_smote_shenn = shuffle(X_train_smoteenn, y_train_smoteenn)\n",
    "axes[0].scatter(X_smote_sh11[:, 3], X_smote_sh11[:, 4], c=y_smote_sh11, alpha=.3)\n",
    "axes[1].scatter(X_smote_shenn[:, 3], X_smote_shenn[:, 4], c=y_smote_shenn, alpha=.3)\n",
    "axes[0].set_title(\"SMOTE\")\n",
    "axes[1].set_title(\"SMOTE ENN\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.bincount(y_train_smote11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.bincount(y_train_smoteenn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import CondensedNearestNeighbour\n",
    "\n",
    "X_resampled, y_resampled = SMOTE().fit_sample(X_syn, y_syn)\n",
    "X_resampled_enn, y_resampled_enn = SMOTEENN().fit_sample(X_syn, y_syn)\n",
    "X_resampled_cnn, y_resampled_cnn = make_imb_pipeline(SMOTE(), CondensedNearestNeighbour()).fit_sample(X_syn, y_syn)\n",
    "\n",
    "fig, axes = plt.subplots(1, 3)\n",
    "\n",
    "axes[0].scatter(X_resampled[:, 0], X_resampled[:, 1], c=plt.cm.Vega10(y_resampled), alpha=.3)\n",
    "axes[0].set_title(\"SMOTE\")\n",
    "axes[1].scatter(X_resampled_enn[:, 0], X_resampled_enn[:, 1], c=plt.cm.Vega10(y_resampled_enn), alpha=.3)\n",
    "axes[1].set_title(\"SMOTE ENN\")\n",
    "axes[2].scatter(X_resampled_cnn[:, 0], X_resampled_cnn[:, 1], c=plt.cm.Vega10(y_resampled_cnn), alpha=.3)\n",
    "axes[2].set_title(\"SMOTE CNN\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}