python/Logistic-Regression-Feature-Importance.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import math"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = load_iris(return_X_y=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_names = [\"sepal_length\", \"sepal_width\", \"petal_length\", \"petal_width\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(x, columns = feature_names)\n",
    "df['target'] = y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sepal_length</th>\n",
       "      <th>sepal_width</th>\n",
       "      <th>petal_length</th>\n",
       "      <th>petal_width</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.7</td>\n",
       "      <td>3.2</td>\n",
       "      <td>1.3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.6</td>\n",
       "      <td>3.1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.6</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sepal_length  sepal_width  petal_length  petal_width  target\n",
       "0           5.1          3.5           1.4          0.2       0\n",
       "1           4.9          3.0           1.4          0.2       0\n",
       "2           4.7          3.2           1.3          0.2       0\n",
       "3           4.6          3.1           1.5          0.2       0\n",
       "4           5.0          3.6           1.4          0.2       0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2], dtype=int64)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['target'].unique()\n",
    "#0: Iris-setosa, 1: Iris-versicolor, 2: Iris-virginica"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert to Binary Classification Problem"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you feed multiclass classification problem to logistic regression, then it will build several binary classification models.\n",
    "\n",
    "To make this study simple, I will ignore Iris-virginica classes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df['target'] != 2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1], dtype=int64)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['target'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sepal_length</th>\n",
       "      <th>sepal_width</th>\n",
       "      <th>petal_length</th>\n",
       "      <th>petal_width</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>5.7</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.2</td>\n",
       "      <td>1.2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>5.7</td>\n",
       "      <td>2.9</td>\n",
       "      <td>4.2</td>\n",
       "      <td>1.3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>6.2</td>\n",
       "      <td>2.9</td>\n",
       "      <td>4.3</td>\n",
       "      <td>1.3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>5.1</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>5.7</td>\n",
       "      <td>2.8</td>\n",
       "      <td>4.1</td>\n",
       "      <td>1.3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    sepal_length  sepal_width  petal_length  petal_width  target\n",
       "95           5.7          3.0           4.2          1.2       1\n",
       "96           5.7          2.9           4.2          1.3       1\n",
       "97           6.2          2.9           4.3          1.3       1\n",
       "98           5.1          2.5           3.0          1.1       1\n",
       "99           5.7          2.8           4.1          1.3       1"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Normalize inputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Features have units. They all represent length and width in centimeters. We can get rid of the units if we divide them to standard deviation values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "for feature_name in feature_names:\n",
    "    #df[feature_name] = (df[feature_name] - df[feature_name].mean()) / df[feature_name].std()\n",
    "    df[feature_name] = df[feature_name] / df[feature_name].std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sepal_length</th>\n",
       "      <th>sepal_width</th>\n",
       "      <th>petal_length</th>\n",
       "      <th>petal_width</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>8.882678</td>\n",
       "      <td>6.301766</td>\n",
       "      <td>2.899422</td>\n",
       "      <td>2.119064</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>8.882678</td>\n",
       "      <td>6.091707</td>\n",
       "      <td>2.899422</td>\n",
       "      <td>2.295653</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>9.661861</td>\n",
       "      <td>6.091707</td>\n",
       "      <td>2.968456</td>\n",
       "      <td>2.295653</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>7.947660</td>\n",
       "      <td>5.251472</td>\n",
       "      <td>2.071016</td>\n",
       "      <td>1.942475</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>8.882678</td>\n",
       "      <td>5.881648</td>\n",
       "      <td>2.830388</td>\n",
       "      <td>2.295653</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    sepal_length  sepal_width  petal_length  petal_width  target\n",
       "95      8.882678     6.301766      2.899422     2.119064       1\n",
       "96      8.882678     6.091707      2.899422     2.295653       1\n",
       "97      9.661861     6.091707      2.968456     2.295653       1\n",
       "98      7.947660     5.251472      2.071016     1.942475       1\n",
       "99      8.882678     5.881648      2.830388     2.295653       1"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LogisticRegression(random_state=0).fit(df[feature_names].values, df['target'].values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate built model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100.0 % on  100 instances\n"
     ]
    }
   ],
   "source": [
    "print(model.score(df[feature_names].values, df['target'].values) * 100,\"% on \",df.shape[0],\"instances\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Equation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "w0 = model.intercept_[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "w = w1, w2, w3, w4 = model.coef_[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'y = -0.102763 + (0.444753 * x1) + (-1.371312 * x2) + (1.544792 * x3) + (1.590001 * x4)'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"y = %f + (%f * x1) + (%f * x2) + (%f * x3) + (%f * x4)\" % (w0, w1, w2, w3, w4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9782192589879745\n"
     ]
    }
   ],
   "source": [
    "idx = 99\n",
    "x = df.iloc[idx][feature_names].values\n",
    "#y = model.predict(x.reshape(1, -1))[0]\n",
    "y = model.predict_proba(x.reshape(1, -1))[0]\n",
    "print(y[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    return 1 / (1 + pow(math.e, -x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9782192589879745\n"
     ]
    }
   ],
   "source": [
    "result = 0\n",
    "\n",
    "result += w0\n",
    "for i in range(0, 4):\n",
    "    result += x[i] * w[i]\n",
    "\n",
    "result = sigmoid(result)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Interpretability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_importance = pd.DataFrame(feature_names, columns = [\"feature\"])\n",
    "feature_importance[\"importance\"] = pow(math.e, w)\n",
    "feature_importance = feature_importance.sort_values(by = [\"importance\"], ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>petal_width</td>\n",
       "      <td>4.903752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>petal_length</td>\n",
       "      <td>4.686994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sepal_length</td>\n",
       "      <td>1.560105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sepal_width</td>\n",
       "      <td>0.253774</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        feature  importance\n",
       "3   petal_width    4.903752\n",
       "2  petal_length    4.686994\n",
       "0  sepal_length    1.560105\n",
       "1   sepal_width    0.253774"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbUAAAD8CAYAAADwijrNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGWtJREFUeJzt3Xu0XnV95/H3JyGQACGUy2ICUQMMF5VAgCQVkFuXBi0wHSXCINMhtgyLolTHpW1mpmN1umxRaUHRgrRiHKUUoepYWFVwSbgrSTAXUiK3OUjEVYXaEOTSJHznj2cnnsbk3HLOeU523q+1ss5+fs/e+/c9e0E++f3279lPqgpJktpgXLcLkCRpuBhqkqTWMNQkSa1hqEmSWsNQkyS1hqEmSWoNQ02S1BqGmiSpNQw1SVJr7NLtAnY2++23X02fPr3bZUjSDmXp0qXPVtX+/e1nqI2y6dOns2TJkm6XIUk7lCRPDWQ/px8lSa1hqEmSWsNQkyS1hvfUJKkP69evZ82aNbz88svdLmWnMHHiRKZNm8aECROGdLyhJkl9WLNmDZMnT2b69Okk6XY5rVZVPPfcc6xZs4aDDz54SOdw+lGS+vDyyy+z7777GmijIAn77rvvdo2KDTVJ6oeBNnq291obapKk1vCemiQNwvQFtw3r+XouP7PffU488UTuv//+Ye23Lz09Pdx///28+93vHrU+h4sjNUka40Yz0DZs2EBPTw9/8zd/M2p9DidDTZLGuD333BOARYsWceqpp3Luuedy+OGHs2DBAm644QbmzJnDjBkzeOKJJwCYP38+l1xyCSeffDKHH344t956K9BZ9PKe97yHGTNmcOyxx3LnnXcCsHDhQt71rndx9tlnM3fuXBYsWMA999zDzJkzufLKK+np6eHkk0/muOOO47jjjtscsosWLeK0005j3rx5HHnkkVxwwQVUFQCLFy/mxBNP5JhjjmHOnDmsW7eOjRs38uEPf5jZs2dz9NFH8/nPf37Yr5XTj5K0A1m+fDmPPPII++yzD4cccggXXXQRDz74IJ/+9Ke5+uqrueqqq4DOFOJdd93FE088wemnn87jjz/O5z73OQBWrlzJ6tWrmTt3Lo8++igADzzwACtWrGCfffZh0aJFXHHFFZvD8MUXX+SOO+5g4sSJPPbYY5x//vmbn2H7gx/8gFWrVnHggQdy0kkncd999zFnzhzOO+88brrpJmbPns3zzz/PpEmT+MIXvsCUKVNYvHgxr7zyCieddBJz584d8vL9rTHURtnKH6/drjn5gcy/S2qv2bNnM3XqVAAOPfRQ5s6dC8CMGTM2j7wAzj33XMaNG8dhhx3GIYccwurVq7n33nu57LLLADjyyCN53etetznU3vrWt7LPPvtstc/169fzvve9j2XLljF+/PjNxwDMmTOHadOmATBz5kx6enqYMmUKU6dOZfbs2QDstddeANx+++2sWLGCW265BYC1a9fy2GOPGWqStLPabbfdNm+PGzdu8+tx48axYcOGze9tuTQ+yeapwa3ZY489tvnelVdeyQEHHMDy5ct59dVXmThx4lbrGT9+PBs2bKCqtro0v6q4+uqrOeOMM/r4DbeP99QkqYVuvvlmXn31VZ544gmefPJJjjjiCE455RRuuOEGAB599FF+9KMfccQRR/zKsZMnT2bdunWbX69du5apU6cybtw4vvzlL7Nx48Y++z7yyCN55plnWLx4MQDr1q1jw4YNnHHGGVxzzTWsX79+cw2/+MUvhutXBhypSdKg7Ci3AI444ghOPfVU/umf/olrr72WiRMncumll3LJJZcwY8YMdtllFxYuXPhvRlqbHH300eyyyy4cc8wxzJ8/n0svvZRzzjmHm2++mdNPP73PUR3Arrvuyk033cRll13GSy+9xKRJk/jOd77DRRddRE9PD8cddxxVxf777883vvGNYf2909dwVMNvt6mH1dQLrxry8TvK/1BSWzzyyCO8/vWv73YZgzJ//nzOOuss5s2b1+1ShmRr1zzJ0qqa1d+xTj9KklrD6UdJapmFCxd2u4SucaQmSf3wNs3o2d5rbahJUh8mTpzIc889Z7CNgk3fp9b7IwOD5fSjJPVh2rRprFmzhp/97GfdLmWnsOmbr4fKUJOkPkyYMGFYn3ihkdWK6cckpyW5dTuOn5XkM9t4ryfJfkn2TnLpcPUpSRp+rQi17VVVS6rq9/vZbW/g0n72kSR10aiFWpI9ktyWZHmSh5Ocl+T4JHclWZrk20mmNvsuSnJVkvubfec07XOath80P3/1+S5b73tlM9JKkueS/Jem/ctJ3tJ71JVk3yS3N318Htj0ALPLgUOTLEvyqaZtzyS3JFmd5Ib4ne+S1FWjOVJ7G/BMVR1TVUcB3wKuBuZV1fHA9cDHe+2/R1WdSGd0dH3Ttho4paqOBT4C/OkA+74POAl4I/AkcHLT/ibge1vs+8fAvU0f3wRe27QvAJ6oqplV9eGm7VjgA8AbgEOaPn5FkouTLEmyZOOLawdYsiRpsEZzochK4IoknwBuBX4OHAXc0QxwxgM/6bX/jQBVdXeSvZLsDUwGvpTkMKCACQPs+x7gFOAp4Brg4iQHAf9cVS9sMcA6BXhn0/dtSX7ex3kfrKo1AEmWAdOBe7fcqaquA66DzmOyBlizJGmQRm2kVlWPAsfTCbc/A84BVjUjn5lVNaOq5vY+ZMtTAH8C3NmM9M4GBvphhrvpjM5OBhYBPwPm0Qm7rZY7wPO+0mt7I64mlaSuGs17agcCL1bVV4ArgF8H9k9yQvP+hCRv7HXIeU37m4G1VbUWmAL8uHl//kD7rqqngf2Aw6rqSTqjqQ+x9VC7G7ig6fvtwK817evojBQlSWPUaI4sZgCfSvIqsB74PWAD8JkkU5pargJWNfv/PMn9wF7A7zRtn6Qz/fhB4LuD7P/7dKY4oRNmf8ZWpgqBjwE3JnkIuAv4EUBVPZfkviQPA/8ADP3rqyVJI2JMfvVMkkXAh6pqSbdrGW5+9YwkDZ5fPSNJ2umMyYUNVXXaUI5L8h7g/Vs031dV793uoiRJY96YDLWhqqovAl/sdh2SpO5w+lGS1BqGmiSpNQw1SVJrGGqSpNZo1UKRHcGMg6awxM+aSdKIcKQmSWoNQ02S1BqGmiSpNQw1SVJrGGqSpNYw1CRJrWGoSZJaw1CTJLWGoSZJag1DTZLUGoaaJKk1DDVJUmsYapKk1jDUJEmtYahJklrDUJMktYahJklqDUNNktQahpokqTUMNUlSaxhqkqTWMNQkSa1hqEmSWsNQkyS1hqEmSWoNQ02S1Bq7dLuAnc3KH69l+oLbul2Geum5/MxulyBpmDhSkyS1hqEmSWoNQ02S1BqGmiSpNQw1SVJrGGqSpNYw1CRJrWGoSZJaw1CTJLWGoSZJao0dMtSSnJbk1j7en5/ksyPQ7/wkB/Z63ZNkv+HuR5I0NDtkqHXRfODA/naSJHXHiD3QOMkewFeBacB44E+Ax4G/APYEngXmV9VPkiwClgFzgL2A36mqB5PMAa4CJgEvAe+pqh8Oso79gWuB1zZNH6iq+5J8tGk7pPl5VVV9pjnmfwEXAE83dS4FeoBZwA1JXgJOaM53WZKzgQnAu6pq9WDqkyQNn5Ecqb0NeKaqjqmqo4BvAVcD86rqeOB64OO99t+jqk4ELm3eA1gNnFJVxwIfAf50CHV8GriyqmYD5wB/3eu9I4Ez6ITpHyeZkGRWs9+xwDvpBBlVdQuwBLigqmZW1UvNOZ6tquOAa4APba2AJBcnWZJkycYX1w7hV5AkDcRIfvXMSuCKJJ8AbgV+DhwF3JEEOqO3n/Ta/0aAqro7yV5J9gYmA19KchhQdEZDg/UW4A1NnwB7JZncbN9WVa8AryT5KXAA8Gbg/24KrSR/38/5v9b8XEonBH9FVV0HXAew29TDagi/gyRpAEYs1Krq0STHA78J/BlwB7Cqqk7Y1iFbef0nwJ1V9Y4k04FFQyhlHHBCr5EVAE3IvdKraSOd6xEGZ9M5Nh0vSeqSEZt+bFYJvlhVXwGuAH4d2D/JCc37E5K8sdch5zXtbwbWVtVaYArw4+b9+UMs5Xbgfb3qmtnP/vcCZyeZmGRPoPc3SK6jM3qUJI1BIzmymAF8KsmrwHrg94ANwGeSTGn6vgpY1ez/8yT30ywUado+SWf68YPAd4dYx+8Dn0uyounzbuCSbe1cVYuTfBNYDjxF5z7aphthC4Frt1goIkkaI1LV/Vs8zerHD1XVkm7XApBkz6p6IcnudELw4qp6aDjOvdvUw2rqhVcNx6k0THouP7P/nSR1VZKlVTWrv/28B7R11yV5AzAR+NJwBZokaWSNiVCrqtOGclyS9wDv36L5vqp673bW8+7tOV6S1B1jItSGqqq+CHyx23VIksYGH5MlSWoNQ02S1BqGmiSpNfoNtSQHJPlCkn9oXr8hye+OfGmSJA3OQBaKLKSzGON/Nq8fBW4CvjBCNbXajIOmsMTPRUnSiBjI9ON+VfVV4FWAqtpA5zmHkiSNKQMJtV8k2ZfmgcNJ3sQvHxslSdKYMZDpxw8C3wQOTXIfsD8wb0SrkiRpCPoMtSTj6Dwq6lTgCDpfy/LDqlo/CrVJkjQofYZaVb2a5M+b70Bb1de+kiR120Duqd2e5Jz0+upoSZLGooHeU9sD2JDkZTpTkFVVe41oZZIkDVK/oVZVftOzJGmH0G+oJTlla+1VdffwlyNJ0tANZPrxw722JwJzgKXAb4xIRZIkDdFAph/P7v06yWuAT45YRZIkDdFQntK/BjhquAuRJGl7DeSe2tU0j8iiE4IzgeUjWZQkSUMxkHtqS3ptbwBurKr7RqgeSZKGbCChtndVfbp3Q5L3b9kmSVK3DeSe2oVbaZs/zHVIkrTdtjlSS3I+8G7g4CTf7PXWZOC5kS5MkqTB6mv68X7gJ8B+wJ/3al8HrBjJoiRJGopthlpVPQU8BZwweuVIkjR0/d5TS/KmJIuTvJDkX5NsTPL8aBQnSdJgDGShyGeB84HHgEnARcDVI1mUJElDMZAl/VTV40nGV9VG4ItJ7h/huiRJGrSBhNqLSXYFliX5JJ3FI3uMbFmSJA3eQKYff7vZ733AL4DXAOeMZFGSJA3FQJ7S/1SSScDUqvrYKNQkSdKQDGT149nAMuBbzeuZW3wYW5KkMWEg048fpfPFoP8CUFXLgOkjV5IkSUMzkIUiG6pqbZIRL2ZnsPLHa5m+4LZulyFJQ9Jz+ZndLqFPAwm1h5O8Gxif5DDg9+k8QkuSpDFlm9OPSb7cbD4BvBF4BbgReB74wMiXJknS4PQ1Ujs+yeuA84DT+bcPNd4deHkkC5MkabD6CrVr6ax4PIR/++3XAapplyRpzNjm9GNVfaaqXg9cX1WH9PpzcFUZaJKkMaffJf1V9XujUYgkSdtrIJ9TkyRph2CoSZJaw1CTJLWGoSZJao0xFWpJ5ic5cAD7LUwyr4/3FyWZNcy17Z3k0l6vT0ty63D2IUnaPmMq1ID5QL+h1iV7A5f2u5ckqWtGNNSSTE+yOsmXkqxIckuS3ZMcn+SuJEuTfDvJ1GbkNQu4IcmyJJOSfCTJ4iQPJ7kuQ3iqcpK5SR5I8lCSm5Ps2bT3JPlY074yyZFN+/5J7mjaP5/kqST7AZcDhza1fao5/Z7N77Q6yQ3bqi/JxUmWJFmy8cW1Q7qWkqT+jcZI7Qjguqo6ms5zI98LXA3Mq6rjgeuBj1fVLXSeXHJBVc2sqpeAz1bV7Ko6CpgEnDWYjpsw+iPgLVV1XHP+D/ba5dmm/RrgQ03bHwPfbdq/Dry2aV8APNHU9uGm7Vg6z8F8A50nrJy0tTqq6rqqmlVVs8bvPmUwv4IkaRAG8pT+7fV0Vd3XbH8F+B/AUcAdzcBmPPCTbRx7epI/oPOsyX2AVcDfD6LvN9EJnPuavnYFHuj1/tean0uBdzbbbwbeAVBV30ry8z7O/2BVrQFIsul75u4dRH2SpGE0GqFWW7xeB6yqqhP6OijJROAvgVlV9XSSjwITB9l3gDuq6vxtvP9K83Mjv7wWg5nifKXXdu9zSJK6YDSmH1+bZFOAnQ98D9h/U1uSCUne2Ly/DpjcbG8KsGeb+2DbXO3Yh+8BJyX5901fuyc5vJ9j7gXObfafC/zaVmqTJI1BoxFqjwAXJllBZwrxajoB9Ykky4FlwInNvguBa5upvFeAvwJWAt8AFg+246r6GZ0VlTc2/X8POLKfwz4GzE3yEPB2OlOj66rqOTrTmA/3WigiSRpDUrXl7OAwnjyZDtzaLPTYISTZDdhYVRua0eQ1VTVzuM6/29TDauqFVw3X6SRpVPVcfmZX+k2ytKr6/fyx94B+1WuBryYZB/wr8F+7XI8kaYBGNNSqqofOSscRkeTrwMFbNP9hVX17qOesqsfoLNWXJO1gduiRWlW9o9s1SJLGjrH2mCxJkobMUJMktYahJklqDUNNktQaO/RCkR3RjIOmsKRLn/OQpLZzpCZJag1DTZLUGoaaJKk1DDVJUmsYapKk1jDUJEmtYahJklrDUJMktYahJklqDUNNktQahpokqTUMNUlSaxhqkqTWMNQkSa1hqEmSWsNQkyS1hqEmSWoNQ02S1BqGmiSpNQw1SVJrGGqSpNYw1CRJrWGoSZJaw1CTJLWGoSZJag1DTZLUGrt0u4Cdzcofr2X6gtu6XYYkjaqey88clX4cqUmSWsNQkyS1hqEmSWoNQ02S1BqGmiSpNQw1SVJrGGqSpNYw1CRJrWGoSZJaw1CTJLXGmA+1JPOTHDiA/RYmmbcd/fzvJG/ZSvtpSW7ttX3icPUpSRpeO8KzH+cDDwPPjGQnVfWRAex2GvACcP9I1iJJGppRH6klmZ5kdZIvJVmR5JYkuyc5PsldSZYm+XaSqc0oaBZwQ5JlSSYl+UiSxUkeTnJdkgygzzlJvtZs/1aSl5LsmmRikieb9s2jriRva2q8F3jnprqBS4D/1tRycnP6U5Lcn+RJR22S1F3dmn48Ariuqo4GngfeC1wNzKuq44HrgY9X1S3AEuCCqppZVS8Bn62q2VV1FDAJOGsA/T0EHNtsn0xn5Dcb+HXg+713TDIR+Cvg7GbffwdQVT3AtcCVTS33NIdMBd7c1HH51jpPcnGSJUmWbHxx7QDKlSQNRbdC7emquq/Z/gpwBnAUcEeSZcAfAdO2cezpSb6fZCXwG8Ab++usqjYAjyd5PTAH+AvgFDqhdc8Wux8J/L+qeqyqqqmvL9+oqler6h+BA7bR/3VVNauqZo3ffUp/5UqShqhb99Rqi9frgFVVdUJfBzWjqL8EZlXV00k+CkwcYJ/3AG8H1gPfARYC44EPDaC+vrzSu8RBHCdJGmbdGqm9NsmmADsf+B6w/6a2JBOSbBqBrQMmN9ubAuzZJHsCg7mHdTfwAeCBqvoZsC+dUdmqLfZbDRyc5NBe9W3SuxZJ0hjTrVB7BLgwyQpgH5r7acAnkiwHlgGbls4vBK5tpiVfoXO/ayXwDWDxIPr8Pp3pwbub1yuAFc0U42ZV9TJwMXBbs1DkqV5v/z3wji0WikiSxohs8Xf6yHfYWUV4a7PQY6ez29TDauqFV3W7DEkaVT2Xn7ldxydZWlWz+ttvzH/4WpKkgRr1hSLN0vgRG6Ul+Tpw8BbNf1hV3x6pPiVJY8OO8ESRQamqd3S7BklSdzj9KElqDUNNktQahpokqTUMNUlSa7RuochYN+OgKSzZzs9rSJK2zpGaJKk1DDVJUmsYapKk1jDUJEmtYahJklrDUJMktYahJklqDUNNktQahpokqTUMNUlSaxhqkqTWMNQkSa1hqEmSWiNV1e0adipJ1gE/7HYdXbYf8Gy3i+gyr4HXALwGMPBr8Lqq2r+/nfzqmdH3w6qa1e0iuinJEq+B18Br4DWA4b8GTj9KklrDUJMktYahNvqu63YBY4DXwGsAXgPwGsAwXwMXikiSWsORmiSpNQy1UZLkbUl+mOTxJAu6XU83JLk+yU+TPNztWrohyWuS3JnkkSSrkry/2zV1Q5KJSR5Msry5Dh/rdk3dkGR8kh8kubXbtXRLkp4kK5MsS7JkWM7p9OPISzIeeBR4K7AGWAycX1X/2NXCRlmSU4AXgP9TVUd1u57RlmQqMLWqHkoyGVgK/Med8L+DAHtU1QtJJgD3Au+vqu91ubRRleSDwCxgr6o6q9v1dEOSHmBWVQ3bZ/UcqY2OOcDjVfVkVf0r8LfAb3W5plFXVXcD/9ztOrqlqn5SVQ812+uAR4CDulvV6KuOF5qXE5o/O9W/rpNMA84E/rrbtbSNoTY6DgKe7vV6DTvhX2b6pSTTgWOB73e3ku5opt6WAT8F7qiqne06XAX8AfBqtwvpsgJuT7I0ycXDcUJDbXRkK2071b9M9UtJ9gT+DvhAVT3f7Xq6oao2VtVMYBowJ8lOMx2d5Czgp1W1tNu1jAEnVdVxwNuB9za3KLaLoTY61gCv6fV6GvBMl2pRFzX3kP4OuKGqvtbterqtqv4FWAS8rculjKaTgP/Q3E/6W+A3knyluyV1R1U90/z8KfB1OrdqtouhNjoWA4clOTjJrsB/Ar7Z5Zo0ypoFEl8AHqmqv+h2Pd2SZP8kezfbk4C3AKu7W9Xoqar/XlXTqmo6nb8LvltV/7nLZY26JHs0C6ZIsgcwF9juldGG2iioqg3A+4Bv01kc8NWqWtXdqkZfkhuBB4AjkqxJ8rvdrmmUnQT8Np1/mS9r/vxmt4vqgqnAnUlW0PkH3x1VtdMua9+JHQDcm2Q58CBwW1V9a3tP6pJ+SVJrOFKTJLWGoSZJag1DTZLUGoaaJKk1DDVJUmsYapKk1jDUJEmtYahJklrj/wM0GfWliS3xNgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax = feature_importance.plot.barh(x='feature', y='importance')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}