case study - amazon employee access.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:0c83e183649cb36da56a91116c71182bd6a221cd47536204f5afacec81b95d06"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "- [main page](http://www.kaggle.com/c/amazon-employee-access-challenge/data)\n",
      "- [solutions](http://www.chioka.in/kaggle-competition-solutions/)"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%load_ext autoreload\n",
      "%autoreload 2\n",
      "%matplotlib inline"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from munging import session\n",
      "from munging import transform\n",
      "\n",
      "\n",
      "import matplotlib.pyplot as plt\n",
      "import pandas as pd\n",
      "import numpy as np\n",
      "\n",
      "from sklearn.metrics import f1_score, roc_auc_score\n",
      "from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\n",
      "from sklearn.linear_model import LogisticRegression"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "## load data\n",
      "data = pd.read_csv(\"data/amazon_employee_access/train.csv\")\n",
      "## make ids recognized as categorical data\n",
      "for f in data.columns[1:]:\n",
      "    data[f] = data[f].astype(np.str)\n",
      "data.head(3)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>ACTION</th>\n",
        "      <th>RESOURCE</th>\n",
        "      <th>MGR_ID</th>\n",
        "      <th>ROLE_ROLLUP_1</th>\n",
        "      <th>ROLE_ROLLUP_2</th>\n",
        "      <th>ROLE_DEPTNAME</th>\n",
        "      <th>ROLE_TITLE</th>\n",
        "      <th>ROLE_FAMILY_DESC</th>\n",
        "      <th>ROLE_FAMILY</th>\n",
        "      <th>ROLE_CODE</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 1</td>\n",
        "      <td> 39353</td>\n",
        "      <td> 85475</td>\n",
        "      <td> 117961</td>\n",
        "      <td> 118300</td>\n",
        "      <td> 123472</td>\n",
        "      <td> 117905</td>\n",
        "      <td> 117906</td>\n",
        "      <td> 290919</td>\n",
        "      <td> 117908</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 1</td>\n",
        "      <td> 17183</td>\n",
        "      <td>  1540</td>\n",
        "      <td> 117961</td>\n",
        "      <td> 118343</td>\n",
        "      <td> 123125</td>\n",
        "      <td> 118536</td>\n",
        "      <td> 118536</td>\n",
        "      <td> 308574</td>\n",
        "      <td> 118539</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 1</td>\n",
        "      <td> 36724</td>\n",
        "      <td> 14457</td>\n",
        "      <td> 118219</td>\n",
        "      <td> 118220</td>\n",
        "      <td> 117884</td>\n",
        "      <td> 117879</td>\n",
        "      <td> 267952</td>\n",
        "      <td>  19721</td>\n",
        "      <td> 117880</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 23,
       "text": [
        "   ACTION RESOURCE MGR_ID ROLE_ROLLUP_1 ROLE_ROLLUP_2 ROLE_DEPTNAME  \\\n",
        "0       1    39353  85475        117961        118300        123472   \n",
        "1       1    17183   1540        117961        118343        123125   \n",
        "2       1    36724  14457        118219        118220        117884   \n",
        "\n",
        "  ROLE_TITLE ROLE_FAMILY_DESC ROLE_FAMILY ROLE_CODE  \n",
        "0     117905           117906      290919    117908  \n",
        "1     118536           118536      308574    118539  \n",
        "2     117879           267952       19721    117880  "
       ]
      }
     ],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "## exploration session\n",
      "dsession = session.Session(data, \"ACTION\", random_state=0)\n",
      "transformers = []\n",
      "print dsession.get_parameters()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'SKEWNESS_THR': 20, 'FRAC_OF_NA_TO_IGNORE': 0.95, 'FRAC_OF_FEAT_TO_BE_NONINFORMATIVE': 0.96, 'REDUNDANT_FEAT_CORR_THR': 0.95, 'MIN_NUM_VALUES_FOR_NUMERICAL': 5}\n"
       ]
      }
     ],
     "prompt_number": 25
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "## numerical and categorical features\n",
      "numerical_feats = dsession.get_features_of(dsession.is_numerical_feature)\n",
      "categorical_feats = dsession.get_features_of(dsession.is_categorical_feature)\n",
      "print len(numerical_feats)\n",
      "print len(categorical_feats)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0\n",
        "9\n"
       ]
      }
     ],
     "prompt_number": 26
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "## knowing what you are dealing with\n",
      "pd.value_counts(data.ACTION) * 1./ data.shape[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 28,
       "text": [
        "1    0.94211\n",
        "0    0.05789\n",
        "dtype: float64"
       ]
      }
     ],
     "prompt_number": 28
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}