05.06_Bin_Counting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- [Click-through ad data from Kaggle competition](https://www.kaggle.com/c/avazu-ctr-prediction/data)\n",
    "- train_subset is first 10K rows of 6+GB set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/train_subset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>click</th>\n",
       "      <th>hour</th>\n",
       "      <th>C1</th>\n",
       "      <th>banner_pos</th>\n",
       "      <th>site_id</th>\n",
       "      <th>site_domain</th>\n",
       "      <th>site_category</th>\n",
       "      <th>app_id</th>\n",
       "      <th>app_domain</th>\n",
       "      <th>...</th>\n",
       "      <th>device_type</th>\n",
       "      <th>device_conn_type</th>\n",
       "      <th>C14</th>\n",
       "      <th>C15</th>\n",
       "      <th>C16</th>\n",
       "      <th>C17</th>\n",
       "      <th>C18</th>\n",
       "      <th>C19</th>\n",
       "      <th>C20</th>\n",
       "      <th>C21</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.000009e+18</td>\n",
       "      <td>0</td>\n",
       "      <td>14102100</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1fbe01fe</td>\n",
       "      <td>f3845767</td>\n",
       "      <td>28905ebd</td>\n",
       "      <td>ecad2386</td>\n",
       "      <td>7801e8d9</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>15706</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>1722</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>-1</td>\n",
       "      <td>79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.000017e+19</td>\n",
       "      <td>0</td>\n",
       "      <td>14102100</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1fbe01fe</td>\n",
       "      <td>f3845767</td>\n",
       "      <td>28905ebd</td>\n",
       "      <td>ecad2386</td>\n",
       "      <td>7801e8d9</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>15704</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>1722</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>100084</td>\n",
       "      <td>79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.000037e+19</td>\n",
       "      <td>0</td>\n",
       "      <td>14102100</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1fbe01fe</td>\n",
       "      <td>f3845767</td>\n",
       "      <td>28905ebd</td>\n",
       "      <td>ecad2386</td>\n",
       "      <td>7801e8d9</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>15704</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>1722</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>100084</td>\n",
       "      <td>79</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id  click      hour    C1  banner_pos   site_id site_domain  \\\n",
       "0  1.000009e+18      0  14102100  1005           0  1fbe01fe    f3845767   \n",
       "1  1.000017e+19      0  14102100  1005           0  1fbe01fe    f3845767   \n",
       "2  1.000037e+19      0  14102100  1005           0  1fbe01fe    f3845767   \n",
       "\n",
       "  site_category    app_id app_domain ...  device_type device_conn_type    C14  \\\n",
       "0      28905ebd  ecad2386   7801e8d9 ...            1                2  15706   \n",
       "1      28905ebd  ecad2386   7801e8d9 ...            1                0  15704   \n",
       "2      28905ebd  ecad2386   7801e8d9 ...            1                0  15704   \n",
       "\n",
       "   C15  C16   C17  C18  C19     C20  C21  \n",
       "0  320   50  1722    0   35      -1   79  \n",
       "1  320   50  1722    0   35  100084   79  \n",
       "2  320   50  1722    0   35  100084   79  \n",
       "\n",
       "[3 rows x 24 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7201"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# how many features should we have after?\n",
    "len(df['device_id'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Features are $\\theta$ = [$N^+$, $N^-$, $log(N^+)-log(N^-)$, isRest]\n",
    "\n",
    "$N^+$ = $p(+)$ = $n^+/(n^+ + n^-)$\n",
    "\n",
    "$N^-$ = $p(-)$ = $n^-/(n^+ + n^-)$\n",
    "\n",
    "$log(N^+)-log(N^-)$ = $\\frac{p(+)}{p(-)}$\n",
    "\n",
    "isRest = back-off bin (not shown here)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def click_counting(x, bin_column):\n",
    "    clicks = pd.Series(x[x['click'] > 0][bin_column].value_counts(), name='clicks')\n",
    "    no_clicks = pd.Series(x[x['click'] < 1][bin_column].value_counts(), name='no_clicks')\n",
    "    \n",
    "    counts = pd.DataFrame([clicks,no_clicks]).T.fillna('0')\n",
    "    counts['total'] = counts['clicks'].astype('int64') + counts['no_clicks'].astype('int64')\n",
    "    \n",
    "    return counts\n",
    "\n",
    "def bin_counting(counts):\n",
    "    counts['N+'] = counts['clicks'].astype('int64').divide(counts['total'].astype('int64'))\n",
    "    counts['N-'] = counts['no_clicks'].astype('int64').divide(counts['total'].astype('int64'))\n",
    "    counts['log_N+'] = counts['N+'].divide(counts['N-'])\n",
    "\n",
    "#    If we wanted to only return bin-counting properties, we would filter here\n",
    "    bin_counts = counts.filter(items= ['N+', 'N-', 'log_N+'])\n",
    "    return counts, bin_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# bin counts example: device_id\n",
    "bin_column = 'device_id'\n",
    "device_clicks = click_counting(df.filter(items= [bin_column, 'click']), bin_column)\n",
    "device_all, device_bin_counts = bin_counting(device_clicks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7201"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check to make sure we have all the devices\n",
    "len(device_bin_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>clicks</th>\n",
       "      <th>no_clicks</th>\n",
       "      <th>total</th>\n",
       "      <th>N+</th>\n",
       "      <th>N-</th>\n",
       "      <th>log_N+</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>a99f214a</th>\n",
       "      <td>15729</td>\n",
       "      <td>71206</td>\n",
       "      <td>86935</td>\n",
       "      <td>0.180928</td>\n",
       "      <td>0.819072</td>\n",
       "      <td>0.220894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c357dbff</th>\n",
       "      <td>33</td>\n",
       "      <td>134</td>\n",
       "      <td>167</td>\n",
       "      <td>0.197605</td>\n",
       "      <td>0.802395</td>\n",
       "      <td>0.246269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31da1bd0</th>\n",
       "      <td>0</td>\n",
       "      <td>62</td>\n",
       "      <td>62</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>936e92fb</th>\n",
       "      <td>5</td>\n",
       "      <td>54</td>\n",
       "      <td>59</td>\n",
       "      <td>0.084746</td>\n",
       "      <td>0.915254</td>\n",
       "      <td>0.092593</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         clicks no_clicks  total        N+        N-    log_N+\n",
       "a99f214a  15729     71206  86935  0.180928  0.819072  0.220894\n",
       "c357dbff     33       134    167  0.197605  0.802395  0.246269\n",
       "31da1bd0      0        62     62  0.000000  1.000000  0.000000\n",
       "936e92fb      5        54     59  0.084746  0.915254  0.092593"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device_all.sort_values(by = 'total', ascending=False).head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Our pandas Series, in bytes:  7300031\n",
      "Our bin-counting feature, in bytes:  525697\n"
     ]
    }
   ],
   "source": [
    "# We can see how this can change model evaluation time by comparing raw vs. bin-counting size\n",
    "from sys import getsizeof\n",
    "\n",
    "print('Our pandas Series, in bytes: ', getsizeof(df.filter(items= ['device_id', 'click'])))\n",
    "print('Our bin-counting feature, in bytes: ', getsizeof(device_bin_counts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}