ethnicolr/examples/ethnicolr_app_contrib20xx.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Application: 2000/2010 Political Campaign Contributions by Race\n",
    "\n",
    "Using ethnicolr, we look to answer three basic questions:\n",
    "<ol>\n",
    "<li>What proportion of contributions were made by blacks, whites, Hispanics, and Asians? \n",
    "<li>What proportion of unique contributors were blacks, whites, Hispanics, and Asians?\n",
    "<li>What proportion of total donations were given by blacks, whites, Hispanics, and Asians?\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([u'cycle', u'transaction_id', u'transaction_type', u'amount', u'date',\n",
       "       u'bonica_cid', u'contributor_name', u'contributor_lname',\n",
       "       u'contributor_fname', u'contributor_mname', u'contributor_suffix',\n",
       "       u'contributor_title', u'contributor_ffname', u'contributor_type',\n",
       "       u'contributor_gender', u'contributor_address', u'contributor_city',\n",
       "       u'contributor_state', u'contributor_zipcode', u'contributor_occupation',\n",
       "       u'contributor_employer', u'contributor_category',\n",
       "       u'contributor_category_order', u'is_corp', u'organization_name',\n",
       "       u'parent_organization_name', u'recipient_name', u'bonica_rid',\n",
       "       u'recipient_party', u'recipient_type', u'recipient_state',\n",
       "       u'recipient_category', u'recipient_category_order',\n",
       "       u'recipient_district', u'seat', u'election_type',\n",
       "       u'contributor_cfscore', u'candidate_cfscore', u'latitude', u'longitude',\n",
       "       u'gis_confidence', u'contributor_district_90s',\n",
       "       u'contributor_district_00s', u'contributor_district_10s',\n",
       "       u'lname_frequency', u'efec_memo', u'efec_memo2',\n",
       "       u'efec_transaction_id_orig', u'efec_org_orig', u'efec_comid_orig',\n",
       "       u'efec_form_type'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('/opt/names/fec_contrib/contribDB_2000.csv', nrows=100)\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/venv/local/lib/python2.7/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from ethnicolr import census_ln, pred_census_ln"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Load and Subset on Individual Contributors**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/venv/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (8,7) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>amount</th>\n",
       "      <th>contributor_name</th>\n",
       "      <th>contributor_lname</th>\n",
       "      <th>contributor_fname</th>\n",
       "      <th>contributor_type</th>\n",
       "      <th>race</th>\n",
       "      <th>api</th>\n",
       "      <th>black</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>white</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>102912</th>\n",
       "      <td>180.0</td>\n",
       "      <td>JOHNSON, KENYIE PAUL</td>\n",
       "      <td>johnson</td>\n",
       "      <td>kenyie</td>\n",
       "      <td>I</td>\n",
       "      <td>white</td>\n",
       "      <td>0.002370</td>\n",
       "      <td>0.364711</td>\n",
       "      <td>0.014732</td>\n",
       "      <td>0.618187</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103990</th>\n",
       "      <td>743.0</td>\n",
       "      <td>KIRSCH, STEVEN T</td>\n",
       "      <td>kirsch</td>\n",
       "      <td>steven</td>\n",
       "      <td>I</td>\n",
       "      <td>white</td>\n",
       "      <td>0.009365</td>\n",
       "      <td>0.004461</td>\n",
       "      <td>0.019418</td>\n",
       "      <td>0.966756</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105298</th>\n",
       "      <td>180.0</td>\n",
       "      <td>MCCOY, TIMOTHY D</td>\n",
       "      <td>mccoy</td>\n",
       "      <td>timothy</td>\n",
       "      <td>I</td>\n",
       "      <td>white</td>\n",
       "      <td>0.006450</td>\n",
       "      <td>0.292733</td>\n",
       "      <td>0.022897</td>\n",
       "      <td>0.677921</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105344</th>\n",
       "      <td>188.0</td>\n",
       "      <td>WILLIAMS, VICTOR K</td>\n",
       "      <td>williams</td>\n",
       "      <td>victor</td>\n",
       "      <td>I</td>\n",
       "      <td>black</td>\n",
       "      <td>0.001726</td>\n",
       "      <td>0.515756</td>\n",
       "      <td>0.020712</td>\n",
       "      <td>0.461805</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105577</th>\n",
       "      <td>211.0</td>\n",
       "      <td>ELDER, CHESTER H</td>\n",
       "      <td>elder</td>\n",
       "      <td>chester</td>\n",
       "      <td>I</td>\n",
       "      <td>white</td>\n",
       "      <td>0.004245</td>\n",
       "      <td>0.165576</td>\n",
       "      <td>0.033800</td>\n",
       "      <td>0.796379</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105659</th>\n",
       "      <td>13000.0</td>\n",
       "      <td>MACARTHUR, GREG</td>\n",
       "      <td>macarthur</td>\n",
       "      <td>greg</td>\n",
       "      <td>I</td>\n",
       "      <td>white</td>\n",
       "      <td>0.057506</td>\n",
       "      <td>0.028473</td>\n",
       "      <td>0.017616</td>\n",
       "      <td>0.896404</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105665</th>\n",
       "      <td>13972.0</td>\n",
       "      <td>ABELE, CHRIS</td>\n",
       "      <td>abele</td>\n",
       "      <td>chris</td>\n",
       "      <td>I</td>\n",
       "      <td>white</td>\n",
       "      <td>0.012272</td>\n",
       "      <td>0.072357</td>\n",
       "      <td>0.048589</td>\n",
       "      <td>0.866782</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105829</th>\n",
       "      <td>15000.0</td>\n",
       "      <td>PRICE, SOL</td>\n",
       "      <td>price</td>\n",
       "      <td>sol</td>\n",
       "      <td>I</td>\n",
       "      <td>white</td>\n",
       "      <td>0.001978</td>\n",
       "      <td>0.194008</td>\n",
       "      <td>0.008592</td>\n",
       "      <td>0.795422</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106029</th>\n",
       "      <td>13600.0</td>\n",
       "      <td>KIRSCH, STEVEN T</td>\n",
       "      <td>kirsch</td>\n",
       "      <td>steven</td>\n",
       "      <td>I</td>\n",
       "      <td>white</td>\n",
       "      <td>0.009365</td>\n",
       "      <td>0.004461</td>\n",
       "      <td>0.019418</td>\n",
       "      <td>0.966756</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106150</th>\n",
       "      <td>22146.0</td>\n",
       "      <td>KIRSCH, STEVEN T</td>\n",
       "      <td>kirsch</td>\n",
       "      <td>steven</td>\n",
       "      <td>I</td>\n",
       "      <td>white</td>\n",
       "      <td>0.009365</td>\n",
       "      <td>0.004461</td>\n",
       "      <td>0.019418</td>\n",
       "      <td>0.966756</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106690</th>\n",
       "      <td>813.0</td>\n",
       "      <td>SHAPP, RICHARD A</td>\n",
       "      <td>shapp</td>\n",
       "      <td>richard</td>\n",
       "      <td>I</td>\n",
       "      <td>white</td>\n",
       "      <td>0.014794</td>\n",
       "      <td>0.109744</td>\n",
       "      <td>0.020067</td>\n",
       "      <td>0.855395</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107082</th>\n",
       "      <td>949.0</td>\n",
       "      <td>WILLIAMS, VICTOR K</td>\n",
       "      <td>williams</td>\n",
       "      <td>victor</td>\n",
       "      <td>I</td>\n",
       "      <td>black</td>\n",
       "      <td>0.001726</td>\n",
       "      <td>0.515756</td>\n",
       "      <td>0.020712</td>\n",
       "      <td>0.461805</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109127</th>\n",
       "      <td>1717.0</td>\n",
       "      <td>HELFRICK, EDWARD W</td>\n",
       "      <td>helfrick</td>\n",
       "      <td>edward</td>\n",
       "      <td>I</td>\n",
       "      <td>white</td>\n",
       "      <td>0.002648</td>\n",
       "      <td>0.020904</td>\n",
       "      <td>0.008069</td>\n",
       "      <td>0.968379</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109266</th>\n",
       "      <td>8.0</td>\n",
       "      <td>MCCOY, TIMOTHY D</td>\n",
       "      <td>mccoy</td>\n",
       "      <td>timothy</td>\n",
       "      <td>I</td>\n",
       "      <td>white</td>\n",
       "      <td>0.006450</td>\n",
       "      <td>0.292733</td>\n",
       "      <td>0.022897</td>\n",
       "      <td>0.677921</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109541</th>\n",
       "      <td>24150.0</td>\n",
       "      <td>KIRSCH, STEVEN T</td>\n",
       "      <td>kirsch</td>\n",
       "      <td>steven</td>\n",
       "      <td>I</td>\n",
       "      <td>white</td>\n",
       "      <td>0.009365</td>\n",
       "      <td>0.004461</td>\n",
       "      <td>0.019418</td>\n",
       "      <td>0.966756</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109586</th>\n",
       "      <td>265.0</td>\n",
       "      <td>WILLIAMS, VICTOR K</td>\n",
       "      <td>williams</td>\n",
       "      <td>victor</td>\n",
       "      <td>I</td>\n",
       "      <td>black</td>\n",
       "      <td>0.001726</td>\n",
       "      <td>0.515756</td>\n",
       "      <td>0.020712</td>\n",
       "      <td>0.461805</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109603</th>\n",
       "      <td>25750.0</td>\n",
       "      <td>SUTHERLAND, DWIGHT D JR</td>\n",
       "      <td>sutherland</td>\n",
       "      <td>dwight</td>\n",
       "      <td>I</td>\n",
       "      <td>white</td>\n",
       "      <td>0.007970</td>\n",
       "      <td>0.081236</td>\n",
       "      <td>0.015958</td>\n",
       "      <td>0.894836</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109664</th>\n",
       "      <td>45795.0</td>\n",
       "      <td>KIRSCH, STEVEN T</td>\n",
       "      <td>kirsch</td>\n",
       "      <td>steven</td>\n",
       "      <td>I</td>\n",
       "      <td>white</td>\n",
       "      <td>0.009365</td>\n",
       "      <td>0.004461</td>\n",
       "      <td>0.019418</td>\n",
       "      <td>0.966756</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109730</th>\n",
       "      <td>28884.0</td>\n",
       "      <td>KIRSCH, STEVEN T</td>\n",
       "      <td>kirsch</td>\n",
       "      <td>steven</td>\n",
       "      <td>I</td>\n",
       "      <td>white</td>\n",
       "      <td>0.009365</td>\n",
       "      <td>0.004461</td>\n",
       "      <td>0.019418</td>\n",
       "      <td>0.966756</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109731</th>\n",
       "      <td>30000.0</td>\n",
       "      <td>WOODS, FRED R</td>\n",
       "      <td>woods</td>\n",
       "      <td>fred</td>\n",
       "      <td>I</td>\n",
       "      <td>white</td>\n",
       "      <td>0.000718</td>\n",
       "      <td>0.373277</td>\n",
       "      <td>0.007999</td>\n",
       "      <td>0.618006</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         amount         contributor_name contributor_lname contributor_fname  \\\n",
       "102912    180.0     JOHNSON, KENYIE PAUL           johnson            kenyie   \n",
       "103990    743.0         KIRSCH, STEVEN T            kirsch            steven   \n",
       "105298    180.0         MCCOY, TIMOTHY D             mccoy           timothy   \n",
       "105344    188.0       WILLIAMS, VICTOR K          williams            victor   \n",
       "105577    211.0         ELDER, CHESTER H             elder           chester   \n",
       "105659  13000.0          MACARTHUR, GREG         macarthur              greg   \n",
       "105665  13972.0             ABELE, CHRIS             abele             chris   \n",
       "105829  15000.0               PRICE, SOL             price               sol   \n",
       "106029  13600.0         KIRSCH, STEVEN T            kirsch            steven   \n",
       "106150  22146.0         KIRSCH, STEVEN T            kirsch            steven   \n",
       "106690    813.0         SHAPP, RICHARD A             shapp           richard   \n",
       "107082    949.0       WILLIAMS, VICTOR K          williams            victor   \n",
       "109127   1717.0       HELFRICK, EDWARD W          helfrick            edward   \n",
       "109266      8.0         MCCOY, TIMOTHY D             mccoy           timothy   \n",
       "109541  24150.0         KIRSCH, STEVEN T            kirsch            steven   \n",
       "109586    265.0       WILLIAMS, VICTOR K          williams            victor   \n",
       "109603  25750.0  SUTHERLAND, DWIGHT D JR        sutherland            dwight   \n",
       "109664  45795.0         KIRSCH, STEVEN T            kirsch            steven   \n",
       "109730  28884.0         KIRSCH, STEVEN T            kirsch            steven   \n",
       "109731  30000.0            WOODS, FRED R             woods              fred   \n",
       "\n",
       "       contributor_type   race       api     black  hispanic     white  year  \n",
       "102912                I  white  0.002370  0.364711  0.014732  0.618187  2000  \n",
       "103990                I  white  0.009365  0.004461  0.019418  0.966756  2000  \n",
       "105298                I  white  0.006450  0.292733  0.022897  0.677921  2000  \n",
       "105344                I  black  0.001726  0.515756  0.020712  0.461805  2000  \n",
       "105577                I  white  0.004245  0.165576  0.033800  0.796379  2000  \n",
       "105659                I  white  0.057506  0.028473  0.017616  0.896404  2000  \n",
       "105665                I  white  0.012272  0.072357  0.048589  0.866782  2000  \n",
       "105829                I  white  0.001978  0.194008  0.008592  0.795422  2000  \n",
       "106029                I  white  0.009365  0.004461  0.019418  0.966756  2000  \n",
       "106150                I  white  0.009365  0.004461  0.019418  0.966756  2000  \n",
       "106690                I  white  0.014794  0.109744  0.020067  0.855395  2000  \n",
       "107082                I  black  0.001726  0.515756  0.020712  0.461805  2000  \n",
       "109127                I  white  0.002648  0.020904  0.008069  0.968379  2000  \n",
       "109266                I  white  0.006450  0.292733  0.022897  0.677921  2000  \n",
       "109541                I  white  0.009365  0.004461  0.019418  0.966756  2000  \n",
       "109586                I  black  0.001726  0.515756  0.020712  0.461805  2000  \n",
       "109603                I  white  0.007970  0.081236  0.015958  0.894836  2000  \n",
       "109664                I  white  0.009365  0.004461  0.019418  0.966756  2000  \n",
       "109730                I  white  0.009365  0.004461  0.019418  0.966756  2000  \n",
       "109731                I  white  0.000718  0.373277  0.007999  0.618006  2000  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('/opt/names/fec_contrib/contribDB_2000.csv', usecols=['amount', 'contributor_type', 'contributor_lname', 'contributor_fname', 'contributor_name'])\n",
    "sdf = df[df.contributor_type=='I'].copy()\n",
    "rdf2000 = pred_census_ln(sdf, 'contributor_lname', 2000)\n",
    "rdf2000['year'] = 2000\n",
    "\n",
    "df = pd.read_csv('/opt/names/fec_contrib/contribDB_2010.csv.zip', usecols=['amount', 'contributor_type', 'contributor_lname', 'contributor_fname', 'contributor_name'])\n",
    "sdf = df[df.contributor_type=='I'].copy()\n",
    "rdf2010 = pred_census_ln(sdf, 'contributor_lname', 2010)\n",
    "rdf2010['year'] = 2010\n",
    "\n",
    "rdf = pd.concat([rdf2000, rdf2010])\n",
    "rdf.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  What proportion of contributions were by blacks, whites, Hispanics, and Asians?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style>  \n",
       "<table id=\"T_335b8e92_4c6a_11e8_8cdc_fa7917587175\" > \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" colspan=4>contributor_lname</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level1\" >race</th> \n",
       "        <th class=\"col_heading level1 col0\" >api</th> \n",
       "        <th class=\"col_heading level1 col1\" >black</th> \n",
       "        <th class=\"col_heading level1 col2\" >hispanic</th> \n",
       "        <th class=\"col_heading level1 col3\" >white</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level0\" >year</th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_335b8e92_4c6a_11e8_8cdc_fa7917587175level0_row0\" class=\"row_heading level0 row0\" >2000</th> \n",
       "        <td id=\"T_335b8e92_4c6a_11e8_8cdc_fa7917587175row0_col0\" class=\"data row0 col0\" >1.38%</td> \n",
       "        <td id=\"T_335b8e92_4c6a_11e8_8cdc_fa7917587175row0_col1\" class=\"data row0 col1\" >1.30%</td> \n",
       "        <td id=\"T_335b8e92_4c6a_11e8_8cdc_fa7917587175row0_col2\" class=\"data row0 col2\" >2.39%</td> \n",
       "        <td id=\"T_335b8e92_4c6a_11e8_8cdc_fa7917587175row0_col3\" class=\"data row0 col3\" >94.93%</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_335b8e92_4c6a_11e8_8cdc_fa7917587175level0_row1\" class=\"row_heading level0 row1\" >2010</th> \n",
       "        <td id=\"T_335b8e92_4c6a_11e8_8cdc_fa7917587175row1_col0\" class=\"data row1 col0\" >2.09%</td> \n",
       "        <td id=\"T_335b8e92_4c6a_11e8_8cdc_fa7917587175row1_col1\" class=\"data row1 col1\" >1.00%</td> \n",
       "        <td id=\"T_335b8e92_4c6a_11e8_8cdc_fa7917587175row1_col2\" class=\"data row1 col2\" >3.61%</td> \n",
       "        <td id=\"T_335b8e92_4c6a_11e8_8cdc_fa7917587175row1_col3\" class=\"data row1 col3\" >93.30%</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f1660c3add0>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adf = rdf.groupby(['year', 'race']).agg({'contributor_lname': 'count'})\n",
    "adf.unstack().apply(lambda r: r / r.sum(), axis=1).style.format(\"{:.2%}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### What proportion of the donors were blacks, whites, Hispanics, and Asians?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style>  \n",
       "<table id=\"T_3ab82a2e_4c6a_11e8_8cdc_fa7917587175\" > \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" colspan=4>contributor_name</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level1\" >race</th> \n",
       "        <th class=\"col_heading level1 col0\" >api</th> \n",
       "        <th class=\"col_heading level1 col1\" >black</th> \n",
       "        <th class=\"col_heading level1 col2\" >hispanic</th> \n",
       "        <th class=\"col_heading level1 col3\" >white</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level0\" >year</th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_3ab82a2e_4c6a_11e8_8cdc_fa7917587175level0_row0\" class=\"row_heading level0 row0\" >2000</th> \n",
       "        <td id=\"T_3ab82a2e_4c6a_11e8_8cdc_fa7917587175row0_col0\" class=\"data row0 col0\" >1.75%</td> \n",
       "        <td id=\"T_3ab82a2e_4c6a_11e8_8cdc_fa7917587175row0_col1\" class=\"data row0 col1\" >1.05%</td> \n",
       "        <td id=\"T_3ab82a2e_4c6a_11e8_8cdc_fa7917587175row0_col2\" class=\"data row0 col2\" >2.88%</td> \n",
       "        <td id=\"T_3ab82a2e_4c6a_11e8_8cdc_fa7917587175row0_col3\" class=\"data row0 col3\" >94.33%</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_3ab82a2e_4c6a_11e8_8cdc_fa7917587175level0_row1\" class=\"row_heading level0 row1\" >2010</th> \n",
       "        <td id=\"T_3ab82a2e_4c6a_11e8_8cdc_fa7917587175row1_col0\" class=\"data row1 col0\" >2.91%</td> \n",
       "        <td id=\"T_3ab82a2e_4c6a_11e8_8cdc_fa7917587175row1_col1\" class=\"data row1 col1\" >0.87%</td> \n",
       "        <td id=\"T_3ab82a2e_4c6a_11e8_8cdc_fa7917587175row1_col2\" class=\"data row1 col2\" >4.43%</td> \n",
       "        <td id=\"T_3ab82a2e_4c6a_11e8_8cdc_fa7917587175row1_col3\" class=\"data row1 col3\" >91.80%</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f165f457cd0>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "udf = rdf.drop_duplicates(subset=['contributor_name']).copy()\n",
    "gdf = udf.groupby(['year', 'race']).agg({'contributor_name': 'count'})\n",
    "gdf.unstack().apply(lambda r: r / r.sum(), axis=1).style.format(\"{:.2%}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### What proportion of the total donation was given by blacks, whites, Hispanics, and Asians?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style>  \n",
       "<table id=\"T_3c190d2a_4c6a_11e8_8cdc_fa7917587175\" > \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" colspan=4>amount</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level1\" >race</th> \n",
       "        <th class=\"col_heading level1 col0\" >api</th> \n",
       "        <th class=\"col_heading level1 col1\" >black</th> \n",
       "        <th class=\"col_heading level1 col2\" >hispanic</th> \n",
       "        <th class=\"col_heading level1 col3\" >white</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level0\" >year</th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_3c190d2a_4c6a_11e8_8cdc_fa7917587175level0_row0\" class=\"row_heading level0 row0\" >2000</th> \n",
       "        <td id=\"T_3c190d2a_4c6a_11e8_8cdc_fa7917587175row0_col0\" class=\"data row0 col0\" >1.47%</td> \n",
       "        <td id=\"T_3c190d2a_4c6a_11e8_8cdc_fa7917587175row0_col1\" class=\"data row0 col1\" >1.28%</td> \n",
       "        <td id=\"T_3c190d2a_4c6a_11e8_8cdc_fa7917587175row0_col2\" class=\"data row0 col2\" >2.23%</td> \n",
       "        <td id=\"T_3c190d2a_4c6a_11e8_8cdc_fa7917587175row0_col3\" class=\"data row0 col3\" >95.02%</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_3c190d2a_4c6a_11e8_8cdc_fa7917587175level0_row1\" class=\"row_heading level0 row1\" >2010</th> \n",
       "        <td id=\"T_3c190d2a_4c6a_11e8_8cdc_fa7917587175row1_col0\" class=\"data row1 col0\" >2.15%</td> \n",
       "        <td id=\"T_3c190d2a_4c6a_11e8_8cdc_fa7917587175row1_col1\" class=\"data row1 col1\" >0.74%</td> \n",
       "        <td id=\"T_3c190d2a_4c6a_11e8_8cdc_fa7917587175row1_col2\" class=\"data row1 col2\" >2.51%</td> \n",
       "        <td id=\"T_3c190d2a_4c6a_11e8_8cdc_fa7917587175row1_col3\" class=\"data row1 col3\" >94.60%</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f1660b67a50>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bdf = rdf.groupby(['year', 'race']).agg({'amount': 'sum'})\n",
    "bdf.unstack().apply(lambda r: r / r.sum(), axis=1).style.format(\"{:.2%}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### What if we estimated by using probabilities for race rather than labels?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  What proportion of contributons were by blacks, whites, Hispanics, and Asians?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style>  \n",
       "<table id=\"T_3c190d2b_4c6a_11e8_8cdc_fa7917587175\" > \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" >black_count</th> \n",
       "        <th class=\"col_heading level0 col1\" >white_count</th> \n",
       "        <th class=\"col_heading level0 col2\" >api_count</th> \n",
       "        <th class=\"col_heading level0 col3\" >hispanic_count</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level0\" >year</th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_3c190d2b_4c6a_11e8_8cdc_fa7917587175level0_row0\" class=\"row_heading level0 row0\" >2000</th> \n",
       "        <td id=\"T_3c190d2b_4c6a_11e8_8cdc_fa7917587175row0_col0\" class=\"data row0 col0\" >11.61%</td> \n",
       "        <td id=\"T_3c190d2b_4c6a_11e8_8cdc_fa7917587175row0_col1\" class=\"data row0 col1\" >82.10%</td> \n",
       "        <td id=\"T_3c190d2b_4c6a_11e8_8cdc_fa7917587175row0_col2\" class=\"data row0 col2\" >2.15%</td> \n",
       "        <td id=\"T_3c190d2b_4c6a_11e8_8cdc_fa7917587175row0_col3\" class=\"data row0 col3\" >4.14%</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_3c190d2b_4c6a_11e8_8cdc_fa7917587175level0_row1\" class=\"row_heading level0 row1\" >2010</th> \n",
       "        <td id=\"T_3c190d2b_4c6a_11e8_8cdc_fa7917587175row1_col0\" class=\"data row1 col0\" >11.90%</td> \n",
       "        <td id=\"T_3c190d2b_4c6a_11e8_8cdc_fa7917587175row1_col1\" class=\"data row1 col1\" >78.69%</td> \n",
       "        <td id=\"T_3c190d2b_4c6a_11e8_8cdc_fa7917587175row1_col2\" class=\"data row1 col2\" >3.08%</td> \n",
       "        <td id=\"T_3c190d2b_4c6a_11e8_8cdc_fa7917587175row1_col3\" class=\"data row1 col3\" >6.33%</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f1660c3c250>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdf['white_count'] = rdf.white\n",
    "rdf['black_count'] = rdf.black\n",
    "rdf['api_count'] = rdf.api\n",
    "rdf['hispanic_count'] = rdf.hispanic\n",
    "gdf = rdf.groupby(['year']).agg({'white_count': 'sum', 'black_count': 'sum', 'api_count': 'sum', 'hispanic_count': 'sum'})\n",
    "gdf.apply(lambda r: r / r.sum(), axis=1).style.format(\"{:.2%}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### What proportion of the donors were blacks, whites, Hispanics, and Asians?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style>  \n",
       "<table id=\"T_3d2fe896_4c6a_11e8_8cdc_fa7917587175\" > \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" >black_count</th> \n",
       "        <th class=\"col_heading level0 col1\" >white_count</th> \n",
       "        <th class=\"col_heading level0 col2\" >api_count</th> \n",
       "        <th class=\"col_heading level0 col3\" >hispanic_count</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level0\" >year</th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_3d2fe896_4c6a_11e8_8cdc_fa7917587175level0_row0\" class=\"row_heading level0 row0\" >2000</th> \n",
       "        <td id=\"T_3d2fe896_4c6a_11e8_8cdc_fa7917587175row0_col0\" class=\"data row0 col0\" >10.68%</td> \n",
       "        <td id=\"T_3d2fe896_4c6a_11e8_8cdc_fa7917587175row0_col1\" class=\"data row0 col1\" >82.18%</td> \n",
       "        <td id=\"T_3d2fe896_4c6a_11e8_8cdc_fa7917587175row0_col2\" class=\"data row0 col2\" >2.54%</td> \n",
       "        <td id=\"T_3d2fe896_4c6a_11e8_8cdc_fa7917587175row0_col3\" class=\"data row0 col3\" >4.61%</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_3d2fe896_4c6a_11e8_8cdc_fa7917587175level0_row1\" class=\"row_heading level0 row1\" >2010</th> \n",
       "        <td id=\"T_3d2fe896_4c6a_11e8_8cdc_fa7917587175row1_col0\" class=\"data row1 col0\" >10.38%</td> \n",
       "        <td id=\"T_3d2fe896_4c6a_11e8_8cdc_fa7917587175row1_col1\" class=\"data row1 col1\" >78.54%</td> \n",
       "        <td id=\"T_3d2fe896_4c6a_11e8_8cdc_fa7917587175row1_col2\" class=\"data row1 col2\" >3.96%</td> \n",
       "        <td id=\"T_3d2fe896_4c6a_11e8_8cdc_fa7917587175row1_col3\" class=\"data row1 col3\" >7.12%</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f1660c1f8d0>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "udf['white_count'] = udf.white\n",
    "udf['black_count'] = udf.black\n",
    "udf['api_count'] = udf.api\n",
    "udf['hispanic_count'] = udf.hispanic\n",
    "gdf = udf.groupby(['year']).agg({'white_count': 'sum', 'black_count': 'sum', 'api_count': 'sum', 'hispanic_count': 'sum'})\n",
    "gdf.apply(lambda r: r / r.sum(), axis=1).style.format(\"{:.2%}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### What proportion of the total donation was given by blacks, whites, Hispanics, and Asians?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style>  \n",
       "<table id=\"T_3d2fe897_4c6a_11e8_8cdc_fa7917587175\" > \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" >api_amount</th> \n",
       "        <th class=\"col_heading level0 col1\" >white_amount</th> \n",
       "        <th class=\"col_heading level0 col2\" >black_amount</th> \n",
       "        <th class=\"col_heading level0 col3\" >hispanic_amount</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level0\" >year</th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_3d2fe897_4c6a_11e8_8cdc_fa7917587175level0_row0\" class=\"row_heading level0 row0\" >2000</th> \n",
       "        <td id=\"T_3d2fe897_4c6a_11e8_8cdc_fa7917587175row0_col0\" class=\"data row0 col0\" >4.16</td> \n",
       "        <td id=\"T_3d2fe897_4c6a_11e8_8cdc_fa7917587175row0_col1\" class=\"data row0 col1\" >147.78</td> \n",
       "        <td id=\"T_3d2fe897_4c6a_11e8_8cdc_fa7917587175row0_col2\" class=\"data row0 col2\" >20.83</td> \n",
       "        <td id=\"T_3d2fe897_4c6a_11e8_8cdc_fa7917587175row0_col3\" class=\"data row0 col3\" >7.16</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_3d2fe897_4c6a_11e8_8cdc_fa7917587175level0_row1\" class=\"row_heading level0 row1\" >2010</th> \n",
       "        <td id=\"T_3d2fe897_4c6a_11e8_8cdc_fa7917587175row1_col0\" class=\"data row1 col0\" >14.12</td> \n",
       "        <td id=\"T_3d2fe897_4c6a_11e8_8cdc_fa7917587175row1_col1\" class=\"data row1 col1\" >363.10</td> \n",
       "        <td id=\"T_3d2fe897_4c6a_11e8_8cdc_fa7917587175row1_col2\" class=\"data row1 col2\" >50.52</td> \n",
       "        <td id=\"T_3d2fe897_4c6a_11e8_8cdc_fa7917587175row1_col3\" class=\"data row1 col3\" >24.06</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f1660c7b050>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdf['white_amount'] = rdf.amount * rdf.white\n",
    "rdf['black_amount'] = rdf.amount * rdf.black\n",
    "rdf['api_amount'] = rdf.amount * rdf.api\n",
    "rdf['hispanic_amount'] = rdf.amount * rdf.hispanic\n",
    "gdf = rdf.groupby(['year']).agg({'white_amount': 'sum', 'black_amount': 'sum', 'api_amount': 'sum', 'hispanic_amount': 'sum'}) / 10e6\n",
    "gdf.style.format(\"{:0.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style>  \n",
       "<table id=\"T_3d2fe898_4c6a_11e8_8cdc_fa7917587175\" > \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" >api_amount</th> \n",
       "        <th class=\"col_heading level0 col1\" >white_amount</th> \n",
       "        <th class=\"col_heading level0 col2\" >black_amount</th> \n",
       "        <th class=\"col_heading level0 col3\" >hispanic_amount</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level0\" >year</th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_3d2fe898_4c6a_11e8_8cdc_fa7917587175level0_row0\" class=\"row_heading level0 row0\" >2000</th> \n",
       "        <td id=\"T_3d2fe898_4c6a_11e8_8cdc_fa7917587175row0_col0\" class=\"data row0 col0\" >2.31%</td> \n",
       "        <td id=\"T_3d2fe898_4c6a_11e8_8cdc_fa7917587175row0_col1\" class=\"data row0 col1\" >82.14%</td> \n",
       "        <td id=\"T_3d2fe898_4c6a_11e8_8cdc_fa7917587175row0_col2\" class=\"data row0 col2\" >11.58%</td> \n",
       "        <td id=\"T_3d2fe898_4c6a_11e8_8cdc_fa7917587175row0_col3\" class=\"data row0 col3\" >3.98%</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_3d2fe898_4c6a_11e8_8cdc_fa7917587175level0_row1\" class=\"row_heading level0 row1\" >2010</th> \n",
       "        <td id=\"T_3d2fe898_4c6a_11e8_8cdc_fa7917587175row1_col0\" class=\"data row1 col0\" >3.12%</td> \n",
       "        <td id=\"T_3d2fe898_4c6a_11e8_8cdc_fa7917587175row1_col1\" class=\"data row1 col1\" >80.37%</td> \n",
       "        <td id=\"T_3d2fe898_4c6a_11e8_8cdc_fa7917587175row1_col2\" class=\"data row1 col2\" >11.18%</td> \n",
       "        <td id=\"T_3d2fe898_4c6a_11e8_8cdc_fa7917587175row1_col3\" class=\"data row1 col3\" >5.33%</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f1660c7b250>"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.apply(lambda r: r / r.sum(), axis=1).style.format(\"{:.2%}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}