census-sample-generator.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 20160110-etl-census-with-python\n",
    "\n",
    "Related post:  \n",
    "https://stharrold.github.io/20160110-etl-census-with-python.html\n",
    "\n",
    "Data documentation:  \n",
    "https://www.census.gov/programs-surveys/acs/technical-documentation/pums/documentation.2013.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Import standard packages.\n",
    "import collections\n",
    "import functools\n",
    "import os\n",
    "import pdb # Debug with pdb.\n",
    "import subprocess\n",
    "import sys\n",
    "import time\n",
    "# Import installed packages.\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "# Import local packages.\n",
    "# Insert current directory into module search path.\n",
    "# Autoreload local packages after editing.\n",
    "# `dsdemos` version: https://github.com/stharrold/dsdemos/releases/tag/v0.0.3\n",
    "sys.path.insert(0, os.path.join(os.path.curdir, r'dsdemos'))\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "import dsdemos as dsd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "### Globals\n",
    "\n",
    "File sources:\n",
    "* 2013 5-year PUMS data dictionary: [PUMS_Data_Dictionary_2009-2013.txt](http://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2009-2013.txt) (<1&nbsp;MB)\n",
    "* 2013 5-year PUMS person and housing records for Washington DC:\n",
    "    * Person records: [csv_pdc.zip](http://www2.census.gov/programs-surveys/acs/data/pums/2013/5-Year/csv_pdc.zip) (5&nbsp;MB compressed, 30&nbsp;MB decompressed)\n",
    "    * Housing records: [csv_hdc.zip](http://www2.census.gov/programs-surveys/acs/data/pums/2013/5-Year/csv_hdc.zip) (2&nbsp;MB compressed, 13&nbsp;MB decompressed)\n",
    "* 2013 5-year PUMS estimates for user verification: [pums_estimates_9_13.csv](http://www2.census.gov/programs-surveys/acs/tech_docs/pums/estimates/pums_estimates_9_13.csv) (<1&nbsp;MB)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# File paths\n",
    "basename = r'20160110-etl-census-with-python'\n",
    "filename = basename\n",
    "#path_ipynb = os.path.join(path_static, basename, filename+'.ipynb')\n",
    "path_disk = os.path.abspath(r'/home/joel/census/')\n",
    "path_acs = path_disk\n",
    "\n",
    "#sample constructed by:\n",
    "#sort -R ss13tot.csv | head -n 200000 > ss13sample.csv\n",
    "#head -n 1 ss13tot.csv > headers\n",
    "#cat headers ss13sample.csv > ss13sample_fixed.csv\n",
    "path_pcsv = os.path.join(path_acs, r'ss13sample_fixed.csv') # 'pdc' = 'person DC'\n",
    "path_hcsv = os.path.join(path_acs, r'ss13hdc.csv') # 'hdc' = 'housing DC'\n",
    "path_ecsv = os.path.join(path_acs, r'pums_estimates_9_13.csv')\n",
    "path_dtxt = os.path.join(path_acs, r'PUMS_Data_Dictionary_2009-2013.txt')\n",
    "\n",
    "# Weights\n",
    "pwt = 'PWGTP' # person weight\n",
    "pwts = [pwt+str(inum) for inum in range(1, 81)]\n",
    "hwt = 'WGTP' # housing weight\n",
    "hwts = [hwt+str(inum) for inum in range(1, 81)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PUMS data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "`dfp`, `dfh`: Load person and housing records.\n",
      "/home/joel/census/ss13sample_fixed.csv:\n",
      "    size (MB)   = 188.4\n",
      "    num lines   = 200001\n",
      "    num columns = 283\n",
      "/home/joel/census/ss13hdc.csv:\n",
      "    size (MB)   = 13.5\n",
      "    num lines   = 17501\n",
      "    num columns = 205\n",
      "\n",
      "dfp RAM usage (MB) = 452.8\n",
      "dfh RAM usage (MB) = 28.7\n",
      "\n",
      "Time elapsed (sec) = 4.9\n"
     ]
    }
   ],
   "source": [
    "\n",
    "time_start = time.perf_counter()\n",
    "for path in [path_pcsv]:\n",
    "    with open(path) as fobj:\n",
    "        nlines = sum(1 for _ in fobj)\n",
    "    with open(path) as fobj:\n",
    "        first_line = fobj.readline()\n",
    "        ncols = first_line.count(',')+1\n",
    "    print(\"{path}:\".format(path=path))\n",
    "    print(\"    size (MB)   = {size:.1f}\".format(size=os.path.getsize(path)/1e6))\n",
    "    print(\"    num lines   = {nlines}\".format(nlines=nlines))\n",
    "    print(\"    num columns = {ncols}\".format(ncols=ncols))\n",
    "print()\n",
    "\n",
    "# For ss13pdc.csv, low_memory=False since otherwise pandas raises DtypeWarning.\n",
    "dfp = pd.read_csv(path_pcsv, low_memory=True)\n",
    "\n",
    "for (name, df) in [('dfp', dfp)]:\n",
    "    print(\"{name} RAM usage (MB) = {mem:.1f}\".format(\n",
    "            name=name, mem=df.memory_usage().sum()/1e6))\n",
    "time_stop = time.perf_counter()\n",
    "print()\n",
    "print(\"Time elapsed (sec) = {diff:.1f}\".format(diff=time_stop-time_start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(144069,)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import us\n",
    "\n",
    "state_set = set()\n",
    "def create_intermediate(sample):\n",
    "    age = int(sample['AGEP'])\n",
    "    _mar = sample['MAR']\n",
    "    _gender = sample['SEX'] \n",
    "    _state = \"%02d\" % sample['ST'] \n",
    "    _educ = sample['SCHL']\n",
    "    \n",
    "    gender = 'other'\n",
    "    if _gender==1:\n",
    "        gender='male'\n",
    "    if _gender==2:\n",
    "        gender='female'\n",
    "    \n",
    "    educ='error'\n",
    "    if _educ <=15:\n",
    "        educ = \"no-hs\"\n",
    "    if _educ == 16 or _educ == 17:\n",
    "        educ = 'hs'\n",
    "    if _educ >= 18 and _educ <=20:\n",
    "        educ = 'some-college'\n",
    "    if _educ == 21:\n",
    "        educ = 'college'\n",
    "    if _educ > 21:\n",
    "        educ = 'post-grad'\n",
    "    \n",
    "    race = 'other'\n",
    "    if sample['RACWHT']==1:\n",
    "        race = 'white'\n",
    "    if sample['RACASN']==1:\n",
    "        race = 'asian'\n",
    "    if sample['RACBLK']==1:\n",
    "        race = 'black'\n",
    "    if sample['RACAIAN']==1:\n",
    "        race = 'native'\n",
    "    if sample['RACNH']==1:\n",
    "        race = 'hawaiian'\n",
    "            \n",
    "    state = us.states.lookup(str(_state)).abbr\n",
    "    if (state=='DC'):\n",
    "        state = 'VA' #sorry DC \n",
    "    state_set.add(state)\n",
    "    \n",
    "    mar = None\n",
    "    if _mar == 1:\n",
    "     mar = 'married'\n",
    "    if _mar == 2:\n",
    "     mar = 'widowed'\n",
    "    if _mar == 3 or _mar == 4:\n",
    "     mar = 'divorced'\n",
    "    if _mar == 5:\n",
    "     mar = 'single'\n",
    "    \n",
    "    return {'age':age,'mar':mar,'state':state,'race':race,'educ':educ,'gender':gender}    \n",
    "\n",
    "def pull_sample(data,normalized_weights):\n",
    "    idx = data.index\n",
    "    sample = np.random.choice(idx,p=normalized_weights)\n",
    "    return data.loc[sample]\n",
    "\n",
    "#of voting age; US citizen\n",
    "US_citizen_filter = np.logical_and(dfp['AGEP']>18,dfp['CIT']!=5)\n",
    "US_citizen_filter = np.logical_and(US_citizen_filter,dfp['ST']!=72)\n",
    "filtered_data = dfp[US_citizen_filter]\n",
    "    \n",
    "weights = filtered_data[pwt]\n",
    "normalized_weights = weights/weights.sum()\n",
    "\n",
    "print(weights.shape)\n",
    "\n",
    "samples = []\n",
    "for _ in range(2000):\n",
    " sample = pull_sample(filtered_data,normalized_weights)\n",
    " intermediate = create_intermediate(sample)\n",
    " samples.append(intermediate)\n",
    "   \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import json\n",
    "outfile = open(\"census_sample.json\",\"w\")\n",
    "json.dump(samples,outfile)\n",
    "outfile.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}