03.0 - Population Projections.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import sys, os, time, math, csv\n",
    "import itertools\n",
    "import collections\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load State FIPS code map\n",
    "\n",
    "This is a list of the states that we care about."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "state_fips_to_name = {}\n",
    "state_name_to_fips = {}\n",
    "with open(\"data/state_fips.csv\", \"r\") as f:\n",
    "    for line in f:\n",
    "        line = line.strip()\n",
    "        if line != \"\":\n",
    "            parts = line.split(\",\")\n",
    "            state_fips_to_name[\"%02d\" % (int(parts[2]))] = parts[0]\n",
    "            state_name_to_fips[parts[0]] = \"%02d\" % (int(parts[2]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load regression results for estimated numbers of housing units"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "county_fip_list = []\n",
    "block_group_fip_list = []\n",
    "block_group_estimates = collections.OrderedDict()\n",
    "\n",
    "\n",
    "f = open(\"data/processed/regression_estimated_housing_units.csv\", \"r\")\n",
    "predicted_years = list(map(int, f.readline().strip().split(\",\")[2:]))\n",
    "n = 0\n",
    "m = len(predicted_years)\n",
    "\n",
    "for line in f:\n",
    "    line = line.strip()\n",
    "    if line != \"\":\n",
    "        parts = line.split(\",\")\n",
    "        county_fips = parts[0]\n",
    "        block_group_fips = parts[1]\n",
    "        values = np.array(list(map(float,parts[2:])))\n",
    "        \n",
    "        county_fip_list.append(county_fips)\n",
    "        block_group_fip_list.append(block_group_fips)\n",
    "        block_group_estimates[block_group_fips] = values\n",
    "        \n",
    "        n += 1\n",
    "        assert len(values) == m\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Census 2010 PPHU and GQ data\n",
    "\n",
    "Here we need to account for the geography changes in 2011 and 2012:\n",
    "\n",
    "- https://www.census.gov/programs-surveys/acs/technical-documentation/table-and-geography-changes/2011/geography-changes.html\n",
    "- https://www.census.gov/programs-surveys/acs/technical-documentation/table-and-geography-changes/2012/geography-changes.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This should only be reported for a single BG in Richmond, NY\n",
      "360850089000 Block Group 0, Census Tract 89, Richmond County, New York 0.0 0.0\n"
     ]
    }
   ],
   "source": [
    "mapping_census_2010_to_acs_2012 = {\n",
    "    \"040190027011\" : \"040190027041\", # Block Group 1, Census Tract 27.01, Pima County, Arizona\n",
    "    \"040190027012\" : \"040190027042\", # Block Group 2, Census Tract 27.01, Pima County, Arizona\n",
    "    \"040190029031\" : \"040190029061\", # Block Group 1, Census Tract 29.03, Pima County, Arizona\n",
    "    \"040194105011\" : \"040190041181\", # Block Group 1, Census Tract 4105.01, Pima County, Arizona\n",
    "    \"040194105021\" : \"040190041211\", # Block Group 1, Census Tract 4105.02, Pima County, Arizona\n",
    "    \"040194105031\" : \"040190041251\", # Block Group 1, Census Tract 4105.03, Pima County, Arizona\n",
    "    \"040194105032\" : \"040190041252\", # Block Group 2, Census Tract 4105.03, Pima County, Arizona\n",
    "    \"040194704001\" : \"040190052001\", # Block Group 1, Census Tract 4704, Pima County, Arizona\n",
    "    \"040194704002\" : \"040190052002\", # Block Group 2, Census Tract 4704, Pima County, Arizona\n",
    "    \"040194704003\" : \"040190052003\", # Block Group 3, Census Tract 4704, Pima County, Arizona\n",
    "    \"040194704004\" : \"040190052004\", # Block Group 4, Census Tract 4704, Pima County, Arizona\n",
    "    \"040194705001\" : \"040190053001\", # Block Group 1, Census Tract 4705, Pima County, Arizona\n",
    "    \"040194705002\" : \"040190053002\", # Block Group 2, Census Tract 4705, Pima County, Arizona\n",
    "    \"060378002043\" : \"060371370001\", # Block Group 3, Census Tract 8002.04, Los Angeles County, California\n",
    "    \"060379304011\" : \"060371370002\", # Block Group 1, Census Tract 9304.01, Los Angeles County, California\n",
    "    \"360539401011\" : \"360530301011\", # Block Group 1, Census Tract 9401.01, Madison County, New York\n",
    "    \"360539401012\" : \"360530301012\", # Block Group 2, Census Tract 9401.01, Madison County, New York\n",
    "    \"360539401021\" : \"360530301021\", # Block Group 1, Census Tract 9401.02, Madison County, New York\n",
    "    \"360539401022\" : \"360530301022\", # Block Group 2, Census Tract 9401.02, Madison County, New York\n",
    "    \"360539401023\" : \"360530301023\", # Block Group 3, Census Tract 9401.02, Madison County, New York\n",
    "    \"360539401024\" : \"360530301024\", # Block Group 4, Census Tract 9401.02, Madison County, New York\n",
    "    \"360539401031\" : \"360530301031\", # Block Group 1, Census Tract 9401.03, Madison County, New York\n",
    "    \"360539401032\" : \"360530301032\", # Block Group 2, Census Tract 9401.03, Madison County, New York\n",
    "    \"360539401033\" : \"360530301033\", # Block Group 3, Census Tract 9401.03, Madison County, New York\n",
    "    \"360539402001\" : \"360530302001\", # Block Group 1, Census Tract 9402, Madison County, New York\n",
    "    \"360539402002\" : \"360530302002\", # Block Group 2, Census Tract 9402, Madison County, New York\n",
    "    \"360539402003\" : \"360530302003\", # Block Group 3, Census Tract 9402, Madison County, New York\n",
    "    \"360539403001\" : \"360530303001\", # Block Group 1, Census Tract 9403, Madison County, New York\n",
    "    \"360539403002\" : \"360530303002\", # Block Group 2, Census Tract 9403, Madison County, New York\n",
    "    \"360539403003\" : \"360530303003\", # Block Group 3, Census Tract 9403, Madison County, New York\n",
    "    \"360539403004\" : \"360530303004\", # Block Group 4, Census Tract 9403, Madison County, New York\n",
    "    \"360539404011\" : \"360530304011\", # Block Group 1, Census Tract 9404.01, Madison County, New York\n",
    "    \"360539404012\" : \"360530304012\", # Block Group 2, Census Tract 9404.01, Madison County, New York\n",
    "    \"360539404013\" : \"360530304013\", # Block Group 3, Census Tract 9404.01, Madison County, New York\n",
    "    \"360539404014\" : \"360530304014\", # Block Group 4, Census Tract 9404.01, Madison County, New York\n",
    "    \"360539404015\" : \"360530304015\", # Block Group 5, Census Tract 9404.01, Madison County, New York\n",
    "    \"360539404031\" : \"360530304031\", # Block Group 1, Census Tract 9404.03, Madison County, New York\n",
    "    \"360539404032\" : \"360530304032\", # Block Group 2, Census Tract 9404.03, Madison County, New York\n",
    "    \"360539404033\" : \"360530304033\", # Block Group 3, Census Tract 9404.03, Madison County, New York\n",
    "    \"360539406001\" : \"360530306001\", # Block Group 1, Census Tract 9406, Madison County, New York\n",
    "    \"360539406002\" : \"360530306002\", # Block Group 2, Census Tract 9406, Madison County, New York\n",
    "    \"360539406003\" : \"360530306003\", # Block Group 3, Census Tract 9406, Madison County, New York\n",
    "    \"360539406004\" : \"360530306004\", # Block Group 4, Census Tract 9406, Madison County, New York\n",
    "    \"360539407001\" : \"360530304021\", # Block Group 1, Census Tract 9407, Madison County, New York\n",
    "    \"360539407002\" : \"360530304022\", # Block Group 2, Census Tract 9407, Madison County, New York\n",
    "    \"360659400001\" : \"360650248001\", # Block Group 1, Census Tract 9400, Oneida County, New York\n",
    "    \"360659400002\" : \"360650248002\", # Block Group 2, Census Tract 9400, Oneida County, New York\n",
    "    \"360659401001\" : \"360650247001\", # Block Group 1, Census Tract 9401, Oneida County, New York\n",
    "    \"360659401002\" : \"360650247002\", # Block Group 2, Census Tract 9401, Oneida County, New York\n",
    "    \"360659401003\" : \"360650247003\", # Block Group 3, Census Tract 9401, Oneida County, New York\n",
    "    \"360659401004\" : \"360650247004\", # Block Group 4, Census Tract 9401, Oneida County, New York\n",
    "    \"360659402001\" : \"360650249001\", # Block Group 1, Census Tract 9402, Oneida County, New York\n",
    "    \"360659402002\" : \"360650249002\", # Block Group 2, Census Tract 9402, Oneida County, New York\n",
    "    \"360659402003\" : \"360650249003\", # Block Group 3, Census Tract 9402, Oneida County, New York\n",
    "    #\"360850089000\" : \"WATER\" # Block Group 0, Census Tract 89, Richmond County, New York\n",
    "}\n",
    "\n",
    "census_2010_gq = collections.OrderedDict()\n",
    "census_2010_pphu = collections.OrderedDict()\n",
    "\n",
    "census_2010_hu = collections.OrderedDict()\n",
    "census_2010_pop_hu = collections.OrderedDict()\n",
    "\n",
    "with open(\"data/raw/R11633875_SL150.txt\", \"r\", encoding=\"latin-1\") as f:\n",
    "    reader = csv.DictReader(f, delimiter=\"\\t\")\n",
    "\n",
    "    for row in reader:\n",
    "        state_code = row[\"Geo_STATE\"]\n",
    "        county_code = row[\"Geo_COUNTY\"]\n",
    "        county_fips = state_code + county_code\n",
    "        fips_code = row[\"Geo_FIPS\"]\n",
    "\n",
    "        if state_code in state_fips_to_name:\n",
    "            \n",
    "            pphu_value = float(row[\"SF1_P0170001\"])\n",
    "            gq_value = float(row[\"SF1_P0420001\"])\n",
    "            \n",
    "            hu_value = float(row[\"SF1_H0010001\"])\n",
    "            pop_hu_value = float(row[\"SF1_H0100001\"])\n",
    "            \n",
    "            if not fips_code in block_group_estimates:\n",
    "            \n",
    "                if fips_code in mapping_census_2010_to_acs_2012:\n",
    "                    fips_code = mapping_census_2010_to_acs_2012[fips_code]\n",
    "                else:\n",
    "                    print(\"This should only be reported for a single BG in Richmond, NY\")\n",
    "                    print(fips_code, row[\"Geo_QName\"], gq_value, pphu_value)\n",
    "\n",
    "            census_2010_gq[fips_code] = gq_value\n",
    "            census_2010_pphu[fips_code] = pphu_value\n",
    "            census_2010_hu[fips_code] = hu_value\n",
    "            census_2010_pop_hu[fips_code] = pop_hu_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for fips_code in block_group_estimates.keys():\n",
    "    assert fips_code in census_2010_gq"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create population estimates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted_population = []\n",
    "for i in range(n):    \n",
    "    block_group_fips = block_group_fip_list[i]\n",
    "    \n",
    "    pphu = census_2010_pphu[block_group_fips]\n",
    "    gq = census_2010_gq[block_group_fips]\n",
    "    estimated_housing_units = block_group_estimates[block_group_fips]\n",
    "    \n",
    "    pphu_2 = 0.0\n",
    "    if census_2010_hu[block_group_fips] == 0:\n",
    "        if not census_2010_pop_hu[block_group_fips] == 0:\n",
    "            print(census_2010_pop_hu[block_group_fips])    \n",
    "    else:\n",
    "        pphu_2 = census_2010_pop_hu[block_group_fips] / census_2010_hu[block_group_fips]\n",
    "    \n",
    "    estimated_population = estimated_housing_units * pphu_2 + gq\n",
    "    predicted_population.append(estimated_population)\n",
    "    \n",
    "predicted_population = np.array(predicted_population)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Write population estimates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open(\"data/processed/regression_estimated_population.csv\", \"w\")\n",
    "f.write(\"Geo_COUNTY_FIPS,Geo_BG_FIPS,%s\\n\" % (\",\".join(map(str, predicted_years))))\n",
    "for i in range(n):\n",
    "    f.write(\"%s,%s,%s\\n\" % (\n",
    "        county_fip_list[i],\n",
    "        block_group_fip_list[i],\n",
    "        \",\".join(map(str, predicted_population[i,:]))\n",
    "    ))\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}