notebooks/Day_10_A_fixed_width_parsing_completed.ipynb

{
 "metadata": {
  "name": "Day_10_A_fixed_width_parsing_completed"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Goals"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "To parse text files with fixed columns such as [Census DataDict.txt](https://raw.github.com/rdhyee/working-open-data/master/data/census/DataDict.txt)"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Reading off the columns\n",
      " \n",
      "* 0, 8\n",
      "* 10, 111\n",
      "* 115, 117\n",
      "* 122\n",
      "* 129, 137\n",
      "* 144, 149\n",
      "* 153, 161\n",
      "* 164, 169\n",
      " \n",
      "[u'Data_Item', u'Item_Description', u'Unit', u'Decimal', u'US_Total', u'Minimum', u'Maximum', u'Source']"
     ]
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Let's use requests to read file from github\n"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Hint:  if you use requests to read the file, you may need to turn verify off for requests.get: http://stackoverflow.com/questions/10667960/python-requests-throwing-up-sslerror"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from itertools import islice\n",
      "import requests\n",
      "import StringIO\n",
      "import os\n",
      "\n",
      "cafile = os.path.join(os.pardir, \"data/cacert.pem\")\n",
      "datadict_url = \"https://raw.github.com/rdhyee/working-open-data/5ef3932b4ff7cadf1f06ca01eb852ad71361894a/data/census/DataDict.txt\"\n",
      "r = requests.get(datadict_url, verify=cafile)\n",
      "\n",
      "f = StringIO.StringIO(r.content.decode(\"iso-8859-1\"))\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "r.text"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 2,
       "text": [
        "u\"Data_Item                                             Item_Description                                             UnitDecimal  US_Total    Minimum     Maximum     Source\\nSTATECOU  FIPS State and County code\\nPST045212 Resident total population estimate (July 1) 2012                                                         ABS    0      313914040      576412   313914040  CENSUS\\nPST045211 Resident total population estimate (July 1) 2011                                                         ABS    0      311587816          90   311587816  CENSUS\\nPST040210 Resident total population, estimates base (April 1) 2010                                                 ABS    0      308747508          82   308747508  CENSUS\\nPST120212 Resident total population, percent change - April 1, 2010 to July 1, 2012                                PCT    1            1.7        -0.2         5.1  CENSUS\\nPST120211 Resident total population, percent change - April 1, 2010 to July 1, 2011                                PCT    1            0.9       -18.1        14.6  CENSUS\\nPOP010210 Resident population (April 1 - complete count) 2010                                                      ABS    0      308745538          82   308745538  CENSUS\\nAGE135211 Resident population under 5 years, percent, 2011                                                         PCT    1            6.5         0.0        13.3  CENSUS\\nAGE295211 Resident population under 18 years, percent, 2011                                                        PCT    1           23.7         0.0        41.0  CENSUS\\nAGE775211 Resident population 65 years and over, percent, 2011                                                     PCT    1           13.3         3.7        45.5  CENSUS\\nSEX255211 Resident population: total females, percent, 2011                                                        PCT    1           50.8        28.7        56.8  CENSUS\\nRHI125211 Resident population: White alone, percent, 2011                                                          PCT    1           78.1         3.9        99.7  CENSUS\\nRHI225211 Resident population: Black alone, percent, 2011                                                          PCT    1           13.1         0.0        84.7  CENSUS\\nRHI325211 Resident population: American Indian and Alaska Native alone, percent, 2011                              PCT    1            1.2         0.0        93.3  CENSUS\\nRHI425211 Resident population: Asian alone, percent, 2011                                                          PCT    1            5.0         0.0        43.6  CENSUS\\nRHI525211 Resident population: Native Hawaiian and Other Pacific Islander alone, percent, 2011                     PCT    1            0.2         0.0        48.9  CENSUS\\nRHI625211 Resident population: Two or more races, percent, 2011                                                    PCT    1            2.3         0.0        29.2  CENSUS\\nRHI725211 Resident population: Hispanic or Latino Origin, percent, 2011                                            PCT    1           16.7         0.0        95.6  CENSUS\\nRHI825211 Resident population: Not Hispanic, White alone, percent, 2011                                            PCT    1           63.4         3.2        98.9  CENSUS\\nPOP715211 Population 1 year and over by residence - same house, one year ago, percent, 2007-2011                   PCT    1           84.6        49.0       100.0  CENSUS\\nPOP645211 Place of birth, foreign born, percent,  2007-2011                                                        PCT    1           12.8         0.0        63.4  CENSUS\\nPOP815211 Population 5 years and over, percent speaking language other than English at home, 2007-2011             PCT    1           20.3         0.0        95.9  CENSUS\\nEDU635211 Educational attainment - persons 25 years and over - percent high school graduate or higher, 2007-2011   PCT    1           85.4        46.3        98.6  CENSUS\\nEDU685211 Educational attainment - persons 25 years and over - percent bachelor's degree or higher, 2007-2011      PCT    1           28.2         4.2        72.0  CENSUS\\nVET605211 Veterans - total, 2007-2011                                                                              ABS    0       22215303           0    22215303  CENSUS\\nLFE305211 Average travel time to work for workers 16 years and over not working at home, 2007-2011                 MIN    1           25.4         4.3        42.5  CENSUS\\nHSG010211 Housing unit estimates, 2011                                                                             ABS    0      132312404          48   132312404  CENSUS\\nHSG445211 Owner-occupied housing units - percent of total occupied housing units, 2007-2011                        PCT    1           66.1         0.0        93.7  CENSUS\\nHSG096211 Housing units by units in structure - multi-dwelling structure, percent, 2007-2011                       PCT    1           25.9         0.0        98.4  CENSUS\\nHSG495211 Median value of specified owner-occupied housing units, 2007-2011                                        DOL    0         186200           0      993900  CENSUS\\nHSD410211 Households, 2007-2011                                                                                    ABS    0      114761359          27   114761359  CENSUS\\nHSD310211 Average household size, 2007-2011                                                                        AVG    2           2.60        1.20        4.77  CENSUS\\nINC910211 Per capita income in the past 12 months (in 2011 inflation-adjusted dollars), 2007-2011                  DOL    0          27915        7887       61290  CENSUS\\nINC110211 Median household income, 2007-2011                                                                       DOL    0          52762       19344      120332  CENSUS\\nPVY020211 People of all ages in poverty, percent, 2007-2011                                                        PCT    1           14.3         0.0        53.5  CENSUS\\nBZA010210 Private nonfarm establishments, 2010                                                                     ABS    0        7396628           0     7396628  CENSUS\\nBZA110210 Private nonfarm employment for pay period including March 12, 2010                                       ABS    0      111970095           0   111970095  CENSUS\\nBZA115210 Private nonfarm employment for pay period including March 12, 2010, percent change, 2000-2010            PCT    1           -1.8       -82.7       385.8  CENSUS\\nNES010210 Nonemployer: total (NAICS 00) - establishments, 2010                                                     ABS    0       22110628          17    22110628  CENSUS\\nSBO001207 Total number of firms, 2007                                                                              ABS    0       27092908          27    27092908  CENSUS\\nSBO315207 Total Black-owned firms, percent, 2007                                                                   PCT    1            7.1         0.2        66.7  CENSUS\\nSBO115207 Total American Indian- and Alaska Native-owned firms, percent, 2007                                      PCT    1            0.9         0.2        71.8  CENSUS\\nSBO215207 Total Asian-owned firms, percent, 2007                                                                   PCT    1            5.7         0.3        56.6  CENSUS\\nSBO515207 Total Native Hawaiian- and Other Pacific Islander-owned firms, percent, 2007                             PCT    1            0.1         0.0        10.5  CENSUS\\nSBO415207 Total Hispanic-owned firms, percent, 2007                                                                PCT    1            8.3         0.3        78.0  CENSUS\\nSBO015207 Total Women-owned firms, percent, 2007                                                                   PCT    1           28.8         6.5        56.2  CENSUS\\nMAN450207 Manufacturing: total (NAICS 31-33) - value of shipments, 2007                                            TH$    0     5338306501       56903  5338306501  CENSUS\\nWTN220207 Wholesale trade: merchant wholesalers (NAICS 42) - sales of establishments with payroll, 2007            TH$    0     4174286516           0  4174286516  CENSUS\\nRTN130207 Retail trade: total (NAICS 44-45) - sales of establishments with payroll, 2007                           TH$    0     3917663456           0  3917663456  CENSUS\\nRTN131207 Retail trade: total (NAICS 44-45) - sales of establishments with payroll per capita, 2007                DOL    0          12990           0       80800  CENSUS\\nAFN120207 Accommodation and Food Services: total (NAICS 72) - sales of establishments with payroll, 2007           TH$    0      613795732           0   613795732  CENSUS\\nBPS030211 New private housing units authorized by building permits - total, 2011 (20,000-place universe)           ABS    0         624061           0      624061  CENSUS\\nLND110210 Land area in square miles, 2010                                                                          SQM    2     3531905.43           2  3531905.43  CENSUS\\nPOP060210 Population per square mile, 2010                                                                         RTE    1           87.4         0.0     69467.5  CENSUS\\n\""
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Parse by counting off the column widths and parsing"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import os\n",
      "import codecs\n",
      "from itertools import islice, izip\n",
      "from pandas import DataFrame"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "datadict_path = os.path.join(os.pardir, \"data/census/DataDict.txt\")\n",
      "\n",
      "# f = islice(codecs.open(datadict_path, mode=\"rU\", encoding=\"iso-8859-1\"), None)\n",
      "\n",
      "cafile = os.path.join(os.pardir, \"data/cacert.pem\")\n",
      "datadict_url = \"https://raw.github.com/rdhyee/working-open-data/5ef3932b4ff7cadf1f06ca01eb852ad71361894a/data/census/DataDict.txt\"\n",
      "r = requests.get(datadict_url, verify=cafile)\n",
      "\n",
      "f = StringIO.StringIO(r.content.decode(\"iso-8859-1\"))\n",
      "\n",
      "header_row = f.next()\n",
      "header_row.split()\n",
      "\n",
      "headers = [u'Data_Item',\n",
      " u'Item_Description',\n",
      " u'Unit',\n",
      " u'Decimal',\n",
      " u'US_Total',\n",
      " u'Minimum',\n",
      " u'Maximum',\n",
      " u'Source']\n",
      "\n",
      "column_boundaries = [(0, 8),\n",
      "  (10, 111),\n",
      "  (115, 117),\n",
      "  (122, 122),\n",
      "  (129, 137),\n",
      "  (144, 149),\n",
      "  (153, 161),\n",
      "  (164, 169)\n",
      "]\n",
      "\n",
      "header_to_columns = dict(izip(headers, column_boundaries))\n",
      "\n",
      "# skip 2nd row -- exceptional\n",
      "\n",
      "rows = islice(f, 1, None)\n",
      "parsed_rows = []\n",
      "\n",
      "for row in rows:\n",
      "    row_dict = {}\n",
      "    for (header, bound) in header_to_columns.iteritems():\n",
      "        row_dict[header] = row[bound[0]:bound[1]+1]\n",
      "    parsed_rows.append(row_dict)\n",
      "    \n",
      "data_dict_df = DataFrame(parsed_rows)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# TEST\n",
      "\n",
      "assert set(data_dict_df.columns) == set([u'Decimal',\n",
      "     u'Maximum',\n",
      "     u'Source',\n",
      "     u'Minimum',\n",
      "     u'Unit',\n",
      "     u'US_Total',\n",
      "     u'Data_Item',\n",
      "     u'Item_Description'])\n",
      "\n",
      "assert set(data_dict_df[\"Data_Item\"]) == set([u'RHI125211',\n",
      "     u'SBO415207', u'VET605211', u'RHI225211', u'PVY020211', u'HSD310211',\n",
      "     u'POP645211', u'EDU635211', u'EDU685211', u'RHI625211', u'SBO215207',\n",
      "     u'PST045212', u'SBO015207', u'POP715211', u'PST120211', u'PST120212',\n",
      "     u'POP010210', u'PST045211', u'SBO315207', u'POP060210', u'RHI425211',\n",
      "     u'POP815211', u'HSD410211', u'HSG495211', u'BZA010210', u'LFE305211',\n",
      "     u'BZA110210', u'AGE775211', u'HSG096211', u'RHI525211', u'LND110210',\n",
      "     u'PST040210', u'RHI825211', u'BZA115210', u'NES010210', u'MAN450207',\n",
      "     u'AGE135211', u'RTN131207', u'RHI725211', u'BPS030211', u'INC110211',\n",
      "     u'AGE295211', u'SBO115207', u'INC910211', u'RHI325211', u'WTN220207',\n",
      "     u'HSG445211', u'SBO515207', u'AFN120207', u'RTN130207', u'HSG010211',\n",
      "     u'SEX255211', u'SBO001207'])\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "k = np.array(list('hi there! '), np.dtype('U1'))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "k.dtype"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 7,
       "text": [
        "dtype('<U1')"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "len(k)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "10"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "k[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 9,
       "text": [
        "u'h'"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "k[1]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 10,
       "text": [
        "u'i'"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "np.char.isspace(k)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 11,
       "text": [
        "array([False, False,  True, False, False, False, False, False, False,  True], dtype=bool)"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "r1 = \"12 123 456\"\n",
      "r2 = \"23 455  xx\"\n",
      "\n",
      "a1= np.frombuffer(r1, dtype='S1')\n",
      "a2 = np.frombuffer(r1, dtype='S1')\n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "np.char.isspace(a1) & np.char.isspace(a2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 13,
       "text": [
        "array([False, False,  True, False, False, False,  True, False, False, False], dtype=bool)"
       ]
      }
     ],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "u\"1\".encode('utf-8')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 14,
       "text": [
        "'1'"
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rows = [\"12 123 456\",\n",
      "        \"23 455  xx\",\n",
      "        \" 4 789 333\"]\n",
      "\n",
      "m = np.char.isspace(np.vstack((np.frombuffer(row, dtype='S1') for row in rows)))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "m"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 16,
       "text": [
        "array([[False, False,  True, False, False, False,  True, False, False,\n",
        "        False],\n",
        "       [False, False,  True, False, False, False,  True,  True, False,\n",
        "        False],\n",
        "       [ True, False,  True, False, False, False,  True, False, False,\n",
        "        False]], dtype=bool)"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "m.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 17,
       "text": [
        "(3, 10)"
       ]
      }
     ],
     "prompt_number": 17
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n",
      "\n",
      "cols_isspace = pd.Series([np.all(col) for col in m.T])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "cols_isspace[cols_isspace].index"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 19,
       "text": [
        "Int64Index([2, 6], dtype=int64)"
       ]
      }
     ],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "m"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 20,
       "text": [
        "array([[False, False,  True, False, False, False,  True, False, False,\n",
        "        False],\n",
        "       [False, False,  True, False, False, False,  True,  True, False,\n",
        "        False],\n",
        "       [ True, False,  True, False, False, False,  True, False, False,\n",
        "        False]], dtype=bool)"
       ]
      }
     ],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "np.where(np.all(m, 0))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 21,
       "text": [
        "(array([2, 6]),)"
       ]
      }
     ],
     "prompt_number": 21
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "parsing using sets"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from itertools import islice, izip, groupby\n",
      "import re\n",
      "import os\n",
      "import codecs\n",
      "import operator\n",
      "\n",
      "DATA_DIR = os.path.join(os.pardir, \"data\")\n",
      "\n",
      "f = codecs.open(os.path.join(DATA_DIR, \"census/DataDict.txt\"), encoding=\"iso-8859-1\")\n",
      "\n",
      "f_sliced = islice(f, None)\n",
      "\n",
      "head_row = f_sliced.next()\n",
      "headers = head_row.split()\n",
      "\n",
      "print headers\n",
      "\n",
      "# Actually, Unit should be broken off from Decimal\n",
      "\n",
      "# skip the second row also\n",
      "header2 = f_sliced.next()\n",
      "\n",
      "# read in all the rows\n",
      "\n",
      "rows = list([r[:-1] for r in islice(f_sliced,None)])\n",
      "\n",
      "# What's the max length of rows?\n",
      "\n",
      "max_len = max([len(row) for row in rows])\n",
      "print max_len\n",
      "\n",
      "# loop through all rows, looking for which columns have spaces exclusively\n",
      "\n",
      "cols_with_space = set(range(max_len))\n",
      "\n",
      "for row in rows:\n",
      "    cols_with_space_in_row = set([m.start() for m in re.finditer(' ', row)])\n",
      "    cols_with_space.intersection_update(cols_with_space_in_row)\n",
      "\n",
      "cols_with_data = set(range(max_len)) - cols_with_space\n",
      "# print sorted(cols_with_data)\n",
      "\n",
      "# http://code.activestate.com/recipes/496682-make-ranges-of-contiguous-numbers-from-a-list-of-i/#c2\n",
      "\n",
      "ranges = [map(operator.itemgetter(1), g) for k, g in groupby(enumerate(sorted(cols_with_data)), lambda (i,x):i-x) ]\n",
      "print [(r[0], r[-1]+1) for r in ranges]\n",
      "\n",
      "for row in rows:\n",
      "    print [row[r[0]:r[-1]+1].strip() for r in ranges]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[u'Data_Item', u'Item_Description', u'UnitDecimal', u'US_Total', u'Minimum', u'Maximum', u'Source']\n",
        "170\n",
        "[(0, 9), (10, 112), (115, 118), (122, 123), (128, 138), (144, 150), (152, 162), (164, 170)]\n",
        "[u'PST045212', u'Resident total population estimate (July 1) 2012', u'ABS', u'0', u'313914040', u'576412', u'313914040', u'CENSUS']\n",
        "[u'PST045211', u'Resident total population estimate (July 1) 2011', u'ABS', u'0', u'311587816', u'90', u'311587816', u'CENSUS']\n",
        "[u'PST040210', u'Resident total population, estimates base (April 1) 2010', u'ABS', u'0', u'308747508', u'82', u'308747508', u'CENSUS']\n",
        "[u'PST120212', u'Resident total population, percent change - April 1, 2010 to July 1, 2012', u'PCT', u'1', u'1.7', u'-0.2', u'5.1', u'CENSUS']\n",
        "[u'PST120211', u'Resident total population, percent change - April 1, 2010 to July 1, 2011', u'PCT', u'1', u'0.9', u'-18.1', u'14.6', u'CENSUS']\n",
        "[u'POP010210', u'Resident population (April 1 - complete count) 2010', u'ABS', u'0', u'308745538', u'82', u'308745538', u'CENSUS']\n",
        "[u'AGE135211', u'Resident population under 5 years, percent, 2011', u'PCT', u'1', u'6.5', u'0.0', u'13.3', u'CENSUS']\n",
        "[u'AGE295211', u'Resident population under 18 years, percent, 2011', u'PCT', u'1', u'23.7', u'0.0', u'41.0', u'CENSUS']\n",
        "[u'AGE775211', u'Resident population 65 years and over, percent, 2011', u'PCT', u'1', u'13.3', u'3.7', u'45.5', u'CENSUS']\n",
        "[u'SEX255211', u'Resident population: total females, percent, 2011', u'PCT', u'1', u'50.8', u'28.7', u'56.8', u'CENSUS']\n",
        "[u'RHI125211', u'Resident population: White alone, percent, 2011', u'PCT', u'1', u'78.1', u'3.9', u'99.7', u'CENSUS']\n",
        "[u'RHI225211', u'Resident population: Black alone, percent, 2011', u'PCT', u'1', u'13.1', u'0.0', u'84.7', u'CENSUS']\n",
        "[u'RHI325211', u'Resident population: American Indian and Alaska Native alone, percent, 2011', u'PCT', u'1', u'1.2', u'0.0', u'93.3', u'CENSUS']\n",
        "[u'RHI425211', u'Resident population: Asian alone, percent, 2011', u'PCT', u'1', u'5.0', u'0.0', u'43.6', u'CENSUS']\n",
        "[u'RHI525211', u'Resident population: Native Hawaiian and Other Pacific Islander alone, percent, 2011', u'PCT', u'1', u'0.2', u'0.0', u'48.9', u'CENSUS']\n",
        "[u'RHI625211', u'Resident population: Two or more races, percent, 2011', u'PCT', u'1', u'2.3', u'0.0', u'29.2', u'CENSUS']\n",
        "[u'RHI725211', u'Resident population: Hispanic or Latino Origin, percent, 2011', u'PCT', u'1', u'16.7', u'0.0', u'95.6', u'CENSUS']\n",
        "[u'RHI825211', u'Resident population: Not Hispanic, White alone, percent, 2011', u'PCT', u'1', u'63.4', u'3.2', u'98.9', u'CENSUS']\n",
        "[u'POP715211', u'Population 1 year and over by residence - same house, one year ago, percent, 2007-2011', u'PCT', u'1', u'84.6', u'49.0', u'100.0', u'CENSUS']\n",
        "[u'POP645211', u'Place of birth, foreign born, percent,  2007-2011', u'PCT', u'1', u'12.8', u'0.0', u'63.4', u'CENSUS']\n",
        "[u'POP815211', u'Population 5 years and over, percent speaking language other than English at home, 2007-2011', u'PCT', u'1', u'20.3', u'0.0', u'95.9', u'CENSUS']\n",
        "[u'EDU635211', u'Educational attainment - persons 25 years and over - percent high school graduate or higher, 2007-2011', u'PCT', u'1', u'85.4', u'46.3', u'98.6', u'CENSUS']\n",
        "[u'EDU685211', u\"Educational attainment - persons 25 years and over - percent bachelor's degree or higher, 2007-2011\", u'PCT', u'1', u'28.2', u'4.2', u'72.0', u'CENSUS']\n",
        "[u'VET605211', u'Veterans - total, 2007-2011', u'ABS', u'0', u'22215303', u'0', u'22215303', u'CENSUS']\n",
        "[u'LFE305211', u'Average travel time to work for workers 16 years and over not working at home, 2007-2011', u'MIN', u'1', u'25.4', u'4.3', u'42.5', u'CENSUS']\n",
        "[u'HSG010211', u'Housing unit estimates, 2011', u'ABS', u'0', u'132312404', u'48', u'132312404', u'CENSUS']\n",
        "[u'HSG445211', u'Owner-occupied housing units - percent of total occupied housing units, 2007-2011', u'PCT', u'1', u'66.1', u'0.0', u'93.7', u'CENSUS']\n",
        "[u'HSG096211', u'Housing units by units in structure - multi-dwelling structure, percent, 2007-2011', u'PCT', u'1', u'25.9', u'0.0', u'98.4', u'CENSUS']\n",
        "[u'HSG495211', u'Median value of specified owner-occupied housing units, 2007-2011', u'DOL', u'0', u'186200', u'0', u'993900', u'CENSUS']\n",
        "[u'HSD410211', u'Households, 2007-2011', u'ABS', u'0', u'114761359', u'27', u'114761359', u'CENSUS']\n",
        "[u'HSD310211', u'Average household size, 2007-2011', u'AVG', u'2', u'2.60', u'1.20', u'4.77', u'CENSUS']\n",
        "[u'INC910211', u'Per capita income in the past 12 months (in 2011 inflation-adjusted dollars), 2007-2011', u'DOL', u'0', u'27915', u'7887', u'61290', u'CENSUS']\n",
        "[u'INC110211', u'Median household income, 2007-2011', u'DOL', u'0', u'52762', u'19344', u'120332', u'CENSUS']\n",
        "[u'PVY020211', u'People of all ages in poverty, percent, 2007-2011', u'PCT', u'1', u'14.3', u'0.0', u'53.5', u'CENSUS']\n",
        "[u'BZA010210', u'Private nonfarm establishments, 2010', u'ABS', u'0', u'7396628', u'0', u'7396628', u'CENSUS']\n",
        "[u'BZA110210', u'Private nonfarm employment for pay period including March 12, 2010', u'ABS', u'0', u'111970095', u'0', u'111970095', u'CENSUS']\n",
        "[u'BZA115210', u'Private nonfarm employment for pay period including March 12, 2010, percent change, 2000-2010', u'PCT', u'1', u'-1.8', u'-82.7', u'385.8', u'CENSUS']\n",
        "[u'NES010210', u'Nonemployer: total (NAICS 00) - establishments, 2010', u'ABS', u'0', u'22110628', u'17', u'22110628', u'CENSUS']\n",
        "[u'SBO001207', u'Total number of firms, 2007', u'ABS', u'0', u'27092908', u'27', u'27092908', u'CENSUS']\n",
        "[u'SBO315207', u'Total Black-owned firms, percent, 2007', u'PCT', u'1', u'7.1', u'0.2', u'66.7', u'CENSUS']\n",
        "[u'SBO115207', u'Total American Indian- and Alaska Native-owned firms, percent, 2007', u'PCT', u'1', u'0.9', u'0.2', u'71.8', u'CENSUS']\n",
        "[u'SBO215207', u'Total Asian-owned firms, percent, 2007', u'PCT', u'1', u'5.7', u'0.3', u'56.6', u'CENSUS']\n",
        "[u'SBO515207', u'Total Native Hawaiian- and Other Pacific Islander-owned firms, percent, 2007', u'PCT', u'1', u'0.1', u'0.0', u'10.5', u'CENSUS']\n",
        "[u'SBO415207', u'Total Hispanic-owned firms, percent, 2007', u'PCT', u'1', u'8.3', u'0.3', u'78.0', u'CENSUS']\n",
        "[u'SBO015207', u'Total Women-owned firms, percent, 2007', u'PCT', u'1', u'28.8', u'6.5', u'56.2', u'CENSUS']\n",
        "[u'MAN450207', u'Manufacturing: total (NAICS 31-33) - value of shipments, 2007', u'TH$', u'0', u'5338306501', u'56903', u'5338306501', u'CENSUS']\n",
        "[u'WTN220207', u'Wholesale trade: merchant wholesalers (NAICS 42) - sales of establishments with payroll, 2007', u'TH$', u'0', u'4174286516', u'0', u'4174286516', u'CENSUS']\n",
        "[u'RTN130207', u'Retail trade: total (NAICS 44-45) - sales of establishments with payroll, 2007', u'TH$', u'0', u'3917663456', u'0', u'3917663456', u'CENSUS']\n",
        "[u'RTN131207', u'Retail trade: total (NAICS 44-45) - sales of establishments with payroll per capita, 2007', u'DOL', u'0', u'12990', u'0', u'80800', u'CENSUS']\n",
        "[u'AFN120207', u'Accommodation and Food Services: total (NAICS 72) - sales of establishments with payroll, 2007', u'TH$', u'0', u'613795732', u'0', u'613795732', u'CENSUS']\n",
        "[u'BPS030211', u'New private housing units authorized by building permits - total, 2011 (20,000-place universe)', u'ABS', u'0', u'624061', u'0', u'624061', u'CENSUS']\n",
        "[u'LND110210', u'Land area in square miles, 2010', u'SQM', u'2', u'3531905.43', u'2', u'3531905.43', u'CENSUS']\n",
        "[u'POP060210', u'Population per square mile, 2010', u'RTE', u'1', u'87.4', u'0.0', u'69467.5', u'CENSUS']\n"
       ]
      }
     ],
     "prompt_number": 22
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "how to cast into standard size string?\n",
      "\n"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "s = \"hello there folks\"\n",
      "print [i for (i, k) in enumerate(list(s)) if k == ' ']\n",
      "\n",
      "# http://stackoverflow.com/a/4664889/7782\n",
      "\n",
      "import re\n",
      "print [m.start() for m in re.finditer(' ', s)]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[5, 11]\n",
        "[5, 11]\n"
       ]
      }
     ],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import sets\n",
      "a = set(range(10))\n",
      "a.intersection_update([2,3])\n",
      "a"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 24,
       "text": [
        "set([2, 3])"
       ]
      }
     ],
     "prompt_number": 24
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# http://code.activestate.com/recipes/496682-make-ranges-of-contiguous-numbers-from-a-list-of-i/#c2\n",
      "\n",
      "from itertools import groupby\n",
      "import operator\n",
      "\n",
      "data = [ 1,  4,5,6, 10, 15,16,17,18, 22, 25,26,27,28]\n",
      "for k, g in groupby(enumerate(data), lambda (i,x):i-x):\n",
      "       print map(operator.itemgetter(1), g)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[1]\n",
        "[4, 5, 6]\n",
        "[10]\n",
        "[15, 16, 17, 18]\n",
        "[22]\n",
        "[25, 26, 27, 28]\n"
       ]
      }
     ],
     "prompt_number": 25
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# masking\n",
      "# http://docs.scipy.org/doc/numpy/reference/maskedarray.generic.html\n",
      "\n",
      "import numpy as np\n",
      "import numpy.ma as ma\n",
      "\n",
      "x = np.array([1, 2, 3, -1, 5])\n",
      "mx = ma.masked_array(x, mask=[0, 0, 0, 1, 0])\n",
      "\n",
      "mx"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 26,
       "text": [
        "masked_array(data = [1 2 3 -- 5],\n",
        "             mask = [False False False  True False],\n",
        "       fill_value = 999999)\n"
       ]
      }
     ],
     "prompt_number": 26
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "mx.count()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 27,
       "text": [
        "4"
       ]
      }
     ],
     "prompt_number": 27
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# np.ma.clump_masked\n",
      "\n",
      "[mx[s].data for s in np.ma.clump_unmasked(mx)]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 28,
       "text": [
        "[array([1, 2, 3]), array([5])]"
       ]
      }
     ],
     "prompt_number": 28
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# http://stackoverflow.com/a/14606271/7782\n",
      "\n",
      "import numpy as np\n",
      "nan = np.nan\n",
      "\n",
      "def using_clump(a):\n",
      "    return [a[s] for s in np.ma.clump_unmasked(np.ma.masked_invalid(a))]\n",
      "\n",
      "x = [nan,nan, 1 , 2 , 3 , nan, nan, 10, 11 , nan, nan, nan, 23, 1, nan, 7, 8]\n",
      "\n",
      "using_clump(x)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 29,
       "text": [
        "[[1, 2, 3], [10, 11], [23, 1], [7, 8]]"
       ]
      }
     ],
     "prompt_number": 29
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from itertools import islice, izip, groupby\n",
      "import re\n",
      "import os\n",
      "import codecs\n",
      "import operator\n",
      "import pandas as pd\n",
      "\n",
      "DATA_DIR = os.path.join(os.pardir, \"data\")\n",
      "\n",
      "f = codecs.open(os.path.join(DATA_DIR, \"census/DataDict.txt\"), encoding=\"iso-8859-1\")\n",
      "\n",
      "f_sliced = islice(f, None)\n",
      "\n",
      "head_row = f_sliced.next()\n",
      "headers = head_row.split()\n",
      "\n",
      "print headers\n",
      "\n",
      "# Actually, Unit should be broken off from Decimal\n",
      "\n",
      "# skip the second row also\n",
      "header2 = f_sliced.next()\n",
      "\n",
      "# read in all the rows\n",
      "\n",
      "rows = list([r[:-1] for r in islice(f_sliced,None)])\n",
      "\n",
      "# What's the max length of rows?\n",
      "\n",
      "max_len = max([len(row) for row in rows])\n",
      "print max_len, len(rows)\n",
      "\n",
      "rows_array = np.vstack((np.array(list(row), dtype='S1') for row in rows))\n",
      "\n",
      "m = np.char.isspace(rows_array)\n",
      "print m.shape\n",
      "mask = np.all(m,0)\n",
      "#mask = np.where(np.all(m,0))\n",
      "mask\n",
      "\n",
      "# mask == np.array([np.all(col) for col in m.T])\n",
      "\n",
      "df = pd.DataFrame([[\"\".join(list(rows_array_row[s])).strip() for s in np.ma.clump_unmasked(np.ma.array(rows_array_row, mask =mask))] for rows_array_row in rows_array],\n",
      "       columns = [u'Data_Item', u'Item_Description', u'Unit', u'Decimal', u'US_Total', u'Minimum', u'Maximum', u'Source'])\n",
      "df"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[u'Data_Item', u'Item_Description', u'UnitDecimal', u'US_Total', u'Minimum', u'Maximum', u'Source']\n",
        "170 53\n",
        "(53, 170)\n"
       ]
      },
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>Data_Item</th>\n",
        "      <th>Item_Description</th>\n",
        "      <th>Unit</th>\n",
        "      <th>Decimal</th>\n",
        "      <th>US_Total</th>\n",
        "      <th>Minimum</th>\n",
        "      <th>Maximum</th>\n",
        "      <th>Source</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0 </th>\n",
        "      <td> PST045212</td>\n",
        "      <td>  Resident total population estimate (July 1) 2012</td>\n",
        "      <td> ABS</td>\n",
        "      <td> 0</td>\n",
        "      <td>  313914040</td>\n",
        "      <td> 576412</td>\n",
        "      <td>  313914040</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1 </th>\n",
        "      <td> PST045211</td>\n",
        "      <td>  Resident total population estimate (July 1) 2011</td>\n",
        "      <td> ABS</td>\n",
        "      <td> 0</td>\n",
        "      <td>  311587816</td>\n",
        "      <td>     90</td>\n",
        "      <td>  311587816</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2 </th>\n",
        "      <td> PST040210</td>\n",
        "      <td> Resident total population, estimates base (Apr...</td>\n",
        "      <td> ABS</td>\n",
        "      <td> 0</td>\n",
        "      <td>  308747508</td>\n",
        "      <td>     82</td>\n",
        "      <td>  308747508</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3 </th>\n",
        "      <td> PST120212</td>\n",
        "      <td> Resident total population, percent change - Ap...</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>        1.7</td>\n",
        "      <td>   -0.2</td>\n",
        "      <td>        5.1</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4 </th>\n",
        "      <td> PST120211</td>\n",
        "      <td> Resident total population, percent change - Ap...</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>        0.9</td>\n",
        "      <td>  -18.1</td>\n",
        "      <td>       14.6</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5 </th>\n",
        "      <td> POP010210</td>\n",
        "      <td> Resident population (April 1 - complete count)...</td>\n",
        "      <td> ABS</td>\n",
        "      <td> 0</td>\n",
        "      <td>  308745538</td>\n",
        "      <td>     82</td>\n",
        "      <td>  308745538</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>6 </th>\n",
        "      <td> AGE135211</td>\n",
        "      <td>  Resident population under 5 years, percent, 2011</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>        6.5</td>\n",
        "      <td>    0.0</td>\n",
        "      <td>       13.3</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>7 </th>\n",
        "      <td> AGE295211</td>\n",
        "      <td> Resident population under 18 years, percent, 2011</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>       23.7</td>\n",
        "      <td>    0.0</td>\n",
        "      <td>       41.0</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>8 </th>\n",
        "      <td> AGE775211</td>\n",
        "      <td> Resident population 65 years and over, percent...</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>       13.3</td>\n",
        "      <td>    3.7</td>\n",
        "      <td>       45.5</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>9 </th>\n",
        "      <td> SEX255211</td>\n",
        "      <td> Resident population: total females, percent, 2011</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>       50.8</td>\n",
        "      <td>   28.7</td>\n",
        "      <td>       56.8</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>10</th>\n",
        "      <td> RHI125211</td>\n",
        "      <td>   Resident population: White alone, percent, 2011</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>       78.1</td>\n",
        "      <td>    3.9</td>\n",
        "      <td>       99.7</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>11</th>\n",
        "      <td> RHI225211</td>\n",
        "      <td>   Resident population: Black alone, percent, 2011</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>       13.1</td>\n",
        "      <td>    0.0</td>\n",
        "      <td>       84.7</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>12</th>\n",
        "      <td> RHI325211</td>\n",
        "      <td> Resident population: American Indian and Alask...</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>        1.2</td>\n",
        "      <td>    0.0</td>\n",
        "      <td>       93.3</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>13</th>\n",
        "      <td> RHI425211</td>\n",
        "      <td>   Resident population: Asian alone, percent, 2011</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>        5.0</td>\n",
        "      <td>    0.0</td>\n",
        "      <td>       43.6</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>14</th>\n",
        "      <td> RHI525211</td>\n",
        "      <td> Resident population: Native Hawaiian and Other...</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>        0.2</td>\n",
        "      <td>    0.0</td>\n",
        "      <td>       48.9</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>15</th>\n",
        "      <td> RHI625211</td>\n",
        "      <td> Resident population: Two or more races, percen...</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>        2.3</td>\n",
        "      <td>    0.0</td>\n",
        "      <td>       29.2</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>16</th>\n",
        "      <td> RHI725211</td>\n",
        "      <td> Resident population: Hispanic or Latino Origin...</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>       16.7</td>\n",
        "      <td>    0.0</td>\n",
        "      <td>       95.6</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>17</th>\n",
        "      <td> RHI825211</td>\n",
        "      <td> Resident population: Not Hispanic, White alone...</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>       63.4</td>\n",
        "      <td>    3.2</td>\n",
        "      <td>       98.9</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>18</th>\n",
        "      <td> POP715211</td>\n",
        "      <td> Population 1 year and over by residence - same...</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>       84.6</td>\n",
        "      <td>   49.0</td>\n",
        "      <td>      100.0</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>19</th>\n",
        "      <td> POP645211</td>\n",
        "      <td> Place of birth, foreign born, percent,  2007-2011</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>       12.8</td>\n",
        "      <td>    0.0</td>\n",
        "      <td>       63.4</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>20</th>\n",
        "      <td> POP815211</td>\n",
        "      <td> Population 5 years and over, percent speaking ...</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>       20.3</td>\n",
        "      <td>    0.0</td>\n",
        "      <td>       95.9</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>21</th>\n",
        "      <td> EDU635211</td>\n",
        "      <td> Educational attainment - persons 25 years and ...</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>       85.4</td>\n",
        "      <td>   46.3</td>\n",
        "      <td>       98.6</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>22</th>\n",
        "      <td> EDU685211</td>\n",
        "      <td> Educational attainment - persons 25 years and ...</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>       28.2</td>\n",
        "      <td>    4.2</td>\n",
        "      <td>       72.0</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>23</th>\n",
        "      <td> VET605211</td>\n",
        "      <td>                       Veterans - total, 2007-2011</td>\n",
        "      <td> ABS</td>\n",
        "      <td> 0</td>\n",
        "      <td>   22215303</td>\n",
        "      <td>      0</td>\n",
        "      <td>   22215303</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>24</th>\n",
        "      <td> LFE305211</td>\n",
        "      <td> Average travel time to work for workers 16 yea...</td>\n",
        "      <td> MIN</td>\n",
        "      <td> 1</td>\n",
        "      <td>       25.4</td>\n",
        "      <td>    4.3</td>\n",
        "      <td>       42.5</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>25</th>\n",
        "      <td> HSG010211</td>\n",
        "      <td>                      Housing unit estimates, 2011</td>\n",
        "      <td> ABS</td>\n",
        "      <td> 0</td>\n",
        "      <td>  132312404</td>\n",
        "      <td>     48</td>\n",
        "      <td>  132312404</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>26</th>\n",
        "      <td> HSG445211</td>\n",
        "      <td> Owner-occupied housing units - percent of tota...</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>       66.1</td>\n",
        "      <td>    0.0</td>\n",
        "      <td>       93.7</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>27</th>\n",
        "      <td> HSG096211</td>\n",
        "      <td> Housing units by units in structure - multi-dw...</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>       25.9</td>\n",
        "      <td>    0.0</td>\n",
        "      <td>       98.4</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>28</th>\n",
        "      <td> HSG495211</td>\n",
        "      <td> Median value of specified owner-occupied housi...</td>\n",
        "      <td> DOL</td>\n",
        "      <td> 0</td>\n",
        "      <td>     186200</td>\n",
        "      <td>      0</td>\n",
        "      <td>     993900</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>29</th>\n",
        "      <td> HSD410211</td>\n",
        "      <td>                             Households, 2007-2011</td>\n",
        "      <td> ABS</td>\n",
        "      <td> 0</td>\n",
        "      <td>  114761359</td>\n",
        "      <td>     27</td>\n",
        "      <td>  114761359</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>30</th>\n",
        "      <td> HSD310211</td>\n",
        "      <td>                 Average household size, 2007-2011</td>\n",
        "      <td> AVG</td>\n",
        "      <td> 2</td>\n",
        "      <td>       2.60</td>\n",
        "      <td>   1.20</td>\n",
        "      <td>       4.77</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>31</th>\n",
        "      <td> INC910211</td>\n",
        "      <td> Per capita income in the past 12 months (in 20...</td>\n",
        "      <td> DOL</td>\n",
        "      <td> 0</td>\n",
        "      <td>      27915</td>\n",
        "      <td>   7887</td>\n",
        "      <td>      61290</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>32</th>\n",
        "      <td> INC110211</td>\n",
        "      <td>                Median household income, 2007-2011</td>\n",
        "      <td> DOL</td>\n",
        "      <td> 0</td>\n",
        "      <td>      52762</td>\n",
        "      <td>  19344</td>\n",
        "      <td>     120332</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>33</th>\n",
        "      <td> PVY020211</td>\n",
        "      <td> People of all ages in poverty, percent, 2007-2011</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>       14.3</td>\n",
        "      <td>    0.0</td>\n",
        "      <td>       53.5</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>34</th>\n",
        "      <td> BZA010210</td>\n",
        "      <td>              Private nonfarm establishments, 2010</td>\n",
        "      <td> ABS</td>\n",
        "      <td> 0</td>\n",
        "      <td>    7396628</td>\n",
        "      <td>      0</td>\n",
        "      <td>    7396628</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>35</th>\n",
        "      <td> BZA110210</td>\n",
        "      <td> Private nonfarm employment for pay period incl...</td>\n",
        "      <td> ABS</td>\n",
        "      <td> 0</td>\n",
        "      <td>  111970095</td>\n",
        "      <td>      0</td>\n",
        "      <td>  111970095</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>36</th>\n",
        "      <td> BZA115210</td>\n",
        "      <td> Private nonfarm employment for pay period incl...</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>       -1.8</td>\n",
        "      <td>  -82.7</td>\n",
        "      <td>      385.8</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>37</th>\n",
        "      <td> NES010210</td>\n",
        "      <td> Nonemployer: total (NAICS 00) - establishments...</td>\n",
        "      <td> ABS</td>\n",
        "      <td> 0</td>\n",
        "      <td>   22110628</td>\n",
        "      <td>     17</td>\n",
        "      <td>   22110628</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>38</th>\n",
        "      <td> SBO001207</td>\n",
        "      <td>                       Total number of firms, 2007</td>\n",
        "      <td> ABS</td>\n",
        "      <td> 0</td>\n",
        "      <td>   27092908</td>\n",
        "      <td>     27</td>\n",
        "      <td>   27092908</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>39</th>\n",
        "      <td> SBO315207</td>\n",
        "      <td>            Total Black-owned firms, percent, 2007</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>        7.1</td>\n",
        "      <td>    0.2</td>\n",
        "      <td>       66.7</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>40</th>\n",
        "      <td> SBO115207</td>\n",
        "      <td> Total American Indian- and Alaska Native-owned...</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>        0.9</td>\n",
        "      <td>    0.2</td>\n",
        "      <td>       71.8</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>41</th>\n",
        "      <td> SBO215207</td>\n",
        "      <td>            Total Asian-owned firms, percent, 2007</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>        5.7</td>\n",
        "      <td>    0.3</td>\n",
        "      <td>       56.6</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>42</th>\n",
        "      <td> SBO515207</td>\n",
        "      <td> Total Native Hawaiian- and Other Pacific Islan...</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>        0.1</td>\n",
        "      <td>    0.0</td>\n",
        "      <td>       10.5</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>43</th>\n",
        "      <td> SBO415207</td>\n",
        "      <td>         Total Hispanic-owned firms, percent, 2007</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>        8.3</td>\n",
        "      <td>    0.3</td>\n",
        "      <td>       78.0</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>44</th>\n",
        "      <td> SBO015207</td>\n",
        "      <td>            Total Women-owned firms, percent, 2007</td>\n",
        "      <td> PCT</td>\n",
        "      <td> 1</td>\n",
        "      <td>       28.8</td>\n",
        "      <td>    6.5</td>\n",
        "      <td>       56.2</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>45</th>\n",
        "      <td> MAN450207</td>\n",
        "      <td> Manufacturing: total (NAICS 31-33) - value of ...</td>\n",
        "      <td> TH$</td>\n",
        "      <td> 0</td>\n",
        "      <td> 5338306501</td>\n",
        "      <td>  56903</td>\n",
        "      <td> 5338306501</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>46</th>\n",
        "      <td> WTN220207</td>\n",
        "      <td> Wholesale trade: merchant wholesalers (NAICS 4...</td>\n",
        "      <td> TH$</td>\n",
        "      <td> 0</td>\n",
        "      <td> 4174286516</td>\n",
        "      <td>      0</td>\n",
        "      <td> 4174286516</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>47</th>\n",
        "      <td> RTN130207</td>\n",
        "      <td> Retail trade: total (NAICS 44-45) - sales of e...</td>\n",
        "      <td> TH$</td>\n",
        "      <td> 0</td>\n",
        "      <td> 3917663456</td>\n",
        "      <td>      0</td>\n",
        "      <td> 3917663456</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>48</th>\n",
        "      <td> RTN131207</td>\n",
        "      <td> Retail trade: total (NAICS 44-45) - sales of e...</td>\n",
        "      <td> DOL</td>\n",
        "      <td> 0</td>\n",
        "      <td>      12990</td>\n",
        "      <td>      0</td>\n",
        "      <td>      80800</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>49</th>\n",
        "      <td> AFN120207</td>\n",
        "      <td> Accommodation and Food Services: total (NAICS ...</td>\n",
        "      <td> TH$</td>\n",
        "      <td> 0</td>\n",
        "      <td>  613795732</td>\n",
        "      <td>      0</td>\n",
        "      <td>  613795732</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>50</th>\n",
        "      <td> BPS030211</td>\n",
        "      <td> New private housing units authorized by buildi...</td>\n",
        "      <td> ABS</td>\n",
        "      <td> 0</td>\n",
        "      <td>     624061</td>\n",
        "      <td>      0</td>\n",
        "      <td>     624061</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>51</th>\n",
        "      <td> LND110210</td>\n",
        "      <td>                   Land area in square miles, 2010</td>\n",
        "      <td> SQM</td>\n",
        "      <td> 2</td>\n",
        "      <td> 3531905.43</td>\n",
        "      <td>      2</td>\n",
        "      <td> 3531905.43</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>52</th>\n",
        "      <td> POP060210</td>\n",
        "      <td>                  Population per square mile, 2010</td>\n",
        "      <td> RTE</td>\n",
        "      <td> 1</td>\n",
        "      <td>       87.4</td>\n",
        "      <td>    0.0</td>\n",
        "      <td>    69467.5</td>\n",
        "      <td> CENSUS</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "output_type": "pyout",
       "prompt_number": 30,
       "text": [
        "    Data_Item                                   Item_Description Unit Decimal  \\\n",
        "0   PST045212   Resident total population estimate (July 1) 2012  ABS       0   \n",
        "1   PST045211   Resident total population estimate (July 1) 2011  ABS       0   \n",
        "2   PST040210  Resident total population, estimates base (Apr...  ABS       0   \n",
        "3   PST120212  Resident total population, percent change - Ap...  PCT       1   \n",
        "4   PST120211  Resident total population, percent change - Ap...  PCT       1   \n",
        "5   POP010210  Resident population (April 1 - complete count)...  ABS       0   \n",
        "6   AGE135211   Resident population under 5 years, percent, 2011  PCT       1   \n",
        "7   AGE295211  Resident population under 18 years, percent, 2011  PCT       1   \n",
        "8   AGE775211  Resident population 65 years and over, percent...  PCT       1   \n",
        "9   SEX255211  Resident population: total females, percent, 2011  PCT       1   \n",
        "10  RHI125211    Resident population: White alone, percent, 2011  PCT       1   \n",
        "11  RHI225211    Resident population: Black alone, percent, 2011  PCT       1   \n",
        "12  RHI325211  Resident population: American Indian and Alask...  PCT       1   \n",
        "13  RHI425211    Resident population: Asian alone, percent, 2011  PCT       1   \n",
        "14  RHI525211  Resident population: Native Hawaiian and Other...  PCT       1   \n",
        "15  RHI625211  Resident population: Two or more races, percen...  PCT       1   \n",
        "16  RHI725211  Resident population: Hispanic or Latino Origin...  PCT       1   \n",
        "17  RHI825211  Resident population: Not Hispanic, White alone...  PCT       1   \n",
        "18  POP715211  Population 1 year and over by residence - same...  PCT       1   \n",
        "19  POP645211  Place of birth, foreign born, percent,  2007-2011  PCT       1   \n",
        "20  POP815211  Population 5 years and over, percent speaking ...  PCT       1   \n",
        "21  EDU635211  Educational attainment - persons 25 years and ...  PCT       1   \n",
        "22  EDU685211  Educational attainment - persons 25 years and ...  PCT       1   \n",
        "23  VET605211                        Veterans - total, 2007-2011  ABS       0   \n",
        "24  LFE305211  Average travel time to work for workers 16 yea...  MIN       1   \n",
        "25  HSG010211                       Housing unit estimates, 2011  ABS       0   \n",
        "26  HSG445211  Owner-occupied housing units - percent of tota...  PCT       1   \n",
        "27  HSG096211  Housing units by units in structure - multi-dw...  PCT       1   \n",
        "28  HSG495211  Median value of specified owner-occupied housi...  DOL       0   \n",
        "29  HSD410211                              Households, 2007-2011  ABS       0   \n",
        "30  HSD310211                  Average household size, 2007-2011  AVG       2   \n",
        "31  INC910211  Per capita income in the past 12 months (in 20...  DOL       0   \n",
        "32  INC110211                 Median household income, 2007-2011  DOL       0   \n",
        "33  PVY020211  People of all ages in poverty, percent, 2007-2011  PCT       1   \n",
        "34  BZA010210               Private nonfarm establishments, 2010  ABS       0   \n",
        "35  BZA110210  Private nonfarm employment for pay period incl...  ABS       0   \n",
        "36  BZA115210  Private nonfarm employment for pay period incl...  PCT       1   \n",
        "37  NES010210  Nonemployer: total (NAICS 00) - establishments...  ABS       0   \n",
        "38  SBO001207                        Total number of firms, 2007  ABS       0   \n",
        "39  SBO315207             Total Black-owned firms, percent, 2007  PCT       1   \n",
        "40  SBO115207  Total American Indian- and Alaska Native-owned...  PCT       1   \n",
        "41  SBO215207             Total Asian-owned firms, percent, 2007  PCT       1   \n",
        "42  SBO515207  Total Native Hawaiian- and Other Pacific Islan...  PCT       1   \n",
        "43  SBO415207          Total Hispanic-owned firms, percent, 2007  PCT       1   \n",
        "44  SBO015207             Total Women-owned firms, percent, 2007  PCT       1   \n",
        "45  MAN450207  Manufacturing: total (NAICS 31-33) - value of ...  TH$       0   \n",
        "46  WTN220207  Wholesale trade: merchant wholesalers (NAICS 4...  TH$       0   \n",
        "47  RTN130207  Retail trade: total (NAICS 44-45) - sales of e...  TH$       0   \n",
        "48  RTN131207  Retail trade: total (NAICS 44-45) - sales of e...  DOL       0   \n",
        "49  AFN120207  Accommodation and Food Services: total (NAICS ...  TH$       0   \n",
        "50  BPS030211  New private housing units authorized by buildi...  ABS       0   \n",
        "51  LND110210                    Land area in square miles, 2010  SQM       2   \n",
        "52  POP060210                   Population per square mile, 2010  RTE       1   \n",
        "\n",
        "      US_Total Minimum     Maximum  Source  \n",
        "0    313914040  576412   313914040  CENSUS  \n",
        "1    311587816      90   311587816  CENSUS  \n",
        "2    308747508      82   308747508  CENSUS  \n",
        "3          1.7    -0.2         5.1  CENSUS  \n",
        "4          0.9   -18.1        14.6  CENSUS  \n",
        "5    308745538      82   308745538  CENSUS  \n",
        "6          6.5     0.0        13.3  CENSUS  \n",
        "7         23.7     0.0        41.0  CENSUS  \n",
        "8         13.3     3.7        45.5  CENSUS  \n",
        "9         50.8    28.7        56.8  CENSUS  \n",
        "10        78.1     3.9        99.7  CENSUS  \n",
        "11        13.1     0.0        84.7  CENSUS  \n",
        "12         1.2     0.0        93.3  CENSUS  \n",
        "13         5.0     0.0        43.6  CENSUS  \n",
        "14         0.2     0.0        48.9  CENSUS  \n",
        "15         2.3     0.0        29.2  CENSUS  \n",
        "16        16.7     0.0        95.6  CENSUS  \n",
        "17        63.4     3.2        98.9  CENSUS  \n",
        "18        84.6    49.0       100.0  CENSUS  \n",
        "19        12.8     0.0        63.4  CENSUS  \n",
        "20        20.3     0.0        95.9  CENSUS  \n",
        "21        85.4    46.3        98.6  CENSUS  \n",
        "22        28.2     4.2        72.0  CENSUS  \n",
        "23    22215303       0    22215303  CENSUS  \n",
        "24        25.4     4.3        42.5  CENSUS  \n",
        "25   132312404      48   132312404  CENSUS  \n",
        "26        66.1     0.0        93.7  CENSUS  \n",
        "27        25.9     0.0        98.4  CENSUS  \n",
        "28      186200       0      993900  CENSUS  \n",
        "29   114761359      27   114761359  CENSUS  \n",
        "30        2.60    1.20        4.77  CENSUS  \n",
        "31       27915    7887       61290  CENSUS  \n",
        "32       52762   19344      120332  CENSUS  \n",
        "33        14.3     0.0        53.5  CENSUS  \n",
        "34     7396628       0     7396628  CENSUS  \n",
        "35   111970095       0   111970095  CENSUS  \n",
        "36        -1.8   -82.7       385.8  CENSUS  \n",
        "37    22110628      17    22110628  CENSUS  \n",
        "38    27092908      27    27092908  CENSUS  \n",
        "39         7.1     0.2        66.7  CENSUS  \n",
        "40         0.9     0.2        71.8  CENSUS  \n",
        "41         5.7     0.3        56.6  CENSUS  \n",
        "42         0.1     0.0        10.5  CENSUS  \n",
        "43         8.3     0.3        78.0  CENSUS  \n",
        "44        28.8     6.5        56.2  CENSUS  \n",
        "45  5338306501   56903  5338306501  CENSUS  \n",
        "46  4174286516       0  4174286516  CENSUS  \n",
        "47  3917663456       0  3917663456  CENSUS  \n",
        "48       12990       0       80800  CENSUS  \n",
        "49   613795732       0   613795732  CENSUS  \n",
        "50      624061       0      624061  CENSUS  \n",
        "51  3531905.43       2  3531905.43  CENSUS  \n",
        "52        87.4     0.0     69467.5  CENSUS  "
       ]
      }
     ],
     "prompt_number": 30
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# compare rows / m\n",
      "\n",
      "from itertools import izip\n",
      "\n",
      "#for (i, (r0, m0)) in enumerate(izip(rows, m)):\n",
      "#    print r0, m0\n",
      "#    # print i, np.all([c == ' ' for c in r0] == m0)\n",
      "\n",
      "rnum = 1\n",
      "all([c == ' ' for c in rows[rnum]] == m[rnum])\n",
      "\n",
      "[c == ' ' for c in rows[rnum]] == m[rnum]\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 31,
       "text": [
        "array([ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
        "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
        "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
        "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
        "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
        "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
        "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
        "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
        "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
        "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
        "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
        "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
        "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
        "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
        "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
        "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
        "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
        "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
        "        True,  True,  True,  True,  True,  True,  True,  True], dtype=bool)"
       ]
      }
     ],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rows_array[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 32,
       "text": [
        "array(['P', 'S', 'T', '0', '4', '5', '2', '1', '2', ' ', 'R', 'e', 's',\n",
        "       'i', 'd', 'e', 'n', 't', ' ', 't', 'o', 't', 'a', 'l', ' ', 'p',\n",
        "       'o', 'p', 'u', 'l', 'a', 't', 'i', 'o', 'n', ' ', 'e', 's', 't',\n",
        "       'i', 'm', 'a', 't', 'e', ' ', '(', 'J', 'u', 'l', 'y', ' ', '1',\n",
        "       ')', ' ', '2', '0', '1', '2', ' ', ' ', ' ', ' ', ' ', ' ', ' ',\n",
        "       ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ',\n",
        "       ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ',\n",
        "       ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ',\n",
        "       ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', 'A', 'B',\n",
        "       'S', ' ', ' ', ' ', ' ', '0', ' ', ' ', ' ', ' ', ' ', ' ', '3',\n",
        "       '1', '3', '9', '1', '4', '0', '4', '0', ' ', ' ', ' ', ' ', ' ',\n",
        "       ' ', '5', '7', '6', '4', '1', '2', ' ', ' ', ' ', '3', '1', '3',\n",
        "       '9', '1', '4', '0', '4', '0', ' ', ' ', 'C', 'E', 'N', 'S', 'U', 'S'], \n",
        "      dtype='|S1')"
       ]
      }
     ],
     "prompt_number": 32
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "np.ma.clump_unmasked"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 33,
       "text": [
        "<function numpy.ma.extras.clump_unmasked>"
       ]
      }
     ],
     "prompt_number": 33
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "[\"\".join(list(rows_array[0][s])).strip() for s in np.ma.clump_unmasked(np.ma.array(rows_array[0], mask =mask))]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 34,
       "text": [
        "['PST045212',\n",
        " 'Resident total population estimate (July 1) 2012',\n",
        " 'ABS',\n",
        " '0',\n",
        " '313914040',\n",
        " '576412',\n",
        " '313914040',\n",
        " 'CENSUS']"
       ]
      }
     ],
     "prompt_number": 34
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Addendum:  creating column markers to read off column numbers"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\n",
      "0...9\n",
      "a...i\n",
      "A...I\n",
      "\n",
      "repeat..."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import string\n",
      "from itertools import islice\n",
      "\n",
      "upper = string.uppercase + \"0\"\n",
      "lower = string.lowercase\n",
      "\n",
      "def column_marker(start=0, stop=None):\n",
      "    n = start\n",
      "    while stop is None or n < stop:\n",
      "        k = n % 10\n",
      "        if k > 0:\n",
      "            yield unicode(k)\n",
      "        else:\n",
      "            if n % 100 == 0:\n",
      "                yield unicode(upper[(n % 1000) / 100 -1 ])\n",
      "            elif n % 10 == 0 :\n",
      "                yield unicode(lower[(n % 100) / 10  - 1])\n",
      "        \n",
      "        n += 1"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 35
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "len(list(column_marker(0,12)))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 36,
       "text": [
        "12"
       ]
      }
     ],
     "prompt_number": 36
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print \"\".join(column_marker(0,180))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0123456789a123456789b123456789c123456789d123456789e123456789f123456789g123456789h123456789i123456789A123456789a123456789b123456789c123456789d123456789e123456789f123456789g123456789\n"
       ]
      }
     ],
     "prompt_number": 37
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "<pre>\n",
      "0123456789a123456789b123456789c123456789d123456789e123456789f123456789g123456789h123456789i123456789A123456789a123456789b123456789c123456789d123456789e123456789f123456789\n",
      "PST045212 Resident total population estimate (July 1) 2012                                                         ABS    0      313914040      576412   313914040  CENSUS\n",
      "PST045211 Resident total population estimate (July 1) 2011                                                         ABS    0      311587816          90   311587816  CENSUS\n",
      "PST040210 Resident total population, estimates base (April 1) 2010                                                 ABS    0      308747508          82   308747508  CENSUS\n",
      "PST120212 Resident total population, percent change - April 1, 2010 to July 1, 2012                                PCT    1            1.7        -0.2         5.1  CENSUS\n",
      "PST120211 Resident total population, percent change - April 1, 2010 to July 1, 2011                                PCT    1            0.9       -18.1        14.6  CENSUS\n",
      "POP010210 Resident population (April 1 - complete count) 2010                                                      ABS    0      308745538          82   308745538  CENSUS\n",
      "AGE135211 Resident population under 5 years, percent, 2011                                                         PCT    1            6.5         0.0        13.3  CENSUS\n",
      "A\n",
      "</pre>"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Reading off the columns\n",
      "\n",
      "* 0, 8\n",
      "* 10, 111\n",
      "* 115, 117\n",
      "* 122\n",
      "* 129, 137\n",
      "* 144, 149\n",
      "* 153, 161\n",
      "* 164, 169\n",
      "\n",
      "[u'Data_Item', u'Item_Description', u'Unit', u'Decimal', u'US_Total', u'Minimum', u'Maximum', u'Source']"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 37
    }
   ],
   "metadata": {}
  }
 ]
}