Merge branch 'master' of github.com:ireapps/census

dataprocessing/batch_20110719.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 # 33 NH 44M
-./batch_sf.sh "New Hampshire" production
+# ./batch_sf.sh "New Hampshire" production
 # 23 ME 56M
-./batch_sf.sh "Maine" production
+# ./batch_sf.sh "Maine" production
 # 21 KY 151M
-./batch_sf.sh "Kentucky" production
+# ./batch_sf.sh "Kentucky" production
 # 51 VA 259M
 ./batch_sf.sh "Virginia" production
 # 13 GA 285M

dataprocessing/batch_20110726.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# 16 ID 
+./batch_sf.sh "Idaho" production
+# 54 WV 
+./batch_sf.sh "West Virginia" production
+# 49 UT 
+./batch_sf.sh "Utah" production
+# 55 WI 
+./batch_sf.sh "Wisconsin" production
+# 29 MO 
+./batch_sf.sh "Missouri" production
+# 39 OH 
+./batch_sf.sh "Ohio" production

dataprocessing/batch_sf.sh

@@ -13,6 +13,7 @@ STATE_NAME_ABBR=`python get_state_abbr.py "${STATE_NAME}"` || exit $?
 STATE_FIPS=`python get_state_fips.py "${STATE_NAME}"` || exit $?
 ENVIRONMENT="${@:2:1}"
 FAKE="${@:3:1}"
+MONGO_DUMP_DIR="/mnt/data/mongodumps"
 
 echo Begin $STATE_NAME at `date`
 echo 'Dropping previous data.'
@@ -72,6 +73,10 @@ echo 'Processing crosswalk'
 echo 'Computing deltas'
 ./compute_deltas_sf.py || exit $?
 
+echo 'Dumping mongo data for ${STATE_NAME}'
+mkdir -p $MONGO_DUMP_DIR/${STATE_FIPS}
+mongodump -d census -o $MONGO_DUMP_DIR/${STATE_FIPS}
+
 echo 'Deploying to S3'
 ./deploy_data.py $ENVIRONMENT || exit $?
 ./deploy_lookups.py $ENVIRONMENT || exit $?
@@ -82,7 +87,5 @@ echo 'Deploying to S3'
 ./deploy_csv.py $STATE_FIPS 140 $ENVIRONMENT || exit $?
 ./deploy_csv.py $STATE_FIPS 160 $ENVIRONMENT || exit $?
 
-mkdir -p /mnt/data/mongodumps/${STATE_FIPS}
-mongodump -d census -o /mnt/data/mongodumps/${STATE_FIPS}
 echo Complete $STATE_NAME at `date`
 

dataprocessing/crosswalk.py

@@ -107,7 +107,7 @@
         geography_2000 = collection_2000.find_one({ 'geoid': geography['geoid'] }, fields=['data'])
 
         if not geography_2000:
-            print 'Couldn\'t find matching 2000 geography for %s (%s)' % (geography['metadata']['NAME'].encode('utf-8'), geography['geoid'])
+            print 'Couldn\'t find matching 2000 geography for %s' % (geography['geoid'])
 
             continue
 

dataprocessing/restore_state.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# Given a state fips code, restore that state to the mongo DB
+# this REPLACES current data, so use with caution
+MONGO_DUMP_DIR="/mnt/data/mongodumps"
+STATE_FIPS="${@:1:1}"
+
+mongorestore -d census --drop ${MONGO_DUMP_DIR}/${STATE_FIPS}/census