Skip to content
Browse files

Add indexes to batch scripts.

  • Loading branch information...
1 parent 3a4ef49 commit 1d6eaa5e5b4b21d37092820d71794e8fc8604cf5 cgroskopf committed May 20, 2011
View
7 dataprocessing/batch_sf.sh
@@ -2,17 +2,14 @@
# See batch.sh for notes.
-echo "use census;
-db.geographies.ensureIndex({ 'geoid': 1 });
-db.geographies.ensureIndex({ 'metadata.STATE': 1 });
-db.geographies.ensureIndex({ 'xrefs': 1 });" | mongo
-
STATE_NAME=$1
STATE_NAME_LOWER=`echo $1 | tr '[A-Z]' '[a-z]'`
STATE_NAME_ABBR=`python get_state_abbr.py $1`
STATE_FIPS=`python get_state_fips.py $1`
FAKE=$2
+./ensure_indexes.sh
+
./fetch_sf_data.sh $STATE_NAME $STATE_NAME_LOWER $STATE_NAME_ABBR $FAKE
./load_sf_geographies_2000.py data/${STATE_NAME_ABBR}geo2000.csv
View
8 dataprocessing/batch_test.sh
@@ -5,6 +5,8 @@
echo "use census;
db.dropDatabase();" | mongo
+./ensure_indexes.sh
+
./fetch_test_data.sh
./load_pl_geographies_2000.py data/degeo2000.csv
@@ -13,13 +15,13 @@ db.dropDatabase();" | mongo
./load_pl_geographies_2010.py data/degeo2010.csv
-./load_crosswalk.py data/us2010trf.csv
+./load_crosswalk.py 10 data/us2010trf.csv
./load_pl_data_2010.py data/pl_data_2010_delaware_1.csv
./load_pl_data_2010.py data/pl_data_2010_delaware_2.csv
./load_pl_labels_2010.py data/pl_2010_data_labels.csv
-./crosswalk.py
-./compute_deltas.py
+./crosswalk.py 10
+./compute_deltas.py 10
./tests.py
View
8 dataprocessing/ensure_indexes.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+echo "use census;
+db.geographies.ensureIndex({ 'geoid': 1 });
+db.geographies.ensureIndex({ 'metadata.STATE': 1 });
+db.geographies.ensureIndex({ 'xrefs': 1 });
+db.geographies_2000.ensureIndex({ 'geoid': 1 });
+db.geographies_2000.ensureIndex({ 'metadata.STATE': 1 });" | mongo
View
2 dataprocessing/load_crosswalk.py
@@ -40,7 +40,7 @@
row_count += 1
row_dict = dict(zip(headers, row))
- if row_dict['STATE'] != STATE_FIPS:
+ if row_dict['STATE10'] != STATE_FIPS:
continue
geography = collection.find_one({ 'geoid': row_dict['GEOID10'] })

0 comments on commit 1d6eaa5

Please sign in to comment.
Something went wrong with that request. Please try again.