Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Updated batch scripts so state is marked available when it is made pu…

…blic. Closes #117.
  • Loading branch information...
commit d85fab2e3c722018477e6bcacc650cf3c4af67a0 1 parent c32d0d8
cgroskopf authored
View
1  dataprocessing/batch_sf.sh
@@ -73,5 +73,4 @@ echo 'Deploying to S3'
./deploy_data.py $ENVIRONMENT
./deploy_lookups.py $ENVIRONMENT
./deploy_labels.py $ENVIRONMENT
-./update_state_list.py $ENVIRONMENT $STATE_NAME
View
6 dataprocessing/make_state_public.py
@@ -6,12 +6,14 @@
import config
import get_state_fips
+import update_state_list
if len(sys.argv) < 3:
sys.exit('You must "staging" or "production" and a state name as arguments to this script.')
ENVIRONMENT = sys.argv[1]
-STATE_FIPS = get_state_fips.STATE_FIPS[sys.argv[2]]
+STATE = sys.argv[2]
+STATE_FIPS = get_state_fips.STATE_FIPS[STATE]
c = S3Connection()
bucket = c.get_bucket(config.S3_BUCKETS[ENVIRONMENT])
@@ -22,3 +24,5 @@
if i % 100 == 0:
print 'Processed %i...' % i
+# Update available states list
+update_state_list.update_state_list(ENVIRONMENT, STATE)
View
99 dataprocessing/update_state_list.py
@@ -6,55 +6,58 @@
from boto.exception import S3ResponseError
from boto.s3.connection import S3Connection
from boto.s3.key import Key
-from pymongo import Connection
import config
import utils
-if len(sys.argv) < 3:
- sys.exit('You must specify either "staging" or "production" and a state as arguments to this script.')
-
-ENVIRONMENT = sys.argv[1]
-STATE = sys.argv[2]
-try: CLEAR = sys.argv[3]
-except: CLEAR = None
-
-connection = Connection()
-db = connection[config.CENSUS_DB]
-collection = db[config.GEOGRAPHIES_COLLECTION]
-
-row_count = 0
-deployed = 0
-
-c = S3Connection()
-bucket = c.get_bucket(config.S3_BUCKETS[ENVIRONMENT])
-
-k = Key(bucket)
-k.key = 'states.jsonp'
-
-try:
- data = k.get_contents_as_string()
-
- # No existing file
- if not data:
- raise S3ResponseError()
-
- # Strip off jsonp wrapper
- contents = utils.gunzip_data(data)
- data = contents[7:-1]
-
- states = json.loads(data)
-except S3ResponseError:
- states = []
-
-if CLEAR == 'CLEAR':
- states = [STATE]
-else:
- if STATE not in states:
- states.append(STATE)
-
-jsonp = 'states(%s)' % json.dumps(states)
-compressed = utils.gzip_data(jsonp)
-
-k.set_contents_from_string(compressed, headers={ 'Content-encoding': 'gzip', 'Content-Type': 'application/json' }, policy='public-read')
-
+def update_state_list(environment, state, clear=False):
+ c = S3Connection()
+ bucket = c.get_bucket(config.S3_BUCKETS[environment])
+
+ k = Key(bucket)
+ k.key = 'states.jsonp'
+
+ try:
+ data = k.get_contents_as_string()
+
+ # No existing file
+ if not data:
+ raise S3ResponseError()
+
+ # Strip off jsonp wrapper
+ contents = utils.gunzip_data(data)
+ data = contents[7:-1]
+
+ states = json.loads(data)
+ except S3ResponseError:
+ states = []
+
+ if clear:
+ states = [state]
+
+ print 'Reset list of avialable states and added %s' % state
+ else:
+ if state not in states:
+ states.append(state)
+
+ print '%s added to available state list' % state
+ else:
+ print '%s is already available' % state
+
+ jsonp = 'states(%s)' % json.dumps(states)
+ compressed = utils.gzip_data(jsonp)
+
+ k.set_contents_from_string(compressed, headers={ 'Content-encoding': 'gzip', 'Content-Type': 'application/json' }, policy='public-read')
+
+if __name__ == '__main__':
+ if len(sys.argv) < 3:
+ sys.exit('You must specify either "staging" or "production" and a state as arguments to this script.')
+
+ ENVIRONMENT = sys.argv[1]
+ STATE = sys.argv[2]
+ try:
+ CLEAR = sys.argv[3]
+ except:
+ CLEAR = None
+
+ update_state_list(ENVIRONMENT, STATE, (CLEAR == 'CLEAR'))
Please sign in to comment.
Something went wrong with that request. Please try again.