Skip to content
This repository
Browse code

Updated batch scripts so state is marked available when it is made pu…

…blic. Closes #117.
  • Loading branch information...
commit d85fab2e3c722018477e6bcacc650cf3c4af67a0 1 parent c32d0d8
cgroskopf authored
1  dataprocessing/batch_sf.sh
@@ -73,5 +73,4 @@ echo 'Deploying to S3'
73 73 ./deploy_data.py $ENVIRONMENT
74 74 ./deploy_lookups.py $ENVIRONMENT
75 75 ./deploy_labels.py $ENVIRONMENT
76   -./update_state_list.py $ENVIRONMENT $STATE_NAME
77 76
6 dataprocessing/make_state_public.py
@@ -6,12 +6,14 @@
6 6
7 7 import config
8 8 import get_state_fips
  9 +import update_state_list
9 10
10 11 if len(sys.argv) < 3:
11 12 sys.exit('You must "staging" or "production" and a state name as arguments to this script.')
12 13
13 14 ENVIRONMENT = sys.argv[1]
14   -STATE_FIPS = get_state_fips.STATE_FIPS[sys.argv[2]]
  15 +STATE = sys.argv[2]
  16 +STATE_FIPS = get_state_fips.STATE_FIPS[STATE]
15 17
16 18 c = S3Connection()
17 19 bucket = c.get_bucket(config.S3_BUCKETS[ENVIRONMENT])
@@ -22,3 +24,5 @@
22 24 if i % 100 == 0:
23 25 print 'Processed %i...' % i
24 26
  27 +# Update available states list
  28 +update_state_list.update_state_list(ENVIRONMENT, STATE)
99 dataprocessing/update_state_list.py
@@ -6,55 +6,58 @@
6 6 from boto.exception import S3ResponseError
7 7 from boto.s3.connection import S3Connection
8 8 from boto.s3.key import Key
9   -from pymongo import Connection
10 9
11 10 import config
12 11 import utils
13 12
14   -if len(sys.argv) < 3:
15   - sys.exit('You must specify either "staging" or "production" and a state as arguments to this script.')
16   -
17   -ENVIRONMENT = sys.argv[1]
18   -STATE = sys.argv[2]
19   -try: CLEAR = sys.argv[3]
20   -except: CLEAR = None
21   -
22   -connection = Connection()
23   -db = connection[config.CENSUS_DB]
24   -collection = db[config.GEOGRAPHIES_COLLECTION]
25   -
26   -row_count = 0
27   -deployed = 0
28   -
29   -c = S3Connection()
30   -bucket = c.get_bucket(config.S3_BUCKETS[ENVIRONMENT])
31   -
32   -k = Key(bucket)
33   -k.key = 'states.jsonp'
34   -
35   -try:
36   - data = k.get_contents_as_string()
37   -
38   - # No existing file
39   - if not data:
40   - raise S3ResponseError()
41   -
42   - # Strip off jsonp wrapper
43   - contents = utils.gunzip_data(data)
44   - data = contents[7:-1]
45   -
46   - states = json.loads(data)
47   -except S3ResponseError:
48   - states = []
49   -
50   -if CLEAR == 'CLEAR':
51   - states = [STATE]
52   -else:
53   - if STATE not in states:
54   - states.append(STATE)
55   -
56   -jsonp = 'states(%s)' % json.dumps(states)
57   -compressed = utils.gzip_data(jsonp)
58   -
59   -k.set_contents_from_string(compressed, headers={ 'Content-encoding': 'gzip', 'Content-Type': 'application/json' }, policy='public-read')
60   -
  13 +def update_state_list(environment, state, clear=False):
  14 + c = S3Connection()
  15 + bucket = c.get_bucket(config.S3_BUCKETS[environment])
  16 +
  17 + k = Key(bucket)
  18 + k.key = 'states.jsonp'
  19 +
  20 + try:
  21 + data = k.get_contents_as_string()
  22 +
  23 + # No existing file
  24 + if not data:
  25 + raise S3ResponseError()
  26 +
  27 + # Strip off jsonp wrapper
  28 + contents = utils.gunzip_data(data)
  29 + data = contents[7:-1]
  30 +
  31 + states = json.loads(data)
  32 + except S3ResponseError:
  33 + states = []
  34 +
  35 + if clear:
  36 + states = [state]
  37 +
  38 + print 'Reset list of avialable states and added %s' % state
  39 + else:
  40 + if state not in states:
  41 + states.append(state)
  42 +
  43 + print '%s added to available state list' % state
  44 + else:
  45 + print '%s is already available' % state
  46 +
  47 + jsonp = 'states(%s)' % json.dumps(states)
  48 + compressed = utils.gzip_data(jsonp)
  49 +
  50 + k.set_contents_from_string(compressed, headers={ 'Content-encoding': 'gzip', 'Content-Type': 'application/json' }, policy='public-read')
  51 +
  52 +if __name__ == '__main__':
  53 + if len(sys.argv) < 3:
  54 + sys.exit('You must specify either "staging" or "production" and a state as arguments to this script.')
  55 +
  56 + ENVIRONMENT = sys.argv[1]
  57 + STATE = sys.argv[2]
  58 + try:
  59 + CLEAR = sys.argv[3]
  60 + except:
  61 + CLEAR = None
  62 +
  63 + update_state_list(ENVIRONMENT, STATE, (CLEAR == 'CLEAR'))

0 comments on commit d85fab2

Please sign in to comment.
Something went wrong with that request. Please try again.