From 7633f253049a1b464ae251b2587ea9ee9e8b05cb Mon Sep 17 00:00:00 2001 From: Will Kahn-Greene Date: Mon, 16 Oct 2017 10:36:14 -0400 Subject: [PATCH] fixes bug 1409038 - Add --dumps/--no-dumps flag to scripts/fetch_crash_data.py --- socorro/scripts/__init__.py | 37 +++++++++++ socorro/scripts/fetch_crash_data.py | 95 ++++++++++++++++------------- 2 files changed, 88 insertions(+), 44 deletions(-) diff --git a/socorro/scripts/__init__.py b/socorro/scripts/__init__.py index 7ebfa00ac3..13759aeb03 100644 --- a/socorro/scripts/__init__.py +++ b/socorro/scripts/__init__.py @@ -24,3 +24,40 @@ def _fill_text(self, text, width, indent): parts[i] = super(WrappedTextHelpFormatter, self)._fill_text(part, width, indent) return '\n\n'.join(parts) + + +class FlagAction(argparse.Action): + """Facilitates --flag, --no-flag arguments + + Usage:: + + parser.add_argument( + '--flag', '--no-flag', action=FlagAction, default=True, + help='whether to enable flag' + ) + + + Which allows you to do:: + + $ command + $ command --flag + $ command --no-flag + + + And the help works nicely, too:: + + $ command --help + ... + optional arguments: + --flag, --no-flag whether to enable flag + + """ + def __init__(self, option_strings, dest, nargs=None, **kwargs): + super(FlagAction, self).__init__(option_strings, dest, nargs=0, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + if option_string.startswith('--no'): + value = False + else: + value = True + setattr(namespace, self.dest, value) diff --git a/socorro/scripts/fetch_crash_data.py b/socorro/scripts/fetch_crash_data.py index 8b5bf1c54b..078355fa2c 100644 --- a/socorro/scripts/fetch_crash_data.py +++ b/socorro/scripts/fetch_crash_data.py @@ -13,7 +13,7 @@ import requests from socorro.lib.datetimeutil import JsonDTEncoder -from socorro.scripts import WrappedTextHelpFormatter +from socorro.scripts import FlagAction, WrappedTextHelpFormatter DESCRIPTION = """ @@ -46,7 +46,12 @@ def create_dir_if_needed(d): os.makedirs(d) -def fetch_crash(outputdir, api_token, crash_id): +def fetch_crash(fetchdumps, outputdir, api_token, crash_id): + """Fetch crash data and save to correct place + + http://antenna.readthedocs.io/en/latest/architecture.html#aws-s3-file-hierarchy + + """ if api_token: headers = { 'Auth-Token': api_token @@ -55,7 +60,6 @@ def fetch_crash(outputdir, api_token, crash_id): headers = {} # Fetch raw crash metadata - print('Fetching %s' % crash_id) resp = requests.get( HOST + '/api/RawCrash/', params={ @@ -69,55 +73,53 @@ def fetch_crash(outputdir, api_token, crash_id): raw_crash = resp.json() - # Fetch dumps - dumps = {} - dump_names = raw_crash.get('dump_checksums', {}).keys() - for dump_name in dump_names: - print('Fetching %s -> %s' % (crash_id, dump_name)) - if dump_name == 'upload_file_minidump': - dump_name = 'dump' - - resp = requests.get( - HOST + '/api/RawCrash/', - params={ - 'crash_id': crash_id, - 'format': 'raw', - 'name': dump_name - }, - headers=headers, - ) - - if resp.status_code != 200: - raise Exception('Something unexpected happened. status_code %s, content %s' % ( - resp.status_code, resp.content) - ) - - dumps[dump_name] = resp.content - - # Save everything to file system in the right place - # http://antenna.readthedocs.io/en/latest/architecture.html#aws-s3-file-hierarchy - # Save raw crash to file system fn = os.path.join(outputdir, 'v2', 'raw_crash', crash_id[0:3], '20' + crash_id[-6:], crash_id) create_dir_if_needed(os.path.dirname(fn)) with open(fn, 'w') as fp: json.dump(raw_crash, fp, cls=JsonDTEncoder, indent=2, sort_keys=True) - # Save dump_names to file system - fn = os.path.join(outputdir, 'v1', 'dump_names', crash_id) - create_dir_if_needed(os.path.dirname(fn)) - with open(fn, 'w') as fp: - json.dump(dumps.keys(), fp) + if fetchdumps: + # Fetch dumps + dumps = {} + dump_names = raw_crash.get('dump_checksums', {}).keys() + for dump_name in dump_names: + print('Fetching %s -> %s' % (crash_id, dump_name)) + if dump_name == 'upload_file_minidump': + dump_name = 'dump' + + resp = requests.get( + HOST + '/api/RawCrash/', + params={ + 'crash_id': crash_id, + 'format': 'raw', + 'name': dump_name + }, + headers=headers, + ) - # Save dumps to file system - for dump_name, data in dumps.items(): - if dump_name == 'upload_file_minidump': - dump_name = 'dump' + if resp.status_code != 200: + raise Exception('Something unexpected happened. status_code %s, content %s' % ( + resp.status_code, resp.content) + ) - fn = os.path.join(outputdir, 'v1', dump_name, crash_id) + dumps[dump_name] = resp.content + + # Save dump_names to file system + fn = os.path.join(outputdir, 'v1', 'dump_names', crash_id) create_dir_if_needed(os.path.dirname(fn)) - with open(fn, 'wb') as fp: - fp.write(data) + with open(fn, 'w') as fp: + json.dump(dumps.keys(), fp) + + # Save dumps to file system + for dump_name, data in dumps.items(): + if dump_name == 'upload_file_minidump': + dump_name = 'dump' + + fn = os.path.join(outputdir, 'v1', dump_name, crash_id) + create_dir_if_needed(os.path.dirname(fn)) + with open(fn, 'wb') as fp: + fp.write(data) def main(argv): @@ -127,6 +129,11 @@ def main(argv): description=DESCRIPTION.strip(), epilog=EPILOG.strip(), ) + parser.add_argument( + '--dumps', '--no-dumps', dest='fetchdumps', action=FlagAction, default=True, + help='whether or not to save dumps' + ) + parser.add_argument('outputdir', help='directory to place crash data in') parser.add_argument('crashid', nargs='*', help='one or more crash ids to fetch data for') @@ -149,6 +156,6 @@ def main(argv): crash_id = crash_id.strip() print('Working on %s...' % crash_id) - fetch_crash(outputdir, api_token, crash_id) + fetch_crash(args.fetchdumps, outputdir, api_token, crash_id) return 0