From cb0ea3353716b5f9d60533d0e25f452909fa9b13 Mon Sep 17 00:00:00 2001 From: aschroed Date: Thu, 26 Jul 2018 12:32:50 -0400 Subject: [PATCH] Couple of tweaks in graceful error handling and tests to match --- tests/test_import_data.py | 12 ++++-------- wranglertools/import_data.py | 32 +++++++++++++++----------------- 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/tests/test_import_data.py b/tests/test_import_data.py index 54382bba..91a0ccf3 100644 --- a/tests/test_import_data.py +++ b/tests/test_import_data.py @@ -994,7 +994,7 @@ def test_check_extra_file_meta_w_format_filename_new_file(mocker): assert ff in seen -def test_check_extra_file_meta_w_filename_seen_format(mocker, capsys): +def test_check_extra_file_meta_w_filename_seen_format(mocker): fn = '/test/path/to/file/test_pairs_index.pairs.gz.px2' ff = 'pairs_px2' md5sum = 'mymd5' @@ -1003,14 +1003,12 @@ def test_check_extra_file_meta_w_filename_seen_format(mocker, capsys): with mocker.patch('wranglertools.import_data.md5', return_value=md5sum): with mocker.patch('wranglertools.import_data.os.path.getsize', return_value=fsize): result, seen = imp.check_extra_file_meta(data, ['pairs_px2'], []) - out = capsys.readouterr()[0] assert result['file_format'] == ff assert result['filename'] == fn assert result['md5sum'] == md5sum assert result['filesize'] == fsize assert result['submitted_filename'] == 'test_pairs_index.pairs.gz.px2' assert ff in seen - assert 'Warning each file in extra_files must have unique file_format' in out def test_check_extra_file_meta_malformed_data(capsys): @@ -1018,16 +1016,14 @@ def test_check_extra_file_meta_malformed_data(capsys): result, _ = imp.check_extra_file_meta(fn, [], []) out = capsys.readouterr()[0] assert not result - assert 'Malformed extrafile field formatting' in out + assert 'WARNING! -- Malformed extrafile field formatting' in out -def test_check_extra_file_meta_no_file_format(capsys): +def test_check_extra_file_meta_no_file_format(): fn = '/test/path/to/file/test_pairs_index.pairs.gz.px2' data = {'filename': fn} result, _ = imp.check_extra_file_meta(data, [], []) - out = capsys.readouterr()[0] - assert not result - assert 'extrafiles.file_format is required' in out + assert result == data def test_check_extra_file_meta_w_filename_existing_format(mocker, capsys): diff --git a/wranglertools/import_data.py b/wranglertools/import_data.py index 05e4f1e8..6e29335f 100755 --- a/wranglertools/import_data.py +++ b/wranglertools/import_data.py @@ -530,13 +530,11 @@ def check_extra_file_meta(ef_info, seen_formats, existing_formats): try: ef_format = ef_info.get('file_format') except AttributeError: - print('Malformed extrafile field formatting', ef_info) + print('WARNING! -- Malformed extrafile field formatting', ef_info) return None, seen_formats - if not ef_format: - print('extrafiles.file_format is required') - return None, seen_formats - if ef_format in seen_formats: - print("Warning each file in extra_files must have unique file_format") + else: + if not ef_format: + return ef_info, seen_formats if ef_format in existing_formats: print("An extrafile with %s format exists - will attempt to patch" % ef_format) @@ -600,20 +598,20 @@ def populate_post_json(post_json, connection, sheet): # , existing_data): if existing_data: if existing_data.get('extra_files'): existing_extrafiles = existing_data.get('extra_files') # to include existing - existing_formats = [ef.get('file_format') for ef in existing_data.get('extra_files') - if ef.get('file_format') is not None] + existing_formats = [ef.get('file_format') for ef in existing_data.get('extra_files')] seen_formats = [] for extrafile in extrafiles: extrafile_meta, seen_formats = check_extra_file_meta(extrafile, seen_formats, existing_formats) - if extrafile_meta is not None: - if extrafile_meta.get('filename'): - extrafiles2upload[extrafile_meta['file_format']] = extrafile_meta['filename'] - del extrafile_meta['filename'] - for ix, eef in enumerate(existing_extrafiles): - if eef['file_format'] == extrafile_meta['file_format']: - # we are patching so want to remove existing entry from existing_extrafiles - del existing_extrafiles[ix] - break + if extrafile_meta: + if extrafile_meta.get('file_format'): + if extrafile_meta.get('filename'): + extrafiles2upload[extrafile_meta['file_format']] = extrafile_meta['filename'] + del extrafile_meta['filename'] + for ix, eef in enumerate(existing_extrafiles): + if eef['file_format'] == extrafile_meta['file_format']: + # we are patching so want to remove existing entry from existing_extrafiles + del existing_extrafiles[ix] + break extrafile_metadata.append(extrafile_meta) if extrafile_metadata: