diff --git a/beetmoverscript/script.py b/beetmoverscript/script.py index 86fd81a8..68700929 100755 --- a/beetmoverscript/script.py +++ b/beetmoverscript/script.py @@ -166,10 +166,12 @@ async def push_to_maven(context): expected_files = list(mapping_manifest['mapping'].keys()) - context.artifacts_to_beetmove = check_and_extract_zip_archives( - context.artifacts_to_beetmove, expected_files, - context.config.get('zip_extract_max_file_size_in_mb', ZIP_MAX_COMPRESSION_RATIO) - ) + context.artifacts_to_beetmove = { + 'en-US': check_and_extract_zip_archives( + context.artifacts_to_beetmove, expected_files, + context.config.get('zip_extract_max_file_size_in_mb', ZIP_MAX_COMPRESSION_RATIO) + ) + } await move_beets(context, context.artifacts_to_beetmove, mapping_manifest) diff --git a/beetmoverscript/templates/maven_geckoview.yml b/beetmoverscript/templates/maven_geckoview.yml index 40008333..f9975094 100644 --- a/beetmoverscript/templates/maven_geckoview.yml +++ b/beetmoverscript/templates/maven_geckoview.yml @@ -7,53 +7,56 @@ metadata: s3_bucket_path: maven2/org/mozilla/{{ artifact_id }}/{{ version }}/ # Maven groupId is org.mozilla mapping: +{% for locale in ['en-US'] %} + "{{ locale }}": # Locale is not needed for geckoview, it's used by move_beets, though {% for product in ['geckoview'] %} - "{{ product }}.aar": - s3_key: {{ artifact_id }}-{{ version }}.aar - destinations: - - {{ artifact_id }}-{{ version }}.aar - "{{ product }}.aar.md5": - s3_key: {{ artifact_id }}-{{ version }}.aar.md5 - destinations: - - {{ artifact_id }}-{{ version }}.aar.md5 - "{{ product }}.aar.sha1": - s3_key: {{ artifact_id }}-{{ version }}.aar.sha1 - destinations: - - {{ artifact_id }}-{{ version }}.aar.sha1 - "{{ product }}.pom": - s3_key: {{ artifact_id }}-{{ version }}.pom - destinations: - - {{ artifact_id }}-{{ version }}.pom - "{{ product }}.pom.md5": - s3_key: {{ artifact_id }}-{{ version }}.pom.md5 - destinations: - - {{ artifact_id }}-{{ version }}.pom.md5 - "{{ product }}.pom.sha1": - s3_key: {{ artifact_id }}-{{ version }}.pom.sha1 - destinations: - - {{ artifact_id }}-{{ version }}.pom.sha1 - "{{ product }}-javadoc.jar": - s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar - destinations: - - {{ artifact_id }}-{{ version }}-javadoc.jar - "{{ product }}-javadoc.jar.md5": - s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar.md5 - destinations: - - {{ artifact_id }}-{{ version }}-javadoc.jar.md5 - "{{ product }}-javadoc.jar.sha1": - s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar.sha1 - destinations: - - {{ artifact_id }}-{{ version }}-javadoc.jar.sha1 - "{{ product }}-sources.jar": - s3_key: {{ artifact_id }}-{{ version }}-sources.jar - destinations: - - {{ artifact_id }}-{{ version }}-sources.jar - "{{ product }}-sources.jar.md5": - s3_key: {{ artifact_id }}-{{ version }}-sources.jar.md5 - destinations: - - {{ artifact_id }}-{{ version }}-sources.jar.md5 - "{{ product }}-sources.jar.sha1": - s3_key: {{ artifact_id }}-{{ version }}-sources.jar.sha1 - destinations: - - {{ artifact_id }}-{{ version }}-sources.jar.sha1 + "{{ product }}.aar": + s3_key: {{ artifact_id }}-{{ version }}.aar + destinations: + - {{ artifact_id }}-{{ version }}.aar + "{{ product }}.aar.md5": + s3_key: {{ artifact_id }}-{{ version }}.aar.md5 + destinations: + - {{ artifact_id }}-{{ version }}.aar.md5 + "{{ product }}.aar.sha1": + s3_key: {{ artifact_id }}-{{ version }}.aar.sha1 + destinations: + - {{ artifact_id }}-{{ version }}.aar.sha1 + "{{ product }}.pom": + s3_key: {{ artifact_id }}-{{ version }}.pom + destinations: + - {{ artifact_id }}-{{ version }}.pom + "{{ product }}.pom.md5": + s3_key: {{ artifact_id }}-{{ version }}.pom.md5 + destinations: + - {{ artifact_id }}-{{ version }}.pom.md5 + "{{ product }}.pom.sha1": + s3_key: {{ artifact_id }}-{{ version }}.pom.sha1 + destinations: + - {{ artifact_id }}-{{ version }}.pom.sha1 + "{{ product }}-javadoc.jar": + s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar + destinations: + - {{ artifact_id }}-{{ version }}-javadoc.jar + "{{ product }}-javadoc.jar.md5": + s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar.md5 + destinations: + - {{ artifact_id }}-{{ version }}-javadoc.jar.md5 + "{{ product }}-javadoc.jar.sha1": + s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar.sha1 + destinations: + - {{ artifact_id }}-{{ version }}-javadoc.jar.sha1 + "{{ product }}-sources.jar": + s3_key: {{ artifact_id }}-{{ version }}-sources.jar + destinations: + - {{ artifact_id }}-{{ version }}-sources.jar + "{{ product }}-sources.jar.md5": + s3_key: {{ artifact_id }}-{{ version }}-sources.jar.md5 + destinations: + - {{ artifact_id }}-{{ version }}-sources.jar.md5 + "{{ product }}-sources.jar.sha1": + s3_key: {{ artifact_id }}-{{ version }}-sources.jar.sha1 + destinations: + - {{ artifact_id }}-{{ version }}-sources.jar.sha1 +{% endfor %} {% endfor %} diff --git a/beetmoverscript/test/test_utils.py b/beetmoverscript/test/test_utils.py index 7d56b816..8e3e1e0f 100644 --- a/beetmoverscript/test/test_utils.py +++ b/beetmoverscript/test/test_utils.py @@ -97,53 +97,55 @@ def test_generate_manifest_maven(context, mocker): assert generate_beetmover_manifest(context) == { 'mapping': { - 'geckoview-javadoc.jar': { - 'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar'], - 's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar', - }, - 'geckoview-javadoc.jar.md5': { - 'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.md5'], - 's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.md5', - }, - 'geckoview-javadoc.jar.sha1': { - 'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.sha1'], - 's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.sha1', - }, - 'geckoview-sources.jar': { - 'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar'], - 's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar', - }, - 'geckoview-sources.jar.md5': { - 'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar.md5'], - 's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar.md5', - }, - 'geckoview-sources.jar.sha1': { - 'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar.sha1'], - 's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar.sha1', - }, - 'geckoview.aar': { - 'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar'], - 's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar', - }, - 'geckoview.aar.md5': { - 'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar.md5'], - 's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar.md5', - }, - 'geckoview.aar.sha1': { - 'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar.sha1'], - 's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar.sha1', - }, - 'geckoview.pom': { - 'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom'], - 's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom', - }, - 'geckoview.pom.md5': { - 'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom.md5'], - 's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom.md5', - }, - 'geckoview.pom.sha1': { - 'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom.sha1'], - 's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom.sha1' + 'en-US': { + 'geckoview-javadoc.jar': { + 'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar'], + 's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar', + }, + 'geckoview-javadoc.jar.md5': { + 'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.md5'], + 's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.md5', + }, + 'geckoview-javadoc.jar.sha1': { + 'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.sha1'], + 's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.sha1', + }, + 'geckoview-sources.jar': { + 'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar'], + 's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar', + }, + 'geckoview-sources.jar.md5': { + 'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar.md5'], + 's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar.md5', + }, + 'geckoview-sources.jar.sha1': { + 'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar.sha1'], + 's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar.sha1', + }, + 'geckoview.aar': { + 'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar'], + 's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar', + }, + 'geckoview.aar.md5': { + 'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar.md5'], + 's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar.md5', + }, + 'geckoview.aar.sha1': { + 'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar.sha1'], + 's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar.sha1', + }, + 'geckoview.pom': { + 'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom'], + 's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom', + }, + 'geckoview.pom.md5': { + 'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom.md5'], + 's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom.md5', + }, + 'geckoview.pom.sha1': { + 'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom.sha1'], + 's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom.sha1' + }, }, }, 'metadata': { diff --git a/beetmoverscript/test/test_zip.py b/beetmoverscript/test/test_zip.py index c3216162..069cd058 100644 --- a/beetmoverscript/test/test_zip.py +++ b/beetmoverscript/test/test_zip.py @@ -90,21 +90,17 @@ def test_check_and_extract_zip_archives(): artifacts_per_task_id, expected_files_per_archive_per_task_id, zip_max_size_in_mb=100 ) - assert files_once_extracted == { - 'firstTaskId': [ - '/a/non/archive', - '/another/non/archive', - os.path.join(d, 'firstTaskId-archive1.zip.out', 'some_file1'), - os.path.join(d, 'firstTaskId-archive1.zip.out', 'some', 'subfolder', 'file1'), - os.path.join(d, 'firstTaskId-archive2.zip.out', 'some_file2'), - os.path.join(d, 'firstTaskId-archive2.zip.out', 'some', 'subfolder', 'file2'), - ], - 'secondTaskId': ['/just/another/regular/file'], - 'thirdTaskId': [ - os.path.join(d, 'thirdTaskId-archive1.zip.out', 'some_file3'), - os.path.join(d, 'thirdTaskId-archive1.zip.out', 'some', 'subfolder', 'file3'), - ], - } + assert sorted(files_once_extracted) == sorted([ + '/a/non/archive', + '/another/non/archive', + os.path.join(d, 'firstTaskId-archive1.zip.out', 'some_file1'), + os.path.join(d, 'firstTaskId-archive1.zip.out', 'some', 'subfolder', 'file1'), + os.path.join(d, 'firstTaskId-archive2.zip.out', 'some_file2'), + os.path.join(d, 'firstTaskId-archive2.zip.out', 'some', 'subfolder', 'file2'), + '/just/another/regular/file', + os.path.join(d, 'thirdTaskId-archive1.zip.out', 'some_file3'), + os.path.join(d, 'thirdTaskId-archive1.zip.out', 'some', 'subfolder', 'file3'), + ]) def test_check_and_extract_zip_archives_for_given_task(): diff --git a/beetmoverscript/zip.py b/beetmoverscript/zip.py index 092d1224..7810554d 100644 --- a/beetmoverscript/zip.py +++ b/beetmoverscript/zip.py @@ -10,26 +10,23 @@ def check_and_extract_zip_archives(artifacts_per_task_id, expected_files_per_archive_per_task_id, zip_max_size_in_mb): - deflated_artifacts_per_task_id = {} + deflated_artifacts = [] for task_id, task_artifacts_params in artifacts_per_task_id.items(): for artifacts_param in task_artifacts_params: paths_for_task = artifacts_param['paths'] - deflated_artifacts = deflated_artifacts_per_task_id.get(task_id, []) if artifacts_param['zip_extract'] is False: log.debug('Skipping artifacts marked as not `zipExtract`able: {}'.format(paths_for_task)) deflated_artifacts.extend(paths_for_task) - deflated_artifacts_per_task_id[task_id] = deflated_artifacts continue expected_files_per_archive = expected_files_per_archive_per_task_id[task_id] deflated_artifacts.extend(_check_and_extract_zip_archives_for_given_task( task_id, expected_files_per_archive, zip_max_size_in_mb )) - deflated_artifacts_per_task_id[task_id] = deflated_artifacts - return deflated_artifacts_per_task_id + return deflated_artifacts def _check_and_extract_zip_archives_for_given_task(task_id, expected_files_per_archive, zip_max_size_in_mb):