From d7f7a1dd1db460cb59909f6f15c93ceb330fab56 Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Mon, 23 Jul 2018 13:45:57 +0200 Subject: [PATCH] Fix wire --- beetmoverscript/script.py | 17 +++++++---- beetmoverscript/templates/maven_geckoview.yml | 24 ++++++++-------- beetmoverscript/test/test_script.py | 28 ++++++++++++++++++- beetmoverscript/test/test_utils.py | 24 ++++++++-------- beetmoverscript/zip.py | 2 +- 5 files changed, 63 insertions(+), 32 deletions(-) diff --git a/beetmoverscript/script.py b/beetmoverscript/script.py index 68700929..c82908a4 100755 --- a/beetmoverscript/script.py +++ b/beetmoverscript/script.py @@ -17,15 +17,18 @@ from scriptworker.exceptions import ScriptWorkerTaskException, ScriptWorkerRetryException from scriptworker.utils import retry_async, raise_future_exceptions +from beetmoverscript import task, zip + from beetmoverscript.constants import ( MIME_MAP, RELEASE_BRANCHES, CACHE_CONTROL_MAXAGE, RELEASE_EXCLUDE, NORMALIZED_BALROG_PLATFORMS, PARTNER_REPACK_PUBLIC_PREFIX_TMPL, PARTNER_REPACK_PRIVATE_REGEXES, PARTNER_REPACK_PUBLIC_REGEXES, BUILDHUB_ARTIFACT, INSTALLER_ARTIFACTS, ZIP_MAX_COMPRESSION_RATIO ) +from beetmoverscript.maven import get_maven_expected_files_per_archive_per_task_id from beetmoverscript.task import ( validate_task_schema, add_balrog_manifest_to_artifacts, - get_upstream_artifacts, get_upstream_artifacts_with_zip_extract_param, get_release_props, + get_upstream_artifacts, get_release_props, add_checksums_to_artifacts, get_task_bucket, get_task_action, validate_bucket_paths, get_updated_buildhub_artifact ) @@ -38,7 +41,6 @@ get_product_name, is_partner_action, is_partner_private_task, is_partner_public_task, write_json ) -from beetmoverscript.zip import check_and_extract_zip_archives log = logging.getLogger(__name__) @@ -157,18 +159,21 @@ async def push_to_releases(context): async def push_to_maven(context): """Push artifacts to locations expected by maven clients (like mvn or gradle)""" - context.artifacts_to_beetmove = get_upstream_artifacts_with_zip_extract_param(context) + context.artifacts_to_beetmove = task.get_upstream_artifacts_with_zip_extract_param(context) context.release_props = get_release_props(context) context.checksums = dict() # Needed by downstream calls mapping_manifest = generate_beetmover_manifest(context) validate_bucket_paths(context.bucket, mapping_manifest['s3_bucket_path']) - expected_files = list(mapping_manifest['mapping'].keys()) + expected_files = get_maven_expected_files_per_archive_per_task_id( + context.task['payload']['upstreamArtifacts'], mapping_manifest + ) context.artifacts_to_beetmove = { - 'en-US': check_and_extract_zip_archives( - context.artifacts_to_beetmove, expected_files, + 'en-US': zip.check_and_extract_zip_archives( + context.artifacts_to_beetmove, + expected_files, context.config.get('zip_extract_max_file_size_in_mb', ZIP_MAX_COMPRESSION_RATIO) ) } diff --git a/beetmoverscript/templates/maven_geckoview.yml b/beetmoverscript/templates/maven_geckoview.yml index f9975094..e198523e 100644 --- a/beetmoverscript/templates/maven_geckoview.yml +++ b/beetmoverscript/templates/maven_geckoview.yml @@ -10,51 +10,51 @@ mapping: {% for locale in ['en-US'] %} "{{ locale }}": # Locale is not needed for geckoview, it's used by move_beets, though {% for product in ['geckoview'] %} - "{{ product }}.aar": + "{{ artifact_id }}-{{ version }}.aar": s3_key: {{ artifact_id }}-{{ version }}.aar destinations: - {{ artifact_id }}-{{ version }}.aar - "{{ product }}.aar.md5": + "{{ artifact_id }}-{{ version }}.aar.md5": s3_key: {{ artifact_id }}-{{ version }}.aar.md5 destinations: - {{ artifact_id }}-{{ version }}.aar.md5 - "{{ product }}.aar.sha1": + "{{ artifact_id }}-{{ version }}.aar.sha1": s3_key: {{ artifact_id }}-{{ version }}.aar.sha1 destinations: - {{ artifact_id }}-{{ version }}.aar.sha1 - "{{ product }}.pom": + "{{ artifact_id }}-{{ version }}.pom": s3_key: {{ artifact_id }}-{{ version }}.pom destinations: - {{ artifact_id }}-{{ version }}.pom - "{{ product }}.pom.md5": + "{{ artifact_id }}-{{ version }}.pom.md5": s3_key: {{ artifact_id }}-{{ version }}.pom.md5 destinations: - {{ artifact_id }}-{{ version }}.pom.md5 - "{{ product }}.pom.sha1": + "{{ artifact_id }}-{{ version }}.pom.sha1": s3_key: {{ artifact_id }}-{{ version }}.pom.sha1 destinations: - {{ artifact_id }}-{{ version }}.pom.sha1 - "{{ product }}-javadoc.jar": + "{{ artifact_id }}-{{ version }}-javadoc.jar": s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar destinations: - {{ artifact_id }}-{{ version }}-javadoc.jar - "{{ product }}-javadoc.jar.md5": + "{{ artifact_id }}-{{ version }}-javadoc.jar.md5": s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar.md5 destinations: - {{ artifact_id }}-{{ version }}-javadoc.jar.md5 - "{{ product }}-javadoc.jar.sha1": + "{{ artifact_id }}-{{ version }}-javadoc.jar.sha1": s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar.sha1 destinations: - {{ artifact_id }}-{{ version }}-javadoc.jar.sha1 - "{{ product }}-sources.jar": + "{{ artifact_id }}-{{ version }}-sources.jar": s3_key: {{ artifact_id }}-{{ version }}-sources.jar destinations: - {{ artifact_id }}-{{ version }}-sources.jar - "{{ product }}-sources.jar.md5": + "{{ artifact_id }}-{{ version }}-sources.jar.md5": s3_key: {{ artifact_id }}-{{ version }}-sources.jar.md5 destinations: - {{ artifact_id }}-{{ version }}-sources.jar.md5 - "{{ product }}-sources.jar.sha1": + "{{ artifact_id }}-{{ version }}-sources.jar.sha1": s3_key: {{ artifact_id }}-{{ version }}-sources.jar.sha1 destinations: - {{ artifact_id }}-{{ version }}-sources.jar.sha1 diff --git a/beetmoverscript/test/test_script.py b/beetmoverscript/test/test_script.py index 255068c1..79ed0bf4 100644 --- a/beetmoverscript/test/test_script.py +++ b/beetmoverscript/test/test_script.py @@ -92,8 +92,34 @@ def fake_list(*args): @pytest.mark.asyncio async def test_push_to_maven(context, mocker): - mocker.patch('beetmoverscript.script.move_beets', new=noop_async) + async def assert_artifacts_to_beetmove(_, artifacts_to_beetmove, __): + assert artifacts_to_beetmove == {'en-US': [ + '/work_dir/cot/someTaskId/public/build/target.maven.zip.out/org/mozilla/geckoview-beta-x86/62.0b3/geckoview-beta-x86-62.0b3.aar', + '/work_dir/cot/someTaskId/public/build/target.maven.zip.out/org/mozilla/geckoview-beta-x86/62.0b3/geckoview-beta-x86-62.0b3.aar.md5', + '/work_dir/cot/someTaskId/public/build/target.maven.zip.out/org/mozilla/geckoview-beta-x86/62.0b3/geckoview-beta-x86-62.0b3.aar.sha1', + ]} + + mocker.patch('beetmoverscript.script.move_beets', new=assert_artifacts_to_beetmove) + mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env()) + context.task['payload']['upstreamArtifacts'] = [{ + 'paths': ['public/build/target.maven.zip'], + 'taskId': 'someTaskId', + 'taskType': 'build', + 'zipExtract': True, + }] + mocker.patch('beetmoverscript.task.get_upstream_artifacts_with_zip_extract_param', new=lambda _: { + 'someTaskId': [{ + 'paths': ['/work_dir/cot/someTaskId/public/build/target.maven.zip'], + 'zip_extract': True, + }] + }) + mocker.patch('beetmoverscript.zip.check_and_extract_zip_archives', new=lambda _, __, ___: [ + '/work_dir/cot/someTaskId/public/build/target.maven.zip.out/org/mozilla/geckoview-beta-x86/62.0b3/geckoview-beta-x86-62.0b3.aar', + '/work_dir/cot/someTaskId/public/build/target.maven.zip.out/org/mozilla/geckoview-beta-x86/62.0b3/geckoview-beta-x86-62.0b3.aar.md5', + '/work_dir/cot/someTaskId/public/build/target.maven.zip.out/org/mozilla/geckoview-beta-x86/62.0b3/geckoview-beta-x86-62.0b3.aar.sha1', + ]) + await push_to_maven(context) diff --git a/beetmoverscript/test/test_utils.py b/beetmoverscript/test/test_utils.py index 8e3e1e0f..478c9d68 100644 --- a/beetmoverscript/test/test_utils.py +++ b/beetmoverscript/test/test_utils.py @@ -98,51 +98,51 @@ def test_generate_manifest_maven(context, mocker): assert generate_beetmover_manifest(context) == { 'mapping': { 'en-US': { - 'geckoview-javadoc.jar': { + 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar': { 'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar'], 's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar', }, - 'geckoview-javadoc.jar.md5': { + 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.md5': { 'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.md5'], 's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.md5', }, - 'geckoview-javadoc.jar.sha1': { + 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.sha1': { 'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.sha1'], 's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.sha1', }, - 'geckoview-sources.jar': { + 'geckoview-nightly-x86-63.0.20181231120000-sources.jar': { 'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar'], 's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar', }, - 'geckoview-sources.jar.md5': { + 'geckoview-nightly-x86-63.0.20181231120000-sources.jar.md5': { 'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar.md5'], 's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar.md5', }, - 'geckoview-sources.jar.sha1': { + 'geckoview-nightly-x86-63.0.20181231120000-sources.jar.sha1': { 'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar.sha1'], 's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar.sha1', }, - 'geckoview.aar': { + 'geckoview-nightly-x86-63.0.20181231120000.aar': { 'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar'], 's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar', }, - 'geckoview.aar.md5': { + 'geckoview-nightly-x86-63.0.20181231120000.aar.md5': { 'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar.md5'], 's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar.md5', }, - 'geckoview.aar.sha1': { + 'geckoview-nightly-x86-63.0.20181231120000.aar.sha1': { 'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar.sha1'], 's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar.sha1', }, - 'geckoview.pom': { + 'geckoview-nightly-x86-63.0.20181231120000.pom': { 'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom'], 's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom', }, - 'geckoview.pom.md5': { + 'geckoview-nightly-x86-63.0.20181231120000.pom.md5': { 'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom.md5'], 's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom.md5', }, - 'geckoview.pom.sha1': { + 'geckoview-nightly-x86-63.0.20181231120000.pom.sha1': { 'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom.sha1'], 's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom.sha1' }, diff --git a/beetmoverscript/zip.py b/beetmoverscript/zip.py index 7810554d..db46bdd7 100644 --- a/beetmoverscript/zip.py +++ b/beetmoverscript/zip.py @@ -142,7 +142,7 @@ def _ensure_all_expected_files_are_present_in_archive(zip_path, files_in_archive ) if file not in unique_expected_files: raise TaskVerificationError( - 'File "{}" is present in archive "{}" is not expected.'.format( + 'File "{}" present in archive "{}" is not expected.'.format( file, zip_path ) )