All tests back to green

mozilla-releng · Jul 23, 2018 · e227115 · e227115
1 parent ed3d38b
commit e227115
Show file tree

Hide file tree

Showing 4 changed files with 44 additions and 43 deletions.
diff --git a/beetmoverscript/script.py b/beetmoverscript/script.py
@@ -21,11 +21,11 @@
     MIME_MAP, RELEASE_BRANCHES, CACHE_CONTROL_MAXAGE, RELEASE_EXCLUDE,
     NORMALIZED_BALROG_PLATFORMS, PARTNER_REPACK_PUBLIC_PREFIX_TMPL,
     PARTNER_REPACK_PRIVATE_REGEXES, PARTNER_REPACK_PUBLIC_REGEXES, BUILDHUB_ARTIFACT,
-    INSTALLER_ARTIFACTS
+    INSTALLER_ARTIFACTS, ZIP_MAX_COMPRESSION_RATIO
 )
 from beetmoverscript.task import (
     validate_task_schema, add_balrog_manifest_to_artifacts,
-    get_upstream_artifacts, get_release_props,
+    get_upstream_artifacts, get_upstream_artifacts_with_zip_extract_param, get_release_props,
     add_checksums_to_artifacts, get_task_bucket, get_task_action, validate_bucket_paths,
     get_updated_buildhub_artifact
 )
@@ -157,7 +157,7 @@ async def push_to_releases(context):
 
 async def push_to_maven(context):
     """Push artifacts to locations expected by maven clients (like mvn or gradle)"""
-    context.artifacts_to_beetmove = get_upstream_artifacts_with_extract_param(context)
+    context.artifacts_to_beetmove = get_upstream_artifacts_with_zip_extract_param(context)
     context.release_props = get_release_props(context)
     context.checksums = dict()  # Needed by downstream calls
 
@@ -167,7 +167,8 @@ async def push_to_maven(context):
     expected_files = list(mapping_manifest['mapping'].keys())
 
     context.artifacts_to_beetmove = check_and_extract_zip_archives(
-        context.artifacts_to_beetmove, expected_files, context.config['zip_extract_max_file_size_in_mb']
+        context.artifacts_to_beetmove, expected_files,
+        context.config.get('zip_extract_max_file_size_in_mb', ZIP_MAX_COMPRESSION_RATIO)
     )
 
     await move_beets(context, context.artifacts_to_beetmove, mapping_manifest)

diff --git a/beetmoverscript/templates/maven_geckoview.yml b/beetmoverscript/templates/maven_geckoview.yml
@@ -8,51 +8,51 @@ s3_bucket_path: maven2/org/mozilla/{{ artifact_id }}/{{ version }}/  # Maven gro
 
 mapping:
 {% for product in ['geckoview'] %}
-  "org/mozilla/{{ artifact_id }}/{{ version }}/{{ product }}.aar":
+  "{{ product }}.aar":
     s3_key: {{ artifact_id }}-{{ version }}.aar
     destinations:
       - {{ artifact_id }}-{{ version }}.aar
-  "org/mozilla/{{ artifact_id }}/{{ version }}/{{ product }}.aar.md5":
+  "{{ product }}.aar.md5":
     s3_key: {{ artifact_id }}-{{ version }}.aar.md5
     destinations:
       - {{ artifact_id }}-{{ version }}.aar.md5
-  "org/mozilla/{{ artifact_id }}/{{ version }}/{{ product }}.aar.sha1":
+  "{{ product }}.aar.sha1":
     s3_key: {{ artifact_id }}-{{ version }}.aar.sha1
     destinations:
       - {{ artifact_id }}-{{ version }}.aar.sha1
-  "org/mozilla/{{ artifact_id }}/{{ version }}/{{ product }}.pom":
+  "{{ product }}.pom":
     s3_key: {{ artifact_id }}-{{ version }}.pom
     destinations:
       - {{ artifact_id }}-{{ version }}.pom
-  "org/mozilla/{{ artifact_id }}/{{ version }}/{{ product }}.pom.md5":
+  "{{ product }}.pom.md5":
     s3_key: {{ artifact_id }}-{{ version }}.pom.md5
     destinations:
       - {{ artifact_id }}-{{ version }}.pom.md5
-  "org/mozilla/{{ artifact_id }}/{{ version }}/{{ product }}.pom.sha1":
+  "{{ product }}.pom.sha1":
     s3_key: {{ artifact_id }}-{{ version }}.pom.sha1
     destinations:
       - {{ artifact_id }}-{{ version }}.pom.sha1
-  "org/mozilla/{{ artifact_id }}/{{ version }}/{{ product }}-javadoc.jar":
+  "{{ product }}-javadoc.jar":
     s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar
     destinations:
       - {{ artifact_id }}-{{ version }}-javadoc.jar
-  "org/mozilla/{{ artifact_id }}/{{ version }}/{{ product }}-javadoc.jar.md5":
+  "{{ product }}-javadoc.jar.md5":
     s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar.md5
     destinations:
       - {{ artifact_id }}-{{ version }}-javadoc.jar.md5
-  "org/mozilla/{{ artifact_id }}/{{ version }}/{{ product }}-javadoc.jar.sha1":
+  "{{ product }}-javadoc.jar.sha1":
     s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar.sha1
     destinations:
       - {{ artifact_id }}-{{ version }}-javadoc.jar.sha1
-  "org/mozilla/{{ artifact_id }}/{{ version }}/{{ product }}-sources.jar":
+  "{{ product }}-sources.jar":
     s3_key: {{ artifact_id }}-{{ version }}-sources.jar
     destinations:
       - {{ artifact_id }}-{{ version }}-sources.jar
-  "org/mozilla/{{ artifact_id }}/{{ version }}/{{ product }}-sources.jar.md5":
+  "{{ product }}-sources.jar.md5":
     s3_key: {{ artifact_id }}-{{ version }}-sources.jar.md5
     destinations:
       - {{ artifact_id }}-{{ version }}-sources.jar.md5
-  "org/mozilla/{{ artifact_id }}/{{ version }}/{{ product }}-sources.jar.sha1":
+  "{{ product }}-sources.jar.sha1":
     s3_key: {{ artifact_id }}-{{ version }}-sources.jar.sha1
     destinations:
       - {{ artifact_id }}-{{ version }}-sources.jar.sha1

diff --git a/beetmoverscript/test/test_utils.py b/beetmoverscript/test/test_utils.py
@@ -98,60 +98,60 @@ def test_generate_manifest_maven(context, mocker):
     assert generate_beetmover_manifest(context) == {
         'mapping': {
             'geckoview-javadoc.jar': {
-                'destinations': ['geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000-javadoc.jar'],
-                's3_key': 'geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000-javadoc.jar',
+                'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar'],
+                's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar',
             },
             'geckoview-javadoc.jar.md5': {
-                'destinations': ['geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.md5'],
-                's3_key': 'geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.md5',
+                'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.md5'],
+                's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.md5',
             },
             'geckoview-javadoc.jar.sha1': {
-                'destinations': ['geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.sha1'],
-                's3_key': 'geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.sha1',
+                'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.sha1'],
+                's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.sha1',
             },
             'geckoview-sources.jar': {
-                'destinations': ['geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000-sources.jar'],
-                's3_key': 'geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000-sources.jar',
+                'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar'],
+                's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar',
             },
             'geckoview-sources.jar.md5': {
-                'destinations': ['geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000-sources.jar.md5'],
-                's3_key': 'geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000-sources.jar.md5',
+                'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar.md5'],
+                's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar.md5',
             },
             'geckoview-sources.jar.sha1': {
-                'destinations': ['geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000-sources.jar.sha1'],
-                's3_key': 'geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000-sources.jar.sha1',
+                'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar.sha1'],
+                's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar.sha1',
             },
             'geckoview.aar': {
-                'destinations': ['geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000.aar'],
-                's3_key': 'geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000.aar',
+                'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar'],
+                's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar',
             },
             'geckoview.aar.md5': {
-                'destinations': ['geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000.aar.md5'],
-                's3_key': 'geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000.aar.md5',
+                'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar.md5'],
+                's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar.md5',
             },
             'geckoview.aar.sha1': {
-                'destinations': ['geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000.aar.sha1'],
-                's3_key': 'geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000.aar.sha1',
+                'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar.sha1'],
+                's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar.sha1',
             },
             'geckoview.pom': {
-                'destinations': ['geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000.pom'],
-                's3_key': 'geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000.pom',
+                'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom'],
+                's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom',
             },
             'geckoview.pom.md5': {
-                'destinations': ['geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000.pom.md5'],
-                's3_key': 'geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000.pom.md5',
+                'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom.md5'],
+                's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom.md5',
             },
             'geckoview.pom.sha1': {
-                'destinations': ['geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000.pom.sha1'],
-                's3_key': 'geckoview-nightly-x86/63.0.20181231120000/geckoview-nightly-x86-63.0.20181231120000.pom.sha1'
+                'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom.sha1'],
+                's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom.sha1'
             },
         },
         'metadata': {
             'description': "Maps artifacts to spec'd maven location",
             'name': 'Maven repository',
             'owner': 'release@mozilla.com',
         },
-        's3_bucket_path': 'maven2/org/mozilla/',
+        's3_bucket_path': 'maven2/org/mozilla/geckoview-nightly-x86/63.0.20181231120000/',
     }
 
 
@@ -307,7 +307,7 @@ def test_beetmover_template_args_maven(context, branch, version, artifact_id, bu
         'artifact_id': artifact_id,
         'branch': branch,
         'product': 'geckoview',
-        'template_key': 'maven',
+        'template_key': 'maven_geckoview',
         'version': expected_version,
         'buildid': build_id,
     }

diff --git a/beetmoverscript/utils.py b/beetmoverscript/utils.py
@@ -177,7 +177,7 @@ def _generate_beetmover_template_args_maven(task, release_props):
         'branch': release_props['branch'],
         'buildid': release_props['buildid'],
         'product': release_props['appName'],
-        'template_key': 'maven',
+        'template_key': 'maven_{}'.format(release_props['appName']),
         'version': task['payload']['version'],
     }