Bug 1470942 - Support maven on S3

beetmoverscript/constants.py

@@ -92,6 +92,12 @@
     'partner': [
         '',  # all legal
     ],
+    'maven': [
+        'maven2/',
+    ],
+    'maven-staging': [
+        'maven2/',
+    ],
 }
 
 # actions that imply actual releases, hence the need of `build_number` and
@@ -108,6 +114,10 @@
     'push-to-partner',
 )
 
+MAVEN_ACTIONS = (
+    'push-to-maven',
+)
+
 # XXX this is a fairly clunky way of specifying which files to copy from
 # candidates to releases -- let's find a nicer way of doing this.
 # XXX if we keep this, let's make it configurable? overridable in config?
@@ -164,3 +174,9 @@
     'target.dmg',
     'target.apk',
 )
+
+# Zip archive can theoretically have a better compression ratio, like when there's a big amount
+# of redundancy (e.g.: files full of zeros). Let beetmover only deal with regular cases. Edge cases
+# are considered too suspicious, so we bail out on them.
+DEFAULT_ZIP_MAX_FILE_SIZE_IN_MB = 100
+ZIP_MAX_COMPRESSION_RATIO = 10

beetmoverscript/data/beetmover_task_schema.json

@@ -56,7 +56,7 @@
                             "type" : "string"
                         }
                     },
-                    "required" : ["appName", "buildid", "appVersion", "hashType", "platform", "branch"]
+                    "required" : ["appName", "buildid", "appVersion", "branch"]
                 },
                 "upstreamArtifacts": {
                     "type": "array",
@@ -79,9 +79,12 @@
                                 "items": {
                                     "type": "string"
                                 }
+                            },
+                            "zipExtract": {
+                                "type": "boolean"
                             }
                         },
-                        "required": ["taskId", "taskType", "paths", "locale"]
+                        "required": ["taskId", "taskType", "paths"]
                     },
                     "minItems": 1,
                     "uniqueItems": true

beetmoverscript/data/release_beetmover_task_schema.json

@@ -46,9 +46,12 @@
                                 "items": {
                                     "type": "string"
                                 }
+                            },
+                            "zipExtract": {
+                                "type": "boolean"
                             }
                         },
-                        "required": ["taskId", "taskType", "paths", "locale"]
+                        "required": ["taskId", "taskType", "paths"]
                     },
                     "minItems": 0,
                     "uniqueItems": true

beetmoverscript/maven_utils.py

@@ -0,0 +1,54 @@
+import os
+
+_MAVEN_ZIP_NAME = 'target.maven.zip'
+
+
+def get_maven_expected_files_per_archive_per_task_id(upstream_artifacts_per_task_id, mapping_manifest):
+    task_id, maven_zip_full_path = _get_task_id_and_full_path_of_maven_archive(upstream_artifacts_per_task_id)
+
+    return {
+        task_id: {
+            maven_zip_full_path: _get_maven_expected_files_in_archive(mapping_manifest)
+        }
+    }
+
+
+def _get_task_id_and_full_path_of_maven_archive(upstream_artifacts_per_task_id):
+    candidate_task_id = ''
+    candidate_path = ''
+
+    for task_id, upstream_definitions in upstream_artifacts_per_task_id.items():
+        for upstream_definition in upstream_definitions:
+            for path in upstream_definition['paths']:
+                if path.endswith(_MAVEN_ZIP_NAME):
+                    if candidate_task_id:
+                        raise ValueError(
+                            'Too many upstream artifact ending with "{}" found: ({}, {}) and ({}, {})'.format(
+                                _MAVEN_ZIP_NAME, candidate_task_id, candidate_path, task_id, path
+                            )
+                        )
+
+                    candidate_task_id = task_id
+                    candidate_path = path
+
+    if not candidate_task_id:
+        raise ValueError('No upstream artifact ending with "{}" found. Given: {}'.format(
+            _MAVEN_ZIP_NAME, upstream_artifacts_per_task_id)
+        )
+
+    return candidate_task_id, candidate_path
+
+
+def _get_maven_expected_files_in_archive(mapping_manifest):
+    files = mapping_manifest['mapping']['en-US'].keys()
+    return [
+        os.path.join(
+            _remove_first_directory_from_bucket(mapping_manifest['s3_bucket_path']),
+            file
+        ) for file in files
+    ]
+
+
+def _remove_first_directory_from_bucket(s3_bucket_path):
+    # remove 'maven2' because it's not in the archive, but it exists on the maven server
+    return '/'.join(s3_bucket_path.split('/')[1:])

beetmoverscript/script.py

@@ -17,11 +17,13 @@
 from scriptworker.exceptions import ScriptWorkerTaskException, ScriptWorkerRetryException
 from scriptworker.utils import retry_async, raise_future_exceptions
 
+from beetmoverscript import task, zip, maven_utils
+
 from beetmoverscript.constants import (
     MIME_MAP, RELEASE_BRANCHES, CACHE_CONTROL_MAXAGE, RELEASE_EXCLUDE,
     NORMALIZED_BALROG_PLATFORMS, PARTNER_REPACK_PUBLIC_PREFIX_TMPL,
     PARTNER_REPACK_PRIVATE_REGEXES, PARTNER_REPACK_PUBLIC_REGEXES, BUILDHUB_ARTIFACT,
-    INSTALLER_ARTIFACTS
+    INSTALLER_ARTIFACTS, DEFAULT_ZIP_MAX_FILE_SIZE_IN_MB
 )
 from beetmoverscript.task import (
     validate_task_schema, add_balrog_manifest_to_artifacts,
@@ -154,6 +156,49 @@ async def push_to_releases(context):
     copy_beets(context, candidates_keys_checksums, releases_keys_checksums)
 
 
+async def push_to_maven(context):
+    """Push artifacts to locations expected by maven clients (like mvn or gradle)"""
+    artifacts_to_beetmove = task.get_upstream_artifacts_with_zip_extract_param(context)
+    context.release_props = get_release_props(context)
+    context.checksums = dict()  # Needed by downstream calls
+    context.raw_balrog_manifest = dict()    # Needed by downstream calls
+
+    mapping_manifest = generate_beetmover_manifest(context)
+    validate_bucket_paths(context.bucket, mapping_manifest['s3_bucket_path'])
+
+    context.artifacts_to_beetmove = _extract_and_check_maven_artifacts_to_beetmove(
+        artifacts_to_beetmove,
+        mapping_manifest,
+        context.config.get('zip_max_file_size_in_mb', DEFAULT_ZIP_MAX_FILE_SIZE_IN_MB)
+    )
+
+    await move_beets(context, context.artifacts_to_beetmove, mapping_manifest)
+
+
+def _extract_and_check_maven_artifacts_to_beetmove(artifacts, mapping_manifest, zip_max_file_size_in_mb):
+    expected_files = maven_utils.get_maven_expected_files_per_archive_per_task_id(
+        artifacts, mapping_manifest
+    )
+
+    extracted_paths_per_archive = zip.check_and_extract_zip_archives(
+        artifacts, expected_files, zip_max_file_size_in_mb
+    )
+
+    number_of_extracted_archives = len(extracted_paths_per_archive)
+    if number_of_extracted_archives == 0:
+        raise ScriptWorkerTaskException('No archive extracted')
+    elif number_of_extracted_archives > 1:
+        raise NotImplementedError('More than 1 archive extracted. Only 1 is supported at once')
+    extracted_paths_per_relative_path = list(extracted_paths_per_archive.values())[0]
+
+    return {
+        'en-US': {
+            os.path.basename(path_in_archive): full_path
+            for path_in_archive, full_path in extracted_paths_per_relative_path.items()
+        }
+    }
+
+
 # copy_beets {{{1
 def copy_beets(context, from_keys_checksums, to_keys_checksums):
     creds = get_creds(context)
@@ -217,6 +262,7 @@ def list_bucket_objects(context, s3_resource, prefix):
     # push to candidates is at this point identical to push_to_nightly
     'push-to-candidates': push_to_nightly,
     'push-to-releases': push_to_releases,
+    'push-to-maven': push_to_maven,
 }
 
 

beetmoverscript/task.py

@@ -14,6 +14,7 @@
     RESTRICTED_BUCKET_PATHS,
     CHECKSUMS_CUSTOM_FILE_NAMING
 )
+from scriptworker import artifacts as scriptworker_artifacts
 from scriptworker.exceptions import ScriptWorkerTaskException
 
 log = logging.getLogger(__name__)
@@ -109,29 +110,45 @@ def add_balrog_manifest_to_artifacts(context):
     utils.write_json(abs_file_path, context.balrog_manifest)
 
 
-def get_upstream_artifact(context, taskid, path):
-    abs_path = os.path.abspath(os.path.join(context.config['work_dir'], 'cot', taskid, path))
-    if not os.path.exists(abs_path):
-        raise ScriptWorkerTaskException(
-            "upstream artifact with path: {}, does not exist".format(abs_path)
-        )
-    return abs_path
-
-
 def get_upstream_artifacts(context, preserve_full_paths=False):
     artifacts = {}
     for artifact_dict in context.task['payload']['upstreamArtifacts']:
         locale = artifact_dict['locale']
         artifacts[locale] = artifacts.get(locale, {})
         for path in artifact_dict['paths']:
-            abs_path = get_upstream_artifact(context, artifact_dict['taskId'], path)
+            abs_path = scriptworker_artifacts.get_and_check_single_upstream_artifact_full_path(
+                context, artifact_dict['taskId'], path
+            )
             if preserve_full_paths:
                 artifacts[locale][path] = abs_path
             else:
                 artifacts[locale][os.path.basename(abs_path)] = abs_path
     return artifacts
 
 
+def get_upstream_artifacts_with_zip_extract_param(context):
+    # XXX A dict comprehension isn't used because upstream_definition would be erased if the same
+    # taskId is present twice in upstreamArtifacts
+    upstream_artifacts_per_task_id = {}
+
+    for artifact_definition in context.task['payload']['upstreamArtifacts']:
+        task_id = artifact_definition['taskId']
+        upstream_definitions = upstream_artifacts_per_task_id.get(task_id, [])
+
+        new_upstream_definition = {
+            'paths': [
+                scriptworker_artifacts.get_and_check_single_upstream_artifact_full_path(context, task_id, path)
+                for path in artifact_definition['paths']
+            ],
+            'zip_extract': artifact_definition.get('zipExtract', False),
+        }
+
+        upstream_definitions.append(new_upstream_definition)
+        upstream_artifacts_per_task_id[task_id] = upstream_definitions
+
+    return upstream_artifacts_per_task_id
+
+
 def get_release_props(context, platform_mapping=STAGE_PLATFORM_MAP):
     """determined via parsing the Nightly build job's payload and
     expanded the properties with props beetmover knows about."""
@@ -151,7 +168,7 @@ def update_props(context, props, platform_mapping):
     `stage_platform` as we need both in the beetmover template manifests."""
     props = deepcopy(props)
 
-    stage_platform = props["platform"]
+    stage_platform = props.get('platform', '')
     # for some products/platforms this mapping is not needed, hence the default
     props["platform"] = platform_mapping.get(stage_platform, stage_platform)
     props["stage_platform"] = stage_platform

beetmoverscript/templates/maven_geckoview.yml

@@ -0,0 +1,62 @@
+---
+metadata:
+    name: "Maven repository"
+    description: "Maps artifacts to spec'd maven location"
+    owner: "release@mozilla.com"
+
+s3_bucket_path: maven2/org/mozilla/{{ artifact_id }}/{{ version }}/  # Maven groupId is org.mozilla
+
+mapping:
+{% for locale in ['en-US'] %}
+  "{{ locale }}":   # Locale is not needed for geckoview, it's used by move_beets, though
+  {% for product in ['geckoview'] %}
+    "{{ artifact_id }}-{{ version }}.aar":
+      s3_key: {{ artifact_id }}-{{ version }}.aar
+      destinations:
+        - {{ artifact_id }}-{{ version }}.aar
+    "{{ artifact_id }}-{{ version }}.aar.md5":
+      s3_key: {{ artifact_id }}-{{ version }}.aar.md5
+      destinations:
+        - {{ artifact_id }}-{{ version }}.aar.md5
+    "{{ artifact_id }}-{{ version }}.aar.sha1":
+      s3_key: {{ artifact_id }}-{{ version }}.aar.sha1
+      destinations:
+        - {{ artifact_id }}-{{ version }}.aar.sha1
+    "{{ artifact_id }}-{{ version }}.pom":
+      s3_key: {{ artifact_id }}-{{ version }}.pom
+      destinations:
+        - {{ artifact_id }}-{{ version }}.pom
+    "{{ artifact_id }}-{{ version }}.pom.md5":
+      s3_key: {{ artifact_id }}-{{ version }}.pom.md5
+      destinations:
+        - {{ artifact_id }}-{{ version }}.pom.md5
+    "{{ artifact_id }}-{{ version }}.pom.sha1":
+      s3_key: {{ artifact_id }}-{{ version }}.pom.sha1
+      destinations:
+        - {{ artifact_id }}-{{ version }}.pom.sha1
+    "{{ artifact_id }}-{{ version }}-javadoc.jar":
+      s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar
+      destinations:
+        - {{ artifact_id }}-{{ version }}-javadoc.jar
+    "{{ artifact_id }}-{{ version }}-javadoc.jar.md5":
+      s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar.md5
+      destinations:
+        - {{ artifact_id }}-{{ version }}-javadoc.jar.md5
+    "{{ artifact_id }}-{{ version }}-javadoc.jar.sha1":
+      s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar.sha1
+      destinations:
+        - {{ artifact_id }}-{{ version }}-javadoc.jar.sha1
+    "{{ artifact_id }}-{{ version }}-sources.jar":
+      s3_key: {{ artifact_id }}-{{ version }}-sources.jar
+      destinations:
+        - {{ artifact_id }}-{{ version }}-sources.jar
+    "{{ artifact_id }}-{{ version }}-sources.jar.md5":
+      s3_key: {{ artifact_id }}-{{ version }}-sources.jar.md5
+      destinations:
+        - {{ artifact_id }}-{{ version }}-sources.jar.md5
+    "{{ artifact_id }}-{{ version }}-sources.jar.sha1":
+      s3_key: {{ artifact_id }}-{{ version }}-sources.jar.sha1
+      destinations:
+        - {{ artifact_id }}-{{ version }}-sources.jar.sha1
+  {% endfor %}
+{% endfor %}