Attempt to support new manifest format and fetch-bmo.py

signing-manifests/bug1639199.0.yml

@@ -1,10 +1,12 @@
 ---
-url: https://bugzilla.mozilla.org/attachment.cgi?id=9153960#/esr78_switch_test.mar
-artifact-name: esr78_switch_test.mar
 bug: 1639199
-private-artifact: false
-signing-formats: ["autograph_hash_only_mar384"]
 sha256: b43c2d8fec3bc98f25a28cabf05147c627c13b7657683a500309f11b393446fb
 filesize: 2207
+private-artifact: false
+signing-formats: ["autograph_hash_only_mar384"]
 requestor: Justin Wood <jwood@mozilla.com>
 reason: test mar signing for esr switch
+artifact-name: esr78_switch_test.mar
+fetch:
+  type: bmo-attachment
+  attachment-id: 9153960

signing-manifests/bug1642701.yml

@@ -1,10 +1,11 @@
 ---
-url: https://github.com/mozilla-releng/staging-adhoc-signing/raw/99e70937e8ea2aac48168e473ae5ba39b47978d1/artifacts/SignableFile.bin
-artifact-name: SignableFile.bin
 bug: 1642701
-private-artifact: false
-signing-formats: ["autograph_authenticode_stub"]
 sha256: 5de3f160913fe1764ea4d572762ad3119969648cd6e80ac40862321b6943a063
 filesize: 4096
+private-artifact: false
+signing-formats: ["autograph_authenticode_stub"]
 requestor: tjr
 reason: new cert
+artifact-name: SignableFile.bin
+fetch:
+  url: https://github.com/mozilla-releng/staging-adhoc-signing/raw/99e70937e8ea2aac48168e473ae5ba39b47978d1/artifacts/SignableFile.bin

signing-manifests/example.yml.tmpl

@@ -1,10 +1,12 @@
 ---
-url: https://evil.com/foo/bar.exe
-artifact-name: bar.exe
 bug: 12345
-private-artifact: false
-signing-formats: ["autograph_gpg"]
 sha256: abcd12345
 filesize: 12345
+private-artifact: false
+signing-formats: ["autograph_gpg"]
 requestor: Dr. Pepper
 reason: sign my evil file!!!
+artifact-name: bar.exe
+fetch:
+    type: static-url
+    url: https://evil.com/foo/bar.exe

signing-manifests/mar.yml

@@ -1,10 +1,11 @@
 ---
-url: https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/Sejw462STvKGjIAQ5dsm2g/runs/0/artifacts/public/build/target.complete.mar
-artifact-name: target.complete.mar
 bug: 12345
-private-artifact: false
-signing-formats: ["autograph_hash_only_mar384"]
 sha256: df3463a5f3f84c9d4a572d0ebbb6dce6b4bb21ebc8ba86400268e9546441ec03
 filesize: 64260074
+private-artifact: false
+signing-formats: ["autograph_hash_only_mar384"]
 requestor: Aki Sasaki <asasaki@mozilla.com>
 reason: test mar signing
+artifact-name: target.complete.mar
+fetch:
+  url: https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/Sejw462STvKGjIAQ5dsm2g/runs/0/artifacts/public/build/target.complete.mar

signing-manifests/stub.yml

@@ -1,10 +1,11 @@
 ---
-url: https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/V6HrQPOuT5mjeaGu1bV9iA/runs/0/artifacts/public/build/setup-stub.exe
-artifact-name: setup-stub.exe
 bug: 12345
-private-artifact: false
-signing-formats: ["autograph_authenticode_stub"]
 sha256: aa78e7e049789fb49ab0128f1195fce7f90258ef46615ba90f94af5784f101a3
 filesize: 451035
+private-artifact: false
+signing-formats: ["autograph_authenticode_stub"]
 requestor: Aki Sasaki <asasaki@mozilla.com>
 reason: test stub signing
+artifact-name: setup-stub.exe
+fetch:
+  url: https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/V6HrQPOuT5mjeaGu1bV9iA/runs/0/artifacts/public/build/setup-stub.exe

taskcluster/adhoc_taskgraph/__init__.py

@@ -19,6 +19,7 @@ def register(graph_config):
         "signing_manifest",
         "target",
         "worker_types",
+        "fetches",
     ])
 
 

taskcluster/adhoc_taskgraph/fetches.py

@@ -0,0 +1,58 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from six import text_type
+
+from voluptuous import Required
+
+from taskgraph.util.schema import taskref_or_string
+from taskgraph.util import path as mozpath
+from taskgraph.transforms.fetch import fetch_builder
+
+
+@fetch_builder('bmo-attachment', schema={
+    # The URL to download.
+    Required('attachment-id'): text_type,
+
+    # The SHA-256 of the downloaded content.
+    Required('sha256'): text_type,
+
+    # Size of the downloaded entity, in bytes.
+    Required('size'): int,
+
+    # The name to give to the generated artifact.
+    Required('artifact-name'): text_type,
+
+})
+def create_fetch_url_task(config, name, fetch):
+
+    artifact_name = fetch['artifact-name']
+
+    workdir = '/builds/worker'
+
+    # Arguments that matter to the cache digest
+    args = [
+        'bmo-attachment',
+        '--sha256', fetch['sha256'],
+        '--size', '%d' % fetch['size'],
+        '--name', artifact_name,
+        fetch['attachment-id']
+    ]
+
+    cmd = [
+        'bash',
+        '-c',
+        'cd {} && '
+        '/usr/bin/python3 {} {}'.format(
+            workdir, '/builds/worker/bin/fetch-bmo.py'
+        )
+    ] + args
+
+    return {
+        'command': cmd,
+        'artifact_name': artifact_name,
+        'digest_data': ['bmo-attachment'] + args
+    }

taskcluster/adhoc_taskgraph/signing_manifest.py

@@ -37,16 +37,25 @@
 
 base_schema = Schema(
     {
-        Required("url"): text_type,
         Required("bug"): int,
-        Required("private-artifact"): bool,
-        Required("signing-formats"): [Any(*SUPPORTED_SIGNING_FORMATS)],
         Required("sha256"): text_type,
         Required("filesize"): int,
+        Required("private-artifact"): bool,
+        Required("signing-formats"): [Any(*SUPPORTED_SIGNING_FORMATS)],
         Required("requestor"): basestring,
         Required("reason"): basestring,
-        Optional("gpg-signature"): basestring,
         Required("artifact-name"): basestring,
+        Optional("gpg-signature"): basestring,
+        Required("fetch"): Any(
+            {
+                Optional('type'): 'static-url',
+                Required('url'): basestring,
+            },
+            {
+                Required('type'): 'bmo-attachment',
+                Required('attachment-id'): Any(basestring, int)
+            }
+        ),
         Required("manifest_name"): basestring,
     }
 )

taskcluster/adhoc_taskgraph/transforms/fetch.py

@@ -19,11 +19,17 @@ def from_manifests(config, jobs):
         manifest = job.pop('manifest')
         job['name'] = manifest['manifest_name']
         fetch = job.setdefault("fetch", {})
-        fetch['type'] = 'static-url'
-        fetch["url"] = manifest["url"]
+        fetch['type'] = manifest["fetch"].get('type', 'static-url')
+        if fetch['type'] == 'static-url':
+            fetch["url"] = manifest["fetch"]["url"]
+            if manifest.get('gpg-signature'):
+                fetch['gpg-signature'] = manifest.get('gpg-signature')
+        elif fetch['type'] == 'bmo-attachment':
+            fetch['attachment-id'] = unicode(manifest["fetch"]['attachment-id'])
         fetch["sha256"] = manifest["sha256"]
         fetch["size"] = manifest["filesize"]
-        for k in ("gpg-signature", "artifact-name"):
+
+        for k in ("artifact-name", ):
             if manifest.get(k):
                 fetch[k] = manifest[k]
         job.setdefault('attributes', {})['manifest'] = manifest

taskcluster/ci/dep-signing/kind.yml

@@ -20,4 +20,4 @@ job-template:
     worker:
         signing-type: dep-signing
         max-run-time: 3600
-    run-on-tasks-for: ['action']
+    run-on-tasks-for: ['action', 'github-pull-request']

taskcluster/docker/fetch/Dockerfile

@@ -24,6 +24,9 @@ RUN apt-get update && \
 
 # %include-run-task
 
+# %include taskcluster/run-task/fetch-bmo.py
+ADD topsrcdir/taskcluster/run-task/fetch-bmo.py /builds/worker/bin/fetch-bmo.py
+
 ENV SHELL=/bin/bash \
     HOME=/builds/worker \
     PATH=/builds/worker/.local/bin:$PATH