Attempt to support new manifest format and fetch-bmo.py (#8)

* Attempt to support new manifest format and fetch-bmo.py

* Attempt to run dep-signing and fetches on PR

* Pass correct values

* Pass correct values

* Pass correct values

* move gpg-signature into `fetch`

* address review comments

* remove unused run-task dir

Co-authored-by: Aki Sasaki <aki@mozilla.com>

signing-manifests/bug1639199.0.yml

@@ -1,10 +1,12 @@
 ---
-url: https://bugzilla.mozilla.org/attachment.cgi?id=9153960#/esr78_switch_test.mar
-artifact-name: esr78_switch_test.mar
 bug: 1639199
-private-artifact: false
-signing-formats: ["autograph_hash_only_mar384"]
 sha256: b43c2d8fec3bc98f25a28cabf05147c627c13b7657683a500309f11b393446fb
 filesize: 2207
+private-artifact: false
+signing-formats: ["autograph_hash_only_mar384"]
 requestor: Justin Wood <jwood@mozilla.com>
 reason: test mar signing for esr switch
+artifact-name: esr78_switch_test.mar
+fetch:
+  type: bmo-attachment
+  attachment-id: 9153960

signing-manifests/bug1642701.yml

@@ -1,10 +1,11 @@
 ---
-url: https://github.com/mozilla-releng/staging-adhoc-signing/raw/99e70937e8ea2aac48168e473ae5ba39b47978d1/artifacts/SignableFile.bin
-artifact-name: SignableFile.bin
 bug: 1642701
-private-artifact: false
-signing-formats: ["autograph_authenticode_stub"]
 sha256: 5de3f160913fe1764ea4d572762ad3119969648cd6e80ac40862321b6943a063
 filesize: 4096
+private-artifact: false
+signing-formats: ["autograph_authenticode_stub"]
 requestor: tjr
 reason: new cert
+artifact-name: SignableFile.bin
+fetch:
+  url: https://github.com/mozilla-releng/staging-adhoc-signing/raw/99e70937e8ea2aac48168e473ae5ba39b47978d1/artifacts/SignableFile.bin

signing-manifests/example.yml.tmpl

@@ -1,10 +1,12 @@
 ---
-url: https://evil.com/foo/bar.exe
-artifact-name: bar.exe
 bug: 12345
-private-artifact: false
-signing-formats: ["autograph_gpg"]
 sha256: abcd12345
 filesize: 12345
+private-artifact: false
+signing-formats: ["autograph_gpg"]
 requestor: Dr. Pepper
 reason: sign my evil file!!!
+artifact-name: bar.exe
+fetch:
+    type: static-url
+    url: https://evil.com/foo/bar.exe

signing-manifests/mar.yml

@@ -1,10 +1,11 @@
 ---
-url: https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/Sejw462STvKGjIAQ5dsm2g/runs/0/artifacts/public/build/target.complete.mar
-artifact-name: target.complete.mar
 bug: 12345
-private-artifact: false
-signing-formats: ["autograph_hash_only_mar384"]
 sha256: df3463a5f3f84c9d4a572d0ebbb6dce6b4bb21ebc8ba86400268e9546441ec03
 filesize: 64260074
+private-artifact: false
+signing-formats: ["autograph_hash_only_mar384"]
 requestor: Aki Sasaki <asasaki@mozilla.com>
 reason: test mar signing
+artifact-name: target.complete.mar
+fetch:
+  url: https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/Sejw462STvKGjIAQ5dsm2g/runs/0/artifacts/public/build/target.complete.mar

signing-manifests/stub.yml

@@ -1,10 +1,11 @@
 ---
-url: https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/V6HrQPOuT5mjeaGu1bV9iA/runs/0/artifacts/public/build/setup-stub.exe
-artifact-name: setup-stub.exe
 bug: 12345
-private-artifact: false
-signing-formats: ["autograph_authenticode_stub"]
 sha256: aa78e7e049789fb49ab0128f1195fce7f90258ef46615ba90f94af5784f101a3
 filesize: 451035
+private-artifact: false
+signing-formats: ["autograph_authenticode_stub"]
 requestor: Aki Sasaki <asasaki@mozilla.com>
 reason: test stub signing
+artifact-name: setup-stub.exe
+fetch:
+  url: https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/V6HrQPOuT5mjeaGu1bV9iA/runs/0/artifacts/public/build/setup-stub.exe

taskcluster/adhoc_taskgraph/__init__.py

@@ -19,6 +19,7 @@ def register(graph_config):
         "signing_manifest",
         "target",
         "worker_types",
+        "fetches",
     ])
 
 

taskcluster/adhoc_taskgraph/fetches.py

@@ -0,0 +1,62 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from six import text_type
+
+from voluptuous import Required
+
+from taskgraph.util.schema import taskref_or_string
+from taskgraph.util import path as mozpath
+from taskgraph.transforms.fetch import fetch_builder
+
+
+@fetch_builder('bmo-attachment', schema={
+    # The URL to download.
+    Required('attachment-id'): text_type,
+
+    # The SHA-256 of the downloaded content.
+    Required('sha256'): text_type,
+
+    # Size of the downloaded entity, in bytes.
+    Required('size'): int,
+
+    # The name to give to the generated artifact.
+    Required('artifact-name'): text_type,
+
+})
+def create_fetch_url_task(config, name, fetch):
+
+    artifact_name = fetch['artifact-name']
+
+    workdir = '/builds/worker'
+
+    # Arguments that matter to the cache digest
+    args = (
+        'bmo-attachment '
+        '--sha256 {} '
+        '--size {} '
+        '--name {} '
+        '{} '
+        '/builds/worker/artifacts/{}'.format(
+            fetch['sha256'],
+            fetch['size'],
+            artifact_name,
+            fetch['attachment-id'],
+            artifact_name
+            )
+    )
+
+    cmd = [
+        'bash',
+        '-c',
+        '/usr/local/bin/fetch-bmo.py {}'.format(args)
+    ]
+
+    return {
+        'command': cmd,
+        'artifact_name': artifact_name,
+        'digest_data': args,
+    }

taskcluster/adhoc_taskgraph/signing_manifest.py

@@ -37,16 +37,25 @@
 
 base_schema = Schema(
     {
-        Required("url"): text_type,
         Required("bug"): int,
-        Required("private-artifact"): bool,
-        Required("signing-formats"): [Any(*SUPPORTED_SIGNING_FORMATS)],
         Required("sha256"): text_type,
         Required("filesize"): int,
+        Required("private-artifact"): bool,
+        Required("signing-formats"): [Any(*SUPPORTED_SIGNING_FORMATS)],
         Required("requestor"): basestring,
         Required("reason"): basestring,
-        Optional("gpg-signature"): basestring,
         Required("artifact-name"): basestring,
+        Required("fetch"): Any(
+            {
+                Optional("gpg-signature"): basestring,
+                Optional('type'): 'static-url',
+                Required('url'): basestring,
+            },
+            {
+                Required('type'): 'bmo-attachment',
+                Required('attachment-id'): Any(basestring, int)
+            }
+        ),
         Required("manifest_name"): basestring,
     }
 )

taskcluster/adhoc_taskgraph/transforms/fetch.py

@@ -19,11 +19,17 @@ def from_manifests(config, jobs):
         manifest = job.pop('manifest')
         job['name'] = manifest['manifest_name']
         fetch = job.setdefault("fetch", {})
-        fetch['type'] = 'static-url'
-        fetch["url"] = manifest["url"]
+        fetch['type'] = manifest["fetch"].get('type', 'static-url')
+        if fetch['type'] == 'static-url':
+            fetch["url"] = manifest["fetch"]["url"]
+            if manifest['fetch'].get('gpg-signature'):
+                fetch['gpg-signature'] = manifest['fetch'].get('gpg-signature')
+        elif fetch['type'] == 'bmo-attachment':
+            fetch['attachment-id'] = unicode(manifest["fetch"]['attachment-id'])
         fetch["sha256"] = manifest["sha256"]
         fetch["size"] = manifest["filesize"]
-        for k in ("gpg-signature", "artifact-name"):
+
+        for k in ("artifact-name", ):
             if manifest.get(k):
                 fetch[k] = manifest[k]
         job.setdefault('attributes', {})['manifest'] = manifest

taskcluster/ci/dep-signing/kind.yml

@@ -14,10 +14,12 @@ transforms:
 
 job-template:
     description: Ad-hoc signing
+    attributes:
+        code-review: true
     index:
         type: dep-signing
     worker-type: dep-signing
     worker:
         signing-type: dep-signing
         max-run-time: 3600
-    run-on-tasks-for: ['action']
+    run-on-tasks-for: ['action', 'github-pull-request']

taskcluster/ci/fetch/kind.yml

@@ -12,3 +12,5 @@ transforms:
 
 job-template:
     description: 'Unsigned artifact'
+    attributes:
+        code-review: true

taskcluster/ci/pr/kind.yml

@@ -6,6 +6,7 @@ loader: taskgraph.loader.transform:loader
 
 kind-dependencies:
     - fetch
+    - dep-signing
 
 transforms:
     - taskgraph.transforms.code_review:transforms

taskcluster/docker/fetch/Dockerfile

@@ -24,6 +24,9 @@ RUN apt-get update && \
 
 # %include-run-task
 
+COPY fetch-bmo.py /usr/local/bin/fetch-bmo.py
+RUN chmod a+x /usr/local/bin/fetch-bmo.py
+
 ENV SHELL=/bin/bash \
     HOME=/builds/worker \
     PATH=/builds/worker/.local/bin:$PATH