Create a release promotion action

taskcluster/app_services_taskgraph/__init__.py

@@ -17,6 +17,7 @@
 def register(graph_config):
     # Import modules to register decorated functions
     _import_modules([
+        "actions.release_promotion",
         "branch_builds",
         "job",
         "target_tasks",

taskcluster/app_services_taskgraph/actions/release_promotion.py

@@ -0,0 +1,127 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from taskgraph.actions.registry import register_callback_action
+from taskgraph.decision import taskgraph_decision
+from taskgraph.parameters import Parameters
+from taskgraph.taskgraph import TaskGraph
+from taskgraph.util.taskcluster import get_artifact
+from taskgraph.util.taskgraph import (
+    find_decision_task,
+    find_existing_tasks_from_previous_kinds,
+)
+
+RELEASE_PROMOTION_PROJECTS = (
+    "https://github.com/mozilla/application-services",
+    "https://github.com/mozilla-releng/staging-application-services",
+)
+
+
+@register_callback_action(
+    name="release-promotion",
+    title="Release Promotion",
+    symbol="${input.release_promotion_flavor}",
+    description="Promote or ship the an application-services release.",
+    generic=False,
+    order=500,
+    context=[],
+    available=lambda p: p["head_repository"] in RELEASE_PROMOTION_PROJECTS,
+    schema=lambda graph_config: {
+        "type": "object",
+        "properties": {
+            "release_promotion_flavor": {
+                "type": "string",
+                "description": "The flavor of release promotion to perform.",
+                "default": "promote",
+                "enum": sorted(graph_config["release-promotion"]["flavors"]),
+            },
+            "do_not_optimize": {
+                "type": "array",
+                "description": (
+                    "Optional: a list of labels to avoid optimizing out "
+                    "of the graph (to force a rerun)."
+                ),
+                "items": {
+                    "type": "string",
+                },
+            },
+            "rebuild_kinds": {
+                "type": "array",
+                "description": (
+                    "Optional: an array of kinds to ignore from the previous "
+                    "graph(s)."
+                ),
+                "items": {
+                    "type": "string",
+                },
+            },
+            "previous_graph_ids": {
+                "type": "array",
+                "description": (
+                    "Optional: an array of taskIds of decision or action "
+                    "tasks from the previous graph(s) to use to populate "
+                    "our `previous_graph_kinds`."
+                ),
+                "items": {
+                    "type": "string",
+                },
+            },
+        },
+        "required": [
+            "release_promotion_flavor",
+        ],
+    },
+)
+def release_promotion_action(parameters, graph_config, input, task_group_id, task_id):
+    release_promotion_flavor = input["release_promotion_flavor"]
+    promotion_config = graph_config["release-promotion"]["flavors"][
+        release_promotion_flavor
+    ]
+
+    target_tasks_method = promotion_config["target-tasks-method"].format(
+        project=parameters["project"]
+    )
+    rebuild_kinds = input.get("rebuild_kinds") or promotion_config.get(
+        "rebuild-kinds", []
+    )
+    do_not_optimize = input.get("do_not_optimize") or promotion_config.get(
+        "do-not-optimize", []
+    )
+
+    # make parameters read-write
+    parameters = dict(parameters)
+    # Build previous_graph_ids from ``previous_graph_ids`` or ``revision``.
+    previous_graph_ids = input.get("previous_graph_ids")
+    if not previous_graph_ids:
+        previous_graph_ids = [find_decision_task(parameters, graph_config)]
+
+    # Download parameters from the first decision task
+    parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml")
+
+    # Download and combine full task graphs from each of the
+    # previous_graph_ids. Sometimes previous relpro action tasks will add
+    # tasks that didn't exist in the first full_task_graph, so combining them
+    # is important. The rightmost graph should take precedence in the case of
+    # conflicts.
+    combined_full_task_graph = {}
+    for graph_id in previous_graph_ids:
+        full_task_graph = get_artifact(graph_id, "public/full-task-graph.json")
+        combined_full_task_graph.update(full_task_graph)
+    _, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph)
+    parameters["existing_tasks"] = find_existing_tasks_from_previous_kinds(
+        combined_full_task_graph, previous_graph_ids, rebuild_kinds
+    )
+    parameters["do_not_optimize"] = do_not_optimize
+    parameters["target_tasks_method"] = target_tasks_method
+
+    # When doing staging releases, we still want to re-use tasks from previous
+    # graphs.
+    parameters["optimize_target_tasks"] = True
+    parameters["shipping_phase"] = input["release_promotion_flavor"]
+    parameters["tasks_for"] = "action"
+
+    # make parameters read-only
+    parameters = Parameters(**parameters)
+
+    taskgraph_decision({"root": graph_config.root_dir}, parameters=parameters)

taskcluster/app_services_taskgraph/target_tasks.py

@@ -22,6 +22,7 @@ def target_tasks_pr_skip(full_task_graph, parameters, graph_config):
 def target_tasks_release(full_task_graph, parameters, graph_config):
     return full_task_graph.tasks
 
+
 @_target_task('pr-full')
 def target_tasks_all(full_task_graph, parameters, graph_config):
     """Target the tasks which have indicated they should be run on this project
@@ -33,6 +34,7 @@ def filter(task):
 
     return [l for l, task in full_task_graph.tasks.items() if filter(task)]
 
+
 @_target_task('pr-normal')
 def target_tasks_default(full_task_graph, parameters, graph_config):
     """Target the tasks which have indicated they should be run on this project
@@ -43,3 +45,41 @@ def filter(task):
                 and task.attributes.get('release-type') != 'release-only')
 
     return [l for l, task in full_task_graph.tasks.items() if filter(task)]
+
+
+def filter_release_promotion(
+    full_task_graph, filtered_for_candidates, shipping_phase
+):
+    def filter(task):
+        # Include promotion tasks; these will be optimized out
+        if task.label in filtered_for_candidates:
+            return True
+
+        if task.attributes.get("shipping_phase") == shipping_phase:
+            return True
+
+    return [
+        label for label, task in full_task_graph.tasks.items()
+        if filter(task)
+    ]
+
+
+@_target_task("promote")
+def target_tasks_promote(full_task_graph, parameters, graph_config):
+    return filter_release_promotion(
+        full_task_graph,
+        filtered_for_candidates=[],
+        shipping_phase="promote",
+    )
+
+
+@_target_task("ship")
+def target_tasks_ship(full_task_graph, parameters, graph_config):
+    filtered_for_candidates = target_tasks_promote(
+        full_task_graph,
+        parameters,
+        graph_config,
+    )
+    return filter_release_promotion(
+        full_task_graph, filtered_for_candidates, shipping_phase="ship"
+    )

taskcluster/app_services_taskgraph/transforms/deps_complete.py

@@ -5,6 +5,7 @@
 from taskgraph.transforms.base import TransformSequence
 
 transforms = TransformSequence()
+alerts = TransformSequence()
 
 @transforms.add
 def deps_complete_script(config, tasks):
@@ -51,14 +52,19 @@ def convert_dependencies(config, tasks):
         ]
         yield task
 
+@alerts.add
 @transforms.add
 def add_alert_routes(config, tasks):
     """
     Add routes to alert channels when this task fails.
     """
     for task in tasks:
-        task.setdefault('routes', [])
         alerts = task.pop("alerts", {})
+        if config.params["level"] != "3":
+            yield task
+            continue
+
+        task.setdefault('routes', [])
         for name, value in alerts.items():
             if name not in ("slack-channel", "email", "pulse", "matrix-room"):
                 raise KeyError("Unknown alert type: {}".format(name))

taskcluster/app_services_taskgraph/transforms/multi_dep.py

@@ -17,7 +17,9 @@ def build_name_and_attributes(config, tasks):
         }
         primary_dep = task["primary-dependency"]
         copy_of_attributes = primary_dep.attributes.copy()
-        task.setdefault("attributes", copy_of_attributes)
+        task.setdefault("attributes", {})
+        for k, v in copy_of_attributes.items():
+            task["attributes"].setdefault(k, v)
         # run_on_tasks_for is set as an attribute later in the pipeline
         task.setdefault("run-on-tasks-for", copy_of_attributes['run_on_tasks_for'])
         task["name"] = _get_dependent_job_name_without_its_kind(primary_dep)

taskcluster/app_services_taskgraph/transforms/release_publish.py

@@ -34,10 +34,11 @@ def setup_command(config, tasks):
                 "type": "file",
             }
         ]
-        task["routes"] = [
-            f"index.project.application-services.v2.{release_type}.latest",
-            f"index.project.application-services.v2.{release_type}.{version}",
-        ]
+        if config.params['level'] == '3':
+            task["routes"] = [
+                f"index.project.application-services.v2.{release_type}.latest",
+                f"index.project.application-services.v2.{release_type}.{version}",
+            ]
         yield task
 
 @transforms.add

taskcluster/ci/beetmover/kind.yml

@@ -6,6 +6,7 @@ loader: app_services_taskgraph.loader.multi_dep:loader
 
 transforms:
   - app_services_taskgraph.transforms.multi_dep:transforms
+  - app_services_taskgraph.transforms.deps_complete:alerts
   - app_services_taskgraph.transforms.beetmover:transforms
   - taskgraph.transforms.task:transforms
 
@@ -20,10 +21,12 @@ primary-dependency: module-build
 group-by: component
 
 task-template:
+  attributes:
+    shipping_phase: ship
   run-on-tasks-for: [github-release, cron]
   description: 'Publish release module {}'
   worker-type: beetmover
   worker:
     app-name: appservices
-  routes:
-    - notify.email.a-s-ci-failures@mozilla.com.on-failed
+  alerts:
+    email: a-s-ci-failures@mozilla.com

taskcluster/ci/config.yml

@@ -43,5 +43,12 @@ workers:
       os: scriptworker
       worker-type: 'app-services-{level}-beetmover'
 
+release-promotion:
+  flavors:
+    promote:
+      target-tasks-method: promote
+    ship:
+      target-tasks-method: ship
+
 scriptworker:
   scope-prefix: project:mozilla:app-services:releng

taskcluster/ci/signing/kind.yml

@@ -6,6 +6,7 @@ loader: app_services_taskgraph.loader.multi_dep:loader
 
 transforms:
   - app_services_taskgraph.transforms.multi_dep:transforms
+  - app_services_taskgraph.transforms.deps_complete:alerts
   - app_services_taskgraph.transforms.signing:transforms
   - taskgraph.transforms.task:transforms
 
@@ -16,6 +17,8 @@ primary-dependency: module-build
 group-by: component
 
 task-template:
+  attributes:
+    shipping_phase: promote
   run-on-tasks-for: [github-release, cron]
   description: 'Sign release module {}'
   worker-type: signing
@@ -26,6 +29,5 @@ task-template:
           release-signing
         default:
           dep-signing
-  routes:
-    - notify.email.a-s-ci-failures@mozilla.com.on-failed
-
+  alerts:
+    email: a-s-ci-failures@mozilla.com

taskcluster/test/params/relpro-promote.yml

@@ -0,0 +1,60 @@
+base_ref: origin/main
+base_repository: https://github.com/mozilla-releng/application-services
+base_rev: c7b530d61ea8dd4a6dd2d876c4f05a7066dfee5f
+branch-build: {}
+build_date: 1683647278
+build_number: 1
+do_not_optimize: []
+enable_always_target: true
+existing_tasks:
+  Build summary task: YEuUOoVbRNeGKtOvBJlInQ
+  build-docker-image-linux: CD--BKGXRAOfpGJviIxY6w
+  fetch-go-1.14.4: LUCoOzsCTTeWJgSSDszEvg
+  fetch-resource-monitor: OYpix6V9TIS_CUBv8zEf-Q
+  lint-detekt: CipDmbrkQH2ccXSGtrStZA
+  lint-ktlint: ckD6DYUOQleJp7ng7OmfZA
+  module-build-autofill: X5xdPX1KSE232byWlLK2_Q
+  module-build-crashtest: VYYDYqGBTzS8d16X3ZB-yQ
+  module-build-errorsupport: Nf8WFWLhRGK9fKVebJkn4w
+  module-build-full-megazord: VvBgm9H8TPKAuO11b-alwQ
+  module-build-fxaclient: NfhMA9mSQQ-wl1h9t-G5cQ
+  module-build-httpconfig: O5rztTUjT32bBWjMyKpcRA
+  module-build-logins: OQfCHW4_QTq4ZaYKRz84rA
+  module-build-native-support: J-h6f8zAQOCnzf0NnCtWbA
+  module-build-nimbus: KL2gtWC3RMKMbWa2zhMLuw
+  module-build-places: CjxF6i5XTUuue3N6hur76A
+  module-build-push: Eb1ryYa0TQas2d6L8shrKg
+  module-build-rust-log-forwarder: PavZP7KCQUa6eIiH7rcguw
+  module-build-rustlog: DzsCxDB2RfmjXjSb9sXUcg
+  module-build-sync15: c-mAP_Z5Tt-jep2NqUVEPQ
+  module-build-syncmanager: aO_QpDq_RoOcNf56oxDtug
+  module-build-tabs: d_M2TbtARdeqZdNuLDGMxA
+  module-build-tooling-nimbus-gradle: OfsUfpFkTRiSJIsPbA0cTA
+  toolchain-android: Tf8GiXQBSpOIpg0KqSmR8Q
+  toolchain-desktop-linux: FXqdOHj-Q72AiSvjQiiXYw
+  toolchain-desktop-macos: VfWl6da-QYaF_Xch88tjdA
+  toolchain-linux64-resource-monitor: M5U6TKbGSSSfzJoSGNEh5g
+  toolchain-macosx64-resource-monitor: MawQ-sNYSVieeRo9djQk8Q
+  toolchain-robolectric: Z9JODYylR0Wjw2GzzaclTg
+  toolchain-rust: XwjhfYmLSMytDOulmyxM0g
+filters:
+- target_tasks_method
+- branch-build
+head_ref: refs/heads/main
+head_repository: https://github.com/mozilla-releng/application-services
+head_rev: c7b530d61ea8dd4a6dd2d876c4f05a7066dfee5f
+head_tag: ''
+level: '1'
+moz_build_date: '20230509154758'
+next_version: null
+optimize_strategies: null
+optimize_target_tasks: true
+owner: user@example.com
+project: application-services
+pushdate: 0
+pushlog_id: '0'
+repository_type: git
+shipping_phase: promote
+target_tasks_method: promote
+tasks_for: action
+version: null