From 9deb6b634b59df01d6f319a3dc32ac79c2e02d95 Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Tue, 18 Apr 2023 10:18:21 -0400 Subject: [PATCH] feat: create transforms for adding dependencies These transforms will implement part of the work that the multi_dep loader is doing in other projects. Issue: #227 --- docs/reference/index.rst | 1 + .../reference/source/taskgraph.transforms.rst | 8 + docs/reference/transforms/from_deps.rst | 191 ++++++++++++++++ docs/reference/transforms/index.rst | 11 + src/taskgraph/transforms/from_deps.py | 205 ++++++++++++++++++ src/taskgraph/util/attributes.py | 28 ++- test/fixtures/gen.py | 4 +- test/test_optimize.py | 56 ++--- test/test_transforms_from_deps.py | 182 ++++++++++++++++ 9 files changed, 651 insertions(+), 35 deletions(-) create mode 100644 docs/reference/transforms/from_deps.rst create mode 100644 docs/reference/transforms/index.rst create mode 100644 src/taskgraph/transforms/from_deps.py create mode 100644 test/test_transforms_from_deps.py diff --git a/docs/reference/index.rst b/docs/reference/index.rst index 4e7179bf5..ab6e87e5d 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -6,6 +6,7 @@ Reference cli parameters + transforms/index optimization-strategies source/modules migrations diff --git a/docs/reference/source/taskgraph.transforms.rst b/docs/reference/source/taskgraph.transforms.rst index 0374fc57d..e55af7f0e 100644 --- a/docs/reference/source/taskgraph.transforms.rst +++ b/docs/reference/source/taskgraph.transforms.rst @@ -20,6 +20,14 @@ taskgraph.transforms.base module :undoc-members: :show-inheritance: +taskgraph.transforms.from\_deps module +-------------------------------------- + +.. automodule:: taskgraph.transforms.from_deps + :members: + :undoc-members: + :show-inheritance: + taskgraph.transforms.cached\_tasks module ----------------------------------------- diff --git a/docs/reference/transforms/from_deps.rst b/docs/reference/transforms/from_deps.rst new file mode 100644 index 000000000..02b811c4b --- /dev/null +++ b/docs/reference/transforms/from_deps.rst @@ -0,0 +1,191 @@ +.. _from_deps: + +From Dependencies +================= + +The :mod:`taskgraph.transforms.from_deps` transforms can be used to create +tasks based on the kind dependencies, filtering on common attributes like the +``build-type``. + +These transforms are useful when you want to create follow-up tasks for some +indeterminate subset of existing tasks. For example, maybe you want to run +a signing task after each build task. + + +Usage +----- + +Add the transform to the ``transforms`` key in your ``kind.yml`` file: + +.. code-block:: yaml + + transforms: + - taskgraph.transforms.from_deps + # ... + +Then create a ``from-deps`` section in your task definition, e.g: + +.. code-block:: yaml + + kind-dependencies: + - build + - toolchain + + tasks: + signing: + from-deps: + kinds: [build] + +This example will split the ``signing`` task into many, one for each build. If +``kinds`` is unspecified, then it defaults to all kinds listed in the +``kind-dependencies`` key. So the following is valid: + +.. code-block:: yaml + + kind-dependencies: + - build + - toolchain + + tasks: + signing: + from-deps: {} + +In this example, a task will be created for each build *and* each toolchain. + +Limiting Dependencies by Attribute +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It may not be desirable to create a new task for *all* tasks in the given kind. +It's possible to limit the tasks given some attribute: + +.. code-block:: yaml + + kind-dependencies: + - build + + tasks: + signing: + from-deps: + with-attributes: + platform: linux + +In the above example, follow-up tasks will only be created for builds whose +``platform`` attribute equals "linux". Multiple attributes can be specified, +these are resolved using the :func:`~taskgraph.util.attributes.attrmatch` +utility function. + +Grouping Dependencies +~~~~~~~~~~~~~~~~~~~~~ + +Sometimes, it may be desirable to run a task after a group of related tasks +rather than an individual task. One example of this, is a task to notify when a +release is finished for each platform we're shipping. We want it to depend on +every task from that particular release phase. + +To accomplish this, we specify the ``group-by`` key: + +.. code-block:: yaml + + kind-dependencies: + - build + - signing + - publish + + tasks: + notify: + from-deps: + group-by: + attribute: platform + +In this example, tasks across the ``build``, ``signing`` and ``publish`` kinds will +be scanned for an attribute called "platform" and sorted into corresponding groups. +Assuming we're shipping on Windows, Mac and Linux, it might create the following +groups: + +.. code-block:: + + - build-windows, signing-windows, publish-windows + - build-mac, signing-mac, publish-mac + - build-linux, signing-linux, publish-linux + +Then the ``notify`` task will be duplicated into three, one for each group. The +notify tasks will depend on each task in its associated group. + +Custom Grouping +~~~~~~~~~~~~~~~ + +Only the default ``single`` and the ``attribute`` group-by functions are +built-in. But if more complex grouping is needed, custom functions can be +implemented as well: + +.. code-block:: python + + from typing import List + + from taskgraph.task import Task + from taskgraph.transforms.from_deps import group_by + from taskgraph.transforms.base import TransformConfig + + @group_by("custom-name") + def group_by(config: TransformConfig, tasks: List[Task]) -> List[List[Task]]: + pass + +This can then be used in a task like so: + +.. code-block:: yaml + + from-deps: + group-by: custom-name + +It's also possible to specify a schema for your custom group-by function, which +allows tasks to pass down additional context (such as with the built-in +``attribute`` function): + +.. code-block:: python + + from typing import List + + from taskgraph.task import Task + from taskgraph.transforms.from_deps import group_by + from taskgraph.transforms.base import TransformConfig + from taskgraph.util.schema import Schema + + @group_by("custom-name", schema=Schema(str)) + def group_by(config: TransformConfig, tasks: List[Task], ctx: str) -> List[List[Task]]: + pass + +The extra context can be passed by turning ``group-by`` into an object +instead of a string: + +.. code-block:: yaml + + from-deps: + group-by: + custom-name: foobar + +In the above example, the value ``foobar`` is what must conform to the schema defined +by the ``group_by`` function. + +Primary Kind +~~~~~~~~~~~~ + +Each task has a ``primary-kind``. This is the kind dependency in each grouping +that comes first in the list of supported kinds (either via the +``kind-dependencies`` in the ``kind.yml`` file, or via the ``from-deps.kinds`` +key). Note that depending how the dependencies get grouped, a given group may +not contain a dependency for each kind. Therefore the list of kind dependencies +are ordered by preference. E.g, kinds earlier in the list will be chosen as the +primary kind before kinds later in the list. + +The primary kind is used to derive the task's label, as well as copy attributes +if the ``copy-attributes`` key is set to ``True`` (see next section). + +Each task created by the ``from_deps`` transforms, will have a +``primary-kind-dependency`` attribute set. + +Copying Attributes +~~~~~~~~~~~~~~~~~~ + +It's often useful to copy attributes from a dependency. When this key is set to ``True``, +all attributes from the ``primary-kind`` (see above) will be copied over to the task. If +the task contain pre-existing attributes, they will not be overwritten. diff --git a/docs/reference/transforms/index.rst b/docs/reference/transforms/index.rst new file mode 100644 index 000000000..840bbc1f5 --- /dev/null +++ b/docs/reference/transforms/index.rst @@ -0,0 +1,11 @@ +Transforms +========== + +Taskgraph includes several transforms out of the box. These can be used to +accomplish common tasks such as adding dependencies programmatically or setting +up notifications. This section contains a reference of the transforms Taskgraph +provides, read below to learn how to use them. + +.. toctree:: + + from_deps diff --git a/src/taskgraph/transforms/from_deps.py b/src/taskgraph/transforms/from_deps.py new file mode 100644 index 000000000..124e7ffbb --- /dev/null +++ b/src/taskgraph/transforms/from_deps.py @@ -0,0 +1,205 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Transforms used to create tasks based on the kind dependencies, filtering on +common attributes like the ``build-type``. + +These transforms are useful when follow-up tasks are needed for some +indeterminate subset of existing tasks. For example, running a signing task +after each build task, whatever builds may exist. +""" +from copy import deepcopy +from textwrap import dedent + +from voluptuous import ALLOW_EXTRA, Any, Optional, Required + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.attributes import attrmatch +from taskgraph.util.schema import Schema + +# Define a collection of group_by functions +GROUP_BY_MAP = {} + + +def group_by(name, schema=None): + def wrapper(func): + GROUP_BY_MAP[name] = func + func.schema = schema + return func + + return wrapper + + +@group_by("single") +def group_by_single(config, tasks): + for task in tasks: + yield [task] + + +@group_by("attribute", schema=Schema(str)) +def group_by_attribute(config, tasks, attr): + groups = {} + for task in tasks: + val = task.attributes.get(attr) + if not val: + continue + groups.setdefault(val, []).append(task) + + return groups.values() + + +FROM_DEPS_SCHEMA = Schema( + { + Required("from-deps"): { + Optional( + "kinds", + description=dedent( + """ + Limit dependencies to specified kinds (defaults to all kinds in + `kind-dependencies`). + + The first kind in the list is the "primary" kind. The + dependency of this kind will be used to derive the label + and copy attributes (if `copy-attributes` is True). + """.lstrip() + ), + ): list, + Optional( + "with-attributes", + description=dedent( + """ + Limit dependencies to tasks whose attributes match + using :func:`~taskgraph.util.attributes.attrmatch`. + """.lstrip() + ), + ): {str: Any(list, str)}, + Optional( + "group-by", + description=dedent( + """ + Group cross-kind dependencies using the given group-by + function. One task will be created for each group. If not + specified, the 'single' function will be used which creates + a new task for each individual dependency. + """.lstrip() + ), + ): Any( + None, + *GROUP_BY_MAP, + {Any(*GROUP_BY_MAP): object}, + ), + Optional( + "copy-attributes", + description=dedent( + """ + If True, copy attributes from the dependency matching the + first kind in the `kinds` list (whether specified explicitly + or taken from `kind-dependencies`). + """.lstrip() + ), + ): bool, + } + }, + extra=ALLOW_EXTRA, +) +"""Schema for from_deps transforms.""" + +transforms = TransformSequence() +transforms.add_validate(FROM_DEPS_SCHEMA) + + +@transforms.add +def from_deps(config, tasks): + for task in tasks: + # Setup and error handling. + from_deps = task.pop("from-deps") + kind_deps = config.config.get("kind-dependencies", []) + kinds = from_deps.get("kinds", kind_deps) + + invalid = set(kinds) - set(kind_deps) + if invalid: + invalid = "\n".join(sorted(invalid)) + raise Exception( + dedent( + f""" + The `from-deps.kinds` key contains the following kinds + that are not defined in `kind-dependencies`: + {invalid} + """.lstrip() + ) + ) + + if not kinds: + raise Exception( + dedent( + """ + The `from_deps` transforms require at least one kind defined + in `kind-dependencies`! + """.lstrip() + ) + ) + + # Resolve desired dependencies. + with_attributes = from_deps.get("with-attributes") + deps = [ + task + for task in config.kind_dependencies_tasks.values() + if task.kind in kinds + if not with_attributes or attrmatch(task.attributes, **with_attributes) + ] + + # Resolve groups. + group_by = from_deps.get("group-by", "single") + groups = set() + + if isinstance(group_by, dict): + assert len(group_by) == 1 + group_by, arg = group_by.popitem() + func = GROUP_BY_MAP[group_by] + if func.schema: + func.schema(arg) + groups = func(config, deps, arg) + else: + func = GROUP_BY_MAP[group_by] + groups = func(config, deps) + + # Split the task, one per group. + copy_attributes = from_deps.get("copy-attributes", False) + for group in groups: + # Verify there is only one task per kind in each group. + group_kinds = {t.kind for t in group} + if len(group_kinds) < len(group): + raise Exception( + "The from_deps transforms only allow a single task per kind in a group!" + ) + + new_task = deepcopy(task) + new_task["dependencies"] = {dep.kind: dep.label for dep in group} + + # Set name and copy attributes from the primary kind. + for kind in kinds: + if kind in group_kinds: + primary_kind = kind + break + else: + raise Exception("Could not detect primary kind!") + + new_task.setdefault("attributes", {})[ + "primary-kind-dependency" + ] = primary_kind + + primary_dep = [dep for dep in group if dep.kind == primary_kind][0] + + if primary_dep.label.startswith(primary_kind): + new_task["name"] = primary_dep.label[len(primary_kind) + 1 :] + else: + new_task["name"] = primary_dep.label + + if copy_attributes: + attrs = new_task.get("attributes", {}) + new_task["attributes"] = primary_dep.attributes.copy() + new_task["attributes"].update(attrs) + + yield new_task diff --git a/src/taskgraph/util/attributes.py b/src/taskgraph/util/attributes.py index cf6f11c57..74d699662 100644 --- a/src/taskgraph/util/attributes.py +++ b/src/taskgraph/util/attributes.py @@ -7,18 +7,30 @@ def attrmatch(attributes, **kwargs): - """Determine whether the given set of task attributes matches. The - conditions are given as keyword arguments, where each keyword names an - attribute. The keyword value can be a literal, a set, or a callable. A - literal must match the attribute exactly. Given a set, the attribute value - must be in the set. A callable is called with the attribute value. If an - attribute is specified as a keyword argument but not present in the - attributes, the result is False.""" + """Determine whether the given set of task attributes matches. + + The conditions are given as keyword arguments, where each keyword names an + attribute. The keyword value can be a literal, a set, or a callable: + + * A literal must match the attribute exactly. + * Given a set or list, the attribute value must be contained within it. + * A callable is called with the attribute value and returns a boolean. + + If an attribute is specified as a keyword argument but not present in the + task's attributes, the result is False. + + Args: + attributes (dict): The task's attributes object. + kwargs (dict): The conditions the task's attributes must satisfy in + order to match. + Returns: + bool: Whether the task's attributes match the conditions or not. + """ for kwkey, kwval in kwargs.items(): if kwkey not in attributes: return False attval = attributes[kwkey] - if isinstance(kwval, set): + if isinstance(kwval, (set, list)): if attval not in kwval: return False elif callable(kwval): diff --git a/test/fixtures/gen.py b/test/fixtures/gen.py index 044a1e801..3edcdef86 100644 --- a/test/fixtures/gen.py +++ b/test/fixtures/gen.py @@ -217,8 +217,8 @@ def inner(func, tasks, config=None): def make_task( label, + kind="test", optimization=None, - optimized=None, task_def=None, task_id=None, dependencies=None, @@ -232,7 +232,7 @@ def make_task( task = Task( attributes=attributes or {}, if_dependencies=if_dependencies or [], - kind="test", + kind=kind, label=label, task=task_def, ) diff --git a/test/test_optimize.py b/test/test_optimize.py index 399a84779..1ad54d910 100644 --- a/test/test_optimize.py +++ b/test/test_optimize.py @@ -51,9 +51,9 @@ def make_triangle(deps=True, **opts): `---- t2 --' """ return make_graph( - make_task("t1", opts.get("t1")), - make_task("t2", opts.get("t2")), - make_task("t3", opts.get("t3")), + make_task("t1", optimization=opts.get("t1")), + make_task("t2", optimization=opts.get("t2")), + make_task("t3", optimization=opts.get("t3")), ("t3", "t2", "dep"), ("t3", "t1", "dep2"), ("t2", "t1", "dep"), @@ -111,8 +111,8 @@ def make_triangle(deps=True, **opts): # Tasks with the 'not' composite strategy are removed when the substrategy says not to pytest.param( make_graph( - make_task("t1", {"not-never": None}), - make_task("t2", {"not-remove": None}), + make_task("t1", optimization={"not-never": None}), + make_task("t2", optimization={"not-remove": None}), ), { "strategies": lambda: { @@ -150,10 +150,12 @@ def make_triangle(deps=True, **opts): # Tasks with 'if_dependencies' are removed when deps are not run pytest.param( make_graph( - make_task("t1", {"remove": None}), - make_task("t2", {"remove": None}), - make_task("t3", {"never": None}, if_dependencies=["t1", "t2"]), - make_task("t4", {"never": None}, if_dependencies=["t1"]), + make_task("t1", optimization={"remove": None}), + make_task("t2", optimization={"remove": None}), + make_task( + "t3", optimization={"never": None}, if_dependencies=["t1", "t2"] + ), + make_task("t4", optimization={"never": None}, if_dependencies=["t1"]), ("t3", "t2", "dep"), ("t3", "t1", "dep2"), ("t2", "t1", "dep"), @@ -167,10 +169,12 @@ def make_triangle(deps=True, **opts): # Parents of tasks with 'if_dependencies' are also removed even if requested pytest.param( make_graph( - make_task("t1", {"remove": None}), - make_task("t2", {"remove": None}), - make_task("t3", {"never": None}, if_dependencies=["t1", "t2"]), - make_task("t4", {"never": None}, if_dependencies=["t1"]), + make_task("t1", optimization={"remove": None}), + make_task("t2", optimization={"remove": None}), + make_task( + "t3", optimization={"never": None}, if_dependencies=["t1", "t2"] + ), + make_task("t4", optimization={"never": None}, if_dependencies=["t1"]), ("t3", "t2", "dep"), ("t3", "t1", "dep2"), ("t2", "t1", "dep"), @@ -184,10 +188,12 @@ def make_triangle(deps=True, **opts): # Tasks with 'if_dependencies' are kept if at least one of said dependencies are kept pytest.param( make_graph( - make_task("t1", {"never": None}), - make_task("t2", {"remove": None}), - make_task("t3", {"never": None}, if_dependencies=["t1", "t2"]), - make_task("t4", {"never": None}, if_dependencies=["t1"]), + make_task("t1", optimization={"never": None}), + make_task("t2", optimization={"remove": None}), + make_task( + "t3", optimization={"never": None}, if_dependencies=["t1", "t2"] + ), + make_task("t4", optimization={"never": None}, if_dependencies=["t1"]), ("t3", "t2", "dep"), ("t3", "t1", "dep2"), ("t2", "t1", "dep"), @@ -201,9 +207,9 @@ def make_triangle(deps=True, **opts): # Ancestor of task with 'if_dependencies' does not cause it to be kept pytest.param( make_graph( - make_task("t1", {"never": None}), - make_task("t2", {"remove": None}), - make_task("t3", {"never": None}, if_dependencies=["t2"]), + make_task("t1", optimization={"never": None}), + make_task("t2", optimization={"remove": None}), + make_task("t3", optimization={"never": None}, if_dependencies=["t2"]), ("t3", "t2", "dep"), ("t2", "t1", "dep2"), ), @@ -216,10 +222,10 @@ def make_triangle(deps=True, **opts): # don't have any dependents and are not in 'requested_tasks' pytest.param( make_graph( - make_task("t1", {"never": None}), - make_task("t2", {"never": None}, if_dependencies=["t1"]), - make_task("t3", {"remove": None}), - make_task("t4", {"never": None}, if_dependencies=["t3"]), + make_task("t1", optimization={"never": None}), + make_task("t2", optimization={"never": None}, if_dependencies=["t1"]), + make_task("t3", optimization={"remove": None}), + make_task("t4", optimization={"never": None}, if_dependencies=["t3"]), ("t2", "t1", "e1"), ("t4", "t2", "e2"), ("t4", "t3", "e3"), @@ -333,7 +339,7 @@ def test_remove_tasks(monkeypatch, graph, kwargs, exp_removed): # A task which expires before a dependents deadline is not a valid replacement. pytest.param( make_graph( - make_task("t1", {"replace": "e1"}), + make_task("t1", optimization={"replace": "e1"}), make_task( "t2", task_def={"deadline": {"relative-datestamp": "2 days"}} ), diff --git a/test/test_transforms_from_deps.py b/test/test_transforms_from_deps.py new file mode 100644 index 000000000..b8a8b3685 --- /dev/null +++ b/test/test_transforms_from_deps.py @@ -0,0 +1,182 @@ +""" +Tests for the 'from_deps' transforms. +""" + +from pprint import pprint + +import pytest + +from taskgraph.transforms import from_deps + +from .conftest import make_task + + +def handle_exception(obj, exc=None): + if exc: + assert isinstance(obj, exc) + elif isinstance(obj, Exception): + raise obj + + +def assert_no_kind_dependencies(e): + handle_exception(e, exc=Exception) + + +def assert_invalid_only_kinds(e): + handle_exception(e, exc=Exception) + + +def assert_defaults(tasks): + handle_exception(tasks) + assert len(tasks) == 2 + assert tasks[0]["attributes"] == {"primary-kind-dependency": "foo"} + assert tasks[0]["dependencies"] == {"foo": "a"} + assert tasks[0]["name"] == "a" + assert tasks[1]["attributes"] == {"primary-kind-dependency": "bar"} + assert tasks[1]["dependencies"] == {"bar": "bar-b"} + assert tasks[1]["name"] == "b" + + +assert_group_by_single = assert_defaults + + +def assert_group_by_attribute(tasks): + handle_exception(tasks) + assert len(tasks) == 2 + assert tasks[0]["dependencies"] == {"foo": "a"} + assert tasks[0]["attributes"] == {"primary-kind-dependency": "foo"} + assert tasks[1]["dependencies"] == {"foo": "b", "bar": "c"} + assert tasks[1]["attributes"] == {"primary-kind-dependency": "foo"} + + +def assert_group_by_attribute_dupe(e): + handle_exception(e, exc=Exception) + + +def assert_copy_attributes(tasks): + handle_exception(tasks) + assert len(tasks) == 1 + + task = tasks[0] + assert task["attributes"] == { + "build-type": "win", + "kind": "foo", + "primary-kind-dependency": "foo", + } + + +@pytest.mark.parametrize( + "task, kind_config, deps", + ( + pytest.param( + # task + {"from-deps": {}}, + # kind config + {}, + # deps + None, + id="no_kind_dependencies", + ), + pytest.param( + # task + {"from-deps": {"kinds": ["foo", "baz"]}}, + # kind config + None, + # deps + None, + id="invalid_only_kinds", + ), + pytest.param( + # task + {"from-deps": {}}, + # kind config + None, + # deps + None, + id="defaults", + ), + pytest.param( + # task + {"from-deps": {}}, + # kind config + None, + # deps + None, + id="group_by_single", + ), + pytest.param( + # task + {"from-deps": {"group-by": {"attribute": "build-type"}}}, + # kind config + None, + # deps + { + "a": make_task("a", attributes={"build-type": "linux"}, kind="foo"), + "b": make_task("b", attributes={"build-type": "win"}, kind="foo"), + "c": make_task("c", attributes={"build-type": "win"}, kind="bar"), + }, + id="group_by_attribute", + ), + pytest.param( + # task + {"from-deps": {"group-by": {"attribute": "build-type"}}}, + # kind config + None, + # deps + { + "a": make_task("a", attributes={"build-type": "linux"}, kind="foo"), + "b": make_task("b", attributes={"build-type": "win"}, kind="foo"), + "c": make_task("c", attributes={"build-type": "win"}, kind="foo"), + }, + id="group_by_attribute_dupe", + ), + pytest.param( + # task + { + "from-deps": { + "group-by": {"attribute": "build-type"}, + "copy-attributes": True, + } + }, + # kind config + None, + # deps + { + "a": make_task( + "a", attributes={"build-type": "win", "kind": "bar"}, kind="bar" + ), + "b": make_task( + "b", attributes={"build-type": "win", "kind": "foo"}, kind="foo" + ), + }, + id="copy_attributes", + ), + ), +) +def test_transforms( + request, make_transform_config, run_transform, task, kind_config, deps +): + task.setdefault("name", "task") + task.setdefault("description", "description") + + if kind_config is None: + kind_config = {"kind-dependencies": ["foo", "bar"]} + + if deps is None: + deps = { + "a": make_task("a", kind="foo"), + "b": make_task("bar-b", kind="bar"), + } + config = make_transform_config(kind_config, deps) + + try: + result = run_transform(from_deps.transforms, task, config) + except Exception as e: + result = e + + print("Dumping result:") + pprint(result, indent=2) + + param_id = request.node.callspec.id + assert_func = globals()[f"assert_{param_id}"] + assert_func(result)