From 6ca7d2e652d7facd725d10760a308e213b4689bc Mon Sep 17 00:00:00 2001 From: Leon Kang Date: Mon, 18 Mar 2024 16:34:29 -0500 Subject: [PATCH 1/3] Initial commit for allowing Pulp tasks to implicitly skip steps - uses depends_on argument in StepDecorator to figure out if a certain step needs to be skipped - skipped_value can be provided in the case where the skipped method should provide some value by default even if method is skipped --- pubtools/_pulp/step.py | 13 ++++++++++--- pubtools/_pulp/task.py | 10 +++++++--- pubtools/_pulp/tasks/common.py | 2 +- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/pubtools/_pulp/step.py b/pubtools/_pulp/step.py index b606a019..7f8d9d29 100644 --- a/pubtools/_pulp/step.py +++ b/pubtools/_pulp/step.py @@ -9,8 +9,10 @@ class StepDecorator(object): """Implementation of PulpTask.step decorator. See that method for more info.""" - def __init__(self, name): + def __init__(self, name, depends_on, skipped_value): self._name = name + self._depends_on = depends_on or [] + self._skipped_value = skipped_value @property def human_name(self): @@ -28,7 +30,11 @@ def new_fn(instance, *args, **kwargs): self.human_name, extra={"event": {"type": "%s-skip" % self.machine_name}}, ) - return args[0] if args else None + return ( + self.skipped_value + if self._skipped_value is not UNSET + else args[0] if args else None + ) logger = StepLogger(self) args = logger.log_start(args) @@ -53,7 +59,8 @@ def new_fn(instance, *args, **kwargs): def should_skip(self, instance): skip = (getattr(instance.args, "skip", None) or "").split(",") - return self.machine_name in skip + deps_and_self = self._depends_on + [self.machine_name] + return any([name in skip for name in deps_and_self]) # helpers used in implementation of decorator diff --git a/pubtools/_pulp/task.py b/pubtools/_pulp/task.py index 8e7dc92f..92a6655e 100644 --- a/pubtools/_pulp/task.py +++ b/pubtools/_pulp/task.py @@ -4,7 +4,7 @@ from pubtools.pluggy import task_context -from .step import StepDecorator +from .step import StepDecorator, UNSET LOG = logging.getLogger("pubtools.pulp") @@ -68,13 +68,17 @@ def args(self): return self._args @classmethod - def step(cls, name): + def step(cls, name, depends_on=None, skipped_value=UNSET): """A decorator to mark an instance method as a discrete workflow step. Marking a method as a step has effects: - Log messages will be produced when entering and leaving the method - The method can be skipped if requested by the caller (via --skip argument) + - Methods that depend on other methods are implicitly skipped if depends_on + is supplied and that dependant is being skipped + - If the method is skipped, it returns either `skipped_value` (if that has + been set), or the method's first argument. Steps may be written as plain blocking functions, as non-blocking functions which accept or return Futures, or as generators. @@ -94,7 +98,7 @@ def step(cls, name): - The step is considered *failed* if it raised an exception. - The step is considered *finished* once all items have been yielded. """ - return StepDecorator(name) + return StepDecorator(name, depends_on, skipped_value) def _basic_args(self): # minimum args required for a pulp CLI task diff --git a/pubtools/_pulp/tasks/common.py b/pubtools/_pulp/tasks/common.py index 9aed2d81..cc7d235e 100644 --- a/pubtools/_pulp/tasks/common.py +++ b/pubtools/_pulp/tasks/common.py @@ -36,7 +36,7 @@ class CDNCache(FastPurgeClientService, CdnClientService): """Provide features to interact with CDN cache.""" - @step("Flush CDN cache") + @step("Flush CDN cache", depends_on=["publish"], skipped_value=[]) def flush_cdn(self, repos): """Clears the CDN cache for the repositories provided From 73fb072235cd62c3eab5345ed0af6dbc837b8362 Mon Sep 17 00:00:00 2001 From: Leon Kang Date: Wed, 20 Mar 2024 07:22:00 -0500 Subject: [PATCH 2/3] Fix missing UNSET initialization and update baselines --- pubtools/_pulp/step.py | 4 ++-- .../test_clear_repo/test_clear_file_skip_publish.jsonl | 3 +-- .../test_clear_repo/test_clear_file_skip_publish.txt | 4 +--- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/pubtools/_pulp/step.py b/pubtools/_pulp/step.py index 7f8d9d29..2715e1e7 100644 --- a/pubtools/_pulp/step.py +++ b/pubtools/_pulp/step.py @@ -4,7 +4,7 @@ from more_executors.futures import f_sequence, f_return LOG = logging.getLogger("pubtools.pulp") - +UNSET = object() class StepDecorator(object): """Implementation of PulpTask.step decorator. See that method for more info.""" @@ -31,7 +31,7 @@ def new_fn(instance, *args, **kwargs): extra={"event": {"type": "%s-skip" % self.machine_name}}, ) return ( - self.skipped_value + self._skipped_value if self._skipped_value is not UNSET else args[0] if args else None ) diff --git a/tests/logs/clear_repo/test_clear_repo/test_clear_file_skip_publish.jsonl b/tests/logs/clear_repo/test_clear_repo/test_clear_file_skip_publish.jsonl index cf4ff11a..18cf76d5 100644 --- a/tests/logs/clear_repo/test_clear_repo/test_clear_file_skip_publish.jsonl +++ b/tests/logs/clear_repo/test_clear_repo/test_clear_file_skip_publish.jsonl @@ -7,5 +7,4 @@ {"event": {"type": "publish-skip"}} {"event": {"type": "flush-ud-cache-start"}} {"event": {"type": "flush-ud-cache-end"}} -{"event": {"type": "flush-cdn-cache-start"}} -{"event": {"type": "flush-cdn-cache-end"}} +{"event": {"type": "flush-cdn-cache-skip"}} diff --git a/tests/logs/clear_repo/test_clear_repo/test_clear_file_skip_publish.txt b/tests/logs/clear_repo/test_clear_repo/test_clear_file_skip_publish.txt index dfa1ed10..0bd6898b 100644 --- a/tests/logs/clear_repo/test_clear_repo/test_clear_file_skip_publish.txt +++ b/tests/logs/clear_repo/test_clear_repo/test_clear_file_skip_publish.txt @@ -9,6 +9,4 @@ [ INFO] Flush UD cache: started [ INFO] UD cache flush is not enabled. [ INFO] Flush UD cache: finished -[ INFO] Flush CDN cache: started -[ INFO] CDN cache flush is not enabled. -[ INFO] Flush CDN cache: finished +[ INFO] Flush CDN cache: skipped From de743a120d156b4c0910337abe898c928eaae653 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 20 Mar 2024 12:24:05 +0000 Subject: [PATCH 3/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pubtools/_pulp/step.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pubtools/_pulp/step.py b/pubtools/_pulp/step.py index 2715e1e7..7d0f9df1 100644 --- a/pubtools/_pulp/step.py +++ b/pubtools/_pulp/step.py @@ -6,6 +6,7 @@ LOG = logging.getLogger("pubtools.pulp") UNSET = object() + class StepDecorator(object): """Implementation of PulpTask.step decorator. See that method for more info."""