From f8b1c2669908d1176db57185aa9f8e8790c32194 Mon Sep 17 00:00:00 2001 From: Matthias Dellweg Date: Mon, 19 Sep 2022 14:04:09 +0200 Subject: [PATCH] Add filter parameters to task list fixes #543 Co-authored-by: Grant Gainey --- CHANGES/543.feature | 1 + pulpcore/cli/common/context.py | 27 ++++++++- pulpcore/cli/common/generic.py | 18 +++--- pulpcore/cli/common/openapi.py | 18 +++--- pulpcore/cli/core/context.py | 39 ++++++++++++- pulpcore/cli/core/generic.py | 91 ++++++++++++++++++++++------- pulpcore/cli/core/task.py | 53 +++++++++++++++-- tests/scripts/pulpcore/test_task.sh | 35 +++++++++++ tests/scripts/test_debug_api.sh | 2 +- 9 files changed, 235 insertions(+), 49 deletions(-) create mode 100644 CHANGES/543.feature diff --git a/CHANGES/543.feature b/CHANGES/543.feature new file mode 100644 index 000000000..539f1d141 --- /dev/null +++ b/CHANGES/543.feature @@ -0,0 +1 @@ +Added task filtering options. diff --git a/pulpcore/cli/common/context.py b/pulpcore/cli/common/context.py index 0cea81d60..c64aeac1b 100644 --- a/pulpcore/cli/common/context.py +++ b/pulpcore/cli/common/context.py @@ -3,7 +3,7 @@ import re import sys import time -from typing import Any, ClassVar, Dict, List, NamedTuple, Optional, Set, Type, Union +from typing import Any, ClassVar, Dict, Iterable, List, NamedTuple, Optional, Set, Type, Union import click import yaml @@ -28,6 +28,21 @@ DEFAULT_LIMIT = 25 BATCH_SIZE = 25 +DATETIME_FORMATS = [ + "%Y-%m-%dT%H:%M:%S.%fZ", # Pulp format + "%Y-%m-%d", # intl. format + "%Y-%m-%d %H:%M:%S", # ... with time + "%Y-%m-%dT%H:%M:%S", # ... with time and T as a separator + "%y/%m/%d", # US format + "%y/%m/%d %h:%M:%S %p", # ... with time + "%y/%m/%dT%h:%M:%S %p", # ... with time and T as a separator + "%y/%m/%d %H:%M:%S", # ... with time 24h + "%y/%m/%dT%H:%M:%S", # ... with time 24h and T as a separator + "%x", # local format + "%x %X", # ... with time + "%xT%X", # ... with time and T as a separator +] + href_regex = re.compile(r"\/([a-z0-9-_]+\/)+", flags=re.IGNORECASE) @@ -66,9 +81,15 @@ def default(self, obj: Any) -> Any: return super().default(obj) -def _preprocess_value(key: str, value: Any) -> Any: +def _preprocess_value(value: Any) -> Any: + if isinstance(value, str): + return value if isinstance(value, PulpEntityContext): return value.pulp_href + if isinstance(value, datetime.datetime): + return value.strftime(DATETIME_FORMATS[0]) + if isinstance(value, Iterable): + return [_preprocess_value(item) for item in value] return value @@ -77,7 +98,7 @@ def preprocess_payload(payload: EntityDefinition) -> EntityDefinition: return payload return PreprocessedEntityDefinition( - {key: _preprocess_value(key, value) for key, value in payload.items() if value is not None} + {key: _preprocess_value(value) for key, value in payload.items() if value is not None} ) diff --git a/pulpcore/cli/common/generic.py b/pulpcore/cli/common/generic.py index a96264c1d..34b27448c 100644 --- a/pulpcore/cli/common/generic.py +++ b/pulpcore/cli/common/generic.py @@ -8,6 +8,7 @@ from click.decorators import FC, F from pulpcore.cli.common.context import ( + DATETIME_FORMATS, DEFAULT_LIMIT, EntityDefinition, EntityFieldDefinition, @@ -526,6 +527,7 @@ def _type_callback(ctx: click.Context, param: click.Parameter, value: Optional[s type=click.IntRange(1), help=_("Limit the number of {entities} to show."), ) + offset_option = pulp_option( "--offset", default=0, @@ -656,29 +658,29 @@ def _type_callback(ctx: click.Context, param: click.Parameter, value: Optional[s pulp_created_gte_option = pulp_option( "--created-after", "pulp_created__gte", - help=_("Search for {entities} created at or after this ISO 8601 date"), - type=str, + help=_("Search for {entities} created at or after this date"), + type=click.DateTime(formats=DATETIME_FORMATS), ) pulp_created_lte_option = pulp_option( "--created-before", "pulp_created__lte", - help=_("Search for {entities} created at or before this ISO 8601 date"), - type=str, + help=_("Search for {entities} created at or before this date"), + type=click.DateTime(formats=DATETIME_FORMATS), ) pulp_last_updated_gte_option = pulp_option( "--updated-after", "pulp_last_updated__gte", - help=_("Search for {entities} last updated at or after this ISO 8601 date"), - type=str, + help=_("Search for {entities} last updated at or after this date"), + type=click.DateTime(formats=DATETIME_FORMATS), ) pulp_last_updated_lte_option = pulp_option( "--updated-before", "pulp_last_updated__lte", - help=_("Search for {entities} last updated at or before this ISO 8601 date"), - type=str, + help=_("Search for {entities} last updated at or before this date"), + type=click.DateTime(formats=DATETIME_FORMATS), ) retained_versions_option = pulp_option( diff --git a/pulpcore/cli/common/openapi.py b/pulpcore/cli/common/openapi.py index 28a785fa4..89d7ac3b4 100644 --- a/pulpcore/cli/common/openapi.py +++ b/pulpcore/cli/common/openapi.py @@ -123,7 +123,7 @@ def _download_api(self) -> bytes: def extract_params( self, - param_type: str, + param_in: str, path_spec: Dict[str, Any], method_spec: Dict[str, Any], params: Dict[str, Any], @@ -131,13 +131,13 @@ def extract_params( param_specs = { entry["name"]: entry for entry in path_spec.get("parameters", []) - if entry["in"] == param_type + if entry["in"] == param_in } param_specs.update( { entry["name"]: entry for entry in method_spec.get("parameters", []) - if entry["in"] == param_type + if entry["in"] == param_in } ) result: Dict[str, Any] = {} @@ -145,9 +145,6 @@ def extract_params( if name in param_specs: param = params.pop(name) param_spec = param_specs.pop(name) - style = param_spec.get( - "style", "form" if param_type in ("query", "cookie") else "simple" - ) param_schema = param_spec.get("schema") if param_schema: param_type = param_schema.get("type", "string") @@ -157,7 +154,12 @@ def extract_params( assert isinstance(param, Iterable) and not isinstance( param, str ), f"Parameter {name} is expected to be a list." - if not param_spec.get("explode", style == "form"): + style = param_spec.get( + "style", "form" if param_in in ("query", "cookie") else "simple" + ) + explode = param_spec.get("explode", style == "form") + if not explode: + # Not exploding means comma separated list param = ",".join(param) elif param_type == "integer": assert isinstance(param, int) @@ -338,7 +340,7 @@ def call( validate_body=validate_body, ) - self.debug_callback(1, f"{method} {request.url}") + self.debug_callback(1, f"{operation_id} : {method} {request.url}") for key, value in request.headers.items(): self.debug_callback(2, f" {key}: {value}") if request.body is not None: diff --git a/pulpcore/cli/core/context.py b/pulpcore/cli/core/context.py index 4f536b11d..2af139946 100644 --- a/pulpcore/cli/core/context.py +++ b/pulpcore/cli/core/context.py @@ -1,3 +1,4 @@ +import datetime import hashlib import os import sys @@ -327,6 +328,32 @@ class PulpTaskContext(PulpEntityContext): resource_context: Optional[PulpEntityContext] = None + def list(self, limit: int, offset: int, parameters: Dict[str, Any]) -> List[Any]: + if not self.pulp_ctx.has_plugin(PluginRequirement("core", min="3.22.0dev")): + parameters = parameters.copy() + reserved_resources = parameters.pop("reserved_resources", None) or [] + exclusive_resources = parameters.pop("exclusive_resources", None) or [] + shared_resources = parameters.pop("shared_resources", None) or [] + parameters.pop("reserved_resources__in", None) + parameters.pop("exclusive_resources__in", None) + parameters.pop("shared_resources__in", None) + reserved_resources_record = ( + reserved_resources + + exclusive_resources + + ["shared:" + item for item in shared_resources] + ) + if len(reserved_resources_record) > 1: + self.pulp_ctx.needs_plugin( + PluginRequirement( + "core", + min="3.22.0dev", + feature=_("specify multiple reserved resources"), + ), + ) + parameters["reserved_resources_record"] = reserved_resources_record + + return super().list(limit=limit, offset=offset, parameters=parameters) + def cancel(self, task_href: str) -> Any: return self.call( "cancel", @@ -337,11 +364,18 @@ def cancel(self, task_href: str) -> Any: @property def scope(self) -> Dict[str, Any]: if self.resource_context: - return {"reserved_resources_record": [self.resource_context.pulp_href]} + if self.pulp_ctx.has_plugin(PluginRequirement("core", min="3.22.0dev")): + return {"reserved_resources": self.resource_context.pulp_href} + else: + return {"reserved_resources_record": [self.resource_context.pulp_href]} else: return {} - def purge(self, finished_before: Optional[str], states: Optional[List[str]]) -> Any: + def purge( + self, + finished_before: Optional[datetime.datetime], + states: Optional[List[str]], + ) -> Any: body: Dict[str, Any] = {} if finished_before: body["finished_before"] = finished_before @@ -455,3 +489,4 @@ class PulpWorkerContext(PulpEntityContext): ENTITIES = _("workers") HREF = "worker_href" ID_PREFIX = "workers" + HREF_PATTERN = r"workers/" diff --git a/pulpcore/cli/core/generic.py b/pulpcore/cli/core/generic.py index 8c7b77aff..63593ee3d 100644 --- a/pulpcore/cli/core/generic.py +++ b/pulpcore/cli/core/generic.py @@ -4,15 +4,16 @@ import click from pulpcore.cli.common.context import ( + DATETIME_FORMATS, PluginRequirement, PulpContext, PulpEntityContext, pass_entity_context, pass_pulp_context, ) -from pulpcore.cli.common.generic import list_command, pulp_group, pulp_option +from pulpcore.cli.common.generic import list_command, pulp_group, pulp_option, resource_option from pulpcore.cli.common.i18n import get_translation -from pulpcore.cli.core.context import PulpTaskContext +from pulpcore.cli.core.context import PulpTaskContext, PulpWorkerContext translation = get_translation(__name__) _ = translation.gettext @@ -22,27 +23,31 @@ # Generic reusable commands -def _task_group_filter_callback( - ctx: click.Context, param: click.Parameter, value: Optional[str] -) -> Optional[str]: - if value is not None: - pulp_ctx = ctx.find_object(PulpContext) - assert pulp_ctx is not None - - # Example: "/pulp/api/v3/task-groups/a69230d2-506e-44c7-9c46-e64a905f85e7/" - match = re.match( - rf"({pulp_ctx.api_path}task-groups/)?" - r"([0-9a-f]{8})-?([0-9a-f]{4})-?([0-9a-f]{4})-?([0-9a-f]{4})-?([0-9a-f]{12})/?", - value, - ) - if match: - value = "{}task-groups/{}-{}-{}-{}-{}/".format( - pulp_ctx.api_path, *match.group(2, 3, 4, 5, 6) +class HrefOrUuidCallback: + def __init__(self, base_href: str) -> None: + self.base_href = base_href + + def __call__( + self, ctx: click.Context, param: click.Parameter, value: Optional[str] + ) -> Optional[str]: + if value is not None: + pulp_ctx = ctx.find_object(PulpContext) + assert pulp_ctx is not None + + # Example: "/pulp/api/v3/tasks/a69230d2-506e-44c7-9c46-e64a905f85e7/" + href_match = re.match( + rf"({pulp_ctx.api_path}{self.base_href}/)?" + r"([0-9a-f]{8})-?([0-9a-f]{4})-?([0-9a-f]{4})-?([0-9a-f]{4})-?([0-9a-f]{12})/?", + value, ) - else: - raise click.ClickException(_("Either an href or a UUID must be provided.")) + if href_match: + value = "{}{}/{}-{}-{}-{}-{}/".format( + pulp_ctx.api_path, self.base_href, *href_match.group(2, 3, 4, 5, 6) + ) + else: + raise click.ClickException(_("Either an href or a UUID must be provided.")) - return value + return value task_filter = [ @@ -79,7 +84,49 @@ def _task_group_filter_callback( click.option( "--task-group", help=_("List only tasks in this task group. Provide pulp_href or UUID."), - callback=_task_group_filter_callback, + callback=HrefOrUuidCallback("task-groups"), + ), + click.option( + "--parent-task", + help=_("Parent task by uuid or href."), + callback=HrefOrUuidCallback("tasks"), + ), + resource_option( + "--worker", + default_plugin="core", + default_type="none", + context_table={"core:none": PulpWorkerContext}, + href_pattern=PulpWorkerContext.HREF_PATTERN, + help=_("Worker used to execute the task by name or href."), + ), + click.option( + "--created-resource", + "created_resources", + help=_("Href of a resource created in the task."), + ), + click.option( + "--started-after", + "started_at__gte", + help=_("Filter for tasks started at or after this date"), + type=click.DateTime(formats=DATETIME_FORMATS), + ), + click.option( + "--started-before", + "started_at__lte", + help=_("Filter for tasks started at or before this date"), + type=click.DateTime(formats=DATETIME_FORMATS), + ), + click.option( + "--finished-after", + "finished_at__gte", + help=_("Filter for tasks finished at or after this date"), + type=click.DateTime(formats=DATETIME_FORMATS), + ), + click.option( + "--finished-before", + "finished_at__lte", + help=_("Filter for tasks finished at or before this date"), + type=click.DateTime(formats=DATETIME_FORMATS), ), ] diff --git a/pulpcore/cli/core/task.py b/pulpcore/cli/core/task.py index d20e493dd..de4f4bcb3 100644 --- a/pulpcore/cli/core/task.py +++ b/pulpcore/cli/core/task.py @@ -6,6 +6,7 @@ import click from pulpcore.cli.common.context import ( + DATETIME_FORMATS, PluginRequirement, PulpContext, PulpEntityContext, @@ -27,8 +28,6 @@ translation = get_translation(__name__) _ = translation.gettext -DATETIME_FORMATS = ["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S"] - def _uuid_callback( ctx: click.Context, param: click.Parameter, value: Optional[str] @@ -55,7 +54,52 @@ def task(ctx: click.Context, pulp_ctx: PulpContext) -> None: ctx.obj = PulpTaskContext(pulp_ctx) -task.add_command(list_command(decorators=task_filter)) +task.add_command( + list_command( + decorators=task_filter + + [ + pulp_option( + "--reserved-resource", + "reserved_resources", + help=_("Href of a resource reserved by the task."), + ), + pulp_option( + "--reserved-resource-in", + "reserved_resources__in", + multiple=True, + help=_("Href of a resource reserved by the task. May be specified multiple times."), + needs_plugins=[PluginRequirement("core", min="3.22.0dev")], + ), + pulp_option( + "--exclusive-resource", + "exclusive_resources", + help=_("Href of a resource reserved exclusively by the task."), + ), + pulp_option( + "--exclusive-resource-in", + "exclusive_resources__in", + multiple=True, + help=_( + "Href of a resource reserved exclusively by the task." + " May be specified multiple times." + ), + needs_plugins=[PluginRequirement("core", min="3.22.0dev")], + ), + pulp_option( + "--shared-resource", + "shared_resources", + help=_("Href of a resource shared by the task."), + ), + pulp_option( + "--shared-resource-in", + "shared_resources__in", + multiple=True, + help=_("Href of a resource shared by the task. May be specified multiple times."), + needs_plugins=[PluginRequirement("core", min="3.22.0dev")], + ), + ] + ) +) task.add_command(destroy_command(decorators=[href_option, uuid_option])) task.add_command( role_command( @@ -155,5 +199,4 @@ def purge( ) -> None: pulp_ctx.needs_plugin(PluginRequirement("core", "3.17.0")) state_list = list(state) if state else None - finished_str = finished.strftime(DATETIME_FORMATS[1]) if finished else None - task_ctx.purge(finished_str, state_list) + task_ctx.purge(finished, state_list) diff --git a/tests/scripts/pulpcore/test_task.sh b/tests/scripts/pulpcore/test_task.sh index 4f4efe551..88e8296e9 100755 --- a/tests/scripts/pulpcore/test_task.sh +++ b/tests/scripts/pulpcore/test_task.sh @@ -16,12 +16,17 @@ trap cleanup EXIT sync_task="pulp_file.app.tasks.synchronizing.synchronize" expect_succ pulp task list --name $sync_task --state canceled count="$(echo "$OUTPUT" | jq -r length)" +expect_succ pulp worker list --limit 1 +worker="$(echo "$OUTPUT" | jq -r '.[0].pulp_href')" +worker_name="$(echo "$OUTPUT" | jq -r '.[0].name')" expect_succ pulp file remote create --name "cli_test_file_remote" \ --url "$FILE_REMOTE_URL" +remote_href="$(echo "$OUTPUT" | jq -r '.pulp_href')" expect_succ pulp file remote create --name "cli_test_file_large_remote" \ --url "$FILE_LARGE_REMOTE_URL" expect_succ pulp file repository create --name "cli_test_file_repository" --remote "cli_test_file_large_remote" +repository_href="$(echo "$OUTPUT" | jq -r '.pulp_href')" # Test canceling a task if pulp debug has-plugin --name "core" --min-version "3.12.0" @@ -42,9 +47,39 @@ task=$(echo "$ERROUTPUT" | grep -E -o "${PULP_API_ROOT}api/v3/tasks/[-[:xdigit:] task_uuid="${task%/}" task_uuid="${task_uuid##*/}" expect_succ pulp task show --wait --uuid "$task_uuid" +created_resource="$(echo "$OUTPUT" | jq -r '.created_resources[0]')" expect_succ test "$(echo "$OUTPUT" | jq -r '.state')" = "completed" expect_succ pulp task list --name-contains file +expect_succ pulp task list --parent-task "$task" --worker "$worker" +expect_succ pulp task list --parent-task "$task_uuid" --worker "$worker_name" +expect_succ pulp task list --started-before "21/01/12" --started-after "22/01/06T00:00:00" +expect_succ pulp task list --finished-before "2021-12-01" --finished-after "2022-06-01 00:00:00" +expect_succ pulp task list --created-resource "$created_resource" + +if pulp debug has-plugin --name "core" --min-version "3.22.0.dev" +then + # New style task resource filters + expect_succ pulp task list --reserved-resource-in "$repository_href" --reserved-resource-in "$remote_href" + expect_succ test "$(echo "$OUTPUT" | jq -r length)" -eq 1 + expect_succ pulp task list --reserved-resource "$repository_href" + expect_succ test "$(echo "$OUTPUT" | jq -r length)" -eq 2 + expect_succ pulp task list --exclusive-resource "$repository_href" + expect_succ test "$(echo "$OUTPUT" | jq -r length)" -eq 2 + expect_succ pulp task list --exclusive-resource "$remote_href" + expect_succ test "$(echo "$OUTPUT" | jq -r length)" -eq 0 + expect_succ pulp task list --shared-resource "$remote_href" + expect_succ test "$(echo "$OUTPUT" | jq -r length)" -eq 1 +else + expect_succ pulp task list --reserved-resource "$repository_href" + if pulp debug has-plugin --name "core" --min-version "3.12.0" + then + expect_succ test "$(echo "$OUTPUT" | jq -r length)" -eq 2 + else + # We didn't run the cancel test in this condition + expect_succ test "$(echo "$OUTPUT" | jq -r length)" -eq 1 + fi +fi expect_fail pulp task list --state=cannotwork expect_succ pulp task list --state=COmPLetED diff --git a/tests/scripts/test_debug_api.sh b/tests/scripts/test_debug_api.sh index 3e8b9405a..8cd3e8c2a 100755 --- a/tests/scripts/test_debug_api.sh +++ b/tests/scripts/test_debug_api.sh @@ -5,4 +5,4 @@ expect_succ pulp -v status -echo "${ERROUTPUT}" | grep -q "^get https\?://\w.*/api/v3/status/$" +echo "${ERROUTPUT}" | grep -q "^status_read : get https\?://\w.*/api/v3/status/$"