From 847142443a9e234c134f3bde5707b91e172955cb Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Fri, 17 Feb 2023 17:39:56 +0100 Subject: [PATCH] [resotocore][feat] AliasTenplate: use default args syntax instead of key/value (#1452) * [resotocore][feat] AliasTenplate: use default args syntax instead of key/value * translate between internal _ and - in args * add test for dash underscore handling --- plugins/aws/resoto_plugin_aws/__init__.py | 9 +++--- resotocore/resotocore/cli/__init__.py | 15 +++++++++ resotocore/resotocore/cli/cli.py | 16 +++++++--- resotocore/resotocore/cli/model.py | 31 +++++++++++++++++-- resotocore/resotocore/core_config.py | 11 +++++++ .../static/report/checks/aws/aws_rds.json | 2 +- resotocore/resotocore/web/api.py | 8 ++++- .../tests/resotocore/cli/command_test.py | 8 ++--- resotocore/tests/resotocore/cli/model_test.py | 4 +-- resotolib/resotolib/parse_util.py | 1 + resotoshell/resotoshell/promptsession.py | 2 +- 11 files changed, 87 insertions(+), 20 deletions(-) diff --git a/plugins/aws/resoto_plugin_aws/__init__.py b/plugins/aws/resoto_plugin_aws/__init__.py index 3fe39b275..813ea00d1 100644 --- a/plugins/aws/resoto_plugin_aws/__init__.py +++ b/plugins/aws/resoto_plugin_aws/__init__.py @@ -122,13 +122,13 @@ def regions(self, profile: Optional[str] = None) -> List[str]: name="aws", info="Execute aws commands on AWS resources", args_description={ + "service": "Defines the AWS service, like ec2, s3, iam, etc.", + "operation": "Defines the operation to execute.", + "operation_args": "Defines the arguments for the operation. The parameters depend on the operation.", "--account": "[Optional] The AWS account identifier.", "--role": "[Optional] The AWS role.", "--profile": "[Optional] The AWS profile to use.", "--region": "[Optional] The AWS region.", - "service": "Defines the AWS service, like ec2, s3, iam, etc.", - "operation": "Defines the operation to execute.", - "operation_args": "Defines the arguments for the operation. The parameters depend on the operation.", }, description="Execute an operation on an AWS resource.\n" "For a list of services with respective operations see " @@ -138,7 +138,8 @@ def regions(self, profile: Optional[str] = None) -> List[str]: "There are two modes of operation:\n" "1. Use a search and then pipe the result of the search into the `aws` command. " "Every resource matched by the search will invoke this command. " - "You can use templating parameter to define the exact invocation arguments.\n" + "You can use templating parameter to define the exact invocation arguments. " + "Account, region, profile and role is defined by the resource if not defined explicitly.\n" "2. Call the `aws` command directly without passing any resource to interact " "with AWS using the credentials defined via configuration.\n\n" "## Examples\n\n" diff --git a/resotocore/resotocore/cli/__init__.py b/resotocore/resotocore/cli/__init__.py index 4d49c60a1..c223ec802 100644 --- a/resotocore/resotocore/cli/__init__.py +++ b/resotocore/resotocore/cli/__init__.py @@ -21,6 +21,8 @@ double_quoted_string_part_dp, backslash_dp, single_quoted_string_part_dp, + dash_dp, + equals_p, ) from resotocore.model.graph_access import Section from resotocore.types import JsonElement @@ -42,6 +44,18 @@ def key_value_parser() -> Parser: return key, value +@make_parser +def arg_with_value_parser() -> Parser: + # parses --foo bla as (foo, bla) and --foo=bla as (foo, bla) + # translates - to _ in the key --foo-test bla -> (foo_test, bla) + yield dash_dp + yield dash_dp + key = yield literal_dp + yield equals_p | space_dp.at_least(1) + value = yield json_value_dp + return key.replace("-", "_"), value + + # for the cli part: unicode and special characters are translated. escaped \\ \' \" are preserved string_esc_dp = backslash_dp >> ( backslash_dp.result(r"\\") @@ -62,6 +76,7 @@ def key_value_parser() -> Parser: # name=value test=true -> {name: value, test: true} key_values_parser: Parser = key_value_parser.sep_by(comma_p | space_dp).map(dict) +args_values_parser: Parser = arg_with_value_parser.sep_by(space_dp).map(dict) # anything that is not: | " ' ; \ cmd_token = regex("[^|\"';\\\\]+") # single and double-quoted string are maintained with quotes: "foo"->"foo", 'foo'->'foo' diff --git a/resotocore/resotocore/cli/cli.py b/resotocore/resotocore/cli/cli.py index 0a867c0af..43df5bcda 100644 --- a/resotocore/resotocore/cli/cli.py +++ b/resotocore/resotocore/cli/cli.py @@ -24,7 +24,7 @@ from resotolib.utils import get_local_tzinfo from resotocore import version from resotocore.analytics import CoreEvent -from resotocore.cli import cmd_with_args_parser, key_values_parser, T, Sink +from resotocore.cli import cmd_with_args_parser, key_values_parser, T, Sink, args_values_parser from resotocore.cli.command import ( SearchPart, PredecessorsPart, @@ -59,6 +59,7 @@ AliasTemplate, ArgsInfo, ArgInfo, + AliasTemplateParameter, ) from resotocore.console_renderer import ConsoleRenderer from resotocore.error import CLIParseError @@ -453,16 +454,23 @@ async def parse_line(parsed: ParsedCommands) -> ParsedCommandLine: def expand_aliases(line: ParsedCommands) -> ParsedCommands: def expand_alias(alias_cmd: ParsedCommand) -> List[ParsedCommand]: alias: AliasTemplate = self.alias_templates[alias_cmd.cmd] + available: Dict[str, AliasTemplateParameter] = {p.name: p for p in alias.parameters} props: Dict[str, JsonElement] = self.replacements(**{**self.cli_env, **context.env}) # type: ignore props["args"] = alias_cmd.args for p in alias.parameters: props[p.name] = p.default # only parse properties, if there are any declared if alias.parameters: - props.update(key_values_parser.parse(alias_cmd.args or "")) - undefined = [k for k, v in props.items() if k != "args" and v is None] + args = (alias_cmd.args or "").strip() + parser = args_values_parser if args.startswith("--") else key_values_parser + props.update(parser.parse(args)) + undefined = [ + available[k].arg_name for k, v in props.items() if k != "args" and v is None and k in available + ] if undefined: - raise AttributeError(f"Alias {alias_cmd.cmd} missing attributes: {', '.join(undefined)}") + raise AttributeError( + f"Alias {alias_cmd.cmd} not enough parameters provided. Missing: {', '.join(undefined)}" + ) rendered = alias.render(props) log.debug(f"The rendered alias template is: {rendered}") return single_commands.parse(rendered) # type: ignore diff --git a/resotocore/resotocore/cli/model.py b/resotocore/resotocore/cli/model.py index e69df2023..c14877937 100644 --- a/resotocore/resotocore/cli/model.py +++ b/resotocore/resotocore/cli/model.py @@ -381,6 +381,10 @@ class AliasTemplateParameter: def example_value(self) -> JsonElement: return self.default if self.default else f"test_{self.name}" + @property + def arg_name(self) -> str: + return "--" + self.name.replace("_", "-") + # pylint: disable=not-an-iterable @define(order=True, hash=True, frozen=True) @@ -390,13 +394,27 @@ class AliasTemplate: template: str parameters: List[AliasTemplateParameter] = field(factory=list) description: Optional[str] = None + # only use args_description if the template does not use explicit parameters args_description: Dict[str, str] = field(factory=dict) + allowed_in_source_position: bool = False def render(self, props: Json) -> str: return render_template(self.template, props) + def args_info(self) -> ArgsInfo: + args_desc = [ArgInfo(name, expects_value=True, help_text=desc) for name, desc in self.args_description.items()] + param = [ + ArgInfo( + p.arg_name, + expects_value=True, + help_text=f"[{'required' if p.default is None else 'optional'}] {p.description}", + ) + for p in sorted(self.parameters, key=lambda p: p.default is not None) # required parameters first + ] + return args_desc + param + def help_with_params(self) -> str: - args = ", ".join(f"{arg.name}=" for arg in self.parameters) + args = " ".join(f"{arg.arg_name} " for arg in self.parameters) def param_info(p: AliasTemplateParameter) -> str: default = f" [default: {p.default}]" if p.default else "" @@ -404,7 +422,7 @@ def param_info(p: AliasTemplateParameter) -> str: indent = " " arg_info = f"\n{indent}".join(param_info(arg) for arg in sorted(self.parameters, key=attrgetter("name"))) - minimal = ", ".join(f'{p.name}="{p.example_value()}"' for p in self.parameters if p.default is None) + minimal = " ".join(f'{p.arg_name} "{p.example_value()}"' for p in self.parameters if p.default is None) desc = "" if self.description: for line in self.description.splitlines(): @@ -458,7 +476,14 @@ def from_config(cfg: AliasTemplateConfig) -> AliasTemplate: def arg(p: AliasTemplateParameterConfig) -> AliasTemplateParameter: return AliasTemplateParameter(p.name, p.description, p.default) - return AliasTemplate(cfg.name, cfg.info, cfg.template, [arg(a) for a in cfg.parameters], cfg.description) + return AliasTemplate( + name=cfg.name, + info=cfg.info, + template=cfg.template, + parameters=[arg(a) for a in cfg.parameters], + description=cfg.description, + allowed_in_source_position=cfg.allowed_in_source_position or False, + ) class InternalPart(ABC): diff --git a/resotocore/resotocore/core_config.py b/resotocore/resotocore/core_config.py index dda8f6ab6..7f9127886 100644 --- a/resotocore/resotocore/core_config.py +++ b/resotocore/resotocore/core_config.py @@ -188,6 +188,12 @@ class AliasTemplateConfig(ConfigObject): factory=list, metadata=dict(description="All template parameters.") ) description: Optional[str] = field(metadata=dict(description="A longer description of the command."), default=None) + allowed_in_source_position: Optional[bool] = field( + metadata=dict( + description="true if this alias can be executed directly, false if it expects input from another command." + ), + default=False, + ) def alias_templates() -> List[AliasTemplateConfig]: @@ -224,6 +230,7 @@ def alias_templates() -> List[AliasTemplateConfig]: AliasTemplateParameterConfig("title", "Alert title"), AliasTemplateParameterConfig("webhook", "Discord webhook URL"), ], + allowed_in_source_position=False, ), AliasTemplateConfig( name="slack", @@ -261,6 +268,7 @@ def alias_templates() -> List[AliasTemplateConfig]: AliasTemplateParameterConfig("title", "Alert title"), AliasTemplateParameterConfig("webhook", "Slack webhook URL"), ], + allowed_in_source_position=False, ), AliasTemplateConfig( name="jira", @@ -301,6 +309,7 @@ def alias_templates() -> List[AliasTemplateConfig]: AliasTemplateParameterConfig("project_id", "Jira project ID"), AliasTemplateParameterConfig("reporter_id", "Jira reporter user ID"), ], + allowed_in_source_position=False, ), AliasTemplateConfig( name="alertmanager", @@ -335,6 +344,7 @@ def alias_templates() -> List[AliasTemplateConfig]: AliasTemplateParameterConfig("duration", "The duration of this alert in alertmanager.", "3h"), AliasTemplateParameterConfig("alertmanager_url", "The complete url to alertmanager."), ], + allowed_in_source_position=False, ), AliasTemplateConfig( name="pagerduty", @@ -409,6 +419,7 @@ def alias_templates() -> List[AliasTemplateConfig]: "https://events.pagerduty.com/v2/enqueue", ), ], + allowed_in_source_position=False, ), ] diff --git a/resotocore/resotocore/static/report/checks/aws/aws_rds.json b/resotocore/resotocore/static/report/checks/aws/aws_rds.json index f574f254e..8a5105bbd 100644 --- a/resotocore/resotocore/static/report/checks/aws/aws_rds.json +++ b/resotocore/resotocore/static/report/checks/aws/aws_rds.json @@ -37,7 +37,7 @@ "title": "Ensure there are no Public Accessible RDS instances.", "result_kind": "aws_rds_instance", "categories": ["security", "compliance"], - "risk": "Auto Minor Version Upgrade is a feature that you can enable to have your database automatically upgraded when a new minor database engine version is available. Minor version upgrades often patch security vulnerabilities and fix bugs and therefore should be applied.", + "risk": "Publicly accessible databases could expose sensitive data to bad actors.", "severity": "critical", "detect": { "resoto": "is(aws_rds_instance) and db_publicly_accessible==true" diff --git a/resotocore/resotocore/web/api.py b/resotocore/resotocore/web/api.py index cd8b5a6b5..366c6927d 100644 --- a/resotocore/resotocore/web/api.py +++ b/resotocore/resotocore/web/api.py @@ -849,7 +849,13 @@ def cmd_json(cmd: CLICommand) -> Json: } def alias_json(cmd: AliasTemplate) -> Json: - return {"name": cmd.name, "info": cmd.info, "help": cmd.help()} + return { + "name": cmd.name, + "info": cmd.info, + "help": cmd.help(), + "args": to_js(cmd.args_info(), force_dict=True), + "source": cmd.allowed_in_source_position, + } commands = [cmd_json(cmd) for cmd in self.cli.direct_commands.values() if not isinstance(cmd, InternalPart)] replacements = self.cli.replacements() diff --git a/resotocore/tests/resotocore/cli/command_test.py b/resotocore/tests/resotocore/cli/command_test.py index 898ab10a1..5e54fdcfc 100644 --- a/resotocore/tests/resotocore/cli/command_test.py +++ b/resotocore/tests/resotocore/cli/command_test.py @@ -816,7 +816,7 @@ def test_if_set(prop: Any, value: Any) -> None: async def test_discord_alias(cli: CLI, echo_http_server: Tuple[int, List[Tuple[Request, Json]]]) -> None: port, requests = echo_http_server result = await cli.execute_cli_command( - f'search is(bla) | discord webhook="http://localhost:{port}/success" title=test message="test message"', + f'search is(bla) | discord --webhook "http://localhost:{port}/success" --title test --message "test message"', stream.list, ) # 100 times bla, discord allows 25 fields -> 4 requests @@ -840,7 +840,7 @@ async def test_discord_alias(cli: CLI, echo_http_server: Tuple[int, List[Tuple[R async def test_slack_alias(cli: CLI, echo_http_server: Tuple[int, List[Tuple[Request, Json]]]) -> None: port, requests = echo_http_server result = await cli.execute_cli_command( - f'search is(bla) | slack webhook="http://localhost:{port}/success" title=test message="test message"', + f'search is(bla) | slack --webhook "http://localhost:{port}/success" --title test --message "test message"', stream.list, ) # 100 times bla, discord allows 25 fields -> 4 requests @@ -861,7 +861,7 @@ async def test_slack_alias(cli: CLI, echo_http_server: Tuple[int, List[Tuple[Req async def test_jira_alias(cli: CLI, echo_http_server: Tuple[int, List[Tuple[Request, Json]]]) -> None: port, requests = echo_http_server result = await cli.execute_cli_command( - f'search is(bla) | jira url="http://localhost:{port}/success" title=test message="test message" username=test token=test project_id=10000 reporter_id=test', + f'search is(bla) | jira --url "http://localhost:{port}/success" --title test --message "test message" --username test --token test --project_id 10000 --reporter_id test', stream.list, ) assert result == [["1 requests with status 200 sent."]] @@ -883,7 +883,7 @@ async def test_jira_alias(cli: CLI, echo_http_server: Tuple[int, List[Tuple[Requ async def test_pagerduty_alias(cli: CLI, echo_http_server: Tuple[int, List[Tuple[Request, Json]]]) -> None: port, requests = echo_http_server result = await cli.execute_cli_command( - f'search is(bla) | pagerduty webhook_url="http://localhost:{port}/success" summary=test routing_key=123 dedup_key=234', + f'search is(bla) | pagerduty --webhook-url "http://localhost:{port}/success" --summary test --routing-key 123 --dedup-key 234', stream.list, ) assert result == [["1 requests with status 200 sent."]] diff --git a/resotocore/tests/resotocore/cli/model_test.py b/resotocore/tests/resotocore/cli/model_test.py index 3d32302cf..c8385a0f5 100644 --- a/resotocore/tests/resotocore/cli/model_test.py +++ b/resotocore/tests/resotocore/cli/model_test.py @@ -36,7 +36,7 @@ def test_alias_template() -> None: """ foo: does foes ```shell - foo a=, b= + foo --a --b ``` ## Parameters @@ -51,7 +51,7 @@ def test_alias_template() -> None: ## Example ```shell # Executing this alias template - > foo a="test_a" + > foo --a "test_a" # Will expand to this command > test_a | bv ``` diff --git a/resotolib/resotolib/parse_util.py b/resotolib/resotolib/parse_util.py index 2f078ddcb..2ca6cc35e 100644 --- a/resotolib/resotolib/parse_util.py +++ b/resotolib/resotolib/parse_util.py @@ -50,6 +50,7 @@ def lexeme(p: Parser) -> Parser: backtick_dp = string("`") backslash_dp = string("\\") pipe_dp = string("|") +dash_dp = string("-") integer_dp = regex(r"[+-]?[0-9]+").map(int) float_dp = regex(r"[+-]?[0-9]+\.[0-9]+").map(float) literal_dp = regex("[A-Za-z0-9][A-Za-z0-9_\\-]*") diff --git a/resotoshell/resotoshell/promptsession.py b/resotoshell/resotoshell/promptsession.py index 91b61e945..eab9e425e 100644 --- a/resotoshell/resotoshell/promptsession.py +++ b/resotoshell/resotoshell/promptsession.py @@ -785,7 +785,7 @@ def path(p: Property) -> List[str]: known_props = {p for v in aggregate_roots.values() for prop in v.properties or [] for p in path(prop)} info = await client.cli_info() - cmds = [jsons.load(cmd, CommandInfo) for cmd in info.get("commands", [])] + cmds = [jsons.load(cmd, CommandInfo) for cmd in (info.get("commands", []) + info.get("alias_templates", []))] return cmds, sorted(aggregate_roots.keys()), sorted(known_props) except Exception as ex: log.warning(