From b61728ea7e47e2f2bbbb91e0a825c70275e476da Mon Sep 17 00:00:00 2001 From: Kevin Van Brunt Date: Sat, 16 Mar 2019 23:27:34 -0400 Subject: [PATCH 1/5] Added Statement object to argparse Namespace passed to wrapped functions --- cmd2/cmd2.py | 35 +++++++++++++------ cmd2/parsing.py | 90 ++++++++++++++++++++++++++++--------------------- 2 files changed, 76 insertions(+), 49 deletions(-) diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py index e41d947d2..d0a86391e 100644 --- a/cmd2/cmd2.py +++ b/cmd2/cmd2.py @@ -49,7 +49,7 @@ from .argparse_completer import AutoCompleter, ACArgumentParser, ACTION_ARG_CHOICES from .clipboard import can_clip, get_paste_buffer, write_to_paste_buffer from .history import History, HistoryItem -from .parsing import StatementParser, Statement, Macro, MacroArg, shlex_split, get_command_arg_list +from .parsing import StatementParser, Statement, Macro, MacroArg, shlex_split # Set up readline from .rl_utils import rl_type, RlType, rl_get_point, rl_set_prompt, vt100_support, rl_make_safe_prompt @@ -174,9 +174,13 @@ def with_argument_list(*args: List[Callable], preserve_quotes: bool = False) -> def arg_decorator(func: Callable): @functools.wraps(func) def cmd_wrapper(cmd2_instance, statement: Union[Statement, str]): - parsed_arglist = get_command_arg_list(statement, preserve_quotes) + _, parsed_arglist = cmd2_instance.statement_parser.get_command_arg_list(command_name, + statement, + preserve_quotes) + return func(cmd2_instance, parsed_arglist) + command_name = func.__name__[len(COMMAND_FUNC_PREFIX):] cmd_wrapper.__doc__ = func.__doc__ return cmd_wrapper @@ -193,7 +197,10 @@ def with_argparser_and_unknown_args(argparser: argparse.ArgumentParser, preserve :param argparser: unique instance of ArgumentParser :param preserve_quotes: if True, then arguments passed to argparse maintain their quotes - :return: function that gets passed argparse-parsed args and a list of unknown argument strings + :return: function that gets passed argparse-parsed args in a Namespace and a list of unknown argument strings + A member called __statement__ is added to the Namespace to provide command functions access to the + Statement object. This can be useful when knowledge of the command line is needed. + """ import functools @@ -201,18 +208,22 @@ def with_argparser_and_unknown_args(argparser: argparse.ArgumentParser, preserve def arg_decorator(func: Callable): @functools.wraps(func) def cmd_wrapper(cmd2_instance, statement: Union[Statement, str]): - parsed_arglist = get_command_arg_list(statement, preserve_quotes) + statement, parsed_arglist = cmd2_instance.statement_parser.get_command_arg_list(command_name, + statement, + preserve_quotes) try: args, unknown = argparser.parse_known_args(parsed_arglist) except SystemExit: return else: + setattr(args, '__statement__', statement) return func(cmd2_instance, args, unknown) # argparser defaults the program name to sys.argv[0] # we want it to be the name of our command - argparser.prog = func.__name__[len(COMMAND_FUNC_PREFIX):] + command_name = func.__name__[len(COMMAND_FUNC_PREFIX):] + argparser.prog = command_name # If the description has not been set, then use the method docstring if one exists if argparser.description is None and func.__doc__: @@ -236,7 +247,9 @@ def with_argparser(argparser: argparse.ArgumentParser, :param argparser: unique instance of ArgumentParser :param preserve_quotes: if True, then arguments passed to argparse maintain their quotes - :return: function that gets passed the argparse-parsed args + :return: function that gets passed the argparse-parsed args in a Namespace + A member called __statement__ is added to the Namespace to provide command functions access to the + Statement object. This can be useful when knowledge of the command line is needed. """ import functools @@ -244,19 +257,21 @@ def with_argparser(argparser: argparse.ArgumentParser, def arg_decorator(func: Callable): @functools.wraps(func) def cmd_wrapper(cmd2_instance, statement: Union[Statement, str]): - - parsed_arglist = get_command_arg_list(statement, preserve_quotes) - + statement, parsed_arglist = cmd2_instance.statement_parser.get_command_arg_list(command_name, + statement, + preserve_quotes) try: args = argparser.parse_args(parsed_arglist) except SystemExit: return else: + setattr(args, '__statement__', statement) return func(cmd2_instance, args) # argparser defaults the program name to sys.argv[0] # we want it to be the name of our command - argparser.prog = func.__name__[len(COMMAND_FUNC_PREFIX):] + command_name = func.__name__[len(COMMAND_FUNC_PREFIX):] + argparser.prog = command_name # If the description has not been set, then use the method docstring if one exists if argparser.description is None and func.__doc__: diff --git a/cmd2/parsing.py b/cmd2/parsing.py index d72ca4ec5..cbb220fba 100644 --- a/cmd2/parsing.py +++ b/cmd2/parsing.py @@ -236,34 +236,6 @@ def argv(self) -> List[str]: return rtn -def get_command_arg_list(to_parse: Union[Statement, str], preserve_quotes: bool) -> List[str]: - """ - Called by the argument_list and argparse wrappers to retrieve just the arguments being - passed to their do_* methods as a list. - - :param to_parse: what is being passed to the do_* method. It can be one of two types: - 1. An already parsed Statement - 2. An argument string in cases where a do_* method is explicitly called - e.g.: Calling do_help('alias create') would cause to_parse to be 'alias create' - - :param preserve_quotes: if True, then quotes will not be stripped from the arguments - :return: the arguments in a list - """ - if isinstance(to_parse, Statement): - # In the case of a Statement, we already have what we need - if preserve_quotes: - return to_parse.arg_list - else: - return to_parse.argv[1:] - else: - # We have the arguments in a string. Use shlex to split it. - parsed_arglist = shlex_split(to_parse) - if not preserve_quotes: - parsed_arglist = [utils.strip_quotes(arg) for arg in parsed_arglist] - - return parsed_arglist - - class StatementParser: """Parse raw text into command components. @@ -371,16 +343,21 @@ def is_valid_command(self, word: str) -> Tuple[bool, str]: errmsg = '' return valid, errmsg - def tokenize(self, line: str) -> List[str]: - """Lex a string into a list of tokens. - - shortcuts and aliases are expanded and comments are removed - - Raises ValueError if there are unclosed quotation marks. + def tokenize(self, line: str, expand: bool = True) -> List[str]: + """ + Lex a string into a list of tokens. Shortcuts and aliases are expanded and comments are removed + + :param line: the command line being lexed + :param expand: if True, then aliases and shortcuts will be expanded + set this to False if the first token does not need to be expanded + because the command name is already known (Defaults to True) + :return: A list of tokens + :raises ValueError if there are unclosed quotation marks. """ # expand shortcuts and aliases - line = self._expand(line) + if expand: + line = self._expand(line) # check if this line is a comment if line.strip().startswith(constants.COMMENT_CHAR): @@ -393,12 +370,18 @@ def tokenize(self, line: str) -> List[str]: tokens = self._split_on_punctuation(tokens) return tokens - def parse(self, line: str) -> Statement: - """Tokenize the input and parse it into a Statement object, stripping + def parse(self, line: str, expand: bool = True) -> Statement: + """ + Tokenize the input and parse it into a Statement object, stripping comments, expanding aliases and shortcuts, and extracting output redirection directives. - Raises ValueError if there are unclosed quotation marks. + :param line: the command line being parsed + :param expand: if True, then aliases and shortcuts will be expanded + set this to False if the first token does not need to be expanded + because the command name is already known (Defaults to True) + :return: A parsed Statement + :raises ValueError if there are unclosed quotation marks """ # handle the special case/hardcoded terminator of a blank line @@ -413,7 +396,7 @@ def parse(self, line: str) -> Statement: arg_list = [] # lex the input into a list of tokens - tokens = self.tokenize(line) + tokens = self.tokenize(line, expand) # of the valid terminators, find the first one to occur in the input terminator_pos = len(tokens) + 1 @@ -594,6 +577,35 @@ def parse_command_only(self, rawinput: str) -> Statement: ) return statement + def get_command_arg_list(self, command_name: str, to_parse: Union[Statement, str], + preserve_quotes: bool) -> Tuple[Statement, List[str]]: + """ + Called by the argument_list and argparse wrappers to retrieve just the arguments being + passed to their do_* methods as a list. + + :param command_name: name of the command being run + :param to_parse: what is being passed to the do_* method. It can be one of two types: + 1. An already parsed Statement + 2. An argument string in cases where a do_* method is explicitly called + e.g.: Calling do_help('alias create') would cause to_parse to be 'alias create' + + In this case, the string will be converted to a Statement and returned along + with the argument list. + + :param preserve_quotes: if True, then quotes will not be stripped from the arguments + :return: A tuple containing: + The Statement used to retrieve the arguments + The argument list + """ + # Check if to_parse needs to be converted to a Statement + if not isinstance(to_parse, Statement): + to_parse = self.parse(command_name + ' ' + to_parse, expand=False) + + if preserve_quotes: + return to_parse, to_parse.arg_list + else: + return to_parse, to_parse.argv[1:] + def _expand(self, line: str) -> str: """Expand shortcuts and aliases""" From 28b7ec2ae2531366d41805b19700ca93677502b4 Mon Sep 17 00:00:00 2001 From: Kevin Van Brunt Date: Sat, 16 Mar 2019 23:48:37 -0400 Subject: [PATCH 2/5] Updated documentation --- CHANGELOG.md | 3 +++ cmd2/cmd2.py | 4 ++-- cmd2/parsing.py | 14 ++++++++------ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d21b33cd..dd6852f12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ also be colored. * `help_error` - the error that prints when no help information can be found * `default_error` - the error that prints when a non-existent command is run + * The `with_argparser` decorators now add a Statement object to the `argparse.Namespace` object they pass + to the `do_*` functions. It is stored in an attribute called `__statement__`. This can be useful if a command + function needs to know the command line for things like logging. * Potentially breaking changes * The following commands now write to stderr instead of stdout when printing an error. This will make catching errors easier in pyscript. diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py index d0a86391e..13278b44b 100644 --- a/cmd2/cmd2.py +++ b/cmd2/cmd2.py @@ -199,7 +199,7 @@ def with_argparser_and_unknown_args(argparser: argparse.ArgumentParser, preserve :param preserve_quotes: if True, then arguments passed to argparse maintain their quotes :return: function that gets passed argparse-parsed args in a Namespace and a list of unknown argument strings A member called __statement__ is added to the Namespace to provide command functions access to the - Statement object. This can be useful when knowledge of the command line is needed. + Statement object. This can be useful if the command function needs to know the command line. """ import functools @@ -249,7 +249,7 @@ def with_argparser(argparser: argparse.ArgumentParser, :param preserve_quotes: if True, then arguments passed to argparse maintain their quotes :return: function that gets passed the argparse-parsed args in a Namespace A member called __statement__ is added to the Namespace to provide command functions access to the - Statement object. This can be useful when knowledge of the command line is needed. + Statement object. This can be useful if the command function needs to know the command line. """ import functools diff --git a/cmd2/parsing.py b/cmd2/parsing.py index cbb220fba..f2fe76280 100644 --- a/cmd2/parsing.py +++ b/cmd2/parsing.py @@ -348,9 +348,10 @@ def tokenize(self, line: str, expand: bool = True) -> List[str]: Lex a string into a list of tokens. Shortcuts and aliases are expanded and comments are removed :param line: the command line being lexed - :param expand: if True, then aliases and shortcuts will be expanded - set this to False if the first token does not need to be expanded - because the command name is already known (Defaults to True) + :param expand: If True, then aliases and shortcuts will be expanded. + Set this to False if no expansion should occur because the command name is already known. + Otherwise the command could be expanded if it matched an alias name. This is for cases where + a do_* function was called manually (e.g do_help('alias'). :return: A list of tokens :raises ValueError if there are unclosed quotation marks. """ @@ -377,9 +378,10 @@ def parse(self, line: str, expand: bool = True) -> Statement: redirection directives. :param line: the command line being parsed - :param expand: if True, then aliases and shortcuts will be expanded - set this to False if the first token does not need to be expanded - because the command name is already known (Defaults to True) + :param expand: If True, then aliases and shortcuts will be expanded. + Set this to False if no expansion should occur because the command name is already known. + Otherwise the command could be expanded if it matched an alias name. This is for cases where + a do_* function was called manually (e.g do_help('alias'). :return: A parsed Statement :raises ValueError if there are unclosed quotation marks """ From 61d95b7cb39551b2ca002c0fd2285c68a02a18a1 Mon Sep 17 00:00:00 2001 From: Kevin Van Brunt Date: Sun, 17 Mar 2019 00:06:16 -0400 Subject: [PATCH 3/5] Added unit tests for expand flag of parse() --- tests/test_parsing.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_parsing.py b/tests/test_parsing.py index 85ee07650..8cea3305c 100644 --- a/tests/test_parsing.py +++ b/tests/test_parsing.py @@ -471,11 +471,18 @@ def test_empty_statement_raises_exception(): ('l', 'shell', 'ls -al') ]) def test_parse_alias_and_shortcut_expansion(parser, line, command, args): + # Test first with expansion statement = parser.parse(line) assert statement.command == command assert statement == args assert statement.args == statement + # Now allow no expansion + statement = parser.parse(line, expand=False) + assert statement.command == line.split()[0] + assert statement.split() == line.split()[1:] + assert statement.args == statement + def test_parse_alias_on_multiline_command(parser): line = 'anothermultiline has > inside an unfinished command' statement = parser.parse(line) From 7db2786706e3f4b5cfbf90c88c29ba1ee1f1f39c Mon Sep 17 00:00:00 2001 From: Kevin Van Brunt Date: Sun, 17 Mar 2019 00:17:27 -0400 Subject: [PATCH 4/5] Added more documentation --- docs/argument_processing.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/argument_processing.rst b/docs/argument_processing.rst index bad683bf9..095b4bda8 100644 --- a/docs/argument_processing.rst +++ b/docs/argument_processing.rst @@ -9,7 +9,9 @@ Argument Processing 1. Parsing input and quoted strings like the Unix shell 2. Parse the resulting argument list using an instance of ``argparse.ArgumentParser`` that you provide -3. Passes the resulting ``argparse.Namespace`` object to your command function +3. Passes the resulting ``argparse.Namespace`` object to your command function. The ``Namespace`` includes the + ``Statement`` object that was parsed from the command line. It is stored in the ``__statement__`` attribute of + the ``Namespace``. 4. Adds the usage message from the argument parser to your command. 5. Checks if the ``-h/--help`` option is present, and if so, display the help message for the command From 40f75f7453c3df83d8e74a281ec1311d53b23eec Mon Sep 17 00:00:00 2001 From: Kevin Van Brunt Date: Sun, 17 Mar 2019 00:52:34 -0400 Subject: [PATCH 5/5] Updated docs and example --- CHANGELOG.md | 6 +++--- cmd2/parsing.py | 4 ++-- docs/argument_processing.rst | 4 ++-- examples/decorator_example.py | 16 +++++++++++----- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd6852f12..01b300278 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,9 @@ also be colored. * `help_error` - the error that prints when no help information can be found * `default_error` - the error that prints when a non-existent command is run - * The `with_argparser` decorators now add a Statement object to the `argparse.Namespace` object they pass - to the `do_*` functions. It is stored in an attribute called `__statement__`. This can be useful if a command - function needs to know the command line for things like logging. + * The `with_argparser` decorators now add the Statement object created when parsing the command line to the + `argparse.Namespace` object they pass to the `do_*` methods. It is stored in an attribute called `__statement__`. + This can be useful if a command function needs to know the command line for things like logging. * Potentially breaking changes * The following commands now write to stderr instead of stdout when printing an error. This will make catching errors easier in pyscript. diff --git a/cmd2/parsing.py b/cmd2/parsing.py index f2fe76280..514f5faf5 100644 --- a/cmd2/parsing.py +++ b/cmd2/parsing.py @@ -351,7 +351,7 @@ def tokenize(self, line: str, expand: bool = True) -> List[str]: :param expand: If True, then aliases and shortcuts will be expanded. Set this to False if no expansion should occur because the command name is already known. Otherwise the command could be expanded if it matched an alias name. This is for cases where - a do_* function was called manually (e.g do_help('alias'). + a do_* method was called manually (e.g do_help('alias'). :return: A list of tokens :raises ValueError if there are unclosed quotation marks. """ @@ -381,7 +381,7 @@ def parse(self, line: str, expand: bool = True) -> Statement: :param expand: If True, then aliases and shortcuts will be expanded. Set this to False if no expansion should occur because the command name is already known. Otherwise the command could be expanded if it matched an alias name. This is for cases where - a do_* function was called manually (e.g do_help('alias'). + a do_* method was called manually (e.g do_help('alias'). :return: A parsed Statement :raises ValueError if there are unclosed quotation marks """ diff --git a/docs/argument_processing.rst b/docs/argument_processing.rst index 095b4bda8..fc1f24332 100644 --- a/docs/argument_processing.rst +++ b/docs/argument_processing.rst @@ -10,8 +10,8 @@ Argument Processing 1. Parsing input and quoted strings like the Unix shell 2. Parse the resulting argument list using an instance of ``argparse.ArgumentParser`` that you provide 3. Passes the resulting ``argparse.Namespace`` object to your command function. The ``Namespace`` includes the - ``Statement`` object that was parsed from the command line. It is stored in the ``__statement__`` attribute of - the ``Namespace``. + ``Statement`` object that was created when parsing the command line. It is stored in the ``__statement__`` + attribute of the ``Namespace``. 4. Adds the usage message from the argument parser to your command. 5. Checks if the ``-h/--help`` option is present, and if so, display the help message for the command diff --git a/examples/decorator_example.py b/examples/decorator_example.py index 5d127619b..79bd76330 100755 --- a/examples/decorator_example.py +++ b/examples/decorator_example.py @@ -12,6 +12,7 @@ """ import argparse import sys +from typing import List import cmd2 @@ -46,7 +47,7 @@ def __init__(self, ip_addr=None, port=None, transcript_files=None): speak_parser.add_argument('words', nargs='+', help='words to say') @cmd2.with_argparser(speak_parser) - def do_speak(self, args): + def do_speak(self, args: argparse.Namespace): """Repeats what you tell me to.""" words = [] for word in args.words: @@ -67,13 +68,18 @@ def do_speak(self, args): tag_parser.add_argument('content', nargs='+', help='content to surround with tag') @cmd2.with_argparser(tag_parser) - def do_tag(self, args): - """create a html tag""" + def do_tag(self, args: argparse.Namespace): + """create an html tag""" + # The Namespace always includes the Statement object created when parsing the command line + statement = args.__statement__ + + self.poutput("The command line you ran was: {}".format(statement.command_and_args)) + self.poutput("It generated this tag:") self.poutput('<{0}>{1}'.format(args.tag, ' '.join(args.content))) @cmd2.with_argument_list - def do_tagg(self, arglist): - """verion of creating an html tag using arglist instead of argparser""" + def do_tagg(self, arglist: List[str]): + """version of creating an html tag using arglist instead of argparser""" if len(arglist) >= 2: tag = arglist[0] content = arglist[1:]