From de3e9b706c81ce0e2e1e737606efdc7c3ac16c00 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Tue, 13 Oct 2020 08:52:12 -0500 Subject: [PATCH 1/5] debug now handles NodeCoords --- yamlpath/wrappers/consoleprinter.py | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/yamlpath/wrappers/consoleprinter.py b/yamlpath/wrappers/consoleprinter.py index 4901f3ce..db8b277b 100644 --- a/yamlpath/wrappers/consoleprinter.py +++ b/yamlpath/wrappers/consoleprinter.py @@ -18,6 +18,8 @@ from ruamel.yaml.comments import CommentedMap +from yamlpath.wrappers.nodecoords import NodeCoords + class ConsolePrinter: """ @@ -222,6 +224,11 @@ def _debug_dump(cls, data: Any, **kwargs) -> Generator[str, None, None]: data, prefix=prefix, **kwargs ): yield line + elif isinstance(data, NodeCoords): + for line in ConsolePrinter._debug_node_coord( + data, prefix=prefix, **kwargs + ): + yield line else: yield ConsolePrinter._debug_scalar(data, prefix=prefix, **kwargs) @@ -240,6 +247,29 @@ def _debug_scalar(cls, data: Any, **kwargs) -> str: return ConsolePrinter._debug_prefix_lines( "{}{}{}".format(prefix, data, (type(data) if print_type else ""))) + @classmethod + def _debug_node_coord( + cls, data: NodeCoords, **kwargs + ) -> Generator[str, None, None]: + """Helper method for debug.""" + prefix = kwargs.pop("prefix", "") + node_prefix = "{}(node)".format(prefix) + parent_prefix = "{}(parent)".format(prefix) + parentref_prefix = "{}(parentref)".format(prefix) + + for line in ConsolePrinter._debug_dump(data.node, prefix=node_prefix): + yield line + + for line in ConsolePrinter._debug_dump( + data.parent, prefix=parent_prefix + ): + yield line + + for line in ConsolePrinter._debug_dump( + data.parentref, prefix=parentref_prefix + ): + yield line + @classmethod def _debug_list(cls, data: List, **kwargs) -> Generator[str, None, None]: """Helper for debug.""" From 85091856bd54bc1a29e7000f36ccc407e3cf20c7 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Tue, 13 Oct 2020 09:58:48 -0500 Subject: [PATCH 2/5] Fix: lists report false-positives on SEARCH nodes --- yamlpath/processor.py | 72 ++++++++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 21 deletions(-) diff --git a/yamlpath/processor.py b/yamlpath/processor.py index 0c9e99ca..d23438e6 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -85,7 +85,7 @@ def get_nodes(self, yaml_path: Union[YAMLPath, str], matched_nodes += 1 self.logger.debug( "Relaying required node:", - prefix="Processor::get_nodes: ", data=node_coords.node) + prefix="Processor::get_nodes: ", data=node_coords) yield node_coords if matched_nodes < 1: @@ -99,7 +99,7 @@ def get_nodes(self, yaml_path: Union[YAMLPath, str], ): self.logger.debug( "Relaying optional node:", - prefix="Processor::get_nodes: ", data=opt_node.node) + prefix="Processor::get_nodes: ", data=opt_node) yield opt_node def set_value(self, yaml_path: Union[YAMLPath, str], @@ -174,10 +174,13 @@ def set_value(self, yaml_path: Union[YAMLPath, str], self.data, yaml_path, value ): self.logger.debug( - ("Processor::set_value: Matched optional node coord, {};" - + " setting its value to {}<{}>.") - .format(node_coord, value, value_format) - ) + "Matched optional node coordinate:" + , data=node_coord + , prefix="Processor::set_value: ") + self.logger.debug( + "Setting its value with format {} to:".format(value_format) + , data=value + , prefix="Processor::set_value: ") try: self._update_node( node_coord.parent, node_coord.parentref, value, @@ -473,9 +476,9 @@ def _get_nodes_by_search( Raises: N/A """ self.logger.debug( - ("Processor::_get_nodes_by_search: Seeking SEARCH nodes matching" - + " {}.") - .format(terms)) + "Seeking SEARCH nodes matching {} in data:".format(terms), + data=data, + prefix="Processor::_get_nodes_by_search: ") parent = kwargs.pop("parent", None) parentref = kwargs.pop("parentref", None) @@ -484,7 +487,10 @@ def _get_nodes_by_search( method = terms.method attr = terms.attribute term = terms.term - if traverse_lists and isinstance(data, list): + if isinstance(data, list): + if not traverse_lists: + return + for lstidx, ele in enumerate(data): if attr == '.': matches = search_matches(method, term, ele) @@ -492,6 +498,10 @@ def _get_nodes_by_search( matches = search_matches(method, term, ele[attr]) if (matches and not invert) or (invert and not matches): + self.logger.debug( + "Yielding list match at index {}:".format(lstidx), + data=ele, + prefix="Processor::_get_nodes_by_search: ") yield NodeCoords(ele, data, lstidx) elif isinstance(data, dict): @@ -500,18 +510,31 @@ def _get_nodes_by_search( for key, val in data.items(): matches = search_matches(method, term, key) if (matches and not invert) or (invert and not matches): + self.logger.debug( + "Yielding dictionary key name match against '{}':" + .format(key), + data=val, + prefix="Processor::_get_nodes_by_search: ") yield NodeCoords(val, data, key) elif attr in data: value = data[attr] matches = search_matches(method, term, value) if (matches and not invert) or (invert and not matches): + self.logger.debug( + "Yielding dictionary attribute match against '{}':" + .format(attr), + data=value, + prefix="Processor::_get_nodes_by_search: ") yield NodeCoords(value, data, attr) else: # Check the passed data itself for a match matches = search_matches(method, term, data) if (matches and not invert) or (invert and not matches): + self.logger.debug( + "Yielding the queried data itself because it matches.", + prefix="Processor::_get_nodes_by_search: ") yield NodeCoords(data, parent, parentref) # pylint: disable=locally-disabled @@ -656,7 +679,7 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, parentref = kwargs.pop("parentref", None) self.logger.debug( - "TRAVERSING the tree at:", + "TRAVERSING the tree at parentref:", prefix="Processor::_get_nodes_by_traversal: ", data=parentref) if data is None: @@ -677,7 +700,7 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, self.logger.debug( "Yielding unfiltered Hash value:", prefix="Processor::_get_nodes_by_traversal: ", - data=node_coord.node) + data=node_coord) yield node_coord elif isinstance(data, list): for idx, ele in enumerate(data): @@ -688,7 +711,7 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, self.logger.debug( "Yielding unfiltered Array value:", prefix="Processor::_get_nodes_by_traversal: ", - data=node_coord.node) + data=node_coord) yield node_coord else: self.logger.debug( @@ -700,6 +723,7 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, # every child against the following segment until there are no more # nodes. For each match, resume normal path function against the # matching node(s). + direct_yielded = False # Because the calling code will continue to process the remainder # of the YAML Path, only the parent of the matched node(s) can be @@ -712,25 +736,31 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, parentref=parentref, traverse_lists=False ): self.logger.debug( - "Yielding filtered DIRECT node at {}:".format(parentref), + "Yielding filtered DIRECT node at parentref {} of coord:" + .format(parentref), prefix="Processor::_get_nodes_by_traversal: ", - data=node_coord.node) + data=node_coord) yield NodeCoords(data, parent, parentref) + direct_yielded = True + + # Do not traverse into parents which already match + if direct_yielded: + return # Then, recurse into each child to perform the same test. if isinstance(data, dict): for key, val in data.items(): self.logger.debug( "Processor::_get_nodes_by_traversal: Recursing into" - " KEY {} at {} for next-segment matches..." + " KEY '{}' at ref '{}' for next-segment matches..." .format(key, parentref)) for node_coord in self._get_nodes_by_traversal( val, yaml_path, segment_index, parent=data, parentref=key ): self.logger.debug( - "Yielding filtered indirect Hash value from KEY {}" - " at {}:".format(key, parentref), + "Yielding filtered indirect Hash value from KEY" + " '{}' at ref '{}':".format(key, parentref), prefix="Processor::_get_nodes_by_traversal: ", data=node_coord.node) yield node_coord @@ -738,7 +768,7 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, for idx, ele in enumerate(data): self.logger.debug( "Processor::_get_nodes_by_traversal: Recursing into" - " INDEX {} at {} for next-segment matches..." + " INDEX '{}' at ref '{}' for next-segment matches..." .format(idx, parentref)) for node_coord in self._get_nodes_by_traversal( ele, yaml_path, segment_index, @@ -748,7 +778,7 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, "Yielding filtered indirect Array value from INDEX" " {} at {}:".format(idx, parentref), prefix="Processor::_get_nodes_by_traversal: ", - data=node_coord.node) + data=node_coord) yield node_coord def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, @@ -820,7 +850,7 @@ def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, .format(type(subnode_coord.node), subnode_coord.parentref), prefix="Processor::_get_required_nodes: ", - data=subnode_coord.node, footer=" ") + data=subnode_coord, footer=" ") yield subnode_coord else: self.logger.debug( From 81b16ab8f3475dfde7a2febfe38896d6b9896553 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Tue, 13 Oct 2020 09:59:04 -0500 Subject: [PATCH 3/5] Consolidate NodeCoords debug logging --- yamlpath/commands/yaml_get.py | 4 +++- yamlpath/commands/yaml_paths.py | 36 ++++++++++++++------------------- yamlpath/commands/yaml_set.py | 13 ++++++++---- yamlpath/merger/mergerconfig.py | 18 +++-------------- 4 files changed, 30 insertions(+), 41 deletions(-) diff --git a/yamlpath/commands/yaml_get.py b/yamlpath/commands/yaml_get.py index b1ffe204..eabbd11c 100644 --- a/yamlpath/commands/yaml_get.py +++ b/yamlpath/commands/yaml_get.py @@ -170,7 +170,9 @@ def main(): publickey=args.publickey, privatekey=args.privatekey) try: for node in processor.get_eyaml_values(yaml_path, mustexist=True): - log.debug("Got {} from {}.".format(repr(node), yaml_path)) + log.debug( + "Got node from {}:".format(yaml_path), data=node, + prefix="yaml_get::main: ") discovered_nodes.append(unwrap_node_coords(node)) except YAMLPathException as ex: log.critical(ex, 1) diff --git a/yamlpath/commands/yaml_paths.py b/yamlpath/commands/yaml_paths.py index 48f1a29a..7e246b18 100644 --- a/yamlpath/commands/yaml_paths.py +++ b/yamlpath/commands/yaml_paths.py @@ -273,10 +273,9 @@ def yield_children(logger: ConsolePrinter, data: Any, include_value_aliases: bool = kwargs.pop("include_value_aliases", False) search_anchors: bool = kwargs.pop("search_anchors", False) logger.debug( - ("yaml_paths::yield_children: " - + "dumping all children in data of type, {}:") - .format(type(data))) - logger.debug(data) + "Dumping all children in data of type, {}:" + .format(type(data)), data=data, + prefix="yaml_paths::yield_children: ") exclude_alias_matchers = [AnchorMatches.UNSEARCHABLE_ALIAS, AnchorMatches.ALIAS_EXCLUDED] @@ -445,11 +444,9 @@ def search_for_paths(logger: ConsolePrinter, processor: EYAMLProcessor, if isinstance(ele, (CommentedSeq, CommentedMap)): logger.debug( - "yaml_paths::search_for_paths:" - + "recursing into complex data:" - ) - logger.debug(ele) - logger.debug(">>>> >>>> >>>> >>>> >>>> >>>> >>>>") + "Recursing into complex data:", data=ele, + prefix="yaml_paths::search_for_paths: ", + footer=">>>> >>>> >>>> >>>> >>>> >>>> >>>>") for subpath in search_for_paths( logger, processor, ele, terms, pathsep, tmp_path, seen_anchors, search_values=search_values, @@ -460,11 +457,10 @@ def search_for_paths(logger: ConsolePrinter, processor: EYAMLProcessor, expand_children=expand_children ): logger.debug( - ("yaml_paths::search_for_paths:" - + "yielding RECURSED match, {}." - ).format(subpath) + "Yielding RECURSED match, {}.".format(subpath), + prefix="yaml_paths::search_for_paths: ", + footer="<<<< <<<< <<<< <<<< <<<< <<<< <<<<" ) - logger.debug("<<<< <<<< <<<< <<<< <<<< <<<< <<<<") yield subpath elif search_values: if (anchor_matched is AnchorMatches.UNSEARCHABLE_ALIAS @@ -588,11 +584,10 @@ def search_for_paths(logger: ConsolePrinter, processor: EYAMLProcessor, if isinstance(val, (CommentedSeq, CommentedMap)): logger.debug( - "yaml_paths::search_for_paths:" - + "recursing into complex data:" + "Recursing into complex data:", data=val, + prefix="yaml_paths::search_for_paths: ", + footer=">>>> >>>> >>>> >>>> >>>> >>>> >>>>" ) - logger.debug(val) - logger.debug(">>>> >>>> >>>> >>>> >>>> >>>> >>>>") for subpath in search_for_paths( logger, processor, val, terms, pathsep, tmp_path, seen_anchors, search_values=search_values, @@ -603,11 +598,10 @@ def search_for_paths(logger: ConsolePrinter, processor: EYAMLProcessor, expand_children=expand_children ): logger.debug( - ("yaml_paths::search_for_paths:" - + "yielding RECURSED match, {}." - ).format(subpath) + "Yielding RECURSED match, {}.".format(subpath), + prefix="yaml_paths::search_for_paths: ", + footer="<<<< <<<< <<<< <<<< <<<< <<<< <<<<" ) - logger.debug("<<<< <<<< <<<< <<<< <<<< <<<< <<<<") yield subpath elif search_values: if (val_anchor_matched is AnchorMatches.UNSEARCHABLE_ALIAS diff --git a/yamlpath/commands/yaml_set.py b/yamlpath/commands/yaml_set.py index 2301af22..3eb7ed4f 100644 --- a/yamlpath/commands/yaml_set.py +++ b/yamlpath/commands/yaml_set.py @@ -262,7 +262,9 @@ def save_to_yaml_file(args, log, yaml_parser, yaml_data, backup_file): if args.backup: remove(backup_file) - log.debug("Assertion error: {}".format(ex)) + log.debug( + "yaml_set::save_to_yaml_file: Assertion error: {}" + .format(ex)) log.critical(( "Indeterminate assertion error encountered while" + " attempting to write updated data to {}. The original" @@ -379,7 +381,9 @@ def main(): for node_coordinate in processor.get_nodes( change_path, mustexist=must_exist, default_value=("" if new_value else " ")): - log.debug('Got "{}" from {}.'.format(node_coordinate, change_path)) + log.debug( + "Got node from {}:".format(change_path), + data=node_coordinate, prefix="yaml_set::main: ") change_node_coordinates.append(node_coordinate) except YAMLPathException as ex: log.critical(ex, 1) @@ -392,8 +396,9 @@ def main(): old_format = YAMLValueFormats.from_node( change_node_coordinates[0].node) - log.debug("Collected nodes:") - log.debug(change_node_coordinates) + log.debug( + "Collected nodes:", data=change_node_coordinates, + prefix="yaml_set::main: ") # Check the value(s), if desired if args.check: diff --git a/yamlpath/merger/mergerconfig.py b/yamlpath/merger/mergerconfig.py index 093dcfb7..0ef81c3e 100644 --- a/yamlpath/merger/mergerconfig.py +++ b/yamlpath/merger/mergerconfig.py @@ -253,14 +253,8 @@ def _prepare_user_rules( "... RULE: {}".format(merge_rule), prefix="MergerConfig::_prepare_user_rules: ") self.log.debug( - "... NODE:", prefix="MergerConfig::_prepare_user_rules: ", - data=node_coord.node) - self.log.debug( - "... PARENT:", prefix="MergerConfig::_prepare_user_rules: ", - data=node_coord.parent) - self.log.debug( - "... REF:", prefix="MergerConfig::_prepare_user_rules: ", - data=node_coord.parentref) + "... NODE:", data=node_coord, + prefix="MergerConfig::_prepare_user_rules: ") def _load_config(self) -> None: """Load the external configuration file.""" @@ -312,13 +306,7 @@ def _get_rule_for(self, node_coord: NodeCoords) -> str: header=" ") self.log.debug( "... NODE:", prefix="MergerConfig::_get_rule_for: ", - data=node_coord.node) - self.log.debug( - "... PARENT:", prefix="MergerConfig::_get_rule_for: ", - data=node_coord.parent) - self.log.debug( - "... REF:", prefix="MergerConfig::_get_rule_for: ", - data=node_coord.parentref, footer=" ") + data=node_coord) return self._get_config_for(node_coord, self.rules) def _get_key_for(self, node_coord: NodeCoords) -> str: From 1bd1afe11ecbd6e5584390b2c71fab9087f4fa1f Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Tue, 13 Oct 2020 10:11:20 -0500 Subject: [PATCH 4/5] Restore parent-sub-parent traversals --- yamlpath/processor.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/yamlpath/processor.py b/yamlpath/processor.py index d23438e6..cc595494 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -723,7 +723,6 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, # every child against the following segment until there are no more # nodes. For each match, resume normal path function against the # matching node(s). - direct_yielded = False # Because the calling code will continue to process the remainder # of the YAML Path, only the parent of the matched node(s) can be @@ -741,11 +740,6 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, prefix="Processor::_get_nodes_by_traversal: ", data=node_coord) yield NodeCoords(data, parent, parentref) - direct_yielded = True - - # Do not traverse into parents which already match - if direct_yielded: - return # Then, recurse into each child to perform the same test. if isinstance(data, dict): From 4c6ca7f4f813a4c22ed2aba2956619baf31b7d1d Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Tue, 13 Oct 2020 10:35:02 -0500 Subject: [PATCH 5/5] Updated tests to cover new code --- tests/test_wrappers_consoleprinter.py | 12 ++++++++++++ tests/test_wrappers_nodecoords.py | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/tests/test_wrappers_consoleprinter.py b/tests/test_wrappers_consoleprinter.py index 5760593b..1c145fd9 100644 --- a/tests/test_wrappers_consoleprinter.py +++ b/tests/test_wrappers_consoleprinter.py @@ -5,6 +5,7 @@ from ruamel.yaml.comments import CommentedMap from ruamel.yaml.scalarstring import PlainScalarString +from yamlpath.wrappers import NodeCoords from yamlpath.wrappers import ConsolePrinter class Test_wrappers_ConsolePrinter(): @@ -128,6 +129,17 @@ def test_debug_noisy(self, capsys): "DEBUG: test_debug_noisy: === FOOTER ===", ]) + "\n" == console.out + nc = NodeCoords("value", dict(key="value"), "key") + logger.debug( + "A node coordinate:", prefix="test_debug_noisy: ", data=nc) + console = capsys.readouterr() + assert "\n".join([ + "DEBUG: test_debug_noisy: A node coordinate:", + "DEBUG: test_debug_noisy: (node)value", + "DEBUG: test_debug_noisy: (parent)[key]value", + "DEBUG: test_debug_noisy: (parentref)key", + ]) + "\n" == console.out + def test_debug_quiet(self, capsys): args = SimpleNamespace(verbose=False, quiet=True, debug=True) logger = ConsolePrinter(args) diff --git a/tests/test_wrappers_nodecoords.py b/tests/test_wrappers_nodecoords.py index 73882f67..82d2c78d 100644 --- a/tests/test_wrappers_nodecoords.py +++ b/tests/test_wrappers_nodecoords.py @@ -11,3 +11,7 @@ def test_generic(self): def test_repr(self): node_coord = NodeCoords([], None, None) assert repr(node_coord) == "NodeCoords('[]', 'None', 'None')" + + def test_str(self): + node_coord = NodeCoords([], None, None) + assert str(node_coord) == "[]"