Merge branch 'metadata-blame'

bundlewrap/cmdline/items.py

@@ -7,7 +7,7 @@
 
 from ..exceptions import FaultUnavailable
 from ..utils.cmdline import get_item, get_node
-from ..utils.statedict import statedict_to_json
+from ..utils.dicts import statedict_to_json
 from ..utils.text import bold, green, mark_for_translation as _, red, yellow
 from ..utils.ui import io
 

bundlewrap/cmdline/metadata.py

@@ -4,9 +4,10 @@
 from decimal import Decimal
 from json import dumps
 
-from ..metadata import MetadataJSONEncoder, value_at_key_path
+from ..metadata import MetadataJSONEncoder
 from ..utils import Fault
 from ..utils.cmdline import get_node, get_target_nodes
+from ..utils.dicts import value_at_key_path
 from ..utils.table import ROW_SEPARATOR, render_table
 from ..utils.text import bold, force_text, mark_for_translation as _, red
 from ..utils.ui import io, page_lines
@@ -39,10 +40,16 @@ def bw_metadata(repo, args):
         page_lines(render_table(table))
     else:
         node = get_node(repo, args['target'], adhoc_nodes=args['adhoc_nodes'])
-        for line in dumps(
-            value_at_key_path(node.metadata, args['keys']),
-            cls=MetadataJSONEncoder,
-            indent=4,
-            sort_keys=True,
-        ).splitlines():
-            io.stdout(force_text(line))
+        if args['blame']:
+            table = [[bold(_("path")), bold(_("source"))], ROW_SEPARATOR]
+            for path, blamed in sorted(node.metadata_blame.items()):
+                table.append([" ".join(path), ", ".join(blamed)])
+            page_lines(render_table(table))
+        else:
+            for line in dumps(
+                value_at_key_path(node.metadata, args['keys']),
+                cls=MetadataJSONEncoder,
+                indent=4,
+                sort_keys=True,
+            ).splitlines():
+                io.stdout(force_text(line))

bundlewrap/cmdline/parser.py

@@ -453,6 +453,12 @@ def build_parser_bw():
         type=str,
         help=_("print only partial metadata from the given space-separated key path (e.g. `bw metadata mynode users jdoe` to show `mynode.metadata['users']['jdoe']`)"),
     )
+    parser_metadata.add_argument(
+        "--blame",
+        action='store_true',
+        dest='blame',
+        help=_("show where each piece of metadata comes from"),
+    )
     parser_metadata.add_argument(
         "-t",
         "--table",
@@ -762,8 +768,8 @@ def build_parser_bw():
                   "If *any* options other than -i are given, *only* the "
                   "tests selected by those options will be run. Otherwise, a "
                   "default selection of tests will be run (that selection may "
-                  "change in future releases). Currently, the default is -IJM "
-                  "if specific nodes are given and -HIJMS if testing the "
+                  "change in future releases). Currently, the default is -IJKM "
+                  "if specific nodes are given and -HIJKMS if testing the "
                   "entire repo.")
     parser_test = subparsers.add_parser("test", description=help_test, help=help_test)
     parser_test.set_defaults(func=bw_test)
@@ -827,6 +833,13 @@ def build_parser_bw():
         dest='hooks_node',
         help=_("run node-level test hooks"),
     )
+    parser_test.add_argument(
+        "-K",
+        "--metadata-keys",
+        action='store_true',
+        dest='metadata_keys',
+        help=_("validate metadata keys"),
+    )
     parser_test.add_argument(
         "-m",
         "--metadata-determinism",

bundlewrap/cmdline/test.py

@@ -8,7 +8,7 @@
 from ..deps import DummyItem
 from ..exceptions import FaultUnavailable, ItemDependencyLoop
 from ..itemqueue import ItemTestQueue
-from ..metadata import check_for_unsolvable_metadata_key_conflicts
+from ..metadata import check_for_unsolvable_metadata_key_conflicts, check_metadata_keys
 from ..plugins import PluginManager
 from ..repo import Repository
 from ..utils.cmdline import count_items, get_target_nodes
@@ -103,6 +103,15 @@ def test_metadata_collisions(node):
     ))
 
 
+def test_metadata_keys(node):
+    with io.job(_("{node}  checking metadata keys").format(node=bold(node.name))):
+        check_metadata_keys(node)
+    io.stdout(_("{x} {node}  has valid metadata keys").format(
+        x=green("✓"),
+        node=bold(node.name),
+    ))
+
+
 def test_orphaned_bundles(repo):
     orphaned_bundles = set(repo.bundle_names)
     for node in repo.nodes:
@@ -285,6 +294,7 @@ def bw_test(repo, args):
         args['hooks_repo'] or
         args['ignore_secret_identifiers'] is not None or
         args['items'] or
+        args['metadata_keys'] or
         args['metadata_collisions'] or
         args['orphaned_bundles'] or
         args['empty_groups'] or
@@ -297,13 +307,15 @@ def bw_test(repo, args):
             args['hooks_node'] = True
             args['items'] = True
             args['metadata_collisions'] = True
+            args['metadata_keys'] = True
     else:
         nodes = copy(list(repo.nodes))
         if not options_selected:
             args['hooks_node'] = True
             args['hooks_repo'] = True
             args['items'] = True
             args['metadata_collisions'] = True
+            args['metadata_keys'] = True
             args['subgroup_loops'] = True
 
     if args['ignore_secret_identifiers'] is not None and not QUIT_EVENT.is_set():
@@ -321,6 +333,15 @@ def bw_test(repo, args):
     if args['orphaned_bundles'] and not QUIT_EVENT.is_set():
         test_orphaned_bundles(repo)
 
+    if args['metadata_keys'] and not QUIT_EVENT.is_set():
+        io.progress_set_total(len(nodes))
+        for node in nodes:
+            if QUIT_EVENT.is_set():
+                break
+            test_metadata_keys(node)
+            io.progress_advance()
+        io.progress_set_total(0)
+
     if args['metadata_collisions'] and not QUIT_EVENT.is_set():
         io.progress_set_total(len(nodes))
         for node in nodes:

bundlewrap/group.py

@@ -5,7 +5,7 @@
 
 from .exceptions import NoSuchGroup, NoSuchNode, RepositoryError
 from .utils import cached_property, names
-from .utils.statedict import hash_statedict
+from .utils.dicts import hash_statedict
 from .utils.text import mark_for_translation as _, validate_name
 
 

bundlewrap/items/__init__.py

@@ -12,7 +12,7 @@
 
 from bundlewrap.exceptions import BundleError, ItemDependencyError, FaultUnavailable
 from bundlewrap.utils import cached_property
-from bundlewrap.utils.statedict import diff_keys, diff_value, hash_statedict, validate_statedict
+from bundlewrap.utils.dicts import diff_keys, diff_value, hash_statedict, validate_statedict
 from bundlewrap.utils.text import force_text, mark_for_translation as _
 from bundlewrap.utils.text import blue, bold, italic, wrap_question
 from bundlewrap.utils.ui import io

bundlewrap/metadata.py

@@ -6,7 +6,8 @@
 from json import dumps, JSONEncoder
 
 from .exceptions import RepositoryError
-from .utils import ATOMIC_TYPES, Fault, merge_dict
+from .utils import Fault
+from .utils.dicts import ATOMIC_TYPES, map_dict_keys, merge_dict, value_at_key_path
 from .utils.text import force_text, mark_for_translation as _
 
 
@@ -48,6 +49,50 @@ def atomic(obj):
         return cls(obj)
 
 
+def blame_changed_paths(old_dict, new_dict, blame_dict, blame_name, defaults=False):
+    def is_mergeable(value1, value2):
+        if isinstance(value1, (list, set, tuple)) and isinstance(value2, (list, set, tuple)):
+            return True
+        elif isinstance(value1, dict) and isinstance(value2, dict):
+            return True
+        return False
+
+    new_paths = map_dict_keys(new_dict)
+
+    # clean up removed paths from blame_dict
+    for path in list(blame_dict.keys()):
+        if path not in new_paths:
+            del blame_dict[path]
+
+    for path in new_paths:
+        new_value = value_at_key_path(new_dict, path)
+        try:
+            old_value = value_at_key_path(old_dict, path)
+        except KeyError:
+            blame_dict[path] = (blame_name,)
+        else:
+            if old_value != new_value:
+                if defaults or is_mergeable(old_value, new_value):
+                    blame_dict[path] += (blame_name,)
+                else:
+                    blame_dict[path] = (blame_name,)
+    return blame_dict
+
+
+def check_metadata_keys(node):
+    try:
+        basestring
+    except NameError:  # Python 2
+        basestring = str
+    for path in map_dict_keys(node.metadata):
+        value = path[-1]
+        if not isinstance(value, basestring):
+            raise TypeError(_("metadata key for {node} at path '{path}' is not a string").format(
+                node=node.name,
+                path="'->'".join(path[:-1]),
+            ))
+
+
 def check_metadata_processor_result(result, node_name, metadata_processor_name):
     """
     Validates the return value of a metadata processor and splits it
@@ -181,7 +226,7 @@ def check_for_unsolvable_metadata_key_conflicts(node):
         chain_metadata.append(metadata)
 
     # create a "key path map" for each chain's metadata
-    chain_metadata_keys = [list(dictionary_key_map(metadata)) for metadata in chain_metadata]
+    chain_metadata_keys = [list(map_dict_keys(metadata)) for metadata in chain_metadata]
 
     # compare all metadata keys with other chains and find matches
     for index1, keymap1 in enumerate(chain_metadata_keys):
@@ -241,42 +286,13 @@ def deepcopy_metadata(obj):
     return new_obj
 
 
-def dictionary_key_map(mapdict):
-    """
-    For the dict
-
-        {
-            "key1": 1,
-            "key2": {
-                "key3": 3,
-                "key4": ["foo"],
-             },
-        }
-
-    the key map would look like this:
-
-        [
-            ("key1",),
-            ("key2",),
-            ("key2", "key3"),
-            ("key2", "key4"),
-        ]
-
-    """
-    for key, value in mapdict.items():
-        if isinstance(value, dict):
-            for child_keys in dictionary_key_map(value):
-                yield (key,) + child_keys
-        yield (key,)
-
-
 def find_groups_causing_metadata_conflict(node_name, chain1, chain2, keypath):
     """
     Given two chains (lists of groups), find one group in each chain
     that has conflicting metadata with the other for the given key path.
     """
-    chain1_metadata = [list(dictionary_key_map(group.metadata)) for group in chain1]
-    chain2_metadata = [list(dictionary_key_map(group.metadata)) for group in chain2]
+    chain1_metadata = [list(map_dict_keys(group.metadata)) for group in chain1]
+    chain2_metadata = [list(map_dict_keys(group.metadata)) for group in chain2]
 
     bad_keypath = None
 
@@ -332,19 +348,3 @@ def hash_metadata(sdict):
         indent=None,
         sort_keys=True,
     ).encode('utf-8')).hexdigest()
-
-
-def value_at_key_path(dict_obj, path):
-    """
-    Given the list of keys in `path`, recursively traverse `dict_obj`
-    and return whatever is found at the end of that path.
-
-    E.g.:
-
-    >>> value_at_key_path({'foo': {'bar': 5}}, ['foo', 'bar'])
-    5
-    """
-    if not path:
-        return dict_obj
-    else:
-        return value_at_key_path(dict_obj[path[0]], path[1:])

bundlewrap/node.py

@@ -26,7 +26,7 @@
 from .lock import NodeLock
 from .metadata import hash_metadata
 from .utils import cached_property, names
-from .utils.statedict import hash_statedict
+from .utils.dicts import hash_statedict
 from .utils.text import (
     blue,
     bold,
@@ -654,6 +654,10 @@ def metadata(self):
         else:
             return self.repo._metadata_for_node(self.name, partial=False)
 
+    @property
+    def metadata_blame(self):
+        return self.repo._metadata_for_node(self.name, partial=False, blame=True)
+
     def metadata_hash(self):
         return hash_metadata(self.metadata)