fix(json): show original JSON key suggestion, without flattening

src/nitpick/constants.py

@@ -13,9 +13,15 @@
 ROOT_PYTHON_FILES = ("setup.py", "app.py", "wsgi.py", "autoapp.py")
 ROOT_FILES = ("requirements*.txt", "Pipfile") + ROOT_PYTHON_FILES
 
+SINGLE_QUOTE = "'"
+DOUBLE_QUOTE = '"'
+
 #: Special unique separator for :py:meth:`flatten()` and :py:meth:`unflatten()`,
 # to avoid collision with existing key values (e.g. the default dot separator "." can be part of a pyproject.toml key).
-UNIQUE_SEPARATOR = "$#@"
+SEPARATOR_FLATTEN = "$#@"
+
+#: Special unique separator for :py:meth:`quoted_split()`.
+SEPARATOR_QUOTED_SPLIT = "#$@"
 
 # Config sections and keys
 TOOL_NITPICK = "tool.nitpick"

src/nitpick/files/json.py

@@ -52,6 +52,7 @@ def get_suggested_json(self, raw_actual: JsonDict = None) -> JsonDict:
         """Return the suggested JSON based on actual values."""
         actual = set(flatten(raw_actual).keys()) if raw_actual else set()
         expected = set(self.file_dict.get(KEY_CONTAINS_KEYS) or [])
+        # TODO: include "contains_json" keys in the suggestion as well
         missing = expected - actual
         if not missing:
             return {}
@@ -83,4 +84,6 @@ def _check_contained_json(self) -> YieldFlake8Error:
                 LOGGER.error("%s on %s while checking %s", err, KEY_CONTAINS_JSON, self.file_path)
                 continue
 
-        yield from self.warn_missing_different(JsonFormat(data=actual_fmt.as_data).compare_with_dictdiffer(expected))
+        yield from self.warn_missing_different(
+            JsonFormat(data=actual_fmt.as_data).compare_with_dictdiffer(expected, unflatten)
+        )

src/nitpick/formats.py

@@ -5,7 +5,7 @@
 import logging
 from collections import OrderedDict
 from pathlib import Path
-from typing import Any, List, Optional, Type, Union
+from typing import Any, Callable, List, Optional, Type, Union
 
 import dictdiffer
 import toml
@@ -176,7 +176,9 @@ def compare_with_flatten(self, expected: Union[JsonDict, "BaseFormat"] = None) -
         )
         return comparison
 
-    def compare_with_dictdiffer(self, expected: Union[JsonDict, "BaseFormat"] = None) -> Comparison:
+    def compare_with_dictdiffer(
+        self, expected: Union[JsonDict, "BaseFormat"] = None, transform_function: Callable = None
+    ) -> Comparison:
         """Compare two structures and compute missing and different items using ``dictdiffer``."""
         comparison = self._create_comparison(expected)
 
@@ -191,7 +193,8 @@ def compare_with_dictdiffer(self, expected: Union[JsonDict, "BaseFormat"] = None
                 if actual_value != expected_value:
                     comparison.update_pair(key, raw_expected)
 
-        comparison.set_missing(comparison.missing_dict)
+        missing = transform_function(comparison.missing_dict) if transform_function else comparison.missing_dict
+        comparison.set_missing(missing)
         comparison.set_diff(comparison.diff_dict)
         return comparison
 

src/nitpick/generic.py

@@ -5,13 +5,14 @@
     from nitpick.generic import *
 """
 import collections
+import re
 from pathlib import Path
 from typing import Any, Dict, Iterable, List, MutableMapping, Optional, Set, Tuple, Union
 
 import jmespath
 from jmespath.parser import ParsedResult
 
-from nitpick.constants import UNIQUE_SEPARATOR
+from nitpick.constants import DOUBLE_QUOTE, SEPARATOR_FLATTEN, SEPARATOR_QUOTED_SPLIT, SINGLE_QUOTE
 from nitpick.typedefs import JsonDict, PathOrStr
 
 
@@ -43,7 +44,7 @@ def flatten(dict_, parent_key="", separator=".", current_lists=None) -> JsonDict
 
     items = []  # type: List[Tuple[str, Any]]
     for key, value in dict_.items():
-        quoted_key = '"{}"'.format(key) if separator in str(key) else key
+        quoted_key = "{quote}{key}{quote}".format(quote=DOUBLE_QUOTE, key=key) if separator in str(key) else key
         new_key = str(parent_key) + separator + str(quoted_key) if parent_key else quoted_key
         if isinstance(value, collections.abc.MutableMapping):
             items.extend(flatten(value, new_key, separator, current_lists).items())
@@ -59,6 +60,40 @@ def flatten(dict_, parent_key="", separator=".", current_lists=None) -> JsonDict
     return dict(items)
 
 
+def quoted_split(string_: str, separator=".") -> List[str]:
+    """Split a string by a separator, but considering quoted parts (single or double quotes).
+
+    >>> quoted_split("my.key.without.quotes")
+    ['my', 'key', 'without', 'quotes']
+    >>> quoted_split('"double.quoted.string"')
+    ['double.quoted.string']
+    >>> quoted_split('"double.quoted.string".and.after')
+    ['double.quoted.string', 'and', 'after']
+    >>> quoted_split('something.before."double.quoted.string"')
+    ['something', 'before', 'double.quoted.string']
+    >>> quoted_split("'single.quoted.string'")
+    ['single.quoted.string']
+    >>> quoted_split("'single.quoted.string'.and.after")
+    ['single.quoted.string', 'and', 'after']
+    >>> quoted_split("something.before.'single.quoted.string'")
+    ['something', 'before', 'single.quoted.string']
+    """
+    if DOUBLE_QUOTE not in string_ and SINGLE_QUOTE not in string_:
+        return string_.split(separator)
+
+    quoted_regex = re.compile(
+        "([{single}{double}][^{single}{double}]+[{single}{double}])".format(single=SINGLE_QUOTE, double=DOUBLE_QUOTE)
+    )
+
+    def remove_quotes(match):
+        return match.group(0).strip("".join([SINGLE_QUOTE, DOUBLE_QUOTE])).replace(separator, SEPARATOR_QUOTED_SPLIT)
+
+    return [
+        part.replace(SEPARATOR_QUOTED_SPLIT, separator)
+        for part in quoted_regex.sub(remove_quotes, string_).split(separator)
+    ]
+
+
 def unflatten(dict_, separator=".") -> collections.OrderedDict:
     """Turn back a flattened dict created by :py:meth:`flatten()` into a nested dict.
 
@@ -72,7 +107,7 @@ def unflatten(dict_, separator=".") -> collections.OrderedDict:
     """
     items = collections.OrderedDict()  # type: collections.OrderedDict[str, Any]
     for root_key, root_value in sorted(dict_.items()):
-        all_keys = root_key.split(separator)
+        all_keys = quoted_split(root_key, separator)
         sub_items = items
         for key in all_keys[:-1]:
             try:
@@ -96,12 +131,12 @@ def __init__(self, original_dict: JsonDict = None) -> None:
 
     def add(self, other: JsonDict) -> None:
         """Add another dictionary to the existing data."""
-        flattened_other = flatten(other, separator=UNIQUE_SEPARATOR, current_lists=self._current_lists)
+        flattened_other = flatten(other, separator=SEPARATOR_FLATTEN, current_lists=self._current_lists)
         self._all_flattened.update(flattened_other)
 
     def merge(self) -> JsonDict:
         """Merge the dictionaries, replacing values with identical keys and extending lists."""
-        return unflatten(self._all_flattened, separator=UNIQUE_SEPARATOR)
+        return unflatten(self._all_flattened, separator=SEPARATOR_FLATTEN)
 
 
 def climb_directory_tree(starting_path: PathOrStr, file_patterns: Iterable[str]) -> Optional[Set[Path]]:

tests/test_json.py

@@ -58,7 +58,7 @@ def test_missing_different_values(request):
         file_names = ["my.json"]
 
         ["my.json".contains_json]
-        some_root_key = """
+        "some.dotted.root.key" = """
             { "valid": "JSON", "content": ["should", "be", "here"] }
         """
         formatting = """ {"doesnt":"matter","here":true,"on the": "config file"} """
@@ -67,14 +67,18 @@ def test_missing_different_values(request):
         """
         NIP348 File my.json has missing values:\x1b[32m
         {
-          "formatting.doesnt": "matter",
-          "formatting.here": true,
-          "some_root_key.content": [
-            "should",
-            "be",
-            "here"
-          ],
-          "some_root_key.valid": "JSON"
+          "formatting": {
+            "doesnt": "matter",
+            "here": true
+          },
+          "some.dotted.root.key": {
+            "content": [
+              "should",
+              "be",
+              "here"
+            ],
+            "valid": "JSON"
+          }
         }\x1b[0m
         """
         # TODO: check different values on JSON files