refactor: replace most of DictBlender by flatten/unflatten

docs/ideas/lab.py

@@ -7,8 +7,7 @@
 import jmespath
 from identify import identify
 
-from nitpick.blender import DictBlender, TomlDoc, YamlDoc, search_json
-from nitpick.constants import DOT
+from nitpick.blender import TomlDoc, YamlDoc, flatten_quotes, search_json
 
 workflow = YamlDoc(path=Path(".github/workflows/python.yaml"))
 
@@ -56,7 +55,7 @@ def main():
         print(json.dumps(config, indent=2))
 
         click.secho("Flattened JSON", fg="yellow")
-        print(json.dumps(DictBlender(config, separator=DOT).flat_dict, indent=2))
+        print(json.dumps(flatten_quotes(config), indent=2))
 
     # find("jobs.build")
     # find("jobs.build.strategy.matrix")

src/nitpick/blender.py

@@ -7,7 +7,8 @@
 import abc
 import json
 import re
-from collections import OrderedDict
+import shlex
+from collections import OrderedDict  # FIXME: remove ordereddict?
 from functools import lru_cache, partial
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, Union, cast
@@ -18,6 +19,7 @@
 import tomlkit
 from attrs import define
 from autorepr import autorepr
+from flatten_dict import flatten, unflatten
 from jmespath.parser import ParsedResult
 from ruamel.yaml import YAML, RoundTripRepresenter, StringIO
 from sortedcontainers import SortedDict
@@ -32,6 +34,7 @@
 SEPARATOR_DOT = "."
 SEPARATOR_COMMA = ","
 SEPARATOR_COLON = ":"
+SEPARATOR_SPACE = " "
 
 #: Special unique separator for :py:meth:`flatten()` and :py:meth:`unflatten()`,
 # to avoid collision with existing key values (e.g. the default SEPARATOR_DOT separator "." can be part of a TOML key).
@@ -192,7 +195,49 @@ def remove_quotes(match):
     ]
 
 
-class DictBlender:
+def quote_if_dotted(key: str) -> str:
+    """Quote the key if it has a dot."""
+    if not isinstance(key, str):
+        return key
+    if SEPARATOR_DOT in key and DOUBLE_QUOTE not in key:
+        return f"{DOUBLE_QUOTE}{key}{DOUBLE_QUOTE}"
+    return key
+
+
+def quote_reducer(key1: Optional[str], key2: str) -> str:
+    """Reducer used to unflatten dicts. Quote keys when they have dots."""
+    if key1 is None:
+        return quote_if_dotted(key2)
+    return f"{key1}{SEPARATOR_DOT}{quote_if_dotted(key2)}"
+
+
+def quotes_splitter(flat_key: str) -> Tuple[str, ...]:
+    """Split keys keeping quotes strings together."""
+    return tuple(
+        piece.replace(SEPARATOR_SPACE, SEPARATOR_DOT) if SEPARATOR_SPACE in piece else piece
+        for piece in shlex.split(flat_key.replace(SEPARATOR_DOT, SEPARATOR_SPACE))
+    )
+
+
+def flatten_quotes(dict_: JsonDict) -> JsonDict:
+    """Flatten a dict keeping quotes in keys."""
+    dict_with_quoted_keys = flatten(dict_, reducer=quote_reducer)
+    clean_dict = {}
+    for key, value in dict_with_quoted_keys.items():  # type: str, Any
+        key_with_stripped_ends = key.strip(DOUBLE_QUOTE)
+        if key_with_stripped_ends.count(DOUBLE_QUOTE):
+            # Key has quotes in the middle; keep all quotes
+            clean_dict[key] = value
+        else:
+            # Key only has quotes in the beginning and end; remove quotes
+            clean_dict[key_with_stripped_ends] = value
+    return clean_dict
+
+
+unflatten_quotes = partial(unflatten, splitter=quotes_splitter)
+
+
+class DictBlender:  # FIXME: remove all references
     """A blender of dictionaries: keep adding dictionaries and mix them all at the end.
 
     .. note::
@@ -279,15 +324,12 @@ def mix(self, sort=True) -> JsonDict:
         return self._unflatten(self._current_flat_dict, sort)
 
 
-DotBlender = partial(DictBlender, separator=SEPARATOR_DOT, flatten_on_add=False)
-
-
 class Comparison:
     """A comparison between two dictionaries, computing missing items and differences."""
 
     def __init__(self, actual: "BaseDoc", expected: JsonDict, special_config: SpecialConfig) -> None:
-        self.flat_actual = DictBlender(actual.as_object, separator=SEPARATOR_DOT).flat_dict
-        self.flat_expected = DictBlender(expected, separator=SEPARATOR_DOT).flat_dict
+        self.flat_actual = flatten_quotes(actual.as_object)
+        self.flat_expected = flatten_quotes(expected)
 
         self.doc_class = actual.__class__
 
@@ -302,21 +344,21 @@ def missing(self) -> Optional["BaseDoc"]:
         """Missing data."""
         if not self.missing_dict:
             return None
-        return self.doc_class(obj=DotBlender(self.missing_dict).mix())
+        return self.doc_class(obj=(unflatten_quotes(self.missing_dict)))
 
     @property
     def diff(self) -> Optional["BaseDoc"]:
         """Different data."""
         if not self.diff_dict:
             return None
-        return self.doc_class(obj=DotBlender(self.diff_dict).mix())
+        return self.doc_class(obj=(unflatten_quotes(self.diff_dict)))
 
     @property
     def replace(self) -> Optional["BaseDoc"]:
         """Data to be replaced."""
         if not self.replace_dict:
             return None
-        return self.doc_class(obj=DotBlender(self.replace_dict).mix())
+        return self.doc_class(obj=unflatten_quotes(self.replace_dict))
 
     @property
     def has_changes(self) -> bool:
@@ -634,7 +676,7 @@ def load(self) -> bool:
         if self.path is not None:
             self._string = Path(self.path).read_text(encoding="UTF-8")
         if self._string is not None:
-            self._object = json.loads(self._string, object_pairs_hook=OrderedDict)
+            self._object = flatten_quotes(json.loads(self._string, object_pairs_hook=OrderedDict))
         if self._object is not None:
             # Every file should end with a blank line
             self._reformatted = json.dumps(self._object, sort_keys=True, indent=2) + "\n"

src/nitpick/plugins/base.py

@@ -9,7 +9,7 @@
 from loguru import logger
 from marshmallow import Schema
 
-from nitpick.blender import SEPARATOR_DOT, BaseDoc, DictBlender, search_json
+from nitpick.blender import BaseDoc, flatten_quotes, search_json
 from nitpick.config import SpecialConfig
 from nitpick.constants import DUNDER_LIST_KEYS
 from nitpick.plugins.info import FileInfo
@@ -66,9 +66,9 @@ def _merge_special_configs(self):
         # The user can override the default list keys (if any) by setting them on the style file.
         # pylint: disable=assigning-non-slot,no-member
         spc.list_keys.from_style = self.expected_config.pop(DUNDER_LIST_KEYS, None) or {}
-        temp_dict.update(DictBlender(spc.list_keys.from_style, separator=SEPARATOR_DOT).flat_dict)
+        temp_dict.update(flatten_quotes(spc.list_keys.from_style))
 
-        flat_config = DictBlender(self.expected_config, separator=SEPARATOR_DOT).flat_dict
+        flat_config = flatten_quotes(self.expected_config)
 
         for key_with_pattern, parent_child_keys in temp_dict.items():
             for expanded_key in fnmatch.filter(flat_config.keys(), key_with_pattern):

src/nitpick/plugins/json.py

@@ -6,12 +6,12 @@
 from loguru import logger
 
 from nitpick import fields
-from nitpick.blender import BaseDoc, Comparison, DictBlender, JsonDoc
-from nitpick.constants import DOT
+from nitpick.blender import BaseDoc, Comparison, JsonDoc, flatten_quotes, unflatten_quotes
 from nitpick.plugins import hookimpl
 from nitpick.plugins.base import NitpickPlugin
 from nitpick.plugins.info import FileInfo
 from nitpick.schemas import BaseNitpickSchema
+from nitpick.typedefs import JsonDict
 from nitpick.violations import Fuss, SharedViolations, ViolationEnum
 
 KEY_CONTAINS_KEYS = "contains_keys"
@@ -41,7 +41,7 @@ class JsonPlugin(NitpickPlugin):
     def enforce_rules(self) -> Iterator[Fuss]:
         """Enforce rules for missing keys and JSON content."""
         json_doc = JsonDoc(path=self.file_path)
-        blender: Optional[DictBlender] = DictBlender(json_doc.as_object, extend_lists=False) if self.autofix else None
+        blender: JsonDict = json_doc.as_object.copy() if self.autofix else {}
 
         comparison = Comparison(json_doc, self.expected_dict_from_contains_keys(), self.special_config)()
         if comparison.missing:
@@ -55,13 +55,13 @@ def enforce_rules(self) -> Iterator[Fuss]:
             )
 
         if self.autofix and self.dirty and blender:
-            self.file_path.write_text(JsonDoc(obj=blender.mix()).reformatted)
+            self.file_path.write_text(JsonDoc(obj=unflatten_quotes(blender)).reformatted)
 
     def expected_dict_from_contains_keys(self):
         """Expected dict created from "contains_keys" values."""
-        blender = DictBlender(separator=DOT, flatten_on_add=False)
-        blender.add({key: VALUE_PLACEHOLDER for key in set(self.expected_config.get(KEY_CONTAINS_KEYS) or [])})
-        return blender.mix()
+        return unflatten_quotes(
+            {key: VALUE_PLACEHOLDER for key in set(self.expected_config.get(KEY_CONTAINS_KEYS) or [])}
+        )
 
     def expected_dict_from_contains_json(self):
         """Expected dict created from "contains_json" values."""
@@ -77,21 +77,21 @@ def expected_dict_from_contains_json(self):
                 continue
         return expected_config
 
-    def report(self, violation: ViolationEnum, blender: Optional[DictBlender], change: Optional[BaseDoc]):
+    def report(self, violation: ViolationEnum, blender: JsonDict, change: Optional[BaseDoc]):
         """Report a violation while optionally modifying the JSON dict."""
         if not change:
             return
         if blender:
-            blender.add(change.as_object)
+            blender.update(flatten_quotes(change.as_object))
             self.dirty = True
         yield self.reporter.make_fuss(violation, change.reformatted, prefix="", fixed=self.autofix)
 
     @property
     def initial_contents(self) -> str:
         """Suggest the initial content for this missing file."""
-        suggestion = DictBlender(self.expected_dict_from_contains_keys())
-        suggestion.add(self.expected_dict_from_contains_json())
-        return self.write_initial_contents(JsonDoc, suggestion.mix())
+        suggestion = flatten_quotes(self.expected_dict_from_contains_keys())
+        suggestion.update(flatten_quotes(self.expected_dict_from_contains_json()))
+        return self.write_initial_contents(JsonDoc, unflatten_quotes(suggestion))
 
 
 @hookimpl

src/nitpick/schemas.py

@@ -6,14 +6,14 @@
 from sortedcontainers import SortedDict
 
 from nitpick import fields
-from nitpick.blender import DictBlender
-from nitpick.constants import DOT, READ_THE_DOCS_URL, SETUP_CFG
+from nitpick.blender import flatten_quotes
+from nitpick.constants import READ_THE_DOCS_URL, SETUP_CFG
 
 
 def flatten_marshmallow_errors(errors: Dict) -> str:
     """Flatten Marshmallow errors to a string."""
     formatted = []
-    for field, data in SortedDict(DictBlender(errors, separator=DOT).flat_dict).items():
+    for field, data in SortedDict(flatten_quotes(errors)).items():
         if isinstance(data, (list, tuple)):
             messages_per_field = [f"{field}: {', '.join(data)}"]
         else:

tests/test_json.py

@@ -25,7 +25,7 @@ def test_suggest_initial_contents(tmp_path, datadir):
         )
     ).assert_file_contents(
         PACKAGE_JSON, expected_package_json
-    )
+    ).api_check_then_fix()
 
 
 def test_missing_different_values_with_contains_json_with_contains_keys(tmp_path, datadir):
@@ -71,7 +71,7 @@ def test_missing_different_values_with_contains_json_with_contains_keys(tmp_path
         ),
     ).assert_file_contents(
         PACKAGE_JSON, expected_package_json
-    )
+    ).api_check_then_fix()
 
 
 def test_missing_different_values_with_contains_json_without_contains_keys(tmp_path, datadir):
@@ -125,7 +125,7 @@ def test_missing_different_values_with_contains_json_without_contains_keys(tmp_p
         ),
     ).assert_file_contents(
         "my.json", datadir / "3-expected.json"
-    )
+    ).api_check_then_fix()
 
 
 def test_invalid_json(tmp_path, datadir):