From 1134de4f76201936e121a9592bc04cf69ef5bcac Mon Sep 17 00:00:00 2001 From: Wilhelm Schuermann Date: Fri, 12 Feb 2021 20:54:40 +0100 Subject: [PATCH 1/6] Fix .yml not being recognized by deep CLI --- deepdiff/serialization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deepdiff/serialization.py b/deepdiff/serialization.py index 4096387d..0c1ba81e 100644 --- a/deepdiff/serialization.py +++ b/deepdiff/serialization.py @@ -364,7 +364,7 @@ def load_path_content(path, file_type=None): if file_type == 'json': with open(path, 'r') as the_file: content = json.load(the_file) - elif file_type in {'yaml', '.yml'}: + elif file_type in {'yaml', 'yml'}: if yaml is None: # pragma: no cover. raise ImportError('Pyyaml needs to be installed.') # pragma: no cover. with open(path, 'r') as the_file: @@ -426,7 +426,7 @@ def _save_content(content, path, file_type, keep_backup=True): if file_type == 'json': with open(path, 'w') as the_file: content = json.dump(content, the_file) - elif file_type in {'yaml', '.yml'}: + elif file_type in {'yaml', 'yml'}: if yaml is None: # pragma: no cover. raise ImportError('Pyyaml needs to be installed.') # pragma: no cover. with open(path, 'w') as the_file: From edff514db1e42eff1e1a83a86998c0bccbc5fd11 Mon Sep 17 00:00:00 2001 From: Tim Klein Date: Fri, 12 Feb 2021 14:51:33 -0500 Subject: [PATCH 2/6] Retain the order of multiple dictionary items added via Delta - This alters the data structure used to determine dictionary items added (via storing the keys from `t1` and `t2` in OrderedSet objects) - This also adds a unit test to verify that after applying a Delta with multiple dict items added, the added keys are not sorted, but are instead added to the resulting dictionary in the same insertion order as their source object (`t2`) --- deepdiff/delta.py | 15 +++++++++++---- deepdiff/diff.py | 12 ++++++------ tests/test_delta.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/deepdiff/delta.py b/deepdiff/delta.py index 81e34e7c..9e6fd78c 100644 --- a/deepdiff/delta.py +++ b/deepdiff/delta.py @@ -267,16 +267,23 @@ def _do_iterable_item_added(self): def _do_dictionary_item_added(self): dictionary_item_added = self.diff.get('dictionary_item_added') if dictionary_item_added: - self._do_item_added(dictionary_item_added) + self._do_item_added(dictionary_item_added, sort=False) def _do_attribute_added(self): attribute_added = self.diff.get('attribute_added') if attribute_added: self._do_item_added(attribute_added) - def _do_item_added(self, items): - # sorting the items by their path so that the items with smaller index are applied first. - for path, new_value in sorted(items.items(), key=lambda x: x[0]): + def _do_item_added(self, items, sort=True): + if sort: + # sorting items by their path so that the items with smaller index + # are applied first (unless `sort` is `False` so that order of + # added items is retained, e.g. for dicts). + items = sorted(items.items(), key=lambda x: x[0]) + else: + items = items.items() + + for path, new_value in items: elem_and_details = self._get_elements_and_details(path) if elem_and_details: elements, parent, parent_to_obj_elem, parent_to_obj_action, obj, elem, action = elem_and_details diff --git a/deepdiff/diff.py b/deepdiff/diff.py index 133661b8..e3eb1e68 100755 --- a/deepdiff/diff.py +++ b/deepdiff/diff.py @@ -450,16 +450,16 @@ def _diff_dict(self, rel_class = DictRelationship if self.ignore_private_variables: - t1_keys = {key for key in t1 if not(isinstance(key, str) and key.startswith('__'))} - t2_keys = {key for key in t2 if not(isinstance(key, str) and key.startswith('__'))} + t1_keys = OrderedSet([key for key in t1 if not(isinstance(key, str) and key.startswith('__'))]) + t2_keys = OrderedSet([key for key in t2 if not(isinstance(key, str) and key.startswith('__'))]) else: - t1_keys = set(t1.keys()) - t2_keys = set(t2.keys()) + t1_keys = OrderedSet(t1.keys()) + t2_keys = OrderedSet(t2.keys()) if self.ignore_string_type_changes or self.ignore_numeric_type_changes: t1_clean_to_keys = self._get_clean_to_keys_mapping(keys=t1_keys, level=level) t2_clean_to_keys = self._get_clean_to_keys_mapping(keys=t2_keys, level=level) - t1_keys = set(t1_clean_to_keys.keys()) - t2_keys = set(t2_clean_to_keys.keys()) + t1_keys = OrderedSet(t1_clean_to_keys.keys()) + t2_keys = OrderedSet(t2_clean_to_keys.keys()) else: t1_clean_to_keys = t2_clean_to_keys = None diff --git a/tests/test_delta.py b/tests/test_delta.py index e91a6463..af5051f9 100644 --- a/tests/test_delta.py +++ b/tests/test_delta.py @@ -335,6 +335,41 @@ def test_list_difference_delta_raises_error_if_prev_value_changed(self): delta2 = Delta(diff, verify_symmetry=False, raise_errors=True) assert t1 + delta2 == t2 + def test_delta_dict_items_added_retain_order(self): + t1 = { + 6: 6 + } + + t2 = { + 6: 6, + 7: 7, + 3: 3, + 5: 5, + 2: 2, + 4: 4 + } + + expected_delta_dict = { + 'dictionary_item_added': { + 'root[7]': 7, + 'root[3]': 3, + 'root[5]': 5, + 'root[2]': 2, + 'root[4]': 4 + } + } + + diff = DeepDiff(t1, t2) + delta_dict = diff._to_delta_dict() + assert expected_delta_dict == delta_dict + delta = Delta(diff, verify_symmetry=False, raise_errors=True) + + result = t1 + delta + assert result == t2 + + assert list(result.keys()) == [6, 7, 3, 5, 2, 4] + assert list(result.keys()) == list(t2.keys()) + picklalbe_obj_without_item = PicklableClass(11) del picklalbe_obj_without_item.item From bbd7c0879d43c2ac1bfb907bed3dc0a2de837552 Mon Sep 17 00:00:00 2001 From: Seperman Date: Mon, 15 Feb 2021 19:01:24 -0800 Subject: [PATCH 3/6] updating authors --- AUTHORS.md | 2 ++ docs/authors.rst | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/AUTHORS.md b/AUTHORS.md index 3fe50f70..4eeb0a88 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -33,3 +33,5 @@ Authors in order of the contributions: - [MyrikLD](https://github.com/MyrikLD) for Bug Fix NoneType in ignore type groups - Stian Jensen [stianjensen](https://github.com/stianjensen) for improving ignoring of NoneType in diff - Florian Klien [flowolf](https://github.com/flowolf) for adding math_epsilon +- Tim Klein [timjklein36](https://github.com/timjklein36) for retaining the order of multiple dictionary items added via Delta. +- Wilhelm Schürmann[wbsch](https://github.com/wbsch) for fixing the typo with yml files. diff --git a/docs/authors.rst b/docs/authors.rst index 847701a1..ee86f776 100644 --- a/docs/authors.rst +++ b/docs/authors.rst @@ -40,6 +40,8 @@ Thanks to the following people for their contributions: - `MyrikLD`_ for Bug Fix NoneType in ignore type groups - Stian Jensen `stianjensen`_ for improving ignoring of NoneType in diff - Florian Klien `flowolf`_ for adding math_epsilon +- Tim Klein `timjklein36`_ for retaining the order of multiple dictionary items added via Delta +- Wilhelm Schürmann `wbsch`_ for fixing the typo with yml files. .. _Sep Dehpour (Seperman): http://www.zepworks.com .. _Victor Hahn Castell: http://hahncastell.de @@ -70,6 +72,8 @@ Thanks to the following people for their contributions: .. _MyrikLD: https://github.com/MyrikLD .. _stianjensen: https://github.com/stianjensen .. _flowolf: https://github.com/flowolf +.. _timjklein36: https://github.com/timjklein36 +.. _wbsch: https://github.com/wbsch Back to :doc:`/index` From 1e1df14a37fbebe4bf07ae16480cf1a18bf37106 Mon Sep 17 00:00:00 2001 From: Seperman Date: Tue, 16 Feb 2021 11:50:51 -0800 Subject: [PATCH 4/6] fixing RecursionError where using non UTF-8 character #227 --- CHANGELOG.md | 3 ++- deepdiff/search.py | 5 ++--- docs/changelog.rst | 1 + tests/test_search.py | 31 +++++++++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e144e6fa..a020156d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ -DeepDiff Change log +# DeepDiff Change log +- v5-2-3: Retaining the order of multiple dictionary items added via Delta. Fixed the typo with yml files in deep cli. Fixing Grep RecursionError where using non UTF-8 character. - v5-2-2: Fixed Delta serialization when None type is present. - v5-2-0: Removed Murmur3 as the preferred hashing method. Using SHA256 by default now. Added commandline for deepdiff. Added group_by. Added math_epsilon. Improved ignoring of NoneType. - v5-0-2: Bug Fix NoneType in ignore type groups https://github.com/seperman/deepdiff/issues/207 diff --git a/deepdiff/search.py b/deepdiff/search.py index dc5ddff7..4226a6a4 100644 --- a/deepdiff/search.py +++ b/deepdiff/search.py @@ -223,7 +223,6 @@ def __search_iterable(self, parent="root", parents_ids=frozenset()): """Search iterables except dictionaries, sets and strings.""" - for i, thing in enumerate(obj): new_parent = "{}[{}]".format(parent, i) if self.__skip_this(thing, parent=new_parent): @@ -271,7 +270,7 @@ def __search_tuple(self, obj, item, parent, parents_ids): def __search(self, obj, item, parent="root", parents_ids=frozenset()): """The main search method""" - + # import pytest; pytest.set_trace() if self.__skip_this(item, parent): return @@ -299,7 +298,7 @@ def __search(self, obj, item, parent="root", parents_ids=frozenset()): self.warning_num += 1 self.__search_iterable(obj, item, parent, parents_ids) - elif isinstance(obj, Iterable): + elif isinstance(obj, Iterable) and not isinstance(obj, strings): self.__search_iterable(obj, item, parent, parents_ids) else: diff --git a/docs/changelog.rst b/docs/changelog.rst index 61350720..1d61ccdd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog DeepDiff Changelog +- v5-2-3: Retaining the order of multiple dictionary items added via Delta. Fixed the typo with yml files in deep cli. Fixing Grep RecursionError where using non UTF-8 character. - v5-2-2: Fixed Delta serialization when None type is present. - v5-2-0: Removed Murmur3 as the preferred hashing method. Using SHA256 by default now. Added commandline for deepdiff. Added group_by. Added math_epsilon. Improved ignoring of NoneType. - v5-0-2: Bug Fix NoneType in ignore type groups https://github.com/seperman/deepdiff/issues/207 diff --git a/tests/test_search.py b/tests/test_search.py index d209bbd2..247e648e 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -29,6 +29,18 @@ def test_number_in_list(self): result = {"matched_values": {'root[1]'}} assert DeepSearch(obj, item, verbose_level=1) == result + def test_number_in_list2(self): + obj = ["a", "10", 10, 20] + item = 10 + result = {"matched_values": {'root[2]'}} + assert DeepSearch(obj, item, verbose_level=1) == result + + def test_number_in_list3(self): + obj = ["a", "10", 10, 20] + item = "10" + result = {"matched_values": {'root[1]'}} + assert DeepSearch(obj, item, verbose_level=1) == result + def test_string_in_root(self): obj = "long string somewhere" result = {"matched_values": {'root'}} @@ -334,3 +346,22 @@ def test_grep_dict(self): } ds = obj | grep(item) assert ds == {'matched_values': {"root['ingredients'][3]"}} + + def test_grep_dict_in_dict(self): + obj = { + "x": { + "y": [ + "aaaaaa\u0142 bbbbb" + ] + }, + "z": "z", + } + item = {"z": "z"} + result = obj | grep(item) + assert {} == result + + def test_grep_with_non_utf8_chars(self): + obj = "aaaaaa\u0142 bbbbb" + item = {"z": "z"} + result = obj | grep(item) + assert {} == result From 9b4fd51a733131a5270159ed588cdff5890c866e Mon Sep 17 00:00:00 2001 From: Seperman Date: Tue, 16 Feb 2021 11:56:45 -0800 Subject: [PATCH 5/6] Pass `json.dumps` parameters in `DeepDiff.to_json` #226 --- .github/FUNDING.yml | 2 ++ CHANGELOG.md | 2 +- deepdiff/serialization.py | 6 ++++-- docs/changelog.rst | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..8b6fb519 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,2 @@ +github: [seperman] +ko_fi: seperman diff --git a/CHANGELOG.md b/CHANGELOG.md index a020156d..5203babc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # DeepDiff Change log -- v5-2-3: Retaining the order of multiple dictionary items added via Delta. Fixed the typo with yml files in deep cli. Fixing Grep RecursionError where using non UTF-8 character. +- v5-2-3: Retaining the order of multiple dictionary items added via Delta. Fixed the typo with yml files in deep cli. Fixing Grep RecursionError where using non UTF-8 character. Allowing kwargs to be passed to to_json method. - v5-2-2: Fixed Delta serialization when None type is present. - v5-2-0: Removed Murmur3 as the preferred hashing method. Using SHA256 by default now. Added commandline for deepdiff. Added group_by. Added math_epsilon. Improved ignoring of NoneType. - v5-0-2: Bug Fix NoneType in ignore type groups https://github.com/seperman/deepdiff/issues/207 diff --git a/deepdiff/serialization.py b/deepdiff/serialization.py index 0c1ba81e..2ce43742 100644 --- a/deepdiff/serialization.py +++ b/deepdiff/serialization.py @@ -116,7 +116,7 @@ def from_json_pickle(cls, value): else: logger.error('jsonpickle library needs to be installed in order to run from_json_pickle') # pragma: no cover. Json pickle is getting deprecated. - def to_json(self, default_mapping=None): + def to_json(self, default_mapping=None, **kwargs): """ Dump json of the text view. **Parameters** @@ -127,6 +127,8 @@ def to_json(self, default_mapping=None): If you have a certain object type that the json serializer can not serialize it, please pass the appropriate type conversion through this dictionary. + kwargs: Any other kwargs you pass will be passed on to Python's json.dumps() + **Example** Serialize custom objects @@ -147,7 +149,7 @@ def to_json(self, default_mapping=None): '{"type_changes": {"root": {"old_type": "A", "new_type": "B", "old_value": "obj A", "new_value": "obj B"}}}' """ dic = self.to_dict(view_override=TEXT_VIEW) - return json.dumps(dic, default=json_convertor_default(default_mapping=default_mapping)) + return json.dumps(dic, default=json_convertor_default(default_mapping=default_mapping), **kwargs) def to_dict(self, view_override=None): """ diff --git a/docs/changelog.rst b/docs/changelog.rst index 1d61ccdd..ba6274de 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,7 +5,7 @@ Changelog DeepDiff Changelog -- v5-2-3: Retaining the order of multiple dictionary items added via Delta. Fixed the typo with yml files in deep cli. Fixing Grep RecursionError where using non UTF-8 character. +- v5-2-3: Retaining the order of multiple dictionary items added via Delta. Fixed the typo with yml files in deep cli. Fixing Grep RecursionError where using non UTF-8 character. Allowing kwargs to be passed to to_json method. - v5-2-2: Fixed Delta serialization when None type is present. - v5-2-0: Removed Murmur3 as the preferred hashing method. Using SHA256 by default now. Added commandline for deepdiff. Added group_by. Added math_epsilon. Improved ignoring of NoneType. - v5-0-2: Bug Fix NoneType in ignore type groups https://github.com/seperman/deepdiff/issues/207 From 085e2394f9d5b52d4a08492f5d3647bbb47c9423 Mon Sep 17 00:00:00 2001 From: Seperman Date: Tue, 16 Feb 2021 11:57:27 -0800 Subject: [PATCH 6/6] =?UTF-8?q?Bump=20version:=205.2.2=20=E2=86=92=205.2.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 26 +++++++++++++------------- deepdiff/__init__.py | 2 +- docs/conf.py | 4 ++-- docs/index.rst | 2 +- setup.cfg | 2 +- setup.py | 2 +- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 41ff975e..9bd2956c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# DeepDiff v 5.2.2 +# DeepDiff v 5.2.3 ![Downloads](https://img.shields.io/pypi/dm/deepdiff.svg?style=flat) ![Python Versions](https://img.shields.io/pypi/pyversions/deepdiff.svg?style=flat) @@ -18,7 +18,7 @@ Tested on Python 3.6+ and PyPy3. **NOTE: The last version of DeepDiff to work on Python 3.5 was DeepDiff 5-0-2** -- [Documentation](https://zepworks.com/deepdiff/5.2.2/) +- [Documentation](https://zepworks.com/deepdiff/5.2.3/) ## Installation @@ -54,13 +54,13 @@ Note: if you want to use DeepDiff via commandline, make sure to run `pip install DeepDiff gets the difference of 2 objects. -> - Please take a look at the [DeepDiff docs](https://zepworks.com/deepdiff/5.2.2/diff.html) -> - The full documentation of all modules can be found on +> - Please take a look at the [DeepDiff docs](https://zepworks.com/deepdiff/5.2.3/diff.html) +> - The full documentation of all modules can be found on > - Tutorials and posts about DeepDiff can be found on ## A few Examples -> Note: This is just a brief overview of what DeepDiff can do. Please visit for full documentation. +> Note: This is just a brief overview of what DeepDiff can do. Please visit for full documentation. ### List difference ignoring order or duplicates @@ -264,8 +264,8 @@ Example: ``` -> - Please take a look at the [DeepDiff docs](https://zepworks.com/deepdiff/5.2.2/diff.html) -> - The full documentation can be found on +> - Please take a look at the [DeepDiff docs](https://zepworks.com/deepdiff/5.2.3/diff.html) +> - The full documentation can be found on # Deep Search @@ -297,8 +297,8 @@ And you can pass all the same kwargs as DeepSearch to grep too: {'matched_paths': {"root['somewhere']": 'around'}, 'matched_values': {"root['long']": 'somewhere'}} ``` -> - Please take a look at the [DeepSearch docs](https://zepworks.com/deepdiff/5.2.2/dsearch.html) -> - The full documentation can be found on +> - Please take a look at the [DeepSearch docs](https://zepworks.com/deepdiff/5.2.3/dsearch.html) +> - The full documentation can be found on # Deep Hash (New in v4-0-0) @@ -306,8 +306,8 @@ And you can pass all the same kwargs as DeepSearch to grep too: DeepHash is designed to give you hash of ANY python object based on its contents even if the object is not considered hashable! DeepHash is supposed to be deterministic in order to make sure 2 objects that contain the same data, produce the same hash. -> - Please take a look at the [DeepHash docs](https://zepworks.com/deepdiff/5.2.2/deephash.html) -> - The full documentation can be found on +> - Please take a look at the [DeepHash docs](https://zepworks.com/deepdiff/5.2.3/deephash.html) +> - The full documentation can be found on Let's say you have a dictionary object. @@ -355,8 +355,8 @@ Which you can write as: At first it might seem weird why DeepHash(obj)[obj] but remember that DeepHash(obj) is a dictionary of hashes of all other objects that obj contains too. -> - Please take a look at the [DeepHash docs](https://zepworks.com/deepdiff/5.2.2/deephash.html) -> - The full documentation can be found on +> - Please take a look at the [DeepHash docs](https://zepworks.com/deepdiff/5.2.3/deephash.html) +> - The full documentation can be found on # Using DeepDiff in unit tests diff --git a/deepdiff/__init__.py b/deepdiff/__init__.py index c152df3f..fcde3bc6 100644 --- a/deepdiff/__init__.py +++ b/deepdiff/__init__.py @@ -1,6 +1,6 @@ """This module offers the DeepDiff, DeepSearch, grep, Delta and DeepHash classes.""" # flake8: noqa -__version__ = '5.2.2' +__version__ = '5.2.3' import logging if __name__ == '__main__': diff --git a/docs/conf.py b/docs/conf.py index 8a253f09..9457fbd7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '5.2.2' +version = '5.2.3' # The full version, including alpha/beta/rc tags. -release = '5.2.2' +release = '5.2.3' load_dotenv(override=True) DOC_VERSION = os.environ.get('DOC_VERSION', version) diff --git a/docs/index.rst b/docs/index.rst index 2a2e1cf5..57b4277e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,7 +4,7 @@ contain the root `toctree` directive. -DeepDiff 5.2.2 documentation! +DeepDiff 5.2.3 documentation! ============================= ***************** diff --git a/setup.cfg b/setup.cfg index cfdfe5f0..2a0b0cf3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.2.2 +current_version = 5.2.3 commit = True tag = True tag_name = {new_version} diff --git a/setup.py b/setup.py index a120132f..50502c25 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ if os.environ.get('USER', '') == 'vagrant': del os.link -version = '5.2.2' +version = '5.2.3' def get_reqs(filename):