From 1dd0aecbc8307842e54ce429ddb254ef4bc97724 Mon Sep 17 00:00:00 2001 From: Seperman Date: Thu, 29 Apr 2021 16:26:07 -0700 Subject: [PATCH 1/8] bye bye readthedocs --- README.md | 12 ++++++------ readthedocs-requirements.txt | 1 - 2 files changed, 6 insertions(+), 7 deletions(-) delete mode 100644 readthedocs-requirements.txt diff --git a/README.md b/README.md index 160c65a2..87e20699 100644 --- a/README.md +++ b/README.md @@ -406,18 +406,18 @@ Example in DeepDiff for the same operation: {'type_changes': {"root['a']['b']['c']": {'old_type': , 'new_value': 42, 'old_value': 'foo', 'new_type': }}} ``` -# Pycon 2016 -I was honored to give a talk about how DeepDiff does what it does at Pycon 2016. Please check out the video and let me know what you think: +# Documentation -[Diff It To Dig It Video](https://www.youtube.com/watch?v=J5r99eJIxF4) -And here is more info: + -# Documentation +# Pycon 2016 - +I was honored to give a talk about the basics of how DeepDiff does what it does at Pycon 2016. Please check out the video and let me know what you think: +[Diff It To Dig It Video](https://www.youtube.com/watch?v=J5r99eJIxF4) +And here is more info: # ChangeLog diff --git a/readthedocs-requirements.txt b/readthedocs-requirements.txt deleted file mode 100644 index aa9bc185..00000000 --- a/readthedocs-requirements.txt +++ /dev/null @@ -1 +0,0 @@ -numpydoc==0.4 \ No newline at end of file From 772220cc5fbfd9ca0edce385b6ca9242892832a2 Mon Sep 17 00:00:00 2001 From: Dustin Torres Date: Fri, 30 Apr 2021 07:51:17 -0700 Subject: [PATCH 2/8] Minor cleanup to documentation wording regarding iterable_compare_func --- docs/diff_doc.rst | 2 +- docs/ignore_order.rst | 2 +- docs/index.rst | 2 +- docs/other.rst | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/diff_doc.rst b/docs/diff_doc.rst index 34ad3569..d57d7ab0 100644 --- a/docs/diff_doc.rst +++ b/docs/diff_doc.rst @@ -93,7 +93,7 @@ ignore_nan_inequality: Boolean, default = False iterable_compare_func: :ref:`iterable_compare_func_label`: - There are times that we want to guide DeepDiff as to what items to compare with other items. In such cases we can pass a iterable_compare_func that takes a function pointer to compare two items. It function takes two parameters and should return True if it is a match, False if it is not a match or raise CannotCompare if it is unable to compare the two. + There are times that we want to guide DeepDiff as to what items to compare with other items. In such cases we can pass a iterable_compare_func that takes a function pointer to compare two items. The function takes three parameters (x, y, level) and should return True if it is a match, False if it is not a match or raise CannotCompare if it is unable to compare the two. ignore_private_variables: Boolean, default = True :ref:`ignore_private_variables_label` diff --git a/docs/ignore_order.rst b/docs/ignore_order.rst index d4bc6956..c0b0eb03 100644 --- a/docs/ignore_order.rst +++ b/docs/ignore_order.rst @@ -227,7 +227,7 @@ Iterable Compare Func New in DeepDiff 5.5.0 -There are times that we want to guide DeepDiff as to what items to compare with other items. In such cases we can pass a `iterable_compare_func` that takes a function pointer to compare two items. It function takes two parameters and should return `True` if it is a match, `False` if it is not a match or raise `CannotCompare` if it is unable to compare the two. +There are times that we want to guide DeepDiff as to what items to compare with other items. In such cases we can pass a `iterable_compare_func` that takes a function pointer to compare two items. The function takes three parameters (x, y, level) and should return `True` if it is a match, `False` if it is not a match or raise `CannotCompare` if it is unable to compare the two. For example take the following objects: diff --git a/docs/index.rst b/docs/index.rst index b868bd2d..776acd82 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -42,7 +42,7 @@ What is New New In DeepDiff 5.5.0 --------------------- -1. New option called `iterable_compare_func` that takes a function pointer to compare two items. It function takes two parameters and should return `True` if it is a match, `False` if it is not a match or raise `CannotCompare` if it is unable to compare the two. If `CannotCompare` is raised then it will revert back to comparing in order. If `iterable_compare_func` is not provided or set to None the behavior defaults to comparing items in order. A new report item called `iterable_item_moved` this will only ever be added if there is a custom compare function. +1. New option called `iterable_compare_func` that takes a function pointer to compare two items. The function takes three parameters (x, y, level) and should return `True` if it is a match, `False` if it is not a match or raise `CannotCompare` if it is unable to compare the two. If `CannotCompare` is raised then it will revert back to comparing in order. If `iterable_compare_func` is not provided or set to None the behavior defaults to comparing items in order. A new report item called `iterable_item_moved` this will only ever be added if there is a custom compare function. >>> from deepdiff import DeepDiff >>> from deepdiff.helper import CannotCompare diff --git a/docs/other.rst b/docs/other.rst index 5f1325d4..fb9055ba 100644 --- a/docs/other.rst +++ b/docs/other.rst @@ -10,7 +10,7 @@ Iterable Compare Func New in DeepDiff 5.5.0 -There are times that we want to guide DeepDiff as to what items to compare with other items. In such cases we can pass a `iterable_compare_func` that takes a function pointer to compare two items. It function takes two parameters and should return `True` if it is a match, `False` if it is not a match or raise `CannotCompare` if it is unable to compare the two. +There are times that we want to guide DeepDiff as to what items to compare with other items. In such cases we can pass a `iterable_compare_func` that takes a function pointer to compare two items. The function takes three parameters (x, y, level) and should return `True` if it is a match, `False` if it is not a match or raise `CannotCompare` if it is unable to compare the two. For example take the following objects: From cb5b480f277eb1b41890e9946e5e57d9c22c64bf Mon Sep 17 00:00:00 2001 From: "sunao.626" Date: Wed, 16 Jun 2021 16:59:30 +0800 Subject: [PATCH 3/8] add ignore_order_func Add ignore_order_func to make ignore-order operation dynamic with level --- deepdiff/diff.py | 18 ++++++-- tests/test_ignore_order.py | 85 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 3 deletions(-) diff --git a/deepdiff/diff.py b/deepdiff/diff.py index 2f349031..e481486a 100755 --- a/deepdiff/diff.py +++ b/deepdiff/diff.py @@ -120,6 +120,7 @@ def __init__(self, hasher=None, hashes=None, ignore_order=False, + ignore_order_func=None, ignore_type_in_groups=None, ignore_string_type_changes=False, ignore_numeric_type_changes=False, @@ -156,12 +157,23 @@ def __init__(self, "cutoff_distance_for_pairs, cutoff_intersection_for_pairs, log_frequency_in_sec, cache_size, " "cache_tuning_sample_size, get_deep_distance, group_by, cache_purge_level, " "math_epsilon, iterable_compare_func, _original_type, " + "ignore_order_func," "_parameters and _shared_parameters.") % ', '.join(kwargs.keys())) if _parameters: + # compatibility + if "ignore_order_func" not in _parameters: + _parameters["ignore_order_func"] = lambda *_args, **_kwargs: _parameters["ignore_order_func"] + self.__dict__.update(_parameters) else: self.ignore_order = ignore_order + + if ignore_order_func is not None: + self.ignore_order_func = ignore_order_func + else: + self.ignore_order_func = lambda *_args, **_kwargs: ignore_order + ignore_type_in_groups = ignore_type_in_groups or [] if numbers == ignore_type_in_groups or numbers in ignore_type_in_groups: ignore_numeric_type_changes = True @@ -556,7 +568,7 @@ def _iterables_subscriptable(t1, t2): def _diff_iterable(self, level, parents_ids=frozenset(), _original_type=None): """Difference of iterables""" - if self.ignore_order: + if self.ignore_order_func(level): self._diff_iterable_with_deephash(level, parents_ids, _original_type=_original_type) else: self._diff_iterable_in_order(level, parents_ids, _original_type=_original_type) @@ -1133,7 +1145,7 @@ def _diff_numpy_array(self, level, parents_ids=frozenset()): # which means numpy module needs to be available. So np can't be None. raise ImportError(CANT_FIND_NUMPY_MSG) # pragma: no cover - if not self.ignore_order: + if not self.ignore_order_func(level): # fast checks if self.significant_digits is None: if np.array_equal(level.t1, level.t2): @@ -1159,7 +1171,7 @@ def _diff_numpy_array(self, level, parents_ids=frozenset()): dimensions = len(shape) if dimensions == 1: self._diff_iterable(level, parents_ids, _original_type=_original_type) - elif self.ignore_order: + elif self.ignore_order_func(level): # arrays are converted to python lists so that certain features of DeepDiff can apply on them easier. # They will be converted back to Numpy at their final dimension. level.t1 = level.t1.tolist() diff --git a/tests/test_ignore_order.py b/tests/test_ignore_order.py index 1c059493..f69a416d 100644 --- a/tests/test_ignore_order.py +++ b/tests/test_ignore_order.py @@ -928,3 +928,88 @@ def compare_func(x, y, level=None): ddiff2 = DeepDiff(t1, t2, ignore_order=True, cutoff_intersection_for_pairs=1, cutoff_distance_for_pairs=1, iterable_compare_func=compare_func) assert expected_with_compare_func == ddiff2 assert ddiff != ddiff2 + + +class TestDynamicIgnoreOrder: + def test_ignore_order_func(self): + t1 = { + "order_matters": [ + {1}, + { + 'id': 2, + 'value': [7, 8, 1] + }, + { + 'id': 3, + 'value': [7, 8], + }, + ], + "order_does_not_matter": [ + {1}, + { + 'id': 2, + 'value': [7, 8, 1] + }, + { + 'id': 3, + 'value': [7, 8], + }, + ] + } + + t2 = { + "order_matters": [ + { + 'id': 2, + 'value': [7, 8] + }, + { + 'id': 3, + 'value': [7, 8, 1], + }, + {}, + ], + "order_does_not_matter": [ + { + 'id': 2, + 'value': [7, 8] + }, + { + 'id': 3, + 'value': [7, 8, 1], + }, + {}, + ] + } + + def ignore_order_func(level): + return "order_does_not_matter" in level.path() + + ddiff = DeepDiff(t1, t2, cutoff_intersection_for_pairs=1, cutoff_distance_for_pairs=1, ignore_order_func=ignore_order_func) + + expected = { + 'type_changes': { + "root['order_matters'][0]": { + 'old_type': set, + 'new_type': dict, + 'old_value': {1}, + 'new_value': {'id': 2, 'value': [7, 8]} + }, + "root['order_does_not_matter'][0]": { + 'old_type': set, + 'new_type': dict, + 'old_value': {1}, + 'new_value': {} + } + }, + 'dictionary_item_removed': [ + "root['order_matters'][2]['id']", + "root['order_matters'][2]['value']" + ], + 'values_changed': { + "root['order_matters'][1]['id']": {'new_value': 3, 'old_value': 2}, + "root['order_does_not_matter'][2]['id']": {'new_value': 2, 'old_value': 3}, + "root['order_does_not_matter'][1]['id']": {'new_value': 3, 'old_value': 2} + } + } + assert expected == ddiff From e55efa003d0e4171506a6644166b0c2a4a7d7745 Mon Sep 17 00:00:00 2001 From: "sunao.626" Date: Wed, 16 Jun 2021 22:11:42 +0800 Subject: [PATCH 4/8] allow custom operators allow custom operators to do/report some custom operations allow --- deepdiff/diff.py | 63 +++++++++++++++++++----- deepdiff/model.py | 31 ++++++++++-- tests/test_operators.py | 104 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 183 insertions(+), 15 deletions(-) create mode 100644 tests/test_operators.py diff --git a/deepdiff/diff.py b/deepdiff/diff.py index 2f349031..d4ea45ef 100755 --- a/deepdiff/diff.py +++ b/deepdiff/diff.py @@ -28,14 +28,13 @@ RemapDict, ResultDict, TextResult, TreeResult, DiffLevel, DictRelationship, AttributeRelationship, SubscriptableIterableRelationship, NonSubscriptableIterableRelationship, - SetRelationship, NumpyArrayRelationship) + SetRelationship, NumpyArrayRelationship, CUSTOM_FILED) from deepdiff.deephash import DeepHash, combine_hashes_lists from deepdiff.base import Base from deepdiff.lfucache import LFUCache, DummyLFU logger = logging.getLogger(__name__) - MAX_PASSES_REACHED_MSG = ( 'DeepDiff has reached the max number of passes of {}. ' 'You can possibly get more accurate results by increasing the max_passes parameter.') @@ -140,6 +139,7 @@ def __init__(self, verbose_level=1, view=TEXT_VIEW, iterable_compare_func=None, + custom_operators=None, _original_type=None, _parameters=None, _shared_parameters=None, @@ -147,20 +147,25 @@ def __init__(self, super().__init__() if kwargs: raise ValueError(( - "The following parameter(s) are not valid: %s\n" - "The valid parameters are ignore_order, report_repetition, significant_digits, " - "number_format_notation, exclude_paths, exclude_types, exclude_regex_paths, ignore_type_in_groups, " - "ignore_string_type_changes, ignore_numeric_type_changes, ignore_type_subclasses, truncate_datetime, " - "ignore_private_variables, ignore_nan_inequality, number_to_string_func, verbose_level, " - "view, hasher, hashes, max_passes, max_diffs, " - "cutoff_distance_for_pairs, cutoff_intersection_for_pairs, log_frequency_in_sec, cache_size, " - "cache_tuning_sample_size, get_deep_distance, group_by, cache_purge_level, " - "math_epsilon, iterable_compare_func, _original_type, " - "_parameters and _shared_parameters.") % ', '.join(kwargs.keys())) + "The following parameter(s) are not valid: %s\n" + "The valid parameters are ignore_order, report_repetition, significant_digits, " + "number_format_notation, exclude_paths, exclude_types, exclude_regex_paths, ignore_type_in_groups, " + "ignore_string_type_changes, ignore_numeric_type_changes, ignore_type_subclasses, truncate_datetime, " + "ignore_private_variables, ignore_nan_inequality, number_to_string_func, verbose_level, " + "view, hasher, hashes, max_passes, max_diffs, " + "cutoff_distance_for_pairs, cutoff_intersection_for_pairs, log_frequency_in_sec, cache_size, " + "cache_tuning_sample_size, get_deep_distance, group_by, cache_purge_level, " + "math_epsilon, iterable_compare_func, _original_type, " + "custom_operators, " + "_parameters and _shared_parameters.") % ', '.join(kwargs.keys())) if _parameters: + if "custom_operators" not in _parameters: + _parameters["custom_operators"] = [] + self.__dict__.update(_parameters) else: + self.custom_operators = custom_operators or [] self.ignore_order = ignore_order ignore_type_in_groups = ignore_type_in_groups or [] if numbers == ignore_type_in_groups or numbers in ignore_type_in_groups: @@ -327,6 +332,24 @@ def _report_result(self, report_type, level): level.report_type = report_type self.tree[report_type].add(level) + def custom_report_result(self, report_type, level, extra_info=None): + """ + Add a detected change to the reference-style result dictionary. + report_type will be added to level. + (We'll create the text-style report from there later.) + :param report_type: A well defined string key describing the type of change. + Examples: "set_item_added", "values_changed" + :param parent: A DiffLevel object describing the objects in question in their + before-change and after-change object structure. + :param extra_info: A dict that describe this result + :rtype: None + """ + + if not self._skip_this(level): + level.report_type = report_type + level.additional[CUSTOM_FILED] = extra_info + self.tree[report_type].add(level) + @staticmethod def _dict_from_slots(object): def unmangle(attribute): @@ -1219,6 +1242,19 @@ def _auto_off_cache(self): self._stats[DISTANCE_CACHE_ENABLED] = False self.progress_logger('Due to minimal cache hits, {} is disabled.'.format('distance cache')) + def _use_custom_operator(self, level): + """ + + """ + used = False + + for operator in self.custom_operators: + if operator.match(level): + prevent_default = operator.diff(level, self) + used = True if prevent_default is None else prevent_default + + return used + def _diff(self, level, parents_ids=frozenset(), _original_type=None): """ The main diff method @@ -1255,6 +1291,9 @@ def _diff(self, level, parents_ids=frozenset(), _original_type=None): if self.ignore_nan_inequality and isinstance(level.t1, float) and str(level.t1) == str(level.t2) == 'nan': return + if self._use_custom_operator(level): + return + if isinstance(level.t1, booleans): self._diff_booleans(level) diff --git a/deepdiff/model.py b/deepdiff/model.py index 80273559..f65208b1 100644 --- a/deepdiff/model.py +++ b/deepdiff/model.py @@ -24,6 +24,8 @@ "repetition_change", } +CUSTOM_FILED = "__internal:custom:extra_info" + class DoesNotExist(Exception): pass @@ -47,6 +49,7 @@ class PrettyOrderedSet(OrderedSet): From the perspective of the users of the library, they are dealing with lists. Behind the scene, we have ordered sets. """ + def __repr__(self): return '[{}]'.format(", ".join(map(str, self))) @@ -85,9 +88,13 @@ def mutual_add_removes_to_become_value_changes(self): if 'iterable_item_added' in self and not self['iterable_item_added']: del self['iterable_item_added'] + def __getitem__(self, item): + if item not in self: + self[item] = PrettyOrderedSet() + return self.get(item) -class TextResult(ResultDict): +class TextResult(ResultDict): ADD_QUOTES_TO_STRINGS = True def __init__(self, tree_results=None, verbose_level=1): @@ -135,6 +142,7 @@ def _from_tree_results(self, tree): self._from_tree_set_item_added(tree) self._from_tree_repetition_change(tree) self._from_tree_deep_distance(tree) + self._from_tree_custom_results(tree) def _from_tree_default(self, tree, report_type): if report_type in tree: @@ -232,16 +240,33 @@ def _from_tree_repetition_change(self, tree): for change in tree['repetition_change']: path = change.path(force=FORCE_DEFAULT) self['repetition_change'][path] = RemapDict(change.additional[ - 'repetition']) + 'repetition']) self['repetition_change'][path]['value'] = change.t1 def _from_tree_deep_distance(self, tree): if 'deep_distance' in tree: self['deep_distance'] = tree['deep_distance'] + def _from_tree_custom_results(self, tree): + for k, _level_list in tree.items(): + if k not in REPORT_KEYS: + if not isinstance(_level_list, PrettyOrderedSet): + continue -class DeltaResult(TextResult): + if len(_level_list) == 0: + continue + if not isinstance(_level_list[0], DiffLevel): + continue + + _custom_dict = {} + for _level in _level_list: + _custom_dict[_level.path( + force=FORCE_DEFAULT)] = _level.additional.get(CUSTOM_FILED, {}) + self[k] = _custom_dict + + +class DeltaResult(TextResult): ADD_QUOTES_TO_STRINGS = False def __init__(self, tree_results=None, ignore_order=None): diff --git a/tests/test_operators.py b/tests/test_operators.py new file mode 100644 index 00000000..9bc6e515 --- /dev/null +++ b/tests/test_operators.py @@ -0,0 +1,104 @@ +import math +import re + +from deepdiff import DeepDiff + + +class TestOperators: + def test_custom_operators_prevent_default(self): + t1 = { + "coordinates": [ + {"x": 5, "y": 5}, + {"x": 8, "y": 8} + ] + } + + t2 = { + "coordinates": [ + {"x": 6, "y": 6}, + {"x": 88, "y": 88} + ] + } + + class L2DistanceDifferWithPreventDefault: + def __init__(self, distance_threshold): + self.distance_threshold = distance_threshold + + def _l2_distance(self, c1, c2): + return math.sqrt( + (c1["x"] - c2["x"]) ** 2 + (c1["y"] - c2["y"]) ** 2 + ) + + def match(self, level): + return re.search(r"^root\['coordinates'\]\[\d+\]$", level.path()) is not None + + def diff(self, level, diff_instance): + l2_distance = self._l2_distance(level.t1, level.t2) + if l2_distance > self.distance_threshold: + diff_instance.custom_report_result('distance_too_far', level, { + "l2_distance": l2_distance + }) + # + return True + + ddiff = DeepDiff(t1, t2, custom_operators=[L2DistanceDifferWithPreventDefault(1)]) + + expected = { + 'distance_too_far': { + "root['coordinates'][0]": {'l2_distance': 1.4142135623730951}, + "root['coordinates'][1]": {'l2_distance': 113.13708498984761} + } + } + assert expected == ddiff + + def test_custom_operators_not_prevent_default(self): + t1 = { + "coordinates": [ + {"x": 5, "y": 5}, + {"x": 8, "y": 8} + ] + } + + t2 = { + "coordinates": [ + {"x": 6, "y": 6}, + {"x": 88, "y": 88} + ] + } + + class L2DistanceDifferWithPreventDefault: + def __init__(self, distance_threshold): + self.distance_threshold = distance_threshold + + def _l2_distance(self, c1, c2): + return math.sqrt( + (c1["x"] - c2["x"]) ** 2 + (c1["y"] - c2["y"]) ** 2 + ) + + def match(self, level): + print(level.path()) + return re.search(r"^root\['coordinates'\]\[\d+\]$", level.path()) is not None + + def diff(self, level, diff_instance): + l2_distance = self._l2_distance(level.t1, level.t2) + if l2_distance > self.distance_threshold: + diff_instance.custom_report_result('distance_too_far', level, { + "l2_distance": l2_distance + }) + # + return False + + ddiff = DeepDiff(t1, t2, custom_operators=[L2DistanceDifferWithPreventDefault(1)]) + expected = { + 'values_changed': { + "root['coordinates'][0]['x']": {'new_value': 6, 'old_value': 5}, + "root['coordinates'][0]['y']": {'new_value': 6, 'old_value': 5}, + "root['coordinates'][1]['x']": {'new_value': 88, 'old_value': 8}, + "root['coordinates'][1]['y']": {'new_value': 88, 'old_value': 8} + }, + 'distance_too_far': { + "root['coordinates'][0]": {'l2_distance': 1.4142135623730951}, + "root['coordinates'][1]": {'l2_distance': 113.13708498984761} + } + } + assert expected == ddiff From ae66cab0bc9c0dbf966d4ff57714c26a798120ab Mon Sep 17 00:00:00 2001 From: "sunao.626" Date: Thu, 17 Jun 2021 14:53:31 +0800 Subject: [PATCH 5/8] give custom operators more flexibiliy --- deepdiff/diff.py | 6 +++--- tests/test_operators.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/deepdiff/diff.py b/deepdiff/diff.py index 26c32b10..b48ca7b8 100755 --- a/deepdiff/diff.py +++ b/deepdiff/diff.py @@ -1279,6 +1279,9 @@ def _diff(self, level, parents_ids=frozenset(), _original_type=None): if self._count_diff() is StopIteration: return + if self._use_custom_operator(level): + return + if level.t1 is level.t2: return @@ -1302,9 +1305,6 @@ def _diff(self, level, parents_ids=frozenset(), _original_type=None): if self.ignore_nan_inequality and isinstance(level.t1, float) and str(level.t1) == str(level.t2) == 'nan': return - if self._use_custom_operator(level): - return - if isinstance(level.t1, booleans): self._diff_booleans(level) diff --git a/tests/test_operators.py b/tests/test_operators.py index 9bc6e515..40a51503 100644 --- a/tests/test_operators.py +++ b/tests/test_operators.py @@ -102,3 +102,39 @@ def diff(self, level, diff_instance): } } assert expected == ddiff + + def test_custom_operators_should_not_equal(self): + t1 = { + "id": 5, + "expect_change_pos": 10, + "expect_change_neg": 10, + } + + t2 = { + "id": 5, + "expect_change_pos": 100, + "expect_change_neg": 10, + } + + class ExpectChangeOperator: + def __init__(self, path_regex): + self.path_regex = path_regex + + def match(self, level): + print(level.path(), re.search(re.compile(self.path_regex), level.path())) + return re.search(re.compile(self.path_regex), level.path()) is not None + + def diff(self, level, diff_instance): + print(level) + if level.t1 == level.t2: + diff_instance.custom_report_result('unexpected:still', level, { + "old": level.t1, + "new": level.t2 + }) + + return True + + ddiff = DeepDiff(t1, t2, custom_operators=[ + ExpectChangeOperator("root\\['expect_change.*'\\]") + ]) + print(ddiff) From e4394705c8cec6f65dd9c1cd3299bd5a3e2ef4ee Mon Sep 17 00:00:00 2001 From: "sunao.626" Date: Thu, 17 Jun 2021 14:56:54 +0800 Subject: [PATCH 6/8] fix assertion --- tests/test_operators.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_operators.py b/tests/test_operators.py index 40a51503..446f6a96 100644 --- a/tests/test_operators.py +++ b/tests/test_operators.py @@ -137,4 +137,5 @@ def diff(self, level, diff_instance): ddiff = DeepDiff(t1, t2, custom_operators=[ ExpectChangeOperator("root\\['expect_change.*'\\]") ]) - print(ddiff) + + assert ddiff == {'unexpected:still': {"root['expect_change_neg']": {'old': 10, 'new': 10}}} From d49c4cef901abfb226580d4dc43c0d8e97c26eaf Mon Sep 17 00:00:00 2001 From: "sunao.626" Date: Thu, 24 Jun 2021 10:43:55 +0800 Subject: [PATCH 7/8] fix some code-issue --- deepdiff/diff.py | 16 +++------------- deepdiff/helper.py | 12 +++++------- deepdiff/model.py | 20 +++++++++++--------- tests/test_delta.py | 2 ++ 4 files changed, 21 insertions(+), 29 deletions(-) diff --git a/deepdiff/diff.py b/deepdiff/diff.py index b48ca7b8..fbc35363 100755 --- a/deepdiff/diff.py +++ b/deepdiff/diff.py @@ -28,7 +28,7 @@ RemapDict, ResultDict, TextResult, TreeResult, DiffLevel, DictRelationship, AttributeRelationship, SubscriptableIterableRelationship, NonSubscriptableIterableRelationship, - SetRelationship, NumpyArrayRelationship, CUSTOM_FILED) + SetRelationship, NumpyArrayRelationship, CUSTOM_FIELD) from deepdiff.deephash import DeepHash, combine_hashes_lists from deepdiff.base import Base from deepdiff.lfucache import LFUCache, DummyLFU @@ -161,22 +161,12 @@ def __init__(self, "_parameters and _shared_parameters.") % ', '.join(kwargs.keys())) if _parameters: - # compatibility - if "ignore_order_func" not in _parameters: - _parameters["ignore_order_func"] = lambda *_args, **_kwargs: _parameters["ignore_order_func"] - - if "custom_operators" not in _parameters: - _parameters["custom_operators"] = [] - self.__dict__.update(_parameters) else: self.custom_operators = custom_operators or [] self.ignore_order = ignore_order - if ignore_order_func is not None: - self.ignore_order_func = ignore_order_func - else: - self.ignore_order_func = lambda *_args, **_kwargs: ignore_order + self.ignore_order_func = ignore_order_func or (lambda *_args, **_kwargs: ignore_order) ignore_type_in_groups = ignore_type_in_groups or [] if numbers == ignore_type_in_groups or numbers in ignore_type_in_groups: @@ -358,7 +348,7 @@ def custom_report_result(self, report_type, level, extra_info=None): if not self._skip_this(level): level.report_type = report_type - level.additional[CUSTOM_FILED] = extra_info + level.additional[CUSTOM_FIELD] = extra_info self.tree[report_type].add(level) @staticmethod diff --git a/deepdiff/helper.py b/deepdiff/helper.py index 8901ea3c..abef7522 100644 --- a/deepdiff/helper.py +++ b/deepdiff/helper.py @@ -65,7 +65,7 @@ class np_type: np_int8, np_int16, np_int32, np_int64, np_uint8, np_uint16, np_uint32, np_uint64, np_intp, np_uintp, np_float32, np_float64, np_float_, np_complex64, - np_complex128, np_complex_, ) + np_complex128, np_complex_,) numpy_dtypes = set(numpy_numbers) numpy_dtypes.add(np_bool_) @@ -112,7 +112,6 @@ def copy(self): # pragma: no cover. Only used in pypy3 and py3.5 else: dict_ = OrderedDictPlus # pragma: no cover. Only used in pypy3 and py3.5 - if py4: logger.warning('Python 4 is not supported yet. Switching logic to Python 3.') # pragma: no cover py3 = True # pragma: no cover @@ -184,6 +183,7 @@ class NotPresent: # pragma: no cover in the future. We previously used None for this but this caused problem when users actually added and removed None. Srsly guys? :D """ + def __repr__(self): return 'not present' # pragma: no cover @@ -202,7 +202,6 @@ class CannotCompare(Exception): not_hashed = NotHashed() notpresent = NotPresent() - # Disabling remapping from old to new keys since the mapping is deprecated. RemapDict = dict_ @@ -316,8 +315,8 @@ def type_in_type_group(item, type_group): def type_is_subclass_of_type_group(item, type_group): return isinstance(item, type_group) \ - or (isinstance(item, type) and issubclass(item, type_group)) \ - or type_in_type_group(item, type_group) + or (isinstance(item, type) and issubclass(item, type_group)) \ + or type_in_type_group(item, type_group) def get_doc(doc_filename): @@ -426,7 +425,6 @@ def __repr__(self): not_found = _NotFound() - warnings.simplefilter('once', DeepDiffDeprecationWarning) @@ -583,7 +581,7 @@ def get_homogeneous_numpy_compatible_type_of_seq(seq): iseq = iter(seq) first_type = type(next(iseq)) if first_type in {int, float, Decimal}: - type_ = first_type if all((type(x) is first_type) for x in iseq ) else False + type_ = first_type if all((type(x) is first_type) for x in iseq) else False return PYTHON_TYPE_TO_NUMPY_TYPE.get(type_, False) else: return False diff --git a/deepdiff/model.py b/deepdiff/model.py index f65208b1..fc5ad135 100644 --- a/deepdiff/model.py +++ b/deepdiff/model.py @@ -24,7 +24,7 @@ "repetition_change", } -CUSTOM_FILED = "__internal:custom:extra_info" +CUSTOM_FIELD = "__internal:custom:extra_info" class DoesNotExist(Exception): @@ -239,8 +239,9 @@ def _from_tree_repetition_change(self, tree): if 'repetition_change' in tree: for change in tree['repetition_change']: path = change.path(force=FORCE_DEFAULT) - self['repetition_change'][path] = RemapDict(change.additional[ - 'repetition']) + self['repetition_change'][path] = RemapDict( + change.additional['repetition'] + ) self['repetition_change'][path]['value'] = change.t1 def _from_tree_deep_distance(self, tree): @@ -253,16 +254,17 @@ def _from_tree_custom_results(self, tree): if not isinstance(_level_list, PrettyOrderedSet): continue - if len(_level_list) == 0: - continue - - if not isinstance(_level_list[0], DiffLevel): - continue + # if len(_level_list) == 0: + # continue + # + # if not isinstance(_level_list[0], DiffLevel): + # continue + # _level_list is a list of DiffLevel _custom_dict = {} for _level in _level_list: _custom_dict[_level.path( - force=FORCE_DEFAULT)] = _level.additional.get(CUSTOM_FILED, {}) + force=FORCE_DEFAULT)] = _level.additional.get(CUSTOM_FIELD, {}) self[k] = _custom_dict diff --git a/tests/test_delta.py b/tests/test_delta.py index 66a6c24a..cecf925b 100644 --- a/tests/test_delta.py +++ b/tests/test_delta.py @@ -1079,6 +1079,8 @@ def test_delta_view_and_to_delta_dict_are_equal_when_parameteres_passed(self): 'cache_size': 500, 'cutoff_intersection_for_pairs': 0.6, 'group_by': None, + 'ignore_order_func': lambda *args, **kwargs: True, + 'custom_operators': [] } expected = {'iterable_items_added_at_indexes': {'root': {1: 1, 2: 1, 3: 1}}, 'iterable_items_removed_at_indexes': {'root': {1: 2, 2: 2}}} From 7e778fd9418108cb3521d56844c3ffb31fec3555 Mon Sep 17 00:00:00 2001 From: "sunao.626" Date: Thu, 24 Jun 2021 11:35:17 +0800 Subject: [PATCH 8/8] docs: add docs for custom operators and ignore_order_func --- deepdiff/operator.py | 16 +++++++++++ docs/ignore_order.rst | 23 +++++++++++++--- docs/other.rst | 59 +++++++++++++++++++++++++++++++++++++++++ tests/test_operators.py | 38 +++++++++++++------------- 4 files changed, 112 insertions(+), 24 deletions(-) create mode 100644 deepdiff/operator.py diff --git a/deepdiff/operator.py b/deepdiff/operator.py new file mode 100644 index 00000000..e9551b49 --- /dev/null +++ b/deepdiff/operator.py @@ -0,0 +1,16 @@ +import re + + +class BaseOperator: + __operator_name__ = "__base__" + + def __init__(self, path_regex): + self.path_regex = path_regex + self.regex = re.compile(f"^{self.path_regex}$") + + def match(self, level) -> bool: + matched = re.search(self.regex, level.path()) is not None + return matched + + def diff(self, level, instance) -> bool: + raise NotImplementedError diff --git a/docs/ignore_order.rst b/docs/ignore_order.rst index c0b0eb03..a02069ab 100644 --- a/docs/ignore_order.rst +++ b/docs/ignore_order.rst @@ -34,6 +34,21 @@ List difference ignoring order or duplicates: (with the same dictionaries as abo >>> print (ddiff) {} +.. _ignore_order_func_label: + +Dynamic Ignore Order +-------------------- + +Sometimes single *ignore_order* parameter is not enough to do a diff job, +you can use *ignore_order_func* to determine whether the order of certain paths should be ignored + +List difference ignoring order with *ignore_order_func* + >>> t1 = {"set": [1,2,3], "list": [1,2,3]} + >>> t2 = {"set": [3,2,1], "list": [3,2,1]} + >>> ddiff = DeepDiff(t1, t2, ignore_order_func=lambda level: "set" in level.path()) + >>> print (ddiff) + { 'values_changed': { "root['list'][0]": {'new_value': 3, 'old_value': 1}, + "root['list'][2]": {'new_value': 1, 'old_value': 3}}} .. _report_repetition_label: @@ -78,7 +93,7 @@ You can control the maximum number of passes that can be run via the max_passes Max Passes Example >>> from pprint import pprint >>> from deepdiff import DeepDiff - >>> + >>> >>> t1 = [ ... { ... 'key3': [[[[[1, 2, 4, 5]]]]], @@ -89,7 +104,7 @@ Max Passes Example ... 'key6': 'val6', ... }, ... ] - >>> + >>> >>> t2 = [ ... { ... 'key5': 'CHANGE', @@ -100,12 +115,12 @@ Max Passes Example ... 'key4': [7, 8], ... }, ... ] - >>> + >>> >>> for max_passes in (1, 2, 62, 65): ... diff = DeepDiff(t1, t2, ignore_order=True, max_passes=max_passes, verbose_level=2) ... print('-\n----- Max Passes = {} -----'.format(max_passes)) ... pprint(diff) - ... + ... DeepDiff has reached the max number of passes of 1. You can possibly get more accurate results by increasing the max_passes parameter. - ----- Max Passes = 1 ----- diff --git a/docs/other.rst b/docs/other.rst index fb9055ba..ac7acab2 100644 --- a/docs/other.rst +++ b/docs/other.rst @@ -122,8 +122,67 @@ For example you could use the level object to further determine if the 2 objects The level parameter of the iterable_compare_func is only used when ignore_order=False which is the default value for ignore_order. +.. _custom_operators_label: + +Custom Operators +--------------------- + +Whether two objects are different or not are largely depend on the context. For example, apple and banana are the same +if you are considering whether the + +*custom_operators* is for the job. + +To define an custom operator, you just need to inherit a *BaseOperator* and + * implement method + * diff(level: DiffLevel, instance: DeepDiff) -> boolean + * to do custom diff logic with full access to DeepDiff instance + * you can use instance.custom_report_result to record info + * to return a boolean value to determine whether the process + should quit or continue with default behavior +An operator that mapping L2:distance as diff criteria + >>> from deepdiff import DeepDiff + >>> from deepdiff.operator import BaseOperator + >>> + >>> t1 = { + ... "coordinates": [ + ... {"x": 5, "y": 5}, + ... {"x": 8, "y": 8} + ... ] + ... } + ... + >>> t2 = { + ... "coordinates": [ + ... {"x": 6, "y": 6}, + ... {"x": 88, "y": 88} + ... ] + ... } + ... + >>> class L2DistanceDifferWithPreventDefault(BaseOperator): + ... def __init__(self, distance_threshold): + ... self.distance_threshold = distance_threshold + ... + ... def _l2_distance(self, c1, c2): + ... return math.sqrt( + ... (c1["x"] - c2["x"]) ** 2 + (c1["y"] - c2["y"]) ** 2 + ... ) + ... # you can also override match method + ... # def match(self, level): + ... # return True + ... + ... def diff(self, level, diff_instance): + ... l2_distance = self._l2_distance(level.t1, level.t2) + ... if l2_distance > self.distance_threshold: + ... diff_instance.custom_report_result('distance_too_far', level, { + ... "l2_distance": l2_distance + ... }) + ... # + ... return True + ... + >>> DeepDiff(t1, t2, custom_operators=[L2DistanceDifferWithPreventDefault(1)]) + {'distance_too_far': {"root['coordinates'][0]": {'l2_distance': 1.4142135623730951}, + "root['coordinates'][1]": {'l2_distance': 113.13708498984761}}} Back to :doc:`/index` diff --git a/tests/test_operators.py b/tests/test_operators.py index 446f6a96..edf025aa 100644 --- a/tests/test_operators.py +++ b/tests/test_operators.py @@ -2,6 +2,7 @@ import re from deepdiff import DeepDiff +from deepdiff.operator import BaseOperator class TestOperators: @@ -20,8 +21,9 @@ def test_custom_operators_prevent_default(self): ] } - class L2DistanceDifferWithPreventDefault: - def __init__(self, distance_threshold): + class L2DistanceDifferWithPreventDefault(BaseOperator): + def __init__(self, path_regex: str, distance_threshold: float): + super().__init__(path_regex) self.distance_threshold = distance_threshold def _l2_distance(self, c1, c2): @@ -29,9 +31,6 @@ def _l2_distance(self, c1, c2): (c1["x"] - c2["x"]) ** 2 + (c1["y"] - c2["y"]) ** 2 ) - def match(self, level): - return re.search(r"^root\['coordinates'\]\[\d+\]$", level.path()) is not None - def diff(self, level, diff_instance): l2_distance = self._l2_distance(level.t1, level.t2) if l2_distance > self.distance_threshold: @@ -41,7 +40,10 @@ def diff(self, level, diff_instance): # return True - ddiff = DeepDiff(t1, t2, custom_operators=[L2DistanceDifferWithPreventDefault(1)]) + ddiff = DeepDiff(t1, t2, custom_operators=[L2DistanceDifferWithPreventDefault( + "^root\\['coordinates'\\]\\[\\d+\\]$", + 1 + )]) expected = { 'distance_too_far': { @@ -66,8 +68,9 @@ def test_custom_operators_not_prevent_default(self): ] } - class L2DistanceDifferWithPreventDefault: - def __init__(self, distance_threshold): + class L2DistanceDifferWithPreventDefault(BaseOperator): + def __init__(self, path_regex, distance_threshold): + super().__init__(path_regex) self.distance_threshold = distance_threshold def _l2_distance(self, c1, c2): @@ -75,10 +78,6 @@ def _l2_distance(self, c1, c2): (c1["x"] - c2["x"]) ** 2 + (c1["y"] - c2["y"]) ** 2 ) - def match(self, level): - print(level.path()) - return re.search(r"^root\['coordinates'\]\[\d+\]$", level.path()) is not None - def diff(self, level, diff_instance): l2_distance = self._l2_distance(level.t1, level.t2) if l2_distance > self.distance_threshold: @@ -88,7 +87,11 @@ def diff(self, level, diff_instance): # return False - ddiff = DeepDiff(t1, t2, custom_operators=[L2DistanceDifferWithPreventDefault(1)]) + ddiff = DeepDiff(t1, t2, custom_operators=[L2DistanceDifferWithPreventDefault( + "^root\\['coordinates'\\]\\[\\d+\\]$", + 1 + ) + ]) expected = { 'values_changed': { "root['coordinates'][0]['x']": {'new_value': 6, 'old_value': 5}, @@ -116,16 +119,11 @@ def test_custom_operators_should_not_equal(self): "expect_change_neg": 10, } - class ExpectChangeOperator: + class ExpectChangeOperator(BaseOperator): def __init__(self, path_regex): - self.path_regex = path_regex - - def match(self, level): - print(level.path(), re.search(re.compile(self.path_regex), level.path())) - return re.search(re.compile(self.path_regex), level.path()) is not None + super().__init__(path_regex) def diff(self, level, diff_instance): - print(level) if level.t1 == level.t2: diff_instance.custom_report_result('unexpected:still', level, { "old": level.t1,