diff --git a/README.md b/README.md index 885d486a..2e9aea1b 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,13 @@ Tested on Python 3.6+ and PyPy3. ## What is new? +DeepDiff 6-1-0 + +- DeepDiff.affected_paths can be used to get the list of all paths where a change, addition, or deletion was reported for. +- DeepDiff.affected_root_keys can be used to get the list of all paths where a change, addition, or deletion was reported for. +- Bugfix: ValueError when using Decimal 0.x #339 by [Enric Pou](https://github.com/epou) +- Serialization of UUID + DeepDiff 6-0-0 - [Exclude obj callback strict](https://github.com/seperman/deepdiff/pull/320/files) parameter is added to DeepDiff by Mikhail Khviyuzov [mskhviyu](https://github.com/mskhviyu). diff --git a/conftest.py b/conftest.py index 0206df11..263b1296 100644 --- a/conftest.py +++ b/conftest.py @@ -46,6 +46,28 @@ def nested_a_result(): return json.load(the_file) +@pytest.fixture(scope='class') +def nested_a_affected_paths(): + return { + 'root[0][0][2][0][1]', 'root[0][1][1][1][5]', 'root[0][2][1]', + 'root[1][1][2][0][1]', 'root[1][2][0]', 'root[1][2][0][1][5]', + 'root[1][0][2][2][3]', 'root[0][0][1][0][0]', 'root[0][1][0][2][3]', + 'root[0][3][0][2][3]', 'root[0][3][1][0][2]', 'root[1][1][1][0][0]', + 'root[1][0][1][2][1]', 'root[1][0][2][1][2]', 'root[1][3][0][2][3]', + 'root[1][3][1][0][2]', 'root[1][2][0][2]', 'root[1][0][2][0][1]', + 'root[0][3][2][0][1]', 'root[0][3][2][1][0]', 'root[1][3][1][1]', + 'root[1][2][1][1][0]', 'root[1][2][1][0]', 'root[1][0][0][0][2]', + 'root[1][3][2][1][0]', 'root[1][0][0][1][1]', 'root[0][1][2][0]', + 'root[0][1][2][1][0]', 'root[0][2][0][1][2]', 'root[1][3][0][1]', + 'root[0][3][1][1]', 'root[1][2][0][0][2]', 'root[1][3][2][0][1]', + 'root[1][0][1][0]', 'root[1][2][0][0][0]', 'root[1][0][0][0][1]', + 'root[1][3][2][2][2]', 'root[0][1][1][2][1]', 'root[0][1][1][2][2]', + 'root[0][2][0][0][2]', 'root[0][2][0][0][3]', 'root[0][3][1][2][1]', + 'root[0][3][1][2][2]', 'root[1][2][1][2][3]', 'root[1][0][0][1][2]', + 'root[1][0][0][2][1]', 'root[1][3][1][2][1]', 'root[1][3][1][2][2]' + } + + @pytest.fixture(scope='class') def nested_b_t1(): with open(os.path.join(FIXTURES_DIR, 'nested_b_t1.json')) as the_file: diff --git a/deepdiff/deephash.py b/deepdiff/deephash.py index 0158c3ae..df7faa1c 100644 --- a/deepdiff/deephash.py +++ b/deepdiff/deephash.py @@ -9,7 +9,7 @@ convert_item_or_items_into_compiled_regexes_else_none, get_id, type_is_subclass_of_type_group, type_in_type_group, number_to_string, datetime_normalize, KEY_TO_VAL_STR, short_repr, - get_truncate_datetime, dict_) + get_truncate_datetime, dict_, add_root_to_paths) from deepdiff.base import Base logger = logging.getLogger(__name__) @@ -88,11 +88,11 @@ def prepare_string_for_hashing( err = er if not encoded: obj_decoded = obj.decode('utf-8', errors='ignore') - start = min(err.start - 10, 0) + start = max(err.start - 20, 0) start_prefix = '' if start > 0: start_prefix = '...' - end = err.end + 10 + end = err.end + 20 end_suffix = '...' if end >= len(obj): end = len(obj) @@ -123,6 +123,7 @@ def __init__(self, hashes=None, exclude_types=None, exclude_paths=None, + include_paths=None, exclude_regex_paths=None, hasher=None, ignore_repetition=True, @@ -146,7 +147,7 @@ def __init__(self, raise ValueError( ("The following parameter(s) are not valid: %s\n" "The valid parameters are obj, hashes, exclude_types, significant_digits, truncate_datetime," - "exclude_paths, exclude_regex_paths, hasher, ignore_repetition, " + "exclude_paths, include_paths, exclude_regex_paths, hasher, ignore_repetition, " "number_format_notation, apply_hash, ignore_type_in_groups, ignore_string_type_changes, " "ignore_numeric_type_changes, ignore_type_subclasses, ignore_string_case " "number_to_string_func, ignore_private_variables, parent " @@ -160,7 +161,8 @@ def __init__(self, exclude_types = set() if exclude_types is None else set(exclude_types) self.exclude_types_tuple = tuple(exclude_types) # we need tuple for checking isinstance self.ignore_repetition = ignore_repetition - self.exclude_paths = convert_item_or_items_into_set_else_none(exclude_paths) + self.exclude_paths = add_root_to_paths(convert_item_or_items_into_set_else_none(exclude_paths)) + self.include_paths = add_root_to_paths(convert_item_or_items_into_set_else_none(include_paths)) self.exclude_regex_paths = convert_item_or_items_into_compiled_regexes_else_none(exclude_regex_paths) self.hasher = default_hasher if hasher is None else hasher self.hashes[UNPROCESSED_KEY] = [] @@ -327,6 +329,13 @@ def _skip_this(self, obj, parent): skip = False if self.exclude_paths and parent in self.exclude_paths: skip = True + if self.include_paths and parent != 'root': + if parent not in self.include_paths: + skip = True + for prefix in self.include_paths: + if parent.startswith(prefix): + skip = False + break elif self.exclude_regex_paths and any( [exclude_regex_path.search(parent) for exclude_regex_path in self.exclude_regex_paths]): skip = True diff --git a/deepdiff/delta.py b/deepdiff/delta.py index 6a94f15b..2a65be7d 100644 --- a/deepdiff/delta.py +++ b/deepdiff/delta.py @@ -7,7 +7,8 @@ from deepdiff.helper import ( strings, short_repr, numbers, np_ndarray, np_array_factory, numpy_dtypes, get_doc, - not_found, numpy_dtype_string_to_type, dict_) + not_found, numpy_dtype_string_to_type, dict_, +) from deepdiff.path import _path_to_elements, _get_nested_obj, GET, GETATTR from deepdiff.anyset import AnySet @@ -70,11 +71,11 @@ def __init__( serializer=pickle_dump, verify_symmetry=False, ): - if 'safe_to_import' not in set(deserializer.__code__.co_varnames): + if hasattr(deserializer, '__code__') and 'safe_to_import' in set(deserializer.__code__.co_varnames): + _deserializer = deserializer + else: def _deserializer(obj, safe_to_import=None): return deserializer(obj) - else: - _deserializer = deserializer if diff is not None: if isinstance(diff, DeepDiff): diff --git a/deepdiff/diff.py b/deepdiff/diff.py index 363b1f53..ba70acac 100755 --- a/deepdiff/diff.py +++ b/deepdiff/diff.py @@ -21,15 +21,15 @@ type_is_subclass_of_type_group, type_in_type_group, get_doc, number_to_string, datetime_normalize, KEY_TO_VAL_STR, booleans, np_ndarray, get_numpy_ndarray_rows, OrderedSetPlus, RepeatedTimer, - TEXT_VIEW, TREE_VIEW, DELTA_VIEW, detailed__dict__, + TEXT_VIEW, TREE_VIEW, DELTA_VIEW, detailed__dict__, add_root_to_paths, np, get_truncate_datetime, dict_, CannotCompare, ENUM_IGNORE_KEYS) from deepdiff.serialization import SerializationMixin from deepdiff.distance import DistanceMixin from deepdiff.model import ( RemapDict, ResultDict, TextResult, TreeResult, DiffLevel, - DictRelationship, AttributeRelationship, + DictRelationship, AttributeRelationship, REPORT_KEYS, SubscriptableIterableRelationship, NonSubscriptableIterableRelationship, - SetRelationship, NumpyArrayRelationship, CUSTOM_FIELD) + SetRelationship, NumpyArrayRelationship, CUSTOM_FIELD, PrettyOrderedSet, ) from deepdiff.deephash import DeepHash, combine_hashes_lists from deepdiff.base import Base from deepdiff.lfucache import LFUCache, DummyLFU @@ -85,6 +85,7 @@ def _report_progress(_stats, progress_logger, duration): DEEPHASH_PARAM_KEYS = ( 'exclude_types', 'exclude_paths', + 'include_paths', 'exclude_regex_paths', 'hasher', 'significant_digits', @@ -119,6 +120,7 @@ def __init__(self, exclude_obj_callback=None, exclude_obj_callback_strict=None, exclude_paths=None, + include_paths=None, exclude_regex_paths=None, exclude_types=None, get_deep_distance=False, @@ -157,7 +159,7 @@ def __init__(self, raise ValueError(( "The following parameter(s) are not valid: %s\n" "The valid parameters are ignore_order, report_repetition, significant_digits, " - "number_format_notation, exclude_paths, exclude_types, exclude_regex_paths, ignore_type_in_groups, " + "number_format_notation, exclude_paths, include_paths, exclude_types, exclude_regex_paths, ignore_type_in_groups, " "ignore_string_type_changes, ignore_numeric_type_changes, ignore_type_subclasses, truncate_datetime, " "ignore_private_variables, ignore_nan_inequality, number_to_string_func, verbose_level, " "view, hasher, hashes, max_passes, max_diffs, " @@ -188,7 +190,8 @@ def __init__(self, ignore_numeric_type_changes=ignore_numeric_type_changes, ignore_type_subclasses=ignore_type_subclasses) self.report_repetition = report_repetition - self.exclude_paths = convert_item_or_items_into_set_else_none(exclude_paths) + self.exclude_paths = add_root_to_paths(convert_item_or_items_into_set_else_none(exclude_paths)) + self.include_paths = add_root_to_paths(convert_item_or_items_into_set_else_none(include_paths)) self.exclude_regex_paths = convert_item_or_items_into_compiled_regexes_else_none(exclude_regex_paths) self.exclude_types = set(exclude_types) if exclude_types else None self.exclude_types_tuple = tuple(exclude_types) if exclude_types else None # we need tuple for checking isinstance @@ -431,21 +434,29 @@ def _skip_this(self, level): Check whether this comparison should be skipped because one of the objects to compare meets exclusion criteria. :rtype: bool """ + level_path = level.path() skip = False - if self.exclude_paths and level.path() in self.exclude_paths: + if self.exclude_paths and level_path in self.exclude_paths: skip = True + if self.include_paths and level_path != 'root': + if level_path not in self.include_paths: + skip = True + for prefix in self.include_paths: + if level_path.startswith(prefix): + skip = False + break elif self.exclude_regex_paths and any( - [exclude_regex_path.search(level.path()) for exclude_regex_path in self.exclude_regex_paths]): + [exclude_regex_path.search(level_path) for exclude_regex_path in self.exclude_regex_paths]): skip = True elif self.exclude_types_tuple and \ (isinstance(level.t1, self.exclude_types_tuple) or isinstance(level.t2, self.exclude_types_tuple)): skip = True elif self.exclude_obj_callback and \ - (self.exclude_obj_callback(level.t1, level.path()) or self.exclude_obj_callback(level.t2, level.path())): + (self.exclude_obj_callback(level.t1, level_path) or self.exclude_obj_callback(level.t2, level_path)): skip = True elif self.exclude_obj_callback_strict and \ - (self.exclude_obj_callback_strict(level.t1, level.path()) and - self.exclude_obj_callback_strict(level.t2, level.path())): + (self.exclude_obj_callback_strict(level.t1, level_path) and + self.exclude_obj_callback_strict(level.t2, level_path)): skip = True return skip @@ -477,12 +488,12 @@ def _get_clean_to_keys_mapping(self, keys, level): return result def _diff_dict(self, - level, - parents_ids=frozenset([]), - print_as_attribute=False, - override=False, - override_t1=None, - override_t2=None): + level, + parents_ids=frozenset([]), + print_as_attribute=False, + override=False, + override_t1=None, + override_t2=None): """Difference of 2 dictionaries""" if override: # for special stuff like custom objects and named tuples we receive preprocessed t1 and t2 @@ -1097,7 +1108,7 @@ def get_other_pair(hash_value, in_t1=True): old_indexes=t1_indexes, new_indexes=t2_indexes) self._report_result('repetition_change', - repetition_change_level) + repetition_change_level) else: for hash_value in hashes_added: @@ -1423,6 +1434,69 @@ def get_stats(self): """ return self._stats + @property + def affected_paths(self): + """ + Get the list of paths that were affected. + Whether a value was changed or they were added or removed. + + Example + >>> t1 = {1: 1, 2: 2, 3: [3], 4: 4} + >>> t2 = {1: 1, 2: 4, 3: [3, 4], 5: 5, 6: 6} + >>> ddiff = DeepDiff(t1, t2) + >>> ddiff + >>> pprint(ddiff, indent=4) + { 'dictionary_item_added': [root[5], root[6]], + 'dictionary_item_removed': [root[4]], + 'iterable_item_added': {'root[3][1]': 4}, + 'values_changed': {'root[2]': {'new_value': 4, 'old_value': 2}}} + >>> ddiff.affected_paths + OrderedSet(['root[3][1]', 'root[4]', 'root[5]', 'root[6]', 'root[2]']) + >>> ddiff.affected_root_keys + OrderedSet([3, 4, 5, 6, 2]) + + """ + result = OrderedSet() + for key in REPORT_KEYS: + value = self.get(key) + if value: + if isinstance(value, PrettyOrderedSet): + result |= value + else: + result |= OrderedSet(value.keys()) + return result + + @property + def affected_root_keys(self): + """ + Get the list of root keys that were affected. + Whether a value was changed or they were added or removed. + + Example + >>> t1 = {1: 1, 2: 2, 3: [3], 4: 4} + >>> t2 = {1: 1, 2: 4, 3: [3, 4], 5: 5, 6: 6} + >>> ddiff = DeepDiff(t1, t2) + >>> ddiff + >>> pprint(ddiff, indent=4) + { 'dictionary_item_added': [root[5], root[6]], + 'dictionary_item_removed': [root[4]], + 'iterable_item_added': {'root[3][1]': 4}, + 'values_changed': {'root[2]': {'new_value': 4, 'old_value': 2}}} + >>> ddiff.affected_paths + OrderedSet(['root[3][1]', 'root[4]', 'root[5]', 'root[6]', 'root[2]']) + >>> ddiff.affected_root_keys + OrderedSet([3, 4, 5, 6, 2]) + """ + result = OrderedSet() + for key in REPORT_KEYS: + value = self.tree.get(key) + if value: + if isinstance(value, PrettyOrderedSet): + result |= OrderedSet([i.get_root_key() for i in value]) + else: + result |= OrderedSet([i.get_root_key() for i in value.keys()]) + return result + if __name__ == "__main__": # pragma: no cover import doctest diff --git a/deepdiff/helper.py b/deepdiff/helper.py index f5a6bc88..2ff8aeb5 100644 --- a/deepdiff/helper.py +++ b/deepdiff/helper.py @@ -5,10 +5,11 @@ import uuid import logging import warnings +import string import time from ast import literal_eval from decimal import Decimal, localcontext -from collections import namedtuple, OrderedDict +from collections import namedtuple from itertools import repeat from ordered_set import OrderedSet from threading import Timer @@ -41,6 +42,7 @@ class np_type: np_complex64 = np_type # pragma: no cover. np_complex128 = np_type # pragma: no cover. np_complex_ = np_type # pragma: no cover. + np_complexfloating = np_type # pragma: no cover. else: np_array_factory = np.array np_ndarray = np.ndarray @@ -61,6 +63,7 @@ class np_type: np_complex64 = np.complex64 np_complex128 = np.complex128 np_complex_ = np.complex_ + np_complexfloating = np.complexfloating numpy_numbers = ( np_int8, np_int16, np_int32, np_int64, np_uint8, @@ -68,6 +71,10 @@ class np_type: np_float32, np_float64, np_float_, np_complex64, np_complex128, np_complex_,) +numpy_complex_numbers = ( + np_complexfloating, np_complex64, np_complex128, np_complex_, +) + numpy_dtypes = set(numpy_numbers) numpy_dtypes.add(np_bool_) @@ -87,6 +94,8 @@ class np_type: py4 = py_major_version == 4 +NUMERICS = frozenset(string.digits) + # we used to use OrderedDictPlus when dictionaries in Python were not ordered. dict_ = dict @@ -102,6 +111,7 @@ class np_type: strings = (str, bytes) # which are both basestring unicode_type = str bytes_type = bytes +only_complex_number = (complex,) + numpy_complex_numbers only_numbers = (int, float, complex, Decimal) + numpy_numbers datetimes = (datetime.datetime, datetime.date, datetime.timedelta, datetime.time) uuids = (uuid.UUID) @@ -115,8 +125,6 @@ class np_type: ID_PREFIX = '!>*id' -ZERO_DECIMAL_CHARACTERS = set("-0.") - KEY_TO_VAL_STR = "{}:{}" TREE_VIEW = 'tree' @@ -220,28 +228,6 @@ class indexed_set(set): """ -JSON_CONVERTOR = { - Decimal: float, - OrderedSet: list, - type: lambda x: x.__name__, - bytes: lambda x: x.decode('utf-8') -} - - -def json_convertor_default(default_mapping=None): - _convertor_mapping = JSON_CONVERTOR.copy() - if default_mapping: - _convertor_mapping.update(default_mapping) - - def _convertor(obj): - for original_type, convert_to in _convertor_mapping.items(): - if isinstance(obj, original_type): - return convert_to(obj) - raise TypeError('We do not know how to convert {} of type {} for json serialization. Please pass the default_mapping parameter with proper mapping of the object to a basic python type.'.format(obj, type(obj))) - - return _convertor - - def add_to_frozen_set(parents_ids, item_id): return parents_ids | {item_id} @@ -257,6 +243,31 @@ def convert_item_or_items_into_set_else_none(items): return items +def add_root_to_paths(paths): + """ + Sometimes the users want to just pass + [key] instead of root[key] for example. + Here we automatically add all sorts of variations that might match + the path they were supposed to pass. + """ + if paths is None: + return + result = OrderedSet() + for path in paths: + if path.startswith('root'): + result.add(path) + else: + if path.isdigit(): + result.add(f"root['{path}']") + result.add(f"root[{path}]") + elif path[0].isdigit(): + result.add(f"root['{path}']") + else: + result.add(f"root.{path}") + result.add(f"root['{path}']") + return result + + RE_COMPILED_TYPE = type(re.compile('')) @@ -323,20 +334,51 @@ def number_to_string(number, significant_digits, number_format_notation="f"): using = number_formatting[number_format_notation] except KeyError: raise ValueError("number_format_notation got invalid value of {}. The valid values are 'f' and 'e'".format(number_format_notation)) from None - if isinstance(number, Decimal): - tup = number.as_tuple() + + if not isinstance(number, numbers): + return number + elif isinstance(number, Decimal): with localcontext() as ctx: - ctx.prec = len(tup.digits) + tup.exponent + significant_digits + # Precision = number of integer digits + significant_digits + # Using number//1 to get the integer part of the number + ctx.prec = len(str(abs(number // 1))) + significant_digits number = number.quantize(Decimal('0.' + '0' * significant_digits)) - elif not isinstance(number, numbers): - return number + elif isinstance(number, only_complex_number): + # Case for complex numbers. + number = number.__class__( + "{real}+{imag}j".format( + real=number_to_string( + number=number.real, + significant_digits=significant_digits, + number_format_notation=number_format_notation + ), + imag=number_to_string( + number=number.imag, + significant_digits=significant_digits, + number_format_notation=number_format_notation + ) + ) + ) + else: + number = round(number=number, ndigits=significant_digits) + + if significant_digits == 0: + number = int(number) + + if number == 0.0: + # Special case for 0: "-0.xx" should compare equal to "0.xx" + number = abs(number) + + # Cast number to string result = (using % significant_digits).format(number) - # Special case for 0: "-0.00" should compare equal to "0.00" - if set(result) <= ZERO_DECIMAL_CHARACTERS: - result = "0.00" # https://bugs.python.org/issue36622 - if number_format_notation == 'e' and isinstance(number, float): - result = result.replace('+0', '+') + if number_format_notation == 'e': + # Removing leading 0 for exponential part. + result = re.sub( + pattern=r'(?<=e(\+|\-))0(?=\d)+', + repl=r'', + string=result + ) return result diff --git a/deepdiff/model.py b/deepdiff/model.py index 5ea8175c..9ab4c9b4 100644 --- a/deepdiff/model.py +++ b/deepdiff/model.py @@ -621,6 +621,21 @@ def all_down(self): def _format_result(root, result): return None if result is None else "{}{}".format(root, result) + def get_root_key(self, use_t2=False): + """ + Get the path's root key value for this change + + For example if the path to the element that is reported to have a change in value is root['X'][0] + then get_root_key should return 'X' + """ + root_level = self.all_up + if(use_t2): + next_rel = root_level.t2_child_rel + else: + next_rel = root_level.t1_child_rel or root_level.t2_child_rel # next relationship object to get a formatted param from + + return next_rel.param + def path(self, root="root", force=None, get_parent_too=False, use_t2=False, output_format='str'): """ A python syntax string describing how to descend to this level, assuming the top level object is called root. diff --git a/deepdiff/serialization.py b/deepdiff/serialization.py index 796c0fcb..9f776ec9 100644 --- a/deepdiff/serialization.py +++ b/deepdiff/serialization.py @@ -1,8 +1,9 @@ -import json import pickle import sys import io import os +import json +import uuid import logging import re # NOQA import builtins # NOQA @@ -23,8 +24,9 @@ except ImportError: # pragma: no cover. clevercsv = None # pragma: no cover. from copy import deepcopy +from functools import partial from collections.abc import Mapping -from deepdiff.helper import (strings, json_convertor_default, get_type, TEXT_VIEW) +from deepdiff.helper import (strings, get_type, TEXT_VIEW) from deepdiff.model import DeltaResult logger = logging.getLogger(__name__) @@ -69,6 +71,7 @@ class UnsupportedFormatErr(TypeError): 'datetime.time', 'datetime.timedelta', 'decimal.Decimal', + 'uuid.UUID', 'ordered_set.OrderedSet', 'collections.namedtuple', 'collections.OrderedDict', @@ -76,6 +79,34 @@ class UnsupportedFormatErr(TypeError): } +TYPE_STR_TO_TYPE = { + 'range': range, + 'complex': complex, + 'set': set, + 'frozenset': frozenset, + 'slice': slice, + 'str': str, + 'bytes': bytes, + 'list': list, + 'tuple': tuple, + 'int': int, + 'float': float, + 'dict': dict, + 'bool': bool, + 'bin': bin, + 'None': None, + 'NoneType': None, + 'datetime': datetime.datetime, + 'time': datetime.time, + 'timedelta': datetime.timedelta, + 'Decimal': decimal.Decimal, + 'OrderedSet': ordered_set.OrderedSet, + 'namedtuple': collections.namedtuple, + 'OrderedDict': collections.OrderedDict, + 'Pattern': re.Pattern, +} + + class ModuleNotFoundError(ImportError): """ Raised when the module is not found in sys.modules @@ -465,3 +496,57 @@ def _save_content(content, path, file_type, keep_backup=True): raise UnsupportedFormatErr('Only json, yaml, toml, csv, tsv and pickle are supported.\n' f' The {file_type} extension is not known.') return content + + +JSON_CONVERTOR = { + decimal.Decimal: float, + ordered_set.OrderedSet: list, + type: lambda x: x.__name__, + bytes: lambda x: x.decode('utf-8'), + datetime.datetime: lambda x: x.isoformat(), + uuid.UUID: lambda x: str(x), +} + + +def json_convertor_default(default_mapping=None): + if default_mapping: + _convertor_mapping = JSON_CONVERTOR.copy() + _convertor_mapping.update(default_mapping) + else: + _convertor_mapping = JSON_CONVERTOR + + def _convertor(obj): + for original_type, convert_to in _convertor_mapping.items(): + if isinstance(obj, original_type): + return convert_to(obj) + raise TypeError('We do not know how to convert {} of type {} for json serialization. Please pass the default_mapping parameter with proper mapping of the object to a basic python type.'.format(obj, type(obj))) + + return _convertor + + +class JSONDecoder(json.JSONDecoder): + + def __init__(self, *args, **kwargs): + json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs) + + def object_hook(self, obj): + if 'old_type' in obj and 'new_type' in obj: + for type_key in ('old_type', 'new_type'): + type_str = obj[type_key] + obj[type_key] = TYPE_STR_TO_TYPE.get(type_str, type_str) + + return obj + + +def json_dumps(item, default_mapping=None, **kwargs): + """ + Dump json with extra details that are not normally json serializable + + Note: I tried to replace json with orjson for its speed. It does work + but the output it makes is a byte object and Postgres couldn't directly use it without + encoding to str. So I switched back to json. + """ + return json.dumps(item, default=json_convertor_default(default_mapping=default_mapping), **kwargs) + + +json_loads = partial(json.loads, cls=JSONDecoder) diff --git a/docs/deephash_doc.rst b/docs/deephash_doc.rst index 20f30992..82e8c361 100644 --- a/docs/deephash_doc.rst +++ b/docs/deephash_doc.rst @@ -32,6 +32,10 @@ exclude_paths: list, default = None List of paths to exclude from the report. If only one item, you can path it as a string instead of a list containing only one path. +include_paths: list, default = None + List of the only paths to include in the report. If only one item, you can path it as a string. + + exclude_regex_paths: list, default = None List of string regex paths or compiled regex paths objects to exclude from the report. If only one item, you can path it as a string instead of a list containing only one regex path. diff --git a/docs/diff_doc.rst b/docs/diff_doc.rst index 1958630e..f7a56ebd 100644 --- a/docs/diff_doc.rst +++ b/docs/diff_doc.rst @@ -43,6 +43,10 @@ exclude_paths: list, default = None :ref:`exclude_paths_label` List of paths to exclude from the report. If only one item, you can path it as a string. +include_paths: list, default = None + :ref:`include_paths_label` + List of the only paths to include in the report. If only one item, you can path it as a string. + exclude_regex_paths: list, default = None :ref:`exclude_regex_paths_label` List of string regex paths or compiled regex paths objects to exclude from the report. If only one item, you can pass it as a string or regex compiled object. diff --git a/docs/exclude_paths.rst b/docs/exclude_paths.rst index 2cc501ef..8e870d05 100644 --- a/docs/exclude_paths.rst +++ b/docs/exclude_paths.rst @@ -16,6 +16,49 @@ Example >>> print (DeepDiff(t1, t2, exclude_paths=["root['ingredients']", "root['ingredients2']"])) # multiple items pass as a list or a set. {} +Also for root keys you don't have to pass as "root['key']". You can instead just pass the key: + +Example + >>> t1 = {"for life": "vegan", "ingredients": ["no meat", "no eggs", "no dairy"]} + >>> t2 = {"for life": "vegan", "ingredients": ["veggies", "tofu", "soy sauce"]} + >>> print (DeepDiff(t1, t2, exclude_paths="ingredients)) # one item pass it as a string + {} + >>> print (DeepDiff(t1, t2, exclude_paths=["ingredients", "ingredients2"])) # multiple items pass as a list or a set. + {} + + +.. _include_paths_label: + +Include Paths +============= + +Only include this part of your object tree in the comparison. +Use include_paths and pass a set or list of paths to limit diffing to only those paths. If only one item is being passed, just put it there as a string—no need to pass it as a list then. + +Example + >>> t1 = {"for life": "vegan", "ingredients": ["no meat", "no eggs", "no dairy"]} + >>> t2 = {"for life": "vegan", "ingredients": ["veggies", "tofu", "soy sauce"]} + >>> print (DeepDiff(t1, t2, include_paths="root['for life']")) # one item pass it as a string + {} + >>> print (DeepDiff(t1, t2, include_paths=["for life", "ingredients2"])) # multiple items pass as a list or a set and you don't need to pass the full path when dealing with root keys. So instead of "root['for life']" you can pass "for life" + {} + + +When passing include_paths, all the children of that path will be included too. + +Example + >>> t1 = { + ... "foo": {"bar": "potato"}, + ... "ingredients": ["no meat", "no eggs", "no dairy"] + ... } + >>> t2 = { + ... "foo": {"bar": "banana"}, + ... "ingredients": ["bread", "cheese"] + ... } + >>> DeepDiff(t1, t2, include_paths="foo") + {'values_changed': {"root['foo']['bar']": {'new_value': 'banana', 'old_value': 'potato'}}} + + .. _exclude_regex_paths_label: Exclude Regex Paths diff --git a/setup.py b/setup.py index 123ddf50..cfbddac6 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ def get_reqs(filename): long_description = file.read() -setup(name='deepdiff6', +setup(name='deepdiff', version=version, description='Deep Difference and Search of any Python object/data.', url='https://github.com/seperman/deepdiff', diff --git a/tests/test_cache.py b/tests/test_cache.py index 9a6ad59b..e9779b42 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -7,7 +7,7 @@ class TestCache: @pytest.mark.slow - def test_cache_deeply_nested_a1(self, nested_a_t1, nested_a_t2, nested_a_result): + def test_cache_deeply_nested_a1(self, nested_a_t1, nested_a_t2, nested_a_result, nested_a_affected_paths): diff = DeepDiff(nested_a_t1, nested_a_t2, ignore_order=True, cache_size=5000, cache_tuning_sample_size=280, @@ -25,6 +25,8 @@ def test_cache_deeply_nested_a1(self, nested_a_t1, nested_a_t2, nested_a_result) assert nested_a_result == diff diff_of_diff = DeepDiff(nested_a_result, diff.to_dict(), ignore_order=False) assert not diff_of_diff + assert nested_a_affected_paths == diff.affected_paths + assert [0, 1] == diff.affected_root_keys @pytest.mark.slow def test_cache_deeply_nested_a2(self, nested_a_t1, nested_a_t2, nested_a_result): diff --git a/tests/test_delta.py b/tests/test_delta.py index 27a37c3d..4bb7329f 100644 --- a/tests/test_delta.py +++ b/tests/test_delta.py @@ -16,7 +16,8 @@ INVALID_ACTION_WHEN_CALLING_SIMPLE_DELETE_ELEM, INDEXES_NOT_FOUND_WHEN_IGNORE_ORDER, FAIL_TO_REMOVE_ITEM_IGNORE_ORDER_MSG, UNABLE_TO_GET_PATH_MSG, NOT_VALID_NUMPY_TYPE) from deepdiff.serialization import ( - DELTA_IGNORE_ORDER_NEEDS_REPETITION_REPORT, DELTA_ERROR_WHEN_GROUP_BY + DELTA_IGNORE_ORDER_NEEDS_REPETITION_REPORT, DELTA_ERROR_WHEN_GROUP_BY, + json_dumps, json_loads, ) from tests import PicklableClass, parameterize_cases, CustomClass, CustomClass2 @@ -24,6 +25,35 @@ class TestBasicsOfDelta: + def test_from_null_delta_json(self): + t1 = None + t2 = [1, 2, 3, 5] + diff = DeepDiff(t1, t2) + delta = Delta(diff, serializer=json_dumps) + dump = delta.dumps() + delta2 = Delta(dump, deserializer=json_loads) + assert delta2 + t1 == t2 + assert t1 + delta2 == t2 + + def test_to_null_delta1_json(self): + t1 = 1 + t2 = None + diff = DeepDiff(t1, t2) + delta = Delta(diff, serializer=json_dumps) + dump = delta.dumps() + delta2 = Delta(dump, deserializer=json_loads) + assert delta2 + t1 == t2 + assert t1 + delta2 == t2 + + def test_to_null_delta2_json(self): + t1 = [1, 2, 3, 5] + t2 = None + diff = DeepDiff(t1, t2) + delta = Delta(diff) + + assert delta + t1 == t2 + assert t1 + delta == t2 + def test_list_difference_add_delta(self): t1 = [1, 2] t2 = [1, 2, 3, 5] @@ -1145,6 +1175,7 @@ def test_delta_view_and_to_delta_dict_are_equal_when_parameteres_passed(self): 'ignore_type_in_groups': [], 'report_repetition': True, 'exclude_paths': None, + 'include_paths': None, 'exclude_regex_paths': None, 'exclude_types': None, 'exclude_types_tuple': None, diff --git a/tests/test_diff_text.py b/tests/test_diff_text.py index c9717a03..bc2c3e8c 100755 --- a/tests/test_diff_text.py +++ b/tests/test_diff_text.py @@ -82,20 +82,25 @@ def test_value_change(self): assert result == DeepDiff(t1, t2) def test_item_added_and_removed(self): - t1 = {1: 1, 2: 2, 3: 3, 4: 4} - t2 = {1: 1, 2: 4, 3: 3, 5: 5, 6: 6} + t1 = {1: 1, 2: 2, 3: [3], 4: 4} + t2 = {1: 1, 2: 4, 3: [3, 4], 5: 5, 6: 6} ddiff = DeepDiff(t1, t2) result = { - 'dictionary_item_added': {'root[5]', 'root[6]'}, - 'dictionary_item_removed': {'root[4]'}, + 'dictionary_item_added': ["root[5]", "root[6]"], + 'dictionary_item_removed': ["root[4]"], 'values_changed': { 'root[2]': { - "old_value": 2, - "new_value": 4 + 'new_value': 4, + 'old_value': 2 } + }, + 'iterable_item_added': { + 'root[3][1]': 4 } } assert result == ddiff + assert {'root[4]', 'root[5]', 'root[6]', 'root[3][1]', 'root[2]'} == ddiff.affected_paths + assert {4, 5, 6, 3, 2} == ddiff.affected_root_keys def test_item_added_and_removed_verbose(self): t1 = {1: 1, 3: 3, 4: 4} @@ -1301,6 +1306,15 @@ def test_skip_path2(self): ddiff = DeepDiff(t1, t2, exclude_paths={"root['ingredients']"}) assert {} == ddiff + def test_skip_path2_key_names(self): + t1 = { + "for life": "vegan", + "ingredients": ["no meat", "no eggs", "no dairy"] + } + t2 = {"for life": "vegan"} + ddiff = DeepDiff(t1, t2, exclude_paths={"ingredients"}) + assert {} == ddiff + def test_skip_path2_reverse(self): t1 = { "for life": "vegan", @@ -1310,6 +1324,43 @@ def test_skip_path2_reverse(self): ddiff = DeepDiff(t2, t1, exclude_paths={"root['ingredients']"}) assert {} == ddiff + def test_include_path3(self): + t1 = { + "for life": "vegan", + "ingredients": ["no meat", "no eggs", "no dairy"] + } + t2 = {"for life": "vegan"} + ddiff = DeepDiff(t2, t1, include_paths={"root['for_life']"}) + assert {} == ddiff + + def test_include_path3_with_just_key_names(self): + t1 = { + "for life": "vegan", + "ingredients": ["no meat", "no eggs", "no dairy"] + } + t2 = {"for life": "vegan"} + ddiff = DeepDiff(t1, t2, include_paths={"for_life"}) + assert {} == ddiff + + def test_include_path4_nested(self): + t1 = { + "foo": {"bar": "potato"}, + "ingredients": ["no meat", "no eggs", "no dairy"] + } + t2 = { + "foo": {"bar": "banana"}, + "ingredients": ["bread", "cheese"] + } + ddiff = DeepDiff(t1, t2, include_paths="foo") + assert { + 'values_changed': { + "root['foo']['bar']": { + 'new_value': 'banana', + 'old_value': 'potato' + } + } + } == ddiff + def test_skip_path4(self): t1 = { "for life": "vegan", @@ -1394,6 +1445,8 @@ def exclude_obj_callback_strict(obj, path): ddiff = DeepDiff(t1, t2, exclude_obj_callback_strict=exclude_obj_callback_strict) result = {'values_changed': {"root['x']": {'new_value': 12, 'old_value': 10}}} assert result == ddiff + assert {"root['x']"} == ddiff.affected_paths + assert {"x"} == ddiff.affected_root_keys def test_skip_str_type_in_dictionary(self): t1 = {1: {2: "a"}} @@ -1447,6 +1500,7 @@ def test_list_none_item_removed(self): 'iterable_item_removed': {'root[2]': None} } assert result == ddiff + assert {"root[2]"} == ddiff.affected_paths def test_non_subscriptable_iterable(self): def gen1(): @@ -1466,6 +1520,7 @@ def gen2(): # Note: In text-style results, we currently pretend this stuff is subscriptable for readability assert result == ddiff + assert {"root[2]"} == ddiff.affected_paths @pytest.mark.parametrize('t1, t2, params, expected_result', [ (float('nan'), float('nan'), {}, ['values_changed']), @@ -1594,6 +1649,7 @@ def test_group_by_not_list_of_dicts(self): diff = DeepDiff(t1, t2, group_by='id') expected = {'values_changed': {'root[1]': {'new_value': 3, 'old_value': 2}}} assert expected == diff + assert {"root[1]"} == diff.affected_paths def test_datetime_in_key(self): diff --git a/tests/test_hash.py b/tests/test_hash.py index c5a90905..9463f318 100755 --- a/tests/test_hash.py +++ b/tests/test_hash.py @@ -357,8 +357,8 @@ def test_same_sets_same_hash(self): assert t1_hash[get_id(t1)] == t2_hash[get_id(t2)] @pytest.mark.parametrize("t1, t2, significant_digits, number_format_notation, result", [ - ({0.012, 0.98}, {0.013, 0.99}, 1, "f", 'set:float:0.00,float:1.0'), - (100000, 100021, 3, "e", 'int:1.000e+05'), + ({0.012, 0.98}, {0.013, 0.99}, 1, "f", 'set:float:0.0,float:1.0'), + (100000, 100021, 3, "e", 'int:1.000e+5'), ]) def test_similar_significant_hash(self, t1, t2, significant_digits, number_format_notation, result): @@ -568,16 +568,18 @@ def test_skip_str_type_in_dict_on_list(self): assert 1 in t1_hash assert t1_hash[dic1] == t2_hash[dic2] - def test_skip_path(self): + def test_skip_path_in_hash(self): dic1 = {1: "a"} t1 = [dic1, 2] dic2 = {} t2 = [dic2, 2] t1_hash = DeepHashPrep(t1, exclude_paths=['root[0]']) t2_hash = DeepHashPrep(t2, exclude_paths='root[0]') + t2_hash_again = DeepHashPrep(t2, include_paths='1') assert 1 not in t1_hash assert 2 in t1_hash assert t1_hash[2] == t2_hash[2] + assert t1_hash[2] == t2_hash_again[2] def test_skip_path2(self): @@ -596,6 +598,23 @@ def test_skip_path2(self): t2_hash = DeepHashPrep(t2, exclude_paths=exclude_paths) assert t1_hash[t1] == t2_hash[t2] + def test_hash_include_path_nested(self): + + obj10 = {'a': 1, 'b': 'f', 'e': "1111", 'foo': {'bar': 'baz'}} + obj11 = {'c': 1, 'd': 'f', 'e': 'Cool'} + + obj20 = {'a': 1, 'b': 'f', 'e': 'Cool', 'foo': {'bar': 'baz'}} + obj21 = {'c': 1, 'd': 'f', 'e': "2222"} + + t1 = [obj10, obj11] + t2 = [obj20, obj21] + + include_paths = ["root[0]['foo']['bar']"] + + t1_hash = DeepHashPrep(t1, include_paths=include_paths) + t2_hash = DeepHashPrep(t2, include_paths=include_paths) + assert t1_hash[t1] == t2_hash[t2] + def test_skip_regex_path(self): dic1 = {1: "a"} t1 = [dic1, 2] @@ -805,9 +824,14 @@ def test_combine_hashes_lists(self, items, prefix, expected): "Please either pass ignore_encoding_errors=True or pass the encoding via encodings=['utf-8', '...'].") EXPECTED_MESSAGE2 = ( - "'utf-8' codec can't decode byte 0xbc in position 0: invalid start byte in 'p of flo...'. " + "'utf-8' codec can't decode byte 0xbc in position 0: invalid start byte in ' cup of flour'. " "Please either pass ignore_encoding_errors=True or pass the encoding via encodings=['utf-8', '...'].") + EXPECTED_MESSAGE3 = ( + "'utf-8' codec can't decode byte 0xc3 in position 34: invalid continuation byte in '...up of potatos. Then ( cup of flour'. Please either pass ignore_encoding_errors=True or " + "pass the encoding via encodings=['utf-8', '...']." + ) + @pytest.mark.parametrize('test_num, item, encodings, ignore_encoding_errors, expected_result, expected_message', [ (1, b'\xc3\x28', None, False, UnicodeDecodeError, EXPECTED_MESSAGE1), (2, b'\xc3\x28', ['utf-8'], False, UnicodeDecodeError, EXPECTED_MESSAGE1), @@ -815,8 +839,9 @@ def test_combine_hashes_lists(self, items, prefix, expected): (4, b"\xbc cup of flour", ['utf-8'], False, UnicodeDecodeError, EXPECTED_MESSAGE2), (5, b"\xbc cup of flour", ['utf-8'], True, {b'\xbc cup of flour': '86ac12eb5e35db88cf93baca1d62098023b2d93d634e75fb4e37657e514f3d51'}, None), (6, b"\xbc cup of flour", ['utf-8', 'latin-1'], False, {b'\xbc cup of flour': 'cfc354ae2232a8983bf59b2004f44fcb4036f57df1d08b9cde9950adea3f8d3e'}, None), + (7, b"First have a cup of potatos. Then \xc3\x28 cup of flour", None, False, UnicodeDecodeError, EXPECTED_MESSAGE3), ]) - def test_encodings(self, test_num, item, encodings, ignore_encoding_errors, expected_result, expected_message): + def test_hash_encodings(self, test_num, item, encodings, ignore_encoding_errors, expected_result, expected_message): if UnicodeDecodeError == expected_result: with pytest.raises(expected_result) as exc_info: DeepHash(item, encodings=encodings, ignore_encoding_errors=ignore_encoding_errors) diff --git a/tests/test_helper.py b/tests/test_helper.py index b0b0b628..282e7f92 100644 --- a/tests/test_helper.py +++ b/tests/test_helper.py @@ -9,7 +9,7 @@ cartesian_product_of_shape, literal_eval_extended, not_found, OrderedSetPlus, diff_numpy_array, cartesian_product_numpy, get_truncate_datetime, datetime_normalize, - detailed__dict__, ENUM_IGNORE_KEYS, + detailed__dict__, ENUM_IGNORE_KEYS, add_root_to_paths, ) @@ -33,30 +33,160 @@ def test_short_repr_when_long(self): output = short_repr(item) assert output == "{'Eat more':...}" - @pytest.mark.parametrize("t1, t2, significant_digits, expected_result", + @pytest.mark.parametrize("t1, t2, significant_digits, number_format_notation, expected_result", [ - (10, 10.0, 5, True), - (10, 10.2, 5, ('10.00000', '10.20000')), - (10, 10.2, 0, True), - (Decimal(10), 10, 0, True), - (Decimal(10), 10, 10, True), - (Decimal(10), 10.0, 0, True), - (Decimal(10), 10.0, 10, True), - (Decimal('10.0'), 10.0, 5, True), - (Decimal('10.01'), 10.01, 1, True), - (Decimal('10.01'), 10.01, 2, True), - (Decimal('10.01'), 10.01, 5, True), - (Decimal('10.01'), 10.01, 8, True), - (Decimal('10.010'), 10.01, 3, True), - (Decimal('100000.1'), 100000.1, 0, True), - (Decimal('100000.1'), 100000.1, 1, True), - (Decimal('100000.1'), 100000.1, 5, True), - (Decimal('100000'), 100000.1, 0, True), - (Decimal('100000'), 100000.1, 1, ('100000.0', '100000.1')), + (10, 10.0, 5, "f", True), + (10, 10.0, 5, "e", True), + (10, 10.2, 5, "f", ('10.00000', '10.20000')), + (10, 10.2, 5, "e", ('1.00000e+1', '1.02000e+1')), + (10, 10.2, 0, "f", True), + (10, 10.2, 0, "e", True), + (Decimal(10), 10, 0, "f", True), + (Decimal(10), 10, 0, "e", True), + (Decimal(10), 10, 10, "f", True), + (Decimal(10), 10, 10, "e", True), + (Decimal(10), 10.0, 0, "f", True), + (Decimal(10), 10.0, 0, "e", True), + (Decimal(10), 10.0, 10, "f", True), + (Decimal(10), 10.0, 10, "e", True), + (Decimal('10.0'), 10.0, 5, "f", True), + (Decimal('10.0'), 10.0, 5, "e", True), + (Decimal('10.01'), 10.01, 1, "f", True), + (Decimal('10.01'), 10.01, 1, "e", True), + (Decimal('10.01'), 10.01, 2, "f", True), + (Decimal('10.01'), 10.01, 2, "e", True), + (Decimal('10.01'), 10.01, 5, "f", True), + (Decimal('10.01'), 10.01, 5, "e", True), + (Decimal('10.01'), 10.01, 8, "f", True), + (Decimal('10.01'), 10.01, 8, "e", True), + (Decimal('10.010'), 10.01, 3, "f", True), + (Decimal('10.010'), 10.01, 3, "e", True), + (Decimal('100000.1'), 100000.1, 0, "f", True), + (Decimal('100000.1'), 100000.1, 0, "e", True), + (Decimal('100000.1'), 100000.1, 1, "f", True), + (Decimal('100000.1'), 100000.1, 1, "e", True), + (Decimal('100000.1'), 100000.1, 5, "f", True), + (Decimal('100000.1'), 100000.1, 5, "e", True), + (Decimal('100000'), 100000.1, 0, "f", True), + (Decimal('100000'), 100000.1, 0, "e", True), + (Decimal('100000'), 100000.1, 1, "f", ('100000.0', '100000.1')), + (Decimal('100000'), 100000.1, 1, "e", True), + (Decimal('-100000'), 100000.1, 1, "f", ('-100000.0', '100000.1')), + (Decimal('-100000'), 100000.1, 1, "e", ("-1.0e+5","1.0e+5")), + (0, 0.0, 5, "f", True), + (0, 0.0, 5, "e", True), + (0, 0.2, 5, "f", ('0.00000', '0.20000')), + (0, 0.2, 5, "e", ('0.00000e+0', '2.00000e-1')), + (0, 0.2, 0, "f", True), + (0, 0.2, 0, "e", True), + (Decimal(0), 0, 0, "f", True), + (Decimal(0), 0, 0, "e", True), + (Decimal(0), 0, 10, "f", True), + (Decimal(0), 0, 10, "e", True), + (Decimal(0), 0.0, 0, "f", True), + (Decimal(0), 0.0, 0, "e", True), + (Decimal(0), 0.0, 10, "f", True), + (Decimal(0), 0.0, 10, "e", True), + (Decimal('0.0'), 0.0, 5, "f", True), + (Decimal('0.0'), 0.0, 5, "e", True), + (Decimal('0.01'), 0.01, 1, "f", True), + (Decimal('0.01'), 0.01, 1, "e", True), + (Decimal('0.01'), 0.01, 2, "f", True), + (Decimal('0.01'), 0.01, 2, "e", True), + (Decimal('0.01'), 0.01, 5, "f", True), + (Decimal('0.01'), 0.01, 5, "e", True), + (Decimal('0.01'), 0.01, 8, "f", True), + (Decimal('0.01'), 0.01, 8, "e", True), + (Decimal('0.010'), 0.01, 3, "f", True), + (Decimal('0.010'), 0.01, 3, "e", True), + (Decimal('0.00002'), 0.00001, 0, "f", True), + (Decimal('0.00002'), 0.00001, 0, "e", True), + (Decimal('0.00002'), 0.00001, 1, "f", True), + (Decimal('0.00002'), 0.00001, 1, "e", True), + (Decimal('0.00002'), 0.00001, 5, "f", ('0.00002', '0.00001')), + (Decimal('0.00002'), 0.00001, 5, "e", ('2.00000e-5', '1.00000e-5')), + (Decimal('0.00002'), 0.00001, 6, "f", ('0.000020', '0.000010')), + (Decimal('0.00002'), 0.00001, 6, "e", ('2.000000e-5', '1.000000e-5')), + (Decimal('0'), 0.1, 0, "f", True), + (Decimal('0'), 0.1, 0, "e", True), + (Decimal('0'), 0.1, 1, "f", ('0.0', '0.1')), + (Decimal('0'), 0.1, 1, "e", ('0.0e+0', '1.0e-1')), + (-0, 0.0, 5, "f", True), + (-0, 0.0, 5, "e", True), + (-0, 0.2, 5, "f", ('0.00000', '0.20000')), + (-0, 0.2, 5, "e", ('0.00000e+0', '2.00000e-1')), + (-0, 0.2, 0, "f", True), + (-0, 0.2, 0, "e", True), + (Decimal(-0), 0, 0, "f", True), + (Decimal(-0), 0, 0, "e", True), + (Decimal(-0), 0, 10, "f", True), + (Decimal(-0), 0, 10, "e", True), + (Decimal(-0), 0.0, 0, "f", True), + (Decimal(-0), 0.0, 0, "e", True), + (Decimal(-0), 0.0, 10, "f", True), + (Decimal(-0), 0.0, 10, "e", True), + (Decimal('-0.0'), 0.0, 5, "f", True), + (Decimal('-0.0'), 0.0, 5, "e", True), + (Decimal('-0.01'), 0.01, 1, "f", True), + (Decimal('-0.01'), 0.01, 1, "e", True), + (Decimal('-0.01'), 0.01, 2, "f", ('-0.01', '0.01')), + (Decimal('-0.01'), 0.01, 2, "e", ('-1.00e-2', '1.00e-2')), + (Decimal('-0.00002'), 0.00001, 0, "f", True), + (Decimal('-0.00002'), 0.00001, 0, "e", True), + (Decimal('-0.00002'), 0.00001, 1, "f", True), + (Decimal('-0.00002'), 0.00001, 1, "e", True), + (Decimal('-0.00002'), 0.00001, 5, "f", ('-0.00002', '0.00001')), + (Decimal('-0.00002'), 0.00001, 5, "e", ('-2.00000e-5', '1.00000e-5')), + (Decimal('-0.00002'), 0.00001, 6, "f", ('-0.000020', '0.000010')), + (Decimal('-0.00002'), 0.00001, 6, "e", ('-2.000000e-5', '1.000000e-5')), + (Decimal('-0'), 0.1, 0, "f", True), + (Decimal('-0'), 0.1, 0, "e", True), + (Decimal('-0'), 0.1, 1, "f", ('0.0', '0.1')), + (Decimal('-0'), 0.1, 1, "e", ('0.0e+0', '1.0e-1')), ]) - def test_number_to_string_decimal_digits(self, t1, t2, significant_digits, expected_result): - st1 = number_to_string(t1, significant_digits=significant_digits, number_format_notation="f") - st2 = number_to_string(t2, significant_digits=significant_digits, number_format_notation="f") + def test_number_to_string_decimal_digits(self, t1, t2, significant_digits, number_format_notation, expected_result): + st1 = number_to_string(t1, significant_digits=significant_digits, number_format_notation=number_format_notation) + st2 = number_to_string(t2, significant_digits=significant_digits, number_format_notation=number_format_notation) + if expected_result is True: + assert st1 == st2 + else: + assert st1 == expected_result[0] + assert st2 == expected_result[1] + + @pytest.mark.parametrize("t1, t2, significant_digits, number_format_notation, expected_result", + [ + (10j, 10.0j, 5, "f", True), + (10j, 10.0j, 5, "e", True), + (4+10j, 4.0000002+10.0000002j, 5, "f", True), + (4+10j, 4.0000002+10.0000002j, 5, "e", True), + (4+10j, 4.0000002+10.0000002j, 7, "f", ('4.0000000+10.0000000j', '4.0000002+10.0000002j')), + (4+10j, 4.0000002+10.0000002j, 7, "e", ('4.0000000e+0+1.0000000e+1j', '4.0000002e+0+1.0000000e+1j')), + (0.00002+0.00002j, 0.00001+0.00001j, 0, "f", True), + (0.00002+0.00002j, 0.00001+0.00001j, 0, "e", True), + (0.00002+0.00002j, 0.00001+0.00001j, 5, "f", ('0.00002+0.00002j', '0.00001+0.00001j')), + (0.00002+0.00002j, 0.00001+0.00001j, 5, "e", ('2.00000e-5+2.00000e-5j', '1.00000e-5+1.00000e-5j')), + (-0.00002-0.00002j, 0.00001+0.00001j, 0, "f", True), + (-0.00002-0.00002j, 0.00001+0.00001j, 0, "e", True), + (10j, 10.2j, 5, "f", ('0.00000+10.00000j', '0.00000+10.20000j')), + (10j, 10.2j, 5, "e", ('0.00000e+0+1.00000e+1j', '0.00000e+0+1.02000e+1j')), + (10j, 10.2j, 0, "f", True), + (10j, 10.2j, 0, "e", True), + (0j, 0.0j, 5, "f", True), + (0j, 0.0j, 5, "e", True), + (0j, 0.2j, 5, "f", ('0.00000', '0.00000+0.20000j')), + (0j, 0.2j, 5, "e", ('0.00000e+0', '0.00000e+0+2.00000e-1j')), + (0j, 0.2j, 0, "f", True), + (0j, 0.2j, 0, "e", True), + (-0j, 0.0j, 5, "f", True), + (-0j, 0.0j, 5, "e", True), + (-0j, 0.2j, 5, "f", ('0.00000', '0.00000+0.20000j')), + (-0j, 0.2j, 5, "e", ('0.00000e+0', '0.00000e+0+2.00000e-1j')), + (-0j, 0.2j, 0, "f", True), + (-0j, 0.2j, 0, "e", True), + ]) + def test_number_to_string_complex_digits(self, t1, t2, significant_digits, number_format_notation, expected_result): + st1 = number_to_string(t1, significant_digits=significant_digits, number_format_notation=number_format_notation) + st2 = number_to_string(t2, significant_digits=significant_digits, number_format_notation=number_format_notation) if expected_result is True: assert st1 == st2 else: @@ -158,3 +288,12 @@ def test_datetime_normalize(self, truncate_datetime, obj, expected): def test_detailed__dict__(self, obj, ignore_keys, expected): result = detailed__dict__(obj, ignore_private_variables=True, ignore_keys=ignore_keys) assert expected == result, f"test_detailed__dict__ failed for {obj}" + + @pytest.mark.parametrize('test_num, value, expected', [ + (1, ['ab'], {'root.ab', "root['ab']"}), + (2, ['11'], {"root['11']", 'root[11]'}), + (3, ['1a'], {"root['1a']"}), + ]) + def test_add_root_to_paths(self, test_num, value, expected): + result = add_root_to_paths(value) + assert expected == result, f"test_add_root_to_paths #{test_num} failed." diff --git a/tests/test_ignore_order.py b/tests/test_ignore_order.py index 52016b3f..00f2b6ad 100644 --- a/tests/test_ignore_order.py +++ b/tests/test_ignore_order.py @@ -47,6 +47,7 @@ def test_ignore_order_depth3(self): t2 = [[{4, 5, 6}], {1, 2, 3}] ddiff = DeepDiff(t1, t2, ignore_order=True) assert {'set_item_added': ["root[1][0][6]"]} == ddiff + assert {"root[1][0][6]"} == ddiff.affected_paths def test_ignore_order_depth4(self): t1 = [[1, 2, 3, 4], [4, 2, 2, 1]] @@ -74,6 +75,7 @@ def test_ignore_order_depth5(self): } } assert expected == ddiff + assert {"root[1]", "root[2]", "root[3]"} == ddiff.affected_paths ddiff = DeepDiff(t1, t2, ignore_order=True, report_repetition=False, cache_purge_level=0) dist = ddiff._get_rough_distance() @@ -124,6 +126,7 @@ def test_dictionary_difference_ignore_order(self): t2 = {"a": [[{"b": 2, "c": 3}, {"b": 2, "c": 4}]]} ddiff = DeepDiff(t1, t2, ignore_order=True) assert {} == ddiff + assert set() == ddiff.affected_paths def test_nested_list_ignore_order(self): t1 = [1, 2, [3, 4]] @@ -190,6 +193,7 @@ def test_nested_list_ignore_order_report_repetition_wrong_currently(self): } } assert result != ddiff + assert {"root[2][0]"} == ddiff.affected_paths def test_list_of_unhashable_difference_ignore_order(self): t1 = [{"a": 2}, {"b": [3, 4, {1: 1}]}] @@ -1126,7 +1130,7 @@ class TestDecodingErrorIgnoreOrder: "Please either pass ignore_encoding_errors=True or pass the encoding via encodings=['utf-8', '...'].") EXPECTED_MESSAGE2 = ( - "'utf-8' codec can't decode byte 0xbc in position 0: Can not produce a hash for root: invalid start byte in 'p of flo...'. " + "'utf-8' codec can't decode byte 0xbc in position 0: Can not produce a hash for root: invalid start byte in ' cup of flour'. " "Please either pass ignore_encoding_errors=True or pass the encoding via encodings=['utf-8', '...'].") @pytest.mark.parametrize('test_num, item, encodings, ignore_encoding_errors, expected_result, expected_message', [ diff --git a/tests/test_serialization.py b/tests/test_serialization.py index c501aa5a..b19177c0 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -11,7 +11,7 @@ from deepdiff.serialization import ( pickle_load, pickle_dump, ForbiddenModule, ModuleNotFoundError, MODULE_NOT_FOUND_MSG, FORBIDDEN_MODULE_MSG, pretty_print_diff, - load_path_content, UnsupportedFormatErr) + load_path_content, UnsupportedFormatErr, json_dumps, json_loads) from conftest import FIXTURES_DIR from ordered_set import OrderedSet from tests import PicklableClass @@ -298,14 +298,14 @@ def test_pretty_print_diff_repetition_change(self, t1, t2, item_path): @pytest.mark.parametrize("expected, verbose_level", ( - ('Item root[5] added to dictionary.' - '\nItem root[3] removed from dictionary.' - '\nType of root[2] changed from int to str and value changed from 2 to "b".' - '\nValue of root[4] changed from 4 to 5.', 0), - ('Item root[5] (5) added to dictionary.' - '\nItem root[3] (3) removed from dictionary.' - '\nType of root[2] changed from int to str and value changed from 2 to "b".' - '\nValue of root[4] changed from 4 to 5.', 2), + ('Item root[5] added to dictionary.' + '\nItem root[3] removed from dictionary.' + '\nType of root[2] changed from int to str and value changed from 2 to "b".' + '\nValue of root[4] changed from 4 to 5.', 0), + ('Item root[5] (5) added to dictionary.' + '\nItem root[3] (3) removed from dictionary.' + '\nType of root[2] changed from int to str and value changed from 2 to "b".' + '\nValue of root[4] changed from 4 to 5.', 2), ), ids=("verbose=0", "verbose=2") ) def test_pretty_form_method(self, expected, verbose_level): @@ -314,3 +314,12 @@ def test_pretty_form_method(self, expected, verbose_level): ddiff = DeepDiff(t1, t2, view='tree', verbose_level=verbose_level) result = ddiff.pretty() assert result == expected + + @pytest.mark.parametrize('test_num, value', [ + (1, {'10': None}), + (2, {"type_changes": {"root": {"old_type": None, "new_type": list, "new_value": ["你好", 2, 3, 5]}}}), + ]) + def test_json_dumps_and_loads(self, test_num, value): + serialized = json_dumps(value) + back = json_loads(serialized) + assert value == back, f"test_json_dumps_and_loads tesst #{test_num} failed"