-
Notifications
You must be signed in to change notification settings - Fork 67
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
dictutils: adds utils for dot lookup and clear
* Adds a method "clear_none()" which helps removing None and empty list/dict values from the dictionary. * Adds a method "dict_lookup()" to support dot key string notation for looking up a key in a dictionary.
- Loading branch information
Showing
6 changed files
with
193 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,98 @@ | ||
# -*- coding: utf-8 -*- | ||
# | ||
# This file is part of Invenio. | ||
# Copyright (C) 2015-2020 CERN. | ||
# | ||
# Invenio is free software; you can redistribute it and/or modify it | ||
# under the terms of the MIT License; see LICENSE file for more details. | ||
|
||
"""Dictionary utilities.""" | ||
|
||
from copy import deepcopy | ||
|
||
|
||
def clear_none(d): | ||
"""Clear None values and empty dicts from a dict.""" | ||
del_keys = [] | ||
for k, v in d.items(): | ||
if v is None: | ||
del_keys.append(k) | ||
elif isinstance(v, dict): | ||
clear_none(v) | ||
if v == {}: | ||
del_keys.append(k) | ||
elif isinstance(v, list): | ||
clear_none_list(v) | ||
if v == []: | ||
del_keys.append(k) | ||
|
||
# Delete the keys (cannot be done during the dict iteration) | ||
for k in del_keys: | ||
del d[k] | ||
|
||
|
||
def clear_none_list(ls): | ||
"""Clear values from a list (in-place).""" | ||
del_idx = [] | ||
for i, v in enumerate(ls): | ||
if v is None: | ||
del_idx.append(i) | ||
elif isinstance(v, list): | ||
clear_none_list(v) | ||
if v == []: | ||
del_idx.append(i) | ||
elif isinstance(v, dict): | ||
clear_none(v) | ||
if v == {}: | ||
del_idx.append(i) | ||
|
||
# Delete the keys (reverse so index stays stable). | ||
for i in reversed(del_idx): | ||
del ls[i] | ||
|
||
|
||
def dict_lookup(source, lookup_key, parent=False): | ||
"""Make a lookup into a dict based on a dot notation. | ||
Examples of the supported dot notation: | ||
- ``'a'`` - Equivalent to ``source['a']`` | ||
- ``'a.b'`` - Equivalent to ``source['a']['b']`` | ||
- ``'a.b.0'`` - Equivalent to ``source['a']['b'][0]`` (for lists) | ||
List notation is also supported: | ||
- `['a']`` | ||
- ``['a','b']`` | ||
- ``['a','b', 0]`` | ||
:param source: The dictionary object to perform the lookup in. | ||
:param parent: If parent argument is True, returns the parent node of | ||
matched object. | ||
:param lookup_key: A string using dot notation, or a list of keys. | ||
""" | ||
# Copied from dictdiffer (CERN contributed part) and slightly modified. | ||
if not lookup_key: | ||
raise KeyError("No lookup key specified") | ||
|
||
# Parse the list of keys | ||
if isinstance(lookup_key, str): | ||
keys = lookup_key.split('.') | ||
elif isinstance(lookup_key, list): | ||
keys = lookup_key | ||
else: | ||
raise TypeError('lookup must be string or list') | ||
|
||
if parent: | ||
keys = keys[:-1] | ||
|
||
# Lookup the key | ||
value = source | ||
for key in keys: | ||
try: | ||
if isinstance(value, list): | ||
key = int(key) | ||
value = value[key] | ||
except (TypeError, IndexError) as exc: | ||
raise KeyError(lookup_key) from exc | ||
return value |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,65 @@ | ||
# -*- coding: utf-8 -*- | ||
# | ||
# This file is part of Invenio. | ||
# Copyright (C) 2020 CERN. | ||
# | ||
# Invenio is free software; you can redistribute it and/or modify it | ||
# under the terms of the MIT License; see LICENSE file for more details. | ||
|
||
"""Test of dictionary utilities.""" | ||
|
||
from copy import deepcopy | ||
|
||
import pytest | ||
|
||
from invenio_records.dictutils import clear_none, dict_lookup | ||
|
||
|
||
def test_clear_none(): | ||
"""Test clearning of the dictionary.""" | ||
d = { | ||
'a': None, | ||
'b': { | ||
'c': None | ||
}, | ||
'd': ['1', None, []], | ||
'e': [{'a': None, 'b': []}], | ||
} | ||
|
||
clear_none(d) | ||
# Modifications are done in place, so gotta test after the function call. | ||
assert d == {'d': ['1']} | ||
|
||
d = { | ||
'a': None, | ||
'b': [ | ||
{'a': '1', 'b': None}, | ||
], | ||
} | ||
clear_none(d) | ||
# Modifications are done in place, so gotta test after the function call. | ||
assert d == {'b': [{'a': '1'}]} | ||
|
||
|
||
def test_dict_lookup(): | ||
"""Test lookup by a key.""" | ||
d = { | ||
'a': 1, | ||
'b': { | ||
'c': None | ||
}, | ||
'd': ['1', '2'], | ||
} | ||
assert dict_lookup(d, 'a') == d['a'] | ||
assert dict_lookup(d, 'b') == d['b'] | ||
assert dict_lookup(d, 'b.c') == d['b']['c'] | ||
assert dict_lookup(d, 'd') == d['d'] | ||
assert dict_lookup(d, 'd.0') == d['d'][0] | ||
assert dict_lookup(d, 'd.1') == d['d'][1] | ||
assert dict_lookup(d, 'd.-1') == d['d'][-1] | ||
|
||
assert pytest.raises(KeyError, dict_lookup, d, 'x') | ||
assert pytest.raises(KeyError, dict_lookup, d, 'a.x') | ||
assert pytest.raises(KeyError, dict_lookup, d, 'b.x') | ||
assert pytest.raises(KeyError, dict_lookup, d, 'b.c.0') | ||
assert pytest.raises(KeyError, dict_lookup, d, 'd.3') |