From 3d778073c3355d9063e96807aa210ebad7328008 Mon Sep 17 00:00:00 2001 From: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Date: Wed, 24 Mar 2021 15:16:30 -0400 Subject: [PATCH 01/17] Add new resolvers `oc.dict.keys` and `oc.dict.values` Fixes #643 --- docs/source/usage.rst | 28 ++++++ news/643.feature | 2 + omegaconf/built_in_resolvers.py | 10 ++- omegaconf/omegaconf.py | 9 +- .../built_in_resolvers/test_dict.py | 87 +++++++++++++++++++ 5 files changed, 132 insertions(+), 4 deletions(-) create mode 100644 news/643.feature create mode 100644 tests/interpolation/built_in_resolvers/test_dict.py diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 0e56495e0..da9b16bb2 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -466,6 +466,34 @@ This can be useful for instance to parse environment variables: type: int, value: 3308 +Extracting lists of keys / values from a dictionary +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Some config options that are stored as a ``DictConfig`` may sometimes be easier to manipulate as lists, +when we care only about the keys or the associated values. + +The resolvers ``oc.dict.keys`` and ``oc.dict.values`` simplify such operations by extracting respectively +the list of keys and values from ``dict``-like objects like ``DictConfig``: + +.. doctest:: + + >>> cfg = OmegaConf.create( + ... { + ... "machines": { + ... "node007": "10.0.0.7", + ... "node012": "10.0.0.3", + ... "node075": "10.0.1.8", + ... }, + ... "nodes": "${oc.dict.keys:${machines}}", + ... "ips": "${oc.dict.values:${machines}}", + ... } + ... ) + >>> show(cfg.nodes) + type: ListConfig, value: ['node007', 'node012', 'node075'] + >>> show(cfg.ips) + type: ListConfig, value: ['10.0.0.7', '10.0.0.3', '10.0.1.8'] + + Custom interpolations ^^^^^^^^^^^^^^^^^^^^^ diff --git a/news/643.feature b/news/643.feature new file mode 100644 index 000000000..78143216d --- /dev/null +++ b/news/643.feature @@ -0,0 +1,2 @@ +New resolvers `oc.dict.keys` and `oc.dict.values` allow extracting the lists of keys and values of a DictConfig + diff --git a/omegaconf/built_in_resolvers.py b/omegaconf/built_in_resolvers.py index 373c01a4d..f253c669c 100644 --- a/omegaconf/built_in_resolvers.py +++ b/omegaconf/built_in_resolvers.py @@ -1,6 +1,6 @@ import os import warnings -from typing import Any, Optional +from typing import Any, Dict, List, Optional from ._utils import _DEFAULT_MARKER_, _get_value, decode_primitive from .base import Container @@ -28,6 +28,14 @@ def decode(expr: Optional[str], _parent_: Container) -> Any: return _get_value(val) +def dict_keys(in_dict: Dict[Any, Any]) -> List[Any]: + return list(in_dict.keys()) + + +def dict_values(in_dict: Dict[Any, Any]) -> List[Any]: + return list(in_dict.values()) + + def env(key: str, default: Any = _DEFAULT_MARKER_) -> Optional[str]: """ :param key: Environment variable key diff --git a/omegaconf/omegaconf.py b/omegaconf/omegaconf.py index 731da2f68..b2390c193 100644 --- a/omegaconf/omegaconf.py +++ b/omegaconf/omegaconf.py @@ -93,11 +93,14 @@ def SI(interpolation: str) -> Any: def register_default_resolvers() -> None: - from .built_in_resolvers import decode, env, legacy_env + from .built_in_resolvers import decode, dict_keys, dict_values, env, legacy_env + + OmegaConf.register_new_resolver("oc.decode", decode) + OmegaConf.register_new_resolver("oc.dict.keys", dict_keys) + OmegaConf.register_new_resolver("oc.dict.values", dict_values) + OmegaConf.register_new_resolver("oc.env", env) OmegaConf.legacy_register_resolver("env", legacy_env) - OmegaConf.register_new_resolver("oc.env", env, use_cache=False) - OmegaConf.register_new_resolver("oc.decode", decode, use_cache=False) class OmegaConf: diff --git a/tests/interpolation/built_in_resolvers/test_dict.py b/tests/interpolation/built_in_resolvers/test_dict.py new file mode 100644 index 000000000..b01a0f117 --- /dev/null +++ b/tests/interpolation/built_in_resolvers/test_dict.py @@ -0,0 +1,87 @@ +from typing import Any + +from pytest import mark, param + +from omegaconf import OmegaConf + + +@mark.parametrize( + ("cfg", "key", "expected"), + [ + param( + {"foo": "${oc.dict.keys:{a: 0, b: 1}}"}, + "foo", + OmegaConf.create(["a", "b"]), + id="dict", + ), + param( + {"foo": "${oc.dict.keys:${bar}}", "bar": {"a": 0, "b": 1}}, + "foo", + OmegaConf.create(["a", "b"]), + id="dictconfig", + ), + param( + {"foo": "${sum:${oc.dict.keys:{1: one, 2: two}}}"}, + "foo", + 3, + id="nested", + ), + ], +) +def test_dict_keys(restore_resolvers: Any, cfg: Any, key: Any, expected: Any) -> None: + OmegaConf.register_new_resolver("sum", lambda x: sum(x)) + + cfg = OmegaConf.create(cfg) + val = cfg[key] + assert val == expected + assert type(val) is type(expected) + + +@mark.parametrize( + ("cfg", "key", "expected"), + [ + param( + {"foo": "${oc.dict.values:{a: 0, b: 1}}"}, + "foo", + OmegaConf.create([0, 1]), + id="dict", + ), + param( + {"foo": "${oc.dict.values:${bar}}", "bar": {"a": 0, "b": 1}}, + "foo", + OmegaConf.create([0, 1]), + id="dictconfig", + ), + param( + {"foo": "${sum:${oc.dict.values:{one: 1, two: 2}}}"}, + "foo", + 3, + id="nested", + ), + param( + { + "foo": "${oc.dict.values:${bar}}", + "bar": {"x": {"x0": 0, "x1": 1}, "y": {"y0": 0}}, + }, + "foo", + OmegaConf.create([{"x0": 0, "x1": 1}, {"y0": 0}]), + id="convert_node_to_list", + ), + param( + { + "foo": "${oc.dict.values:{key: ${val_ref}}}", + "val_ref": "value", + }, + "foo", + OmegaConf.create(["value"]), + id="dict_with_interpolated_value", + ), + ], +) +def test_dict_values(restore_resolvers: Any, cfg: Any, key: Any, expected: Any) -> None: + OmegaConf.register_new_resolver("sum", lambda x: sum(x)) + + cfg = OmegaConf.create(cfg) + val = cfg[key] + assert val == expected + assert type(val) is type(expected) From aa495025ddb26924fe726074fea298cee8e9835b Mon Sep 17 00:00:00 2001 From: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Date: Wed, 24 Mar 2021 17:19:37 -0400 Subject: [PATCH 02/17] Improvements to `oc.dict.{keys,values}` * Can now take a string as input for convenience * The output is always a ListConfig, whose parent is the parent of the node being processed --- docs/source/usage.rst | 7 ++- omegaconf/built_in_resolvers.py | 48 ++++++++++++++-- .../built_in_resolvers/test_dict.py | 55 ++++++++++++++++++- 3 files changed, 100 insertions(+), 10 deletions(-) diff --git a/docs/source/usage.rst b/docs/source/usage.rst index da9b16bb2..6c0fb546c 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -473,7 +473,8 @@ Some config options that are stored as a ``DictConfig`` may sometimes be easier when we care only about the keys or the associated values. The resolvers ``oc.dict.keys`` and ``oc.dict.values`` simplify such operations by extracting respectively -the list of keys and values from ``dict``-like objects like ``DictConfig``: +the list of keys and values from ``dict``-like objects like ``DictConfig``. +If a string is given as input, ``OmegaConf.select()`` is used to access the corresponding config node. .. doctest:: @@ -484,8 +485,10 @@ the list of keys and values from ``dict``-like objects like ``DictConfig``: ... "node012": "10.0.0.3", ... "node075": "10.0.1.8", ... }, + ... # Explicit interpolation `${machines}` as input. ... "nodes": "${oc.dict.keys:${machines}}", - ... "ips": "${oc.dict.values:${machines}}", + ... # Config node name `machines` as input. + ... "ips": "${oc.dict.values:machines}", ... } ... ) >>> show(cfg.nodes) diff --git a/omegaconf/built_in_resolvers.py b/omegaconf/built_in_resolvers.py index f253c669c..df6960256 100644 --- a/omegaconf/built_in_resolvers.py +++ b/omegaconf/built_in_resolvers.py @@ -1,11 +1,16 @@ import os import warnings -from typing import Any, Dict, List, Optional + +# from collections.abc import Mapping, MutableMapping +from typing import Any, Mapping, Optional, Union from ._utils import _DEFAULT_MARKER_, _get_value, decode_primitive from .base import Container +from .basecontainer import BaseContainer from .errors import ValidationError from .grammar_parser import parse +from .listconfig import ListConfig +from .omegaconf import OmegaConf def decode(expr: Optional[str], _parent_: Container) -> Any: @@ -28,12 +33,22 @@ def decode(expr: Optional[str], _parent_: Container) -> Any: return _get_value(val) -def dict_keys(in_dict: Dict[Any, Any]) -> List[Any]: - return list(in_dict.keys()) +def dict_keys( + in_dict: Union[str, Mapping[Any, Any]], + _root_: BaseContainer, + _parent_: Container, +) -> ListConfig: + return _dict_impl( + keys_or_values="keys", in_dict=in_dict, _root_=_root_, _parent_=_parent_ + ) -def dict_values(in_dict: Dict[Any, Any]) -> List[Any]: - return list(in_dict.values()) +def dict_values( + in_dict: Union[str, Mapping[Any, Any]], _root_: BaseContainer, _parent_: Container +) -> ListConfig: + return _dict_impl( + keys_or_values="values", in_dict=in_dict, _root_=_root_, _parent_=_parent_ + ) def env(key: str, default: Any = _DEFAULT_MARKER_) -> Optional[str]: @@ -68,3 +83,26 @@ def legacy_env(key: str, default: Optional[str] = None) -> Any: return decode_primitive(default) else: raise ValidationError(f"Environment variable '{key}' not found") + + +def _dict_impl( + keys_or_values: str, + in_dict: Union[str, Mapping[Any, Any]], + _root_: BaseContainer, + _parent_: Container, +) -> ListConfig: + if isinstance(in_dict, str): + # Path to an existing key in the config: use `select()`. + in_dict = OmegaConf.select(_root_, in_dict, throw_on_missing=True) + + if not isinstance(in_dict, Mapping): + raise TypeError( + f"`oc.dict.{keys_or_values}` cannot be applied to objects of type: " + f"{type(in_dict).__name__}" + ) + + dict_method = getattr(in_dict, keys_or_values) + assert isinstance(_parent_, BaseContainer) + ret = OmegaConf.create(list(dict_method()), parent=_parent_) + assert isinstance(ret, ListConfig) + return ret diff --git a/tests/interpolation/built_in_resolvers/test_dict.py b/tests/interpolation/built_in_resolvers/test_dict.py index b01a0f117..6c9a9219c 100644 --- a/tests/interpolation/built_in_resolvers/test_dict.py +++ b/tests/interpolation/built_in_resolvers/test_dict.py @@ -1,8 +1,9 @@ from typing import Any -from pytest import mark, param +from pytest import mark, param, raises -from omegaconf import OmegaConf +from omegaconf import ListConfig, OmegaConf +from omegaconf.errors import InterpolationResolutionError @mark.parametrize( @@ -18,7 +19,13 @@ {"foo": "${oc.dict.keys:${bar}}", "bar": {"a": 0, "b": 1}}, "foo", OmegaConf.create(["a", "b"]), - id="dictconfig", + id="dictconfig_interpolation", + ), + param( + {"foo": "${oc.dict.keys:bar}", "bar": {"a": 0, "b": 1}}, + "foo", + OmegaConf.create(["a", "b"]), + id="dictconfig_select", ), param( {"foo": "${sum:${oc.dict.keys:{1: one, 2: two}}}"}, @@ -36,6 +43,9 @@ def test_dict_keys(restore_resolvers: Any, cfg: Any, key: Any, expected: Any) -> assert val == expected assert type(val) is type(expected) + if isinstance(val, ListConfig): + assert val._parent is cfg + @mark.parametrize( ("cfg", "key", "expected"), @@ -52,6 +62,12 @@ def test_dict_keys(restore_resolvers: Any, cfg: Any, key: Any, expected: Any) -> OmegaConf.create([0, 1]), id="dictconfig", ), + param( + {"foo": "${oc.dict.values:bar}", "bar": {"a": 0, "b": 1}}, + "foo", + OmegaConf.create([0, 1]), + id="dictconfig_select", + ), param( {"foo": "${sum:${oc.dict.values:{one: 1, two: 2}}}"}, "foo", @@ -85,3 +101,36 @@ def test_dict_values(restore_resolvers: Any, cfg: Any, key: Any, expected: Any) val = cfg[key] assert val == expected assert type(val) is type(expected) + + if isinstance(val, ListConfig): + assert val._parent is cfg + + +@mark.parametrize( + "cfg", + [ + param({"x": "${oc.dict.keys:[]}"}, id="list"), + param({"x": "${oc.dict.keys:${bool}}", "bool": True}, id="bool_interpolation"), + param({"x": "${oc.dict.keys:int}", "int": 0}, id="int_select"), + ], +) +def test_dict_keys_invalid_type(cfg: Any) -> None: + cfg = OmegaConf.create(cfg) + with raises(InterpolationResolutionError, match="TypeError"): + cfg.x + + +@mark.parametrize( + "cfg", + [ + param({"x": "${oc.dict.values:[]}"}, id="list"), + param( + {"x": "${oc.dict.values:${bool}}", "bool": True}, id="bool_interpolation" + ), + param({"x": "${oc.dict.values:int}", "int": 0}, id="int_select"), + ], +) +def test_dict_values_invalid_type(cfg: Any) -> None: + cfg = OmegaConf.create(cfg) + with raises(InterpolationResolutionError, match="TypeError"): + cfg.x From 50247446b89246c1e6dc11bbc06c0490d574f727 Mon Sep 17 00:00:00 2001 From: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Date: Wed, 24 Mar 2021 22:05:59 -0400 Subject: [PATCH 03/17] Un-factorize some code --- omegaconf/built_in_resolvers.py | 35 ++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/omegaconf/built_in_resolvers.py b/omegaconf/built_in_resolvers.py index df6960256..1d5157664 100644 --- a/omegaconf/built_in_resolvers.py +++ b/omegaconf/built_in_resolvers.py @@ -38,17 +38,25 @@ def dict_keys( _root_: BaseContainer, _parent_: Container, ) -> ListConfig: - return _dict_impl( - keys_or_values="keys", in_dict=in_dict, _root_=_root_, _parent_=_parent_ + in_dict = _get_and_validate_dict_input( + in_dict, root=_root_, resolver_name="oc.dict.keys" ) + assert isinstance(_parent_, BaseContainer) + ret = OmegaConf.create(list(in_dict.keys()), parent=_parent_) + assert isinstance(ret, ListConfig) + return ret def dict_values( in_dict: Union[str, Mapping[Any, Any]], _root_: BaseContainer, _parent_: Container ) -> ListConfig: - return _dict_impl( - keys_or_values="values", in_dict=in_dict, _root_=_root_, _parent_=_parent_ + in_dict = _get_and_validate_dict_input( + in_dict, root=_root_, resolver_name="oc.dict.values" ) + assert isinstance(_parent_, BaseContainer) + ret = OmegaConf.create(list(in_dict.values()), parent=_parent_) + assert isinstance(ret, ListConfig) + return ret def env(key: str, default: Any = _DEFAULT_MARKER_) -> Optional[str]: @@ -85,24 +93,19 @@ def legacy_env(key: str, default: Optional[str] = None) -> Any: raise ValidationError(f"Environment variable '{key}' not found") -def _dict_impl( - keys_or_values: str, +def _get_and_validate_dict_input( in_dict: Union[str, Mapping[Any, Any]], - _root_: BaseContainer, - _parent_: Container, -) -> ListConfig: + root: BaseContainer, + resolver_name: str, +) -> Mapping[Any, Any]: if isinstance(in_dict, str): # Path to an existing key in the config: use `select()`. - in_dict = OmegaConf.select(_root_, in_dict, throw_on_missing=True) + in_dict = OmegaConf.select(root, in_dict, throw_on_missing=True) if not isinstance(in_dict, Mapping): raise TypeError( - f"`oc.dict.{keys_or_values}` cannot be applied to objects of type: " + f"`{resolver_name}` cannot be applied to objects of type: " f"{type(in_dict).__name__}" ) - dict_method = getattr(in_dict, keys_or_values) - assert isinstance(_parent_, BaseContainer) - ret = OmegaConf.create(list(dict_method()), parent=_parent_) - assert isinstance(ret, ListConfig) - return ret + return in_dict From 34a33b3e196751c3cf87584af5f626e7990f4a6b Mon Sep 17 00:00:00 2001 From: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Date: Wed, 24 Mar 2021 22:14:31 -0400 Subject: [PATCH 04/17] Add tests for missing values --- .../interpolation/built_in_resolvers/test_dict.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/interpolation/built_in_resolvers/test_dict.py b/tests/interpolation/built_in_resolvers/test_dict.py index 6c9a9219c..e3334d7c7 100644 --- a/tests/interpolation/built_in_resolvers/test_dict.py +++ b/tests/interpolation/built_in_resolvers/test_dict.py @@ -134,3 +134,17 @@ def test_dict_values_invalid_type(cfg: Any) -> None: cfg = OmegaConf.create(cfg) with raises(InterpolationResolutionError, match="TypeError"): cfg.x + + +@mark.parametrize( + "cfg", + [ + param({"x": "${oc.dict.keys:y}", "y": "???"}, id="keys_select_missing"), + param({"x": "${oc.dict.values:y}", "y": "???"}, id="values_select_missing"), + param({"x": "${oc.dict.values:y}", "y": {"m": "???"}}, id="missing_value"), + ], +) +def test_dict_missing(cfg: Any) -> None: + cfg = OmegaConf.create(cfg) + with raises(InterpolationResolutionError, match="MissingMandatoryValue"): + cfg.x From 62e0bd408714950355cfea1dd09593050b8839f1 Mon Sep 17 00:00:00 2001 From: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Date: Wed, 24 Mar 2021 22:19:38 -0400 Subject: [PATCH 05/17] Clarify comments in doc --- docs/source/usage.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 6c0fb546c..4972e1cbe 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -485,9 +485,9 @@ If a string is given as input, ``OmegaConf.select()`` is used to access the corr ... "node012": "10.0.0.3", ... "node075": "10.0.1.8", ... }, - ... # Explicit interpolation `${machines}` as input. + ... # Obtaining keys with explicit interpolation as input. ... "nodes": "${oc.dict.keys:${machines}}", - ... # Config node name `machines` as input. + ... # Obtaining values with node name as input. ... "ips": "${oc.dict.values:machines}", ... } ... ) From 19954b59fcc97f1aba762a64a7ec23331b142b91 Mon Sep 17 00:00:00 2001 From: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Date: Wed, 24 Mar 2021 22:26:25 -0400 Subject: [PATCH 06/17] Refactor some tests --- .../built_in_resolvers/test_dict.py | 57 ++++++++----------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/tests/interpolation/built_in_resolvers/test_dict.py b/tests/interpolation/built_in_resolvers/test_dict.py index e3334d7c7..cd83649bc 100644 --- a/tests/interpolation/built_in_resolvers/test_dict.py +++ b/tests/interpolation/built_in_resolvers/test_dict.py @@ -12,39 +12,29 @@ param( {"foo": "${oc.dict.keys:{a: 0, b: 1}}"}, "foo", - OmegaConf.create(["a", "b"]), + ["a", "b"], id="dict", ), param( {"foo": "${oc.dict.keys:${bar}}", "bar": {"a": 0, "b": 1}}, "foo", - OmegaConf.create(["a", "b"]), + ["a", "b"], id="dictconfig_interpolation", ), param( {"foo": "${oc.dict.keys:bar}", "bar": {"a": 0, "b": 1}}, "foo", - OmegaConf.create(["a", "b"]), + ["a", "b"], id="dictconfig_select", ), - param( - {"foo": "${sum:${oc.dict.keys:{1: one, 2: two}}}"}, - "foo", - 3, - id="nested", - ), ], ) -def test_dict_keys(restore_resolvers: Any, cfg: Any, key: Any, expected: Any) -> None: - OmegaConf.register_new_resolver("sum", lambda x: sum(x)) - +def test_dict_keys(cfg: Any, key: Any, expected: Any) -> None: cfg = OmegaConf.create(cfg) val = cfg[key] assert val == expected - assert type(val) is type(expected) - - if isinstance(val, ListConfig): - assert val._parent is cfg + assert isinstance(val, ListConfig) + assert val._parent is cfg @mark.parametrize( @@ -53,34 +43,28 @@ def test_dict_keys(restore_resolvers: Any, cfg: Any, key: Any, expected: Any) -> param( {"foo": "${oc.dict.values:{a: 0, b: 1}}"}, "foo", - OmegaConf.create([0, 1]), + [0, 1], id="dict", ), param( {"foo": "${oc.dict.values:${bar}}", "bar": {"a": 0, "b": 1}}, "foo", - OmegaConf.create([0, 1]), + [0, 1], id="dictconfig", ), param( {"foo": "${oc.dict.values:bar}", "bar": {"a": 0, "b": 1}}, "foo", - OmegaConf.create([0, 1]), + [0, 1], id="dictconfig_select", ), - param( - {"foo": "${sum:${oc.dict.values:{one: 1, two: 2}}}"}, - "foo", - 3, - id="nested", - ), param( { "foo": "${oc.dict.values:${bar}}", "bar": {"x": {"x0": 0, "x1": 1}, "y": {"y0": 0}}, }, "foo", - OmegaConf.create([{"x0": 0, "x1": 1}, {"y0": 0}]), + [{"x0": 0, "x1": 1}, {"y0": 0}], id="convert_node_to_list", ), param( @@ -89,21 +73,30 @@ def test_dict_keys(restore_resolvers: Any, cfg: Any, key: Any, expected: Any) -> "val_ref": "value", }, "foo", - OmegaConf.create(["value"]), + ["value"], id="dict_with_interpolated_value", ), ], ) -def test_dict_values(restore_resolvers: Any, cfg: Any, key: Any, expected: Any) -> None: - OmegaConf.register_new_resolver("sum", lambda x: sum(x)) +def test_dict_values(cfg: Any, key: Any, expected: Any) -> None: cfg = OmegaConf.create(cfg) val = cfg[key] assert val == expected - assert type(val) is type(expected) + assert isinstance(val, ListConfig) + assert val._parent is cfg - if isinstance(val, ListConfig): - assert val._parent is cfg + +def test_dict_keys_nested(restore_resolvers: Any) -> None: + OmegaConf.register_new_resolver("sum", lambda x: sum(x)) + cfg = OmegaConf.create({"x": "${sum:${oc.dict.keys:{1: one, 2: two}}}"}) + assert cfg.x == 3 + + +def test_dict_values_nested(restore_resolvers: Any) -> None: + OmegaConf.register_new_resolver("sum", lambda x: sum(x)) + cfg = OmegaConf.create({"x": "${sum:${oc.dict.values:{one: 1, two: 2}}}"}) + assert cfg.x == 3 @mark.parametrize( From 5fc26305018c3b4e8a241ea6a36dc05d9da263f8 Mon Sep 17 00:00:00 2001 From: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Date: Wed, 24 Mar 2021 22:54:30 -0400 Subject: [PATCH 07/17] More tests refactoring --- .../built_in_resolvers/test_dict.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/interpolation/built_in_resolvers/test_dict.py b/tests/interpolation/built_in_resolvers/test_dict.py index cd83649bc..fbb9e477d 100644 --- a/tests/interpolation/built_in_resolvers/test_dict.py +++ b/tests/interpolation/built_in_resolvers/test_dict.py @@ -87,16 +87,17 @@ def test_dict_values(cfg: Any, key: Any, expected: Any) -> None: assert val._parent is cfg -def test_dict_keys_nested(restore_resolvers: Any) -> None: - OmegaConf.register_new_resolver("sum", lambda x: sum(x)) - cfg = OmegaConf.create({"x": "${sum:${oc.dict.keys:{1: one, 2: two}}}"}) - assert cfg.x == 3 - - -def test_dict_values_nested(restore_resolvers: Any) -> None: - OmegaConf.register_new_resolver("sum", lambda x: sum(x)) - cfg = OmegaConf.create({"x": "${sum:${oc.dict.values:{one: 1, two: 2}}}"}) - assert cfg.x == 3 +@mark.parametrize( + ("cfg", "expected"), + [ + param({"x": "${sum:${oc.dict.values:{one: 1, two: 2}}}"}, 3, id="values"), + param({"x": "${sum:${oc.dict.keys:{1: one, 2: two}}}"}, 3, id="keys"), + ], +) +def test_nested_oc_dict(restore_resolvers: Any, cfg: Any, expected: Any) -> None: + OmegaConf.register_new_resolver("sum", sum) + cfg = OmegaConf.create(cfg) + assert cfg.x == expected @mark.parametrize( From ac6a32917ad34ef09bec35c3d40c50e52424ee20 Mon Sep 17 00:00:00 2001 From: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Date: Thu, 25 Mar 2021 08:23:35 -0400 Subject: [PATCH 08/17] New implementation of `oc.dict.values` based on interpolations Of particular note: * When the result of an interpolation is a node whose parent is the current node's parent, but has no key, then we set its key to the current node's key. This makes it possible to use its full key as an identifier. * _get_and_validate_dict_input() now properly raises an exception if the desired key does not exist --- docs/source/usage.rst | 16 +- omegaconf/base.py | 11 + omegaconf/built_in_resolvers.py | 63 +++- .../built_in_resolvers/test_dict.py | 283 ++++++++++++++++-- 4 files changed, 347 insertions(+), 26 deletions(-) diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 4972e1cbe..59d202125 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -1,6 +1,6 @@ .. testsetup:: * - from omegaconf import OmegaConf, DictConfig, open_dict, read_write + from omegaconf import OmegaConf, DictConfig, ListConfig, open_dict, read_write import os import sys import tempfile @@ -491,10 +491,16 @@ If a string is given as input, ``OmegaConf.select()`` is used to access the corr ... "ips": "${oc.dict.values:machines}", ... } ... ) - >>> show(cfg.nodes) - type: ListConfig, value: ['node007', 'node012', 'node075'] - >>> show(cfg.ips) - type: ListConfig, value: ['10.0.0.7', '10.0.0.3', '10.0.1.8'] + >>> nodes = cfg.nodes + >>> ips = cfg.ips + >>> # The corresponding lists of keys / values are ListConfig nodes. + >>> assert isinstance(nodes, ListConfig) + >>> assert isinstance(ips, ListConfig) + >>> assert nodes == ['node007', 'node012', 'node075'] + >>> assert ips == ['10.0.0.7', '10.0.0.3', '10.0.1.8'] + >>> # Values are dynamically updated with the underlying dict. + >>> cfg.machines.node012 = "10.0.0.5" + >>> assert ips[1] == "10.0.0.5" Custom interpolations diff --git a/omegaconf/base.py b/omegaconf/base.py index 33bc9f996..68050ca26 100644 --- a/omegaconf/base.py +++ b/omegaconf/base.py @@ -451,6 +451,7 @@ def _resolve_interpolation_from_parse_tree( node that is created to wrap the interpolated value. It is `None` if and only if `throw_on_resolution_failure` is `False` and an error occurs during resolution. """ + try: resolved = self.resolve_parse_tree( parse_tree=parse_tree, @@ -516,6 +517,16 @@ def _validate_and_convert_interpolation_result( ) else: assert isinstance(resolved, Node) + if ( + parent is not None + and resolved._parent is parent + and resolved._key() is None + and value._key() is not None + ): + # The interpolation is returning a transient (key-less) node attached + # to the current parent. By setting its key to this node's key, we make + # it possible to refer to it through an interpolation path. + resolved._set_key(value._key()) return resolved def _wrap_interpolation_result( diff --git a/omegaconf/built_in_resolvers.py b/omegaconf/built_in_resolvers.py index 1d5157664..49ea891b0 100644 --- a/omegaconf/built_in_resolvers.py +++ b/omegaconf/built_in_resolvers.py @@ -2,16 +2,22 @@ import warnings # from collections.abc import Mapping, MutableMapping -from typing import Any, Mapping, Optional, Union +from typing import Any, List, Mapping, Optional, Union -from ._utils import _DEFAULT_MARKER_, _get_value, decode_primitive +from ._utils import _DEFAULT_MARKER_, Marker, _get_value, decode_primitive from .base import Container from .basecontainer import BaseContainer -from .errors import ValidationError +from .dictconfig import DictConfig +from .errors import ConfigKeyError, ValidationError from .grammar_parser import parse from .listconfig import ListConfig +from .nodes import AnyNode from .omegaconf import OmegaConf +# Special marker use as default value when calling `OmegaConf.select()`. It must be +# different from `_DEFAULT_MARKER_`, which is used by `OmegaConf.select()`. +_DEFAULT_SELECT_MARKER_: Any = Marker("_DEFAULT_SELECT_MARKER_") + def decode(expr: Optional[str], _parent_: Container) -> Any: """ @@ -42,6 +48,7 @@ def dict_keys( in_dict, root=_root_, resolver_name="oc.dict.keys" ) assert isinstance(_parent_, BaseContainer) + ret = OmegaConf.create(list(in_dict.keys()), parent=_parent_) assert isinstance(ret, ListConfig) return ret @@ -53,6 +60,49 @@ def dict_values( in_dict = _get_and_validate_dict_input( in_dict, root=_root_, resolver_name="oc.dict.values" ) + + if isinstance(in_dict, DictConfig): + # DictConfig objects are handled in a special way: the goal is to make the + # returned ListConfig point to the DictConfig nodes through interpolations. + + dict_key: Optional[str] = None + if in_dict._get_root() is _root_: + # Try to obtain the full key through which we can access `in_dict`. + if in_dict is _root_: + dict_key = "" + else: + dict_key = in_dict._get_full_key(None) + if dict_key: + dict_key += "." # append dot for future concatenation + else: + # This can happen e.g. if `in_dict` is a transient node. + dict_key = None + + if dict_key is None: + # No path to `in_dict` in the existing config. + raise NotImplementedError( + "`oc.dict.values` currently only supports input config nodes that " + "are accessible through the root config. See " + "https://github.com/omry/omegaconf/issues/650 for details." + ) + + ret = ListConfig([]) + content = in_dict._content + assert isinstance(content, dict) + + for key, node in content.items(): + ref_node = AnyNode(f"${{{dict_key}{key}}}") + ret.append(ref_node) + + # Finalize result by setting proper type and parent. + element_type: Any = in_dict._metadata.element_type + ret._metadata.element_type = element_type + ret._metadata.ref_type = List[element_type] + ret._set_parent(_parent_) + + return ret + + # Other dict-like object: simply create a ListConfig from its values. assert isinstance(_parent_, BaseContainer) ret = OmegaConf.create(list(in_dict.values()), parent=_parent_) assert isinstance(ret, ListConfig) @@ -100,7 +150,12 @@ def _get_and_validate_dict_input( ) -> Mapping[Any, Any]: if isinstance(in_dict, str): # Path to an existing key in the config: use `select()`. - in_dict = OmegaConf.select(root, in_dict, throw_on_missing=True) + key = in_dict + in_dict = OmegaConf.select( + root, key, throw_on_missing=True, default=_DEFAULT_SELECT_MARKER_ + ) + if in_dict is _DEFAULT_SELECT_MARKER_: + raise ConfigKeyError(f"Key not found: '{key}'") if not isinstance(in_dict, Mapping): raise TypeError( diff --git a/tests/interpolation/built_in_resolvers/test_dict.py b/tests/interpolation/built_in_resolvers/test_dict.py index fbb9e477d..bdc6d01b4 100644 --- a/tests/interpolation/built_in_resolvers/test_dict.py +++ b/tests/interpolation/built_in_resolvers/test_dict.py @@ -1,9 +1,16 @@ -from typing import Any +import re +from typing import Any, List from pytest import mark, param, raises -from omegaconf import ListConfig, OmegaConf -from omegaconf.errors import InterpolationResolutionError +from omegaconf import DictConfig, ListConfig, OmegaConf +from omegaconf.built_in_resolvers import _get_and_validate_dict_input +from omegaconf.errors import ( + InterpolationKeyError, + InterpolationResolutionError, + InterpolationToMissingValueError, +) +from tests import User, Users @mark.parametrize( @@ -39,23 +46,102 @@ def test_dict_keys(cfg: Any, key: Any, expected: Any) -> None: @mark.parametrize( ("cfg", "key", "expected"), + [ + param( + {"x": "${oc.dict.keys_or_values:y}", "y": "???"}, + "x", + raises( + InterpolationResolutionError, + match=re.escape( + "MissingMandatoryValue raised while resolving interpolation: " + "Missing mandatory value : y" + ), + ), + id="select_missing", + ), + param( + {"foo": "${oc.dict.keys_or_values:${bar}}"}, + "foo", + raises( + InterpolationKeyError, + match=re.escape("Interpolation key 'bar' not found"), + ), + id="interpolation_key_error", + ), + param( + {"foo": "${oc.dict.keys_or_values:bar}"}, + "foo", + raises( + InterpolationResolutionError, + match=re.escape( + "ConfigKeyError raised while resolving interpolation: " + "Key not found: 'bar'" + ), + ), + id="config_key_error", + ), + param( + {"foo": "${oc.dict.keys_or_values:${bar}}", "bar": 0}, + "foo", + raises( + InterpolationResolutionError, + match=re.escape( + "TypeError raised while resolving interpolation: " + "`oc.dict.keys_or_values` cannot be applied to objects of type: int" + ), + ), + id="type_error", + ), + param( + {"foo": "${oc.dict.keys_or_values:${bar}}", "bar": DictConfig(None)}, + "foo", + raises( + InterpolationResolutionError, + match=re.escape( + "TypeError raised while resolving interpolation: " + "`oc.dict.keys_or_values` cannot be applied to objects of type: NoneType" + ), + ), + id="type_error_dictconfig", + ), + ], +) +def test_get_and_validate_dict_input( + restore_resolvers: Any, cfg: Any, key: Any, expected: Any +) -> None: + OmegaConf.register_new_resolver( + "oc.dict.keys_or_values", + lambda in_dict, _root_: _get_and_validate_dict_input( + in_dict, root=_root_, resolver_name="oc.dict.keys_or_values" + ), + ) + cfg = OmegaConf.create(cfg) + with expected: + cfg[key] + + +@mark.parametrize( + ("cfg", "key", "expected_val", "expected_content"), [ param( {"foo": "${oc.dict.values:{a: 0, b: 1}}"}, "foo", [0, 1], + [0, 1], id="dict", ), param( {"foo": "${oc.dict.values:${bar}}", "bar": {"a": 0, "b": 1}}, "foo", [0, 1], + ["${bar.a}", "${bar.b}"], id="dictconfig", ), param( {"foo": "${oc.dict.values:bar}", "bar": {"a": 0, "b": 1}}, "foo", [0, 1], + ["${bar.a}", "${bar.b}"], id="dictconfig_select", ), param( @@ -65,6 +151,7 @@ def test_dict_keys(cfg: Any, key: Any, expected: Any) -> None: }, "foo", [{"x0": 0, "x1": 1}, {"y0": 0}], + ["${bar.x}", "${bar.y}"], id="convert_node_to_list", ), param( @@ -74,17 +161,183 @@ def test_dict_keys(cfg: Any, key: Any, expected: Any) -> None: }, "foo", ["value"], + ["value"], id="dict_with_interpolated_value", ), + param( + { + "foo": "${oc.dict.values:${bar}}", + "bar": {"key": "${val_ref}"}, + "val_ref": "value", + }, + "foo", + ["value"], + ["${bar.key}"], + id="dictconfig_with_interpolated_value", + ), + param( + { + "foo": "${oc.dict.values:${bar}}", + "bar": "${boz}", + "boz": {"a": 0, "b": 1}, + }, + "foo", + [0, 1], + ["${boz.a}", "${boz.b}"], + id="dictconfig_chained_interpolation", + ), ], ) -def test_dict_values(cfg: Any, key: Any, expected: Any) -> None: +def test_dict_values( + cfg: Any, key: Any, expected_val: Any, expected_content: Any +) -> None: cfg = OmegaConf.create(cfg) val = cfg[key] - assert val == expected + assert val == expected_val assert isinstance(val, ListConfig) assert val._parent is cfg + content = val._content + assert content == expected_content + + +def test_dict_values_with_missing_value() -> None: + cfg = OmegaConf.create({"foo": "${oc.dict.values:bar}", "bar": {"missing": "???"}}) + foo = cfg.foo + with raises(InterpolationToMissingValueError): + foo[0] + cfg.bar.missing = 1 + assert foo[0] == 1 + + +@mark.parametrize( + ("make_resolver", "key", "expected"), + [ + param( + lambda _parent_: OmegaConf.create({"x": 1}, parent=_parent_), + 0, + 1, + id="basic", + ), + param( + lambda _parent_: OmegaConf.create({"x": "${y}", "y": 1}, parent=_parent_), + 0, + 999, # now referring to the `y` node from the global config + id="inter_abs", + ), + param( + lambda _parent_: OmegaConf.create({"x": "${.y}", "y": 1}, parent=_parent_), + 0, + 1, + id="inter_rel", + ), + ], +) +def test_dict_values_dictconfig_resolver_output( + restore_resolvers: Any, make_resolver: Any, key: Any, expected: Any +) -> None: + OmegaConf.register_new_resolver("make", make_resolver) + cfg = OmegaConf.create( + { + "foo": "${oc.dict.values:${make:}}", + "bar": "${oc.dict.values:${boz}}", + "boz": "${make:}", + "y": 999, + } + ) + + # Directly using the resolver output without an intermediate node to store the + # result is not currently supported. + with raises(InterpolationResolutionError, match=re.escape("NotImplementedError")): + cfg.foo + + assert cfg.bar[key] == expected + + +@mark.parametrize( + ("make_resolver", "key", "expected"), + [ + param(lambda: OmegaConf.create({"x": 1}), 0, 1, id="basic"), + param(lambda: OmegaConf.create({"x": "${y}", "y": 1}), 0, 1, id="inter_abs"), + param(lambda: OmegaConf.create({"x": "${.y}", "y": 1}), 0, 1, id="inter_rel"), + ], +) +def test_dict_values_dictconfig_resolver_output_no_parent( + restore_resolvers: Any, make_resolver: Any, key: Any, expected: Any +) -> None: + OmegaConf.register_new_resolver("make", make_resolver) + cfg = OmegaConf.create( + { + "bar": "${oc.dict.values:${boz}}", + "boz": "${make:}", + "y": 999, + } + ) + + # Using transient nodes whose parent is not set is not currently supported. + with raises(InterpolationResolutionError, match=re.escape("NotImplementedError")): + cfg.bar + + +@mark.parametrize( + ("make_resolver", "expected_value", "expected_content"), + [ + param( + lambda _parent_: OmegaConf.create({"a": 0, "b": 1}, parent=_parent_), + [0, 1], + ["${y.a}", "${y.b}"], + id="dictconfig_with_parent", + ), + param( + lambda: {"a": 0, "b": 1}, + [0, 1], + ["${y.a}", "${y.b}"], + id="plain_dict", + ), + ], +) +def test_dict_values_transient_interpolation( + restore_resolvers: Any, + make_resolver: Any, + expected_value: Any, + expected_content: Any, +) -> None: + OmegaConf.register_new_resolver("make", make_resolver) + cfg = OmegaConf.create({"x": "${oc.dict.values:y}", "y": "${make:}"}) + assert cfg.x == expected_value + assert cfg.x._content == expected_content + + +def test_dict_values_are_typed() -> None: + cfg = OmegaConf.create( + { + "x": "${oc.dict.values:${y.name2user}}", + "y": Users( + name2user={ + "john": User(name="john", age=30), + "jane": User(name="jane", age=33), + } + ), + } + ) + x = cfg.x + assert x._metadata.ref_type == List[User] + assert x._metadata.element_type == User + + +@mark.parametrize( + ("cfg", "expected"), + [ + param({"x": "${oc.dict.values:{a: 1, b: 2}}"}, [1, 2], id="values_inline"), + param({"x": "${oc.dict.keys:{a: 1, b: 2}}"}, ["a", "b"], id="keys_inline"), + param({"x": "${oc.dict.values:y}", "y": {"a": 1}}, [1], id="values_inter"), + param({"x": "${oc.dict.keys:y}", "y": {"a": 1}}, ["a"], id="keys_inter"), + ], +) +def test_readonly_parent(cfg: Any, expected: Any) -> None: + cfg = OmegaConf.create(cfg) + cfg._set_flag("readonly", True) + assert cfg.x == expected @mark.parametrize( @@ -130,15 +383,11 @@ def test_dict_values_invalid_type(cfg: Any) -> None: cfg.x -@mark.parametrize( - "cfg", - [ - param({"x": "${oc.dict.keys:y}", "y": "???"}, id="keys_select_missing"), - param({"x": "${oc.dict.values:y}", "y": "???"}, id="values_select_missing"), - param({"x": "${oc.dict.values:y}", "y": {"m": "???"}}, id="missing_value"), - ], -) -def test_dict_missing(cfg: Any) -> None: - cfg = OmegaConf.create(cfg) - with raises(InterpolationResolutionError, match="MissingMandatoryValue"): - cfg.x +def test_dict_values_of_root(restore_resolvers: Any) -> None: + OmegaConf.register_new_resolver("root", lambda _root_: _root_) + cfg = OmegaConf.create({"x": {"a": "${oc.dict.values:${root:}}"}, "y": 0}) + a = cfg.x.a + assert a._content == ["${x}", "${y}"] + assert a[1] == 0 + # We can recurse indefinitely within the first value if we fancy it. + assert cfg.x.a[0].a[0].a[1] == 0 From 26b23722191d10b064a8f3fe7b78e92a28946373 Mon Sep 17 00:00:00 2001 From: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Date: Tue, 30 Mar 2021 08:04:26 -0400 Subject: [PATCH 09/17] Add test for merging into a resolver's output --- .../built_in_resolvers/test_dict.py | 2 +- tests/interpolation/test_custom_resolvers.py | 24 ++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/tests/interpolation/built_in_resolvers/test_dict.py b/tests/interpolation/built_in_resolvers/test_dict.py index bdc6d01b4..a9cea43cc 100644 --- a/tests/interpolation/built_in_resolvers/test_dict.py +++ b/tests/interpolation/built_in_resolvers/test_dict.py @@ -54,7 +54,7 @@ def test_dict_keys(cfg: Any, key: Any, expected: Any) -> None: InterpolationResolutionError, match=re.escape( "MissingMandatoryValue raised while resolving interpolation: " - "Missing mandatory value : y" + "Missing mandatory value: y" ), ), id="select_missing", diff --git a/tests/interpolation/test_custom_resolvers.py b/tests/interpolation/test_custom_resolvers.py index 3e3d9e351..42c7ce0ec 100644 --- a/tests/interpolation/test_custom_resolvers.py +++ b/tests/interpolation/test_custom_resolvers.py @@ -2,7 +2,7 @@ import re from typing import Any, Dict, List -from pytest import mark, raises, warns +from pytest import mark, param, raises, warns from omegaconf import DictConfig, ListConfig, OmegaConf, Resolver from tests.interpolation import dereference_node @@ -466,3 +466,25 @@ def parent_and_default(default: int = 10, *, _parent_: Any) -> Any: assert cfg.no_param == 20 assert cfg.param == 30 + + +@mark.parametrize( + ("cfg2", "expected"), + [ + param({"foo": {"b": 1}}, {"foo": {"a": 0, "b": 1}}, id="extend"), + param({"foo": {"b": "${.a}"}}, {"foo": {"a": 0, "b": 0}}, id="extend_inter"), + param({"foo": {"a": 1}}, {"foo": {"a": 1}}, id="override_int"), + param({"foo": {"a": {"b": 1}}}, {"foo": {"a": {"b": 1}}}, id="override_dict"), + param({"foo": 10}, {"foo": 10}, id="replace_interpolation"), + param({"bar": 10}, {"foo": {"a": 0}, "bar": 10}, id="other_node"), + ], +) +def test_merge_into_resolver_output( + restore_resolvers: Any, cfg2: Any, expected: Any +) -> None: + OmegaConf.register_new_resolver( + "make", lambda _parent_: OmegaConf.create({"a": 0}, parent=_parent_) + ) + + cfg = OmegaConf.create({"foo": "${make:}"}) + assert OmegaConf.merge(cfg, cfg2) == expected From 890017c5dceffda00c4dd7809d294bd5459c8bfa Mon Sep 17 00:00:00 2001 From: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Date: Wed, 31 Mar 2021 21:12:25 -0400 Subject: [PATCH 10/17] Update omegaconf/built_in_resolvers.py Co-authored-by: Omry Yadan --- omegaconf/built_in_resolvers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omegaconf/built_in_resolvers.py b/omegaconf/built_in_resolvers.py index 49ea891b0..ddad5e265 100644 --- a/omegaconf/built_in_resolvers.py +++ b/omegaconf/built_in_resolvers.py @@ -81,7 +81,7 @@ def dict_values( if dict_key is None: # No path to `in_dict` in the existing config. raise NotImplementedError( - "`oc.dict.values` currently only supports input config nodes that " + "`oc.dict.values` only supports input config nodes that " "are accessible through the root config. See " "https://github.com/omry/omegaconf/issues/650 for details." ) From 89496b020b059491bfad146876cebfb6141dd470 Mon Sep 17 00:00:00 2001 From: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Date: Wed, 31 Mar 2021 22:11:47 -0400 Subject: [PATCH 11/17] Update doc to show interpolations for `oc.dict.values` --- docs/source/usage.rst | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 59d202125..56cdfff31 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -480,27 +480,23 @@ If a string is given as input, ``OmegaConf.select()`` is used to access the corr >>> cfg = OmegaConf.create( ... { - ... "machines": { - ... "node007": "10.0.0.7", - ... "node012": "10.0.0.3", - ... "node075": "10.0.1.8", + ... "workers": { + ... "node3": "10.0.0.2", + ... "node7": "10.0.0.9", ... }, ... # Obtaining keys with explicit interpolation as input. - ... "nodes": "${oc.dict.keys:${machines}}", + ... "nodes": "${oc.dict.keys:${workers}}", ... # Obtaining values with node name as input. - ... "ips": "${oc.dict.values:machines}", + ... "ips": "${oc.dict.values:workers}", ... } ... ) - >>> nodes = cfg.nodes - >>> ips = cfg.ips - >>> # The corresponding lists of keys / values are ListConfig nodes. - >>> assert isinstance(nodes, ListConfig) - >>> assert isinstance(ips, ListConfig) - >>> assert nodes == ['node007', 'node012', 'node075'] - >>> assert ips == ['10.0.0.7', '10.0.0.3', '10.0.1.8'] - >>> # Values are dynamically updated with the underlying dict. - >>> cfg.machines.node012 = "10.0.0.5" - >>> assert ips[1] == "10.0.0.5" + >>> # Keys are copied from the DictConfig: + >>> show(cfg.nodes) + type: ListConfig, value: ['node3', 'node7'] + >>> # Values are dynamically fetched through interpolations: + >>> show(cfg.ips) + type: ListConfig, value: ['${workers.node3}', '${workers.node7}'] + >>> assert cfg.ips == ["10.0.0.2", "10.0.0.9"] Custom interpolations From b0fadcd5fc3ad6a33abed0895f8b55870f378ab5 Mon Sep 17 00:00:00 2001 From: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Date: Wed, 31 Mar 2021 22:27:39 -0400 Subject: [PATCH 12/17] Clearer error when trying to use a relative interpolation --- omegaconf/built_in_resolvers.py | 6 ++++++ tests/interpolation/built_in_resolvers/test_dict.py | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/omegaconf/built_in_resolvers.py b/omegaconf/built_in_resolvers.py index ddad5e265..6fd963580 100644 --- a/omegaconf/built_in_resolvers.py +++ b/omegaconf/built_in_resolvers.py @@ -151,11 +151,17 @@ def _get_and_validate_dict_input( if isinstance(in_dict, str): # Path to an existing key in the config: use `select()`. key = in_dict + if key.startswith("."): + raise NotImplementedError( + f"To use relative interpolations with `{resolver_name}`, please use " + f"the explicit interpolation syntax: ${{{resolver_name}:${{{key}}}}}" + ) in_dict = OmegaConf.select( root, key, throw_on_missing=True, default=_DEFAULT_SELECT_MARKER_ ) if in_dict is _DEFAULT_SELECT_MARKER_: raise ConfigKeyError(f"Key not found: '{key}'") + assert in_dict is not None if not isinstance(in_dict, Mapping): raise TypeError( diff --git a/tests/interpolation/built_in_resolvers/test_dict.py b/tests/interpolation/built_in_resolvers/test_dict.py index a9cea43cc..17251ca19 100644 --- a/tests/interpolation/built_in_resolvers/test_dict.py +++ b/tests/interpolation/built_in_resolvers/test_dict.py @@ -104,6 +104,15 @@ def test_dict_keys(cfg: Any, key: Any, expected: Any) -> None: ), id="type_error_dictconfig", ), + param( + {"foo": "${oc.dict.keys_or_values:.bar}", "bar": {"x": 0}}, + "foo", + raises( + InterpolationResolutionError, + match=re.escape("NotImplementedError"), + ), + id="relative_with_select", + ), ], ) def test_get_and_validate_dict_input( From 754134393dad1cda0017416e71acb985e179ae6f Mon Sep 17 00:00:00 2001 From: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Date: Wed, 7 Apr 2021 11:04:50 -0400 Subject: [PATCH 13/17] Only support the select syntax in oc.dict.{keys,values} --- omegaconf/built_in_resolvers.py | 115 ++++++++++++-------------------- 1 file changed, 42 insertions(+), 73 deletions(-) diff --git a/omegaconf/built_in_resolvers.py b/omegaconf/built_in_resolvers.py index 6fd963580..0dee46229 100644 --- a/omegaconf/built_in_resolvers.py +++ b/omegaconf/built_in_resolvers.py @@ -1,8 +1,6 @@ import os import warnings - -# from collections.abc import Mapping, MutableMapping -from typing import Any, List, Mapping, Optional, Union +from typing import Any, List, Optional from ._utils import _DEFAULT_MARKER_, Marker, _get_value, decode_primitive from .base import Container @@ -40,72 +38,40 @@ def decode(expr: Optional[str], _parent_: Container) -> Any: def dict_keys( - in_dict: Union[str, Mapping[Any, Any]], - _root_: BaseContainer, + key: str, _parent_: Container, ) -> ListConfig: + assert isinstance(_parent_, BaseContainer) + in_dict = _get_and_validate_dict_input( - in_dict, root=_root_, resolver_name="oc.dict.keys" + key, parent=_parent_, resolver_name="oc.dict.keys" ) - assert isinstance(_parent_, BaseContainer) ret = OmegaConf.create(list(in_dict.keys()), parent=_parent_) assert isinstance(ret, ListConfig) return ret -def dict_values( - in_dict: Union[str, Mapping[Any, Any]], _root_: BaseContainer, _parent_: Container -) -> ListConfig: +def dict_values(key: str, _root_: BaseContainer, _parent_: Container) -> ListConfig: + assert isinstance(_parent_, BaseContainer) in_dict = _get_and_validate_dict_input( - in_dict, root=_root_, resolver_name="oc.dict.values" + key, parent=_parent_, resolver_name="oc.dict.values" ) - if isinstance(in_dict, DictConfig): - # DictConfig objects are handled in a special way: the goal is to make the - # returned ListConfig point to the DictConfig nodes through interpolations. - - dict_key: Optional[str] = None - if in_dict._get_root() is _root_: - # Try to obtain the full key through which we can access `in_dict`. - if in_dict is _root_: - dict_key = "" - else: - dict_key = in_dict._get_full_key(None) - if dict_key: - dict_key += "." # append dot for future concatenation - else: - # This can happen e.g. if `in_dict` is a transient node. - dict_key = None - - if dict_key is None: - # No path to `in_dict` in the existing config. - raise NotImplementedError( - "`oc.dict.values` only supports input config nodes that " - "are accessible through the root config. See " - "https://github.com/omry/omegaconf/issues/650 for details." - ) - - ret = ListConfig([]) - content = in_dict._content - assert isinstance(content, dict) - - for key, node in content.items(): - ref_node = AnyNode(f"${{{dict_key}{key}}}") - ret.append(ref_node) - - # Finalize result by setting proper type and parent. - element_type: Any = in_dict._metadata.element_type - ret._metadata.element_type = element_type - ret._metadata.ref_type = List[element_type] - ret._set_parent(_parent_) - - return ret - - # Other dict-like object: simply create a ListConfig from its values. - assert isinstance(_parent_, BaseContainer) - ret = OmegaConf.create(list(in_dict.values()), parent=_parent_) - assert isinstance(ret, ListConfig) + ret = ListConfig([]) + content = in_dict._content + assert isinstance(content, dict) + + for k, node in content.items(): + ref_node = AnyNode(f"${{{key}.{k}}}") + ret.append(ref_node) + + # Finalize result by setting proper type and parent. + element_type: Any = in_dict._metadata.element_type + ret._metadata.element_type = element_type + ret._metadata.ref_type = List[element_type] + ret._set_parent(_parent_) + return ret @@ -144,26 +110,29 @@ def legacy_env(key: str, default: Optional[str] = None) -> Any: def _get_and_validate_dict_input( - in_dict: Union[str, Mapping[Any, Any]], - root: BaseContainer, + key: str, + parent: BaseContainer, resolver_name: str, -) -> Mapping[Any, Any]: - if isinstance(in_dict, str): - # Path to an existing key in the config: use `select()`. - key = in_dict - if key.startswith("."): - raise NotImplementedError( - f"To use relative interpolations with `{resolver_name}`, please use " - f"the explicit interpolation syntax: ${{{resolver_name}:${{{key}}}}}" - ) - in_dict = OmegaConf.select( - root, key, throw_on_missing=True, default=_DEFAULT_SELECT_MARKER_ +) -> DictConfig: + if not isinstance(key, str): + raise TypeError( + f"`{resolver_name}` requires a string as input, but obtained `{key}` " + f"of type: {type(key).__name__}" ) - if in_dict is _DEFAULT_SELECT_MARKER_: - raise ConfigKeyError(f"Key not found: '{key}'") - assert in_dict is not None - if not isinstance(in_dict, Mapping): + in_dict = OmegaConf.select( + parent, + key, + throw_on_missing=True, + absolute_key=True, + default=_DEFAULT_SELECT_MARKER_, + ) + + if in_dict is _DEFAULT_SELECT_MARKER_: + raise ConfigKeyError(f"Key not found: '{key}'") + assert in_dict is not None + + if not isinstance(in_dict, DictConfig): raise TypeError( f"`{resolver_name}` cannot be applied to objects of type: " f"{type(in_dict).__name__}" From ce794a60e33aed715792512a44b3b58483147869 Mon Sep 17 00:00:00 2001 From: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Date: Wed, 7 Apr 2021 11:16:42 -0400 Subject: [PATCH 14/17] Update tests and documentation Plus some additional code clean-up. --- docs/source/usage.rst | 11 +- news/643.feature | 2 +- omegaconf/base.py | 10 -- omegaconf/built_in_resolvers.py | 3 +- .../built_in_resolvers/test_dict.py | 165 ++++++------------ 5 files changed, 65 insertions(+), 126 deletions(-) diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 56cdfff31..b40532c67 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -473,8 +473,9 @@ Some config options that are stored as a ``DictConfig`` may sometimes be easier when we care only about the keys or the associated values. The resolvers ``oc.dict.keys`` and ``oc.dict.values`` simplify such operations by extracting respectively -the list of keys and values from ``dict``-like objects like ``DictConfig``. -If a string is given as input, ``OmegaConf.select()`` is used to access the corresponding config node. +the list of keys and values from existing config nodes. +They take as input a string that is the path to another config node (using the same syntax +as interpolations) and return a ``ListConfig`` with its keys / values. .. doctest:: @@ -484,10 +485,8 @@ If a string is given as input, ``OmegaConf.select()`` is used to access the corr ... "node3": "10.0.0.2", ... "node7": "10.0.0.9", ... }, - ... # Obtaining keys with explicit interpolation as input. - ... "nodes": "${oc.dict.keys:${workers}}", - ... # Obtaining values with node name as input. - ... "ips": "${oc.dict.values:workers}", + ... "nodes": "${oc.dict.keys: workers}", + ... "ips": "${oc.dict.values: workers}", ... } ... ) >>> # Keys are copied from the DictConfig: diff --git a/news/643.feature b/news/643.feature index 78143216d..cd3e4fc19 100644 --- a/news/643.feature +++ b/news/643.feature @@ -1,2 +1,2 @@ -New resolvers `oc.dict.keys` and `oc.dict.values` allow extracting the lists of keys and values of a DictConfig +New resolvers `oc.dict.keys` and `oc.dict.values` allow extracting the lists of keys and values of config nodes diff --git a/omegaconf/base.py b/omegaconf/base.py index 68050ca26..1a9082525 100644 --- a/omegaconf/base.py +++ b/omegaconf/base.py @@ -517,16 +517,6 @@ def _validate_and_convert_interpolation_result( ) else: assert isinstance(resolved, Node) - if ( - parent is not None - and resolved._parent is parent - and resolved._key() is None - and value._key() is not None - ): - # The interpolation is returning a transient (key-less) node attached - # to the current parent. By setting its key to this node's key, we make - # it possible to refer to it through an interpolation path. - resolved._set_key(value._key()) return resolved def _wrap_interpolation_result( diff --git a/omegaconf/built_in_resolvers.py b/omegaconf/built_in_resolvers.py index 0dee46229..877adbb6a 100644 --- a/omegaconf/built_in_resolvers.py +++ b/omegaconf/built_in_resolvers.py @@ -58,10 +58,10 @@ def dict_values(key: str, _root_: BaseContainer, _parent_: Container) -> ListCon key, parent=_parent_, resolver_name="oc.dict.values" ) - ret = ListConfig([]) content = in_dict._content assert isinstance(content, dict) + ret = ListConfig([]) for k, node in content.items(): ref_node = AnyNode(f"${{{key}.{k}}}") ret.append(ref_node) @@ -130,7 +130,6 @@ def _get_and_validate_dict_input( if in_dict is _DEFAULT_SELECT_MARKER_: raise ConfigKeyError(f"Key not found: '{key}'") - assert in_dict is not None if not isinstance(in_dict, DictConfig): raise TypeError( diff --git a/tests/interpolation/built_in_resolvers/test_dict.py b/tests/interpolation/built_in_resolvers/test_dict.py index 17251ca19..d96cfa92a 100644 --- a/tests/interpolation/built_in_resolvers/test_dict.py +++ b/tests/interpolation/built_in_resolvers/test_dict.py @@ -6,7 +6,6 @@ from omegaconf import DictConfig, ListConfig, OmegaConf from omegaconf.built_in_resolvers import _get_and_validate_dict_input from omegaconf.errors import ( - InterpolationKeyError, InterpolationResolutionError, InterpolationToMissingValueError, ) @@ -17,22 +16,16 @@ ("cfg", "key", "expected"), [ param( - {"foo": "${oc.dict.keys:{a: 0, b: 1}}"}, - "foo", - ["a", "b"], - id="dict", - ), - param( - {"foo": "${oc.dict.keys:${bar}}", "bar": {"a": 0, "b": 1}}, + {"foo": "${oc.dict.keys:bar}", "bar": {"a": 0, "b": 1}}, "foo", ["a", "b"], - id="dictconfig_interpolation", + id="dictconfig", ), param( - {"foo": "${oc.dict.keys:bar}", "bar": {"a": 0, "b": 1}}, + {"foo": "${oc.dict.keys:bar}", "bar": "${boz}", "boz": {"a": 0, "b": 1}}, "foo", ["a", "b"], - id="dictconfig_select", + id="dictconfig_chained_interpolation", ), ], ) @@ -60,28 +53,32 @@ def test_dict_keys(cfg: Any, key: Any, expected: Any) -> None: id="select_missing", ), param( - {"foo": "${oc.dict.keys_or_values:${bar}}"}, + {"foo": "${oc.dict.keys_or_values:bar}"}, "foo", raises( - InterpolationKeyError, - match=re.escape("Interpolation key 'bar' not found"), + InterpolationResolutionError, + match=re.escape( + "ConfigKeyError raised while resolving interpolation: " + "Key not found: 'bar'" + ), ), - id="interpolation_key_error", + id="config_key_error", ), param( - {"foo": "${oc.dict.keys_or_values:bar}"}, + # This might be allowed in the future. Currently it fails. + {"foo": "${oc.dict.keys_or_values:''}"}, "foo", raises( InterpolationResolutionError, match=re.escape( "ConfigKeyError raised while resolving interpolation: " - "Key not found: 'bar'" + "Key not found: ''" ), ), - id="config_key_error", + id="config_key_error_empty", ), param( - {"foo": "${oc.dict.keys_or_values:${bar}}", "bar": 0}, + {"foo": "${oc.dict.keys_or_values:bar}", "bar": 0}, "foo", raises( InterpolationResolutionError, @@ -93,7 +90,7 @@ def test_dict_keys(cfg: Any, key: Any, expected: Any) -> None: id="type_error", ), param( - {"foo": "${oc.dict.keys_or_values:${bar}}", "bar": DictConfig(None)}, + {"foo": "${oc.dict.keys_or_values:bar}", "bar": DictConfig(None)}, "foo", raises( InterpolationResolutionError, @@ -104,15 +101,6 @@ def test_dict_keys(cfg: Any, key: Any, expected: Any) -> None: ), id="type_error_dictconfig", ), - param( - {"foo": "${oc.dict.keys_or_values:.bar}", "bar": {"x": 0}}, - "foo", - raises( - InterpolationResolutionError, - match=re.escape("NotImplementedError"), - ), - id="relative_with_select", - ), ], ) def test_get_and_validate_dict_input( @@ -120,8 +108,8 @@ def test_get_and_validate_dict_input( ) -> None: OmegaConf.register_new_resolver( "oc.dict.keys_or_values", - lambda in_dict, _root_: _get_and_validate_dict_input( - in_dict, root=_root_, resolver_name="oc.dict.keys_or_values" + lambda in_dict, _parent_: _get_and_validate_dict_input( + in_dict, parent=_parent_, resolver_name="oc.dict.keys_or_values" ), ) cfg = OmegaConf.create(cfg) @@ -132,50 +120,26 @@ def test_get_and_validate_dict_input( @mark.parametrize( ("cfg", "key", "expected_val", "expected_content"), [ - param( - {"foo": "${oc.dict.values:{a: 0, b: 1}}"}, - "foo", - [0, 1], - [0, 1], - id="dict", - ), - param( - {"foo": "${oc.dict.values:${bar}}", "bar": {"a": 0, "b": 1}}, - "foo", - [0, 1], - ["${bar.a}", "${bar.b}"], - id="dictconfig", - ), param( {"foo": "${oc.dict.values:bar}", "bar": {"a": 0, "b": 1}}, "foo", [0, 1], ["${bar.a}", "${bar.b}"], - id="dictconfig_select", - ), - param( - { - "foo": "${oc.dict.values:${bar}}", - "bar": {"x": {"x0": 0, "x1": 1}, "y": {"y0": 0}}, - }, - "foo", - [{"x0": 0, "x1": 1}, {"y0": 0}], - ["${bar.x}", "${bar.y}"], - id="convert_node_to_list", + id="dictconfig", ), param( { - "foo": "${oc.dict.values:{key: ${val_ref}}}", - "val_ref": "value", + "foo": "${oc.dict.values:bar}", + "bar": {"a": {"x": 0, "y": 1}, "b": {"x": 0}}, }, "foo", - ["value"], - ["value"], - id="dict_with_interpolated_value", + [{"x": 0, "y": 1}, {"x": 0}], + ["${bar.a}", "${bar.b}"], + id="dictconfig_deep", ), param( { - "foo": "${oc.dict.values:${bar}}", + "foo": "${oc.dict.values:bar}", "bar": {"key": "${val_ref}"}, "val_ref": "value", }, @@ -186,13 +150,13 @@ def test_get_and_validate_dict_input( ), param( { - "foo": "${oc.dict.values:${bar}}", + "foo": "${oc.dict.values:bar}", "bar": "${boz}", "boz": {"a": 0, "b": 1}, }, "foo", [0, 1], - ["${boz.a}", "${boz.b}"], + ["${bar.a}", "${bar.b}"], id="dictconfig_chained_interpolation", ), ], @@ -231,7 +195,7 @@ def test_dict_values_with_missing_value() -> None: param( lambda _parent_: OmegaConf.create({"x": "${y}", "y": 1}, parent=_parent_), 0, - 999, # now referring to the `y` node from the global config + 999, # referring to the `y` node from the global config id="inter_abs", ), param( @@ -240,6 +204,19 @@ def test_dict_values_with_missing_value() -> None: 1, id="inter_rel", ), + param(lambda: OmegaConf.create({"x": 1}), 0, 1, id="basic_no_parent"), + param( + lambda: OmegaConf.create({"x": "${y}", "y": 1}), + 0, + 1, # no parent => referring to the local `y` node in the generated config + id="inter_abs_no_parent", + ), + param( + lambda: OmegaConf.create({"x": "${.y}", "y": 1}), + 0, + 1, + id="inter_rel_no_parent", + ), ], ) def test_dict_values_dictconfig_resolver_output( @@ -248,44 +225,13 @@ def test_dict_values_dictconfig_resolver_output( OmegaConf.register_new_resolver("make", make_resolver) cfg = OmegaConf.create( { - "foo": "${oc.dict.values:${make:}}", - "bar": "${oc.dict.values:${boz}}", - "boz": "${make:}", + "foo": "${oc.dict.values:bar}", + "bar": "${make:}", "y": 999, } ) - # Directly using the resolver output without an intermediate node to store the - # result is not currently supported. - with raises(InterpolationResolutionError, match=re.escape("NotImplementedError")): - cfg.foo - - assert cfg.bar[key] == expected - - -@mark.parametrize( - ("make_resolver", "key", "expected"), - [ - param(lambda: OmegaConf.create({"x": 1}), 0, 1, id="basic"), - param(lambda: OmegaConf.create({"x": "${y}", "y": 1}), 0, 1, id="inter_abs"), - param(lambda: OmegaConf.create({"x": "${.y}", "y": 1}), 0, 1, id="inter_rel"), - ], -) -def test_dict_values_dictconfig_resolver_output_no_parent( - restore_resolvers: Any, make_resolver: Any, key: Any, expected: Any -) -> None: - OmegaConf.register_new_resolver("make", make_resolver) - cfg = OmegaConf.create( - { - "bar": "${oc.dict.values:${boz}}", - "boz": "${make:}", - "y": 999, - } - ) - - # Using transient nodes whose parent is not set is not currently supported. - with raises(InterpolationResolutionError, match=re.escape("NotImplementedError")): - cfg.bar + assert cfg.foo[key] == expected @mark.parametrize( @@ -320,7 +266,7 @@ def test_dict_values_transient_interpolation( def test_dict_values_are_typed() -> None: cfg = OmegaConf.create( { - "x": "${oc.dict.values:${y.name2user}}", + "x": "${oc.dict.values: y.name2user}", "y": Users( name2user={ "john": User(name="john", age=30), @@ -337,8 +283,6 @@ def test_dict_values_are_typed() -> None: @mark.parametrize( ("cfg", "expected"), [ - param({"x": "${oc.dict.values:{a: 1, b: 2}}"}, [1, 2], id="values_inline"), - param({"x": "${oc.dict.keys:{a: 1, b: 2}}"}, ["a", "b"], id="keys_inline"), param({"x": "${oc.dict.values:y}", "y": {"a": 1}}, [1], id="values_inter"), param({"x": "${oc.dict.keys:y}", "y": {"a": 1}}, ["a"], id="keys_inter"), ], @@ -352,8 +296,16 @@ def test_readonly_parent(cfg: Any, expected: Any) -> None: @mark.parametrize( ("cfg", "expected"), [ - param({"x": "${sum:${oc.dict.values:{one: 1, two: 2}}}"}, 3, id="values"), - param({"x": "${sum:${oc.dict.keys:{1: one, 2: two}}}"}, 3, id="keys"), + param( + {"x": "${sum:${oc.dict.values:y}}", "y": {"one": 1, "two": 2}}, + 3, + id="values", + ), + param( + {"x": "${sum:${oc.dict.keys:y}}", "y": {1: "one", 2: "two"}}, + 3, + id="keys", + ), ], ) def test_nested_oc_dict(restore_resolvers: Any, cfg: Any, expected: Any) -> None: @@ -393,10 +345,9 @@ def test_dict_values_invalid_type(cfg: Any) -> None: def test_dict_values_of_root(restore_resolvers: Any) -> None: - OmegaConf.register_new_resolver("root", lambda _root_: _root_) - cfg = OmegaConf.create({"x": {"a": "${oc.dict.values:${root:}}"}, "y": 0}) + cfg = OmegaConf.create({"x": {"a": "${oc.dict.values:z}"}, "y": 0, "z": "${}"}) a = cfg.x.a - assert a._content == ["${x}", "${y}"] + assert a._content == ["${z.x}", "${z.y}", "${z.z}"] assert a[1] == 0 # We can recurse indefinitely within the first value if we fancy it. assert cfg.x.a[0].a[0].a[1] == 0 From 5692ed9a8da01c85868b34f25e33fbb49485e533 Mon Sep 17 00:00:00 2001 From: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Date: Sat, 10 Apr 2021 08:59:09 -0400 Subject: [PATCH 15/17] Update doc --- docs/source/usage.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/usage.rst b/docs/source/usage.rst index b40532c67..e21afe276 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -472,8 +472,8 @@ Extracting lists of keys / values from a dictionary Some config options that are stored as a ``DictConfig`` may sometimes be easier to manipulate as lists, when we care only about the keys or the associated values. -The resolvers ``oc.dict.keys`` and ``oc.dict.values`` simplify such operations by extracting respectively -the list of keys and values from existing config nodes. +The resolvers ``oc.dict.keys`` and ``oc.dict.values`` simplify such operations by offering an alternative +view of a dictionary's keys or values as a list. They take as input a string that is the path to another config node (using the same syntax as interpolations) and return a ``ListConfig`` with its keys / values. From 517b852567203a3517f8b66d67868d3374e99e79 Mon Sep 17 00:00:00 2001 From: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Date: Sat, 10 Apr 2021 09:01:25 -0400 Subject: [PATCH 16/17] Update news item --- news/643.feature | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/news/643.feature b/news/643.feature index cd3e4fc19..a7fd0f083 100644 --- a/news/643.feature +++ b/news/643.feature @@ -1,2 +1 @@ -New resolvers `oc.dict.keys` and `oc.dict.values` allow extracting the lists of keys and values of config nodes - +New resolvers `oc.dict.keys` and `oc.dict.values` provide a list view of the keys or values of a DictConfig node. From d1a4ef71d984a5145bc66a53f610f8fae0f4628b Mon Sep 17 00:00:00 2001 From: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Date: Sat, 10 Apr 2021 09:19:23 -0400 Subject: [PATCH 17/17] Small code simplification --- omegaconf/built_in_resolvers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omegaconf/built_in_resolvers.py b/omegaconf/built_in_resolvers.py index 877adbb6a..885d6e4b3 100644 --- a/omegaconf/built_in_resolvers.py +++ b/omegaconf/built_in_resolvers.py @@ -62,7 +62,7 @@ def dict_values(key: str, _root_: BaseContainer, _parent_: Container) -> ListCon assert isinstance(content, dict) ret = ListConfig([]) - for k, node in content.items(): + for k in content: ref_node = AnyNode(f"${{{key}.{k}}}") ret.append(ref_node)