add pathlib.Path support (#873)

Co-authored-by: Jasha <8935917+Jasha10@users.noreply.github.com>

docs/source/structured_config.rst

@@ -6,6 +6,7 @@
     from enum import Enum
     from dataclasses import dataclass, field
     import os
+    import pathlib
     from pytest import raises
     from typing import Dict, Any
     import sys
@@ -29,7 +30,7 @@ in the input class.
 
 
 Currently, type hints supported in OmegaConf’s structured configs include:
- - primitive types (int, float, bool, str) and enum types (user-defined
+ - primitive types (int, float, bool, str, Path) and enum types (user-defined
    subclasses of enum.Enum). See the :ref:`simple_types` section below.
  - structured config fields (i.e. MyConfig.x can have type hint MySubConfig).
    See the :ref:`nesting_structured_configs` section below.
@@ -50,6 +51,7 @@ Simple types include
  - bool: boolean values (True, False, On, Off etc)
  - str: any string
  - bytes: an immutable sequence of numbers in [0, 255]
+ - pathlib.Path: filesystem paths as represented by python's standard library `pathlib`
  - Enums: User defined enums
 
 The following class defines fields with all simple types:
@@ -68,6 +70,7 @@ The following class defines fields with all simple types:
     ...     height: Height = Height.SHORT
     ...     description: str = "text"
     ...     data: bytes = b"bin_data"
+    ...     path: pathlib.Path = pathlib.Path("hello.txt")
 
 You can create a config based on the SimpleTypes class itself or an instance of it.
 Those would be equivalent by default, but the Object variant allows you to set the values of specific
@@ -91,6 +94,8 @@ fields during construction.
     description: text
     data: !!binary |
       YmluX2RhdGE=
+    path: !!python/object/apply:pathlib.PosixPath
+    - hello.txt
     <BLANKLINE>
 
 The resulting object is a regular OmegaConf ``DictConfig``, except that it will utilize the type information in the input class/object
@@ -229,7 +234,7 @@ You can assign subclasses:
 Lists
 ^^^^^
 Structured Config fields annotated with ``typing.List`` or ``typing.Tuple`` can hold any type
-supported by OmegaConf (``int``, ``float``. ``bool``, ``str``, ``bytes``, ``Enum`` or Structured configs).
+supported by OmegaConf (``int``, ``float``. ``bool``, ``str``, ``bytes``, ``pathlib.Path``, ``Enum`` or Structured configs).
 
 .. doctest::
 
@@ -242,7 +247,7 @@ supported by OmegaConf (``int``, ``float``. ``bool``, ``str``, ``bytes``, ``Enum
     >>> @dataclass
     ... class ListsExample:
     ...     # Typed list can hold Any, int, float, bool, str,
-    ...     # bytes and Enums as well as arbitrary Structured configs.
+    ...     # bytes, pathlib.Path and Enums as well as arbitrary Structured configs.
     ...     ints: List[int] = field(default_factory=lambda: [10, 20, 30])
     ...     bools: Tuple[bool, bool] = field(default_factory=lambda: (True, False))
     ...     users: List[User] = field(default_factory=lambda: [User(name="omry")])
@@ -271,17 +276,15 @@ Dictionaries
 ^^^^^^^^^^^^
 Dictionaries are supported via annotation of structured config fields with ``typing.Dict``.
 Keys must be typed as one of ``str``, ``int``, ``Enum``, ``float``, ``bytes``, or ``bool``. Values can
-be any of the types supported by OmegaConf (``Any``, ``int``, ``float``, ``bool``, ``bytes``, ``str`` and ``Enum`` as well
-as arbitrary Structured configs)
+be any of the types supported by OmegaConf (``Any``, ``int``, ``float``, ``bool``, ``bytes``,
+``pathlib.Path``, ``str`` and ``Enum`` as well as arbitrary Structured configs)
 
 .. doctest::
 
     >>> from dataclasses import dataclass, field
     >>> from typing import Dict
     >>> @dataclass
     ... class DictExample:
-    ...     # Typed dict keys are strings; values can be typed as Any, int, float, bool, str, bytes and Enums or
-    ...     # arbitrary Structured configs
     ...     ints: Dict[str, int] = field(default_factory=lambda: {"a": 10, "b": 20, "c": 30})
     ...     bools: Dict[str, bool] = field(default_factory=lambda: {"Uno": True, "Zoro": False})
     ...     users: Dict[str, User] = field(default_factory=lambda: {"omry": User(name="omry")})

news/97.feature

@@ -0,0 +1 @@
+Add support for `pathlib.Path`-typed values.

omegaconf/__init__.py

@@ -15,6 +15,7 @@
     EnumNode,
     FloatNode,
     IntegerNode,
+    PathNode,
     StringNode,
     ValueNode,
 )
@@ -53,6 +54,7 @@
     "IntegerNode",
     "StringNode",
     "BytesNode",
+    "PathNode",
     "BooleanNode",
     "EnumNode",
     "FloatNode",

omegaconf/_utils.py

@@ -1,5 +1,6 @@
 import copy
 import os
+import pathlib
 import re
 import string
 import sys
@@ -151,6 +152,20 @@ def construct_mapping(self, node: yaml.Node, deep: bool = False) -> Any:
         ]
         for key, resolvers in loader.yaml_implicit_resolvers.items()
     }
+
+    loader.add_constructor(
+        "tag:yaml.org,2002:python/object/apply:pathlib.Path",
+        lambda loader, node: pathlib.Path(*loader.construct_sequence(node)),
+    )
+    loader.add_constructor(
+        "tag:yaml.org,2002:python/object/apply:pathlib.PosixPath",
+        lambda loader, node: pathlib.PosixPath(*loader.construct_sequence(node)),
+    )
+    loader.add_constructor(
+        "tag:yaml.org,2002:python/object/apply:pathlib.WindowsPath",
+        lambda loader, node: pathlib.WindowsPath(*loader.construct_sequence(node)),
+    )
+
     return loader
 
 
@@ -680,16 +695,19 @@ def _valid_dict_key_annotation_type(type_: Any) -> bool:
     return type_ is None or type_ is Any or issubclass(type_, DictKeyType.__args__)  # type: ignore
 
 
+BASE_TYPES = (
+    int,
+    float,
+    bool,
+    bytes,
+    str,
+    type(None),
+)
+
+
 def is_primitive_type_annotation(type_: Any) -> bool:
     type_ = get_type_of(type_)
-    return issubclass(type_, Enum) or type_ in (
-        int,
-        float,
-        bool,
-        str,
-        bytes,
-        type(None),
-    )
+    return issubclass(type_, (Enum, pathlib.Path)) or type_ in BASE_TYPES
 
 
 def _get_value(value: Any) -> Any:

omegaconf/nodes.py

@@ -3,6 +3,7 @@
 import sys
 from abc import abstractmethod
 from enum import Enum
+from pathlib import Path
 from typing import Any, Dict, Optional, Type, Union
 
 from omegaconf._utils import (
@@ -174,7 +175,7 @@ def _validate_and_convert_impl(self, value: Any) -> str:
         if (
             OmegaConf.is_config(value)
             or is_primitive_container(value)
-            or isinstance(value, bytes)
+            or isinstance(value, (bytes, Path))
         ):
             raise ValidationError("Cannot convert '$VALUE_TYPE' to string: '$VALUE'")
         return str(value)
@@ -185,6 +186,41 @@ def __deepcopy__(self, memo: Dict[int, Any]) -> "StringNode":
         return res
 
 
+class PathNode(ValueNode):
+    def __init__(
+        self,
+        value: Any = None,
+        key: Any = None,
+        parent: Optional[Container] = None,
+        is_optional: bool = True,
+        flags: Optional[Dict[str, bool]] = None,
+    ):
+        super().__init__(
+            parent=parent,
+            value=value,
+            metadata=Metadata(
+                key=key,
+                optional=is_optional,
+                ref_type=Path,
+                object_type=Path,
+                flags=flags,
+            ),
+        )
+
+    def _validate_and_convert_impl(self, value: Any) -> Path:
+        if not isinstance(value, (str, Path)):
+            raise ValidationError(
+                "Value '$VALUE' of type '$VALUE_TYPE' could not be converted to Path"
+            )
+
+        return Path(value)
+
+    def __deepcopy__(self, memo: Dict[int, Any]) -> "PathNode":
+        res = PathNode()
+        self._deepcopy_impl(res, memo)
+        return res
+
+
 class IntegerNode(ValueNode):
     def __init__(
         self,

omegaconf/omegaconf.py

@@ -67,6 +67,7 @@
     EnumNode,
     FloatNode,
     IntegerNode,
+    PathNode,
     StringNode,
     ValueNode,
 )
@@ -1077,6 +1078,8 @@ def _node_wrap(
         node = StringNode(value=value, key=key, parent=parent, is_optional=is_optional)
     elif ref_type == bytes:
         node = BytesNode(value=value, key=key, parent=parent, is_optional=is_optional)
+    elif ref_type == pathlib.Path:
+        node = PathNode(value=value, key=key, parent=parent, is_optional=is_optional)
     else:
         if parent is not None and parent._get_flag("allow_objects") is True:
             node = AnyNode(value=value, key=key, parent=parent)

tests/examples/dataclass_postponed_annotations.py

@@ -4,6 +4,7 @@
 
 from dataclasses import dataclass, fields
 from enum import Enum
+from pathlib import Path
 
 from pytest import raises
 
@@ -23,6 +24,7 @@ class SimpleTypes:
     height: "Height" = Height.SHORT  # test forward ref
     description: str = "text"
     data: bytes = b"bin_data"
+    path: Path = Path("hello.txt")
 
 
 def simple_types_class() -> None:
@@ -36,6 +38,7 @@ def simple_types_class() -> None:
     assert conf.num == 10
     assert conf.pi == 3.1415
     assert conf.data == b"bin_data"
+    assert conf.path == Path("hello.txt")
     assert conf.is_awesome is True
     assert conf.height == Height.SHORT
     assert conf.description == "text"

tests/interpolation/test_interpolation.py

@@ -1,5 +1,6 @@
 import copy
 import re
+from pathlib import Path
 from textwrap import dedent
 from typing import Any, Tuple
 
@@ -142,6 +143,7 @@ def test_indirect_interpolation2() -> None:
         param({"a": "${b}", "b": 3.14, "s": "foo_${b}"}, id="float"),
         param({"a": "${b}", "b": Color.RED, "s": "foo_${b}"}, id="enum"),
         param({"a": "${b}", "b": b"binary", "s": "foo_${b}"}, id="bytes"),
+        param({"a": "${b}", "b": Path("hello.txt"), "s": "foo_${b}"}, id="path"),
     ],
 )
 def test_type_inherit_type(cfg: Any) -> None:

tests/structured_conf/data/attr_classes.py

@@ -1,4 +1,5 @@
 import sys
+from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, Union
 
 import attr
@@ -175,6 +176,21 @@ class BytesConfig:
     interpolation: bytes = II("with_default")
 
 
+@attr.s(auto_attribs=True)
+class PathConfig:
+    # with default value
+    with_default: Path = Path("hello.txt")
+
+    # default is None
+    null_default: Optional[Path] = None
+
+    # explicit no default
+    mandatory_missing: Path = MISSING
+
+    # interpolation, will inherit the type and value of `with_default'
+    interpolation: Path = II("with_default")
+
+
 @attr.s(auto_attribs=True)
 class EnumConfig:
     # with default value

tests/structured_conf/data/dataclasses.py

@@ -1,6 +1,7 @@
 import dataclasses
 import sys
 from dataclasses import dataclass, field
+from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, Union
 
 from pytest import importorskip
@@ -176,6 +177,21 @@ class BytesConfig:
     interpolation: bytes = II("with_default")
 
 
+@dataclass
+class PathConfig:
+    # with default value
+    with_default: Path = Path("hello.txt")
+
+    # default is None
+    null_default: Optional[Path] = None
+
+    # explicit no default
+    mandatory_missing: Path = MISSING
+
+    # interpolation, will inherit the type and value of `with_default'
+    interpolation: Path = II("with_default")
+
+
 @dataclass
 class EnumConfig:
     # with default value