-
Notifications
You must be signed in to change notification settings - Fork 11
/
_implementations.py
2371 lines (1941 loc) · 86.1 KB
/
_implementations.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Copyright (c) 2021 Massachusetts Institute of Technology
# SPDX-License-Identifier: MIT
import inspect
import sys
import warnings
from collections import Counter, defaultdict, deque
from dataclasses import (
MISSING,
Field,
InitVar,
dataclass,
field,
fields,
is_dataclass,
make_dataclass,
)
from enum import Enum
from functools import wraps
from itertools import chain
from typing import (
TYPE_CHECKING,
Any,
Callable,
Dict,
FrozenSet,
List,
Mapping,
Optional,
Sequence,
Set,
Tuple,
Type,
TypeVar,
Union,
cast,
get_type_hints,
overload,
)
from omegaconf import DictConfig, ListConfig
from typing_extensions import Final, Literal, TypeGuard
from hydra_zen.errors import (
HydraZenDeprecationWarning,
HydraZenUnsupportedPrimitiveError,
HydraZenValidationError,
)
from hydra_zen.funcs import get_obj, partial, zen_processing
from hydra_zen.structured_configs import _utils
from hydra_zen.typing import Builds, Importable, Just, PartialBuilds, SupportedPrimitive
from hydra_zen.typing._implementations import (
DataClass,
HasPartialTarget,
HasTarget,
_DataClass,
)
from ._value_conversion import ZEN_SUPPORTED_PRIMITIVES, ZEN_VALUE_CONVERSION
_T = TypeVar("_T")
_T2 = TypeVar("_T2", bound=Callable)
ZenWrapper = Union[
None,
Builds[Callable[[_T2], _T2]],
PartialBuilds[Callable[[_T2], _T2]],
Just[Callable[[_T2], _T2]],
Type[Builds[Callable[[_T2], _T2]]],
Type[PartialBuilds[Callable[[_T2], _T2]]],
Type[Just[Callable[[_T2], _T2]]],
Callable[[_T2], _T2],
str,
]
if TYPE_CHECKING: # pragma: no cover
ZenWrappers = Union[ZenWrapper, Sequence[ZenWrapper]]
else:
ZenWrappers = TypeVar("ZenWrappers")
# Hydra-specific fields
_TARGET_FIELD_NAME: Final[str] = "_target_"
_RECURSIVE_FIELD_NAME: Final[str] = "_recursive_"
_CONVERT_FIELD_NAME: Final[str] = "_convert_"
_POS_ARG_FIELD_NAME: Final[str] = "_args_"
_HYDRA_FIELD_NAMES: FrozenSet[str] = frozenset(
(
_TARGET_FIELD_NAME,
_RECURSIVE_FIELD_NAME,
_CONVERT_FIELD_NAME,
_POS_ARG_FIELD_NAME,
)
)
# hydra-zen-specific fields
_ZEN_PROCESSING_LOCATION: Final[str] = _utils.get_obj_path(zen_processing)
_ZEN_TARGET_FIELD_NAME: Final[str] = "_zen_target"
_PARTIAL_TARGET_FIELD_NAME: Final[str] = "_zen_partial"
_META_FIELD_NAME: Final[str] = "_zen_exclude"
_ZEN_WRAPPERS_FIELD_NAME: Final[str] = "_zen_wrappers"
_JUST_FIELD_NAME: Final[str] = "path"
# TODO: add _JUST_Target
# signature param-types
_POSITIONAL_ONLY: Final = inspect.Parameter.POSITIONAL_ONLY
_POSITIONAL_OR_KEYWORD: Final = inspect.Parameter.POSITIONAL_OR_KEYWORD
_VAR_POSITIONAL: Final = inspect.Parameter.VAR_POSITIONAL
_KEYWORD_ONLY: Final = inspect.Parameter.KEYWORD_ONLY
_VAR_KEYWORD: Final = inspect.Parameter.VAR_KEYWORD
NoneType = type(None)
HYDRA_SUPPORTED_PRIMITIVES = {int, float, bool, str, list, tuple, dict, NoneType}
_builtin_function_or_method_type = type(len)
def _cast_via_tuple(dest_type: Type[_T]) -> Callable[[_T], Type[Builds[Type[_T]]]]:
def converter(value):
return builds(dest_type, tuple(value))
return converter
ZEN_VALUE_CONVERSION[set] = _cast_via_tuple(set)
ZEN_VALUE_CONVERSION[frozenset] = _cast_via_tuple(frozenset)
ZEN_VALUE_CONVERSION[deque] = _cast_via_tuple(deque)
ZEN_VALUE_CONVERSION[bytes] = _cast_via_tuple(bytes)
ZEN_VALUE_CONVERSION[bytearray] = _cast_via_tuple(bytearray)
ZEN_VALUE_CONVERSION[range] = lambda value: builds(
range, value.start, value.stop, value.step
)
ZEN_VALUE_CONVERSION[Counter] = lambda counter: builds(Counter, dict(counter))
def _get_target(x):
return getattr(x, _TARGET_FIELD_NAME)
def _target_as_kwarg_deprecation(func: _T2) -> Callable[..., _T2]:
@wraps(func)
def wrapped(*args, **kwargs):
if not args and "target" in kwargs:
# builds(target=<>, ...) is deprecated
warnings.warn(
HydraZenDeprecationWarning(
"Specifying the target of `builds` as a keyword argument is deprecated "
"as of 2021-10-27. Change `builds(target=<target>, ...)` to `builds(<target>, ...)`."
"\n\nThis will be an error in hydra-zen 1.0.0, or by 2021-01-27 — whichever "
"comes first.\n\nNote: This deprecation does not impact yaml configs "
"produced by `builds`."
),
stacklevel=2,
)
target = kwargs.pop("target")
return func(target, *args, **kwargs)
return func(*args, **kwargs)
return wrapped
def _hydra_partial_deprecation(func: _T2) -> Callable[..., _T2]:
@wraps(func)
def wrapped(*args, **kwargs):
if "hydra_partial" in kwargs:
if "zen_partial" in kwargs:
raise TypeError(
"Both `hydra_partial` and `zen_partial` are specified. "
"Specifying `hydra_partial` is deprecated, use `zen_partial` "
"instead."
)
# builds(..., hydra_partial=...) is deprecated
warnings.warn(
HydraZenDeprecationWarning(
"The argument `hydra_partial` is deprecated as of 2021-10-27.\n"
"Change `builds(..., hydra_partial=<..>)` to `builds(..., zen_partial=<..>)`."
"\n\nThis will be an error in hydra-zen 1.0.0, or by 2022-01-27 — whichever "
"comes first.\n\nNote: This deprecation does not impact yaml configs "
"produced by `builds`."
),
stacklevel=2,
)
kwargs["zen_partial"] = kwargs.pop("hydra_partial")
return func(*args, **kwargs)
return wrapped
def mutable_value(x: _T) -> _T:
"""Used to set a mutable object as a default value for a field
in a dataclass.
This is an alias for ``field(default_factory=lambda: type(x)(x))``
Note that ``type(x)(...)`` serves to make a copy
Examples
--------
>>> from hydra_zen import mutable_value
>>> from dataclasses import dataclass
See https://docs.python.org/3/library/dataclasses.html#mutable-default-values
>>> @dataclass # doctest: +SKIP
... class HasMutableDefault:
... a_list: list = [1, 2, 3] # error: mutable default
Using `mutable_value` to specify the default list:
>>> @dataclass
... class HasMutableDefault:
... a_list: list = mutable_value([1, 2, 3]) # ok
>>> x = HasMutableDefault()
>>> x.a_list.append(-1) # does not append to `HasMutableDefault.a_list`
>>> x
HasMutableDefault(a_list=[1, 2, 3, -1])
>>> HasMutableDefault()
HasMutableDefault(a_list=[1, 2, 3])"""
cast = type(x) # ensure that we return a copy of the default value
x = sanitize_collection(x)
return field(default_factory=lambda: cast(x))
Field_Entry = Tuple[str, type, Field]
# Alternate form, from PEP proposal:
# https://github.com/microsoft/pyright/blob/master/specs/dataclass_transforms.md
#
# This enables static checkers to work with third-party decorators that create
# dataclass-like objects
def __dataclass_transform__(
*,
eq_default: bool = True,
order_default: bool = False,
kw_only_default: bool = False,
field_descriptors: Tuple[Union[type, Callable[..., Any]], ...] = (()),
) -> Callable[[_T], _T]:
# If used within a stub file, the following implementation can be
# replaced with "...".
return lambda a: a
@__dataclass_transform__()
def hydrated_dataclass(
target: Callable,
*pos_args: SupportedPrimitive,
zen_partial: bool = False,
zen_wrappers: ZenWrappers = tuple(),
zen_meta: Optional[Mapping[str, Any]] = None,
populate_full_signature: bool = False,
hydra_recursive: Optional[bool] = None,
hydra_convert: Optional[Literal["none", "partial", "all"]] = None,
frozen: bool = False,
**_kw, # reserved to deprecate hydra_partial
) -> Callable[[Type[_T]], Type[_T]]:
"""A decorator that uses `builds` to create a dataclass with the appropriate
Hydra-specific fields for specifying a targeted config [1]_.
This provides similar functionality to `builds`, but enables a user to define
a config explicitly using the :func:`dataclasses.dataclass` syntax.
Parameters
----------
hydra_target : T (Callable)
The target-object to be configured. This is a required, positional-only argument.
*pos_args : SupportedPrimitive
Positional arguments passed as ``hydra_target(*pos_args, ...)`` upon instantiation.
Arguments specified positionally are not included in the dataclass' signature and
are stored as a tuple bound to in the ``_args_`` field.
zen_partial : bool, optional (default=False)
If ``True``, then the resulting config will instantiate as
``functools.partial(hydra_target, *pos_args, **kwargs_for_target)`` rather than
``hydra_target(*pos_args, **kwargs_for_target)``. Thus this enables the
partial-configuration of objects.
Specifying ``zen_partial=True`` and ``populate_full_signature=True`` together
will populate the config's signature only with parameters that: are explicitly
specified by the user, or that have default values specified in the target's
signature. I.e. it is presumed that un-specified parameters that have no
default values are to be excluded from the config.
zen_wrappers : None | Callable | Builds | InterpStr | Sequence[None | Callable | Builds | InterpStr]
One or more wrappers, which will wrap ``hydra_target`` prior to instantiation.
E.g. specifying the wrappers ``[f1, f2, f3]`` will instantiate as::
f3(f2(f1(hydra_target)))(*args, **kwargs)
Wrappers can also be specified as interpolated strings [2]_ or targeted
configs.
zen_meta : Optional[Mapping[str, SupportedPrimitive]]
Specifies field-names and corresponding values that will be included in the
resulting config, but that will *not* be used to builds ``<hydra_target>``
via instantiation. These are called "meta" fields.
populate_full_signature : bool, optional (default=False)
If ``True``, then the resulting config's signature and fields will be populated
according to the signature of ``hydra_target``; values also specified in
``**kwargs_for_target`` take precedent.
This option is not available for objects with inaccessible signatures, such as
NumPy's various ufuncs.
hydra_recursive : Optional[bool], optional (default=True)
If ``True``, then Hydra will recursively instantiate all other
hydra-config objects nested within this config [3]_.
If ``None``, the ``_recursive_`` attribute is not set on the resulting config.
- ``"none"``: No conversion occurs; omegaconf containers are passed through (Default)
- ``"partial"``: ``DictConfig`` and ``ListConfig`` objects converted to ``dict`` and
``list``, respectively. Structured configs and their fields are passed without conversion.
- ``"all"``: All passed objects are converted to dicts, lists, and primitives, without
a trace of OmegaConf containers.
If ``None``, the ``_convert_`` attribute is not set on the resulting config.
frozen : bool, optional (default=False)
If ``True``, the resulting config will create frozen (i.e. immutable) instances.
I.e. setting/deleting an attribute of an instance will raise
:py:class:`dataclasses.FrozenInstanceError` at runtime.
See Also
--------
builds : Create a targeted structured config designed to "build" a particular object.
Raises
------
hydra_zen.errors.HydraZenUnsupportedPrimitiveError
The provided configured value cannot be serialized by Hydra, nor does hydra-zen
provide specialized support for it. See :ref:`valid-types` for more details.
Notes
-----
Unlike `builds`, `hydrated_dataclass` enables config fields to be set explicitly
with custom type annotations. Additionally, the resulting config' attributes
can be analyzed by static tooling, which can help to warn about errors prior
to running one's code.
For details of the annotation `SupportedPrimitive`, see :ref:`valid-types`.
References
----------
.. [1] https://hydra.cc/docs/next/tutorials/structured_config/intro/
.. [2] https://omegaconf.readthedocs.io/en/2.1_branch/usage.html#variable-interpolation
.. [3] https://hydra.cc/docs/next/advanced/instantiate_objects/overview/#recursive-instantiation
.. [4] https://hydra.cc/docs/next/advanced/instantiate_objects/overview/#parameter-conversion-strategies
Examples
--------
**Basic usage**
>>> from hydra_zen import hydrated_dataclass, instantiate
Here, we specify a config that is designed to "build" a dictionary
upon instantiation
>>> @hydrated_dataclass(target=dict)
... class DictConf:
... x: int = 2
... y: str = 'hello'
>>> instantiate(DictConf(x=10)) # override default `x`
{'x': 10, 'y': 'hello'}
For more detailed examples, refer to `builds`.
"""
if "hydra_partial" in _kw:
if zen_partial is True:
raise TypeError(
"Both `hydra_partial` and `zen_partial` are specified. "
"Specifying `hydra_partial` is deprecated, use `zen_partial` "
"instead."
)
# builds(..., hydra_partial=...) is deprecated
warnings.warn(
HydraZenDeprecationWarning(
"The argument `hydra_partial` is deprecated as of 2021-10-27.\n"
"Change `builds(..., hydra_partial=<..>)` to `builds(..., zen_partial=<..>)`."
"\n\nThis will be an error in hydra-zen 1.0.0, or by 2022-01-27 — whichever "
"comes first.\n\nNote: This deprecation does not impact yaml configs "
"produced by `builds`."
),
stacklevel=2,
)
zen_partial = _kw.pop("hydra_partial")
if _kw:
raise TypeError(
f"hydrated_dataclass got an unexpected argument: {', '.join(_kw)}"
)
def wrapper(decorated_obj: Any) -> Any:
if not isinstance(decorated_obj, type):
raise NotImplementedError(
"Class instances are not supported by `hydrated_dataclass`."
)
# TODO: We should mutate `decorated_obj` directly like @dataclass does.
# Presently, we create an intermediate dataclass that we inherit
# from, which gets the job done for the most part but there are
# practical differences. E.g. you cannot delete an attribute that
# was declared in the definition of `decorated_obj`.
decorated_obj = cast(Any, decorated_obj)
decorated_obj = dataclass(frozen=frozen)(decorated_obj)
if _utils.PATCH_OMEGACONF_830 and 2 < len(decorated_obj.__mro__):
parents = decorated_obj.__mro__[1:-1]
# this class inherits from a parent
for field_ in fields(decorated_obj):
if field_.default_factory is not MISSING and any(
hasattr(p, field_.name) for p in parents
):
# TODO: update error message with fixed omegaconf version
_value = field_.default_factory()
raise HydraZenValidationError(
"This config will not instantiate properly.\nThis is due to a "
"known bug in omegaconf: The config specifies a "
f"default-factory for field {field_.name}, and inherits from a "
"parent that specifies the same field with a non-factory value "
"-- the parent's value will take precedence.\nTo circumvent "
f"this, specify {field_.name} using: "
f"`builds({type(_value).__name__}, {_value})`\n\nFor more "
"information, see: https://github.com/omry/omegaconf/issues/830"
)
if populate_full_signature:
# we need to ensure that the fields specified via the class definition
# take precedence over the fields that will be auto-populated by builds
kwargs = {
f.name: f.default if f.default is not MISSING else f.default_factory()
for f in fields(decorated_obj)
if not (f.default is MISSING and f.default_factory is MISSING)
and f.name not in _HYDRA_FIELD_NAMES
and not f.name.startswith("_zen_")
}
else:
kwargs = {}
return builds(
target,
*pos_args,
**kwargs,
populate_full_signature=populate_full_signature,
hydra_recursive=hydra_recursive,
hydra_convert=hydra_convert,
zen_wrappers=zen_wrappers,
zen_partial=zen_partial,
zen_meta=zen_meta,
builds_bases=(decorated_obj,),
dataclass_name=decorated_obj.__name__,
frozen=frozen,
)
return wrapper
def just(obj: Importable) -> Type[Just[Importable]]:
"""Produces a targeted config that, when instantiated by Hydra, 'just'
returns the target (un-instantiated).
Parameters
----------
obj : Importable
The object that will be instantiated from this config.
Returns
-------
config : Type[Just[Importable]]
See Also
--------
builds : Create a targeted structured config designed to "build" a particular object.
make_config: Creates a general config with customized field names, default values, and annotations.
Notes
-----
The configs produced by `just` introduce an explicit dependency on hydra-zen. I.e.
hydra-zen must be installed in order to instantiate any config that used `just`.
Examples
--------
**Basic usage**
>>> from hydra_zen import just, instantiate, to_yaml
>>> Conf = just(range)
>>> instantiate(Conf) is range
True
The config produced by `just` describes how to import the target,
not how to instantiate/call the object.
>>> print(to_yaml(Conf))
_target_: hydra_zen.funcs.get_obj
path: builtins.range
**Auto-Application of just**
Both `builds` and `make_config` will automatically apply `just` to default values
that are function-objects or class objects. E.g. in the following example `just`
will be applied to ``sum``.
>>> from hydra_zen import make_config
>>> Conf2 = make_config(data=[1, 2, 3], reduction_fn=sum)
>>> print(to_yaml(Conf2))
data:
- 1
- 2
- 3
reduction_fn:
_target_: hydra_zen.funcs.get_obj
path: builtins.sum
>>> conf = instantiate(Conf2)
>>> conf.reduction_fn(conf.data)
6
"""
try:
obj_path = _utils.get_obj_path(obj)
except AttributeError:
raise AttributeError(
f"`just({obj})`: `obj` is not importable; it is missing the attributes `__module__` and/or `__qualname__`"
)
out_class = make_dataclass(
("Just_" + _utils.safe_name(obj)),
[
(
_TARGET_FIELD_NAME,
str,
_utils.field(default=_utils.get_obj_path(get_obj), init=False),
),
(
"path",
str,
_utils.field(
default=obj_path,
init=False,
),
),
],
)
out_class.__doc__ = (
f"A structured config designed to return {obj} when it is instantiated by hydra"
)
return cast(Type[Just[Importable]], out_class)
_KEY_ERROR_PREFIX = "Configuring dictionary key:"
def _is_ufunc(value) -> bool:
# checks without importing numpy
numpy = sys.modules.get("numpy")
if numpy is None: # pragma: no cover
# we do actually cover this branch some runs of our CI,
# but our coverage job installs numpy
return False
return isinstance(value, numpy.ufunc) # type: ignore
def sanitized_default_value(
value: Any,
allow_zen_conversion: bool = True,
*,
error_prefix: str = "",
field_name: str = "",
structured_conf_permitted: bool = True,
) -> Any:
value = sanitize_collection(value)
if (
structured_conf_permitted
and callable(value)
and (
inspect.isfunction(value)
or (not is_dataclass(value) and inspect.isclass(value))
or isinstance(value, _builtin_function_or_method_type)
or _is_ufunc(value)
)
):
return just(value)
resolved_value = value
type_of_value = type(resolved_value)
# we don't use isinstance because we don't permit subclasses of supported
# primitives
if allow_zen_conversion and type_of_value in ZEN_SUPPORTED_PRIMITIVES:
type_ = type(resolved_value)
conversion_fn = ZEN_VALUE_CONVERSION.get(type_)
if conversion_fn is not None:
resolved_value = conversion_fn(resolved_value)
type_of_value = type(resolved_value)
if type_of_value in HYDRA_SUPPORTED_PRIMITIVES or (
structured_conf_permitted
and (
is_dataclass(resolved_value)
or isinstance(resolved_value, (Enum, ListConfig, DictConfig))
)
):
return resolved_value
if field_name:
field_name = f", for field `{field_name}`,"
err_msg = (
error_prefix
+ f" The configured value {value}{field_name} is not supported by Hydra -- "
f"serializing or instantiating this config would ultimately result in an error."
)
if structured_conf_permitted:
err_msg += f"\n\nConsider using `hydra_zen.builds({type(value)}, ...)` to "
"create a config for this particular value."
raise HydraZenUnsupportedPrimitiveError(err_msg)
def sanitize_collection(x: _T) -> _T:
"""Pass contents of lists, tuples, or dicts through sanitized_default_values"""
type_x = type(x)
if type_x in {list, tuple}:
return type_x(sanitized_default_value(_x) for _x in x) # type: ignore
elif type_x is dict:
return {
# Hydra doesn't permit structured configs for keys, thus we only
# support its basic primitives here.
sanitized_default_value(
k,
allow_zen_conversion=False,
structured_conf_permitted=False,
error_prefix=_KEY_ERROR_PREFIX,
): sanitized_default_value(v)
for k, v in x.items() # type: ignore
}
else:
# pass-through
return x
def sanitized_field(
value: Any,
init=True,
allow_zen_conversion: bool = True,
*,
error_prefix: str = "",
field_name: str = "",
_mutable_default_permitted: bool = True,
) -> Field:
type_value = type(value)
if (
type_value in _utils.KNOWN_MUTABLE_TYPES
and type_value in HYDRA_SUPPORTED_PRIMITIVES
):
if _mutable_default_permitted:
return cast(Field, mutable_value(value))
value = builds(type(value), value)
return _utils.field(
default=sanitized_default_value(
value,
allow_zen_conversion=allow_zen_conversion,
error_prefix=error_prefix,
field_name=field_name,
),
init=init,
)
# overloads when `zen_partial=False`
@overload
def builds(
hydra_target: Importable,
*pos_args: SupportedPrimitive,
zen_partial: Literal[False] = False,
zen_wrappers: ZenWrappers = tuple(),
zen_meta: Optional[Mapping[str, SupportedPrimitive]] = None,
populate_full_signature: bool = False,
hydra_recursive: Optional[bool] = None,
hydra_convert: Optional[Literal["none", "partial", "all"]] = None,
dataclass_name: Optional[str] = None,
builds_bases: Tuple[Type[_DataClass], ...] = (),
frozen: bool = False,
**kwargs_for_target: SupportedPrimitive,
) -> Type[Builds[Importable]]: # pragma: no cover
...
# overloads when `zen_partial=True`
@overload
def builds(
hydra_target: Importable,
*pos_args: SupportedPrimitive,
zen_partial: Literal[True],
zen_wrappers: ZenWrappers = tuple(),
zen_meta: Optional[Mapping[str, SupportedPrimitive]] = None,
populate_full_signature: bool = False,
hydra_recursive: Optional[bool] = None,
hydra_convert: Optional[Literal["none", "partial", "all"]] = None,
dataclass_name: Optional[str] = None,
builds_bases: Tuple[Type[_DataClass], ...] = (),
frozen: bool = False,
**kwargs_for_target: SupportedPrimitive,
) -> Type[PartialBuilds[Importable]]: # pragma: no cover
...
# overloads when `zen_partial: bool`
@overload
def builds(
hydra_target: Importable,
*pos_args: SupportedPrimitive,
zen_partial: bool,
zen_wrappers: ZenWrappers = tuple(),
zen_meta: Optional[Mapping[str, SupportedPrimitive]] = None,
populate_full_signature: bool = False,
hydra_recursive: Optional[bool] = None,
hydra_convert: Optional[Literal["none", "partial", "all"]] = None,
dataclass_name: Optional[str] = None,
builds_bases: Tuple[Type[_DataClass], ...] = (),
frozen: bool = False,
**kwargs_for_target: SupportedPrimitive,
) -> Union[
Type[Builds[Importable]], Type[PartialBuilds[Importable]]
]: # pragma: no cover
...
@_hydra_partial_deprecation
@_target_as_kwarg_deprecation
def builds(
*pos_args: Any,
zen_partial: bool = False,
zen_wrappers: ZenWrappers = tuple(),
zen_meta: Optional[Mapping[str, SupportedPrimitive]] = None,
populate_full_signature: bool = False,
hydra_recursive: Optional[bool] = None,
hydra_convert: Optional[Literal["none", "partial", "all"]] = None,
frozen: bool = False,
builds_bases: Tuple[Type[_DataClass], ...] = (),
dataclass_name: Optional[str] = None,
**kwargs_for_target: SupportedPrimitive,
) -> Union[Type[Builds[Importable]], Type[PartialBuilds[Importable]]]:
"""builds(hydra_target, /, *pos_args, zen_partial=False, zen_meta=None,
hydra_recursive=None, populate_full_signature=False, hydra_convert=None,
frozen=False, dataclass_name=None, builds_bases=(), **kwargs_for_target)
Returns a config, which describes how to instantiate/call ``<hydra_target>`` with
user-specified and auto-populated parameter values.
Consult the Examples section of the docstring to see the various features of
`builds` in action.
Parameters
----------
hydra_target : T (Callable)
The target object to be configured. This is a required, **positional-only**
argument.
*pos_args : SupportedPrimitive
Positional arguments passed as ``<hydra_target>(*pos_args, ...)`` upon
instantiation.
Arguments specified positionally are not included in the dataclass' signature
and are stored as a tuple bound to in the ``_args_`` field.
**kwargs_for_target : SupportedPrimitive
The keyword arguments passed as ``<hydra_target>(..., **kwargs_for_target)``
upon instantiation.
The arguments specified here solely determine the signature of the resulting
config, unless ``populate_full_signature=True`` is specified (see below).
Named parameters of the forms that have the prefixes ``hydra_``, ``zen_`` or
``_zen_`` are reserved to ensure future-compatibility, and thus cannot be
specified by the user.
zen_partial : bool, optional (default=False)
If ``True``, then the resulting config will instantiate as
``functools.partial(<hydra_target>, *pos_args, **kwargs_for_target)``. Thus
this enables the partial-configuration of objects.
Specifying ``zen_partial=True`` and ``populate_full_signature=True`` together
will populate the config's signature only with parameters that: are explicitly
specified by the user, or that have default values specified in the target's
signature. I.e. it is presumed that un-specified parameters that have no
default values are to be excluded from the config.
zen_wrappers : None | Callable | Builds | InterpStr | Sequence[None | Callable | Builds | InterpStr]
One or more wrappers, which will wrap ``hydra_target`` prior to instantiation.
E.g. specifying the wrappers ``[f1, f2, f3]`` will instantiate as::
f3(f2(f1(<hydra_target>)))(*args, **kwargs)
Wrappers can also be specified as interpolated strings [2]_ or targeted
configs.
zen_meta : Optional[Mapping[str, SupportedPrimitive]]
Specifies field-names and corresponding values that will be included in the
resulting config, but that will *not* be used to instantiate
``<hydra_target>``. These are called "meta" fields.
populate_full_signature : bool, optional (default=False)
If ``True``, then the resulting config's signature and fields will be populated
according to the signature of ``<hydra_target>``; values also specified in
``**kwargs_for_target`` take precedent.
This option is not available for objects with inaccessible signatures, such as
NumPy's various ufuncs.
hydra_recursive : Optional[bool], optional (default=True)
If ``True``, then Hydra will recursively instantiate all other
hydra-config objects nested within this config [3]_.
If ``None``, the ``_recursive_`` attribute is not set on the resulting config.
hydra_convert : Optional[Literal["none", "partial", "all"]], optional (default="none")
Determines how Hydra handles the non-primitive, omegaconf-specific objects passed to
``<hydra_target>`` [4]_.
- ``"none"``: No conversion occurs; omegaconf containers are passed through (Default)
- ``"partial"``: ``DictConfig`` and ``ListConfig`` objects converted to ``dict`` and
``list``, respectively. Structured configs and their fields are passed without conversion.
- ``"all"``: All passed objects are converted to dicts, lists, and primitives, without
a trace of OmegaConf containers.
If ``None``, the ``_convert_`` attribute is not set on the resulting config.
frozen : bool, optional (default=False)
If ``True``, the resulting config will create frozen (i.e. immutable) instances.
I.e. setting/deleting an attribute of an instance will raise
:py:class:`dataclasses.FrozenInstanceError` at runtime.
builds_bases : Tuple[DataClass, ...]
Specifies a tuple of parent classes that the resulting config inherits from.
A ``PartialBuilds`` class (resulting from ``zen_partial=True``) cannot be a
parent of a ``Builds`` class (i.e. where `zen_partial=False` was specified).
dataclass_name : Optional[str]
If specified, determines the name of the returned class object.
Returns
-------
Config : Type[Builds[Type[T]]] | Type[PartialBuilds[Type[T]]]
A structured config that describes how to build ``hydra_target``.
Raises
------
hydra_zen.errors.HydraZenUnsupportedPrimitiveError
The provided configured value cannot be serialized by Hydra, nor does hydra-zen
provide specialized support for it. See :ref:`valid-types` for more details.
Notes
-----
The resulting "config" is a dataclass-object [5]_ with Hydra-specific attributes
attached to it [1]_.
Using any of the ``zen_xx`` features will result in a config that depends
explicitly on hydra-zen. I.e. hydra-zen must be installed in order to
instantiate the resulting config, including its yaml version.
For details of the annotation `SupportedPrimitive`, see :ref:`valid-types`.
Type annotations are inferred from the target's signature and are only
retained if they are compatible with Hydra's limited set of supported
annotations; otherwise an annotation is automatically 'broadened' until
it is made compatible with Hydra.
`builds` provides runtime validation of user-specified arguments against
the target's signature. E.g. specifying mis-named arguments or too many
arguments will cause `builds` to raise.
References
----------
.. [1] https://hydra.cc/docs/next/tutorials/structured_config/intro/
.. [2] https://omegaconf.readthedocs.io/en/2.1_branch/usage.html#variable-interpolation
.. [3] https://hydra.cc/docs/next/advanced/instantiate_objects/overview/#recursive-instantiation
.. [4] https://hydra.cc/docs/next/advanced/instantiate_objects/overview/#parameter-conversion-strategies
.. [5] https://docs.python.org/3/library/dataclasses.html
.. [6] https://omegaconf.readthedocs.io/en/2.1_branch/usage.html#variable-interpolation
See Also
--------
instantiate: Instantiates a configuration created by `builds`, returning the instantiated target.
make_custom_builds_fn: Returns the `builds` function, but one with customized default values.
make_config: Creates a general config with customized field names, default values, and annotations.
get_target: Returns the target-object from a targeted structured config.
just: Produces a config that, when instantiated by Hydra, "just" returns the un-instantiated target-object.
to_yaml: Serialize a config as a yaml-formatted string.
Examples
--------
**Basic Usage**
Lets create a basic config that describes how to 'build' a particular dictionary.
>>> from hydra_zen import builds, instantiate
>>> Conf = builds(dict, a=1, b='x')
The resulting config is a dataclass with the following signature and attributes:
>>> Conf # signature: Conf(a: Any = 1, b: Any = 'x')
<class 'types.Builds_dict'>
>>> Conf.a
1
>>> Conf.b
'x'
The `instantiate` function is used to enact this build – to create the dictionary.
>>> instantiate(Conf) # calls: `dict(a=1, b='x')`
{'a': 1, 'b': 'x'}
The default parameters that we provided can be overridden.
>>> new_conf = Conf(a=10, b="hi") # an instance of our dataclass
>>> instantiate(new_conf) # calls: `dict(a=10, b='hi')`
{'a': 10, 'b': 'hi'}
Positional arguments can be provided too.
>>> Conf = builds(len, [1, 2, 3]) # specifying positional arguments
>>> instantiate(Conf)
3
**Creating a Partial Config**
`builds` can be used to partially-configure a target. Let's
create a config for the following function
>>> def a_two_tuple(x: int, y: float): return x, y
such that we only configure the parameter ``x``.
>>> PartialConf = builds(a_two_tuple, x=1, zen_partial=True) # configures only `x`
Instantiating this conf will return ``functools.partial(a_two_tuple, x=1)``.
>>> partial_func = instantiate(PartialConf)
>>> partial_func
functools.partial(<function a_two_tuple at 0x00000220A7820EE0>, x=1)
And thus the remaining parameter can be provided post-instantiation.
>>> partial_func(y=22.0) # providing the remaining parameter
(1, 22.0)
**Auto-populating parameters**
The configurable parameters of a target can be auto-populated in our config.
Suppose we want to configure the following function.
>>> def f(x: bool, y: str = 'foo'): return x, y
The following config will have a signature that matches ``f``; the
annotations and default values of the parameters of ``f`` are explicitly
incorporated into the config.
>>> Conf = builds(f, populate_full_signature=True) # signature: `Builds_f(x: bool, y: str = 'foo')`
>>> Conf.y
'foo'
Annotations will be used by Hydra to provide limited runtime type-checking during
instantiation. Here, we'll pass a float for ``x``, which expects a boolean value.
>>> instantiate(Conf(x=10.0))
ValidationError: Value '10.0' is not a valid bool (type float)
full_key: x
object_type=Builds_f
**Composing configs via inheritance**
Because a config produced via `builds` is simply a class-object, we can
compose configs via class inheritance.
>>> ParentConf = builds(dict, a=1, b=2)
>>> ChildConf = builds(dict, b=-2, c=-3, builds_bases=(ParentConf,))
>>> instantiate(ChildConf)
{'a': 1, 'b': -2, 'c': -3}
>>> issubclass(ChildConf, ParentConf)
True
.. _meta-field:
**Using meta-fields**
Meta-fields are fields that are included in a config but are excluded by the
instantiation process. Thus arbitrary metadata can be attached to a config.