Skip to content

Commit

Permalink
Merge 857563f into 5b097d1
Browse files Browse the repository at this point in the history
  • Loading branch information
ncilfone committed Jan 18, 2022
2 parents 5b097d1 + 857563f commit 1e5fb27
Show file tree
Hide file tree
Showing 19 changed files with 511 additions and 80 deletions.
1 change: 1 addition & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
include versioneer.py
include spock/_version.py
include "README.md", "LICENSE.txt", "NOTICE.txt", "CONTRIBUTING.md"
4 changes: 4 additions & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@
packages=setuptools.find_packages(
exclude=["*.tests", "*.tests.*", "tests.*", "tests"]
),
package_data={
"spock": ["py.typed", "*.pyi"],
},
include_package_data=True,
python_requires=">=3.6",
install_requires=install_reqs,
extras_require={"s3": s3_reqs, "tune": tune_reqs},
Expand Down
6 changes: 5 additions & 1 deletion spock/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,12 @@
"""

from spock._version import get_versions
from spock.builder import ConfigArgBuilder
from spock.config import spock, spock_attr

__all__ = ["args", "builder", "config"]
SpockBuilder = ConfigArgBuilder

__all__ = ["args", "builder", "config", "spock", "spock_attr", "SpockBuilder"]

__version__ = get_versions()["version"]
del get_versions
60 changes: 51 additions & 9 deletions spock/addons/tune/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,30 +44,72 @@ class OptunaTunerConfig:
directions: Optional[Sequence[Union[str, optuna.study.StudyDirection]]] = None


def _spock_tune(cls):
"""Ovverides basic spock_attr decorator with another name
Using a different name allows spock to easily determine which parameters are normal and which are
meant to be used in a hyper-parameter tuning backend
def _process_class(cls, kw_only: bool, make_init: bool, dynamic: bool):
"""Process a given class
Args:
cls: basic class def
cls: basic class definition
kw_only: set kwarg only
make_init: make an init function
dynamic: allows inherited classes to not be @spock decorated
Returns:
cls: slotted attrs class that is frozen and kw only
cls with attrs dunder methods added
"""
bases, attrs_dict = _base_attr(cls)
# Handles the MRO and gets old annotations
bases, attrs_dict, merged_annotations = _base_attr(cls, kw_only, make_init, dynamic)
# Dynamically make an attr class
obj = attr.make_class(
name=cls.__name__, bases=bases, attrs=attrs_dict, kw_only=True, frozen=True
name=cls.__name__,
bases=bases,
attrs=attrs_dict,
kw_only=kw_only,
frozen=True,
auto_attribs=True,
init=make_init,
)
# For each class we dynamically create we need to register it within the system modules for pickle to work
setattr(sys.modules["spock"].addons.tune.config, obj.__name__, obj)
# Swap the __doc__ string from cls to obj
obj.__doc__ = cls.__doc__
# Set the __init__ function
# Handle __annotations__ from the MRO
obj.__annotations__ = merged_annotations
return obj


def _spock_tune(
maybe_cls=None,
kw_only=True,
make_init=True,
):
"""Ovverides basic spock_attr decorator with another name
Using a different name allows spock to easily determine which parameters are normal and which are
meant to be used in a hyper-parameter tuning backend
Args:
maybe_cls: maybe a basic class def maybe None depending on call type
kw_only: Make all attributes keyword-only
make_init: bool, define a __init__() method
Returns:
cls: attrs class that is frozen and kw only
"""

def wrap(cls):
return _process_class(cls, kw_only=kw_only, make_init=make_init, dynamic=False)

# Note: Taken from dataclass/attr definition(s)
# maybe_cls's type depends on the usage of the decorator. It's a class
# if it's used as `@spockTuner` but ``None`` if used as `@spockTuner()`.
if maybe_cls is None:
return wrap
else:
return wrap(maybe_cls)


# Make the alias for the decorator
spockTuner = _spock_tune

Expand Down
39 changes: 39 additions & 0 deletions spock/addons/tune/config.pyi
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
from typing import Any, Callable, Tuple, TypeVar, Union, overload

from attr import attrib, field

_T = TypeVar("_T")
_C = TypeVar("_C", bound=type)

# Note: from here
# https://github.com/python-attrs/attrs/blob/main/src/attr/__init__.pyi

# Static type inference support via __dataclass_transform__ implemented as per:
# https://github.com/microsoft/pyright/blob/main/specs/dataclass_transforms.md
# This annotation must be applied to all overloads of "spock_attr"

# NOTE: This is a typing construct and does not exist at runtime. Extensions
# wrapping attrs decorators should declare a separate __dataclass_transform__
# signature in the extension module using the specification linked above to
# provide pyright support -- this currently doesn't work in PyCharm
def __dataclass_transform__(
*,
eq_default: bool = True,
order_default: bool = False,
kw_only_default: bool = False,
field_descriptors: Tuple[Union[type, Callable[..., Any]], ...] = (()),
) -> Callable[[_T], _T]: ...
@overload
@__dataclass_transform__(kw_only_default=True, field_descriptors=(attrib, field))
def _spock_tune(
maybe_cls: _C,
kw_only: bool = True,
make_init: bool = True,
) -> _C: ...
@overload
@__dataclass_transform__(kw_only_default=True, field_descriptors=(attrib, field))
def _spock_tune(
maybe_cls: None = ...,
kw_only: bool = True,
make_init: bool = True,
) -> Callable[[_C], _C]: ...
13 changes: 9 additions & 4 deletions spock/backend/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,13 @@ class BaseBuilder(ABC): # pylint: disable=too-few-public-methods
_max_indent: maximum to indent between help prints
_module_name: module name to register in the spock module space
save_path: list of path(s) to save the configs to
_lazy: attempts to lazily find @spock decorated classes registered within sys.modules["spock"].backend.config
"""

def __init__(self, *args, max_indent: int = 4, module_name: str, **kwargs):
def __init__(
self, *args, max_indent: int = 4, module_name: str, lazy: bool, **kwargs
):
"""Init call for BaseBuilder
Args:
Expand All @@ -46,7 +49,10 @@ def __init__(self, *args, max_indent: int = 4, module_name: str, **kwargs):
**kwargs: keyword args
"""
self._input_classes = args
self._graph = Graph(input_classes=self.input_classes)
self._lazy = lazy
self._graph = Graph(input_classes=self.input_classes, lazy=self._lazy)
# Make sure the input classes are updated -- lazy evaluation
self._input_classes = self._graph.nodes
self._module_name = module_name
self._max_indent = max_indent
self.save_path = None
Expand Down Expand Up @@ -103,8 +109,7 @@ def generate(self, dict_args):
Returns:
namespace containing automatically generated instances of the classes
"""
graph = Graph(input_classes=self.input_classes)
spock_space_kwargs = self.resolve_spock_space_kwargs(graph, dict_args)
spock_space_kwargs = self.resolve_spock_space_kwargs(self._graph, dict_args)
return Spockspace(**spock_space_kwargs)

def resolve_spock_space_kwargs(self, graph: Graph, dict_args: dict) -> dict:
Expand Down
137 changes: 115 additions & 22 deletions spock/backend/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,60 +10,153 @@
import attr

from spock.backend.typed import katra
from spock.exceptions import _SpockUndecoratedClass
from spock.utils import _is_spock_instance


def _base_attr(cls):
def _base_attr(cls, kw_only, make_init, dynamic):
"""Map type hints to katras
Connector function that maps type hinting style to the defined katra style which uses the more strict
attr.ib() definition
Handles dynamic decorators which allows for inheritance of non @spock decorated classes
Args:
cls: basic class def
dynamic: allows inherited classes to not be @spock decorated
Returns:
cls: slotted attrs class that is frozen and kw only
cls: base spock classes derived from the MRO
attrs_dict: the current dictionary of attr.attribute values
merged_annotations: dictionary of type annotations
"""
# Since we are not using the @attr.s decorator we need to get the parent classes for inheritance
# We do this by using the mro and grabbing anything that is not the first and last indices in the list and wrapping
# it into a tuple
bases = ()
base_annotation = {}
base_defaults = {}
if len(cls.__mro__[1:-1]) > 0:
bases = tuple(cls.__mro__[1:-1])
# if there are not parents pass a blank tuple
# Get bases minus self and python class object
bases = list(cls.__mro__[1:-1])
for idx, base_cls in enumerate(bases):
if not _is_spock_instance(base_cls) and not dynamic:
raise _SpockUndecoratedClass(
f"Class `{base_cls.__name__}` was not decorated with the @spock decorator "
f"and `dynamic={dynamic}` was set for child class `{cls.__name__}` -- Please remedy one of these"
)
elif not _is_spock_instance(base_cls) and dynamic:
bases[idx] = _process_class(base_cls, kw_only, make_init, dynamic)
bases = tuple(bases)
base_annotation = {}
for val in bases:
for attribute in val.__attrs_attrs__:
# Since we are moving left to right via the MRO only update if not currently present
# this maintains parity to how the MRO is handled in base python
if attribute.name not in base_annotation:
if "type" in attribute.metadata:
base_annotation.update(
{attribute.name: attribute.metadata["og_type"]}
)
else:
base_annotation.update({attribute.name: attribute.type})
base_defaults = {
attribute.name: attribute.default
for val in bases
for attribute in val.__attrs_attrs__
if attribute.default is not (None or attr.NOTHING)
}
# Merge the annotations -- always override as this is the lowest level of the MRO
if hasattr(cls, "__annotations__"):
new_annotations = {k: v for k, v in cls.__annotations__.items()}
else:
bases = ()
new_annotations = {}
merged_annotations = {**base_annotation, **new_annotations}

# Make a blank attrs dict for new attrs
attrs_dict = {}
if hasattr(cls, "__annotations__"):
for k, v in cls.__annotations__.items():
# If the cls has the attribute then a default was set
if hasattr(cls, k):
default = getattr(cls, k)
else:
default = None
attrs_dict.update({k: katra(typed=v, default=default)})
return bases, attrs_dict

for k, v in merged_annotations.items():
# If the cls has the attribute then a default was set
if hasattr(cls, k):
default = getattr(cls, k)
elif k in base_defaults:
default = base_defaults[k]
else:
default = None
attrs_dict.update({k: katra(typed=v, default=default)})
return bases, attrs_dict, merged_annotations

def spock_attr(cls):
"""Map type hints to katras

Connector function that maps type hinting style to the defined katra style which uses the more strict
attr.ib() definition
def _process_class(cls, kw_only: bool, make_init: bool, dynamic: bool):
"""Process a given class
Args:
cls: basic class def
cls: basic class definition
kw_only: set kwarg only
make_init: make an init function
dynamic: allows inherited classes to not be @spock decorated
Returns:
cls: slotted attrs class that is frozen and kw only
cls with attrs dunder methods added
"""
bases, attrs_dict = _base_attr(cls)
# Handles the MRO and gets old annotations
bases, attrs_dict, merged_annotations = _base_attr(cls, kw_only, make_init, dynamic)
# Dynamically make an attr class
obj = attr.make_class(
name=cls.__name__, bases=bases, attrs=attrs_dict, kw_only=True, frozen=True
name=cls.__name__,
bases=bases,
attrs=attrs_dict,
kw_only=kw_only,
frozen=True,
auto_attribs=True,
init=make_init,
)
# For each class we dynamically create we need to register it within the system modules for pickle to work
setattr(sys.modules["spock"].backend.config, obj.__name__, obj)
# Swap the __doc__ string from cls to obj
obj.__doc__ = cls.__doc__
# Set the __init__ function
# Handle __annotations__ from the MRO
obj.__annotations__ = merged_annotations
return obj


def spock_attr(
maybe_cls=None,
kw_only=True,
make_init=True,
dynamic=False,
):
"""Map type hints to katras
Connector function that maps type hinting style to the defined katra style which uses the more strict
attr.ib() definition -- this allows us to attach the correct validators for types before the attrs class is
built
Args:
maybe_cls: maybe a basic class def maybe None depending on call type
kw_only: Make all attributes keyword-only
make_init: bool, define a __init__() method
dynamic: allows inherited classes to not be @spock decorated -- will automatically cast parent classes to a
spock class by traversing the MRO
Returns:
cls: attrs class that is frozen and kw only
"""

def wrap(cls):
return _process_class(
cls, kw_only=kw_only, make_init=make_init, dynamic=dynamic
)

# Note: Taken from dataclass/attr definition(s)
# maybe_cls's type depends on the usage of the decorator. It's a class
# if it's used as `@spock` but ``None`` if used as `@spock()`.
if maybe_cls is None:
return wrap
else:
return wrap(maybe_cls)
Loading

0 comments on commit 1e5fb27

Please sign in to comment.