Merge 0c606c9 into 2efe7d0

.github/workflows/python-pytest-s3.yaml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [3.6, 3.7, 3.8, 3.9, '3.10']
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/python-pytest-tune.yaml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.7, 3.8, 3.9, '3.10']
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/python-pytest.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [3.6, 3.7, 3.8, 3.9, '3.10']
 
     steps:
     - uses: actions/checkout@v2

spock/backend/builder.py

@@ -4,7 +4,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 """Handles the building/saving of the configurations from the Spock config classes"""
-
+import sys
+import typing
 from abc import ABC, abstractmethod
 from enum import EnumMeta
 from typing import List
@@ -19,6 +20,12 @@
 from spock.graph import Graph
 from spock.utils import make_argument
 
+minor = sys.version_info.minor
+if minor < 7:
+    from typing import CallableMeta as _VariadicGenericAlias
+else:
+    from typing import _GenericAlias, _VariadicGenericAlias
+
 
 class BaseBuilder(ABC):  # pylint: disable=too-few-public-methods
     """Base class for building the backend specific builders
@@ -255,10 +262,11 @@ def _make_group_override_parser(parser, class_obj, class_name):
         )
         for val in class_obj.__attrs_attrs__:
             val_type = val.metadata["type"] if "type" in val.metadata else val.type
-            # Check if the val type has __args__ -- this catches lists?
+            # Check if the val type has __args__ -- this catches GenericAlias classes
             # TODO (ncilfone): Fix up this super super ugly logic
             if (
-                hasattr(val_type, "__args__")
+                not isinstance(val_type, _VariadicGenericAlias)
+                and hasattr(val_type, "__args__")
                 and ((list(set(val_type.__args__))[0]).__module__ == class_name)
                 and attr.has((list(set(val_type.__args__))[0]))
             ):

spock/backend/config.py

@@ -117,6 +117,9 @@ def _process_class(cls, kw_only: bool, make_init: bool, dynamic: bool):
         auto_attribs=True,
         init=make_init,
     )
+    # Copy over the post init function
+    if hasattr(cls, "__post_hook__"):
+        obj.__post_hook__ = cls.__post_hook__
     # For each class we dynamically create we need to register it within the system modules for pickle to work
     setattr(sys.modules["spock"].backend.config, obj.__name__, obj)
     # Swap the __doc__ string from cls to obj

spock/backend/field_handlers.py

@@ -5,17 +5,28 @@
 
 """Handles registering field attributes for spock classes -- deals with the recursive nature of dependencies"""
 
+import importlib
+import sys
 from abc import ABC, abstractmethod
 from enum import EnumMeta
 from typing import List, Type
 
 from attr import NOTHING, Attribute
 
-from spock.args import SpockArguments
 from spock.backend.spaces import AttributeSpace, BuilderSpace, ConfigSpace
-from spock.exceptions import _SpockInstantiationError, _SpockNotOptionalError
+from spock.exceptions import (
+    _SpockInstantiationError,
+    _SpockNotOptionalError,
+    _SpockValueError,
+)
 from spock.utils import _check_iterable, _is_spock_instance, _is_spock_tune_instance
 
+minor = sys.version_info.minor
+if minor < 7:
+    from typing import CallableMeta as _VariadicGenericAlias
+else:
+    from typing import _VariadicGenericAlias
+
 
 class RegisterFieldTemplate(ABC):
     """Base class for handing different field types
@@ -318,6 +329,69 @@ def _handle_and_register_enum(
         builder_space.spock_space[enum_cls.__name__] = attr_space.field
 
 
+class RegisterCallableField(RegisterFieldTemplate):
+    """Class that registers callable types
+
+    Attributes:
+        special_keys: dictionary to check special keys
+
+    """
+
+    def __init__(self):
+        """Init call to RegisterSimpleField
+
+        Args:
+        """
+        super(RegisterCallableField, self).__init__()
+
+    def handle_attribute_from_config(
+        self, attr_space: AttributeSpace, builder_space: BuilderSpace
+    ):
+        """Handles setting a simple attribute when it is a spock class type
+
+        Args:
+            attr_space: holds information about a single attribute that is mapped to a ConfigSpace
+            builder_space: named_tuple containing the arguments and spock_space
+
+        Returns:
+        """
+        # These are always going to be strings... cast just in case
+        str_field = str(
+            builder_space.arguments[attr_space.config_space.name][
+                attr_space.attribute.name
+            ]
+        )
+        module, fn = str_field.rsplit(".", 1)
+        try:
+            call_ref = getattr(importlib.import_module(module), fn)
+            attr_space.field = call_ref
+        except Exception as e:
+            raise _SpockValueError(
+                f"Attempted to import module {module} and callable {fn} however it could not be found on the current "
+                f"python path: {e}"
+            )
+
+    def handle_optional_attribute_type(
+        self, attr_space: AttributeSpace, builder_space: BuilderSpace
+    ):
+        """Not implemented for this type
+
+        Args:
+            attr_space: holds information about a single attribute that is mapped to a ConfigSpace
+            builder_space: named_tuple containing the arguments and spock_space
+
+        Raises:
+            _SpockNotOptionalError
+
+        """
+        print("hi")
+        raise _SpockNotOptionalError(
+            f"Parameter `{attr_space.attribute.name}` within `{attr_space.config_space.name}` is of "
+            f"type `{type(attr_space.attribute.type)}` which seems to be unsupported -- "
+            f"are you missing an @spock decorator on a base python class?"
+        )
+
+
 class RegisterSimpleField(RegisterFieldTemplate):
     """Class that registers basic python types
 
@@ -606,6 +680,9 @@ def recurse_generate(cls, spock_cls, builder_space: BuilderSpace):
             # References to tuner classes
             elif _is_spock_tune_instance(attribute.type):
                 handler = RegisterTuneCls()
+            # References to callables
+            elif isinstance(attribute.type, _VariadicGenericAlias):
+                handler = RegisterCallableField()
             # Basic field
             else:
                 handler = RegisterSimpleField()
@@ -617,6 +694,9 @@ def recurse_generate(cls, spock_cls, builder_space: BuilderSpace):
         # error on instantiation
         try:
             spock_instance = spock_cls(**fields)
+            # If there is a __post_hook__ dunder method then call it
+            if hasattr(spock_cls, "__post_hook__"):
+                spock_instance.__post_hook__()
         except Exception as e:
             raise _SpockInstantiationError(
                 f"Spock class `{spock_cls.__name__}` could not be instantiated -- attrs message: {e}"

spock/backend/saver.py

@@ -146,18 +146,31 @@ def _clean_output(self, out_dict):
                 for idx, list_val in enumerate(val):
                     tmp_dict = {}
                     for inner_key, inner_val in list_val.items():
-                        tmp_dict = self._convert(tmp_dict, inner_val, inner_key)
+                        tmp_dict = self._convert_tuples_2_lists(
+                            tmp_dict, inner_val, inner_key
+                        )
                     val[idx] = tmp_dict
                 clean_inner_dict = val
             else:
                 for inner_key, inner_val in val.items():
-                    clean_inner_dict = self._convert(
+                    clean_inner_dict = self._convert_tuples_2_lists(
                         clean_inner_dict, inner_val, inner_key
                     )
             clean_dict.update({key: clean_inner_dict})
         return clean_dict
 
-    def _convert(self, clean_inner_dict, inner_val, inner_key):
+    def _convert_tuples_2_lists(self, clean_inner_dict, inner_val, inner_key):
+        """Convert tuples to lists
+
+        Args:
+            clean_inner_dict: dictionary to update
+            inner_val: current value
+            inner_key: current key
+
+        Returns:
+            updated dictionary where tuples are cast back to lists
+
+        """
         # Convert tuples to lists so they get written correctly
         if isinstance(inner_val, tuple):
             clean_inner_dict.update(
@@ -277,6 +290,10 @@ def _recursively_handle_clean(
                 if repeat_flag:
                     clean_val = list(set(clean_val))[-1]
                 out_dict.update({key: clean_val})
+            # Catch any callables -- convert back to the str representation
+            elif callable(val):
+                call_2_str = f"{val.__module__}.{val.__name__}"
+                out_dict.update({key: call_2_str})
             # If it's a spock class but has a parent then just use the class name to reference the values
             elif (val_name in all_cls) and parent_name is not None:
                 out_dict.update({key: val_name})

spock/backend/typed.py

@@ -12,11 +12,14 @@
 
 import attr
 
+# from spock.exceptions import _SpockValueError
+
 minor = sys.version_info.minor
 if minor < 7:
+    from typing import CallableMeta as _VariadicGenericAlias
     from typing import GenericMeta as _GenericAlias
 else:
-    from typing import _GenericAlias
+    from typing import _GenericAlias, _VariadicGenericAlias
 
 
 class SavePath(str):
@@ -431,6 +434,55 @@ def _handle_optional_typing(typed):
     return typed, optional
 
 
+def _callable_katra(typed, default=None, optional=False):
+    """Private interface to create a Callable katra
+
+    Here we handle the callable type katra that allows us to force a callable check on the value provided
+
+    A 'katra' is the basic functional unit of `spock`. It defines a parameter using attrs as the backend, type checks
+    both simple types and subscripted GenericAlias types (e.g. lists and tuples), handles setting default parameters,
+    and deals with parameter optionality
+
+    Handles: bool, string, float, int, List, and Tuple
+
+    Args:
+        typed: the type of the parameter to define
+        default: the default value to assign if given
+        optional: whether to make the parameter optional or not (thus allowing None)
+
+    Returns:
+        x: Attribute from attrs
+
+    """
+    if default is not None:
+        # if a default is provided, that takes precedence
+        x = attr.ib(
+            validator=attr.validators.is_callable(),
+            default=default,
+            type=typed,
+            metadata={"base": _get_name_py_version(typed)},
+        )
+    elif optional:
+        x = attr.ib(
+            validator=attr.validators.optional(attr.validators.is_callable()),
+            default=default,
+            type=typed,
+            metadata={"optional": True, "base": _get_name_py_version(typed)},
+        )
+    else:
+        x = attr.ib(
+            validator=attr.validators.is_callable(),
+            type=typed,
+            metadata={"base": _get_name_py_version(typed)},
+        )
+
+        # raise _SpockValueError(f"Types of `{_get_name_py_version(typed)}` must have a default value or be optional -- "
+        #                        f"Spock currently has no way to map from a markup style file to a "
+        #                        f"{_get_name_py_version(typed)} type")
+
+    return x
+
+
 def katra(typed, default=None):
     """Public interface to create a katra
 
@@ -449,9 +501,11 @@ def katra(typed, default=None):
     """
     # Handle optionals
     typed, optional = _handle_optional_typing(typed)
+    if isinstance(typed, _VariadicGenericAlias):
+        x = _callable_katra(typed=typed, default=default, optional=optional)
     # We need to check if the type is a _GenericAlias so that we can handle subscripted general types
     # If it is subscript typed it will not be T which python uses as a generic type name
-    if isinstance(typed, _GenericAlias) and (
+    elif isinstance(typed, _GenericAlias) and (
         not isinstance(typed.__args__[0], TypeVar)
     ):
         x = _generic_alias_katra(typed=typed, default=default, optional=optional)

spock/exceptions.py

@@ -1,4 +1,9 @@
 # -*- coding: utf-8 -*-
+
+# Copyright FMR LLC <opensource@fidelity.com>
+# SPDX-License-Identifier: Apache-2.0
+
+
 class _SpockUndecoratedClass(Exception):
     """Custom exception type for non spock decorated classes and not dynamic"""
 
@@ -25,3 +30,11 @@ class _SpockDuplicateArgumentError(Exception):
 
 class _SpockEvolveError(Exception):
     """Custom exception for when evolve errors occur"""
+
+    pass
+
+
+class _SpockValueError(Exception):
+    """Custom exception for throwing value errors"""
+
+    pass