From e29a61878bb610f514db2306aab58f460a661c69 Mon Sep 17 00:00:00 2001 From: Nicholas Cilfone Date: Fri, 11 Mar 2022 12:06:56 -0500 Subject: [PATCH 1/8] Adds support for typing.Callable types. Adds post init hooks to run validation on the instantiated classes. Adds some common validations to utils. Adds unit tests. Updates all unit tests to run on python 3.10 --- .github/workflows/python-pytest-s3.yaml | 2 +- .github/workflows/python-pytest-tune.yaml | 2 +- .github/workflows/python-pytest.yml | 2 +- spock/backend/builder.py | 15 +- spock/backend/config.py | 3 + spock/backend/field_handlers.py | 80 +++++++++- spock/backend/saver.py | 21 ++- spock/backend/typed.py | 58 ++++++- spock/exceptions.py | 12 ++ spock/utils.py | 106 ++++++++++++- tests/base/attr_configs_test.py | 14 +- tests/base/base_asserts_test.py | 8 +- tests/base/test_cmd_line.py | 10 +- tests/base/test_config_arg_builder.py | 20 ++- tests/base/test_post_hooks.py | 150 ++++++++++++++++++ tests/conf/json/test.json | 3 +- tests/conf/toml/test.toml | 3 +- tests/conf/yaml/inherited.yaml | 2 +- tests/conf/yaml/test.yaml | 2 +- tests/conf/yaml/test_fail_callable.yaml | 70 ++++++++ .../conf/yaml/test_fail_callable_module.yaml | 70 ++++++++ 21 files changed, 626 insertions(+), 27 deletions(-) create mode 100644 tests/base/test_post_hooks.py create mode 100644 tests/conf/yaml/test_fail_callable.yaml create mode 100644 tests/conf/yaml/test_fail_callable_module.yaml diff --git a/.github/workflows/python-pytest-s3.yaml b/.github/workflows/python-pytest-s3.yaml index 3cc65eaa..d6c825ec 100644 --- a/.github/workflows/python-pytest-s3.yaml +++ b/.github/workflows/python-pytest-s3.yaml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9, 3.10] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/python-pytest-tune.yaml b/.github/workflows/python-pytest-tune.yaml index 811f799b..62d0f900 100644 --- a/.github/workflows/python-pytest-tune.yaml +++ b/.github/workflows/python-pytest-tune.yaml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: [3.7, 3.8, 3.9, 3.10] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/python-pytest.yml b/.github/workflows/python-pytest.yml index 468f5db8..d67adb16 100644 --- a/.github/workflows/python-pytest.yml +++ b/.github/workflows/python-pytest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9, 3.10] steps: - uses: actions/checkout@v2 diff --git a/spock/backend/builder.py b/spock/backend/builder.py index b06c3bd4..16cbc2f8 100644 --- a/spock/backend/builder.py +++ b/spock/backend/builder.py @@ -4,7 +4,8 @@ # SPDX-License-Identifier: Apache-2.0 """Handles the building/saving of the configurations from the Spock config classes""" - +import sys +import typing from abc import ABC, abstractmethod from enum import EnumMeta from typing import List @@ -20,6 +21,13 @@ from spock.utils import make_argument +minor = sys.version_info.minor +if minor < 7: + from typing import CallableMeta as _VariadicGenericAlias +else: + from typing import _GenericAlias, _VariadicGenericAlias + + class BaseBuilder(ABC): # pylint: disable=too-few-public-methods """Base class for building the backend specific builders @@ -255,10 +263,11 @@ def _make_group_override_parser(parser, class_obj, class_name): ) for val in class_obj.__attrs_attrs__: val_type = val.metadata["type"] if "type" in val.metadata else val.type - # Check if the val type has __args__ -- this catches lists? + # Check if the val type has __args__ -- this catches GenericAlias classes # TODO (ncilfone): Fix up this super super ugly logic if ( - hasattr(val_type, "__args__") + not isinstance(val_type, _VariadicGenericAlias) + and hasattr(val_type, "__args__") and ((list(set(val_type.__args__))[0]).__module__ == class_name) and attr.has((list(set(val_type.__args__))[0])) ): diff --git a/spock/backend/config.py b/spock/backend/config.py index e0af6245..20f3c8d2 100644 --- a/spock/backend/config.py +++ b/spock/backend/config.py @@ -117,6 +117,9 @@ def _process_class(cls, kw_only: bool, make_init: bool, dynamic: bool): auto_attribs=True, init=make_init, ) + # Copy over the post init function + if hasattr(cls, '__post_hook__'): + obj.__post_hook__ = cls.__post_hook__ # For each class we dynamically create we need to register it within the system modules for pickle to work setattr(sys.modules["spock"].backend.config, obj.__name__, obj) # Swap the __doc__ string from cls to obj diff --git a/spock/backend/field_handlers.py b/spock/backend/field_handlers.py index 48b4ff9d..42c25281 100644 --- a/spock/backend/field_handlers.py +++ b/spock/backend/field_handlers.py @@ -5,18 +5,27 @@ """Handles registering field attributes for spock classes -- deals with the recursive nature of dependencies""" +import sys from abc import ABC, abstractmethod from enum import EnumMeta from typing import List, Type from attr import NOTHING, Attribute -from spock.args import SpockArguments +import importlib + from spock.backend.spaces import AttributeSpace, BuilderSpace, ConfigSpace -from spock.exceptions import _SpockInstantiationError, _SpockNotOptionalError +from spock.exceptions import _SpockInstantiationError, _SpockNotOptionalError, _SpockValueError from spock.utils import _check_iterable, _is_spock_instance, _is_spock_tune_instance +minor = sys.version_info.minor +if minor < 7: + from typing import CallableMeta as _VariadicGenericAlias +else: + from typing import _VariadicGenericAlias + + class RegisterFieldTemplate(ABC): """Base class for handing different field types @@ -318,6 +327,67 @@ def _handle_and_register_enum( builder_space.spock_space[enum_cls.__name__] = attr_space.field +class RegisterCallableField(RegisterFieldTemplate): + """Class that registers callable types + + Attributes: + special_keys: dictionary to check special keys + + """ + + def __init__(self): + """Init call to RegisterSimpleField + + Args: + """ + super(RegisterCallableField, self).__init__() + + def handle_attribute_from_config( + self, attr_space: AttributeSpace, builder_space: BuilderSpace + ): + """Handles setting a simple attribute when it is a spock class type + + Args: + attr_space: holds information about a single attribute that is mapped to a ConfigSpace + builder_space: named_tuple containing the arguments and spock_space + + Returns: + """ + # These are always going to be strings... cast just in case + str_field = str(builder_space.arguments[attr_space.config_space.name][ + attr_space.attribute.name + ]) + module, fn = str_field.rsplit('.', 1) + try: + call_ref = getattr(importlib.import_module(module), fn) + attr_space.field = call_ref + except Exception as e: + raise _SpockValueError( + f"Attempted to import module {module} and callable {fn} however it could not be found on the current " + f"python path: {e}" + ) + + def handle_optional_attribute_type( + self, attr_space: AttributeSpace, builder_space: BuilderSpace + ): + """Not implemented for this type + + Args: + attr_space: holds information about a single attribute that is mapped to a ConfigSpace + builder_space: named_tuple containing the arguments and spock_space + + Raises: + _SpockNotOptionalError + + """ + print("hi") + raise _SpockNotOptionalError( + f"Parameter `{attr_space.attribute.name}` within `{attr_space.config_space.name}` is of " + f"type `{type(attr_space.attribute.type)}` which seems to be unsupported -- " + f"are you missing an @spock decorator on a base python class?" + ) + + class RegisterSimpleField(RegisterFieldTemplate): """Class that registers basic python types @@ -606,6 +676,9 @@ def recurse_generate(cls, spock_cls, builder_space: BuilderSpace): # References to tuner classes elif _is_spock_tune_instance(attribute.type): handler = RegisterTuneCls() + # References to callables + elif isinstance(attribute.type, _VariadicGenericAlias): + handler = RegisterCallableField() # Basic field else: handler = RegisterSimpleField() @@ -617,6 +690,9 @@ def recurse_generate(cls, spock_cls, builder_space: BuilderSpace): # error on instantiation try: spock_instance = spock_cls(**fields) + # If there is a __post_hook__ dunder method then call it + if hasattr(spock_cls, "__post_hook__"): + spock_instance.__post_hook__() except Exception as e: raise _SpockInstantiationError( f"Spock class `{spock_cls.__name__}` could not be instantiated -- attrs message: {e}" diff --git a/spock/backend/saver.py b/spock/backend/saver.py index 89b0b2b5..439fbcc6 100644 --- a/spock/backend/saver.py +++ b/spock/backend/saver.py @@ -146,18 +146,29 @@ def _clean_output(self, out_dict): for idx, list_val in enumerate(val): tmp_dict = {} for inner_key, inner_val in list_val.items(): - tmp_dict = self._convert(tmp_dict, inner_val, inner_key) + tmp_dict = self._convert_tuples_2_lists(tmp_dict, inner_val, inner_key) val[idx] = tmp_dict clean_inner_dict = val else: for inner_key, inner_val in val.items(): - clean_inner_dict = self._convert( + clean_inner_dict = self._convert_tuples_2_lists( clean_inner_dict, inner_val, inner_key ) clean_dict.update({key: clean_inner_dict}) return clean_dict - def _convert(self, clean_inner_dict, inner_val, inner_key): + def _convert_tuples_2_lists(self, clean_inner_dict, inner_val, inner_key): + """Convert tuples to lists + + Args: + clean_inner_dict: dictionary to update + inner_val: current value + inner_key: current key + + Returns: + updated dictionary where tuples are cast back to lists + + """ # Convert tuples to lists so they get written correctly if isinstance(inner_val, tuple): clean_inner_dict.update( @@ -277,6 +288,10 @@ def _recursively_handle_clean( if repeat_flag: clean_val = list(set(clean_val))[-1] out_dict.update({key: clean_val}) + # Catch any callables -- convert back to the str representation + elif callable(val): + call_2_str = f"{val.__module__}.{val.__name__}" + out_dict.update({key: call_2_str}) # If it's a spock class but has a parent then just use the class name to reference the values elif (val_name in all_cls) and parent_name is not None: out_dict.update({key: val_name}) diff --git a/spock/backend/typed.py b/spock/backend/typed.py index 23cea1e8..c2d5a3b1 100644 --- a/spock/backend/typed.py +++ b/spock/backend/typed.py @@ -12,11 +12,14 @@ import attr +# from spock.exceptions import _SpockValueError + minor = sys.version_info.minor if minor < 7: from typing import GenericMeta as _GenericAlias + from typing import CallableMeta as _VariadicGenericAlias else: - from typing import _GenericAlias + from typing import _GenericAlias, _VariadicGenericAlias class SavePath(str): @@ -431,6 +434,55 @@ def _handle_optional_typing(typed): return typed, optional +def _callable_katra(typed, default=None, optional=False): + """Private interface to create a Callable katra + + Here we handle the callable type katra that allows us to force a callable check on the value provided + + A 'katra' is the basic functional unit of `spock`. It defines a parameter using attrs as the backend, type checks + both simple types and subscripted GenericAlias types (e.g. lists and tuples), handles setting default parameters, + and deals with parameter optionality + + Handles: bool, string, float, int, List, and Tuple + + Args: + typed: the type of the parameter to define + default: the default value to assign if given + optional: whether to make the parameter optional or not (thus allowing None) + + Returns: + x: Attribute from attrs + + """ + if default is not None: + # if a default is provided, that takes precedence + x = attr.ib( + validator=attr.validators.is_callable(), + default=default, + type=typed, + metadata={"base": _get_name_py_version(typed)}, + ) + elif optional: + x = attr.ib( + validator=attr.validators.optional(attr.validators.is_callable()), + default=default, + type=typed, + metadata={"optional": True, "base": _get_name_py_version(typed)}, + ) + else: + x = attr.ib( + validator=attr.validators.is_callable(), + type=typed, + metadata={"base": _get_name_py_version(typed)}, + ) + + # raise _SpockValueError(f"Types of `{_get_name_py_version(typed)}` must have a default value or be optional -- " + # f"Spock currently has no way to map from a markup style file to a " + # f"{_get_name_py_version(typed)} type") + + return x + + def katra(typed, default=None): """Public interface to create a katra @@ -449,9 +501,11 @@ def katra(typed, default=None): """ # Handle optionals typed, optional = _handle_optional_typing(typed) + if isinstance(typed, _VariadicGenericAlias): + x = _callable_katra(typed=typed, default=default, optional=optional) # We need to check if the type is a _GenericAlias so that we can handle subscripted general types # If it is subscript typed it will not be T which python uses as a generic type name - if isinstance(typed, _GenericAlias) and ( + elif isinstance(typed, _GenericAlias) and ( not isinstance(typed.__args__[0], TypeVar) ): x = _generic_alias_katra(typed=typed, default=default, optional=optional) diff --git a/spock/exceptions.py b/spock/exceptions.py index 748c39d0..920edb5d 100644 --- a/spock/exceptions.py +++ b/spock/exceptions.py @@ -1,4 +1,8 @@ # -*- coding: utf-8 -*- + +# Copyright FMR LLC +# SPDX-License-Identifier: Apache-2.0 + class _SpockUndecoratedClass(Exception): """Custom exception type for non spock decorated classes and not dynamic""" @@ -25,3 +29,11 @@ class _SpockDuplicateArgumentError(Exception): class _SpockEvolveError(Exception): """Custom exception for when evolve errors occur""" + + pass + + +class _SpockValueError(Exception): + """Custom exception for throwing value errors""" + + pass diff --git a/spock/utils.py b/spock/utils.py index 35efadd9..23f322f9 100644 --- a/spock/utils.py +++ b/spock/utils.py @@ -7,14 +7,18 @@ import ast import os -import re import socket import subprocess import sys -from enum import EnumMeta from time import localtime, strftime from warnings import warn +from spock.exceptions import _SpockValueError + +from enum import EnumMeta +from pathlib import Path +from typing import List, Type, Union + import attr import git @@ -24,9 +28,101 @@ else: from typing import _GenericAlias -from enum import EnumMeta -from pathlib import Path -from typing import List, Type, Union + +def within(val: Union[float, int], low_bound: Union[float, int], upper_bound: Union[float, int], inclusive_lower: bool = False, inclusive_upper: bool = False) -> None: + """Checks that a value is within a defined range + + Args: + val: value to check against + low_bound: lower bound of range + upper_bound: upper bound of range + inclusive_lower: if the check includes the bound value (i.e. >=) + inclusive_upper: if the check includes the bound value (i.e. <=) + + Returns: + None + + Raises: + _SpockValueError + + """ + # Check lower bounds + lower_fn = le if inclusive_upper else lt + upper_fn = ge if inclusive_lower else gt + upper_fn(val=val, bound=low_bound) + lower_fn(val=val, bound=upper_bound) + + +def ge(val: Union[float, int], bound: Union[float, int]) -> None: + """Checks that a value is greater than or equal to (inclusive) a lower bound + + Args: + val: value to check against + bound: lower bound + + Returns: + None + + Raises: + _SpockValueError + + """ + if val < bound: + raise _SpockValueError(f"Set value `{val}` is not >= given bound value `{bound}`") + + +def gt(val: Union[float, int], bound: Union[float, int]) -> None: + """Checks that a value is greater (non-inclusive) than a lower bound + + Args: + val: value to check against + bound: lower bound + + Returns: + None + + Raises: + _SpockValueError + + """ + if val <= bound: + raise _SpockValueError(f"Set value `{val}` is not > given bound value `{bound}`") + + +def le(val: Union[float, int], bound: Union[float, int], ) -> None: + """Checks that a value is less than or equal to (inclusive) an upper bound + + Args: + val: value to check against + bound: upper bound + + Returns: + None + + Raises: + _SpockValueError + + """ + if val > bound: + raise _SpockValueError(f"Set value `{val}` is not <= given bound value `{bound}`") + + +def lt(val: Union[float, int], bound: Union[float, int]) -> None: + """Checks that a value is less (non-inclusive) than an upper bound + + Args: + val: value to check against + bound: upper bound + + Returns: + None + + Raises: + _SpockValueError + + """ + if val >= bound: + raise _SpockValueError(f"Set value `{val}` is not < given bound value `{bound}`") def _find_all_spock_classes(attr_class: Type): diff --git a/tests/base/attr_configs_test.py b/tests/base/attr_configs_test.py index 54b697e6..70d035fd 100644 --- a/tests/base/attr_configs_test.py +++ b/tests/base/attr_configs_test.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: Apache-2.0 from enum import Enum -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, Callable from spock.backend.typed import SavePath from spock.config import spock @@ -156,6 +156,8 @@ class TypeConfig: class_enum: ClassChoice # Double Nested class ref high_config: SingleNestedConfig + # Callable + call_me: Callable @spock @@ -199,6 +201,8 @@ class TypeOptConfig: class_enum_opt_no_def: Optional[ClassChoice] # Additional dummy argument int_p_2: Optional[int] + # Optional Callable + call_me_maybe: Optional[Callable] @spock @@ -207,6 +211,10 @@ class NestedStuffDefault: goals: int = 0 +def foo(val: int): + return val * 2 + + @spock class TypeDefaultConfig: """This creates a test Spock config of all supported variable types as required parameters and falls back @@ -257,6 +265,8 @@ class TypeDefaultConfig: class_enum_def: ClassChoice = NestedStuff # Double Nested class ref high_config_def: SingleNestedConfig = SingleNestedConfig + # Optional Callable + call_me_maybe: Callable = foo @spock @@ -301,6 +311,8 @@ class TypeDefaultOptConfig: nested_list_opt_def: Optional[List[NestedListStuff]] = [NestedListStuff] # Class Enum class_enum_opt_def: Optional[ClassChoice] = NestedStuff + # Optional Callable + call_me_maybe: Optional[Callable] = foo @spock diff --git a/tests/base/base_asserts_test.py b/tests/base/base_asserts_test.py index f21dc6e1..0f2398d7 100644 --- a/tests/base/base_asserts_test.py +++ b/tests/base/base_asserts_test.py @@ -2,7 +2,7 @@ # Copyright FMR LLC # SPDX-License-Identifier: Apache-2.0 -from tests.base.attr_configs_test import FirstDoubleNestedConfig +from tests.base.attr_configs_test import FirstDoubleNestedConfig, foo class AllTypes: @@ -51,6 +51,7 @@ def test_all_set(self, arg_builder): ) assert arg_builder.TypeConfig.high_config.double_nested_config.h_factor == 0.99 assert arg_builder.TypeConfig.high_config.double_nested_config.v_factor == 0.90 + assert arg_builder.TypeConfig.call_me == foo # Optional # assert arg_builder.TypeOptConfig.int_p_opt_no_def is None @@ -70,6 +71,7 @@ def test_all_set(self, arg_builder): assert arg_builder.TypeOptConfig.nested_opt_no_def is None assert arg_builder.TypeOptConfig.nested_list_opt_no_def is None assert arg_builder.TypeOptConfig.class_enum_opt_no_def is None + assert arg_builder.TypeOptConfig.call_me_maybe is None class AllDefaults: @@ -122,6 +124,7 @@ def test_all_defaults(self, arg_builder): arg_builder.TypeDefaultConfig.high_config_def.double_nested_config.v_factor == 0.90 ) + assert arg_builder.TypeDefaultConfig.call_me_maybe == foo # Optional w/ Defaults # assert arg_builder.TypeDefaultOptConfig.int_p_opt_def == 10 @@ -157,6 +160,7 @@ def test_all_defaults(self, arg_builder): assert arg_builder.TypeDefaultOptConfig.nested_list_opt_def[1].two == "bye" assert arg_builder.TypeDefaultOptConfig.class_enum_opt_def.one == 11 assert arg_builder.TypeDefaultOptConfig.class_enum_opt_def.two == "ciao" + assert arg_builder.TypeDefaultOptConfig.call_me_maybe == foo class AllInherited: @@ -207,6 +211,7 @@ def test_all_inherited(self, arg_builder): assert ( arg_builder.TypeInherited.high_config.double_nested_config.v_factor == 0.90 ) + assert arg_builder.TypeInherited.call_me == foo # Optional w/ Defaults # assert arg_builder.TypeInherited.int_p_opt_def == 10 @@ -220,6 +225,7 @@ def test_all_inherited(self, arg_builder): assert arg_builder.TypeInherited.tuple_p_opt_def_int == (10, 20) assert arg_builder.TypeInherited.tuple_p_opt_def_str == ("Spock", "Package") assert arg_builder.TypeInherited.tuple_p_opt_def_bool == (True, False) + assert arg_builder.TypeInherited.call_me_maybe == foo class AllDynamic: diff --git a/tests/base/test_cmd_line.py b/tests/base/test_cmd_line.py index 4464aef0..7d76b028 100644 --- a/tests/base/test_cmd_line.py +++ b/tests/base/test_cmd_line.py @@ -77,7 +77,9 @@ def arg_builder(monkeypatch): "--SingleNestedConfig.double_nested_config", "SecondDoubleNestedConfig", "--SecondDoubleNestedConfig.morph_tolerance", - "0.2" + "0.2", + "--TypeConfig.call_me", + "'tests.base.attr_configs_test.foo'" ], ) config = ConfigArgBuilder( @@ -119,6 +121,7 @@ def test_class_overrides(self, arg_builder): assert arg_builder.NestedListStuff[1].two == "Working" assert isinstance(arg_builder.SingleNestedConfig.double_nested_config, SecondDoubleNestedConfig) is True assert arg_builder.SecondDoubleNestedConfig.morph_tolerance == 0.2 + assert arg_builder.TypeConfig.call_me == foo class TestClassOnlyCmdLine: @@ -187,7 +190,9 @@ def arg_builder(monkeypatch): "--TypeConfig.nested_list.NestedListStuff.two", "['Hooray', 'Working']", "--TypeConfig.high_config", - "SingleNestedConfig" + "SingleNestedConfig", + "--TypeConfig.call_me", + "'tests.base.attr_configs_test.foo'" ], ) config = ConfigArgBuilder( @@ -231,6 +236,7 @@ def test_class_overrides(self, arg_builder): assert arg_builder.NestedListStuff[0].two == "Hooray" assert arg_builder.NestedListStuff[1].one == 21 assert arg_builder.NestedListStuff[1].two == "Working" + assert arg_builder.TypeConfig.call_me == foo class TestRaiseCmdLineNoKey: diff --git a/tests/base/test_config_arg_builder.py b/tests/base/test_config_arg_builder.py index 57c19eca..fcd2dcf0 100644 --- a/tests/base/test_config_arg_builder.py +++ b/tests/base/test_config_arg_builder.py @@ -4,7 +4,7 @@ import pytest from spock.builder import ConfigArgBuilder -from spock.exceptions import _SpockUndecoratedClass, _SpockNotOptionalError +from spock.exceptions import _SpockUndecoratedClass, _SpockNotOptionalError, _SpockValueError from tests.base.attr_configs_test import * from tests.base.base_asserts_test import * @@ -266,3 +266,21 @@ class TestConfigDefaultsFail(Foo, Bar): config = ConfigArgBuilder(TestConfigDefaultsFail) return config.generate() + + +class TestCallableModulePathRaise: + def test_callable_module(self, monkeypatch): + with monkeypatch.context() as m: + m.setattr(sys, "argv", ["", "--config", "./tests/conf/yaml/test_fail_callable_module.yaml"]) + with pytest.raises(_SpockValueError): + config = ConfigArgBuilder(*all_configs) + return config.generate() + + +class TestCallableModuleCallableRaise: + def test_callable_module(self, monkeypatch): + with monkeypatch.context() as m: + m.setattr(sys, "argv", ["", "--config", "./tests/conf/yaml/test_fail_callable.yaml"]) + with pytest.raises(_SpockValueError): + config = ConfigArgBuilder(*all_configs) + return config.generate() \ No newline at end of file diff --git a/tests/base/test_post_hooks.py b/tests/base/test_post_hooks.py new file mode 100644 index 00000000..89a617fc --- /dev/null +++ b/tests/base/test_post_hooks.py @@ -0,0 +1,150 @@ +# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- +import sys + +import pytest + +from spock import spock +from spock import SpockBuilder +from spock.utils import within, gt, ge, lt, le +from spock.exceptions import _SpockInstantiationError + + +@spock +class WithinLowFailConfig: + other: float = 0.9 + + def __post_hook__(self): + within(self.other, 0.9, 1.1, inclusive_lower=False, inclusive_upper=False) + + +@spock +class WithinHighFailConfig: + other: float = 1.1 + + def __post_hook__(self): + within(self.other, 0.9, 1.1, inclusive_lower=False, inclusive_upper=False) + + +@spock +class GTFailConfig: + other: float = 1.0 + + def __post_hook__(self): + gt(self.other, bound=1.1) + + +@spock +class GEFailConfig: + other: float = 0.9 + + def __post_hook__(self): + ge(self.other, bound=1.0) + + +@spock +class LTFailConfig: + other: float = 1.0 + + def __post_hook__(self): + lt(self.other, bound=0.9) + + +@spock +class LEFailConfig: + other: float = 0.9 + + def __post_hook__(self): + le(self.other, bound=0.8) + + +class TestPostHooks: + def test_within_low(self, monkeypatch, tmp_path): + """Test serialization/de-serialization""" + with monkeypatch.context() as m: + m.setattr( + sys, + "argv", + [""], + ) + with pytest.raises(_SpockInstantiationError): + config = SpockBuilder( + WithinLowFailConfig, + desc="Test Builder", + ) + config.generate() + + def test_within_high(self, monkeypatch, tmp_path): + """Test serialization/de-serialization""" + with monkeypatch.context() as m: + m.setattr( + sys, + "argv", + [""], + ) + with pytest.raises(_SpockInstantiationError): + config = SpockBuilder( + WithinHighFailConfig, + desc="Test Builder", + ) + config.generate() + + def test_gt(self, monkeypatch, tmp_path): + """Test serialization/de-serialization""" + with monkeypatch.context() as m: + m.setattr( + sys, + "argv", + [""], + ) + with pytest.raises(_SpockInstantiationError): + config = SpockBuilder( + GTFailConfig, + desc="Test Builder", + ) + config.generate() + + def test_ge(self, monkeypatch, tmp_path): + """Test serialization/de-serialization""" + with monkeypatch.context() as m: + m.setattr( + sys, + "argv", + [""], + ) + with pytest.raises(_SpockInstantiationError): + config = SpockBuilder( + GEFailConfig, + desc="Test Builder", + ) + config.generate() + + def test_lt(self, monkeypatch, tmp_path): + """Test serialization/de-serialization""" + with monkeypatch.context() as m: + m.setattr( + sys, + "argv", + [""], + ) + with pytest.raises(_SpockInstantiationError): + config = SpockBuilder( + LTFailConfig, + desc="Test Builder", + ) + config.generate() + + def test_le(self, monkeypatch, tmp_path): + """Test serialization/de-serialization""" + with monkeypatch.context() as m: + m.setattr( + sys, + "argv", + [""], + ) + with pytest.raises(_SpockInstantiationError): + config = SpockBuilder( + LEFailConfig, + desc="Test Builder", + ) + config.generate() \ No newline at end of file diff --git a/tests/conf/json/test.json b/tests/conf/json/test.json index ddc76c3b..1ba3a8f3 100644 --- a/tests/conf/json/test.json +++ b/tests/conf/json/test.json @@ -46,5 +46,6 @@ }, "TypeConfig": { "float_p": 12.0 - } + }, + "call_me": "tests.base.attr_configs_test.foo" } \ No newline at end of file diff --git a/tests/conf/toml/test.toml b/tests/conf/toml/test.toml index 859c1738..99eaea9f 100644 --- a/tests/conf/toml/test.toml +++ b/tests/conf/toml/test.toml @@ -57,4 +57,5 @@ high_config = 'SingleNestedConfig' v_factor = 0.90 # Overrride general definition [TypeConfig] - float_p = 12.0 \ No newline at end of file + float_p = 12.0 +call_me = 'tests.base.attr_configs_test.foo' \ No newline at end of file diff --git a/tests/conf/yaml/inherited.yaml b/tests/conf/yaml/inherited.yaml index 1931ec26..204075ec 100644 --- a/tests/conf/yaml/inherited.yaml +++ b/tests/conf/yaml/inherited.yaml @@ -63,4 +63,4 @@ SingleNestedConfig: FirstDoubleNestedConfig: h_factor: 0.99 v_factor: 0.90 - +call_me: "tests.base.attr_configs_test.foo" diff --git a/tests/conf/yaml/test.yaml b/tests/conf/yaml/test.yaml index 1fc55154..e1c19f4a 100644 --- a/tests/conf/yaml/test.yaml +++ b/tests/conf/yaml/test.yaml @@ -63,7 +63,7 @@ SingleNestedConfig: FirstDoubleNestedConfig: h_factor: 0.99 v_factor: 0.90 - +call_me: "tests.base.attr_configs_test.foo" # Override general definition TypeConfig: diff --git a/tests/conf/yaml/test_fail_callable.yaml b/tests/conf/yaml/test_fail_callable.yaml new file mode 100644 index 00000000..55066236 --- /dev/null +++ b/tests/conf/yaml/test_fail_callable.yaml @@ -0,0 +1,70 @@ +# conf file for all YAML tests +### Required or Boolean Base Types ### +# Boolean - Set +bool_p_set: true +# Required Int +int_p: 10 +# Required Float +float_p: 1e1 +# Required String +string_p: Spock +# Required List -- Float +list_p_float: [10.0, 20.0] +# Required List -- Int +list_p_int: [10, 20] +# Required List of Lists +list_list_p_int: [[10, 20], [10, 20]] +# Required List -- Str +list_p_str: [Spock, Package] +# Required List -- Bool +list_p_bool: [True, False] +# Required Tuple -- Float +tuple_p_float: [10.0, 20.0] +# Required Tuple -- Int +tuple_p_int: [10, 20] +# Required Tuple -- Str +tuple_p_str: [Spock, Package] +# Required Tuple -- Bool +tuple_p_bool: [True, False] +# Required Tuple -- mixed +tuple_p_mixed: [5, 11.5] +# Required Choice -- Str +choice_p_str: option_1 +# Required Choice -- Int +choice_p_int: 10 +# Required Choice -- Str +choice_p_float: 10.0 +# Required List of Choice -- Str +list_choice_p_str: [option_1] +# Required List of List of Choice -- Str +list_list_choice_p_str: [[option_1], [option_1]] +# Required List of Choice -- Int +list_choice_p_int: [10] +# Required List of Choice -- Float +list_choice_p_float: [10.0] +# Nested Configuration +nested: NestedStuff +NestedStuff: + one: 11 + two: ciao +# Nested List configuration +nested_list: NestedListStuff +NestedListStuff: + - one: 10 + two: hello + - one: 20 + two: bye +# Class Enum +class_enum: NestedStuff +# Nested classes +high_config: SingleNestedConfig +SingleNestedConfig: + double_nested_config: FirstDoubleNestedConfig +FirstDoubleNestedConfig: + h_factor: 0.99 + v_factor: 0.90 +call_me: "tests.base.attr_configs_test.fool" + +# Override general definition +TypeConfig: + float_p: 12.0 diff --git a/tests/conf/yaml/test_fail_callable_module.yaml b/tests/conf/yaml/test_fail_callable_module.yaml new file mode 100644 index 00000000..7de84fa2 --- /dev/null +++ b/tests/conf/yaml/test_fail_callable_module.yaml @@ -0,0 +1,70 @@ +# conf file for all YAML tests +### Required or Boolean Base Types ### +# Boolean - Set +bool_p_set: true +# Required Int +int_p: 10 +# Required Float +float_p: 1e1 +# Required String +string_p: Spock +# Required List -- Float +list_p_float: [10.0, 20.0] +# Required List -- Int +list_p_int: [10, 20] +# Required List of Lists +list_list_p_int: [[10, 20], [10, 20]] +# Required List -- Str +list_p_str: [Spock, Package] +# Required List -- Bool +list_p_bool: [True, False] +# Required Tuple -- Float +tuple_p_float: [10.0, 20.0] +# Required Tuple -- Int +tuple_p_int: [10, 20] +# Required Tuple -- Str +tuple_p_str: [Spock, Package] +# Required Tuple -- Bool +tuple_p_bool: [True, False] +# Required Tuple -- mixed +tuple_p_mixed: [5, 11.5] +# Required Choice -- Str +choice_p_str: option_1 +# Required Choice -- Int +choice_p_int: 10 +# Required Choice -- Str +choice_p_float: 10.0 +# Required List of Choice -- Str +list_choice_p_str: [option_1] +# Required List of List of Choice -- Str +list_list_choice_p_str: [[option_1], [option_1]] +# Required List of Choice -- Int +list_choice_p_int: [10] +# Required List of Choice -- Float +list_choice_p_float: [10.0] +# Nested Configuration +nested: NestedStuff +NestedStuff: + one: 11 + two: ciao +# Nested List configuration +nested_list: NestedListStuff +NestedListStuff: + - one: 10 + two: hello + - one: 20 + two: bye +# Class Enum +class_enum: NestedStuff +# Nested classes +high_config: SingleNestedConfig +SingleNestedConfig: + double_nested_config: FirstDoubleNestedConfig +FirstDoubleNestedConfig: + h_factor: 0.99 + v_factor: 0.90 +call_me: "tests.base.attr_configs_tests.foo" + +# Override general definition +TypeConfig: + float_p: 12.0 From 7d63093099c8c245d37ee0228f52f1ff87c8347b Mon Sep 17 00:00:00 2001 From: Nicholas Cilfone Date: Fri, 11 Mar 2022 12:08:33 -0500 Subject: [PATCH 2/8] linted --- spock/backend/builder.py | 1 - spock/backend/config.py | 2 +- spock/backend/field_handlers.py | 20 ++++++++++------- spock/backend/saver.py | 4 +++- spock/backend/typed.py | 2 +- spock/exceptions.py | 1 + spock/utils.py | 38 +++++++++++++++++++++++---------- 7 files changed, 45 insertions(+), 23 deletions(-) diff --git a/spock/backend/builder.py b/spock/backend/builder.py index 16cbc2f8..11bd9d67 100644 --- a/spock/backend/builder.py +++ b/spock/backend/builder.py @@ -20,7 +20,6 @@ from spock.graph import Graph from spock.utils import make_argument - minor = sys.version_info.minor if minor < 7: from typing import CallableMeta as _VariadicGenericAlias diff --git a/spock/backend/config.py b/spock/backend/config.py index 20f3c8d2..33ee29fc 100644 --- a/spock/backend/config.py +++ b/spock/backend/config.py @@ -118,7 +118,7 @@ def _process_class(cls, kw_only: bool, make_init: bool, dynamic: bool): init=make_init, ) # Copy over the post init function - if hasattr(cls, '__post_hook__'): + if hasattr(cls, "__post_hook__"): obj.__post_hook__ = cls.__post_hook__ # For each class we dynamically create we need to register it within the system modules for pickle to work setattr(sys.modules["spock"].backend.config, obj.__name__, obj) diff --git a/spock/backend/field_handlers.py b/spock/backend/field_handlers.py index 42c25281..805eb3d7 100644 --- a/spock/backend/field_handlers.py +++ b/spock/backend/field_handlers.py @@ -5,6 +5,7 @@ """Handles registering field attributes for spock classes -- deals with the recursive nature of dependencies""" +import importlib import sys from abc import ABC, abstractmethod from enum import EnumMeta @@ -12,13 +13,14 @@ from attr import NOTHING, Attribute -import importlib - from spock.backend.spaces import AttributeSpace, BuilderSpace, ConfigSpace -from spock.exceptions import _SpockInstantiationError, _SpockNotOptionalError, _SpockValueError +from spock.exceptions import ( + _SpockInstantiationError, + _SpockNotOptionalError, + _SpockValueError, +) from spock.utils import _check_iterable, _is_spock_instance, _is_spock_tune_instance - minor = sys.version_info.minor if minor < 7: from typing import CallableMeta as _VariadicGenericAlias @@ -354,10 +356,12 @@ def handle_attribute_from_config( Returns: """ # These are always going to be strings... cast just in case - str_field = str(builder_space.arguments[attr_space.config_space.name][ - attr_space.attribute.name - ]) - module, fn = str_field.rsplit('.', 1) + str_field = str( + builder_space.arguments[attr_space.config_space.name][ + attr_space.attribute.name + ] + ) + module, fn = str_field.rsplit(".", 1) try: call_ref = getattr(importlib.import_module(module), fn) attr_space.field = call_ref diff --git a/spock/backend/saver.py b/spock/backend/saver.py index 439fbcc6..9f72dde0 100644 --- a/spock/backend/saver.py +++ b/spock/backend/saver.py @@ -146,7 +146,9 @@ def _clean_output(self, out_dict): for idx, list_val in enumerate(val): tmp_dict = {} for inner_key, inner_val in list_val.items(): - tmp_dict = self._convert_tuples_2_lists(tmp_dict, inner_val, inner_key) + tmp_dict = self._convert_tuples_2_lists( + tmp_dict, inner_val, inner_key + ) val[idx] = tmp_dict clean_inner_dict = val else: diff --git a/spock/backend/typed.py b/spock/backend/typed.py index c2d5a3b1..fa0ded7f 100644 --- a/spock/backend/typed.py +++ b/spock/backend/typed.py @@ -16,8 +16,8 @@ minor = sys.version_info.minor if minor < 7: - from typing import GenericMeta as _GenericAlias from typing import CallableMeta as _VariadicGenericAlias + from typing import GenericMeta as _GenericAlias else: from typing import _GenericAlias, _VariadicGenericAlias diff --git a/spock/exceptions.py b/spock/exceptions.py index 920edb5d..e060490b 100644 --- a/spock/exceptions.py +++ b/spock/exceptions.py @@ -3,6 +3,7 @@ # Copyright FMR LLC # SPDX-License-Identifier: Apache-2.0 + class _SpockUndecoratedClass(Exception): """Custom exception type for non spock decorated classes and not dynamic""" diff --git a/spock/utils.py b/spock/utils.py index 23f322f9..45cfc15f 100644 --- a/spock/utils.py +++ b/spock/utils.py @@ -10,18 +10,17 @@ import socket import subprocess import sys -from time import localtime, strftime -from warnings import warn - -from spock.exceptions import _SpockValueError - from enum import EnumMeta from pathlib import Path +from time import localtime, strftime from typing import List, Type, Union +from warnings import warn import attr import git +from spock.exceptions import _SpockValueError + minor = sys.version_info.minor if minor < 7: from typing import GenericMeta as _GenericAlias @@ -29,7 +28,13 @@ from typing import _GenericAlias -def within(val: Union[float, int], low_bound: Union[float, int], upper_bound: Union[float, int], inclusive_lower: bool = False, inclusive_upper: bool = False) -> None: +def within( + val: Union[float, int], + low_bound: Union[float, int], + upper_bound: Union[float, int], + inclusive_lower: bool = False, + inclusive_upper: bool = False, +) -> None: """Checks that a value is within a defined range Args: @@ -68,7 +73,9 @@ def ge(val: Union[float, int], bound: Union[float, int]) -> None: """ if val < bound: - raise _SpockValueError(f"Set value `{val}` is not >= given bound value `{bound}`") + raise _SpockValueError( + f"Set value `{val}` is not >= given bound value `{bound}`" + ) def gt(val: Union[float, int], bound: Union[float, int]) -> None: @@ -86,10 +93,15 @@ def gt(val: Union[float, int], bound: Union[float, int]) -> None: """ if val <= bound: - raise _SpockValueError(f"Set value `{val}` is not > given bound value `{bound}`") + raise _SpockValueError( + f"Set value `{val}` is not > given bound value `{bound}`" + ) -def le(val: Union[float, int], bound: Union[float, int], ) -> None: +def le( + val: Union[float, int], + bound: Union[float, int], +) -> None: """Checks that a value is less than or equal to (inclusive) an upper bound Args: @@ -104,7 +116,9 @@ def le(val: Union[float, int], bound: Union[float, int], ) -> None: """ if val > bound: - raise _SpockValueError(f"Set value `{val}` is not <= given bound value `{bound}`") + raise _SpockValueError( + f"Set value `{val}` is not <= given bound value `{bound}`" + ) def lt(val: Union[float, int], bound: Union[float, int]) -> None: @@ -122,7 +136,9 @@ def lt(val: Union[float, int], bound: Union[float, int]) -> None: """ if val >= bound: - raise _SpockValueError(f"Set value `{val}` is not < given bound value `{bound}`") + raise _SpockValueError( + f"Set value `{val}` is not < given bound value `{bound}`" + ) def _find_all_spock_classes(attr_class: Type): From 0c606c9d417549246ec2726299aeb4a592dd1747 Mon Sep 17 00:00:00 2001 From: Nicholas Cilfone Date: Fri, 11 Mar 2022 12:12:55 -0500 Subject: [PATCH 3/8] python version in quotes to fix wrong version reference --- .github/workflows/python-pytest-s3.yaml | 2 +- .github/workflows/python-pytest-tune.yaml | 2 +- .github/workflows/python-pytest.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-pytest-s3.yaml b/.github/workflows/python-pytest-s3.yaml index d6c825ec..f2b32333 100644 --- a/.github/workflows/python-pytest-s3.yaml +++ b/.github/workflows/python-pytest-s3.yaml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9, 3.10] + python-version: [3.6, 3.7, 3.8, 3.9, '3.10'] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/python-pytest-tune.yaml b/.github/workflows/python-pytest-tune.yaml index 62d0f900..e0ac2a8d 100644 --- a/.github/workflows/python-pytest-tune.yaml +++ b/.github/workflows/python-pytest-tune.yaml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9, 3.10] + python-version: [3.7, 3.8, 3.9, '3.10'] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/python-pytest.yml b/.github/workflows/python-pytest.yml index d67adb16..3b4c3c01 100644 --- a/.github/workflows/python-pytest.yml +++ b/.github/workflows/python-pytest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9, 3.10] + python-version: [3.6, 3.7, 3.8, 3.9, '3.10'] steps: - uses: actions/checkout@v2 From 7a855dd0ace9958ce2713459a2b3d79295078596 Mon Sep 17 00:00:00 2001 From: Nicholas Cilfone Date: Fri, 11 Mar 2022 14:01:07 -0500 Subject: [PATCH 4/8] fixing issue with actual typing.Callable type across all python versions --- requirements/S3_REQUIREMENTS.txt | 2 +- spock/backend/builder.py | 8 ++------ spock/backend/field_handlers.py | 10 ++-------- spock/backend/typed.py | 21 ++++----------------- spock/utils.py | 31 ++++++++++++++++++++++++++----- 5 files changed, 35 insertions(+), 37 deletions(-) diff --git a/requirements/S3_REQUIREMENTS.txt b/requirements/S3_REQUIREMENTS.txt index 83074115..f1efd1a7 100644 --- a/requirements/S3_REQUIREMENTS.txt +++ b/requirements/S3_REQUIREMENTS.txt @@ -1,4 +1,4 @@ boto3~=1.20 botocore~=1.24 -hurry.filesize==0.9 +hurry.filesize~=0.9 s3transfer~=0.5 diff --git a/spock/backend/builder.py b/spock/backend/builder.py index 11bd9d67..a8ae7ea6 100644 --- a/spock/backend/builder.py +++ b/spock/backend/builder.py @@ -20,11 +20,7 @@ from spock.graph import Graph from spock.utils import make_argument -minor = sys.version_info.minor -if minor < 7: - from typing import CallableMeta as _VariadicGenericAlias -else: - from typing import _GenericAlias, _VariadicGenericAlias +from spock.utils import _SpockVariadicGenericAlias class BaseBuilder(ABC): # pylint: disable=too-few-public-methods @@ -265,7 +261,7 @@ def _make_group_override_parser(parser, class_obj, class_name): # Check if the val type has __args__ -- this catches GenericAlias classes # TODO (ncilfone): Fix up this super super ugly logic if ( - not isinstance(val_type, _VariadicGenericAlias) + not isinstance(val_type, _SpockVariadicGenericAlias) and hasattr(val_type, "__args__") and ((list(set(val_type.__args__))[0]).__module__ == class_name) and attr.has((list(set(val_type.__args__))[0])) diff --git a/spock/backend/field_handlers.py b/spock/backend/field_handlers.py index 805eb3d7..fa4287c6 100644 --- a/spock/backend/field_handlers.py +++ b/spock/backend/field_handlers.py @@ -19,13 +19,7 @@ _SpockNotOptionalError, _SpockValueError, ) -from spock.utils import _check_iterable, _is_spock_instance, _is_spock_tune_instance - -minor = sys.version_info.minor -if minor < 7: - from typing import CallableMeta as _VariadicGenericAlias -else: - from typing import _VariadicGenericAlias +from spock.utils import _check_iterable, _is_spock_instance, _is_spock_tune_instance, _SpockVariadicGenericAlias class RegisterFieldTemplate(ABC): @@ -681,7 +675,7 @@ def recurse_generate(cls, spock_cls, builder_space: BuilderSpace): elif _is_spock_tune_instance(attribute.type): handler = RegisterTuneCls() # References to callables - elif isinstance(attribute.type, _VariadicGenericAlias): + elif isinstance(attribute.type, _SpockVariadicGenericAlias): handler = RegisterCallableField() # Basic field else: diff --git a/spock/backend/typed.py b/spock/backend/typed.py index fa0ded7f..7ec34f8b 100644 --- a/spock/backend/typed.py +++ b/spock/backend/typed.py @@ -9,18 +9,10 @@ from enum import Enum, EnumMeta from functools import partial from typing import TypeVar, Union +from spock.utils import _SpockGenericAlias, _SpockVariadicGenericAlias import attr -# from spock.exceptions import _SpockValueError - -minor = sys.version_info.minor -if minor < 7: - from typing import CallableMeta as _VariadicGenericAlias - from typing import GenericMeta as _GenericAlias -else: - from typing import _GenericAlias, _VariadicGenericAlias - class SavePath(str): """Spock special key for saving the Spock config to file @@ -367,7 +359,7 @@ def _type_katra(typed, default=None, optional=False): # Grab the name first based on if it is a base type or GenericAlias if isinstance(typed, type): name = typed.__name__ - elif isinstance(typed, _GenericAlias): + elif isinstance(typed, _SpockGenericAlias): name = _get_name_py_version(typed=typed) else: raise TypeError("Encountered an unexpected type in _type_katra") @@ -475,11 +467,6 @@ def _callable_katra(typed, default=None, optional=False): type=typed, metadata={"base": _get_name_py_version(typed)}, ) - - # raise _SpockValueError(f"Types of `{_get_name_py_version(typed)}` must have a default value or be optional -- " - # f"Spock currently has no way to map from a markup style file to a " - # f"{_get_name_py_version(typed)} type") - return x @@ -501,11 +488,11 @@ def katra(typed, default=None): """ # Handle optionals typed, optional = _handle_optional_typing(typed) - if isinstance(typed, _VariadicGenericAlias): + if isinstance(typed, _SpockVariadicGenericAlias): x = _callable_katra(typed=typed, default=default, optional=optional) # We need to check if the type is a _GenericAlias so that we can handle subscripted general types # If it is subscript typed it will not be T which python uses as a generic type name - elif isinstance(typed, _GenericAlias) and ( + elif isinstance(typed, _SpockGenericAlias) and ( not isinstance(typed.__args__[0], TypeVar) ): x = _generic_alias_katra(typed=typed, default=default, optional=optional) diff --git a/spock/utils.py b/spock/utils.py index 45cfc15f..efb16bf2 100644 --- a/spock/utils.py +++ b/spock/utils.py @@ -22,10 +22,31 @@ from spock.exceptions import _SpockValueError minor = sys.version_info.minor -if minor < 7: - from typing import GenericMeta as _GenericAlias -else: - from typing import _GenericAlias + + +def _get_alias_type(): + if minor < 7: + from typing import GenericMeta as _GenericAlias + else: + from typing import _GenericAlias + + return _GenericAlias + + +def _get_callable_type(): + if minor == 6: + from typing import CallableMeta as _VariadicGenericAlias + elif (minor > 6) and (minor < 9): + from typing import _VariadicGenericAlias + elif minor > 8: + from typing import _CallableType as _VariadicGenericAlias + else: + raise RuntimeError(f"Attempting to use spock with python version `3.{minor}` which is unsupported") + return _VariadicGenericAlias + + +_SpockGenericAlias = _get_alias_type() +_SpockVariadicGenericAlias = _get_callable_type() def within( @@ -283,7 +304,7 @@ def make_argument(arg_name, arg_type, parser): """ # For generic alias we take the input string and use a custom type callable to convert - if isinstance(arg_type, _GenericAlias): + if isinstance(arg_type, _SpockGenericAlias): parser.add_argument(arg_name, required=False, type=_handle_generic_type_args) # For Unions -- python 3.6 can't deal with them correctly -- use the same ast method that generics require elif ( From d5879a68c48aba3f79541e67e3799f3d09fba30a Mon Sep 17 00:00:00 2001 From: Nicholas Cilfone Date: Fri, 11 Mar 2022 14:11:13 -0500 Subject: [PATCH 5/8] fixing issue with typing.Callable types at the command line --- spock/backend/builder.py | 4 ++++ tests/base/test_cmd_line.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/spock/backend/builder.py b/spock/backend/builder.py index a8ae7ea6..a159ac51 100644 --- a/spock/backend/builder.py +++ b/spock/backend/builder.py @@ -278,6 +278,10 @@ def _make_group_override_parser(parser, class_obj, class_name): arg_name = f"--{str(attr_name)}.{val.name}" val_type = str group_parser = make_argument(arg_name, val_type, group_parser) + # This catches callables -- need to be of type str which will be use in importlib + elif isinstance(val.type, _SpockVariadicGenericAlias): + arg_name = f"--{str(attr_name)}.{val.name}" + group_parser = make_argument(arg_name, str, group_parser) else: arg_name = f"--{str(attr_name)}.{val.name}" group_parser = make_argument(arg_name, val_type, group_parser) diff --git a/tests/base/test_cmd_line.py b/tests/base/test_cmd_line.py index 7d76b028..b024f069 100644 --- a/tests/base/test_cmd_line.py +++ b/tests/base/test_cmd_line.py @@ -79,7 +79,7 @@ def arg_builder(monkeypatch): "--SecondDoubleNestedConfig.morph_tolerance", "0.2", "--TypeConfig.call_me", - "'tests.base.attr_configs_test.foo'" + 'tests.base.attr_configs_test.foo' ], ) config = ConfigArgBuilder( @@ -192,7 +192,7 @@ def arg_builder(monkeypatch): "--TypeConfig.high_config", "SingleNestedConfig", "--TypeConfig.call_me", - "'tests.base.attr_configs_test.foo'" + 'tests.base.attr_configs_test.foo' ], ) config = ConfigArgBuilder( From 4b916fc325bdd005491700c76cca4547a00166ac Mon Sep 17 00:00:00 2001 From: Nicholas Cilfone Date: Fri, 11 Mar 2022 14:18:38 -0500 Subject: [PATCH 6/8] linted. attempting to bust the actions cache since I can't replicate the 3.6 tests failing locally in a venv --- .github/workflows/python-pytest-s3.yaml | 4 ++-- .github/workflows/python-pytest-tune.yaml | 4 ++-- .github/workflows/python-pytest.yml | 4 ++-- spock/backend/builder.py | 4 +--- spock/backend/field_handlers.py | 7 ++++++- spock/backend/typed.py | 3 ++- spock/utils.py | 4 +++- 7 files changed, 18 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python-pytest-s3.yaml b/.github/workflows/python-pytest-s3.yaml index f2b32333..70007890 100644 --- a/.github/workflows/python-pytest-s3.yaml +++ b/.github/workflows/python-pytest-s3.yaml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9, '3.10'] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 @@ -26,7 +26,7 @@ jobs: - uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/DEV_REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/S3_REQUIREMENTS.txt') }} + key: cache-v1-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/DEV_REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/S3_REQUIREMENTS.txt') }} - name: Install dependencies run: | diff --git a/.github/workflows/python-pytest-tune.yaml b/.github/workflows/python-pytest-tune.yaml index e0ac2a8d..f0dc1348 100644 --- a/.github/workflows/python-pytest-tune.yaml +++ b/.github/workflows/python-pytest-tune.yaml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9, '3.10'] + python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 @@ -26,7 +26,7 @@ jobs: - uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/DEV_REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/S3_REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/TUNE_REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/TEST_EXTRAS_REQUIREMENTS_REQUIREMENTS.txt') }} + key: cache-v1-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/DEV_REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/S3_REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/TUNE_REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/TEST_EXTRAS_REQUIREMENTS_REQUIREMENTS.txt') }} - name: Install dependencies run: | diff --git a/.github/workflows/python-pytest.yml b/.github/workflows/python-pytest.yml index 3b4c3c01..2b17f20e 100644 --- a/.github/workflows/python-pytest.yml +++ b/.github/workflows/python-pytest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9, '3.10'] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 @@ -26,7 +26,7 @@ jobs: - uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/DEV_REQUIREMENTS.txt') }} + key: cache-v1-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/DEV_REQUIREMENTS.txt') }} - name: Install dependencies run: | diff --git a/spock/backend/builder.py b/spock/backend/builder.py index a159ac51..860e59f8 100644 --- a/spock/backend/builder.py +++ b/spock/backend/builder.py @@ -18,9 +18,7 @@ from spock.backend.spaces import BuilderSpace from spock.backend.wrappers import Spockspace from spock.graph import Graph -from spock.utils import make_argument - -from spock.utils import _SpockVariadicGenericAlias +from spock.utils import _SpockVariadicGenericAlias, make_argument class BaseBuilder(ABC): # pylint: disable=too-few-public-methods diff --git a/spock/backend/field_handlers.py b/spock/backend/field_handlers.py index fa4287c6..e4cff0aa 100644 --- a/spock/backend/field_handlers.py +++ b/spock/backend/field_handlers.py @@ -19,7 +19,12 @@ _SpockNotOptionalError, _SpockValueError, ) -from spock.utils import _check_iterable, _is_spock_instance, _is_spock_tune_instance, _SpockVariadicGenericAlias +from spock.utils import ( + _check_iterable, + _is_spock_instance, + _is_spock_tune_instance, + _SpockVariadicGenericAlias, +) class RegisterFieldTemplate(ABC): diff --git a/spock/backend/typed.py b/spock/backend/typed.py index 7ec34f8b..19dce9a7 100644 --- a/spock/backend/typed.py +++ b/spock/backend/typed.py @@ -9,10 +9,11 @@ from enum import Enum, EnumMeta from functools import partial from typing import TypeVar, Union -from spock.utils import _SpockGenericAlias, _SpockVariadicGenericAlias import attr +from spock.utils import _SpockGenericAlias, _SpockVariadicGenericAlias + class SavePath(str): """Spock special key for saving the Spock config to file diff --git a/spock/utils.py b/spock/utils.py index efb16bf2..635df88d 100644 --- a/spock/utils.py +++ b/spock/utils.py @@ -41,7 +41,9 @@ def _get_callable_type(): elif minor > 8: from typing import _CallableType as _VariadicGenericAlias else: - raise RuntimeError(f"Attempting to use spock with python version `3.{minor}` which is unsupported") + raise RuntimeError( + f"Attempting to use spock with python version `3.{minor}` which is unsupported" + ) return _VariadicGenericAlias From abe0a87b58a832f1b251239aefd58f2fd3de15ba Mon Sep 17 00:00:00 2001 From: Nicholas Cilfone Date: Fri, 11 Mar 2022 16:19:29 -0500 Subject: [PATCH 7/8] fixing issue with optional callables on 3.6 --- spock/backend/typed.py | 5 +++-- tests/base/attr_configs_test.py | 2 +- tests/base/base_asserts_test.py | 10 +++++----- tests/base/test_config_arg_builder.py | 4 ++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/spock/backend/typed.py b/spock/backend/typed.py index 19dce9a7..6653ea5e 100644 --- a/spock/backend/typed.py +++ b/spock/backend/typed.py @@ -414,8 +414,9 @@ def _handle_optional_typing(typed): """ # Set optional to false optional = False - # Check if it has __args__ to look for optionality as it is a GenericAlias - if hasattr(typed, "__args__"): + # Check if it has __args__ to look for optionality as it is a GenericAlias -- also make sure it is not + # callable as that also has __args__ + if hasattr(typed, "__args__") and not isinstance(typed, _SpockVariadicGenericAlias): # If it is more than one than it is most likely optional but check against NoneType in the tuple to verify # Check the length of type __args__ type_args = typed.__args__ diff --git a/tests/base/attr_configs_test.py b/tests/base/attr_configs_test.py index 70d035fd..f0503de2 100644 --- a/tests/base/attr_configs_test.py +++ b/tests/base/attr_configs_test.py @@ -331,7 +331,7 @@ class Bar: @spock(dynamic=True) -class TestConfigDynamicDefaults(Foo, Bar): +class ConfigDynamicDefaults(Foo, Bar): x: int = 235 y: str = 'yarghhh' z: List[int] = [10, 20] diff --git a/tests/base/base_asserts_test.py b/tests/base/base_asserts_test.py index 0f2398d7..c3cdd1dc 100644 --- a/tests/base/base_asserts_test.py +++ b/tests/base/base_asserts_test.py @@ -230,8 +230,8 @@ def test_all_inherited(self, arg_builder): class AllDynamic: def test_all_dynamic(self, arg_builder): - assert arg_builder.TestConfigDynamicDefaults.x == 235 - assert arg_builder.TestConfigDynamicDefaults.y == "yarghhh" - assert arg_builder.TestConfigDynamicDefaults.z == [10, 20] - assert arg_builder.TestConfigDynamicDefaults.p == 1 - assert arg_builder.TestConfigDynamicDefaults.q == 'shhh' \ No newline at end of file + assert arg_builder.ConfigDynamicDefaults.x == 235 + assert arg_builder.ConfigDynamicDefaults.y == "yarghhh" + assert arg_builder.ConfigDynamicDefaults.z == [10, 20] + assert arg_builder.ConfigDynamicDefaults.p == 1 + assert arg_builder.ConfigDynamicDefaults.q == 'shhh' \ No newline at end of file diff --git a/tests/base/test_config_arg_builder.py b/tests/base/test_config_arg_builder.py index fcd2dcf0..5f40121e 100644 --- a/tests/base/test_config_arg_builder.py +++ b/tests/base/test_config_arg_builder.py @@ -205,7 +205,7 @@ class TestDynamic(AllDynamic): def arg_builder(monkeypatch): with monkeypatch.context() as m: m.setattr(sys, "argv", [""]) - config = ConfigArgBuilder(TestConfigDynamicDefaults) + config = ConfigArgBuilder(ConfigDynamicDefaults) return config.generate() @@ -247,7 +247,7 @@ class TestDynamic(AllDynamic): def arg_builder(monkeypatch): with monkeypatch.context() as m: m.setattr(sys, "argv", [""]) - config = ConfigArgBuilder(TestConfigDynamicDefaults) + config = ConfigArgBuilder(ConfigDynamicDefaults) return config.generate() From 497b16f5516bac09924cf1a5691d325b3cbd2973 Mon Sep 17 00:00:00 2001 From: Nicholas Cilfone Date: Fri, 11 Mar 2022 16:26:11 -0500 Subject: [PATCH 8/8] updating readme --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 04f1bebb..5f1577f8 100644 --- a/README.md +++ b/README.md @@ -7,12 +7,16 @@

+ + + +

+ +

- - - +