diff --git a/skpro/tests/scenarios/__init__.py b/skpro/tests/scenarios/__init__.py new file mode 100644 index 00000000..5643ff79 --- /dev/null +++ b/skpro/tests/scenarios/__init__.py @@ -0,0 +1 @@ +"""Test scenarios for estimators.""" diff --git a/skpro/tests/scenarios/scenarios.py b/skpro/tests/scenarios/scenarios.py new file mode 100644 index 00000000..ffa33c7f --- /dev/null +++ b/skpro/tests/scenarios/scenarios.py @@ -0,0 +1,271 @@ +"""Testing utility to play back usage scenarios for estimators. + +Contains TestScenario class which applies method/args subsequently +""" +# copied from sktime. Should be jointly refactored to scikit-base. + +__author__ = ["fkiraly"] + +__all__ = ["TestScenario"] + + +from copy import deepcopy +from inspect import isclass + + +class TestScenario: + """Class to run pre-defined method execution scenarios for objects. + + Parameters + ---------- + args : dict of dict, default = None + dict of argument dicts to be used in methods + names for keys need not equal names of methods these are used in + but scripted method will look at key with same name as default + must be passed to constructor, set in a child class + or dynamically created in get_args + default_method_sequence : list of str, default = None + default sequence for methods to be called + optional, if given, default method sequence to use in `run` + if not provided, at least one of the sequence arguments must be passed in `run` + or default_arg_sequence must be provided + default_arg_sequence : list of str, default = None + default sequence of keys for keyword argument dicts to be used + names for keys need not equal names of methods + if not provided, at least one of the sequence arguments must be passed in `run` + or default_method_sequence must be provided + + Methods + ------- + run(obj, args=None, default_method_sequence=None) + Run a call(args) scenario on obj, and retrieve method outputs. + is_applicable(obj) + Check whether scenario is applicable to obj. + get_args(key, obj) + Dynamically create args for call defined by key and obj. + Defaults to self.args[key] if not overridden. + """ + + def __init__( + self, args=None, default_method_sequence=None, default_arg_sequence=None + ): + if default_method_sequence is not None: + self.default_method_sequence = _check_list_of_str(default_method_sequence) + elif not hasattr(self, "default_method_sequence"): + self.default_method_sequence = None + if default_arg_sequence is not None: + self.default_arg_sequence = _check_list_of_str(default_arg_sequence) + elif not hasattr(self, "default_arg_sequence"): + self.default_arg_sequence = None + if args is not None: + self.args = _check_dict_of_dict(args) + else: + if not hasattr(self, "args"): + raise RuntimeError( + f"{self.__class__.__name__} (scenario class) failed to construct, " + "args must either be given to __init__ or set as an attribute" + ) + _check_dict_of_dict(self.args) + + def get_args(self, key, obj=None, deepcopy_args=True): + """Return args for key. Can be overridden for dynamic arg generation. + + If overridden, must not have any side effects on self.args + e.g., avoid assignments args[key] = x without deepcopying self.args first + + Parameters + ---------- + key : str, argument key to construct/retrieve args for + obj : obj, optional, default=None. Object to construct args for. + deepcopy_args : bool, optional, default=True. Whether to deepcopy return. + + Returns + ------- + args : argument dict to be used for a method, keyed by `key` + names for keys need not equal names of methods these are used in + but scripted method will look at key with same name as default + """ + args = self.args.get(key, {}) + if deepcopy_args: + args = deepcopy(args) + return args + + def run( + self, + obj, + method_sequence=None, + arg_sequence=None, + return_all=False, + return_args=False, + deepcopy_return=False, + ): + """Run a call(args) scenario on obj, and retrieve method outputs. + + Runs a sequence of commands + res_1 = obj.method_1(**args_1) + res_2 = obj.method_2(**args_2) + etc, where method_i is method_sequence[i], + and args_i is self.args[arg_sequence[i]] + and returns results. Args are passed as deepcopy to avoid side effects. + + if method_i is __init__ (a constructor), + obj is changed to obj.__init__(**args_i) from the next line on + + Parameters + ---------- + obj : class or object with methods in method_sequence + method_sequence : list of str, default = arg_sequence if passed + if arg_sequence is also None, then default = self.default_method_sequence + sequence of method names to be run + arg_sequence : list of str, default = method_sequence if passed + if method_sequence is also None, then default = self.default_arg_sequence + sequence of keys for keyword argument dicts to be used + names for keys need not equal names of methods + return_all : bool, default = False + whether all or only the last result should be returned + if False, only the last result is returned + if True, list of deepcopies of intermediate results is returned + return_args : bool, default = False + whether arguments should also be returned + if False, there is no second return argument + if True, "args_after_call" return argument is returned + deepcopy_return : bool, default = False + whether returns are deepcopied before returned + if True, returns are deepcopies of return + if False, returns are references/assignments, not deepcopies + NOTE: if self is returned (e.g., in fit), and deepcopy_return=False + method calls may continue to have side effects on that return + + Returns + ------- + results : output of the last method call, if return_all = False + list of deepcopies of all outputs, if return_all = True + args_after_call : list of args after method call, only if return_args = True + i-th element is deepcopy of args of i-th method call, after method call + this is possibly subject to side effects by the method + """ + # if both None, fill with defaults if exist + if method_sequence is None and arg_sequence is None: + method_sequence = getattr(self, "default_method_sequence", None) + arg_sequence = getattr(self, "default_arg_sequence", None) + + # if both are still None, raise an error + if method_sequence is None and arg_sequence is None: + raise ValueError( + "at least one of method_sequence, arg_sequence must be not None " + "if no defaults are set in the class" + ) + + # if only one is None, fill one with the other + if method_sequence is None: + method_sequence = _check_list_of_str(arg_sequence) + else: + method_sequence = _check_list_of_str(method_sequence) + if arg_sequence is None: + arg_sequence = _check_list_of_str(method_sequence) + else: + arg_sequence = _check_list_of_str(arg_sequence) + + # check that length of sequences is the same + num_calls = len(arg_sequence) + if not num_calls == len(method_sequence): + raise ValueError("arg_sequence and method_sequence must have same length") + + # execute the commands in sequence, report result(s) + results = [] + args_after_call = [] + for i in range(num_calls): + methodname = method_sequence[i] + args = deepcopy(self.get_args(key=arg_sequence[i], obj=obj)) + + if methodname != "__init__": + res = getattr(obj, methodname)(**args) + # if constructor is called, run directly and replace obj + else: + if isclass(obj): + res = obj(**args) + else: + res = type(obj)(**args) + obj = res + + args_after_call += [args] + + if deepcopy_return: + res = deepcopy(res) + + if return_all: + results += [res] + else: + results = res + + if return_args: + return results, args_after_call + else: + return results + + def is_applicable(self, obj): + """Check whether scenario is applicable to obj. + + Abstract method, children should implement. This just returns "true". + + Example for child class: scenario is univariate time series forecasting. + Then, this returns False on multivariate, True on univariate forecasters. + + Parameters + ---------- + obj : class or object to check against scenario + + Returns + ------- + applicable: bool + True if self is applicable to obj, False if not + "applicable" is defined as the implementer chooses, as output of this method + False is typically used as a "skip" flag in unit or integration testing + """ + return True + + +def _check_list_of_str(obj, name="obj"): + """Check whether obj is a list of str. + + Parameters + ---------- + obj : any object, check whether is list of str + name : str, default="obj", name of obj to display in error message + + Returns + ------- + obj, unaltered + + Raises + ------ + TypeError if obj is not list of str + """ + if not isinstance(obj, list) or not all(isinstance(x, str) for x in obj): + raise TypeError(f"{name} must be a list of str") + return obj + + +def _check_dict_of_dict(obj, name="obj"): + """Check whether obj is a dict of dict, with str keys. + + Parameters + ---------- + obj : any object, check whether is dict of dict, with str keys + name : str, default="obj", name of obj to display in error message + + Returns + ------- + obj, unaltered + + Raises + ------ + TypeError if obj is not dict of dict, with str keys + """ + if not ( + isinstance(obj, dict) + and all(isinstance(x, dict) for x in obj.values()) + and all(isinstance(x, str) for x in obj.keys()) + ): + raise TypeError(f"{name} must be a dict of dict, with str keys") + return obj diff --git a/skpro/tests/scenarios/scenarios_getter.py b/skpro/tests/scenarios/scenarios_getter.py new file mode 100644 index 00000000..a867fafd --- /dev/null +++ b/skpro/tests/scenarios/scenarios_getter.py @@ -0,0 +1,108 @@ +"""Retrieval utility for test scenarios.""" +# copied from sktime. Should be jointly refactored to scikit-base. + +__author__ = ["fkiraly"] + +__all__ = ["retrieve_scenarios"] + + +from inspect import isclass + +from skpro.tests.scenarios.scenarios_regressor_proba import scenarios_regressor_proba + +scenarios = dict() +scenarios["regressor_proba"] = scenarios_regressor_proba + + +def retrieve_scenarios(obj, filter_tags=None): + """Retrieve test scenarios for obj, or by estimator scitype string. + + Exactly one of the arguments obj, estimator_type must be provided. + + Parameters + ---------- + obj : class or object, or string, or list of str. + Which kind of estimator/object to retrieve scenarios for. + If object, must be a class or object inheriting from BaseObject. + If string(s), must be in registry.BASE_CLASS_REGISTER (first col) + for instance 'classifier', 'regressor', 'transformer', 'forecaster' + filter_tags: dict of (str or list of str), default=None + subsets the returned objectss as follows: + each key/value pair is statement in "and"/conjunction + key is tag name to sub-set on + value str or list of string are tag values + condition is "key must be equal to value, or in set(value)" + + Returns + ------- + scenarios : list of objects, instances of BaseScenario + """ + # if class, get scitypes from inference; otherwise, str or list of str + if not isinstance(obj, str): + if isclass(obj): + if hasattr(obj, "get_class_tag"): + estimator_type = obj.get_class_tag("object_type", "object") + else: + estimator_type = "object" + else: + if hasattr(obj, "get_tag"): + estimator_type = obj.get_tag("object_type", "object", False) + else: + estimator_type = "object" + else: + estimator_type = obj + + # coerce to list, ensure estimator_type is list of str + if not isinstance(estimator_type, list): + estimator_type = [estimator_type] + + # now loop through types and retrieve scenarios + scenarios_for_type = [] + for est_type in estimator_type: + scens = scenarios.get(est_type) + if scens is not None: + scenarios_for_type += scenarios.get(est_type) + + # instantiate all scenarios by calling constructor + scenarios_for_type = [x() for x in scenarios_for_type] + + # if obj was an object, filter to applicable scenarios + if not isinstance(obj, str) and not isinstance(obj, list): + scenarios_for_type = [x for x in scenarios_for_type if x.is_applicable(obj)] + + if filter_tags is not None: + scenarios_for_type = [ + scen for scen in scenarios_for_type if _check_tag_cond(scen, filter_tags) + ] + + return scenarios_for_type + + +def _check_tag_cond(obj, filter_tags=None): + """Check whether object satisfies filter_tags condition. + + Parameters + ---------- + obj: object inheriting from sktime BaseObject + filter_tags: dict of (str or list of str), default=None + subsets the returned objectss as follows: + each key/value pair is statement in "and"/conjunction + key is tag name to sub-set on + value str or list of string are tag values + condition is "key must be equal to value, or in set(value)" + + Returns + ------- + cond_sat: bool, whether estimator satisfies condition in filter_tags + """ + if not isinstance(filter_tags, dict): + raise TypeError("filter_tags must be a dict") + + cond_sat = True + + for key, value in filter_tags.items(): + if not isinstance(value, list): + value = [value] + cond_sat = cond_sat and obj.get_class_tag(key) in set(value) + + return cond_sat diff --git a/skpro/tests/scenarios/scenarios_regressor_proba.py b/skpro/tests/scenarios/scenarios_regressor_proba.py new file mode 100644 index 00000000..1f1f6689 --- /dev/null +++ b/skpro/tests/scenarios/scenarios_regressor_proba.py @@ -0,0 +1,114 @@ +"""Test scenarios for classification and regression. + +Contains TestScenario concrete children to run in tests for classifiers/regressirs. +""" + +__author__ = ["fkiraly"] + +__all__ = ["scenarios_regressor_proba"] + +from inspect import isclass + +from skpro.base import BaseObject +from skpro.tests.scenarios.scenarios import TestScenario + + +class _ProbaRegressorTestScenario(TestScenario, BaseObject): + """Generic test scenario for classifiers.""" + + def get_args(self, key, obj=None, deepcopy_args=True): + """Return args for key. Can be overridden for dynamic arg generation. + + If overridden, must not have any side effects on self.args + e.g., avoid assignments args[key] = x without deepcopying self.args first + + Parameters + ---------- + key : str, argument key to construct/retrieve args for + obj : obj, optional, default=None. Object to construct args for. + deepcopy_args : bool, optional, default=True. Whether to deepcopy return. + + Returns + ------- + args : argument dict to be used for a method, keyed by `key` + names for keys need not equal names of methods these are used in + but scripted method will look at key with same name as default + """ + PREDICT_LIKE_FUNCTIONS = ["predict", "predict_var", "predict_proba"] + # use same args for predict-like functions as for predict + if key in PREDICT_LIKE_FUNCTIONS: + key = "predict" + + return super().get_args(key=key, obj=obj, deepcopy_args=deepcopy_args) + + def is_applicable(self, obj): + """Check whether scenario is applicable to obj. + + Parameters + ---------- + obj : class or object to check against scenario + + Returns + ------- + applicable: bool + True if self is applicable to obj, False if not + """ + + def get_tag(obj, tag_name): + if isclass(obj): + return obj.get_class_tag(tag_name) + else: + return obj.get_tag(tag_name) + + # applicable only if object is a BaseProbaRegressor + if not get_tag(obj, "object_type") == "regressor_proba": + return False + + # if X has missing data, applicable only if can handle missing data + has_missing_data = self.get_tag("X_missing", False, False) + if has_missing_data and not get_tag(obj, "capability:missing"): + return False + + return True + + +# as a function to ensure we can move to fixture-like structure later +def _get_Xy_traintest(): + import pandas as pd + from sklearn.datasets import load_diabetes + from sklearn.model_selection import train_test_split + + X, y = load_diabetes(return_X_y=True, as_frame=True) + X = X.iloc[:50] + y = y.iloc[:50] + y = pd.DataFrame(y) + + X_train, X_test, y_train, y_test = train_test_split(X, y) + + return X_train, X_test, y_train, y_test + + +X_train, X_test, y_train, y_test = _get_Xy_traintest() + + +class ProbaRegressorBasic(_ProbaRegressorTestScenario): + """Fit/predict with univariate panel X, numpy3D mtype, and labels y.""" + + _tags = { + "X_univariate": False, + "X_missing": False, + "y_univariate": True, + "is_enabled": True, + } + + args = { + "fit": {"X": X_train, "y": y_train}, + "predict": {"X": X_test}, + } + default_method_sequence = ["fit", "predict", "predict_proba"] + default_arg_sequence = ["fit", "predict", "predict"] + + +scenarios_regressor_proba = [ + ProbaRegressorBasic, +] diff --git a/skpro/tests/test_all_estimators.py b/skpro/tests/test_all_estimators.py index 6017202e..abb89b62 100644 --- a/skpro/tests/test_all_estimators.py +++ b/skpro/tests/test_all_estimators.py @@ -1,14 +1,21 @@ """Automated tests based on the skbase test suite template.""" import numbers import types +from copy import deepcopy from inspect import getfullargspec, signature +import joblib import numpy as np +import pandas as pd +from skbase.testing import BaseFixtureGenerator as _BaseFixtureGenerator from skbase.testing import TestAllObjects as _TestAllObjects from skbase.testing.utils.inspect import _get_args +from skbase.utils import deep_equals -from skpro.registry import OBJECT_TAG_LIST +from skpro.registry import OBJECT_TAG_LIST, all_objects +from skpro.tests.scenarios.scenarios_getter import retrieve_scenarios from skpro.utils.git_diff import is_class_changed +from skpro.utils.random_state import set_random_state # whether to test only estimators from modules that are changed w.r.t. main # default is False, can be set to True by pytest --only_changed_modules True flag @@ -34,20 +41,110 @@ class PackageConfig: valid_tags = OBJECT_TAG_LIST -class BaseFixtureGenerator: - """Base class for fixture generation, overrides skbase object retrieval.""" - +class BaseFixtureGenerator(_BaseFixtureGenerator): + """Fixture generator for base testing functionality in sktime. + + Test classes inheriting from this and not overriding pytest_generate_tests + will have estimator and scenario fixtures parametrized out of the box. + + Descendants can override: + estimator_type_filter: str, class variable; None or scitype string + e.g., "forecaster", "transformer", "classifier", see BASE_CLASS_SCITYPE_LIST + which estimators are being retrieved and tested + fixture_sequence: list of str + sequence of fixture variable names in conditional fixture generation + _generate_[variable]: object methods, all (test_name: str, **kwargs) -> list + generating list of fixtures for fixture variable with name [variable] + to be used in test with name test_name + can optionally use values for fixtures earlier in fixture_sequence, + these must be input as kwargs in a call + is_excluded: static method (test_name: str, est: class) -> bool + whether test with name test_name should be excluded for estimator est + should be used only for encoding general rules, not individual skips + individual skips should go on the EXCLUDED_TESTS list in _config + requires _generate_estimator_class and _generate_estimator_instance as is + _excluded_scenario: static method (test_name: str, scenario) -> bool + whether scenario should be skipped in test with test_name test_name + requires _generate_estimator_scenario as is + + Fixtures parametrized + --------------------- + object_class: estimator inheriting from BaseObject + ranges over estimator classes not excluded by EXCLUDE_ESTIMATORS, EXCLUDED_TESTS + object_instance: instance of estimator inheriting from BaseObject + ranges over estimator classes not excluded by EXCLUDE_ESTIMATORS, EXCLUDED_TESTS + instances are generated by create_test_instance class method of estimator_class + scenario: instance of TestScenario + ranges over all scenarios returned by retrieve_scenarios + applicable for estimator_class or estimator_instance + """ + + # overrides object retrieval in scikit-base def _all_objects(self): """Retrieve list of all object classes of type self.object_type_filter.""" - obj_list = super()._all_objects() + obj_list = all_objects( + object_types=getattr(self, "object_type_filter", None), + return_names=False, + exclude_objects=self.exclude_objects, + ) # this setting ensures that only estimators are tested that have changed # in the sense that any line in the module is different from main - if ONLY_CHANGED_MODULES: + # exception: if this module has changed, we always run all tests by override + if ONLY_CHANGED_MODULES and not is_class_changed(type(self)): obj_list = [obj for obj in obj_list if is_class_changed(obj)] return obj_list + # which sequence the conditional fixtures are generated in + fixture_sequence = [ + "object_class", + "object_instance", + "scenario", + ] + + def _generate_scenario(self, test_name, **kwargs): + """Return estimator test scenario. + + Fixtures parametrized + --------------------- + scenario: instance of TestScenario + ranges over all scenarios returned by retrieve_scenarios + """ + if "object_class" in kwargs.keys(): + obj = kwargs["object_class"] + elif "object_instance" in kwargs.keys(): + obj = kwargs["object_instance"] + else: + return [] + + scenarios = retrieve_scenarios(obj) + scenarios = [s for s in scenarios if not self._excluded_scenario(test_name, s)] + scenario_names = [type(scen).__name__ for scen in scenarios] + + return scenarios, scenario_names + + @staticmethod + def _excluded_scenario(test_name, scenario): + """Skip list generator for scenarios to skip in test_name. + + Arguments + --------- + test_name : str, name of test + scenario : instance of TestScenario, to be used in test + + Returns + ------- + bool, whether scenario should be skipped in test_name + """ + # this line excludes all scenarios that do not have "is_enabled" flag + # we should slowly enable more scenarios for better coverage + # comment out to run the full test suite with new scenarios + if not scenario.get_tag("is_enabled", False, raise_error=False): + return True + + return False + class TestAllObjects(PackageConfig, BaseFixtureGenerator, _TestAllObjects): """Generic tests for all objects in the mini package.""" @@ -182,3 +279,78 @@ def unreserved(params): f"Reason for discrepancy: {equals_msg}" ) assert is_equal, msg + + +class TestAllEstimators(PackageConfig, BaseFixtureGenerator): + """Package level tests for all sktime estimators, i.e., objects with fit.""" + + def test_fit_updates_state(self, object_instance, scenario): + """Check fit/update state change.""" + # Check that fit updates the is-fitted states + attrs = ["_is_fitted", "is_fitted"] + + estimator = object_instance + object_class = type(object_instance) + + msg = ( + f"{object_class.__name__}.__init__ should call " + f"super({object_class.__name__}, self).__init__, " + "but that does not seem to be the case. Please ensure to call the " + f"parent class's constructor in {object_class.__name__}.__init__" + ) + assert hasattr(estimator, "_is_fitted"), msg + + # Check is_fitted attribute is set correctly to False before fit, at init + for attr in attrs: + assert not getattr( + estimator, attr + ), f"Estimator: {estimator} does not initiate attribute: {attr} to False" + + fitted_estimator = scenario.run(object_instance, method_sequence=["fit"]) + + # Check is_fitted attributes are updated correctly to True after calling fit + for attr in attrs: + assert getattr( + fitted_estimator, attr + ), f"Estimator: {estimator} does not update attribute: {attr} during fit" + + def test_fit_returns_self(self, object_instance, scenario): + """Check that fit returns self.""" + fit_return = scenario.run(object_instance, method_sequence=["fit"]) + assert ( + fit_return is object_instance + ), f"Estimator: {object_instance} does not return self when calling fit" + + def test_fit_does_not_overwrite_hyper_params(self, object_instance, scenario): + """Check that we do not overwrite hyper-parameters in fit.""" + estimator = object_instance + set_random_state(estimator) + + # Make a physical copy of the original estimator parameters before fitting. + params = estimator.get_params() + original_params = deepcopy(params) + + # Fit the model + fitted_est = scenario.run(object_instance, method_sequence=["fit"]) + + # Compare the state of the model parameters with the original parameters + new_params = fitted_est.get_params() + for param_name, original_value in original_params.items(): + new_value = new_params[param_name] + + # We should never change or mutate the internal state of input + # parameters by default. To check this we use the joblib.hash function + # that introspects recursively any subobjects to compute a checksum. + # The only exception to this rule of immutable constructor parameters + # is possible RandomState instance but in this check we explicitly + # fixed the random_state params recursively to be integer seeds. + msg = ( + "Estimator %s should not change or mutate " + " the parameter %s from %s to %s during fit." + % (estimator.__class__.__name__, param_name, original_value, new_value) + ) + # joblib.hash has problems with pandas objects, so we use deep_equals then + if isinstance(original_value, (pd.DataFrame, pd.Series)): + assert deep_equals(new_value, original_value), msg + else: + assert joblib.hash(new_value) == joblib.hash(original_value), msg diff --git a/skpro/utils/random_state.py b/skpro/utils/random_state.py new file mode 100644 index 00000000..a55c9ee3 --- /dev/null +++ b/skpro/utils/random_state.py @@ -0,0 +1,36 @@ +"""Utilities for handling the random_state variable.""" +# copied from scikit-learn to avoid dependency on sklearn private methods + +import numpy as np +from sklearn.utils import check_random_state + + +def set_random_state(estimator, random_state=0): + """Set fixed random_state parameters for an estimator. + + Finds all parameters ending ``random_state`` and sets them to integers + derived from ``random_state``. + + Parameters + ---------- + estimator : estimator supporting get_params, set_params + Estimator with potential randomness managed by random_state parameters. + + random_state : int, RandomState instance or None, default=None + Pseudo-random number generator to control the generation of the random + integers. Pass an int for reproducible output across multiple function calls. + + Notes + ----- + This does not necessarily set *all* ``random_state`` attributes that + control an estimator's randomness, only those accessible through + ``estimator.get_params()``. + """ + random_state = check_random_state(random_state) + to_set = {} + for key in sorted(estimator.get_params(deep=True)): + if key == "random_state" or key.endswith("__random_state"): + to_set[key] = random_state.randint(np.iinfo(np.int32).max) + + if to_set: + estimator.set_params(**to_set)