diff --git a/smt/applications/mfk.py b/smt/applications/mfk.py index 604e9ae7a..7da2ce83b 100644 --- a/smt/applications/mfk.py +++ b/smt/applications/mfk.py @@ -296,6 +296,9 @@ def _new_train_init(self): _, self.cat_features = compute_X_cont( np.concatenate(xt, axis=0), self.design_space ) + self.X_offset[self.cat_features] *= 0 + self.X_scale[self.cat_features] *= 0 + self.X_scale[self.cat_features] += 1 nlevel = self.nlvl @@ -761,7 +764,9 @@ def predict_variances_all_levels(self, X, is_acting=None): sigma2 = self.optimal_par[0]["sigma2"] / self.y_std**2 MSE[:, 0] = sigma2 * ( # 1 + self.optimal_noise_all[0] - (r_t ** 2).sum(axis=0) + (u_ ** 2).sum(axis=0) - 1 - (r_t**2).sum(axis=0) + (u_**2).sum(axis=0) + 1 + - (r_t**2).sum(axis=0) + + (u_**2).sum(axis=0) ) # Calculate recursively kriging variance at level i @@ -845,8 +850,7 @@ def predict_variances_all_levels(self, X, is_acting=None): Q_ = (np.dot((yt - np.dot(Ft, beta)).T, yt - np.dot(Ft, beta)))[0, 0] MSE[:, i] = ( # sigma2_rho * MSE[:, i - 1] - +Q_ - / (2 * (self.nt_all[i] - p - q)) + +Q_ / (2 * (self.nt_all[i] - p - q)) # * (1 + self.optimal_noise_all[i] - (r_t ** 2).sum(axis=0)) * (1 - (r_t**2).sum(axis=0)) + sigma2 * (u_**2).sum(axis=0) @@ -854,7 +858,9 @@ def predict_variances_all_levels(self, X, is_acting=None): else: MSE[:, i] = sigma2 * ( # 1 + self.optimal_noise_all[i] - (r_t ** 2).sum(axis=0) + (u_ ** 2).sum(axis=0) - 1 - (r_t**2).sum(axis=0) + (u_**2).sum(axis=0) + 1 + - (r_t**2).sum(axis=0) + + (u_**2).sum(axis=0) ) # + sigma2_rho * MSE[:, i - 1] if self.options["propagate_uncertainty"]: MSE[:, i] = MSE[:, i] + sigma2_rho * MSE[:, i - 1] diff --git a/smt/applications/mixed_integer.py b/smt/applications/mixed_integer.py index 181152329..d03cde662 100644 --- a/smt/applications/mixed_integer.py +++ b/smt/applications/mixed_integer.py @@ -222,9 +222,9 @@ def __init__( ) and self._surrogate.options["categorical_kernel"] is None ): - self._surrogate.options[ - "categorical_kernel" - ] = MixIntKernelType.HOMO_HSPHERE + self._surrogate.options["categorical_kernel"] = ( + MixIntKernelType.HOMO_HSPHERE + ) warnings.warn( "Using MixedIntegerSurrogateModel integer model with Continuous Relaxation is not supported. \ Switched to homoscedastic hypersphere kernel instead." diff --git a/smt/applications/tests/test_mixed_integer.py b/smt/applications/tests/test_mixed_integer.py index 59ca82523..6da363b64 100644 --- a/smt/applications/tests/test_mixed_integer.py +++ b/smt/applications/tests/test_mixed_integer.py @@ -1316,26 +1316,26 @@ def test_hierarchical_variables_NN(self): sm.predict_values( np.array( [ - [0, -1, -2, 8, 0, 2, 0, 0], - [1, -1, -2, 16, 1, 2, 1, 0], - [2, -1, -2, 32, 2, 2, 1, -2], + [0, -1, -2, 0, 0, 2, 0, 0], + [1, -1, -2, 1, 1, 2, 1, 0], + [2, -1, -2, 2, 2, 2, 1, -2], ] ) )[:, 0], sm.predict_values( np.array( [ - [0, -1, -2, 8, 0, 2, 10, 10], - [1, -1, -2, 16, 1, 2, 1, 10], - [2, -1, -2, 32, 2, 2, 1, -2], + [0, -1, -2, 0, 0, 2, 10, 10], + [1, -1, -2, 1, 1, 2, 1, 10], + [2, -1, -2, 2, 2, 2, 1, -2], ] ) )[:, 0], ) self.assertTrue( np.linalg.norm( - sm.predict_values(np.array([[0, -1, -2, 8, 0, 2, 0, 0]])) - - sm.predict_values(np.array([[0, -1, -2, 8, 0, 12, 10, 10]])) + sm.predict_values(np.array([[0, -1, -2, 0, 0, 2, 0, 0]])) + - sm.predict_values(np.array([[0, -1, -2, 0, 0, 12, 10, 10]])) ) > 1e-8 ) @@ -1789,7 +1789,7 @@ def test_compound_hetero_noise_auto(self): ) def test_mixed_homo_gaussian_3D_ord_cate(self): - xt = np.array([[0.5, 0, 5], [2, 3, 4], [5, 2, -1], [-2, 4, 0.5]]) + xt = np.array([[0, 5, 0], [2, 4, 3], [4, -1, 2], [2, 0.5, 1]]) yt = np.array([[0.0], [3], [1.0], [1.5]]) design_space = DesignSpace( [ diff --git a/smt/surrogate_models/genn.py b/smt/surrogate_models/genn.py index d7500ede7..207a6e203 100644 --- a/smt/surrogate_models/genn.py +++ b/smt/surrogate_models/genn.py @@ -200,9 +200,11 @@ def _train(self): beta1=self.options["beta1"], beta2=self.options["beta2"], epochs=self.options["num_epochs"], - batch_size=None - if self.options["mini_batch_size"] < 0 - else self.options["mini_batch_size"], + batch_size=( + None + if self.options["mini_batch_size"] < 0 + else self.options["mini_batch_size"] + ), max_iter=self.options["num_iterations"], is_backtracking=self.options["is_backtracking"], is_verbose=self.options["is_print"], diff --git a/smt/surrogate_models/krg_based.py b/smt/surrogate_models/krg_based.py index 198cac945..357f425b2 100644 --- a/smt/surrogate_models/krg_based.py +++ b/smt/surrogate_models/krg_based.py @@ -718,16 +718,16 @@ def _matrix_data_corr( d_cont = d[:, np.logical_not(cat_features)] if self.options["corr"] == "squar_sin_exp": if self.options["categorical_kernel"] != MixIntKernelType.GOWER: - theta_cont_features[ - -len([self.design_space.is_cat_mask]) : - ] = np.atleast_2d( - np.array([True] * len([self.design_space.is_cat_mask])) - ).T - theta_cat_features[1][ - -len([self.design_space.is_cat_mask]) : - ] = np.atleast_2d( - np.array([False] * len([self.design_space.is_cat_mask])) - ).T + theta_cont_features[-len([self.design_space.is_cat_mask]) :] = ( + np.atleast_2d( + np.array([True] * len([self.design_space.is_cat_mask])) + ).T + ) + theta_cat_features[1][-len([self.design_space.is_cat_mask]) :] = ( + np.atleast_2d( + np.array([False] * len([self.design_space.is_cat_mask])) + ).T + ) theta_cont = theta[theta_cont_features[:, 0]] r_cont = _correlation_types[corr](theta_cont, d_cont) diff --git a/smt/surrogate_models/tests/test_surrogate_model_examples.py b/smt/surrogate_models/tests/test_surrogate_model_examples.py index e9bce514a..c5b39b8a0 100644 --- a/smt/surrogate_models/tests/test_surrogate_model_examples.py +++ b/smt/surrogate_models/tests/test_surrogate_model_examples.py @@ -322,7 +322,7 @@ def test_mixed_gower_krg(self): sm.train() # DOE for validation - x = np.linspace(0, 5, 5) + x = np.linspace(0, 4, 5) y = sm.predict_values(x) plt.plot(xt, yt, "o", label="data") @@ -725,9 +725,9 @@ def df_dx(x): genn.options["hidden_layer_sizes"] = [6, 6] genn.options["alpha"] = 0.1 genn.options["lambd"] = 0.1 - genn.options[ - "gamma" - ] = 1.0 # 1 = gradient-enhanced on, 0 = gradient-enhanced off + genn.options["gamma"] = ( + 1.0 # 1 = gradient-enhanced on, 0 = gradient-enhanced off + ) genn.options["num_iterations"] = 1000 genn.options["is_backtracking"] = True genn.options["is_normalize"] = False diff --git a/smt/utils/design_space.py b/smt/utils/design_space.py index d467e38ef..50e00316e 100644 --- a/smt/utils/design_space.py +++ b/smt/utils/design_space.py @@ -269,6 +269,12 @@ def correct_get_acting(self, x: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: if x_is_unfolded: x, _ = self.fold_x(x) + indi = 0 + for i in self.design_variables: + if not ((isinstance(i, FloatVariable))): + x[:, indi] = np.int64(np.round(x[:, indi], 0)) + indi += 1 + # Correct and get the is_acting matrix x_corrected, is_acting = self._correct_get_acting(x) @@ -443,7 +449,7 @@ def fold_x( i_x_unfold = 0 for i, dv in enumerate(self.design_variables): - if isinstance(dv, CategoricalVariable) and ( + if (isinstance(dv, CategoricalVariable)) and ( fold_mask is None or fold_mask[i] ): n_dim_cat = dv.n_values @@ -517,9 +523,9 @@ def unfold_x( # The is_acting matrix is simply repeated column-wise if is_acting is not None: - is_acting_unfolded[ - :, i_x_unfold : i_x_unfold + n_dim_cat - ] = np.tile(is_acting[:, [i]], (1, n_dim_cat)) + is_acting_unfolded[:, i_x_unfold : i_x_unfold + n_dim_cat] = ( + np.tile(is_acting[:, [i]], (1, n_dim_cat)) + ) i_x_unfold += n_dim_cat @@ -726,27 +732,43 @@ def _is_num(val): self.random_state = random_state # For testing self._cs = None + self._cs2 = None if HAS_CONFIG_SPACE: cs_vars = {} + cs_vars2 = {} + self.isinteger = False for i, dv in enumerate(design_variables): name = f"x{i}" if isinstance(dv, FloatVariable): cs_vars[name] = UniformFloatHyperparameter( name, lower=dv.lower, upper=dv.upper ) + cs_vars2[name] = UniformFloatHyperparameter( + name, lower=dv.lower, upper=dv.upper + ) elif isinstance(dv, IntegerVariable): cs_vars[name] = FixedIntegerParam( name, lower=dv.lower, upper=dv.upper ) + listvalues = [] + for i in range(int(dv.upper - dv.lower + 1)): + listvalues.append(str(int(i + dv.lower))) + cs_vars2[name] = CategoricalHyperparameter(name, choices=listvalues) + self.isinteger = True elif isinstance(dv, OrdinalVariable): cs_vars[name] = OrdinalHyperparameter(name, sequence=dv.values) + cs_vars2[name] = CategoricalHyperparameter(name, choices=dv.values) + elif isinstance(dv, CategoricalVariable): cs_vars[name] = CategoricalHyperparameter(name, choices=dv.values) + cs_vars2[name] = CategoricalHyperparameter(name, choices=dv.values) + else: raise ValueError(f"Unknown variable type: {dv!r}") seed = self._to_seed(random_state) self._cs = NoDefaultConfigurationSpace(space=cs_vars, seed=seed) + self._cs2 = NoDefaultConfigurationSpace(space=cs_vars2, seed=seed) # dict[int, dict[any, list[int]]]: {meta_var_idx: {value: [decreed_var_idx, ...], ...}, ...} self._meta_vars = {} @@ -783,6 +805,31 @@ def declare_decreed_var( condition = EqualsCondition(decreed_param, meta_param, meta_value) self._cs.add_condition(condition) + decreed_param = self._get_param2(decreed_var) + meta_param = self._get_param2(meta_var) + # Add a condition that checks for equality (if single value given) or in-collection (if sequence given) + if isinstance(meta_value, Sequence): + try: + condition = InCondition( + decreed_param, + meta_param, + list(np.atleast_1d(np.array(meta_value, dtype=str))), + ) + except ValueError: + condition = InCondition( + decreed_param, + meta_param, + list(np.atleast_1d(np.array(meta_value, dtype=float))), + ) + else: + try: + condition = EqualsCondition( + decreed_param, meta_param, str(meta_value) + ) + except ValueError: + condition = EqualsCondition(decreed_param, meta_param, meta_value) + + self._cs2.add_condition(condition) # Simplified implementation else: @@ -854,12 +901,45 @@ def add_value_constraint( constraint_clause = ForbiddenAndConjunction(clause1, clause2) self._cs.add_forbidden_clause(constraint_clause) + # Get parameters + param1 = self._get_param2(var1) + param2 = self._get_param2(var2) + # Add forbidden clauses + if isinstance(value1, Sequence): + clause1 = ForbiddenInClause(param1, str(value1)) + else: + clause1 = ForbiddenEqualsClause(param1, str(value1)) + + if isinstance(value2, Sequence): + try: + clause2 = ForbiddenInClause( + param2, list(np.atleast_1d(np.array(value2, dtype=str))) + ) + except ValueError: + clause2 = ForbiddenInClause( + param2, list(np.atleast_1d(np.array(value2, dtype=float))) + ) + else: + try: + clause2 = ForbiddenEqualsClause(param2, str(value2)) + except ValueError: + clause2 = ForbiddenEqualsClause(param2, value2) + + constraint_clause = ForbiddenAndConjunction(clause1, clause2) + self._cs2.add_forbidden_clause(constraint_clause) + def _get_param(self, idx): try: return self._cs.get_hyperparameter(f"x{idx}") except KeyError: raise KeyError(f"Variable not found: {idx}") + def _get_param2(self, idx): + try: + return self._cs2.get_hyperparameter(f"x{idx}") + except KeyError: + raise KeyError(f"Variable not found: {idx}") + @property def _cs_var_idx(self): """ @@ -978,6 +1058,26 @@ def _get_correct_config(self, vector: np.ndarray) -> Configuration: # to find out which parameters should be inactive while True: try: + if self.isinteger: + vector2 = np.copy(vector) + self._cs_denormalize_x_ordered(np.atleast_2d(vector2)) + indvec = 0 + for hp in self._cs2: + if ( + (str(self._cs.get_hyperparameter(hp)).split()[2]) + == "UniformInteger," + and (str(self._cs2.get_hyperparameter(hp)).split()[2][:3]) + == "Cat" + and not (np.isnan(vector2[indvec])) + ): + vector2[indvec] = int(vector2[indvec]) - int( + str(self._cs2.get_hyperparameter(hp)).split()[4][1:-1] + ) + indvec += 1 + self._normalize_x_no_integer(np.atleast_2d(vector2)) + config2 = Configuration(self._cs2, vector=vector2) + config2.is_valid_configuration() + config.is_valid_configuration() return config @@ -1001,9 +1101,22 @@ def _get_correct_config(self, vector: np.ndarray) -> Configuration: if self.seed is None: seed = self._to_seed(self.random_state) self.seed = seed - - return get_random_neighbor(config, seed=self.seed) - + vector = config.get_array().copy() + indvec = 0 + vector2 = np.copy(vector) + for hp in self._cs2: + if ( + str(self._cs2.get_hyperparameter(hp)).split()[2][:3] + ) == "Cat" and not (np.isnan(vector2[indvec])): + + vector2[indvec] = int(vector2[indvec]) + indvec += 1 + + config2 = Configuration(self._cs2, vector=vector2) + config3 = get_random_neighbor(config2, seed=self.seed) + vector3 = config3.get_array().copy() + config4 = Configuration(self._cs, vector=vector3) + return config4 else: raise @@ -1059,6 +1172,17 @@ def _normalize_x(self, x: np.ndarray, cs_normalize=True): dv.upper - dv.lower + 0.9999 ) + def _normalize_x_no_integer(self, x: np.ndarray, cs_normalize=True): + ordereddesign_variables = [ + self.design_variables[i] for i in self._inv_cs_var_idx + ] + for i, dv in enumerate(ordereddesign_variables): + if isinstance(dv, FloatVariable): + if cs_normalize: + x[:, i] = np.clip( + (x[:, i] - dv.lower) / (dv.upper - dv.lower + 1e-16), 0, 1 + ) + elif isinstance(dv, (OrdinalVariable, CategoricalVariable)): # To ensure equal distribution of continuous values to discrete values, we first stretch-out the # continuous values to extend to 0.5 beyond the integer limits and then round. This ensures that the @@ -1066,6 +1190,7 @@ def _normalize_x(self, x: np.ndarray, cs_normalize=True): x[:, i] = self._round_equally_distributed(x[:, i], dv.lower, dv.upper) def _cs_denormalize_x(self, x: np.ndarray): + for i, dv in enumerate(self.design_variables): if isinstance(dv, FloatVariable): x[:, i] = x[:, i] * (dv.upper - dv.lower) + dv.lower @@ -1076,6 +1201,20 @@ def _cs_denormalize_x(self, x: np.ndarray): x[:, i] * (dv.upper - dv.lower + 0.9999) + dv.lower - 0.49999 ) + def _cs_denormalize_x_ordered(self, x: np.ndarray): + ordereddesign_variables = [ + self.design_variables[i] for i in self._inv_cs_var_idx + ] + for i, dv in enumerate(ordereddesign_variables): + if isinstance(dv, FloatVariable): + x[:, i] = x[:, i] * (dv.upper - dv.lower) + dv.lower + + elif isinstance(dv, IntegerVariable): + # Integer values are normalized similarly to what is done in _round_equally_distributed + x[:, i] = np.round( + x[:, i] * (dv.upper - dv.lower + 0.9999) + dv.lower - 0.49999 + ) + def __str__(self): dvs = "\n".join([f"x{i}: {dv!s}" for i, dv in enumerate(self.design_variables)]) return f"Design space:\n{dvs}" diff --git a/smt/utils/test/test_design_space.py b/smt/utils/test/test_design_space.py index 4f624681a..aed18b896 100644 --- a/smt/utils/test/test_design_space.py +++ b/smt/utils/test/test_design_space.py @@ -194,7 +194,7 @@ def test_design_space(self): ds = DesignSpace( [ CategoricalVariable(["A", "B", "C"]), - OrdinalVariable(["E", "F"]), + OrdinalVariable(["0", "1"]), IntegerVariable(-1, 2), FloatVariable(0.5, 1.5), ], @@ -233,18 +233,18 @@ def test_design_space(self): self.assertEqual(is_acting.shape, x.shape) self.assertEqual(ds.decode_values(x, i_dv=0), ["B", "C", "C"]) - self.assertEqual(ds.decode_values(x, i_dv=1), ["E", "E", "E"]) + self.assertEqual(ds.decode_values(x, i_dv=1), ["0", "0", "0"]) self.assertEqual(ds.decode_values(np.array([0, 1, 2]), i_dv=0), ["A", "B", "C"]) - self.assertEqual(ds.decode_values(np.array([0, 1]), i_dv=1), ["E", "F"]) + self.assertEqual(ds.decode_values(np.array([0, 1]), i_dv=1), ["0", "1"]) - self.assertEqual(ds.decode_values(x[0, :]), ["B", "E", 0, x[0, 3]]) - self.assertEqual(ds.decode_values(x[[0], :]), [["B", "E", 0, x[0, 3]]]) + self.assertEqual(ds.decode_values(x[0, :]), ["B", "0", 0, x[0, 3]]) + self.assertEqual(ds.decode_values(x[[0], :]), [["B", "0", 0, x[0, 3]]]) self.assertEqual( ds.decode_values(x), [ - ["B", "E", 0, x[0, 3]], - ["C", "E", -1, x[1, 3]], - ["C", "E", 0, x[2, 3]], + ["B", "0", 0, x[0, 3]], + ["C", "0", -1, x[1, 3]], + ["C", "0", 0, x[2, 3]], ], ) @@ -263,7 +263,7 @@ def test_design_space(self): [ [2.0, 0.0, -1.0, 1.34158548], [0.0, 1.0, -0.0, 0.55199817], - [1.0, 1.0, 2.0, 1.15663662], + [1.0, 1.0, 1.0, 1.15663662], ] ), atol=1e-8, @@ -561,7 +561,31 @@ def test_check_conditionally_acting_2(self): @unittest.skipIf( not HAS_CONFIG_SPACE, "Hierarchy ConfigSpace dependency not installed" ) - def test_restrictive_value_constraint(self): + def test_restrictive_value_constraint_ordinal(self): + ds = DesignSpace( + [ + OrdinalVariable(["0", "1", "2"]), + OrdinalVariable(["0", "1", "2"]), + ] + ) + assert list(ds._cs.values())[0].default_value == "0" + + ds.add_value_constraint(var1=0, value1="1", var2=1, value2="1") + ds.sample_valid_x(100, random_state=42) + + x_cartesian = np.array(list(itertools.product([0, 1, 2], [0, 1, 2]))) + x_cartesian2, _ = ds.correct_get_acting(x_cartesian) + np.testing.assert_array_equal( + np.array( + [[0, 0], [0, 1], [0, 2], [1, 0], [0, 1], [1, 2], [2, 0], [2, 1], [2, 2]] + ), + x_cartesian2, + ) + + @unittest.skipIf( + not HAS_CONFIG_SPACE, "Hierarchy ConfigSpace dependency not installed" + ) + def test_restrictive_value_constraint_integer(self): ds = DesignSpace( [ IntegerVariable(0, 2), @@ -570,11 +594,43 @@ def test_restrictive_value_constraint(self): ) assert list(ds._cs.values())[0].default_value == 1 - ds.add_value_constraint(var1=0, value1=1, var2=0, value2=1) + ds.add_value_constraint(var1=0, value1=1, var2=1, value2=1) ds.sample_valid_x(100, random_state=42) x_cartesian = np.array(list(itertools.product([0, 1, 2], [0, 1, 2]))) ds.correct_get_acting(x_cartesian) + x_cartesian2, _ = ds.correct_get_acting(x_cartesian) + np.testing.assert_array_equal( + np.array( + [[0, 0], [0, 1], [0, 2], [1, 0], [2, 0], [1, 2], [2, 0], [2, 1], [2, 2]] + ), + x_cartesian2, + ) + + @unittest.skipIf( + not HAS_CONFIG_SPACE, "Hierarchy ConfigSpace dependency not installed" + ) + def test_restrictive_value_constraint_categorical(self): + ds = DesignSpace( + [ + CategoricalVariable(["a", "b", "c"]), + CategoricalVariable(["a", "b", "c"]), + ] + ) + assert list(ds._cs.values())[0].default_value == "a" + + ds.add_value_constraint(var1=0, value1="b", var2=1, value2="b") + ds.sample_valid_x(100, random_state=42) + + x_cartesian = np.array(list(itertools.product([0, 1, 2], [0, 1, 2]))) + ds.correct_get_acting(x_cartesian) + x_cartesian2, _ = ds.correct_get_acting(x_cartesian) + np.testing.assert_array_equal( + np.array( + [[0, 0], [0, 1], [0, 2], [1, 0], [0, 1], [1, 2], [2, 0], [2, 1], [2, 2]] + ), + x_cartesian2, + ) if __name__ == "__main__":