Use all trials in TPESampler even when multivariate=True

optuna/samplers/_tpe/sampler.py

@@ -386,14 +386,14 @@ def _sample_relative(
         values, scores, violations = _get_observation_pairs(
             study,
             param_names,
-            self._multivariate,
             self._constant_liar,
             self._constraints_func is not None,
         )
 
         # If the number of samples is insufficient, we run random trial.
         n = len(scores)
-        if n < self._n_startup_trials:
+        n_sampled = sum(v is not None for v in list(values.values())[0])
+        if n_sampled < self._n_startup_trials:
             return {}
 
         # We divide data into below and above.
@@ -436,16 +436,16 @@ def sample_independent(
         values, scores, violations = _get_observation_pairs(
             study,
             [param_name],
-            self._multivariate,
             self._constant_liar,
             self._constraints_func is not None,
         )
 
+        n_sampled = sum(v is not None for v in values[param_name])
         n = len(scores)
 
-        self._log_independent_sampling(n, trial, param_name)
+        self._log_independent_sampling(n_sampled, trial, param_name)
 
-        if n < self._n_startup_trials:
+        if n_sampled < self._n_startup_trials:
             return self._random_sampler.sample_independent(
                 study, trial, param_name, param_distribution
             )
@@ -580,7 +580,6 @@ def _calculate_nondomination_rank(loss_vals: np.ndarray) -> np.ndarray:
 def _get_observation_pairs(
     study: Study,
     param_names: List[str],
-    multivariate: bool,
     constant_liar: bool = False,  # TODO(hvy): Remove default value and fix unit tests.
     constraints_enabled: bool = False,
 ) -> Tuple[
@@ -610,9 +609,6 @@ def _get_observation_pairs(
     trial is feasible if and only if its violation score is 0.
     """
 
-    if len(param_names) > 1:
-        assert multivariate
-
     signs = []
     for d in study.directions:
         if d == StudyDirection.MINIMIZE:
@@ -630,12 +626,6 @@ def _get_observation_pairs(
     values: Dict[str, List[Optional[float]]] = {param_name: [] for param_name in param_names}
     violations: Optional[List[float]] = [] if constraints_enabled else None
     for trial in study.get_trials(deepcopy=False, states=states):
-        # If ``multivariate`` = True and ``group`` = True, we ignore the trials that are not
-        # included in each subspace.
-        # If ``multivariate`` = False, we skip the check.
-        if multivariate and any([param_name not in trial.params for param_name in param_names]):
-            continue
-
         # We extract score from the trial.
         if trial.state is TrialState.COMPLETE:
             if trial.values is None:

tests/samplers_tests/tpe_tests/test_multi_objective_sampler.py

@@ -338,29 +338,25 @@ def objective(trial: optuna.trial.Trial) -> Tuple[float, float]:
         )
     )
 
-    assert _tpe.sampler._get_observation_pairs(study, ["x"], multivariate, constant_liar) == (
+    assert _tpe.sampler._get_observation_pairs(study, ["x"], constant_liar) == (
         {"x": [int_value, int_value]},
         [(-float("inf"), [objective_value, -objective_value]) for _ in range(2)],
         None,
     )
-    assert _tpe.sampler._get_observation_pairs(study, ["y"], multivariate, constant_liar) == (
+    assert _tpe.sampler._get_observation_pairs(study, ["y"], constant_liar) == (
         {"y": [0, 0]},
         [(-float("inf"), [objective_value, -objective_value]) for _ in range(2)],
         None,
     )
-    assert _tpe.sampler._get_observation_pairs(study, ["x", "y"], True, constant_liar) == (
+    assert _tpe.sampler._get_observation_pairs(study, ["x", "y"], constant_liar) == (
         {"x": [int_value, int_value], "y": [0, 0]},
         [(-float("inf"), [objective_value, -objective_value]) for _ in range(2)],
         None,
     )
-    assert _tpe.sampler._get_observation_pairs(study, ["z"], multivariate, constant_liar) == (
-        (
-            {"z": [None, None]},
-            [(-float("inf"), [objective_value, -objective_value]) for _ in range(2)],
-            None,
-        )
-        if not multivariate
-        else ({"z": []}, [], None)
+    assert _tpe.sampler._get_observation_pairs(study, ["z"], constant_liar) == (
+        {"z": [None, None]},
+        [(-float("inf"), [objective_value, -objective_value]) for _ in range(2)],
+        None,
     )
 
 
@@ -376,26 +372,16 @@ def objective(trial: optuna.trial.Trial) -> Tuple[float, float]:
     study.optimize(objective, n_trials=5)
 
     violations = [max(0, constraint_value) for _ in range(5)]
-    assert _tpe.sampler._get_observation_pairs(study, ["x"], False, constraints_enabled=True) == (
+    assert _tpe.sampler._get_observation_pairs(study, ["x"], constraints_enabled=True) == (
         {"x": [5.0, 5.0, 5.0, 5.0, 5.0]},
         [(-float("inf"), [5.0, -5.0]) for _ in range(5)],
         violations,
     )
-    assert _tpe.sampler._get_observation_pairs(study, ["y"], False, constraints_enabled=True) == (
+    assert _tpe.sampler._get_observation_pairs(study, ["y"], constraints_enabled=True) == (
         {"y": [None, None, None, None, None]},
         [(-float("inf"), [5.0, -5.0]) for _ in range(5)],
         violations,
     )
-    assert _tpe.sampler._get_observation_pairs(study, ["x"], True, constraints_enabled=True) == (
-        {"x": [5.0, 5.0, 5.0, 5.0, 5.0]},
-        [(-float("inf"), [5.0, -5.0]) for _ in range(5)],
-        violations,
-    )
-    assert _tpe.sampler._get_observation_pairs(study, ["y"], True, constraints_enabled=True) == (
-        {"y": []},
-        [],
-        [],
-    )
 
 
 def test_multi_objective_split_observation_pairs() -> None:

tests/samplers_tests/tpe_tests/test_sampler.py

@@ -764,103 +764,28 @@ def objective(trial: Trial) -> float:
         (float("inf"), [sign * 0.0]),  # PRUNED (without intermediate values)
     ]
     assert _tpe.sampler._get_observation_pairs(
-        study, ["x"], False, constraints_enabled=constraints_enabled
+        study, ["x"], constraints_enabled=constraints_enabled
     ) == (
         {"x": [5.0, 5.0, 5.0, 5.0]},
         scores,
         expected_violations,
     )
     assert _tpe.sampler._get_observation_pairs(
-        study, ["y"], False, constraints_enabled=constraints_enabled
+        study, ["y"], constraints_enabled=constraints_enabled
     ) == (
         {"y": [None, None, None, None]},
         scores,
         expected_violations,
     )
     assert _tpe.sampler._get_observation_pairs(
-        study, ["z"], False, constraints_enabled=constraints_enabled
-    ) == (
-        {"z": [0, 0, 0, 0]},  # The internal representation of 'None' for z is 0
-        scores,
-        expected_violations,
-    )
-    assert _tpe.sampler._get_observation_pairs(
-        study, ["x"], True, constraints_enabled=constraints_enabled
-    ) == (
-        {"x": [5.0, 5.0, 5.0, 5.0]},
-        scores,
-        expected_violations,
-    )
-    assert _tpe.sampler._get_observation_pairs(
-        study, ["y"], True, constraints_enabled=constraints_enabled
-    ) == (
-        {"y": []},
-        [],
-        [] if constraints_enabled else expected_violations,
-    )
-    assert _tpe.sampler._get_observation_pairs(
-        study, ["z"], True, constraints_enabled=constraints_enabled
+        study, ["z"], constraints_enabled=constraints_enabled
     ) == (
         {"z": [0, 0, 0, 0]},  # The internal representation of 'None' for z is 0
         scores,
         expected_violations,
     )
 
 
-@pytest.mark.parametrize("direction", ["minimize", "maximize"])
-@pytest.mark.parametrize(
-    "constraints_enabled, constraints_func, expected_violations",
-    [
-        (False, None, None),
-        (True, lambda trial: [(-1, -1), (0, -1), (1, -1), (2, -1)][trial.number], [0, 0, 1, 2]),
-    ],
-)
-def test_get_observation_pairs_multi(
-    direction: str,
-    constraints_enabled: bool,
-    constraints_func: Optional[Callable[[optuna.trial.FrozenTrial], Sequence[float]]],
-    expected_violations: List[float],
-) -> None:
-    def objective(trial: Trial) -> float:
-
-        x = trial.suggest_int("x", 5, 5)
-        y = trial.suggest_int("y", 6, 6)
-        if trial.number == 0:
-            return x + y
-        elif trial.number == 1:
-            trial.report(1, 4)
-            trial.report(2, 7)
-            raise TrialPruned()
-        elif trial.number == 2:
-            trial.report(float("nan"), 3)
-            raise TrialPruned()
-        elif trial.number == 3:
-            raise TrialPruned()
-        else:
-            raise RuntimeError()
-
-    sampler = TPESampler(constraints_func=constraints_func)
-    study = optuna.create_study(direction=direction, sampler=sampler)
-    study.optimize(objective, n_trials=5, catch=(RuntimeError,))
-
-    sign = 1 if direction == "minimize" else -1
-    assert _tpe.sampler._get_observation_pairs(
-        study, ["x", "y"], True, constraints_enabled=constraints_enabled
-    ) == (
-        {"x": [5.0, 5.0, 5.0, 5.0], "y": [6.0, 6.0, 6.0, 6.0]},
-        [
-            (-float("inf"), [sign * 11.0]),  # COMPLETE
-            (-7, [sign * 2]),  # PRUNED (with intermediate values)
-            (
-                -3,
-                [float("inf")],
-            ),  # PRUNED (with a NaN intermediate value; it's treated as infinity)
-            (float("inf"), [sign * 0.0]),  # PRUNED (without intermediate values)
-        ],
-        expected_violations,
-    )
-
-
 def test_split_observation_pairs() -> None:
     indices_below, indices_above = _tpe.sampler._split_observation_pairs(
         [
@@ -1066,8 +991,7 @@ def test_group_experimental_warning() -> None:
 
 
 @pytest.mark.parametrize("direction", ["minimize", "maximize"])
-@pytest.mark.parametrize("multivariate", [True, False])
-def test_constant_liar_observation_pairs(direction: str, multivariate: bool) -> None:
+def test_constant_liar_observation_pairs(direction: str) -> None:
     with warnings.catch_warnings():
         warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning)
         sampler = TPESampler(constant_liar=True)
@@ -1085,14 +1009,13 @@ def test_constant_liar_observation_pairs(direction: str, multivariate: bool) ->
     # and `-float("inf")` during maximization.
     expected_values = [(-float("inf"), [float("inf") * (-1 if direction == "maximize" else 1)])]
 
-    assert _tpe.sampler._get_observation_pairs(
-        study, ["x"], multivariate, constant_liar=False
-    ) == (
+    assert _tpe.sampler._get_observation_pairs(study, ["x"], constant_liar=False) == (
         {"x": []},
         [],
         None,
     )
-    assert _tpe.sampler._get_observation_pairs(study, ["x"], multivariate, constant_liar=True) == (
+
+    assert _tpe.sampler._get_observation_pairs(study, ["x"], constant_liar=True) == (
         {"x": [2]},
         expected_values,
         None,