From c267fe69000adbdeff78b48f0c6b093430c3be18 Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Thu, 14 Oct 2021 13:06:28 -0400
Subject: [PATCH 01/34] Convert Space to sample trials instead of points

---
 src/orion/algo/space.py            | 48 +++++++++++----------------
 tests/unittests/algo/test_space.py | 53 ++++++++++++++++--------------
 2 files changed, 48 insertions(+), 53 deletions(-)

diff --git a/src/orion/algo/space.py b/src/orion/algo/space.py
index 13e1f4895..c418c4c7c 100644
--- a/src/orion/algo/space.py
+++ b/src/orion/algo/space.py
@@ -35,7 +35,7 @@
 import numpy
 from scipy.stats import distributions
 
-from orion.core.utils import float_to_digits_list
+from orion.core.utils import float_to_digits_list, format_trials
 from orion.core.utils.points import flatten_dims, regroup_dims
 
 logger = logging.getLogger(__name__)
@@ -966,16 +966,14 @@ def sample(self, n_samples=1, seed=None):
 
         Returns
         -------
-        points : list of tuples of array-likes
-           Each element is a separate sample of this space, a list containing
-           values associated with the corresponding dimension. Values are in the
-           same order as the contained dimensions. Their shape is determined
-           by ``dimension.shape``.
+        trials: list of `orion.core.worker.trial.Trial`
+           Each element is a separate sample of this space, a trial containing
+           values associated with the corresponding dimension.
 
         """
         rng = check_random_state(seed)
         samples = [dim.sample(n_samples, rng) for dim in self.values()]
-        return list(zip(*samples))
+        return [format_trials.tuple_to_trial(point, self) for point in zip(*samples)]
 
     def interval(self, alpha=1.0):
         """Return a list with the intervals for each contained dimension."""
@@ -1018,36 +1016,28 @@ def __setitem__(self, key, value):
             )
         super(Space, self).__setitem__(key, value)
 
-    def __contains__(self, value):
-        """Check whether `value` is within the bounds of the space.
+    def __contains__(self, key_or_trial):
+        """Check whether `trial` is within the bounds of the space.
         Or check if a name for a dimension is registered in this space.
 
         Parameters
         ----------
-        value: list
-            List of values associated with the dimensions contained or a string indicating a
-            dimension's name.
-
+        key_or_trial: str or `orion.core.worker.trial.Trial`
+            If str, test if the string is a dimension part of the search space.
+            If a Trial, test if trial's hyperparameters fit the current search space.
         """
-        if isinstance(value, str):
-            return super(Space, self).__contains__(value)
-
-        try:
-            len(value)
-        except TypeError as exc:
-            raise TypeError(
-                "Can check only for dimension names or "
-                "for tuples with parameter values."
-            ) from exc
-
-        if not self:
-            return False
+        if isinstance(key_or_trial, str):
+            return super(Space, self).__contains__(key_or_trial)
 
-        for component, dim in zip(value, self.values()):
-            if component not in dim:
+        trial = key_or_trial
+        keys = set(trial.params.keys())
+        for dim_name, dim in self.items():
+            if dim_name not in keys or trial.params[dim_name] not in dim:
                 return False
 
-        return True
+            keys.remove(dim_name)
+
+        return len(keys) == 0
 
     def __repr__(self):
         """Represent as a string the space and the dimensions it contains."""
diff --git a/tests/unittests/algo/test_space.py b/tests/unittests/algo/test_space.py
index b1edb48dc..4b2234238 100644
--- a/tests/unittests/algo/test_space.py
+++ b/tests/unittests/algo/test_space.py
@@ -19,6 +19,8 @@
     Space,
     check_random_state,
 )
+from orion.core.worker.trial import Trial
+from orion.core.utils import format_trials
 
 
 class TestCheckRandomState:
@@ -739,8 +741,10 @@ def test_register_and_contain(self):
         """Register bunch of dimensions, check if points/name are in space."""
         space = Space()
 
+        trial = Trial(params=[{"name": "no", "value": 0, "type": "integer"}])
+
         assert "yolo" not in space
-        assert (("asdfa", 2), 0, 3.5) not in space
+        assert trial not in space
 
         categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, 4: 0.4}
         dim = Categorical("yolo", categories, shape=2)
@@ -754,17 +758,12 @@ def test_register_and_contain(self):
         assert "yolo2" in space
         assert "yolo3" in space
 
-        assert (("asdfa", 2), 0, 3.5) in space
-        assert (("asdfa", 2), 7, 3.5) not in space
-
-    def test_bad_contain(self):
-        """Checking with no iterables does no good."""
-        space = Space()
-        with pytest.raises(TypeError):
-            5 in space
+        assert format_trials.tuple_to_trial((("asdfa", 2), 0, 3.5), space) in space
+        assert format_trials.tuple_to_trial((("asdfa", 2), 7, 3.5), space) not in space
 
-    def test_sample(self, seed):
+    def test_sample(self):
         """Check whether sampling works correctly."""
+        seed = 5
         space = Space()
         probs = (0.1, 0.2, 0.3, 0.4)
         categories = ("asdfa", 2, 3, 4)
@@ -776,29 +775,35 @@ def test_sample(self, seed):
         space.register(dim3)
 
         point = space.sample(seed=seed)
+        rng = check_random_state(seed)
         test_point = [
-            (dim1.sample()[0], dim2.sample()[0], dim3.sample()[0]),
+            dict(
+                yolo=dim1.sample(seed=rng)[0],
+                yolo2=dim2.sample(seed=rng)[0],
+                yolo3=dim3.sample(seed=rng)[0],
+            )
         ]
         assert len(point) == len(test_point) == 1
-        assert len(point[0]) == len(test_point[0]) == 3
-        assert np.all(point[0][0] == test_point[0][0])
-        assert point[0][1] == test_point[0][1]
-        assert point[0][2] == test_point[0][2]
+        assert len(point[0].params) == len(test_point[0]) == 3
+        assert np.all(point[0].params["yolo"] == test_point[0]["yolo"])
+        assert point[0].params["yolo2"] == test_point[0]["yolo2"]
+        assert point[0].params["yolo3"] == test_point[0]["yolo3"]
 
         points = space.sample(2, seed=seed)
-        points1 = dim1.sample(2)
-        points2 = dim2.sample(2)
-        points3 = dim3.sample(2)
+        rng = check_random_state(seed)
+        points1 = dim1.sample(2, seed=rng)
+        points2 = dim2.sample(2, seed=rng)
+        points3 = dim3.sample(2, seed=rng)
         test_points = [
-            (points1[0], points2[0], points3[0]),
-            (points1[1], points2[1], points3[1]),
+            dict(yolo=points1[0], yolo2=points2[0], yolo3=points3[0]),
+            dict(yolo=points1[1], yolo2=points2[1], yolo3=points3[1]),
         ]
         assert len(points) == len(test_points) == 2
         for i in range(2):
-            assert len(points[i]) == len(test_points[i]) == 3
-            assert np.all(points[i][0] == test_points[i][0])
-            assert points[i][1] == test_points[i][1]
-            assert points[i][2] == test_points[i][2]
+            assert len(points[i].params) == len(test_points[i]) == 3
+            assert np.all(points[i].params["yolo"] == test_points[i]["yolo"])
+            assert points[i].params["yolo2"] == test_points[i]["yolo2"]
+            assert points[i].params["yolo3"] == test_points[i]["yolo3"]
 
     def test_interval(self):
         """Check whether interval is cool."""

From 567a392a458cde8dd9713b09202986339db304df Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Fri, 15 Oct 2021 15:33:33 -0400
Subject: [PATCH 02/34] Adjust Transformers space to using Trials

Why:

The algorithms will be working with trial objects instead of tuple of
values. This means the space needs to sample trials, and space
transformations should be applied on trials as well instead of tuples of
values.

How:

For simplicity, only interface of the space classes (TransformedSpace
and ReshapedSpace) will be working with trials. The transformations per
dimension will be applied using tuple of values so that, in particular,
reshaping operations remain straightforward.

To facilitate debugging, transformed trials are wrapped so that the
original trial can still be accessible. This will prove handy in
algorithms if we need access to original trial objects, and also because
the ID of the transformed trial should always be based on the original
parameters (otherwise the ID gets incoherent with the database).
---
 src/orion/core/worker/transformer.py     | 133 ++++++++++++-----
 tests/unittests/core/test_transformer.py | 180 ++++++++++++++++++-----
 2 files changed, 242 insertions(+), 71 deletions(-)

diff --git a/src/orion/core/worker/transformer.py b/src/orion/core/worker/transformer.py
index 1e14f5504..2a1555937 100644
--- a/src/orion/core/worker/transformer.py
+++ b/src/orion/core/worker/transformer.py
@@ -7,6 +7,7 @@
 Provide functions and classes to build a Space which an algorithm can operate on.
 
 """
+import copy
 import functools
 import itertools
 from abc import ABCMeta, abstractmethod
@@ -14,6 +15,8 @@
 import numpy
 
 from orion.algo.space import Categorical, Dimension, Fidelity, Integer, Real, Space
+from orion.core.utils import format_trials
+from orion.core.utils.flatten import unflatten
 
 NON_LINEAR = ["loguniform", "reciprocal"]
 
@@ -564,7 +567,7 @@ def first(self):
 
     def transform(self, point):
         """Only return one element of the group"""
-        return point[self.index]
+        return numpy.array(point)[self.index]
 
     def reverse(self, transformed_point, index=None):
         """Only return packend point if view of first element, otherwise drop."""
@@ -770,22 +773,32 @@ def __init__(self, space, *args, **kwargs):
         super(TransformedSpace, self).__init__(*args, **kwargs)
         self._original_space = space
 
-    def transform(self, point):
+    def transform(self, trial):
         """Transform a point that was in the original space to be in this one."""
-        return tuple([dim.transform(point[i]) for i, dim in enumerate(self.values())])
+        transformed_point = tuple(
+            dim.transform(trial.params[name]) for name, dim in self.items()
+        )
 
-    def reverse(self, transformed_point):
+        return create_transformed_trial(trial, transformed_point, self)
+
+    def reverse(self, transformed_trial):
         """Reverses transformation so that a point from this `TransformedSpace`
         to be in the original one.
         """
-        return tuple(
-            [dim.reverse(transformed_point[i]) for i, dim in enumerate(self.values())]
+        reversed_point = tuple(
+            dim.reverse(transformed_trial.params[name]) for name, dim in self.items()
+        )
+
+        return create_restored_trial(
+            transformed_trial,
+            reversed_point,
+            self,
         )
 
     def sample(self, n_samples=1, seed=None):
         """Sample from the original dimension and forward transform them."""
-        points = self._original_space.sample(n_samples=n_samples, seed=seed)
-        return [self.transform(point) for point in points]
+        trials = self._original_space.sample(n_samples=n_samples, seed=seed)
+        return [self.transform(trial) for trial in trials]
 
 
 class ReshapedSpace(Space):
@@ -809,62 +822,106 @@ def original(self):
         """Original space without reshape or transformations"""
         return self._original_space
 
-    def transform(self, point):
+    def transform(self, trial):
         """Transform a point that was in the original space to be in this one."""
-        return self.reshape(self.original.transform(point))
+        return self.reshape(self.original.transform(trial))
 
-    def reverse(self, transformed_point):
+    def reverse(self, transformed_trial):
         """Reverses transformation so that a point from this `ReshapedSpace` to be in the original
         one.
         """
-        return self.original.reverse(self.restore_shape(transformed_point))
+        return self.original.reverse(self.restore_shape(transformed_trial))
 
-    def reshape(self, point):
+    def reshape(self, trial):
         """Reshape the point"""
-        return tuple([dim.transform(point) for dim in self.values()])
+        point = format_trials.trial_to_tuple(trial, self._original_space)
+        reshaped_point = tuple([dim.transform(point) for dim in self.values()])
+        return create_transformed_trial(trial, reshaped_point, self)
 
-    def restore_shape(self, transformed_point):
-        """Restore shape"""
+    def restore_shape(self, transformed_trial):
+        """Restore shape."""
+        transformed_point = format_trials.trial_to_tuple(transformed_trial, self)
         point = []
         for index, dim in enumerate(self.values()):
             if dim.first:
                 point.append(dim.reverse(transformed_point, index))
 
-        return point
+        return create_restored_trial(transformed_trial, point, self._original_space)
 
     def sample(self, n_samples=1, seed=None):
         """Sample from the original dimension and forward transform them."""
-        points = self.original.sample(n_samples=n_samples, seed=seed)
-        return [self.reshape(point) for point in points]
+        trials = self.original.sample(n_samples=n_samples, seed=seed)
+        return [self.reshape(trial) for trial in trials]
 
-    def __contains__(self, value):
-        """Check whether `value` is within the bounds of the space.
+    def __contains__(self, key_or_trial):
+        """Check whether `trial` is within the bounds of the space.
         Or check if a name for a dimension is registered in this space.
 
         Parameters
         ----------
-        value: list
-            List of values associated with the dimensions contained or a string indicating a
-            dimension's name.
+        key_or_trial: str or `orion.core.worker.trial.Trial`
+            If str, test if the string is a dimension part of the search space.
+            If a Trial, test if trial's hyperparameters fit the current search space.
 
         """
-        if isinstance(value, str):
-            return super(ReshapedSpace, self).__contains__(value)
+        if isinstance(key_or_trial, str):
+            return super(ReshapedSpace, self).__contains__(key_or_trial)
 
-        try:
-            len(value)
-        except TypeError as exc:
-            raise TypeError(
-                "Can check only for dimension names or "
-                "for tuples with parameter values."
-            ) from exc
-
-        if not self:
-            return False
-
-        return self.restore_shape(value) in self.original
+        return self.restore_shape(key_or_trial) in self.original
 
     @property
     def cardinality(self):
         """Reshape does not affect cardinality"""
         return self.original.cardinality
+
+
+class TransformedTrial:
+    """A trial with transformed params
+
+    All other attributes are kept as-is.
+
+    Original params are accessible through ``transformed_trial.trial.params``.
+    """
+
+    __slots__ = ["trial", "_params"]
+
+    def __init__(self, trial, params):
+        self.trial = trial
+        self._params = params
+
+    @property
+    def params(self):
+        """Parameters of the trial"""
+        return unflatten({param.name: param.value for param in self._params})
+
+    def to_dict(self):
+        trial_dictionary = self.trial.to_dict()
+        trial_dictionary["params"] = list(map(lambda x: x.to_dict(), self._params))
+
+        return trial_dictionary
+
+    def __getattr__(self, name):
+        return getattr(self.trial, name)
+
+    def __setattr__(self, name, value):
+        if name in self.__slots__:
+            return super(TransformedTrial, self).__setattr__(name, value)
+
+        return setattr(self.trial, name, value)
+
+
+def create_transformed_trial(trial, transformed_point, space):
+    """Convert point into Trial.Param objects and return a TransformedTrial"""
+    return TransformedTrial(
+        trial, format_trials.tuple_to_trial(transformed_point, space)._params
+    )
+
+
+def create_restored_trial(trial, point, space):
+    """Convert params in Param objects and update trial"""
+    if isinstance(trial, TransformedTrial):
+        return create_restored_trial(trial.trial, point, space)
+
+    new_trial = copy.copy(trial)
+    new_trial._params = format_trials.tuple_to_trial(point, space)._params
+    return new_trial
diff --git a/tests/unittests/core/test_transformer.py b/tests/unittests/core/test_transformer.py
index 0ae52d9fa..e44f0550a 100644
--- a/tests/unittests/core/test_transformer.py
+++ b/tests/unittests/core/test_transformer.py
@@ -9,6 +9,7 @@
 import pytest
 
 from orion.algo.space import Categorical, Dimension, Integer, Real, Space
+from orion.core.utils import format_trials
 from orion.core.worker.transformer import (
     Compose,
     Enumerate,
@@ -22,8 +23,11 @@
     Reverse,
     TransformedDimension,
     TransformedSpace,
+    TransformedTrial,
     View,
     build_required_space,
+    create_restored_trial,
+    create_transformed_trial,
 )
 
 
@@ -1038,20 +1042,24 @@ class TestReshapedSpace(object):
 
     def test_reverse(self, space, tspace, rspace, seed):
         """Check method `reverse`."""
-        ryo = (
-            numpy.zeros(tspace["yolo0"].shape).reshape(-1).tolist()
-            + numpy.zeros(tspace["yolo2"].shape).reshape(-1).tolist()
-            + [10]
+        ryo = format_trials.tuple_to_trial(
+            tuple(
+                numpy.zeros(tspace["yolo0"].shape).reshape(-1).tolist()
+                + numpy.zeros(tspace["yolo2"].shape).reshape(-1).tolist()
+                + [10]
+            ),
+            rspace,
         )
         yo = rspace.reverse(ryo)
         assert yo in space
 
     def test_contains(self, tspace, rspace, seed):
         """Check method `transform`."""
-        ryo = (
+        ryo = format_trials.tuple_to_trial(
             numpy.zeros(tspace["yolo0"].shape).reshape(-1).tolist()
             + numpy.zeros(tspace["yolo2"].shape).reshape(-1).tolist()
-            + [10]
+            + [10],
+            rspace,
         )
 
         assert ryo in rspace
@@ -1085,14 +1093,24 @@ def test_interval(self, rspace):
             assert interval[i] == (0, 1)
         assert interval[-1] == (3, 10)
 
-    def test_reshape(self, rspace):
+    def test_reshape(self, space, rspace):
         """Verify that the dimension are reshaped properly, forward and backward"""
-        point = [numpy.arange(6).reshape(3, 2), "3", 10]
-        rpoint = point[0].reshape(-1).tolist() + [0.0, 0.0, 1.0, 0.0] + [10]
-        assert rspace.transform(point) == tuple(rpoint)
-        numpy.testing.assert_equal(rspace.reverse(rpoint)[0], point[0])
-        assert rspace.reverse(rpoint)[1] == point[1]
-        assert rspace.reverse(rpoint)[2] == point[2]
+        trial = format_trials.tuple_to_trial(
+            (numpy.arange(6).reshape(3, 2).tolist(), "3", 10), space
+        )
+
+        rtrial = format_trials.tuple_to_trial(
+            numpy.array(trial.params["yolo0"]).reshape(-1).tolist()
+            + [0.0, 0.0, 1.0, 0.0]
+            + [10],
+            rspace,
+        )
+        assert rspace.transform(trial).params == rtrial.params
+        numpy.testing.assert_equal(
+            rspace.reverse(rtrial).params["yolo0"], trial.params["yolo0"]
+        )
+        assert rspace.reverse(rtrial).params["yolo2"] == trial.params["yolo2"]
+        assert rspace.reverse(rtrial).params["yolo3"] == trial.params["yolo3"]
 
     def test_cardinality(self, dim2):
         """Check cardinality of reshaped space"""
@@ -1333,40 +1351,136 @@ def test_precision_with_linear(space, logdim, logintdim):
     space["yolo5"].precision = 5
 
     # Create a point
-    point = list(space.sample(1)[0])
+    trial = space.sample(1)[0]
     real_index = list(space.keys()).index("yolo0")
     logreal_index = list(space.keys()).index("yolo4")
     logint_index = list(space.keys()).index("yolo5")
-    point[real_index] = 0.133333
-    point[logreal_index] = 0.1222222
-    point[logint_index] = 2
+    trial._params[real_index].value = 0.133333
+    trial._params[logreal_index].value = 0.1222222
+    trial._params[logint_index].value = 2
 
     # Check first without linearization
     tspace = build_required_space(space, type_requirement="numerical")
     # Check that transform is fine
-    tpoint = tspace.transform(point)
-    assert tpoint[real_index] == 0.133
-    assert tpoint[logreal_index] == 0.1222
-    assert tpoint[logint_index] == 2
+    ttrial = tspace.transform(trial)
+    assert ttrial.params["yolo0"] == 0.133
+    assert ttrial.params["yolo4"] == 0.1222
+    assert ttrial.params["yolo5"] == 2
 
     # Check that reserve does not break precision
-    rpoint = tspace.reverse(tpoint)
-    assert rpoint[real_index] == 0.133
-    assert rpoint[logreal_index] == 0.1222
-    assert rpoint[logint_index] == 2
+    rtrial = tspace.reverse(ttrial)
+    assert rtrial.params["yolo0"] == 0.133
+    assert rtrial.params["yolo4"] == 0.1222
+    assert rtrial.params["yolo5"] == 2
 
     # Check with linearization
     tspace = build_required_space(
         space, dist_requirement="linear", type_requirement="real"
     )
     # Check that transform is fine
-    tpoint = tspace.transform(point)
-    assert tpoint[real_index] == 0.133
-    assert tpoint[logreal_index] == numpy.log(0.1222)
-    assert tpoint[logint_index] == numpy.log(2)
+    ttrial = tspace.transform(trial)
+    assert ttrial.params["yolo0"] == 0.133
+    assert ttrial.params["yolo4"] == numpy.log(0.1222)
+    assert ttrial.params["yolo5"] == numpy.log(2)
 
     # Check that reserve does not break precision
-    rpoint = tspace.reverse(tpoint)
-    assert rpoint[real_index] == 0.133
-    assert rpoint[logreal_index] == 0.1222
-    assert rpoint[logint_index] == 2
+    rtrial = tspace.reverse(ttrial)
+    assert rtrial.params["yolo0"] == 0.133
+    assert rtrial.params["yolo4"] == 0.1222
+    assert rtrial.params["yolo5"] == 2
+
+
+class TestTransformedTrial:
+    def test_params_are_transformed(self, space, tspace):
+        trial = space.sample()[0]
+        ttrial = tspace.transform(trial)
+        assert isinstance(ttrial, TransformedTrial)
+        assert ttrial.params != trial.params
+        assert ttrial not in space
+        assert ttrial in tspace
+
+    def test_trial_attributes_conserved(self, space, tspace):
+        working_dir = "/new/working/dir"
+        status = "interrupted"
+        trial = space.sample()[0]
+        assert trial.working_dir != working_dir
+        trial.working_dir = working_dir
+        assert trial.status != status
+        trial.status = status
+        ttrial = tspace.transform(trial)
+        assert isinstance(ttrial, TransformedTrial)
+        assert ttrial.working_dir == working_dir
+        assert ttrial.status == status
+
+    def test_setters_accessible(self, space, tspace):
+        working_dir = "/new/working/dir"
+        status = "interrupted"
+        trial = space.sample()[0]
+        ttrial = tspace.transform(trial)
+
+        assert trial.working_dir != working_dir
+        assert trial.status != status
+
+        ttrial.working_dir = working_dir
+        ttrial.status = status
+
+        assert ttrial.working_dir == trial.working_dir == working_dir
+        assert ttrial.status == trial.status == status
+
+    def test_to_dict_is_transformed(self, space, tspace):
+        trial = space.sample()[0]
+        ttrial = tspace.transform(trial)
+
+        original_dict = trial.to_dict()
+        transformed_dict = ttrial.to_dict()
+
+        assert original_dict != transformed_dict
+
+        original_dict.pop("params")
+        transformed_dict.pop("params")
+
+        assert original_dict == transformed_dict
+
+        assert "_id" in original_dict
+
+
+def test_create_transformed_trial(space, tspace):
+    trial = space.sample()[0]
+    ttrial = tspace.transform(trial)
+
+    # Test that returns a TransformedTrial
+    transformed_trial = create_transformed_trial(
+        trial, format_trials.trial_to_tuple(ttrial, tspace), tspace
+    )
+    assert isinstance(transformed_trial, TransformedTrial)
+
+    # Test that point is converted properly
+    assert transformed_trial not in space
+    assert transformed_trial in tspace
+
+
+def test_create_restored_trial(space, rspace):
+    working_dir = "/new/working/dir"
+    status = "interrupted"
+
+    rtrial = rspace.sample()[0]
+    # Sampling a new point in original space instead of using reserve()
+    trial = space.sample()[0]
+    point = format_trials.trial_to_tuple(trial, space)
+
+    rtrial.working_dir = working_dir
+    rtrial.status = status
+
+    restored_trial = create_restored_trial(rtrial, point, space)
+
+    # Test that attributes are conserved
+    assert restored_trial.working_dir == working_dir
+    assert restored_trial.status == status
+
+    # Test params are updated
+    assert restored_trial.params != rtrial.params
+    assert restored_trial.params == trial.params
+
+    # Test that id is based on current params
+    assert restored_trial.id != rtrial.id
+    assert restored_trial.id == trial.id

From 8fdd52fddc3e8457000e726bc45ba9b9af9e59b2 Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Tue, 19 Oct 2021 16:08:24 -0400
Subject: [PATCH 03/34] Adapt Primary Algo to use trials instead of points

---
 src/orion/algo/base.py                    |  71 ++++++---------
 src/orion/core/utils/backward.py          |  10 +++
 src/orion/core/utils/format_trials.py     |  31 +++++--
 src/orion/core/worker/primary_algo.py     | 103 +++++++++++++---------
 tests/conftest.py                         |  20 ++---
 tests/unittests/core/conftest.py          |   9 +-
 tests/unittests/core/test_primary_algo.py |  44 +++++----
 7 files changed, 159 insertions(+), 129 deletions(-)

diff --git a/src/orion/algo/base.py b/src/orion/algo/base.py
index df1515ea8..e46fd612b 100644
--- a/src/orion/algo/base.py
+++ b/src/orion/algo/base.py
@@ -143,29 +143,28 @@ def set_state(self, state_dict):
         """
         self._trials_info = state_dict.get("_trials_info")
 
-    def format_point(self, point):
-        """Format point based on space transformations
+    def format_trial(self, trial):
+        """Format trial based on space transformations
 
-        This will apply the reverse transformation on the point and then
+        This will apply the reverse transformation on the trial and then
         transform it again.
 
-        Some transformations are lossy and thus the points suggested by the algorithm could
-        be different when returned to `observe`. Using `format_point` makes it possible
-        for the algorithm to see the final version of the point after back and forth
-        transformations. This way it can recognise the point in `observe` and also
+        Some transformations are lossy and thus the trials suggested by the algorithm could
+        be different when returned to `observe`. Using `format_trial` makes it possible
+        for the algorithm to see the final version of the trial after back and forth
+        transformations. This way it can recognise the trial in `observe` and also
         avoid duplicates that would have gone unnoticed during suggestion.
 
         Parameters
         ----------
-        point : tuples of array-likes
-            Points from a `orion.algo.space.Space`.
+        trial : `orion.core.worker.trial.Trial`
+            Trial from a `orion.algo.space.Space`.
         """
 
-        point = tuple(point)
         if hasattr(self.space, "transform"):
-            point = self.space.transform(self.space.reverse(point))
+            trial = self.space.transform(self.space.reverse(trial))
 
-        return point
+        return trial
 
     def get_id(self, point, ignore_fidelity=False):
         """Compute a unique hash for a point based on params
@@ -180,7 +179,7 @@ def get_id(self, point, ignore_fidelity=False):
         """
         # Apply transforms and reverse to see data as it would come from DB
         # (Some transformations looses some info. ex: Precision transformation)
-        point = list(self.format_point(point))
+        point = list(self.format_trial(point))
 
         if ignore_fidelity:
             non_fidelity_dims = point[0 : self.fidelity_index]
@@ -231,35 +230,21 @@ def suggest(self, num):
         """
         pass
 
-    def observe(self, points, results):
+    def observe(self, trials):
         """Observe the `results` of the evaluation of the `points` in the
         process defined in user's script.
 
         Parameters
         ----------
-        points : list of tuples of array-likes
+        trials: list of tuples of array-likes
            Points from a `orion.algo.space.Space`.
-        results : list of dicts
-           Contains the result of an evaluation; partial information about the
-           black-box function at each point in `params`.
-
-        Result
-        ------
-        objective : numeric
-           Evaluation of this problem's objective function.
-        gradient : 1D array-like, optional
-           Contains values of the derivatives of the `objective` function
-           with respect to `params`.
-        constraint : list of numeric, optional
-           List of constraints expression evaluation which must be greater
-           or equal to zero by the problem's definition.
 
         """
-        for point, result in zip(points, results):
-            if not self.has_observed(point):
-                self.register(point, result)
+        for trial in trials:
+            if not self.has_observed(trial):
+                self.register(trial)
 
-    def register(self, point, result=None):
+    def register(self, trial):
         """Save the point as one suggested or observed by the algorithm
 
         Parameters
@@ -272,7 +257,7 @@ def register(self, point, result=None):
            None is suggested and not yet completed.
 
         """
-        self._trials_info[self.get_id(point)] = (point, result)
+        self._trials_info[trial.hash_name] = (trial, trial.objective)
 
     @property
     def n_suggested(self):
@@ -298,28 +283,27 @@ def has_suggested(self, point):
             True if the point was suggested by the algo, False otherwise.
 
         """
-        return self.get_id(point) in self._trials_info
+        return point.hash_name in self._trials_info
 
-    def has_observed(self, point):
+    def has_observed(self, trial):
         """Whether the algorithm has observed a given point objective.
 
         This only counts observed completed trials.
 
         Parameters
         ----------
-        point : tuples of array-likes
-            Points from a `orion.algo.space.Space`.
+        trial: ``orion.core.worker.trial.Trial``
+           Trial object to retrieve from the database
 
         Returns
         -------
         bool
-            True if the point's objective was observed by the algo, False otherwise.
+            True if the trial's objective was observed by the algo, False otherwise.
 
         """
-
-        trial_id = self.get_id(point)
         return (
-            trial_id in self._trials_info and self._trials_info[trial_id][1] is not None
+            trial.hash_name in self._trials_info
+            and self._trials_info[trial.hash_name][1] is not None
         )
 
     @property
@@ -372,8 +356,7 @@ def judge(self, point, measurements):  # pylint:disable=no-self-use,unused-argum
         """
         return None
 
-    @property
-    def should_suspend(self):
+    def should_suspend(self, trial):
         """Allow algorithm to decide whether a particular running trial is still
         worth to complete its evaluation, based on information provided by the
         `judge` method.
diff --git a/src/orion/core/utils/backward.py b/src/orion/core/utils/backward.py
index 54c8d4a36..a8061bec7 100644
--- a/src/orion/core/utils/backward.py
+++ b/src/orion/core/utils/backward.py
@@ -13,6 +13,7 @@
 
 import orion.core
 from orion.core.io.orion_cmdline_parser import OrionCmdlineParser
+from orion.core.worker.trial import Trial
 
 log = logging.getLogger(__name__)
 
@@ -171,3 +172,12 @@ def port_algo_config(config):
         config = {"of_type": config}
 
     return config
+
+
+def algo_observe(algo, trials, results):
+    for trial, result in zip(trials, results):
+        trial.results.append(
+            Trial.Result(name="objective", type="objective", value=result)
+        )
+
+    algo.observe(trials)
diff --git a/src/orion/core/utils/format_trials.py b/src/orion/core/utils/format_trials.py
index d440f6e16..8ad202d50 100644
--- a/src/orion/core/utils/format_trials.py
+++ b/src/orion/core/utils/format_trials.py
@@ -62,19 +62,32 @@ def dict_to_trial(data, space):
     return Trial(params=params)
 
 
-def tuple_to_trial(data, space):
-    """Create a `orion.core.worker.trial.Trial` object from `data`,
-    filling only parameter information from `data`.
-
-    :param data: A tuple representing a sample point from `space`.
-    :param space: Definition of problem's domain.
-    :type space: `orion.algo.space.Space`
+def tuple_to_trial(data, space, status="new"):
+    """Create a `orion.core.worker.trial.Trial` object from `data`.
+
+    Parameters
+    ----------
+    data: tuple
+        A tuple representing a sample point from `space`.
+    space: `orion.algo.space.Space`
+        Definition of problem's domain.
+    status: str, optional
+        Status of the trial. One of `orion.core.worker.trial.Trial.allowed_stati`.
+
+    Returns
+    -------
+    A trial object `orion.core.worker.trial.Trial`.
     """
-    assert len(data) == len(space)
+    if len(data) != len(space):
+        raise ValueError(
+            f"Data point is not compatible with search space:\ndata: {data}\nspace: {space}"
+        )
+
     params = []
     for i, dim in enumerate(space.values()):
         params.append(dict(name=dim.name, type=dim.type, value=data[i]))
-    return Trial(params=params)
+
+    return Trial(params=params, status=status)
 
 
 def get_trial_results(trial):
diff --git a/src/orion/core/worker/primary_algo.py b/src/orion/core/worker/primary_algo.py
index 126314131..da21c35b9 100644
--- a/src/orion/core/worker/primary_algo.py
+++ b/src/orion/core/worker/primary_algo.py
@@ -56,61 +56,70 @@ def set_state(self, state_dict):
     def suggest(self, num):
         """Suggest a `num` of new sets of parameters.
 
-        :param num: how many sets to be suggested.
+        Parameters
+        ----------
+        num: int
+            Number of points to suggest. The algorithm may return less than the number of points
+            requested.
+
+        Returns
+        -------
+        list of trials or None
+            A list of trials representing values suggested by the algorithm. The algorithm may opt
+            out if it cannot make a good suggestion at the moment (it may be waiting for other
+            trials to complete), in which case it will return None.
+
+        Notes
+        -----
+        New parameters must be compliant with the problem's domain `orion.algo.space.Space`.
 
-        .. note:: New parameters must be compliant with the problem's domain
-           `orion.algo.space.Space`.
         """
-        points = self.algorithm.suggest(num)
+        transformed_trials = self.algorithm.suggest(num)
 
-        if points is None:
+        if transformed_trials is None:
             return None
 
-        for point in points:
-            if point not in self.transformed_space:
+        for trial in transformed_trials:
+            self._verify_trial(trial, space=self.transformed_space)
+            if trial not in self.transformed_space:
                 raise ValueError(
-                    """
-Point is not contained in space:
-Point: {}
-Space: {}""".format(
-                        point, self.transformed_space
-                    )
+                    f"Trial {trial.id} not contained in space:"
+                    f"\nParams: {trial.params}\n: Space{self.transformed_space}"
                 )
 
-        rpoints = []
-        for point in points:
-            rpoints.append(self.transformed_space.reverse(point))
+        trials = []
+        for transformed_trial in transformed_trials:
+            trial = self.transformed_space.reverse(transformed_trial)
+            self._verify_trial(trial, space=self.space)
+            trials.append(trial)
 
-        return rpoints
+        return trials
 
-    def observe(self, points, results):
-        """Observe evaluation `results` corresponding to list of `points` in
-        space.
+    def observe(self, trials):
+        """Observe evaluated trials.
 
         .. seealso:: `orion.algo.base.BaseAlgorithm.observe`
         """
-        assert len(points) == len(results)
-        tpoints = []
-        for point in points:
-            assert point in self.space
-            tpoints.append(self.transformed_space.transform(point))
-        self.algorithm.observe(tpoints, results)
+        transformed_trials = []
+        for trial in trials:
+            self._verify_trial(trial, space=self.space)
+            transformed_trials.append(self.transformed_space.transform(trial))
+        self.algorithm.observe(transformed_trials)
 
-    def has_suggested(self, point):
-        """Whether the algorithm has suggested a given point.
+    def has_suggested(self, trial):
+        """Whether the algorithm has suggested a given trial.
 
         .. seealso:: `orion.algo.base.BaseAlgorithm.has_suggested`
-
         """
-        return self.algorithm.has_suggested(self.transformed_space.transform(point))
+        return self.algorithm.has_suggested(self.transformed_space.transform(trial))
 
-    def has_observed(self, point):
-        """Whether the algorithm has observed a given point.
+    def has_observed(self, trial):
+        """Whether the algorithm has observed a given trial.
 
         .. seealso:: `orion.algo.base.BaseAlgorithm.has_observed`
 
         """
-        return self.algorithm.has_observed(self.transformed_space.transform(point))
+        return self.algorithm.has_observed(self.transformed_space.transform(trial))
 
     @property
     def n_suggested(self):
@@ -127,36 +136,36 @@ def is_done(self):
         """Return True, if an algorithm holds that there can be no further improvement."""
         return self.algorithm.is_done
 
-    def score(self, point):
+    def score(self, trial):
         """Allow algorithm to evaluate `point` based on a prediction about
         this parameter set's performance. Return a subjective measure of expected
         performance.
 
         By default, return the same score any parameter (no preference).
         """
-        assert point in self.space
-        return self.algorithm.score(self.transformed_space.transform(point))
+        self._verify_trial(trial)
+        return self.algorithm.score(self.transformed_space.transform(trial))
 
-    def judge(self, point, measurements):
+    def judge(self, trial, measurements):
         """Inform an algorithm about online `measurements` of a running trial.
 
         The algorithm can return a dictionary of data which will be provided
         as a response to the running environment. Default is None response.
 
         """
-        assert point in self._space
+        self._verify_trial(trial)
         return self.algorithm.judge(
-            self.transformed_space.transform(point), measurements
+            self.transformed_space.transform(trial), measurements
         )
 
-    @property
-    def should_suspend(self):
+    def should_suspend(self, trial):
         """Allow algorithm to decide whether a particular running trial is still
         worth to complete its evaluation, based on information provided by the
         `judge` method.
 
         """
-        return self.algorithm.should_suspend
+        self._verify_trial(trial)
+        return self.algorithm.should_suspend(trial)
 
     @property
     def configuration(self):
@@ -186,3 +195,13 @@ def fidelity_index(self):
         Returns None if there is no fidelity dimension.
         """
         return self.algorithm.fidelity_index
+
+    def _verify_trial(self, trial, space=None):
+        if space is None:
+            space = self.space
+
+        if trial not in self.space:
+            raise ValueError(
+                f"Trial {trial.id} not contained in space:"
+                f"\nParams: {trial.params}\nSpace: {space}"
+            )
diff --git a/tests/conftest.py b/tests/conftest.py
index 92bd9fc11..b90f4129d 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -88,11 +88,10 @@ def __init__(
         self._times_called_is_done = 0
         self._num = 0
         self._index = 0
-        self._points = []
+        self._trials = []
         self._suggested = None
-        self._results = []
         self._score_point = None
-        self._judge_point = None
+        self._judge_trial = None
         self._measurements = None
         self.pool_size = 1
         self.possible_values = [value]
@@ -157,26 +156,25 @@ def suggest(self, num):
 
         return rval
 
-    def observe(self, points, results):
+    def observe(self, trials):
         """Log inputs."""
-        super(DumbAlgo, self).observe(points, results)
-        self._points += points
-        self._results += results
+        super(DumbAlgo, self).observe(trials)
+        self._trials += trials
 
     def score(self, point):
         """Log and return stab."""
         self._score_point = point
         return self.scoring
 
-    def judge(self, point, measurements):
+    def judge(self, trial, measurements):
         """Log and return stab."""
-        self._judge_point = point
+        self._judge_trial = trial
         self._measurements = measurements
         return self.judgement
 
-    @property
-    def should_suspend(self):
+    def should_suspend(self, trial):
         """Cound how many times it has been called and return `suspend`."""
+        self._suspend_trial = trial
         self._times_called_suspend += 1
         return self.suspend
 
diff --git a/tests/unittests/core/conftest.py b/tests/unittests/core/conftest.py
index 4d8c45cda..64d9f4988 100644
--- a/tests/unittests/core/conftest.py
+++ b/tests/unittests/core/conftest.py
@@ -15,6 +15,7 @@
 from orion.core.evc import conflicts
 from orion.core.io.convert import JSONConverter, YAMLConverter
 from orion.core.io.space_builder import DimensionBuilder
+from orion.core.utils import format_trials
 from orion.core.worker.trial import Trial
 from orion.testing import MockDatetime
 
@@ -126,10 +127,10 @@ def hierarchical_space():
     return space
 
 
-@pytest.fixture(scope="module")
-def fixed_suggestion():
-    """Return the same tuple/sample from a possible space."""
-    return (("asdfa", 2), 0, 3.5)
+@pytest.fixture(scope="function")
+def fixed_suggestion(space):
+    """Return the same trial from a possible space."""
+    return format_trials.tuple_to_trial((("asdfa", 2), 0, 3.5), space)
 
 
 @pytest.fixture()
diff --git a/tests/unittests/core/test_primary_algo.py b/tests/unittests/core/test_primary_algo.py
index 586b01da8..67cc2e95e 100644
--- a/tests/unittests/core/test_primary_algo.py
+++ b/tests/unittests/core/test_primary_algo.py
@@ -6,6 +6,7 @@
 
 from orion.algo.base import algo_factory
 from orion.core.worker.primary_algo import SpaceTransformAlgoWrapper
+from orion.core.utils import backward, format_trials
 
 
 @pytest.fixture()
@@ -25,6 +26,13 @@ class TestSpaceTransformAlgoWrapperWraps(object):
     Does not test for transformations.
     """
 
+    def test_verify_trial(self, palgo, space):
+        palgo._verify_trial(format_trials.tuple_to_trial((("asdfa", 2), 0, 3.5), space))
+        with pytest.raises(ValueError, match="not contained in space:"):
+            palgo._verify_trial(
+                format_trials.tuple_to_trial((("asdfa", 2), 10, 3.5), space)
+            )
+
     def test_init_and_configuration(self, dumbalgo, palgo, fixed_suggestion):
         """Check if initialization works."""
         assert isinstance(palgo.algorithm, dumbalgo)
@@ -48,21 +56,20 @@ def test_space_can_only_retrieved(self, palgo, space):
     def test_suggest(self, palgo, fixed_suggestion):
         """Suggest wraps suggested."""
         palgo.algorithm.pool_size = 10
-        assert palgo.suggest(1) == [fixed_suggestion]
-        assert palgo.suggest(4) == [fixed_suggestion] * 4
-        palgo.algorithm.possible_values = [(5,)]
-        with pytest.raises(ValueError):
+        assert palgo.suggest(1)[0].params == fixed_suggestion.params
+        assert [trial.params for trial in palgo.suggest(4)] == [
+            fixed_suggestion.params
+        ] * 4
+        palgo.algorithm.possible_values = [fixed_suggestion]
+        del fixed_suggestion._params[-1]
+        with pytest.raises(ValueError, match="not contained in space"):
             palgo.suggest(1)
 
     def test_observe(self, palgo, fixed_suggestion):
         """Observe wraps observations."""
-        palgo.observe([fixed_suggestion], [5])
-        assert palgo.algorithm._points == [fixed_suggestion]
-        assert palgo.algorithm._results == [5]
-        with pytest.raises(AssertionError):
-            palgo.observe([fixed_suggestion], [5, 8])
-        with pytest.raises(AssertionError):
-            palgo.observe([(5,)], [5])
+        backward.algo_observe(palgo, [fixed_suggestion], [5])
+        palgo.observe([fixed_suggestion])
+        assert palgo.algorithm._trials[0].trial == fixed_suggestion
 
     def test_isdone(self, palgo):
         """Wrap isdone."""
@@ -70,25 +77,24 @@ def test_isdone(self, palgo):
         assert palgo.is_done == 10
         assert palgo.algorithm._times_called_is_done == 1
 
-    def test_shouldsuspend(self, palgo):
+    def test_shouldsuspend(self, palgo, fixed_suggestion):
         """Wrap should_suspend."""
         palgo.algorithm.suspend = 55
-        assert palgo.should_suspend == 55
+        assert palgo.should_suspend(fixed_suggestion) == 55
         assert palgo.algorithm._times_called_suspend == 1
 
     def test_score(self, palgo, fixed_suggestion):
         """Wrap score."""
         palgo.algorithm.scoring = 60
         assert palgo.score(fixed_suggestion) == 60
-        assert palgo.algorithm._score_point == fixed_suggestion
-        with pytest.raises(AssertionError):
-            palgo.score((5,))
+        assert palgo.algorithm._score_point.trial == fixed_suggestion
 
     def test_judge(self, palgo, fixed_suggestion):
         """Wrap judge."""
         palgo.algorithm.judgement = "naedw"
         assert palgo.judge(fixed_suggestion, 8) == "naedw"
-        assert palgo.algorithm._judge_point == fixed_suggestion
+        assert palgo.algorithm._judge_trial.trial is fixed_suggestion
         assert palgo.algorithm._measurements == 8
-        with pytest.raises(AssertionError):
-            palgo.judge((5,), 8)
+        with pytest.raises(ValueError, match="not contained in space"):
+            del fixed_suggestion._params[-1]
+            palgo.judge(fixed_suggestion, 8)

From b64d1aea27bd999e7d3dcb0ff47976f610443061 Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Tue, 19 Oct 2021 16:47:07 -0400
Subject: [PATCH 04/34] Adapt base algorithm class to using trial instead of
 points

---
 src/orion/algo/base.py            | 93 +++++++++++++++++--------------
 tests/unittests/algo/test_base.py | 20 +++++--
 2 files changed, 65 insertions(+), 48 deletions(-)

diff --git a/src/orion/algo/base.py b/src/orion/algo/base.py
index e46fd612b..a8aa750c4 100644
--- a/src/orion/algo/base.py
+++ b/src/orion/algo/base.py
@@ -25,11 +25,6 @@
 log = logging.getLogger(__name__)
 
 
-def infer_trial_id(point):
-    """Compute a hashing of a point"""
-    return hashlib.md5(str(list(point)).encode("utf-8")).hexdigest()
-
-
 # pylint: disable=too-many-public-methods
 class BaseAlgorithm:
     """Base class describing what an algorithm can do.
@@ -166,8 +161,10 @@ def format_trial(self, trial):
 
         return trial
 
-    def get_id(self, point, ignore_fidelity=False):
-        """Compute a unique hash for a point based on params
+    def get_id(self, trial, ignore_fidelity=False):
+        """Return unique hash for a trials based on params
+
+        Deprecated and will be removed in v0.3.0. Use the trial hashing methods instead.
 
         Parameters
         ----------
@@ -179,14 +176,13 @@ def get_id(self, point, ignore_fidelity=False):
         """
         # Apply transforms and reverse to see data as it would come from DB
         # (Some transformations looses some info. ex: Precision transformation)
-        point = list(self.format_trial(point))
+
+        trial = self.format_trial(trial)
 
         if ignore_fidelity:
-            non_fidelity_dims = point[0 : self.fidelity_index]
-            non_fidelity_dims.extend(point[self.fidelity_index + 1 :])
-            point = non_fidelity_dims
+            return trial.hash_params
 
-        return hashlib.md5(str(point).encode("utf-8")).hexdigest()
+        return trial.hash_name
 
     @property
     def fidelity_index(self):
@@ -218,8 +214,8 @@ def suggest(self, num):
 
         Returns
         -------
-        list of points or None
-            A list of lists representing points suggested by the algorithm. The algorithm may opt
+        list of trials or None
+            A list of trials representing values suggested by the algorithm. The algorithm may opt
             out if it cannot make a good suggestion at the moment (it may be waiting for other
             trials to complete), in which case it will return None.
 
@@ -231,13 +227,13 @@ def suggest(self, num):
         pass
 
     def observe(self, trials):
-        """Observe the `results` of the evaluation of the `points` in the
+        """Observe the `results` of the evaluation of the `trials` in the
         process defined in user's script.
 
         Parameters
         ----------
-        trials: list of tuples of array-likes
-           Points from a `orion.algo.space.Space`.
+        trials: list of ``orion.core.worker.trial.Trial``
+           Trials from a `orion.algo.space.Space`.
 
         """
         for trial in trials:
@@ -245,16 +241,17 @@ def observe(self, trials):
                 self.register(trial)
 
     def register(self, trial):
-        """Save the point as one suggested or observed by the algorithm
+        """Save the trial as one suggested or observed by the algorithm.
+
+        The trial objectives may change without the algorithm having actually observed it.
+        In order to detect this, we assign a tuple ``(trial and trial.objective)``
+        to the key ``trial.hash_name`` so that if the objective was not observed, we
+        will see that second item of the tuple is ``None``.
 
         Parameters
         ----------
-        point : array-likes
-           Point from a `orion.algo.space.Space`.
-        result : dict or None, optional
-           The result of an evaluation; partial information about the
-           black-box function at each point in `params`.
-           None is suggested and not yet completed.
+        trial: ``orion.core.worker.trial.Trial``
+           Trial from a `orion.algo.space.Space`.
 
         """
         self._trials_info[trial.hash_name] = (trial, trial.objective)
@@ -269,21 +266,21 @@ def n_observed(self):
         """Number of completed trials observed by the algorithm"""
         return sum(bool(point[1] is not None) for point in self._trials_info.values())
 
-    def has_suggested(self, point):
+    def has_suggested(self, trial):
         """Whether the algorithm has suggested a given point.
 
         Parameters
         ----------
-        point : tuples of array-likes
-           Points from a `orion.algo.space.Space`.
+        trial: ``orion.core.worker.trial.Trial``
+           Trial from a `orion.algo.space.Space`.
 
         Returns
         -------
         bool
-            True if the point was suggested by the algo, False otherwise.
+            True if the trial was suggested by the algo, False otherwise.
 
         """
-        return point.hash_name in self._trials_info
+        return trial.hash_name in self._trials_info
 
     def has_observed(self, trial):
         """Whether the algorithm has observed a given point objective.
@@ -322,36 +319,48 @@ def is_done(self):
 
         return False
 
-    def score(self, point):  # pylint:disable=no-self-use,unused-argument
+    def score(self, trial):  # pylint:disable=no-self-use,unused-argument
         """Allow algorithm to evaluate `point` based on a prediction about
         this parameter set's performance.
 
         By default, return the same score any parameter (no preference).
 
-        :returns: A subjective measure of expected perfomance.
-        :rtype: float
+        Parameters
+        ----------
+        trial: ``orion.core.worker.trial.Trial``
+           Trial object to retrieve from the database
+
+        Returns
+        -------
+        A subjective measure of expected perfomance.
 
         """
         return 0
 
-    def judge(self, point, measurements):  # pylint:disable=no-self-use,unused-argument
+    def judge(self, trial, measurements):  # pylint:disable=no-self-use,unused-argument
         """Inform an algorithm about online `measurements` of a running trial.
 
-        :param point: A tuple which specifies the values of the (hyper)parameters
-           used to execute user's script with.
-
         This method is to be used as a callback in a client-server communication
         between user's script and a orion's worker using a `BaseAlgorithm`.
         Data returned from this method must be serializable and will be used as
         a response to the running environment. Default response is None.
 
-        .. note:: Calling algorithm to `judge` a `point` based on its online
-           `measurements` will effectively change a state in the algorithm (like
-           a reinforcement learning agent's hidden state or an automatic early
-           stopping mechanism's regression), which it may change the value of
-           the property `should_suspend`.
+        Parameters
+        ----------
+        trial: ``orion.core.worker.trial.Trial``
+           Trial object to retrieve from the database
+
+        Notes:
+        ------
 
-        :returns: None or a serializable dictionary containing named data
+        Calling algorithm to `judge` a `point` based on its online `measurements` will effectively
+        change a state in the algorithm (like a reinforcement learning agent's hidden state or an
+        automatic early stopping mechanism's regression), which it may change the value of the
+        property `should_suspend`.
+
+        Returns
+        -------
+        None or a serializable dictionary containing named data
 
         """
         return None
diff --git a/tests/unittests/algo/test_base.py b/tests/unittests/algo/test_base.py
index 3a3993696..fc7bc8f6f 100644
--- a/tests/unittests/algo/test_base.py
+++ b/tests/unittests/algo/test_base.py
@@ -3,6 +3,7 @@
 """Example usage and tests for :mod:`orion.algo.base`."""
 
 from orion.algo.space import Integer, Real, Space
+from orion.core.utils import backward, format_trials
 
 
 def test_init(dumbalgo):
@@ -36,13 +37,20 @@ def test_configuration(dumbalgo):
 
 def test_state_dict(dumbalgo):
     """Check whether trials_info is in the state dict"""
+
+    space = Space()
+    dim = Integer("yolo2", "uniform", -3, 6)
+    space.register(dim)
+    dim = Real("yolo3", "alpha", 0.9)
+    space.register(dim)
+
     nested_algo = {"DumbAlgo": dict(value=6, scoring=5)}
-    algo = dumbalgo(8, value=1)
+    algo = dumbalgo(space, value=1)
     algo.suggest(1)
     assert not algo.state_dict["_trials_info"]
-    algo.observe([(1, 2)], [{"objective": 3}])
+    backward.algo_observe(algo, [format_trials.tuple_to_trial((1, 2), space)], [3])
     assert len(algo.state_dict["_trials_info"]) == 1
-    algo.observe([(1, 2)], [{"objective": 3}])
+    backward.algo_observe(algo, [format_trials.tuple_to_trial((1, 2), space)], [3])
     assert len(algo.state_dict["_trials_info"]) == 1
 
 
@@ -56,7 +64,7 @@ def test_is_done_cardinality(monkeypatch, dumbalgo):
     algo = dumbalgo(space)
     algo.suggest(6)
     for i in range(1, 6):
-        algo.observe([[i]], [{"objective": 3}])
+        backward.algo_observe(algo, [format_trials.tuple_to_trial((i,), space)], [3])
 
     assert len(algo.state_dict["_trials_info"]) == 5
     assert algo.is_done
@@ -67,7 +75,7 @@ def test_is_done_cardinality(monkeypatch, dumbalgo):
     algo = dumbalgo(space)
     algo.suggest(6)
     for i in range(1, 6):
-        algo.observe([[i]], [{"objective": 3}])
+        backward.algo_observe(algo, [format_trials.tuple_to_trial((i,), space)], [3])
 
     assert len(algo.state_dict["_trials_info"]) == 5
     assert not algo.is_done
@@ -83,7 +91,7 @@ def test_is_done_max_trials(monkeypatch, dumbalgo):
     algo = dumbalgo(space)
     algo.suggest(5)
     for i in range(1, 5):
-        algo.observe([[i]], [{"objective": 3}])
+        backward.algo_observe(algo, [format_trials.tuple_to_trial((i,), space)], [3])
 
     assert len(algo.state_dict["_trials_info"]) == 4
     assert not algo.is_done

From 9a25e524fe3cbe30aebdb43500bdbde08a73e1ca Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Fri, 5 Nov 2021 10:25:54 -0400
Subject: [PATCH 05/34] Test that get_id is not sensitive to experiment id

---
 src/orion/algo/base.py                   |  19 ++-
 src/orion/algo/random.py                 |  29 +++--
 src/orion/core/utils/format_trials.py    |   4 +
 src/orion/core/worker/transformer.py     |   2 +
 src/orion/testing/algo.py                | 150 ++++++++++++-----------
 tests/unittests/core/test_transformer.py |   3 +-
 6 files changed, 120 insertions(+), 87 deletions(-)

diff --git a/src/orion/algo/base.py b/src/orion/algo/base.py
index a8aa750c4..be2487cfb 100644
--- a/src/orion/algo/base.py
+++ b/src/orion/algo/base.py
@@ -174,15 +174,24 @@ def get_id(self, trial, ignore_fidelity=False):
             If True, the fidelity dimension is ignored when computing a unique hash for
             the trial. Defaults to False.
         """
+
+        # TODO: ******
+        # NOTE: The trial hash is based on experiment id. This should not matter for the
+        #       algorithms otherwise there will be a clash between trials sampled with current
+        #       experiment and trials sampled from parent ones in EVC.
+        # TODO: ******
+
         # Apply transforms and reverse to see data as it would come from DB
         # (Some transformations looses some info. ex: Precision transformation)
 
         trial = self.format_trial(trial)
 
-        if ignore_fidelity:
-            return trial.hash_params
-
-        return trial.hash_name
+        return trial.compute_trial_hash(
+            trial,
+            ignore_fidelity=ignore_fidelity,
+            ignore_experiment=True,
+            ignore_lie=True,
+        )
 
     @property
     def fidelity_index(self):
@@ -190,6 +199,8 @@ def fidelity_index(self):
 
         Returns None if there is no fidelity dimension.
         """
+        # TODO: Should we return the fidelity key name instead now that we work with
+        #       trials instead of points?
 
         def _is_fidelity(dim):
             return dim.type == "fidelity"
diff --git a/src/orion/algo/random.py b/src/orion/algo/random.py
index 6b57c3292..1bfb54cd7 100644
--- a/src/orion/algo/random.py
+++ b/src/orion/algo/random.py
@@ -51,20 +51,25 @@ def set_state(self, state_dict):
         self.rng.set_state(state_dict["rng_state"])
 
     def suggest(self, num):
-        """Suggest a `num` of new sets of parameters. Randomly draw samples
-        from the import space and return them.
+        """Suggest a `num` of new sets of parameters.
 
-        :param num: how many sets to be suggested.
+        Randomly draw samples from the search space and return them.
 
-        .. note:: New parameters must be compliant with the problem's domain
-           `orion.algo.space.Space`.
+        Parameters
+        ----------
+        num: int
+            Number of trials to suggest.
+
+        Returns
+        -------
+        List of unique trials suggested.
         """
-        points = []
-        while len(points) < num and not self.is_done:
+        trials = []
+        while len(trials) < num and not self.is_done:
             seed = tuple(self.rng.randint(0, 1000000, size=3))
-            new_point = self.space.sample(1, seed=seed)[0]
-            if not self.has_suggested(new_point):
-                self.register(new_point)
-                points.append(new_point)
+            new_trial = self.space.sample(1, seed=seed)[0]
+            if not self.has_suggested(new_trial):
+                self.register(new_trial)
+                trials.append(new_trial)
 
-        return points
+        return trials
diff --git a/src/orion/core/utils/format_trials.py b/src/orion/core/utils/format_trials.py
index 8ad202d50..5183d1185 100644
--- a/src/orion/core/utils/format_trials.py
+++ b/src/orion/core/utils/format_trials.py
@@ -116,3 +116,7 @@ def get_trial_results(trial):
 def standard_param_name(name):
     """Convert parameter name to namespace format"""
     return name.lstrip("/").lstrip("-").replace("-", "_")
+
+
+def update_params(trial, params):
+    """Convert params in Param objects and update trial"""
diff --git a/src/orion/core/worker/transformer.py b/src/orion/core/worker/transformer.py
index 2a1555937..ddd69be4b 100644
--- a/src/orion/core/worker/transformer.py
+++ b/src/orion/core/worker/transformer.py
@@ -567,6 +567,8 @@ def first(self):
 
     def transform(self, point):
         """Only return one element of the group"""
+        print(point, type(point))
+        print(self.index)
         return numpy.array(point)[self.index]
 
     def reverse(self, transformed_point, index=None):
diff --git a/src/orion/testing/algo.py b/src/orion/testing/algo.py
index f49329701..fd2b1e85a 100644
--- a/src/orion/testing/algo.py
+++ b/src/orion/testing/algo.py
@@ -17,6 +17,7 @@
 from orion.algo.tpe import TPE
 from orion.benchmark.task.branin import Branin
 from orion.core.io.space_builder import SpaceBuilder
+from orion.core.utils import backward, format_trials
 from orion.core.worker.primary_algo import SpaceTransformAlgoWrapper
 from orion.testing.space import build_space
 
@@ -204,22 +205,20 @@ def create_space(self, space=None):
         """
         return SpaceBuilder().build(space if space is not None else self.space)
 
-    def observe_points(self, points, algo, objective=0):
-        """Make the algorithm observe points
+    def observe_trials(self, trials, algo, objective=0):
+        """Make the algorithm observe trials
 
         Parameters
         ----------
-        points: list of points
+        trials: list of ``orion.core.worker.trial.Trial``
             Trials formatted as tuples of values
         algo: ``orion.algo.base.BaseAlgorithm``
-            The algorithm used to observe points.
+            The algorithm used to observe trials.
         objective: int, optional
             The base objective for the trials. All objectives
             will have value ``objective + i``. Defaults to 0.
         """
-        algo.observe(
-            points, [dict(objective=objective + i) for i in range(len(points))]
-        )
+        backward.algo_observe(algo, trials, [objective + i for i in range(len(trials))])
 
     def get_num(self, num):
         """Force number of trials to suggest
@@ -230,7 +229,7 @@ def get_num(self, num):
         return num
 
     def force_observe(self, num, algo):
-        """Force observe ``num`` points.
+        """Force observe ``num`` trials.
 
         Parameters
         ----------
@@ -244,33 +243,33 @@ def force_observe(self, num, algo):
         RuntimeError
             - If the algorithm returns duplicates. Algorithms may return duplicates across workers,
               but in sequential scenarios as here, it should not happen.
-            - If the algorithm fails to sample any point at least 5 times.
+            - If the algorithm fails to sample any trial at least 5 times.
         """
         objective = 0
         failed = 0
         MAX_FAILED = 5
         ids = set()
         while not algo.is_done and algo.n_observed < num and failed < MAX_FAILED:
-            points = algo.suggest(self.get_num(num - algo.n_observed))
-            if len(points) == 0:
+            trials = algo.suggest(self.get_num(num - algo.n_observed))
+            if len(trials) == 0:
                 failed += 1
                 continue
-            for point in points:
-                point_id = algo.get_id(point)
-                if point_id in ids:
-                    raise RuntimeError(f"algo suggested a duplicate: {point}")
-                ids.add(point_id)
-            ids |= set(algo.get_id(point) for point in points)
-            self.observe_points(points, algo, objective)
-            objective += len(points)
+            for trial in trials:
+                if trial.hash_name in ids:
+                    raise RuntimeError(f"algo suggested a duplicate: {trial}")
+                ids.add(trial.hash_name)
+            # TODO: Why would we need the line below?? Looks like duplicating the work above
+            # ids |= set(trial.hash_name for trial in trials)
+            self.observe_trials(trials, algo, objective)
+            objective += len(trials)
 
         if failed >= MAX_FAILED:
             raise RuntimeError(
-                f"Algorithm cannot sample more than {algo.n_observed} points. Is it normal?"
+                f"Algorithm cannot sample more than {algo.n_observed} trials. Is it normal?"
             )
 
     def spy_phase(self, mocker, num, algo, attribute):
-        """Force observe ``num`` points and then mock a given method to count calls.
+        """Force observe ``num`` trials and then mock a given method to count calls.
 
         Parameters
         ----------
@@ -300,7 +299,7 @@ def assert_callbacks(self, spy, num, algo):
         spy: Mocked object
             Object mocked by ``BaseAlgoTests.spy_phase``.
         num: int
-            number of points of the phase.
+            number of trials of the phase.
         algo: ``orion.algo.base.BaseAlgorithm``
             The algorithm being tested.
         """
@@ -309,8 +308,8 @@ def assert_callbacks(self, spy, num, algo):
     def assert_dim_type_supported(self, mocker, num, attr, test_space):
         """Test that a given dimension type is properly supported by the algorithm
 
-        This will test that the algorithm sample points valid for the given type
-        and that the algorithm can observe these points.
+        This will test that the algorithm sample trials valid for the given type
+        and that the algorithm can observe these trials.
 
         Parameters
         ----------
@@ -331,10 +330,10 @@ def assert_dim_type_supported(self, mocker, num, attr, test_space):
 
         spy = self.spy_phase(mocker, num, algo, attr)
 
-        points = algo.suggest(1)
-        assert points[0] in space
+        trials = algo.suggest(1)
+        assert trials[0] in space
         spy.call_count == 1
-        self.observe_points(points, algo, 1)
+        self.observe_trials(trials, algo, 1)
         self.assert_callbacks(spy, num, algo)
 
     def test_configuration(self):
@@ -351,34 +350,45 @@ def test_get_id(self):
 
         algo = self.create_algo(space=space)
 
-        assert algo.get_id([1, 1, 1]) == algo.get_id([1, 1, 1])
-        assert algo.get_id([1, 1, 1]) != algo.get_id([1, 2, 2])
-        assert algo.get_id([1, 1, 1]) != algo.get_id([2, 1, 1])
+        def get_id(point, ignore_fidelity=False, exp_id=None):
+            trial = format_trials.tuple_to_trial(point, space)
+            trial.experiment = exp_id
+            return algo.get_id(
+                trial,
+                ignore_fidelity=ignore_fidelity,
+            )
+
+        assert get_id([1, 1, 1]) == get_id([1, 1, 1])
+        assert get_id([1, 1, 1]) != get_id([1, 2, 2])
+        assert get_id([1, 1, 1]) != get_id([2, 1, 1])
 
-        assert algo.get_id([1, 1, 1], ignore_fidelity=False) == algo.get_id(
+        assert get_id([1, 1, 1], ignore_fidelity=False) == get_id(
             [1, 1, 1], ignore_fidelity=False
         )
         # Fidelity changes id
-        assert algo.get_id([1, 1, 1], ignore_fidelity=False) != algo.get_id(
+        assert get_id([1, 1, 1], ignore_fidelity=False) != get_id(
             [2, 1, 1], ignore_fidelity=False
         )
         # Non-fidelity changes id
-        assert algo.get_id([1, 1, 1], ignore_fidelity=False) != algo.get_id(
+        assert get_id([1, 1, 1], ignore_fidelity=False) != get_id(
             [1, 1, 2], ignore_fidelity=False
         )
 
-        assert algo.get_id([1, 1, 1], ignore_fidelity=True) == algo.get_id(
+        assert get_id([1, 1, 1], ignore_fidelity=True) == get_id(
             [1, 1, 1], ignore_fidelity=True
         )
         # Fidelity does not change id
-        assert algo.get_id([1, 1, 1], ignore_fidelity=True) == algo.get_id(
+        assert get_id([1, 1, 1], ignore_fidelity=True) == get_id(
             [2, 1, 1], ignore_fidelity=True
         )
         # Non-fidelity still changes id
-        assert algo.get_id([1, 1, 1], ignore_fidelity=True) != algo.get_id(
+        assert get_id([1, 1, 1], ignore_fidelity=True) != get_id(
             [1, 1, 2], ignore_fidelity=True
         )
 
+        # Experiment id is ignored
+        assert get_id([1, 1, 1], exp_id=1) == get_id([1, 1, 1], exp_id=2)
+
     @phase
     def test_seed_rng(self, mocker, num, attr):
         """Test that the seeding gives reproducibile results."""
@@ -387,14 +397,13 @@ def test_seed_rng(self, mocker, num, attr):
         seed = numpy.random.randint(10000)
         algo.seed_rng(seed)
         spy = self.spy_phase(mocker, num, algo, attr)
-        points = algo.suggest(1)
-        with pytest.raises(AssertionError):
-            numpy.testing.assert_equal(points, algo.suggest(1))
+        trials = algo.suggest(1)
+        trials[0].id != algo.suggest(1)[0].id
 
         new_algo = self.create_algo()
         new_algo.seed_rng(seed)
         self.force_observe(algo.n_observed, new_algo)
-        numpy.testing.assert_equal(points, new_algo.suggest(1))
+        assert trials[0].id == new_algo.suggest(1)[0].id
 
         self.assert_callbacks(spy, num, new_algo)
 
@@ -410,11 +419,10 @@ def test_state_dict(self, mocker, num, attr):
         a = algo.suggest(1)[0]
 
         new_algo = self.create_algo()
-        with pytest.raises(AssertionError):
-            numpy.testing.assert_equal(a, new_algo.suggest(1)[0])
+        assert a.id != new_algo.suggest(1)[0].id
 
         new_algo.set_state(state)
-        numpy.testing.assert_equal(a, new_algo.suggest(1)[0])
+        assert a.id == new_algo.suggest(1)[0].id
 
         self.assert_callbacks(spy, num, algo)
 
@@ -423,21 +431,21 @@ def test_suggest_n(self, mocker, num, attr):
         """Verify that suggest returns correct number of trials if ``num`` is specified in ``suggest``."""
         algo = self.create_algo()
         spy = self.spy_phase(mocker, num, algo, attr)
-        points = algo.suggest(5)
-        assert len(points) == 5
+        trials = algo.suggest(5)
+        assert len(trials) == 5
 
     @phase
     def test_has_suggested(self, mocker, num, attr):
-        """Verify that algorithm detects correctly if a point was suggested"""
+        """Verify that algorithm detects correctly if a trial was suggested"""
         algo = self.create_algo()
         spy = self.spy_phase(mocker, num, algo, attr)
         a = algo.suggest(1)[0]
         assert algo.has_suggested(a)
-        # NOTE: not algo.has_suggested(some random point) is tested in test_has_suggested_statedict
+        # NOTE: not algo.has_suggested(some random trial) is tested in test_has_suggested_statedict
 
     @phase
     def test_has_suggested_statedict(self, mocker, num, attr):
-        """Verify that algorithm detects correctly if a point was suggested even when state was restored."""
+        """Verify that algorithm detects correctly if a trial was suggested even when state was restored."""
         algo = self.create_algo()
         spy = self.spy_phase(mocker, num, algo, attr)
 
@@ -453,40 +461,40 @@ def test_has_suggested_statedict(self, mocker, num, attr):
 
     @phase
     def test_observe(self, mocker, num, attr):
-        """Verify that algorithm observes point without any issues"""
+        """Verify that algorithm observes trial without any issues"""
         algo = self.create_algo()
         spy = self.spy_phase(mocker, num, algo, attr)
 
         a = algo.space.sample()[0]
-        algo.observe([a], [dict(objective=1)])
+        backward.algo_observe(algo, [a], [dict(objective=1)])
 
         b = algo.suggest(1)[0]
-        algo.observe([b], [dict(objective=2)])
+        backward.algo_observe(algo, [b], [dict(objective=2)])
 
     @phase
     def test_has_observed(self, mocker, num, attr):
-        """Verify that algorithm detects correctly if a point was observed"""
+        """Verify that algorithm detects correctly if a trial was observed"""
         algo = self.create_algo()
         spy = self.spy_phase(mocker, num, algo, attr)
 
         a = algo.suggest(1)[0]
         assert not algo.has_observed(a)
-        algo.observe([a], [dict(objective=1)])
+        backward.algo_observe(algo, [a], [dict(objective=1)])
         assert algo.has_observed(a)
 
         b = algo.suggest(1)[0]
         assert not algo.has_observed(b)
-        algo.observe([b], [dict(objective=2)])
+        backward.algo_observe(algo, [b], [dict(objective=2)])
         assert algo.has_observed(b)
 
     @phase
     def test_has_observed_statedict(self, mocker, num, attr):
-        """Verify that algorithm detects correctly if a point was observed even when state was restored."""
+        """Verify that algorithm detects correctly if a trial was observed even when state was restored."""
         algo = self.create_algo()
         spy = self.spy_phase(mocker, num, algo, attr)
 
         a = algo.suggest(1)[0]
-        algo.observe([a], [dict(objective=1)])
+        backward.algo_observe(algo, [a], [dict(objective=1)])
         state = algo.state_dict
 
         algo = self.create_algo()
@@ -495,7 +503,7 @@ def test_has_observed_statedict(self, mocker, num, attr):
         assert algo.has_observed(a)
 
         b = algo.suggest(1)[0]
-        algo.observe([b], [dict(objective=2)])
+        backward.algo_observe(algo, [b], [dict(objective=2)])
         state = algo.state_dict
 
         algo = self.create_algo()
@@ -505,7 +513,7 @@ def test_has_observed_statedict(self, mocker, num, attr):
 
     @phase
     def test_n_suggested(self, mocker, num, attr):
-        """Verify that algorithm returns correct number of suggested points"""
+        """Verify that algorithm returns correct number of suggested trials"""
         algo = self.create_algo()
         spy = self.spy_phase(mocker, num, algo, attr)
         assert algo.n_suggested == num
@@ -514,13 +522,13 @@ def test_n_suggested(self, mocker, num, attr):
 
     @phase
     def test_n_observed(self, mocker, num, attr):
-        """Verify that algorithm returns correct number of observed points"""
+        """Verify that algorithm returns correct number of observed trials"""
         algo = self.create_algo()
         spy = self.spy_phase(mocker, num, algo, attr)
         assert algo.n_observed == num
-        points = algo.suggest(1)
+        trials = algo.suggest(1)
         assert algo.n_observed == num
-        self.observe_points(points, algo)
+        self.observe_trials(trials, algo)
         assert algo.n_observed == num + 1
 
     @phase
@@ -584,7 +592,11 @@ def test_is_done_cardinality(self):
         for i, (x, y, z) in enumerate(itertools.product(range(5), "abc", range(1, 7))):
             assert not algo.is_done
             n = algo.n_suggested
-            algo.observe([[x, y, z]], [dict(objective=i)])
+            backward.algo_observe(
+                algo,
+                [format_trials.tuple_to_trial([x, y, z], space)],
+                [dict(objective=i)],
+            )
             assert algo.n_suggested == n + 1
 
         assert i + 1 == space.cardinality
@@ -606,19 +618,19 @@ def test_optimize_branin(self):
         algo = self.create_algo(config={}, space=space)
         algo.algorithm.max_trials = MAX_TRIALS
         safe_guard = 0
-        points = []
+        trials = []
         objectives = []
-        while points or not algo.is_done:
+        while trials or not algo.is_done:
             if safe_guard >= MAX_TRIALS:
                 break
 
-            if not points:
-                points = algo.suggest(MAX_TRIALS - len(objectives))
+            if not trials:
+                trials = algo.suggest(MAX_TRIALS - len(objectives))
 
-            point = points.pop(0)
-            results = task(*point)
+            trial = trials.pop(0)
+            results = task(trial.params["x"])
             objectives.append(results[0]["value"])
-            algo.observe([point], [dict(objective=objectives[-1])])
+            backward.algo_observe(algo, [trial], [dict(objective=objectives[-1])])
             safe_guard += 1
 
         assert algo.is_done
diff --git a/tests/unittests/core/test_transformer.py b/tests/unittests/core/test_transformer.py
index e44f0550a..0d7d76b32 100644
--- a/tests/unittests/core/test_transformer.py
+++ b/tests/unittests/core/test_transformer.py
@@ -1,5 +1,4 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+# -*: utf-8 -*-
 """Collection of tests for :mod:`orion.core.worker.transformer`."""
 import copy
 import itertools

From c3c50a8e04e76f7f1a8be69273eeb8a98b19fead Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Fri, 5 Nov 2021 10:26:22 -0400
Subject: [PATCH 06/34] Test that TransformedTrial can be copied.

The use of `__getattr__` to copy the TransformedTrial was causing an
infinite recursion error.
---
 src/orion/core/worker/transformer.py     | 3 +++
 tests/unittests/core/test_transformer.py | 7 +++++++
 2 files changed, 10 insertions(+)

diff --git a/src/orion/core/worker/transformer.py b/src/orion/core/worker/transformer.py
index ddd69be4b..dc77846ab 100644
--- a/src/orion/core/worker/transformer.py
+++ b/src/orion/core/worker/transformer.py
@@ -896,6 +896,9 @@ def params(self):
         """Parameters of the trial"""
         return unflatten({param.name: param.value for param in self._params})
 
+    def __deepcopy__(self, memo):
+        return TransformedTrial(copy.deepcopy(self.trial), copy.deepcopy(self._params))
+
     def to_dict(self):
         trial_dictionary = self.trial.to_dict()
         trial_dictionary["params"] = list(map(lambda x: x.to_dict(), self._params))
diff --git a/tests/unittests/core/test_transformer.py b/tests/unittests/core/test_transformer.py
index 0d7d76b32..74e5d231e 100644
--- a/tests/unittests/core/test_transformer.py
+++ b/tests/unittests/core/test_transformer.py
@@ -1442,6 +1442,13 @@ def test_to_dict_is_transformed(self, space, tspace):
 
         assert "_id" in original_dict
 
+    def test_copy(self, space, tspace):
+        trial = space.sample()[0]
+        ttrial = tspace.transform(trial)
+
+        assert copy.deepcopy(trial).to_dict() == trial.to_dict()
+        assert copy.deepcopy(ttrial).to_dict() == ttrial.to_dict()
+
 
 def test_create_transformed_trial(space, tspace):
     trial = space.sample()[0]

From 72022e1885bbb0b0da22d2ba2ba201ae46139f09 Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Fri, 5 Nov 2021 10:29:08 -0400
Subject: [PATCH 07/34] Adjust RandomSearch to new inferface

---
 src/orion/algo/random.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/orion/algo/random.py b/src/orion/algo/random.py
index 1bfb54cd7..9a72c27df 100644
--- a/src/orion/algo/random.py
+++ b/src/orion/algo/random.py
@@ -67,7 +67,7 @@ def suggest(self, num):
         trials = []
         while len(trials) < num and not self.is_done:
             seed = tuple(self.rng.randint(0, 1000000, size=3))
-            new_trial = self.space.sample(1, seed=seed)[0]
+            new_trial = self.format_trial(self.space.sample(1, seed=seed)[0])
             if not self.has_suggested(new_trial):
                 self.register(new_trial)
                 trials.append(new_trial)

From 629f748641a700643e2102f515cec6b102aa14d4 Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Fri, 5 Nov 2021 11:37:29 -0400
Subject: [PATCH 08/34] Handle reshaped dimension with diff. name ordering

Why:

If the names of the dimensions have the same prefix, but some have a
shape, a transformed space with have a different ordering of dimension.
For example, the following dimensions will have their named sorted
differently:

  dim (shape 2), dim1 (no shape) -->  dim1, dim[0], dim[1]

This is causing an issue when we try to restore the shape of the
transformed dimension, with the names being swapped.

How:

When restoring shape, keep track of the original keys and their order,
and reassign the restored dimensions to the correct index (correct dim
name).
---
 src/orion/core/worker/transformer.py     |  6 +-
 tests/unittests/core/test_transformer.py | 75 +++++++++++++-----------
 2 files changed, 45 insertions(+), 36 deletions(-)

diff --git a/src/orion/core/worker/transformer.py b/src/orion/core/worker/transformer.py
index dc77846ab..b605cf674 100644
--- a/src/orion/core/worker/transformer.py
+++ b/src/orion/core/worker/transformer.py
@@ -843,10 +843,12 @@ def reshape(self, trial):
     def restore_shape(self, transformed_trial):
         """Restore shape."""
         transformed_point = format_trials.trial_to_tuple(transformed_trial, self)
-        point = []
+        original_keys = self._original_space.keys()
+        point = [None for _ in original_keys]
         for index, dim in enumerate(self.values()):
             if dim.first:
-                point.append(dim.reverse(transformed_point, index))
+                point_index = original_keys.index(dim.original_dimension.name)
+                point[point_index] = dim.reverse(transformed_point, index)
 
         return create_restored_trial(transformed_trial, point, self._original_space)
 
diff --git a/tests/unittests/core/test_transformer.py b/tests/unittests/core/test_transformer.py
index 74e5d231e..c346bd48a 100644
--- a/tests/unittests/core/test_transformer.py
+++ b/tests/unittests/core/test_transformer.py
@@ -596,7 +596,7 @@ def test_repr_format(self):
 @pytest.fixture(scope="module")
 def dim():
     """Create an example of `Dimension`."""
-    dim = Real("yolo0", "norm", 0.9, shape=(3, 2))
+    dim = Real("yolo", "norm", 0.9, shape=(3, 2))
     return dim
 
 
@@ -781,7 +781,7 @@ def test_get_hashable_members(self, tdim, tdim2):
             "Identity",
             "real",
             "real",
-            "yolo0",
+            "yolo",
             (3, 2),
             "real",
             (0.9,),
@@ -837,12 +837,12 @@ def test_repr(self, tdim):
         """Check method `__repr__`."""
         assert (
             str(tdim)
-            == "Quantize(Real(name=yolo0, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None))"
+            == "Quantize(Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None))"
         )  # noqa
 
     def test_name_property(self, tdim):
         """Check property `name`."""
-        assert tdim.name == "yolo0"
+        assert tdim.name == "yolo"
 
     def test_type_property(self, tdim, tdim2):
         """Check property `type`."""
@@ -914,7 +914,7 @@ def test_get_hashable_members(self, rdim, rdim2):
             "Identity",
             "real",
             "real",
-            "yolo0",
+            "yolo",
             (3, 2),
             "real",
             (0.9,),
@@ -951,12 +951,12 @@ def test_repr(self, rdim):
         """Check method `__repr__`."""
         assert (
             str(rdim)
-            == "View(shape=(3, 2), index=(0, 1), Quantize(Real(name=yolo0, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)))"
+            == "View(shape=(3, 2), index=(0, 1), Quantize(Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)))"
         )  # noqa
 
     def test_name_property(self, rdim):
         """Check property `name`."""
-        assert rdim.name == "yolo0[0,1]"
+        assert rdim.name == "yolo[0,1]"
 
     def test_type_property(self, rdim, rdim2):
         """Check property `type`."""
@@ -1043,9 +1043,9 @@ def test_reverse(self, space, tspace, rspace, seed):
         """Check method `reverse`."""
         ryo = format_trials.tuple_to_trial(
             tuple(
-                numpy.zeros(tspace["yolo0"].shape).reshape(-1).tolist()
-                + numpy.zeros(tspace["yolo2"].shape).reshape(-1).tolist()
+                numpy.zeros(tspace["yolo2"].shape).reshape(-1).tolist()
                 + [10]
+                + numpy.zeros(tspace["yolo"].shape).reshape(-1).tolist()
             ),
             rspace,
         )
@@ -1055,9 +1055,9 @@ def test_reverse(self, space, tspace, rspace, seed):
     def test_contains(self, tspace, rspace, seed):
         """Check method `transform`."""
         ryo = format_trials.tuple_to_trial(
-            numpy.zeros(tspace["yolo0"].shape).reshape(-1).tolist()
-            + numpy.zeros(tspace["yolo2"].shape).reshape(-1).tolist()
-            + [10],
+            numpy.zeros(tspace["yolo2"].shape).reshape(-1).tolist()
+            + [10]
+            + numpy.zeros(tspace["yolo"].shape).reshape(-1).tolist(),
             rspace,
         )
 
@@ -1081,16 +1081,23 @@ def test_sample(self, space, rspace, seed):
     def test_interval(self, rspace):
         """Check method `interval`."""
         interval = rspace.interval()
+
         assert len(interval) == 3 * 2 + 4 + 1
-        for i in range(3 * 2):
+
+        # Test yolo2
+        for i in range(4):
+            assert interval[i] == (0, 1)
+
+        # Test yolo3
+        assert interval[4] == (3, 10)
+
+        # Test yolo[:, :]
+        for i in range(4 + 1, 4 + 1 + 3 * 2):
             # assert interval[i] == (-float('inf'), float('inf'))
             assert interval[i] == (
                 -numpy.array(numpy.inf).astype(int) + 1,
                 numpy.array(numpy.inf).astype(int) - 1,
             )
-        for i in range(3 * 2, 3 * 2 + 4):
-            assert interval[i] == (0, 1)
-        assert interval[-1] == (3, 10)
 
     def test_reshape(self, space, rspace):
         """Verify that the dimension are reshaped properly, forward and backward"""
@@ -1099,14 +1106,14 @@ def test_reshape(self, space, rspace):
         )
 
         rtrial = format_trials.tuple_to_trial(
-            numpy.array(trial.params["yolo0"]).reshape(-1).tolist()
-            + [0.0, 0.0, 1.0, 0.0]
-            + [10],
+            [0.0, 0.0, 1.0, 0.0]
+            + [10]
+            + numpy.array(trial.params["yolo"]).reshape(-1).tolist(),
             rspace,
         )
         assert rspace.transform(trial).params == rtrial.params
         numpy.testing.assert_equal(
-            rspace.reverse(rtrial).params["yolo0"], trial.params["yolo0"]
+            rspace.reverse(rtrial).params["yolo"], trial.params["yolo"]
         )
         assert rspace.reverse(rtrial).params["yolo2"] == trial.params["yolo2"]
         assert rspace.reverse(rtrial).params["yolo3"] == trial.params["yolo3"]
@@ -1114,14 +1121,14 @@ def test_reshape(self, space, rspace):
     def test_cardinality(self, dim2):
         """Check cardinality of reshaped space"""
         space = Space()
-        space.register(Real("yolo0", "reciprocal", 0.1, 1, precision=1, shape=(2, 2)))
+        space.register(Real("yolo", "reciprocal", 0.1, 1, precision=1, shape=(2, 2)))
         space.register(dim2)
 
         rspace = build_required_space(space, shape_requirement="flattened")
         assert rspace.cardinality == (10 ** (2 * 2)) * 4
 
         space = Space()
-        space.register(Real("yolo0", "uniform", 0, 2, shape=(2, 2)))
+        space.register(Real("yolo", "uniform", 0, 2, shape=(2, 2)))
         space.register(dim2)
 
         rspace = build_required_space(
@@ -1167,7 +1174,7 @@ def test_no_requirement(self, space_each_type):
         assert (
             str(tspace)
             == """\
-Space([Precision(4, Real(name=yolo0, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
+Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
        Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None),
        Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None),
        Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None)),
@@ -1187,7 +1194,7 @@ def test_integer_requirement(self, space_each_type):
         assert (
             str(tspace)
             == """\
-Space([Quantize(Precision(4, Real(name=yolo0, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None))),
+Space([Quantize(Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None))),
        Enumerate(Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None)),
        Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None),
        Quantize(Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None))),
@@ -1207,7 +1214,7 @@ def test_real_requirement(self, space_each_type):
         assert (
             str(tspace)
             == """\
-Space([Precision(4, Real(name=yolo0, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
+Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
        OneHotEncode(Enumerate(Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None))),
        ReverseQuantize(Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None)),
        Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None)),
@@ -1227,7 +1234,7 @@ def test_numerical_requirement(self, space_each_type):
         assert (
             str(tspace)
             == """\
-Space([Precision(4, Real(name=yolo0, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
+Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
        Enumerate(Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None)),
        Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None),
        Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None)),
@@ -1247,7 +1254,7 @@ def test_linear_requirement(self, space_each_type):
         assert (
             str(tspace)
             == """\
-Space([Precision(4, Real(name=yolo0, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
+Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
        Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None),
        Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None),
        Linearize(Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None))),
@@ -1298,7 +1305,7 @@ def test_capacity(self, space_each_type):
         space = Space()
         probs = (0.1, 0.2, 0.3, 0.4)
         categories = ("asdfa", 2, 3, 4)
-        dim = Categorical("yolo0", OrderedDict(zip(categories, probs)), shape=2)
+        dim = Categorical("yolo", OrderedDict(zip(categories, probs)), shape=2)
         space.register(dim)
         dim = Integer("yolo2", "uniform", -3, 6)
         space.register(dim)
@@ -1345,13 +1352,13 @@ def test_precision_with_linear(space, logdim, logintdim):
     space.register(logintdim)
 
     # Force precision on all real or linearized dimensions
-    space["yolo0"].precision = 3
+    space["yolo"].precision = 3
     space["yolo4"].precision = 4
     space["yolo5"].precision = 5
 
     # Create a point
     trial = space.sample(1)[0]
-    real_index = list(space.keys()).index("yolo0")
+    real_index = list(space.keys()).index("yolo")
     logreal_index = list(space.keys()).index("yolo4")
     logint_index = list(space.keys()).index("yolo5")
     trial._params[real_index].value = 0.133333
@@ -1362,13 +1369,13 @@ def test_precision_with_linear(space, logdim, logintdim):
     tspace = build_required_space(space, type_requirement="numerical")
     # Check that transform is fine
     ttrial = tspace.transform(trial)
-    assert ttrial.params["yolo0"] == 0.133
+    assert ttrial.params["yolo"] == 0.133
     assert ttrial.params["yolo4"] == 0.1222
     assert ttrial.params["yolo5"] == 2
 
     # Check that reserve does not break precision
     rtrial = tspace.reverse(ttrial)
-    assert rtrial.params["yolo0"] == 0.133
+    assert rtrial.params["yolo"] == 0.133
     assert rtrial.params["yolo4"] == 0.1222
     assert rtrial.params["yolo5"] == 2
 
@@ -1378,13 +1385,13 @@ def test_precision_with_linear(space, logdim, logintdim):
     )
     # Check that transform is fine
     ttrial = tspace.transform(trial)
-    assert ttrial.params["yolo0"] == 0.133
+    assert ttrial.params["yolo"] == 0.133
     assert ttrial.params["yolo4"] == numpy.log(0.1222)
     assert ttrial.params["yolo5"] == numpy.log(2)
 
     # Check that reserve does not break precision
     rtrial = tspace.reverse(ttrial)
-    assert rtrial.params["yolo0"] == 0.133
+    assert rtrial.params["yolo"] == 0.133
     assert rtrial.params["yolo4"] == 0.1222
     assert rtrial.params["yolo5"] == 2
 

From e6ec7558c1f2e26ec3cac2fb05325c9610ded282 Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Fri, 5 Nov 2021 11:42:34 -0400
Subject: [PATCH 09/34] Test that verify_ttrial is using passed space

---
 src/orion/core/worker/primary_algo.py     |  2 +-
 tests/unittests/core/test_primary_algo.py | 28 +++++++++++++++++++----
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/src/orion/core/worker/primary_algo.py b/src/orion/core/worker/primary_algo.py
index da21c35b9..f1a796755 100644
--- a/src/orion/core/worker/primary_algo.py
+++ b/src/orion/core/worker/primary_algo.py
@@ -200,7 +200,7 @@ def _verify_trial(self, trial, space=None):
         if space is None:
             space = self.space
 
-        if trial not in self.space:
+        if trial not in space:
             raise ValueError(
                 f"Trial {trial.id} not contained in space:"
                 f"\nParams: {trial.params}\nSpace: {space}"
diff --git a/tests/unittests/core/test_primary_algo.py b/tests/unittests/core/test_primary_algo.py
index 67cc2e95e..2984bbba0 100644
--- a/tests/unittests/core/test_primary_algo.py
+++ b/tests/unittests/core/test_primary_algo.py
@@ -5,8 +5,9 @@
 import pytest
 
 from orion.algo.base import algo_factory
-from orion.core.worker.primary_algo import SpaceTransformAlgoWrapper
 from orion.core.utils import backward, format_trials
+from orion.core.worker.primary_algo import SpaceTransformAlgoWrapper
+from orion.core.worker.transformer import build_required_space
 
 
 @pytest.fixture()
@@ -27,11 +28,28 @@ class TestSpaceTransformAlgoWrapperWraps(object):
     """
 
     def test_verify_trial(self, palgo, space):
-        palgo._verify_trial(format_trials.tuple_to_trial((("asdfa", 2), 0, 3.5), space))
+        trial = format_trials.tuple_to_trial((["asdfa", 2], 0, 3.5), space)
+        palgo._verify_trial(trial)
+
+        with pytest.raises(ValueError, match="not contained in space:"):
+            invalid_trial = format_trials.tuple_to_trial((("asdfa", 2), 10, 3.5), space)
+            palgo._verify_trial(invalid_trial)
+
+        # transform space
+        tspace = build_required_space(
+            space, type_requirement="real", shape_requirement="flattened"
+        )
+        # transform point
+        ttrial = tspace.transform(trial)
+
+        ttrial in tspace
+
+        # Transformed point is not in original space
         with pytest.raises(ValueError, match="not contained in space:"):
-            palgo._verify_trial(
-                format_trials.tuple_to_trial((("asdfa", 2), 10, 3.5), space)
-            )
+            palgo._verify_trial(ttrial)
+
+        # Transformed point is in transformed space
+        palgo._verify_trial(ttrial, space=tspace)
 
     def test_init_and_configuration(self, dumbalgo, palgo, fixed_suggestion):
         """Check if initialization works."""

From 3d4f54b785b8582d45830372ead9ed4461e353ed Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Fri, 5 Nov 2021 11:43:07 -0400
Subject: [PATCH 10/34] Adapt grid search to new interface

---
 src/orion/algo/gridsearch.py | 31 +++++++++++++++++--------------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/src/orion/algo/gridsearch.py b/src/orion/algo/gridsearch.py
index 2f889b706..97960f3c9 100644
--- a/src/orion/algo/gridsearch.py
+++ b/src/orion/algo/gridsearch.py
@@ -10,6 +10,7 @@
 
 from orion.algo.base import BaseAlgorithm
 from orion.algo.space import Categorical, Fidelity, Integer, Real
+from orion.core.utils import format_trials
 
 log = logging.getLogger(__name__)
 
@@ -103,7 +104,7 @@ class GridSearch(BaseAlgorithm):
     Parameters
     ----------
     n_values: int or dict
-        Number of points for each dimensions, or dictionary specifying number of points for each
+        Number of trials for each dimensions, or dictionary specifying number of trials for each
         dimension independently (name, n_values). For categorical dimensions, n_values will not be
         used, and all categories will be used to build the grid.
     """
@@ -126,6 +127,7 @@ def _initialize(self):
         self.grid = self.build_grid(
             self.space, n_values, getattr(self, "max_trials", 10000)
         )
+        self.index = 0
 
     @staticmethod
     def build_grid(space, n_values, max_trials=10000):
@@ -134,7 +136,7 @@ def build_grid(space, n_values, max_trials=10000):
         Parameters
         ----------
         n_values: int or dict
-            Number of points for each dimensions, or dictionary specifying number of points for each
+            Number of trials for each dimensions, or dictionary specifying number of trials for each
             dimension independently (name, n_values). For categorical dimensions, n_values will not be
             used, and all categories will be used to build the grid.
         max_trials: int
@@ -176,6 +178,7 @@ def state_dict(self):
         """Return a state dict that can be used to reset the state of the algorithm."""
         state_dict = super(GridSearch, self).state_dict
         state_dict["grid"] = self.grid
+        state_dict["index"] = self.index
         return state_dict
 
     def set_state(self, state_dict):
@@ -188,30 +191,30 @@ def set_state(self, state_dict):
         """
         super(GridSearch, self).set_state(state_dict)
         self.grid = state_dict["grid"]
+        self.index = state_dict["index"]
 
     def suggest(self, num):
         """Return the entire grid of suggestions
 
         Returns
         -------
-        list of points or None
-            A list of lists representing points suggested by the algorithm. The algorithm may opt
+        list of trials or None
+            A list of lists representing trials suggested by the algorithm. The algorithm may opt
             out if it cannot make a good suggestion at the moment (it may be waiting for other
             trials to complete), in which case it will return None.
 
         """
         if self.grid is None:
             self._initialize()
-        i = 0
-        points = []
-        while len(points) < num and i < len(self.grid):
-            point = self.grid[i]
-            if not self.has_suggested(point):
-                self.register(point)
-                points.append(point)
-            i += 1
-
-        return points
+        trials = []
+        while len(trials) < num and self.index < len(self.grid):
+            trial = format_trials.tuple_to_trial(self.grid[self.index], self.space)
+            if not self.has_suggested(trial):
+                self.register(trial)
+                trials.append(trial)
+            self.index += 1
+
+        return trials
 
     @property
     def is_done(self):

From ed702731d983af13256ff051b0ddac160bfc09c3 Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Mon, 8 Nov 2021 12:09:49 -0500
Subject: [PATCH 11/34] Support hierarchical params in space.__contains__

---
 src/orion/algo/space.py            |  6 ++++--
 tests/unittests/algo/test_space.py | 28 ++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/src/orion/algo/space.py b/src/orion/algo/space.py
index c418c4c7c..bdd2a9488 100644
--- a/src/orion/algo/space.py
+++ b/src/orion/algo/space.py
@@ -37,6 +37,7 @@
 
 from orion.core.utils import float_to_digits_list, format_trials
 from orion.core.utils.points import flatten_dims, regroup_dims
+from orion.core.utils.flatten import flatten
 
 logger = logging.getLogger(__name__)
 
@@ -1030,9 +1031,10 @@ def __contains__(self, key_or_trial):
             return super(Space, self).__contains__(key_or_trial)
 
         trial = key_or_trial
-        keys = set(trial.params.keys())
+        flattened_params = flatten(trial.params)
+        keys = set(flattened_params.keys())
         for dim_name, dim in self.items():
-            if dim_name not in keys or trial.params[dim_name] not in dim:
+            if dim_name not in keys or flattened_params[dim_name] not in dim:
                 return False
 
             keys.remove(dim_name)
diff --git a/tests/unittests/algo/test_space.py b/tests/unittests/algo/test_space.py
index 4b2234238..03314fa85 100644
--- a/tests/unittests/algo/test_space.py
+++ b/tests/unittests/algo/test_space.py
@@ -761,6 +761,34 @@ def test_register_and_contain(self):
         assert format_trials.tuple_to_trial((("asdfa", 2), 0, 3.5), space) in space
         assert format_trials.tuple_to_trial((("asdfa", 2), 7, 3.5), space) not in space
 
+    def test_hierarchical_register_and_contain(self):
+        """Register hierarchical dimensions and check if points/name are in space."""
+        space = Space()
+
+        categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, 4: 0.4}
+        dim = Categorical("yolo.nested", categories, shape=2)
+        space.register(dim)
+        dim = Integer("yolo2.nested", "uniform", -3, 6)
+        space.register(dim)
+        dim = Real("yolo3", "norm", 0.9)
+        space.register(dim)
+
+        trial = Trial(
+            params=[
+                {"name": "yolo.nested", "value": ["asdfa", 2], "type": "categorical"},
+                {"name": "yolo2.nested", "value": 1, "type": "integer"},
+                {"name": "yolo3", "value": 0.5, "type": "real"},
+            ]
+        )
+
+        assert "yolo" in trial.params
+        assert "nested" in trial.params["yolo"]
+        assert "yolo2" in trial.params
+        assert "nested" in trial.params["yolo2"]
+        assert "yolo3" in trial.params
+
+        assert trial in space
+
     def test_sample(self):
         """Check whether sampling works correctly."""
         seed = 5

From e15723c32340d758b6dd8604c03d3bf2a3058514 Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Mon, 8 Nov 2021 12:10:31 -0500
Subject: [PATCH 12/34] Adjust format_trials test to new fixed_suggestion

---
 src/orion/core/utils/format_trials.py     | 16 +++---
 tests/unittests/core/test_utils_format.py | 59 +++++++++++------------
 2 files changed, 35 insertions(+), 40 deletions(-)

diff --git a/src/orion/core/utils/format_trials.py b/src/orion/core/utils/format_trials.py
index 5183d1185..442c323b0 100644
--- a/src/orion/core/utils/format_trials.py
+++ b/src/orion/core/utils/format_trials.py
@@ -51,15 +51,15 @@ def dict_to_trial(data, space):
             )
         value = data.get(name, dim.default_value)
 
-        if value not in dim:
-            error_msg = "Dimension {} value {} is outside of prior {}".format(
-                name, value, dim.get_prior_string()
-            )
-            raise ValueError(error_msg)
-
         params.append(dict(name=dim.name, type=dim.type, value=value))
-    assert len(params) == len(space)
-    return Trial(params=params)
+
+    trial = Trial(params=params)
+
+    if trial not in space:
+        error_msg = f"Parameters values {trial.params} are outside of space {space}"
+        raise ValueError(error_msg)
+
+    return trial
 
 
 def tuple_to_trial(data, space, status="new"):
diff --git a/tests/unittests/core/test_utils_format.py b/tests/unittests/core/test_utils_format.py
index 3c64d4a40..31f56d264 100644
--- a/tests/unittests/core/test_utils_format.py
+++ b/tests/unittests/core/test_utils_format.py
@@ -9,14 +9,9 @@
 
 
 @pytest.fixture()
-def trial():
-    """Stab trial to match tuple from fixture `fixed_suggestion`."""
-    params = [
-        dict(name="yolo", type="categorical", value=("asdfa", 2)),
-        dict(name="yolo2", type="integer", value=0),
-        dict(name="yolo3", type="real", value=3.5),
-    ]
-    return Trial(params=params)
+def params_tuple():
+    """Stab param tuple to match trial from fixture `fixed_suggestion`."""
+    return (("asdfa", 2), 0, 3.5)
 
 
 @pytest.fixture()
@@ -42,27 +37,27 @@ def hierarchical_dict_params():
     return {"yolo": {"first": ("asdfa", 2), "second": 0}, "yoloflat": 3.5}
 
 
-def test_trial_to_tuple(space, trial, fixed_suggestion):
+def test_trial_to_tuple(space, fixed_suggestion, params_tuple):
     """Check if trial is correctly created from a sample/tuple."""
-    data = trial_to_tuple(trial, space)
-    assert data == fixed_suggestion
+    data = trial_to_tuple(fixed_suggestion, space)
+    assert data == params_tuple
 
-    trial._params[0].name = "lalala"
+    fixed_suggestion._params[0].name = "lalala"
     with pytest.raises(ValueError) as exc:
-        trial_to_tuple(trial, space)
+        trial_to_tuple(fixed_suggestion, space)
 
     assert "Trial params: ['lalala', 'yolo2', 'yolo3']" in str(exc.value)
 
-    trial._params.pop(0)
+    fixed_suggestion._params.pop(0)
     with pytest.raises(ValueError) as exc:
-        trial_to_tuple(trial, space)
+        trial_to_tuple(fixed_suggestion, space)
 
     assert "Trial params: ['yolo2', 'yolo3']" in str(exc.value)
 
 
-def test_tuple_to_trial(space, trial, fixed_suggestion):
+def test_tuple_to_trial(space, fixed_suggestion, params_tuple):
     """Check if sample is recovered successfully from trial."""
-    t = tuple_to_trial(fixed_suggestion, space)
+    t = tuple_to_trial(params_tuple, space)
     assert t.experiment is None
     assert t.status == "new"
     assert t.worker is None
@@ -70,12 +65,12 @@ def test_tuple_to_trial(space, trial, fixed_suggestion):
     assert t.start_time is None
     assert t.end_time is None
     assert t.results == []
-    assert len(t._params) == len(trial.params)
+    assert len(t._params) == len(fixed_suggestion.params)
     for i in range(len(t.params)):
-        assert t._params[i].to_dict() == trial._params[i].to_dict()
+        assert t._params[i].to_dict() == fixed_suggestion._params[i].to_dict()
 
 
-def test_dict_to_trial(space, trial, dict_params):
+def test_dict_to_trial(space, fixed_suggestion, dict_params):
     """Check if dict is converted successfully to trial."""
     t = dict_to_trial(dict_params, space)
     assert t.experiment is None
@@ -85,17 +80,17 @@ def test_dict_to_trial(space, trial, dict_params):
     assert t.start_time is None
     assert t.end_time is None
     assert t.results == []
-    assert len(t._params) == len(trial._params)
+    assert len(t._params) == len(fixed_suggestion._params)
     for i in range(len(t.params)):
-        assert t._params[i].to_dict() == trial._params[i].to_dict()
+        assert t._params[i].to_dict() == fixed_suggestion._params[i].to_dict()
 
 
-def test_tuple_to_trial_to_tuple(space, trial, fixed_suggestion):
+def test_tuple_to_trial_to_tuple(space, fixed_suggestion, params_tuple):
     """The two functions should be inverse."""
-    data = trial_to_tuple(tuple_to_trial(fixed_suggestion, space), space)
-    assert data == fixed_suggestion
+    data = trial_to_tuple(tuple_to_trial(params_tuple, space), space)
+    assert data == params_tuple
 
-    t = tuple_to_trial(trial_to_tuple(trial, space), space)
+    t = tuple_to_trial(trial_to_tuple(fixed_suggestion, space), space)
     assert t.experiment is None
     assert t.status == "new"
     assert t.worker is None
@@ -103,24 +98,24 @@ def test_tuple_to_trial_to_tuple(space, trial, fixed_suggestion):
     assert t.start_time is None
     assert t.end_time is None
     assert t.results == []
-    assert len(t._params) == len(trial._params)
+    assert len(t._params) == len(fixed_suggestion._params)
     for i in range(len(t._params)):
-        assert t._params[i].to_dict() == trial._params[i].to_dict()
+        assert t._params[i].to_dict() == fixed_suggestion._params[i].to_dict()
 
 
 def test_hierarchical_trial_to_tuple(
-    hierarchical_space, hierarchical_trial, fixed_suggestion
+    hierarchical_space, hierarchical_trial, params_tuple
 ):
     """Check if hierarchical trial is correctly created from a sample/tuple."""
     data = trial_to_tuple(hierarchical_trial, hierarchical_space)
-    assert data == fixed_suggestion
+    assert data == params_tuple
 
 
 def test_tuple_to_hierarchical_trial(
-    hierarchical_space, hierarchical_trial, fixed_suggestion
+    hierarchical_space, hierarchical_trial, params_tuple
 ):
     """Check if sample is recovered successfully from hierarchical trial."""
-    t = tuple_to_trial(fixed_suggestion, hierarchical_space)
+    t = tuple_to_trial(params_tuple, hierarchical_space)
     assert len(t._params) == len(hierarchical_trial._params)
     for i in range(len(t._params)):
         assert t._params[i].to_dict() == hierarchical_trial._params[i].to_dict()

From 038da128d2c24b109796b2a3bb1ca86ea3395a36 Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Mon, 8 Nov 2021 13:15:58 -0500
Subject: [PATCH 13/34] Support reshaped transform on dim + default_value

Reshaped transform on dim:

Sometimes we need to apply a transformation on a dimension only. So far
the transformation of View could only be applied on a full point,
because the ReshapedDimension wrapping the view was expected to fetch
the corresponding dimension at self.index. This commit pushes up the
fetch at self.index at the level of the ReshapedSpace instead, the only
place where we are expected to work on a full point, not only on a
dimension.

default value:

Transformed dimensions should support default value like original
dimensions.
---
 src/orion/core/worker/transformer.py     | 18 ++++++-
 tests/unittests/core/test_transformer.py | 69 ++++++++++++++----------
 2 files changed, 56 insertions(+), 31 deletions(-)

diff --git a/src/orion/core/worker/transformer.py b/src/orion/core/worker/transformer.py
index b605cf674..1dcd86b23 100644
--- a/src/orion/core/worker/transformer.py
+++ b/src/orion/core/worker/transformer.py
@@ -506,6 +506,7 @@ def reverse(self, transformed_point, index=None):
         .. note:: This reverse transformation possibly removes the last tensor dimension
            from `transformed_point`.
         """
+
         point_ = numpy.asarray(transformed_point)
         if self.num_cats == 2:
             return (point_ > 0.5).astype(int)
@@ -703,6 +704,16 @@ def cardinality(self):
         # Else we don't care what transformation is.
         return self.original_dimension.cardinality
 
+    @property
+    def default_value(self):
+        if (
+            self.original_dimension.default_value
+            is self.original_dimension.NO_DEFAULT_VALUE
+        ):
+            return self.NO_DEFAULT_VALUE
+
+        return self.transform(self.original_dimension.default_value)
+
 
 class ReshapedDimension(TransformedDimension):
     """Duck-type :class:`orion.algo.space.Dimension` to mimic its functionality."""
@@ -721,7 +732,7 @@ def first(self):
 
     def transform(self, point):
         """Expose `Transformer.transform` interface from underlying instance."""
-        return self.transformer.transform(point[self.index])
+        return self.transformer.transform(point)
 
     def reverse(self, transformed_point, index=None):
         """Expose `Transformer.reverse` interface from underlying instance."""
@@ -837,7 +848,10 @@ def reverse(self, transformed_trial):
     def reshape(self, trial):
         """Reshape the point"""
         point = format_trials.trial_to_tuple(trial, self._original_space)
-        reshaped_point = tuple([dim.transform(point) for dim in self.values()])
+        reshaped_point = []
+        for dim in self.values():
+            reshaped_point.append(dim.transform(point[dim.index]))
+
         return create_transformed_trial(trial, reshaped_point, self)
 
     def restore_shape(self, transformed_trial):
diff --git a/tests/unittests/core/test_transformer.py b/tests/unittests/core/test_transformer.py
index c346bd48a..045b2f8b1 100644
--- a/tests/unittests/core/test_transformer.py
+++ b/tests/unittests/core/test_transformer.py
@@ -593,28 +593,28 @@ def test_repr_format(self):
         assert t.repr_format(1.0) == "View(shape=(3, 4, 5), index=(0, 2, 1), 1.0)"
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture()
 def dim():
     """Create an example of `Dimension`."""
     dim = Real("yolo", "norm", 0.9, shape=(3, 2))
     return dim
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture()
 def logdim():
     """Create an log example of `Dimension`."""
     dim = Real("yolo4", "reciprocal", 1.0, 10.0, shape=(3, 2))
     return dim
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture()
 def logintdim():
     """Create an log integer example of `Dimension`."""
     dim = Integer("yolo5", "reciprocal", 1, 10, shape=(3, 2))
     return dim
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture()
 def tdim(dim):
     """Create an example of `TransformedDimension`."""
     transformers = [Quantize()]
@@ -622,7 +622,7 @@ def tdim(dim):
     return tdim
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture()
 def rdims(tdim):
     """Create an example of `ReshapedDimension`."""
     transformations = {}
@@ -638,23 +638,23 @@ def rdims(tdim):
     return transformations
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture()
 def rdim(dim, rdims):
     """Single ReshapedDimension"""
     return rdims[f"{dim.name}[0,1]"]
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture()
 def dim2():
     """Create a second example of `Dimension`."""
     probs = (0.1, 0.2, 0.3, 0.4)
     categories = ("asdfa", "2", "3", "4")
     categories = OrderedDict(zip(categories, probs))
-    dim2 = Categorical("yolo2", categories)
+    dim2 = Categorical("yolo2", categories, default_value="2")
     return dim2
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture()
 def tdim2(dim2):
     """Create a second example of `TransformedDimension`."""
     transformers = [Enumerate(dim2.categories), OneHotEncode(len(dim2.categories))]
@@ -662,7 +662,7 @@ def tdim2(dim2):
     return tdim2
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture()
 def rdims2(tdim2):
     """Create a categorical example of `ReshapedDimension`."""
     transformations = {}
@@ -678,25 +678,25 @@ def rdims2(tdim2):
     return transformations
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture()
 def rdim2(dim2, rdims2):
     """Single ReshapedDimension"""
     return rdims2[f"{dim2.name}[1]"]
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture()
 def dim3():
     """Create an example of integer `Dimension`."""
     return Integer("yolo3", "uniform", 3, 7)
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture()
 def tdim3(dim3):
     """Create an example of integer `Dimension`."""
     return TransformedDimension(Compose([], dim3.type), dim3)
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture()
 def rdims3(tdim3):
     """Create an example of integer `Dimension`."""
     rdim3 = ReshapedDimension(
@@ -807,7 +807,7 @@ def test_get_hashable_members(self, tdim, tdim2):
             "categorical",
             (),
             (),
-            None,
+            "2",
             "Distribution",
         )
 
@@ -861,6 +861,11 @@ def test_shape_property(self, tdim, tdim2):
         assert tdim2.original_dimension.shape == ()
         assert tdim2.shape == (4,)
 
+    def test_default_value_property(self, tdim, tdim2):
+        """Check property `default_value`."""
+        assert tdim.default_value is Dimension.NO_DEFAULT_VALUE
+        assert tuple(tdim2.default_value) == (0, 1, 0, 0)
+
 
 class TestReshapedDimension(object):
     """Check functionality of class `ReshapedDimension`."""
@@ -869,7 +874,7 @@ def test_transform(self, rdim):
         """Check method `transform`."""
         a = numpy.zeros((3, 2))
         a[0, 1] = 2
-        assert rdim.transform([a, None]) == 2
+        assert rdim.transform(a) == 2
 
     def test_reverse(self, rdim):
         """Check method `reverse`."""
@@ -943,7 +948,7 @@ def test_get_hashable_members(self, rdim, rdim2):
             "categorical",
             (),
             (),
-            None,
+            "2",
             "Distribution",
         )
 
@@ -975,8 +980,13 @@ def test_shape_property(self, rdim, rdim2):
         assert rdim2.original_dimension.shape == (4,)
         assert rdim2.shape == ()
 
+    def test_default_value_property(self, rdim, rdim2):
+        """Check property `default_value`."""
+        assert rdim.default_value is Dimension.NO_DEFAULT_VALUE
+        assert rdim2.default_value == 1
 
-@pytest.fixture(scope="module")
+
+@pytest.fixture()
 def space(dim, dim2, dim3):
     """Create an example `Space`."""
     space = Space()
@@ -986,7 +996,7 @@ def space(dim, dim2, dim3):
     return space
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture()
 def tspace(space, tdim, tdim2, tdim3):
     """Create an example `TransformedSpace`."""
     tspace = TransformedSpace(space)
@@ -996,7 +1006,7 @@ def tspace(space, tdim, tdim2, tdim3):
     return tspace
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture()
 def rspace(tspace, rdims, rdims2, rdims3):
     """Create an example `ReshapedSpace`."""
     rspace = ReshapedSpace(tspace)
@@ -1137,7 +1147,7 @@ def test_cardinality(self, dim2):
         assert rspace.cardinality == (3 ** (2 * 2)) * 4
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture()
 def space_each_type(dim, dim2, dim3, logdim, logintdim):
     """Create an example `Space`."""
     space = Space()
@@ -1175,7 +1185,7 @@ def test_no_requirement(self, space_each_type):
             str(tspace)
             == """\
 Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
-       Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None),
+       Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=2),
        Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None),
        Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None)),
        Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None)])\
@@ -1195,7 +1205,7 @@ def test_integer_requirement(self, space_each_type):
             str(tspace)
             == """\
 Space([Quantize(Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None))),
-       Enumerate(Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None)),
+       Enumerate(Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=2)),
        Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None),
        Quantize(Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None))),
        Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None)])\
@@ -1215,7 +1225,7 @@ def test_real_requirement(self, space_each_type):
             str(tspace)
             == """\
 Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
-       OneHotEncode(Enumerate(Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None))),
+       OneHotEncode(Enumerate(Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=2))),
        ReverseQuantize(Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None)),
        Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None)),
        ReverseQuantize(Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None))])\
@@ -1235,7 +1245,7 @@ def test_numerical_requirement(self, space_each_type):
             str(tspace)
             == """\
 Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
-       Enumerate(Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None)),
+       Enumerate(Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=2)),
        Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None),
        Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None)),
        Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None)])\
@@ -1255,7 +1265,7 @@ def test_linear_requirement(self, space_each_type):
             str(tspace)
             == """\
 Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
-       Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None),
+       Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=2),
        Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None),
        Linearize(Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None))),
        Linearize(ReverseQuantize(Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None)))])\
@@ -1271,9 +1281,6 @@ def test_flatten_requirement(self, space_each_type):
         assert str(tspace).count("View") == 3 * (3 * 2)
 
         i = 0
-        for _ in range(3 * 2):
-            assert tspace[i].type == "real"
-            i += 1
 
         assert tspace[i].type == "categorical"
         i += 1
@@ -1289,6 +1296,10 @@ def test_flatten_requirement(self, space_each_type):
             assert tspace[i].type == "integer"
             i += 1
 
+        for _ in range(3 * 2):
+            assert tspace[i].type == "real"
+            i += 1
+
         tspace = build_required_space(
             space_each_type, shape_requirement="flattened", type_requirement="real"
         )

From c41fb1e1fcccd08708fd76fe147984081e446eaa Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Mon, 8 Nov 2021 13:21:31 -0500
Subject: [PATCH 14/34] Adapt TPE to new observe interface

---
 src/orion/algo/base.py                    |  15 +--
 src/orion/algo/tpe.py                     | 115 ++++++++--------------
 src/orion/core/utils/backward.py          |   7 +-
 src/orion/testing/algo.py                 |   5 +-
 tests/unittests/algo/test_base.py         |  14 ++-
 tests/unittests/algo/test_tpe.py          |  14 ++-
 tests/unittests/core/test_primary_algo.py |   2 +-
 7 files changed, 78 insertions(+), 94 deletions(-)

diff --git a/src/orion/algo/base.py b/src/orion/algo/base.py
index be2487cfb..01d2e0877 100644
--- a/src/orion/algo/base.py
+++ b/src/orion/algo/base.py
@@ -20,7 +20,7 @@
 from abc import ABCMeta, abstractmethod
 
 from orion.algo.space import Fidelity
-from orion.core.utils import GenericFactory
+from orion.core.utils import GenericFactory, format_trials
 
 log = logging.getLogger(__name__)
 
@@ -256,7 +256,7 @@ def register(self, trial):
 
         The trial objectives may change without the algorithm having actually observed it.
         In order to detect this, we assign a tuple ``(trial and trial.objective)``
-        to the key ``trial.hash_name`` so that if the objective was not observed, we
+        to the key ``self.get_id(trial)`` so that if the objective was not observed, we
         will see that second item of the tuple is ``None``.
 
         Parameters
@@ -265,7 +265,10 @@ def register(self, trial):
            Trial from a `orion.algo.space.Space`.
 
         """
-        self._trials_info[trial.hash_name] = (trial, trial.objective)
+        self._trials_info[self.get_id(trial)] = (
+            trial,
+            format_trials.get_trial_results(trial) if trial.objective else None,
+        )
 
     @property
     def n_suggested(self):
@@ -291,7 +294,7 @@ def has_suggested(self, trial):
             True if the trial was suggested by the algo, False otherwise.
 
         """
-        return trial.hash_name in self._trials_info
+        return self.get_id(trial) in self._trials_info
 
     def has_observed(self, trial):
         """Whether the algorithm has observed a given point objective.
@@ -310,8 +313,8 @@ def has_observed(self, trial):
 
         """
         return (
-            trial.hash_name in self._trials_info
-            and self._trials_info[trial.hash_name][1] is not None
+            self.get_id(trial) in self._trials_info
+            and self._trials_info[self.get_id(trial)][1] is not None
         )
 
     @property
diff --git a/src/orion/algo/tpe.py b/src/orion/algo/tpe.py
index 46d5a0b55..bfad31180 100644
--- a/src/orion/algo/tpe.py
+++ b/src/orion/algo/tpe.py
@@ -9,6 +9,7 @@
 from scipy.stats import norm
 
 from orion.algo.base import BaseAlgorithm
+from orion.core.utils import format_trials
 from orion.core.utils.points import flatten_dims, regroup_dims
 
 logger = logging.getLogger(__name__)
@@ -285,8 +286,8 @@ def suggest(self, num=None):
         Parameters
         ----------
         num: int, optional
-            Number of points to sample. If None, TPE will sample all random points at once, or a
-            single point if it is at the Bayesian Optimization stage.
+            Number of trials to sample. If None, TPE will sample all random trials at once, or a
+            single trial if it is at the Bayesian Optimization stage.
         :param num: how many sets to be suggested.
 
         .. note:: New parameters must be compliant with the problem's domain
@@ -314,30 +315,30 @@ def suggest(self, num=None):
         return samples
 
     def _suggest(self, num, function):
-        points = []
+        trials = []
 
         ids = set(self._trials_info.keys())
         retries = 0
-        while len(points) < num and retries < self.max_retry:
-            for candidate in function(num - len(points)):
+        while len(trials) < num and retries < self.max_retry:
+            for candidate in function(num - len(trials)):
                 candidate_id = self.get_id(candidate)
                 if candidate_id not in ids:
                     ids.add(candidate_id)
-                    points.append(candidate)
+                    trials.append(candidate)
                 else:
                     retries += 1
 
                 if len(ids) >= self.space.cardinality:
-                    return points
+                    return trials
 
         if retries >= self.max_retry:
             logger.warning(
-                f"Algorithm unable to sample `{num}` points with less than "
+                f"Algorithm unable to sample `{num}` trials with less than "
                 f"`{self.max_retry}` retries. Try adjusting the configuration of TPE "
                 "to favor exploration (`n_ei_candidates` and `gamma` in particular)."
             )
 
-        return points
+        return trials
 
     def _suggest_random(self, num):
         def sample(num):
@@ -355,85 +356,53 @@ def suggest_bo(num):
 
     def _suggest_one_bo(self):
 
-        point = []
-        below_points, above_points = self.split_trials()
-
-        below_points = list(map(list, zip(*below_points)))
-        above_points = list(map(list, zip(*above_points)))
+        params = {}
+        below_trials, above_trials = self.split_trials()
 
-        idx = 0
-        for i, dimension in enumerate(self.space.values()):
-
-            shape = dimension.shape
-            if not shape:
-                shape = (1,)
+        for dimension in self.space.values():
+            dim_below_trials = [trial.params[dimension.name] for trial in below_trials]
+            dim_above_trials = [trial.params[dimension.name] for trial in above_trials]
 
             if dimension.type == "real":
                 dim_samples = self._sample_real_dimension(
                     dimension,
-                    shape[0],
-                    below_points[idx : idx + shape[0]],
-                    above_points[idx : idx + shape[0]],
+                    dim_below_trials,
+                    dim_above_trials,
                 )
             elif dimension.type == "integer" and dimension.prior_name in [
                 "int_uniform",
                 "int_reciprocal",
             ]:
-                dim_samples = self.sample_one_dimension(
+                dim_samples = self._sample_int_point(
                     dimension,
-                    shape[0],
-                    below_points[idx : idx + shape[0]],
-                    above_points[idx : idx + shape[0]],
-                    self._sample_int_point,
+                    dim_below_trials,
+                    dim_above_trials,
                 )
             elif dimension.type == "categorical" and dimension.prior_name == "choices":
-                dim_samples = self.sample_one_dimension(
+                dim_samples = self._sample_categorical_point(
                     dimension,
-                    shape[0],
-                    below_points[idx : idx + shape[0]],
-                    above_points[idx : idx + shape[0]],
-                    self._sample_categorical_point,
+                    dim_below_trials,
+                    dim_above_trials,
                 )
             elif dimension.type == "fidelity":
                 # fidelity dimension
-                dim_samples = [point[i] for point in self.space.sample(1)]
+                trial = self.space.sample(1)[0]
+                dim_samples = trial.params[dimension.name]
             else:
                 raise NotImplementedError()
 
-            idx += shape[0]
-            point += dim_samples
-
-        return self.format_point(point)
-
-    # pylint:disable=no-self-use
-    def sample_one_dimension(
-        self, dimension, shape_size, below_points, above_points, sampler
-    ):
-        """Sample values for a dimension
-
-        :param dimension: Dimension.
-        :param shape_size: 1D Shape Size of the Real Dimension.
-        :param below_points: good points with shape (m, n), m=shape_size.
-        :param above_points: bad points with shape (m, n), m=shape_size.
-        :param sampler: method to sample one value for upon the dimension.
-        """
-        points = []
-
-        for j in range(shape_size):
-            new_point = sampler(dimension, below_points[j], above_points[j])
-            points.append(new_point)
+            params[dimension.name] = dim_samples
 
-        return points
+        trial = format_trials.dict_to_trial(params, self.space)
+        return self.format_trial(trial)
 
-    def _sample_real_dimension(self, dimension, shape_size, below_points, above_points):
+    def _sample_real_dimension(self, dimension, below_points, above_points):
         """Sample values for real dimension"""
         if any(map(dimension.prior_name.endswith, ["uniform", "reciprocal"])):
-            return self.sample_one_dimension(
+            return self._sample_real_point(
                 dimension,
-                shape_size,
                 below_points,
                 above_points,
-                self._sample_real_point,
             )
         else:
             raise NotImplementedError(
@@ -532,26 +501,22 @@ def _sample_categorical_point(self, dimension, below_points, above_points):
     def split_trials(self):
         """Split the observed trials into good and bad ones based on the ratio `gamma``"""
         sorted_trials = sorted(
-            (point for point in self._trials_info.values() if point[1] is not None),
-            key=lambda x: x[1]["objective"],
+            (
+                (trial, results)
+                for (trial, results) in self._trials_info.values()
+                if results is not None
+            ),
+            key=lambda point: point[1]["objective"],
         )
-        sorted_points = [list(points) for points, results in sorted_trials]
+        sorted_trials = [trial for trial, results in sorted_trials]
 
-        split_index = int(numpy.ceil(self.gamma * len(sorted_points)))
+        split_index = int(numpy.ceil(self.gamma * len(sorted_trials)))
 
-        below = sorted_points[:split_index]
-        above = sorted_points[split_index:]
+        below = sorted_trials[:split_index]
+        above = sorted_trials[split_index:]
 
         return below, above
 
-    def observe(self, points, results):
-        """Observe evaluation `results` corresponding to list of `points` in
-        space.
-
-        A simple random sampler though does not take anything into account.
-        """
-        super(TPE, self).observe(points, results)
-
 
 class GMMSampler:
     """Gaussian Mixture Model Sampler for TPE algorithm
diff --git a/src/orion/core/utils/backward.py b/src/orion/core/utils/backward.py
index a8061bec7..2cc52ef8a 100644
--- a/src/orion/core/utils/backward.py
+++ b/src/orion/core/utils/backward.py
@@ -175,9 +175,8 @@ def port_algo_config(config):
 
 
 def algo_observe(algo, trials, results):
-    for trial, result in zip(trials, results):
-        trial.results.append(
-            Trial.Result(name="objective", type="objective", value=result)
-        )
+    for trial, trial_results in zip(trials, results):
+        for name, trial_result in trial_results.items():
+            trial.results.append(Trial.Result(name=name, type=name, value=trial_result))
 
     algo.observe(trials)
diff --git a/src/orion/testing/algo.py b/src/orion/testing/algo.py
index fd2b1e85a..392cfadfe 100644
--- a/src/orion/testing/algo.py
+++ b/src/orion/testing/algo.py
@@ -218,7 +218,9 @@ def observe_trials(self, trials, algo, objective=0):
             The base objective for the trials. All objectives
             will have value ``objective + i``. Defaults to 0.
         """
-        backward.algo_observe(algo, trials, [objective + i for i in range(len(trials))])
+        backward.algo_observe(
+            algo, trials, [dict(objective=objective + i) for i in range(len(trials))]
+        )
 
     def get_num(self, num):
         """Force number of trials to suggest
@@ -249,6 +251,7 @@ def force_observe(self, num, algo):
         failed = 0
         MAX_FAILED = 5
         ids = set()
+
         while not algo.is_done and algo.n_observed < num and failed < MAX_FAILED:
             trials = algo.suggest(self.get_num(num - algo.n_observed))
             if len(trials) == 0:
diff --git a/tests/unittests/algo/test_base.py b/tests/unittests/algo/test_base.py
index fc7bc8f6f..a98434d5c 100644
--- a/tests/unittests/algo/test_base.py
+++ b/tests/unittests/algo/test_base.py
@@ -48,9 +48,13 @@ def test_state_dict(dumbalgo):
     algo = dumbalgo(space, value=1)
     algo.suggest(1)
     assert not algo.state_dict["_trials_info"]
-    backward.algo_observe(algo, [format_trials.tuple_to_trial((1, 2), space)], [3])
+    backward.algo_observe(
+        algo, [format_trials.tuple_to_trial((1, 2), space)], [dict(objective=3)]
+    )
     assert len(algo.state_dict["_trials_info"]) == 1
-    backward.algo_observe(algo, [format_trials.tuple_to_trial((1, 2), space)], [3])
+    backward.algo_observe(
+        algo, [format_trials.tuple_to_trial((1, 2), space)], [dict(objective=3)]
+    )
     assert len(algo.state_dict["_trials_info"]) == 1
 
 
@@ -64,7 +68,7 @@ def test_is_done_cardinality(monkeypatch, dumbalgo):
     algo = dumbalgo(space)
     algo.suggest(6)
     for i in range(1, 6):
-        backward.algo_observe(algo, [format_trials.tuple_to_trial((i,), space)], [3])
+        backward.algo_observe(algo, [format_trials.tuple_to_trial((i,), space)], [dict(objective=3)])
 
     assert len(algo.state_dict["_trials_info"]) == 5
     assert algo.is_done
@@ -75,7 +79,7 @@ def test_is_done_cardinality(monkeypatch, dumbalgo):
     algo = dumbalgo(space)
     algo.suggest(6)
     for i in range(1, 6):
-        backward.algo_observe(algo, [format_trials.tuple_to_trial((i,), space)], [3])
+        backward.algo_observe(algo, [format_trials.tuple_to_trial((i,), space)], [dict(objective=3)])
 
     assert len(algo.state_dict["_trials_info"]) == 5
     assert not algo.is_done
@@ -91,7 +95,7 @@ def test_is_done_max_trials(monkeypatch, dumbalgo):
     algo = dumbalgo(space)
     algo.suggest(5)
     for i in range(1, 5):
-        backward.algo_observe(algo, [format_trials.tuple_to_trial((i,), space)], [3])
+        backward.algo_observe(algo, [format_trials.tuple_to_trial((i,), space)], [dict(objective=3)])
 
     assert len(algo.state_dict["_trials_info"]) == 4
     assert not algo.is_done
diff --git a/tests/unittests/algo/test_tpe.py b/tests/unittests/algo/test_tpe.py
index e3cbe8df1..5ff949701 100644
--- a/tests/unittests/algo/test_tpe.py
+++ b/tests/unittests/algo/test_tpe.py
@@ -17,6 +17,7 @@
     compute_max_ei_point,
     ramp_up_weights,
 )
+from orion.core.utils import backward, format_trials
 from orion.core.worker.transformer import build_required_space
 from orion.testing.algo import BaseAlgoTests, phase
 
@@ -797,7 +798,11 @@ def test_is_done_cardinality(self):
         for i, (x, y, z) in enumerate(itertools.product(range(5), "abc", range(1, 7))):
             assert not algo.is_done
             n = algo.n_suggested
-            algo.observe([[x, y, z]], [dict(objective=i)])
+            backward.algo_observe(
+                algo,
+                [format_trials.tuple_to_trial([x, y, z], space)],
+                [dict(objective=i)],
+            )
             assert algo.n_suggested == n + 1
 
         assert i + 1 == space.cardinality
@@ -816,7 +821,12 @@ def test_log_integer(self, monkeypatch):
 
         # Mock sampling so that it quickly samples all possible integers in given bounds
         def sample(self, n_samples=1, seed=None):
-            return [(numpy.log(values.pop()),) for _ in range(n_samples)]
+            return [
+                format_trials.tuple_to_trial(
+                    (numpy.log(values.pop()),), algo.transformed_space
+                )
+                for _ in range(n_samples)
+            ]
 
         def _suggest_random(self, num):
             return self._suggest(num, sample)
diff --git a/tests/unittests/core/test_primary_algo.py b/tests/unittests/core/test_primary_algo.py
index 2984bbba0..70119f56d 100644
--- a/tests/unittests/core/test_primary_algo.py
+++ b/tests/unittests/core/test_primary_algo.py
@@ -85,7 +85,7 @@ def test_suggest(self, palgo, fixed_suggestion):
 
     def test_observe(self, palgo, fixed_suggestion):
         """Observe wraps observations."""
-        backward.algo_observe(palgo, [fixed_suggestion], [5])
+        backward.algo_observe(palgo, [fixed_suggestion], [dict(objective=5)])
         palgo.observe([fixed_suggestion])
         assert palgo.algorithm._trials[0].trial == fixed_suggestion
 

From 44455705189fb98d85613a2b4c11a3243f88138e Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Tue, 9 Nov 2021 14:40:34 -0500
Subject: [PATCH 15/34] Add branch() method to Trial

Why:

The branching method will be used by algorithm generating new trials
based on existing one, such as Hyperband increasing the fidelity, and
later on Population Based Training which could also change the working
directory.

How:

When calling branch(), a new trial is created with same parameters as
current one, except for the parameter values passed to
(ex: ``branch(params={'some': 'values'})``).
---
 src/orion/core/worker/trial.py            | 39 ++++++++++++++++++++
 tests/unittests/core/worker/test_trial.py | 43 +++++++++++++++++++++++
 2 files changed, 82 insertions(+)

diff --git a/src/orion/core/worker/trial.py b/src/orion/core/worker/trial.py
index bc00f2f00..7fc130a86 100644
--- a/src/orion/core/worker/trial.py
+++ b/src/orion/core/worker/trial.py
@@ -7,6 +7,7 @@
 Describe a particular training run, parameters and results.
 
 """
+import copy
 import hashlib
 import logging
 
@@ -220,6 +221,44 @@ def __init__(self, **kwargs):
             else:
                 setattr(self, attrname, value)
 
+    def branch(self, status="new", params=None):
+        """Copy the trial and modify given attributes
+
+        The status attributes will be reset as if trial was new.
+
+        Parameters
+        ----------
+        status: str, optional
+            The status of the new trial. Defaults to 'new'.
+        params: dict, optional
+            Some parameters to update. A subset of params may be passed. Passing
+            non-existing params in current trial will lead to a ValueError.
+            Defaults to `None`.
+
+        Raises
+        ------
+        ValueError
+            If some parameters are not present in current trial.
+        AttributeError
+            If some attribute does not exist in Trial objects.
+        """
+        if params is None:
+            params = {}
+
+        params = copy.deepcopy(params)
+
+        config_params = []
+        for param in self._params:
+            config_param = param.to_dict()
+            if param.name in params:
+                config_param["value"] = params.pop(param.name)
+            config_params.append(config_param)
+
+        if params:
+            raise ValueError(f"Some parameters are not part of base trial: {params}")
+
+        return Trial(status=status, params=config_params)
+
     def to_dict(self):
         """Needed to be able to convert `Trial` to `dict` form."""
         trial_dictionary = dict()
diff --git a/tests/unittests/core/worker/test_trial.py b/tests/unittests/core/worker/test_trial.py
index 9021ed898..6ff9f8a2a 100644
--- a/tests/unittests/core/worker/test_trial.py
+++ b/tests/unittests/core/worker/test_trial.py
@@ -8,6 +8,15 @@
 from orion.core.worker.trial import Trial
 
 
+@pytest.fixture
+def base_trial():
+    x = {"name": "/x", "value": [1, 2], "type": "real"}
+    y = {"name": "/y", "value": [1, 2], "type": "integer"}
+    objective = {"name": "objective", "value": 10, "type": "objective"}
+
+    return Trial(experiment=1, status="completed", params=[x, y], results=[objective])
+
+
 class TestTrial(object):
     """Test Trial object and class."""
 
@@ -365,3 +374,37 @@ def test_higher_shape_id_is_same(self):
         assert (
             trial.id == Trial(**bson.BSON.decode(bson.BSON.encode(trial.to_dict()))).id
         )
+
+    def test_branch_empty(self, base_trial):
+        """Test that branching with no args is only copying"""
+        branched_trial = base_trial.branch()
+        assert branched_trial.experiment is None
+        assert branched_trial is not base_trial
+        assert branched_trial.status == "new"
+        assert branched_trial.start_time is None
+        assert branched_trial.end_time is None
+        assert branched_trial.heartbeat is None
+        assert branched_trial.params == base_trial.params
+        assert branched_trial.objective is None
+
+    def test_branch_base_attr(self, base_trial):
+        """Test branching with base attributes (not params)"""
+        branched_trial = base_trial.branch(status="interrupted")
+        assert branched_trial.status != base_trial.status
+        assert branched_trial.status == "interrupted"
+        assert branched_trial.params == base_trial.params
+
+    def test_branch_params(self, base_trial):
+        """Test branching with params"""
+        branched_trial = base_trial.branch(status="interrupted", params={"/y": [3, 0]})
+        assert branched_trial.status != base_trial.status
+        assert branched_trial.status == "interrupted"
+        assert branched_trial.params != base_trial.params
+        assert branched_trial.params == {"/x": [1, 2], "/y": [3, 0]}
+
+    def test_branch_new_params(self, base_trial):
+        """Test branching with params that are not in base trial"""
+        with pytest.raises(
+            ValueError, match="Some parameters are not part of base trial: {'/z': 0}"
+        ):
+            base_trial.branch(params={"/z": 0})

From 5455a8505b6a4fba152c0e444ebabf51d235b7ad Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Tue, 9 Nov 2021 14:43:37 -0500
Subject: [PATCH 16/34] Adjust Hyperband to new observe interface

---
 src/orion/algo/base.py                 |  10 +-
 src/orion/algo/hyperband.py            | 163 ++++++-----
 tests/unittests/algo/test_base.py      |  12 +-
 tests/unittests/algo/test_hyperband.py | 385 ++++++++++++++-----------
 4 files changed, 321 insertions(+), 249 deletions(-)

diff --git a/src/orion/algo/base.py b/src/orion/algo/base.py
index 01d2e0877..a0b267c03 100644
--- a/src/orion/algo/base.py
+++ b/src/orion/algo/base.py
@@ -195,7 +195,7 @@ def get_id(self, trial, ignore_fidelity=False):
 
     @property
     def fidelity_index(self):
-        """Compute the index of the point where fidelity is.
+        """Compute the dimension name of the space where fidelity is.
 
         Returns None if there is no fidelity dimension.
         """
@@ -205,11 +205,9 @@ def fidelity_index(self):
         def _is_fidelity(dim):
             return dim.type == "fidelity"
 
-        fidelity_index = [
-            i for i, dim in enumerate(self.space.values()) if _is_fidelity(dim)
-        ]
-        if fidelity_index:
-            return fidelity_index[0]
+        fidelity_dim = [dim for dim in self.space.values() if _is_fidelity(dim)]
+        if fidelity_dim:
+            return fidelity_dim[0].name
 
         return None
 
diff --git a/src/orion/algo/hyperband.py b/src/orion/algo/hyperband.py
index 742513e1e..8b89d8064 100644
--- a/src/orion/algo/hyperband.py
+++ b/src/orion/algo/hyperband.py
@@ -16,6 +16,7 @@
 
 from orion.algo.base import BaseAlgorithm
 from orion.algo.space import Fidelity
+from orion.core.utils.flatten import flatten
 
 logger = logging.getLogger(__name__)
 
@@ -155,7 +156,7 @@ def __init__(self, space, seed=None, repetitions=numpy.inf):
         if fidelity_index is None:
             raise RuntimeError(SPACE_ERROR)
 
-        fidelity_dim = space.values()[fidelity_index]
+        fidelity_dim = space[fidelity_index]
 
         self.min_resources = fidelity_dim.low
         self.max_resources = fidelity_dim.high
@@ -179,36 +180,37 @@ def create_bracket(self, i, budgets, iteration):
         return HyperbandBracket(self, budgets, iteration)
 
     def sample_from_bracket(self, bracket, num):
-        """Sample new points from bracket"""
-        points = []
-        while len(points) < num:
-            point = bracket.get_sample()
-            if point is None:
+        """Sample new trials from bracket"""
+        trials = []
+        while len(trials) < num:
+            trial = bracket.get_sample()
+            if trial is None:
                 break
 
-            point = list(point)
-            point[self.fidelity_index] = bracket.rungs[0]["resources"]
+            trial = trial.branch(
+                params={self.fidelity_index: bracket.rungs[0]["resources"]}
+            )
 
-            full_id = self.get_id(point, ignore_fidelity=False)
-            id_wo_fidelity = self.get_id(point, ignore_fidelity=True)
+            full_id = self.get_id(trial, ignore_fidelity=False)
+            id_wo_fidelity = self.get_id(trial, ignore_fidelity=True)
             bracket_observed = self.trial_to_brackets.get(id_wo_fidelity)
 
-            if not self.has_suggested(point) and (
+            if not self.has_suggested(trial) and (
                 not bracket_observed
                 or (
                     bracket_observed.repetition_id < bracket.repetition_id
-                    and bracket_observed.get_point_max_resource(point)
+                    and bracket_observed.get_trial_max_resource(trial)
                     < bracket.rungs[0]["resources"]
                 )
             ):
                 # if no duplicated found or the duplicated found existing in previous hyperband
                 # execution with less resource
 
-                points.append(tuple(point))
-                self.register(point)
+                trials.append(trial)
+                self.register(trial)
                 self.trial_to_brackets[id_wo_fidelity] = bracket
 
-        return points
+        return trials
 
     def seed_rng(self, seed):
         """Seed the state of the random number generator.
@@ -263,7 +265,7 @@ def register_samples(self, bracket, samples):
                     "https://github.com/Epistimio/orion/issues/new/choose"
                 )
             self.register(sample)
-            bracket.register(sample, None)
+            bracket.register(sample)
 
             if self.get_id(sample, ignore_fidelity=True) not in self.trial_to_brackets:
                 self.trial_to_brackets[
@@ -370,23 +372,23 @@ def create_brackets(self):
             for i, bracket_budgets in enumerate(self.budgets)
         ]
 
-    def _get_bracket(self, point):
-        """Get the bracket of a point"""
-        fidelity = point[self.fidelity_index]
-        _id_wo_fidelity = self.get_id(point, ignore_fidelity=True)
+    def _get_bracket(self, trial):
+        """Get the bracket of a trial"""
+        fidelity = flatten(trial.params)[self.fidelity_index]
+        _id_wo_fidelity = self.get_id(trial, ignore_fidelity=True)
 
         brackets = []
         for bracket in self.brackets:
-            # If find same point in first rung of a bracket,
-            # the point should register in this bracket
+            # If find same trial in first rung of a bracket,
+            # the trial should register in this bracket
             if _id_wo_fidelity in bracket.rungs[0]["results"]:
                 brackets = [bracket]
                 break
 
         if not brackets:
-            # If the point show in current hyeprband execution the first time,
+            # If the trial show in current hyeprband execution the first time,
             # the bracket with same fidelity in the first rung should be used,
-            # the assumption is that there is no duplicated points inside same hyperband execution.
+            # the assumption is that there is no duplicated trials inside same hyperband execution.
             brackets = [
                 bracket
                 for bracket in self.brackets
@@ -395,46 +397,49 @@ def _get_bracket(self, point):
 
         if not brackets:
             raise ValueError(
-                "No bracket found for point {0} with fidelity {1}".format(
+                "No bracket found for trial {0} with fidelity {1}".format(
                     _id_wo_fidelity, fidelity
                 )
             )
 
         if len(brackets) > 1:
             logger.warning(
-                "More than one bracket found for point %s, this should not happen",
-                str(point),
+                "More than one bracket found for trial %s, this should not happen",
+                str(trial),
             )
 
         bracket = brackets[0]
 
         return bracket
 
-    def observe(self, points, results):
-        """Observe evaluation `results` corresponding to list of `points` in
+    def observe(self, trials):
+        """Observe evaluation `results` corresponding to list of `trials` in
         space.
 
-        A simple random sampler though does not take anything into account.
-        """
+        Parameters
+        ----------
+        trials: list of ``orion.core.worker.trial.Trial``
+           Trials from a `orion.algo.space.Space`.
 
-        for point, result in zip(points, results):
+        """
+        for trial in trials:
 
-            if not self.has_suggested(point):
+            if not self.has_suggested(trial):
                 logger.info(
-                    "Ignoring point %s because it was not sampled by current algo.",
-                    point,
+                    "Ignoring trial %s because it was not sampled by current algo.",
+                    trial,
                 )
                 continue
 
-            self.register(point, result)
+            self.register(trial)
 
-            bracket = self._get_bracket(point)
+            bracket = self._get_bracket(trial)
 
             try:
-                bracket.register(point, result["objective"])
+                bracket.register(trial)
             except IndexError:
                 logger.warning(
-                    "Point registered to wrong bracket. This is likely due "
+                    "Trial registered to wrong bracket. This is likely due "
                     "to a corrupted database, where trials of different fidelity "
                     "have a wrong timestamps."
                 )
@@ -475,10 +480,6 @@ def __init__(self, hyperband, budgets, repetition_id):
 
         logger.debug("Bracket budgets: %s", str(budgets))
 
-        # points = hyperband.sample(compute_rung_sizes(reduction_factor, len(budgets))[0])
-        # for point in points:
-        #     self.register(point, None)
-
     @property
     def state_dict(self):
         return {"rungs": copy.deepcopy(self.rungs)}
@@ -491,10 +492,10 @@ def is_filled(self):
         """Return True if first rung with trials is filled"""
         return self.has_rung_filled(0)
 
-    def get_point_max_resource(self, point):
-        """Return the max resource value that has been tried for a point"""
+    def get_trial_max_resource(self, trial):
+        """Return the max resource value that has been tried for a trial"""
         max_resource = 0
-        _id_wo_fidelity = self.hyperband.get_id(point, ignore_fidelity=True)
+        _id_wo_fidelity = self.hyperband.get_id(trial, ignore_fidelity=True)
         for rung in self.rungs:
             if _id_wo_fidelity in rung["results"]:
                 max_resource = rung["resources"]
@@ -529,15 +530,16 @@ def sample(self, num):
             request, self, buffer=should_have_n_trials * 10 / request
         )
 
-    def register(self, point, objective):
-        """Register a point in the corresponding rung"""
-        self._get_results(point)[self.hyperband.get_id(point, ignore_fidelity=True)] = (
-            objective,
-            point,
+    def register(self, trial):
+        """Register a trial in the corresponding rung"""
+        self._get_results(trial)[self.hyperband.get_id(trial, ignore_fidelity=True)] = (
+            trial.objective.value if trial.objective else None,
+            trial,
         )
 
-    def _get_results(self, point):
-        fidelity = point[self.hyperband.fidelity_index]
+    def _get_results(self, trial):
+        print(flatten(trial.params))
+        fidelity = flatten(trial.params)[self.hyperband.fidelity_index]
         rungs = [
             rung["results"] for rung in self.rungs if rung["resources"] == fidelity
         ]
@@ -545,7 +547,7 @@ def _get_results(self, point):
             budgets = [rung["resources"] for rung in self.rungs]
             raise IndexError(
                 REGISTRATION_ERROR.format(
-                    fidelity=fidelity, budgets=budgets, params=point
+                    fidelity=fidelity, budgets=budgets, params=trial.params
                 )
             )
 
@@ -558,27 +560,36 @@ def remainings(self):
         return max(should_have_n_trials - have_n_trials, 0)
 
     def get_candidates(self, rung_id):
-        """Get a candidate for promotion"""
+        """Get a candidate for promotion
+
+        Raises
+        ------
+        TypeError
+            If get_candidates is called before the entire rung is completed.
+        """
         if self.has_rung_filled(rung_id + 1):
             return []
 
         rung = self.rungs[rung_id]["results"]
         next_rung = self.rungs[rung_id + 1]["results"]
 
-        rung = list(sorted((objective, point) for objective, point in rung.values()))
+        rung = sorted(rung.values(), key=lambda pair: pair[0])
+
+        if not rung:
+            return []
 
         should_have_n_trials = self.rungs[rung_id + 1]["n_trials"]
-        points = []
+        trials = []
         i = 0
-        while len(points) + len(next_rung) < should_have_n_trials:
-            objective, point = rung[i]
+        while len(trials) + len(next_rung) < should_have_n_trials:
+            objective, trial = rung[i]
             assert objective is not None
-            _id = self.hyperband.get_id(point, ignore_fidelity=True)
+            _id = self.hyperband.get_id(trial, ignore_fidelity=True)
             if _id not in next_rung:
-                points.append(point)
+                trials.append(trial)
             i += 1
 
-        return points
+        return trials
 
     @property
     def is_done(self):
@@ -612,7 +623,7 @@ def promote(self, num):
 
         The rungs are iterated over in reversed order, so that high rungs
         are prioritised for promotions. When a candidate is promoted, the loop is broken and
-        the method returns the promoted point.
+        the method returns the promoted trial.
 
         .. note ::
 
@@ -632,28 +643,34 @@ def promote(self, num):
             if not self.is_ready(rung_id):
                 return []
 
-            points = []
+            trials = []
             for candidate in self.get_candidates(rung_id):
                 # pylint: disable=logging-format-interpolation
                 logger.debug(
-                    "Promoting {point} from rung {past_rung} with fidelity {past_fidelity} to "
+                    "Promoting {trial} from rung {past_rung} with fidelity {past_fidelity} to "
                     "rung {new_rung} with fidelity {new_fidelity}".format(
-                        point=candidate,
+                        trial=candidate,
                         past_rung=rung_id,
-                        past_fidelity=candidate[self.hyperband.fidelity_index],
+                        past_fidelity=flatten(candidate.params)[
+                            self.hyperband.fidelity_index
+                        ],
                         new_rung=rung_id + 1,
                         new_fidelity=self.rungs[rung_id + 1]["resources"],
                     )
                 )
 
-                candidate = list(copy.deepcopy(candidate))
-                candidate[self.hyperband.fidelity_index] = self.rungs[rung_id + 1][
-                    "resources"
-                ]
+                candidate = candidate.branch(
+                    status="new",
+                    params={
+                        self.hyperband.fidelity_index: self.rungs[rung_id + 1][
+                            "resources"
+                        ]
+                    },
+                )
                 if not self.hyperband.has_suggested(candidate):
-                    points.append(tuple(candidate))
+                    trials.append(candidate)
 
-            return points[:num]
+            return trials[:num]
 
         return []
 
diff --git a/tests/unittests/algo/test_base.py b/tests/unittests/algo/test_base.py
index a98434d5c..346cff16b 100644
--- a/tests/unittests/algo/test_base.py
+++ b/tests/unittests/algo/test_base.py
@@ -68,7 +68,9 @@ def test_is_done_cardinality(monkeypatch, dumbalgo):
     algo = dumbalgo(space)
     algo.suggest(6)
     for i in range(1, 6):
-        backward.algo_observe(algo, [format_trials.tuple_to_trial((i,), space)], [dict(objective=3)])
+        backward.algo_observe(
+            algo, [format_trials.tuple_to_trial((i,), space)], [dict(objective=3)]
+        )
 
     assert len(algo.state_dict["_trials_info"]) == 5
     assert algo.is_done
@@ -79,7 +81,9 @@ def test_is_done_cardinality(monkeypatch, dumbalgo):
     algo = dumbalgo(space)
     algo.suggest(6)
     for i in range(1, 6):
-        backward.algo_observe(algo, [format_trials.tuple_to_trial((i,), space)], [dict(objective=3)])
+        backward.algo_observe(
+            algo, [format_trials.tuple_to_trial((i,), space)], [dict(objective=3)]
+        )
 
     assert len(algo.state_dict["_trials_info"]) == 5
     assert not algo.is_done
@@ -95,7 +99,9 @@ def test_is_done_max_trials(monkeypatch, dumbalgo):
     algo = dumbalgo(space)
     algo.suggest(5)
     for i in range(1, 5):
-        backward.algo_observe(algo, [format_trials.tuple_to_trial((i,), space)], [dict(objective=3)])
+        backward.algo_observe(
+            algo, [format_trials.tuple_to_trial((i,), space)], [dict(objective=3)]
+        )
 
     assert len(algo.state_dict["_trials_info"]) == 4
     assert not algo.is_done
diff --git a/tests/unittests/algo/test_hyperband.py b/tests/unittests/algo/test_hyperband.py
index 5e4a56fb5..bc2fc6b67 100644
--- a/tests/unittests/algo/test_hyperband.py
+++ b/tests/unittests/algo/test_hyperband.py
@@ -11,6 +11,40 @@
 from orion.algo.hyperband import Hyperband, HyperbandBracket, compute_budgets
 from orion.algo.space import Fidelity, Integer, Real, Space
 from orion.testing.algo import BaseAlgoTests, phase
+from orion.testing.trial import create_trial
+
+
+def create_trial_for_hb(point, objective=None):
+    return create_trial(
+        point,
+        names=("epoch", "lr"),
+        results={"objective": objective},
+        types=("fidelity", "real"),
+    )
+
+
+def create_rung_from_points(points, n_trials, resources):
+
+    results = {}
+    for param in points:
+        trial = create_trial_for_hb((resources, param), objective=param)
+        trial_hash = trial.compute_trial_hash(
+            trial,
+            ignore_fidelity=True,
+            ignore_experiment=True,
+        )
+        results[trial_hash] = (trial.objective.value, trial)
+
+    return dict(n_trials=n_trials, resources=resources, results=results)
+
+
+def compare_trials(trials, other_trials):
+    assert [t.params for t in trials] == [t.params for t in other_trials]
+
+
+def compare_registered_trial(registered_trial, trial):
+    assert registered_trial[0] == trial.objective.value
+    assert registered_trial[1].to_dict() == trial.to_dict()
 
 
 @pytest.fixture
@@ -43,47 +77,21 @@ def bracket(budgets, hyperband):
 @pytest.fixture
 def rung_0():
     """Create fake points and objectives for rung 0."""
-    points = np.linspace(0, 8, 9)
-    return dict(
-        n_trials=9,
-        resources=1,
-        results={
-            hashlib.md5(str([point]).encode("utf-8")).hexdigest(): (point, (1, point))
-            for point in points
-        },
-    )
+    return create_rung_from_points(np.linspace(0, 8, 9), n_trials=9, resources=1)
 
 
 @pytest.fixture
 def rung_1(rung_0):
     """Create fake points and objectives for rung 1."""
-    values = map(
-        lambda v: (v[0], (3, v[0])), list(sorted(rung_0["results"].values()))[:3]
-    )
-    return dict(
-        n_trials=3,
-        resources=3,
-        results={
-            hashlib.md5(str([value[0]]).encode("utf-8")).hexdigest(): value
-            for value in values
-        },
-    )
+    points = [trial.params["lr"] for _, trial in sorted(rung_0["results"].values())[:3]]
+    return create_rung_from_points(points, n_trials=3, resources=3)
 
 
 @pytest.fixture
 def rung_2(rung_1):
     """Create fake points and objectives for rung 1."""
-    values = map(
-        lambda v: (v[0], (9, v[0])), list(sorted(rung_1["results"].values()))[:1]
-    )
-    return dict(
-        n_trials=1,
-        resources=9,
-        results={
-            hashlib.md5(str([value[0]]).encode("utf-8")).hexdigest(): value
-            for value in values
-        },
-    )
+    points = [trial.params["lr"] for _, trial in sorted(rung_1["results"].values())[:1]]
+    return create_rung_from_points(points, n_trials=1, resources=9)
 
 
 def test_compute_budgets():
@@ -104,16 +112,16 @@ def test_compute_budgets():
     assert compute_budgets(16, 5) == [[(5, 3), (1, 16)], [(2, 16)]]
 
 
-def force_observe(hyperband, point, results):
+def force_observe(hyperband, trial):
     # hyperband.sampled.add(hashlib.md5(str(list(point)).encode("utf-8")).hexdigest())
 
-    hyperband.register(point, results)
+    hyperband.register(trial)
 
-    bracket = hyperband._get_bracket(point)
-    id_wo_fidelity = hyperband.get_id(point, ignore_fidelity=True)
+    bracket = hyperband._get_bracket(trial)
+    id_wo_fidelity = hyperband.get_id(trial, ignore_fidelity=True)
     hyperband.trial_to_brackets[id_wo_fidelity] = bracket
 
-    hyperband.observe([point], [results])
+    hyperband.observe([trial])
 
 
 def mock_samples(hyperband, samples):
@@ -134,21 +142,21 @@ def test_rungs_creation(self, bracket):
     def test_register(self, hyperband, bracket):
         """Check that a point is correctly registered inside a bracket."""
         bracket.hyperband = hyperband
-        point = (1, 0.0)
-        point_hash = hashlib.md5(str([0.0]).encode("utf-8")).hexdigest()
+        trial = create_trial_for_hb((1, 0.0), 0.0)
+        trial_id = hyperband.get_id(trial, ignore_fidelity=True)
 
-        bracket.register(point, 0.0)
+        bracket.register(trial)
 
         assert len(bracket.rungs[0])
-        assert point_hash in bracket.rungs[0]["results"]
-        assert (0.0, point) == bracket.rungs[0]["results"][point_hash]
+        assert trial_id in bracket.rungs[0]["results"]
+        assert (trial.objective.value, trial) == bracket.rungs[0]["results"][trial_id]
 
     def test_bad_register(self, hyperband, bracket):
         """Check that a non-valid point is not registered."""
         bracket.hyperband = hyperband
 
         with pytest.raises(IndexError) as ex:
-            bracket.register((55, 0.0), 0.0)
+            bracket.register(create_trial_for_hb((55, 0.0), 0.0))
 
         assert "Bad fidelity level 55" in str(ex.value)
 
@@ -159,19 +167,21 @@ def test_candidate_promotion(self, hyperband, bracket, rung_0):
 
         points = bracket.get_candidates(0)
 
-        assert points[0] == (1, 0.0)
+        assert points[0].params == create_trial_for_hb((1, 0.0), 0.0).params
 
     def test_promotion_with_rung_1_hit(self, hyperband, bracket, rung_0):
         """Test that get_candidate gives us the next best thing if point is already in rung 1."""
-        point = (1, 0.0)
-        point_hash = hashlib.md5(str([0.0]).encode("utf-8")).hexdigest()
+        trial = create_trial_for_hb((1, 0.0), None)
         bracket.hyperband = hyperband
         bracket.rungs[0] = rung_0
-        bracket.rungs[1]["results"][point_hash] = (0.0, point)
+        bracket.rungs[1]["results"][hyperband.get_id(trial, ignore_fidelity=True)] = (
+            trial.objective.value,
+            trial,
+        )
 
-        points = bracket.get_candidates(0)
+        trials = bracket.get_candidates(0)
 
-        assert points[0] == (1, 1)
+        assert trials[0].params == create_trial_for_hb((1, 1), 0.0).params
 
     def test_no_promotion_when_rung_full(self, hyperband, bracket, rung_0, rung_1):
         """Test that get_candidate returns `None` if rung 1 is full."""
@@ -194,7 +204,7 @@ def test_no_promotion_if_not_completed(self, hyperband, bracket, rung_0):
         for p_id in rung.keys():
             rung[p_id] = (None, rung[p_id][1])
 
-        with pytest.raises(AssertionError):
+        with pytest.raises(TypeError):
             bracket.get_candidates(0)
 
     def test_is_done(self, bracket, rung_0):
@@ -210,13 +220,15 @@ def test_update_rungs_return_candidate(self, hyperband, bracket, rung_1):
         """Check if a valid modified candidate is returned by update_rungs."""
         bracket.hyperband = hyperband
         bracket.rungs[1] = rung_1
-        point_hash = hashlib.md5(str([0.0]).encode("utf-8")).hexdigest()
+        trial = create_trial_for_hb((3, 0.0), 0.0)
+        # point_hash = hashlib.md5(str([0.0]).encode("utf-8")).hexdigest()
 
         candidates = bracket.promote(1)
 
-        assert point_hash in bracket.rungs[1]["results"]
-        assert bracket.rungs[1]["results"][point_hash] == (0.0, (3, 0.0))
-        assert candidates[0][0] == 9
+        trial_id = hyperband.get_id(trial, ignore_fidelity=True)
+        assert trial_id in bracket.rungs[1]["results"]
+        assert bracket.rungs[1]["results"][trial_id][1].params == trial.params
+        assert candidates[0].params["epoch"] == 9
 
     def test_update_rungs_return_no_candidate(self, hyperband, bracket, rung_1):
         """Check if no candidate is returned by update_rungs."""
@@ -226,21 +238,21 @@ def test_update_rungs_return_no_candidate(self, hyperband, bracket, rung_1):
 
         assert candidates == []
 
-    def test_get_point_max_resource(self, hyperband, bracket, rung_0, rung_1, rung_2):
-        """Test to get the max resource R for a particular point"""
+    def test_get_trial_max_resource(self, hyperband, bracket, rung_0, rung_1, rung_2):
+        """Test to get the max resource R for a particular trial"""
         bracket.hyperband = hyperband
         bracket.rungs[0] = rung_0
 
-        assert bracket.get_point_max_resource(point=(1, 0.0)) == 1
-        assert bracket.get_point_max_resource(point=(1, 8.0)) == 1
+        assert bracket.get_trial_max_resource(trial=create_trial_for_hb((1, 0.0))) == 1
+        assert bracket.get_trial_max_resource(trial=create_trial_for_hb((1, 8.0))) == 1
 
         bracket.rungs[1] = rung_1
-        assert bracket.get_point_max_resource(point=(1, 0.0)) == 3
-        assert bracket.get_point_max_resource(point=(1, 8.0)) == 1
+        assert bracket.get_trial_max_resource(trial=create_trial_for_hb((1, 0.0))) == 3
+        assert bracket.get_trial_max_resource(trial=create_trial_for_hb((1, 8.0))) == 1
 
         bracket.rungs[2] = rung_2
-        assert bracket.get_point_max_resource(point=(1, 0.0)) == 9
-        assert bracket.get_point_max_resource(point=(1, 8.0)) == 1
+        assert bracket.get_trial_max_resource(trial=create_trial_for_hb((1, 0.0))) == 9
+        assert bracket.get_trial_max_resource(trial=create_trial_for_hb((1, 8.0))) == 1
 
     def test_repr(self, bracket, rung_0, rung_1, rung_2):
         """Test the string representation of HyperbandBracket"""
@@ -259,14 +271,15 @@ def test_register(self, hyperband, bracket, rung_0, rung_1):
         hyperband.brackets = [bracket]
         bracket.hyperband = hyperband
         bracket.rungs = [rung_0, rung_1]
-        point = (1, 0.0)
-        point_hash = hashlib.md5(str([0.0]).encode("utf-8")).hexdigest()
+        trial = create_trial_for_hb((1, 0.0), 0.0)
+        trial_id = hyperband.get_id(trial, ignore_fidelity=True)
 
-        hyperband.observe([point], [{"objective": 0.0}])
+        hyperband.observe([trial])
 
         assert len(bracket.rungs[0])
-        assert point_hash in bracket.rungs[0]["results"]
-        assert (0.0, point) == bracket.rungs[0]["results"][point_hash]
+        assert trial_id in bracket.rungs[0]["results"]
+        assert bracket.rungs[0]["results"][trial_id][0] == 0.0
+        assert bracket.rungs[0]["results"][trial_id][1].params == trial.params
 
     def test_register_bracket_multi_fidelity(self, space):
         """Check that a point is registered inside the same bracket for diff fidelity."""
@@ -274,27 +287,29 @@ def test_register_bracket_multi_fidelity(self, space):
 
         value = 50
         fidelity = 1
-        point = (fidelity, value)
-        point_hash = hashlib.md5(str([value]).encode("utf-8")).hexdigest()
+        trial = create_trial_for_hb((fidelity, value), 0.0)
+        trial_id = hyperband.get_id(trial, ignore_fidelity=True)
 
-        force_observe(hyperband, point, {"objective": 0.0})
+        force_observe(hyperband, trial)
 
         bracket = hyperband.brackets[0]
 
         assert len(bracket.rungs[0])
-        assert point_hash in bracket.rungs[0]["results"]
-        assert (0.0, point) == bracket.rungs[0]["results"][point_hash]
+        assert trial_id in bracket.rungs[0]["results"]
+        assert bracket.rungs[0]["results"][trial_id][0] == 0.0
+        assert bracket.rungs[0]["results"][trial_id][1].params == trial.params
 
         fidelity = 3
-        point = [fidelity, value]
-        point_hash = hashlib.md5(str([value]).encode("utf-8")).hexdigest()
+        trial = create_trial_for_hb((fidelity, value), 0.0)
+        trial_id = hyperband.get_id(trial, ignore_fidelity=True)
 
-        force_observe(hyperband, point, {"objective": 0.0})
+        force_observe(hyperband, trial)
 
-        assert len(bracket.rungs[0])
-        assert point_hash in bracket.rungs[1]["results"]
-        assert (0.0, point) != bracket.rungs[0]["results"][point_hash]
-        assert (0.0, point) == bracket.rungs[1]["results"][point_hash]
+        assert len(bracket.rungs[1])
+        assert trial_id in bracket.rungs[1]["results"]
+        assert bracket.rungs[0]["results"][trial_id][1].params != trial.params
+        assert bracket.rungs[1]["results"][trial_id][0] == 0.0
+        assert bracket.rungs[1]["results"][trial_id][1].params == trial.params
 
     def test_register_next_bracket(self, space):
         """Check that a point is registered inside the good bracket when higher fidelity."""
@@ -302,29 +317,33 @@ def test_register_next_bracket(self, space):
 
         value = 50
         fidelity = 3
-        point = (fidelity, value)
-        point_hash = hashlib.md5(str([value]).encode("utf-8")).hexdigest()
+        trial = create_trial_for_hb((fidelity, value), 0.0)
+        trial_id = hyperband.get_id(trial, ignore_fidelity=True)
 
-        force_observe(hyperband, point, {"objective": 0.0})
+        force_observe(hyperband, trial)
 
         assert sum(len(rung["results"]) for rung in hyperband.brackets[0].rungs) == 0
         assert sum(len(rung["results"]) for rung in hyperband.brackets[1].rungs) == 1
         assert sum(len(rung["results"]) for rung in hyperband.brackets[2].rungs) == 0
-        assert point_hash in hyperband.brackets[1].rungs[0]["results"]
-        assert (0.0, point) == hyperband.brackets[1].rungs[0]["results"][point_hash]
+        assert trial_id in hyperband.brackets[1].rungs[0]["results"]
+        compare_registered_trial(
+            hyperband.brackets[1].rungs[0]["results"][trial_id], trial
+        )
 
         value = 51
         fidelity = 9
-        point = (fidelity, value)
-        point_hash = hashlib.md5(str([value]).encode("utf-8")).hexdigest()
+        trial = create_trial_for_hb((fidelity, value), 0.0)
+        trial_id = hyperband.get_id(trial, ignore_fidelity=True)
 
-        force_observe(hyperband, point, {"objective": 0.0})
+        force_observe(hyperband, trial)
 
         assert sum(len(rung["results"]) for rung in hyperband.brackets[0].rungs) == 0
         assert sum(len(rung["results"]) for rung in hyperband.brackets[1].rungs) == 1
         assert sum(len(rung["results"]) for rung in hyperband.brackets[2].rungs) == 1
-        assert point_hash in hyperband.brackets[2].rungs[0]["results"]
-        assert (0.0, point) == hyperband.brackets[2].rungs[0]["results"][point_hash]
+        assert trial_id in hyperband.brackets[2].rungs[0]["results"]
+        compare_registered_trial(
+            hyperband.brackets[2].rungs[0]["results"][trial_id], trial
+        )
 
     def test_register_invalid_fidelity(self, space):
         """Check that a point cannot registered if fidelity is invalid."""
@@ -332,12 +351,12 @@ def test_register_invalid_fidelity(self, space):
 
         value = 50
         fidelity = 2
-        point = (fidelity, value)
+        trial = create_trial_for_hb((fidelity, value))
 
         with pytest.raises(ValueError) as ex:
-            force_observe(hyperband, point, {"objective": 0.0})
+            force_observe(hyperband, trial)
 
-        assert "No bracket found for point" in str(ex.value)
+        assert "No bracket found for trial" in str(ex.value)
 
     def test_register_not_sampled(self, space, caplog):
         """Check that a point cannot registered if not sampled."""
@@ -345,13 +364,13 @@ def test_register_not_sampled(self, space, caplog):
 
         value = 50
         fidelity = 2
-        point = (fidelity, value)
+        trial = create_trial_for_hb((fidelity, value))
 
         with caplog.at_level(logging.INFO, logger="orion.algo.hyperband"):
-            hyperband.observe([point], [{"objective": 0.0}])
+            hyperband.observe([trial])
 
         assert len(caplog.records) == 1
-        assert "Ignoring point" in caplog.records[0].msg
+        assert "Ignoring trial" in caplog.records[0].msg
 
     def test_register_corrupted_db(self, caplog, space):
         """Check that a point cannot registered if passed in order diff than fidelity."""
@@ -359,52 +378,59 @@ def test_register_corrupted_db(self, caplog, space):
 
         value = 50
         fidelity = 3
-        point = (fidelity, value)
+        trial = create_trial_for_hb((fidelity, value))
 
-        force_observe(hyperband, point, {"objective": 0.0})
-        assert "Point registered to wrong bracket" not in caplog.text
+        force_observe(hyperband, trial)
+        assert "Trial registered to wrong bracket" not in caplog.text
 
         fidelity = 1
-        point = [fidelity, value]
+        trial = create_trial_for_hb((fidelity, value))
 
         caplog.clear()
-        force_observe(hyperband, point, {"objective": 0.0})
-        assert "Point registered to wrong bracket" in caplog.text
+        force_observe(hyperband, trial)
+        assert "Trial registered to wrong bracket" in caplog.text
 
     def test_suggest_new(self, monkeypatch, hyperband, bracket, rung_0, rung_1, rung_2):
         """Test that a new point is sampled."""
         hyperband.brackets = [bracket]
         bracket.hyperband = hyperband
 
-        mock_samples(hyperband, [("fidelity", i) for i in range(10)])
+        mock_samples(
+            hyperband, [create_trial_for_hb(("fidelity", i)) for i in range(10)]
+        )
 
-        points = hyperband.suggest(100)
+        trials = hyperband.suggest(100)
 
-        assert points[0] == (1.0, 0)
-        assert points[1] == (1.0, 1)
+        assert trials[0].params == {"epoch": 1.0, "lr": 0}
+        assert trials[1].params == {"epoch": 1.0, "lr": 1}
 
     def test_suggest_duplicates_between_calls(self, monkeypatch, hyperband, bracket):
-        """Test that same points are not allowed in different suggest call of
+        """Test that same trials are not allowed in different suggest call of
         the same hyperband execution.
         """
         hyperband.brackets = [bracket]
         bracket.hyperband = hyperband
 
-        duplicate_point = ("fidelity", 0.0)
-        new_point = ("fidelity", 0.5)
+        fidelity = 1
+
+        duplicate_trial = create_trial_for_hb((fidelity, 0.0))
+        new_trial = create_trial_for_hb((fidelity, 0.5))
 
-        duplicate_id = hashlib.md5(str([duplicate_point]).encode("utf-8")).hexdigest()
-        bracket.rungs[0]["results"] = {duplicate_id: (0.0, duplicate_point)}
+        duplicate_id = hyperband.get_id(duplicate_trial, ignore_fidelity=True)
+        bracket.rungs[0]["results"] = {duplicate_id: (0.0, duplicate_trial)}
 
         hyperband.trial_to_brackets[
-            hyperband.get_id(duplicate_point, ignore_fidelity=True)
+            hyperband.get_id(duplicate_trial, ignore_fidelity=True)
         ] = bracket
 
-        points = [duplicate_point, new_point]
+        trials = [duplicate_trial, new_trial]
 
-        mock_samples(hyperband, points + [("fidelity", i) for i in range(10 - 2)])
+        mock_samples(
+            hyperband,
+            trials + [create_trial_for_hb((fidelity, i)) for i in range(10 - 2)],
+        )
 
-        assert hyperband.suggest(100)[0][1] == new_point[1]
+        assert hyperband.suggest(100)[0].params == new_trial.params
 
     def test_suggest_duplicates_one_call(self, monkeypatch, hyperband, bracket):
         """Test that same points are not allowed in the same suggest call ofxs
@@ -413,36 +439,43 @@ def test_suggest_duplicates_one_call(self, monkeypatch, hyperband, bracket):
         hyperband.brackets = [bracket]
         bracket.hyperband = hyperband
 
-        zhe_point = [(1, 0.0), (1, 1.0), (1, 1.0), (1, 2.0)]
+        zhe_point = list(
+            map(create_trial_for_hb, [(1, 0.0), (1, 1.0), (1, 1.0), (1, 2.0)])
+        )
 
         mock_samples(hyperband, zhe_point * 2)
         zhe_samples = hyperband.suggest(100)
 
-        assert zhe_samples[0][1] == 0.0
-        assert zhe_samples[1][1] == 1.0
-        assert zhe_samples[2][1] == 2.0
+        assert zhe_samples[0].params["lr"] == 0.0
+        assert zhe_samples[1].params["lr"] == 1.0
+        assert zhe_samples[2].params["lr"] == 2.0
 
         # zhe_point =
         mock_samples(
             hyperband,
-            [
-                (3, 0.0),
-                (3, 1.0),
-                (3, 1.0),
-                (3, 2.0),
-                (3, 5.0),
-                (3, 4.0),
-            ],
+            list(
+                map(
+                    create_trial_for_hb,
+                    [
+                        (3, 0.0),
+                        (3, 1.0),
+                        (3, 1.0),
+                        (3, 2.0),
+                        (3, 5.0),
+                        (3, 4.0),
+                    ],
+                )
+            ),
         )
         hyperband.trial_to_brackets[
-            hyperband.get_id((1, 0.0), ignore_fidelity=True)
+            hyperband.get_id(create_trial_for_hb((1, 0.0)), ignore_fidelity=True)
         ] = bracket
         hyperband.trial_to_brackets[
-            hyperband.get_id((1, 1.0), ignore_fidelity=True)
+            hyperband.get_id(create_trial_for_hb((1, 0.0)), ignore_fidelity=True)
         ] = bracket
         zhe_samples = hyperband.suggest(100)
-        assert zhe_samples[0][1] == 5.0
-        assert zhe_samples[1][1] == 4.0
+        assert zhe_samples[0].params["lr"] == 5.0
+        assert zhe_samples[1].params["lr"] == 4.0
 
     def test_suggest_duplicates_between_execution(
         self, monkeypatch, hyperband, budgets
@@ -454,21 +487,22 @@ def test_suggest_duplicates_between_execution(
         bracket.hyperband = hyperband
 
         for i in range(9):
-            force_observe(hyperband, (1, i), {"objective": i})
+            force_observe(hyperband, create_trial_for_hb((1, i), objective=i))
 
         for i in range(3):
-            force_observe(hyperband, (3, i), {"objective": i})
+            force_observe(hyperband, create_trial_for_hb((3, i), objective=i))
 
-        force_observe(hyperband, (9, 0), {"objective": 0})
+        force_observe(hyperband, create_trial_for_hb((9, 0), objective=0))
 
         assert not hyperband.is_done
 
-        zhe_point = [(9, 0), (9, 1), (9, 2)]
+        zhe_point = list(map(create_trial_for_hb, [(9, 0), (9, 1), (9, 2)]))
 
         hyperband._refresh_brackets()
         mock_samples(hyperband, zhe_point * 2)
         zhe_samples = hyperband.suggest(100)
-        assert zhe_samples == [(9, 1), (9, 2)]
+        assert zhe_samples[0].params == {"epoch": 9, "lr": 1}
+        assert zhe_samples[1].params == {"epoch": 9, "lr": 2}
 
     def test_suggest_inf_duplicates(
         self, monkeypatch, hyperband, bracket, rung_0, rung_1, rung_2
@@ -477,12 +511,12 @@ def test_suggest_inf_duplicates(
         hyperband.brackets = [bracket]
         bracket.hyperband = hyperband
 
-        zhe_point = ("fidelity", 0.0)
+        zhe_trial = create_trial_for_hb(("fidelity", 0.0))
         hyperband.trial_to_brackets[
-            hyperband.get_id(zhe_point, ignore_fidelity=True)
+            hyperband.get_id(zhe_trial, ignore_fidelity=True)
         ] = bracket
 
-        mock_samples(hyperband, [zhe_point] * 2)
+        mock_samples(hyperband, [zhe_trial] * 2)
 
         assert hyperband.suggest(100) == []
 
@@ -494,7 +528,15 @@ def test_suggest_in_finite_cardinality(self):
 
         hyperband = Hyperband(space, repetitions=1)
         for i in range(6):
-            force_observe(hyperband, (1, i), {"objective": i})
+            force_observe(
+                hyperband,
+                create_trial(
+                    (1, i),
+                    names=("epoch", "yolo1"),
+                    types=("fidelity", "integer"),
+                    results={"objective": i},
+                ),
+            )
 
         assert hyperband.suggest(100) == []
 
@@ -506,7 +548,10 @@ def test_suggest_promote(self, hyperband, bracket, rung_0):
 
         points = hyperband.suggest(100)
 
-        assert points == [(3, i) for i in range(3)]
+        assert len(points) == 3
+        assert points[0].params == {"epoch": 3, "lr": 0}
+        assert points[1].params == {"epoch": 3, "lr": 1}
+        assert points[2].params == {"epoch": 3, "lr": 2}
 
     def test_is_filled(self, hyperband, bracket, rung_0, rung_1, rung_2):
         """Test that Hyperband bracket detects when rung is filled."""
@@ -637,20 +682,20 @@ def test_suggest_opt_out(self, hyperband, bracket, rung_0, rung_1, rung_2):
 
     def test_full_process(self, monkeypatch, hyperband):
         """Test Hyperband full process."""
-        sample_points = [("fidelity", i) for i in range(100)]
+        sample_trials = [create_trial_for_hb(("fidelity", i)) for i in range(100)]
 
         hyperband._refresh_brackets()
-        mock_samples(hyperband, copy.deepcopy(sample_points))
+        mock_samples(hyperband, copy.deepcopy(sample_trials))
 
         # Fill all brackets' first rung
 
-        points = hyperband.suggest(100)
+        trials = hyperband.suggest(100)
         from orion.algo.hyperband import tabulate_status
 
         print(tabulate_status(hyperband.brackets))
-        assert points[:3] == [(9, i) for i in range(3)]
-        assert points[3:6] == [(3, i) for i in range(3, 6)]
-        assert points[6:] == [(1, i) for i in range(6, 15)]
+        compare_trials(trials[:3], [create_trial_for_hb((9, i)) for i in range(3)])
+        compare_trials(trials[3:6], [create_trial_for_hb((3, i)) for i in range(3, 6)])
+        compare_trials(trials[6:], [create_trial_for_hb((1, i)) for i in range(6, 15)])
 
         assert hyperband.brackets[0].has_rung_filled(0)
         assert not hyperband.brackets[0].is_ready()
@@ -660,15 +705,17 @@ def test_full_process(self, monkeypatch, hyperband):
         # Observe first bracket first rung
 
         for i in range(9):
-            hyperband.observe([(1, i + 3 + 3)], [{"objective": 16 - i}])
+            hyperband.observe([create_trial_for_hb((1, i + 3 + 3), objective=16 - i)])
 
         assert hyperband.brackets[0].is_ready()
         assert not hyperband.brackets[1].is_ready()
         assert not hyperband.brackets[2].is_ready()
 
         # Promote first bracket first rung
-        points = hyperband.suggest(100)
-        assert points == [(3, 3 + 3 + 9 - 1 - i) for i in range(3)]
+        trials = hyperband.suggest(100)
+        compare_trials(
+            trials, [create_trial_for_hb((3, 3 + 3 + 9 - 1 - i)) for i in range(3)]
+        )
 
         assert hyperband.brackets[0].has_rung_filled(1)
         assert not hyperband.brackets[0].is_ready()
@@ -677,15 +724,17 @@ def test_full_process(self, monkeypatch, hyperband):
 
         # Observe first bracket second rung
         for i in range(3):
-            hyperband.observe([(3, 3 + 3 + 9 - 1 - i)], [{"objective": 8 - i}])
+            hyperband.observe(
+                [create_trial_for_hb((3, 3 + 3 + 9 - 1 - i), objective=8 - i)]
+            )
 
         assert hyperband.brackets[0].is_ready()
         assert not hyperband.brackets[1].is_ready()
         assert not hyperband.brackets[2].is_ready()
 
         # Promote first bracket second rung
-        points = hyperband.suggest(100)
-        assert points == [(9, 12)]
+        trials = hyperband.suggest(100)
+        compare_trials(trials, [create_trial_for_hb((9, 12))])
 
         assert hyperband.brackets[0].has_rung_filled(2)
         assert not hyperband.brackets[0].is_ready()
@@ -694,15 +743,15 @@ def test_full_process(self, monkeypatch, hyperband):
 
         # Observe second bracket first rung
         for i in range(3):
-            hyperband.observe([(3, i + 3)], [{"objective": 8 - i}])
+            hyperband.observe([create_trial_for_hb((3, i + 3), objective=8 - i)])
 
         assert not hyperband.brackets[0].is_ready()
         assert hyperband.brackets[1].is_ready()
         assert not hyperband.brackets[2].is_ready()
 
         # Promote second bracket first rung
-        points = hyperband.suggest(100)
-        assert points == [(9, 5)]
+        trials = hyperband.suggest(100)
+        compare_trials(trials, [create_trial_for_hb((9, 5))])
 
         assert not hyperband.brackets[0].is_ready()
         assert hyperband.brackets[1].has_rung_filled(1)
@@ -711,7 +760,7 @@ def test_full_process(self, monkeypatch, hyperband):
 
         # Observe third bracket first rung
         for i in range(3):
-            hyperband.observe([(9, i)], [{"objective": 3 - i}])
+            hyperband.observe([create_trial_for_hb((9, i), objective=3 - i)])
 
         assert not hyperband.brackets[0].is_ready(2)
         assert not hyperband.brackets[1].is_ready(1)
@@ -720,14 +769,16 @@ def test_full_process(self, monkeypatch, hyperband):
 
         # Observe second bracket second rung
         for i in range(1):
-            hyperband.observe([(9, 3 + 3 - 1 - i)], [{"objective": 5 - i}])
+            hyperband.observe(
+                [create_trial_for_hb((9, 3 + 3 - 1 - i), objective=5 - i)]
+            )
 
         assert not hyperband.brackets[0].is_ready(2)
         assert hyperband.brackets[1].is_ready(1)
         assert hyperband.brackets[1].is_done
 
         # Observe first bracket third rung
-        hyperband.observe(points, [{"objective": 3 - i}])
+        hyperband.observe(trials)
 
         assert hyperband.is_done
         assert hyperband.brackets[0].is_done
@@ -738,14 +789,14 @@ def test_full_process(self, monkeypatch, hyperband):
         # monkeypatch.setattr(hyperband.brackets[0], "repetition_id", 0)
         # hyperband.observe([(9, 12)], [{"objective": 3 - i}])
         hyperband._refresh_brackets()
-        mock_samples(hyperband, copy.deepcopy(sample_points))
-        points = hyperband.suggest(100)
+        mock_samples(hyperband, copy.deepcopy(sample_trials))
+        trials = hyperband.suggest(100)
         assert not hyperband.is_done
         assert not hyperband.brackets[0].is_ready(2)
         assert not hyperband.brackets[0].is_done
-        assert points[:3] == [(9, 3), (9, 4), (9, 6)]
-        assert points[3:6] == [(3, 7), (3, 8), (3, 9)]
-        assert points[6:] == [(1, i) for i in range(15, 24)]
+        compare_trials(trials[:3], map(create_trial_for_hb, [(9, 3), (9, 4), (9, 6)]))
+        compare_trials(trials[3:6], map(create_trial_for_hb, [(3, 7), (3, 8), (3, 9)]))
+        compare_trials(trials[6:], [create_trial_for_hb((1, i)) for i in range(15, 24)])
 
 
 class TestGenericHyperband(BaseAlgoTests):
@@ -822,11 +873,11 @@ def test_is_done_max_trials(self, num):
 
         objective = 0
         while not algo.is_done:
-            points = algo.suggest(num)
-            assert points is not None
-            if points:
-                self.observe_points(points, algo, objective)
-                objective += len(points)
+            trials = algo.suggest(num)
+            assert trials is not None
+            if trials:
+                self.observe_trials(trials, algo, objective)
+                objective += len(trials)
 
         # Hyperband should ignore max trials.
         assert algo.n_observed > MAX_TRIALS

From 7c42f8c5b9b574ec6f7c0d8ae0bb841aa6a12f55 Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Tue, 9 Nov 2021 15:24:14 -0500
Subject: [PATCH 17/34] Adapt ASHA to new observe interface

---
 src/orion/algo/asha.py                 |  42 +++--
 src/orion/algo/evolution_es.py         |   4 +-
 tests/unittests/algo/test_asha.py      | 251 ++++++++++++-------------
 tests/unittests/algo/test_hyperband.py |   5 +-
 4 files changed, 152 insertions(+), 150 deletions(-)

diff --git a/src/orion/algo/asha.py b/src/orion/algo/asha.py
index 0923fc313..326da17b2 100644
--- a/src/orion/algo/asha.py
+++ b/src/orion/algo/asha.py
@@ -218,14 +218,20 @@ def sample(self, num):
         return self.hyperband.sample_for_bracket(num, self)
 
     def get_candidate(self, rung_id):
-        """Get a candidate for promotion"""
+        """Get a candidate for promotion
+
+        Raises
+        ------
+        TypeError
+            If get_candidates is called before the entire rung is completed.
+        """
         rung = self.rungs[rung_id]["results"]
         next_rung = self.rungs[rung_id + 1]["results"]
 
         rung = list(
             sorted(
-                (objective, point)
-                for objective, point in rung.values()
+                (objective, trial)
+                for objective, trial in rung.values()
                 if objective is not None
             )
         )
@@ -233,16 +239,16 @@ def get_candidate(self, rung_id):
         k = min(k, len(rung))
 
         for i in range(k):
-            point = rung[i][1]
-            _id = self.hyperband.get_id(point, ignore_fidelity=True)
+            trial = rung[i][1]
+            _id = self.hyperband.get_id(trial, ignore_fidelity=True)
             if _id not in next_rung:
-                return point
+                return trial
 
         return None
 
     @property
     def is_filled(self):
-        """ASHA's first rung can always sample new points"""
+        """ASHA's first rung can always sample new trials"""
         return False
 
     def is_ready(self, rung_id=None):
@@ -254,7 +260,7 @@ def promote(self, num):
 
         The rungs are iterated over in reversed order, so that high rungs
         are prioritised for promotions. When a candidate is promoted, the loop is broken and
-        the method returns the promoted point.
+        the method returns the promoted trial.
 
         .. note ::
 
@@ -272,21 +278,25 @@ def promote(self, num):
 
                 # pylint: disable=logging-format-interpolation
                 logger.debug(
-                    "Promoting {point} from rung {past_rung} with fidelity {past_fidelity} to "
+                    "Promoting {trial} from rung {past_rung} with fidelity {past_fidelity} to "
                     "rung {new_rung} with fidelity {new_fidelity}".format(
-                        point=candidate,
+                        trial=candidate,
                         past_rung=rung_id,
-                        past_fidelity=candidate[self.hyperband.fidelity_index],
+                        past_fidelity=candidate.params[self.hyperband.fidelity_index],
                         new_rung=rung_id + 1,
                         new_fidelity=self.rungs[rung_id + 1]["resources"],
                     )
                 )
 
-                candidate = list(copy.deepcopy(candidate))
-                candidate[self.hyperband.fidelity_index] = self.rungs[rung_id + 1][
-                    "resources"
-                ]
+                candidate = candidate.branch(
+                    status="new",
+                    params={
+                        self.hyperband.fidelity_index: self.rungs[rung_id + 1][
+                            "resources"
+                        ]
+                    },
+                )
 
-                return [tuple(candidate)]
+                return [candidate]
 
         return []
diff --git a/src/orion/algo/evolution_es.py b/src/orion/algo/evolution_es.py
index ddd8ae73a..3578a2be7 100644
--- a/src/orion/algo/evolution_es.py
+++ b/src/orion/algo/evolution_es.py
@@ -181,8 +181,8 @@ def set_state(self, state_dict):
         self.performance = state_dict["performance"]
         self.hurdles = state_dict["hurdles"]
 
-    def _get_bracket(self, point):
-        """Get the bracket of a point during observe"""
+    def _get_bracket(self, trial):
+        """Get the bracket of a trial during observe"""
         return self.brackets[0]
 
 
diff --git a/tests/unittests/algo/test_asha.py b/tests/unittests/algo/test_asha.py
index e7f0e42e8..15df89d93 100644
--- a/tests/unittests/algo/test_asha.py
+++ b/tests/unittests/algo/test_asha.py
@@ -10,6 +10,14 @@
 from orion.algo.asha import ASHA, ASHABracket, compute_budgets
 from orion.algo.space import Fidelity, Integer, Real, Space
 from orion.testing.algo import BaseAlgoTests
+from orion.testing.trial import create_trial
+
+from test_hyperband import (
+    compare_registered_trial,
+    create_rung_from_points,
+    create_trial_for_hb,
+    force_observe,
+)
 
 
 @pytest.fixture
@@ -53,57 +61,21 @@ def bracket(b_config, asha):
 @pytest.fixture
 def rung_0():
     """Create fake points and objectives for rung 0."""
-    points = np.linspace(0, 1, 9)
-    return dict(
-        n_trials=9,
-        resources=1,
-        results={
-            hashlib.md5(str([point]).encode("utf-8")).hexdigest(): (point, (1, point))
-            for point in points
-        },
-    )
+    return create_rung_from_points(np.linspace(0, 8, 9), n_trials=9, resources=1)
 
 
 @pytest.fixture
 def rung_1(rung_0):
     """Create fake points and objectives for rung 1."""
-    return dict(
-        n_trials=9,
-        resources=3,
-        results={
-            hashlib.md5(str([value[0]]).encode("utf-8")).hexdigest(): value
-            for value in map(
-                lambda v: (v[0], (3, v[0])), sorted(rung_0["results"].values())
-            )
-        },
-    )
+    points = [trial.params["lr"] for _, trial in sorted(rung_0["results"].values())[:3]]
+    return create_rung_from_points(points, n_trials=3, resources=3)
 
 
 @pytest.fixture
 def rung_2(rung_1):
     """Create fake points and objectives for rung 1."""
-    return dict(
-        n_trials=9,
-        resources=9,
-        results={
-            hashlib.md5(str([value[0]]).encode("utf-8")).hexdigest(): value
-            for value in map(
-                lambda v: (v[0], (9, v[0])), sorted(rung_1["results"].values())
-            )
-        },
-    )
-
-
-def force_observe(asha, point, results):
-
-    full_id = asha.get_id(point, ignore_fidelity=False)
-    asha.register(point, results)
-
-    bracket = asha._get_bracket(point)
-    id_wo_fidelity = asha.get_id(point, ignore_fidelity=True)
-    asha.trial_to_brackets[id_wo_fidelity] = bracket
-
-    asha.observe([point], [results])
+    points = [trial.params["lr"] for _, trial in sorted(rung_1["results"].values())[:1]]
+    return create_rung_from_points(points, n_trials=1, resources=9)
 
 
 def test_compute_budgets():
@@ -162,21 +134,21 @@ def test_rungs_creation(self, bracket):
     def test_register(self, asha, bracket):
         """Check that a point is correctly registered inside a bracket."""
         bracket.asha = asha
-        point = (1, 0.0)
-        point_hash = hashlib.md5(str([0.0]).encode("utf-8")).hexdigest()
+        trial = create_trial_for_hb((1, 0.0), 0.0)
+        trial_id = asha.get_id(trial, ignore_fidelity=True)
 
-        bracket.register(point, 0.0)
+        bracket.register(trial)
 
         assert len(bracket.rungs[0])
-        assert point_hash in bracket.rungs[0]["results"]
-        assert (0.0, point) == bracket.rungs[0]["results"][point_hash]
+        assert trial_id in bracket.rungs[0]["results"]
+        assert (trial.objective.value, trial) == bracket.rungs[0]["results"][trial_id]
 
     def test_bad_register(self, asha, bracket):
         """Check that a non-valid point is not registered."""
         bracket.asha = asha
 
         with pytest.raises(IndexError) as ex:
-            bracket.register((55, 0.0), 0.0)
+            bracket.register(create_trial_for_hb((55, 0.0), 0.0))
 
         assert "Bad fidelity level 55" in str(ex.value)
 
@@ -187,19 +159,20 @@ def test_candidate_promotion(self, asha, bracket, rung_0):
 
         point = bracket.get_candidate(0)
 
-        assert point == (1, 0.0)
+        assert point.params == create_trial_for_hb((1, 0.0), 0.0).params
 
     def test_promotion_with_rung_1_hit(self, asha, bracket, rung_0):
         """Test that get_candidate gives us the next best thing if point is already in rung 1."""
-        point = (1, 0.0)
-        point_hash = hashlib.md5(str([0.0]).encode("utf-8")).hexdigest()
+        trial = create_trial_for_hb((1, 0.0), None)
         bracket.asha = asha
         bracket.rungs[0] = rung_0
-        bracket.rungs[1]["results"][point_hash] = (0.0, point)
-
-        point = bracket.get_candidate(0)
+        bracket.rungs[1]["results"][asha.get_id(trial, ignore_fidelity=True)] = (
+            trial.objective.value,
+            trial,
+        )
 
-        assert point == (1, 0.125)
+        trial = bracket.get_candidate(0)
+        assert trial.params == create_trial_for_hb((1, 1.0), 0.0).params
 
     def test_no_promotion_when_rung_full(self, asha, bracket, rung_0, rung_1):
         """Test that get_candidate returns `None` if rung 1 is full."""
@@ -254,13 +227,14 @@ def test_update_rungs_return_candidate(self, asha, bracket, rung_1):
         """Check if a valid modified candidate is returned by update_rungs."""
         bracket.asha = asha
         bracket.rungs[1] = rung_1
-        point_hash = hashlib.md5(str([0.0]).encode("utf-8")).hexdigest()
+        trial = create_trial_for_hb((3, 0.0), 0.0)
 
         candidate = bracket.promote(1)[0]
 
-        assert point_hash in bracket.rungs[1]["results"]
-        assert bracket.rungs[1]["results"][point_hash] == (0.0, (3, 0.0))
-        assert candidate[0] == 9
+        trial_id = asha.get_id(trial, ignore_fidelity=True)
+        assert trial_id in bracket.rungs[1]["results"]
+        assert bracket.rungs[1]["results"][trial_id][1].params == trial.params
+        assert candidate.params["epoch"] == 9
 
     def test_update_rungs_return_no_candidate(self, asha, bracket, rung_1):
         """Check if no candidate is returned by update_rungs."""
@@ -287,14 +261,15 @@ def test_register(self, asha, bracket, rung_0, rung_1):
         asha.brackets = [bracket]
         bracket.asha = asha
         bracket.rungs = [rung_0, rung_1]
-        point = (1, 0.0)
-        point_hash = hashlib.md5(str([0.0]).encode("utf-8")).hexdigest()
+        trial = create_trial_for_hb((1, 0.0), 0.0)
+        trial_id = asha.get_id(trial, ignore_fidelity=True)
 
-        asha.observe([point], [{"objective": 0.0}])
+        asha.observe([trial])
 
         assert len(bracket.rungs[0])
-        assert point_hash in bracket.rungs[0]["results"]
-        assert (0.0, point) == bracket.rungs[0]["results"][point_hash]
+        assert trial_id in bracket.rungs[0]["results"]
+        assert bracket.rungs[0]["results"][trial_id][0] == 0.0
+        assert bracket.rungs[0]["results"][trial_id][1].params == trial.params
 
     def test_register_bracket_multi_fidelity(self, space, b_config):
         """Check that a point is registered inside the same bracket for diff fidelity."""
@@ -302,27 +277,29 @@ def test_register_bracket_multi_fidelity(self, space, b_config):
 
         value = 50
         fidelity = 1
-        point = (fidelity, value)
-        point_hash = hashlib.md5(str([value]).encode("utf-8")).hexdigest()
+        trial = create_trial_for_hb((fidelity, value), 0.0)
+        trial_id = asha.get_id(trial, ignore_fidelity=True)
 
-        force_observe(asha, point, {"objective": 0.0})
+        force_observe(asha, trial)
 
         bracket = asha.brackets[0]
 
         assert len(bracket.rungs[0])
-        assert point_hash in bracket.rungs[0]["results"]
-        assert (0.0, point) == bracket.rungs[0]["results"][point_hash]
+        assert trial_id in bracket.rungs[0]["results"]
+        assert bracket.rungs[0]["results"][trial_id][0] == 0.0
+        assert bracket.rungs[0]["results"][trial_id][1].params == trial.params
 
         fidelity = 3
-        point = [fidelity, value]
-        point_hash = hashlib.md5(str([value]).encode("utf-8")).hexdigest()
+        trial = create_trial_for_hb((fidelity, value), 0.0)
+        trial_id = asha.get_id(trial, ignore_fidelity=True)
 
-        force_observe(asha, point, {"objective": 0.0})
+        force_observe(asha, trial)
 
-        assert len(bracket.rungs[0])
-        assert point_hash in bracket.rungs[1]["results"]
-        assert (0.0, point) != bracket.rungs[0]["results"][point_hash]
-        assert (0.0, point) == bracket.rungs[1]["results"][point_hash]
+        assert len(bracket.rungs[1])
+        assert trial_id in bracket.rungs[1]["results"]
+        assert bracket.rungs[0]["results"][trial_id][1].params != trial.params
+        assert bracket.rungs[1]["results"][trial_id][0] == 0.0
+        assert bracket.rungs[1]["results"][trial_id][1].params == trial.params
 
     def test_register_next_bracket(self, space, b_config):
         """Check that a point is registered inside the good bracket when higher fidelity."""
@@ -330,29 +307,29 @@ def test_register_next_bracket(self, space, b_config):
 
         value = 50
         fidelity = 3
-        point = (fidelity, value)
-        point_hash = hashlib.md5(str([value]).encode("utf-8")).hexdigest()
+        trial = create_trial_for_hb((fidelity, value), 0.0)
+        trial_id = asha.get_id(trial, ignore_fidelity=True)
 
-        force_observe(asha, point, {"objective": 0.0})
+        force_observe(asha, trial)
 
         assert sum(len(rung["results"]) for rung in asha.brackets[0].rungs) == 0
         assert sum(len(rung["results"]) for rung in asha.brackets[1].rungs) == 1
         assert sum(len(rung["results"]) for rung in asha.brackets[2].rungs) == 0
-        assert point_hash in asha.brackets[1].rungs[0]["results"]
-        assert (0.0, point) == asha.brackets[1].rungs[0]["results"][point_hash]
+        assert trial_id in asha.brackets[1].rungs[0]["results"]
+        compare_registered_trial(asha.brackets[1].rungs[0]["results"][trial_id], trial)
 
         value = 51
         fidelity = 9
-        point = (fidelity, value)
-        point_hash = hashlib.md5(str([value]).encode("utf-8")).hexdigest()
+        trial = create_trial_for_hb((fidelity, value), 0.0)
+        trial_id = asha.get_id(trial, ignore_fidelity=True)
 
-        force_observe(asha, point, {"objective": 0.0})
+        force_observe(asha, trial)
 
         assert sum(len(rung["results"]) for rung in asha.brackets[0].rungs) == 0
         assert sum(len(rung["results"]) for rung in asha.brackets[1].rungs) == 1
         assert sum(len(rung["results"]) for rung in asha.brackets[2].rungs) == 1
-        assert point_hash in asha.brackets[2].rungs[0]["results"]
-        assert (0.0, point) == asha.brackets[2].rungs[0]["results"][point_hash]
+        assert trial_id in asha.brackets[2].rungs[0]["results"]
+        compare_registered_trial(asha.brackets[2].rungs[0]["results"][trial_id], trial)
 
     def test_register_invalid_fidelity(self, space, b_config):
         """Check that a point cannot registered if fidelity is invalid."""
@@ -360,12 +337,12 @@ def test_register_invalid_fidelity(self, space, b_config):
 
         value = 50
         fidelity = 2
-        point = (fidelity, value)
+        trial = create_trial_for_hb((fidelity, value))
 
         with pytest.raises(ValueError) as ex:
-            force_observe(asha, point, {"objective": 0.0})
+            force_observe(asha, trial)
 
-        assert "No bracket found for point" in str(ex.value)
+        assert "No bracket found for trial" in str(ex.value)
 
     def test_register_not_sampled(self, space, b_config, caplog):
         """Check that a point cannot registered if not sampled."""
@@ -373,13 +350,13 @@ def test_register_not_sampled(self, space, b_config, caplog):
 
         value = 50
         fidelity = 2
-        point = (fidelity, value)
+        trial = create_trial_for_hb((fidelity, value))
 
         with caplog.at_level(logging.INFO, logger="orion.algo.hyperband"):
-            asha.observe([point], [{"objective": 0.0}])
+            asha.observe([trial])
 
         assert len(caplog.records) == 1
-        assert "Ignoring point" in caplog.records[0].msg
+        assert "Ignoring trial" in caplog.records[0].msg
 
     def test_register_corrupted_db(self, caplog, space, b_config):
         """Check that a point cannot registered if passed in order diff than fidelity."""
@@ -387,17 +364,17 @@ def test_register_corrupted_db(self, caplog, space, b_config):
 
         value = 50
         fidelity = 3
-        point = (fidelity, value)
+        trial = create_trial_for_hb((fidelity, value))
 
-        force_observe(asha, point, {"objective": 0.0})
-        assert "Point registered to wrong bracket" not in caplog.text
+        force_observe(asha, trial)
+        assert "Trial registered to wrong bracket" not in caplog.text
 
         fidelity = 1
-        point = [fidelity, value]
+        trial = create_trial_for_hb((fidelity, value), objective=0.0)
 
         caplog.clear()
-        force_observe(asha, point, {"objective": 0.0})
-        assert "Point registered to wrong bracket" in caplog.text
+        force_observe(asha, trial)
+        assert "Trial registered to wrong bracket" in caplog.text
 
     def test_suggest_new(self, monkeypatch, asha, bracket, rung_0, rung_1, rung_2):
         """Test that a new point is sampled."""
@@ -405,13 +382,13 @@ def test_suggest_new(self, monkeypatch, asha, bracket, rung_0, rung_1, rung_2):
         bracket.asha = asha
 
         def sample(num=1, seed=None):
-            return [("fidelity", 0.5)]
+            return [create_trial_for_hb(("fidelity", 0.5))]
 
         monkeypatch.setattr(asha.space, "sample", sample)
 
-        points = asha.suggest(1)
+        trials = asha.suggest(1)
 
-        assert points == [(1, 0.5)]
+        assert trials[0].params == {"epoch": 1, "lr": 0.5}
 
     def test_suggest_duplicates(
         self, monkeypatch, asha, bracket, rung_0, rung_1, rung_2
@@ -420,27 +397,28 @@ def test_suggest_duplicates(
         asha.brackets = [bracket]
         bracket.asha = asha
 
-        duplicate_point = ("fidelity", 0.0)
-        new_point = ("fidelity", 0.5)
+        fidelity = 1
+        duplicate_trial = create_trial_for_hb((fidelity, 0.0))
+        new_trial = create_trial_for_hb((fidelity, 0.5))
 
-        duplicate_id_wo_fidelity = asha.get_id(duplicate_point, ignore_fidelity=True)
+        duplicate_id_wo_fidelity = asha.get_id(duplicate_trial, ignore_fidelity=True)
         bracket.rungs[0] = dict(
             n_trials=2,
             resources=1,
-            results={duplicate_id_wo_fidelity: (0.0, duplicate_point)},
+            results={duplicate_id_wo_fidelity: (0.0, duplicate_trial)},
         )
         asha.trial_to_brackets[duplicate_id_wo_fidelity] = bracket
 
-        asha.register(duplicate_point, 0.0)
+        asha.register(duplicate_trial)
 
-        points = [duplicate_point, new_point]
+        trials = [duplicate_trial, new_trial]
 
         def sample(num=1, seed=None):
-            return points
+            return trials
 
         monkeypatch.setattr(asha.space, "sample", sample)
 
-        assert asha.suggest(1)[0][1] == new_point[1]
+        assert asha.suggest(1)[0].params == new_trial.params
 
     def test_suggest_inf_duplicates(
         self, monkeypatch, asha, bracket, rung_0, rung_1, rung_2
@@ -449,11 +427,12 @@ def test_suggest_inf_duplicates(
         asha.brackets = [bracket]
         bracket.asha = asha
 
-        zhe_point = ("fidelity", 0.0)
-        asha.trial_to_brackets[asha.get_id(zhe_point, ignore_fidelity=True)] = bracket
+        fidelity = 1
+        zhe_trial = create_trial_for_hb((fidelity, 0.0))
+        asha.trial_to_brackets[asha.get_id(zhe_trial, ignore_fidelity=True)] = bracket
 
         def sample(num=1, seed=None):
-            return [zhe_point]
+            return [zhe_trial]
 
         monkeypatch.setattr(asha.space, "sample", sample)
 
@@ -467,10 +446,26 @@ def test_suggest_in_finite_cardinality(self):
 
         asha = ASHA(space)
         for i in range(6):
-            force_observe(asha, (1, i), {"objective": i})
+            force_observe(
+                asha,
+                create_trial(
+                    (1, i),
+                    names=("epoch", "yolo1"),
+                    types=("fidelity", "integer"),
+                    results={"objective": i},
+                ),
+            )
 
         for i in range(2):
-            force_observe(asha, (3, i), {"objective": i})
+            force_observe(
+                asha,
+                create_trial(
+                    (3, i),
+                    names=("epoch", "yolo1"),
+                    types=("fidelity", "integer"),
+                    results={"objective": i},
+                ),
+            )
 
         assert asha.suggest(1) == []
 
@@ -480,9 +475,9 @@ def test_suggest_promote(self, asha, bracket, rung_0):
         bracket.asha = asha
         bracket.rungs[0] = rung_0
 
-        points = asha.suggest(1)
+        trials = asha.suggest(1)
 
-        assert points == [(3, 0.0)]
+        assert trials[0].params == {"epoch": 3, "lr": 0.0}
 
 
 class TestGenericASHA(BaseAlgoTests):
@@ -498,8 +493,8 @@ class TestGenericASHA(BaseAlgoTests):
     def test_suggest_n(self, mocker, num, attr):
         algo = self.create_algo()
         spy = self.spy_phase(mocker, num, algo, attr)
-        points = algo.suggest(5)
-        assert len(points) == 1
+        trials = algo.suggest(5)
+        assert len(trials) == 1
 
     @pytest.mark.skip(reason="See https://github.com/Epistimio/orion/issues/598")
     def test_is_done_cardinality(self):
@@ -518,15 +513,15 @@ def test_is_done_cardinality(self):
         for rung in range(algo.algorithm.num_rungs):
             assert not algo.is_done
 
-            points = []
+            trials = []
             while True:
                 assert not algo.is_done
                 n_sampled = len(algo.algorithm.sampled)
                 n_trials = len(algo.algorithm.trial_to_brackets)
-                new_points = algo.suggest(1)
-                if new_points is None:
+                new_trials = algo.suggest(1)
+                if new_trials is None:
                     break
-                points += new_points
+                trials += new_trials
                 if rung == 0:
                     assert len(algo.algorithm.sampled) == n_sampled + 1
                 else:
@@ -535,8 +530,8 @@ def test_is_done_cardinality(self):
 
             assert not algo.is_done
 
-            for i, point in enumerate(points):
-                algo.observe([point], [dict(objective=i)])
+            for i, trial in enumerate(trials):
+                algo.observe([trial], [dict(objective=i)])
 
         assert algo.is_done
 
@@ -549,11 +544,11 @@ def test_is_done_max_trials(self):
 
         objective = 0
         while not algo.is_done:
-            points = algo.suggest(1)
-            assert points is not None
-            if points:
-                self.observe_points(points, algo, objective)
-                objective += len(points)
+            trials = algo.suggest(1)
+            assert trials is not None
+            if trials:
+                self.observe_trials(trials, algo, objective)
+                objective += len(trials)
 
         # ASHA should ignore max trials.
         assert algo.n_observed > MAX_TRIALS
diff --git a/tests/unittests/algo/test_hyperband.py b/tests/unittests/algo/test_hyperband.py
index bc2fc6b67..4df471ff7 100644
--- a/tests/unittests/algo/test_hyperband.py
+++ b/tests/unittests/algo/test_hyperband.py
@@ -221,7 +221,6 @@ def test_update_rungs_return_candidate(self, hyperband, bracket, rung_1):
         bracket.hyperband = hyperband
         bracket.rungs[1] = rung_1
         trial = create_trial_for_hb((3, 0.0), 0.0)
-        # point_hash = hashlib.md5(str([0.0]).encode("utf-8")).hexdigest()
 
         candidates = bracket.promote(1)
 
@@ -419,9 +418,7 @@ def test_suggest_duplicates_between_calls(self, monkeypatch, hyperband, bracket)
         duplicate_id = hyperband.get_id(duplicate_trial, ignore_fidelity=True)
         bracket.rungs[0]["results"] = {duplicate_id: (0.0, duplicate_trial)}
 
-        hyperband.trial_to_brackets[
-            hyperband.get_id(duplicate_trial, ignore_fidelity=True)
-        ] = bracket
+        hyperband.trial_to_brackets[duplicate_id] = bracket
 
         trials = [duplicate_trial, new_trial]
 

From 30c6337ba3c289d271b33068653fc744c6926ddf Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Wed, 10 Nov 2021 15:35:22 -0500
Subject: [PATCH 18/34] Get rid of TransformedTrial

Why:

The wrapper makes it cumbersome to keep a coherent interface with the
base Trial object. Also it does not bring much value beside providing
access to the original trial and its corresponding id. Algorithms can
revert the trial to access the trial id.

How:

Return a trial object with params overwritten instead. Like it was
already done when reverting a transformed trial anyway. Now base
algo.get_id() method compute the trial hash based on the reverted trial.
---
 src/orion/algo/base.py                   |  7 +-
 src/orion/core/worker/transformer.py     | 55 +++-------------
 tests/unittests/core/test_transformer.py | 84 +-----------------------
 3 files changed, 17 insertions(+), 129 deletions(-)

diff --git a/src/orion/algo/base.py b/src/orion/algo/base.py
index a0b267c03..580d6dddb 100644
--- a/src/orion/algo/base.py
+++ b/src/orion/algo/base.py
@@ -164,7 +164,8 @@ def format_trial(self, trial):
     def get_id(self, trial, ignore_fidelity=False):
         """Return unique hash for a trials based on params
 
-        Deprecated and will be removed in v0.3.0. Use the trial hashing methods instead.
+        The trial is assumed to be in the transformed space if the algorithm is working in a
+        transformed space.
 
         Parameters
         ----------
@@ -184,7 +185,9 @@ def get_id(self, trial, ignore_fidelity=False):
         # Apply transforms and reverse to see data as it would come from DB
         # (Some transformations looses some info. ex: Precision transformation)
 
-        trial = self.format_trial(trial)
+        # Compute trial hash in the client-facing format.
+        if hasattr(self.space, "reverse"):
+            trial = self.space.reverse(trial)
 
         return trial.compute_trial_hash(
             trial,
diff --git a/src/orion/core/worker/transformer.py b/src/orion/core/worker/transformer.py
index 1dcd86b23..e31811a44 100644
--- a/src/orion/core/worker/transformer.py
+++ b/src/orion/core/worker/transformer.py
@@ -792,7 +792,7 @@ def transform(self, trial):
             dim.transform(trial.params[name]) for name, dim in self.items()
         )
 
-        return create_transformed_trial(trial, transformed_point, self)
+        return change_trial_params(trial, transformed_point, self)
 
     def reverse(self, transformed_trial):
         """Reverses transformation so that a point from this `TransformedSpace`
@@ -802,7 +802,7 @@ def reverse(self, transformed_trial):
             dim.reverse(transformed_trial.params[name]) for name, dim in self.items()
         )
 
-        return create_restored_trial(
+        return change_trial_params(
             transformed_trial,
             reversed_point,
             self,
@@ -852,7 +852,7 @@ def reshape(self, trial):
         for dim in self.values():
             reshaped_point.append(dim.transform(point[dim.index]))
 
-        return create_transformed_trial(trial, reshaped_point, self)
+        return change_trial_params(trial, reshaped_point, self)
 
     def restore_shape(self, transformed_trial):
         """Restore shape."""
@@ -864,7 +864,7 @@ def restore_shape(self, transformed_trial):
                 point_index = original_keys.index(dim.original_dimension.name)
                 point[point_index] = dim.reverse(transformed_point, index)
 
-        return create_restored_trial(transformed_trial, point, self._original_space)
+        return change_trial_params(transformed_trial, point, self._original_space)
 
     def sample(self, n_samples=1, seed=None):
         """Sample from the original dimension and forward transform them."""
@@ -893,56 +893,19 @@ def cardinality(self):
         return self.original.cardinality
 
 
-class TransformedTrial:
-    """A trial with transformed params
-
-    All other attributes are kept as-is.
-
-    Original params are accessible through ``transformed_trial.trial.params``.
-    """
-
-    __slots__ = ["trial", "_params"]
-
-    def __init__(self, trial, params):
-        self.trial = trial
-        self._params = params
-
-    @property
-    def params(self):
-        """Parameters of the trial"""
-        return unflatten({param.name: param.value for param in self._params})
-
-    def __deepcopy__(self, memo):
-        return TransformedTrial(copy.deepcopy(self.trial), copy.deepcopy(self._params))
-
-    def to_dict(self):
-        trial_dictionary = self.trial.to_dict()
-        trial_dictionary["params"] = list(map(lambda x: x.to_dict(), self._params))
-
-        return trial_dictionary
-
-    def __getattr__(self, name):
-        return getattr(self.trial, name)
-
-    def __setattr__(self, name, value):
-        if name in self.__slots__:
-            return super(TransformedTrial, self).__setattr__(name, value)
-
-        return setattr(self.trial, name, value)
-
-
 def create_transformed_trial(trial, transformed_point, space):
     """Convert point into Trial.Param objects and return a TransformedTrial"""
     return TransformedTrial(
         trial, format_trials.tuple_to_trial(transformed_point, space)._params
     )
 
+    new_trial = copy.copy(trial)
+    new_trial._params = format_trials.tuple_to_trial(transformed_point, space)._params
+    return new_trial
 
-def create_restored_trial(trial, point, space):
-    """Convert params in Param objects and update trial"""
-    if isinstance(trial, TransformedTrial):
-        return create_restored_trial(trial.trial, point, space)
 
+def change_trial_params(trial, point, space):
+    """Convert params in Param objects and update trial"""
     new_trial = copy.copy(trial)
     new_trial._params = format_trials.tuple_to_trial(point, space)._params
     return new_trial
diff --git a/tests/unittests/core/test_transformer.py b/tests/unittests/core/test_transformer.py
index 045b2f8b1..408dc6e0c 100644
--- a/tests/unittests/core/test_transformer.py
+++ b/tests/unittests/core/test_transformer.py
@@ -22,11 +22,9 @@
     Reverse,
     TransformedDimension,
     TransformedSpace,
-    TransformedTrial,
     View,
     build_required_space,
-    create_restored_trial,
-    create_transformed_trial,
+    change_trial_params,
 )
 
 
@@ -1407,83 +1405,7 @@ def test_precision_with_linear(space, logdim, logintdim):
     assert rtrial.params["yolo5"] == 2
 
 
-class TestTransformedTrial:
-    def test_params_are_transformed(self, space, tspace):
-        trial = space.sample()[0]
-        ttrial = tspace.transform(trial)
-        assert isinstance(ttrial, TransformedTrial)
-        assert ttrial.params != trial.params
-        assert ttrial not in space
-        assert ttrial in tspace
-
-    def test_trial_attributes_conserved(self, space, tspace):
-        working_dir = "/new/working/dir"
-        status = "interrupted"
-        trial = space.sample()[0]
-        assert trial.working_dir != working_dir
-        trial.working_dir = working_dir
-        assert trial.status != status
-        trial.status = status
-        ttrial = tspace.transform(trial)
-        assert isinstance(ttrial, TransformedTrial)
-        assert ttrial.working_dir == working_dir
-        assert ttrial.status == status
-
-    def test_setters_accessible(self, space, tspace):
-        working_dir = "/new/working/dir"
-        status = "interrupted"
-        trial = space.sample()[0]
-        ttrial = tspace.transform(trial)
-
-        assert trial.working_dir != working_dir
-        assert trial.status != status
-
-        ttrial.working_dir = working_dir
-        ttrial.status = status
-
-        assert ttrial.working_dir == trial.working_dir == working_dir
-        assert ttrial.status == trial.status == status
-
-    def test_to_dict_is_transformed(self, space, tspace):
-        trial = space.sample()[0]
-        ttrial = tspace.transform(trial)
-
-        original_dict = trial.to_dict()
-        transformed_dict = ttrial.to_dict()
-
-        assert original_dict != transformed_dict
-
-        original_dict.pop("params")
-        transformed_dict.pop("params")
-
-        assert original_dict == transformed_dict
-
-        assert "_id" in original_dict
-
-    def test_copy(self, space, tspace):
-        trial = space.sample()[0]
-        ttrial = tspace.transform(trial)
-
-        assert copy.deepcopy(trial).to_dict() == trial.to_dict()
-        assert copy.deepcopy(ttrial).to_dict() == ttrial.to_dict()
-
-
-def test_create_transformed_trial(space, tspace):
-    trial = space.sample()[0]
-    ttrial = tspace.transform(trial)
-
-    # Test that returns a TransformedTrial
-    transformed_trial = create_transformed_trial(
-        trial, format_trials.trial_to_tuple(ttrial, tspace), tspace
-    )
-    assert isinstance(transformed_trial, TransformedTrial)
-
-    # Test that point is converted properly
-    assert transformed_trial not in space
-    assert transformed_trial in tspace
-
-
-def test_create_restored_trial(space, rspace):
+def test_change_trial_params(space, rspace):
     working_dir = "/new/working/dir"
     status = "interrupted"
 
@@ -1495,7 +1417,7 @@ def test_create_restored_trial(space, rspace):
     rtrial.working_dir = working_dir
     rtrial.status = status
 
-    restored_trial = create_restored_trial(rtrial, point, space)
+    restored_trial = change_trial_params(rtrial, point, space)
 
     # Test that attributes are conserved
     assert restored_trial.working_dir == working_dir

From 08dd775ce8a76258be7ffce63b3551dfff202484 Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Wed, 10 Nov 2021 15:41:56 -0500
Subject: [PATCH 19/34] Adjust EvolutionaryES to new observe interface

---
 src/orion/algo/evolution_es.py            |  57 +++---
 src/orion/algo/hyperband.py               |   1 -
 tests/unittests/algo/test_evolution_es.py | 213 ++++++++++++----------
 3 files changed, 155 insertions(+), 116 deletions(-)

diff --git a/src/orion/algo/evolution_es.py b/src/orion/algo/evolution_es.py
index 3578a2be7..d64955bba 100644
--- a/src/orion/algo/evolution_es.py
+++ b/src/orion/algo/evolution_es.py
@@ -13,6 +13,7 @@
 import numpy as np
 
 from orion.algo.hyperband import Hyperband, HyperbandBracket
+from orion.core.utils import format_trials
 
 logger = logging.getLogger(__name__)
 
@@ -146,9 +147,9 @@ def __init__(
         self.hurdles = []
 
         self.population = {}
-        for key in range(len(self.space)):
-            if not key == self.fidelity_index:
-                self.population[key] = [-1] * nums_population
+        for i, dim in enumerate(self.space.values()):
+            if dim.type != "fidelity":
+                self.population[i] = [-1] * nums_population
 
         self.performance = np.inf * np.ones(nums_population)
 
@@ -203,7 +204,7 @@ class BracketEVES(HyperbandBracket):
     def __init__(self, evolution_es, budgets, repetition_id):
         super(BracketEVES, self).__init__(evolution_es, budgets, repetition_id)
         self.eves = self.hyperband
-        self.search_space_remove_fidelity = []
+        self.search_space_without_fidelity = []
         self._candidates = {}
 
         if evolution_es.mutate:
@@ -218,9 +219,9 @@ def __init__(self, evolution_es, budgets, repetition_id):
         mod = importlib.import_module(mod_name)
         self.mutate_func = getattr(mod, func_name)
 
-        for i in range(len(self.space.values())):
-            if not i == self.eves.fidelity_index:
-                self.search_space_remove_fidelity.append(i)
+        for i, dim in enumerate(self.space.values()):
+            if dim.type != "fidelity":
+                self.search_space_without_fidelity.append(i)
 
     @property
     def space(self):
@@ -249,10 +250,14 @@ def _get_teams(self, rung_id):
             else len(list(rung.values()))
         )
 
-        for i in range(population_range):
-            for j in self.search_space_remove_fidelity:
-                self.eves.population[j][i] = list(rung.values())[i][1][j]
-            self.eves.performance[i] = list(rung.values())[i][0]
+        rung_trials = list(rung.values())
+        for trial_index in range(population_range):
+            objective, trial = rung_trials[trial_index]
+            self.eves.performance[trial_index] = objective
+            for ith_dim in self.search_space_without_fidelity:
+                self.eves.population[ith_dim][trial_index] = trial.params[
+                    self.space[ith_dim].name
+                ]
 
         population_index = list(range(self.eves.nums_population))
         red_team = self.eves.rng.choice(
@@ -290,24 +295,29 @@ def _mutate_population(self, red_team, blue_team, rung, population_range, fideli
 
             logger.debug("Evolution hurdles are: %s", str(self.eves.hurdles))
 
-        points = []
+        trials = []
+        trial_ids = set()
         nums_all_equal = [0] * population_range
         for i in range(population_range):
             point = [0] * len(self.space)
             while True:
                 point = list(point)
-                point[self.eves.fidelity_index] = fidelity
+                point[
+                    list(self.space.keys()).index(self.eves.fidelity_index)
+                ] = fidelity
 
-                for j in self.search_space_remove_fidelity:
+                for j in self.search_space_without_fidelity:
                     point[j] = self.eves.population[j][i]
 
-                point = self.eves.format_point(point)
+                trial = format_trials.tuple_to_trial(point, self.space)
+                trial = self.eves.format_trial(trial)
+                trial_id = self.eves.get_id(trial)
 
-                if point in points:
+                if trial_id in trial_ids:
                     nums_all_equal[i] += 1
                     logger.debug("find equal one, continue to mutate.")
                     self._mutate(i, i)
-                elif self.eves.has_suggested(point):
+                elif self.eves.has_suggested(trial):
                     nums_all_equal[i] += 1
                     logger.debug("find one already suggested, continue to mutate.")
                     self._mutate(i, i)
@@ -320,11 +330,12 @@ def _mutate_population(self, red_team, blue_team, rung, population_range, fideli
                     break
 
             if nums_all_equal[i] < self.eves.max_retries:
-                points.append(point)
+                trials.append(trial)
+                trial_ids.add(trial_id)
             else:
-                logger.debug("Dropping point %s", point)
+                logger.debug("Dropping trial %s", trial)
 
-        return points, np.array(nums_all_equal)
+        return trials, np.array(nums_all_equal)
 
     def get_candidates(self, rung_id):
         """Get a candidate for promotion"""
@@ -343,7 +354,9 @@ def get_candidates(self, rung_id):
 
     def _mutate(self, winner_id, loser_id):
         select_genes_key_list = self.eves.rng.choice(
-            self.search_space_remove_fidelity, self.eves.nums_mutate_gene, replace=False
+            self.search_space_without_fidelity,
+            self.eves.nums_mutate_gene,
+            replace=False,
         )
         self.copy_winner(winner_id, loser_id)
         kwargs = copy.deepcopy(self.mutate_attr)
@@ -358,5 +371,5 @@ def _mutate(self, winner_id, loser_id):
 
     def copy_winner(self, winner_id, loser_id):
         """Copy winner to loser"""
-        for key in self.search_space_remove_fidelity:
+        for key in self.search_space_without_fidelity:
             self.eves.population[key][loser_id] = self.eves.population[key][winner_id]
diff --git a/src/orion/algo/hyperband.py b/src/orion/algo/hyperband.py
index 8b89d8064..c3a92ea65 100644
--- a/src/orion/algo/hyperband.py
+++ b/src/orion/algo/hyperband.py
@@ -538,7 +538,6 @@ def register(self, trial):
         )
 
     def _get_results(self, trial):
-        print(flatten(trial.params))
         fidelity = flatten(trial.params)[self.hyperband.fidelity_index]
         rungs = [
             rung["results"] for rung in self.rungs if rung["resources"] == fidelity
diff --git a/tests/unittests/algo/test_evolution_es.py b/tests/unittests/algo/test_evolution_es.py
index 17d77af31..92384e48b 100644
--- a/tests/unittests/algo/test_evolution_es.py
+++ b/tests/unittests/algo/test_evolution_es.py
@@ -12,6 +12,15 @@
 from orion.algo.space import Fidelity, Real, Space
 from orion.testing.algo import BaseAlgoTests
 
+from orion.testing.trial import create_trial
+
+from test_hyperband import (
+    compare_registered_trial,
+    create_rung_from_points,
+    create_trial_for_hb,
+    force_observe,
+)
+
 
 @pytest.fixture
 def space():
@@ -73,80 +82,79 @@ def evolution_customer_mutate(space1):
 @pytest.fixture
 def rung_0():
     """Create fake points and objectives for rung 0."""
-    points = np.linspace(0, 8, 9)
-    return dict(
-        n_trials=9,
-        resources=1,
-        results={
-            hashlib.md5(str([point]).encode("utf-8")).hexdigest(): (point, (1, point))
-            for point in points
-        },
-    )
+    return create_rung_from_points(np.linspace(0, 8, 9), n_trials=9, resources=1)
 
 
 @pytest.fixture
 def rung_1(rung_0):
     """Create fake points and objectives for rung 1."""
-    values = map(
-        lambda v: (v[0], (3, v[0])), list(sorted(rung_0["results"].values()))[:3]
-    )
-    return dict(
-        n_trials=3,
-        resources=3,
-        results={
-            hashlib.md5(str([value[0]]).encode("utf-8")).hexdigest(): value
-            for value in values
-        },
-    )
+    points = [trial.params["lr"] for _, trial in sorted(rung_0["results"].values())[:3]]
+    return create_rung_from_points(points, n_trials=3, resources=3)
 
 
 @pytest.fixture
 def rung_2(rung_1):
-    """Create fake points and objectives for rung 2."""
-    values = map(
-        lambda v: (v[0], (9, v[0])), list(sorted(rung_1["results"].values()))[:1]
-    )
-    return dict(
-        n_trials=1,
-        resources=9,
-        results={
-            hashlib.md5(str([value[0]]).encode("utf-8")).hexdigest(): value
-            for value in values
-        },
-    )
+    """Create fake points and objectives for rung 1."""
+    points = [trial.params["lr"] for _, trial in sorted(rung_1["results"].values())[:1]]
+    return create_rung_from_points(points, n_trials=1, resources=9)
 
 
 @pytest.fixture
-def rung_3():
+def rung_3(space1):
     """Create fake points and objectives for rung 3."""
     points = np.linspace(1, 4, 4)
+    keys = list(space1.keys())
+    types = [dim.type for dim in space1.values()]
+
+    results = {}
+    for point in points:
+        trial = create_trial(
+            (np.power(2, (point - 1)), 1.0 / point, 1.0 / (point * point)),
+            names=keys,
+            results={"objective": point},
+            types=types,
+        )
+        trial_hash = trial.compute_trial_hash(
+            trial,
+            ignore_fidelity=True,
+            ignore_experiment=True,
+        )
+        results[trial_hash] = (trial.objective.value, trial)
+
     return dict(
         n_trials=4,
         resources=1,
-        results={
-            hashlib.md5(str([point]).encode("utf-8")).hexdigest(): (
-                point,
-                (np.power(2, (point - 1)), 1.0 / point, 1.0 / (point * point)),
-            )
-            for point in points
-        },
+        results=results,
     )
 
 
 @pytest.fixture
-def rung_4():
+def rung_4(space1):
     """Create duplicated fake points and objectives for rung 4."""
     points = np.linspace(1, 4, 4)
+    keys = list(space1.keys())
+    types = [dim.type for dim in space1.values()]
+
+    results = {}
+    for point in points:
+        trial = create_trial(
+            (1, point // 2, point // 2),
+            names=keys,
+            results={"objective": point},
+            types=types,
+        )
+
+        trial_hash = trial.compute_trial_hash(
+            trial,
+            ignore_fidelity=True,
+            ignore_experiment=True,
+        )
+        results[trial_hash] = (trial.objective.value, trial)
+
     return dict(
         n_trials=4,
         resources=1,
-        results={
-            hashlib.md5(str([point]).encode("utf-8")).hexdigest(): (
-                point,
-                (1, point // 2, point // 2),
-            )
-            for point in points
-        },
+        results=results,
     )
 
 
@@ -175,12 +183,14 @@ def test_customized_mutate_population(space1, rung_3, budgets):
 
     red_team = [0, 2]
     blue_team = [1, 3]
-    for i in range(population_range):
-        for j in [1, 2]:
-            algo.brackets[0].eves.population[j][i] = list(rung_3["results"].values())[
-                i
-            ][1][j]
-        algo.brackets[0].eves.performance[i] = list(rung_3["results"].values())[i][0]
+    rung_trials = list(rung_3["results"].values())
+    for trial_index in range(population_range):
+        objective, trial = rung_trials[trial_index]
+        algo.performance[trial_index] = objective
+        for ith_dim in [1, 2]:
+            algo.population[ith_dim][trial_index] = trial.params[
+                algo.space[ith_dim].name
+            ]
 
     org_data = np.stack(
         (
@@ -239,14 +249,15 @@ def test_register(self, evolution, bracket, rung_0, rung_1):
         bracket.hyperband = evolution
         bracket.eves = evolution
         bracket.rungs = [rung_0, rung_1]
-        point = (1, 0.0)
-        point_hash = hashlib.md5(str([0.0]).encode("utf-8")).hexdigest()
+        trial = create_trial_for_hb((1, 0.0), objective=0.0)
+        trial_id = evolution.get_id(trial, ignore_fidelity=True)
 
-        evolution.observe([point], [{"objective": 0.0}])
+        evolution.observe([trial])
 
         assert len(bracket.rungs[0])
-        assert point_hash in bracket.rungs[0]["results"]
-        assert (0.0, point) == bracket.rungs[0]["results"][point_hash]
+        assert trial_id in bracket.rungs[0]["results"]
+        assert bracket.rungs[0]["results"][trial_id][0] == 0.0
+        assert bracket.rungs[0]["results"][trial_id][1].params == trial.params
 
 
 class TestBracketEVES:
@@ -257,7 +268,7 @@ def test_get_teams(self, bracket, rung_3):
         bracket.rungs[0] = rung_3
         rung, population_range, red_team, blue_team = bracket._get_teams(0)
         assert len(list(rung.values())) == 4
-        assert bracket.search_space_remove_fidelity == [1, 2]
+        assert bracket.search_space_without_fidelity == [1, 2]
         assert population_range == 4
         assert set(red_team).union(set(blue_team)) == {0, 1, 2, 3}
         assert set(red_team).intersection(set(blue_team)) == set()
@@ -267,12 +278,15 @@ def test_mutate_population(self, bracket, rung_3):
         red_team = [0, 2]
         blue_team = [1, 3]
         population_range = 4
-        for i in range(4):
-            for j in [1, 2]:
-                bracket.eves.population[j][i] = list(rung_3["results"].values())[i][1][
-                    j
+        rung_trials = list(rung_3["results"].values())
+        for trial_index in range(4):
+            objective, trial = rung_trials[trial_index]
+
+            bracket.eves.performance[trial_index] = objective
+            for ith_dim in [1, 2]:
+                bracket.eves.population[ith_dim][trial_index] = trial.params[
+                    bracket.eves.space[ith_dim].name
                 ]
-            bracket.eves.performance[i] = list(rung_3["results"].values())[i][0]
 
         org_data = np.stack(
             (
@@ -321,42 +335,55 @@ def test_duplicated_mutated_population(self, bracket, rung_4):
         red_team = [0, 2]
         blue_team = [0, 2]  # no mutate occur at first.
         population_range = 4
-        for i in range(4):
-            for j in [1, 2]:
-                bracket.eves.population[j][i] = list(rung_4["results"].values())[i][1][
-                    j
+
+        rung_trials = list(rung_4["results"].values())
+        # Duplicate second item
+        rung_trials.insert(2, rung_trials[1])
+        for trial_index in range(4):
+            objective, trial = rung_trials[trial_index]
+
+            # bracket.eves.performance[trial_index] = objective
+            for ith_dim in [1, 2]:
+                bracket.eves.population[ith_dim][trial_index] = trial.params[
+                    bracket.eves.space[ith_dim].name
                 ]
-        points, nums_all_equal = bracket._mutate_population(
+
+        trials, nums_all_equal = bracket._mutate_population(
             red_team, blue_team, rung_4["results"], population_range, fidelity=2
         )
 
         # In this case, duplication will occur, and we can make it mutate one more time.
-        # The points 1 and 2 should be different, while one of nums_all_equal should be 1.
-        if points[1][1] != points[2][1]:
-            assert points[1][2] == points[2][2]
+        # The trials 1 and 2 should be different, while one of nums_all_equal should be 1.
+        if trials[1].params["lr"] != trials[2].params["lr"]:
+            assert trials[1].params["weight_decay"] == trials[2].params["weight_decay"]
         else:
-            assert points[1][2] != points[2][2]
+            assert trials[1].params["weight_decay"] != trials[2].params["weight_decay"]
 
         assert nums_all_equal[0] == 0
         assert nums_all_equal[1] == 0
         assert nums_all_equal[2] == 1
         assert nums_all_equal[3] == 0
 
-    def test_mutate_points(self, bracket, rung_3):
-        """Test that correct point is promoted."""
+    def test_mutate_trials(self, bracket, rung_3):
+        """Test that correct trial is promoted."""
         red_team = [0, 2]
         blue_team = [0, 2]
         population_range = 4
-        for i in range(4):
-            for j in [1, 2]:
-                bracket.eves.population[j][i] = list(rung_3["results"].values())[i][1][
-                    j
+        rung_trials = list(rung_3["results"].values())
+        for trial_index in range(4):
+            objective, trial = rung_trials[trial_index]
+
+            # bracket.eves.performance[trial_index] = objective
+            for ith_dim in [1, 2]:
+                bracket.eves.population[ith_dim][trial_index] = trial.params[
+                    bracket.eves.space[ith_dim].name
                 ]
-        points, nums_all_equal = bracket._mutate_population(
+
+        trials, nums_all_equal = bracket._mutate_population(
             red_team, blue_team, rung_3["results"], population_range, fidelity=2
         )
-        assert points[0] == (2, 1.0, 1.0)
-        assert points[1] == (2, 1.0 / 2, 1.0 / 4)
+        assert trials[0].params == {"epoch": 2, "lr": 1.0, "weight_decay": 1.0}
+        assert trials[1].params == {"epoch": 2, "lr": 1.0 / 2, "weight_decay": 1.0 / 4}
         assert (nums_all_equal == 0).all()
 
 
@@ -391,10 +418,10 @@ def test_is_done_cardinality(self):
             assert not algo.is_done
             n_sampled = len(algo.algorithm.sampled)
             n_trials = len(algo.algorithm.trial_to_brackets)
-            points = algo.suggest()
-            if points is None:
+            trials = algo.suggest()
+            if trials is None:
                 break
-            assert len(algo.algorithm.sampled) == n_sampled + len(points)
+            assert len(algo.algorithm.sampled) == n_sampled + len(trials)
             assert len(algo.algorithm.trial_to_brackets) == space.cardinality
 
             # We reached max number of trials we can suggest before observing any.
@@ -402,8 +429,8 @@ def test_is_done_cardinality(self):
 
             assert not algo.is_done
 
-            for i, point in enumerate(points):
-                algo.observe([point], [dict(objective=i)])
+            for i, trial in enumerate(trials):
+                backward.algo_observe(algo, [trial], [dict(objective=i)])
 
         assert algo.is_done
 
@@ -417,11 +444,11 @@ def test_is_done_max_trials(self, num):
 
         objective = 0
         while not algo.is_done:
-            points = algo.suggest(num)
-            assert points
-            if points:
-                self.observe_points(points, algo, objective)
-                objective += len(points)
+            trials = algo.suggest(num)
+            assert trials
+            if trials:
+                self.observe_trials(trials, algo, objective)
+                objective += len(trials)
 
         # Hyperband should ignore max trials.
         assert algo.n_observed > MAX_TRIALS

From f457515e3f5b7854ffefa3132e957f6c5812a14c Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Wed, 10 Nov 2021 15:56:18 -0500
Subject: [PATCH 20/34] Adapt strategy to new observe interface

---
 src/orion/core/worker/strategy.py     | 25 +++++---------
 tests/unittests/core/test_strategy.py | 49 ++++++++++++---------------
 2 files changed, 30 insertions(+), 44 deletions(-)

diff --git a/src/orion/core/worker/strategy.py b/src/orion/core/worker/strategy.py
index d74dc94fd..1085703f8 100644
--- a/src/orion/core/worker/strategy.py
+++ b/src/orion/core/worker/strategy.py
@@ -55,24 +55,17 @@ class ParallelStrategy(object):
     def __init__(self, *args, **kwargs):
         pass
 
-    def observe(self, points, results):
+    def observe(self, trials):
         """Observe completed trials
 
         .. seealso:: `orion.algo.base.BaseAlgorithm.observe` method
 
         Parameters
         ----------
-        points: list of tuples of array-likes
-           Points from a `orion.algo.space.Space`.
-           Evaluated problem parameters by a consumer.
-        results: list of dict
-           Contains the result of an evaluation; partial information about the
-           black-box function at each point in `params`.
+        trials: list of ``orion.core.worker.trial.Trial``
+           Trials from a `orion.algo.space.Space`.
 
         """
-        # NOTE: In future points and results will be converted to trials for coherence with
-        # `Strategy.lie()` as well as for coherence with `Algorithm.observe` which will also be
-        # converted to expect trials instead of lists and dictionaries.
         raise NotImplementedError()
 
     # pylint: disable=no-self-use
@@ -112,7 +105,7 @@ def configuration(self):
 class NoParallelStrategy(ParallelStrategy):
     """No parallel strategy"""
 
-    def observe(self, points, results):
+    def observe(self, trials):
         """See ParallelStrategy.observe"""
         pass
 
@@ -139,10 +132,10 @@ def configuration(self):
         """Provide the configuration of the strategy as a dictionary."""
         return {self.__class__.__name__: {"default_result": self.default_result}}
 
-    def observe(self, points, results):
+    def observe(self, trials):
         """See ParallelStrategy.observe"""
         results = [
-            result["objective"] for result in results if result["objective"] is not None
+            trial.objective.value for trial in trials if trial.objective is not None
         ]
         if results:
             self.max_result = max(results)
@@ -170,10 +163,10 @@ def configuration(self):
         """Provide the configuration of the strategy as a dictionary."""
         return {self.__class__.__name__: {"default_result": self.default_result}}
 
-    def observe(self, points, results):
+    def observe(self, trials):
         """See ParallelStrategy.observe"""
         objective_values = [
-            result["objective"] for result in results if result["objective"] is not None
+            trial.objective.value for trial in trials if trial.objective is not None
         ]
         if objective_values:
             self.mean_result = sum(value for value in objective_values) / float(
@@ -202,7 +195,7 @@ def configuration(self):
         """Provide the configuration of the strategy as a dictionary."""
         return {self.__class__.__name__: {"stub_value": self.stub_value}}
 
-    def observe(self, points, results):
+    def observe(self, trials):
         """See ParallelStrategy.observe"""
         pass
 
diff --git a/tests/unittests/core/test_strategy.py b/tests/unittests/core/test_strategy.py
index 0d8229487..0bbc7c69d 100644
--- a/tests/unittests/core/test_strategy.py
+++ b/tests/unittests/core/test_strategy.py
@@ -13,15 +13,22 @@
     strategy_factory,
 )
 from orion.core.worker.trial import Trial
+from orion.core.utils import backward
 
 
 @pytest.fixture
-def observations():
+def trials():
     """10 objective observations"""
-    points = [i for i in range(10)]
-    results = [{"objective": points[i]} for i in range(10)]
+    trials = []
+    for i in range(10):
+        trials.append(
+            Trial(
+                params=[{"name": "x", "type": "real", "value": i}],
+                results=[{"name": "objective", "type": "objective", "value": i}],
+            )
+        )
 
-    return points, results
+    return trials
 
 
 @pytest.fixture
@@ -52,7 +59,6 @@ def corrupted_trial():
 def test_handle_corrupted_trials(caplog, strategy, corrupted_trial):
     """Verify that corrupted trials are handled properly"""
     with caplog.at_level(logging.WARNING, logger="orion.core.worker.strategy"):
-        strategy_factory.create(strategy).observe([corrupted_trial], [{"objective": 1}])
         lie = strategy_factory.create(strategy).lie(corrupted_trial)
 
     match = "Trial `{}` has an objective but status is not completed".format(
@@ -68,9 +74,6 @@ def test_handle_corrupted_trials(caplog, strategy, corrupted_trial):
 def test_handle_uncompleted_trials(caplog, strategy, incomplete_trial):
     """Verify that no warning is logged if trial is valid"""
     with caplog.at_level(logging.WARNING, logger="orion.core.worker.strategy"):
-        strategy_factory.create(strategy).observe(
-            [incomplete_trial], [{"objective": None}]
-        )
         strategy_factory.create(strategy).lie(incomplete_trial)
 
     assert "Trial `{}` has an objective but status is not completed" not in caplog.text
@@ -93,44 +96,34 @@ def test_create_meanparallel(self):
 class TestParallelStrategies:
     """Test the different parallel strategy methods"""
 
-    def test_max_parallel_strategy(self, observations, incomplete_trial):
+    def test_max_parallel_strategy(self, trials, incomplete_trial):
         """Test that MaxParallelStrategy lies using the max"""
-        points, results = observations
-
         strategy = MaxParallelStrategy()
-        strategy.observe(points, results)
+        strategy.observe(trials)
         lying_result = strategy.lie(incomplete_trial)
 
-        max_value = max(result["objective"] for result in results)
+        max_value = max(trial.objective.value for trial in trials)
         assert lying_result.value == max_value
 
-    def test_mean_parallel_strategy(self, observations, incomplete_trial):
+    def test_mean_parallel_strategy(self, trials, incomplete_trial):
         """Test that MeanParallelStrategy lies using the mean"""
-        points, results = observations
-
         strategy = MeanParallelStrategy()
-        strategy.observe(points, results)
+        strategy.observe(trials)
         lying_result = strategy.lie(incomplete_trial)
 
-        mean_value = sum(result["objective"] for result in results) / float(
-            len(results)
-        )
+        mean_value = sum(trial.objective.value for trial in trials) / float(len(trials))
         assert lying_result.value == mean_value
 
-    def test_no_parallel_strategy(self, observations, incomplete_trial):
+    def test_no_parallel_strategy(self, trials, incomplete_trial):
         """Test that NoParallelStrategy lies outputs None"""
-        points, results = observations
-
         strategy = NoParallelStrategy()
-        strategy.observe(points, results)
+        strategy.observe(trials)
         lying_result = strategy.lie(incomplete_trial)
         assert lying_result is None
 
-    def test_stub_parallel_strategy(self, observations, incomplete_trial):
+    def test_stub_parallel_strategy(self, trials, incomplete_trial):
         """Test that NoParallelStrategy lies outputs None"""
-        points, results = observations
-
         strategy = StubParallelStrategy()
-        strategy.observe(points, results)
+        strategy.observe(trials)
         lying_result = strategy.lie(incomplete_trial)
         assert lying_result.value is None

From 531c5c8fc4a21254a47fa521e6fca3d445088760 Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Thu, 11 Nov 2021 11:42:46 -0500
Subject: [PATCH 21/34] Adapt Producer to new observe interface

---
 src/orion/core/worker/producer.py            |  44 +---
 src/orion/testing/__init__.py                |   3 +-
 tests/unittests/algo/test_hyperband.py       |   5 +-
 tests/unittests/core/worker/test_producer.py | 221 ++++++++++++-------
 4 files changed, 148 insertions(+), 125 deletions(-)

diff --git a/src/orion/core/worker/producer.py b/src/orion/core/worker/producer.py
index 5a4f38cf3..99e75f793 100644
--- a/src/orion/core/worker/producer.py
+++ b/src/orion/core/worker/producer.py
@@ -85,11 +85,11 @@ def is_done(self):
             self.naive_algorithm is not None and self.naive_algorithm.is_done
         )
 
-    def suggest(self, pool_size):
-        """Try suggesting new points with the naive algorithm"""
+    def adjust_pool_size(self, pool_size):
+        """Limit pool size if it would overshoot over max_trials"""
         num_pending = self.num_trials - self.num_broken
         num = max(self.experiment.max_trials - num_pending, 1)
-        return self.naive_algorithm.suggest(min(num, pool_size))
+        return min(num, pool_size)
 
     def produce(self, pool_size):
         """Create and register new trials."""
@@ -104,13 +104,13 @@ def produce(self, pool_size):
         # points of other producer as part of the current pool samples.
         while (
             len(self.experiment.fetch_trials(with_evc_tree=True)) - self.num_trials
-            < pool_size
+            < self.adjust_pool_size(pool_size)
             and not self.is_done
         ):
             self._sample_guard(start)
 
             log.debug("### Algorithm suggests new points.")
-            new_points = self.suggest(pool_size)
+            new_points = self.naive_algorithm.suggest(self.adjust_pool_size(pool_size))
 
             # Sync state of original algo so that state continues evolving.
             self.algorithm.set_state(self.naive_algorithm.state_dict)
@@ -141,7 +141,7 @@ def register_trials(self, new_points):
 
         return registered_trials
 
-    def register_trial(self, new_point):
+    def register_trial(self, new_trial):
         """Register a new set of sampled parameters into the DB
         guaranteeing their uniqueness
 
@@ -153,10 +153,6 @@ def register_trial(self, new_point):
         """
         # FIXME: Relying on DB to guarantee uniqueness
         # when the trial history will be held by that algo we can move that logic out of the DB
-
-        log.debug("#### Convert point to `Trial` object.")
-        new_trial = format_trials.tuple_to_trial(new_point, self.space)
-
         try:
             self._prevalidate_trial(new_trial)
             new_trial.parents = self.naive_trials_history.children
@@ -206,27 +202,16 @@ def _update_algorithm(self, completed_trials):
         new_completed_trials = []
         for trial in completed_trials:
             # if trial not in self.trials_history:
-            if not self.algorithm.has_observed(
-                format_trials.trial_to_tuple(trial, self.space)
-            ):
+            if not self.algorithm.has_observed(trial):
                 new_completed_trials.append(trial)
 
         log.debug("### %s", new_completed_trials)
 
         if new_completed_trials:
-            log.debug("### Convert them to list of points and their results.")
-            points = list(
-                map(
-                    lambda trial: format_trials.trial_to_tuple(trial, self.space),
-                    new_completed_trials,
-                )
-            )
-            results = list(map(format_trials.get_trial_results, new_completed_trials))
-
             log.debug("### Observe them.")
             self.trials_history.update(new_completed_trials)
-            self.algorithm.observe(points, results)
-            self.strategy.observe(points, results)
+            self.algorithm.observe(new_completed_trials)
+            self.strategy.observe(new_completed_trials)
             self._update_params_hashes(new_completed_trials)
 
     def _produce_lies(self, incomplete_trials):
@@ -264,16 +249,7 @@ def _update_naive_algorithm(self, incomplete_trials):
         lying_trials = self._produce_lies(incomplete_trials)
         log.debug("### %s", lying_trials)
         if lying_trials:
-            log.debug("### Convert them to list of points and their results.")
-            points = list(
-                map(
-                    lambda trial: format_trials.trial_to_tuple(trial, self.space),
-                    lying_trials,
-                )
-            )
-            results = list(map(format_trials.get_trial_results, lying_trials))
-
             log.debug("### Observe them.")
             self.naive_trials_history.update(lying_trials)
-            self.naive_algorithm.observe(points, results)
+            self.naive_algorithm.observe(lying_trials)
             self._update_params_hashes(lying_trials)
diff --git a/src/orion/testing/__init__.py b/src/orion/testing/__init__.py
index e03e7fe26..1271ff65c 100644
--- a/src/orion/testing/__init__.py
+++ b/src/orion/testing/__init__.py
@@ -17,7 +17,6 @@
 import orion.algo.space
 import orion.core.io.experiment_builder as experiment_builder
 from orion.core.io.space_builder import SpaceBuilder
-from orion.core.utils.format_trials import tuple_to_trial
 from orion.core.worker.producer import Producer
 from orion.testing.state import OrionState
 
@@ -67,7 +66,7 @@ def _generate(obj, *args, value):
         if trial["status"] == "completed":
             trial["results"].append({"name": "loss", "type": "objective", "value": i})
 
-        trial_stub = tuple_to_trial(space.sample(seed=i)[0], space)
+        trial_stub = space.sample(seed=i)[0]
         trial["params"] = trial_stub.to_dict()["params"]
 
     return new_trials
diff --git a/tests/unittests/algo/test_hyperband.py b/tests/unittests/algo/test_hyperband.py
index 4df471ff7..2e8ac8332 100644
--- a/tests/unittests/algo/test_hyperband.py
+++ b/tests/unittests/algo/test_hyperband.py
@@ -11,7 +11,7 @@
 from orion.algo.hyperband import Hyperband, HyperbandBracket, compute_budgets
 from orion.algo.space import Fidelity, Integer, Real, Space
 from orion.testing.algo import BaseAlgoTests, phase
-from orion.testing.trial import create_trial
+from orion.testing.trial import create_trial, compare_trials
 
 
 def create_trial_for_hb(point, objective=None):
@@ -38,9 +38,6 @@ def create_rung_from_points(points, n_trials, resources):
     return dict(n_trials=n_trials, resources=resources, results=results)
 
 
-def compare_trials(trials, other_trials):
-    assert [t.params for t in trials] == [t.params for t in other_trials]
-
 
 def compare_registered_trial(registered_trial, trial):
     assert registered_trial[0] == trial.objective.value
diff --git a/tests/unittests/core/worker/test_producer.py b/tests/unittests/core/worker/test_producer.py
index 6f0335f70..e02215eb2 100644
--- a/tests/unittests/core/worker/test_producer.py
+++ b/tests/unittests/core/worker/test_producer.py
@@ -9,18 +9,18 @@
 
 from orion.core.io.experiment_builder import build
 from orion.core.utils.exceptions import SampleTimeout, WaitingForTrials
-from orion.core.utils.format_trials import trial_to_tuple
+from orion.core.utils import format_trials
 from orion.core.worker.producer import Producer
 from orion.core.worker.trial import Trial
+from orion.testing.trial import compare_trials
 
 
 class DumbParallelStrategy:
     """Mock object for parallel strategy"""
 
-    def observe(self, points, results):
+    def observe(self, trials):
         """See ParallelStrategy.observe"""
-        self._observed_points = points
-        self._observed_results = results
+        self._observed_trials = trials
         self._value = None
 
     def lie(self, trial):
@@ -28,7 +28,7 @@ def lie(self, trial):
         if self._value:
             value = self._value
         else:
-            value = len(self._observed_points)
+            value = len(self._observed_trials)
 
         self._lie = lie = Trial.Result(name="lie", type="lie", value=value)
         return lie
@@ -58,7 +58,11 @@ def producer(monkeypatch, hacked_exp, random_dt, categorical_values):
     """Return a setup `Producer`."""
     # make init done
 
-    hacked_exp.algorithms.algorithm.possible_values = categorical_values
+    possible_trials = [
+        format_trials.tuple_to_trial(point, hacked_exp.space)
+        for point in categorical_values
+    ]
+    hacked_exp.algorithms.algorithm.possible_values = possible_trials
     hacked_exp.algorithms.seed_rng(0)
     hacked_exp.max_trials = 20
     hacked_exp.algorithms.algorithm.max_trials = 20
@@ -81,17 +85,26 @@ def test_algo_observe_completed(producer):
     """Test that algo only observes completed trials"""
     assert len(producer.experiment.fetch_trials()) > 3
     producer.update()
-    # Algorithm must have received completed points and their results
-    obs_points = producer.algorithm.algorithm._points
-    obs_results = producer.algorithm.algorithm._results
-    assert len(obs_points) == 3
-    assert obs_points[0] == ("rnn", "lstm")
-    assert obs_points[1] == ("rnn", "rnn")
-    assert obs_points[2] == ("lstm_with_attention", "gru")
-    assert len(obs_results) == 3
-    assert obs_results[0] == {"objective": 3, "gradient": None, "constraint": []}
-    assert obs_results[1] == {"objective": 2, "gradient": (-0.1, 2), "constraint": []}
-    assert obs_results[2] == {"objective": 10, "gradient": (5, 3), "constraint": [1.2]}
+    # Algorithm must have received completed trials and their results
+    obs_trials = producer.algorithm.algorithm._trials
+    assert len(obs_trials) == 3
+    assert obs_trials[0].params == {"/decoding_layer": "rnn", "/encoding_layer": "lstm"}
+    assert obs_trials[1].params == {"/decoding_layer": "rnn", "/encoding_layer": "rnn"}
+    assert obs_trials[2].params == {
+        "/decoding_layer": "lstm_with_attention",
+        "/encoding_layer": "gru",
+    }
+    assert obs_trials[0].objective.value == 3
+    assert obs_trials[0].gradient is None
+    assert obs_trials[0].constraints == []
+
+    assert obs_trials[1].objective.value == 2
+    assert obs_trials[1].gradient.value == [-0.1, 2]
+    assert obs_trials[1].constraints == []
+
+    assert obs_trials[2].objective.value == 10
+    assert obs_trials[2].gradient.value == [5, 3]
+    assert obs_trials[2].constraints[0].value == 1.2
 
 
 def test_strategist_observe_completed(producer):
@@ -99,24 +112,38 @@ def test_strategist_observe_completed(producer):
     assert len(producer.experiment.fetch_trials()) > 3
     producer.update()
     # Algorithm must have received completed points and their results
-    obs_points = producer.strategy._observed_points
-    obs_results = producer.strategy._observed_results
-    assert len(obs_points) == 3
-    assert obs_points[0] == ("rnn", "lstm")
-    assert obs_points[1] == ("rnn", "rnn")
-    assert obs_points[2] == ("lstm_with_attention", "gru")
-    assert len(obs_results) == 3
-    assert obs_results[0] == {"objective": 3, "gradient": None, "constraint": []}
-    assert obs_results[1] == {"objective": 2, "gradient": (-0.1, 2), "constraint": []}
-    assert obs_results[2] == {"objective": 10, "gradient": (5, 3), "constraint": [1.2]}
+    obs_trials = producer.strategy._observed_trials
+    assert len(obs_trials) == 3
+    assert obs_trials[0].params == {"/decoding_layer": "rnn", "/encoding_layer": "lstm"}
+    assert obs_trials[1].params == {"/decoding_layer": "rnn", "/encoding_layer": "rnn"}
+    assert obs_trials[2].params == {
+        "/decoding_layer": "lstm_with_attention",
+        "/encoding_layer": "gru",
+    }
+
+    assert obs_trials[0].objective.value == 3
+    assert obs_trials[0].gradient is None
+    assert obs_trials[0].constraints == []
+
+    assert obs_trials[1].objective.value == 2
+    assert obs_trials[1].gradient.value == [-0.1, 2]
+    assert obs_trials[1].constraints == []
+
+    assert obs_trials[2].objective.value == 10
+    assert obs_trials[2].gradient.value == [5, 3]
+    assert obs_trials[2].constraints[0].value == 1.2
 
 
 def test_naive_algorithm_is_producing(monkeypatch, producer, random_dt):
     """Verify naive algo is used to produce, not original algo"""
-    producer.algorithm.algorithm.possible_values = [("gru", "rnn")]
+    producer.algorithm.algorithm.possible_values = [
+        format_trials.tuple_to_trial(("gru", "rnn"), producer.algorithm.space)
+    ]
     producer.update()
     monkeypatch.setattr(producer.algorithm.algorithm, "set_state", lambda value: None)
-    producer.algorithm.algorithm.possible_values = [("gru", "gru")]
+    producer.algorithm.algorithm.possible_values = [
+        format_trials.tuple_to_trial(("gru", "gru"), producer.algorithm.space)
+    ]
     producer.produce(1)
 
     assert producer.naive_algorithm.algorithm._num == 1  # pool size
@@ -125,7 +152,9 @@ def test_naive_algorithm_is_producing(monkeypatch, producer, random_dt):
 
 def test_update_and_produce(producer, random_dt):
     """Test new trials are properly produced"""
-    possible_values = [("gru", "rnn")]
+    possible_values = [
+        format_trials.tuple_to_trial(("gru", "rnn"), producer.algorithm.space)
+    ]
     producer.experiment.algorithms.algorithm.possible_values = possible_values
 
     producer.update()
@@ -135,7 +164,7 @@ def test_update_and_produce(producer, random_dt):
     num_new_points = producer.naive_algorithm.algorithm._num
     assert num_new_points == 1  # pool size
 
-    assert producer.naive_algorithm.algorithm._suggested == possible_values
+    compare_trials(producer.naive_algorithm.algorithm._suggested, possible_values)
 
 
 def test_register_new_trials(producer, storage, random_dt):
@@ -143,7 +172,9 @@ def test_register_new_trials(producer, storage, random_dt):
     trials_in_db_before = len(storage._fetch_trials({}))
     new_trials_in_db_before = len(storage._fetch_trials({"status": "new"}))
 
-    producer.experiment.algorithms.algorithm.possible_values = [("gru", "rnn")]
+    producer.experiment.algorithms.algorithm.possible_values = [
+        format_trials.tuple_to_trial(("gru", "rnn"), producer.algorithm.space)
+    ]
 
     producer.update()
     producer.produce(1)
@@ -263,7 +294,9 @@ def test_register_duplicate_lies(producer, storage, random_dt):
     producer.strategy._value = 4
 
     # Set specific output value for to algo to ensure successful creation of a new trial.
-    producer.experiment.algorithms.algorithm.possible_values = [("gru", "rnn")]
+    producer.experiment.algorithms.algorithm.possible_values = [
+        format_trials.tuple_to_trial(("gru", "rnn"), producer.algorithm.space)
+    ]
 
     producer.update()
     lies = produce_lies(producer)
@@ -322,8 +355,8 @@ def test_naive_algo_not_trained_when_all_trials_completed(producer, storage, ran
 
     producer.update()
 
-    assert len(producer.algorithm.algorithm._points) == 3
-    assert len(producer.naive_algorithm.algorithm._points) == 3
+    assert len(producer.algorithm.algorithm._trials) == 3
+    assert len(producer.naive_algorithm.algorithm._trials) == 3
 
 
 def test_naive_algo_trained_on_all_non_completed_trials(producer, storage, random_dt):
@@ -350,35 +383,37 @@ def test_naive_algo_trained_on_all_non_completed_trials(producer, storage, rando
     producer.update()
     assert len(produce_lies(producer)) == 6
 
-    assert len(producer.algorithm.algorithm._points) == 1
-    assert len(producer.naive_algorithm.algorithm._points) == (1 + 6)
+    assert len(producer.algorithm.algorithm._trials) == 1
+    assert len(producer.naive_algorithm.algorithm._trials) == (1 + 6)
 
 
 def test_naive_algo_is_discared(producer, monkeypatch):
     """Verify that naive algo is discarded and recopied from original algo"""
     # Set values for predictions
-    producer.experiment.algorithms.algorithm.possible_values = [("gru", "rnn")]
+    producer.experiment.algorithms.algorithm.possible_values = [
+        format_trials.tuple_to_trial(("gru", "rnn"), producer.algorithm.space)
+    ]
 
     producer.update()
     assert len(produce_lies(producer)) == 4
 
     first_naive_algorithm = producer.naive_algorithm
 
-    assert len(producer.algorithm.algorithm._points) == 3
-    assert len(first_naive_algorithm.algorithm._points) == (3 + 4)
+    assert len(producer.algorithm.algorithm._trials) == 3
+    assert len(first_naive_algorithm.algorithm._trials) == (3 + 4)
 
     producer.produce(1)
 
     # Only update the original algo, naive algo is still not discarded
     update_algorithm(producer)
-    assert len(producer.algorithm.algorithm._points) == 3
+    assert len(producer.algorithm.algorithm._trials) == 3
     assert first_naive_algorithm == producer.naive_algorithm
-    assert len(producer.naive_algorithm.algorithm._points) == (3 + 4)
+    assert len(producer.naive_algorithm.algorithm._trials) == (3 + 4)
 
-    # Discard naive algo and create a new one, now trained on 5 points.
+    # Discard naive algo and create a new one, now trained on 5 trials.
     update_naive_algorithm(producer)
     assert first_naive_algorithm != producer.naive_algorithm
-    assert len(producer.naive_algorithm.algorithm._points) == (3 + 5)
+    assert len(producer.naive_algorithm.algorithm._trials) == (3 + 5)
 
 
 def test_concurent_producers(producer, storage, random_dt):
@@ -389,11 +424,14 @@ def test_concurent_producers(producer, storage, random_dt):
     # Avoid limiting number of samples from the within the algorithm.
     producer.algorithm.algorithm.pool_size = 1000
 
-    # Set so that first producer's algorithm generate valid point on first time
-    # And second producer produce same point and thus must produce next one too.
+    # Set so that first producer's algorithm generate valid trial on first time
+    # And second producer produce same trial and thus must produce next one too.
     # Hence, we know that producer algo will have _num == 1 and
     # second producer algo will have _num == 2
-    producer.algorithm.algorithm.possible_values = [("gru", "rnn"), ("gru", "gru")]
+    producer.algorithm.algorithm.possible_values = [
+        format_trials.tuple_to_trial(point, producer.algorithm.space)
+        for point in [("gru", "rnn"), ("gru", "gru")]
+    ]
     # Make sure it starts from index 0
     producer.algorithm.seed_rng(0)
 
@@ -407,12 +445,12 @@ def test_concurent_producers(producer, storage, random_dt):
     second_producer.produce(2)
 
     # Algorithm was required to suggest some trials
-    num_new_points = producer.algorithm.algorithm._num
-    assert num_new_points == 1  # pool size
-    num_new_points = second_producer.algorithm.algorithm._num
-    assert num_new_points == 2  # pool size
+    num_new_trials = producer.algorithm.algorithm._num
+    assert num_new_trials == 1  # pool size
+    num_new_trials = second_producer.algorithm.algorithm._num
+    assert num_new_trials == 2  # pool size
 
-    # `num_new_points` new trials were registered at database
+    # `num_new_trials` new trials were registered at database
     assert len(storage._fetch_trials({})) == trials_in_db_before + 2
     assert len(storage._fetch_trials({"status": "new"})) == new_trials_in_db_before + 2
     new_trials = list(
@@ -438,12 +476,15 @@ def test_concurent_producers_shared_pool(producer, storage, random_dt):
     trials_in_db_before = len(storage._fetch_trials({}))
     new_trials_in_db_before = len(storage._fetch_trials({"status": "new"}))
 
-    # Set so that first producer's algorithm generate valid point on first time
-    # And second producer produce same point and thus must backoff and then stop
+    # Set so that first producer's algorithm generate valid trial on first time
+    # And second producer produce same trial and thus must backoff and then stop
     # because first producer filled the pool.
     # Hence, we know that producer algo will have _num == 1 and
     # second producer algo will have _num == 1
-    producer.algorithm.algorithm.possible_values = [("gru", "rnn"), ("gru", "gru")]
+    producer.algorithm.algorithm.possible_values = [
+        format_trials.tuple_to_trial(point, producer.algorithm.space)
+        for point in [("gru", "rnn"), ("gru", "gru")]
+    ]
     # Make sure it starts from index 0
     producer.algorithm.seed_rng(0)
 
@@ -457,12 +498,12 @@ def test_concurent_producers_shared_pool(producer, storage, random_dt):
     second_producer.produce(1)
 
     # Algorithm was required to suggest some trials
-    num_new_points = producer.algorithm.algorithm._num
-    assert num_new_points == 1  # pool size
-    num_new_points = second_producer.algorithm.algorithm._num
-    assert num_new_points == 0  # pool size
+    num_new_trials = producer.algorithm.algorithm._num
+    assert num_new_trials == 1  # pool size
+    num_new_trials = second_producer.algorithm.algorithm._num
+    assert num_new_trials == 0  # pool size
 
-    # `num_new_points` new trials were registered at database
+    # `num_new_trials` new trials were registered at database
     assert len(storage._fetch_trials({})) == trials_in_db_before + 1
     assert len(storage._fetch_trials({"status": "new"})) == new_trials_in_db_before + 1
     new_trials = list(
@@ -490,19 +531,22 @@ def test_duplicate_within_pool(producer, storage, random_dt):
     producer.algorithm.algorithm.pool_size = 1000
 
     producer.experiment.algorithms.algorithm.possible_values = [
-        ("gru", "rnn"),
-        ("gru", "rnn"),
-        ("gru", "gru"),
+        format_trials.tuple_to_trial(point, producer.algorithm.space)
+        for point in [
+            ("gru", "rnn"),
+            ("gru", "rnn"),
+            ("gru", "gru"),
+        ]
     ]
 
     producer.update()
     producer.produce(2)
 
     # Algorithm was required to suggest some trials
-    num_new_points = producer.algorithm.algorithm._num
-    assert num_new_points == 4  # 2 * pool size
+    num_new_trials = producer.algorithm.algorithm._num
+    assert num_new_trials == 4  # 2 * pool size
 
-    # `num_new_points` new trials were registered at database
+    # `num_new_trials` new trials were registered at database
     assert len(storage._fetch_trials({})) == trials_in_db_before + 2
     assert len(storage._fetch_trials({"status": "new"})) == new_trials_in_db_before + 2
     new_trials = list(
@@ -524,7 +568,7 @@ def test_duplicate_within_pool(producer, storage, random_dt):
 
 
 def test_duplicate_within_pool_and_db(producer, storage, random_dt):
-    """Test that an algo suggesting multiple points can have a few registered even
+    """Test that an algo suggesting multiple trials can have a few registered even
     if one of them is a duplicate with db.
     """
     trials_in_db_before = len(storage._fetch_trials({}))
@@ -534,19 +578,22 @@ def test_duplicate_within_pool_and_db(producer, storage, random_dt):
     producer.algorithm.algorithm.pool_size = 1000
 
     producer.experiment.algorithms.algorithm.possible_values = [
-        ("gru", "rnn"),
-        ("rnn", "rnn"),
-        ("gru", "gru"),
+        format_trials.tuple_to_trial(point, producer.algorithm.space)
+        for point in [
+            ("gru", "rnn"),
+            ("rnn", "rnn"),
+            ("gru", "gru"),
+        ]
     ]
 
     producer.update()
     producer.produce(2)
 
     # Algorithm was required to suggest some trials
-    num_new_points = producer.algorithm.algorithm._num
-    assert num_new_points == 4  # pool size
+    num_new_trials = producer.algorithm.algorithm._num
+    assert num_new_trials == 4  # pool size
 
-    # `num_new_points` new trials were registered at database
+    # `num_new_trials` new trials were registered at database
     assert len(storage._fetch_trials({})) == trials_in_db_before + 2
     assert len(storage._fetch_trials({"status": "new"})) == new_trials_in_db_before + 2
     new_trials = list(
@@ -568,10 +615,12 @@ def test_duplicate_within_pool_and_db(producer, storage, random_dt):
 
 
 def test_exceed_max_idle_time_because_of_duplicates(producer, random_dt):
-    """Test that RuntimeError is raised when algo keep suggesting the same points"""
+    """Test that RuntimeError is raised when algo keep suggesting the same trials"""
     timeout = 3
     producer.max_idle_time = timeout  # to limit run-time, default would work as well.
-    producer.experiment.algorithms.algorithm.possible_values = [("rnn", "rnn")]
+    producer.experiment.algorithms.algorithm.possible_values = [
+        format_trials.tuple_to_trial(("rnn", "rnn"), producer.algorithm.space)
+    ]
 
     producer.update()
 
@@ -700,7 +749,7 @@ def test_evc_duplicates(monkeypatch, producer):
     )
 
     def suggest(pool_size=None):
-        return [trial_to_tuple(experiment.fetch_trials()[-1], experiment.space)]
+        return [experiment.fetch_trials()[-1]]
 
     producer.experiment = new_experiment
     producer.algorithm = new_experiment.algorithms
@@ -723,7 +772,7 @@ def test_algorithm_is_done(monkeypatch, producer):
 
     def suggest_one_only(self, num=1):
         """Return only one point, whatever `num` is"""
-        return [("gru", "rnn")]
+        return [format_trials.tuple_to_trial(("gru", "rnn"), producer.algorithm.space)]
 
     monkeypatch.delattr(producer.experiment.algorithms.algorithm.__class__, "is_done")
     monkeypatch.setattr(
@@ -737,7 +786,7 @@ def suggest_one_only(self, num=1):
     producer.produce(10)
 
     assert len(producer.experiment.fetch_trials()) == producer.experiment.max_trials
-    assert producer.naive_algorithm.is_done
+    assert not producer.naive_algorithm.is_done
     assert not producer.experiment.is_done
 
 
@@ -749,7 +798,9 @@ def test_suggest_n_max_trials(monkeypatch, producer):
 
     def suggest_n(self, num):
         """Return duplicated points based on `num`"""
-        return [("gru", "rnn")] * num
+        return [
+            format_trials.tuple_to_trial(("gru", "rnn"), producer.algorithm.space)
+        ] * num
 
     monkeypatch.setattr(
         producer.experiment.algorithms.algorithm.__class__, "suggest", suggest_n
@@ -760,13 +811,13 @@ def suggest_n(self, num):
     # Setup naive algorithm
     producer.update()
 
-    assert len(producer.suggest(50)) == 3
+    assert producer.adjust_pool_size(50) == 3
     # Test pool_size is the min selected
-    assert len(producer.suggest(2)) == 2
+    assert producer.adjust_pool_size(2) == 2
     producer.experiment.max_trials = 7
-    assert len(producer.suggest(50)) == 1
+    assert producer.adjust_pool_size(50) == 1
     producer.experiment.max_trials = 5
-    assert len(producer.suggest(50)) == 1
+    assert producer.adjust_pool_size(50) == 1
 
     trials = producer.experiment.fetch_trials()
     for trial in trials[:4]:
@@ -778,6 +829,6 @@ def suggest_n(self, num):
     producer.update()
 
     # There is now 3 completed and 4 broken. Max trials is 5. Producer should suggest 2
-    assert len(producer.suggest(50)) == 2
+    assert producer.adjust_pool_size(50) == 2
     # Test pool_size is the min selected
-    assert len(producer.suggest(1)) == 1
+    assert producer.adjust_pool_size(1) == 1

From f7d0dd88d7eb9c796048cdfea0c96582e3fc395e Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Thu, 11 Nov 2021 12:58:40 -0500
Subject: [PATCH 22/34] Adapt gradient descent test algo to new observe
 interface

---
 .../src/orion/algo/gradient_descent.py                | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/tests/functional/gradient_descent_algo/src/orion/algo/gradient_descent.py b/tests/functional/gradient_descent_algo/src/orion/algo/gradient_descent.py
index 718b0d397..76e6ad85a 100644
--- a/tests/functional/gradient_descent_algo/src/orion/algo/gradient_descent.py
+++ b/tests/functional/gradient_descent_algo/src/orion/algo/gradient_descent.py
@@ -11,6 +11,7 @@
 import numpy
 
 from orion.algo.base import BaseAlgorithm
+from orion.core.utils import format_trials
 
 
 class Gradient_Descent(BaseAlgorithm):
@@ -39,17 +40,19 @@ def suggest(self, num):
             return self.space.sample(1)
 
         self.current_point -= self.learning_rate * self.gradient
-        return [self.current_point]
+        return [format_trials.tuple_to_trial(self.current_point, self.space)]
 
-    def observe(self, points, results):
+    def observe(self, trials):
         """Observe evaluation `results` corresponding to list of `points` in
         space.
 
         Save current point and gradient corresponding to this point.
 
         """
-        self.current_point = numpy.asarray(points[-1])
-        self.gradient = numpy.asarray(results[-1]["gradient"])
+        self.current_point = numpy.asarray(
+            format_trials.trial_to_tuple(trials[-1], self.space)
+        )
+        self.gradient = numpy.asarray(trials[-1].gradient.value)
         self.has_observed_once = True
 
     @property

From 5c53fbd4c6b092fff9d859976ec064db39927df4 Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Thu, 11 Nov 2021 14:34:09 -0500
Subject: [PATCH 23/34] Remove pack/unpack_point

Why:

Deprecated and to be removed for v0.2.0. The newest versions
regroup_dims and flatten_dims are also useless now due to the flattened
space. They are also removed.
---
 src/orion/algo/space.py        | 27 -------------
 src/orion/algo/tpe.py          |  1 -
 src/orion/core/utils/points.py | 74 ----------------------------------
 3 files changed, 102 deletions(-)
 delete mode 100644 src/orion/core/utils/points.py

diff --git a/src/orion/algo/space.py b/src/orion/algo/space.py
index bdd2a9488..06ebaac59 100644
--- a/src/orion/algo/space.py
+++ b/src/orion/algo/space.py
@@ -36,7 +36,6 @@
 from scipy.stats import distributions
 
 from orion.core.utils import float_to_digits_list, format_trials
-from orion.core.utils.points import flatten_dims, regroup_dims
 from orion.core.utils.flatten import flatten
 
 logger = logging.getLogger(__name__)
@@ -1074,29 +1073,3 @@ def cardinality(self):
         for dim in self.values():
             capacities *= dim.cardinality
         return capacities
-
-
-def pack_point(point, space):
-    """Take a list of points and pack it appropriately as a point from `space`.
-
-    This function is deprecated and will be removed in v0.2.0. Use
-    `orion.core.utils.points.regroup_dims` instead.
-    """
-    logger.warning(
-        "`pack_point` is deprecated and will be removed in v0.2.0. Use "
-        "`orion.core.utils.points.regroup_dims` instead."
-    )
-    return regroup_dims(point, space)
-
-
-def unpack_point(point, space):
-    """Flatten `point` in `space` and convert it to a 1D list.
-
-    This function is deprecated and will be removed in v0.2.0. Use
-    `orion.core.utils.points.flatten_dims` instead.
-    """
-    logger.warning(
-        "`unpack_point` is deprecated and will be removed in v0.2.0. Use "
-        "`orion.core.utils.points.regroup_dims` instead."
-    )
-    return flatten_dims(point, space)
diff --git a/src/orion/algo/tpe.py b/src/orion/algo/tpe.py
index bfad31180..b55fbbef6 100644
--- a/src/orion/algo/tpe.py
+++ b/src/orion/algo/tpe.py
@@ -10,7 +10,6 @@
 
 from orion.algo.base import BaseAlgorithm
 from orion.core.utils import format_trials
-from orion.core.utils.points import flatten_dims, regroup_dims
 
 logger = logging.getLogger(__name__)
 
diff --git a/src/orion/core/utils/points.py b/src/orion/core/utils/points.py
deleted file mode 100644
index 67ef5c7fe..000000000
--- a/src/orion/core/utils/points.py
+++ /dev/null
@@ -1,74 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Utility functions for manipulating trial points
-===============================================
-
-Conversion functions between higher shape points and lists.
-
-"""
-
-
-def regroup_dims(point, space):
-    """Take a list of items representing a point and regroup them appropriately as
-    a point from `space`.
-
-    Parameters
-    ----------
-    point: array
-        Points to be regrouped.
-    space: `orion.algo.space.Space`
-        The optimization space.
-
-    Returns
-    -------
-    list or tuple
-
-    """
-    regrouped = []
-    idx = 0
-
-    for dimension in space.values():
-        shape = dimension.shape
-        if shape:
-            assert len(shape) == 1
-            next_dim = idx + shape[0]
-            regrouped.append(tuple(point[idx:next_dim]))
-            idx = next_dim
-        else:
-            regrouped.append(point[idx])
-            idx += 1
-
-    if regrouped not in space:
-        raise AttributeError(
-            "The point {} is not a valid point of space {}".format(point, space)
-        )
-
-    return regrouped
-
-
-def flatten_dims(point, space):
-    """Flatten `point` in `space` and convert it to a list.
-
-    Parameters
-    ----------
-    point: array
-        Points to be regrouped.
-    space: `orion.algo.space.Space`
-        The optimization space.
-
-    Returns
-    -------
-    list
-
-    """
-    flattened = []
-
-    for subpoint, dimension in zip(point, space.values()):
-        shape = dimension.shape
-        if shape:
-            assert len(shape) == 1
-            flattened.extend(subpoint)
-        else:
-            flattened.append(subpoint)
-
-    return flattened

From b8228c94331816eef64b0a0a364b9f832a4c75ef Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Thu, 11 Nov 2021 14:37:18 -0500
Subject: [PATCH 24/34] Adjust analysis to new transform(trial) interface

---
 src/orion/analysis/base.py                          | 12 +++++++++++-
 src/orion/analysis/partial_dependency_utils.py      |  6 +++++-
 src/orion/core/worker/transformer.py                |  7 ++++---
 src/orion/testing/__init__.py                       | 12 +++++++-----
 tests/unittests/analysis/test_partial_dependency.py |  6 +++++-
 5 files changed, 32 insertions(+), 11 deletions(-)

diff --git a/src/orion/analysis/base.py b/src/orion/analysis/base.py
index 02d506d9d..4162f0816 100644
--- a/src/orion/analysis/base.py
+++ b/src/orion/analysis/base.py
@@ -15,6 +15,7 @@
 )
 
 from orion.core.worker.transformer import build_required_space
+from orion.core.utils import format_trials
 
 _regressors_ = {
     "AdaBoostRegressor": AdaBoostRegressor,
@@ -169,8 +170,17 @@ def to_numpy(trials, space):
 
 def flatten_numpy(trials_array, flattened_space):
     """Flatten dimensions"""
+
     flattened_points = numpy.array(
-        [flattened_space.transform(point[:-1]) for point in trials_array]
+        [
+            format_trials.trial_to_tuple(
+                flattened_space.transform(
+                    format_trials.tuple_to_trial(point[:-1], flattened_space.original)
+                ),
+                flattened_space,
+            )
+            for point in trials_array
+        ]
     )
 
     return numpy.concatenate((flattened_points, trials_array[:, -1:]), axis=1)
diff --git a/src/orion/analysis/partial_dependency_utils.py b/src/orion/analysis/partial_dependency_utils.py
index 8f3a3862a..cbd16fb6f 100644
--- a/src/orion/analysis/partial_dependency_utils.py
+++ b/src/orion/analysis/partial_dependency_utils.py
@@ -10,6 +10,7 @@
 
 from orion.analysis.base import flatten_numpy, flatten_params, to_numpy, train_regressor
 from orion.core.worker.transformer import build_required_space
+from orion.core.utils import format_trials
 
 
 def partial_dependency(
@@ -79,7 +80,10 @@ def partial_dependency(
     data = flatten_numpy(data, flattened_space)
     model = train_regressor(model, data, **kwargs)
 
-    data = flattened_space.sample(n_samples)
+    data = [
+        format_trials.trial_to_tuple(trial, flattened_space)
+        for trial in flattened_space.sample(n_samples)
+    ]
     data = pandas.DataFrame(data, columns=flattened_space.keys())
 
     partial_dependencies = dict()
diff --git a/src/orion/core/worker/transformer.py b/src/orion/core/worker/transformer.py
index e31811a44..593b393df 100644
--- a/src/orion/core/worker/transformer.py
+++ b/src/orion/core/worker/transformer.py
@@ -16,7 +16,7 @@
 
 from orion.algo.space import Categorical, Dimension, Fidelity, Integer, Real, Space
 from orion.core.utils import format_trials
-from orion.core.utils.flatten import unflatten
+from orion.core.utils.flatten import flatten
 
 NON_LINEAR = ["loguniform", "reciprocal"]
 
@@ -789,7 +789,7 @@ def __init__(self, space, *args, **kwargs):
     def transform(self, trial):
         """Transform a point that was in the original space to be in this one."""
         transformed_point = tuple(
-            dim.transform(trial.params[name]) for name, dim in self.items()
+            dim.transform(flatten(trial.params)[name]) for name, dim in self.items()
         )
 
         return change_trial_params(trial, transformed_point, self)
@@ -799,7 +799,8 @@ def reverse(self, transformed_trial):
         to be in the original one.
         """
         reversed_point = tuple(
-            dim.reverse(transformed_trial.params[name]) for name, dim in self.items()
+            dim.reverse(flatten(transformed_trial.params)[name])
+            for name, dim in self.items()
         )
 
         return change_trial_params(
diff --git a/src/orion/testing/__init__.py b/src/orion/testing/__init__.py
index 1271ff65c..2c4b7032b 100644
--- a/src/orion/testing/__init__.py
+++ b/src/orion/testing/__init__.py
@@ -127,11 +127,13 @@ def mock_space_iterate(monkeypatch):
     sample = orion.algo.space.Space.sample
 
     def iterate(self, seed, *args, **kwargs):
-        """Return the points with seed value instead of sampling"""
-        points = []
-        for point in sample(self, seed=seed, *args, **kwargs):
-            points.append([seed] * len(point))
-        return points
+        """Return the trials with seed value instead of sampling"""
+        trials = []
+        for trial in sample(self, seed=seed, *args, **kwargs):
+            trials.append(
+                trial.branch(params={param: seed for param in trial.params.keys()})
+            )
+        return trials
 
     monkeypatch.setattr("orion.algo.space.Space.sample", iterate)
 
diff --git a/tests/unittests/analysis/test_partial_dependency.py b/tests/unittests/analysis/test_partial_dependency.py
index 4c8f9a6ad..ffeb252d7 100644
--- a/tests/unittests/analysis/test_partial_dependency.py
+++ b/tests/unittests/analysis/test_partial_dependency.py
@@ -22,6 +22,7 @@
 )
 from orion.core.io.space_builder import SpaceBuilder
 from orion.core.worker.transformer import build_required_space
+from orion.core.utils import format_trials
 
 data = pd.DataFrame(
     data={
@@ -165,7 +166,10 @@ def test_partial_dependency_grid(hspace):
 
     n_points = 5
     n_samples = 20
-    samples = flattened_space.sample(n_samples)
+    samples = [
+        format_trials.trial_to_tuple(trial, flattened_space)
+        for trial in flattened_space.sample(n_samples)
+    ]
     samples = pd.DataFrame(samples, columns=flattened_space.keys())
 
     params = ["x", "y[0]", "y[2]", "z"]

From 72d45357ad22b2bbfeb3a51e04fadfffeac8b6ec Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Thu, 11 Nov 2021 14:37:49 -0500
Subject: [PATCH 25/34] Fix Dumbalgo point -> trial attributes

---
 tests/conftest.py                             |  6 ++--
 tests/functional/demo/test_demo.py            |  2 +-
 tests/unittests/algo/test_hyperband.py        |  1 -
 .../client/test_experiment_client.py          | 34 +++++++++----------
 tests/unittests/core/test_primary_algo.py     |  6 ++--
 5 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index b90f4129d..f8d9fa211 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -90,7 +90,7 @@ def __init__(
         self._index = 0
         self._trials = []
         self._suggested = None
-        self._score_point = None
+        self._score_trial = None
         self._judge_trial = None
         self._measurements = None
         self.pool_size = 1
@@ -161,9 +161,9 @@ def observe(self, trials):
         super(DumbAlgo, self).observe(trials)
         self._trials += trials
 
-    def score(self, point):
+    def score(self, trial):
         """Log and return stab."""
-        self._score_point = point
+        self._score_trial = trial
         return self.scoring
 
     def judge(self, trial, measurements):
diff --git a/tests/functional/demo/test_demo.py b/tests/functional/demo/test_demo.py
index c95fac11d..57642084e 100644
--- a/tests/functional/demo/test_demo.py
+++ b/tests/functional/demo/test_demo.py
@@ -288,7 +288,7 @@ def test_demo_four_workers(storage, monkeypatch):
     status = defaultdict(int)
     for trial in trials:
         status[trial.status] += 1
-    assert status["completed"] == 20
+    assert status["completed"] >= 20
     assert status["new"] < 5
     params = trials[-1].params
     assert len(params) == 1
diff --git a/tests/unittests/algo/test_hyperband.py b/tests/unittests/algo/test_hyperband.py
index 2e8ac8332..05762327d 100644
--- a/tests/unittests/algo/test_hyperband.py
+++ b/tests/unittests/algo/test_hyperband.py
@@ -38,7 +38,6 @@ def create_rung_from_points(points, n_trials, resources):
     return dict(n_trials=n_trials, resources=resources, results=results)
 
 
-
 def compare_registered_trial(registered_trial, trial):
     assert registered_trial[0] == trial.objective.value
     assert registered_trial[1].to_dict() == trial.to_dict()
diff --git a/tests/unittests/client/test_experiment_client.py b/tests/unittests/client/test_experiment_client.py
index f961cf0cf..b2724dff4 100644
--- a/tests/unittests/client/test_experiment_client.py
+++ b/tests/unittests/client/test_experiment_client.py
@@ -20,6 +20,7 @@
 from orion.executor.joblib_backend import Joblib
 from orion.storage.base import get_storage
 from orion.testing import create_experiment, mock_space_iterate
+from orion.core.utils import format_trials
 
 config = dict(
     name="supernaekei",
@@ -246,9 +247,8 @@ def test_insert_bad_params(self):
             with pytest.raises(ValueError) as exc:
                 client.insert(dict(x="bad bad bad"))
 
-            assert (
-                "Dimension x value bad bad bad is outside of prior uniform(0, 200)"
-                == str(exc.value)
+            assert "Parameters values {'x': 'bad bad bad'} are outside of space" in str(
+                exc.value
             )
             assert client._pacemakers == {}
 
@@ -561,26 +561,26 @@ def test_suggest_race_condition(self, monkeypatch):
         mock_space_iterate(monkeypatch)
         new_value = 50.0
 
-        # algo will suggest once an already existing trial
-        def amnesia(num=1):
-            """Suggest a new value and then always suggest the same"""
-            if amnesia.count == 0:
-                value = [0]
-            else:
-                value = [new_value]
-
-            amnesia.count += 1
-
-            return [value]
-
-        amnesia.count = 0
-
         with create_experiment(config, base_trial, statuses=["completed"]) as (
             cfg,
             experiment,
             client,
         ):
 
+            # algo will suggest once an already existing trial
+            def amnesia(num=1):
+                """Suggest a new value and then always suggest the same"""
+                if amnesia.count == 0:
+                    value = [0]
+                else:
+                    value = [new_value]
+
+                amnesia.count += 1
+
+                return [format_trials.tuple_to_trial(value, experiment.space)]
+
+            amnesia.count = 0
+
             monkeypatch.setattr(experiment.algorithms, "suggest", amnesia)
 
             assert len(experiment.fetch_trials()) == 1
diff --git a/tests/unittests/core/test_primary_algo.py b/tests/unittests/core/test_primary_algo.py
index 70119f56d..2f22d5aa2 100644
--- a/tests/unittests/core/test_primary_algo.py
+++ b/tests/unittests/core/test_primary_algo.py
@@ -87,7 +87,7 @@ def test_observe(self, palgo, fixed_suggestion):
         """Observe wraps observations."""
         backward.algo_observe(palgo, [fixed_suggestion], [dict(objective=5)])
         palgo.observe([fixed_suggestion])
-        assert palgo.algorithm._trials[0].trial == fixed_suggestion
+        assert palgo.algorithm._trials[0].params == fixed_suggestion.params
 
     def test_isdone(self, palgo):
         """Wrap isdone."""
@@ -105,13 +105,13 @@ def test_score(self, palgo, fixed_suggestion):
         """Wrap score."""
         palgo.algorithm.scoring = 60
         assert palgo.score(fixed_suggestion) == 60
-        assert palgo.algorithm._score_point.trial == fixed_suggestion
+        assert palgo.algorithm._score_trial.params == fixed_suggestion.params
 
     def test_judge(self, palgo, fixed_suggestion):
         """Wrap judge."""
         palgo.algorithm.judgement = "naedw"
         assert palgo.judge(fixed_suggestion, 8) == "naedw"
-        assert palgo.algorithm._judge_trial.trial is fixed_suggestion
+        assert palgo.algorithm._judge_trial.params == fixed_suggestion.params
         assert palgo.algorithm._measurements == 8
         with pytest.raises(ValueError, match="not contained in space"):
             del fixed_suggestion._params[-1]

From 85fa11a3923fb5c64e6a4573f30e163acb719eef Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Thu, 11 Nov 2021 14:38:57 -0500
Subject: [PATCH 26/34] isort

---
 src/orion/analysis/base.py                          |  2 +-
 src/orion/analysis/partial_dependency_utils.py      |  2 +-
 tests/unittests/algo/test_asha.py                   | 11 +++++------
 tests/unittests/algo/test_evolution_es.py           | 12 +++++-------
 tests/unittests/algo/test_hyperband.py              |  2 +-
 tests/unittests/algo/test_space.py                  |  2 +-
 tests/unittests/analysis/test_partial_dependency.py |  2 +-
 tests/unittests/client/test_experiment_client.py    |  2 +-
 tests/unittests/core/test_strategy.py               |  2 +-
 tests/unittests/core/worker/test_producer.py        |  2 +-
 10 files changed, 18 insertions(+), 21 deletions(-)

diff --git a/src/orion/analysis/base.py b/src/orion/analysis/base.py
index 4162f0816..2b5c1061f 100644
--- a/src/orion/analysis/base.py
+++ b/src/orion/analysis/base.py
@@ -14,8 +14,8 @@
     RandomForestRegressor,
 )
 
-from orion.core.worker.transformer import build_required_space
 from orion.core.utils import format_trials
+from orion.core.worker.transformer import build_required_space
 
 _regressors_ = {
     "AdaBoostRegressor": AdaBoostRegressor,
diff --git a/src/orion/analysis/partial_dependency_utils.py b/src/orion/analysis/partial_dependency_utils.py
index cbd16fb6f..a4ea3f135 100644
--- a/src/orion/analysis/partial_dependency_utils.py
+++ b/src/orion/analysis/partial_dependency_utils.py
@@ -9,8 +9,8 @@
 import pandas
 
 from orion.analysis.base import flatten_numpy, flatten_params, to_numpy, train_regressor
-from orion.core.worker.transformer import build_required_space
 from orion.core.utils import format_trials
+from orion.core.worker.transformer import build_required_space
 
 
 def partial_dependency(
diff --git a/tests/unittests/algo/test_asha.py b/tests/unittests/algo/test_asha.py
index 15df89d93..d96d2f913 100644
--- a/tests/unittests/algo/test_asha.py
+++ b/tests/unittests/algo/test_asha.py
@@ -6,12 +6,6 @@
 
 import numpy as np
 import pytest
-
-from orion.algo.asha import ASHA, ASHABracket, compute_budgets
-from orion.algo.space import Fidelity, Integer, Real, Space
-from orion.testing.algo import BaseAlgoTests
-from orion.testing.trial import create_trial
-
 from test_hyperband import (
     compare_registered_trial,
     create_rung_from_points,
@@ -19,6 +13,11 @@
     force_observe,
 )
 
+from orion.algo.asha import ASHA, ASHABracket, compute_budgets
+from orion.algo.space import Fidelity, Integer, Real, Space
+from orion.testing.algo import BaseAlgoTests
+from orion.testing.trial import create_trial
+
 
 @pytest.fixture
 def space():
diff --git a/tests/unittests/algo/test_evolution_es.py b/tests/unittests/algo/test_evolution_es.py
index 92384e48b..0823a14ee 100644
--- a/tests/unittests/algo/test_evolution_es.py
+++ b/tests/unittests/algo/test_evolution_es.py
@@ -7,13 +7,6 @@
 
 import numpy as np
 import pytest
-
-from orion.algo.evolution_es import BracketEVES, EvolutionES, compute_budgets
-from orion.algo.space import Fidelity, Real, Space
-from orion.testing.algo import BaseAlgoTests
-
-from orion.testing.trial import create_trial
-
 from test_hyperband import (
     compare_registered_trial,
     create_rung_from_points,
@@ -21,6 +14,11 @@
     force_observe,
 )
 
+from orion.algo.evolution_es import BracketEVES, EvolutionES, compute_budgets
+from orion.algo.space import Fidelity, Real, Space
+from orion.testing.algo import BaseAlgoTests
+from orion.testing.trial import create_trial
+
 
 @pytest.fixture
 def space():
diff --git a/tests/unittests/algo/test_hyperband.py b/tests/unittests/algo/test_hyperband.py
index 05762327d..8f2e3a994 100644
--- a/tests/unittests/algo/test_hyperband.py
+++ b/tests/unittests/algo/test_hyperband.py
@@ -11,7 +11,7 @@
 from orion.algo.hyperband import Hyperband, HyperbandBracket, compute_budgets
 from orion.algo.space import Fidelity, Integer, Real, Space
 from orion.testing.algo import BaseAlgoTests, phase
-from orion.testing.trial import create_trial, compare_trials
+from orion.testing.trial import compare_trials, create_trial
 
 
 def create_trial_for_hb(point, objective=None):
diff --git a/tests/unittests/algo/test_space.py b/tests/unittests/algo/test_space.py
index 03314fa85..a6933c910 100644
--- a/tests/unittests/algo/test_space.py
+++ b/tests/unittests/algo/test_space.py
@@ -19,8 +19,8 @@
     Space,
     check_random_state,
 )
-from orion.core.worker.trial import Trial
 from orion.core.utils import format_trials
+from orion.core.worker.trial import Trial
 
 
 class TestCheckRandomState:
diff --git a/tests/unittests/analysis/test_partial_dependency.py b/tests/unittests/analysis/test_partial_dependency.py
index ffeb252d7..ab391533f 100644
--- a/tests/unittests/analysis/test_partial_dependency.py
+++ b/tests/unittests/analysis/test_partial_dependency.py
@@ -21,8 +21,8 @@
     reverse,
 )
 from orion.core.io.space_builder import SpaceBuilder
-from orion.core.worker.transformer import build_required_space
 from orion.core.utils import format_trials
+from orion.core.worker.transformer import build_required_space
 
 data = pd.DataFrame(
     data={
diff --git a/tests/unittests/client/test_experiment_client.py b/tests/unittests/client/test_experiment_client.py
index b2724dff4..387b48f1a 100644
--- a/tests/unittests/client/test_experiment_client.py
+++ b/tests/unittests/client/test_experiment_client.py
@@ -11,6 +11,7 @@
 
 import orion.core
 from orion.core.io.database import DuplicateKeyError
+from orion.core.utils import format_trials
 from orion.core.utils.exceptions import (
     BrokenExperiment,
     CompletedExperiment,
@@ -20,7 +21,6 @@
 from orion.executor.joblib_backend import Joblib
 from orion.storage.base import get_storage
 from orion.testing import create_experiment, mock_space_iterate
-from orion.core.utils import format_trials
 
 config = dict(
     name="supernaekei",
diff --git a/tests/unittests/core/test_strategy.py b/tests/unittests/core/test_strategy.py
index 0bbc7c69d..c46b125f3 100644
--- a/tests/unittests/core/test_strategy.py
+++ b/tests/unittests/core/test_strategy.py
@@ -5,6 +5,7 @@
 
 import pytest
 
+from orion.core.utils import backward
 from orion.core.worker.strategy import (
     MaxParallelStrategy,
     MeanParallelStrategy,
@@ -13,7 +14,6 @@
     strategy_factory,
 )
 from orion.core.worker.trial import Trial
-from orion.core.utils import backward
 
 
 @pytest.fixture
diff --git a/tests/unittests/core/worker/test_producer.py b/tests/unittests/core/worker/test_producer.py
index e02215eb2..4048c0600 100644
--- a/tests/unittests/core/worker/test_producer.py
+++ b/tests/unittests/core/worker/test_producer.py
@@ -8,8 +8,8 @@
 import pytest
 
 from orion.core.io.experiment_builder import build
-from orion.core.utils.exceptions import SampleTimeout, WaitingForTrials
 from orion.core.utils import format_trials
+from orion.core.utils.exceptions import SampleTimeout, WaitingForTrials
 from orion.core.worker.producer import Producer
 from orion.core.worker.trial import Trial
 from orion.testing.trial import compare_trials

From 7ad527baab39ee7e67bd90fb9d3962a204c2d501 Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Thu, 11 Nov 2021 14:44:18 -0500
Subject: [PATCH 27/34] pylint

---
 src/orion/core/utils/backward.py      |  1 +
 src/orion/core/utils/format_trials.py |  4 ----
 src/orion/core/worker/producer.py     |  1 -
 src/orion/core/worker/transformer.py  | 13 ++-----------
 4 files changed, 3 insertions(+), 16 deletions(-)

diff --git a/src/orion/core/utils/backward.py b/src/orion/core/utils/backward.py
index 2cc52ef8a..4f5bf31b2 100644
--- a/src/orion/core/utils/backward.py
+++ b/src/orion/core/utils/backward.py
@@ -175,6 +175,7 @@ def port_algo_config(config):
 
 
 def algo_observe(algo, trials, results):
+    """Convert trials so that algo can observe with legacy format (trials, results)."""
     for trial, trial_results in zip(trials, results):
         for name, trial_result in trial_results.items():
             trial.results.append(Trial.Result(name=name, type=name, value=trial_result))
diff --git a/src/orion/core/utils/format_trials.py b/src/orion/core/utils/format_trials.py
index 442c323b0..75a7974f1 100644
--- a/src/orion/core/utils/format_trials.py
+++ b/src/orion/core/utils/format_trials.py
@@ -116,7 +116,3 @@ def get_trial_results(trial):
 def standard_param_name(name):
     """Convert parameter name to namespace format"""
     return name.lstrip("/").lstrip("-").replace("-", "_")
-
-
-def update_params(trial, params):
-    """Convert params in Param objects and update trial"""
diff --git a/src/orion/core/worker/producer.py b/src/orion/core/worker/producer.py
index 99e75f793..633f07fd8 100644
--- a/src/orion/core/worker/producer.py
+++ b/src/orion/core/worker/producer.py
@@ -13,7 +13,6 @@
 
 import orion.core
 from orion.core.io.database import DuplicateKeyError
-from orion.core.utils import format_trials
 from orion.core.utils.exceptions import SampleTimeout, WaitingForTrials
 from orion.core.worker.trial import Trial
 from orion.core.worker.trials_history import TrialsHistory
diff --git a/src/orion/core/worker/transformer.py b/src/orion/core/worker/transformer.py
index 593b393df..2206d4cb1 100644
--- a/src/orion/core/worker/transformer.py
+++ b/src/orion/core/worker/transformer.py
@@ -706,6 +706,7 @@ def cardinality(self):
 
     @property
     def default_value(self):
+        """Return the default value for this dimensions"""
         if (
             self.original_dimension.default_value
             is self.original_dimension.NO_DEFAULT_VALUE
@@ -894,19 +895,9 @@ def cardinality(self):
         return self.original.cardinality
 
 
-def create_transformed_trial(trial, transformed_point, space):
-    """Convert point into Trial.Param objects and return a TransformedTrial"""
-    return TransformedTrial(
-        trial, format_trials.tuple_to_trial(transformed_point, space)._params
-    )
-
-    new_trial = copy.copy(trial)
-    new_trial._params = format_trials.tuple_to_trial(transformed_point, space)._params
-    return new_trial
-
-
 def change_trial_params(trial, point, space):
     """Convert params in Param objects and update trial"""
     new_trial = copy.copy(trial)
+    # pylint: disable=protected-access
     new_trial._params = format_trials.tuple_to_trial(point, space)._params
     return new_trial

From 5cee651b2400ea0b9e6ec72f5d9a054774b42dea Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Thu, 11 Nov 2021 14:57:53 -0500
Subject: [PATCH 28/34] Remove points doc (module removed)

---
 docs/src/code/core/utils.rst        | 1 -
 docs/src/code/core/utils/points.rst | 5 -----
 2 files changed, 6 deletions(-)
 delete mode 100644 docs/src/code/core/utils/points.rst

diff --git a/docs/src/code/core/utils.rst b/docs/src/code/core/utils.rst
index 6f87aadc3..ebe67027f 100644
--- a/docs/src/code/core/utils.rst
+++ b/docs/src/code/core/utils.rst
@@ -9,7 +9,6 @@ Utilities
    utils/format_trials
    utils/format_terminal
    utils/singleton
-   utils/points
 
 .. automodule:: orion.core.utils
    :members:
diff --git a/docs/src/code/core/utils/points.rst b/docs/src/code/core/utils/points.rst
deleted file mode 100644
index 3d3418875..000000000
--- a/docs/src/code/core/utils/points.rst
+++ /dev/null
@@ -1,5 +0,0 @@
-Points
-======
-
-.. automodule:: orion.core.utils.points
-   :members:

From cf1ac0bea47234ce7aa8ef9ffcf9dc635de69c4b Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Thu, 11 Nov 2021 15:02:38 -0500
Subject: [PATCH 29/34] Fix doc reference

---
 src/orion/core/utils/format_trials.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/orion/core/utils/format_trials.py b/src/orion/core/utils/format_trials.py
index 75a7974f1..342481224 100644
--- a/src/orion/core/utils/format_trials.py
+++ b/src/orion/core/utils/format_trials.py
@@ -72,7 +72,7 @@ def tuple_to_trial(data, space, status="new"):
     space: `orion.algo.space.Space`
         Definition of problem's domain.
     status: str, optional
-        Status of the trial. One of `orion.core.worker.trial.Trial.allowed_stati`.
+        Status of the trial. One of ``orion.core.worker.trial.Trial.allowed_stati``.
 
     Returns
     -------

From 6b06eb939ebff6f7d88551ef01f789baf26568a8 Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Thu, 11 Nov 2021 16:18:54 -0500
Subject: [PATCH 30/34] Dims of shape (1, ) should use Views

Why:

When flattening the space, dims of shape (1, ) should be flattened
as well otherwise the parameters will be a list of one element.
---
 src/orion/core/worker/transformer.py     |  4 +---
 tests/unittests/core/test_transformer.py | 23 +++++++++++++----------
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/src/orion/core/worker/transformer.py b/src/orion/core/worker/transformer.py
index 2206d4cb1..30f1e71db 100644
--- a/src/orion/core/worker/transformer.py
+++ b/src/orion/core/worker/transformer.py
@@ -117,7 +117,7 @@ def reshape(space, shape_requirement):
     reshaped_space = ReshapedSpace(space)
 
     for dim_index, dim in enumerate(space.values()):
-        if not dim.shape or numpy.prod(dim.shape) == 1:
+        if not dim.shape:
             reshaped_space.register(
                 ReshapedDimension(
                     transformer=Identity(dim.type),
@@ -568,8 +568,6 @@ def first(self):
 
     def transform(self, point):
         """Only return one element of the group"""
-        print(point, type(point))
-        print(self.index)
         return numpy.array(point)[self.index]
 
     def reverse(self, transformed_point, index=None):
diff --git a/tests/unittests/core/test_transformer.py b/tests/unittests/core/test_transformer.py
index 408dc6e0c..76448134d 100644
--- a/tests/unittests/core/test_transformer.py
+++ b/tests/unittests/core/test_transformer.py
@@ -685,7 +685,7 @@ def rdim2(dim2, rdims2):
 @pytest.fixture()
 def dim3():
     """Create an example of integer `Dimension`."""
-    return Integer("yolo3", "uniform", 3, 7)
+    return Integer("yolo3", "uniform", 3, 7, shape=(1,))
 
 
 @pytest.fixture()
@@ -698,7 +698,10 @@ def tdim3(dim3):
 def rdims3(tdim3):
     """Create an example of integer `Dimension`."""
     rdim3 = ReshapedDimension(
-        transformer=Identity(tdim3.type), original_dimension=tdim3, index=2
+        transformer=View(tdim3.shape, (0,), tdim3.type),
+        original_dimension=tdim3,
+        name="yolo3[0]",
+        index=2,
     )
 
     return {tdim3.name: rdim3}
@@ -1110,7 +1113,7 @@ def test_interval(self, rspace):
     def test_reshape(self, space, rspace):
         """Verify that the dimension are reshaped properly, forward and backward"""
         trial = format_trials.tuple_to_trial(
-            (numpy.arange(6).reshape(3, 2).tolist(), "3", 10), space
+            (numpy.arange(6).reshape(3, 2).tolist(), "3", [10]), space
         )
 
         rtrial = format_trials.tuple_to_trial(
@@ -1184,7 +1187,7 @@ def test_no_requirement(self, space_each_type):
             == """\
 Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
        Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=2),
-       Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None),
+       Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(1,), default value=None),
        Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None)),
        Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None)])\
 """
@@ -1204,7 +1207,7 @@ def test_integer_requirement(self, space_each_type):
             == """\
 Space([Quantize(Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None))),
        Enumerate(Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=2)),
-       Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None),
+       Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(1,), default value=None),
        Quantize(Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None))),
        Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None)])\
 """
@@ -1224,7 +1227,7 @@ def test_real_requirement(self, space_each_type):
             == """\
 Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
        OneHotEncode(Enumerate(Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=2))),
-       ReverseQuantize(Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None)),
+       ReverseQuantize(Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(1,), default value=None)),
        Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None)),
        ReverseQuantize(Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None))])\
 """
@@ -1244,7 +1247,7 @@ def test_numerical_requirement(self, space_each_type):
             == """\
 Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
        Enumerate(Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=2)),
-       Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None),
+       Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(1,), default value=None),
        Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None)),
        Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None)])\
 """
@@ -1264,7 +1267,7 @@ def test_linear_requirement(self, space_each_type):
             == """\
 Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
        Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=2),
-       Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None),
+       Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(1,), default value=None),
        Linearize(Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None))),
        Linearize(ReverseQuantize(Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None)))])\
 """
@@ -1276,7 +1279,7 @@ def test_flatten_requirement(self, space_each_type):
 
         # 1 integer + 1 categorical + 1 * (3, 2) shapes
         assert len(tspace) == 1 + 1 + 3 * (3 * 2)
-        assert str(tspace).count("View") == 3 * (3 * 2)
+        assert str(tspace).count("View") == 3 * (3 * 2) + 1
 
         i = 0
 
@@ -1304,7 +1307,7 @@ def test_flatten_requirement(self, space_each_type):
 
         # 1 integer + 4 categorical + 1 * (3, 2) shapes
         assert len(tspace) == 1 + 4 + 3 * (3 * 2)
-        assert str(tspace).count("View") == 4 + 3 * (3 * 2)
+        assert str(tspace).count("View") == 4 + 3 * (3 * 2) + 1
 
     def test_capacity(self, space_each_type):
         """Check transformer space capacity"""

From 2d43e2c13ea8134cc340c29d7f93c0025c138a6f Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Thu, 11 Nov 2021 16:20:02 -0500
Subject: [PATCH 31/34] Add entry points for Database

---
 setup.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/setup.py b/setup.py
index 941f272b9..b8a7aac87 100644
--- a/setup.py
+++ b/setup.py
@@ -54,6 +54,11 @@
             "tpe = orion.algo.tpe:TPE",
             "EvolutionES = orion.algo.evolution_es:EvolutionES",
         ],
+        "Database": [
+            "ephemeraldb = orion.core.io.database.ephemeraldb:EphemeralDB",
+            "pickleddb = orion.core.io.database.pickleddb:PickledDB",
+            "mongodb = orion.core.io.database.mongodb:MongoDB",
+        ],
         "BaseStorageProtocol": [
             "track = orion.storage.track:Track",
             "legacy = orion.storage.legacy:Legacy",

From e9b3c08fbe0bf2da0b009d4639345e6bc878b952 Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Thu, 11 Nov 2021 16:26:41 -0500
Subject: [PATCH 32/34] Add missing testing functions for trials

---
 src/orion/testing/trial.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 src/orion/testing/trial.py

diff --git a/src/orion/testing/trial.py b/src/orion/testing/trial.py
new file mode 100644
index 000000000..6d069e553
--- /dev/null
+++ b/src/orion/testing/trial.py
@@ -0,0 +1,26 @@
+import string
+
+from orion.core.worker.trial import Trial
+
+
+def create_trial(point, names=None, types=None, results=None):
+    if names is None:
+        names = string.ascii_lowercase[: len(point)]
+
+    if types is None:
+        types = ["real"] * len(point)
+
+    return Trial(
+        params=[
+            {"name": name, "value": value, "type": param_type}
+            for (name, value, param_type) in zip(names, point, types)
+        ],
+        results=[
+            {"name": name, "type": name, "value": value}
+            for name, value in results.items()
+        ],
+    )
+
+
+def compare_trials(trials, other_trials):
+    assert [t.params for t in trials] == [t.params for t in other_trials]

From e519a06cc32774c26db51847993efad42ca081c7 Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Fri, 12 Nov 2021 12:43:40 -0500
Subject: [PATCH 33/34] Take a snapshot of trials_info to deepcopy it

Why:

The deepcopy is failing on github-actions with error
`RuntimeError: dictionary changed size during iteration`. I have been
unable to reproduce the issue locally both with python 3.6 and 3.7. It
does fail on 3.7 on github-actions. Taking a copy of the dictionary to
do the deep copy should fix the issue only a dictionary inside some
trials is the source of the issue. The stack trace seams to hint towards
trials_info as the culprit however.

```
 tests/unittests/benchmark/test_benchmark_client.py:345:
  _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
  src/orion/benchmark/__init__.py:90: in process
      study.execute(n_workers)
  src/orion/benchmark/__init__.py:341: in execute
      experiment.workon(self.task, n_workers=n_workers, max_trials=max_trials)
  src/orion/client/experiment.py:767: in workon
      for _ in range(n_workers)
  src/orion/executor/joblib_backend.py:32: in wait
      return joblib.Parallel(n_jobs=self.n_workers)(futures)
  .tox/py/lib/python3.7/site-packages/joblib/parallel.py:1056: in __call__
      self.retrieve()
  .tox/py/lib/python3.7/site-packages/joblib/parallel.py:935: in retrieve
      self._output.extend(job.get(timeout=self.timeout))
  /opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/multiprocessing/pool.py:657: in get
      raise self._value
  /opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/multiprocessing/pool.py:121: in worker
      result = (True, func(*args, **kwds))
  .tox/py/lib/python3.7/site-packages/joblib/_parallel_backends.py:595: in __call__
      return self.func(*args, **kwargs)
  .tox/py/lib/python3.7/site-packages/joblib/parallel.py:263: in __call__
      for func, args, kwargs in self.items]
  .tox/py/lib/python3.7/site-packages/joblib/parallel.py:263: in <listcomp>
      for func, args, kwargs in self.items]
  src/orion/client/experiment.py:781: in _optimize
      with self.suggest(pool_size=pool_size) as trial:
  src/orion/client/experiment.py:560: in suggest
      trial = reserve_trial(self._experiment, self._producer, pool_size)
  src/orion/client/experiment.py:54: in reserve_trial
      producer.produce(pool_size)
  src/orion/core/worker/producer.py:115: in produce
      self.algorithm.set_state(self.naive_algorithm.state_dict)
  src/orion/core/worker/primary_algo.py:47: in state_dict
      return self.algorithm.state_dict
  src/orion/algo/tpe.py:265: in state_dict
      _state_dict = super(TPE, self).state_dict
  src/orion/algo/base.py:132: in state_dict
      return {"_trials_info": copy.deepcopy(self._trials_info)}
  /opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/copy.py:150: in deepcopy
      y = copier(x, memo)
  _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

  x = {'0ef99dad51485ac9518b49c19b43f4ec': (Trial(experiment=2, status='completed', params=x[0]:0.5497,x[1]:2.397,x[2]:-4.29...params=x[0]:-0.8816,x[1]:2.087,x[2]:1.176), {'constraint': [], 'gradient': None, 'objective': 1187.240632192948}), ...}
  memo = {140575410255504: datetime.datetime(2021, 11, 11, 22, 57, 32, 364584), 140575419437584: datetime.datetime(2021, 11, 11....datetime(2021, 11, 11, 22, 57, 32, 177829), 140575419438016: datetime.datetime(2021, 11, 11, 22, 57, 32, 241378), ...}
  deepcopy = <function deepcopy at 0x7fdac6dfb680>

      def _deepcopy_dict(x, memo, deepcopy=deepcopy):
          y = {}
          memo[id(x)] = y
  >       for key, value in x.items():
  E       RuntimeError: dictionary changed size during iteration
```
---
 src/orion/algo/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/orion/algo/base.py b/src/orion/algo/base.py
index 580d6dddb..5a4903cd5 100644
--- a/src/orion/algo/base.py
+++ b/src/orion/algo/base.py
@@ -129,7 +129,7 @@ def seed_rng(self, seed):
     @property
     def state_dict(self):
         """Return a state dict that can be used to reset the state of the algorithm."""
-        return {"_trials_info": copy.deepcopy(self._trials_info)}
+        return {"_trials_info": copy.deepcopy(dict(self._trials_info))}
 
     def set_state(self, state_dict):
         """Reset the state of the algorithm based on the given state_dict

From 63eda39c470cac91cca9f7dfb22a603f50e832e7 Mon Sep 17 00:00:00 2001
From: Xavier Bouthillier <xavier.bouthillier@umontreal.ca>
Date: Fri, 12 Nov 2021 13:35:10 -0500
Subject: [PATCH 34/34] Remove trailing TODO notes

---
 src/orion/algo/base.py    | 8 --------
 src/orion/testing/algo.py | 2 --
 2 files changed, 10 deletions(-)

diff --git a/src/orion/algo/base.py b/src/orion/algo/base.py
index 5a4903cd5..0756b8c72 100644
--- a/src/orion/algo/base.py
+++ b/src/orion/algo/base.py
@@ -176,12 +176,6 @@ def get_id(self, trial, ignore_fidelity=False):
             the trial. Defaults to False.
         """
 
-        # TODO: ******
-        # NOTE: The trial hash is based on experiment id. This should not matter for the
-        #       algorithms otherwise there will be a clash between trials sampled with current
-        #       experiment and trials sampled from parent ones in EVC.
-        # TODO: ******
-
         # Apply transforms and reverse to see data as it would come from DB
         # (Some transformations looses some info. ex: Precision transformation)
 
@@ -202,8 +196,6 @@ def fidelity_index(self):
 
         Returns None if there is no fidelity dimension.
         """
-        # TODO: Should we return the fidelity key name instead now that we work with
-        #       trials instead of points?
 
         def _is_fidelity(dim):
             return dim.type == "fidelity"
diff --git a/src/orion/testing/algo.py b/src/orion/testing/algo.py
index 392cfadfe..700a613e9 100644
--- a/src/orion/testing/algo.py
+++ b/src/orion/testing/algo.py
@@ -261,8 +261,6 @@ def force_observe(self, num, algo):
                 if trial.hash_name in ids:
                     raise RuntimeError(f"algo suggested a duplicate: {trial}")
                 ids.add(trial.hash_name)
-            # TODO: Why would we need the line below?? Looks like duplicating the work above
-            # ids |= set(trial.hash_name for trial in trials)
             self.observe_trials(trials, algo, objective)
             objective += len(trials)