feat: Add method to get the best configuration directly from Tuner, add com…

docs/source/faq.rst

@@ -337,6 +337,14 @@ You can take a look at this example
 `examples/launch_checkpoint_example.py <examples.html#retrieving-the-best-checkpoint>`__
 which shows how to retrieve the best checkpoint obtained after tuning an XGBoost model.
 
+How can I retrain the best model found after tuning?
+====================================================
+
+You can call ``tuner.trial_backend.start_trial(config=tuner.best_config())`` after tuning to retrain the best config,
+you can take a look at this example
+`examples/launch_plot_example.py <examples.html#plot-results-of-tuning-experiment>`__
+which shows how to retrain the best model found while tuning.
+
 Which schedulers make use of checkpointing?
 ===========================================
 

examples/launch_plot_results.py

@@ -25,7 +25,7 @@
 )
 
 if __name__ == "__main__":
-    logging.getLogger().setLevel(logging.DEBUG)
+    logging.getLogger().setLevel(logging.INFO)
 
     random_seed = 31415927
     max_steps = 100
@@ -64,8 +64,13 @@
     tuner.run()
 
     tuning_experiment = load_experiment(tuner.name)
-    print(tuning_experiment)
 
+    # Print the best configuration found from experiment-results
     print(f"best result found: {tuning_experiment.best_config()}")
 
+    # Plot the best value found over time
     tuning_experiment.plot()
+
+    # Print the best configuration found from the tuner and retrain it
+    trial_id, best_config = tuner.best_config()
+    tuner.trial_backend.start_trial(config=best_config)

syne_tune/experiments/experiment_result.py

@@ -30,7 +30,7 @@
     ST_TUNER_TIME,
 )
 from syne_tune.try_import import try_import_aws_message, try_import_visual_message
-from syne_tune.util import experiment_path, s3_experiment_path
+from syne_tune.util import experiment_path, s3_experiment_path, metric_name_mode
 
 try:
     import boto3
@@ -98,7 +98,7 @@ def plot_hypervolume(
             len(metrics_to_plot) > 1
         ), "Only one metric defined, cannot compute hypervolume"
 
-        metrics, metric_names, metric_modes = zip(
+        metric_names, metric_modes = zip(
             *[self._metric_name_mode(metric) for metric in metrics_to_plot]
         )
         assert np.all(
@@ -139,9 +139,7 @@ def plot(
             If None, the figure is shown
         :param plt_kwargs: Arguments to :func:`matplotlib.pyplot.plot`
         """
-        metric, metric_name, metric_mode = self._metric_name_mode(
-            metric_to_plot, verbose=True
-        )
+        metric_name, metric_mode = self._metric_name_mode(metric_to_plot)
 
         df = self.results
         if df is not None and len(df) > 0:
@@ -178,7 +176,7 @@ def best_config(self, metric: Union[str, int] = 0) -> Dict[str, Any]:
             default to 0 - first metric defined in the Scheduler
         :return: Configuration corresponding to best metric value
         """
-        metric, metric_name, metric_mode = self._metric_name_mode(metric, verbose=True)
+        metric_name, metric_mode = self._metric_name_mode(metric)
 
         # locate best result
         if metric_mode == "min":
@@ -190,41 +188,16 @@ def best_config(self, metric: Union[str, int] = 0) -> Dict[str, Any]:
         # Don't include internal fields
         return {k: v for k, v in res.items() if not k.startswith("st_")}
 
-    def _metric_name_mode(
-        self, metric: Union[str, int], verbose: bool = False
-    ) -> Tuple[int, str, str]:
+    def _metric_name_mode(self, metric: Union[str, int]) -> Tuple[str, str]:
         """
-        Determine the metric, name and its mode given ambiguous input.
+        Determine the name and its mode given ambiguous input.
         :param metric: Index or name of the selected metric
-        :param verbose: If True, prints a warning message when only one metric is selected from many
         """
-        if isinstance(metric, str):
-            assert (
-                metric in self.metric_names()
-            ), f"Attepted to use {metric} while available metrics are {self.metric_names()}"
-            metric_name = metric
-            metric = self.metric_names().index(metric)
-        elif isinstance(metric, int):
-            assert metric < len(
-                self.metric_names()
-            ), f"Attepted to use metric index={metric} with {len(self.metric_names())} availale metrics"
-            metric_name = self.metric_names()[metric]
-        else:
-            raise AttributeError(
-                f"metic must be <int> or <str> but {type(metric)} was provided"
-            )
-
-        if len(self.metric_names()) > 1 and verbose:
-            logger.warning(
-                "Several metrics exists, this will "
-                f"use metric={metric_name} (index={metric}) out of {self.metric_names()}."
-            )
-
-        metric_mode = self.metric_mode()
-        if isinstance(metric_mode, list):
-            metric_mode = metric_mode[metric]
-
-        return metric, metric_name, metric_mode
+        return metric_name_mode(
+            metric_names=self.metric_names(),
+            metric_mode=self.metric_mode(),
+            metric=metric,
+        )
 
 
 def download_single_experiment(

syne_tune/tuner.py

@@ -14,7 +14,7 @@
 import time
 from collections import OrderedDict
 from pathlib import Path
-from typing import Callable, Dict, List, Optional, Set, Tuple, Any
+from typing import Callable, Dict, List, Optional, Set, Tuple, Any, Union
 
 import dill as dill
 
@@ -44,6 +44,7 @@
     experiment_path,
     name_from_base,
     dump_json_with_numpy,
+    metric_name_mode,
 )
 
 logger = logging.getLogger(__name__)
@@ -683,3 +684,30 @@ def _default_callback():
         :return: Default callback to store results
         """
         return StoreResultsCallback()
+
+    def best_config(
+        self, metric: Optional[Union[str, int]] = 0
+    ) -> Tuple[int, Dict[str, Any]]:
+        """
+        :param metric: Indicates which metric to use, can be the index or a name of the metric.
+            default to 0 - first metric defined in the Scheduler
+        :return: the best configuration found while tuning for the metric given and the associated trial-id
+        """
+        metric_name, metric_mode = metric_name_mode(
+            metric_names=self.scheduler.metric_names(),
+            metric_mode=self.scheduler.metric_mode(),
+            metric=metric,
+        )
+        trial_id, best_metric = print_best_metric_found(
+            self.tuning_status, metric_names=[metric_name], mode=metric_mode
+        )
+        config = self.trial_backend._trial_dict[trial_id].config
+
+        logger.info(
+            f"If you want to retrain the best configuration found, you can run: \n"
+            f"```tuner.trial_backend.start_trial(config={config})``` to start training from scratch\n"
+            f"or\n"
+            f"```tuner.trial_backend.start_trial(config={config}, checkpoint_trial_id={trial_id})``` to start from "
+            f"last checkpoint (your script should have stored a checkpoint)"
+        )
+        return trial_id, config

syne_tune/util.py

@@ -21,6 +21,8 @@
 from typing import Optional, List, Union, Dict, Any, Iterable
 from time import perf_counter
 from contextlib import contextmanager
+from typing import Tuple, Union, List
+import logging
 
 import numpy as np
 
@@ -31,6 +33,8 @@
 )
 from syne_tune.try_import import try_import_aws_message
 
+logger = logging.getLogger(__name__)
+
 try:
     import sagemaker
 except ImportError:
@@ -319,3 +323,43 @@ def find_first_of_type(a: Iterable[Any], typ) -> Optional[Any]:
         return next(x for x in a if isinstance(x, typ))
     except StopIteration:
         return None
+
+
+def metric_name_mode(
+    metric_names: List[str], metric_mode: Union[str, List[str]], metric: Union[str, int]
+) -> Tuple[str, str]:
+    """
+    Retrieve the metric mode given a metric queried by either index or name.
+    :param metric_names: metrics names defined in a scheduler
+    :param metric_mode: metric mode or modes of a scheduler
+    :param metric: Index or name of the selected metric
+    :return the name and the mode of the queried metric
+    """
+    if isinstance(metric, str):
+        assert (
+            metric in metric_names
+        ), f"Attempted to use {metric} while available metrics are {metric_names}"
+        metric_name = metric
+    elif isinstance(metric, int):
+        assert metric < len(
+            metric_names
+        ), f"Attempted to use metric index={metric} with {len(metric_names)} available metrics"
+        metric_name = metric_names[metric]
+    else:
+        raise AttributeError(
+            f"metric must be <int> or <str> but {type(metric)} was provided"
+        )
+
+    if len(metric_names) > 1:
+        logger.warning(
+            "Several metrics exists, this will "
+            f"use metric={metric_name} (index={metric}) out of {metric_names}."
+        )
+
+    if isinstance(metric_mode, list):
+        metric_index = (
+            metric_names.index(metric_name) if isinstance(metric, str) else metric
+        )
+        metric_mode = metric_mode[metric_index]
+
+    return metric_name, metric_mode

tst/experiments/test_metric_name_mode.py

@@ -0,0 +1,26 @@
+import pytest
+
+from syne_tune.util import metric_name_mode
+
+metric_names = ["m1", "m2", "m3"]
+
+
+@pytest.mark.parametrize(
+    "metric_mode, query_metric, expected_metric, expected_mode,",
+    [
+        ("max", "m2", "m2", "max"),
+        ("min", "m2", "m2", "min"),
+        (["max", "min", "max"], "m2", "m2", "min"),
+        (["max", "min", "max"], "m3", "m3", "max"),
+        ("max", 1, "m2", "max"),
+        ("min", 1, "m2", "min"),
+        (["max", "min", "max"], 1, "m2", "min"),
+        (["max", "min", "max"], 2, "m3", "max"),
+    ],
+)
+def test_metric_name_mode(metric_mode, query_metric, expected_metric, expected_mode):
+    metric_name, metric_mode = metric_name_mode(
+        metric_names=metric_names, metric_mode=metric_mode, metric=query_metric
+    )
+    assert metric_name == expected_metric
+    assert metric_mode == expected_mode