Merge branch 'main' into checkpoint

examples/launch_plot_results.py

@@ -65,12 +65,16 @@
 
     tuning_experiment = load_experiment(tuner.name)
 
-    # Print the best configuration found from experiment-results
+    # prints the best configuration found from experiment-results
     print(f"best result found: {tuning_experiment.best_config()}")
 
-    # Plot the best value found over time
+    # plots the best metric over time
     tuning_experiment.plot()
 
-    # Print the best configuration found from the tuner and retrain it
+    # plots values found by all trials over time
+    tuning_experiment.plot_trials_over_time()
+
+    # prints the best configuration found from the tuner and retrains it
     trial_id, best_config = tuner.best_config()
     tuner.trial_backend.start_trial(config=best_config)
+

examples/launch_smac_example.py

@@ -83,7 +83,6 @@ def run_smac_simulated_backend():
         dataset=dataset_name,
     )
 
-    # Asynchronous successive halving (ASHA)
     blackbox = trial_backend.blackbox
 
     scheduler = SMAC(

setup.py

@@ -43,7 +43,9 @@ def load_benchmark_requirements():
 required_moo = load_requirements("requirements-moo.txt")
 required_visual = load_requirements("requirements-visual.txt")
 required_sklearn = load_requirements("requirements-sklearn.txt")
+required_smac = load_requirements("requirements-smac.txt")
 
+# Do not add SMAC for now in "extra" as it can easily conflict with Yahpo config-space version
 required_extra = (
     required_gpsearchers
     + required_kde
@@ -95,6 +97,7 @@ def load_benchmark_requirements():
         "bore": required_bore,
         "extra": required_extra,
         "basic": required_basic,
+        "smac": required_smac,
     },
     install_requires=required_core,
     include_package_data=True,

syne_tune/blackbox_repository/conversion_scripts/scripts/requirements-yahpo.txt

@@ -1,3 +1,5 @@
+# TODO uncomment once https://github.com/slds-lmu/yahpo_gym/pull/83 is released in Yahpo
+# onnxruntime>=1.10.0
 onnxruntime==1.10.0
 pyyaml
 configspace<=0.6.1

syne_tune/experiments/experiment_result.py

@@ -160,6 +160,45 @@ def plot(
             else:
                 fig.show()
 
+    def plot_trials_over_time(
+        self, metric_to_plot: Union[str, int] = 0, figure_path: str = None, figsize=None
+    ):
+        """Plot trials results over as function of wallclock time
+
+        :param metric_to_plot: Indicates which metric to plot, can be the index or a name of the metric.
+            default to 0 - first metric defined
+        :param figure_path: If specified, defines the path where the figure will be saved.
+            If None, the figure is shown
+        :param figsize: width and height of figure
+        """
+        _, metric_name, metric_mode = self._metric_name_mode(
+            metric_to_plot, verbose=True
+        )
+        df = self.results
+
+        fig, ax = plt.subplots(1, 1, figsize=figsize if figsize else (12, 4))
+        for trial_id in sorted(df.trial_id.unique()):
+            df_trial = df[df.trial_id == trial_id]
+            df_trial.plot(
+                x=ST_TUNER_TIME,
+                y=metric_name,
+                marker=".",
+                ax=ax,
+                legend=None,
+                alpha=0.5,
+            )
+        df_stop = df[df["st_decision"] == "STOP"]
+        plt.scatter(
+            df_stop[ST_TUNER_TIME], df_stop[metric_name], marker="x", color="red"
+        )
+        plt.xlabel("Wallclock time (s)")
+        plt.ylabel(metric_name)
+        plt.title("Trials value over time")
+        if figure_path is not None:
+            fig.savefig(figure_path)
+        else:
+            fig.show()
+
     def metric_mode(self) -> Union[str, List[str]]:
         return self.metadata["metric_mode"]