From 62aadcedf84d00acb67dd5d27e578aa82dd113e6 Mon Sep 17 00:00:00 2001 From: Ema Mekic Date: Mon, 9 Jun 2025 13:10:26 +0200 Subject: [PATCH 01/27] Rename overwrite_working_directory => overwrite_root_directory --- docs/reference/declarative_usage.md | 2 +- docs/reference/neps_run.md | 8 ++++---- neps/api.py | 6 +++--- neps/utils/cli.py | 4 ++-- neps_examples/convenience/declarative_usage/config.yaml | 2 +- neps_examples/experimental/freeze_thaw.py | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/reference/declarative_usage.md b/docs/reference/declarative_usage.md index 57b53c476..6d1c1c3df 100644 --- a/docs/reference/declarative_usage.md +++ b/docs/reference/declarative_usage.md @@ -111,7 +111,7 @@ but also advanced settings for more complex setups. max_cost_total: # Debug and Monitoring - overwrite_working_directory: true + overwrite_root_directory: true post_run_summary: false # Parallelization Setup diff --git a/docs/reference/neps_run.md b/docs/reference/neps_run.md index 2885770ed..07785890d 100644 --- a/docs/reference/neps_run.md +++ b/docs/reference/neps_run.md @@ -108,13 +108,13 @@ If the run previously stopped due to reaching a budget and you specify the same ## Overwriting a Run -To overwrite a run, simply provide the same `root_directory=` to [`neps.run()`][neps.api.run] as before, with the `overwrite_working_directory=True` argument. +To overwrite a run, simply provide the same `root_directory=` to [`neps.run()`][neps.api.run] as before, with the `overwrite_root_directory=True` argument. ```python neps.run( ..., root_directory="path/to/previous_result_dir", - overwrite_working_directory=True, + overwrite_root_directory=True, ) ``` @@ -176,7 +176,7 @@ Any new workers that come online will automatically pick up work and work togeth max_evaluations_total=100, max_evaluations_per_run=10, # (1)! continue_until_max_evaluation_completed=True, # (2)! - overwrite_working_directory=False, #!!! + overwrite_root_directory=False, #!!! ) ``` @@ -186,7 +186,7 @@ Any new workers that come online will automatically pick up work and work togeth !!! warning - Ensure `overwrite_working_directory=False` to prevent newly spawned workers from deleting the shared directory! + Ensure `overwrite_root_directory=False` to prevent newly spawned workers from deleting the shared directory! === "Shell" diff --git a/neps/api.py b/neps/api.py index 77c8ebcf2..ca1bdc5b8 100644 --- a/neps/api.py +++ b/neps/api.py @@ -33,7 +33,7 @@ def run( # noqa: PLR0913 ), *, root_directory: str | Path = "neps_results", - overwrite_working_directory: bool = False, + overwrite_root_directory: bool = False, post_run_summary: bool = True, max_evaluations_total: int | None = None, max_evaluations_per_run: int | None = None, @@ -189,7 +189,7 @@ def evaluate_pipeline(some_parameter: float) -> float: root_directory: The directory to save progress to. - overwrite_working_directory: If true, delete the working directory at the start of + overwrite_root_directory: If true, delete the working directory at the start of the run. This is, e.g., useful when debugging a evaluate_pipeline function. post_run_summary: If True, creates a csv file after each worker is done, @@ -443,7 +443,7 @@ def __call__( objective_value_on_error=objective_value_on_error, cost_value_on_error=cost_value_on_error, ignore_errors=ignore_errors, - overwrite_optimization_dir=overwrite_working_directory, + overwrite_optimization_dir=overwrite_root_directory, sample_batch_size=sample_batch_size, ) diff --git a/neps/utils/cli.py b/neps/utils/cli.py index cd58b461b..8edf8b38c 100644 --- a/neps/utils/cli.py +++ b/neps/utils/cli.py @@ -166,7 +166,7 @@ def init_config(args: argparse.Namespace) -> None: root_directory: "set/path/for/root_directory" max_evaluations_total: -overwrite_working_directory: +overwrite_root_directory: """ ) elif template == "complete": @@ -195,7 +195,7 @@ def init_config(args: argparse.Namespace) -> None: max_cost_total: # Debug and Monitoring -overwrite_working_directory: false +overwrite_root_directory: false post_run_summary: true # Parallelization Setup diff --git a/neps_examples/convenience/declarative_usage/config.yaml b/neps_examples/convenience/declarative_usage/config.yaml index 858eb6e52..7bbe27b8a 100644 --- a/neps_examples/convenience/declarative_usage/config.yaml +++ b/neps_examples/convenience/declarative_usage/config.yaml @@ -1,7 +1,7 @@ experiment: root_directory: "results/example_run" max_evaluations_total: 20 - overwrite_working_directory: true + overwrite_root_directory: true post_run_summary: true pipeline_space: diff --git a/neps_examples/experimental/freeze_thaw.py b/neps_examples/experimental/freeze_thaw.py index c75cdc346..66113f04b 100644 --- a/neps_examples/experimental/freeze_thaw.py +++ b/neps_examples/experimental/freeze_thaw.py @@ -168,7 +168,7 @@ def training_pipeline( optimizer="ifbo", max_evaluations_total=50, root_directory="./debug/ifbo-mnist/", - overwrite_working_directory=False, # set to False for a multi-worker run + overwrite_root_directory=False, # set to False for a multi-worker run ) # NOTE: this is `experimental` and may not work as expected From 876e5f69172e226aa4f4776bdc7f290b6fce506c Mon Sep 17 00:00:00 2001 From: Ema Mekic Date: Wed, 11 Jun 2025 16:01:33 +0200 Subject: [PATCH 02/27] neps.plot fixed --- neps/plot/plotting.py | 4 +- neps/plot/read_results.py | 77 ++++++++++++++++++--------------------- 2 files changed, 38 insertions(+), 43 deletions(-) diff --git a/neps/plot/plotting.py b/neps/plot/plotting.py index 612fad9d8..f87461b56 100644 --- a/neps/plot/plotting.py +++ b/neps/plot/plotting.py @@ -148,7 +148,7 @@ def _interpolate_time( _df.index = [max_b] df = pd.concat((df, _df)).sort_index() - df = df.fillna(method="backfill", axis=0).fillna(method="ffill", axis=0) + df = df.bfill(axis=0).ffill(axis=0) if x_range is not None: df = df.query(f"{x_range[0]} <= index <= {x_range[1]}") @@ -161,7 +161,7 @@ def _df_to_x_range(df: pd.DataFrame, x_range: tuple | None = None) -> pd.DataFra _df = pd.DataFrame.from_dict(new_entry, orient="index").T _df.index = [x_max] df = pd.concat((df, _df)).sort_index() - return df.fillna(method="backfill", axis=0).fillna(method="ffill", axis=0) + return df.bfill(axis=0).ffill(axis=0) def _set_legend( diff --git a/neps/plot/read_results.py b/neps/plot/read_results.py index 4e01f8b71..42107842f 100644 --- a/neps/plot/read_results.py +++ b/neps/plot/read_results.py @@ -3,9 +3,10 @@ from __future__ import annotations from pathlib import Path +from neps.state.trial import State import neps - +import numpy as np def process_seed( *, @@ -21,51 +22,45 @@ def process_seed( path = path / str(seed) / "neps_root_directory" _fulldf, _summary = neps.status(path, print_summary=False) - raise NotImplementedError( - "I'm sorry, I broke this. We now dump all the information neps has available" - " into the above dataframe `fulldf`." - ) - # > sorted_stats = sorted(sorted(stats.items()), key=lambda x: len(x[0])) - # > stats = OrderedDict(sorted_stats) + if _fulldf.empty: + raise ValueError(f"No trials found in {path}") + + _fulldf = _fulldf.sort_values("time_sampled") - # > # max_cost only relevant for scaling x-axis when using fidelity on the x-axis - # > max_cost: float = -1.0 - # > if key_to_extract == "fidelity": - # > # TODO(eddiebergman): This can crash for a number of reasons, namely if the - # > # config crased and it's result is an error, or if the `"info_dict"` and/or - # > # `key_to_extract` doesn't exist - # > max_cost = max(s.result["info_dict"][key_to_extract] for s in stats.values()) + def get_cost(idx: str | int) -> float: + row = _fulldf.loc[idx] + if key_to_extract and key_to_extract in row: + return float(row[key_to_extract]) + return 1.0 - # > global_start = stats[min(stats.keys())].metadata["time_sampled"] + losses = [] + costs = [] - # > def get_cost(idx: str) -> float: - # > if key_to_extract is not None: - # > # TODO(eddiebergman): This can crash for a number of reasons, namely if - # > # the config crased and it's result is an error, or if the `"info_dict"` - # > # and/or `key_to_extract` doesn't exist - # > return float(stats[idx].result["info_dict"][key_to_extract]) + # max_cost only relevant for scaling x-axis when using fidelity on the x-axis + max_cost: float = -1.0 + global_start = _fulldf["time_sampled"].min() - # > return 1.0 + for config_id, config_result in _fulldf.iterrows(): + if config_result["state"] != State.SUCCESS: + continue - # > losses = [] - # > costs = [] + cost = get_cost(config_id) - # > for config_id, config_result in stats.items(): - # > config_cost = get_cost(config_id) - # > if consider_continuations: - # > if n_workers == 1: - # > # calculates continuation costs for MF algorithms NOTE: assumes that - # > # all recorded evaluations are black-box evaluations where - # > # continuations or freeze-thaw was not accounted for during opt - # > if "previous_config_id" in config_result.metadata: - # > previous_config_id = config_result.metadata["previous_config_id"] - # > config_cost -= get_cost(previous_config_id) - # > else: - # > config_cost = config_result.metadata["time_end"] - global_start + if consider_continuations: + if n_workers == 1 and "previous_config_id" in config_result["metadata"]: + # calculates continuation costs for MF algorithms NOTE: assumes that + # all recorded evaluations are black-box evaluations where + # continuations or freeze-thaw was not accounted for during optimization + previous_id = config_result["metadata"]["previous_config_id"] + + if previous_id in _fulldf.index and key_to_extract: + cost -= get_cost(config_id) + else: + cost = float(config_result["time_end"] - global_start) - # > # TODO(eddiebergman): Assumes it never crashed and there's a - # > # objective_to_minimize available,not fixing now but it should be addressed - # > losses.append(config_result.result["objective_to_minimize"]) # type: ignore - # > costs.append(config_cost) + loss = float(config_result["objective_to_minimize"]) + losses.append(loss) + costs.append(cost) + max_cost = max(max_cost, cost) - # > return list(np.minimum.accumulate(losses)), costs, max_cost + return list(np.minimum.accumulate(losses)), costs, max_cost \ No newline at end of file From 1c7e99c42223a3c0300437f223f531635a6e5296 Mon Sep 17 00:00:00 2001 From: Ema Mekic Date: Tue, 24 Jun 2025 11:49:09 +0200 Subject: [PATCH 03/27] Added documentation and logging of incumbent --- .trace.lock | 0 docs/reference/analyse.md | 8 +++- neps/runtime.py | 78 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 78 insertions(+), 8 deletions(-) create mode 100644 .trace.lock diff --git a/.trace.lock b/.trace.lock new file mode 100644 index 000000000..e69de29bb diff --git a/docs/reference/analyse.md b/docs/reference/analyse.md index 3fd8f694b..92c42341c 100644 --- a/docs/reference/analyse.md +++ b/docs/reference/analyse.md @@ -55,6 +55,8 @@ NePS will also generate a summary CSV file for you. ├── summary │ ├── full.csv │ └── short.csv + ├── best_objective_trajectory.txt + ├── best_objective.txt ├── optimizer_info.yaml └── optimizer_state.pkl ``` @@ -69,6 +71,8 @@ NePS will also generate a summary CSV file for you. │ ├── config.yaml │ ├── metadata.yaml │ └── report.yaml + ├── best_objective_trajectory.txt + ├── best_objective.txt ├── optimizer_info.yaml └── optimizer_state.pkl ``` @@ -77,8 +81,8 @@ NePS will also generate a summary CSV file for you. The `full.csv` contains all configuration details in CSV format. Details include configuration hyperparameters and any returned result and cost from the `evaluate_pipeline` function. -The `run_status.csv` provides general run details, such as the number of failed and successful configurations, -and the best configuration with its corresponding objective value. +The `best_objective_trajectory.txt` contains logging of the incumbent trajectory. +The `best_objective.txt` records current incumbent. # TensorBoard Integration diff --git a/neps/runtime.py b/neps/runtime.py index e3958322b..b23c21297 100644 --- a/neps/runtime.py +++ b/neps/runtime.py @@ -7,6 +7,7 @@ import os import shutil import time +import math from collections.abc import Callable, Iterator, Mapping from contextlib import contextmanager from dataclasses import dataclass @@ -14,6 +15,8 @@ from typing import TYPE_CHECKING, ClassVar, Literal from portalocker import portalocker +from pathlib import Path +from filelock import FileLock from neps.env import ( FS_SYNC_GRACE_BASE, @@ -321,12 +324,22 @@ def _check_global_stopping_criterion( ) -> str | Literal[False]: if self.settings.max_evaluations_total is not None: if self.settings.include_in_progress_evaluations_towards_maximum: - count = sum( - 1 - for _, trial in trials.items() - if trial.metadata.state - not in (Trial.State.PENDING, Trial.State.SUBMITTED) - ) + if self.optimizer.space.fidelities: + count = sum( + trial.report.cost + for _, trial in trials.items() + if trial.report is not None and trial.report.cost is not None + ) + for name, fidelity_param in self.optimizer.space.fidelities.items(): + count = math.ceil(count / fidelity_param.upper) + else: + count = sum( + 1 + for _, trial in trials.items() + if trial.metadata.state + not in (Trial.State.PENDING, Trial.State.SUBMITTED) + ) + else: # This indicates they have completed. count = sum(1 for _, trial in trials.items() if trial.report is not None) @@ -340,11 +353,14 @@ def _check_global_stopping_criterion( ) if self.settings.max_cost_total is not None: + # for _, trial in trials.items(): + # print("TRIAL: ", trial) cost = sum( trial.report.cost for _, trial in trials.items() if trial.report is not None and trial.report.cost is not None ) + # print("COST ", cost) if cost >= self.settings.max_cost_total: return ( f"The maximum cost `{self.settings.max_cost_total=}` has been" @@ -493,8 +509,24 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 """ _set_workers_neps_state(self.state) + main_dir = Path(self.state.path) + improvement_trace_path = main_dir / "best_objective_trajectory.txt" + _trace_lock = FileLock(".trace.lock") + best_config_path = main_dir / "best_objective.txt" + + if not improvement_trace_path.exists(): + with _trace_lock: + with open(improvement_trace_path, mode='w') as f: + f.write("Best configs and their objectives across evaluations:\n") + f.write("-" * 80 + "\n") + + _best_score_so_far = float("inf") + logger.info("Launching NePS") + optimizer_name = self.state._optimizer_info["name"] + logger.info("Using optimizer: %s", optimizer_name) + _time_monotonic_start = time.monotonic() _error_from_evaluation: Exception | None = None @@ -599,6 +631,40 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 evaluated_trial.metadata.state, ) + if report.objective_to_minimize is not None and report.err is None: + new_score = report.objective_to_minimize + if new_score < _best_score_so_far: + _best_score_so_far = new_score + logger.info( + "Evaluated trial: %s with objective %s is the new best trial.", + evaluated_trial.id, + new_score, + ) + logger.info("Trajectory of best objectives can be found in %s", improvement_trace_path) + logger.info("New best incumbent can be found in %s", best_config_path) + config_text = ( + f"Objective to minimize: {new_score}\n" + f"Config ID: {evaluated_trial.id}\n" + f"Config: {evaluated_trial.config}\n" + + "-" * 80 + "\n" + ) + + # Log improvement to trace + with _trace_lock: + with open(improvement_trace_path, mode='a') as f: + f.write(config_text) + + # Write best config to file (overwrite mode) + best_summary = ( + f"# Best incumbent:" + "\n" + f"\n Config ID: {evaluated_trial.id}" + f"\n Objective to minimize: {new_score}" + f"\n Config: {evaluated_trial.config}" + ) + with open(best_config_path, "w") as f: + f.write(best_summary) + if report.cost is not None: self.worker_cumulative_eval_cost += report.cost From b17306e820a341cfca9153773e012f442e981ce6 Mon Sep 17 00:00:00 2001 From: Ema Mekic Date: Tue, 24 Jun 2025 11:51:48 +0200 Subject: [PATCH 04/27] Removed comment --- neps/runtime.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/neps/runtime.py b/neps/runtime.py index b23c21297..3a1130a74 100644 --- a/neps/runtime.py +++ b/neps/runtime.py @@ -353,14 +353,11 @@ def _check_global_stopping_criterion( ) if self.settings.max_cost_total is not None: - # for _, trial in trials.items(): - # print("TRIAL: ", trial) cost = sum( trial.report.cost for _, trial in trials.items() if trial.report is not None and trial.report.cost is not None ) - # print("COST ", cost) if cost >= self.settings.max_cost_total: return ( f"The maximum cost `{self.settings.max_cost_total=}` has been" From 222955f330f827dc6bd613c6c23a443dccfa4f73 Mon Sep 17 00:00:00 2001 From: Ema Mekic Date: Tue, 24 Jun 2025 12:24:22 +0200 Subject: [PATCH 05/27] Renaming max_evalutaions_total to evaluations_to_spend --- README.md | 2 +- docs/index.md | 2 +- docs/reference/neps_run.md | 16 ++++++++-------- docs/reference/optimizers.md | 8 ++++---- neps/api.py | 16 ++++++++-------- neps/runtime.py | 14 +++++++------- neps/state/settings.py | 4 ++-- neps_examples/basic_usage/hyperparameters.py | 2 +- .../convenience/logging_additional_info.py | 2 +- .../convenience/neps_tblogger_tutorial.py | 2 +- .../convenience/running_on_slurm_scripts.py | 2 +- .../working_directory_per_pipeline.py | 2 +- .../expert_priors_for_hyperparameters.py | 2 +- neps_examples/efficiency/multi_fidelity.py | 2 +- .../multi_fidelity_and_expert_priors.py | 2 +- .../efficiency/pytorch_lightning_ddp.py | 2 +- .../efficiency/pytorch_lightning_fsdp.py | 2 +- neps_examples/efficiency/pytorch_native_ddp.py | 2 +- neps_examples/efficiency/pytorch_native_fsdp.py | 2 +- neps_examples/experimental/freeze_thaw.py | 2 +- .../real_world/image_segmentation_hpo.py | 2 +- tests/test_runtime/test_default_report_values.py | 6 +++--- .../test_error_handling_strategies.py | 6 +++--- tests/test_runtime/test_stopping_criterion.py | 16 ++++++++-------- 24 files changed, 59 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 4e549326f..5356de78f 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ neps.run( evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="path/to/save/results", # Replace with the actual path. - max_evaluations_total=100, + evaluations_to_spend=100, ) ``` diff --git a/docs/index.md b/docs/index.md index d6e618e19..e9721ecd7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -87,7 +87,7 @@ neps.run( evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="path/to/save/results", # Replace with the actual path. - max_evaluations_total=100, + evaluations_to_spend=100, ) ``` diff --git a/docs/reference/neps_run.md b/docs/reference/neps_run.md index 5e023253f..de008039d 100644 --- a/docs/reference/neps_run.md +++ b/docs/reference/neps_run.md @@ -45,7 +45,7 @@ See the following for more: * What goes in and what goes out of [`evaluate_pipeline()`](../reference/evaluate_pipeline.md)? ## Budget, how long to run? -To define a budget, provide `max_evaluations_total=` to [`neps.run()`][neps.api.run], +To define a budget, provide `evaluations_to_spend=` to [`neps.run()`][neps.api.run], to specify the total number of evaluations to conduct before halting the optimization process, or `max_cost_total=` to specify a cost threshold for your own custom cost metric, such as time, energy, or monetary, as returned by each evaluation of the pipeline . @@ -60,7 +60,7 @@ def evaluate_pipeline(learning_rate: float, epochs: int) -> float: return {"objective_function_to_minimize": loss, "cost": duration} neps.run( - max_evaluations_total=10, # (1)! + evaluations_to_spend=10, # (1)! max_cost_total=1000, # (2)! ) ``` @@ -87,7 +87,7 @@ Please refer to Python's [logging documentation](https://docs.python.org/3/libra ## Continuing Runs To continue a run, all you need to do is provide the same `root_directory=` to [`neps.run()`][neps.api.run] as before, -with an increased `max_evaluations_total=` or `max_cost_total=`. +with an increased `evaluations_to_spend=` or `max_cost_total=`. ```python def run(learning_rate: float, epochs: int) -> float: @@ -100,7 +100,7 @@ def run(learning_rate: float, epochs: int) -> float: neps.run( # Increase the total number of trials from 10 as set previously to 50 - max_evaluations_total=50, + evaluations_to_spend=50, ) ``` @@ -174,7 +174,7 @@ Any new workers that come online will automatically pick up work and work togeth evaluate_pipeline=..., pipeline_space=..., root_directory="some/path", - max_evaluations_total=100, + evaluations_to_spend=100, max_evaluations_per_run=10, # (1)! continue_until_max_evaluation_completed=True, # (2)! overwrite_working_directory=False, #!!! @@ -182,8 +182,8 @@ Any new workers that come online will automatically pick up work and work togeth ``` 1. Limits the number of evaluations for this specific call of [`neps.run()`][neps.api.run]. - 2. Evaluations in-progress count towards max_evaluations_total, halting new ones when this limit is reached. - Setting this to `True` enables continuous sampling of new evaluations until the total of completed ones meets max_evaluations_total, optimizing resource use in time-sensitive scenarios. + 2. Evaluations in-progress count towards evaluations_to_spend, halting new ones when this limit is reached. + Setting this to `True` enables continuous sampling of new evaluations until the total of completed ones meets evaluations_to_spend, optimizing resource use in time-sensitive scenarios. !!! warning @@ -227,7 +227,7 @@ neps.run( !!! note - Any runs that error will still count towards the total `max_evaluations_total` or `max_evaluations_per_run`. + Any runs that error will still count towards the total `evaluations_to_spend` or `max_evaluations_per_run`. ### Re-running Failed Configurations diff --git a/docs/reference/optimizers.md b/docs/reference/optimizers.md index 3c6502624..6edef593a 100644 --- a/docs/reference/optimizers.md +++ b/docs/reference/optimizers.md @@ -72,7 +72,7 @@ neps.run( evaluate_pipeline=run_function, pipeline_space=pipeline_space, root_directory="results/", - max_evaluations_total=25, + evaluations_to_spend=25, # no optimizer specified ) ``` @@ -87,7 +87,7 @@ neps.run( evaluate_pipeline=run_function, pipeline_space=pipeline_space, root_directory="results/", - max_evaluations_total=25, + evaluations_to_spend=25, # optimizer specified, along with an argument optimizer=neps.algorithms.bayesian_optimization, # or as string: "bayesian_optimization" ) @@ -104,7 +104,7 @@ neps.run( evaluate_pipeline=run_function, pipeline_space=pipeline_space, root_directory="results/", - max_evaluations_total=25, + evaluations_to_spend=25, optimizer=("bayesian_optimization", {"initial_design_size": 5}) ) ``` @@ -137,7 +137,7 @@ neps.run( evaluate_pipeline=run_function, pipeline_space=pipeline_space, root_directory="results/", - max_evaluations_total=25, + evaluations_to_spend=25, optimizer=MyOptimizer, ) ``` diff --git a/neps/api.py b/neps/api.py index 77c8ebcf2..79aeb495c 100644 --- a/neps/api.py +++ b/neps/api.py @@ -35,7 +35,7 @@ def run( # noqa: PLR0913 root_directory: str | Path = "neps_results", overwrite_working_directory: bool = False, post_run_summary: bool = True, - max_evaluations_total: int | None = None, + evaluations_to_spend: int | None = None, max_evaluations_per_run: int | None = None, continue_until_max_evaluation_completed: bool = False, max_cost_total: int | float | None = None, @@ -96,7 +96,7 @@ def evaluate_pipeline(some_parameter: float) -> float: ) }, root_directory="usage_example", - max_evaluations_total=5, + evaluations_to_spend=5, ) ``` @@ -197,18 +197,18 @@ def evaluate_pipeline(some_parameter: float) -> float: max_evaluations_per_run: Number of evaluations this specific call should do. - max_evaluations_total: Number of evaluations after which to terminate. + evaluations_to_spend: Number of evaluations after which to terminate. This is shared between all workers operating in the same `root_directory`. continue_until_max_evaluation_completed: - If true, only stop after max_evaluations_total have been completed. + If true, only stop after evaluations_to_spend have been completed. This is only relevant in the parallel setting. max_cost_total: No new evaluations will start when this cost is exceeded. Requires returning a cost in the evaluate_pipeline function, e.g., `return dict(loss=loss, cost=cost)`. ignore_errors: Ignore hyperparameter settings that threw an error and do not raise - an error. Error configs still count towards max_evaluations_total. + an error. Error configs still count towards evaluations_to_spend. objective_value_on_error: Setting this and cost_value_on_error to any float will supress any error and will use given objective_to_minimize value instead. default: None cost_value_on_error: Setting this and objective_value_on_error to any float will @@ -396,14 +396,14 @@ def __call__( """ # noqa: E501 if ( - max_evaluations_total is None + evaluations_to_spend is None and max_evaluations_per_run is None and max_cost_total is None ): warnings.warn( "None of the following were set, this will run idefinitely until the worker" " process is stopped." - f"\n * {max_evaluations_total=}" + f"\n * {evaluations_to_spend=}" f"\n * {max_evaluations_per_run=}" f"\n * {max_cost_total=}", UserWarning, @@ -437,7 +437,7 @@ def __call__( optimizer_info=_optimizer_info, max_cost_total=max_cost_total, optimization_dir=Path(root_directory), - max_evaluations_total=max_evaluations_total, + evaluations_to_spend=evaluations_to_spend, max_evaluations_for_worker=max_evaluations_per_run, continue_until_max_evaluation_completed=continue_until_max_evaluation_completed, objective_value_on_error=objective_value_on_error, diff --git a/neps/runtime.py b/neps/runtime.py index e3958322b..97fc40f25 100644 --- a/neps/runtime.py +++ b/neps/runtime.py @@ -319,7 +319,7 @@ def _check_global_stopping_criterion( self, trials: Mapping[str, Trial], ) -> str | Literal[False]: - if self.settings.max_evaluations_total is not None: + if self.settings.evaluations_to_spend is not None: if self.settings.include_in_progress_evaluations_towards_maximum: count = sum( 1 @@ -331,10 +331,10 @@ def _check_global_stopping_criterion( # This indicates they have completed. count = sum(1 for _, trial in trials.items() if trial.report is not None) - if count >= self.settings.max_evaluations_total: + if count >= self.settings.evaluations_to_spend: return ( "The total number of evaluations has reached the maximum allowed of" - f" `{self.settings.max_evaluations_total=}`." + f" `{self.settings.evaluations_to_spend=}`." " To allow more evaluations, increase this value or use a different" " stopping criterion." ) @@ -372,7 +372,7 @@ def _check_global_stopping_criterion( @property def _requires_global_stopping_criterion(self) -> bool: return ( - self.settings.max_evaluations_total is not None + self.settings.evaluations_to_spend is not None or self.settings.max_cost_total is not None or self.settings.max_evaluation_time_total_seconds is not None ) @@ -697,7 +697,7 @@ def _launch_runtime( # noqa: PLR0913 cost_value_on_error: float | None, continue_until_max_evaluation_completed: bool, overwrite_optimization_dir: bool, - max_evaluations_total: int | None, + evaluations_to_spend: int | None, max_evaluations_for_worker: int | None, sample_batch_size: int | None, ) -> None: @@ -738,7 +738,7 @@ def _launch_runtime( # noqa: PLR0913 BudgetInfo( max_cost_total=max_cost_total, used_cost_budget=0, - max_evaluations=max_evaluations_total, + max_evaluations=evaluations_to_spend, used_evaluations=0, ) ), @@ -767,7 +767,7 @@ def _launch_runtime( # noqa: PLR0913 ), batch_size=sample_batch_size, default_report_values=default_report_values, - max_evaluations_total=max_evaluations_total, + evaluations_to_spend=evaluations_to_spend, include_in_progress_evaluations_towards_maximum=( not continue_until_max_evaluation_completed ), diff --git a/neps/state/settings.py b/neps/state/settings.py index bbc7c0498..49a39ff65 100644 --- a/neps/state/settings.py +++ b/neps/state/settings.py @@ -78,7 +78,7 @@ class WorkerSettings: """The number of configurations to sample in a single batch.""" # --------- Global Stopping Criterion --------- - max_evaluations_total: int | None + evaluations_to_spend: int | None """The maximum number of evaluations to run in total. Once this evaluation total is reached, **all** workers will stop evaluating @@ -95,7 +95,7 @@ class WorkerSettings: include_in_progress_evaluations_towards_maximum: bool """Whether to include currently evaluating configurations towards the stopping criterion - [`max_evaluations_total`][neps.state.settings.WorkerSettings.max_evaluations_total] + [`evaluations_to_spend`][neps.state.settings.WorkerSettings.evaluations_to_spend] """ max_cost_total: float | None diff --git a/neps_examples/basic_usage/hyperparameters.py b/neps_examples/basic_usage/hyperparameters.py index 6b736fcd3..023de5d4f 100644 --- a/neps_examples/basic_usage/hyperparameters.py +++ b/neps_examples/basic_usage/hyperparameters.py @@ -28,5 +28,5 @@ def evaluate_pipeline(float1, float2, categorical, integer1, integer2): pipeline_space=pipeline_space, root_directory="results/hyperparameters_example", post_run_summary=True, - max_evaluations_total=30, + evaluations_to_spend=30, ) diff --git a/neps_examples/convenience/logging_additional_info.py b/neps_examples/convenience/logging_additional_info.py index 6756e03c7..3120c7db5 100644 --- a/neps_examples/convenience/logging_additional_info.py +++ b/neps_examples/convenience/logging_additional_info.py @@ -34,5 +34,5 @@ def evaluate_pipeline(float1, float2, categorical, integer1, integer2): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/logging_additional_info", - max_evaluations_total=5, + evaluations_to_spend=5, ) diff --git a/neps_examples/convenience/neps_tblogger_tutorial.py b/neps_examples/convenience/neps_tblogger_tutorial.py index fd9bc8144..938ca0e2c 100644 --- a/neps_examples/convenience/neps_tblogger_tutorial.py +++ b/neps_examples/convenience/neps_tblogger_tutorial.py @@ -342,7 +342,7 @@ def evaluate_pipeline(lr, optim, weight_decay): neps.run( **run_args, - max_evaluations_total=3, + evaluations_to_spend=3, ) """ diff --git a/neps_examples/convenience/running_on_slurm_scripts.py b/neps_examples/convenience/running_on_slurm_scripts.py index 86fe41ac2..26dac7082 100644 --- a/neps_examples/convenience/running_on_slurm_scripts.py +++ b/neps_examples/convenience/running_on_slurm_scripts.py @@ -60,5 +60,5 @@ def evaluate_pipeline_via_slurm( evaluate_pipeline=evaluate_pipeline_via_slurm, pipeline_space=pipeline_space, root_directory="results/slurm_script_example", - max_evaluations_total=5, + evaluations_to_spend=5, ) diff --git a/neps_examples/convenience/working_directory_per_pipeline.py b/neps_examples/convenience/working_directory_per_pipeline.py index de2ec9fd9..7b1b5ad13 100644 --- a/neps_examples/convenience/working_directory_per_pipeline.py +++ b/neps_examples/convenience/working_directory_per_pipeline.py @@ -29,5 +29,5 @@ def evaluate_pipeline(pipeline_directory: Path, float1, categorical, integer1): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/working_directory_per_pipeline", - max_evaluations_total=5, + evaluations_to_spend=5, ) diff --git a/neps_examples/efficiency/expert_priors_for_hyperparameters.py b/neps_examples/efficiency/expert_priors_for_hyperparameters.py index 5980668a5..32633930b 100644 --- a/neps_examples/efficiency/expert_priors_for_hyperparameters.py +++ b/neps_examples/efficiency/expert_priors_for_hyperparameters.py @@ -48,5 +48,5 @@ def evaluate_pipeline(some_float, some_integer, some_cat): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/user_priors_example", - max_evaluations_total=15, + evaluations_to_spend=15, ) diff --git a/neps_examples/efficiency/multi_fidelity.py b/neps_examples/efficiency/multi_fidelity.py index c85d411e1..394340c15 100644 --- a/neps_examples/efficiency/multi_fidelity.py +++ b/neps_examples/efficiency/multi_fidelity.py @@ -94,5 +94,5 @@ def evaluate_pipeline( root_directory="results/multi_fidelity_example", # Optional: Do not start another evaluation after <=50 epochs, corresponds to cost # field above. - max_cost_total=50, + max_cost_total=10, ) diff --git a/neps_examples/efficiency/multi_fidelity_and_expert_priors.py b/neps_examples/efficiency/multi_fidelity_and_expert_priors.py index 96f7b2b3e..074b8aaa2 100644 --- a/neps_examples/efficiency/multi_fidelity_and_expert_priors.py +++ b/neps_examples/efficiency/multi_fidelity_and_expert_priors.py @@ -43,5 +43,5 @@ def evaluate_pipeline(float1, float2, integer1, fidelity): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/multifidelity_priors", - max_evaluations_total=25, # For an alternate stopping method see multi_fidelity.py + evaluations_to_spend=25, # For an alternate stopping method see multi_fidelity.py ) diff --git a/neps_examples/efficiency/pytorch_lightning_ddp.py b/neps_examples/efficiency/pytorch_lightning_ddp.py index 96b620de1..21a4fba01 100644 --- a/neps_examples/efficiency/pytorch_lightning_ddp.py +++ b/neps_examples/efficiency/pytorch_lightning_ddp.py @@ -100,5 +100,5 @@ def evaluate_pipeline(lr=0.1, epoch=20): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/pytorch_lightning_ddp", - max_evaluations_total=5 + evaluations_to_spend=5 ) diff --git a/neps_examples/efficiency/pytorch_lightning_fsdp.py b/neps_examples/efficiency/pytorch_lightning_fsdp.py index 6af3d6746..aa568c4d0 100644 --- a/neps_examples/efficiency/pytorch_lightning_fsdp.py +++ b/neps_examples/efficiency/pytorch_lightning_fsdp.py @@ -74,5 +74,5 @@ def evaluate_pipeline(lr=0.1, epoch=20): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/pytorch_lightning_fsdp", - max_evaluations_total=5 + evaluations_to_spend=5 ) diff --git a/neps_examples/efficiency/pytorch_native_ddp.py b/neps_examples/efficiency/pytorch_native_ddp.py index 9ced5dc86..9477d6e95 100644 --- a/neps_examples/efficiency/pytorch_native_ddp.py +++ b/neps_examples/efficiency/pytorch_native_ddp.py @@ -112,4 +112,4 @@ def evaluate_pipeline(learning_rate, epochs): neps.run(evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/pytorch_ddp", - max_evaluations_total=25) + evaluations_to_spend=25) diff --git a/neps_examples/efficiency/pytorch_native_fsdp.py b/neps_examples/efficiency/pytorch_native_fsdp.py index 1fec7bef3..b549fd0c7 100644 --- a/neps_examples/efficiency/pytorch_native_fsdp.py +++ b/neps_examples/efficiency/pytorch_native_fsdp.py @@ -212,5 +212,5 @@ def evaluate_pipeline(lr=0.1, epoch=20): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/pytorch_fsdp", - max_evaluations_total=20 + evaluations_to_spend=20 ) diff --git a/neps_examples/experimental/freeze_thaw.py b/neps_examples/experimental/freeze_thaw.py index 597e1df3d..e9134e93c 100644 --- a/neps_examples/experimental/freeze_thaw.py +++ b/neps_examples/experimental/freeze_thaw.py @@ -166,7 +166,7 @@ def training_pipeline( pipeline_space=pipeline_space, evaluate_pipeline=training_pipeline, optimizer="ifbo", - max_evaluations_total=50, + evaluations_to_spend=50, root_directory="./results/ifbo-mnist/", overwrite_working_directory=False, # set to False for a multi-worker run ) diff --git a/neps_examples/real_world/image_segmentation_hpo.py b/neps_examples/real_world/image_segmentation_hpo.py index 2320f20f1..3d7e481f1 100644 --- a/neps_examples/real_world/image_segmentation_hpo.py +++ b/neps_examples/real_world/image_segmentation_hpo.py @@ -125,5 +125,5 @@ def evaluate_pipeline(**kwargs): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/hpo_image_segmentation", - max_evaluations_total=500 + evaluations_to_spend=500 ) diff --git a/tests/test_runtime/test_default_report_values.py b/tests/test_runtime/test_default_report_values.py index 3b37d5254..091ac8c6e 100644 --- a/tests/test_runtime/test_default_report_values.py +++ b/tests/test_runtime/test_default_report_values.py @@ -41,7 +41,7 @@ def test_default_values_on_error( cost_value_on_error=2.4, # <- Highlight learning_curve_on_error=[2.4, 2.5], # <- Highlight ), - max_evaluations_total=None, + evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, max_cost_total=None, max_evaluations_for_worker=1, @@ -92,7 +92,7 @@ def test_default_values_on_not_specified( cost_if_not_provided=2.4, learning_curve_if_not_provided=[2.4, 2.5], ), - max_evaluations_total=None, + evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, max_cost_total=None, max_evaluations_for_worker=1, @@ -141,7 +141,7 @@ def test_default_value_objective_to_minimize_curve_take_objective_to_minimize_va default_report_values=DefaultReportValues( learning_curve_if_not_provided="objective_to_minimize" ), - max_evaluations_total=None, + evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, max_cost_total=None, max_evaluations_for_worker=1, diff --git a/tests/test_runtime/test_error_handling_strategies.py b/tests/test_runtime/test_error_handling_strategies.py index c1762a02a..5aa83882a 100644 --- a/tests/test_runtime/test_error_handling_strategies.py +++ b/tests/test_runtime/test_error_handling_strategies.py @@ -48,7 +48,7 @@ def test_worker_raises_when_error_in_self( settings = WorkerSettings( on_error=on_error, # <- Highlight default_report_values=DefaultReportValues(), - max_evaluations_total=None, + evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, max_cost_total=None, max_evaluations_for_worker=1, @@ -88,7 +88,7 @@ def test_worker_raises_when_error_in_other_worker(neps_state: NePSState) -> None settings = WorkerSettings( on_error=OnErrorPossibilities.RAISE_ANY_ERROR, # <- Highlight default_report_values=DefaultReportValues(), - max_evaluations_total=None, + evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, max_cost_total=None, max_evaluations_for_worker=1, @@ -148,7 +148,7 @@ def test_worker_does_not_raise_when_error_in_other_worker( settings = WorkerSettings( on_error=on_error, # <- Highlight default_report_values=DefaultReportValues(), - max_evaluations_total=None, + evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, max_cost_total=None, max_evaluations_for_worker=1, diff --git a/tests/test_runtime/test_stopping_criterion.py b/tests/test_runtime/test_stopping_criterion.py index 08fc3dbf3..850b44339 100644 --- a/tests/test_runtime/test_stopping_criterion.py +++ b/tests/test_runtime/test_stopping_criterion.py @@ -40,7 +40,7 @@ def test_max_evaluations_total_stopping_criterion( settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), - max_evaluations_total=3, # <- Highlight + evaluations_to_spend=3, # <- Highlight include_in_progress_evaluations_towards_maximum=False, max_cost_total=None, max_evaluations_for_worker=None, @@ -92,7 +92,7 @@ def test_worker_evaluations_total_stopping_criterion( settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), - max_evaluations_total=None, + evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, max_cost_total=None, max_evaluations_for_worker=2, @@ -153,7 +153,7 @@ def test_include_in_progress_evaluations_towards_maximum_with_work_eval_count( settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), - max_evaluations_total=2, # <- Highlight, only 2 maximum evaluations allowed + evaluations_to_spend=2, # <- Highlight, only 2 maximum evaluations allowed include_in_progress_evaluations_towards_maximum=True, # <- inprogress trial max_cost_total=None, max_evaluations_for_worker=None, @@ -207,7 +207,7 @@ def test_max_cost_total(neps_state: NePSState) -> None: settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), - max_evaluations_total=10, # Safety incase it doesn't work that we eventually stop + evaluations_to_spend=10, # Safety incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, max_cost_total=2, # <- Highlight, only 2 maximum evaluations allowed max_evaluations_for_worker=None, @@ -255,7 +255,7 @@ def test_worker_cost_total(neps_state: NePSState) -> None: settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), - max_evaluations_total=10, # Safety incase it doesn't work that we eventually stop + evaluations_to_spend=10, # Safety incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, max_cost_total=None, max_evaluations_for_worker=None, @@ -311,7 +311,7 @@ def test_worker_wallclock_time(neps_state: NePSState) -> None: settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), - max_evaluations_total=1000, # Incase it doesn't work that we eventually stop + evaluations_to_spend=1000, # Incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, max_cost_total=None, max_evaluations_for_worker=None, @@ -366,7 +366,7 @@ def test_max_worker_evaluation_time(neps_state: NePSState) -> None: settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), - max_evaluations_total=10, # Safety incase it doesn't work that we eventually stop + evaluations_to_spend=10, # Safety incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, max_cost_total=None, max_evaluations_for_worker=None, @@ -422,7 +422,7 @@ def test_max_evaluation_time_global(neps_state: NePSState) -> None: settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), - max_evaluations_total=10, # Safety incase it doesn't work that we eventually stop + evaluations_to_spend=10, # Safety incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, max_cost_total=None, max_evaluations_for_worker=None, From 23e5130f20c34e5dc477ed808d124e742c08ec29 Mon Sep 17 00:00:00 2001 From: Ema Mekic Date: Tue, 24 Jun 2025 12:36:02 +0200 Subject: [PATCH 06/27] Renaming max_cost_total to cost_to_spend --- docs/reference/evaluate_pipeline.md | 6 +++--- docs/reference/neps_run.md | 6 +++--- neps/api.py | 10 +++++----- neps/optimizers/bayesian_optimization.py | 4 ++-- neps/runtime.py | 14 +++++++------- neps/state/optimizer.py | 2 +- neps/state/settings.py | 2 +- neps_examples/efficiency/multi_fidelity.py | 2 +- tests/test_runtime/test_default_report_values.py | 6 +++--- .../test_error_handling_strategies.py | 6 +++--- tests/test_runtime/test_stopping_criterion.py | 16 ++++++++-------- tests/test_state/test_filebased_neps_state.py | 4 ++-- tests/test_state/test_neps_state.py | 6 +++--- 13 files changed, 42 insertions(+), 42 deletions(-) diff --git a/docs/reference/evaluate_pipeline.md b/docs/reference/evaluate_pipeline.md index f23b6d663..b26142f98 100644 --- a/docs/reference/evaluate_pipeline.md +++ b/docs/reference/evaluate_pipeline.md @@ -63,12 +63,12 @@ def evaluate_pipeline( #### Cost -Along with the return of the `loss`, the `evaluate_pipeline=` function would optionally need to return a `cost` in certain cases. Specifically when the `max_cost_total` parameter is being utilized in the `neps.run` function. +Along with the return of the `loss`, the `evaluate_pipeline=` function would optionally need to return a `cost` in certain cases. Specifically when the `cost_to_spend` parameter is being utilized in the `neps.run` function. !!! note - `max_cost_total` sums the cost from all returned configuration results and checks whether the maximum allowed cost has been reached (if so, the search will come to an end). + `cost_to_spend` sums the cost from all returned configuration results and checks whether the maximum allowed cost has been reached (if so, the search will come to an end). ```python import neps @@ -97,7 +97,7 @@ if __name__ == "__main__": evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, # Assuming the pipeline space is defined root_directory="results/bo", - max_cost_total=10, + cost_to_spend=10, optimizer="bayesian_optimization", ) ``` diff --git a/docs/reference/neps_run.md b/docs/reference/neps_run.md index de008039d..2e8342643 100644 --- a/docs/reference/neps_run.md +++ b/docs/reference/neps_run.md @@ -47,7 +47,7 @@ See the following for more: ## Budget, how long to run? To define a budget, provide `evaluations_to_spend=` to [`neps.run()`][neps.api.run], to specify the total number of evaluations to conduct before halting the optimization process, -or `max_cost_total=` to specify a cost threshold for your own custom cost metric, such as time, energy, or monetary, as returned by each evaluation of the pipeline . +or `cost_to_spend=` to specify a cost threshold for your own custom cost metric, such as time, energy, or monetary, as returned by each evaluation of the pipeline . ```python @@ -61,7 +61,7 @@ def evaluate_pipeline(learning_rate: float, epochs: int) -> float: neps.run( evaluations_to_spend=10, # (1)! - max_cost_total=1000, # (2)! + cost_to_spend=1000, # (2)! ) ``` @@ -87,7 +87,7 @@ Please refer to Python's [logging documentation](https://docs.python.org/3/libra ## Continuing Runs To continue a run, all you need to do is provide the same `root_directory=` to [`neps.run()`][neps.api.run] as before, -with an increased `evaluations_to_spend=` or `max_cost_total=`. +with an increased `evaluations_to_spend=` or `cost_to_spend=`. ```python def run(learning_rate: float, epochs: int) -> float: diff --git a/neps/api.py b/neps/api.py index 79aeb495c..01d6d2bf0 100644 --- a/neps/api.py +++ b/neps/api.py @@ -38,7 +38,7 @@ def run( # noqa: PLR0913 evaluations_to_spend: int | None = None, max_evaluations_per_run: int | None = None, continue_until_max_evaluation_completed: bool = False, - max_cost_total: int | float | None = None, + cost_to_spend: int | float | None = None, ignore_errors: bool = False, objective_value_on_error: float | None = None, cost_value_on_error: float | None = None, @@ -204,7 +204,7 @@ def evaluate_pipeline(some_parameter: float) -> float: If true, only stop after evaluations_to_spend have been completed. This is only relevant in the parallel setting. - max_cost_total: No new evaluations will start when this cost is exceeded. Requires + cost_to_spend: No new evaluations will start when this cost is exceeded. Requires returning a cost in the evaluate_pipeline function, e.g., `return dict(loss=loss, cost=cost)`. ignore_errors: Ignore hyperparameter settings that threw an error and do not raise @@ -398,14 +398,14 @@ def __call__( if ( evaluations_to_spend is None and max_evaluations_per_run is None - and max_cost_total is None + and cost_to_spend is None ): warnings.warn( "None of the following were set, this will run idefinitely until the worker" " process is stopped." f"\n * {evaluations_to_spend=}" f"\n * {max_evaluations_per_run=}" - f"\n * {max_cost_total=}", + f"\n * {cost_to_spend=}", UserWarning, stacklevel=2, ) @@ -435,7 +435,7 @@ def __call__( evaluation_fn=_eval, # type: ignore optimizer=_optimizer_ask, optimizer_info=_optimizer_info, - max_cost_total=max_cost_total, + cost_to_spend=cost_to_spend, optimization_dir=Path(root_directory), evaluations_to_spend=evaluations_to_spend, max_evaluations_for_worker=max_evaluations_per_run, diff --git a/neps/optimizers/bayesian_optimization.py b/neps/optimizers/bayesian_optimization.py index 0c8f84e86..e12e875f8 100644 --- a/neps/optimizers/bayesian_optimization.py +++ b/neps/optimizers/bayesian_optimization.py @@ -170,9 +170,9 @@ def __call__( # noqa: C901, PLR0912, PLR0915 # noqa: C901, PLR0912 "Must provide a 'cost' to configurations if using cost" " with BayesianOptimization." ) - if budget_info.max_cost_total is None: + if budget_info.cost_to_spend is None: raise ValueError("Cost budget must be set if using cost") - cost_percent = budget_info.used_cost_budget / budget_info.max_cost_total + cost_percent = budget_info.used_cost_budget / budget_info.cost_to_spend # If we should use the prior, weight the acquisition function by # the probability of it being sampled from the prior. diff --git a/neps/runtime.py b/neps/runtime.py index 97fc40f25..a8b8e664d 100644 --- a/neps/runtime.py +++ b/neps/runtime.py @@ -339,15 +339,15 @@ def _check_global_stopping_criterion( " stopping criterion." ) - if self.settings.max_cost_total is not None: + if self.settings.cost_to_spend is not None: cost = sum( trial.report.cost for _, trial in trials.items() if trial.report is not None and trial.report.cost is not None ) - if cost >= self.settings.max_cost_total: + if cost >= self.settings.cost_to_spend: return ( - f"The maximum cost `{self.settings.max_cost_total=}` has been" + f"The maximum cost `{self.settings.cost_to_spend=}` has been" " reached by all of the evaluated trials. To allow more evaluations," " increase this value or use a different stopping criterion." ) @@ -373,7 +373,7 @@ def _check_global_stopping_criterion( def _requires_global_stopping_criterion(self) -> bool: return ( self.settings.evaluations_to_spend is not None - or self.settings.max_cost_total is not None + or self.settings.cost_to_spend is not None or self.settings.max_evaluation_time_total_seconds is not None ) @@ -691,7 +691,7 @@ def _launch_runtime( # noqa: PLR0913 optimizer: AskFunction, optimizer_info: OptimizerInfo, optimization_dir: Path, - max_cost_total: float | None, + cost_to_spend: float | None, ignore_errors: bool = False, objective_value_on_error: float | None, cost_value_on_error: float | None, @@ -736,7 +736,7 @@ def _launch_runtime( # noqa: PLR0913 seed_snapshot=SeedSnapshot.new_capture(), budget=( BudgetInfo( - max_cost_total=max_cost_total, + cost_to_spend=cost_to_spend, used_cost_budget=0, max_evaluations=evaluations_to_spend, used_evaluations=0, @@ -771,7 +771,7 @@ def _launch_runtime( # noqa: PLR0913 include_in_progress_evaluations_towards_maximum=( not continue_until_max_evaluation_completed ), - max_cost_total=max_cost_total, + cost_to_spend=cost_to_spend, max_evaluations_for_worker=max_evaluations_for_worker, max_evaluation_time_total_seconds=None, # TODO: User can't specify yet max_wallclock_time_for_worker_seconds=None, # TODO: User can't specify yet diff --git a/neps/state/optimizer.py b/neps/state/optimizer.py index 5e51c21e7..d3ed771e4 100644 --- a/neps/state/optimizer.py +++ b/neps/state/optimizer.py @@ -13,7 +13,7 @@ class BudgetInfo: """Information about the budget of an optimizer.""" - max_cost_total: float | None = None + cost_to_spend: float | None = None used_cost_budget: float = 0.0 max_evaluations: int | None = None used_evaluations: int = 0 diff --git a/neps/state/settings.py b/neps/state/settings.py index 49a39ff65..242966cb3 100644 --- a/neps/state/settings.py +++ b/neps/state/settings.py @@ -98,7 +98,7 @@ class WorkerSettings: [`evaluations_to_spend`][neps.state.settings.WorkerSettings.evaluations_to_spend] """ - max_cost_total: float | None + cost_to_spend: float | None """The maximum cost to run in total. Once this cost total is reached, **all** workers will stop evaluating new diff --git a/neps_examples/efficiency/multi_fidelity.py b/neps_examples/efficiency/multi_fidelity.py index 394340c15..4c4d3245e 100644 --- a/neps_examples/efficiency/multi_fidelity.py +++ b/neps_examples/efficiency/multi_fidelity.py @@ -94,5 +94,5 @@ def evaluate_pipeline( root_directory="results/multi_fidelity_example", # Optional: Do not start another evaluation after <=50 epochs, corresponds to cost # field above. - max_cost_total=10, + cost_to_spend=10, ) diff --git a/tests/test_runtime/test_default_report_values.py b/tests/test_runtime/test_default_report_values.py index 091ac8c6e..0fd43e5e4 100644 --- a/tests/test_runtime/test_default_report_values.py +++ b/tests/test_runtime/test_default_report_values.py @@ -43,7 +43,7 @@ def test_default_values_on_error( ), evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, max_evaluations_for_worker=1, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -94,7 +94,7 @@ def test_default_values_on_not_specified( ), evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, max_evaluations_for_worker=1, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -143,7 +143,7 @@ def test_default_value_objective_to_minimize_curve_take_objective_to_minimize_va ), evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, max_evaluations_for_worker=1, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, diff --git a/tests/test_runtime/test_error_handling_strategies.py b/tests/test_runtime/test_error_handling_strategies.py index 5aa83882a..aeaeba0d9 100644 --- a/tests/test_runtime/test_error_handling_strategies.py +++ b/tests/test_runtime/test_error_handling_strategies.py @@ -50,7 +50,7 @@ def test_worker_raises_when_error_in_self( default_report_values=DefaultReportValues(), evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, max_evaluations_for_worker=1, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -90,7 +90,7 @@ def test_worker_raises_when_error_in_other_worker(neps_state: NePSState) -> None default_report_values=DefaultReportValues(), evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, max_evaluations_for_worker=1, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -150,7 +150,7 @@ def test_worker_does_not_raise_when_error_in_other_worker( default_report_values=DefaultReportValues(), evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, max_evaluations_for_worker=1, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, diff --git a/tests/test_runtime/test_stopping_criterion.py b/tests/test_runtime/test_stopping_criterion.py index 850b44339..40212cd8f 100644 --- a/tests/test_runtime/test_stopping_criterion.py +++ b/tests/test_runtime/test_stopping_criterion.py @@ -42,7 +42,7 @@ def test_max_evaluations_total_stopping_criterion( default_report_values=DefaultReportValues(), evaluations_to_spend=3, # <- Highlight include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -94,7 +94,7 @@ def test_worker_evaluations_total_stopping_criterion( default_report_values=DefaultReportValues(), evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, max_evaluations_for_worker=2, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -155,7 +155,7 @@ def test_include_in_progress_evaluations_towards_maximum_with_work_eval_count( default_report_values=DefaultReportValues(), evaluations_to_spend=2, # <- Highlight, only 2 maximum evaluations allowed include_in_progress_evaluations_towards_maximum=True, # <- inprogress trial - max_cost_total=None, + cost_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -209,7 +209,7 @@ def test_max_cost_total(neps_state: NePSState) -> None: default_report_values=DefaultReportValues(), evaluations_to_spend=10, # Safety incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, - max_cost_total=2, # <- Highlight, only 2 maximum evaluations allowed + cost_to_spend=2, # <- Highlight, only 2 maximum evaluations allowed max_evaluations_for_worker=None, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -257,7 +257,7 @@ def test_worker_cost_total(neps_state: NePSState) -> None: default_report_values=DefaultReportValues(), evaluations_to_spend=10, # Safety incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -313,7 +313,7 @@ def test_worker_wallclock_time(neps_state: NePSState) -> None: default_report_values=DefaultReportValues(), evaluations_to_spend=1000, # Incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=1, # <- highlight, 1 second @@ -368,7 +368,7 @@ def test_max_worker_evaluation_time(neps_state: NePSState) -> None: default_report_values=DefaultReportValues(), evaluations_to_spend=10, # Safety incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -424,7 +424,7 @@ def test_max_evaluation_time_global(neps_state: NePSState) -> None: default_report_values=DefaultReportValues(), evaluations_to_spend=10, # Safety incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=0.5, # <- Highlight max_wallclock_time_for_worker_seconds=None, diff --git a/tests/test_state/test_filebased_neps_state.py b/tests/test_state/test_filebased_neps_state.py index 448b0f393..5572abb4d 100644 --- a/tests/test_state/test_filebased_neps_state.py +++ b/tests/test_state/test_filebased_neps_state.py @@ -20,7 +20,7 @@ @fixture -@parametrize("budget_info", [BudgetInfo(max_cost_total=10, used_cost_budget=0), None]) +@parametrize("budget_info", [BudgetInfo(cost_to_spend=10, used_cost_budget=0), None]) @parametrize("shared_state", [{"a": "b"}, {}]) def optimizer_state( budget_info: BudgetInfo | None, @@ -81,7 +81,7 @@ def test_create_or_load_with_load_filebased_neps_state( # that we prioritize what's in the existing data over what # was passed in. different_state = OptimizationState( - budget=BudgetInfo(max_cost_total=20, used_cost_budget=10), + budget=BudgetInfo(cost_to_spend=20, used_cost_budget=10), seed_snapshot=SeedSnapshot.new_capture(), shared_state=None, ) diff --git a/tests/test_state/test_neps_state.py b/tests/test_state/test_neps_state.py index 98efd5026..0cb3c823e 100644 --- a/tests/test_state/test_neps_state.py +++ b/tests/test_state/test_neps_state.py @@ -154,11 +154,11 @@ def optimizer_and_key_and_search_space( @parametrize("optimizer_info", [OptimizerInfo(name="blah", info={"a": "b"})]) -@parametrize("max_cost_total", [BudgetInfo(max_cost_total=10, used_cost_budget=0), None]) +@parametrize("cost_to_spend", [BudgetInfo(cost_to_spend=10, used_cost_budget=0), None]) @parametrize("shared_state", [{"a": "b"}, {}]) def case_neps_state_filebased( tmp_path: Path, - max_cost_total: BudgetInfo | None, + cost_to_spend: BudgetInfo | None, optimizer_info: OptimizerInfo, shared_state: dict[str, Any], ) -> NePSState: @@ -167,7 +167,7 @@ def case_neps_state_filebased( path=new_path, optimizer_info=optimizer_info, optimizer_state=OptimizationState( - budget=max_cost_total, + budget=cost_to_spend, seed_snapshot=SeedSnapshot.new_capture(), shared_state=shared_state, ), From 092d7716445ced817ee91a1c7d5c1718a0a09b02 Mon Sep 17 00:00:00 2001 From: Ema Mekic Date: Sun, 29 Jun 2025 19:25:53 +0200 Subject: [PATCH 07/27] Introduction of fidelities_to_spend --- neps/api.py | 63 +++++++++++++++++++++++++++++++++++++++++- neps/runtime.py | 18 ++++++++++++ neps/state/settings.py | 14 ++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/neps/api.py b/neps/api.py index 01d6d2bf0..7f9f4bd0e 100644 --- a/neps/api.py +++ b/neps/api.py @@ -36,9 +36,12 @@ def run( # noqa: PLR0913 overwrite_working_directory: bool = False, post_run_summary: bool = True, evaluations_to_spend: int | None = None, + max_evaluations_total: int | None = None, max_evaluations_per_run: int | None = None, continue_until_max_evaluation_completed: bool = False, cost_to_spend: int | float | None = None, + max_cost_total: int | float | None = None, + fidelities_to_spend: int | None = None, ignore_errors: bool = False, objective_value_on_error: float | None = None, cost_value_on_error: float | None = None, @@ -207,6 +210,9 @@ def evaluate_pipeline(some_parameter: float) -> float: cost_to_spend: No new evaluations will start when this cost is exceeded. Requires returning a cost in the evaluate_pipeline function, e.g., `return dict(loss=loss, cost=cost)`. + + fidelities_to_spend: Number of evaluations in case of multi-fidelity after which to terminate. + ignore_errors: Ignore hyperparameter settings that threw an error and do not raise an error. Error configs still count towards evaluations_to_spend. objective_value_on_error: Setting this and cost_value_on_error to any float will @@ -397,23 +403,77 @@ def __call__( """ # noqa: E501 if ( evaluations_to_spend is None + and max_evaluations_total is None and max_evaluations_per_run is None and cost_to_spend is None + and max_cost_total is None + and fidelities_to_spend is None ): warnings.warn( "None of the following were set, this will run idefinitely until the worker" " process is stopped." f"\n * {evaluations_to_spend=}" f"\n * {max_evaluations_per_run=}" - f"\n * {cost_to_spend=}", + f"\n * {cost_to_spend=}" + f"\n * {fidelities_to_spend}", UserWarning, stacklevel=2, ) + if max_evaluations_total is not None: + warnings.warn( + "`max_evaluations_total` is deprecated and will be removed in a future release. " + "Please use `evaluations_to_spend` instead.", + DeprecationWarning, + stacklevel=2, + ) + evaluations_to_spend = max_evaluations_total + + if max_cost_total is not None: + warnings.warn( + "`max_cost_total` is deprecated and will be removed in a future release. " + "Please use `cost_to_spend` instead.", + DeprecationWarning, + stacklevel=2, + ) + cost_to_spend = max_cost_total + + criteria = { + "evaluations_to_spend": evaluations_to_spend, + "max_evaluations_per_run": max_evaluations_per_run, + "cost_to_spend": cost_to_spend, + "fidelities_to_spend": fidelities_to_spend, + } + set_criteria = [k for k, v in criteria.items() if v is not None] + if len(set_criteria) > 1: + raise ValueError( + f"Multiple stopping criteria specified: {', '.join(set_criteria)}. " + "Only one is allowed." + ) + logger.info(f"Starting neps.run using root directory {root_directory}") space = convert_to_space(pipeline_space) _optimizer_ask, _optimizer_info = load_optimizer(optimizer=optimizer, space=space) + multi_fidelity_optimizers = { + "successive_halving", + "asha", + "hyperband", + "async_hb", + "ifbo", + "priorband", + } + + is_multi_fidelity = _optimizer_info["name"] in multi_fidelity_optimizers + + if is_multi_fidelity: + if evaluations_to_spend is not None: + raise ValueError("`evaluations_to_spend` is not allowed for multi-fidelity optimizers. Only `fidelities_to_spend` or `cost_to_spend`") + else: + if fidelities_to_spend is not None: + raise ValueError("`fidelities_to_spend` is not allowed for non-multi-fidelity optimizers.") + + _eval: Callable if isinstance(evaluate_pipeline, str): module, funcname = evaluate_pipeline.rsplit(":", 1) @@ -436,6 +496,7 @@ def __call__( optimizer=_optimizer_ask, optimizer_info=_optimizer_info, cost_to_spend=cost_to_spend, + fidelities_to_spend=fidelities_to_spend, optimization_dir=Path(root_directory), evaluations_to_spend=evaluations_to_spend, max_evaluations_for_worker=max_evaluations_per_run, diff --git a/neps/runtime.py b/neps/runtime.py index a8b8e664d..37377a6a3 100644 --- a/neps/runtime.py +++ b/neps/runtime.py @@ -338,6 +338,20 @@ def _check_global_stopping_criterion( " To allow more evaluations, increase this value or use a different" " stopping criterion." ) + + if self.settings.fidelities_to_spend is not None: + count = sum( + trial.report.cost + for _, trial in trials.items() + if trial.report is not None and trial.report.cost is not None + ) + if count >= self.settings.fidelities_to_spend: + return ( + "The total number of fidelity evaluations has reached the maximum allowed of" + f" `{self.settings.fidelities_to_spend=}`." + " To allow more evaluations, increase this value or use a different" + " stopping criterion." + ) if self.settings.cost_to_spend is not None: cost = sum( @@ -374,6 +388,7 @@ def _requires_global_stopping_criterion(self) -> bool: return ( self.settings.evaluations_to_spend is not None or self.settings.cost_to_spend is not None + or self.settings.fidelities_to_spend is not None or self.settings.max_evaluation_time_total_seconds is not None ) @@ -698,6 +713,7 @@ def _launch_runtime( # noqa: PLR0913 continue_until_max_evaluation_completed: bool, overwrite_optimization_dir: bool, evaluations_to_spend: int | None, + fidelities_to_spend: int | None, max_evaluations_for_worker: int | None, sample_batch_size: int | None, ) -> None: @@ -739,6 +755,7 @@ def _launch_runtime( # noqa: PLR0913 cost_to_spend=cost_to_spend, used_cost_budget=0, max_evaluations=evaluations_to_spend, + fidelities_to_spend=fidelities_to_spend, used_evaluations=0, ) ), @@ -768,6 +785,7 @@ def _launch_runtime( # noqa: PLR0913 batch_size=sample_batch_size, default_report_values=default_report_values, evaluations_to_spend=evaluations_to_spend, + fidelities_to_spend=fidelities_to_spend, include_in_progress_evaluations_towards_maximum=( not continue_until_max_evaluation_completed ), diff --git a/neps/state/settings.py b/neps/state/settings.py index 242966cb3..7de86ab36 100644 --- a/neps/state/settings.py +++ b/neps/state/settings.py @@ -111,6 +111,20 @@ class WorkerSettings: indefinitely or until another stopping criterion is met. """ + fidelities_to_spend: int | None + """The maximum number of evaluations to run in case of multi-fidelity. + + Once this evaluation total is reached, **all** workers will stop evaluating + new configurations. + + To control whether currently evaluating configurations are included in this + total, see + [`include_in_progress_evaluations_towards_maximum`][neps.state.settings.WorkerSettings.include_in_progress_evaluations_towards_maximum]. + + If `None`, there is no limit and workers will continue to evaluate + indefinitely. + """ + max_evaluation_time_total_seconds: float | None """The maximum wallclock time allowed for evaluation in total. From 459cb884c52522f22ffcd16427733dab60dde84b Mon Sep 17 00:00:00 2001 From: Ema Mekic Date: Sun, 29 Jun 2025 19:37:45 +0200 Subject: [PATCH 08/27] Mf example --- neps_examples/efficiency/multi_fidelity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neps_examples/efficiency/multi_fidelity.py b/neps_examples/efficiency/multi_fidelity.py index 4c4d3245e..1a850e837 100644 --- a/neps_examples/efficiency/multi_fidelity.py +++ b/neps_examples/efficiency/multi_fidelity.py @@ -94,5 +94,5 @@ def evaluate_pipeline( root_directory="results/multi_fidelity_example", # Optional: Do not start another evaluation after <=50 epochs, corresponds to cost # field above. - cost_to_spend=10, + fidelities_to_spend=10 ) From f56f43312c1ff179c8d7c18d61c2e42afd839c3d Mon Sep 17 00:00:00 2001 From: Ema Mekic Date: Sun, 29 Jun 2025 19:45:18 +0200 Subject: [PATCH 09/27] Optimizer - fidelities to spend --- neps/state/optimizer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/neps/state/optimizer.py b/neps/state/optimizer.py index d3ed771e4..7d4787cce 100644 --- a/neps/state/optimizer.py +++ b/neps/state/optimizer.py @@ -17,6 +17,7 @@ class BudgetInfo: used_cost_budget: float = 0.0 max_evaluations: int | None = None used_evaluations: int = 0 + fidelities_to_spend: int | None = None def clone(self) -> BudgetInfo: """Create a copy of the budget info.""" From 1e8fdbd852258876bd1ccc81f48cbb9a6329b1b1 Mon Sep 17 00:00:00 2001 From: Ema Mekic Date: Sun, 6 Jul 2025 14:14:56 +0200 Subject: [PATCH 10/27] Added trajectory and best incumbent. Solved warning in plot --- docs/reference/analyse.md | 20 ++--- docs/reference/neps_run.md | 9 +- neps/api.py | 29 +++---- neps/optimizers/bracket_optimizer.py | 16 ++-- neps/plot/plotting.py | 6 +- neps/runtime.py | 118 +++++++++++++++++---------- neps/state/neps_state.py | 9 +- neps/state/settings.py | 5 ++ neps/status/status.py | 77 ++++++++++++++++- 9 files changed, 200 insertions(+), 89 deletions(-) diff --git a/docs/reference/analyse.md b/docs/reference/analyse.md index 92c42341c..2edb4ed4b 100644 --- a/docs/reference/analyse.md +++ b/docs/reference/analyse.md @@ -40,10 +40,10 @@ Currently, this creates one plot that shows the best error value across the numb ## What's on disk? In the root directory, NePS maintains several files at all times that are human readable and can be useful -If you pass the `post_run_summary=` argument to [`neps.run()`][neps.api.run], -NePS will also generate a summary CSV file for you. +If you pass the `write_summary_to_disk=` argument to [`neps.run()`][neps.api.run], +NePS will generate a summary CSV and TXT files for you. -=== "`neps.run(..., post_run_summary=True)`" +=== "`neps.run(..., write_summary_to_disk=True)`" ``` ROOT_DIRECTORY @@ -54,15 +54,15 @@ NePS will also generate a summary CSV file for you. │ └── report.yaml ├── summary │ ├── full.csv - │ └── short.csv - ├── best_objective_trajectory.txt - ├── best_objective.txt + │ ├── short.csv + │ ├── best_config_trajectory.txt + │ └── best_config.txt ├── optimizer_info.yaml └── optimizer_state.pkl ``` -=== "`neps.run(..., post_run_summary=False)`" +=== "`neps.run(..., write_summary_to_disk=False)`" ``` ROOT_DIRECTORY @@ -71,8 +71,6 @@ NePS will also generate a summary CSV file for you. │ ├── config.yaml │ ├── metadata.yaml │ └── report.yaml - ├── best_objective_trajectory.txt - ├── best_objective.txt ├── optimizer_info.yaml └── optimizer_state.pkl ``` @@ -81,8 +79,8 @@ NePS will also generate a summary CSV file for you. The `full.csv` contains all configuration details in CSV format. Details include configuration hyperparameters and any returned result and cost from the `evaluate_pipeline` function. -The `best_objective_trajectory.txt` contains logging of the incumbent trajectory. -The `best_objective.txt` records current incumbent. +The `best_config_trajectory.txt` contains logging of the incumbent trajectory. +The `best_config.txt` records current incumbent. # TensorBoard Integration diff --git a/docs/reference/neps_run.md b/docs/reference/neps_run.md index bb968aa25..1793ea16c 100644 --- a/docs/reference/neps_run.md +++ b/docs/reference/neps_run.md @@ -125,9 +125,6 @@ neps.run( ## Getting the results The results of the optimization process are stored in the `root_directory=` provided to [`neps.run()`][neps.api.run]. -To obtain a summary of the optimization process, you can enable the -`post_run_summary=True` argument in [`neps.run()`][neps.api.run], -while will generate a summary csv after the run has finished. === "Result Directory" @@ -143,9 +140,11 @@ while will generate a summary csv after the run has finished. │ └── config_2 │ ├── config.yaml │ └── metadata.json - ├── summary # Only if post_run_summary=True + ├── summary │ ├── full.csv │ └── short.csv + │ ├── best_config_trajectory.txt + │ └── best_config.txt ├── optimizer_info.yaml # The optimizer's configuration └── optimizer_state.pkl # The optimizer's state, shared between workers ``` @@ -153,7 +152,7 @@ while will generate a summary csv after the run has finished. === "python" ```python - neps.run(..., post_run_summary=True) + neps.run(..., write_summary_to_disk=True) ``` To capture the results of the optimization process, you can use tensorbaord logging with various utilities to integrate diff --git a/neps/api.py b/neps/api.py index ca1bdc5b8..9cb9d8893 100644 --- a/neps/api.py +++ b/neps/api.py @@ -11,7 +11,7 @@ from neps.optimizers import AskFunction, OptimizerChoice, load_optimizer from neps.runtime import _launch_runtime from neps.space.parsing import convert_to_space -from neps.status.status import post_run_csv +from neps.status.status import post_run_csv, trajectory_of_improvements from neps.utils.common import dynamic_load_object if TYPE_CHECKING: @@ -34,7 +34,7 @@ def run( # noqa: PLR0913 *, root_directory: str | Path = "neps_results", overwrite_root_directory: bool = False, - post_run_summary: bool = True, + write_summary_to_disk: bool = True, max_evaluations_total: int | None = None, max_evaluations_per_run: int | None = None, continue_until_max_evaluation_completed: bool = False, @@ -192,7 +192,7 @@ def evaluate_pipeline(some_parameter: float) -> float: overwrite_root_directory: If true, delete the working directory at the start of the run. This is, e.g., useful when debugging a evaluate_pipeline function. - post_run_summary: If True, creates a csv file after each worker is done, + write_summary_to_disk: If True, creates a csv and txt files after each worker is done, holding summary information about the configs and results. max_evaluations_per_run: Number of evaluations this specific call should do. @@ -445,22 +445,19 @@ def __call__( ignore_errors=ignore_errors, overwrite_optimization_dir=overwrite_root_directory, sample_batch_size=sample_batch_size, + write_summary_to_disk=write_summary_to_disk, ) - if post_run_summary: - full_frame_path, short_path = post_run_csv(root_directory) + post_run_csv(root_directory) + root_directory = Path(root_directory) + summary_dir = root_directory / "summary" + if write_summary_to_disk==False: + trajectory_of_improvements(root_directory) logger.info( - "The post run summary has been created, which is a csv file with the " - "output of all data in the run." - f"\nYou can find a full dataframe at: {full_frame_path}." - f"\nYou can find a quick summary at: {short_path}." + "The summary folder has been created, which contains csv and txt files with the " + "output of all data in the run (short.csv - only the best; full.csv - all runs" + "; best_config_trajectory.txt for incumbent trajectory; and best_config.txt for final incumbent)." + f"\nYou can find summary folder at: {summary_dir}." ) - else: - logger.info( - "Skipping the creation of the post run summary, which is a csv file with the " - " output of all data in the run." - "\nSet `post_run_summary=True` to enable it." - ) - __all__ = ["run"] diff --git a/neps/optimizers/bracket_optimizer.py b/neps/optimizers/bracket_optimizer.py index baeea37ff..da9c0cc70 100644 --- a/neps/optimizers/bracket_optimizer.py +++ b/neps/optimizers/bracket_optimizer.py @@ -44,7 +44,7 @@ def trials_to_table(trials: Mapping[str, Trial]) -> pd.DataFrame: configs = np.empty(len(trials), dtype=object) for i, (trial_id, trial) in enumerate(trials.items()): - config_id_str, rung_str = trial_id.split("_") + config_id_str, rung_str = trial_id.split("_rung_") _id, _rung = int(config_id_str), int(rung_str) if trial.report is None: @@ -272,7 +272,7 @@ def __call__( # noqa: C901, PLR0912 config.update(space.constants) config[self.fid_name] = self.fid_max rung = max(self.rung_to_fid) - return SampledConfig(id=f"1_{rung}", config=config) + return SampledConfig(id=f"1_rung_{rung}", config=config) case True: # fid_min config = { name: p.prior if p.prior is not None else p.center @@ -281,7 +281,7 @@ def __call__( # noqa: C901, PLR0912 config.update(space.constants) config[self.fid_name] = self.fid_min rung = min(self.rung_to_fid) - return SampledConfig(id=f"1_{rung}", config=config) + return SampledConfig(id=f"1_rung_{rung}", config=config) case False: pass @@ -333,9 +333,9 @@ def __call__( # noqa: C901, PLR0912 self.fid_name: self.rung_to_fid[new_rung], } return SampledConfig( - id=f"{config_id}_{new_rung}", + id=f"{config_id}_rung_{new_rung}", config=config, - previous_config_id=f"{config_id}_{new_rung - 1}", + previous_config_id=f"{config_id}_rung_{new_rung - 1}", ) # The bracket would like us to sample a new configuration for a rung @@ -352,7 +352,7 @@ def __call__( # noqa: C901, PLR0912 target_fidelity=target_fidelity, ) config.update(space.constants) - return SampledConfig(id=f"{nxt_id}_{rung}", config=config) + return SampledConfig(id=f"{nxt_id}_rung_{rung}", config=config) # We need to sample for a new rung, with either no gp or it has # not yet kicked in. @@ -366,7 +366,7 @@ def __call__( # noqa: C901, PLR0912 **space.constants, self.fid_name: self.rung_to_fid[rung], } - return SampledConfig(id=f"{nxt_id}_{rung}", config=config) + return SampledConfig(id=f"{nxt_id}_rung_{rung}", config=config) case PriorBandSampler(): config = self.sampler.sample_config(table, rung=rung) @@ -375,7 +375,7 @@ def __call__( # noqa: C901, PLR0912 **space.constants, self.fid_name: self.rung_to_fid[rung], } - return SampledConfig(id=f"{nxt_id}_{rung}", config=config) + return SampledConfig(id=f"{nxt_id}_rung_{rung}", config=config) case _: raise RuntimeError(f"Unknown sampler: {self.sampler}") case _: diff --git a/neps/plot/plotting.py b/neps/plot/plotting.py index f87461b56..c5054595a 100644 --- a/neps/plot/plotting.py +++ b/neps/plot/plotting.py @@ -139,11 +139,11 @@ def _interpolate_time( if x_range is not None: min_b, max_b = x_range - new_entry = {c: np.nan for c in df.columns} + new_entry = dict.fromkeys(df.columns, np.nan) _df = pd.DataFrame.from_dict(new_entry, orient="index").T _df.index = [min_b] df = pd.concat((df, _df)).sort_index() - new_entry = {c: np.nan for c in df.columns} + new_entry = dict.fromkeys(df.columns, np.nan) _df = pd.DataFrame.from_dict(new_entry, orient="index").T _df.index = [max_b] df = pd.concat((df, _df)).sort_index() @@ -157,7 +157,7 @@ def _interpolate_time( def _df_to_x_range(df: pd.DataFrame, x_range: tuple | None = None) -> pd.DataFrame: x_max = np.inf if x_range is None else int(x_range[-1]) - new_entry = {c: np.nan for c in df.columns} + new_entry = dict.fromkeys(df.columns, np.nan) _df = pd.DataFrame.from_dict(new_entry, orient="index").T _df.index = [x_max] df = pd.concat((df, _df)).sort_index() diff --git a/neps/runtime.py b/neps/runtime.py index 3a1130a74..69a93fbd1 100644 --- a/neps/runtime.py +++ b/neps/runtime.py @@ -44,6 +44,7 @@ WorkerSettings, evaluate_trial, ) +from neps.status.status import post_run_csv, _initiate_summary_csv, status from neps.utils.common import gc_disabled if TYPE_CHECKING: @@ -507,19 +508,30 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 _set_workers_neps_state(self.state) main_dir = Path(self.state.path) - improvement_trace_path = main_dir / "best_objective_trajectory.txt" - _trace_lock = FileLock(".trace.lock") - best_config_path = main_dir / "best_objective.txt" + if self.settings.write_summary_to_disk: + full_df_path, short_path, csv_locker = _initiate_summary_csv(main_dir) - if not improvement_trace_path.exists(): - with _trace_lock: - with open(improvement_trace_path, mode='w') as f: - f.write("Best configs and their objectives across evaluations:\n") - f.write("-" * 80 + "\n") + # Create empty CSV files + with csv_locker.lock(): + full_df_path.parent.mkdir(parents=True, exist_ok=True) + full_df_path.touch(exist_ok=True) + short_path.touch(exist_ok=True) - _best_score_so_far = float("inf") + summary_dir = main_dir / "summary" + summary_dir.mkdir(parents=True, exist_ok=True) + + improvement_trace_path = summary_dir / "best_config_trajectory.txt" + improvement_trace_path.touch(exist_ok=True) + best_config_path = summary_dir / "best_config.txt" + best_config_path.touch(exist_ok=True) + _trace_lock = FileLock(".trace.lock") + _trace_lock_path = Path(str(_trace_lock.lock_file)) + _trace_lock_path.touch(exist_ok=True) - logger.info("Launching NePS") + all_best_configs = [] + logger.info("Summary files of evaluations can be found in folder `Summary` in the main directory: %s", main_dir) + + _best_score_so_far = float("inf") optimizer_name = self.state._optimizer_info["name"] logger.info("Using optimizer: %s", optimizer_name) @@ -628,40 +640,6 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 evaluated_trial.metadata.state, ) - if report.objective_to_minimize is not None and report.err is None: - new_score = report.objective_to_minimize - if new_score < _best_score_so_far: - _best_score_so_far = new_score - logger.info( - "Evaluated trial: %s with objective %s is the new best trial.", - evaluated_trial.id, - new_score, - ) - logger.info("Trajectory of best objectives can be found in %s", improvement_trace_path) - logger.info("New best incumbent can be found in %s", best_config_path) - config_text = ( - f"Objective to minimize: {new_score}\n" - f"Config ID: {evaluated_trial.id}\n" - f"Config: {evaluated_trial.config}\n" - + "-" * 80 + "\n" - ) - - # Log improvement to trace - with _trace_lock: - with open(improvement_trace_path, mode='a') as f: - f.write(config_text) - - # Write best config to file (overwrite mode) - best_summary = ( - f"# Best incumbent:" - "\n" - f"\n Config ID: {evaluated_trial.id}" - f"\n Objective to minimize: {new_score}" - f"\n Config: {evaluated_trial.config}" - ) - with open(best_config_path, "w") as f: - f.write(best_summary) - if report.cost is not None: self.worker_cumulative_eval_cost += report.cost @@ -686,6 +664,56 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 for _key, callback in _TRIAL_END_CALLBACKS.items(): callback(trial_to_eval) + if report.objective_to_minimize is not None and report.err is None: + new_score = report.objective_to_minimize + if new_score < _best_score_so_far: + _best_score_so_far = new_score + logger.info( + "Evaluated trial: %s with objective %s is the new best trial.", + evaluated_trial.id, + new_score, + ) + + if self.settings.write_summary_to_disk: + # Store in memory for later file re-writing + all_best_configs.append({ + "score": new_score, + "trial_id": evaluated_trial.id, + "config": evaluated_trial.config + }) + + # Build trace text and best config text + trace_text = "Best configs and their objectives across evaluations:\n" + "-" * 80 + "\n" + for best in all_best_configs: + trace_text += ( + f"Objective to minimize: {best['score']}\n" + f"Config ID: {best['trial_id']}\n" + f"Config: {best['config']}\n" + + "-" * 80 + "\n" + ) + + best_config = all_best_configs[-1] # Latest best + best_config_text = ( + f"# Best config:" + f"\n\n Config ID: {best_config['trial_id']}" + f"\n Objective to minimize: {best_config['score']}" + f"\n Config: {best_config['config']}" + ) + + # Write files from scratch + with _trace_lock: + with open(improvement_trace_path, mode='w') as f: + f.write(trace_text) + + with open(best_config_path, mode='w') as f: + f.write(best_config_text) + + if self.settings.write_summary_to_disk: + full_df, short = status(main_dir) + with csv_locker.lock(): + full_df.to_csv(full_df_path) + short.to_frame().to_csv(short_path) + logger.debug("Config %s: %s", evaluated_trial.id, evaluated_trial.config) logger.debug("Loss %s: %s", evaluated_trial.id, report.objective_to_minimize) logger.debug("Cost %s: %s", evaluated_trial.id, report.objective_to_minimize) @@ -763,6 +791,7 @@ def _launch_runtime( # noqa: PLR0913 max_evaluations_total: int | None, max_evaluations_for_worker: int | None, sample_batch_size: int | None, + write_summary_to_disk: bool = True, ) -> None: default_report_values = DefaultReportValues( objective_value_on_error=objective_value_on_error, @@ -840,6 +869,7 @@ def _launch_runtime( # noqa: PLR0913 max_wallclock_time_for_worker_seconds=None, # TODO: User can't specify yet max_evaluation_time_for_worker_seconds=None, # TODO: User can't specify yet max_cost_for_worker=None, # TODO: User can't specify yet + write_summary_to_disk=write_summary_to_disk ) # HACK: Due to nfs file-systems, locking with the default `flock()` is not reliable. diff --git a/neps/state/neps_state.py b/neps/state/neps_state.py index 0e684a428..149cfe1a5 100644 --- a/neps/state/neps_state.py +++ b/neps/state/neps_state.py @@ -382,9 +382,16 @@ def _sample_trial( else: previous_trial_location = None + id_str = sampled_config.id + config_name = f"{sampled_config.id}" + parts = id_str.split('_rung_') + if len(parts) == 2 and all(p.isdigit() for p in parts): + config_id, rung_id = map(int, parts) + config_name = f"{config_id}_rung_{rung_id}" + trial = Trial.new( trial_id=sampled_config.id, - location=str(self._trial_repo.directory / f"config_{sampled_config.id}"), + location=str(self._trial_repo.directory / f"config_{config_name}"), config=sampled_config.config, previous_trial=sampled_config.previous_config_id, previous_trial_location=previous_trial_location, diff --git a/neps/state/settings.py b/neps/state/settings.py index bbc7c0498..ca20de648 100644 --- a/neps/state/settings.py +++ b/neps/state/settings.py @@ -174,3 +174,8 @@ class WorkerSettings: If `None`, there is no limit and this worker will continue to evaluate indefinitely or until another stopping criterion is met. """ + + write_summary_to_disk: bool = True + """If True, creates a csv and txt files after each worker is done, + holding summary information about the configs and results. + """ diff --git a/neps/status/status.py b/neps/status/status.py index 4d8d4421e..0b51cff6a 100644 --- a/neps/status/status.py +++ b/neps/status/status.py @@ -11,7 +11,6 @@ import numpy as np import pandas as pd -from neps.runtime import get_workers_neps_state from neps.state.neps_state import FileLocker, NePSState from neps.state.trial import State, Trial @@ -141,6 +140,7 @@ def from_directory(cls, root_directory: str | Path) -> Summary: # NOTE: We don't lock the shared state since we are just reading and don't need to # make decisions based on the state try: + from neps.runtime import get_workers_neps_state shared_state = get_workers_neps_state() except RuntimeError: shared_state = NePSState.create_or_load(root_directory, load_only=True) @@ -220,6 +220,80 @@ def status( return df, short +def trajectory_of_improvements( + root_directory: str | Path, +) -> list[dict]: + """Track and write the trajectory of improving configurations over time. + + Args: + root_directory: The root directory given to neps.run. + + Returns: + List of dicts with improving scores and their configurations. + """ + root_directory = Path(root_directory) + summary = Summary.from_directory(root_directory) + + if summary.is_multiobjective: + return [] + + df = summary.df() + + if "time_sampled" not in df.columns: + raise ValueError("Missing `time_sampled` column in summary DataFrame.") + + df = df.sort_values("time_sampled") + + all_best_configs = [] + best_score = float("inf") + trace_text = "" + + for trial_id, row in df.iterrows(): + if "objective_to_minimize" not in row or pd.isna(row["objective_to_minimize"]): + continue + + score = row["objective_to_minimize"] + if score < best_score: + best_score = score + config = { + k.replace("config.", ""): v + for k, v in row.items() + if k.startswith("config.") + } + + best = { + "score": score, + "trial_id": trial_id, + "config": config, + } + all_best_configs.append(best) + + trace_text += ( + f"Objective to minimize: {best['score']}\n" + f"Config ID: {best['trial_id']}\n" + f"Config: {best['config']}\n" + + "-" * 80 + "\n" + ) + + summary_dir = root_directory / "summary" + summary_dir.mkdir(parents=True, exist_ok=True) + output_path = summary_dir / "best_config_trajectory.txt" + with open(output_path, "w") as f: + f.write(trace_text) + + if all_best_configs: + final_best = all_best_configs[-1] + best_path = summary_dir / "best_config.txt" + with open(best_path, "w") as f: + f.write( + f"Objective to minimize: {final_best['score']}\n" + f"Config ID: {final_best['trial_id']}\n" + f"Config: {final_best['config']}\n" + ) + + return all_best_configs + + def _initiate_summary_csv(root_directory: str | Path) -> tuple[Path, Path, FileLocker]: """Initializes a summary CSV and an associated locker for file access control. @@ -262,3 +336,4 @@ def post_run_csv(root_directory: str | Path) -> tuple[Path, Path]: short.to_frame().to_csv(short_path) return full_df_path, short_path + From e2639f46688d8b8b1bef7377d69eb1e105e294a4 Mon Sep 17 00:00:00 2001 From: Ema Mekic Date: Sun, 6 Jul 2025 18:18:24 +0200 Subject: [PATCH 11/27] Fixed multi-fidelity stopping criteria --- neps/runtime.py | 5 +++-- neps_examples/efficiency/multi_fidelity.py | 2 +- neps_examples/efficiency/multi_fidelity_and_expert_priors.py | 2 +- neps_examples/efficiency/pytorch_lightning_ddp.py | 2 +- neps_examples/efficiency/pytorch_lightning_fsdp.py | 2 +- neps_examples/efficiency/pytorch_native_fsdp.py | 2 +- neps_examples/experimental/freeze_thaw.py | 2 +- neps_examples/real_world/image_segmentation_hpo.py | 2 +- 8 files changed, 10 insertions(+), 9 deletions(-) diff --git a/neps/runtime.py b/neps/runtime.py index 37377a6a3..70dcf48a8 100644 --- a/neps/runtime.py +++ b/neps/runtime.py @@ -340,10 +340,11 @@ def _check_global_stopping_criterion( ) if self.settings.fidelities_to_spend is not None: + fidelity_name = list(self.optimizer.space.fidelities.keys())[0] count = sum( - trial.report.cost + trial.config[fidelity_name] for _, trial in trials.items() - if trial.report is not None and trial.report.cost is not None + if trial.report is not None and trial.config[fidelity_name] is not None ) if count >= self.settings.fidelities_to_spend: return ( diff --git a/neps_examples/efficiency/multi_fidelity.py b/neps_examples/efficiency/multi_fidelity.py index 1a850e837..b067eac2b 100644 --- a/neps_examples/efficiency/multi_fidelity.py +++ b/neps_examples/efficiency/multi_fidelity.py @@ -94,5 +94,5 @@ def evaluate_pipeline( root_directory="results/multi_fidelity_example", # Optional: Do not start another evaluation after <=50 epochs, corresponds to cost # field above. - fidelities_to_spend=10 + fidelities_to_spend=20 ) diff --git a/neps_examples/efficiency/multi_fidelity_and_expert_priors.py b/neps_examples/efficiency/multi_fidelity_and_expert_priors.py index 074b8aaa2..5e8ff2aab 100644 --- a/neps_examples/efficiency/multi_fidelity_and_expert_priors.py +++ b/neps_examples/efficiency/multi_fidelity_and_expert_priors.py @@ -43,5 +43,5 @@ def evaluate_pipeline(float1, float2, integer1, fidelity): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/multifidelity_priors", - evaluations_to_spend=25, # For an alternate stopping method see multi_fidelity.py + fidelities_to_spend=25, # For an alternate stopping method see multi_fidelity.py ) diff --git a/neps_examples/efficiency/pytorch_lightning_ddp.py b/neps_examples/efficiency/pytorch_lightning_ddp.py index 21a4fba01..d40a29cdd 100644 --- a/neps_examples/efficiency/pytorch_lightning_ddp.py +++ b/neps_examples/efficiency/pytorch_lightning_ddp.py @@ -100,5 +100,5 @@ def evaluate_pipeline(lr=0.1, epoch=20): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/pytorch_lightning_ddp", - evaluations_to_spend=5 + fidelities_to_spend=5 ) diff --git a/neps_examples/efficiency/pytorch_lightning_fsdp.py b/neps_examples/efficiency/pytorch_lightning_fsdp.py index aa568c4d0..c7317bfc6 100644 --- a/neps_examples/efficiency/pytorch_lightning_fsdp.py +++ b/neps_examples/efficiency/pytorch_lightning_fsdp.py @@ -74,5 +74,5 @@ def evaluate_pipeline(lr=0.1, epoch=20): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/pytorch_lightning_fsdp", - evaluations_to_spend=5 + fidelities_to_spend=5 ) diff --git a/neps_examples/efficiency/pytorch_native_fsdp.py b/neps_examples/efficiency/pytorch_native_fsdp.py index b549fd0c7..4b40e318a 100644 --- a/neps_examples/efficiency/pytorch_native_fsdp.py +++ b/neps_examples/efficiency/pytorch_native_fsdp.py @@ -212,5 +212,5 @@ def evaluate_pipeline(lr=0.1, epoch=20): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/pytorch_fsdp", - evaluations_to_spend=20 + fidelities_to_spend=20 ) diff --git a/neps_examples/experimental/freeze_thaw.py b/neps_examples/experimental/freeze_thaw.py index e9134e93c..bcf4fb618 100644 --- a/neps_examples/experimental/freeze_thaw.py +++ b/neps_examples/experimental/freeze_thaw.py @@ -166,7 +166,7 @@ def training_pipeline( pipeline_space=pipeline_space, evaluate_pipeline=training_pipeline, optimizer="ifbo", - evaluations_to_spend=50, + fidelities_to_spend=50, root_directory="./results/ifbo-mnist/", overwrite_working_directory=False, # set to False for a multi-worker run ) diff --git a/neps_examples/real_world/image_segmentation_hpo.py b/neps_examples/real_world/image_segmentation_hpo.py index 3d7e481f1..51ff27a06 100644 --- a/neps_examples/real_world/image_segmentation_hpo.py +++ b/neps_examples/real_world/image_segmentation_hpo.py @@ -125,5 +125,5 @@ def evaluate_pipeline(**kwargs): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/hpo_image_segmentation", - evaluations_to_spend=500 + fidelities_to_spend=500 ) From eff33c72161df87b0ac737dec8b5a8cd74f9e631 Mon Sep 17 00:00:00 2001 From: Sohambasu07 Date: Thu, 17 Jul 2025 12:18:39 +0530 Subject: [PATCH 12/27] feat: update api with MO and fix runtime issues --- neps/api.py | 31 +++++++++++++--------- neps/runtime.py | 68 +++++++++++++++++++++++++++++-------------------- 2 files changed, 60 insertions(+), 39 deletions(-) diff --git a/neps/api.py b/neps/api.py index 8ab35dd0d..f0bd5124a 100644 --- a/neps/api.py +++ b/neps/api.py @@ -24,7 +24,7 @@ logger = logging.getLogger(__name__) -def run( # noqa: PLR0913 +def run( # noqa: C901, D417, PLR0913 evaluate_pipeline: Callable[..., EvaluatePipelineReturn] | str, pipeline_space: ( Mapping[str, dict | str | int | float | Parameter] @@ -422,8 +422,8 @@ def __call__( if max_evaluations_total is not None: warnings.warn( - "`max_evaluations_total` is deprecated and will be removed in a future release. " - "Please use `evaluations_to_spend` instead.", + "`max_evaluations_total` is deprecated and will be removed in" + " a future release. Please use `evaluations_to_spend` instead.", DeprecationWarning, stacklevel=2, ) @@ -462,17 +462,22 @@ def __call__( "async_hb", "ifbo", "priorband", + "moasha", + "mo_hyperband", } is_multi_fidelity = _optimizer_info["name"] in multi_fidelity_optimizers if is_multi_fidelity: if evaluations_to_spend is not None: - raise ValueError("`evaluations_to_spend` is not allowed for multi-fidelity optimizers. Only `fidelities_to_spend` or `cost_to_spend`") - else: - if fidelities_to_spend is not None: - raise ValueError("`fidelities_to_spend` is not allowed for non-multi-fidelity optimizers.") - + raise ValueError( + "`evaluations_to_spend` is not allowed for multi-fidelity optimizers. " + "Only `fidelities_to_spend` or `cost_to_spend`" + ) + elif fidelities_to_spend is not None: + raise ValueError( + "`fidelities_to_spend` is not allowed for non-multi-fidelity optimizers." + ) _eval: Callable if isinstance(evaluate_pipeline, str): @@ -512,13 +517,15 @@ def __call__( post_run_csv(root_directory) root_directory = Path(root_directory) summary_dir = root_directory / "summary" - if write_summary_to_disk==False: + if not write_summary_to_disk: trajectory_of_improvements(root_directory) logger.info( - "The summary folder has been created, which contains csv and txt files with the " - "output of all data in the run (short.csv - only the best; full.csv - all runs" - "; best_config_trajectory.txt for incumbent trajectory; and best_config.txt for final incumbent)." + "The summary folder has been created, which contains csv and txt files with" + "the output of all data in the run (short.csv - only the best; full.csv - " + "all runs; best_config_trajectory.txt for incumbent trajectory; and " + "best_config.txt for final incumbent)." f"\nYou can find summary folder at: {summary_dir}." ) + __all__ = ["run"] diff --git a/neps/runtime.py b/neps/runtime.py index adf5b4998..98e8d56d2 100644 --- a/neps/runtime.py +++ b/neps/runtime.py @@ -4,19 +4,18 @@ import datetime import logging +import math import os import shutil import time -import math from collections.abc import Callable, Iterator, Mapping from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, ClassVar, Literal -from portalocker import portalocker -from pathlib import Path from filelock import FileLock +from portalocker import portalocker from neps.env import ( FS_SYNC_GRACE_BASE, @@ -44,7 +43,7 @@ WorkerSettings, evaluate_trial, ) -from neps.status.status import post_run_csv, _initiate_summary_csv, status +from neps.status.status import _initiate_summary_csv, status from neps.utils.common import gc_disabled if TYPE_CHECKING: @@ -319,19 +318,19 @@ def _check_shared_error_stopping_criterion(self) -> str | Literal[False]: return False - def _check_global_stopping_criterion( + def _check_global_stopping_criterion( # noqa: C901, PLR0912 self, trials: Mapping[str, Trial], ) -> str | Literal[False]: if self.settings.evaluations_to_spend is not None: if self.settings.include_in_progress_evaluations_towards_maximum: - if self.optimizer.space.fidelities: + if hasattr(self.optimizer, "space") and self.optimizer.space.fidelities: count = sum( trial.report.cost for _, trial in trials.items() if trial.report is not None and trial.report.cost is not None ) - for name, fidelity_param in self.optimizer.space.fidelities.items(): + for name, fidelity_param in self.optimizer.space.fidelities.items(): # noqa: B007 count = math.ceil(count / fidelity_param.upper) else: count = sum( @@ -340,7 +339,7 @@ def _check_global_stopping_criterion( if trial.metadata.state not in (Trial.State.PENDING, Trial.State.SUBMITTED) ) - + else: # This indicates they have completed. count = sum(1 for _, trial in trials.items() if trial.report is not None) @@ -352,9 +351,11 @@ def _check_global_stopping_criterion( " To allow more evaluations, increase this value or use a different" " stopping criterion." ) - - if self.settings.fidelities_to_spend is not None: - fidelity_name = list(self.optimizer.space.fidelities.keys())[0] + + if self.settings.fidelities_to_spend is not None and hasattr( + self.optimizer, "space" + ): + fidelity_name = next(iter(self.optimizer.space.fidelities.keys())) count = sum( trial.config[fidelity_name] for _, trial in trials.items() @@ -362,8 +363,8 @@ def _check_global_stopping_criterion( ) if count >= self.settings.fidelities_to_spend: return ( - "The total number of fidelity evaluations has reached the maximum allowed of" - f" `{self.settings.fidelities_to_spend=}`." + "The total number of fidelity evaluations has reached the maximum" + f" allowed of `{self.settings.fidelities_to_spend=}`." " To allow more evaluations, increase this value or use a different" " stopping criterion." ) @@ -545,7 +546,11 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 _trace_lock_path.touch(exist_ok=True) all_best_configs = [] - logger.info("Summary files of evaluations can be found in folder `Summary` in the main directory: %s", main_dir) + logger.info( + "Summary files of evaluations can be found in folder" + "`Summary` in the main directory: %s", + main_dir, + ) _best_score_so_far = float("inf") @@ -680,7 +685,11 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 for _key, callback in _TRIAL_END_CALLBACKS.items(): callback(trial_to_eval) - if report.objective_to_minimize is not None and report.err is None: + if ( + report.objective_to_minimize is not None + and report.err is None + and not isinstance(report.objective_to_minimize, list) + ): new_score = report.objective_to_minimize if new_score < _best_score_so_far: _best_score_so_far = new_score @@ -689,23 +698,28 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 evaluated_trial.id, new_score, ) - + if self.settings.write_summary_to_disk: # Store in memory for later file re-writing - all_best_configs.append({ - "score": new_score, - "trial_id": evaluated_trial.id, - "config": evaluated_trial.config - }) + all_best_configs.append( + { + "score": new_score, + "trial_id": evaluated_trial.id, + "config": evaluated_trial.config, + } + ) # Build trace text and best config text - trace_text = "Best configs and their objectives across evaluations:\n" + "-" * 80 + "\n" + trace_text = ( + "Best configs and their objectives across evaluations:\n" + + "-" * 80 + + "\n" + ) for best in all_best_configs: trace_text += ( f"Objective to minimize: {best['score']}\n" f"Config ID: {best['trial_id']}\n" - f"Config: {best['config']}\n" - + "-" * 80 + "\n" + f"Config: {best['config']}\n" + "-" * 80 + "\n" ) best_config = all_best_configs[-1] # Latest best @@ -718,10 +732,10 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 # Write files from scratch with _trace_lock: - with open(improvement_trace_path, mode='w') as f: + with improvement_trace_path.open(mode="w") as f: f.write(trace_text) - with open(best_config_path, mode='w') as f: + with best_config_path.open(mode="w") as f: f.write(best_config_text) if self.settings.write_summary_to_disk: @@ -888,7 +902,7 @@ def _launch_runtime( # noqa: PLR0913 max_wallclock_time_for_worker_seconds=None, # TODO: User can't specify yet max_evaluation_time_for_worker_seconds=None, # TODO: User can't specify yet max_cost_for_worker=None, # TODO: User can't specify yet - write_summary_to_disk=write_summary_to_disk + write_summary_to_disk=write_summary_to_disk, ) # HACK: Due to nfs file-systems, locking with the default `flock()` is not reliable. From 5fae231d1d9674c8c78fc4b047f32ad6004754ee Mon Sep 17 00:00:00 2001 From: Sohambasu07 Date: Thu, 17 Jul 2025 12:28:03 +0530 Subject: [PATCH 13/27] feat: update primo --- neps/api.py | 1 + neps/optimizers/primo.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/neps/api.py b/neps/api.py index f0bd5124a..b7c37ca1b 100644 --- a/neps/api.py +++ b/neps/api.py @@ -464,6 +464,7 @@ def __call__( "priorband", "moasha", "mo_hyperband", + "primo", } is_multi_fidelity = _optimizer_info["name"] in multi_fidelity_optimizers diff --git a/neps/optimizers/primo.py b/neps/optimizers/primo.py index bfdabc264..fba87bf81 100644 --- a/neps/optimizers/primo.py +++ b/neps/optimizers/primo.py @@ -155,7 +155,7 @@ def __call__( # noqa: C901, PLR0912 # Get the next ID for the sampled configuration if "_" in trial_id: - config_id_str, _ = trial_id.split("_") + config_id_str, _, _ = trial_id.split("_") else: config_id_str = trial_id From 376feeb6401afc28dd3def058a0fc4e635754e6d Mon Sep 17 00:00:00 2001 From: Ema Mekic Date: Thu, 17 Jul 2025 12:33:37 +0200 Subject: [PATCH 14/27] Logging messages changed --- neps/runtime.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neps/runtime.py b/neps/runtime.py index adf5b4998..00ffaf401 100644 --- a/neps/runtime.py +++ b/neps/runtime.py @@ -545,7 +545,7 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 _trace_lock_path.touch(exist_ok=True) all_best_configs = [] - logger.info("Summary files of evaluations can be found in folder `Summary` in the main directory: %s", main_dir) + logger.info("Summary files can be found in the “summary” folder inside the root directory: %s", summary_dir) _best_score_so_far = float("inf") @@ -685,7 +685,7 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 if new_score < _best_score_so_far: _best_score_so_far = new_score logger.info( - "Evaluated trial: %s with objective %s is the new best trial.", + "New best: trial %s with objective %s", evaluated_trial.id, new_score, ) From a4c5f438d511d4768a4869c24575842e4e78cba6 Mon Sep 17 00:00:00 2001 From: Ema Mekic Date: Thu, 17 Jul 2025 16:53:17 +0200 Subject: [PATCH 15/27] Removed info_dict from logging --- neps/runtime.py | 22 ++++++---------------- neps/state/pipeline_eval.py | 3 ++- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/neps/runtime.py b/neps/runtime.py index 00ffaf401..251a0f648 100644 --- a/neps/runtime.py +++ b/neps/runtime.py @@ -325,22 +325,12 @@ def _check_global_stopping_criterion( ) -> str | Literal[False]: if self.settings.evaluations_to_spend is not None: if self.settings.include_in_progress_evaluations_towards_maximum: - if self.optimizer.space.fidelities: - count = sum( - trial.report.cost - for _, trial in trials.items() - if trial.report is not None and trial.report.cost is not None - ) - for name, fidelity_param in self.optimizer.space.fidelities.items(): - count = math.ceil(count / fidelity_param.upper) - else: - count = sum( - 1 - for _, trial in trials.items() - if trial.metadata.state - not in (Trial.State.PENDING, Trial.State.SUBMITTED) - ) - + count = sum( + 1 + for _, trial in trials.items() + if trial.metadata.state + not in (Trial.State.PENDING, Trial.State.SUBMITTED) + ) else: # This indicates they have completed. count = sum(1 for _, trial in trials.items() if trial.report is not None) diff --git a/neps/state/pipeline_eval.py b/neps/state/pipeline_eval.py index 7903fec18..b6a120b15 100644 --- a/neps/state/pipeline_eval.py +++ b/neps/state/pipeline_eval.py @@ -402,7 +402,8 @@ def _eval_trial( else: duration = time.monotonic() - start time_end = time.time() - logger.info(f"Successful evaluation of '{trial.id}': {user_result}.") + filtered_data = {k: v for k, v in user_result.items() if k != 'info_dict'} + logger.info(f"Successful evaluation of '{trial.id}': {filtered_data}.") result = UserResult.parse( user_result, From fb9930b196f2c57fb11ffd5c55dd0b4b619fec4f Mon Sep 17 00:00:00 2001 From: Sohambasu07 Date: Fri, 18 Jul 2025 17:31:17 +0530 Subject: [PATCH 16/27] feat: allow None confidence centers for MO priors --- neps/optimizers/algorithms.py | 2 +- neps/optimizers/mopriors.py | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/neps/optimizers/algorithms.py b/neps/optimizers/algorithms.py index 7f9fab489..a59346d20 100644 --- a/neps/optimizers/algorithms.py +++ b/neps/optimizers/algorithms.py @@ -1173,7 +1173,7 @@ def primo( sample_prior_first: bool | Literal["highest_fidelity"] = False, # noqa: ARG001 eta: int = 3, epsilon: float = 0.25, - prior_centers: Mapping[str, Mapping[str, Any]] | None = None, + prior_centers: Mapping[str, Mapping[str, Any]], mo_selector: Literal["nsga2", "epsnet"] = "epsnet", prior_confidences: Mapping[str, Mapping[str, float]] | None = None, initial_design_size: int | Literal["ndim"] = "ndim", diff --git a/neps/optimizers/mopriors.py b/neps/optimizers/mopriors.py index f78ffdeb4..f5a61bc3d 100644 --- a/neps/optimizers/mopriors.py +++ b/neps/optimizers/mopriors.py @@ -29,7 +29,7 @@ def dists_from_centers_and_confidences( cls, parameters: Mapping[str, Parameter], prior_centers: Mapping[str, Mapping[str, float]], - confidence_values: Mapping[str, Mapping[str, float]], + confidence_values: Mapping[str, Mapping[str, float]] | None = None, ) -> Mapping[str, Prior]: """Creates a mapping of prior distributions from the given centers and confidence values. @@ -41,19 +41,17 @@ def dists_from_centers_and_confidences( A mapping of prior distributions. """ _priors = {} - for key, _prior_center in prior_centers.items(): assert isinstance(_prior_center, dict), ( f"Expected prior center values to be a dict, got {type(_prior_center)}" ) - assert key in confidence_values, ( - f"Expected confidence values to contain {key}, " - f"got {confidence_values.keys()}" - ) + _default_confidence = dict.fromkeys(prior_centers.keys(), 0.25) _priors[key] = Prior.from_parameters( parameters=parameters, center_values=_prior_center, - confidence_values=confidence_values[key], + confidence_values=( + confidence_values[key] if confidence_values else _default_confidence + ), ) return _priors From b50a4627fc7dc8d3bd2b2dbe9401a164a61213b6 Mon Sep 17 00:00:00 2001 From: Sohambasu07 Date: Fri, 18 Jul 2025 21:43:15 +0530 Subject: [PATCH 17/27] fix: disable info_dict logging --- neps/runtime.py | 42 ++++++++++++++----------------------- neps/state/neps_state.py | 2 +- neps/state/pipeline_eval.py | 7 ++++++- 3 files changed, 23 insertions(+), 28 deletions(-) diff --git a/neps/runtime.py b/neps/runtime.py index 98e8d56d2..6b7aa6dd4 100644 --- a/neps/runtime.py +++ b/neps/runtime.py @@ -4,7 +4,6 @@ import datetime import logging -import math import os import shutil import time @@ -318,27 +317,18 @@ def _check_shared_error_stopping_criterion(self) -> str | Literal[False]: return False - def _check_global_stopping_criterion( # noqa: C901, PLR0912 + def _check_global_stopping_criterion( self, trials: Mapping[str, Trial], ) -> str | Literal[False]: if self.settings.evaluations_to_spend is not None: if self.settings.include_in_progress_evaluations_towards_maximum: - if hasattr(self.optimizer, "space") and self.optimizer.space.fidelities: - count = sum( - trial.report.cost - for _, trial in trials.items() - if trial.report is not None and trial.report.cost is not None - ) - for name, fidelity_param in self.optimizer.space.fidelities.items(): # noqa: B007 - count = math.ceil(count / fidelity_param.upper) - else: - count = sum( - 1 - for _, trial in trials.items() - if trial.metadata.state - not in (Trial.State.PENDING, Trial.State.SUBMITTED) - ) + count = sum( + 1 + for _, trial in trials.items() + if trial.metadata.state + not in (Trial.State.PENDING, Trial.State.SUBMITTED) + ) else: # This indicates they have completed. @@ -547,11 +537,10 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 all_best_configs = [] logger.info( - "Summary files of evaluations can be found in folder" - "`Summary` in the main directory: %s", - main_dir, + "Summary files can be found in the “summary” folder inside" + "the root directory: %s", + summary_dir, ) - _best_score_so_far = float("inf") optimizer_name = self.state._optimizer_info["name"] @@ -693,11 +682,12 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 new_score = report.objective_to_minimize if new_score < _best_score_so_far: _best_score_so_far = new_score - logger.info( - "Evaluated trial: %s with objective %s is the new best trial.", - evaluated_trial.id, - new_score, - ) + # This was a bug + # logger.info( + # "New best: trial %s with objective %s", # noqa: ERA001 + # evaluated_trial.id, + # new_score, + # ) if self.settings.write_summary_to_disk: # Store in memory for later file re-writing diff --git a/neps/state/neps_state.py b/neps/state/neps_state.py index 149cfe1a5..da29201cd 100644 --- a/neps/state/neps_state.py +++ b/neps/state/neps_state.py @@ -384,7 +384,7 @@ def _sample_trial( id_str = sampled_config.id config_name = f"{sampled_config.id}" - parts = id_str.split('_rung_') + parts = id_str.split("_rung_") if len(parts) == 2 and all(p.isdigit() for p in parts): config_id, rung_id = map(int, parts) config_name = f"{config_id}_rung_{rung_id}" diff --git a/neps/state/pipeline_eval.py b/neps/state/pipeline_eval.py index 7903fec18..086de800e 100644 --- a/neps/state/pipeline_eval.py +++ b/neps/state/pipeline_eval.py @@ -402,7 +402,12 @@ def _eval_trial( else: duration = time.monotonic() - start time_end = time.time() - logger.info(f"Successful evaluation of '{trial.id}': {user_result}.") + match user_result: + case dict(): + filtered_data = {k: v for k, v in user_result.items() if k != "info_dict"} + logger.info(f"Successful evaluation of '{trial.id}': {filtered_data}.") + case _: # TODO: Revisit this and check all possible cases + logger.info(f"Successful evaluation of '{trial.id}': {user_result}.") result = UserResult.parse( user_result, From 0d08ad4850deed856aad1fbea20bba353c430ca5 Mon Sep 17 00:00:00 2001 From: Sohambasu07 Date: Fri, 18 Jul 2025 21:43:32 +0530 Subject: [PATCH 18/27] fix: disable info_dict logging --- neps/runtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neps/runtime.py b/neps/runtime.py index 6b7aa6dd4..63b37614c 100644 --- a/neps/runtime.py +++ b/neps/runtime.py @@ -684,7 +684,7 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 _best_score_so_far = new_score # This was a bug # logger.info( - # "New best: trial %s with objective %s", # noqa: ERA001 + # "New best: trial %s with objective %s", # evaluated_trial.id, # new_score, # ) From 4da5e8d3a15664d6087c9b714b2223fcc5833d5f Mon Sep 17 00:00:00 2001 From: Ema Mekic Date: Thu, 24 Jul 2025 10:50:22 +0200 Subject: [PATCH 19/27] Fixed trace error in case of multiple runs --- neps/runtime.py | 73 +++++++++++++++++++++++++++++++++------- neps/state/neps_state.py | 6 ++++ 2 files changed, 66 insertions(+), 13 deletions(-) diff --git a/neps/runtime.py b/neps/runtime.py index 251a0f648..6145a3bca 100644 --- a/neps/runtime.py +++ b/neps/runtime.py @@ -10,9 +10,9 @@ import math from collections.abc import Callable, Iterator, Mapping from contextlib import contextmanager -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path -from typing import TYPE_CHECKING, ClassVar, Literal +from typing import TYPE_CHECKING, ClassVar, Literal, List, Any, Dict from portalocker import portalocker from pathlib import Path @@ -534,10 +534,15 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 _trace_lock_path = Path(str(_trace_lock.lock_file)) _trace_lock_path.touch(exist_ok=True) - all_best_configs = [] logger.info("Summary files can be found in the “summary” folder inside the root directory: %s", summary_dir) + previous_trials = self.state.lock_and_read_trials() + if len(previous_trials): + load_incumbent_trace(previous_trials, _trace_lock, self.state, self.settings, improvement_trace_path, best_config_path) + _best_score_so_far = float("inf") + if self.state.new_score != _best_score_so_far: + _best_score_so_far = self.state.new_score optimizer_name = self.state._optimizer_info["name"] logger.info("Using optimizer: %s", optimizer_name) @@ -669,28 +674,28 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 # This is mostly for `tblogger` for _key, callback in _TRIAL_END_CALLBACKS.items(): callback(trial_to_eval) - + if report.objective_to_minimize is not None and report.err is None: - new_score = report.objective_to_minimize - if new_score < _best_score_so_far: - _best_score_so_far = new_score + self.state.new_score = report.objective_to_minimize + if self.state.new_score < _best_score_so_far: + _best_score_so_far = self.state.new_score logger.info( "New best: trial %s with objective %s", evaluated_trial.id, - new_score, + self.state.new_score, ) if self.settings.write_summary_to_disk: # Store in memory for later file re-writing - all_best_configs.append({ - "score": new_score, + self.state.all_best_configs.append({ + "score": self.state.new_score, "trial_id": evaluated_trial.id, "config": evaluated_trial.config }) # Build trace text and best config text trace_text = "Best configs and their objectives across evaluations:\n" + "-" * 80 + "\n" - for best in all_best_configs: + for best in self.state.all_best_configs: trace_text += ( f"Objective to minimize: {best['score']}\n" f"Config ID: {best['trial_id']}\n" @@ -698,7 +703,7 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 + "-" * 80 + "\n" ) - best_config = all_best_configs[-1] # Latest best + best_config = self.state.all_best_configs[-1] # Latest best best_config_text = ( f"# Best config:" f"\n\n Config ID: {best_config['trial_id']}" @@ -710,7 +715,6 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 with _trace_lock: with open(improvement_trace_path, mode='w') as f: f.write(trace_text) - with open(best_config_path, mode='w') as f: f.write(best_config_text) @@ -727,6 +731,49 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 "Learning Curve %s: %s", evaluated_trial.id, report.learning_curve ) +def load_incumbent_trace( + previous_trials: dict[str, Trial], + _trace_lock: FileLock, + state: NePSState, + settings: WorkerSettings, + improvement_trace_path: Path, + best_config_path: Path +) -> None: + _best_score_so_far = float("inf") + + for evaluated_trial in previous_trials.values(): + state.new_score = evaluated_trial.report.objective_to_minimize + if state.new_score < _best_score_so_far: + _best_score_so_far = state.new_score + state.all_best_configs.append({ + "score": state.new_score, + "trial_id": evaluated_trial.metadata.id, + "config": evaluated_trial.config + }) + + trace_text = "Best configs and their objectives across evaluations:\n" + "-" * 80 + "\n" + for best in state.all_best_configs: + trace_text += ( + f"Objective to minimize: {best['score']}\n" + f"Config ID: {best['trial_id']}\n" + f"Config: {best['config']}\n" + + "-" * 80 + "\n" + ) + + best_config = state.all_best_configs[-1] # Latest best + best_config_text = ( + f"# Best config:" + f"\n\n Config ID: {best_config['trial_id']}" + f"\n Objective to minimize: {best_config['score']}" + f"\n Config: {best_config['config']}" + ) + + with _trace_lock: + with open(improvement_trace_path, mode='w') as f: + f.write(trace_text) + with open(best_config_path, mode='w') as f: + f.write(best_config_text) + def _launch_ddp_runtime( *, diff --git a/neps/state/neps_state.py b/neps/state/neps_state.py index 149cfe1a5..fb6678106 100644 --- a/neps/state/neps_state.py +++ b/neps/state/neps_state.py @@ -251,6 +251,12 @@ class NePSState: _shared_errors_path: Path = field(repr=False) _shared_errors: ErrDump = field(repr=False) + new_score: float = float("inf") + """Tracking of the new incumbent""" + + all_best_configs: list = field(default_factory=list) + """Trajectory to the newest incbumbent""" + def lock_and_read_trials(self) -> dict[str, Trial]: """Acquire the state lock and read the trials.""" with self._trial_lock.lock(): From d1c0af04d2788a17525ebf866b1e6587fd226702 Mon Sep 17 00:00:00 2001 From: Ema Mekic Date: Thu, 24 Jul 2025 21:48:03 +0200 Subject: [PATCH 20/27] Fixed tests --- neps/runtime.py | 39 +++++++++++-------- neps/state/pipeline_eval.py | 8 +++- .../test_default_report_values.py | 3 ++ .../test_error_handling_strategies.py | 3 ++ tests/test_runtime/test_stopping_criterion.py | 8 ++++ 5 files changed, 43 insertions(+), 18 deletions(-) diff --git a/neps/runtime.py b/neps/runtime.py index 6145a3bca..7afdeb7a0 100644 --- a/neps/runtime.py +++ b/neps/runtime.py @@ -541,7 +541,7 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 load_incumbent_trace(previous_trials, _trace_lock, self.state, self.settings, improvement_trace_path, best_config_path) _best_score_so_far = float("inf") - if self.state.new_score != _best_score_so_far: + if self.state.new_score is not None and self.state.new_score != _best_score_so_far: _best_score_so_far = self.state.new_score optimizer_name = self.state._optimizer_info["name"] @@ -742,14 +742,15 @@ def load_incumbent_trace( _best_score_so_far = float("inf") for evaluated_trial in previous_trials.values(): - state.new_score = evaluated_trial.report.objective_to_minimize - if state.new_score < _best_score_so_far: - _best_score_so_far = state.new_score - state.all_best_configs.append({ - "score": state.new_score, - "trial_id": evaluated_trial.metadata.id, - "config": evaluated_trial.config - }) + if evaluated_trial.report is not None and evaluated_trial.report.objective_to_minimize is not None: + state.new_score = evaluated_trial.report.objective_to_minimize + if state.new_score is not None and state.new_score < _best_score_so_far: + _best_score_so_far = state.new_score + state.all_best_configs.append({ + "score": state.new_score, + "trial_id": evaluated_trial.metadata.id, + "config": evaluated_trial.config + }) trace_text = "Best configs and their objectives across evaluations:\n" + "-" * 80 + "\n" for best in state.all_best_configs: @@ -760,13 +761,19 @@ def load_incumbent_trace( + "-" * 80 + "\n" ) - best_config = state.all_best_configs[-1] # Latest best - best_config_text = ( - f"# Best config:" - f"\n\n Config ID: {best_config['trial_id']}" - f"\n Objective to minimize: {best_config['score']}" - f"\n Config: {best_config['config']}" - ) + best_config_text = "" + if state.all_best_configs: + best_config = state.all_best_configs[-1] + best_config_text = ( + f"# Best config:" + f"\n\n Config ID: {best_config['trial_id']}" + f"\n Objective to minimize: {best_config['score']}" + f"\n Config: {best_config['config']}" + ) + else: + best_config = None + + with _trace_lock: with open(improvement_trace_path, mode='w') as f: diff --git a/neps/state/pipeline_eval.py b/neps/state/pipeline_eval.py index b6a120b15..086de800e 100644 --- a/neps/state/pipeline_eval.py +++ b/neps/state/pipeline_eval.py @@ -402,8 +402,12 @@ def _eval_trial( else: duration = time.monotonic() - start time_end = time.time() - filtered_data = {k: v for k, v in user_result.items() if k != 'info_dict'} - logger.info(f"Successful evaluation of '{trial.id}': {filtered_data}.") + match user_result: + case dict(): + filtered_data = {k: v for k, v in user_result.items() if k != "info_dict"} + logger.info(f"Successful evaluation of '{trial.id}': {filtered_data}.") + case _: # TODO: Revisit this and check all possible cases + logger.info(f"Successful evaluation of '{trial.id}': {user_result}.") result = UserResult.parse( user_result, diff --git a/tests/test_runtime/test_default_report_values.py b/tests/test_runtime/test_default_report_values.py index 0fd43e5e4..45ad5e5cc 100644 --- a/tests/test_runtime/test_default_report_values.py +++ b/tests/test_runtime/test_default_report_values.py @@ -44,6 +44,7 @@ def test_default_values_on_error( evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=1, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -95,6 +96,7 @@ def test_default_values_on_not_specified( evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=1, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -144,6 +146,7 @@ def test_default_value_objective_to_minimize_curve_take_objective_to_minimize_va evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=1, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, diff --git a/tests/test_runtime/test_error_handling_strategies.py b/tests/test_runtime/test_error_handling_strategies.py index aeaeba0d9..0549f87b5 100644 --- a/tests/test_runtime/test_error_handling_strategies.py +++ b/tests/test_runtime/test_error_handling_strategies.py @@ -51,6 +51,7 @@ def test_worker_raises_when_error_in_self( evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=1, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -91,6 +92,7 @@ def test_worker_raises_when_error_in_other_worker(neps_state: NePSState) -> None evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=1, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -151,6 +153,7 @@ def test_worker_does_not_raise_when_error_in_other_worker( evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=1, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, diff --git a/tests/test_runtime/test_stopping_criterion.py b/tests/test_runtime/test_stopping_criterion.py index 40212cd8f..67f15365f 100644 --- a/tests/test_runtime/test_stopping_criterion.py +++ b/tests/test_runtime/test_stopping_criterion.py @@ -43,6 +43,7 @@ def test_max_evaluations_total_stopping_criterion( evaluations_to_spend=3, # <- Highlight include_in_progress_evaluations_towards_maximum=False, cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -95,6 +96,7 @@ def test_worker_evaluations_total_stopping_criterion( evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=2, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -156,6 +158,7 @@ def test_include_in_progress_evaluations_towards_maximum_with_work_eval_count( evaluations_to_spend=2, # <- Highlight, only 2 maximum evaluations allowed include_in_progress_evaluations_towards_maximum=True, # <- inprogress trial cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -210,6 +213,7 @@ def test_max_cost_total(neps_state: NePSState) -> None: evaluations_to_spend=10, # Safety incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, cost_to_spend=2, # <- Highlight, only 2 maximum evaluations allowed + fidelities_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -258,6 +262,7 @@ def test_worker_cost_total(neps_state: NePSState) -> None: evaluations_to_spend=10, # Safety incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -314,6 +319,7 @@ def test_worker_wallclock_time(neps_state: NePSState) -> None: evaluations_to_spend=1000, # Incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=1, # <- highlight, 1 second @@ -369,6 +375,7 @@ def test_max_worker_evaluation_time(neps_state: NePSState) -> None: evaluations_to_spend=10, # Safety incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -425,6 +432,7 @@ def test_max_evaluation_time_global(neps_state: NePSState) -> None: evaluations_to_spend=10, # Safety incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=0.5, # <- Highlight max_wallclock_time_for_worker_seconds=None, From f21aa694bc6f983f051a07c264d42ef4fd90e651 Mon Sep 17 00:00:00 2001 From: Ema Mekic Date: Thu, 24 Jul 2025 21:56:09 +0200 Subject: [PATCH 21/27] Example fixed --- neps_examples/basic_usage/hyperparameters.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/neps_examples/basic_usage/hyperparameters.py b/neps_examples/basic_usage/hyperparameters.py index 023de5d4f..6b1c6a5d7 100644 --- a/neps_examples/basic_usage/hyperparameters.py +++ b/neps_examples/basic_usage/hyperparameters.py @@ -12,7 +12,7 @@ def evaluate_pipeline(float1, float2, categorical, integer1, integer2): np.sum([float1, float2, int(categorical), integer1, integer2]) ) return objective_to_minimize - + pipeline_space = dict( float1=neps.Float(lower=0, upper=1), @@ -27,6 +27,5 @@ def evaluate_pipeline(float1, float2, categorical, integer1, integer2): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/hyperparameters_example", - post_run_summary=True, evaluations_to_spend=30, ) From fb9b146469b62a4687373a687626ada419571dd8 Mon Sep 17 00:00:00 2001 From: Ema Mekic Date: Fri, 25 Jul 2025 17:07:57 +0200 Subject: [PATCH 22/27] Numpy version update --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index afd6f5ecd..f96d32a19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ classifiers = [ requires-python = ">=3.10,<3.14" dependencies = [ - "numpy>=2.0", + "numpy>=1.24,<2.0", "pandas>=2.0,<3.0", "networkx>=2.6.3,<3.0", "scipy>=1.13.1", From 4c01c025f949566a5a3aa73b1848783033ac1a63 Mon Sep 17 00:00:00 2001 From: Sohambasu07 Date: Fri, 1 Aug 2025 00:44:56 +0530 Subject: [PATCH 23/27] feat: Modify tests for MOMF opts --- tests/test_state/test_neps_state.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/test_state/test_neps_state.py b/tests/test_state/test_neps_state.py index 0cb3c823e..1fa0ae644 100644 --- a/tests/test_state/test_neps_state.py +++ b/tests/test_state/test_neps_state.py @@ -96,8 +96,6 @@ def case_search_space_fid_with_prior() -> SearchSpace: "async_hb", "ifbo", "priorband", - "moasha", - "mo_hyperband", ] NO_DEFAULT_FIDELITY_SUPPORT = [ "random_search", @@ -114,14 +112,17 @@ def case_search_space_fid_with_prior() -> SearchSpace: "hyperband", "async_hb", "random_search", - "moasha", - "mo_hyperband", ] REQUIRES_PRIOR = [ "pibo", "priorband", ] +REQUIRES_FIDELITY_MO = [ + "moasha", + "mo_hyperband", +] + @fixture @parametrize("key", list(PredefinedOptimizers.keys())) @@ -148,6 +149,9 @@ def optimizer_and_key_and_search_space( ): pytest.xfail(f"{key} requires a prior") + if key in REQUIRES_FIDELITY_MO and search_space.fidelity is None: + pytest.xfail(f"Multi-objective optimizer {key} requires a fidelity parameter") + kwargs: dict[str, Any] = {} opt, _ = load_optimizer((key, kwargs), search_space) # type: ignore return opt, key, search_space @@ -226,4 +230,7 @@ def test_optimizers_work_roughly( for _ in range(20): trial = ask_and_tell.ask() - ask_and_tell.tell(trial, 1.0) + if key in REQUIRES_FIDELITY_MO: + ask_and_tell.tell(trial, [1.0, 2.0]) + else: + ask_and_tell.tell(trial, 1.0) From 285b4400411c7576f1f5e5d43769a9134876d154 Mon Sep 17 00:00:00 2001 From: Sohambasu07 Date: Fri, 1 Aug 2025 00:48:59 +0530 Subject: [PATCH 24/27] feat: fix ruff-format in status.py --- neps/status/status.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/neps/status/status.py b/neps/status/status.py index 0b51cff6a..32a741e3c 100644 --- a/neps/status/status.py +++ b/neps/status/status.py @@ -141,6 +141,7 @@ def from_directory(cls, root_directory: str | Path) -> Summary: # make decisions based on the state try: from neps.runtime import get_workers_neps_state + shared_state = get_workers_neps_state() except RuntimeError: shared_state = NePSState.create_or_load(root_directory, load_only=True) @@ -250,7 +251,7 @@ def trajectory_of_improvements( for trial_id, row in df.iterrows(): if "objective_to_minimize" not in row or pd.isna(row["objective_to_minimize"]): - continue + continue score = row["objective_to_minimize"] if score < best_score: @@ -271,20 +272,19 @@ def trajectory_of_improvements( trace_text += ( f"Objective to minimize: {best['score']}\n" f"Config ID: {best['trial_id']}\n" - f"Config: {best['config']}\n" - + "-" * 80 + "\n" + f"Config: {best['config']}\n" + "-" * 80 + "\n" ) summary_dir = root_directory / "summary" summary_dir.mkdir(parents=True, exist_ok=True) output_path = summary_dir / "best_config_trajectory.txt" - with open(output_path, "w") as f: + with output_path.open("w") as f: f.write(trace_text) if all_best_configs: final_best = all_best_configs[-1] best_path = summary_dir / "best_config.txt" - with open(best_path, "w") as f: + with best_path.open("w") as f: f.write( f"Objective to minimize: {final_best['score']}\n" f"Config ID: {final_best['trial_id']}\n" @@ -336,4 +336,3 @@ def post_run_csv(root_directory: str | Path) -> tuple[Path, Path]: short.to_frame().to_csv(short_path) return full_df_path, short_path - From 9d1f45172de81482a9360f0c558a34ef47710678 Mon Sep 17 00:00:00 2001 From: Sohambasu07 Date: Fri, 1 Aug 2025 00:54:15 +0530 Subject: [PATCH 25/27] feat: fix more ruff-formatting --- neps/plot/read_results.py | 43 ++++++++++++++++++++++----------------- neps/state/settings.py | 2 +- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/neps/plot/read_results.py b/neps/plot/read_results.py index 42107842f..c29cffbff 100644 --- a/neps/plot/read_results.py +++ b/neps/plot/read_results.py @@ -3,18 +3,20 @@ from __future__ import annotations from pathlib import Path -from neps.state.trial import State -import neps import numpy as np +import neps +from neps.state.trial import State + + def process_seed( *, path: str | Path, seed: str | int | None, - key_to_extract: str | None = None, # noqa: ARG001 - consider_continuations: bool = False, # noqa: ARG001 - n_workers: int = 1, # noqa: ARG001 + key_to_extract: str | None = None, + consider_continuations: bool = False, + n_workers: int = 1, ) -> tuple[list[float], list[float], float]: """Reads and processes data per seed.""" path = Path(path) @@ -24,13 +26,13 @@ def process_seed( _fulldf, _summary = neps.status(path, print_summary=False) if _fulldf.empty: raise ValueError(f"No trials found in {path}") - + _fulldf = _fulldf.sort_values("time_sampled") def get_cost(idx: str | int) -> float: row = _fulldf.loc[idx] if key_to_extract and key_to_extract in row: - return float(row[key_to_extract]) + return float(row[key_to_extract]) return 1.0 losses = [] @@ -46,21 +48,24 @@ def get_cost(idx: str | int) -> float: cost = get_cost(config_id) - if consider_continuations: - if n_workers == 1 and "previous_config_id" in config_result["metadata"]: - # calculates continuation costs for MF algorithms NOTE: assumes that - # all recorded evaluations are black-box evaluations where - # continuations or freeze-thaw was not accounted for during optimization - previous_id = config_result["metadata"]["previous_config_id"] - - if previous_id in _fulldf.index and key_to_extract: - cost -= get_cost(config_id) - else: - cost = float(config_result["time_end"] - global_start) + if ( + consider_continuations + and n_workers == 1 + and "previous_config_id" in config_result["metadata"] + ): + # calculates continuation costs for MF algorithms NOTE: assumes that + # all recorded evaluations are black-box evaluations where + # continuations or freeze-thaw was not accounted for during optimization + previous_id = config_result["metadata"]["previous_config_id"] + + if previous_id in _fulldf.index and key_to_extract: + cost -= get_cost(config_id) + else: + cost = float(config_result["time_end"] - global_start) loss = float(config_result["objective_to_minimize"]) losses.append(loss) costs.append(cost) max_cost = max(max_cost, cost) - return list(np.minimum.accumulate(losses)), costs, max_cost \ No newline at end of file + return list(np.minimum.accumulate(losses)), costs, max_cost diff --git a/neps/state/settings.py b/neps/state/settings.py index be78e1e4b..aefec74f6 100644 --- a/neps/state/settings.py +++ b/neps/state/settings.py @@ -188,7 +188,7 @@ class WorkerSettings: If `None`, there is no limit and this worker will continue to evaluate indefinitely or until another stopping criterion is met. """ - + write_summary_to_disk: bool = True """If True, creates a csv and txt files after each worker is done, holding summary information about the configs and results. From 0e39bbe074a2beb6d4419f8ef9a9faf302629177 Mon Sep 17 00:00:00 2001 From: Sohambasu07 Date: Fri, 1 Aug 2025 01:00:36 +0530 Subject: [PATCH 26/27] fix: skip tests for PriMO for now --- tests/test_state/test_neps_state.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_state/test_neps_state.py b/tests/test_state/test_neps_state.py index 1fa0ae644..bf7512fae 100644 --- a/tests/test_state/test_neps_state.py +++ b/tests/test_state/test_neps_state.py @@ -123,6 +123,10 @@ def case_search_space_fid_with_prior() -> SearchSpace: "mo_hyperband", ] +REQUIRES_MO_PRIOR = [ + "primo", +] + @fixture @parametrize("key", list(PredefinedOptimizers.keys())) @@ -152,6 +156,9 @@ def optimizer_and_key_and_search_space( if key in REQUIRES_FIDELITY_MO and search_space.fidelity is None: pytest.xfail(f"Multi-objective optimizer {key} requires a fidelity parameter") + if key in REQUIRES_MO_PRIOR: + pytest.xfail("No tests defined for PriMO yet") + kwargs: dict[str, Any] = {} opt, _ = load_optimizer((key, kwargs), search_space) # type: ignore return opt, key, search_space From 7d983fe5f6c50431db72745811d3d243ad2baaa6 Mon Sep 17 00:00:00 2001 From: Sohambasu07 Date: Tue, 5 Aug 2025 17:48:46 +0530 Subject: [PATCH 27/27] numpy>=2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f96d32a19..afd6f5ecd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ classifiers = [ requires-python = ">=3.10,<3.14" dependencies = [ - "numpy>=1.24,<2.0", + "numpy>=2.0", "pandas>=2.0,<3.0", "networkx>=2.6.3,<3.0", "scipy>=1.13.1",