diff --git a/.trace.lock b/.trace.lock new file mode 100644 index 000000000..e69de29bb diff --git a/README.md b/README.md index 4e549326f..5356de78f 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ neps.run( evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="path/to/save/results", # Replace with the actual path. - max_evaluations_total=100, + evaluations_to_spend=100, ) ``` diff --git a/docs/index.md b/docs/index.md index d6e618e19..e9721ecd7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -87,7 +87,7 @@ neps.run( evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="path/to/save/results", # Replace with the actual path. - max_evaluations_total=100, + evaluations_to_spend=100, ) ``` diff --git a/docs/reference/analyse.md b/docs/reference/analyse.md index 3fd8f694b..2edb4ed4b 100644 --- a/docs/reference/analyse.md +++ b/docs/reference/analyse.md @@ -40,10 +40,10 @@ Currently, this creates one plot that shows the best error value across the numb ## What's on disk? In the root directory, NePS maintains several files at all times that are human readable and can be useful -If you pass the `post_run_summary=` argument to [`neps.run()`][neps.api.run], -NePS will also generate a summary CSV file for you. +If you pass the `write_summary_to_disk=` argument to [`neps.run()`][neps.api.run], +NePS will generate a summary CSV and TXT files for you. -=== "`neps.run(..., post_run_summary=True)`" +=== "`neps.run(..., write_summary_to_disk=True)`" ``` ROOT_DIRECTORY @@ -54,13 +54,15 @@ NePS will also generate a summary CSV file for you. │ └── report.yaml ├── summary │ ├── full.csv - │ └── short.csv + │ ├── short.csv + │ ├── best_config_trajectory.txt + │ └── best_config.txt ├── optimizer_info.yaml └── optimizer_state.pkl ``` -=== "`neps.run(..., post_run_summary=False)`" +=== "`neps.run(..., write_summary_to_disk=False)`" ``` ROOT_DIRECTORY @@ -77,8 +79,8 @@ NePS will also generate a summary CSV file for you. The `full.csv` contains all configuration details in CSV format. Details include configuration hyperparameters and any returned result and cost from the `evaluate_pipeline` function. -The `run_status.csv` provides general run details, such as the number of failed and successful configurations, -and the best configuration with its corresponding objective value. +The `best_config_trajectory.txt` contains logging of the incumbent trajectory. +The `best_config.txt` records current incumbent. # TensorBoard Integration diff --git a/docs/reference/evaluate_pipeline.md b/docs/reference/evaluate_pipeline.md index f23b6d663..b26142f98 100644 --- a/docs/reference/evaluate_pipeline.md +++ b/docs/reference/evaluate_pipeline.md @@ -63,12 +63,12 @@ def evaluate_pipeline( #### Cost -Along with the return of the `loss`, the `evaluate_pipeline=` function would optionally need to return a `cost` in certain cases. Specifically when the `max_cost_total` parameter is being utilized in the `neps.run` function. +Along with the return of the `loss`, the `evaluate_pipeline=` function would optionally need to return a `cost` in certain cases. Specifically when the `cost_to_spend` parameter is being utilized in the `neps.run` function. !!! note - `max_cost_total` sums the cost from all returned configuration results and checks whether the maximum allowed cost has been reached (if so, the search will come to an end). + `cost_to_spend` sums the cost from all returned configuration results and checks whether the maximum allowed cost has been reached (if so, the search will come to an end). ```python import neps @@ -97,7 +97,7 @@ if __name__ == "__main__": evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, # Assuming the pipeline space is defined root_directory="results/bo", - max_cost_total=10, + cost_to_spend=10, optimizer="bayesian_optimization", ) ``` diff --git a/docs/reference/neps_run.md b/docs/reference/neps_run.md index 5e023253f..356f27c47 100644 --- a/docs/reference/neps_run.md +++ b/docs/reference/neps_run.md @@ -45,9 +45,9 @@ See the following for more: * What goes in and what goes out of [`evaluate_pipeline()`](../reference/evaluate_pipeline.md)? ## Budget, how long to run? -To define a budget, provide `max_evaluations_total=` to [`neps.run()`][neps.api.run], +To define a budget, provide `evaluations_to_spend=` to [`neps.run()`][neps.api.run], to specify the total number of evaluations to conduct before halting the optimization process, -or `max_cost_total=` to specify a cost threshold for your own custom cost metric, such as time, energy, or monetary, as returned by each evaluation of the pipeline . +or `cost_to_spend=` to specify a cost threshold for your own custom cost metric, such as time, energy, or monetary, as returned by each evaluation of the pipeline . ```python @@ -60,8 +60,8 @@ def evaluate_pipeline(learning_rate: float, epochs: int) -> float: return {"objective_function_to_minimize": loss, "cost": duration} neps.run( - max_evaluations_total=10, # (1)! - max_cost_total=1000, # (2)! + evaluations_to_spend=10, # (1)! + cost_to_spend=1000, # (2)! ) ``` @@ -87,7 +87,7 @@ Please refer to Python's [logging documentation](https://docs.python.org/3/libra ## Continuing Runs To continue a run, all you need to do is provide the same `root_directory=` to [`neps.run()`][neps.api.run] as before, -with an increased `max_evaluations_total=` or `max_cost_total=`. +with an increased `evaluations_to_spend=` or `cost_to_spend=`. ```python def run(learning_rate: float, epochs: int) -> float: @@ -100,7 +100,7 @@ def run(learning_rate: float, epochs: int) -> float: neps.run( # Increase the total number of trials from 10 as set previously to 50 - max_evaluations_total=50, + evaluations_to_spend=50, ) ``` @@ -108,13 +108,13 @@ If the run previously stopped due to reaching a budget and you specify the same ## Overwriting a Run -To overwrite a run, simply provide the same `root_directory=` to [`neps.run()`][neps.api.run] as before, with the `overwrite_working_directory=True` argument. +To overwrite a run, simply provide the same `root_directory=` to [`neps.run()`][neps.api.run] as before, with the `overwrite_root_directory=True` argument. ```python neps.run( ..., root_directory="path/to/previous_result_dir", - overwrite_working_directory=True, + overwrite_root_directory=True, ) ``` @@ -125,9 +125,6 @@ neps.run( ## Getting the results The results of the optimization process are stored in the `root_directory=` provided to [`neps.run()`][neps.api.run]. -To obtain a summary of the optimization process, you can enable the -`post_run_summary=True` argument in [`neps.run()`][neps.api.run], -while will generate a summary csv after the run has finished. === "Result Directory" @@ -143,9 +140,11 @@ while will generate a summary csv after the run has finished. │ └── config_2 │ ├── config.yaml │ └── metadata.json - ├── summary # Only if post_run_summary=True + ├── summary │ ├── full.csv │ └── short.csv + │ ├── best_config_trajectory.txt + │ └── best_config.txt ├── optimizer_info.yaml # The optimizer's configuration └── optimizer_state.pkl # The optimizer's state, shared between workers ``` @@ -153,7 +152,7 @@ while will generate a summary csv after the run has finished. === "python" ```python - neps.run(..., post_run_summary=True) + neps.run(..., write_summary_to_disk=True) ``` To capture the results of the optimization process, you can use tensorbaord logging with various utilities to integrate @@ -174,20 +173,20 @@ Any new workers that come online will automatically pick up work and work togeth evaluate_pipeline=..., pipeline_space=..., root_directory="some/path", - max_evaluations_total=100, + evaluations_to_spend=100, max_evaluations_per_run=10, # (1)! continue_until_max_evaluation_completed=True, # (2)! - overwrite_working_directory=False, #!!! + overwrite_root_directory=False, #!!! ) ``` 1. Limits the number of evaluations for this specific call of [`neps.run()`][neps.api.run]. - 2. Evaluations in-progress count towards max_evaluations_total, halting new ones when this limit is reached. - Setting this to `True` enables continuous sampling of new evaluations until the total of completed ones meets max_evaluations_total, optimizing resource use in time-sensitive scenarios. + 2. Evaluations in-progress count towards evaluations_to_spend, halting new ones when this limit is reached. + Setting this to `True` enables continuous sampling of new evaluations until the total of completed ones meets evaluations_to_spend, optimizing resource use in time-sensitive scenarios. !!! warning - Ensure `overwrite_working_directory=False` to prevent newly spawned workers from deleting the shared directory! + Ensure `overwrite_root_directory=False` to prevent newly spawned workers from deleting the shared directory! === "Shell" @@ -227,7 +226,7 @@ neps.run( !!! note - Any runs that error will still count towards the total `max_evaluations_total` or `max_evaluations_per_run`. + Any runs that error will still count towards the total `evaluations_to_spend` or `max_evaluations_per_run`. ### Re-running Failed Configurations diff --git a/docs/reference/optimizers.md b/docs/reference/optimizers.md index 3c6502624..6edef593a 100644 --- a/docs/reference/optimizers.md +++ b/docs/reference/optimizers.md @@ -72,7 +72,7 @@ neps.run( evaluate_pipeline=run_function, pipeline_space=pipeline_space, root_directory="results/", - max_evaluations_total=25, + evaluations_to_spend=25, # no optimizer specified ) ``` @@ -87,7 +87,7 @@ neps.run( evaluate_pipeline=run_function, pipeline_space=pipeline_space, root_directory="results/", - max_evaluations_total=25, + evaluations_to_spend=25, # optimizer specified, along with an argument optimizer=neps.algorithms.bayesian_optimization, # or as string: "bayesian_optimization" ) @@ -104,7 +104,7 @@ neps.run( evaluate_pipeline=run_function, pipeline_space=pipeline_space, root_directory="results/", - max_evaluations_total=25, + evaluations_to_spend=25, optimizer=("bayesian_optimization", {"initial_design_size": 5}) ) ``` @@ -137,7 +137,7 @@ neps.run( evaluate_pipeline=run_function, pipeline_space=pipeline_space, root_directory="results/", - max_evaluations_total=25, + evaluations_to_spend=25, optimizer=MyOptimizer, ) ``` diff --git a/neps/api.py b/neps/api.py index 77c8ebcf2..b7c37ca1b 100644 --- a/neps/api.py +++ b/neps/api.py @@ -11,7 +11,7 @@ from neps.optimizers import AskFunction, OptimizerChoice, load_optimizer from neps.runtime import _launch_runtime from neps.space.parsing import convert_to_space -from neps.status.status import post_run_csv +from neps.status.status import post_run_csv, trajectory_of_improvements from neps.utils.common import dynamic_load_object if TYPE_CHECKING: @@ -24,7 +24,7 @@ logger = logging.getLogger(__name__) -def run( # noqa: PLR0913 +def run( # noqa: C901, D417, PLR0913 evaluate_pipeline: Callable[..., EvaluatePipelineReturn] | str, pipeline_space: ( Mapping[str, dict | str | int | float | Parameter] @@ -33,12 +33,15 @@ def run( # noqa: PLR0913 ), *, root_directory: str | Path = "neps_results", - overwrite_working_directory: bool = False, - post_run_summary: bool = True, + overwrite_root_directory: bool = False, + evaluations_to_spend: int | None = None, + write_summary_to_disk: bool = True, max_evaluations_total: int | None = None, max_evaluations_per_run: int | None = None, continue_until_max_evaluation_completed: bool = False, + cost_to_spend: int | float | None = None, max_cost_total: int | float | None = None, + fidelities_to_spend: int | None = None, ignore_errors: bool = False, objective_value_on_error: float | None = None, cost_value_on_error: float | None = None, @@ -96,7 +99,7 @@ def evaluate_pipeline(some_parameter: float) -> float: ) }, root_directory="usage_example", - max_evaluations_total=5, + evaluations_to_spend=5, ) ``` @@ -189,26 +192,29 @@ def evaluate_pipeline(some_parameter: float) -> float: root_directory: The directory to save progress to. - overwrite_working_directory: If true, delete the working directory at the start of + overwrite_root_directory: If true, delete the working directory at the start of the run. This is, e.g., useful when debugging a evaluate_pipeline function. - post_run_summary: If True, creates a csv file after each worker is done, + write_summary_to_disk: If True, creates a csv and txt files after each worker is done, holding summary information about the configs and results. max_evaluations_per_run: Number of evaluations this specific call should do. - max_evaluations_total: Number of evaluations after which to terminate. + evaluations_to_spend: Number of evaluations after which to terminate. This is shared between all workers operating in the same `root_directory`. continue_until_max_evaluation_completed: - If true, only stop after max_evaluations_total have been completed. + If true, only stop after evaluations_to_spend have been completed. This is only relevant in the parallel setting. - max_cost_total: No new evaluations will start when this cost is exceeded. Requires + cost_to_spend: No new evaluations will start when this cost is exceeded. Requires returning a cost in the evaluate_pipeline function, e.g., `return dict(loss=loss, cost=cost)`. + + fidelities_to_spend: Number of evaluations in case of multi-fidelity after which to terminate. + ignore_errors: Ignore hyperparameter settings that threw an error and do not raise - an error. Error configs still count towards max_evaluations_total. + an error. Error configs still count towards evaluations_to_spend. objective_value_on_error: Setting this and cost_value_on_error to any float will supress any error and will use given objective_to_minimize value instead. default: None cost_value_on_error: Setting this and objective_value_on_error to any float will @@ -396,24 +402,84 @@ def __call__( """ # noqa: E501 if ( - max_evaluations_total is None + evaluations_to_spend is None + and max_evaluations_total is None and max_evaluations_per_run is None + and cost_to_spend is None and max_cost_total is None + and fidelities_to_spend is None ): warnings.warn( "None of the following were set, this will run idefinitely until the worker" " process is stopped." - f"\n * {max_evaluations_total=}" + f"\n * {evaluations_to_spend=}" f"\n * {max_evaluations_per_run=}" - f"\n * {max_cost_total=}", + f"\n * {cost_to_spend=}" + f"\n * {fidelities_to_spend}", UserWarning, stacklevel=2, ) + if max_evaluations_total is not None: + warnings.warn( + "`max_evaluations_total` is deprecated and will be removed in" + " a future release. Please use `evaluations_to_spend` instead.", + DeprecationWarning, + stacklevel=2, + ) + evaluations_to_spend = max_evaluations_total + + if max_cost_total is not None: + warnings.warn( + "`max_cost_total` is deprecated and will be removed in a future release. " + "Please use `cost_to_spend` instead.", + DeprecationWarning, + stacklevel=2, + ) + cost_to_spend = max_cost_total + + criteria = { + "evaluations_to_spend": evaluations_to_spend, + "max_evaluations_per_run": max_evaluations_per_run, + "cost_to_spend": cost_to_spend, + "fidelities_to_spend": fidelities_to_spend, + } + set_criteria = [k for k, v in criteria.items() if v is not None] + if len(set_criteria) > 1: + raise ValueError( + f"Multiple stopping criteria specified: {', '.join(set_criteria)}. " + "Only one is allowed." + ) + logger.info(f"Starting neps.run using root directory {root_directory}") space = convert_to_space(pipeline_space) _optimizer_ask, _optimizer_info = load_optimizer(optimizer=optimizer, space=space) + multi_fidelity_optimizers = { + "successive_halving", + "asha", + "hyperband", + "async_hb", + "ifbo", + "priorband", + "moasha", + "mo_hyperband", + "primo", + } + + is_multi_fidelity = _optimizer_info["name"] in multi_fidelity_optimizers + + if is_multi_fidelity: + if evaluations_to_spend is not None: + raise ValueError( + "`evaluations_to_spend` is not allowed for multi-fidelity optimizers. " + "Only `fidelities_to_spend` or `cost_to_spend`" + ) + elif fidelities_to_spend is not None: + raise ValueError( + "`fidelities_to_spend` is not allowed for non-multi-fidelity optimizers." + ) + _eval: Callable if isinstance(evaluate_pipeline, str): module, funcname = evaluate_pipeline.rsplit(":", 1) @@ -435,31 +501,31 @@ def __call__( evaluation_fn=_eval, # type: ignore optimizer=_optimizer_ask, optimizer_info=_optimizer_info, - max_cost_total=max_cost_total, + cost_to_spend=cost_to_spend, + fidelities_to_spend=fidelities_to_spend, optimization_dir=Path(root_directory), - max_evaluations_total=max_evaluations_total, + evaluations_to_spend=evaluations_to_spend, max_evaluations_for_worker=max_evaluations_per_run, continue_until_max_evaluation_completed=continue_until_max_evaluation_completed, objective_value_on_error=objective_value_on_error, cost_value_on_error=cost_value_on_error, ignore_errors=ignore_errors, - overwrite_optimization_dir=overwrite_working_directory, + overwrite_optimization_dir=overwrite_root_directory, sample_batch_size=sample_batch_size, + write_summary_to_disk=write_summary_to_disk, ) - if post_run_summary: - full_frame_path, short_path = post_run_csv(root_directory) - logger.info( - "The post run summary has been created, which is a csv file with the " - "output of all data in the run." - f"\nYou can find a full dataframe at: {full_frame_path}." - f"\nYou can find a quick summary at: {short_path}." - ) - else: + post_run_csv(root_directory) + root_directory = Path(root_directory) + summary_dir = root_directory / "summary" + if not write_summary_to_disk: + trajectory_of_improvements(root_directory) logger.info( - "Skipping the creation of the post run summary, which is a csv file with the " - " output of all data in the run." - "\nSet `post_run_summary=True` to enable it." + "The summary folder has been created, which contains csv and txt files with" + "the output of all data in the run (short.csv - only the best; full.csv - " + "all runs; best_config_trajectory.txt for incumbent trajectory; and " + "best_config.txt for final incumbent)." + f"\nYou can find summary folder at: {summary_dir}." ) diff --git a/neps/optimizers/algorithms.py b/neps/optimizers/algorithms.py index 7f9fab489..a59346d20 100644 --- a/neps/optimizers/algorithms.py +++ b/neps/optimizers/algorithms.py @@ -1173,7 +1173,7 @@ def primo( sample_prior_first: bool | Literal["highest_fidelity"] = False, # noqa: ARG001 eta: int = 3, epsilon: float = 0.25, - prior_centers: Mapping[str, Mapping[str, Any]] | None = None, + prior_centers: Mapping[str, Mapping[str, Any]], mo_selector: Literal["nsga2", "epsnet"] = "epsnet", prior_confidences: Mapping[str, Mapping[str, float]] | None = None, initial_design_size: int | Literal["ndim"] = "ndim", diff --git a/neps/optimizers/bayesian_optimization.py b/neps/optimizers/bayesian_optimization.py index 0c8f84e86..e12e875f8 100644 --- a/neps/optimizers/bayesian_optimization.py +++ b/neps/optimizers/bayesian_optimization.py @@ -170,9 +170,9 @@ def __call__( # noqa: C901, PLR0912, PLR0915 # noqa: C901, PLR0912 "Must provide a 'cost' to configurations if using cost" " with BayesianOptimization." ) - if budget_info.max_cost_total is None: + if budget_info.cost_to_spend is None: raise ValueError("Cost budget must be set if using cost") - cost_percent = budget_info.used_cost_budget / budget_info.max_cost_total + cost_percent = budget_info.used_cost_budget / budget_info.cost_to_spend # If we should use the prior, weight the acquisition function by # the probability of it being sampled from the prior. diff --git a/neps/optimizers/bracket_optimizer.py b/neps/optimizers/bracket_optimizer.py index 9e382fe8c..d034f3dbb 100644 --- a/neps/optimizers/bracket_optimizer.py +++ b/neps/optimizers/bracket_optimizer.py @@ -45,7 +45,7 @@ def trials_to_table(trials: Mapping[str, Trial]) -> pd.DataFrame: configs = np.empty(len(trials), dtype=object) for i, (trial_id, trial) in enumerate(trials.items()): - config_id_str, rung_str = trial_id.split("_") + config_id_str, rung_str = trial_id.split("_rung_") _id, _rung = int(config_id_str), int(rung_str) if trial.report is None: @@ -273,7 +273,7 @@ def __call__( # noqa: C901, PLR0912 config.update(space.constants) config[self.fid_name] = self.fid_max rung = max(self.rung_to_fid) - return SampledConfig(id=f"1_{rung}", config=config) + return SampledConfig(id=f"1_rung_{rung}", config=config) case True: # fid_min config = { name: p.prior if p.prior is not None else p.center @@ -282,7 +282,7 @@ def __call__( # noqa: C901, PLR0912 config.update(space.constants) config[self.fid_name] = self.fid_min rung = min(self.rung_to_fid) - return SampledConfig(id=f"1_{rung}", config=config) + return SampledConfig(id=f"1_rung_{rung}", config=config) case False: pass @@ -334,9 +334,9 @@ def __call__( # noqa: C901, PLR0912 self.fid_name: self.rung_to_fid[new_rung], } return SampledConfig( - id=f"{config_id}_{new_rung}", + id=f"{config_id}_rung_{new_rung}", config=config, - previous_config_id=f"{config_id}_{new_rung - 1}", + previous_config_id=f"{config_id}_rung_{new_rung - 1}", ) # The bracket would like us to sample a new configuration for a rung @@ -353,7 +353,7 @@ def __call__( # noqa: C901, PLR0912 target_fidelity=target_fidelity, ) config.update(space.constants) - return SampledConfig(id=f"{nxt_id}_{rung}", config=config) + return SampledConfig(id=f"{nxt_id}_rung_{rung}", config=config) # We need to sample for a new rung, with either no gp or it has # not yet kicked in. @@ -367,7 +367,7 @@ def __call__( # noqa: C901, PLR0912 **space.constants, self.fid_name: self.rung_to_fid[rung], } - return SampledConfig(id=f"{nxt_id}_{rung}", config=config) + return SampledConfig(id=f"{nxt_id}_rung_{rung}", config=config) case PriorBandSampler(): config = self.sampler.sample_config(table, rung=rung) @@ -376,7 +376,7 @@ def __call__( # noqa: C901, PLR0912 **space.constants, self.fid_name: self.rung_to_fid[rung], } - return SampledConfig(id=f"{nxt_id}_{rung}", config=config) + return SampledConfig(id=f"{nxt_id}_rung_{rung}", config=config) case _: raise RuntimeError(f"Unknown sampler: {self.sampler}") case _: diff --git a/neps/optimizers/mopriors.py b/neps/optimizers/mopriors.py index f78ffdeb4..f5a61bc3d 100644 --- a/neps/optimizers/mopriors.py +++ b/neps/optimizers/mopriors.py @@ -29,7 +29,7 @@ def dists_from_centers_and_confidences( cls, parameters: Mapping[str, Parameter], prior_centers: Mapping[str, Mapping[str, float]], - confidence_values: Mapping[str, Mapping[str, float]], + confidence_values: Mapping[str, Mapping[str, float]] | None = None, ) -> Mapping[str, Prior]: """Creates a mapping of prior distributions from the given centers and confidence values. @@ -41,19 +41,17 @@ def dists_from_centers_and_confidences( A mapping of prior distributions. """ _priors = {} - for key, _prior_center in prior_centers.items(): assert isinstance(_prior_center, dict), ( f"Expected prior center values to be a dict, got {type(_prior_center)}" ) - assert key in confidence_values, ( - f"Expected confidence values to contain {key}, " - f"got {confidence_values.keys()}" - ) + _default_confidence = dict.fromkeys(prior_centers.keys(), 0.25) _priors[key] = Prior.from_parameters( parameters=parameters, center_values=_prior_center, - confidence_values=confidence_values[key], + confidence_values=( + confidence_values[key] if confidence_values else _default_confidence + ), ) return _priors diff --git a/neps/optimizers/primo.py b/neps/optimizers/primo.py index bfdabc264..fba87bf81 100644 --- a/neps/optimizers/primo.py +++ b/neps/optimizers/primo.py @@ -155,7 +155,7 @@ def __call__( # noqa: C901, PLR0912 # Get the next ID for the sampled configuration if "_" in trial_id: - config_id_str, _ = trial_id.split("_") + config_id_str, _, _ = trial_id.split("_") else: config_id_str = trial_id diff --git a/neps/plot/plotting.py b/neps/plot/plotting.py index 612fad9d8..c5054595a 100644 --- a/neps/plot/plotting.py +++ b/neps/plot/plotting.py @@ -139,16 +139,16 @@ def _interpolate_time( if x_range is not None: min_b, max_b = x_range - new_entry = {c: np.nan for c in df.columns} + new_entry = dict.fromkeys(df.columns, np.nan) _df = pd.DataFrame.from_dict(new_entry, orient="index").T _df.index = [min_b] df = pd.concat((df, _df)).sort_index() - new_entry = {c: np.nan for c in df.columns} + new_entry = dict.fromkeys(df.columns, np.nan) _df = pd.DataFrame.from_dict(new_entry, orient="index").T _df.index = [max_b] df = pd.concat((df, _df)).sort_index() - df = df.fillna(method="backfill", axis=0).fillna(method="ffill", axis=0) + df = df.bfill(axis=0).ffill(axis=0) if x_range is not None: df = df.query(f"{x_range[0]} <= index <= {x_range[1]}") @@ -157,11 +157,11 @@ def _interpolate_time( def _df_to_x_range(df: pd.DataFrame, x_range: tuple | None = None) -> pd.DataFrame: x_max = np.inf if x_range is None else int(x_range[-1]) - new_entry = {c: np.nan for c in df.columns} + new_entry = dict.fromkeys(df.columns, np.nan) _df = pd.DataFrame.from_dict(new_entry, orient="index").T _df.index = [x_max] df = pd.concat((df, _df)).sort_index() - return df.fillna(method="backfill", axis=0).fillna(method="ffill", axis=0) + return df.bfill(axis=0).ffill(axis=0) def _set_legend( diff --git a/neps/plot/read_results.py b/neps/plot/read_results.py index 4e01f8b71..c29cffbff 100644 --- a/neps/plot/read_results.py +++ b/neps/plot/read_results.py @@ -4,16 +4,19 @@ from pathlib import Path +import numpy as np + import neps +from neps.state.trial import State def process_seed( *, path: str | Path, seed: str | int | None, - key_to_extract: str | None = None, # noqa: ARG001 - consider_continuations: bool = False, # noqa: ARG001 - n_workers: int = 1, # noqa: ARG001 + key_to_extract: str | None = None, + consider_continuations: bool = False, + n_workers: int = 1, ) -> tuple[list[float], list[float], float]: """Reads and processes data per seed.""" path = Path(path) @@ -21,51 +24,48 @@ def process_seed( path = path / str(seed) / "neps_root_directory" _fulldf, _summary = neps.status(path, print_summary=False) - raise NotImplementedError( - "I'm sorry, I broke this. We now dump all the information neps has available" - " into the above dataframe `fulldf`." - ) - # > sorted_stats = sorted(sorted(stats.items()), key=lambda x: len(x[0])) - # > stats = OrderedDict(sorted_stats) - - # > # max_cost only relevant for scaling x-axis when using fidelity on the x-axis - # > max_cost: float = -1.0 - # > if key_to_extract == "fidelity": - # > # TODO(eddiebergman): This can crash for a number of reasons, namely if the - # > # config crased and it's result is an error, or if the `"info_dict"` and/or - # > # `key_to_extract` doesn't exist - # > max_cost = max(s.result["info_dict"][key_to_extract] for s in stats.values()) - - # > global_start = stats[min(stats.keys())].metadata["time_sampled"] - - # > def get_cost(idx: str) -> float: - # > if key_to_extract is not None: - # > # TODO(eddiebergman): This can crash for a number of reasons, namely if - # > # the config crased and it's result is an error, or if the `"info_dict"` - # > # and/or `key_to_extract` doesn't exist - # > return float(stats[idx].result["info_dict"][key_to_extract]) - - # > return 1.0 - - # > losses = [] - # > costs = [] - - # > for config_id, config_result in stats.items(): - # > config_cost = get_cost(config_id) - # > if consider_continuations: - # > if n_workers == 1: - # > # calculates continuation costs for MF algorithms NOTE: assumes that - # > # all recorded evaluations are black-box evaluations where - # > # continuations or freeze-thaw was not accounted for during opt - # > if "previous_config_id" in config_result.metadata: - # > previous_config_id = config_result.metadata["previous_config_id"] - # > config_cost -= get_cost(previous_config_id) - # > else: - # > config_cost = config_result.metadata["time_end"] - global_start - - # > # TODO(eddiebergman): Assumes it never crashed and there's a - # > # objective_to_minimize available,not fixing now but it should be addressed - # > losses.append(config_result.result["objective_to_minimize"]) # type: ignore - # > costs.append(config_cost) - - # > return list(np.minimum.accumulate(losses)), costs, max_cost + if _fulldf.empty: + raise ValueError(f"No trials found in {path}") + + _fulldf = _fulldf.sort_values("time_sampled") + + def get_cost(idx: str | int) -> float: + row = _fulldf.loc[idx] + if key_to_extract and key_to_extract in row: + return float(row[key_to_extract]) + return 1.0 + + losses = [] + costs = [] + + # max_cost only relevant for scaling x-axis when using fidelity on the x-axis + max_cost: float = -1.0 + global_start = _fulldf["time_sampled"].min() + + for config_id, config_result in _fulldf.iterrows(): + if config_result["state"] != State.SUCCESS: + continue + + cost = get_cost(config_id) + + if ( + consider_continuations + and n_workers == 1 + and "previous_config_id" in config_result["metadata"] + ): + # calculates continuation costs for MF algorithms NOTE: assumes that + # all recorded evaluations are black-box evaluations where + # continuations or freeze-thaw was not accounted for during optimization + previous_id = config_result["metadata"]["previous_config_id"] + + if previous_id in _fulldf.index and key_to_extract: + cost -= get_cost(config_id) + else: + cost = float(config_result["time_end"] - global_start) + + loss = float(config_result["objective_to_minimize"]) + losses.append(loss) + costs.append(cost) + max_cost = max(max_cost, cost) + + return list(np.minimum.accumulate(losses)), costs, max_cost diff --git a/neps/runtime.py b/neps/runtime.py index e3958322b..30915a7fb 100644 --- a/neps/runtime.py +++ b/neps/runtime.py @@ -13,6 +13,7 @@ from pathlib import Path from typing import TYPE_CHECKING, ClassVar, Literal +from filelock import FileLock from portalocker import portalocker from neps.env import ( @@ -41,6 +42,7 @@ WorkerSettings, evaluate_trial, ) +from neps.status.status import _initiate_summary_csv, status from neps.utils.common import gc_disabled if TYPE_CHECKING: @@ -319,7 +321,7 @@ def _check_global_stopping_criterion( self, trials: Mapping[str, Trial], ) -> str | Literal[False]: - if self.settings.max_evaluations_total is not None: + if self.settings.evaluations_to_spend is not None: if self.settings.include_in_progress_evaluations_towards_maximum: count = sum( 1 @@ -331,23 +333,40 @@ def _check_global_stopping_criterion( # This indicates they have completed. count = sum(1 for _, trial in trials.items() if trial.report is not None) - if count >= self.settings.max_evaluations_total: + if count >= self.settings.evaluations_to_spend: return ( "The total number of evaluations has reached the maximum allowed of" - f" `{self.settings.max_evaluations_total=}`." + f" `{self.settings.evaluations_to_spend=}`." " To allow more evaluations, increase this value or use a different" " stopping criterion." ) - if self.settings.max_cost_total is not None: + if self.settings.fidelities_to_spend is not None and hasattr( + self.optimizer, "space" + ): + fidelity_name = next(iter(self.optimizer.space.fidelities.keys())) + count = sum( + trial.config[fidelity_name] + for _, trial in trials.items() + if trial.report is not None and trial.config[fidelity_name] is not None + ) + if count >= self.settings.fidelities_to_spend: + return ( + "The total number of fidelity evaluations has reached the maximum" + f" allowed of `{self.settings.fidelities_to_spend=}`." + " To allow more evaluations, increase this value or use a different" + " stopping criterion." + ) + + if self.settings.cost_to_spend is not None: cost = sum( trial.report.cost for _, trial in trials.items() if trial.report is not None and trial.report.cost is not None ) - if cost >= self.settings.max_cost_total: + if cost >= self.settings.cost_to_spend: return ( - f"The maximum cost `{self.settings.max_cost_total=}` has been" + f"The maximum cost `{self.settings.cost_to_spend=}` has been" " reached by all of the evaluated trials. To allow more evaluations," " increase this value or use a different stopping criterion." ) @@ -372,8 +391,9 @@ def _check_global_stopping_criterion( @property def _requires_global_stopping_criterion(self) -> bool: return ( - self.settings.max_evaluations_total is not None - or self.settings.max_cost_total is not None + self.settings.evaluations_to_spend is not None + or self.settings.cost_to_spend is not None + or self.settings.fidelities_to_spend is not None or self.settings.max_evaluation_time_total_seconds is not None ) @@ -493,7 +513,53 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 """ _set_workers_neps_state(self.state) - logger.info("Launching NePS") + main_dir = Path(self.state.path) + if self.settings.write_summary_to_disk: + full_df_path, short_path, csv_locker = _initiate_summary_csv(main_dir) + + # Create empty CSV files + with csv_locker.lock(): + full_df_path.parent.mkdir(parents=True, exist_ok=True) + full_df_path.touch(exist_ok=True) + short_path.touch(exist_ok=True) + + summary_dir = main_dir / "summary" + summary_dir.mkdir(parents=True, exist_ok=True) + + improvement_trace_path = summary_dir / "best_config_trajectory.txt" + improvement_trace_path.touch(exist_ok=True) + best_config_path = summary_dir / "best_config.txt" + best_config_path.touch(exist_ok=True) + _trace_lock = FileLock(".trace.lock") + _trace_lock_path = Path(str(_trace_lock.lock_file)) + _trace_lock_path.touch(exist_ok=True) + + logger.info( + "Summary files can be found in the “summary” folder inside" + "the root directory: %s", + summary_dir, + ) + + previous_trials = self.state.lock_and_read_trials() + if len(previous_trials): + load_incumbent_trace( + previous_trials, + _trace_lock, + self.state, + self.settings, + improvement_trace_path, + best_config_path, + ) + + _best_score_so_far = float("inf") + if ( + self.state.new_score is not None + and self.state.new_score != _best_score_so_far + ): + _best_score_so_far = self.state.new_score + + optimizer_name = self.state._optimizer_info["name"] + logger.info("Using optimizer: %s", optimizer_name) _time_monotonic_start = time.monotonic() _error_from_evaluation: Exception | None = None @@ -623,6 +689,65 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 for _key, callback in _TRIAL_END_CALLBACKS.items(): callback(trial_to_eval) + if ( + report.objective_to_minimize is not None + and report.err is None + and not isinstance(report.objective_to_minimize, list) + ): + self.state.new_score = report.objective_to_minimize + if self.state.new_score < _best_score_so_far: + _best_score_so_far = self.state.new_score + logger.info( + "New best: trial %s with objective %s", + evaluated_trial.id, + self.state.new_score, + ) + + if self.settings.write_summary_to_disk: + # Store in memory for later file re-writing + self.state.all_best_configs.append( + { + "score": self.state.new_score, + "trial_id": evaluated_trial.id, + "config": evaluated_trial.config, + } + ) + + # Build trace text and best config text + trace_text = ( + "Best configs and their objectives across evaluations:\n" + + "-" * 80 + + "\n" + ) + for best in self.state.all_best_configs: + trace_text += ( + f"Objective to minimize: {best['score']}\n" + f"Config ID: {best['trial_id']}\n" + f"Config: {best['config']}\n" + "-" * 80 + "\n" + ) + + best_config = self.state.all_best_configs[-1] # Latest best + best_config_text = ( + f"# Best config:" + f"\n\n Config ID: {best_config['trial_id']}" + f"\n Objective to minimize: {best_config['score']}" + f"\n Config: {best_config['config']}" + ) + + # Write files from scratch + with _trace_lock: + with improvement_trace_path.open(mode="w") as f: + f.write(trace_text) + + with best_config_path.open(mode="w") as f: + f.write(best_config_text) + + if self.settings.write_summary_to_disk: + full_df, short = status(main_dir) + with csv_locker.lock(): + full_df.to_csv(full_df_path) + short.to_frame().to_csv(short_path) + logger.debug("Config %s: %s", evaluated_trial.id, evaluated_trial.config) logger.debug("Loss %s: %s", evaluated_trial.id, report.objective_to_minimize) logger.debug("Cost %s: %s", evaluated_trial.id, report.objective_to_minimize) @@ -631,6 +756,61 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 ) +def load_incumbent_trace( # noqa: D103 + previous_trials: dict[str, Trial], + _trace_lock: FileLock, + state: NePSState, + settings: WorkerSettings, # noqa: ARG001 + improvement_trace_path: Path, + best_config_path: Path, +) -> None: + _best_score_so_far = float("inf") + + for evaluated_trial in previous_trials.values(): + if ( + evaluated_trial.report is not None + and evaluated_trial.report.objective_to_minimize is not None + ): + state.new_score = evaluated_trial.report.objective_to_minimize + if state.new_score is not None and state.new_score < _best_score_so_far: + _best_score_so_far = state.new_score + state.all_best_configs.append( + { + "score": state.new_score, + "trial_id": evaluated_trial.metadata.id, + "config": evaluated_trial.config, + } + ) + + trace_text = ( + "Best configs and their objectives across evaluations:\n" + "-" * 80 + "\n" + ) + for best in state.all_best_configs: + trace_text += ( + f"Objective to minimize: {best['score']}\n" + f"Config ID: {best['trial_id']}\n" + f"Config: {best['config']}\n" + "-" * 80 + "\n" + ) + + best_config_text = "" + if state.all_best_configs: + best_config = state.all_best_configs[-1] + best_config_text = ( + f"# Best config:" + f"\n\n Config ID: {best_config['trial_id']}" + f"\n Objective to minimize: {best_config['score']}" + f"\n Config: {best_config['config']}" + ) + else: + best_config = None + + with _trace_lock: + with improvement_trace_path.open(mode="w") as f: + f.write(trace_text) + with best_config_path.open(mode="w") as f: + f.write(best_config_text) + + def _launch_ddp_runtime( *, evaluation_fn: Callable[..., EvaluatePipelineReturn], @@ -691,15 +871,17 @@ def _launch_runtime( # noqa: PLR0913 optimizer: AskFunction, optimizer_info: OptimizerInfo, optimization_dir: Path, - max_cost_total: float | None, + cost_to_spend: float | None, ignore_errors: bool = False, objective_value_on_error: float | None, cost_value_on_error: float | None, continue_until_max_evaluation_completed: bool, overwrite_optimization_dir: bool, - max_evaluations_total: int | None, + evaluations_to_spend: int | None, + fidelities_to_spend: int | None, max_evaluations_for_worker: int | None, sample_batch_size: int | None, + write_summary_to_disk: bool = True, ) -> None: default_report_values = DefaultReportValues( objective_value_on_error=objective_value_on_error, @@ -736,9 +918,10 @@ def _launch_runtime( # noqa: PLR0913 seed_snapshot=SeedSnapshot.new_capture(), budget=( BudgetInfo( - max_cost_total=max_cost_total, + cost_to_spend=cost_to_spend, used_cost_budget=0, - max_evaluations=max_evaluations_total, + max_evaluations=evaluations_to_spend, + fidelities_to_spend=fidelities_to_spend, used_evaluations=0, ) ), @@ -767,16 +950,18 @@ def _launch_runtime( # noqa: PLR0913 ), batch_size=sample_batch_size, default_report_values=default_report_values, - max_evaluations_total=max_evaluations_total, + evaluations_to_spend=evaluations_to_spend, + fidelities_to_spend=fidelities_to_spend, include_in_progress_evaluations_towards_maximum=( not continue_until_max_evaluation_completed ), - max_cost_total=max_cost_total, + cost_to_spend=cost_to_spend, max_evaluations_for_worker=max_evaluations_for_worker, max_evaluation_time_total_seconds=None, # TODO: User can't specify yet max_wallclock_time_for_worker_seconds=None, # TODO: User can't specify yet max_evaluation_time_for_worker_seconds=None, # TODO: User can't specify yet max_cost_for_worker=None, # TODO: User can't specify yet + write_summary_to_disk=write_summary_to_disk, ) # HACK: Due to nfs file-systems, locking with the default `flock()` is not reliable. diff --git a/neps/state/neps_state.py b/neps/state/neps_state.py index 0e684a428..67d8bd9a8 100644 --- a/neps/state/neps_state.py +++ b/neps/state/neps_state.py @@ -251,6 +251,12 @@ class NePSState: _shared_errors_path: Path = field(repr=False) _shared_errors: ErrDump = field(repr=False) + new_score: float = float("inf") + """Tracking of the new incumbent""" + + all_best_configs: list = field(default_factory=list) + """Trajectory to the newest incbumbent""" + def lock_and_read_trials(self) -> dict[str, Trial]: """Acquire the state lock and read the trials.""" with self._trial_lock.lock(): @@ -382,9 +388,16 @@ def _sample_trial( else: previous_trial_location = None + id_str = sampled_config.id + config_name = f"{sampled_config.id}" + parts = id_str.split("_rung_") + if len(parts) == 2 and all(p.isdigit() for p in parts): + config_id, rung_id = map(int, parts) + config_name = f"{config_id}_rung_{rung_id}" + trial = Trial.new( trial_id=sampled_config.id, - location=str(self._trial_repo.directory / f"config_{sampled_config.id}"), + location=str(self._trial_repo.directory / f"config_{config_name}"), config=sampled_config.config, previous_trial=sampled_config.previous_config_id, previous_trial_location=previous_trial_location, diff --git a/neps/state/optimizer.py b/neps/state/optimizer.py index 5e51c21e7..7d4787cce 100644 --- a/neps/state/optimizer.py +++ b/neps/state/optimizer.py @@ -13,10 +13,11 @@ class BudgetInfo: """Information about the budget of an optimizer.""" - max_cost_total: float | None = None + cost_to_spend: float | None = None used_cost_budget: float = 0.0 max_evaluations: int | None = None used_evaluations: int = 0 + fidelities_to_spend: int | None = None def clone(self) -> BudgetInfo: """Create a copy of the budget info.""" diff --git a/neps/state/pipeline_eval.py b/neps/state/pipeline_eval.py index 7903fec18..086de800e 100644 --- a/neps/state/pipeline_eval.py +++ b/neps/state/pipeline_eval.py @@ -402,7 +402,12 @@ def _eval_trial( else: duration = time.monotonic() - start time_end = time.time() - logger.info(f"Successful evaluation of '{trial.id}': {user_result}.") + match user_result: + case dict(): + filtered_data = {k: v for k, v in user_result.items() if k != "info_dict"} + logger.info(f"Successful evaluation of '{trial.id}': {filtered_data}.") + case _: # TODO: Revisit this and check all possible cases + logger.info(f"Successful evaluation of '{trial.id}': {user_result}.") result = UserResult.parse( user_result, diff --git a/neps/state/settings.py b/neps/state/settings.py index bbc7c0498..aefec74f6 100644 --- a/neps/state/settings.py +++ b/neps/state/settings.py @@ -78,7 +78,7 @@ class WorkerSettings: """The number of configurations to sample in a single batch.""" # --------- Global Stopping Criterion --------- - max_evaluations_total: int | None + evaluations_to_spend: int | None """The maximum number of evaluations to run in total. Once this evaluation total is reached, **all** workers will stop evaluating @@ -95,10 +95,10 @@ class WorkerSettings: include_in_progress_evaluations_towards_maximum: bool """Whether to include currently evaluating configurations towards the stopping criterion - [`max_evaluations_total`][neps.state.settings.WorkerSettings.max_evaluations_total] + [`evaluations_to_spend`][neps.state.settings.WorkerSettings.evaluations_to_spend] """ - max_cost_total: float | None + cost_to_spend: float | None """The maximum cost to run in total. Once this cost total is reached, **all** workers will stop evaluating new @@ -111,6 +111,20 @@ class WorkerSettings: indefinitely or until another stopping criterion is met. """ + fidelities_to_spend: int | None + """The maximum number of evaluations to run in case of multi-fidelity. + + Once this evaluation total is reached, **all** workers will stop evaluating + new configurations. + + To control whether currently evaluating configurations are included in this + total, see + [`include_in_progress_evaluations_towards_maximum`][neps.state.settings.WorkerSettings.include_in_progress_evaluations_towards_maximum]. + + If `None`, there is no limit and workers will continue to evaluate + indefinitely. + """ + max_evaluation_time_total_seconds: float | None """The maximum wallclock time allowed for evaluation in total. @@ -174,3 +188,8 @@ class WorkerSettings: If `None`, there is no limit and this worker will continue to evaluate indefinitely or until another stopping criterion is met. """ + + write_summary_to_disk: bool = True + """If True, creates a csv and txt files after each worker is done, + holding summary information about the configs and results. + """ diff --git a/neps/status/status.py b/neps/status/status.py index 4d8d4421e..32a741e3c 100644 --- a/neps/status/status.py +++ b/neps/status/status.py @@ -11,7 +11,6 @@ import numpy as np import pandas as pd -from neps.runtime import get_workers_neps_state from neps.state.neps_state import FileLocker, NePSState from neps.state.trial import State, Trial @@ -141,6 +140,8 @@ def from_directory(cls, root_directory: str | Path) -> Summary: # NOTE: We don't lock the shared state since we are just reading and don't need to # make decisions based on the state try: + from neps.runtime import get_workers_neps_state + shared_state = get_workers_neps_state() except RuntimeError: shared_state = NePSState.create_or_load(root_directory, load_only=True) @@ -220,6 +221,79 @@ def status( return df, short +def trajectory_of_improvements( + root_directory: str | Path, +) -> list[dict]: + """Track and write the trajectory of improving configurations over time. + + Args: + root_directory: The root directory given to neps.run. + + Returns: + List of dicts with improving scores and their configurations. + """ + root_directory = Path(root_directory) + summary = Summary.from_directory(root_directory) + + if summary.is_multiobjective: + return [] + + df = summary.df() + + if "time_sampled" not in df.columns: + raise ValueError("Missing `time_sampled` column in summary DataFrame.") + + df = df.sort_values("time_sampled") + + all_best_configs = [] + best_score = float("inf") + trace_text = "" + + for trial_id, row in df.iterrows(): + if "objective_to_minimize" not in row or pd.isna(row["objective_to_minimize"]): + continue + + score = row["objective_to_minimize"] + if score < best_score: + best_score = score + config = { + k.replace("config.", ""): v + for k, v in row.items() + if k.startswith("config.") + } + + best = { + "score": score, + "trial_id": trial_id, + "config": config, + } + all_best_configs.append(best) + + trace_text += ( + f"Objective to minimize: {best['score']}\n" + f"Config ID: {best['trial_id']}\n" + f"Config: {best['config']}\n" + "-" * 80 + "\n" + ) + + summary_dir = root_directory / "summary" + summary_dir.mkdir(parents=True, exist_ok=True) + output_path = summary_dir / "best_config_trajectory.txt" + with output_path.open("w") as f: + f.write(trace_text) + + if all_best_configs: + final_best = all_best_configs[-1] + best_path = summary_dir / "best_config.txt" + with best_path.open("w") as f: + f.write( + f"Objective to minimize: {final_best['score']}\n" + f"Config ID: {final_best['trial_id']}\n" + f"Config: {final_best['config']}\n" + ) + + return all_best_configs + + def _initiate_summary_csv(root_directory: str | Path) -> tuple[Path, Path, FileLocker]: """Initializes a summary CSV and an associated locker for file access control. diff --git a/neps_examples/basic_usage/hyperparameters.py b/neps_examples/basic_usage/hyperparameters.py index 6b736fcd3..6b1c6a5d7 100644 --- a/neps_examples/basic_usage/hyperparameters.py +++ b/neps_examples/basic_usage/hyperparameters.py @@ -12,7 +12,7 @@ def evaluate_pipeline(float1, float2, categorical, integer1, integer2): np.sum([float1, float2, int(categorical), integer1, integer2]) ) return objective_to_minimize - + pipeline_space = dict( float1=neps.Float(lower=0, upper=1), @@ -27,6 +27,5 @@ def evaluate_pipeline(float1, float2, categorical, integer1, integer2): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/hyperparameters_example", - post_run_summary=True, - max_evaluations_total=30, + evaluations_to_spend=30, ) diff --git a/neps_examples/convenience/logging_additional_info.py b/neps_examples/convenience/logging_additional_info.py index 6756e03c7..3120c7db5 100644 --- a/neps_examples/convenience/logging_additional_info.py +++ b/neps_examples/convenience/logging_additional_info.py @@ -34,5 +34,5 @@ def evaluate_pipeline(float1, float2, categorical, integer1, integer2): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/logging_additional_info", - max_evaluations_total=5, + evaluations_to_spend=5, ) diff --git a/neps_examples/convenience/neps_tblogger_tutorial.py b/neps_examples/convenience/neps_tblogger_tutorial.py index fd9bc8144..938ca0e2c 100644 --- a/neps_examples/convenience/neps_tblogger_tutorial.py +++ b/neps_examples/convenience/neps_tblogger_tutorial.py @@ -342,7 +342,7 @@ def evaluate_pipeline(lr, optim, weight_decay): neps.run( **run_args, - max_evaluations_total=3, + evaluations_to_spend=3, ) """ diff --git a/neps_examples/convenience/running_on_slurm_scripts.py b/neps_examples/convenience/running_on_slurm_scripts.py index 86fe41ac2..26dac7082 100644 --- a/neps_examples/convenience/running_on_slurm_scripts.py +++ b/neps_examples/convenience/running_on_slurm_scripts.py @@ -60,5 +60,5 @@ def evaluate_pipeline_via_slurm( evaluate_pipeline=evaluate_pipeline_via_slurm, pipeline_space=pipeline_space, root_directory="results/slurm_script_example", - max_evaluations_total=5, + evaluations_to_spend=5, ) diff --git a/neps_examples/convenience/working_directory_per_pipeline.py b/neps_examples/convenience/working_directory_per_pipeline.py index de2ec9fd9..7b1b5ad13 100644 --- a/neps_examples/convenience/working_directory_per_pipeline.py +++ b/neps_examples/convenience/working_directory_per_pipeline.py @@ -29,5 +29,5 @@ def evaluate_pipeline(pipeline_directory: Path, float1, categorical, integer1): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/working_directory_per_pipeline", - max_evaluations_total=5, + evaluations_to_spend=5, ) diff --git a/neps_examples/efficiency/expert_priors_for_hyperparameters.py b/neps_examples/efficiency/expert_priors_for_hyperparameters.py index 5980668a5..32633930b 100644 --- a/neps_examples/efficiency/expert_priors_for_hyperparameters.py +++ b/neps_examples/efficiency/expert_priors_for_hyperparameters.py @@ -48,5 +48,5 @@ def evaluate_pipeline(some_float, some_integer, some_cat): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/user_priors_example", - max_evaluations_total=15, + evaluations_to_spend=15, ) diff --git a/neps_examples/efficiency/multi_fidelity.py b/neps_examples/efficiency/multi_fidelity.py index c85d411e1..b067eac2b 100644 --- a/neps_examples/efficiency/multi_fidelity.py +++ b/neps_examples/efficiency/multi_fidelity.py @@ -94,5 +94,5 @@ def evaluate_pipeline( root_directory="results/multi_fidelity_example", # Optional: Do not start another evaluation after <=50 epochs, corresponds to cost # field above. - max_cost_total=50, + fidelities_to_spend=20 ) diff --git a/neps_examples/efficiency/multi_fidelity_and_expert_priors.py b/neps_examples/efficiency/multi_fidelity_and_expert_priors.py index 96f7b2b3e..5e8ff2aab 100644 --- a/neps_examples/efficiency/multi_fidelity_and_expert_priors.py +++ b/neps_examples/efficiency/multi_fidelity_and_expert_priors.py @@ -43,5 +43,5 @@ def evaluate_pipeline(float1, float2, integer1, fidelity): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/multifidelity_priors", - max_evaluations_total=25, # For an alternate stopping method see multi_fidelity.py + fidelities_to_spend=25, # For an alternate stopping method see multi_fidelity.py ) diff --git a/neps_examples/efficiency/pytorch_lightning_ddp.py b/neps_examples/efficiency/pytorch_lightning_ddp.py index 96b620de1..d40a29cdd 100644 --- a/neps_examples/efficiency/pytorch_lightning_ddp.py +++ b/neps_examples/efficiency/pytorch_lightning_ddp.py @@ -100,5 +100,5 @@ def evaluate_pipeline(lr=0.1, epoch=20): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/pytorch_lightning_ddp", - max_evaluations_total=5 + fidelities_to_spend=5 ) diff --git a/neps_examples/efficiency/pytorch_lightning_fsdp.py b/neps_examples/efficiency/pytorch_lightning_fsdp.py index 6af3d6746..c7317bfc6 100644 --- a/neps_examples/efficiency/pytorch_lightning_fsdp.py +++ b/neps_examples/efficiency/pytorch_lightning_fsdp.py @@ -74,5 +74,5 @@ def evaluate_pipeline(lr=0.1, epoch=20): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/pytorch_lightning_fsdp", - max_evaluations_total=5 + fidelities_to_spend=5 ) diff --git a/neps_examples/efficiency/pytorch_native_ddp.py b/neps_examples/efficiency/pytorch_native_ddp.py index 9ced5dc86..9477d6e95 100644 --- a/neps_examples/efficiency/pytorch_native_ddp.py +++ b/neps_examples/efficiency/pytorch_native_ddp.py @@ -112,4 +112,4 @@ def evaluate_pipeline(learning_rate, epochs): neps.run(evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/pytorch_ddp", - max_evaluations_total=25) + evaluations_to_spend=25) diff --git a/neps_examples/efficiency/pytorch_native_fsdp.py b/neps_examples/efficiency/pytorch_native_fsdp.py index 1fec7bef3..4b40e318a 100644 --- a/neps_examples/efficiency/pytorch_native_fsdp.py +++ b/neps_examples/efficiency/pytorch_native_fsdp.py @@ -212,5 +212,5 @@ def evaluate_pipeline(lr=0.1, epoch=20): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/pytorch_fsdp", - max_evaluations_total=20 + fidelities_to_spend=20 ) diff --git a/neps_examples/experimental/freeze_thaw.py b/neps_examples/experimental/freeze_thaw.py index 597e1df3d..bcf4fb618 100644 --- a/neps_examples/experimental/freeze_thaw.py +++ b/neps_examples/experimental/freeze_thaw.py @@ -166,7 +166,7 @@ def training_pipeline( pipeline_space=pipeline_space, evaluate_pipeline=training_pipeline, optimizer="ifbo", - max_evaluations_total=50, + fidelities_to_spend=50, root_directory="./results/ifbo-mnist/", overwrite_working_directory=False, # set to False for a multi-worker run ) diff --git a/neps_examples/real_world/image_segmentation_hpo.py b/neps_examples/real_world/image_segmentation_hpo.py index 2320f20f1..51ff27a06 100644 --- a/neps_examples/real_world/image_segmentation_hpo.py +++ b/neps_examples/real_world/image_segmentation_hpo.py @@ -125,5 +125,5 @@ def evaluate_pipeline(**kwargs): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="results/hpo_image_segmentation", - max_evaluations_total=500 + fidelities_to_spend=500 ) diff --git a/tests/test_runtime/test_default_report_values.py b/tests/test_runtime/test_default_report_values.py index 3b37d5254..45ad5e5cc 100644 --- a/tests/test_runtime/test_default_report_values.py +++ b/tests/test_runtime/test_default_report_values.py @@ -41,9 +41,10 @@ def test_default_values_on_error( cost_value_on_error=2.4, # <- Highlight learning_curve_on_error=[2.4, 2.5], # <- Highlight ), - max_evaluations_total=None, + evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=1, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -92,9 +93,10 @@ def test_default_values_on_not_specified( cost_if_not_provided=2.4, learning_curve_if_not_provided=[2.4, 2.5], ), - max_evaluations_total=None, + evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=1, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -141,9 +143,10 @@ def test_default_value_objective_to_minimize_curve_take_objective_to_minimize_va default_report_values=DefaultReportValues( learning_curve_if_not_provided="objective_to_minimize" ), - max_evaluations_total=None, + evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=1, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, diff --git a/tests/test_runtime/test_error_handling_strategies.py b/tests/test_runtime/test_error_handling_strategies.py index c1762a02a..0549f87b5 100644 --- a/tests/test_runtime/test_error_handling_strategies.py +++ b/tests/test_runtime/test_error_handling_strategies.py @@ -48,9 +48,10 @@ def test_worker_raises_when_error_in_self( settings = WorkerSettings( on_error=on_error, # <- Highlight default_report_values=DefaultReportValues(), - max_evaluations_total=None, + evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=1, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -88,9 +89,10 @@ def test_worker_raises_when_error_in_other_worker(neps_state: NePSState) -> None settings = WorkerSettings( on_error=OnErrorPossibilities.RAISE_ANY_ERROR, # <- Highlight default_report_values=DefaultReportValues(), - max_evaluations_total=None, + evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=1, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -148,9 +150,10 @@ def test_worker_does_not_raise_when_error_in_other_worker( settings = WorkerSettings( on_error=on_error, # <- Highlight default_report_values=DefaultReportValues(), - max_evaluations_total=None, + evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=1, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, diff --git a/tests/test_runtime/test_stopping_criterion.py b/tests/test_runtime/test_stopping_criterion.py index 08fc3dbf3..67f15365f 100644 --- a/tests/test_runtime/test_stopping_criterion.py +++ b/tests/test_runtime/test_stopping_criterion.py @@ -40,9 +40,10 @@ def test_max_evaluations_total_stopping_criterion( settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), - max_evaluations_total=3, # <- Highlight + evaluations_to_spend=3, # <- Highlight include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -92,9 +93,10 @@ def test_worker_evaluations_total_stopping_criterion( settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), - max_evaluations_total=None, + evaluations_to_spend=None, include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=2, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -153,9 +155,10 @@ def test_include_in_progress_evaluations_towards_maximum_with_work_eval_count( settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), - max_evaluations_total=2, # <- Highlight, only 2 maximum evaluations allowed + evaluations_to_spend=2, # <- Highlight, only 2 maximum evaluations allowed include_in_progress_evaluations_towards_maximum=True, # <- inprogress trial - max_cost_total=None, + cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -207,9 +210,10 @@ def test_max_cost_total(neps_state: NePSState) -> None: settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), - max_evaluations_total=10, # Safety incase it doesn't work that we eventually stop + evaluations_to_spend=10, # Safety incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, - max_cost_total=2, # <- Highlight, only 2 maximum evaluations allowed + cost_to_spend=2, # <- Highlight, only 2 maximum evaluations allowed + fidelities_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -255,9 +259,10 @@ def test_worker_cost_total(neps_state: NePSState) -> None: settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), - max_evaluations_total=10, # Safety incase it doesn't work that we eventually stop + evaluations_to_spend=10, # Safety incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -311,9 +316,10 @@ def test_worker_wallclock_time(neps_state: NePSState) -> None: settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), - max_evaluations_total=1000, # Incase it doesn't work that we eventually stop + evaluations_to_spend=1000, # Incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=1, # <- highlight, 1 second @@ -366,9 +372,10 @@ def test_max_worker_evaluation_time(neps_state: NePSState) -> None: settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), - max_evaluations_total=10, # Safety incase it doesn't work that we eventually stop + evaluations_to_spend=10, # Safety incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=None, max_wallclock_time_for_worker_seconds=None, @@ -422,9 +429,10 @@ def test_max_evaluation_time_global(neps_state: NePSState) -> None: settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), - max_evaluations_total=10, # Safety incase it doesn't work that we eventually stop + evaluations_to_spend=10, # Safety incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, - max_cost_total=None, + cost_to_spend=None, + fidelities_to_spend=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=0.5, # <- Highlight max_wallclock_time_for_worker_seconds=None, diff --git a/tests/test_state/test_filebased_neps_state.py b/tests/test_state/test_filebased_neps_state.py index 448b0f393..5572abb4d 100644 --- a/tests/test_state/test_filebased_neps_state.py +++ b/tests/test_state/test_filebased_neps_state.py @@ -20,7 +20,7 @@ @fixture -@parametrize("budget_info", [BudgetInfo(max_cost_total=10, used_cost_budget=0), None]) +@parametrize("budget_info", [BudgetInfo(cost_to_spend=10, used_cost_budget=0), None]) @parametrize("shared_state", [{"a": "b"}, {}]) def optimizer_state( budget_info: BudgetInfo | None, @@ -81,7 +81,7 @@ def test_create_or_load_with_load_filebased_neps_state( # that we prioritize what's in the existing data over what # was passed in. different_state = OptimizationState( - budget=BudgetInfo(max_cost_total=20, used_cost_budget=10), + budget=BudgetInfo(cost_to_spend=20, used_cost_budget=10), seed_snapshot=SeedSnapshot.new_capture(), shared_state=None, ) diff --git a/tests/test_state/test_neps_state.py b/tests/test_state/test_neps_state.py index f25788bd6..bf7512fae 100644 --- a/tests/test_state/test_neps_state.py +++ b/tests/test_state/test_neps_state.py @@ -121,10 +121,9 @@ def case_search_space_fid_with_prior() -> SearchSpace: REQUIRES_FIDELITY_MO = [ "moasha", "mo_hyperband", - "primo", ] -REQUIRES_PRIOR_MO = [ +REQUIRES_MO_PRIOR = [ "primo", ] @@ -157,10 +156,8 @@ def optimizer_and_key_and_search_space( if key in REQUIRES_FIDELITY_MO and search_space.fidelity is None: pytest.xfail(f"Multi-objective optimizer {key} requires a fidelity parameter") - if key in REQUIRES_PRIOR_MO and all( - parameter.prior is None for parameter in search_space.searchables.values() - ): - pytest.xfail(f"Multi-objective optimizer {key} requires a prior") + if key in REQUIRES_MO_PRIOR: + pytest.xfail("No tests defined for PriMO yet") kwargs: dict[str, Any] = {} opt, _ = load_optimizer((key, kwargs), search_space) # type: ignore @@ -168,11 +165,11 @@ def optimizer_and_key_and_search_space( @parametrize("optimizer_info", [OptimizerInfo(name="blah", info={"a": "b"})]) -@parametrize("max_cost_total", [BudgetInfo(max_cost_total=10, used_cost_budget=0), None]) +@parametrize("cost_to_spend", [BudgetInfo(cost_to_spend=10, used_cost_budget=0), None]) @parametrize("shared_state", [{"a": "b"}, {}]) def case_neps_state_filebased( tmp_path: Path, - max_cost_total: BudgetInfo | None, + cost_to_spend: BudgetInfo | None, optimizer_info: OptimizerInfo, shared_state: dict[str, Any], ) -> NePSState: @@ -181,7 +178,7 @@ def case_neps_state_filebased( path=new_path, optimizer_info=optimizer_info, optimizer_state=OptimizationState( - budget=max_cost_total, + budget=cost_to_spend, seed_snapshot=SeedSnapshot.new_capture(), shared_state=shared_state, ), @@ -240,7 +237,7 @@ def test_optimizers_work_roughly( for _ in range(20): trial = ask_and_tell.ask() - if key in (REQUIRES_FIDELITY_MO + REQUIRES_PRIOR_MO): + if key in REQUIRES_FIDELITY_MO: ask_and_tell.tell(trial, [1.0, 2.0]) else: ask_and_tell.tell(trial, 1.0)