From 0c38d215e7cb2dba65012ed197ae64298df90bdc Mon Sep 17 00:00:00 2001 From: Kristina Thieme Date: Thu, 5 Sep 2024 16:51:04 +0200 Subject: [PATCH 01/12] functional, but needs polishing/refactoring --- README.md | 4 +- deepcave/layouts/general.py | 61 +- .../hyperparameter/configuration_cube 2.py | 440 +++ .../hyperparameter/parallel_coordinates 2.py | 431 +++ deepcave/runs/converters/optuna 2.py | 266 ++ docs/plugins/symbolic_explanations 2.rst | 22 + examples/api/importances 2.py | 36 + logs/README 2.rst | 11 + .../run_1/configspace 2.json | 1 + .../run_1/intensifier 2.json | 145 + .../run_1/optimization 2.json | 6 + .../run_1/runhistory 2.json | 2393 +++++++++++++++++ .../run_1/scenario 2.json | 155 ++ .../run_2/configspace 2.json | 1 + .../run_2/intensifier 2.json | 74 + .../run_2/optimization 2.json | 6 + .../run_2/runhistory 2.json | 766 ++++++ .../run_2/scenario 2.json | 155 ++ tests/test_runs/__init__ 2.py | 0 tests/test_utils/__init__ 2.py | 0 20 files changed, 4953 insertions(+), 20 deletions(-) create mode 100644 deepcave/plugins/hyperparameter/configuration_cube 2.py create mode 100644 deepcave/plugins/hyperparameter/parallel_coordinates 2.py create mode 100644 deepcave/runs/converters/optuna 2.py create mode 100644 docs/plugins/symbolic_explanations 2.rst create mode 100644 examples/api/importances 2.py create mode 100644 logs/README 2.rst create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/configspace 2.json create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/intensifier 2.json create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/optimization 2.json create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/runhistory 2.json create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/scenario 2.json create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/configspace 2.json create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/intensifier 2.json create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/optimization 2.json create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/runhistory 2.json create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/scenario 2.json create mode 100644 tests/test_runs/__init__ 2.py create mode 100644 tests/test_utils/__init__ 2.py diff --git a/README.md b/README.md index 3354fcc5..41493d6f 100644 --- a/README.md +++ b/README.md @@ -69,8 +69,8 @@ DeepCAVE comes with some pre-evaluated runs to get a feeling for what DeepCAVE c If you cloned the repository from GitHub via `git clone https://github.com/automl/DeepCAVE.git`, you can try out some examples by exploring the `logs` directory inside the DeepCAVE dashboard. -For example, if you navigate to `logs/DeepCAVE`, you can view the run `mnist_pytorch` if you hit -the `+` button left to it. +For example, if you navigate to `logs/DeepCAVE/mnist_pytorch`, you can view its runs if you hit +the `+` button left to them. ## Features diff --git a/deepcave/layouts/general.py b/deepcave/layouts/general.py index 20c03da9..4f4dd755 100644 --- a/deepcave/layouts/general.py +++ b/deepcave/layouts/general.py @@ -142,8 +142,9 @@ def callback(run_paths: List[str]): # Add text to go to parent directory new_element = html.Div( [ - dbc.Button( - "+", id={"type": "general-dynamic-add-run", "index": -1}, disabled=True + html.I( + className="fas fa-folder-open fa-lg", + id={"type": "general-dynamic-add-run", "index": -1}, ), dbc.Button( "..", @@ -161,23 +162,47 @@ def callback(run_paths: List[str]): for i, run_path in enumerate(run_paths): run_name = run_handler.get_run_name(run_path) - new_element = html.Div( - [ - dbc.Button("+", id={"type": "general-dynamic-add-run", "index": i}), - dbc.Button( - run_name, - id={"type": "general-dynamic-change-directory", "index": i}, - color="link", - ), - dcc.Store( - id={"type": "general-dynamic-available-run-path", "index": i}, - data=run_path, - ), - ], - className="mb-1", - ) - children.append(new_element) + is_run = run_handler.add_run(run_path) + # Differenciate between run and directory for visibility reasons + if is_run: + run_handler.remove_run(run_path) + new_element = html.Div( + [ + dbc.Button( + "+", id={"type": "general-dynamic-add-run", "index": i}, size="sm" + ), + dbc.Button( + run_name, + id={"type": "general-dynamic-change-directory", "index": i}, + color="light", + disabled=True, + ), + dcc.Store( + id={"type": "general-dynamic-available-run-path", "index": i}, + data=run_path, + ), + ], + className="mb-1", + ) + else: + new_element = html.Div( + [ + html.I(className="fas fa-folder fa-lg"), + dbc.Button( + run_name, + id={"type": "general-dynamic-change-directory", "index": i}, + color="link", + ), + dcc.Store( + id={"type": "general-dynamic-available-run-path", "index": i}, + data=run_path, + ), + ], + className="mb-1", + ) + print(new_element) + children.append(new_element) if len(children) == 0: return html.Div("No runs found.") diff --git a/deepcave/plugins/hyperparameter/configuration_cube 2.py b/deepcave/plugins/hyperparameter/configuration_cube 2.py new file mode 100644 index 00000000..ee6a4051 --- /dev/null +++ b/deepcave/plugins/hyperparameter/configuration_cube 2.py @@ -0,0 +1,440 @@ +# noqa: D400 +""" +# ConfigurationCube + +This module provides utilities for visualizing and creating a configuration cube. + +The configuration cube displays configurations and their score on the objective. + +## Classes + - ConfigurationCube: A plugin for visualizing a configuration cube. +""" + +from typing import Any, Callable, Dict, List, Tuple + +import dash_bootstrap_components as dbc +import numpy as np +import pandas as pd +import plotly.graph_objs as go +from dash import dcc, html +from dash.exceptions import PreventUpdate + +from deepcave import config +from deepcave.plugins.dynamic import DynamicPlugin +from deepcave.runs import AbstractRun, Status +from deepcave.utils.compression import deserialize, serialize +from deepcave.utils.layout import ( + get_checklist_options, + get_select_options, + get_slider_marks, + help_button, +) +from deepcave.utils.logs import get_logger +from deepcave.utils.styled_plotty import ( + get_hovertext_from_config, + get_hyperparameter_ticks, + save_image, +) + +logger = get_logger(__name__) + + +class ConfigurationCube(DynamicPlugin): + """A plugin for visualizing a configuration cube.""" + + id = "ccube" + name = "Configuration Cube" + icon = "fas fa-cube" + activate_run_selection = True + help = "docs/plugins/configuration_cube.rst" + + @staticmethod + def get_input_layout(register: Callable) -> List[dbc.Row]: + """ + Get the layout for the input block. + + Parameters + ---------- + register : Callable + Method to register (user) variables. + The register_input function is located in the Plugin superclass. + + Returns + ------- + List[dbc.Row] + Layouts for the input block. + """ + return [ + dbc.Row( + [ + dbc.Col( + [ + dbc.Label("Objective"), + dbc.Select( + id=register("objective_id", ["value", "options"], type=int), + placeholder="Select objective ...", + ), + ], + md=6, + ), + dbc.Col( + [ + dbc.Label("Budget"), + help_button( + "Budget refers to the multi-fidelity budget. " + "Combined budget means that the trial on the highest" + " evaluated budget is used. \n " + "Note: Selecting combined budget might be misleading if" + " a time objective is used. Often, higher budget take " + " longer to evaluate, which might negatively influence " + " the results." + ), + dbc.Select( + id=register("budget_id", ["value", "options"], type=int), + placeholder="Select budget ...", + ), + ], + md=6, + ), + ], + ), + ] + + @staticmethod + def get_filter_layout(register: Callable) -> List[html.Div]: + """ + Get the layout for the filter block. + + Parameters + ---------- + register : Callable + Method to register (user) variables. + The register_input function is located in the Plugin superclass. + + Returns + ------- + List[html.Div] + Layouts for the filter block. + """ + return [ + html.Div( + [ + dbc.Label("Number of Configurations"), + help_button( + "The number of configurations to show, these are ordered based on" + " the time at which they were evaluated." + ), + dcc.Slider( + id=register("n_configs", ["value", "min", "max", "marks"]), step=None + ), + ], + className="mb-3", + ), + html.Div( + [ + dbc.Label("Hyperparameters"), + help_button("Which hyperparameters to show. Maxium 3 active."), + dbc.Checklist( + id=register("hyperparameter_names", ["value", "options"]), inline=True + ), + ] + ), + ] + + def load_inputs(self) -> Dict[str, Any]: + """ + Load the content for the defined inputs in 'get_input_layout' and 'get_filter_layout'. + + This method is necessary to pre-load contents for the inputs. + So, if the plugin is called for the first time or there are no results in the cache, + the plugin gets its content from this method. + + Returns + ------- + Dict[str, Any] + The content to be filled. + """ + return { + "n_configs": {"min": 0, "max": 0, "marks": get_slider_marks(), "value": 0}, + "hyperparameter_names": {"options": get_checklist_options(), "value": []}, + } + + def load_dependency_inputs(self, run, _, inputs) -> Dict[str, Any]: # type: ignore + """ + Work like 'load_inputs' but called after inputs have changed. + + It is restricted to three Hyperparameters. + + Note + ---- + Only the changes have to be returned. + The returned dictionary will be merged with the inputs. + + Parameters + ---------- + run + The selected run. + inputs + Current content of the inputs. + + Returns + ------- + Dict[str, Any] + The dictionary with the changes. + """ + # Prepare objectives + objective_names = run.get_objective_names() + objective_ids = run.get_objective_ids() + objective_options = get_select_options(objective_names, objective_ids) + objective_value = inputs["objective_id"]["value"] + + # Prepare budgets + budgets = run.get_budgets(human=True) + budget_ids = run.get_budget_ids() + budget_options = get_select_options(budgets, budget_ids) + budget_value = inputs["budget_id"]["value"] + + # Prepare others + hp_names = list(run.configspace.keys()) + + # Get selected values + n_configs_value = inputs["n_configs"]["value"] + + # Pre-set values + if objective_value is None: + objective_value = objective_ids[0] + if budget_value is None: + budget_value = budget_ids[-1] + else: + budget_value = inputs["budget_id"]["value"] + + budget = run.get_budget(budget_value) + configs = run.get_configs(budget=budget) + if n_configs_value == 0: + n_configs_value = len(configs) + else: + if n_configs_value > len(configs): + n_configs_value = len(configs) + + # Restrict to three hyperparameters + selected_hps = inputs["hyperparameter_names"]["value"] + n_selected = len(selected_hps) + if n_selected > 3: + del selected_hps[0] + + return { + "objective_id": { + "options": objective_options, + "value": objective_value, + }, + "budget_id": { + "options": budget_options, + "value": budget_value, + }, + "n_configs": { + "min": 0, + "max": len(configs), + "marks": get_slider_marks(list(range(0, len(configs) + 1))), + "value": n_configs_value, + }, + "hyperparameter_names": { + "options": get_select_options(hp_names), + "value": selected_hps, + }, + } + + @staticmethod + def process(run: AbstractRun, inputs: Dict[str, Any]) -> Dict[str, str]: + """ + Return raw data based on the run and input data. + + Warning + ------- + The returned data must be JSON serializable. + + Note + ---- + The passed inputs are cleaned and therefore differs compared to 'load_inputs' + or 'load_dependency_inputs'. + Please see '_clean_inputs' for more information. + + Parameters + ---------- + run : AbstractRun + The selected run. + inputs : Dict[str, Any] + The input data. + + Returns + ------- + Dict[str, str] + The serialized dictionary. + """ + budget = run.get_budget(inputs["budget_id"]) + objective = run.get_objective(inputs["objective_id"]) + + df = run.get_encoded_data( + objectives=objective, budget=budget, statuses=Status.SUCCESS, include_config_ids=True + ) + return {"df": serialize(df)} + + @staticmethod + def get_output_layout(register: Callable) -> Tuple[dcc.Graph,]: + """ + Get the layout for the output block. + + Parameters + ---------- + register : Callable + Method to register output. + The register_output function is located in the Plugin superclass. + + Returns + ------- + Tuple[dcc.Graph,] + Layout for the output block. + """ + return ( + dcc.Graph( + register("graph", "figure"), + style={"height": config.FIGURE_HEIGHT}, + config={"toImageButtonOptions": {"scale": config.FIGURE_DOWNLOAD_SCALE}}, + ), + ) + + @staticmethod + def load_outputs(run, inputs, outputs) -> go.Figure: # type: ignore + """ + Read in the raw data and prepares them for the layout. + + Note + ---- + The passed inputs are cleaned and therefore differs compared to 'load_inputs' + or 'load_dependency_inputs'. + Please see '_clean_inputs' for more information. + + Parameters + ---------- + run + The selected run. + inputs + Input and filter values from the user. + outputs + Raw output from the run. + + Returns + ------- + go.Figure + The output figure. + """ + df = deserialize(outputs["df"], dtype=pd.DataFrame) + hp_names = inputs["hyperparameter_names"] + n_configs = inputs["n_configs"] + objective_id = inputs["objective_id"] + objective = run.get_objective(objective_id) + budget = run.get_budget(inputs["budget_id"]) + df = df.groupby(df.columns.drop(objective.name).to_list(), as_index=False).mean() + df.index = df.index.astype("str") + + # Limit to n_configs + idx = [str(i) for i in range(n_configs, len(df))] + df = df.drop(idx) + + costs = df[objective.name].values.tolist() + config_ids = df["config_id"].values.tolist() + data = [] + + # Specify layout kwargs + layout_kwargs = {} + if n_configs > 0 and len(hp_names) > 0: + for i, (hp_name, axis_name) in enumerate(zip(hp_names, ["xaxis", "yaxis", "zaxis"])): + hp = run.configspace[hp_name] + values = df[hp_name].values.tolist() + + tickvals, ticktext = get_hyperparameter_ticks(hp, ticks=4, include_nan=True) + layout_kwargs[axis_name] = { + "tickvals": tickvals, + "ticktext": ticktext, + "title": hp_name, + } + data.append(values) + + if len(data) == 0: + raise PreventUpdate + + # Transforms data to values + values = np.transpose(np.array(data)).tolist() + + # Now the duplicates are filtered + filtered_data: Dict[Tuple[int, float], Tuple] = {} + for config_id, cost, v in zip(config_ids, costs, values): + v = tuple(v) # Make hashable + key = (config_id, cost) + + if v in filtered_data.values(): + old_key = list(filtered_data.keys())[list(filtered_data.values()).index(v)] + old_cost = old_key[1] + + if objective.optimize == "lower": + if old_cost < cost: + continue + + if objective.optimize == "upper": + if old_cost > cost: + continue + + # Otherwise it is replaced + del filtered_data[old_key] + + filtered_data[key] = v + + # Replace data + config_ids = [k[0] for k in filtered_data.keys()] + costs = [k[1] for k in filtered_data.keys()] + + # Specify scatter kwargs + scatter_kwargs = { + "mode": "markers", + "marker": { + "size": 5, + "color": costs, + "colorbar": {"thickness": 30, "title": objective.name}, + }, + "hovertext": [ + get_hovertext_from_config(run, config_id, budget) for config_id in config_ids + ], + "meta": {"colorbar": costs}, + "hoverinfo": "text", + } + + if len(data) >= 1: + x = [v[0] for v in filtered_data.values()] + if len(data) >= 2: + y = [v[1] for v in filtered_data.values()] + if len(data) >= 3: + z = [v[2] for v in filtered_data.values()] + + if len(data) == 3: + trace = go.Scatter3d(x=x, y=y, z=z, **scatter_kwargs) + layout = go.Layout( + {"scene": {**layout_kwargs}}, + font=dict(size=config.FIGURE_FONT_SIZE), + ) + else: + if len(data) == 1: + y = [0 for _ in x] + trace = go.Scatter(x=x, y=y, **scatter_kwargs) + elif len(data) == 2: + trace = go.Scatter(x=x, y=y, **scatter_kwargs) + else: + trace = go.Scatter(x=[], y=[]) + layout = go.Layout( + **layout_kwargs, + font=dict(size=config.FIGURE_FONT_SIZE), + ) + + figure = go.Figure(data=trace, layout=layout) + figure.update_layout(dict(margin=config.FIGURE_MARGIN)) + save_image(figure, "configuration_cube.pdf") + + return figure diff --git a/deepcave/plugins/hyperparameter/parallel_coordinates 2.py b/deepcave/plugins/hyperparameter/parallel_coordinates 2.py new file mode 100644 index 00000000..b2b0bb84 --- /dev/null +++ b/deepcave/plugins/hyperparameter/parallel_coordinates 2.py @@ -0,0 +1,431 @@ +# noqa: D400 + +""" +# ParallelCoordinates + +This module provides utilities for visualizing the parallel coordinates. + +## Classes + - ParallelCoordinates : Can be used for visualizing the parallel coordinates. +""" + +from typing import Any, Callable, Dict, List + +from collections import defaultdict + +import dash_bootstrap_components as dbc +import numpy as np +import pandas as pd +import plotly.graph_objs as go +from dash import dcc, html +from dash.exceptions import PreventUpdate + +from deepcave import config +from deepcave.constants import VALUE_RANGE +from deepcave.evaluators.fanova import fANOVA +from deepcave.plugins.static import StaticPlugin +from deepcave.utils.compression import deserialize, serialize +from deepcave.utils.layout import get_checklist_options, get_select_options, help_button +from deepcave.utils.logs import get_logger +from deepcave.utils.styled_plotty import get_hyperparameter_ticks, save_image + +logger = get_logger(__name__) + + +class ParallelCoordinates(StaticPlugin): + """Can be used for visualizing the parallel coordinates.""" + + id = "parallel_coordinates" + name = "Parallel Coordinates" + icon = "far fa-map" + activate_run_selection = True + help = "docs/plugins/parallel_coordinates.rst" + + @staticmethod + def get_input_layout(register: Callable) -> List[Any]: + """ + Get the layout for the input block. + + Parameters + ---------- + register : Callable + Method to regsiter (user) variables. + The register_input function is located in the Plugin superclass. + + Returns + ------- + List[Any] + The layouts for the input block. + """ + return [ + dbc.Row( + [ + dbc.Col( + [ + dbc.Label("Objective"), + dbc.Select( + id=register("objective_id", ["value", "options"], type=int), + placeholder="Select objective ...", + ), + ], + md=6, + ), + dbc.Col( + [ + dbc.Label("Budget"), + help_button( + "Budget refers to the multi-fidelity budget. " + "Combined budget means that the trial on the highest evaluated" + " budget is used. \n " + "Note: Selecting combined budget might be misleading if a time" + " objective is used. Often, higher budget take longer to evaluate," + " which might negatively influence the results." + ), + dbc.Select( + id=register("budget_id", ["value", "options"], type=int), + placeholder="Select budget ...", + ), + ], + md=6, + ), + ], + className="mb-3", + ), + html.Div( + [ + dbc.Label("Show Important Hyperparameters"), + help_button( + "Only the most important hyperparameters are shown which are " + "calculated by fANOVA using 10 trees. The more left a " + "hyperparameter stands, the more important it is. However, activating " + "this option might take longer." + ), + dbc.Select( + id=register("show_important_only", ["value", "options"]), + placeholder="Select ...", + ), + ] + ), + ] + + @staticmethod + def get_filter_layout(register: Callable) -> List[Any]: + """ + Get the layout for the filter block. + + Parameters + ---------- + register : Callable + Method to register (user) variables. + The register_input function is located in the Plugin superclass. + + Returns + ------- + List[Any] + The layouts for the filter block. + """ + return [ + dbc.Row( + [ + dbc.Col( + [ + dbc.Label("Limit Hyperparameters"), + help_button( + "Shows either the n most important hyperparameters (if show " + "important hyperparameters is true) or the first n selected " + "hyperparameters." + ), + dbc.Input(id=register("n_hps", "value"), type="number"), + ], + md=6, + ), + dbc.Col( + [ + dbc.Label("Show Unsuccessful Configurations"), + help_button("Whether to show all configurations or only failed ones."), + dbc.Select( + id=register("show_unsuccessful", ["value", "options"]), + placeholder="Select ...", + ), + ], + md=6, + ), + ], + ), + html.Div( + [ + dbc.Label("Hyperparameters"), + dbc.Checklist( + id=register("hyperparameter_names", ["value", "options"]), inline=True + ), + ], + className="mt-3", + id=register("hide_hps", ["hidden"]), + ), + ] + + def load_inputs(self) -> Dict[str, Dict[str, Any]]: + """ + Load the content for the defined inputs in 'get_input_layout' and 'get_filter_layout'. + + This method is necessary to pre-load contents for the inputs. + So, if the plugin is called for the first time or there are no results in the cache, + the plugin gets its content from this method. + + Returns + ------- + Dict[str, Dict[str, Any]] + Content to be filled. + """ + return { + "show_important_only": {"options": get_select_options(binary=True), "value": "true"}, + "show_unsuccessful": {"options": get_select_options(binary=True), "value": "false"}, + "n_hps": {"value": 0}, + "hyperparameter_names": {"options": get_checklist_options(), "value": []}, + "hide_hps": {"hidden": True}, + } + + def load_dependency_inputs(self, run, _, inputs) -> Dict[str, Any]: # type: ignore + """ + Work like 'load_inputs' but called after inputs have changed. + + Note + ---- + Only the changes have to be returned. + The returned dictionary will be merged with the inputs. + + Parameters + ---------- + run + The selected run. + inputs + Current content of the inputs. + + Returns + ------- + Dict[str, Any] + The dictionary with the changes. + """ + # Prepare objectives + objective_names = run.get_objective_names() + objective_ids = run.get_objective_ids() + objective_options = get_select_options(objective_names, objective_ids) + objective_value = inputs["objective_id"]["value"] + + # Prepare budgets + budgets = run.get_budgets(human=True) + budget_ids = run.get_budget_ids() + budget_options = get_checklist_options(budgets, budget_ids) + budget_value = inputs["budget_id"]["value"] + + # Prepare others + n_hps = inputs["n_hps"]["value"] + hp_names = list(run.configspace.keys()) + + if inputs["show_important_only"]["value"] == "true": + hp_options = [] + hp_value = inputs["hyperparameter_names"]["value"] + hidden = True + else: + hp_options = get_select_options(hp_names) + + values = inputs["hyperparameter_names"]["value"] + if len(values) == 0: + values = hp_names + + hp_value = values + hidden = False + + if objective_value is None: + objective_value = objective_ids[0] + budget_value = budget_ids[-1] + hp_value = hp_names + + if n_hps == 0: + n_hps = len(hp_names) + + return { + "objective_id": { + "options": objective_options, + "value": objective_value, + }, + "budget_id": { + "options": budget_options, + "value": budget_value, + }, + "hyperparameter_names": { + "options": hp_options, + "value": hp_value, + }, + "n_hps": {"value": n_hps}, + "hide_hps": {"hidden": hidden}, + } + + @staticmethod + def process(run, inputs) -> Dict[str, Any]: # type: ignore + """ + Return raw data based on a run and input data. + + Warning + ------- + The returned data must be JSON serializable. + + Note + ---- + The passed inputs are cleaned and therefore differs compared to 'load_inputs' + or 'load_dependency_inputs'. + Please see '_clean_inputs' for more information. + + Parameters + ---------- + run : AbstractRun + The run to process. + inputs : Dict[str, Any] + The input data. + + Returns + ------- + Dict[str, Any] + The serialized dictionary. + """ + budget = run.get_budget(inputs["budget_id"]) + objective = run.get_objective(inputs["objective_id"]) + df = run.get_encoded_data(objective, budget) + df = df.groupby(df.columns.drop(objective.name).to_list(), as_index=False).mean() + df = serialize(df) + result: Dict[str, Any] = {"df": df} + + if inputs["show_important_only"]: + # Let's run a quick fANOVA here + evaluator = fANOVA(run) + evaluator.calculate(objective, budget, n_trees=10, seed=0) + importances_dict = evaluator.get_importances() + importances = {u: v[0] for u, v in importances_dict.items()} + important_hp_names = sorted( + importances, key=lambda key: importances[key], reverse=False + ) + result["important_hp_names"] = important_hp_names + + return result + + @staticmethod + def get_output_layout(register: Callable) -> dcc.Graph: + """ + Get the layout for the output block. + + Parameters + ---------- + register : Callable + Method to register outputs. + The register_output function is located in the Plugin superclass. + + Returns + ------- + dcc.Graph + The layouts for the output block. + """ + return dcc.Graph( + register("graph", "figure"), + style={"height": config.FIGURE_HEIGHT}, + config={"toImageButtonOptions": {"scale": config.FIGURE_DOWNLOAD_SCALE}}, + ) + + @staticmethod + def load_outputs(run, inputs, outputs) -> go.Figure: # type: ignore + """ + Read in the raw data and prepare them for the layout. + + Note + ---- + The passed inputs are cleaned and therefore differs compared to 'load_inputs' + or 'load_dependency_inputs'. + Please see '_clean_inputs' for more information. + + Parameters + ---------- + run + The selected run. + inputs + The inputs and filter values fromt the user. + outputs + Raw output from the run. + + Returns + ------- + go.Figure + The output figure. + """ + objective = run.get_objective(inputs["objective_id"]) + objective_name = objective.name + + show_important_only = inputs["show_important_only"] + show_unsuccessful = inputs["show_unsuccessful"] + n_hps = inputs["n_hps"] + + if n_hps == "" or n_hps is None: + raise PreventUpdate + else: + n_hps = int(n_hps) + + if show_important_only: + hp_names = outputs["important_hp_names"] + else: + hp_names = inputs["hyperparameter_names"] + + hp_names = hp_names[:n_hps] + + df = outputs["df"] + df = deserialize(df, dtype=pd.DataFrame) + objective_values = [] + for value in df[objective_name].values: + b = np.isnan(value) + if not show_unsuccessful: + b = not b + if b: + objective_values += [value] + + data: defaultdict = defaultdict(dict) + for hp_name in hp_names: + values = [] + for hp_v, objective_v in zip(df[hp_name].values, df[objective_name].values): + b = np.isnan(objective_v) + if not show_unsuccessful: + b = not b + if b: + values += [hp_v] + + data[hp_name]["values"] = values + data[hp_name]["label"] = hp_name + data[hp_name]["range"] = VALUE_RANGE + + hp = run.configspace[hp_name] + tickvals, ticktext = get_hyperparameter_ticks(hp, ticks=4, include_nan=True) + + data[hp_name]["tickvals"] = tickvals + data[hp_name]["ticktext"] = ticktext + + if show_unsuccessful: + line = dict() + else: + data[objective_name]["values"] = objective_values + data[objective_name]["label"] = objective_name + line = dict( + color=data[objective_name]["values"], + showscale=True, + colorscale="aggrnyl", + ) + + figure = go.Figure( + data=go.Parcoords( + line=line, + dimensions=list([d for d in data.values()]), + labelangle=45, + ), + layout=dict( + margin=dict(t=150, b=50, l=100, r=0), + font=dict(size=config.FIGURE_FONT_SIZE), + ), + ) + save_image(figure, "parallel_coordinates.pdf") + + return figure diff --git a/deepcave/runs/converters/optuna 2.py b/deepcave/runs/converters/optuna 2.py new file mode 100644 index 00000000..a6cd1497 --- /dev/null +++ b/deepcave/runs/converters/optuna 2.py @@ -0,0 +1,266 @@ +# noqa: D400 +""" +# OptunaRun + +This module provides utilities to create an Optuna run. + +## Classes + - OptunaRun: Define an Optuna run object. +""" + +from typing import List, Union + +import pickle +from pathlib import Path + +from ConfigSpace import ( + Categorical, + Configuration, + ConfigurationSpace, + Float, + Integer, + Uniform, +) +from ConfigSpace.hyperparameters import Hyperparameter +from optuna.distributions import ( + CategoricalDistribution, + FloatDistribution, + IntDistribution, +) +from optuna.search_space import IntersectionSearchSpace +from optuna.study import StudyDirection +from optuna.trial import TrialState + +from deepcave.runs import Status +from deepcave.runs.objective import Objective +from deepcave.runs.run import Run +from deepcave.utils.hash import file_to_hash +from deepcave.utils.logs import get_logger + +logger = get_logger(__name__) + + +class OptunaRun(Run): + """ + Define an Optuna run object. + + Properties + ---------- + path : Path + The path to the run. + """ + + prefix = "Optuna" + _initial_order = 2 + + @staticmethod + def _get_pickle_file(path: Path) -> Path: + """ + Get the path to the pickle file from the directory path. + + Parameters + ---------- + path : Path + The path to the directory containing the pickle file. + + Returns + ------- + Path + The path to the pickle file. + """ + pickle_files = list(path.glob("*.pkl")) + if len(pickle_files) != 1: + raise RuntimeError("There should be exactly one pickle file in the directory.") + else: + return pickle_files[0] + + @property + def hash(self) -> str: + """ + Hash of the current run. + + If the hash changes, the cache has to be cleared. + This ensures that the cache always holds the latest results of the run. + + Returns + ------- + str + The hash of the run. + """ + if self.path is None: + return "" + + pickle_file = OptunaRun._get_pickle_file(self.path) + return file_to_hash(pickle_file) + + @property + def latest_change(self) -> Union[float, int]: + """ + Get the timestamp of the latest change. + + Returns + ------- + Union[float, int] + The latest change. + """ + if self.path is None: + return 0 + + pickle_file = OptunaRun._get_pickle_file(self.path) + return Path(pickle_file).stat().st_mtime + + @classmethod + def from_path(cls, path: Union[Path, str]) -> "OptunaRun": + """ + Based on working_dir/run_name/*, return a new trials object. + + Parameters + ---------- + path : Union[Path, str] + The path to base the trial object on. + + Returns + ------- + The Optuna run. + + Raises + ------ + RuntimeError + Instances are not supported. + """ + path = Path(path) + + # Load the optuna study from the file path + pickle_file_path = OptunaRun._get_pickle_file(path) + with open(pickle_file_path, "rb") as f: + optuna_study = pickle.load(f) + + # Read configspace + optuna_space = IntersectionSearchSpace(include_pruned=True).calculate(study=optuna_study) + configspace = ConfigurationSpace() + + hyperparameters: List[Hyperparameter] = [] + + for hp_name, hp in optuna_space.items(): + if isinstance(hp, FloatDistribution) or isinstance(hp, IntDistribution): + if hp.step is not None and hp.step != 1: + logger.warning( + f"Step is not supported. " + f'Step={hp.step} will be ignored for hyperparameter "{hp_name}".' + ) + + if isinstance(hp, FloatDistribution): + hyperparameters.append( + Float( + name=hp_name, + bounds=(hp.low, hp.high), + distribution=Uniform(), + default=None, + log=hp.log, + ) + ) + elif isinstance(hp, IntDistribution): + hyperparameters.append( + Integer( + name=hp_name, + bounds=(hp.low, hp.high), + distribution=Uniform(), + default=None, + log=hp.log, + ) + ) + elif isinstance(hp, CategoricalDistribution): + hyperparameters.append( + Categorical( + name=hp_name, + default=hp.choices[0], + items=hp.choices, + ) + ) + else: + raise ValueError( + ( + "The hyperparameters in the Optuna study must be of type " + "`FloatDistribution`, `IntDistribution` or `CategoricalDistribution`, " + f"but a hyperparameter of type {type(hp)} was given." + ) + ) + configspace.add(hyperparameters) + + n_objectives = max(len(trial.values) for trial in optuna_study.trials) + obj_list = list() + for i in range(n_objectives): + if optuna_study.metric_names is not None: + metric_name = optuna_study.metric_names[i] + else: + metric_name = f"Objective{i}" + optimize = "lower" if optuna_study.directions[i] == StudyDirection.MINIMIZE else "upper" + + obj_list.append( + Objective( + name=metric_name, + lower=None, + upper=None, + optimize=optimize, + ) + ) + + obj_list.append(Objective("Time")) + + # Let's create a new run object + run = OptunaRun(name=path.stem, configspace=configspace, objectives=obj_list, meta=None) + + # The path has to be set manually + run._path = path + + first_starttime = None + for trial in optuna_study.trials: + try: + config = Configuration(configspace, trial.params) + except ValueError: + raise ValueError( + f"Could not convert the configuration of trial {trial.number} to " + f"a ConfigSpace configuration.\nThis might be due to the " + f"configuration space containing conditionals or dynamic " + f"hyperparameter value ranges, which are currently not supported." + ) + + if first_starttime is None: + first_starttime = trial.datetime_start.timestamp() + + starttime = trial.datetime_start.timestamp() - first_starttime + endtime = trial.datetime_complete.timestamp() - first_starttime + + if trial.state == TrialState.COMPLETE: + status = Status.SUCCESS + elif trial.state == TrialState.FAIL: + status = Status.FAILED + elif trial.state == TrialState.PRUNED: + status = Status.PRUNED + elif trial.state == TrialState.RUNNING or trial["status"] == TrialState.WAITING: + continue + else: + status = Status.UNKNOWN + + cost = trial.values + + if status != Status.SUCCESS: + # Costs which failed, should not be included + cost = [None] * len(cost) if isinstance(cost, list) else None + time = None + else: + time = float(endtime - starttime) + + run.add( + costs=cost + [time] if isinstance(cost, list) else [cost, time], # type: ignore + config=config, + budget=0.0, + seed=-1, + start_time=starttime, + end_time=endtime, + status=status, + origin=None, + additional=None, + ) + + return run diff --git a/docs/plugins/symbolic_explanations 2.rst b/docs/plugins/symbolic_explanations 2.rst new file mode 100644 index 00000000..1b78e02b --- /dev/null +++ b/docs/plugins/symbolic_explanations 2.rst @@ -0,0 +1,22 @@ +Symbolic Explanations +==================== + +Symbolic Explanations allow to obtain explicit formulas quantifying the relation between +hyperparameter values and model performance by applying symbolic regression to meta-data collected +during hyperparameter optimization. + +The plugin is capable of answering similar questions as the Partial Dependencies plugin, i.e.: + +* How does the objective change with respect to one or two hyperparameters? For example, does the + accuracy increase if the learning rate decreases? +* Do multiple trials show similar behavior? + +While the Partial Dependencies plugin provides a plot describing the effects of hyperparameters on +the model performance, the Symbolic Explanations plugin additionally allows to obtain an explicit +formula capturing these effects. + +To learn more about Symbolic Explanations, please see the paper +`Symbolic Explanations for Hyperparameter Optimization +`_. + +.. image:: ../images/plugins/symbolic_explanations.png diff --git a/examples/api/importances 2.py b/examples/api/importances 2.py new file mode 100644 index 00000000..de4dad38 --- /dev/null +++ b/examples/api/importances 2.py @@ -0,0 +1,36 @@ +""" +Importances +^^^^^^^^^^^^^^^^^^^^ + +This example shows how to use the plugin Importances. +Note that other plugins use the same interfaces and can be used in the same fashion. +""" + +from deepcave.plugins.hyperparameter.importances import Importances +from deepcave.runs.converters.deepcave import DeepCAVERun +from pathlib import Path + + +if __name__ == "__main__": + # Instantiate the run + run = DeepCAVERun.from_path(Path("logs/DeepCAVE/minimal/run_2")) + + objective_id = run.get_objective_ids()[0] + budget_ids = run.get_budget_ids() + + # Instantiate the plugin + plugin = Importances() + inputs = plugin.generate_inputs( + hyperparameter_names=list(run.configspace.keys()), + objective_id=objective_id, + budget_ids=budget_ids, + method="global", + n_hps=3, + n_trees=10, + ) + # Note: Filter variables are not considered. + outputs = plugin.generate_outputs(run, inputs) + + # Finally, you can load the figure. Here, the filter variables play a role. + figure = plugin.load_outputs(run, inputs, outputs) + figure.write_image("examples/api/importances.png", scale=2.0) diff --git a/logs/README 2.rst b/logs/README 2.rst new file mode 100644 index 00000000..2cadd357 --- /dev/null +++ b/logs/README 2.rst @@ -0,0 +1,11 @@ +Example Logs +============ + +This directory contains exemplary run logs from hyperparameter optimization runs created with +AMLTK, BOHB, Optuna, and SMAC3. +Furthermore, it contains example logs created with the DeepCAVE recorder and example logs in +CSV format. + +After starting DeepCAVE, you can load these logs by navigating to the directory containing the run +log you would like to load on the `General` site and then selecting the run log via clicking on +the `+` next to it. \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/configspace 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/configspace 2.json new file mode 100644 index 00000000..be8aa231 --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_1/configspace 2.json @@ -0,0 +1 @@ +{"name": null, "hyperparameters": [{"type": "categorical", "name": "activation", "choices": ["logistic", "tanh", "relu"], "weights": null, "default_value": "tanh", "meta": null}, {"type": "uniform_int", "name": "n_layer", "lower": 1, "upper": 5, "default_value": 1, "log": false, "meta": null}, {"type": "uniform_int", "name": "n_neurons", "lower": 8, "upper": 256, "default_value": 10, "log": true, "meta": null}, {"type": "categorical", "name": "solver", "choices": ["lbfgs", "sgd", "adam"], "weights": null, "default_value": "adam", "meta": null}, {"type": "uniform_int", "name": "batch_size", "lower": 30, "upper": 300, "default_value": 200, "log": false, "meta": null}, {"type": "categorical", "name": "learning_rate", "choices": ["constant", "invscaling", "adaptive"], "weights": null, "default_value": "constant", "meta": null}, {"type": "uniform_float", "name": "learning_rate_init", "lower": 0.0001, "upper": 1.0, "default_value": 0.001, "log": true, "meta": null}], "conditions": [{"type": "IN", "child": "batch_size", "parent": "solver", "values": ["sgd", "adam"]}, {"type": "EQ", "child": "learning_rate", "parent": "solver", "value": "sgd"}, {"type": "IN", "child": "learning_rate_init", "parent": "solver", "values": ["sgd", "adam"]}], "forbiddens": [], "python_module_version": "1.2.0", "format_version": 0.4} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/intensifier 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/intensifier 2.json new file mode 100644 index 00000000..d7b218ec --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_1/intensifier 2.json @@ -0,0 +1,145 @@ +{ + "incumbent_ids": [ + 20 + ], + "rejected_config_ids": [ + 1, + 6, + 8, + 3, + 10, + 4, + 12, + 13, + 7, + 2, + 15, + 17, + 18, + 19, + 14, + 24, + 21, + 9, + 22, + 16, + 26, + 29, + 28, + 31, + 32, + 34, + 27, + 36, + 37, + 35, + 23, + 11, + 38, + 39, + 43, + 5, + 41, + 40, + 42, + 45, + 46, + 48, + 30, + 50, + 52, + 53, + 55, + 54, + 44, + 57, + 56, + 25, + 49 + ], + "incumbents_changed": 5, + "trajectory": [ + { + "config_ids": [ + 1 + ], + "costs": [ + 0.6699473847106159 + ], + "trial": 1, + "walltime": 0.4147176742553711 + }, + { + "config_ids": [ + 8 + ], + "costs": [ + 0.12855926957598263 + ], + "trial": 2, + "walltime": 0.7400827407836914 + }, + { + "config_ids": [ + 2 + ], + "costs": [ + 0.025597338285360594 + ], + "trial": 5, + "walltime": 1.215728759765625 + }, + { + "config_ids": [ + 14 + ], + "costs": [ + 0.02392757660167144 + ], + "trial": 13, + "walltime": 2.366025686264038 + }, + { + "config_ids": [ + 20 + ], + "costs": [ + 0.02115031466006397 + ], + "trial": 37, + "walltime": 8.94394326210022 + } + ], + "state": { + "queue": [ + [ + 33, + 4 + ], + [ + 51, + 4 + ], + [ + 58, + 2 + ], + [ + 47, + 4 + ], + [ + 59, + 2 + ], + [ + 60, + 2 + ], + [ + 61, + 2 + ] + ] + } +} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/optimization 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/optimization 2.json new file mode 100644 index 00000000..cd5ebb87 --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_1/optimization 2.json @@ -0,0 +1,6 @@ +{ + "used_walltime": 67.33127307891846, + "used_target_function_walltime": 494.0225176811218, + "last_update": 1712565076.753763, + "finished": true +} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/runhistory 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/runhistory 2.json new file mode 100644 index 00000000..5ab1ec0f --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_1/runhistory 2.json @@ -0,0 +1,2393 @@ +{ + "stats": { + "submitted": 154, + "finished": 154, + "running": 0 + }, + "data": [ + [ + 1, + null, + 209652396, + null, + 0.6699473847106159, + 0.32405996322631836, + 1, + 1712565009.4599066, + 1712565009.7847676, + {} + ], + [ + 2, + null, + 209652396, + null, + 0.025597338285360594, + 1.0813992023468018, + 1, + 1712565009.4976892, + 1712565010.5797143, + {} + ], + [ + 3, + null, + 209652396, + null, + 0.6583333333333333, + 1.1021058559417725, + 1, + 1712565009.5342155, + 1712565010.6368752, + {} + ], + [ + 4, + null, + 209652396, + null, + 0.02560352831940571, + 1.2457053661346436, + 1, + 1712565009.573759, + 1712565010.8200808, + {} + ], + [ + 5, + null, + 209652396, + null, + 0.8992773135252243, + 10.674755573272705, + 1, + 1712565009.611309, + 1712565020.286695, + {} + ], + [ + 6, + null, + 209652396, + null, + 0.19250851129681212, + 0.5179076194763184, + 1, + 1712565009.6513555, + 1712565010.1701534, + {} + ], + [ + 7, + null, + 209652396, + null, + 0.89983286908078, + 1.9669127464294434, + 1, + 1712565009.704494, + 1712565011.67224, + {} + ], + [ + 8, + null, + 209652396, + null, + 0.12855926957598263, + 0.35997724533081055, + 1, + 1712565009.744616, + 1712565010.1052234, + {} + ], + [ + 9, + null, + 209652396, + null, + 0.05064376354069944, + 2.6614692211151123, + 1, + 1712565009.8414927, + 1712565012.5038698, + {} + ], + [ + 1, + null, + 398764591, + null, + 0.6861915815536985, + 0.3299448490142822, + 1, + 1712565010.1649523, + 1712565010.4956682, + {} + ], + [ + 10, + null, + 209652396, + null, + 0.2904967502321263, + 0.49783873558044434, + 1, + 1712565010.2330816, + 1712565010.7317798, + {} + ], + [ + 8, + null, + 398764591, + null, + 0.09072887650882078, + 0.3993697166442871, + 1, + 1712565010.5594473, + 1712565010.959439, + {} + ], + [ + 11, + null, + 209652396, + null, + 0.030058805323429305, + 8.363003253936768, + 1, + 1712565010.6421216, + 1712565019.005703, + {} + ], + [ + 2, + null, + 398764591, + null, + 0.03006035283194064, + 1.5795397758483887, + 1, + 1712565010.7179625, + 1712565012.2989554, + {} + ], + [ + 12, + null, + 209652396, + null, + 0.4312689569792634, + 0.41001152992248535, + 1, + 1712565010.7946854, + 1712565011.2055905, + {} + ], + [ + 13, + null, + 209652396, + null, + 0.8987186629526462, + 0.787147045135498, + 1, + 1712565010.8761115, + 1712565011.6642888, + {} + ], + [ + 14, + null, + 209652396, + null, + 0.02392757660167144, + 0.7371306419372559, + 1, + 1712565011.0186121, + 1712565011.7567017, + {} + ], + [ + 15, + null, + 209652396, + null, + 0.27155834107087595, + 0.5496902465820312, + 1, + 1712565011.2593076, + 1712565011.8098226, + {} + ], + [ + 16, + null, + 209652396, + null, + 0.027825750541627947, + 2.965526819229126, + 1, + 1712565011.714806, + 1712565014.681264, + {} + ], + [ + 17, + null, + 209652396, + null, + 0.2031012070566388, + 0.9088141918182373, + 1, + 1712565011.782877, + 1712565012.692426, + {} + ], + [ + 14, + null, + 398764591, + null, + 0.03616527390900648, + 0.5853779315948486, + 1, + 1712565011.84823, + 1712565012.43451, + {} + ], + [ + 18, + null, + 209652396, + null, + 0.043975549365521616, + 1.1089684963226318, + 1, + 1712565011.9250941, + 1712565013.034664, + {} + ], + [ + 19, + null, + 209652396, + null, + 0.0473026926648098, + 0.8433613777160645, + 1, + 1712565012.3523204, + 1712565013.196589, + {} + ], + [ + 20, + null, + 209652396, + null, + 0.019481584648715633, + 1.379931926727295, + 1, + 1712565012.4961815, + 1712565013.877354, + {} + ], + [ + 14, + null, + 924231285, + null, + 0.034498607242339774, + 0.5644114017486572, + 1, + 1712565012.5570464, + 1712565013.1220315, + {} + ], + [ + 9, + null, + 398764591, + null, + 0.046202414113277634, + 3.3080027103424072, + 1, + 1712565012.7521577, + 1712565016.0607708, + {} + ], + [ + 17, + null, + 398764591, + null, + 0.2303899721448468, + 0.8374693393707275, + 1, + 1712565013.0987794, + 1712565013.9371634, + {} + ], + [ + 18, + null, + 398764591, + null, + 0.04397709687403284, + 1.4642667770385742, + 1, + 1712565013.1837516, + 1712565014.6489024, + {} + ], + [ + 21, + null, + 209652396, + null, + 0.8102460538532962, + 1.399742841720581, + 1, + 1712565013.24768, + 1712565014.6482792, + {} + ], + [ + 19, + null, + 398764591, + null, + 0.04674558960074293, + 0.9420022964477539, + 1, + 1712565013.938809, + 1712565014.881975, + {} + ], + [ + 20, + null, + 398764591, + null, + 0.023378211080160916, + 1.4037482738494873, + 1, + 1712565014.003451, + 1712565015.4077935, + {} + ], + [ + 17, + null, + 924231285, + null, + 0.2531925100588053, + 0.8451352119445801, + 1, + 1712565014.7065277, + 1712565015.5526712, + {} + ], + [ + 16, + null, + 398764591, + null, + 0.02560662333642827, + 3.018314838409424, + 1, + 1712565014.7788284, + 1712565017.7980478, + {} + ], + [ + 18, + null, + 924231285, + null, + 0.033389043639740046, + 1.1432137489318848, + 1, + 1712565014.83027, + 1712565015.9741292, + {} + ], + [ + 21, + null, + 398764591, + null, + 0.8770318786753327, + 1.9530634880065918, + 1, + 1712565014.9338968, + 1712565016.8878474, + {} + ], + [ + 19, + null, + 924231285, + null, + 0.07066542865985759, + 1.0578465461730957, + 1, + 1712565015.468166, + 1712565016.5266423, + {} + ], + [ + 20, + null, + 924231285, + null, + 0.02059114825131536, + 2.661302089691162, + 1, + 1712565015.615147, + 1712565018.2773883, + {} + ], + [ + 22, + null, + 209652396, + null, + 0.04287062828845567, + 1.5211269855499268, + 1, + 1712565016.0558727, + 1712565017.5779333, + {} + ], + [ + 23, + null, + 209652396, + null, + 0.04841999380996587, + 8.354167461395264, + 1, + 1712565016.1641889, + 1712565024.5193043, + {} + ], + [ + 9, + null, + 924231285, + null, + 0.0456313834726092, + 3.8456690311431885, + 1, + 1712565016.6118283, + 1712565020.459012, + {} + ], + [ + 24, + null, + 209652396, + null, + 0.8008805323429279, + 0.46524930000305176, + 1, + 1712565016.9718251, + 1712565017.438463, + {} + ], + [ + 21, + null, + 924231285, + null, + 0.8920519962859795, + 2.354919672012329, + 1, + 1712565017.5199628, + 1712565019.8761735, + {} + ], + [ + 24, + null, + 398764591, + null, + 0.726819870009285, + 0.6692235469818115, + 1, + 1712565017.6583474, + 1712565018.328519, + {} + ], + [ + 22, + null, + 398764591, + null, + 0.032836583101207184, + 1.1873645782470703, + 1, + 1712565017.88121, + 1712565019.0695643, + {} + ], + [ + 16, + null, + 924231285, + null, + 0.02894769421231813, + 3.8686747550964355, + 1, + 1712565018.3620129, + 1712565022.2322352, + {} + ], + [ + 24, + null, + 924231285, + null, + 0.7039430516867843, + 0.5134823322296143, + 1, + 1712565018.45915, + 1712565018.9735632, + {} + ], + [ + 25, + null, + 209652396, + null, + 0.0473026926648098, + 16.41287326812744, + 1, + 1712565019.0678082, + 1712565035.4815993, + {} + ], + [ + 11, + null, + 398764591, + null, + 0.032836583101207184, + 11.42868709564209, + 1, + 1712565019.1586893, + 1712565030.5886083, + {} + ], + [ + 22, + null, + 924231285, + null, + 0.042299597647787124, + 1.5826566219329834, + 1, + 1712565019.2559285, + 1712565020.8400044, + {} + ], + [ + 26, + null, + 209652396, + null, + 0.7974450634478489, + 0.5728342533111572, + 1, + 1712565019.9510424, + 1712565020.5249114, + {} + ], + [ + 27, + null, + 209652396, + null, + 0.09015010832559567, + 4.364055395126343, + 1, + 1712565020.3812418, + 1712565024.7467766, + {} + ], + [ + 5, + null, + 398764591, + null, + 0.9003884246363355, + 14.183544158935547, + 1, + 1712565020.5460055, + 1712565034.730529, + {} + ], + [ + 28, + null, + 209652396, + null, + 0.0984989167440421, + 0.9867184162139893, + 1, + 1712565020.6601338, + 1712565021.6482403, + {} + ], + [ + 26, + null, + 398764591, + null, + 0.6844769421231817, + 0.5313403606414795, + 1, + 1712565020.927448, + 1712565021.4603171, + {} + ], + [ + 29, + null, + 209652396, + null, + 0.8981615598885794, + 0.22849583625793457, + 1, + 1712565021.561606, + 1712565021.7911196, + {} + ], + [ + 26, + null, + 924231285, + null, + 0.823571649644073, + 0.731696605682373, + 1, + 1712565021.7452338, + 1712565022.4780235, + {} + ], + [ + 28, + null, + 398764591, + null, + 0.10796193129062215, + 1.0691678524017334, + 1, + 1712565021.888155, + 1712565022.9592311, + {} + ], + [ + 29, + null, + 398764591, + null, + 0.89983286908078, + 0.24839043617248535, + 1, + 1712565022.327761, + 1712565022.57766, + {} + ], + [ + 30, + null, + 209652396, + null, + 0.8992742185082018, + 10.086802959442139, + 1, + 1712565022.5642378, + 1712565032.651993, + {} + ], + [ + 31, + null, + 209652396, + null, + 0.1001562983596409, + 1.2912817001342773, + 1, + 1712565022.6596029, + 1712565023.9524426, + {} + ], + [ + 29, + null, + 924231285, + null, + 0.9059610027855153, + 0.22541332244873047, + 1, + 1712565023.0410864, + 1712565023.268075, + {} + ], + [ + 28, + null, + 924231285, + null, + 0.09126121943670695, + 1.0557401180267334, + 1, + 1712565023.348831, + 1712565024.4059231, + {} + ], + [ + 32, + null, + 209652396, + null, + 0.14357319715258432, + 0.7567057609558105, + 1, + 1712565024.0348003, + 1712565024.7930315, + {} + ], + [ + 31, + null, + 398764591, + null, + 0.12243268337975866, + 0.9943161010742188, + 1, + 1712565024.4760678, + 1712565025.4713368, + {} + ], + [ + 33, + null, + 209652396, + null, + 0.16188022284122572, + 25.683335304260254, + 1, + 1712565024.600618, + 1712565050.285301, + {} + ], + [ + 23, + null, + 398764591, + null, + 0.051753327143299166, + 7.385228157043457, + 1, + 1712565024.8224359, + 1712565032.2086697, + {} + ], + [ + 27, + null, + 398764591, + null, + 0.1191024450634478, + 3.959066867828369, + 1, + 1712565024.9297316, + 1712565028.8901627, + {} + ], + [ + 32, + null, + 398764591, + null, + 0.135792324357784, + 0.838543176651001, + 1, + 1712565025.5465531, + 1712565026.3867607, + {} + ], + [ + 31, + null, + 924231285, + null, + 0.10852986691426803, + 1.264732837677002, + 1, + 1712565026.4774003, + 1712565027.743719, + {} + ], + [ + 32, + null, + 924231285, + null, + 0.1413370473537604, + 0.9844629764556885, + 1, + 1712565027.8312953, + 1712565028.8173516, + {} + ], + [ + 34, + null, + 209652396, + null, + 0.14081089445991957, + 0.7884180545806885, + 1, + 1712565028.910618, + 1712565029.700526, + {} + ], + [ + 27, + null, + 924231285, + null, + 0.10739863819251005, + 4.667412996292114, + 1, + 1712565029.015657, + 1712565033.6845143, + {} + ], + [ + 35, + null, + 209652396, + null, + 0.04230114515629835, + 3.2498931884765625, + 1, + 1712565029.786574, + 1712565033.0380797, + {} + ], + [ + 34, + null, + 398764591, + null, + 0.16361188486536682, + 0.6655764579772949, + 1, + 1712565030.678432, + 1712565031.34571, + {} + ], + [ + 11, + null, + 924231285, + null, + 0.03283194057567318, + 12.322223424911499, + 1, + 1712565031.4208908, + 1712565043.7444239, + {} + ], + [ + 34, + null, + 924231285, + null, + 0.10627050448777475, + 0.6005818843841553, + 1, + 1712565032.2954042, + 1712565032.8974326, + {} + ], + [ + 23, + null, + 924231285, + null, + 0.06121324667285677, + 7.619780540466309, + 1, + 1712565032.744003, + 1712565040.3652062, + {} + ], + [ + 30, + null, + 398764591, + null, + 0.8948297740637574, + 11.035215616226196, + 1, + 1712565032.9881554, + 1712565044.0248187, + {} + ], + [ + 36, + null, + 209652396, + null, + 0.33456979263385933, + 0.9639077186584473, + 1, + 1712565033.1345205, + 1712565034.1002426, + {} + ], + [ + 35, + null, + 398764591, + null, + 0.04396007428040849, + 3.2995266914367676, + 1, + 1712565033.7999034, + 1712565037.1014593, + {} + ], + [ + 37, + null, + 209652396, + null, + 0.8992773135252244, + 0.7176535129547119, + 1, + 1712565034.1942468, + 1712565034.9135132, + {} + ], + [ + 36, + null, + 398764591, + null, + 0.5536861652739089, + 0.7682046890258789, + 1, + 1712565034.8139176, + 1712565035.583699, + {} + ], + [ + 5, + null, + 924231285, + null, + 0.9003899721448468, + 14.44285535812378, + 1, + 1712565035.018067, + 1712565049.4626539, + {} + ], + [ + 37, + null, + 398764591, + null, + 0.8987202104611576, + 0.7350404262542725, + 1, + 1712565035.604154, + 1712565036.3408573, + {} + ], + [ + 25, + null, + 398764591, + null, + 0.04230114515629835, + 16.373348712921143, + 1, + 1712565035.7092667, + 1712565052.084024, + {} + ], + [ + 36, + null, + 924231285, + null, + 0.4841782729805014, + 0.6209244728088379, + 1, + 1712565036.4488175, + 1712565037.071179, + {} + ], + [ + 37, + null, + 924231285, + null, + 0.89983286908078, + 0.8147656917572021, + 1, + 1712565037.180723, + 1712565037.9965858, + {} + ], + [ + 35, + null, + 924231285, + null, + 0.03784122562674097, + 2.6988790035247803, + 1, + 1712565037.301929, + 1712565040.002422, + {} + ], + [ + 38, + null, + 209652396, + null, + 0.8992757660167131, + 2.005662679672241, + 1, + 1712565038.0986674, + 1712565040.10565, + {} + ], + [ + 39, + null, + 209652396, + null, + 0.10461931290622106, + 1.514883041381836, + 1, + 1712565040.0788782, + 1712565041.5947666, + {} + ], + [ + 40, + null, + 209652396, + null, + 0.023930671618693777, + 2.3705286979675293, + 1, + 1712565040.184661, + 1712565042.556817, + {} + ], + [ + 38, + null, + 398764591, + null, + 0.8987186629526462, + 1.8411645889282227, + 1, + 1712565040.4387646, + 1712565042.2815857, + {} + ], + [ + 41, + null, + 209652396, + null, + 0.5219870009285051, + 2.545016050338745, + 1, + 1712565041.699431, + 1712565044.2464228, + {} + ], + [ + 39, + null, + 398764591, + null, + 0.06066233364283513, + 1.1961593627929688, + 1, + 1712565042.3568745, + 1712565043.5545325, + {} + ], + [ + 38, + null, + 924231285, + null, + 0.8987186629526462, + 2.093280792236328, + 1, + 1712565042.63884, + 1712565044.7334743, + {} + ], + [ + 40, + null, + 398764591, + null, + 0.02560352831940571, + 2.676013469696045, + 1, + 1712565043.6318994, + 1712565046.3088646, + {} + ], + [ + 39, + null, + 924231285, + null, + 0.07399566697616833, + 1.149909496307373, + 1, + 1712565043.8234403, + 1712565044.9747822, + {} + ], + [ + 42, + null, + 209652396, + null, + 0.041186939028164615, + 1.3158142566680908, + 1, + 1712565044.1248553, + 1712565045.4421346, + {} + ], + [ + 30, + null, + 924231285, + null, + 0.9009470752089137, + 10.608931303024292, + 1, + 1712565044.3304265, + 1712565054.9409497, + {} + ], + [ + 41, + null, + 398764591, + null, + 0.4590931600123801, + 2.614492177963257, + 1, + 1712565044.812999, + 1712565047.429369, + {} + ], + [ + 43, + null, + 209652396, + null, + 0.12633859486227172, + 1.004054069519043, + 1, + 1712565045.064711, + 1712565046.0705123, + {} + ], + [ + 44, + null, + 209652396, + null, + 0.15694367069018877, + 6.814570188522339, + 1, + 1712565045.5238674, + 1712565052.3401663, + {} + ], + [ + 42, + null, + 398764591, + null, + 0.03673320953265247, + 1.7806165218353271, + 1, + 1712565046.185416, + 1712565047.967166, + {} + ], + [ + 43, + null, + 398764591, + null, + 0.16193283813060977, + 1.155543565750122, + 1, + 1712565046.4128385, + 1712565047.570328, + {} + ], + [ + 40, + null, + 924231285, + null, + 0.024484679665738307, + 2.6976280212402344, + 1, + 1712565047.529584, + 1712565050.2290654, + {} + ], + [ + 41, + null, + 924231285, + null, + 0.3995605075827917, + 1.997565507888794, + 1, + 1712565047.6774108, + 1712565049.6766438, + {} + ], + [ + 43, + null, + 924231285, + null, + 0.13798978644382542, + 0.9540266990661621, + 1, + 1712565048.050342, + 1712565049.0060203, + {} + ], + [ + 42, + null, + 924231285, + null, + 0.03227793252862887, + 1.4512040615081787, + 1, + 1712565049.0885086, + 1712565050.5413556, + {} + ], + [ + 45, + null, + 209652396, + null, + 0.8753203342618384, + 0.847496747970581, + 1, + 1712565049.543191, + 1712565050.392285, + {} + ], + [ + 46, + null, + 209652396, + null, + 0.2489322191272052, + 0.6393148899078369, + 1, + 1712565049.7643716, + 1712565050.4047034, + {} + ], + [ + 47, + null, + 209652396, + null, + 0.08013927576601676, + 15.927346467971802, + 1, + 1712565050.3042223, + 1712565066.232525, + {} + ], + [ + 33, + null, + 398764591, + null, + 0.1925379139585267, + 23.66932439804077, + 1, + 1712565050.3959663, + 1712565074.0666122, + {} + ], + [ + 48, + null, + 209652396, + null, + 0.8124187558031569, + 0.910470724105835, + 1, + 1712565050.4891617, + 1712565051.4010818, + {} + ], + [ + 45, + null, + 398764591, + null, + 0.8914763231197772, + 0.7565186023712158, + 1, + 1712565050.623894, + 1712565051.3820372, + {} + ], + [ + 46, + null, + 398764591, + null, + 0.35610492107706604, + 0.8270492553710938, + 1, + 1712565050.7091923, + 1712565051.5371573, + {} + ], + [ + 49, + null, + 209652396, + null, + 0.2715707211389663, + 6.957829475402832, + 1, + 1712565051.466912, + 1712565058.4263031, + {} + ], + [ + 48, + null, + 398764591, + null, + 0.764634787991334, + 0.8151421546936035, + 1, + 1712565051.5597835, + 1712565052.3764203, + {} + ], + [ + 45, + null, + 924231285, + null, + 0.8747632311977716, + 0.941584587097168, + 1, + 1712565051.6266928, + 1712565052.5692742, + {} + ], + [ + 46, + null, + 924231285, + null, + 0.10686939028164644, + 0.5962722301483154, + 1, + 1712565052.165133, + 1712565052.762847, + {} + ], + [ + 25, + null, + 924231285, + null, + 0.04730424017332102, + 16.94296383857727, + 1, + 1712565052.4514196, + 1712565069.3960361, + {} + ], + [ + 44, + null, + 398764591, + null, + 0.18422624574435154, + 6.634417295455933, + 1, + 1712565052.544118, + 1712565059.1801126, + {} + ], + [ + 48, + null, + 924231285, + null, + 0.8602785515320335, + 0.7708423137664795, + 1, + 1712565052.6536405, + 1712565053.4260578, + {} + ], + [ + 50, + null, + 209652396, + null, + 0.04842463633549987, + 0.9821035861968994, + 1, + 1712565052.8549998, + 1712565053.8382864, + {} + ], + [ + 51, + null, + 209652396, + null, + 0.03673475704116369, + 10.69734239578247, + 1, + 1712565053.5026176, + 1712565064.2009585, + {} + ], + [ + 52, + null, + 209652396, + null, + 0.9031723924481585, + 0.40129661560058594, + 1, + 1712565053.932928, + 1712565054.335735, + {} + ], + [ + 50, + null, + 398764591, + null, + 0.04452955741256592, + 0.7838237285614014, + 1, + 1712565054.4255831, + 1712565055.2110846, + {} + ], + [ + 52, + null, + 398764591, + null, + 0.9031723924481585, + 0.40575075149536133, + 1, + 1712565055.0314689, + 1712565055.4387197, + {} + ], + [ + 53, + null, + 209652396, + null, + 0.8987202104611576, + 0.95816969871521, + 1, + 1712565055.2899451, + 1712565056.2494361, + {} + ], + [ + 50, + null, + 924231285, + null, + 0.04896626431445372, + 0.717515230178833, + 1, + 1712565055.52647, + 1712565056.2451823, + {} + ], + [ + 52, + null, + 924231285, + null, + 0.9037264004952027, + 0.4451889991760254, + 1, + 1712565056.3189044, + 1712565056.7654727, + {} + ], + [ + 53, + null, + 398764591, + null, + 0.8992757660167131, + 1.097594976425171, + 1, + 1712565056.4288678, + 1712565057.5279367, + {} + ], + [ + 54, + null, + 209652396, + null, + 0.08237233054781812, + 1.8947386741638184, + 1, + 1712565056.8644137, + 1712565058.76014, + {} + ], + [ + 55, + null, + 209652396, + null, + 0.0801470133085731, + 1.0197317600250244, + 1, + 1712565057.613952, + 1712565058.6350822, + {} + ], + [ + 53, + null, + 924231285, + null, + 0.888133704735376, + 0.9786050319671631, + 1, + 1712565058.522568, + 1712565059.502105, + {} + ], + [ + 49, + null, + 398764591, + null, + 0.5036165273909007, + 7.000282049179077, + 1, + 1712565058.730699, + 1712565065.7326648, + {} + ], + [ + 55, + null, + 398764591, + null, + 0.08347415660786128, + 0.8997397422790527, + 1, + 1712565058.8603768, + 1712565059.7616763, + {} + ], + [ + 54, + null, + 398764591, + null, + 0.08682141751779626, + 2.5480194091796875, + 1, + 1712565059.2588532, + 1712565061.808606, + {} + ], + [ + 44, + null, + 924231285, + null, + 0.1307644692045805, + 7.21750807762146, + 1, + 1712565059.5947275, + 1712565066.8132136, + {} + ], + [ + 56, + null, + 209652396, + null, + 0.024493964716805983, + 2.156345844268799, + 1, + 1712565059.8427088, + 1712565062.0000536, + {} + ], + [ + 55, + null, + 924231285, + null, + 0.06342927886103378, + 0.9959497451782227, + 1, + 1712565061.8952117, + 1712565062.892992, + {} + ], + [ + 54, + null, + 924231285, + null, + 0.08180903744970591, + 2.1299283504486084, + 1, + 1712565062.0780683, + 1712565064.2096357, + {} + ], + [ + 56, + null, + 398764591, + null, + 0.027270194986072527, + 2.475947856903076, + 1, + 1712565062.9701085, + 1712565065.4473305, + {} + ], + [ + 57, + null, + 209652396, + null, + 0.8281089445991954, + 0.6765487194061279, + 1, + 1712565064.2821856, + 1712565064.9597, + {} + ], + [ + 51, + null, + 398764591, + null, + 0.04229495512225334, + 10.195435047149658, + 1, + 1712565064.3825853, + 1712565074.5789762, + {} + ], + [ + 58, + null, + 209652396, + null, + 0.8280609718353451, + 7.688541412353516, + 1, + 1712565065.0443525, + 1712565072.7342741, + {} + ], + [ + 57, + null, + 398764591, + null, + 0.8636691426802847, + 1.2278459072113037, + 1, + 1712565065.5273557, + 1712565066.756225, + {} + ], + [ + 56, + null, + 924231285, + null, + 0.02893995666976168, + 2.2545557022094727, + 1, + 1712565065.8200572, + 1712565068.075699, + {} + ], + [ + 49, + null, + 924231285, + null, + 0.48469978334880837, + 6.26426362991333, + 1, + 1712565066.338829, + 1712565072.6052105, + {} + ], + [ + 47, + null, + 398764591, + null, + 0.08179975239863813, + 9.90377926826477, + 1, + 1712565066.839424, + 1712565076.744739, + {} + ], + [ + 57, + null, + 924231285, + null, + 0.8580532342927887, + 0.9290714263916016, + 1, + 1712565066.9271002, + 1712565067.857914, + {} + ], + [ + 59, + null, + 209652396, + null, + 0.07180594243268335, + 0.8639552593231201, + 1, + 1712565067.9361448, + 1712565068.801029, + {} + ], + [ + 60, + null, + 209652396, + null, + 0.025601980810894487, + 1.736633539199829, + 1, + 1712565068.1614459, + 1712565069.8996112, + {} + ], + [ + 61, + null, + 209652396, + null, + 0.7572144846796658, + 4.349518537521362, + 1, + 1712565068.9006317, + 1712565073.2513459, + {} + ], + [ + 59, + null, + 398764591, + null, + 0.09125502940266172, + 1.0945045948028564, + 1, + 1712565069.482275, + 1712565070.5787363, + {} + ] + ], + "configs": { + "1": { + "activation": "logistic", + "n_layer": 4, + "n_neurons": 11, + "solver": "lbfgs" + }, + "2": { + "activation": "tanh", + "n_layer": 1, + "n_neurons": 146, + "solver": "adam", + "batch_size": 77, + "learning_rate_init": 0.000534923804864797 + }, + "3": { + "activation": "relu", + "n_layer": 5, + "n_neurons": 43, + "solver": "sgd", + "batch_size": 157, + "learning_rate": "invscaling", + "learning_rate_init": 0.08903539757637764 + }, + "4": { + "activation": "logistic", + "n_layer": 2, + "n_neurons": 90, + "solver": "adam", + "batch_size": 175, + "learning_rate_init": 0.004156370184967407 + }, + "5": { + "activation": "logistic", + "n_layer": 4, + "n_neurons": 200, + "solver": "sgd", + "batch_size": 204, + "learning_rate": "constant", + "learning_rate_init": 0.016071068084706618 + }, + "6": { + "activation": "relu", + "n_layer": 2, + "n_neurons": 12, + "solver": "sgd", + "batch_size": 119, + "learning_rate": "adaptive", + "learning_rate_init": 0.0022428359458905173 + }, + "7": { + "activation": "relu", + "n_layer": 4, + "n_neurons": 51, + "solver": "adam", + "batch_size": 31, + "learning_rate_init": 0.11617003217050809 + }, + "8": { + "activation": "tanh", + "n_layer": 2, + "n_neurons": 20, + "solver": "lbfgs" + }, + "9": { + "activation": "tanh", + "n_layer": 5, + "n_neurons": 61, + "solver": "lbfgs" + }, + "10": { + "activation": "tanh", + "n_layer": 3, + "n_neurons": 26, + "solver": "sgd", + "batch_size": 233, + "learning_rate": "adaptive", + "learning_rate_init": 0.001104760089338138 + }, + "11": { + "activation": "relu", + "n_layer": 3, + "n_neurons": 214, + "solver": "lbfgs" + }, + "12": { + "activation": "logistic", + "n_layer": 1, + "n_neurons": 18, + "solver": "adam", + "batch_size": 135, + "learning_rate_init": 0.00015265008040823863 + }, + "13": { + "activation": "logistic", + "n_layer": 4, + "n_neurons": 29, + "solver": "adam", + "batch_size": 106, + "learning_rate_init": 0.33335905507564456 + }, + "14": { + "activation": "relu", + "n_layer": 1, + "n_neurons": 84, + "solver": "lbfgs" + }, + "15": { + "activation": "tanh", + "n_layer": 5, + "n_neurons": 8, + "solver": "adam", + "batch_size": 279, + "learning_rate_init": 0.04285190354891313 + }, + "16": { + "activation": "tanh", + "n_layer": 2, + "n_neurons": 122, + "solver": "sgd", + "batch_size": 61, + "learning_rate": "constant", + "learning_rate_init": 0.00630547594663167 + }, + "17": { + "activation": "tanh", + "n_layer": 5, + "n_neurons": 23, + "solver": "adam", + "batch_size": 172, + "learning_rate_init": 0.00040121151561322107 + }, + "18": { + "activation": "tanh", + "n_layer": 2, + "n_neurons": 66, + "solver": "lbfgs" + }, + "19": { + "activation": "relu", + "n_layer": 4, + "n_neurons": 16, + "solver": "sgd", + "batch_size": 66, + "learning_rate": "constant", + "learning_rate_init": 0.005542589894513166 + }, + "20": { + "activation": "logistic", + "n_layer": 1, + "n_neurons": 232, + "solver": "sgd", + "batch_size": 253, + "learning_rate": "adaptive", + "learning_rate_init": 0.06675517414146671 + }, + "21": { + "activation": "logistic", + "n_layer": 3, + "n_neurons": 74, + "solver": "sgd", + "batch_size": 300, + "learning_rate": "adaptive", + "learning_rate_init": 0.0029904795631048244 + }, + "22": { + "activation": "relu", + "n_layer": 1, + "n_neurons": 32, + "solver": "adam", + "batch_size": 45, + "learning_rate_init": 0.012048758384365402 + }, + "23": { + "activation": "relu", + "n_layer": 5, + "n_neurons": 106, + "solver": "lbfgs" + }, + "24": { + "activation": "tanh", + "n_layer": 3, + "n_neurons": 9, + "solver": "adam", + "batch_size": 210, + "learning_rate_init": 0.1549368466139337 + }, + "25": { + "activation": "tanh", + "n_layer": 4, + "n_neurons": 158, + "solver": "sgd", + "batch_size": 143, + "learning_rate": "adaptive", + "learning_rate_init": 0.00147326235260944 + }, + "26": { + "activation": "relu", + "n_layer": 2, + "n_neurons": 10, + "solver": "sgd", + "batch_size": 193, + "learning_rate": "invscaling", + "learning_rate_init": 0.018836814953733322 + }, + "27": { + "activation": "relu", + "n_layer": 4, + "n_neurons": 97, + "solver": "adam", + "batch_size": 249, + "learning_rate_init": 0.00011444921049693927 + }, + "28": { + "activation": "logistic", + "n_layer": 2, + "n_neurons": 38, + "solver": "lbfgs" + }, + "29": { + "activation": "logistic", + "n_layer": 5, + "n_neurons": 13, + "solver": "lbfgs" + }, + "30": { + "activation": "relu", + "n_layer": 3, + "n_neurons": 175, + "solver": "adam", + "batch_size": 271, + "learning_rate_init": 0.4444919521413565 + }, + "31": { + "activation": "tanh", + "n_layer": 3, + "n_neurons": 21, + "solver": "lbfgs" + }, + "32": { + "activation": "logistic", + "n_layer": 1, + "n_neurons": 45, + "solver": "sgd", + "batch_size": 106, + "learning_rate": "invscaling", + "learning_rate_init": 0.03214161586827769 + }, + "33": { + "activation": "logistic", + "n_layer": 5, + "n_neurons": 224, + "solver": "adam", + "batch_size": 290, + "learning_rate_init": 0.004563292654888671 + }, + "34": { + "activation": "tanh", + "n_layer": 2, + "n_neurons": 17, + "solver": "lbfgs" + }, + "35": { + "activation": "relu", + "n_layer": 4, + "n_neurons": 57, + "solver": "adam", + "batch_size": 116, + "learning_rate_init": 0.00035489292891332955 + }, + "36": { + "activation": "logistic", + "n_layer": 1, + "n_neurons": 27, + "solver": "sgd", + "batch_size": 203, + "learning_rate": "adaptive", + "learning_rate_init": 0.7609797301059412 + }, + "37": { + "activation": "logistic", + "n_layer": 4, + "n_neurons": 8, + "solver": "sgd", + "batch_size": 182, + "learning_rate": "adaptive", + "learning_rate_init": 0.00019717209149247594 + }, + "38": { + "activation": "relu", + "n_layer": 2, + "n_neurons": 130, + "solver": "sgd", + "batch_size": 163, + "learning_rate": "constant", + "learning_rate_init": 0.14078837052759072 + }, + "39": { + "activation": "relu", + "n_layer": 4, + "n_neurons": 31, + "solver": "lbfgs" + }, + "40": { + "activation": "tanh", + "n_layer": 2, + "n_neurons": 80, + "solver": "adam", + "batch_size": 260, + "learning_rate_init": 0.01019194380084181 + }, + "41": { + "activation": "tanh", + "n_layer": 3, + "n_neurons": 40, + "solver": "sgd", + "batch_size": 58, + "learning_rate": "adaptive", + "learning_rate_init": 0.00012594180136887576 + }, + "42": { + "activation": "tanh", + "n_layer": 1, + "n_neurons": 94, + "solver": "lbfgs" + }, + "43": { + "activation": "relu", + "n_layer": 5, + "n_neurons": 12, + "solver": "adam", + "batch_size": 232, + "learning_rate_init": 0.0017417895884476993 + }, + "44": { + "activation": "logistic", + "n_layer": 3, + "n_neurons": 137, + "solver": "lbfgs" + }, + "45": { + "activation": "logistic", + "n_layer": 5, + "n_neurons": 54, + "solver": "lbfgs" + }, + "46": { + "activation": "relu", + "n_layer": 2, + "n_neurons": 19, + "solver": "adam", + "batch_size": 185, + "learning_rate_init": 0.039036592000729606 + }, + "47": { + "activation": "tanh", + "n_layer": 4, + "n_neurons": 191, + "solver": "sgd", + "batch_size": 240, + "learning_rate": "constant", + "learning_rate_init": 0.0007368727861046234 + }, + "48": { + "activation": "tanh", + "n_layer": 1, + "n_neurons": 13, + "solver": "sgd", + "batch_size": 88, + "learning_rate": "adaptive", + "learning_rate_init": 0.5025399902870405 + }, + "49": { + "activation": "logistic", + "n_layer": 4, + "n_neurons": 102, + "solver": "lbfgs" + }, + "50": { + "activation": "tanh", + "n_layer": 1, + "n_neurons": 35, + "solver": "adam", + "batch_size": 129, + "learning_rate_init": 0.0034213963211558564 + }, + "51": { + "activation": "relu", + "n_layer": 5, + "n_neurons": 148, + "solver": "lbfgs" + }, + "52": { + "activation": "logistic", + "n_layer": 2, + "n_neurons": 10, + "solver": "sgd", + "batch_size": 294, + "learning_rate": "constant", + "learning_rate_init": 0.00047328481860027055 + }, + "53": { + "activation": "logistic", + "n_layer": 5, + "n_neurons": 20, + "solver": "adam", + "batch_size": 256, + "learning_rate_init": 0.10560248678966794 + }, + "54": { + "activation": "relu", + "n_layer": 3, + "n_neurons": 48, + "solver": "sgd", + "batch_size": 68, + "learning_rate": "adaptive", + "learning_rate_init": 0.00026295202043791785 + }, + "55": { + "activation": "tanh", + "n_layer": 3, + "n_neurons": 14, + "solver": "adam", + "batch_size": 150, + "learning_rate_init": 0.013590087590816565 + }, + "56": { + "activation": "tanh", + "n_layer": 1, + "n_neurons": 167, + "solver": "lbfgs" + }, + "57": { + "activation": "tanh", + "n_layer": 5, + "n_neurons": 15, + "solver": "sgd", + "batch_size": 99, + "learning_rate": "constant", + "learning_rate_init": 0.2144498574918249 + }, + "58": { + "activation": "relu", + "n_layer": 2, + "n_neurons": 243, + "solver": "sgd", + "batch_size": 217, + "learning_rate": "invscaling", + "learning_rate_init": 0.00016793749976650653 + }, + "59": { + "activation": "relu", + "n_layer": 4, + "n_neurons": 24, + "solver": "lbfgs" + }, + "60": { + "activation": "logistic", + "n_layer": 1, + "n_neurons": 62, + "solver": "adam", + "batch_size": 51, + "learning_rate_init": 0.0013063520380558128 + }, + "61": { + "activation": "logistic", + "n_layer": 3, + "n_neurons": 114, + "solver": "adam", + "batch_size": 90, + "learning_rate_init": 0.0520608529688821 + } + }, + "config_origins": { + "1": "Initial Design: Sobol", + "2": "Initial Design: Sobol", + "3": "Initial Design: Sobol", + "4": "Initial Design: Sobol", + "5": "Initial Design: Sobol", + "6": "Initial Design: Sobol", + "7": "Initial Design: Sobol", + "8": "Initial Design: Sobol", + "9": "Initial Design: Sobol", + "10": "Initial Design: Sobol", + "11": "Initial Design: Sobol", + "12": "Initial Design: Sobol", + "13": "Initial Design: Sobol", + "14": "Initial Design: Sobol", + "15": "Initial Design: Sobol", + "16": "Initial Design: Sobol", + "17": "Initial Design: Sobol", + "18": "Initial Design: Sobol", + "19": "Initial Design: Sobol", + "20": "Initial Design: Sobol", + "21": "Initial Design: Sobol", + "22": "Initial Design: Sobol", + "23": "Initial Design: Sobol", + "24": "Initial Design: Sobol", + "25": "Initial Design: Sobol", + "26": "Initial Design: Sobol", + "27": "Initial Design: Sobol", + "28": "Initial Design: Sobol", + "29": "Initial Design: Sobol", + "30": "Initial Design: Sobol", + "31": "Initial Design: Sobol", + "32": "Initial Design: Sobol", + "33": "Initial Design: Sobol", + "34": "Initial Design: Sobol", + "35": "Initial Design: Sobol", + "36": "Initial Design: Sobol", + "37": "Initial Design: Sobol", + "38": "Initial Design: Sobol", + "39": "Initial Design: Sobol", + "40": "Initial Design: Sobol", + "41": "Initial Design: Sobol", + "42": "Initial Design: Sobol", + "43": "Initial Design: Sobol", + "44": "Initial Design: Sobol", + "45": "Initial Design: Sobol", + "46": "Initial Design: Sobol", + "47": "Initial Design: Sobol", + "48": "Initial Design: Sobol", + "49": "Initial Design: Sobol", + "50": "Initial Design: Sobol", + "51": "Initial Design: Sobol", + "52": "Initial Design: Sobol", + "53": "Initial Design: Sobol", + "54": "Initial Design: Sobol", + "55": "Initial Design: Sobol", + "56": "Initial Design: Sobol", + "57": "Initial Design: Sobol", + "58": "Initial Design: Sobol", + "59": "Initial Design: Sobol", + "60": "Initial Design: Sobol", + "61": "Initial Design: Sobol" + } +} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/scenario 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/scenario 2.json new file mode 100644 index 00000000..d6ceef93 --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_1/scenario 2.json @@ -0,0 +1,155 @@ +{ + "name": "bcdb4a7cedc37b0caa608ab570d77c03", + "deterministic": false, + "objectives": "cost", + "crash_cost": Infinity, + "termination_cost_threshold": Infinity, + "walltime_limit": 60, + "cputime_limit": Infinity, + "trial_walltime_limit": null, + "trial_memory_limit": null, + "n_trials": 500, + "use_default_config": false, + "instances": null, + "instance_features": null, + "min_budget": null, + "max_budget": null, + "seed": 0, + "n_workers": 8, + "_meta": { + "facade": { + "name": "HyperparameterOptimizationFacade" + }, + "runner": { + "name": "DaskParallelRunner" + }, + "model": { + "name": "RandomForest", + "types": [ + 3, + 0, + 0, + 3, + 0, + 4, + 0 + ], + "bounds": [ + [ + 3, + NaN + ], + [ + 0, + 1.0 + ], + [ + 0, + 1.0 + ], + [ + 3, + NaN + ], + [ + -1.0, + 1.0 + ], + [ + 4, + NaN + ], + [ + -1.0, + 1.0 + ] + ], + "pca_components": 7, + "n_trees": 10, + "n_points_per_tree": -1, + "ratio_features": 1.0, + "min_samples_split": 2, + "min_samples_leaf": 1, + "max_depth": 1048576, + "eps_purity": 1e-08, + "max_nodes": 1048576, + "bootstrapping": true + }, + "acquisition_maximizer": { + "name": "LocalAndSortedRandomSearch", + "acquisition_function": { + "name": "EI", + "xi": 0.0, + "log": true + }, + "challengers": 10000, + "seed": 0, + "random_search": { + "name": "RandomSearch", + "acquisition_function": { + "name": "EI", + "xi": 0.0, + "log": true + }, + "challengers": 5000, + "seed": 0 + }, + "local_search": { + "name": "LocalSearch", + "acquisition_function": { + "name": "EI", + "xi": 0.0, + "log": true + }, + "challengers": 5000, + "seed": 0, + "max_steps": null, + "n_steps_plateau_walk": 10, + "vectorization_min_obtain": 2, + "vectorization_max_obtain": 64 + } + }, + "acquisition_function": { + "name": "EI", + "xi": 0.0, + "log": true + }, + "intensifier": { + "name": "Intensifier", + "max_incumbents": 10, + "seed": 0 + }, + "initial_design": { + "name": "SobolInitialDesign", + "n_configs": 70, + "n_configs_per_hyperparameter": 10, + "additional_configs": [], + "seed": 0 + }, + "random_design": { + "name": "ProbabilityRandomDesign", + "seed": 0, + "probability": 0.2 + }, + "runhistory_encoder": { + "name": "RunHistoryLogScaledEncoder", + "considered_states": [ + 1, + 2, + 4 + ], + "lower_budget_states": [], + "scale_percentage": 5, + "seed": 0 + }, + "multi_objective_algorithm": null, + "config_selector": { + "name": "ConfigSelector", + "retrain_after": 8, + "retries": 16, + "min_trials": 1 + }, + "version": "2.0.2" + }, + "output_directory": "smac3_output/bcdb4a7cedc37b0caa608ab570d77c03/0" +} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/configspace 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/configspace 2.json new file mode 100644 index 00000000..be8aa231 --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_2/configspace 2.json @@ -0,0 +1 @@ +{"name": null, "hyperparameters": [{"type": "categorical", "name": "activation", "choices": ["logistic", "tanh", "relu"], "weights": null, "default_value": "tanh", "meta": null}, {"type": "uniform_int", "name": "n_layer", "lower": 1, "upper": 5, "default_value": 1, "log": false, "meta": null}, {"type": "uniform_int", "name": "n_neurons", "lower": 8, "upper": 256, "default_value": 10, "log": true, "meta": null}, {"type": "categorical", "name": "solver", "choices": ["lbfgs", "sgd", "adam"], "weights": null, "default_value": "adam", "meta": null}, {"type": "uniform_int", "name": "batch_size", "lower": 30, "upper": 300, "default_value": 200, "log": false, "meta": null}, {"type": "categorical", "name": "learning_rate", "choices": ["constant", "invscaling", "adaptive"], "weights": null, "default_value": "constant", "meta": null}, {"type": "uniform_float", "name": "learning_rate_init", "lower": 0.0001, "upper": 1.0, "default_value": 0.001, "log": true, "meta": null}], "conditions": [{"type": "IN", "child": "batch_size", "parent": "solver", "values": ["sgd", "adam"]}, {"type": "EQ", "child": "learning_rate", "parent": "solver", "value": "sgd"}, {"type": "IN", "child": "learning_rate_init", "parent": "solver", "values": ["sgd", "adam"]}], "forbiddens": [], "python_module_version": "1.2.0", "format_version": 0.4} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/intensifier 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/intensifier 2.json new file mode 100644 index 00000000..57b6338e --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_2/intensifier 2.json @@ -0,0 +1,74 @@ +{ + "incumbent_ids": [ + 4 + ], + "rejected_config_ids": [ + 8, + 1, + 7, + 3, + 13, + 10, + 14, + 12, + 2, + 17, + 6, + 15, + 16, + 9 + ], + "incumbents_changed": 3, + "trajectory": [ + { + "config_ids": [ + 1 + ], + "costs": [ + 0.6699473847106159 + ], + "trial": 1, + "walltime": 0.7647695541381836 + }, + { + "config_ids": [ + 10 + ], + "costs": [ + 0.05601722892809243 + ], + "trial": 14, + "walltime": 5.185967445373535 + }, + { + "config_ids": [ + 4 + ], + "costs": [ + 0.02671515526668729 + ], + "trial": 28, + "walltime": 9.271684408187866 + } + ], + "state": { + "queue": [ + [ + 5, + 2 + ], + [ + 19, + 2 + ], + [ + 11, + 4 + ], + [ + 18, + 4 + ] + ] + } +} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/optimization 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/optimization 2.json new file mode 100644 index 00000000..445f0dfa --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_2/optimization 2.json @@ -0,0 +1,6 @@ +{ + "used_walltime": 15.669282674789429, + "used_target_function_walltime": 102.76791548728943, + "last_update": 1706277639.8103335, + "finished": true +} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/runhistory 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/runhistory 2.json new file mode 100644 index 00000000..5be2d9d8 --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_2/runhistory 2.json @@ -0,0 +1,766 @@ +{ + "stats": { + "submitted": 50, + "finished": 50, + "running": 0 + }, + "data": [ + [ + 1, + null, + 209652396, + null, + 0.6699473847106159, + 0.6026918888092041, + 1, + 1706277624.222565, + 1706277624.8264945, + {} + ], + [ + 2, + null, + 209652396, + null, + 0.025597338285360594, + 3.1097147464752197, + 1, + 1706277624.2967758, + 1706277627.407669, + {} + ], + [ + 3, + null, + 209652396, + null, + 0.6583333333333333, + 2.336819648742676, + 1, + 1706277624.3608909, + 1706277626.6990438, + {} + ], + [ + 4, + null, + 209652396, + null, + 0.02560352831940571, + 2.9831438064575195, + 1, + 1706277624.415114, + 1706277627.3992622, + {} + ], + [ + 5, + null, + 209652396, + null, + 0.8992773135252243, + 14.455097913742065, + 1, + 1706277624.4994059, + 1706277638.9561086, + {} + ], + [ + 6, + null, + 209652396, + null, + 0.3678195605075827, + 4.080667018890381, + 1, + 1706277624.5847695, + 1706277628.6664875, + {} + ], + [ + 7, + null, + 209652396, + null, + 0.8992757660167131, + 1.912712574005127, + 1, + 1706277624.6662445, + 1706277626.5800583, + {} + ], + [ + 8, + null, + 209652396, + null, + 0.8241210151655834, + 0.5521142482757568, + 1, + 1706277624.7443416, + 1706277625.2985408, + {} + ], + [ + 9, + null, + 209652396, + null, + 0.0317254719900959, + 5.659100532531738, + 1, + 1706277624.910818, + 1706277630.5716987, + {} + ], + [ + 1, + null, + 398764591, + null, + 0.6861915815536985, + 0.8483016490936279, + 1, + 1706277625.409777, + 1706277626.2611463, + {} + ], + [ + 10, + null, + 209652396, + null, + 0.056199319096254974, + 0.7128767967224121, + 1, + 1706277626.3443322, + 1706277627.058281, + {} + ], + [ + 1, + null, + 924231285, + null, + 0.7017641597028783, + 0.723477840423584, + 1, + 1706277626.6957843, + 1706277627.4203732, + {} + ], + [ + 7, + null, + 398764591, + null, + 0.8987202104611576, + 1.744448184967041, + 1, + 1706277626.7947023, + 1706277628.5401833, + {} + ], + [ + 3, + null, + 398764591, + null, + 0.8458000619003405, + 1.7513632774353027, + 1, + 1706277627.1429644, + 1706277628.896084, + {} + ], + [ + 10, + null, + 398764591, + null, + 0.056761064685855844, + 0.6708414554595947, + 1, + 1706277627.5016356, + 1706277628.1742969, + {} + ], + [ + 2, + null, + 398764591, + null, + 0.03006035283194064, + 3.3852593898773193, + 1, + 1706277627.64613, + 1706277631.0330572, + {} + ], + [ + 4, + null, + 398764591, + null, + 0.027273290003095085, + 2.520305633544922, + 1, + 1706277627.740861, + 1706277630.2628653, + {} + ], + [ + 11, + null, + 209652396, + null, + 0.03840916125038696, + 7.205214738845825, + 1, + 1706277628.2559705, + 1706277635.4626877, + {} + ], + [ + 10, + null, + 924231285, + null, + 0.05509130300216647, + 0.6167201995849609, + 1, + 1706277628.640889, + 1706277629.2592533, + {} + ], + [ + 7, + null, + 924231285, + null, + 0.9009470752089137, + 1.3385493755340576, + 1, + 1706277628.76265, + 1706277630.102818, + {} + ], + [ + 6, + null, + 398764591, + null, + 0.398457134014237, + 3.8393754959106445, + 1, + 1706277628.9893534, + 1706277632.830835, + {} + ], + [ + 3, + null, + 924231285, + null, + Infinity, + 0.0, + 2, + 1706277629.3351586, + 1706277630.2660835, + { + "traceback": "Traceback (most recent call last):\n File \"/home/skrebs/Projects/Software/SMAC3/smac/runner/target_function_runner.py\", line 184, in run\n rval = self(config_copy, target_function, kwargs)\n File \"/home/skrebs/Projects/Software/SMAC3/smac/runner/target_function_runner.py\", line 257, in __call__\n return algorithm(config, **algorithm_kwargs)\n File \"/home/skrebs/Projects/Software/SMAC3/examples/2_multi_fidelity/1_mlp_epochs.py\", line 107, in train\n score = cross_val_score(classifier, dataset.data, dataset.target, cv=cv, error_score=\"raise\")\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/model_selection/_validation.py\", line 562, in cross_val_score\n cv_results = cross_validate(\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/utils/_param_validation.py\", line 211, in wrapper\n return func(*args, **kwargs)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/model_selection/_validation.py\", line 309, in cross_validate\n results = parallel(\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/utils/parallel.py\", line 65, in __call__\n return super().__call__(iterable_with_config)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 1088, in __call__\n while self.dispatch_one_batch(iterator):\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 901, in dispatch_one_batch\n self._dispatch(tasks)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 819, in _dispatch\n job = self._backend.apply_async(batch, callback=cb)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/_parallel_backends.py\", line 208, in apply_async\n result = ImmediateResult(func)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/_parallel_backends.py\", line 597, in __init__\n self.results = batch()\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 288, in __call__\n return [func(*args, **kwargs)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 288, in \n return [func(*args, **kwargs)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/utils/parallel.py\", line 127, in __call__\n return self.function(*args, **kwargs)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/model_selection/_validation.py\", line 732, in _fit_and_score\n estimator.fit(X_train, y_train, **fit_params)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/base.py\", line 1151, in wrapper\n return fit_method(estimator, *args, **kwargs)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py\", line 753, in fit\n return self._fit(X, y, incremental=False)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py\", line 496, in _fit\n raise ValueError(\nValueError: Solver produced non-finite parameter weights. The input data may contain large values and need to be preprocessed.\n", + "error": "ValueError('Solver produced non-finite parameter weights. The input data may contain large values and need to be preprocessed.')" + } + ], + [ + 12, + null, + 209652396, + null, + 0.10016403590219747, + 1.307952642440796, + 1, + 1706277630.1908402, + 1706277631.4997926, + {} + ], + [ + 13, + null, + 209652396, + null, + 0.89983286908078, + 0.2807650566101074, + 1, + 1706277630.3577998, + 1706277630.6401331, + {} + ], + [ + 4, + null, + 924231285, + null, + 0.02726864747756108, + 2.861912250518799, + 1, + 1706277630.4543989, + 1706277633.3173153, + {} + ], + [ + 14, + null, + 209652396, + null, + 0.3895543175487466, + 0.808574914932251, + 1, + 1706277630.6644979, + 1706277631.4741547, + {} + ], + [ + 9, + null, + 398764591, + null, + 0.032271742494583755, + 5.011699914932251, + 1, + 1706277630.76806, + 1706277635.7808084, + {} + ], + [ + 13, + null, + 398764591, + null, + 0.8886908077994429, + 0.32169437408447266, + 1, + 1706277631.1505885, + 1706277631.4740863, + {} + ], + [ + 2, + null, + 924231285, + null, + 0.029497059733828435, + 2.967978000640869, + 1, + 1706277631.5600839, + 1706277634.529425, + {} + ], + [ + 12, + null, + 398764591, + null, + 0.08014391829155054, + 1.0976686477661133, + 1, + 1706277631.6530569, + 1706277632.7523105, + {} + ], + [ + 14, + null, + 398764591, + null, + 0.495255338904364, + 0.6123650074005127, + 1, + 1706277631.7122657, + 1706277632.3262582, + {} + ], + [ + 13, + null, + 924231285, + null, + 0.9048467966573817, + 0.22585248947143555, + 1, + 1706277632.4026914, + 1706277632.6302176, + {} + ], + [ + 14, + null, + 924231285, + null, + 0.31948467966573824, + 0.7365078926086426, + 1, + 1706277632.7140539, + 1706277633.4522555, + {} + ], + [ + 15, + null, + 209652396, + null, + 0.04341844630145464, + 1.4842357635498047, + 1, + 1706277632.830031, + 1706277634.3161619, + {} + ], + [ + 12, + null, + 924231285, + null, + 0.08125657691117305, + 1.5107638835906982, + 1, + 1706277632.9116373, + 1706277634.4242318, + {} + ], + [ + 6, + null, + 924231285, + null, + 0.36005261528938415, + 3.516299247741699, + 1, + 1706277633.4184535, + 1706277636.9357994, + {} + ], + [ + 16, + null, + 209652396, + null, + 0.05231352522438881, + 1.0166974067687988, + 1, + 1706277633.539396, + 1706277634.5570924, + {} + ], + [ + 17, + null, + 209652396, + null, + 0.15859950479727636, + 0.510662317276001, + 1, + 1706277634.4225478, + 1706277634.9349742, + {} + ], + [ + 15, + null, + 398764591, + null, + 0.043413803775920856, + 1.1191012859344482, + 1, + 1706277634.5313032, + 1706277635.651467, + {} + ], + [ + 18, + null, + 209652396, + null, + 0.03673320953265247, + 1.036933183670044, + 1, + 1706277635.221057, + 1706277636.2595854, + {} + ], + [ + 16, + null, + 398764591, + null, + 0.042296502630764565, + 1.2329812049865723, + 1, + 1706277635.3001854, + 1706277636.5348752, + {} + ], + [ + 17, + null, + 398764591, + null, + 0.1229944289693593, + 0.5270049571990967, + 1, + 1706277635.3790934, + 1706277635.907605, + {} + ], + [ + 19, + null, + 209652396, + null, + 0.04006809037449699, + 1.2701239585876465, + 1, + 1706277635.5504642, + 1706277636.8224409, + {} + ], + [ + 11, + null, + 398764591, + null, + 0.0395094398019189, + 3.738813638687134, + 1, + 1706277635.738447, + 1706277639.4788716, + {} + ], + [ + 15, + null, + 924231285, + null, + 0.036725471990095904, + 1.2338676452636719, + 1, + 1706277635.8669333, + 1706277637.1028805, + {} + ], + [ + 9, + null, + 924231285, + null, + 0.03617301145156304, + 3.797776222229004, + 1, + 1706277635.9984384, + 1706277639.797911, + {} + ], + [ + 17, + null, + 924231285, + null, + 0.11129990714948934, + 0.5645942687988281, + 1, + 1706277636.3598018, + 1706277636.9253795, + {} + ], + [ + 18, + null, + 398764591, + null, + 0.040072732900030994, + 1.1205317974090576, + 1, + 1706277636.6408405, + 1706277637.7630773, + {} + ], + [ + 16, + null, + 924231285, + null, + 0.03450324976787367, + 1.0402038097381592, + 1, + 1706277636.9236588, + 1706277637.9659233, + {} + ], + [ + 19, + null, + 398764591, + null, + 0.038952336737852145, + 0.7655572891235352, + 1, + 1706277637.010183, + 1706277637.7774007, + {} + ] + ], + "configs": { + "1": { + "activation": "logistic", + "n_layer": 4, + "n_neurons": 11, + "solver": "lbfgs" + }, + "2": { + "activation": "tanh", + "n_layer": 1, + "n_neurons": 146, + "solver": "adam", + "batch_size": 77, + "learning_rate_init": 0.000534923804864797 + }, + "3": { + "activation": "relu", + "n_layer": 5, + "n_neurons": 43, + "solver": "sgd", + "batch_size": 157, + "learning_rate": "invscaling", + "learning_rate_init": 0.08903539757637764 + }, + "4": { + "activation": "logistic", + "n_layer": 2, + "n_neurons": 90, + "solver": "adam", + "batch_size": 175, + "learning_rate_init": 0.004156370184967407 + }, + "5": { + "activation": "logistic", + "n_layer": 4, + "n_neurons": 200, + "solver": "sgd", + "batch_size": 204, + "learning_rate": "constant", + "learning_rate_init": 0.016071068084706618 + }, + "6": { + "activation": "tanh", + "n_layer": 4, + "n_neurons": 63, + "solver": "sgd", + "batch_size": 144, + "learning_rate": "invscaling", + "learning_rate_init": 0.0056279320474151686 + }, + "7": { + "activation": "relu", + "n_layer": 5, + "n_neurons": 29, + "solver": "adam", + "batch_size": 173, + "learning_rate_init": 0.503948959867121 + }, + "8": { + "activation": "logistic", + "n_layer": 1, + "n_neurons": 8, + "solver": "adam", + "batch_size": 240, + "learning_rate_init": 0.8212461922256864 + }, + "9": { + "activation": "relu", + "n_layer": 3, + "n_neurons": 118, + "solver": "lbfgs" + }, + "10": { + "activation": "tanh", + "n_layer": 1, + "n_neurons": 35, + "solver": "lbfgs" + }, + "11": { + "activation": "tanh", + "n_layer": 4, + "n_neurons": 80, + "solver": "lbfgs" + }, + "12": { + "activation": "tanh", + "n_layer": 3, + "n_neurons": 27, + "solver": "lbfgs" + }, + "13": { + "activation": "logistic", + "n_layer": 5, + "n_neurons": 16, + "solver": "lbfgs" + }, + "14": { + "activation": "relu", + "n_layer": 4, + "n_neurons": 12, + "solver": "lbfgs" + }, + "15": { + "activation": "tanh", + "n_layer": 1, + "n_neurons": 63, + "solver": "lbfgs" + }, + "16": { + "activation": "relu", + "n_layer": 2, + "n_neurons": 43, + "solver": "lbfgs" + }, + "17": { + "activation": "relu", + "n_layer": 2, + "n_neurons": 18, + "solver": "lbfgs" + }, + "18": { + "activation": "tanh", + "n_layer": 1, + "n_neurons": 82, + "solver": "lbfgs" + }, + "19": { + "activation": "relu", + "n_layer": 1, + "n_neurons": 57, + "solver": "lbfgs" + } + }, + "config_origins": { + "1": "Initial Design: Sobol", + "2": "Initial Design: Sobol", + "3": "Initial Design: Sobol", + "4": "Initial Design: Sobol", + "5": "Initial Design: Sobol", + "6": null, + "7": null, + "8": null, + "9": null, + "10": "Acquisition Function Maximizer: Random Search (sorted)", + "11": "Acquisition Function Maximizer: Random Search (sorted)", + "12": "Acquisition Function Maximizer: Random Search (sorted)", + "13": "Acquisition Function Maximizer: Random Search (sorted)", + "14": "Acquisition Function Maximizer: Random Search (sorted)", + "15": "Acquisition Function Maximizer: Local Search", + "16": "Acquisition Function Maximizer: Local Search", + "17": "Acquisition Function Maximizer: Local Search", + "18": "Acquisition Function Maximizer: Random Search (sorted)", + "19": "Acquisition Function Maximizer: Random Search (sorted)" + } +} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/scenario 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/scenario 2.json new file mode 100644 index 00000000..679a91c6 --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_2/scenario 2.json @@ -0,0 +1,155 @@ +{ + "name": "e262f9639f7cb897f1e597cb5b717d0c", + "deterministic": false, + "objectives": "cost", + "crash_cost": Infinity, + "termination_cost_threshold": Infinity, + "walltime_limit": Infinity, + "cputime_limit": Infinity, + "trial_walltime_limit": null, + "trial_memory_limit": null, + "n_trials": 50, + "use_default_config": false, + "instances": null, + "instance_features": null, + "min_budget": null, + "max_budget": null, + "seed": 0, + "n_workers": 8, + "_meta": { + "facade": { + "name": "HyperparameterOptimizationFacade" + }, + "runner": { + "name": "DaskParallelRunner" + }, + "model": { + "name": "RandomForest", + "types": [ + 3, + 0, + 0, + 3, + 0, + 4, + 0 + ], + "bounds": [ + [ + 3, + NaN + ], + [ + 0, + 1.0 + ], + [ + 0, + 1.0 + ], + [ + 3, + NaN + ], + [ + -1.0, + 1.0 + ], + [ + 4, + NaN + ], + [ + -1.0, + 1.0 + ] + ], + "pca_components": 7, + "n_trees": 10, + "n_points_per_tree": -1, + "ratio_features": 1.0, + "min_samples_split": 2, + "min_samples_leaf": 1, + "max_depth": 1048576, + "eps_purity": 1e-08, + "max_nodes": 1048576, + "bootstrapping": true + }, + "acquisition_maximizer": { + "name": "LocalAndSortedRandomSearch", + "acquisition_function": { + "name": "EI", + "xi": 0.0, + "log": true + }, + "challengers": 10000, + "seed": 0, + "random_search": { + "name": "RandomSearch", + "acquisition_function": { + "name": "EI", + "xi": 0.0, + "log": true + }, + "challengers": 5000, + "seed": 0 + }, + "local_search": { + "name": "LocalSearch", + "acquisition_function": { + "name": "EI", + "xi": 0.0, + "log": true + }, + "challengers": 5000, + "seed": 0, + "max_steps": null, + "n_steps_plateau_walk": 10, + "vectorization_min_obtain": 2, + "vectorization_max_obtain": 64 + } + }, + "acquisition_function": { + "name": "EI", + "xi": 0.0, + "log": true + }, + "intensifier": { + "name": "Intensifier", + "max_incumbents": 10, + "seed": 0 + }, + "initial_design": { + "name": "SobolInitialDesign", + "n_configs": 5, + "n_configs_per_hyperparameter": 10, + "additional_configs": [], + "seed": 0 + }, + "random_design": { + "name": "ProbabilityRandomDesign", + "seed": 0, + "probability": 0.2 + }, + "runhistory_encoder": { + "name": "RunHistoryLogScaledEncoder", + "considered_states": [ + 1, + 2, + 4 + ], + "lower_budget_states": [], + "scale_percentage": 5, + "seed": 0 + }, + "multi_objective_algorithm": null, + "config_selector": { + "name": "ConfigSelector", + "retrain_after": 8, + "retries": 16, + "min_trials": 1 + }, + "version": "2.0.2" + }, + "output_directory": "smac3_output/e262f9639f7cb897f1e597cb5b717d0c/0" +} \ No newline at end of file diff --git a/tests/test_runs/__init__ 2.py b/tests/test_runs/__init__ 2.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_utils/__init__ 2.py b/tests/test_utils/__init__ 2.py new file mode 100644 index 00000000..e69de29b From c6fd863f3e19b365efdc00cf1ad0834f822e6928 Mon Sep 17 00:00:00 2001 From: Kristina Thieme Date: Mon, 7 Oct 2024 13:42:13 +0200 Subject: [PATCH 02/12] added run validation method to each converter --- deepcave/layouts/general.py | 7 +++---- deepcave/runs/converters/amltk.py | 23 +++++++++++++++++++++++ deepcave/runs/converters/bohb.py | 21 +++++++++++++++++++++ deepcave/runs/converters/dataframe.py | 22 ++++++++++++++++++++++ deepcave/runs/converters/deepcave.py | 21 +++++++++++++++++++++ deepcave/runs/converters/optuna 2.py | 22 ++++++++++++++++++++++ deepcave/runs/converters/optuna.py | 22 ++++++++++++++++++++++ deepcave/runs/converters/smac3v1.py | 23 +++++++++++++++++++++++ deepcave/runs/converters/smac3v2.py | 23 +++++++++++++++++++++++ deepcave/runs/handler.py | 20 ++++++++++++++++++++ deepcave/runs/run.py | 19 +++++++++++++++++++ 11 files changed, 219 insertions(+), 4 deletions(-) diff --git a/deepcave/layouts/general.py b/deepcave/layouts/general.py index 4f4dd755..6088d58c 100644 --- a/deepcave/layouts/general.py +++ b/deepcave/layouts/general.py @@ -162,11 +162,11 @@ def callback(run_paths: List[str]): for i, run_path in enumerate(run_paths): run_name = run_handler.get_run_name(run_path) - is_run = run_handler.add_run(run_path) - # Differenciate between run and directory for visibility reasons + is_run = run_handler.is_run(run_path) + # Differenciate between run and directory for visibility and usability reasons if is_run: - run_handler.remove_run(run_path) + # run_handler.remove_run(run_path) new_element = html.Div( [ dbc.Button( @@ -201,7 +201,6 @@ def callback(run_paths: List[str]): ], className="mb-1", ) - print(new_element) children.append(new_element) if len(children) == 0: return html.Div("No runs found.") diff --git a/deepcave/runs/converters/amltk.py b/deepcave/runs/converters/amltk.py index 3f1becef..56017c26 100644 --- a/deepcave/runs/converters/amltk.py +++ b/deepcave/runs/converters/amltk.py @@ -24,6 +24,7 @@ from typing import Optional, Sequence, Union +import os import re from pathlib import Path @@ -215,3 +216,25 @@ def from_path(cls, path: Union[Path, str]) -> "AMLTKRun": ) return run + + @classmethod + def is_valid_run(cls, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid amltk run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + if os.path.isfile(path_name + "/history.parquet") and os.path.isfile( + path_name + "/configspace.json" + ): + return True + return False diff --git a/deepcave/runs/converters/bohb.py b/deepcave/runs/converters/bohb.py index 08b2f37c..4a39f69d 100644 --- a/deepcave/runs/converters/bohb.py +++ b/deepcave/runs/converters/bohb.py @@ -25,6 +25,7 @@ from typing import Any, Dict, Union +import os from pathlib import Path from ConfigSpace.configuration_space import ConfigurationSpace @@ -177,3 +178,23 @@ def from_path(cls, path: Union[Path, str]) -> "BOHBRun": ) return run + + @classmethod + def is_valid_run(cls, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid bohb run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + if os.path.isfile(path_name + "/configspace.json"): + return True + return False diff --git a/deepcave/runs/converters/dataframe.py b/deepcave/runs/converters/dataframe.py index 11e34048..7af0fb73 100644 --- a/deepcave/runs/converters/dataframe.py +++ b/deepcave/runs/converters/dataframe.py @@ -385,3 +385,25 @@ def _extract_additional( additional = data.drop(hyperparameters + costs_metrics + budgets + seeds + meta) additional = dict(additional) return {key: value if pd.notna(value) else None for key, value in additional.items()} + + @classmethod + def is_valid_run(cls, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid dataframe run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + if os.path.isfile(path_name + "/trials.csv") and os.path.isfile( + path_name + "/configspace.csv" + ): + return True + return False diff --git a/deepcave/runs/converters/deepcave.py b/deepcave/runs/converters/deepcave.py index 9fedc619..3e3c6324 100644 --- a/deepcave/runs/converters/deepcave.py +++ b/deepcave/runs/converters/deepcave.py @@ -24,6 +24,7 @@ from typing import Union +import os from pathlib import Path from deepcave.runs.run import Run @@ -92,3 +93,23 @@ def from_path(cls, path: Path) -> "DeepCAVERun": The DeepCAVE run. """ return DeepCAVERun(path.stem, path=Path(path)) + + @classmethod + def is_valid_run(cls, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid deepcave run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + if os.path.isfile(path_name + "history.jsonl"): + return True + return False diff --git a/deepcave/runs/converters/optuna 2.py b/deepcave/runs/converters/optuna 2.py index a6cd1497..d010f130 100644 --- a/deepcave/runs/converters/optuna 2.py +++ b/deepcave/runs/converters/optuna 2.py @@ -264,3 +264,25 @@ def from_path(cls, path: Union[Path, str]) -> "OptunaRun": ) return run + + @classmethod + def is_valid_run(cls, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid optuna2 run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + path = Path(path_name) + pickle_files = list(path.glob("*.pkl")) + if len(pickle_files) != 1: + return False + return True diff --git a/deepcave/runs/converters/optuna.py b/deepcave/runs/converters/optuna.py index 2d570744..bce2159a 100644 --- a/deepcave/runs/converters/optuna.py +++ b/deepcave/runs/converters/optuna.py @@ -285,3 +285,25 @@ def from_path(cls, path: Union[Path, str]) -> "OptunaRun": ) return run + + @classmethod + def is_valid_run(cls, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid optuna run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + path = Path(path_name) + pickle_files = list(path.glob("*.pkl")) + if len(pickle_files) != 1: + return False + return True diff --git a/deepcave/runs/converters/smac3v1.py b/deepcave/runs/converters/smac3v1.py index 21afe20c..c90142da 100644 --- a/deepcave/runs/converters/smac3v1.py +++ b/deepcave/runs/converters/smac3v1.py @@ -27,6 +27,7 @@ from typing import Optional, Union import json +import os from pathlib import Path import numpy as np @@ -217,3 +218,25 @@ def from_path(cls, path: Union[Path, str]) -> "SMAC3v1Run": ) return run + + @classmethod + def is_valid_run(cls, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid smac3v1 run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + if os.path.isfile(path_name + "/runhistory.json") and os.path.isfile( + path_name + "/configspace.json" + ): + return True + return False diff --git a/deepcave/runs/converters/smac3v2.py b/deepcave/runs/converters/smac3v2.py index 0c933526..7ccf0f25 100644 --- a/deepcave/runs/converters/smac3v2.py +++ b/deepcave/runs/converters/smac3v2.py @@ -28,6 +28,7 @@ from typing import Union import json +import os from pathlib import Path import numpy as np @@ -216,3 +217,25 @@ def from_path(cls, path: Union[Path, str]) -> "SMAC3v2Run": ) return run + + @classmethod + def is_valid_run(cls, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid smac3v2 run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + if os.path.isfile(path_name + "/runhistory.json") and os.path.isfile( + path_name + "/configspace.json" + ): + return True + return False diff --git a/deepcave/runs/handler.py b/deepcave/runs/handler.py index 27387348..6b8057c7 100644 --- a/deepcave/runs/handler.py +++ b/deepcave/runs/handler.py @@ -523,3 +523,23 @@ def get_runs(self, include_groups: bool = False) -> List[AbstractRun]: runs += list(self.groups.values()) return runs + + def is_run(self, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + for run_class in self.available_run_classes: + if run_class.is_valid_run(path_name): + return True + return False diff --git a/deepcave/runs/run.py b/deepcave/runs/run.py index 18e3475c..912db646 100644 --- a/deepcave/runs/run.py +++ b/deepcave/runs/run.py @@ -139,6 +139,25 @@ def from_path(cls, path: Path) -> "Run": """ pass + @classmethod + @abstractmethod + def is_valid_run(cls, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + pass + @property def id(self) -> str: """ From 1cf53f378c058f2c11b45e87fc0d0ee4f44961ca Mon Sep 17 00:00:00 2001 From: Kristina Thieme Date: Mon, 7 Oct 2024 14:32:57 +0200 Subject: [PATCH 03/12] added documentation, fixed minor bugs --- CHANGELOG.md | 10 ++++++++++ deepcave/layouts/general.py | 2 +- docs/converters/bohb.rst | 8 +++++++- docs/converters/custom_converters.rst | 19 +++++++++++++++---- docs/converters/deepcave.rst | 8 +++++++- docs/converters/optuna.rst | 3 +++ docs/converters/smac.rst | 11 ++++++++++- 7 files changed, 53 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a8cfbdd..59288d62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +# Version 1.3.3 + +## Converters +- Add for each converter a method to check whether a run is valid. +- Change rst docs of converters. + +## General Layout +- Fix navigation and button problems in general layout. +- Change general layout to be more intuitiv. + # Version 1.3.2 ## Features diff --git a/deepcave/layouts/general.py b/deepcave/layouts/general.py index 6088d58c..3191892b 100644 --- a/deepcave/layouts/general.py +++ b/deepcave/layouts/general.py @@ -145,6 +145,7 @@ def callback(run_paths: List[str]): html.I( className="fas fa-folder-open fa-lg", id={"type": "general-dynamic-add-run", "index": -1}, + style={"pointer-events": "none"}, ), dbc.Button( "..", @@ -166,7 +167,6 @@ def callback(run_paths: List[str]): is_run = run_handler.is_run(run_path) # Differenciate between run and directory for visibility and usability reasons if is_run: - # run_handler.remove_run(run_path) new_element = html.Div( [ dbc.Button( diff --git a/docs/converters/bohb.rst b/docs/converters/bohb.rst index 0f289929..89d1ae0b 100644 --- a/docs/converters/bohb.rst +++ b/docs/converters/bohb.rst @@ -5,4 +5,10 @@ To load an BOHB run into DeepCAVE, it is necessary to install HpBandSter, e.g. v .. code-block:: bash - pip install deepcave[bohb] \ No newline at end of file + pip install deepcave[bohb] + +To be able to load it correctly into DeepCAVE, you need save the configuration space as a ConfigSpace JSON file: + + .. code-block:: python + + "configspace.json" \ No newline at end of file diff --git a/docs/converters/custom_converters.rst b/docs/converters/custom_converters.rst index 015892ca..9f5467ff 100644 --- a/docs/converters/custom_converters.rst +++ b/docs/converters/custom_converters.rst @@ -27,23 +27,31 @@ Step 1: Create the converter class @property def hash(self) -> str: """ - Returns a unique hash for the run (e.g., hashing the trial history). + Return a unique hash for the run (e.g., hashing the trial history). """ pass @property def latest_change(self) -> float: """ - Returns the timestamp of the latest change. + Return the timestamp of the latest change. """ pass @classmethod def from_path(cls, path: str) -> 'Run': """ - Returns a Run object from a given path. + Return a Run object from a given path. """ pass + + @classmethod + def is_valid_run(cls, path: str) -> bool: + """ + Check if the path belongs to a valid run. + """ + pass + Step 2: Register your converter ------------------------------- @@ -63,7 +71,10 @@ Step 3: Implement the methods Inside this method, you have to provide a configspace, objectives, and meta data, as well as the trials, which have to be added to the newly created run object. -Step 4: Document your converter +4. **is_valid_run**: This method sould return True, if the given path belongs to a valid run. Valid means, that the given run + directory contains all files that are necessary for further processing (e.g. a configspace.json). + +Step 5: Document your converter ------------------------------- To document your converter, create a corresponding `.rst` file under `docs/converters` and add it to diff --git a/docs/converters/deepcave.rst b/docs/converters/deepcave.rst index e89ad522..09d2e27e 100644 --- a/docs/converters/deepcave.rst +++ b/docs/converters/deepcave.rst @@ -2,4 +2,10 @@ DeepCAVE --------- You can incorporate DeepCAVE into your AutoML tool directly to save your results in the native -DeepCAVE format and then analyze it in DeepCAVE. Please see the :ref:`examples` on this. \ No newline at end of file +DeepCAVE format and then analyze it in DeepCAVE. Please see the :ref:`examples` on this. + +To be able to load it correctly into DeepCAVE, your run needs to at least contain a history JSONL file: + + .. code-block:: python + + "history.jsonl" \ No newline at end of file diff --git a/docs/converters/optuna.rst b/docs/converters/optuna.rst index 89467972..932a30f5 100644 --- a/docs/converters/optuna.rst +++ b/docs/converters/optuna.rst @@ -14,6 +14,9 @@ consider when running Optuna: Replace ```` with the directory where you want to store the Pickle file. +.. note:: + There has to be exactly one Pickle file. + 2. Optional: If you want to display the names of your objectives in DeepCAVE, you can set metric names in Optuna using the following command: .. code-block:: python diff --git a/docs/converters/smac.rst b/docs/converters/smac.rst index dabb3937..e6f75eda 100644 --- a/docs/converters/smac.rst +++ b/docs/converters/smac.rst @@ -5,4 +5,13 @@ For the SMAC framework, we support loading runs created with the older SMAC vers runs created with the newer SMAC version >= 2.0.0. As Auto-Sklearn and Auto-PyTorch rely on SMAC for hyperparameter optimization, the SMAC -converter also allows to load runs created with these tools. \ No newline at end of file +converter also allows to load runs created with these tools. + +To be able to load it correctly into DeepCAVE, you need save the configuration space as a ConfigSpace JSON file, +as well as the run history: + + .. code-block:: python + + "configspace.json" + "runhistory.json" + From 6bb1eeeba1a8ec2f58e3bc314e71f95dd12264c2 Mon Sep 17 00:00:00 2001 From: Kristina Thieme Date: Tue, 8 Oct 2024 13:09:39 +0200 Subject: [PATCH 04/12] deleted weird duplicated files --- .../hyperparameter/configuration_cube 2.py | 440 --- .../hyperparameter/parallel_coordinates 2.py | 431 --- deepcave/runs/converters/optuna 2.py | 288 -- docs/plugins/symbolic_explanations 2.rst | 22 - docs/sg_execution_times.rst | 58 + examples/api/importances 2.py | 36 - logs/README 2.rst | 11 - .../run_1/configspace 2.json | 1 - .../run_1/intensifier 2.json | 145 - .../run_1/optimization 2.json | 6 - .../run_1/runhistory 2.json | 2393 ----------------- .../run_1/scenario 2.json | 155 -- .../run_2/configspace 2.json | 1 - .../run_2/intensifier 2.json | 74 - .../run_2/optimization 2.json | 6 - .../run_2/runhistory 2.json | 766 ------ .../run_2/scenario 2.json | 155 -- tests/test_runs/__init__ 2.py | 0 tests/test_utils/__init__ 2.py | 0 19 files changed, 58 insertions(+), 4930 deletions(-) delete mode 100644 deepcave/plugins/hyperparameter/configuration_cube 2.py delete mode 100644 deepcave/plugins/hyperparameter/parallel_coordinates 2.py delete mode 100644 deepcave/runs/converters/optuna 2.py delete mode 100644 docs/plugins/symbolic_explanations 2.rst create mode 100644 docs/sg_execution_times.rst delete mode 100644 examples/api/importances 2.py delete mode 100644 logs/README 2.rst delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/configspace 2.json delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/intensifier 2.json delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/optimization 2.json delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/runhistory 2.json delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/scenario 2.json delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/configspace 2.json delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/intensifier 2.json delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/optimization 2.json delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/runhistory 2.json delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/scenario 2.json delete mode 100644 tests/test_runs/__init__ 2.py delete mode 100644 tests/test_utils/__init__ 2.py diff --git a/deepcave/plugins/hyperparameter/configuration_cube 2.py b/deepcave/plugins/hyperparameter/configuration_cube 2.py deleted file mode 100644 index ee6a4051..00000000 --- a/deepcave/plugins/hyperparameter/configuration_cube 2.py +++ /dev/null @@ -1,440 +0,0 @@ -# noqa: D400 -""" -# ConfigurationCube - -This module provides utilities for visualizing and creating a configuration cube. - -The configuration cube displays configurations and their score on the objective. - -## Classes - - ConfigurationCube: A plugin for visualizing a configuration cube. -""" - -from typing import Any, Callable, Dict, List, Tuple - -import dash_bootstrap_components as dbc -import numpy as np -import pandas as pd -import plotly.graph_objs as go -from dash import dcc, html -from dash.exceptions import PreventUpdate - -from deepcave import config -from deepcave.plugins.dynamic import DynamicPlugin -from deepcave.runs import AbstractRun, Status -from deepcave.utils.compression import deserialize, serialize -from deepcave.utils.layout import ( - get_checklist_options, - get_select_options, - get_slider_marks, - help_button, -) -from deepcave.utils.logs import get_logger -from deepcave.utils.styled_plotty import ( - get_hovertext_from_config, - get_hyperparameter_ticks, - save_image, -) - -logger = get_logger(__name__) - - -class ConfigurationCube(DynamicPlugin): - """A plugin for visualizing a configuration cube.""" - - id = "ccube" - name = "Configuration Cube" - icon = "fas fa-cube" - activate_run_selection = True - help = "docs/plugins/configuration_cube.rst" - - @staticmethod - def get_input_layout(register: Callable) -> List[dbc.Row]: - """ - Get the layout for the input block. - - Parameters - ---------- - register : Callable - Method to register (user) variables. - The register_input function is located in the Plugin superclass. - - Returns - ------- - List[dbc.Row] - Layouts for the input block. - """ - return [ - dbc.Row( - [ - dbc.Col( - [ - dbc.Label("Objective"), - dbc.Select( - id=register("objective_id", ["value", "options"], type=int), - placeholder="Select objective ...", - ), - ], - md=6, - ), - dbc.Col( - [ - dbc.Label("Budget"), - help_button( - "Budget refers to the multi-fidelity budget. " - "Combined budget means that the trial on the highest" - " evaluated budget is used. \n " - "Note: Selecting combined budget might be misleading if" - " a time objective is used. Often, higher budget take " - " longer to evaluate, which might negatively influence " - " the results." - ), - dbc.Select( - id=register("budget_id", ["value", "options"], type=int), - placeholder="Select budget ...", - ), - ], - md=6, - ), - ], - ), - ] - - @staticmethod - def get_filter_layout(register: Callable) -> List[html.Div]: - """ - Get the layout for the filter block. - - Parameters - ---------- - register : Callable - Method to register (user) variables. - The register_input function is located in the Plugin superclass. - - Returns - ------- - List[html.Div] - Layouts for the filter block. - """ - return [ - html.Div( - [ - dbc.Label("Number of Configurations"), - help_button( - "The number of configurations to show, these are ordered based on" - " the time at which they were evaluated." - ), - dcc.Slider( - id=register("n_configs", ["value", "min", "max", "marks"]), step=None - ), - ], - className="mb-3", - ), - html.Div( - [ - dbc.Label("Hyperparameters"), - help_button("Which hyperparameters to show. Maxium 3 active."), - dbc.Checklist( - id=register("hyperparameter_names", ["value", "options"]), inline=True - ), - ] - ), - ] - - def load_inputs(self) -> Dict[str, Any]: - """ - Load the content for the defined inputs in 'get_input_layout' and 'get_filter_layout'. - - This method is necessary to pre-load contents for the inputs. - So, if the plugin is called for the first time or there are no results in the cache, - the plugin gets its content from this method. - - Returns - ------- - Dict[str, Any] - The content to be filled. - """ - return { - "n_configs": {"min": 0, "max": 0, "marks": get_slider_marks(), "value": 0}, - "hyperparameter_names": {"options": get_checklist_options(), "value": []}, - } - - def load_dependency_inputs(self, run, _, inputs) -> Dict[str, Any]: # type: ignore - """ - Work like 'load_inputs' but called after inputs have changed. - - It is restricted to three Hyperparameters. - - Note - ---- - Only the changes have to be returned. - The returned dictionary will be merged with the inputs. - - Parameters - ---------- - run - The selected run. - inputs - Current content of the inputs. - - Returns - ------- - Dict[str, Any] - The dictionary with the changes. - """ - # Prepare objectives - objective_names = run.get_objective_names() - objective_ids = run.get_objective_ids() - objective_options = get_select_options(objective_names, objective_ids) - objective_value = inputs["objective_id"]["value"] - - # Prepare budgets - budgets = run.get_budgets(human=True) - budget_ids = run.get_budget_ids() - budget_options = get_select_options(budgets, budget_ids) - budget_value = inputs["budget_id"]["value"] - - # Prepare others - hp_names = list(run.configspace.keys()) - - # Get selected values - n_configs_value = inputs["n_configs"]["value"] - - # Pre-set values - if objective_value is None: - objective_value = objective_ids[0] - if budget_value is None: - budget_value = budget_ids[-1] - else: - budget_value = inputs["budget_id"]["value"] - - budget = run.get_budget(budget_value) - configs = run.get_configs(budget=budget) - if n_configs_value == 0: - n_configs_value = len(configs) - else: - if n_configs_value > len(configs): - n_configs_value = len(configs) - - # Restrict to three hyperparameters - selected_hps = inputs["hyperparameter_names"]["value"] - n_selected = len(selected_hps) - if n_selected > 3: - del selected_hps[0] - - return { - "objective_id": { - "options": objective_options, - "value": objective_value, - }, - "budget_id": { - "options": budget_options, - "value": budget_value, - }, - "n_configs": { - "min": 0, - "max": len(configs), - "marks": get_slider_marks(list(range(0, len(configs) + 1))), - "value": n_configs_value, - }, - "hyperparameter_names": { - "options": get_select_options(hp_names), - "value": selected_hps, - }, - } - - @staticmethod - def process(run: AbstractRun, inputs: Dict[str, Any]) -> Dict[str, str]: - """ - Return raw data based on the run and input data. - - Warning - ------- - The returned data must be JSON serializable. - - Note - ---- - The passed inputs are cleaned and therefore differs compared to 'load_inputs' - or 'load_dependency_inputs'. - Please see '_clean_inputs' for more information. - - Parameters - ---------- - run : AbstractRun - The selected run. - inputs : Dict[str, Any] - The input data. - - Returns - ------- - Dict[str, str] - The serialized dictionary. - """ - budget = run.get_budget(inputs["budget_id"]) - objective = run.get_objective(inputs["objective_id"]) - - df = run.get_encoded_data( - objectives=objective, budget=budget, statuses=Status.SUCCESS, include_config_ids=True - ) - return {"df": serialize(df)} - - @staticmethod - def get_output_layout(register: Callable) -> Tuple[dcc.Graph,]: - """ - Get the layout for the output block. - - Parameters - ---------- - register : Callable - Method to register output. - The register_output function is located in the Plugin superclass. - - Returns - ------- - Tuple[dcc.Graph,] - Layout for the output block. - """ - return ( - dcc.Graph( - register("graph", "figure"), - style={"height": config.FIGURE_HEIGHT}, - config={"toImageButtonOptions": {"scale": config.FIGURE_DOWNLOAD_SCALE}}, - ), - ) - - @staticmethod - def load_outputs(run, inputs, outputs) -> go.Figure: # type: ignore - """ - Read in the raw data and prepares them for the layout. - - Note - ---- - The passed inputs are cleaned and therefore differs compared to 'load_inputs' - or 'load_dependency_inputs'. - Please see '_clean_inputs' for more information. - - Parameters - ---------- - run - The selected run. - inputs - Input and filter values from the user. - outputs - Raw output from the run. - - Returns - ------- - go.Figure - The output figure. - """ - df = deserialize(outputs["df"], dtype=pd.DataFrame) - hp_names = inputs["hyperparameter_names"] - n_configs = inputs["n_configs"] - objective_id = inputs["objective_id"] - objective = run.get_objective(objective_id) - budget = run.get_budget(inputs["budget_id"]) - df = df.groupby(df.columns.drop(objective.name).to_list(), as_index=False).mean() - df.index = df.index.astype("str") - - # Limit to n_configs - idx = [str(i) for i in range(n_configs, len(df))] - df = df.drop(idx) - - costs = df[objective.name].values.tolist() - config_ids = df["config_id"].values.tolist() - data = [] - - # Specify layout kwargs - layout_kwargs = {} - if n_configs > 0 and len(hp_names) > 0: - for i, (hp_name, axis_name) in enumerate(zip(hp_names, ["xaxis", "yaxis", "zaxis"])): - hp = run.configspace[hp_name] - values = df[hp_name].values.tolist() - - tickvals, ticktext = get_hyperparameter_ticks(hp, ticks=4, include_nan=True) - layout_kwargs[axis_name] = { - "tickvals": tickvals, - "ticktext": ticktext, - "title": hp_name, - } - data.append(values) - - if len(data) == 0: - raise PreventUpdate - - # Transforms data to values - values = np.transpose(np.array(data)).tolist() - - # Now the duplicates are filtered - filtered_data: Dict[Tuple[int, float], Tuple] = {} - for config_id, cost, v in zip(config_ids, costs, values): - v = tuple(v) # Make hashable - key = (config_id, cost) - - if v in filtered_data.values(): - old_key = list(filtered_data.keys())[list(filtered_data.values()).index(v)] - old_cost = old_key[1] - - if objective.optimize == "lower": - if old_cost < cost: - continue - - if objective.optimize == "upper": - if old_cost > cost: - continue - - # Otherwise it is replaced - del filtered_data[old_key] - - filtered_data[key] = v - - # Replace data - config_ids = [k[0] for k in filtered_data.keys()] - costs = [k[1] for k in filtered_data.keys()] - - # Specify scatter kwargs - scatter_kwargs = { - "mode": "markers", - "marker": { - "size": 5, - "color": costs, - "colorbar": {"thickness": 30, "title": objective.name}, - }, - "hovertext": [ - get_hovertext_from_config(run, config_id, budget) for config_id in config_ids - ], - "meta": {"colorbar": costs}, - "hoverinfo": "text", - } - - if len(data) >= 1: - x = [v[0] for v in filtered_data.values()] - if len(data) >= 2: - y = [v[1] for v in filtered_data.values()] - if len(data) >= 3: - z = [v[2] for v in filtered_data.values()] - - if len(data) == 3: - trace = go.Scatter3d(x=x, y=y, z=z, **scatter_kwargs) - layout = go.Layout( - {"scene": {**layout_kwargs}}, - font=dict(size=config.FIGURE_FONT_SIZE), - ) - else: - if len(data) == 1: - y = [0 for _ in x] - trace = go.Scatter(x=x, y=y, **scatter_kwargs) - elif len(data) == 2: - trace = go.Scatter(x=x, y=y, **scatter_kwargs) - else: - trace = go.Scatter(x=[], y=[]) - layout = go.Layout( - **layout_kwargs, - font=dict(size=config.FIGURE_FONT_SIZE), - ) - - figure = go.Figure(data=trace, layout=layout) - figure.update_layout(dict(margin=config.FIGURE_MARGIN)) - save_image(figure, "configuration_cube.pdf") - - return figure diff --git a/deepcave/plugins/hyperparameter/parallel_coordinates 2.py b/deepcave/plugins/hyperparameter/parallel_coordinates 2.py deleted file mode 100644 index b2b0bb84..00000000 --- a/deepcave/plugins/hyperparameter/parallel_coordinates 2.py +++ /dev/null @@ -1,431 +0,0 @@ -# noqa: D400 - -""" -# ParallelCoordinates - -This module provides utilities for visualizing the parallel coordinates. - -## Classes - - ParallelCoordinates : Can be used for visualizing the parallel coordinates. -""" - -from typing import Any, Callable, Dict, List - -from collections import defaultdict - -import dash_bootstrap_components as dbc -import numpy as np -import pandas as pd -import plotly.graph_objs as go -from dash import dcc, html -from dash.exceptions import PreventUpdate - -from deepcave import config -from deepcave.constants import VALUE_RANGE -from deepcave.evaluators.fanova import fANOVA -from deepcave.plugins.static import StaticPlugin -from deepcave.utils.compression import deserialize, serialize -from deepcave.utils.layout import get_checklist_options, get_select_options, help_button -from deepcave.utils.logs import get_logger -from deepcave.utils.styled_plotty import get_hyperparameter_ticks, save_image - -logger = get_logger(__name__) - - -class ParallelCoordinates(StaticPlugin): - """Can be used for visualizing the parallel coordinates.""" - - id = "parallel_coordinates" - name = "Parallel Coordinates" - icon = "far fa-map" - activate_run_selection = True - help = "docs/plugins/parallel_coordinates.rst" - - @staticmethod - def get_input_layout(register: Callable) -> List[Any]: - """ - Get the layout for the input block. - - Parameters - ---------- - register : Callable - Method to regsiter (user) variables. - The register_input function is located in the Plugin superclass. - - Returns - ------- - List[Any] - The layouts for the input block. - """ - return [ - dbc.Row( - [ - dbc.Col( - [ - dbc.Label("Objective"), - dbc.Select( - id=register("objective_id", ["value", "options"], type=int), - placeholder="Select objective ...", - ), - ], - md=6, - ), - dbc.Col( - [ - dbc.Label("Budget"), - help_button( - "Budget refers to the multi-fidelity budget. " - "Combined budget means that the trial on the highest evaluated" - " budget is used. \n " - "Note: Selecting combined budget might be misleading if a time" - " objective is used. Often, higher budget take longer to evaluate," - " which might negatively influence the results." - ), - dbc.Select( - id=register("budget_id", ["value", "options"], type=int), - placeholder="Select budget ...", - ), - ], - md=6, - ), - ], - className="mb-3", - ), - html.Div( - [ - dbc.Label("Show Important Hyperparameters"), - help_button( - "Only the most important hyperparameters are shown which are " - "calculated by fANOVA using 10 trees. The more left a " - "hyperparameter stands, the more important it is. However, activating " - "this option might take longer." - ), - dbc.Select( - id=register("show_important_only", ["value", "options"]), - placeholder="Select ...", - ), - ] - ), - ] - - @staticmethod - def get_filter_layout(register: Callable) -> List[Any]: - """ - Get the layout for the filter block. - - Parameters - ---------- - register : Callable - Method to register (user) variables. - The register_input function is located in the Plugin superclass. - - Returns - ------- - List[Any] - The layouts for the filter block. - """ - return [ - dbc.Row( - [ - dbc.Col( - [ - dbc.Label("Limit Hyperparameters"), - help_button( - "Shows either the n most important hyperparameters (if show " - "important hyperparameters is true) or the first n selected " - "hyperparameters." - ), - dbc.Input(id=register("n_hps", "value"), type="number"), - ], - md=6, - ), - dbc.Col( - [ - dbc.Label("Show Unsuccessful Configurations"), - help_button("Whether to show all configurations or only failed ones."), - dbc.Select( - id=register("show_unsuccessful", ["value", "options"]), - placeholder="Select ...", - ), - ], - md=6, - ), - ], - ), - html.Div( - [ - dbc.Label("Hyperparameters"), - dbc.Checklist( - id=register("hyperparameter_names", ["value", "options"]), inline=True - ), - ], - className="mt-3", - id=register("hide_hps", ["hidden"]), - ), - ] - - def load_inputs(self) -> Dict[str, Dict[str, Any]]: - """ - Load the content for the defined inputs in 'get_input_layout' and 'get_filter_layout'. - - This method is necessary to pre-load contents for the inputs. - So, if the plugin is called for the first time or there are no results in the cache, - the plugin gets its content from this method. - - Returns - ------- - Dict[str, Dict[str, Any]] - Content to be filled. - """ - return { - "show_important_only": {"options": get_select_options(binary=True), "value": "true"}, - "show_unsuccessful": {"options": get_select_options(binary=True), "value": "false"}, - "n_hps": {"value": 0}, - "hyperparameter_names": {"options": get_checklist_options(), "value": []}, - "hide_hps": {"hidden": True}, - } - - def load_dependency_inputs(self, run, _, inputs) -> Dict[str, Any]: # type: ignore - """ - Work like 'load_inputs' but called after inputs have changed. - - Note - ---- - Only the changes have to be returned. - The returned dictionary will be merged with the inputs. - - Parameters - ---------- - run - The selected run. - inputs - Current content of the inputs. - - Returns - ------- - Dict[str, Any] - The dictionary with the changes. - """ - # Prepare objectives - objective_names = run.get_objective_names() - objective_ids = run.get_objective_ids() - objective_options = get_select_options(objective_names, objective_ids) - objective_value = inputs["objective_id"]["value"] - - # Prepare budgets - budgets = run.get_budgets(human=True) - budget_ids = run.get_budget_ids() - budget_options = get_checklist_options(budgets, budget_ids) - budget_value = inputs["budget_id"]["value"] - - # Prepare others - n_hps = inputs["n_hps"]["value"] - hp_names = list(run.configspace.keys()) - - if inputs["show_important_only"]["value"] == "true": - hp_options = [] - hp_value = inputs["hyperparameter_names"]["value"] - hidden = True - else: - hp_options = get_select_options(hp_names) - - values = inputs["hyperparameter_names"]["value"] - if len(values) == 0: - values = hp_names - - hp_value = values - hidden = False - - if objective_value is None: - objective_value = objective_ids[0] - budget_value = budget_ids[-1] - hp_value = hp_names - - if n_hps == 0: - n_hps = len(hp_names) - - return { - "objective_id": { - "options": objective_options, - "value": objective_value, - }, - "budget_id": { - "options": budget_options, - "value": budget_value, - }, - "hyperparameter_names": { - "options": hp_options, - "value": hp_value, - }, - "n_hps": {"value": n_hps}, - "hide_hps": {"hidden": hidden}, - } - - @staticmethod - def process(run, inputs) -> Dict[str, Any]: # type: ignore - """ - Return raw data based on a run and input data. - - Warning - ------- - The returned data must be JSON serializable. - - Note - ---- - The passed inputs are cleaned and therefore differs compared to 'load_inputs' - or 'load_dependency_inputs'. - Please see '_clean_inputs' for more information. - - Parameters - ---------- - run : AbstractRun - The run to process. - inputs : Dict[str, Any] - The input data. - - Returns - ------- - Dict[str, Any] - The serialized dictionary. - """ - budget = run.get_budget(inputs["budget_id"]) - objective = run.get_objective(inputs["objective_id"]) - df = run.get_encoded_data(objective, budget) - df = df.groupby(df.columns.drop(objective.name).to_list(), as_index=False).mean() - df = serialize(df) - result: Dict[str, Any] = {"df": df} - - if inputs["show_important_only"]: - # Let's run a quick fANOVA here - evaluator = fANOVA(run) - evaluator.calculate(objective, budget, n_trees=10, seed=0) - importances_dict = evaluator.get_importances() - importances = {u: v[0] for u, v in importances_dict.items()} - important_hp_names = sorted( - importances, key=lambda key: importances[key], reverse=False - ) - result["important_hp_names"] = important_hp_names - - return result - - @staticmethod - def get_output_layout(register: Callable) -> dcc.Graph: - """ - Get the layout for the output block. - - Parameters - ---------- - register : Callable - Method to register outputs. - The register_output function is located in the Plugin superclass. - - Returns - ------- - dcc.Graph - The layouts for the output block. - """ - return dcc.Graph( - register("graph", "figure"), - style={"height": config.FIGURE_HEIGHT}, - config={"toImageButtonOptions": {"scale": config.FIGURE_DOWNLOAD_SCALE}}, - ) - - @staticmethod - def load_outputs(run, inputs, outputs) -> go.Figure: # type: ignore - """ - Read in the raw data and prepare them for the layout. - - Note - ---- - The passed inputs are cleaned and therefore differs compared to 'load_inputs' - or 'load_dependency_inputs'. - Please see '_clean_inputs' for more information. - - Parameters - ---------- - run - The selected run. - inputs - The inputs and filter values fromt the user. - outputs - Raw output from the run. - - Returns - ------- - go.Figure - The output figure. - """ - objective = run.get_objective(inputs["objective_id"]) - objective_name = objective.name - - show_important_only = inputs["show_important_only"] - show_unsuccessful = inputs["show_unsuccessful"] - n_hps = inputs["n_hps"] - - if n_hps == "" or n_hps is None: - raise PreventUpdate - else: - n_hps = int(n_hps) - - if show_important_only: - hp_names = outputs["important_hp_names"] - else: - hp_names = inputs["hyperparameter_names"] - - hp_names = hp_names[:n_hps] - - df = outputs["df"] - df = deserialize(df, dtype=pd.DataFrame) - objective_values = [] - for value in df[objective_name].values: - b = np.isnan(value) - if not show_unsuccessful: - b = not b - if b: - objective_values += [value] - - data: defaultdict = defaultdict(dict) - for hp_name in hp_names: - values = [] - for hp_v, objective_v in zip(df[hp_name].values, df[objective_name].values): - b = np.isnan(objective_v) - if not show_unsuccessful: - b = not b - if b: - values += [hp_v] - - data[hp_name]["values"] = values - data[hp_name]["label"] = hp_name - data[hp_name]["range"] = VALUE_RANGE - - hp = run.configspace[hp_name] - tickvals, ticktext = get_hyperparameter_ticks(hp, ticks=4, include_nan=True) - - data[hp_name]["tickvals"] = tickvals - data[hp_name]["ticktext"] = ticktext - - if show_unsuccessful: - line = dict() - else: - data[objective_name]["values"] = objective_values - data[objective_name]["label"] = objective_name - line = dict( - color=data[objective_name]["values"], - showscale=True, - colorscale="aggrnyl", - ) - - figure = go.Figure( - data=go.Parcoords( - line=line, - dimensions=list([d for d in data.values()]), - labelangle=45, - ), - layout=dict( - margin=dict(t=150, b=50, l=100, r=0), - font=dict(size=config.FIGURE_FONT_SIZE), - ), - ) - save_image(figure, "parallel_coordinates.pdf") - - return figure diff --git a/deepcave/runs/converters/optuna 2.py b/deepcave/runs/converters/optuna 2.py deleted file mode 100644 index d010f130..00000000 --- a/deepcave/runs/converters/optuna 2.py +++ /dev/null @@ -1,288 +0,0 @@ -# noqa: D400 -""" -# OptunaRun - -This module provides utilities to create an Optuna run. - -## Classes - - OptunaRun: Define an Optuna run object. -""" - -from typing import List, Union - -import pickle -from pathlib import Path - -from ConfigSpace import ( - Categorical, - Configuration, - ConfigurationSpace, - Float, - Integer, - Uniform, -) -from ConfigSpace.hyperparameters import Hyperparameter -from optuna.distributions import ( - CategoricalDistribution, - FloatDistribution, - IntDistribution, -) -from optuna.search_space import IntersectionSearchSpace -from optuna.study import StudyDirection -from optuna.trial import TrialState - -from deepcave.runs import Status -from deepcave.runs.objective import Objective -from deepcave.runs.run import Run -from deepcave.utils.hash import file_to_hash -from deepcave.utils.logs import get_logger - -logger = get_logger(__name__) - - -class OptunaRun(Run): - """ - Define an Optuna run object. - - Properties - ---------- - path : Path - The path to the run. - """ - - prefix = "Optuna" - _initial_order = 2 - - @staticmethod - def _get_pickle_file(path: Path) -> Path: - """ - Get the path to the pickle file from the directory path. - - Parameters - ---------- - path : Path - The path to the directory containing the pickle file. - - Returns - ------- - Path - The path to the pickle file. - """ - pickle_files = list(path.glob("*.pkl")) - if len(pickle_files) != 1: - raise RuntimeError("There should be exactly one pickle file in the directory.") - else: - return pickle_files[0] - - @property - def hash(self) -> str: - """ - Hash of the current run. - - If the hash changes, the cache has to be cleared. - This ensures that the cache always holds the latest results of the run. - - Returns - ------- - str - The hash of the run. - """ - if self.path is None: - return "" - - pickle_file = OptunaRun._get_pickle_file(self.path) - return file_to_hash(pickle_file) - - @property - def latest_change(self) -> Union[float, int]: - """ - Get the timestamp of the latest change. - - Returns - ------- - Union[float, int] - The latest change. - """ - if self.path is None: - return 0 - - pickle_file = OptunaRun._get_pickle_file(self.path) - return Path(pickle_file).stat().st_mtime - - @classmethod - def from_path(cls, path: Union[Path, str]) -> "OptunaRun": - """ - Based on working_dir/run_name/*, return a new trials object. - - Parameters - ---------- - path : Union[Path, str] - The path to base the trial object on. - - Returns - ------- - The Optuna run. - - Raises - ------ - RuntimeError - Instances are not supported. - """ - path = Path(path) - - # Load the optuna study from the file path - pickle_file_path = OptunaRun._get_pickle_file(path) - with open(pickle_file_path, "rb") as f: - optuna_study = pickle.load(f) - - # Read configspace - optuna_space = IntersectionSearchSpace(include_pruned=True).calculate(study=optuna_study) - configspace = ConfigurationSpace() - - hyperparameters: List[Hyperparameter] = [] - - for hp_name, hp in optuna_space.items(): - if isinstance(hp, FloatDistribution) or isinstance(hp, IntDistribution): - if hp.step is not None and hp.step != 1: - logger.warning( - f"Step is not supported. " - f'Step={hp.step} will be ignored for hyperparameter "{hp_name}".' - ) - - if isinstance(hp, FloatDistribution): - hyperparameters.append( - Float( - name=hp_name, - bounds=(hp.low, hp.high), - distribution=Uniform(), - default=None, - log=hp.log, - ) - ) - elif isinstance(hp, IntDistribution): - hyperparameters.append( - Integer( - name=hp_name, - bounds=(hp.low, hp.high), - distribution=Uniform(), - default=None, - log=hp.log, - ) - ) - elif isinstance(hp, CategoricalDistribution): - hyperparameters.append( - Categorical( - name=hp_name, - default=hp.choices[0], - items=hp.choices, - ) - ) - else: - raise ValueError( - ( - "The hyperparameters in the Optuna study must be of type " - "`FloatDistribution`, `IntDistribution` or `CategoricalDistribution`, " - f"but a hyperparameter of type {type(hp)} was given." - ) - ) - configspace.add(hyperparameters) - - n_objectives = max(len(trial.values) for trial in optuna_study.trials) - obj_list = list() - for i in range(n_objectives): - if optuna_study.metric_names is not None: - metric_name = optuna_study.metric_names[i] - else: - metric_name = f"Objective{i}" - optimize = "lower" if optuna_study.directions[i] == StudyDirection.MINIMIZE else "upper" - - obj_list.append( - Objective( - name=metric_name, - lower=None, - upper=None, - optimize=optimize, - ) - ) - - obj_list.append(Objective("Time")) - - # Let's create a new run object - run = OptunaRun(name=path.stem, configspace=configspace, objectives=obj_list, meta=None) - - # The path has to be set manually - run._path = path - - first_starttime = None - for trial in optuna_study.trials: - try: - config = Configuration(configspace, trial.params) - except ValueError: - raise ValueError( - f"Could not convert the configuration of trial {trial.number} to " - f"a ConfigSpace configuration.\nThis might be due to the " - f"configuration space containing conditionals or dynamic " - f"hyperparameter value ranges, which are currently not supported." - ) - - if first_starttime is None: - first_starttime = trial.datetime_start.timestamp() - - starttime = trial.datetime_start.timestamp() - first_starttime - endtime = trial.datetime_complete.timestamp() - first_starttime - - if trial.state == TrialState.COMPLETE: - status = Status.SUCCESS - elif trial.state == TrialState.FAIL: - status = Status.FAILED - elif trial.state == TrialState.PRUNED: - status = Status.PRUNED - elif trial.state == TrialState.RUNNING or trial["status"] == TrialState.WAITING: - continue - else: - status = Status.UNKNOWN - - cost = trial.values - - if status != Status.SUCCESS: - # Costs which failed, should not be included - cost = [None] * len(cost) if isinstance(cost, list) else None - time = None - else: - time = float(endtime - starttime) - - run.add( - costs=cost + [time] if isinstance(cost, list) else [cost, time], # type: ignore - config=config, - budget=0.0, - seed=-1, - start_time=starttime, - end_time=endtime, - status=status, - origin=None, - additional=None, - ) - - return run - - @classmethod - def is_valid_run(cls, path_name: str) -> bool: - """ - Check whether the path name belongs to a valid optuna2 run. - - Parameters - ---------- - path_name: str - The path to check. - - Returns - ------- - bool - True if path is valid run. - False otherwise. - """ - path = Path(path_name) - pickle_files = list(path.glob("*.pkl")) - if len(pickle_files) != 1: - return False - return True diff --git a/docs/plugins/symbolic_explanations 2.rst b/docs/plugins/symbolic_explanations 2.rst deleted file mode 100644 index 1b78e02b..00000000 --- a/docs/plugins/symbolic_explanations 2.rst +++ /dev/null @@ -1,22 +0,0 @@ -Symbolic Explanations -==================== - -Symbolic Explanations allow to obtain explicit formulas quantifying the relation between -hyperparameter values and model performance by applying symbolic regression to meta-data collected -during hyperparameter optimization. - -The plugin is capable of answering similar questions as the Partial Dependencies plugin, i.e.: - -* How does the objective change with respect to one or two hyperparameters? For example, does the - accuracy increase if the learning rate decreases? -* Do multiple trials show similar behavior? - -While the Partial Dependencies plugin provides a plot describing the effects of hyperparameters on -the model performance, the Symbolic Explanations plugin additionally allows to obtain an explicit -formula capturing these effects. - -To learn more about Symbolic Explanations, please see the paper -`Symbolic Explanations for Hyperparameter Optimization -`_. - -.. image:: ../images/plugins/symbolic_explanations.png diff --git a/docs/sg_execution_times.rst b/docs/sg_execution_times.rst new file mode 100644 index 00000000..1654b0ec --- /dev/null +++ b/docs/sg_execution_times.rst @@ -0,0 +1,58 @@ + +:orphan: + +.. _sphx_glr_sg_execution_times: + + +Computation times +================= +**00:00.000** total execution time for 8 files **from all galleries**: + +.. container:: + + .. raw:: html + + + + + + + + .. list-table:: + :header-rows: 1 + :class: table table-striped sg-datatable + + * - Example + - Time + - Mem (MB) + * - :ref:`sphx_glr_examples_api_ablation_paths.py` (``../examples/api/ablation_paths.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_examples_api_importances 2.py` (``../examples/api/importances 2.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_examples_api_importances.py` (``../examples/api/importances.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_examples_api_parallel_coordinates.py` (``../examples/api/parallel_coordinates.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_examples_api_pdp.py` (``../examples/api/pdp.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_examples_record_digits_sklearn.py` (``../examples/record/digits_sklearn.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_examples_record_minimal.py` (``../examples/record/minimal.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_examples_record_mnist_pytorch.py` (``../examples/record/mnist_pytorch.py``) + - 00:00.000 + - 0.0 diff --git a/examples/api/importances 2.py b/examples/api/importances 2.py deleted file mode 100644 index de4dad38..00000000 --- a/examples/api/importances 2.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Importances -^^^^^^^^^^^^^^^^^^^^ - -This example shows how to use the plugin Importances. -Note that other plugins use the same interfaces and can be used in the same fashion. -""" - -from deepcave.plugins.hyperparameter.importances import Importances -from deepcave.runs.converters.deepcave import DeepCAVERun -from pathlib import Path - - -if __name__ == "__main__": - # Instantiate the run - run = DeepCAVERun.from_path(Path("logs/DeepCAVE/minimal/run_2")) - - objective_id = run.get_objective_ids()[0] - budget_ids = run.get_budget_ids() - - # Instantiate the plugin - plugin = Importances() - inputs = plugin.generate_inputs( - hyperparameter_names=list(run.configspace.keys()), - objective_id=objective_id, - budget_ids=budget_ids, - method="global", - n_hps=3, - n_trees=10, - ) - # Note: Filter variables are not considered. - outputs = plugin.generate_outputs(run, inputs) - - # Finally, you can load the figure. Here, the filter variables play a role. - figure = plugin.load_outputs(run, inputs, outputs) - figure.write_image("examples/api/importances.png", scale=2.0) diff --git a/logs/README 2.rst b/logs/README 2.rst deleted file mode 100644 index 2cadd357..00000000 --- a/logs/README 2.rst +++ /dev/null @@ -1,11 +0,0 @@ -Example Logs -============ - -This directory contains exemplary run logs from hyperparameter optimization runs created with -AMLTK, BOHB, Optuna, and SMAC3. -Furthermore, it contains example logs created with the DeepCAVE recorder and example logs in -CSV format. - -After starting DeepCAVE, you can load these logs by navigating to the directory containing the run -log you would like to load on the `General` site and then selecting the run log via clicking on -the `+` next to it. \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/configspace 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/configspace 2.json deleted file mode 100644 index be8aa231..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_1/configspace 2.json +++ /dev/null @@ -1 +0,0 @@ -{"name": null, "hyperparameters": [{"type": "categorical", "name": "activation", "choices": ["logistic", "tanh", "relu"], "weights": null, "default_value": "tanh", "meta": null}, {"type": "uniform_int", "name": "n_layer", "lower": 1, "upper": 5, "default_value": 1, "log": false, "meta": null}, {"type": "uniform_int", "name": "n_neurons", "lower": 8, "upper": 256, "default_value": 10, "log": true, "meta": null}, {"type": "categorical", "name": "solver", "choices": ["lbfgs", "sgd", "adam"], "weights": null, "default_value": "adam", "meta": null}, {"type": "uniform_int", "name": "batch_size", "lower": 30, "upper": 300, "default_value": 200, "log": false, "meta": null}, {"type": "categorical", "name": "learning_rate", "choices": ["constant", "invscaling", "adaptive"], "weights": null, "default_value": "constant", "meta": null}, {"type": "uniform_float", "name": "learning_rate_init", "lower": 0.0001, "upper": 1.0, "default_value": 0.001, "log": true, "meta": null}], "conditions": [{"type": "IN", "child": "batch_size", "parent": "solver", "values": ["sgd", "adam"]}, {"type": "EQ", "child": "learning_rate", "parent": "solver", "value": "sgd"}, {"type": "IN", "child": "learning_rate_init", "parent": "solver", "values": ["sgd", "adam"]}], "forbiddens": [], "python_module_version": "1.2.0", "format_version": 0.4} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/intensifier 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/intensifier 2.json deleted file mode 100644 index d7b218ec..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_1/intensifier 2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "incumbent_ids": [ - 20 - ], - "rejected_config_ids": [ - 1, - 6, - 8, - 3, - 10, - 4, - 12, - 13, - 7, - 2, - 15, - 17, - 18, - 19, - 14, - 24, - 21, - 9, - 22, - 16, - 26, - 29, - 28, - 31, - 32, - 34, - 27, - 36, - 37, - 35, - 23, - 11, - 38, - 39, - 43, - 5, - 41, - 40, - 42, - 45, - 46, - 48, - 30, - 50, - 52, - 53, - 55, - 54, - 44, - 57, - 56, - 25, - 49 - ], - "incumbents_changed": 5, - "trajectory": [ - { - "config_ids": [ - 1 - ], - "costs": [ - 0.6699473847106159 - ], - "trial": 1, - "walltime": 0.4147176742553711 - }, - { - "config_ids": [ - 8 - ], - "costs": [ - 0.12855926957598263 - ], - "trial": 2, - "walltime": 0.7400827407836914 - }, - { - "config_ids": [ - 2 - ], - "costs": [ - 0.025597338285360594 - ], - "trial": 5, - "walltime": 1.215728759765625 - }, - { - "config_ids": [ - 14 - ], - "costs": [ - 0.02392757660167144 - ], - "trial": 13, - "walltime": 2.366025686264038 - }, - { - "config_ids": [ - 20 - ], - "costs": [ - 0.02115031466006397 - ], - "trial": 37, - "walltime": 8.94394326210022 - } - ], - "state": { - "queue": [ - [ - 33, - 4 - ], - [ - 51, - 4 - ], - [ - 58, - 2 - ], - [ - 47, - 4 - ], - [ - 59, - 2 - ], - [ - 60, - 2 - ], - [ - 61, - 2 - ] - ] - } -} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/optimization 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/optimization 2.json deleted file mode 100644 index cd5ebb87..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_1/optimization 2.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "used_walltime": 67.33127307891846, - "used_target_function_walltime": 494.0225176811218, - "last_update": 1712565076.753763, - "finished": true -} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/runhistory 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/runhistory 2.json deleted file mode 100644 index 5ab1ec0f..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_1/runhistory 2.json +++ /dev/null @@ -1,2393 +0,0 @@ -{ - "stats": { - "submitted": 154, - "finished": 154, - "running": 0 - }, - "data": [ - [ - 1, - null, - 209652396, - null, - 0.6699473847106159, - 0.32405996322631836, - 1, - 1712565009.4599066, - 1712565009.7847676, - {} - ], - [ - 2, - null, - 209652396, - null, - 0.025597338285360594, - 1.0813992023468018, - 1, - 1712565009.4976892, - 1712565010.5797143, - {} - ], - [ - 3, - null, - 209652396, - null, - 0.6583333333333333, - 1.1021058559417725, - 1, - 1712565009.5342155, - 1712565010.6368752, - {} - ], - [ - 4, - null, - 209652396, - null, - 0.02560352831940571, - 1.2457053661346436, - 1, - 1712565009.573759, - 1712565010.8200808, - {} - ], - [ - 5, - null, - 209652396, - null, - 0.8992773135252243, - 10.674755573272705, - 1, - 1712565009.611309, - 1712565020.286695, - {} - ], - [ - 6, - null, - 209652396, - null, - 0.19250851129681212, - 0.5179076194763184, - 1, - 1712565009.6513555, - 1712565010.1701534, - {} - ], - [ - 7, - null, - 209652396, - null, - 0.89983286908078, - 1.9669127464294434, - 1, - 1712565009.704494, - 1712565011.67224, - {} - ], - [ - 8, - null, - 209652396, - null, - 0.12855926957598263, - 0.35997724533081055, - 1, - 1712565009.744616, - 1712565010.1052234, - {} - ], - [ - 9, - null, - 209652396, - null, - 0.05064376354069944, - 2.6614692211151123, - 1, - 1712565009.8414927, - 1712565012.5038698, - {} - ], - [ - 1, - null, - 398764591, - null, - 0.6861915815536985, - 0.3299448490142822, - 1, - 1712565010.1649523, - 1712565010.4956682, - {} - ], - [ - 10, - null, - 209652396, - null, - 0.2904967502321263, - 0.49783873558044434, - 1, - 1712565010.2330816, - 1712565010.7317798, - {} - ], - [ - 8, - null, - 398764591, - null, - 0.09072887650882078, - 0.3993697166442871, - 1, - 1712565010.5594473, - 1712565010.959439, - {} - ], - [ - 11, - null, - 209652396, - null, - 0.030058805323429305, - 8.363003253936768, - 1, - 1712565010.6421216, - 1712565019.005703, - {} - ], - [ - 2, - null, - 398764591, - null, - 0.03006035283194064, - 1.5795397758483887, - 1, - 1712565010.7179625, - 1712565012.2989554, - {} - ], - [ - 12, - null, - 209652396, - null, - 0.4312689569792634, - 0.41001152992248535, - 1, - 1712565010.7946854, - 1712565011.2055905, - {} - ], - [ - 13, - null, - 209652396, - null, - 0.8987186629526462, - 0.787147045135498, - 1, - 1712565010.8761115, - 1712565011.6642888, - {} - ], - [ - 14, - null, - 209652396, - null, - 0.02392757660167144, - 0.7371306419372559, - 1, - 1712565011.0186121, - 1712565011.7567017, - {} - ], - [ - 15, - null, - 209652396, - null, - 0.27155834107087595, - 0.5496902465820312, - 1, - 1712565011.2593076, - 1712565011.8098226, - {} - ], - [ - 16, - null, - 209652396, - null, - 0.027825750541627947, - 2.965526819229126, - 1, - 1712565011.714806, - 1712565014.681264, - {} - ], - [ - 17, - null, - 209652396, - null, - 0.2031012070566388, - 0.9088141918182373, - 1, - 1712565011.782877, - 1712565012.692426, - {} - ], - [ - 14, - null, - 398764591, - null, - 0.03616527390900648, - 0.5853779315948486, - 1, - 1712565011.84823, - 1712565012.43451, - {} - ], - [ - 18, - null, - 209652396, - null, - 0.043975549365521616, - 1.1089684963226318, - 1, - 1712565011.9250941, - 1712565013.034664, - {} - ], - [ - 19, - null, - 209652396, - null, - 0.0473026926648098, - 0.8433613777160645, - 1, - 1712565012.3523204, - 1712565013.196589, - {} - ], - [ - 20, - null, - 209652396, - null, - 0.019481584648715633, - 1.379931926727295, - 1, - 1712565012.4961815, - 1712565013.877354, - {} - ], - [ - 14, - null, - 924231285, - null, - 0.034498607242339774, - 0.5644114017486572, - 1, - 1712565012.5570464, - 1712565013.1220315, - {} - ], - [ - 9, - null, - 398764591, - null, - 0.046202414113277634, - 3.3080027103424072, - 1, - 1712565012.7521577, - 1712565016.0607708, - {} - ], - [ - 17, - null, - 398764591, - null, - 0.2303899721448468, - 0.8374693393707275, - 1, - 1712565013.0987794, - 1712565013.9371634, - {} - ], - [ - 18, - null, - 398764591, - null, - 0.04397709687403284, - 1.4642667770385742, - 1, - 1712565013.1837516, - 1712565014.6489024, - {} - ], - [ - 21, - null, - 209652396, - null, - 0.8102460538532962, - 1.399742841720581, - 1, - 1712565013.24768, - 1712565014.6482792, - {} - ], - [ - 19, - null, - 398764591, - null, - 0.04674558960074293, - 0.9420022964477539, - 1, - 1712565013.938809, - 1712565014.881975, - {} - ], - [ - 20, - null, - 398764591, - null, - 0.023378211080160916, - 1.4037482738494873, - 1, - 1712565014.003451, - 1712565015.4077935, - {} - ], - [ - 17, - null, - 924231285, - null, - 0.2531925100588053, - 0.8451352119445801, - 1, - 1712565014.7065277, - 1712565015.5526712, - {} - ], - [ - 16, - null, - 398764591, - null, - 0.02560662333642827, - 3.018314838409424, - 1, - 1712565014.7788284, - 1712565017.7980478, - {} - ], - [ - 18, - null, - 924231285, - null, - 0.033389043639740046, - 1.1432137489318848, - 1, - 1712565014.83027, - 1712565015.9741292, - {} - ], - [ - 21, - null, - 398764591, - null, - 0.8770318786753327, - 1.9530634880065918, - 1, - 1712565014.9338968, - 1712565016.8878474, - {} - ], - [ - 19, - null, - 924231285, - null, - 0.07066542865985759, - 1.0578465461730957, - 1, - 1712565015.468166, - 1712565016.5266423, - {} - ], - [ - 20, - null, - 924231285, - null, - 0.02059114825131536, - 2.661302089691162, - 1, - 1712565015.615147, - 1712565018.2773883, - {} - ], - [ - 22, - null, - 209652396, - null, - 0.04287062828845567, - 1.5211269855499268, - 1, - 1712565016.0558727, - 1712565017.5779333, - {} - ], - [ - 23, - null, - 209652396, - null, - 0.04841999380996587, - 8.354167461395264, - 1, - 1712565016.1641889, - 1712565024.5193043, - {} - ], - [ - 9, - null, - 924231285, - null, - 0.0456313834726092, - 3.8456690311431885, - 1, - 1712565016.6118283, - 1712565020.459012, - {} - ], - [ - 24, - null, - 209652396, - null, - 0.8008805323429279, - 0.46524930000305176, - 1, - 1712565016.9718251, - 1712565017.438463, - {} - ], - [ - 21, - null, - 924231285, - null, - 0.8920519962859795, - 2.354919672012329, - 1, - 1712565017.5199628, - 1712565019.8761735, - {} - ], - [ - 24, - null, - 398764591, - null, - 0.726819870009285, - 0.6692235469818115, - 1, - 1712565017.6583474, - 1712565018.328519, - {} - ], - [ - 22, - null, - 398764591, - null, - 0.032836583101207184, - 1.1873645782470703, - 1, - 1712565017.88121, - 1712565019.0695643, - {} - ], - [ - 16, - null, - 924231285, - null, - 0.02894769421231813, - 3.8686747550964355, - 1, - 1712565018.3620129, - 1712565022.2322352, - {} - ], - [ - 24, - null, - 924231285, - null, - 0.7039430516867843, - 0.5134823322296143, - 1, - 1712565018.45915, - 1712565018.9735632, - {} - ], - [ - 25, - null, - 209652396, - null, - 0.0473026926648098, - 16.41287326812744, - 1, - 1712565019.0678082, - 1712565035.4815993, - {} - ], - [ - 11, - null, - 398764591, - null, - 0.032836583101207184, - 11.42868709564209, - 1, - 1712565019.1586893, - 1712565030.5886083, - {} - ], - [ - 22, - null, - 924231285, - null, - 0.042299597647787124, - 1.5826566219329834, - 1, - 1712565019.2559285, - 1712565020.8400044, - {} - ], - [ - 26, - null, - 209652396, - null, - 0.7974450634478489, - 0.5728342533111572, - 1, - 1712565019.9510424, - 1712565020.5249114, - {} - ], - [ - 27, - null, - 209652396, - null, - 0.09015010832559567, - 4.364055395126343, - 1, - 1712565020.3812418, - 1712565024.7467766, - {} - ], - [ - 5, - null, - 398764591, - null, - 0.9003884246363355, - 14.183544158935547, - 1, - 1712565020.5460055, - 1712565034.730529, - {} - ], - [ - 28, - null, - 209652396, - null, - 0.0984989167440421, - 0.9867184162139893, - 1, - 1712565020.6601338, - 1712565021.6482403, - {} - ], - [ - 26, - null, - 398764591, - null, - 0.6844769421231817, - 0.5313403606414795, - 1, - 1712565020.927448, - 1712565021.4603171, - {} - ], - [ - 29, - null, - 209652396, - null, - 0.8981615598885794, - 0.22849583625793457, - 1, - 1712565021.561606, - 1712565021.7911196, - {} - ], - [ - 26, - null, - 924231285, - null, - 0.823571649644073, - 0.731696605682373, - 1, - 1712565021.7452338, - 1712565022.4780235, - {} - ], - [ - 28, - null, - 398764591, - null, - 0.10796193129062215, - 1.0691678524017334, - 1, - 1712565021.888155, - 1712565022.9592311, - {} - ], - [ - 29, - null, - 398764591, - null, - 0.89983286908078, - 0.24839043617248535, - 1, - 1712565022.327761, - 1712565022.57766, - {} - ], - [ - 30, - null, - 209652396, - null, - 0.8992742185082018, - 10.086802959442139, - 1, - 1712565022.5642378, - 1712565032.651993, - {} - ], - [ - 31, - null, - 209652396, - null, - 0.1001562983596409, - 1.2912817001342773, - 1, - 1712565022.6596029, - 1712565023.9524426, - {} - ], - [ - 29, - null, - 924231285, - null, - 0.9059610027855153, - 0.22541332244873047, - 1, - 1712565023.0410864, - 1712565023.268075, - {} - ], - [ - 28, - null, - 924231285, - null, - 0.09126121943670695, - 1.0557401180267334, - 1, - 1712565023.348831, - 1712565024.4059231, - {} - ], - [ - 32, - null, - 209652396, - null, - 0.14357319715258432, - 0.7567057609558105, - 1, - 1712565024.0348003, - 1712565024.7930315, - {} - ], - [ - 31, - null, - 398764591, - null, - 0.12243268337975866, - 0.9943161010742188, - 1, - 1712565024.4760678, - 1712565025.4713368, - {} - ], - [ - 33, - null, - 209652396, - null, - 0.16188022284122572, - 25.683335304260254, - 1, - 1712565024.600618, - 1712565050.285301, - {} - ], - [ - 23, - null, - 398764591, - null, - 0.051753327143299166, - 7.385228157043457, - 1, - 1712565024.8224359, - 1712565032.2086697, - {} - ], - [ - 27, - null, - 398764591, - null, - 0.1191024450634478, - 3.959066867828369, - 1, - 1712565024.9297316, - 1712565028.8901627, - {} - ], - [ - 32, - null, - 398764591, - null, - 0.135792324357784, - 0.838543176651001, - 1, - 1712565025.5465531, - 1712565026.3867607, - {} - ], - [ - 31, - null, - 924231285, - null, - 0.10852986691426803, - 1.264732837677002, - 1, - 1712565026.4774003, - 1712565027.743719, - {} - ], - [ - 32, - null, - 924231285, - null, - 0.1413370473537604, - 0.9844629764556885, - 1, - 1712565027.8312953, - 1712565028.8173516, - {} - ], - [ - 34, - null, - 209652396, - null, - 0.14081089445991957, - 0.7884180545806885, - 1, - 1712565028.910618, - 1712565029.700526, - {} - ], - [ - 27, - null, - 924231285, - null, - 0.10739863819251005, - 4.667412996292114, - 1, - 1712565029.015657, - 1712565033.6845143, - {} - ], - [ - 35, - null, - 209652396, - null, - 0.04230114515629835, - 3.2498931884765625, - 1, - 1712565029.786574, - 1712565033.0380797, - {} - ], - [ - 34, - null, - 398764591, - null, - 0.16361188486536682, - 0.6655764579772949, - 1, - 1712565030.678432, - 1712565031.34571, - {} - ], - [ - 11, - null, - 924231285, - null, - 0.03283194057567318, - 12.322223424911499, - 1, - 1712565031.4208908, - 1712565043.7444239, - {} - ], - [ - 34, - null, - 924231285, - null, - 0.10627050448777475, - 0.6005818843841553, - 1, - 1712565032.2954042, - 1712565032.8974326, - {} - ], - [ - 23, - null, - 924231285, - null, - 0.06121324667285677, - 7.619780540466309, - 1, - 1712565032.744003, - 1712565040.3652062, - {} - ], - [ - 30, - null, - 398764591, - null, - 0.8948297740637574, - 11.035215616226196, - 1, - 1712565032.9881554, - 1712565044.0248187, - {} - ], - [ - 36, - null, - 209652396, - null, - 0.33456979263385933, - 0.9639077186584473, - 1, - 1712565033.1345205, - 1712565034.1002426, - {} - ], - [ - 35, - null, - 398764591, - null, - 0.04396007428040849, - 3.2995266914367676, - 1, - 1712565033.7999034, - 1712565037.1014593, - {} - ], - [ - 37, - null, - 209652396, - null, - 0.8992773135252244, - 0.7176535129547119, - 1, - 1712565034.1942468, - 1712565034.9135132, - {} - ], - [ - 36, - null, - 398764591, - null, - 0.5536861652739089, - 0.7682046890258789, - 1, - 1712565034.8139176, - 1712565035.583699, - {} - ], - [ - 5, - null, - 924231285, - null, - 0.9003899721448468, - 14.44285535812378, - 1, - 1712565035.018067, - 1712565049.4626539, - {} - ], - [ - 37, - null, - 398764591, - null, - 0.8987202104611576, - 0.7350404262542725, - 1, - 1712565035.604154, - 1712565036.3408573, - {} - ], - [ - 25, - null, - 398764591, - null, - 0.04230114515629835, - 16.373348712921143, - 1, - 1712565035.7092667, - 1712565052.084024, - {} - ], - [ - 36, - null, - 924231285, - null, - 0.4841782729805014, - 0.6209244728088379, - 1, - 1712565036.4488175, - 1712565037.071179, - {} - ], - [ - 37, - null, - 924231285, - null, - 0.89983286908078, - 0.8147656917572021, - 1, - 1712565037.180723, - 1712565037.9965858, - {} - ], - [ - 35, - null, - 924231285, - null, - 0.03784122562674097, - 2.6988790035247803, - 1, - 1712565037.301929, - 1712565040.002422, - {} - ], - [ - 38, - null, - 209652396, - null, - 0.8992757660167131, - 2.005662679672241, - 1, - 1712565038.0986674, - 1712565040.10565, - {} - ], - [ - 39, - null, - 209652396, - null, - 0.10461931290622106, - 1.514883041381836, - 1, - 1712565040.0788782, - 1712565041.5947666, - {} - ], - [ - 40, - null, - 209652396, - null, - 0.023930671618693777, - 2.3705286979675293, - 1, - 1712565040.184661, - 1712565042.556817, - {} - ], - [ - 38, - null, - 398764591, - null, - 0.8987186629526462, - 1.8411645889282227, - 1, - 1712565040.4387646, - 1712565042.2815857, - {} - ], - [ - 41, - null, - 209652396, - null, - 0.5219870009285051, - 2.545016050338745, - 1, - 1712565041.699431, - 1712565044.2464228, - {} - ], - [ - 39, - null, - 398764591, - null, - 0.06066233364283513, - 1.1961593627929688, - 1, - 1712565042.3568745, - 1712565043.5545325, - {} - ], - [ - 38, - null, - 924231285, - null, - 0.8987186629526462, - 2.093280792236328, - 1, - 1712565042.63884, - 1712565044.7334743, - {} - ], - [ - 40, - null, - 398764591, - null, - 0.02560352831940571, - 2.676013469696045, - 1, - 1712565043.6318994, - 1712565046.3088646, - {} - ], - [ - 39, - null, - 924231285, - null, - 0.07399566697616833, - 1.149909496307373, - 1, - 1712565043.8234403, - 1712565044.9747822, - {} - ], - [ - 42, - null, - 209652396, - null, - 0.041186939028164615, - 1.3158142566680908, - 1, - 1712565044.1248553, - 1712565045.4421346, - {} - ], - [ - 30, - null, - 924231285, - null, - 0.9009470752089137, - 10.608931303024292, - 1, - 1712565044.3304265, - 1712565054.9409497, - {} - ], - [ - 41, - null, - 398764591, - null, - 0.4590931600123801, - 2.614492177963257, - 1, - 1712565044.812999, - 1712565047.429369, - {} - ], - [ - 43, - null, - 209652396, - null, - 0.12633859486227172, - 1.004054069519043, - 1, - 1712565045.064711, - 1712565046.0705123, - {} - ], - [ - 44, - null, - 209652396, - null, - 0.15694367069018877, - 6.814570188522339, - 1, - 1712565045.5238674, - 1712565052.3401663, - {} - ], - [ - 42, - null, - 398764591, - null, - 0.03673320953265247, - 1.7806165218353271, - 1, - 1712565046.185416, - 1712565047.967166, - {} - ], - [ - 43, - null, - 398764591, - null, - 0.16193283813060977, - 1.155543565750122, - 1, - 1712565046.4128385, - 1712565047.570328, - {} - ], - [ - 40, - null, - 924231285, - null, - 0.024484679665738307, - 2.6976280212402344, - 1, - 1712565047.529584, - 1712565050.2290654, - {} - ], - [ - 41, - null, - 924231285, - null, - 0.3995605075827917, - 1.997565507888794, - 1, - 1712565047.6774108, - 1712565049.6766438, - {} - ], - [ - 43, - null, - 924231285, - null, - 0.13798978644382542, - 0.9540266990661621, - 1, - 1712565048.050342, - 1712565049.0060203, - {} - ], - [ - 42, - null, - 924231285, - null, - 0.03227793252862887, - 1.4512040615081787, - 1, - 1712565049.0885086, - 1712565050.5413556, - {} - ], - [ - 45, - null, - 209652396, - null, - 0.8753203342618384, - 0.847496747970581, - 1, - 1712565049.543191, - 1712565050.392285, - {} - ], - [ - 46, - null, - 209652396, - null, - 0.2489322191272052, - 0.6393148899078369, - 1, - 1712565049.7643716, - 1712565050.4047034, - {} - ], - [ - 47, - null, - 209652396, - null, - 0.08013927576601676, - 15.927346467971802, - 1, - 1712565050.3042223, - 1712565066.232525, - {} - ], - [ - 33, - null, - 398764591, - null, - 0.1925379139585267, - 23.66932439804077, - 1, - 1712565050.3959663, - 1712565074.0666122, - {} - ], - [ - 48, - null, - 209652396, - null, - 0.8124187558031569, - 0.910470724105835, - 1, - 1712565050.4891617, - 1712565051.4010818, - {} - ], - [ - 45, - null, - 398764591, - null, - 0.8914763231197772, - 0.7565186023712158, - 1, - 1712565050.623894, - 1712565051.3820372, - {} - ], - [ - 46, - null, - 398764591, - null, - 0.35610492107706604, - 0.8270492553710938, - 1, - 1712565050.7091923, - 1712565051.5371573, - {} - ], - [ - 49, - null, - 209652396, - null, - 0.2715707211389663, - 6.957829475402832, - 1, - 1712565051.466912, - 1712565058.4263031, - {} - ], - [ - 48, - null, - 398764591, - null, - 0.764634787991334, - 0.8151421546936035, - 1, - 1712565051.5597835, - 1712565052.3764203, - {} - ], - [ - 45, - null, - 924231285, - null, - 0.8747632311977716, - 0.941584587097168, - 1, - 1712565051.6266928, - 1712565052.5692742, - {} - ], - [ - 46, - null, - 924231285, - null, - 0.10686939028164644, - 0.5962722301483154, - 1, - 1712565052.165133, - 1712565052.762847, - {} - ], - [ - 25, - null, - 924231285, - null, - 0.04730424017332102, - 16.94296383857727, - 1, - 1712565052.4514196, - 1712565069.3960361, - {} - ], - [ - 44, - null, - 398764591, - null, - 0.18422624574435154, - 6.634417295455933, - 1, - 1712565052.544118, - 1712565059.1801126, - {} - ], - [ - 48, - null, - 924231285, - null, - 0.8602785515320335, - 0.7708423137664795, - 1, - 1712565052.6536405, - 1712565053.4260578, - {} - ], - [ - 50, - null, - 209652396, - null, - 0.04842463633549987, - 0.9821035861968994, - 1, - 1712565052.8549998, - 1712565053.8382864, - {} - ], - [ - 51, - null, - 209652396, - null, - 0.03673475704116369, - 10.69734239578247, - 1, - 1712565053.5026176, - 1712565064.2009585, - {} - ], - [ - 52, - null, - 209652396, - null, - 0.9031723924481585, - 0.40129661560058594, - 1, - 1712565053.932928, - 1712565054.335735, - {} - ], - [ - 50, - null, - 398764591, - null, - 0.04452955741256592, - 0.7838237285614014, - 1, - 1712565054.4255831, - 1712565055.2110846, - {} - ], - [ - 52, - null, - 398764591, - null, - 0.9031723924481585, - 0.40575075149536133, - 1, - 1712565055.0314689, - 1712565055.4387197, - {} - ], - [ - 53, - null, - 209652396, - null, - 0.8987202104611576, - 0.95816969871521, - 1, - 1712565055.2899451, - 1712565056.2494361, - {} - ], - [ - 50, - null, - 924231285, - null, - 0.04896626431445372, - 0.717515230178833, - 1, - 1712565055.52647, - 1712565056.2451823, - {} - ], - [ - 52, - null, - 924231285, - null, - 0.9037264004952027, - 0.4451889991760254, - 1, - 1712565056.3189044, - 1712565056.7654727, - {} - ], - [ - 53, - null, - 398764591, - null, - 0.8992757660167131, - 1.097594976425171, - 1, - 1712565056.4288678, - 1712565057.5279367, - {} - ], - [ - 54, - null, - 209652396, - null, - 0.08237233054781812, - 1.8947386741638184, - 1, - 1712565056.8644137, - 1712565058.76014, - {} - ], - [ - 55, - null, - 209652396, - null, - 0.0801470133085731, - 1.0197317600250244, - 1, - 1712565057.613952, - 1712565058.6350822, - {} - ], - [ - 53, - null, - 924231285, - null, - 0.888133704735376, - 0.9786050319671631, - 1, - 1712565058.522568, - 1712565059.502105, - {} - ], - [ - 49, - null, - 398764591, - null, - 0.5036165273909007, - 7.000282049179077, - 1, - 1712565058.730699, - 1712565065.7326648, - {} - ], - [ - 55, - null, - 398764591, - null, - 0.08347415660786128, - 0.8997397422790527, - 1, - 1712565058.8603768, - 1712565059.7616763, - {} - ], - [ - 54, - null, - 398764591, - null, - 0.08682141751779626, - 2.5480194091796875, - 1, - 1712565059.2588532, - 1712565061.808606, - {} - ], - [ - 44, - null, - 924231285, - null, - 0.1307644692045805, - 7.21750807762146, - 1, - 1712565059.5947275, - 1712565066.8132136, - {} - ], - [ - 56, - null, - 209652396, - null, - 0.024493964716805983, - 2.156345844268799, - 1, - 1712565059.8427088, - 1712565062.0000536, - {} - ], - [ - 55, - null, - 924231285, - null, - 0.06342927886103378, - 0.9959497451782227, - 1, - 1712565061.8952117, - 1712565062.892992, - {} - ], - [ - 54, - null, - 924231285, - null, - 0.08180903744970591, - 2.1299283504486084, - 1, - 1712565062.0780683, - 1712565064.2096357, - {} - ], - [ - 56, - null, - 398764591, - null, - 0.027270194986072527, - 2.475947856903076, - 1, - 1712565062.9701085, - 1712565065.4473305, - {} - ], - [ - 57, - null, - 209652396, - null, - 0.8281089445991954, - 0.6765487194061279, - 1, - 1712565064.2821856, - 1712565064.9597, - {} - ], - [ - 51, - null, - 398764591, - null, - 0.04229495512225334, - 10.195435047149658, - 1, - 1712565064.3825853, - 1712565074.5789762, - {} - ], - [ - 58, - null, - 209652396, - null, - 0.8280609718353451, - 7.688541412353516, - 1, - 1712565065.0443525, - 1712565072.7342741, - {} - ], - [ - 57, - null, - 398764591, - null, - 0.8636691426802847, - 1.2278459072113037, - 1, - 1712565065.5273557, - 1712565066.756225, - {} - ], - [ - 56, - null, - 924231285, - null, - 0.02893995666976168, - 2.2545557022094727, - 1, - 1712565065.8200572, - 1712565068.075699, - {} - ], - [ - 49, - null, - 924231285, - null, - 0.48469978334880837, - 6.26426362991333, - 1, - 1712565066.338829, - 1712565072.6052105, - {} - ], - [ - 47, - null, - 398764591, - null, - 0.08179975239863813, - 9.90377926826477, - 1, - 1712565066.839424, - 1712565076.744739, - {} - ], - [ - 57, - null, - 924231285, - null, - 0.8580532342927887, - 0.9290714263916016, - 1, - 1712565066.9271002, - 1712565067.857914, - {} - ], - [ - 59, - null, - 209652396, - null, - 0.07180594243268335, - 0.8639552593231201, - 1, - 1712565067.9361448, - 1712565068.801029, - {} - ], - [ - 60, - null, - 209652396, - null, - 0.025601980810894487, - 1.736633539199829, - 1, - 1712565068.1614459, - 1712565069.8996112, - {} - ], - [ - 61, - null, - 209652396, - null, - 0.7572144846796658, - 4.349518537521362, - 1, - 1712565068.9006317, - 1712565073.2513459, - {} - ], - [ - 59, - null, - 398764591, - null, - 0.09125502940266172, - 1.0945045948028564, - 1, - 1712565069.482275, - 1712565070.5787363, - {} - ] - ], - "configs": { - "1": { - "activation": "logistic", - "n_layer": 4, - "n_neurons": 11, - "solver": "lbfgs" - }, - "2": { - "activation": "tanh", - "n_layer": 1, - "n_neurons": 146, - "solver": "adam", - "batch_size": 77, - "learning_rate_init": 0.000534923804864797 - }, - "3": { - "activation": "relu", - "n_layer": 5, - "n_neurons": 43, - "solver": "sgd", - "batch_size": 157, - "learning_rate": "invscaling", - "learning_rate_init": 0.08903539757637764 - }, - "4": { - "activation": "logistic", - "n_layer": 2, - "n_neurons": 90, - "solver": "adam", - "batch_size": 175, - "learning_rate_init": 0.004156370184967407 - }, - "5": { - "activation": "logistic", - "n_layer": 4, - "n_neurons": 200, - "solver": "sgd", - "batch_size": 204, - "learning_rate": "constant", - "learning_rate_init": 0.016071068084706618 - }, - "6": { - "activation": "relu", - "n_layer": 2, - "n_neurons": 12, - "solver": "sgd", - "batch_size": 119, - "learning_rate": "adaptive", - "learning_rate_init": 0.0022428359458905173 - }, - "7": { - "activation": "relu", - "n_layer": 4, - "n_neurons": 51, - "solver": "adam", - "batch_size": 31, - "learning_rate_init": 0.11617003217050809 - }, - "8": { - "activation": "tanh", - "n_layer": 2, - "n_neurons": 20, - "solver": "lbfgs" - }, - "9": { - "activation": "tanh", - "n_layer": 5, - "n_neurons": 61, - "solver": "lbfgs" - }, - "10": { - "activation": "tanh", - "n_layer": 3, - "n_neurons": 26, - "solver": "sgd", - "batch_size": 233, - "learning_rate": "adaptive", - "learning_rate_init": 0.001104760089338138 - }, - "11": { - "activation": "relu", - "n_layer": 3, - "n_neurons": 214, - "solver": "lbfgs" - }, - "12": { - "activation": "logistic", - "n_layer": 1, - "n_neurons": 18, - "solver": "adam", - "batch_size": 135, - "learning_rate_init": 0.00015265008040823863 - }, - "13": { - "activation": "logistic", - "n_layer": 4, - "n_neurons": 29, - "solver": "adam", - "batch_size": 106, - "learning_rate_init": 0.33335905507564456 - }, - "14": { - "activation": "relu", - "n_layer": 1, - "n_neurons": 84, - "solver": "lbfgs" - }, - "15": { - "activation": "tanh", - "n_layer": 5, - "n_neurons": 8, - "solver": "adam", - "batch_size": 279, - "learning_rate_init": 0.04285190354891313 - }, - "16": { - "activation": "tanh", - "n_layer": 2, - "n_neurons": 122, - "solver": "sgd", - "batch_size": 61, - "learning_rate": "constant", - "learning_rate_init": 0.00630547594663167 - }, - "17": { - "activation": "tanh", - "n_layer": 5, - "n_neurons": 23, - "solver": "adam", - "batch_size": 172, - "learning_rate_init": 0.00040121151561322107 - }, - "18": { - "activation": "tanh", - "n_layer": 2, - "n_neurons": 66, - "solver": "lbfgs" - }, - "19": { - "activation": "relu", - "n_layer": 4, - "n_neurons": 16, - "solver": "sgd", - "batch_size": 66, - "learning_rate": "constant", - "learning_rate_init": 0.005542589894513166 - }, - "20": { - "activation": "logistic", - "n_layer": 1, - "n_neurons": 232, - "solver": "sgd", - "batch_size": 253, - "learning_rate": "adaptive", - "learning_rate_init": 0.06675517414146671 - }, - "21": { - "activation": "logistic", - "n_layer": 3, - "n_neurons": 74, - "solver": "sgd", - "batch_size": 300, - "learning_rate": "adaptive", - "learning_rate_init": 0.0029904795631048244 - }, - "22": { - "activation": "relu", - "n_layer": 1, - "n_neurons": 32, - "solver": "adam", - "batch_size": 45, - "learning_rate_init": 0.012048758384365402 - }, - "23": { - "activation": "relu", - "n_layer": 5, - "n_neurons": 106, - "solver": "lbfgs" - }, - "24": { - "activation": "tanh", - "n_layer": 3, - "n_neurons": 9, - "solver": "adam", - "batch_size": 210, - "learning_rate_init": 0.1549368466139337 - }, - "25": { - "activation": "tanh", - "n_layer": 4, - "n_neurons": 158, - "solver": "sgd", - "batch_size": 143, - "learning_rate": "adaptive", - "learning_rate_init": 0.00147326235260944 - }, - "26": { - "activation": "relu", - "n_layer": 2, - "n_neurons": 10, - "solver": "sgd", - "batch_size": 193, - "learning_rate": "invscaling", - "learning_rate_init": 0.018836814953733322 - }, - "27": { - "activation": "relu", - "n_layer": 4, - "n_neurons": 97, - "solver": "adam", - "batch_size": 249, - "learning_rate_init": 0.00011444921049693927 - }, - "28": { - "activation": "logistic", - "n_layer": 2, - "n_neurons": 38, - "solver": "lbfgs" - }, - "29": { - "activation": "logistic", - "n_layer": 5, - "n_neurons": 13, - "solver": "lbfgs" - }, - "30": { - "activation": "relu", - "n_layer": 3, - "n_neurons": 175, - "solver": "adam", - "batch_size": 271, - "learning_rate_init": 0.4444919521413565 - }, - "31": { - "activation": "tanh", - "n_layer": 3, - "n_neurons": 21, - "solver": "lbfgs" - }, - "32": { - "activation": "logistic", - "n_layer": 1, - "n_neurons": 45, - "solver": "sgd", - "batch_size": 106, - "learning_rate": "invscaling", - "learning_rate_init": 0.03214161586827769 - }, - "33": { - "activation": "logistic", - "n_layer": 5, - "n_neurons": 224, - "solver": "adam", - "batch_size": 290, - "learning_rate_init": 0.004563292654888671 - }, - "34": { - "activation": "tanh", - "n_layer": 2, - "n_neurons": 17, - "solver": "lbfgs" - }, - "35": { - "activation": "relu", - "n_layer": 4, - "n_neurons": 57, - "solver": "adam", - "batch_size": 116, - "learning_rate_init": 0.00035489292891332955 - }, - "36": { - "activation": "logistic", - "n_layer": 1, - "n_neurons": 27, - "solver": "sgd", - "batch_size": 203, - "learning_rate": "adaptive", - "learning_rate_init": 0.7609797301059412 - }, - "37": { - "activation": "logistic", - "n_layer": 4, - "n_neurons": 8, - "solver": "sgd", - "batch_size": 182, - "learning_rate": "adaptive", - "learning_rate_init": 0.00019717209149247594 - }, - "38": { - "activation": "relu", - "n_layer": 2, - "n_neurons": 130, - "solver": "sgd", - "batch_size": 163, - "learning_rate": "constant", - "learning_rate_init": 0.14078837052759072 - }, - "39": { - "activation": "relu", - "n_layer": 4, - "n_neurons": 31, - "solver": "lbfgs" - }, - "40": { - "activation": "tanh", - "n_layer": 2, - "n_neurons": 80, - "solver": "adam", - "batch_size": 260, - "learning_rate_init": 0.01019194380084181 - }, - "41": { - "activation": "tanh", - "n_layer": 3, - "n_neurons": 40, - "solver": "sgd", - "batch_size": 58, - "learning_rate": "adaptive", - "learning_rate_init": 0.00012594180136887576 - }, - "42": { - "activation": "tanh", - "n_layer": 1, - "n_neurons": 94, - "solver": "lbfgs" - }, - "43": { - "activation": "relu", - "n_layer": 5, - "n_neurons": 12, - "solver": "adam", - "batch_size": 232, - "learning_rate_init": 0.0017417895884476993 - }, - "44": { - "activation": "logistic", - "n_layer": 3, - "n_neurons": 137, - "solver": "lbfgs" - }, - "45": { - "activation": "logistic", - "n_layer": 5, - "n_neurons": 54, - "solver": "lbfgs" - }, - "46": { - "activation": "relu", - "n_layer": 2, - "n_neurons": 19, - "solver": "adam", - "batch_size": 185, - "learning_rate_init": 0.039036592000729606 - }, - "47": { - "activation": "tanh", - "n_layer": 4, - "n_neurons": 191, - "solver": "sgd", - "batch_size": 240, - "learning_rate": "constant", - "learning_rate_init": 0.0007368727861046234 - }, - "48": { - "activation": "tanh", - "n_layer": 1, - "n_neurons": 13, - "solver": "sgd", - "batch_size": 88, - "learning_rate": "adaptive", - "learning_rate_init": 0.5025399902870405 - }, - "49": { - "activation": "logistic", - "n_layer": 4, - "n_neurons": 102, - "solver": "lbfgs" - }, - "50": { - "activation": "tanh", - "n_layer": 1, - "n_neurons": 35, - "solver": "adam", - "batch_size": 129, - "learning_rate_init": 0.0034213963211558564 - }, - "51": { - "activation": "relu", - "n_layer": 5, - "n_neurons": 148, - "solver": "lbfgs" - }, - "52": { - "activation": "logistic", - "n_layer": 2, - "n_neurons": 10, - "solver": "sgd", - "batch_size": 294, - "learning_rate": "constant", - "learning_rate_init": 0.00047328481860027055 - }, - "53": { - "activation": "logistic", - "n_layer": 5, - "n_neurons": 20, - "solver": "adam", - "batch_size": 256, - "learning_rate_init": 0.10560248678966794 - }, - "54": { - "activation": "relu", - "n_layer": 3, - "n_neurons": 48, - "solver": "sgd", - "batch_size": 68, - "learning_rate": "adaptive", - "learning_rate_init": 0.00026295202043791785 - }, - "55": { - "activation": "tanh", - "n_layer": 3, - "n_neurons": 14, - "solver": "adam", - "batch_size": 150, - "learning_rate_init": 0.013590087590816565 - }, - "56": { - "activation": "tanh", - "n_layer": 1, - "n_neurons": 167, - "solver": "lbfgs" - }, - "57": { - "activation": "tanh", - "n_layer": 5, - "n_neurons": 15, - "solver": "sgd", - "batch_size": 99, - "learning_rate": "constant", - "learning_rate_init": 0.2144498574918249 - }, - "58": { - "activation": "relu", - "n_layer": 2, - "n_neurons": 243, - "solver": "sgd", - "batch_size": 217, - "learning_rate": "invscaling", - "learning_rate_init": 0.00016793749976650653 - }, - "59": { - "activation": "relu", - "n_layer": 4, - "n_neurons": 24, - "solver": "lbfgs" - }, - "60": { - "activation": "logistic", - "n_layer": 1, - "n_neurons": 62, - "solver": "adam", - "batch_size": 51, - "learning_rate_init": 0.0013063520380558128 - }, - "61": { - "activation": "logistic", - "n_layer": 3, - "n_neurons": 114, - "solver": "adam", - "batch_size": 90, - "learning_rate_init": 0.0520608529688821 - } - }, - "config_origins": { - "1": "Initial Design: Sobol", - "2": "Initial Design: Sobol", - "3": "Initial Design: Sobol", - "4": "Initial Design: Sobol", - "5": "Initial Design: Sobol", - "6": "Initial Design: Sobol", - "7": "Initial Design: Sobol", - "8": "Initial Design: Sobol", - "9": "Initial Design: Sobol", - "10": "Initial Design: Sobol", - "11": "Initial Design: Sobol", - "12": "Initial Design: Sobol", - "13": "Initial Design: Sobol", - "14": "Initial Design: Sobol", - "15": "Initial Design: Sobol", - "16": "Initial Design: Sobol", - "17": "Initial Design: Sobol", - "18": "Initial Design: Sobol", - "19": "Initial Design: Sobol", - "20": "Initial Design: Sobol", - "21": "Initial Design: Sobol", - "22": "Initial Design: Sobol", - "23": "Initial Design: Sobol", - "24": "Initial Design: Sobol", - "25": "Initial Design: Sobol", - "26": "Initial Design: Sobol", - "27": "Initial Design: Sobol", - "28": "Initial Design: Sobol", - "29": "Initial Design: Sobol", - "30": "Initial Design: Sobol", - "31": "Initial Design: Sobol", - "32": "Initial Design: Sobol", - "33": "Initial Design: Sobol", - "34": "Initial Design: Sobol", - "35": "Initial Design: Sobol", - "36": "Initial Design: Sobol", - "37": "Initial Design: Sobol", - "38": "Initial Design: Sobol", - "39": "Initial Design: Sobol", - "40": "Initial Design: Sobol", - "41": "Initial Design: Sobol", - "42": "Initial Design: Sobol", - "43": "Initial Design: Sobol", - "44": "Initial Design: Sobol", - "45": "Initial Design: Sobol", - "46": "Initial Design: Sobol", - "47": "Initial Design: Sobol", - "48": "Initial Design: Sobol", - "49": "Initial Design: Sobol", - "50": "Initial Design: Sobol", - "51": "Initial Design: Sobol", - "52": "Initial Design: Sobol", - "53": "Initial Design: Sobol", - "54": "Initial Design: Sobol", - "55": "Initial Design: Sobol", - "56": "Initial Design: Sobol", - "57": "Initial Design: Sobol", - "58": "Initial Design: Sobol", - "59": "Initial Design: Sobol", - "60": "Initial Design: Sobol", - "61": "Initial Design: Sobol" - } -} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/scenario 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/scenario 2.json deleted file mode 100644 index d6ceef93..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_1/scenario 2.json +++ /dev/null @@ -1,155 +0,0 @@ -{ - "name": "bcdb4a7cedc37b0caa608ab570d77c03", - "deterministic": false, - "objectives": "cost", - "crash_cost": Infinity, - "termination_cost_threshold": Infinity, - "walltime_limit": 60, - "cputime_limit": Infinity, - "trial_walltime_limit": null, - "trial_memory_limit": null, - "n_trials": 500, - "use_default_config": false, - "instances": null, - "instance_features": null, - "min_budget": null, - "max_budget": null, - "seed": 0, - "n_workers": 8, - "_meta": { - "facade": { - "name": "HyperparameterOptimizationFacade" - }, - "runner": { - "name": "DaskParallelRunner" - }, - "model": { - "name": "RandomForest", - "types": [ - 3, - 0, - 0, - 3, - 0, - 4, - 0 - ], - "bounds": [ - [ - 3, - NaN - ], - [ - 0, - 1.0 - ], - [ - 0, - 1.0 - ], - [ - 3, - NaN - ], - [ - -1.0, - 1.0 - ], - [ - 4, - NaN - ], - [ - -1.0, - 1.0 - ] - ], - "pca_components": 7, - "n_trees": 10, - "n_points_per_tree": -1, - "ratio_features": 1.0, - "min_samples_split": 2, - "min_samples_leaf": 1, - "max_depth": 1048576, - "eps_purity": 1e-08, - "max_nodes": 1048576, - "bootstrapping": true - }, - "acquisition_maximizer": { - "name": "LocalAndSortedRandomSearch", - "acquisition_function": { - "name": "EI", - "xi": 0.0, - "log": true - }, - "challengers": 10000, - "seed": 0, - "random_search": { - "name": "RandomSearch", - "acquisition_function": { - "name": "EI", - "xi": 0.0, - "log": true - }, - "challengers": 5000, - "seed": 0 - }, - "local_search": { - "name": "LocalSearch", - "acquisition_function": { - "name": "EI", - "xi": 0.0, - "log": true - }, - "challengers": 5000, - "seed": 0, - "max_steps": null, - "n_steps_plateau_walk": 10, - "vectorization_min_obtain": 2, - "vectorization_max_obtain": 64 - } - }, - "acquisition_function": { - "name": "EI", - "xi": 0.0, - "log": true - }, - "intensifier": { - "name": "Intensifier", - "max_incumbents": 10, - "seed": 0 - }, - "initial_design": { - "name": "SobolInitialDesign", - "n_configs": 70, - "n_configs_per_hyperparameter": 10, - "additional_configs": [], - "seed": 0 - }, - "random_design": { - "name": "ProbabilityRandomDesign", - "seed": 0, - "probability": 0.2 - }, - "runhistory_encoder": { - "name": "RunHistoryLogScaledEncoder", - "considered_states": [ - 1, - 2, - 4 - ], - "lower_budget_states": [], - "scale_percentage": 5, - "seed": 0 - }, - "multi_objective_algorithm": null, - "config_selector": { - "name": "ConfigSelector", - "retrain_after": 8, - "retries": 16, - "min_trials": 1 - }, - "version": "2.0.2" - }, - "output_directory": "smac3_output/bcdb4a7cedc37b0caa608ab570d77c03/0" -} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/configspace 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/configspace 2.json deleted file mode 100644 index be8aa231..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_2/configspace 2.json +++ /dev/null @@ -1 +0,0 @@ -{"name": null, "hyperparameters": [{"type": "categorical", "name": "activation", "choices": ["logistic", "tanh", "relu"], "weights": null, "default_value": "tanh", "meta": null}, {"type": "uniform_int", "name": "n_layer", "lower": 1, "upper": 5, "default_value": 1, "log": false, "meta": null}, {"type": "uniform_int", "name": "n_neurons", "lower": 8, "upper": 256, "default_value": 10, "log": true, "meta": null}, {"type": "categorical", "name": "solver", "choices": ["lbfgs", "sgd", "adam"], "weights": null, "default_value": "adam", "meta": null}, {"type": "uniform_int", "name": "batch_size", "lower": 30, "upper": 300, "default_value": 200, "log": false, "meta": null}, {"type": "categorical", "name": "learning_rate", "choices": ["constant", "invscaling", "adaptive"], "weights": null, "default_value": "constant", "meta": null}, {"type": "uniform_float", "name": "learning_rate_init", "lower": 0.0001, "upper": 1.0, "default_value": 0.001, "log": true, "meta": null}], "conditions": [{"type": "IN", "child": "batch_size", "parent": "solver", "values": ["sgd", "adam"]}, {"type": "EQ", "child": "learning_rate", "parent": "solver", "value": "sgd"}, {"type": "IN", "child": "learning_rate_init", "parent": "solver", "values": ["sgd", "adam"]}], "forbiddens": [], "python_module_version": "1.2.0", "format_version": 0.4} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/intensifier 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/intensifier 2.json deleted file mode 100644 index 57b6338e..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_2/intensifier 2.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "incumbent_ids": [ - 4 - ], - "rejected_config_ids": [ - 8, - 1, - 7, - 3, - 13, - 10, - 14, - 12, - 2, - 17, - 6, - 15, - 16, - 9 - ], - "incumbents_changed": 3, - "trajectory": [ - { - "config_ids": [ - 1 - ], - "costs": [ - 0.6699473847106159 - ], - "trial": 1, - "walltime": 0.7647695541381836 - }, - { - "config_ids": [ - 10 - ], - "costs": [ - 0.05601722892809243 - ], - "trial": 14, - "walltime": 5.185967445373535 - }, - { - "config_ids": [ - 4 - ], - "costs": [ - 0.02671515526668729 - ], - "trial": 28, - "walltime": 9.271684408187866 - } - ], - "state": { - "queue": [ - [ - 5, - 2 - ], - [ - 19, - 2 - ], - [ - 11, - 4 - ], - [ - 18, - 4 - ] - ] - } -} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/optimization 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/optimization 2.json deleted file mode 100644 index 445f0dfa..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_2/optimization 2.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "used_walltime": 15.669282674789429, - "used_target_function_walltime": 102.76791548728943, - "last_update": 1706277639.8103335, - "finished": true -} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/runhistory 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/runhistory 2.json deleted file mode 100644 index 5be2d9d8..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_2/runhistory 2.json +++ /dev/null @@ -1,766 +0,0 @@ -{ - "stats": { - "submitted": 50, - "finished": 50, - "running": 0 - }, - "data": [ - [ - 1, - null, - 209652396, - null, - 0.6699473847106159, - 0.6026918888092041, - 1, - 1706277624.222565, - 1706277624.8264945, - {} - ], - [ - 2, - null, - 209652396, - null, - 0.025597338285360594, - 3.1097147464752197, - 1, - 1706277624.2967758, - 1706277627.407669, - {} - ], - [ - 3, - null, - 209652396, - null, - 0.6583333333333333, - 2.336819648742676, - 1, - 1706277624.3608909, - 1706277626.6990438, - {} - ], - [ - 4, - null, - 209652396, - null, - 0.02560352831940571, - 2.9831438064575195, - 1, - 1706277624.415114, - 1706277627.3992622, - {} - ], - [ - 5, - null, - 209652396, - null, - 0.8992773135252243, - 14.455097913742065, - 1, - 1706277624.4994059, - 1706277638.9561086, - {} - ], - [ - 6, - null, - 209652396, - null, - 0.3678195605075827, - 4.080667018890381, - 1, - 1706277624.5847695, - 1706277628.6664875, - {} - ], - [ - 7, - null, - 209652396, - null, - 0.8992757660167131, - 1.912712574005127, - 1, - 1706277624.6662445, - 1706277626.5800583, - {} - ], - [ - 8, - null, - 209652396, - null, - 0.8241210151655834, - 0.5521142482757568, - 1, - 1706277624.7443416, - 1706277625.2985408, - {} - ], - [ - 9, - null, - 209652396, - null, - 0.0317254719900959, - 5.659100532531738, - 1, - 1706277624.910818, - 1706277630.5716987, - {} - ], - [ - 1, - null, - 398764591, - null, - 0.6861915815536985, - 0.8483016490936279, - 1, - 1706277625.409777, - 1706277626.2611463, - {} - ], - [ - 10, - null, - 209652396, - null, - 0.056199319096254974, - 0.7128767967224121, - 1, - 1706277626.3443322, - 1706277627.058281, - {} - ], - [ - 1, - null, - 924231285, - null, - 0.7017641597028783, - 0.723477840423584, - 1, - 1706277626.6957843, - 1706277627.4203732, - {} - ], - [ - 7, - null, - 398764591, - null, - 0.8987202104611576, - 1.744448184967041, - 1, - 1706277626.7947023, - 1706277628.5401833, - {} - ], - [ - 3, - null, - 398764591, - null, - 0.8458000619003405, - 1.7513632774353027, - 1, - 1706277627.1429644, - 1706277628.896084, - {} - ], - [ - 10, - null, - 398764591, - null, - 0.056761064685855844, - 0.6708414554595947, - 1, - 1706277627.5016356, - 1706277628.1742969, - {} - ], - [ - 2, - null, - 398764591, - null, - 0.03006035283194064, - 3.3852593898773193, - 1, - 1706277627.64613, - 1706277631.0330572, - {} - ], - [ - 4, - null, - 398764591, - null, - 0.027273290003095085, - 2.520305633544922, - 1, - 1706277627.740861, - 1706277630.2628653, - {} - ], - [ - 11, - null, - 209652396, - null, - 0.03840916125038696, - 7.205214738845825, - 1, - 1706277628.2559705, - 1706277635.4626877, - {} - ], - [ - 10, - null, - 924231285, - null, - 0.05509130300216647, - 0.6167201995849609, - 1, - 1706277628.640889, - 1706277629.2592533, - {} - ], - [ - 7, - null, - 924231285, - null, - 0.9009470752089137, - 1.3385493755340576, - 1, - 1706277628.76265, - 1706277630.102818, - {} - ], - [ - 6, - null, - 398764591, - null, - 0.398457134014237, - 3.8393754959106445, - 1, - 1706277628.9893534, - 1706277632.830835, - {} - ], - [ - 3, - null, - 924231285, - null, - Infinity, - 0.0, - 2, - 1706277629.3351586, - 1706277630.2660835, - { - "traceback": "Traceback (most recent call last):\n File \"/home/skrebs/Projects/Software/SMAC3/smac/runner/target_function_runner.py\", line 184, in run\n rval = self(config_copy, target_function, kwargs)\n File \"/home/skrebs/Projects/Software/SMAC3/smac/runner/target_function_runner.py\", line 257, in __call__\n return algorithm(config, **algorithm_kwargs)\n File \"/home/skrebs/Projects/Software/SMAC3/examples/2_multi_fidelity/1_mlp_epochs.py\", line 107, in train\n score = cross_val_score(classifier, dataset.data, dataset.target, cv=cv, error_score=\"raise\")\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/model_selection/_validation.py\", line 562, in cross_val_score\n cv_results = cross_validate(\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/utils/_param_validation.py\", line 211, in wrapper\n return func(*args, **kwargs)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/model_selection/_validation.py\", line 309, in cross_validate\n results = parallel(\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/utils/parallel.py\", line 65, in __call__\n return super().__call__(iterable_with_config)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 1088, in __call__\n while self.dispatch_one_batch(iterator):\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 901, in dispatch_one_batch\n self._dispatch(tasks)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 819, in _dispatch\n job = self._backend.apply_async(batch, callback=cb)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/_parallel_backends.py\", line 208, in apply_async\n result = ImmediateResult(func)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/_parallel_backends.py\", line 597, in __init__\n self.results = batch()\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 288, in __call__\n return [func(*args, **kwargs)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 288, in \n return [func(*args, **kwargs)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/utils/parallel.py\", line 127, in __call__\n return self.function(*args, **kwargs)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/model_selection/_validation.py\", line 732, in _fit_and_score\n estimator.fit(X_train, y_train, **fit_params)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/base.py\", line 1151, in wrapper\n return fit_method(estimator, *args, **kwargs)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py\", line 753, in fit\n return self._fit(X, y, incremental=False)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py\", line 496, in _fit\n raise ValueError(\nValueError: Solver produced non-finite parameter weights. The input data may contain large values and need to be preprocessed.\n", - "error": "ValueError('Solver produced non-finite parameter weights. The input data may contain large values and need to be preprocessed.')" - } - ], - [ - 12, - null, - 209652396, - null, - 0.10016403590219747, - 1.307952642440796, - 1, - 1706277630.1908402, - 1706277631.4997926, - {} - ], - [ - 13, - null, - 209652396, - null, - 0.89983286908078, - 0.2807650566101074, - 1, - 1706277630.3577998, - 1706277630.6401331, - {} - ], - [ - 4, - null, - 924231285, - null, - 0.02726864747756108, - 2.861912250518799, - 1, - 1706277630.4543989, - 1706277633.3173153, - {} - ], - [ - 14, - null, - 209652396, - null, - 0.3895543175487466, - 0.808574914932251, - 1, - 1706277630.6644979, - 1706277631.4741547, - {} - ], - [ - 9, - null, - 398764591, - null, - 0.032271742494583755, - 5.011699914932251, - 1, - 1706277630.76806, - 1706277635.7808084, - {} - ], - [ - 13, - null, - 398764591, - null, - 0.8886908077994429, - 0.32169437408447266, - 1, - 1706277631.1505885, - 1706277631.4740863, - {} - ], - [ - 2, - null, - 924231285, - null, - 0.029497059733828435, - 2.967978000640869, - 1, - 1706277631.5600839, - 1706277634.529425, - {} - ], - [ - 12, - null, - 398764591, - null, - 0.08014391829155054, - 1.0976686477661133, - 1, - 1706277631.6530569, - 1706277632.7523105, - {} - ], - [ - 14, - null, - 398764591, - null, - 0.495255338904364, - 0.6123650074005127, - 1, - 1706277631.7122657, - 1706277632.3262582, - {} - ], - [ - 13, - null, - 924231285, - null, - 0.9048467966573817, - 0.22585248947143555, - 1, - 1706277632.4026914, - 1706277632.6302176, - {} - ], - [ - 14, - null, - 924231285, - null, - 0.31948467966573824, - 0.7365078926086426, - 1, - 1706277632.7140539, - 1706277633.4522555, - {} - ], - [ - 15, - null, - 209652396, - null, - 0.04341844630145464, - 1.4842357635498047, - 1, - 1706277632.830031, - 1706277634.3161619, - {} - ], - [ - 12, - null, - 924231285, - null, - 0.08125657691117305, - 1.5107638835906982, - 1, - 1706277632.9116373, - 1706277634.4242318, - {} - ], - [ - 6, - null, - 924231285, - null, - 0.36005261528938415, - 3.516299247741699, - 1, - 1706277633.4184535, - 1706277636.9357994, - {} - ], - [ - 16, - null, - 209652396, - null, - 0.05231352522438881, - 1.0166974067687988, - 1, - 1706277633.539396, - 1706277634.5570924, - {} - ], - [ - 17, - null, - 209652396, - null, - 0.15859950479727636, - 0.510662317276001, - 1, - 1706277634.4225478, - 1706277634.9349742, - {} - ], - [ - 15, - null, - 398764591, - null, - 0.043413803775920856, - 1.1191012859344482, - 1, - 1706277634.5313032, - 1706277635.651467, - {} - ], - [ - 18, - null, - 209652396, - null, - 0.03673320953265247, - 1.036933183670044, - 1, - 1706277635.221057, - 1706277636.2595854, - {} - ], - [ - 16, - null, - 398764591, - null, - 0.042296502630764565, - 1.2329812049865723, - 1, - 1706277635.3001854, - 1706277636.5348752, - {} - ], - [ - 17, - null, - 398764591, - null, - 0.1229944289693593, - 0.5270049571990967, - 1, - 1706277635.3790934, - 1706277635.907605, - {} - ], - [ - 19, - null, - 209652396, - null, - 0.04006809037449699, - 1.2701239585876465, - 1, - 1706277635.5504642, - 1706277636.8224409, - {} - ], - [ - 11, - null, - 398764591, - null, - 0.0395094398019189, - 3.738813638687134, - 1, - 1706277635.738447, - 1706277639.4788716, - {} - ], - [ - 15, - null, - 924231285, - null, - 0.036725471990095904, - 1.2338676452636719, - 1, - 1706277635.8669333, - 1706277637.1028805, - {} - ], - [ - 9, - null, - 924231285, - null, - 0.03617301145156304, - 3.797776222229004, - 1, - 1706277635.9984384, - 1706277639.797911, - {} - ], - [ - 17, - null, - 924231285, - null, - 0.11129990714948934, - 0.5645942687988281, - 1, - 1706277636.3598018, - 1706277636.9253795, - {} - ], - [ - 18, - null, - 398764591, - null, - 0.040072732900030994, - 1.1205317974090576, - 1, - 1706277636.6408405, - 1706277637.7630773, - {} - ], - [ - 16, - null, - 924231285, - null, - 0.03450324976787367, - 1.0402038097381592, - 1, - 1706277636.9236588, - 1706277637.9659233, - {} - ], - [ - 19, - null, - 398764591, - null, - 0.038952336737852145, - 0.7655572891235352, - 1, - 1706277637.010183, - 1706277637.7774007, - {} - ] - ], - "configs": { - "1": { - "activation": "logistic", - "n_layer": 4, - "n_neurons": 11, - "solver": "lbfgs" - }, - "2": { - "activation": "tanh", - "n_layer": 1, - "n_neurons": 146, - "solver": "adam", - "batch_size": 77, - "learning_rate_init": 0.000534923804864797 - }, - "3": { - "activation": "relu", - "n_layer": 5, - "n_neurons": 43, - "solver": "sgd", - "batch_size": 157, - "learning_rate": "invscaling", - "learning_rate_init": 0.08903539757637764 - }, - "4": { - "activation": "logistic", - "n_layer": 2, - "n_neurons": 90, - "solver": "adam", - "batch_size": 175, - "learning_rate_init": 0.004156370184967407 - }, - "5": { - "activation": "logistic", - "n_layer": 4, - "n_neurons": 200, - "solver": "sgd", - "batch_size": 204, - "learning_rate": "constant", - "learning_rate_init": 0.016071068084706618 - }, - "6": { - "activation": "tanh", - "n_layer": 4, - "n_neurons": 63, - "solver": "sgd", - "batch_size": 144, - "learning_rate": "invscaling", - "learning_rate_init": 0.0056279320474151686 - }, - "7": { - "activation": "relu", - "n_layer": 5, - "n_neurons": 29, - "solver": "adam", - "batch_size": 173, - "learning_rate_init": 0.503948959867121 - }, - "8": { - "activation": "logistic", - "n_layer": 1, - "n_neurons": 8, - "solver": "adam", - "batch_size": 240, - "learning_rate_init": 0.8212461922256864 - }, - "9": { - "activation": "relu", - "n_layer": 3, - "n_neurons": 118, - "solver": "lbfgs" - }, - "10": { - "activation": "tanh", - "n_layer": 1, - "n_neurons": 35, - "solver": "lbfgs" - }, - "11": { - "activation": "tanh", - "n_layer": 4, - "n_neurons": 80, - "solver": "lbfgs" - }, - "12": { - "activation": "tanh", - "n_layer": 3, - "n_neurons": 27, - "solver": "lbfgs" - }, - "13": { - "activation": "logistic", - "n_layer": 5, - "n_neurons": 16, - "solver": "lbfgs" - }, - "14": { - "activation": "relu", - "n_layer": 4, - "n_neurons": 12, - "solver": "lbfgs" - }, - "15": { - "activation": "tanh", - "n_layer": 1, - "n_neurons": 63, - "solver": "lbfgs" - }, - "16": { - "activation": "relu", - "n_layer": 2, - "n_neurons": 43, - "solver": "lbfgs" - }, - "17": { - "activation": "relu", - "n_layer": 2, - "n_neurons": 18, - "solver": "lbfgs" - }, - "18": { - "activation": "tanh", - "n_layer": 1, - "n_neurons": 82, - "solver": "lbfgs" - }, - "19": { - "activation": "relu", - "n_layer": 1, - "n_neurons": 57, - "solver": "lbfgs" - } - }, - "config_origins": { - "1": "Initial Design: Sobol", - "2": "Initial Design: Sobol", - "3": "Initial Design: Sobol", - "4": "Initial Design: Sobol", - "5": "Initial Design: Sobol", - "6": null, - "7": null, - "8": null, - "9": null, - "10": "Acquisition Function Maximizer: Random Search (sorted)", - "11": "Acquisition Function Maximizer: Random Search (sorted)", - "12": "Acquisition Function Maximizer: Random Search (sorted)", - "13": "Acquisition Function Maximizer: Random Search (sorted)", - "14": "Acquisition Function Maximizer: Random Search (sorted)", - "15": "Acquisition Function Maximizer: Local Search", - "16": "Acquisition Function Maximizer: Local Search", - "17": "Acquisition Function Maximizer: Local Search", - "18": "Acquisition Function Maximizer: Random Search (sorted)", - "19": "Acquisition Function Maximizer: Random Search (sorted)" - } -} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/scenario 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/scenario 2.json deleted file mode 100644 index 679a91c6..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_2/scenario 2.json +++ /dev/null @@ -1,155 +0,0 @@ -{ - "name": "e262f9639f7cb897f1e597cb5b717d0c", - "deterministic": false, - "objectives": "cost", - "crash_cost": Infinity, - "termination_cost_threshold": Infinity, - "walltime_limit": Infinity, - "cputime_limit": Infinity, - "trial_walltime_limit": null, - "trial_memory_limit": null, - "n_trials": 50, - "use_default_config": false, - "instances": null, - "instance_features": null, - "min_budget": null, - "max_budget": null, - "seed": 0, - "n_workers": 8, - "_meta": { - "facade": { - "name": "HyperparameterOptimizationFacade" - }, - "runner": { - "name": "DaskParallelRunner" - }, - "model": { - "name": "RandomForest", - "types": [ - 3, - 0, - 0, - 3, - 0, - 4, - 0 - ], - "bounds": [ - [ - 3, - NaN - ], - [ - 0, - 1.0 - ], - [ - 0, - 1.0 - ], - [ - 3, - NaN - ], - [ - -1.0, - 1.0 - ], - [ - 4, - NaN - ], - [ - -1.0, - 1.0 - ] - ], - "pca_components": 7, - "n_trees": 10, - "n_points_per_tree": -1, - "ratio_features": 1.0, - "min_samples_split": 2, - "min_samples_leaf": 1, - "max_depth": 1048576, - "eps_purity": 1e-08, - "max_nodes": 1048576, - "bootstrapping": true - }, - "acquisition_maximizer": { - "name": "LocalAndSortedRandomSearch", - "acquisition_function": { - "name": "EI", - "xi": 0.0, - "log": true - }, - "challengers": 10000, - "seed": 0, - "random_search": { - "name": "RandomSearch", - "acquisition_function": { - "name": "EI", - "xi": 0.0, - "log": true - }, - "challengers": 5000, - "seed": 0 - }, - "local_search": { - "name": "LocalSearch", - "acquisition_function": { - "name": "EI", - "xi": 0.0, - "log": true - }, - "challengers": 5000, - "seed": 0, - "max_steps": null, - "n_steps_plateau_walk": 10, - "vectorization_min_obtain": 2, - "vectorization_max_obtain": 64 - } - }, - "acquisition_function": { - "name": "EI", - "xi": 0.0, - "log": true - }, - "intensifier": { - "name": "Intensifier", - "max_incumbents": 10, - "seed": 0 - }, - "initial_design": { - "name": "SobolInitialDesign", - "n_configs": 5, - "n_configs_per_hyperparameter": 10, - "additional_configs": [], - "seed": 0 - }, - "random_design": { - "name": "ProbabilityRandomDesign", - "seed": 0, - "probability": 0.2 - }, - "runhistory_encoder": { - "name": "RunHistoryLogScaledEncoder", - "considered_states": [ - 1, - 2, - 4 - ], - "lower_budget_states": [], - "scale_percentage": 5, - "seed": 0 - }, - "multi_objective_algorithm": null, - "config_selector": { - "name": "ConfigSelector", - "retrain_after": 8, - "retries": 16, - "min_trials": 1 - }, - "version": "2.0.2" - }, - "output_directory": "smac3_output/e262f9639f7cb897f1e597cb5b717d0c/0" -} \ No newline at end of file diff --git a/tests/test_runs/__init__ 2.py b/tests/test_runs/__init__ 2.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/test_utils/__init__ 2.py b/tests/test_utils/__init__ 2.py deleted file mode 100644 index e69de29b..00000000 From a400d2fd1a9add55e638cb38ba04f7baee3eb6af Mon Sep 17 00:00:00 2001 From: Kristina Thieme Date: Tue, 8 Oct 2024 13:40:07 +0200 Subject: [PATCH 05/12] deleted another unnecessary file --- docs/sg_execution_times.rst | 58 ------------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 docs/sg_execution_times.rst diff --git a/docs/sg_execution_times.rst b/docs/sg_execution_times.rst deleted file mode 100644 index 1654b0ec..00000000 --- a/docs/sg_execution_times.rst +++ /dev/null @@ -1,58 +0,0 @@ - -:orphan: - -.. _sphx_glr_sg_execution_times: - - -Computation times -================= -**00:00.000** total execution time for 8 files **from all galleries**: - -.. container:: - - .. raw:: html - - - - - - - - .. list-table:: - :header-rows: 1 - :class: table table-striped sg-datatable - - * - Example - - Time - - Mem (MB) - * - :ref:`sphx_glr_examples_api_ablation_paths.py` (``../examples/api/ablation_paths.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_examples_api_importances 2.py` (``../examples/api/importances 2.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_examples_api_importances.py` (``../examples/api/importances.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_examples_api_parallel_coordinates.py` (``../examples/api/parallel_coordinates.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_examples_api_pdp.py` (``../examples/api/pdp.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_examples_record_digits_sklearn.py` (``../examples/record/digits_sklearn.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_examples_record_minimal.py` (``../examples/record/minimal.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_examples_record_mnist_pytorch.py` (``../examples/record/mnist_pytorch.py``) - - 00:00.000 - - 0.0 From 4147a5a0bace98eacacd13d3472bcacf1f4d2822 Mon Sep 17 00:00:00 2001 From: Kristina Thieme Date: Mon, 21 Oct 2024 11:43:14 +0200 Subject: [PATCH 06/12] changed spelling errors --- docs/converters/bohb.rst | 2 +- docs/converters/custom_converters.rst | 2 +- docs/converters/smac.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/converters/bohb.rst b/docs/converters/bohb.rst index 89d1ae0b..681da493 100644 --- a/docs/converters/bohb.rst +++ b/docs/converters/bohb.rst @@ -7,7 +7,7 @@ To load an BOHB run into DeepCAVE, it is necessary to install HpBandSter, e.g. v pip install deepcave[bohb] -To be able to load it correctly into DeepCAVE, you need save the configuration space as a ConfigSpace JSON file: +To be able to load it correctly into DeepCAVE, you need to save the configuration space as a ConfigSpace JSON file: .. code-block:: python diff --git a/docs/converters/custom_converters.rst b/docs/converters/custom_converters.rst index 9f5467ff..d8820a6a 100644 --- a/docs/converters/custom_converters.rst +++ b/docs/converters/custom_converters.rst @@ -71,7 +71,7 @@ Step 3: Implement the methods Inside this method, you have to provide a configspace, objectives, and meta data, as well as the trials, which have to be added to the newly created run object. -4. **is_valid_run**: This method sould return True, if the given path belongs to a valid run. Valid means, that the given run +4. **is_valid_run**: This method should return True if the given path belongs to a valid run. Valid means that the given run directory contains all files that are necessary for further processing (e.g. a configspace.json). Step 5: Document your converter diff --git a/docs/converters/smac.rst b/docs/converters/smac.rst index e6f75eda..7d0e7e92 100644 --- a/docs/converters/smac.rst +++ b/docs/converters/smac.rst @@ -7,7 +7,7 @@ runs created with the newer SMAC version >= 2.0.0. As Auto-Sklearn and Auto-PyTorch rely on SMAC for hyperparameter optimization, the SMAC converter also allows to load runs created with these tools. -To be able to load it correctly into DeepCAVE, you need save the configuration space as a ConfigSpace JSON file, +To be able to load it correctly into DeepCAVE, you need to save the configuration space as a ConfigSpace JSON file, as well as the run history: .. code-block:: python From df0e01a91de816a00f1bda8f7cd97ddcea72f9ba Mon Sep 17 00:00:00 2001 From: Lukas Fehring Date: Fri, 13 Dec 2024 12:43:29 +0100 Subject: [PATCH 07/12] Update requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c4f6fe31..abf3bcd7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ dash==2.0.0 dash-extensions==0.0.71 dash-bootstrap-components==1.0.3 redis>=4.1.4 -rq>=1.10.1 +rq==1.10.1 # Pinned due to https://github.com/plotly/dash/issues/1992 # Pinning might be removed for dash>2.3.0 werkzeug==2.0.3 From b062727e301757e3b0daf7fa98db20a8e341c2af Mon Sep 17 00:00:00 2001 From: Lukas Fehring Date: Fri, 13 Dec 2024 13:52:24 +0100 Subject: [PATCH 08/12] Prepare addition of newer runhistry. Splitting SMAC3v2Run.from_path --- deepcave/runs/converters/smac3v2.py | 180 +++++++++++++++++----------- 1 file changed, 112 insertions(+), 68 deletions(-) diff --git a/deepcave/runs/converters/smac3v2.py b/deepcave/runs/converters/smac3v2.py index 7ccf0f25..ac88ee88 100644 --- a/deepcave/runs/converters/smac3v2.py +++ b/deepcave/runs/converters/smac3v2.py @@ -25,7 +25,7 @@ - SMAC3v2Run: Define a SMAC3v2 run object. """ -from typing import Union +from typing import Dict, List, Optional, Union import json import os @@ -145,79 +145,123 @@ def from_path(cls, path: Union[Path, str]) -> "SMAC3v2Run": config_origins = all_data["config_origins"] configs = all_data["configs"] - instance_ids = [] + instance_ids: List[int] = [] first_starttime = None - for ( - config_id, - instance_id, - seed, - budget, - cost, - time, - status, - starttime, - endtime, - additional_info, - ) in data: - if instance_id not in instance_ids: - instance_ids += [instance_id] - - if len(instance_ids) > 1: - raise RuntimeError("Instances are not supported.") - - config_id = str(config_id) - config = configs[config_id] - - if first_starttime is None: - first_starttime = starttime - - starttime = starttime - first_starttime - endtime = endtime - first_starttime - - if status == 0: - # still running - continue - elif status == 1: - status = Status.SUCCESS - elif status == 3: - status = Status.TIMEOUT - elif status == 4: - status = Status.MEMORYOUT - else: - status = Status.CRASHED - - if status != Status.SUCCESS: - # Costs which failed, should not be included - cost = [None] * len(cost) if isinstance(cost, list) else None - time = None - else: - time = endtime - starttime - - # Round budget - if budget: - budget = np.round(budget, 2) - else: - budget = 0.0 - - origin = None - if config_id in config_origins: - origin = config_origins[config_id] - - run.add( - costs=cost + [time] if isinstance(cost, list) else [cost, time], - config=config, - budget=budget, - seed=seed, - start_time=starttime, - end_time=endtime, - status=status, - origin=origin, - additional=additional_info, + + if type(data) is list: + import warnings + + warnings.warn( + "The runhistory.json file is in an outdated format.", + DeprecationWarning, + stacklevel=2, # Adjusts the stack level to point to the caller. ) + for ( + config_id, + instance_id, + seed, + budget, + cost, + time, + status, + starttime, + endtime, + additional_info, + ) in data: + run_dict = run._process_data_entry( + str(config_id), + instance_id, + seed, + budget, + cost, + time, + status, + starttime, + endtime, + additional_info, + first_starttime, + instance_ids, + configs, + config_origins, + ) + if run_dict is not None: + run.add(**run_dict) return run + def _process_data_entry( + self, + config_id: str, + instance_id: int, + seed: int, + budget: Optional[float], + cost: Optional[Union[List[Union[float, None]], float]], + time: Optional[float], + status: int, + starttime: float, + endtime: float, + additional_info: Optional[Dict], + first_starttime: Optional[float], + instance_ids: List[int], + configs: Dict, + config_origins: Dict[str, str], + ) -> Optional[Dict]: + if instance_id not in instance_ids: + instance_ids += [instance_id] + + if len(instance_ids) > 1: + raise RuntimeError("Instances are not supported.") + + config = configs[config_id] + + if first_starttime is None: + first_starttime = starttime + + starttime = starttime - first_starttime + endtime = endtime - first_starttime + + if status == 0: + # still running + return None + elif status == 1: + status = Status.SUCCESS + elif status == 3: + status = Status.TIMEOUT + elif status == 4: + status = Status.MEMORYOUT + else: + status = Status.CRASHED + + if status != Status.SUCCESS: + # Costs which failed, should not be included + cost = [None] * len(cost) if isinstance(cost, list) else None + time = None + else: + time = endtime - starttime + + # Round budget + if budget: + budget = np.round(budget, 2) + else: + budget = 0.0 + + origin = None + if config_id in config_origins: + origin = config_origins[config_id] + + return { + "costs": cost + [time] if isinstance(cost, list) else [cost, time], + "config": config, + "budget": budget, + "seed": seed, + "start_time": starttime, + "end_time": endtime, + "status": status, + "origin": origin, + "additional": additional_info, + } + @classmethod def is_valid_run(cls, path_name: str) -> bool: """ From 2c09febdf43259505d7d0e3e2b04ec3b47da611d Mon Sep 17 00:00:00 2001 From: Lukas Fehring Date: Fri, 13 Dec 2024 14:58:28 +0100 Subject: [PATCH 09/12] Add leading of new run_history --- deepcave/runs/converters/smac3v2.py | 34 +++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/deepcave/runs/converters/smac3v2.py b/deepcave/runs/converters/smac3v2.py index ac88ee88..42a903db 100644 --- a/deepcave/runs/converters/smac3v2.py +++ b/deepcave/runs/converters/smac3v2.py @@ -149,7 +149,7 @@ def from_path(cls, path: Union[Path, str]) -> "SMAC3v2Run": first_starttime = None - if type(data) is list: + if isinstance(data, list): import warnings warnings.warn( @@ -187,7 +187,37 @@ def from_path(cls, path: Union[Path, str]) -> "SMAC3v2Run": ) if run_dict is not None: run.add(**run_dict) - + elif isinstance(data, dict): + for config_id, config_data in data.items(): + instance_id = config_data["instance"] + seed = config_data["seed"] + budget = config_data["budget"] + cost = config_data["cost"] + time = config_data["time"] + status = config_data["status"] + starttime = config_data["starttime"] + endtime = config_data["endtime"] + additional_info = config_data["additional_info"] + run_dict = run._process_data_entry( + config_id, + instance_id, + seed, + budget, + cost, + time, + status, + starttime, + endtime, + additional_info, + first_starttime, + instance_ids, + configs, + config_origins, + ) + if run_dict is not None: + run.add(**run_dict) + else: + raise RuntimeError("Data in runhistory.json is not in a valid format.") return run def _process_data_entry( From 00195b297a25d67f9215747bb144ef691ce50d3c Mon Sep 17 00:00:00 2001 From: Lukas Fehring Date: Mon, 16 Dec 2024 16:14:01 +0100 Subject: [PATCH 10/12] Update permissions for the docs.yml file --- .github/workflows/docs.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2db6efef..9ac74b4b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -63,6 +63,9 @@ jobs: cp -r ../${{ env.name }}/docs/build/html $branch_name - name: Push to gh-pages + permissions: + contents: write + id: ${{ github.actor }} if: (contains(github.ref, 'development') || contains(github.ref, 'main')) run: | last_commit=$(git log --pretty=format:"%an: %s") From 1407e961219f5134d197f034beee87c928dae2e2 Mon Sep 17 00:00:00 2001 From: Lukas Fehring Date: Mon, 16 Dec 2024 16:16:40 +0100 Subject: [PATCH 11/12] Change permissions location --- .github/workflows/docs.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9ac74b4b..560a142a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -26,6 +26,9 @@ env: jobs: build-and-deploy: runs-on: ubuntu-latest + permissions: + contents: write + actions: read defaults: run: shell: bash # Default to using bash on all @@ -63,9 +66,6 @@ jobs: cp -r ../${{ env.name }}/docs/build/html $branch_name - name: Push to gh-pages - permissions: - contents: write - id: ${{ github.actor }} if: (contains(github.ref, 'development') || contains(github.ref, 'main')) run: | last_commit=$(git log --pretty=format:"%an: %s") From ba6f0b94fc8e2d6919804588f22d9182010db5dd Mon Sep 17 00:00:00 2001 From: Kristina Thieme Date: Wed, 8 Jan 2025 11:28:41 +0100 Subject: [PATCH 12/12] fixed numpy install and github page --- CHANGELOG.md | 5 +++++ Makefile | 2 ++ docs/installation.rst | 3 +++ requirements.txt | 4 ++-- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59288d62..8d170491 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# Version 1.3.4 + +## Installation +- Numpy will be installed via conda instead of pip to fix linkage + # Version 1.3.3 ## Converters diff --git a/Makefile b/Makefile index c150b3c0..92a1bdd5 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,9 @@ FLAKE8 ?= flake8 install: $(PIP) install -e . +# Fix numpy as version 2.1.0 will drop support for Python 3.9 install-dev: + conda install -y numpy=2.0.1 $(PIP) install -e ".[dev]" pre-commit install diff --git a/docs/installation.rst b/docs/installation.rst index f5bc4ead..d50f9f79 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -52,6 +52,9 @@ If you want to contribute to DeepCAVE, you can clone it from GitHub and install While it is generally expected to function correctly on MacOS, some issues may arise due to compatibility with Swig. Specifically, users may encounter problems with the Parallel Coordinates and Importance Plugin on MacOS. + NumPy currently needs to be installed with conda (as opposed to pip) to ensure it being linked to openblas + and not the Mac specific accelerate. The latter, if used, will result in a plugin failure for + Configuration Footprint. Currently, DeepCAVE cannot be run on Windows due to its reliance on a bash script for starting services such as Redis, workers, and the webserver. diff --git a/requirements.txt b/requirements.txt index c4f6fe31..f710af33 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ setuptools==68.2.2 absl-py>=1.0.0 jsonlines>=3.0.0 pandas>=1.3.4 -numpy==2.0.1 # Fix numpy as version 2.1.0 will drop support for Python 3.9 +# numpy needs to be installed via conda instead of pip matplotlib==3.9.0 seaborn>=0.13.0 pyyaml>=6.0.1 @@ -24,7 +24,7 @@ dash==2.0.0 dash-extensions==0.0.71 dash-bootstrap-components==1.0.3 redis>=4.1.4 -rq>=1.10.1 +rq==1.10.1 # Pinned due to https://github.com/plotly/dash/issues/1992 # Pinning might be removed for dash>2.3.0 werkzeug==2.0.3