diff --git a/SCHEMA.md b/SCHEMA.md index ae9c18b..60bf7b1 100644 --- a/SCHEMA.md +++ b/SCHEMA.md @@ -22,6 +22,7 @@ Three design choices shape everything below: - [Library](#library) - [Thing types](#thing-types) - [Implementation](#implementation) + - [Reference](#reference) - [ProblemLike](#problemlike) (shared fields) - [Problem](#problem) - [Suite](#suite) @@ -29,7 +30,7 @@ Three design choices shape everything below: - [Shared building blocks](#shared-building-blocks) - [Variable](#variable) / [VariableType](#variabletype) - [Constraint](#constraint) / [ConstraintType](#constrainttype) - - [Reference](#reference) / [Link](#link) + - [Link](#link) - [ValueRange](#valuerange) - [YesNoSome](#yesnosome) @@ -43,26 +44,33 @@ IDs are free-form but must be unique and the convention is to add a prefix marki | Prefix | Type | |---------|------------------| | `impl_` | Implementation | +| `ref_` | Reference | | `fn_` | Problem | | `suite_`| Suite | | `gen_` | Generator | -On load the library validates that every ID referenced by a suite (`problems`) or problem (`implementations`) exists and has the correct type. Suites also have their `fidelity_levels` auto-populated from their problems. +On load the library validates that every ID referenced by a suite (`problems`), problem (`implementations`), or any [ProblemLike](#problemlike) (`references`) exists and has the correct type. Suites also have their `fidelity_levels` auto-populated from their problems. ```yaml impl_coco: type: implementation name: COCO description: Comparing Continuous Optimisers +ref_bbob: + type: reference + title: "Real-Parameter Black-Box Optimization Benchmarking: Experimental Setup" + link: {type: url, url: "https://numbbo.github.io/coco/"} fn_sphere: type: problem name: Sphere objectives: [1] implementations: [impl_coco] + references: [ref_bbob] suite_bbob: type: suite name: BBOB problems: [fn_sphere] + references: [ref_bbob] ``` --- @@ -112,6 +120,32 @@ impl_py_cocoex: - {type: package, url: https://pypi.org/project/coco-experiment/} ``` +### Reference + +A bibliographic pointer stored at the top level of the [Library](#library) so that a single reference can be shared by multiple problems, suites, generators, and implementations. +Requires either a `title` or a `link` and optionally a list of `authors`. + +| Field | Type | Notes | +|-----------|------------------------|------------------------------------------| +| `title` | str? | required if `link` is not given | +| `authors` | list of str? | | +| `link` | [Link](#link)? | required if `title` is not given | + +```yaml +ref_honey_badger: + type: reference + title: "Honey Badger Algorithm: New metaheuristic algorithm for solving optimization problems." + authors: + - Fatma A. Hashim + - Essam H. Houssein + - Kashif Hussain + - Mai S. Mabrouk + - Walid Al-Atabany + link: {type: doi, url: "https://doi.org/10.1016/j.matcom.2021.08.013"} +``` + +[ProblemLike](#problemlike) entities refer to a reference by its ID in the `references` field. + ### ProblemLike Fields shared by [Problem](#problem), [Suite](#suite), and [Generator](#generator). @@ -123,7 +157,7 @@ The schema deliberately puts most descriptive fields here so suites can be chara | `long_name` | str? | | | `description` | str? (markdown) | longer prose | | `tags` | set of str? | free-form keywords | -| `references` | set of [Reference](#reference)? | | +| `references` | set of IDs? | must resolve to [Reference](#reference)s | | `implementations` | set of IDs? | must resolve to [Implementation](#implementation)s | | `objectives` | set of int? | e.g. `{1}`, `{2, 3}` — **not** a ValueRange | | `variables` | set of [Variable](#variable)? | | @@ -252,23 +286,6 @@ constraints: `box | linear | function | unknown`. `function` covers non-linear/black-box constraints. -### Reference - -Bibliographic pointer. -Requires either a `title` or a `link` and optionally a list of `authors`. - -```yaml -references: - - title: "Honey Badger Algorithm: New metaheuristic algorithm for solving optimization problems." - authors: - - Fatma A. Hashim - - Essam H. Houssein - - Kashif Hussain - - Mai S. Mabrouk - - Walid Al-Atabany - link: {type: doi, url: "https://doi.org/10.1016/j.matcom.2021.08.013"] -``` - ### Link `{type?: str, url: str}`. diff --git a/examples/cobi.py b/examples/cobi.py index 2e534e4..571bba8 100644 --- a/examples/cobi.py +++ b/examples/cobi.py @@ -20,6 +20,12 @@ requirements="https://github.com/numbbo/cobi-problem-generator/blob/main/requirements.txt", ) +things["cobi_ref"] = Reference( + title="Pareto Set Characterization in Constrained Multiobjective Optimization and the COBI Problem Generator", + authors=["Anne Auger", "Dimo Brockhoff", "Luka Opravš", "Tea Tušar"], + link={"type": "arxiv", "url": "https://arxiv.org/abs/2604.09131"}, +) + things["cobi_problem"] = Generator( name="COBI Problem", description="Generator of COnstrained BI-objective optimization problems", @@ -32,13 +38,7 @@ "multi-peak", "convex-quadratic", }, - references=[ - Reference( - title="Pareto Set Characterization in Constrained Multiobjective Optimization and the COBI Problem Generator", - authors=["Anne Auger", "Dimo Brockhoff", "Luka Opravš", "Tea Tušar"], - link={"type": "arxiv", "url": "https://arxiv.org/abs/2604.09131"}, - ) - ], + references={"cobi_ref"}, objectives={2}, variables=[Variable(type="continuous", dim={"min": 1})], implementations={"cobi_impl"}, diff --git a/examples/emdo.py b/examples/emdo.py index 7a4504d..4f51f06 100644 --- a/examples/emdo.py +++ b/examples/emdo.py @@ -1,4 +1,4 @@ -from opltools import Library, Problem, Implementation +from opltools import Library, Problem, Implementation, Reference from pydantic_yaml import to_yaml_str #! - name: Electric Motor Design Optimization @@ -39,6 +39,11 @@ language="python", evaluation_time=["8 minutes"] ), + "ref_emdo": Reference( + title="A Multi-Step Evaluation Process in Electric Motor Design", + authors=["Tea Tušar", "Peter Korošec", "Bogdan Filipič"], + link={"url": "https://dis.ijs.si/tea/Publications/Tusar23Multistep.pdf"}, + ), "fn_emdo": Problem( name="Electric Motor Design Optimization", description="""# Goal @@ -67,14 +72,7 @@ noise_type=["yes"], fidelity_levels=[1], source=["real-world"], - references=[ - { - "title": "A Multi-Step Evaluation Process in Electric Motor Design", - "lang": "sj", - "authors": ["Tea Tušar", "Peter Korošec", "Bogdan Filipič"], - "link": {"url": "https://dis.ijs.si/tea/Publications/Tusar23Multistep.pdf"} - } - ], + references={"ref_emdo"}, implementations=["impl_emdo"] ) }) diff --git a/examples/problems.py b/examples/problems.py index 0a2d4ea..7011a89 100644 --- a/examples/problems.py +++ b/examples/problems.py @@ -81,6 +81,271 @@ ) +# ===================================================================== +# Shared references (reused by multiple YAML entries). +# ===================================================================== + +things["ref_coco_a_platform_for"] = Reference( + title="COCO: a platform for comparing continuous optimizers in a black-box setting", + link=Link(url="https://doi.org/10.1080/10556788.2020.1808977"), +) +things["ref_bbob_bi_objective_test_suite"] = Reference( + title="BBOB bi-objective test suite", + link=Link(url="https://doi.org/10.48550/arXiv.1604.00359"), +) +things["ref_real_parameter_black_box"] = Reference( + title="Real-parameter black-box optimization benchmarking: noisy functions definitions", + link=Link(url="https://hal.inria.fr/inria-00369466"), +) +things["ref_bbob_large_scale_test_suite"] = Reference( + title="BBOB large-scale test suite", + link=Link(url="https://doi.org/10.48550/arXiv.1903.06396"), +) +things["ref_bbob_mixed_integer_test_suite"] = Reference( + title="BBOB bi-objective mixed-integer test suite", + link=Link(url="https://doi.org/10.1145/3321707.3321868"), +) +things["ref_bbob_constrained_documentation"] = Reference( + title="bbob-constrained documentation", + link=Link(url="http://numbbo.github.io/coco-doc/bbob-constrained/"), +) +things["ref_comparison_of_multiobjective_evolutionary"] = Reference( + title="Comparison of multiobjective evolutionary algorithms: empirical results", + authors=["Eckart Zitzler", "Kalyanmoy Deb", "Lothar Thiele"], + link=Link(url="https://doi.org/10.1162/106365600568202"), +) +things["ref_scalable_multi_objective_optimization"] = Reference( + title="Scalable multi-objective optimization test problems", + authors=["Kalyanmoy Deb", "Lothar Thiele", "Marco Laumanns", "Eckart Zitzler"], + link=Link(url="https://doi.org/10.1109/CEC.2002.1007032"), +) +things["ref_a_review_of_multiobjective"] = Reference( + title="A review of multiobjective test problems and a scalable test problem toolkit", + authors=["Simon Huband", "Philip Hingston", "Luigi Barone", "Lyndon While"], + link=Link(url="https://doi.org/10.1109/TEVC.2005.861417"), +) +things["ref_cdmp_benchmark"] = Reference( + title="CDMP benchmark", + link=Link(url="https://doi.org/10.1145/3321707.3321878"), +) +things["ref_sdp_dynamic_multi_objective_benchmark"] = Reference( + title="SDP dynamic multi-objective benchmark", + link=Link(url="https://doi.org/10.1109/TCYB.2019.2896021"), +) +things["ref_maop_benchmark"] = Reference( + title="MaOP benchmark", + link=Link(url="https://doi.org/10.1016/j.swevo.2019.02.003"), +) +things["ref_bp_benchmark"] = Reference( + title="BP benchmark", + link=Link(url="https://doi.org/10.1109/CEC.2019.8790277"), +) +things["ref_gpd_generator"] = Reference( + title="GPD generator", + link=Link(url="https://doi.org/10.1016/j.asoc.2020.106139"), +) +things["ref_evolutionary_many_task_optimization"] = Reference( + title="Evolutionary many-task optimization framework", + link=Link(url="https://doi.org/10.48550/arXiv.2110.08033"), +) +things["ref_mmopp_technical_report"] = Reference( + title="MMOPP technical report", + link=Link(url="http://www5.zzu.edu.cn/system/_content/download.jsp?urltype=news.DownloadAttachUrl&owner=1327567121&wbfileid=4764412"), +) +things["ref_cfd_test_problem_suite"] = Reference( + title="CFD test problem suite", + link=Link(url="https://doi.org/10.1007/978-3-319-99259-4_24"), +) +things["ref_game_benchmark_for_evolutionary"] = Reference( + title="Game benchmark for evolutionary algorithms", + link=Link(url="https://doi.org/10.1145/3321707.3321805"), +) +things["ref_car_structure_design_benchmark"] = Reference( + title="Car structure design benchmark", + link=Link(url="https://doi.org/10.1145/3205651.3205702"), +) +things["ref_bbcomp_emo_2017"] = Reference( + title="BBComp EMO 2017", + link=Link(url="https://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/"), +) +things["ref_jpnsec_ec_symposium_2019_competition"] = Reference( + title="JPNSEC EC-Symposium 2019 competition", + link=Link(url="http://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.html"), +) +things["ref_easy_to_evaluate_real"] = Reference( + title="Easy-to-evaluate real-world multi-objective optimization problems", + authors=["Ryoji Tanabe", "Hisao Ishibuchi"], + link=Link(url="https://doi.org/10.1016/j.asoc.2020.106078"), +) +things["ref_radar_waveform_design"] = Reference( + title="Radar waveform design", + link=Link(url="https://doi.org/10.1007/978-3-540-70928-2_53"), +) +things["ref_mf2_a_collection_of"] = Reference( + title="mf2: a collection of multi-fidelity benchmark functions in Python", + link=Link(url="https://doi.org/10.21105/joss.02049"), +) +things["ref_amvop"] = Reference( + title="RWMVOP", + link=Link(url="https://doi.org/10.1109/TEVC.2013.2281531"), +) +things["ref_sbox_cost"] = Reference( + title="SBOX-COST", + link=Link(url="https://doi.org/10.48550/arXiv.2305.12221"), +) +things["ref_on_the_design_of"] = Reference( + title="On the design of multi-objective evolutionary algorithms based on NK-landscapes", + link=Link(url="https://doi.org/10.1016/j.ejor.2012.12.019"), +) +things["ref_mubqp_benchmark"] = Reference( + title="mUBQP benchmark", + link=Link(url="https://doi.org/10.1016/j.asoc.2013.11.008"), +) +things["ref_on_the_impact_of"] = Reference( + title="On the impact of multi-objective scalability for the ρmTSP", + link=Link(url="https://doi.org/10.1007/978-3-319-45823-6_40"), +) +things["ref_benchmark_functions_for_cec"] = Reference( + title="Benchmark Functions for CEC 2015 Special Session and Competition on Dynamic Multi-objective Optimization", +) +things["ref_ealain"] = Reference( + title="Ealain", + link=Link(url="https://doi.org/10.1145/3638530.3654299"), +) +things["ref_ma_bbob"] = Reference( + title="MA-BBOB", + link=Link(url="https://doi.org/10.1145/3673908"), +) +things["ref_mpm2_technical_report_tr15_01"] = Reference( + title="MPM2 technical report TR15-01", + link=Link(url="https://ls11-www.cs.tu-dortmund.de/_media/techreports/tr15-01.pdf"), +) +things["ref_convex_dtlz2"] = Reference( + title="Convex DTLZ2", + link=Link(url="https://doi.org/10.1109/TEVC.2013.2281535"), +) +things["ref_inverted_dtlz1"] = Reference( + title="Inverted DTLZ1", + link=Link(url="https://doi.org/10.1109/TEVC.2013.2281534"), +) +things["ref_minus_dtlz_minus_wfg"] = Reference( + title="Minus DTLZ / Minus WFG", + link=Link(url="https://doi.org/10.1109/TEVC.2016.2587749"), +) +things["ref_linkage_zdt_dtlz_variants"] = Reference( + title="Linkage ZDT/DTLZ variants", + link=Link(url="https://doi.org/10.1145/1143997.1144179"), +) +things["ref_cec2018_dmop_competition_tr"] = Reference( + title="CEC2018 DMOP Competition TR", + link=Link(url="https://www.academia.edu/download/94499025/TR-CEC2018-DMOP-Competition.pdf"), +) +things["ref_modact"] = Reference( + title="MODAct", + link=Link(url="https://doi.org/10.1109/TEVC.2020.3020046"), +) +things["ref_iohclustering"] = Reference( + title="IOHClustering", + link=Link(url="https://arxiv.org/pdf/2505.09233"), +) +things["ref_gnbg_ii"] = Reference( + title="GNBG-II", + link=Link(url="https://dl.acm.org/doi/pdf/10.1145/3712255.3734271"), +) +things["ref_gnbg"] = Reference( + title="GNBG", + link=Link(url="https://arxiv.org/abs/2312.07083"), +) +things["ref_dynamicbinval"] = Reference( + title="DynamicBinVal", + link=Link(url="https://arxiv.org/pdf/2404.15837"), +) +things["ref_pbo_benchmarks"] = Reference( + title="PBO benchmarks", + link=Link(url="https://dl.acm.org/doi/pdf/10.1145/3319619.3326810"), +) +things["ref_w_model"] = Reference( + title="W-model", + link=Link(url="https://dl.acm.org/doi/abs/10.1145/3205651.3208240"), +) +things["ref_submodular_optimization_benchmark"] = Reference( + title="Submodular optimization benchmark", + link=Link(url="https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=10254181"), +) +things["ref_cec2013_definitions"] = Reference( + title="CEC2013 definitions", + link=Link(url="https://peerj.com/articles/cs-2671/CEC2013.pdf"), +) +things["ref_cec2022_tr"] = Reference( + title="CEC2022 TR", + link=Link(url="https://github.com/P-N-Suganthan/2022-SO-BO/blob/main/CEC2022%20TR.pdf"), +) +things["ref_onemax_sphere_zeromax_sphere"] = Reference( + title="Mixed-variable multi-objective test problems", + link=Link(url="https://doi.org/10.1145/3449726.3459521"), +) +things["ref_porkchop_plot_interplanetary_trajectory"] = Reference( + title="Porkchop plot interplanetary trajectory benchmark", + link=Link(url="https://doi.org/10.1109/CEC65147.2025.11042973"), +) +things["ref_kinematics_of_a_robot_arm"] = Reference( + title="Kinematics of a robot arm", + link=Link(url="https://doi.org/10.1023/A:1013258808932"), +) +things["ref_vehicledynamics_benchmark"] = Reference( + title="VehicleDynamics benchmark", + link=Link(url="https://www.scitepress.org/Papers/2023/121580/121580.pdf"), +) +things["ref_mechbench"] = Reference( + title="MECHBench", + link=Link(url="https://arxiv.org/abs/2511.10821"), +) +things["ref_expobench"] = Reference( + title="EXPObench", + link=Link(url="https://doi.org/10.1016/j.asoc.2023.110744"), +) +things["ref_gasoline_direct_injection_engine_design"] = Reference( + title="Gasoline direct injection engine design", + link=Link(url="https://doi.org/10.1016/j.ejor.2022.08.032"), +) +things["ref_beacon"] = Reference( + title="BEACON", + link=Link(url="https://dl.acm.org/doi/10.1145/3712255.3734303"), +) +things["ref_tulipaenergymodel_jl_scientific_references"] = Reference( + title="TulipaEnergyModel.jl scientific references", + link=Link(url="https://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/40-scientific-foundation/45-scientific-references"), +) +things["ref_brachytherapy_treatment_planning"] = Reference( + title="Brachytherapy treatment planning", + link=Link(url="https://www.sciencedirect.com/science/article/pii/S1538472123016781"), +) +things["ref_fleetopt"] = Reference( + title="FleetOpt", + link=Link(url="https://dl.acm.org/doi/abs/10.1145/3638530.3664137"), +) +things["ref_building_spatial_design"] = Reference( + title="Building spatial design", + link=Link(url="https://hdl.handle.net/1887/81789"), +) +things["ref_a_multi_step_evaluation"] = Reference( + title="A Multi-Step Evaluation Process in Electric Motor Design", + authors=["Tea Tušar", "Peter Korošec", "Bogdan Filipič"], + link=Link(url="https://dis.ijs.si/tea/Publications/Tusar23Multistep.pdf"), +) +things["ref_cuter"] = Reference( + title="CUTEr", + link=Link(url="https://dl.acm.org/doi/10.1145/962437.962439"), +) +things["ref_cutest"] = Reference( + title="CUTEst", + link=Link(url="https://link.springer.com/article/10.1007/s10589-014-9687-3"), +) +things["ref_puboi"] = Reference( + title="PUBOi", + link=Link(url="https://link.springer.com/chapter/10.1007/978-3-031-04148-8_12"), +) + # ===================================================================== # Entries # ===================================================================== @@ -104,13 +369,7 @@ objectives={1}, variables=[Variable(type="continuous", dim=ValueRange(min=1))], modality={"multimodal"}, - references=[ - Reference( - title="COCO: a platform for comparing continuous optimizers in a black-box setting", - authors=[], - link=Link(url="https://doi.org/10.1080/10556788.2020.1808977"), - ) - ], + references={"ref_coco_a_platform_for"}, implementations={"impl_coco"}, ) @@ -133,13 +392,7 @@ objectives={2}, variables=[Variable(type="continuous", dim=ValueRange(min=2, max=40))], modality={"multimodal"}, - references=[ - Reference( - title="BBOB bi-objective test suite", - authors=[], - link=Link(url="https://doi.org/10.48550/arXiv.1604.00359"), - ) - ], + references={"ref_bbob_bi_objective_test_suite"}, implementations={"impl_coco"}, ) @@ -163,13 +416,7 @@ variables=[Variable(type="continuous", dim=ValueRange(min=1))], modality={"multimodal"}, noise_type={"noisy"}, - references=[ - Reference( - title="Real-parameter black-box optimization benchmarking: noisy functions definitions", - authors=[], - link=Link(url="https://hal.inria.fr/inria-00369466"), - ) - ], + references={"ref_real_parameter_black_box"}, implementations={"impl_coco_legacy"}, ) @@ -192,13 +439,7 @@ objectives={1}, variables=[Variable(type="continuous", dim=ValueRange(min=20, max=640))], modality={"multimodal"}, - references=[ - Reference( - title="BBOB large-scale test suite", - authors=[], - link=Link(url="https://doi.org/10.48550/arXiv.1903.06396"), - ) - ], + references={"ref_bbob_large_scale_test_suite"}, implementations={"impl_coco"}, ) @@ -224,13 +465,7 @@ Variable(type="integer", dim=ValueRange(min=5, max=160)), ], modality={"multimodal"}, - references=[ - Reference( - title="BBOB mixed-integer test suite", - authors=[], - link=Link(url="https://doi.org/10.1145/3321707.3321868"), - ) - ], + references={"ref_bbob_mixed_integer_test_suite"}, implementations={"impl_coco"}, ) @@ -256,13 +491,7 @@ Variable(type="integer", dim=ValueRange(min=5, max=160)), ], modality={"multimodal"}, - references=[ - Reference( - title="BBOB bi-objective mixed-integer test suite", - authors=[], - link=Link(url="https://doi.org/10.1145/3321707.3321868"), - ) - ], + references={"ref_bbob_mixed_integer_test_suite"}, implementations={"impl_coco"}, ) @@ -286,13 +515,7 @@ variables=[Variable(type="continuous", dim=ValueRange(min=2, max=40))], constraints=[Constraint(hard="yes")], modality={"multimodal"}, - references=[ - Reference( - title="bbob-constrained documentation", - authors=[], - link=Link(url="http://numbbo.github.io/coco-doc/bbob-constrained/"), - ) - ], + references={"ref_bbob_constrained_documentation"}, implementations={"impl_coco"}, ) @@ -347,13 +570,7 @@ Variable(type="continuous", dim=ValueRange(min=1)), Variable(type="binary", dim=ValueRange(min=1)), ], - references=[ - Reference( - title="Comparison of multiobjective evolutionary algorithms: empirical results", - authors=["Eckart Zitzler", "Kalyanmoy Deb", "Lothar Thiele"], - link=Link(url="https://doi.org/10.1162/106365600568202"), - ) - ], + references={"ref_comparison_of_multiobjective_evolutionary"}, implementations={"impl_pymoo"}, ) @@ -376,13 +593,7 @@ # FIXME: original "2+" - schema requires set[int]; truncated to 2..10. objectives=set(range(2, 11)), variables=[Variable(type="continuous", dim=ValueRange(min=1))], - references=[ - Reference( - title="Scalable multi-objective optimization test problems", - authors=["Kalyanmoy Deb", "Lothar Thiele", "Marco Laumanns", "Eckart Zitzler"], - link=Link(url="https://doi.org/10.1109/CEC.2002.1007032"), - ) - ], + references={"ref_scalable_multi_objective_optimization"}, implementations={"impl_pymoo"}, ) @@ -405,13 +616,7 @@ # FIXME: original "2+" - truncated to 2..10. objectives=set(range(2, 11)), variables=[Variable(type="continuous", dim=ValueRange(min=1))], - references=[ - Reference( - title="A review of multiobjective test problems and a scalable test problem toolkit", - authors=["Simon Huband", "Philip Hingston", "Luigi Barone", "Lyndon While"], - link=Link(url="https://doi.org/10.1109/TEVC.2005.861417"), - ) - ], + references={"ref_a_review_of_multiobjective"}, implementations={"impl_pymoo"}, ) @@ -438,13 +643,7 @@ constraints=[Constraint(hard="yes")], dynamic_type={"unknown"}, noise_type={"unknown"}, - references=[ - Reference( - title="CDMP benchmark", - authors=[], - link=Link(url="https://doi.org/10.1145/3321707.3321878"), - ) - ], + references={"ref_cdmp_benchmark"}, ) #! - name: SDP @@ -469,13 +668,7 @@ variables=[Variable(type="continuous", dim=ValueRange(min=1))], dynamic_type={"dynamic"}, noise_type={"unknown"}, - references=[ - Reference( - title="SDP dynamic multi-objective benchmark", - authors=[], - link=Link(url="https://doi.org/10.1109/TCYB.2019.2896021"), - ) - ], + references={"ref_sdp_dynamic_multi_objective_benchmark"}, ) #! - name: MaOP @@ -499,13 +692,7 @@ objectives=set(range(2, 11)), variables=[Variable(type="continuous", dim=ValueRange(min=1))], noise_type={"unknown"}, - references=[ - Reference( - title="MaOP benchmark", - authors=[], - link=Link(url="https://doi.org/10.1016/j.swevo.2019.02.003"), - ) - ], + references={"ref_maop_benchmark"}, ) #! - name: BP @@ -529,13 +716,7 @@ objectives=set(range(2, 11)), variables=[Variable(type="continuous", dim=ValueRange(min=1))], noise_type={"unknown"}, - references=[ - Reference( - title="BP benchmark", - authors=[], - link=Link(url="https://doi.org/10.1109/CEC.2019.8790277"), - ) - ], + references={"ref_bp_benchmark"}, ) #! - name: GPD @@ -560,13 +741,7 @@ variables=[Variable(type="continuous", dim=ValueRange(min=1))], constraints=[Constraint(hard="some")], noise_type={"optional"}, - references=[ - Reference( - title="GPD generator", - authors=[], - link=Link(url="https://doi.org/10.1016/j.asoc.2020.106139"), - ) - ], + references={"ref_gpd_generator"}, ) #! - name: ETMOF @@ -593,13 +768,7 @@ objectives=set(range(2, 51)), variables=[Variable(type="continuous", dim=ValueRange(min=25, max=10000))], dynamic_type={"dynamic"}, - references=[ - Reference( - title="Evolutionary many-task optimization framework", - authors=[], - link=Link(url="https://doi.org/10.48550/arXiv.2110.08033"), - ) - ], + references={"ref_evolutionary_many_task_optimization"}, implementations={"impl_etmof"}, ) @@ -629,15 +798,7 @@ variables=[Variable(type="unknown")], constraints=[Constraint(hard="yes")], modality={"multimodal"}, - references=[ - Reference( - title="MMOPP technical report", - authors=[], - link=Link( - url="http://www5.zzu.edu.cn/system/_content/download.jsp?urltype=news.DownloadAttachUrl&owner=1327567121&wbfileid=4764412" - ), - ) - ], + references={"ref_mmopp_technical_report"}, implementations={"impl_mmopp"}, ) @@ -669,13 +830,7 @@ variables=[Variable(type="unknown", dim=ValueRange(min=1))], constraints=[Constraint(hard="yes")], source={"real-world"}, - references=[ - Reference( - title="CFD test problem suite", - authors=[], - link=Link(url="https://doi.org/10.1007/978-3-319-99259-4_24"), - ) - ], + references={"ref_cfd_test_problem_suite"}, implementations={"impl_cfd"}, ) @@ -707,13 +862,7 @@ noise_type={"noisy"}, modality={"multimodal"}, source={"real-world"}, - references=[ - Reference( - title="Game benchmark for evolutionary algorithms", - authors=[], - link=Link(url="https://doi.org/10.1145/3321707.3321805"), - ) - ], + references={"ref_game_benchmark_for_evolutionary"}, implementations={"impl_gbea"}, ) @@ -744,13 +893,7 @@ variables=[Variable(type="integer", dim=ValueRange(min=144, max=222))], constraints=[Constraint(hard="yes", number=54)], source={"real-world"}, - references=[ - Reference( - title="Car structure design benchmark", - authors=[], - link=Link(url="https://doi.org/10.1145/3205651.3205702"), - ) - ], + references={"ref_car_structure_design_benchmark"}, implementations={"impl_car_structure"}, ) @@ -783,13 +926,7 @@ objectives={2}, variables=[Variable(type="continuous", dim=ValueRange(min=4, max=24))], source={"real-world"}, - references=[ - Reference( - title="BBComp EMO 2017", - authors=[], - link=Link(url="https://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/"), - ) - ], + references={"ref_bbcomp_emo_2017"}, implementations={"impl_emo2017"}, ) @@ -825,15 +962,7 @@ variables=[Variable(type="continuous", dim=32)], constraints=[Constraint(hard="yes", number=22)], source={"real-world"}, - references=[ - Reference( - title="JPNSEC EC-Symposium 2019 competition", - authors=[], - link=Link( - url="http://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.html" - ), - ) - ], + references={"ref_jpnsec_ec_symposium_2019_competition"}, implementations={"impl_jsec2019"}, ) @@ -859,13 +988,7 @@ Variable(type="integer", dim=ValueRange(min=2, max=7)), ], source={"real-world-like"}, - references=[ - Reference( - title="Easy-to-evaluate real-world multi-objective optimization problems", - authors=["Ryoji Tanabe", "Hisao Ishibuchi"], - link=Link(url="https://doi.org/10.1016/j.asoc.2020.106078"), - ) - ], + references={"ref_easy_to_evaluate_real"}, implementations={"impl_reproblems"}, ) @@ -892,13 +1015,7 @@ ], constraints=[Constraint(hard="yes")], source={"real-world-like"}, - references=[ - Reference( - title="Easy-to-evaluate real-world multi-objective optimization problems", - authors=["Ryoji Tanabe", "Hisao Ishibuchi"], - link=Link(url="https://doi.org/10.1016/j.asoc.2020.106078"), - ) - ], + references={"ref_easy_to_evaluate_real"}, implementations={"impl_reproblems"}, ) @@ -927,13 +1044,7 @@ variables=[Variable(type="integer", dim=ValueRange(min=4, max=12))], constraints=[Constraint(hard="yes")], source={"real-world"}, - references=[ - Reference( - title="Radar waveform design", - authors=[], - link=Link(url="https://doi.org/10.1007/978-3-540-70928-2_53"), - ) - ], + references={"ref_radar_waveform_design"}, implementations={"impl_radar_waveform"}, ) @@ -962,13 +1073,7 @@ objectives={1}, variables=[Variable(type="continuous", dim=ValueRange(min=1))], fidelity_levels={1, 2}, - references=[ - Reference( - title="mf2: a collection of multi-fidelity benchmark functions in Python", - authors=[], - link=Link(url="https://doi.org/10.21105/joss.02049"), - ) - ], + references={"ref_mf2_a_collection_of"}, implementations={"impl_mf2"}, ) @@ -996,13 +1101,7 @@ Variable(type="categorical", dim=ValueRange(min=1)), ], modality={"multimodal"}, - references=[ - Reference( - title="AMVOP", - authors=[], - link=Link(url="https://doi.org/10.1109/TEVC.2013.2281531"), - ) - ], + references={"ref_amvop"}, ) #! - name: RWMVOP @@ -1030,13 +1129,7 @@ ], constraints=[Constraint(hard="yes")], source={"real-world"}, - references=[ - Reference( - title="RWMVOP", - authors=[], - link=Link(url="https://doi.org/10.1109/TEVC.2013.2281531"), - ) - ], + references={"ref_amvop"}, ) #! - name: SBOX-COST @@ -1060,13 +1153,7 @@ objectives={1}, variables=[Variable(type="continuous", dim=ValueRange(min=1))], modality={"multimodal"}, - references=[ - Reference( - title="SBOX-COST", - authors=[], - link=Link(url="https://doi.org/10.48550/arXiv.2305.12221"), - ) - ], + references={"ref_sbox_cost"}, implementations={"impl_iohexperimenter"}, ) @@ -1092,13 +1179,7 @@ objectives=set(range(1, 11)), variables=[Variable(type="binary", dim=ValueRange(min=1))], modality={"multimodal"}, - references=[ - Reference( - title="On the design of multi-objective evolutionary algorithms based on NK-landscapes", - authors=[], - link=Link(url="https://doi.org/10.1016/j.ejor.2012.12.019"), - ) - ], + references={"ref_on_the_design_of"}, implementations={"impl_mocobench"}, ) @@ -1124,13 +1205,7 @@ objectives=set(range(1, 11)), variables=[Variable(type="binary", dim=ValueRange(min=1))], modality={"multimodal", "quadratic"}, - references=[ - Reference( - title="mUBQP benchmark", - authors=[], - link=Link(url="https://doi.org/10.1016/j.asoc.2013.11.008"), - ) - ], + references={"ref_mubqp_benchmark"}, implementations={"impl_mocobench"}, ) @@ -1157,13 +1232,7 @@ objectives=set(range(1, 11)), variables=[Variable(type="unknown", dim=ValueRange(min=1))], modality={"multimodal", "quadratic"}, - references=[ - Reference( - title="On the impact of multi-objective scalability for the ρmTSP", - authors=[], - link=Link(url="https://doi.org/10.1007/978-3-319-45823-6_40"), - ) - ], + references={"ref_on_the_impact_of"}, implementations={"impl_mocobench"}, ) @@ -1189,12 +1258,7 @@ variables=[Variable(type="continuous")], constraints=[Constraint(hard="?")], dynamic_type={"dynamic"}, - references=[ - Reference( - title="Benchmark Functions for CEC 2015 Special Session and Competition on Dynamic Multi-objective Optimization", - authors=[], - ) - ], + references={"ref_benchmark_functions_for_cec"}, ) #! - name: Ealain @@ -1230,13 +1294,7 @@ dynamic_type={"optional"}, fidelity_levels={1, 2}, source={"real-world-like"}, - references=[ - Reference( - title="Ealain", - authors=[], - link=Link(url="https://doi.org/10.1145/3638530.3654299"), - ) - ], + references={"ref_ealain"}, implementations={"impl_ealain"}, ) @@ -1271,13 +1329,7 @@ variables=[Variable(type="continuous", dim=ValueRange(min=1))], modality={"multimodal"}, source={"artificial"}, - references=[ - Reference( - title="MA-BBOB", - authors=[], - link=Link(url="https://doi.org/10.1145/3673908"), - ) - ], + references={"ref_ma_bbob"}, implementations={"impl_ma_bbob", "impl_iohexperimenter"}, ) @@ -1313,13 +1365,7 @@ objectives={1}, variables=[Variable(type="continuous", dim=ValueRange(min=1))], modality={"multimodal"}, - references=[ - Reference( - title="MPM2 technical report TR15-01", - authors=[], - link=Link(url="https://ls11-www.cs.tu-dortmund.de/_media/techreports/tr15-01.pdf"), - ) - ], + references={"ref_mpm2_technical_report_tr15_01"}, implementations={"impl_mpm2"}, ) @@ -1344,13 +1390,7 @@ # FIXME: original "2+" - truncated to 2..10. objectives=set(range(2, 11)), variables=[Variable(type="continuous", dim=ValueRange(min=1))], - references=[ - Reference( - title="Convex DTLZ2", - authors=[], - link=Link(url="https://doi.org/10.1109/TEVC.2013.2281535"), - ) - ], + references={"ref_convex_dtlz2"}, ) #! - name: Inverted DTLZ1 @@ -1374,13 +1414,7 @@ # FIXME: original "2+" - truncated to 2..10. objectives=set(range(2, 11)), variables=[Variable(type="continuous", dim=ValueRange(min=1))], - references=[ - Reference( - title="Inverted DTLZ1", - authors=[], - link=Link(url="https://doi.org/10.1109/TEVC.2013.2281534"), - ) - ], + references={"ref_inverted_dtlz1"}, ) #! - name: Minus DTLZ @@ -1404,13 +1438,7 @@ # FIXME: original "2+" - truncated to 2..10. objectives=set(range(2, 11)), variables=[Variable(type="continuous", dim=ValueRange(min=1))], - references=[ - Reference( - title="Minus DTLZ / Minus WFG", - authors=[], - link=Link(url="https://doi.org/10.1109/TEVC.2016.2587749"), - ) - ], + references={"ref_minus_dtlz_minus_wfg"}, ) #! - name: Minus WFG @@ -1434,13 +1462,7 @@ # FIXME: original "2+" - truncated to 2..10. objectives=set(range(2, 11)), variables=[Variable(type="continuous", dim=ValueRange(min=1))], - references=[ - Reference( - title="Minus DTLZ / Minus WFG", - authors=[], - link=Link(url="https://doi.org/10.1109/TEVC.2016.2587749"), - ) - ], + references={"ref_minus_dtlz_minus_wfg"}, ) #! - name: L1-ZDT @@ -1468,13 +1490,7 @@ Variable(type="continuous", dim=ValueRange(min=1)), Variable(type="binary", dim=ValueRange(min=1)), ], - references=[ - Reference( - title="Linkage ZDT/DTLZ variants", - authors=[], - link=Link(url="https://doi.org/10.1145/1143997.1144179"), - ) - ], + references={"ref_linkage_zdt_dtlz_variants"}, ) #! - name: L2-ZDT @@ -1501,13 +1517,7 @@ Variable(type="continuous", dim=ValueRange(min=1)), Variable(type="binary", dim=ValueRange(min=1)), ], - references=[ - Reference( - title="Linkage ZDT/DTLZ variants", - authors=[], - link=Link(url="https://doi.org/10.1145/1143997.1144179"), - ) - ], + references={"ref_linkage_zdt_dtlz_variants"}, ) #! - name: L3-ZDT @@ -1534,13 +1544,7 @@ Variable(type="continuous", dim=ValueRange(min=1)), Variable(type="binary", dim=ValueRange(min=1)), ], - references=[ - Reference( - title="Linkage ZDT/DTLZ variants", - authors=[], - link=Link(url="https://doi.org/10.1145/1143997.1144179"), - ) - ], + references={"ref_linkage_zdt_dtlz_variants"}, ) #! - name: L2-DTLZ @@ -1565,13 +1569,7 @@ # FIXME: original "2+" - truncated to 2..10. objectives=set(range(2, 11)), variables=[Variable(type="continuous", dim=ValueRange(min=1))], - references=[ - Reference( - title="Linkage ZDT/DTLZ variants", - authors=[], - link=Link(url="https://doi.org/10.1145/1143997.1144179"), - ) - ], + references={"ref_linkage_zdt_dtlz_variants"}, ) #! - name: L3-DTLZ @@ -1596,13 +1594,7 @@ # FIXME: original "2+" - truncated to 2..10. objectives=set(range(2, 11)), variables=[Variable(type="continuous", dim=ValueRange(min=1))], - references=[ - Reference( - title="Linkage ZDT/DTLZ variants", - authors=[], - link=Link(url="https://doi.org/10.1145/1143997.1144179"), - ) - ], + references={"ref_linkage_zdt_dtlz_variants"}, ) #! - name: CEC2018 DT - CEC2018 Competition on Dynamic Multiobjective Optimisation @@ -1630,13 +1622,7 @@ variables=[Variable(type="unknown", dim=ValueRange(min=1))], dynamic_type={"dynamic"}, source={"artificial"}, - references=[ - Reference( - title="CEC2018 DMOP Competition TR", - authors=[], - link=Link(url="https://www.academia.edu/download/94499025/TR-CEC2018-DMOP-Competition.pdf"), - ) - ], + references={"ref_cec2018_dmop_competition_tr"}, implementations={"impl_pymoo"}, ) @@ -1673,13 +1659,7 @@ ], constraints=[Constraint(hard="yes")], source={"real-world"}, - references=[ - Reference( - title="MODAct", - authors=[], - link=Link(url="https://doi.org/10.1109/TEVC.2020.3020046"), - ) - ], + references={"ref_modact"}, implementations={"impl_modact", "impl_pymoo"}, ) @@ -1711,13 +1691,7 @@ variables=[Variable(type="continuous", dim=ValueRange(min=1))], modality={"multimodal"}, source={"artificial-from-real-data"}, - references=[ - Reference( - title="IOHClustering", - authors=[], - link=Link(url="https://arxiv.org/pdf/2505.09233"), - ) - ], + references={"ref_iohclustering"}, implementations={"impl_iohclustering"}, ) things["gen_iohclustering"] = Generator( @@ -1727,13 +1701,7 @@ variables=[Variable(type="continuous", dim=ValueRange(min=1))], modality={"multimodal"}, source={"artificial-from-real-data"}, - references=[ - Reference( - title="IOHClustering", - authors=[], - link=Link(url="https://arxiv.org/pdf/2505.09233"), - ) - ], + references={"ref_iohclustering"}, implementations={"impl_iohclustering"}, ) @@ -1767,13 +1735,7 @@ objectives={1}, variables=[Variable(type="continuous", dim=ValueRange(min=1))], source={"artificial"}, - references=[ - Reference( - title="GNBG-II", - authors=[], - link=Link(url="https://dl.acm.org/doi/pdf/10.1145/3712255.3734271"), - ) - ], + references={"ref_gnbg_ii"}, implementations={"impl_gnbg_ii", "impl_iohgnbg"}, ) things["gen_gnbg_ii"] = Generator( @@ -1782,13 +1744,7 @@ objectives={1}, variables=[Variable(type="continuous", dim=ValueRange(min=1))], source={"artificial"}, - references=[ - Reference( - title="GNBG-II", - authors=[], - link=Link(url="https://dl.acm.org/doi/pdf/10.1145/3712255.3734271"), - ) - ], + references={"ref_gnbg_ii"}, implementations={"impl_gnbg_ii", "impl_iohgnbg"}, ) @@ -1817,13 +1773,7 @@ objectives={1}, variables=[Variable(type="continuous", dim=ValueRange(min=1))], source={"artificial"}, - references=[ - Reference( - title="GNBG", - authors=[], - link=Link(url="https://arxiv.org/abs/2312.07083"), - ) - ], + references={"ref_gnbg"}, implementations={"impl_gnbg"}, ) things["gen_gnbg"] = Generator( @@ -1832,13 +1782,7 @@ objectives={1}, variables=[Variable(type="continuous", dim=ValueRange(min=1))], source={"artificial"}, - references=[ - Reference( - title="GNBG", - authors=[], - link=Link(url="https://arxiv.org/abs/2312.07083"), - ) - ], + references={"ref_gnbg"}, implementations={"impl_gnbg"}, ) @@ -1863,13 +1807,7 @@ variables=[Variable(type="binary", dim=ValueRange(min=1))], dynamic_type={"dynamic"}, source={"artificial"}, - references=[ - Reference( - title="DynamicBinVal", - authors=[], - link=Link(url="https://arxiv.org/pdf/2404.15837"), - ) - ], + references={"ref_dynamicbinval"}, implementations={"impl_iohexperimenter"}, ) @@ -1893,13 +1831,7 @@ objectives={1}, variables=[Variable(type="binary", dim=ValueRange(min=1))], source={"artificial"}, - references=[ - Reference( - title="PBO benchmarks", - authors=[], - link=Link(url="https://dl.acm.org/doi/pdf/10.1145/3319619.3326810"), - ) - ], + references={"ref_pbo_benchmarks"}, implementations={"impl_iohexperimenter"}, ) @@ -1929,15 +1861,7 @@ objectives={1}, variables=[Variable(type="binary", dim=ValueRange(min=1))], source={"artificial"}, - references=[ - Reference( - title="W-model", - authors=[], - link=Link( - url="https://dl.acm.org/doi/abs/10.1145/3205651.3208240" - ), - ) - ], + references={"ref_w_model"}, implementations={"impl_wmodel"}, ) @@ -1962,13 +1886,7 @@ objectives={1}, variables=[Variable(type="binary", dim=ValueRange(min=1))], source={"artificial"}, - references=[ - Reference( - title="Submodular optimization benchmark", - authors=[], - link=Link(url="https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=10254181"), - ) - ], + references={"ref_submodular_optimization_benchmark"}, implementations={"impl_iohexperimenter"}, ) @@ -1997,13 +1915,7 @@ objectives={1}, variables=[Variable(type="continuous", dim=ValueRange(min=1))], source={"artificial"}, - references=[ - Reference( - title="CEC2013 definitions", - authors=[], - link=Link(url="https://peerj.com/articles/cs-2671/CEC2013.pdf"), - ) - ], + references={"ref_cec2013_definitions"}, implementations={"impl_cec2013", "impl_iohexperimenter"}, ) @@ -2032,13 +1944,7 @@ objectives={1}, variables=[Variable(type="continuous", dim=ValueRange(min=1))], source={"artificial"}, - references=[ - Reference( - title="CEC2022 TR", - authors=[], - link=Link(url="https://github.com/P-N-Suganthan/2022-SO-BO/blob/main/CEC2022%20TR.pdf"), - ) - ], + references={"ref_cec2022_tr"}, implementations={"impl_cec2022", "impl_iohexperimenter"}, ) @@ -2065,13 +1971,7 @@ Variable(type="continuous", dim=ValueRange(min=1)), ], source={"artificial"}, - references=[ - Reference( - title="Onemax+Sphere / Zeromax+Sphere", - authors=[], - link=Link(url="https://doi.org/10.1145/3449726.3459521"), - ) - ], + references={"ref_onemax_sphere_zeromax_sphere"}, ) #! - name: Onemax+Sphere / DeceptiveTrap+RotatedEllipsoid @@ -2097,13 +1997,7 @@ Variable(type="continuous", dim=ValueRange(min=1)), ], source={"artificial"}, - references=[ - Reference( - title="Mixed-variable multi-objective test problems", - authors=[], - link=Link(url="https://doi.org/10.1145/3449726.3459521"), - ) - ], + references={"ref_onemax_sphere_zeromax_sphere"}, ) #! - name: InverseDeceptiveTrap+RotatedEllipsoid / DeceptiveTrap+RotatedEllipsoid @@ -2129,13 +2023,7 @@ Variable(type="continuous", dim=ValueRange(min=1)), ], source={"artificial"}, - references=[ - Reference( - title="Mixed-variable multi-objective test problems", - authors=[], - link=Link(url="https://doi.org/10.1145/3449726.3459521"), - ) - ], + references={"ref_onemax_sphere_zeromax_sphere"}, ) #! - name: PorkchopPlotInterplanetaryTrajectory @@ -2168,13 +2056,7 @@ variables=[Variable(type="continuous", dim=2)], modality={"multimodal"}, source={"real-world"}, - references=[ - Reference( - title="Porkchop plot interplanetary trajectory benchmark", - authors=[], - link=Link(url="https://doi.org/10.1109/CEC65147.2025.11042973"), - ) - ], + references={"ref_porkchop_plot_interplanetary_trajectory"}, implementations={"impl_transfer_rf_bbob_rw"}, ) @@ -2198,13 +2080,7 @@ variables=[Variable(type="continuous", dim=21)], modality={"unimodal"}, source={"real-world"}, - references=[ - Reference( - title="Kinematics of a robot arm", - authors=[], - link=Link(url="https://doi.org/10.1023/A:1013258808932"), - ) - ], + references={"ref_kinematics_of_a_robot_arm"}, implementations={"impl_transfer_rf_bbob_rw"}, ) @@ -2233,13 +2109,7 @@ variables=[Variable(type="continuous", dim=2)], modality={"multimodal"}, source={"real-world"}, - references=[ - Reference( - title="VehicleDynamics benchmark", - authors=[], - link=Link(url="https://www.scitepress.org/Papers/2023/121580/121580.pdf"), - ) - ], + references={"ref_vehicledynamics_benchmark"}, implementations={"impl_vehicle_dynamics"}, ) @@ -2290,13 +2160,7 @@ modality={"multimodal"}, allows_partial_evaluation="no", source={"real-world"}, - references=[ - Reference( - title="MECHBench", - authors=[], - link=Link(url="https://arxiv.org/abs/2511.10821"), - ) - ], + references={"ref_mechbench"}, implementations={"impl_mechbench"}, ) @@ -2355,13 +2219,7 @@ noise_type={"observational", "real-life"}, allows_partial_evaluation="no", source={"real-world"}, - references=[ - Reference( - title="EXPObench", - authors=[], - link=Link(url="https://doi.org/10.1016/j.asoc.2023.110744"), - ) - ], + references={"ref_expobench"}, implementations={"impl_expobench"}, ) @@ -2403,13 +2261,7 @@ constraints=[Constraint(hard="yes", number=5)], fidelity_levels={1, 2}, source={"real-world"}, - references=[ - Reference( - title="Gasoline direct injection engine design", - authors=[], - link=Link(url="https://doi.org/10.1016/j.ejor.2022.08.032"), - ) - ], + references={"ref_gasoline_direct_injection_engine_design"}, implementations={"impl_gasoline"}, ) @@ -2457,13 +2309,7 @@ modality={"multimodal"}, allows_partial_evaluation="no", source={"artificial"}, - references=[ - Reference( - title="BEACON", - authors=[], - link=Link(url="https://dl.acm.org/doi/10.1145/3712255.3734303"), - ) - ], + references={"ref_beacon"}, implementations={"impl_beacon"}, ) @@ -2518,15 +2364,7 @@ modality={"unimodal"}, fidelity_levels={1, 2}, source={"real-world"}, - references=[ - Reference( - title="TulipaEnergyModel.jl scientific references", - authors=[], - link=Link( - url="https://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/40-scientific-foundation/45-scientific-references" - ), - ) - ], + references={"ref_tulipaenergymodel_jl_scientific_references"}, implementations={"impl_tulipa"}, ) @@ -2590,13 +2428,7 @@ fidelity_levels={1, 2}, allows_partial_evaluation="yes", source={"real-world"}, - references=[ - Reference( - title="Brachytherapy treatment planning", - authors=[], - link=Link(url="https://www.sciencedirect.com/science/article/pii/S1538472123016781"), - ) - ], + references={"ref_brachytherapy_treatment_planning"}, ) #! - name: FleetOpt @@ -2624,13 +2456,7 @@ constraints=[Constraint(hard="yes")], allows_partial_evaluation="yes", source={"real-world"}, - references=[ - Reference( - title="FleetOpt", - authors=[], - link=Link(url="https://dl.acm.org/doi/abs/10.1145/3638530.3664137"), - ) - ], + references={"ref_fleetopt"}, ) #! - name: Building spatial design @@ -2677,13 +2503,7 @@ ], allows_partial_evaluation="no", source={"real-world"}, - references=[ - Reference( - title="Building spatial design", - authors=[], - link=Link(url="https://hdl.handle.net/1887/81789"), - ) - ], + references={"ref_building_spatial_design"}, implementations={"impl_bso_toolbox"}, ) @@ -2750,13 +2570,7 @@ modality={"multimodal"}, allows_partial_evaluation="no", source={"real-world"}, - references=[ - Reference( - title="A Multi-Step Evaluation Process in Electric Motor Design", - authors=["Tea Tušar", "Peter Korošec", "Bogdan Filipič"], - link=Link(url="https://dis.ijs.si/tea/Publications/Tusar23Multistep.pdf"), - ) - ], + references={"ref_a_multi_step_evaluation"}, implementations={"impl_emdo"}, ) @@ -2876,13 +2690,7 @@ constraints=[Constraint(hard="yes")], allows_partial_evaluation="no", source={"artificial"}, - references=[ - Reference( - title="CUTEr", - authors=[], - link=Link(url="https://dl.acm.org/doi/10.1145/962437.962439"), - ) - ], + references={"ref_cuter"}, ) #! - name: CUTEst @@ -2930,13 +2738,7 @@ modality={"multimodal"}, allows_partial_evaluation="no", source={"artificial"}, - references=[ - Reference( - title="CUTEst", - authors=[], - link=Link(url="https://link.springer.com/article/10.1007/s10589-014-9687-3"), - ) - ], + references={"ref_cutest"}, implementations={"impl_pycutest"}, ) @@ -2975,13 +2777,7 @@ variables=[Variable(type="binary", dim=ValueRange(min=1))], allows_partial_evaluation="no", source={"artificial"}, - references=[ - Reference( - title="PUBOi", - authors=[], - link=Link(url="https://link.springer.com/chapter/10.1007/978-3-031-04148-8_12"), - ) - ], + references={"ref_puboi"}, implementations={"impl_puboi"}, ) diff --git a/problems.yaml b/problems.yaml index ea3b7af..c348956 100644 --- a/problems.yaml +++ b/problems.yaml @@ -44,8 +44,8 @@ fn_building_spatial: evaluations. dynamic_type: null evaluation_time: - - 1 second - 40 seconds + - 1 second fidelity_levels: null implementations: - impl_bso_toolbox @@ -57,11 +57,7 @@ fn_building_spatial: objectives: - 2 references: - - authors: [] - link: - type: null - url: https://hdl.handle.net/1887/81789 - title: Building spatial design + - ref_building_spatial_design source: - real-world tags: null @@ -70,11 +66,11 @@ fn_building_spatial: - dim: max: null min: 1 - type: binary + type: continuous - dim: max: null min: 1 - type: continuous + type: binary fn_convex_dtlz2: allows_partial_evaluation: null can_evaluate_objectives_independently: null @@ -101,11 +97,7 @@ fn_convex_dtlz2: - 9 - 10 references: - - authors: [] - link: - type: null - url: https://doi.org/10.1109/TEVC.2013.2281535 - title: Convex DTLZ2 + - ref_convex_dtlz2 source: null tags: null type: problem @@ -153,23 +145,16 @@ fn_emdo: objectives: - 1 references: - - authors: - - Tea Tušar - - Peter Korošec - - Bogdan Filipič - link: - type: null - url: https://dis.ijs.si/tea/Publications/Tusar23Multistep.pdf - title: A Multi-Step Evaluation Process in Electric Motor Design + - ref_a_multi_step_evaluation source: - real-world tags: null type: problem variables: - - dim: 13 - type: continuous - dim: 13 type: integer + - dim: 13 + type: continuous fn_fleetopt: allows_partial_evaluation: yes can_evaluate_objectives_independently: null @@ -194,11 +179,7 @@ fn_fleetopt: objectives: - 1 references: - - authors: [] - link: - type: null - url: https://dl.acm.org/doi/abs/10.1145/3638530.3664137 - title: FleetOpt + - ref_fleetopt source: - real-world tags: null @@ -237,20 +218,16 @@ fn_gasoline: objectives: - 2 references: - - authors: [] - link: - type: null - url: https://doi.org/10.1016/j.ejor.2022.08.032 - title: Gasoline direct injection engine design + - ref_gasoline_direct_injection_engine_design source: - real-world tags: null type: problem variables: - - dim: 7 - type: integer - dim: 7 type: continuous + - dim: 7 + type: integer fn_invdeceptive_deceptive_rotell: allows_partial_evaluation: null can_evaluate_objectives_independently: null @@ -269,11 +246,7 @@ fn_invdeceptive_deceptive_rotell: objectives: - 2 references: - - authors: [] - link: - type: null - url: https://doi.org/10.1145/3449726.3459521 - title: Mixed-variable multi-objective test problems + - ref_onemax_sphere_zeromax_sphere source: - artificial tags: null @@ -282,11 +255,11 @@ fn_invdeceptive_deceptive_rotell: - dim: max: null min: 1 - type: binary + type: continuous - dim: max: null min: 1 - type: continuous + type: binary fn_inverted_dtlz1: allows_partial_evaluation: null can_evaluate_objectives_independently: null @@ -313,11 +286,7 @@ fn_inverted_dtlz1: - 9 - 10 references: - - authors: [] - link: - type: null - url: https://doi.org/10.1109/TEVC.2013.2281534 - title: Inverted DTLZ1 + - ref_inverted_dtlz1 source: null tags: null type: problem @@ -354,12 +323,7 @@ fn_jsec2019: - 4 - 5 references: - - authors: [] - link: - type: null - url: - http://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.html - title: JPNSEC EC-Symposium 2019 competition + - ref_jpnsec_ec_symposium_2019_competition source: - real-world tags: null @@ -385,11 +349,7 @@ fn_onemax_sphere_deceptive_rotell: objectives: - 2 references: - - authors: [] - link: - type: null - url: https://doi.org/10.1145/3449726.3459521 - title: Mixed-variable multi-objective test problems + - ref_onemax_sphere_zeromax_sphere source: - artificial tags: null @@ -398,11 +358,11 @@ fn_onemax_sphere_deceptive_rotell: - dim: max: null min: 1 - type: binary + type: continuous - dim: max: null min: 1 - type: continuous + type: binary fn_onemax_sphere_zeromax_sphere: allows_partial_evaluation: null can_evaluate_objectives_independently: null @@ -421,11 +381,7 @@ fn_onemax_sphere_zeromax_sphere: objectives: - 2 references: - - authors: [] - link: - type: null - url: https://doi.org/10.1145/3449726.3459521 - title: Onemax+Sphere / Zeromax+Sphere + - ref_onemax_sphere_zeromax_sphere source: - artificial tags: null @@ -434,11 +390,11 @@ fn_onemax_sphere_zeromax_sphere: - dim: max: null min: 1 - type: binary + type: continuous - dim: max: null min: 1 - type: continuous + type: binary fn_radar_waveform: allows_partial_evaluation: null can_evaluate_objectives_independently: null @@ -462,11 +418,7 @@ fn_radar_waveform: objectives: - 9 references: - - authors: [] - link: - type: null - url: https://doi.org/10.1007/978-3-540-70928-2_53 - title: Radar waveform design + - ref_radar_waveform_design source: - real-world tags: null @@ -502,11 +454,7 @@ gen_beacon: objectives: - 2 references: - - authors: [] - link: - type: null - url: https://dl.acm.org/doi/10.1145/3712255.3734303 - title: BEACON + - ref_beacon source: - artificial tags: null @@ -585,11 +533,7 @@ gen_ealain: - 9 - 10 references: - - authors: [] - link: - type: null - url: https://doi.org/10.1145/3638530.3654299 - title: Ealain + - ref_ealain source: - real-world-like tags: null @@ -598,15 +542,15 @@ gen_ealain: - dim: max: null min: 1 - type: integer + type: continuous - dim: max: null min: 1 - type: binary + type: integer - dim: max: null min: 1 - type: continuous + type: binary gen_gnbg: allows_partial_evaluation: null can_evaluate_objectives_independently: null @@ -625,11 +569,7 @@ gen_gnbg: objectives: - 1 references: - - authors: [] - link: - type: null - url: https://arxiv.org/abs/2312.07083 - title: GNBG + - ref_gnbg source: - artificial tags: null @@ -649,8 +589,8 @@ gen_gnbg_ii: evaluation_time: null fidelity_levels: null implementations: - - impl_gnbg_ii - impl_iohgnbg + - impl_gnbg_ii long_name: null modality: null name: GNBG-II @@ -658,11 +598,7 @@ gen_gnbg_ii: objectives: - 1 references: - - authors: [] - link: - type: null - url: https://dl.acm.org/doi/pdf/10.1145/3712255.3734271 - title: GNBG-II + - ref_gnbg_ii source: - artificial tags: null @@ -702,11 +638,7 @@ gen_gpd: - 9 - 10 references: - - authors: [] - link: - type: null - url: https://doi.org/10.1016/j.asoc.2020.106139 - title: GPD generator + - ref_gpd_generator source: null tags: null type: generator @@ -734,11 +666,7 @@ gen_iohclustering: objectives: - 1 references: - - authors: [] - link: - type: null - url: https://arxiv.org/pdf/2505.09233 - title: IOHClustering + - ref_iohclustering source: - artificial-from-real-data tags: null @@ -758,8 +686,8 @@ gen_ma_bbob: evaluation_time: null fidelity_levels: null implementations: - - impl_ma_bbob - impl_iohexperimenter + - impl_ma_bbob long_name: null modality: - multimodal @@ -768,11 +696,7 @@ gen_ma_bbob: objectives: - 1 references: - - authors: [] - link: - type: null - url: https://doi.org/10.1145/3673908 - title: MA-BBOB + - ref_ma_bbob source: - artificial tags: null @@ -802,11 +726,7 @@ gen_mpm2: objectives: - 1 references: - - authors: [] - link: - type: null - url: https://ls11-www.cs.tu-dortmund.de/_media/techreports/tr15-01.pdf - title: MPM2 technical report TR15-01 + - ref_mpm2_technical_report_tr15_01 source: null tags: null type: generator @@ -845,11 +765,7 @@ gen_mubqp: - 9 - 10 references: - - authors: [] - link: - type: null - url: https://doi.org/10.1016/j.asoc.2013.11.008 - title: mUBQP benchmark + - ref_mubqp_benchmark source: null tags: null type: generator @@ -878,11 +794,7 @@ gen_puboi: objectives: - 1 references: - - authors: [] - link: - type: null - url: https://link.springer.com/chapter/10.1007/978-3-031-04148-8_12 - title: PUBOi + - ref_puboi source: - artificial tags: null @@ -929,15 +841,15 @@ gen_randoptgen: - dim: max: null min: 1 - type: integer + type: continuous - dim: max: null min: 1 - type: binary + type: integer - dim: max: null min: 1 - type: continuous + type: binary gen_rho_mnk_landscapes: allows_partial_evaluation: null can_evaluate_objectives_independently: null @@ -967,12 +879,7 @@ gen_rho_mnk_landscapes: - 9 - 10 references: - - authors: [] - link: - type: null - url: https://doi.org/10.1016/j.ejor.2012.12.019 - title: On the design of multi-objective evolutionary algorithms based on - NK-landscapes + - ref_on_the_design_of source: null tags: null type: generator @@ -1011,11 +918,7 @@ gen_rho_mtsp: - 9 - 10 references: - - authors: [] - link: - type: null - url: https://doi.org/10.1007/978-3-319-45823-6_40 - title: On the impact of multi-objective scalability for the ρmTSP + - ref_on_the_impact_of source: null tags: null type: generator @@ -1043,11 +946,7 @@ gen_wmodel: objectives: - 1 references: - - authors: [] - link: - type: null - url: https://dl.acm.org/doi/abs/10.1145/3205651.3208240 - title: W-model + - ref_w_model source: - artificial tags: null @@ -1082,8 +981,8 @@ impl_bonobench: impl_bso_toolbox: description: Building Spatial Design toolbox (TU/e) evaluation_time: - - 1 second - 40 seconds + - 1 second language: C++ links: - type: repository @@ -1199,8 +1098,8 @@ impl_expobench: description: EXPensive Optimization benchmark library (wind farm layout, gas filter design, pipe shape, hyperparameter tuning, hospital simulation) evaluation_time: - - 80 seconds - 2 seconds + - 80 seconds language: Python links: - type: repository @@ -1221,8 +1120,8 @@ impl_gasoline: impl_gbea: description: Game-Benchmark for Evolutionary Algorithms (COCO fork) evaluation_time: - - 34 seconds - 5 seconds + - 34 seconds language: null links: - type: repository @@ -1306,8 +1205,8 @@ impl_ma_bbob: impl_mechbench: description: Structural mechanics design optimization benchmark evaluation_time: - - 1 minute - 7 minutes + - 1 minute language: Python links: - type: repository @@ -1487,6 +1386,475 @@ impl_wmodel: name: BBDOB W-Model requirements: null type: implementation +ref_a_multi_step_evaluation: + authors: + - Tea Tušar + - Peter Korošec + - Bogdan Filipič + link: + type: null + url: https://dis.ijs.si/tea/Publications/Tusar23Multistep.pdf + title: A Multi-Step Evaluation Process in Electric Motor Design + type: reference +ref_a_review_of_multiobjective: + authors: + - Simon Huband + - Philip Hingston + - Luigi Barone + - Lyndon While + link: + type: null + url: https://doi.org/10.1109/TEVC.2005.861417 + title: A review of multiobjective test problems and a scalable test problem + toolkit + type: reference +ref_amvop: + authors: null + link: + type: null + url: https://doi.org/10.1109/TEVC.2013.2281531 + title: RWMVOP + type: reference +ref_bbcomp_emo_2017: + authors: null + link: + type: null + url: https://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/ + title: BBComp EMO 2017 + type: reference +ref_bbob_bi_objective_test_suite: + authors: null + link: + type: null + url: https://doi.org/10.48550/arXiv.1604.00359 + title: BBOB bi-objective test suite + type: reference +ref_bbob_constrained_documentation: + authors: null + link: + type: null + url: http://numbbo.github.io/coco-doc/bbob-constrained/ + title: bbob-constrained documentation + type: reference +ref_bbob_large_scale_test_suite: + authors: null + link: + type: null + url: https://doi.org/10.48550/arXiv.1903.06396 + title: BBOB large-scale test suite + type: reference +ref_bbob_mixed_integer_test_suite: + authors: null + link: + type: null + url: https://doi.org/10.1145/3321707.3321868 + title: BBOB bi-objective mixed-integer test suite + type: reference +ref_beacon: + authors: null + link: + type: null + url: https://dl.acm.org/doi/10.1145/3712255.3734303 + title: BEACON + type: reference +ref_benchmark_functions_for_cec: + authors: null + link: null + title: Benchmark Functions for CEC 2015 Special Session and Competition on + Dynamic Multi-objective Optimization + type: reference +ref_bp_benchmark: + authors: null + link: + type: null + url: https://doi.org/10.1109/CEC.2019.8790277 + title: BP benchmark + type: reference +ref_brachytherapy_treatment_planning: + authors: null + link: + type: null + url: https://www.sciencedirect.com/science/article/pii/S1538472123016781 + title: Brachytherapy treatment planning + type: reference +ref_building_spatial_design: + authors: null + link: + type: null + url: https://hdl.handle.net/1887/81789 + title: Building spatial design + type: reference +ref_car_structure_design_benchmark: + authors: null + link: + type: null + url: https://doi.org/10.1145/3205651.3205702 + title: Car structure design benchmark + type: reference +ref_cdmp_benchmark: + authors: null + link: + type: null + url: https://doi.org/10.1145/3321707.3321878 + title: CDMP benchmark + type: reference +ref_cec2013_definitions: + authors: null + link: + type: null + url: https://peerj.com/articles/cs-2671/CEC2013.pdf + title: CEC2013 definitions + type: reference +ref_cec2018_dmop_competition_tr: + authors: null + link: + type: null + url: + https://www.academia.edu/download/94499025/TR-CEC2018-DMOP-Competition.pdf + title: CEC2018 DMOP Competition TR + type: reference +ref_cec2022_tr: + authors: null + link: + type: null + url: https://github.com/P-N-Suganthan/2022-SO-BO/blob/main/CEC2022%20TR.pdf + title: CEC2022 TR + type: reference +ref_cfd_test_problem_suite: + authors: null + link: + type: null + url: https://doi.org/10.1007/978-3-319-99259-4_24 + title: CFD test problem suite + type: reference +ref_coco_a_platform_for: + authors: null + link: + type: null + url: https://doi.org/10.1080/10556788.2020.1808977 + title: 'COCO: a platform for comparing continuous optimizers in a black-box setting' + type: reference +ref_comparison_of_multiobjective_evolutionary: + authors: + - Eckart Zitzler + - Kalyanmoy Deb + - Lothar Thiele + link: + type: null + url: https://doi.org/10.1162/106365600568202 + title: 'Comparison of multiobjective evolutionary algorithms: empirical results' + type: reference +ref_convex_dtlz2: + authors: null + link: + type: null + url: https://doi.org/10.1109/TEVC.2013.2281535 + title: Convex DTLZ2 + type: reference +ref_cuter: + authors: null + link: + type: null + url: https://dl.acm.org/doi/10.1145/962437.962439 + title: CUTEr + type: reference +ref_cutest: + authors: null + link: + type: null + url: https://link.springer.com/article/10.1007/s10589-014-9687-3 + title: CUTEst + type: reference +ref_dynamicbinval: + authors: null + link: + type: null + url: https://arxiv.org/pdf/2404.15837 + title: DynamicBinVal + type: reference +ref_ealain: + authors: null + link: + type: null + url: https://doi.org/10.1145/3638530.3654299 + title: Ealain + type: reference +ref_easy_to_evaluate_real: + authors: + - Ryoji Tanabe + - Hisao Ishibuchi + link: + type: null + url: https://doi.org/10.1016/j.asoc.2020.106078 + title: Easy-to-evaluate real-world multi-objective optimization problems + type: reference +ref_evolutionary_many_task_optimization: + authors: null + link: + type: null + url: https://doi.org/10.48550/arXiv.2110.08033 + title: Evolutionary many-task optimization framework + type: reference +ref_expobench: + authors: null + link: + type: null + url: https://doi.org/10.1016/j.asoc.2023.110744 + title: EXPObench + type: reference +ref_fleetopt: + authors: null + link: + type: null + url: https://dl.acm.org/doi/abs/10.1145/3638530.3664137 + title: FleetOpt + type: reference +ref_game_benchmark_for_evolutionary: + authors: null + link: + type: null + url: https://doi.org/10.1145/3321707.3321805 + title: Game benchmark for evolutionary algorithms + type: reference +ref_gasoline_direct_injection_engine_design: + authors: null + link: + type: null + url: https://doi.org/10.1016/j.ejor.2022.08.032 + title: Gasoline direct injection engine design + type: reference +ref_gnbg: + authors: null + link: + type: null + url: https://arxiv.org/abs/2312.07083 + title: GNBG + type: reference +ref_gnbg_ii: + authors: null + link: + type: null + url: https://dl.acm.org/doi/pdf/10.1145/3712255.3734271 + title: GNBG-II + type: reference +ref_gpd_generator: + authors: null + link: + type: null + url: https://doi.org/10.1016/j.asoc.2020.106139 + title: GPD generator + type: reference +ref_inverted_dtlz1: + authors: null + link: + type: null + url: https://doi.org/10.1109/TEVC.2013.2281534 + title: Inverted DTLZ1 + type: reference +ref_iohclustering: + authors: null + link: + type: null + url: https://arxiv.org/pdf/2505.09233 + title: IOHClustering + type: reference +ref_jpnsec_ec_symposium_2019_competition: + authors: null + link: + type: null + url: + http://www.jpnsec.org/files/competition2019/EC-Symposium-2019-Competition-English.html + title: JPNSEC EC-Symposium 2019 competition + type: reference +ref_kinematics_of_a_robot_arm: + authors: null + link: + type: null + url: https://doi.org/10.1023/A:1013258808932 + title: Kinematics of a robot arm + type: reference +ref_linkage_zdt_dtlz_variants: + authors: null + link: + type: null + url: https://doi.org/10.1145/1143997.1144179 + title: Linkage ZDT/DTLZ variants + type: reference +ref_ma_bbob: + authors: null + link: + type: null + url: https://doi.org/10.1145/3673908 + title: MA-BBOB + type: reference +ref_maop_benchmark: + authors: null + link: + type: null + url: https://doi.org/10.1016/j.swevo.2019.02.003 + title: MaOP benchmark + type: reference +ref_mechbench: + authors: null + link: + type: null + url: https://arxiv.org/abs/2511.10821 + title: MECHBench + type: reference +ref_mf2_a_collection_of: + authors: null + link: + type: null + url: https://doi.org/10.21105/joss.02049 + title: 'mf2: a collection of multi-fidelity benchmark functions in Python' + type: reference +ref_minus_dtlz_minus_wfg: + authors: null + link: + type: null + url: https://doi.org/10.1109/TEVC.2016.2587749 + title: Minus DTLZ / Minus WFG + type: reference +ref_mmopp_technical_report: + authors: null + link: + type: null + url: + http://www5.zzu.edu.cn/system/_content/download.jsp?urltype=news.DownloadAttachUrl&owner=1327567121&wbfileid=4764412 + title: MMOPP technical report + type: reference +ref_modact: + authors: null + link: + type: null + url: https://doi.org/10.1109/TEVC.2020.3020046 + title: MODAct + type: reference +ref_mpm2_technical_report_tr15_01: + authors: null + link: + type: null + url: https://ls11-www.cs.tu-dortmund.de/_media/techreports/tr15-01.pdf + title: MPM2 technical report TR15-01 + type: reference +ref_mubqp_benchmark: + authors: null + link: + type: null + url: https://doi.org/10.1016/j.asoc.2013.11.008 + title: mUBQP benchmark + type: reference +ref_on_the_design_of: + authors: null + link: + type: null + url: https://doi.org/10.1016/j.ejor.2012.12.019 + title: On the design of multi-objective evolutionary algorithms based on + NK-landscapes + type: reference +ref_on_the_impact_of: + authors: null + link: + type: null + url: https://doi.org/10.1007/978-3-319-45823-6_40 + title: On the impact of multi-objective scalability for the ρmTSP + type: reference +ref_onemax_sphere_zeromax_sphere: + authors: null + link: + type: null + url: https://doi.org/10.1145/3449726.3459521 + title: Mixed-variable multi-objective test problems + type: reference +ref_pbo_benchmarks: + authors: null + link: + type: null + url: https://dl.acm.org/doi/pdf/10.1145/3319619.3326810 + title: PBO benchmarks + type: reference +ref_porkchop_plot_interplanetary_trajectory: + authors: null + link: + type: null + url: https://doi.org/10.1109/CEC65147.2025.11042973 + title: Porkchop plot interplanetary trajectory benchmark + type: reference +ref_puboi: + authors: null + link: + type: null + url: https://link.springer.com/chapter/10.1007/978-3-031-04148-8_12 + title: PUBOi + type: reference +ref_radar_waveform_design: + authors: null + link: + type: null + url: https://doi.org/10.1007/978-3-540-70928-2_53 + title: Radar waveform design + type: reference +ref_real_parameter_black_box: + authors: null + link: + type: null + url: https://hal.inria.fr/inria-00369466 + title: 'Real-parameter black-box optimization benchmarking: noisy functions definitions' + type: reference +ref_sbox_cost: + authors: null + link: + type: null + url: https://doi.org/10.48550/arXiv.2305.12221 + title: SBOX-COST + type: reference +ref_scalable_multi_objective_optimization: + authors: + - Kalyanmoy Deb + - Lothar Thiele + - Marco Laumanns + - Eckart Zitzler + link: + type: null + url: https://doi.org/10.1109/CEC.2002.1007032 + title: Scalable multi-objective optimization test problems + type: reference +ref_sdp_dynamic_multi_objective_benchmark: + authors: null + link: + type: null + url: https://doi.org/10.1109/TCYB.2019.2896021 + title: SDP dynamic multi-objective benchmark + type: reference +ref_submodular_optimization_benchmark: + authors: null + link: + type: null + url: https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=10254181 + title: Submodular optimization benchmark + type: reference +ref_tulipaenergymodel_jl_scientific_references: + authors: null + link: + type: null + url: + https://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/40-scientific-foundation/45-scientific-references + title: TulipaEnergyModel.jl scientific references + type: reference +ref_vehicledynamics_benchmark: + authors: null + link: + type: null + url: https://www.scitepress.org/Papers/2023/121580/121580.pdf + title: VehicleDynamics benchmark + type: reference +ref_w_model: + authors: null + link: + type: null + url: https://dl.acm.org/doi/abs/10.1145/3205651.3208240 + title: W-model + type: reference suite_amvop: allows_partial_evaluation: null can_evaluate_objectives_independently: null @@ -1506,11 +1874,7 @@ suite_amvop: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1109/TEVC.2013.2281531 - title: AMVOP + - ref_amvop source: null tags: null type: suite @@ -1518,7 +1882,7 @@ suite_amvop: - dim: max: null min: 1 - type: categorical + type: continuous - dim: max: null min: 1 @@ -1526,7 +1890,7 @@ suite_amvop: - dim: max: null min: 1 - type: continuous + type: categorical suite_bbob: allows_partial_evaluation: null can_evaluate_objectives_independently: null @@ -1547,11 +1911,7 @@ suite_bbob: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1080/10556788.2020.1808977 - title: 'COCO: a platform for comparing continuous optimizers in a black-box setting' + - ref_coco_a_platform_for source: null tags: null type: suite @@ -1580,11 +1940,7 @@ suite_bbob_biobj: - 2 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.48550/arXiv.1604.00359 - title: BBOB bi-objective test suite + - ref_bbob_bi_objective_test_suite source: null tags: null type: suite @@ -1613,11 +1969,7 @@ suite_bbob_biobj_mixint: - 2 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1145/3321707.3321868 - title: BBOB bi-objective mixed-integer test suite + - ref_bbob_mixed_integer_test_suite source: null tags: null type: suite @@ -1625,11 +1977,11 @@ suite_bbob_biobj_mixint: - dim: max: 160 min: 5 - type: integer + type: continuous - dim: max: 160 min: 5 - type: continuous + type: integer suite_bbob_constrained: allows_partial_evaluation: null can_evaluate_objectives_independently: null @@ -1654,11 +2006,7 @@ suite_bbob_constrained: - 1 problems: null references: - - authors: [] - link: - type: null - url: http://numbbo.github.io/coco-doc/bbob-constrained/ - title: bbob-constrained documentation + - ref_bbob_constrained_documentation source: null tags: null type: suite @@ -1687,11 +2035,7 @@ suite_bbob_largescale: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.48550/arXiv.1903.06396 - title: BBOB large-scale test suite + - ref_bbob_large_scale_test_suite source: null tags: null type: suite @@ -1720,11 +2064,7 @@ suite_bbob_mixint: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1145/3321707.3321868 - title: BBOB mixed-integer test suite + - ref_bbob_mixed_integer_test_suite source: null tags: null type: suite @@ -1732,11 +2072,11 @@ suite_bbob_mixint: - dim: max: 160 min: 5 - type: integer + type: continuous - dim: max: 160 min: 5 - type: continuous + type: integer suite_bbob_noisy: allows_partial_evaluation: null can_evaluate_objectives_independently: null @@ -1758,11 +2098,7 @@ suite_bbob_noisy: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://hal.inria.fr/inria-00369466 - title: 'Real-parameter black-box optimization benchmarking: noisy functions definitions' + - ref_real_parameter_black_box source: null tags: null type: suite @@ -1798,11 +2134,7 @@ suite_bp: - 10 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1109/CEC.2019.8790277 - title: BP benchmark + - ref_bp_benchmark source: null tags: null type: suite @@ -1840,11 +2172,7 @@ suite_brachytherapy: - 3 problems: null references: - - authors: [] - link: - type: null - url: https://www.sciencedirect.com/science/article/pii/S1538472123016781 - title: Brachytherapy treatment planning + - ref_brachytherapy_treatment_planning source: - real-world tags: null @@ -1877,11 +2205,7 @@ suite_car_structure: - 2 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1145/3205651.3205702 - title: Car structure design benchmark + - ref_car_structure_design_benchmark source: - real-world tags: null @@ -1923,11 +2247,7 @@ suite_cdmp: - 10 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1145/3321707.3321878 - title: CDMP benchmark + - ref_cdmp_benchmark source: null tags: null type: suite @@ -1946,8 +2266,8 @@ suite_cec2013: evaluation_time: null fidelity_levels: null implementations: - - impl_cec2013 - impl_iohexperimenter + - impl_cec2013 long_name: null modality: null name: CEC2013 @@ -1956,11 +2276,7 @@ suite_cec2013: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://peerj.com/articles/cs-2671/CEC2013.pdf - title: CEC2013 definitions + - ref_cec2013_definitions source: - artificial tags: null @@ -1994,10 +2310,7 @@ suite_cec2015_dmoo: - 3 problems: null references: - - authors: [] - link: null - title: Benchmark Functions for CEC 2015 Special Session and Competition on - Dynamic Multi-objective Optimization + - ref_benchmark_functions_for_cec source: null tags: null type: suite @@ -2027,12 +2340,7 @@ suite_cec2018_dt: - 3 problems: null references: - - authors: [] - link: - type: null - url: - https://www.academia.edu/download/94499025/TR-CEC2018-DMOP-Competition.pdf - title: CEC2018 DMOP Competition TR + - ref_cec2018_dmop_competition_tr source: - artificial tags: null @@ -2062,12 +2370,7 @@ suite_cec2022: - 1 problems: null references: - - authors: [] - link: - type: null - url: - https://github.com/P-N-Suganthan/2022-SO-BO/blob/main/CEC2022%20TR.pdf - title: CEC2022 TR + - ref_cec2022_tr source: - artificial tags: null @@ -2101,11 +2404,7 @@ suite_cfd: - 2 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1007/978-3-319-99259-4_24 - title: CFD test problem suite + - ref_cfd_test_problem_suite source: - real-world tags: null @@ -2141,13 +2440,7 @@ suite_cre: - 5 problems: null references: - - authors: - - Ryoji Tanabe - - Hisao Ishibuchi - link: - type: null - url: https://doi.org/10.1016/j.asoc.2020.106078 - title: Easy-to-evaluate real-world multi-objective optimization problems + - ref_easy_to_evaluate_real source: - real-world-like tags: null @@ -2183,11 +2476,7 @@ suite_cuter: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://dl.acm.org/doi/10.1145/962437.962439 - title: CUTEr + - ref_cuter source: - artificial tags: null @@ -2196,15 +2485,15 @@ suite_cuter: - dim: max: null min: 1 - type: integer + type: continuous - dim: max: null min: 1 - type: binary + type: integer - dim: max: null min: 1 - type: continuous + type: binary suite_cutest: allows_partial_evaluation: no can_evaluate_objectives_independently: null @@ -2235,11 +2524,7 @@ suite_cutest: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://link.springer.com/article/10.1007/s10589-014-9687-3 - title: CUTEst + - ref_cutest source: - artificial tags: null @@ -2248,15 +2533,15 @@ suite_cutest: - dim: max: null min: 1 - type: integer + type: continuous - dim: max: null min: 1 - type: binary + type: integer - dim: max: null min: 1 - type: continuous + type: binary suite_dtlz: allows_partial_evaluation: null can_evaluate_objectives_independently: null @@ -2284,15 +2569,7 @@ suite_dtlz: - 10 problems: null references: - - authors: - - Kalyanmoy Deb - - Lothar Thiele - - Marco Laumanns - - Eckart Zitzler - link: - type: null - url: https://doi.org/10.1109/CEC.2002.1007032 - title: Scalable multi-objective optimization test problems + - ref_scalable_multi_objective_optimization source: null tags: null type: suite @@ -2321,11 +2598,7 @@ suite_dynamicbinval: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://arxiv.org/pdf/2404.15837 - title: DynamicBinVal + - ref_dynamicbinval source: - artificial tags: null @@ -2354,11 +2627,7 @@ suite_emo2017: - 2 problems: null references: - - authors: [] - link: - type: null - url: https://www.ini.rub.de/PEOPLE/glasmtbl/projects/bbcomp/ - title: BBComp EMO 2017 + - ref_bbcomp_emo_2017 source: - real-world tags: null @@ -2436,11 +2705,7 @@ suite_etmof: - 50 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.48550/arXiv.2110.08033 - title: Evolutionary many-task optimization framework + - ref_evolutionary_many_task_optimization source: null tags: null type: suite @@ -2479,11 +2744,7 @@ suite_expobench: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1016/j.asoc.2023.110744 - title: EXPObench + - ref_expobench source: - real-world tags: null @@ -2492,7 +2753,7 @@ suite_expobench: - dim: max: 135 min: 10 - type: categorical + type: continuous - dim: max: 135 min: 10 @@ -2500,7 +2761,7 @@ suite_expobench: - dim: max: 135 min: 10 - type: continuous + type: categorical suite_gbea: allows_partial_evaluation: null can_evaluate_objectives_independently: null @@ -2524,11 +2785,7 @@ suite_gbea: - 2 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1145/3321707.3321805 - title: Game benchmark for evolutionary algorithms + - ref_game_benchmark_for_evolutionary source: - real-world tags: null @@ -2557,11 +2814,7 @@ suite_gnbg: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://arxiv.org/abs/2312.07083 - title: GNBG + - ref_gnbg source: - artificial tags: null @@ -2582,8 +2835,8 @@ suite_gnbg_ii: evaluation_time: null fidelity_levels: null implementations: - - impl_gnbg_ii - impl_iohgnbg + - impl_gnbg_ii long_name: null modality: null name: GNBG-II @@ -2592,11 +2845,7 @@ suite_gnbg_ii: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://dl.acm.org/doi/pdf/10.1145/3712255.3734271 - title: GNBG-II + - ref_gnbg_ii source: - artificial tags: null @@ -2627,11 +2876,7 @@ suite_iohclustering: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://arxiv.org/pdf/2505.09233 - title: IOHClustering + - ref_iohclustering source: - artificial-from-real-data tags: null @@ -2661,11 +2906,7 @@ suite_kinematics_robotarm: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1023/A:1013258808932 - title: Kinematics of a robot arm + - ref_kinematics_of_a_robot_arm source: - real-world tags: null @@ -2691,11 +2932,7 @@ suite_l1_zdt: - 2 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1145/1143997.1144179 - title: Linkage ZDT/DTLZ variants + - ref_linkage_zdt_dtlz_variants source: null tags: null type: suite @@ -2703,11 +2940,11 @@ suite_l1_zdt: - dim: max: null min: 1 - type: binary + type: continuous - dim: max: null min: 1 - type: continuous + type: binary suite_l2_dtlz: allows_partial_evaluation: null can_evaluate_objectives_independently: null @@ -2734,11 +2971,7 @@ suite_l2_dtlz: - 10 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1145/1143997.1144179 - title: Linkage ZDT/DTLZ variants + - ref_linkage_zdt_dtlz_variants source: null tags: null type: suite @@ -2765,11 +2998,7 @@ suite_l2_zdt: - 2 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1145/1143997.1144179 - title: Linkage ZDT/DTLZ variants + - ref_linkage_zdt_dtlz_variants source: null tags: null type: suite @@ -2777,11 +3006,11 @@ suite_l2_zdt: - dim: max: null min: 1 - type: binary + type: continuous - dim: max: null min: 1 - type: continuous + type: binary suite_l3_dtlz: allows_partial_evaluation: null can_evaluate_objectives_independently: null @@ -2808,11 +3037,7 @@ suite_l3_dtlz: - 10 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1145/1143997.1144179 - title: Linkage ZDT/DTLZ variants + - ref_linkage_zdt_dtlz_variants source: null tags: null type: suite @@ -2839,11 +3064,7 @@ suite_l3_zdt: - 2 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1145/1143997.1144179 - title: Linkage ZDT/DTLZ variants + - ref_linkage_zdt_dtlz_variants source: null tags: null type: suite @@ -2851,11 +3072,11 @@ suite_l3_zdt: - dim: max: null min: 1 - type: binary + type: continuous - dim: max: null min: 1 - type: continuous + type: binary suite_maop: allows_partial_evaluation: null can_evaluate_objectives_independently: null @@ -2883,11 +3104,7 @@ suite_maop: - 10 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1016/j.swevo.2019.02.003 - title: MaOP benchmark + - ref_maop_benchmark source: null tags: null type: suite @@ -2924,11 +3141,7 @@ suite_mechbench: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://arxiv.org/abs/2511.10821 - title: MECHBench + - ref_mechbench source: - real-world tags: null @@ -2959,11 +3172,7 @@ suite_mf2: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.21105/joss.02049 - title: 'mf2: a collection of multi-fidelity benchmark functions in Python' + - ref_mf2_a_collection_of source: null tags: null type: suite @@ -2999,11 +3208,7 @@ suite_minus_dtlz: - 10 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1109/TEVC.2016.2587749 - title: Minus DTLZ / Minus WFG + - ref_minus_dtlz_minus_wfg source: null tags: null type: suite @@ -3039,11 +3244,7 @@ suite_minus_wfg: - 10 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1109/TEVC.2016.2587749 - title: Minus DTLZ / Minus WFG + - ref_minus_dtlz_minus_wfg source: null tags: null type: suite @@ -3081,12 +3282,7 @@ suite_mmopp: - 7 problems: null references: - - authors: [] - link: - type: null - url: - http://www5.zzu.edu.cn/system/_content/download.jsp?urltype=news.DownloadAttachUrl&owner=1327567121&wbfileid=4764412 - title: MMOPP technical report + - ref_mmopp_technical_report source: null tags: null type: suite @@ -3121,11 +3317,7 @@ suite_modact: - 5 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1109/TEVC.2020.3020046 - title: MODAct + - ref_modact source: - real-world tags: null @@ -3185,11 +3377,7 @@ suite_pbo: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://dl.acm.org/doi/pdf/10.1145/3319619.3326810 - title: PBO benchmarks + - ref_pbo_benchmarks source: - artificial tags: null @@ -3219,11 +3407,7 @@ suite_porkchop: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1109/CEC65147.2025.11042973 - title: Porkchop plot interplanetary trajectory benchmark + - ref_porkchop_plot_interplanetary_trajectory source: - real-world tags: null @@ -3257,13 +3441,7 @@ suite_re: - 9 problems: null references: - - authors: - - Ryoji Tanabe - - Hisao Ishibuchi - link: - type: null - url: https://doi.org/10.1016/j.asoc.2020.106078 - title: Easy-to-evaluate real-world multi-objective optimization problems + - ref_easy_to_evaluate_real source: - real-world-like tags: null @@ -3299,11 +3477,7 @@ suite_rwmvop: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1109/TEVC.2013.2281531 - title: RWMVOP + - ref_amvop source: - real-world tags: null @@ -3312,7 +3486,7 @@ suite_rwmvop: - dim: max: null min: 1 - type: categorical + type: continuous - dim: max: null min: 1 @@ -3320,7 +3494,7 @@ suite_rwmvop: - dim: max: null min: 1 - type: continuous + type: categorical suite_sbox_cost: allows_partial_evaluation: null can_evaluate_objectives_independently: null @@ -3342,11 +3516,7 @@ suite_sbox_cost: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.48550/arXiv.2305.12221 - title: SBOX-COST + - ref_sbox_cost source: null tags: null type: suite @@ -3383,11 +3553,7 @@ suite_sdp: - 10 problems: null references: - - authors: [] - link: - type: null - url: https://doi.org/10.1109/TCYB.2019.2896021 - title: SDP dynamic multi-objective benchmark + - ref_sdp_dynamic_multi_objective_benchmark source: null tags: null type: suite @@ -3416,11 +3582,7 @@ suite_submodular: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=10254181 - title: Submodular optimization benchmark + - ref_submodular_optimization_benchmark source: - artificial tags: null @@ -3464,12 +3626,7 @@ suite_tulipa_energy: - 1 problems: null references: - - authors: [] - link: - type: null - url: - https://tulipaenergy.github.io/TulipaEnergyModel.jl/stable/40-scientific-foundation/45-scientific-references - title: TulipaEnergyModel.jl scientific references + - ref_tulipaenergymodel_jl_scientific_references source: - real-world tags: null @@ -3499,11 +3656,7 @@ suite_vehicle_dynamics: - 1 problems: null references: - - authors: [] - link: - type: null - url: https://www.scitepress.org/Papers/2023/121580/121580.pdf - title: VehicleDynamics benchmark + - ref_vehicledynamics_benchmark source: - real-world tags: null @@ -3538,16 +3691,7 @@ suite_wfg: - 10 problems: null references: - - authors: - - Simon Huband - - Philip Hingston - - Luigi Barone - - Lyndon While - link: - type: null - url: https://doi.org/10.1109/TEVC.2005.861417 - title: A review of multiobjective test problems and a scalable test problem - toolkit + - ref_a_review_of_multiobjective source: null tags: null type: suite @@ -3575,14 +3719,7 @@ suite_zdt: - 2 problems: null references: - - authors: - - Eckart Zitzler - - Kalyanmoy Deb - - Lothar Thiele - link: - type: null - url: https://doi.org/10.1162/106365600568202 - title: 'Comparison of multiobjective evolutionary algorithms: empirical results' + - ref_comparison_of_multiobjective_evolutionary source: null tags: null type: suite @@ -3590,8 +3727,8 @@ suite_zdt: - dim: max: null min: 1 - type: binary + type: continuous - dim: max: null min: 1 - type: continuous + type: binary diff --git a/src/opltools/schema.py b/src/opltools/schema.py index a2c5b5b..f4adc54 100644 --- a/src/opltools/schema.py +++ b/src/opltools/schema.py @@ -12,6 +12,7 @@ class OPLType(Enum): suite = "suite" generator = "generator" implementation = "implementation" + reference = "reference" class Link(BaseModel): @@ -73,7 +74,8 @@ def __hash__(self): return hash((self.type, self.hard, self.equality, number)) -class Reference(BaseModel): +class Reference(Thing): + type: OPLType = OPLType.reference title: str | None = None authors: list[str] | None = None link: Link | None = None @@ -108,7 +110,7 @@ class ProblemLike(Thing): long_name: str | None = None description: str | None = None tags: set[str] | None = None - references: set[Reference] | None = None + references: set[str] | None = None implementations: set[str] | None = None objectives: set[int] | None = None variables: set[Variable] | None = None @@ -142,7 +144,7 @@ class Generator(ProblemLike): class Library(RootModel): - root: dict[str, Problem | Generator | Suite | Implementation] = {} + root: dict[str, Problem | Generator | Suite | Implementation | Reference] = {} def _check_id_references(self, ids, type: OPLType) -> None: for id in ids: @@ -174,6 +176,19 @@ def _percolate_set(self, thing: Any, children: set | None, property: str): @model_validator(mode="after") def _validate(self) -> Self: + # Check reference ids on all ProblemLike things + for id, thing in self.root.items(): + if isinstance(thing, ProblemLike) and thing.references: + for ref_id in thing.references: + if ref_id not in self.root: + raise ValueError( + f"{thing.type.name} {id} references reference with undefined id '{ref_id}'." + ) + if self.root[ref_id].type != OPLType.reference: + raise ValueError( + f"{thing.type.name} {id} references reference with id '{ref_id}' but id is a {self.root[ref_id].type.name}." + ) + # First check and fixup all problems for id, thing in self.root.items(): if isinstance(thing, Problem) and thing.implementations: diff --git a/tests/test_library.py b/tests/test_library.py index 542bc7b..c525eb3 100644 --- a/tests/test_library.py +++ b/tests/test_library.py @@ -6,6 +6,7 @@ Implementation, Library, Problem, + Reference, Suite, ) @@ -81,6 +82,35 @@ def test_fixup_fidelity_all_problems_without_levels(self): }) assert lib.root["s1"].fidelity_levels == set() + def test_reference_as_top_level_thing(self): + lib = Library(root={ + "ref1": Reference(title="A paper"), + "p1": Problem(name="P1", references={"ref1"}), + }) + assert isinstance(lib.root["ref1"], Reference) + assert lib.root["p1"].references == {"ref1"} + + def test_problem_references_missing_reference(self): + with pytest.raises(ValidationError, match="undefined id"): + Library(root={ + "p1": Problem(name="P1", references={"does-not-exist"}), + }) + + def test_problem_references_non_reference(self): + with pytest.raises(ValidationError, match="but id is a"): + Library(root={ + "p1": Problem(name="P1"), + "p2": Problem(name="P2", references={"p1"}), + }) + + def test_suite_references_a_reference(self): + lib = Library(root={ + "ref1": Reference(title="A paper"), + "p1": Problem(name="P1"), + "s1": Suite(name="S1", problems={"p1"}, references={"ref1"}), + }) + assert lib.root["s1"].references == {"ref1"} + def test_fixup_evaluation_time_percolates_from_implementation_to_suite(self): lib = Library(root={ "impl1": Implementation( diff --git a/tests/test_reference.py b/tests/test_reference.py index 0d388a6..f3af998 100644 --- a/tests/test_reference.py +++ b/tests/test_reference.py @@ -1,12 +1,13 @@ import pytest from pydantic import ValidationError -from opltools.schema import Link, Reference +from opltools.schema import Link, OPLType, Reference class TestReference: def test_only_author(self): ref = Reference(title="A paper") + assert ref.type is OPLType.reference assert ref.title == "A paper" assert ref.authors is None assert ref.link is None