Skip to content

Commit

Permalink
Merge branch 'master' into minuit_upgrade
Browse files Browse the repository at this point in the history
  • Loading branch information
@tyronerees
tyronerees committed Feb 20, 2024
2 parents f94dc87 + 0b24c7c commit d7f1515
Show file tree
Hide file tree
Showing 8 changed files with 86 additions and 50 deletions.
84 changes: 52 additions & 32 deletions fitbenchmarking/core/fitting_benchmarking.py
View file
Expand Up @@ -8,7 +8,6 @@
import timeit
import warnings

from contextlib import nullcontext
import numpy as np
from codecarbon import EmissionsTracker
from tqdm import tqdm, trange
Expand Down Expand Up @@ -126,6 +125,13 @@ def loop_over_benchmark_problems(problem_group, options, checkpointer):
problems.append((p, parsed_problem))

name_index = {key: 0 for key in name_count}

track_emissions = 'emissions' in options.table_type
if track_emissions:
emissions_tracker = EmissionsTracker()
else:
emissions_tracker = None

LOGGER.info('Running problems')

if options.pbar:
Expand Down Expand Up @@ -156,14 +162,19 @@ def loop_over_benchmark_problems(problem_group, options, checkpointer):
loop_over_starting_values(problem,
options=options,
grabbed_output=grabbed_output,
checkpointer=checkpointer)
checkpointer=checkpointer,
emissions_tracker=emissions_tracker)
results.extend(problem_results)
failed_problems.extend(problem_fails)

if emissions_tracker:
_ = emissions_tracker.stop()

return results, failed_problems, unselected_minimizers


def loop_over_starting_values(problem, options, grabbed_output, checkpointer):
def loop_over_starting_values(problem, options, grabbed_output, checkpointer,
emissions_tracker):
"""
Loops over starting values from the fitting problem.
Expand Down Expand Up @@ -209,7 +220,8 @@ def loop_over_starting_values(problem, options, grabbed_output, checkpointer):
options=options,
start_values_index=index,
grabbed_output=grabbed_output,
checkpointer=checkpointer)
checkpointer=checkpointer,
emissions_tracker=emissions_tracker)

# Checks to see if all of the minimizers from every software raised an
# exception and record the problem name if that is the case
Expand All @@ -226,7 +238,7 @@ def loop_over_starting_values(problem, options, grabbed_output, checkpointer):


def loop_over_cost_function(problem, options, start_values_index,
grabbed_output, checkpointer):
grabbed_output, checkpointer, emissions_tracker):
"""
Run benchmarking for each cost function given in options.
Expand Down Expand Up @@ -266,14 +278,16 @@ def loop_over_cost_function(problem, options, start_values_index,
options=options,
start_values_index=start_values_index,
grabbed_output=grabbed_output,
checkpointer=checkpointer)
checkpointer=checkpointer,
emissions_tracker=emissions_tracker)
problem_results.extend(individual_problem_results)

return problem_results, unselected_minimizers


def loop_over_fitting_software(cost_func, options, start_values_index,
grabbed_output, checkpointer):
def loop_over_fitting_software(cost_func, options,
start_values_index, grabbed_output,
checkpointer, emissions_tracker):
"""
Loops over fitting software selected in the options
Expand Down Expand Up @@ -330,15 +344,16 @@ def loop_over_fitting_software(cost_func, options, start_values_index,
minimizers=minimizers,
options=options,
grabbed_output=grabbed_output,
checkpointer=checkpointer)
checkpointer=checkpointer,
emissions_tracker=emissions_tracker)

unselected_minimizers[s] = minimizer_failed
results.extend(problem_result)
return results, unselected_minimizers


def loop_over_minimizers(controller, minimizers, options, grabbed_output,
checkpointer):
checkpointer, emissions_tracker):
"""
Loops over minimizers in fitting software
Expand Down Expand Up @@ -404,13 +419,15 @@ def loop_over_minimizers(controller, minimizers, options, grabbed_output,
results = loop_over_jacobians(controller,
options=options,
grabbed_output=grabbed_output,
checkpointer=checkpointer)
checkpointer=checkpointer,
emissions_tracker=emissions_tracker)
results_problem.extend(results)

return results_problem, minimizer_failed


def loop_over_jacobians(controller, options, grabbed_output, checkpointer):
def loop_over_jacobians(controller, options, grabbed_output, checkpointer,
emissions_tracker):
"""
Loops over Jacobians set from the options file
Expand Down Expand Up @@ -460,10 +477,13 @@ def loop_over_jacobians(controller, options, grabbed_output, checkpointer):
#######################
# Loops over Hessians #
#######################
new_result = loop_over_hessians(controller,
options=options,
grabbed_output=grabbed_output,
checkpointer=checkpointer)
new_result = loop_over_hessians(
controller,
options=options,
grabbed_output=grabbed_output,
checkpointer=checkpointer,
emissions_tracker=emissions_tracker
)

results.extend(new_result)
# For minimizers that do not accept jacobians we raise an
Expand All @@ -478,7 +498,8 @@ def loop_over_jacobians(controller, options, grabbed_output, checkpointer):
return results


def loop_over_hessians(controller, options, grabbed_output, checkpointer):
def loop_over_hessians(controller, options, grabbed_output, checkpointer,
emissions_tracker):
"""
Loops over Hessians set from the options file
Expand Down Expand Up @@ -533,7 +554,7 @@ def loop_over_hessians(controller, options, grabbed_output, checkpointer):

# Perform the fit a number of times specified by num_runs
accuracy, runtimes, emissions = perform_fit(
controller, options, grabbed_output)
controller, options, grabbed_output, emissions_tracker)
result_args = {'controller': controller,
'accuracy': accuracy,
'runtimes': runtimes,
Expand Down Expand Up @@ -561,7 +582,7 @@ def loop_over_hessians(controller, options, grabbed_output, checkpointer):
return new_result


def perform_fit(controller, options, grabbed_output):
def perform_fit(controller, options, grabbed_output, emissions_tracker):
"""
Performs a fit using the provided controller and its data. It
will be run a number of times specified by num_runs.
Expand All @@ -577,24 +598,19 @@ def perform_fit(controller, options, grabbed_output):
"""
num_runs = options.num_runs

track_emissions = 'emissions' in options.table_type
if track_emissions:
emissions_tracker = EmissionsTracker()
else:
emissions_tracker = nullcontext()
emissions = np.inf
emissions = np.nan
try:
with grabbed_output:
controller.validate()
controller.prepare()
with emissions_tracker:
# Calls timeit repeat with repeat = num_runs and number = 1
runtimes = timeit.Timer(
stmt=controller.execute
).repeat(num_runs, 1)
if track_emissions:
if emissions_tracker:
emissions_tracker.start_task()
runtimes = timeit.Timer(
stmt=controller.execute
).repeat(num_runs, 1)
if emissions_tracker:
# stop emissions tracking after all runs have completed
emissions = emissions_tracker.final_emissions / num_runs
emissions = emissions_tracker.stop_task().emissions / num_runs

controller.cleanup()
controller.check_attributes()
Expand Down Expand Up @@ -650,6 +666,10 @@ def perform_fit(controller, options, grabbed_output):
# Reset the controller timer once exceptions have been handled
controller.timer.reset()

# ensure emissions tracker has been stopped if emissions not set
if emissions == np.nan and emissions_tracker:
_ = emissions_tracker.stop_task()

if controller.flag in [3, 6, 7]:
# If there was an exception, set the runtimes and
# cost function value to be infinite
Expand Down
1 change: 1 addition & 0 deletions fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark_problems.py
View file
Expand Up @@ -66,6 +66,7 @@ def setUp(self):
"""
cost_func = make_cost_function()
self.options = Options()
self.options.table_type = ['acc', 'runtime', 'compare', 'local_min']
self.options.software = ["scipy"]
scipy_len = len(self.options.minimizers["scipy"])
bench_prob_dir = os.path.dirname(inspect.getfile(test_files))
Expand Down
22 changes: 14 additions & 8 deletions fitbenchmarking/core/tests/test_fitting_benchmarking_hessians.py
View file
Expand Up @@ -117,7 +117,6 @@ def setUp(self):
self.grabbed_output = output_grabber.OutputGrabber(self.options)
self.controller.parameter_set = 0
self.cp = Checkpoint(self.options)
self.options.table_type = ['acc', 'runtime', 'compare', 'local_min']

def tearDown(self) -> None:
"""
Expand All @@ -135,7 +134,8 @@ def test_single_hessian(self):
_ = loop_over_hessians(self.controller,
options=self.options,
grabbed_output=self.grabbed_output,
checkpointer=self.cp)
checkpointer=self.cp,
emissions_tracker=None)
self.assertEqual(self.controller.count, 1)

@patch.object(DummyController, "check_bounds_respected")
Expand All @@ -152,7 +152,8 @@ def test_bounds_respected_func_called(
_ = loop_over_hessians(self.controller,
options=self.options,
grabbed_output=self.grabbed_output,
checkpointer=self.cp)
checkpointer=self.cp,
emissions_tracker=None)
check_bounds_respected.assert_called()

@patch.object(DummyController, "check_bounds_respected")
Expand All @@ -169,7 +170,8 @@ def test_bounds_respected_func_not_called(
_ = loop_over_hessians(self.controller,
options=self.options,
grabbed_output=self.grabbed_output,
checkpointer=self.cp)
checkpointer=self.cp,
emissions_tracker=None)
check_bounds_respected.assert_not_called()

def test_max_runtime_exceeded(self):
Expand All @@ -191,7 +193,8 @@ def test_max_runtime_exceeded(self):
results = loop_over_hessians(controller,
options=options,
grabbed_output=grabbed_output,
checkpointer=self.cp)
checkpointer=self.cp,
emissions_tracker=None)
self.assertEqual(results[0].error_flag, 6)

@patch.object(DummyController, "eval_chisq")
Expand All @@ -207,7 +210,8 @@ def test_eval_chisq_called(self, fit, eval_chisq):
_ = loop_over_hessians(self.controller,
options=self.options,
grabbed_output=self.grabbed_output,
checkpointer=self.cp)
checkpointer=self.cp,
emissions_tracker=None)
eval_chisq.assert_called_once()

@patch.object(DummyController, "eval_confidence")
Expand All @@ -224,7 +228,8 @@ def test_eval_confidence_called(self, eval_confidence):
_ = loop_over_hessians(self.controller,
options=self.options,
grabbed_output=self.grabbed_output,
checkpointer=self.cp)
checkpointer=self.cp,
emissions_tracker=None)
eval_confidence.assert_called_once()

@run_for_test_types(TEST_TYPE, 'all')
Expand All @@ -249,7 +254,8 @@ def test_multifit_num_results(self, perform_fit):
results = loop_over_hessians(controller=controller,
options=options,
grabbed_output=grabbed_output,
checkpointer=self.cp)
checkpointer=self.cp,
emissions_tracker=None)
self.assertTrue(len(results) == 2)


Expand Down
6 changes: 4 additions & 2 deletions fitbenchmarking/core/tests/test_fitting_benchmarking_jacobians.py
View file
Expand Up @@ -120,7 +120,8 @@ def test_single_jacobian(self, loop_over_hessians):
_ = loop_over_jacobians(self.controller,
options=self.options,
grabbed_output=self.grabbed_output,
checkpointer=self.cp)
checkpointer=self.cp,
emissions_tracker=None)
loop_over_hessians.assert_called_once()

@patch(f'{FITTING_DIR}.loop_over_hessians')
Expand All @@ -135,7 +136,8 @@ def test_multiple_jacobian(self, loop_over_hessians):
_ = loop_over_jacobians(self.controller,
options=self.options,
grabbed_output=self.grabbed_output,
checkpointer=self.cp)
checkpointer=self.cp,
emissions_tracker=None)
self.assertEqual(loop_over_hessians.call_count, 2)


Expand Down
12 changes: 8 additions & 4 deletions fitbenchmarking/core/tests/test_fitting_benchmarking_minimizers.py
View file
Expand Up @@ -123,7 +123,8 @@ def test_run_minimzers_none_selected(self):
self.minimizers,
options=self.options,
grabbed_output=self.grabbed_output,
checkpointer=self.cp)
checkpointer=self.cp,
emissions_tracker=None)
assert results_problem == []
assert minimizer_failed == self.minimizers

Expand All @@ -142,7 +143,8 @@ def test_run_minimzers_selected(self, loop_over_hessians):
self.minimizers,
options=self.options,
grabbed_output=self.grabbed_output,
checkpointer=self.cp)
checkpointer=self.cp,
emissions_tracker=None)
assert all(isinstance(x, fitbm_result.FittingResult)
for x in results_problem)
assert minimizer_failed == ["deriv_free_algorithm"]
Expand All @@ -161,7 +163,8 @@ def test_run_minimzers_all(self, loop_over_hessians):
self.minimizers,
options=self.options,
grabbed_output=self.grabbed_output,
checkpointer=self.cp)
checkpointer=self.cp,
emissions_tracker=None)
assert all(isinstance(x, fitbm_result.FittingResult)
for x in results_problem)
assert minimizer_failed == []
Expand All @@ -180,7 +183,8 @@ def test_no_bounds_minimizer(self):
self.minimizers,
options=self.options,
grabbed_output=self.grabbed_output,
checkpointer=self.cp)
checkpointer=self.cp,
emissions_tracker=None)

assert results_problem[0].error_flag == 4
assert minimizer_failed == []
Expand Down
6 changes: 4 additions & 2 deletions fitbenchmarking/core/tests/test_fitting_benchmarking_software.py
View file
Expand Up @@ -105,7 +105,8 @@ def shared_test(self, expected_list_len, expected_minimizer_failed):
options=self.options,
start_values_index=self.start_values_index,
grabbed_output=self.grabbed_output,
checkpointer=self.cp)
checkpointer=self.cp,
emissions_tracker=None)
assert len(results) == expected_list_len

dict_test(unselected_minimzers, expected_minimizer_failed)
Expand Down Expand Up @@ -207,7 +208,8 @@ def test_incorrect_software(self):
options=self.options,
start_values_index=self.start_values_index,
grabbed_output=self.grabbed_output,
checkpointer=self.cp)
checkpointer=self.cp,
emissions_tracker=None)


if __name__ == "__main__":
Expand Down
3 changes: 2 additions & 1 deletion fitbenchmarking/core/tests/test_fitting_benchmarking_starting_values.py
View file
Expand Up @@ -105,7 +105,8 @@ def shared_tests(self, expected_list_len, expected_problem_fails,
= loop_over_starting_values(self.problem,
options=self.options,
grabbed_output=self.grabbed_output,
checkpointer=self.cp)
checkpointer=self.cp,
emissions_tracker=None)
assert len(problem_results) == expected_list_len
assert problem_fails == expected_problem_fails

Expand Down
2 changes: 1 addition & 1 deletion setup.py
View file
Expand Up @@ -30,7 +30,7 @@
'pandas>=1.3',
'jinja2',
'configparser',
'codecarbon<2.2.4',
'codecarbon>=2.3.4',
'dash'],
extras_require={'DFO': ['DFO-LS', 'dfogn'],
'SAS': ['sasmodels',
Expand Down

0 comments on commit d7f1515

Please sign in to comment.