257 Adding averaging time using timeit

docs/source/users/options.rst

@@ -118,7 +118,7 @@ Scipy:
   - ``"dogbox"``
   - ``"lm"``
   - and ``"trf"``
- 
+
 
   Information about these can be found on the
   `Scipy documentation
@@ -128,9 +128,9 @@ DFO-GN:
   - ``"dfogn"``
   Information about this can be found on the
   `DFO-GN documentation
-  <http://people.maths.ox.ac.uk/robertsl/dfogn/>`__  
-  
-  
+  <http://people.maths.ox.ac.uk/robertsl/dfogn/>`__
+
+
 ``comparison_mode``
 -------------------
 The comparison mode is used when displaying results to select the value
@@ -152,3 +152,11 @@ Available options are ``"abs"``, ``"rel"``, or ``"both"``.
   Return both absolute and relative values.
   Values will be shown as an absolute value followed by a relative value in
   parentheses.
+
+
+``num_runs``
+-------------------
+
+Number of runs is used to define how many times FitBenchmarking calls a minimizer and thus calculates an average elapsed time using `timeit`.
+
+Default set as `5`.

fitbenchmarking/fitbenchmark_one_problem.py

@@ -4,7 +4,7 @@
 
 from __future__ import absolute_import, division, print_function
 
-import time
+import timeit
 
 import numpy as np
 
@@ -28,14 +28,16 @@
     ScipyController = None
 
 
-def fitbm_one_prob(user_input, problem):
+def fitbm_one_prob(user_input, problem, num_runs):
     """
     Sets up the controller for a particular problem and fits the models
     provided in the problem object. The best fit, along with the data and a
     starting guess is then plotted on a visual display page.
 
     @param user_input :: all the information specified by the user
     @param problem :: a problem object containing information used in fitting
+    @param num_runs :: number of times controller.fit() is run to
+                       generate an average runtime
 
     @returns :: nested array of result objects, per function definition
                 containing the fit information
@@ -66,7 +68,8 @@ def fitbm_one_prob(user_input, problem):
         controller.function_id = i
 
         results_problem, best_fit = benchmark(controller=controller,
-                                              minimizers=user_input.minimizers)
+                                              minimizers=user_input.minimizers,
+                                              num_runs=num_runs)
 
         if best_fit is not None:
             # Make the plot of the best fit
@@ -80,13 +83,15 @@ def fitbm_one_prob(user_input, problem):
     return results_fit_problem
 
 
-def benchmark(controller, minimizers):
+def benchmark(controller, minimizers, num_runs):
     """
     Fit benchmark one problem, with one function definition and all
     the selected minimizers, using the chosen fitting software.
 
     @param controller :: The software controller for the fitting
     @param minimizers :: array of minimizers used in fitting
+    @param num_runs :: number of times controller.fit() is run to
+                       generate an average runtime
 
     @returns :: nested array of result objects, per minimizer
                 and data object for the best fit data
@@ -102,15 +107,11 @@ def benchmark(controller, minimizers):
         init_function_def = controller.problem.get_function_def(params=controller.initial_params,
                                                                 function_id=controller.function_id)
         try:
-            start_time = time.time()
-            controller.fit()
-            end_time = time.time()
+            runtime = timeit.timeit(controller.fit, number=num_runs) / num_runs
         except Exception as e:
             print(e.message)
             controller.success = False
-            end_time = np.inf
-
-        runtime = end_time - start_time
+            runtime = np.inf
 
         controller.cleanup()
 

fitbenchmarking/fitbenchmarking_default_options.json

@@ -11,5 +11,7 @@
         "dfogn" : ["dfogn"]
         },
 
-    "comparison_mode" : "both" 
+    "comparison_mode" : "both",
+
+    "num_runs" : 5
 }

fitbenchmarking/fitting_benchmarking.py

@@ -11,82 +11,97 @@
 from fitbenchmarking.utils.logging_setup import logger
 
 from fitbenchmarking.parsing import parse
-from fitbenchmarking.utils import create_dirs, misc
+from fitbenchmarking.utils import create_dirs, misc, options
 from fitbenchmarking.fitbenchmark_one_problem import fitbm_one_prob
 
 
 def fitbenchmark_group(group_name, software_options, data_dir,
                        use_errors=True, results_dir=None):
-    """
-    Gather the user input and list of paths. Call benchmarking on these.
+  """
+  Gather the user input and list of paths. Call benchmarking on these.
+
+  @param group_name :: is the name (label) for a group. E.g. the name for the group of problems in
+                       "NIST/low_difficulty" may be picked to be NIST_low_difficulty
+  @param software_options :: dictionary containing software used in fitting the problem, list of minimizers and
+                             location of json file contain minimizers
+  @param data_dir :: full path of a directory that holds a group of problem definition files
+  @param use_errors :: whether to use errors on the data or not
+  @param results_dir :: directory in which to put the results. None
+                        means results directory is created for you
+
+  @returns :: array of fitting results for the problem group and
+              the path to the results directory
+  """
+
+  logger.info("Loading minimizers from {0}".format(
+      software_options['software']))
+  minimizers, software = misc.get_minimizers(software_options)
+  num_runs = software_options.get('num_runs', None)
+
+  if num_runs is None:
+    if 'num_runs' in software_options:
+      options_file = software_options['num_runs']
+      num_runs = options.get_option(options_file=options_file,
+                                    option='num_runs')
+    else:
+      num_runs = options.get_option(option='num_runs')
 
-    @param group_name :: is the name (label) for a group. E.g. the name for the group of problems in
-                         "NIST/low_difficulty" may be picked to be NIST_low_difficulty
-    @param software_options :: dictionary containing software used in fitting the problem, list of minimizers and
-                               location of json file contain minimizers
-    @param data_dir :: full path of a directory that holds a group of problem definition files
-    @param use_errors :: whether to use errors on the data or not
-    @param results_dir :: directory in which to put the results. None
-                          means results directory is created for you
+    if num_runs is None:
+      num_runs = 5
 
-    @returns :: array of fitting results for the problem group and
-                the path to the results directory
-    """
+  # create list of paths to all problem definitions in data_dir
+  problem_group = misc.get_problem_files(data_dir)
 
-    logger.info("Loading minimizers from {0}".format(
-        software_options['software']))
-    minimizers, software = misc.get_minimizers(software_options)
+  results_dir = create_dirs.results(results_dir)
+  group_results_dir = create_dirs.group_results(results_dir, group_name)
 
-    # create list of paths to all problem definitions in data_dir
-    problem_group = misc.get_problem_files(data_dir)
+  user_input = misc.save_user_input(software, minimizers, group_name,
+                                    group_results_dir, use_errors)
 
-    results_dir = create_dirs.results(results_dir)
-    group_results_dir = create_dirs.group_results(results_dir, group_name)
+  prob_results = _benchmark(user_input, problem_group, num_runs)
 
-    user_input = misc.save_user_input(software, minimizers, group_name,
-                                      group_results_dir, use_errors)
+  return prob_results, results_dir
 
-    prob_results = _benchmark(user_input, problem_group)
 
-    return prob_results, results_dir
+def _benchmark(user_input, problem_group, num_runs=5):
+  """
+  Loops through software and benchmarks each problem within the problem
+  group.
 
+  @param user_input :: all the information specified by the user
+  @param problem_group :: list of paths to problem files in the group
+                          e.g. ['NIST/low_difficulty/file1.dat',
+                                'NIST/low_difficulty/file2.dat',
+                                ...]
+  @param num_runs :: number of times controller.fit() is run to
+                     generate an average runtime
 
-def _benchmark(user_input, problem_group):
-    """
-    Loops through software and benchmarks each problem within the problem
-    group.
 
-    @param user_input :: all the information specified by the user
-    @param problem_group :: list of paths to problem files in the group
-                            e.g. ['NIST/low_difficulty/file1.dat',
-                                  'NIST/low_difficulty/file2.dat',
-                                  ...]
+  @returns :: array of result objects, per problem per user_input
+  """
 
-    @returns :: array of result objects, per problem per user_input
-    """
+  parsed_problems = [parse.parse_problem_file(p) for p in problem_group]
 
-    parsed_problems = [parse.parse_problem_file(p) for p in problem_group]
+  if not isinstance(user_input, list):
+    list_prob_results = [per_func
+                         for p in parsed_problems
+                         for per_func in fitbm_one_prob(user_input,
+                                                        p, num_runs)]
 
-    if not isinstance(user_input, list):
-        list_prob_results = [per_func
-                             for p in parsed_problems
-                             for per_func in fitbm_one_prob(user_input, p)]
+  else:
+    list_prob_results = [[fitbm_one_prob(u, p, num_runs)
+                          for u in user_input]
+                         for p in parsed_problems]
 
-    else:
-        list_prob_results = [[fitbm_one_prob(u, p)
-                              for u in user_input]
-                             for p in parsed_problems]
-
-        # reorganise loop structure from:
-        # [[[val per minimizer] per function] per input] per problem]
-        # to:
-        # [[val per input per minimizer] per function per problem]
-        list_prob_results = \
-            [[list_prob_results[prob_idx][input_idx][func_idx][minim_idx]
-              for input_idx in range(len(user_input))
-              for minim_idx in range(len(list_prob_results[prob_idx][input_idx][func_idx]))]
-             for prob_idx in range(len(parsed_problems))
-             for func_idx in range(len(list_prob_results[prob_idx][0]))]
-
-    return list_prob_results
+    # reorganise loop structure from:
+    # [[[val per minimizer] per function] per input] per problem]
+    # to:
+    # [[val per input per minimizer] per function per problem]
+    list_prob_results = \
+        [[list_prob_results[prob_idx][input_idx][func_idx][minim_idx]
+          for input_idx in range(len(user_input))
+          for minim_idx in range(len(list_prob_results[prob_idx][input_idx][func_idx]))]
+         for prob_idx in range(len(parsed_problems))
+         for func_idx in range(len(list_prob_results[prob_idx][0]))]
 
+  return list_prob_results