257 Adding averaging time using timeit

docs/source/users/options.rst

@@ -159,3 +159,11 @@ Available options are ``"abs"``, ``"rel"``, or ``"both"``.
   Return both absolute and relative values.
   Values will be shown as an absolute value followed by a relative value in
   parentheses.
+
+
+``num_runs``
+-------------------
+
+Number of runs is defines how many times FitBenchmarking calls a minimizer and thus calculates an average elapsed time using ``timeit``.
+
+Default set as ``5``.

example_scripts/example_runScripts_expert.py

@@ -11,8 +11,7 @@
 import glob
 
 from fitbenchmarking.fitting_benchmarking import _benchmark
-from fitbenchmarking.utils import misc
-from fitbenchmarking.utils import create_dirs
+from fitbenchmarking.utils import create_dirs, misc, options
 from fitbenchmarking.results_output import save_tables, generate_tables, \
     create_acc_tbl, create_runtime_tbl
 from fitbenchmarking.resproc import visual_pages
@@ -89,7 +88,15 @@ def main(argv):
 
         # Processes software_options dictionary into Fitbenchmarking format
         minimizers, software = misc.get_minimizers(software_options)
-
+        num_runs = software_options.get('num_runs', None)
+
+        if num_runs is None:
+            if 'num_runs' in software_options:
+                options_file = software_options['num_runs']
+                num_runs = options.get_option(options_file=options_file,
+                                              option='num_runs')
+            else:
+                num_runs = options.get_option(option='num_runs')
         # Sets up the problem group specified by the user by providing
         # a respective data directory.
         problem_group = misc.get_problem_files(data_dir)
@@ -103,7 +110,7 @@ def main(argv):
                                           group_results_dir, use_errors)
 
         # Loops through group of problems and benchmark them
-        prob_results = _benchmark(user_input, problem_group)
+        prob_results = _benchmark(user_input, problem_group, num_runs)
 
         print('\nProducing output for the {} problem set\n'.format(group_name))
 

fitbenchmarking/fitbenchmark_one_problem.py

@@ -4,8 +4,8 @@
 
 from __future__ import absolute_import, division, print_function
 
-import time
-
+import timeit
+import warnings
 import numpy as np
 
 from fitbenchmarking.fitting import misc
@@ -32,17 +32,23 @@
     ScipyController = None
 
 
-def fitbm_one_prob(user_input, problem):
+def fitbm_one_prob(user_input, problem, num_runs):
     """
     Sets up the controller for a particular problem and fits the models
     provided in the problem object. The best fit, along with the data and a
     starting guess is then plotted on a visual display page.
 
-    @param user_input :: all the information specified by the user
-    @param problem :: a problem object containing information used in fitting
-
-    @returns :: nested array of result objects, per function definition
-                containing the fit information
+    :param user_input :: all the information specified by the user
+    :type user_input :: UserInput
+    :param problem :: a problem object containing information used in fitting
+    :type problem :: FittingProblem
+    :param num_runs :: number of times controller.fit() is run to
+                       generate an average runtime
+    :type num_runs :: int
+
+    :return :: nested array of result objects, per function definition
+               containing the fit information
+    :rtype :: list
     """
 
     results_fit_problem = []
@@ -58,7 +64,8 @@ def fitbm_one_prob(user_input, problem):
     if software in controllers:
         controller = controllers[software](problem, user_input.use_errors)
     else:
-        raise NotImplementedError('The chosen software is not implemented yet: {}'.format(user_input.software))
+        raise NotImplementedError('The chosen software is not implemented yet:'
+                                  '{}'.format(user_input.software))
 
     # The controller reformats the data to fit within a start- and end-x bound
     # It also estimates errors if not provided.
@@ -71,7 +78,8 @@ def fitbm_one_prob(user_input, problem):
         controller.function_id = i
 
         results_problem, best_fit = benchmark(controller=controller,
-                                              minimizers=user_input.minimizers)
+                                              minimizers=user_input.minimizers,
+                                              num_runs=num_runs)
 
         if best_fit is not None:
             # Make the plot of the best fit
@@ -85,47 +93,65 @@ def fitbm_one_prob(user_input, problem):
     return results_fit_problem
 
 
-def benchmark(controller, minimizers):
+def benchmark(controller, minimizers, num_runs):
     """
     Fit benchmark one problem, with one function definition and all
     the selected minimizers, using the chosen fitting software.
 
-    @param controller :: The software controller for the fitting
+    :param controller :: The software controller for the fitting
+    :type controller :: Object derived from BaseSoftwareController
     @param minimizers :: array of minimizers used in fitting
+    :type minimizers :: list
+    @param num_runs :: number of times controller.fit() is run to
+                       generate an average runtime
+    :type num_runs :: int
+
 
-    @returns :: nested array of result objects, per minimizer
-                and data object for the best fit data
+    :return :: tuple(results_problem, best_fit) nested array of
+               result objects, per minimizer and data object for
+               the best fit data
+    :rtype :: (list of FittingResult, plot_helper.data instance)
     """
     min_chi_sq, best_fit = None, None
     results_problem = []
 
     for minimizer in minimizers:
         controller.minimizer = minimizer
 
-        controller.prepare()
-
-        init_function_def = controller.problem.get_function_def(params=controller.initial_params,
-                                                                function_id=controller.function_id)
+        init_function_def = controller.problem.get_function_def(
+            params=controller.initial_params,
+            function_id=controller.function_id)
         try:
-            start_time = time.time()
-            controller.fit()
-            end_time = time.time()
-        except Exception as e:
-            print(e.message)
+            # Calls timeit repeat with repeat = num_runs and number = 1
+            runtime_list = \
+                timeit.Timer(setup=controller.prepare,
+                             stmt=controller.fit).repeat(num_runs, 1)
+            runtime = sum(runtime_list) / num_runs
+
+        # Catching all exceptions as this means runtime cannot be calculated
+        # pylint: disable=broad-except
+        except Exception as excp:
+            print(str(excp))
             controller.success = False
-            end_time = np.inf
-
-        runtime = end_time - start_time
+            runtime = np.inf
 
         controller.cleanup()
 
-        fin_function_def = controller.problem.get_function_def(params=controller.final_params,
-                                                               function_id=controller.function_id)
+        fin_function_def = controller.problem.get_function_def(
+            params=controller.final_params,
+            function_id=controller.function_id)
 
         if not controller.success:
             chi_sq = np.nan
             status = 'failed'
         else:
+            ratio = np.max(runtime_list) / np.min(runtime_list)
+            tol = 4
+            if ratio > tol:
+                warnings.warn('The ratio of the max time to the min is {0}'
+                              ' which is  larger than the tolerance of {1},'
+                              ' which may indicate that caching has occurred'
+                              ' in the timing results'.format(ratio, tol))
             chi_sq = misc.compute_chisq(fitted=controller.results,
                                         actual=controller.data_y,
                                         errors=controller.data_e)

fitbenchmarking/fitbenchmarking_default_options.json

@@ -12,5 +12,7 @@
         "ralfit" : ["ralfit"]
         },
 
-    "comparison_mode" : "both"
+    "comparison_mode" : "both",
+
+    "num_runs" : 5
 }

fitbenchmarking/fitting_benchmarking.py

@@ -11,7 +11,7 @@
 from fitbenchmarking.utils.logging_setup import logger
 
 from fitbenchmarking.parsing import parse
-from fitbenchmarking.utils import create_dirs, misc
+from fitbenchmarking.utils import create_dirs, misc, options
 from fitbenchmarking.fitbenchmark_one_problem import fitbm_one_prob
 
 
@@ -20,22 +20,42 @@ def fitbenchmark_group(group_name, software_options, data_dir,
     """
     Gather the user input and list of paths. Call benchmarking on these.
 
-    @param group_name :: is the name (label) for a group. E.g. the name for the group of problems in
-                         "NIST/low_difficulty" may be picked to be NIST_low_difficulty
-    @param software_options :: dictionary containing software used in fitting the problem, list of minimizers and
-                               location of json file contain minimizers
-    @param data_dir :: full path of a directory that holds a group of problem definition files
-    @param use_errors :: whether to use errors on the data or not
-    @param results_dir :: directory in which to put the results. None
-                          means results directory is created for you
-
-    @returns :: array of fitting results for the problem group and
-                the path to the results directory
+    :param group_name :: is the name (label) for a group. E.g. the name for the
+                         group of problems in "NIST/low_difficulty" may be
+                         picked to be NIST_low_difficulty
+    :type group_name :: str
+    :param software_options :: dictionary containing software used in fitting
+                               the problem, list of minimizers and location of
+                               json file contain minimizers
+    :type software_options :: dict
+    :param data_dir :: full path of a directory that holds a group of problem
+                       definition files
+    :type date_dir :: str
+    :param use_errors :: whether to use errors on the data or not
+    :type use_errors :: bool
+    :param results_dir :: directory in which to put the results. None means
+                          results directory is created for you
+    :type results_dir :: str/NoneType
+
+    :return :: tuple(prob_results, results_dir) array of fitting results for
+                the problem group and the path to the results directory
+    :rtype :: (list of FittingResult, str)
     """
 
-    logger.info("Loading minimizers from {0}".format(
-        software_options['software']))
+    logger.info("Loading minimizers from %s", software_options['software'])
     minimizers, software = misc.get_minimizers(software_options)
+    num_runs = software_options.get('num_runs', None)
+
+    if num_runs is None:
+        if 'num_runs' in software_options:
+            options_file = software_options['options_file']
+            num_runs = options.get_option(options_file=options_file,
+                                          option='num_runs')
+        else:
+            num_runs = options.get_option(option='num_runs')
+
+        if num_runs is None:
+            num_runs = 5
 
     # create list of paths to all problem definitions in data_dir
     problem_group = misc.get_problem_files(data_dir)
@@ -46,34 +66,43 @@ def fitbenchmark_group(group_name, software_options, data_dir,
     user_input = misc.save_user_input(software, minimizers, group_name,
                                       group_results_dir, use_errors)
 
-    prob_results = _benchmark(user_input, problem_group)
+    prob_results = _benchmark(user_input, problem_group, num_runs)
 
     return prob_results, results_dir
 
 
-def _benchmark(user_input, problem_group):
+def _benchmark(user_input, problem_group, num_runs):
     """
     Loops through software and benchmarks each problem within the problem
     group.
 
-    @param user_input :: all the information specified by the user
-    @param problem_group :: list of paths to problem files in the group
-                            e.g. ['NIST/low_difficulty/file1.dat',
-                                  'NIST/low_difficulty/file2.dat',
-                                  ...]
+    :param user_input :: all the information specified by the user
+    :type user_input :: UserInput
+    :param problem_group :: list of paths to problem files in the group
+                          e.g. ['NIST/low_difficulty/file1.dat',
+                                'NIST/low_difficulty/file2.dat',
+                                ...]
+    :type problem_group :: list
+    :param num_runs :: number of times controller.fit() is run to
+                     generate an average runtime
+    :type num_runs :: str
+
+
+    :return :: array of result objects, per problem per user_input
+    :rtype :: list of FittingResult
 
-    @returns :: array of result objects, per problem per user_input
     """
 
     parsed_problems = [parse.parse_problem_file(p) for p in problem_group]
 
     if not isinstance(user_input, list):
         list_prob_results = [per_func
                              for p in parsed_problems
-                             for per_func in fitbm_one_prob(user_input, p)]
+                             for per_func in fitbm_one_prob(user_input,
+                                                            p, num_runs)]
 
     else:
-        list_prob_results = [[fitbm_one_prob(u, p)
+        list_prob_results = [[fitbm_one_prob(u, p, num_runs)
                               for u in user_input]
                              for p in parsed_problems]
 
@@ -84,9 +113,9 @@ def _benchmark(user_input, problem_group):
         list_prob_results = \
             [[list_prob_results[prob_idx][input_idx][func_idx][minim_idx]
               for input_idx in range(len(user_input))
-              for minim_idx in range(len(list_prob_results[prob_idx][input_idx][func_idx]))]
+              for minim_idx
+              in range(len(list_prob_results[prob_idx][input_idx][func_idx]))]
              for prob_idx in range(len(parsed_problems))
              for func_idx in range(len(list_prob_results[prob_idx][0]))]
 
     return list_prob_results
-