Merge 9e6692f into 8ccfdab

benchmark_problems/Neutron_data/ENGINX236516_vanadium_bank1_10brk.txt

@@ -1,6 +1,6 @@
 # An example of fitting of a spline background on a Vanadium run from ENGIN-X
 name = 'ENGINX 236516 vanadium, bank 1, 10 brk'
 input_file = 'ENGINX236516_vanadium_bank1.txt'
-function = 'name=BSpline, Order=3, NBreak=10'
+function = 'name=BSpline, Order=3, NBreak=10, StartX=0.0, EndX=5.4'
 fit_parameters = {'StartX': 0, 'EndX': 5.4}
 description = ''

benchmark_problems/Neutron_data/ENGINX236516_vanadium_bank1_20brk.txt

@@ -1,6 +1,6 @@
 # An example of fitting of a spline background on a Vanadium run from ENGIN-X
 name = 'ENGINX 236516 vanadium, bank 1, 30 brk'
 input_file = 'ENGINX236516_vanadium_bank1.txt'
-function = 'name=BSpline, Order=3, NBreak=20'
+function = 'name=BSpline, Order=3, NBreak=20, StartX=0.0, EndX=5.4'
 fit_parameters = {'StartX': 0, 'EndX': 5.4}
 description = ''

example_scripts/README.md

@@ -2,6 +2,10 @@
 
 Folder that holds all the example scripts for fitBenchmarking. The user can modify these files and run a benchmarking on their machine. For more details, please read the comments in the code of each file.
 
+Here is the list of all the example scripts and the difference between them:
+1. `example_runScripts.py` is designed to be the first script to run once FitBenchmarking is properly installed. This script will benchmark Scipy minimizers against NIST-type problem definition file. User should be able to run this script without having to install any other software or packages
+2. `example_runScripts_mantid.py`is designed to be run once Mantid is installed on your computer. This script will benchmark Mantid minimizers against various different type of problem definition files.
+3. `example_runScripts_expert.py` is designed to be run once Mantid and SasView (future development) are installed. This script will benchmark all the minimizers from different softwares against various type of problem definition files.
 ### Example usage
 
 For default minimizers use (which loads minimizers_list_defaults.json):

example_scripts/example_runScripts.py

@@ -1,31 +1,13 @@
 """
-Script that runs the fitbenchmarking tool with various problems and minimizers.
+This example script is designed to demonstrate the features of fitbenchmarking to benchmark
+the performance of Scipy minimizers against NIST-type problem definition files.
 """
 
-# Copyright © 2016 ISIS Rutherford Appleton Laboratory, NScD
-# Oak Ridge National Laboratory & European Spallation Source
-#
-# This file is part of Mantid.
-# Mantid is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# Mantid is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# File change history is stored at: <https://github.com/mantidproject/mantid>.
-# Code Documentation is available at: <http://doxygen.mantidproject.org>
-
 
 from __future__ import (absolute_import, division, print_function)
 import os
 import sys
+import glob
 
 
 # Avoid reaching the maximum recursion depth by setting recursion limit
@@ -97,22 +79,28 @@
 # problem_sets = ["CUTEst", "NIST/average_difficulty", "NIST/high_difficulty", "NIST/low_difficulty"]
 problem_sets = ['CUTEst']
 for sub_dir in problem_sets:
-  # generate group label/name used for problem set
-  label = sub_dir.replace('/', '_')
+    # generate group label/name used for problem set
+    label = sub_dir.replace('/', '_')
+
+    # Problem data directory
+    data_dir = os.path.join(benchmark_probs_dir, sub_dir)
+
+    test_data = glob.glob(data_dir + '/*.*')
 
-  # Problem data directory
-  data_dir = os.path.join(benchmark_probs_dir, sub_dir)
+    if test_data == []:
+        print('Problem set {} not found'.format(sub_dir))
+        continue
 
-  print('\nRunning the benchmarking on the {} problem set\n'.format(label))
-  results_per_group, results_dir = fitBenchmarking(group_name=label, software_options=software_options,
+    print('\nRunning the benchmarking on the {} problem set\n'.format(label))
+    results_per_group, results_dir = fitBenchmarking(group_name=label, software_options=software_options,
                                                    data_dir=data_dir,
                                                    use_errors=use_errors, results_dir=results_dir)
 
-  print('\nProducing output for the {} problem set\n'.format(label))
-  for idx, group_results in enumerate(results_per_group):
-    # Display the runtime and accuracy results in a table
-    printTables(software_options, group_results,
+    print('\nProducing output for the {} problem set\n'.format(label))
+    for idx, group_results in enumerate(results_per_group):
+        # Display the runtime and accuracy results in a table
+        printTables(software_options, group_results,
                 group_name=label, use_errors=use_errors,
                 color_scale=color_scale, results_dir=results_dir)
 
-  print('\nCompleted benchmarking for {} problem set\n'.format(sub_dir))
+    print('\nCompleted benchmarking for {} problem set\n'.format(sub_dir))

example_scripts/example_runScripts_expert.py

@@ -4,30 +4,11 @@
 at each stage to enable a user to customize their problem to their needs.
 """
 
-# Copyright © 2016 ISIS Rutherford Appleton Laboratory, NScD
-# Oak Ridge National Laboratory & European Spallation Source
-#
-# This file is part of Mantid.
-# Mantid is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# Mantid is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# File change history is stored at: <https://github.com/mantidproject/mantid>.
-# Code Documentation is available at: <http://doxygen.mantidproject.org>
-
 
 from __future__ import (absolute_import, division, print_function)
 import os
 import sys
+import glob
 
 
 # Avoid reaching the maximum recursion depth by setting recursion limit
@@ -107,6 +88,12 @@
     # Problem data directory
     data_dir = os.path.join(benchmark_probs_dir, sub_dir)
 
+    test_data = glob.glob(data_dir + '/*.*')
+
+    if test_data == []:
+        print('Problem set {} not found'.format(sub_dir))
+        continue
+
     print('\nRunning the benchmarking on the {} problem set\n'.format(group_name))
 
     # Processes software_options dictionary into Fitbenchmarking format

example_scripts/example_runScripts_mantid.py

@@ -1,31 +1,15 @@
 """
-Script that runs the fitbenchmarking tool with various problems and minimizers.
-"""
-
-# Copyright © 2016 ISIS Rutherford Appleton Laboratory, NScD
-# Oak Ridge National Laboratory & European Spallation Source
-#
-# This file is part of Mantid.
-# Mantid is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# Mantid is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# File change history is stored at: <https://github.com/mantidproject/mantid>.
-# Code Documentation is available at: <http://doxygen.mantidproject.org>
+This example script is designed to demonstrate the features of fitbenchmarking to benchmark
+the performance of Mantid minimizers against various different problem definition files.
 
+This example script can also be modified to benchmark against Scipy minimizers as well.
+To do that, simply change the variable "software" from "mantid" to "scipy".
+"""
 
 from __future__ import (absolute_import, division, print_function)
 import os
 import sys
+import glob
 
 # Avoid reaching the maximum recursion depth by setting recursion limit
 # This is useful when running multiple data set benchmarking
@@ -39,9 +23,6 @@
 scripts_folder = os.path.join(fitbenchmarking_folder, 'fitbenchmarking')
 sys.path.insert(0, scripts_folder)
 
-from fitting_benchmarking import do_fitting_benchmark as fitBenchmarking
-from results_output import save_results_tables as printTables
-
 try:
     import mantid.simpleapi as msapi
 except:
@@ -52,6 +33,9 @@
           '******************************************')
     sys.exit()
 
+from fitting_benchmarking import do_fitting_benchmark as fitBenchmarking
+from results_output import save_results_tables as printTables
+
 # SPECIFY THE SOFTWARE/PACKAGE CONTAINING THE MINIMIZERS YOU WANT TO BENCHMARK
 software = 'mantid'
 software_options = {'software': software}
@@ -110,6 +94,12 @@
   # Problem data directory
   data_dir = os.path.join(benchmark_probs_dir, sub_dir)
 
+  test_data = glob.glob(data_dir + '/*.*')
+
+  if test_data == []:
+      print('Problem set {} not found'.format(sub_dir))
+      continue
+
   print('\nRunning the benchmarking on the {} problem set\n'.format(label))
   results_per_group, results_dir = fitBenchmarking(group_name=label, software_options=software_options,
                                                    data_dir=data_dir,

fitbenchmarking/fitting/mantid/externals.py

@@ -29,15 +29,17 @@
 from utils.logging_setup import logger
 
 
-def gen_func_obj(function_name):
+def gen_func_obj(function_name, params_set):
     """
     Generates a mantid function object.
 
     @param function_name :: the name of the function to be generated
+    @params_set :: set of parameters per function extracted from the problem definition file
 
     @returns :: mantid function object that can be called in python
     """
-    exec "function_object = msapi." + function_name + "()"
+    params_set = (params_set.split(', ties'))[0]
+    exec "function_object = msapi." + function_name + "("+ params_set +")"
     return function_object
 
 

fitbenchmarking/fitting/scipy/fitbenchmark_data_functions.py

@@ -46,8 +46,8 @@ def fitbenchmark_func_definitions(functions_string):
     function_params = get_all_fitbenchmark_func_params(functions_string)
     params, ties = get_fitbenchmark_initial_params_values(function_params)
     fit_function = None
-    for name in function_names:
-        fit_function = make_fitbenchmark_fit_function(name, fit_function)
+    for name, params_set in zip(function_names, function_params):
+        fit_function = make_fitbenchmark_fit_function(name, fit_function, params_set)
     fit_function = set_ties(fit_function, ties)
 
 
@@ -128,11 +128,11 @@ def get_fitbenchmark_initial_params_values(function_params):
     return params, ties
 
 
-def make_fitbenchmark_fit_function(func_name, fit_function):
+def make_fitbenchmark_fit_function(func_name, fit_function, params_set):
     """
     Create the fitbenchmark fit function object that is used by scipy.
     """
-    func_obj = gen_func_obj(func_name)
+    func_obj = gen_func_obj(func_name, params_set)
     if fit_function == None:
         fit_function = func_obj
     else:

fitbenchmarking/resproc/rst_table.py

@@ -92,7 +92,11 @@ def create_table_body(cells, items_link, rows_txt, first_col_len, cell_len,
     tbl_body = ''
     for row in range(0, cells.shape[0]):
         link = items_link
-        tbl_body += '|' + rows_txt[row].ljust(first_col_len, ' ') + '|'
+        all_fit_failed_status = ''
+        if np.isnan(cells[row, :]).all():
+            all_fit_failed_status = '(all fit failed)'
+        tbl_body += '|' + rows_txt[row].ljust(first_col_len-len(all_fit_failed_status), ' ')\
+                    + all_fit_failed_status +'|'
         for col in range(0, cells.shape[1]):
             tbl_body += format_cell_value(cells[row, col], cell_len,
                                           color_scale, link)
@@ -162,6 +166,8 @@ def calc_first_col_len(cell_len, rows_txt):
         if name_len > first_col_len:
             first_col_len = name_len
 
+    first_col_len += 20
+
     return first_col_len
 
 

fitbenchmarking/resproc/tests/test_rst_table.py

@@ -36,8 +36,8 @@ def PrepareTableHeader(self):
     linked_problem = "`Misra1a 1 <file:///" + current_dir + \
                      "/dump/nist/VDPages/nist_lower_misra1a.html>`__"
     length_table = len(linked_problem)
-    tbl_header_top = ("+" + "-" * length_table + "+" + ("-" * 21 + "+") * 10)
-    tbl_header_text = ("|" + " " * length_table + "|" + "Minimizer1" + " " * 11 + "|" +
+    tbl_header_top = ("+" + "-" * (length_table+20) + "+" + ("-" * 21 + "+") * 10)
+    tbl_header_text = ("|" + " " * (length_table+20) + "|" + "Minimizer1" + " " * 11 + "|" +
                        "Minimizer2" + 11 * " " + "|" + "Minimizer3" + " " * 11 +
                        "|" +
                        "Minimizer4" + 11 * " " + "|" + "Minimizer5" + " " * 11 +
@@ -47,7 +47,7 @@ def PrepareTableHeader(self):
                        "Minimizer8" + 11 * " " + "|" + "Minimizer9" + " " * 11 +
                        "|" +
                        "Trust Region" + " " * 9 + "|")
-    tbl_header_bottom = ("+" + "=" * length_table + "+" + ("=" * 21 + "+") * 10)
+    tbl_header_bottom = ("+" + "=" * (length_table+20) + "+" + ("=" * 21 + "+") * 10)
 
     return tbl_header_top, tbl_header_text, tbl_header_bottom
 
@@ -58,11 +58,11 @@ def GenerateTableBody(self):
     tbl_footer = tbl_htop + '\n'
     tbl_body = '|`Misra1a 1 <file:///' + current_dir + \
                '/dump/nist/VDPages/' + \
-               'nist_lower_misra1a.html>`__| :ranking-top-1:`1`  | ' + \
+               'nist_lower_misra1a.html>`__'+' '*20 +'| :ranking-top-1:`1`  | ' + \
                ':ranking-low-4:`2`  | :ranking-low-4:`3`  |\n' + \
                tbl_footer + '|`Misra1a 2 <file:///' + current_dir + \
                '/dump/' + \
-               'nist/VDPages/nist_lower_misra1a.html>`__| ' + \
+               'nist/VDPages/nist_lower_misra1a.html>`__'+' '*20 + '| ' + \
                ':ranking-low-5:`5`  | :ranking-low-5:`10` |' + \
                ' :ranking-low-5:`13` |\n'
 
@@ -128,7 +128,7 @@ def test_createTableBody_produces_right_body(self):
 
     columns_txt, rows_txt, cells, color_scale = self.CreateTableInputData()
     items_link = 'FittingMinimizersComparisonDetailedWithWeights'
-    first_col_len = len(rows_txt[0])
+    first_col_len = len(rows_txt[0])+20
     tbl_header_top, tbl_header_text, tbl_header_bottom = \
         self.PrepareTableHeader()
     tbl_footer = tbl_header_top + '\n'
@@ -220,7 +220,7 @@ def test_calcFirstColLen_for_NIST_problem_Misra1a_and_Lanczos3(self):
                 "/dump/nist/VDPages/nist_lower_misra1a.html>`__"]
 
     first_col_len = calc_first_col_len(cell_len, rows_txt)
-    first_col_len_expected = len(rows_txt[0])
+    first_col_len_expected = len(rows_txt[0]) + 20
 
     self.assertEqual(first_col_len_expected, first_col_len)
 
@@ -231,7 +231,7 @@ def test_buildHeaderChunks_return_header_chucks_Misra1a_prob_results(self):
                 "/dump/nist/VDPages/nist_lower_misra1a.html>`__",
                 "`Misra1a 2 <file:///" + current_dir +
                 "/dump/nist/VDPages/nist_lower_misra1a.html>`__"]
-    first_col_len = len(rows_txt[0])
+    first_col_len = len(rows_txt[0]) + 20
     cell_len = 21
     columns_txt = ['Minimizer1', 'Minimizer2', 'Minimizer3', 'Minimizer4',
                    'Minimizer5', 'Minimizer6', 'Minimizer7', 'Minimizer8',

fitbenchmarking/utils/misc.py

@@ -107,7 +107,7 @@ def save_user_input(software, minimizers, group_name, results_dir, use_errors):
 
 def get_problem_files(data_dir):
     """
-    Gets all the problem definition files from the neutron directory.
+    Gets all the problem definition files from the specified problem set directory.
 
     @param dirs :: array of directories that contain the problems