Merge branch 'development' of ssh://github.com:mfeurer/automl/HPOlib …

…into development

HPOlib/Plotting/getTopK.py

@@ -41,14 +41,17 @@
                         default=False, help="Plot worst k configs")
     parser.add_argument("-a", dest="additional", action="store_true",
                         default=False, help="Show additional info")
+    parser.add_argument("--csv", dest="csv", action="store_true",
+                        default=False, help="Print info as csv")
 
     args, unknown = parser.parse_known_args()
 
     sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n")
 
     pkl_list = list()
     result_dict = dict()
-    print unknown
+    if not args.csv:
+        print unknown
     for pkl in unknown:
         if not os.path.exists(pkl):
             print "%s does not exist" % pkl
@@ -77,11 +80,31 @@
     while len(topK) < args.k:
         topK.append(result_dict[results[ct]])
         ct += 1
-    print "Found %d different results" % len(result_dict)
+    if not args.csv:
+        print "Found %d different results" % len(result_dict)
+
+    header = False
     for k in topK:
-        print "Result = %10f, Time = %10f, " % (k[0]['result'], k[0]['duration']),\
-            ", ".join(["%s = %3s" % (key.strip('-'), k[0]['params'][key]) for key in k[0]['params']]),
-        if args.additional:
-            print ", additional_info = %s" % str(k[0]['additional_data'])
+        if args.csv:
+            if not header:
+                header = True
+                print ",".join(["Result", "Time"]),
+                print "," + ",".join([key.strip('-') for key in k[0]['params']]),
+                if args.additional:
+                    print ",additional_info"
+                else:
+                    print
+            print ",".join(["%10.5f" % k[0]['result'], "%10.5f" % k[0]['duration']]),
+            print "," + ",".join([k[0]['params'][key] for key in k[0]['params']]),
+            if args.additional:
+                print ", additional_info = %s" % str(k[0]['additional_data'])
+            else:
+                print
+
         else:
-            print
+            print "Result = %10f, Time = %10f, " % (k[0]['result'], k[0]['duration']),\
+                ", ".join(["%s = %3s" % (key.strip('-'), k[0]['params'][key]) for key in k[0]['params']]),
+            if args.additional:
+                print ", additional_info = %s" % str(k[0]['additional_data'])
+            else:
+                print

HPOlib/Plotting/plotParam.py

@@ -270,6 +270,8 @@ def main(pkl_list, name_list, param=None, min_=0.0, max_=0.0,
                             key = len(string_to_value_map.keys()) + 1
                             string_to_value_map[value] = key
                             value = key
+                    if np.isnan(t["result"]):
+                        continue
                     value_list.append(value)
                     result_list.append(t["result"])
                     param_set.add(k)

HPOlib/Plotting/plot_util.py

@@ -224,7 +224,10 @@ def get_best_value_and_index(trials, cut=sys.maxint):
     """Return the best value found and its index in experiment.
 
     If `cut` is given, look at the first `cut` results. Raise ValueError if cut
-    is equal or less than zero. Important: The index is zero-based!"""
+    is equal or less than zero. Important: The index is zero-based!
+
+    if test is given, look for the best trial and report testperformance
+    """
     if not isinstance(cut, int):
         raise ValueError("Argument cut must be an Integer value but is %s" %
             type(cut))

HPOlib/Plotting/results.py

@@ -85,7 +85,7 @@ def collect_results(directory):
                     cfg = ConfigParser.ConfigParser()
                     cfg.read(os.path.join(subdir, "config.cfg"))
 
-                    optimizer = pkl["optimizer"]
+                    optimizer = pkl["experiment_name"]
                     optimizer = optimizer.split("/")[-1]
 
                     configurations = len(pkl["trials"])

HPOlib/benchmarks/svm_on_grid.py

@@ -1473,8 +1473,6 @@ def svm_on_grid(c, alpha, epsilon, ret_time=False):
 
     print c, alpha, epsilon
     if ret_time:
-        return config_tree[c_values[c]][alpha_values[alpha]]
-        [epsilon_values[epsilon]][1]
+        return config_tree[c_values[c]][alpha_values[alpha]][epsilon_values[epsilon]][1]
     else:
-        return config_tree[c_values[c]][alpha_values[alpha]]
-        [epsilon_values[epsilon]][0]
+        return config_tree[c_values[c]][alpha_values[alpha]][epsilon_values[epsilon]][0]

HPOlib/check_before_start.py

@@ -60,16 +60,6 @@ def _check_modules():
     except:
         raise ImportError("Scipy cannot be imported. Are you sure that it's installed?")
 
-    try:
-        import theano
-        logger.debug("\tTheano: %s" % str(theano.__version__))
-    except ImportError:
-        logger.warning("Theano not found. You might need this to run some "
-                       "more complex benchmarks!")
-
-    if 'cuda' not in os.environ['PATH']:
-        logger.warning("CUDA not in $PATH")
-
 
 def _check_config(experiment_dir):
     # check whether config file exists

benchmarks/branin/ROAR_smac_2_06_01-dev

@@ -0,0 +1 @@
+smac_2_06_01-dev

benchmarks/branin/ROAR_smac_2_08_00-master

@@ -0,0 +1 @@
+smac_2_08_00-master

benchmarks/camelback/ROAR_smac_2_06_01-dev

@@ -0,0 +1 @@
+smac_2_06_01-dev

benchmarks/camelback/ROAR_smac_2_08_00-master

@@ -0,0 +1 @@
+smac_2_08_00-master

benchmarks/har6/ROAR_smac_2_06_01-dev

@@ -0,0 +1 @@
+smac_2_06_01-dev

benchmarks/har6/ROAR_smac_2_08_00-master

@@ -0,0 +1 @@
+smac_2_08_00-master

benchmarks/lda_on_grid/ROAR_smac_2_06_01-dev

@@ -0,0 +1 @@
+smac_2_06_01-dev

benchmarks/lda_on_grid/ROAR_smac_2_08_00-master

@@ -0,0 +1 @@
+smac_2_08_00-master

benchmarks/lda_on_grid/lda.py

@@ -17,8 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
-import HPOlib.benchmarks.benchmark_functions as benchmark_functions
-
+import HPOlib.benchmarks.lda_on_grid
 import HPOlib.benchmarks.benchmark_util as benchmark_util
 
 
@@ -29,7 +28,7 @@
 
 def main(params, ret_time=False, **kwargs):
     print 'Params: ', params
-    y = benchmark_functions.save_lda_on_grid(params, ret_time=ret_time, **kwargs)
+    y = HPOlib.benchmarks.lda_on_grid.save_lda_on_grid(params, ret_time=ret_time, **kwargs)
     print 'Result: ', y
     return y
 

benchmarks/logreg_on_grid/smac_2_08_00-master

@@ -1 +1 @@
-smac_2_06_01-dev/
+smac_2_06_01-dev

benchmarks/logreg_on_grid/spearmint_gitfork_mod

@@ -1 +1 @@
-/home/feurerm/HPOlib/Software/HPOlib/benchmarks/lda_on_grid/spearmint_april2013_mod
+spearmint_april2013_mod

benchmarks/svm_on_grid/ROAR_smac_2_06_01-dev

@@ -0,0 +1 @@
+smac_2_06_01-dev

benchmarks/svm_on_grid/ROAR_smac_2_08_00-master

@@ -0,0 +1 @@
+smac_2_08_00-master

benchmarks/svm_on_grid/svm.py

@@ -16,7 +16,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import HPOlib.benchmarks.benchmark_functions as benchmark_functions
+import HPOlib.benchmarks.svm_on_grid
 
 import HPOlib.benchmarks.benchmark_util as benchmark_util
 
@@ -28,7 +28,9 @@
 
 def main(params, ret_time=False, **kwargs):
     print 'Params: ', params, '\n'
-    y = benchmark_functions.save_svm_on_grid(params, opt_time=ret_time, **kwargs)
+    y = HPOlib.benchmarks.svm_on_grid.save_svm_on_grid(params,
+                                                       ret_time=ret_time,
+                                                       **kwargs)
     print 'Result: ', y
     return y
 

optimizers/smac/ROAR_smac_2_06_01-dev.py

@@ -0,0 +1 @@
+smac_2_06_01-dev.py

optimizers/smac/ROAR_smac_2_06_01-devDefault.cfg

@@ -0,0 +1 @@
+smac_2_06_01-devDefault.cfg

optimizers/smac/ROAR_smac_2_06_01-dev_parser.py

@@ -0,0 +1,40 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import os
+import sys
+
+sys.path.append(os.path.dirname(__file__))
+smac_2_06_01_dev_parser = __import__('smac_2_06_01-dev_parser')
+
+logger = logging.getLogger("HPOlib.optimizers.smac.ROAR_smac_2_06_01-dev_parser")
+
+
+def manipulate_config(config):
+    '''
+    This method wraps the smac config parser in order to run ROAR
+    '''
+
+    logger.debug("Running in ROAR mode")
+    config = smac_2_06_01_dev_parser.manipulate_config(config=config)
+    config.set('SMAC', 'exec_mode', 'ROAR')
+
+    return config
+
+

optimizers/smac/ROAR_smac_2_08_00-master.py

@@ -0,0 +1 @@
+ROAR_smac_2_06_01-dev.py

optimizers/smac/ROAR_smac_2_08_00-masterDefault.cfg

@@ -0,0 +1 @@
+smac_2_08_00-masterDefault.cfg

optimizers/smac/ROAR_smac_2_08_00-master_parser.py

@@ -0,0 +1,38 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import os
+import sys
+
+sys.path.append(os.path.dirname(__file__))
+smac_2_08_00_master_parser = __import__('smac_2_08_00-master_parser')
+
+logger = logging.getLogger("HPOlib.optimizers.smac.ROAR_smac_2_08_00-master_parser")
+
+
+def manipulate_config(config):
+    '''
+    This method wraps the smac config parser in order to run ROAR
+    '''
+
+    logger.debug("Running in ROAR mode")
+    config = smac_2_08_00_master_parser.manipulate_config(config=config)
+    config.set('SMAC', 'exec_mode', 'ROAR')
+
+    return config

optimizers/smac/smac_2_06_01-dev.py

@@ -40,6 +40,7 @@
 
 #optimizer_str = "smac_2_06_01-dev"
 
+
 def get_algo_exec():
     return '"python ' + os.path.join(os.path.dirname(__file__),
                                      'SMAC_to_HPOlib.py') + '"'
@@ -65,7 +66,7 @@ def _get_state_run(optimizer_dir):
     if len(rungroups) == 1:
         rungroup = rungroups[0]
     else:
-        logger.warning("Found multiple rugroups, take the newest one.")
+        logger.warning("Found multiple rungroups, take the newest one.")
         creation_times = []
         for i, filename in enumerate(rungroups):
             creation_times.append(float(os.path.getctime(filename)))
@@ -97,7 +98,6 @@ def build_smac_call(config, options, optimizer_dir):
                     '--algoExec', get_algo_exec(),
                     '--execDir', optimizer_dir,
                     '-p', os.path.join(optimizer_dir, os.path.basename(config.get('SMAC', 'p'))),
-                    '--initial-incumbent', config.get('SMAC', 'initial_incumbent'),
                     # The experiment dir MUST not be specified when restarting, it is set
                     # further down in the code
                     # '--experimentDir', optimizer_dir,
@@ -111,8 +111,11 @@ def build_smac_call(config, options, optimizer_dir):
                     config.get('SMAC', 'retry_target_algorithm_run_count'),
                     '--intensification-percentage',
                     config.get('SMAC', 'intensification_percentage'),
+                    '--initial-incumbent', config.get('SMAC', 'initial_incumbent'),
                     '--rf-split-min', config.get('SMAC', 'rf_split_min'),
-                    '--validation', config.get('SMAC', 'validation')])
+                    '--validation', config.get('SMAC', 'validation'),
+                    '--runtime-limit', config.get('SMAC', 'runtime_limit'),
+                    '--exec-mode', config.get('SMAC', 'exec_mode')])
 
     if config.getboolean('SMAC', 'save_runs_every_iteration'):
         call = " ".join([call, '--save-runs-every-iteration true'])

optimizers/smac/smac_2_06_01-devDefault.cfg

@@ -3,7 +3,6 @@
 handles_cv=1
 
 [SMAC]
-
 intra_instance_obj = MEAN
 run_obj = QUALITY
 p = smac_2_06_01-dev/params.pcs
@@ -14,6 +13,7 @@ initial_incumbent = DEFAULT
 adaptive_capping = false
 max_incumbent_runs = 2000
 num_iterations = 2147483647
+runtime_limit = 2147483647
 
 deterministic = True
 retry_target_algorithm_run_count = 0

optimizers/smac/smac_2_06_01-dev_parser.py

@@ -20,8 +20,6 @@
 import os
 import sys
 
-import ConfigParser
-
 logger = logging.getLogger("HPOlib.optimizers.smac.smac_2_06_01-dev_parser")
 
 
@@ -51,6 +49,8 @@ def manipulate_config(config):
         config.set('SMAC', 'num_concurrent_algo_execs',
                    config.get('HPOLIB', 'number_of_concurrent_jobs'))
 
+    config.set('SMAC', 'exec_mode', 'SMAC')
+
     path_to_optimizer = config.get('SMAC', 'path_to_optimizer')
     if not os.path.isabs(path_to_optimizer):
         path_to_optimizer = os.path.join(os.path.dirname(os.path.realpath(__file__)), path_to_optimizer)

optimizers/smac/smac_2_08_00-master.py

@@ -71,7 +71,9 @@ def check_dependencies():
 
 
 def _get_state_run(optimizer_dir):
-    rungroups = glob.glob(optimizer_dir + "scenario-SMAC*")
+    rungroups = glob.glob(optimizer_dir + "/" + "scenario-SMAC*")
+    if len(rungroups) == 0:
+        raise Exception("Could not find a rungroup in %s" % optimizer_dir)
     if len(rungroups) == 1:
         rungroup = rungroups[0]
     else:
@@ -121,7 +123,8 @@ def build_smac_call(config, options, optimizer_dir):
                     '--initial-incumbent', config.get('SMAC', 'initial_incumbent'),
                     '--rf-split-min', config.get('SMAC', 'rf_split_min'),
                     '--validation', config.get('SMAC', 'validation'),
-                    '--runtime-limit', config.get('SMAC', 'runtime_limit')])
+                    '--runtime-limit', config.get('SMAC', 'runtime_limit'),
+                    '--exec-mode', config.get('SMAC', 'exec_mode')])
 
     if config.getboolean('SMAC', 'save_runs_every_iteration'):
         call = " ".join([call, '--save-runs-every-iteration true'])
@@ -236,7 +239,7 @@ def main(config, options, experiment_dir, experiment_directory_prefix, **kwargs)
     time_string = wrapping_util.get_time_string()
 
     optimizer_str = os.path.splitext(os.path.basename(__file__))[0]
-
+    logger.debug("Optimizer_str: %s" % optimizer_str)
     # Find experiment directory
     if options.restore:
         if not os.path.exists(options.restore):