Merge branch 'development' of https://github.com/automl/HPOlib into d…

…evelopment

HPOlib/Plotting/plotParam.py

@@ -22,6 +22,7 @@
 
 import cPickle
 import re
+import os
 import sys
 
 from matplotlib.pyplot import tight_layout, figure, subplots_adjust, subplot, savefig, show
@@ -64,7 +65,8 @@ def translate_para(key, value):
     return new_name, value
 
 
-def plot_params(value_list, result_list, name, save="", title="", jitter=0):
+def plot_params(value_list, result_list, name, min_=0.0, max_=0.0, save="",
+                title="", jitter=0):
     color = 'k'
     marker = 'o'
     size = 1
@@ -78,11 +80,17 @@ def plot_params(value_list, result_list, name, save="", title="", jitter=0):
     ax.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
 
     # Define xlims
-    min_y = min(result_list)
-    max_y = max(result_list)
-    y_offset = np.abs(max_y - min_y) * 0.1
-    min_y -= y_offset
-    max_y += y_offset
+    if min_ == max_:
+        min_y = min(result_list)
+        max_y = max(result_list)
+        y_offset = np.abs(max_y - min_y) * 0.1
+        min_y -= y_offset
+        max_y += y_offset
+    else:
+        min_y = min_
+        max_y = max_
+        result_list = np.array(result_list)
+        result_list[result_list > max_y] = max_y
 
     min_x = min(value_list)
     max_x = max(value_list)
@@ -129,49 +137,74 @@ def plot_params(value_list, result_list, name, save="", title="", jitter=0):
         show()
 
 
-def main(pkl_list, name_list, param, save="", title="", jitter=0):
+def main(pkl_list, name_list, param, min_=0.0, max_=0.0,
+         save="", title="", jitter=0):
     if len(name_list) > 1:
         raise NotImplementedError("No support for more than one experiment")
 
-    mod_param = param
-
-    string_to_value_map = dict()
-
-    value_list = list()
-    result_list = list()
-    param_set = set()
-    for pkl in pkl_list[0]:
-        fh = open(pkl, "r")
-        trials = cPickle.load(fh)
-        fh.close()
-        for t in trials["trials"]:
-            if mod_param in t["params"]:
-                k, value = translate_para(param, t["params"][mod_param])
-                k = re.sub('^-', '', k)
-                print k, value
-                value = value.strip()
-                try:
-                    value = float(value)
-                except:
-                    if value in string_to_value_map:
-                        value = string_to_value_map[value]
-                    else:
-                        key = len(string_to_value_map.keys()) + 1
-                        string_to_value_map[value] = key
-                        value = key
-                value_list.append(value)
-                result_list.append(t["result"])
-                param_set.add(k)
-
-    if len(value_list) == 0:
-        print("No values found for param '%s', Available params:\n%s" %
-              (param, "\n".join([p for p in param_set])))
-        sys.exit(1)
-    else:
-        print "Found %s values for %s" % (str(len(value_list)), param)
+    if not param:
+        params = set()
 
-    plot_params(value_list=value_list, result_list=result_list, name=param,
-                save=save, title=title, jitter=jitter)
+        for pkl in pkl_list[0]:
+            fh = open(pkl, "r")
+            trials = cPickle.load(fh)
+            fh.close()
+            for t in trials["trials"]:
+                for k in t["params"]:
+                    k = re.sub('^-', '', k)
+                    params.add(k)
+
+    else:
+        params = [param]
+
+    for param in params:
+        if len(params) > 1:
+            try:
+                os.mkdir("params")
+            except:
+                pass
+
+            save = "params/%s.png" % param
+
+        mod_param = param
+
+        string_to_value_map = dict()
+
+        value_list = list()
+        result_list = list()
+        param_set = set()
+        for pkl in pkl_list[0]:
+            fh = open(pkl, "r")
+            trials = cPickle.load(fh)
+            fh.close()
+            for t in trials["trials"]:
+                if mod_param in t["params"]:
+                    k, value = translate_para(param, t["params"][mod_param])
+                    k = re.sub('^-', '', k)
+                    print k, value
+                    value = value.strip()
+                    try:
+                        value = float(value)
+                    except:
+                        if value in string_to_value_map:
+                            value = string_to_value_map[value]
+                        else:
+                            key = len(string_to_value_map.keys()) + 1
+                            string_to_value_map[value] = key
+                            value = key
+                    value_list.append(value)
+                    result_list.append(t["result"])
+                    param_set.add(k)
+
+        if len(value_list) == 0:
+            print("No values found for param '%s', Available params:\n%s" %
+                  (param, "\n".join([p for p in param_set])))
+            sys.exit(1)
+        else:
+            print "Found %s values for %s" % (str(len(value_list)), param)
+
+        plot_params(value_list=value_list, result_list=result_list, name=param,
+                    min_=min_, max_=max_, save=save, title=title, jitter=jitter)
 
 if __name__ == "__main__":
     prog = "python plot_param.py WhatIsThis <pathTo.pkl>* "
@@ -180,6 +213,10 @@ def main(pkl_list, name_list, param, save="", title="", jitter=0):
     parser = ArgumentParser(description=description, prog=prog)
     parser.add_argument("-s", "--save", dest="save", default="",
                         help="Where to save plot instead of showing it?")
+    parser.add_argument("--max", dest="max", type=float,
+                        default=0, help="Maximum value of the y axis.")
+    parser.add_argument("--min", dest="min", type=float,
+                        default=0, help="Minimum value of the y axis.")
     parser.add_argument("-p", "--parameter", dest="param", default="",
                         help="Which parameter to plot")
     parser.add_argument("-j", "--jitter", dest="jitter", default=0, type=float,
@@ -192,4 +229,5 @@ def main(pkl_list, name_list, param, save="", title="", jitter=0):
     sys.stdout.write("\nFound " + str(len(unknown)) + " argument[s]...\n")
     pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown)
     main(pkl_list=pkl_list_main, name_list=name_list_main, param=args.param,
-         save=args.save, title=args.title, jitter=args.jitter)
+         min_=args.min, max_=args.max, save=args.save, title=args.title,
+         jitter=args.jitter)

HPOlib/config_parser/generalDefault.cfg

@@ -56,3 +56,4 @@ result_on_terminate =
 [MYSQLDBTAE]
 pool = example
 database =
+rc_cutoff = 3600

HPOlib/wrapping_util.py

@@ -139,7 +139,8 @@ def load_experiment_config_file():
         sys.exit(1)
 
 
-def get_configuration(experiment_dir, optimizer_version, unknown_arguments):
+def get_configuration(experiment_dir, optimizer_version, unknown_arguments,
+                      strict=True):
     """How the configuration is parsed:
     1. The command line is already parsed, we have a list of unknown arguments
     2. Assemble list of all config files related to this experiment, these are
@@ -206,9 +207,11 @@ def get_configuration(experiment_dir, optimizer_version, unknown_arguments):
         fh.seek(0)
     else:
         fh = None
+
     config = parse.parse_config(config_files, allow_no_value=True,
                                 optimizer_version=optimizer_version,
                                 cli_values=fh)
+
     if fh is not None:
         fh.close()
     if optimizer_version != "" and optimizer_version is not None:
@@ -271,24 +274,30 @@ def save_config_to_file(file_handle, config, write_nones=True):
                     file_handle.write("%s = %s\n" % (key, config.get(section, key)))
 
 
-def kill_children(sig):
+def kill_processes(sig, processes):
+    """Kill a number of processes with a specified signal.
+
+    Parameters
+    ----------
+    sig : int
+        Send this signal to the processes
+
+    processes : list of psutil.Process
+    """
     # TODO: somehow wait, until the Experiment pickle is written to disk
-    process = psutil.Process(os.getpid())
-    # Ubuntu 14.04 and older versions ship with a deprecated version of
-    # psutil. Use the old interface only if the old version is installed:
-    if psutil.version_info[0] >= 2:
-        children = process.children()
-    else:
-        children = process.get_children()
+
 
     pids_with_commands = []
-    for child in children:
-        pids_with_commands.append((child.pid, child.cmdline()))
+    for process in processes:
+        try:
+            pids_with_commands.append((process.pid, process.cmdline()))
+        except:
+            pass
 
-    logger.debug("Running %s" % str(pids_with_commands))
-    for child in children:
+    logger.debug("Running: %s" % "\n".join(pids_with_commands))
+    for process in processes:
         try:
-            os.kill(child.pid, sig)
+            os.kill(process.pid, sig)
         except Exception as e:
             logger.error(type(e))
             logger.error(e)

benchmarks/branin/config.cfg

@@ -14,4 +14,3 @@ python_module = HPOlib.benchmarks.benchmark_functions
 python_function = save_branin
 number_of_jobs = 200
 result_on_terminate = 1000
-loglevel = 1

tests/workflowtests/test_optimizer.py

@@ -4,7 +4,6 @@
 import shlex
 import subprocess
 import shutil
-import sys
 import unittest
 
 import HPOlib.wrapping_util as wrapping_util
@@ -17,14 +16,16 @@ def setUp(self):
         self.hpolib_dir = os.path.join(os.path.dirname(__file__), "..", "..")
         self.optimizer_dir = os.path.join(self.hpolib_dir, "optimizers")
         self.benchmarks_dir = os.path.join(self.hpolib_dir, "benchmarks")
-        # Add new optimizers only in this kind of format and watch out that
+        # Add new optimizers only in this kind of format and keep in mind that
         # everything after the last '/' is treated as the optimizer name
         self.optimizers = ["smac/smac_2_06_01",
                            "smac/smac_2_08_00",
                            "spearmint/spearmint_april2013",
                            "tpe/hyperopt",
                            "tpe/random"]
         self.experiment_dir_prefix = wrapping_util.get_time_string()
+        os.environ["PATH"] = os.environ["PATH"] + os.pathsep + \
+            os.path.join(self.hpolib_dir, "runsolver/src")
 
     def tearDown(self):
         glob_string = os.path.join(self.benchmarks_dir, "*",
@@ -36,7 +37,8 @@ def tearDown(self):
 
     def test_run_and_test(self):
         for optimizer in self.optimizers:
-            cmd = "HPOlib-run -o %s/%s -s 10 --cwd %s/logreg_on_grid " \
+            cmd = "python -m HPOlib.wrapping " \
+                  "-o %s/%s -s 10 --cwd %s/logreg_on_grid " \
                   "--HPOLIB:number_of_jobs 5 " \
                   "--HPOLIB:experiment_directory_prefix %s" % \
                   (self.optimizer_dir, optimizer, self.benchmarks_dir,
@@ -48,11 +50,10 @@ def test_run_and_test(self):
             print
 
             proc = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE,
-                                    stdin=subprocess.PIPE,
-                                    env=os.environ.copy())
+                                    stdin=subprocess.PIPE)
 
             stdout, stderr = proc.communicate()
-            self.assertEqual(0, proc.returncode)
+            self.assertEqual(0, proc.returncode, stderr)
 
             # And now test that stuff
 
@@ -69,13 +70,14 @@ def test_run_and_test(self):
             print cmd
             print
             proc = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE,
-                                    stdin=subprocess.PIPE, env=os.environ.copy())
+                                    stdin=subprocess.PIPE)
             stdout, stderr = proc.communicate()
             self.assertEqual(0, proc.returncode)
 
     def test_crossvalidation(self):
         # This is only a dummy crossvalidation!
-        cmd = "HPOlib-run -o %s/%s -s 10 --cwd %s/logreg_on_grid " \
+        cmd = "python -m HPOlib.wrapping" \
+              " -o %s/%s -s 10 --cwd %s/logreg_on_grid " \
               "--HPOLIB:number_of_jobs 5 " \
               "--HPOLIB:experiment_directory_prefix %s " \
               "--HPOLIB:number_cv_folds 2" % \