Add ConfigurationRunner as new optimizer

HPOlib/optimization_interceptor.py

@@ -44,11 +44,16 @@
 # TODO: This should be in a util function sometime in the future
 #       Is duplicated in runsolver_wrapper.py
 def get_optimizer():
-    return "_".join(os.getcwd().split("/")[-1].split("_")[0:-2])
+    optimizer_dir = os.getcwd().split("/")[-1]
+    if optimizer_dir.count("_") == 1:
+        return optimizer_dir.split("_")[0]
+    else:
+        return optimizer_dir.split("_")[0:-2]
 
 
 def load_experiment_file():
     optimizer = get_optimizer()
+    print os.getcwd().split("/")[-1]
     experiment = Experiment(".", optimizer)
     return experiment
 

docs/source/index.rst

@@ -14,6 +14,7 @@ Contents:
    install
    algorithms_and_datasets
    manual
+   optimizers
    plotting
    structure
    cite

docs/source/optimizers.rst

@@ -0,0 +1,65 @@
+=========================
+Optimizeration algorithms
+=========================
+
+.. _bergstra and bengio, 2012: http://www.jmlr.org/papers/volume13/bergstra12a/bergstra12a.pdf
+.. _bergstra et al., 2011: http://papers.nips.cc/paper/4443-algorithms-for-hyper-parameter-optimization.pdf
+.. _hutter et al., 2011: http://www.cs.ubc.ca/labs/beta/Projects/SMAC/papers/11-LION5-SMAC.pdf
+.. _snoek et al. (2012): http://papers.nips.cc/paper/4522-practical-bayesian-optimization-of-machine-learning-algorithms.pdf
+
+HPOlib ships several optimization packages by default. These are:
+
+* :ref:`ConfigurationRunner <configurationrunner>`
+    Executes configurations which are saved in a csv file.
+* SMAC v2.06.01
+    Includes the ROAR and SMAC algorithm (`Hutter et al., 2011`_).
+* SMAC v2.08.00
+    Includes the ROAR and SMAC algorithm (`Hutter et al., 2011`_).
+* SMAC v2.10.00
+    Includes the ROAR and SMAC algorithm (`Hutter et al., 2011`_).
+* Spearmint (github clone from april 2013)
+    Performs Bayesian optimization with Gaussian Processes as described in
+    `Snoek et al. (2012)`_.
+* Hyperopt (github clone from august 2013)
+    Includes random search (`Bergstra and Bengio, 2012`_) and the Tree Parzen
+    Estimator (`Bergstra et al., 2011`_)
+
+
+.. _configurationrunner:
+
+Configuration Runner
+====================
+
+The `ConfigurationRunner` is an optimizer which runs configurations saved in
+a csv file. It is useful to evaluate configurations which do not come from an
+optimization algorithm and still benefit from HPOlib's functionality.
+
+By default, it expects a csv file called `configurations` as input. The first
+line determines the names of the hyperparameters, every following line
+determines a single configuration.
+
+The following is an example file for the branin function::
+
+    x,y
+    0,0
+    1,1
+    2,2
+    3,3
+    4,4
+    5,5
+    6,6
+    7,7
+    8,8
+    9,9
+    10,10
+
+**WARNING**: `ConfigurationRunner` does not check if the configurations
+adhere to any configuration space. This must be done by the user.
+
+Furthermore, `ConfigurationRunner` can execute the function evaluations in
+parallel. This is governed by the argument `n_jobs` and only useful if the
+target machine has enough processors/cores or the jobs are distributed across
+several machines.
+
+
+

optimizers/ConfigurationRunner/ConfigurationRunner.py

@@ -0,0 +1,103 @@
+##
+# HPOlib: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013-2015 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#!/usr/bin/env python
+
+import os
+
+import HPOlib.wrapping_util as wrapping_util
+
+
+version_info = ("0.1")
+__authors__ = ["Matthias Feurer"]
+__contact__ = "automl.org"
+
+optimizer_str = "ConfigurationRunner"
+
+
+def check_dependencies():
+    # Check the dependencies. The ConfigurationRunner has no external
+    # dependencies!
+    pass
+
+
+def build_call(config, options, optimizer_dir):
+    # Build the call to the ConfigurationRunner
+    this_file = __file__
+    this_directory = os.path.dirname(this_file)
+    ConfigurationRunner_file = os.path.join(this_directory,
+                                            "ConfigurationRunner_optimizer.py")
+    call = []
+    call.append("python")
+    call.append(ConfigurationRunner_file)
+    call.append(os.path.join(optimizer_dir,
+                os.path.basename(config.get("ConfigurationRunner",
+                                            "configurations"))))
+    return " ".join(call)
+
+
+#noinspection PyUnusedLocal
+def restore(config, optimizer_dir, **kwargs):
+    # This will not be used
+    raise NotImplementedError("Restore is not implemented for the "
+                              "ConfigurationRunner")
+
+
+#noinspection PyUnusedLocal
+def main(config, options, experiment_dir, experiment_directory_prefix, **kwargs):
+    # config:           Loaded .cfg file
+    # options:          Options containing seed, restore_dir,
+    # experiment_dir:   Experiment directory/Benchmark_directory
+    # **kwargs:         Nothing so far
+    time_string = wrapping_util.get_time_string()
+
+    path_to_optimizer = os.path.abspath(os.path.dirname(__file__))
+
+    # Find experiment directory
+    if options.restore:
+        raise NotImplementedError("Restore is not implemented for the "
+                                  "ConfigurationRunner")
+    else:
+        optimizer_dir = os.path.join(experiment_dir,
+                                     experiment_directory_prefix
+                                     + optimizer_str + "_" + time_string)
+
+    # Set up experiment directory
+    if not os.path.exists(optimizer_dir):
+        os.mkdir(optimizer_dir)
+
+    space = config.get('ConfigurationRunner', "configurations")
+    abs_space = os.path.abspath(space)
+    parent_space = os.path.join(experiment_dir, optimizer_str, space)
+    if os.path.exists(abs_space):
+        space = abs_space
+    elif os.path.exists(parent_space):
+        space = parent_space
+    else:
+        raise Exception("Configurations for the ConfigurationRunner not found. "
+                        "Searched at %s and "
+                        "%s" % (abs_space, parent_space))
+
+    # if not os.path.exists(os.path.join(optimizer_dir, os.path.basename(space))):
+    os.symlink(os.path.join(experiment_dir, optimizer_str, space),
+               os.path.join(optimizer_dir, os.path.basename(space)))
+
+    # Build call
+    cmd = build_call(config, options, optimizer_dir)
+
+    return cmd, optimizer_dir

optimizers/ConfigurationRunner/ConfigurationRunnerDefault.cfg

@@ -0,0 +1,3 @@
+[ConfigurationRunner]
+configurations = configurations
+n_jobs = 1

optimizers/ConfigurationRunner/ConfigurationRunner_optimizer.py

@@ -0,0 +1,82 @@
+import argparse
+from collections import OrderedDict
+import csv
+import logging
+import subprocess
+from multiprocessing import Pool
+
+import HPOlib.optimization_interceptor
+
+
+def command_line_function(params):
+    call = construct_cli_call(params)
+    ret = subprocess.call(call, shell=True)
+    return ret
+
+
+def construct_cli_call(params):
+    optimization_interceptor_file = HPOlib.optimization_interceptor.__file__
+    if optimization_interceptor_file.endswith("pyc"):
+        optimization_interceptor_file = optimization_interceptor_file[:-1]
+    cli_target = optimization_interceptor_file
+
+    cli_call = []
+    cli_call.append("python")
+    cli_call.append(cli_target)
+    cli_call.append("--params")
+    params = OrderedDict(sorted(params.items(), key=lambda t: t[0]))
+    for param in params:
+        value = params[param]
+        if value:
+            cli_call.append(" -" + param)
+            cli_call.append("\"'" + str(params[param]) + "'\"")
+    return " ".join(cli_call)
+
+
+class ConfigurationRunner(object):
+    """
+    Run all hyperparameter configurations from a list sequentially.
+
+    This optimizer can be used to do the following:
+        * Perform a grid search
+        * Evaluate a list of hyperparameter configurations
+
+    It can read the following formats:
+        * A csv in which every line specifies one hyperparameter configuration
+    """
+    def __init__(self, configurations_file, n_jobs):
+        self.logger = logging.getLogger("ConfigurationRunner")
+        self.logger.setLevel(logging.INFO)
+
+        self.configurations_file = configurations_file
+        self.n_jobs = n_jobs
+        self.configurations = []
+
+        with open(self.configurations_file) as fh:
+            csv_reader = csv.DictReader(fh)
+            for row in csv_reader:
+                self.configurations.append(row)
+
+        for configuration in self.configurations:
+            self.logger.info(configuration)
+
+    def run(self):
+        pool = Pool(processes=4)
+        pool.map(command_line_function, self.configurations)
+        pool.close()
+        pool.join()
+
+
+if __name__ == "__main__":
+    description = "Evaluate hyperparameter configurations via HPOlib."
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument("configurations_file", metavar="configurations-file",
+                        help="List of configurations as a csv file.")
+    parser.add_argument("--n-jobs", default=1, type=int,
+                        help="Number of parallel function evaluations.")
+    args = parser.parse_args()
+
+    runner = ConfigurationRunner(args.configurations_file, args.n_jobs)
+    runner.run()
+
+

optimizers/ConfigurationRunner/ConfigurationRunner_parser.py

@@ -0,0 +1,44 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+import ConfigParser
+
+
+def manipulate_config(config):
+    # This module reads sequentialDefault.cfg and adds this defaults to a
+    # given config
+
+    assert isinstance(config, ConfigParser.RawConfigParser), \
+        'config is not a valid instance'
+
+    config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                             'ConfigurationRunnerDefault.cfg')
+    if not os.path.isfile(config_file):
+        raise Exception('%s is not a valid file\n' % config_file)
+
+    sequential_config = ConfigParser.SafeConfigParser(allow_no_value=True)
+    sequential_config.read(config_file)
+
+    if not config.has_section('ConfigurationRunner'):
+        config.add_section('ConfigurationRunner')
+
+    config.set("DEFAULT", "optimizer_version", "0.1")
+
+    return config