FEATURE implement #79

HPOlib/Experiment.py

@@ -316,6 +316,35 @@ def add_job(self, params):
         self._sanity_check()
         return len(self.trials) - 1
 
+    def clean_test_outputs(self, _id):
+        """Revert all information for the run with `_id` to the default so
+        the evaluation can be carried out again.
+
+        Parameters
+        ----------
+        _id : int
+            The ID of the trial dictionary
+
+        Returns
+        -------
+        """
+        trial = self.get_trial_from_id(_id)
+
+        trial['test_status'] = CANDIDATE_STATE
+        trial['test_result'] = np.NaN
+        trial['test_std'] = np.NaN
+        trial['test_duration'] = np.NaN
+
+        duration = np.nansum(trial['test_instance_durations'])
+        self.total_wallclock_time -= duration
+
+        for fold in range(len(trial['test_instance_status'])):
+            trial['test_instance_status'][fold] = CANDIDATE_STATE
+            trial['test_instance_durations'][fold] = np.NaN
+            trial['test_instance_results'][fold] = np.NaN
+            trial['test_additional_data'][fold] = ""
+
+
     def set_one_fold_running(self, _id, fold):
         """Change the status of one fold to running.
 

HPOlib/testing.py

@@ -65,6 +65,10 @@ def use_arg_parser():
     parser.add_argument("--cwd", action="store", type=str, dest="working_dir",
                         default=None, help="Change the working directory to "
                                            "<working_directory> prior to running the experiment")
+    parser.add_argument("--redo-runs", action="store_true",
+                        help="If argument is given, previous runs will be "
+                             "executed again and the previous results will be "
+                             "overwritten.")
     group = parser.add_mutually_exclusive_group()
     group.add_argument("-q", "--silent", action="store_true",
                        dest="silent", default=False,
@@ -101,21 +105,29 @@ def main():
     os.chdir(experiment_dir)
 
     # TODO check if the testing directory exists
-    # TODO check if the test function is there!
     check_before_start.check_first(experiment_dir)
 
     # Now we can safely import non standard things
     import numpy as np
+    global np
     import HPOlib.Experiment as Experiment  # Wants numpy and scipy
+    global Experiment
 
 
     if not config.has_option("HPOLIB", "is_not_original_config_file"):
-        logger.critical("The directory you're in seems to be no directory in "
+        logger.error("The directory you're in seems to be no directory in "
                         "which an HPOlib run was executed: %s" % experiment_dir)
+        exit(1)
     is_not_original_config_file = config.get("HPOLIB", "is_not_original_config_file")
     if not is_not_original_config_file:
-        logger.critical("The directory you're in seems to be no directory in "
+        logger.error("The directory you're in seems to be no directory in "
                         "which an HPOlib run was executed: %s" % experiment_dir)
+        exit(1)
+
+    if not config.has_option("HPOLIB", "test_function"):
+        logger.error("The configuration file does not define a test "
+                        "function.")
+        exit(1)
 
     experiment_directory_prefix = config.get("HPOLIB", "experiment_directory_prefix")
     optimizer = wrapping_util.get_optimizer()
@@ -139,14 +151,16 @@ def main():
                                                   runsolver_output)
 
     configurations_to_test = []
-    # TODO this should not rerun configurations which were already run!
     # Find the configurations to test on!
     if args.all:
         for idx in range(len(trials.trials)):
             configurations_to_test.append(idx)
+            if args.redo_runs:
+                trials.clean_test_outputs(idx)
     elif args.best:
         id_ = trials.get_arg_best(consider_incomplete=False)
         configurations_to_test.append(id_)
+        trials.clean_test_outputs(id_)
     elif args.trajectory:
         raise NotImplementedError("Evaluating the runs along a trajectory is "
                                   "not implemented yet!")

docs/source/manual.rst

@@ -543,14 +543,20 @@ configurations the HPOlib will call the test script.
 * :bash:`--best`: Call the test-script only for the best configuration.
 * :bash:`--trajectory`: Not yet implemented!
 
-As an example, consider the usecase that we ran SMAC to optimize the
-:ref:`logistic regression <logreg>` and want to get the test performance for
-the best configuration.
+The second argument :bash:`--cwd` tells HPOlib in which experiment directory
+it should run test the configurations. As an example, consider the usecase that
+we ran SMAC to optimize the :ref:`logistic regression <logreg>` and want to get
+the test performance for the best configuration.
 
 .. code:: bash
 
     HPOlib-testbest --best --cwd logreg/nocv/smac_2_08_00-master_2000_2014-11-7--16-49-28-166127/
 
+Further options are:
+
+* :bash:`--redo-runs`: If argument is given, previous runs will be executed
+  again and the previous results will be overwritten.
+
 Dispatchers: Different ways to invoke the Target Algorithm
 ==========================================================
 

tests/unittests/test_experiment.py

@@ -180,6 +180,28 @@ def test_add_job(self):
     # works, as this is implicitly tested in test_set_one_fold_crashed and
     # test_set_one_fold_complete
 
+    def test_clean_test_outputs(self):
+        experiment = Experiment.Experiment(".", "test_exp", folds=2)
+        for i in range(2):
+            _id = experiment.add_job({"x": i})
+            experiment.set_one_fold_running(_id, 0)
+            experiment.set_one_fold_running(_id, 1)
+            experiment.set_one_fold_complete(_id, 0, 1, 1, "")
+            experiment.set_one_fold_complete(_id, 1, 2, 2, "")
+            experiment.set_one_test_fold_running(_id, 0)
+            experiment.set_one_test_fold_complete(_id, 0, 1, 5, "")
+        self.assertEqual(experiment.total_wallclock_time, 16)
+        experiment.clean_test_outputs(0)
+        trial = experiment.get_trial_from_id(0)
+        self.assertEqual(experiment.total_wallclock_time, 11)
+        self.assertFalse(np.isfinite(trial['test_duration']))
+        self.assertFalse(np.isfinite(trial['test_result']))
+        self.assertFalse(np.isfinite(trial['test_std']))
+        self.assertEqual(trial['test_status'], 0)
+        self.assertFalse(all(np.isfinite(trial['test_instance_durations'])))
+        self.assertFalse(all(np.isfinite(trial['test_instance_results'])))
+        self.assertEqual(np.sum(trial['test_instance_status']), 0)
+
     def test_set_one_fold_complete(self):
         experiment = Experiment.Experiment(".", "test_exp", folds=1)
         experiment.add_job({"x": 0})