Merge pull request #32 from automl/fix/#17/#26

FIX #17/#26

smac/runhistory/runhistory.py

@@ -73,16 +73,11 @@ def add(self, config, cost, time,
                 information from TA or fields such as start time and host_id)
         '''
 
-        # TODO: replace str casting of config when we have something hashable
-        # as a config object
-        # TODO JTS: We might have to execute one config multiple times
-        #           since the results can be noisy and then we can't simply
-        #           overwrite the old config result here!
-        config_id = self.config_ids.get(config.__repr__())
+        config_id = self.config_ids.get(config)
         if config_id is None:
             self._n_id += 1
-            self.config_ids[config.__repr__()] = self._n_id
-            config_id = self.config_ids.get(config.__repr__())
+            self.config_ids[config] = self._n_id
+            config_id = self.config_ids.get(config)
             self.ids_config[self._n_id] = config
 
         k = self.RunKey(config_id, instance_id, seed)
@@ -91,11 +86,11 @@ def add(self, config, cost, time,
         self.data[k] = v
 
     def update_cost(self, config, cost):
-        config_id = self.config_ids[config.__repr__()]
+        config_id = self.config_ids[config]
         self.cost_per_config[config_id] = cost
 
     def get_cost(self, config):
-        config_id = self.config_ids[config.__repr__()]
+        config_id = self.config_ids[config]
         return self.cost_per_config[config_id]
 
     def get_runs_for_config(self, config):
@@ -139,8 +134,8 @@ def save_json(self, fn="runhistory.json"):
             file name
         '''
 
-        id_vec = dict([(id_, conf.get_array().tolist())
-                       for id_, conf in self.ids_config.items()])
+        configs = {id_: conf.get_dictionary()
+                   for id_, conf in self.ids_config.items()}
 
         data = [([int(k.config_id),
                   str(k.instance_id) if k.instance_id is not None else None,
@@ -149,7 +144,7 @@ def save_json(self, fn="runhistory.json"):
 
         with open(fn, "w") as fp:
             json.dump({"data": data,
-                       "id_config": id_vec}, fp)
+                       "configs": configs}, fp)
 
     def load_json(self, fn, cs):
         """Load and runhistory in json representation from disk.
@@ -167,18 +162,18 @@ def load_json(self, fn, cs):
         with open(fn) as fp:
             all_data = json.load(fp)
 
-        self.ids_config = dict([(int(id_), Configuration(
-            cs, vector=numpy.array(vec))) for id_, vec in all_data["id_config"].items()])
+        self.ids_config = {int(id_): Configuration(cs, values=values)
+                           for id_, values in all_data["configs"].items()}
 
 
-        self.config_ids = dict([(Configuration(
-            cs, vector=numpy.array(vec)).__repr__(), id_) for id_, vec in all_data["id_config"].items()])
+        self.config_ids = {Configuration(cs, values=values): int(id_)
+                           for id_, values in all_data["configs"].items()}
+
         self._n_id = len(self.config_ids)
 
-        self.data = dict([(self.RunKey(int(k[0]), k[1], int(k[2])),
-                           self.RunValue(float(v[0]), float(v[1]), v[2], v[3]))
-                          for k, v in all_data["data"]
-                          ])
+        self.data = {self.RunKey(int(k[0]), k[1], int(k[2])):
+                     self.RunValue(float(v[0]), float(v[1]), v[2], v[3])
+                     for k, v in all_data["data"]}
 
     def update_from_json(self, fn, cs):
         """Update the current runhistory by adding new runs from a json file.

smac/smbo/objective.py

@@ -28,7 +28,7 @@ def _runtime(config, run_history, instance_seed_pairs=None):
     list
     """
     try:
-        id_ = run_history.config_ids[config.__repr__()]
+        id_ = run_history.config_ids[config]
     except KeyError:  # challenger was not running so far
         return []
 
@@ -82,7 +82,7 @@ def _cost(config, run_history, instance_seed_pairs=None):
     list
     """
     try:
-        id_ = run_history.config_ids[config.__repr__()]
+        id_ = run_history.config_ids[config]
     except KeyError:  # challenger was not running so far
         return []
 

smac/utils/io/traj_logging.py

@@ -7,10 +7,8 @@
 
 import os
 import logging
-import numpy
 import json
 
-from smac.stats.stats import Stats
 
 
 class TrajLogger(object):
@@ -92,7 +90,7 @@ def _add_in_old_format(self, train_perf,
         conf = []
         for p in incumbent:
             if not incumbent[p] is None:
-                conf.append("%s='%s'" % (p, incumbent[p]))
+                conf.append("%s='%s'" % (p, repr(incumbent[p])))
 
         ta_time_used = self.stats.ta_time_used
         wallclock_time = self.stats.get_used_wallclock_time()
@@ -125,7 +123,7 @@ def _add_in_aclib_format(self, train_perf,
         conf = []
         for p in incumbent:
             if not incumbent[p] is None:
-                conf.append("%s='%s'" % (p, incumbent[p]))
+                conf.append("%s='%s'" % (p, repr(incumbent[p])))
 
         ta_time_used = self.stats.ta_time_used
         wallclock_time = self.stats.get_used_wallclock_time()

test/test_smbo/test_pSMAC.py

@@ -32,10 +32,11 @@ def test_write(self):
                   '[[2, null, 1], [1, 1, 1, null]], ' \
                   '[[3, "branin-hoo", 1], [1, 1, 1, null]], ' \
                   '[[4, null, 1], [1, 1, 1, null]]], ' \
-                  '"id_config": {"4": [0.1862602113776709, 0.34556072704304774], ' \
-                  '"3": [0.14675589081711304, 0.0923385947687978], ' \
-                  '"1": [0.417022004702574, 0.7203244934421581], ' \
-                  '"2": [0.00011437481734488664, 0.30233257263183977]}}'
+                  '"configs": {' \
+                  '"4": {"x": -2.2060968293349363, "y": 5.183410905645716}, ' \
+                  '"3": {"x": -2.7986616377433045, "y": 1.385078921531967}, ' \
+                  '"1": {"x": 1.2553300705386103, "y": 10.804867401632372}, ' \
+                  '"2": {"x": -4.998284377739827, "y": 4.534988589477597}}}'
 
         run_history = RunHistory()
         configuration_space = test_helpers.get_branin_config_space()
@@ -83,10 +84,11 @@ def test_load(self):
         '[[2, null, 1], [1, 1, 1, null]], ' \
         '[[1, "branini", 1], [1, 1, 1, null]], ' \
         '[[4, null, 1], [1, 1, 1, null]]], ' \
-        '"id_config": {"1": [0.417022004702574, 0.7203244934421581], ' \
-        '"2": [0.00011437481734488664, 0.30233257263183977], ' \
-        '"3": [0.14675589081711304, 0.0923385947687978], ' \
-        '"4": [0.1862602113776709, 0.34556072704304774]}}'
+        '"configs": {' \
+        '"4": {"x": -2.2060968293349363, "y": 5.183410905645716}, ' \
+        '"3": {"x": -2.7986616377433045, "y": 1.385078921531967}, ' \
+        '"1": {"x": 1.2553300705386103, "y": 10.804867401632372}, ' \
+        '"2": {"x": -4.998284377739827, "y": 4.534988589477597}}}'
 
         other_runhistory_filename = os.path.join(self.tmp_dir,
                                                  '.runhistory_20.json')

test/test_utils/io/test_traj_logging.py

@@ -37,8 +37,8 @@ def test_init(self):
         self.assertFalse(os.path.exists('smac3-output'))
         self.assertTrue(os.path.exists('tmp_test_folder'))
 
-    @patch('smac.utils.io.traj_logging.Stats')
-    def test_add_entry(self,mock_stats):
+    @patch('smac.stats.stats.Stats')
+    def test_add_entry(self, mock_stats):
 
         tl = TrajLogger(output_dir='./tmp_test_folder', stats=mock_stats)
         test_config = {'param_a': 0.5,
@@ -75,7 +75,7 @@ def test_add_entry(self,mock_stats):
         self.assertEquals(len(json_dict['incumbent']), 3)
         self.assertTrue("param_a='0.5'" in json_dict['incumbent'])
 
-    @patch('smac.utils.io.traj_logging.Stats')
+    @patch('smac.stats.stats.Stats')
     def test_add_multiple_entries(self, mock_stats):
         tl = TrajLogger(output_dir='./tmp_test_folder', stats=mock_stats)