Config (#52)

Reworked internal configuration loading to make Hydra more consistent and flexible.
It is now possible to override all of the Hydra configuration option from:
 * .hydra/hydra.yaml in the config path.
 * directly from the job config
 * from the command line
Merging is done in this order:
 - default Hydra configuration
 - .hydra/hydra.yaml in the config path
 - job configuration hydra branch
 - command line overrides starting with hydra.


#### Incompatible changes
If you were using anything on the left below you need to change your code / config / command line.
- ${job:name} -> hydra.job.name
- ${job:override_dirname} -> hydra.job.override_dirname
- ${job:num} -> hydra.job.num
- ${job:id} -> hydra.job.id
- ${hydra:num_jobs) -> hydra.job.num_jobs
- Logging configuration node for job changed from `hydra.task_logging` to `hydra.job_logging`

.circleci/config.yml

@@ -62,4 +62,4 @@ workflows:
       - deploy-website:
           filters:
             branches:
-              only: master  
+              only: master

CHANGELOG.md

@@ -1,4 +1,26 @@
 ## Hydra
+### 08/10/2019
+Reworked internal configuration loading to make Hydra more consistent and flexible.
+It is now possible to override all of the Hydra configuration option from:
+ * .hydra/hydra.yaml in the config path.
+ * directly from the job config
+ * from the command line
+Merging is done in this order:
+ - default Hydra configuration
+ - .hydra/hydra.yaml in the config path
+ - job configuration hydra branch
+ - command line overrides starting with hydra.
+
+
+#### Incompatible changes
+If you were using anything on the left below you need to change your code / config / command line.
+- ${job:name} -> hydra.job.name
+- ${job:override_dirname} -> hydra.job.override_dirname
+- ${job:num} -> hydra.job.num
+- ${job:id} -> hydra.job.id
+- ${hydra:num_jobs) -> hydra.job.num_jobs
+- Logging configuration node for job changed from `hydra.task_logging` to `hydra.job_logging`
+
 ### 07/24/2019
 * Separated logging into hydra_logging and task_logging [#24](https://github.com/fairinternal/hydra/issues/24)
 * Integrated code coverage into nox (current coverage 80%)

demos/4_config_splitting/experiment.py

@@ -2,7 +2,7 @@
 import hydra
 
 
-@hydra.main(config_path='config.yaml')
+@hydra.main(config_path='conf/config.yaml')
 def experiment(cfg):
     print(cfg.pretty())
 

demos/6_sweep/conf/.hydra/launcher/fairtask.yaml

@@ -13,10 +13,10 @@ hydra:
         slurm:
           class: fairtask_slurm.slurm.SLURMQueueConfig
           params:
-            num_jobs: ${hydra:num_jobs}
+            num_jobs: ${hydra.job.num_jobs}
             num_nodes_per_job: 1
             num_workers_per_node: 1
-            name: ${hydra.name}
+            name: ${hydra.job.name}
             maxtime_mins: 4320
             partition: learnfair
             cpus_per_worker: 10

demos/6_sweep/conf/.hydra/launcher/submitit.yaml

@@ -31,5 +31,5 @@ hydra:
           tasks_per_node: 1
           timeout_min: 60
 
-    # variables used by various queues above
+    # variables used queues above
     mem_limit: 24

demos/99_hydra_configuration/task_name/no_config_file_override.py → demos/99_hydra_configuration/job_name/no_config_file_override.py

@@ -1,11 +1,11 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import hydra
-from hydra.utils import JobRuntime
+from hydra import HydraConfig
 
 
 @hydra.main()
 def experiment(_cfg):
-    print(JobRuntime().get('name'))
+    print(HydraConfig.get().hydra.job.name)
 
 
 if __name__ == "__main__":

demos/99_hydra_configuration/task_name/with_config_file_override.py → demos/99_hydra_configuration/job_name/with_config_file_override.py

@@ -1,11 +1,11 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import hydra
-from hydra.utils import JobRuntime
+from hydra import HydraConfig
 
 
 @hydra.main(config_path='config.yaml')
 def experiment(_cfg):
-    print(JobRuntime().get('name'))
+    print(HydraConfig().hydra.job.name)
 
 
 if __name__ == "__main__":

demos/99_hydra_configuration/logging/.hydra/hydra.yaml

@@ -1,2 +1,2 @@
 defaults:
-  - task_logging
+  - job_logging

demos/99_hydra_configuration/logging/.hydra/task_logging.yaml

@@ -1,6 +1,6 @@
 hydra:
   # python logging configuration for the tasks
-  task_logging:
+  job_logging:
     formatters:
       simple:
         format: '[%(levelname)s] - %(message)s'

demos/99_hydra_configuration/workdir/.hydra/hydra.yaml

@@ -5,5 +5,5 @@ hydra:
   run:
     dir: ./outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
   sweep:
-    dir: /checkpoint/${env:USER}/outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}/${hydra.name}
-    subdir: ${job:num}_${job:id}_${job:override_dirname}
+    dir: /checkpoint/${env:USER}/outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}/${hydra.job.name}
+    subdir: ${hydra.job.num}_${hydra.job.num}_${hydra.job.override_dirname}

demos/99_hydra_configuration/workdir/.hydra/launcher/fairtask.yaml

@@ -13,10 +13,10 @@ hydra:
         slurm:
           class: fairtask_slurm.slurm.SLURMQueueConfig
           params:
-            num_jobs: ${hydra.launcher.concurrent}
+            num_jobs: ${hydra.job.num_jobs}
             num_nodes_per_job: 1
             num_workers_per_node: 1
-            name: ${hydra.name}
+            name: ${hydra.job.name}
             maxtime_mins: 4320
             partition: learnfair
             cpus_per_worker: 10

demos/99_hydra_configuration/workdir/custom_workdir.py

@@ -1,16 +1,13 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-import logging
 import os
 import sys
 
 import hydra
 
-log = logging.getLogger(__name__)
-
 
 @hydra.main()
 def experiment(_cfg):
-    log.info("Working directory : {}".format(os.getcwd()))
+    print(os.getcwd())
 
 
 if __name__ == "__main__":

hydra/__init__.py

@@ -6,6 +6,7 @@
 from .launcher import Launcher
 from .plugins import Plugins
 from .sweeper import Sweeper
+from .utils import HydraConfig
 
 __all__ = [
     'Hydra',
@@ -16,4 +17,5 @@
     'Launcher',
     'Plugins',
     'Sweeper',
+    'HydraConfig'
 ]

hydra/config_loader.py

@@ -6,10 +6,11 @@
 import copy
 import os
 
-from omegaconf import OmegaConf, DictConfig, ListConfig
 from pkg_resources import resource_stream, resource_exists
 
+from omegaconf import OmegaConf, DictConfig, ListConfig
 from .errors import MissingConfigException
+from .utils import JobRuntime
 
 
 class ConfigLoader:
@@ -36,7 +37,7 @@ def load_hydra_cfg(self, overrides):
         :param overrides: overrides from command line.
         :return:
         """
-        hydra_cfg_defaults = self._create_cfg(
+        hydra_cfg_defaults, _ = self._create_cfg(
             cfg_dir='pkg://hydra.default_conf',
             cfg_filename='hydra.yaml',
             strict=False)
@@ -47,8 +48,9 @@ def load_hydra_cfg(self, overrides):
         cfg_dir = os.path.join(self.cfg_dir, '.hydra')
 
         hydra_cfg_path = os.path.join(cfg_dir, "hydra.yaml")
+        consumed_defaults = []
         if os.path.exists(hydra_cfg_path):
-            hydra_cfg = self._create_cfg(
+            hydra_cfg, consumed_defaults = self._create_cfg(
                 cfg_dir=cfg_dir,
                 cfg_filename="hydra.yaml",
                 strict=False,
@@ -57,37 +59,46 @@ def load_hydra_cfg(self, overrides):
             hydra_cfg = OmegaConf.from_dotlist(overrides)
         # strip everything outside of the hydra tree from the hydra config
         hydra_cfg = OmegaConf.merge(hydra_cfg_defaults, hydra_cfg)
+        for consumed in consumed_defaults:
+            hydra_cfg.hydra.overrides.hydra.append(consumed)
+            overrides.remove(consumed)
+
         clean = OmegaConf.create()
         clean.hydra = hydra_cfg.hydra
         hydra_cfg = clean
         return hydra_cfg
 
-    def load_task_cfg(self, cli_overrides):
-        """
-        Loads the task configuration
-        :param cli_overrides: config overrides from CLI
-        """
-        return self._create_cfg(
-            cfg_dir=self.cfg_dir,
-            cfg_filename=self.conf_filename,
-            cli_overrides=cli_overrides,
-            strict=self.strict_task_cfg)
-
-    def load_configuration(self, overrides=None):
-        """
-        Load both the Hydra and the task configuraitons
-        :param overrides:
-        :return: a dictionary with both configs
-        """
+    def load_configuration(self, overrides=[]):
         assert overrides is None or isinstance(overrides, list)
 
-        hydra_cfg = self.load_hydra_cfg(overrides or [])
+        overrides = overrides or []
+        hydra_cfg = self.load_hydra_cfg(overrides)
+
+        task_cfg, consumed_defaults = self._create_cfg(cfg_dir=self.cfg_dir,
+                                                       cfg_filename=self.conf_filename,
+                                                       cli_overrides=overrides or [],
+                                                       strict=self.strict_task_cfg)
+        cfg = OmegaConf.merge(hydra_cfg, task_cfg)
+        for item in [x for x in overrides if not x.startswith('hydra.')]:
+            cfg.hydra.overrides.task.append(item)
+        for item in [x for x in overrides if x.startswith('hydra.')]:
+            cfg.hydra.overrides.hydra.append(item)
+
+        job = OmegaConf.create(dict(
+            name=JobRuntime().get('name')
+        ))
+        cfg.hydra.job = OmegaConf.merge(job, cfg.hydra.job)
+        OmegaConf.set_struct(cfg, self.strict_task_cfg)
+        return cfg
 
-        task_cfg = self._create_cfg(cfg_dir=self.cfg_dir,
-                                    cfg_filename=self.conf_filename,
-                                    cli_overrides=overrides or [],
-                                    strict=self.strict_task_cfg)
-        return dict(hydra_cfg=hydra_cfg, task_cfg=task_cfg)
+    def load_sweep_config(self, master_config, sweep_overrides):
+        # Recreate the config for this sweep instance with the appropriate overrides
+        sweep_config = self.load_configuration(master_config.hydra.overrides.hydra.to_container() + sweep_overrides)
+
+        # Copy old config cache to ensure we get the same resolved values (for things like timestamps etc)
+        OmegaConf.copy_cache(from_config=master_config, to_config=sweep_config)
+
+        return sweep_config
 
     def _load_config_impl(self, is_pkg, filename):
         loaded_cfg = None
@@ -164,14 +175,15 @@ def _create_cfg(self, cfg_dir, cfg_filename, strict, cli_overrides=[]):
         # config tree
         overrides = []
         defaults_changes = {}
+        consumed_defaults = []
         for override in copy.deepcopy(cli_overrides):
             key, value = override.split('=')
             assert key != 'optional', "optional is a reserved keyword and cannot be used as a " \
                                       "config group name"
             path = os.path.join(cfg_dir, key)
             if ConfigLoader._exists(is_pkg, path):
                 defaults_changes[key] = value
-                cli_overrides.remove(override)
+                consumed_defaults.append(override)
             else:
                 overrides.append(override)
 
@@ -206,7 +218,7 @@ def _create_cfg(self, cfg_dir, cfg_filename, strict, cli_overrides=[]):
         cfg.merge_with_dotlist(overrides)
         # remove config block from resulting cfg.
         del cfg['defaults']
-        return cfg
+        return cfg, consumed_defaults
 
     @staticmethod
     def _validate_config(cfg):