Merge branch 'main' of https://github.com/sara-nl/hydra into main

.circleci/config.yml

@@ -92,9 +92,8 @@ commands:
             choco install -y --no-progress openssl javaruntime
       - run:
           name: Preparing environment - Hydra
-          # Using virtualenv==20.0.33 higher versions of virtualenv are not compatible with conda on windows. Relevant issue: https://github.com/ContinuumIO/anaconda-issues/issues/12094
           command: |
-            conda create -n hydra python=<< parameters.py_version >> virtualenv==20.0.33 -qy
+            conda create -n hydra python=<< parameters.py_version >> -qy
             conda activate hydra
             pip install nox dataclasses --progress-bar off
       - save_cache:
@@ -250,7 +249,7 @@ workflows:
       - test_win:
           matrix:
             parameters:
-              py_version: ["3.6", "3.7", "3.8"]
+              py_version: ["3.6", "3.7", "3.8", "3.9"]
 
 
   plugin_tests:
@@ -269,7 +268,7 @@ workflows:
       - test_plugin_win:
           matrix:
             parameters:
-              py_version: ["3.6", "3.7", "3.8",]
+              py_version: ["3.6", "3.7", "3.8", "3.9"]
               test_plugin: [<< pipeline.parameters.test_plugins >>]
 
 

.flake8

@@ -7,6 +7,7 @@ exclude =
     ,tools/configen/example/gen
     ,tools/configen/tests/test_modules/expected
     ,temp
+    ,build
 
     # flake8-copyright does not support unicode, savoirfairelinux/flake8-copyright#15
     ,examples/plugins/example_configsource_plugin/hydra_plugins/example_configsource_plugin/example_configsource_plugin.py

.isort.cfg

@@ -24,3 +24,4 @@ skip=
     ,tools/configen/example/gen
     ,tools/configen/tests/test_modules/expected
     ,temp
+    ,build

examples/tutorials/basic/running_your_hydra_app/5_basic_sweep/conf/config.yaml

@@ -0,0 +1,9 @@
+defaults:
+  - db: ???
+  - _self_
+
+hydra:
+  sweeper:
+    params:
+      db: glob(*)
+      db.timeout: 5,10

examples/tutorials/basic/running_your_hydra_app/5_basic_sweep/conf/db/mysql.yaml

@@ -0,0 +1,4 @@
+driver: mysql
+user: omry
+password: secret
+timeout: 5

examples/tutorials/basic/running_your_hydra_app/5_basic_sweep/conf/db/postgresql.yaml

@@ -0,0 +1,4 @@
+driver: postgresql
+user: postgres_user
+password: drowssap
+timeout: 10

examples/tutorials/basic/running_your_hydra_app/5_basic_sweep/my_app.py

@@ -0,0 +1,13 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+from omegaconf import DictConfig
+
+import hydra
+
+
+@hydra.main(config_path="conf", config_name="config")
+def my_app(cfg: DictConfig) -> None:
+    print(f"driver={cfg.db.driver}, timeout={cfg.db.timeout}")
+
+
+if __name__ == "__main__":
+    my_app()

hydra/_internal/core_plugins/basic_sweeper.py

@@ -19,9 +19,10 @@
 import itertools
 import logging
 import time
+from collections import OrderedDict
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Iterable, List, Optional, Sequence
+from typing import Any, Dict, Iterable, List, Optional, Sequence
 
 from omegaconf import DictConfig, OmegaConf
 
@@ -39,6 +40,7 @@
 class BasicSweeperConf:
     _target_: str = "hydra._internal.core_plugins.basic_sweeper.BasicSweeper"
     max_batch_size: Optional[int] = None
+    params: Optional[Dict[str, str]] = None
 
 
 ConfigStore.instance().store(
@@ -54,14 +56,20 @@ class BasicSweeper(Sweeper):
     Basic sweeper
     """
 
-    def __init__(self, max_batch_size: Optional[int]) -> None:
+    def __init__(
+        self, max_batch_size: Optional[int], params: Optional[Dict[str, str]] = None
+    ) -> None:
         """
         Instantiates
         """
         super(BasicSweeper, self).__init__()
+
+        if params is None:
+            params = {}
         self.overrides: Optional[Sequence[Sequence[Sequence[str]]]] = None
         self.batch_index = 0
         self.max_batch_size = max_batch_size
+        self.params = params
 
         self.hydra_context: Optional[HydraContext] = None
         self.config: Optional[DictConfig] = None
@@ -101,12 +109,13 @@ def split_arguments(
     ) -> List[List[List[str]]]:
 
         lists = []
+        final_overrides = OrderedDict()
         for override in overrides:
             if override.is_sweep_override():
                 if override.is_discrete_sweep():
                     key = override.get_key_element()
                     sweep = [f"{key}={val}" for val in override.sweep_string_iterator()]
-                    lists.append(sweep)
+                    final_overrides[key] = sweep
                 else:
                     assert override.value_type is not None
                     raise HydraException(
@@ -115,7 +124,10 @@ def split_arguments(
             else:
                 key = override.get_key_element()
                 value = override.get_value_element_as_str()
-                lists.append([f"{key}={value}"])
+                final_overrides[key] = [f"{key}={value}"]
+
+        for _, v in final_overrides.items():
+            lists.append(v)
 
         all_batches = [list(x) for x in itertools.product(*lists)]
         assert max_batch_size is None or max_batch_size > 0
@@ -127,13 +139,22 @@ def split_arguments(
             )
             return [x for x in chunks_iter]
 
+    def _parse_config(self) -> List[str]:
+        params_conf = []
+        for k, v in self.params.items():
+            params_conf.append(f"{k}={v}")
+        return params_conf
+
     def sweep(self, arguments: List[str]) -> Any:
         assert self.config is not None
         assert self.launcher is not None
         assert self.hydra_context is not None
 
+        params_conf = self._parse_config()
+        params_conf.extend(arguments)
+
         parser = OverridesParser.create(config_loader=self.hydra_context.config_loader)
-        overrides = parser.parse_overrides(arguments)
+        overrides = parser.parse_overrides(params_conf)
 
         self.overrides = self.split_arguments(overrides, self.max_batch_size)
         returns: List[Sequence[JobReturn]] = []

hydra/_internal/core_plugins/importlib_resources_config_source.py

@@ -63,18 +63,15 @@ def load_config(self, config_path: str) -> ConfigResult:
 
     def available(self) -> bool:
         try:
-            ret = resources.is_resource(self.path, "__init__.py")  # type:ignore
-            assert isinstance(ret, bool)
-            return ret
-        except ValueError:
-            return False
-        except ModuleNotFoundError:
+            files = resources.files(self.path)  # type: ignore
+        except (ValueError, ModuleNotFoundError, TypeError):
             return False
+        return any(f.name == "__init__.py" and f.is_file() for f in files.iterdir())
 
     def is_group(self, config_path: str) -> bool:
         try:
             files = resources.files(self.path)  # type:ignore
-        except Exception:
+        except (ValueError, ModuleNotFoundError, TypeError):
             return False
 
         res = files.joinpath(config_path)
@@ -86,7 +83,7 @@ def is_config(self, config_path: str) -> bool:
         config_path = self._normalize_file_name(config_path)
         try:
             files = resources.files(self.path)  # type:ignore
-        except Exception:
+        except (ValueError, ModuleNotFoundError, TypeError):
             return False
         res = files.joinpath(config_path)
         ret = res.exists() and res.is_file()

hydra/_internal/utils.py

@@ -559,43 +559,45 @@ def _locate(path: str) -> Union[type, Callable[..., Any]]:
     """
     if path == "":
         raise ImportError("Empty path")
-    import builtins
     from importlib import import_module
+    from types import ModuleType
 
     parts = [part for part in path.split(".") if part]
-    module = None
-    for n in reversed(range(len(parts))):
+    for n in reversed(range(1, len(parts) + 1)):
+        mod = ".".join(parts[:n])
         try:
-            mod = ".".join(parts[:n])
-            module = import_module(mod)
-        except Exception as e:
-            if n == 0:
-                raise ImportError(f"Error loading module '{path}'") from e
+            obj = import_module(mod)
+        except Exception as exc_import:
+            if n == 1:
+                raise ImportError(f"Error loading module '{path}'") from exc_import
             continue
-        if module:
-            break
-    if module:
-        obj = module
-    else:
-        obj = builtins
-    for part in parts[n:]:
-        mod = mod + "." + part
-        if not hasattr(obj, part):
-            try:
-                import_module(mod)
-            except Exception as e:
-                raise ImportError(
-                    f"Encountered error: `{e}` when loading module '{path}'"
-                ) from e
-        obj = getattr(obj, part)
+        break
+    for m in range(n, len(parts)):
+        part = parts[m]
+        try:
+            obj = getattr(obj, part)
+        except AttributeError as exc_attr:
+            if isinstance(obj, ModuleType):
+                mod = ".".join(parts[: m + 1])
+                try:
+                    import_module(mod)
+                except ModuleNotFoundError:
+                    pass
+                except Exception as exc_import:
+                    raise ImportError(
+                        f"Error loading '{path}': '{repr(exc_import)}'"
+                    ) from exc_import
+            raise ImportError(
+                f"Encountered AttributeError while loading '{path}': {exc_attr}"
+            ) from exc_attr
     if isinstance(obj, type):
         obj_type: type = obj
         return obj_type
     elif callable(obj):
         obj_callable: Callable[..., Any] = obj
         return obj_callable
     else:
-        # dummy case
+        # reject if not callable & not a type
         raise ValueError(f"Invalid type ({type(obj)}) found for {path}")
 
 

hydra/conf/__init__.py

@@ -46,6 +46,10 @@ class JobConf:
     # Job name, populated automatically unless specified by the user (in config or cli)
     name: str = MISSING
 
+    # Change current working dir to the output dir.
+    # Will be non-optional and default to False in Hydra 1.3
+    chdir: Optional[bool] = None
+
     # Populated automatically by Hydra.
     # Concatenation of job overrides that can be used as a part
     # of the directory name.
@@ -93,6 +97,7 @@ class RuntimeConf:
     version: str = MISSING
     cwd: str = MISSING
     config_sources: List[ConfigSourceInfo] = MISSING
+    output_dir: str = MISSING
 
     # Composition choices dictionary
     # Ideally, the value type would be Union[str, List[str], None]

hydra/conf/hydra/job_logging/default.yaml

@@ -11,8 +11,8 @@ handlers:
   file:
     class: logging.FileHandler
     formatter: simple
-    # relative to the job log directory
-    filename: ${hydra.job.name}.log
+    # absolute file path
+    filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
 root:
   level: INFO
   handlers: [console, file]

hydra/core/utils.py

@@ -118,17 +118,23 @@ def run_job(
 
     old_cwd = os.getcwd()
     orig_hydra_cfg = HydraConfig.instance().cfg
-    HydraConfig.instance().set_config(config)
-    working_dir = str(OmegaConf.select(config, job_dir_key))
+
+    output_dir = str(OmegaConf.select(config, job_dir_key))
     if job_subdir_key is not None:
         # evaluate job_subdir_key lazily.
         # this is running on the client side in sweep and contains things such as job:id which
         # are only available there.
         subdir = str(OmegaConf.select(config, job_subdir_key))
-        working_dir = os.path.join(working_dir, subdir)
+        output_dir = os.path.join(output_dir, subdir)
+
+    with read_write(config.hydra.runtime):
+        with open_dict(config.hydra.runtime):
+            config.hydra.runtime.output_dir = os.path.abspath(output_dir)
+
+    HydraConfig.instance().set_config(config)
+    _chdir = None
     try:
         ret = JobReturn()
-        ret.working_dir = working_dir
         task_cfg = copy.deepcopy(config)
         with read_write(task_cfg):
             with open_dict(task_cfg):
@@ -142,14 +148,35 @@ def run_job(
         assert isinstance(overrides, list)
         ret.overrides = overrides
         # handle output directories here
-        Path(str(working_dir)).mkdir(parents=True, exist_ok=True)
-        os.chdir(working_dir)
+        Path(str(output_dir)).mkdir(parents=True, exist_ok=True)
+
+        _chdir = hydra_cfg.hydra.job.chdir
+
+        if _chdir is None:
+            url = "https://hydra.cc/docs/upgrades/1.1_to_1.2/changes_to_job_working_dir"
+            deprecation_warning(
+                message=dedent(
+                    f"""\
+                    Hydra 1.3 will no longer change working directory at job runtime by default.
+                    See {url} for more information."""
+                ),
+                stacklevel=2,
+            )
+            _chdir = True
+
+        if _chdir:
+            os.chdir(output_dir)
+            ret.working_dir = output_dir
+        else:
+            ret.working_dir = os.getcwd()
 
         if configure_logging:
             configure_log(config.hydra.job_logging, config.hydra.verbose)
 
         if config.hydra.output_subdir is not None:
-            hydra_output = Path(config.hydra.output_subdir)
+            hydra_output = Path(config.hydra.runtime.output_dir) / Path(
+                config.hydra.output_subdir
+            )
             _save_config(task_cfg, "config.yaml", hydra_output)
             _save_config(hydra_cfg, "hydra.yaml", hydra_output)
             _save_config(config.hydra.overrides.task, "overrides.yaml", hydra_output)
@@ -172,7 +199,8 @@ def run_job(
         return ret
     finally:
         HydraConfig.instance().cfg = orig_hydra_cfg
-        os.chdir(old_cwd)
+        if _chdir:
+            os.chdir(old_cwd)
 
 
 def get_valid_filename(s: str) -> str:

hydra/grammar/OverrideLexer.g4

@@ -59,7 +59,7 @@ BOOL:
 
 NULL: [Nn][Uu][Ll][Ll];
 
-UNQUOTED_CHAR: [/\-\\+.$%*@?];  // other characters allowed in unquoted strings
+UNQUOTED_CHAR: [/\-\\+.$%*@?|];  // other characters allowed in unquoted strings
 ID: (CHAR|'_') (CHAR|DIGIT|'_')*;
 // Note: when adding more characters to the ESC rule below, also add them to
 // the `_ESC` string in `_internal/grammar/utils.py`.