Revise config loading to avoid using a singleton (#604)

* Do not create Config as a singleton

This allows the config to be copied and passed around without
being updated in-place whenever a new config is loaded

* Make (re)create independent of global config
* Remove dependency on global cfg from duqduq cli tools
* Update path always

src/duqtools/cli.py

@@ -10,7 +10,7 @@
 
 from ._click_opt_groups import GroupCmd, GroupOpt
 from ._logging_utils import TermEscapeCodeFormatter, duqlog_screen
-from .config import cfg
+from .config import cfg, load_config
 from .operations import op_queue, op_queue_context
 
 logger = logging.getLogger(__name__)
@@ -169,7 +169,7 @@ def parse_dry_run(self, *, dry_run, **kwargs):
 
     def parse_config(self, *, config, **kwargs):
         try:
-            cfg.parse_file(config)
+            load_config(config)
         except ValidationError as e:
             exit(e)
 
@@ -279,9 +279,9 @@ def cli_setup(**kwargs):
 @common_options(*all_options)
 def cli_create(**kwargs):
     """Create the UQ run files."""
-    from .create import create
+    from .create import create_entry
     with op_queue_context():
-        create(**kwargs)
+        create_entry(**kwargs)
 
 
 @cli.command('recreate', cls=GroupCmd)

src/duqtools/config/__init__.py

@@ -1,9 +1,10 @@
-from ._config import Config, cfg
+from ._config import Config, cfg, load_config
 from ._variables import lookup_vars, var_lookup
 
 __all__ = [
     'cfg',
     'Config',
+    'load_config',
     'lookup_vars',
     'var_lookup',
 ]

src/duqtools/config/_config.py

@@ -1,32 +1,41 @@
+"""Config class containing all configs, can be used with:
+
+    from duqtools.config import cfg
+    cfg.<variable you want>
+
+To update the config:
+
+    load_config('duqtools.yaml')
+"""
+
 from __future__ import annotations
 
+from pathlib import Path
+from typing import Union
+
 from ..schema.cli import ConfigModel
 
 
 class Config(ConfigModel):
-    """Config class containing all configs, can be used with:
+    ...
+
+
+def load_config(path: Union[str, Path]) -> Config:
+    global cfg
+
+    new_cfg = Config.parse_file(path)
 
-        from duqtools.config import cfg
-        cfg.<variable you want>
+    from ._variables import var_lookup
 
-    To update the config:
+    if new_cfg.extra_variables:
+        var_lookup.update(new_cfg.extra_variables.to_variable_dict())
 
-        cfg.parse_file('duqtools.yaml')
-    """
-    _instance = None
+    cfg.__dict__.update(new_cfg.__dict__)
 
-    def __new__(cls, *args, **kwargs):
-        # Make it a singleton
-        if not Config._instance:
-            Config._instance = object.__new__(cls)
-        return Config._instance
+    for obj in (cfg, new_cfg):
+        obj._path = path
 
-    def parse_file(self, *args, **kwargs):
-        """Add extra variables to variable lookup table."""
-        super().parse_file(*args, **kwargs)
-        from ._variables import var_lookup
-        if self.extra_variables:
-            var_lookup.update(self.extra_variables.to_variable_dict())
+    return new_cfg
 
 
 cfg = Config.construct()

src/duqtools/create.py

@@ -8,7 +8,7 @@
 
 from .apply_model import apply_model
 from .cleanup import remove_run
-from .config import Config, cfg
+from .config import Config
 from .ids import ImasHandle
 from .matrix_samplers import get_matrix_sampler
 from .models import Locations, Run, Runs
@@ -27,13 +27,15 @@ class CreateError(Exception):
 class CreateManager:
     """Docstring for CreateManager."""
 
-    def __init__(self):
-        self.options = cfg.create
+    def __init__(self, cfg: Config):
+        self.cfg = cfg
 
-        if not self.options:
+        if not self.cfg.create:
             logger.warning('No create options specified.')
             raise CreateError('No create options specified in config.')
 
+        self.options = self.cfg.create
+
         self.template_drc = self.options.template
         self.system = get_system()
         self.runs_dir = self.system.get_runs_dir()
@@ -176,10 +178,10 @@ def write_runs_file(self, runs: Sequence[Run]) -> None:
                     runs.yaml(stream=f)
 
     @add_to_op_queue('Writing csv', quiet=True)
-    def write_runs_csv(self, runs: Sequence[Run], cfg: Config):
+    def write_runs_csv(self, runs: Sequence[Run]):
         fname = self.data_csv
 
-        prefix = f'{cfg.tag}.' if cfg.tag else ''
+        prefix = f'{self.cfg.tag}.' if self.cfg.tag else ''
 
         run_map = {
             f'{prefix}{run.shortname}': run.data_out.dict()
@@ -192,9 +194,9 @@ def write_runs_csv(self, runs: Sequence[Run], cfg: Config):
             df.to_csv(self.runs_dir / fname)
 
     @add_to_op_queue('Storing duqtools.yaml inside runs_dir', quiet=True)
-    def copy_config(self, config):
+    def copy_config(self):
         if self._is_runs_dir_different_from_config_dir():
-            shutil.copyfile(Path.cwd() / config,
+            shutil.copyfile(Path.cwd() / self.cfg._path,
                             self.runs_dir / 'duqtools.yaml')
 
     def _is_runs_dir_different_from_config_dir(self) -> bool:
@@ -218,7 +220,7 @@ def create_run(self, model: Run, *, force: bool = False):
 
         self.apply_operations(model.data_in, model.dirname, model.operations)
 
-        self.system.write_batchfile(model.dirname, cfg)
+        self.system.write_batchfile(model.dirname, self.cfg)
 
         self.system.update_imas_locations(run=model.dirname,
                                           inp=model.data_in,
@@ -227,7 +229,7 @@ def create_run(self, model: Run, *, force: bool = False):
 
 def create(*,
            force,
-           config,
+           cfg: Config,
            no_sampling: bool = False,
            absolute_dirpath: bool = False,
            **kwargs):
@@ -237,15 +239,15 @@ def create(*,
     ----------
     force : bool
         Override protection if data and directories already exist.
-    config : Path
-        Config file location
+    cfg : Config
+        Duqtools config
     no_sampling : bool
         If true, create base run by ignoring `sampler`/`dimensions`.
 
     **kwargs
         Unused.
     """
-    create_mgr = CreateManager()
+    create_mgr = CreateManager(cfg)
 
     ops_dict = create_mgr.generate_ops_dict(base_only=no_sampling)
 
@@ -270,8 +272,14 @@ def create(*,
         create_mgr.create_run(model, force=force)
 
     create_mgr.write_runs_file(runs)
-    create_mgr.write_runs_csv(runs, cfg)
-    create_mgr.copy_config(config)
+    create_mgr.write_runs_csv(runs)
+    create_mgr.copy_config()
+
+
+def create_entry(*args, **kwargs):
+    """Entry point for duqtools cli."""
+    from .config import cfg
+    return create(cfg=cfg, *args, **kwargs)
 
 
 def recreate(*, runs: Sequence[Path], **kwargs):
@@ -284,7 +292,8 @@ def recreate(*, runs: Sequence[Path], **kwargs):
     **kwargs
         Unused.
     """
-    create_mgr = CreateManager()
+    from ..config import cfg
+    create_mgr = CreateManager(cfg)
 
     run_dict = {run.shortname: run for run in Locations().runs}
 

src/duqtools/large_scale_validation/create.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 
-from ..config import cfg
-from ..create import create as duqtools_create
+from ..config import load_config
+from ..create import create as create_entry
 from ..utils import read_imas_handles_from_file, work_directory
 
 
@@ -29,15 +29,17 @@ def create(*, no_sampling: bool, input_file: str, pattern: str, **kwargs):
     config_files = cwd.glob(f'{pattern}/duqtools.yaml')
 
     for config_file in config_files:
-        cfg.parse_file(config_file)
+        cfg = load_config(config_file)
+
+        assert cfg.create
 
         if handles and (cfg.create.template_data not in handles):
             continue
 
         config_dir = config_file.parent
 
         with work_directory(config_dir):
-            duqtools_create(config=config_file,
-                            absolute_dirpath=True,
-                            no_sampling=no_sampling,
-                            **kwargs)
+            create_entry(cfg=cfg,
+                         absolute_dirpath=True,
+                         no_sampling=no_sampling,
+                         **kwargs)

src/duqtools/large_scale_validation/merge.py

@@ -6,7 +6,7 @@
 
 from duqtools.api import ImasHandle
 
-from ..config import cfg
+from ..config import load_config
 from ..merge import _merge, _resolve_variables
 from ..models import Job, Locations
 from ..operations import add_to_op_queue, op_queue
@@ -33,7 +33,10 @@ def merge(force: bool, var_names: Sequence[str], **kwargs):
     for config_file in config_files:
         run_name = config_file.parent.name
 
-        cfg.parse_file(config_file)
+        cfg = load_config(config_file)
+
+        assert cfg.create
+        assert cfg.create.runs_dir
 
         config_dir = config_file.parent
 

src/duqtools/large_scale_validation/status.py

@@ -2,7 +2,7 @@
 
 import click
 
-from ..config import cfg
+from ..config import load_config
 from ..models import Job, Locations
 from ..status import Status, StatusError
 
@@ -32,7 +32,7 @@ def status(*, progress: bool, detailed: bool, pattern: str, **kwargs):
     click.echo()
 
     for config_file in config_files:
-        cfg.parse_file(config_file)
+        cfg = load_config(config_file)
 
         if not cfg.status:
             raise StatusError(

src/duqtools/large_scale_validation/submit.py

@@ -2,10 +2,9 @@
 from pathlib import Path
 from typing import Deque, Sequence
 
-from ..config import cfg
+from ..config import load_config
 from ..models import Job, Locations
 from ..submit import (
-    SubmitError,
     job_array_submitter,
     job_scheduler,
     job_submitter,
@@ -53,15 +52,14 @@ def submit(*, array, force, max_jobs, schedule, max_array_size: int,
 
     for drc in dirs:
         config_file = drc / 'duqtools.yaml'
-        cfg.parse_file(config_file)
+        cfg = load_config(config_file)
+
+        assert cfg.create
+        assert cfg.submit
 
         if handles and (cfg.create.template_data not in handles):
             continue
 
-        if not cfg.submit:
-            raise SubmitError(
-                f'Submit field required in config file: {config_file}')
-
         config_dir = config_file.parent
 
         jobs.extend(Job(run.dirname) for run in Locations(config_dir).runs)

src/duqtools/list_variables.py

@@ -1,7 +1,7 @@
 import click
 from pydantic import ValidationError
 
-from duqtools.config import cfg, var_lookup
+from duqtools.config import load_config, var_lookup
 
 cs = click.style
 
@@ -38,7 +38,7 @@ def list_variables(*, config, **kwargs):
         Unused.
     """
     try:
-        cfg.parse_file(config)
+        cfg = load_config(config)
     except FileNotFoundError:
         print(f'Could not find: {config}')
     except ValidationError as e:

src/duqtools/schema/cli.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 from typing import Literal, Optional, Union
 
-from pydantic import DirectoryPath, Field
+from pydantic import DirectoryPath, Field, PrivateAttr
 
 from ._basemodel import BaseModel
 from ._description_helpers import formatter as f
@@ -196,3 +196,5 @@ class ConfigModel(BaseModel):
         False,
         description=
         'If true, do not output to stdout, except for mandatory prompts.')
+
+    _path: Union[Path, str] = PrivateAttr(None)