DM-14191: Merge butler configuration files with defaults

python/lsst/daf/butler/core/butlerConfig.py

@@ -26,6 +26,10 @@
 import os
 
 from .config import Config
+from .datastore import DatastoreConfig
+from .schema import SchemaConfig
+from .registry import RegistryConfig
+from .storageClass import StorageClassConfig
 from .utils import doImport
 
 __all__ = ("ButlerConfig",)
@@ -36,9 +40,17 @@ class ButlerConfig(Config):
 
     The configuration is read and merged with default configurations for
     the particular classes. The defaults are read from
-    ``$DAF_BUTLER_DIR/config`` using names specified by the appropriate classes
-    for registry and datastore, and including the standard schema and storage
-    class definitions.
+    ``$DAF_BUTLER_DIR/config`` and ``$DAF_BULTER_CONFIG_PATH``. The defaults
+    are constructed by reading first the global defaults, and then adding
+    in overrides from each entry in the colon-separated
+    ``$DAF_BUTLER_CONFIG_PATH`` in reverse order such that the entries ahead
+    in the list take priority. The registry and datastore configurations
+    are read using the names specified by the appropriate classes defined
+    in the supplied butler configuration.
+
+    The externally supplied butler configuration must include definitions
+    for ``registry.cls`` and ``datastore.cls`` to enable the correct defaults
+    to be located.
 
     Parameters
     ----------
@@ -51,54 +63,64 @@ def __init__(self, other):
         # Create an empty config for us to populate
         super().__init__()
 
-        # Find the butler configs
-        self.defaultsDir = None
-        if "DAF_BUTLER_DIR" in os.environ:
-            self.defaultsDir = os.path.join(os.environ["DAF_BUTLER_DIR"], "config")
-
-        # Storage classes
-        storageClasses = Config(os.path.join(self.defaultsDir, "storageClasses.yaml"))
-        self.update(storageClasses)
-
-        # Default schema
-        schema = Config(os.path.join(self.defaultsDir, "registry", "default_schema.yaml"))
-        self.update(schema)
-
         # Read the supplied config so that we can work out which other
         # defaults to use.
         butlerConfig = Config(other)
 
-        # Check that fundamental keys are present
-        self._validate(butlerConfig)
+        # All the configs that can be associated with default files
+        configComponents = (SchemaConfig, StorageClassConfig, DatastoreConfig, RegistryConfig)
+
+        # This is a list of all the config files to search
+        defaultsFiles = [(c, c.defaultConfigFile) for c in configComponents]
 
-        # Look for class specific defaults
-        for section in ("datastore", "registry"):
-            k = f"{section}.cls"
-            print("Checking {}: {}".format(k, butlerConfig[k]))
+        # Components that derive their configurations from implementation
+        # classes rather than configuration classes
+        specializedConfigs = (DatastoreConfig, RegistryConfig)
+
+        # Look for class specific default files to be added to the file list
+        # These class definitions must come from the suppliedbutler config and
+        # are not defaulted.
+        for componentConfig in specializedConfigs:
+            k = "{}.cls".format(componentConfig.component)
+            if k not in butlerConfig:
+                raise ValueError("Required configuration {} not present in {}".format(k, other))
             cls = doImport(butlerConfig[k])
-            defaultsPath = cls.defaults
-            if defaultsPath is not None:
-                if not os.path.isabs(defaultsPath):
-                    defaultsPath = os.path.join(self.defaultsDir, defaultsPath)
-                c = Config(defaultsPath)
-                # Merge into baseline
-                self.update(c)
+            defaultsFile = cls.defaultConfigFile
+            if defaultsFile is not None:
+                defaultsFiles.append((componentConfig, defaultsFile))
+
+        # We can pick up defaults from multiple search paths
+        # We fill defaults by using the butler config path and then
+        # the config path environment variable in reverse order.
+        self.defaultsPaths = []
+
+        # Find the butler configs
+        if "DAF_BUTLER_DIR" in os.environ:
+            self.defaultsPaths.append(os.path.join(os.environ["DAF_BUTLER_DIR"], "config"))
+
+        if "DAF_BUTLER_CONFIG_PATH" in os.environ:
+            externalPaths = list(reversed(os.environ["DAF_BUTLER_CONFIG_PATH"].split(os.pathsep)))
+            self.defaultsPaths.append(externalPaths)
+
+        # Search each directory for each of the default files
+        for pathDir in self.defaultsPaths:
+            for configClass, configFile in defaultsFiles:
+                # Assume external paths have same config files as global config
+                # directory. Absolute paths are possible for external
+                # code.
+                # Should this be a log message? Are we using lsst.log?
+                # print("Checking path {} for {} ({})".format(pathDir, configClass, configFile))
+                if not os.path.isabs(configFile):
+                    configFile = os.path.join(pathDir, configFile)
+                if os.path.exists(configFile):
+                    # this checks a specific part of the tree
+                    # We may need to turn off validation since we do not
+                    # require that each defaults file found is fully
+                    # consistent.
+                    config = configClass(configFile)
+                    # Attach it using the global namespace
+                    self.update({configClass.component: config})
 
         # Now that we have all the defaults we can merge the externally
         # provided config into the defaults.
         self.update(butlerConfig)
-
-    def _validate(self, config=None):
-        """Check a butler config contains mandatory keys.
-
-        Parameters
-        ----------
-        config : `Config`, optional
-            By default checks itself, but if ``config`` is given, this
-            config will be checked instead.
-        """
-        if config is None:
-            config = self
-        for k in ['datastore.cls', 'registry.cls']:
-            if k not in config:
-                raise ValueError(f"Missing ButlerConfig parameter: {k}")

python/lsst/daf/butler/core/config.py

@@ -357,6 +357,10 @@ class ConfigSubset(Config):
     """Keys that are required to be specified in the configuration.
     """
 
+    defaultConfigFile = None
+    """Name of the file containing defaults for this config class.
+    """
+
     def __init__(self, other=None):
         super().__init__(other)
         if self.component is not None and self.component in self.data:

python/lsst/daf/butler/core/datastore.py

@@ -35,6 +35,7 @@
 class DatastoreConfig(ConfigSubset):
     component = "datastore"
     requiredKeys = ("cls",)
+    defaultConfigFile = "datastore.yaml"
 
 
 class Datastore(metaclass=ABCMeta):
@@ -55,7 +56,7 @@ class Datastore(metaclass=ABCMeta):
         Load configuration
     """
 
-    defaults = None
+    defaultConfigFile = None
     """Path to configuration defaults. Relative to $DAF_BUTLER_DIR/config or
     absolute path. Can be None if no defaults specified.
     """

python/lsst/daf/butler/core/registry.py

@@ -22,55 +22,75 @@
 from abc import ABCMeta
 
 from .utils import doImport
-from .butlerConfig import ButlerConfig
-from .config import Config
+from .config import Config, ConfigSubset
+from .schema import SchemaConfig
 
 __all__ = ("RegistryConfig", "Registry")
 
 
-class RegistryConfig(ButlerConfig):
-    pass
+class RegistryConfig(ConfigSubset):
+    component = "registry"
+    requiredKeys = ("cls",)
+    defaultConfigFile = "registry.yaml"
 
 
 class Registry(metaclass=ABCMeta):
     """Registry interface.
 
     Parameters
     ----------
-    config : `RegistryConfig`
+    registryConfig : `RegistryConfig`
+        Registry configuration.
+    schemaConfig : `SchemaConfig`, optional
+        Schema configuration.
     """
 
-    defaults = None
+    defaultConfigFile = None
     """Path to configuration defaults. Relative to $DAF_BUTLER_DIR/config or
     absolute path. Can be None if no defaults specified.
     """
 
     @staticmethod
-    def fromConfig(config):
+    def fromConfig(registryConfig, schemaConfig=None):
         """Create `Registry` subclass instance from `config`.
 
         Uses ``registry.cls`` from `config` to determine which subclass to instantiate.
 
         Parameters
         ----------
-        config : `RegistryConfig`, `Config` or `str`
+        registryConfig : `ButlerConfig`, `RegistryConfig`, `Config` or `str`
             Registry configuration
+        schemaConfig : `SchemaConfig`, `Config` or `str`, optional.
+            Schema configuration. Can be read from supplied registryConfig
+            if the relevant component is defined and ``schemaConfig`` is
+            `None`.
 
         Returns
         -------
         registry : `Registry` (subclass)
             A new `Registry` subclass instance.
         """
-        if not isinstance(config, RegistryConfig):
-            if isinstance(config, str) or isinstance(config, Config):
-                config = RegistryConfig(config)
+        if schemaConfig is None:
+            # Try to instantiate a schema configuration from the supplied
+            # registry configuration.
+            schemaConfig = SchemaConfig(registryConfig)
+        elif not isinstance(schemaConfig, SchemaConfig):
+            if isinstance(schemaConfig, str) or isinstance(schemaConfig, Config):
+                schemaConfig = SchemaConfig(schemaConfig)
             else:
-                raise ValueError("Incompatible Registry configuration: {}".format(config))
-        cls = doImport(config['registry.cls'])
-        return cls(config=config)
+                raise ValueError("Incompatible Schema configuration: {}".format(schemaConfig))
 
-    def __init__(self, config):
-        assert isinstance(config, RegistryConfig)
-        self.config = config
+        if not isinstance(registryConfig, RegistryConfig):
+            if isinstance(registryConfig, str) or isinstance(registryConfig, Config):
+                registryConfig = RegistryConfig(registryConfig)
+            else:
+                raise ValueError("Incompatible Registry configuration: {}".format(registryConfig))
+
+        cls = doImport(registryConfig['cls'])
+        return cls(registryConfig, schemaConfig)
+
+    def __init__(self, registryConfig, schemaConfig=None):
+        assert isinstance(registryConfig, RegistryConfig)
+        self.config = registryConfig
 
     #  TODO Add back all interfaces (copied from SqlRegistry) once that is stabalized

python/lsst/daf/butler/core/schema.py

@@ -33,6 +33,7 @@
 class SchemaConfig(ConfigSubset):
     component = "schema"
     requiredKeys = ("version", "dataUnits")
+    defaultConfigFile = "registry_schema.yaml"
 
 
 class Schema:

python/lsst/daf/butler/core/storageClass.py

@@ -32,6 +32,7 @@
 
 class StorageClassConfig(ConfigSubset):
     component = "storageClasses"
+    defaultConfigFile = "storageClasses.yaml"
 
 
 class StorageClass:

python/lsst/daf/butler/datastores/posixDatastore.py

@@ -75,7 +75,7 @@ class PosixDatastore(Datastore):
         configuration.
     """
 
-    defaults = "datastores/posixDatastore.yaml"
+    defaultConfigFile = "datastores/posixDatastore.yaml"
     """Path to configuration defaults. Relative to $DAF_BUTLER_DIR/config or
     absolute path. Can be None if no defaults specified.
     """

python/lsst/daf/butler/registries/sqlRegistry.py

@@ -50,20 +50,22 @@ class SqlRegistry(Registry):
     ----------
     config : `SqlRegistryConfig` or `str`
         Load configuration
+    schemaConfig : `SchemaConfig` or `str`
+        Definition of the schema to use.
     """
 
-    defaults = None
+    defaultConfigFile = None
     """Path to configuration defaults. Relative to $DAF_BUTLER_DIR/config or
     absolute path. Can be None if no defaults specified.
     """
 
-    def __init__(self, config):
-        super().__init__(config)
+    def __init__(self, registryConfig, schemaConfig):
+        super().__init__(registryConfig)
 
-        self.config = SqlRegistryConfig(config)
+        self.config = SqlRegistryConfig(registryConfig)
         self.storageClasses = StorageClassFactory()
-        self._schema = Schema(self.config['schema'])
-        self._engine = create_engine(self.config['registry.db'])
+        self._schema = Schema(schemaConfig)
+        self._engine = create_engine(self.config['db'])
         self._schema.metadata.create_all(self._engine)
         self._datasetTypes = {}
 

python/lsst/daf/butler/registries/sqliteRegistry.py

@@ -47,5 +47,5 @@ class SqliteRegistry(SqlRegistry):
         Load configuration
     """
 
-    def __init__(self, config):
-        super().__init__(config)
+    def __init__(self, registryConfig, schemaConfig):
+        super().__init__(registryConfig, schemaConfig)