Add --options-file for command line bulk overrides via file

python/lsst/daf/butler/cli/cmd/commands.py

@@ -22,8 +22,8 @@
 import click
 import yaml
 
-from ..opt import (collection_type_option, dataset_type_option, directory_argument, glob_argument,
-                   repo_argument, run_option, transfer_option, verbose_option)
+from ..opt import (collection_type_option, dataset_type_option, directory_argument, options_file_option,
+                   glob_argument, repo_argument, run_option, transfer_option, verbose_option)
 from ..utils import cli_handle_exception, split_commas, typeStrAcceptsMultiple, unwrap
 from ...script import (butlerImport, createRepo, configDump, configValidate, pruneCollection,
                        queryCollections, queryDatasetTypes)
@@ -51,6 +51,7 @@
 @click.option("--skip-dimensions", "-s", type=str, multiple=True, callback=split_commas,
               metavar=typeStrAcceptsMultiple,
               help="Dimensions that should be skipped during import")
+@options_file_option()
 def butler_import(*args, **kwargs):
     """Import data into a butler repository."""
     cli_handle_exception(butlerImport, *args, **kwargs)
@@ -65,6 +66,7 @@ def butler_import(*args, **kwargs):
               "repo settings.")
 @click.option("--outfile", "-f", default=None, type=str, help="Name of output file to receive repository "
               "configuration. Default is to write butler.yaml into the specified repo.")
+@options_file_option()
 def create(*args, **kwargs):
     """Create an empty Gen3 Butler repository."""
     cli_handle_exception(createRepo, *args, **kwargs)
@@ -81,6 +83,7 @@ def create(*args, **kwargs):
 @click.option("--file", "outfile", type=click.File("w"), default="-",
               help="Print the (possibly-expanded) configuration for a repository to a file, or to stdout "
               "by default.")
+@options_file_option()
 def config_dump(*args, **kwargs):
     """Dump either a subset or full Butler configuration to standard output."""
     cli_handle_exception(configDump, *args, **kwargs)
@@ -93,6 +96,7 @@ def config_dump(*args, **kwargs):
 @click.option("--ignore", "-i", type=str, multiple=True, callback=split_commas,
               metavar=typeStrAcceptsMultiple,
               help="DatasetType(s) to ignore for validation.")
+@options_file_option()
 def config_validate(*args, **kwargs):
     """Validate the configuration files for a Gen3 Butler repository."""
     is_good = cli_handle_exception(configValidate, *args, **kwargs)
@@ -115,6 +119,7 @@ def config_validate(*args, **kwargs):
 @click.option("--unstore",
               help=("""Remove all datasets in the collection from all datastores in which they appear."""),
               is_flag=True)
+@options_file_option()
 def prune_collection(**kwargs):
     """Remove a collection and possibly prune datasets within it."""
     cli_handle_exception(pruneCollection, **kwargs)
@@ -134,6 +139,7 @@ def prune_collection(**kwargs):
                    "--no-include-chains do not return records for CHAINED collections. Default is the "
                    "opposite of --flatten-chains: include either CHAINED collections or their children, but "
                    "not both.")
+@options_file_option()
 def query_collections(*args, **kwargs):
     """Get the collections whose names match an expression."""
     print(yaml.dump(cli_handle_exception(queryCollections, *args, **kwargs)))
@@ -151,6 +157,7 @@ def query_collections(*args, **kwargs):
                    "specified) is to apply patterns to components only if their parent datasets were not "
                    "matched by the expression. Fully-specified component datasets (`str` or `DatasetType` "
                    "instances) are always included.")
+@options_file_option()
 def query_dataset_types(*args, **kwargs):
     """Get the dataset types in a repository."""
     print(yaml.dump(cli_handle_exception(queryDatasetTypes, *args, **kwargs), sort_keys=False))

python/lsst/daf/butler/cli/opt/options.py

@@ -23,7 +23,7 @@
 import click
 from functools import partial
 
-from ..utils import MWOptionDecorator, split_commas, split_kv, unwrap
+from ..utils import MWOptionDecorator, split_commas, split_kv, unwrap, yaml_presets
 from lsst.daf.butler.registry import CollectionType
 
 
@@ -103,3 +103,12 @@ def makeCollectionType(context, param, value):
 verbose_option = MWOptionDecorator("-v", "--verbose",
                                    help="Increase verbosity.",
                                    is_flag=True)
+
+
+options_file_option = MWOptionDecorator("--options-file", "-@",
+                                        expose_value=False,  # This option should not be forwarded
+                                        help=unwrap("""URI to YAML file containing overrides
+                                                    of command line options. The YAML should be organized
+                                                    as a hierarchy with subcommand names at the top
+                                                    level options for that subcommand below."""),
+                                        callback=yaml_presets)

python/lsst/daf/butler/cli/utils.py

@@ -31,10 +31,13 @@
 from unittest.mock import MagicMock, patch
 import uuid
 import yaml
+import logging
 
 from .cliLog import CliLog
 from ..core.utils import iterable
+from ..core.config import Config
 
+log = logging.getLogger(__name__)
 
 # CLI_MOCK_ENV is set by some tests as an environment variable, it
 # indicates to the cli_handle_exception function that instead of executing the
@@ -667,3 +670,50 @@ def getFrom(ctx):
             return ctx.obj
         ctx.obj = MWCtxObj()
         return ctx.obj
+
+
+def yaml_presets(ctx, param, value):
+    """Click callback that reads additional values from the supplied
+    YAML file.
+
+    Parameters
+    ----------
+    ctx : `click.context`
+        The context for the click operation. Used to extract the subcommand
+        name.
+    param : `str`
+        The parameter name.
+    value : `object`
+        The value of the parameter.
+    """
+    ctx.default_map = ctx.default_map or {}
+    cmd_name = ctx.info_name
+    if value:
+        try:
+            overrides = _read_yaml_presets(value, cmd_name)
+        except Exception as e:
+            raise click.BadOptionUsage(param.name, f"Error reading overrides file: {e}", ctx)
+        # Override the defaults for this subcommand
+        ctx.default_map.update(overrides)
+    return
+
+
+def _read_yaml_presets(file_uri, cmd_name):
+    """Read file command line overrides from YAML config file.
+
+    Parameters
+    ----------
+    file_uri : `str`
+        URI of override YAML file containing the command line overrides.
+        They should be grouped by command name.
+    cmd_name : `str`
+        The subcommand name that is being modified.
+
+    Returns
+    -------
+    overrides : `dict` of [`str`, Any]
+        The relevant command line options read from the override file.
+    """
+    log.debug("Reading command line overrides for subcommand %s from URI %s", cmd_name, file_uri)
+    config = Config(file_uri)
+    return config[cmd_name]

tests/data/config-overrides.yaml

@@ -0,0 +1,3 @@
+# Each subcommand has its own section here
+config-dump:
+  subset: .datastore

tests/test_cliCmdConfigDump.py

@@ -23,6 +23,7 @@
 """Unit tests for daf_butler CLI config-dump command.
 """
 
+import os.path
 import unittest
 import yaml
 
@@ -31,6 +32,8 @@
 from lsst.daf.butler.cli.utils import clickResultMsg, LogCliRunner
 from lsst.daf.butler.tests import CliCmdTestBase
 
+TESTDIR = os.path.abspath(os.path.dirname(__file__))
+
 
 class ConfigDumpTest(CliCmdTestBase, unittest.TestCase):
 
@@ -105,6 +108,41 @@ def test_invalidSubset(self):
             self.assertEqual(result.exit_code, 1)
             self.assertIn("Error: 'foo not found in config at here'", result.output)
 
+    def test_presets(self):
+        """Test that file overrides can set command line options in bulk.
+        """
+        with self.runner.isolated_filesystem():
+            result = self.runner.invoke(butler.cli, ["create", "here"])
+            self.assertEqual(result.exit_code, 0, clickResultMsg(result))
+            overrides_path = os.path.join(TESTDIR, "data", "config-overrides.yaml")
+
+            # Run with a presets file
+            result = self.runner.invoke(butler.cli, ["config-dump", "here", "--options-file", overrides_path])
+            self.assertEqual(result.exit_code, 0, clickResultMsg(result))
+            cfg = yaml.safe_load(result.stdout)
+            # Look for datastore information
+            self.assertIn("formatters", cfg)
+            self.assertIn("root", cfg)
+
+            # Now run with an explicit subset and presets
+            result = self.runner.invoke(butler.cli, ["config-dump", "here", f"-@{overrides_path}",
+                                                     "--subset", ".registry"])
+            self.assertEqual(result.exit_code, 0, clickResultMsg(result))
+            cfg = yaml.safe_load(result.stdout)
+            # Look for datastore information
+            self.assertNotIn("formatters", cfg)
+            self.assertIn("managers", cfg)
+
+            # Now with subset before presets -- explicit always trumps
+            # presets.
+            result = self.runner.invoke(butler.cli, ["config-dump", "here", "--subset", ".registry",
+                                                     "--options-file", overrides_path])
+            self.assertEqual(result.exit_code, 0, clickResultMsg(result))
+            cfg = yaml.safe_load(result.stdout)
+            # Look for datastore information
+            self.assertNotIn("formatters", cfg)
+            self.assertIn("managers", cfg)
+
 
 if __name__ == "__main__":
     unittest.main()