executionButler: add option to transfer files

In some instances, it is helpful/important to transfer files
into the execution butler (e.g., original butler datastore is
on a slow disk). Added option:
    pipetask qgraph ... --target-datastore-root=<root>
to allow transfer of files into a new datastore. Also added
a --transfer option for fine-grained control of the transfer
mode, which defaults to "copy" if there's a new datastore.

doc/changes/DM-35494.feature.md

@@ -0,0 +1 @@
+* Added support for transferring files into execution butler.

python/lsst/ctrl/mpexec/cli/opt/optionGroups.py

@@ -33,6 +33,7 @@
 import click
 import lsst.daf.butler.cli.opt as dafButlerOpts
 import lsst.pipe.base.cli.opt as pipeBaseOpts
+from lsst.daf.butler.cli.opt import transfer_option
 from lsst.daf.butler.cli.utils import OptionGroup, option_section, unwrap
 
 from . import options as ctrlMpExecOpts
@@ -88,6 +89,14 @@ def __init__(self) -> None:
             ctrlMpExecOpts.qgraph_dot_option(),
             ctrlMpExecOpts.save_execution_butler_option(),
             ctrlMpExecOpts.clobber_execution_butler_option(),
+            ctrlMpExecOpts.target_datastore_root_option(),
+            transfer_option(
+                help=unwrap(
+                    """Data transfer mode for the execution butler datastore.
+                    Defaults to "copy" if --target-datastore-root is provided.
+                    """
+                ),
+            ),
             ctrlMpExecOpts.dataset_query_constraint(),
             ctrlMpExecOpts.qgraph_header_data_option(),
         ]

python/lsst/ctrl/mpexec/cli/opt/options.py

@@ -415,6 +415,16 @@
     ),
     is_flag=True,
 )
+
+target_datastore_root_option = MWOptionDecorator(
+    "--target-datastore-root",
+    help=unwrap(
+        """Root directory for datastore of execution butler.
+        Default is to use the original datastore.
+        """
+    ),
+)
+
 dataset_query_constraint = MWOptionDecorator(
     "--dataset-query-constraint",
     help=unwrap(

python/lsst/ctrl/mpexec/cli/script/qgraph.py

@@ -51,6 +51,8 @@ def qgraph(  # type: ignore
     show,
     save_execution_butler,
     clobber_execution_butler,
+    target_datastore_root,
+    transfer,
     clobber_outputs,
     dataset_query_constraint,
     show_qgraph_header=False,
@@ -135,6 +137,12 @@ def qgraph(  # type: ignore
         QuantumGraph.
     clobber_execution_butler : `bool`
         It True overwrite existing execution butler files if present.
+    target_datastore_root : `str` or `None`
+        URI location for the execution butler's datastore.
+    transfer : `str` or `None`
+        Transfer mode for execution butler creation.  This should be a
+        ``transfer`` string recognized by
+        :func:`lsst.resources.ResourcePath.transfer_from`.
     clobber_outputs : `bool`
         Remove outputs from previous execution of the same quantum before new
         execution.  If ``skip_existing`` is also passed, then only failed
@@ -178,6 +186,8 @@ def qgraph(  # type: ignore
         skip_existing=skip_existing,
         execution_butler_location=save_execution_butler,
         clobber_execution_butler=clobber_execution_butler,
+        target_datastore_root=target_datastore_root,
+        transfer=transfer,
         clobber_outputs=clobber_outputs,
         dataset_query_constraint=dataset_query_constraint,
         show_qgraph_header=show_qgraph_header,

python/lsst/ctrl/mpexec/cmdLineFwk.py

@@ -667,6 +667,8 @@ def builderShim(butler: Butler) -> Butler:
                 butlerModifier=builderShim,
                 collections=all_inputs,
                 clobber=args.clobber_execution_butler,
+                datastoreRoot=args.target_datastore_root,
+                transfer=args.transfer,
             )
 
         return qgraph