diff --git a/doc/changes/DM-52360.perf.md b/doc/changes/DM-52360.perf.md new file mode 100644 index 00000000..31976c64 --- /dev/null +++ b/doc/changes/DM-52360.perf.md @@ -0,0 +1 @@ +Switch the default `finalJob` implementation to the new `aggregate-graph` command, which makes use of multiple cores much more effectively than `transfer-from-graph`. diff --git a/doc/lsst.ctrl.bps/quickstart.rst b/doc/lsst.ctrl.bps/quickstart.rst index b3692bcd..422f0de2 100644 --- a/doc/lsst.ctrl.bps/quickstart.rst +++ b/doc/lsst.ctrl.bps/quickstart.rst @@ -1161,7 +1161,7 @@ New YAML Section implementation: JOB concurrencyLimit: db_limit command1: >- - ${DAF_BUTLER_DIR}/bin/butler transfer-from-graph + ${DAF_BUTLER_DIR}/bin/butler aggregate-graph {fileDistributionEndPoint}{qgraphFile} {butlerConfig} --register-dataset-types diff --git a/python/lsst/ctrl/bps/etc/bps_defaults.yaml b/python/lsst/ctrl/bps/etc/bps_defaults.yaml index 1344f439..7b1e39a0 100644 --- a/python/lsst/ctrl/bps/etc/bps_defaults.yaml +++ b/python/lsst/ctrl/bps/etc/bps_defaults.yaml @@ -114,6 +114,7 @@ useLazyCommands: True executionButler: whenCreate: "NEVER" +extraAggregateOptions: "" finalJob: retryUnlessExit: 2 updateOutputChain: "--update-output-chain" @@ -124,12 +125,16 @@ finalJob: implementation: JOB concurrencyLimit: db_limit finalPreCmdOpts: "{defaultPreCmdOpts}" + requestCpus: 16 + requestMemory: 32768 command1: >- - ${DAF_BUTLER_DIR}/bin/butler {finalPreCmdOpts} transfer-from-graph + ${DAF_BUTLER_DIR}/bin/butler {finalPreCmdOpts} aggregate-graph {fileDistributionEndPoint}{qgraphFile} {butlerConfig} + -j {requestCpus} --register-dataset-types {updateOutputChain} + {extraAggregateOptions} # Set a global default memory limit (in MiB) for the automatic memory scaling # mechanism. The value, 480 GiB, picked based on the cluster specifications