From 42917987b4bd86645e3e2239543bbba334aa6089 Mon Sep 17 00:00:00 2001
From: Jim Bosch <jbosch@astro.princeton.edu>
Date: Tue, 7 Oct 2025 12:22:50 -0400
Subject: [PATCH 1/3] Switch default finalJob to aggregate-graph.

---
 doc/changes/DM-52360.perf.md               | 1 +
 doc/lsst.ctrl.bps/quickstart.rst           | 2 +-
 python/lsst/ctrl/bps/etc/bps_defaults.yaml | 8 +++++++-
 3 files changed, 9 insertions(+), 2 deletions(-)
 create mode 100644 doc/changes/DM-52360.perf.md

diff --git a/doc/changes/DM-52360.perf.md b/doc/changes/DM-52360.perf.md
new file mode 100644
index 00000000..31976c64
--- /dev/null
+++ b/doc/changes/DM-52360.perf.md
@@ -0,0 +1 @@
+Switch the default `finalJob` implementation to the new `aggregate-graph` command, which makes use of multiple cores much more effectively than `transfer-from-graph`.
diff --git a/doc/lsst.ctrl.bps/quickstart.rst b/doc/lsst.ctrl.bps/quickstart.rst
index b3692bcd..422f0de2 100644
--- a/doc/lsst.ctrl.bps/quickstart.rst
+++ b/doc/lsst.ctrl.bps/quickstart.rst
@@ -1161,7 +1161,7 @@ New YAML Section
      implementation: JOB
      concurrencyLimit: db_limit
      command1: >-
-       ${DAF_BUTLER_DIR}/bin/butler transfer-from-graph
+       ${DAF_BUTLER_DIR}/bin/butler aggregate-graph
        {fileDistributionEndPoint}{qgraphFile}
        {butlerConfig}
        --register-dataset-types
diff --git a/python/lsst/ctrl/bps/etc/bps_defaults.yaml b/python/lsst/ctrl/bps/etc/bps_defaults.yaml
index 1344f439..dad3f211 100644
--- a/python/lsst/ctrl/bps/etc/bps_defaults.yaml
+++ b/python/lsst/ctrl/bps/etc/bps_defaults.yaml
@@ -114,6 +114,8 @@ useLazyCommands: True
 executionButler:
   whenCreate: "NEVER"
 
+finalJobNumProcesses: 8
+extraAggregateOptions: ""
 finalJob:
   retryUnlessExit: 2
   updateOutputChain: "--update-output-chain"
@@ -124,12 +126,16 @@ finalJob:
   implementation: JOB
   concurrencyLimit: db_limit
   finalPreCmdOpts: "{defaultPreCmdOpts}"
+  requestCpus: "{finalJobNumProcesses}"
+  requestMemory: 16384
   command1: >-
-    ${DAF_BUTLER_DIR}/bin/butler {finalPreCmdOpts} transfer-from-graph
+    ${DAF_BUTLER_DIR}/bin/butler {finalPreCmdOpts} aggregate-graph
     {fileDistributionEndPoint}{qgraphFile}
     {butlerConfig}
+    -j {finalJobNumProcesses}
     --register-dataset-types
     {updateOutputChain}
+    {extraAggregateOptions}
 
 # Set a global default memory limit (in MiB) for the automatic memory scaling
 # mechanism. The value, 480 GiB, picked based on the cluster specifications

From 0b4ad5b17c0e28cc50f6bbf130218ed192a3edf2 Mon Sep 17 00:00:00 2001
From: Jim Bosch <jbosch@astro.princeton.edu>
Date: Mon, 20 Oct 2025 14:54:28 -0400
Subject: [PATCH 2/3] Update finalJob resource requests based on USDF
 benchmarks/sizes.

---
 python/lsst/ctrl/bps/etc/bps_defaults.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/lsst/ctrl/bps/etc/bps_defaults.yaml b/python/lsst/ctrl/bps/etc/bps_defaults.yaml
index dad3f211..27a55e70 100644
--- a/python/lsst/ctrl/bps/etc/bps_defaults.yaml
+++ b/python/lsst/ctrl/bps/etc/bps_defaults.yaml
@@ -114,7 +114,7 @@ useLazyCommands: True
 executionButler:
   whenCreate: "NEVER"
 
-finalJobNumProcesses: 8
+finalJobNumProcesses: 16
 extraAggregateOptions: ""
 finalJob:
   retryUnlessExit: 2
@@ -127,7 +127,7 @@ finalJob:
   concurrencyLimit: db_limit
   finalPreCmdOpts: "{defaultPreCmdOpts}"
   requestCpus: "{finalJobNumProcesses}"
-  requestMemory: 16384
+  requestMemory: 32768
   command1: >-
     ${DAF_BUTLER_DIR}/bin/butler {finalPreCmdOpts} aggregate-graph
     {fileDistributionEndPoint}{qgraphFile}

From 789878930b91f5f2568d89cfe5f930d0166052d6 Mon Sep 17 00:00:00 2001
From: Michelle Gower <mgower@illinois.edu>
Date: Mon, 20 Oct 2025 14:52:17 -0700
Subject: [PATCH 3/3] Replace finalJobNumProcesses with requestCpus.

---
 python/lsst/ctrl/bps/etc/bps_defaults.yaml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/python/lsst/ctrl/bps/etc/bps_defaults.yaml b/python/lsst/ctrl/bps/etc/bps_defaults.yaml
index 27a55e70..7b1e39a0 100644
--- a/python/lsst/ctrl/bps/etc/bps_defaults.yaml
+++ b/python/lsst/ctrl/bps/etc/bps_defaults.yaml
@@ -114,7 +114,6 @@ useLazyCommands: True
 executionButler:
   whenCreate: "NEVER"
 
-finalJobNumProcesses: 16
 extraAggregateOptions: ""
 finalJob:
   retryUnlessExit: 2
@@ -126,13 +125,13 @@ finalJob:
   implementation: JOB
   concurrencyLimit: db_limit
   finalPreCmdOpts: "{defaultPreCmdOpts}"
-  requestCpus: "{finalJobNumProcesses}"
+  requestCpus: 16
   requestMemory: 32768
   command1: >-
     ${DAF_BUTLER_DIR}/bin/butler {finalPreCmdOpts} aggregate-graph
     {fileDistributionEndPoint}{qgraphFile}
     {butlerConfig}
-    -j {finalJobNumProcesses}
+    -j {requestCpus}
     --register-dataset-types
     {updateOutputChain}
     {extraAggregateOptions}