From 22cc9dfb33dc70e1034b974f5c79194b2429f2b9 Mon Sep 17 00:00:00 2001
From: Peter Josef Scheibel <scheibel1@llnl.gov>
Date: Fri, 27 Jun 2025 16:25:13 -0700
Subject: [PATCH 01/16] partial script for aggregation

---
 aggregate.py | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 aggregate.py

diff --git a/aggregate.py b/aggregate.py
new file mode 100644
index 000000000..c0b14d047
--- /dev/null
+++ b/aggregate.py
@@ -0,0 +1,32 @@
+import argparse
+import os
+import re
+
+def main():
+    parser = argparse.ArgumentParser(description="Aggregate experiments from one or more workspaces")
+    parser.add_argument("output", help="Directory to generate scripts in")
+    parser.add_argument("workspaces", nargs=argparse.REMAINDER, help="One or more benchpark workspaces")
+
+    args = parser.parse_args()
+
+    if os.path.exists(args.output):
+        raise ValueError(f"Directory must not already exist: {args.output}")
+
+    experiments = list()
+    for workspace_dir in args.workspaces:
+        experiments.extend(collect_experiments(workspace_dir))
+
+    import pdb; pdb.set_trace()
+
+def collect_experiments(workspace_dir):
+    experiments = list()
+    for entry in os.listdir(workspace_dir):
+        if entry in ["spack", "ramble"] or not os.path.isdir(entry):
+            continue
+        for dirpath, dirnames, filenames in os.walk(os.path.join(workspace_dir, entry)):
+            for fname in filenames:
+                experiments.append(os.path.join(dirpath, fname)
+    return experiments
+
+if __name__ == "__main__":
+    main()

From 095d64bc8d4166b3df8e0c12b616131a807b2b30 Mon Sep 17 00:00:00 2001
From: Peter Josef Scheibel <scheibel1@llnl.gov>
Date: Fri, 27 Jun 2025 16:27:50 -0700
Subject: [PATCH 02/16] bugfix and look for just experiment scripts

---
 aggregate.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/aggregate.py b/aggregate.py
index c0b14d047..24f30e93d 100644
--- a/aggregate.py
+++ b/aggregate.py
@@ -25,7 +25,8 @@ def collect_experiments(workspace_dir):
             continue
         for dirpath, dirnames, filenames in os.walk(os.path.join(workspace_dir, entry)):
             for fname in filenames:
-                experiments.append(os.path.join(dirpath, fname)
+                if fname == "execute_experiment":
+                    experiments.append(os.path.join(dirpath, fname))
     return experiments
 
 if __name__ == "__main__":

From 20f99cc23b2d06b27161da69bc6b6dce15084a2a Mon Sep 17 00:00:00 2001
From: Peter Josef Scheibel <scheibel1@llnl.gov>
Date: Fri, 27 Jun 2025 17:06:30 -0700
Subject: [PATCH 03/16] completed script for flux: generates 1 grouped script
 for each unique batch request (so e.g. all 1-node requests will go into one
 script)

---
 aggregate.py | 45 ++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 44 insertions(+), 1 deletion(-)

diff --git a/aggregate.py b/aggregate.py
index 24f30e93d..1ea3e0fad 100644
--- a/aggregate.py
+++ b/aggregate.py
@@ -1,4 +1,5 @@
 import argparse
+from collections import defaultdict
 import os
 import re
 
@@ -16,7 +17,49 @@ def main():
     for workspace_dir in args.workspaces:
         experiments.extend(collect_experiments(workspace_dir))
 
-    import pdb; pdb.set_trace()
+    opts_to_request = {}
+    opts_to_scripts = defaultdict(list)
+    for experiment_script in experiments:
+        batch_opts, batch_lines = collect_scheduler_options(experiment_script)
+        if batch_opts:
+            opts_to_scripts[batch_opts].append(experiment_script)
+            opts_to_request[batch_opts] = batch_lines
+        else:
+            #import pdb; pdb.set_trace()
+            raise Exception(f"Not expected: no batch opts in {experiment_script}")
+
+    os.mkdir(args.output)
+    script_id = 0
+    for opts_group, scripts in opts_to_scripts.items():
+        script_path = os.path.join(args.output, f"{script_id}.sh")
+        make_aggregate_script(script_path, opts_to_request[opts_group], scripts)
+        script_id += 1
+
+def make_aggregate_script(script_path, batch_lines, to_aggregate):
+    with open(script_path, "w") as f:
+        for line in batch_lines:
+            f.write(line + "\n")
+        for experiment in to_aggregate:
+            f.write(os.path.abspath(experiment) + "\n")
+
+def collect_scheduler_options(experiment_script):
+    batch_patterns = [
+        r"\s*#\s*(flux:.*$)"
+    ]
+    batch_opts = list()
+    batch_lines = list()
+    with open(experiment_script, "r") as f:
+        for line in f.readlines():
+            for p in batch_patterns:
+                m = re.match(p, line)
+                if m:
+                    batch_opts.append(tuple(m.group(1).strip().split()))
+                    batch_lines.append(line.strip())
+                #elif "flux" in line:
+                #    import pdb; pdb.set_trace()
+                #    print('hi')
+
+    return tuple(sorted(batch_opts)), batch_lines
 
 def collect_experiments(workspace_dir):
     experiments = list()

From e5bd59bee074b2f346321ee83ac206ea84e3f233 Mon Sep 17 00:00:00 2001
From: Peter Josef Scheibel <scheibel1@llnl.gov>
Date: Mon, 30 Jun 2025 13:28:40 -0700
Subject: [PATCH 04/16] rm debugging code

---
 aggregate.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/aggregate.py b/aggregate.py
index 1ea3e0fad..1817cba49 100644
--- a/aggregate.py
+++ b/aggregate.py
@@ -25,7 +25,6 @@ def main():
             opts_to_scripts[batch_opts].append(experiment_script)
             opts_to_request[batch_opts] = batch_lines
         else:
-            #import pdb; pdb.set_trace()
             raise Exception(f"Not expected: no batch opts in {experiment_script}")
 
     os.mkdir(args.output)
@@ -55,9 +54,6 @@ def collect_scheduler_options(experiment_script):
                 if m:
                     batch_opts.append(tuple(m.group(1).strip().split()))
                     batch_lines.append(line.strip())
-                #elif "flux" in line:
-                #    import pdb; pdb.set_trace()
-                #    print('hi')
 
     return tuple(sorted(batch_opts)), batch_lines
 

From bbda84efbcfd7003c67ebe75c8f801dd938dc53f Mon Sep 17 00:00:00 2001
From: Peter Josef Scheibel <scheibel1@llnl.gov>
Date: Mon, 30 Jun 2025 13:30:24 -0700
Subject: [PATCH 05/16] add comment about handling schedulers besides flux

---
 aggregate.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/aggregate.py b/aggregate.py
index 1817cba49..f8127ae06 100644
--- a/aggregate.py
+++ b/aggregate.py
@@ -42,6 +42,7 @@ def make_aggregate_script(script_path, batch_lines, to_aggregate):
             f.write(os.path.abspath(experiment) + "\n")
 
 def collect_scheduler_options(experiment_script):
+    # Should only take 1 line per scheduler to handle all schedulers
     batch_patterns = [
         r"\s*#\s*(flux:.*$)"
     ]

From b094169d125dc25269d759d3cde9b8e1e507bc6f Mon Sep 17 00:00:00 2001
From: Michael McKinsey <michaelmckinsey1@gmail.com>
Date: Fri, 5 Sep 2025 15:40:22 -0700
Subject: [PATCH 06/16] Work for slurm

---
 aggregate.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/aggregate.py b/aggregate.py
index f8127ae06..3e2c06363 100644
--- a/aggregate.py
+++ b/aggregate.py
@@ -44,7 +44,8 @@ def make_aggregate_script(script_path, batch_lines, to_aggregate):
 def collect_scheduler_options(experiment_script):
     # Should only take 1 line per scheduler to handle all schedulers
     batch_patterns = [
-        r"\s*#\s*(flux:.*$)"
+        r"\s*#\s*(flux:.*$)",
+        r"\s*#SBATCH\s+(.*)$",
     ]
     batch_opts = list()
     batch_lines = list()

From 64ebe00542cc017ea7b32510e8e298d33e5767cd Mon Sep 17 00:00:00 2001
From: Michael McKinsey <michaelmckinsey1@gmail.com>
Date: Mon, 10 Nov 2025 18:37:54 -0800
Subject: [PATCH 07/16] Make command

---
 aggregate.py                   |  74 --------------------
 lib/benchpark/cmd/aggregate.py | 120 +++++++++++++++++++++++++++++++++
 lib/main.py                    |  16 ++++-
 3 files changed, 133 insertions(+), 77 deletions(-)
 delete mode 100644 aggregate.py
 create mode 100644 lib/benchpark/cmd/aggregate.py

diff --git a/aggregate.py b/aggregate.py
deleted file mode 100644
index 3e2c06363..000000000
--- a/aggregate.py
+++ /dev/null
@@ -1,74 +0,0 @@
-import argparse
-from collections import defaultdict
-import os
-import re
-
-def main():
-    parser = argparse.ArgumentParser(description="Aggregate experiments from one or more workspaces")
-    parser.add_argument("output", help="Directory to generate scripts in")
-    parser.add_argument("workspaces", nargs=argparse.REMAINDER, help="One or more benchpark workspaces")
-
-    args = parser.parse_args()
-
-    if os.path.exists(args.output):
-        raise ValueError(f"Directory must not already exist: {args.output}")
-
-    experiments = list()
-    for workspace_dir in args.workspaces:
-        experiments.extend(collect_experiments(workspace_dir))
-
-    opts_to_request = {}
-    opts_to_scripts = defaultdict(list)
-    for experiment_script in experiments:
-        batch_opts, batch_lines = collect_scheduler_options(experiment_script)
-        if batch_opts:
-            opts_to_scripts[batch_opts].append(experiment_script)
-            opts_to_request[batch_opts] = batch_lines
-        else:
-            raise Exception(f"Not expected: no batch opts in {experiment_script}")
-
-    os.mkdir(args.output)
-    script_id = 0
-    for opts_group, scripts in opts_to_scripts.items():
-        script_path = os.path.join(args.output, f"{script_id}.sh")
-        make_aggregate_script(script_path, opts_to_request[opts_group], scripts)
-        script_id += 1
-
-def make_aggregate_script(script_path, batch_lines, to_aggregate):
-    with open(script_path, "w") as f:
-        for line in batch_lines:
-            f.write(line + "\n")
-        for experiment in to_aggregate:
-            f.write(os.path.abspath(experiment) + "\n")
-
-def collect_scheduler_options(experiment_script):
-    # Should only take 1 line per scheduler to handle all schedulers
-    batch_patterns = [
-        r"\s*#\s*(flux:.*$)",
-        r"\s*#SBATCH\s+(.*)$",
-    ]
-    batch_opts = list()
-    batch_lines = list()
-    with open(experiment_script, "r") as f:
-        for line in f.readlines():
-            for p in batch_patterns:
-                m = re.match(p, line)
-                if m:
-                    batch_opts.append(tuple(m.group(1).strip().split()))
-                    batch_lines.append(line.strip())
-
-    return tuple(sorted(batch_opts)), batch_lines
-
-def collect_experiments(workspace_dir):
-    experiments = list()
-    for entry in os.listdir(workspace_dir):
-        if entry in ["spack", "ramble"] or not os.path.isdir(entry):
-            continue
-        for dirpath, dirnames, filenames in os.walk(os.path.join(workspace_dir, entry)):
-            for fname in filenames:
-                if fname == "execute_experiment":
-                    experiments.append(os.path.join(dirpath, fname))
-    return experiments
-
-if __name__ == "__main__":
-    main()
diff --git a/lib/benchpark/cmd/aggregate.py b/lib/benchpark/cmd/aggregate.py
new file mode 100644
index 000000000..702fe9fad
--- /dev/null
+++ b/lib/benchpark/cmd/aggregate.py
@@ -0,0 +1,120 @@
+# Copyright 2025 Lawrence Livermore National Security, LLC
+# and other Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import re
+from collections import defaultdict
+
+from benchpark.error import BenchparkError
+
+
+def _make_aggregate_script(script_path, batch_lines, to_aggregate):
+    with open(script_path, "w") as f:
+        for line in batch_lines:
+            f.write(line.rstrip("\n") + "\n")
+        for experiment in to_aggregate:
+            f.write(os.path.abspath(experiment) + "\n")
+
+
+def _collect_scheduler_options(experiment_script):
+    batch_patterns = [
+        r"\s*#\s*(flux:.*$)",   # flux-style header (kept verbatim)
+        r"\s*#SBATCH\s+(.*)$",  # SLURM
+    ]
+
+    batch_opts = []
+    batch_lines = []
+
+    with open(experiment_script, "r") as f:
+        for line in f:
+            for p in batch_patterns:
+                m = re.match(p, line)
+                if m:
+                    batch_opts.append(tuple(m.group(1).strip().split()))
+                    batch_lines.append(line.strip())
+
+    return tuple(sorted(batch_opts)), batch_lines
+
+
+def _collect_experiments(workspace_dir):
+    if not os.path.isdir(workspace_dir):
+        raise BenchparkError(
+            f"Workspace path does not exist or is not a directory: {workspace_dir}"
+        )
+
+    experiments = []
+    skip_roots = {"spack", "ramble"}
+
+    for entry in os.listdir(workspace_dir):
+        entry_path = os.path.join(workspace_dir, entry)
+        if entry in skip_roots or not os.path.isdir(entry_path):
+            continue
+
+        for dirpath, _dirnames, filenames in os.walk(entry_path):
+            for fname in filenames:
+                if fname == "execute_experiment":
+                    experiments.append(os.path.join(dirpath, fname))
+
+    return experiments
+
+
+def _aggregate(args):
+    output_dir = args.dest
+    workspaces = args.workspaces
+
+    if not workspaces:
+        raise BenchparkError("No workspaces provided.")
+
+    if os.path.exists(output_dir):
+        raise BenchparkError(f"Directory must not already exist: {output_dir}")
+
+    experiments = []
+    for ws in workspaces:
+        experiments.extend(_collect_experiments(ws))
+
+    if not experiments:
+        raise BenchparkError(
+            "No 'execute_experiment' scripts found in the given workspaces."
+        )
+
+    opts_to_request = {}
+    opts_to_scripts = defaultdict(list)
+
+    for experiment_script in experiments:
+        batch_opts, batch_lines = _collect_scheduler_options(experiment_script)
+        if not batch_opts:
+            raise BenchparkError(f"Not expected: no batch opts in {experiment_script}")
+
+        opts_to_scripts[batch_opts].append(experiment_script)
+        if batch_opts not in opts_to_request:
+            opts_to_request[batch_opts] = batch_lines
+
+    os.mkdir(output_dir)
+    script_id = 0
+    for opts_group, scripts in opts_to_scripts.items():
+        script_path = os.path.join(output_dir, f"{script_id}.sh")
+        _make_aggregate_script(script_path, opts_to_request[opts_group], scripts)
+        script_id += 1
+
+def setup_parser(root_parser):
+    """
+    Register arguments for `benchpark aggregate` directly (no subcommands).
+    Usage:
+        benchpark aggregate --dest OUTDIR WS1 [WS2 ...]
+    """
+    root_parser.add_argument(
+        "--dest",
+        required=True,
+        help="Directory to generate aggregate scripts in",
+    )
+    root_parser.add_argument(
+        "workspaces",
+        nargs="+",
+        help="One or more Benchpark workspace directories",
+    )
+
+
+def command(args):
+    _aggregate(args)
diff --git a/lib/main.py b/lib/main.py
index 33d6d03e5..ecf3b34c7 100755
--- a/lib/main.py
+++ b/lib/main.py
@@ -17,7 +17,7 @@
 if "-V" in sys.argv or "--version" in sys.argv:
     print(__version__)
     exit()
-helpstr = """usage: main.py [-h] [-V] {tags,system,experiment,setup,unit-test,audit,mirror,info,show-build,list,bootstrap,analyze} ...
+helpstr = """usage: main.py [-h] [-V] {tags,system,experiment,setup,unit-test,audit,mirror,info,show-build,list,bootstrap,analyze,aggregate} ...
 
 Benchpark
 
@@ -26,7 +26,7 @@
   -V, --version         show version number and exit
 
 Subcommands:
-  {tags,system,experiment,setup,unit-test,audit,mirror,info,show-build,list,bootstrap,analyze}
+  {tags,system,experiment,setup,unit-test,audit,mirror,info,show-build,list,bootstrap,analyze,aggregate}
     tags                Tags in Benchpark experiments
     system              Initialize a system config
     experiment          Interact with experiments
@@ -38,7 +38,9 @@
     show-build          Show how spack built a benchmark
     list                List experiments, systems, benchmarks, and modifiers
     bootstrap           Bootstrap benchpark or update an existing bootstrap
-    analyze             Perform pre-defined analysis on the performance data (caliper files) after 'ramble on'"""
+    analyze             Perform pre-defined analysis on the performance data (caliper files) after 'ramble on'
+    aggregate           Aggregate multiple experiments (even across workspaces) into the same submission script
+    """
 if len(sys.argv) == 1 or "-h" == sys.argv[1] or "--help" == sys.argv[1]:
     print(helpstr)
     exit()
@@ -59,6 +61,7 @@
 import benchpark.cmd.show_build  # noqa: E402
 import benchpark.cmd.system  # noqa: E402
 import benchpark.cmd.unit_test  # noqa: E402
+import benchpark.cmd.aggregate # noqa: E402
 from benchpark.accounting import benchpark_benchmarks  # noqa: E402
 
 try:
@@ -225,6 +228,12 @@ def init_commands(subparsers, actions_dict):
         help="Perform pre-defined analysis on the performance data (caliper files) after 'ramble on'",
     )
 
+    aggregate_parser = subparsers.add_parser(
+        "aggregate",
+        help="Aggregate multiple experiments (even across workspaces) into the same submission script",
+    )
+    benchpark.cmd.aggregate.setup_parser(aggregate_parser)
+
     actions_dict["system"] = benchpark.cmd.system.command
     actions_dict["experiment"] = benchpark.cmd.experiment.command
     actions_dict["setup"] = benchpark.cmd.setup.command
@@ -235,6 +244,7 @@ def init_commands(subparsers, actions_dict):
     actions_dict["show-build"] = benchpark.cmd.show_build.command
     actions_dict["list"] = benchpark.cmd.list.command
     actions_dict["bootstrap"] = benchpark.cmd.bootstrap.command
+    actions_dict["aggregate"] = benchpark.cmd.aggregate.command
     if analyze_installed:
         benchpark.cmd.analyze.setup_parser(analyze_parser)
         actions_dict["analyze"] = benchpark.cmd.analyze.command

From 5962425fab916405f463dbb38879f514e85d9b1e Mon Sep 17 00:00:00 2001
From: Michael McKinsey <michaelmckinsey1@gmail.com>
Date: Mon, 10 Nov 2025 18:47:52 -0800
Subject: [PATCH 08/16] Add docs

---
 docs/run-experiment.rst | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/docs/run-experiment.rst b/docs/run-experiment.rst
index 500c35eb2..997e39154 100644
--- a/docs/run-experiment.rst
+++ b/docs/run-experiment.rst
@@ -48,4 +48,23 @@ the experiments benchpark would run in the second run. Generally, we would advis
 user to remove the ``$workspace/experiments`` directory before re-running the
 experiments using ``ramble --workspace-dir . on``.
 
+Running Different Experiments in the Same Allocation
+====================================================
+
+The ``benchpark aggregate`` command provides the functionality to batch experiments from
+the same workspace or multiple workspaces together. To use ``benchpark aggregate``,
+provide a destination for the job script, and the workspaces as positional arguments:
+
+::
+
+    $ benchpark aggregate --dest agg/ workspace1/ workspace2/ ...
+
+Then, submit the newly created job script to the scheduler:
+
+::
+    $ flux batch agg/0.sh
+
+Output from the experiments will still be written to the respective experiment
+directory.
+
 Once you have run your experiment you can try :doc:`analyze-experiment`.

From 2953e045a537a43311ae32bb7d0b3bd4767e28eb Mon Sep 17 00:00:00 2001
From: Michael McKinsey <michaelmckinsey1@gmail.com>
Date: Mon, 10 Nov 2025 18:53:07 -0800
Subject: [PATCH 09/16] lint

---
 lib/benchpark/cmd/aggregate.py | 3 ++-
 lib/main.py                    | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/benchpark/cmd/aggregate.py b/lib/benchpark/cmd/aggregate.py
index 702fe9fad..83411786b 100644
--- a/lib/benchpark/cmd/aggregate.py
+++ b/lib/benchpark/cmd/aggregate.py
@@ -20,7 +20,7 @@ def _make_aggregate_script(script_path, batch_lines, to_aggregate):
 
 def _collect_scheduler_options(experiment_script):
     batch_patterns = [
-        r"\s*#\s*(flux:.*$)",   # flux-style header (kept verbatim)
+        r"\s*#\s*(flux:.*$)",  # flux-style header (kept verbatim)
         r"\s*#SBATCH\s+(.*)$",  # SLURM
     ]
 
@@ -98,6 +98,7 @@ def _aggregate(args):
         _make_aggregate_script(script_path, opts_to_request[opts_group], scripts)
         script_id += 1
 
+
 def setup_parser(root_parser):
     """
     Register arguments for `benchpark aggregate` directly (no subcommands).
diff --git a/lib/main.py b/lib/main.py
index ecf3b34c7..57241e3ec 100755
--- a/lib/main.py
+++ b/lib/main.py
@@ -61,7 +61,7 @@
 import benchpark.cmd.show_build  # noqa: E402
 import benchpark.cmd.system  # noqa: E402
 import benchpark.cmd.unit_test  # noqa: E402
-import benchpark.cmd.aggregate # noqa: E402
+import benchpark.cmd.aggregate  # noqa: E402
 from benchpark.accounting import benchpark_benchmarks  # noqa: E402
 
 try:

From 17c1a300a8f87a9ab06d49279ac54940984d55cd Mon Sep 17 00:00:00 2001
From: Michael McKinsey <michaelmckinsey1@gmail.com>
Date: Mon, 10 Nov 2025 18:58:01 -0800
Subject: [PATCH 10/16] Add test

---
 .github/workflows/run.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml
index 4cc7a2d9c..bd3a02535 100644
--- a/.github/workflows/run.yml
+++ b/.github/workflows/run.yml
@@ -132,6 +132,11 @@ jobs:
             on \
             --executor '{execute_experiment}' \
             --where '{n_nodes} == 1'
+      - name: Test aggregate Command
+        run: |
+          ./bin/benchpark experiment init --dest=saxpy-agg --system=generic-x86 saxpy n_repeats=5
+          ./bin/benchpark setup saxpy-agg workspace2/
+          ./bin/benchpark aggregate agg workspace2 workspace  # Aggregate multi-trial and single-trial workspace
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         with:

From d4086f18ff4653d8b98989dbbc4a7a362a0ea513 Mon Sep 17 00:00:00 2001
From: Michael McKinsey <michaelmckinsey1@gmail.com>
Date: Mon, 10 Nov 2025 19:00:25 -0800
Subject: [PATCH 11/16] isort

---
 lib/main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/main.py b/lib/main.py
index 57241e3ec..42ecfc716 100755
--- a/lib/main.py
+++ b/lib/main.py
@@ -51,6 +51,7 @@
 bootstrapper = RuntimeResources(benchpark.paths.benchpark_home)  # noqa
 bootstrapper.bootstrap()  # noqa
 
+import benchpark.cmd.aggregate  # noqa: E402
 import benchpark.cmd.audit  # noqa: E402
 import benchpark.cmd.bootstrap  # noqa: E402
 import benchpark.cmd.experiment  # noqa: E402
@@ -61,7 +62,6 @@
 import benchpark.cmd.show_build  # noqa: E402
 import benchpark.cmd.system  # noqa: E402
 import benchpark.cmd.unit_test  # noqa: E402
-import benchpark.cmd.aggregate  # noqa: E402
 from benchpark.accounting import benchpark_benchmarks  # noqa: E402
 
 try:

From 13e513d0ed6b136c7c50c47e5139e8bcc1d4b3a1 Mon Sep 17 00:00:00 2001
From: Michael McKinsey <michaelmckinsey1@gmail.com>
Date: Mon, 10 Nov 2025 19:46:47 -0800
Subject: [PATCH 12/16] Fix test

---
 .github/workflows/run.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml
index bd3a02535..ecab47a4b 100644
--- a/.github/workflows/run.yml
+++ b/.github/workflows/run.yml
@@ -136,7 +136,7 @@ jobs:
         run: |
           ./bin/benchpark experiment init --dest=saxpy-agg --system=generic-x86 saxpy n_repeats=5
           ./bin/benchpark setup saxpy-agg workspace2/
-          ./bin/benchpark aggregate agg workspace2 workspace  # Aggregate multi-trial and single-trial workspace
+          ./bin/benchpark aggregate --dest agg workspace2 workspace  # Aggregate multi-trial and single-trial workspace
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         with:

From bf5835bfb651cf16f328d5a7cfe2f154ceeeec67 Mon Sep 17 00:00:00 2001
From: Michael McKinsey <michaelmckinsey1@gmail.com>
Date: Tue, 11 Nov 2025 09:29:08 -0800
Subject: [PATCH 13/16] Need system with scheduler

---
 .github/workflows/run.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml
index ecab47a4b..89ea79286 100644
--- a/.github/workflows/run.yml
+++ b/.github/workflows/run.yml
@@ -134,9 +134,10 @@ jobs:
             --where '{n_nodes} == 1'
       - name: Test aggregate Command
         run: |
-          ./bin/benchpark experiment init --dest=saxpy-agg --system=generic-x86 saxpy n_repeats=5
+          ./bin/benchpark system init --dest=dane llnl-cluster cluster=dane
+          ./bin/benchpark experiment init --dest=saxpy-agg --system=dane saxpy n_repeats=5
           ./bin/benchpark setup saxpy-agg workspace2/
-          ./bin/benchpark aggregate --dest agg workspace2 workspace  # Aggregate multi-trial and single-trial workspace
+          ./bin/benchpark aggregate --dest agg workspace2  # Aggregate multi-trial workspace
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         with:

From e75ee51a3413f57a7e109b1b471db7fc93213117 Mon Sep 17 00:00:00 2001
From: Michael McKinsey <michaelmckinsey1@gmail.com>
Date: Tue, 11 Nov 2025 09:31:42 -0800
Subject: [PATCH 14/16] Add aggregate to codecov

---
 .github/workflows/run.yml | 12 ++++++++++--
 codecov.yml               |  2 +-
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml
index 89ea79286..1e59306ac 100644
--- a/.github/workflows/run.yml
+++ b/.github/workflows/run.yml
@@ -137,8 +137,8 @@ jobs:
           ./bin/benchpark system init --dest=dane llnl-cluster cluster=dane
           ./bin/benchpark experiment init --dest=saxpy-agg --system=dane saxpy n_repeats=5
           ./bin/benchpark setup saxpy-agg workspace2/
-          ./bin/benchpark aggregate --dest agg workspace2  # Aggregate multi-trial workspace
-      - name: Upload coverage to Codecov
+          BENCHPARK_RUN_COVERAGE=aggregate ./bin/benchpark aggregate --dest agg workspace2  # Aggregate multi-trial workspace
+      - name: Upload binary coverage to Codecov
         uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.BENCHPARK_CODECOV_TOKEN }}
@@ -146,6 +146,14 @@ jobs:
           flags: dryrunexperiments-binary
           verbose: true
           fail_ci_if_error: true
+      - name: Upload aggregate coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.BENCHPARK_CODECOV_TOKEN }}
+          directory: ./coverage-data-aggregate
+          flags: dryrunexperiments-aggregate
+          verbose: true
+          fail_ci_if_error: true
   dryrunexperiments:
     runs-on: ubuntu-24.04
     strategy:
diff --git a/codecov.yml b/codecov.yml
index ebf9e1603..246a2b4c8 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -7,4 +7,4 @@ comment:
   require_changes: false
   require_base: false
   require_head: true
-  after_n_builds: 10  # Number of SEPARATE codecov uploads expected, e.g. "Upload coverage to Codecov" jobs
+  after_n_builds: 11  # Number of SEPARATE codecov uploads expected, e.g. "Upload coverage to Codecov" jobs

From 7320323a15fb2d91f56fb73e4822b6bc4549dbc0 Mon Sep 17 00:00:00 2001
From: Michael McKinsey <MichaelMcKinsey1@gmail.com>
Date: Tue, 11 Nov 2025 14:03:55 -0800
Subject: [PATCH 15/16] Update aggregate.py

---
 lib/benchpark/cmd/aggregate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/benchpark/cmd/aggregate.py b/lib/benchpark/cmd/aggregate.py
index 83411786b..fc60eff51 100644
--- a/lib/benchpark/cmd/aggregate.py
+++ b/lib/benchpark/cmd/aggregate.py
@@ -45,7 +45,7 @@ def _collect_experiments(workspace_dir):
         )
 
     experiments = []
-    skip_roots = {"spack", "ramble"}
+    skip_roots = {"spack", "ramble", "spack-packages"}
 
     for entry in os.listdir(workspace_dir):
         entry_path = os.path.join(workspace_dir, entry)

From 8dcd5ac848c4a8fdea1218fbd9e9482ab56245f5 Mon Sep 17 00:00:00 2001
From: Michael McKinsey <michaelmckinsey1@gmail.com>
Date: Wed, 12 Nov 2025 09:15:51 -0800
Subject: [PATCH 16/16] Fix test

---
 .github/workflows/run.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml
index 1e59306ac..0aca51f84 100644
--- a/.github/workflows/run.yml
+++ b/.github/workflows/run.yml
@@ -137,6 +137,7 @@ jobs:
           ./bin/benchpark system init --dest=dane llnl-cluster cluster=dane
           ./bin/benchpark experiment init --dest=saxpy-agg --system=dane saxpy n_repeats=5
           ./bin/benchpark setup saxpy-agg workspace2/
+          ./workspace2/ramble/bin/ramble --workspace-dir /home/runner/work/benchpark/benchpark/workspace2/saxpy-agg/dane/workspace workspace setup --dry-run
           BENCHPARK_RUN_COVERAGE=aggregate ./bin/benchpark aggregate --dest agg workspace2  # Aggregate multi-trial workspace
       - name: Upload binary coverage to Codecov
         uses: codecov/codecov-action@v4