$ conbench list (for orchestration)

conbench/cli.py

@@ -2,7 +2,7 @@
 
 import click
 
-from .runner import REGISTRY
+from .runner import REGISTRY, LIST
 from .util import Connection, register_benchmarks
 
 
@@ -53,6 +53,15 @@ def compare(baseline, contender, kind, threshold):
     print(json.dumps(result, indent=2))
 
 
+@conbench.command(name="list")
+def list_benchmarks():
+    """List of registered benchmarks to run (for orchestration)."""
+    benchmarks = []
+    if LIST:
+        benchmarks = LIST[0]().list(BENCHMARKS)
+    print(json.dumps(benchmarks, indent=2))
+
+
 def _option(params, name, default, _type, help_msg=None):
     params.append(
         click.Option(
@@ -100,8 +109,8 @@ def _to_cli_name(name):
 
     instance = benchmark()
     fields, cases = instance.fields, instance.cases
-    options = benchmark.options if hasattr(benchmark, "options") else {}
-    arguments = benchmark.arguments if hasattr(benchmark, "arguments") else []
+    options = getattr(benchmark, "options", {})
+    arguments = getattr(benchmark, "arguments", [])
     tags = [_to_cli_name(tag) for tag in fields]
     external = getattr(benchmark, "external", False)
 

conbench/runner.py

@@ -13,13 +13,20 @@
 
 
 REGISTRY = []
+LIST = []
 
 
 def register_benchmark(cls):
     REGISTRY.append(cls)
     return cls
 
 
+def register_list(cls):
+    if not LIST:
+        LIST.append(cls)
+    return cls
+
+
 def _now_formatted():
     now = datetime.datetime.now(datetime.timezone.utc)
     return now.isoformat()
@@ -71,6 +78,12 @@ def _get_case(self, case, options):
         return case
 
 
+class BenchmarkList(abc.ABC):
+    @abc.abstractmethod
+    def list(self, classes):
+        pass
+
+
 class Conbench(Connection):
     def __init__(self):
         super().__init__()

conbench/tests/benchmark/_example_benchmarks.py

@@ -1,6 +1,19 @@
 import conbench.runner
 
 
+@conbench.runner.register_list
+class BenchmarkList(conbench.runner.BenchmarkList):
+    def list(self, classes):
+        benchmarks = []
+        for name, benchmark in classes.items():
+            instance, parts = benchmark(), [name]
+            if instance.cases:
+                parts.append("--all=true")
+            parts.append(f"--iterations=2")
+            benchmarks.append({"command": " ".join(parts)})
+        return sorted(benchmarks, key=lambda k: k["command"])
+
+
 @conbench.runner.register_benchmark
 class WithoutCasesBenchmark(conbench.runner.Benchmark):
     name = "addition"

conbench/tests/benchmark/test_cli.py

@@ -14,9 +14,21 @@
 Commands:
   addition     Run addition benchmark.
   compare      Compare benchmark runs.
+  list         List of registered benchmarks to run (for orchestration).
   subtraction  Run subtraction benchmark(s).
 """
 
+CONBENCH_LIST = """
+[
+  {
+    "command": "addition --iterations=2"
+  },
+  {
+    "command": "subtraction --all=true --iterations=2"
+  }
+]
+"""
+
 
 CONBENCH_COMPARE = """
 TODO
@@ -199,6 +211,13 @@ def test_conbench_command_with_cases_help(runner):
     assert_command_output(result, CONBENCH_SUBTRACTION_HELP)
 
 
+def test_conbench_list(runner):
+    from conbench.cli import conbench
+
+    result = runner.invoke(conbench, "list")
+    assert_command_output(result, CONBENCH_LIST)
+
+
 def test_conbench_compare(runner):
     from conbench.cli import conbench
 

conbench/util.py

@@ -156,19 +156,25 @@ def register_benchmarks(directory=None):
     """Look for files matching the following patterns in the current
     working directory and import them.
 
-        bench_*.py
-        benchmark_*.py
-        *_bench.py
-        *_benchmark.py
-        *_benchmarks.py
+        benchmark*.py
+        *benchmark.py
+        *benchmarks.py
 
     This registers benchmarks that are decorated as conbench benchmarks.
 
         import conbench.runner
 
         @conbench.runner.register_benchmark
-        class ExampleBenchmark():
-            name = "example"
+        class ExampleBenchmark:
+            ...
+
+    It also registers the benchmark list class.
+
+        import conbench.runner
+
+        @conbench.runner.register_list
+        class ExampleBenchmarkList:
+            ...
     """
     if directory is None:
         directory = os.getcwd()
@@ -182,11 +188,9 @@ class ExampleBenchmark():
             ):
                 continue
             if (
-                filename.startswith("bench_")
-                or filename.startswith("benchmark_")
-                or filename.endswith("_bench.py")
-                or filename.endswith("_benchmark.py")
-                or filename.endswith("_benchmarks.py")
+                filename.startswith("benchmark")
+                or filename.endswith("benchmark.py")
+                or filename.endswith("benchmarks.py")
             ):
                 import_module(directory, filename)