CLI - Add command sb benchmark [list,list-parameters] (#279)

__Description__

Add command `sb benchmark list` and `sb benchmark list-parameters` to support listing all optional parameters for benchmarks.

<details>
<summary>Examples</summary>
<pre>
$ sb benchmark list -n [a-z]+-bw -o table
Result
--------
mem-bw
nccl-bw
rccl-bw
</pre>
<pre>
$ sb benchmark list-parameters -n mem-bw
=== mem-bw ===
optional arguments:
  --bin_dir str         Specify the directory of the benchmark binary.
  --duration int        The elapsed time of benchmark in seconds.
  --mem_type str [str ...]
                        Memory types to benchmark. E.g. htod dtoh dtod.
  --memory str          Memory argument for bandwidthtest. E.g. pinned unpinned.
  --run_count int       The run count of benchmark.
  --shmoo_mode          Enable shmoo mode for bandwidthtest.
default values:
{'bin_dir': None,
 'duration': 0,
 'mem_type': ['htod', 'dtoh'],
 'memory': 'pinned',
 'run_count': 1}
</pre>
</details>

__Major Revisions__
* Add `sb benchmark list` to list benchmarks matching given name.
* Add `sb benchmark list-parameters` to list parameters for benchmarks which match given name.

__Minor Revisions__
* Sort format help text for argparse.

docs/cli.md

@@ -24,6 +24,68 @@ Welcome to the SB CLI!
 
 The following lists `sb` commands usages and examples:
 
+### `sb benchmark list`
+
+List benchmarks which match the regular expression.
+```bash title="SB CLI"
+sb benchmark list [--name]
+```
+
+#### Optional arguments
+
+| Name          | Default | Description                           |
+|---------------|---------|---------------------------------------|
+| `--name` `-n` | `None`  | Benchmark name or regular expression. |
+
+#### Global arguments
+
+| Name          | Default | Description        |
+|---------------|---------|--------------------|
+| `--help` `-h` | N/A     | Show help message. |
+
+#### Examples
+
+List all benchmarks:
+```bash title="SB CLI"
+sb benchmark list
+```
+
+List all benchmarks ending with "-bw":
+```bash title="SB CLI"
+sb benchmark list --name [a-z]+-bw
+```
+
+### `sb benchmark list-parameters`
+
+List parameters for benchmarks which match the regular expression.
+```bash title="SB CLI"
+sb benchmark list-parameters [--name]
+```
+
+#### Optional arguments
+
+| Name          | Default | Description                           |
+|---------------|---------|---------------------------------------|
+| `--name` `-n` | `None`  | Benchmark name or regular expression. |
+
+#### Global arguments
+
+| Name          | Default | Description        |
+|---------------|---------|--------------------|
+| `--help` `-h` | N/A     | Show help message. |
+
+#### Examples
+
+List parameters for all benchmarks:
+```bash title="SB CLI"
+sb benchmark list-parameters
+```
+
+List parameters for all benchmarks which starts with "pytorch-":
+```bash title="SB CLI"
+sb benchmark list-parameters --name pytorch-[a-z]+
+```
+
 ### `sb deploy`
 
 Deploy the SuperBench environments to all managed nodes.
@@ -217,4 +279,3 @@ Print version:
 ```bash title="SB CLI"
 sb version
 ```
-

superbench/benchmarks/base.py

@@ -6,13 +6,25 @@
 import argparse
 import numbers
 from datetime import datetime
+from operator import attrgetter
 from abc import ABC, abstractmethod
 
 from superbench.common.utils import logger
 from superbench.benchmarks import BenchmarkType, ReturnCode
 from superbench.benchmarks.result import BenchmarkResult
 
 
+class SortedMetavarTypeHelpFormatter(argparse.MetavarTypeHelpFormatter):
+    """Custom HelpFormatter class for argparse which sorts option strings."""
+    def add_arguments(self, actions):
+        """Sort option strings before original add_arguments.
+
+        Args:
+            actions (argparse.Action): Argument parser actions.
+        """
+        super(SortedMetavarTypeHelpFormatter, self).add_arguments(sorted(actions, key=attrgetter('option_strings')))
+
+
 class Benchmark(ABC):
     """The base class of all benchmarks."""
     def __init__(self, name, parameters=''):
@@ -29,7 +41,7 @@ def __init__(self, name, parameters=''):
             add_help=False,
             usage=argparse.SUPPRESS,
             allow_abbrev=False,
-            formatter_class=argparse.MetavarTypeHelpFormatter
+            formatter_class=SortedMetavarTypeHelpFormatter,
         )
         self._args = None
         self._curr_run_index = 0

superbench/cli/_benchmark_handler.py

@@ -0,0 +1,59 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""SuperBench CLI benchmark subgroup command handler."""
+
+import re
+from pprint import pformat
+
+from knack.util import CLIError
+
+from superbench.benchmarks import Platform, BenchmarkRegistry
+
+
+def benchmark_list_command_handler(name=None):
+    """List benchmarks which match the regular expression.
+
+    Args:
+        name (str, optional): Benchmark name or regular expression. Defaults to None.
+
+    Raises:
+        CLIError: If cannot find the matching benchmark.
+
+    Returns:
+        list: Benchmark list.
+    """
+    benchmark_list = list(BenchmarkRegistry.get_all_benchmark_predefine_settings().keys())
+    if name is None:
+        return benchmark_list
+    filter_list = list(filter(re.compile(name).match, benchmark_list))
+    if not filter_list:
+        raise CLIError('Benchmark {} does not exist.'.format(name))
+    return filter_list
+
+
+def benchmark_list_params_command_handler(name=None):
+    """List parameters for benchmarks which match the regular expression.
+
+    Args:
+        name (str, optional): Benchmark name or regular expression. Defaults to None.
+
+    Raises:
+        CLIError: If cannot find the matching benchmark.
+    """
+    for benchmark_name in benchmark_list_command_handler(name):
+        format_help = ''
+        for platform in Platform:
+            if platform in BenchmarkRegistry.benchmarks[benchmark_name]:
+                format_help = BenchmarkRegistry.get_benchmark_configurable_settings(
+                    BenchmarkRegistry.create_benchmark_context(benchmark_name, platform=platform)
+                )
+                break
+        print(
+            (
+                f'=== {benchmark_name} ===\n\n'
+                f'{format_help}\n\n'
+                f'default values:\n'
+                f'{pformat(BenchmarkRegistry.benchmarks[benchmark_name]["predefine_param"])}\n'
+            )
+        )

superbench/cli/_commands.py

@@ -23,6 +23,9 @@ def load_command_table(self, args):
             g.command('deploy', 'deploy_command_handler')
             g.command('exec', 'exec_command_handler')
             g.command('run', 'run_command_handler')
+        with CommandGroup(self, 'benchmark', 'superbench.cli._benchmark_handler#{}') as g:
+            g.command('list', 'benchmark_list_command_handler')
+            g.command('list-parameters', 'benchmark_list_params_command_handler')
         with CommandGroup(self, 'node', 'superbench.cli._node_handler#{}') as g:
             g.command('info', 'info_command_handler')
         with CommandGroup(self, 'result', 'superbench.cli._result_handler#{}') as g:
@@ -61,6 +64,10 @@ def load_arguments(self, command):
                 nargs='+',
                 help='Extra arguments to override config_file.'
             )
+
+        with ArgumentsContext(self, 'benchmark') as ac:
+            ac.argument('name', options_list=('--name', '-n'), type=str, help='Benchmark name or regular expression.')
+
         with ArgumentsContext(self, 'result') as ac:
             ac.argument('raw_data_file', options_list=('--data-file', '-d'), type=str, help='Path to raw data file.')
             ac.argument('rule_file', options_list=('--rule-file', '-r'), type=str, help='Path to rule file.')
@@ -73,4 +80,5 @@ def load_arguments(self, command):
                 help='Path to output directory, outputs/{datetime} will be used if not specified.'
             )
             ac.argument('output_file_format', type=str, help='Format of output file, excel or json.')
+
         super().load_arguments(command)

superbench/cli/_help.py

@@ -61,8 +61,31 @@
           text: {cli_name} run --docker-image superbench/cuda:11.1 --host-file ./host.ini
 """.format(cli_name=CLI_NAME)
 
+helps['benchmark'] = """
+    type: group
+    short-summary: Commands to manage benchmarks.
+"""
+
+helps['benchmark list'] = """
+    type: command
+    examples:
+        - name: list all benchmarks
+          text: {cli_name} benchmark list
+        - name: list all benchmarks ending with "-bw"
+          text: {cli_name} benchmark list --name [a-z]+-bw
+""".format(cli_name=CLI_NAME)
+
+helps['benchmark list-parameters'] = """
+    type: command
+    examples:
+        - name: list parameters for all benchmarks
+          text: {cli_name} benchmark list-parameters
+        - name: list parameters for all benchmarks which starts with "pytorch-"
+          text: {cli_name} benchmark list-parameters --name pytorch-[a-z]+
+""".format(cli_name=CLI_NAME)
+
 helps['node'] = """
-    type: Group
+    type: group
     short-summary: Get detailed information or configurations on the local node.
 """
 
@@ -75,19 +98,31 @@
 """.format(cli_name=CLI_NAME)
 
 helps['result'] = """
-    type: Group
+    type: group
     short-summary: Process or analyze the results of SuperBench benchmarks.
 """
 
 helps['result diagnosis'] = """
     type: command
-    short-summary: Filter the defective machines automatically from benchmarking results according to rules defined in rule file.
+    short-summary: >
+        Filter the defective machines automatically from benchmarking results
+        according to rules defined in rule file.
     examples:
         - name: run data diagnosis and output the results in excel format
-          text: {cli_name} result diagnosis --data-file 'outputs/results-summary.jsonl' --rule-file 'rule.yaml' --baseline-file 'baseline.json' --output-file-foramt 'excel'
+          text: >
+            {cli_name} result diagnosis
+            --data-file outputs/results-summary.jsonl
+            --rule-file rule.yaml
+            --baseline-file baseline.json
+            --output-file-foramt excel
         - name: run data diagnosis and output the results in jsonl format
-          text: {cli_name} result diagnosis --data-file 'outputs/results-summary.jsonl' --rule-file 'rule.yaml' --baseline-file 'baseline.json' --output-file-foramt 'json'
-""".format(cli_name=CLI_NAME)    # noqa: E501
+          text: >
+            {cli_name} result diagnosis
+            --data-file outputs/results-summary.jsonl
+            --rule-file rule.yaml
+            --baseline-file baseline.json
+            --output-file-foramt json
+""".format(cli_name=CLI_NAME)
 
 
 class SuperBenchCLIHelp(CLIHelp):

tests/benchmarks/model_benchmarks/test_model_base.py

@@ -147,26 +147,26 @@ def test_arguments_related_interfaces():
     settings = benchmark.get_configurable_settings()
     expected_settings = (
         """optional arguments:
-  --run_count int       The run count of benchmark.
-  --duration int        The elapsed time of benchmark in seconds.
-  --num_warmup int      The number of warmup step.
-  --num_steps int       The number of test step.
-  --sample_count int    The number of data samples in dataset.
   --batch_size int      The number of batch size.
-  --precision Precision [Precision ...]
-                        Model precision. E.g. float16 float32 float64 bfloat16
-                        uint8 int8 int16 int32 int64.
-  --model_action ModelAction [ModelAction ...]
-                        Benchmark model process. E.g. train inference.
+  --distributed_backend DistributedBackend
+                        Distributed backends. E.g. nccl mpi gloo.
   --distributed_impl DistributedImpl
                         Distributed implementations. E.g. ddp mirrored
                         multiworkermirrored parameterserver horovod.
-  --distributed_backend DistributedBackend
-                        Distributed backends. E.g. nccl mpi gloo.
-  --no_gpu              Disable GPU training.
-  --pin_memory          Enable option to pin memory in data loader.
+  --duration int        The elapsed time of benchmark in seconds.
   --force_fp32          Enable option to use full float32 precision.
   --hidden_size int     Hidden size.
+  --model_action ModelAction [ModelAction ...]
+                        Benchmark model process. E.g. train inference.
+  --no_gpu              Disable GPU training.
+  --num_steps int       The number of test step.
+  --num_warmup int      The number of warmup step.
+  --pin_memory          Enable option to pin memory in data loader.
+  --precision Precision [Precision ...]
+                        Model precision. E.g. float16 float32 float64 bfloat16
+                        uint8 int8 int16 int32 int64.
+  --run_count int       The run count of benchmark.
+  --sample_count int    The number of data samples in dataset.
   --seq_len int         Sequence length."""
     )
     assert (settings == expected_settings)
@@ -181,26 +181,26 @@ def test_preprocess():
     settings = benchmark.get_configurable_settings()
     expected_settings = (
         """optional arguments:
-  --run_count int       The run count of benchmark.
-  --duration int        The elapsed time of benchmark in seconds.
-  --num_warmup int      The number of warmup step.
-  --num_steps int       The number of test step.
-  --sample_count int    The number of data samples in dataset.
   --batch_size int      The number of batch size.
-  --precision Precision [Precision ...]
-                        Model precision. E.g. float16 float32 float64 bfloat16
-                        uint8 int8 int16 int32 int64.
-  --model_action ModelAction [ModelAction ...]
-                        Benchmark model process. E.g. train inference.
+  --distributed_backend DistributedBackend
+                        Distributed backends. E.g. nccl mpi gloo.
   --distributed_impl DistributedImpl
                         Distributed implementations. E.g. ddp mirrored
                         multiworkermirrored parameterserver horovod.
-  --distributed_backend DistributedBackend
-                        Distributed backends. E.g. nccl mpi gloo.
-  --no_gpu              Disable GPU training.
-  --pin_memory          Enable option to pin memory in data loader.
+  --duration int        The elapsed time of benchmark in seconds.
   --force_fp32          Enable option to use full float32 precision.
   --hidden_size int     Hidden size.
+  --model_action ModelAction [ModelAction ...]
+                        Benchmark model process. E.g. train inference.
+  --no_gpu              Disable GPU training.
+  --num_steps int       The number of test step.
+  --num_warmup int      The number of warmup step.
+  --pin_memory          Enable option to pin memory in data loader.
+  --precision Precision [Precision ...]
+                        Model precision. E.g. float16 float32 float64 bfloat16
+                        uint8 int8 int16 int32 int64.
+  --run_count int       The run count of benchmark.
+  --sample_count int    The number of data samples in dataset.
   --seq_len int         Sequence length."""
     )
     assert (settings == expected_settings)

tests/benchmarks/test_registry.py

@@ -113,9 +113,9 @@ def test_get_benchmark_configurable_settings():
     settings = BenchmarkRegistry.get_benchmark_configurable_settings(context)
 
     expected = """optional arguments:
-  --run_count int    The run count of benchmark.
   --duration int     The elapsed time of benchmark in seconds.
   --lower_bound int  The lower bound for accumulation.
+  --run_count int    The run count of benchmark.
   --upper_bound int  The upper bound for accumulation."""
     assert (settings == expected)
 

tests/cli/test_sb.py

@@ -6,11 +6,12 @@
 import io
 import contextlib
 from functools import wraps
-from knack.testsdk import ScenarioTest, StringCheck, NoneCheck
+from knack.testsdk import ScenarioTest, StringCheck, NoneCheck, JMESPathCheck
 from pathlib import Path
 
 import superbench
 from superbench.cli import SuperBenchCLI
+from superbench.benchmarks import BenchmarkRegistry
 
 
 def capture_system_exit(func):
@@ -83,6 +84,20 @@ def test_sb_run_nonexist_host_file(self):
         result = self.cmd('sb run --host-file ./nonexist.yaml', expect_failure=True)
         self.assertEqual(result.exit_code, 1)
 
+    def test_sb_benchmark_list(self):
+        """Test sb benchmark list."""
+        self.cmd('sb benchmark list', checks=[JMESPathCheck('length(@)', len(BenchmarkRegistry.benchmarks))])
+
+    def test_sb_benchmark_list_nonexist(self):
+        """Test sb benchmark list, give a non-exist benchmark name, should fail."""
+        result = self.cmd('sb benchmark list -n non-exist-name', expect_failure=True)
+        self.assertEqual(result.exit_code, 1)
+
+    def test_sb_benchmark_list_parameters(self):
+        """Test sb benchmark list-parameters."""
+        self.cmd('sb benchmark list-parameters', checks=[NoneCheck()])
+        self.cmd('sb benchmark list-parameters -n pytorch-[a-z]+', checks=[NoneCheck()])
+
     def test_sb_node_info(self):
         """Test sb node info, should fail."""
         self.cmd('sb node info', expect_failure=False)