Add build and discover commands

[jeremyfowers]

Problem

The turnkey benchmark command has --build-only and --analyze-only flags that provide an "early exit" from benchmarking to support standalone analysis and analysis+build.

However, this syntax is confusing: "use turnkey benchmark --build-only to export the ONNX model zoo".

It also leads to a lot of mutually exclusive arguments to turnkey benchmark. For example, --analyze-only is mutually exclusive with --sequence. This makes the help pages needlessly convoluted.

We also want to rename analyze to discover, which is a more accurate term for what is actually happening.

Proposal

Part 1: Discover

Rename analyze to discover throughout the docs and code.

Part 2: New Commands

Create two new commands:

• turnkey build: like turnkey benchmark --build-only, except that all of the benchmarking-specific flags are removed.
• turnkey discover: like turnkey benchmark --analyze-only, except that all of the build- and benchmarking-specific flags are removed.

Part 3: Help Page

Broken out into a separate issue #73

cc @danielholanda

[jeremyfowers]

This could be an interesting parser:

import sys
import argparse

discover_parser = argparse.ArgumentParser(
    prog="discover",
    description="Find the model(s) in each input and send them into the toolchain.",
    epilog="Any additional arguments to `discover` are forwarded the input Python scripts (.py files)",
)
discover_parser.add_argument("--max-depth")
discover_parser.add_argument("--labels")

build_parser = argparse.ArgumentParser(
    prog="Stage build",
    description="Automatically select stages to build your model(s) based on the device and runtime arguments.",
)

export_parser = argparse.ArgumentParser(prog="Stage export")
export_parser.add_argument("--opset")
export_parser.add_argument("--validate", action="store_true")

vaiq_parser = argparse.ArgumentParser(prog="Stage vaiq_onnx")

vitisep_parser = argparse.ArgumentParser(prog="Stage vitisep_compile")
vitisep_parser.add_argument(
    "--xclbin", choices=["phx::1x4", "phx::4x4", "stx::1x4", "stx::4x4"]
)

quant_parser = argparse.ArgumentParser(prog="Stage ipex_build")
quant_parser.add_argument("--dtype", choices=["float32", "float16", "bfloat16", "int8"])

benchmark_parser = argparse.ArgumentParser(prog="Stage benchmark")
benchmark_parser.add_argument("--iterations", default=100)

commands = {
    "discover": discover_parser,
    "build": build_parser,
    "export": export_parser,
    "ipex_optimize": quant_parser,
    "vitisep_compile": vitisep_parser,
    "vaiq_onnx": vaiq_parser,
    "benchmark": benchmark_parser,
}

parser = argparse.ArgumentParser(
    description="By default, turnkey will discover the models in each input file. To customize the  "
    "toolchain, specify a sequence of stages at the end of the command. "
    "For exmaple: `turnkey model.py export` will "
    "export an ONNX model for each PyTorch model in model.py."
)
input_files_action = parser.add_argument(
    "input_files",
    nargs="*",
    help="Python scripts (.py), ONNX files (.onnx), and build directories (.tkb) to be evaluated. ",
)
parser.add_argument("--device")
parser.add_argument("--runtime")
parser.add_argument("--cache-dir")
parser.add_argument("--lean-cache")
parser.add_argument("--retry", choices=["attempted", "failed", "successful", "timeout"])
parser.add_argument("--verbosity")
parser.add_argument("--process-isolation")
parser.add_argument("--use-slurm")
parser.add_argument("--timeout")
parser.add_argument(
    "stages",
    nargs="?",
    help="Available toolchain stages. Call `turnkey STAGE -h` to learn more about that stage. Default: discover.",
    choices=commands.keys(),
)


current_group = "globals"
groups = {current_group: []}
cmd = sys.argv[1:]
while len(cmd):
    if cmd[0] in commands.keys():
        current_group = cmd.pop(0)
        groups[current_group] = []
    else:
        groups[current_group].append(cmd.pop(0))

if len(groups) == 1:
    # Assign default stages
    groups["discover"] = []

# Do one pass of parsing to figure out if -h was used
global_args = parser.parse_args(groups["globals"])
for cmd, argv in groups.items():
    if cmd != "globals":
        if cmd == "benchmark":
            if global_args.device == "ryzenai" or global_args.runtime == "vitisep":
                commands["benchmark"].add_argument("--threads")
                commands["benchmark"].prog = "Stage benchmark vitisep"
        commands[cmd].parse_args(argv)

# Now we make sure that there's at least one input
input_files_action.nargs = "+"

print("\n\nParse result:\n")

for cmd, argv in groups.items():
    if cmd == "discover":
        args, script_args = commands[cmd].parse_known_args(argv)
        print("Discovering models")
        print("\tWith settings", args)
        print("\tWith script args:", script_args)
        print()
    elif cmd != "globals":
        args = commands[cmd].parse_args(argv)
        print("Running stage:", cmd)
        print("\tWith settings", args)
        print()
    else:
        args = parser.parse_args(argv)
        input_files = args.__dict__.pop("input_files")
        args.__dict__.pop("stages")
        print("Input files:", input_files)
        print()
        print(f"Arguments that apply to all stages:", args)
        print()

[jeremyfowers]

Completed in the refresh!