llvm-bolt-wrapper.py

#!/usr/bin/env python3
import argparse
import subprocess
from typing import *
import tempfile
import copy
import os
import shutil
import sys
import re
import configparser
from types import SimpleNamespace
from textwrap import dedent

# USAGE:
# 0. Prepare two BOLT build versions: base and compare.
# 1. Create the config by invoking this script with required options.
#    Save the config as `llvm-bolt-wrapper.ini` next to the script or
#    in the testing directory.
# In the base BOLT build directory:
# 2. Rename `llvm-bolt` to `llvm-bolt.real`
# 3. Create a symlink from this script to `llvm-bolt`
# 4. Create `llvm-bolt-wrapper.ini` and fill it using the example below.
#
# This script will compare binaries produced by base and compare BOLT, and
# report elapsed processing time and max RSS.

# read options from config file llvm-bolt-wrapper.ini in script CWD
#
# [config]
# # mandatory
# base_bolt = /full/path/to/llvm-bolt.real
# cmp_bolt = /full/path/to/other/llvm-bolt
# # optional, default to False
# verbose
# keep_tmp
# no_minimize
# run_sequentially
# compare_output
# skip_binary_cmp
# # optional, defaults to timing.log in CWD
# timing_file = timing1.log


def read_cfg():
    src_dir = os.path.dirname(os.path.abspath(__file__))
    cfg = configparser.ConfigParser(allow_no_value=True)
    cfgs = cfg.read("llvm-bolt-wrapper.ini")
    if not cfgs:
        cfgs = cfg.read(os.path.join(src_dir, "llvm-bolt-wrapper.ini"))
    assert cfgs, f"llvm-bolt-wrapper.ini is not found in {os.getcwd()}"

    def get_cfg(key):
        # if key is not present in config, assume False
        if key not in cfg["config"]:
            return False
        # if key is present, but has no value, assume True
        if not cfg["config"][key]:
            return True
        # if key has associated value, interpret the value
        return cfg["config"].getboolean(key)

    d = {
        # BOLT binary locations
        "BASE_BOLT": cfg["config"]["base_bolt"],
        "CMP_BOLT": cfg["config"]["cmp_bolt"],
        # optional
        "VERBOSE": get_cfg("verbose"),
        "KEEP_TMP": get_cfg("keep_tmp"),
        "NO_MINIMIZE": get_cfg("no_minimize"),
        "RUN_SEQUENTIALLY": get_cfg("run_sequentially"),
        "COMPARE_OUTPUT": get_cfg("compare_output"),
        "SKIP_BINARY_CMP": get_cfg("skip_binary_cmp"),
        "TIMING_FILE": cfg["config"].get("timing_file", "timing.log"),
    }
    if d["VERBOSE"]:
        print(f"Using config {os.path.abspath(cfgs[0])}")
    return SimpleNamespace(**d)


# perf2bolt mode
PERF2BOLT_MODE = ["-aggregate-only", "-ignore-build-id"]

# boltdiff mode
BOLTDIFF_MODE = ["-diff-only", "-o", "/dev/null"]

# options to suppress binary differences as much as possible
MINIMIZE_DIFFS = ["-bolt-info=0"]

# bolt output options that need to be intercepted
BOLT_OUTPUT_OPTS = {
    "-o": "BOLT output binary",
    "-w": "BOLT recorded profile",
}

# regex patterns to exclude the line from log comparison
SKIP_MATCH = [
    "BOLT-INFO: BOLT version",
    r"^Args: ",
    r"^BOLT-DEBUG:",
    r"BOLT-INFO:.*data.*output data",
    "WARNING: reading perf data directly",
]


def run_cmd(cmd, out_f, cfg):
    if cfg.VERBOSE:
        print(" ".join(cmd))
    return subprocess.Popen(cmd, stdout=out_f, stderr=subprocess.STDOUT)


def run_bolt(bolt_path, bolt_args, out_f, cfg):
    p2b = os.path.basename(sys.argv[0]) == "perf2bolt"  # perf2bolt mode
    bd = os.path.basename(sys.argv[0]) == "llvm-boltdiff"  # boltdiff mode
    cmd = ["/usr/bin/time", "-f", "%e %M", bolt_path] + bolt_args
    if p2b:
        # -ignore-build-id can occur at most once, hence remove it from cmd
        if "-ignore-build-id" in cmd:
            cmd.remove("-ignore-build-id")
        cmd += PERF2BOLT_MODE
    elif bd:
        cmd += BOLTDIFF_MODE
    elif not cfg.NO_MINIMIZE:
        cmd += MINIMIZE_DIFFS
    return run_cmd(cmd, out_f, cfg)


def prepend_dash(args: Mapping[AnyStr, AnyStr]) -> Sequence[AnyStr]:
    """
    Accepts parsed arguments and returns flat list with dash prepended to
    the option.
    Example: Namespace(o='test.tmp') -> ['-o', 'test.tmp']
    """
    dashed = [("-" + key, value) for (key, value) in args.items()]
    flattened = list(sum(dashed, ()))
    return flattened


def replace_cmp_path(tmp: AnyStr, args: Mapping[AnyStr, AnyStr]) -> Sequence[AnyStr]:
    """
    Keeps file names, but replaces the path to a temp folder.
    Example: Namespace(o='abc/test.tmp') -> Namespace(o='/tmp/tmpf9un/test.tmp')
    Except preserve /dev/null.
    """
    replace_path = (
        lambda x: os.path.join(tmp, os.path.basename(x))
        if x != "/dev/null"
        else "/dev/null"
    )
    new_args = {key: replace_path(value) for key, value in args.items()}
    return prepend_dash(new_args)


def preprocess_args(args: argparse.Namespace) -> Mapping[AnyStr, AnyStr]:
    """
    Drop options that weren't parsed (e.g. -w), convert to a dict
    """
    return {key: value for key, value in vars(args).items() if value}


def write_to(txt, filename, mode="w"):
    with open(filename, mode) as f:
        f.write(txt)


def wait(proc, fdesc):
    proc.wait()
    fdesc.close()
    return open(fdesc.name)


def compare_logs(main, cmp, skip_begin=0, skip_end=0, str_input=True):
    """
    Compares logs but allows for certain lines to be excluded from comparison.
    If str_input is True (default), the input it assumed to be a string,
    which is split into lines. Otherwise the input is assumed to be a file.
    Returns None on success, mismatch otherwise.
    """
    main_inp = main.splitlines() if str_input else main.readlines()
    cmp_inp = cmp.splitlines() if str_input else cmp.readlines()
    # rewind logs after consumption
    if not str_input:
        main.seek(0)
        cmp.seek(0)
    for lhs, rhs in list(zip(main_inp, cmp_inp))[skip_begin : -skip_end or None]:
        if lhs != rhs:
            # check skip patterns
            for skip in SKIP_MATCH:
                # both lines must contain the pattern
                if re.search(skip, lhs) and re.search(skip, rhs):
                    break
            # otherwise return mismatching lines
            else:
                return (lhs, rhs)
    return None


def fmt_cmp(cmp_tuple):
    if not cmp_tuple:
        return ""
    return f"main:\n{cmp_tuple[0]}\ncmp:\n{cmp_tuple[1]}\n"


def compare_with(lhs, rhs, cmd, skip_begin=0, skip_end=0):
    """
    Runs cmd on both lhs and rhs and compares stdout.
    Returns tuple (mismatch, lhs_stdout):
        - if stdout matches between two files, mismatch is None,
        - otherwise mismatch is a tuple of mismatching lines.
    """
    run = lambda binary: subprocess.run(
        cmd.split() + [binary], text=True, check=True, capture_output=True
    ).stdout
    run_lhs = run(lhs)
    run_rhs = run(rhs)
    cmp = compare_logs(run_lhs, run_rhs, skip_begin, skip_end)
    return cmp, run_lhs


def parse_cmp_offset(cmp_out):
    """
    Extracts byte number from cmp output:
    file1 file2 differ: byte X, line Y
    """
    # NOTE: cmp counts bytes starting from 1!
    return int(re.search(r"byte (\d+),", cmp_out).groups()[0]) - 1


def report_real_time(binary, main_err, cmp_err, cfg):
    """
    Extracts real time from stderr and appends it to TIMING FILE it as csv:
    "output binary; base bolt; cmp bolt"
    """

    def get_real_from_stderr(logline):
        return "; ".join(logline.split())

    for line in main_err:
        pass
    main = get_real_from_stderr(line)
    for line in cmp_err:
        pass
    cmp = get_real_from_stderr(line)
    write_to(f"{binary}; {main}; {cmp}\n", cfg.TIMING_FILE, "a")
    # rewind logs after consumption
    main_err.seek(0)
    cmp_err.seek(0)


def clean_exit(tmp, out, exitcode, cfg):
    # temp files are only cleaned on success
    if not cfg.KEEP_TMP:
        shutil.rmtree(tmp)

    # report stdout and stderr from the main process
    shutil.copyfileobj(out, sys.stdout)
    sys.exit(exitcode)


def find_section(offset, readelf_hdr):
    hdr = readelf_hdr.split("\n")
    section = None
    # extract sections table (parse objdump -hw output)
    for line in hdr[5:-1]:
        cols = line.strip().split()
        # extract section offset
        file_offset = int(cols[5], 16)
        # section size
        size = int(cols[2], 16)
        if offset >= file_offset and offset < file_offset + size:
            if sys.stdout.isatty():  # terminal supports colors
                print(f"\033[1m{line}\033[0m")
            else:
                print(f">{line}")
            section = cols[1]
        else:
            print(line)
    return section


def main_config_generator():
    parser = argparse.ArgumentParser()
    parser.add_argument("base_bolt", help="Full path to base llvm-bolt binary")
    parser.add_argument("cmp_bolt", help="Full path to cmp llvm-bolt binary")
    parser.add_argument(
        "--verbose",
        action="store_true",
        help="Print subprocess invocation cmdline (default False)",
    )
    parser.add_argument(
        "--keep_tmp",
        action="store_true",
        help="Preserve tmp folder on a clean exit "
        "(tmp directory is preserved on crash by default)",
    )
    parser.add_argument(
        "--no_minimize",
        action="store_true",
        help=f"Do not add `{MINIMIZE_DIFFS}` that is used "
        "by default to reduce binary differences",
    )
    parser.add_argument(
        "--run_sequentially",
        action="store_true",
        help="Run both binaries sequentially (default "
        "in parallel). Use for timing comparison",
    )
    parser.add_argument(
        "--compare_output",
        action="store_true",
        help="Compare bolt stdout/stderr (disabled by default)",
    )
    parser.add_argument(
        "--skip_binary_cmp", action="store_true", help="Disable output comparison"
    )
    parser.add_argument(
        "--timing_file",
        help="Override path to timing log " "file (default `timing.log` in CWD)",
    )
    args = parser.parse_args()

    print(
        dedent(
            f"""\
    [config]
    # mandatory
    base_bolt = {args.base_bolt}
    cmp_bolt = {args.cmp_bolt}"""
        )
    )
    del args.base_bolt
    del args.cmp_bolt
    d = vars(args)
    if any(d.values()):
        print("# optional")
        for key, value in d.items():
            if value:
                print(key)


def main():
    cfg = read_cfg()
    # intercept output arguments
    parser = argparse.ArgumentParser(add_help=False)
    for option, help in BOLT_OUTPUT_OPTS.items():
        parser.add_argument(option, help=help)
    args, unknownargs = parser.parse_known_args()
    args = preprocess_args(args)
    cmp_args = copy.deepcopy(args)
    tmp = tempfile.mkdtemp()
    cmp_args = replace_cmp_path(tmp, cmp_args)

    # reconstruct output arguments: prepend dash
    args = prepend_dash(args)

    # run both BOLT binaries
    main_f = open(os.path.join(tmp, "main_bolt.stdout"), "w")
    cmp_f = open(os.path.join(tmp, "cmp_bolt.stdout"), "w")
    main_bolt = run_bolt(cfg.BASE_BOLT, unknownargs + args, main_f, cfg)
    if cfg.RUN_SEQUENTIALLY:
        main_out = wait(main_bolt, main_f)
        cmp_bolt = run_bolt(cfg.CMP_BOLT, unknownargs + cmp_args, cmp_f, cfg)
    else:
        cmp_bolt = run_bolt(cfg.CMP_BOLT, unknownargs + cmp_args, cmp_f, cfg)
        main_out = wait(main_bolt, main_f)
    cmp_out = wait(cmp_bolt, cmp_f)

    # check exit code
    if main_bolt.returncode != cmp_bolt.returncode:
        print(tmp)
        exit("exitcode mismatch")

    # don't compare output upon unsuccessful exit
    if main_bolt.returncode != 0:
        cfg.SKIP_BINARY_CMP = True

    # compare logs, skip_end=1 skips the line with time
    out = (
        compare_logs(main_out, cmp_out, skip_end=1, str_input=False)
        if cfg.COMPARE_OUTPUT
        else None
    )
    if out:
        print(tmp)
        print(fmt_cmp(out))
        write_to(fmt_cmp(out), os.path.join(tmp, "summary.txt"))
        exit("logs mismatch")

    if os.path.basename(sys.argv[0]) == "llvm-boltdiff":  # boltdiff mode
        # no output binary to compare, so just exit
        clean_exit(tmp, main_out, main_bolt.returncode, cfg)

    # compare binaries (using cmp)
    main_binary = args[args.index("-o") + 1]
    cmp_binary = cmp_args[cmp_args.index("-o") + 1]
    if main_binary == "/dev/null":
        assert cmp_binary == "/dev/null"
        cfg.SKIP_BINARY_CMP = True

    # report binary timing as csv: output binary; base bolt real; cmp bolt real
    report_real_time(main_binary, main_out, cmp_out, cfg)

    if not cfg.SKIP_BINARY_CMP:
        # check if files exist
        main_exists = os.path.exists(main_binary)
        cmp_exists = os.path.exists(cmp_binary)
        if main_exists and cmp_exists:
            # proceed to comparison
            pass
        elif not main_exists and not cmp_exists:
            # both don't exist, assume it's intended, skip comparison
            clean_exit(tmp, main_out, main_bolt.returncode, cfg)
        elif main_exists:
            assert not cmp_exists
            exit(f"{cmp_binary} doesn't exist")
        else:
            assert not main_exists
            exit(f"{main_binary} doesn't exist")

        cmp_proc = subprocess.run(
            ["cmp", "-b", main_binary, cmp_binary], capture_output=True, text=True
        )
        if cmp_proc.returncode:
            # check if output is an ELF file (magic bytes)
            with open(main_binary, "rb") as f:
                magic = f.read(4)
                if magic != b"\x7fELF":
                    exit("output mismatch")
            # check if ELF headers match
            mismatch, _ = compare_with(main_binary, cmp_binary, "readelf -We")
            if mismatch:
                print(fmt_cmp(mismatch))
                write_to(fmt_cmp(mismatch), os.path.join(tmp, "headers.txt"))
                exit("headers mismatch")
            # if headers match, compare sections (skip line with filename)
            mismatch, hdr = compare_with(
                main_binary, cmp_binary, "objdump -hw", skip_begin=2
            )
            assert not mismatch
            # check which section has the first mismatch
            mismatch_offset = parse_cmp_offset(cmp_proc.stdout)
            section = find_section(mismatch_offset, hdr)
            exit(f"binary mismatch @{hex(mismatch_offset)} ({section})")

    clean_exit(tmp, main_out, main_bolt.returncode, cfg)


if __name__ == "__main__":
    # config generator mode if the script is launched as is
    if os.path.basename(__file__) == "llvm-bolt-wrapper.py":
        main_config_generator()
    else:
        # llvm-bolt interceptor mode otherwise
        main()