fabfile.py

#!/usr/bin/env python3

import sys
import boto3
import botocore
import time
import concurrent.futures
import io
import os
import pathlib
import queue
from fabric import Connection
import patchwork.transfers
import invoke.exceptions
import tempfile
# import paramiko.rsakey
import json
from invoke import run as local
from functools import reduce

# # DEBUGGING:
# import logging
# logging.basicConfig(level=logging.DEBUG)


#### Environment/arguments: ####

# We'll upload reports to:
#   f"{RESULTS_S3_BUCKET}/mono-pr-{PR_NUMBER}/{benchmark_set}.json"
# Result sets can be retreived efficiently with
#   s3.list_objects(Bucket=RESULTS_S3_BUCKET, Prefix=f"{THIS_S3_BUCKET_PREFIX}/")['Contents']
PR_NUMBER = os.environ['PR_NUMBER']
THIS_S3_BUCKET_PREFIX = f"mono-pr-{PR_NUMBER}"

PR_TARGET_BRANCH = os.environ['PR_TARGET_BRANCH']

# The image file we'll ship to the benchmark runner to test:
# You can pull get a local image file from docker with e.g.:
#    docker image save -o /tmp/hasura-blah.tar hasura/graphql-engine:latest
HASURA_DOCKER_IMAGE = os.environ['HASURA_DOCKER_IMAGE']
if not os.path.isfile(HASURA_DOCKER_IMAGE):
    sys.exit(f"Could not find a docker image file at {HASURA_DOCKER_IMAGE}")

# For boto3:
AWS_ACCESS_KEY_ID = os.environ['BENCHMARKS_AWS_ACCESS_KEY_ID']
AWS_SECRET_ACCESS_KEY = os.environ['BENCHMARKS_AWS_SECRET_ACCESS_KEY']

#### Globals: ####

# This key just has privileges to SSH into the benchmarks runner instance.
# This variable is the entire PEM file (hasura-benchmarks-runner.pem) as a
# string:
BENCHMARKS_RUNNER_PRIVATE_KEY = os.environ['BENCHMARKS_RUNNER_PRIVATE_KEY']

# Keep track of EC2 instances launched in 'run_benchmark_set' so we can
# clean them up on exit/error (NOTE: 'atexit' is useless for this):
LAUNCHED_INSTANCES = queue.Queue()

# Global mutable to allow us to abort restarts when the user initiates a shutdown (e.g. CTRL-C)
SHUTTING_DOWN = False

RESULTS_S3_BUCKET = 'hasura-benchmark-results'
# NOTE: Reports uploaded will be PUBLICLY ACCESSIBLE at:
def s3_url(filename, bucket_prefix=THIS_S3_BUCKET_PREFIX):
    return f"https://{RESULTS_S3_BUCKET}.s3.us-east-2.amazonaws.com/{bucket_prefix}/{filename}"
# This short identifier format (e.g. 'mono-pr-1998/chinook') is understood by
# graphql-bench viewer:
def s3_short_id(filename, bucket_prefix=THIS_S3_BUCKET_PREFIX):
    return f"{bucket_prefix}/{filename[:-5]}"
def graphql_bench_url(short_ids):
    return f"https://hasura.github.io/graphql-bench/app/web-app/#{','.join(short_ids)}"

# We'll write to this, CI will look for it and insert it into the PR comment thread:
REGRESSION_REPORT_COMMENT_FILENAME = "/tmp/hasura_regression_report_comment.md"


def main():
    try:
        benchmark_sets_basepath = abs_path("benchmark_sets")
        # Collect the benchmarks we'll run in CI:
        benchmark_sets = [ dir for dir in os.listdir(benchmark_sets_basepath)
            if not pathlib.Path(benchmark_sets_basepath, dir, 'SKIP_CI').is_file()]

        # Theses benchmark sets won't have raw regression numbers displayed
        # (likely because they are unstable for now)
        skip_pr_report_names = [ dir for dir in os.listdir(benchmark_sets_basepath)
            if pathlib.Path(benchmark_sets_basepath, dir, 'SKIP_PR_REPORT').is_file()]

        # Start all the instances we need and run benchmarks in parallel
        # NOTE: ProcessPoolExecutor doesn't work with shared queue
        with concurrent.futures.ThreadPoolExecutor() as executor:
             bench_result = executor.map(run_benchmark_set, benchmark_sets)

        # This just surfaces any exceptions from concurrent.futures; we might
        # do something else with these later:
        #   http://docs.pyinvoke.org/en/latest/api/runners.html#invoke.runners.Result
        print(list(bench_result))

        report, merge_base_pr = generate_regression_report()
        pretty_print_regression_report_github_comment(
            report,
            skip_pr_report_names,
            merge_base_pr,
            REGRESSION_REPORT_COMMENT_FILENAME
        )
        say("Success!")

    # NOTE: without this 'except', any exceptions are silently swallowed... ugh
    except Exception as e:
        raise
    finally:
        global SHUTTING_DOWN
        SHUTTING_DOWN = True
        print("Cleaning up resources and shutting down:")
        for i in list(LAUNCHED_INSTANCES.queue):
            print(f"  * {i.instance_id}")
            # NOTE: this is idempotent, so okay to run on success
            i.terminate()
        print("")
    sys.exit()

def new_boto_session():
    # For other auth options:
    #     https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html
    # NOTE: we must start a new 'Session' per thread:
    return boto3.Session(
        aws_access_key_id=AWS_ACCESS_KEY_ID,
        aws_secret_access_key=AWS_SECRET_ACCESS_KEY
    )

# Start an EC2 instance that will run 'benchmark_set'. use_spot indicates
# whether we should try to start spot instances. If use_spot=True we'll retry
# once in case this fails, with an on-demand instance instead
def run_benchmark_set(benchmark_set, use_spot=True):
    # Prefer this instead of 'print()' so we can follow along with concurrent execution:
    def say(s):
        print(f"*** \033[92m {benchmark_set}: {s} \033[0m")
    def warn(s):
        print(f"*** \033[93m {benchmark_set}: {s} \033[0m")
    boto3_session = new_boto_session()
    # boto3_session = boto3.Session(profile_name='benchmarks')
    ec2_client = boto3_session.client('ec2', region_name='us-east-2')
    ec2 = boto3_session.resource('ec2', region_name='us-east-2')
    s3 = boto3_session.client('s3')

    # Get benchmark-runner AMI (see README_AMI.md)
    runner_images_dirty = ec2_client.describe_images(Filters=[{'Name':'tag:Name', 'Values':['hasura-benchmarks-runner']}])['Images']
    if len(runner_images_dirty) > 1:
        sys.exit("More than one instance tagged 'hasura-benchmarks-runner'; please delete tag from old image")
    elif len(runner_images_dirty) == 0:
        sys.exit("The 'hasura-benchmarks-runner' image needs to be copied to this region.")

    runner_image_id = runner_images_dirty[0]['ImageId']

    # We can and do run into capacity issues. Try our best to find a region
    # with spot availability (much cheaper), else try on-demand price
    spot = {
               'MarketType': 'spot',
               'SpotOptions': {
                   # A bit over the on-demand price
                   # NOTE: at time of this writing spot price has been very stable around $0.35/hr
                   'MaxPrice': '1.80',
                   'SpotInstanceType': 'one-time',
                   'InstanceInterruptionBehavior': 'terminate'
               }
           }
    # Regions in which we can run benchmarks, in order (approximately) from
    # cheap to more expensive. With c4.8xlarge we run into capacity issues from
    # time to time.
    # NOTE: if you want to add a new region here you'll need to copy the
    # hasura-benchmarks-runner AMI, security group, and keypair to that region
    # also.
    ok_regions = [
        "us-east-2",
        "us-west-2",
        "ap-south-1",
        "ca-central-1",
        "eu-west-2",
        "eu-west-1",
    ]
    # the sequence of spot/on-demand requests we'll make:
    market_types = [spot, {}, {}, {}, "FAIL"] if use_spot else [{}, {}, {}, "FAIL"]
    def launch_instance():
        # We'll try on-demand instances three times, hoping capacity gets added, before giving up
        for market_type in market_types:
            for region in ok_regions:
                if market_type == "FAIL":
                    sys.exit("All regions are out of capacity! We'll just need to wait and try again, sorry.")

                market_type_str = "on-demand" if market_type == {} else "spot"
                say(f"Trying to launch in {region} as {market_type_str}")

                try:
                    # Launch beefy ec2 instances that will run the actual benchmarks:
                    instance = ec2.create_instances(
                        ImageId=runner_image_id,
                        MinCount=1, MaxCount=1,
                        # NOTE: benchmarks are tuned very specifically to this instance type  and
                        # the other settings here (see bench.sh):
                        #   Lately AWS seems to be running out of capacity and so we may need to research 
                        # (check numa configuration, etc) and switch to one of these:
                        #   https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/processor_state_control.html
                        InstanceType='c4.8xlarge',
                        KeyName='hasura-benchmarks-runner',
                        InstanceInitiatedShutdownBehavior='terminate',
                        # Disable hyperthreading:
                        CpuOptions={
                            'CoreCount': 18,
                            'ThreadsPerCore': 1
                        },
                        # AFAICT this is always true for c4 instances and comes at no additional
                        # charge, but the console shows 'false' if we don't set this...
                        EbsOptimized=True,
                        InstanceMarketOptions=market_type,
                        TagSpecifications=[{
                            'ResourceType': 'instance',
                            'Tags': [
                               # Informational. This will show up in console:
                               {'Key': 'Name',
                                'Value': 'hasura-benchmarks-runner-'+benchmark_set
                               },
                               # "Owner" here is an arbitrary name; this tag allows us to define an
                               # IAM policy that effectively restricts hasura-benchmarks-runner to
                               # only terminating instances that it has started (here):
                               {'Key': 'Owner',
                                'Value': 'hasura-benchmarks-runner'
                               }
                            ]
                        }],
                        SecurityGroupIds=[ 'hasura-benchmarks-runner' ]
                     )[0]

                except botocore.exceptions.ClientError as error:
                    if error.response['Error']['Code'] == 'InsufficientInstanceCapacity':
                        say(f"Warning, got InsufficientInstanceCapacity in region {region}. Trying the next one")
                        if region == ok_regions[-1]:
                            say('Waiting a bit, hoping capacity gets added before going through regions again')
                            time.sleep(20)
                        continue
                    else:
                        raise

                # Above succeeded, presumably, so we can return...

                # Ensure we clean up instances even on error:
                LAUNCHED_INSTANCES.put(instance)

                instance.wait_until_running()
                instance.load()
                # NOTE: at this point we may still not be able to SSH in
                return instance
    try:
      # for reasons of ergonomics and compatibility on CI, we want to supply the SSH key as an environment variable. Unfortunately I'm not sure how to do that without writing to a file
      with tempfile.NamedTemporaryFile(mode='w+') as key_file:
        key_file.write(BENCHMARKS_RUNNER_PRIVATE_KEY)
        key_file.seek(0)

        instance = launch_instance()
        c = Connection(
            instance.public_dns_name,
            user="ubuntu",
            connect_timeout=10,
            connect_kwargs={
                # "key_filename": "
                "key_filename": key_file.name,
                ## NOTE: I couldn't figure out how to take the key from a string:
                ##    https://github.com/paramiko/paramiko/issues/1866
                # "pkey": paramiko.rsakey.RSAKey.from_private_key(io.StringIO(BENCHMARKS_AWS_PRIVATE_KEY)),
            }
        )
        # It can take some time for our EC2 instances to become available, so
        # we need to retry SSH connections for a while:
        say("Waiting for SSH to come up")
        conn_attempts = range(0,20)
        for n in conn_attempts:
            try:
                c.run("whoami", hide='out')
            except:
                if n == conn_attempts[-1]:
                    raise
                else:
                    time.sleep(1)
                    continue
            break

        # Install any extra dependencies (TODO: bake these into AMI)
        c.sudo('apt-get update')
        c.sudo('apt-get upgrade -y')
        c.sudo('apt-get install -y jq')

        # In case our heroic exception handling and cleanup attempts here fail,
        # make sure this instance shuts down (and is terminated, per
        # InstanceInitiatedShutdownBehavior) after X minutes:
        c.sudo('shutdown -P +20 "Oops, we failed to clean up this instance; terminating now"')

        say("Uploading and loading docker image under test")
        patchwork.transfers.rsync(c, HASURA_DOCKER_IMAGE, '/tmp/hasura_image.tar', rsync_opts="--quiet")
        hasura_docker_image_name = c.run(
            "docker load -i /tmp/hasura_image.tar | grep '^Loaded image: ' | sed 's/Loaded image: //g'",
            pty=True
        ).stdout.strip()

        say(f"Running benchmarks for: {hasura_docker_image_name}")
        # Upload the benchmarks directory to remote (though we only care about 'benchmark_set')
        patchwork.transfers.rsync(c, abs_path('../benchmarks'), '/tmp', exclude='venv', rsync_opts="--quiet")
        with c.cd("/tmp/benchmarks"):
            # We'll sleep for the 'huge_schema' case to allow memory to settle,
            # since measuring idle residency to support the schema is the main
            # point of this test. Since 'chinook' takes much longer we don't
            # lose any wallclock CI time by waiting here
            # TODO the fact that we're mentioning a specific benchmark set here is a wart:
            post_setup_sleep = 90 if benchmark_set == 'huge_schema' else 0
            # NOTE: it seems like K6 is what requires pty here:
            # NOTE: add hide='both' here if we decide to suppress output
            lkey = os.environ['HASURA_GRAPHQL_EE_LICENSE_KEY']
            bench_result = c.run(f"HASURA_GRAPHQL_EE_LICENSE_KEY={lkey} ./bench.sh {benchmark_set} {hasura_docker_image_name} {post_setup_sleep}", pty=True)

        with tempfile.TemporaryDirectory("-hasura-benchmarks") as tmp:
            filename = f"{benchmark_set}.json"
            say(f"Fetching results and uploading to S3. Available at: {s3_url(filename)}")

            local_path = os.path.join(tmp, filename)
            c.get(f"/tmp/benchmarks/benchmark_sets/{benchmark_set}/report.json", local=local_path)

            s3.upload_file(
                local_path, RESULTS_S3_BUCKET, f"{THIS_S3_BUCKET_PREFIX}/{filename}",
                ExtraArgs={'ACL': 'public-read'}
            )

        # Terminate ASAP, to save money, even though we also ensure cleanup in main():
        say("Success! Shutting down")
        instance.terminate()

        return bench_result

    # If AWS evicted our spot instance (probably), try again with on-demand
    except invoke.exceptions.UnexpectedExit:
        if SHUTTING_DOWN:
            warn("interrupted, exiting")
            return None
        if use_spot:
            warn("Dang, it looks like our spot instance was evicted! Retrying with on-demand")
            run_benchmark_set(benchmark_set, use_spot=False)
        else:
            raise

# Create a regression report between the benchmarks we just ran (for PR_NUMBER;
# assumes 'run_benchmark_set' has been run for all)
#
# NOTE: this is a little inefficient (since we fetch from S3 the report we just
# generated) in the service of simplicity and so that this function might be
# re-used if this becomes a proper fabfile
def generate_regression_report():
    boto3_session = new_boto_session()
    s3 = boto3_session.client('s3')

    def fetch_report_json(prefix, bench_name):
        tmp = io.BytesIO()
        s3.download_fileobj(RESULTS_S3_BUCKET, f"{prefix}/{bench_name}", tmp)
        tmp.seek(0)
        return json.load(tmp)

    # Find the PR number of the nearest parent commit, to compare against.
    # NOTE: we need a somewhat long history here, because there may be a long
    # string of e.g. console commits which never resulted in a benchmark run in CI:
    merge_base_candidates = (
        local(f"utils/pr-merge-bases.sh {PR_TARGET_BRANCH} 30", hide='stdout')
        .stdout
        .splitlines()
    )
    # Find the most recent PR in merge base with some benchmark results:
    for pr_num in merge_base_candidates:
        if 'Contents' in s3.list_objects(Bucket=RESULTS_S3_BUCKET, Prefix=f"mono-pr-{pr_num}/"):
            merge_base_pr = pr_num
            break
    try:
      if merge_base_pr != merge_base_candidates[0]:
          warn("Showing regression report against older PR in merge base!")
      say(f"Comparing performance to changes from https://github.com/hasura/graphql-engine-mono/pull/{merge_base_pr}")
    except UnboundLocalError:
      warn(f"Could not find a commit in merge base with associated benchmarks! (among {merge_base_candidates}")
      warn(f"Exiting")
      raise

    # We'll accumulate a structure like this, with all non-empty maps:
    #   :: Map BenchSetName (MemInUsePctChg, LiveBytesPctChg, [ (BenchmarkName, Map Metric PctChg) ]
    results = {}

    # For each benchmark set we uploaded, for PR_NUMBER...
    for o in s3.list_objects(Bucket=RESULTS_S3_BUCKET, Prefix=f"{THIS_S3_BUCKET_PREFIX}/")['Contents']:
        this_prefix, benchmark_set_name = o['Key'].split('/')
        this_report = fetch_report_json(this_prefix, benchmark_set_name)
        try:
            merge_base_report = fetch_report_json(f"mono-pr-{merge_base_pr}", benchmark_set_name)
        except botocore.exceptions.ClientError:
            # This will happen, e.g. when a new benchmark set is added in this change set
            warn(f"No results for {benchmark_set_name} found for PR #{merge_base_pr}. Skipping")
            continue

        # A benchmark set may contain no queries (e.g. formerly, If it was just
        # using the ad hoc operation mode), in which case the results are an
        # empty array.  Skip in those cases for now
        if not (this_report and merge_base_report):
            continue

        benchmark_set_results = []

        # So we can look up by benchmark name:
        merge_base_report_dict = {}
        for bench in merge_base_report:
            merge_base_report_dict[bench['name']] = bench

        # Record residency stats before any benchmarks have run, to present as
        # a baseline for serving this particular schema:
        def mem_regression(ix, stat):
            this_bytes       =       this_report[ix]["extended_hasura_checks"][stat]
            merge_base_bytes = merge_base_report[ix]["extended_hasura_checks"][stat]
            return pct_change(merge_base_bytes, this_bytes)
        mem_in_use_before_diff = mem_regression(0, "mem_in_use_bytes_before")
        live_bytes_before_diff = mem_regression(0, "live_bytes_before")
        # ...and also the live_bytes after, which lets us see e.g. whether a
        # memory improvement was just creation of thunks that get evaluated
        # when we do actual work:
        live_bytes_after_diff = mem_regression(-1, "live_bytes_after")
        mem_in_use_after_diff = mem_regression(-1, "mem_in_use_bytes_after")
        # ^^^ NOTE: ideally we'd want to pause before collecting mem_in_use
        #     here too I guess, to allow RTS to reclaim

        for this_bench in this_report:
            # this_bench['requests']['count'] # TODO use this to normalize allocations
            name = this_bench['name']

            # Skip if: this is a "low load" variation with few samples since these are 
            #          likely redundant / less useful for the purpose of finding regressions
            #          (see mono #5942)
            if "low_load" in name:
                warn(f"Skipping '{name}' which has 'low_load' in name")
                continue

            # Skip if: no result in merge base report to compare to:
            try:
                merge_base_bench = merge_base_report_dict[name]
            except KeyError:
                warn(f"Skipping '{name}' which is not found in the old report")
                continue

            # NOTE: below we want to skip any metrics not present in both reports,
            # since we might decide to add or need to remove some:
            metrics = {}

            # if this is a throughput benchmark set ( identified by the word
            # "throughput" in the name)  then for now just look at the average
            # RPS for the purposes of this regression report
            if "throughput" in benchmark_set_name:
                try:
                    metrics['avg_peak_rps'] = pct_change(
                        merge_base_bench["requests"]["average"],
                        this_bench[      "requests"]["average"]
                    )
                    benchmark_set_results.append((name, metrics))
                except KeyError:
                    pass
                # skip remaining metrics:
                continue

            try:
                metrics['bytes_alloc_per_req'] = pct_change(
                    merge_base_bench["extended_hasura_checks"]["bytes_allocated_per_request"],
                    this_bench[      "extended_hasura_checks"]["bytes_allocated_per_request"]
                )
            except KeyError:
                continue

            # For now just report regressions in the stable bytes-allocated metric for adhoc
            if name.startswith("ADHOC-"):
                warn(f"Just reporting regressions in bytes_alloc_per_req for '{name}' which is adhoc")
                benchmark_set_results.append((name, metrics))
                # Skip everything else:
                continue

            # Response body size:
            try:
                merge_base_body_size = float(merge_base_bench['response']['totalBytes']) / float(merge_base_bench['requests']['count'])
                this_body_size       = float(      this_bench['response']['totalBytes']) / float(      this_bench['requests']['count'])
                response_body_change = pct_change(merge_base_body_size, this_body_size)
                # filter response body size unless it changes significantly, since this is rare:
                if abs(response_body_change) > 1:
                    metrics['response_body_size'] = response_body_change
            # We need to catch division by zero here for adhoc mode queries
            # (where we just set total_bytes to 0 for now), but probably want
            # to keep this in even if that changes.
            except (ZeroDivisionError, KeyError):
                pass
            # NOTE: we decided to omit higher-percentile latencies here since
            # they are noisy (which might lead to people ignoring benchmarks)
            # NOTE: we originally had `min` here, thinking it should be an
            # asymptote (we can only get so fast doing a particular workload),
            # but this hasn't turned out to be a useful summary statistic (we
            # might need several times more samples for it to stabilize)
            for m in ['p50']:
                try:
                    this_hist = this_bench['histogram']['json']
                    merge_base_hist = merge_base_bench['histogram']['json']

                    # Store percent difference from this to merge base:
                    metrics[m] = pct_change(merge_base_hist[m], this_hist[m])

                # We only expect ZeroDivisionError for old reports, before we
                # fixed precision issue in graphql-bench:
                except (KeyError, ZeroDivisionError):
                    continue

            if metrics == {}:
                # again, this should only happen with old reports with zeros
                warn(f"Skipping {name} since metrics are empty")
                continue
            benchmark_set_results.append((name, metrics))

        results[benchmark_set_name] = (mem_in_use_before_diff, live_bytes_before_diff, mem_in_use_after_diff, live_bytes_after_diff, benchmark_set_results)

    return results, merge_base_pr

# We (ab)use githubs syntax highlighting for displaying the regression report
# table as a github comment, that can be easily scanned
def pretty_print_regression_report_github_comment(results, skip_pr_report_names, merge_base_pr, output_filename):
    f = open(output_filename, "w")
    def out(s): f.write(s+"\n")

    out(f"## Benchmark Results (graphql-engine-pro)") # NOTE: We use this header to identify benchmark reports in `hide-benchmark-reports.sh`
    out(f"<details closed><summary>Click for detailed reports, and help docs</summary>")
    out(f"")
    out((f"The regression report below shows, for each benchmark, the **percent change** for "
         f"different metrics, between the merge base (the changes from **PR {merge_base_pr}**) and "
         # NOTE: we don't use #{merge_base_pr} because we want to avoid backlinks from the target PRs
         f"this PR. For advice on interpreting benchmarks, please see [benchmarks/README.md]"
         f"(https://github.com/hasura/graphql-engine-mono/blob/main/server/benchmarks/README.md)."))
    out(f"")
    out(f"More significant regressions or improvements will be colored with `#b31d28` or `#22863a`, respectively.")
    out(f"NOTE: throughput benchmarks are quite variable for now, and have a looser threshold for highlighting.")
    out(f"")
    out(f"You can view graphs of the full reports here:")
    for benchmark_set_name, _ in results.items():
        these_id = s3_short_id(benchmark_set_name)
        base_id  = s3_short_id(benchmark_set_name, 'mono-pr-'+merge_base_pr)
        out(f"- **{benchmark_set_name}**: "
            f"[:bar_chart: these changes]({graphql_bench_url([these_id])})... "
            f"[:bar_chart: merge base]({graphql_bench_url([base_id])})... "
            f"[:bar_chart: both compared]({graphql_bench_url([these_id, base_id])})")
    out(f"")
    out(f"</details>")
    out(f"")

    # Return what should be the first few chars of the line, which will detemine its styling:
    def highlight_sensitive(val=None):
        if val == None:        return "#   "  # GRAY
        elif abs(val) <= 2.0:  return "#   "  # GRAY
        elif abs(val) <= 3.5:  return "*   "  # NORMAL
        # ^^^ So far variation in min, bytes, and median seem to stay within this range.
        elif 0 < val <= 15.0:  return "-   "  # RED
        elif 0 < val <= 25.0:  return "--  "  # RED
        elif 0 < val:          return "--- "  # RED
        elif -15.0 <= val < 0: return "+   "  # GREEN
        elif -25.0 <= val < 0: return "++  "  # GREEN
        else:                  return "+++ "  # GREEN
    # For noisier benchmarks (tuned for throughput benchmarks, for now)
    def highlight_lax(val=None):
        if val == None:        return "#   "  # GRAY
        elif abs(val) <= 8.0:  return "#   "  # GRAY
        elif abs(val) <= 12.0: return "*   "  # NORMAL
        elif 0 < val <= 20.0:  return "-   "  # RED
        elif 0 < val <= 35.0:  return "--  "  # RED
        elif 0 < val:          return "--- "  # RED
        elif -20.0 <= val < 0: return "+   "  # GREEN
        elif -35.0 <= val < 0: return "++  "  # GREEN
        else:                  return "+++ "  # GREEN

    out(f"``` diff")  # START DIFF SYNTAX
    for benchmark_set_name, (mem_in_use_before_diff, live_bytes_before_diff, mem_in_use_after_diff, live_bytes_after_diff, benchmarks) in results.items():
        if benchmark_set_name[:-5] in skip_pr_report_names: continue
        l0 = live_bytes_before_diff
        l1 = live_bytes_after_diff
        u0 = mem_in_use_before_diff
        # u1 = mem_in_use_after_diff

        col = highlight_sensitive
        out(        f"{col(u0)} {benchmark_set_name[:-5]+'  ':─<21s}{'┤ MEMORY RESIDENCY (from RTS)': <30}{'mem_in_use (BEFORE benchmarks)': >38}{u0:>12.1f} ┐")
        out(        f"{col(l0)} {                        '  ': <21s}{'│'                            : <30}{'live_bytes (BEFORE benchmarks)': >38}{l0:>12.1f} │")
        out(        f"{col(l1)} {                        '  ': <21s}{'│'                              }{'   live_bytes  (AFTER benchmarks)':_>67}{l1:>12.1f} ┘")
        for bench_name, metrics in benchmarks:
            bench_name_pretty = bench_name.replace('-k6-custom','').replace('_',' ') # need at least 40 chars
            if "throughput" in benchmark_set_name:
                # invert the sign so we color properly, since higher throughput is better:
                col = lambda v: highlight_lax(-v)
            else:
                col = highlight_sensitive

            for metric_name, d in metrics.items():
              if len(list(metrics.items())) == 1:  # if only one metric:
                out(f"{col(d )} {                        '  ': <21s}{'│_'+bench_name_pretty+' '     :_<40}{                     metric_name:_>28}{d :>12.1f}  ")
              elif metric_name == list(metrics.items())[0][0]:  # first:
                out(f"{col(d )} {                        '  ': <21s}{'│ '+bench_name_pretty         : <40}{                     metric_name: >28}{d :>12.1f} ┐")
              elif metric_name == list(metrics.items())[-1][0]:  # last:
                out(f"{col(d )} {                        '  ': <21s}{'│'                                 }{               '   '+metric_name:_>67}{d :>12.1f} ┘")
              else:   # middle, omit name
                out(f"{col(d )} {                        '  ': <21s}{'│ '                           : <40}{                     metric_name: >28}{d :>12.1f} │")


    out(f"```")  # END DIFF SYNTAX

    say(f"Wrote github comment to {REGRESSION_REPORT_COMMENT_FILENAME}")
    f.close()


def pct_change(previous, current):
  return ((float(current)-previous)/previous)*100

# return an absolute path, from one relative to this file
def abs_path(p):
    return os.path.join(os.path.dirname(__file__), p)

# Traverse a dictionary using tuple of keys as path
def get_path(d, path):
    return reduce(dict.get, path, d)

def say(s):
    print(f"***\033[92m {s} \033[0m")
def warn(s):
    print(f"***\033[93m {s} \033[0m")


if __name__ == "__main__":
    main()