From 3eff4cb5a56ff9c9405247cd820a3c4994c39b17 Mon Sep 17 00:00:00 2001 From: Malmahrouqi3 Date: Mon, 10 Nov 2025 13:16:18 -0500 Subject: [PATCH 1/4] bench_diff improve --- toolchain/mfc/args.py | 10 ++++++--- toolchain/mfc/bench.py | 51 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/toolchain/mfc/args.py b/toolchain/mfc/args.py index 169ee076a4..4658b7cb23 100644 --- a/toolchain/mfc/args.py +++ b/toolchain/mfc/args.py @@ -138,6 +138,8 @@ def add_common_arguments(p: argparse.ArgumentParser, mask = None): add_common_arguments(bench_diff, "t") bench_diff.add_argument("lhs", metavar="LHS", type=str, help="Path to a benchmark result YAML file.") bench_diff.add_argument("rhs", metavar="RHS", type=str, help="Path to a benchmark result YAML file.") + bench_diff.add_argument("-f", "--file", metavar="FILE", type=str, required=False, default=None, help="Path to the data file (e.g., data.js).") + bench_diff.add_argument("-n", "--name", metavar="NAME", nargs="+", type=str, required=False, default=[], help="Test name (e.g. GT Phoenix (CPU)).") # COUNT add_common_arguments(count, "g") @@ -170,9 +172,6 @@ def add_common_arguments(p: argparse.ArgumentParser, mask = None): parser.print_help() exit(-1) - # "Slugify" the name of the job - args["name"] = re.sub(r'[\W_]+', '-', args["name"]) - # We need to check for some invalid combinations of arguments because of # the limitations of argparse. if args["command"] == "build": @@ -181,6 +180,11 @@ def add_common_arguments(p: argparse.ArgumentParser, mask = None): if args["command"] == "run": if args["binary"] is not None and args["engine"] != "interactive": raise MFCException("./mfc.sh run's --binary can only be used with --engine=interactive.") + if isinstance(args["name"], str): + # "Slugify" the name of the job + args["name"] = re.sub(r'[\W_]+', '-', args["name"]).strip('-') + elif args["command"] == "bench_diff" and len(args["name"]) > 0: + args["name"] = " ".join(args["name"]) # Input files to absolute paths for e in ["input", "input1", "input2"]: diff --git a/toolchain/mfc/bench.py b/toolchain/mfc/bench.py index eb1b120035..970664b77f 100644 --- a/toolchain/mfc/bench.py +++ b/toolchain/mfc/bench.py @@ -1,4 +1,4 @@ -import os, sys, uuid, subprocess, dataclasses, typing, math +import os, sys, uuid, subprocess, dataclasses, typing, math, json import rich.table @@ -106,16 +106,51 @@ def diff(): Using intersection: {slugs} with {len(slugs)} elements. """) + cb_stats = {} + if ARG("file") is not None and ARG("name") is not None: + try: + with open(ARG("file"), 'r') as f: + data_json = json.load(f) + + cb_test = ARG("name") + if "entries" in data_json and cb_test in data_json["entries"]: + benchmark_runs = data_json["entries"][cb_test] + case_times = {} + for run in benchmark_runs: + if "benches" not in run: + continue + for bench in run["benches"]: + case_name = bench.get("name") + grind_value = bench.get("value") + if case_name is None or grind_value is None: + continue + if case_name not in case_times: + case_times[case_name] = [] + case_times[case_name].append(grind_value) + for case_name, values in case_times.items(): + if len(values) > 0: + avg = sum(values) / len(values) + cb_stats[case_name] = {"avg": avg, "count": len(values)} + + cons.print(f"[bold]Loaded cb data for test: [bold]{cb_test}[/bold] ({len(cb_stats)} cases)[/bold]") + else: + cons.print(f"[bold yellow]Warning[/bold yellow]: Test '[bold]{cb_test}[/bold]' not found in data file.") + except Exception as e: + cons.print(f"[bold yellow]Warning[/bold yellow]: Could not load data file: {e}") + table = rich.table.Table(show_header=True, box=rich.table.box.SIMPLE) table.add_column("[bold]Case[/bold]", justify="left") table.add_column("[bold]Pre Process[/bold]", justify="right") table.add_column("[bold]Simulation[/bold]", justify="right") table.add_column("[bold]Post Process[/bold]", justify="right") + if cb_stats: + table.add_column("[bold] CB (Grind)[/bold]", justify="right") err = 0 for slug in slugs: lhs_summary, rhs_summary = lhs["cases"][slug]["output_summary"], rhs["cases"][slug]["output_summary"] speedups = ['N/A', 'N/A', 'N/A'] + grind_comparison = 'N/A' for i, target in enumerate(sorted(DEFAULT_TARGETS, key=lambda t: t.runOrder)): if (target.name not in lhs_summary) or (target.name not in rhs_summary): @@ -140,11 +175,23 @@ def diff(): if grind_time_value <0.95: cons.print(f"[bold red]Error[/bold red]: Benchmarking failed since grind time speedup for {target.name} below acceptable threshold (<0.95) - Case: {slug}") err = 1 + if slug in cb_stats: + rhs_grind = rhs_summary[target.name]["grind"] + stats = cb_stats[slug] + avg = stats["avg"] + offset_pct = ((rhs_grind - avg) / avg * 100) + color = "red" if (offset_pct) > 0 else "yellow" if abs(offset_pct) == 0 else "green" + grind_comparison = f"[{color}]{offset_pct:+.2f}%[/{color}] (avg: {avg:.2f})" + except Exception as _: pass - table.add_row(f"[magenta]{slug}[/magenta]", *speedups) + row = [f"[magenta]{slug}[/magenta]", *speedups] + if cb_stats: + row.append(grind_comparison) + table.add_row(*row) cons.raw.print(table) if err: raise MFCException("Benchmarking failed") + \ No newline at end of file From a2eaec7c4d2dc700a40dc96b1d5d8045ebb4ebed Mon Sep 17 00:00:00 2001 From: Malmahrouqi3 Date: Mon, 10 Nov 2025 13:29:36 -0500 Subject: [PATCH 2/4] implement CB on bench.yml --- .github/workflows/bench.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 7198b20e52..41bb4f7396 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -97,8 +97,10 @@ jobs: - name: Generate & Post Comment run: | + git clone "${{ secrets.DOC_PUSH_URL }}" ../www + cp ../www/documentation/data.js ./pr/data.js (cd pr && . ./mfc.sh load -c ${{ matrix.flag }} -m g) - (cd pr && ./mfc.sh bench_diff ../master/bench-${{ matrix.device }}-${{ matrix.interface }}.yaml ../pr/bench-${{ matrix.device }}-${{ matrix.interface }}.yaml) + (cd pr && ./mfc.sh bench_diff ../master/bench-${{ matrix.device }}-${{ matrix.interface }}.yaml ../pr/bench-${{ matrix.device }}-${{ matrix.interface }}.yaml -f data.js -n "${{ matrix.name }} (${{ matrix.device }}/${{ matrix.interface }})") - name: Print Logs if: always() From 8499ec53b8ec4c322a2d48eb88d0f468293c0735 Mon Sep 17 00:00:00 2001 From: Malmahrouqi3 Date: Mon, 10 Nov 2025 14:10:01 -0500 Subject: [PATCH 3/4] lint fixes --- bench.yaml | 285 +++++++++++++++ data.js | 790 +++++++++++++++++++++++++++++++++++++++++ toolchain/mfc/args.py | 4 +- toolchain/mfc/bench.py | 60 ++-- 4 files changed, 1102 insertions(+), 37 deletions(-) create mode 100755 bench.yaml create mode 100755 data.js diff --git a/bench.yaml b/bench.yaml new file mode 100755 index 0000000000..a6d83042c1 --- /dev/null +++ b/bench.yaml @@ -0,0 +1,285 @@ +cases: + 5eq_rk3_weno3_hllc: + description: + args: + - -c + - phoenix + - --gpu + - -g + - '0' + - '1' + - -n + - '2' + path: /opt/MFC/benchmarks/5eq_rk3_weno3_hllc/case.py + slug: 5eq_rk3_weno3_hllc + output_summary: + invocation: + - run + - /opt/MFC/benchmarks/5eq_rk3_weno3_hllc/case.py + - --case-optimization + - --targets + - pre_process + - simulation + - post_process + - --output-summary + - /opt/MFC/build/benchmarks/9de1/5eq_rk3_weno3_hllc.yaml + - -c + - phoenix + - --gpu + - -g + - '0' + - '1' + - -n + - '2' + - -- + - --gbpp + - '12' + lock: + debug: false + fastmath: false + gcov: false + gpu: true + mpi: true + single: false + unified: false + post_process: + exec: 4.8194991 + pre_process: + exec: 10.0581012 + simulation: + exec: 47.2468157 + grind: 0.67073686 + syscheck: + exec: 1.3798784 + hypo_hll: + description: + args: + - -c + - phoenix + - --gpu + - -g + - '0' + - '1' + - -n + - '2' + path: /opt/MFC/benchmarks/hypo_hll/case.py + slug: hypo_hll + output_summary: + invocation: + - run + - /opt/MFC/benchmarks/hypo_hll/case.py + - --case-optimization + - --targets + - pre_process + - simulation + - post_process + - --output-summary + - /opt/MFC/build/benchmarks/9de1/hypo_hll.yaml + - -c + - phoenix + - --gpu + - -g + - '0' + - '1' + - -n + - '2' + - -- + - --gbpp + - '12' + lock: + debug: false + fastmath: false + gcov: false + gpu: true + mpi: true + single: false + unified: false + post_process: + exec: 9.2257145 + pre_process: + exec: 92.3468801 + simulation: + exec: 245.5551634 + grind: 323170 + syscheck: + exec: 1.2918744 + ibm: + description: + args: + - -c + - phoenix + - --gpu + - -g + - '0' + - '1' + - -n + - '2' + path: /opt/MFC/benchmarks/ibm/case.py + slug: ibm + output_summary: + invocation: + - run + - /opt/MFC/benchmarks/ibm/case.py + - --case-optimization + - --targets + - pre_process + - simulation + - post_process + - --output-summary + - /opt/MFC/build/benchmarks/9de1/ibm.yaml + - -c + - phoenix + - --gpu + - -g + - '0' + - '1' + - -n + - '2' + - -- + - --gbpp + - '12' + lock: + debug: false + fastmath: false + gcov: false + gpu: true + mpi: true + single: false + unified: false + post_process: + exec: 4.9753983 + pre_process: + exec: 11.9773121 + simulation: + exec: 148.6407971 + grind: 32317 + syscheck: + exec: 1.230999 + igr: + description: + args: + - -c + - phoenix + - --gpu + - -g + - '0' + - '1' + - -n + - '2' + path: /opt/MFC/benchmarks/igr/case.py + slug: igr + output_summary: + invocation: + - run + - /opt/MFC/benchmarks/igr/case.py + - --case-optimization + - --targets + - pre_process + - simulation + - post_process + - --output-summary + - /opt/MFC/build/benchmarks/9de1/igr.yaml + - -c + - phoenix + - --gpu + - -g + - '0' + - '1' + - -n + - '2' + - -- + - --gbpp + - '12' + lock: + debug: false + fastmath: false + gcov: false + gpu: true + mpi: true + single: false + unified: false + pre_process: + exec: 8.1699418 + simulation: + exec: 36.3594534 + grind: 0.72979167 + syscheck: + exec: 1.3865383 + viscous_weno5_sgb_acoustic: + description: + args: + - -c + - phoenix + - --gpu + - -g + - '0' + - '1' + - -n + - '2' + path: /opt/MFC/benchmarks/viscous_weno5_sgb_acoustic/case.py + slug: viscous_weno5_sgb_acoustic + output_summary: + invocation: + - run + - /opt/MFC/benchmarks/viscous_weno5_sgb_acoustic/case.py + - --case-optimization + - --targets + - pre_process + - simulation + - post_process + - --output-summary + - /opt/MFC/build/benchmarks/9de1/viscous_weno5_sgb_acoustic.yaml + - -c + - phoenix + - --gpu + - -g + - '0' + - '1' + - -n + - '2' + - -- + - --gbpp + - '12' + lock: + debug: false + fastmath: false + gcov: false + gpu: true + mpi: true + single: false + unified: false + post_process: + exec: 4.8518692 + pre_process: + exec: 8.7731736 + simulation: + exec: 94.3440505 + grind: 1.99630224 + syscheck: + exec: 1.2763789 +metadata: + invocation: + - bench + - --mem + - '12' + - -j + - '24' + - -o + - v5.0.6-gpu.yaml + - -- + - -c + - phoenix + - --gpu + - -g + - '0' + - '1' + - -n + - '2' + lock: + debug: false + fastmath: false + gcov: false + gpu: true + mpi: true + single: false + unified: false diff --git a/data.js b/data.js new file mode 100755 index 0000000000..59c00b241e --- /dev/null +++ b/data.js @@ -0,0 +1,790 @@ +{ + "lastUpdate": 1762153869098, + "repoUrl": "https://github.com/Malmahrouqi3/MFC-mo2", + "entries": { + "GT Phoenix (CPU)": [ + { + "commit": { + "author": { + "name": "Mohammed S. Al-Mahrouqi", + "username": "Malmahrouqi3", + "email": "145478595+Malmahrouqi3@users.noreply.github.com" + }, + "committer": { + "name": "GitHub", + "username": "web-flow", + "email": "noreply@github.com" + }, + "id": "e2a8ccc6a426a0575a9fdb8752f1f85913f700f4", + "message": "Update report.yml to include GPU benchmarking\n\nAdded separate benchmark storage for GPU results.", + "timestamp": "2025-10-28T21:44:51Z", + "url": "https://github.com/Malmahrouqi3/MFC-mo2/commit/e2a8ccc6a426a0575a9fdb8752f1f85913f700f4" + }, + "date": 1761688402289, + "tool": "googlecpp", + "benches": [ + { + "name": "5eq_rk3_weno3_hllc", + "value": 29275, + "unit": "s", + "extra": "time steps: 94877\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "hypo_hll", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "ibm", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "igr", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "viscous_weno5_sgb_acoustic", + "value": 32724, + "unit": "s", + "extra": "time steps: 21393\nruntime: 29275 s\nthreads: 1" + } + ] + }, + { + "commit": { + "author": { + "name": "Mohammed S. Al-Mahrouqi", + "username": "Malmahrouqi3", + "email": "145478595+Malmahrouqi3@users.noreply.github.com" + }, + "committer": { + "name": "GitHub", + "username": "web-flow", + "email": "noreply@github.com" + }, + "id": "093adb218db7ff5e50e10bf5616d6a9c434935c9", + "message": "Add clean step for benchmark data repository\n\nAdd step to clean benchmark data repository directory before storing results.", + "timestamp": "2025-10-28T21:54:39Z", + "url": "https://github.com/Malmahrouqi3/MFC-mo2/commit/093adb218db7ff5e50e10bf5616d6a9c434935c9" + }, + "date": 1761688557903, + "tool": "googlecpp", + "benches": [ + { + "name": "5eq_rk3_weno3_hllc", + "value": 29275, + "unit": "s", + "extra": "time steps: 94877\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "hypo_hll", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "ibm", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "igr", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "viscous_weno5_sgb_acoustic", + "value": 32724, + "unit": "s", + "extra": "time steps: 21393\nruntime: 29275 s\nthreads: 1" + } + ] + }, + { + "commit": { + "author": { + "name": "Mohammed S. Al-Mahrouqi", + "username": "Malmahrouqi3", + "email": "145478595+Malmahrouqi3@users.noreply.github.com" + }, + "committer": { + "name": "GitHub", + "username": "web-flow", + "email": "noreply@github.com" + }, + "id": "c074f4144e40da4bda67bea415e45e574015c096", + "message": "Restore data.js in docs.yml workflow\n\nRestore documentation/data.js in the GitHub Actions workflow.", + "timestamp": "2025-11-03T06:33:40Z", + "url": "https://github.com/Malmahrouqi3/MFC-mo2/commit/c074f4144e40da4bda67bea415e45e574015c096" + }, + "date": 1762152228006, + "tool": "googlecpp", + "benches": [ + { + "name": "5eq_rk3_weno3_hllc", + "value": 29275, + "unit": "s", + "extra": "time steps: 94877\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "hypo_hll", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "ibm", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "igr", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "viscous_weno5_sgb_acoustic", + "value": 32724, + "unit": "s", + "extra": "time steps: 21393\nruntime: 29275 s\nthreads: 1" + } + ] + }, + { + "commit": { + "author": { + "name": "Mohammed S. Al-Mahrouqi", + "username": "Malmahrouqi3", + "email": "145478595+Malmahrouqi3@users.noreply.github.com" + }, + "committer": { + "name": "GitHub", + "username": "web-flow", + "email": "noreply@github.com" + }, + "id": "c074f4144e40da4bda67bea415e45e574015c096", + "message": "Restore data.js in docs.yml workflow\n\nRestore documentation/data.js in the GitHub Actions workflow.", + "timestamp": "2025-11-03T06:33:40Z", + "url": "https://github.com/Malmahrouqi3/MFC-mo2/commit/c074f4144e40da4bda67bea415e45e574015c096" + }, + "date": 1762152238684, + "tool": "googlecpp", + "benches": [ + { + "name": "5eq_rk3_weno3_hllc", + "value": 29275, + "unit": "s", + "extra": "time steps: 94877\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "hypo_hll", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "ibm", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "igr", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "viscous_weno5_sgb_acoustic", + "value": 32724, + "unit": "s", + "extra": "time steps: 21393\nruntime: 29275 s\nthreads: 1" + } + ] + }, + { + "commit": { + "author": { + "name": "Mohammed S. Al-Mahrouqi", + "username": "Malmahrouqi3", + "email": "145478595+Malmahrouqi3@users.noreply.github.com" + }, + "committer": { + "name": "GitHub", + "username": "web-flow", + "email": "noreply@github.com" + }, + "id": "c074f4144e40da4bda67bea415e45e574015c096", + "message": "Restore data.js in docs.yml workflow\n\nRestore documentation/data.js in the GitHub Actions workflow.", + "timestamp": "2025-11-03T06:33:40Z", + "url": "https://github.com/Malmahrouqi3/MFC-mo2/commit/c074f4144e40da4bda67bea415e45e574015c096" + }, + "date": 1762152289204, + "tool": "googlecpp", + "benches": [ + { + "name": "5eq_rk3_weno3_hllc", + "value": 29275, + "unit": "s", + "extra": "time steps: 94877\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "hypo_hll", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "ibm", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "igr", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "viscous_weno5_sgb_acoustic", + "value": 32724, + "unit": "s", + "extra": "time steps: 21393\nruntime: 29275 s\nthreads: 1" + } + ] + }, + { + "commit": { + "author": { + "name": "Mohammed S. Al-Mahrouqi", + "username": "Malmahrouqi3", + "email": "145478595+Malmahrouqi3@users.noreply.github.com" + }, + "committer": { + "name": "GitHub", + "username": "web-flow", + "email": "noreply@github.com" + }, + "id": "c074f4144e40da4bda67bea415e45e574015c096", + "message": "Restore data.js in docs.yml workflow\n\nRestore documentation/data.js in the GitHub Actions workflow.", + "timestamp": "2025-11-03T06:33:40Z", + "url": "https://github.com/Malmahrouqi3/MFC-mo2/commit/c074f4144e40da4bda67bea415e45e574015c096" + }, + "date": 1762152302280, + "tool": "googlecpp", + "benches": [ + { + "name": "5eq_rk3_weno3_hllc", + "value": 29275, + "unit": "s", + "extra": "time steps: 94877\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "hypo_hll", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "ibm", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "igr", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "viscous_weno5_sgb_acoustic", + "value": 32724, + "unit": "s", + "extra": "time steps: 21393\nruntime: 29275 s\nthreads: 1" + } + ] + }, + { + "commit": { + "author": { + "name": "Mohammed S. Al-Mahrouqi", + "username": "Malmahrouqi3", + "email": "145478595+Malmahrouqi3@users.noreply.github.com" + }, + "committer": { + "name": "GitHub", + "username": "web-flow", + "email": "noreply@github.com" + }, + "id": "f7feb4edce2d124e726e718f185b77e4c2172c83", + "message": "Add step to polish data.js in report workflow\n\nAdded a step to polish data.js in the workflow.", + "timestamp": "2025-11-03T06:56:10Z", + "url": "https://github.com/Malmahrouqi3/MFC-mo2/commit/f7feb4edce2d124e726e718f185b77e4c2172c83" + }, + "date": 1762153152507, + "tool": "googlecpp", + "benches": [ + { + "name": "5eq_rk3_weno3_hllc", + "value": 29275, + "unit": "s", + "extra": "time steps: 94877\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "hypo_hll", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "ibm", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "igr", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "viscous_weno5_sgb_acoustic", + "value": 32724, + "unit": "s", + "extra": "time steps: 21393\nruntime: 29275 s\nthreads: 1" + } + ] + }, + { + "commit": { + "author": { + "name": "Mohammed S. Al-Mahrouqi", + "username": "Malmahrouqi3", + "email": "145478595+Malmahrouqi3@users.noreply.github.com" + }, + "committer": { + "name": "GitHub", + "username": "web-flow", + "email": "noreply@github.com" + }, + "id": "46ae0c686d9e9e22b9d8c13d6d0bc58afab28f91", + "message": "Update sed command to replace 'iterations' with 'time steps'", + "timestamp": "2025-11-03T07:09:45Z", + "url": "https://github.com/Malmahrouqi3/MFC-mo2/commit/46ae0c686d9e9e22b9d8c13d6d0bc58afab28f91" + }, + "date": 1762153863640, + "tool": "googlecpp", + "benches": [ + { + "name": "5eq_rk3_weno3_hllc", + "value": 29275, + "unit": "s", + "extra": "time steps: 94877\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "hypo_hll", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "ibm", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "igr", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "viscous_weno5_sgb_acoustic", + "value": 32724, + "unit": "s", + "extra": "time steps: 21393\nruntime: 29275 s\nthreads: 1" + } + ] + } + ], + "GT Phoenix (GPU)": [ + { + "commit": { + "author": { + "name": "Mohammed S. Al-Mahrouqi", + "username": "Malmahrouqi3", + "email": "145478595+Malmahrouqi3@users.noreply.github.com" + }, + "committer": { + "name": "GitHub", + "username": "web-flow", + "email": "noreply@github.com" + }, + "id": "093adb218db7ff5e50e10bf5616d6a9c434935c9", + "message": "Add clean step for benchmark data repository\n\nAdd step to clean benchmark data repository directory before storing results.", + "timestamp": "2025-10-28T21:54:39Z", + "url": "https://github.com/Malmahrouqi3/MFC-mo2/commit/093adb218db7ff5e50e10bf5616d6a9c434935c9" + }, + "date": 1761688561579, + "tool": "googlecpp", + "benches": [ + { + "name": "5eq_rk3_weno3_hllc", + "value": 29275, + "unit": "s", + "extra": "time steps: 94877\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "hypo_hll", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "ibm", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "igr", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "viscous_weno5_sgb_acoustic", + "value": 32724, + "unit": "s", + "extra": "time steps: 21393\nruntime: 29275 s\nthreads: 1" + } + ] + }, + { + "commit": { + "author": { + "name": "Mohammed S. Al-Mahrouqi", + "username": "Malmahrouqi3", + "email": "145478595+Malmahrouqi3@users.noreply.github.com" + }, + "committer": { + "name": "GitHub", + "username": "web-flow", + "email": "noreply@github.com" + }, + "id": "c074f4144e40da4bda67bea415e45e574015c096", + "message": "Restore data.js in docs.yml workflow\n\nRestore documentation/data.js in the GitHub Actions workflow.", + "timestamp": "2025-11-03T06:33:40Z", + "url": "https://github.com/Malmahrouqi3/MFC-mo2/commit/c074f4144e40da4bda67bea415e45e574015c096" + }, + "date": 1762152231329, + "tool": "googlecpp", + "benches": [ + { + "name": "5eq_rk3_weno3_hllc", + "value": 29275, + "unit": "s", + "extra": "time steps: 94877\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "hypo_hll", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "ibm", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "igr", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "viscous_weno5_sgb_acoustic", + "value": 32724, + "unit": "s", + "extra": "time steps: 21393\nruntime: 29275 s\nthreads: 1" + } + ] + }, + { + "commit": { + "author": { + "name": "Mohammed S. Al-Mahrouqi", + "username": "Malmahrouqi3", + "email": "145478595+Malmahrouqi3@users.noreply.github.com" + }, + "committer": { + "name": "GitHub", + "username": "web-flow", + "email": "noreply@github.com" + }, + "id": "c074f4144e40da4bda67bea415e45e574015c096", + "message": "Restore data.js in docs.yml workflow\n\nRestore documentation/data.js in the GitHub Actions workflow.", + "timestamp": "2025-11-03T06:33:40Z", + "url": "https://github.com/Malmahrouqi3/MFC-mo2/commit/c074f4144e40da4bda67bea415e45e574015c096" + }, + "date": 1762152242305, + "tool": "googlecpp", + "benches": [ + { + "name": "5eq_rk3_weno3_hllc", + "value": 29275, + "unit": "s", + "extra": "time steps: 94877\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "hypo_hll", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "ibm", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "igr", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "viscous_weno5_sgb_acoustic", + "value": 32724, + "unit": "s", + "extra": "time steps: 21393\nruntime: 29275 s\nthreads: 1" + } + ] + }, + { + "commit": { + "author": { + "name": "Mohammed S. Al-Mahrouqi", + "username": "Malmahrouqi3", + "email": "145478595+Malmahrouqi3@users.noreply.github.com" + }, + "committer": { + "name": "GitHub", + "username": "web-flow", + "email": "noreply@github.com" + }, + "id": "c074f4144e40da4bda67bea415e45e574015c096", + "message": "Restore data.js in docs.yml workflow\n\nRestore documentation/data.js in the GitHub Actions workflow.", + "timestamp": "2025-11-03T06:33:40Z", + "url": "https://github.com/Malmahrouqi3/MFC-mo2/commit/c074f4144e40da4bda67bea415e45e574015c096" + }, + "date": 1762152292023, + "tool": "googlecpp", + "benches": [ + { + "name": "5eq_rk3_weno3_hllc", + "value": 29275, + "unit": "s", + "extra": "time steps: 94877\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "hypo_hll", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "ibm", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "igr", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "viscous_weno5_sgb_acoustic", + "value": 32724, + "unit": "s", + "extra": "time steps: 21393\nruntime: 29275 s\nthreads: 1" + } + ] + }, + { + "commit": { + "author": { + "name": "Mohammed S. Al-Mahrouqi", + "username": "Malmahrouqi3", + "email": "145478595+Malmahrouqi3@users.noreply.github.com" + }, + "committer": { + "name": "GitHub", + "username": "web-flow", + "email": "noreply@github.com" + }, + "id": "c074f4144e40da4bda67bea415e45e574015c096", + "message": "Restore data.js in docs.yml workflow\n\nRestore documentation/data.js in the GitHub Actions workflow.", + "timestamp": "2025-11-03T06:33:40Z", + "url": "https://github.com/Malmahrouqi3/MFC-mo2/commit/c074f4144e40da4bda67bea415e45e574015c096" + }, + "date": 1762152305287, + "tool": "googlecpp", + "benches": [ + { + "name": "5eq_rk3_weno3_hllc", + "value": 29275, + "unit": "s", + "extra": "time steps: 94877\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "hypo_hll", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "ibm", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "igr", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "viscous_weno5_sgb_acoustic", + "value": 32724, + "unit": "s", + "extra": "time steps: 21393\nruntime: 29275 s\nthreads: 1" + } + ] + }, + { + "commit": { + "author": { + "name": "Mohammed S. Al-Mahrouqi", + "username": "Malmahrouqi3", + "email": "145478595+Malmahrouqi3@users.noreply.github.com" + }, + "committer": { + "name": "GitHub", + "username": "web-flow", + "email": "noreply@github.com" + }, + "id": "f7feb4edce2d124e726e718f185b77e4c2172c83", + "message": "Add step to polish data.js in report workflow\n\nAdded a step to polish data.js in the workflow.", + "timestamp": "2025-11-03T06:56:10Z", + "url": "https://github.com/Malmahrouqi3/MFC-mo2/commit/f7feb4edce2d124e726e718f185b77e4c2172c83" + }, + "date": 1762153155190, + "tool": "googlecpp", + "benches": [ + { + "name": "5eq_rk3_weno3_hllc", + "value": 29275, + "unit": "s", + "extra": "time steps: 94877\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "hypo_hll", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "ibm", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "igr", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "viscous_weno5_sgb_acoustic", + "value": 32724, + "unit": "s", + "extra": "time steps: 21393\nruntime: 29275 s\nthreads: 1" + } + ] + }, + { + "commit": { + "author": { + "name": "Mohammed S. Al-Mahrouqi", + "username": "Malmahrouqi3", + "email": "145478595+Malmahrouqi3@users.noreply.github.com" + }, + "committer": { + "name": "GitHub", + "username": "web-flow", + "email": "noreply@github.com" + }, + "id": "46ae0c686d9e9e22b9d8c13d6d0bc58afab28f91", + "message": "Update sed command to replace 'iterations' with 'time steps'", + "timestamp": "2025-11-03T07:09:45Z", + "url": "https://github.com/Malmahrouqi3/MFC-mo2/commit/46ae0c686d9e9e22b9d8c13d6d0bc58afab28f91" + }, + "date": 1762153867124, + "tool": "googlecpp", + "benches": [ + { + "name": "5eq_rk3_weno3_hllc", + "value": 29275, + "unit": "s", + "extra": "time steps: 94877\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "hypo_hll", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "ibm", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "igr", + "value": 32317, + "unit": "s", + "extra": "time steps: 21609\nruntime: 29275 s\nthreads: 1" + }, + { + "name": "viscous_weno5_sgb_acoustic", + "value": 32724, + "unit": "s", + "extra": "time steps: 21393\nruntime: 29275 s\nthreads: 1" + } + ] + } + ] + } +} \ No newline at end of file diff --git a/toolchain/mfc/args.py b/toolchain/mfc/args.py index 4658b7cb23..05ed663a84 100644 --- a/toolchain/mfc/args.py +++ b/toolchain/mfc/args.py @@ -181,8 +181,8 @@ def add_common_arguments(p: argparse.ArgumentParser, mask = None): if args["binary"] is not None and args["engine"] != "interactive": raise MFCException("./mfc.sh run's --binary can only be used with --engine=interactive.") if isinstance(args["name"], str): - # "Slugify" the name of the job - args["name"] = re.sub(r'[\W_]+', '-', args["name"]).strip('-') + # "Slugify" the name of the job + args["name"] = re.sub(r'[\W_]+', '-', args["name"]).strip('-') elif args["command"] == "bench_diff" and len(args["name"]) > 0: args["name"] = " ".join(args["name"]) diff --git a/toolchain/mfc/bench.py b/toolchain/mfc/bench.py index 970664b77f..fd8a17996a 100644 --- a/toolchain/mfc/bench.py +++ b/toolchain/mfc/bench.py @@ -107,36 +107,28 @@ def diff(): """) cb_stats = {} - if ARG("file") is not None and ARG("name") is not None: - try: - with open(ARG("file"), 'r') as f: - data_json = json.load(f) - - cb_test = ARG("name") - if "entries" in data_json and cb_test in data_json["entries"]: - benchmark_runs = data_json["entries"][cb_test] - case_times = {} - for run in benchmark_runs: - if "benches" not in run: - continue - for bench in run["benches"]: - case_name = bench.get("name") - grind_value = bench.get("value") - if case_name is None or grind_value is None: - continue - if case_name not in case_times: - case_times[case_name] = [] - case_times[case_name].append(grind_value) - for case_name, values in case_times.items(): - if len(values) > 0: - avg = sum(values) / len(values) - cb_stats[case_name] = {"avg": avg, "count": len(values)} - - cons.print(f"[bold]Loaded cb data for test: [bold]{cb_test}[/bold] ({len(cb_stats)} cases)[/bold]") - else: - cons.print(f"[bold yellow]Warning[/bold yellow]: Test '[bold]{cb_test}[/bold]' not found in data file.") - except Exception as e: - cons.print(f"[bold yellow]Warning[/bold yellow]: Could not load data file: {e}") + try: + with open(ARG("file"), 'r') as f: + data_json = json.load(f) + cb_test = ARG("name") + if "entries" in data_json and cb_test in data_json["entries"]: + benchmark_runs = data_json["entries"][cb_test] + case_times = {} + for run in benchmark_runs: + for benches in run["benches"]: + case_name = benches.get("name") + grind_value = benches.get("value") + if (case_name is None or case_name not in case_times) or grind_value is None: + case_times[case_name] = [] + case_times[case_name].append(grind_value) + for case_name, values in case_times.items(): + avg = sum(values) / len(values) + cb_stats[case_name] = {"avg": avg, "count": len(values)} + cons.print(f"[bold]Loaded cb data for test: [bold]{cb_test}[/bold] ({len(cb_stats)} cases)[/bold]") + else: + cons.print(f"[bold yellow]Warning[/bold yellow]: Test '[bold]{cb_test}[/bold]' not found in data file.") + except Exception as e: + cons.print(f"[bold yellow]Warning[/bold yellow]: Could not load data file: {e}") table = rich.table.Table(show_header=True, box=rich.table.box.SIMPLE) table.add_column("[bold]Case[/bold]", justify="left") @@ -177,10 +169,9 @@ def diff(): err = 1 if slug in cb_stats: rhs_grind = rhs_summary[target.name]["grind"] - stats = cb_stats[slug] - avg = stats["avg"] - offset_pct = ((rhs_grind - avg) / avg * 100) - color = "red" if (offset_pct) > 0 else "yellow" if abs(offset_pct) == 0 else "green" + avg = cb_stats[slug]["avg"] + offset_pct = (rhs_grind - avg) / avg * 100 + color = "red" if (offset_pct) > 0 else "yellow" if (offset_pct) == 0 else "green" grind_comparison = f"[{color}]{offset_pct:+.2f}%[/{color}] (avg: {avg:.2f})" except Exception as _: @@ -194,4 +185,3 @@ def diff(): cons.raw.print(table) if err: raise MFCException("Benchmarking failed") - \ No newline at end of file From 83fd1b24cd22645a97edc07da79bbc68c22be722 Mon Sep 17 00:00:00 2001 From: Malmahrouqi3 Date: Mon, 10 Nov 2025 14:40:10 -0500 Subject: [PATCH 4/4] All Necessary Changes Implemented --- .github/workflows/cont-bench.yml | 168 +++++++++++++++++++++++++++++++ .github/workflows/docs.yml | 1 + 2 files changed, 169 insertions(+) create mode 100644 .github/workflows/cont-bench.yml diff --git a/.github/workflows/cont-bench.yml b/.github/workflows/cont-bench.yml new file mode 100644 index 0000000000..e42e5b3ced --- /dev/null +++ b/.github/workflows/cont-bench.yml @@ -0,0 +1,168 @@ +name: 'Continuous Benchmarking' + +on: + pull_request_review: + types: [submitted] + workflow_dispatch: + inputs: + tag: + description: 'tag to CB' + required: true + +permissions: + contents: write + deployments: write + pages: write + id-token: write + +jobs: + file-changes: + name: Detect File Changes + runs-on: 'ubuntu-latest' + outputs: + checkall: ${{ steps.changes.outputs.checkall }} + steps: + - name: Clone + uses: actions/checkout@v4 + + - name: Detect Changes + uses: dorny/paths-filter@v3 + id: changes + with: + filters: ".github/file-filter.yml" + + self: + name: "${{ matrix.name }} (${{ matrix.device }})" + needs: file-changes + strategy: + fail-fast: false + matrix: + include: + - cluster: phoenix + name: Georgia Tech | Phoenix (NVHPC) + group: phoenix + labels: gt + flag: p + device: cpu + interface: none + build_script: "" + - cluster: phoenix + name: Georgia Tech | Phoenix (NVHPC) + group: phoenix + labels: gt + flag: p + device: gpu + interface: acc + build_script: "" + - cluster: phoenix + name: Georgia Tech | Phoenix (NVHPC) + group: phoenix + labels: gt + flag: p + device: gpu + interface: omp + build_script: "" + - cluster: frontier + name: Oak Ridge | Frontier (CCE) + group: phoenix + labels: frontier + flag: f + device: gpu + interface: acc + build_script: "bash .github/workflows/frontier/build.sh gpu acc bench" + runs-on: + group: ${{ matrix.group }} + labels: ${{ matrix.labels }} + timeout-minutes: 1400 + env: + ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + steps: + - name: Clone - PR + uses: actions/checkout@v4 + with: + path: pr + + - name: Clone - Master + uses: actions/checkout@v4 + with: + repository: MFlowCode/MFC + ref: master + path: master + + - name: Setup & Build + if: matrix.build_script != '' + run: | + (cd pr && ${{ matrix.build_script }}) & + (cd master && ${{ matrix.build_script }}) & + wait %1 && wait %2 + + - name: Bench (Master v. PR) + run: | + (cd pr && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }} ${{ matrix.interface }}) & + (cd master && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }} ${{ matrix.interface }}) & + wait %1 && wait %2 + + - name: Generate & Post Comment + run: | + git clone "${{ secrets.DOC_PUSH_URL }}" ../www + cp ../www/documentation/data.js ./pr/data.js + (cd pr && . ./mfc.sh load -c ${{ matrix.flag }} -m g) + (cd pr && ./mfc.sh bench_diff ../master/bench-${{ matrix.device }}-${{ matrix.interface }}.yaml ../pr/bench-${{ matrix.device }}-${{ matrix.interface }}.yaml -f data.js -n "${{ matrix.name }} (${{ matrix.device }}/${{ matrix.interface }})") + + - name: Prep Benchmark Results + run: | + # should fill in the template in pr/cont-bench-template.json + + - name: Post Benchmark Results + uses: benchmark-action/github-action-benchmark@v1 + with: + name: "${{ matrix.name }} (${{ matrix.device }}/${{ matrix.interface }})" + tool: 'googlecpp' + output-file-path: pr/report.json + github-token: ${{ secrets.TOKEN }} + auto-push: true + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true + gh-repository: 'github.com/MFlowCode/MFlowCode.github.io' + gh-pages-branch: 'main' + benchmark-data-dir-path: 'documentation' + + - name: Polishing Results + run: | + TAG="${{ github.event.inputs.tag || github.ref_name }}" + echo "tag=$TAG" >> $GITHUB_OUTPUT + echo "TAG=$TAG" >> $GITHUB_ENV + + set +e + git ls-remote "${{ secrets.DOC_PUSH_URL }}" -q + if [ "$?" -ne "0" ]; then exit 0; fi + set -e + git config --global user.name 'MFC Action' + git config --global user.email '<>' + git clone "${{ secrets.DOC_PUSH_URL }}" ../www + sed -i 's/"unit": "s\/iter"/"unit": "s"/g; s/cpu: /runtime: /g; s/iterations: /time steps: /g' ../www/documentation/data.js + sed -i "s/${GITHUB_SHA::7}/$TAG/g" ../www/documentation/data.js + + git -C ../www add -A + git -C ../www commit -m "Docs @ ${GITHUB_SHA::7}" || true + git -C ../www push + + - name: Print Logs + if: always() + run: | + cat pr/bench-${{ matrix.device }}-${{ matrix.interface }}.* 2>/dev/null || true + cat master/bench-${{ matrix.device }}-${{ matrix.interface }}.* 2>/dev/null || true + + # All other runners (non-Phoenix) just run without special env + - name: Archive Logs (Frontier) + if: always() && matrix.cluster != 'phoenix' + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.cluster }}-${{ matrix.device }}-${{ matrix.interface }} + path: | + pr/bench-${{ matrix.device }}-${{ matrix.interface }}.* + pr/build/benchmarks/* + master/bench-${{ matrix.device }}-${{ matrix.interface }}.* + master/build/benchmarks/* diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d161d80342..856ee2c19f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -69,6 +69,7 @@ jobs: rm -rf ../www/* mv build/install/docs/mfc/* ../www/ git -C ../www add -A + git -C ../www restore documentation/data.js git -C ../www commit -m "Docs @ ${GITHUB_SHA::7}" || true git -C ../www push