From 3fb40f5658c53d89915852337c7b424711c8f321 Mon Sep 17 00:00:00 2001
From: Oguz Ulgen <oulgen@meta.com>
Date: Mon, 6 Oct 2025 11:14:28 -0700
Subject: [PATCH] [Benchmark] Keep going when a single benchmark fails

stack-info: PR: https://github.com/pytorch/helion/pull/820, branch: oulgen/stack/117
---
 .github/workflows/benchmark.yml | 4 ++--
 benchmarks/run.py               | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 1df1e47db..1a1d2fa2a 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -169,7 +169,7 @@ jobs:
                 --only $IMPLS \
                 --only-match-mode prefix-with-baseline \
                 --baseline $BASELINE \
-                --exit-on-exception
+                --keep-going
 
             # Relax the GPU
             sleep 2m
@@ -185,7 +185,7 @@ jobs:
                 --baseline $BASELINE \
                 --output "$TEST_REPORTS_DIR/helionbench.json" \
                 --append-to-output \
-                --exit-on-exception
+                --keep-going
 
             echo "✅ Completed benchmark for kernel: $kernel"
           done
diff --git a/benchmarks/run.py b/benchmarks/run.py
index b2db48034..4195b16e8 100644
--- a/benchmarks/run.py
+++ b/benchmarks/run.py
@@ -989,6 +989,9 @@ def process_result(
                 if name not in KERNEL_METRIC_MAPPINGS[kernel_name]:
                     logger.info(f"ignoring {name}")
                 else:
+                    if item == "":
+                        # if benchmark failed, tritonbench emits empty string
+                        item = 0.0
                     metrics[KERNEL_METRIC_MAPPINGS[kernel_name][name]].append(
                         float(item)
                     )