Skip to content

[Small bug fix] benchmark stats by language - pull the correct pass rate for each language #32

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dwash96 merged 4 commits into from
Oct 5, 2025
Merged

[Small bug fix] benchmark stats by language - pull the correct pass rate for each language #32

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d25c73c
fixed benchmark report per language stats bug
itsmeknt Sep 30, 2025
7a63e61
remove space
itsmeknt Sep 30, 2025
8430ae2
changed to printing of pass_rate_i to pass_rate_i+1 for per language …
itsmeknt Oct 2, 2025
3eac8bc
fix whitespace
itsmeknt Oct 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions benchmark/benchmark.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -548,9 +548,9 @@ def add(attr_name, increment, global_stats, lang_stats):
lang_stats.avg_duration_per_test = 0
lang_stats.cost = 0
for i in range(tries):
setattr(lang_stats, f"pass_rate_{i}", 0)
setattr(lang_stats, f"pass_rate_{i + 1}", 0)
for i in range(tries):
setattr(lang_stats, f"pass_num_{i}", 0)
setattr(lang_stats, f"pass_num_{i + 1}", 0)
lang_stats.error_outputs = 0
lang_stats.user_asks = 0
lang_stats.test_timeouts = 0
Expand Down Expand Up @@ -636,7 +636,7 @@ def show(stat, red="red"):
for i in range(tries):
pass_rate = 100 * passed_tests[i] / res.completed_tests
percents[i] = pass_rate
# console.print(f"{pass_rate:.1f}% correct after try {i+1}")
# console.print(f"{pass_rate:.1f}% correct after try {i + 1}")
setattr(res, f"pass_rate_{i + 1}", f"{pass_rate:.1f}")
setattr(res, f"pass_num_{i + 1}", passed_tests[i])

Expand Down Expand Up @@ -705,17 +705,17 @@ def show(stat, red="red"):

if verbose and len(lang_to_stats) > 0:

def format_lang_stats(lang_stats):
def format_lang_stats(lang, lang_stats):
# First, postprocess attributes for easier printing
if lang_stats.completed_tests > 0:
lang_stats.avg_duration_per_test = lang_stats.duration / float(
lang_stats.completed_tests
)
for i in range(tries):
num_passed = lang_to_passed_tests[lang][i]
setattr(lang_stats, f"pass_num_{i}", num_passed)
setattr(lang_stats, f"pass_num_{i + 1}", num_passed)
pass_rate = 100 * num_passed / float(lang_stats.completed_tests)
setattr(lang_stats, f"pass_rate_{i}", pass_rate)
setattr(lang_stats, f"pass_rate_{i + 1}", pass_rate)

# Then format attributes into ready-to-print strings
for attr in lang_stats.__dict__:
Expand All @@ -742,7 +742,7 @@ def compute_lang_to_col_widths(lang_to_stats):
print("======== Stats by language ========")
print()

[format_lang_stats(lang_stats) for lang_stats in lang_to_stats.values()]
[format_lang_stats(lang, lang_stats) for lang, lang_stats in lang_to_stats.items()]
lang_to_col_widths = compute_lang_to_col_widths(lang_to_stats)

any_stats = list(lang_to_stats.values())[0]
Expand Down
Loading