From 483e0213e89e0e9d605f0c9121c811f2556293d2 Mon Sep 17 00:00:00 2001 From: ngc92 <7938269+ngc92@users.noreply.github.com> Date: Sat, 5 Apr 2025 00:10:08 +0200 Subject: [PATCH 1/3] include leaderboard results in report --- src/discord-cluster-manager/report.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/discord-cluster-manager/report.py b/src/discord-cluster-manager/report.py index 14886120..7c23aead 100644 --- a/src/discord-cluster-manager/report.py +++ b/src/discord-cluster-manager/report.py @@ -26,7 +26,8 @@ async def _send_split_log(thread: discord.Thread, partial_message: str, header: if len(partial_message) + len(line) < 1900: partial_message += line + "\n" else: - chunks.append(partial_message) + if partial_message != "": + chunks.append(partial_message) partial_message = line if partial_message != "": @@ -258,6 +259,24 @@ async def generate_report(thread: discord.Thread, runs: dict[str, EvalResult]): make_benchmark_log(bench_run), ) + if "leaderboard" in runs: + bench_run = runs["leaderboard"] + if bench_run.compilation is not None and not bench_run.compilation.success: + await _generate_compile_report(thread, bench_run.compilation) + return + + bench_run = bench_run.run + if not bench_run.success: + await _generate_crash_report(thread, bench_run) + return + + message = await _send_split_log( + thread, + message, + "Ranked Benchmark", + make_benchmark_log(bench_run), + ) + if "script" in runs: run = runs["script"] if run.compilation is not None and not run.compilation.success: From ca23fa4d5643f49580f52668301c66288f0c4564 Mon Sep 17 00:00:00 2001 From: ngc92 <7938269+ngc92@users.noreply.github.com> Date: Sat, 5 Apr 2025 01:43:17 +0200 Subject: [PATCH 2/3] better summary message for submit test/benchmark --- .../cogs/submit_cog.py | 13 ++-- src/discord-cluster-manager/report.py | 59 +++++++++++-------- 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/src/discord-cluster-manager/cogs/submit_cog.py b/src/discord-cluster-manager/cogs/submit_cog.py index 9da5a82e..0a2648f2 100644 --- a/src/discord-cluster-manager/cogs/submit_cog.py +++ b/src/discord-cluster-manager/cogs/submit_cog.py @@ -9,7 +9,7 @@ from consts import SubmissionMode from discord import app_commands from discord.ext import commands -from report import MultiProgressReporter, RunProgressReporter, generate_report, private_run_report +from report import MultiProgressReporter, RunProgressReporter, generate_report, make_short_report from run_eval import FullResult from task import LeaderboardTask from utils import build_task_config, send_discord_message, setup_logging, with_error_handling @@ -150,12 +150,11 @@ async def _handle_submission( else: await reporter.update_title(reporter.title + " ✅ success") - if mode == SubmissionMode.PRIVATE: - await reporter.push(private_run_report(result.runs)) - else: - if mode == SubmissionMode.LEADERBOARD: - await reporter.push(private_run_report(result.runs)) - + await reporter.push(make_short_report( + result.runs, + full=mode in [SubmissionMode.PRIVATE, SubmissionMode.LEADERBOARD]) + ) + if mode != SubmissionMode.PRIVATE: try: await generate_report(thread, result.runs) await reporter.push(f"See results at {thread.jump_url}") diff --git a/src/discord-cluster-manager/report.py b/src/discord-cluster-manager/report.py index 7c23aead..02786f0a 100644 --- a/src/discord-cluster-manager/report.py +++ b/src/discord-cluster-manager/report.py @@ -112,7 +112,7 @@ async def _generate_test_report(thread: discord.Thread, run: RunResult): return -def private_run_report(runs: dict[str, EvalResult]) -> list[str]: # noqa: C901 +def make_short_report(runs: dict[str, EvalResult], full=True) -> list[str]: # noqa: C901 """ Creates a minimalistic report for `runs`, returned as a list of status strings @@ -128,31 +128,42 @@ def private_run_report(runs: dict[str, EvalResult]) -> list[str]: # noqa: C901 if any_compile: result.append("✅ Compilation successful") - if "test" not in runs or not runs["test"].run.success: - result.append("❌ Running tests failed") - return result - elif not runs["test"].run.passed: - result.append("❌ Testing failed") - return result - else: - result.append("✅ Testing successful") - - if "benchmark" not in runs or not runs["benchmark"].run.success: - result.append("❌ Running benchmarks failed") - return result - elif not runs["benchmark"].run.passed: - result.append("❌ Benchmarking failed") - return result - else: - result.append("✅ Benchmarking successful") + if "test" in runs: + test_run = runs["test"].run + if not test_run.success: + result.append("❌ Running tests failed") + return result + elif not test_run.passed: + result.append("❌ Testing failed") + return result + else: + result.append("✅ Testing successful") + elif full: + result.append("❌ Tests missing") - if "leaderboard" not in runs or not runs["leaderboard"].run.success: - result.append("❌ Running leaderboard failed") - elif not runs["leaderboard"].run.passed: - result.append("❌ Leaderboard run failed") - else: - result.append("✅ Leaderboard run successful") + if "benchmark" in runs: + bench_run = runs["benchmark"].run + if bench_run.success: + result.append("❌ Running benchmarks failed") + return result + elif not bench_run.passed: + result.append("❌ Benchmarking failed") + return result + else: + result.append("✅ Benchmarking successful") + elif full: + result.append("❌ Benchmarks missing") + if "leaderboard" in runs: + lb_run = runs["leaderboard"].run + if not lb_run.success: + result.append("❌ Running leaderboard failed") + elif not lb_run.passed: + result.append("❌ Leaderboard run failed") + else: + result.append("✅ Leaderboard run successful") + elif full: + result.append("❌ Leaderboard missing") return result From 5eccd2017695f80901b6c6d3b287119b1b062d80 Mon Sep 17 00:00:00 2001 From: ngc92 <7938269+ngc92@users.noreply.github.com> Date: Wed, 9 Apr 2025 09:32:21 +0200 Subject: [PATCH 3/3] Condition fixed by copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/discord-cluster-manager/report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/discord-cluster-manager/report.py b/src/discord-cluster-manager/report.py index 02786f0a..9fc7920b 100644 --- a/src/discord-cluster-manager/report.py +++ b/src/discord-cluster-manager/report.py @@ -143,7 +143,7 @@ def make_short_report(runs: dict[str, EvalResult], full=True) -> list[str]: # n if "benchmark" in runs: bench_run = runs["benchmark"].run - if bench_run.success: + if not bench_run.success: result.append("❌ Running benchmarks failed") return result elif not bench_run.passed: