diff --git a/src/discord-cluster-manager/cogs/submit_cog.py b/src/discord-cluster-manager/cogs/submit_cog.py index 9da5a82e..0a2648f2 100644 --- a/src/discord-cluster-manager/cogs/submit_cog.py +++ b/src/discord-cluster-manager/cogs/submit_cog.py @@ -9,7 +9,7 @@ from consts import SubmissionMode from discord import app_commands from discord.ext import commands -from report import MultiProgressReporter, RunProgressReporter, generate_report, private_run_report +from report import MultiProgressReporter, RunProgressReporter, generate_report, make_short_report from run_eval import FullResult from task import LeaderboardTask from utils import build_task_config, send_discord_message, setup_logging, with_error_handling @@ -150,12 +150,11 @@ async def _handle_submission( else: await reporter.update_title(reporter.title + " ✅ success") - if mode == SubmissionMode.PRIVATE: - await reporter.push(private_run_report(result.runs)) - else: - if mode == SubmissionMode.LEADERBOARD: - await reporter.push(private_run_report(result.runs)) - + await reporter.push(make_short_report( + result.runs, + full=mode in [SubmissionMode.PRIVATE, SubmissionMode.LEADERBOARD]) + ) + if mode != SubmissionMode.PRIVATE: try: await generate_report(thread, result.runs) await reporter.push(f"See results at {thread.jump_url}") diff --git a/src/discord-cluster-manager/report.py b/src/discord-cluster-manager/report.py index 14886120..9fc7920b 100644 --- a/src/discord-cluster-manager/report.py +++ b/src/discord-cluster-manager/report.py @@ -26,7 +26,8 @@ async def _send_split_log(thread: discord.Thread, partial_message: str, header: if len(partial_message) + len(line) < 1900: partial_message += line + "\n" else: - chunks.append(partial_message) + if partial_message != "": + chunks.append(partial_message) partial_message = line if partial_message != "": @@ -111,7 +112,7 @@ async def _generate_test_report(thread: discord.Thread, run: RunResult): return -def private_run_report(runs: dict[str, EvalResult]) -> list[str]: # noqa: C901 +def make_short_report(runs: dict[str, EvalResult], full=True) -> list[str]: # noqa: C901 """ Creates a minimalistic report for `runs`, returned as a list of status strings @@ -127,31 +128,42 @@ def private_run_report(runs: dict[str, EvalResult]) -> list[str]: # noqa: C901 if any_compile: result.append("✅ Compilation successful") - if "test" not in runs or not runs["test"].run.success: - result.append("❌ Running tests failed") - return result - elif not runs["test"].run.passed: - result.append("❌ Testing failed") - return result - else: - result.append("✅ Testing successful") - - if "benchmark" not in runs or not runs["benchmark"].run.success: - result.append("❌ Running benchmarks failed") - return result - elif not runs["benchmark"].run.passed: - result.append("❌ Benchmarking failed") - return result - else: - result.append("✅ Benchmarking successful") - - if "leaderboard" not in runs or not runs["leaderboard"].run.success: - result.append("❌ Running leaderboard failed") - elif not runs["leaderboard"].run.passed: - result.append("❌ Leaderboard run failed") - else: - result.append("✅ Leaderboard run successful") + if "test" in runs: + test_run = runs["test"].run + if not test_run.success: + result.append("❌ Running tests failed") + return result + elif not test_run.passed: + result.append("❌ Testing failed") + return result + else: + result.append("✅ Testing successful") + elif full: + result.append("❌ Tests missing") + if "benchmark" in runs: + bench_run = runs["benchmark"].run + if not bench_run.success: + result.append("❌ Running benchmarks failed") + return result + elif not bench_run.passed: + result.append("❌ Benchmarking failed") + return result + else: + result.append("✅ Benchmarking successful") + elif full: + result.append("❌ Benchmarks missing") + + if "leaderboard" in runs: + lb_run = runs["leaderboard"].run + if not lb_run.success: + result.append("❌ Running leaderboard failed") + elif not lb_run.passed: + result.append("❌ Leaderboard run failed") + else: + result.append("✅ Leaderboard run successful") + elif full: + result.append("❌ Leaderboard missing") return result @@ -258,6 +270,24 @@ async def generate_report(thread: discord.Thread, runs: dict[str, EvalResult]): make_benchmark_log(bench_run), ) + if "leaderboard" in runs: + bench_run = runs["leaderboard"] + if bench_run.compilation is not None and not bench_run.compilation.success: + await _generate_compile_report(thread, bench_run.compilation) + return + + bench_run = bench_run.run + if not bench_run.success: + await _generate_crash_report(thread, bench_run) + return + + message = await _send_split_log( + thread, + message, + "Ranked Benchmark", + make_benchmark_log(bench_run), + ) + if "script" in runs: run = runs["script"] if run.compilation is not None and not run.compilation.success: