Add nightly application validation for top kernels

README.md

@@ -34,6 +34,8 @@ yoyo apply src/migrations -d postgresql://user:password@localhost/clusterdev
 See [docs/database.md](docs/database.md) for migration patterns and creating new migrations.
 
 For production incident triage, see [docs/production-debugging.md](docs/production-debugging.md).
+For nightly end-to-end kernel checks, see
+[docs/application-validation.md](docs/application-validation.md).
 
 ### Environment Variables
 
@@ -52,6 +54,7 @@ PROBLEM_DEV_DIR=examples
 DISABLE_SSL=1           # Set for local development
 GITHUB_TOKEN_BACKUP=    # Fallback token for rate limiting
 ADMIN_TOKEN=            # Token for admin API endpoints
+APPLICATION_VALIDATION_ENABLED=true
 
 # Discord bot (only needed if testing Discord integration)
 # See docs/discord.md for setup instructions

docs/application-validation.md

@@ -0,0 +1,66 @@
+# Application validation
+
+KernelBot can run a small, problem-owned workload against the current top 10
+submissions. A problem opts in with two fields in `reference-kernels`:
+
+```yaml
+validation:
+  version: cholesky-natural-gradient-v1
+  script: validation.py
+```
+
+The version invalidates old results when the workload changes. KernelBot loads
+the script beside the problem, adds it to the problem's normal source files,
+and runs it on B200 through the existing Modal app.
+
+The script receives the version in `KERNELBOT_VALIDATION_CONFIG` and prints one
+JSON result:
+
+```json
+{
+  "passed_shapes": 8,
+  "total_shapes": 8,
+  "fully_validated": true,
+  "geomean_sync_wall_speedup": 1.42,
+  "results": []
+}
+```
+
+## Schedule
+
+Every day at 22:00 `America/Los_Angeles`, one KernelBot replica claims each
+`(leaderboard, GPU, contract version, local date)` sweep. It snapshots the
+current best submission from each of the top 10 users and runs at most two
+Modal jobs concurrently. The database claim prevents duplicate sweeps.
+
+Set `APPLICATION_VALIDATION_ENABLED=false` to disable the scheduler. A failed
+job is recorded as `VALIDATION ERROR`; a completed job is stored as `X/Y
+VALIDATED`.
+
+## Local debug
+
+Modal already reads `MODAL_ENVIRONMENT`, so KernelBot does not need separate
+environment plumbing:
+
+```bash
+modal environment create cholesky-validation-debug
+modal deploy --env cholesky-validation-debug src/runners/modal_runner_archs.py
+
+MODAL_ENVIRONMENT=cholesky-validation-debug \
+APPLICATION_VALIDATION_ENABLED=false \
+python src/kernelbot/main.py --api-only --debug
+```
+
+Run one sweep synchronously:
+
+```bash
+kernelbot-admin --api-url http://localhost:8000 \
+  validate-top10 cholesky B200 --wait
+```
+
+Omit `--wait` to enqueue the sweep and return immediately. The command reads
+`ADMIN_TOKEN` and, unless `--api-url` is supplied,
+`DISCORD_CLUSTER_MANAGER_API_BASE_URL` from the environment.
+
+Roll out KernelBot's migration and runner first, then the reference-kernels
+contract, then the Kernelboard badge.

pyproject.toml

@@ -37,6 +37,9 @@ dev = [
     "pytest-asyncio"
 ]
 
+[project.scripts]
+kernelbot-admin = "kernelbot.admin_cli:main"
+
 [tool.setuptools.packages.find]
 where = ["src"]
 

src/kernelbot/admin_cli.py

@@ -0,0 +1,62 @@
+import argparse
+import json
+import os
+from urllib.parse import quote
+
+import requests
+
+
+def _validate_top10(args: argparse.Namespace) -> int:
+    api_url = args.api_url or os.getenv("DISCORD_CLUSTER_MANAGER_API_BASE_URL")
+    token = os.getenv("ADMIN_TOKEN")
+    if not api_url:
+        raise SystemExit(
+            "Set DISCORD_CLUSTER_MANAGER_API_BASE_URL or pass --api-url."
+        )
+    if not token:
+        raise SystemExit("Set ADMIN_TOKEN.")
+
+    path = "/admin/application-validations/{}/{}".format(
+        quote(args.leaderboard, safe=""),
+        quote(args.gpu, safe=""),
+    )
+    response = requests.post(
+        f"{api_url.rstrip('/')}{path}",
+        headers={"Authorization": f"Bearer {token}"},
+        params={"wait": str(args.wait).lower()},
+        timeout=None if args.wait else 30,
+    )
+    response.raise_for_status()
+    print(json.dumps(response.json(), indent=2, sort_keys=True))
+    return 0
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="kernelbot-admin")
+    parser.add_argument(
+        "--api-url",
+        help="Kernelbot API URL (defaults to DISCORD_CLUSTER_MANAGER_API_BASE_URL)",
+    )
+    commands = parser.add_subparsers(required=True)
+    validate = commands.add_parser(
+        "validate-top10",
+        help="Run application validation for a leaderboard's current top 10",
+    )
+    validate.add_argument("leaderboard")
+    validate.add_argument("gpu")
+    validate.add_argument(
+        "--wait",
+        action="store_true",
+        help="Wait for all validation jobs and print their results",
+    )
+    validate.set_defaults(run=_validate_top10)
+    return parser
+
+
+def main() -> int:
+    args = _parser().parse_args()
+    return args.run(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/kernelbot/api/main.py

@@ -12,6 +12,7 @@
 from fastapi.responses import JSONResponse, StreamingResponse
 
 from kernelbot.env import env
+from libkernelbot.application_validation import ApplicationValidationService
 from libkernelbot.backend import KernelBackend
 from libkernelbot.background_submission_manager import BackgroundSubmissionManager
 from libkernelbot.consts import SubmissionMode
@@ -54,6 +55,7 @@ def json_serializer(obj):
 
 backend_instance: KernelBackend = None
 background_submission_manager: BackgroundSubmissionManager = None
+application_validation_service: ApplicationValidationService = None
 
 _last_action = time.time()
 _submit_limiter = asyncio.Semaphore(3)
@@ -89,6 +91,12 @@ def init_background_submission_manager(_manager: BackgroundSubmissionManager):
     return background_submission_manager
 
 
+def init_application_validation_service(_service: ApplicationValidationService):
+    global application_validation_service
+    application_validation_service = _service
+    return application_validation_service
+
+
 @app.exception_handler(KernelBotError)
 async def kernel_bot_error_handler(req: Request, exc: KernelBotError):
     return JSONResponse(status_code=exc.http_code, content={"message": str(exc)})
@@ -477,6 +485,35 @@ async def admin_unban_user(
     return {"status": "ok", "user_id": user_id, "banned": False}
 
 
+@app.post("/admin/application-validations/{leaderboard_name}/{gpu_type}")
+async def admin_run_application_validation(
+    leaderboard_name: str,
+    gpu_type: str,
+    _: Annotated[None, Depends(require_admin)],
+    wait: bool = Query(False),
+) -> dict:
+    if application_validation_service is None:
+        raise HTTPException(
+            status_code=503,
+            detail="Application validation service is not initialized",
+        )
+    if wait:
+        return await application_validation_service.run_sweep(
+            leaderboard_name,
+            gpu_type,
+            scheduled_for=None,
+        )
+    application_validation_service.enqueue_manual_sweep(
+        leaderboard_name,
+        gpu_type,
+    )
+    return {
+        "status": "accepted",
+        "leaderboard": leaderboard_name,
+        "gpu_type": gpu_type,
+    }
+
+
 @app.post("/{leaderboard_name}/{gpu_type}/{submission_mode}")
 async def run_submission(  # noqa: C901
     leaderboard_name: str,

src/kernelbot/env.py

@@ -42,6 +42,10 @@
 # PostgreSQL-specific constants
 env.DATABASE_URL = os.getenv("DATABASE_URL")
 env.DISABLE_SSL = os.getenv("DISABLE_SSL")
+env.APPLICATION_VALIDATION_ENABLED = os.getenv(
+    "APPLICATION_VALIDATION_ENABLED",
+    "true",
+).lower() in {"1", "true", "yes"}
 
 
 def init_environment(skip_discord: bool = False):

src/kernelbot/main.py

@@ -4,7 +4,12 @@
 
 import discord
 import uvicorn
-from api.main import app, init_api, init_background_submission_manager
+from api.main import (
+    app,
+    init_api,
+    init_application_validation_service,
+    init_background_submission_manager,
+)
 from cogs.admin_cog import AdminCog
 from cogs.leaderboard_cog import LeaderboardCog
 from cogs.misc_cog import BotManagerCog
@@ -15,6 +20,7 @@
 from env import env, init_environment
 
 from libkernelbot import consts
+from libkernelbot.application_validation import ApplicationValidationService
 from libkernelbot.backend import KernelBackend
 from libkernelbot.background_submission_manager import BackgroundSubmissionManager
 from libkernelbot.launchers import GitHubLauncher, ModalLauncher
@@ -50,11 +56,19 @@ async def run_api_server(backend: KernelBackend):
     init_api(backend)
     manager = init_background_submission_manager(BackgroundSubmissionManager(backend))
     await manager.start()
+    validation_service = init_application_validation_service(
+        ApplicationValidationService(
+            backend,
+            enabled=env.APPLICATION_VALIDATION_ENABLED,
+        )
+    )
+    await validation_service.start()
 
     server = create_uvicorn_server()
     try:
         await server.serve()
     finally:
+        await validation_service.stop()
         await manager.stop()
 
 
@@ -241,9 +255,9 @@ async def start_bot(self, token: str):
             raise e
 
 
-async def start_api_only():
+async def start_api_only(debug_mode: bool = False):
     """Start only the FastAPI server without Discord bot."""
-    backend = create_backend(debug_mode=False)
+    backend = create_backend(debug_mode=debug_mode)
     await run_api_server(backend)
 
 
@@ -258,6 +272,13 @@ async def start_bot_and_api(debug_mode: bool):
     init_api(bot_instance.backend)
     manager = init_background_submission_manager(BackgroundSubmissionManager(bot_instance.backend))
     await manager.start()
+    validation_service = init_application_validation_service(
+        ApplicationValidationService(
+            bot_instance.backend,
+            enabled=env.APPLICATION_VALIDATION_ENABLED,
+        )
+    )
+    await validation_service.start()
 
     server = create_uvicorn_server()
     try:
@@ -266,6 +287,7 @@ async def start_bot_and_api(debug_mode: bool):
             server.serve(),
         )
     finally:
+        await validation_service.stop()
         await manager.stop()
 
 def on_unhandled_exception(loop, context):
@@ -283,9 +305,9 @@ def main():
 
     if args.api_only:
         logger.info("Starting API server only (no Discord bot)...")
-        with asyncio.Runner() as runner:
+        with asyncio.Runner(debug=args.debug) as runner:
             runner.get_loop().set_exception_handler(on_unhandled_exception)
-            runner.run(start_api_only())
+            runner.run(start_api_only(debug_mode=args.debug))
     else:
         logger.info("Starting kernelbot and API server...")
         with asyncio.Runner(debug=args.debug) as runner: