diff --git a/scripts/native_eval/aggregate.py b/scripts/native_eval/aggregate.py index d316090..0ebf428 100644 --- a/scripts/native_eval/aggregate.py +++ b/scripts/native_eval/aggregate.py @@ -91,6 +91,9 @@ "trial_name", "result_path", "classification", + "execution_outcome", + "execution_exit_code", + "execution_reason", "reward", "exception_type", "exception_message", @@ -142,6 +145,8 @@ "infra_dominated", "harness_wide_failure", "harness_wide_failure_signature", + "run_accepted", + "run_acceptance_reason", "canonical_model_identity", "trajectory_complete", "parity_validated", @@ -278,8 +283,18 @@ def _is_infra(exception_type: str, exception_message: str) -> bool: return any(pattern in combined for pattern in INFRA_MESSAGE_PATTERNS) -def _classify(result: dict[str, Any], reward: int | float | None) -> str: +def _classify( + result: dict[str, Any], + reward: int | float | None, + execution_kind: str, +) -> str: exception_type, exception_message, _ = _exception_fields(result) + if execution_kind in {"harness_error", "infra_error"}: + return "infra" + if execution_kind == "verifier_error": + return "verifier_missing_reward" + if execution_kind == "agent_error": + return "agent_exit" if exception_type in MISSING_REWARD_EXCEPTION_TYPES: return "verifier_missing_reward" if exception_type: @@ -295,6 +310,40 @@ def _classify(result: dict[str, Any], reward: int | float | None) -> str: return "pass" +def _execution_fields( + result: dict[str, Any], + *, + harness: str, +) -> tuple[str, int | None, str]: + outcome = result.get("execution_outcome") + if isinstance(outcome, dict): + kind = str(outcome.get("kind") or "") + if kind and kind != "pending": + exit_code = _number(outcome.get("exit_code")) + return ( + kind, + int(exit_code) if exit_code is not None else None, + str(outcome.get("reason") or ""), + ) + + exception_type, exception_message, _ = _exception_fields(result) + exit_match = re.search(r"\bcode\s+(\d+)\b", exception_message) + exit_code = int(exit_match.group(1)) if exit_match else None + if ( + harness == "openclaw" + and exception_type == "NonZeroAgentExitCodeError" + and exit_code in {70, 71} + ): + return "harness_error", exit_code, exception_message + if exception_type in MISSING_REWARD_EXCEPTION_TYPES: + return "verifier_error", exit_code, exception_message + if exception_type and _is_infra(exception_type, exception_message): + return "infra_error", exit_code, exception_message + if exception_type: + return "agent_error", exit_code, exception_message + return "clean", None, "" + + def _scorecard_infra_error(trial_dir: Path) -> tuple[str, str] | None: scorecard_path = trial_dir / "verifier" / "scorecard.json" if not scorecard_path.is_file(): @@ -525,6 +574,7 @@ def _normalize_result( run_label: str, pair_label: str, repetition: int | None, + harness: str, ) -> dict[str, Any]: task_name, task_path, trial_name = _task_identity(result, result_path.parent) reward = _reward(result) @@ -532,6 +582,13 @@ def _normalize_result( scorecard_infra = _scorecard_infra_error(result_path.parent) if scorecard_infra is not None: exception_type, exception_message = scorecard_infra + execution_kind, execution_exit_code, execution_reason = _execution_fields( + result, + harness=harness, + ) + if scorecard_infra is not None: + execution_kind = "infra_error" + execution_reason = exception_message agent_result = result.get("agent_result") agent_result = agent_result if isinstance(agent_result, dict) else {} row: dict[str, Any] = { @@ -543,8 +600,13 @@ def _normalize_result( "trial_name": trial_name, "result_path": str(result_path), "classification": ( - "infra" if scorecard_infra is not None else _classify(result, reward) + "infra" + if scorecard_infra is not None + else _classify(result, reward, execution_kind) ), + "execution_outcome": execution_kind, + "execution_exit_code": execution_exit_code, + "execution_reason": execution_reason, "reward": reward, "exception_type": exception_type, "exception_message": exception_message, @@ -654,6 +716,7 @@ def _load_run( run_label = str(manifest.get("run_label") or run_label) pair_label = _pair_label(manifest, run_label) repetition = _repetition(manifest, run_label) + harness = str(manifest.get("harness") or "") rows: list[dict[str, Any]] = [] trial_dirs = sorted(path for path in run_dir.iterdir() if path.is_dir()) @@ -699,6 +762,7 @@ def _load_run( run_label=run_label, pair_label=pair_label, repetition=repetition, + harness=harness, ) ) @@ -840,6 +904,10 @@ def _summarize_run( classifications = [str(row["classification"]) for row in scored_rows] rewards = [_number(row.get("reward")) for row in scored_rows] all_classifications = [str(row["classification"]) for row in rows] + execution_kinds = [ + str(row.get("execution_outcome") or "") + for row in scored_rows + ] result_file_count = sum(bool(row.get("_has_result_file")) for row in rows) valid_result_count = sum(bool(row.get("_valid_result")) for row in rows) completed_result_count = sum(bool(row.get("_completed_result")) for row in rows) @@ -874,6 +942,25 @@ def _summarize_run( rows, expected_count, ) + acceptance = manifest.get("execution_acceptance") + acceptance = acceptance if isinstance(acceptance, dict) else {} + accepted_value = acceptance.get("accepted") + if isinstance(accepted_value, bool): + run_accepted = accepted_value + else: + invalid_execution_kinds = { + "harness_error", + "infra_error", + "verifier_error", + } + run_accepted = not ( + expected_count > 0 + and len(execution_kinds) == expected_count + and all(kind in invalid_execution_kinds for kind in execution_kinds) + ) + run_acceptance_reason = str(acceptance.get("reason") or "") + if not run_accepted and not run_acceptance_reason: + run_acceptance_reason = "all_trials_harness_or_infrastructure_errors" native_run = ( manifest.get("runner") == "shellbench-native" or any(row.get("source") == "shellbench-native" for row in rows) @@ -917,6 +1004,7 @@ def _summarize_run( ) = _parity_validation(manifest, native_run=native_run) eligible = ( not incomplete + and run_accepted and not infra_dominated and not harness_wide_failure and canonical_model_identity @@ -926,6 +1014,8 @@ def _summarize_run( exclusion_reason = "incomplete" elif infra_dominated: exclusion_reason = "infra_dominated" + elif not run_accepted: + exclusion_reason = "run_rejected" elif harness_wide_failure: exclusion_reason = "harness_wide_failure" elif not canonical_model_identity: @@ -962,6 +1052,8 @@ def _summarize_run( "infra_dominated": infra_dominated, "harness_wide_failure": harness_wide_failure, "harness_wide_failure_signature": harness_wide_failure_signature, + "run_accepted": run_accepted, + "run_acceptance_reason": run_acceptance_reason, "canonical_model_identity": canonical_model_identity, "trajectory_complete": trajectory_complete, "parity_validated": parity_validated, @@ -1184,9 +1276,9 @@ def _write_leaderboard(path: Path, pairs: Sequence[dict[str, Any]]) -> None: lines = [ "# Cleaned Native ShellBench Leaderboard", "", - "Incomplete, infra-dominated, harness-wide failure, identity-invalid, and " - "trajectory-incomplete repetitions are excluded. Harbor parity validation is " - "reported separately and is not an eligibility gate.", + "Incomplete, rejected, infra-dominated, harness-wide failure, identity-invalid, " + "and trajectory-incomplete repetitions are excluded. Harbor parity validation " + "is reported separately and is not an eligibility gate.", "", "| Rank | Pair | Repetitions | Parity validated | Mean | Stdev | Min | Max | Mean exact passes | Pass rate | Clean complete |", "| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index 433d775..a2dac75 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -1246,16 +1246,6 @@ def _archived_exit_status(self, run_label: str) -> int | None: raise FleetError(f"conflicting archived exit_status values for {run_label}") return candidates[0] if candidates else None - def _checkpoint_log_has_final(self, run_label: str) -> bool: - log_path = self.config.local_root / "logs" / f"{run_label}.checkpoints.log" - if not log_path.is_file(): - return False - for line in log_path.read_text(encoding="utf-8").splitlines(): - fields = line.split("\t", 4) - if len(fields) >= 2 and fields[1] == "final": - return True - return False - def _finish_exported(self, entry: dict[str, Any], run: RunSpec) -> bool: verified, result_count, artifacts = self._verify_final(run.run_label) if not verified: @@ -1282,19 +1272,6 @@ def _finish_exported(self, entry: dict[str, Any], run: RunSpec) -> bool: "run_exit_code": run_exit_code, "run_exit_code_source": "archived_exit_status", } - elif ( - result_count == run.expected_task_count - and self._checkpoint_log_has_final(run.run_label) - ): - run_exit_code = 0 - exit_code_changes = { - "run_exit_code": 0, - "run_exit_code_source": "recovered_full_coverage_remote_done", - "run_exit_code_inference": ( - "inferred zero from verified full-coverage archive and " - "checkpoint final event after remote done" - ), - } stop_command = [ self.config.crabbox_bin, "stop", diff --git a/scripts/native_eval/run_job.py b/scripts/native_eval/run_job.py index ccafe5e..d1d8ab4 100644 --- a/scripts/native_eval/run_job.py +++ b/scripts/native_eval/run_job.py @@ -6,6 +6,7 @@ import os import subprocess import uuid +from collections import Counter from dataclasses import asdict from pathlib import Path from typing import Any @@ -127,9 +128,12 @@ async def execute(task: TaskSpec) -> None: state["finished_at"] = utc_now() state["updated_at"] = state["finished_at"] _update_job_result(state, results, len(tasks)) + acceptance = _execution_acceptance(results, len(tasks)) + state["execution_acceptance"] = acceptance atomic_write_json(job_dir / "result.json", state) manifest["finished_at_utc"] = state["finished_at"] manifest["result_json_count"] = len(results) + manifest["execution_acceptance"] = acceptance agent_results = [ result.get("agent_result") for result in results @@ -204,6 +208,30 @@ def _update_job_result( state["updated_at"] = utc_now() +def _execution_acceptance( + results: list[dict[str, Any]], + total: int, +) -> dict[str, Any]: + kinds = Counter( + str((result.get("execution_outcome") or {}).get("kind") or "unknown") + for result in results + ) + invalid_kinds = {"harness_error", "infra_error", "verifier_error"} + invalid_count = sum(kinds[kind] for kind in invalid_kinds) + accepted = len(results) == total and invalid_count < total + if len(results) != total: + reason = "incomplete_result_coverage" + elif invalid_count == total and total: + reason = "all_trials_harness_or_infrastructure_errors" + else: + reason = None + return { + "accepted": accepted, + "reason": reason, + "outcome_counts": dict(sorted(kinds.items())), + } + + def _run_manifest( run: RunSpec, *, @@ -293,6 +321,11 @@ def _run_manifest( "started_at_utc": started_at, "finished_at_utc": None, "result_json_count": 0, + "execution_acceptance": { + "accepted": None, + "reason": "run_in_progress", + "outcome_counts": {}, + }, } @@ -434,6 +467,8 @@ def main() -> None: ) ) print(json.dumps(state, indent=2)) + if state["execution_acceptance"]["accepted"] is not True: + raise SystemExit(2) if __name__ == "__main__": diff --git a/scripts/native_eval/runtime.py b/scripts/native_eval/runtime.py index f26f693..287e3a4 100644 --- a/scripts/native_eval/runtime.py +++ b/scripts/native_eval/runtime.py @@ -52,6 +52,12 @@ class RewardFileNotFoundError(NativeEvalError): pass +OPENCLAW_HARNESS_EXIT_REASONS = { + 70: "gateway_start_failed", + 71: "terminal_session_evidence_unavailable", +} + + CODEX_DIAGNOSTIC_LINE = re.compile( r"^\d{4}-\d{2}-\d{2}T\S+\s+" r"(?:TRACE|DEBUG|INFO|WARN|ERROR)\s+codex[\w:.-]*:" @@ -473,6 +479,7 @@ async def run_trial( toolchain_root=toolchain_root, ) recorded_exception: BaseException | None = None + agent_exit_code: int | None = None agent_command = build_harness_command( run, proxy_url=proxy_url, @@ -517,6 +524,7 @@ async def run_trial( stderr_path=trial_dir / "agent" / "stderr.txt", ) if agent.returncode: + agent_exit_code = agent.returncode raise NonZeroAgentExitCodeError( f"Agent exited with code {agent.returncode}" ) @@ -584,6 +592,11 @@ async def run_trial( if recorded_exception is None: recorded_exception = exc result["finished_at"] = utc_now() + result["execution_outcome"] = execution_outcome( + harness=run.harness, + exception=recorded_exception, + agent_exit_code=agent_exit_code, + ) if recorded_exception is not None: result["exception_info"] = exception_info(recorded_exception) (trial_dir / "exception.txt").write_text( @@ -640,6 +653,11 @@ def _initial_trial_result( }, "agent_result": None, "verifier_result": None, + "execution_outcome": { + "kind": "pending", + "exit_code": None, + "reason": None, + }, "verifier_environment_mode": "shared", "exception_info": None, "started_at": started_at, @@ -1287,6 +1305,42 @@ def exception_info(exc: BaseException) -> dict[str, str]: } +def execution_outcome( + *, + harness: str, + exception: BaseException | None, + agent_exit_code: int | None, +) -> dict[str, Any]: + if exception is None: + return {"kind": "clean", "exit_code": None, "reason": None} + if ( + harness == "openclaw" + and agent_exit_code in OPENCLAW_HARNESS_EXIT_REASONS + ): + return { + "kind": "harness_error", + "exit_code": agent_exit_code, + "reason": OPENCLAW_HARNESS_EXIT_REASONS[agent_exit_code], + } + if isinstance(exception, NonZeroAgentExitCodeError): + return { + "kind": "agent_error", + "exit_code": agent_exit_code, + "reason": str(exception), + } + if isinstance(exception, (AgentSetupError, AgentSetupTimeoutError)): + kind = "harness_error" + elif isinstance(exception, RewardFileNotFoundError): + kind = "verifier_error" + else: + kind = "infra_error" + return { + "kind": kind, + "exit_code": agent_exit_code, + "reason": str(exception), + } + + def _timing(result: CommandResult) -> dict[str, str]: return { "started_at": result.started_at, diff --git a/tests/test_native_eval_aggregate.py b/tests/test_native_eval_aggregate.py index c1e26d1..d3c793c 100644 --- a/tests/test_native_eval_aggregate.py +++ b/tests/test_native_eval_aggregate.py @@ -509,6 +509,48 @@ def test_native_identity_checks_ignore_unavailable_agent_exit_trajectory( assert run["eligible"] is True +def test_all_openclaw_harness_errors_preserve_rewards_but_reject_run( + tmp_path: Path, +): + jobs_root = tmp_path / "native" + summaries_dir = tmp_path / "summaries" + _write_run( + jobs_root, + "openclaw-gpt55-terminal-evidence-failure", + expected_task_count=2, + results=[ + _result( + "a", + reward=0.25, + exception_type="NonZeroAgentExitCodeError", + exception_message="Agent exited with code 71", + ), + _result( + "b", + reward=0.75, + exception_type="NonZeroAgentExitCodeError", + exception_message="Agent exited with code 71", + ), + ], + harness="openclaw", + ) + + report = aggregate(jobs_root, summaries_dir) + + run = report["runs"][0] + assert run["score"] == pytest.approx(0.5) + assert run["nonzero"] == 2 + assert run["run_accepted"] is False + assert run["run_acceptance_reason"] == ( + "all_trials_harness_or_infrastructure_errors" + ) + assert run["eligible"] is False + assert run["exclusion_reason"] == "infra_dominated" + with (summaries_dir / "per_task_results.csv").open(newline="") as handle: + rows = list(csv.DictReader(handle)) + assert {row["execution_outcome"] for row in rows} == {"harness_error"} + + def test_native_identity_checks_real_agent_exit_trajectory(tmp_path: Path): jobs_root = tmp_path / "native" summaries_dir = tmp_path / "summaries" diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index b9855ff..08bd113 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -1273,7 +1273,7 @@ def test_recovery_required_resumes_existing_remote_run(tmp_path: Path) -> None: assert final["status"] == "completed" -def test_recovery_infers_success_from_verified_full_archive_and_done_log( +def test_recovery_does_not_infer_success_from_result_count( tmp_path: Path, ) -> None: label = "openclaw-gpt55-full-2-r1-20260727" @@ -1287,7 +1287,7 @@ def test_recovery_infers_success_from_verified_full_archive_and_done_log( } run_index = tmp_path / "manifests" / "run_index.json" _write_index(run_index, [run]) - config = _config(tmp_path, run_index) + config = _config(tmp_path, run_index, max_attempts=1) _write_final(config.local_root, label, 2) checkpoint_log = config.local_root / "logs" / f"{label}.checkpoints.log" checkpoint_log.parent.mkdir(parents=True, exist_ok=True) @@ -1309,13 +1309,12 @@ def test_recovery_infers_success_from_verified_full_archive_and_done_log( } executor.active_leases = 1 - assert FleetController(config, executor=executor).run() == 0 + assert FleetController(config, executor=executor).run() == 1 recovered = json.loads(run_index.read_text(encoding="utf-8"))["runs"][0] - assert recovered["status"] == "completed" - assert recovered["run_exit_code"] == 0 - assert recovered["run_exit_code_source"] == "recovered_full_coverage_remote_done" - assert "inferred zero" in recovered["run_exit_code_inference"] + assert recovered["status"] == "failed" + assert recovered.get("run_exit_code") is None + assert recovered["last_error"] == "run exit unknown; result coverage 2/2" def test_recovery_preserves_archived_nonzero_exit_status(tmp_path: Path) -> None: diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 6aef78f..601d22f 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -33,6 +33,7 @@ from scripts.native_eval import plan as native_plan from scripts.native_eval.proxy import JUDGE_PROXY_MODEL_NAME, write_proxy_config from scripts.native_eval.run_job import ( + _execution_acceptance, _git_commit, _run_manifest, build_run_spec, @@ -40,8 +41,10 @@ ) from scripts.native_eval.runtime import ( DockerTaskEnvironment, + NonZeroAgentExitCodeError, build_judge_env, collect_agent_metrics, + execution_outcome, read_reward, write_agent_trajectory, ) @@ -59,6 +62,48 @@ def test_matrix_plan_contains_only_requested_models_and_harnesses() -> None: assert {run.repetition for run in plan} == {1, 2, 3} +def test_openclaw_terminal_evidence_exit_is_a_harness_error() -> None: + outcome = execution_outcome( + harness="openclaw", + exception=NonZeroAgentExitCodeError("Agent exited with code 71"), + agent_exit_code=71, + ) + + assert outcome == { + "kind": "harness_error", + "exit_code": 71, + "reason": "terminal_session_evidence_unavailable", + } + + +def test_all_harness_errors_reject_run_but_agent_errors_do_not() -> None: + rejected = _execution_acceptance( + [ + {"execution_outcome": {"kind": "harness_error"}}, + {"execution_outcome": {"kind": "infra_error"}}, + ], + 2, + ) + accepted = _execution_acceptance( + [ + {"execution_outcome": {"kind": "clean"}}, + {"execution_outcome": {"kind": "agent_error"}}, + ], + 2, + ) + + assert rejected == { + "accepted": False, + "reason": "all_trials_harness_or_infrastructure_errors", + "outcome_counts": {"harness_error": 1, "infra_error": 1}, + } + assert accepted == { + "accepted": True, + "reason": None, + "outcome_counts": {"agent_error": 1, "clean": 1}, + } + + def test_run_index_records_agent_and_judge_reasoning( tmp_path: Path, monkeypatch,