feat(native-eval): support OpenClaw candidate packages

.agents/skills/shellbench-research-runbook/references/runbook.md

@@ -85,6 +85,22 @@ Before every campaign:
 Do not update a pin during a campaign. Start a new campaign ID when a harness,
 runner, task, provider route, or judge version changes.
 
+For a trusted, unpublished OpenClaw build, pass its npm package tarball to the
+fleet controller:
+
+```sh
+python -m scripts.native_eval.fleet \
+  ... \
+  --openclaw-package-tarball /path/to/openclaw.tgz
+```
+
+Candidate campaigns must contain only OpenClaw runs. The controller validates
+the npm package name and version, stages it as
+`manifests/openclaw-candidate.tgz`, and pins its SHA-256 in campaign, toolchain,
+and run provenance. Resume with the same tarball identity; a missing or changed
+candidate is rejected before leasing. Registry installation remains the
+default when the option is absent.
+
 ## Campaign Identity
 
 Use a stable campaign ID:

scripts/native_eval/audit.py

@@ -23,6 +23,17 @@ def _read_object(path: Path) -> dict[str, Any]:
     return value
 
 
+def _candidate_status(
+    expected: dict[str, Any] | None,
+    observed: Any,
+) -> str:
+    if expected is None:
+        return "not_applicable"
+    if not isinstance(observed, dict):
+        return "missing"
+    return "match" if observed == expected else "mismatch"
+
+
 def build_metadata_supplements(
     run_index_path: Path,
     extracted_root: Path,
@@ -36,8 +47,12 @@ def build_metadata_supplements(
     missing = [field for field, value in provenance.items() if value in (None, "")]
     if missing:
         raise ValueError(f"fleet metadata is missing: {', '.join(missing)}")
+    expected_candidate = fleet.get("openclaw_package")
+    if expected_candidate is not None and not isinstance(expected_candidate, dict):
+        raise ValueError("fleet openclaw_package must be an object")
 
     supplements = []
+    candidate_counts = {"match": 0, "mismatch": 0, "missing": 0}
     for manifest_path in sorted(extracted_root.glob("*/shellbench_meta-*/run_manifest.json")):
         manifest = _read_object(manifest_path)
         run_label = str(manifest.get("run_label") or "")
@@ -54,22 +69,31 @@ def build_metadata_supplements(
                     f"{run_label} has conflicting {field}: "
                     f"archive={archived!r}, fleet={expected!r}"
                 )
-        if additions:
-            supplements.append(
-                {
-                    "run_label": run_label,
-                    "archived_manifest": str(manifest_path.relative_to(extracted_root)),
-                    "supplements": additions,
-                }
-            )
+        candidate_status = _candidate_status(
+            expected_candidate,
+            manifest.get("openclaw_package"),
+        )
+        if candidate_status != "not_applicable":
+            candidate_counts[candidate_status] += 1
+        if additions or candidate_status != "not_applicable":
+            supplement = {
+                "run_label": run_label,
+                "archived_manifest": str(manifest_path.relative_to(extracted_root)),
+                "supplements": additions,
+            }
+            if candidate_status != "not_applicable":
+                supplement["openclaw_candidate_status"] = candidate_status
+            supplements.append(supplement)
 
     return {
-        "schema_version": 1,
+        "schema_version": 2,
         "generated_at_utc": datetime.now(UTC).isoformat(),
         "source_run_index": str(run_index_path),
         "extracted_root": str(extracted_root),
         "raw_archives_mutated": False,
         "provenance": provenance,
+        "openclaw_package": expected_candidate,
+        "openclaw_candidate_counts": candidate_counts,
         "runs": supplements,
     }
 

scripts/native_eval/bootstrap_beast.sh

@@ -8,9 +8,23 @@ CODEX_VERSION="${CODEX_VERSION:-0.145.0}"
 CLAUDE_CODE_VERSION="${CLAUDE_CODE_VERSION:-2.1.220}"
 HERMES_COMMIT="${HERMES_COMMIT:-cb06017b1d6e1b9ae0cb35f99a48ffa6bcbaa828}"
 LITELLM_VERSION="${LITELLM_VERSION:-1.93.0}"
+OPENCLAW_PACKAGE_TARBALL="${OPENCLAW_PACKAGE_TARBALL:-}"
+OPENCLAW_PACKAGE_SHA256="${OPENCLAW_PACKAGE_SHA256:-}"
+OPENCLAW_PACKAGE_VERSION="${OPENCLAW_PACKAGE_VERSION:-}"
 
 if [[ "$(id -u)" -ne 0 ]]; then
-  exec sudo -E bash "$0" "$@"
+  exec sudo -E env \
+    TOOLCHAIN_ROOT="$TOOLCHAIN_ROOT" \
+    NODE_VERSION="$NODE_VERSION" \
+    OPENCLAW_VERSION="$OPENCLAW_VERSION" \
+    CODEX_VERSION="$CODEX_VERSION" \
+    CLAUDE_CODE_VERSION="$CLAUDE_CODE_VERSION" \
+    HERMES_COMMIT="$HERMES_COMMIT" \
+    LITELLM_VERSION="$LITELLM_VERSION" \
+    OPENCLAW_PACKAGE_TARBALL="$OPENCLAW_PACKAGE_TARBALL" \
+    OPENCLAW_PACKAGE_SHA256="$OPENCLAW_PACKAGE_SHA256" \
+    OPENCLAW_PACKAGE_VERSION="$OPENCLAW_PACKAGE_VERSION" \
+    bash "$0" "$@"
 fi
 
 export DEBIAN_FRONTEND=noninteractive
@@ -67,6 +81,7 @@ EOF
 
 install_node_tools() {
   local node_root="$TOOLCHAIN_ROOT/node"
+  local openclaw_spec="openclaw@$OPENCLAW_VERSION"
   if [[ ! -x "$node_root/bin/node" ]] || \
     [[ "$("$node_root/bin/node" --version)" != "v$NODE_VERSION" ]]; then
     rm -rf "$node_root"
@@ -79,9 +94,19 @@ install_node_tools() {
   fi
 
   export PATH="$node_root/bin:$PATH"
+  if [[ -n "$OPENCLAW_PACKAGE_TARBALL" ]]; then
+    [[ -n "$OPENCLAW_PACKAGE_SHA256" && -n "$OPENCLAW_PACKAGE_VERSION" ]]
+    [[ -f "$OPENCLAW_PACKAGE_TARBALL" ]]
+    printf '%s  %s\n' \
+      "$OPENCLAW_PACKAGE_SHA256" "$OPENCLAW_PACKAGE_TARBALL" \
+      | sha256sum -c -
+    [[ "$(tar -xOf "$OPENCLAW_PACKAGE_TARBALL" package/package.json | jq -r '.name')" == "openclaw" ]]
+    [[ "$(tar -xOf "$OPENCLAW_PACKAGE_TARBALL" package/package.json | jq -r '.version')" == "$OPENCLAW_PACKAGE_VERSION" ]]
+    openclaw_spec="$OPENCLAW_PACKAGE_TARBALL"
+  fi
   rm -rf "$TOOLCHAIN_ROOT/npm-packages"
   npm install --prefix "$TOOLCHAIN_ROOT/npm-packages" \
-    "openclaw@$OPENCLAW_VERSION" \
+    "$openclaw_spec" \
     "@openai/codex@$CODEX_VERSION" \
     "@anthropic-ai/claude-code@$CLAUDE_CODE_VERSION"
 }
@@ -125,6 +150,16 @@ install_litellm() {
 }
 
 write_manifest() {
+  local openclaw_source_kind="registry"
+  local openclaw_package_version="$OPENCLAW_VERSION"
+  local openclaw_package_sha256=""
+  local openclaw_artifact_filename=""
+  if [[ -n "$OPENCLAW_PACKAGE_TARBALL" ]]; then
+    openclaw_source_kind="npm_tarball"
+    openclaw_package_version="$OPENCLAW_PACKAGE_VERSION"
+    openclaw_package_sha256="$OPENCLAW_PACKAGE_SHA256"
+    openclaw_artifact_filename="$(basename "$OPENCLAW_PACKAGE_TARBALL")"
+  fi
   export PATH="$TOOLCHAIN_ROOT/node/bin:$TOOLCHAIN_ROOT/npm-packages/node_modules/.bin:$TOOLCHAIN_ROOT/home/.local/bin:$PATH"
   jq -n \
     --arg created_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
@@ -135,10 +170,21 @@ write_manifest() {
     --arg hermes "$(hermes version | head -1)" \
     --arg litellm "$("$TOOLCHAIN_ROOT/litellm-venv/bin/python" -c 'from importlib.metadata import version; print(version("litellm"))')" \
     --arg hermes_commit "$HERMES_COMMIT" \
+    --arg openclaw_source_kind "$openclaw_source_kind" \
+    --arg openclaw_package_version "$openclaw_package_version" \
+    --arg openclaw_package_sha256 "$openclaw_package_sha256" \
+    --arg openclaw_artifact_filename "$openclaw_artifact_filename" \
     '{
       created_at_utc: $created_at,
       node: $node,
       openclaw: $openclaw,
+      openclaw_package: {
+        source_kind: $openclaw_source_kind,
+        package_name: "openclaw",
+        package_version: $openclaw_package_version,
+        sha256: (if $openclaw_package_sha256 == "" then null else $openclaw_package_sha256 end),
+        artifact_filename: (if $openclaw_artifact_filename == "" then null else $openclaw_artifact_filename end)
+      },
       codex: $codex,
       claude_code: $claude_code,
       hermes: $hermes,