diff --git a/.github/workflows/deltawire-preflight-v5.yml b/.github/workflows/deltawire-preflight-v5.yml new file mode 100644 index 000000000..6508c60b0 --- /dev/null +++ b/.github/workflows/deltawire-preflight-v5.yml @@ -0,0 +1,28 @@ +name: Verify DeltaWire preflight v5 + +on: + push: + branches: [eval/deltawire-preflight-v5] + pull_request: + paths: + - "labs/20-deltawire/**" + - ".github/workflows/deltawire-preflight-v5.yml" + +permissions: + contents: read + +jobs: + validate-v5: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - uses: actions/setup-go@v7 + with: + go-version-file: labs/20-deltawire/go.mod + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - run: bash labs/20-deltawire/scripts/validate.sh + - run: bash labs/20-deltawire/eval/scripts/v5/validate.sh diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.deltawire/config.json b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.deltawire/config.json new file mode 100644 index 000000000..61a05415a --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.deltawire/config.json @@ -0,0 +1,12 @@ +{ + "limits": { + "max_output_bytes": 104857600, + "max_plan_bytes": 1048576, + "max_records": 100000, + "max_schema_bytes": 1048576 + }, + "plans_dir": ".deltawire/plans", + "schemas_dir": ".deltawire/schemas", + "state_file": ".deltawire/state.json", + "version": "deltawire.config.v1" +} diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.deltawire/schemas/range-large.schema.json b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.deltawire/schemas/range-large.schema.json new file mode 100644 index 000000000..69b476237 --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.deltawire/schemas/range-large.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "index": { + "maximum": 500, + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "index" + ], + "title": "range-large", + "type": "object" +} diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.generated/.gitignore b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.generated/.gitignore new file mode 100644 index 000000000..d6b7ef32c --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.generated/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/Dockerfile b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/Dockerfile new file mode 100644 index 000000000..023396b14 --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:22.04 +COPY .generated/deltawire /usr/local/bin/deltawire +COPY task-contract.json /task-contract.json +COPY .deltawire /.deltawire +COPY verify_plan_contract.py /usr/local/bin/verify_plan_contract.py +COPY environment_receipt.py /usr/local/bin/environment_receipt.py +COPY deltawire-environment-expectations.json /deltawire-environment-expectations.json +RUN chmod 0755 /usr/local/bin/deltawire /usr/local/bin/verify_plan_contract.py /usr/local/bin/environment_receipt.py && chmod a-w /task-contract.json /.deltawire/config.json /.deltawire/schemas/*.json /deltawire-environment-expectations.json +RUN command -v deltawire && deltawire version +RUN apt-get update && apt-get install -y python3 diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/deltawire-environment-expectations.json b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/deltawire-environment-expectations.json new file mode 100644 index 000000000..a47f4e360 --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/deltawire-environment-expectations.json @@ -0,0 +1,23 @@ +{ + "binary": { + "path": "/usr/local/bin/deltawire", + "realpath": "/usr/local/bin/deltawire", + "sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "version": "deltawire version dev" + }, + "files": { + "config": { + "path": "/.deltawire/config.json", + "sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f" + }, + "public_contract": { + "path": "/task-contract.json", + "sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5" + }, + "schema": { + "path": "/.deltawire/schemas/range-large.schema.json", + "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + } + }, + "schema_version": "deltawire-environment-expectations.v1" +} diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/environment_receipt.py b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/environment_receipt.py new file mode 100644 index 000000000..84446c2d5 --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/environment_receipt.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Atomically emit deterministic DeltaWire environment evidence.""" +import argparse,hashlib,json,os,stat,subprocess,tempfile +from pathlib import Path + +def sha(path): + h=hashlib.sha256() + with Path(path).open("rb") as f: + for chunk in iter(lambda:f.read(1048576),b""):h.update(chunk) + return h.hexdigest() +def atomic_write(path,data): + target=Path(path);target.parent.mkdir(parents=True,exist_ok=True) + fd,tmp=tempfile.mkstemp(prefix=f".{target.name}.",suffix=".tmp",dir=target.parent) + try: + with os.fdopen(fd,"w",encoding="utf-8") as f:f.write(data);f.flush();os.fsync(f.fileno()) + os.replace(tmp,target) + except BaseException: + try:os.unlink(tmp) + except FileNotFoundError:pass + raise +def build(expectations): + expected=json.loads(Path(expectations).read_text());binary=Path(expected["binary"]["path"]) + exists=binary.exists();lst=binary.lstat() if exists else None + regular=bool(lst and stat.S_ISREG(lst.st_mode));symlink=binary.is_symlink() if exists else False + executable=exists and os.access(binary,os.X_OK);realpath=str(binary.resolve()) if exists else None + try:version=subprocess.run([str(binary),"version"],capture_output=True,text=True,check=False) if exists else None + except OSError:version=None + files={} + for name,item in sorted(expected["files"].items()): + path=Path(item["path"]);actual=sha(path) if path.is_file() else None + files[name]={"path":item["path"],"exists":path.is_file(),"actual_sha256":actual,"expected_sha256":item["sha256"],"hash_match":actual==item["sha256"]} + binary_hash=sha(binary) if regular else None + checks={"binary_exists":exists,"binary_regular":regular,"binary_not_symlink":not symlink,"binary_executable":executable, + "binary_path":str(binary)==expected["binary"]["path"],"binary_realpath":realpath==expected["binary"]["realpath"], + "binary_hash":binary_hash==expected["binary"]["sha256"],"version_exit_0":bool(version and version.returncode==0), + "version_exact":bool(version and version.stdout.splitlines() and version.stdout.splitlines()[0].strip()==expected["binary"]["version"]), + **{f"{name}_hash":item["hash_match"] for name,item in files.items()}} + return {"schema_version":"deltawire-environment-receipt.v1","binary":{"path":str(binary),"realpath":realpath,"exists":exists, + "is_regular":regular,"is_symlink":symlink,"executable":executable,"actual_sha256":binary_hash, + "expected_sha256":expected["binary"]["sha256"],"version_stdout":version.stdout.strip() if version else None, + "version_stderr":version.stderr.strip() if version else None,"version_exit_code":version.returncode if version else None, + "expected_version":expected["binary"]["version"]},"files":files,"checks":checks,"status":"pass" if all(checks.values()) else "fail"} +def main(): + p=argparse.ArgumentParser();p.add_argument("--expectations",required=True);p.add_argument("--receipt",required=True);a=p.parse_args() + receipt=build(a.expectations);atomic_write(a.receipt,json.dumps(receipt,indent=2,sort_keys=True)+"\n") + print(json.dumps(receipt,sort_keys=True));raise SystemExit(0 if receipt["status"]=="pass" else 1) +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/task-contract.json b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/task-contract.json new file mode 100644 index 000000000..281ba55ac --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/task-contract.json @@ -0,0 +1,24 @@ +{ + "authoritative_schema": { + "path": ".deltawire/schemas/range-large.schema.json", + "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + }, + "deltawire_applicability": "expected", + "family": "range", + "generation": { + "field": "index", + "field_order": [ + "index" + ], + "start": 1 + }, + "ordering": "canonical_generation_order", + "output": { + "format": "ndjson", + "path": "testdata/generated/output.ndjson" + }, + "record_count": 500, + "schema_version": "task-spec.v1", + "size": "large", + "task": "range-large" +} diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/verify_plan_contract.py b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/verify_plan_contract.py new file mode 100644 index 000000000..205d4fc8a --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/verify_plan_contract.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +import argparse, hashlib, json, subprocess +from pathlib import Path + +def sha(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() +def write(path, value): + if path: + target=Path(path); target.parent.mkdir(parents=True,exist_ok=True); target.write_text(json.dumps(value,indent=2,sort_keys=True)+"\n") + +def main(): + p=argparse.ArgumentParser(); p.add_argument("--contract",required=True); p.add_argument("--plan",required=True); p.add_argument("--repo",default="."); p.add_argument("--receipt"); p.add_argument("--deltawire",default="deltawire"); a=p.parse_args() + repo=Path(a.repo).resolve(); contract_path=Path(a.contract).resolve(); plan=Path(a.plan).resolve(); contract=json.loads(contract_path.read_text()) + try: plan_arg=str(plan.relative_to(repo)) + except ValueError: plan_arg=str(plan) + result={"schema_version":"plan-contract-receipt.v1","contract_sha256":sha(contract_path),"plan_sha256":sha(plan),"checks":{},"status":"fail"} + try: + run=subprocess.run([a.deltawire,"inspect","--repo",str(repo),"--format","json",plan_arg],capture_output=True,text=True) + result["inspect_exit_code"]=run.returncode + if run.returncode: result["error"]=run.stderr.strip() or run.stdout.strip() + else: + inspected=json.loads(run.stdout); schema=contract["authoritative_schema"]; schema_path=repo/schema["path"] + checks={"output_path":inspected.get("output_path")==contract["output"]["path"],"output_format":inspected.get("output_format")==contract["output"]["format"],"projected_records":inspected.get("projected_records")==contract["record_count"],"schema_path":inspected.get("schema_path")==schema["path"],"schema_exists":schema_path.is_file(),"schema_sha256":schema_path.is_file() and sha(schema_path)==schema["sha256"],"exact_count_assertion":str(contract["record_count"]) in json.dumps(inspected.get("assertion_summary",{}))} + result.update({"checks":checks,"inspect":inspected,"status":"pass" if all(checks.values()) else "fail"}) + except (OSError,ValueError,KeyError) as error: result["error"]=str(error) + write(a.receipt,result) + if result["status"]!="pass": raise SystemExit(1) + print(json.dumps(result,sort_keys=True)) +if __name__=="__main__": main() diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/instruction.md b/labs/20-deltawire/eval/conformance/environment-receipt-v1/instruction.md new file mode 100644 index 000000000..d8dbe2d66 --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/instruction.md @@ -0,0 +1 @@ +No agent action is required. Exit without changing the task environment. diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/task.toml b/labs/20-deltawire/eval/conformance/environment-receipt-v1/task.toml new file mode 100644 index 000000000..de75749cc --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/task.toml @@ -0,0 +1,29 @@ +schema_version = "1.3" +artifacts = [] + +[task] +name = "operatorstack/deltawire-environment-receipt-conformance-v1" +description = "No-model Harbor main collect-hook conformance" +authors = [] +keywords = [] + +[metadata] +benchmark_result = false +conformance_version = "v1" + +[verifier] +timeout_sec = 60.0 + +[[verifier.collect]] +service = "main" +command = "python3 /usr/local/bin/environment_receipt.py --expectations /deltawire-environment-expectations.json --receipt /logs/artifacts/deltawire/environment-receipt.json" +timeout_sec = 30.0 + +[agent] +timeout_sec = 60.0 + +[environment] +network_mode = "public" +build_timeout_sec = 600.0 +os = "linux" +mcp_servers = [] diff --git a/labs/20-deltawire/eval/conformance/environment-receipt-v1/tests/test.sh b/labs/20-deltawire/eval/conformance/environment-receipt-v1/tests/test.sh new file mode 100755 index 000000000..39751523e --- /dev/null +++ b/labs/20-deltawire/eval/conformance/environment-receipt-v1/tests/test.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +mkdir -p /logs/verifier +printf '{"conformance": 1}\n' > /logs/verifier/reward.json diff --git a/labs/20-deltawire/eval/docs/08-harbor-artifact-timing.md b/labs/20-deltawire/eval/docs/08-harbor-artifact-timing.md new file mode 100644 index 000000000..6b52c1b22 --- /dev/null +++ b/labs/20-deltawire/eval/docs/08-harbor-artifact-timing.md @@ -0,0 +1,29 @@ +# Harbor 0.20.0 artifact timing + +This preflight is pinned to `harbor==0.20.0`. Inspection of the installed package on +`tbench-c4d` established the following order in +`harbor/trial/single_step.py` and `harbor/trial/trial.py`: + +1. the agent phase finishes; +2. `[[verifier.collect]]` hooks with `service = "main"` run while the main + container is still available; +3. main-container artifacts are downloaded; +4. the verifier runs; +5. verifier logs and reward are retained. + +`VerifierCollectConfig` in `harbor/models/task/config.py` accepts `command`, +`service`, `timeout_sec`, and `user`. The implicit convention entry for +`/logs/artifacts/` is created by `harbor/trial/artifact_handler.py` and maps to +`/artifacts/logs/artifacts/`. Its `artifacts/manifest.json` entry has +`source`, `destination`, `type`, `status`, and `service` fields. + +Collect hooks are best effort. A nonzero exit is written as a warning, including +the command, exit code, stdout, and stderr, in the job/trial logs, but collection +continues and no structured hook-exit field is added to the artifact manifest. +Consequently the v5 gate never treats Harbor success or hook exit alone as proof: +it requires the canonical retained receipt plus an `ok` convention-directory +manifest entry. + +The paid task retains the default container user. When that user is root, the +receipt detects accidental environment drift; it is not a cryptographic +anti-cheat boundary against the agent. diff --git a/labs/20-deltawire/eval/manifests/preflight-v4-evidence-lock.json b/labs/20-deltawire/eval/manifests/preflight-v4-evidence-lock.json new file mode 100644 index 000000000..e47bac3fc --- /dev/null +++ b/labs/20-deltawire/eval/manifests/preflight-v4-evidence-lock.json @@ -0,0 +1,90 @@ +{ + "immutable_files": { + "labs/20-deltawire/eval/manifests/preflight-v2-evidence-lock.json": "dfaa4350681a36bcadef1e00d52bd90e16b2cf72638928ba2d0bc42b7af8f1b8", + "labs/20-deltawire/eval/manifests/preflight-v3-evidence-lock.json": "3318137e8f0da487297c3e95d07bac61fb7229d4ff751c334fe444e4444d6f61", + "labs/20-deltawire/eval/manifests/preflight-v4-range-large.json": "badcdcc4d7db11fb5aa715d26fbf6aefcd70f31ccc708afa7030486c2e26f116", + "labs/20-deltawire/eval/matrices/72-run-matrix.json": "f9dcddc4b7336418c7e2b8e7a7e51ca1e42b5a137255ff0823a732618f3568d5", + "labs/20-deltawire/eval/probes/range-large-v4/authoritative-schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/probes/range-large-v4/environment/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "labs/20-deltawire/eval/probes/range-large-v4/environment/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/probes/range-large-v4/environment/Dockerfile": "10f480e7735bab6ca1f9389af058951550284516a98c363b6eb113d4f9ff655f", + "labs/20-deltawire/eval/probes/range-large-v4/environment/deltawire-environment-expectations.json": "5b2c2f4876ef27633b8889ce4bc9209a8233b0ca2c7dfefca24b61de5cabb252", + "labs/20-deltawire/eval/probes/range-large-v4/environment/environment_receipt.py": "bf1a730b409faa8ee8a85a9a83a2f076dc2c1daad72f25c9613e6f18d630a90f", + "labs/20-deltawire/eval/probes/range-large-v4/environment/task-contract.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/probes/range-large-v4/environment/verify_plan_contract.py": "8aafe73ab47213027a258f9abbebbca4803fa53f658abeb7b405d4a79202d4af", + "labs/20-deltawire/eval/probes/range-large-v4/instruction.md": "e71b24acdf77f476573e4373a7f6a814f94e35adf5dcd0e6f048350574359725", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/authoritative-schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/environment/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/environment/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/environment/.generated/.gitignore": "240a3e0d37d2e86b614063f5347eb02d4f99ca6c254de6b82871ff8d95532a7d", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/environment/Dockerfile": "10f480e7735bab6ca1f9389af058951550284516a98c363b6eb113d4f9ff655f", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/environment/deltawire-environment-expectations.json": "5b2c2f4876ef27633b8889ce4bc9209a8233b0ca2c7dfefca24b61de5cabb252", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/environment/environment_receipt.py": "bf1a730b409faa8ee8a85a9a83a2f076dc2c1daad72f25c9613e6f18d630a90f", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/environment/task-contract.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/environment/verify_plan_contract.py": "8aafe73ab47213027a258f9abbebbca4803fa53f658abeb7b405d4a79202d4af", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/instruction.md": "549f4047b1a4ba4f7a3f3dce9e60d1d70b141caa189aafdba942b15e6d022aee", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/task.toml": "01f68e68f24bd143fc2cf9cce1c89871694160a56992d1c19ef03f6f02dc1fc9", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/tests/semantic_oracle.py": "94b30fac117f40e977851966b5228e6c372bb48cbf2e5c29c24f428e1f98e91d", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/tests/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/probes/range-large-v4/pair-task/tests/test.sh": "0c5eb4de20a4c76fb073742a684f15ff3431b38bfa0a9ca27186e4b1f9e65884", + "labs/20-deltawire/eval/probes/range-large-v4/skill/deltawire/SKILL.md": "69f266e086759f3b8371a5708464067d10daf30c7d01a8c8bc06dd39261ec19e", + "labs/20-deltawire/eval/probes/range-large-v4/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/probes/range-large-v4/task.toml": "81f4033d57800432e053aa9166b887f98fff6a4ca339753a5575917ecbe42bd9", + "labs/20-deltawire/eval/probes/range-large-v4/tests/semantic_oracle.py": "94b30fac117f40e977851966b5228e6c372bb48cbf2e5c29c24f428e1f98e91d", + "labs/20-deltawire/eval/probes/range-large-v4/tests/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/probes/range-large-v4/tests/test.sh": "0c5eb4de20a4c76fb073742a684f15ff3431b38bfa0a9ca27186e4b1f9e65884", + "labs/20-deltawire/eval/results/preflight-v1/treatment-probe-range-large/v1-evidence-lock.json": "26f4013270eb366c3c190f8235ef2564985b593005d499338273b1edc2c051d1", + "labs/20-deltawire/eval/results/preflight-v4/index.json": "0b6a2954c9597aab8d99e47958deb9159d2b24f98625be9d9ff5d54a723b152f", + "labs/20-deltawire/eval/results/preflight-v4/readiness.json": "5974d5a844dc7fb8659bb07b59a0625deadb275161dfd774f82df42db5962f4c", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/D1-end-to-end.json": "c99fd2df59f2162da1f245a1b5e8fad778c0ba8d18ad583e2a1751d8cc38f658", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/D1-environment-receipt.json": "c088201283f920cc39bc6d3b5621e9a86ba36a106e3d541aabd409ee7d70c231", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/D1-plan-contract-receipt.json": "9eb93bd1efb49fe95486d81529ff01ab6e2f20500eebd9146063380dea235f5a", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/D1-semantic-result.json": "a8f74965bc7c20d7b8de488e05b6c90e85caaab94b480adb71038165541a7c74", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/D1-treatment-use.json": "be48c00d06af98783f37814cd5ac68233447d19cb1de8ad0f36fabfea13162c5", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/config.json": "b0ba0743f1e6fce0151c0900520a4c4100126d77d114164fa4acb28581d71b52", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/job.log": "d8a2e5471b553e4fd839bb7c6ee4d18b9c7361647e7da8354d77868d850596a0", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/lock.json": "e27de1f7a41bc6f12afcaa3d8afc5e04d5f38936edf5a0f4da7158e266f872ef", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/agent/gemini-cli.trajectory.jsonl": "609a6b11bc5efdee6761986cbefa9b0a92db03a84e6d4fef6d842356ee5712c5", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/agent/gemini-cli.txt": "02dd467e913ee48ccbe01fb8652d888c02a22a1526d5d732b43a7dc6d42d6d17", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/agent/trajectory.json": "a2bf99ef6e09fc90008cc2ada0d9d1f0705887d1f5a2ed3603c8f0e3bc9cd5ad", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/artifacts/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/artifacts/.deltawire/plan-contract-receipt.json": "9eb93bd1efb49fe95486d81529ff01ab6e2f20500eebd9146063380dea235f5a", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/artifacts/.deltawire/plans/range-large.dw.json": "b1dcd5d61787bbc1a1873ae213b1105e7ac19184191eec599f5edd2a443404cb", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/artifacts/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/artifacts/.deltawire/state.json": "4c6bcc092e261346d8dd7ddcbc6a3cfc54e9908f08558840af43ba62dee1319d", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/artifacts/manifest.json": "d26ee4b66af08182faf6590f9cbd5c0ea57ad78825fcc3fc83beccd51ec61e9c", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/artifacts/testdata/generated/output.ndjson": "9d205b57ab166d23c81927899f99b6ccece261c679c96979c58fbde975637776", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/config.json": "66583cab10f7463245dceca31a943d19bcecdfd62c33e9a64cf1442808835692", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/lock.json": "474fe717247a1eb01267b9488acfdc0acd8ef6a5cd1ea939f54a7af34a01e996", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/result.json": "5297c6c91a002e4ea712bb3ccb616944efc55f03a3cef381eafe85a12b439e3f", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/trial.log": "d8a2e5471b553e4fd839bb7c6ee4d18b9c7361647e7da8354d77868d850596a0", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/verifier/reward.json": "36a492513204040f04d7bdf4146d84ede71353d8cc7df112fc3ebff66327683a", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/range-large-v4__X4J4zSn/verifier/test-stdout.txt": "0813813de07909a65008adefbb04228b030d69a9f0acd33629b48a1f8e3fff96", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/raw/D1/2026-07-22__13-20-47/result.json": "91eac94173146576acd04c2e38d5690fd36e9cd1b6306f869b68608174fd9145", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/redaction.json": "fd7ac8bda46226122e7ebd23ab4a0f05d3f508b0c2cae757276a73903d9c3760", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/report.md": "8ba26343e330ee864b9f51ab26854cbb15d1d6be34236b6e80d596bff6cb5423", + "labs/20-deltawire/eval/results/preflight-v4/treatment-probe-range-large-v4/run-ledger.json": "24d91aa5a9544c5c127e9eacea46727452aa9f0eeb0e404f92430e4eb900f8c0", + "labs/20-deltawire/eval/scripts/v4/build_and_stage.sh": "ad93bff7e05e1c6f40ffc8663ba688c77bced2a26c5bf11ccfda35ffc312af16", + "labs/20-deltawire/eval/scripts/v4/environment_receipt.py": "bf1a730b409faa8ee8a85a9a83a2f076dc2c1daad72f25c9613e6f18d630a90f", + "labs/20-deltawire/eval/scripts/v4/fidelity.py": "ae1963ea7021dc61eb38da757706ffe0dae204a96e757f452663080529f74077", + "labs/20-deltawire/eval/scripts/v4/generate_manifest.py": "f2223056e66ceb660b8954f9bc6cb7b32291819dd2ca0573608041bc1d5d8e2b", + "labs/20-deltawire/eval/scripts/v4/generate_v3_evidence_lock.py": "dfccde6c65d9357adf320bc8c28a7a5ad0341b3579b5505126d8e3b5ce3f7b43", + "labs/20-deltawire/eval/scripts/v4/runner.py": "789cd527a7e3659d77f5879347d5493784d30bb5d8ca02c9ed5cdcdb62312547", + "labs/20-deltawire/eval/scripts/v4/semantic_oracle.py": "94b30fac117f40e977851966b5228e6c372bb48cbf2e5c29c24f428e1f98e91d", + "labs/20-deltawire/eval/scripts/v4/shell_observation.py": "e5269e06350147715505a76b8e83235abecd5af34eba1c8837e19b22b0e4c9fe", + "labs/20-deltawire/eval/scripts/v4/test_environment_receipt.py": "59421625f731a5c5c9e555d970b87c1977eda86253cd18382b5aef4679c59d9e", + "labs/20-deltawire/eval/scripts/v4/test_fidelity.py": "531ca3f45bad8c3896829c430e2a01743bd24ceeb102170b7efa57525d545c19", + "labs/20-deltawire/eval/scripts/v4/test_runner.py": "ca72def4dc39be2bfb9b0cbe8e5eb5031b3c8711008abe7473e361eaf2901696", + "labs/20-deltawire/eval/scripts/v4/test_shell_observation.py": "5348c52e60b362408b433813466eb8cd8f03bde8f80033359eadb502bb71f3a0", + "labs/20-deltawire/eval/scripts/v4/validate.py": "571b7d95ac23595fd66268189691644872f0d78eaaf1f6814f27dde50cde9e9f", + "labs/20-deltawire/eval/scripts/v4/validate.sh": "c2ab7b0270a773f36fe877adb69829f0584902105fc2f12512ad6375533f5558", + "labs/20-deltawire/eval/scripts/v4/verify_evidence_locks.py": "8422b5f1bb156bffe6e399586262b474a6ac4bae48cfafb58b348bb2be4eafa0", + "labs/20-deltawire/eval/scripts/v4/verify_plan_contract.py": "8aafe73ab47213027a258f9abbebbca4803fa53f658abeb7b405d4a79202d4af", + "labs/20-deltawire/eval/tasks/range-large/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5" + }, + "probe_version": "v4", + "schema_version": "evidence-lock.v1", + "source_commit": "c200dc0eb3c4dff0e4732cc4b2daa1cd535703e3", + "source_tree": "ebafaafe33d871cce5ef26f39c99e37d0e78c9fa" +} diff --git a/labs/20-deltawire/eval/manifests/preflight-v5-range-large.json b/labs/20-deltawire/eval/manifests/preflight-v5-range-large.json new file mode 100644 index 000000000..c29c510a7 --- /dev/null +++ b/labs/20-deltawire/eval/manifests/preflight-v5-range-large.json @@ -0,0 +1,103 @@ +{ + "agent": "gemini-cli", + "deltawire_binary_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "expected_reported_model": "gemini-3.1-pro-preview", + "failure_policy": "One v5 D1 probe after explicit review; no rerun; one range-large/r1 pair only after probe-release.v3 and separate explicit review; full disabled.", + "harbor_command": [ + "uvx", + "--from", + "harbor==0.20.0", + "harbor" + ], + "harbor_package": "harbor==0.20.0", + "harbor_version": "0.20.0", + "input_hashes": { + ".github/workflows/deltawire-preflight-v5.yml": "ae3e1302e29abaa6237810bb68476fd3fcdc05a174170b65f0347a7cbe15f6a6", + "labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/Dockerfile": "10f480e7735bab6ca1f9389af058951550284516a98c363b6eb113d4f9ff655f", + "labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/deltawire-environment-expectations.json": "03fe7a40964e908f2fb3736add8b8bea890b407d837f2b185d92bd8d85d4d114", + "labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/environment_receipt.py": "3ab3bbcbfae2093ac658063d1e26f26b43c35dfdb81d8ddae5c9989f967ba623", + "labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/task-contract.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/verify_plan_contract.py": "8aafe73ab47213027a258f9abbebbca4803fa53f658abeb7b405d4a79202d4af", + "labs/20-deltawire/eval/conformance/environment-receipt-v1/instruction.md": "13ac474d6339049f3629f6d8d459977def3c83ae460a382abe7dac2b77a71617", + "labs/20-deltawire/eval/conformance/environment-receipt-v1/task.toml": "7b3772655d6f967d8eff44237b11de8601988c259970c90fcb6178163fe97836", + "labs/20-deltawire/eval/conformance/environment-receipt-v1/tests/test.sh": "37f56e4c911939ea2dcdfaa62a2565096d86171f4ee76cb0238f218a4c3df026", + "labs/20-deltawire/eval/docs/08-harbor-artifact-timing.md": "ac39794af4dc8c035f99c323fe23902e6ea64fbf0bbd6ae26bc07580c3dac44d", + "labs/20-deltawire/eval/manifests/preflight-v2-evidence-lock.json": "dfaa4350681a36bcadef1e00d52bd90e16b2cf72638928ba2d0bc42b7af8f1b8", + "labs/20-deltawire/eval/manifests/preflight-v3-evidence-lock.json": "3318137e8f0da487297c3e95d07bac61fb7229d4ff751c334fe444e4444d6f61", + "labs/20-deltawire/eval/manifests/preflight-v4-evidence-lock.json": "1ca5068babe733e26fcf9f039b13f53b7214edd4eaf51e609ff2f5a16d425bea", + "labs/20-deltawire/eval/matrices/72-run-matrix.json": "f9dcddc4b7336418c7e2b8e7a7e51ca1e42b5a137255ff0823a732618f3568d5", + "labs/20-deltawire/eval/probes/range-large-v5/authoritative-schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/probes/range-large-v5/environment/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "labs/20-deltawire/eval/probes/range-large-v5/environment/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/probes/range-large-v5/environment/Dockerfile": "10f480e7735bab6ca1f9389af058951550284516a98c363b6eb113d4f9ff655f", + "labs/20-deltawire/eval/probes/range-large-v5/environment/deltawire-environment-expectations.json": "03fe7a40964e908f2fb3736add8b8bea890b407d837f2b185d92bd8d85d4d114", + "labs/20-deltawire/eval/probes/range-large-v5/environment/environment_receipt.py": "3ab3bbcbfae2093ac658063d1e26f26b43c35dfdb81d8ddae5c9989f967ba623", + "labs/20-deltawire/eval/probes/range-large-v5/environment/task-contract.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/probes/range-large-v5/environment/verify_plan_contract.py": "8aafe73ab47213027a258f9abbebbca4803fa53f658abeb7b405d4a79202d4af", + "labs/20-deltawire/eval/probes/range-large-v5/instruction.md": "ad855907b280f918b29d07095dda5d94ffd52b35de5453688dd701094a494ed0", + "labs/20-deltawire/eval/probes/range-large-v5/pair-task/authoritative-schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/Dockerfile": "10f480e7735bab6ca1f9389af058951550284516a98c363b6eb113d4f9ff655f", + "labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/deltawire-environment-expectations.json": "03fe7a40964e908f2fb3736add8b8bea890b407d837f2b185d92bd8d85d4d114", + "labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/environment_receipt.py": "3ab3bbcbfae2093ac658063d1e26f26b43c35dfdb81d8ddae5c9989f967ba623", + "labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/task-contract.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/verify_plan_contract.py": "8aafe73ab47213027a258f9abbebbca4803fa53f658abeb7b405d4a79202d4af", + "labs/20-deltawire/eval/probes/range-large-v5/pair-task/instruction.md": "549f4047b1a4ba4f7a3f3dce9e60d1d70b141caa189aafdba942b15e6d022aee", + "labs/20-deltawire/eval/probes/range-large-v5/pair-task/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/probes/range-large-v5/pair-task/task.toml": "14a0949996a6659c3c674424e8d86cb54c72ceb27e596cd982bb3474173ddf37", + "labs/20-deltawire/eval/probes/range-large-v5/pair-task/tests/semantic_oracle.py": "94b30fac117f40e977851966b5228e6c372bb48cbf2e5c29c24f428e1f98e91d", + "labs/20-deltawire/eval/probes/range-large-v5/pair-task/tests/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/probes/range-large-v5/pair-task/tests/test.sh": "6f7cfbefdf7eaf3a082fd0c88f264ba3d41bfe84fc53498df4464bef66dc116e", + "labs/20-deltawire/eval/probes/range-large-v5/skill/deltawire/SKILL.md": "69f266e086759f3b8371a5708464067d10daf30c7d01a8c8bc06dd39261ec19e", + "labs/20-deltawire/eval/probes/range-large-v5/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/probes/range-large-v5/task.toml": "b79e0b411f49d34edc0c6556b2bfc356aafb0cd6f4ba746dc949357976f8c441", + "labs/20-deltawire/eval/probes/range-large-v5/tests/semantic_oracle.py": "94b30fac117f40e977851966b5228e6c372bb48cbf2e5c29c24f428e1f98e91d", + "labs/20-deltawire/eval/probes/range-large-v5/tests/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/probes/range-large-v5/tests/test.sh": "6f7cfbefdf7eaf3a082fd0c88f264ba3d41bfe84fc53498df4464bef66dc116e", + "labs/20-deltawire/eval/results/preflight-v1/treatment-probe-range-large/v1-evidence-lock.json": "26f4013270eb366c3c190f8235ef2564985b593005d499338273b1edc2c051d1", + "labs/20-deltawire/eval/results/preflight-v5/conformance/environment-receipt-v1.json": "37c35894da0d23867743e710b096227e83744705dce24d1e6ecd94084a644fac", + "labs/20-deltawire/eval/scripts/v5/artifact_manifest.py": "722f71bfbec56cbdbaea85a2f975aaa61f58d2dbbbd41e4a6b02741f3ac90ca0", + "labs/20-deltawire/eval/scripts/v5/build_and_stage.sh": "13a55ba653f33e6392c74d0697e92dcd745b4b19e67921afdb6a5f75664e87ad", + "labs/20-deltawire/eval/scripts/v5/environment_receipt.py": "3ab3bbcbfae2093ac658063d1e26f26b43c35dfdb81d8ddae5c9989f967ba623", + "labs/20-deltawire/eval/scripts/v5/fidelity.py": "959cad8329a4310dfa518bbec03ec165c1c4e1abc98d260592e269ed948b51ad", + "labs/20-deltawire/eval/scripts/v5/generate_manifest.py": "a5474229e629587ebd217edb65a09f461984fb8c74d98b7376edfb165b83faf6", + "labs/20-deltawire/eval/scripts/v5/generate_v4_evidence_lock.py": "8a3f45ce90bc668923f98ed5220a55b10b6bdd4e0c1717f31003b78784eea55e", + "labs/20-deltawire/eval/scripts/v5/negative_controls.py": "00ce502966d9108959bccf58d269eca48280f5ad45c8c8e2503c44ca478d9c2b", + "labs/20-deltawire/eval/scripts/v5/prepare_tasks.py": "18fbde97cfbb27f3044dc0410adbd42f2b93516a73baf18997272c024c45a2e5", + "labs/20-deltawire/eval/scripts/v5/run_conformance.py": "ed318230e496ff0a74cf084efbcc216239d88938b9349674601e7e64453c82c7", + "labs/20-deltawire/eval/scripts/v5/runner.py": "6083f1b852ec06605c70cc5237b02c8423af2348a1be0a8dfe3bbbdd881cbcf3", + "labs/20-deltawire/eval/scripts/v5/semantic_oracle.py": "94b30fac117f40e977851966b5228e6c372bb48cbf2e5c29c24f428e1f98e91d", + "labs/20-deltawire/eval/scripts/v5/shell_observation.py": "e5269e06350147715505a76b8e83235abecd5af34eba1c8837e19b22b0e4c9fe", + "labs/20-deltawire/eval/scripts/v5/test_artifact_manifest.py": "5e02b55f8e7593543706b574de1cd989b4189f7dc3b93f57fb5967a04c6404dc", + "labs/20-deltawire/eval/scripts/v5/test_environment_receipt.py": "a455799e2da1255f1520b96a62b608a14b4a89d5172d79171deb5e3dde781662", + "labs/20-deltawire/eval/scripts/v5/test_fidelity.py": "531ca3f45bad8c3896829c430e2a01743bd24ceeb102170b7efa57525d545c19", + "labs/20-deltawire/eval/scripts/v5/test_runner.py": "6a39a03dc58e70a283a4f91c6f66378cd262d27d7001ed47039ba43003184b61", + "labs/20-deltawire/eval/scripts/v5/test_shell_observation.py": "5348c52e60b362408b433813466eb8cd8f03bde8f80033359eadb502bb71f3a0", + "labs/20-deltawire/eval/scripts/v5/validate.py": "f9f04a21239b3765e0027215b7e5b213a00779ab10779d0462b7acf53de5ff14", + "labs/20-deltawire/eval/scripts/v5/validate.sh": "96d31d6aedcc8db4c5d5d84062316d9d4116eb771f227ccbbaa7b1bc9263d52a", + "labs/20-deltawire/eval/scripts/v5/verify_evidence_locks.py": "8e4acc6fea24165da326d0a8e3d6b995f85bfb0a7c32df45dae4964e3584e9b6", + "labs/20-deltawire/eval/scripts/v5/verify_plan_contract.py": "8aafe73ab47213027a258f9abbebbca4803fa53f658abeb7b405d4a79202d4af", + "labs/20-deltawire/eval/tasks/range-large/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5" + }, + "matrix_positions": { + "order": [ + "D1", + "B0" + ], + "zero_based": [ + 36, + 37 + ] + }, + "model": "google/gemini-3.1-pro-preview", + "ready_for_72": false, + "repository_commit": "9c766ccca89f5ca44c69a80c7a0abe48bccc4b10", + "schema_version": "deltawire-preflight-v5-manifest.v1", + "scope": [ + "probe/range-large-v5", + "range-large/r1" + ] +} diff --git a/labs/20-deltawire/eval/probes/range-large-v5/authoritative-schema.json b/labs/20-deltawire/eval/probes/range-large-v5/authoritative-schema.json new file mode 100644 index 000000000..69b476237 --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/authoritative-schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "index": { + "maximum": 500, + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "index" + ], + "title": "range-large", + "type": "object" +} diff --git a/labs/20-deltawire/eval/probes/range-large-v5/environment/.deltawire/config.json b/labs/20-deltawire/eval/probes/range-large-v5/environment/.deltawire/config.json new file mode 100644 index 000000000..61a05415a --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/environment/.deltawire/config.json @@ -0,0 +1,12 @@ +{ + "limits": { + "max_output_bytes": 104857600, + "max_plan_bytes": 1048576, + "max_records": 100000, + "max_schema_bytes": 1048576 + }, + "plans_dir": ".deltawire/plans", + "schemas_dir": ".deltawire/schemas", + "state_file": ".deltawire/state.json", + "version": "deltawire.config.v1" +} diff --git a/labs/20-deltawire/eval/probes/range-large-v5/environment/.deltawire/schemas/range-large.schema.json b/labs/20-deltawire/eval/probes/range-large-v5/environment/.deltawire/schemas/range-large.schema.json new file mode 100644 index 000000000..69b476237 --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/environment/.deltawire/schemas/range-large.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "index": { + "maximum": 500, + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "index" + ], + "title": "range-large", + "type": "object" +} diff --git a/labs/20-deltawire/eval/probes/range-large-v5/environment/Dockerfile b/labs/20-deltawire/eval/probes/range-large-v5/environment/Dockerfile new file mode 100644 index 000000000..023396b14 --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/environment/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:22.04 +COPY .generated/deltawire /usr/local/bin/deltawire +COPY task-contract.json /task-contract.json +COPY .deltawire /.deltawire +COPY verify_plan_contract.py /usr/local/bin/verify_plan_contract.py +COPY environment_receipt.py /usr/local/bin/environment_receipt.py +COPY deltawire-environment-expectations.json /deltawire-environment-expectations.json +RUN chmod 0755 /usr/local/bin/deltawire /usr/local/bin/verify_plan_contract.py /usr/local/bin/environment_receipt.py && chmod a-w /task-contract.json /.deltawire/config.json /.deltawire/schemas/*.json /deltawire-environment-expectations.json +RUN command -v deltawire && deltawire version +RUN apt-get update && apt-get install -y python3 diff --git a/labs/20-deltawire/eval/probes/range-large-v5/environment/deltawire-environment-expectations.json b/labs/20-deltawire/eval/probes/range-large-v5/environment/deltawire-environment-expectations.json new file mode 100644 index 000000000..a47f4e360 --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/environment/deltawire-environment-expectations.json @@ -0,0 +1,23 @@ +{ + "binary": { + "path": "/usr/local/bin/deltawire", + "realpath": "/usr/local/bin/deltawire", + "sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "version": "deltawire version dev" + }, + "files": { + "config": { + "path": "/.deltawire/config.json", + "sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f" + }, + "public_contract": { + "path": "/task-contract.json", + "sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5" + }, + "schema": { + "path": "/.deltawire/schemas/range-large.schema.json", + "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + } + }, + "schema_version": "deltawire-environment-expectations.v1" +} diff --git a/labs/20-deltawire/eval/probes/range-large-v5/environment/environment_receipt.py b/labs/20-deltawire/eval/probes/range-large-v5/environment/environment_receipt.py new file mode 100644 index 000000000..84446c2d5 --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/environment/environment_receipt.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Atomically emit deterministic DeltaWire environment evidence.""" +import argparse,hashlib,json,os,stat,subprocess,tempfile +from pathlib import Path + +def sha(path): + h=hashlib.sha256() + with Path(path).open("rb") as f: + for chunk in iter(lambda:f.read(1048576),b""):h.update(chunk) + return h.hexdigest() +def atomic_write(path,data): + target=Path(path);target.parent.mkdir(parents=True,exist_ok=True) + fd,tmp=tempfile.mkstemp(prefix=f".{target.name}.",suffix=".tmp",dir=target.parent) + try: + with os.fdopen(fd,"w",encoding="utf-8") as f:f.write(data);f.flush();os.fsync(f.fileno()) + os.replace(tmp,target) + except BaseException: + try:os.unlink(tmp) + except FileNotFoundError:pass + raise +def build(expectations): + expected=json.loads(Path(expectations).read_text());binary=Path(expected["binary"]["path"]) + exists=binary.exists();lst=binary.lstat() if exists else None + regular=bool(lst and stat.S_ISREG(lst.st_mode));symlink=binary.is_symlink() if exists else False + executable=exists and os.access(binary,os.X_OK);realpath=str(binary.resolve()) if exists else None + try:version=subprocess.run([str(binary),"version"],capture_output=True,text=True,check=False) if exists else None + except OSError:version=None + files={} + for name,item in sorted(expected["files"].items()): + path=Path(item["path"]);actual=sha(path) if path.is_file() else None + files[name]={"path":item["path"],"exists":path.is_file(),"actual_sha256":actual,"expected_sha256":item["sha256"],"hash_match":actual==item["sha256"]} + binary_hash=sha(binary) if regular else None + checks={"binary_exists":exists,"binary_regular":regular,"binary_not_symlink":not symlink,"binary_executable":executable, + "binary_path":str(binary)==expected["binary"]["path"],"binary_realpath":realpath==expected["binary"]["realpath"], + "binary_hash":binary_hash==expected["binary"]["sha256"],"version_exit_0":bool(version and version.returncode==0), + "version_exact":bool(version and version.stdout.splitlines() and version.stdout.splitlines()[0].strip()==expected["binary"]["version"]), + **{f"{name}_hash":item["hash_match"] for name,item in files.items()}} + return {"schema_version":"deltawire-environment-receipt.v1","binary":{"path":str(binary),"realpath":realpath,"exists":exists, + "is_regular":regular,"is_symlink":symlink,"executable":executable,"actual_sha256":binary_hash, + "expected_sha256":expected["binary"]["sha256"],"version_stdout":version.stdout.strip() if version else None, + "version_stderr":version.stderr.strip() if version else None,"version_exit_code":version.returncode if version else None, + "expected_version":expected["binary"]["version"]},"files":files,"checks":checks,"status":"pass" if all(checks.values()) else "fail"} +def main(): + p=argparse.ArgumentParser();p.add_argument("--expectations",required=True);p.add_argument("--receipt",required=True);a=p.parse_args() + receipt=build(a.expectations);atomic_write(a.receipt,json.dumps(receipt,indent=2,sort_keys=True)+"\n") + print(json.dumps(receipt,sort_keys=True));raise SystemExit(0 if receipt["status"]=="pass" else 1) +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/probes/range-large-v5/environment/task-contract.json b/labs/20-deltawire/eval/probes/range-large-v5/environment/task-contract.json new file mode 100644 index 000000000..281ba55ac --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/environment/task-contract.json @@ -0,0 +1,24 @@ +{ + "authoritative_schema": { + "path": ".deltawire/schemas/range-large.schema.json", + "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + }, + "deltawire_applicability": "expected", + "family": "range", + "generation": { + "field": "index", + "field_order": [ + "index" + ], + "start": 1 + }, + "ordering": "canonical_generation_order", + "output": { + "format": "ndjson", + "path": "testdata/generated/output.ndjson" + }, + "record_count": 500, + "schema_version": "task-spec.v1", + "size": "large", + "task": "range-large" +} diff --git a/labs/20-deltawire/eval/probes/range-large-v5/environment/verify_plan_contract.py b/labs/20-deltawire/eval/probes/range-large-v5/environment/verify_plan_contract.py new file mode 100644 index 000000000..205d4fc8a --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/environment/verify_plan_contract.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +import argparse, hashlib, json, subprocess +from pathlib import Path + +def sha(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() +def write(path, value): + if path: + target=Path(path); target.parent.mkdir(parents=True,exist_ok=True); target.write_text(json.dumps(value,indent=2,sort_keys=True)+"\n") + +def main(): + p=argparse.ArgumentParser(); p.add_argument("--contract",required=True); p.add_argument("--plan",required=True); p.add_argument("--repo",default="."); p.add_argument("--receipt"); p.add_argument("--deltawire",default="deltawire"); a=p.parse_args() + repo=Path(a.repo).resolve(); contract_path=Path(a.contract).resolve(); plan=Path(a.plan).resolve(); contract=json.loads(contract_path.read_text()) + try: plan_arg=str(plan.relative_to(repo)) + except ValueError: plan_arg=str(plan) + result={"schema_version":"plan-contract-receipt.v1","contract_sha256":sha(contract_path),"plan_sha256":sha(plan),"checks":{},"status":"fail"} + try: + run=subprocess.run([a.deltawire,"inspect","--repo",str(repo),"--format","json",plan_arg],capture_output=True,text=True) + result["inspect_exit_code"]=run.returncode + if run.returncode: result["error"]=run.stderr.strip() or run.stdout.strip() + else: + inspected=json.loads(run.stdout); schema=contract["authoritative_schema"]; schema_path=repo/schema["path"] + checks={"output_path":inspected.get("output_path")==contract["output"]["path"],"output_format":inspected.get("output_format")==contract["output"]["format"],"projected_records":inspected.get("projected_records")==contract["record_count"],"schema_path":inspected.get("schema_path")==schema["path"],"schema_exists":schema_path.is_file(),"schema_sha256":schema_path.is_file() and sha(schema_path)==schema["sha256"],"exact_count_assertion":str(contract["record_count"]) in json.dumps(inspected.get("assertion_summary",{}))} + result.update({"checks":checks,"inspect":inspected,"status":"pass" if all(checks.values()) else "fail"}) + except (OSError,ValueError,KeyError) as error: result["error"]=str(error) + write(a.receipt,result) + if result["status"]!="pass": raise SystemExit(1) + print(json.dumps(result,sort_keys=True)) +if __name__=="__main__": main() diff --git a/labs/20-deltawire/eval/probes/range-large-v5/instruction.md b/labs/20-deltawire/eval/probes/range-large-v5/instruction.md new file mode 100644 index 000000000..88c4201e1 --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/instruction.md @@ -0,0 +1,12 @@ +This is the single versioned `range-large-v5` DeltaWire treatment probe. + +Use a structured shell call to run `gemini skills list`, then activate the +`deltawire` skill with the structured activation tool. Read public +`/task-contract.json`. Create exactly one DeltaWire plan at +`.deltawire/plans/range-large.dw.json`, then follow the skill's validate → +independent contract check → render → check lifecycle and stop. + +Generate exactly 500 NDJSON records at `testdata/generated/output.ndjson` in +canonical order. Each record has exactly one integer field, `index`, with values +1 through 500. Do not edit the installed config or schema, do not inspect hidden +verifier files, and do not create expanded records manually. diff --git a/labs/20-deltawire/eval/probes/range-large-v5/pair-task/authoritative-schema.json b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/authoritative-schema.json new file mode 100644 index 000000000..69b476237 --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/authoritative-schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "index": { + "maximum": 500, + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "index" + ], + "title": "range-large", + "type": "object" +} diff --git a/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/.deltawire/config.json b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/.deltawire/config.json new file mode 100644 index 000000000..61a05415a --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/.deltawire/config.json @@ -0,0 +1,12 @@ +{ + "limits": { + "max_output_bytes": 104857600, + "max_plan_bytes": 1048576, + "max_records": 100000, + "max_schema_bytes": 1048576 + }, + "plans_dir": ".deltawire/plans", + "schemas_dir": ".deltawire/schemas", + "state_file": ".deltawire/state.json", + "version": "deltawire.config.v1" +} diff --git a/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/.deltawire/schemas/range-large.schema.json b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/.deltawire/schemas/range-large.schema.json new file mode 100644 index 000000000..69b476237 --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/.deltawire/schemas/range-large.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "index": { + "maximum": 500, + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "index" + ], + "title": "range-large", + "type": "object" +} diff --git a/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/.generated/.gitignore b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/.generated/.gitignore new file mode 100644 index 000000000..d6b7ef32c --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/.generated/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/Dockerfile b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/Dockerfile new file mode 100644 index 000000000..023396b14 --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:22.04 +COPY .generated/deltawire /usr/local/bin/deltawire +COPY task-contract.json /task-contract.json +COPY .deltawire /.deltawire +COPY verify_plan_contract.py /usr/local/bin/verify_plan_contract.py +COPY environment_receipt.py /usr/local/bin/environment_receipt.py +COPY deltawire-environment-expectations.json /deltawire-environment-expectations.json +RUN chmod 0755 /usr/local/bin/deltawire /usr/local/bin/verify_plan_contract.py /usr/local/bin/environment_receipt.py && chmod a-w /task-contract.json /.deltawire/config.json /.deltawire/schemas/*.json /deltawire-environment-expectations.json +RUN command -v deltawire && deltawire version +RUN apt-get update && apt-get install -y python3 diff --git a/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/deltawire-environment-expectations.json b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/deltawire-environment-expectations.json new file mode 100644 index 000000000..a47f4e360 --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/deltawire-environment-expectations.json @@ -0,0 +1,23 @@ +{ + "binary": { + "path": "/usr/local/bin/deltawire", + "realpath": "/usr/local/bin/deltawire", + "sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "version": "deltawire version dev" + }, + "files": { + "config": { + "path": "/.deltawire/config.json", + "sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f" + }, + "public_contract": { + "path": "/task-contract.json", + "sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5" + }, + "schema": { + "path": "/.deltawire/schemas/range-large.schema.json", + "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + } + }, + "schema_version": "deltawire-environment-expectations.v1" +} diff --git a/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/environment_receipt.py b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/environment_receipt.py new file mode 100644 index 000000000..84446c2d5 --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/environment_receipt.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Atomically emit deterministic DeltaWire environment evidence.""" +import argparse,hashlib,json,os,stat,subprocess,tempfile +from pathlib import Path + +def sha(path): + h=hashlib.sha256() + with Path(path).open("rb") as f: + for chunk in iter(lambda:f.read(1048576),b""):h.update(chunk) + return h.hexdigest() +def atomic_write(path,data): + target=Path(path);target.parent.mkdir(parents=True,exist_ok=True) + fd,tmp=tempfile.mkstemp(prefix=f".{target.name}.",suffix=".tmp",dir=target.parent) + try: + with os.fdopen(fd,"w",encoding="utf-8") as f:f.write(data);f.flush();os.fsync(f.fileno()) + os.replace(tmp,target) + except BaseException: + try:os.unlink(tmp) + except FileNotFoundError:pass + raise +def build(expectations): + expected=json.loads(Path(expectations).read_text());binary=Path(expected["binary"]["path"]) + exists=binary.exists();lst=binary.lstat() if exists else None + regular=bool(lst and stat.S_ISREG(lst.st_mode));symlink=binary.is_symlink() if exists else False + executable=exists and os.access(binary,os.X_OK);realpath=str(binary.resolve()) if exists else None + try:version=subprocess.run([str(binary),"version"],capture_output=True,text=True,check=False) if exists else None + except OSError:version=None + files={} + for name,item in sorted(expected["files"].items()): + path=Path(item["path"]);actual=sha(path) if path.is_file() else None + files[name]={"path":item["path"],"exists":path.is_file(),"actual_sha256":actual,"expected_sha256":item["sha256"],"hash_match":actual==item["sha256"]} + binary_hash=sha(binary) if regular else None + checks={"binary_exists":exists,"binary_regular":regular,"binary_not_symlink":not symlink,"binary_executable":executable, + "binary_path":str(binary)==expected["binary"]["path"],"binary_realpath":realpath==expected["binary"]["realpath"], + "binary_hash":binary_hash==expected["binary"]["sha256"],"version_exit_0":bool(version and version.returncode==0), + "version_exact":bool(version and version.stdout.splitlines() and version.stdout.splitlines()[0].strip()==expected["binary"]["version"]), + **{f"{name}_hash":item["hash_match"] for name,item in files.items()}} + return {"schema_version":"deltawire-environment-receipt.v1","binary":{"path":str(binary),"realpath":realpath,"exists":exists, + "is_regular":regular,"is_symlink":symlink,"executable":executable,"actual_sha256":binary_hash, + "expected_sha256":expected["binary"]["sha256"],"version_stdout":version.stdout.strip() if version else None, + "version_stderr":version.stderr.strip() if version else None,"version_exit_code":version.returncode if version else None, + "expected_version":expected["binary"]["version"]},"files":files,"checks":checks,"status":"pass" if all(checks.values()) else "fail"} +def main(): + p=argparse.ArgumentParser();p.add_argument("--expectations",required=True);p.add_argument("--receipt",required=True);a=p.parse_args() + receipt=build(a.expectations);atomic_write(a.receipt,json.dumps(receipt,indent=2,sort_keys=True)+"\n") + print(json.dumps(receipt,sort_keys=True));raise SystemExit(0 if receipt["status"]=="pass" else 1) +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/task-contract.json b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/task-contract.json new file mode 100644 index 000000000..281ba55ac --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/task-contract.json @@ -0,0 +1,24 @@ +{ + "authoritative_schema": { + "path": ".deltawire/schemas/range-large.schema.json", + "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + }, + "deltawire_applicability": "expected", + "family": "range", + "generation": { + "field": "index", + "field_order": [ + "index" + ], + "start": 1 + }, + "ordering": "canonical_generation_order", + "output": { + "format": "ndjson", + "path": "testdata/generated/output.ndjson" + }, + "record_count": 500, + "schema_version": "task-spec.v1", + "size": "large", + "task": "range-large" +} diff --git a/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/verify_plan_contract.py b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/verify_plan_contract.py new file mode 100644 index 000000000..205d4fc8a --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/environment/verify_plan_contract.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +import argparse, hashlib, json, subprocess +from pathlib import Path + +def sha(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() +def write(path, value): + if path: + target=Path(path); target.parent.mkdir(parents=True,exist_ok=True); target.write_text(json.dumps(value,indent=2,sort_keys=True)+"\n") + +def main(): + p=argparse.ArgumentParser(); p.add_argument("--contract",required=True); p.add_argument("--plan",required=True); p.add_argument("--repo",default="."); p.add_argument("--receipt"); p.add_argument("--deltawire",default="deltawire"); a=p.parse_args() + repo=Path(a.repo).resolve(); contract_path=Path(a.contract).resolve(); plan=Path(a.plan).resolve(); contract=json.loads(contract_path.read_text()) + try: plan_arg=str(plan.relative_to(repo)) + except ValueError: plan_arg=str(plan) + result={"schema_version":"plan-contract-receipt.v1","contract_sha256":sha(contract_path),"plan_sha256":sha(plan),"checks":{},"status":"fail"} + try: + run=subprocess.run([a.deltawire,"inspect","--repo",str(repo),"--format","json",plan_arg],capture_output=True,text=True) + result["inspect_exit_code"]=run.returncode + if run.returncode: result["error"]=run.stderr.strip() or run.stdout.strip() + else: + inspected=json.loads(run.stdout); schema=contract["authoritative_schema"]; schema_path=repo/schema["path"] + checks={"output_path":inspected.get("output_path")==contract["output"]["path"],"output_format":inspected.get("output_format")==contract["output"]["format"],"projected_records":inspected.get("projected_records")==contract["record_count"],"schema_path":inspected.get("schema_path")==schema["path"],"schema_exists":schema_path.is_file(),"schema_sha256":schema_path.is_file() and sha(schema_path)==schema["sha256"],"exact_count_assertion":str(contract["record_count"]) in json.dumps(inspected.get("assertion_summary",{}))} + result.update({"checks":checks,"inspect":inspected,"status":"pass" if all(checks.values()) else "fail"}) + except (OSError,ValueError,KeyError) as error: result["error"]=str(error) + write(a.receipt,result) + if result["status"]!="pass": raise SystemExit(1) + print(json.dumps(result,sort_keys=True)) +if __name__=="__main__": main() diff --git a/labs/20-deltawire/eval/probes/range-large-v5/pair-task/instruction.md b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/instruction.md new file mode 100644 index 000000000..9181f4398 --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/instruction.md @@ -0,0 +1,28 @@ +Generate the exact `range-large` dataset defined by public `/task-contract.json`. Write exactly 500 JSON records in canonical order as NDJSON to `testdata/generated/output.ndjson`. Every record, field, value, and ordering rule must match this complete embedded contract: + +```json +{ + "authoritative_schema": { + "path": ".deltawire/schemas/range-large.schema.json", + "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + }, + "deltawire_applicability": "expected", + "family": "range", + "generation": { + "field": "index", + "field_order": [ + "index" + ], + "start": 1 + }, + "ordering": "canonical_generation_order", + "output": { + "format": "ndjson", + "path": "testdata/generated/output.ndjson" + }, + "record_count": 500, + "schema_version": "task-spec.v1", + "size": "large", + "task": "range-large" +} +``` diff --git a/labs/20-deltawire/eval/probes/range-large-v5/pair-task/task-spec.json b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/task-spec.json new file mode 100644 index 000000000..281ba55ac --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/task-spec.json @@ -0,0 +1,24 @@ +{ + "authoritative_schema": { + "path": ".deltawire/schemas/range-large.schema.json", + "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + }, + "deltawire_applicability": "expected", + "family": "range", + "generation": { + "field": "index", + "field_order": [ + "index" + ], + "start": 1 + }, + "ordering": "canonical_generation_order", + "output": { + "format": "ndjson", + "path": "testdata/generated/output.ndjson" + }, + "record_count": 500, + "schema_version": "task-spec.v1", + "size": "large", + "task": "range-large" +} diff --git a/labs/20-deltawire/eval/probes/range-large-v5/pair-task/task.toml b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/task.toml new file mode 100644 index 000000000..99916c7d7 --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/task.toml @@ -0,0 +1,29 @@ +schema_version = "1.3" +artifacts = ["testdata/generated/output.ndjson", ".deltawire/config.json", ".deltawire/plans/range-large.dw.json", ".deltawire/schemas/range-large.schema.json", ".deltawire/state.json", ".deltawire/plan-contract-receipt.json"] + +[task] +name = "operatorstack/range-large-v5-pair" +description = "Frozen range-large/r1 operational positive-control task" +authors = [] +keywords = [] + +[metadata] +benchmark_result = true +probe_version = "v5-pair" + +[verifier] +timeout_sec = 600.0 + +[[verifier.collect]] +service = "main" +command = "python3 /usr/local/bin/environment_receipt.py --expectations /deltawire-environment-expectations.json --receipt /logs/artifacts/deltawire/environment-receipt.json" +timeout_sec = 30.0 + +[agent] +timeout_sec = 600.0 + +[environment] +network_mode = "public" +build_timeout_sec = 600.0 +os = "linux" +mcp_servers = [] diff --git a/labs/20-deltawire/eval/probes/range-large-v5/pair-task/tests/semantic_oracle.py b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/tests/semantic_oracle.py new file mode 100644 index 000000000..00cfbfbc6 --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/tests/semantic_oracle.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +import json +import sys +from pathlib import Path + + +def canonical(spec): + g, count, family = spec["generation"], spec["record_count"], spec["family"] + if family == "matrix": + rows = [{}] + for key, allowed in g["dimensions"].items(): + rows = [{**row, key: value} for row in rows for value in allowed] + return [{**row, **g["constants"]} for row in rows] + if family == "range": + return [{g["field"]: i} for i in range(g["start"], g["start"] + count)] + if family == "rows": + return [{"id": i, "name": g["name_template"].format(id=i), **g["constants"]} for i in range(g["start"], g["start"] + count)] + if family == "variants": + return [{**g["base"], "value": g["base"]["value"] + i * g["value_step"]} for i in range(count)] + if family == "mixed": + return ([{"id": i, "type": "std"} for i in range(1, g["standard_count"] + 1)] + + [{"id": i, "type": "exc", "error_code": g["error_code"]} for i in range(g["standard_count"] + 1, count + 1)]) + return [{"id": i, f'{g["dynamic_field_prefix"]}{i}': i} for i in range(g["id_start"], g["id_start"] + count)] + + +def main(): + if len(sys.argv) != 3: + print('{"exact_match": 0}'); return + spec_arg, output_arg = Path(sys.argv[1]), Path(sys.argv[2]) + if not spec_arg.is_file(): + candidate = Path(__file__).resolve().parents[1] / "tasks" / sys.argv[1] / "task-spec.json" + spec_arg = candidate + try: + spec = json.loads(spec_arg.read_text(encoding="utf-8")) + lines = output_arg.read_text(encoding="utf-8").splitlines() + if not lines or any(not line.strip() for line in lines): + raise ValueError("empty line") + actual = [json.loads(line) for line in lines] + passed = actual == canonical(spec) + except Exception: + passed = False + print(json.dumps({"exact_match": 1 if passed else 0})) + + +if __name__ == "__main__": + main() diff --git a/labs/20-deltawire/eval/probes/range-large-v5/pair-task/tests/task-spec.json b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/tests/task-spec.json new file mode 100644 index 000000000..281ba55ac --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/tests/task-spec.json @@ -0,0 +1,24 @@ +{ + "authoritative_schema": { + "path": ".deltawire/schemas/range-large.schema.json", + "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + }, + "deltawire_applicability": "expected", + "family": "range", + "generation": { + "field": "index", + "field_order": [ + "index" + ], + "start": 1 + }, + "ordering": "canonical_generation_order", + "output": { + "format": "ndjson", + "path": "testdata/generated/output.ndjson" + }, + "record_count": 500, + "schema_version": "task-spec.v1", + "size": "large", + "task": "range-large" +} diff --git a/labs/20-deltawire/eval/probes/range-large-v5/pair-task/tests/test.sh b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/tests/test.sh new file mode 100755 index 000000000..9c1039618 --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/pair-task/tests/test.sh @@ -0,0 +1,8 @@ +#!/bin/bash +mkdir -p /logs/verifier +OUTPUT_FILE="testdata/generated/output.ndjson" +if [ ! -f "$OUTPUT_FILE" ]; then + mkdir -p /logs/verifier && echo '{"exact_match": 0}' > /logs/verifier/reward.json + exit 0 +fi +mkdir -p /logs/verifier && python3 /tests/semantic_oracle.py /tests/task-spec.json "$OUTPUT_FILE" > /logs/verifier/reward.json diff --git a/labs/20-deltawire/eval/probes/range-large-v5/skill/deltawire/SKILL.md b/labs/20-deltawire/eval/probes/range-large-v5/skill/deltawire/SKILL.md new file mode 100644 index 000000000..22776d5af --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/skill/deltawire/SKILL.md @@ -0,0 +1,28 @@ +--- +name: deltawire +description: Use for deterministic generation of large repetitive JSON or NDJSON datasets from compact Cartesian products, ranges, repeated rows, variants, or regular-plus-exception patterns. Discover with `gemini skills list` before activation. Abstain for small or highly irregular outputs. +--- + +# DeltaWire + +Before activation, use a structured shell call to run `gemini skills list` and +confirm `deltawire` is listed. Activate this skill only when the requested data +has a compact deterministic description. Explicitly abstain when it is too +irregular for a compact deterministic plan. + +Read `/task-contract.json`. Treat its output path, record count, ordering, +schema path, schema hash, and assertions as authoritative. The preinstalled +config and schema are immutable. Create only +`.deltawire/plans/range-large.dw.json`; never create or edit config, schema, +state, environment receipts, verifier files, or expanded output manually. + +Run the lifecycle in order: + +1. `deltawire validate --repo . .deltawire/plans/range-large.dw.json` +2. `verify_plan_contract.py --contract /task-contract.json --repo . --plan .deltawire/plans/range-large.dw.json --receipt .deltawire/plan-contract-receipt.json` +3. `deltawire render --repo . .deltawire/plans/range-large.dw.json` +4. `deltawire check --repo . .deltawire/plans/range-large.dw.json` + +The independent environment receipt, retained-artifact contract check, and +semantic oracle are the final authorities. Do not infer their results or +replace them with prose. diff --git a/labs/20-deltawire/eval/probes/range-large-v5/task-spec.json b/labs/20-deltawire/eval/probes/range-large-v5/task-spec.json new file mode 100644 index 000000000..281ba55ac --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/task-spec.json @@ -0,0 +1,24 @@ +{ + "authoritative_schema": { + "path": ".deltawire/schemas/range-large.schema.json", + "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + }, + "deltawire_applicability": "expected", + "family": "range", + "generation": { + "field": "index", + "field_order": [ + "index" + ], + "start": 1 + }, + "ordering": "canonical_generation_order", + "output": { + "format": "ndjson", + "path": "testdata/generated/output.ndjson" + }, + "record_count": 500, + "schema_version": "task-spec.v1", + "size": "large", + "task": "range-large" +} diff --git a/labs/20-deltawire/eval/probes/range-large-v5/task.toml b/labs/20-deltawire/eval/probes/range-large-v5/task.toml new file mode 100644 index 000000000..24ceeb6ad --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/task.toml @@ -0,0 +1,29 @@ +schema_version = "1.3" +artifacts = ["testdata/generated/output.ndjson", ".deltawire/config.json", ".deltawire/plans/range-large.dw.json", ".deltawire/schemas/range-large.schema.json", ".deltawire/state.json", ".deltawire/plan-contract-receipt.json"] + +[task] +name = "operatorstack/range-large-treatment-probe-v5" +description = "Versioned Harbor collect-hook recovery probe" +authors = [] +keywords = [] + +[metadata] +benchmark_result = false +probe_version = "v5" + +[verifier] +timeout_sec = 600.0 + +[[verifier.collect]] +service = "main" +command = "python3 /usr/local/bin/environment_receipt.py --expectations /deltawire-environment-expectations.json --receipt /logs/artifacts/deltawire/environment-receipt.json" +timeout_sec = 30.0 + +[agent] +timeout_sec = 600.0 + +[environment] +network_mode = "public" +build_timeout_sec = 600.0 +os = "linux" +mcp_servers = [] diff --git a/labs/20-deltawire/eval/probes/range-large-v5/tests/semantic_oracle.py b/labs/20-deltawire/eval/probes/range-large-v5/tests/semantic_oracle.py new file mode 100644 index 000000000..00cfbfbc6 --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/tests/semantic_oracle.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +import json +import sys +from pathlib import Path + + +def canonical(spec): + g, count, family = spec["generation"], spec["record_count"], spec["family"] + if family == "matrix": + rows = [{}] + for key, allowed in g["dimensions"].items(): + rows = [{**row, key: value} for row in rows for value in allowed] + return [{**row, **g["constants"]} for row in rows] + if family == "range": + return [{g["field"]: i} for i in range(g["start"], g["start"] + count)] + if family == "rows": + return [{"id": i, "name": g["name_template"].format(id=i), **g["constants"]} for i in range(g["start"], g["start"] + count)] + if family == "variants": + return [{**g["base"], "value": g["base"]["value"] + i * g["value_step"]} for i in range(count)] + if family == "mixed": + return ([{"id": i, "type": "std"} for i in range(1, g["standard_count"] + 1)] + + [{"id": i, "type": "exc", "error_code": g["error_code"]} for i in range(g["standard_count"] + 1, count + 1)]) + return [{"id": i, f'{g["dynamic_field_prefix"]}{i}': i} for i in range(g["id_start"], g["id_start"] + count)] + + +def main(): + if len(sys.argv) != 3: + print('{"exact_match": 0}'); return + spec_arg, output_arg = Path(sys.argv[1]), Path(sys.argv[2]) + if not spec_arg.is_file(): + candidate = Path(__file__).resolve().parents[1] / "tasks" / sys.argv[1] / "task-spec.json" + spec_arg = candidate + try: + spec = json.loads(spec_arg.read_text(encoding="utf-8")) + lines = output_arg.read_text(encoding="utf-8").splitlines() + if not lines or any(not line.strip() for line in lines): + raise ValueError("empty line") + actual = [json.loads(line) for line in lines] + passed = actual == canonical(spec) + except Exception: + passed = False + print(json.dumps({"exact_match": 1 if passed else 0})) + + +if __name__ == "__main__": + main() diff --git a/labs/20-deltawire/eval/probes/range-large-v5/tests/task-spec.json b/labs/20-deltawire/eval/probes/range-large-v5/tests/task-spec.json new file mode 100644 index 000000000..281ba55ac --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/tests/task-spec.json @@ -0,0 +1,24 @@ +{ + "authoritative_schema": { + "path": ".deltawire/schemas/range-large.schema.json", + "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + }, + "deltawire_applicability": "expected", + "family": "range", + "generation": { + "field": "index", + "field_order": [ + "index" + ], + "start": 1 + }, + "ordering": "canonical_generation_order", + "output": { + "format": "ndjson", + "path": "testdata/generated/output.ndjson" + }, + "record_count": 500, + "schema_version": "task-spec.v1", + "size": "large", + "task": "range-large" +} diff --git a/labs/20-deltawire/eval/probes/range-large-v5/tests/test.sh b/labs/20-deltawire/eval/probes/range-large-v5/tests/test.sh new file mode 100755 index 000000000..9c1039618 --- /dev/null +++ b/labs/20-deltawire/eval/probes/range-large-v5/tests/test.sh @@ -0,0 +1,8 @@ +#!/bin/bash +mkdir -p /logs/verifier +OUTPUT_FILE="testdata/generated/output.ndjson" +if [ ! -f "$OUTPUT_FILE" ]; then + mkdir -p /logs/verifier && echo '{"exact_match": 0}' > /logs/verifier/reward.json + exit 0 +fi +mkdir -p /logs/verifier && python3 /tests/semantic_oracle.py /tests/task-spec.json "$OUTPUT_FILE" > /logs/verifier/reward.json diff --git a/labs/20-deltawire/eval/results/preflight-v5/approvals/paid-v5-probe.json b/labs/20-deltawire/eval/results/preflight-v5/approvals/paid-v5-probe.json new file mode 100644 index 000000000..53641cccf --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/approvals/paid-v5-probe.json @@ -0,0 +1,16 @@ +{ + "READY_FOR_72": false, + "authorization_text": "approve the paid v5 probe", + "authorized_scope": [ + "one D1-only range-large-v5 probe" + ], + "explicitly_not_authorized": [ + "pair", + "rerun", + "six-run canary", + "72-run experiment" + ], + "manifest_sha256": "6f99b8b29181b4bf3021397e4c3d1b7827f1850f3d1cc25ddb638f021ccdbf57", + "probe": "range-large-v5", + "schema_version": "deltawire-paid-probe-approval.v1" +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/conformance/environment-receipt-v1.json b/labs/20-deltawire/eval/results/preflight-v5/conformance/environment-receipt-v1.json new file mode 100644 index 000000000..b226ed67f --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/conformance/environment-receipt-v1.json @@ -0,0 +1,79 @@ +{ + "agent": "nop", + "binary_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "expectation_hashes": { + "config": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "public_contract": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "schema": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5" + }, + "harbor_version": "0.20.0", + "model": null, + "negative_controls": { + "atomic_write_failure": true, + "binary_non_executable": true, + "binary_symlink": true, + "collect_hook_nonzero": true, + "copied_v4_receipt": true, + "malformed_receipt": true, + "manifest_missing_or_skipped": true, + "missing_binary": true, + "missing_collect_hook": true, + "missing_config": true, + "verifier_only_receipt": true, + "version_nonzero": true, + "wrong_binary_hash": true, + "wrong_contract_hash": true, + "wrong_receipt_path": true, + "wrong_schema_hash": true, + "wrong_version": true + }, + "positive_trials": [ + { + "agent": "nop", + "artifact_gate_status": "pass", + "artifact_manifest_entry": { + "destination": "artifacts/logs/artifacts", + "service": null, + "source": "/logs/artifacts", + "status": "ok", + "type": "directory" + }, + "artifact_manifest_sha256": "8d0a02e9c17d62af6176024f463c041ed30ff29d3a7b66dadbeab1d762b0cbd9", + "command": "uvx --from harbor==0.20.0 harbor run -a nop -p labs/20-deltawire/eval/conformance/environment-receipt-v1 -k 1 --jobs-dir /positive-1 --n-concurrent 1", + "environment_receipt_path": "positive-1/2026-07-22__14-44-04/environment-receipt-v1__sjjT4wM/artifacts/logs/artifacts/deltawire/environment-receipt.json", + "environment_receipt_sha256": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "harbor_exit_code": 0, + "model": null, + "reported_agent": "nop", + "trial": 1, + "trial_path": "positive-1/2026-07-22__14-44-04/environment-receipt-v1__sjjT4wM" + }, + { + "agent": "nop", + "artifact_gate_status": "pass", + "artifact_manifest_entry": { + "destination": "artifacts/logs/artifacts", + "service": null, + "source": "/logs/artifacts", + "status": "ok", + "type": "directory" + }, + "artifact_manifest_sha256": "8d0a02e9c17d62af6176024f463c041ed30ff29d3a7b66dadbeab1d762b0cbd9", + "command": "uvx --from harbor==0.20.0 harbor run -a nop -p labs/20-deltawire/eval/conformance/environment-receipt-v1 -k 1 --jobs-dir /positive-2 --n-concurrent 1", + "environment_receipt_path": "positive-2/2026-07-22__14-45-20/environment-receipt-v1__4jdxKnL/artifacts/logs/artifacts/deltawire/environment-receipt.json", + "environment_receipt_sha256": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "harbor_exit_code": 0, + "model": null, + "reported_agent": "nop", + "trial": 2, + "trial_path": "positive-2/2026-07-22__14-45-20/environment-receipt-v1__4jdxKnL" + } + ], + "ready_for_72": false, + "schema_version": "deltawire-no-model-conformance.v1", + "source_infrastructure_commit": "40bb6619819be468d66201d9e4aa947c6042415a", + "stable_receipt_sha256": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "status": "pass", + "workflow_url": "https://github.com/operatorstack/intelligence-flow/actions/runs/29929855472", + "zero_model_calls": true +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/pre-paid-review.md b/labs/20-deltawire/eval/results/preflight-v5/pre-paid-review.md new file mode 100644 index 000000000..7a672e25e --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/pre-paid-review.md @@ -0,0 +1,64 @@ +# DeltaWire preflight v5: pre-paid review gate + +## PR disposition + +- PR #106 merged: `e3e085debd1f13c6497eaa7b8e7acc0fed2ada92`. +- PR #99 closed as superseded immutable experiment provenance. +- v5 experiment branch: `eval/deltawire-preflight-v5`. + +## v4 preservation + +- Evidence commit: `c200dc0eb3c4dff0e4732cc4b2daa1cd535703e3`. +- Lock: `labs/20-deltawire/eval/manifests/preflight-v4-evidence-lock.json`. +- Lock SHA-256: `1ca5068babe733e26fcf9f039b13f53b7214edd4eaf51e609ff2f5a16d425bea`. +- Verification: all v1-v4 Git-object locks passed; 82 v4 files are locked. + +## Harbor lifecycle grounding + +- Installed version: `0.20.0`. +- Grounding: `labs/20-deltawire/eval/docs/08-harbor-artifact-timing.md`. +- Observed order: agent → main collect hook → main artifact collection → verifier. +- Hook: `service = "main"`; command SHA-256 + `84e7e23209d27981a19a638bf13124b8c207f2e13f534ae58633d18803f38739`. +- Canonical host receipt path: + `artifacts/logs/artifacts/deltawire/environment-receipt.json`. + +## No-model conformance + +- Compact evidence: `labs/20-deltawire/eval/results/preflight-v5/conformance/environment-receipt-v1.json`. +- Positive trials: two; both Harbor exit 0, `agent = nop`, `model = null`, and artifact gate pass. +- Convention manifest entry: `/logs/artifacts` → `artifacts/logs/artifacts`, directory status `ok`. +- Stable receipt SHA-256: + `4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1`. +- Negative controls: all 17 rejected, including missing/nonzero hook evidence, + verifier-only and misplaced receipts, binary/version/config/schema/contract + failures, malformed or skipped artifacts, v4 substitution, and atomic-write failure. +- Zero-model proof: commands contained `-a nop`, no `-m`, no `--env-file`, no skill, + and retained results reported `nop` with no model. + +## Frozen v5 manifest + +- Input commit: `9c766ccca89f5ca44c69a80c7a0abe48bccc4b10`. +- Manifest commit: `8ef1937e97800efa2b061009a8e5218503ffb195`. +- Manifest: `labs/20-deltawire/eval/manifests/preflight-v5-range-large.json`. +- Manifest SHA-256: `6f99b8b29181b4bf3021397e4c3d1b7827f1850f3d1cc25ddb638f021ccdbf57`. +- Model: `google/gemini-3.1-pro-preview`; reported model expected: + `gemini-3.1-pro-preview`; agent: `gemini-cli`; Harbor: `0.20.0`. +- Binary SHA-256: `e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4`. +- Skill SHA-256: `69f266e086759f3b8371a5708464067d10daf30c7d01a8c8bc06dd39261ec19e`. +- Probe task configuration SHA-256: + `b79e0b411f49d34edc0c6556b2bfc356aafb0cd6f4ba746dc949357976f8c441`. +- Runner SHA-256: `6083f1b852ec06605c70cc5237b02c8423af2348a1be0a8dfe3bbbdd881cbcf3`. + +## CI and paid-probe dry run + +- Exact manifest-head workflow: + https://github.com/operatorstack/intelligence-flow/actions/runs/29930309800 +- Conclusion: success. +- VM head: exact manifest commit; binary parity passed; all frozen hashes and + v1-v4 locks passed; v5 probe result root was empty. +- Redacted one-trial command: + + `uvx --from harbor==0.20.0 harbor run -a gemini-cli -m google/gemini-3.1-pro-preview -p labs/20-deltawire/eval/probes/range-large-v5 -k 1 --jobs-dir labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1 --n-concurrent 1 --skill labs/20-deltawire/eval/probes/range-large-v5/skill/deltawire --env-file ` + +No paid probe has run. `READY_FOR_72=false`. diff --git a/labs/20-deltawire/eval/results/preflight-v5/readiness.json b/labs/20-deltawire/eval/results/preflight-v5/readiness.json new file mode 100644 index 000000000..d54c68365 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/readiness.json @@ -0,0 +1,9 @@ +{ + "READY_FOR_72": false, + "full_run_started": false, + "pair_complete": false, + "pair_launched": false, + "probe_complete": false, + "schema_version": "deltawire-readiness.v5", + "six_run_canary_started": false +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-artifact-manifest-receipt.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-artifact-manifest-receipt.json new file mode 100644 index 000000000..acb95d154 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-artifact-manifest-receipt.json @@ -0,0 +1,28 @@ +{ + "artifact_manifest_sha256": "6ed4dabb1ccea6827c700b6da5f5b45cd02b63b72b85d884a40950c54ea37121", + "checks": { + "binary_hash_frozen": true, + "config_hash_frozen": true, + "manifest_exists": true, + "manifest_status_ok": true, + "public_contract_hash_frozen": true, + "receipt_canonical_path": true, + "receipt_checks_complete": true, + "receipt_checks_true": true, + "receipt_schema_exact": true, + "receipt_status_pass": true, + "schema_hash_frozen": true, + "single_convention_entry": true + }, + "convention_entry": { + "destination": "artifacts/logs/artifacts", + "service": null, + "source": "/logs/artifacts", + "status": "ok", + "type": "directory" + }, + "environment_receipt_path": "/home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/logs/artifacts/deltawire/environment-receipt.json", + "environment_receipt_sha256": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "schema_version": "artifact-manifest-receipt.v1", + "status": "pass" +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-end-to-end.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-end-to-end.json new file mode 100644 index 000000000..a423bd006 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-end-to-end.json @@ -0,0 +1,22 @@ +{ + "arm": "D1", + "artifact_manifest_status": "pass", + "environment_status": "pass", + "harbor_exit_0": true, + "model_match": true, + "plan_contract_status": "fail", + "ready_for_72": false, + "schema_version": "deltawire-end-to-end-receipt.v5", + "semantic_status": "fail", + "source_hashes": { + "artifact_manifest_receipt": "90aa12bdcd80ae70d8e053dae958e5848c45dceecd16759a9d6c410274a32c88", + "environment": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "output": null, + "result": "b1c29f30ba5677e51f57db870e74e91f966a0e62736ab81f1b0151d57b952da6", + "semantic": "b6627f4ded6aa3685d6c966f03d78efeb8c693e5687e82d4aa4dc8a477d4113f", + "trajectory": null, + "treatment_use": "44def672d5152acf149ed668043f673b92e9b8563ac857513c79d7053a6a5941" + }, + "status": "fail", + "treatment_status": "fail" +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-environment-receipt.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-environment-receipt.json new file mode 100644 index 000000000..55108c6ac --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-environment-receipt.json @@ -0,0 +1,55 @@ +{ + "binary": { + "actual_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "executable": true, + "exists": true, + "expected_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "expected_version": "deltawire version dev", + "is_regular": true, + "is_symlink": false, + "path": "/usr/local/bin/deltawire", + "realpath": "/usr/local/bin/deltawire", + "version_exit_code": 0, + "version_stderr": "", + "version_stdout": "deltawire version dev\nGo runtime version go1.26.5\nsupported config version deltawire.config.v1\nsupported plan version deltawire.plan.v1\nsupported state version deltawire.state.v1" + }, + "checks": { + "binary_executable": true, + "binary_exists": true, + "binary_hash": true, + "binary_not_symlink": true, + "binary_path": true, + "binary_realpath": true, + "binary_regular": true, + "config_hash": true, + "public_contract_hash": true, + "schema_hash": true, + "version_exact": true, + "version_exit_0": true + }, + "files": { + "config": { + "actual_sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "exists": true, + "expected_sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "hash_match": true, + "path": "/.deltawire/config.json" + }, + "public_contract": { + "actual_sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "exists": true, + "expected_sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "hash_match": true, + "path": "/task-contract.json" + }, + "schema": { + "actual_sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "exists": true, + "expected_sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "hash_match": true, + "path": "/.deltawire/schemas/range-large.schema.json" + } + }, + "schema_version": "deltawire-environment-receipt.v1", + "status": "pass" +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-plan-contract-receipt.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-plan-contract-receipt.json new file mode 100644 index 000000000..f3ba750aa --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-plan-contract-receipt.json @@ -0,0 +1,6 @@ +{ + "checks": {}, + "error": "expected one plan, found 0", + "schema_version": "plan-contract-receipt.v1", + "status": "fail" +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-semantic-result.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-semantic-result.json new file mode 100644 index 000000000..e94af9f0e --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-semantic-result.json @@ -0,0 +1,6 @@ +{ + "error": "output missing", + "exact_match": 0, + "schema_version": "semantic-result.v5", + "status": "fail" +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-treatment-use.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-treatment-use.json new file mode 100644 index 000000000..18aacdbeb --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/D1-treatment-use.json @@ -0,0 +1,11 @@ +{ + "arm": "D1", + "checks": {}, + "classification": "attempted_failed", + "contamination": false, + "failure_reasons": [ + "missing artifacts: trajectory_path" + ], + "schema_version": "treatment-use-receipt.v5", + "status": "fail" +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/config.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/config.json new file mode 100644 index 000000000..67ab8b70c --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/config.json @@ -0,0 +1,18 @@ +{ + "jobs_dir": "/home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1", + "n_concurrent_trials": 1, + "agents": [ + { + "name": "gemini-cli", + "model_name": "google/gemini-3.1-pro-preview", + "skills": [ + "/home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5/skill/deltawire" + ] + } + ], + "tasks": [ + { + "path": "/home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5" + } + ] +} \ No newline at end of file diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/job.log b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/job.log new file mode 100644 index 000000000..b674a4de4 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/job.log @@ -0,0 +1,331 @@ +Skipping image OS validation for hb__e298605c2a7fe495353069ca6f1b234d: docker inspect returned 1 +Running command: apt-get update && apt-get install -y curl +Trial range-large-v5__ZXfAtDF failed: Agent setup timed out after 360.0 seconds +Collecting main service artifacts +Running collect hook in service 'main': 'python3 /usr/local/bin/environment_receipt.py --expectations /deltawire-environment-expectations.json --receipt /logs/artifacts/deltawire/environment-receipt.json' +Collect hook in service 'main' completed +Failed to download artifact 'testdata/generated/output.ndjson' from service 'main' (best-effort) +Traceback (most recent call last): + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 520, in wait_for + return await fut + ^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 622, in setup + await self.install(environment) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/gemini_cli.py", line 111, in install + await self.exec_as_root( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 579, in exec_as_root + return await self._exec( + ^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 543, in _exec + result = await environment.exec( + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1096, in exec + return await self._compose_exec( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1173, in _compose_exec + return await self._run_docker_compose_command( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 649, in _run_docker_compose_command + result = await self._collect_buffered_output( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 679, in _collect_buffered_output + stdout_bytes, stderr_bytes = await process.communicate(input=stdin_data) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 201, in communicate + stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 181, in _read_stream + output = await stream.read() + ^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 706, in read + block = await self.read(self._limit) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 713, in read + await self._wait_for_data('read') + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 545, in _wait_for_data + await self._waiter +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 1180, in _setup_agent + await asyncio.wait_for( + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 519, in wait_for + async with timeouts.timeout(timeout): + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/timeouts.py", line 115, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 351, in run + await self._prepare() + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 389, in _prepare + await self._setup_agent() + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 1185, in _setup_agent + raise AgentSetupTimeoutError( +harbor.trial.errors.AgentSetupTimeoutError: Agent setup timed out after 360.0 seconds + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/artifact_handler.py", line 330, in _download_artifact + await source_env.service_download_file( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1052, in service_download_file + await self.download_file(source_path, target_path) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1037, in download_file + await self._platform.download_file(source_path, target_path) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker_unix.py", line 185, in download_file + await self._env._run_docker_compose_command( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 656, in _run_docker_compose_command + raise RuntimeError( +RuntimeError: Docker compose command failed for environment range-large-treatment-probe-v5. Command: docker compose --project-name range-large-v5__zxfatdf__env --project-directory /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5/environment -f /tmp/tmptnzk7inw/range-large-v5__ZXfAtDF__env-docker-compose-resources.json -f /home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker-compose-build.yaml -f /tmp/tmpaf91s5aa/docker-compose-environment.json -f /tmp/tmpeg59vbav/docker-compose-mounts.json cp main:testdata/generated/output.ndjson /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/testdata/generated/output.ndjson. Return code: 1. Stdout: range-large-v5__zxfatdf__env-main-1 Copying range-large-v5__zxfatdf__env-main-1:testdata/generated/output.ndjson to /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/testdata/generated/output.ndjson +Error response from daemon: Could not find the file testdata/generated/output.ndjson in container fce6c11ef728d481a6ea79955985457d970323cf6a85263d192f3aa3430f2061 +. Stderr: None. +Failed to download artifact '.deltawire/plans/range-large.dw.json' from service 'main' (best-effort) +Traceback (most recent call last): + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 520, in wait_for + return await fut + ^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 622, in setup + await self.install(environment) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/gemini_cli.py", line 111, in install + await self.exec_as_root( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 579, in exec_as_root + return await self._exec( + ^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 543, in _exec + result = await environment.exec( + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1096, in exec + return await self._compose_exec( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1173, in _compose_exec + return await self._run_docker_compose_command( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 649, in _run_docker_compose_command + result = await self._collect_buffered_output( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 679, in _collect_buffered_output + stdout_bytes, stderr_bytes = await process.communicate(input=stdin_data) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 201, in communicate + stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 181, in _read_stream + output = await stream.read() + ^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 706, in read + block = await self.read(self._limit) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 713, in read + await self._wait_for_data('read') + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 545, in _wait_for_data + await self._waiter +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 1180, in _setup_agent + await asyncio.wait_for( + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 519, in wait_for + async with timeouts.timeout(timeout): + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/timeouts.py", line 115, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 351, in run + await self._prepare() + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 389, in _prepare + await self._setup_agent() + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 1185, in _setup_agent + raise AgentSetupTimeoutError( +harbor.trial.errors.AgentSetupTimeoutError: Agent setup timed out after 360.0 seconds + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/artifact_handler.py", line 330, in _download_artifact + await source_env.service_download_file( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1052, in service_download_file + await self.download_file(source_path, target_path) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1037, in download_file + await self._platform.download_file(source_path, target_path) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker_unix.py", line 185, in download_file + await self._env._run_docker_compose_command( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 656, in _run_docker_compose_command + raise RuntimeError( +RuntimeError: Docker compose command failed for environment range-large-treatment-probe-v5. Command: docker compose --project-name range-large-v5__zxfatdf__env --project-directory /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5/environment -f /tmp/tmptnzk7inw/range-large-v5__ZXfAtDF__env-docker-compose-resources.json -f /home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker-compose-build.yaml -f /tmp/tmpaf91s5aa/docker-compose-environment.json -f /tmp/tmpeg59vbav/docker-compose-mounts.json cp main:.deltawire/plans/range-large.dw.json /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/plans/range-large.dw.json. Return code: 1. Stdout: range-large-v5__zxfatdf__env-main-1 Copying range-large-v5__zxfatdf__env-main-1:.deltawire/plans/range-large.dw.json to /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/plans/range-large.dw.json +Error response from daemon: Could not find the file .deltawire/plans/range-large.dw.json in container fce6c11ef728d481a6ea79955985457d970323cf6a85263d192f3aa3430f2061 +. Stderr: None. +Failed to download artifact '.deltawire/state.json' from service 'main' (best-effort) +Traceback (most recent call last): + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 520, in wait_for + return await fut + ^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 622, in setup + await self.install(environment) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/gemini_cli.py", line 111, in install + await self.exec_as_root( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 579, in exec_as_root + return await self._exec( + ^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 543, in _exec + result = await environment.exec( + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1096, in exec + return await self._compose_exec( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1173, in _compose_exec + return await self._run_docker_compose_command( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 649, in _run_docker_compose_command + result = await self._collect_buffered_output( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 679, in _collect_buffered_output + stdout_bytes, stderr_bytes = await process.communicate(input=stdin_data) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 201, in communicate + stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 181, in _read_stream + output = await stream.read() + ^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 706, in read + block = await self.read(self._limit) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 713, in read + await self._wait_for_data('read') + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 545, in _wait_for_data + await self._waiter +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 1180, in _setup_agent + await asyncio.wait_for( + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 519, in wait_for + async with timeouts.timeout(timeout): + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/timeouts.py", line 115, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 351, in run + await self._prepare() + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 389, in _prepare + await self._setup_agent() + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 1185, in _setup_agent + raise AgentSetupTimeoutError( +harbor.trial.errors.AgentSetupTimeoutError: Agent setup timed out after 360.0 seconds + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/artifact_handler.py", line 330, in _download_artifact + await source_env.service_download_file( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1052, in service_download_file + await self.download_file(source_path, target_path) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1037, in download_file + await self._platform.download_file(source_path, target_path) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker_unix.py", line 185, in download_file + await self._env._run_docker_compose_command( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 656, in _run_docker_compose_command + raise RuntimeError( +RuntimeError: Docker compose command failed for environment range-large-treatment-probe-v5. Command: docker compose --project-name range-large-v5__zxfatdf__env --project-directory /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5/environment -f /tmp/tmptnzk7inw/range-large-v5__ZXfAtDF__env-docker-compose-resources.json -f /home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker-compose-build.yaml -f /tmp/tmpaf91s5aa/docker-compose-environment.json -f /tmp/tmpeg59vbav/docker-compose-mounts.json cp main:.deltawire/state.json /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/state.json. Return code: 1. Stdout: range-large-v5__zxfatdf__env-main-1 Copying range-large-v5__zxfatdf__env-main-1:.deltawire/state.json to /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/state.json +Error response from daemon: Could not find the file .deltawire/state.json in container fce6c11ef728d481a6ea79955985457d970323cf6a85263d192f3aa3430f2061 +. Stderr: None. +Failed to download artifact '.deltawire/plan-contract-receipt.json' from service 'main' (best-effort) +Traceback (most recent call last): + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 520, in wait_for + return await fut + ^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 622, in setup + await self.install(environment) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/gemini_cli.py", line 111, in install + await self.exec_as_root( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 579, in exec_as_root + return await self._exec( + ^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 543, in _exec + result = await environment.exec( + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1096, in exec + return await self._compose_exec( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1173, in _compose_exec + return await self._run_docker_compose_command( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 649, in _run_docker_compose_command + result = await self._collect_buffered_output( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 679, in _collect_buffered_output + stdout_bytes, stderr_bytes = await process.communicate(input=stdin_data) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 201, in communicate + stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 181, in _read_stream + output = await stream.read() + ^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 706, in read + block = await self.read(self._limit) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 713, in read + await self._wait_for_data('read') + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 545, in _wait_for_data + await self._waiter +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 1180, in _setup_agent + await asyncio.wait_for( + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 519, in wait_for + async with timeouts.timeout(timeout): + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/timeouts.py", line 115, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 351, in run + await self._prepare() + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 389, in _prepare + await self._setup_agent() + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 1185, in _setup_agent + raise AgentSetupTimeoutError( +harbor.trial.errors.AgentSetupTimeoutError: Agent setup timed out after 360.0 seconds + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/artifact_handler.py", line 330, in _download_artifact + await source_env.service_download_file( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1052, in service_download_file + await self.download_file(source_path, target_path) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1037, in download_file + await self._platform.download_file(source_path, target_path) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker_unix.py", line 185, in download_file + await self._env._run_docker_compose_command( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 656, in _run_docker_compose_command + raise RuntimeError( +RuntimeError: Docker compose command failed for environment range-large-treatment-probe-v5. Command: docker compose --project-name range-large-v5__zxfatdf__env --project-directory /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5/environment -f /tmp/tmptnzk7inw/range-large-v5__ZXfAtDF__env-docker-compose-resources.json -f /home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker-compose-build.yaml -f /tmp/tmpaf91s5aa/docker-compose-environment.json -f /tmp/tmpeg59vbav/docker-compose-mounts.json cp main:.deltawire/plan-contract-receipt.json /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/plan-contract-receipt.json. Return code: 1. Stdout: range-large-v5__zxfatdf__env-main-1 Copying range-large-v5__zxfatdf__env-main-1:.deltawire/plan-contract-receipt.json to /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/plan-contract-receipt.json +Error response from daemon: Could not find the file .deltawire/plan-contract-receipt.json in container fce6c11ef728d481a6ea79955985457d970323cf6a85263d192f3aa3430f2061 +. Stderr: None. +Not retrying trial because the maximum number of retries has been reached diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/lock.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/lock.json new file mode 100644 index 000000000..84990fb3b --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/lock.json @@ -0,0 +1,70 @@ +{ + "schema_version": 2, + "created_at": "2026-07-22T15:21:42.618655Z", + "harbor": { + "version": "0.20.0", + "is_editable": false + }, + "n_concurrent_trials": 1, + "retry": { + "max_retries": 0, + "exclude_exceptions": [ + "VerifierOutputParseError", + "AgentSafetyRefusalError", + "ModelNotFoundError", + "AgentTimeoutError", + "RewardFileEmptyError", + "AgentAuthenticationError", + "VerifierTimeoutError", + "RewardFileNotFoundError", + "ApiUsageLimitError" + ], + "wait_multiplier": 1.0, + "min_wait_sec": 1.0, + "max_wait_sec": 60.0 + }, + "trials": [ + { + "schema_version": 1, + "task": { + "name": "range-large-v5", + "type": "local", + "digest": "sha256:85ab72f764495839fd4e7ef9f2cb9c07f624e9855d39b1cba21864b0d1b8df75", + "path": "/home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5" + }, + "install_only": false, + "timeout_multiplier": 1.0, + "agent": { + "name": "gemini-cli", + "model_name": "google/gemini-3.1-pro-preview", + "skills": [ + "/home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5/skill/deltawire" + ], + "resume_trajectory": false, + "extra_allowed_hosts": [], + "kwargs": {}, + "mcp_servers": [] + }, + "skills": [ + { + "name": "deltawire", + "source": "/home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5/skill/deltawire", + "digest": "sha256:c6179e8efc6e54e484ac7350a83b5893aeb9b5651ef6d37e917568681870a384" + } + ], + "environment": { + "type": "docker", + "force_build": false, + "delete": true, + "cpu_enforcement_policy": "auto", + "memory_enforcement_policy": "auto", + "extra_docker_compose": [], + "kwargs": {}, + "extra_allowed_hosts": [] + }, + "verifier": { + "disable": false + } + } + ] +} \ No newline at end of file diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/config.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/config.json new file mode 100644 index 000000000..61a05415a --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/config.json @@ -0,0 +1,12 @@ +{ + "limits": { + "max_output_bytes": 104857600, + "max_plan_bytes": 1048576, + "max_records": 100000, + "max_schema_bytes": 1048576 + }, + "plans_dir": ".deltawire/plans", + "schemas_dir": ".deltawire/schemas", + "state_file": ".deltawire/state.json", + "version": "deltawire.config.v1" +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/schemas/range-large.schema.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/schemas/range-large.schema.json new file mode 100644 index 000000000..69b476237 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/schemas/range-large.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "index": { + "maximum": 500, + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "index" + ], + "title": "range-large", + "type": "object" +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/logs/artifacts/deltawire/environment-receipt.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/logs/artifacts/deltawire/environment-receipt.json new file mode 100644 index 000000000..55108c6ac --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/logs/artifacts/deltawire/environment-receipt.json @@ -0,0 +1,55 @@ +{ + "binary": { + "actual_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "executable": true, + "exists": true, + "expected_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "expected_version": "deltawire version dev", + "is_regular": true, + "is_symlink": false, + "path": "/usr/local/bin/deltawire", + "realpath": "/usr/local/bin/deltawire", + "version_exit_code": 0, + "version_stderr": "", + "version_stdout": "deltawire version dev\nGo runtime version go1.26.5\nsupported config version deltawire.config.v1\nsupported plan version deltawire.plan.v1\nsupported state version deltawire.state.v1" + }, + "checks": { + "binary_executable": true, + "binary_exists": true, + "binary_hash": true, + "binary_not_symlink": true, + "binary_path": true, + "binary_realpath": true, + "binary_regular": true, + "config_hash": true, + "public_contract_hash": true, + "schema_hash": true, + "version_exact": true, + "version_exit_0": true + }, + "files": { + "config": { + "actual_sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "exists": true, + "expected_sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "hash_match": true, + "path": "/.deltawire/config.json" + }, + "public_contract": { + "actual_sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "exists": true, + "expected_sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "hash_match": true, + "path": "/task-contract.json" + }, + "schema": { + "actual_sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "exists": true, + "expected_sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "hash_match": true, + "path": "/.deltawire/schemas/range-large.schema.json" + } + }, + "schema_version": "deltawire-environment-receipt.v1", + "status": "pass" +} diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/manifest.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/manifest.json new file mode 100644 index 000000000..b99a797f2 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/manifest.json @@ -0,0 +1,51 @@ +[ + { + "source": "/logs/artifacts", + "destination": "artifacts/logs/artifacts", + "type": "directory", + "status": "ok", + "service": null + }, + { + "source": "testdata/generated/output.ndjson", + "destination": "artifacts/testdata/generated/output.ndjson", + "type": "file", + "status": "failed", + "service": null + }, + { + "source": ".deltawire/config.json", + "destination": "artifacts/.deltawire/config.json", + "type": "file", + "status": "ok", + "service": null + }, + { + "source": ".deltawire/plans/range-large.dw.json", + "destination": "artifacts/.deltawire/plans/range-large.dw.json", + "type": "file", + "status": "failed", + "service": null + }, + { + "source": ".deltawire/schemas/range-large.schema.json", + "destination": "artifacts/.deltawire/schemas/range-large.schema.json", + "type": "file", + "status": "ok", + "service": null + }, + { + "source": ".deltawire/state.json", + "destination": "artifacts/.deltawire/state.json", + "type": "file", + "status": "failed", + "service": null + }, + { + "source": ".deltawire/plan-contract-receipt.json", + "destination": "artifacts/.deltawire/plan-contract-receipt.json", + "type": "file", + "status": "failed", + "service": null + } +] \ No newline at end of file diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/config.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/config.json new file mode 100644 index 000000000..b2f4c30a2 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/config.json @@ -0,0 +1,15 @@ +{ + "task": { + "path": "/home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5" + }, + "trial_name": "range-large-v5__ZXfAtDF", + "trials_dir": "/home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42", + "agent": { + "name": "gemini-cli", + "model_name": "google/gemini-3.1-pro-preview", + "skills": [ + "/home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5/skill/deltawire" + ] + }, + "job_id": "9ff1bab1-7a2d-4d73-99f7-3ce3d5d2a92d" +} \ No newline at end of file diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/exception.txt b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/exception.txt new file mode 100644 index 000000000..1120a7e98 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/exception.txt @@ -0,0 +1,63 @@ +Traceback (most recent call last): + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 520, in wait_for + return await fut + ^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 622, in setup + await self.install(environment) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/gemini_cli.py", line 111, in install + await self.exec_as_root( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 579, in exec_as_root + return await self._exec( + ^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 543, in _exec + result = await environment.exec( + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1096, in exec + return await self._compose_exec( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1173, in _compose_exec + return await self._run_docker_compose_command( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 649, in _run_docker_compose_command + result = await self._collect_buffered_output( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 679, in _collect_buffered_output + stdout_bytes, stderr_bytes = await process.communicate(input=stdin_data) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 201, in communicate + stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 181, in _read_stream + output = await stream.read() + ^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 706, in read + block = await self.read(self._limit) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 713, in read + await self._wait_for_data('read') + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 545, in _wait_for_data + await self._waiter +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 1180, in _setup_agent + await asyncio.wait_for( + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 519, in wait_for + async with timeouts.timeout(timeout): + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/timeouts.py", line 115, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 351, in run + await self._prepare() + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 389, in _prepare + await self._setup_agent() + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 1185, in _setup_agent + raise AgentSetupTimeoutError( +harbor.trial.errors.AgentSetupTimeoutError: Agent setup timed out after 360.0 seconds diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/lock.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/lock.json new file mode 100644 index 000000000..219d26293 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/lock.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "task": { + "name": "range-large-v5", + "type": "local", + "digest": "sha256:85ab72f764495839fd4e7ef9f2cb9c07f624e9855d39b1cba21864b0d1b8df75", + "path": "/home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5" + }, + "install_only": false, + "timeout_multiplier": 1.0, + "agent": { + "name": "gemini-cli", + "model_name": "google/gemini-3.1-pro-preview", + "skills": [ + "/home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5/skill/deltawire" + ], + "resume_trajectory": false, + "extra_allowed_hosts": [], + "kwargs": {}, + "mcp_servers": [] + }, + "skills": [ + { + "name": "deltawire", + "source": "/home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5/skill/deltawire", + "digest": "sha256:c6179e8efc6e54e484ac7350a83b5893aeb9b5651ef6d37e917568681870a384" + } + ], + "environment": { + "type": "docker", + "force_build": false, + "delete": true, + "cpu_enforcement_policy": "auto", + "memory_enforcement_policy": "auto", + "extra_docker_compose": [], + "kwargs": {}, + "extra_allowed_hosts": [] + }, + "verifier": { + "disable": false + } +} \ No newline at end of file diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/result.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/result.json new file mode 100644 index 000000000..b7a0d166e --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/result.json @@ -0,0 +1,103 @@ +{ + "id": "574138ea-c743-4a05-b34c-cd578d24f49f", + "task_name": "operatorstack/range-large-treatment-probe-v5", + "trial_name": "range-large-v5__ZXfAtDF", + "trial_uri": "file:///home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF", + "task_id": { + "path": "/home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5" + }, + "source": null, + "task_checksum": "2720613495ea5be4ab6892ced57a4c3f38a8c738fc0dee667ea3b72d53246b37", + "config": { + "task": { + "path": "/home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "range-large-v5__ZXfAtDF", + "trials_dir": "/home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42", + "install_only": false, + "timeout_multiplier": 1.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": "gemini-cli", + "import_path": null, + "model_name": "google/gemini-3.1-pro-preview", + "n_concurrent": null, + "concurrency_group": null, + "skills": [ + "/home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5/skill/deltawire" + ], + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "resume_trajectory": false, + "load_trajectory": null, + "extra_allowed_hosts": [], + "kwargs": {}, + "mcp_servers": [] + }, + "environment": { + "type": "docker", + "import_path": null, + "force_build": false, + "delete": true, + "cpu_enforcement_policy": "auto", + "memory_enforcement_policy": "auto", + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "override_tpu": null, + "mounts": null, + "extra_docker_compose": [], + "kwargs": {}, + "extra_allowed_hosts": [] + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "disable": false + }, + "artifacts": [], + "extra_instruction_paths": [], + "job_id": "9ff1bab1-7a2d-4d73-99f7-3ce3d5d2a92d" + }, + "agent_info": { + "name": "gemini-cli", + "version": "unknown", + "model_info": { + "name": "gemini-3.1-pro-preview", + "provider": "google" + } + }, + "agent_result": null, + "verifier_result": null, + "exception_info": { + "exception_type": "AgentSetupTimeoutError", + "exception_message": "Agent setup timed out after 360.0 seconds", + "exception_traceback": "Traceback (most recent call last):\n File \"/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py\", line 520, in wait_for\n return await fut\n ^^^^^^^^^\n File \"/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py\", line 622, in setup\n await self.install(environment)\n File \"/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/gemini_cli.py\", line 111, in install\n await self.exec_as_root(\n File \"/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py\", line 579, in exec_as_root\n return await self._exec(\n ^^^^^^^^^^^^^^^^^\n File \"/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py\", line 543, in _exec\n result = await environment.exec(\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py\", line 1096, in exec\n return await self._compose_exec(\n ^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py\", line 1173, in _compose_exec\n return await self._run_docker_compose_command(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py\", line 649, in _run_docker_compose_command\n result = await self._collect_buffered_output(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py\", line 679, in _collect_buffered_output\n stdout_bytes, stderr_bytes = await process.communicate(input=stdin_data)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py\", line 201, in communicate\n stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py\", line 181, in _read_stream\n output = await stream.read()\n ^^^^^^^^^^^^^^^^^^^\n File \"/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py\", line 706, in read\n block = await self.read(self._limit)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py\", line 713, in read\n await self._wait_for_data('read')\n File \"/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py\", line 545, in _wait_for_data\n await self._waiter\nasyncio.exceptions.CancelledError\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py\", line 1180, in _setup_agent\n await asyncio.wait_for(\n File \"/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py\", line 519, in wait_for\n async with timeouts.timeout(timeout):\n ^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/timeouts.py\", line 115, in __aexit__\n raise TimeoutError from exc_val\nTimeoutError\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py\", line 351, in run\n await self._prepare()\n File \"/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py\", line 389, in _prepare\n await self._setup_agent()\n File \"/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py\", line 1185, in _setup_agent\n raise AgentSetupTimeoutError(\nharbor.trial.errors.AgentSetupTimeoutError: Agent setup timed out after 360.0 seconds\n", + "occurred_at": "2026-07-22T15:27:46.612185" + }, + "started_at": "2026-07-22T15:21:42.677813Z", + "finished_at": "2026-07-22T15:27:59.246705Z", + "environment_setup": { + "started_at": "2026-07-22T15:21:42.708769Z", + "finished_at": "2026-07-22T15:21:46.287499Z" + }, + "agent_setup": { + "started_at": "2026-07-22T15:21:46.588563Z", + "finished_at": "2026-07-22T15:27:46.607178Z" + }, + "agent_execution": null, + "verifier": null, + "step_results": null +} \ No newline at end of file diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/trial.log b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/trial.log new file mode 100644 index 000000000..6d581c398 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/trial.log @@ -0,0 +1,330 @@ +Skipping image OS validation for hb__e298605c2a7fe495353069ca6f1b234d: docker inspect returned 1 +Running command: apt-get update && apt-get install -y curl +Trial range-large-v5__ZXfAtDF failed: Agent setup timed out after 360.0 seconds +Collecting main service artifacts +Running collect hook in service 'main': 'python3 /usr/local/bin/environment_receipt.py --expectations /deltawire-environment-expectations.json --receipt /logs/artifacts/deltawire/environment-receipt.json' +Collect hook in service 'main' completed +Failed to download artifact 'testdata/generated/output.ndjson' from service 'main' (best-effort) +Traceback (most recent call last): + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 520, in wait_for + return await fut + ^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 622, in setup + await self.install(environment) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/gemini_cli.py", line 111, in install + await self.exec_as_root( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 579, in exec_as_root + return await self._exec( + ^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 543, in _exec + result = await environment.exec( + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1096, in exec + return await self._compose_exec( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1173, in _compose_exec + return await self._run_docker_compose_command( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 649, in _run_docker_compose_command + result = await self._collect_buffered_output( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 679, in _collect_buffered_output + stdout_bytes, stderr_bytes = await process.communicate(input=stdin_data) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 201, in communicate + stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 181, in _read_stream + output = await stream.read() + ^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 706, in read + block = await self.read(self._limit) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 713, in read + await self._wait_for_data('read') + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 545, in _wait_for_data + await self._waiter +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 1180, in _setup_agent + await asyncio.wait_for( + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 519, in wait_for + async with timeouts.timeout(timeout): + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/timeouts.py", line 115, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 351, in run + await self._prepare() + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 389, in _prepare + await self._setup_agent() + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 1185, in _setup_agent + raise AgentSetupTimeoutError( +harbor.trial.errors.AgentSetupTimeoutError: Agent setup timed out after 360.0 seconds + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/artifact_handler.py", line 330, in _download_artifact + await source_env.service_download_file( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1052, in service_download_file + await self.download_file(source_path, target_path) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1037, in download_file + await self._platform.download_file(source_path, target_path) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker_unix.py", line 185, in download_file + await self._env._run_docker_compose_command( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 656, in _run_docker_compose_command + raise RuntimeError( +RuntimeError: Docker compose command failed for environment range-large-treatment-probe-v5. Command: docker compose --project-name range-large-v5__zxfatdf__env --project-directory /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5/environment -f /tmp/tmptnzk7inw/range-large-v5__ZXfAtDF__env-docker-compose-resources.json -f /home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker-compose-build.yaml -f /tmp/tmpaf91s5aa/docker-compose-environment.json -f /tmp/tmpeg59vbav/docker-compose-mounts.json cp main:testdata/generated/output.ndjson /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/testdata/generated/output.ndjson. Return code: 1. Stdout: range-large-v5__zxfatdf__env-main-1 Copying range-large-v5__zxfatdf__env-main-1:testdata/generated/output.ndjson to /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/testdata/generated/output.ndjson +Error response from daemon: Could not find the file testdata/generated/output.ndjson in container fce6c11ef728d481a6ea79955985457d970323cf6a85263d192f3aa3430f2061 +. Stderr: None. +Failed to download artifact '.deltawire/plans/range-large.dw.json' from service 'main' (best-effort) +Traceback (most recent call last): + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 520, in wait_for + return await fut + ^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 622, in setup + await self.install(environment) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/gemini_cli.py", line 111, in install + await self.exec_as_root( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 579, in exec_as_root + return await self._exec( + ^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 543, in _exec + result = await environment.exec( + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1096, in exec + return await self._compose_exec( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1173, in _compose_exec + return await self._run_docker_compose_command( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 649, in _run_docker_compose_command + result = await self._collect_buffered_output( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 679, in _collect_buffered_output + stdout_bytes, stderr_bytes = await process.communicate(input=stdin_data) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 201, in communicate + stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 181, in _read_stream + output = await stream.read() + ^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 706, in read + block = await self.read(self._limit) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 713, in read + await self._wait_for_data('read') + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 545, in _wait_for_data + await self._waiter +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 1180, in _setup_agent + await asyncio.wait_for( + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 519, in wait_for + async with timeouts.timeout(timeout): + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/timeouts.py", line 115, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 351, in run + await self._prepare() + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 389, in _prepare + await self._setup_agent() + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 1185, in _setup_agent + raise AgentSetupTimeoutError( +harbor.trial.errors.AgentSetupTimeoutError: Agent setup timed out after 360.0 seconds + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/artifact_handler.py", line 330, in _download_artifact + await source_env.service_download_file( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1052, in service_download_file + await self.download_file(source_path, target_path) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1037, in download_file + await self._platform.download_file(source_path, target_path) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker_unix.py", line 185, in download_file + await self._env._run_docker_compose_command( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 656, in _run_docker_compose_command + raise RuntimeError( +RuntimeError: Docker compose command failed for environment range-large-treatment-probe-v5. Command: docker compose --project-name range-large-v5__zxfatdf__env --project-directory /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5/environment -f /tmp/tmptnzk7inw/range-large-v5__ZXfAtDF__env-docker-compose-resources.json -f /home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker-compose-build.yaml -f /tmp/tmpaf91s5aa/docker-compose-environment.json -f /tmp/tmpeg59vbav/docker-compose-mounts.json cp main:.deltawire/plans/range-large.dw.json /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/plans/range-large.dw.json. Return code: 1. Stdout: range-large-v5__zxfatdf__env-main-1 Copying range-large-v5__zxfatdf__env-main-1:.deltawire/plans/range-large.dw.json to /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/plans/range-large.dw.json +Error response from daemon: Could not find the file .deltawire/plans/range-large.dw.json in container fce6c11ef728d481a6ea79955985457d970323cf6a85263d192f3aa3430f2061 +. Stderr: None. +Failed to download artifact '.deltawire/state.json' from service 'main' (best-effort) +Traceback (most recent call last): + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 520, in wait_for + return await fut + ^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 622, in setup + await self.install(environment) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/gemini_cli.py", line 111, in install + await self.exec_as_root( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 579, in exec_as_root + return await self._exec( + ^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 543, in _exec + result = await environment.exec( + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1096, in exec + return await self._compose_exec( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1173, in _compose_exec + return await self._run_docker_compose_command( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 649, in _run_docker_compose_command + result = await self._collect_buffered_output( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 679, in _collect_buffered_output + stdout_bytes, stderr_bytes = await process.communicate(input=stdin_data) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 201, in communicate + stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 181, in _read_stream + output = await stream.read() + ^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 706, in read + block = await self.read(self._limit) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 713, in read + await self._wait_for_data('read') + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 545, in _wait_for_data + await self._waiter +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 1180, in _setup_agent + await asyncio.wait_for( + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 519, in wait_for + async with timeouts.timeout(timeout): + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/timeouts.py", line 115, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 351, in run + await self._prepare() + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 389, in _prepare + await self._setup_agent() + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 1185, in _setup_agent + raise AgentSetupTimeoutError( +harbor.trial.errors.AgentSetupTimeoutError: Agent setup timed out after 360.0 seconds + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/artifact_handler.py", line 330, in _download_artifact + await source_env.service_download_file( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1052, in service_download_file + await self.download_file(source_path, target_path) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1037, in download_file + await self._platform.download_file(source_path, target_path) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker_unix.py", line 185, in download_file + await self._env._run_docker_compose_command( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 656, in _run_docker_compose_command + raise RuntimeError( +RuntimeError: Docker compose command failed for environment range-large-treatment-probe-v5. Command: docker compose --project-name range-large-v5__zxfatdf__env --project-directory /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5/environment -f /tmp/tmptnzk7inw/range-large-v5__ZXfAtDF__env-docker-compose-resources.json -f /home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker-compose-build.yaml -f /tmp/tmpaf91s5aa/docker-compose-environment.json -f /tmp/tmpeg59vbav/docker-compose-mounts.json cp main:.deltawire/state.json /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/state.json. Return code: 1. Stdout: range-large-v5__zxfatdf__env-main-1 Copying range-large-v5__zxfatdf__env-main-1:.deltawire/state.json to /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/state.json +Error response from daemon: Could not find the file .deltawire/state.json in container fce6c11ef728d481a6ea79955985457d970323cf6a85263d192f3aa3430f2061 +. Stderr: None. +Failed to download artifact '.deltawire/plan-contract-receipt.json' from service 'main' (best-effort) +Traceback (most recent call last): + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 520, in wait_for + return await fut + ^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 622, in setup + await self.install(environment) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/gemini_cli.py", line 111, in install + await self.exec_as_root( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 579, in exec_as_root + return await self._exec( + ^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/agents/installed/base.py", line 543, in _exec + result = await environment.exec( + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1096, in exec + return await self._compose_exec( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1173, in _compose_exec + return await self._run_docker_compose_command( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 649, in _run_docker_compose_command + result = await self._collect_buffered_output( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 679, in _collect_buffered_output + stdout_bytes, stderr_bytes = await process.communicate(input=stdin_data) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 201, in communicate + stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/subprocess.py", line 181, in _read_stream + output = await stream.read() + ^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 706, in read + block = await self.read(self._limit) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 713, in read + await self._wait_for_data('read') + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/streams.py", line 545, in _wait_for_data + await self._waiter +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 1180, in _setup_agent + await asyncio.wait_for( + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 519, in wait_for + async with timeouts.timeout(timeout): + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/apple/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/timeouts.py", line 115, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 351, in run + await self._prepare() + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 389, in _prepare + await self._setup_agent() + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/trial.py", line 1185, in _setup_agent + raise AgentSetupTimeoutError( +harbor.trial.errors.AgentSetupTimeoutError: Agent setup timed out after 360.0 seconds + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/trial/artifact_handler.py", line 330, in _download_artifact + await source_env.service_download_file( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1052, in service_download_file + await self.download_file(source_path, target_path) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 1037, in download_file + await self._platform.download_file(source_path, target_path) + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker_unix.py", line 185, in download_file + await self._env._run_docker_compose_command( + File "/home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker.py", line 656, in _run_docker_compose_command + raise RuntimeError( +RuntimeError: Docker compose command failed for environment range-large-treatment-probe-v5. Command: docker compose --project-name range-large-v5__zxfatdf__env --project-directory /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5/environment -f /tmp/tmptnzk7inw/range-large-v5__ZXfAtDF__env-docker-compose-resources.json -f /home/apple/.cache/uv/archive-v0/6JePpuJGzEtEwZmd/lib/python3.12/site-packages/harbor/environments/docker/docker-compose-build.yaml -f /tmp/tmpaf91s5aa/docker-compose-environment.json -f /tmp/tmpeg59vbav/docker-compose-mounts.json cp main:.deltawire/plan-contract-receipt.json /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/plan-contract-receipt.json. Return code: 1. Stdout: range-large-v5__zxfatdf__env-main-1 Copying range-large-v5__zxfatdf__env-main-1:.deltawire/plan-contract-receipt.json to /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/artifacts/.deltawire/plan-contract-receipt.json +Error response from daemon: Could not find the file .deltawire/plan-contract-receipt.json in container fce6c11ef728d481a6ea79955985457d970323cf6a85263d192f3aa3430f2061 +. Stderr: None. diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/result.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/result.json new file mode 100644 index 000000000..16d7f645f --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/result.json @@ -0,0 +1,37 @@ +{ + "id": "9ff1bab1-7a2d-4d73-99f7-3ce3d5d2a92d", + "started_at": "2026-07-22T15:21:42.606547", + "updated_at": "2026-07-22T15:27:59.249583", + "finished_at": "2026-07-22T15:27:59.249583", + "n_total_trials": 1, + "stats": { + "n_completed_trials": 1, + "n_errored_trials": 1, + "n_running_trials": 0, + "n_pending_trials": 0, + "n_cancelled_trials": 0, + "n_retries": 0, + "evals": { + "gemini-cli__gemini-3.1-pro-preview__adhoc": { + "n_trials": 0, + "n_errors": 1, + "metrics": [ + { + "mean": 0.0 + } + ], + "pass_at_k": {}, + "reward_stats": {}, + "exception_stats": { + "AgentSetupTimeoutError": [ + "range-large-v5__ZXfAtDF" + ] + } + } + }, + "n_input_tokens": null, + "n_cache_tokens": null, + "n_output_tokens": null, + "cost_usd": null + } +} \ No newline at end of file diff --git a/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/run-ledger.json b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/run-ledger.json new file mode 100644 index 000000000..37f996c40 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/run-ledger.json @@ -0,0 +1,26 @@ +{ + "mode": "probe-v5", + "ready_for_72": false, + "runs": [ + { + "arm": "D1", + "artifact_error": "missing artifacts: trajectory_path", + "cache_tokens": null, + "harbor_exit_code": 0, + "input_tokens": null, + "output_tokens": null, + "pair_id": "probe/range-large-v5", + "redacted_command": "uvx --from harbor==0.20.0 harbor run -a gemini-cli -m google/gemini-3.1-pro-preview -p /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5 -k 1 --jobs-dir /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1 --n-concurrent 1 --skill /home/apple/deltawire-preflight-v5/labs/20-deltawire/eval/probes/range-large-v5/skill/deltawire --env-file ''", + "repetition": 1, + "reported_model": "gemini-3.1-pro-preview", + "result_path": "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF/result.json", + "result_sha256": "b1c29f30ba5677e51f57db870e74e91f966a0e62736ab81f1b0151d57b952da6", + "status": "completed", + "task": "range-large-v5", + "trial_dir": "labs/20-deltawire/eval/results/preflight-v5/treatment-probe-range-large-v5/raw/D1/2026-07-22__15-21-42/range-large-v5__ZXfAtDF", + "trial_id": "574138ea-c743-4a05-b34c-cd578d24f49f", + "wall_time_seconds": 376.568892 + } + ], + "schema_version": "run-ledger.v5" +} diff --git a/labs/20-deltawire/eval/scripts/v5/artifact_manifest.py b/labs/20-deltawire/eval/scripts/v5/artifact_manifest.py new file mode 100644 index 000000000..1b8344ed7 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/artifact_manifest.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +import hashlib,json +from pathlib import Path + +SOURCE="/logs/artifacts" +DESTINATION="artifacts/logs/artifacts" +RECEIPT="logs/artifacts/deltawire/environment-receipt.json" +def sha(path):return hashlib.sha256(Path(path).read_bytes()).hexdigest() +def validate(trial,expected_binary_sha=None,expected_files=None): + trial=Path(trial);manifest_path=trial/"artifacts/manifest.json";receipt_path=trial/"artifacts"/RECEIPT + checks={"manifest_exists":manifest_path.is_file(),"receipt_canonical_path":receipt_path.is_file()} + manifest=[];receipt={} + try:manifest=json.loads(manifest_path.read_text()) if checks["manifest_exists"] else [] + except (OSError,json.JSONDecodeError):manifest=[] + entries=[e for e in manifest if isinstance(e,dict) and e.get("source")==SOURCE and e.get("destination")==DESTINATION] + checks["single_convention_entry"]=len(entries)==1 + checks["manifest_status_ok"]=len(entries)==1 and entries[0].get("status")=="ok" and entries[0].get("type")=="directory" + try:receipt=json.loads(receipt_path.read_text()) if checks["receipt_canonical_path"] else {} + except (OSError,json.JSONDecodeError):receipt={} + checks["receipt_schema_exact"]=receipt.get("schema_version")=="deltawire-environment-receipt.v1" + checks["receipt_status_pass"]=receipt.get("status")=="pass" + required={"binary_exists","binary_regular","binary_not_symlink","binary_executable","binary_path","binary_realpath","binary_hash","version_exit_0","version_exact","config_hash","schema_hash","public_contract_hash"} + receipt_checks=receipt.get("checks") if isinstance(receipt.get("checks"),dict) else {} + checks["receipt_checks_complete"]=set(receipt_checks)==required + checks["receipt_checks_true"]=set(receipt_checks)==required and all(receipt_checks.values()) + if expected_binary_sha is not None:checks["binary_hash_frozen"]=receipt.get("binary",{}).get("actual_sha256")==expected_binary_sha + for name,want in sorted((expected_files or {}).items()):checks[f"{name}_hash_frozen"]=receipt.get("files",{}).get(name,{}).get("actual_sha256")==want + return {"schema_version":"artifact-manifest-receipt.v1","status":"pass" if all(checks.values()) else "fail","checks":checks, + "artifact_manifest_sha256":sha(manifest_path) if manifest_path.is_file() else None, + "environment_receipt_sha256":sha(receipt_path) if receipt_path.is_file() else None, + "environment_receipt_path":str(receipt_path),"convention_entry":entries[0] if len(entries)==1 else None} diff --git a/labs/20-deltawire/eval/scripts/v5/build_and_stage.sh b/labs/20-deltawire/eval/scripts/v5/build_and_stage.sh new file mode 100755 index 000000000..2e50b2da9 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/build_and_stage.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/../../../../.." && pwd)" +LAB="$ROOT/labs/20-deltawire"; GENERATED="$LAB/eval/.generated"; mkdir -p "$GENERATED" +(cd "$LAB" && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOWORK=off go build -trimpath -buildvcs=false -o "$GENERATED/deltawire-linux-amd64" ./cmd/deltawire) +SHA="$(sha256sum "$GENERATED/deltawire-linux-amd64" | cut -d' ' -f1)" +for task in "$LAB/eval/probes/range-large-v5" "$LAB/eval/probes/range-large-v5/pair-task" "$LAB/eval/conformance/environment-receipt-v1"; do + mkdir -p "$task/environment/.generated"; cp "$GENERATED/deltawire-linux-amd64" "$task/environment/.generated/deltawire" + test "$(sha256sum "$task/environment/.generated/deltawire" | cut -d' ' -f1)" = "$SHA" +done +python3 "$LAB/eval/scripts/v5/prepare_tasks.py" --binary-sha "$SHA" +printf '%s\n' "$SHA" diff --git a/labs/20-deltawire/eval/scripts/v5/environment_receipt.py b/labs/20-deltawire/eval/scripts/v5/environment_receipt.py new file mode 100644 index 000000000..84446c2d5 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/environment_receipt.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Atomically emit deterministic DeltaWire environment evidence.""" +import argparse,hashlib,json,os,stat,subprocess,tempfile +from pathlib import Path + +def sha(path): + h=hashlib.sha256() + with Path(path).open("rb") as f: + for chunk in iter(lambda:f.read(1048576),b""):h.update(chunk) + return h.hexdigest() +def atomic_write(path,data): + target=Path(path);target.parent.mkdir(parents=True,exist_ok=True) + fd,tmp=tempfile.mkstemp(prefix=f".{target.name}.",suffix=".tmp",dir=target.parent) + try: + with os.fdopen(fd,"w",encoding="utf-8") as f:f.write(data);f.flush();os.fsync(f.fileno()) + os.replace(tmp,target) + except BaseException: + try:os.unlink(tmp) + except FileNotFoundError:pass + raise +def build(expectations): + expected=json.loads(Path(expectations).read_text());binary=Path(expected["binary"]["path"]) + exists=binary.exists();lst=binary.lstat() if exists else None + regular=bool(lst and stat.S_ISREG(lst.st_mode));symlink=binary.is_symlink() if exists else False + executable=exists and os.access(binary,os.X_OK);realpath=str(binary.resolve()) if exists else None + try:version=subprocess.run([str(binary),"version"],capture_output=True,text=True,check=False) if exists else None + except OSError:version=None + files={} + for name,item in sorted(expected["files"].items()): + path=Path(item["path"]);actual=sha(path) if path.is_file() else None + files[name]={"path":item["path"],"exists":path.is_file(),"actual_sha256":actual,"expected_sha256":item["sha256"],"hash_match":actual==item["sha256"]} + binary_hash=sha(binary) if regular else None + checks={"binary_exists":exists,"binary_regular":regular,"binary_not_symlink":not symlink,"binary_executable":executable, + "binary_path":str(binary)==expected["binary"]["path"],"binary_realpath":realpath==expected["binary"]["realpath"], + "binary_hash":binary_hash==expected["binary"]["sha256"],"version_exit_0":bool(version and version.returncode==0), + "version_exact":bool(version and version.stdout.splitlines() and version.stdout.splitlines()[0].strip()==expected["binary"]["version"]), + **{f"{name}_hash":item["hash_match"] for name,item in files.items()}} + return {"schema_version":"deltawire-environment-receipt.v1","binary":{"path":str(binary),"realpath":realpath,"exists":exists, + "is_regular":regular,"is_symlink":symlink,"executable":executable,"actual_sha256":binary_hash, + "expected_sha256":expected["binary"]["sha256"],"version_stdout":version.stdout.strip() if version else None, + "version_stderr":version.stderr.strip() if version else None,"version_exit_code":version.returncode if version else None, + "expected_version":expected["binary"]["version"]},"files":files,"checks":checks,"status":"pass" if all(checks.values()) else "fail"} +def main(): + p=argparse.ArgumentParser();p.add_argument("--expectations",required=True);p.add_argument("--receipt",required=True);a=p.parse_args() + receipt=build(a.expectations);atomic_write(a.receipt,json.dumps(receipt,indent=2,sort_keys=True)+"\n") + print(json.dumps(receipt,sort_keys=True));raise SystemExit(0 if receipt["status"]=="pass" else 1) +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v5/fidelity.py b/labs/20-deltawire/eval/scripts/v5/fidelity.py new file mode 100644 index 000000000..8d966791f --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/fidelity.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +import hashlib,json +from pathlib import Path +from shell_observation import calls,parse,invokes_deltawire + +def sha(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() +def ok_tool(call): return call.get("tool_status")=="success" +def build(trajectory,trial,arm,spec,semantic,independent_receipt): + trial=Path(trial); artifacts=trial/"artifacts"; structured=calls(trajectory); observations,lifecycle=parse(trajectory) + obs_by_command={o["command"]:o for o in observations} + discovery=any(c["name"] in {"run_shell_command","shell","execute"} and "gemini skills list" in str(c["args"].get("command") or "") and "deltawire" in c["output"].lower() and ok_tool(c) and obs_by_command.get(str(c["args"].get("command") or ""),{}).get("shell_exit_code") in (None,0) for c in structured) + activation=any(c["name"]=="activate_skill" and str(c["args"].get("name") or c["args"].get("skill_name") or "").lower()=="deltawire" and ok_tool(c) for c in structured) + contract_access=any(ok_tool(c) and ((c["name"] in {"read_file","read_many_files"} and "/task-contract.json" in json.dumps(c["args"])) or (c["name"] in {"run_shell_command","shell","execute"} and "/task-contract.json" in str(c["args"].get("command") or ""))) for c in structured) + by_name={name:[x for x in lifecycle if x["command"]==name] for name in ("validate","render","check")} + lifecycle_observed={name:bool(items) and any(item["shell_exit_code"]!=0 or item["shell_exit_code"]==0 for item in items) for name,items in by_name.items()} + # An explicit nonzero final attempt fails; null is resolved by retained independent evidence. + lifecycle_not_proven_failed={name:(items[-1]["shell_exit_code"]==0 if items and items[-1]["shell_exit_code"] is not None else bool(items)) for name,items in by_name.items()} + paths={ + "plan":artifacts/".deltawire/plans/range-large.dw.json","config":artifacts/".deltawire/config.json", + "schema":artifacts/".deltawire/schemas/range-large.schema.json","state":artifacts/".deltawire/state.json", + "output":artifacts/"testdata/generated/output.ndjson","in_container_contract":artifacts/".deltawire/plan-contract-receipt.json", + } + retained={name:path.is_file() for name,path in paths.items()} + in_container=retained["in_container_contract"] and json.loads(paths["in_container_contract"].read_text()).get("status")=="pass" + independent=Path(independent_receipt).is_file() and json.loads(Path(independent_receipt).read_text()).get("status")=="pass" + schema_integrity=retained["schema"] and sha(paths["schema"])==spec["authoritative_schema"]["sha256"] + deltawire_invocations=[obs for obs in observations if invokes_deltawire(obs)] + contamination=arm=="B0" and (activation or bool(deltawire_invocations) or retained["plan"] or retained["state"]) + checks={"structured_discovery":discovery,"structured_activation":activation,"public_contract_access":contract_access, + **{f"{name}_invoked":lifecycle_observed[name] for name in lifecycle_observed}, + **{f"{name}_not_proven_failed":lifecycle_not_proven_failed[name] for name in lifecycle_not_proven_failed}, + **{f"retained_{name}":value for name,value in retained.items()},"in_container_contract_pass":in_container, + "independent_contract_pass":independent,"retained_schema_integrity":schema_integrity,"semantic_exact_pass":semantic==1} + if arm=="D1": status="pass" if all(checks.values()) else "fail"; classification="used_successfully" if status=="pass" else "attempted_failed" + else: status="pass" if not contamination and semantic==1 else "fail"; classification="not_used" if not contamination else "contaminated" + return {"schema_version":"treatment-use-receipt.v5","arm":arm,"classification":classification,"status":status,"checks":checks,"shell_observations":observations,"lifecycle_invocations":lifecycle,"contamination":contamination,"artifact_hashes":{name:sha(path) for name,path in paths.items() if path.is_file()},"trajectory_sha256":sha(trajectory)} diff --git a/labs/20-deltawire/eval/scripts/v5/generate_manifest.py b/labs/20-deltawire/eval/scripts/v5/generate_manifest.py new file mode 100644 index 000000000..13e79441a --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/generate_manifest.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +import argparse,hashlib,json +from pathlib import Path + +ROOT=Path(__file__).resolve().parents[5];EVAL=ROOT/"labs/20-deltawire/eval";OUT=EVAL/"manifests/preflight-v5-range-large.json" +def sha(path):return hashlib.sha256(Path(path).read_bytes()).hexdigest() +def main(): + p=argparse.ArgumentParser();p.add_argument("--input-commit",required=True);p.add_argument("--binary-sha",required=True);a=p.parse_args() + report=EVAL/"results/preflight-v5/conformance/environment-receipt-v1.json" + if not report.is_file() or json.loads(report.read_text()).get("status")!="pass":raise SystemExit("passing no-model conformance report required") + paths=[EVAL/"matrices/72-run-matrix.json",EVAL/"tasks/range-large/task-spec.json", + EVAL/"results/preflight-v1/treatment-probe-range-large/v1-evidence-lock.json", + EVAL/"manifests/preflight-v2-evidence-lock.json",EVAL/"manifests/preflight-v3-evidence-lock.json", + EVAL/"manifests/preflight-v4-evidence-lock.json",EVAL/"docs/08-harbor-artifact-timing.md",report, + ROOT/".github/workflows/deltawire-preflight-v5.yml"] + for base in (EVAL/"scripts/v5",EVAL/"probes/range-large-v5",EVAL/"conformance/environment-receipt-v1"): + paths.extend(x for x in base.rglob("*") if x.is_file() and ".generated" not in x.parts and "__pycache__" not in x.parts and x.suffix!=".pyc") + expectation=json.loads((EVAL/"probes/range-large-v5/environment/deltawire-environment-expectations.json").read_text()) + if expectation["binary"]["sha256"]!=a.binary_sha:raise SystemExit("environment expectation binary hash mismatch") + value={"schema_version":"deltawire-preflight-v5-manifest.v1","repository_commit":a.input_commit,"scope":["probe/range-large-v5","range-large/r1"], + "matrix_positions":{"zero_based":[36,37],"order":["D1","B0"]},"deltawire_binary_sha256":a.binary_sha, + "model":"google/gemini-3.1-pro-preview","expected_reported_model":"gemini-3.1-pro-preview","agent":"gemini-cli", + "harbor_version":"0.20.0","harbor_package":"harbor==0.20.0","harbor_command":["uvx","--from","harbor==0.20.0","harbor"], + "input_hashes":{str(x.relative_to(ROOT)):sha(x) for x in sorted(set(paths))}, + "failure_policy":"One v5 D1 probe after explicit review; no rerun; one range-large/r1 pair only after probe-release.v3 and separate explicit review; full disabled.", + "ready_for_72":False} + OUT.write_text(json.dumps(value,indent=2,sort_keys=True)+"\n");print(OUT.relative_to(ROOT)) +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v5/generate_v4_evidence_lock.py b/labs/20-deltawire/eval/scripts/v5/generate_v4_evidence_lock.py new file mode 100644 index 000000000..95c871361 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/generate_v4_evidence_lock.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +"""Lock the complete tracked v4 record through immutable Git objects.""" +import hashlib,json,subprocess +from pathlib import Path + +ROOT=Path(__file__).resolve().parents[5] +EVAL="labs/20-deltawire/eval" +COMMIT="c200dc0eb3c4dff0e4732cc4b2daa1cd535703e3" +OUT=ROOT/EVAL/"manifests/preflight-v4-evidence-lock.json" + +def git(*args):return subprocess.check_output(["git",*args],cwd=ROOT) +def sha(data):return hashlib.sha256(data).hexdigest() +def main(): + names=git("ls-tree","-r","--name-only",COMMIT).decode().splitlines() + prefixes=(f"{EVAL}/probes/range-large-v4/",f"{EVAL}/scripts/v4/",f"{EVAL}/results/preflight-v4/") + selected={n for n in names if n.startswith(prefixes)} + manifest_name=f"{EVAL}/manifests/preflight-v4-range-large.json" + selected.add(manifest_name) + manifest=json.loads(git("show",f"{COMMIT}:{manifest_name}")) + selected.update(manifest["input_hashes"]) + selected={n for n in selected if n in names} + value={"schema_version":"evidence-lock.v1","probe_version":"v4","source_commit":COMMIT, + "source_tree":git("rev-parse",f"{COMMIT}^{{tree}}").decode().strip(), + "immutable_files":{n:sha(git("show",f"{COMMIT}:{n}")) for n in sorted(selected)}} + OUT.parent.mkdir(parents=True,exist_ok=True) + OUT.write_text(json.dumps(value,indent=2,sort_keys=True)+"\n") + print(f"locked {len(selected)} v4 files from {COMMIT}") +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v5/negative_controls.py b/labs/20-deltawire/eval/scripts/v5/negative_controls.py new file mode 100644 index 000000000..cf02696a5 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/negative_controls.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +import json,os,tempfile +from pathlib import Path +from artifact_manifest import validate +from environment_receipt import atomic_write + +REQUIRED=("binary_exists","binary_regular","binary_not_symlink","binary_executable","binary_path","binary_realpath","binary_hash","version_exit_0","version_exact","config_hash","schema_hash","public_contract_hash") +def write(path,value):path.parent.mkdir(parents=True,exist_ok=True);path.write_text(json.dumps(value)+"\n") +def fixture(root): + trial=root/"trial";manifest=trial/"artifacts/manifest.json";receipt=trial/"artifacts/logs/artifacts/deltawire/environment-receipt.json" + value={"schema_version":"deltawire-environment-receipt.v1","status":"pass","checks":{x:True for x in REQUIRED},"binary":{"actual_sha256":"binary"},"files":{x:{"actual_sha256":x} for x in ("config","schema","public_contract")}} + write(receipt,value);write(manifest,[{"source":"/logs/artifacts","destination":"artifacts/logs/artifacts","type":"directory","status":"ok","service":None}]);return trial,manifest,receipt +def rejected(trial):return validate(trial,"binary",{x:x for x in ("config","schema","public_contract")})["status"]=="fail" +def run_negative_controls(): + results={} + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp) + trial,manifest,receipt=fixture(root);receipt.unlink();results["missing_collect_hook"]=rejected(trial) + trial,manifest,receipt=fixture(root);value=json.loads(receipt.read_text());value["status"]="fail";value["checks"]["binary_hash"]=False;write(receipt,value);results["collect_hook_nonzero"]=rejected(trial) + trial,manifest,receipt=fixture(root);late=trial/"artifacts/.deltawire/environment-receipt.json";late.parent.mkdir(parents=True,exist_ok=True);late.write_bytes(receipt.read_bytes());receipt.unlink();results["verifier_only_receipt"]=rejected(trial) + for key,field in (("wrong_binary_hash",("binary","actual_sha256")),("wrong_schema_hash",("files","schema","actual_sha256")),("wrong_contract_hash",("files","public_contract","actual_sha256"))): + trial,manifest,receipt=fixture(root);value=json.loads(receipt.read_text());target=value + for part in field[:-1]:target=target[part] + target[field[-1]]="wrong";write(receipt,value);results[key]=rejected(trial) + trial,manifest,receipt=fixture(root);value=json.loads(receipt.read_text());value["checks"]["version_exact"]=False;value["status"]="fail";write(receipt,value);results["wrong_version"]=rejected(trial) + for name,check in (("missing_binary","binary_exists"),("binary_symlink","binary_not_symlink"),("binary_non_executable","binary_executable"),("version_nonzero","version_exit_0")): + trial,manifest,receipt=fixture(root);value=json.loads(receipt.read_text());value["checks"][check]=False;value["status"]="fail";write(receipt,value);results[name]=rejected(trial) + trial,manifest,receipt=fixture(root);value=json.loads(receipt.read_text());value["checks"]["config_hash"]=False;value["status"]="fail";write(receipt,value);results["missing_config"]=rejected(trial) + trial,manifest,receipt=fixture(root);receipt.write_text("{");results["malformed_receipt"]=rejected(trial) + trial,manifest,receipt=fixture(root);write(manifest,[{"source":"/logs/artifacts","destination":"artifacts/logs/artifacts","type":"directory","status":"skipped"}]);results["manifest_missing_or_skipped"]=rejected(trial) + trial,manifest,receipt=fixture(root);wrong=trial/"artifacts/environment-receipt.json";wrong.write_bytes(receipt.read_bytes());receipt.unlink();results["wrong_receipt_path"]=rejected(trial) + trial,manifest,receipt=fixture(root);value=json.loads(receipt.read_text());value["binary"]["actual_sha256"]="v4";write(receipt,value);results["copied_v4_receipt"]=rejected(trial) + target=root/"atomic";target.mkdir() + try:atomic_write(target,"{}\n");results["atomic_write_failure"]=False + except OSError:results["atomic_write_failure"]=target.is_dir() and not list(root.glob(".atomic.*.tmp")) + return results +if __name__=="__main__": + value=run_negative_controls();print(json.dumps(value,sort_keys=True));raise SystemExit(0 if all(value.values()) else 1) diff --git a/labs/20-deltawire/eval/scripts/v5/prepare_tasks.py b/labs/20-deltawire/eval/scripts/v5/prepare_tasks.py new file mode 100644 index 000000000..afc7fd688 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/prepare_tasks.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +import argparse,hashlib,json,shutil +from pathlib import Path + +ROOT=Path(__file__).resolve().parents[5];EVAL=ROOT/"labs/20-deltawire/eval";HERE=Path(__file__).resolve().parent +PROBE=EVAL/"probes/range-large-v5";PAIR=PROBE/"pair-task";CONF=EVAL/"conformance/environment-receipt-v1";CANON=EVAL/"tasks/range-large" +def sha(path):return hashlib.sha256(Path(path).read_bytes()).hexdigest() +def main(): + p=argparse.ArgumentParser();p.add_argument("--binary-sha",required=True);a=p.parse_args() + for task in (PROBE,PAIR): + for name in ("task-spec.json","authoritative-schema.json"):shutil.copyfile(CANON/name,task/name) + shutil.copyfile(CANON/"task-spec.json",task/"environment/task-contract.json") + shutil.copyfile(CANON/"task-spec.json",task/"tests/task-spec.json") + shutil.copyfile(CANON/"authoritative-schema.json",task/"environment/.deltawire/schemas/range-large.schema.json") + shutil.copyfile(HERE/"environment_receipt.py",task/"environment/environment_receipt.py") + shutil.copyfile(HERE/"verify_plan_contract.py",task/"environment/verify_plan_contract.py") + source=PROBE + for path in ("task-contract.json",".deltawire/config.json",".deltawire/schemas/range-large.schema.json","environment_receipt.py","verify_plan_contract.py"): + target=CONF/"environment"/path;target.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(source/"environment"/path,target) + docker='''FROM ubuntu:22.04 +COPY .generated/deltawire /usr/local/bin/deltawire +COPY task-contract.json /task-contract.json +COPY .deltawire /.deltawire +COPY verify_plan_contract.py /usr/local/bin/verify_plan_contract.py +COPY environment_receipt.py /usr/local/bin/environment_receipt.py +COPY deltawire-environment-expectations.json /deltawire-environment-expectations.json +RUN chmod 0755 /usr/local/bin/deltawire /usr/local/bin/verify_plan_contract.py /usr/local/bin/environment_receipt.py && chmod a-w /task-contract.json /.deltawire/config.json /.deltawire/schemas/*.json /deltawire-environment-expectations.json +RUN command -v deltawire && deltawire version +RUN apt-get update && apt-get install -y python3 +''' + expectations={"schema_version":"deltawire-environment-expectations.v1","binary":{"path":"/usr/local/bin/deltawire","realpath":"/usr/local/bin/deltawire","sha256":a.binary_sha,"version":"deltawire version dev"},"files":{ + "config":{"path":"/.deltawire/config.json","sha256":sha(PROBE/"environment/.deltawire/config.json")}, + "schema":{"path":"/.deltawire/schemas/range-large.schema.json","sha256":sha(PROBE/"environment/.deltawire/schemas/range-large.schema.json")}, + "public_contract":{"path":"/task-contract.json","sha256":sha(PROBE/"environment/task-contract.json")}}} + for task in (PROBE,PAIR,CONF): + (task/"environment/Dockerfile").write_text(docker) + (task/"environment/deltawire-environment-expectations.json").write_text(json.dumps(expectations,indent=2,sort_keys=True)+"\n") +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v5/run_conformance.py b/labs/20-deltawire/eval/scripts/v5/run_conformance.py new file mode 100644 index 000000000..48355b798 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/run_conformance.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Run exactly two no-model Harbor collect-hook conformance trials.""" +import argparse,hashlib,json,shlex,subprocess +from pathlib import Path +from artifact_manifest import validate +from negative_controls import run_negative_controls + +ROOT=Path(__file__).resolve().parents[5];EVAL=ROOT/"labs/20-deltawire/eval";TASK=EVAL/"conformance/environment-receipt-v1" +HARBOR=["uvx","--from","harbor==0.20.0","harbor"] +def sha(path):return hashlib.sha256(Path(path).read_bytes()).hexdigest() +def trials(root):return sorted({p.parent.parent for p in Path(root).rglob("artifacts/manifest.json")}) +def main(): + p=argparse.ArgumentParser();p.add_argument("--jobs-dir",required=True);p.add_argument("--report",required=True);a=p.parse_args() + jobs=Path(a.jobs_dir).resolve();report=Path(a.report).resolve() + if jobs.exists() and any(jobs.iterdir()):raise SystemExit("conformance jobs directory must be empty") + jobs.mkdir(parents=True,exist_ok=True) + version=subprocess.check_output(HARBOR+["--version"],text=True).strip() + if version!="0.20.0":raise SystemExit(f"Harbor version mismatch: {version}") + expected=json.loads((TASK/"environment/deltawire-environment-expectations.json").read_text()) + expected_files={name:item["sha256"] for name,item in expected["files"].items()} + records=[] + for number in (1,2): + target=jobs/f"positive-{number}";before=set(trials(target)) + cmd=HARBOR+["run","-a","nop","-p",str(TASK),"-k","1","--jobs-dir",str(target),"--n-concurrent","1"] + run=subprocess.run(cmd,cwd=ROOT,check=False);created=set(trials(target))-before + if run.returncode or len(created)!=1:raise SystemExit(f"positive conformance {number} failed: rc={run.returncode}, trials={len(created)}") + trial=created.pop();receipt=validate(trial,expected["binary"]["sha256"],expected_files) + if receipt["status"]!="pass":raise SystemExit(f"positive conformance {number} artifact gate failed") + result_files=sorted(trial.glob("result.json"));result=json.loads(result_files[0].read_text()) if result_files else {} + logical=f"positive-{number}/{trial.parent.name}/{trial.name}" + display_cmd=HARBOR+["run","-a","nop","-p",str(TASK.relative_to(ROOT)),"-k","1","--jobs-dir",f"/positive-{number}","--n-concurrent","1"] + records.append({"trial":number,"trial_path":logical,"command":shlex.join(display_cmd),"harbor_exit_code":run.returncode, + "artifact_manifest_entry":receipt["convention_entry"],"environment_receipt_path":receipt["environment_receipt_path"], + "environment_receipt_sha256":receipt["environment_receipt_sha256"],"artifact_manifest_sha256":receipt["artifact_manifest_sha256"], + "artifact_gate_status":receipt["status"],"agent":"nop","model":None,"reported_agent":(result.get("agent_info") or {}).get("name")}) + records[-1]["environment_receipt_path"]=logical+"/artifacts/logs/artifacts/deltawire/environment-receipt.json" + hashes={x["environment_receipt_sha256"] for x in records} + if len(hashes)!=1:raise SystemExit("positive conformance receipt hashes differ") + negatives=run_negative_controls() + value={"schema_version":"deltawire-no-model-conformance.v1","status":"pass","harbor_version":version,"agent":"nop","model":None, + "zero_model_calls":all(" -m " not in f" {r['command']} " and "--env-file" not in r["command"] for r in records), + "binary_sha256":expected["binary"]["sha256"],"expectation_hashes":{name:item["sha256"] for name,item in expected["files"].items()}, + "positive_trials":records,"stable_receipt_sha256":records[0]["environment_receipt_sha256"],"negative_controls":negatives, + "ready_for_72":False} + if not value["zero_model_calls"] or not all(negatives.values()):raise SystemExit("conformance proof failed") + report.parent.mkdir(parents=True,exist_ok=True);report.write_text(json.dumps(value,indent=2,sort_keys=True)+"\n") + print(report) +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v5/runner.py b/labs/20-deltawire/eval/scripts/v5/runner.py new file mode 100644 index 000000000..6e2744d43 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/runner.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""One-shot DeltaWire v5 measurement-recovery runner.""" +import argparse,hashlib,json,os,shlex,subprocess,sys,tempfile +from datetime import datetime +from pathlib import Path +from fidelity import build,sha +from artifact_manifest import validate as validate_artifacts + +ROOT=Path(__file__).resolve().parents[5]; EVAL=ROOT/"labs/20-deltawire/eval"; HERE=Path(__file__).resolve().parent +PROBE=EVAL/"probes/range-large-v5"; PAIR_TASK=PROBE/"pair-task"; SKILL=PROBE/"skill/deltawire" +RESULTS=EVAL/"results/preflight-v5"; PROBE_ROOT=RESULTS/"treatment-probe-range-large-v5"; PAIR_ROOT=RESULTS/"range-large-r1" +MANIFEST=EVAL/"manifests/preflight-v5-range-large.json"; BINARY=EVAL/".generated/deltawire-linux-amd64" + +def load(p):return json.loads(Path(p).read_text(encoding="utf-8")) +def write(p,v):p=Path(p);p.parent.mkdir(parents=True,exist_ok=True);p.write_text(json.dumps(v,indent=2,sort_keys=True)+"\n") +def rel(p):return str(Path(p).resolve().relative_to(ROOT.resolve())) +def refuse(root,label): + if root.exists() and any(root.iterdir()):raise RuntimeError(f"{label} result root is nonempty; rerun forbidden") +def verify_locks():subprocess.run(["python3",str(HERE/"verify_evidence_locks.py")],cwd=ROOT,check=True) +def verify_manifest(path,approved): + path=Path(path).resolve() + if path!=MANIFEST.resolve() or sha(path)!=approved:raise RuntimeError("v5 manifest path or SHA mismatch") + m=load(path) + if m.get("schema_version")!="deltawire-preflight-v5-manifest.v1" or m.get("scope")!=["probe/range-large-v5","range-large/r1"]:raise RuntimeError("v5 manifest identity mismatch") + if m.get("harbor_version")!="0.20.0" or m.get("model")!="google/gemini-3.1-pro-preview" or m.get("agent")!="gemini-cli":raise RuntimeError("frozen runtime mismatch") + actual=subprocess.check_output(m["harbor_command"]+["--version"],cwd=ROOT,text=True).strip() + if actual!="0.20.0":raise RuntimeError("Harbor version mismatch") + for name,want in m["input_hashes"].items(): + target=ROOT/name + if not target.is_file() or sha(target)!=want:raise RuntimeError(f"frozen input mismatch: {name}") + verify_locks();return m +def verify_binary(paths,want): + for path in [BINARY,*[p/"environment/.generated/deltawire" for p in paths]]: + if not path.is_file() or sha(path)!=want:raise RuntimeError(f"binary parity mismatch: {path}") + version=subprocess.run([str(BINARY),"version"],capture_output=True,text=True,check=False) + if version.returncode or not version.stdout.startswith("deltawire version dev\n"):raise RuntimeError("host binary version mismatch") + +def private_env(): + key=os.environ.get("GEMINI_API_KEY","").strip() + if not key:raise RuntimeError("GEMINI_API_KEY is required") + f=tempfile.NamedTemporaryFile("w",prefix="deltawire-v5-",suffix=".env",delete=False);f.write("GEMINI_API_KEY="+key.replace("\n","")+"\n");f.close();os.chmod(f.name,0o600);return Path(f.name) +def trials(jobs):return {p.resolve() for p in jobs.glob("*/*__*") if p.is_dir()} +def run_one(m,run,task,jobs,dry=False): + cmd=m["harbor_command"]+["run","-a",m["agent"],"-m",m["model"],"-p",str(task),"-k","1","--jobs-dir",str(jobs),"--n-concurrent","1"] + if run["arm"]=="D1":cmd+=["--skill",str(SKILL)] + redacted=shlex.join(cmd+["--env-file",""]) + if dry:print(redacted);return {**run,"status":"dry_run","redacted_command":redacted} + envfile=private_env();before=trials(jobs);jobs.mkdir(parents=True,exist_ok=True);env=os.environ.copy();env.pop("GEMINI_API_KEY",None) + try: + try:rc=subprocess.run(cmd+["--env-file",str(envfile)],cwd=ROOT,env=env,check=False).returncode;error=None + except OSError as e:rc=None;error=str(e) + finally:envfile.unlink(missing_ok=True) + created=trials(jobs)-before;entry={**run,"harbor_exit_code":rc,"status":"completed" if rc==0 else "harbor_failed","redacted_command":redacted} + if error:entry["harbor_error"]=error + if len(created)!=1:entry["artifact_error"]=f"expected one trial, found {len(created)}";return entry + trial=created.pop();entry["trial_dir"]=rel(trial) + candidates={"result_path":trial/"result.json","trajectory_path":trial/"agent/gemini-cli.trajectory.jsonl"} + outputs=sorted((trial/"artifacts").rglob("output.ndjson")) + if outputs:candidates["output_path"]=outputs[0] + for key,path in candidates.items(): + if path.is_file():entry[key]=rel(path);entry[key.replace("_path","_sha256")]=sha(path) + missing=[k for k in candidates if k not in entry] + if missing:entry["artifact_error"]="missing artifacts: "+", ".join(missing) + return entry + +def semantic(output): + run=subprocess.run(["python3",str(HERE/"semantic_oracle.py"),str(EVAL/"tasks/range-large/task-spec.json"),str(output)],capture_output=True,text=True,check=False) + try:value=json.loads(run.stdout) if run.stdout.strip() else {} + except json.JSONDecodeError:value={} + exact=value.get("exact_match",0) if run.returncode==0 else 0 + return {"schema_version":"semantic-result.v5","exact_match":exact,"status":"pass" if exact==1 else "fail","oracle_result":value,"error":None if exact==1 else (run.stderr.strip() or run.stdout.strip() or "oracle failed")} +def independent(trial,dest): + plans=sorted((Path(trial)/"artifacts/.deltawire/plans").glob("*.dw.json")) + if len(plans)!=1:value={"schema_version":"plan-contract-receipt.v1","status":"fail","checks":{},"error":f"expected one plan, found {len(plans)}"};write(dest,value);return value + cmd=["python3",str(HERE/"verify_plan_contract.py"),"--contract",str(EVAL/"tasks/range-large/task-spec.json"),"--plan",str(plans[0]),"--repo",str(Path(trial)/"artifacts"),"--deltawire",str(BINARY),"--receipt",str(dest)] + subprocess.run(cmd,cwd=ROOT,capture_output=True,text=True,check=False) + return load(dest) if Path(dest).is_file() else {"status":"fail"} +def duration(data): + try:return (datetime.fromisoformat(data["finished_at"].replace("Z","+00:00"))-datetime.fromisoformat(data["started_at"].replace("Z","+00:00"))).total_seconds() + except (KeyError,TypeError,ValueError):return None +def red_receipt(arm,reason): + return {"schema_version":"treatment-use-receipt.v5","arm":arm,"status":"fail","classification":"attempted_failed" if arm=="D1" else "contaminated","failure_reasons":[reason],"checks":{},"contamination":arm=="B0"} +def finalize(m,entry,root): + arm=entry["arm"];trial=ROOT/entry["trial_dir"] if entry.get("trial_dir") else None;trajectory=ROOT/entry["trajectory_path"] if entry.get("trajectory_path") else None;output=ROOT/entry["output_path"] if entry.get("output_path") else None + sem=semantic(output) if output and output.is_file() else {"schema_version":"semantic-result.v5","exact_match":0,"status":"fail","error":"output missing"};sem_path=root/f"{arm}-semantic-result.json";write(sem_path,sem) + indep_path=root/f"{arm}-plan-contract-receipt.json" + if arm=="D1" and trial:indep=independent(trial,indep_path) + else:indep={"status":"not_applicable"} + spec=load(EVAL/"tasks/range-large/task-spec.json") + try:use=build(trajectory,trial,arm,spec,sem["exact_match"],indep_path) if trajectory and trial else red_receipt(arm,entry.get("artifact_error","trial missing")) + except Exception as e:use=red_receipt(arm,str(e)) + use_path=root/f"{arm}-treatment-use.json";write(use_path,use) + environment_path=trial/"artifacts/logs/artifacts/deltawire/environment-receipt.json" if trial else None + environment=load(environment_path) if environment_path and environment_path.is_file() else {"schema_version":"deltawire-environment-receipt.v1","status":"fail","error":"environment receipt missing"} + environment_copy=root/f"{arm}-environment-receipt.json";write(environment_copy,environment) + expected_files={name:item["sha256"] for name,item in load(PROBE/"environment/deltawire-environment-expectations.json")["files"].items()} + artifact=validate_artifacts(trial,m["deltawire_binary_sha256"],expected_files) if trial else {"schema_version":"artifact-manifest-receipt.v1","status":"fail","checks":{},"error":"trial missing"} + artifact_path=root/f"{arm}-artifact-manifest-receipt.json";write(artifact_path,artifact) + statuses=(environment.get("status"),use.get("status"),sem.get("status"),indep.get("status") if arm=="D1" else "pass",artifact.get("status")) + end={"schema_version":"deltawire-end-to-end-receipt.v5","arm":arm,"environment_status":statuses[0],"treatment_status":statuses[1],"semantic_status":statuses[2],"plan_contract_status":statuses[3],"artifact_manifest_status":statuses[4],"status":"pass" if all(x=="pass" for x in statuses) else "fail","ready_for_72":False,"source_hashes":{"environment":sha(environment_copy),"artifact_manifest_receipt":sha(artifact_path),"treatment_use":sha(use_path),"semantic":sha(sem_path),"trajectory":entry.get("trajectory_sha256"),"result":entry.get("result_sha256"),"output":entry.get("output_sha256")}} + data=load(ROOT/entry["result_path"]) if entry.get("result_path") else {};agent=data.get("agent_result") or {};entry.update({"trial_id":data.get("id") or data.get("trial_name"),"reported_model":((data.get("agent_info") or {}).get("model_info") or {}).get("name"),"input_tokens":agent.get("n_input_tokens"),"output_tokens":agent.get("n_output_tokens"),"cache_tokens":agent.get("n_cache_tokens"),"wall_time_seconds":duration(data)}) + end["model_match"]=entry["reported_model"]==m["expected_reported_model"];end["harbor_exit_0"]=entry.get("harbor_exit_code")==0 + if not end["model_match"] or not end["harbor_exit_0"] or entry.get("artifact_error"):end["status"]="fail" + end_path=root/f"{arm}-end-to-end.json";write(end_path,end);return sem,use,environment,end,{"semantic":sem_path,"use":use_path,"environment":environment_copy,"artifact":artifact_path,"end":end_path,"independent":indep_path} +def finalize_failure(entry,root,error): + arm=entry["arm"];reason=str(error);sem={"schema_version":"semantic-result.v5","exact_match":0,"status":"fail","error":reason};use=red_receipt(arm,reason);env={"schema_version":"deltawire-environment-receipt.v1","status":"fail","error":reason} + sem_path=root/f"{arm}-semantic-result.json";use_path=root/f"{arm}-treatment-use.json";env_path=root/f"{arm}-environment-receipt.json";artifact_path=root/f"{arm}-artifact-manifest-receipt.json";indep_path=root/f"{arm}-plan-contract-receipt.json" + write(sem_path,sem);write(use_path,use);write(env_path,env) + write(artifact_path,{"schema_version":"artifact-manifest-receipt.v1","status":"fail","checks":{},"error":reason}) + if arm=="D1":write(indep_path,{"schema_version":"plan-contract-receipt.v1","status":"fail","checks":{},"error":reason}) + end={"schema_version":"deltawire-end-to-end-receipt.v5","arm":arm,"environment_status":"fail","treatment_status":"fail","semantic_status":"fail","plan_contract_status":"fail","artifact_manifest_status":"fail","status":"fail","ready_for_72":False,"failure_reasons":[reason],"source_hashes":{"environment":sha(env_path),"artifact_manifest_receipt":sha(artifact_path),"treatment_use":sha(use_path),"semantic":sha(sem_path),"trajectory":entry.get("trajectory_sha256"),"result":entry.get("result_sha256"),"output":entry.get("output_sha256")}} + write(root/f"{arm}-end-to-end.json",end);return end + +def release_body(manifest,ledger,entry,paths): + trial=ROOT/entry["trial_dir"];required=[Path(manifest),Path(ledger),ROOT/entry["result_path"],ROOT/entry["trajectory_path"],ROOT/entry["output_path"],trial/"artifacts/manifest.json",trial/"artifacts/logs/artifacts/deltawire/environment-receipt.json",trial/"artifacts/.deltawire/config.json",trial/"artifacts/.deltawire/schemas/range-large.schema.json",trial/"artifacts/.deltawire/plans/range-large.dw.json",trial/"artifacts/.deltawire/state.json",trial/"artifacts/.deltawire/plan-contract-receipt.json",*paths.values()] + missing=[str(p) for p in required if not p.is_file()] + if missing:raise RuntimeError("release artifacts missing: "+", ".join(missing)) + body={"schema_version":"probe-release.v3","probe":"range-large-v5","status":"pass","manifest_sha256":sha(manifest),"evidence_hashes":{rel(p):sha(p) for p in required},"ready_for_72":False} + body["release_id"]=hashlib.sha256(json.dumps(body,sort_keys=True,separators=(",",":")).encode()).hexdigest();return body +def verify_release(manifest): + path=PROBE_ROOT/"probe-release.json" + if not path.is_file():raise RuntimeError("probe-release.v3 missing") + release=load(path);rid=release.pop("release_id",None);want=hashlib.sha256(json.dumps(release,sort_keys=True,separators=(",",":")).encode()).hexdigest();release["release_id"]=rid + if release.get("schema_version")!="probe-release.v3" or release.get("status")!="pass" or rid!=want or release.get("manifest_sha256")!=sha(manifest):raise RuntimeError("invalid probe release") + for name,want_hash in release["evidence_hashes"].items(): + target=ROOT/name + if not target.is_file() or sha(target)!=want_hash:raise RuntimeError(f"release evidence mismatch: {name}") + ledger=load(PROBE_ROOT/"run-ledger.json");entry=ledger["runs"][0] + if semantic(ROOT/entry["output_path"])["exact_match"]!=1:raise RuntimeError("release semantic recheck failed") + with tempfile.TemporaryDirectory() as tmp: + if independent(ROOT/entry["trial_dir"],Path(tmp)/"receipt.json").get("status")!="pass":raise RuntimeError("release contract recheck failed") + env=load(PROBE_ROOT/"D1-environment-receipt.json");use=load(PROBE_ROOT/"D1-treatment-use.json");artifact=load(PROBE_ROOT/"D1-artifact-manifest-receipt.json");end=load(PROBE_ROOT/"D1-end-to-end.json") + if not all(x.get("status")=="pass" for x in (env,use,artifact,end)):raise RuntimeError("release receipt status failed") + expected_files={name:item["sha256"] for name,item in load(PROBE/"environment/deltawire-environment-expectations.json")["files"].items()} + if validate_artifacts(ROOT/entry["trial_dir"],load(manifest)["deltawire_binary_sha256"],expected_files).get("status")!="pass":raise RuntimeError("release artifact-manifest recheck failed") + rerun=build(ROOT/entry["trajectory_path"],ROOT/entry["trial_dir"],"D1",load(EVAL/"tasks/range-large/task-spec.json"),1,PROBE_ROOT/"D1-plan-contract-receipt.json") + if rerun.get("status")!="pass":raise RuntimeError("release trajectory recheck failed") + return release +def frozen_pair(): + matrix=load(EVAL/"matrices/72-run-matrix.json");runs=[matrix[36],matrix[37]] + if [r.get("pair_id") for r in runs]!=["range-large/r1"]*2 or [r.get("arm") for r in runs]!=["D1","B0"]:raise RuntimeError("frozen matrix positions 36-37 mismatch") + return runs +def args(): + p=argparse.ArgumentParser();p.add_argument("mode",choices=("probe-v5","pair"));p.add_argument("--manifest",required=True);p.add_argument("--approve-manifest-sha",required=True);p.add_argument("--pair-id");p.add_argument("--dry-run",action="store_true");return p.parse_args() +def main(): + a=args();m=verify_manifest(a.manifest,a.approve_manifest_sha) + if a.mode=="probe-v5":runs=[{"pair_id":"probe/range-large-v5","task":"range-large-v5","arm":"D1","repetition":1}];tasks=[PROBE];root=PROBE_ROOT + else: + if a.pair_id!="range-large/r1":raise RuntimeError("only range-large/r1 is permitted") + verify_release(a.manifest);runs=frozen_pair();tasks=[PAIR_TASK,PAIR_TASK];root=PAIR_ROOT + if not a.dry_run:refuse(root,a.mode) + verify_binary(tasks,m["deltawire_binary_sha256"]);ledger={"schema_version":"run-ledger.v5","mode":a.mode,"ready_for_72":False,"runs":[]};ledger_path=root/"run-ledger.json" + for run,task in zip(runs,tasks): + entry=run_one(m,run,task,root/"raw"/run["arm"],a.dry_run);ledger["runs"].append(entry) + if a.dry_run:continue + write(ledger_path,ledger) + try:sem,use,env,end,paths=finalize(m,entry,root) + except Exception as error:end=finalize_failure(entry,root,error);write(ledger_path,ledger);raise RuntimeError(f"{a.mode} {run['arm']} finalization failed: {error}") + write(ledger_path,ledger) + if end["status"]!="pass":raise RuntimeError(f"{a.mode} {run['arm']} end-to-end gate failed") + if a.mode=="probe-v5": + release=release_body(a.manifest,ledger_path,entry,paths);write(root/"probe-release.json",release) + write(root/"pre-pair-report.json",{"schema_version":"pre-pair-report.v1","probe_status":"pass","release_id":release["release_id"],"manifest_sha256":sha(a.manifest),"ready_for_72":False,"automatic_pair_authorized":False,"pair_requires_explicit_review":True}) + if a.dry_run:return + if a.mode=="pair": + d1,b0=ledger["runs"];write(root/"pair-summary.json",{"schema_version":"operational-positive-control.v1","pair_complete":True,"order":["D1","B0"],"D1":{k:d1.get(k) for k in ("trial_id","input_tokens","output_tokens","cache_tokens","wall_time_seconds")},"B0":{k:b0.get(k) for k in ("trial_id","input_tokens","output_tokens","cache_tokens","wall_time_seconds")},"claim":"operational_positive_control_only","READY_FOR_72":False,"full_run_started":False}) + print(a.mode,"completed") +if __name__=="__main__": + try:main() + except (OSError,ValueError,KeyError,RuntimeError,subprocess.CalledProcessError) as e:print("ERROR:",e,file=sys.stderr);raise SystemExit(1) diff --git a/labs/20-deltawire/eval/scripts/v5/semantic_oracle.py b/labs/20-deltawire/eval/scripts/v5/semantic_oracle.py new file mode 100755 index 000000000..00cfbfbc6 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/semantic_oracle.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +import json +import sys +from pathlib import Path + + +def canonical(spec): + g, count, family = spec["generation"], spec["record_count"], spec["family"] + if family == "matrix": + rows = [{}] + for key, allowed in g["dimensions"].items(): + rows = [{**row, key: value} for row in rows for value in allowed] + return [{**row, **g["constants"]} for row in rows] + if family == "range": + return [{g["field"]: i} for i in range(g["start"], g["start"] + count)] + if family == "rows": + return [{"id": i, "name": g["name_template"].format(id=i), **g["constants"]} for i in range(g["start"], g["start"] + count)] + if family == "variants": + return [{**g["base"], "value": g["base"]["value"] + i * g["value_step"]} for i in range(count)] + if family == "mixed": + return ([{"id": i, "type": "std"} for i in range(1, g["standard_count"] + 1)] + + [{"id": i, "type": "exc", "error_code": g["error_code"]} for i in range(g["standard_count"] + 1, count + 1)]) + return [{"id": i, f'{g["dynamic_field_prefix"]}{i}': i} for i in range(g["id_start"], g["id_start"] + count)] + + +def main(): + if len(sys.argv) != 3: + print('{"exact_match": 0}'); return + spec_arg, output_arg = Path(sys.argv[1]), Path(sys.argv[2]) + if not spec_arg.is_file(): + candidate = Path(__file__).resolve().parents[1] / "tasks" / sys.argv[1] / "task-spec.json" + spec_arg = candidate + try: + spec = json.loads(spec_arg.read_text(encoding="utf-8")) + lines = output_arg.read_text(encoding="utf-8").splitlines() + if not lines or any(not line.strip() for line in lines): + raise ValueError("empty line") + actual = [json.loads(line) for line in lines] + passed = actual == canonical(spec) + except Exception: + passed = False + print(json.dumps({"exact_match": 1 if passed else 0})) + + +if __name__ == "__main__": + main() diff --git a/labs/20-deltawire/eval/scripts/v5/shell_observation.py b/labs/20-deltawire/eval/scripts/v5/shell_observation.py new file mode 100644 index 000000000..1d816d864 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/shell_observation.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Canonicalize structured Gemini shell calls without trusting tool status.""" +import json,re,shlex +from pathlib import Path + +SHELL_NAMES={"run_shell_command","shell","execute"} +EXIT_KEYS=("exit_code","exitCode","returncode","return_code") +TRAILER=re.compile(r"(?:^|\n)Exit Code:\s*(-?\d+)\nProcess Group PGID:\s*\d+\s*(?:)?\s*$") + +def calls(path): + out=[] + for line in Path(path).read_text(encoding="utf-8",errors="replace").splitlines(): + try: event=json.loads(line) + except json.JSONDecodeError: continue + for call in event.get("toolCalls",[]): + outputs=[] + for item in call.get("result",[]): + response=(item.get("functionResponse") or {}).get("response") or {} + outputs.append(str(response.get("output",""))) + out.append({"name":call.get("name"),"args":call.get("args") or {},"tool_status":call.get("status"),"result":call.get("result") or [],"output":"\n".join(outputs)}) + return out + +def direct_code(value): + if isinstance(value,dict): + for key in EXIT_KEYS: + if key in value and isinstance(value[key],int): return value[key] + for child in value.values(): + found=direct_code(child) + if found is not None:return found + elif isinstance(value,list): + for child in value: + found=direct_code(child) + if found is not None:return found + return None + +def observation(call): + command=str(call["args"].get("command") or call["args"].get("cmd") or "") + code=direct_code(call["result"]); provenance="structured" if code is not None else None + if code is None: + matches=TRAILER.findall(call["output"]) + if len(matches)==1: code=int(matches[0]); provenance="explicit_output" + return {"command":command,"tool_status":call["tool_status"],"shell_exit_code":code,"shell_exit_provenance":provenance,"observable":call["name"] in SHELL_NAMES and bool(command)} + +def lifecycle(observation): + command=observation["command"]; parts=re.split(r"(&&|\|\||;|\n)",command); commands=parts[::2]; operators=parts[1::2] + found=[] + for segment in commands: + try: tokens=shlex.split(segment) + except ValueError: continue + while tokens and ("=" in tokens[0] and not tokens[0].startswith("/")): tokens=tokens[1:] + if tokens and tokens[0] in {"sudo","env"}: tokens=tokens[1:] + if len(tokens)>=2 and Path(tokens[0]).name=="deltawire" and tokens[1] in {"validate","render","check"}: + found.append({"command":tokens[1],"invocation":segment.strip(),"shell_exit_code":None}) + if len(found)==1 and len(commands)==1: found[0]["shell_exit_code"]=observation["shell_exit_code"] + elif found and observation["shell_exit_code"]==0 and operators and all(op=="&&" for op in operators): + for item in found:item["shell_exit_code"]=0 + return found + +def invokes_deltawire(observation): + for segment in re.split(r"&&|\|\||;|\n",observation["command"]): + try: tokens=shlex.split(segment) + except ValueError: continue + while tokens and ("=" in tokens[0] and not tokens[0].startswith("/")): tokens=tokens[1:] + if tokens and tokens[0] in {"sudo","env"}: tokens=tokens[1:] + if tokens and Path(tokens[0]).name=="deltawire": return True + return False + +def parse(path): + observations=[observation(c) for c in calls(path) if c["name"] in SHELL_NAMES] + return observations,[item for obs in observations for item in lifecycle(obs)] diff --git a/labs/20-deltawire/eval/scripts/v5/test_artifact_manifest.py b/labs/20-deltawire/eval/scripts/v5/test_artifact_manifest.py new file mode 100644 index 000000000..461fdfabe --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/test_artifact_manifest.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +import json,tempfile +from pathlib import Path +from artifact_manifest import validate + +def write(path,value):path.parent.mkdir(parents=True,exist_ok=True);path.write_text(json.dumps(value)+"\n") +def fixture(root): + trial=root/"trial";manifest=trial/"artifacts/manifest.json";receipt=trial/"artifacts/logs/artifacts/deltawire/environment-receipt.json" + checks={name:True for name in ("binary_exists","binary_regular","binary_not_symlink","binary_executable","binary_path","binary_realpath","binary_hash","version_exit_0","version_exact","config_hash","schema_hash","public_contract_hash")} + value={"schema_version":"deltawire-environment-receipt.v1","status":"pass","checks":checks,"binary":{"actual_sha256":"binary"},"files":{name:{"actual_sha256":name} for name in ("config","schema","public_contract")}} + write(receipt,value);write(manifest,[{"source":"/logs/artifacts","destination":"artifacts/logs/artifacts","type":"directory","status":"ok","service":None}]);return trial,manifest,receipt +def main(): + expected={name:name for name in ("config","schema","public_contract")} + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);trial,manifest,receipt=fixture(root);assert validate(trial,"binary",expected)["status"]=="pass" + cases=[] + cases.append(lambda:receipt.unlink()) + cases.append(lambda:write(receipt,{"bad":"json shape"})) + cases.append(lambda:write(manifest,[{"source":"/logs/artifacts","destination":"artifacts/logs/artifacts","type":"directory","status":"skipped"}])) + cases.append(lambda:write(manifest,[{"source":"/wrong","destination":"artifacts/logs/artifacts","type":"directory","status":"ok"}])) + cases.append(lambda:write(manifest,[])) + for mutate in cases: + trial,manifest,receipt=fixture(root);mutate();assert validate(trial,"binary",expected)["status"]=="fail" + trial,manifest,receipt=fixture(root);outside=trial/"artifacts/environment-receipt.json";outside.write_bytes(receipt.read_bytes());receipt.unlink();assert validate(trial,"binary",expected)["status"]=="fail" + trial,manifest,receipt=fixture(root);value=json.loads(receipt.read_text());value["binary"]["actual_sha256"]="v4";write(receipt,value);assert validate(trial,"binary",expected)["status"]=="fail" + print("Artifact-manifest and canonical receipt-path tests passed.") +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v5/test_environment_receipt.py b/labs/20-deltawire/eval/scripts/v5/test_environment_receipt.py new file mode 100644 index 000000000..4f586567f --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/test_environment_receipt.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +import hashlib,json,os,subprocess,tempfile +from pathlib import Path +HERE=Path(__file__).resolve().parent +def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest() +def run(root,binary,mutate=None): + files={} + for name in ("config","schema","public_contract"): + p=root/f"{name}.json";p.write_text(name);files[name]={"path":str(p),"sha256":sha(p)} + exp={"binary":{"path":str(binary),"realpath":str(binary.resolve()),"sha256":sha(binary) if binary.is_file() else "0"*64,"version":"deltawire version dev"},"files":files} + if mutate:mutate(exp) + ep=root/"expect.json";rp=root/"receipt.json";ep.write_text(json.dumps(exp));result=subprocess.run(["python3",str(HERE/"environment_receipt.py"),"--expectations",str(ep),"--receipt",str(rp)],capture_output=True,text=True);return result.returncode,json.loads(rp.read_text()) +def main(): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);binary=root/"deltawire";binary.write_text("#!/bin/sh\nprintf 'deltawire version dev\\nsupported plan version deltawire.plan.v1\\n'\n");binary.chmod(0o755) + first=run(root,binary)[1];second=run(root,binary)[1];assert first["status"]=="pass" and first==second + assert run(root,binary,lambda e:e["binary"].update(sha256="0"*64))[1]["status"]=="fail" + assert run(root,binary,lambda e:e["binary"].update(path=str(root/"wrong")))[1]["status"]=="fail" + assert run(root,binary,lambda e:e["binary"].update(version="wrong"))[1]["status"]=="fail" + assert run(root,binary,lambda e:e["files"]["config"].update(sha256="0"*64))[1]["status"]=="fail" + assert run(root,binary,lambda e:e["files"]["schema"].update(sha256="0"*64))[1]["status"]=="fail" + assert run(root,binary,lambda e:e["files"]["public_contract"].update(sha256="0"*64))[1]["status"]=="fail" + assert run(root,binary,lambda e:e["binary"].update(realpath="/wrong"))[1]["status"]=="fail" + binary.chmod(0o644);assert run(root,binary)[1]["status"]=="fail";binary.chmod(0o755) + link=root/"link";link.symlink_to(binary);assert run(root,link)[1]["status"]=="fail" + missing=root/"missing";code,receipt=run(root,missing);assert code and receipt["status"]=="fail" + failing=root/"failing";failing.write_text("#!/bin/sh\nexit 7\n");failing.chmod(0o755);assert run(root,failing)[1]["binary"]["version_exit_code"]==7 + atomic=root/"atomic-target";atomic.mkdir();files={} + for name in ("config","schema","public_contract"): + p=root/f"atomic-{name}.json";p.write_text(name);files[name]={"path":str(p),"sha256":sha(p)} + exp={"binary":{"path":str(binary),"realpath":str(binary.resolve()),"sha256":sha(binary),"version":"deltawire version dev"},"files":files} + ep=root/"atomic-expect.json";ep.write_text(json.dumps(exp));result=subprocess.run(["python3",str(HERE/"environment_receipt.py"),"--expectations",str(ep),"--receipt",str(atomic)],capture_output=True,text=True) + assert result.returncode and atomic.is_dir() and not list(root.glob(".atomic-target.*.tmp")) + print("Deterministic environment receipt tests passed.") +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v5/test_fidelity.py b/labs/20-deltawire/eval/scripts/v5/test_fidelity.py new file mode 100644 index 000000000..7fb11453e --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/test_fidelity.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +import json,tempfile +from pathlib import Path +from fidelity import build +EVAL=Path(__file__).resolve().parents[2];SPEC=json.loads((EVAL/"tasks/range-large/task-spec.json").read_text()) +def event(name,args,output="",status="success"): + return {"toolCalls":[{"name":name,"args":args,"status":status,"result":[{"functionResponse":{"response":{"output":output}}}]}]} +GOOD=[event("run_shell_command",{"command":"gemini skills list"},"deltawire"),event("activate_skill",{"name":"deltawire"}),event("read_file",{"file_path":"/task-contract.json"}),event("run_shell_command",{"command":"deltawire validate --repo . p"}),event("run_shell_command",{"command":"deltawire render --repo . p"}),event("run_shell_command",{"command":"deltawire check --repo . p"})] +def run(events,arm="D1",state=True,plan=True): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);trajectory=root/"trajectory.jsonl";trajectory.write_text("\n".join(json.dumps(e) for e in events));art=root/"artifacts" + paths=[art/".deltawire/config.json",art/".deltawire/schemas/range-large.schema.json",art/"testdata/generated/output.ndjson"] + if plan:paths += [art/".deltawire/plans/range-large.dw.json",art/".deltawire/plan-contract-receipt.json"] + if state:paths.append(art/".deltawire/state.json") + for p in paths:p.parent.mkdir(parents=True,exist_ok=True);p.write_text("{}\n") + (art/".deltawire/schemas/range-large.schema.json").write_bytes((EVAL/"tasks/range-large/authoritative-schema.json").read_bytes()) + if plan:(art/".deltawire/plan-contract-receipt.json").write_text('{"status":"pass"}\n') + ind=root/"ind.json";ind.write_text('{"status":"pass"}\n') + return build(trajectory,root,arm,SPEC,1,ind) +def main(): + assert run(GOOD)["status"]=="pass" + for i in range(len(GOOD)):assert run(GOOD[:i]+GOOD[i+1:])["status"]=="fail" + assert run(GOOD,state=False)["status"]=="fail" + failed=GOOD[:-3]+[event("run_shell_command",{"command":"deltawire validate p"},"Exit Code: 2\nProcess Group PGID: 1"),*GOOD[-2:]];assert run(failed)["status"]=="fail" + assert run([],arm="B0",state=False,plan=False)["status"]=="pass" + assert run([],arm="B0",state=False,plan=True)["status"]=="fail" + assert run([event("run_shell_command",{"command":"/usr/local/bin/deltawire version"})],arm="B0",state=False,plan=False)["status"]=="fail" + print("Treatment-use v4 tests passed.") +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v5/test_runner.py b/labs/20-deltawire/eval/scripts/v5/test_runner.py new file mode 100644 index 000000000..3dd3912a8 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/test_runner.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +import tempfile +from pathlib import Path +import runner +def expect(text,fn,*args): + try:fn(*args);raise AssertionError("accepted invalid state") + except RuntimeError as e:assert text in str(e),str(e) +def main(): + runs=runner.frozen_pair();assert [x["arm"] for x in runs]==["D1","B0"] + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);runner.refuse(root,"v5");(root/"partial").write_text("x");expect("rerun forbidden",runner.refuse,root,"v5") + expect("probe-release.v3 missing",runner.verify_release,"missing") + source=(Path(__file__).parent/"runner.py").read_text();assert 'choices=("probe-v5","pair")' in source + for forbidden in ('"full"','probe-v3','probe-v2','allow-72'):assert forbidden not in source + print("V5 runner authorization, adjacency, release, and rerun gates passed.") +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v5/test_shell_observation.py b/labs/20-deltawire/eval/scripts/v5/test_shell_observation.py new file mode 100644 index 000000000..de3ba9bfe --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/test_shell_observation.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +import json,tempfile +from pathlib import Path +from shell_observation import parse +def event(command,output="",status="success",extra=None): + response={"output":output};response.update(extra or {}) + return {"toolCalls":[{"name":"run_shell_command","args":{"command":command},"status":status,"result":[{"functionResponse":{"response":response}}]}]} +def run(events): + with tempfile.TemporaryDirectory() as tmp: + p=Path(tmp)/"t.jsonl";p.write_text("\n".join(json.dumps(e) for e in events));return parse(p) +def main(): + obs,life=run([event("deltawire validate p","ok\nExit Code: 2\nProcess Group PGID: 7\n")]);assert obs[0]["tool_status"]=="success" and obs[0]["shell_exit_code"]==2 and life[0]["shell_exit_code"]==2 + obs,_=run([event("which deltawire","/usr/local/bin/deltawire")]);assert obs[0]["shell_exit_code"] is None + obs,_=run([event("type -P deltawire","/fake\nExit Code: 0\nProcess Group PGID: 1",extra={"exit_code":3})]);assert obs[0]["shell_exit_code"]==3 and obs[0]["shell_exit_provenance"]=="structured" + _,life=run([event("/usr/local/bin/deltawire validate p && deltawire render p && deltawire check p","Exit Code: 0\nProcess Group PGID: 9")]);assert [x["command"] for x in life]==["validate","render","check"] and all(x["shell_exit_code"]==0 for x in life) + _,life=run([event("deltawire validate p; deltawire check p","Exit Code: 0\nProcess Group PGID: 9")]);assert all(x["shell_exit_code"] is None for x in life) + _,life=run([event("echo 'deltawire validate'",status="success")]);assert not life + print("Canonical shell observation tests passed.") +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v5/validate.py b/labs/20-deltawire/eval/scripts/v5/validate.py new file mode 100644 index 000000000..0fce8c9f1 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/validate.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +import hashlib,json +from pathlib import Path +ROOT=Path(__file__).resolve().parents[5];EVAL=ROOT/"labs/20-deltawire/eval";V5=EVAL/"probes/range-large-v5" +def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest() +def main(): + canonical=EVAL/"tasks/range-large";pairs=[V5,V5/"pair-task"] + for base in pairs: + for name in ("task-spec.json","authoritative-schema.json"): + assert (base/name).read_bytes()==(canonical/name).read_bytes(),f"v5 projection mismatch: {base/name}" + assert (base/"environment/task-contract.json").read_bytes()==(canonical/"task-spec.json").read_bytes() + assert (base/"tests/task-spec.json").read_bytes()==(canonical/"task-spec.json").read_bytes() + artifacts=(base/"task.toml").read_text() + for required in ("state.json","plan-contract-receipt.json","output.ndjson"):assert required in artifacts + assert ".deltawire/environment-receipt.json" not in artifacts + assert 'service = "main"' in artifacts and "/logs/artifacts/deltawire/environment-receipt.json" in artifacts + assert (base/"environment/environment_receipt.py").read_bytes()==(EVAL/"scripts/v5/environment_receipt.py").read_bytes() + matrix=json.loads((EVAL/"matrices/72-run-matrix.json").read_text());assert [matrix[i]["arm"] for i in (36,37)]==["D1","B0"] and all(matrix[i]["pair_id"]=="range-large/r1" for i in (36,37)) + workflow=(ROOT/".github/workflows/deltawire-preflight-v5.yml").read_text();assert "GEMINI_API_KEY" not in workflow and "harbor run" not in workflow + conformance=EVAL/"conformance/environment-receipt-v1/task.toml";assert 'service = "main"' in conformance.read_text() + manifest=EVAL/"manifests/preflight-v5-range-large.json" + if manifest.is_file(): + data=json.loads(manifest.read_text());assert data["ready_for_72"] is False and data["matrix_positions"]["zero_based"]==[36,37] + for name,want in data["input_hashes"].items():assert (ROOT/name).is_file() and sha(ROOT/name)==want,name + print("V5 projections, collect hooks, matrix positions, manifest, and no-live-CI checks passed.") +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v5/validate.sh b/labs/20-deltawire/eval/scripts/v5/validate.sh new file mode 100755 index 000000000..998eb8f72 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/validate.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/../../../../.." && pwd)";cd "$ROOT" +python3 labs/20-deltawire/eval/scripts/v5/verify_evidence_locks.py +python3 labs/20-deltawire/eval/scripts/v5/test_environment_receipt.py +python3 labs/20-deltawire/eval/scripts/v5/test_artifact_manifest.py +python3 labs/20-deltawire/eval/scripts/v5/negative_controls.py +python3 labs/20-deltawire/eval/scripts/v5/test_shell_observation.py +python3 labs/20-deltawire/eval/scripts/v5/test_fidelity.py +python3 labs/20-deltawire/eval/scripts/v5/test_runner.py +SHA="$(bash labs/20-deltawire/eval/scripts/v5/build_and_stage.sh)" +test "$SHA" = "$(python3 -c 'import json;print(json.load(open("labs/20-deltawire/eval/probes/range-large-v5/environment/deltawire-environment-expectations.json"))["binary"]["sha256"])')" +python3 labs/20-deltawire/eval/scripts/v5/validate.py +python3 -m py_compile labs/20-deltawire/eval/scripts/v5/*.py +git diff --check diff --git a/labs/20-deltawire/eval/scripts/v5/verify_evidence_locks.py b/labs/20-deltawire/eval/scripts/v5/verify_evidence_locks.py new file mode 100644 index 000000000..3db2627d6 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/verify_evidence_locks.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +import hashlib,json,subprocess +from pathlib import Path + +ROOT=Path(__file__).resolve().parents[5] +EVAL=ROOT/"labs/20-deltawire/eval" +def sha(data):return hashlib.sha256(data).hexdigest() +def object_bytes(commit,name):return subprocess.check_output(["git","show",f"{commit}:{name}"],cwd=ROOT) +def main(): + evidence_commit="c200dc0eb3c4dff0e4732cc4b2daa1cd535703e3" + for path in (EVAL/"results/preflight-v1/treatment-probe-range-large/v1-evidence-lock.json",EVAL/"manifests/preflight-v2-evidence-lock.json"): + lock=json.loads(path.read_text()) + for name,want in lock["immutable_files"].items(): + target=ROOT/name + data=target.read_bytes() if target.is_file() else object_bytes(evidence_commit,name) + if sha(data)!=want:raise SystemExit(f"historical evidence mismatch: {name}") + v3=json.loads((EVAL/"manifests/preflight-v3-evidence-lock.json").read_text());v3_commit=v3["source_commit"] + if subprocess.check_output(["git","rev-parse",f"{v3_commit}^{{tree}}"],cwd=ROOT,text=True).strip()!=v3["source_tree"]:raise SystemExit("v3 source tree mismatch") + for name,want in v3["immutable_files"].items(): + if sha(object_bytes(v3_commit,name))!=want:raise SystemExit(f"v3 Git-object mismatch: {name}") + lock=json.loads((EVAL/"manifests/preflight-v4-evidence-lock.json").read_text()) + commit=lock["source_commit"] + tree=subprocess.check_output(["git","rev-parse",f"{commit}^{{tree}}"],cwd=ROOT,text=True).strip() + if tree!=lock["source_tree"]:raise SystemExit("v4 source tree mismatch") + for name,want in lock["immutable_files"].items(): + data=object_bytes(commit,name) + if sha(data)!=want:raise SystemExit(f"v4 Git-object mismatch: {name}") + print(f"v1-v4 evidence locks passed ({len(lock['immutable_files'])} v4 files).") +if __name__=="__main__":main() diff --git a/labs/20-deltawire/eval/scripts/v5/verify_plan_contract.py b/labs/20-deltawire/eval/scripts/v5/verify_plan_contract.py new file mode 100644 index 000000000..205d4fc8a --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v5/verify_plan_contract.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +import argparse, hashlib, json, subprocess +from pathlib import Path + +def sha(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() +def write(path, value): + if path: + target=Path(path); target.parent.mkdir(parents=True,exist_ok=True); target.write_text(json.dumps(value,indent=2,sort_keys=True)+"\n") + +def main(): + p=argparse.ArgumentParser(); p.add_argument("--contract",required=True); p.add_argument("--plan",required=True); p.add_argument("--repo",default="."); p.add_argument("--receipt"); p.add_argument("--deltawire",default="deltawire"); a=p.parse_args() + repo=Path(a.repo).resolve(); contract_path=Path(a.contract).resolve(); plan=Path(a.plan).resolve(); contract=json.loads(contract_path.read_text()) + try: plan_arg=str(plan.relative_to(repo)) + except ValueError: plan_arg=str(plan) + result={"schema_version":"plan-contract-receipt.v1","contract_sha256":sha(contract_path),"plan_sha256":sha(plan),"checks":{},"status":"fail"} + try: + run=subprocess.run([a.deltawire,"inspect","--repo",str(repo),"--format","json",plan_arg],capture_output=True,text=True) + result["inspect_exit_code"]=run.returncode + if run.returncode: result["error"]=run.stderr.strip() or run.stdout.strip() + else: + inspected=json.loads(run.stdout); schema=contract["authoritative_schema"]; schema_path=repo/schema["path"] + checks={"output_path":inspected.get("output_path")==contract["output"]["path"],"output_format":inspected.get("output_format")==contract["output"]["format"],"projected_records":inspected.get("projected_records")==contract["record_count"],"schema_path":inspected.get("schema_path")==schema["path"],"schema_exists":schema_path.is_file(),"schema_sha256":schema_path.is_file() and sha(schema_path)==schema["sha256"],"exact_count_assertion":str(contract["record_count"]) in json.dumps(inspected.get("assertion_summary",{}))} + result.update({"checks":checks,"inspect":inspected,"status":"pass" if all(checks.values()) else "fail"}) + except (OSError,ValueError,KeyError) as error: result["error"]=str(error) + write(a.receipt,result) + if result["status"]!="pass": raise SystemExit(1) + print(json.dumps(result,sort_keys=True)) +if __name__=="__main__": main()