Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .github/workflows/deltawire-runtime.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,5 +33,12 @@ jobs:
- run: python3 labs/20-deltawire/eval/scripts/v6/test_setup_receipt.py
- run: python3 labs/20-deltawire/eval/scripts/v6/test_trial_status.py
- run: python3 labs/20-deltawire/eval/scripts/v6/test_shell_observation.py
- run: python3 labs/20-deltawire/eval/scripts/v7/verify_terminal_evidence_lock.py
- run: python3 labs/20-deltawire/eval/scripts/v7/test_terminal_evidence_lock.py
- name: Run v7 tests in isolated processes
run: |
for test_file in labs/20-deltawire/eval/scripts/v7/test_*.py; do
python3 "$test_file"
done
- run: python3 labs/20-deltawire/eval/scripts/v6/negative_controls.py --self-test
- run: PYTHONPATH=labs/20-deltawire/eval uv run --with harbor==0.20.0 python labs/20-deltawire/eval/agents/test_preinstalled_gemini_cli.py
10 changes: 9 additions & 1 deletion .github/workflows/deltawire-v7.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,15 @@ jobs:
cmp /tmp/skill-conformance.json labs/20-deltawire/eval/results/preflight-v7/skill-conformance.json
fi
- name: No-live v7 validation
run: bash labs/20-deltawire/eval/scripts/v7/validate.sh
run: |
python3 labs/20-deltawire/eval/scripts/v6/verify_terminal_evidence_lock.py
python3 labs/20-deltawire/eval/scripts/v7/validate.py
python3 labs/20-deltawire/eval/scripts/v7/verify_terminal_evidence_lock.py
python3 labs/20-deltawire/eval/scripts/v7/test_terminal_evidence_lock.py
for test_file in labs/20-deltawire/eval/scripts/v7/test_*.py; do
DELTAWIRE_TEST_BINARY=labs/20-deltawire/eval/.generated/deltawire-linux-amd64 \
python3 "$test_file"
done
- name: Prove workflow has no model invocation
run: |
live_runner='runner.py .*pair'"-v7"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
{
"files": {
"labs/20-deltawire/eval/manifests/deltawire-range-large-v7-pair.json": "67789b85c08802a87f5d14bb72aa2d36068c43a45aa27187ddab5d66dba3eca4",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/authoritative-schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.generated/.gitignore": "240a3e0d37d2e86b614063f5347eb02d4f99ca6c254de6b82871ff8d95532a7d",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/Dockerfile": "cd768e64efba435604b265088bce6269e8dbd09e54bf51a4c5bf0dd94d10c05f",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/deltawire-environment-expectations.json": "03fe7a40964e908f2fb3736add8b8bea890b407d837f2b185d92bd8d85d4d114",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/environment_receipt.py": "3ab3bbcbfae2093ac658063d1e26f26b43c35dfdb81d8ddae5c9989f967ba623",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/task-contract.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/verify_plan_contract.py": "8aafe73ab47213027a258f9abbebbca4803fa53f658abeb7b405d4a79202d4af",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/instruction.md": "549f4047b1a4ba4f7a3f3dce9e60d1d70b141caa189aafdba942b15e6d022aee",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/SKILL.md": "e7d3a1e8ccef1b72338d5dac7a7b7ac412164846018ab55b7f2ecadf5a07a89a",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/matrix.dw.json": "56d16a5b20086c8202cbc1bbdccf41d467a5d4c3806d701227ebcdbf72e54b1a",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/range.dw.json": "b31e078409547f4fbb5a855f3bf0b1bd6f52386696c07401622f34d224fe148b",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/rows.dw.json": "e172cc7d6224e4566a0cd46a73465392976338fea0e225512a089322ba81aca3",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/variants.dw.json": "3183405b76e11874309728f3a4025687af6c54add8d2b97b94daff2d6ef8cdca",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/references/plan-v1.md": "f78d41266db73592526a3c5e7d6e4c883f549f99b01c6387c7f5337fc269dd5e",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/task.toml": "4adc78ba9c59f8c743770ee9b1b31a55364e7303010e3c3f109da3b3d61d96e3",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/semantic_oracle.py": "94b30fac117f40e977851966b5228e6c372bb48cbf2e5c29c24f428e1f98e91d",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5",
"labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/test.sh": "6f7cfbefdf7eaf3a082fd0c88f264ba3d41bfe84fc53498df4464bef66dc116e",
"labs/20-deltawire/eval/results/preflight-v7/pair-approval.json": "c6b493851951e72ead0d15e38231f47d036331f94ded94d9d3dceb38db7c3154",
"labs/20-deltawire/eval/results/preflight-v7/pair-security-scan.json": "4ab32a7795109d3973bd8665dbe60df485c6ac0af25b0b2e752e15a448650411",
"labs/20-deltawire/eval/results/preflight-v7/pre-live-report.json": "dcf77e739288edbd358470458582e960224fee620f92f0f19ea7415303b268d7",
"labs/20-deltawire/eval/results/preflight-v7/skill-conformance.json": "4d0e142a1bc801642b5de8c77b9cbebc9680a42f6f64da95b3d48b8624fe7c46",
"labs/20-deltawire/eval/results/range-large-v7-r1/D1-artifact-manifest.json": "40cf6ddc54e657621f177afcb92e88a94eec553ef2f29d829174d920b86ab469",
"labs/20-deltawire/eval/results/range-large-v7-r1/D1-end-to-end.json": "207e6e300158c9d87b92589c412a7ae15aa732549dbb7bdaa8654a366efc62e1",
"labs/20-deltawire/eval/results/range-large-v7-r1/D1-environment.json": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1",
"labs/20-deltawire/eval/results/range-large-v7-r1/D1-plan-contract-receipt.json": "d3c6d569d9353120583551f280753e457683c2b886594cda291b93b4e30515a3",
"labs/20-deltawire/eval/results/range-large-v7-r1/D1-semantic-result.json": "09654564d33620bb4b39ae7cb64af13243078013118e6595adaf3a40af59cba0",
"labs/20-deltawire/eval/results/range-large-v7-r1/D1-setup.json": "adca6d256d43f320bb542745bc37f58fcfa97db736eebcc2aa1ce885d84c5240",
"labs/20-deltawire/eval/results/range-large-v7-r1/D1-treatment.json": "298daa00fb33afe6f7a93b67028bcc243fe1ef809f58199f89b36d35438148aa",
"labs/20-deltawire/eval/results/range-large-v7-r1/evidence-closure.json": "95dc20298e27fb2b295ac694688f7d8b14a78aaaa16c854e6642158e15f45240",
"labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/config.json": "27de86441290364f9bc0c6c66854b4bc0194f591b909dbaf275b628f40471728",
"labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/lock.json": "0a58888ac484406279a52b2109f9aa8215cc7890204b7ff2f17022bd5d703286",
"labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/agent/gemini-cli.trajectory.jsonl": "d6659287d950b37b9ea1364e6256d715ce0220b652c45689355b79e623be7af5",
"labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/agent/gemini-cli.txt": "54e1b5e6d019e43be0e7141827f3dccb32b4a613792551ca6b80e08452e9ee6c",
"labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/agent/trajectory.json": "d6c4bae360f51d98c467028fa0354f351e8b98d6b157b19baae328ea5f514db5",
"labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f",
"labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5",
"labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/.deltawire/state.json": "5e1cf20f1d0945a30accb87152086fc61c848cdf18ac46ad2f0fa65fbd5b8c76",
"labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/logs/artifacts/deltawire/environment-receipt.json": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1",
"labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json": "a88d221e573265997797dcad31136aec0ed74a5bd37e50887595e5ea85cfa0d1",
"labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/manifest.json": "0c8e85115c5bc9a68deb3aa39b84e4f734bdeb8b7742a97b72bc139b1192b386",
"labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/testdata/generated/output.ndjson": "9d205b57ab166d23c81927899f99b6ccece261c679c96979c58fbde975637776",
"labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/config.json": "0768e4213b3a05725c0ef62c851d2c0bd902b102dfa134de749cfe2e992f5bad",
"labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/lock.json": "5a931c5b1058e58b3792de2f7a9303139778180b3efedc2d3b91081ef14679a3",
"labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/result.json": "22f50c59c4eac9f5f1a680572304f25a11270c3fb2caa66be584bbd9d7a3525b",
"labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/verifier/reward.json": "36a492513204040f04d7bdf4146d84ede71353d8cc7df112fc3ebff66327683a",
"labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/verifier/test-stdout.txt": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/result.json": "5596535bf83df1d66ae5fa4f6388734fd456a6825505e7f6ad11295552c48cb3",
"labs/20-deltawire/eval/results/range-large-v7-r1/readiness.json": "93124d41c2c3c66043bf5fa46673893faf31737bdfcada75656320a98ea543e9",
"labs/20-deltawire/eval/results/range-large-v7-r1/run-ledger.json": "8b4438d59fffcc5b193c68cae268d400f30200bc28b2156249d978f544b5f541",
"labs/20-deltawire/eval/results/range-large-v7-r1/terminal-report.json": "86dcc3abf7f998089ebdfed173322a7db96e28a36eac7eb9a5087630b89a85aa",
"labs/20-deltawire/eval/scripts/v7/generate_manifest.py": "60fbcd9a31514b94f606e87b7df07c4442259a9ed15c00f169b344675596b1c6",
"labs/20-deltawire/eval/scripts/v7/runner.py": "cc551f8f943f39c767c0cc88bc07f584972777b4d775ca586d3502eb653399b8",
"labs/20-deltawire/eval/scripts/v7/skill_conformance.py": "31617b88fef9185635ea3bc78f60368df8a894212a804c4f5e1d60b2365d9491",
"labs/20-deltawire/eval/scripts/v7/test_runner.py": "6e0ca10b09f1fd2624bbbe70892c9faf31ef630644f710ba9cd05fbaa92556ab",
"labs/20-deltawire/eval/scripts/v7/test_skill.py": "6fd6b49e06de82e4af7a97a449a9981fd860dca96c806c069834cf96287d288e",
"labs/20-deltawire/eval/scripts/v7/test_treatment.py": "4f2cb7744ce65d5cfe8ec1da8b1aca804609977a44c20fdbafa12397110409ce",
"labs/20-deltawire/eval/scripts/v7/treatment.py": "5587514cd678223105a2aff6b1a01f58fff6b6e138d06b9461db0fc7f77e7ee8",
"labs/20-deltawire/eval/scripts/v7/validate.py": "c96eadbac126fb6546816f83513e3a38dea0b29091b43fec620b929768e1e06a",
"labs/20-deltawire/eval/scripts/v7/validate.sh": "a24c183a3824df10cf5dffcdd07c0bd336b7d51705a9e311127587d56aa87200"
},
"schema_version": "deltawire-v7-terminal-evidence-lock.v1",
"scopes": [
"labs/20-deltawire/eval/manifests/deltawire-range-large-v7-pair.json",
"labs/20-deltawire/eval/pairs/range-large-v7-r1",
"labs/20-deltawire/eval/scripts/v7",
"labs/20-deltawire/eval/results/preflight-v7",
"labs/20-deltawire/eval/results/range-large-v7-r1"
],
"source_commit": "c340db338a2758fbb33efd84209d43e4e2fd6be3",
"terminal_classification": {
"B0": "withheld",
"D1_activation": "pass",
"D1_assignment": "pass",
"D1_contract_receipts": "fail",
"D1_end_to_end": "fail",
"D1_environment": "pass",
"D1_lifecycle": "pass",
"D1_plan_retention": "fail",
"D1_semantic": "pass",
"D1_treatment_fidelity": "fail",
"READY_FOR_72": false,
"pair_attempt_consumed": true,
"pair_valid": false,
"rerun_authorized": false
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"artifact_manifest_sha256": "0c8e85115c5bc9a68deb3aa39b84e4f734bdeb8b7742a97b72bc139b1192b386",
"checks": {
"binary_hash_frozen": true,
"config_hash_frozen": true,
"manifest_exists": true,
"manifest_status_ok": true,
"public_contract_hash_frozen": true,
"receipt_canonical_path": true,
"receipt_checks_complete": true,
"receipt_checks_true": true,
"receipt_schema_exact": true,
"receipt_status_pass": true,
"schema_hash_frozen": true,
"single_convention_entry": true
},
"convention_entry": {
"destination": "artifacts/logs/artifacts",
"service": null,
"source": "/logs/artifacts",
"status": "ok",
"type": "directory"
},
"environment_receipt_path": "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/logs/artifacts/deltawire/environment-receipt.json",
"environment_receipt_sha256": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1",
"schema_version": "artifact-manifest-receipt.v1",
"status": "pass"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
{
"READY_FOR_72": false,
"arm": "D1",
"component_checks": {
"artifact_manifest": true,
"environment": true,
"plan_contract": false,
"semantic": true,
"setup": true,
"treatment": false
},
"identity_checks": {
"agent_match": true,
"harbor_exit_zero": true,
"model_match": true,
"trial_completed": true
},
"schema_version": "deltawire-end-to-end-receipt.v7",
"source_hashes": {
"artifact_manifest": "40cf6ddc54e657621f177afcb92e88a94eec553ef2f29d829174d920b86ab469",
"environment": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1",
"output": "9d205b57ab166d23c81927899f99b6ccece261c679c96979c58fbde975637776",
"result": "22f50c59c4eac9f5f1a680572304f25a11270c3fb2caa66be584bbd9d7a3525b",
"semantic": "09654564d33620bb4b39ae7cb64af13243078013118e6595adaf3a40af59cba0",
"setup": "adca6d256d43f320bb542745bc37f58fcfa97db736eebcc2aa1ce885d84c5240",
"trajectory": "d6659287d950b37b9ea1364e6256d715ce0220b652c45689355b79e623be7af5",
"treatment": "298daa00fb33afe6f7a93b67028bcc243fe1ef809f58199f89b36d35438148aa"
},
"status": "fail"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
{
"binary": {
"actual_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4",
"executable": true,
"exists": true,
"expected_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4",
"expected_version": "deltawire version dev",
"is_regular": true,
"is_symlink": false,
"path": "/usr/local/bin/deltawire",
"realpath": "/usr/local/bin/deltawire",
"version_exit_code": 0,
"version_stderr": "",
"version_stdout": "deltawire version dev\nGo runtime version go1.26.5\nsupported config version deltawire.config.v1\nsupported plan version deltawire.plan.v1\nsupported state version deltawire.state.v1"
},
"checks": {
"binary_executable": true,
"binary_exists": true,
"binary_hash": true,
"binary_not_symlink": true,
"binary_path": true,
"binary_realpath": true,
"binary_regular": true,
"config_hash": true,
"public_contract_hash": true,
"schema_hash": true,
"version_exact": true,
"version_exit_0": true
},
"files": {
"config": {
"actual_sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f",
"exists": true,
"expected_sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f",
"hash_match": true,
"path": "/.deltawire/config.json"
},
"public_contract": {
"actual_sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5",
"exists": true,
"expected_sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5",
"hash_match": true,
"path": "/task-contract.json"
},
"schema": {
"actual_sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5",
"exists": true,
"expected_sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5",
"hash_match": true,
"path": "/.deltawire/schemas/range-large.schema.json"
}
},
"schema_version": "deltawire-environment-receipt.v1",
"status": "pass"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"checks": {},
"error": "expected one plan; found 0",
"schema_version": "plan-contract-receipt.v1",
"status": "fail"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"exact_match": 1,
"oracle": {
"exact_match": 1
},
"schema_version": "semantic-result.v7",
"status": "pass"
}
44 changes: 44 additions & 0 deletions labs/20-deltawire/eval/results/preflight-v7-terminal/D1-setup.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
{
"checks": {
"agent_exact": true,
"checks_exact": true,
"checks_true": true,
"gemini_path_exact": true,
"gemini_version_exact": true,
"image_id_exact": true,
"no_timestamps": true,
"node_path_exact": true,
"node_version_exact": true,
"npm_version_exact": true,
"package_version_exact": true,
"receipt_canonical_path": true,
"receipt_exists": true,
"schema_exact": true,
"status_pass": true
},
"receipt": {
"agent": "preinstalled-gemini-cli-v1",
"checks": {
"agent_identity": true,
"gemini_executable": true,
"gemini_version": true,
"image_id": true,
"node_executable": true,
"node_version": true,
"npm_version": true,
"package_identity": true
},
"gemini_path": "/usr/local/bin/gemini",
"gemini_version": "0.51.0",
"image_id": "sha256:589fce58d7ddf09910876afe0198bcee0086a840af6de7c08515bbce3e8367a7",
"node_path": "/usr/local/bin/node",
"node_version": "v22.23.1",
"npm_version": "10.9.8",
"package_version": "0.51.0",
"schema_version": "preinstalled-gemini-setup-receipt.v1",
"status": "pass"
},
"receipt_sha256": "a88d221e573265997797dcad31136aec0ed74a5bd37e50887595e5ea85cfa0d1",
"schema_version": "preinstalled-gemini-setup-validation.v1",
"status": "pass"
}
Loading
Loading