diff --git a/.github/workflows/deltawire-runtime.yml b/.github/workflows/deltawire-runtime.yml index be4c3525c..973ea160e 100644 --- a/.github/workflows/deltawire-runtime.yml +++ b/.github/workflows/deltawire-runtime.yml @@ -33,5 +33,12 @@ jobs: - run: python3 labs/20-deltawire/eval/scripts/v6/test_setup_receipt.py - run: python3 labs/20-deltawire/eval/scripts/v6/test_trial_status.py - run: python3 labs/20-deltawire/eval/scripts/v6/test_shell_observation.py + - run: python3 labs/20-deltawire/eval/scripts/v7/verify_terminal_evidence_lock.py + - run: python3 labs/20-deltawire/eval/scripts/v7/test_terminal_evidence_lock.py + - name: Run v7 tests in isolated processes + run: | + for test_file in labs/20-deltawire/eval/scripts/v7/test_*.py; do + python3 "$test_file" + done - run: python3 labs/20-deltawire/eval/scripts/v6/negative_controls.py --self-test - run: PYTHONPATH=labs/20-deltawire/eval uv run --with harbor==0.20.0 python labs/20-deltawire/eval/agents/test_preinstalled_gemini_cli.py diff --git a/.github/workflows/deltawire-v7.yml b/.github/workflows/deltawire-v7.yml index 0b0f02827..cc534b0ca 100644 --- a/.github/workflows/deltawire-v7.yml +++ b/.github/workflows/deltawire-v7.yml @@ -46,7 +46,15 @@ jobs: cmp /tmp/skill-conformance.json labs/20-deltawire/eval/results/preflight-v7/skill-conformance.json fi - name: No-live v7 validation - run: bash labs/20-deltawire/eval/scripts/v7/validate.sh + run: | + python3 labs/20-deltawire/eval/scripts/v6/verify_terminal_evidence_lock.py + python3 labs/20-deltawire/eval/scripts/v7/validate.py + python3 labs/20-deltawire/eval/scripts/v7/verify_terminal_evidence_lock.py + python3 labs/20-deltawire/eval/scripts/v7/test_terminal_evidence_lock.py + for test_file in labs/20-deltawire/eval/scripts/v7/test_*.py; do + DELTAWIRE_TEST_BINARY=labs/20-deltawire/eval/.generated/deltawire-linux-amd64 \ + python3 "$test_file" + done - name: Prove workflow has no model invocation run: | live_runner='runner.py .*pair'"-v7" diff --git a/labs/20-deltawire/eval/manifests/preflight-v7-terminal-evidence-lock.json b/labs/20-deltawire/eval/manifests/preflight-v7-terminal-evidence-lock.json new file mode 100644 index 000000000..97bd67a5d --- /dev/null +++ b/labs/20-deltawire/eval/manifests/preflight-v7-terminal-evidence-lock.json @@ -0,0 +1,93 @@ +{ + "files": { + "labs/20-deltawire/eval/manifests/deltawire-range-large-v7-pair.json": "67789b85c08802a87f5d14bb72aa2d36068c43a45aa27187ddab5d66dba3eca4", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/authoritative-schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/.generated/.gitignore": "240a3e0d37d2e86b614063f5347eb02d4f99ca6c254de6b82871ff8d95532a7d", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/Dockerfile": "cd768e64efba435604b265088bce6269e8dbd09e54bf51a4c5bf0dd94d10c05f", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/deltawire-environment-expectations.json": "03fe7a40964e908f2fb3736add8b8bea890b407d837f2b185d92bd8d85d4d114", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/environment_receipt.py": "3ab3bbcbfae2093ac658063d1e26f26b43c35dfdb81d8ddae5c9989f967ba623", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/task-contract.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/environment/verify_plan_contract.py": "8aafe73ab47213027a258f9abbebbca4803fa53f658abeb7b405d4a79202d4af", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/instruction.md": "549f4047b1a4ba4f7a3f3dce9e60d1d70b141caa189aafdba942b15e6d022aee", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/SKILL.md": "e7d3a1e8ccef1b72338d5dac7a7b7ac412164846018ab55b7f2ecadf5a07a89a", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/matrix.dw.json": "56d16a5b20086c8202cbc1bbdccf41d467a5d4c3806d701227ebcdbf72e54b1a", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/range.dw.json": "b31e078409547f4fbb5a855f3bf0b1bd6f52386696c07401622f34d224fe148b", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/rows.dw.json": "e172cc7d6224e4566a0cd46a73465392976338fea0e225512a089322ba81aca3", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/examples/variants.dw.json": "3183405b76e11874309728f3a4025687af6c54add8d2b97b94daff2d6ef8cdca", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire/references/plan-v1.md": "f78d41266db73592526a3c5e7d6e4c883f549f99b01c6387c7f5337fc269dd5e", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/task.toml": "4adc78ba9c59f8c743770ee9b1b31a55364e7303010e3c3f109da3b3d61d96e3", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/semantic_oracle.py": "94b30fac117f40e977851966b5228e6c372bb48cbf2e5c29c24f428e1f98e91d", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/task-spec.json": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "labs/20-deltawire/eval/pairs/range-large-v7-r1/tests/test.sh": "6f7cfbefdf7eaf3a082fd0c88f264ba3d41bfe84fc53498df4464bef66dc116e", + "labs/20-deltawire/eval/results/preflight-v7/pair-approval.json": "c6b493851951e72ead0d15e38231f47d036331f94ded94d9d3dceb38db7c3154", + "labs/20-deltawire/eval/results/preflight-v7/pair-security-scan.json": "4ab32a7795109d3973bd8665dbe60df485c6ac0af25b0b2e752e15a448650411", + "labs/20-deltawire/eval/results/preflight-v7/pre-live-report.json": "dcf77e739288edbd358470458582e960224fee620f92f0f19ea7415303b268d7", + "labs/20-deltawire/eval/results/preflight-v7/skill-conformance.json": "4d0e142a1bc801642b5de8c77b9cbebc9680a42f6f64da95b3d48b8624fe7c46", + "labs/20-deltawire/eval/results/range-large-v7-r1/D1-artifact-manifest.json": "40cf6ddc54e657621f177afcb92e88a94eec553ef2f29d829174d920b86ab469", + "labs/20-deltawire/eval/results/range-large-v7-r1/D1-end-to-end.json": "207e6e300158c9d87b92589c412a7ae15aa732549dbb7bdaa8654a366efc62e1", + "labs/20-deltawire/eval/results/range-large-v7-r1/D1-environment.json": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "labs/20-deltawire/eval/results/range-large-v7-r1/D1-plan-contract-receipt.json": "d3c6d569d9353120583551f280753e457683c2b886594cda291b93b4e30515a3", + "labs/20-deltawire/eval/results/range-large-v7-r1/D1-semantic-result.json": "09654564d33620bb4b39ae7cb64af13243078013118e6595adaf3a40af59cba0", + "labs/20-deltawire/eval/results/range-large-v7-r1/D1-setup.json": "adca6d256d43f320bb542745bc37f58fcfa97db736eebcc2aa1ce885d84c5240", + "labs/20-deltawire/eval/results/range-large-v7-r1/D1-treatment.json": "298daa00fb33afe6f7a93b67028bcc243fe1ef809f58199f89b36d35438148aa", + "labs/20-deltawire/eval/results/range-large-v7-r1/evidence-closure.json": "95dc20298e27fb2b295ac694688f7d8b14a78aaaa16c854e6642158e15f45240", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/config.json": "27de86441290364f9bc0c6c66854b4bc0194f591b909dbaf275b628f40471728", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/lock.json": "0a58888ac484406279a52b2109f9aa8215cc7890204b7ff2f17022bd5d703286", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/agent/gemini-cli.trajectory.jsonl": "d6659287d950b37b9ea1364e6256d715ce0220b652c45689355b79e623be7af5", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/agent/gemini-cli.txt": "54e1b5e6d019e43be0e7141827f3dccb32b4a613792551ca6b80e08452e9ee6c", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/agent/trajectory.json": "d6c4bae360f51d98c467028fa0354f351e8b98d6b157b19baae328ea5f514db5", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/.deltawire/config.json": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/.deltawire/schemas/range-large.schema.json": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/.deltawire/state.json": "5e1cf20f1d0945a30accb87152086fc61c848cdf18ac46ad2f0fa65fbd5b8c76", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/logs/artifacts/deltawire/environment-receipt.json": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/logs/artifacts/preinstalled-gemini/setup-receipt.json": "a88d221e573265997797dcad31136aec0ed74a5bd37e50887595e5ea85cfa0d1", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/manifest.json": "0c8e85115c5bc9a68deb3aa39b84e4f734bdeb8b7742a97b72bc139b1192b386", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/testdata/generated/output.ndjson": "9d205b57ab166d23c81927899f99b6ccece261c679c96979c58fbde975637776", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/config.json": "0768e4213b3a05725c0ef62c851d2c0bd902b102dfa134de749cfe2e992f5bad", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/lock.json": "5a931c5b1058e58b3792de2f7a9303139778180b3efedc2d3b91081ef14679a3", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/result.json": "22f50c59c4eac9f5f1a680572304f25a11270c3fb2caa66be584bbd9d7a3525b", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/verifier/reward.json": "36a492513204040f04d7bdf4146d84ede71353d8cc7df112fc3ebff66327683a", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/verifier/test-stdout.txt": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/result.json": "5596535bf83df1d66ae5fa4f6388734fd456a6825505e7f6ad11295552c48cb3", + "labs/20-deltawire/eval/results/range-large-v7-r1/readiness.json": "93124d41c2c3c66043bf5fa46673893faf31737bdfcada75656320a98ea543e9", + "labs/20-deltawire/eval/results/range-large-v7-r1/run-ledger.json": "8b4438d59fffcc5b193c68cae268d400f30200bc28b2156249d978f544b5f541", + "labs/20-deltawire/eval/results/range-large-v7-r1/terminal-report.json": "86dcc3abf7f998089ebdfed173322a7db96e28a36eac7eb9a5087630b89a85aa", + "labs/20-deltawire/eval/scripts/v7/generate_manifest.py": "60fbcd9a31514b94f606e87b7df07c4442259a9ed15c00f169b344675596b1c6", + "labs/20-deltawire/eval/scripts/v7/runner.py": "cc551f8f943f39c767c0cc88bc07f584972777b4d775ca586d3502eb653399b8", + "labs/20-deltawire/eval/scripts/v7/skill_conformance.py": "31617b88fef9185635ea3bc78f60368df8a894212a804c4f5e1d60b2365d9491", + "labs/20-deltawire/eval/scripts/v7/test_runner.py": "6e0ca10b09f1fd2624bbbe70892c9faf31ef630644f710ba9cd05fbaa92556ab", + "labs/20-deltawire/eval/scripts/v7/test_skill.py": "6fd6b49e06de82e4af7a97a449a9981fd860dca96c806c069834cf96287d288e", + "labs/20-deltawire/eval/scripts/v7/test_treatment.py": "4f2cb7744ce65d5cfe8ec1da8b1aca804609977a44c20fdbafa12397110409ce", + "labs/20-deltawire/eval/scripts/v7/treatment.py": "5587514cd678223105a2aff6b1a01f58fff6b6e138d06b9461db0fc7f77e7ee8", + "labs/20-deltawire/eval/scripts/v7/validate.py": "c96eadbac126fb6546816f83513e3a38dea0b29091b43fec620b929768e1e06a", + "labs/20-deltawire/eval/scripts/v7/validate.sh": "a24c183a3824df10cf5dffcdd07c0bd336b7d51705a9e311127587d56aa87200" + }, + "schema_version": "deltawire-v7-terminal-evidence-lock.v1", + "scopes": [ + "labs/20-deltawire/eval/manifests/deltawire-range-large-v7-pair.json", + "labs/20-deltawire/eval/pairs/range-large-v7-r1", + "labs/20-deltawire/eval/scripts/v7", + "labs/20-deltawire/eval/results/preflight-v7", + "labs/20-deltawire/eval/results/range-large-v7-r1" + ], + "source_commit": "c340db338a2758fbb33efd84209d43e4e2fd6be3", + "terminal_classification": { + "B0": "withheld", + "D1_activation": "pass", + "D1_assignment": "pass", + "D1_contract_receipts": "fail", + "D1_end_to_end": "fail", + "D1_environment": "pass", + "D1_lifecycle": "pass", + "D1_plan_retention": "fail", + "D1_semantic": "pass", + "D1_treatment_fidelity": "fail", + "READY_FOR_72": false, + "pair_attempt_consumed": true, + "pair_valid": false, + "rerun_authorized": false + } +} diff --git a/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-artifact-manifest.json b/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-artifact-manifest.json new file mode 100644 index 000000000..279615f53 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-artifact-manifest.json @@ -0,0 +1,28 @@ +{ + "artifact_manifest_sha256": "0c8e85115c5bc9a68deb3aa39b84e4f734bdeb8b7742a97b72bc139b1192b386", + "checks": { + "binary_hash_frozen": true, + "config_hash_frozen": true, + "manifest_exists": true, + "manifest_status_ok": true, + "public_contract_hash_frozen": true, + "receipt_canonical_path": true, + "receipt_checks_complete": true, + "receipt_checks_true": true, + "receipt_schema_exact": true, + "receipt_status_pass": true, + "schema_hash_frozen": true, + "single_convention_entry": true + }, + "convention_entry": { + "destination": "artifacts/logs/artifacts", + "service": null, + "source": "/logs/artifacts", + "status": "ok", + "type": "directory" + }, + "environment_receipt_path": "labs/20-deltawire/eval/results/range-large-v7-r1/raw/D1/2026-07-22__19-57-08/range-large-v7-r1__3wEHYmv/artifacts/logs/artifacts/deltawire/environment-receipt.json", + "environment_receipt_sha256": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "schema_version": "artifact-manifest-receipt.v1", + "status": "pass" +} diff --git a/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-end-to-end.json b/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-end-to-end.json new file mode 100644 index 000000000..d7d22f367 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-end-to-end.json @@ -0,0 +1,30 @@ +{ + "READY_FOR_72": false, + "arm": "D1", + "component_checks": { + "artifact_manifest": true, + "environment": true, + "plan_contract": false, + "semantic": true, + "setup": true, + "treatment": false + }, + "identity_checks": { + "agent_match": true, + "harbor_exit_zero": true, + "model_match": true, + "trial_completed": true + }, + "schema_version": "deltawire-end-to-end-receipt.v7", + "source_hashes": { + "artifact_manifest": "40cf6ddc54e657621f177afcb92e88a94eec553ef2f29d829174d920b86ab469", + "environment": "4a30e8e33d3dee9c4fcb4761da3e10c2a27c5dc4c2aca296e185e1eb758620b1", + "output": "9d205b57ab166d23c81927899f99b6ccece261c679c96979c58fbde975637776", + "result": "22f50c59c4eac9f5f1a680572304f25a11270c3fb2caa66be584bbd9d7a3525b", + "semantic": "09654564d33620bb4b39ae7cb64af13243078013118e6595adaf3a40af59cba0", + "setup": "adca6d256d43f320bb542745bc37f58fcfa97db736eebcc2aa1ce885d84c5240", + "trajectory": "d6659287d950b37b9ea1364e6256d715ce0220b652c45689355b79e623be7af5", + "treatment": "298daa00fb33afe6f7a93b67028bcc243fe1ef809f58199f89b36d35438148aa" + }, + "status": "fail" +} diff --git a/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-environment.json b/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-environment.json new file mode 100644 index 000000000..55108c6ac --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-environment.json @@ -0,0 +1,55 @@ +{ + "binary": { + "actual_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "executable": true, + "exists": true, + "expected_sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4", + "expected_version": "deltawire version dev", + "is_regular": true, + "is_symlink": false, + "path": "/usr/local/bin/deltawire", + "realpath": "/usr/local/bin/deltawire", + "version_exit_code": 0, + "version_stderr": "", + "version_stdout": "deltawire version dev\nGo runtime version go1.26.5\nsupported config version deltawire.config.v1\nsupported plan version deltawire.plan.v1\nsupported state version deltawire.state.v1" + }, + "checks": { + "binary_executable": true, + "binary_exists": true, + "binary_hash": true, + "binary_not_symlink": true, + "binary_path": true, + "binary_realpath": true, + "binary_regular": true, + "config_hash": true, + "public_contract_hash": true, + "schema_hash": true, + "version_exact": true, + "version_exit_0": true + }, + "files": { + "config": { + "actual_sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "exists": true, + "expected_sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "hash_match": true, + "path": "/.deltawire/config.json" + }, + "public_contract": { + "actual_sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "exists": true, + "expected_sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5", + "hash_match": true, + "path": "/task-contract.json" + }, + "schema": { + "actual_sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "exists": true, + "expected_sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "hash_match": true, + "path": "/.deltawire/schemas/range-large.schema.json" + } + }, + "schema_version": "deltawire-environment-receipt.v1", + "status": "pass" +} diff --git a/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-plan-contract-receipt.json b/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-plan-contract-receipt.json new file mode 100644 index 000000000..f1e010ed5 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-plan-contract-receipt.json @@ -0,0 +1,6 @@ +{ + "checks": {}, + "error": "expected one plan; found 0", + "schema_version": "plan-contract-receipt.v1", + "status": "fail" +} diff --git a/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-semantic-result.json b/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-semantic-result.json new file mode 100644 index 000000000..14b571937 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-semantic-result.json @@ -0,0 +1,8 @@ +{ + "exact_match": 1, + "oracle": { + "exact_match": 1 + }, + "schema_version": "semantic-result.v7", + "status": "pass" +} diff --git a/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-setup.json b/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-setup.json new file mode 100644 index 000000000..2e3d844f2 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-setup.json @@ -0,0 +1,44 @@ +{ + "checks": { + "agent_exact": true, + "checks_exact": true, + "checks_true": true, + "gemini_path_exact": true, + "gemini_version_exact": true, + "image_id_exact": true, + "no_timestamps": true, + "node_path_exact": true, + "node_version_exact": true, + "npm_version_exact": true, + "package_version_exact": true, + "receipt_canonical_path": true, + "receipt_exists": true, + "schema_exact": true, + "status_pass": true + }, + "receipt": { + "agent": "preinstalled-gemini-cli-v1", + "checks": { + "agent_identity": true, + "gemini_executable": true, + "gemini_version": true, + "image_id": true, + "node_executable": true, + "node_version": true, + "npm_version": true, + "package_identity": true + }, + "gemini_path": "/usr/local/bin/gemini", + "gemini_version": "0.51.0", + "image_id": "sha256:589fce58d7ddf09910876afe0198bcee0086a840af6de7c08515bbce3e8367a7", + "node_path": "/usr/local/bin/node", + "node_version": "v22.23.1", + "npm_version": "10.9.8", + "package_version": "0.51.0", + "schema_version": "preinstalled-gemini-setup-receipt.v1", + "status": "pass" + }, + "receipt_sha256": "a88d221e573265997797dcad31136aec0ed74a5bd37e50887595e5ea85cfa0d1", + "schema_version": "preinstalled-gemini-setup-validation.v1", + "status": "pass" +} diff --git a/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-treatment.json b/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-treatment.json new file mode 100644 index 000000000..bbe547126 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v7-terminal/D1-treatment.json @@ -0,0 +1,131 @@ +{ + "arm": "D1", + "artifact_hashes": { + "config": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f", + "output": "9d205b57ab166d23c81927899f99b6ccece261c679c96979c58fbde975637776", + "schema": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5", + "state": "5e1cf20f1d0945a30accb87152086fc61c848cdf18ac46ad2f0fa65fbd5b8c76" + }, + "assignment": { + "checks": { + "config_exactly_one_skill": true, + "config_skill_name": true, + "local_tree_hash": true, + "lock_exactly_one_skill": true, + "lock_skill_digest": true, + "lock_skill_name": true + }, + "status": "pass" + }, + "checks": { + "assignment_pass": true, + "check_invoked": true, + "check_not_proven_failed": true, + "in_container_contract_pass": false, + "independent_contract_pass": false, + "public_contract_access": false, + "render_invoked": true, + "render_not_proven_failed": true, + "retained_config": true, + "retained_in_container_contract": false, + "retained_output": true, + "retained_plan": false, + "retained_schema": true, + "retained_state": true, + "schema_integrity": true, + "semantic_exact_pass": true, + "structured_activation": true, + "validate_invoked": true, + "validate_not_proven_failed": true + }, + "classification": "attempted_failed", + "contamination": false, + "diagnostics": { + "binary_introspection": false, + "direct_skill_read": true, + "direct_validate_count": 1, + "failed_direct_validate_count": 0, + "gemini_skills_list_observed": false, + "matching_example_read": true, + "reference_read": true, + "subprocess_bruteforce_loops": 0 + }, + "lifecycle_invocations": [ + { + "command": "validate", + "invocation": "deltawire validate --repo . plan.json", + "shell_exit_code": null + }, + { + "command": "render", + "invocation": "deltawire render --repo . plan.json", + "shell_exit_code": null + }, + { + "command": "check", + "invocation": "deltawire check --repo . plan.json", + "shell_exit_code": null + } + ], + "schema_version": "deltawire-treatment-receipt.v7", + "shell_observations": [ + { + "command": "deltawire validate --repo . plan.json", + "observable": true, + "shell_exit_code": null, + "shell_exit_provenance": null, + "tool_status": "success" + }, + { + "command": "find / -maxdepth 3 -name \"*contract*check*\" -o -name \"*check*contract*\" 2>/dev/null", + "observable": true, + "shell_exit_code": null, + "shell_exit_provenance": null, + "tool_status": "success" + }, + { + "command": "ls -l /usr/local/bin | grep check", + "observable": true, + "shell_exit_code": 1, + "shell_exit_provenance": "explicit_output", + "tool_status": "success" + }, + { + "command": "ls -l /usr/local/bin", + "observable": true, + "shell_exit_code": null, + "shell_exit_provenance": null, + "tool_status": "success" + }, + { + "command": "verify_plan_contract.py plan.json", + "observable": true, + "shell_exit_code": 2, + "shell_exit_provenance": "explicit_output", + "tool_status": "success" + }, + { + "command": "verify_plan_contract.py --contract task-contract.json --plan plan.json", + "observable": true, + "shell_exit_code": null, + "shell_exit_provenance": null, + "tool_status": "success" + }, + { + "command": "deltawire render --repo . plan.json", + "observable": true, + "shell_exit_code": null, + "shell_exit_provenance": null, + "tool_status": "success" + }, + { + "command": "deltawire check --repo . plan.json", + "observable": true, + "shell_exit_code": null, + "shell_exit_provenance": null, + "tool_status": "success" + } + ], + "status": "fail", + "trajectory_sha256": "d6659287d950b37b9ea1364e6256d715ce0220b652c45689355b79e623be7af5" +} diff --git a/labs/20-deltawire/eval/results/preflight-v7-terminal/readiness.json b/labs/20-deltawire/eval/results/preflight-v7-terminal/readiness.json new file mode 100644 index 000000000..6531e477f --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v7-terminal/readiness.json @@ -0,0 +1,20 @@ +{ + "B0_launched": false, + "D1_launched": true, + "READY_FOR_72": false, + "blockers": [ + "v7 D1 treatment fidelity failed", + "v7 B0 was withheld", + "no valid causal pair", + "no six-run canary", + "no final confirmatory manifest", + "no explicit 72-run authorization" + ], + "full_run_started": false, + "pair_complete": false, + "pair_launched": true, + "pair_valid": false, + "rerun_authorized": false, + "schema_version": "deltawire-readiness.v7", + "six_run_canary_started": false +} diff --git a/labs/20-deltawire/eval/results/preflight-v7-terminal/security-scan.json b/labs/20-deltawire/eval/results/preflight-v7-terminal/security-scan.json new file mode 100644 index 000000000..470e2b88a --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v7-terminal/security-scan.json @@ -0,0 +1,11 @@ +{ + "active_pair_processes": 0, + "approval_sha256": "c6b493851951e72ead0d15e38231f47d036331f94ded94d9d3dceb38db7c3154", + "bytes_scanned": 143725, + "exact_credential_matches": 0, + "files_scanned": 29, + "generic_secret_matches": 0, + "schema_version": "deltawire-v7-security-scan.v1", + "status": "pass", + "temporary_credential_files_remaining": 0 +} diff --git a/labs/20-deltawire/eval/results/preflight-v7-terminal/summary.json b/labs/20-deltawire/eval/results/preflight-v7-terminal/summary.json new file mode 100644 index 000000000..137fd93f5 --- /dev/null +++ b/labs/20-deltawire/eval/results/preflight-v7-terminal/summary.json @@ -0,0 +1,58 @@ +{ + "B0": { + "launched": false, + "reason": "withheld after D1 end-to-end failure", + "trial_count": 0 + }, + "D1": { + "activation": true, + "agent": "preinstalled-gemini-cli-v1", + "artifact_manifest_status": "pass", + "assignment_status": "pass", + "end_to_end_status": "fail", + "environment_status": "pass", + "failure_reasons": [ + "plan was created as plan.json instead of retained .deltawire/plans/range-large.dw.json", + "in-container plan-contract receipt was not retained", + "independent retained-plan contract check found zero canonical plans", + "structured public-contract access gate was not proven" + ], + "harbor_exit_code": 0, + "mechanism": { + "binary_introspection": false, + "contract_checker_failed_attempts": 1, + "direct_validates": 1, + "failed_direct_validates": 0, + "lifecycle_calls": 3, + "matching_example_read": true, + "reference_read": true, + "subprocess_bruteforce_loops": 0, + "tool_calls": 18 + }, + "model": "gemini-3.1-pro-preview", + "semantic_status": "pass", + "setup_status": "pass", + "treatment_status": "fail", + "trial_exception_type": null, + "trial_id": "db50c949-e6b2-4fcf-8ba0-3d0f56b0109e", + "usage": { + "cache_tokens": 112782, + "harbor_reported_cost": 0.1736744, + "input_plus_output_tokens": 173291, + "input_tokens": 170281, + "output_tokens": 3010, + "wall_time_seconds": 64.163724 + } + }, + "READY_FOR_72": false, + "claim": "no causal comparison; D1-only failed-fidelity observation", + "comparison_available": false, + "full_run_started": false, + "outcome": "D1_semantic_pass_treatment_fidelity_fail_B0_withheld", + "pair_attempt_consumed": true, + "pair_id": "range-large/v7-r1", + "pair_valid": false, + "rerun_authorized": false, + "schema_version": "deltawire-v7-terminal-report.v1", + "six_run_canary_started": false +} diff --git a/labs/20-deltawire/eval/scripts/v7/generate_terminal_evidence_lock.py b/labs/20-deltawire/eval/scripts/v7/generate_terminal_evidence_lock.py new file mode 100644 index 000000000..4a2c176bb --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v7/generate_terminal_evidence_lock.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Generate the Git-object-backed terminal v7 evidence lock.""" +import argparse +import hashlib +import json +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[5] +COMMIT = "c340db338a2758fbb33efd84209d43e4e2fd6be3" +SCOPES = [ + "labs/20-deltawire/eval/manifests/deltawire-range-large-v7-pair.json", + "labs/20-deltawire/eval/pairs/range-large-v7-r1", + "labs/20-deltawire/eval/scripts/v7", + "labs/20-deltawire/eval/results/preflight-v7", + "labs/20-deltawire/eval/results/range-large-v7-r1", +] + + +def git(*args: str) -> bytes: + return subprocess.check_output(["git", *args], cwd=ROOT) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--output", required=True) + args = parser.parse_args() + names = set() + for scope in SCOPES: + names.update(git("ls-tree", "-r", "--name-only", COMMIT, "--", scope).decode().splitlines()) + names.discard("") + value = { + "schema_version": "deltawire-v7-terminal-evidence-lock.v1", + "source_commit": COMMIT, + "scopes": SCOPES, + "files": {name: hashlib.sha256(git("show", f"{COMMIT}:{name}")).hexdigest() for name in sorted(names)}, + "terminal_classification": { + "D1_assignment": "pass", + "D1_activation": "pass", + "D1_environment": "pass", + "D1_lifecycle": "pass", + "D1_semantic": "pass", + "D1_plan_retention": "fail", + "D1_contract_receipts": "fail", + "D1_treatment_fidelity": "fail", + "D1_end_to_end": "fail", + "B0": "withheld", + "pair_attempt_consumed": True, + "pair_valid": False, + "rerun_authorized": False, + "READY_FOR_72": False, + }, + } + target = Path(args.output) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + print(f"locked {len(names)} terminal v7 files") + + +if __name__ == "__main__": + main() diff --git a/labs/20-deltawire/eval/scripts/v7/test_skill.py b/labs/20-deltawire/eval/scripts/v7/test_skill.py index 109d63ef2..30582645a 100644 --- a/labs/20-deltawire/eval/scripts/v7/test_skill.py +++ b/labs/20-deltawire/eval/scripts/v7/test_skill.py @@ -9,7 +9,7 @@ HERE = Path(__file__).resolve().parent ROOT = HERE.parents[4] -SKILL = ROOT / "labs/20-deltawire/eval/pairs/range-large-v7-r1/skill/deltawire" +SKILL = ROOT / "labs/20-deltawire/eval/skills/deltawire" class SkillBundleTest(unittest.TestCase): @@ -21,7 +21,9 @@ def test_frontmatter_and_host_neutral_guidance(self): self.assertNotIn("gemini skills list", lowered) self.assertNotIn("range-large", lowered) self.assertNotIn("500", lowered) - self.assertIn("not inspect or reverse\nengineer", lowered) + self.assertIn("not generate expanded output manually or inspect/reverse", lowered) + self.assertIn(".deltawire/plans/.dw.json", lowered) + self.assertIn(".deltawire/plan-contract-receipt.json", lowered) self.assertIn("references/plan-v1.md", text) for name in ("range", "matrix", "rows", "variants"): self.assertIn(f"examples/{name}.dw.json", text) diff --git a/labs/20-deltawire/eval/scripts/v7/test_terminal_evidence_lock.py b/labs/20-deltawire/eval/scripts/v7/test_terminal_evidence_lock.py new file mode 100644 index 000000000..0ee3e7df9 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v7/test_terminal_evidence_lock.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +import copy +import json + +import verify_terminal_evidence_lock as verifier + + +def rejected(value): + try: + verifier.verify(value) + except RuntimeError: + return True + return False + + +def main(): + value = json.loads(verifier.LOCK.read_text()) + verifier.verify(value) + changed = copy.deepcopy(value) + changed["terminal_classification"]["pair_valid"] = True + assert rejected(changed) + changed = copy.deepcopy(value) + name = next(iter(changed["files"])) + changed["files"][name] = "0" * 64 + assert rejected(changed) + changed = copy.deepcopy(value) + changed["files"].pop(name) + assert rejected(changed) + print("v7 terminal evidence-lock mutation tests passed.") + + +if __name__ == "__main__": + main() diff --git a/labs/20-deltawire/eval/scripts/v7/test_validate_skill.py b/labs/20-deltawire/eval/scripts/v7/test_validate_skill.py new file mode 100644 index 000000000..b1c7519a4 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v7/test_validate_skill.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +import tempfile +import unittest +from pathlib import Path + +import sys +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from validate_skill import validate + +ROOT = Path(__file__).resolve().parents[5] +SKILL = ROOT / "labs/20-deltawire/eval/skills/deltawire/SKILL.md" + + +class SkillValidatorTest(unittest.TestCase): + def check_rejected(self, old, new): + with tempfile.TemporaryDirectory() as raw: + target = Path(raw) / "SKILL.md" + text = SKILL.read_text() + self.assertIn(old, text) + target.write_text(text.replace(old, new)) + self.assertTrue(validate(target)) + + def test_current_skill_passes(self): + self.assertEqual(validate(SKILL), []) + + def test_host_specific_discovery_rejected(self): + self.check_rejected("Read `/task-contract.json`", "Run `gemini skills list`. Read `/task-contract.json`") + + def test_missing_canonical_plan_rejected(self): + self.check_rejected(".deltawire/plans/.dw.json", "plan.json") + + def test_task_specific_answer_rejected(self): + self.check_rejected("integer sequences", "500 range-large integer sequences") + + def test_manual_output_or_binary_introspection_guidance_rejected(self): + self.check_rejected("Do not generate expanded output manually or inspect/reverse", "Generate expanded output manually and inspect/reverse") + + +if __name__ == "__main__": + unittest.main() diff --git a/labs/20-deltawire/eval/scripts/v7/verify_terminal_evidence_lock.py b/labs/20-deltawire/eval/scripts/v7/verify_terminal_evidence_lock.py new file mode 100644 index 000000000..eb1079d65 --- /dev/null +++ b/labs/20-deltawire/eval/scripts/v7/verify_terminal_evidence_lock.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Verify terminal v7 evidence from Git objects and compact local projections.""" +import hashlib +import json +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[5] +LOCK = ROOT / "labs/20-deltawire/eval/manifests/preflight-v7-terminal-evidence-lock.json" +COMMIT = "c340db338a2758fbb33efd84209d43e4e2fd6be3" +EXPECTED = { + "D1_assignment": "pass", "D1_activation": "pass", "D1_environment": "pass", + "D1_lifecycle": "pass", "D1_semantic": "pass", "D1_plan_retention": "fail", + "D1_contract_receipts": "fail", "D1_treatment_fidelity": "fail", + "D1_end_to_end": "fail", "B0": "withheld", "pair_attempt_consumed": True, + "pair_valid": False, "rerun_authorized": False, "READY_FOR_72": False, +} +PROJECTIONS = { + "D1-artifact-manifest.json": "labs/20-deltawire/eval/results/range-large-v7-r1/D1-artifact-manifest.json", + "D1-end-to-end.json": "labs/20-deltawire/eval/results/range-large-v7-r1/D1-end-to-end.json", + "D1-environment.json": "labs/20-deltawire/eval/results/range-large-v7-r1/D1-environment.json", + "D1-plan-contract-receipt.json": "labs/20-deltawire/eval/results/range-large-v7-r1/D1-plan-contract-receipt.json", + "D1-semantic-result.json": "labs/20-deltawire/eval/results/range-large-v7-r1/D1-semantic-result.json", + "D1-setup.json": "labs/20-deltawire/eval/results/range-large-v7-r1/D1-setup.json", + "D1-treatment.json": "labs/20-deltawire/eval/results/range-large-v7-r1/D1-treatment.json", + "summary.json": "labs/20-deltawire/eval/results/range-large-v7-r1/terminal-report.json", + "readiness.json": "labs/20-deltawire/eval/results/range-large-v7-r1/readiness.json", + "security-scan.json": "labs/20-deltawire/eval/results/preflight-v7/pair-security-scan.json", +} + + +def git(*args: str) -> bytes: + return subprocess.check_output(["git", *args], cwd=ROOT) + + +def verify(value: dict) -> int: + if value.get("schema_version") != "deltawire-v7-terminal-evidence-lock.v1": + raise RuntimeError("v7 terminal lock schema mismatch") + if value.get("source_commit") != COMMIT or value.get("terminal_classification") != EXPECTED: + raise RuntimeError("v7 terminal identity or classification mismatch") + git("rev-parse", f"{COMMIT}^{{tree}}") + found = set() + for scope in value.get("scopes", []): + found.update(git("ls-tree", "-r", "--name-only", COMMIT, "--", scope).decode().splitlines()) + found.discard("") + recorded = value.get("files", {}) + if set(recorded) != found: + raise RuntimeError("v7 terminal evidence closure mismatch") + for name, wanted in recorded.items(): + if hashlib.sha256(git("show", f"{COMMIT}:{name}")).hexdigest() != wanted: + raise RuntimeError(f"v7 Git-object mismatch: {name}") + compact = ROOT / "labs/20-deltawire/eval/results/preflight-v7-terminal" + for local_name, source_name in PROJECTIONS.items(): + source = git("show", f"{COMMIT}:{source_name}") + if local_name == "D1-artifact-manifest.json": + normalized = json.loads(source) + path = normalized["environment_receipt_path"] + normalized["environment_receipt_path"] = "labs/" + path.split("/labs/", 1)[1] + source = (json.dumps(normalized, indent=2, sort_keys=True) + "\n").encode() + if (compact / local_name).read_bytes() != source: + raise RuntimeError(f"v7 compact projection mismatch: {local_name}") + return len(recorded) + + +def main() -> None: + count = verify(json.loads(LOCK.read_text())) + print(f"v7 terminal evidence lock passed ({count} Git-object files, {len(PROJECTIONS)} compact projections).") + + +if __name__ == "__main__": + main() diff --git a/labs/20-deltawire/eval/scripts/validate_dataset.sh b/labs/20-deltawire/eval/scripts/validate_dataset.sh index a96eb2f33..eb7c2a812 100755 --- a/labs/20-deltawire/eval/scripts/validate_dataset.sh +++ b/labs/20-deltawire/eval/scripts/validate_dataset.sh @@ -21,9 +21,16 @@ python3 labs/20-deltawire/eval/scripts/v5/verify_evidence_locks.py python3 labs/20-deltawire/eval/scripts/v6/verify_terminal_evidence_lock.py python3 labs/20-deltawire/eval/scripts/v6/test_terminal_evidence_lock.py python3 labs/20-deltawire/eval/scripts/v6/test_release.py -if [ -d labs/20-deltawire/eval/scripts/v7 ]; then - bash labs/20-deltawire/eval/scripts/v7/validate.sh -fi +python3 labs/20-deltawire/eval/scripts/v7/verify_terminal_evidence_lock.py +python3 labs/20-deltawire/eval/scripts/v7/test_terminal_evidence_lock.py +for test_file in labs/20-deltawire/eval/scripts/v7/test_*.py; do + if [ "$(uname -s)" = "Linux" ]; then + DELTAWIRE_TEST_BINARY=labs/20-deltawire/eval/.generated/deltawire-linux-amd64 \ + python3 "$test_file" + else + python3 "$test_file" + fi +done python3 labs/20-deltawire/eval/scripts/v5/test_environment_receipt.py python3 labs/20-deltawire/eval/scripts/v5/test_artifact_manifest.py python3 labs/20-deltawire/eval/scripts/v5/validate_integration.py diff --git a/labs/20-deltawire/eval/scripts/validate_skill.py b/labs/20-deltawire/eval/scripts/validate_skill.py index 46c488869..e3fd92913 100755 --- a/labs/20-deltawire/eval/scripts/validate_skill.py +++ b/labs/20-deltawire/eval/scripts/validate_skill.py @@ -5,22 +5,25 @@ REQUIRED_BODY = { - "discovery": "gemini skills list", - "availability": "command -v deltawire", - "version": "deltawire version", - "applicability": "compact deterministic description", - "abstention": "too irregular", "public contract": "/task-contract.json", - "immutable schema": "schema is immutable", - "plan": ".deltawire/plans/", + "reference": "references/plan-v1.md", + "range example": "examples/range.dw.json", + "matrix example": "examples/matrix.dw.json", + "rows example": "examples/rows.dw.json", + "variants example": "examples/variants.dw.json", + "immutable config and schema": "do not edit the installed config or schema", + "canonical plan": ".deltawire/plans/.dw.json", "validate": "deltawire validate", "contract check": "verify_plan_contract.py", + "retained contract receipt": ".deltawire/plan-contract-receipt.json", "render": "deltawire render", "check": "deltawire check", - "manual generation prohibition": "manual expanded-data generation", - "schema-edit prohibition": "schema edits", + "manual generation prohibition": "do not generate expanded output manually", + "binary introspection prohibition": "inspect/reverse engineer the binary", + "hidden verifier prohibition": "do not access hidden verifier files", "oracle": "semantic oracle", } +FORBIDDEN = ("gemini skills list", "command -v deltawire", "type -p deltawire", "range-large", "500") def validate(path: Path) -> list[str]: @@ -39,14 +42,17 @@ def validate(path: Path) -> list[str]: errors.append("missing name") if not fields.get("description"): errors.append("missing description") - elif "gemini skills list" not in fields["description"] or "before activating" not in fields["description"].lower(): - errors.append("frontmatter must require pre-activation skill discovery") + elif "compact exact plan" not in fields["description"].lower() or "too irregular" not in fields["description"].lower(): + errors.append("frontmatter must define applicability and abstention") if not body.strip(): errors.append("empty body") - lower = body.lower() + lower = " ".join(body.lower().split()) for label, guidance in REQUIRED_BODY.items(): if guidance not in lower: errors.append(f"missing {label} guidance") + for forbidden in FORBIDDEN: + if forbidden in " ".join(text.lower().split()): + errors.append(f"forbidden host- or task-specific guidance: {forbidden}") return errors diff --git a/labs/20-deltawire/eval/skills/deltawire/SKILL.md b/labs/20-deltawire/eval/skills/deltawire/SKILL.md index 597e9532e..041dff872 100644 --- a/labs/20-deltawire/eval/skills/deltawire/SKILL.md +++ b/labs/20-deltawire/eval/skills/deltawire/SKILL.md @@ -1,25 +1,21 @@ --- name: deltawire -description: Before activating, use the shell to run `gemini skills list` and confirm `deltawire` is listed. Then use this skill for deterministic generation of large repetitive JSON or NDJSON datasets from Cartesian products, ranges, repeated rows, variants, or regular-plus-exception patterns. Do not use for very small outputs or highly irregular records. +description: Generate large deterministic JSON or NDJSON datasets from compact ranges, Cartesian products, repeated rows, variants, defaults, and regular exceptions. Use when the requested output has a compact exact plan; abstain when records are small or too irregular for a deterministic description. --- # DeltaWire -Before activating this skill, discovery must be proven by a successful -structured shell call to `gemini skills list` whose output lists `deltawire`. -Before choosing DeltaWire, run `command -v deltawire` and `deltawire version`. -Use it when a dataset has a compact deterministic description: Cartesian -products, numeric ranges, repeated rows, systematic variants, or a regular -base plus explicit exceptions. Abstain for very small outputs and records that -are too irregular to express compactly; state that reason explicitly. +Read `/task-contract.json`, then read `references/plan-v1.md` and the matching +bundled example. Use `examples/range.dw.json` for integer sequences, +`examples/matrix.dw.json` for Cartesian products, `examples/rows.dw.json` for +explicit rows, or `examples/variants.dw.json` for base records with small +changes. -Read the authoritative public contract at `/task-contract.json`. Inspect the -preinstalled schema named by `authoritative_schema.path`, and prove its SHA-256 -matches `authoritative_schema.sha256`. The schema is immutable: do not create, -replace, or edit it. Create only the compact plan under `.deltawire/plans/`, -binding its schema, output, record count, and assertions to the public contract. -Also prove the SHA-256 of `/task-contract.json` and `.deltawire/config.json`; -both are immutable and must remain byte-identical to their preinstalled copies. +Preserve the contract's output, format, schema path/hash, count, fields, values, +and order. Do not edit the installed config or schema. Create exactly one plan +at `.deltawire/plans/.dw.json`; a root-level or temporary plan is not a +retained plan. Do not generate expanded output manually or inspect/reverse +engineer the binary. Resolve errors from the reference and matching example. Run the complete lifecycle: @@ -28,6 +24,7 @@ Run the complete lifecycle: 3. `deltawire render --repo . ` 4. `deltawire check --repo . ` -After choosing DeltaWire, manual expanded-data generation, manual output edits, -schema edits, and access to hidden verifier files are prohibited. The task's -independent semantic oracle remains the final correctness authority. +Stop only after the canonical plan, config, schema, state, output, and +`.deltawire/plan-contract-receipt.json` are retained and all four steps pass. +Do not access hidden verifier files. The independent semantic oracle remains +the final correctness authority. diff --git a/labs/20-deltawire/eval/skills/deltawire/examples/matrix.dw.json b/labs/20-deltawire/eval/skills/deltawire/examples/matrix.dw.json new file mode 100644 index 000000000..ca1c24521 --- /dev/null +++ b/labs/20-deltawire/eval/skills/deltawire/examples/matrix.dw.json @@ -0,0 +1,17 @@ +{ + "version": "deltawire.plan.v1", + "id": "matrix-example", + "description": "A small Cartesian product", + "record_schema": ".deltawire/schemas/example.schema.json", + "output": {"path": "testdata/generated/matrix.ndjson", "format": "ndjson", "pretty": false}, + "generators": [{ + "kind": "matrix", + "name": "requests", + "dimensions": [ + {"name": "role", "values": ["guest", "member"]}, + {"name": "method", "values": ["GET", "POST"]} + ], + "record": {"role": "${role}", "method": "${method}"} + }], + "assertions": {"count": 4} +} diff --git a/labs/20-deltawire/eval/skills/deltawire/examples/range.dw.json b/labs/20-deltawire/eval/skills/deltawire/examples/range.dw.json new file mode 100644 index 000000000..9c467bf42 --- /dev/null +++ b/labs/20-deltawire/eval/skills/deltawire/examples/range.dw.json @@ -0,0 +1,14 @@ +{ + "version": "deltawire.plan.v1", + "id": "range-example", + "description": "Three ordered integer records", + "record_schema": ".deltawire/schemas/example.schema.json", + "output": {"path": "testdata/generated/range.ndjson", "format": "ndjson", "pretty": false}, + "generators": [{ + "kind": "matrix", + "name": "records", + "dimensions": [{"name": "index", "range": {"start": 1, "end_exclusive": 4, "step": 1}}], + "record": {"index": "${index}"} + }], + "assertions": {"count": 3} +} diff --git a/labs/20-deltawire/eval/skills/deltawire/examples/rows.dw.json b/labs/20-deltawire/eval/skills/deltawire/examples/rows.dw.json new file mode 100644 index 000000000..543ddac69 --- /dev/null +++ b/labs/20-deltawire/eval/skills/deltawire/examples/rows.dw.json @@ -0,0 +1,15 @@ +{ + "version": "deltawire.plan.v1", + "id": "rows-example", + "description": "Two explicit records with defaults", + "record_schema": ".deltawire/schemas/example.schema.json", + "output": {"path": "testdata/generated/rows.ndjson", "format": "ndjson", "pretty": false}, + "defaults": {"enabled": true}, + "generators": [{ + "kind": "rows", + "name": "items", + "columns": ["/name", "/rank"], + "rows": [["alpha", 1], ["beta", 2]] + }], + "assertions": {"count": 2} +} diff --git a/labs/20-deltawire/eval/skills/deltawire/examples/variants.dw.json b/labs/20-deltawire/eval/skills/deltawire/examples/variants.dw.json new file mode 100644 index 000000000..c9adb90bb --- /dev/null +++ b/labs/20-deltawire/eval/skills/deltawire/examples/variants.dw.json @@ -0,0 +1,18 @@ +{ + "version": "deltawire.plan.v1", + "id": "variants-example", + "description": "Base records with small deterministic changes", + "record_schema": ".deltawire/schemas/example.schema.json", + "output": {"path": "testdata/generated/variants.ndjson", "format": "ndjson", "pretty": false}, + "defaults": {"active": true}, + "generators": [{ + "kind": "variants", + "name": "cases", + "base": {"role": "member", "status": "allow"}, + "variants": [ + {"name": "deny", "set": {"/status": "deny"}}, + {"name": "inactive", "set": {"/active": false}} + ] + }], + "assertions": {"count": 2} +} diff --git a/labs/20-deltawire/eval/skills/deltawire/references/plan-v1.md b/labs/20-deltawire/eval/skills/deltawire/references/plan-v1.md new file mode 100644 index 000000000..8a10230ed --- /dev/null +++ b/labs/20-deltawire/eval/skills/deltawire/references/plan-v1.md @@ -0,0 +1,45 @@ +# DeltaWire plan v1 + +A plan is strict JSON. Unknown fields fail decoding. + +## Plan fields + +- `version`: exactly `deltawire.plan.v1`. +- `id`, `description`: descriptive strings. +- `record_schema`: repository-relative JSON Schema path used by `check`. +- `output.path`: repository-relative output path. +- `output.format`: `json` or `ndjson`. +- `output.pretty`: JSON pretty-printing flag. +- `defaults`: optional object deep-merged into each record. +- `sets`: optional named arrays of scalar dimension values. +- `generators`: ordered array of `matrix`, `rows`, or `variants` generators. +- `assertions.count`: exact projected record count. +- `assertions.unique`: optional JSON Pointer paths that must be unique. +- `assertions.coverage`: optional JSON Pointer/value coverage checks. + +## Matrix + +A matrix generator has `kind`, `name`, `dimensions`, and `record`. Each +dimension has a `name` plus exactly one of `values`, `set`, or `range`. An +inline integer range uses `start`, `end_exclusive`, and nonzero `step`. + +Reference dimensions from the record with `${name}`. When a string is exactly +one placeholder, the generated scalar retains its original type; for example, +an integer range produces JSON integers. Embedded placeholders stringify their +scalar values. Every dimension must be referenced by the record. + +## Rows + +A rows generator has `kind`, `name`, `columns`, and `rows`. Each column is a +JSON Pointer such as `/role`; every row must have exactly one value per column. +Rows begin with plan defaults, then column values are assigned in order. + +## Variants + +A variants generator has `kind`, `name`, `base`, and `variants`. Each variant +has a unique `name`, optional `set` JSON Pointer/value assignments, and optional +`omit` JSON Pointer paths. Records are built by deep-merging plan defaults and +the generator base, then applying the variant changes. + +Generators emit records in plan order. Matrix dimensions vary from the last +dimension inward, preserving each values/range order.