eval(deltawire): superseded v5 experiment provenance

.github/workflows/deltawire-preflight-v5.yml

@@ -0,0 +1,28 @@
+name: Verify DeltaWire preflight v5
+
+on:
+  push:
+    branches: [eval/deltawire-preflight-v5]
+  pull_request:
+    paths:
+      - "labs/20-deltawire/**"
+      - ".github/workflows/deltawire-preflight-v5.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  validate-v5:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v7
+        with:
+          go-version-file: labs/20-deltawire/go.mod
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+      - run: bash labs/20-deltawire/scripts/validate.sh
+      - run: bash labs/20-deltawire/eval/scripts/v5/validate.sh

labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.deltawire/config.json

@@ -0,0 +1,12 @@
+{
+  "limits": {
+    "max_output_bytes": 104857600,
+    "max_plan_bytes": 1048576,
+    "max_records": 100000,
+    "max_schema_bytes": 1048576
+  },
+  "plans_dir": ".deltawire/plans",
+  "schemas_dir": ".deltawire/schemas",
+  "state_file": ".deltawire/state.json",
+  "version": "deltawire.config.v1"
+}

labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.deltawire/schemas/range-large.schema.json

@@ -0,0 +1,16 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "additionalProperties": false,
+  "properties": {
+    "index": {
+      "maximum": 500,
+      "minimum": 1,
+      "type": "integer"
+    }
+  },
+  "required": [
+    "index"
+  ],
+  "title": "range-large",
+  "type": "object"
+}

labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/.generated/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/Dockerfile

@@ -0,0 +1,10 @@
+FROM ubuntu:22.04
+COPY .generated/deltawire /usr/local/bin/deltawire
+COPY task-contract.json /task-contract.json
+COPY .deltawire /.deltawire
+COPY verify_plan_contract.py /usr/local/bin/verify_plan_contract.py
+COPY environment_receipt.py /usr/local/bin/environment_receipt.py
+COPY deltawire-environment-expectations.json /deltawire-environment-expectations.json
+RUN chmod 0755 /usr/local/bin/deltawire /usr/local/bin/verify_plan_contract.py /usr/local/bin/environment_receipt.py && chmod a-w /task-contract.json /.deltawire/config.json /.deltawire/schemas/*.json /deltawire-environment-expectations.json
+RUN command -v deltawire && deltawire version
+RUN apt-get update && apt-get install -y python3

labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/deltawire-environment-expectations.json

@@ -0,0 +1,23 @@
+{
+  "binary": {
+    "path": "/usr/local/bin/deltawire",
+    "realpath": "/usr/local/bin/deltawire",
+    "sha256": "e5198d15000e093a2a28e57ad4a093dde8f66bcab21462549ddf9114064c25f4",
+    "version": "deltawire version dev"
+  },
+  "files": {
+    "config": {
+      "path": "/.deltawire/config.json",
+      "sha256": "673b9cf053e3ba2263bfbc49034360ed8dba8d959bd167f8ff4b1f7cb4302b6f"
+    },
+    "public_contract": {
+      "path": "/task-contract.json",
+      "sha256": "d3e1b7b3e5a4576540b3a2969d59f49818c7c0a12197b26d271e7e277b0054e5"
+    },
+    "schema": {
+      "path": "/.deltawire/schemas/range-large.schema.json",
+      "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5"
+    }
+  },
+  "schema_version": "deltawire-environment-expectations.v1"
+}

labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/environment_receipt.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""Atomically emit deterministic DeltaWire environment evidence."""
+import argparse,hashlib,json,os,stat,subprocess,tempfile
+from pathlib import Path
+
+def sha(path):
+ h=hashlib.sha256()
+ with Path(path).open("rb") as f:
+  for chunk in iter(lambda:f.read(1048576),b""):h.update(chunk)
+ return h.hexdigest()
+def atomic_write(path,data):
+ target=Path(path);target.parent.mkdir(parents=True,exist_ok=True)
+ fd,tmp=tempfile.mkstemp(prefix=f".{target.name}.",suffix=".tmp",dir=target.parent)
+ try:
+  with os.fdopen(fd,"w",encoding="utf-8") as f:f.write(data);f.flush();os.fsync(f.fileno())
+  os.replace(tmp,target)
+ except BaseException:
+  try:os.unlink(tmp)
+  except FileNotFoundError:pass
+  raise
+def build(expectations):
+ expected=json.loads(Path(expectations).read_text());binary=Path(expected["binary"]["path"])
+ exists=binary.exists();lst=binary.lstat() if exists else None
+ regular=bool(lst and stat.S_ISREG(lst.st_mode));symlink=binary.is_symlink() if exists else False
+ executable=exists and os.access(binary,os.X_OK);realpath=str(binary.resolve()) if exists else None
+ try:version=subprocess.run([str(binary),"version"],capture_output=True,text=True,check=False) if exists else None
+ except OSError:version=None
+ files={}
+ for name,item in sorted(expected["files"].items()):
+  path=Path(item["path"]);actual=sha(path) if path.is_file() else None
+  files[name]={"path":item["path"],"exists":path.is_file(),"actual_sha256":actual,"expected_sha256":item["sha256"],"hash_match":actual==item["sha256"]}
+ binary_hash=sha(binary) if regular else None
+ checks={"binary_exists":exists,"binary_regular":regular,"binary_not_symlink":not symlink,"binary_executable":executable,
+         "binary_path":str(binary)==expected["binary"]["path"],"binary_realpath":realpath==expected["binary"]["realpath"],
+         "binary_hash":binary_hash==expected["binary"]["sha256"],"version_exit_0":bool(version and version.returncode==0),
+         "version_exact":bool(version and version.stdout.splitlines() and version.stdout.splitlines()[0].strip()==expected["binary"]["version"]),
+         **{f"{name}_hash":item["hash_match"] for name,item in files.items()}}
+ return {"schema_version":"deltawire-environment-receipt.v1","binary":{"path":str(binary),"realpath":realpath,"exists":exists,
+         "is_regular":regular,"is_symlink":symlink,"executable":executable,"actual_sha256":binary_hash,
+         "expected_sha256":expected["binary"]["sha256"],"version_stdout":version.stdout.strip() if version else None,
+         "version_stderr":version.stderr.strip() if version else None,"version_exit_code":version.returncode if version else None,
+         "expected_version":expected["binary"]["version"]},"files":files,"checks":checks,"status":"pass" if all(checks.values()) else "fail"}
+def main():
+ p=argparse.ArgumentParser();p.add_argument("--expectations",required=True);p.add_argument("--receipt",required=True);a=p.parse_args()
+ receipt=build(a.expectations);atomic_write(a.receipt,json.dumps(receipt,indent=2,sort_keys=True)+"\n")
+ print(json.dumps(receipt,sort_keys=True));raise SystemExit(0 if receipt["status"]=="pass" else 1)
+if __name__=="__main__":main()

labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/task-contract.json

@@ -0,0 +1,24 @@
+{
+  "authoritative_schema": {
+    "path": ".deltawire/schemas/range-large.schema.json",
+    "sha256": "2fe7bdc8de996a09ccc651f42611dc88faee60c7390b36c62443ed3e523800b5"
+  },
+  "deltawire_applicability": "expected",
+  "family": "range",
+  "generation": {
+    "field": "index",
+    "field_order": [
+      "index"
+    ],
+    "start": 1
+  },
+  "ordering": "canonical_generation_order",
+  "output": {
+    "format": "ndjson",
+    "path": "testdata/generated/output.ndjson"
+  },
+  "record_count": 500,
+  "schema_version": "task-spec.v1",
+  "size": "large",
+  "task": "range-large"
+}

labs/20-deltawire/eval/conformance/environment-receipt-v1/environment/verify_plan_contract.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+import argparse, hashlib, json, subprocess
+from pathlib import Path
+
+def sha(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest()
+def write(path, value):
+    if path:
+        target=Path(path); target.parent.mkdir(parents=True,exist_ok=True); target.write_text(json.dumps(value,indent=2,sort_keys=True)+"\n")
+
+def main():
+    p=argparse.ArgumentParser(); p.add_argument("--contract",required=True); p.add_argument("--plan",required=True); p.add_argument("--repo",default="."); p.add_argument("--receipt"); p.add_argument("--deltawire",default="deltawire"); a=p.parse_args()
+    repo=Path(a.repo).resolve(); contract_path=Path(a.contract).resolve(); plan=Path(a.plan).resolve(); contract=json.loads(contract_path.read_text())
+    try: plan_arg=str(plan.relative_to(repo))
+    except ValueError: plan_arg=str(plan)
+    result={"schema_version":"plan-contract-receipt.v1","contract_sha256":sha(contract_path),"plan_sha256":sha(plan),"checks":{},"status":"fail"}
+    try:
+        run=subprocess.run([a.deltawire,"inspect","--repo",str(repo),"--format","json",plan_arg],capture_output=True,text=True)
+        result["inspect_exit_code"]=run.returncode
+        if run.returncode: result["error"]=run.stderr.strip() or run.stdout.strip()
+        else:
+            inspected=json.loads(run.stdout); schema=contract["authoritative_schema"]; schema_path=repo/schema["path"]
+            checks={"output_path":inspected.get("output_path")==contract["output"]["path"],"output_format":inspected.get("output_format")==contract["output"]["format"],"projected_records":inspected.get("projected_records")==contract["record_count"],"schema_path":inspected.get("schema_path")==schema["path"],"schema_exists":schema_path.is_file(),"schema_sha256":schema_path.is_file() and sha(schema_path)==schema["sha256"],"exact_count_assertion":str(contract["record_count"]) in json.dumps(inspected.get("assertion_summary",{}))}
+            result.update({"checks":checks,"inspect":inspected,"status":"pass" if all(checks.values()) else "fail"})
+    except (OSError,ValueError,KeyError) as error: result["error"]=str(error)
+    write(a.receipt,result)
+    if result["status"]!="pass": raise SystemExit(1)
+    print(json.dumps(result,sort_keys=True))
+if __name__=="__main__": main()

labs/20-deltawire/eval/conformance/environment-receipt-v1/instruction.md

@@ -0,0 +1 @@
+No agent action is required. Exit without changing the task environment.

labs/20-deltawire/eval/conformance/environment-receipt-v1/task.toml

@@ -0,0 +1,29 @@
+schema_version = "1.3"
+artifacts = []
+
+[task]
+name = "operatorstack/deltawire-environment-receipt-conformance-v1"
+description = "No-model Harbor main collect-hook conformance"
+authors = []
+keywords = []
+
+[metadata]
+benchmark_result = false
+conformance_version = "v1"
+
+[verifier]
+timeout_sec = 60.0
+
+[[verifier.collect]]
+service = "main"
+command = "python3 /usr/local/bin/environment_receipt.py --expectations /deltawire-environment-expectations.json --receipt /logs/artifacts/deltawire/environment-receipt.json"
+timeout_sec = 30.0
+
+[agent]
+timeout_sec = 60.0
+
+[environment]
+network_mode = "public"
+build_timeout_sec = 600.0
+os = "linux"
+mcp_servers = []

labs/20-deltawire/eval/conformance/environment-receipt-v1/tests/test.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+mkdir -p /logs/verifier
+printf '{"conformance": 1}\n' > /logs/verifier/reward.json

labs/20-deltawire/eval/docs/08-harbor-artifact-timing.md

@@ -0,0 +1,29 @@
+# Harbor 0.20.0 artifact timing
+
+This preflight is pinned to `harbor==0.20.0`. Inspection of the installed package on
+`tbench-c4d` established the following order in
+`harbor/trial/single_step.py` and `harbor/trial/trial.py`:
+
+1. the agent phase finishes;
+2. `[[verifier.collect]]` hooks with `service = "main"` run while the main
+   container is still available;
+3. main-container artifacts are downloaded;
+4. the verifier runs;
+5. verifier logs and reward are retained.
+
+`VerifierCollectConfig` in `harbor/models/task/config.py` accepts `command`,
+`service`, `timeout_sec`, and `user`. The implicit convention entry for
+`/logs/artifacts/` is created by `harbor/trial/artifact_handler.py` and maps to
+`<trial>/artifacts/logs/artifacts/`. Its `artifacts/manifest.json` entry has
+`source`, `destination`, `type`, `status`, and `service` fields.
+
+Collect hooks are best effort. A nonzero exit is written as a warning, including
+the command, exit code, stdout, and stderr, in the job/trial logs, but collection
+continues and no structured hook-exit field is added to the artifact manifest.
+Consequently the v5 gate never treats Harbor success or hook exit alone as proof:
+it requires the canonical retained receipt plus an `ok` convention-directory
+manifest entry.
+
+The paid task retains the default container user. When that user is root, the
+receipt detects accidental environment drift; it is not a cryptographic
+anti-cheat boundary against the agent.