diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index 759c764..ca6f7fd 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -80,6 +80,10 @@ jobs: (github.event_name == 'schedule' || inputs.channel == 'stable' || inputs.channel == 'both') }} runs-on: ubuntu-latest + # Consumed by `stable-verify-assets` below, which must know WHETHER a tag was actually cut + # (a no-op run must not verify anything) and WHICH tag to verify. + outputs: + tag: ${{ steps.ver.outputs.tag }} steps: - name: Require RELEASE_TOKEN (else no-op) id: token @@ -193,7 +197,107 @@ jobs: git push origin "HEAD:main" git push origin "${{ steps.ver.outputs.tag }}" fi - echo "Pushed changelog commit + ${{ steps.ver.outputs.tag }} — release.yml (stable build) will fire." + echo "Pushed changelog commit + ${{ steps.ver.outputs.tag }} — release.yml (stable build) should now fire." + + # THE TAG-PUSH EVENT IS NOT A GUARANTEE (dig_ecosystem#2290). + # + # Everything the stable channel produces — the binaries (release.yml) and the native install + # packages (package.yml) — hangs off ONE thing: GitHub delivering a `push` event for the tag + # above and creating a workflow run from it. That is an at-most-once delivery this repo does + # not control, and on 2026-08-06 it simply did not happen. The push itself succeeded + # (`* [new tag] v0.99.9 -> v0.99.9`) and the branch push one second earlier DID create a run, + # but the tag created NONE. Neither release.yml nor package.yml ran. The release was then + # repaired by hand by dispatching release.yml alone, which produced a `latest` release of + # bare binaries — and because feedsign resolves dig-node by its native-package names, that + # froze the stable signed feed for every product until the manifest expired. + # + # So the tag push is treated as a REQUEST, and this step confirms it was honoured. Where a + # run is missing, it is dispatched explicitly against the tag ref, which is equivalent: both + # workflows gate their publish on `github.ref_type == 'tag'`, which a dispatch against a tag + # satisfies. The step is idempotent — it dispatches only what is absent, so on the normal + # path (both runs present) it does nothing at all. + # + # This deliberately does NOT restructure the release into `workflow_call` jobs. Both + # workflows already publish to the tag's release, and adding a second publisher for the same + # assets would trade a rare lost event for a routine race. + - name: Confirm the tag push actually created the release runs + if: steps.token.outputs.present == 'true' && steps.ver.outputs.skip == 'false' + shell: bash + env: + GH_TOKEN: ${{ secrets.RELEASE_TOKEN }} + REPO: ${{ github.repository }} + TAG: ${{ steps.ver.outputs.tag }} + FORCE_TAG: ${{ steps.ver.outputs.force }} + run: | + set -euo pipefail + + # A run "for this tag" must mean a run for THIS CUT of the tag, not merely one that + # carried the same tag NAME at some point. `gh run list --branch` matches on the name + # alone, and a tag name is not stable: a `force` re-cut moves it onto a new commit. So + # the commit is the identity, and the run's `headSha` is what is compared. + # + # Without that, a force re-cut whose tag event is lost finds the PREVIOUS cut's runs, + # dispatches nothing, and the asset guard then passes on the PREVIOUS cut's packages — + # a green release run shipping stale binaries under a moved tag, wearing exactly the + # success signature of a correct release. That is the failure class this step exists to + # remove, so it must not be reachable through the step itself. + TAG_COMMIT="$(git rev-parse "$TAG^{commit}")" + + runs_for_this_cut() { + gh run list --repo "$REPO" --workflow "$1" --branch "$TAG" \ + --json headSha --jq "[.[] | select(.headSha == \"$TAG_COMMIT\")] | length" + } + + # A force re-cut ALWAYS dispatches, without consulting history. The documented reason to + # force is "the build failed, re-fire it" — and a failed run is still a run at the right + # commit, so any count-based check reads the wreckage of the previous attempt as success + # and turns the retry into a silent no-op that then burns the guard's full timeout before + # going red. Re-cutting is an explicit request for fresh builds; treat it as one. + if [ "${FORCE_TAG:-}" = "true" ]; then + echo "force re-cut of $TAG at $TAG_COMMIT — dispatching both release workflows unconditionally (a re-cut is a request for fresh builds; an earlier run at this tag may be the failed one being retried)." + for wf in release.yml package.yml; do + gh workflow run "$wf" --repo "$REPO" --ref "$TAG" + done + exit 0 + fi + + # Run creation is asynchronous; give the event a fair chance before declaring it lost. + # A false "lost" is cheap here (the dispatch is idempotent and the publish is tag-gated), + # but waiting avoids a pointless duplicate run on every single release. + for _ in $(seq 1 12); do + sleep 10 + [ "$(runs_for_this_cut release.yml)" -gt 0 ] && [ "$(runs_for_this_cut package.yml)" -gt 0 ] && break + done + + for wf in release.yml package.yml; do + if [ "$(runs_for_this_cut "$wf")" -gt 0 ]; then + echo "$wf: the tag push created a run for $TAG at $TAG_COMMIT." + continue + fi + echo "::warning::$wf did NOT run for $TAG at $TAG_COMMIT — the tag-push event was not delivered. Dispatching it explicitly against the tag ref so the release is not published incomplete." + gh workflow run "$wf" --repo "$REPO" --ref "$TAG" + done + + # The stable channel's ACCEPTANCE TEST (dig_ecosystem#2290). Cutting a tag and dispatching the + # builds is a request; this is the confirmation. It blocks until the published release carries + # the native install packages dig-updater's feedsign resolves dig-node by, and reddens this + # release run if it never does. + # + # It exists because every layer above it can report success while the release is unusable: the + # stable job succeeds when the tag is pushed, release.yml succeeds when the binaries are + # attached, and neither knows or cares whether the packages arrived. The only signal that a + # stable release is actually shippable is the asset list itself, and until now nobody read it. + # The consequence landed in a different repo hours later (dig-updater's feed.yml failing closed), + # which is the worst possible place for it to surface. + # + # Skipped on a no-op run (`stable` found the version already tagged, so `tag` is empty). + stable-verify-assets: + name: Stable — verify release assets + needs: stable + if: needs.stable.outputs.tag != '' + uses: ./.github/workflows/verify-release-assets.yml + with: + tag: ${{ needs.stable.outputs.tag }} # ───────────────────────────────── NIGHTLY channel ───────────────────────────────── # 1) meta: gate on the token + synthesize the nightly version and tags (nothing is committed). diff --git a/.github/workflows/verify-release-assets.yml b/.github/workflows/verify-release-assets.yml new file mode 100644 index 0000000..3e3b8c9 --- /dev/null +++ b/.github/workflows/verify-release-assets.yml @@ -0,0 +1,146 @@ +# The RELEASE ASSET GUARD (dig_ecosystem#2290). +# +# A `vX.Y.Z` release of dig-node is not shippable just because its binaries exist. dig-updater's +# feedsign resolves dig-node by the NATIVE INSTALL PACKAGE file names — the beacon installs +# dig-node by handing a package to msiexec/installer/dpkg, it never places a bare binary — and it +# FAILS CLOSED on the whole signed manifest when even one component cannot be resolved. So a +# stable release that carries binaries but no `.msi`/`.pkg`/`.deb` does not merely ship an +# incomplete dig-node: it freezes auto-update for EVERY product on the stable channel. +# +# That is not hypothetical. `v0.99.9` shipped binaries only, feedsign failed closed on four +# consecutive runs, and the stable manifest sat frozen and then EXPIRED for ~15 hours while every +# individual workflow run in this repo reported success. Nothing was red, because nothing was +# asking the one question that mattered: does the published release actually carry the assets the +# feed needs? +# +# This workflow asks it, and is the only place that does. +# +# * workflow_call — the stable release path (nightly-release.yml) waits on this after cutting a +# tag, so a package-less stable release reddens the release run itself rather than surfacing +# hours later as a feed failure in another repo. +# * workflow_dispatch — point it at ANY tag on demand. This is what makes the guard falsifiable: +# dispatching it at a release known to lack packages MUST fail, and at a complete release MUST +# pass. A guard that cannot be shown to go red is not a guard. +# +# It polls rather than sampling once, because the binary build and the package build are separate +# workflows that finish at different times; a single sample would race them and produce a red that +# only means "not finished yet". +name: Verify release assets + +on: + workflow_call: + inputs: + tag: + description: "The release tag to verify (e.g. v0.99.9)." + type: string + required: true + timeout_minutes: + description: >- + How long to wait for the assets to appear before failing. The default spans a cold + cross-OS package build (the .msi and .pkg legs dominate). + type: number + required: false + default: 75 + workflow_dispatch: + inputs: + tag: + description: "The release tag to verify (e.g. v0.99.9)." + type: string + required: true + timeout_minutes: + description: "How long to wait for the assets to appear before failing." + type: number + required: false + default: 5 + +permissions: + contents: read + +jobs: + verify: + name: Verify ${{ inputs.tag }} carries the native install packages + runs-on: ubuntu-latest + steps: + - name: Assert the feed-resolvable assets are present + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + TAG: ${{ inputs.tag }} + TIMEOUT_MINUTES: ${{ inputs.timeout_minutes }} + run: | + set -euo pipefail + + # The version as it appears in asset names: the tag without its leading `v`. + VERSION="${TAG#v}" + + # EXACTLY the names dig-updater's feedsign looks for. Kept as a literal list rather than + # derived from a glob: a glob would happily accept a `.deb` for the wrong arch or a + # stale version and call the release complete, which is the failure this guard exists to + # catch. + # + # THIS IS THE THIRD COPY OF A CROSS-REPO CONTRACT, and a shell step cannot import the + # Rust constant that owns it. Producer: `package.yml` in this repo. Consumer: + # `dig-updater/crates/dig-updater-feedsign/src/resolve.rs` (`asset_name_parts`). + # Verifier: here. Nothing enforces that the three agree, so they are held together by + # `SYSTEM.md` (dig-updater section, "dig-node release-asset file names") and the + # `canonical` skill (beacon/update trust anchors). Change one, change all three. + # + # Note macOS contributes ONE name, not two: the `.pkg` is universal and carries no arch + # token, so `macos/arm64` and `macos/x64` both resolve to it — feedsign's five platforms + # yield four distinct file names. + # + # `arm64.deb` is required here DELIBERATELY, and this is stricter than feedsign's own + # failure condition. feedsign fails closed only when a component resolves ZERO assets, so + # a release missing just `arm64.deb` would still publish — silently dropping linux/arm64 + # hosts from auto-update rather than reddening anything. That silent drop is exactly the + # arm64 platform floor (#1741/#1736/#2126), so the stable channel treats a missing arm64 + # package as a failed release. Do not relax this to match feedsign. + EXPECTED=( + "dig-node_${VERSION}_amd64.deb" + "dig-node_${VERSION}_arm64.deb" + "dig-node-${VERSION}-macos.pkg" + "dig-node-${VERSION}-windows-x64.msi" + ) + + deadline=$(( $(date +%s) + TIMEOUT_MINUTES * 60 )) + attempt=0 + + while :; do + attempt=$(( attempt + 1 )) + + # A missing release is a legitimate "not yet" while the release workflow is still + # running, so it is treated the same as a missing asset rather than aborting early. + assets="$(gh release view "$TAG" --repo "$REPO" --json assets --jq '.assets[].name' 2>/dev/null || true)" + + missing=() + for name in "${EXPECTED[@]}"; do + printf '%s\n' "$assets" | grep -qxF "$name" || missing+=("$name") + done + + if [ ${#missing[@]} -eq 0 ]; then + echo "$TAG carries all ${#EXPECTED[@]} native install packages — feedsign can resolve dig-node." + { + echo "### Release assets verified — \`$TAG\`" + echo + for name in "${EXPECTED[@]}"; do echo "- \`$name\`"; done + } >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + if [ "$(date +%s)" -ge "$deadline" ]; then + { + echo "### Release assets MISSING — \`$TAG\`" + echo + echo "dig-updater's feedsign cannot resolve dig-node from this release, so the" + echo "STABLE signed feed will fail closed for every component until it is fixed." + echo + echo "Missing:" + for name in "${missing[@]}"; do echo "- \`$name\`"; done + } >> "$GITHUB_STEP_SUMMARY" + echo "::error::release $TAG is missing ${#missing[@]} native install package(s): ${missing[*]}. dig-updater feedsign resolves dig-node by these names and fails closed on the ENTIRE stable manifest when they are absent. Attach them (dispatch package.yml against the $TAG ref) before this release is allowed to stand as latest." + exit 1 + fi + + echo "attempt $attempt: still missing ${#missing[@]} of ${#EXPECTED[@]} (${missing[*]}); retrying…" + sleep 30 + done diff --git a/Cargo.lock b/Cargo.lock index 55553fe..337efc4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2295,7 +2295,7 @@ dependencies = [ [[package]] name = "dig-node-service" -version = "0.100.1" +version = "0.100.2" dependencies = [ "async-trait", "axum", diff --git a/Cargo.toml b/Cargo.toml index e8cea60..9564be8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ edition = "2021" # the ROOT manifest (`[workspace.package].version`), so it MUST be set here for a # release to fire (§3.6). The library crates (dig-node-core/dig-runtime/dig-wallet) # keep their own independent versions — only the released binary tracks the workspace version. -version = "0.100.1" +version = "0.100.2" # Release hardening, matching digstore: keep integer-overflow checks ON in release. # The node parses untrusted serialized input and does offset/length arithmetic over diff --git a/DEVELOPMENT_LOG.md b/DEVELOPMENT_LOG.md index 4f58eb3..eca8781 100644 --- a/DEVELOPMENT_LOG.md +++ b/DEVELOPMENT_LOG.md @@ -4,6 +4,28 @@ High-signal realizations from debugging/development: non-obvious cross-system co sharp edges, and gotchas. Concise durable facts with context — NOT a change diary. See `CLAUDE.md` §4.5 for the maintenance contract (a curator periodically re-verifies + prunes). +## A tag push can succeed and create ZERO workflow runs (dig_ecosystem#2290) + +`git push origin vX.Y.Z` reporting `* [new tag] v0.99.9 -> v0.99.9` does NOT mean a `push: tags:` +workflow ran. On 2026-08-06 that push landed and GitHub created **no** runs from it — neither +`release.yml` nor `package.yml` — while the `HEAD:main` push one second earlier in the same step +DID create one. So this was not an outage, a disabled workflow, or the documented +`GITHUB_TOKEN`-does-not-retrigger rule (the tag was pushed by `RELEASE_TOKEN`, as the eleven +preceding tags were, and `package.yml` has fired on tags 55 times). Run creation from a push event +is effectively at-most-once, and a release path that assumes otherwise has a silent single point +of failure. **Never treat a successful tag push as proof the release fired — confirm the run +exists, and dispatch against the tag ref if it does not** (`ref_type == 'tag'` publish gates are +satisfied by a dispatch selected against a tag, which is what makes the repair equivalent). + +Two traps around it. **Reading run history by recency lies:** `package.yml`'s ten most recent runs +were all `event=pull_request`, which reads as "this has never fired on a tag" — filter by +`?event=push` before concluding a trigger is dead. And **a partial manual repair is worse than +none:** dispatching `release.yml` alone produced a `latest` release of bare binaries, and because +dig-updater's feedsign resolves dig-node by native-package file names and fails closed on the +ENTIRE manifest, that froze — then expired — stable auto-update for all five components, dig-app +included. A dig-node release without its `.msi`/`.pkg`/`.deb` is not a partial dig-node release, +it is an ecosystem-wide auto-update outage. + ## Local-RPC authz — holder-REVEALING reads gate too, not just mutators (#2108) Holder-revealing `cache.*` READS must be control-token-gated over the HTTP (loopback) surface, not diff --git a/SPEC.md b/SPEC.md index b7cd48b..8fcfaee 100644 --- a/SPEC.md +++ b/SPEC.md @@ -2866,6 +2866,21 @@ boolean, default `false`). It MUST NOT trigger on `push` to `main`. force-moved tag breaks git tag-immutability; because dig-node updates are gated by the dig-updater signed feed (an Ed25519 signature over the update descriptor, verified before apply), that signature — not the mutable tag — is the integrity anchor. Ship new code by bumping the version. +- **The tag push is a request, not a guarantee.** Creating a workflow run from a pushed tag is an + event delivery this repo does not control, and it has been observed to not occur even though the + push succeeded. After pushing the tag the stable job MUST confirm that both `release.yml` and + `package.yml` have a run for that tag, and MUST dispatch — against the tag ref — whichever is + absent. Both workflows gate publication on `github.ref_type == 'tag'`, which a dispatch against a + tag satisfies, so the dispatched run is equivalent to the event-triggered one. The confirmation + MUST be idempotent: where the event was delivered normally, it dispatches nothing. +- **A stable release MUST carry the native install packages.** dig-updater's feedsign resolves + dig-node by the `.deb`/`.pkg`/`.msi` file names and fails closed on the ENTIRE signed manifest + when they are absent, so a stable release of bare binaries does not ship a partial dig-node — it + freezes auto-update for every component on the channel. The stable path MUST therefore verify the + published release's asset list (`verify-release-assets.yml`) and MUST fail the release run when + `dig-node__amd64.deb`, `dig-node__arm64.deb`, `dig-node--macos.pkg`, or + `dig-node--windows-x64.msi` is missing. Repairing a failed release by publishing only the + binaries is NOT a repair. 11.1a. **Doc-only commits never release** (the version is unchanged → the tag exists → the stable job is a no-op). The manual-dispatch `workflow_dispatch` on `release.yml` is a build-only "does main diff --git a/runbooks/release.md b/runbooks/release.md index 70b2b04..2bea3af 100644 --- a/runbooks/release.md +++ b/runbooks/release.md @@ -104,6 +104,7 @@ a bug in the beacon: | `release.yml` | `push: tags: v*` (+ dispatch canary) | Builds + publishes the stable Release for a `vX.Y.Z` tag. | | `build-binaries.yml` | `workflow_call` | Reusable cross-OS build, dual-named + `dign` (both channels call it). | | `package.yml` | PR + `push: tags: v*` + `workflow_call` | Builds the `.deb`/`.pkg`/`.msi`. Attaches them itself on a `v*` tag; on a `workflow_call` (the nightly channel) it leaves them as run artifacts for the caller to publish. | +| `verify-release-assets.yml` | `workflow_call` (stable path) + `workflow_dispatch` | Asserts a `vX.Y.Z` release carries the four native install packages dig-updater's feedsign resolves dig-node by. Dispatch it at any tag to check a release by hand — a package-less release freezes the stable signed feed for every product. | | `ci.yml` | PR + push to main | fmt/clippy + `cargo llvm-cov nextest --workspace` (pre-merge). NOTE: `ubuntu-latest` only — Windows/macOS build breaks are first caught by the nightly channel, not PR CI (SPEC §11 / follow-up). | ## Local build (dev)