Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 117 additions & 0 deletions .github/workflows/smoke-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
name: Base Std Smoke Tests (Vibenet)

# Advisory, MANUALLY-triggered smoke run of the FULL b20 suite against a LIVE Base network
# (Vibenet by default).
#
# workflow_dispatch ONLY — deliberately not pull_request / merge_group / schedule. Vibenet is a
# live chain, manually deployed per hardfork; CI cannot guarantee which precompile set is live, so
# an automated run would flap. This job never gates merges: it is a bring-up check you run by hand
# (e.g. after a hardfork ships to Vibenet) to confirm the b20 surface behaves against the real Rust
# precompiles. It always runs the whole suite against the ref you dispatch from (latest = main) —
# there is no journey picker and no fork/ref selector (branch-per-fork: dispatch from the branch/tag
# that matches the fork you want). Each journey self-skips when the live chain is not yet on the fork
# that ships its surface (e.g. the ERC-8056 multiplier journey on a pre-Cobalt chain).

on:
workflow_dispatch:
inputs:
rpc_url:
description: "JSON-RPC endpoint of the live chain"
required: true
default: https://rpc.vibes.base.org/

concurrency:
group: ${{ github.workflow }}-${{ github.run_id }}
cancel-in-progress: false

permissions:
contents: read

jobs:
smoke:
name: Vibenet smoke
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
with:
egress-policy: audit

- name: Checkout base-std
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 1
submodules: recursive

- name: Install Foundry
uses: foundry-rs/foundry-toolchain@50d5a8956f2e319df19e6b57539d7e2acb9f8c1e # v1.5.0
with:
version: stable

- name: Set up Python 3.13
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: '3.13'

- name: Set up smoke runner venv
shell: bash
# setup-python makes python3 == 3.13; make python-check enforces it.
run: make smoke-setup PYTHON=python3

- name: Build contracts (interface ABIs the harness binds to)
shell: bash
run: forge build

- name: Run the full Vibenet smoke suite
id: smoke
shell: bash
# RPC_URL comes from the dispatch input; the two keys are repo secrets (never echoed). `-k`
# (keep-going) runs every journey and prints a summary, so one journey's failure/skip never
# masks the rest; it always exits 0, so the (advisory) job stays green and the summary below
# reports the real per-journey status. continue-on-error still guards against an infra crash.
continue-on-error: true
env:
RPC_URL: ${{ inputs.rpc_url }}
DEPLOYER_PK: ${{ secrets.SMOKE_DEPLOYER_PK }}
USER2_PK: ${{ secrets.SMOKE_USER2_PK }}
run: |
set -o pipefail
PYTHONPATH=script script/smoke/.venv/bin/python -m smoke all -k \
2>&1 | tee "$RUNNER_TEMP/smoke-output.txt"
- name: Summarize smoke results
if: always()
shell: bash
run: |
output="$RUNNER_TEMP/smoke-output.txt"
# Vibenet chain id, as logged by the harness preflight ("preflight ok — chain=<id> ...").
chain_id=$(grep -oE 'chain=[0-9]+' "$output" 2>/dev/null | head -1 | cut -d= -f2)
chain_id=${chain_id:-unknown}
# The -k runner prints "smoke summary: P passed, F failed, S skipped (of N)".
summary=$(grep -oE 'smoke summary: [0-9]+ passed, [0-9]+ failed, [0-9]+ skipped' "$output" 2>/dev/null | tail -1)
passed=$(echo "$summary" | grep -oE '[0-9]+ passed' | grep -oE '[0-9]+'); passed=${passed:-0}
failed=$(echo "$summary" | grep -oE '[0-9]+ failed' | grep -oE '[0-9]+'); failed=${failed:-0}
skipped=$(echo "$summary" | grep -oE '[0-9]+ skipped' | grep -oE '[0-9]+'); skipped=${skipped:-0}
{
echo "## Vibenet Smoke Results"
echo ""
echo "| Field | Value |"
echo "|---|---|"
echo "| Ref | \`${{ github.ref_name }}\` |"
echo "| RPC endpoint | \`${{ inputs.rpc_url }}\` |"
echo "| Chain id | \`${chain_id}\` |"
echo "| Passed / Failed / Skipped | ${passed} / ${failed} / ${skipped} |"
echo ""
if [ -z "$summary" ]; then
echo "❓ **Did not run** — no summary produced (likely an infra error before the suite ran, e.g. RPC unreachable). Check the run log."
elif [ "$failed" -ne 0 ]; then
echo "❌ **${failed} journey(s) failed** — an assertion or expected revert did not match. Advisory only; does not gate merges. See the run log for the failing journey and its trace."
elif [ "$passed" -ne 0 ]; then
echo "✅ **${passed} passed** (${skipped} skipped) — the b20 surface behaved against the live precompiles."
else
echo "⏭️ **All ${skipped} skipped** — Vibenet is not on the expected fork yet (feature inactive or pre-fork). Chain/fork state, not a defect."
fi
} >> "$GITHUB_STEP_SUMMARY"
40 changes: 12 additions & 28 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ VENV = script/smoke/.venv
SMOKE_RUN = $(LOAD_ENV) PYTHONPATH=script $(VENV)/bin/python -m smoke
FORK_RUN = $(LOAD_ENV) PYTHONPATH=script $(VENV)/bin/python -m fork

.PHONY: build coverage smoke smoke-all smoke-factory smoke-asset smoke-stablecoin smoke-policy smoke-invariants python-check smoke-setup fork-tests
.PHONY: build coverage smoke smoke-all python-check smoke-setup fork-tests

# Generate an lcov coverage report and open it in the browser.
# Scoped to src/ and test/lib/mocks/ (excludes test runner files and the smoke probe helper).
Expand Down Expand Up @@ -58,31 +58,15 @@ fork-tests:
build:
forge build

# b20 precompile bring-up smoketest (web3.py + the interface ABIs read from
# `out/`). Sends real txs to $RPC_URL; requires env RPC_URL, DEPLOYER_PK,
# USER2_PK and a venv (`make smoke-setup`). `make smoke` runs every journey
# fail-fast.
smoke: smoke-factory smoke-asset smoke-stablecoin smoke-policy smoke-invariants

# Run every journey in a single process. KEEP_GOING=1 runs them all and reports a
# summary without erroring on failure (audit/triage mode); default fails fast and
# exits non-zero on the first failure (CI gating).
# make smoke-all # fail-fast
# make smoke-all KEEP_GOING=1 # run all, report, exit 0
smoke-all: build
# b20 precompile bring-up smoketest (web3.py + the interface ABIs read from `out/`). Sends real txs
# to $RPC_URL; requires env RPC_URL, DEPLOYER_PK, USER2_PK and a venv (`make smoke-setup`). The suite
# always runs as a whole — there are deliberately no per-journey targets. Fail-fast by default (CI
# gating); KEEP_GOING=1 runs every journey and prints a summary without erroring (audit/triage).
# make smoke # full suite, fail-fast
# make smoke KEEP_GOING=1 # full suite, run all, report, exit 0
# For ad-hoc single-journey debugging (cheaper/faster on a live chain), call the CLI directly, e.g.
# PYTHONPATH=script script/smoke/.venv/bin/python -m smoke asset
# The `multiplier` journey skips on a pre-Cobalt chain; SMOKE_OBSERVE_FLIP=1 also observes its lazy
# flip via a bounded real-time poll (off by default).
smoke smoke-all: build
@$(SMOKE_RUN) all $(if $(KEEP_GOING),--keep-going,)

smoke-factory: build
@$(SMOKE_RUN) factory

smoke-asset: build
@$(SMOKE_RUN) asset

smoke-stablecoin: build
@$(SMOKE_RUN) stablecoin

smoke-policy: build
@$(SMOKE_RUN) policy

smoke-invariants: build
@$(SMOKE_RUN) invariants
81 changes: 59 additions & 22 deletions script/smoke/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ Requires **Python 3.13** (`make smoke-setup` enforces it; override with `PYTHON=
```bash
make smoke-setup # one-time: create the venv + install web3
cp .env.template .env # then set RPC_URL, DEPLOYER_PK, USER2_PK
make smoke-all KEEP_GOING=1 # all journeys, audit summary; or one: make smoke-factory
make smoke KEEP_GOING=1 # run the full suite, audit summary
```

`DEPLOYER_PK` must hold enough ether to sign the setup and admin txs (it also
Expand All @@ -53,17 +53,25 @@ shell env wins over `.env` values.
### Make targets

```bash
make smoke # run every journey, fail-fast (CI gating default)
make smoke-all # all journeys, single process, fail-fast
make smoke-all KEEP_GOING=1 # all journeys, summarize, exit 0 regardless
make smoke-factory # one journey at a time: factory|asset|stablecoin|policy|invariants
make smoke-setup # create the venv + install web3 (one-time)
make smoke # run the FULL suite, fail-fast (CI gating default)
make smoke KEEP_GOING=1 # full suite, summarize, exit 0 regardless (audit/triage)
make smoke-all # alias of `make smoke` (also honors KEEP_GOING)
make smoke-setup # create the venv + install web3 (one-time)
```

> The `smoke-*` targets set `PYTHONPATH=script` for you. Running `python -m smoke`
> by hand needs that too (and the env exported), else you get `No module named
> smoke` — prefer the Make targets. The raw CLI takes an arbitrary subset and a
> fail-fast/keep-going flag, e.g. `python -m smoke asset policy -k`.
The suite always runs as a whole — there are deliberately **no per-journey `make`
targets**, so you can't get false confidence from a partial run. For ad-hoc
single-journey debugging (cheaper and faster on a live chain, where every journey
spends real gas and time), call the CLI directly — it takes an arbitrary subset and
a fail-fast/keep-going flag:

```bash
PYTHONPATH=script python -m smoke asset # one journey
PYTHONPATH=script python -m smoke asset policy -k # a subset, keep-going
```

> `PYTHONPATH=script` is required (the Make targets set it for you); without it you
> get `No module named smoke`.

### Environment / config knobs

Expand All @@ -75,17 +83,34 @@ make smoke-setup # create the venv + install web3 (one-time)
| `GAS_FLOAT_ETHER` | no | `0.01` | One-time gas float the deployer sends user2. |
| `SMOKE_SALT` | no | random | Pin the per-run salt namespace (reproducible addresses). |
| `SMOKE_TRACE` | no | `1` | On failure, dump a `debug_traceCall/Transaction` call tree. Set `0` for just the request + replayed revert data. |
| `SMOKE_OBSERVE_FLIP` | no | `0` | `multiplier` journey: also observe the scheduled multiplier's lazy flip via a bounded real-time poll (off by default — real-time coupling would flap on a slow chain; pending state is asserted read-only regardless). |
| `SMOKE_FLIP_WINDOW_S` / `SMOKE_FLIP_TIMEOUT_S` | no | `30` / `150` | When `SMOKE_OBSERVE_FLIP=1`: how far in the future to schedule the flip, and how long to poll for it. |
| `FAUCET_URL` / `FAUCET_NETWORK` | no | — | Optional deployer top-up when underfunded. |
| `FAUCET_AMOUNT` / `FAUCET_MIN_ETHER` | no | `0.05` / `0.02` | Faucet amount and balance floor. |

### Advisory CI against Vibenet

`.github/workflows/smoke-tests.yml` runs the **full suite** against a live Base network on demand. It
is **`workflow_dispatch` only** — deliberately not wired to pull requests, merge groups, or a schedule.
Vibenet is a live chain that is manually deployed per hardfork, so CI can't guarantee which precompile
set is live; an automated run would flap. The workflow is **advisory and never gates merges**: run it
by hand (Actions → *Base Std Smoke Tests (Vibenet)* → *Run workflow*) after a hardfork ships. Its only
input is the RPC endpoint (default `https://rpc.vibes.base.org/`); it always runs every journey (`-k`)
against the ref you dispatch from — latest is `main`, and to run an older fork you dispatch from the
matching branch/tag (branch-per-fork; there is no journey picker and no fork/ref selector). It exports
`RPC_URL` from the input and `DEPLOYER_PK` / `USER2_PK` from repo secrets (`SMOKE_DEPLOYER_PK` /
`SMOKE_USER2_PK`), then reports per-journey **passed / failed / skipped** plus the chain id in the run
summary. A journey whose surface the live chain does not yet ship is reported as *skipped*, not failed.

## What it checks

Five "journeys", each runnable on its own or all together:
Six "journeys", run as a whole suite (a single journey can still be run via the CLI for debugging):

| Journey | What it exercises |
|---|---|
| `factory` | Deterministic create + address prediction, the `isB20` / `isB20Initialized` query surface, and creation-time reverts (duplicate salt, bad decimals, bad currency, unknown variant). |
| `asset` | Full Asset-variant lifecycle (18 decimals): mint, transfer, `transferWithMemo`, delegated `transferFrom`, `announce` + `batchMint`, rebase via `updateMultiplier`, metadata, burn, then the gates that must reject (supply cap, pause, role, announcement-id reuse). |
| `asset` | Full Asset-variant lifecycle (18 decimals): mint, transfer, `transferWithMemo`, delegated `transferFrom`, `announce` + `batchMint`, rebase via `updateMultiplier`, metadata, burn, then the gates that must reject (supply cap, pause, role, announcement-id reuse). The rebase event is fork-aware (V1 `MultiplierUpdated` vs Cobalt `UIMultiplierUpdated`). |
| `multiplier` | ERC-8056 scheduled multiplier (AssetV2 @ Cobalt): `setUIMultiplier` scheduling + its guards (`InvalidMultiplier`, `EffectiveAtInPast`, `EffectiveAtTooFar`, `ScheduleOverlap`), `cancelScheduledMultiplier` (+ `NoScheduledMultiplier`), the `updateMultiplier` instant-failsafe V2 event semantics (`UIMultiplierUpdated` + `MultiplierUpdateCancelled`, *not* `MultiplierUpdated`), the read aliases (`uiMultiplier`/`balanceOfUI`/`totalSupplyUI`), and ERC-165 advertisement. **Skips** cleanly on a pre-Cobalt chain (probed via `supportsInterface(0xa60bf13d)`). |
| `stablecoin` | Stablecoin-variant deltas (fixed 6 decimals, immutable currency) plus the regulated freeze-and-seize path (blocklist policy + `burnBlocked`). |
| `policy` | Policy creation (both types), membership, built-in sentinels, the two-step admin transfer lifecycle, and a token actually *enforcing* a policy (`PolicyForbids` on transfer + mint). |
| `invariants` | EVM-context invariants a precompile must implement explicitly: payable rejection, unknown-selector revert, strict ABI decode, dirty-bit canonicalization, `STATICCALL` read-only enforcement, returndata fidelity, OOG containment, revert atomicity, and gas independence from a force-fed balance. Uses the `PrecompileProbe` + `ForceFeeder` helpers under `test/lib/`. |
Expand All @@ -105,17 +130,29 @@ journey:
- **fail** — an assertion or expected revert did not match. For lifecycle
journeys this is fail-fast; the harness dumps the offending call (and a trace
when `SMOKE_TRACE=1`).
- **skip** — the preflight found the b20 features are **not activated** on the
target chain. Reported as chain/fork state, *not* a contract defect:
- **skip** — reported as chain/fork state, *not* a contract defect. Two cases:

1. The preflight found the b20 features are **not activated** on the target chain:

```
[smoke] b20 features NOT ACTIVE on chain <id>: ActivationRegistry not installed ... fork < Beryl?
[smoke] ... skipping (use the ActivationRegistry to enable).
```

If everything skips, your RPC simply doesn't have the precompiles active.
Activate the b20 features in the ActivationRegistry, or point `RPC_URL` at a
node that already has them.

2. A journey's surface only exists on a **later fork** than the target chain — e.g. the
`multiplier` journey against a pre-Cobalt chain (the ERC-8056 scheduled multiplier is
AssetV2 @ Cobalt). It probes `supportsInterface(0xa60bf13d)` and opts out:

```
[smoke] b20 features NOT ACTIVE on chain <id>: ActivationRegistry not installed ... fork < Beryl?
[smoke] ... skipping (use the ActivationRegistry to enable).
```
```
[smoke] multiplier: not applicable on chain <id> — Asset does not advertise ERC-8056 ... pre-Cobalt
```

If everything skips, your RPC simply doesn't have the precompiles active.
Activate the b20 features in the ActivationRegistry, or point `RPC_URL` at a
node that already has them.
Point `RPC_URL` at a chain on the fork that ships the surface (branch-per-fork: run the
matching base-std ref).

The `invariants` journey is special: it collects all findings and prints
`N/12 invariants held`. A finding is a precompile behavior to triage, not a flaky
Expand Down Expand Up @@ -182,6 +219,6 @@ script/smoke/
abis.py # interface ABIs + probe/feeder artifacts, read from out/
codec.py # the one hand-written encode: createB20 params + initCalls
errors.py # selector -> custom-error-name map (from the ABIs)
journeys/ # factory, asset_lifecycle, stablecoin_lifecycle, policy_registry, precompile_invariants
journeys/ # factory, asset_lifecycle, scheduled_multiplier, stablecoin_lifecycle, policy_registry, precompile_invariants
requirements.txt
```
15 changes: 11 additions & 4 deletions script/smoke/__main__.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""CLI: python -m smoke <journey> [<journey> ...] [-k]

Journeys: factory, asset, stablecoin, policy, invariants — or `all` to run every
journey in sequence. Env (RPC_URL / DEPLOYER_PK / USER2_PK) is sourced by the
Journeys: factory, asset, multiplier, stablecoin, policy, invariants — or `all` to run
every journey in sequence. Env (RPC_URL / DEPLOYER_PK / USER2_PK) is sourced by the
Makefile from .env; running directly requires it exported. A preflight liveness
probe checks the b20 precompiles are actually active on the target chain (fork
>= Beryl); if not, the journey is skipped rather than reporting environment state
Expand All @@ -20,18 +20,19 @@
import sys

from . import config
from .chain import Chain, log
from .chain import Chain, Skip, log

JOURNEYS = {
"factory": "smoke.journeys.factory",
"asset": "smoke.journeys.asset_lifecycle",
"multiplier": "smoke.journeys.scheduled_multiplier",
"stablecoin": "smoke.journeys.stablecoin_lifecycle",
"policy": "smoke.journeys.policy_registry",
"invariants": "smoke.journeys.precompile_invariants",
}

# Canonical run order; also the expansion of `all`.
ORDER = ["factory", "asset", "stablecoin", "policy", "invariants"]
ORDER = ["factory", "asset", "multiplier", "stablecoin", "policy", "invariants"]


def _usage() -> str:
Expand Down Expand Up @@ -78,6 +79,12 @@ def main(argv: list[str]) -> None:
try:
module.run(chain)
results.append((name, "pass", ""))
except Skip as exc:
# A journey opting out because its surface only exists on a later fork (e.g. the ERC-8056
# scheduled multiplier at Cobalt). Fork/activation state, not a defect — record and continue
# even in fail-fast mode, mirroring the features-not-active preflight skip above.
log(f"{name}: not applicable on chain {chain.chain_id} \u2014 {exc}")
results.append((name, "skip", str(exc)))
except SystemExit as exc:
if not keep_going:
raise
Expand Down
Loading
Loading