feat(test): forest + lotus-miner devnet in CI

.github/workflows/forest.yml

@@ -456,10 +456,20 @@ jobs:
         run: ./scripts/tests/calibnet_db_migration.sh
         timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}
   local-devnet-check:
-    name: Devnet checks
+    name: Devnet checks${{ matrix.label }}
     runs-on: ubuntu-24.04
     needs:
       - build-ubuntu
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - label: ""
+            dir: scripts/devnet
+            proofs_volume: devnet_filecoin-proofs
+          - label: " (Forest miner)"
+            dir: scripts/devnet-forest-miner
+            proofs_volume: devnet-forest-miner_filecoin-proofs
     env: # We use a custom Dockerfile for CI to speed up the build process.
       FOREST_DOCKERFILE_OVERRIDE: scripts/devnet/forest_ci.dockerfile
     steps:
@@ -471,9 +481,9 @@ jobs:
       - name: Load cache into volume
         if: steps.cache.outputs.cache-hit == 'true'
         run: |
-          docker volume create devnet_filecoin-proofs
+          docker volume create "${{ matrix.proofs_volume }}"
           docker run --rm \
-            -v devnet_filecoin-proofs:/proofs \
+            -v "${{ matrix.proofs_volume }}":/proofs \
             -v "$FIL_PROOFS_PARAMETER_CACHE":/cache \
             "$SHELL_IMAGE" \
             sh -c "cp /cache/* /proofs"
@@ -482,10 +492,10 @@ jobs:
         with:
           name: "forest-${{ runner.os }}"
       - name: Devnet setup
-        run: ./scripts/devnet/setup.sh
+        run: ./${{ matrix.dir }}/setup.sh
         timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}
       - name: Devnet check
-        run: ./scripts/devnet/check.sh
+        run: ./${{ matrix.dir }}/check.sh
         timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}
       - name: Add forest binaries to PATH
         run: |
@@ -494,6 +504,7 @@ jobs:
       - name: Devnet tests
         run: |
           set -euo pipefail
+          export DEVNET_DIR=${{ matrix.dir }}
           source ./scripts/devnet/test_harness.sh
           devnet_test_env_init
           forest-dev tests mpool
@@ -508,7 +519,7 @@ jobs:
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
           docker run --rm \
-            -v devnet_filecoin-proofs:/proofs \
+            -v "${{ matrix.proofs_volume }}":/proofs \
             -v "$FIL_PROOFS_PARAMETER_CACHE":/cache \
             "$SHELL_IMAGE" \
             sh -c "cp /proofs/* /cache"
@@ -601,7 +612,6 @@ jobs:
       - calibnet-api-test-stateful
       - db-migration-checks
       - local-devnet-check
-      # - local-devnet-curio-check
       - bootstrap-checks-forest
       - bootstrap-checks-lotus
     name: Integration tests status

.github/workflows/lotus-api-bump.yml

@@ -23,10 +23,12 @@ jobs:
           # largest version for calibration network compatibility.
           if [[ "$(printf '%s\n' "$TAG" "$CURRENT_TAG" | sort -V | tail -n1)" != "$CURRENT_TAG" || "$CURRENT_TAG" == "$TAG"* ]]; then
             echo "Updating Lotus version to $TAG"
-            sed -i "s/\(LOTUS_IMAGE=filecoin.*\):.*/\1:$TAG-calibnet/" ./scripts/tests/api_compare/.env
-            sed -i "s/\(LOTUS_IMAGE=filecoin.*\):.*/\1:$TAG-calibnet/" ./scripts/tests/bootstrapper/.env
-            sed -i "s/\(LOTUS_IMAGE=filecoin.*\):.*/\1:$TAG-calibnet/" ./scripts/tests/snapshot_parity/.env
-            sed -i "s/\(LOTUS_IMAGE=filecoin.*\):.*/\1:$TAG-2k/" ./scripts/devnet/.env
+            for env in ./scripts/tests/api_compare/.env ./scripts/tests/bootstrapper/.env ./scripts/tests/snapshot_parity/.env; do
+              sed -i "s/\(LOTUS_IMAGE=filecoin.*\):.*/\1:$TAG-calibnet/" "$env"
+            done
+            for env in ./scripts/devnet/.env ./scripts/devnet-forest-miner/.env; do
+              sed -i "s/\(LOTUS_IMAGE=filecoin.*\):.*/\1:$TAG-2k/" "$env"
+            done
           else
             echo "Lotus version is already up to date"
           fi

mise.toml

@@ -237,20 +237,11 @@ forest-dev tests wallet
 
 [tasks."test:devnet"]
 description = "Run the integration test suites against a local docker devnet."
-shell = "bash -c"
-run = '''
-set -euo pipefail
-pushd scripts/devnet
-./setup.sh
-./check.sh
-popd
-source ./scripts/devnet/test_harness.sh
-devnet_test_env_init
-forest-dev tests mpool
-forest-dev tests wallet
-forest-dev devnet eth-gas
-forest-dev devnet eth-skip-sender
-'''
+run = "scripts/devnet/run_integration_tests.sh scripts/devnet"
+
+[tasks."test:devnet:forest-miner"]
+description = "Run the integration test suites against the Forest-produced local docker devnet."
+run = "scripts/devnet/run_integration_tests.sh scripts/devnet-forest-miner"
 
 [tasks."codecov:nextest"]
 description = "Run codecov with nextest"

scripts/devnet-forest-miner/.env

@@ -0,0 +1,28 @@
+LOTUS_IMAGE=filecoin/lotus-all-in-one:c4e3b46b1-2k
+# Genesis network name. Lotus stamps it into the genesis block; Forest joins the
+# same network via `--chain ${NETWORK_NAME}`, so the two must agree (this is what
+# makes the `/fil/blocks/${NETWORK_NAME}` gossip topic line up).
+NETWORK_NAME=devnet
+FOREST_DATA_DIR=/forest_data
+LOTUS_DATA_DIR=/lotus_data
+FIL_PROOFS_PARAMETER_CACHE=/var/tmp/filecoin-proof-parameters
+MINER_ACTOR_ADDRESS=t01000
+LOTUS_RPC_PORT=1234
+LOTUS_P2P_PORT=1235
+MINER_RPC_PORT=2345
+FOREST_RPC_PORT=3456
+FOREST_P2P_PORT=4903
+GENESIS_NETWORK_VERSION=18
+SHARK_HEIGHT=-10
+HYGGE_HEIGHT=-9
+LIGHTNING_HEIGHT=3
+THUNDER_HEIGHT=6
+WATERMELON_HEIGHT=9
+DRAGON_HEIGHT=12
+WAFFLE_HEIGHT=18
+TUKTUK_HEIGHT=20
+TEEP_HEIGHT=22
+TOCK_HEIGHT=24
+GOLDEN_WEEK_HEIGHT=26
+FIREHORSE_HEIGHT=28
+TARGET_HEIGHT=32

scripts/devnet-forest-miner/README.md

@@ -0,0 +1,18 @@
+# Forest-produced devnet (Lotus validates)
+
+Mirrors [`../devnet`](../devnet) with Lotus and Forest roles swapped:
+
+|                 | `../devnet`              | this devnet              |
+| --------------- | ------------------------ | ------------------------ |
+| Block producer  | Lotus node + Lotus Miner | **Forest** + Lotus Miner |
+| Validating node | Forest                   | **Lotus**                |
+
+Lotus Miner produces blocks entirely through Forest's RPC (`MinerGetBaseInfo` ->
+`MinerCreateBlock` -> `SyncSubmitBlock`); Forest gossips them and the Lotus node
+validates and follows.
+
+## Integration tests
+
+```shell
+mise run test:devnet:forest-miner
+```

scripts/devnet-forest-miner/check.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Assert that Lotus Miner drives Forest past the target height AND that the Lotus
+# validating node follows it there. Both climbing proves block production through
+# Forest's RPC works and that Lotus accepts (validates) Forest-produced blocks.
+
+set -eux
+
+# Path to the directory containing this script.
+PARENT_PATH=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
+pushd "${PARENT_PATH}"
+source .env
+
+function get_sync_height {
+  local port=$1
+  curl --silent -X POST -H "Content-Type: application/json" \
+       --data '{"jsonrpc":"2.0","id":2,"method":"Filecoin.ChainHead","param":"null"}' \
+       "http://127.0.0.1:${port}/rpc/v1" | jq '.result.Height'
+}
+
+start_time=$(date +%s)
+timeout=$((start_time + 360))  # 6 minutes
+
+# Target height chosen so that every migration in .env is crossed.
+target_height=$TARGET_HEIGHT
+
+while true; do
+  forest_height=$(get_sync_height "${FOREST_RPC_PORT}")
+  lotus_height=$(get_sync_height "${LOTUS_RPC_PORT}")
+  if [ "$forest_height" -gt "$target_height" ] && [ "$lotus_height" -gt "$target_height" ]; then
+    echo "Both nodes past $target_height (Forest: $forest_height, Lotus: $lotus_height): Lotus validated Forest-produced blocks"
+    break
+  fi
+
+  current_time=$(date +%s)
+  if [ "$current_time" -gt "$timeout" ]; then
+    echo "Timeout reached (Forest: $forest_height, Lotus: $lotus_height), target $target_height not reached by both"
+    exit 1
+  fi
+
+  sleep 1
+done