From 83b1f6615e794d481750b369d0599a4b874d567c Mon Sep 17 00:00:00 2001 From: dicethedev Date: Tue, 12 May 2026 21:10:32 +0100 Subject: [PATCH 1/6] ci: run Lean Hive simulators in CI --- .github/config/hive/clients.yaml | 2 + .github/scripts/check-hive-results.sh | 82 +++++++++++++++++ .github/workflows/ci.yml | 126 ++++++++++++++++++++++++++ 3 files changed, 210 insertions(+) create mode 100644 .github/config/hive/clients.yaml create mode 100644 .github/scripts/check-hive-results.sh diff --git a/.github/config/hive/clients.yaml b/.github/config/hive/clients.yaml new file mode 100644 index 00000000..1a196dfc --- /dev/null +++ b/.github/config/hive/clients.yaml @@ -0,0 +1,2 @@ +- client: ethlambda + nametag: devnet4 diff --git a/.github/scripts/check-hive-results.sh b/.github/scripts/check-hive-results.sh new file mode 100644 index 00000000..d1cb3e21 --- /dev/null +++ b/.github/scripts/check-hive-results.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +set -euo pipefail + +results_dir="${1:-src/results}" + +if ! command -v jq >/dev/null 2>&1; then + echo "jq is required to parse Hive results but was not found in PATH" + exit 1 +fi + +if [ ! -d "${results_dir}" ]; then + echo "Hive results directory '${results_dir}' not found" + exit 1 +fi + +shopt -s nullglob +json_files=("${results_dir}"/*.json) +shopt -u nullglob + +if [ "${#json_files[@]}" -eq 0 ]; then + echo "No Hive JSON result files found in ${results_dir}" + exit 1 +fi + +failures=0 +failed_logs_root="${results_dir}/failed_logs" +rm -rf "${failed_logs_root}" +mkdir -p "${failed_logs_root}" + +for json_file in "${json_files[@]}"; do + if [[ "${json_file}" == *"hive.json" ]]; then + continue + fi + + suite_name="$(jq -r '.name // empty' "${json_file}")" + suite_label="${suite_name:-$(basename "${json_file}" .json)}" + failed_cases="$(jq '[.testCases[]? | select(.summaryResult.pass != true)] | length' "${json_file}")" + + if [ "${failed_cases}" -eq 0 ]; then + continue + fi + + failures=$((failures + failed_cases)) + echo "Detected ${failed_cases} failing Hive test case(s) in ${suite_label}" + + failure_list="$( + jq -r ' + .testCases[]? + | select(.summaryResult.pass != true) + | . as $case + | ($case.summaryResult // {}) as $summary + | ($summary.message // $summary.reason // $summary.error // "") as $message + | "- " + ($case.name // "unknown test") + + (if $message != "" then ": " + $message else "" end) + ' "${json_file}" + )" + + printf '%s\n' "${failure_list}" + + suite_slug="$(printf '%s' "${suite_label}" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9._-]+/-/g; s/^-//; s/-$//')" + suite_dir="${failed_logs_root}/${suite_slug:-suite}" + mkdir -p "${suite_dir}" + + cp "${json_file}" "${suite_dir}/" + printf '%s\n\n%s\n' "Detected ${failed_cases} failing Hive test case(s) in ${suite_label}" "${failure_list}" > "${suite_dir}/failed-tests.txt" + + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### Hive failures: ${suite_label}" + printf '%s\n' "${failure_list}" + echo + } >> "${GITHUB_STEP_SUMMARY}" + fi +done + +if [ "${failures}" -gt 0 ]; then + echo "Hive reported ${failures} failing test case(s) in total" + exit 1 +fi + +echo "Hive reported no failing test cases." diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0c70608..773ed7c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,3 +146,129 @@ jobs: - name: Run tests run: make test + + docker_build: + name: Build Docker + runs-on: ubuntu-latest + if: ${{ needs.detect-changes.outputs.run_tests == 'true' }} + steps: + - uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build ethlambda Docker image + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + build-args: | + GIT_COMMIT=${{ github.sha }} + GIT_BRANCH=${{ github.ref_name }} + push: false + tags: | + ghcr.io/lambdaclass/ethlambda:devnet3 + ghcr.io/lambdaclass/ethlambda:devnet4 + outputs: type=docker,dest=/tmp/ethlambda_image.tar + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Upload image artifact + uses: actions/upload-artifact@v6 + with: + name: ethlambda_image + path: /tmp/ethlambda_image.tar + + run-hive: + name: Hive - ${{ matrix.name }} + runs-on: ubuntu-latest + needs: docker_build + strategy: + fail-fast: false + matrix: + include: + - name: "Lean RPC Compat" + limit: "rpc-compat" + artifact_prefix: lean_rpc_compat + - name: "Lean Sync" + limit: "sync" + artifact_prefix: lean_sync + - name: "Lean Client Interop" + limit: "client-interop" + artifact_prefix: lean_client_interop + - name: "Lean Validation" + limit: "validation" + artifact_prefix: lean_validation + - name: "Lean Gossip" + limit: "gossip" + artifact_prefix: lean_gossip + - name: "Lean ReqResp" + limit: "reqresp" + artifact_prefix: lean_reqresp + - name: "Lean Fork Choice Spec Tests" + limit: "lean-spec-tests-fork-choice" + artifact_prefix: lean_spec_fork_choice + - name: "Lean State Transition Spec Tests" + limit: "lean-spec-tests-state-transition" + artifact_prefix: lean_spec_state_transition + - name: "Lean Verify Signatures Spec Tests" + limit: "lean-spec-tests-verify-signatures" + artifact_prefix: lean_spec_verify_signatures + steps: + - uses: actions/checkout@v6 + + - name: Download ethlambda image artifact + uses: actions/download-artifact@v6 + with: + name: ethlambda_image + path: /tmp + + - name: Load image + run: docker load --input /tmp/ethlambda_image.tar + + - name: Load hive client config + id: client-config + shell: bash + run: | + { + echo "config<> "$GITHUB_OUTPUT" + + - name: Determine hive flags + id: hive-flags + shell: bash + env: + SIM_LIMIT: ${{ matrix.limit }} + run: | + FLAGS="--sim.parallelism 4 --sim.loglevel 3" + if [[ -n "$SIM_LIMIT" ]]; then + escaped_limit=${SIM_LIMIT//\'/\'\\\'\'} + FLAGS+=" --sim.limit '$escaped_limit'" + fi + echo "flags=$FLAGS" >> "$GITHUB_OUTPUT" + + - name: Run Hive Simulation + uses: ethpandaops/hive-github-action@v0.5.0 + with: + hive_repository: ethereum/hive + hive_version: 6f704d0cd8fb4dfd3f635abcc5a3ea6cd395e61b + simulator: lean + client: ethlambda + client_config: ${{ steps.client-config.outputs.config }} + extra_flags: ${{ steps.hive-flags.outputs.flags }} + + - name: Check Hive Results For Failures + id: verify-hive-results + if: ${{ success() }} + shell: bash + run: bash ./.github/scripts/check-hive-results.sh src/results + + - name: Upload Hive Failure Logs + if: ${{ failure() && steps.verify-hive-results.conclusion == 'failure' }} + uses: actions/upload-artifact@v6 + with: + name: hive_failed_logs_${{ matrix.artifact_prefix }} + path: src/results/failed_logs + if-no-files-found: warn From 8647c25dce44375bddd91b22d0e8c52782728452 Mon Sep 17 00:00:00 2001 From: dicethedev Date: Tue, 12 May 2026 21:24:26 +0100 Subject: [PATCH 2/6] ci: add Hive config for both ethlambda devnets --- .github/config/hive/clients.yaml | 2 ++ .github/workflows/ci.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/config/hive/clients.yaml b/.github/config/hive/clients.yaml index 1a196dfc..8d25fc55 100644 --- a/.github/config/hive/clients.yaml +++ b/.github/config/hive/clients.yaml @@ -1,2 +1,4 @@ +- client: ethlambda + nametag: devnet3 - client: ethlambda nametag: devnet4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 773ed7c1..e7059f91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -255,7 +255,7 @@ jobs: hive_repository: ethereum/hive hive_version: 6f704d0cd8fb4dfd3f635abcc5a3ea6cd395e61b simulator: lean - client: ethlambda + client: ethlambda_devnet4 client_config: ${{ steps.client-config.outputs.config }} extra_flags: ${{ steps.hive-flags.outputs.flags }} From 7319fb86a53b2e672d32884923ef5d977f6c53d9 Mon Sep 17 00:00:00 2001 From: dicethedev Date: Tue, 12 May 2026 22:44:00 +0100 Subject: [PATCH 3/6] update ci --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7059f91..4d8a3117 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,7 +150,6 @@ jobs: docker_build: name: Build Docker runs-on: ubuntu-latest - if: ${{ needs.detect-changes.outputs.run_tests == 'true' }} steps: - uses: actions/checkout@v6 @@ -173,7 +172,7 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max - - name: Upload image artifact + - name: Upload image artifacts uses: actions/upload-artifact@v6 with: name: ethlambda_image @@ -255,7 +254,7 @@ jobs: hive_repository: ethereum/hive hive_version: 6f704d0cd8fb4dfd3f635abcc5a3ea6cd395e61b simulator: lean - client: ethlambda_devnet4 + client: ethlambda client_config: ${{ steps.client-config.outputs.config }} extra_flags: ${{ steps.hive-flags.outputs.flags }} From 1a227e5b10c0e8f2c2b9baaf28ffb66e033751ce Mon Sep 17 00:00:00 2001 From: dicethedev Date: Fri, 15 May 2026 03:31:17 +0100 Subject: [PATCH 4/6] ci: harden hive workflow permissions and action pinning --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d8a3117..1d9f4b41 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,6 +150,8 @@ jobs: docker_build: name: Build Docker runs-on: ubuntu-latest + permissions: + contents: read steps: - uses: actions/checkout@v6 @@ -181,6 +183,8 @@ jobs: run-hive: name: Hive - ${{ matrix.name }} runs-on: ubuntu-latest + permissions: + contents: read needs: docker_build strategy: fail-fast: false @@ -249,7 +253,7 @@ jobs: echo "flags=$FLAGS" >> "$GITHUB_OUTPUT" - name: Run Hive Simulation - uses: ethpandaops/hive-github-action@v0.5.0 + uses: ethpandaops/hive-github-action@1aa8d73dad34de13afbb3113ab16c1a462d2fbc3 # v0.5.0 with: hive_repository: ethereum/hive hive_version: 6f704d0cd8fb4dfd3f635abcc5a3ea6cd395e61b From 05af189ce70fb9d3d5f8f5fed69d46eb57edc77e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 15 May 2026 16:01:20 -0300 Subject: [PATCH 5/6] chore: run devnet4 only --- .github/config/hive/clients.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/config/hive/clients.yaml b/.github/config/hive/clients.yaml index 8d25fc55..1a196dfc 100644 --- a/.github/config/hive/clients.yaml +++ b/.github/config/hive/clients.yaml @@ -1,4 +1,2 @@ -- client: ethlambda - nametag: devnet3 - client: ethlambda nametag: devnet4 From 6f3ccdbdf25c75009f271e931fecbbdbab337bce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 15 May 2026 16:01:34 -0300 Subject: [PATCH 6/6] chore: tag devnet4 only --- .github/workflows/ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d9f4b41..8b701979 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,9 +167,7 @@ jobs: GIT_COMMIT=${{ github.sha }} GIT_BRANCH=${{ github.ref_name }} push: false - tags: | - ghcr.io/lambdaclass/ethlambda:devnet3 - ghcr.io/lambdaclass/ethlambda:devnet4 + tags: ghcr.io/lambdaclass/ethlambda:devnet4 outputs: type=docker,dest=/tmp/ethlambda_image.tar cache-from: type=gha cache-to: type=gha,mode=max