change(ci): Generate mainnet checkpoints in CI (#6550)

* Add extra test type modes to support zebra-checkpoints * Add Mainnet and Testnet zebra-checkpoints test harnesses * Add zebra-checkpoints to test docker images * Add zebra-checkpoints test entrypoints * Add Mainnet CI workflow for zebra-checkpoints * Enable zebra-checkpoints feature in the test image * Use the same features for (almost) all the docker tests * Make workflow features match Docker features * Add a feature note * Add a zebra-checkpoints test feature to zebrad * Remove the "no cached state" testnet code * Log a startup message to standard error when launching zebra-checkpoints * Rename tests to avoid partial name conflicts * Fix log formatting * Add sentry feature to experimental docker image build * Explain what ENTRYPOINT_FEATURES is used for * Use the correct zebra-checkpoints path * Silence zebrad logs while generating checkpoints * Fix zebra-checkpoints log handling * Re-enable waiting for zebrad to fully sync * Add documentation for how to run these tests individually * Start generating checkpoints from the last compiled-in checkpoint * Fix clippy lints * Revert changes to TestType * Wait for all the checkpoints before finishing * Add more stderr debugging to zebra-checkpoints * Fix an outdated module comment * Add a workaround for zebra-checkpoints launch/run issues * Use temp dir and log what it is * Log extra metadata about the zebra-checkpoints binary * Add note about unstable feature -Z bindeps * Temporarily make the test run faster and with debug info * Log the original test command name when showing stdout and stderr * Try zebra-checkpoints in the system path first, then the cargo path * Fix slow thread close bug in dual process test harness * If the logs are shown, don't say they are hidden * Run `zebra-checkpoints --help` to work out what's going on in CI * Build `zebra-utils` binaries for `zebrad` integration tests * Revert temporary debugging changes * Revert changes that were moved to another PR
ZcashFoundation · Apr 27, 2023 · 1461c91 · 1461c91
1 parent 0ffd31e
commit 1461c91
Show file tree

Hide file tree

Showing 20 changed files with 818 additions and 83 deletions.
diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml
@@ -37,12 +37,15 @@ on:
         required: false
         type: string
         default: info
+      # keep these in sync with:
+      # https://github.com/ZcashFoundation/zebra/blob/main/docker/Dockerfile#L83
       features:
         required: false
+        default: "sentry"
         type: string
       test_features: 
         required: false
-        default: "lightwalletd-grpc-tests"
+        default: "lightwalletd-grpc-tests zebra-checkpoints"
         type: string
       rpc_port: 
         required: false

diff --git a/.github/workflows/continous-integration-docker.patch.yml b/.github/workflows/continous-integration-docker.patch.yml
@@ -90,6 +90,12 @@ jobs:
     steps:
       - run: 'echo "No build required"'
 
+  generate-checkpoints-mainnet:
+    name: Generate checkpoints mainnet / Run generate-checkpoints-mainnet test
+    runs-on: ubuntu-latest
+    steps:
+      - run: 'echo "No build required"'
+
   lightwalletd-rpc-test:
     name: Zebra tip JSON-RPC / Run fully-synced-rpc test
     runs-on: ubuntu-latest

diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml
@@ -461,6 +461,38 @@ jobs:
       height_grep_text: 'current_height.*=.*Height.*\('
     secrets: inherit
 
+  # zebra checkpoint generation tests
+
+  # Test that Zebra can generate mainnet checkpoints after syncing to the chain tip,
+  # using a cached Zebra tip state,
+  #
+  # Runs:
+  # - after every PR is merged to `main`
+  # - on every PR update
+  #
+  # If the state version has changed, waits for the new cached state to be created.
+  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
+  generate-checkpoints-mainnet:
+    name: Generate checkpoints mainnet
+    needs: test-full-sync
+    uses: ./.github/workflows/deploy-gcp-tests.yml
+    if: ${{ !cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
+    with:
+      app_name: zebrad
+      test_id: generate-checkpoints-mainnet
+      test_description: Generate Zebra checkpoints on mainnet
+      test_variables: '-e GENERATE_CHECKPOINTS_MAINNET=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
+      needs_zebra_state: true
+      # update the disk on every PR, to increase CI speed
+      saves_to_disk: true
+      disk_suffix: tip
+      root_state_path: '/var/cache'
+      zebra_state_dir: 'zebrad-cache'
+      height_grep_text: 'current_height.*=.*Height.*\('
+    secrets: inherit
+
+  # TODO: testnet checkpoints, test-full-sync, test-update-sync, get-available-disks (+ reusable)
+
   # lightwalletd cached tip state tests
 
   # Test full sync of lightwalletd with a Zebra tip state

diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
@@ -46,7 +46,7 @@ jobs:
       tag_suffix: .experimental
       network: Testnet
       rpc_port: '18232'
-      features: "getblocktemplate-rpcs"
+      features: "sentry getblocktemplate-rpcs"
       test_features: ""
       checkpoint_sync: true
       rust_backtrace: '1'

diff --git a/Cargo.lock b/Cargo.lock
@@ -6254,6 +6254,7 @@ dependencies = [
  "zebra-rpc",
  "zebra-state",
  "zebra-test",
+ "zebra-utils",
 ]
 
 [[package]]

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -80,8 +80,13 @@ ARG CHECKPOINT_SYNC
 ENV CHECKPOINT_SYNC ${CHECKPOINT_SYNC:-true}
 
 # Build zebrad with these features
-ARG FEATURES
-ARG TEST_FEATURES="lightwalletd-grpc-tests"
+# Keep these in sync with:
+# https://github.com/ZcashFoundation/zebra/blob/main/.github/workflows/build-docker-image.yml#L42
+ARG FEATURES="sentry"
+ARG TEST_FEATURES="lightwalletd-grpc-tests zebra-checkpoints"
+# Use ENTRYPOINT_FEATURES to override the specific features used to run tests in entrypoint.sh,
+# separately from the test and production image builds.
+ENV ENTRYPOINT_FEATURES "$TEST_FEATURES $FEATURES"
 
 ARG NETWORK
 ENV NETWORK ${NETWORK:-Mainnet}
@@ -104,11 +109,12 @@ COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/lightwalletd /opt/lightwalletd
 # This is the caching Docker layer for Rust!
 #
 # TODO: is it faster to use --tests here?
-RUN cargo chef cook --release --features "sentry ${TEST_FEATURES} ${FEATURES}" --workspace --recipe-path recipe.json
+RUN cargo chef cook --release --features "${TEST_FEATURES} ${FEATURES}" --workspace --recipe-path recipe.json
 
 COPY . .
 RUN cargo test --locked --release --features "${TEST_FEATURES} ${FEATURES}" --workspace --no-run
 RUN cp /opt/zebrad/target/release/zebrad /usr/local/bin
+RUN cp /opt/zebrad/target/release/zebra-checkpoints /usr/local/bin
 
 COPY ./docker/entrypoint.sh /
 RUN chmod u+x /entrypoint.sh
@@ -122,11 +128,11 @@ ENTRYPOINT [ "/entrypoint.sh" ]
 # `test` stage. This step is a dependency for the `runtime` stage, which uses the resulting
 # zebrad binary from this step.
 FROM deps AS release
-RUN cargo chef cook --release --features "sentry ${FEATURES}" --recipe-path recipe.json
+RUN cargo chef cook --release --features "${FEATURES}" --recipe-path recipe.json
 
 COPY . .
 # Build zebra
-RUN cargo build --locked --release --features "sentry ${FEATURES}" --package zebrad --bin zebrad
+RUN cargo build --locked --release --features "${FEATURES}" --package zebrad --bin zebrad
 
 # This stage is only used when deploying nodes or when only the resulting zebrad binary is needed
 #

diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
@@ -13,82 +13,106 @@ echo "ZEBRA_TEST_LIGHTWALLETD=$ZEBRA_TEST_LIGHTWALLETD"
 echo "Hard-coded Zebra full sync directory: /zebrad-cache"
 echo "ZEBRA_CACHED_STATE_DIR=$ZEBRA_CACHED_STATE_DIR"
 echo "LIGHTWALLETD_DATA_DIR=$LIGHTWALLETD_DATA_DIR"
+echo "ENTRYPOINT_FEATURES=$ENTRYPOINT_FEATURES"
 
 case "$1" in
     --* | -*)
         exec zebrad "$@"
         ;;
     *)
-        # For these tests, we activate the gRPC feature to avoid recompiling `zebrad`,
-        # but we might not actually run any gRPC tests.
+        # For these tests, we activate the test features to avoid recompiling `zebrad`,
+        # but we don't actually run any gRPC tests.
         if [[ "$RUN_ALL_TESTS" -eq "1" ]]; then
             # Run all the available tests for the current environment.
             # If the lightwalletd environmental variables are set, we will also run those tests.
-            cargo test --locked --release --features lightwalletd-grpc-tests --workspace -- --nocapture --include-ignored
+            cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --workspace -- --nocapture --include-ignored
 
         # For these tests, we activate the gRPC feature to avoid recompiling `zebrad`,
         # but we don't actually run any gRPC tests.
         elif [[ "$TEST_FULL_SYNC" -eq "1" ]]; then
             # Run a Zebra full sync test.
-            cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored full_sync_mainnet
+            cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored full_sync_mainnet
             # List directory generated by test
             # TODO: replace with $ZEBRA_CACHED_STATE_DIR in Rust and workflows
             ls -lh "/zebrad-cache"/*/* || (echo "No /zebrad-cache/*/*"; ls -lhR "/zebrad-cache" | head -50 || echo "No /zebrad-cache directory")
         elif [[ "$TEST_DISK_REBUILD" -eq "1" ]]; then
             # Run a Zebra sync up to the mandatory checkpoint.
             #
             # TODO: use environmental variables instead of Rust features (part of #2995)
-            cargo test --locked --release --features "test_sync_to_mandatory_checkpoint_${NETWORK,,},lightwalletd-grpc-tests" --package zebrad --test acceptance -- --nocapture --include-ignored "sync_to_mandatory_checkpoint_${NETWORK,,}"
+            cargo test --locked --release --features "test_sync_to_mandatory_checkpoint_${NETWORK,,},$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored "sync_to_mandatory_checkpoint_${NETWORK,,}"
             # TODO: replace with $ZEBRA_CACHED_STATE_DIR in Rust and workflows
             ls -lh "/zebrad-cache"/*/* || (echo "No /zebrad-cache/*/*"; ls -lhR "/zebrad-cache" | head -50 || echo "No /zebrad-cache directory")
         elif [[ "$TEST_UPDATE_SYNC" -eq "1" ]]; then
             # Run a Zebra sync starting at the cached tip, and syncing to the latest tip.
             #
             # List directory used by test
             ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
-            cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored zebrad_update_sync
+            cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored zebrad_update_sync
         elif [[ "$TEST_CHECKPOINT_SYNC" -eq "1" ]]; then
             # Run a Zebra sync starting at the cached mandatory checkpoint, and syncing past it.
             #
             # List directory used by test
             # TODO: replace with $ZEBRA_CACHED_STATE_DIR in Rust and workflows
             ls -lh "/zebrad-cache"/*/* || (echo "No /zebrad-cache/*/*"; ls -lhR "/zebrad-cache" | head -50 || echo "No /zebrad-cache directory")
             # TODO: use environmental variables instead of Rust features (part of #2995)
-            cargo test --locked --release --features "test_sync_past_mandatory_checkpoint_${NETWORK,,},lightwalletd-grpc-tests" --package zebrad --test acceptance -- --nocapture --include-ignored "sync_past_mandatory_checkpoint_${NETWORK,,}"
+            cargo test --locked --release --features "test_sync_past_mandatory_checkpoint_${NETWORK,,},$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored "sync_past_mandatory_checkpoint_${NETWORK,,}"
+
+        elif [[ "$GENERATE_CHECKPOINTS_MAINNET" -eq "1" ]]; then
+            # Generate checkpoints after syncing Zebra from a cached state on mainnet.
+            #
+            # TODO: disable or filter out logs like:
+            # test generate_checkpoints_mainnet has been running for over 60 seconds
+            #
+            # List directory used by test
+            ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
+            cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored generate_checkpoints_mainnet
+        elif [[ "$GENERATE_CHECKPOINTS_TESTNET" -eq "1" ]]; then
+            # Generate checkpoints after syncing Zebra on testnet.
+            #
+            # This test might fail if testnet is unstable.
+            #
+            # List directory used by test
+            ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
+            cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored generate_checkpoints_testnet
+
         elif [[ "$TEST_LWD_RPC_CALL" -eq "1" ]]; then
             # Starting at a cached Zebra tip, test a JSON-RPC call to Zebra.
             ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
-            cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored fully_synced_rpc_test
+            cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored fully_synced_rpc_test
         elif [[ "$TEST_LWD_FULL_SYNC" -eq "1" ]]; then
             # Starting at a cached Zebra tip, run a lightwalletd sync to tip.
             ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
-            cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_full_sync
+            cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_full_sync
             ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory")
         elif [[ "$TEST_LWD_UPDATE_SYNC" -eq "1" ]]; then
             # Starting with a cached Zebra and lightwalletd tip, run a quick update sync.
             ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
             ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory")
-            cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_update_sync
+            cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_update_sync
 
         # These tests actually use gRPC.
         elif [[ "$TEST_LWD_GRPC" -eq "1" ]]; then
             # Starting with a cached Zebra and lightwalletd tip, test all gRPC calls to lightwalletd, which calls Zebra.
             ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
             ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory")
-            cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_wallet_grpc_tests
+            cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_wallet_grpc_tests
         elif [[ "$TEST_LWD_TRANSACTIONS" -eq "1" ]]; then
             # Starting with a cached Zebra and lightwalletd tip, test sending transactions gRPC call to lightwalletd, which calls Zebra.
             ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
             ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory")
-            cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored sending_transactions_using_lightwalletd
+            cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored sending_transactions_using_lightwalletd
+
+        # These tests use mining code, but don't use gRPC.
+        # We add the mining feature here because our other code needs to pass tests without it.
         elif [[ "$TEST_GET_BLOCK_TEMPLATE" -eq "1" ]]; then
             # Starting with a cached Zebra tip, test getting a block template from Zebra's RPC server.
             ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
-            cargo test --locked --release --features getblocktemplate-rpcs --package zebrad --test acceptance -- --nocapture --include-ignored get_block_template
+            cargo test --locked --release --features "getblocktemplate-rpcs,$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored get_block_template
         elif [[ "$TEST_SUBMIT_BLOCK" -eq "1" ]]; then
             # Starting with a cached Zebra tip, test sending a block to Zebra's RPC port.
             ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
-            cargo test --locked --release --features getblocktemplate-rpcs --package zebrad --test acceptance -- --nocapture --include-ignored submit_block
+            cargo test --locked --release --features "getblocktemplate-rpcs,$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored submit_block
+
         else
             exec "$@"
         fi