diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml index 3ee759378fc..e1e1f871797 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/build-docker-image.yml @@ -37,12 +37,15 @@ on: required: false type: string default: info + # keep these in sync with: + # https://github.com/ZcashFoundation/zebra/blob/main/docker/Dockerfile#L83 features: required: false + default: "sentry" type: string test_features: required: false - default: "lightwalletd-grpc-tests" + default: "lightwalletd-grpc-tests zebra-checkpoints" type: string rpc_port: required: false diff --git a/.github/workflows/continous-integration-docker.patch.yml b/.github/workflows/continous-integration-docker.patch.yml index ae7f8f2a894..bb92b0592d0 100644 --- a/.github/workflows/continous-integration-docker.patch.yml +++ b/.github/workflows/continous-integration-docker.patch.yml @@ -90,6 +90,12 @@ jobs: steps: - run: 'echo "No build required"' + generate-checkpoints-mainnet: + name: Generate checkpoints mainnet / Run generate-checkpoints-mainnet test + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + lightwalletd-rpc-test: name: Zebra tip JSON-RPC / Run fully-synced-rpc test runs-on: ubuntu-latest diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index 8989c0817a5..47a21c4f00d 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -461,6 +461,38 @@ jobs: height_grep_text: 'current_height.*=.*Height.*\(' secrets: inherit + # zebra checkpoint generation tests + + # Test that Zebra can generate mainnet checkpoints after syncing to the chain tip, + # using a cached Zebra tip state, + # + # Runs: + # - after every PR is merged to `main` + # - on every PR update + # + # If the state version has changed, waits for the new cached state to be created. + # Otherwise, if the state rebuild was skipped, runs immediately after the build job. + generate-checkpoints-mainnet: + name: Generate checkpoints mainnet + needs: test-full-sync + uses: ./.github/workflows/deploy-gcp-tests.yml + if: ${{ !cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + with: + app_name: zebrad + test_id: generate-checkpoints-mainnet + test_description: Generate Zebra checkpoints on mainnet + test_variables: '-e GENERATE_CHECKPOINTS_MAINNET=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache' + needs_zebra_state: true + # update the disk on every PR, to increase CI speed + saves_to_disk: true + disk_suffix: tip + root_state_path: '/var/cache' + zebra_state_dir: 'zebrad-cache' + height_grep_text: 'current_height.*=.*Height.*\(' + secrets: inherit + + # TODO: testnet checkpoints, test-full-sync, test-update-sync, get-available-disks (+ reusable) + # lightwalletd cached tip state tests # Test full sync of lightwalletd with a Zebra tip state diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 35a777e0229..d307fd1ee51 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -46,7 +46,7 @@ jobs: tag_suffix: .experimental network: Testnet rpc_port: '18232' - features: "getblocktemplate-rpcs" + features: "sentry getblocktemplate-rpcs" test_features: "" checkpoint_sync: true rust_backtrace: '1' diff --git a/Cargo.lock b/Cargo.lock index 434faa3e080..0fb0140efb0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6254,6 +6254,7 @@ dependencies = [ "zebra-rpc", "zebra-state", "zebra-test", + "zebra-utils", ] [[package]] diff --git a/docker/Dockerfile b/docker/Dockerfile index caa9a53860c..180a50e52b8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -80,8 +80,13 @@ ARG CHECKPOINT_SYNC ENV CHECKPOINT_SYNC ${CHECKPOINT_SYNC:-true} # Build zebrad with these features -ARG FEATURES -ARG TEST_FEATURES="lightwalletd-grpc-tests" +# Keep these in sync with: +# https://github.com/ZcashFoundation/zebra/blob/main/.github/workflows/build-docker-image.yml#L42 +ARG FEATURES="sentry" +ARG TEST_FEATURES="lightwalletd-grpc-tests zebra-checkpoints" +# Use ENTRYPOINT_FEATURES to override the specific features used to run tests in entrypoint.sh, +# separately from the test and production image builds. +ENV ENTRYPOINT_FEATURES "$TEST_FEATURES $FEATURES" ARG NETWORK ENV NETWORK ${NETWORK:-Mainnet} @@ -104,11 +109,12 @@ COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/lightwalletd /opt/lightwalletd # This is the caching Docker layer for Rust! # # TODO: is it faster to use --tests here? -RUN cargo chef cook --release --features "sentry ${TEST_FEATURES} ${FEATURES}" --workspace --recipe-path recipe.json +RUN cargo chef cook --release --features "${TEST_FEATURES} ${FEATURES}" --workspace --recipe-path recipe.json COPY . . RUN cargo test --locked --release --features "${TEST_FEATURES} ${FEATURES}" --workspace --no-run RUN cp /opt/zebrad/target/release/zebrad /usr/local/bin +RUN cp /opt/zebrad/target/release/zebra-checkpoints /usr/local/bin COPY ./docker/entrypoint.sh / RUN chmod u+x /entrypoint.sh @@ -122,11 +128,11 @@ ENTRYPOINT [ "/entrypoint.sh" ] # `test` stage. This step is a dependency for the `runtime` stage, which uses the resulting # zebrad binary from this step. FROM deps AS release -RUN cargo chef cook --release --features "sentry ${FEATURES}" --recipe-path recipe.json +RUN cargo chef cook --release --features "${FEATURES}" --recipe-path recipe.json COPY . . # Build zebra -RUN cargo build --locked --release --features "sentry ${FEATURES}" --package zebrad --bin zebrad +RUN cargo build --locked --release --features "${FEATURES}" --package zebrad --bin zebrad # This stage is only used when deploying nodes or when only the resulting zebrad binary is needed # diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index e17bfbcc163..75044351919 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -13,24 +13,25 @@ echo "ZEBRA_TEST_LIGHTWALLETD=$ZEBRA_TEST_LIGHTWALLETD" echo "Hard-coded Zebra full sync directory: /zebrad-cache" echo "ZEBRA_CACHED_STATE_DIR=$ZEBRA_CACHED_STATE_DIR" echo "LIGHTWALLETD_DATA_DIR=$LIGHTWALLETD_DATA_DIR" +echo "ENTRYPOINT_FEATURES=$ENTRYPOINT_FEATURES" case "$1" in --* | -*) exec zebrad "$@" ;; *) - # For these tests, we activate the gRPC feature to avoid recompiling `zebrad`, - # but we might not actually run any gRPC tests. + # For these tests, we activate the test features to avoid recompiling `zebrad`, + # but we don't actually run any gRPC tests. if [[ "$RUN_ALL_TESTS" -eq "1" ]]; then # Run all the available tests for the current environment. # If the lightwalletd environmental variables are set, we will also run those tests. - cargo test --locked --release --features lightwalletd-grpc-tests --workspace -- --nocapture --include-ignored + cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --workspace -- --nocapture --include-ignored # For these tests, we activate the gRPC feature to avoid recompiling `zebrad`, # but we don't actually run any gRPC tests. elif [[ "$TEST_FULL_SYNC" -eq "1" ]]; then # Run a Zebra full sync test. - cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored full_sync_mainnet + cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored full_sync_mainnet # List directory generated by test # TODO: replace with $ZEBRA_CACHED_STATE_DIR in Rust and workflows ls -lh "/zebrad-cache"/*/* || (echo "No /zebrad-cache/*/*"; ls -lhR "/zebrad-cache" | head -50 || echo "No /zebrad-cache directory") @@ -38,7 +39,7 @@ case "$1" in # Run a Zebra sync up to the mandatory checkpoint. # # TODO: use environmental variables instead of Rust features (part of #2995) - cargo test --locked --release --features "test_sync_to_mandatory_checkpoint_${NETWORK,,},lightwalletd-grpc-tests" --package zebrad --test acceptance -- --nocapture --include-ignored "sync_to_mandatory_checkpoint_${NETWORK,,}" + cargo test --locked --release --features "test_sync_to_mandatory_checkpoint_${NETWORK,,},$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored "sync_to_mandatory_checkpoint_${NETWORK,,}" # TODO: replace with $ZEBRA_CACHED_STATE_DIR in Rust and workflows ls -lh "/zebrad-cache"/*/* || (echo "No /zebrad-cache/*/*"; ls -lhR "/zebrad-cache" | head -50 || echo "No /zebrad-cache directory") elif [[ "$TEST_UPDATE_SYNC" -eq "1" ]]; then @@ -46,7 +47,7 @@ case "$1" in # # List directory used by test ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") - cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored zebrad_update_sync + cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored zebrad_update_sync elif [[ "$TEST_CHECKPOINT_SYNC" -eq "1" ]]; then # Run a Zebra sync starting at the cached mandatory checkpoint, and syncing past it. # @@ -54,41 +55,64 @@ case "$1" in # TODO: replace with $ZEBRA_CACHED_STATE_DIR in Rust and workflows ls -lh "/zebrad-cache"/*/* || (echo "No /zebrad-cache/*/*"; ls -lhR "/zebrad-cache" | head -50 || echo "No /zebrad-cache directory") # TODO: use environmental variables instead of Rust features (part of #2995) - cargo test --locked --release --features "test_sync_past_mandatory_checkpoint_${NETWORK,,},lightwalletd-grpc-tests" --package zebrad --test acceptance -- --nocapture --include-ignored "sync_past_mandatory_checkpoint_${NETWORK,,}" + cargo test --locked --release --features "test_sync_past_mandatory_checkpoint_${NETWORK,,},$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored "sync_past_mandatory_checkpoint_${NETWORK,,}" + + elif [[ "$GENERATE_CHECKPOINTS_MAINNET" -eq "1" ]]; then + # Generate checkpoints after syncing Zebra from a cached state on mainnet. + # + # TODO: disable or filter out logs like: + # test generate_checkpoints_mainnet has been running for over 60 seconds + # + # List directory used by test + ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") + cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored generate_checkpoints_mainnet + elif [[ "$GENERATE_CHECKPOINTS_TESTNET" -eq "1" ]]; then + # Generate checkpoints after syncing Zebra on testnet. + # + # This test might fail if testnet is unstable. + # + # List directory used by test + ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") + cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored generate_checkpoints_testnet + elif [[ "$TEST_LWD_RPC_CALL" -eq "1" ]]; then # Starting at a cached Zebra tip, test a JSON-RPC call to Zebra. ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") - cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored fully_synced_rpc_test + cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored fully_synced_rpc_test elif [[ "$TEST_LWD_FULL_SYNC" -eq "1" ]]; then # Starting at a cached Zebra tip, run a lightwalletd sync to tip. ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") - cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_full_sync + cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_full_sync ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory") elif [[ "$TEST_LWD_UPDATE_SYNC" -eq "1" ]]; then # Starting with a cached Zebra and lightwalletd tip, run a quick update sync. ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory") - cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_update_sync + cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_update_sync # These tests actually use gRPC. elif [[ "$TEST_LWD_GRPC" -eq "1" ]]; then # Starting with a cached Zebra and lightwalletd tip, test all gRPC calls to lightwalletd, which calls Zebra. ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory") - cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_wallet_grpc_tests + cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_wallet_grpc_tests elif [[ "$TEST_LWD_TRANSACTIONS" -eq "1" ]]; then # Starting with a cached Zebra and lightwalletd tip, test sending transactions gRPC call to lightwalletd, which calls Zebra. ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory") - cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored sending_transactions_using_lightwalletd + cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored sending_transactions_using_lightwalletd + + # These tests use mining code, but don't use gRPC. + # We add the mining feature here because our other code needs to pass tests without it. elif [[ "$TEST_GET_BLOCK_TEMPLATE" -eq "1" ]]; then # Starting with a cached Zebra tip, test getting a block template from Zebra's RPC server. ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") - cargo test --locked --release --features getblocktemplate-rpcs --package zebrad --test acceptance -- --nocapture --include-ignored get_block_template + cargo test --locked --release --features "getblocktemplate-rpcs,$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored get_block_template elif [[ "$TEST_SUBMIT_BLOCK" -eq "1" ]]; then # Starting with a cached Zebra tip, test sending a block to Zebra's RPC port. ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") - cargo test --locked --release --features getblocktemplate-rpcs --package zebrad --test acceptance -- --nocapture --include-ignored submit_block + cargo test --locked --release --features "getblocktemplate-rpcs,$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored submit_block + else exec "$@" fi diff --git a/zebra-test/src/command.rs b/zebra-test/src/command.rs index 9549dff08af..dae47defba3 100644 --- a/zebra-test/src/command.rs +++ b/zebra-test/src/command.rs @@ -52,8 +52,8 @@ pub trait CommandExt { fn output2(&mut self) -> Result, Report>; /// wrapper for `spawn` fn on `Command` that constructs informative error - /// reports - fn spawn2(&mut self, dir: T) -> Result, Report>; + /// reports using the original `command_path` + fn spawn2(&mut self, dir: T, command_path: impl ToString) -> Result, Report>; } impl CommandExt for Command { @@ -89,18 +89,19 @@ impl CommandExt for Command { } /// wrapper for `spawn` fn on `Command` that constructs informative error - /// reports - fn spawn2(&mut self, dir: T) -> Result, Report> { - let cmd = format!("{self:?}"); + /// reports using the original `command_path` + fn spawn2(&mut self, dir: T, command_path: impl ToString) -> Result, Report> { + let command_and_args = format!("{self:?}"); let child = self.spawn(); let child = child .wrap_err("failed to execute process") - .with_section(|| cmd.clone().header("Command:"))?; + .with_section(|| command_and_args.clone().header("Command:"))?; Ok(TestChild { dir: Some(dir), - cmd, + cmd: command_and_args, + command_path: command_path.to_string(), child: Some(child), stdout: None, stderr: None, @@ -133,14 +134,18 @@ where Self: AsRef + Sized, { #[allow(clippy::unwrap_in_result)] - fn spawn_child_with_command(self, cmd: &str, args: Arguments) -> Result> { - let mut cmd = test_cmd(cmd, self.as_ref())?; + fn spawn_child_with_command( + self, + command_path: &str, + args: Arguments, + ) -> Result> { + let mut cmd = test_cmd(command_path, self.as_ref())?; Ok(cmd .args(args.into_arguments()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) - .spawn2(self) + .spawn2(self, command_path) .unwrap()) } } @@ -183,9 +188,12 @@ pub struct TestChild { /// and its output has been taken. pub dir: Option, - /// The original command string. + /// The full command string, including arguments and working directory. pub cmd: String, + /// The path of the command, as passed to spawn2(). + pub command_path: String, + /// The child process itself. /// /// `None` when the command has been waited on, @@ -533,7 +541,8 @@ impl TestChild { // // This checks for failure logs, and prevents some test hangs and deadlocks. if self.child.is_some() || self.stdout.is_some() { - let wrote_lines = self.wait_for_stdout_line("\nChild Stdout:".to_string()); + let wrote_lines = + self.wait_for_stdout_line(format!("\n{} Child Stdout:", self.command_path)); while self.wait_for_stdout_line(None) {} @@ -544,7 +553,8 @@ impl TestChild { } if self.child.is_some() || self.stderr.is_some() { - let wrote_lines = self.wait_for_stderr_line("\nChild Stderr:".to_string()); + let wrote_lines = + self.wait_for_stderr_line(format!("\n{} Child Stderr:", self.command_path)); while self.wait_for_stderr_line(None) {} @@ -559,7 +569,7 @@ impl TestChild { /// Waits until a line of standard output is available, then consumes it. /// /// If there is a line, and `write_context` is `Some`, writes the context to the test logs. - /// Then writes the line to the test logs. + /// Always writes the line to the test logs. /// /// Returns `true` if a line was available, /// or `false` if the standard output has finished. @@ -592,7 +602,7 @@ impl TestChild { /// Waits until a line of standard error is available, then consumes it. /// /// If there is a line, and `write_context` is `Some`, writes the context to the test logs. - /// Then writes the line to the test logs. + /// Always writes the line to the test logs. /// /// Returns `true` if a line was available, /// or `false` if the standard error has finished. @@ -686,21 +696,21 @@ impl TestChild { self } - /// Configures testrunner to forward stdout and stderr to the true stdout, + /// Configures this test runner to forward stdout and stderr to the true stdout, /// rather than the fakestdout used by cargo tests. pub fn bypass_test_capture(mut self, cond: bool) -> Self { self.bypass_test_capture = cond; self } - /// Checks each line of the child's stdout against `success_regex`, and returns Ok - /// if a line matches. + /// Checks each line of the child's stdout against `success_regex`, + /// and returns the first matching line. Prints all stdout lines. /// /// Kills the child on error, or after the configured timeout has elapsed. /// See [`Self::expect_line_matching_regex_set`] for details. #[instrument(skip(self))] #[allow(clippy::unwrap_in_result)] - pub fn expect_stdout_line_matches(&mut self, success_regex: R) -> Result<&mut Self> + pub fn expect_stdout_line_matches(&mut self, success_regex: R) -> Result where R: ToRegex + Debug, { @@ -711,11 +721,11 @@ impl TestChild { .take() .expect("child must capture stdout to call expect_stdout_line_matches, and it can't be called again after an error"); - match self.expect_line_matching_regex_set(&mut lines, success_regex, "stdout") { - Ok(()) => { + match self.expect_line_matching_regex_set(&mut lines, success_regex, "stdout", true) { + Ok(line) => { // Replace the log lines for the next check self.stdout = Some(lines); - Ok(self) + Ok(line) } Err(report) => { // Read all the log lines for error context @@ -725,14 +735,14 @@ impl TestChild { } } - /// Checks each line of the child's stderr against `success_regex`, and returns Ok - /// if a line matches. + /// Checks each line of the child's stderr against `success_regex`, + /// and returns the first matching line. Prints all stderr lines to stdout. /// /// Kills the child on error, or after the configured timeout has elapsed. /// See [`Self::expect_line_matching_regex_set`] for details. #[instrument(skip(self))] #[allow(clippy::unwrap_in_result)] - pub fn expect_stderr_line_matches(&mut self, success_regex: R) -> Result<&mut Self> + pub fn expect_stderr_line_matches(&mut self, success_regex: R) -> Result where R: ToRegex + Debug, { @@ -743,11 +753,75 @@ impl TestChild { .take() .expect("child must capture stderr to call expect_stderr_line_matches, and it can't be called again after an error"); - match self.expect_line_matching_regex_set(&mut lines, success_regex, "stderr") { - Ok(()) => { + match self.expect_line_matching_regex_set(&mut lines, success_regex, "stderr", true) { + Ok(line) => { // Replace the log lines for the next check self.stderr = Some(lines); - Ok(self) + Ok(line) + } + Err(report) => { + // Read all the log lines for error context + self.stderr = Some(lines); + Err(report).context_from(self) + } + } + } + + /// Checks each line of the child's stdout against `success_regex`, + /// and returns the first matching line. Does not print any output. + /// + /// Kills the child on error, or after the configured timeout has elapsed. + /// See [`Self::expect_line_matching_regex_set`] for details. + #[instrument(skip(self))] + #[allow(clippy::unwrap_in_result)] + pub fn expect_stdout_line_matches_silent(&mut self, success_regex: R) -> Result + where + R: ToRegex + Debug, + { + self.apply_failure_regexes_to_outputs(); + + let mut lines = self + .stdout + .take() + .expect("child must capture stdout to call expect_stdout_line_matches, and it can't be called again after an error"); + + match self.expect_line_matching_regex_set(&mut lines, success_regex, "stdout", false) { + Ok(line) => { + // Replace the log lines for the next check + self.stdout = Some(lines); + Ok(line) + } + Err(report) => { + // Read all the log lines for error context + self.stdout = Some(lines); + Err(report).context_from(self) + } + } + } + + /// Checks each line of the child's stderr against `success_regex`, + /// and returns the first matching line. Does not print any output. + /// + /// Kills the child on error, or after the configured timeout has elapsed. + /// See [`Self::expect_line_matching_regex_set`] for details. + #[instrument(skip(self))] + #[allow(clippy::unwrap_in_result)] + pub fn expect_stderr_line_matches_silent(&mut self, success_regex: R) -> Result + where + R: ToRegex + Debug, + { + self.apply_failure_regexes_to_outputs(); + + let mut lines = self + .stderr + .take() + .expect("child must capture stderr to call expect_stderr_line_matches, and it can't be called again after an error"); + + match self.expect_line_matching_regex_set(&mut lines, success_regex, "stderr", false) { + Ok(line) => { + // Replace the log lines for the next check + self.stderr = Some(lines); + Ok(line) } Err(report) => { // Read all the log lines for error context @@ -767,7 +841,8 @@ impl TestChild { lines: &mut L, success_regexes: R, stream_name: &str, - ) -> Result<()> + write_to_logs: bool, + ) -> Result where L: Iterator>, R: ToRegexSet, @@ -776,7 +851,7 @@ impl TestChild { .to_regex_set() .expect("regexes must be valid"); - self.expect_line_matching_regexes(lines, success_regexes, stream_name) + self.expect_line_matching_regexes(lines, success_regexes, stream_name, write_to_logs) } /// Checks each line in `lines` against a regex set, and returns Ok if a line matches. @@ -788,7 +863,8 @@ impl TestChild { lines: &mut L, success_regexes: I, stream_name: &str, - ) -> Result<()> + write_to_logs: bool, + ) -> Result where L: Iterator>, I: CollectRegexSet, @@ -797,7 +873,7 @@ impl TestChild { .collect_regex_set() .expect("regexes must be valid"); - self.expect_line_matching_regexes(lines, success_regexes, stream_name) + self.expect_line_matching_regexes(lines, success_regexes, stream_name, write_to_logs) } /// Checks each line in `lines` against `success_regexes`, and returns Ok if a line @@ -814,7 +890,8 @@ impl TestChild { lines: &mut L, success_regexes: RegexSet, stream_name: &str, - ) -> Result<()> + write_to_logs: bool, + ) -> Result where L: Iterator>, { @@ -831,11 +908,13 @@ impl TestChild { break; }; - // Since we're about to discard this line write it to stdout. - Self::write_to_test_logs(&line, self.bypass_test_capture); + if write_to_logs { + // Since we're about to discard this line write it to stdout. + Self::write_to_test_logs(&line, self.bypass_test_capture); + } if success_regexes.is_match(&line) { - return Ok(()); + return Ok(line); } } @@ -1295,8 +1374,8 @@ impl ContextFrom<&mut TestChild> for Report { } } - self.section(stdout_buf.header("Unread Stdout:")) - .section(stderr_buf.header("Unread Stderr:")) + self.section(stdout_buf.header(format!("{} Unread Stdout:", source.command_path))) + .section(stderr_buf.header(format!("{} Unread Stderr:", source.command_path))) } } @@ -1313,6 +1392,7 @@ impl ContextFrom<&Output> for Report { type Return = Report; fn context_from(self, source: &Output) -> Self::Return { + // TODO: add TestChild.command_path before Stdout and Stderr header names let stdout = || { String::from_utf8_lossy(&source.stdout) .into_owned() diff --git a/zebra-utils/src/bin/zebra-checkpoints/main.rs b/zebra-utils/src/bin/zebra-checkpoints/main.rs index 9dcd84e71b2..6481df8b7e5 100644 --- a/zebra-utils/src/bin/zebra-checkpoints/main.rs +++ b/zebra-utils/src/bin/zebra-checkpoints/main.rs @@ -137,14 +137,19 @@ where /// Process entry point for `zebra-checkpoints` #[tokio::main] -#[allow(clippy::print_stdout)] +#[allow(clippy::print_stdout, clippy::print_stderr)] async fn main() -> Result<()> { + eprintln!("zebra-checkpoints launched"); + // initialise init_tracing(); color_eyre::install()?; let args = args::Args::from_args(); + eprintln!("Command-line arguments: {args:?}"); + eprintln!("Fetching block info and calculating checkpoints...\n\n"); + // get the current block count let get_block_chain_info = rpc_output(&args, "getblockchaininfo", None) .await diff --git a/zebra-utils/tests/build_utils_for_zebrad_tests.rs b/zebra-utils/tests/build_utils_for_zebrad_tests.rs new file mode 100644 index 00000000000..5edcd3061d5 --- /dev/null +++ b/zebra-utils/tests/build_utils_for_zebrad_tests.rs @@ -0,0 +1,17 @@ +//! # Dependency Workaround +//! +//! This empty integration test makes `cargo` build the `zebra-checkpoints` binary for the `zebrad` +//! integration tests: +//! +//! > Binary targets are automatically built if there is an integration test or benchmark being +//! > selected to test. +//! +//! +//! +//! Each utility binary will only be built if its corresponding Rust feature is activated. +//! +//! +//! # Unstable `cargo` Feature +//! +//! When `cargo -Z bindeps` is stabilised, add a binary dependency to `zebrad/Cargo.toml` instead: +//! https://github.com/rust-lang/cargo/issues/9096 diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index e469600e96b..3f11c94b6e7 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -80,6 +80,11 @@ proptest-impl = [ "zebra-chain/proptest-impl", ] +# Build the zebra-checkpoints utility for checkpoint generation tests +zebra-checkpoints = [ + "zebra-utils/zebra-checkpoints", +] + # The gRPC tests also need an installed lightwalletd binary lightwalletd-grpc-tests = ["tonic-build"] @@ -219,3 +224,14 @@ zebra-state = { path = "../zebra-state", features = ["proptest-impl"] } zebra-node-services = { path = "../zebra-node-services", features = ["rpc-client"] } zebra-test = { path = "../zebra-test" } + +# Used by the checkpoint generation tests via the zebra-checkpoints feature +# (the binaries in this crate won't be built unless their features are enabled). +# +# Currently, we use zebra-utils/tests/build_utils_for_zebrad_tests.rs as a workaround +# to build the zebra-checkpoints utility for the zebrad acceptance tests. +# +# When `-Z bindeps` is stabilised, enable this binary dependency instead: +# https://github.com/rust-lang/cargo/issues/9096 +# zebra-utils { path = "../zebra-utils", artifact = "bin:zebra-checkpoints" } +zebra-utils = { path = "../zebra-utils" } diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index c6bb51d6883..5db592146ef 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -75,28 +75,28 @@ //! $ cargo test lightwalletd_integration -- --nocapture //! //! $ export ZEBRA_TEST_LIGHTWALLETD=true -//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/chain" +//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state" //! $ export LIGHTWALLETD_DATA_DIR="/path/to/lightwalletd/database" //! $ cargo test lightwalletd_update_sync -- --nocapture //! //! $ export ZEBRA_TEST_LIGHTWALLETD=true -//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/chain" +//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state" //! $ cargo test lightwalletd_full_sync -- --ignored --nocapture //! //! $ export ZEBRA_TEST_LIGHTWALLETD=true //! $ cargo test lightwalletd_test_suite -- --ignored --nocapture //! //! $ export ZEBRA_TEST_LIGHTWALLETD=true -//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/chain" +//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state" //! $ cargo test fully_synced_rpc_test -- --ignored --nocapture //! //! $ export ZEBRA_TEST_LIGHTWALLETD=true -//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/chain" +//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state" //! $ export LIGHTWALLETD_DATA_DIR="/path/to/lightwalletd/database" //! $ cargo test sending_transactions_using_lightwalletd --features lightwalletd-grpc-tests -- --ignored --nocapture //! //! $ export ZEBRA_TEST_LIGHTWALLETD=true -//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/chain" +//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state" //! $ export LIGHTWALLETD_DATA_DIR="/path/to/lightwalletd/database" //! $ cargo test lightwalletd_wallet_grpc_tests --features lightwalletd-grpc-tests -- --ignored --nocapture //! ``` @@ -106,17 +106,25 @@ //! Example of how to run the get_block_template test: //! //! ```console -//! ZEBRA_CACHED_STATE_DIR=/path/to/zebra/chain cargo test get_block_template --features getblocktemplate-rpcs --release -- --ignored --nocapture +//! ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state cargo test get_block_template --features getblocktemplate-rpcs --release -- --ignored --nocapture //! ``` //! //! Example of how to run the submit_block test: //! //! ```console -//! ZEBRA_CACHED_STATE_DIR=/path/to/zebra/chain cargo test submit_block --features getblocktemplate-rpcs --release -- --ignored --nocapture +//! ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state cargo test submit_block --features getblocktemplate-rpcs --release -- --ignored --nocapture //! ``` //! //! Please refer to the documentation of each test for more information. //! +//! ## Checkpoint Generation Tests +//! +//! Generate checkpoints on mainnet and testnet using a cached state: +//! ```console +//! GENERATE_CHECKPOINTS_MAINNET=1 ENTRYPOINT_FEATURES=zebra-checkpoints ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state docker/entrypoint.sh +//! GENERATE_CHECKPOINTS_TESTNET=1 ENTRYPOINT_FEATURES=zebra-checkpoints ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state docker/entrypoint.sh +//! ``` +//! //! ## Disk Space for Testing //! //! The full sync and lightwalletd tests with cached state expect a temporary directory with @@ -2205,3 +2213,26 @@ async fn get_block_template() -> Result<()> { async fn submit_block() -> Result<()> { common::get_block_template_rpcs::submit_block::run().await } + +/// Test `zebra-checkpoints` on mainnet. +/// +/// If you want to run this test individually, see the module documentation. +/// See [`common::checkpoints`] for more information. +#[tokio::test] +#[ignore] +#[cfg(feature = "zebra-checkpoints")] +async fn generate_checkpoints_mainnet() -> Result<()> { + common::checkpoints::run(Mainnet).await +} + +/// Test `zebra-checkpoints` on testnet. +/// This test might fail if testnet is unstable. +/// +/// If you want to run this test individually, see the module documentation. +/// See [`common::checkpoints`] for more information. +#[tokio::test] +#[ignore] +#[cfg(feature = "zebra-checkpoints")] +async fn generate_checkpoints_testnet() -> Result<()> { + common::checkpoints::run(Testnet).await +} diff --git a/zebrad/tests/common/checkpoints.rs b/zebrad/tests/common/checkpoints.rs new file mode 100644 index 00000000000..b083d2126ca --- /dev/null +++ b/zebrad/tests/common/checkpoints.rs @@ -0,0 +1,489 @@ +//! Test generating checkpoints using `zebra-checkpoints` directly connected to `zebrad`. +//! +//! This test requires a cached chain state that is synchronized close to the network chain tip +//! height. It will finish the sync and update the cached chain state. + +use std::{ + env, fs, + net::SocketAddr, + path::{Path, PathBuf}, + sync::atomic::{AtomicBool, Ordering}, +}; + +use color_eyre::eyre::Result; +use tempfile::TempDir; + +use zebra_chain::{ + block::{Height, HeightDiff, TryIntoHeight}, + parameters::Network, + transparent::MIN_TRANSPARENT_COINBASE_MATURITY, +}; +use zebra_consensus::MAX_CHECKPOINT_HEIGHT_GAP; +use zebra_node_services::rpc_client::RpcRequestClient; +use zebra_test::{ + args, + command::{Arguments, TestDirExt, NO_MATCHES_REGEX_ITER}, + prelude::TestChild, +}; + +use crate::common::{ + launch::spawn_zebrad_for_rpc, + sync::{CHECKPOINT_VERIFIER_REGEX, SYNC_FINISHED_REGEX}, + test_type::TestType::*, +}; + +use super::{ + config::testdir, + failure_messages::{ + PROCESS_FAILURE_MESSAGES, ZEBRA_CHECKPOINTS_FAILURE_MESSAGES, ZEBRA_FAILURE_MESSAGES, + }, + launch::ZebradTestDirExt, + test_type::TestType, +}; + +/// The environmental variable used to activate zebrad logs in the checkpoint generation test. +/// +/// We use a constant so the compiler detects typos. +pub const LOG_ZEBRAD_CHECKPOINTS: &str = "LOG_ZEBRAD_CHECKPOINTS"; + +/// The test entry point. +#[allow(clippy::print_stdout)] +pub async fn run(network: Network) -> Result<()> { + let _init_guard = zebra_test::init(); + + // We want a Zebra state dir, but we don't need `lightwalletd`. + let test_type = UpdateZebraCachedStateWithRpc; + let test_name = "zebra_checkpoints_test"; + + // Skip the test unless the user supplied the correct cached state env vars + let Some(zebrad_state_path) = test_type.zebrad_state_path(test_name) else { + return Ok(()); + }; + + tracing::info!( + ?network, + ?test_type, + ?zebrad_state_path, + "running zebra_checkpoints test, spawning zebrad...", + ); + + // Sync zebrad to the network chain tip + let (mut zebrad, zebra_rpc_address) = if let Some(zebrad_and_address) = + spawn_zebrad_for_rpc(network, test_name, test_type, true)? + { + zebrad_and_address + } else { + // Skip the test, we don't have the required cached state + return Ok(()); + }; + + let zebra_rpc_address = zebra_rpc_address.expect("zebra_checkpoints test must have RPC port"); + + tracing::info!( + ?network, + ?zebra_rpc_address, + "spawned zebrad, waiting for it to load compiled-in checkpoints...", + ); + + let last_checkpoint = zebrad.expect_stdout_line_matches(CHECKPOINT_VERIFIER_REGEX)?; + // TODO: do this with a regex? + let (_prefix, last_checkpoint) = last_checkpoint + .split_once("max_checkpoint_height") + .expect("just checked log format"); + let (_prefix, last_checkpoint) = last_checkpoint + .split_once('(') + .expect("unexpected log format"); + let (last_checkpoint, _suffix) = last_checkpoint + .split_once(')') + .expect("unexpected log format"); + + tracing::info!( + ?network, + ?zebra_rpc_address, + ?last_checkpoint, + "found zebrad's current last checkpoint", + ); + + tracing::info!( + ?network, + ?zebra_rpc_address, + "waiting for zebrad to open its RPC port...", + ); + zebrad.expect_stdout_line_matches(&format!("Opened RPC endpoint at {zebra_rpc_address}"))?; + + tracing::info!( + ?network, + ?zebra_rpc_address, + "zebrad opened its RPC port, waiting for it to sync...", + ); + + zebrad.expect_stdout_line_matches(SYNC_FINISHED_REGEX)?; + + let zebra_tip_height = zebrad_tip_height(zebra_rpc_address).await?; + tracing::info!( + ?network, + ?zebra_rpc_address, + ?zebra_tip_height, + ?last_checkpoint, + "zebrad synced to the tip, launching zebra-checkpoints...", + ); + + let zebra_checkpoints = + spawn_zebra_checkpoints_direct(network, test_type, zebra_rpc_address, last_checkpoint)?; + + let show_zebrad_logs = env::var(LOG_ZEBRAD_CHECKPOINTS).is_ok(); + if !show_zebrad_logs { + tracing::info!( + "zebrad logs are hidden, show them using {LOG_ZEBRAD_CHECKPOINTS}=1 and RUST_LOG=debug" + ); + } + + tracing::info!( + ?network, + ?zebra_rpc_address, + ?zebra_tip_height, + ?last_checkpoint, + "spawned zebra-checkpoints connected to zebrad, checkpoints should appear here...", + ); + println!("\n\n"); + + let (_zebra_checkpoints, _zebrad) = wait_for_zebra_checkpoints_generation( + zebra_checkpoints, + zebrad, + zebra_tip_height, + test_type, + show_zebrad_logs, + )?; + + println!("\n\n"); + tracing::info!( + ?network, + ?zebra_tip_height, + ?last_checkpoint, + "finished generating Zebra checkpoints", + ); + + Ok(()) +} + +/// Spawns a `zebra-checkpoints` instance on `network`, connected to `zebrad_rpc_address`. +/// +/// Returns: +/// - `Ok(zebra_checkpoints)` on success, +/// - `Err(_)` if spawning `zebra-checkpoints` fails. +#[tracing::instrument] +pub fn spawn_zebra_checkpoints_direct( + network: Network, + test_type: TestType, + zebrad_rpc_address: SocketAddr, + last_checkpoint: &str, +) -> Result> { + let zebrad_rpc_address = zebrad_rpc_address.to_string(); + + let arguments = args![ + "--addr": zebrad_rpc_address, + "--last-checkpoint": last_checkpoint, + ]; + + // TODO: add logs for different kinds of zebra_checkpoints failures + let zebra_checkpoints_failure_messages = PROCESS_FAILURE_MESSAGES + .iter() + .chain(ZEBRA_FAILURE_MESSAGES) + .chain(ZEBRA_CHECKPOINTS_FAILURE_MESSAGES) + .cloned(); + let zebra_checkpoints_ignore_messages = NO_MATCHES_REGEX_ITER.iter().cloned(); + + // Currently unused, but we might put a copy of the checkpoints file in it later + let zebra_checkpoints_dir = testdir()?; + + let mut zebra_checkpoints = zebra_checkpoints_dir + .spawn_zebra_checkpoints_child(arguments)? + .with_timeout(test_type.zebrad_timeout()) + .with_failure_regex_iter( + zebra_checkpoints_failure_messages, + zebra_checkpoints_ignore_messages, + ); + + // zebra-checkpoints logs to stderr when it launches. + // + // This log happens very quickly, so it is ok to block for a short while here. + zebra_checkpoints.expect_stderr_line_matches(regex::escape("calculating checkpoints"))?; + + Ok(zebra_checkpoints) +} + +/// Extension trait for methods on `tempfile::TempDir` for using it as a test +/// directory for `zebra-checkpoints`. +pub trait ZebraCheckpointsTestDirExt: ZebradTestDirExt +where + Self: AsRef + Sized, +{ + /// Spawn `zebra-checkpoints` with `extra_args`, as a child process in this test directory, + /// potentially taking ownership of the tempdir for the duration of the child process. + /// + /// By default, launch an instance that connects directly to `zebrad`. + fn spawn_zebra_checkpoints_child(self, extra_args: Arguments) -> Result>; +} + +impl ZebraCheckpointsTestDirExt for TempDir { + #[allow(clippy::unwrap_in_result)] + fn spawn_zebra_checkpoints_child(mut self, extra_args: Arguments) -> Result> { + // By default, launch an instance that connects directly to `zebrad`. + let mut args = Arguments::new(); + args.set_parameter("--transport", "direct"); + + // Apply user provided arguments + args.merge_with(extra_args); + + // Create debugging info + let temp_dir = self.as_ref().display().to_string(); + + // Try searching the system $PATH first, that's what the test Docker image uses + let zebra_checkpoints_path = "zebra-checkpoints"; + + // Make sure we have the right zebra-checkpoints binary. + // + // When we were creating this test, we spent a lot of time debugging a build issue where + // `zebra-checkpoints` had an empty `main()` function. This check makes sure that doesn't + // happen again. + let debug_checkpoints = env::var(LOG_ZEBRAD_CHECKPOINTS).is_ok(); + if debug_checkpoints { + let mut args = Arguments::new(); + args.set_argument("--help"); + + let help_dir = testdir()?; + + tracing::info!( + ?zebra_checkpoints_path, + ?args, + ?help_dir, + system_path = ?env::var("PATH"), + // TODO: disable when the tests are working well + usr_local_zebra_checkpoints_info = ?fs::metadata("/usr/local/bin/zebra-checkpoints"), + "Trying to launch `zebra-checkpoints --help` by searching system $PATH...", + ); + + let zebra_checkpoints = help_dir.spawn_child_with_command(zebra_checkpoints_path, args); + + if let Err(help_error) = zebra_checkpoints { + tracing::info!(?help_error, "Failed to launch `zebra-checkpoints --help`"); + } else { + tracing::info!("Launched `zebra-checkpoints --help`, output is:"); + + let mut zebra_checkpoints = zebra_checkpoints.unwrap(); + let mut output_is_empty = true; + + // Get the help output + while zebra_checkpoints.wait_for_stdout_line(None) { + output_is_empty = false; + } + while zebra_checkpoints.wait_for_stderr_line(None) { + output_is_empty = false; + } + + if output_is_empty { + tracing::info!( + "`zebra-checkpoints --help` did not log any output. \ + Is the binary being built during tests? Are its required-features active?" + ); + } + } + } + + // Try the `zebra-checkpoints` binary the Docker image copied just after it built the tests. + tracing::info!( + ?zebra_checkpoints_path, + ?args, + ?temp_dir, + system_path = ?env::var("PATH"), + // TODO: disable when the tests are working well + usr_local_zebra_checkpoints_info = ?fs::metadata("/usr/local/bin/zebra-checkpoints"), + "Trying to launch zebra-checkpoints by searching system $PATH...", + ); + + let zebra_checkpoints = self.spawn_child_with_command(zebra_checkpoints_path, args.clone()); + + let Err(system_path_error) = zebra_checkpoints else { + return zebra_checkpoints; + }; + + // Fall back to assuming zebra-checkpoints is in the same directory as zebrad. + let mut zebra_checkpoints_path: PathBuf = env!("CARGO_BIN_EXE_zebrad").into(); + assert!( + zebra_checkpoints_path.pop(), + "must have at least one path component", + ); + zebra_checkpoints_path.push("zebra-checkpoints"); + + if zebra_checkpoints_path.exists() { + // Create a new temporary directory, because the old one has been used up. + // + // TODO: instead, return the TempDir from spawn_child_with_command() on error. + self = testdir()?; + + // Create debugging info + let temp_dir = self.as_ref().display().to_string(); + + tracing::info!( + ?zebra_checkpoints_path, + ?args, + ?temp_dir, + ?system_path_error, + // TODO: disable when the tests are working well + zebra_checkpoints_info = ?fs::metadata(&zebra_checkpoints_path), + "Launching from system $PATH failed, \ + trying to launch zebra-checkpoints from cargo path...", + ); + + self.spawn_child_with_command( + zebra_checkpoints_path.to_str().expect( + "internal test harness error: path is not UTF-8 \ + TODO: change spawn child methods to take &OsStr not &str", + ), + args, + ) + } else { + tracing::info!( + cargo_path = ?zebra_checkpoints_path, + ?system_path_error, + // TODO: disable when the tests are working well + cargo_path_info = ?fs::metadata(&zebra_checkpoints_path), + "Launching from system $PATH failed, \ + and zebra-checkpoints cargo path does not exist...", + ); + + // Return the original error + Err(system_path_error) + } + } +} + +/// Wait for `zebra-checkpoints` to generate checkpoints, clearing Zebra's logs at the same time. +#[tracing::instrument] +pub fn wait_for_zebra_checkpoints_generation< + P: ZebradTestDirExt + std::fmt::Debug + std::marker::Send + 'static, +>( + mut zebra_checkpoints: TestChild, + mut zebrad: TestChild

, + zebra_tip_height: Height, + test_type: TestType, + show_zebrad_logs: bool, +) -> Result<(TestChild, TestChild

)> { + let last_checkpoint_gap = HeightDiff::try_from(MIN_TRANSPARENT_COINBASE_MATURITY) + .expect("constant fits in HeightDiff") + + HeightDiff::try_from(MAX_CHECKPOINT_HEIGHT_GAP).expect("constant fits in HeightDiff"); + let expected_final_checkpoint_height = + (zebra_tip_height - last_checkpoint_gap).expect("network tip is high enough"); + + let is_zebra_checkpoints_finished = AtomicBool::new(false); + let is_zebra_checkpoints_finished = &is_zebra_checkpoints_finished; + + // Check Zebra's logs for errors. + // + // Checkpoint generation can take a long time, so we need to check `zebrad` for errors + // in parallel. + let zebrad_mut = &mut zebrad; + let zebrad_wait_fn = || -> Result<_> { + tracing::debug!( + ?test_type, + "zebrad is waiting for zebra-checkpoints to generate checkpoints...", + ); + while !is_zebra_checkpoints_finished.load(Ordering::SeqCst) { + // Just keep silently checking the Zebra logs for errors, + // so the checkpoint list can be copied from the output. + // + // Make sure the sync is still finished, this is logged every minute or so. + if env::var(LOG_ZEBRAD_CHECKPOINTS).is_ok() { + zebrad_mut.expect_stdout_line_matches(SYNC_FINISHED_REGEX)?; + } else { + zebrad_mut.expect_stdout_line_matches_silent(SYNC_FINISHED_REGEX)?; + } + } + + Ok(zebrad_mut) + }; + + // Wait until `zebra-checkpoints` has generated a full set of checkpoints. + // Also checks `zebra-checkpoints` logs for errors. + // + // Checkpoints generation can take a long time, so we need to run it in parallel with `zebrad`. + let zebra_checkpoints_mut = &mut zebra_checkpoints; + let zebra_checkpoints_wait_fn = || -> Result<_> { + tracing::debug!( + ?test_type, + "waiting for zebra_checkpoints to generate checkpoints...", + ); + + // zebra-checkpoints does not log anything when it finishes, it just prints checkpoints. + // + // We know that checkpoints are always less than 1000 blocks apart, but they can happen + // anywhere in that range due to block sizes. So we ignore the last 3 digits of the height. + let expected_final_checkpoint_prefix = expected_final_checkpoint_height.0 / 1000; + + // Mainnet and testnet checkpoints always have at least one leading zero in their hash. + let expected_final_checkpoint = + format!("{expected_final_checkpoint_prefix}[0-9][0-9][0-9] 0"); + zebra_checkpoints_mut.expect_stdout_line_matches(&expected_final_checkpoint)?; + + // Write the rest of the checkpoints: there can be 0-2 more checkpoints. + while zebra_checkpoints_mut.wait_for_stdout_line(None) {} + + // Tell the other thread that `zebra_checkpoints` has finished + is_zebra_checkpoints_finished.store(true, Ordering::SeqCst); + + Ok(zebra_checkpoints_mut) + }; + + // Run both threads in parallel, automatically propagating any panics to this thread. + std::thread::scope(|s| { + // Launch the sync-waiting threads + let zebrad_thread = s.spawn(|| { + zebrad_wait_fn().expect("test failed while waiting for zebrad to sync"); + }); + + let zebra_checkpoints_thread = s.spawn(|| { + let zebra_checkpoints_result = zebra_checkpoints_wait_fn(); + + is_zebra_checkpoints_finished.store(true, Ordering::SeqCst); + + zebra_checkpoints_result + .expect("test failed while waiting for zebra_checkpoints to sync."); + }); + + // Mark the sync-waiting threads as finished if they fail or panic. + // This tells the other thread that it can exit. + // + // TODO: use `panic::catch_unwind()` instead, + // when `&mut zebra_test::command::TestChild` is unwind-safe + s.spawn(|| { + let zebrad_result = zebrad_thread.join(); + zebrad_result.expect("test panicked or failed while waiting for zebrad to sync"); + }); + s.spawn(|| { + let zebra_checkpoints_result = zebra_checkpoints_thread.join(); + is_zebra_checkpoints_finished.store(true, Ordering::SeqCst); + + zebra_checkpoints_result + .expect("test panicked or failed while waiting for zebra_checkpoints to sync"); + }); + }); + + Ok((zebra_checkpoints, zebrad)) +} + +/// Returns an approximate `zebrad` tip height, using JSON-RPC. +#[tracing::instrument] +pub async fn zebrad_tip_height(zebra_rpc_address: SocketAddr) -> Result { + let client = RpcRequestClient::new(zebra_rpc_address); + + let zebrad_blockchain_info = client + .text_from_call("getblockchaininfo", "[]".to_string()) + .await?; + let zebrad_blockchain_info: serde_json::Value = serde_json::from_str(&zebrad_blockchain_info)?; + + let zebrad_tip_height = zebrad_blockchain_info["result"]["blocks"] + .try_into_height() + .expect("unexpected block height: invalid Height value"); + + Ok(zebrad_tip_height) +} diff --git a/zebrad/tests/common/failure_messages.rs b/zebrad/tests/common/failure_messages.rs index 8ff3a84eb36..1b6ff6f986b 100644 --- a/zebrad/tests/common/failure_messages.rs +++ b/zebrad/tests/common/failure_messages.rs @@ -118,3 +118,26 @@ pub const LIGHTWALLETD_EMPTY_ZEBRA_STATE_IGNORE_MESSAGES: &[&str] = &[ // but we expect Zebra to start with an empty state. r#"No Chain tip available yet","level":"warning","msg":"error with getblockchaininfo rpc, retrying"#, ]; + +/// Failure log messages from `zebra-checkpoints`. +/// +/// These `zebra-checkpoints` messages show that checkpoint generation has failed. +/// So when we see them in the logs, we make the test fail. +#[cfg(feature = "zebra-checkpoints")] +pub const ZEBRA_CHECKPOINTS_FAILURE_MESSAGES: &[&str] = &[ + // Rust-specific panics + "The application panicked", + // RPC port errors + "Unable to start RPC server", + // RPC argument errors: parsing and data + // + // These logs are produced by jsonrpc_core inside Zebra, + // but it doesn't log them yet. + // + // TODO: log these errors in Zebra, and check for them in the Zebra logs? + "Invalid params", + "Method not found", + // Incorrect command-line arguments + "USAGE", + "Invalid value", +]; diff --git a/zebrad/tests/common/lightwalletd.rs b/zebrad/tests/common/lightwalletd.rs index a9f2a196915..62282ed52c4 100644 --- a/zebrad/tests/common/lightwalletd.rs +++ b/zebrad/tests/common/lightwalletd.rs @@ -148,7 +148,7 @@ pub fn can_spawn_lightwalletd_for_rpc + std::fmt::Debug>( } /// Extension trait for methods on `tempfile::TempDir` for using it as a test -/// directory for `zebrad`. +/// directory for `lightwalletd`. pub trait LightWalletdTestDirExt: ZebradTestDirExt where Self: AsRef + Sized, diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index ac6a9d6e7e6..1db9d1211d4 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -52,11 +52,6 @@ fn max_sent_transactions() -> usize { const MAX_NUM_FUTURE_BLOCKS: u32 = 50; /// The test entry point. -// -// TODO: -// - check output of zebrad and lightwalletd in different threads, -// to avoid test hangs due to full output pipes -// (see lightwalletd_integration_test for an example) pub async fn run() -> Result<()> { let _init_guard = zebra_test::init(); diff --git a/zebrad/tests/common/lightwalletd/sync.rs b/zebrad/tests/common/lightwalletd/sync.rs index 096f481252a..a44a76f78af 100644 --- a/zebrad/tests/common/lightwalletd/sync.rs +++ b/zebrad/tests/common/lightwalletd/sync.rs @@ -132,11 +132,17 @@ pub fn wait_for_zebrad_and_lightwalletd_sync< std::thread::scope(|s| { // Launch the sync-waiting threads let zebrad_thread = s.spawn(|| { - zebrad_wait_fn().expect("test failed while waiting for zebrad to sync"); + let zebrad_result = zebrad_wait_fn(); + is_zebrad_finished.store(true, Ordering::SeqCst); + + zebrad_result.expect("test failed while waiting for zebrad to sync"); }); let lightwalletd_thread = s.spawn(|| { - lightwalletd_wait_fn().expect("test failed while waiting for lightwalletd to sync."); + let lightwalletd_result = lightwalletd_wait_fn(); + is_lightwalletd_finished.store(true, Ordering::SeqCst); + + lightwalletd_result.expect("test failed while waiting for lightwalletd to sync."); }); // Mark the sync-waiting threads as finished if they fail or panic. diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs index 08b9f23c3d0..c583863215c 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs @@ -59,11 +59,6 @@ use crate::common::{ }; /// The test entry point. -// -// TODO: -// - check output of zebrad and lightwalletd in different threads, -// to avoid test hangs due to full output pipes -// (see lightwalletd_integration_test for an example) pub async fn run() -> Result<()> { let _init_guard = zebra_test::init(); diff --git a/zebrad/tests/common/mod.rs b/zebrad/tests/common/mod.rs index 0aac0afaafb..77627d0275e 100644 --- a/zebrad/tests/common/mod.rs +++ b/zebrad/tests/common/mod.rs @@ -18,5 +18,8 @@ pub mod lightwalletd; pub mod sync; pub mod test_type; +#[cfg(feature = "zebra-checkpoints")] +pub mod checkpoints; + #[cfg(feature = "getblocktemplate-rpcs")] pub mod get_block_template_rpcs; diff --git a/zebrad/tests/common/sync.rs b/zebrad/tests/common/sync.rs index 3d5920619b1..2da33067f2c 100644 --- a/zebrad/tests/common/sync.rs +++ b/zebrad/tests/common/sync.rs @@ -46,11 +46,14 @@ pub const SYNC_FINISHED_REGEX: &str = r"finished initial sync to chain tip, using gossiped blocks .*sync_percent.*=.*100\."; /// The text that should be logged every time Zebra checks the sync progress. -// -// This is only used with `--feature lightwalletd-grpc-tests` -#[allow(dead_code)] +#[cfg(feature = "lightwalletd-grpc-tests")] pub const SYNC_PROGRESS_REGEX: &str = r"sync_percent"; +/// The text that should be logged when Zebra loads its compiled-in checkpoints. +#[cfg(feature = "zebra-checkpoints")] +pub const CHECKPOINT_VERIFIER_REGEX: &str = + r"initializing chain verifier.*max_checkpoint_height.*=.*Height"; + /// The maximum amount of time Zebra should take to reload after shutting down. /// /// This should only take a second, but sometimes CI VMs or RocksDB can be slow.