Merge branch 'main' into openapi-fix-descriptions

.github/workflows/cd-deploy-nodes-gcp.yml

@@ -136,6 +136,8 @@ jobs:
       image_name: zebrad
       no_cache: ${{ inputs.no_cache || false }}
       rust_log: info
+    # This step needs access to Docker Hub secrets to run successfully
+    secrets: inherit
 
   # Test that Zebra works using the default config with the latest Zebra version.
   test-configuration-file:

.github/workflows/ci-tests.yml

@@ -130,6 +130,8 @@ jobs:
       rust_backtrace: full
       rust_lib_backtrace: full
       rust_log: info
+    # This step needs access to Docker Hub secrets to run successfully
+    secrets: inherit
 
   # Runs Zebra unit tests
   unit-tests:

.github/workflows/sub-build-docker-image.yml

@@ -68,7 +68,7 @@ jobs:
   build:
     name: Build images
     timeout-minutes: 210
-    runs-on: ubuntu-latest-xl
+    runs-on: ubuntu-latest
     outputs:
       image_digest: ${{ steps.docker_build.outputs.digest }}
       image_name: ${{ fromJSON(steps.docker_build.outputs.metadata)['image.name'] }}
@@ -92,6 +92,8 @@ jobs:
         uses: docker/metadata-action@v5.5.1
         with:
           # list of Docker images to use as base name for tags
+          # We only publish images to DockerHub if a release is not a pre-release
+          # Ref: https://github.com/orgs/community/discussions/26281#discussioncomment-3251177
           images: |
             us-docker.pkg.dev/${{ vars.GCP_PROJECT }}/zebra/${{ inputs.image_name }}
             zfnd/${{ inputs.image_name }},enable=${{ github.event_name == 'release' && !github.event.release.prerelease }}
@@ -118,11 +120,6 @@ jobs:
             type=edge,enable={{is_default_branch}}
             type=schedule
 
-      # Setup Docker Buildx to allow use of docker cache layers from GH
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v3
-
       - name: Authenticate to Google Cloud
         id: auth
         uses: google-github-actions/auth@v2.1.3
@@ -144,14 +141,20 @@ jobs:
           password: ${{ steps.auth.outputs.access_token }}
 
       - name: Login to DockerHub
-        # We only publish images to DockerHub if a release is not a pre-release
-        # Ref: https://github.com/orgs/community/discussions/26281#discussioncomment-3251177
-        if: ${{ github.event_name == 'release' && !github.event.release.prerelease }}
         uses: docker/login-action@v3.2.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      # Setup Docker Buildx to use Docker Build Cloud
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: "lab:latest"
+          driver: cloud
+          endpoint: "zfnd/zebra"
+
       # Build and push image to Google Artifact Registry, and possibly DockerHub
       - name: Build & push
         id: docker_build
@@ -171,17 +174,3 @@ jobs:
           # Don't read from the cache if the caller disabled it.
           # https://docs.docker.com/engine/reference/commandline/buildx_build/#options
           no-cache: ${{ inputs.no_cache }}
-          # To improve build speeds, for each branch we push an additional image to the registry,
-          # to be used as the caching layer, using the `max` caching mode.
-          #
-          # We use multiple cache sources to confirm a cache hit, starting from a per-branch cache.
-          # If there's no hit, we continue with a `main` branch cache, which helps us avoid
-          # rebuilding cargo-chef, most dependencies, and possibly some Zebra crates.
-          #
-          # The caches are tried in top-down order, the first available cache is used:
-          # https://github.com/moby/moby/pull/26839#issuecomment-277383550
-          cache-from: |
-            type=registry,ref=us-docker.pkg.dev/${{ vars.GCP_PROJECT }}/zebra-caching/${{ inputs.image_name }}${{ inputs.tag_suffix }}:${{ env.GITHUB_REF_SLUG_URL }}-cache
-            type=registry,ref=us-docker.pkg.dev/${{ vars.GCP_PROJECT }}/zebra-caching/${{ inputs.image_name }}${{ inputs.tag_suffix }}:main-cache
-          cache-to: |
-            type=registry,ref=us-docker.pkg.dev/${{ vars.GCP_PROJECT }}/zebra-caching/${{ inputs.image_name }}${{ inputs.tag_suffix }}:${{ env.GITHUB_REF_SLUG_URL }}-cache,mode=max

CHANGELOG.md

@@ -9,8 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org).
 
 ### Changed
 
-- We realized that a longer than `zcashd` end of support could be problematic in some cases so we reverted back from 20 to 16 weeks ([#8530](https://github.com/ZcashFoundation/zebra/pull/8530))
-
+- We realized that a longer than `zcashd` end of support could be problematic in
+  some cases so we reverted back from 20 to 16 weeks
+  ([#8530](https://github.com/ZcashFoundation/zebra/pull/8530))
+- The `zebrad` binary no longer contains the scanner of shielded transactions.
+  This means `zebrad` no longer contains users' viewing keys.
+- We're no longer using general conditional compilation attributes for `tor`,
+  but only feature flags instead.
 - Fixed a bug with trailing characters in the openapi spec method descriptions((#8597)[https://github.com/ZcashFoundation/zebra/pull/8597])
 
 ## [Zebra 1.7.0](https://github.com/ZcashFoundation/zebra/releases/tag/v1.7.0) - 2024-05-07

tower-batch-control/Cargo.toml

@@ -47,3 +47,6 @@ tower-fallback = { path = "../tower-fallback/", version = "0.2.41-beta.13" }
 tower-test = "0.4.0"
 
 zebra-test = { path = "../zebra-test/", version = "1.0.0-beta.37" }
+
+[lints.rust]
+unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] }

zebra-network/src/isolated.rs

@@ -16,8 +16,7 @@ use crate::{
 
 // Wait until `arti-client`'s dependency `x25519-dalek v1.2.0` is updated to a higher version. (#5492)
 // #[cfg(feature = "tor")]
-#[cfg(tor)]
-pub(crate) mod tor;
+// pub(crate) mod tor;
 
 #[cfg(test)]
 mod tests;

zebra-network/src/lib.rs

@@ -165,13 +165,11 @@ mod protocol;
 
 // Wait until `arti-client`'s dependency `x25519-dalek v1.2.0` is updated to a higher version. (#5492)
 // #[cfg(feature = "tor")]
-#[cfg(tor)]
-pub use crate::isolated::tor::connect_isolated_tor;
+// pub use crate::isolated::tor::connect_isolated_tor;
 
 // Wait until `arti-client`'s dependency `x25519-dalek v1.2.0` is updated to a higher version. (#5492)
 // #[cfg(all(feature = "tor", any(test, feature = "proptest-impl")))]
-#[cfg(tor)]
-pub use crate::isolated::tor::connect_isolated_tor_with_inbound;
+// pub use crate::isolated::tor::connect_isolated_tor_with_inbound;
 
 #[cfg(any(test, feature = "proptest-impl"))]
 pub use crate::{

zebra-state/src/arbitrary.rs

@@ -50,27 +50,6 @@ where
     }
 }
 
-impl From<SemanticallyVerifiedBlock> for ChainTipBlock {
-    fn from(prepared: SemanticallyVerifiedBlock) -> Self {
-        let SemanticallyVerifiedBlock {
-            block,
-            hash,
-            height,
-            new_outputs: _,
-            transaction_hashes,
-        } = prepared;
-
-        Self {
-            hash,
-            height,
-            time: block.header.time,
-            transactions: block.transactions.clone(),
-            transaction_hashes,
-            previous_block_hash: block.header.previous_block_hash,
-        }
-    }
-}
-
 impl SemanticallyVerifiedBlock {
     /// Returns a [`ContextuallyVerifiedBlock`] created from this block,
     /// with fake zero-valued spent UTXOs.

zebra-state/src/lib.rs

@@ -46,8 +46,10 @@ pub use request::{
 };
 pub use response::{KnownBlock, MinedTx, ReadResponse, Response};
 pub use service::{
-    chain_tip::{ChainTipChange, LatestChainTip, TipAction},
-    check, init, spawn_init,
+    chain_tip::{ChainTipBlock, ChainTipChange, ChainTipSender, LatestChainTip, TipAction},
+    check, init, init_read_only,
+    non_finalized_state::NonFinalizedState,
+    spawn_init, spawn_init_read_only,
     watch_receiver::WatchReceiver,
     OutputIndex, OutputLocation, TransactionIndex, TransactionLocation,
 };
@@ -76,7 +78,6 @@ pub use response::GetBlockTemplateChainInfo;
 #[cfg(any(test, feature = "proptest-impl"))]
 pub use service::{
     arbitrary::{populated_state, CHAIN_TIP_UPDATE_WAIT_LIMIT},
-    chain_tip::{ChainTipBlock, ChainTipSender},
     finalized_state::{RawBytes, KV, MAX_ON_DISK_HEIGHT},
     init_test, init_test_services,
 };
@@ -96,4 +97,4 @@ pub(crate) use config::hidden::{
     write_database_format_version_to_disk, write_state_database_format_version_to_disk,
 };
 
-pub(crate) use request::ContextuallyVerifiedBlock;
+pub use request::ContextuallyVerifiedBlock;

zebra-state/src/service.rs

@@ -24,15 +24,6 @@ use std::{
     time::{Duration, Instant},
 };
 
-#[cfg(feature = "elasticsearch")]
-use elasticsearch::{
-    auth::Credentials::Basic,
-    cert::CertificateValidation,
-    http::transport::{SingleNodeConnectionPool, TransportBuilder},
-    http::Url,
-    Elasticsearch,
-};
-
 use futures::future::FutureExt;
 use tokio::sync::{oneshot, watch};
 use tower::{util::BoxService, Service, ServiceExt};
@@ -319,29 +310,12 @@ impl StateService {
         checkpoint_verify_concurrency_limit: usize,
     ) -> (Self, ReadStateService, LatestChainTip, ChainTipChange) {
         let timer = CodeTimer::start();
-
-        #[cfg(feature = "elasticsearch")]
-        let finalized_state = {
-            let conn_pool = SingleNodeConnectionPool::new(
-                Url::parse(config.elasticsearch_url.as_str())
-                    .expect("configured elasticsearch url is invalid"),
-            );
-            let transport = TransportBuilder::new(conn_pool)
-                .cert_validation(CertificateValidation::None)
-                .auth(Basic(
-                    config.clone().elasticsearch_username,
-                    config.clone().elasticsearch_password,
-                ))
-                .build()
-                .expect("elasticsearch transport builder should not fail");
-            let elastic_db = Some(Elasticsearch::new(transport));
-
-            FinalizedState::new(&config, network, elastic_db)
-        };
-
-        #[cfg(not(feature = "elasticsearch"))]
-        let finalized_state = { FinalizedState::new(&config, network) };
-
+        let finalized_state = FinalizedState::new(
+            &config,
+            network,
+            #[cfg(feature = "elasticsearch")]
+            true,
+        );
         timer.finish(module_path!(), line!(), "opening finalized state database");
 
         let timer = CodeTimer::start();
@@ -387,7 +361,7 @@ impl StateService {
 
         let read_service = ReadStateService::new(
             &finalized_state,
-            block_write_task,
+            Some(block_write_task),
             non_finalized_state_receiver,
         );
 
@@ -828,14 +802,14 @@ impl ReadStateService {
     /// and a watch channel for updating the shared recent non-finalized chain.
     pub(crate) fn new(
         finalized_state: &FinalizedState,
-        block_write_task: Arc<std::thread::JoinHandle<()>>,
+        block_write_task: Option<Arc<std::thread::JoinHandle<()>>>,
         non_finalized_state_receiver: watch::Receiver<NonFinalizedState>,
     ) -> Self {
         let read_service = Self {
             network: finalized_state.network(),
             db: finalized_state.db.clone(),
             non_finalized_state_receiver: WatchReceiver::new(non_finalized_state_receiver),
-            block_write_task: Some(block_write_task),
+            block_write_task,
         };
 
         tracing::debug!("created new read-only state service");
@@ -1945,6 +1919,52 @@ pub fn init(
     )
 }
 
+/// Initialize a read state service from the provided [`Config`].
+/// Returns a read-only state service,
+///
+/// Each `network` has its own separate on-disk database.
+///
+/// To share access to the state, clone the returned [`ReadStateService`].
+pub fn init_read_only(
+    config: Config,
+    network: &Network,
+) -> (
+    ReadStateService,
+    ZebraDb,
+    tokio::sync::watch::Sender<NonFinalizedState>,
+) {
+    let finalized_state = FinalizedState::new_with_debug(
+        &config,
+        network,
+        true,
+        #[cfg(feature = "elasticsearch")]
+        false,
+        true,
+    );
+    let (non_finalized_state_sender, non_finalized_state_receiver) =
+        tokio::sync::watch::channel(NonFinalizedState::new(network));
+
+    (
+        ReadStateService::new(&finalized_state, None, non_finalized_state_receiver),
+        finalized_state.db.clone(),
+        non_finalized_state_sender,
+    )
+}
+
+/// Calls [`init_read_only`] with the provided [`Config`] and [`Network`] from a blocking task.
+/// Returns a [`tokio::task::JoinHandle`] with a read state service and chain tip sender.
+pub fn spawn_init_read_only(
+    config: Config,
+    network: &Network,
+) -> tokio::task::JoinHandle<(
+    ReadStateService,
+    ZebraDb,
+    tokio::sync::watch::Sender<NonFinalizedState>,
+)> {
+    let network = network.clone();
+    tokio::task::spawn_blocking(move || init_read_only(config, &network))
+}
+
 /// Calls [`init`] with the provided [`Config`] and [`Network`] from a blocking task.
 /// Returns a [`tokio::task::JoinHandle`] with a boxed state service,
 /// a read state service, and receivers for state chain tip updates.

zebra-state/src/service/chain_tip.rs

@@ -107,15 +107,15 @@ impl From<ContextuallyVerifiedBlock> for ChainTipBlock {
     }
 }
 
-impl From<CheckpointVerifiedBlock> for ChainTipBlock {
-    fn from(finalized: CheckpointVerifiedBlock) -> Self {
-        let CheckpointVerifiedBlock(SemanticallyVerifiedBlock {
+impl From<SemanticallyVerifiedBlock> for ChainTipBlock {
+    fn from(prepared: SemanticallyVerifiedBlock) -> Self {
+        let SemanticallyVerifiedBlock {
             block,
             hash,
             height,
+            new_outputs: _,
             transaction_hashes,
-            ..
-        }) = finalized;
+        } = prepared;
 
         Self {
             hash,
@@ -128,6 +128,12 @@ impl From<CheckpointVerifiedBlock> for ChainTipBlock {
     }
 }
 
+impl From<CheckpointVerifiedBlock> for ChainTipBlock {
+    fn from(CheckpointVerifiedBlock(prepared): CheckpointVerifiedBlock) -> Self {
+        prepared.into()
+    }
+}
+
 /// A sender for changes to the non-finalized and finalized chain tips.
 #[derive(Debug)]
 pub struct ChainTipSender {