Skip to content

Commit

Permalink
feat: added unauthenticated version of gcs object store (#916)
Browse files Browse the repository at this point in the history
Adding unauthenticated version of GCS object store.
It's needed for EN to be able to download snapshots from our GCS
  • Loading branch information
tomg10 committed Jan 24, 2024
1 parent f833b6c commit 638a813
Show file tree
Hide file tree
Showing 3 changed files with 42 additions and 17 deletions.
1 change: 1 addition & 0 deletions core/lib/config/src/configs/object_store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ pub enum ObjectStoreMode {
GCS,
GCSWithCredentialFile,
FileBacked,
GCSAnonymousReadOnly,
}

/// Configuration for the object store
Expand Down
35 changes: 21 additions & 14 deletions core/lib/object_store/src/gcs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -63,17 +63,22 @@ impl fmt::Debug for GoogleCloudStorage {
}
}

#[derive(Debug, Clone)]
pub enum GoogleCloudStorageAuthMode {
AuthenticatedWithCredentialFile(String),
Authenticated,
Anonymous,
}

impl GoogleCloudStorage {
pub async fn new(
credential_file_path: Option<String>,
auth_mode: GoogleCloudStorageAuthMode,
bucket_prefix: String,
max_retries: u16,
) -> Self {
let client_config = retry(max_retries, || {
Self::get_client_config(credential_file_path.clone())
})
.await
.expect("failed fetching GCS client config after retries");
let client_config = retry(max_retries, || Self::get_client_config(auth_mode.clone()))
.await
.expect("failed fetching GCS client config after retries");

Self {
client: Client::new(client_config),
Expand All @@ -83,15 +88,17 @@ impl GoogleCloudStorage {
}

async fn get_client_config(
credential_file_path: Option<String>,
auth_mode: GoogleCloudStorageAuthMode,
) -> Result<ClientConfig, Error> {
if let Some(path) = credential_file_path {
let cred_file = CredentialsFile::new_from_file(path)
.await
.expect("failed loading GCS credential file");
ClientConfig::default().with_credentials(cred_file).await
} else {
ClientConfig::default().with_auth().await
match auth_mode {
GoogleCloudStorageAuthMode::AuthenticatedWithCredentialFile(path) => {
let cred_file = CredentialsFile::new_from_file(path)
.await
.expect("failed loading GCS credential file");
ClientConfig::default().with_credentials(cred_file).await
}
GoogleCloudStorageAuthMode::Authenticated => ClientConfig::default().with_auth().await,
GoogleCloudStorageAuthMode::Anonymous => Ok(ClientConfig::default().anonymous()),
}
}

Expand Down
23 changes: 20 additions & 3 deletions core/lib/object_store/src/raw.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,11 @@ use std::{error, fmt, sync::Arc};
use async_trait::async_trait;
use zksync_config::configs::object_store::{ObjectStoreConfig, ObjectStoreMode};

use crate::{file::FileBackedObjectStore, gcs::GoogleCloudStorage, mock::MockStore};
use crate::{
file::FileBackedObjectStore,
gcs::{GoogleCloudStorage, GoogleCloudStorageAuthMode},
mock::MockStore,
};

/// Bucket for [`ObjectStore`] in which objects can be placed.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
Expand Down Expand Up @@ -196,7 +200,7 @@ impl ObjectStoreFactory {
"Initialized GoogleCloudStorage Object store without credential file"
);
let store = GoogleCloudStorage::new(
gcs_credential_file_path,
GoogleCloudStorageAuthMode::Authenticated,
config.bucket_base_url.clone(),
config.max_retries,
)
Expand All @@ -206,7 +210,10 @@ impl ObjectStoreFactory {
ObjectStoreMode::GCSWithCredentialFile => {
tracing::trace!("Initialized GoogleCloudStorage Object store with credential file");
let store = GoogleCloudStorage::new(
gcs_credential_file_path,
GoogleCloudStorageAuthMode::AuthenticatedWithCredentialFile(
gcs_credential_file_path
.expect("Credentials path must be provided for GCSWithCredentialFile"),
),
config.bucket_base_url.clone(),
config.max_retries,
)
Expand All @@ -218,6 +225,16 @@ impl ObjectStoreFactory {
let store = FileBackedObjectStore::new(config.file_backed_base_path.clone()).await;
Arc::new(store)
}
ObjectStoreMode::GCSAnonymousReadOnly => {
tracing::trace!("Initialized GoogleCloudStoragePublicReadOnly store");
let store = GoogleCloudStorage::new(
GoogleCloudStorageAuthMode::Anonymous,
config.bucket_base_url.clone(),
config.max_retries,
)
.await;
Arc::new(store)
}
}
}
}

0 comments on commit 638a813

Please sign in to comment.