diff --git a/rust/cubestore/src/config/mod.rs b/rust/cubestore/src/config/mod.rs index 66adcb45000dd..0bd85fc4b5cdc 100644 --- a/rust/cubestore/src/config/mod.rs +++ b/rust/cubestore/src/config/mod.rs @@ -158,7 +158,9 @@ pub enum FileStoreProvider { }, S3 { region: String, + endpoint: Option, bucket_name: String, + path_style: bool, sub_path: Option, }, GCS { @@ -408,8 +410,10 @@ impl Config { store_provider: { if let Ok(bucket_name) = env::var("CUBESTORE_S3_BUCKET") { FileStoreProvider::S3 { - bucket_name, + bucket_name: bucket_name, region: env::var("CUBESTORE_S3_REGION").unwrap(), + endpoint: env::var("CUBESTORE_S3_ENDPOINT").ok(), + path_style: env_bool("CUBESTORE_S3_PATH_STYLE", false), sub_path: env::var("CUBESTORE_S3_SUB_PATH").ok(), } } else if let Ok(bucket_name) = env::var("CUBESTORE_GCS_BUCKET") { @@ -643,17 +647,28 @@ impl Config { } FileStoreProvider::S3 { region, + endpoint, bucket_name, + path_style, sub_path, } => { let data_dir = self.config_obj.data_dir.clone(); let region = region.to_string(); let bucket_name = bucket_name.to_string(); + let endpoint = endpoint.clone(); + let path_style = path_style.clone(); let sub_path = sub_path.clone(); self.injector .register("original_remote_fs", async move |_| { - let arc: Arc = - S3RemoteFs::new(data_dir, region, bucket_name, sub_path).unwrap(); + let arc: Arc = S3RemoteFs::new( + data_dir, + region, + path_style, + endpoint, + bucket_name, + sub_path, + ) + .unwrap(); arc }) .await; diff --git a/rust/cubestore/src/remotefs/s3.rs b/rust/cubestore/src/remotefs/s3.rs index 6c479b47ba879..f8977c4979359 100644 --- a/rust/cubestore/src/remotefs/s3.rs +++ b/rust/cubestore/src/remotefs/s3.rs @@ -7,7 +7,7 @@ use chrono::{DateTime, Utc}; use log::{debug, info}; use regex::{NoExpand, Regex}; use s3::creds::Credentials; -use s3::Bucket; +use s3::{Bucket, Region}; use std::env; use std::io::Write; use std::path::{Path, PathBuf}; @@ -29,6 +29,8 @@ impl S3RemoteFs { pub fn new( dir: PathBuf, region: String, + path_style: bool, + endpoint: Option, bucket_name: String, sub_path: Option, ) -> Result, CubeError> { @@ -39,7 +41,18 @@ impl S3RemoteFs { None, None, )?; - let bucket = Bucket::new(&bucket_name, region.parse()?, credentials)?; + let bucket = if path_style == true { + Bucket::new_with_path_style( + &bucket_name, + Region::Custom { + endpoint: endpoint.unwrap(), + region, + }, + credentials, + )? + } else { + Bucket::new(&bucket_name, region.parse()?, credentials)? + }; Ok(Arc::new(Self { dir, bucket, diff --git a/rust/cubestore/src/sql/mod.rs b/rust/cubestore/src/sql/mod.rs index ccc19d083c69b..8f26fac74a44d 100644 --- a/rust/cubestore/src/sql/mod.rs +++ b/rust/cubestore/src/sql/mod.rs @@ -1241,6 +1241,8 @@ mod tests { c.compaction_chunks_count_threshold = 100; c.store_provider = FileStoreProvider::S3 { region: "us-west-2".to_string(), + endpoint: None, + path_style: false, bucket_name: "cube-store-ci-test".to_string(), sub_path: Some("high_frequency_inserts_s3".to_string()), }; @@ -1256,6 +1258,8 @@ mod tests { c.server_name = "127.0.0.1:4306".to_string(); c.store_provider = FileStoreProvider::S3 { region: "us-west-2".to_string(), + endpoint: None, + path_style: false, bucket_name: "cube-store-ci-test".to_string(), sub_path: Some("high_frequency_inserts_s3".to_string()), }; @@ -1558,6 +1562,8 @@ mod tests { c.compaction_chunks_count_threshold = 100; c.store_provider = FileStoreProvider::S3 { region: "us-west-2".to_string(), + path_style:false, + endpoint:None, bucket_name: "cube-store-ci-test".to_string(), sub_path: Some("create_table_with_location_cluster".to_string()), }; @@ -1574,6 +1580,8 @@ mod tests { c.server_name = "127.0.0.1:24306".to_string(); c.store_provider = FileStoreProvider::S3 { region: "us-west-2".to_string(), + endpoint:None, + path_style:false, bucket_name: "cube-store-ci-test".to_string(), sub_path: Some("create_table_with_location_cluster".to_string()), };