From 3d8dc6c0e339c1775263feb05e57b675b0349818 Mon Sep 17 00:00:00 2001 From: rong fengliang <1141591465@qq.com> Date: Thu, 6 May 2021 13:45:34 +0800 Subject: [PATCH 1/6] add minio support --- rust/cubestore/src/config/mod.rs | 13 +++++++++++-- rust/cubestore/src/remotefs/s3.rs | 17 +++++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/rust/cubestore/src/config/mod.rs b/rust/cubestore/src/config/mod.rs index 66adcb45000dd..481ab84ba7c6b 100644 --- a/rust/cubestore/src/config/mod.rs +++ b/rust/cubestore/src/config/mod.rs @@ -38,6 +38,7 @@ use std::sync::Arc; use std::{env, fs}; use tokio::sync::broadcast; use tokio::time::{timeout_at, Duration, Instant}; +use std::env::VarError; #[derive(Clone)] pub struct CubeServices { @@ -158,7 +159,9 @@ pub enum FileStoreProvider { }, S3 { region: String, + endpoint:Option, bucket_name: String, + path_style:bool, sub_path: Option, }, GCS { @@ -408,8 +411,10 @@ impl Config { store_provider: { if let Ok(bucket_name) = env::var("CUBESTORE_S3_BUCKET") { FileStoreProvider::S3 { - bucket_name, + bucket_name:bucket_name, region: env::var("CUBESTORE_S3_REGION").unwrap(), + endpoint: env::var("CUBESTORE_S3_ENDPOINT").ok(), + path_style:env_bool("CURBSTONE_S3_PATH_STYLE",false), sub_path: env::var("CUBESTORE_S3_SUB_PATH").ok(), } } else if let Ok(bucket_name) = env::var("CUBESTORE_GCS_BUCKET") { @@ -643,17 +648,21 @@ impl Config { } FileStoreProvider::S3 { region, + endpoint, bucket_name, + path_style, sub_path, } => { let data_dir = self.config_obj.data_dir.clone(); let region = region.to_string(); let bucket_name = bucket_name.to_string(); + let endpoint = endpoint.clone(); + let path_style = path_style.to_owned(); let sub_path = sub_path.clone(); self.injector .register("original_remote_fs", async move |_| { let arc: Arc = - S3RemoteFs::new(data_dir, region, bucket_name, sub_path).unwrap(); + S3RemoteFs::new(data_dir, region,path_style,endpoint, bucket_name, sub_path).unwrap(); arc }) .await; diff --git a/rust/cubestore/src/remotefs/s3.rs b/rust/cubestore/src/remotefs/s3.rs index 6c479b47ba879..6ceb8b6b5848a 100644 --- a/rust/cubestore/src/remotefs/s3.rs +++ b/rust/cubestore/src/remotefs/s3.rs @@ -7,7 +7,7 @@ use chrono::{DateTime, Utc}; use log::{debug, info}; use regex::{NoExpand, Regex}; use s3::creds::Credentials; -use s3::Bucket; +use s3::{Bucket, Region}; use std::env; use std::io::Write; use std::path::{Path, PathBuf}; @@ -29,6 +29,8 @@ impl S3RemoteFs { pub fn new( dir: PathBuf, region: String, + path_style:bool, + endpoint: Option, bucket_name: String, sub_path: Option, ) -> Result, CubeError> { @@ -39,7 +41,18 @@ impl S3RemoteFs { None, None, )?; - let bucket = Bucket::new(&bucket_name, region.parse()?, credentials)?; + let bucket = if path_style==true { + Bucket::new_with_path_style( + &bucket_name, + Region::Custom { + endpoint:endpoint.unwrap(), + region, + }, + credentials + )? + } else{ + Bucket::new(&bucket_name, region.parse()?, credentials)? + }; Ok(Arc::new(Self { dir, bucket, From 06a08497cbd5b9e04851acb83083d9a8801a92e6 Mon Sep 17 00:00:00 2001 From: rong fengliang <1141591465@qq.com> Date: Thu, 6 May 2021 13:54:34 +0800 Subject: [PATCH 2/6] delete used code --- rust/cubestore/src/config/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/cubestore/src/config/mod.rs b/rust/cubestore/src/config/mod.rs index 481ab84ba7c6b..985c21595eee1 100644 --- a/rust/cubestore/src/config/mod.rs +++ b/rust/cubestore/src/config/mod.rs @@ -38,7 +38,6 @@ use std::sync::Arc; use std::{env, fs}; use tokio::sync::broadcast; use tokio::time::{timeout_at, Duration, Instant}; -use std::env::VarError; #[derive(Clone)] pub struct CubeServices { From b9ce572f8b9a708b45d8e309f83b1bb53aefd14a Mon Sep 17 00:00:00 2001 From: rong fengliang <1141591465@qq.com> Date: Thu, 6 May 2021 14:18:47 +0800 Subject: [PATCH 3/6] fmt code --- rust/cubestore/src/config/mod.rs | 19 +++++++++++++------ rust/cubestore/src/remotefs/s3.rs | 10 +++++----- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/rust/cubestore/src/config/mod.rs b/rust/cubestore/src/config/mod.rs index 985c21595eee1..b67bbc68cdb9c 100644 --- a/rust/cubestore/src/config/mod.rs +++ b/rust/cubestore/src/config/mod.rs @@ -158,9 +158,9 @@ pub enum FileStoreProvider { }, S3 { region: String, - endpoint:Option, + endpoint: Option, bucket_name: String, - path_style:bool, + path_style: bool, sub_path: Option, }, GCS { @@ -410,10 +410,10 @@ impl Config { store_provider: { if let Ok(bucket_name) = env::var("CUBESTORE_S3_BUCKET") { FileStoreProvider::S3 { - bucket_name:bucket_name, + bucket_name: bucket_name, region: env::var("CUBESTORE_S3_REGION").unwrap(), endpoint: env::var("CUBESTORE_S3_ENDPOINT").ok(), - path_style:env_bool("CURBSTONE_S3_PATH_STYLE",false), + path_style: env_bool("CURBSTONE_S3_PATH_STYLE", false), sub_path: env::var("CUBESTORE_S3_SUB_PATH").ok(), } } else if let Ok(bucket_name) = env::var("CUBESTORE_GCS_BUCKET") { @@ -660,8 +660,15 @@ impl Config { let sub_path = sub_path.clone(); self.injector .register("original_remote_fs", async move |_| { - let arc: Arc = - S3RemoteFs::new(data_dir, region,path_style,endpoint, bucket_name, sub_path).unwrap(); + let arc: Arc = S3RemoteFs::new( + data_dir, + region, + path_style, + endpoint, + bucket_name, + sub_path, + ) + .unwrap(); arc }) .await; diff --git a/rust/cubestore/src/remotefs/s3.rs b/rust/cubestore/src/remotefs/s3.rs index 6ceb8b6b5848a..f8977c4979359 100644 --- a/rust/cubestore/src/remotefs/s3.rs +++ b/rust/cubestore/src/remotefs/s3.rs @@ -29,7 +29,7 @@ impl S3RemoteFs { pub fn new( dir: PathBuf, region: String, - path_style:bool, + path_style: bool, endpoint: Option, bucket_name: String, sub_path: Option, @@ -41,16 +41,16 @@ impl S3RemoteFs { None, None, )?; - let bucket = if path_style==true { + let bucket = if path_style == true { Bucket::new_with_path_style( &bucket_name, Region::Custom { - endpoint:endpoint.unwrap(), + endpoint: endpoint.unwrap(), region, }, - credentials + credentials, )? - } else{ + } else { Bucket::new(&bucket_name, region.parse()?, credentials)? }; Ok(Arc::new(Self { From 77cfb822b24060eb32d7ebc163d983d7a055b0f1 Mon Sep 17 00:00:00 2001 From: rong fengliang <1141591465@qq.com> Date: Thu, 6 May 2021 15:25:57 +0800 Subject: [PATCH 4/6] fix test --- rust/cubestore/src/sql/mod.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rust/cubestore/src/sql/mod.rs b/rust/cubestore/src/sql/mod.rs index ccc19d083c69b..f2294c61155e7 100644 --- a/rust/cubestore/src/sql/mod.rs +++ b/rust/cubestore/src/sql/mod.rs @@ -1241,6 +1241,8 @@ mod tests { c.compaction_chunks_count_threshold = 100; c.store_provider = FileStoreProvider::S3 { region: "us-west-2".to_string(), + endpoint:None, + path_style:false, bucket_name: "cube-store-ci-test".to_string(), sub_path: Some("high_frequency_inserts_s3".to_string()), }; @@ -1256,6 +1258,8 @@ mod tests { c.server_name = "127.0.0.1:4306".to_string(); c.store_provider = FileStoreProvider::S3 { region: "us-west-2".to_string(), + endpoint:None, + path_style:false, bucket_name: "cube-store-ci-test".to_string(), sub_path: Some("high_frequency_inserts_s3".to_string()), }; @@ -1558,6 +1562,8 @@ mod tests { c.compaction_chunks_count_threshold = 100; c.store_provider = FileStoreProvider::S3 { region: "us-west-2".to_string(), + path_style:false, + endpoint:None, bucket_name: "cube-store-ci-test".to_string(), sub_path: Some("create_table_with_location_cluster".to_string()), }; @@ -1574,6 +1580,8 @@ mod tests { c.server_name = "127.0.0.1:24306".to_string(); c.store_provider = FileStoreProvider::S3 { region: "us-west-2".to_string(), + endpoint:None, + path_style:false, bucket_name: "cube-store-ci-test".to_string(), sub_path: Some("create_table_with_location_cluster".to_string()), }; From 090facd2c484ae0b639d146f9bcb978281592454 Mon Sep 17 00:00:00 2001 From: rong fengliang <1141591465@qq.com> Date: Thu, 6 May 2021 15:38:57 +0800 Subject: [PATCH 5/6] fmt code --- rust/cubestore/src/sql/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rust/cubestore/src/sql/mod.rs b/rust/cubestore/src/sql/mod.rs index f2294c61155e7..8f26fac74a44d 100644 --- a/rust/cubestore/src/sql/mod.rs +++ b/rust/cubestore/src/sql/mod.rs @@ -1241,8 +1241,8 @@ mod tests { c.compaction_chunks_count_threshold = 100; c.store_provider = FileStoreProvider::S3 { region: "us-west-2".to_string(), - endpoint:None, - path_style:false, + endpoint: None, + path_style: false, bucket_name: "cube-store-ci-test".to_string(), sub_path: Some("high_frequency_inserts_s3".to_string()), }; @@ -1258,8 +1258,8 @@ mod tests { c.server_name = "127.0.0.1:4306".to_string(); c.store_provider = FileStoreProvider::S3 { region: "us-west-2".to_string(), - endpoint:None, - path_style:false, + endpoint: None, + path_style: false, bucket_name: "cube-store-ci-test".to_string(), sub_path: Some("high_frequency_inserts_s3".to_string()), }; From 941d919a8d2f02b2cb56cdf1f59da42056fddf49 Mon Sep 17 00:00:00 2001 From: rong fengliang <1141591465@qq.com> Date: Thu, 6 May 2021 16:36:41 +0800 Subject: [PATCH 6/6] change env name --- rust/cubestore/src/config/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/cubestore/src/config/mod.rs b/rust/cubestore/src/config/mod.rs index b67bbc68cdb9c..0bd85fc4b5cdc 100644 --- a/rust/cubestore/src/config/mod.rs +++ b/rust/cubestore/src/config/mod.rs @@ -413,7 +413,7 @@ impl Config { bucket_name: bucket_name, region: env::var("CUBESTORE_S3_REGION").unwrap(), endpoint: env::var("CUBESTORE_S3_ENDPOINT").ok(), - path_style: env_bool("CURBSTONE_S3_PATH_STYLE", false), + path_style: env_bool("CUBESTORE_S3_PATH_STYLE", false), sub_path: env::var("CUBESTORE_S3_SUB_PATH").ok(), } } else if let Ok(bucket_name) = env::var("CUBESTORE_GCS_BUCKET") { @@ -656,7 +656,7 @@ impl Config { let region = region.to_string(); let bucket_name = bucket_name.to_string(); let endpoint = endpoint.clone(); - let path_style = path_style.to_owned(); + let path_style = path_style.clone(); let sub_path = sub_path.clone(); self.injector .register("original_remote_fs", async move |_| {