Skip to content

[opt](test) Support IAM Role for s3 client ut test #62303

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wyxxxcat wants to merge 1 commit into
base: master
Choose a base branch
from
+90 −14
Open

[opt](test) Support IAM Role for s3 client ut test #62303

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 45 additions & 7 deletions be/test/io/client/s3_file_system_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,15 @@ namespace doris {

static const char* MOCK_S3_FS_ID = "mock_s3_fs_id_6";

static std::string get_env_with_fallback(const char* primary, const char* fallback) {
const char* value = std::getenv(primary);
if (value != nullptr) {
return value;
}
value = std::getenv(fallback);
return value != nullptr ? value : "";
}

class S3TestConfig {
public:
S3TestConfig() {
Expand All @@ -58,18 +67,25 @@ class S3TestConfig {

access_key = GET_ENV_IF_DEFINED(S3_AK);
secret_key = GET_ENV_IF_DEFINED(S3_SK);
endpoint = GET_ENV_IF_DEFINED(S3_ENDPOINT);
role_arn = GET_ENV_IF_DEFINED(AWS_ROLE_ARN);
external_id = GET_ENV_IF_DEFINED(AWS_EXTERNAL_ID);
endpoint = get_env_with_fallback("S3_ENDPOINT", "AWS_ENDPOINT");
provider = GET_ENV_IF_DEFINED(S3_PROVIDER);
bucket = GET_ENV_IF_DEFINED(S3_BUCKET);
region = GET_ENV_IF_DEFINED(S3_REGION);
prefix = GET_ENV_IF_DEFINED(S3_PREFIX);
bucket = get_env_with_fallback("S3_BUCKET", "AWS_BUCKET");
region = get_env_with_fallback("S3_REGION", "AWS_REGION");
prefix = get_env_with_fallback("S3_PREFIX", "AWS_PREFIX");
}

bool is_enabled() const { return enabled; }

bool is_valid() const {
return enabled && !access_key.empty() && !secret_key.empty() && !endpoint.empty() &&
!region.empty() && !bucket.empty();
if (!enabled || endpoint.empty() || region.empty() || bucket.empty() || prefix.empty()) {
return false;
}
if (has_partial_static_credentials()) {
return false;
}
return has_static_credentials() || use_instance_profile();
}

std::string get_access_key() const { return access_key; }
Expand All @@ -79,15 +95,23 @@ class S3TestConfig {
std::string get_bucket() const { return bucket; }
std::string get_prefix() const { return prefix; }
std::string get_region() const { return region; }
std::string get_role_arn() const { return role_arn; }
std::string get_external_id() const { return external_id; }
bool use_instance_profile() const { return !has_static_credentials(); }

void print_config() const {
std::cout << "S3 Test Configuration:" << std::endl;
std::cout << " Enabled: " << (enabled ? "Yes" : "No") << std::endl;
if (enabled) {
std::cout << " Auth Mode: " << (has_static_credentials() ? "AK/SK" : "IAM Role")
<< std::endl;
std::cout << " Access Key: " << (access_key.empty() ? "<empty>" : "<set>")
<< std::endl;
std::cout << " Secret Key: " << (secret_key.empty() ? "<empty>" : "<hidden>")
<< std::endl;
std::cout << " Role Arn: " << (role_arn.empty() ? "<empty>" : role_arn) << std::endl;
std::cout << " External Id: " << (external_id.empty() ? "<empty>" : "<hidden>")
<< std::endl;
std::cout << " Endpoint: " << endpoint << std::endl;
std::cout << " Provider: " << provider << std::endl;
std::cout << " Bucket: " << bucket << std::endl;
Expand All @@ -97,9 +121,17 @@ class S3TestConfig {
}

private:
bool has_static_credentials() const { return !access_key.empty() && !secret_key.empty(); }

bool has_partial_static_credentials() const {
return !has_static_credentials() && (!access_key.empty() || !secret_key.empty());
}

bool enabled = false;
std::string access_key;
std::string secret_key;
std::string role_arn;
std::string external_id;
std::string endpoint;
std::string provider;
std::string bucket;
Expand All @@ -121,7 +153,8 @@ class S3FileSystemTest : public ::testing::Test {
}

if (!config_->is_valid()) {
GTEST_SKIP() << "S3 configuration is incomplete. Required: AK, SK, ENDPOINT, BUCKET.";
GTEST_SKIP() << "S3 configuration is incomplete. Required: ENDPOINT, REGION, BUCKET, "
"PREFIX and either AK/SK or iam role credentials.";
}

// Attempt to create S3 client
Expand Down Expand Up @@ -159,6 +192,11 @@ class S3FileSystemTest : public ::testing::Test {
s3_conf.client_conf.bucket = config_->get_bucket();
s3_conf.client_conf.region = config_->get_region();
s3_conf.client_conf.provider = convert_provider(config_->get_provider());
if (config_->use_instance_profile()) {
s3_conf.client_conf.cred_provider_type = CredProviderType::InstanceProfile;
s3_conf.client_conf.role_arn = config_->get_role_arn();
s3_conf.client_conf.external_id = config_->get_external_id();
}

s3_fs_ = DORIS_TRY(io::S3FileSystem::create(s3_conf, MOCK_S3_FS_ID));
return Status::OK();
Expand Down
52 changes: 45 additions & 7 deletions cloud/test/s3_accessor_client_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,15 @@ namespace doris::cloud {
return val ? std::string(val) : ""; \
}())

static std::string get_env_with_fallback(const char* primary, const char* fallback) {
const char* value = std::getenv(primary);
if (value != nullptr) {
return value;
}
value = std::getenv(fallback);
return value != nullptr ? value : "";
}

class S3TestConfig {
public:
S3TestConfig() {
Expand All @@ -72,18 +81,25 @@ class S3TestConfig {

access_key = GET_ENV_IF_DEFINED(S3_AK);
secret_key = GET_ENV_IF_DEFINED(S3_SK);
endpoint = GET_ENV_IF_DEFINED(S3_ENDPOINT);
role_arn = GET_ENV_IF_DEFINED(AWS_ROLE_ARN);
external_id = GET_ENV_IF_DEFINED(AWS_EXTERNAL_ID);
endpoint = get_env_with_fallback("S3_ENDPOINT", "AWS_ENDPOINT");
provider = GET_ENV_IF_DEFINED(S3_PROVIDER);
bucket = GET_ENV_IF_DEFINED(S3_BUCKET);
region = GET_ENV_IF_DEFINED(S3_REGION);
prefix = GET_ENV_IF_DEFINED(S3_PREFIX);
bucket = get_env_with_fallback("S3_BUCKET", "AWS_BUCKET");
region = get_env_with_fallback("S3_REGION", "AWS_REGION");
prefix = get_env_with_fallback("S3_PREFIX", "AWS_PREFIX");
}

bool is_enabled() const { return enabled; }

bool is_valid() const {
return enabled && !access_key.empty() && !secret_key.empty() && !endpoint.empty() &&
!region.empty() && !bucket.empty();
if (!enabled || endpoint.empty() || region.empty() || bucket.empty() || prefix.empty()) {
return false;
}
if (has_partial_static_credentials()) {
return false;
}
return has_static_credentials() || use_instance_profile();
}

std::string get_access_key() const { return access_key; }
Expand All @@ -93,15 +109,23 @@ class S3TestConfig {
std::string get_bucket() const { return bucket; }
std::string get_prefix() const { return prefix; }
std::string get_region() const { return region; }
std::string get_role_arn() const { return role_arn; }
std::string get_external_id() const { return external_id; }
bool use_instance_profile() const { return !has_static_credentials(); }

void print_config() const {
std::cout << "S3 Test Configuration:" << std::endl;
std::cout << " Enabled: " << (enabled ? "Yes" : "No") << std::endl;
if (enabled) {
std::cout << " Auth Mode: " << (has_static_credentials() ? "AK/SK" : "IAM Role")
<< std::endl;
std::cout << " Access Key: " << (access_key.empty() ? "<empty>" : "<set>")
<< std::endl;
std::cout << " Secret Key: " << (secret_key.empty() ? "<empty>" : "<hidden>")
<< std::endl;
std::cout << " Role Arn: " << (role_arn.empty() ? "<empty>" : role_arn) << std::endl;
std::cout << " External Id: " << (external_id.empty() ? "<empty>" : "<hidden>")
<< std::endl;
std::cout << " Endpoint: " << endpoint << std::endl;
std::cout << " Provider: " << provider << std::endl;
std::cout << " Bucket: " << bucket << std::endl;
Expand All @@ -111,9 +135,17 @@ class S3TestConfig {
}

private:
bool has_static_credentials() const { return !access_key.empty() && !secret_key.empty(); }

bool has_partial_static_credentials() const {
return !has_static_credentials() && (!access_key.empty() || !secret_key.empty());
}

bool enabled = false;
std::string access_key;
std::string secret_key;
std::string role_arn;
std::string external_id;
std::string endpoint;
std::string provider;
std::string bucket;
Expand All @@ -135,7 +167,8 @@ class S3AccessorClientTest : public ::testing::Test {
}

if (!config_->is_valid()) {
GTEST_SKIP() << "S3 configuration is incomplete. Required: AK, SK, ENDPOINT, BUCKET.";
GTEST_SKIP() << "S3 configuration is incomplete. Required: ENDPOINT, REGION, BUCKET, "
"PREFIX and either AK/SK or iam role credentials.";
}

// Attempt to create S3 accessor
Expand Down Expand Up @@ -169,6 +202,11 @@ class S3AccessorClientTest : public ::testing::Test {
s3_conf.bucket = config_->get_bucket();
s3_conf.prefix = config_->get_prefix();
s3_conf.provider = convert_provider(config_->get_provider());
if (config_->use_instance_profile()) {
s3_conf.cred_provider_type = CredProviderType::InstanceProfile;
s3_conf.role_arn = config_->get_role_arn();
s3_conf.external_id = config_->get_external_id();
}

std::shared_ptr<S3Accessor> accessor;
int ret = S3Accessor::create(s3_conf, &accessor);
Expand Down
Loading