Skip to content

branch-4.0: [opt](cloud) Support fdb locality aware load balancing #61312 #61375

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
github-actions wants to merge 1 commit into
base: branch-4.0
Choose a base branch
from
Open

branch-4.0: [opt](cloud) Support fdb locality aware load balancing #61312 #61375

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions cloud/src/common/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ CONF_Bool(enable_fdb_external_client_directory, "true");
// The directory path of external foundationdb client library.
// eg: /path/to/dir1:/path/to/dir2:...
CONF_String(fdb_external_client_directory, "./lib/fdb/7.3.69/");
// Enable FDB locality-aware load balancing. When enabled, fdb_zone_id and fdb_dc_id
// will be set on the database for better location-aware request routing.
CONF_Bool(enable_fdb_locality_load_balance, "false");
CONF_String(fdb_zone_id, "");
CONF_String(fdb_dc_id, "");
CONF_String(http_token, "greedisgood9999");
// use volatile mem kv for test. MUST NOT be `true` in production environment.
CONF_Bool(use_mem_kv, "false");
Expand Down
57 changes: 56 additions & 1 deletion cloud/src/meta-store/txn_kv.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,10 @@ inline constexpr size_t write_clear_range_approximate_size(size_t begin_size, si

} // anonymous namespace

std::string_view KNOB_LOAD_BALANCE_ZONE_ID_LOCALITY_ENABLED =
"load_balance_zone_id_locality_enabled=1";
std::string_view KNOB_LOAD_BALANCE_DC_ID_LOCALITY_ENABLED = "load_balance_dc_id_locality_enabled=1";

// Ref https://apple.github.io/foundationdb/api-error-codes.html#developer-guide-error-codes.
constexpr fdb_error_t FDB_ERROR_CODE_TIMED_OUT = 1004;
constexpr fdb_error_t FDB_ERROR_CODE_TXN_TOO_OLD = 1007;
Expand Down Expand Up @@ -406,6 +410,33 @@ int Network::init() {
LOG(INFO) << "set fdb external client directory: " << config::fdb_external_client_directory;
}

if (config::enable_fdb_locality_load_balance) {
if (!config::fdb_zone_id.empty()) {
err = fdb_network_set_option(
FDB_NET_OPTION_KNOB,
(const uint8_t*)KNOB_LOAD_BALANCE_ZONE_ID_LOCALITY_ENABLED.data(),
KNOB_LOAD_BALANCE_ZONE_ID_LOCALITY_ENABLED.size());
if (err) {
LOG(WARNING) << "failed to set fdb load balance zone id locality enabled, err: "
<< fdb_get_error(err);
return 1;
}
LOG(INFO) << "set fdb load balance zone id locality enabled";
}
if (!config::fdb_dc_id.empty()) {
err = fdb_network_set_option(
FDB_NET_OPTION_KNOB,
(const uint8_t*)KNOB_LOAD_BALANCE_DC_ID_LOCALITY_ENABLED.data(),
KNOB_LOAD_BALANCE_DC_ID_LOCALITY_ENABLED.size());
if (err) {
LOG(WARNING) << "failed to set fdb load balance dc id locality enabled, err: "
<< fdb_get_error(err);
return 1;
}
LOG(INFO) << "set fdb load balance dc id locality enabled";
}
}

// ATTN: Network can be configured only once,
// even if fdb_stop_network() is called successfully
err = fdb_setup_network(); // Must be called only once before any
Expand Down Expand Up @@ -457,14 +488,38 @@ void Network::stop() {
// =============================================================================

int Database::init() {
// TODO: process opt
fdb_error_t err = fdb_create_database(cluster_file_path_.c_str(), &db_);
if (err) {
LOG(WARNING) << __PRETTY_FUNCTION__ << " fdb_create_database error: " << fdb_get_error(err)
<< " conf: " << cluster_file_path_;
return 1;
}

if (config::enable_fdb_locality_load_balance) {
if (!config::fdb_zone_id.empty()) {
err = fdb_database_set_option(db_, FDB_DB_OPTION_MACHINE_ID,
(const uint8_t*)config::fdb_zone_id.c_str(),
config::fdb_zone_id.size());
if (err) {
LOG(WARNING) << "failed to set FDB_DB_OPTION_MACHINE_ID: " << fdb_get_error(err)
<< ", zone_id: " << config::fdb_zone_id;
return 1;
}
LOG(INFO) << "set FDB_DB_OPTION_MACHINE_ID (zone_id): " << config::fdb_zone_id;
}
if (!config::fdb_dc_id.empty()) {
err = fdb_database_set_option(db_, FDB_DB_OPTION_DATACENTER_ID,
(const uint8_t*)config::fdb_dc_id.c_str(),
config::fdb_dc_id.size());
if (err) {
LOG(WARNING) << "failed to set FDB_DB_OPTION_DATACENTER_ID: " << fdb_get_error(err)
<< ", dc_id: " << config::fdb_dc_id;
return 1;
}
LOG(INFO) << "set FDB_DB_OPTION_DATACENTER_ID (dc_id): " << config::fdb_dc_id;
}
}

return 0;
}

Expand Down
Loading