From 2e26eb28d748292c98736bec7e063042c5a01cea Mon Sep 17 00:00:00 2001 From: Trond Norbye Date: Tue, 22 Nov 2022 10:40:22 +0100 Subject: [PATCH] MB-54641: Backport tuning of TCP Keepalive This is a backport of MB-53939 and MB-54477 to allow tuning of TCP Keepalive on the user ports Change-Id: Iadfef3faad120ef40094ecd13885fd08e2f9281a Reviewed-on: https://review.couchbase.org/c/kv_engine/+/183116 Well-Formed: Restriction Checker Tested-by: Build Bot Reviewed-by: Dave Rigby --- daemon/server_socket.cc | 49 ++++++++++++++++++++++++++++++++++++++++ daemon/settings.cc | 42 ++++++++++++++++++++++++++++++++++ daemon/settings.h | 48 +++++++++++++++++++++++++++++++++++++++ docs/memcached.json.adoc | 18 +++++++++++++++ 4 files changed, 157 insertions(+) diff --git a/daemon/server_socket.cc b/daemon/server_socket.cc index c86d7d099d..94d3f4bda3 100644 --- a/daemon/server_socket.cc +++ b/daemon/server_socket.cc @@ -73,6 +73,9 @@ class LibeventServerSocketImpl : public ServerSocket { } protected: + /// Set the various TCP Keepalive options to the provided socket. + void setTcpKeepalive(SOCKET client); + /// The socket object to accept clients from const SOCKET sfd; @@ -233,6 +236,12 @@ void LibeventServerSocketImpl::acceptNewClient() { return; } + // Connections connecting to a system port should use the TCP Keepalive + // configuration configured in the OS. + if (!interface->system) { + setTcpKeepalive(client); + } + uniqueSslPtr ssl; bool failed = false; if (interface->tls) { @@ -261,6 +270,46 @@ void LibeventServerSocketImpl::acceptNewClient() { FrontEndThread::dispatch(client, interface, std::move(ssl)); } +void LibeventServerSocketImpl::setTcpKeepalive(SOCKET client) { + auto& settings = Settings::instance(); + const auto idle = settings.getTcpKeepAliveIdle(); + if (idle.count() != 0) { + uint32_t t = idle.count(); +#ifdef __APPLE__ +#define TCP_KEEPIDLE TCP_KEEPALIVE +#endif + if (cb::net::setsockopt( + client, IPPROTO_TCP, TCP_KEEPIDLE, &t, sizeof(t)) == -1) { + LOG_WARNING("{} Failed to set TCP_KEEPIDLE: {}", + client, + cb_strerror(cb::net::get_socket_error())); + } + } + const auto interval = settings.getTcpKeepAliveInterval(); + if (interval.count() != 0) { + uint32_t val = interval.count(); + if (cb::net::setsockopt( + client, IPPROTO_TCP, TCP_KEEPINTVL, &val, sizeof(val)) == + -1) { + LOG_WARNING("{} Failed to set TCP_KEEPINTVL: {}", + client, + cb_strerror(cb::net::get_socket_error())); + } + } + const auto probes = settings.getTcpKeepAliveProbes(); + if (probes) { + if (cb::net::setsockopt(client, + IPPROTO_TCP, + TCP_KEEPCNT, + &probes, + sizeof(probes)) == -1) { + LOG_WARNING("{} Failed to set TCP_KEEPCNT: {}", + client, + cb_strerror(cb::net::get_socket_error())); + } + } +} + nlohmann::json LibeventServerSocketImpl::toJson() const { nlohmann::json ret; diff --git a/daemon/settings.cc b/daemon/settings.cc index b1e1b969e7..5ac99b998a 100644 --- a/daemon/settings.cc +++ b/daemon/settings.cc @@ -246,6 +246,18 @@ static void handle_low_reqs_event(Settings& s, const nlohmann::json& obj) { s, obj, EventPriority::Low, "reqs_per_event_low_priority"); } +static void handle_tcp_keepalive_idle(Settings& s, const nlohmann::json& obj) { + s.setTcpKeepAliveIdle(std::chrono::seconds(obj.get())); +} +static void handle_tcp_keepalive_interval(Settings& s, + const nlohmann::json& obj) { + s.setTcpKeepAliveInterval(std::chrono::seconds(obj.get())); +} +static void handle_tcp_keepalive_probes(Settings& s, + const nlohmann::json& obj) { + s.setTcpKeepAliveProbes(obj.get()); +} + /** * Handle the "verbosity" tag in the settings * @@ -697,6 +709,9 @@ void Settings::reconfigure(const nlohmann::json& json) { {"ssl_sasl_mechanisms", handle_ssl_sasl_mechanisms}, {"stdin_listener", handle_stdin_listener}, {"dedupe_nmvb_maps", handle_dedupe_nmvb_maps}, + {"tcp_keepalive_idle", handle_tcp_keepalive_idle}, + {"tcp_keepalive_interval", handle_tcp_keepalive_interval}, + {"tcp_keepalive_probes", handle_tcp_keepalive_probes}, {"xattr_enabled", handle_xattr_enabled}, {"client_cert_auth", handle_client_cert_auth}, {"collections_enabled", handle_collections_enabled}, @@ -818,6 +833,33 @@ void Settings::updateSettings(const Settings& other, bool apply) { } // Ok, go ahead and update the settings!! + if (other.has.tcp_keepalive_idle) { + if (other.getTcpKeepAliveIdle() != getTcpKeepAliveIdle()) { + LOG_INFO("Change TCP_KEEPIDLE time from {}s to {}s", + getTcpKeepAliveIdle().count(), + other.getTcpKeepAliveIdle().count()); + setTcpKeepAliveIdle(other.getTcpKeepAliveIdle()); + } + } + + if (other.has.tcp_keepalive_interval) { + if (other.getTcpKeepAliveInterval() != getTcpKeepAliveInterval()) { + LOG_INFO("Change TCP_KEEPINTVL interval from {}s to {}s", + getTcpKeepAliveInterval().count(), + other.getTcpKeepAliveInterval().count()); + setTcpKeepAliveInterval(other.getTcpKeepAliveInterval()); + } + } + + if (other.has.tcp_keepalive_probes) { + if (other.tcp_keepalive_probes != tcp_keepalive_probes) { + LOG_INFO("Change TCP_KEEPCNT from {} to {}", + getTcpKeepAliveProbes(), + other.getTcpKeepAliveProbes()); + setTcpKeepAliveProbes(other.getTcpKeepAliveProbes()); + } + } + if (other.has.always_collect_trace_info) { if (other.alwaysCollectTraceInfo() != alwaysCollectTraceInfo()) { if (other.alwaysCollectTraceInfo()) { diff --git a/daemon/settings.h b/daemon/settings.h index 930a87d3bf..4943fc1f67 100644 --- a/daemon/settings.h +++ b/daemon/settings.h @@ -623,6 +623,42 @@ class Settings { notify_changed("xattr_enabled"); } + /// Set time until the first probe is sent + void setTcpKeepAliveIdle(std::chrono::seconds val) { + tcp_keepalive_idle.store(val, std::memory_order_release); + has.tcp_keepalive_idle = true; + notify_changed("tcp_keepalive_idle"); + } + + /// Get the time before the first probe is sent + std::chrono::seconds getTcpKeepAliveIdle() const { + return tcp_keepalive_idle.load(std::memory_order_acquire); + } + + /// Set the interval between the probes is sent + void setTcpKeepAliveInterval(std::chrono::seconds val) { + tcp_keepalive_interval.store(val, std::memory_order_release); + has.tcp_keepalive_interval = true; + notify_changed("tcp_keepalive_interval"); + } + + /// Get the interval between the probes is sent + std::chrono::seconds getTcpKeepAliveInterval() const { + return tcp_keepalive_interval.load(std::memory_order_acquire); + } + + /// Set the number of probes sent before the connection is marked as dead + void setTcpKeepAliveProbes(uint32_t val) { + tcp_keepalive_probes.store(val, std::memory_order_release); + has.tcp_keepalive_probes = true; + notify_changed("tcp_keepalive_probes"); + } + + /// Get the number of probes sent before the connection is marked as dead + uint32_t getTcpKeepAliveProbes() const { + return tcp_keepalive_probes.load(std::memory_order_acquire); + } + /** * Collections prototype means certain work-in-progress parts of collections * are enabled/disabled and also means DCP auto-enables collections for @@ -987,6 +1023,15 @@ class Settings { /// Number of storage backend threads std::atomic num_storage_threads{0}; + /// The number of seconds before keepalive kicks in + std::atomic tcp_keepalive_idle{ + std::chrono::seconds{360}}; + /// The number of seconds between each probe sent in keepalive + std::atomic tcp_keepalive_interval{ + std::chrono::seconds{10}}; + /// The number of missing probes before the connection is considered dead + std::atomic tcp_keepalive_probes{3}; + folly::Synchronized phosphor_config{ "buffer-mode:ring;buffer-size:20971520;enabled-categories:*"}; @@ -1036,6 +1081,9 @@ class Settings { bool stdin_listener = false; bool scramsha_fallback_salt = false; bool external_auth_service = false; + bool tcp_keepalive_idle = false; + bool tcp_keepalive_interval = false; + bool tcp_keepalive_probes = false; bool active_external_users_push_interval = false; bool max_connections = false; bool system_connections = false; diff --git a/docs/memcached.json.adoc b/docs/memcached.json.adoc index e51521cb82..ba58706410 100644 --- a/docs/memcached.json.adoc +++ b/docs/memcached.json.adoc @@ -76,6 +76,24 @@ clients. By default this number is set to 75% of the number of cores available on the system (but no less than 4). The value for threads should be specified as an integral number. +=== tcp_keepalive_idle + +The *tcp_keepalive_idle* attribute is an integral value specifying the +number of seconds before the first TCP keepalive probe gets sent. Set the +value to 0 to use the operating system default value. + +=== tcp_keepalive_interval + +The *tcp_keepalive_interval* attribute is an integral value specifying the +number of seconds between each TCP keepalive probe gets sent.Set the +value to 0 to use the operating system default value. + +=== tcp_keepalive_probes + +The *tcp_keepalive_probes* attribute is an integral value specifying the +number keepalive probes which should be sent before marking the connection +as dead. Set the value to 0 to use the operating system default value. + === prometheus The *prometheus* is a object with the following properties: