MB-54641: Backport tuning of TCP Keepalive

This is a backport of MB-53939 and MB-54477 to allow tuning of
TCP Keepalive on the user ports

Change-Id: Iadfef3faad120ef40094ecd13885fd08e2f9281a
Reviewed-on: https://review.couchbase.org/c/kv_engine/+/183116
Well-Formed: Restriction Checker
Tested-by: Build Bot <build@couchbase.com>
Reviewed-by: Dave Rigby <daver@couchbase.com>

daemon/server_socket.cc

@@ -73,6 +73,9 @@ class LibeventServerSocketImpl : public ServerSocket {
     }
 
 protected:
+    /// Set the various TCP Keepalive options to the provided socket.
+    void setTcpKeepalive(SOCKET client);
+
     /// The socket object to accept clients from
     const SOCKET sfd;
 
@@ -233,6 +236,12 @@ void LibeventServerSocketImpl::acceptNewClient() {
         return;
     }
 
+    // Connections connecting to a system port should use the TCP Keepalive
+    // configuration configured in the OS.
+    if (!interface->system) {
+        setTcpKeepalive(client);
+    }
+
     uniqueSslPtr ssl;
     bool failed = false;
     if (interface->tls) {
@@ -261,6 +270,46 @@ void LibeventServerSocketImpl::acceptNewClient() {
     FrontEndThread::dispatch(client, interface, std::move(ssl));
 }
 
+void LibeventServerSocketImpl::setTcpKeepalive(SOCKET client) {
+    auto& settings = Settings::instance();
+    const auto idle = settings.getTcpKeepAliveIdle();
+    if (idle.count() != 0) {
+        uint32_t t = idle.count();
+#ifdef __APPLE__
+#define TCP_KEEPIDLE TCP_KEEPALIVE
+#endif
+        if (cb::net::setsockopt(
+                    client, IPPROTO_TCP, TCP_KEEPIDLE, &t, sizeof(t)) == -1) {
+            LOG_WARNING("{} Failed to set TCP_KEEPIDLE: {}",
+                        client,
+                        cb_strerror(cb::net::get_socket_error()));
+        }
+    }
+    const auto interval = settings.getTcpKeepAliveInterval();
+    if (interval.count() != 0) {
+        uint32_t val = interval.count();
+        if (cb::net::setsockopt(
+                    client, IPPROTO_TCP, TCP_KEEPINTVL, &val, sizeof(val)) ==
+            -1) {
+            LOG_WARNING("{} Failed to set TCP_KEEPINTVL: {}",
+                        client,
+                        cb_strerror(cb::net::get_socket_error()));
+        }
+    }
+    const auto probes = settings.getTcpKeepAliveProbes();
+    if (probes) {
+        if (cb::net::setsockopt(client,
+                                IPPROTO_TCP,
+                                TCP_KEEPCNT,
+                                &probes,
+                                sizeof(probes)) == -1) {
+            LOG_WARNING("{} Failed to set TCP_KEEPCNT: {}",
+                        client,
+                        cb_strerror(cb::net::get_socket_error()));
+        }
+    }
+}
+
 nlohmann::json LibeventServerSocketImpl::toJson() const {
     nlohmann::json ret;
 

daemon/settings.cc

@@ -246,6 +246,18 @@ static void handle_low_reqs_event(Settings& s, const nlohmann::json& obj) {
             s, obj, EventPriority::Low, "reqs_per_event_low_priority");
 }
 
+static void handle_tcp_keepalive_idle(Settings& s, const nlohmann::json& obj) {
+    s.setTcpKeepAliveIdle(std::chrono::seconds(obj.get<uint32_t>()));
+}
+static void handle_tcp_keepalive_interval(Settings& s,
+                                          const nlohmann::json& obj) {
+    s.setTcpKeepAliveInterval(std::chrono::seconds(obj.get<uint32_t>()));
+}
+static void handle_tcp_keepalive_probes(Settings& s,
+                                        const nlohmann::json& obj) {
+    s.setTcpKeepAliveProbes(obj.get<uint32_t>());
+}
+
 /**
  * Handle the "verbosity" tag in the settings
  *
@@ -697,6 +709,9 @@ void Settings::reconfigure(const nlohmann::json& json) {
             {"ssl_sasl_mechanisms", handle_ssl_sasl_mechanisms},
             {"stdin_listener", handle_stdin_listener},
             {"dedupe_nmvb_maps", handle_dedupe_nmvb_maps},
+            {"tcp_keepalive_idle", handle_tcp_keepalive_idle},
+            {"tcp_keepalive_interval", handle_tcp_keepalive_interval},
+            {"tcp_keepalive_probes", handle_tcp_keepalive_probes},
             {"xattr_enabled", handle_xattr_enabled},
             {"client_cert_auth", handle_client_cert_auth},
             {"collections_enabled", handle_collections_enabled},
@@ -818,6 +833,33 @@ void Settings::updateSettings(const Settings& other, bool apply) {
     }
 
     // Ok, go ahead and update the settings!!
+    if (other.has.tcp_keepalive_idle) {
+        if (other.getTcpKeepAliveIdle() != getTcpKeepAliveIdle()) {
+            LOG_INFO("Change TCP_KEEPIDLE time from {}s to {}s",
+                     getTcpKeepAliveIdle().count(),
+                     other.getTcpKeepAliveIdle().count());
+            setTcpKeepAliveIdle(other.getTcpKeepAliveIdle());
+        }
+    }
+
+    if (other.has.tcp_keepalive_interval) {
+        if (other.getTcpKeepAliveInterval() != getTcpKeepAliveInterval()) {
+            LOG_INFO("Change TCP_KEEPINTVL interval from {}s to {}s",
+                     getTcpKeepAliveInterval().count(),
+                     other.getTcpKeepAliveInterval().count());
+            setTcpKeepAliveInterval(other.getTcpKeepAliveInterval());
+        }
+    }
+
+    if (other.has.tcp_keepalive_probes) {
+        if (other.tcp_keepalive_probes != tcp_keepalive_probes) {
+            LOG_INFO("Change TCP_KEEPCNT from {} to {}",
+                     getTcpKeepAliveProbes(),
+                     other.getTcpKeepAliveProbes());
+            setTcpKeepAliveProbes(other.getTcpKeepAliveProbes());
+        }
+    }
+
     if (other.has.always_collect_trace_info) {
         if (other.alwaysCollectTraceInfo() != alwaysCollectTraceInfo()) {
             if (other.alwaysCollectTraceInfo()) {

daemon/settings.h

@@ -623,6 +623,42 @@ class Settings {
         notify_changed("xattr_enabled");
     }
 
+    /// Set time until the first probe is sent
+    void setTcpKeepAliveIdle(std::chrono::seconds val) {
+        tcp_keepalive_idle.store(val, std::memory_order_release);
+        has.tcp_keepalive_idle = true;
+        notify_changed("tcp_keepalive_idle");
+    }
+
+    /// Get the time before the first probe is sent
+    std::chrono::seconds getTcpKeepAliveIdle() const {
+        return tcp_keepalive_idle.load(std::memory_order_acquire);
+    }
+
+    /// Set the interval between the probes is sent
+    void setTcpKeepAliveInterval(std::chrono::seconds val) {
+        tcp_keepalive_interval.store(val, std::memory_order_release);
+        has.tcp_keepalive_interval = true;
+        notify_changed("tcp_keepalive_interval");
+    }
+
+    /// Get the interval between the probes is sent
+    std::chrono::seconds getTcpKeepAliveInterval() const {
+        return tcp_keepalive_interval.load(std::memory_order_acquire);
+    }
+
+    /// Set the number of probes sent before the connection is marked as dead
+    void setTcpKeepAliveProbes(uint32_t val) {
+        tcp_keepalive_probes.store(val, std::memory_order_release);
+        has.tcp_keepalive_probes = true;
+        notify_changed("tcp_keepalive_probes");
+    }
+
+    /// Get the number of probes sent before the connection is marked as dead
+    uint32_t getTcpKeepAliveProbes() const {
+        return tcp_keepalive_probes.load(std::memory_order_acquire);
+    }
+
     /**
      * Collections prototype means certain work-in-progress parts of collections
      * are enabled/disabled and also means DCP auto-enables collections for
@@ -987,6 +1023,15 @@ class Settings {
     /// Number of storage backend threads
     std::atomic<int> num_storage_threads{0};
 
+    /// The number of seconds before keepalive kicks in
+    std::atomic<std::chrono::seconds> tcp_keepalive_idle{
+            std::chrono::seconds{360}};
+    /// The number of seconds between each probe sent in keepalive
+    std::atomic<std::chrono::seconds> tcp_keepalive_interval{
+            std::chrono::seconds{10}};
+    /// The number of missing probes before the connection is considered dead
+    std::atomic<uint32_t> tcp_keepalive_probes{3};
+
     folly::Synchronized<std::string> phosphor_config{
             "buffer-mode:ring;buffer-size:20971520;enabled-categories:*"};
 
@@ -1036,6 +1081,9 @@ class Settings {
         bool stdin_listener = false;
         bool scramsha_fallback_salt = false;
         bool external_auth_service = false;
+        bool tcp_keepalive_idle = false;
+        bool tcp_keepalive_interval = false;
+        bool tcp_keepalive_probes = false;
         bool active_external_users_push_interval = false;
         bool max_connections = false;
         bool system_connections = false;

docs/memcached.json.adoc

@@ -76,6 +76,24 @@ clients. By default this number is set to 75% of the number of cores
 available on the system (but no less than 4). The value for threads
 should be specified as an integral number.
 
+=== tcp_keepalive_idle
+
+The *tcp_keepalive_idle* attribute is an integral value specifying the
+number of seconds before the first TCP keepalive probe gets sent. Set the
+value to 0 to use the operating system default value.
+
+=== tcp_keepalive_interval
+
+The *tcp_keepalive_interval* attribute is an integral value specifying the
+number of seconds between each TCP keepalive probe gets sent.Set the
+value to 0 to use the operating system default value.
+
+=== tcp_keepalive_probes
+
+The *tcp_keepalive_probes* attribute is an integral value specifying the
+number keepalive probes which should be sent before marking the connection
+as dead. Set the value to 0 to use the operating system default value.
+
 === prometheus
 
 The *prometheus* is a object with the following properties: