SERVER-44676 Enable TCP_FASTOPEN

jstests/core/tcp_fastopen.js

@@ -0,0 +1,34 @@
+// Attempt to verify that connections can make use of TCP_FASTOPEN
+
+(function() {
+'use strict';
+
+// Does it make sense to expect TFO support?
+try {
+    // Both client and server bits must be set to run this test.
+    const val = cat("/proc/sys/net/ipv4/tcp_fastopen");
+    if ((Number.parseInt(val) & 3) != 3) {
+        print("==Skipping test, tcp_fastopen not enabled: " + val);
+        return;
+    }
+} catch (e) {
+    // File not found or unreadable, assume no TFO support.
+    print("==Skipping test, unable to read /proc/sys/net/ipv4/tcp_fastopen");
+    return;
+}
+
+const tcpFastOpen = db.serverStatus().network.tcpFastOpen;
+printjson(tcpFastOpen);
+
+const confused = "proc file suggests this kernel is capable, but setsockopt failed";
+assert.eq(true, tcpFastOpen.serverSupported, confused);
+assert.eq(true, tcpFastOpen.clientSupported, confused);
+
+const countBefore = tcpFastOpen.accepted;
+
+const netConn = runMongoProgram('mongo', '--port', myPort(), '--eval', ';');
+assert.eq(0, netConn);
+
+const countAfter = db.serverStatus().network.tcpFastOpen.accepted;
+assert.gt(countAfter, countBefore, "Second connection did not trigger TFO");
+})();

src/mongo/db/stats/counters.cpp

@@ -147,14 +147,26 @@ void NetworkCounter::hitLogicalOut(long long bytes) {
     }
 }
 
+void NetworkCounter::acceptedTFOIngress() {
+    _tfo.accepted.fetchAndAddRelaxed(1);
+}
+
 void NetworkCounter::append(BSONObjBuilder& b) {
     b.append("bytesIn", static_cast<long long>(_together.logicalBytesIn.loadRelaxed()));
     b.append("bytesOut", static_cast<long long>(_logicalBytesOut.loadRelaxed()));
     b.append("physicalBytesIn", static_cast<long long>(_physicalBytesIn.loadRelaxed()));
     b.append("physicalBytesOut", static_cast<long long>(_physicalBytesOut.loadRelaxed()));
     b.append("numRequests", static_cast<long long>(_together.requests.loadRelaxed()));
-}
 
+    BSONObjBuilder tfo;
+#ifdef __linux__
+    tfo.append("kernelSetting", _tfo.kernelSetting);
+#endif
+    tfo.append("serverSupported", _tfo.kernelSupportServer);
+    tfo.append("clientSupported", _tfo.kernelSupportClient);
+    tfo.append("accepted", _tfo.accepted.loadRelaxed());
+    b.append("tcpFastOpen", tfo.obj());
+}
 
 OpCounters globalOpCounters;
 OpCounters replOpCounters;

src/mongo/db/stats/counters.h

@@ -119,6 +119,21 @@ class NetworkCounter {
     void hitLogicalIn(long long bytes);
     void hitLogicalOut(long long bytes);
 
+    // TFO Counters and Status;
+    void acceptedTFOIngress();
+
+    void setTFOKernelSetting(std::int64_t val) {
+        _tfo.kernelSetting = val;
+    }
+
+    void setTFOServerSupport(bool val) {
+        _tfo.kernelSupportServer = val;
+    }
+
+    void setTFOClientSupport(bool val) {
+        _tfo.kernelSupportClient = val;
+    }
+
     void append(BSONObjBuilder& b);
 
 private:
@@ -136,6 +151,17 @@ class NetworkCounter {
                   "cache line spill");
 
     CacheAligned<AtomicWord<long long>> _logicalBytesOut{0};
+
+    struct TFO {
+        // Counter of inbound connections at runtime.
+        AtomicWord<std::int64_t> accepted{0};
+
+        // Info determined at startup.
+        std::int64_t kernelSetting;
+        bool kernelSupportServer{false};
+        bool kernelSupportClient{false};
+    };
+    CacheAligned<TFO> _tfo{};
 };
 
 extern NetworkCounter networkCounter;

src/mongo/transport/SConscript

@@ -51,6 +51,7 @@ tlEnv.Library(
     target='transport_layer',
     source=[
         'transport_layer_asio.cpp',
+        env.Idlc('transport_options.idl')[0],
     ],
     LIBDEPS=[
         'transport_layer_common',
@@ -61,7 +62,9 @@ tlEnv.Library(
         '$BUILD_DIR/mongo/db/stats/counters',
     ],
     LIBDEPS_PRIVATE=[
+        '$BUILD_DIR/mongo/idl/server_parameter',
         '$BUILD_DIR/mongo/util/net/ssl_manager',
+        '$BUILD_DIR/mongo/util/options_parser/options_parser',
         '$BUILD_DIR/third_party/shim_asio',
     ],
 )

src/mongo/transport/session_asio.h

@@ -78,12 +78,16 @@ class TransportLayerASIO::ASIOSession final : public Session {
     ASIOSession& operator=(const ASIOSession&) = delete;
 
 public:
+    using Endpoint = asio::generic::stream_protocol::endpoint;
+
     // If the socket is disconnected while any of these options are being set, this constructor
     // may throw, but it is guaranteed to throw a mongo DBException.
-    ASIOSession(TransportLayerASIO* tl, GenericSocket socket, bool isIngressSession) try
-        : _socket(std::move(socket)),
-          _tl(tl),
-          _isIngressSession(isIngressSession) {
+    ASIOSession(TransportLayerASIO* tl,
+                GenericSocket socket,
+                bool isIngressSession,
+                Endpoint endpoint = Endpoint()) try : _socket(std::move(socket)),
+                                                      _tl(tl),
+                                                      _isIngressSession(isIngressSession) {
         auto family = endpointToSockAddr(_socket.local_endpoint()).getType();
         if (family == AF_INET || family == AF_INET6) {
             _socket.set_option(asio::ip::tcp::no_delay(true));
@@ -92,7 +96,16 @@ class TransportLayerASIO::ASIOSession final : public Session {
         }
 
         _localAddr = endpointToSockAddr(_socket.local_endpoint());
-        _remoteAddr = endpointToSockAddr(_socket.remote_endpoint());
+
+        if (endpoint == Endpoint()) {
+            // Inbound connection, query socket for remote.
+            _remoteAddr = endpointToSockAddr(_socket.remote_endpoint());
+        } else {
+            // Outbound connection, get remote from resolved endpoint.
+            // Necessary for TCP_FASTOPEN where the remote isn't connected yet.
+            _remoteAddr = endpointToSockAddr(endpoint);
+        }
+
         _local = HostAndPort(_localAddr.toString(true));
         _remote = HostAndPort(_remoteAddr.toString(true));
     } catch (const DBException&) {