Refactor work with Tcp Fast Open (both sides); docs edited

docs/conf_options_reference/searchd_program_configuration_options.rst

@@ -490,11 +490,11 @@ Unix-domain sockets are not supported on Windows.
 listen_tfo
 ~~~~~~~~~~
 
-This setting allows TCP_FASTOPEN flag for all listeners. By default it is 0,
-but it is safe to have it set '1' always.
+This setting allows TCP_FASTOPEN flag for all listeners. By default it is managed by system,
+but may be explicitly switched off by setting to '0'.
 
 For general knowledge about TCP Fast Open extension you can visit Wikipedia.
-Shortly speaking, it allows to eliminate one TCP packages round-trip when establishing
+Shortly speaking, it allows to eliminate one TCP round-trip when establishing
 connection.
 
 In practice using TFO in many situation may optimize client-agent network efficiency
@@ -506,9 +506,9 @@ not the rule. Linux (as most progressive) supports it since 2011, on kernels sta
 (for server side). Windows supports it from some build of Windows 10. Anothers (FreeBSD, MacOS)
 also in game.
 
-For Linux system check variable ``/proc/sys/net/ipv4/tcp_fastopen`` and ensure that bit 1 (which
-manages server side of TFO) is set (i.e. value is 2 or 3). Remote clients, in turn, must have bit 0
-to be set on their side, but this is common default.
+For Linux system daemon checks variable ``/proc/sys/net/ipv4/tcp_fastopen`` and behaves according
+ to it. Bit 0 manages client side, bit 1 rules listeners. By default system has this param set to 1,
+ i.e. clients enabled, listeners disabled.
 
 .. _log:
 

src/searchd.cpp

@@ -175,7 +175,7 @@ static int				g_iThdPoolCount		= 2;
 static int				g_iThdQueueMax		= 0;
 static int				g_tmWait = 1;
 bool					g_bGroupingInUtc	= false;
-static bool				g_bTcpFastOpen		= false; ///< whether to use TFO on listeners
+static auto&			g_iTFO = sphGetTFO ();
 static CSphString		g_sShutdownToken;
 
 struct Listener_t
@@ -2159,16 +2159,13 @@ int sphCreateInetSocket ( DWORD uAddr, int iPort )
 		sphWarning ( "setsockopt(TCP_NODELAY) failed: %s", sphSockError() );
 #endif
 
-	// TFO
-	if ( g_bTcpFastOpen )
-	{
 #ifdef TCP_FASTOPEN
+	if ( ( g_iTFO!=TFO_ABSENT ) && ( g_iTFO & TFO_LISTEN ) )
+	{
 		if ( setsockopt ( iSock, IPPROTO_TCP, TCP_FASTOPEN, ( char * ) &iOn, sizeof ( iOn ) ) )
-			sphWarning ( "can't set TCP_FASTOPEN option for listener: %s", sphSockError () );
-#else
-		sphWarning ( "daemon wasn't build with TCP Fast Open support, option 'listen_tfo' ignored");
-#endif
+			sphWarning ( "setsockopt(TCP_FASTOPEN) failed: %s", sphSockError () );
 	}
+#endif
 
 	int iTries = 12;
 	int iRes;
@@ -22880,6 +22877,45 @@ static void ParsePredictedTimeCosts ( const char * p )
 	}
 }
 
+// read system TFO settings and init g_ITFO according to it.
+/* From https://www.kernel.org/doc/Documentation/networking/ip-sysctl.txt
+ * possible bitmask values are:
+
+	  0x1: (client) enables sending data in the opening SYN on the client.
+	  0x2: (server) enables the server support, i.e., allowing data in
+			a SYN packet to be accepted and passed to the
+			application before 3-way handshake finishes.
+	  0x4: (client) send data in the opening SYN regardless of cookie
+			availability and without a cookie option.
+	0x200: (server) accept data-in-SYN w/o any cookie option present.
+	0x400: (server) enable all listeners to support Fast Open by
+			default without explicit TCP_FASTOPEN socket option.
+
+ Actually we interested only in first 2 bits.
+ */
+static void CheckSystemTFO ()
+{
+#if defined (MSG_FASTOPEN)
+	char sBuf[20] = { 0 };
+	g_iTFO = TFO_ABSENT;
+
+	FILE * fp = fopen ( "/proc/sys/net/ipv4/tcp_fastopen", "rb" );
+	if ( !fp )
+	{
+		sphWarning ( "TCP fast open unavailable (can't read /proc/sys/net/ipv4/tcp_fastopen, unsupported kernel?)" );
+		return;
+	}
+
+	auto szResult = fgets ( sBuf, 20, fp );
+	fclose ( fp );
+	if ( !szResult )
+		return;
+
+	g_iTFO = atoi ( szResult );
+#else
+	g_iTFO = 3; // suggest it is available.
+#endif
+}
 
 void ConfigureSearchd ( const CSphConfig & hConf, bool bOptPIDFile )
 {
@@ -22964,8 +23000,11 @@ void ConfigureSearchd ( const CSphConfig & hConf, bool bOptPIDFile )
 	g_iThrottleAccept = hSearchd.GetInt ( "net_throttle_accept", g_iThrottleAccept );
 	g_iNetWorkers = hSearchd.GetInt ( "net_workers", g_iNetWorkers );
 	g_iNetWorkers = Max ( g_iNetWorkers, 1 );
-	g_bTcpFastOpen = ( hSearchd.GetInt ( "listen_tfo", 0 )==1 );
-
+	CheckSystemTFO();
+	if ( g_iTFO!=TFO_ABSENT && hSearchd.GetInt ( "listen_tfo" )==0 )
+	{
+		g_iTFO &= ~TFO_LISTEN;
+	}
 	if ( hSearchd ( "collation_libc_locale" ) )
 	{
 		const char * sLocale = hSearchd.GetStr ( "collation_libc_locale" );

src/searchdha.cpp

@@ -45,6 +45,7 @@ DWORD			g_uHAPeriodKarma	= 60;		// by default use the last 1 minute statistic to
 int				g_iPersistentPoolSize	= 0;
 
 static auto& g_bShutdown = sphGetShutdown();
+static auto& g_iTFO = sphGetTFO ();
 
 CSphString HostDesc_t::GetMyUrl() const
 {
@@ -1989,7 +1990,7 @@ inline SSIZE_T AgentConn_t::SendChunk ()
 /// @return 1 on success, 0 if need fallback into usual (::connect), -1 on failure.
 int AgentConn_t::DoTFO ( struct sockaddr * pSs, int iLen )
 {
-	if ( pSs->sa_family==AF_UNIX )
+	if ( pSs->sa_family==AF_UNIX || g_iTFO==TFO_ABSENT || !(g_iTFO & TFO_CONNECT) )
 		return 0;
 	m_iStartQuery = sphMicroTimer (); // copied old behaviour
 #if USE_WINDOWS
@@ -2081,6 +2082,13 @@ int AgentConn_t::DoTFO ( struct sockaddr * pSs, int iLen )
 	auto iErr = sphSockGetErrno ();
 	if ( iErr!=EINPROGRESS )
 	{
+		if ( iErr==EOPNOTSUPP )
+		{
+			assert ( g_iTFO!=TFO_ABSENT );
+			sphWarning("TFO client supoport unavailable, switch to usual connect()");
+			g_iTFO &= ~TFO_CONNECT;
+			return 0;
+		}
 		Fatal ( eConnectFailures, "sendmsg/connectx() failed: errno=%d, %s", iErr, sphSockError ( iErr ) );
 		return -1;
 	}

src/sphinx.cpp

@@ -33100,3 +33100,9 @@ volatile bool& sphGetShutdown ()
 	static bool bShutdown = false;
 	return bShutdown;
 }
+
+volatile int &sphGetTFO ()
+{
+	static int iTFO = 0;
+	return iTFO;
+}

src/sphinx.h

@@ -3564,6 +3564,11 @@ extern CSphString g_sLemmatizerBase;
 // Get global shutdown flag
 volatile bool& sphGetShutdown();
 
+// Access to global TFO settings
+volatile int& sphGetTFO();
+#define TFO_CONNECT 1
+#define TFO_LISTEN 2
+#define TFO_ABSENT (-1)
 /////////////////////////////////////////////////////////////////////////////
 
 // workaround to suppress C4511/C4512 warnings (copy ctor and assignment operator) in VS 2003