win: add tcp loopback fastpath support

Starting with Windows 8 / Server 2012, an new IOCTL
(SIO_LOOPBACK_FAST_PATH) is available to increase loopback
performance.

This feature is enabled by default on all server sockets and loopback
client socket when using either IPv4 or IPv6. It's forcibly disabled
when dualstack IPv4/IPv6 is enabled as it cause crashes. It can be
disabled explicitly using the new function uv_tcp_fastpath.

See also:
<http://blogs.technet.com/b/wincat/archive/2012/12/05/fast-tcp-loopback-performance-and-low-latency-with-windows-server-2012-tcp-loopback-fast-path.aspx>

Benchmarks numbers :

     test name      | without fastpath | with fastpath  |  ratio  |
--------------------|------------------|----------------|---------|
tcp_pump1_client    |   3.8 gbit/s     |  8.9 gbit/s    |  +134%  |
tcp_pump100_client  |   3.8 gbit/s     | 10.7 gbit/s    |  +181%  |
tcp-conn-pound-100  | 11926 accepts/s  | 9601 accepts/s |  - 19%  |
tcp-conn-pound-1000 | 10216 accepts/s  | 9144 accepts/s |  - 10%  |

include/uv.h

@@ -518,6 +518,7 @@ UV_EXTERN int uv_tcp_nodelay(uv_tcp_t* handle, int enable);
 UV_EXTERN int uv_tcp_keepalive(uv_tcp_t* handle,
                                int enable,
                                unsigned int delay);
+UV_EXTERN int uv_tcp_fastpath(uv_tcp_t* handle, int enable);
 UV_EXTERN int uv_tcp_simultaneous_accepts(uv_tcp_t* handle, int enable);
 
 enum uv_tcp_flags {

src/unix/tcp.c

@@ -348,6 +348,12 @@ int uv_tcp_keepalive(uv_tcp_t* handle, int on, unsigned int delay) {
 }
 
 
+int uv_tcp_fastpath(uv_tcp_t* handle, int enable) {
+  /* Windows-only feature */
+  return UV_ENOSYS;
+}
+
+
 int uv_tcp_simultaneous_accepts(uv_tcp_t* handle, int enable) {
   if (enable)
     handle->flags &= ~UV_TCP_SINGLE_ACCEPT;

src/win/internal.h

@@ -94,6 +94,7 @@ extern UV_THREAD_LOCAL int uv__crt_assert_enabled;
 #define UV_HANDLE_TCP_ACCEPT_STATE_CHANGING     0x10000000
 #define UV_HANDLE_TCP_SOCKET_CLOSED             0x20000000
 #define UV_HANDLE_SHARED_TCP_SOCKET             0x40000000
+#define UV_HANDLE_TCP_LOOPBACK_FASTPATH         0x80000000
 
 /* Only used by uv_pipe_t handles. */
 #define UV_HANDLE_NON_OVERLAPPED_PIPE           0x01000000

src/win/tcp.c

@@ -45,6 +45,29 @@ const unsigned int uv_simultaneous_server_accepts = 32;
 /* A zero-size buffer for use by uv_tcp_read */
 static char uv_zero_[] = "";
 
+/* Get the windows version once, used to test fastpath feature compatibility */
+static uv_once_t fastpath_unsupported_guard = UV_ONCE_INIT;
+static BOOL fastpath_supported = FALSE;
+static BOOL fastpath_default_enable = FALSE;
+
+static void uv__init_fastpath_support(void) {
+  RTL_OSVERSIONINFOW osvi = {0};
+  osvi.dwOSVersionInfoSize = sizeof(osvi);
+
+  if(pRtlGetVersion(&osvi) == STATUS_SUCCESS) {
+    /* TCP fast path unsupported if < Windows 6.2 */
+    fastpath_supported =
+          osvi.dwMajorVersion > 6 ||
+          (osvi.dwMajorVersion == 6 && osvi.dwMinorVersion >= 2);
+
+    /* TCP fast path is slower on Windows 10 so enable it by default only on Windows 8/8.1 (>= 6.2 && < 6.4) */
+    fastpath_default_enable =
+          osvi.dwMajorVersion == 6 &&
+          osvi.dwMinorVersion >= 2 && osvi.dwMinorVersion < 4;
+  }
+}
+
+
 static int uv__tcp_nodelay(uv_tcp_t* handle, SOCKET socket, int enable) {
   if (setsockopt(socket,
                  IPPROTO_TCP,
@@ -78,6 +101,34 @@ static int uv__tcp_keepalive(uv_tcp_t* handle, SOCKET socket, int enable, unsign
 }
 
 
+static int uv__tcp_fastpath(uv_tcp_t* handle, int enable) {
+  SOCKET socket = handle->socket;
+  DWORD bytes;
+
+  /* return value of WSAIoctl is not consistent accross Windows versions so check it here */
+  if (!fastpath_supported)
+    return WSAEOPNOTSUPP;
+
+  /* Inihibit TCP fastpath if using ipv6 as it causes crashes with dualstack */
+  if (handle->flags & UV_HANDLE_IPV6)
+    return WSAEOPNOTSUPP;
+
+  if (WSAIoctl(socket,
+               SIO_LOOPBACK_FAST_PATH,
+               &enable,
+               sizeof(enable),
+               NULL,
+               0,
+               &bytes,
+               NULL,
+               NULL) != 0) {
+    return WSAGetLastError();
+  }
+
+  return 0;
+}
+
+
 static int uv_tcp_set_socket(uv_loop_t* loop,
                              uv_tcp_t* handle,
                              SOCKET socket,
@@ -150,6 +201,9 @@ static int uv_tcp_set_socket(uv_loop_t* loop,
     assert(!(handle->flags & UV_HANDLE_IPV6));
   }
 
+  if (fastpath_default_enable && fastpath_supported)
+      handle->flags |= UV_HANDLE_TCP_LOOPBACK_FASTPATH;
+
   return 0;
 }
 
@@ -199,6 +253,9 @@ int uv_tcp_init_ex(uv_loop_t* loop, uv_tcp_t* handle, unsigned int flags) {
 
   }
 
+  /* Check if fastpath is supported */
+  uv_once(&fastpath_unsupported_guard, uv__init_fastpath_support);
+
   return 0;
 }
 
@@ -583,6 +640,13 @@ int uv_tcp_listen(uv_tcp_t* handle, int backlog, uv_connection_cb cb) {
       return handle->delayed_error;
   }
 
+  if (handle->flags & UV_HANDLE_TCP_LOOPBACK_FASTPATH) {
+    uv__tcp_fastpath(handle, 1);
+
+    /* ignore errors while enabling fast path as it only gives a performance boost */
+    /* and has no effects on failure */
+  }
+
   if (!handle->tcp.serv.func_acceptex) {
     if (!uv_get_acceptex_function(handle->socket, &handle->tcp.serv.func_acceptex)) {
       return WSAEAFNOSUPPORT;
@@ -771,6 +835,17 @@ static int uv_tcp_try_connect(uv_connect_t* req,
       return handle->delayed_error;
   }
 
+  if (handle->flags & UV_HANDLE_TCP_LOOPBACK_FASTPATH) {
+    /* Only enable fastpath when connecting to a loopback address (ignore IPv6 as it can cause crashes) */
+    if (addrlen == sizeof(uv_addr_ip4_any_)) {
+        const struct sockaddr_in *addr4 = (const struct sockaddr_in*)addr;
+
+        /* loopback in IPv4 is 127.0.0.0/8 */
+        if (addr4->sin_addr.s_net == 127)
+          uv__tcp_fastpath(handle, 1);
+    }
+  }
+
   if (!handle->tcp.conn.func_connectex) {
     if (!uv_get_connectex_function(handle->socket, &handle->tcp.conn.func_connectex)) {
       return WSAEAFNOSUPPORT;
@@ -1269,6 +1344,20 @@ int uv_tcp_keepalive(uv_tcp_t* handle, int enable, unsigned int delay) {
 }
 
 
+int uv_tcp_fastpath(uv_tcp_t* handle, int enable) {
+  if (!fastpath_supported)
+    return UV_ENOSYS;
+
+  if (enable) {
+    handle->flags |= UV_HANDLE_TCP_LOOPBACK_FASTPATH;
+  } else {
+    handle->flags &= ~UV_HANDLE_TCP_LOOPBACK_FASTPATH;
+  }
+
+  return 0;
+}
+
+
 int uv_tcp_duplicate_socket(uv_tcp_t* handle, int pid,
     LPWSAPROTOCOL_INFOW protocol_info) {
   if (!(handle->flags & UV_HANDLE_CONNECTION)) {

src/win/winapi.c

@@ -27,6 +27,7 @@
 
 /* Ntdll function pointers */
 sRtlNtStatusToDosError pRtlNtStatusToDosError;
+sRtlGetVersion pRtlGetVersion;
 sNtDeviceIoControlFile pNtDeviceIoControlFile;
 sNtQueryInformationFile pNtQueryInformationFile;
 sNtSetInformationFile pNtSetInformationFile;
@@ -65,6 +66,13 @@ void uv_winapi_init() {
     uv_fatal_error(GetLastError(), "GetProcAddress");
   }
 
+  pRtlGetVersion = (sRtlGetVersion) GetProcAddress(
+      ntdll_module,
+      "RtlGetVersion");
+  if (pRtlGetVersion == NULL) {
+    uv_fatal_error(GetLastError(), "GetProcAddress");
+  }
+
   pNtDeviceIoControlFile = (sNtDeviceIoControlFile) GetProcAddress(
       ntdll_module,
       "NtDeviceIoControlFile");

src/win/winapi.h

@@ -4507,6 +4507,10 @@ typedef struct _SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION {
 # define IO_REPARSE_TAG_SYMLINK (0xA000000CL)
 #endif
 
+#if defined(__MINGW32__) && !defined(__MINGW64_VERSION_MAJOR)
+  typedef struct _OSVERSIONINFOW RTL_OSVERSIONINFOW, *PRTL_OSVERSIONINFOW;
+#endif
+
 typedef VOID (NTAPI *PIO_APC_ROUTINE)
              (PVOID ApcContext,
               PIO_STATUS_BLOCK IoStatusBlock,
@@ -4515,6 +4519,9 @@ typedef VOID (NTAPI *PIO_APC_ROUTINE)
 typedef ULONG (NTAPI *sRtlNtStatusToDosError)
               (NTSTATUS Status);
 
+typedef NTSTATUS (NTAPI *sRtlGetVersion)
+                 (PRTL_OSVERSIONINFOW lpVersionInformation);
+
 typedef NTSTATUS (NTAPI *sNtDeviceIoControlFile)
                  (HANDLE FileHandle,
                   HANDLE Event,
@@ -4686,6 +4693,7 @@ typedef DWORD (WINAPI* sGetFinalPathNameByHandleW)
 
 /* Ntdll function pointers */
 extern sRtlNtStatusToDosError pRtlNtStatusToDosError;
+extern sRtlGetVersion pRtlGetVersion;
 extern sNtDeviceIoControlFile pNtDeviceIoControlFile;
 extern sNtQueryInformationFile pNtQueryInformationFile;
 extern sNtSetInformationFile pNtSetInformationFile;

src/win/winsock.h

@@ -54,6 +54,10 @@
 # define SIO_BASE_HANDLE 0x48000022
 #endif
 
+#ifndef SIO_LOOPBACK_FAST_PATH
+# define SIO_LOOPBACK_FAST_PATH 0x98000010
+#endif
+
 /*
  * TDI defines that are only in the DDK.
  * We only need receive flags so far.

test/test-tcp-flags.c

@@ -42,6 +42,13 @@ TEST_IMPL(tcp_flags) {
   r = uv_tcp_keepalive(&handle, 1, 60);
   ASSERT(r == 0);
 
+  r = uv_tcp_fastpath(&handle, 1);
+#ifdef WIN32
+  ASSERT(r == 0);
+#else
+  ASSERT(r == UV_ENOSYS);
+#endif
+
   uv_close((uv_handle_t*)&handle, NULL);
 
   r = uv_run(loop, UV_RUN_DEFAULT);