Add io_uring support

.github/workflows/build-reuse-unix.yml

@@ -52,7 +52,7 @@ on:
         type: string
         # options:
         #   - quictls
-        #   - openssl 
+        #   - openssl
       static:
         required: false
         default: ''
@@ -73,6 +73,10 @@ on:
         required: false
         default: ''
         type: string
+      iouring:
+        required: false
+        default: ''
+        type: string
       sanitize:
         required: false
         default: ''
@@ -107,17 +111,17 @@ jobs:
     - name: Build For Test
       if: inputs.build == '-Test'
       shell: pwsh
-      run: scripts/build.ps1 -Config ${{ inputs.config }} -Platform ${{ inputs.plat }} -Arch ${{ inputs.arch }} -Tls ${{ inputs.tls }} -DisablePerf ${{ inputs.static }} ${{ inputs.clang }} ${{ inputs.systemcrypto }} ${{ inputs.codecheck }} ${{ inputs.sanitize }} ${{ inputs.xdp }} -OneBranch
+      run: scripts/build.ps1 -Config ${{ inputs.config }} -Platform ${{ inputs.plat }} -Arch ${{ inputs.arch }} -Tls ${{ inputs.tls }} -DisablePerf ${{ inputs.static }} ${{ inputs.clang }} ${{ inputs.systemcrypto }} ${{ inputs.codecheck }} ${{ inputs.sanitize }} ${{ inputs.xdp }} ${{ inputs.iouring }} -OneBranch
     - name: Build For Perf
       if: inputs.build == '-Perf'
       shell: pwsh
-      run: scripts/build.ps1 -Config ${{ inputs.config }} -Platform ${{ inputs.plat }} -Arch ${{ inputs.arch }} -Tls ${{ inputs.tls }} -DisableTools -DisableTest ${{ inputs.static }} ${{ inputs.clang }} ${{ inputs.systemcrypto }} ${{ inputs.codecheck }} ${{ inputs.sanitize }} ${{ inputs.xdp }}
+      run: scripts/build.ps1 -Config ${{ inputs.config }} -Platform ${{ inputs.plat }} -Arch ${{ inputs.arch }} -Tls ${{ inputs.tls }} -DisableTools -DisableTest ${{ inputs.static }} ${{ inputs.clang }} ${{ inputs.systemcrypto }} ${{ inputs.codecheck }} ${{ inputs.sanitize }} ${{ inputs.xdp }} ${{ inputs.iouring }}
     - name: Build
       if: inputs.build == ''
       shell: pwsh
-      run: scripts/build.ps1 -Config ${{ inputs.config }} -Platform ${{ inputs.plat }} -Arch ${{ inputs.arch }} -Tls ${{ inputs.tls }} ${{ inputs.static }} ${{ inputs.clang }} ${{ inputs.systemcrypto }} ${{ inputs.codecheck }} ${{ inputs.sanitize }} ${{ inputs.xdp }} -OneBranch
+      run: scripts/build.ps1 -Config ${{ inputs.config }} -Platform ${{ inputs.plat }} -Arch ${{ inputs.arch }} -Tls ${{ inputs.tls }} ${{ inputs.static }} ${{ inputs.clang }} ${{ inputs.systemcrypto }} ${{ inputs.codecheck }} ${{ inputs.sanitize }} ${{ inputs.xdp }} ${{ inputs.iouring }} -OneBranch
     - name: Upload build artifacts
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
       with:
-        name: ${{ inputs.config }}-${{ inputs.plat }}-${{ inputs.os }}-${{ inputs.arch }}-${{ inputs.tls }}${{ inputs.static }}${{ inputs.clang }}${{ inputs.systemcrypto }}${{ inputs.codecheck }}${{ inputs.sanitize }}${{ inputs.xdp }}${{ inputs.build }}
+        name: ${{ inputs.config }}-${{ inputs.plat }}-${{ inputs.os }}-${{ inputs.arch }}-${{ inputs.tls }}${{ inputs.static }}${{ inputs.clang }}${{ inputs.systemcrypto }}${{ inputs.codecheck }}${{ inputs.sanitize }}${{ inputs.xdp }}${{ inputs.iouring }}${{ inputs.build }}
         path: artifacts

.github/workflows/build.yml

@@ -129,6 +129,8 @@ jobs:
         clang: ['', '-Clang']
         codecheck: ['', '-CodeCheck']
         xdp: ['', '-UseXdp']
+        iouring: ['', '-UseIoUring']
+
         exclude:
         # Android doesn't support x86, XDP, Clang, CodeCheck or SystemCrypto
         - plat: android
@@ -158,6 +160,9 @@ jobs:
           clang: '-Clang'
         - plat: android
           codecheck: '-CodeCheck'
+        # linux io_uring is for ubuntu24.04 only for now
+        - plat: android
+          iouring: "-UseIoUring"
         # CodeCheck doesn't work with SystemCrypto, Clang, Static or Release builds
         - codecheck: '-CodeCheck'
           systemcrypto: '-UseSystemOpenSSLCrypto'
@@ -180,6 +185,14 @@ jobs:
           os: 'ubuntu-22.04'
         - xdp: "-UseXdp"
           arch: x86
+        # Linux io_uring is for x64 ubuntu24.04 only for now
+        - iouring: "-UseIoUring"
+          os: 'ubuntu-22.04'
+        - iouring: "-UseIoUring"
+          arch: x86
+        # Linux io_uring is incompatible with XDP for now
+        - iouring: "-UseIoUring"
+          xdp: "-UseXdp"
     uses: ./.github/workflows/build-reuse-unix.yml
     with:
       config: ${{ matrix.config }}
@@ -192,6 +205,7 @@ jobs:
       clang: ${{ matrix.clang }}
       codecheck: ${{ matrix.codecheck }}
       xdp: ${{ matrix.xdp }}
+      iouring: ${{ matrix.iouring }}
       repo: ${{ github.repository }}
 
   build-darwin:

.github/workflows/stress.yml

@@ -52,11 +52,12 @@ jobs:
           { config: "Debug", plat: "linux", os: "ubuntu-22.04", arch: "x64", tls: "quictls", sanitize: "-Sanitize", build: "-Test" },
           { config: "Debug", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", sanitize: "-Sanitize", build: "-Test" },
           { config: "Debug", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", build: "-Test", xdp: "-UseXdp" },
+          { config: "Debug", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", build: "-Test", iouring: "-UseIoUring" },
           { config: "Debug", plat: "macos", os: "macos-13", arch: "x64", tls: "openssl", build: "-Test" },
           { config: "Debug", plat: "linux", os: "ubuntu-22.04", arch: "x64", tls: "openssl", sanitize: "-Sanitize", build: "-Test" },
           { config: "Debug", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "openssl", sanitize: "-Sanitize", build: "-Test" },
           { config: "Debug", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "openssl", build: "-Test", xdp: "-UseXdp" },
-
+          { config: "Debug", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "openssl", build: "-Test", iouring: "-UseIoUring" },
         ]
     uses: ./.github/workflows/build-reuse-unix.yml
     with:
@@ -69,6 +70,7 @@ jobs:
       sanitize: ${{ matrix.vec.sanitize }}
       build: ${{ matrix.vec.build }}
       xdp: ${{ matrix.vec.xdp }}
+      iouring: ${{ matrix.vec.iouring }}
       repo: ${{ github.repository }}
 
   stress:
@@ -86,7 +88,7 @@ jobs:
           { config: "Debug", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "openssl", sanitize: "-Sanitize", build: "-Test"  },
           { config: "Debug", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "openssl", build: "-Test", xdp: "-UseXdp"  },
           { config: "Debug", plat: "macos", os: "macos-13", arch: "x64", tls: "openssl", build: "-Test" },
-         
+
           { config: "Debug", plat: "windows", os: "windows-2022", arch: "x64", tls: "schannel", sanitize: "-Sanitize", build: "-Test" },
           { config: "Debug", plat: "windows", os: "windows-2022", arch: "x64", tls: "schannel", xdp: "-UseXdp", sanitize: "-Sanitize", build: "-Test" },
           { config: "Debug", plat: "windows", os: "windows-2022", arch: "x64", tls: "quictls", build: "-Test" },
@@ -197,4 +199,3 @@ jobs:
       with:
         name: recvfuzz-${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.xdp }}${{ matrix.vec.sanitize }}
         path: artifacts
-

.github/workflows/test.yml

@@ -105,6 +105,8 @@ jobs:
           { config: "Release", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", build: "-Test" },
           { config: "Debug", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", build: "-Test", xdp: "-UseXdp" },
           { config: "Release", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", build: "-Test", xdp: "-UseXdp" },
+          { config: "Debug", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", build: "-Test", iouring: "-UseIoUring" },
+          { config: "Release", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", build: "-Test", iouring: "-UseIoUring" },
           { config: "Debug", plat: "linux", os: "ubuntu-22.04", arch: "x64", tls: "openssl", sanitize: "-Sanitize", build: "-Test" },
           { config: "Debug", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "openssl", sanitize: "-Sanitize", build: "-Test" },
         ]
@@ -119,6 +121,7 @@ jobs:
       sanitize: ${{ matrix.vec.sanitize }}
       build: ${{ matrix.vec.build }}
       xdp: ${{ matrix.vec.xdp }}
+      iouring: ${{ matrix.vec.iouring }}
       ref: ${{ inputs.ref || '' }}
       repo: ${{ github.repository }}
 
@@ -136,6 +139,8 @@ jobs:
           { config: "Debug", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", build: "-Test", xdp: "-UseXdp"  },
           { config: "Release", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", build: "-Test"  },
           { config: "Release", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", build: "-Test", xdp: "-UseXdp"  },
+          { config: "Debug", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", build: "-Test", iouring: "-UseIoUring"  },
+          { config: "Release", plat: "linux", os: "ubuntu-24.04", arch: "x64", tls: "quictls", systemcrypto: "-UseSystemOpenSSLCrypto", build: "-Test", iouring: "-UseIoUring"  },
           { config: "Debug", plat: "linux", os: "ubuntu-22.04", arch: "x64", tls: "openssl", sanitize: "-Sanitize", build: "-Test"  },
           { config: "Debug", plat: "windows", os: "windows-2022", arch: "x64", tls: "schannel", sanitize: "-Sanitize", build: "-Test" },
           { config: "Debug", plat: "windows", os: "windows-2022", arch: "x64", tls: "schannel", xdp: "-UseXdp", sanitize: "-Sanitize", build: "-Test" },
@@ -172,7 +177,7 @@ jobs:
       uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
       if: matrix.vec.plat == 'linux'
       with:
-        name: ${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.systemcrypto }}${{ matrix.vec.sanitize }}${{ matrix.vec.xdp }}${{ matrix.vec.build }}
+        name: ${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.systemcrypto }}${{ matrix.vec.sanitize }}${{ matrix.vec.xdp }}${{ matrix.vec.iouring }}${{ matrix.vec.build }}
         path: artifacts
     - name: Fix permissions for Unix
       if: matrix.vec.plat == 'linux' || matrix.vec.plat == 'macos'
@@ -207,7 +212,7 @@ jobs:
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
       if: failure() || cancelled()
       with:
-        name: BVT-${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.xdp }}${{ matrix.vec.qtip }}${{ matrix.vec.systemcrypto }}${{ matrix.vec.sanitize }}
+        name: BVT-${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.xdp }}${{ matrix.vec.iouring }}${{ matrix.vec.qtip }}${{ matrix.vec.systemcrypto }}${{ matrix.vec.sanitize }}
         path: artifacts
 
   bvt-kernel:

CMakeLists.txt

@@ -94,6 +94,7 @@ option(QUIC_UWP_BUILD "Build for UWP" OFF)
 option(QUIC_GAMECORE_BUILD "Build for GameCore" OFF)
 option(QUIC_EXTERNAL_TOOLCHAIN "Enable if system libs and include paths are configured by CMake toolchain" OFF)
 option(QUIC_PGO "Enables profile guided optimizations" OFF)
+option(QUIC_LINUX_IOURING_ENABLED "Enables io_uring support" OFF)
 option(QUIC_LINUX_XDP_ENABLED "Enables XDP support" OFF)
 option(QUIC_SOURCE_LINK "Enables source linking on MSVC" ON)
 option(QUIC_EMBED_GIT_HASH "Embed git commit hash in the binary" ON)
@@ -455,6 +456,15 @@ if (NOT MSVC AND NOT APPLE AND NOT ANDROID)
     endif()
 endif()
 
+if (QUIC_LINUX_IOURING_ENABLED)
+    find_library(LIBURING NAMES liburing-ffi.so)
+    if (LIBURING)
+        message(STATUS "Found liburing: ${LIBURING}")
+    else()
+        message(STATUS "liburing not found. If build fails, install liburing")
+    endif()
+endif()
+
 if (CMAKE_GENERATOR_PLATFORM STREQUAL "")
 string(TOLOWER ${CMAKE_SYSTEM_PROCESSOR} SYSTEM_PROCESSOR)
 else()
@@ -713,6 +723,10 @@ if (QUIC_USE_SYSTEM_LIBCRYPTO)
     list(APPEND QUIC_COMMON_DEFINES CXPLAT_SYSTEM_CRYPTO)
 endif()
 
+if (QUIC_LINUX_IOURING_ENABLED)
+    list(APPEND QUIC_COMMON_DEFINES CXPLAT_USE_IO_URING)
+endif()
+
 if(QUIC_CODE_CHECK)
     find_program(CLANGTIDY NAMES clang-tidy)
     if(CLANGTIDY)

scripts/build.ps1

@@ -60,6 +60,9 @@ This script provides helpers for building msquic.
 .PARAMETER UseXdp
     Enables XDP support (Linux-only).
 
+.PARAMETER UseIoUring
+    Enables io_uring support (Linux-only).
+
 .PARAMETER Generator
     Specifies a specific cmake generator (Only supported on unix)
 
@@ -174,6 +177,9 @@ param (
     [Parameter(Mandatory = $false)]
     [switch]$UseXdp = $false,
 
+    [Parameter(Mandatory = $false)]
+    [switch]$UseIoUring = $false,
+
     [Parameter(Mandatory = $false)]
     [string]$Generator = "",
 
@@ -481,6 +487,9 @@ function CMake-Generate {
     if ($UseXdp) {
         $Arguments += " -DQUIC_LINUX_XDP_ENABLED=on"
     }
+    if ($UseIoUring) {
+        $Arguments += " -DQUIC_LINUX_IOURING_ENABLED=on"
+    }
     if ($Platform -eq "uwp") {
         $Arguments += " -DCMAKE_SYSTEM_NAME=WindowsStore -DCMAKE_SYSTEM_VERSION=10.0 -DQUIC_UWP_BUILD=on"
     }

scripts/clog.inputs

@@ -3,6 +3,7 @@
 ../src/platform/crypt.c
 ../src/platform/datapath_winkernel.c
 ../src/platform/datapath_epoll.c
+../src/platform/datapath_iouring.c
 ../src/platform/tls_schannel.c
 ../src/platform/selfsign_capi.c
 ../src/platform/cert_capi.c

scripts/prepare-machine.ps1

@@ -546,6 +546,7 @@ if ($IsLinux) {
         sudo apt-get install -y babeltrace
         sudo apt-get install -y libssl-dev
         sudo apt-get install -y libnuma-dev
+        sudo apt-get install -y liburing-dev
         if ($InstallArm64Toolchain) {
             sudo apt-get install -y gcc-aarch64-linux-gnu
             sudo apt-get install -y binutils-aarch64-linux-gnu
@@ -576,6 +577,7 @@ if ($IsLinux) {
         sudo apt-get install -y lttng-tools
         sudo apt-get install -y liblttng-ust-dev
         sudo apt-get install -y gdb
+        sudo apt-get install -y liburing2
         if ($UseXdp) {
             if (!$IsUbuntu2404) {
                 sudo apt-add-repository "deb http://mirrors.kernel.org/ubuntu noble main" -y

src/generated/CMakeLists.txt

@@ -23,13 +23,13 @@ if(QUIC_ENABLE_LOGGING)
         target_link_libraries(logging PRIVATE inc)
 
     elseif(QUIC_LOGGING_TYPE STREQUAL "lttng")
-        target_include_directories(logging_inc INTERFACE 
+        target_include_directories(logging_inc INTERFACE
             $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/common>
             $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
-        target_include_directories(logging_inc INTERFACE 
+        target_include_directories(logging_inc INTERFACE
             $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/linux>
             $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
-        target_include_directories(logging_inc INTERFACE 
+        target_include_directories(logging_inc INTERFACE
             $<BUILD_INTERFACE:${LTTNGUST_INCLUDE_DIRS}>
             $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
 

src/generated/linux/datapath_epoll.c.clog.h

@@ -69,12 +69,12 @@ tracepoint(CLOG_DATAPATH_EPOLL_C, AllocFailure , arg2, arg3);\
 // QuicTraceEvent(
             DatapathErrorStatus,
             "[data][%p] ERROR, %u, %s.",
-            SocketContext->Binding,
+            Binding,
             Status,
-            "setsockopt(SO_ATTACH_REUSEPORT_CBPF) failed");
-// arg2 = arg2 = SocketContext->Binding = arg2
+            "CxPlatSqeInitialize failed");
+// arg2 = arg2 = Binding = arg2
 // arg3 = arg3 = Status = arg3
-// arg4 = arg4 = "setsockopt(SO_ATTACH_REUSEPORT_CBPF) failed" = arg4
+// arg4 = arg4 = "CxPlatSqeInitialize failed" = arg4
 ----------------------------------------------------------*/
 #ifndef _clog_5_ARGS_TRACE_DatapathErrorStatus
 #define _clog_5_ARGS_TRACE_DatapathErrorStatus(uniqueId, encoded_arg_string, arg2, arg3, arg4)\

src/generated/linux/datapath_epoll.c.clog.h.lttng.h

@@ -49,12 +49,12 @@ TRACEPOINT_EVENT(CLOG_DATAPATH_EPOLL_C, AllocFailure,
 // QuicTraceEvent(
             DatapathErrorStatus,
             "[data][%p] ERROR, %u, %s.",
-            SocketContext->Binding,
+            Binding,
             Status,
-            "setsockopt(SO_ATTACH_REUSEPORT_CBPF) failed");
-// arg2 = arg2 = SocketContext->Binding = arg2
+            "CxPlatSqeInitialize failed");
+// arg2 = arg2 = Binding = arg2
 // arg3 = arg3 = Status = arg3
-// arg4 = arg4 = "setsockopt(SO_ATTACH_REUSEPORT_CBPF) failed" = arg4
+// arg4 = arg4 = "CxPlatSqeInitialize failed" = arg4
 ----------------------------------------------------------*/
 TRACEPOINT_EVENT(CLOG_DATAPATH_EPOLL_C, DatapathErrorStatus,
     TP_ARGS(