From 44e2cf83a6e76b19cfe73eeae7e28386f6b27851 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Fri, 21 Feb 2025 14:02:46 -0800 Subject: [PATCH] Update the io_uring module for Linux changes Add new opcodes `ReadMultishot`, `Waitid`, `FixedFdInstall`, `Ftruncate`, `Bind`, and `Listen`, and add support for `IoringFeatureFlags::RECVSEND_BUNDLE`. Also, use `io_uring_ptr` and `io_uring_user_data` instead of `u64` in more fields, to better preserve pointer provenance, and add more utility functions for working with `io_uring_ptr` and `io_uring_user_data`. --- src/io_uring/mod.rs | 199 ++++++++++++++++++++++++++++++++++--- tests/io_uring/register.rs | 2 +- 2 files changed, 184 insertions(+), 17 deletions(-) diff --git a/src/io_uring/mod.rs b/src/io_uring/mod.rs index e0fda2c32..99e919b27 100644 --- a/src/io_uring/mod.rs +++ b/src/io_uring/mod.rs @@ -29,7 +29,9 @@ mod bindgen_types; use crate::fd::{AsFd, BorrowedFd, OwnedFd, RawFd}; use crate::{backend, io}; use bindgen_types::*; +use core::cmp::Ordering; use core::ffi::c_void; +use core::hash::{Hash, Hasher}; use core::mem::MaybeUninit; use core::ptr::{null_mut, write_bytes}; use linux_raw_sys::net; @@ -428,6 +430,12 @@ pub enum IoringOp { /// `IORING_OP_SENDMSG_ZC` SendmsgZc = sys::io_uring_op::IORING_OP_SENDMSG_ZC as _, + /// `IORING_OP_READ_MULTISHOT` + ReadMultishot = sys::io_uring_op::IORING_OP_READ_MULTISHOT as _, + + /// `IORING_OP_WAITID` + Waitid = sys::io_uring_op::IORING_OP_WAITID as _, + /// `IORING_OP_FUTEX_WAIT` FutexWait = sys::io_uring_op::IORING_OP_FUTEX_WAIT as _, @@ -436,6 +444,18 @@ pub enum IoringOp { /// `IORING_OP_FUTEX_WAITV` FutexWaitv = sys::io_uring_op::IORING_OP_FUTEX_WAITV as _, + + /// `IORING_OP_FIXED_FD_INSTALL` + FixedFdInstall = sys::io_uring_op::IORING_OP_FIXED_FD_INSTALL as _, + + /// `IORING_OP_FTRUNCATE` + Ftruncate = sys::io_uring_op::IORING_OP_FTRUNCATE as _, + + /// `IORING_OP_BIND` + Bind = sys::io_uring_op::IORING_OP_BIND as _, + + /// `IORING_OP_LISTEN` + Listen = sys::io_uring_op::IORING_OP_LISTEN as _, } impl Default for IoringOp { @@ -714,6 +734,19 @@ bitflags::bitflags! { } } +bitflags::bitflags! { + /// `IORING_FIXED_FD_*` flags for use with [`io_uring_sqe`]. + #[repr(transparent)] + #[derive(Default, Copy, Clone, Eq, PartialEq, Hash, Debug)] + pub struct IoringFixedFdFlags: u32 { + /// `IORING_FIXED_FD_NO_CLOEXEC` + const NO_CLOEXEC = sys::IORING_FIXED_FD_NO_CLOEXEC; + + /// + const _ = !0; + } +} + bitflags::bitflags! { /// `IORING_FEAT_*` flags for use with [`io_uring_params`]. #[repr(transparent)] @@ -761,6 +794,9 @@ bitflags::bitflags! { /// `IORING_FEAT_REG_REG_RING` const REG_REG_RING = sys::IORING_FEAT_REG_REG_RING; + /// `IORING_FEAT_RECVSEND_BUNDLE` + const RECVSEND_BUNDLE = sys::IORING_FEAT_RECVSEND_BUNDLE; + /// const _ = !0; } @@ -864,6 +900,11 @@ bitflags::bitflags! { /// `IORING_SEND_ZC_REPORT_USAGE` (since Linux 6.2) const ZC_REPORT_USAGE = sys::IORING_SEND_ZC_REPORT_USAGE as _; + /// `IORING_RECVSEND_BUNDLE` + /// + /// See also [`IoringRecvFlags::BUNDLE`]. + const BUNDLE = sys::IORING_RECVSEND_BUNDLE as _; + /// const _ = !0; } @@ -887,6 +928,11 @@ bitflags::bitflags! { /// See also [`IoringSendFlags::FIXED_BUF`]. const FIXED_BUF = sys::IORING_RECVSEND_FIXED_BUF as _; + /// `IORING_RECVSEND_BUNDLE` + /// + /// See also [`IoringSendFlags::BUNDLE`]. + const BUNDLE = sys::IORING_RECVSEND_BUNDLE as _; + /// const _ = !0; } @@ -981,24 +1027,74 @@ pub struct io_uring_ptr { pub __pad32: u32, } -impl From<*mut c_void> for io_uring_ptr { +impl io_uring_ptr { + /// Construct a null `io_uring_ptr`. #[inline] - fn from(ptr: *mut c_void) -> Self { + pub const fn null() -> Self { + Self::new(null_mut()) + } + + /// Construct a new `io_uring_ptr`. + #[inline] + pub const fn new(ptr: *mut c_void) -> Self { Self { ptr, #[cfg(target_pointer_width = "16")] - __pad16: Default::default(), + __pad16: 0, #[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))] - __pad32: Default::default(), + __pad32: 0, } } } +impl From<*mut c_void> for io_uring_ptr { + #[inline] + fn from(ptr: *mut c_void) -> Self { + Self::new(ptr) + } +} + +impl PartialEq for io_uring_ptr { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.ptr.eq(&other.ptr) + } +} + +impl Eq for io_uring_ptr {} + +impl PartialOrd for io_uring_ptr { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + self.ptr.partial_cmp(&other.ptr) + } +} + +impl Ord for io_uring_ptr { + #[inline] + fn cmp(&self, other: &Self) -> Ordering { + self.ptr.cmp(&other.ptr) + } +} + +impl Hash for io_uring_ptr { + #[inline] + fn hash(&self, state: &mut H) { + self.ptr.hash(state) + } +} + impl Default for io_uring_ptr { #[inline] fn default() -> Self { - Self::from(null_mut()) + Self::null() + } +} + +impl core::fmt::Pointer for io_uring_ptr { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + self.ptr.fmt(f) } } @@ -1026,33 +1122,81 @@ pub union io_uring_user_data { impl io_uring_user_data { /// Return the `u64` value. #[inline] - pub fn u64_(self) -> u64 { + pub const fn u64_(self) -> u64 { // SAFETY: All the fields have the same underlying representation. unsafe { self.u64_ } } /// Create a `Self` from a `u64` value. #[inline] - pub fn from_u64(u64_: u64) -> Self { + pub const fn from_u64(u64_: u64) -> Self { Self { u64_ } } /// Return the `ptr` pointer value. #[inline] - pub fn ptr(self) -> *mut c_void { + pub const fn ptr(self) -> *mut c_void { // SAFETY: All the fields have the same underlying representation. unsafe { self.ptr }.ptr } /// Create a `Self` from a pointer value. #[inline] - pub fn from_ptr(ptr: *mut c_void) -> Self { + pub const fn from_ptr(ptr: *mut c_void) -> Self { Self { - ptr: io_uring_ptr::from(ptr), + ptr: io_uring_ptr::new(ptr), } } } +impl From for io_uring_user_data { + #[inline] + fn from(u64_: u64) -> Self { + Self::from_u64(u64_) + } +} + +impl From<*mut c_void> for io_uring_user_data { + #[inline] + fn from(ptr: *mut c_void) -> Self { + Self::from_ptr(ptr) + } +} + +impl PartialEq for io_uring_user_data { + #[inline] + fn eq(&self, other: &Self) -> bool { + // SAFETY: `io_uring_ptr` and `u64` have the same layout. + unsafe { self.u64_.eq(&other.u64_) } + } +} + +impl Eq for io_uring_user_data {} + +impl PartialOrd for io_uring_user_data { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + // SAFETY: `io_uring_ptr` and `u64` have the same layout. + unsafe { self.u64_.partial_cmp(&other.u64_) } + } +} + +impl Ord for io_uring_user_data { + #[inline] + fn cmp(&self, other: &Self) -> Ordering { + // SAFETY: `io_uring_ptr` and `u64` have the same layout. + unsafe { self.u64_.cmp(&other.u64_) } + } +} + +impl Hash for io_uring_user_data { + #[inline] + fn hash(&self, state: &mut H) { + // SAFETY: `io_uring_ptr` and `u64` have the same layout. + unsafe { self.u64_.hash(state) } + } +} + impl Default for io_uring_user_data { #[inline] fn default() -> Self { @@ -1188,6 +1332,7 @@ pub union op_flags_union { pub msg_ring_flags: IoringMsgringFlags, pub uring_cmd_flags: IoringUringCmdFlags, pub futex_flags: FutexWaitvFlags, + pub install_fd_flags: IoringFixedFdFlags, } #[allow(missing_docs)] @@ -1222,7 +1367,7 @@ pub struct addr_len_struct { #[derive(Copy, Clone)] #[non_exhaustive] pub struct io_uring_sync_cancel_reg { - pub addr: u64, + pub addr: io_uring_user_data, pub fd: i32, pub flags: IoringAsyncCancelFlags, pub timeout: Timespec, @@ -1312,7 +1457,7 @@ pub struct io_sqring_offsets { pub dropped: u32, pub array: u32, pub resv1: u32, - pub user_addr: u64, + pub user_addr: io_uring_ptr, } #[allow(missing_docs)] @@ -1327,7 +1472,7 @@ pub struct io_cqring_offsets { pub cqes: u32, pub flags: u32, pub resv1: u32, - pub user_addr: u64, + pub user_addr: io_uring_ptr, } #[allow(missing_docs)] @@ -1437,7 +1582,7 @@ pub struct open_how { #[repr(C)] #[derive(Debug, Copy, Clone, Default)] pub struct io_uring_buf_reg { - pub ring_addr: u64, + pub ring_addr: io_uring_ptr, pub ring_entries: u32, pub bgid: u16, pub flags: u16, @@ -1448,7 +1593,7 @@ pub struct io_uring_buf_reg { #[repr(C)] #[derive(Debug, Copy, Clone, Default)] pub struct io_uring_buf { - pub addr: u64, + pub addr: io_uring_ptr, pub len: u32, pub bid: u16, pub resv: u16, @@ -1568,7 +1713,7 @@ mod tests { // io_uring stores them in a `u64`. unsafe { const MAGIC: u64 = !0x0123456789abcdef; - let ptr = io_uring_ptr::from(MAGIC as usize as *mut c_void); + let ptr = io_uring_ptr::new(MAGIC as usize as *mut c_void); assert_eq!(ptr.ptr, MAGIC as usize as *mut c_void); #[cfg(target_pointer_width = "16")] assert_eq!(ptr.__pad16, 0); @@ -1578,6 +1723,28 @@ mod tests { assert_eq!(int, MAGIC as usize as u64); } + // `io_uring_user_data` is a replacement for `u64`. + assert_eq_size!(io_uring_user_data, u64); + assert_eq_align!(io_uring_user_data, u64); + + // Test that `u64`s and pointers are properly stored in + // io_uring_user_data`. + unsafe { + const MAGIC: u64 = !0x0123456789abcdef; + let user_data = io_uring_user_data::from_u64(MAGIC); + assert_eq!(user_data.u64_(), MAGIC); + assert_eq!( + core::mem::transmute::(user_data), + MAGIC + ); + let user_data = io_uring_user_data::from_ptr(MAGIC as usize as *mut c_void); + assert_eq!(user_data.ptr(), MAGIC as usize as *mut c_void); + assert_eq!( + core::mem::transmute::(user_data), + MAGIC as usize as u64 + ); + } + check_renamed_type!(off_or_addr2_union, io_uring_sqe__bindgen_ty_1); check_renamed_type!(addr_or_splice_off_in_union, io_uring_sqe__bindgen_ty_2); check_renamed_type!(addr3_or_cmd_union, io_uring_sqe__bindgen_ty_6); diff --git a/tests/io_uring/register.rs b/tests/io_uring/register.rs index 643cbb60d..0af4f6365 100644 --- a/tests/io_uring/register.rs +++ b/tests/io_uring/register.rs @@ -158,7 +158,7 @@ fn io_uring_buf_ring_can_be_registered() { let br = unsafe { br_ptr.as_mut() }.expect("A valid io_uring_buf_ring struct"); let reg = io_uring_buf_reg { - ring_addr: br_ptr as u64, + ring_addr: br_ptr.cast::().into(), ring_entries: ENTRIES as u32, bgid: BGID, flags: 0,