epoll wakeup fixes

src/concurrency/thread.rs

@@ -515,10 +515,13 @@ impl<'tcx> ThreadManager<'tcx> {
         &mut self.threads[self.active_thread].stack
     }
 
-    pub fn all_stacks(
+    pub fn all_blocked_stacks(
         &self,
     ) -> impl Iterator<Item = (ThreadId, &[Frame<'tcx, Provenance, FrameExtra<'tcx>>])> {
-        self.threads.iter_enumerated().map(|(id, t)| (id, &t.stack[..]))
+        self.threads
+            .iter_enumerated()
+            .filter(|(_id, t)| matches!(t.state, ThreadState::Blocked { .. }))
+            .map(|(id, t)| (id, &t.stack[..]))
     }
 
     /// Create a new thread and returns its id.

src/diagnostics.rs

@@ -469,7 +469,7 @@ pub fn report_result<'tcx>(
     eprint!("{extra}"); // newlines are already in the string
 
     if show_all_threads {
-        for (thread, stack) in ecx.machine.threads.all_stacks() {
+        for (thread, stack) in ecx.machine.threads.all_blocked_stacks() {
             if thread != ecx.active_thread() {
                 let stacktrace = Frame::generate_stacktrace_from_stack(stack);
                 let (stacktrace, was_pruned) = prune_stacktrace(stacktrace, &ecx.machine);

src/shims/unix/linux_like/epoll.rs

@@ -1,5 +1,5 @@
 use std::cell::RefCell;
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, VecDeque};
 use std::io;
 use std::time::Duration;
 
@@ -22,8 +22,8 @@ struct Epoll {
     /// "ready" list; instead, a boolean flag in this list tracks which subset is ready. This makes
     /// `epoll_wait` less efficient, but also requires less bookkeeping.
     interest_list: RefCell<BTreeMap<EpollEventKey, EpollEventInterest>>,
-    /// A list of thread ids blocked on this epoll instance.
-    blocked: RefCell<Vec<ThreadId>>,
+    /// The queue of threads blocked on this epoll instance, and how many events they'd like to get.
+    queue: RefCell<VecDeque<(ThreadId, u32)>>,
 }
 
 impl VisitProvenance for Epoll {
@@ -459,7 +459,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 }
             };
             // Record this thread as blocked.
-            epfd.blocked.borrow_mut().push(this.active_thread());
+            epfd.queue
+                .borrow_mut()
+                .push_back((this.active_thread(), maxevents.try_into().unwrap()));
             // And block it.
             let dest = dest.clone();
             // We keep a strong ref to the underlying `Epoll` to make sure it sticks around.
@@ -477,14 +479,15 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                     |this, unblock: UnblockKind| {
                         match unblock {
                             UnblockKind::Ready => {
-                                return_ready_list(&epfd, &dest, &event, this)?;
+                                let events = return_ready_list(&epfd, &dest, &event, this)?;
+                                assert!(events > 0, "we got woken up with no events to deliver");
                                 interp_ok(())
                             },
                             UnblockKind::TimedOut => {
                                 // Remove the current active thread_id from the blocked thread_id list.
                                 epfd
-                                    .blocked.borrow_mut()
-                                    .retain(|&id| id != this.active_thread());
+                                    .queue.borrow_mut()
+                                    .retain(|&(id, _events)| id != this.active_thread());
                                 this.write_int(0, &dest)?;
                                 interp_ok(())
                             },
@@ -550,7 +553,7 @@ fn update_readiness<'tcx>(
         &mut dyn FnMut(&mut EpollEventInterest) -> InterpResult<'tcx>,
     ) -> InterpResult<'tcx>,
 ) -> InterpResult<'tcx> {
-    let mut wakeup = false;
+    let mut num_ready = 0u32; // how many events we have ready to deliver
     for_each_interest(&mut |interest| {
         // Update the ready events tracked in this interest.
         let new_readiness = interest.relevant_events & active_events;
@@ -565,19 +568,29 @@ fn update_readiness<'tcx>(
             ecx.release_clock(|clock| {
                 interest.clock.join(clock);
             })?;
-            wakeup = true;
+            num_ready = num_ready.saturating_add(1);
         }
         interp_ok(())
     })?;
-    if wakeup {
-        // Wake up threads that may have been waiting for events on this epoll.
-        // Do this only once for all the interests.
-        // Edge-triggered notification only notify one thread even if there are
-        // multiple threads blocked on the same epoll.
-        if let Some(thread_id) = epoll.blocked.borrow_mut().pop() {
-            ecx.unblock_thread(thread_id, BlockReason::Epoll)?;
+    // Edge-triggered notifications only wake up as many threads as are needed to deliver
+    // all the events.
+    while num_ready > 0
+        && let Some((thread_id, events)) = epoll.queue.borrow_mut().pop_front()
+    {
+        ecx.unblock_thread(thread_id, BlockReason::Epoll)?;
+        // Keep track of how many events we have left to deliver (except if we saturated;
+        // in that case we just wake up everybody).
+        if num_ready != u32::MAX {
+            num_ready = num_ready.saturating_sub(events);
         }
     }
+    // Sanity-check: if there are threads left to wake up, then there are no more ready events.
+    if !epoll.queue.borrow().is_empty() {
+        assert!(
+            epoll.interest_list.borrow().values().all(|i| !i.ready),
+            "there are unconsumed ready events and threads ready to take them"
+        );
+    }
 
     interp_ok(())
 }
@@ -589,7 +602,7 @@ fn return_ready_list<'tcx>(
     dest: &MPlaceTy<'tcx>,
     events: &MPlaceTy<'tcx>,
     ecx: &mut MiriInterpCx<'tcx>,
-) -> InterpResult<'tcx> {
+) -> InterpResult<'tcx, i32> {
     let mut interest_list = epfd.interest_list.borrow_mut();
     let mut num_of_events: i32 = 0;
     let mut array_iter = ecx.project_array_fields(events)?;
@@ -629,5 +642,5 @@ fn return_ready_list<'tcx>(
         }
     }
     ecx.write_int(num_of_events, dest)?;
-    interp_ok(())
+    interp_ok(num_of_events)
 }

tests/fail-dep/libc/eventfd_block_read_twice.rs

@@ -1,6 +1,4 @@
 //@only-target: linux android illumos
-//~^ERROR: deadlocked
-//~^^ERROR: deadlocked
 //@compile-flags: -Zmiri-deterministic-concurrency
 //@error-in-other-file: deadlock
 

tests/fail-dep/libc/eventfd_block_read_twice.stderr

@@ -14,24 +14,13 @@ note: inside `main`
 LL |     thread2.join().unwrap();
    |     ^^^^^^^^^^^^^^
 
-error: the evaluated program deadlocked
-   |
-   = note: this thread got stuck here
-   = note: (no span available)
-
 error: the evaluated program deadlocked
   --> tests/fail-dep/libc/eventfd_block_read_twice.rs:LL:CC
    |
 LL |         let res: i64 = unsafe { libc::read(fd, buf.as_mut_ptr().cast(), 8).try_into().unwrap() };
    |                                                                          ^ this thread got stuck here
 
-error: the evaluated program deadlocked
-   |
-   = note: this thread got stuck here
-   = note: (no span available)
-   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
-
 note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
 
-error: aborting due to 4 previous errors
+error: aborting due to 2 previous errors
 

tests/fail-dep/libc/eventfd_block_write_twice.rs

@@ -1,6 +1,4 @@
 //@only-target: linux android illumos
-//~^ERROR: deadlocked
-//~^^ERROR: deadlocked
 //@compile-flags: -Zmiri-deterministic-concurrency
 //@error-in-other-file: deadlock
 
@@ -38,7 +36,7 @@ fn main() {
 
     let thread2 = thread::spawn(move || {
         let sized_8_data = (u64::MAX - 1).to_ne_bytes();
-        // Write u64::MAX - 1, so the all subsequent write will block.
+        // Write u64::MAX - 1, so that all subsequent writes will block.
         let res: i64 = unsafe {
             // This `write` will initially blocked, then get unblocked by thread3, then get blocked again
             // because the `write` in thread1 executes first.

tests/fail-dep/libc/eventfd_block_write_twice.stderr

@@ -14,24 +14,13 @@ note: inside `main`
 LL |     thread2.join().unwrap();
    |     ^^^^^^^^^^^^^^
 
-error: the evaluated program deadlocked
-   |
-   = note: this thread got stuck here
-   = note: (no span available)
-
 error: the evaluated program deadlocked
   --> tests/fail-dep/libc/eventfd_block_write_twice.rs:LL:CC
    |
 LL |             libc::write(fd, sized_8_data.as_ptr() as *const libc::c_void, 8).try_into().unwrap()
    |                                                                            ^ this thread got stuck here
 
-error: the evaluated program deadlocked
-   |
-   = note: this thread got stuck here
-   = note: (no span available)
-   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
-
 note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
 
-error: aborting due to 4 previous errors
+error: aborting due to 2 previous errors
 

tests/fail-dep/libc/libc_epoll_block_two_thread.rs

@@ -1,11 +1,12 @@
 //@compile-flags: -Zmiri-deterministic-concurrency
-//~^ERROR: deadlocked
-//~^^ERROR: deadlocked
 //@only-target: linux android illumos
 //@error-in-other-file: deadlock
 
 use std::convert::TryInto;
-use std::thread::spawn;
+use std::thread;
+
+#[path = "../../utils/libc.rs"]
+mod libc_utils;
 
 // Using `as` cast since `EPOLLET` wraps around
 const EPOLL_IN_OUT_ET: u32 = (libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET) as _;
@@ -49,39 +50,37 @@ fn main() {
     let epfd = unsafe { libc::epoll_create1(0) };
     assert_ne!(epfd, -1);
 
-    // Create a socketpair instance.
-    let mut fds = [-1, -1];
-    let res = unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) };
-    assert_eq!(res, 0);
+    // Create an eventfd instance.
+    let flags = libc::EFD_NONBLOCK | libc::EFD_CLOEXEC;
+    let fd1 = unsafe { libc::eventfd(0, flags) };
+    // Make a duplicate so that we have two file descriptors for the same file description.
+    let fd2 = unsafe { libc::dup(fd1) };
 
-    // Register one side of the socketpair with epoll.
-    let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fds[0] as u64 };
-    let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fds[0], &mut ev) };
+    // Register both with epoll.
+    let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fd1 as u64 };
+    let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fd1, &mut ev) };
+    assert_eq!(res, 0);
+    let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fd2 as u64 };
+    let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fd2, &mut ev) };
     assert_eq!(res, 0);
 
-    // epoll_wait to clear notification.
-    let expected_event = u32::try_from(libc::EPOLLOUT).unwrap();
-    let expected_value = fds[0] as u64;
-    check_epoll_wait::<1>(epfd, &[(expected_event, expected_value)], 0);
+    // Consume the initial events.
+    let expected = [(libc::EPOLLOUT as u32, fd1 as u64), (libc::EPOLLOUT as u32, fd2 as u64)];
+    check_epoll_wait::<8>(epfd, &expected, -1);
 
-    let expected_event = u32::try_from(libc::EPOLLIN | libc::EPOLLOUT).unwrap();
-    let expected_value = fds[0] as u64;
-    let thread1 = spawn(move || {
-        check_epoll_wait::<1>(epfd, &[(expected_event, expected_value)], -1);
-        //~^ERROR: deadlocked
+    let thread1 = thread::spawn(move || {
+        check_epoll_wait::<2>(epfd, &expected, -1);
     });
-    let thread2 = spawn(move || {
-        check_epoll_wait::<1>(epfd, &[(expected_event, expected_value)], -1);
+    let thread2 = thread::spawn(move || {
+        check_epoll_wait::<2>(epfd, &expected, -1);
+        //~^ERROR: deadlocked
     });
+    // Yield so the threads are both blocked.
+    thread::yield_now();
 
-    let thread3 = spawn(move || {
-        // Just a single write, so we only wake up one of them.
-        let data = "abcde".as_bytes().as_ptr();
-        let res = unsafe { libc::write(fds[1], data as *const libc::c_void, 5) };
-        assert!(res > 0 && res <= 5);
-    });
+    // Create two events at once.
+    libc_utils::write_all_from_slice(fd1, &0_u64.to_ne_bytes()).unwrap();
 
     thread1.join().unwrap();
     thread2.join().unwrap();
-    thread3.join().unwrap();
 }

tests/fail-dep/libc/libc_epoll_block_two_thread.stderr

@@ -1,8 +1,3 @@
-error: the evaluated program deadlocked
-   |
-   = note: this thread got stuck here
-   = note: (no span available)
-
 error: the evaluated program deadlocked
   --> RUSTLIB/std/src/sys/thread/PLATFORM.rs:LL:CC
    |
@@ -16,22 +11,16 @@ LL |         let ret = unsafe { libc::pthread_join(id, ptr::null_mut()) };
 note: inside `main`
   --> tests/fail-dep/libc/libc_epoll_block_two_thread.rs:LL:CC
    |
-LL |     thread1.join().unwrap();
+LL |     thread2.join().unwrap();
    |     ^^^^^^^^^^^^^^
 
 error: the evaluated program deadlocked
   --> tests/fail-dep/libc/libc_epoll_block_two_thread.rs:LL:CC
    |
-LL |         check_epoll_wait::<TAG>(epfd, &[(expected_event, expected_value)], -1);
-   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ this thread got stuck here
-
-error: the evaluated program deadlocked
-   |
-   = note: this thread got stuck here
-   = note: (no span available)
-   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+LL |         check_epoll_wait::<TAG>(epfd, &expected, -1);
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ this thread got stuck here
 
 note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
 
-error: aborting due to 4 previous errors
+error: aborting due to 2 previous errors
 

tests/fail-dep/libc/socketpair-close-while-blocked.rs

@@ -1,6 +1,5 @@
 //! This is a regression test for <https://github.com/rust-lang/miri/issues/3947>: we had some
 //! faulty logic around `release_clock` that led to this code not reporting a data race.
-//~^^ERROR: deadlock
 //@ignore-target: windows # no libc socketpair on Windows
 //@compile-flags: -Zmiri-deterministic-concurrency
 //@error-in-other-file: deadlock

tests/fail-dep/libc/socketpair-close-while-blocked.stderr

@@ -20,12 +20,7 @@ error: the evaluated program deadlocked
 LL |             libc::read(fds[1], buf.as_mut_ptr().cast(), buf.len() as libc::size_t)
    |                                                                                  ^ this thread got stuck here
 
-error: the evaluated program deadlocked
-   |
-   = note: this thread got stuck here
-   = note: (no span available)
-
 note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
 
-error: aborting due to 3 previous errors
+error: aborting due to 2 previous errors
 

tests/fail-dep/libc/socketpair_block_read_twice.rs

@@ -1,6 +1,4 @@
 //@ignore-target: windows # No libc socketpair on Windows
-//~^ERROR: deadlocked
-//~^^ERROR: deadlocked
 // test_race depends on a deterministic schedule.
 //@compile-flags: -Zmiri-deterministic-concurrency
 //@error-in-other-file: deadlock

tests/fail-dep/libc/socketpair_block_read_twice.stderr

@@ -14,24 +14,13 @@ note: inside `main`
 LL |     thread2.join().unwrap();
    |     ^^^^^^^^^^^^^^
 
-error: the evaluated program deadlocked
-   |
-   = note: this thread got stuck here
-   = note: (no span available)
-
 error: the evaluated program deadlocked
   --> tests/fail-dep/libc/socketpair_block_read_twice.rs:LL:CC
    |
 LL |             libc::read(fds[1], buf.as_mut_ptr().cast(), buf.len() as libc::size_t)
    |                                                                                  ^ this thread got stuck here
 
-error: the evaluated program deadlocked
-   |
-   = note: this thread got stuck here
-   = note: (no span available)
-   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
-
 note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
 
-error: aborting due to 4 previous errors
+error: aborting due to 2 previous errors
 

tests/fail-dep/libc/socketpair_block_write_twice.rs

@@ -1,6 +1,4 @@
 //@ignore-target: windows # No libc socketpair on Windows
-//~^ERROR: deadlocked
-//~^^ERROR: deadlocked
 // test_race depends on a deterministic schedule.
 //@compile-flags: -Zmiri-deterministic-concurrency
 //@error-in-other-file: deadlock