From 895958f8ba90f480b7d5b4dd52a10bbfc0c30f0a Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Tue, 12 Sep 2023 00:19:01 +0800 Subject: [PATCH 01/94] reduce the lock contention in task spawn. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The origin version: spawn_tasks_current_thread time: [654.87 ns 664.26 ns 672.13 ns] change: [-4.5814% -1.2511% +2.1604%] (p = 0.48 > 0.05) No change in performance detected. spawn_tasks_current_thread_parallel time: [537.55 ns 540.00 ns 542.20 ns] change: [-1.0603% -0.2591% +0.6283%] (p = 0.58 > 0.05) No change in performance detected. Found 1 outliers among 100 measurements (1.00%) 1 (1.00%) high severe spawn_tasks time: [1.0447 µs 1.0533 µs 1.0610 µs] change: [+0.9379% +2.2768% +3.7191%] (p = 0.00 < 0.05) Change within noise threshold. Found 3 outliers among 100 measurements (3.00%) 1 (1.00%) low severe 2 (2.00%) low mild spawn_tasks_parallel time: [992.38 ns 1.0002 µs 1.0073 µs] change: [+0.1973% +1.4194% +2.5819%] (p = 0.02 < 0.05) Change within noise threshold. Found 6 outliers among 100 measurements (6.00%) 1 (1.00%) low severe 2 (2.00%) low mild 1 (1.00%) high mild 2 (2.00%) high severe This version: spawn_tasks_current_thread time: [705.92 ns 724.31 ns 739.78 ns] spawn_tasks_current_thread_parallel time: [529.33 ns 531.00 ns 532.61 ns] Found 2 outliers among 100 measurements (2.00%) 1 (1.00%) high mild 1 (1.00%) high severe spawn_tasks time: [881.56 ns 892.21 ns 902.10 ns] Found 3 outliers among 100 measurements (3.00%) 2 (2.00%) low mild 1 (1.00%) high mild spawn_tasks_parallel time: [815.00 ns 819.87 ns 824.60 ns] Found 4 outliers among 100 measurements (4.00%) 2 (2.00%) high mild 2 (2.00%) high severe --- benches/Cargo.toml | 9 ++ benches/spawn_concurrent.rs | 93 ++++++++++++++ .../runtime/scheduler/current_thread/mod.rs | 2 +- .../runtime/scheduler/multi_thread/worker.rs | 2 +- tokio/src/runtime/task/core.rs | 6 + tokio/src/runtime/task/id.rs | 2 +- tokio/src/runtime/task/list.rs | 114 ++++++++++++------ tokio/src/util/linked_list.rs | 67 ---------- 8 files changed, 187 insertions(+), 108 deletions(-) create mode 100644 benches/spawn_concurrent.rs diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 47a830416ab..4daeb9da56c 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -12,6 +12,8 @@ tokio = { version = "1.5.0", path = "../tokio", features = ["full"] } bencher = "0.1.5" rand = "0.8" rand_chacha = "0.3" +criterion = "0.5.1" +num_cpus = "1.16.0" [dev-dependencies] tokio-util = { version = "0.7.0", path = "../tokio-util", features = ["full"] } @@ -84,3 +86,10 @@ harness = false name = "time_now" path = "time_now.rs" harness = false + +[[bench]] +name = "spawn_concurrent" +path = "spawn_concurrent.rs" +harness = false + + diff --git a/benches/spawn_concurrent.rs b/benches/spawn_concurrent.rs new file mode 100644 index 00000000000..ca8f586703e --- /dev/null +++ b/benches/spawn_concurrent.rs @@ -0,0 +1,93 @@ +use std::time::Instant; + +use criterion::*; + +fn spawn_tasks_current_thread(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + + c.bench_function("spawn_tasks_current_thread", move |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + runtime.block_on(async { + black_box(job(iters as usize, 1).await); + }); + start.elapsed() + }) + }); +} + +fn spawn_tasks_current_thread_parallel(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + + c.bench_function("spawn_tasks_current_thread_parallel", move |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + runtime.block_on(async { + black_box(job(iters as usize, num_cpus::get_physical() * 2).await); + }); + start.elapsed() + }) + }); +} + +fn spawn_tasks(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_multi_thread().build().unwrap(); + + c.bench_function("spawn_tasks", move |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + runtime.block_on(async { + black_box(job(iters as usize, 1).await); + }); + start.elapsed() + }) + }); +} + +fn spawn_tasks_parallel(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_multi_thread().build().unwrap(); + c.bench_function("spawn_tasks_parallel", move |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + runtime.block_on(async { + black_box(job(iters as usize, num_cpus::get_physical()).await); + }); + start.elapsed() + }) + }); +} + +async fn job(iters: usize, procs: usize) { + for _ in 0..procs { + let mut threads_handles = Vec::with_capacity(procs); + threads_handles.push(tokio::spawn(async move { + let mut thread_handles = Vec::with_capacity(iters / procs); + for _ in 0..iters / procs { + thread_handles.push(tokio::spawn(async { + let val = 1 + 1; + tokio::task::yield_now().await; + black_box(val) + })); + } + for handle in thread_handles { + handle.await.unwrap(); + } + })); + for handle in threads_handles { + handle.await.unwrap(); + } + } +} + +criterion_group!( + benches, + spawn_tasks_current_thread, + spawn_tasks_current_thread_parallel, + spawn_tasks, + spawn_tasks_parallel +); +criterion_main!(benches); diff --git a/tokio/src/runtime/scheduler/current_thread/mod.rs b/tokio/src/runtime/scheduler/current_thread/mod.rs index 30b17c0e8ed..3e57aa8a374 100644 --- a/tokio/src/runtime/scheduler/current_thread/mod.rs +++ b/tokio/src/runtime/scheduler/current_thread/mod.rs @@ -132,7 +132,7 @@ impl CurrentThread { let handle = Arc::new(Handle { shared: Shared { inject: Inject::new(), - owned: OwnedTasks::new(), + owned: OwnedTasks::new(1), woken: AtomicBool::new(false), config, scheduler_metrics: SchedulerMetrics::new(), diff --git a/tokio/src/runtime/scheduler/multi_thread/worker.rs b/tokio/src/runtime/scheduler/multi_thread/worker.rs index 8f3181ffcc3..418f6c9ad5a 100644 --- a/tokio/src/runtime/scheduler/multi_thread/worker.rs +++ b/tokio/src/runtime/scheduler/multi_thread/worker.rs @@ -287,7 +287,7 @@ pub(super) fn create( remotes: remotes.into_boxed_slice(), inject, idle, - owned: OwnedTasks::new(), + owned: OwnedTasks::new(16), synced: Mutex::new(Synced { idle: idle_synced, inject: inject_synced, diff --git a/tokio/src/runtime/task/core.rs b/tokio/src/runtime/task/core.rs index d62ea965659..12d5dbc0360 100644 --- a/tokio/src/runtime/task/core.rs +++ b/tokio/src/runtime/task/core.rs @@ -179,6 +179,9 @@ pub(crate) struct Header { /// The tracing ID for this instrumented task. #[cfg(all(tokio_unstable, feature = "tracing"))] pub(super) tracing_id: Option, + + /// The task's ID, is used for deciding which list to put it in. + pub(super) task_id: Id, } unsafe impl Send for Header {} @@ -216,12 +219,14 @@ impl Cell { fn new_header( state: State, vtable: &'static Vtable, + task_id: Id, #[cfg(all(tokio_unstable, feature = "tracing"))] tracing_id: Option, ) -> Header { Header { state, queue_next: UnsafeCell::new(None), vtable, + task_id, owner_id: UnsafeCell::new(None), #[cfg(all(tokio_unstable, feature = "tracing"))] tracing_id, @@ -235,6 +240,7 @@ impl Cell { header: new_header( state, vtable, + task_id, #[cfg(all(tokio_unstable, feature = "tracing"))] tracing_id, ), diff --git a/tokio/src/runtime/task/id.rs b/tokio/src/runtime/task/id.rs index 2b0d95c0243..dd5ae504582 100644 --- a/tokio/src/runtime/task/id.rs +++ b/tokio/src/runtime/task/id.rs @@ -24,7 +24,7 @@ use std::fmt; #[cfg_attr(docsrs, doc(cfg(all(feature = "rt", tokio_unstable))))] #[cfg_attr(not(tokio_unstable), allow(unreachable_pub))] #[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)] -pub struct Id(u64); +pub struct Id(pub(crate) u64); /// Returns the [`Id`] of the currently running task. /// diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 3a1fcce2ec4..373a47de8e0 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -8,12 +8,14 @@ use crate::future::Future; use crate::loom::cell::UnsafeCell; +use crate::loom::sync::atomic::AtomicUsize; use crate::loom::sync::Mutex; use crate::runtime::task::{JoinHandle, LocalNotified, Notified, Schedule, Task}; -use crate::util::linked_list::{CountedLinkedList, Link, LinkedList}; +use crate::util::linked_list::{Link, LinkedList}; use std::marker::PhantomData; use std::num::NonZeroU64; +use std::sync::atomic::AtomicBool; // The id from the module below is used to verify whether a given task is stored // in this OwnedTasks, or some other task. The counter starts at one so we can @@ -55,12 +57,14 @@ cfg_not_has_atomic_u64! { } pub(crate) struct OwnedTasks { - inner: Mutex>, + lists: Vec>>, pub(crate) id: NonZeroU64, + closed: AtomicBool, + grain: usize, + count: AtomicUsize, } struct CountedOwnedTasksInner { - list: CountedLinkedList, as Link>::Target>, - closed: bool, + list: LinkedList, as Link>::Target>, } pub(crate) struct LocalOwnedTasks { inner: UnsafeCell>, @@ -73,13 +77,25 @@ struct OwnedTasksInner { } impl OwnedTasks { - pub(crate) fn new() -> Self { + /// grain must be an integer power of 2 + pub(crate) fn new(grain: usize) -> Self { + assert_eq!( + grain & (grain - 1), + 0, + "grain must be an integer power of 2" + ); + let mut lists = Vec::with_capacity(grain); + for _ in 0..grain { + lists.push(Mutex::new(CountedOwnedTasksInner { + list: LinkedList::new(), + })) + } Self { - inner: Mutex::new(CountedOwnedTasksInner { - list: CountedLinkedList::new(), - closed: false, - }), + lists, + closed: AtomicBool::new(false), id: get_next_id(), + grain, + count: AtomicUsize::new(0), } } @@ -97,12 +113,17 @@ impl OwnedTasks { T::Output: Send + 'static, { let (task, notified, join) = super::new_task(task, scheduler, id); - let notified = unsafe { self.bind_inner(task, notified) }; + let notified = unsafe { self.bind_inner(task, notified, id) }; (join, notified) } /// The part of `bind` that's the same for every type of future. - unsafe fn bind_inner(&self, task: Task, notified: Notified) -> Option> + unsafe fn bind_inner( + &self, + task: Task, + notified: Notified, + id: super::Id, + ) -> Option> where S: Schedule, { @@ -111,17 +132,16 @@ impl OwnedTasks { // to the field. task.header().set_owner_id(self.id); } - - let mut lock = self.inner.lock(); - if lock.closed { - drop(lock); - drop(notified); + // check close flag + if self.closed.load(Ordering::Relaxed) { task.shutdown(); - None - } else { - lock.list.push_front(task); - Some(notified) + return None; } + + let mut lock = self.lists[id.0 as usize & (self.grain - 1)].lock(); + lock.list.push_front(task); + self.count.fetch_add(1, Ordering::Relaxed); + Some(notified) } /// Asserts that the given task is owned by this OwnedTasks and convert it to @@ -129,7 +149,6 @@ impl OwnedTasks { #[inline] pub(crate) fn assert_owner(&self, task: Notified) -> LocalNotified { debug_assert_eq!(task.header().get_owner_id(), Some(self.id)); - // safety: All tasks bound to this OwnedTasks are Send, so it is safe // to poll it on this thread no matter what thread we are on. LocalNotified { @@ -146,28 +165,35 @@ impl OwnedTasks { { // The first iteration of the loop was unrolled so it can set the // closed bool. - let first_task = { - let mut lock = self.inner.lock(); - lock.closed = true; - lock.list.pop_back() - }; - match first_task { - Some(task) => task.shutdown(), - None => return, - } + self.closed.fetch_and(true, Ordering::SeqCst); - loop { - let task = match self.inner.lock().list.pop_back() { - Some(task) => task, - None => return, + for inner in &self.lists { + let first_task = { + let mut lock = inner.lock(); + lock.list.pop_back() }; + match first_task { + Some(task) => { + self.count.fetch_sub(1, Ordering::Relaxed); + task.shutdown(); + } + None => return, + } - task.shutdown(); + loop { + match inner.lock().list.pop_back() { + Some(task) => { + self.count.fetch_sub(1, Ordering::Relaxed); + task.shutdown(); + } + None => return, + }; + } } } pub(crate) fn active_tasks_count(&self) -> usize { - self.inner.lock().list.count() + self.count.load(Ordering::Relaxed) } pub(crate) fn remove(&self, task: &Task) -> Option> { @@ -179,11 +205,23 @@ impl OwnedTasks { // safety: We just checked that the provided task is not in some other // linked list. - unsafe { self.inner.lock().list.remove(task.header_ptr()) } + unsafe { + match self.lists[(task.header().task_id.0) as usize & (self.grain - 1)] + .lock() + .list + .remove(task.header_ptr()) + { + Some(t) => { + self.count.fetch_sub(1, Ordering::Relaxed); + Some(t) + } + None => None, + } + } } pub(crate) fn is_empty(&self) -> bool { - self.inner.lock().list.is_empty() + self.count.load(Ordering::SeqCst) == 0 } } diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index 1f9bdf4b811..a4a1a42af9d 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -228,53 +228,6 @@ impl fmt::Debug for LinkedList { } } -// ===== impl CountedLinkedList ==== - -// Delegates operations to the base LinkedList implementation, and adds a counter to the elements -// in the list. -pub(crate) struct CountedLinkedList { - list: LinkedList, - count: usize, -} - -impl CountedLinkedList { - pub(crate) fn new() -> CountedLinkedList { - CountedLinkedList { - list: LinkedList::new(), - count: 0, - } - } - - pub(crate) fn push_front(&mut self, val: L::Handle) { - self.list.push_front(val); - self.count += 1; - } - - pub(crate) fn pop_back(&mut self) -> Option { - let val = self.list.pop_back(); - if val.is_some() { - self.count -= 1; - } - val - } - - pub(crate) fn is_empty(&self) -> bool { - self.list.is_empty() - } - - pub(crate) unsafe fn remove(&mut self, node: NonNull) -> Option { - let val = self.list.remove(node); - if val.is_some() { - self.count -= 1; - } - val - } - - pub(crate) fn count(&self) -> usize { - self.count - } -} - #[cfg(any( feature = "fs", feature = "rt", @@ -796,26 +749,6 @@ pub(crate) mod tests { } } - #[test] - fn count() { - let mut list = CountedLinkedList::<&Entry, <&Entry as Link>::Target>::new(); - assert_eq!(0, list.count()); - - let a = entry(5); - let b = entry(7); - list.push_front(a.as_ref()); - list.push_front(b.as_ref()); - assert_eq!(2, list.count()); - - list.pop_back(); - assert_eq!(1, list.count()); - - unsafe { - list.remove(ptr(&b)); - } - assert_eq!(0, list.count()); - } - /// This is a fuzz test. You run it by entering `cargo fuzz run fuzz_linked_list` in CLI in `/tokio/` module. #[cfg(fuzzing)] pub fn fuzz_linked_list(ops: &[u8]) { From 5ffbf018395c07d98c71f7ec7e63dfac407d6417 Mon Sep 17 00:00:00 2001 From: wathen Date: Tue, 12 Sep 2023 14:26:43 +0800 Subject: [PATCH 02/94] rm extra criterion in dependencies --- benches/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/benches/Cargo.toml b/benches/Cargo.toml index fcbafa254a1..8845d1ff1f2 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -12,7 +12,6 @@ tokio = { version = "1.5.0", path = "../tokio", features = ["full"] } criterion = "0.5.1" rand = "0.8" rand_chacha = "0.3" -criterion = "0.5.1" num_cpus = "1.16.0" [dev-dependencies] From 8e2c0b2a708493940dcfd2f1a2824559e8563faa Mon Sep 17 00:00:00 2001 From: wathen Date: Tue, 12 Sep 2023 14:31:35 +0800 Subject: [PATCH 03/94] restart a ci --- .DS_Store | Bin 0 -> 6148 bytes .../runtime/scheduler/multi_thread_alt/worker.rs | 2 +- tokio/src/runtime/tests/task.rs | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 Date: Tue, 12 Sep 2023 19:29:52 +0800 Subject: [PATCH 04/94] fix for each --- tokio/src/runtime/task/list.rs | 5 ++++- tokio/src/util/linked_list.rs | 13 ++----------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 373a47de8e0..d8ca1d5c023 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -232,7 +232,10 @@ cfg_taskdump! { where F: FnMut(&Task) { - self.inner.lock().list.for_each(f) + let mut f = f; + for list in &self.lists{ + f = list.lock().list.for_each(f); + } } } } diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index a4a1a42af9d..b39e662dd54 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -295,21 +295,11 @@ cfg_io_driver_impl! { } cfg_taskdump! { - impl CountedLinkedList { - pub(crate) fn for_each(&mut self, f: F) - where - F: FnMut(&T::Handle), - { - self.list.for_each(f) - } - } - impl LinkedList { - pub(crate) fn for_each(&mut self, mut f: F) + pub(crate) fn for_each(&mut self, mut f: F) -> F where F: FnMut(&T::Handle), { - use std::mem::ManuallyDrop; let mut next = self.head; @@ -320,6 +310,7 @@ cfg_taskdump! { next = T::pointers(curr).as_ref().get_next(); } } + f } } } From a500a7957249d874b53c441c19493eac59b4c730 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Tue, 12 Sep 2023 21:38:06 +0800 Subject: [PATCH 05/94] reduce the size of header --- tokio/src/runtime/id.rs | 8 +++- .../runtime/scheduler/current_thread/mod.rs | 4 +- .../runtime/scheduler/multi_thread/handle.rs | 4 +- .../scheduler/multi_thread_alt/handle.rs | 4 +- tokio/src/runtime/task/core.rs | 8 ++-- tokio/src/runtime/task/list.rs | 42 ++++++------------- 6 files changed, 29 insertions(+), 41 deletions(-) diff --git a/tokio/src/runtime/id.rs b/tokio/src/runtime/id.rs index 58551d49989..8c6df7fcefb 100644 --- a/tokio/src/runtime/id.rs +++ b/tokio/src/runtime/id.rs @@ -1,5 +1,5 @@ use std::fmt; -use std::num::NonZeroU64; +use std::num::{NonZeroU32, NonZeroU64}; /// An opaque ID that uniquely identifies a runtime relative to all other currently /// running runtimes. @@ -39,6 +39,12 @@ impl From for Id { } } +impl From for Id { + fn from(value: NonZeroU32) -> Self { + Id(value.into()) + } +} + impl fmt::Display for Id { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) diff --git a/tokio/src/runtime/scheduler/current_thread/mod.rs b/tokio/src/runtime/scheduler/current_thread/mod.rs index 3e57aa8a374..c913e6e0b35 100644 --- a/tokio/src/runtime/scheduler/current_thread/mod.rs +++ b/tokio/src/runtime/scheduler/current_thread/mod.rs @@ -546,10 +546,10 @@ cfg_metrics! { } cfg_unstable! { - use std::num::NonZeroU64; + use std::num::NonZeroU32; impl Handle { - pub(crate) fn owned_id(&self) -> NonZeroU64 { + pub(crate) fn owned_id(&self) -> NonZeroU32 { self.shared.owned.id } } diff --git a/tokio/src/runtime/scheduler/multi_thread/handle.rs b/tokio/src/runtime/scheduler/multi_thread/handle.rs index 568eb80af8b..957673d749f 100644 --- a/tokio/src/runtime/scheduler/multi_thread/handle.rs +++ b/tokio/src/runtime/scheduler/multi_thread/handle.rs @@ -60,10 +60,10 @@ impl Handle { } cfg_unstable! { - use std::num::NonZeroU64; + use std::num::NonZeroU32; impl Handle { - pub(crate) fn owned_id(&self) -> NonZeroU64 { + pub(crate) fn owned_id(&self) -> NonZeroU32 { self.shared.owned.id } } diff --git a/tokio/src/runtime/scheduler/multi_thread_alt/handle.rs b/tokio/src/runtime/scheduler/multi_thread_alt/handle.rs index d746bca1a18..b7288fe2302 100644 --- a/tokio/src/runtime/scheduler/multi_thread_alt/handle.rs +++ b/tokio/src/runtime/scheduler/multi_thread_alt/handle.rs @@ -59,10 +59,10 @@ impl Handle { } cfg_unstable! { - use std::num::NonZeroU64; + use std::num::NonZeroU32; impl Handle { - pub(crate) fn owned_id(&self) -> NonZeroU64 { + pub(crate) fn owned_id(&self) -> NonZeroU32 { self.shared.owned.id } } diff --git a/tokio/src/runtime/task/core.rs b/tokio/src/runtime/task/core.rs index 4c274e3a0e3..c198badf3d4 100644 --- a/tokio/src/runtime/task/core.rs +++ b/tokio/src/runtime/task/core.rs @@ -17,7 +17,7 @@ use crate::runtime::task::state::State; use crate::runtime::task::{Id, Schedule}; use crate::util::linked_list; -use std::num::NonZeroU64; +use std::num::NonZeroU32; use std::pin::Pin; use std::ptr::NonNull; use std::task::{Context, Poll, Waker}; @@ -168,7 +168,7 @@ pub(crate) struct Header { /// The id is not unset when removed from a list because we want to be able /// to read the id without synchronization, even if it is concurrently being /// removed from the list. - pub(super) owner_id: UnsafeCell>, + pub(super) owner_id: UnsafeCell>, /// The tracing ID for this instrumented task. #[cfg(all(tokio_unstable, feature = "tracing"))] @@ -397,11 +397,11 @@ impl Header { // safety: The caller must guarantee exclusive access to this field, and // must ensure that the id is either `None` or the id of the OwnedTasks // containing this task. - pub(super) unsafe fn set_owner_id(&self, owner: NonZeroU64) { + pub(super) unsafe fn set_owner_id(&self, owner: NonZeroU32) { self.owner_id.with_mut(|ptr| *ptr = Some(owner)); } - pub(super) fn get_owner_id(&self) -> Option { + pub(super) fn get_owner_id(&self) -> Option { // safety: If there are concurrent writes, then that write has violated // the safety requirements on `set_owner_id`. unsafe { self.owner_id.with(|ptr| *ptr) } diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index d8ca1d5c023..e1abcc1533a 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -14,8 +14,9 @@ use crate::runtime::task::{JoinHandle, LocalNotified, Notified, Schedule, Task}; use crate::util::linked_list::{Link, LinkedList}; use std::marker::PhantomData; -use std::num::NonZeroU64; +use std::num::NonZeroU32; use std::sync::atomic::AtomicBool; +use std::sync::atomic::{AtomicU32, Ordering}; // The id from the module below is used to verify whether a given task is stored // in this OwnedTasks, or some other task. The counter starts at one so we can @@ -26,39 +27,20 @@ use std::sync::atomic::AtomicBool; // bug in Tokio, so we accept that certain bugs would not be caught if the two // mixed up runtimes happen to have the same id. -cfg_has_atomic_u64! { - use std::sync::atomic::{AtomicU64, Ordering}; +static NEXT_OWNED_TASKS_ID: AtomicU32 = AtomicU32::new(1); - static NEXT_OWNED_TASKS_ID: AtomicU64 = AtomicU64::new(1); - - fn get_next_id() -> NonZeroU64 { - loop { - let id = NEXT_OWNED_TASKS_ID.fetch_add(1, Ordering::Relaxed); - if let Some(id) = NonZeroU64::new(id) { - return id; - } - } - } -} - -cfg_not_has_atomic_u64! { - use std::sync::atomic::{AtomicU32, Ordering}; - - static NEXT_OWNED_TASKS_ID: AtomicU32 = AtomicU32::new(1); - - fn get_next_id() -> NonZeroU64 { - loop { - let id = NEXT_OWNED_TASKS_ID.fetch_add(1, Ordering::Relaxed); - if let Some(id) = NonZeroU64::new(u64::from(id)) { - return id; - } +fn get_next_id() -> NonZeroU32 { + loop { + let id = NEXT_OWNED_TASKS_ID.fetch_add(1, Ordering::Relaxed); + if let Some(id) = NonZeroU32::new(id) { + return id; } } } pub(crate) struct OwnedTasks { lists: Vec>>, - pub(crate) id: NonZeroU64, + pub(crate) id: NonZeroU32, closed: AtomicBool, grain: usize, count: AtomicUsize, @@ -68,7 +50,7 @@ struct CountedOwnedTasksInner { } pub(crate) struct LocalOwnedTasks { inner: UnsafeCell>, - pub(crate) id: NonZeroU64, + pub(crate) id: NonZeroU32, _not_send_or_sync: PhantomData<*const ()>, } struct OwnedTasksInner { @@ -233,8 +215,8 @@ cfg_taskdump! { F: FnMut(&Task) { let mut f = f; - for list in &self.lists{ - f = list.lock().list.for_each(f); + for inner in &self.lists{ + f = inner.lock().list.for_each(f); } } } From 632a8d36996f4937741cd90f8a007970ad81b4df Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Wed, 13 Sep 2023 12:21:42 +0800 Subject: [PATCH 06/94] code refactor in list.rs --- tokio/src/runtime/task/list.rs | 67 +++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index e1abcc1533a..27c8d1c560e 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -64,7 +64,7 @@ impl OwnedTasks { assert_eq!( grain & (grain - 1), 0, - "grain must be an integer power of 2" + "the grain of OwnedTasks must be an integer power of 2" ); let mut lists = Vec::with_capacity(grain); for _ in 0..grain { @@ -119,11 +119,15 @@ impl OwnedTasks { task.shutdown(); return None; } + self.push_inner(id, task); + Some(notified) + } - let mut lock = self.lists[id.0 as usize & (self.grain - 1)].lock(); + #[inline] + fn push_inner(&self, task_id: super::Id, task: Task) { + let mut lock = self.lists[task_id.0 as usize & (self.grain - 1)].lock(); lock.list.push_front(task); self.count.fetch_add(1, Ordering::Relaxed); - Some(notified) } /// Asserts that the given task is owned by this OwnedTasks and convert it to @@ -149,28 +153,30 @@ impl OwnedTasks { // closed bool. self.closed.fetch_and(true, Ordering::SeqCst); - for inner in &self.lists { - let first_task = { - let mut lock = inner.lock(); - lock.list.pop_back() - }; + for i in 0..self.lists.len() { + let first_task = self.pop_back_inner(i); match first_task { - Some(task) => { - self.count.fetch_sub(1, Ordering::Relaxed); - task.shutdown(); - } + Some(task) => task.shutdown(), None => return, } - loop { - match inner.lock().list.pop_back() { - Some(task) => { - self.count.fetch_sub(1, Ordering::Relaxed); - task.shutdown(); - } + let task = match self.pop_back_inner(i) { + Some(task) => task, None => return, }; + task.shutdown(); + } + } + } + + fn pop_back_inner(&self, index: usize) -> Option> { + debug_assert!(index < self.lists.len()); + match self.lists[index].lock().list.pop_back() { + Some(task) => { + self.count.fetch_sub(1, Ordering::Relaxed); + Some(task) } + None => None, } } @@ -187,18 +193,21 @@ impl OwnedTasks { // safety: We just checked that the provided task is not in some other // linked list. - unsafe { - match self.lists[(task.header().task_id.0) as usize & (self.grain - 1)] - .lock() - .list - .remove(task.header_ptr()) - { - Some(t) => { - self.count.fetch_sub(1, Ordering::Relaxed); - Some(t) - } - None => None, + unsafe { self.remove_inner(task) } + } + + #[inline] + unsafe fn remove_inner(&self, task: &Task) -> Option> { + match self.lists[(task.header().task_id.0) as usize & (self.grain - 1)] + .lock() + .list + .remove(task.header_ptr()) + { + Some(task) => { + self.count.fetch_sub(1, Ordering::Relaxed); + Some(task) } + None => None, } } From 6453017547037b575b755462bdbaed454b58a9f1 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Wed, 13 Sep 2023 21:51:34 +0800 Subject: [PATCH 07/94] change ordering of atomic --- tokio/src/runtime/scheduler/current_thread/mod.rs | 1 + tokio/src/runtime/scheduler/multi_thread/worker.rs | 1 + tokio/src/runtime/task/list.rs | 10 +++++++--- tokio/src/runtime/tests/task.rs | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tokio/src/runtime/scheduler/current_thread/mod.rs b/tokio/src/runtime/scheduler/current_thread/mod.rs index c913e6e0b35..7d27a5c20e9 100644 --- a/tokio/src/runtime/scheduler/current_thread/mod.rs +++ b/tokio/src/runtime/scheduler/current_thread/mod.rs @@ -264,6 +264,7 @@ fn shutdown2(mut core: Box, handle: &Handle) -> Box { drop(task); } + assert!(handle.shared.owned.is_closed()); assert!(handle.shared.owned.is_empty()); // Submit metrics diff --git a/tokio/src/runtime/scheduler/multi_thread/worker.rs b/tokio/src/runtime/scheduler/multi_thread/worker.rs index 418f6c9ad5a..90503169cb3 100644 --- a/tokio/src/runtime/scheduler/multi_thread/worker.rs +++ b/tokio/src/runtime/scheduler/multi_thread/worker.rs @@ -1152,6 +1152,7 @@ impl Handle { return; } + debug_assert!(self.shared.owned.is_closed()); debug_assert!(self.shared.owned.is_empty()); for mut core in cores.drain(..) { diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 27c8d1c560e..66d6f29d57c 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -115,7 +115,7 @@ impl OwnedTasks { task.header().set_owner_id(self.id); } // check close flag - if self.closed.load(Ordering::Relaxed) { + if self.closed.load(Ordering::Acquire) { task.shutdown(); return None; } @@ -151,7 +151,7 @@ impl OwnedTasks { { // The first iteration of the loop was unrolled so it can set the // closed bool. - self.closed.fetch_and(true, Ordering::SeqCst); + self.closed.fetch_and(true, Ordering::Release); for i in 0..self.lists.len() { let first_task = self.pop_back_inner(i); @@ -211,8 +211,12 @@ impl OwnedTasks { } } + pub(crate) fn is_closed(&self) -> bool { + self.closed.load(Ordering::Acquire) + } + pub(crate) fn is_empty(&self) -> bool { - self.count.load(Ordering::SeqCst) == 0 + self.count.load(Ordering::Relaxed) == 0 } } diff --git a/tokio/src/runtime/tests/task.rs b/tokio/src/runtime/tests/task.rs index 4d77638a43b..b2d3ef8fe18 100644 --- a/tokio/src/runtime/tests/task.rs +++ b/tokio/src/runtime/tests/task.rs @@ -315,7 +315,7 @@ impl Runtime { } drop(core); - + assert!(self.0.owned.is_closed()); assert!(self.0.owned.is_empty()); } } From 9bfb4f19ce78c755ea808167cea45753000da502 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Wed, 13 Sep 2023 22:19:14 +0800 Subject: [PATCH 08/94] fix iterate in close_and_shutdown --- tokio/src/runtime/task/list.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 66d6f29d57c..291faebe3ec 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -154,15 +154,10 @@ impl OwnedTasks { self.closed.fetch_and(true, Ordering::Release); for i in 0..self.lists.len() { - let first_task = self.pop_back_inner(i); - match first_task { - Some(task) => task.shutdown(), - None => return, - } loop { let task = match self.pop_back_inner(i) { Some(task) => task, - None => return, + None => continue, }; task.shutdown(); } From b4ac885e5a09f570a29f0afbf436580ec3461c9f Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Wed, 13 Sep 2023 22:45:49 +0800 Subject: [PATCH 09/94] fix iterate in close_and_shutdown --- tokio/src/runtime/task/list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 291faebe3ec..b463e239b08 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -157,7 +157,7 @@ impl OwnedTasks { loop { let task = match self.pop_back_inner(i) { Some(task) => task, - None => continue, + None => break, }; task.shutdown(); } From 4b386c53530a9397ff33a1912ec6f01aad9c7382 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Thu, 14 Sep 2023 00:01:24 +0800 Subject: [PATCH 10/94] fix atomic method --- tokio/src/runtime/task/list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index b463e239b08..1269d1a3845 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -151,7 +151,7 @@ impl OwnedTasks { { // The first iteration of the loop was unrolled so it can set the // closed bool. - self.closed.fetch_and(true, Ordering::Release); + self.closed.store(true, Ordering::Release); for i in 0..self.lists.len() { loop { From 9ded74b29e337a0951d6e4ecf945e81d547e2d63 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Wed, 20 Sep 2023 11:45:40 +0800 Subject: [PATCH 11/94] rm type CountedOwnedTasksInner --- tokio/src/runtime/task/list.rs | 37 ++++++++++++---------------------- 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 1269d1a3845..cdae876f566 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -39,15 +39,15 @@ fn get_next_id() -> NonZeroU32 { } pub(crate) struct OwnedTasks { - lists: Vec>>, + lists: Vec>>, pub(crate) id: NonZeroU32, closed: AtomicBool, grain: usize, count: AtomicUsize, } -struct CountedOwnedTasksInner { - list: LinkedList, as Link>::Target>, -} + +type ListInner = LinkedList, as Link>::Target>; + pub(crate) struct LocalOwnedTasks { inner: UnsafeCell>, pub(crate) id: NonZeroU32, @@ -68,9 +68,7 @@ impl OwnedTasks { ); let mut lists = Vec::with_capacity(grain); for _ in 0..grain { - lists.push(Mutex::new(CountedOwnedTasksInner { - list: LinkedList::new(), - })) + lists.push(Mutex::new(LinkedList::new())) } Self { lists, @@ -126,7 +124,7 @@ impl OwnedTasks { #[inline] fn push_inner(&self, task_id: super::Id, task: Task) { let mut lock = self.lists[task_id.0 as usize & (self.grain - 1)].lock(); - lock.list.push_front(task); + lock.push_front(task); self.count.fetch_add(1, Ordering::Relaxed); } @@ -153,10 +151,13 @@ impl OwnedTasks { // closed bool. self.closed.store(true, Ordering::Release); - for i in 0..self.lists.len() { + for list in &self.lists { loop { - let task = match self.pop_back_inner(i) { - Some(task) => task, + let task = match list.lock().pop_back() { + Some(task) => { + self.count.fetch_sub(1, Ordering::Relaxed); + task + } None => break, }; task.shutdown(); @@ -164,17 +165,6 @@ impl OwnedTasks { } } - fn pop_back_inner(&self, index: usize) -> Option> { - debug_assert!(index < self.lists.len()); - match self.lists[index].lock().list.pop_back() { - Some(task) => { - self.count.fetch_sub(1, Ordering::Relaxed); - Some(task) - } - None => None, - } - } - pub(crate) fn active_tasks_count(&self) -> usize { self.count.load(Ordering::Relaxed) } @@ -195,7 +185,6 @@ impl OwnedTasks { unsafe fn remove_inner(&self, task: &Task) -> Option> { match self.lists[(task.header().task_id.0) as usize & (self.grain - 1)] .lock() - .list .remove(task.header_ptr()) { Some(task) => { @@ -224,7 +213,7 @@ cfg_taskdump! { { let mut f = f; for inner in &self.lists{ - f = inner.lock().list.for_each(f); + f = inner.lock().for_each(f); } } } From 191edf608fb73bd83cb7199af72ca696ba7646a5 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Wed, 20 Sep 2023 11:59:05 +0800 Subject: [PATCH 12/94] refactor for_each --- tokio/src/runtime/task/list.rs | 9 ++++----- tokio/src/util/linked_list.rs | 4 +--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index cdae876f566..ff109a4a3e4 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -207,13 +207,12 @@ impl OwnedTasks { cfg_taskdump! { impl OwnedTasks { /// Locks the tasks, and calls `f` on an iterator over them. - pub(crate) fn for_each(&self, f: F) + pub(crate) fn for_each(&self, mut f: F) where - F: FnMut(&Task) + F: FnMut(&Task), { - let mut f = f; - for inner in &self.lists{ - f = inner.lock().for_each(f); + for list in &self.lists { + list.lock().for_each(&mut f); } } } diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index b39e662dd54..035b315fe73 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -296,11 +296,10 @@ cfg_io_driver_impl! { cfg_taskdump! { impl LinkedList { - pub(crate) fn for_each(&mut self, mut f: F) -> F + pub(crate) fn for_each(&mut self, f: &mut F) where F: FnMut(&T::Handle), { - let mut next = self.head; while let Some(curr) = next { @@ -310,7 +309,6 @@ cfg_taskdump! { next = T::pointers(curr).as_ref().get_next(); } } - f } } } From 2be70c4178abb7c56f250d5622656dfceafbe414 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Wed, 20 Sep 2023 13:48:54 +0800 Subject: [PATCH 13/94] use atomic type in loom instead --- tokio/src/runtime/task/list.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index ff109a4a3e4..293ec4d9781 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -13,10 +13,9 @@ use crate::loom::sync::Mutex; use crate::runtime::task::{JoinHandle, LocalNotified, Notified, Schedule, Task}; use crate::util::linked_list::{Link, LinkedList}; +use crate::loom::sync::atomic::{AtomicBool, Ordering}; use std::marker::PhantomData; use std::num::NonZeroU32; -use std::sync::atomic::AtomicBool; -use std::sync::atomic::{AtomicU32, Ordering}; // The id from the module below is used to verify whether a given task is stored // in this OwnedTasks, or some other task. The counter starts at one so we can @@ -27,7 +26,7 @@ use std::sync::atomic::{AtomicU32, Ordering}; // bug in Tokio, so we accept that certain bugs would not be caught if the two // mixed up runtimes happen to have the same id. -static NEXT_OWNED_TASKS_ID: AtomicU32 = AtomicU32::new(1); +static NEXT_OWNED_TASKS_ID: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(1); fn get_next_id() -> NonZeroU32 { loop { From 17b0be9cac78b267750550a74322582b2e608b9a Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Wed, 20 Sep 2023 23:40:23 +0800 Subject: [PATCH 14/94] fix: put read closed flag after got the lock for avoid concurrency problems --- tokio/src/runtime/task/list.rs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 293ec4d9781..e234dcbfd3e 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -111,20 +111,28 @@ impl OwnedTasks { // to the field. task.header().set_owner_id(self.id); } + self.push_inner(id, task, notified) + } + + #[inline] + pub(crate) fn push_inner( + &self, + task_id: super::Id, + task: Task, + notified: Notified, + ) -> Option> + where + S: Schedule, + { + let mut lock = self.lists[task_id.0 as usize & (self.grain - 1)].lock(); // check close flag if self.closed.load(Ordering::Acquire) { task.shutdown(); return None; } - self.push_inner(id, task); - Some(notified) - } - - #[inline] - fn push_inner(&self, task_id: super::Id, task: Task) { - let mut lock = self.lists[task_id.0 as usize & (self.grain - 1)].lock(); lock.push_front(task); self.count.fetch_add(1, Ordering::Relaxed); + Some(notified) } /// Asserts that the given task is owned by this OwnedTasks and convert it to From 3bb484ee8f03098517eb1d8f55b9e64c4d6b7b26 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Thu, 21 Sep 2023 00:00:44 +0800 Subject: [PATCH 15/94] introduce random make shutdown faster --- .../runtime/scheduler/current_thread/mod.rs | 4 +-- .../runtime/scheduler/multi_thread/worker.rs | 11 ++++++-- .../scheduler/multi_thread_alt/worker.rs | 4 ++- tokio/src/runtime/task/list.rs | 25 +++++++++---------- tokio/src/runtime/tests/task.rs | 2 +- 5 files changed, 27 insertions(+), 19 deletions(-) diff --git a/tokio/src/runtime/scheduler/current_thread/mod.rs b/tokio/src/runtime/scheduler/current_thread/mod.rs index 7d27a5c20e9..e8cff36b1cf 100644 --- a/tokio/src/runtime/scheduler/current_thread/mod.rs +++ b/tokio/src/runtime/scheduler/current_thread/mod.rs @@ -248,7 +248,7 @@ fn shutdown2(mut core: Box, handle: &Handle) -> Box { // Drain the OwnedTasks collection. This call also closes the // collection, ensuring that no tasks are ever pushed after this // call returns. - handle.shared.owned.close_and_shutdown_all(); + handle.shared.owned.close_and_shutdown_all(0); // Drain local queue // We already shut down every task, so we just need to drop the task. @@ -615,7 +615,7 @@ impl Schedule for Arc { // If `None`, the runtime is shutting down, so there is no need to signal shutdown if let Some(core) = core.as_mut() { core.unhandled_panic = true; - self.shared.owned.close_and_shutdown_all(); + self.shared.owned.close_and_shutdown_all(0); } } _ => unreachable!("runtime core not set in CURRENT thread-local"), diff --git a/tokio/src/runtime/scheduler/multi_thread/worker.rs b/tokio/src/runtime/scheduler/multi_thread/worker.rs index 90503169cb3..59a918c8c55 100644 --- a/tokio/src/runtime/scheduler/multi_thread/worker.rs +++ b/tokio/src/runtime/scheduler/multi_thread/worker.rs @@ -547,7 +547,6 @@ impl Context { } core.pre_shutdown(&self.worker); - // Signal shutdown self.worker.handle.shutdown_core(core); Err(()) @@ -954,8 +953,16 @@ impl Core { /// Signals all tasks to shut down, and waits for them to complete. Must run /// before we enter the single-threaded phase of shutdown processing. fn pre_shutdown(&mut self, worker: &Worker) { + // Start from a random inner list + let start = self + .rand + .fastrand_n(worker.handle.shared.owned.grain as u32); // Signal to all tasks to shut down. - worker.handle.shared.owned.close_and_shutdown_all(); + worker + .handle + .shared + .owned + .close_and_shutdown_all(start as usize); self.stats .submit(&worker.handle.shared.worker_metrics[worker.index]); diff --git a/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs b/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs index 73f0a131f21..79a52dc177b 100644 --- a/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs +++ b/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs @@ -1460,7 +1460,9 @@ impl Shared { } pub(super) fn shutdown_core(&self, handle: &Handle, mut core: Box) { - self.owned.close_and_shutdown_all(); + // Start from a random inner list + let start = core.rand.fastrand_n(self.owned.grain as u32); + self.owned.close_and_shutdown_all(start as usize); core.stats.submit(&self.worker_metrics[core.index]); diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index e234dcbfd3e..d9ea132f4b7 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -38,10 +38,10 @@ fn get_next_id() -> NonZeroU32 { } pub(crate) struct OwnedTasks { - lists: Vec>>, + lists: Box<[Mutex>]>, pub(crate) id: NonZeroU32, closed: AtomicBool, - grain: usize, + pub(crate) grain: u32, count: AtomicUsize, } @@ -59,18 +59,18 @@ struct OwnedTasksInner { impl OwnedTasks { /// grain must be an integer power of 2 - pub(crate) fn new(grain: usize) -> Self { + pub(crate) fn new(grain: u32) -> Self { assert_eq!( grain & (grain - 1), 0, "the grain of OwnedTasks must be an integer power of 2" ); - let mut lists = Vec::with_capacity(grain); + let mut lists = Vec::with_capacity(grain as usize); for _ in 0..grain { lists.push(Mutex::new(LinkedList::new())) } Self { - lists, + lists: lists.into_boxed_slice(), closed: AtomicBool::new(false), id: get_next_id(), grain, @@ -124,7 +124,7 @@ impl OwnedTasks { where S: Schedule, { - let mut lock = self.lists[task_id.0 as usize & (self.grain - 1)].lock(); + let mut lock = self.lists[task_id.0 as usize & (self.grain - 1) as usize].lock(); // check close flag if self.closed.load(Ordering::Acquire) { task.shutdown(); @@ -150,17 +150,16 @@ impl OwnedTasks { /// Shuts down all tasks in the collection. This call also closes the /// collection, preventing new items from being added. - pub(crate) fn close_and_shutdown_all(&self) + pub(crate) fn close_and_shutdown_all(&self, start: usize) where S: Schedule, { // The first iteration of the loop was unrolled so it can set the // closed bool. self.closed.store(true, Ordering::Release); - - for list in &self.lists { + for i in start..self.lists.len() + start { loop { - let task = match list.lock().pop_back() { + let task = match self.lists[i % (self.lists.len())].lock().pop_back() { Some(task) => { self.count.fetch_sub(1, Ordering::Relaxed); task @@ -190,7 +189,7 @@ impl OwnedTasks { #[inline] unsafe fn remove_inner(&self, task: &Task) -> Option> { - match self.lists[(task.header().task_id.0) as usize & (self.grain - 1)] + match self.lists[(task.header().task_id.0) as usize & (self.grain - 1) as usize] .lock() .remove(task.header_ptr()) { @@ -218,8 +217,8 @@ cfg_taskdump! { where F: FnMut(&Task), { - for list in &self.lists { - list.lock().for_each(&mut f); + for i in 0 .. self.lists.len() { + self.lists[i].lock().for_each(&mut f); } } } diff --git a/tokio/src/runtime/tests/task.rs b/tokio/src/runtime/tests/task.rs index b2d3ef8fe18..8db5aadb901 100644 --- a/tokio/src/runtime/tests/task.rs +++ b/tokio/src/runtime/tests/task.rs @@ -308,7 +308,7 @@ impl Runtime { fn shutdown(&self) { let mut core = self.0.core.try_lock().unwrap(); - self.0.owned.close_and_shutdown_all(); + self.0.owned.close_and_shutdown_all(0); while let Some(task) = core.queue.pop_back() { drop(task); From e325825f01f47a89ca450945dd58e8d184984360 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Thu, 21 Sep 2023 00:41:33 +0800 Subject: [PATCH 16/94] use grain instead --- tokio/src/runtime/task/list.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index d9ea132f4b7..7736abd331a 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -157,9 +157,9 @@ impl OwnedTasks { // The first iteration of the loop was unrolled so it can set the // closed bool. self.closed.store(true, Ordering::Release); - for i in start..self.lists.len() + start { + for i in start..self.grain as usize + start { loop { - let task = match self.lists[i % (self.lists.len())].lock().pop_back() { + let task = match self.lists[i & (self.grain - 1) as usize].lock().pop_back() { Some(task) => { self.count.fetch_sub(1, Ordering::Relaxed); task From a11f80c44cfcc72f922cf2c567c771a3ab316444 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Thu, 21 Sep 2023 00:56:10 +0800 Subject: [PATCH 17/94] fix: dead lock --- tokio/src/runtime/task/list.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 7736abd331a..f28b902a3cc 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -127,6 +127,7 @@ impl OwnedTasks { let mut lock = self.lists[task_id.0 as usize & (self.grain - 1) as usize].lock(); // check close flag if self.closed.load(Ordering::Acquire) { + drop(lock); task.shutdown(); return None; } @@ -159,13 +160,15 @@ impl OwnedTasks { self.closed.store(true, Ordering::Release); for i in start..self.grain as usize + start { loop { - let task = match self.lists[i & (self.grain - 1) as usize].lock().pop_back() { + let mut lock = self.lists[i & (self.grain - 1) as usize].lock(); + let task = match lock.pop_back() { Some(task) => { self.count.fetch_sub(1, Ordering::Relaxed); task } None => break, }; + drop(lock); task.shutdown(); } } From cab1f5892ee8a942ebd981d3c226208bc103c02e Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Thu, 21 Sep 2023 08:45:55 +0800 Subject: [PATCH 18/94] fix: use list_inner to offer lock of list --- tokio/src/runtime/task/list.rs | 44 +++++++++++++++------------------- tokio/src/runtime/task/mod.rs | 4 ++++ 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index f28b902a3cc..2d874ac8d89 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -9,7 +9,7 @@ use crate::future::Future; use crate::loom::cell::UnsafeCell; use crate::loom::sync::atomic::AtomicUsize; -use crate::loom::sync::Mutex; +use crate::loom::sync::{Mutex, MutexGuard}; use crate::runtime::task::{JoinHandle, LocalNotified, Notified, Schedule, Task}; use crate::util::linked_list::{Link, LinkedList}; @@ -92,17 +92,12 @@ impl OwnedTasks { T::Output: Send + 'static, { let (task, notified, join) = super::new_task(task, scheduler, id); - let notified = unsafe { self.bind_inner(task, notified, id) }; + let notified = unsafe { self.bind_inner(task, notified) }; (join, notified) } /// The part of `bind` that's the same for every type of future. - unsafe fn bind_inner( - &self, - task: Task, - notified: Notified, - id: super::Id, - ) -> Option> + unsafe fn bind_inner(&self, task: Task, notified: Notified) -> Option> where S: Schedule, { @@ -111,27 +106,24 @@ impl OwnedTasks { // to the field. task.header().set_owner_id(self.id); } - self.push_inner(id, task, notified) + self.push_inner(task, notified) } #[inline] - pub(crate) fn push_inner( - &self, - task_id: super::Id, - task: Task, - notified: Notified, - ) -> Option> + pub(crate) fn push_inner(&self, task: Task, notified: Notified) -> Option> where S: Schedule, { - let mut lock = self.lists[task_id.0 as usize & (self.grain - 1) as usize].lock(); - // check close flag + let mut lock = self.list_inner(task.task_id() as usize); + // check close flag, + // it must be checked in the lock, for ensuring all tasks will shutdown after OwnedTasks has beem closed if self.closed.load(Ordering::Acquire) { drop(lock); task.shutdown(); return None; } lock.push_front(task); + drop(lock); self.count.fetch_add(1, Ordering::Relaxed); Some(notified) } @@ -155,20 +147,18 @@ impl OwnedTasks { where S: Schedule, { - // The first iteration of the loop was unrolled so it can set the - // closed bool. self.closed.store(true, Ordering::Release); for i in start..self.grain as usize + start { loop { - let mut lock = self.lists[i & (self.grain - 1) as usize].lock(); + let mut lock = self.list_inner(i); let task = match lock.pop_back() { Some(task) => { + drop(lock); self.count.fetch_sub(1, Ordering::Relaxed); task } None => break, }; - drop(lock); task.shutdown(); } } @@ -192,11 +182,10 @@ impl OwnedTasks { #[inline] unsafe fn remove_inner(&self, task: &Task) -> Option> { - match self.lists[(task.header().task_id.0) as usize & (self.grain - 1) as usize] - .lock() - .remove(task.header_ptr()) - { + let mut lock = self.list_inner(task.task_id() as usize); + match lock.remove(task.header_ptr()) { Some(task) => { + drop(lock); self.count.fetch_sub(1, Ordering::Relaxed); Some(task) } @@ -204,6 +193,11 @@ impl OwnedTasks { } } + #[inline] + fn list_inner(&self, id: usize) -> MutexGuard<'_, ListInner> { + self.lists[id & (self.grain - 1) as usize].lock() + } + pub(crate) fn is_closed(&self) -> bool { self.closed.load(Ordering::Acquire) } diff --git a/tokio/src/runtime/task/mod.rs b/tokio/src/runtime/task/mod.rs index abf7cc266e7..61b9af77b59 100644 --- a/tokio/src/runtime/task/mod.rs +++ b/tokio/src/runtime/task/mod.rs @@ -361,6 +361,10 @@ impl Task { fn header_ptr(&self) -> NonNull
{ self.raw.header_ptr() } + + fn task_id(&self) -> u64 { + self.header().task_id.0 + } } impl Notified { From 389e6b9c7fb4ebdbcd4ad6764250380e938f2419 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Thu, 21 Sep 2023 09:15:35 +0800 Subject: [PATCH 19/94] fix: use segment_size instead of grain --- .../runtime/scheduler/multi_thread/worker.rs | 2 +- .../scheduler/multi_thread_alt/worker.rs | 2 +- tokio/src/runtime/task/list.rs | 51 ++++++++++++------- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/tokio/src/runtime/scheduler/multi_thread/worker.rs b/tokio/src/runtime/scheduler/multi_thread/worker.rs index 59a918c8c55..1bbb1073621 100644 --- a/tokio/src/runtime/scheduler/multi_thread/worker.rs +++ b/tokio/src/runtime/scheduler/multi_thread/worker.rs @@ -956,7 +956,7 @@ impl Core { // Start from a random inner list let start = self .rand - .fastrand_n(worker.handle.shared.owned.grain as u32); + .fastrand_n(worker.handle.shared.owned.segment_size as u32); // Signal to all tasks to shut down. worker .handle diff --git a/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs b/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs index 79a52dc177b..af11dfed764 100644 --- a/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs +++ b/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs @@ -1461,7 +1461,7 @@ impl Shared { pub(super) fn shutdown_core(&self, handle: &Handle, mut core: Box) { // Start from a random inner list - let start = core.rand.fastrand_n(self.owned.grain as u32); + let start = core.rand.fastrand_n(self.owned.segment_size as u32); self.owned.close_and_shutdown_all(start as usize); core.stats.submit(&self.worker_metrics[core.index]); diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 2d874ac8d89..5f599d95565 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -38,14 +38,15 @@ fn get_next_id() -> NonZeroU32 { } pub(crate) struct OwnedTasks { - lists: Box<[Mutex>]>, + lists: Box<[Mutex>]>, pub(crate) id: NonZeroU32, closed: AtomicBool, - pub(crate) grain: u32, + pub(crate) segment_size: u32, + segment_mask: u32, count: AtomicUsize, } -type ListInner = LinkedList, as Link>::Target>; +type ListSement = LinkedList, as Link>::Target>; pub(crate) struct LocalOwnedTasks { inner: UnsafeCell>, @@ -58,22 +59,36 @@ struct OwnedTasksInner { } impl OwnedTasks { - /// grain must be an integer power of 2 - pub(crate) fn new(grain: u32) -> Self { - assert_eq!( - grain & (grain - 1), - 0, - "the grain of OwnedTasks must be an integer power of 2" + /// The concurrency_level should be set according to the expected scale of multi-thread concurrency. + /// A large concurrency_level should not affect performance, but might affect the CPU's cache. + /// The concurrency_level is at least one, otherwise it will panic. + /// The maximum concurrency_level is 65536, + /// if the parameter is larger than this value, OwnedTasks will actually select 65536 internally. + pub(crate) fn new(mut concurrency_level: u32) -> Self { + if concurrency_level > 1 << 16 { + concurrency_level = 1 << 16; + } + assert!( + concurrency_level > 0, + "concurrency_level must be at least one" ); - let mut lists = Vec::with_capacity(grain as usize); - for _ in 0..grain { + // Find power-of-two sizes best matching arguments + let mut segment_size = 1; + while segment_size < concurrency_level { + segment_size = segment_size << 1; + } + let segment_mask = segment_size - 1; + + let mut lists = Vec::with_capacity(segment_size as usize); + for _ in 0..segment_size { lists.push(Mutex::new(LinkedList::new())) } Self { lists: lists.into_boxed_slice(), closed: AtomicBool::new(false), id: get_next_id(), - grain, + segment_size, + segment_mask, count: AtomicUsize::new(0), } } @@ -114,7 +129,7 @@ impl OwnedTasks { where S: Schedule, { - let mut lock = self.list_inner(task.task_id() as usize); + let mut lock = self.segment_inner(task.task_id() as usize); // check close flag, // it must be checked in the lock, for ensuring all tasks will shutdown after OwnedTasks has beem closed if self.closed.load(Ordering::Acquire) { @@ -148,9 +163,9 @@ impl OwnedTasks { S: Schedule, { self.closed.store(true, Ordering::Release); - for i in start..self.grain as usize + start { + for i in start..self.segment_size as usize + start { loop { - let mut lock = self.list_inner(i); + let mut lock = self.segment_inner(i); let task = match lock.pop_back() { Some(task) => { drop(lock); @@ -182,7 +197,7 @@ impl OwnedTasks { #[inline] unsafe fn remove_inner(&self, task: &Task) -> Option> { - let mut lock = self.list_inner(task.task_id() as usize); + let mut lock = self.segment_inner(task.task_id() as usize); match lock.remove(task.header_ptr()) { Some(task) => { drop(lock); @@ -194,8 +209,8 @@ impl OwnedTasks { } #[inline] - fn list_inner(&self, id: usize) -> MutexGuard<'_, ListInner> { - self.lists[id & (self.grain - 1) as usize].lock() + fn segment_inner(&self, id: usize) -> MutexGuard<'_, ListSement> { + self.lists[id & (self.segment_mask) as usize].lock() } pub(crate) fn is_closed(&self) -> bool { From 833377c5e45ee25306ff1bd2240517e71c372179 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Thu, 21 Sep 2023 10:25:58 +0800 Subject: [PATCH 20/94] clippy --- tokio/src/runtime/task/list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 5f599d95565..754d10afef7 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -75,7 +75,7 @@ impl OwnedTasks { // Find power-of-two sizes best matching arguments let mut segment_size = 1; while segment_size < concurrency_level { - segment_size = segment_size << 1; + segment_size <<= 1; } let segment_mask = segment_size - 1; From 542289503777778d9acc7fdfdaae5ca28a3db707 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Thu, 21 Sep 2023 11:42:00 +0800 Subject: [PATCH 21/94] feat: let spawn_concurrency_level configurable --- tokio/src/runtime/builder.rs | 56 +++++++++++++++++++ .../src/runtime/scheduler/multi_thread/mod.rs | 2 + .../runtime/scheduler/multi_thread/worker.rs | 5 +- .../runtime/scheduler/multi_thread_alt/mod.rs | 2 + .../scheduler/multi_thread_alt/worker.rs | 3 +- tokio/src/runtime/task/list.rs | 14 +---- 6 files changed, 66 insertions(+), 16 deletions(-) diff --git a/tokio/src/runtime/builder.rs b/tokio/src/runtime/builder.rs index 03f1678dcb1..882e6a864d9 100644 --- a/tokio/src/runtime/builder.rs +++ b/tokio/src/runtime/builder.rs @@ -57,6 +57,11 @@ pub struct Builder { /// Only used when not using the current-thread executor. worker_threads: Option, + /// Configures the global OwnedTasks's concurrency level + /// + /// Only used when not using the current-thread executor. + pub(super) spawn_concurrency_level: Option, + /// Cap on thread usage. max_blocking_threads: usize, @@ -278,6 +283,9 @@ impl Builder { // Default to lazy auto-detection (one thread per CPU core) worker_threads: None, + // Default to lazy auto-detection (twice the number of worker threads) + spawn_concurrency_level: None, + max_blocking_threads: 512, // Default thread name @@ -401,6 +409,49 @@ impl Builder { self } + /// Sets the spawn concurrency level the `Runtime` will use. + /// + /// This can be any number greater than 0 and less than or equal to 65536, + /// if the parameter is larger than this value, concurrency level will actually select 65536 internally. + /// + /// This should be set according to the expected scale of multi-thread concurrency of `tokio::spawn`, + /// This requires a trade-off between concurrency scale and CPU's cache. + /// + /// # Default + /// + /// The default value is twice the number of worker threads. + /// + /// When using the `current_thread` runtime this method has no effect. + /// + /// # Examples + /// + /// ## Multi threaded runtime with spawn_concurrency_level 8 + /// + /// ``` + /// use tokio::runtime; + /// + /// // This will spawn a work-stealing runtime with 4 worker threads. + /// let rt = runtime::Builder::new_multi_thread() + /// .spawn_concurrency_level(8) + /// .build() + /// .unwrap(); + /// + /// rt.spawn(async move {}); + /// ``` + /// + /// # Panics + /// + /// This will panic if `val` is not larger than `0`. + #[track_caller] + pub fn spawn_concurrency_level(&mut self, mut val: usize) -> &mut Self { + assert!(val > 0, "spawn concurrency level cannot be set to 0"); + if val > 1 << 16 { + val = 1 << 16; + } + self.spawn_concurrency_level = Some(val); + self + } + /// Specifies the limit for additional threads spawned by the Runtime. /// /// These threads are used for blocking operations like tasks spawned @@ -1231,6 +1282,7 @@ cfg_rt_multi_thread! { use crate::runtime::scheduler::{self, MultiThread}; let core_threads = self.worker_threads.unwrap_or_else(num_cpus); + let spawn_concurrency_level = self.spawn_concurrency_level.unwrap_or(core_threads * 2); let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?; @@ -1249,6 +1301,7 @@ cfg_rt_multi_thread! { driver_handle, blocking_spawner, seed_generator_2, + spawn_concurrency_level, Config { before_park: self.before_park.clone(), after_unpark: self.after_unpark.clone(), @@ -1279,6 +1332,7 @@ cfg_rt_multi_thread! { use crate::runtime::scheduler::MultiThreadAlt; let core_threads = self.worker_threads.unwrap_or_else(num_cpus); + let spawn_concurrency_level = self.spawn_concurrency_level.unwrap_or(core_threads * 2); let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?; @@ -1297,6 +1351,7 @@ cfg_rt_multi_thread! { driver_handle, blocking_spawner, seed_generator_2, + spawn_concurrency_level, Config { before_park: self.before_park.clone(), after_unpark: self.after_unpark.clone(), @@ -1321,6 +1376,7 @@ impl fmt::Debug for Builder { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt.debug_struct("Builder") .field("worker_threads", &self.worker_threads) + .field("spawn_concurrency_level", &self.spawn_concurrency_level) .field("max_blocking_threads", &self.max_blocking_threads) .field( "thread_name", diff --git a/tokio/src/runtime/scheduler/multi_thread/mod.rs b/tokio/src/runtime/scheduler/multi_thread/mod.rs index d85a0ae0a2a..24d353cdfac 100644 --- a/tokio/src/runtime/scheduler/multi_thread/mod.rs +++ b/tokio/src/runtime/scheduler/multi_thread/mod.rs @@ -60,6 +60,7 @@ impl MultiThread { driver_handle: driver::Handle, blocking_spawner: blocking::Spawner, seed_generator: RngSeedGenerator, + spawn_concurrency_level: usize, config: Config, ) -> (MultiThread, Arc, Launch) { let parker = Parker::new(driver); @@ -69,6 +70,7 @@ impl MultiThread { driver_handle, blocking_spawner, seed_generator, + spawn_concurrency_level, config, ); diff --git a/tokio/src/runtime/scheduler/multi_thread/worker.rs b/tokio/src/runtime/scheduler/multi_thread/worker.rs index 1bbb1073621..6e55f5df215 100644 --- a/tokio/src/runtime/scheduler/multi_thread/worker.rs +++ b/tokio/src/runtime/scheduler/multi_thread/worker.rs @@ -245,6 +245,7 @@ pub(super) fn create( driver_handle: driver::Handle, blocking_spawner: blocking::Spawner, seed_generator: RngSeedGenerator, + spawn_concurrency_level: usize, config: Config, ) -> (Arc, Launch) { let mut cores = Vec::with_capacity(size); @@ -287,7 +288,7 @@ pub(super) fn create( remotes: remotes.into_boxed_slice(), inject, idle, - owned: OwnedTasks::new(16), + owned: OwnedTasks::new(spawn_concurrency_level as u32), synced: Mutex::new(Synced { idle: idle_synced, inject: inject_synced, @@ -956,7 +957,7 @@ impl Core { // Start from a random inner list let start = self .rand - .fastrand_n(worker.handle.shared.owned.segment_size as u32); + .fastrand_n(worker.handle.shared.owned.segment_size); // Signal to all tasks to shut down. worker .handle diff --git a/tokio/src/runtime/scheduler/multi_thread_alt/mod.rs b/tokio/src/runtime/scheduler/multi_thread_alt/mod.rs index e30c9b4783b..86c6f96087d 100644 --- a/tokio/src/runtime/scheduler/multi_thread_alt/mod.rs +++ b/tokio/src/runtime/scheduler/multi_thread_alt/mod.rs @@ -49,6 +49,7 @@ impl MultiThread { driver_handle: driver::Handle, blocking_spawner: blocking::Spawner, seed_generator: RngSeedGenerator, + spawn_concurrency_level: usize, config: Config, ) -> (MultiThread, runtime::Handle) { let handle = worker::create( @@ -57,6 +58,7 @@ impl MultiThread { driver_handle, blocking_spawner, seed_generator, + spawn_concurrency_level, config, ); diff --git a/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs b/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs index af11dfed764..1533546e807 100644 --- a/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs +++ b/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs @@ -259,6 +259,7 @@ pub(super) fn create( driver_handle: driver::Handle, blocking_spawner: blocking::Spawner, seed_generator: RngSeedGenerator, + spawn_concurrency_level: usize, config: Config, ) -> runtime::Handle { let mut num_workers = num_cores; @@ -307,7 +308,7 @@ pub(super) fn create( remotes: remotes.into_boxed_slice(), inject, idle, - owned: OwnedTasks::new(16), + owned: OwnedTasks::new(spawn_concurrency_level as u32), synced: Mutex::new(Synced { assigned_cores: (0..num_workers).map(|_| None).collect(), shutdown_cores: Vec::with_capacity(num_cores), diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 754d10afef7..082c8c37215 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -59,19 +59,7 @@ struct OwnedTasksInner { } impl OwnedTasks { - /// The concurrency_level should be set according to the expected scale of multi-thread concurrency. - /// A large concurrency_level should not affect performance, but might affect the CPU's cache. - /// The concurrency_level is at least one, otherwise it will panic. - /// The maximum concurrency_level is 65536, - /// if the parameter is larger than this value, OwnedTasks will actually select 65536 internally. - pub(crate) fn new(mut concurrency_level: u32) -> Self { - if concurrency_level > 1 << 16 { - concurrency_level = 1 << 16; - } - assert!( - concurrency_level > 0, - "concurrency_level must be at least one" - ); + pub(crate) fn new(concurrency_level: u32) -> Self { // Find power-of-two sizes best matching arguments let mut segment_size = 1; while segment_size < concurrency_level { From b90101a7de03efc8aa6232b8f118f13ae720513c Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Thu, 21 Sep 2023 15:49:11 +0800 Subject: [PATCH 22/94] feat: update benchmark --- benches/spawn_concurrent.rs | 72 ++++++++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 4 deletions(-) diff --git a/benches/spawn_concurrent.rs b/benches/spawn_concurrent.rs index ca8f586703e..148c79b8cdd 100644 --- a/benches/spawn_concurrent.rs +++ b/benches/spawn_concurrent.rs @@ -48,9 +48,12 @@ fn spawn_tasks(c: &mut Criterion) { }); } -fn spawn_tasks_parallel(c: &mut Criterion) { - let runtime = tokio::runtime::Builder::new_multi_thread().build().unwrap(); - c.bench_function("spawn_tasks_parallel", move |b| { +fn spawn_tasks_parallel(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_multi_thread() + .spawn_concurrency_level(S) + .build() + .unwrap(); + c.bench_function(format!("spawn_tasks_parallel {}", S).as_str(), move |b| { b.iter_custom(|iters| { let start = Instant::now(); runtime.block_on(async { @@ -83,11 +86,72 @@ async fn job(iters: usize, procs: usize) { } } +fn shutdown_tasks_parallel(c: &mut Criterion) { + c.bench_function( + format!("showdown_tasks_parallel {}", S).as_str(), + move |b| { + b.iter_custom(|iters| { + let runtime = tokio::runtime::Builder::new_multi_thread() + .spawn_concurrency_level(S) + .enable_time() + .build() + .unwrap(); + runtime.block_on(async { + black_box(job_shutdown(iters as usize, num_cpus::get_physical()).await); + }); + let start = Instant::now(); + drop(runtime); + start.elapsed() + }) + }, + ); +} + +async fn job_shutdown(iters: usize, procs: usize) { + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + for _ in 0..procs { + let tx = tx.clone(); + tokio::spawn(async move { + for _ in 0..iters / procs { + let tx = tx.clone(); + tokio::spawn(async move { + let val = 1 + 1; + tx.send(()).unwrap(); + tokio::time::sleep(tokio::time::Duration::from_secs(1000)).await; // it will never return + black_box(val) + }); + } + }); + } + for _ in 0..(iters / procs) * procs { + rx.recv().await; + } +} + criterion_group!( benches, spawn_tasks_current_thread, spawn_tasks_current_thread_parallel, spawn_tasks, - spawn_tasks_parallel + spawn_tasks_parallel<1>, + spawn_tasks_parallel<4>, + spawn_tasks_parallel<8>, + spawn_tasks_parallel<16>, + spawn_tasks_parallel<32>, + spawn_tasks_parallel<64>, + spawn_tasks_parallel<128>, + spawn_tasks_parallel<256>, + spawn_tasks_parallel<512>, + spawn_tasks_parallel<1024>, + shutdown_tasks_parallel<1>, + shutdown_tasks_parallel<4>, + shutdown_tasks_parallel<8>, + shutdown_tasks_parallel<16>, + shutdown_tasks_parallel<32>, + shutdown_tasks_parallel<64>, + shutdown_tasks_parallel<128>, + shutdown_tasks_parallel<256>, + shutdown_tasks_parallel<512>, + shutdown_tasks_parallel<1024>, ); criterion_main!(benches); From 68af71a0a165072fb99d393ad05ce8f1cd760c27 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Thu, 21 Sep 2023 22:35:58 +0800 Subject: [PATCH 23/94] feat: add benchmarks for spawn_concurrency_level --- benches/spawn_concurrent.rs | 134 +++++++++++++++++++++--------------- 1 file changed, 77 insertions(+), 57 deletions(-) diff --git a/benches/spawn_concurrent.rs b/benches/spawn_concurrent.rs index 148c79b8cdd..d000c65a18a 100644 --- a/benches/spawn_concurrent.rs +++ b/benches/spawn_concurrent.rs @@ -1,6 +1,6 @@ use std::time::Instant; -use criterion::*; +use criterion::{measurement::WallTime, *}; fn spawn_tasks_current_thread(c: &mut Criterion) { let runtime = tokio::runtime::Builder::new_current_thread() @@ -11,7 +11,7 @@ fn spawn_tasks_current_thread(c: &mut Criterion) { b.iter_custom(|iters| { let start = Instant::now(); runtime.block_on(async { - black_box(job(iters as usize, 1).await); + black_box(spawn_job(iters as usize, 1).await); }); start.elapsed() }) @@ -27,44 +27,70 @@ fn spawn_tasks_current_thread_parallel(c: &mut Criterion) { b.iter_custom(|iters| { let start = Instant::now(); runtime.block_on(async { - black_box(job(iters as usize, num_cpus::get_physical() * 2).await); + black_box(spawn_job(iters as usize, num_cpus::get_physical() * 2).await); }); start.elapsed() }) }); } -fn spawn_tasks(c: &mut Criterion) { - let runtime = tokio::runtime::Builder::new_multi_thread().build().unwrap(); +fn bench_create_runtime_multi_thread(c: &mut Criterion) { + let mut group = c.benchmark_group("create_runtime_multi_thread"); + create_multi_thread_runtime::<1>(&mut group); + create_multi_thread_runtime::<8>(&mut group); + create_multi_thread_runtime::<32>(&mut group); + create_multi_thread_runtime::<64>(&mut group); + create_multi_thread_runtime::<128>(&mut group); + create_multi_thread_runtime::<256>(&mut group); + create_multi_thread_runtime::<512>(&mut group); + create_multi_thread_runtime::<1024>(&mut group); + create_multi_thread_runtime::<2048>(&mut group); + create_multi_thread_runtime::<4096>(&mut group); +} - c.bench_function("spawn_tasks", move |b| { - b.iter_custom(|iters| { - let start = Instant::now(); - runtime.block_on(async { - black_box(job(iters as usize, 1).await); - }); - start.elapsed() +fn create_multi_thread_runtime(g: &mut BenchmarkGroup) { + g.bench_function(format!("{:04}", S), |b| { + b.iter(|| { + let runtime = tokio::runtime::Builder::new_multi_thread() + .spawn_concurrency_level(black_box(S)) + .build() + .unwrap(); + drop(runtime); }) }); } -fn spawn_tasks_parallel(c: &mut Criterion) { +fn bench_parallel_spawn_multi_thread(c: &mut Criterion) { + let mut group = c.benchmark_group("spawn_parallel_multi_thread"); + spawn_tasks_parallel_multi_thread::<1>(&mut group); + spawn_tasks_parallel_multi_thread::<8>(&mut group); + spawn_tasks_parallel_multi_thread::<32>(&mut group); + spawn_tasks_parallel_multi_thread::<64>(&mut group); + spawn_tasks_parallel_multi_thread::<128>(&mut group); + spawn_tasks_parallel_multi_thread::<256>(&mut group); + spawn_tasks_parallel_multi_thread::<512>(&mut group); + spawn_tasks_parallel_multi_thread::<1024>(&mut group); + spawn_tasks_parallel_multi_thread::<2048>(&mut group); + spawn_tasks_parallel_multi_thread::<4096>(&mut group); +} + +fn spawn_tasks_parallel_multi_thread(g: &mut BenchmarkGroup) { let runtime = tokio::runtime::Builder::new_multi_thread() - .spawn_concurrency_level(S) + .spawn_concurrency_level(black_box(S)) .build() .unwrap(); - c.bench_function(format!("spawn_tasks_parallel {}", S).as_str(), move |b| { + g.bench_function(format!("{:04}", S), |b| { b.iter_custom(|iters| { let start = Instant::now(); runtime.block_on(async { - black_box(job(iters as usize, num_cpus::get_physical()).await); + black_box(spawn_job(iters as usize, num_cpus::get_physical()).await); }); start.elapsed() }) }); } -async fn job(iters: usize, procs: usize) { +async fn spawn_job(iters: usize, procs: usize) { for _ in 0..procs { let mut threads_handles = Vec::with_capacity(procs); threads_handles.push(tokio::spawn(async move { @@ -86,25 +112,37 @@ async fn job(iters: usize, procs: usize) { } } -fn shutdown_tasks_parallel(c: &mut Criterion) { - c.bench_function( - format!("showdown_tasks_parallel {}", S).as_str(), - move |b| { - b.iter_custom(|iters| { - let runtime = tokio::runtime::Builder::new_multi_thread() - .spawn_concurrency_level(S) - .enable_time() - .build() - .unwrap(); - runtime.block_on(async { - black_box(job_shutdown(iters as usize, num_cpus::get_physical()).await); - }); - let start = Instant::now(); - drop(runtime); - start.elapsed() - }) - }, - ); +fn bench_shutdown_parallel_multi_thread(c: &mut Criterion) { + let mut group = c.benchmark_group("shutdown_parallel_multi_thread"); + shutdown_tasks_parallel::<1>(&mut group); + shutdown_tasks_parallel::<8>(&mut group); + shutdown_tasks_parallel::<32>(&mut group); + shutdown_tasks_parallel::<64>(&mut group); + shutdown_tasks_parallel::<128>(&mut group); + shutdown_tasks_parallel::<256>(&mut group); + shutdown_tasks_parallel::<512>(&mut group); + shutdown_tasks_parallel::<1024>(&mut group); + shutdown_tasks_parallel::<2048>(&mut group); + shutdown_tasks_parallel::<4096>(&mut group); + group.finish(); +} + +fn shutdown_tasks_parallel(g: &mut BenchmarkGroup) { + g.bench_function(format!("{:04}", S), |b| { + b.iter_custom(|iters| { + let runtime = tokio::runtime::Builder::new_multi_thread() + .spawn_concurrency_level(black_box(S)) + .enable_time() + .build() + .unwrap(); + runtime.block_on(async { + black_box(job_shutdown(iters as usize, num_cpus::get_physical()).await); + }); + let start = Instant::now(); + drop(runtime); + start.elapsed() + }) + }); } async fn job_shutdown(iters: usize, procs: usize) { @@ -132,26 +170,8 @@ criterion_group!( benches, spawn_tasks_current_thread, spawn_tasks_current_thread_parallel, - spawn_tasks, - spawn_tasks_parallel<1>, - spawn_tasks_parallel<4>, - spawn_tasks_parallel<8>, - spawn_tasks_parallel<16>, - spawn_tasks_parallel<32>, - spawn_tasks_parallel<64>, - spawn_tasks_parallel<128>, - spawn_tasks_parallel<256>, - spawn_tasks_parallel<512>, - spawn_tasks_parallel<1024>, - shutdown_tasks_parallel<1>, - shutdown_tasks_parallel<4>, - shutdown_tasks_parallel<8>, - shutdown_tasks_parallel<16>, - shutdown_tasks_parallel<32>, - shutdown_tasks_parallel<64>, - shutdown_tasks_parallel<128>, - shutdown_tasks_parallel<256>, - shutdown_tasks_parallel<512>, - shutdown_tasks_parallel<1024>, + bench_create_runtime_multi_thread, + bench_parallel_spawn_multi_thread, + bench_shutdown_parallel_multi_thread, ); criterion_main!(benches); From 8e4716a7e7450faf20e5425afcb758d99d3e0fd8 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Fri, 22 Sep 2023 11:53:05 +0800 Subject: [PATCH 24/94] change spawn_concurrency_level to be 4 times the number of worker threads --- tokio/src/runtime/builder.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tokio/src/runtime/builder.rs b/tokio/src/runtime/builder.rs index 882e6a864d9..6f0408ffbb7 100644 --- a/tokio/src/runtime/builder.rs +++ b/tokio/src/runtime/builder.rs @@ -414,12 +414,14 @@ impl Builder { /// This can be any number greater than 0 and less than or equal to 65536, /// if the parameter is larger than this value, concurrency level will actually select 65536 internally. /// - /// This should be set according to the expected scale of multi-thread concurrency of `tokio::spawn`, - /// This requires a trade-off between concurrency scale and CPU's cache. + /// When the value of this is small compared to the number of concurrent threads, increasing it + /// will help improve the performanc of concurrently spawn tasks. However, when the value is + /// already large enough, further increasing it will not continue to improve performance. + /// Instead, it may result in longer time of the Runtime creation. /// /// # Default /// - /// The default value is twice the number of worker threads. + /// The default value for this is 4 times the number of worker threads. /// /// When using the `current_thread` runtime this method has no effect. /// @@ -1282,7 +1284,7 @@ cfg_rt_multi_thread! { use crate::runtime::scheduler::{self, MultiThread}; let core_threads = self.worker_threads.unwrap_or_else(num_cpus); - let spawn_concurrency_level = self.spawn_concurrency_level.unwrap_or(core_threads * 2); + let spawn_concurrency_level = self.spawn_concurrency_level.unwrap_or(core_threads * 4); let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?; @@ -1332,7 +1334,7 @@ cfg_rt_multi_thread! { use crate::runtime::scheduler::MultiThreadAlt; let core_threads = self.worker_threads.unwrap_or_else(num_cpus); - let spawn_concurrency_level = self.spawn_concurrency_level.unwrap_or(core_threads * 2); + let spawn_concurrency_level = self.spawn_concurrency_level.unwrap_or(core_threads * 4); let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?; From 7bbc2e4186e09fddd83e3d8976439c74807b6f19 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Fri, 22 Sep 2023 23:33:15 +0800 Subject: [PATCH 25/94] change benchmark tests name from shutdown_parallel_multi_thread to shutdown_runtime_multi_thread --- benches/spawn_concurrent.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benches/spawn_concurrent.rs b/benches/spawn_concurrent.rs index d000c65a18a..4fd59bfa7a7 100644 --- a/benches/spawn_concurrent.rs +++ b/benches/spawn_concurrent.rs @@ -113,7 +113,7 @@ async fn spawn_job(iters: usize, procs: usize) { } fn bench_shutdown_parallel_multi_thread(c: &mut Criterion) { - let mut group = c.benchmark_group("shutdown_parallel_multi_thread"); + let mut group = c.benchmark_group("shutdown_runtime_multi_thread"); shutdown_tasks_parallel::<1>(&mut group); shutdown_tasks_parallel::<8>(&mut group); shutdown_tasks_parallel::<32>(&mut group); From c15c0bdda28a7ff20081c9e0406fa43b43471ce3 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Fri, 22 Sep 2023 23:51:10 +0800 Subject: [PATCH 26/94] fix the comments on spawn_concurrency_level --- tokio/src/runtime/builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/runtime/builder.rs b/tokio/src/runtime/builder.rs index 6f0408ffbb7..cc3fcf06a8d 100644 --- a/tokio/src/runtime/builder.rs +++ b/tokio/src/runtime/builder.rs @@ -283,7 +283,7 @@ impl Builder { // Default to lazy auto-detection (one thread per CPU core) worker_threads: None, - // Default to lazy auto-detection (twice the number of worker threads) + // Default to lazy auto-detection (4 times the number of worker threads) spawn_concurrency_level: None, max_blocking_threads: 512, From 6706a35f4ec0ff22b2251b6728264e082c4d6270 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Sat, 23 Sep 2023 00:09:06 +0800 Subject: [PATCH 27/94] add comments for parameter in OwnedTasks.close_and_shutdown_all --- tokio/src/runtime/task/list.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 082c8c37215..a8ef5e43689 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -146,6 +146,10 @@ impl OwnedTasks { /// Shuts down all tasks in the collection. This call also closes the /// collection, preventing new items from being added. + /// The parameter start should be random among different worker threads + /// to reduce lock conflicts during shutdown. + /// Initiate shutting down the segment indexed by the start, and reset to 0 + /// once the segment_size is reached, continuing until start - 1, it works like a ring. pub(crate) fn close_and_shutdown_all(&self, start: usize) where S: Schedule, From d2c7668b34beac99966d95d5ee59158c7087cb4f Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Sat, 23 Sep 2023 00:19:55 +0800 Subject: [PATCH 28/94] fix comments --- tokio/src/runtime/task/list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index a8ef5e43689..bd5f7bd144c 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -119,7 +119,7 @@ impl OwnedTasks { { let mut lock = self.segment_inner(task.task_id() as usize); // check close flag, - // it must be checked in the lock, for ensuring all tasks will shutdown after OwnedTasks has beem closed + // it must be checked in the lock, for ensuring all tasks will shutdown after OwnedTasks has been closed if self.closed.load(Ordering::Acquire) { drop(lock); task.shutdown(); From 5105610d043344b5226f34fe63575f9ca21172e5 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Mon, 25 Sep 2023 00:51:04 +0800 Subject: [PATCH 29/94] style nit: simplify code, apply suggestions from hawkw --- tokio/src/runtime/task/list.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index bd5f7bd144c..96baaae1ad6 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -158,15 +158,14 @@ impl OwnedTasks { for i in start..self.segment_size as usize + start { loop { let mut lock = self.segment_inner(i); - let task = match lock.pop_back() { + match lock.pop_back() { Some(task) => { drop(lock); self.count.fetch_sub(1, Ordering::Relaxed); - task + task.shutdown(); } None => break, }; - task.shutdown(); } } } @@ -190,14 +189,10 @@ impl OwnedTasks { #[inline] unsafe fn remove_inner(&self, task: &Task) -> Option> { let mut lock = self.segment_inner(task.task_id() as usize); - match lock.remove(task.header_ptr()) { - Some(task) => { - drop(lock); - self.count.fetch_sub(1, Ordering::Relaxed); - Some(task) - } - None => None, - } + let task = lock.remove(task.header_ptr())?; + drop(lock); + self.count.fetch_sub(1, Ordering::Relaxed); + Some(task) } #[inline] From a30df11e225d2fc9d720d951e8593e7a4737d8f2 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Mon, 25 Sep 2023 14:29:58 +0800 Subject: [PATCH 30/94] make spawn_concurrency_level is constant 4 in loom test --- tokio/src/runtime/builder.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tokio/src/runtime/builder.rs b/tokio/src/runtime/builder.rs index cc3fcf06a8d..e69b22950aa 100644 --- a/tokio/src/runtime/builder.rs +++ b/tokio/src/runtime/builder.rs @@ -1284,6 +1284,11 @@ cfg_rt_multi_thread! { use crate::runtime::scheduler::{self, MultiThread}; let core_threads = self.worker_threads.unwrap_or_else(num_cpus); + // Shrink the size of spawn_concurrency_level when using loom. This shouldn't impact + // logic, but allows loom to test more edge cases in a reasoable a mount of time + #[cfg(loom)] + let spawn_concurrency_level = 4; + #[cfg(not(loom))] let spawn_concurrency_level = self.spawn_concurrency_level.unwrap_or(core_threads * 4); let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?; @@ -1334,6 +1339,12 @@ cfg_rt_multi_thread! { use crate::runtime::scheduler::MultiThreadAlt; let core_threads = self.worker_threads.unwrap_or_else(num_cpus); + + // Shrink the size of spawn_concurrency_level when using loom. This shouldn't impact + // logic, but allows loom to test more edge cases in a reasoable a mount of time + #[cfg(loom)] + let spawn_concurrency_level = 4; + #[cfg(not(loom))] let spawn_concurrency_level = self.spawn_concurrency_level.unwrap_or(core_threads * 4); let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?; From df4ab6110fa442d8dfe7b50f8ebb85f2e3a71968 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Thu, 28 Sep 2023 21:42:22 +0800 Subject: [PATCH 31/94] use Header::get_id to get task_id --- tokio/src/runtime/task/core.rs | 6 ------ tokio/src/runtime/task/list.rs | 11 +++++++++-- tokio/src/runtime/task/mod.rs | 4 ---- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/tokio/src/runtime/task/core.rs b/tokio/src/runtime/task/core.rs index c198badf3d4..e62f1160ee0 100644 --- a/tokio/src/runtime/task/core.rs +++ b/tokio/src/runtime/task/core.rs @@ -173,9 +173,6 @@ pub(crate) struct Header { /// The tracing ID for this instrumented task. #[cfg(all(tokio_unstable, feature = "tracing"))] pub(super) tracing_id: Option, - - /// The task's ID, is used for deciding which list to put it in. - pub(super) task_id: Id, } unsafe impl Send for Header {} @@ -213,14 +210,12 @@ impl Cell { fn new_header( state: State, vtable: &'static Vtable, - task_id: Id, #[cfg(all(tokio_unstable, feature = "tracing"))] tracing_id: Option, ) -> Header { Header { state, queue_next: UnsafeCell::new(None), vtable, - task_id, owner_id: UnsafeCell::new(None), #[cfg(all(tokio_unstable, feature = "tracing"))] tracing_id, @@ -234,7 +229,6 @@ impl Cell { header: new_header( state, vtable, - task_id, #[cfg(all(tokio_unstable, feature = "tracing"))] tracing_id, ), diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 96baaae1ad6..44ebe679773 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -17,6 +17,8 @@ use crate::loom::sync::atomic::{AtomicBool, Ordering}; use std::marker::PhantomData; use std::num::NonZeroU32; +use super::core::Header; + // The id from the module below is used to verify whether a given task is stored // in this OwnedTasks, or some other task. The counter starts at one so we can // use `None` for tasks not owned by any list. @@ -117,7 +119,10 @@ impl OwnedTasks { where S: Schedule, { - let mut lock = self.segment_inner(task.task_id() as usize); + // Safety: it is safe, because every task has one task_id + let task_id = unsafe { Header::get_id(task.header_ptr()) }; + let mut lock = self.segment_inner(task_id.0 as usize); + // check close flag, // it must be checked in the lock, for ensuring all tasks will shutdown after OwnedTasks has been closed if self.closed.load(Ordering::Acquire) { @@ -188,7 +193,9 @@ impl OwnedTasks { #[inline] unsafe fn remove_inner(&self, task: &Task) -> Option> { - let mut lock = self.segment_inner(task.task_id() as usize); + // Safety: it is safe, because every task has one task_id + let task_id = unsafe{Header::get_id(task.header_ptr())}; + let mut lock = self.segment_inner(task_id.0 as usize); let task = lock.remove(task.header_ptr())?; drop(lock); self.count.fetch_sub(1, Ordering::Relaxed); diff --git a/tokio/src/runtime/task/mod.rs b/tokio/src/runtime/task/mod.rs index 61b9af77b59..abf7cc266e7 100644 --- a/tokio/src/runtime/task/mod.rs +++ b/tokio/src/runtime/task/mod.rs @@ -361,10 +361,6 @@ impl Task { fn header_ptr(&self) -> NonNull
{ self.raw.header_ptr() } - - fn task_id(&self) -> u64 { - self.header().task_id.0 - } } impl Notified { From 63a76796f6120499c093855b0d49770e831b4eb2 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Thu, 28 Sep 2023 21:47:16 +0800 Subject: [PATCH 32/94] change owned_id to u64 back --- tokio/src/runtime/task/core.rs | 8 +++---- tokio/src/runtime/task/list.rs | 39 +++++++++++++++++++++++++--------- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/tokio/src/runtime/task/core.rs b/tokio/src/runtime/task/core.rs index e62f1160ee0..6f5867df574 100644 --- a/tokio/src/runtime/task/core.rs +++ b/tokio/src/runtime/task/core.rs @@ -17,7 +17,7 @@ use crate::runtime::task::state::State; use crate::runtime::task::{Id, Schedule}; use crate::util::linked_list; -use std::num::NonZeroU32; +use std::num::NonZeroU64; use std::pin::Pin; use std::ptr::NonNull; use std::task::{Context, Poll, Waker}; @@ -168,7 +168,7 @@ pub(crate) struct Header { /// The id is not unset when removed from a list because we want to be able /// to read the id without synchronization, even if it is concurrently being /// removed from the list. - pub(super) owner_id: UnsafeCell>, + pub(super) owner_id: UnsafeCell>, /// The tracing ID for this instrumented task. #[cfg(all(tokio_unstable, feature = "tracing"))] @@ -391,11 +391,11 @@ impl Header { // safety: The caller must guarantee exclusive access to this field, and // must ensure that the id is either `None` or the id of the OwnedTasks // containing this task. - pub(super) unsafe fn set_owner_id(&self, owner: NonZeroU32) { + pub(super) unsafe fn set_owner_id(&self, owner: NonZeroU64) { self.owner_id.with_mut(|ptr| *ptr = Some(owner)); } - pub(super) fn get_owner_id(&self) -> Option { + pub(super) fn get_owner_id(&self) -> Option { // safety: If there are concurrent writes, then that write has violated // the safety requirements on `set_owner_id`. unsafe { self.owner_id.with(|ptr| *ptr) } diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 44ebe679773..5b44f5c6e9b 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -15,7 +15,7 @@ use crate::util::linked_list::{Link, LinkedList}; use crate::loom::sync::atomic::{AtomicBool, Ordering}; use std::marker::PhantomData; -use std::num::NonZeroU32; +use std::num::NonZeroU64; use super::core::Header; @@ -28,20 +28,39 @@ use super::core::Header; // bug in Tokio, so we accept that certain bugs would not be caught if the two // mixed up runtimes happen to have the same id. -static NEXT_OWNED_TASKS_ID: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(1); +cfg_has_atomic_u64! { + use std::sync::atomic::AtomicU64; -fn get_next_id() -> NonZeroU32 { - loop { - let id = NEXT_OWNED_TASKS_ID.fetch_add(1, Ordering::Relaxed); - if let Some(id) = NonZeroU32::new(id) { - return id; + static NEXT_OWNED_TASKS_ID: AtomicU64 = AtomicU64::new(1); + + fn get_next_id() -> NonZeroU64 { + loop { + let id = NEXT_OWNED_TASKS_ID.fetch_add(1, Ordering::Relaxed); + if let Some(id) = NonZeroU64::new(id) { + return id; + } + } + } +} + +cfg_not_has_atomic_u64! { + use std::sync::atomic::AtomicU32; + + static NEXT_OWNED_TASKS_ID: AtomicU32 = AtomicU32::new(1); + + fn get_next_id() -> NonZeroU64 { + loop { + let id = NEXT_OWNED_TASKS_ID.fetch_add(1, Ordering::Relaxed); + if let Some(id) = NonZeroU64::new(u64::from(id)) { + return id; + } } } } pub(crate) struct OwnedTasks { lists: Box<[Mutex>]>, - pub(crate) id: NonZeroU32, + pub(crate) id: NonZeroU64, closed: AtomicBool, pub(crate) segment_size: u32, segment_mask: u32, @@ -52,7 +71,7 @@ type ListSement = LinkedList, as Link>::Target>; pub(crate) struct LocalOwnedTasks { inner: UnsafeCell>, - pub(crate) id: NonZeroU32, + pub(crate) id: NonZeroU64, _not_send_or_sync: PhantomData<*const ()>, } struct OwnedTasksInner { @@ -194,7 +213,7 @@ impl OwnedTasks { #[inline] unsafe fn remove_inner(&self, task: &Task) -> Option> { // Safety: it is safe, because every task has one task_id - let task_id = unsafe{Header::get_id(task.header_ptr())}; + let task_id = unsafe { Header::get_id(task.header_ptr()) }; let mut lock = self.segment_inner(task_id.0 as usize); let task = lock.remove(task.header_ptr())?; drop(lock); From b2010d7ae2cf64fbc4ea40b4a81c5d7d26017460 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Thu, 28 Sep 2023 23:07:26 +0800 Subject: [PATCH 33/94] refactor: use local_staic in loom --- .../runtime/scheduler/current_thread/mod.rs | 6 +++--- .../runtime/scheduler/multi_thread/handle.rs | 4 ++-- .../runtime/scheduler/multi_thread/worker.rs | 2 +- .../scheduler/multi_thread_alt/handle.rs | 4 ++-- tokio/src/runtime/task/id.rs | 17 ++++++++++++++--- tokio/src/runtime/task/list.rs | 18 +++++++++++------- tokio/src/runtime/tests/task.rs | 2 +- 7 files changed, 34 insertions(+), 19 deletions(-) diff --git a/tokio/src/runtime/scheduler/current_thread/mod.rs b/tokio/src/runtime/scheduler/current_thread/mod.rs index e8cff36b1cf..cb31e0fe887 100644 --- a/tokio/src/runtime/scheduler/current_thread/mod.rs +++ b/tokio/src/runtime/scheduler/current_thread/mod.rs @@ -264,7 +264,7 @@ fn shutdown2(mut core: Box, handle: &Handle) -> Box { drop(task); } - assert!(handle.shared.owned.is_closed()); + assert!(handle.shared.owned.is_shutdown()); assert!(handle.shared.owned.is_empty()); // Submit metrics @@ -547,10 +547,10 @@ cfg_metrics! { } cfg_unstable! { - use std::num::NonZeroU32; + use std::num::NonZeroU64; impl Handle { - pub(crate) fn owned_id(&self) -> NonZeroU32 { + pub(crate) fn owned_id(&self) -> NonZeroU64 { self.shared.owned.id } } diff --git a/tokio/src/runtime/scheduler/multi_thread/handle.rs b/tokio/src/runtime/scheduler/multi_thread/handle.rs index 957673d749f..568eb80af8b 100644 --- a/tokio/src/runtime/scheduler/multi_thread/handle.rs +++ b/tokio/src/runtime/scheduler/multi_thread/handle.rs @@ -60,10 +60,10 @@ impl Handle { } cfg_unstable! { - use std::num::NonZeroU32; + use std::num::NonZeroU64; impl Handle { - pub(crate) fn owned_id(&self) -> NonZeroU32 { + pub(crate) fn owned_id(&self) -> NonZeroU64 { self.shared.owned.id } } diff --git a/tokio/src/runtime/scheduler/multi_thread/worker.rs b/tokio/src/runtime/scheduler/multi_thread/worker.rs index 6e55f5df215..b94d1c08ba1 100644 --- a/tokio/src/runtime/scheduler/multi_thread/worker.rs +++ b/tokio/src/runtime/scheduler/multi_thread/worker.rs @@ -1160,7 +1160,7 @@ impl Handle { return; } - debug_assert!(self.shared.owned.is_closed()); + debug_assert!(self.shared.owned.is_shutdown()); debug_assert!(self.shared.owned.is_empty()); for mut core in cores.drain(..) { diff --git a/tokio/src/runtime/scheduler/multi_thread_alt/handle.rs b/tokio/src/runtime/scheduler/multi_thread_alt/handle.rs index b7288fe2302..d746bca1a18 100644 --- a/tokio/src/runtime/scheduler/multi_thread_alt/handle.rs +++ b/tokio/src/runtime/scheduler/multi_thread_alt/handle.rs @@ -59,10 +59,10 @@ impl Handle { } cfg_unstable! { - use std::num::NonZeroU32; + use std::num::NonZeroU64; impl Handle { - pub(crate) fn owned_id(&self) -> NonZeroU32 { + pub(crate) fn owned_id(&self) -> NonZeroU64 { self.shared.owned.id } } diff --git a/tokio/src/runtime/task/id.rs b/tokio/src/runtime/task/id.rs index dd5ae504582..82c8a7e7e90 100644 --- a/tokio/src/runtime/task/id.rs +++ b/tokio/src/runtime/task/id.rs @@ -74,11 +74,22 @@ impl fmt::Display for Id { impl Id { pub(crate) fn next() -> Self { - use crate::loom::sync::atomic::{Ordering::Relaxed, StaticAtomicU64}; + use crate::loom::sync::atomic::Ordering::Relaxed; + use crate::loom::sync::atomic::StaticAtomicU64; - static NEXT_ID: StaticAtomicU64 = StaticAtomicU64::new(1); + #[cfg(all(test, loom))] + { + crate::loom::lazy_static! { + static ref NEXT_ID: StaticAtomicU64 = StaticAtomicU64::new(1); + } + Self(NEXT_ID.fetch_add(1, Relaxed)) + } - Self(NEXT_ID.fetch_add(1, Relaxed)) + #[cfg(not(all(test, loom)))] + { + static NEXT_ID: StaticAtomicU64 = StaticAtomicU64::new(1); + Self(NEXT_ID.fetch_add(1, Relaxed)) + } } pub(crate) fn as_u64(&self) -> u64 { diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 5b44f5c6e9b..023867066da 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -8,10 +8,10 @@ use crate::future::Future; use crate::loom::cell::UnsafeCell; -use crate::loom::sync::atomic::AtomicUsize; use crate::loom::sync::{Mutex, MutexGuard}; use crate::runtime::task::{JoinHandle, LocalNotified, Notified, Schedule, Task}; use crate::util::linked_list::{Link, LinkedList}; +use std::sync::atomic::AtomicUsize; use crate::loom::sync::atomic::{AtomicBool, Ordering}; use std::marker::PhantomData; @@ -61,7 +61,8 @@ cfg_not_has_atomic_u64! { pub(crate) struct OwnedTasks { lists: Box<[Mutex>]>, pub(crate) id: NonZeroU64, - closed: AtomicBool, + closing: AtomicBool, + shutdown: AtomicBool, pub(crate) segment_size: u32, segment_mask: u32, count: AtomicUsize, @@ -94,7 +95,8 @@ impl OwnedTasks { } Self { lists: lists.into_boxed_slice(), - closed: AtomicBool::new(false), + closing: AtomicBool::new(false), + shutdown: AtomicBool::new(false), id: get_next_id(), segment_size, segment_mask, @@ -144,7 +146,7 @@ impl OwnedTasks { // check close flag, // it must be checked in the lock, for ensuring all tasks will shutdown after OwnedTasks has been closed - if self.closed.load(Ordering::Acquire) { + if self.closing.load(Ordering::Acquire) { drop(lock); task.shutdown(); return None; @@ -178,7 +180,7 @@ impl OwnedTasks { where S: Schedule, { - self.closed.store(true, Ordering::Release); + self.closing.store(true, Ordering::Release); for i in start..self.segment_size as usize + start { loop { let mut lock = self.segment_inner(i); @@ -192,6 +194,8 @@ impl OwnedTasks { }; } } + // we have shut down all tasks + self.shutdown.store(true, Ordering::Release) } pub(crate) fn active_tasks_count(&self) -> usize { @@ -226,8 +230,8 @@ impl OwnedTasks { self.lists[id & (self.segment_mask) as usize].lock() } - pub(crate) fn is_closed(&self) -> bool { - self.closed.load(Ordering::Acquire) + pub(crate) fn is_shutdown(&self) -> bool { + self.shutdown.load(Ordering::Acquire) } pub(crate) fn is_empty(&self) -> bool { diff --git a/tokio/src/runtime/tests/task.rs b/tokio/src/runtime/tests/task.rs index 8db5aadb901..8eafb48ef89 100644 --- a/tokio/src/runtime/tests/task.rs +++ b/tokio/src/runtime/tests/task.rs @@ -315,7 +315,7 @@ impl Runtime { } drop(core); - assert!(self.0.owned.is_closed()); + assert!(self.0.owned.is_shutdown()); assert!(self.0.owned.is_empty()); } } From 052e141071d14e7dca51448de7eef2b3d71f04f8 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Fri, 29 Sep 2023 16:11:36 +0800 Subject: [PATCH 34/94] fix: OwnedTasks get all locks first --- tokio/src/runtime/task/list.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 023867066da..96ea40528d1 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -246,13 +246,17 @@ cfg_taskdump! { where F: FnMut(&Task), { - for i in 0 .. self.lists.len() { - self.lists[i].lock().for_each(&mut f); + // while tracing, new tasks are not allowed to add, so we get all locks first + let mut guards = Vec::with_capacity(self.segment_size as usize); + for list in self.lists.as_ref() { + guards.push(list.lock()); + } + for guard in &mut guards{ + guard.for_each(&mut f); } } } } - impl LocalOwnedTasks { pub(crate) fn new() -> Self { Self { From 65670eb5513c2e8e8c16e9c15d7de8095c8cdae0 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Fri, 29 Sep 2023 22:14:41 +0800 Subject: [PATCH 35/94] fix: rm segment_size field of OwnedTasks, use method to return it instead --- tokio/src/runtime/scheduler/multi_thread/worker.rs | 2 +- .../src/runtime/scheduler/multi_thread_alt/worker.rs | 2 +- tokio/src/runtime/task/list.rs | 11 +++++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tokio/src/runtime/scheduler/multi_thread/worker.rs b/tokio/src/runtime/scheduler/multi_thread/worker.rs index b94d1c08ba1..776a42a4061 100644 --- a/tokio/src/runtime/scheduler/multi_thread/worker.rs +++ b/tokio/src/runtime/scheduler/multi_thread/worker.rs @@ -957,7 +957,7 @@ impl Core { // Start from a random inner list let start = self .rand - .fastrand_n(worker.handle.shared.owned.segment_size); + .fastrand_n(worker.handle.shared.owned.get_segment_size() as u32); // Signal to all tasks to shut down. worker .handle diff --git a/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs b/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs index 1533546e807..5f3dc0d0a6f 100644 --- a/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs +++ b/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs @@ -1462,7 +1462,7 @@ impl Shared { pub(super) fn shutdown_core(&self, handle: &Handle, mut core: Box) { // Start from a random inner list - let start = core.rand.fastrand_n(self.owned.segment_size as u32); + let start = core.rand.fastrand_n(self.owned.get_segment_size() as u32); self.owned.close_and_shutdown_all(start as usize); core.stats.submit(&self.worker_metrics[core.index]); diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 96ea40528d1..d5bf89cd38a 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -63,7 +63,6 @@ pub(crate) struct OwnedTasks { pub(crate) id: NonZeroU64, closing: AtomicBool, shutdown: AtomicBool, - pub(crate) segment_size: u32, segment_mask: u32, count: AtomicUsize, } @@ -98,7 +97,6 @@ impl OwnedTasks { closing: AtomicBool::new(false), shutdown: AtomicBool::new(false), id: get_next_id(), - segment_size, segment_mask, count: AtomicUsize::new(0), } @@ -181,7 +179,7 @@ impl OwnedTasks { S: Schedule, { self.closing.store(true, Ordering::Release); - for i in start..self.segment_size as usize + start { + for i in start..self.get_segment_size() + start { loop { let mut lock = self.segment_inner(i); match lock.pop_back() { @@ -198,6 +196,11 @@ impl OwnedTasks { self.shutdown.store(true, Ordering::Release) } + #[inline] + pub(crate) fn get_segment_size(&self) -> usize { + self.lists.len() + } + pub(crate) fn active_tasks_count(&self) -> usize { self.count.load(Ordering::Relaxed) } @@ -247,7 +250,7 @@ cfg_taskdump! { F: FnMut(&Task), { // while tracing, new tasks are not allowed to add, so we get all locks first - let mut guards = Vec::with_capacity(self.segment_size as usize); + let mut guards = Vec::with_capacity(self.get_segment_size() as usize); for list in self.lists.as_ref() { guards.push(list.lock()); } From 08d7d0c4fe5f6eee4df802de82543ee4c730e026 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Sun, 1 Oct 2023 20:01:04 +0800 Subject: [PATCH 36/94] feat: make spawn_concurrency_level to be a unstable api --- tokio/src/runtime/builder.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tokio/src/runtime/builder.rs b/tokio/src/runtime/builder.rs index e69b22950aa..2d4a3fdd2e4 100644 --- a/tokio/src/runtime/builder.rs +++ b/tokio/src/runtime/builder.rs @@ -445,6 +445,7 @@ impl Builder { /// /// This will panic if `val` is not larger than `0`. #[track_caller] + #[cfg(tokio_unstable)] pub fn spawn_concurrency_level(&mut self, mut val: usize) -> &mut Self { assert!(val > 0, "spawn concurrency level cannot be set to 0"); if val > 1 << 16 { From 26621d4d080af0abdd88536b58a1632a9f0c2b57 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Sun, 1 Oct 2023 20:14:33 +0800 Subject: [PATCH 37/94] feat: rm shutdown flag --- .../runtime/scheduler/current_thread/mod.rs | 1 - .../runtime/scheduler/multi_thread/worker.rs | 1 - tokio/src/runtime/task/list.rs | 18 +++++------------- tokio/src/runtime/tests/task.rs | 1 - 4 files changed, 5 insertions(+), 16 deletions(-) diff --git a/tokio/src/runtime/scheduler/current_thread/mod.rs b/tokio/src/runtime/scheduler/current_thread/mod.rs index cb31e0fe887..d4365711bde 100644 --- a/tokio/src/runtime/scheduler/current_thread/mod.rs +++ b/tokio/src/runtime/scheduler/current_thread/mod.rs @@ -264,7 +264,6 @@ fn shutdown2(mut core: Box, handle: &Handle) -> Box { drop(task); } - assert!(handle.shared.owned.is_shutdown()); assert!(handle.shared.owned.is_empty()); // Submit metrics diff --git a/tokio/src/runtime/scheduler/multi_thread/worker.rs b/tokio/src/runtime/scheduler/multi_thread/worker.rs index 776a42a4061..c9a561c09c3 100644 --- a/tokio/src/runtime/scheduler/multi_thread/worker.rs +++ b/tokio/src/runtime/scheduler/multi_thread/worker.rs @@ -1160,7 +1160,6 @@ impl Handle { return; } - debug_assert!(self.shared.owned.is_shutdown()); debug_assert!(self.shared.owned.is_empty()); for mut core in cores.drain(..) { diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index d5bf89cd38a..3f951ef08d0 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -61,8 +61,7 @@ cfg_not_has_atomic_u64! { pub(crate) struct OwnedTasks { lists: Box<[Mutex>]>, pub(crate) id: NonZeroU64, - closing: AtomicBool, - shutdown: AtomicBool, + closed: AtomicBool, segment_mask: u32, count: AtomicUsize, } @@ -94,8 +93,7 @@ impl OwnedTasks { } Self { lists: lists.into_boxed_slice(), - closing: AtomicBool::new(false), - shutdown: AtomicBool::new(false), + closed: AtomicBool::new(false), id: get_next_id(), segment_mask, count: AtomicUsize::new(0), @@ -144,7 +142,7 @@ impl OwnedTasks { // check close flag, // it must be checked in the lock, for ensuring all tasks will shutdown after OwnedTasks has been closed - if self.closing.load(Ordering::Acquire) { + if self.closed.load(Ordering::Acquire) { drop(lock); task.shutdown(); return None; @@ -178,7 +176,7 @@ impl OwnedTasks { where S: Schedule, { - self.closing.store(true, Ordering::Release); + self.closed.store(true, Ordering::Release); for i in start..self.get_segment_size() + start { loop { let mut lock = self.segment_inner(i); @@ -189,11 +187,9 @@ impl OwnedTasks { task.shutdown(); } None => break, - }; + } } } - // we have shut down all tasks - self.shutdown.store(true, Ordering::Release) } #[inline] @@ -233,10 +229,6 @@ impl OwnedTasks { self.lists[id & (self.segment_mask) as usize].lock() } - pub(crate) fn is_shutdown(&self) -> bool { - self.shutdown.load(Ordering::Acquire) - } - pub(crate) fn is_empty(&self) -> bool { self.count.load(Ordering::Relaxed) == 0 } diff --git a/tokio/src/runtime/tests/task.rs b/tokio/src/runtime/tests/task.rs index 8eafb48ef89..a0604505ccc 100644 --- a/tokio/src/runtime/tests/task.rs +++ b/tokio/src/runtime/tests/task.rs @@ -315,7 +315,6 @@ impl Runtime { } drop(core); - assert!(self.0.owned.is_shutdown()); assert!(self.0.owned.is_empty()); } } From 5b010bcbf7654e41535915387086a5446a1b9b48 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Sun, 1 Oct 2023 20:30:13 +0800 Subject: [PATCH 38/94] rm benches/spawn_concurrent.rs because it is unstable now in tokio --- benches/Cargo.toml | 7 -- benches/spawn_concurrent.rs | 177 ----------------------------------- tokio/src/runtime/builder.rs | 1 + 3 files changed, 1 insertion(+), 184 deletions(-) delete mode 100644 benches/spawn_concurrent.rs diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 8845d1ff1f2..1eea2e04489 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -85,10 +85,3 @@ harness = false name = "time_now" path = "time_now.rs" harness = false - -[[bench]] -name = "spawn_concurrent" -path = "spawn_concurrent.rs" -harness = false - - diff --git a/benches/spawn_concurrent.rs b/benches/spawn_concurrent.rs deleted file mode 100644 index 4fd59bfa7a7..00000000000 --- a/benches/spawn_concurrent.rs +++ /dev/null @@ -1,177 +0,0 @@ -use std::time::Instant; - -use criterion::{measurement::WallTime, *}; - -fn spawn_tasks_current_thread(c: &mut Criterion) { - let runtime = tokio::runtime::Builder::new_current_thread() - .build() - .unwrap(); - - c.bench_function("spawn_tasks_current_thread", move |b| { - b.iter_custom(|iters| { - let start = Instant::now(); - runtime.block_on(async { - black_box(spawn_job(iters as usize, 1).await); - }); - start.elapsed() - }) - }); -} - -fn spawn_tasks_current_thread_parallel(c: &mut Criterion) { - let runtime = tokio::runtime::Builder::new_current_thread() - .build() - .unwrap(); - - c.bench_function("spawn_tasks_current_thread_parallel", move |b| { - b.iter_custom(|iters| { - let start = Instant::now(); - runtime.block_on(async { - black_box(spawn_job(iters as usize, num_cpus::get_physical() * 2).await); - }); - start.elapsed() - }) - }); -} - -fn bench_create_runtime_multi_thread(c: &mut Criterion) { - let mut group = c.benchmark_group("create_runtime_multi_thread"); - create_multi_thread_runtime::<1>(&mut group); - create_multi_thread_runtime::<8>(&mut group); - create_multi_thread_runtime::<32>(&mut group); - create_multi_thread_runtime::<64>(&mut group); - create_multi_thread_runtime::<128>(&mut group); - create_multi_thread_runtime::<256>(&mut group); - create_multi_thread_runtime::<512>(&mut group); - create_multi_thread_runtime::<1024>(&mut group); - create_multi_thread_runtime::<2048>(&mut group); - create_multi_thread_runtime::<4096>(&mut group); -} - -fn create_multi_thread_runtime(g: &mut BenchmarkGroup) { - g.bench_function(format!("{:04}", S), |b| { - b.iter(|| { - let runtime = tokio::runtime::Builder::new_multi_thread() - .spawn_concurrency_level(black_box(S)) - .build() - .unwrap(); - drop(runtime); - }) - }); -} - -fn bench_parallel_spawn_multi_thread(c: &mut Criterion) { - let mut group = c.benchmark_group("spawn_parallel_multi_thread"); - spawn_tasks_parallel_multi_thread::<1>(&mut group); - spawn_tasks_parallel_multi_thread::<8>(&mut group); - spawn_tasks_parallel_multi_thread::<32>(&mut group); - spawn_tasks_parallel_multi_thread::<64>(&mut group); - spawn_tasks_parallel_multi_thread::<128>(&mut group); - spawn_tasks_parallel_multi_thread::<256>(&mut group); - spawn_tasks_parallel_multi_thread::<512>(&mut group); - spawn_tasks_parallel_multi_thread::<1024>(&mut group); - spawn_tasks_parallel_multi_thread::<2048>(&mut group); - spawn_tasks_parallel_multi_thread::<4096>(&mut group); -} - -fn spawn_tasks_parallel_multi_thread(g: &mut BenchmarkGroup) { - let runtime = tokio::runtime::Builder::new_multi_thread() - .spawn_concurrency_level(black_box(S)) - .build() - .unwrap(); - g.bench_function(format!("{:04}", S), |b| { - b.iter_custom(|iters| { - let start = Instant::now(); - runtime.block_on(async { - black_box(spawn_job(iters as usize, num_cpus::get_physical()).await); - }); - start.elapsed() - }) - }); -} - -async fn spawn_job(iters: usize, procs: usize) { - for _ in 0..procs { - let mut threads_handles = Vec::with_capacity(procs); - threads_handles.push(tokio::spawn(async move { - let mut thread_handles = Vec::with_capacity(iters / procs); - for _ in 0..iters / procs { - thread_handles.push(tokio::spawn(async { - let val = 1 + 1; - tokio::task::yield_now().await; - black_box(val) - })); - } - for handle in thread_handles { - handle.await.unwrap(); - } - })); - for handle in threads_handles { - handle.await.unwrap(); - } - } -} - -fn bench_shutdown_parallel_multi_thread(c: &mut Criterion) { - let mut group = c.benchmark_group("shutdown_runtime_multi_thread"); - shutdown_tasks_parallel::<1>(&mut group); - shutdown_tasks_parallel::<8>(&mut group); - shutdown_tasks_parallel::<32>(&mut group); - shutdown_tasks_parallel::<64>(&mut group); - shutdown_tasks_parallel::<128>(&mut group); - shutdown_tasks_parallel::<256>(&mut group); - shutdown_tasks_parallel::<512>(&mut group); - shutdown_tasks_parallel::<1024>(&mut group); - shutdown_tasks_parallel::<2048>(&mut group); - shutdown_tasks_parallel::<4096>(&mut group); - group.finish(); -} - -fn shutdown_tasks_parallel(g: &mut BenchmarkGroup) { - g.bench_function(format!("{:04}", S), |b| { - b.iter_custom(|iters| { - let runtime = tokio::runtime::Builder::new_multi_thread() - .spawn_concurrency_level(black_box(S)) - .enable_time() - .build() - .unwrap(); - runtime.block_on(async { - black_box(job_shutdown(iters as usize, num_cpus::get_physical()).await); - }); - let start = Instant::now(); - drop(runtime); - start.elapsed() - }) - }); -} - -async fn job_shutdown(iters: usize, procs: usize) { - let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); - for _ in 0..procs { - let tx = tx.clone(); - tokio::spawn(async move { - for _ in 0..iters / procs { - let tx = tx.clone(); - tokio::spawn(async move { - let val = 1 + 1; - tx.send(()).unwrap(); - tokio::time::sleep(tokio::time::Duration::from_secs(1000)).await; // it will never return - black_box(val) - }); - } - }); - } - for _ in 0..(iters / procs) * procs { - rx.recv().await; - } -} - -criterion_group!( - benches, - spawn_tasks_current_thread, - spawn_tasks_current_thread_parallel, - bench_create_runtime_multi_thread, - bench_parallel_spawn_multi_thread, - bench_shutdown_parallel_multi_thread, -); -criterion_main!(benches); diff --git a/tokio/src/runtime/builder.rs b/tokio/src/runtime/builder.rs index 2d4a3fdd2e4..77dcae5b79f 100644 --- a/tokio/src/runtime/builder.rs +++ b/tokio/src/runtime/builder.rs @@ -446,6 +446,7 @@ impl Builder { /// This will panic if `val` is not larger than `0`. #[track_caller] #[cfg(tokio_unstable)] + #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))] pub fn spawn_concurrency_level(&mut self, mut val: usize) -> &mut Self { assert!(val > 0, "spawn concurrency level cannot be set to 0"); if val > 1 << 16 { From 66fa190064344e80a65454cbf01412567da4bf55 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Tue, 3 Oct 2023 11:37:29 +0800 Subject: [PATCH 39/94] use get_unchecked to get segment lock --- tokio/src/runtime/task/list.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 3f951ef08d0..affd9b099b5 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -226,7 +226,12 @@ impl OwnedTasks { #[inline] fn segment_inner(&self, id: usize) -> MutexGuard<'_, ListSement> { - self.lists[id & (self.segment_mask) as usize].lock() + // Safety: this modulo operation ensures it is safe here. + unsafe { + self.lists + .get_unchecked(id & (self.segment_mask) as usize) + .lock() + } } pub(crate) fn is_empty(&self) -> bool { From 2ac0b9637d0e62c4d60b9eb361cb336ac1c7106e Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Sun, 15 Oct 2023 22:50:49 +0800 Subject: [PATCH 40/94] feat: drop lock promptly and explicitly --- tokio/src/runtime/task/list.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index affd9b099b5..1741b97c4f5 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -180,9 +180,10 @@ impl OwnedTasks { for i in start..self.get_segment_size() + start { loop { let mut lock = self.segment_inner(i); - match lock.pop_back() { + let task = lock.pop_back(); + drop(lock); + match task { Some(task) => { - drop(lock); self.count.fetch_sub(1, Ordering::Relaxed); task.shutdown(); } @@ -218,10 +219,12 @@ impl OwnedTasks { // Safety: it is safe, because every task has one task_id let task_id = unsafe { Header::get_id(task.header_ptr()) }; let mut lock = self.segment_inner(task_id.0 as usize); - let task = lock.remove(task.header_ptr())?; + let task = lock.remove(task.header_ptr()); drop(lock); - self.count.fetch_sub(1, Ordering::Relaxed); - Some(task) + if task.is_some() { + self.count.fetch_sub(1, Ordering::Relaxed); + } + task } #[inline] From 47820b383eb0e24f5dfc814712985ff7a88602a8 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Mon, 16 Oct 2023 01:21:07 +0800 Subject: [PATCH 41/94] feat: move the atomic operations of count into the lock --- tokio/src/runtime/task/list.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 1741b97c4f5..87c026315d4 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -148,7 +148,6 @@ impl OwnedTasks { return None; } lock.push_front(task); - drop(lock); self.count.fetch_add(1, Ordering::Relaxed); Some(notified) } @@ -181,10 +180,10 @@ impl OwnedTasks { loop { let mut lock = self.segment_inner(i); let task = lock.pop_back(); - drop(lock); match task { Some(task) => { self.count.fetch_sub(1, Ordering::Relaxed); + drop(lock); task.shutdown(); } None => break, @@ -220,7 +219,6 @@ impl OwnedTasks { let task_id = unsafe { Header::get_id(task.header_ptr()) }; let mut lock = self.segment_inner(task_id.0 as usize); let task = lock.remove(task.header_ptr()); - drop(lock); if task.is_some() { self.count.fetch_sub(1, Ordering::Relaxed); } From 7ef0265cc6144b09f52e493b8e97e8414b393a5c Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Mon, 23 Oct 2023 14:40:20 +0800 Subject: [PATCH 42/94] first commit --- .../runtime/scheduler/multi_thread/worker.rs | 2 +- .../scheduler/multi_thread_alt/worker.rs | 2 +- tokio/src/runtime/task/list.rs | 106 ++++----------- tokio/src/runtime/task/mod.rs | 15 +++ tokio/src/util/mod.rs | 11 ++ tokio/src/util/shared_list.rs | 121 ++++++++++++++++++ 6 files changed, 173 insertions(+), 84 deletions(-) create mode 100644 tokio/src/util/shared_list.rs diff --git a/tokio/src/runtime/scheduler/multi_thread/worker.rs b/tokio/src/runtime/scheduler/multi_thread/worker.rs index ce7e7150ba8..e96ae973b57 100644 --- a/tokio/src/runtime/scheduler/multi_thread/worker.rs +++ b/tokio/src/runtime/scheduler/multi_thread/worker.rs @@ -957,7 +957,7 @@ impl Core { // Start from a random inner list let start = self .rand - .fastrand_n(worker.handle.shared.owned.get_segment_size() as u32); + .fastrand_n(worker.handle.shared.owned.get_shard_size() as u32); // Signal to all tasks to shut down. worker .handle diff --git a/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs b/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs index 5f3dc0d0a6f..000f646e0c8 100644 --- a/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs +++ b/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs @@ -1462,7 +1462,7 @@ impl Shared { pub(super) fn shutdown_core(&self, handle: &Handle, mut core: Box) { // Start from a random inner list - let start = core.rand.fastrand_n(self.owned.get_segment_size() as u32); + let start = core.rand.fastrand_n(self.owned.get_shard_size() as u32); self.owned.close_and_shutdown_all(start as usize); core.stats.submit(&self.worker_metrics[core.index]); diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 87c026315d4..22ea845afc4 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -8,17 +8,14 @@ use crate::future::Future; use crate::loom::cell::UnsafeCell; -use crate::loom::sync::{Mutex, MutexGuard}; use crate::runtime::task::{JoinHandle, LocalNotified, Notified, Schedule, Task}; use crate::util::linked_list::{Link, LinkedList}; -use std::sync::atomic::AtomicUsize; +use crate::util::shared_list; use crate::loom::sync::atomic::{AtomicBool, Ordering}; use std::marker::PhantomData; use std::num::NonZeroU64; -use super::core::Header; - // The id from the module below is used to verify whether a given task is stored // in this OwnedTasks, or some other task. The counter starts at one so we can // use `None` for tasks not owned by any list. @@ -59,14 +56,12 @@ cfg_not_has_atomic_u64! { } pub(crate) struct OwnedTasks { - lists: Box<[Mutex>]>, + list: List, pub(crate) id: NonZeroU64, closed: AtomicBool, - segment_mask: u32, - count: AtomicUsize, } -type ListSement = LinkedList, as Link>::Target>; +type List = shared_list::ShardedList, as Link>::Target>; pub(crate) struct LocalOwnedTasks { inner: UnsafeCell>, @@ -80,23 +75,10 @@ struct OwnedTasksInner { impl OwnedTasks { pub(crate) fn new(concurrency_level: u32) -> Self { - // Find power-of-two sizes best matching arguments - let mut segment_size = 1; - while segment_size < concurrency_level { - segment_size <<= 1; - } - let segment_mask = segment_size - 1; - - let mut lists = Vec::with_capacity(segment_size as usize); - for _ in 0..segment_size { - lists.push(Mutex::new(LinkedList::new())) - } Self { - lists: lists.into_boxed_slice(), + list: List::new(concurrency_level as usize), closed: AtomicBool::new(false), id: get_next_id(), - segment_mask, - count: AtomicUsize::new(0), } } @@ -128,27 +110,18 @@ impl OwnedTasks { // to the field. task.header().set_owner_id(self.id); } - self.push_inner(task, notified) - } - - #[inline] - pub(crate) fn push_inner(&self, task: Task, notified: Notified) -> Option> - where - S: Schedule, - { - // Safety: it is safe, because every task has one task_id - let task_id = unsafe { Header::get_id(task.header_ptr()) }; - let mut lock = self.segment_inner(task_id.0 as usize); - - // check close flag, - // it must be checked in the lock, for ensuring all tasks will shutdown after OwnedTasks has been closed + // check close flag if self.closed.load(Ordering::Acquire) { - drop(lock); task.shutdown(); return None; } - lock.push_front(task); - self.count.fetch_add(1, Ordering::Relaxed); + self.list.push(task); + + // double check close flag for ensuring all tasks will shutdown after OwnedTasks has been closed, + // it should be completed quickly. + if self.closed.load(Ordering::Acquire) { + self.close_and_shutdown_all(0); + } Some(notified) } @@ -167,23 +140,21 @@ impl OwnedTasks { /// Shuts down all tasks in the collection. This call also closes the /// collection, preventing new items from being added. + /// /// The parameter start should be random among different worker threads /// to reduce lock conflicts during shutdown. - /// Initiate shutting down the segment indexed by the start, and reset to 0 - /// once the segment_size is reached, continuing until start - 1, it works like a ring. + /// Initiate shutting down the shard indexed by the start, and reset to 0 + /// once the shar_size is reached, continuing until start - 1, it works like a ring. pub(crate) fn close_and_shutdown_all(&self, start: usize) where S: Schedule, { self.closed.store(true, Ordering::Release); - for i in start..self.get_segment_size() + start { + for i in start..self.get_shard_size() + start { loop { - let mut lock = self.segment_inner(i); - let task = lock.pop_back(); + let task = self.list.pop_back(i); match task { Some(task) => { - self.count.fetch_sub(1, Ordering::Relaxed); - drop(lock); task.shutdown(); } None => break, @@ -193,12 +164,12 @@ impl OwnedTasks { } #[inline] - pub(crate) fn get_segment_size(&self) -> usize { - self.lists.len() + pub(crate) fn get_shard_size(&self) -> usize { + self.list.shard_size() } pub(crate) fn active_tasks_count(&self) -> usize { - self.count.load(Ordering::Relaxed) + self.list.len() } pub(crate) fn remove(&self, task: &Task) -> Option> { @@ -210,54 +181,25 @@ impl OwnedTasks { // safety: We just checked that the provided task is not in some other // linked list. - unsafe { self.remove_inner(task) } - } - - #[inline] - unsafe fn remove_inner(&self, task: &Task) -> Option> { - // Safety: it is safe, because every task has one task_id - let task_id = unsafe { Header::get_id(task.header_ptr()) }; - let mut lock = self.segment_inner(task_id.0 as usize); - let task = lock.remove(task.header_ptr()); - if task.is_some() { - self.count.fetch_sub(1, Ordering::Relaxed); - } - task - } - - #[inline] - fn segment_inner(&self, id: usize) -> MutexGuard<'_, ListSement> { - // Safety: this modulo operation ensures it is safe here. - unsafe { - self.lists - .get_unchecked(id & (self.segment_mask) as usize) - .lock() - } + unsafe { self.list.remove(task.header_ptr()) } } pub(crate) fn is_empty(&self) -> bool { - self.count.load(Ordering::Relaxed) == 0 + self.list.is_empty() } } cfg_taskdump! { impl OwnedTasks { - /// Locks the tasks, and calls `f` on an iterator over them. pub(crate) fn for_each(&self, mut f: F) where F: FnMut(&Task), { - // while tracing, new tasks are not allowed to add, so we get all locks first - let mut guards = Vec::with_capacity(self.get_segment_size() as usize); - for list in self.lists.as_ref() { - guards.push(list.lock()); - } - for guard in &mut guards{ - guard.for_each(&mut f); - } + self.list.for_each(&mut f); } } } + impl LocalOwnedTasks { pub(crate) fn new() -> Self { Self { diff --git a/tokio/src/runtime/task/mod.rs b/tokio/src/runtime/task/mod.rs index ff1a455df3b..32b719e131b 100644 --- a/tokio/src/runtime/task/mod.rs +++ b/tokio/src/runtime/task/mod.rs @@ -208,6 +208,7 @@ cfg_taskdump! { use crate::future::Future; use crate::util::linked_list; +use crate::util::shared_list; use std::marker::PhantomData; use std::ptr::NonNull; @@ -503,3 +504,17 @@ unsafe impl linked_list::Link for Task { self::core::Trailer::addr_of_owned(Header::get_trailer(target)) } } + +/// # Safety +/// +/// Tasks are pinned. +unsafe impl shared_list::ShardedListItem for Task { + unsafe fn get_shared_id(target: NonNull) -> usize { + let task_id = unsafe { Header::get_id(target) }.0; + if cfg!(target_pointer_width = "32") { + (task_id & 0xFFFFFFFF) as usize + } else { + task_id as usize + } + } +} diff --git a/tokio/src/util/mod.rs b/tokio/src/util/mod.rs index 7d4cd5f9c7c..95d3998ff7b 100644 --- a/tokio/src/util/mod.rs +++ b/tokio/src/util/mod.rs @@ -42,6 +42,17 @@ pub(crate) use wake_list::WakeList; ))] pub(crate) mod linked_list; +#[cfg(any( + feature = "fs", + feature = "net", + feature = "process", + feature = "rt", + feature = "sync", + feature = "signal", + feature = "time", +))] +pub(crate) mod shared_list; + #[cfg(any(feature = "rt", feature = "macros"))] pub(crate) mod rand; diff --git a/tokio/src/util/shared_list.rs b/tokio/src/util/shared_list.rs new file mode 100644 index 00000000000..82137b2f533 --- /dev/null +++ b/tokio/src/util/shared_list.rs @@ -0,0 +1,121 @@ +#![cfg_attr(not(feature = "full"), allow(dead_code))] + +use std::{ptr::NonNull, sync::atomic::Ordering}; + +use crate::loom::sync::{atomic::AtomicUsize, Mutex, MutexGuard}; + +use super::linked_list::{Link, LinkedList}; + +/// An intrusive linked list supporting high concurrent updates. +/// +/// It relies on `LinkedList`, so it is the caller's +/// responsibility to ensure the list is empty before dropping it. +/// +/// Note: Due to its inner sharded design, the order of node cannot be guaranteed. +pub(crate) struct ShardedList { + lists: Box<[Mutex>]>, + count: AtomicUsize, + shard_mask: usize, +} + +pub(crate) unsafe trait ShardedListItem: Link { + // The returned id is used to pick which list this item should go into. + unsafe fn get_shared_id(target: NonNull) -> usize; +} + +impl ShardedList { + /// Creates a new and empty sharded linked list under specified size + pub(crate) fn new(sharded_size: usize) -> Self { + // Find power-of-two sizes best matching arguments + let mut size = 1; + while size < sharded_size { + size <<= 1; + } + let shard_mask = size - 1; + + let mut lists = Vec::with_capacity(size as usize); + for _ in 0..size { + lists.push(Mutex::new(LinkedList::::new())) + } + Self { + lists: lists.into_boxed_slice(), + count: AtomicUsize::new(0), + shard_mask, + } + } +} + +impl ShardedList { + /// Adds an element into the list. + pub(crate) fn push(&self, val: L::Handle) { + // Safety: it is safe, because every ShardedListItem has an id for shard. + let id = unsafe { L::get_shared_id(L::as_raw(&val)) }; + let mut lock = self.shard_inner(id); + lock.push_front(val); + self.count.fetch_add(1, Ordering::Relaxed); + } + + /// Removes the last element from a list specified by shard_id and returns it, or None if it is + /// empty. + pub(crate) fn pop_back(&self, shard_id: usize) -> Option { + let mut lock = self.shard_inner(shard_id); + lock.pop_back() + } + + /// Returns whether the linked list does not contain any node + pub(crate) fn is_empty(&self) -> bool { + self.count.load(Ordering::Relaxed) == 0 + } + + /// Removes the specified node from the list + /// + /// # Safety + /// + /// The caller **must** ensure that exactly one of the following is true: + /// - `node` is currently contained by `self`, + /// - `node` is not contained by any list, + /// - `node` is currently contained by some other `GuardedLinkedList`. + pub(crate) unsafe fn remove(&self, node: NonNull) -> Option { + let id = L::get_shared_id(node); + let mut lock = self.shard_inner(id); + let node = lock.remove(node); + if node.is_some() { + self.count.fetch_sub(1, Ordering::Relaxed); + } + node + } + + /// Gets the count of elements in this list. + pub(crate) fn len(&self) -> usize { + self.count.load(Ordering::Relaxed) + } + + /// Gets the shard size of this SharedList. + /// Used to help us to decide the parameter `shard_id`` of the `pop_back` method. + pub(crate) fn shard_size(&self) -> usize { + self.shard_mask + 1 + } + + #[inline] + fn shard_inner(&self, id: usize) -> MutexGuard<'_, LinkedList::Target>> { + // Safety: this modulo operation ensures it is safe here. + unsafe { self.lists.get_unchecked(id & self.shard_mask).lock() } + } +} + +cfg_taskdump! { + impl ShardedList { + pub(crate) fn for_each(&mut self, f: &mut F) + where + F: FnMut(&L::Handle), + { + let mut guards = vec![]; + for list in self.lists.iter(){ + guards.push(list.lock()); + } + for g in &mut guards{ + g.for_each(f); + } + } + } +} From 01da1ed5b3695121174f58ea3cf4538a8e00bd31 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Mon, 23 Oct 2023 16:40:11 +0800 Subject: [PATCH 43/94] add Safety --- tokio/src/util/shared_list.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tokio/src/util/shared_list.rs b/tokio/src/util/shared_list.rs index 82137b2f533..5ddbe969330 100644 --- a/tokio/src/util/shared_list.rs +++ b/tokio/src/util/shared_list.rs @@ -8,7 +8,7 @@ use super::linked_list::{Link, LinkedList}; /// An intrusive linked list supporting high concurrent updates. /// -/// It relies on `LinkedList`, so it is the caller's +/// It currently relies on `LinkedList`, so it is the caller's /// responsibility to ensure the list is empty before dropping it. /// /// Note: Due to its inner sharded design, the order of node cannot be guaranteed. @@ -18,6 +18,14 @@ pub(crate) struct ShardedList { shard_mask: usize, } +/// Defines the id of a node in a linked list, different ids cause inner list nodes +/// to be scattered in different mutexs. +/// +/// # Safety +/// +/// Implementations must guarantee that `Target` types are pinned in memory. In +/// other words, when a node is inserted, the value will not be moved as long as +/// it is stored in the list. pub(crate) unsafe trait ShardedListItem: Link { // The returned id is used to pick which list this item should go into. unsafe fn get_shared_id(target: NonNull) -> usize; From 6f5eaa2d8b03057c5ed9945e424431e2629b961b Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Mon, 23 Oct 2023 16:43:28 +0800 Subject: [PATCH 44/94] mutable ref to immutable ref --- tokio/src/util/shared_list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/util/shared_list.rs b/tokio/src/util/shared_list.rs index 5ddbe969330..649ec3b552c 100644 --- a/tokio/src/util/shared_list.rs +++ b/tokio/src/util/shared_list.rs @@ -113,7 +113,7 @@ impl ShardedList { cfg_taskdump! { impl ShardedList { - pub(crate) fn for_each(&mut self, f: &mut F) + pub(crate) fn for_each(&self, f: &mut F) where F: FnMut(&L::Handle), { From 3257cb72c1f71c247c23afd74ca8fb76d80d922e Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Mon, 23 Oct 2023 16:51:19 +0800 Subject: [PATCH 45/94] use std AtomicUsize --- tokio/src/util/shared_list.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tokio/src/util/shared_list.rs b/tokio/src/util/shared_list.rs index 649ec3b552c..a2adfa3bade 100644 --- a/tokio/src/util/shared_list.rs +++ b/tokio/src/util/shared_list.rs @@ -2,7 +2,8 @@ use std::{ptr::NonNull, sync::atomic::Ordering}; -use crate::loom::sync::{atomic::AtomicUsize, Mutex, MutexGuard}; +use crate::loom::sync::{Mutex, MutexGuard}; +use std::sync::atomic::AtomicUsize; use super::linked_list::{Link, LinkedList}; From 7b101ee629de520bf081130c0a82b74120fda85b Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Mon, 23 Oct 2023 17:15:19 +0800 Subject: [PATCH 46/94] fix: count sub in pop_back --- tokio/src/util/shared_list.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tokio/src/util/shared_list.rs b/tokio/src/util/shared_list.rs index a2adfa3bade..d9f02de4f3b 100644 --- a/tokio/src/util/shared_list.rs +++ b/tokio/src/util/shared_list.rs @@ -68,7 +68,11 @@ impl ShardedList { /// empty. pub(crate) fn pop_back(&self, shard_id: usize) -> Option { let mut lock = self.shard_inner(shard_id); - lock.pop_back() + let node = lock.pop_back(); + if node.is_some() { + self.count.fetch_sub(1, Ordering::Relaxed); + } + node } /// Returns whether the linked list does not contain any node From 608d2c66a2385ccb3da13d48adb8cbc7fa76a603 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Mon, 23 Oct 2023 18:52:10 +0800 Subject: [PATCH 47/94] refactor doube check closed flag in bind_inner --- tokio/src/runtime/task/list.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 22ea845afc4..50e44743f2a 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -120,7 +120,11 @@ impl OwnedTasks { // double check close flag for ensuring all tasks will shutdown after OwnedTasks has been closed, // it should be completed quickly. if self.closed.load(Ordering::Acquire) { - self.close_and_shutdown_all(0); + for i in 0..self.get_shard_size() { + if let Some(task) = self.list.pop_back(i) { + task.shutdown(); + } + } } Some(notified) } From 1c214e0964d7c4c236745b76024bd6186542e98a Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Mon, 23 Oct 2023 20:36:41 +0800 Subject: [PATCH 48/94] cast task_id to usize --- tokio/src/runtime/task/list.rs | 14 ++++++++------ tokio/src/runtime/task/mod.rs | 9 +++------ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 50e44743f2a..7ab8d1f6863 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -16,6 +16,8 @@ use crate::loom::sync::atomic::{AtomicBool, Ordering}; use std::marker::PhantomData; use std::num::NonZeroU64; +use super::core::Header; + // The id from the module below is used to verify whether a given task is stored // in this OwnedTasks, or some other task. The counter starts at one so we can // use `None` for tasks not owned by any list. @@ -110,6 +112,9 @@ impl OwnedTasks { // to the field. task.header().set_owner_id(self.id); } + // safety: We just set task_id for this task + let task_id = unsafe { Header::get_id(task.header_ptr()) }; + // check close flag if self.closed.load(Ordering::Acquire) { task.shutdown(); @@ -117,13 +122,10 @@ impl OwnedTasks { } self.list.push(task); - // double check close flag for ensuring all tasks will shutdown after OwnedTasks has been closed, - // it should be completed quickly. + // check close flag again, for ensuring all tasks will shutdown after OwnedTasks has been closed. if self.closed.load(Ordering::Acquire) { - for i in 0..self.get_shard_size() { - if let Some(task) = self.list.pop_back(i) { - task.shutdown(); - } + if let Some(task) = self.list.pop_back(task_id.0 as usize) { + task.shutdown(); } } Some(notified) diff --git a/tokio/src/runtime/task/mod.rs b/tokio/src/runtime/task/mod.rs index 32b719e131b..30f0da0e0a9 100644 --- a/tokio/src/runtime/task/mod.rs +++ b/tokio/src/runtime/task/mod.rs @@ -510,11 +510,8 @@ unsafe impl linked_list::Link for Task { /// Tasks are pinned. unsafe impl shared_list::ShardedListItem for Task { unsafe fn get_shared_id(target: NonNull) -> usize { - let task_id = unsafe { Header::get_id(target) }.0; - if cfg!(target_pointer_width = "32") { - (task_id & 0xFFFFFFFF) as usize - } else { - task_id as usize - } + let task_id = unsafe { Header::get_id(target) }; + // in 32-bits machine, the lower 32 bits will be reserved + return task_id.0 as usize; } } From e24db6de8a46eb84f991836d53041d7c4ae3ef54 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Mon, 23 Oct 2023 20:58:49 +0800 Subject: [PATCH 49/94] use ShardGuard --- tokio/src/runtime/task/list.rs | 18 ++------ tokio/src/runtime/task/mod.rs | 4 +- tokio/src/util/mod.rs | 2 +- .../util/{shared_list.rs => sharded_list.rs} | 45 ++++++++++++++----- 4 files changed, 42 insertions(+), 27 deletions(-) rename tokio/src/util/{shared_list.rs => sharded_list.rs} (82%) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 7ab8d1f6863..cae52cd3e4d 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -10,14 +10,12 @@ use crate::future::Future; use crate::loom::cell::UnsafeCell; use crate::runtime::task::{JoinHandle, LocalNotified, Notified, Schedule, Task}; use crate::util::linked_list::{Link, LinkedList}; -use crate::util::shared_list; +use crate::util::sharded_list; use crate::loom::sync::atomic::{AtomicBool, Ordering}; use std::marker::PhantomData; use std::num::NonZeroU64; -use super::core::Header; - // The id from the module below is used to verify whether a given task is stored // in this OwnedTasks, or some other task. The counter starts at one so we can // use `None` for tasks not owned by any list. @@ -63,7 +61,7 @@ pub(crate) struct OwnedTasks { closed: AtomicBool, } -type List = shared_list::ShardedList, as Link>::Target>; +type List = sharded_list::ShardedList, as Link>::Target>; pub(crate) struct LocalOwnedTasks { inner: UnsafeCell>, @@ -112,22 +110,14 @@ impl OwnedTasks { // to the field. task.header().set_owner_id(self.id); } - // safety: We just set task_id for this task - let task_id = unsafe { Header::get_id(task.header_ptr()) }; + let shard = self.list.lock_shard(&task); // check close flag if self.closed.load(Ordering::Acquire) { task.shutdown(); return None; } - self.list.push(task); - - // check close flag again, for ensuring all tasks will shutdown after OwnedTasks has been closed. - if self.closed.load(Ordering::Acquire) { - if let Some(task) = self.list.pop_back(task_id.0 as usize) { - task.shutdown(); - } - } + shard.push(task); Some(notified) } diff --git a/tokio/src/runtime/task/mod.rs b/tokio/src/runtime/task/mod.rs index 30f0da0e0a9..85cf8979025 100644 --- a/tokio/src/runtime/task/mod.rs +++ b/tokio/src/runtime/task/mod.rs @@ -208,7 +208,7 @@ cfg_taskdump! { use crate::future::Future; use crate::util::linked_list; -use crate::util::shared_list; +use crate::util::sharded_list; use std::marker::PhantomData; use std::ptr::NonNull; @@ -508,7 +508,7 @@ unsafe impl linked_list::Link for Task { /// # Safety /// /// Tasks are pinned. -unsafe impl shared_list::ShardedListItem for Task { +unsafe impl sharded_list::ShardedListItem for Task { unsafe fn get_shared_id(target: NonNull) -> usize { let task_id = unsafe { Header::get_id(target) }; // in 32-bits machine, the lower 32 bits will be reserved diff --git a/tokio/src/util/mod.rs b/tokio/src/util/mod.rs index 95d3998ff7b..59257a6ab95 100644 --- a/tokio/src/util/mod.rs +++ b/tokio/src/util/mod.rs @@ -51,7 +51,7 @@ pub(crate) mod linked_list; feature = "signal", feature = "time", ))] -pub(crate) mod shared_list; +pub(crate) mod sharded_list; #[cfg(any(feature = "rt", feature = "macros"))] pub(crate) mod rand; diff --git a/tokio/src/util/shared_list.rs b/tokio/src/util/sharded_list.rs similarity index 82% rename from tokio/src/util/shared_list.rs rename to tokio/src/util/sharded_list.rs index d9f02de4f3b..5256de48f53 100644 --- a/tokio/src/util/shared_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -54,13 +54,18 @@ impl ShardedList { } } +pub(crate) struct ShardGuard<'a, L, T> { + lock: MutexGuard<'a, LinkedList>, + count: &'a AtomicUsize, + id: usize, +} + impl ShardedList { - /// Adds an element into the list. + #[allow(dead_code)] + /// Push a value to this shard. pub(crate) fn push(&self, val: L::Handle) { - // Safety: it is safe, because every ShardedListItem has an id for shard. - let id = unsafe { L::get_shared_id(L::as_raw(&val)) }; - let mut lock = self.shard_inner(id); - lock.push_front(val); + let shard = self.lock_shard(&val); + shard.push(val); self.count.fetch_add(1, Ordering::Relaxed); } @@ -75,11 +80,6 @@ impl ShardedList { node } - /// Returns whether the linked list does not contain any node - pub(crate) fn is_empty(&self) -> bool { - self.count.load(Ordering::Relaxed) == 0 - } - /// Removes the specified node from the list /// /// # Safety @@ -98,11 +98,26 @@ impl ShardedList { node } + /// Gets the lock of ShardedList, lets us have the write permission. + pub(crate) fn lock_shard(&self, val: &L::Handle) -> ShardGuard<'_, L, L::Target> { + let id = unsafe { L::get_shared_id(L::as_raw(&val)) }; + ShardGuard { + lock: self.shard_inner(id), + count: &self.count, + id, + } + } + /// Gets the count of elements in this list. pub(crate) fn len(&self) -> usize { self.count.load(Ordering::Relaxed) } + /// Returns whether the linked list does not contain any node + pub(crate) fn is_empty(&self) -> bool { + self.len() == 0 + } + /// Gets the shard size of this SharedList. /// Used to help us to decide the parameter `shard_id`` of the `pop_back` method. pub(crate) fn shard_size(&self) -> usize { @@ -116,6 +131,16 @@ impl ShardedList { } } +impl<'a, L: ShardedListItem> ShardGuard<'a, L, L::Target> { + /// Push a value to this shard. + pub(crate) fn push(mut self, val: L::Handle) { + let id = unsafe { L::get_shared_id(L::as_raw(&val)) }; + assert_eq!(id, self.id); + self.lock.push_front(val); + self.count.fetch_add(1, Ordering::Relaxed); + } +} + cfg_taskdump! { impl ShardedList { pub(crate) fn for_each(&self, f: &mut F) From 3c90918fecb660d03f5d56e2821f92e412c46606 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Mon, 23 Oct 2023 21:06:15 +0800 Subject: [PATCH 50/94] update comments --- tokio/src/runtime/task/list.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index cae52cd3e4d..ae307ee0d0a 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -112,7 +112,8 @@ impl OwnedTasks { } let shard = self.list.lock_shard(&task); - // check close flag + // check the closed flag in the lock for ensuring all tasks + // will shutdown after ownedTasks has been closed. if self.closed.load(Ordering::Acquire) { task.shutdown(); return None; From cd5fb2047c03399df8a51af27a669a416e4eda55 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Mon, 23 Oct 2023 21:07:49 +0800 Subject: [PATCH 51/94] update comments --- tokio/src/runtime/task/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tokio/src/runtime/task/mod.rs b/tokio/src/runtime/task/mod.rs index 85cf8979025..2c1a8d73ca6 100644 --- a/tokio/src/runtime/task/mod.rs +++ b/tokio/src/runtime/task/mod.rs @@ -511,7 +511,7 @@ unsafe impl linked_list::Link for Task { unsafe impl sharded_list::ShardedListItem for Task { unsafe fn get_shared_id(target: NonNull) -> usize { let task_id = unsafe { Header::get_id(target) }; - // in 32-bits machine, the lower 32 bits will be reserved - return task_id.0 as usize; + // safety: in 32-bits machine, the lower 32 bits will be reserved + task_id.0 as usize } } From 38c9eba5b5fea249f7f214111d676315472116b1 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Mon, 23 Oct 2023 21:13:54 +0800 Subject: [PATCH 52/94] fix: remove needless reference --- tokio/src/util/sharded_list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index 5256de48f53..6dcfbec6400 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -100,7 +100,7 @@ impl ShardedList { /// Gets the lock of ShardedList, lets us have the write permission. pub(crate) fn lock_shard(&self, val: &L::Handle) -> ShardGuard<'_, L, L::Target> { - let id = unsafe { L::get_shared_id(L::as_raw(&val)) }; + let id = unsafe { L::get_shared_id(L::as_raw(val)) }; ShardGuard { lock: self.shard_inner(id), count: &self.count, From 7968b51da57d7559076b23468aa607637ee1fe19 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Mon, 23 Oct 2023 21:34:27 +0800 Subject: [PATCH 53/94] fix: release lock as far as possible --- tokio/src/runtime/task/list.rs | 1 + tokio/src/util/sharded_list.rs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index ae307ee0d0a..a271fbf8dbd 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -115,6 +115,7 @@ impl OwnedTasks { // check the closed flag in the lock for ensuring all tasks // will shutdown after ownedTasks has been closed. if self.closed.load(Ordering::Acquire) { + drop(shard); task.shutdown(); return None; } diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index 6dcfbec6400..b45d770c0a4 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -54,6 +54,7 @@ impl ShardedList { } } +/// Used to get the lock of shard pub(crate) struct ShardGuard<'a, L, T> { lock: MutexGuard<'a, LinkedList>, count: &'a AtomicUsize, @@ -98,7 +99,7 @@ impl ShardedList { node } - /// Gets the lock of ShardedList, lets us have the write permission. + /// Gets the lock of ShardedList, makes us have the write permission. pub(crate) fn lock_shard(&self, val: &L::Handle) -> ShardGuard<'_, L, L::Target> { let id = unsafe { L::get_shared_id(L::as_raw(val)) }; ShardGuard { From 5d3da9edf82409aaee16da4f8cea8bbb5d8ba314 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Mon, 6 Nov 2023 21:17:06 +0800 Subject: [PATCH 54/94] Update tokio/src/runtime/task/list.rs Co-authored-by: Alice Ryhl --- tokio/src/runtime/task/list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index a271fbf8dbd..9301374cc67 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -142,7 +142,7 @@ impl OwnedTasks { /// The parameter start should be random among different worker threads /// to reduce lock conflicts during shutdown. /// Initiate shutting down the shard indexed by the start, and reset to 0 - /// once the shar_size is reached, continuing until start - 1, it works like a ring. + /// once the `shard_size` is reached, continuing until `start - 1`, it works like a ring. pub(crate) fn close_and_shutdown_all(&self, start: usize) where S: Schedule, From 1d1a7a3d297469d49c2c9dc0d53e1619f3f73b1f Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Mon, 6 Nov 2023 21:17:35 +0800 Subject: [PATCH 55/94] Update tokio/src/util/sharded_list.rs Co-authored-by: Alice Ryhl --- tokio/src/util/sharded_list.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index b45d770c0a4..4c83a478b0e 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -1,6 +1,7 @@ #![cfg_attr(not(feature = "full"), allow(dead_code))] -use std::{ptr::NonNull, sync::atomic::Ordering}; +use std::ptr::NonNull; +use std::sync::atomic::Ordering; use crate::loom::sync::{Mutex, MutexGuard}; use std::sync::atomic::AtomicUsize; From 13c4b93f0b3785aa092c3bd3b7bf8a0f644320c7 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Mon, 6 Nov 2023 21:22:24 +0800 Subject: [PATCH 56/94] Update tokio/src/util/sharded_list.rs Co-authored-by: Alice Ryhl --- tokio/src/util/sharded_list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index 4c83a478b0e..0be60f42fb2 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -128,7 +128,7 @@ impl ShardedList { #[inline] fn shard_inner(&self, id: usize) -> MutexGuard<'_, LinkedList::Target>> { - // Safety: this modulo operation ensures it is safe here. + // Safety: This modulo operation ensures that the index is not out of bounds. unsafe { self.lists.get_unchecked(id & self.shard_mask).lock() } } } From ee53e2373b6ca6ad2eee4672dc2f2e29ee1e5242 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Mon, 6 Nov 2023 21:23:36 +0800 Subject: [PATCH 57/94] Update tokio/src/util/sharded_list.rs Co-authored-by: Alice Ryhl --- tokio/src/util/sharded_list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index 0be60f42fb2..1620792bbab 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -149,7 +149,7 @@ cfg_taskdump! { where F: FnMut(&L::Handle), { - let mut guards = vec![]; + let mut guards = Vec::with_capacity(self.lists.len()); for list in self.lists.iter(){ guards.push(list.lock()); } From 01afd7b2d550f9383d6c5a8458b06c68137a4b0c Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Mon, 6 Nov 2023 21:23:44 +0800 Subject: [PATCH 58/94] Update tokio/src/util/sharded_list.rs Co-authored-by: Alice Ryhl --- tokio/src/util/sharded_list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index 1620792bbab..4e06437a5af 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -150,7 +150,7 @@ cfg_taskdump! { F: FnMut(&L::Handle), { let mut guards = Vec::with_capacity(self.lists.len()); - for list in self.lists.iter(){ + for list in self.lists.iter() { guards.push(list.lock()); } for g in &mut guards{ From 36c2355c842d331e2cedede7c6179d9c326fbed1 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Mon, 6 Nov 2023 21:23:53 +0800 Subject: [PATCH 59/94] Update tokio/src/util/sharded_list.rs Co-authored-by: Alice Ryhl --- tokio/src/util/sharded_list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index 4e06437a5af..f0614afe89f 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -153,7 +153,7 @@ cfg_taskdump! { for list in self.lists.iter() { guards.push(list.lock()); } - for g in &mut guards{ + for g in &mut guards { g.for_each(f); } } From d6606b89f41e3d646c6ad82669a0cd3b9b3778c8 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Mon, 6 Nov 2023 21:24:12 +0800 Subject: [PATCH 60/94] Update tokio/src/util/sharded_list.rs Co-authored-by: Alice Ryhl --- tokio/src/util/sharded_list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index f0614afe89f..9f5f1cd3194 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -147,7 +147,7 @@ cfg_taskdump! { impl ShardedList { pub(crate) fn for_each(&self, f: &mut F) where - F: FnMut(&L::Handle), + F: FnMut(&L::Handle), { let mut guards = Vec::with_capacity(self.lists.len()); for list in self.lists.iter() { From c9f32d2891c481d58d9481120fd5c498d24b1909 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Mon, 6 Nov 2023 21:34:56 +0800 Subject: [PATCH 61/94] Update tokio/src/runtime/task/list.rs Co-authored-by: Alice Ryhl --- tokio/src/runtime/task/list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 9301374cc67..ae27d0582f1 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -193,7 +193,7 @@ cfg_taskdump! { where F: FnMut(&Task), { - self.list.for_each(&mut f); + self.list.for_each(&mut f); } } } From bbefb7059ba2631b35d0df47d4656da4916d012b Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Mon, 6 Nov 2023 21:43:00 +0800 Subject: [PATCH 62/94] fix: accept ownedship of closue in method for_each of OwnedTasks --- tokio/src/runtime/task/list.rs | 4 ++-- tokio/src/util/sharded_list.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index ae27d0582f1..d3246d71a08 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -189,11 +189,11 @@ impl OwnedTasks { cfg_taskdump! { impl OwnedTasks { - pub(crate) fn for_each(&self, mut f: F) + pub(crate) fn for_each(&self, f: F) where F: FnMut(&Task), { - self.list.for_each(&mut f); + self.list.for_each(f); } } } diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index 9f5f1cd3194..05da3534900 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -145,7 +145,7 @@ impl<'a, L: ShardedListItem> ShardGuard<'a, L, L::Target> { cfg_taskdump! { impl ShardedList { - pub(crate) fn for_each(&self, f: &mut F) + pub(crate) fn for_each(&self, mut f: F) where F: FnMut(&L::Handle), { @@ -154,7 +154,7 @@ cfg_taskdump! { guards.push(list.lock()); } for g in &mut guards { - g.for_each(f); + g.for_each(&mut f); } } } From 8baf79eb5f513d4c752db46b834a5d6398d87fb6 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Tue, 7 Nov 2023 17:05:46 +0800 Subject: [PATCH 63/94] Apply suggestions from code review Co-authored-by: Alice Ryhl --- tokio/src/util/sharded_list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index 05da3534900..a062c794a6b 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -8,7 +8,7 @@ use std::sync::atomic::AtomicUsize; use super::linked_list::{Link, LinkedList}; -/// An intrusive linked list supporting high concurrent updates. +/// An intrusive linked list supporting highly concurrent updates. /// /// It currently relies on `LinkedList`, so it is the caller's /// responsibility to ensure the list is empty before dropping it. From 87e70c37a99813eab4425f29476da1707fc5eb5c Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Tue, 7 Nov 2023 17:06:07 +0800 Subject: [PATCH 64/94] Apply suggestions from code review Co-authored-by: Alice Ryhl --- tokio/src/util/sharded_list.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index a062c794a6b..99dff2ba06d 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -13,21 +13,19 @@ use super::linked_list::{Link, LinkedList}; /// It currently relies on `LinkedList`, so it is the caller's /// responsibility to ensure the list is empty before dropping it. /// -/// Note: Due to its inner sharded design, the order of node cannot be guaranteed. +/// Note: Due to its inner sharded design, the order of nodes cannot be guaranteed. pub(crate) struct ShardedList { lists: Box<[Mutex>]>, count: AtomicUsize, shard_mask: usize, } -/// Defines the id of a node in a linked list, different ids cause inner list nodes -/// to be scattered in different mutexs. +/// Determines which linked list an item should be stored in. /// /// # Safety /// -/// Implementations must guarantee that `Target` types are pinned in memory. In -/// other words, when a node is inserted, the value will not be moved as long as -/// it is stored in the list. +/// Implementations must guarantee that the id of an item does not change from +/// call to call. pub(crate) unsafe trait ShardedListItem: Link { // The returned id is used to pick which list this item should go into. unsafe fn get_shared_id(target: NonNull) -> usize; From af92f20f9d622f373b0fc98433800587ed95fe50 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Tue, 7 Nov 2023 17:06:48 +0800 Subject: [PATCH 65/94] Apply suggestions from code review Co-authored-by: Alice Ryhl --- tokio/src/util/sharded_list.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index 99dff2ba06d..d18762ec3cf 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -27,7 +27,8 @@ pub(crate) struct ShardedList { /// Implementations must guarantee that the id of an item does not change from /// call to call. pub(crate) unsafe trait ShardedListItem: Link { - // The returned id is used to pick which list this item should go into. + /// # Safety + /// The provided pointer must point at a valid list item. unsafe fn get_shared_id(target: NonNull) -> usize; } From 60104b87f7d4eb4cfaca1a38a817f126865e0ca6 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Tue, 7 Nov 2023 17:07:06 +0800 Subject: [PATCH 66/94] Apply suggestions from code review Co-authored-by: Alice Ryhl --- tokio/src/util/sharded_list.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index d18762ec3cf..bb724dcdeb8 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -29,11 +29,11 @@ pub(crate) struct ShardedList { pub(crate) unsafe trait ShardedListItem: Link { /// # Safety /// The provided pointer must point at a valid list item. - unsafe fn get_shared_id(target: NonNull) -> usize; + unsafe fn get_shard_id(target: NonNull) -> usize; } impl ShardedList { - /// Creates a new and empty sharded linked list under specified size + /// Creates a new and empty sharded linked list with the specified size. pub(crate) fn new(sharded_size: usize) -> Self { // Find power-of-two sizes best matching arguments let mut size = 1; From bb4458b9ac7d4dc9fae27122153e08151e120346 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Tue, 7 Nov 2023 17:08:25 +0800 Subject: [PATCH 67/94] rm unused push method --- tokio/src/util/sharded_list.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index bb724dcdeb8..875c9c34c6c 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -62,14 +62,6 @@ pub(crate) struct ShardGuard<'a, L, T> { } impl ShardedList { - #[allow(dead_code)] - /// Push a value to this shard. - pub(crate) fn push(&self, val: L::Handle) { - let shard = self.lock_shard(&val); - shard.push(val); - self.count.fetch_add(1, Ordering::Relaxed); - } - /// Removes the last element from a list specified by shard_id and returns it, or None if it is /// empty. pub(crate) fn pop_back(&self, shard_id: usize) -> Option { From 1adad7076fec4a4cafb7578a1bd176d0ed23d47f Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Tue, 7 Nov 2023 17:09:04 +0800 Subject: [PATCH 68/94] rename get_sharded_id to get_shard_id --- tokio/src/util/sharded_list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index 875c9c34c6c..2ffd69aff51 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -93,7 +93,7 @@ impl ShardedList { /// Gets the lock of ShardedList, makes us have the write permission. pub(crate) fn lock_shard(&self, val: &L::Handle) -> ShardGuard<'_, L, L::Target> { - let id = unsafe { L::get_shared_id(L::as_raw(val)) }; + let id = unsafe { L::get_shard_id(L::as_raw(val)) }; ShardGuard { lock: self.shard_inner(id), count: &self.count, From ef8d2b7c577db9c98ff2ac9f2a3e310d3d693ad5 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Tue, 7 Nov 2023 17:09:39 +0800 Subject: [PATCH 69/94] Apply suggestions from code review Co-authored-by: Alice Ryhl --- tokio/src/util/sharded_list.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index 2ffd69aff51..aea13493c6a 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -106,12 +106,13 @@ impl ShardedList { self.count.load(Ordering::Relaxed) } - /// Returns whether the linked list does not contain any node + /// Returns whether the linked list does not contain any node. pub(crate) fn is_empty(&self) -> bool { self.len() == 0 } /// Gets the shard size of this SharedList. + /// /// Used to help us to decide the parameter `shard_id`` of the `pop_back` method. pub(crate) fn shard_size(&self) -> usize { self.shard_mask + 1 From 2d4fbf6b9c6e5ff82f2d6a134af306828a442303 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Tue, 7 Nov 2023 17:10:34 +0800 Subject: [PATCH 70/94] rename get_sharded_id to get_shard_id --- tokio/src/runtime/task/mod.rs | 2 +- tokio/src/util/sharded_list.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tokio/src/runtime/task/mod.rs b/tokio/src/runtime/task/mod.rs index b041f9a6805..2f51f5eaf47 100644 --- a/tokio/src/runtime/task/mod.rs +++ b/tokio/src/runtime/task/mod.rs @@ -509,7 +509,7 @@ unsafe impl linked_list::Link for Task { /// /// Tasks are pinned. unsafe impl sharded_list::ShardedListItem for Task { - unsafe fn get_shared_id(target: NonNull) -> usize { + unsafe fn get_shard_id(target: NonNull) -> usize { let task_id = unsafe { Header::get_id(target) }; // safety: in 32-bits machine, the lower 32 bits will be reserved task_id.0 as usize diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index aea13493c6a..d961211aed8 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -82,7 +82,7 @@ impl ShardedList { /// - `node` is not contained by any list, /// - `node` is currently contained by some other `GuardedLinkedList`. pub(crate) unsafe fn remove(&self, node: NonNull) -> Option { - let id = L::get_shared_id(node); + let id = L::get_shard_id(node); let mut lock = self.shard_inner(id); let node = lock.remove(node); if node.is_some() { @@ -128,7 +128,7 @@ impl ShardedList { impl<'a, L: ShardedListItem> ShardGuard<'a, L, L::Target> { /// Push a value to this shard. pub(crate) fn push(mut self, val: L::Handle) { - let id = unsafe { L::get_shared_id(L::as_raw(&val)) }; + let id = unsafe { L::get_shard_id(L::as_raw(&val)) }; assert_eq!(id, self.id); self.lock.push_front(val); self.count.fetch_add(1, Ordering::Relaxed); From 777d97e8be28c7a6530097d379179f99433e0182 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Tue, 7 Nov 2023 17:13:54 +0800 Subject: [PATCH 71/94] add sentence in comments --- tokio/src/util/sharded_list.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index d961211aed8..6d6db8bfd7a 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -35,7 +35,7 @@ pub(crate) unsafe trait ShardedListItem: Link { impl ShardedList { /// Creates a new and empty sharded linked list with the specified size. pub(crate) fn new(sharded_size: usize) -> Self { - // Find power-of-two sizes best matching arguments + // Find power-of-two sizes best matching arguments. let mut size = 1; while size < sharded_size { size <<= 1; @@ -54,7 +54,7 @@ impl ShardedList { } } -/// Used to get the lock of shard +/// Used to get the lock of shard. pub(crate) struct ShardGuard<'a, L, T> { lock: MutexGuard<'a, LinkedList>, count: &'a AtomicUsize, @@ -73,7 +73,7 @@ impl ShardedList { node } - /// Removes the specified node from the list + /// Removes the specified node from the list. /// /// # Safety /// From 5406a7e452aba7d4bc7a0c32ddb8e587de219e26 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Tue, 7 Nov 2023 17:14:58 +0800 Subject: [PATCH 72/94] rm dead_code attr --- tokio/src/util/sharded_list.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index 6d6db8bfd7a..f81423cb8e1 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -1,5 +1,3 @@ -#![cfg_attr(not(feature = "full"), allow(dead_code))] - use std::ptr::NonNull; use std::sync::atomic::Ordering; From 680848e7986718257d480f9961df0bb02f0021e5 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Tue, 7 Nov 2023 19:22:02 +0800 Subject: [PATCH 73/94] move spawn_concurrent_level size from shardedList to builder --- tokio/src/runtime/builder.rs | 38 ++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/tokio/src/runtime/builder.rs b/tokio/src/runtime/builder.rs index f4011d0ab45..184c75bb7da 100644 --- a/tokio/src/runtime/builder.rs +++ b/tokio/src/runtime/builder.rs @@ -414,6 +414,8 @@ impl Builder { /// This can be any number greater than 0 and less than or equal to 65536, /// if the parameter is larger than this value, concurrency level will actually select 65536 internally. /// + /// This value must be power of two. + /// /// When the value of this is small compared to the number of concurrent threads, increasing it /// will help improve the performanc of concurrently spawn tasks. However, when the value is /// already large enough, further increasing it will not continue to improve performance. @@ -448,14 +450,38 @@ impl Builder { #[cfg(tokio_unstable)] #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))] pub fn spawn_concurrency_level(&mut self, mut val: usize) -> &mut Self { + const MAX_SPAWN_CONCURRENCY_LEVEL: usize = 1 << 16; + assert!(val > 0, "spawn concurrency level cannot be set to 0"); - if val > 1 << 16 { - val = 1 << 16; + assert!( + sharded_size.is_power_of_two(), + "spawn concurrency level must be power of two" + ); + if val > MAX_SPAWN_CONCURRENCY_LEVEL { + val = MAX_SPAWN_CONCURRENCY_LEVEL; } self.spawn_concurrency_level = Some(val); self } + fn spawn_concurrency_level(&self) -> usize { + const MAX_SPAWN_CONCURRENCY_LEVEL: usize = 1 << 16; + + match self.spawn_concurrency_level { + Some(i) => i, + None => { + use crate::loom::sys::num_cpus; + let core_threads = self.worker_threads.unwrap_or_else(num_cpus); + + let mut size = 1; + while size / 4 < core_threads && size < MAX_SPAWN_CONCURRENCY_LEVEL { + size <<= 1; + } + size.min(MAX_SPAWN_CONCURRENCY_LEVEL) + } + } + } + /// Specifies the limit for additional threads spawned by the Runtime. /// /// These threads are used for blocking operations like tasks spawned @@ -1287,11 +1313,11 @@ cfg_rt_multi_thread! { let core_threads = self.worker_threads.unwrap_or_else(num_cpus); // Shrink the size of spawn_concurrency_level when using loom. This shouldn't impact - // logic, but allows loom to test more edge cases in a reasoable a mount of time + // logic, but allows loom to test more edge cases in a reasoable a mount of time. #[cfg(loom)] let spawn_concurrency_level = 4; #[cfg(not(loom))] - let spawn_concurrency_level = self.spawn_concurrency_level.unwrap_or(core_threads * 4); + let spawn_concurrency_level = self.spawn_concurrency_level(); let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?; @@ -1343,11 +1369,11 @@ cfg_rt_multi_thread! { let core_threads = self.worker_threads.unwrap_or_else(num_cpus); // Shrink the size of spawn_concurrency_level when using loom. This shouldn't impact - // logic, but allows loom to test more edge cases in a reasoable a mount of time + // logic, but allows loom to test more edge cases in a reasoable a mount of time. #[cfg(loom)] let spawn_concurrency_level = 4; #[cfg(not(loom))] - let spawn_concurrency_level = self.spawn_concurrency_level.unwrap_or(core_threads * 4); + let spawn_concurrency_level = spawn_concurrency_level; let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?; From 6fb70b1f22bd01b0b9d4585733753cd20b2031f9 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Tue, 7 Nov 2023 19:27:02 +0800 Subject: [PATCH 74/94] update comments --- tokio/src/runtime/task/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/runtime/task/mod.rs b/tokio/src/runtime/task/mod.rs index 2f51f5eaf47..8f74749c680 100644 --- a/tokio/src/runtime/task/mod.rs +++ b/tokio/src/runtime/task/mod.rs @@ -511,7 +511,7 @@ unsafe impl linked_list::Link for Task { unsafe impl sharded_list::ShardedListItem for Task { unsafe fn get_shard_id(target: NonNull) -> usize { let task_id = unsafe { Header::get_id(target) }; - // safety: in 32-bits machine, the lower 32 bits will be reserved + // Safety: The task_id is 64-bits in all platforms, but lower 32 bits will be reserved in 32-bits machine. task_id.0 as usize } } From 0ba87dbe860dcbcc79cdc1b4e13fc74b0d938cf7 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Tue, 7 Nov 2023 19:33:58 +0800 Subject: [PATCH 75/94] update comments --- tokio/src/runtime/task/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tokio/src/runtime/task/mod.rs b/tokio/src/runtime/task/mod.rs index 8f74749c680..55b89781318 100644 --- a/tokio/src/runtime/task/mod.rs +++ b/tokio/src/runtime/task/mod.rs @@ -511,7 +511,8 @@ unsafe impl linked_list::Link for Task { unsafe impl sharded_list::ShardedListItem for Task { unsafe fn get_shard_id(target: NonNull) -> usize { let task_id = unsafe { Header::get_id(target) }; - // Safety: The task_id is 64-bits in all platforms, but lower 32 bits will be reserved in 32-bits machine. + // Safety: The `task_id` is 64-bits in all platforms, but lower 32 bits will be reserved in 32-bits machine, + // it is safe to take `task_id` as `shard_id` in this casting. task_id.0 as usize } } From 8bb106fe28115cb7f33a3d2720f5bdc229d9003d Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Tue, 7 Nov 2023 19:36:28 +0800 Subject: [PATCH 76/94] update comments --- tokio/src/runtime/task/mod.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tokio/src/runtime/task/mod.rs b/tokio/src/runtime/task/mod.rs index 55b89781318..e6023dbe36a 100644 --- a/tokio/src/runtime/task/mod.rs +++ b/tokio/src/runtime/task/mod.rs @@ -507,12 +507,13 @@ unsafe impl linked_list::Link for Task { /// # Safety /// -/// Tasks are pinned. +/// The id of a task is never changed after creation of the task, so the return value of +/// `get_shard_id` will not change. (The cast may throw away the upper 32 bits of the task id, but +/// the shard still won't change from call to call.) unsafe impl sharded_list::ShardedListItem for Task { unsafe fn get_shard_id(target: NonNull) -> usize { + // SAFETY: The caller guarantees that `target` points at a valid task. let task_id = unsafe { Header::get_id(target) }; - // Safety: The `task_id` is 64-bits in all platforms, but lower 32 bits will be reserved in 32-bits machine, - // it is safe to take `task_id` as `shard_id` in this casting. task_id.0 as usize } } From e0fb9e2a6b70ac4f91326b7ea32d5aab1affa225 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Tue, 7 Nov 2023 19:40:57 +0800 Subject: [PATCH 77/94] rm loop in ShardedList::new --- tokio/src/util/sharded_list.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index f81423cb8e1..e59caef32f5 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -33,15 +33,11 @@ pub(crate) unsafe trait ShardedListItem: Link { impl ShardedList { /// Creates a new and empty sharded linked list with the specified size. pub(crate) fn new(sharded_size: usize) -> Self { - // Find power-of-two sizes best matching arguments. - let mut size = 1; - while size < sharded_size { - size <<= 1; - } - let shard_mask = size - 1; - - let mut lists = Vec::with_capacity(size as usize); - for _ in 0..size { + assert!(sharded_size.is_power_of_two()); + + let shard_mask = sharded_size - 1; + let mut lists = Vec::with_capacity(sharded_size); + for _ in 0..sharded_size { lists.push(Mutex::new(LinkedList::::new())) } Self { From 57133a50521e54eab5a14836065b9c38ab13c86c Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Tue, 7 Nov 2023 19:45:14 +0800 Subject: [PATCH 78/94] fix rustfmt --- tokio/src/util/sharded_list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index e59caef32f5..abbbf04eeb9 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -34,7 +34,7 @@ impl ShardedList { /// Creates a new and empty sharded linked list with the specified size. pub(crate) fn new(sharded_size: usize) -> Self { assert!(sharded_size.is_power_of_two()); - + let shard_mask = sharded_size - 1; let mut lists = Vec::with_capacity(sharded_size); for _ in 0..sharded_size { From 71e89830bd62463c9bffbf7676f058477a571787 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Tue, 7 Nov 2023 19:46:45 +0800 Subject: [PATCH 79/94] fix --- tokio/src/runtime/builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/runtime/builder.rs b/tokio/src/runtime/builder.rs index 184c75bb7da..bf25e42b36f 100644 --- a/tokio/src/runtime/builder.rs +++ b/tokio/src/runtime/builder.rs @@ -454,7 +454,7 @@ impl Builder { assert!(val > 0, "spawn concurrency level cannot be set to 0"); assert!( - sharded_size.is_power_of_two(), + val.is_power_of_two(), "spawn concurrency level must be power of two" ); if val > MAX_SPAWN_CONCURRENCY_LEVEL { From d1ce613cda5fdb35ed0be52118a9b1768d4bf2dd Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Tue, 7 Nov 2023 19:48:44 +0800 Subject: [PATCH 80/94] fix spawn_concurrency_level --- tokio/src/runtime/builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/runtime/builder.rs b/tokio/src/runtime/builder.rs index bf25e42b36f..3d67b20563e 100644 --- a/tokio/src/runtime/builder.rs +++ b/tokio/src/runtime/builder.rs @@ -1373,7 +1373,7 @@ cfg_rt_multi_thread! { #[cfg(loom)] let spawn_concurrency_level = 4; #[cfg(not(loom))] - let spawn_concurrency_level = spawn_concurrency_level; + let spawn_concurrency_level = self.spawn_concurrency_level(); let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?; From e6b8db12ca14fb8369c447c7f0ce89ea799b92a2 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Tue, 7 Nov 2023 19:50:39 +0800 Subject: [PATCH 81/94] fix spawn_concurrency_level --- tokio/src/runtime/builder.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tokio/src/runtime/builder.rs b/tokio/src/runtime/builder.rs index 3d67b20563e..4b5d5d290e3 100644 --- a/tokio/src/runtime/builder.rs +++ b/tokio/src/runtime/builder.rs @@ -464,7 +464,7 @@ impl Builder { self } - fn spawn_concurrency_level(&self) -> usize { + fn get_spawn_concurrency_level(&self) -> usize { const MAX_SPAWN_CONCURRENCY_LEVEL: usize = 1 << 16; match self.spawn_concurrency_level { @@ -1317,7 +1317,7 @@ cfg_rt_multi_thread! { #[cfg(loom)] let spawn_concurrency_level = 4; #[cfg(not(loom))] - let spawn_concurrency_level = self.spawn_concurrency_level(); + let spawn_concurrency_level = self.get_spawn_concurrency_level(); let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?; @@ -1373,7 +1373,7 @@ cfg_rt_multi_thread! { #[cfg(loom)] let spawn_concurrency_level = 4; #[cfg(not(loom))] - let spawn_concurrency_level = self.spawn_concurrency_level(); + let spawn_concurrency_level = self.get_spawn_concurrency_level(); let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?; From 592f4324cdcafdf306ec876f82342da53b3535e0 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Tue, 7 Nov 2023 20:00:46 +0800 Subject: [PATCH 82/94] rm get_spawn_concurrency_level to cfg_rt_multi_thread --- tokio/src/runtime/builder.rs | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tokio/src/runtime/builder.rs b/tokio/src/runtime/builder.rs index 4b5d5d290e3..970658c7124 100644 --- a/tokio/src/runtime/builder.rs +++ b/tokio/src/runtime/builder.rs @@ -464,24 +464,6 @@ impl Builder { self } - fn get_spawn_concurrency_level(&self) -> usize { - const MAX_SPAWN_CONCURRENCY_LEVEL: usize = 1 << 16; - - match self.spawn_concurrency_level { - Some(i) => i, - None => { - use crate::loom::sys::num_cpus; - let core_threads = self.worker_threads.unwrap_or_else(num_cpus); - - let mut size = 1; - while size / 4 < core_threads && size < MAX_SPAWN_CONCURRENCY_LEVEL { - size <<= 1; - } - size.min(MAX_SPAWN_CONCURRENCY_LEVEL) - } - } - } - /// Specifies the limit for additional threads spawned by the Runtime. /// /// These threads are used for blocking operations like tasks spawned @@ -1410,6 +1392,24 @@ cfg_rt_multi_thread! { Ok(Runtime::from_parts(Scheduler::MultiThreadAlt(scheduler), handle, blocking_pool)) } } + + fn get_spawn_concurrency_level(&self) -> usize { + const MAX_SPAWN_CONCURRENCY_LEVEL: usize = 1 << 16; + + match self.spawn_concurrency_level { + Some(i) => i, + None => { + use crate::loom::sys::num_cpus; + let core_threads = self.worker_threads.unwrap_or_else(num_cpus); + + let mut size = 1; + while size / 4 < core_threads && size < MAX_SPAWN_CONCURRENCY_LEVEL { + size <<= 1; + } + size.min(MAX_SPAWN_CONCURRENCY_LEVEL) + } + } + } } } From f083757481475f9a1b74abee2547d66ee114ce22 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Tue, 7 Nov 2023 20:09:58 +0800 Subject: [PATCH 83/94] add allow(dead_code))] --- tokio/src/util/sharded_list.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index abbbf04eeb9..905dfbb7557 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -1,3 +1,4 @@ +#![cfg_attr(not(feature = "full"), allow(dead_code))] use std::ptr::NonNull; use std::sync::atomic::Ordering; From 3105af7b3f72b3ff5c7c8896892c1dbfd5bbaaeb Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Wed, 8 Nov 2023 15:03:07 +0800 Subject: [PATCH 84/94] rm dead_code attr --- tokio/src/util/mod.rs | 13 +++---------- tokio/src/util/sharded_list.rs | 1 - 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/tokio/src/util/mod.rs b/tokio/src/util/mod.rs index 59257a6ab95..abdb70406d2 100644 --- a/tokio/src/util/mod.rs +++ b/tokio/src/util/mod.rs @@ -42,16 +42,9 @@ pub(crate) use wake_list::WakeList; ))] pub(crate) mod linked_list; -#[cfg(any( - feature = "fs", - feature = "net", - feature = "process", - feature = "rt", - feature = "sync", - feature = "signal", - feature = "time", -))] -pub(crate) mod sharded_list; +cfg_rt! { + pub(crate) mod sharded_list; +} #[cfg(any(feature = "rt", feature = "macros"))] pub(crate) mod rand; diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index 905dfbb7557..abbbf04eeb9 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -1,4 +1,3 @@ -#![cfg_attr(not(feature = "full"), allow(dead_code))] use std::ptr::NonNull; use std::sync::atomic::Ordering; From 8e189cfe72f30838f5df1da2c92c2d2a5450af36 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Wed, 22 Nov 2023 20:32:22 +0800 Subject: [PATCH 85/94] make spawn_concurrency_level unconfigurable --- tokio/src/runtime/builder.rs | 79 ++---------------------------------- 1 file changed, 3 insertions(+), 76 deletions(-) diff --git a/tokio/src/runtime/builder.rs b/tokio/src/runtime/builder.rs index 970658c7124..2629065f14d 100644 --- a/tokio/src/runtime/builder.rs +++ b/tokio/src/runtime/builder.rs @@ -57,11 +57,6 @@ pub struct Builder { /// Only used when not using the current-thread executor. worker_threads: Option, - /// Configures the global OwnedTasks's concurrency level - /// - /// Only used when not using the current-thread executor. - pub(super) spawn_concurrency_level: Option, - /// Cap on thread usage. max_blocking_threads: usize, @@ -283,9 +278,6 @@ impl Builder { // Default to lazy auto-detection (one thread per CPU core) worker_threads: None, - // Default to lazy auto-detection (4 times the number of worker threads) - spawn_concurrency_level: None, - max_blocking_threads: 512, // Default thread name @@ -409,61 +401,6 @@ impl Builder { self } - /// Sets the spawn concurrency level the `Runtime` will use. - /// - /// This can be any number greater than 0 and less than or equal to 65536, - /// if the parameter is larger than this value, concurrency level will actually select 65536 internally. - /// - /// This value must be power of two. - /// - /// When the value of this is small compared to the number of concurrent threads, increasing it - /// will help improve the performanc of concurrently spawn tasks. However, when the value is - /// already large enough, further increasing it will not continue to improve performance. - /// Instead, it may result in longer time of the Runtime creation. - /// - /// # Default - /// - /// The default value for this is 4 times the number of worker threads. - /// - /// When using the `current_thread` runtime this method has no effect. - /// - /// # Examples - /// - /// ## Multi threaded runtime with spawn_concurrency_level 8 - /// - /// ``` - /// use tokio::runtime; - /// - /// // This will spawn a work-stealing runtime with 4 worker threads. - /// let rt = runtime::Builder::new_multi_thread() - /// .spawn_concurrency_level(8) - /// .build() - /// .unwrap(); - /// - /// rt.spawn(async move {}); - /// ``` - /// - /// # Panics - /// - /// This will panic if `val` is not larger than `0`. - #[track_caller] - #[cfg(tokio_unstable)] - #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))] - pub fn spawn_concurrency_level(&mut self, mut val: usize) -> &mut Self { - const MAX_SPAWN_CONCURRENCY_LEVEL: usize = 1 << 16; - - assert!(val > 0, "spawn concurrency level cannot be set to 0"); - assert!( - val.is_power_of_two(), - "spawn concurrency level must be power of two" - ); - if val > MAX_SPAWN_CONCURRENCY_LEVEL { - val = MAX_SPAWN_CONCURRENCY_LEVEL; - } - self.spawn_concurrency_level = Some(val); - self - } - /// Specifies the limit for additional threads spawned by the Runtime. /// /// These threads are used for blocking operations like tasks spawned @@ -1299,7 +1236,7 @@ cfg_rt_multi_thread! { #[cfg(loom)] let spawn_concurrency_level = 4; #[cfg(not(loom))] - let spawn_concurrency_level = self.get_spawn_concurrency_level(); + let spawn_concurrency_level = Self::get_spawn_concurrency_level(core_threads); let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?; @@ -1355,7 +1292,7 @@ cfg_rt_multi_thread! { #[cfg(loom)] let spawn_concurrency_level = 4; #[cfg(not(loom))] - let spawn_concurrency_level = self.get_spawn_concurrency_level(); + let spawn_concurrency_level = Self::get_spawn_concurrency_level(core_threads); let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?; @@ -1393,23 +1330,14 @@ cfg_rt_multi_thread! { } } - fn get_spawn_concurrency_level(&self) -> usize { + fn get_spawn_concurrency_level(core_threads : usize) -> usize { const MAX_SPAWN_CONCURRENCY_LEVEL: usize = 1 << 16; - - match self.spawn_concurrency_level { - Some(i) => i, - None => { - use crate::loom::sys::num_cpus; - let core_threads = self.worker_threads.unwrap_or_else(num_cpus); - let mut size = 1; while size / 4 < core_threads && size < MAX_SPAWN_CONCURRENCY_LEVEL { size <<= 1; } size.min(MAX_SPAWN_CONCURRENCY_LEVEL) - } } - } } } @@ -1417,7 +1345,6 @@ impl fmt::Debug for Builder { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt.debug_struct("Builder") .field("worker_threads", &self.worker_threads) - .field("spawn_concurrency_level", &self.spawn_concurrency_level) .field("max_blocking_threads", &self.max_blocking_threads) .field( "thread_name", From 75d308136bb66606358b6620afd334364fb035fc Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Fri, 24 Nov 2023 08:26:16 +0800 Subject: [PATCH 86/94] Apply suggestions from code review Co-authored-by: Alice Ryhl --- tokio/src/runtime/task/list.rs | 10 ++++------ tokio/src/runtime/task/mod.rs | 2 +- tokio/src/util/sharded_list.rs | 4 +++- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index d3246d71a08..265c91b27a7 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -112,8 +112,8 @@ impl OwnedTasks { } let shard = self.list.lock_shard(&task); - // check the closed flag in the lock for ensuring all tasks - // will shutdown after ownedTasks has been closed. + // Check the closed flag in the lock for ensuring all that tasks + // will shut down after the OwnedTasks has been closed. if self.closed.load(Ordering::Acquire) { drop(shard); task.shutdown(); @@ -139,10 +139,8 @@ impl OwnedTasks { /// Shuts down all tasks in the collection. This call also closes the /// collection, preventing new items from being added. /// - /// The parameter start should be random among different worker threads - /// to reduce lock conflicts during shutdown. - /// Initiate shutting down the shard indexed by the start, and reset to 0 - /// once the `shard_size` is reached, continuing until `start - 1`, it works like a ring. + /// The parameter start determines which shard this method will start at. + /// Using different values for each worker thread reduces contention. pub(crate) fn close_and_shutdown_all(&self, start: usize) where S: Schedule, diff --git a/tokio/src/runtime/task/mod.rs b/tokio/src/runtime/task/mod.rs index e6023dbe36a..e48788567e4 100644 --- a/tokio/src/runtime/task/mod.rs +++ b/tokio/src/runtime/task/mod.rs @@ -509,7 +509,7 @@ unsafe impl linked_list::Link for Task { /// /// The id of a task is never changed after creation of the task, so the return value of /// `get_shard_id` will not change. (The cast may throw away the upper 32 bits of the task id, but -/// the shard still won't change from call to call.) +/// the shard id still won't change from call to call.) unsafe impl sharded_list::ShardedListItem for Task { unsafe fn get_shard_id(target: NonNull) -> usize { // SAFETY: The caller guarantees that `target` points at a valid task. diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index abbbf04eeb9..140c8efb1d9 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -78,7 +78,9 @@ impl ShardedList { pub(crate) unsafe fn remove(&self, node: NonNull) -> Option { let id = L::get_shard_id(node); let mut lock = self.shard_inner(id); - let node = lock.remove(node); + // SAFETY: Since the shard id cannot change, it's not possible for this node + // to be in any other list of the same sharded list. + let node = unsafe { lock.remove(node) }; if node.is_some() { self.count.fetch_sub(1, Ordering::Relaxed); } From ed27f709449effd98d8bcf7fc0be4f9f3aad4f9a Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Fri, 24 Nov 2023 08:33:32 +0800 Subject: [PATCH 87/94] apply suggestions from core review --- tokio/src/runtime/task/list.rs | 1 + tokio/src/util/linked_list.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 265c91b27a7..44873540986 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -187,6 +187,7 @@ impl OwnedTasks { cfg_taskdump! { impl OwnedTasks { + /// Locks the tasks, and calls `f` on an iterator over them. pub(crate) fn for_each(&self, f: F) where F: FnMut(&Task), diff --git a/tokio/src/util/linked_list.rs b/tokio/src/util/linked_list.rs index dab9828ccbd..0ed2b616456 100644 --- a/tokio/src/util/linked_list.rs +++ b/tokio/src/util/linked_list.rs @@ -296,7 +296,7 @@ cfg_io_driver_impl! { cfg_taskdump! { impl LinkedList { - pub(crate) fn for_each(&mut self, f: &mut F) + pub(crate) fn for_each(&mut self, mut f: F) where F: FnMut(&T::Handle), { From db580761241b96623ff3cc0b11c9b3295bcda104 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Fri, 24 Nov 2023 08:58:33 +0800 Subject: [PATCH 88/94] move get_spawn_concurrency_level from builder to task/list.rs --- tokio/src/runtime/builder.rs | 25 ------------------- .../src/runtime/scheduler/multi_thread/mod.rs | 2 -- .../runtime/scheduler/multi_thread/worker.rs | 3 +-- .../runtime/scheduler/multi_thread_alt/mod.rs | 2 -- .../scheduler/multi_thread_alt/worker.rs | 3 +-- tokio/src/runtime/task/list.rs | 22 ++++++++++++++-- 6 files changed, 22 insertions(+), 35 deletions(-) diff --git a/tokio/src/runtime/builder.rs b/tokio/src/runtime/builder.rs index 2629065f14d..adee919656e 100644 --- a/tokio/src/runtime/builder.rs +++ b/tokio/src/runtime/builder.rs @@ -1231,12 +1231,6 @@ cfg_rt_multi_thread! { use crate::runtime::scheduler::{self, MultiThread}; let core_threads = self.worker_threads.unwrap_or_else(num_cpus); - // Shrink the size of spawn_concurrency_level when using loom. This shouldn't impact - // logic, but allows loom to test more edge cases in a reasoable a mount of time. - #[cfg(loom)] - let spawn_concurrency_level = 4; - #[cfg(not(loom))] - let spawn_concurrency_level = Self::get_spawn_concurrency_level(core_threads); let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?; @@ -1255,7 +1249,6 @@ cfg_rt_multi_thread! { driver_handle, blocking_spawner, seed_generator_2, - spawn_concurrency_level, Config { before_park: self.before_park.clone(), after_unpark: self.after_unpark.clone(), @@ -1286,14 +1279,6 @@ cfg_rt_multi_thread! { use crate::runtime::scheduler::MultiThreadAlt; let core_threads = self.worker_threads.unwrap_or_else(num_cpus); - - // Shrink the size of spawn_concurrency_level when using loom. This shouldn't impact - // logic, but allows loom to test more edge cases in a reasoable a mount of time. - #[cfg(loom)] - let spawn_concurrency_level = 4; - #[cfg(not(loom))] - let spawn_concurrency_level = Self::get_spawn_concurrency_level(core_threads); - let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?; // Create the blocking pool @@ -1311,7 +1296,6 @@ cfg_rt_multi_thread! { driver_handle, blocking_spawner, seed_generator_2, - spawn_concurrency_level, Config { before_park: self.before_park.clone(), after_unpark: self.after_unpark.clone(), @@ -1329,15 +1313,6 @@ cfg_rt_multi_thread! { Ok(Runtime::from_parts(Scheduler::MultiThreadAlt(scheduler), handle, blocking_pool)) } } - - fn get_spawn_concurrency_level(core_threads : usize) -> usize { - const MAX_SPAWN_CONCURRENCY_LEVEL: usize = 1 << 16; - let mut size = 1; - while size / 4 < core_threads && size < MAX_SPAWN_CONCURRENCY_LEVEL { - size <<= 1; - } - size.min(MAX_SPAWN_CONCURRENCY_LEVEL) - } } } diff --git a/tokio/src/runtime/scheduler/multi_thread/mod.rs b/tokio/src/runtime/scheduler/multi_thread/mod.rs index 24d353cdfac..d85a0ae0a2a 100644 --- a/tokio/src/runtime/scheduler/multi_thread/mod.rs +++ b/tokio/src/runtime/scheduler/multi_thread/mod.rs @@ -60,7 +60,6 @@ impl MultiThread { driver_handle: driver::Handle, blocking_spawner: blocking::Spawner, seed_generator: RngSeedGenerator, - spawn_concurrency_level: usize, config: Config, ) -> (MultiThread, Arc, Launch) { let parker = Parker::new(driver); @@ -70,7 +69,6 @@ impl MultiThread { driver_handle, blocking_spawner, seed_generator, - spawn_concurrency_level, config, ); diff --git a/tokio/src/runtime/scheduler/multi_thread/worker.rs b/tokio/src/runtime/scheduler/multi_thread/worker.rs index d4aed2ec231..c1dd6df9865 100644 --- a/tokio/src/runtime/scheduler/multi_thread/worker.rs +++ b/tokio/src/runtime/scheduler/multi_thread/worker.rs @@ -245,7 +245,6 @@ pub(super) fn create( driver_handle: driver::Handle, blocking_spawner: blocking::Spawner, seed_generator: RngSeedGenerator, - spawn_concurrency_level: usize, config: Config, ) -> (Arc, Launch) { let mut cores = Vec::with_capacity(size); @@ -288,7 +287,7 @@ pub(super) fn create( remotes: remotes.into_boxed_slice(), inject, idle, - owned: OwnedTasks::new(spawn_concurrency_level as u32), + owned: OwnedTasks::new(size), synced: Mutex::new(Synced { idle: idle_synced, inject: inject_synced, diff --git a/tokio/src/runtime/scheduler/multi_thread_alt/mod.rs b/tokio/src/runtime/scheduler/multi_thread_alt/mod.rs index 86c6f96087d..e30c9b4783b 100644 --- a/tokio/src/runtime/scheduler/multi_thread_alt/mod.rs +++ b/tokio/src/runtime/scheduler/multi_thread_alt/mod.rs @@ -49,7 +49,6 @@ impl MultiThread { driver_handle: driver::Handle, blocking_spawner: blocking::Spawner, seed_generator: RngSeedGenerator, - spawn_concurrency_level: usize, config: Config, ) -> (MultiThread, runtime::Handle) { let handle = worker::create( @@ -58,7 +57,6 @@ impl MultiThread { driver_handle, blocking_spawner, seed_generator, - spawn_concurrency_level, config, ); diff --git a/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs b/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs index 6db95a935d0..8d16418a80b 100644 --- a/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs +++ b/tokio/src/runtime/scheduler/multi_thread_alt/worker.rs @@ -259,7 +259,6 @@ pub(super) fn create( driver_handle: driver::Handle, blocking_spawner: blocking::Spawner, seed_generator: RngSeedGenerator, - spawn_concurrency_level: usize, config: Config, ) -> runtime::Handle { let mut num_workers = num_cores; @@ -308,7 +307,7 @@ pub(super) fn create( remotes: remotes.into_boxed_slice(), inject, idle, - owned: OwnedTasks::new(spawn_concurrency_level as u32), + owned: OwnedTasks::new(num_cores), synced: Mutex::new(Synced { assigned_cores: (0..num_workers).map(|_| None).collect(), shutdown_cores: Vec::with_capacity(num_cores), diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 44873540986..8a446656d05 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -68,15 +68,17 @@ pub(crate) struct LocalOwnedTasks { pub(crate) id: NonZeroU64, _not_send_or_sync: PhantomData<*const ()>, } + struct OwnedTasksInner { list: LinkedList, as Link>::Target>, closed: bool, } impl OwnedTasks { - pub(crate) fn new(concurrency_level: u32) -> Self { + pub(crate) fn new(num_cores: usize) -> Self { + let shard_size = Self::gen_shared_list_size(num_cores); Self { - list: List::new(concurrency_level as usize), + list: List::new(shard_size), closed: AtomicBool::new(false), id: get_next_id(), } @@ -183,6 +185,22 @@ impl OwnedTasks { pub(crate) fn is_empty(&self) -> bool { self.list.is_empty() } + + fn gen_shared_list_size(num_cores: usize) -> usize { + // Shrink the size of shared_list_size when using loom. This shouldn't impact + // logic, but allows loom to test more edge cases in a reasoable a mount of time. + #[cfg(loom)] + return 4; + #[cfg(not(loom))] + { + const MAX_SHARED_LIST_SIZE: usize = 1 << 16; + let mut size = 1; + while size / 4 < num_cores && size < MAX_SHARED_LIST_SIZE { + size <<= 1; + } + size.min(MAX_SHARED_LIST_SIZE) + } + } } cfg_taskdump! { From 06656b923c4939b930410c793716f020411fd523 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Fri, 24 Nov 2023 10:24:19 +0800 Subject: [PATCH 89/94] feat: add comments and rm loom cfg --- tokio/src/runtime/task/list.rs | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 8a446656d05..78b211c4f0b 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -185,21 +185,23 @@ impl OwnedTasks { pub(crate) fn is_empty(&self) -> bool { self.list.is_empty() } - + /// Generates the size of the shared list based on the number of worker threads. + /// + /// The sharded lock design can effectively alleviate + /// lock contention performance problems caused by high concurrency. + /// + /// However, as the number of shards increases, the memory continuity between nodes in + /// the intrusive linked list will decrease. Moreover, as the number of shards increases, + /// the construction time of the sharded list will increase. + /// + /// For the above reasons, we set a maximum value for the shared list size `MAX_SHARED_LIST_SIZE`. fn gen_shared_list_size(num_cores: usize) -> usize { - // Shrink the size of shared_list_size when using loom. This shouldn't impact - // logic, but allows loom to test more edge cases in a reasoable a mount of time. - #[cfg(loom)] - return 4; - #[cfg(not(loom))] - { - const MAX_SHARED_LIST_SIZE: usize = 1 << 16; - let mut size = 1; - while size / 4 < num_cores && size < MAX_SHARED_LIST_SIZE { - size <<= 1; - } - size.min(MAX_SHARED_LIST_SIZE) + const MAX_SHARED_LIST_SIZE: usize = 1 << 16; + let mut size = 1; + while size / 4 < num_cores && size < MAX_SHARED_LIST_SIZE { + size <<= 1; } + size.min(MAX_SHARED_LIST_SIZE) } } From caa74c944ef729689f87c89df09c1d9d7ccfb24d Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Fri, 24 Nov 2023 10:42:31 +0800 Subject: [PATCH 90/94] feat: update comments for gen_shared_list_size --- tokio/src/runtime/task/list.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index 78b211c4f0b..a72be4343d4 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -190,9 +190,9 @@ impl OwnedTasks { /// The sharded lock design can effectively alleviate /// lock contention performance problems caused by high concurrency. /// - /// However, as the number of shards increases, the memory continuity between nodes in - /// the intrusive linked list will decrease. Moreover, as the number of shards increases, - /// the construction time of the sharded list will increase. + /// However, as the number of shards increases, the memory continuity between + /// nodes in the intrusive linked list will diminish. Furthermore, + /// the construction time of the sharded list will also increase with a higher number of shards. /// /// For the above reasons, we set a maximum value for the shared list size `MAX_SHARED_LIST_SIZE`. fn gen_shared_list_size(num_cores: usize) -> usize { From 865de083f2fc564eea92e10156d5279da23e4f45 Mon Sep 17 00:00:00 2001 From: wathenjiang Date: Fri, 24 Nov 2023 10:45:17 +0800 Subject: [PATCH 91/94] feat: update comments for gen_shared_list_size --- tokio/src/runtime/task/list.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index a72be4343d4..de1b33584cb 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -194,7 +194,8 @@ impl OwnedTasks { /// nodes in the intrusive linked list will diminish. Furthermore, /// the construction time of the sharded list will also increase with a higher number of shards. /// - /// For the above reasons, we set a maximum value for the shared list size `MAX_SHARED_LIST_SIZE`. + /// Due to the above reasons, we set a maximum value for the shared list size, + /// denoted as `MAX_SHARED_LIST_SIZE`. fn gen_shared_list_size(num_cores: usize) -> usize { const MAX_SHARED_LIST_SIZE: usize = 1 << 16; let mut size = 1; From 7a76ad50e01e23c9d2163edbc1ab8c7876d34fe3 Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Sun, 26 Nov 2023 09:18:18 +0800 Subject: [PATCH 92/94] Apply suggestions from code review Co-authored-by: Alice Ryhl --- tokio/src/runtime/task/list.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index de1b33584cb..c30764e3cb6 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -185,6 +185,7 @@ impl OwnedTasks { pub(crate) fn is_empty(&self) -> bool { self.list.is_empty() } + /// Generates the size of the shared list based on the number of worker threads. /// /// The sharded lock design can effectively alleviate @@ -198,11 +199,7 @@ impl OwnedTasks { /// denoted as `MAX_SHARED_LIST_SIZE`. fn gen_shared_list_size(num_cores: usize) -> usize { const MAX_SHARED_LIST_SIZE: usize = 1 << 16; - let mut size = 1; - while size / 4 < num_cores && size < MAX_SHARED_LIST_SIZE { - size <<= 1; - } - size.min(MAX_SHARED_LIST_SIZE) + usize::min(MAX_SHARED_LIST_SIZE, num_cores.next_power_of_two()*4) } } From 78d1deaa23e5cb0f8dca4e13275e8b34a6d2611c Mon Sep 17 00:00:00 2001 From: Weijia Jiang Date: Sun, 26 Nov 2023 09:35:35 +0800 Subject: [PATCH 93/94] fix: fmt and typo fix --- tokio/src/runtime/task/list.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tokio/src/runtime/task/list.rs b/tokio/src/runtime/task/list.rs index c30764e3cb6..3d2a121cf1d 100644 --- a/tokio/src/runtime/task/list.rs +++ b/tokio/src/runtime/task/list.rs @@ -186,7 +186,7 @@ impl OwnedTasks { self.list.is_empty() } - /// Generates the size of the shared list based on the number of worker threads. + /// Generates the size of the sharded list based on the number of worker threads. /// /// The sharded lock design can effectively alleviate /// lock contention performance problems caused by high concurrency. @@ -199,7 +199,7 @@ impl OwnedTasks { /// denoted as `MAX_SHARED_LIST_SIZE`. fn gen_shared_list_size(num_cores: usize) -> usize { const MAX_SHARED_LIST_SIZE: usize = 1 << 16; - usize::min(MAX_SHARED_LIST_SIZE, num_cores.next_power_of_two()*4) + usize::min(MAX_SHARED_LIST_SIZE, num_cores.next_power_of_two() * 4) } } From 3844bd321eeeba0f4100ddb11767fa8dc9c4bc96 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 7 Dec 2023 11:32:27 +0100 Subject: [PATCH 94/94] Update tokio/src/util/sharded_list.rs --- tokio/src/util/sharded_list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokio/src/util/sharded_list.rs b/tokio/src/util/sharded_list.rs index 140c8efb1d9..c1009db94c9 100644 --- a/tokio/src/util/sharded_list.rs +++ b/tokio/src/util/sharded_list.rs @@ -109,7 +109,7 @@ impl ShardedList { /// Gets the shard size of this SharedList. /// - /// Used to help us to decide the parameter `shard_id`` of the `pop_back` method. + /// Used to help us to decide the parameter `shard_id` of the `pop_back` method. pub(crate) fn shard_size(&self) -> usize { self.shard_mask + 1 }