Deadlock on PeerPessimisticLocks between raftstore and shceduler workers

[overvenus]

Bug Report

Found dead locks between scheduler workers and raftstore.
This is also the root cause of #16337 #16290 and #16235

Raftstore backtrace

Thread 148 (Thread 0x7fa6e632d640 (LWP 177) "raftstore-80024"):
#0  0x00007fa7893d9e5d in syscall () from /lib64/libc.so.6
#1  0x000055df4c6135c4 in parking_lot::raw_rwlock::RawRwLock::lock_exclusive_slow (self=0x7fa553ba1df0, timeout=...) at /rust/registry/src/index.crates.io-6f17d22bba15001f/parking_lot_core-0.9.1/src/thread_parker/linux.rs:112
#2  0x000055df4d959ced in parking_lot::raw_rwlock::{impl#0}::lock_exclusive (self=0x7fa553ba1df0) at /rust/registry/src/index.crates.io-6f17d22bba15001f/parking_lot-0.12.1/src/raw_rwlock.rs:73
#3  lock_api::rwlock::RwLock<parking_lot::raw_rwlock::RawRwLock, raftstore::store::txn_ext::PeerPessimisticLocks>::write<parking_lot::raw_rwlock::RawRwLock, raftstore::store::txn_ext::PeerPessimisticLocks> (self=<optimized out>) at /rust/registry/src/index.crates.io-6f17d22bba15001f/lock_api-0.4.6/src/rwlock.rs:480
#4  raftstore::store::peer::Peer<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>::clear_in_memory_pessimistic_locks<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine> (self=0x7fa63747fc00) at components/raftstore/src/store/peer.rs:5560
#5  raftstore::store::peer::Peer<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>::on_role_changed<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine, tikv::server::transport::ServerTransport<tikv::server::raftkv::raft_extension::RaftRouterWrap<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, engine_rocks::engine::RocksEngine>, tikv::server::resolve::PdStoreAddrResolver>> (self=0x7fa63747fc00, ctx=0x7fa6e62f87e0, ready=0x7fa6e62f7210) at components/raftstore/src/store/peer.rs:2222
#6  raftstore::store::peer::Peer<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>::handle_raft_ready_append<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine, tikv::server::transport::ServerTransport<tikv::server::raftkv::raft_extension::RaftRouterWrap<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, engine_rocks::engine::RocksEngine>, tikv::server::resolve::PdStoreAddrResolver>> (self=0x7fa63747fc00, ctx=0x7fa6e62f87e0) at components/raftstore/src/store/peer.rs:2675
#7  0x000055df4d900f3e in raftstore::store::fsm::peer::PeerFsmDelegate<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine, tikv::server::transport::ServerTransport<tikv::server::raftkv::raft_extension::RaftRouterWrap<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, engine_rocks::engine::RocksEngine>, tikv::server::resolve::PdStoreAddrResolver>>::collect_ready<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine, tikv::server::transport::ServerTransport<tikv::server::raftkv::raft_extension::RaftRouterWrap<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, engine_rocks::engine::RocksEngine>, tikv::server::resolve::PdStoreAddrResolver>> (self=0x7fa6e62f7c68) at components/raftstore/src/store/fsm/peer.rs:2053
#8  raftstore::store::fsm::store::{impl#14}::handle_normal<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine, tikv::server::transport::ServerTransport<tikv::server::raftkv::raft_extension::RaftRouterWrap<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, engine_rocks::engine::RocksEngine>, tikv::server::resolve::PdStoreAddrResolver>, batch_system::batch::NormalFsm<raftstore::store::fsm::peer::PeerFsm<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>> (self=0x7fa6e62f87c8, peer=<optimized out>) at components/raftstore/src/store/fsm/store.rs:1055
#9  0x000055df4d8f222f in batch_system::batch::Poller<raftstore::store::fsm::peer::PeerFsm<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, raftstore::store::fsm::store::StoreFsm<engine_rocks::engine::RocksEngine>, raftstore::store::fsm::store::RaftPoller<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine, tikv::server::transport::ServerTransport<tikv::server::raftkv::raft_extension::RaftRouterWrap<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, engine_rocks::engine::RocksEngine>, tikv::server::resolve::PdStoreAddrResolver>>>::poll<raftstore::store::fsm::peer::PeerFsm<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, raftstore::store::fsm::store::StoreFsm<engine_rocks::engine::RocksEngine>, raftstore::store::fsm::store::RaftPoller<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine, tikv::server::transport::ServerTransport<tikv::server::raftkv::raft_extension::RaftRouterWrap<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, engine_rocks::engine::RocksEngine>, tikv::server::resolve::PdStoreAddrResolver>>> (self=0x7fa6e62f87c8) at components/batch-system/src/batch.rs:380
#10 0x000055df4d8f1792 in raftstore::store::worker::refresh_config::{impl#1}::increase_by::{closure#0}<raftstore::store::fsm::peer::PeerFsm<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, raftstore::store::fsm::store::StoreFsm<engine_rocks::engine::RocksEngine>, raftstore::store::fsm::store::RaftPollerBuilder<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine, tikv::server::transport::ServerTransport<tikv::server::raftkv::raft_extension::RaftRouterWrap<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, engine_rocks::engine::RocksEngine>, tikv::server::resolve::PdStoreAddrResolver>>> () at components/raftstore/src/store/worker/refresh_config.rs:84
#11 tikv_util::sys::thread::{impl#1}::spawn_wrapper::{closure#0}<raftstore::store::worker::refresh_config::{impl#1}::increase_by::{closure_env#0}<raftstore::store::fsm::peer::PeerFsm<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, raftstore::store::fsm::store::StoreFsm<engine_rocks::engine::RocksEngine>, raftstore::store::fsm::store::RaftPollerBuilder<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine, tikv::server::transport::ServerTransport<tikv::server::raftkv::raft_extension::RaftRouterWrap<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, engine_rocks::engine::RocksEngine>, tikv::server::resolve::PdStoreAddrResolver>>>, ()> () at components/tikv_util/src/sys/thread.rs:438
#12 std::sys_common::backtrace::__rust_begin_short_backtrace<tikv_util::sys::thread::{impl#1}::spawn_wrapper::{closure_env#0}<raftstore::store::worker::refresh_config::{impl#1}::increase_by::{closure_env#0}<raftstore::store::fsm::peer::PeerFsm<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, raftstore::store::fsm::store::StoreFsm<engine_rocks::engine::RocksEngine>, raftstore::store::fsm::store::RaftPollerBuilder<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine, tikv::server::transport::ServerTransport<tikv::server::raftkv::raft_extension::RaftRouterWrap<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, engine_rocks::engine::RocksEngine>, tikv::server::resolve::PdStoreAddrResolver>>>, ()>, ()> (f=...) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155
#13 0x000055df4d9dd9c8 in std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure#0}<tikv_util::sys::thread::{impl#1}::spawn_wrapper::{closure_env#0}<batch_system::batch::{impl#7}::start_poller::{closure_env#0}<raftstore::store::fsm::peer::PeerFsm<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, raftstore::store::fsm::store::StoreFsm<engine_rocks::engine::RocksEngine>, raftstore::store::fsm::store::RaftPollerBuilder<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine, tikv::server::transport::ServerTransport<tikv::server::raftkv::raft_extension::RaftRouterWrap<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, engine_rocks::engine::RocksEngine>, tikv::server::resolve::PdStoreAddrResolver>>>, ()>, ()> () at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529
#14 core::panic::unwind_safe::{impl#23}::call_once<(), std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<tikv_util::sys::thread::{impl#1}::spawn_wrapper::{closure_env#0}<batch_system::batch::{impl#7}::start_poller::{closure_env#0}<raftstore::store::fsm::peer::PeerFsm<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, raftstore::store::fsm::store::StoreFsm<engine_rocks::engine::RocksEngine>, raftstore::store::fsm::store::RaftPollerBuilder<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine, tikv::server::transport::ServerTransport<tikv::server::raftkv::raft_extension::RaftRouterWrap<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, engine_rocks::engine::RocksEngine>, tikv::server::resolve::PdStoreAddrResolver>>>, ()>, ()>> (self=<error reading variable: Cannot access memory at address 0x29c0>) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272
#15 std::panicking::try::do_call<core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<tikv_util::sys::thread::{impl#1}::spawn_wrapper::{closure_env#0}<batch_system::batch::{impl#7}::start_poller::{closure_env#0}<raftstore::store::fsm::peer::PeerFsm<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, raftstore::store::fsm::store::StoreFsm<engine_rocks::engine::RocksEngine>, raftstore::store::fsm::store::RaftPollerBuilder<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine, tikv::server::transport::ServerTransport<tikv::server::raftkv::raft_extension::RaftRouterWrap<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, engine_rocks::engine::RocksEngine>, tikv::server::resolve::PdStoreAddrResolver>>>, ()>, ()>>, ()> (data=<optimized out>) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552
#16 std::panicking::try<(), core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<tikv_util::sys::thread::{impl#1}::spawn_wrapper::{closure_env#0}<batch_system::batch::{impl#7}::start_poller::{closure_env#0}<raftstore::store::fsm::peer::PeerFsm<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, raftstore::store::fsm::store::StoreFsm<engine_rocks::engine::RocksEngine>, raftstore::store::fsm::store::RaftPollerBuilder<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine, tikv::server::transport::ServerTransport<tikv::server::raftkv::raft_extension::RaftRouterWrap<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, engine_rocks::engine::RocksEngine>, tikv::server::resolve::PdStoreAddrResolver>>>, ()>, ()>>> (f=<error reading variable: Cannot access memory at address 0x29c0>) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516
#17 std::panic::catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<tikv_util::sys::thread::{impl#1}::spawn_wrapper::{closure_env#0}<batch_system::batch::{impl#7}::start_poller::{closure_env#0}<raftstore::store::fsm::peer::PeerFsm<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, raftstore::store::fsm::store::StoreFsm<engine_rocks::engine::RocksEngine>, raftstore::store::fsm::store::RaftPollerBuilder<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine, tikv::server::transport::ServerTransport<tikv::server::raftkv::raft_extension::RaftRouterWrap<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, engine_rocks::engine::RocksEngine>, tikv::server::resolve::PdStoreAddrResolver>>>, ()>, ()>>, ()> (f=<error reading variable: Cannot access memory at address 0x29c0>) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142
#18 std::thread::{impl#0}::spawn_unchecked_::{closure#1}<tikv_util::sys::thread::{impl#1}::spawn_wrapper::{closure_env#0}<batch_system::batch::{impl#7}::start_poller::{closure_env#0}<raftstore::store::fsm::peer::PeerFsm<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, raftstore::store::fsm::store::StoreFsm<engine_rocks::engine::RocksEngine>, raftstore::store::fsm::store::RaftPollerBuilder<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine, tikv::server::transport::ServerTransport<tikv::server::raftkv::raft_extension::RaftRouterWrap<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, engine_rocks::engine::RocksEngine>, tikv::server::resolve::PdStoreAddrResolver>>>, ()>, ()> () at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528
#19 core::ops::function::FnOnce::call_once<std::thread::{impl#0}::spawn_unchecked_::{closure_env#1}<tikv_util::sys::thread::{impl#1}::spawn_wrapper::{closure_env#0}<batch_system::batch::{impl#7}::start_poller::{closure_env#0}<raftstore::store::fsm::peer::PeerFsm<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, raftstore::store::fsm::store::StoreFsm<engine_rocks::engine::RocksEngine>, raftstore::store::fsm::store::RaftPollerBuilder<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine, tikv::server::transport::ServerTransport<tikv::server::raftkv::raft_extension::RaftRouterWrap<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>, engine_rocks::engine::RocksEngine>, tikv::server::resolve::PdStoreAddrResolver>>>, ()>, ()>, ()> () at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250
#20 0x000055df4f09c619 in std::sys::unix::thread::{impl#2}::new::thread_start (main=0x7fa6eff8e750) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015
#21 0x00007fa78943a802 in start_thread () from /lib64/libc.so.6
#22 0x00007fa7893da314 in clone () from /lib64/libc.so.6

Scheduler worker backtrace

Thread 69 (Thread 0x7fa6f0dfc640 (LWP 98) "sched-worker-po"):
#0  0x00007fa7893d9e5d in syscall () from /lib64/libc.so.6
#1  0x000055df4c6140f4 in parking_lot::raw_rwlock::RawRwLock::lock_shared_slow (self=0x7fa553ba1df0, recursive=false, timeout=...) at /rust/registry/src/index.crates.io-6f17d22bba15001f/parking_lot_core-0.9.1/src/thread_parker/linux.rs:112
#2  0x000055df4d6f20ed in parking_lot::raw_rwlock::{impl#0}::lock_shared (self=<optimized out>) at /rust/registry/src/index.crates.io-6f17d22bba15001f/parking_lot-0.12.1/src/raw_rwlock.rs:109
#3  lock_api::rwlock::RwLock<parking_lot::raw_rwlock::RawRwLock, raftstore::store::txn_ext::PeerPessimisticLocks>::read<parking_lot::raw_rwlock::RawRwLock, raftstore::store::txn_ext::PeerPessimisticLocks> (self=<optimized out>) at /rust/registry/src/index.crates.io-6f17d22bba15001f/lock_api-0.4.6/src/rwlock.rs:448
#4  tikv::storage::mvcc::reader::reader::MvccReader<raftstore::store::region_snapshot::RegionSnapshot<engine_rocks::snapshot::RocksSnapshot>>::load_in_memory_pessimistic_lock<raftstore::store::region_snapshot::RegionSnapshot<engine_rocks::snapshot::RocksSnapshot>> (self=0x7fa6f0dc9220, key=<optimized out>) at src/storage/mvcc/reader/reader.rs:421
#5  tikv::storage::mvcc::reader::reader::MvccReader<raftstore::store::region_snapshot::RegionSnapshot<engine_rocks::snapshot::RocksSnapshot>>::load_lock<raftstore::store::region_snapshot::RegionSnapshot<engine_rocks::snapshot::RocksSnapshot>> (self=0x7fa6f0dc9220, key=0x7fa6f0dc9bc0) at src/storage/mvcc/reader/reader.rs:235
#6  0x000055df4d6f0796 in tikv::storage::mvcc::reader::reader::SnapshotReader<raftstore::store::region_snapshot::RegionSnapshot<engine_rocks::snapshot::RocksSnapshot>>::load_lock<raftstore::store::region_snapshot::RegionSnapshot<engine_rocks::snapshot::RocksSnapshot>> (self=<optimized out>, key=<optimized out>) at src/storage/mvcc/reader/reader.rs:68
#7  tikv::storage::txn::actions::commit::commit<raftstore::store::region_snapshot::RegionSnapshot<engine_rocks::snapshot::RocksSnapshot>> (txn=0x7fa6f0dc96a0, reader=0x7fa6f0dc9220, key=..., commit_ts=...) at src/storage/txn/actions/commit.rs:24
#8  0x000055df4d6c40fb in tikv::storage::txn::commands::commit::{impl#1}::process_write<raftstore::store::region_snapshot::RegionSnapshot<engine_rocks::snapshot::RocksSnapshot>, tikv::server::lock_manager::LockManager> (self=..., snapshot=..., context=...) at src/storage/txn/commands/commit.rs:64
#9  tikv::storage::txn::commands::Command::process_write<raftstore::store::region_snapshot::RegionSnapshot<engine_rocks::snapshot::RocksSnapshot>, tikv::server::lock_manager::LockManager> (self=..., snapshot=..., context=...) at src/storage/txn/commands/mod.rs:713
#10 0x000055df4d6b4361 in tikv::storage::txn::scheduler::{impl#6}::process_write::{async_fn#0}::{closure#1}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager> () at src/storage/txn/scheduler.rs:1287
#11 tikv::storage::metrics::with_perf_context::{closure#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::storage::txn::scheduler::{impl#6}::process_write::{async_fn#0}::{closure_env#1}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>, core::result::Result<tikv::storage::txn::commands::WriteResult, tikv::storage::errors::Error>> (c=...) at src/storage/metrics.rs:361
#12 std::thread::local::LocalKey<core::cell::RefCell<core::option::Option<alloc::boxed::Box<dyn engine_traits::perf_context::PerfContext, alloc::alloc::Global>>>>::try_with<core::cell::RefCell<core::option::Option<alloc::boxed::Box<dyn engine_traits::perf_context::PerfContext, alloc::alloc::Global>>>, tikv::storage::metrics::with_perf_context::{closure_env#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::storage::txn::scheduler::{impl#6}::process_write::{async_fn#0}::{closure_env#1}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>, core::result::Result<tikv::storage::txn::commands::WriteResult, tikv::storage::errors::Error>>, core::result::Result<tikv::storage::txn::commands::WriteResult, tikv::storage::errors::Error>> (f=..., self=<optimized out>) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:270
#13 std::thread::local::LocalKey<core::cell::RefCell<core::option::Option<alloc::boxed::Box<dyn engine_traits::perf_context::PerfContext, alloc::alloc::Global>>>>::with<core::cell::RefCell<core::option::Option<alloc::boxed::Box<dyn engine_traits::perf_context::PerfContext, alloc::alloc::Global>>>, tikv::storage::metrics::with_perf_context::{closure_env#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::storage::txn::scheduler::{impl#6}::process_write::{async_fn#0}::{closure_env#1}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>, core::result::Result<tikv::storage::txn::commands::WriteResult, tikv::storage::errors::Error>>, core::result::Result<tikv::storage::txn::commands::WriteResult, tikv::storage::errors::Error>> (self=<optimized out>, f=<error reading variable: Cannot access memory at address 0x0>) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:246
#14 tikv::storage::metrics::with_perf_context<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::storage::txn::scheduler::{impl#6}::process_write::{async_fn#0}::{closure_env#1}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>, core::result::Result<tikv::storage::txn::commands::WriteResult, tikv::storage::errors::Error>> (cmd=<optimized out>, f=...) at src/storage/metrics.rs:352
#15 tikv::storage::txn::scheduler::{impl#6}::process_write::{async_fn#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager> () at src/storage/txn/scheduler.rs:1286
#16 0x000055df4d6a9ad4 in tikv::storage::txn::scheduler::{impl#6}::process::{async_fn#0}::{async_block#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager> () at src/storage/txn/scheduler.rs:1177
#17 resource_metering::{impl#5}::poll<tikv::storage::txn::scheduler::{impl#6}::process::{async_fn#0}::{async_block_env#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>> (self=..., cx=0x7fa6f0dcef58) at components/resource_metering/src/lib.rs:267
#18 tikv::storage::txn::scheduler::{impl#6}::process::{async_fn#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager> () at src/storage/txn/scheduler.rs:1191
#19 tikv::storage::txn::scheduler::{impl#6}::execute::{async_block#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager> () at src/storage/txn/scheduler.rs:748
#20 0x000055df4d709f51 in tracker::tls::{impl#1}::poll::{closure#0}<tikv::storage::txn::scheduler::{impl#6}::execute::{async_block_env#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>> (c=<optimized out>) at components/tracker/src/tls.rs:64
#21 std::thread::local::LocalKey<core::cell::Cell<tracker::slab::TrackerToken>>::try_with<core::cell::Cell<tracker::slab::TrackerToken>, tracker::tls::{impl#1}::poll::{closure_env#0}<tikv::storage::txn::scheduler::{impl#6}::execute::{async_block_env#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>>, core::task::poll::Poll<()>> (f=..., self=<optimized out>) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:270
#22 std::thread::local::LocalKey<core::cell::Cell<tracker::slab::TrackerToken>>::with<core::cell::Cell<tracker::slab::TrackerToken>, tracker::tls::{impl#1}::poll::{closure_env#0}<tikv::storage::txn::scheduler::{impl#6}::execute::{async_block_env#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>>, core::task::poll::Poll<()>> (f=...) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:246
#23 tracker::tls::{impl#1}::poll<tikv::storage::txn::scheduler::{impl#6}::execute::{async_block_env#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>> (self=..., cx=0x7fa6f0dcef58) at components/tracker/src/tls.rs:62
#24 tikv_util::yatp_pool::future_pool::{impl#4}::spawn::{async_block#1}<tracker::tls::TrackedFuture<tikv::storage::txn::scheduler::{impl#6}::execute::{async_block_env#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>>> () at components/tikv_util/src/yatp_pool/future_pool.rs:220
#25 yatp::task::future::RawTask<tikv_util::yatp_pool::future_pool::{impl#4}::spawn::{async_block_env#1}<tracker::tls::TrackedFuture<tikv::storage::txn::scheduler::{impl#6}::execute::{async_block_env#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>>>>::poll<tikv_util::yatp_pool::future_pool::{impl#4}::spawn::{async_block_env#1}<tracker::tls::TrackedFuture<tikv::storage::txn::scheduler::{impl#6}::execute::{async_block_env#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>>>> (task=<optimized out>, cx=0x7fa6f0dcef58) at /rust/git/checkouts/yatp-e704b73c3ee279b6/5572a78/src/task/future.rs:59
#26 0x000055df4f96b74a in yatp::task::future::{impl#15}::handle (self=<optimized out>, local=0x7fa6f0dcf1a0, task_cell=...) at src/task/future.rs:103
#27 0x000055df4dd38ae4 in tikv_util::yatp_pool::{impl#4}::handle<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>> (self=0x7fa6f0dcf080, local=0x80, task_cell=...) at components/tikv_util/src/yatp_pool/mod.rs:198
#28 0x000055df4dd397ff in yatp::pool::worker::WorkerThread<yatp::task::future::TaskCell, tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>::run<yatp::task::future::TaskCell, tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>> (self=...) at /rust/git/checkouts/yatp-e704b73c3ee279b6/5572a78/src/pool/worker.rs:48
#29 yatp::pool::builder::{impl#3}::build::{closure#0}<yatp::task::future::TaskCell, yatp::pool::runner::CloneRunnerBuilder<tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>> () at /rust/git/checkouts/yatp-e704b73c3ee279b6/5572a78/src/pool/builder.rs:114
#30 std::sys_common::backtrace::__rust_begin_short_backtrace<yatp::pool::builder::{impl#3}::build::{closure_env#0}<yatp::task::future::TaskCell, yatp::pool::runner::CloneRunnerBuilder<tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>>, ()> (f=...) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155
#31 0x000055df4dd394bd in std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure#0}<yatp::pool::builder::{impl#3}::build::{closure_env#0}<yatp::task::future::TaskCell, yatp::pool::runner::CloneRunnerBuilder<tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>>, ()> () at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529
#32 core::panic::unwind_safe::{impl#23}::call_once<(), std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<yatp::pool::builder::{impl#3}::build::{closure_env#0}<yatp::task::future::TaskCell, yatp::pool::runner::CloneRunnerBuilder<tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>>, ()>> (self=...) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272
#33 std::panicking::try::do_call<core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<yatp::pool::builder::{impl#3}::build::{closure_env#0}<yatp::task::future::TaskCell, yatp::pool::runner::CloneRunnerBuilder<tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>>, ()>>, ()> (data=<optimized out>) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552
#34 std::panicking::try<(), core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<yatp::pool::builder::{impl#3}::build::{closure_env#0}<yatp::task::future::TaskCell, yatp::pool::runner::CloneRunnerBuilder<tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>>, ()>>> (f=<error reading variable: Cannot access memory at address 0x170>) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516
#35 std::panic::catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<yatp::pool::builder::{impl#3}::build::{closure_env#0}<yatp::task::future::TaskCell, yatp::pool::runner::CloneRunnerBuilder<tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>>, ()>>, ()> (f=<error reading variable: Cannot access memory at address 0x170>) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142
#36 std::thread::{impl#0}::spawn_unchecked_::{closure#1}<yatp::pool::builder::{impl#3}::build::{closure_env#0}<yatp::task::future::TaskCell, yatp::pool::runner::CloneRunnerBuilder<tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>>, ()> () at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528
#37 core::ops::function::FnOnce::call_once<std::thread::{impl#0}::spawn_unchecked_::{closure_env#1}<yatp::pool::builder::{impl#3}::build::{closure_env#0}<yatp::task::future::TaskCell, yatp::pool::runner::CloneRunnerBuilder<tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>>, ()>, ()> () at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250
#38 0x000055df4f09c619 in std::sys::unix::thread::{impl#2}::new::thread_start (main=0x7fa788f66db0) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015
#39 0x00007fa78943a802 in start_thread () from /lib64/libc.so.6
#40 0x00007fa7893da314 in clone () from /lib64/libc.so.6

Thread 62 (Thread 0x7fa6f2dff640 (LWP 91) "sched-worker-po"):
#0  0x00007fa7893d9e5d in syscall () from /lib64/libc.so.6
#1  0x000055df4ce32d1f in parking_lot::raw_rwlock::RawRwLock::wait_for_readers (self=<optimized out>, timeout=..., prev_value=<optimized out>) at /rust/registry/src/index.crates.io-6f17d22bba15001f/parking_lot_core-0.9.1/src/thread_parker/linux.rs:112
#2  0x000055df4c613619 in parking_lot::raw_rwlock::RawRwLock::lock_exclusive_slow (self=0x7fa553ba1df0, timeout=...) at src/raw_rwlock.rs:644
#3  0x000055df4d6b7f63 in parking_lot::raw_rwlock::{impl#0}::lock_exclusive (self=0x7fa553ba1df0) at /rust/registry/src/index.crates.io-6f17d22bba15001f/parking_lot-0.12.1/src/raw_rwlock.rs:73
#4  lock_api::rwlock::RwLock<parking_lot::raw_rwlock::RawRwLock, raftstore::store::txn_ext::PeerPessimisticLocks>::write<parking_lot::raw_rwlock::RawRwLock, raftstore::store::txn_ext::PeerPessimisticLocks> (self=<optimized out>) at /rust/registry/src/index.crates.io-6f17d22bba15001f/lock_api-0.4.6/src/rwlock.rs:480
#5  tikv::storage::txn::scheduler::{impl#6}::process_write::{async_fn#0}::{closure#3}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager> (txn_ext=<optimized out>) at src/storage/txn/scheduler.rs:1548
#6  core::option::Option<&alloc::sync::Arc<raftstore::store::txn_ext::TxnExt, alloc::alloc::Global>>::map<&alloc::sync::Arc<raftstore::store::txn_ext::TxnExt, alloc::alloc::Global>, lock_api::rwlock::RwLockWriteGuard<parking_lot::raw_rwlock::RawRwLock, raftstore::store::txn_ext::PeerPessimisticLocks>, tikv::storage::txn::scheduler::{impl#6}::process_write::{async_fn#0}::{closure_env#3}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>> (self=..., f=...) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/option.rs:1072
#7  tikv::storage::txn::scheduler::{impl#6}::process_write::{async_fn#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager> () at src/storage/txn/scheduler.rs:1546
#8  0x000055df4d6a9ad4 in tikv::storage::txn::scheduler::{impl#6}::process::{async_fn#0}::{async_block#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager> () at src/storage/txn/scheduler.rs:1177
#9  resource_metering::{impl#5}::poll<tikv::storage::txn::scheduler::{impl#6}::process::{async_fn#0}::{async_block_env#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>> (self=..., cx=0x7fa6f2dd1f58) at components/resource_metering/src/lib.rs:267
#10 tikv::storage::txn::scheduler::{impl#6}::process::{async_fn#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager> () at src/storage/txn/scheduler.rs:1191
#11 tikv::storage::txn::scheduler::{impl#6}::execute::{async_block#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager> () at src/storage/txn/scheduler.rs:748
#12 0x000055df4d709f51 in tracker::tls::{impl#1}::poll::{closure#0}<tikv::storage::txn::scheduler::{impl#6}::execute::{async_block_env#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>> (c=<optimized out>) at components/tracker/src/tls.rs:64
#13 std::thread::local::LocalKey<core::cell::Cell<tracker::slab::TrackerToken>>::try_with<core::cell::Cell<tracker::slab::TrackerToken>, tracker::tls::{impl#1}::poll::{closure_env#0}<tikv::storage::txn::scheduler::{impl#6}::execute::{async_block_env#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>>, core::task::poll::Poll<()>> (f=..., self=<optimized out>) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:270
#14 std::thread::local::LocalKey<core::cell::Cell<tracker::slab::TrackerToken>>::with<core::cell::Cell<tracker::slab::TrackerToken>, tracker::tls::{impl#1}::poll::{closure_env#0}<tikv::storage::txn::scheduler::{impl#6}::execute::{async_block_env#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>>, core::task::poll::Poll<()>> (f=...) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:246
#15 tracker::tls::{impl#1}::poll<tikv::storage::txn::scheduler::{impl#6}::execute::{async_block_env#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>> (self=..., cx=0x7fa6f2dd1f58) at components/tracker/src/tls.rs:62
#16 tikv_util::yatp_pool::future_pool::{impl#4}::spawn::{async_block#1}<tracker::tls::TrackedFuture<tikv::storage::txn::scheduler::{impl#6}::execute::{async_block_env#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>>> () at components/tikv_util/src/yatp_pool/future_pool.rs:220
#17 yatp::task::future::RawTask<tikv_util::yatp_pool::future_pool::{impl#4}::spawn::{async_block_env#1}<tracker::tls::TrackedFuture<tikv::storage::txn::scheduler::{impl#6}::execute::{async_block_env#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>>>>::poll<tikv_util::yatp_pool::future_pool::{impl#4}::spawn::{async_block_env#1}<tracker::tls::TrackedFuture<tikv::storage::txn::scheduler::{impl#6}::execute::{async_block_env#0}<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine, raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>, tikv::server::lock_manager::LockManager>>>> (task=<optimized out>, cx=0x7fa6f2dd1f58) at /rust/git/checkouts/yatp-e704b73c3ee279b6/5572a78/src/task/future.rs:59
#18 0x000055df4f96b74a in yatp::task::future::{impl#15}::handle (self=<optimized out>, local=0x7fa6f2dd21a0, task_cell=...) at src/task/future.rs:103
#19 0x000055df4dd38ae4 in tikv_util::yatp_pool::{impl#4}::handle<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>> (self=0x7fa6f2dd2080, local=0x80, task_cell=...) at components/tikv_util/src/yatp_pool/mod.rs:198
#20 0x000055df4dd397ff in yatp::pool::worker::WorkerThread<yatp::task::future::TaskCell, tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>::run<yatp::task::future::TaskCell, tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>> (self=...) at /rust/git/checkouts/yatp-e704b73c3ee279b6/5572a78/src/pool/worker.rs:48
#21 yatp::pool::builder::{impl#3}::build::{closure#0}<yatp::task::future::TaskCell, yatp::pool::runner::CloneRunnerBuilder<tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>> () at /rust/git/checkouts/yatp-e704b73c3ee279b6/5572a78/src/pool/builder.rs:114
#22 std::sys_common::backtrace::__rust_begin_short_backtrace<yatp::pool::builder::{impl#3}::build::{closure_env#0}<yatp::task::future::TaskCell, yatp::pool::runner::CloneRunnerBuilder<tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>>, ()> (f=...) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155
#23 0x000055df4dd394bd in std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure#0}<yatp::pool::builder::{impl#3}::build::{closure_env#0}<yatp::task::future::TaskCell, yatp::pool::runner::CloneRunnerBuilder<tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>>, ()> () at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529
#24 core::panic::unwind_safe::{impl#23}::call_once<(), std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<yatp::pool::builder::{impl#3}::build::{closure_env#0}<yatp::task::future::TaskCell, yatp::pool::runner::CloneRunnerBuilder<tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>>, ()>> (self=...) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272
#25 std::panicking::try::do_call<core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<yatp::pool::builder::{impl#3}::build::{closure_env#0}<yatp::task::future::TaskCell, yatp::pool::runner::CloneRunnerBuilder<tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>>, ()>>, ()> (data=<optimized out>) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552
#26 std::panicking::try<(), core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<yatp::pool::builder::{impl#3}::build::{closure_env#0}<yatp::task::future::TaskCell, yatp::pool::runner::CloneRunnerBuilder<tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>>, ()>>> (f=<error reading variable: Cannot access memory at address 0x170>) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516
#27 std::panic::catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<yatp::pool::builder::{impl#3}::build::{closure_env#0}<yatp::task::future::TaskCell, yatp::pool::runner::CloneRunnerBuilder<tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>>, ()>>, ()> (f=<error reading variable: Cannot access memory at address 0x170>) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142
#28 std::thread::{impl#0}::spawn_unchecked_::{closure#1}<yatp::pool::builder::{impl#3}::build::{closure_env#0}<yatp::task::future::TaskCell, yatp::pool::runner::CloneRunnerBuilder<tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>>, ()> () at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528
#29 core::ops::function::FnOnce::call_once<std::thread::{impl#0}::spawn_unchecked_::{closure_env#1}<yatp::pool::builder::{impl#3}::build::{closure_env#0}<yatp::task::future::TaskCell, yatp::pool::runner::CloneRunnerBuilder<tikv_util::yatp_pool::YatpPoolRunner<tikv::storage::txn::sched_pool::SchedTicker<tikv_util::worker::pool::Scheduler<raftstore::store::worker::pd::Task<engine_rocks::engine::RocksEngine, raft_log_engine::engine::RaftLogEngine>>>>>>, ()>, ()> () at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250
#30 0x000055df4f09c619 in std::sys::unix::thread::{impl#2}::new::thread_start (main=0x7fa788f66a00) at /rust/toolchains/nightly-2023-12-10-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015
#31 0x00007fa78943a802 in start_thread () from /lib64/libc.so.6
#32 0x00007fa7893da314 in clone () from /lib64/libc.so.6

What version of TiKV are you using?

nightly 2024-01-08

What did you expect?

No deadlock

What did happened?

Deadlock

[cfzjywxk]

Fixed by #16342