tikv panic due to: there is no reactor running, must be called from the context of a Tokio 1.x runtime

[glorv]

Bug Report

What version of TiKV are you using?

v6.5.6, but I think the master code still have this bug

What operating system and CPU are you using?

Steps to reproduce

panic log:

[2024/05/14 21:52:05.575 +08:00] [FATAL] [lib.rs:509] ["there is no reactor running, must be called from the context of a Tokio 1.x runtime"] [backtrace="   0: tikv_util::set_panic_hook::{{closure}}\n             at /home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/tikv_util/src/lib.rs:508:18\n   1: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2032:9\n      std::panicking::rust_panic_with_hook\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:692:13\n   2: std::panicking::begin_panic_handler::{{closure}}\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:579:13\n   3: std::sys_common::backtrace::__rust_end_short_backtrace\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:137:18\n   4: rust_begin_unwind\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:575:5\n   5: core::panicking::panic_fmt\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panicking.rs:65:14\n   6: core::panicking::panic_display\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panicking.rs:138:5\n   7: core::panicking::panic_str\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panicking.rs:122:5\n   8: core::option::expect_failed\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/option.rs:1879:5\n   9: core::option::Option<T>::expect\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/option.rs:741:21\n      tokio::task::spawn::spawn_inner\n             at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-1.21.2/src/task/spawn.rs:147:28\n      tokio::task::spawn::spawn\n             at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-1.21.2/src/task/spawn.rs:135:13\n      backup_stream::endpoint::Endpoint<S,R,E,RT,PDC>::on_resume\n             at /home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/backup-stream/src/endpoint.rs:690:17\n      backup_stream::endpoint::Endpoint<S,R,E,RT,PDC>::handle_watch_task\n             at /home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/backup-stream/src/endpoint.rs:543:17\n      backup_stream::endpoint::Endpoint<S,R,E,RT,PDC>::run_task\n             at /home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/backup-stream/src/endpoint.rs:897:36\n  10: <backup_stream::endpoint::Endpoint<S,R,E,RT,PDC> as tikv_util::worker::pool::Runnable>::run\n             at /home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/backup-stream/src/endpoint.rs:1287:9\n      tikv_util::worker::pool::Worker::start_impl::{{closure}}\n             at /home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/tikv_util/src/worker/pool.rs:468:25\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/mod.rs:91:19\n      yatp::task::future::RawTask<F>::poll\n             at /rust/git/checkouts/yatp-e704b73c3ee279b6/39cb495/src/task/future.rs:59:9\n  11: yatp::task::future::TaskCell::poll\n             at /rust/git/checkouts/yatp-e704b73c3ee279b6/39cb495/src/task/future.rs:103:9\n      <yatp::task::future::Runner as yatp::pool::runner::Runner>::handle\n             at /rust/git/checkouts/yatp-e704b73c3ee279b6/39cb495/src/task/future.rs:387:20\n  12: <tikv_util::yatp_pool::YatpPoolRunner<T> as yatp::pool::runner::Runner>::handle\n             at /home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/tikv_util/src/yatp_pool/mod.rs:122:24\n      yatp::pool::worker::WorkerThread<T,R>::run\n             at /rust/git/checkouts/yatp-e704b73c3ee279b6/39cb495/src/pool/worker.rs:48:13\n      yatp::pool::builder::LazyBuilder<T>::build::{{closure}}\n             at /rust/git/checkouts/yatp-e704b73c3ee279b6/39cb495/src/pool/builder.rs:114:25\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:121:18\n  13: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:551:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:483:40\n      std::panicking::try\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:447:19\n      std::panic::catch_unwind\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:137:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:550:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:513:5\n  14: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2000:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2000:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at /rust/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n  15: start_thread\n  16: __clone\n"] [location=/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/backup-stream/src/endpoint.rs:690] [thread_name=backup-stream-0] 

What did you expect?

What did happened?

[seiya-annie]

/found customer