In-memory Engine: assertion failed: self.direction == Direction::Backward

[overvenus]

Bug Report

TiKV panic when preparing 1000 tpcc warehouses.

[2024/05/29 08:40:28.734 +00:00] [FATAL] [lib.rs:477] ["assertion failed: self.direction == Direction::Backward"] [backtrace="   0: tikv_util::set_panic_hook::{{closure}}
             at home/stn/tikv/components/tikv_util/src/lib.rs:476:18
   1: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call
             at home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2029:9
      std::panicking::rust_panic_with_hook
             at home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:783:13
   2: std::panicking::begin_panic_handler::{{closure}}
             at home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:649:13
   3: std::sys_common::backtrace::__rust_end_short_backtrace
             at home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:171:18
   4: rust_begin_unwind
             at home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:645:5
   5: core::panicking::panic_fmt
             at home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panicking.rs:72:14
   6: core::panicking::panic
             at home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panicking.rs:144:5
   7: <region_cache_memory_engine::read::RangeCacheIterator as engine_traits::iterable::Iterator>::prev
             at home/stn/tikv/components/region_cache_memory_engine/src/read.rs:489:9
   8: <hybrid_engine::engine_iterator::HybridEngineIterator<EK,EC> as engine_traits::iterable::Iterator>::prev
             at home/stn/tikv/components/hybrid_engine/src/engine_iterator.rs:70:44
      raftstore::store::region_snapshot::RegionIterator<S>::prev
             at home/stn/tikv/components/raftstore/src/store/region_snapshot.rs:361:9
      tikv_kv::raftstore_impls::<impl tikv_kv::Iterator for raftstore::store::region_snapshot::RegionIterator<S>>::prev
             at home/stn/tikv/components/tikv_kv/src/raftstore_impls.rs:126:9
      tikv_kv::cursor::Cursor<I>::prev
             at home/stn/tikv/components/tikv_kv/src/cursor.rs:386:9
   9: tikv_kv::cursor::Cursor<I>::near_seek
             at home/stn/tikv/components/tikv_kv/src/cursor.rs:143:17
  10: tikv::storage::mvcc::reader::scanner::backward::BackwardKvScanner<S>::reverse_get
             at home/stn/tikv/src/storage/mvcc/reader/scanner/backward.rs:345:13
      tikv::storage::mvcc::reader::scanner::backward::BackwardKvScanner<S>::read_next
             at home/stn/tikv/src/storage/mvcc/reader/scanner/backward.rs:205:30
      <tikv::storage::mvcc::reader::scanner::Scanner<S> as tikv::storage::txn::store::Scanner>::next
             at home/stn/tikv/src/storage/mvcc/reader/scanner/mod.rs:232:46
  11: <tikv::coprocessor::dag::storage_impl::TikvStorage<S> as tidb_query_common::storage::Storage>::scan_next
             at home/stn/tikv/src/coprocessor/dag/storage_impl.rs:73:18
  12: tidb_query_common::storage::scanner::RangesScanner<T,F>::next_opt::{{closure}}
             at home/stn/tikv/components/tidb_query_common/src/storage/scanner.rs:145:21
  13: tidb_query_executors::util::scan_executor::ScanExecutor<S,I,F>::fill_column_vec::{{closure}}
             at home/stn/tikv/components/tidb_query_executors/src/util/scan_executor.rs:111:70
      <tidb_query_executors::util::scan_executor::ScanExecutor<S,I,F> as tidb_query_executors::interface::BatchExecutor>::next_batch::{{closure}}
             at home/stn/tikv/components/tidb_query_executors/src/util/scan_executor.rs:179:80
  14: <core::pin::Pin<P> as core::future::future::Future>::poll
             at home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/future.rs:124:9
      <tidb_query_executors::table_scan_executor::BatchTableScanExecutor<S,F> as tidb_query_executors::interface::BatchExecutor>::next_batch::{{closure}}
             at home/stn/tikv/components/tidb_query_executors/src/table_scan_executor.rs:126:38
  15: <core::pin::Pin<P> as core::future::future::Future>::poll
             at home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/future.rs:124:9
      <tidb_query_common::execute_stats::WithSummaryCollector<C,T> as tidb_query_executors::interface::BatchExecutor>::next_batch::{{closure}}
             at home/stn/tikv/components/tidb_query_executors/src/interface.rs:114:55
  16: <core::pin::Pin<P> as core::future::future::Future>::poll
             at home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/future.rs:124:9
      <alloc::boxed::Box<T> as tidb_query_executors::interface::BatchExecutor>::next_batch::{{closure}}
             at home/stn/tikv/components/tidb_query_executors/src/interface.rs:82:40
  17: <core::pin::Pin<P> as core::future::future::Future>::poll
             at home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/future.rs:124:9
      <tidb_query_executors::selection_executor::BatchSelectionExecutor<Src> as tidb_query_executors::interface::BatchExecutor>::next_batch::{{closure}}
             at home/stn/tikv/components/tidb_query_executors/src/selection_executor.rs:180:61
  18: <core::pin::Pin<P> as core::future::future::Future>::poll
             at home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/future.rs:124:9
      <tidb_query_common::execute_stats::WithSummaryCollector<C,T> as tidb_query_executors::interface::BatchExecutor>::next_batch::{{closure}}
             at home/stn/tikv/components/tidb_query_executors/src/interface.rs:114:55
  19: <core::pin::Pin<P> as core::future::future::Future>::poll
             at home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/future.rs:124:9
      <alloc::boxed::Box<T> as tidb_query_executors::interface::BatchExecutor>::next_batch::{{closure}}
             at home/stn/tikv/components/tidb_query_executors/src/interface.rs:82:40
  20: <core::pin::Pin<P> as core::future::future::Future>::poll
             at home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/future.rs:124:9
      <tidb_query_executors::limit_executor::BatchLimitExecutor<Src> as tidb_query_executors::interface::BatchExecutor>::next_batch::{{closure}}
             at home/stn/tikv/components/tidb_query_executors/src/limit_executor.rs:43:62
  21: <core::pin::Pin<P> as core::future::future::Future>::poll
             at home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/future.rs:124:9
      <tidb_query_common::execute_stats::WithSummaryCollector<C,T> as tidb_query_executors::interface::BatchExecutor>::next_batch::{{closure}}
             at home/stn/tikv/components/tidb_query_executors/src/interface.rs:114:55
  22: <core::pin::Pin<P> as core::future::future::Future>::poll
             at home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/future.rs:124:9
      <alloc::boxed::Box<T> as tidb_query_executors::interface::BatchExecutor>::next_batch::{{closure}}
             at home/stn/tikv/components/tidb_query_executors/src/interface.rs:82:40
  23: <core::pin::Pin<P> as core::future::future::Future>::poll
             at home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/future.rs:124:9
      tidb_query_executors::runner::BatchExecutorsRunner<SS>::internal_handle_request::{{closure}}
             at home/stn/tikv/components/tidb_query_executors/src/runner.rs:655:72
  24: <tikv_util::quota_limiter::CpuObserveFuture<F> as core::future::future::Future>::poll
             at home/stn/tikv/components/tikv_util/src/quota_limiter.rs:177:19
      tidb_query_executors::runner::BatchExecutorsRunner<SS>::handle_request::{{closure}}
             at home/stn/tikv/components/tidb_query_executors/src/runner.rs:519:22
      <tikv::coprocessor::dag::BatchDagHandler as tikv::coprocessor::RequestHandler>::handle_request::{{closure}}
             at home/stn/tikv/src/coprocessor/dag/mod.rs:124:51
  25: <core::pin::Pin<P> as core::future::future::Future>::poll
             at home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/future.rs:124:9
      <tikv::coprocessor::interceptors::deadline::DeadlineChecker<F> as core::future::future::Future>::poll
             at home/stn/tikv/src/coprocessor/interceptors/deadline.rs:37:9
      <tikv::coprocessor::interceptors::tracker::Tracker<F,E> as core::future::future::Future>::poll
             at home/stn/tikv/src/coprocessor/interceptors/tracker.rs:54:19
      <tikv::coprocessor::interceptors::concurrency_limiter::ConcurrencyLimiter<PF,F> as core::future::future::Future>::poll
             at home/stn/tikv/src/coprocessor/inte

What version of TiKV are you using?

Nightly, built on #16868

Steps to reproduce

TiKV Config

[range-cache-engine]
  enabled = true
  hard-limit-threshold = "8GB"
  soft-limit-threshold = "5GB"

TPCC commands

/root/.tiup/components/bench/v1.12.0/go-tpc \
    tpcc prepare \
    -H $HOST -P $PORT \
    -D tpcc100 --warehouses 100 -T 50

What did you expect?

No such error.

[SpadeA-Tang]

This is due to that it does not support reverse the direction now: first seek and next, then prev. I will complement this part later.