segmentation fault in SortedPageIter

[zojw]

(gdb) bt
#0  __memcmp_avx2_movbe () at ../sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S:248
#1  0x00005632e6470eb6 in <u8 as core::slice::cmp::SliceOrd>::compare (left=..., right=...) at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/core/src/slice/cmp.rs:198
#2  0x00005632e647107d in core::slice::cmp::<impl core::cmp::Ord for [T]>::cmp (self=..., other=...) at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/core/src/slice/cmp.rs:41
#3  0x00005632e63477c2 in <photondb::page::data::Key as core::cmp::Ord>::cmp (self=0x7fe0380182b0, other=0x7fe038018248) at photondb/src/page/data.rs:18
#4  0x00005632e6379842 in <photondb::page::iter::OrderedIter<I> as core::cmp::Ord>::cmp (self=0x7fe038018270, other=0x7fe038018208) at photondb/src/page/iter.rs:158
#5  0x00005632e63798f4 in <photondb::page::iter::OrderedIter<I> as core::cmp::PartialOrd>::partial_cmp (self=0x7fe038018270, other=0x7fe038018208) at photondb/src/page/iter.rs:176
#6  0x00005632e6378ec4 in core::cmp::PartialOrd::le (self=0x7fe038018270, other=0x7fe038018208) at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/core/src/cmp.rs:1139
#7  0x00005632e6387551 in <core::cmp::Reverse<T> as core::cmp::PartialOrd>::le (self=0x7fe038018208, other=0x7fe038018270) at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/core/src/cmp.rs:616
#8  0x00005632e63896dc in core::cmp::impls::<impl core::cmp::PartialOrd<&B> for &A>::le () at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/alloc/src/collections/binary_heap.rs:598
#9  alloc::collections::binary_heap::BinaryHeap<T>::sift_down_range (self=0x7fe051841bc0, pos=0, end=63) at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/alloc/src/collections/binary_heap.rs:598
#10 0x00005632e638a3ad in alloc::collections::binary_heap::BinaryHeap<T>::sift_down (self=0x7fe051841bc0, pos=0) at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/alloc/src/collections/binary_heap.rs:628
#11 0x00005632e635b00b in <alloc::collections::binary_heap::PeekMut<T> as core::ops::drop::Drop>::drop (self=0x7fe0518407d8)
    at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/alloc/src/collections/binary_heap.rs:297
#12 0x00005632e63543db in core::ptr::drop_in_place<alloc::collections::binary_heap::PeekMut<core::cmp::Reverse<photondb::page::iter::OrderedIter<photondb::page::sorted_page::SortedPageIter<photondb::page::data::Key,photondb::page::data::Value>>>>> () at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/core/src/ptr/mod.rs:487
#13 0x00005632e637a49c in <photondb::page::iter::MergingIter<I> as core::iter::traits::iterator::Iterator>::next (self=0x7fe051841bc0) at photondb/src/page/iter.rs:260
#14 0x00005632e63a14f9 in <photondb::tree::page::MergingPageIter<K,V> as core::iter::traits::iterator::Iterator>::next (self=0x7fe051841bc0) at photondb/src/tree/page.rs:93
#15 0x00005632e63a118a in <&mut I as core::iter::traits::iterator::Iterator>::next (self=0x7fe051840978) at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/core/src/iter/traits/iterator.rs:3878
#16 0x00005632e6380bcb in <photondb::tree::page::MergingLeafPageIter as core::iter::traits::iterator::Iterator>::next (self=0x7fe051841bc0) at photondb/src/tree/page.rs:156
#17 0x00005632e63a11ba in <&mut I as core::iter::traits::iterator::Iterator>::next (self=0x7fe051840b80) at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/core/src/iter/traits/iterator.rs:3878
#18 0x00005632e633d2fd in photondb::page::sorted_page::SortedPageBuilder<I>::build (self=..., page=0x7fe051841b30) at photondb/src/page/sorted_page.rs:72
#19 0x00005632e5ffa9d6 in photondb::tree::TreeTxn<E>::consolidate_page_impl::{{closure}} () at photondb/src/tree/mod.rs:608
#20 0x00005632e60308b3 in <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll (self=..., cx=0x7fe05184fd68)
    at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/core/src/future/mod.rs:91
#21 0x00005632e5ff779b in photondb::tree::TreeTxn<E>::consolidate_page::{{closure}} () at photondb/src/tree/mod.rs:582
#22 0x00005632e6030433 in <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll (self=..., cx=0x7fe05184fd68)
    at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/core/src/future/mod.rs:91
#23 0x00005632e5ffcd58 in photondb::tree::TreeTxn<E>::consolidate_and_restructure_page::{{closure}} () at photondb/src/tree/mod.rs:685
#24 0x00005632e602a3cf in <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll (self=..., cx=0x7fe05184fd68)
    at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/core/src/future/mod.rs:91
#25 0x00005632e6000455 in photondb::tree::TreeTxn<E>::try_write::{{closure}} () at photondb/src/tree/mod.rs:182
#26 0x00005632e602bd5f in <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll (self=..., cx=0x7fe05184fd68)
    at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/core/src/future/mod.rs:91
#27 0x00005632e5ffe22a in photondb::tree::TreeTxn<E>::write::{{closure}} () at photondb/src/tree/mod.rs:131
#28 0x00005632e60300df in <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll (self=..., cx=0x7fe05184fd68)
    at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/core/src/future/mod.rs:91
#29 0x00005632e5feced1 in photondb::raw::table::Table<E>::put::{{closure}} () at photondb/src/raw/table.rs:74
#30 0x00005632e602f33c in <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll (self=..., cx=0x7fe05184fd68)
    at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/core/src/future/mod.rs:91
#31 0x00005632e614e5e3 in photondb_tools::stress::write_task::{{closure}} () at photondb-tools/src/stress/mod.rs:289
#32 0x00005632e602f7fc in <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll (self=..., cx=0x7fe05184fd68)
--Type <RET> for more, q to quit, c to continue without paging--
    at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/core/src/future/mod.rs:91
#33 0x00005632e60b7e7d in photonio_uring::task::raw::poll::{{closure}} () at /home/robi/.cargo/git/checkouts/photonio-4d2b22930312944b/8aab9a4/photonio-uring/src/task/raw.rs:209
#34 0x00005632e5fee233 in <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once (self=..., _args=())
    at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/core/src/panic/unwind_safe.rs:271
#35 0x00005632e601b5cd in std::panicking::try::do_call (data=0x7fe05184fc70 "0 ") at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/std/src/panicking.rs:464
#36 0x00005632e601c2ab in __rust_try ()
#37 0x00005632e601ac15 in std::panicking::try (f=...) at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/std/src/panicking.rs:428
#38 0x00005632e60b97fa in std::panic::catch_unwind (f=...) at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/std/src/panic.rs:137
#39 0x00005632e60b7b19 in photonio_uring::task::raw::poll (head=0x7fe05184fee8) at /home/robi/.cargo/git/checkouts/photonio-4d2b22930312944b/8aab9a4/photonio-uring/src/task/raw.rs:209
#40 0x00005632e64862b0 in photonio_uring::task::raw::Head::poll (self=0x7fe04c001ff0, this=0x7fe05184fee8) at src/task/raw.rs:30
#41 0x00005632e647870a in photonio_uring::task::Task::poll (self=0x7fe05184fee8) at src/task/mod.rs:61
#42 0x00005632e647ca1d in photonio_uring::runtime::worker::Local::poll (self=0x7fe0518502b8) at src/runtime/worker.rs:82
#43 0x00005632e647c110 in photonio_uring::runtime::worker::Local::run (self=0x7fe0518502b8) at src/runtime/worker.rs:53
#44 0x00005632e647d7a3 in photonio_uring::runtime::worker::enter::{{closure}} () at src/runtime/worker.rs:154
#45 0x00005632e6479b04 in scoped_tls::ScopedKey<T>::set (self=0x5632e6841e58 <photonio_uring::runtime::worker::CURRENT>, t=0x7fe0518502b8, f=...)
    at /home/robi/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/scoped-tls-1.0.1/src/lib.rs:137
#46 0x00005632e647d735 in photonio_uring::runtime::worker::enter (local=<error reading variable: Cannot access memory at address 0x7fdfc4532c74>) at src/runtime/worker.rs:154
#47 0x00005632e647d6a3 in photonio_uring::runtime::worker::Worker::launch::{{closure}} () at src/runtime/worker.rs:129
#48 0x00005632e6485373 in std::sys_common::backtrace::__rust_begin_short_backtrace (f=...) at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/std/src/sys_common/backtrace.rs:122
#49 0x00005632e6479684 in std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} () at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/std/src/thread/mod.rs:514
#50 0x00005632e6485334 in <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once (self=..., _args=())
    at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/core/src/panic/unwind_safe.rs:271
#51 0x00005632e647e379 in std::panicking::try::do_call (data=0x7fe051851140 "\005") at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/std/src/panicking.rs:464
#52 0x00005632e647ec5b in __rust_try ()
#53 0x00005632e647e1e3 in std::panicking::try (f=...) at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/std/src/panicking.rs:428
#54 0x00005632e64814b0 in std::panic::catch_unwind (f=...) at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/std/src/panic.rs:137
#55 0x00005632e6479446 in std::thread::Builder::spawn_unchecked_::{{closure}} () at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/std/src/thread/mod.rs:513
#56 0x00005632e647523f in core::ops::function::FnOnce::call_once{{vtable-shim}} () at /rustc/8ce3204af9463db3192ea1eb31c45c2f6d4b5ae6/library/core/src/ops/function.rs:251
#57 0x00005632e662b773 in <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once () at library/alloc/src/boxed.rs:1938
#58 <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once () at library/alloc/src/boxed.rs:1938
#59 std::sys::unix::thread::Thread::new::thread_start () at library/std/src/sys/unix/thread.rs:108
#60 0x00007fe0527a4609 in start_thread (arg=<optimized out>) at pthread_create.c:477
#61 0x00007fe052574133 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

git hash: 8140ae7
workload: stress

[huachaohuang]

Maybe related to #247

[huachaohuang]

I can reproduce this problem. Looks like it is accessing a released write buffer. This should not happen since the tree transaction holds the guard during the whole operation. /cc @w41ter