assert failure in journal vdev during replication I/O load

[yamingk]

So far this is seen on testing with file once (not able to reproduce often).

test_raft_repl_dev: /tmp/source/yk_hs/src/lib/device/journal_vdev.cpp:642: std::tuple<std::shared_ptr<homestore::Chunk>, unsigned int, long int> homestore::JournalVirtualDev::Descriptor::offset_to_chunk(off_t, bool) const: Assertion `0' failed.
[05/03/24 11:16:43-07:00] [C] [test_raft_repl_dev_replica_1] [646987] Input log_offset is invalid: 39200256 log_dev=0;ds=0x1fdf600;end=0x4000000;writesz=5779968;tail=0x2562800;rsvdsz=0;chunks=1;trunc=true;total=33554432;seek=0x0
test_raft_repl_dev: /tmp/source/yk_hs/src/lib/device/journal_vdev.cpp:642: std::tuple<std::shared_ptr<homestore::Chunk>, unsigned int, long int> homestore::JournalVirtualDev::Descriptor::offset_to_chunk(off_t, bool) const: Assertion `0' failed.
*** Aborted at 1714760203 (Unix time, try 'date -d @1714760203') ***

full stack:

   @ 0000000002f4ba78 _ZN5folly10symbolizer12_GLOBAL__N_118innerSignalHandlerEiP9siginfo_tPv
                       /home/bszmyd/.conan/data/folly/nu2.2023.12.18.00/_/_/build/544aa6f60fd65ca05acaa5d692aa40636124c932/src/folly/experimental/symbolizer/SignalHandler.cpp:449
    @ 0000000002f4bb5e _ZN5folly10symbolizer12_GLOBAL__N_113signalHandlerEiP9siginfo_tPv
                       /home/bszmyd/.conan/data/folly/nu2.2023.12.18.00/_/_/build/544aa6f60fd65ca05acaa5d692aa40636124c932/src/folly/experimental/symbolizer/SignalHandler.cpp:470
    @ 000000000004251f (unknown)
    @ 00000000000969fc pthread_kill
    @ 0000000000042475 raise
    @ 00000000000287f2 abort
    @ 000000000002871a (unknown)
    @ 0000000000039e95 __assert_fail
    @ 000000000268b6f7 _ZNK9homestore17JournalVirtualDev10Descriptor15offset_to_chunkElb
                       /tmp/source/yk_hs/src/lib/device/journal_vdev.cpp:642
    @ 000000000268749a _ZN9homestore17JournalVirtualDev10Descriptor10sync_preadEPhml
                       /tmp/source/yk_hs/src/lib/device/journal_vdev.cpp:402
    @ 0000000002709e0a _ZN9homestore6LogDev4readERKNS_10logdev_keyERNS_21serialized_log_recordE
                       /tmp/source/yk_hs/src/lib/logstore/log_dev.cpp:271
    @ 000000000276695d _ZN9homestore12HomeLogStore9read_syncEl
                       /tmp/source/yk_hs/src/lib/logstore/log_store.cpp:149
    @ 000000000261db2b _ZN9homestore16HomeRaftLogStore8entry_atEm
                       /tmp/source/yk_hs/src/lib/replication/log_store/home_raft_log_store.cpp:195
    @ 000000000396714d _ZN6nuraft11raft_server17commit_in_bg_execEm
                       /root/.conan/data/nuraft/2.3.0/_/_/build/a0e0d7402084eed2ac2c68ace47a8ae12d43f8b2/src/src/handle_commit.cxx:225
    @ 0000000003966783 _ZN6nuraft11raft_server12commit_in_bgEv
                       /root/.conan/data/nuraft/2.3.0/_/_/build/a0e0d7402084eed2ac2c68ace47a8ae12d43f8b2/src/src/handle_commit.cxx:144
    @ 00000000039bc6e0 _ZSt13__invoke_implIvRMN6nuraft11raft_serverEFvvERPS1_JEET_St21__invoke_memfun_derefOT0_OT1_DpOT2_
                       /usr/include/c++/11/bits/invoke.h:74
                       -> /root/.conan/data/nuraft/2.3.0/_/_/build/a0e0d7402084eed2ac2c68ace47a8ae12d43f8b2/src/src/raft_server.cxx
    @ 00000000039bbcc9 _ZSt8__invokeIRMN6nuraft11raft_serverEFvvEJRPS1_EENSt15__invoke_resultIT_JDpT0_EE4typeEOS8_DpOS9_
                       /usr/include/c++/11/bits/invoke.h:96
                       -> /root/.conan/data/nuraft/2.3.0/_/_/build/a0e0d7402084eed2ac2c68ace47a8ae12d43f8b2/src/src/raft_server.cxx
    @ 00000000039baf8c _ZNSt5_BindIFMN6nuraft11raft_serverEFvvEPS1_EE6__callIvJEJLm0EEEET_OSt5tupleIJDpT0_EESt12_Index_tupleIJXspT1_EEE
                       /usr/include/c++/11/functional:420
                       -> /root/.conan/data/nuraft/2.3.0/_/_/build/a0e0d7402084eed2ac2c68ace47a8ae12d43f8b2/src/src/raft_server.cxx
    @ 00000000039ba5ce _ZNSt5_BindIFMN6nuraft11raft_serverEFvvEPS1_EEclIJEvEET0_DpOT_
                       /usr/include/c++/11/functional:503
                       -> /root/.conan/data/nuraft/2.3.0/_/_/build/a0e0d7402084eed2ac2c68ace47a8ae12d43f8b2/src/src/raft_server.cxx
    @ 00000000039bd22d _ZSt13__invoke_implIvSt5_BindIFMN6nuraft11raft_serverEFvvEPS2_EEJEET_St14__invoke_otherOT0_DpOT1_
                       /usr/include/c++/11/bits/invoke.h:61
                       -> /root/.conan/data/nuraft/2.3.0/_/_/build/a0e0d7402084eed2ac2c68ace47a8ae12d43f8b2/src/src/raft_server.cxx
    @ 00000000039bd1e8 _ZSt8__invokeISt5_BindIFMN6nuraft11raft_serverEFvvEPS2_EEJEENSt15__invoke_resultIT_JDpT0_EE4typeEOS9_DpOSA_
                       /usr/include/c++/11/bits/invoke.h:96
                       -> /root/.conan/data/nuraft/2.3.0/_/_/build/a0e0d7402084eed2ac2c68ace47a8ae12d43f8b2/src/src/raft_server.cxx
    @ 00000000039bd189 _ZNSt6thread8_InvokerISt5tupleIJSt5_BindIFMN6nuraft11raft_serverEFvvEPS4_EEEEE9_M_invokeIJLm0EEEEvSt12_Index_tupleIJXspT_EEE
                       /usr/include/c++/11/bits/std_thread.h:259
                       -> /root/.conan/data/nuraft/2.3.0/_/_/build/a0e0d7402084eed2ac2c68ace47a8ae12d43f8b2/src/src/raft_server.cxx
    @ 00000000039bd135 _ZNSt6thread8_InvokerISt5tupleIJSt5_BindIFMN6nuraft11raft_serverEFvvEPS4_EEEEEclEv
                       /usr/include/c++/11/bits/std_thread.h:266
                       -> /root/.conan/data/nuraft/2.3.0/_/_/build/a0e0d7402084eed2ac2c68ace47a8ae12d43f8b2/src/src/raft_server.cxx
    @ 00000000039bd115 _ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJSt5_BindIFMN6nuraft11raft_serverEFvvEPS5_EEEEEEE6_M_runEv
                       /usr/include/c++/11/bits/std_thread.h:211
                       -> /root/.conan/data/nuraft/2.3.0/_/_/build/a0e0d7402084eed2ac2c68ace47a8ae12d43f8b2/src/src/raft_server.cxx
    @ 00000000000dc252 (unknown)

[sanebay]

Will reopen if we could repro this with latest changes.

[xiaoxichen]

@sanebay do we see this?