heap-use-after-free failure on log_cas_test

[dutor]

This bug appears not every time.

It seems some resources are being accessed after the service already stopped.

17: ==4393==ERROR: AddressSanitizer: heap-use-after-free on address 0x60400011de10 at pc 0x0000006dc8b9 bp 0x7f914d6743c0 sp 0x7f914d6743b0
17: Could not create logging file: No such file or directory
17: COULD NOT CREATE A LOGGINGFILE 20190114-163350.4393!I0114 16:33:50.038925  4393 RaftexService.cpp:89] Stopping the raftex service on port 35935
17: I0114 16:33:50.039160  4393 RaftexService.cpp:96] All partitions have stopped
17: READ of size 8 at 0x60400011de10 thread T1101
17: I0114 16:33:50.039450  5677 RaftexService.cpp:61] The Raftex Service stopped
17:     #0 0x6dc8b8 in std::__detail::_Node_iterator_base<std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> >, true>::_M_incr() /usr/include/c++/8/bits/hashtable_policy.h:300
17:     #1 0x6dc8b8 in std::__detail::_Node_iterator<std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> >, false, true>::operator++() /usr/include/c++/8/bits/hashtable_policy.h:355
17:     #2 0x6dc8b8 in foreach<folly::gen::detail::Map<Predicate>::Generator<Value, Source, Result>::foreach(Body&&) const [with Body = folly::gen::detail::CollectTemplate<Collection, Allocator>::compose(const folly::gen::GenImpl<Value, Source>&) const [with Value = folly::Future<nebula::raftex::cpp2::AppendLogResponse>&&; Source = folly::gen::detail::Map<nebula::raftex::RaftPart::sendHeartbeat()::<lambda(PeerHostEntry&)> >::Generator<std::pair<const std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >&, folly::gen::detail::ReferencedSource<std::unordered_map<std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >, std::pair<const std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >&>, folly::Future<nebula::raftex::cpp2::AppendLogResponse>&&>; StorageType = folly::Future<nebula::raftex::cpp2::AppendLogResponse>; Collection = std::vector<folly::Future<nebula::raftex::cpp2::AppendLogResponse>, std::allocator<folly::Future<nebula::raftex::cpp2::AppendLogResponse> > >; Container = std::vector; Allocator = std::allocator]::<lambda(folly::Future<nebula::raftex::cpp2::AppendLogResponse>&&)>; Value = std::pair<const std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >&; Source = folly::gen::detail::ReferencedSource<std::unordered_map<std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >, std::pair<const std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >&>; Result = folly::Future<nebula::raftex::cpp2::AppendLogResponse>&&; Predicate = nebula::raftex::RaftPart::sendHeartbeat()::<lambda(PeerHostEntry&)>]::<lambda(std::pair<const std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >&)> > /home/dutor/Wdir/nebula/third-party/folly/_install/include/folly/gen/Base-inl.h:130
17:     #3 0x6dc8b8 in foreach<folly::gen::detail::CollectTemplate<Collection, Allocator>::compose(const folly::gen::GenImpl<Value, Source>&) const [with Value = folly::Future<nebula::raftex::cpp2::AppendLogResponse>&&; Source = folly::gen::detail::Map<nebula::raftex::RaftPart::sendHeartbeat()::<lambda(PeerHostEntry&)> >::Generator<std::pair<const std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >&, folly::gen::detail::ReferencedSource<std::unordered_map<std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >, std::pair<const std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >&>, folly::Future<nebula::raftex::cpp2::AppendLogResponse>&&>; StorageType = folly::Future<nebula::raftex::cpp2::AppendLogResponse>; Collection = std::vector<folly::Future<nebula::raftex::cpp2::AppendLogResponse>, std::allocator<folly::Future<nebula::raftex::cpp2::AppendLogResponse> > >; Container = std::vector; Allocator = std::allocator]::<lambda(folly::Future<nebula::raftex::cpp2::AppendLogResponse>&&)> > /home/dutor/Wdir/nebula/third-party/folly/_install/include/folly/gen/Base-inl.h:500
17:     #4 0x6dc8b8 in operator|<folly::Future<nebula::raftex::cpp2::AppendLogResponse>&&, folly::gen::detail::Map<nebula::raftex::RaftPart::sendHeartbeat()::<lambda(PeerHostEntry&)> >::Generator<std::pair<const std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >&, folly::gen::detail::ReferencedSource<std::unordered_map<std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >, std::pair<const std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >&>, folly::Future<nebula::raftex::cpp2::AppendLogResponse>&&>, folly::gen::detail::CollectTemplate<Collection, Allocator>::compose(const folly::gen::GenImpl<Value, Source>&) const [with Value = folly::Future<nebula::raftex::cpp2::AppendLogResponse>&&; Source = folly::gen::detail::Map<nebula::raftex::RaftPart::sendHeartbeat()::<lambda(PeerHostEntry&)> >::Generator<std::pair<const std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >&, folly::gen::detail::ReferencedSource<std::unordered_map<std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >, std::pair<const std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >&>, folly::Future<nebula::raftex::cpp2::AppendLogResponse>&&>; StorageType = folly::Future<nebula::raftex::cpp2::AppendLogResponse>; Collection = std::vector<folly::Future<nebula::raftex::cpp2::AppendLogResponse>, std::allocator<folly::Future<nebula::raftex::cpp2::AppendLogResponse> > >; Container = std::vector; Allocator = std::allocator]::<lambda(folly::Future<nebula::raftex::cpp2::AppendLogResponse>&&)> > /home/dutor/Wdir/nebula/third-party/folly/_install/include/folly/gen/Core-inl.h:264
17:     #5 0x6dc8b8 in compose<folly::Future<nebula::raftex::cpp2::AppendLogResponse>&&, folly::gen::detail::Map<nebula::raftex::RaftPart::sendHeartbeat()::<lambda(PeerHostEntry&)> >::Generator<std::pair<const std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >&, folly::gen::detail::ReferencedSource<std::unordered_map<std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >, std::pair<const std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >&>, folly::Future<nebula::raftex::cpp2::AppendLogResponse>&&> > /home/dutor/Wdir/nebula/third-party/folly/_install/include/folly/gen/Base-inl.h:2204
17:     #6 0x6dc8b8 in operator|<folly::Future<nebula::raftex::cpp2::AppendLogResponse>&&, folly::gen::detail::Map<nebula::raftex::RaftPart::sendHeartbeat()::<lambda(PeerHostEntry&)> >::Generator<std::pair<const std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >&, folly::gen::detail::ReferencedSource<std::unordered_map<std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >, std::pair<const std::pair<int, int>, std::shared_ptr<nebula::raftex::Host> >&>, folly::Future<nebula::raftex::cpp2::AppendLogResponse>&&>, folly::gen::detail::CollectTemplate<std::vector> > /home/dutor/Wdir/nebula/third-party/folly/_install/include/folly/gen/Core-inl.h:292
17:     #7 0x6dc8b8 in nebula::raftex::RaftPart::sendHeartbeat() /home/dutor/Wdir/nebula/src/raftex/RaftPart.cpp:1220
17:     #8 0x6e76f8 in nebula::raftex::RaftPart::statusPolling() /home/dutor/Wdir/nebula/src/raftex/RaftPart.cpp:831
17:     #9 0x6e7c4e in operator() /home/dutor/Wdir/nebula/src/raftex/RaftPart.cpp:840
17:     #10 0x6e7c4e in __invoke_impl<void, nebula::raftex::RaftPart::statusPolling()::<lambda()>&> /usr/include/c++/8/bits/invoke.h:60
17:     #11 0x6e7c4e in __invoke<nebula::raftex::RaftPart::statusPolling()::<lambda()>&> /usr/include/c++/8/bits/invoke.h:95
17:     #12 0x6e7c4e in __call<void> /usr/include/c++/8/functional:400
17:     #13 0x6e7c4e in operator()<> /usr/include/c++/8/functional:484
17:     #14 0x6e7c4e in _M_invoke /usr/include/c++/8/bits/std_function.h:297
17:     #15 0x6c6d31 in std::function<void ()>::operator()() const /usr/include/c++/8/bits/std_function.h:687
17:     #16 0x6c6d31 in operator() /home/dutor/Wdir/nebula/src/common/thread/GenericWorker.h:236
17:     #17 0x6c6d31 in __invoke_impl<void, nebula::thread::GenericWorker::addDelayTask(size_t, F&&, Args&& ...) [with F = nebula::raftex::RaftPart::statusPolling()::<lambda()>; Args = {}]::<lambda()>&> /usr/include/c++/8/bits/invoke.h:60
17:     #18 0x6c6d31 in __invoke<nebula::thread::GenericWorker::addDelayTask(size_t, F&&, Args&& ...) [with F = nebula::raftex::RaftPart::statusPolling()::<lambda()>; Args = {}]::<lambda()>&> /usr/include/c++/8/bits/invoke.h:95
17:     #19 0x6c6d31 in __call<void> /usr/include/c++/8/functional:400
17:     #20 0x6c6d31 in operator()<> /usr/include/c++/8/functional:484
17:     #21 0x6c6d31 in _M_invoke /usr/include/c++/8/bits/std_function.h:297
17:     #22 0x8bbd97 in std::function<void ()>::operator()() const /usr/include/c++/8/bits/std_function.h:687
17:     #23 0x8bbd97 in operator() /home/dutor/Wdir/nebula/src/common/thread/GenericWorker.cpp:125
17:     #24 0x8bbd97 in _FUN /home/dutor/Wdir/nebula/src/common/thread/GenericWorker.cpp:129
17:     #25 0xb715a0 in event_process_active_single_queue (/home/dutor/Wdir/nebula/src/raftex/test/_build/log_cas_test+0xb715a0)
17:     #26 0xb71cf6 in event_base_loop (/home/dutor/Wdir/nebula/src/raftex/test/_build/log_cas_test+0xb71cf6)
17:     #27 0x8be25f in void std::__invoke_impl<void, void (nebula::thread::GenericWorker::*&)(), nebula::thread::GenericWorker*&>(std::__invoke_memfun_deref, void (nebula::thread::GenericWorker::*&)(), nebula::thread::GenericWorker*&) /usr/include/c++/8/bits/invoke.h:73
17:     #28 0x8be25f in std::__invoke_result<void (nebula::thread::GenericWorker::*&)(), nebula::thread::GenericWorker*&>::type std::__invoke<void (nebula::thread::GenericWorker::*&)(), nebula::thread::GenericWorker*&>(void (nebula::thread::GenericWorker::*&)(), nebula::thread::GenericWorker*&) /usr/include/c++/8/bits/invoke.h:95
17:     #29 0x8be25f in void std::_Bind<void (nebula::thread::GenericWorker::*(nebula::thread::GenericWorker*))()>::__call<void, , 0ul>(std::tuple<>&&, std::_Index_tuple<0ul>) /usr/include/c++/8/functional:400
17:     #30 0x8be25f in void std::_Bind<void (nebula::thread::GenericWorker::*(nebula::thread::GenericWorker*))()>::operator()<, void>() /usr/include/c++/8/functional:484
17:     #31 0x8be25f in std::_Function_handler<void (), std::_Bind<void (nebula::thread::GenericWorker::*(nebula::thread::GenericWorker*))()> >::_M_invoke(std::_Any_data const&) /usr/include/c++/8/bits/std_function.h:297
17:     #32 0x8bddf3 in void std::__invoke_impl<void, void (*)(nebula::thread::NamedThread*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::function<void ()> const&), nebula::thread::NamedThread*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::_Bind<void (nebula::thread::GenericWorker::*(nebula::thread::GenericWorker*))()> >(std::__invoke_other, void (*&&)(nebula::thread::NamedThread*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::function<void ()> const&), nebula::thread::NamedThread*&&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&&, std::_Bind<void (nebula::thread::GenericWorker::*(nebula::thread::GenericWorker*))()>&&) /usr/include/c++/8/bits/invoke.h:60
17:     #33 0x8bddf3 in std::__invoke_result<void (*)(nebula::thread::NamedThread*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::function<void ()> const&), nebula::thread::NamedThread*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::_Bind<void (nebula::thread::GenericWorker::*(nebula::thread::GenericWorker*))()> >::type std::__invoke<void (*)(nebula::thread::NamedThread*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::function<void ()> const&), nebula::thread::NamedThread*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::_Bind<void (nebula::thread::GenericWorker::*(nebula::thread::GenericWorker*))()> >(void (*&&)(nebula::thread::NamedThread*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::function<void ()> const&), nebula::thread::NamedThread*&&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&&, std::_Bind<void (nebula::thread::GenericWorker::*(nebula::thread::GenericWorker*))()>&&) /usr/include/c++/8/bits/invoke.h:95
17:     #34 0x8bddf3 in decltype (__invoke((_S_declval<0ul>)(), (_S_declval<1ul>)(), (_S_declval<2ul>)(), (_S_declval<3ul>)())) std::thread::_Invoker<std::tuple<void (*)(nebula::thread::NamedThread*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::function<void ()> const&), nebula::thread::NamedThread*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::_Bind<void (nebula::thread::GenericWorker::*(nebula::thread::GenericWorker*))()> > >::_M_invoke<0ul, 1ul, 2ul, 3ul>(std::_Index_tuple<0ul, 1ul, 2ul, 3ul>) /usr/include/c++/8/thread:244
17:     #35 0x8bddf3 in std::thread::_Invoker<std::tuple<void (*)(nebula::thread::NamedThread*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::function<void ()> const&), nebula::thread::NamedThread*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::_Bind<void (nebula::thread::GenericWorker::*(nebula::thread::GenericWorker*))()> > >::operator()() /usr/include/c++/8/thread:253
17:     #36 0x8bddf3 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(nebula::thread::NamedThread*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::function<void ()> const&), nebula::thread::NamedThread*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::_Bind<void (nebula::thread::GenericWorker::*(nebula::thread::GenericWorker*))()> > > >::_M_run() /usr/include/c++/8/thread:196
17:     #37 0xbff282 in execute_native_thread_routine (/home/dutor/Wdir/nebula/src/raftex/test/_build/log_cas_test+0xbff282)
17:     #38 0x7f918448b58d in start_thread /usr/src/debug/glibc-2.28/nptl/pthread_create.c:486
17:     #39 0x7f91843ba512 in clone (/lib64/libc.so.6+0xfd512)
17:
17: 0x60400011de10 is located 0 bytes inside of 40-byte region [0x60400011de10,0x60400011de38)
17: freed by thread T0 here:
17:     #0 0x7f9184c4d348 in operator delete(void*) (/lib64/libasan.so.5+0xf2348)
17:     #1 0x6fdb13 in std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> >, true> > >::_M_deallocate_nodes(std::__detail::_Hash_node<std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> >, true>*) /usr/include/c++/8/bits/hashtable_policy.h:2113
17:     #2 0x6fdb13 in std::_Hashtable<std::pair<int, int>, std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> >, std::allocator<std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> > >, std::__detail::_Select1st, std::equal_to<std::pair<int, int> >, std::hash<std::pair<int, int> >, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >::clear() /usr/include/c++/8/bits/hashtable.h:2047
17:     #3 0x6c0137 in std::unordered_map<std::pair<int, int>, std::shared_ptr<nebula::raftex::Host>, std::hash<std::pair<int, int> >, std::equal_to<std::pair<int, int> >, std::allocator<std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> > > >::clear() /usr/include/c++/8/bits/unordered_map.h:843
17:     #4 0x6c0137 in nebula::raftex::RaftPart::stop() /home/dutor/Wdir/nebula/src/raftex/RaftPart.cpp:279
17:     #5 0x72cadd in nebula::raftex::RaftexService::stop() /home/dutor/Wdir/nebula/src/raftex/RaftexService.cpp:93
17:     #6 0x6aaf1b in nebula::raftex::finishRaft(std::vector<std::shared_ptr<nebula::raftex::RaftexService>, std::allocator<std::shared_ptr<nebula::raftex::RaftexService> > >&, std::vector<std::shared_ptr<nebula::raftex::test::TestShard>, std::allocator<std::shared_ptr<nebula::raftex::test::TestShard> > >&, std::shared_ptr<nebula::thread::GenericThreadPool>&, std::shared_ptr<nebula::raftex::test::TestShard>&) /home/dutor/Wdir/nebula/src/raftex/test/RaftexTestBase.cpp:183
17:     #7 0x696154 in nebula::raftex::RaftexTestFixture::TearDown() /home/dutor/Wdir/nebula/src/raftex/test/RaftexTestBase.h:95
17:     #8 0xb49d99 in void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) (/home/dutor/Wdir/nebula/src/raftex/test/_build/log_cas_test+0xb49d99)
17:
17: previously allocated by thread T0 here:
17:     #0 0x7f9184c4c470 in operator new(unsigned long) (/lib64/libasan.so.5+0xf1470)
17:     #1 0x7036e5 in __gnu_cxx::new_allocator<std::__detail::_Hash_node<std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> >, true> >::allocate(unsigned long, void const*) /usr/include/c++/8/ext/new_allocator.h:111
17:     #2 0x7036e5 in std::allocator_traits<std::allocator<std::__detail::_Hash_node<std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> >, true> > >::allocate(std::allocator<std::__detail::_Hash_node<std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> >, true> >&, unsigned long) /usr/include/c++/8/bits/alloc_traits.h:436
17:     #3 0x7036e5 in std::__detail::_Hash_node<std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> >, true>* std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> >, true> > >::_M_allocate_node<std::pair<int, int>&, std::shared_ptr<nebula::raftex::Host> >(std::pair<int, int>&, std::shared_ptr<nebula::raftex::Host>&&) /usr/include/c++/8/bits/hashtable_policy.h:2077
17:     #4 0x7036e5 in std::pair<std::__detail::_Node_iterator<std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> >, false, true>, bool> std::_Hashtable<std::pair<int, int>, std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> >, std::allocator<std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> > >, std::__detail::_Select1st, std::equal_to<std::pair<int, int> >, std::hash<std::pair<int, int> >, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >::_M_emplace<std::pair<int, int>&, std::shared_ptr<nebula::raftex::Host> >(std::integral_constant<bool, true>, std::pair<int, int>&, std::shared_ptr<nebula::raftex::Host>&&) /usr/include/c++/8/bits/hashtable.h:1657
17:     #5 0x6c8d16 in std::pair<std::__detail::_Node_iterator<std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> >, false, true>, bool> std::_Hashtable<std::pair<int, int>, std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> >, std::allocator<std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> > >, std::__detail::_Select1st, std::equal_to<std::pair<int, int> >, std::hash<std::pair<int, int> >, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >::emplace<std::pair<int, int>&, std::shared_ptr<nebula::raftex::Host> >(std::pair<int, int>&, std::shared_ptr<nebula::raftex::Host>&&) /usr/include/c++/8/bits/hashtable.h:748
17:     #6 0x6c8d16 in std::pair<std::__detail::_Node_iterator<std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> >, false, true>, bool> std::unordered_map<std::pair<int, int>, std::shared_ptr<nebula::raftex::Host>, std::hash<std::pair<int, int> >, std::equal_to<std::pair<int, int> >, std::allocator<std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> > > >::emplace<std::pair<int, int>&, std::shared_ptr<nebula::raftex::Host> >(std::pair<int, int>&, std::shared_ptr<nebula::raftex::Host>&&) /usr/include/c++/8/bits/unordered_map.h:388
17:     #7 0x6c8d16 in nebula::raftex::RaftPart::start(std::vector<std::pair<int, int>, std::allocator<std::pair<int, int> > >&&) /home/dutor/Wdir/nebula/src/raftex/RaftPart.cpp:245
17:     #8 0x6af23e in nebula::raftex::setupRaft(nebula::fs::TempDir&, std::shared_ptr<nebula::thread::GenericThreadPool>&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&, std::vector<std::pair<int, int>, std::allocator<std::pair<int, int> > >&, std::vector<std::shared_ptr<nebula::raftex::RaftexService>, std::allocator<std::shared_ptr<nebula::raftex::RaftexService> > >&, std::vector<std::shared_ptr<nebula::raftex::test::TestShard>, std::allocator<std::shared_ptr<nebula::raftex::test::TestShard> > >&, std::shared_ptr<nebula::raftex::test::TestShard>&) /home/dutor/Wdir/nebula/src/raftex/test/RaftexTestBase.cpp:166
17:     #9 0x696422 in nebula::raftex::RaftexTestFixture::SetUp() /home/dutor/Wdir/nebula/src/raftex/test/RaftexTestBase.h:88
17:     #10 0xb49d99 in void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) (/home/dutor/Wdir/nebula/src/raftex/test/_build/log_cas_test+0xb49d99)
17:
17: Thread T1101 created by T0 here:
17:     #0 0x7f9184ba7043 in __interceptor_pthread_create (/lib64/libasan.so.5+0x4c043)
17:     #1 0xbff358 in std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) (/home/dutor/Wdir/nebula/src/raftex/test/_build/log_cas_test+0xbff358)
17:     #2 0x8bf6dc in nebula::thread::GenericThreadPool::start(unsigned long, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) /home/dutor/Wdir/nebula/src/common/thread/GenericThreadPool.cpp:29
17:     #3 0x6ad21c in nebula::raftex::setupRaft(nebula::fs::TempDir&, std::shared_ptr<nebula::thread::GenericThreadPool>&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&, std::vector<std::pair<int, int>, std::allocator<std::pair<int, int> > >&, std::vector<std::shared_ptr<nebula::raftex::RaftexService>, std::allocator<std::shared_ptr<nebula::raftex::RaftexService> > >&, std::vector<std::shared_ptr<nebula::raftex::test::TestShard>, std::allocator<std::shared_ptr<nebula::raftex::test::TestShard> > >&, std::shared_ptr<nebula::raftex::test::TestShard>&) /home/dutor/Wdir/nebula/src/raftex/test/RaftexTestBase.cpp:123
17:     #4 0x696422 in nebula::raftex::RaftexTestFixture::SetUp() /home/dutor/Wdir/nebula/src/raftex/test/RaftexTestBase.h:88
17:     #5 0xb49d99 in void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) (/home/dutor/Wdir/nebula/src/raftex/test/_build/log_cas_test+0xb49d99)
17:
17: SUMMARY: AddressSanitizer: heap-use-after-free /usr/include/c++/8/bits/hashtable_policy.h:300 in std::__detail::_Node_iterator_base<std::pair<std::pair<int, int> const, std::shared_ptr<nebula::raftex::Host> >, true>::_M_incr()
17: Shadow bytes around the buggy address:
17:   0x0c088001bb70: fa fa fd fd fd fd fd fa fa fa fd fd fd fd fd fa
17:   0x0c088001bb80: fa fa 00 00 00 00 00 fa fa fa fd fd fd fd fd fa
17:   0x0c088001bb90: fa fa fd fd fd fd fd fa fa fa 00 00 00 00 00 fa
17:   0x0c088001bba0: fa fa fd fd fd fd fd fa fa fa 00 00 00 00 02 fa
17:   0x0c088001bbb0: fa fa fd fd fd fd fd fa fa fa fd fd fd fd fd fa
17: =>0x0c088001bbc0: fa fa[fd]fd fd fd fd fa fa fa fd fd fd fd fd fd
17:   0x0c088001bbd0: fa fa fd fd fd fd fd fa fa fa 00 00 00 00 02 fa
17:   0x0c088001bbe0: fa fa fd fd fd fd fd fa fa fa fd fd fd fd fd fa
17:   0x0c088001bbf0: fa fa fd fd fd fd fd fa fa fa fd fd fd fd fd fd
17:   0x0c088001bc00: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
17:   0x0c088001bc10: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
17: Shadow byte legend (one shadow byte represents 8 application bytes):
17:   Addressable:           00
17:   Partially addressable: 01 02 03 04 05 06 07
17:   Heap left redzone:       fa
17:   Freed heap region:       fd
17:   Stack left redzone:      f1
17:   Stack mid redzone:       f2
17:   Stack right redzone:     f3
17:   Stack after return:      f5
17:   Stack use after scope:   f8
17:   Global redzone:          f9
17:   Global init order:       f6
17:   Poisoned by user:        f7
17:   Container overflow:      fc
17:   Array cookie:            ac
17:   Intra object redzone:    bb
17:   ASan internal:           fe
17:   Left alloca redzone:     ca
17:   Right alloca redzone:    cb
17: ==4393==ABORTING
1/1 Test #17: log_cas_test .....................***Failed   27.97 sec

[sherman-the-tank]

This is a dup of #129 (Same root cause)