Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fix tsan issues found during ep add/del and svc add/del in loop (#305)
Pattern1: WARNING: ThreadSanitizer: data race (pid=23522) Write of size 8 at 0x7fff79e44c50 by thread T13 (mutexes: write M1028): #0 std::_Hashtable<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexagent::Endpoint #1 std::_Hashtable<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexagent::Endpoint #2 std::__detail::_Map_base<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexagent: #3 std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::shared_ptr<opflexagent::Endpoint const>, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<cha #4 opflexagent::EndpointManager::updateEndpointLocal(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::optional<std::set<opflex::modb::URI, std::less<opflex::modb::URI>, std::allocator<opfl #5 opflexagent::EndpointManager::updateEndpoint(opflexagent::Endpoint const&) lib/EndpointManager.cpp:311 (libopflex_agent.so.0+0x1bf259) #6 opflexagent::EndpointSource::updateEndpoint(opflexagent::Endpoint const&) lib/EndpointSource.cpp:23 (libopflex_agent.so.0+0x1fae1a) #7 opflexagent::FSEndpointSource::updated(boost::filesystem::path const&) lib/FSEndpointSource.cpp:521 (libopflex_agent.so.0+0x204af0) #8 opflexagent::FSWatcher::operator()() lib/FSWatcher.cpp:215 (libopflex_agent.so.0+0x19e743) Previous read of size 8 at 0x7fff79e44c50 by thread T8 (mutexes: write M1365): #0 std::_Hashtable<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexagent::Endpoint #1 std::_Hashtable<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexagent::Endpoint #2 std::_Hashtable<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexagent::Endpoint #3 std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::shared_ptr<opflexagent::Endpoint const>, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<cha #4 opflexagent::IntFlowManager::updateSvcTgtStatsFlows(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool const&, bool const&) ovs/IntFlowManager.cpp:3774 (libopflex_agent_renderer_openvswitch #5 opflexagent::IntFlowManager::updateSvcTgtStatsFlows(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool const&, bool const&) ovs/IntFlowManager.cpp:3821 (libopflex_agent_renderer_openvswitch #6 opflexagent::IntFlowManager::handleUpdateSvcStatsFlows(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) ovs/IntFlowManager.cpp:3027 (libopflex_agent_renderer_openvswitch.so+0x142442) #7 operator() ovs/IntFlowManager.cpp:3080 (libopflex_agent_renderer_openvswitch.so+0x142852) Pattern2: WARNING: ThreadSanitizer: data race (pid=23522) Write of size 8 at 0x7b1000044688 by thread T13 (mutexes: write M1028): #0 operator delete(void*) <null> (libtsan.so.0+0x7747e) #1 __gnu_cxx::new_allocator<std::__detail::_Hash_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexagent::Endpoint const> >, true> >::deallocate(std::__detail:: #2 std::allocator_traits<std::allocator<std::__detail::_Hash_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexagent::Endpoint const> >, true> > >::deallocate(s #3 std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexagent::Endpoint const> >, true> > >::_M #4 std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexagent::Endpoint const> >, true> > >::_M #5 std::_Hashtable<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexagent::Endpoint #6 std::_Hashtable<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexagent::Endpoint #7 std::_Hashtable<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexagent::Endpoint #8 std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::shared_ptr<opflexagent::Endpoint const>, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<cha #9 opflexagent::EndpointManager::removeEndpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) lib/EndpointManager.cpp:353 (libopflex_agent.so.0+0x1c3972) #10 opflexagent::EndpointSource::removeEndpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) lib/EndpointSource.cpp:27 (libopflex_agent.so.0+0x1fae5a) #11 opflexagent::FSEndpointSource::deleted(boost::filesystem::path const&) lib/FSEndpointSource.cpp:544 (libopflex_agent.so.0+0x1fb49b) Previous read of size 8 at 0x7b1000044688 by thread T8 (mutexes: write M1365): #0 std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::data() const /usr/include/c++/9/bits/basic_string.h:2313 (libopflex_agent_renderer_openvswitch.so+0x169c83) #1 __gnu_cxx::__enable_if<std::__is_char<char>::__value, bool>::__type std::operator==<char>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<cha #2 std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >::operator()(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, st #3 std::__detail::_Equal_helper<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexag #4 std::__detail::_Hashtable_base<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflex #5 std::_Hashtable<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexagent::Endpoint #6 std::_Hashtable<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexagent::Endpoint #7 std::_Hashtable<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexagent::Endpoint #8 std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::shared_ptr<opflexagent::Endpoint const>, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<cha #9 opflexagent::IntFlowManager::updateSvcNodeStatsFlows(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool const&, bool const&) ovs/IntFlowManager.cpp:3520 (libopflex_agent_renderer_openvswitc #10 opflexagent::IntFlowManager::updateSvcNodeStatsFlows(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool const&, bool const&) ovs/IntFlowManager.cpp:3567 (libopflex_agent_renderer_openvswit #11 opflexagent::IntFlowManager::handleUpdateSvcStatsFlows(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) ovs/IntFlowManager.cpp:3028 (libopflex_agent_renderer_openvswitch.so+0x142453) #12 operator() ovs/IntFlowManager.cpp:3080 (libopflex_agent_renderer_openvswitch.so+0x142852) Pattern3: #0 std::_Hashtable<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexagent::Endpoint #1 std::_Hashtable<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexagent::Endpoint [2020-Dec-08 19:51:10.927044] [error] [lib/FSEndpointSource.cpp:527:updated] Could not load endpoint from: "/usr/local/var/lib/opflex-agent-ovs/endpoints/h1.ep": /usr/local/var/lib/opflex-agent-ovs/endpoints/h1.ep: cannot open file #2 std::_Hashtable<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::shared_ptr<opflexagent::Endpoint #3 std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::shared_ptr<opflexagent::Endpoint const>, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<cha #4 opflexagent::IntFlowManager::updateSvcTgtStatsFlows(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool const&, bool const&) ovs/IntFlowManager.cpp:3774 (libopflex_agent_renderer_openvswitch #5 opflexagent::IntFlowManager::updateSvcTgtStatsFlows(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool const&, bool const&) ovs/IntFlowManager.cpp:3821 (libopflex_agent_renderer_openvswitch #6 opflexagent::IntFlowManager::handleUpdateSvcStatsFlows(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) ovs/IntFlowManager.cpp:3027 (libopflex_agent_renderer_openvswitch.so+0x142442) #7 operator() ovs/IntFlowManager.cpp:3080 (libopflex_agent_renderer_openvswitch.so+0x142852) Previous write of size 8 at 0x7b0c0000fdf0 by thread T12: [failed to restore the stack] Mutex M1365 (0x7ba400000920) created at: [2020-Dec-08 19:51:10.927490] [debug] [ovs/TableState.cpp:489:apply] DEL|cookie=0x0, duration=0s, table=5, n_packets=0, n_bytes=0, idle_age=0, priority=20,icmp6,reg4=0x1,reg6=0x1,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00,icmp_type=135,ic #0 pthread_mutex_lock <null> (libtsan.so.0+0x41d5b) #1 __gthread_mutex_lock /usr/include/x86_64-linux-gnu/c++/9/bits/gthr-default.h:749 (libopflex_agent_renderer_openvswitch.so+0x141ae4) #2 std::mutex::lock() /usr/include/c++/9/bits/std_mutex.h:100 (libopflex_agent_renderer_openvswitch.so+0x141ae4) #3 std::lock_guard<std::mutex>::lock_guard(std::mutex&) /usr/include/c++/9/bits/std_mutex.h:159 (libopflex_agent_renderer_openvswitch.so+0x141ae4) #4 opflexagent::IntFlowManager::handleUpdateSvcStatsFlows(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) ovs/IntFlowManager.cpp:3015 (libopflex_agent_renderer_openvswitch.so+0x141ae4) #5 operator() ovs/IntFlowManager.cpp:3080 (libopflex_agent_renderer_openvswitch.so+0x142852) Pattern4: WARNING: ThreadSanitizer: heap-use-after-free (pid=10699) Read of size 8 at 0x7b6400021e88 by thread T8 (mutexes: write M1379, write M182): #0 std::_Hashtable<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std: #1 std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::hash<std::__cxx11::basic_string<char, std::ch #2 opflexagent::AgentPrometheusManager::createLabelMapFromSvcTargetAttr(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<ch #3 opflexagent::AgentPrometheusManager::createDynamicGaugeSvcTarget(opflexagent::AgentPrometheusManager::SVC_TARGET_METRICS, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_s #4 opflexagent::AgentPrometheusManager::addNUpdateSvcTargetCounter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > #5 opflexagent::IntFlowManager::updateSvcTgtStatsCounters(unsigned long const&, bool const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long const&, unsigned long const&, std::unor #6 operator() ovs/IntFlowManager.cpp:3703 (libopflex_agent_renderer_openvswitch.so+0x14127f) #7 opflexagent::IntFlowManager::updateSvcTgtStatsFlows(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool const&, bool const&) ovs/IntFlowManager.cpp:3776 (libopflex_agent_renderer_openvswitch #8 opflexagent::IntFlowManager::updateSvcTgtStatsFlows(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool const&, bool const&) ovs/IntFlowManager.cpp:3821 (libopflex_agent_renderer_openvswitch #9 opflexagent::IntFlowManager::handleUpdateSvcStatsFlows(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) ovs/IntFlowManager.cpp:3027 (libopflex_agent_renderer_openvswitch.so+0x142a72) #10 operator() ovs/IntFlowManager.cpp:3080 (libopflex_agent_renderer_openvswitch.so+0x142e82) Previous write of size 8 at 0x7b6400021e88 by thread T12: #0 operator delete(void*) <null> (libtsan.so.0+0x7747e) #1 std::_Sp_counted_ptr_inplace<opflexagent::Endpoint const, std::allocator<opflexagent::Endpoint>, (__gnu_cxx::_Lock_policy)2>::_M_destroy() /usr/include/c++/9/ext/new_allocator.h:128 (libopflex_agent.so.0+0x1c2f75) #2 std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release() /usr/include/c++/9/bits/shared_ptr_base.h:171 (libopflex_agent_renderer_openvswitch.so+0x1904e8) #3 std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release() /usr/include/c++/9/bits/shared_ptr_base.h:148 (libopflex_agent_renderer_openvswitch.so+0x1904e8) #4 std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count() /usr/include/c++/9/bits/shared_ptr_base.h:730 (libopflex_agent_renderer_openvswitch.so+0x1904e8) #5 std::__shared_ptr<opflexagent::Endpoint const, (__gnu_cxx::_Lock_policy)2>::~__shared_ptr() /usr/include/c++/9/bits/shared_ptr_base.h:1169 (libopflex_agent_renderer_openvswitch.so+0x1904e8) #6 std::shared_ptr<opflexagent::Endpoint const>::~shared_ptr() /usr/include/c++/9/bits/shared_ptr.h:103 (libopflex_agent_renderer_openvswitch.so+0x1904e8) #7 opflexagent::AccessFlowManager::handleEndpointUpdate(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) ovs/AccessFlowManager.cpp:411 (libopflex_agent_renderer_openvswitch.so+0x1904e8) Signed-off-by: Gautam Venkataramanan <gautam.chennai@gmail.com>
- Loading branch information