Guarantee failed: [pair.second] key for entry_t already exists

[timmaxw]

2015-05-22T22:44:29.012970266 124.263009s error: Error in ../src/concurrency/watchable_map.tcc at line 98:
2015-05-22T22:44:29.015198513 124.265236s error: Guarantee failed: [pair.second] key for entry_t already exists
2015-05-22T22:44:29.022269817 124.272307s error: Backtrace:
2015-05-22T22:45:16.504757329 171.754797s error: Fri May 22 22:44:29 2015

1: rethinkdb_backtrace(void**, int) at rethinkdb_backtrace.cc:100
2: backtrace_t::backtrace_t() at backtrace.cc:203
3: lazy_backtrace_formatter_t::lazy_backtrace_formatter_t() at backtrace.cc:283
4: format_backtrace(bool) at backtrace.cc:198
5: report_fatal_error(char const*, int, char const*, ...) at errors.cc:83
6: watchable_map_var_t<std::pair<uuid_u, uuid_u>, contract_execution_bcard_t>::entry_t::entry_t(watchable_map_var_t<std::pair<uuid_u, uuid_u>, contract_execution_bcard_t>*, std::pair<uuid_u, uuid_u> const&, contract_execution_bcard_t const&) at watchable_map.tcc:98
7: minidir_read_manager_t<std::pair<uuid_u, uuid_u>, contract_execution_bcard_t>::on_update(signal_t*, peer_id_t const&, uuid_u const&, fifo_enforcer_write_token_t, bool, boost::optional<std::pair<uuid_u, uuid_u> > const&, boost::optional<contract_execution_bcard_t> const&) at minidir.tcc:89
8: std::_Mem_fn<void (minidir_read_manager_t<std::pair<uuid_u, uuid_u>, contract_execution_bcard_t>::*)(signal_t*, peer_id_t const&, uuid_u const&, fifo_enforcer_write_token_t, bool, boost::optional<std::pair<uuid_u, uuid_u> > const&, boost::optional<contract_execution_bcard_t> const&)>::operator()(minidir_read_manager_t<std::pair<uuid_u, uuid_u>, contract_execution_bcard_t>*, signal_t*, peer_id_t const&, uuid_u const&, fifo_enforcer_write_token_t, bool, boost::optional<std::pair<uuid_u, uuid_u> > const&, boost::optional<contract_execution_bcard_t> const&) const at functional:551
9: void std::_Bind<std::_Mem_fn<void (minidir_read_manager_t<std::pair<uuid_u, uuid_u>, contract_execution_bcard_t>::*)(signal_t*, peer_id_t const&, uuid_u const&, fifo_enforcer_write_token_t, bool, boost::optional<std::pair<uuid_u, uuid_u> > const&, boost::optional<contract_execution_bcard_t> const&)> (minidir_read_manager_t<std::pair<uuid_u, uuid_u>, contract_execution_bcard_t>*, std::_Placeholder<1>, std::_Placeholder<2>, std::_Placeholder<3>, std::_Placeholder<4>, std::_Placeholder<5>, std::_Placeholder<6>, std::_Placeholder<7>)>::__call<void, signal_t*&&, peer_id_t&&, uuid_u&&, fifo_enforcer_write_token_t&&, bool&&, boost::optional<std::pair<uuid_u, uuid_u> >&&, boost::optional<contract_execution_bcard_t>&&, 0, 1, 2, 3, 4, 5, 6, 7>(std::tuple<signal_t*&&, peer_id_t&&, uuid_u&&, fifo_enforcer_write_token_t&&, bool&&, boost::optional<std::pair<uuid_u, uuid_u> >&&, boost::optional<contract_execution_bcard_t>&&>&&, std::_Index_tuple<0, 1, 2, 3, 4, 5, 6, 7>) at functional:1145
10: void std::_Bind<std::_Mem_fn<void (minidir_read_manager_t<std::pair<uuid_u, uuid_u>, contract_execution_bcard_t>::*)(signal_t*, peer_id_t const&, uuid_u const&, fifo_enforcer_write_token_t, bool, boost::optional<std::pair<uuid_u, uuid_u> > const&, boost::optional<contract_execution_bcard_t> const&)> (minidir_read_manager_t<std::pair<uuid_u, uuid_u>, contract_execution_bcard_t>*, std::_Placeholder<1>, std::_Placeholder<2>, std::_Placeholder<3>, std::_Placeholder<4>, std::_Placeholder<5>, std::_Placeholder<6>, std::_Placeholder<7>)>::operator()<signal_t*, peer_id_t, uuid_u, fifo_enforcer_write_token_t, bool, boost::optional<std::pair<uuid_u, uuid_u> >, boost::optional<contract_execution_bcard_t>, void>(signal_t*&&, peer_id_t&&, uuid_u&&, fifo_enforcer_write_token_t&&, bool&&, boost::optional<std::pair<uuid_u, uuid_u> >&&, boost::optional<contract_execution_bcard_t>&&) at functional:1203
11: std::_Function_handler<void (signal_t*, peer_id_t, uuid_u, fifo_enforcer_write_token_t, bool, boost::optional<std::pair<uuid_u, uuid_u> >, boost::optional<contract_execution_bcard_t>), std::_Bind<std::_Mem_fn<void (minidir_read_manager_t<std::pair<uuid_u, uuid_u>, contract_execution_bcard_t>::*)(signal_t*, peer_id_t const&, uuid_u const&, fifo_enforcer_write_token_t, bool, boost::optional<std::pair<uuid_u, uuid_u> > const&, boost::optional<contract_execution_bcard_t> const&)> (minidir_read_manager_t<std::pair<uuid_u, uuid_u>, contract_execution_bcard_t>*, std::_Placeholder<1>, std::_Placeholder<2>, std::_Placeholder<3>, std::_Placeholder<4>, std::_Placeholder<5>, std::_Placeholder<6>, std::_Placeholder<7>)> >::_M_invoke(std::_Any_data const&, signal_t*, peer_id_t, uuid_u, fifo_enforcer_write_token_t, bool, boost::optional<std::pair<uuid_u, uuid_u> >, boost::optional<contract_execution_bcard_t>) at functional:1780
12: std::function<void (signal_t*, peer_id_t, uuid_u, fifo_enforcer_write_token_t, bool, boost::optional<std::pair<uuid_u, uuid_u> >, boost::optional<contract_execution_bcard_t>)>::operator()(signal_t*, peer_id_t, uuid_u, fifo_enforcer_write_token_t, bool, boost::optional<std::pair<uuid_u, uuid_u> >, boost::optional<contract_execution_bcard_t>) const at functional:2161
13: mailbox_t<void (peer_id_t, uuid_u, fifo_enforcer_write_token_t, bool, boost::optional<std::pair<uuid_u, uuid_u> >, boost::optional<contract_execution_bcard_t>)>::read_impl_t::read(read_stream_t*, signal_t*) at typed.hpp:556
14: mailbox_manager_t::mailbox_read_coroutine(connectivity_cluster_t::connection_t*, auto_drainer_t::lock_t, threadnum_t, unsigned long, std::vector<char, std::allocator<char> >*, long, mailbox_manager_t::force_yield_t) at mailbox.cc:270
15: /home/tim/rethinkdb/build/debug_clang/rethinkdb() [0x14d3c8f] at 0x14d3c8f ()
16: /home/tim/rethinkdb/build/debug_clang/rethinkdb() [0x14d3b7c] at 0x14d3b7c ()
17: callable_action_wrapper_t::run() at runtime_utils.cc:43
18: coro_t::run() at coroutines.cc:205
19: /home/tim/rethinkdb/build/debug_clang/rethinkdb() [0x14d346d] at 0x14d346d ()
20: mailbox_manager_t::on_message(connectivity_cluster_t::connection_t*, auto_drainer_t::lock_t, read_stream_t*) at mailbox.cc:232
21: connectivity_cluster_t::run_t::handle(keepalive_tcp_conn_stream_t*, boost::optional<peer_id_t>, boost::optional<peer_address_t>, auto_drainer_t::lock_t, bool*) at cluster.cc:1126
22: connectivity_cluster_t::run_t::connect_to_peer(peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*) at cluster.cc:323
23: std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*)>::operator()(connectivity_cluster_t::run_t*, peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*) const at functional:551
24: void std::_Bind<std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*)> (connectivity_cluster_t::run_t*, peer_address_t const*, std::_Placeholder<1>, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, static_semaphore_t*)>::__call_c<void, long&, 0, 1, 2, 3, 4, 5, 6>(std::tuple<long&>&&, std::_Index_tuple<0, 1, 2, 3, 4, 5, 6>) const at functional:1155
25: void std::_Bind<std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*)> (connectivity_cluster_t::run_t*, peer_address_t const*, std::_Placeholder<1>, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, static_semaphore_t*)>::operator()<long&, void>(long&) const at functional:1218
26: pmap_runner_one_arg_t<std::_Bind<std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*)> (connectivity_cluster_t::run_t*, peer_address_t const*, std::_Placeholder<1>, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, static_semaphore_t*)>, long>::operator()() at pmap.hpp:21
27: callable_action_instance_t<pmap_runner_one_arg_t<std::_Bind<std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*)> (connectivity_cluster_t::run_t*, peer_address_t const*, std::_Placeholder<1>, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, static_semaphore_t*)>, long> >::run_action() at callable_action.hpp:31
28: callable_action_wrapper_t::run() at runtime_utils.cc:43
29: coro_t::run() at coroutines.cc:205
30: void coro_t::spawn_now_dangerously<pmap_runner_one_arg_t<std::_Bind<std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*)> (connectivity_cluster_t::run_t*, peer_address_t const*, std::_Placeholder<1>, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, static_semaphore_t*)>, long> >(pmap_runner_one_arg_t<std::_Bind<std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*)> (connectivity_cluster_t::run_t*, peer_address_t const*, std::_Placeholder<1>, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, static_semaphore_t*)>, long>&&) at coroutines.hpp:52
31: void pmap<std::_Bind<std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*)> (connectivity_cluster_t::run_t*, peer_address_t const*, std::_Placeholder<1>, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, static_semaphore_t*)> >(long, long, std::_Bind<std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*)> (connectivity_cluster_t::run_t*, peer_address_t const*, std::_Placeholder<1>, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, static_semaphore_t*)> const&) at pmap.hpp:44
32: void pmap<std::_Bind<std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*)> (connectivity_cluster_t::run_t*, peer_address_t const*, std::_Placeholder<1>, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, static_semaphore_t*)> >(long, std::_Bind<std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*)> (connectivity_cluster_t::run_t*, peer_address_t const*, std::_Placeholder<1>, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, static_semaphore_t*)> const&) at pmap.hpp:54
33: connectivity_cluster_t::run_t::join_blocking(peer_address_t, boost::optional<peer_id_t>, auto_drainer_t::lock_t) at cluster.cc:359
34: std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t, boost::optional<peer_id_t>, auto_drainer_t::lock_t)>::operator()(connectivity_cluster_t::run_t*, peer_address_t, boost::optional<peer_id_t>, auto_drainer_t::lock_t) const at functional:551
35: void std::_Bind<std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t, boost::optional<peer_id_t>, auto_drainer_t::lock_t)> (connectivity_cluster_t::run_t*, peer_address_t, int boost::detail::none_helper::*, auto_drainer_t::lock_t)>::__call<void, , 0, 1, 2, 3>(std::tuple<>&&, std::_Index_tuple<0, 1, 2, 3>) at functional:1146
36: void std::_Bind<std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t, boost::optional<peer_id_t>, auto_drainer_t::lock_t)> (connectivity_cluster_t::run_t*, peer_address_t, int boost::detail::none_helper::*, auto_drainer_t::lock_t)>::operator()<, void>() at functional:1204
37: callable_action_instance_t<std::_Bind<std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t, boost::optional<peer_id_t>, auto_drainer_t::lock_t)> (connectivity_cluster_t::run_t*, peer_address_t, int boost::detail::none_helper::*, auto_drainer_t::lock_t)> >::run_action() at callable_action.hpp:31
38: callable_action_wrapper_t::run() at runtime_utils.cc:43
39: coro_t::run() at coroutines.cc:205
40: void coro_t::spawn_now_dangerously<std::_Bind<std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t, boost::optional<peer_id_t>, auto_drainer_t::lock_t)> (connectivity_cluster_t::run_t*, peer_address_t, int boost::detail::none_helper::*, auto_drainer_t::lock_t)> >(std::_Bind<std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t, boost::optional<peer_id_t>, auto_drainer_t::lock_t)> (connectivity_cluster_t::run_t*, peer_address_t, int boost::detail::none_helper::*, auto_drainer_t::lock_t)>&&) at coroutines.hpp:52
41: connectivity_cluster_t::run_t::join(peer_address_t const&) at cluster.cc:266
42: auto_reconnector_t::try_reconnect(uuid_u, auto_drainer_t::lock_t) at auto_reconnect.cc:98
43: boost::_mfi::mf2<void, auto_reconnector_t, uuid_u, auto_drainer_t::lock_t>::operator()(auto_reconnector_t*, uuid_u, auto_drainer_t::lock_t) const at mem_fn_template.hpp:280
44: void boost::_bi::list3<boost::_bi::value<auto_reconnector_t*>, boost::_bi::value<uuid_u>, boost::_bi::value<auto_drainer_t::lock_t> >::operator()<boost::_mfi::mf2<void, auto_reconnector_t, uuid_u, auto_drainer_t::lock_t>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf2<void, auto_reconnector_t, uuid_u, auto_drainer_t::lock_t>&, boost::_bi::list0&, int) at bind.hpp:392
45: boost::_bi::bind_t<void, boost::_mfi::mf2<void, auto_reconnector_t, uuid_u, auto_drainer_t::lock_t>, boost::_bi::list3<boost::_bi::value<auto_reconnector_t*>, boost::_bi::value<uuid_u>, boost::_bi::value<auto_drainer_t::lock_t> > >::operator()() at bind_template.hpp:20
46: callable_action_instance_t<boost::_bi::bind_t<void, boost::_mfi::mf2<void, auto_reconnector_t, uuid_u, auto_drainer_t::lock_t>, boost::_bi::list3<boost::_bi::value<auto_reconnector_t*>, boost::_bi::value<uuid_u>, boost::_bi::value<auto_drainer_t::lock_t> > > >::run_action() at callable_action.hpp:31
47: callable_action_wrapper_t::run() at runtime_utils.cc:43
48: coro_t::run() at coroutines.cc:205
49: coro_t* coro_t::spawn_sometime<boost::_bi::bind_t<void, boost::_mfi::mf2<void, auto_reconnector_t, uuid_u, auto_drainer_t::lock_t>, boost::_bi::list3<boost::_bi::value<auto_reconnector_t*>, boost::_bi::value<uuid_u>, boost::_bi::value<auto_drainer_t::lock_t> > > >(boost::_bi::bind_t<void, boost::_mfi::mf2<void, auto_reconnector_t, uuid_u, auto_drainer_t::lock_t>, boost::_bi::list3<boost::_bi::value<auto_reconnector_t*>, boost::_bi::value<uuid_u>, boost::_bi::value<auto_drainer_t::lock_t> > >&&) at coroutines.hpp:58
50: auto_reconnector_t::on_connect_or_disconnect() at auto_reconnect.cc:54
51: boost::_mfi::mf0<void, auto_reconnector_t>::operator()(auto_reconnector_t*) const at mem_fn_template.hpp:49
52: void boost::_bi::list1<boost::_bi::value<auto_reconnector_t*> >::operator()<boost::_mfi::mf0<void, auto_reconnector_t>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, auto_reconnector_t>&, boost::_bi::list0&, int) at bind.hpp:254
53: boost::_bi::bind_t<void, boost::_mfi::mf0<void, auto_reconnector_t>, boost::_bi::list1<boost::_bi::value<auto_reconnector_t*> > >::operator()() at bind_template.hpp:20
54: std::_Function_handler<void (), boost::_bi::bind_t<void, boost::_mfi::mf0<void, auto_reconnector_t>, boost::_bi::list1<boost::_bi::value<auto_reconnector_t*> > > >::_M_invoke(std::_Any_data const&) at functional:1780
55: std::function<void ()>::operator()() const at functional:2161
56: call_function(std::function<void ()> const&) at watchable.hpp:185
57: void publisher_controller_t<std::function<void ()> >::publish<void (*)(std::function<void ()> const&)>(void (* const&)(std::function<void ()> const&)) at pubsub.hpp:139
58: subview_watchable_t<change_tracking_map_t<peer_id_t, uuid_u>, change_tracking_map_t<peer_id_t, cluster_directory_metadata_t>, incremental_field_getter_t<uuid_u, cluster_directory_metadata_t> >::lensed_value_cache_t::on_parent_changed() at watchable.tcc:106
59: std::_Mem_fn<void (subview_watchable_t<change_tracking_map_t<peer_id_t, uuid_u>, change_tracking_map_t<peer_id_t, cluster_directory_metadata_t>, incremental_field_getter_t<uuid_u, cluster_directory_metadata_t> >::lensed_value_cache_t::*)()>::operator()(subview_watchable_t<change_tracking_map_t<peer_id_t, uuid_u>, change_tracking_map_t<peer_id_t, cluster_directory_metadata_t>, incremental_field_getter_t<uuid_u, cluster_directory_metadata_t> >::lensed_value_cache_t*) const at functional:551
60: void std::_Bind<std::_Mem_fn<void (subview_watchable_t<change_tracking_map_t<peer_id_t, uuid_u>, change_tracking_map_t<peer_id_t, cluster_directory_metadata_t>, incremental_field_getter_t<uuid_u, cluster_directory_metadata_t> >::lensed_value_cache_t::*)()> (subview_watchable_t<change_tracking_map_t<peer_id_t, uuid_u>, change_tracking_map_t<peer_id_t, cluster_directory_metadata_t>, incremental_field_getter_t<uuid_u, cluster_directory_metadata_t> >::lensed_value_cache_t*)>::__call<void, , 0>(std::tuple<>&&, std::_Index_tuple<0>) at functional:1146
61: void std::_Bind<std::_Mem_fn<void (subview_watchable_t<change_tracking_map_t<peer_id_t, uuid_u>, change_tracking_map_t<peer_id_t, cluster_directory_metadata_t>, incremental_field_getter_t<uuid_u, cluster_directory_metadata_t> >::lensed_value_cache_t::*)()> (subview_watchable_t<change_tracking_map_t<peer_id_t, uuid_u>, change_tracking_map_t<peer_id_t, cluster_directory_metadata_t>, incremental_field_getter_t<uuid_u, cluster_directory_metadata_t> >::lensed_value_cache_t*)>::operator()<, void>() at functional:1204
62: std::_Function_handler<void (), std::_Bind<std::_Mem_fn<void (subview_watchable_t<change_tracking_map_t<peer_id_t, uuid_u>, change_tracking_map_t<peer_id_t, cluster_directory_metadata_t>, incremental_field_getter_t<uuid_u, cluster_directory_metadata_t> >::lensed_value_cache_t::*)()> (subview_watchable_t<change_tracking_map_t<peer_id_t, uuid_u>, change_tracking_map_t<peer_id_t, cluster_directory_metadata_t>, incremental_field_getter_t<uuid_u, cluster_directory_metadata_t> >::lensed_value_cache_t*)> >::_M_invoke(std::_Any_data const&) at functional:1780
63: std::function<void ()>::operator()() const at functional:2161
64: call_function(std::function<void ()> const&) at watchable.hpp:185
65: void publisher_controller_t<std::function<void ()> >::publish<void (*)(std::function<void ()> const&)>(void (* const&)(std::function<void ()> const&)) at pubsub.hpp:139
66: watchable_variable_t<change_tracking_map_t<peer_id_t, cluster_directory_metadata_t> >::apply_atomic_op(std::function<bool (change_tracking_map_t<peer_id_t, cluster_directory_metadata_t>*)> const&) at watchable.hpp:223
67: directory_read_manager_t<cluster_directory_metadata_t>::handle_connection(connectivity_cluster_t::connection_t*, auto_drainer_t::lock_t, boost::shared_ptr<cluster_directory_metadata_t> const&, fifo_enforcer_state_t, auto_drainer_t::lock_t) at read_manager.tcc:166
68: std::_Mem_fn<void (directory_read_manager_t<cluster_directory_metadata_t>::*)(connectivity_cluster_t::connection_t*, auto_drainer_t::lock_t, boost::shared_ptr<cluster_directory_metadata_t> const&, fifo_enforcer_state_t, auto_drainer_t::lock_t)>::operator()(directory_read_manager_t<cluster_directory_metadata_t>*, connectivity_cluster_t::connection_t*, auto_drainer_t::lock_t, boost::shared_ptr<cluster_directory_metadata_t> const&, fifo_enforcer_state_t, auto_drainer_t::lock_t) const at functional:551
69: void std::_Bind<std::_Mem_fn<void (directory_read_manager_t<cluster_directory_metadata_t>::*)(connectivity_cluster_t::connection_t*, auto_drainer_t::lock_t, boost::shared_ptr<cluster_directory_metadata_t> const&, fifo_enforcer_state_t, auto_drainer_t::lock_t)> (directory_read_manager_t<cluster_directory_metadata_t>*, connectivity_cluster_t::connection_t*, auto_drainer_t::lock_t, boost::shared_ptr<cluster_directory_metadata_t>, fifo_enforcer_state_t, auto_drainer_t::lock_t)>::__call<void, , 0, 1, 2, 3, 4, 5>(std::tuple<>&&, std::_Index_tuple<0, 1, 2, 3, 4, 5>) at functional:1146
70: void std::_Bind<std::_Mem_fn<void (directory_read_manager_t<cluster_directory_metadata_t>::*)(connectivity_cluster_t::connection_t*, auto_drainer_t::lock_t, boost::shared_ptr<cluster_directory_metadata_t> const&, fifo_enforcer_state_t, auto_drainer_t::lock_t)> (directory_read_manager_t<cluster_directory_metadata_t>*, connectivity_cluster_t::connection_t*, auto_drainer_t::lock_t, boost::shared_ptr<cluster_directory_metadata_t>, fifo_enforcer_state_t, auto_drainer_t::lock_t)>::operator()<, void>() at functional:1204
71: callable_action_instance_t<std::_Bind<std::_Mem_fn<void (directory_read_manager_t<cluster_directory_metadata_t>::*)(connectivity_cluster_t::connection_t*, auto_drainer_t::lock_t, boost::shared_ptr<cluster_directory_metadata_t> const&, fifo_enforcer_state_t, auto_drainer_t::lock_t)> (directory_read_manager_t<cluster_directory_metadata_t>*, connectivity_cluster_t::connection_t*, auto_drainer_t::lock_t, boost::shared_ptr<cluster_directory_metadata_t>, fifo_enforcer_state_t, auto_drainer_t::lock_t)> >::run_action() at callable_action.hpp:31
72: callable_action_wrapper_t::run() at runtime_utils.cc:43
73: coro_t::run() at coroutines.cc:205
74: coro_t* coro_t::spawn_sometime<std::_Bind<std::_Mem_fn<void (directory_read_manager_t<cluster_directory_metadata_t>::*)(connectivity_cluster_t::connection_t*, auto_drainer_t::lock_t, boost::shared_ptr<cluster_directory_metadata_t> const&, fifo_enforcer_state_t, auto_drainer_t::lock_t)> (directory_read_manager_t<cluster_directory_metadata_t>*, connectivity_cluster_t::connection_t*, auto_drainer_t::lock_t, boost::shared_ptr<cluster_directory_metadata_t>, fifo_enforcer_state_t, auto_drainer_t::lock_t)> >(std::_Bind<std::_Mem_fn<void (directory_read_manager_t<cluster_directory_metadata_t>::*)(connectivity_cluster_t::connection_t*, auto_drainer_t::lock_t, boost::shared_ptr<cluster_directory_metadata_t> const&, fifo_enforcer_state_t, auto_drainer_t::lock_t)> (directory_read_manager_t<cluster_directory_metadata_t>*, connectivity_cluster_t::connection_t*, auto_drainer_t::lock_t, boost::shared_ptr<cluster_directory_metadata_t>, fifo_enforcer_state_t, auto_drainer_t::lock_t)>&&) at coroutines.hpp:58
75: directory_read_manager_t<cluster_directory_metadata_t>::on_message(connectivity_cluster_t::connection_t*, auto_drainer_t::lock_t, read_stream_t*) at read_manager.tcc:69
76: connectivity_cluster_t::run_t::handle(keepalive_tcp_conn_stream_t*, boost::optional<peer_id_t>, boost::optional<peer_address_t>, auto_drainer_t::lock_t, bool*) at cluster.cc:1126
77: connectivity_cluster_t::run_t::connect_to_peer(peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*) at cluster.cc:323
78: std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*)>::operator()(connectivity_cluster_t::run_t*, peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*) const at functional:551
79: void std::_Bind<std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*)> (connectivity_cluster_t::run_t*, peer_address_t const*, std::_Placeholder<1>, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, static_semaphore_t*)>::__call_c<void, long&, 0, 1, 2, 3, 4, 5, 6>(std::tuple<long&>&&, std::_Index_tuple<0, 1, 2, 3, 4, 5, 6>) const at functional:1155
80: void std::_Bind<std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*)> (connectivity_cluster_t::run_t*, peer_address_t const*, std::_Placeholder<1>, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, static_semaphore_t*)>::operator()<long&, void>(long&) const at functional:1218
81: pmap_runner_one_arg_t<std::_Bind<std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*)> (connectivity_cluster_t::run_t*, peer_address_t const*, std::_Placeholder<1>, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, static_semaphore_t*)>, long>::operator()() at pmap.hpp:21
82: callable_action_instance_t<pmap_runner_one_arg_t<std::_Bind<std::_Mem_fn<void (connectivity_cluster_t::run_t::*)(peer_address_t const*, int, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, co_semaphore_t*)> (connectivity_cluster_t::run_t*, peer_address_t const*, std::_Placeholder<1>, boost::optional<peer_id_t>, auto_drainer_t::lock_t, bool*, static_semaphore_t*)>, long> >::run_action() at callable_action.hpp:31
2015-05-22T22:45:16.509564170 171.759602s error: Exiting.

This happened while running a cluster of 26 servers on my laptop.

[timmaxw]

At the time that this occurred, some of the connections were failing and rejoining. Here's a theory: We lost a connection to a server and then regained the connection before we noticed we had lost the original connection. So we had two connections to the same server, and so we got a duplicate entry in the minidir.

Wait, no, that's impossible. None of the processed had restarted, so the reconnection attempt would use the same peer ID and so it would be rejected.

[danielmewes]

We haven't been able to reproduce this. Moving to polish.

[danielmewes]

Just saw this again when killing a server in a three-node cluster.

[danielmewes]

Looks like this can at least intermittently be reproduced with the table fuzzer:
#4872 (comment)

[VeXocide]

In CR 3309 by @danielmewes.

[VeXocide]

Merged into next via commit 3c2473f.