Rare out_of_range exception during init somewhere in db::system_keyspace::build_dc_rack_info

[gleb-cloudius]

While running cluster I sometimes get std::out_of_range exception during init of a seed
node. Here is a stack trace:

#0  0x00007ffff74b1500 in std::out_of_range::out_of_range(std::string const&)@plt () from /lib64/libstdc++.so.6
#1  0x00007ffff7510d3e in std::__throw_out_of_range(char const*) ()
   from /lib64/libstdc++.so.6
#2  0x00000000008b8cda in std::__detail::_Map_base<basic_sstring<char, unsigned int, 15u>, std::pair<basic_sstring<char, unsigned int, 15u> const, std::experimental::optional<basic_sstring<signed char, unsigned int, 31u> > >, std::allocator<std::pair<basic_sstring<char, unsigned int, 15u> const, std::experimental::optional<basic_sstring<signed char, unsigned int, 31u> > > >, std::__detail::_Select1st, std::equal_to<basic_sstring<char, unsigned int, 15u> >, std::hash<basic_sstring<char, unsigned int, 15u> >, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, true>::at (this=this@entry=0x6000000b0568, 
    __k=...) at /usr/include/c++/4.9.2/bits/hashtable_policy.h:672
#3  0x00000000008b9c3c in at (__k=..., this=0x6000000b0568)
    at /usr/include/c++/4.9.2/bits/unordered_map.h:648
#4  get_blob (name=..., this=0x6000000b0550) at ./cql3/untyped_result_set.hh:48
#5  cql3::untyped_result_set::row::get_as<net::ipv4_address> (
    this=this@entry=0x6000000b0550, name=...)
    at ./cql3/untyped_result_set.hh:52
#6  0x000000000089cc74 in operator()<const cql3::untyped_result_set::row> (
    __closure=<optimized out>, row=...) at db/system_keyspace.cc:462
#7  do_for_each<__gnu_cxx::__normal_iterator<const cql3::untyped_result_set::row*, std::vector<cql3::untyped_result_set::row> >, db::system_keyspace::build_dc_rack_info()::<lambda()>::<lambda(shared_ptr<cql3::untyped_result_set>)>::<lambda(auto:28&)> >(__gnu_cxx::__normal_iterator<cql3::untyped_result_set::row const*, std::vector<cql3::untyped_result_set::row, std::allocator<cql3::untyped_result_set::row> > >, __gnu_cxx::__normal_iterator<cql3::untyped_result_set::row const*, std::vector<cql3::untyped_result_set::row, std::allocator<cql3::untyped_result_set::row> > >, <unknown type in /home/gleb/work/seastar/build/release/scylla, CU 0x6718d88, DIE 0x6905033>) (begin=..., end=..., action=<optimized out>)
    at /home/gleb/work/seastar/seastar/core/future-util.hh:271
#8  0x000000000089e285 in operator() (__closure=__closure@entry=0x7fffffffd420)
    at /home/gleb/work/seastar/seastar/core/future-util.hh:279
#9  apply (args=<optimized out>, 
    func=func@entry=<unknown type in /home/gleb/work/seastar/build/release/scylla, CU 0x6718d88, DIE 0x6905c72>)
    at /home/gleb/work/seastar/seastar/core/apply.hh:34
#10 apply<do_for_each(Iterator, Iterator, AsyncAction&&) [with Iterator = __gnu_cxx::__normal_iterator<const cql3::untyped_result_set::row*, std::vector<cql3::untyped_result_set::row> >; AsyncAction = db::system_keyspace::build_dc_rack_info()::<lambda()>::<lambda(shared_ptr<cql3::untyped_result_set>)>::<lambda(auto:28&)>]::<lambda()> > (args=<optimized out>, 
    func=func@entry=<unknown type in /home/gleb/work/seastar/build/release/scylla, CU 0x6718d88, DIE 0x6905ce0>)
    at /home/gleb/work/seastar/seastar/core/apply.hh:42
#11 futurize<future<> >::apply<do_for_each(Iterator, Iterator, AsyncAction&&) [with Iterator = __gnu_cxx::__normal_iterator<const cql3::untyped_result_set::row*, std::vector<cql3::untyped_result_set::row> >; AsyncAction = db::system_keyspace::build_dc_rack_info()::<lambda()>::<lambda(shared_ptr<cql3::untyped_result_set>)>::<lambda(auto:28&)>]::<lambda()> >(<unknown type in /home/gleb/work/seastar/build/release/scylla, CU 0x6718d88, DIE 0x6905d20>, <unknown type in /home/gleb/work/seastar/build/release/scylla, CU 0x6718d88, DIE 0x6905d33>) (
    func=func@entry=<unknown type in /home/gleb/work/seastar/build/release/scylla, CU 0x6718d88, DIE 0x6905d20>, args=<optimized out>)
    at /home/gleb/work/seastar/seastar/core/future.hh:1021
#12 0x00000000008b2a84 in operator()<future_state<> > (
    state=<unknown type in /home/gleb/work/seastar/build/release/scylla, CU 0x6718d88, DIE 0x69e2c49>, __closure=0x60000086d320)
    at /home/gleb/work/seastar/seastar/core/future.hh:637
#13 continuation<future<T>::then(Func&&, Param&&) [with Ret = future<>; Func = do_for_each(Iterator, Iterator, AsyncAction&&) [with Iterator = __gnu_cxx::__normal_iterator<const cql3::untyped_result_set::row*, std::vector<cql3::untyped_result_set::row> >; AsyncAction = db::system_keyspace::build_dc_rack_info()::<lambda()>::<lambda(shared_ptr<cql3::untyped_result_set>)>::<lambda(auto:28&)>]::<lambda()>; Param = future<T>::then(Func&&) [with Func = do_for_each(Iterator, Iterator, AsyncAction&&) [with Iterator = __gnu_cxx::__normal_iterator<const cql3::untyped_result_set::row*, std::vector<cql3::untyped_result_set::row> >; AsyncAction = db::system_keyspace::build_dc_rack_info()::<lambda()>::<lambda(shared_ptr<cql3::untyped_result_set>)>::<lambda(auto:28&)>]::<lambda()>; Result = future<>; T = {}]::<lambda(future_state<>&&)>; T = {}; futurize_t<Ret> = future<>]::<lambda(auto:3&&)> >::run(void) (this=0x60000086d310)
    at /home/gleb/work/seastar/seastar/core/future.hh:333
#14 0x00000000004437f2 in reactor::run_tasks (this=this@entry=0x6000001d0000, 
    tasks=..., quota=<optimized out>) at core/reactor.cc:1054
#15 0x000000000045ed8b in reactor::run (this=0x6000001d0000)
    at core/reactor.cc:1151
#16 0x00000000004b064e in app_template::run(int, char**, std::function<void ()>&&) (this=this@entry=0x7fffffffdb20, ac=ac@entry=17, 
    av=av@entry=0x7fffffffdd68, 
    func=func@entry=<unknown type in /home/gleb/work/seastar/build/release/scylla, CU 0x46c5f8, DIE 0x505a6b>) at core/app-template.cc:104
#17 0x000000000041ac78 in main (ac=17, av=0x7fffffffdd68) at main.cc:208

[avikivity]

@asias

[slivne]

@gleb-cloudius are you still running into this - I am not able to reproduce, if you are unable to reproduce - please close