panic: no entry found for key as message_sink_connector_ready attempts to fetch a sink that was just deleted

[philip-stoev]

What version of Materialize are you using?

v0.8.4-dev (7cfb9bf58)

What was the issue?

@mjibson

There seems to be a race condition in the way message_sink_connector_ready operates. It attempts to do a catalog::Op::DropItem but the sink no longer exists in the catalog and panic ensues:

2021-07-27T07:12:54.110827Z ERROR panic: no entry found for key
thread: coordinator
location: src/coord/src/catalog.rs:1008:10
version: 0.8.4-dev (7cfb9bf58a42d545a0b814031a6ed9ea7a03f460)
backtrace:
   0: materialized::handle_panic
             at src/materialized/src/bin/materialized/main.rs:660:21
   1: core::ops::function::Fn::call
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/core/src/ops/function.rs:70:5
   2: std::panicking::rust_panic_with_hook
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/std/src/panicking.rs:595:17
   3: std::panicking::begin_panic_handler::{{closure}}
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/std/src/panicking.rs:497:13
   4: std::sys_common::backtrace::__rust_end_short_backtrace
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/std/src/sys_common/backtrace.rs:141:18
   5: rust_begin_unwind
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/std/src/panicking.rs:493:5
   6: core::panicking::panic_fmt
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/core/src/panicking.rs:92:14
   7: core::option::expect_failed
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/core/src/option.rs:1241:5
   8: core::option::Option<T>::expect
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/core/src/option.rs:349:21
      <alloc::collections::btree::map::BTreeMap<K,V> as core::ops::index::Index<&Q>>::index
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/alloc/src/collections/btree/map.rs:2028:9
      coord::catalog::Catalog::get_by_id
             at src/coord/src/catalog.rs:1008:10
   9: coord::coord::Coordinator::catalog_transact::{{closure}}
             at src/coord/src/coord.rs:2909:23
      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/core/src/future/mod.rs:80:19
  10: coord::coord::Coordinator::message_sink_connector_ready::{{closure}}
             at src/coord/src/coord.rs:690:17
      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/core/src/future/mod.rs:80:19
      coord::coord::Coordinator::serve::{{closure}}
             at src/coord/src/coord.rs:506:21
  11: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/core/src/future/mod.rs:80:19
      tokio::park::thread::CachedParkThread::block_on::{{closure}}
             at /home/pstoev/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.9.0/src/park/thread.rs:263:54
      tokio::coop::with_budget::{{closure}}
             at /home/pstoev/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.9.0/src/coop.rs:106:9
      std::thread::local::LocalKey<T>::try_with
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/std/src/thread/local.rs:376:16
      std::thread::local::LocalKey<T>::with
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/std/src/thread/local.rs:352:9
      tokio::coop::with_budget
             at /home/pstoev/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.9.0/src/coop.rs:99:5
      tokio::coop::budget
             at /home/pstoev/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.9.0/src/coop.rs:76:5
      tokio::park::thread::CachedParkThread::block_on
             at /home/pstoev/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.9.0/src/park/thread.rs:263:31
      tokio::runtime::enter::Enter::block_on
             at /home/pstoev/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.9.0/src/runtime/enter.rs:151:13
  12: tokio::runtime::handle::Handle::block_on
             at /home/pstoev/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.9.0/src/runtime/handle.rs:299:9
  13: coord::coord::serve::{{closure}}::{{closure}}
             at src/coord/src/coord.rs:3449:13
      std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/std/src/sys_common/backtrace.rs:125:18
  14: std::thread::Builder::spawn_unchecked::{{closure}}::{{closure}}
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/std/src/thread/mod.rs:481:17
      <std::panic::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/std/src/panic.rs:344:9
      std::panicking::try::do_call
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/std/src/panicking.rs:379:40
      std::panicking::try
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/std/src/panicking.rs:343:19
      std::panic::catch_unwind
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/std/src/panic.rs:431:14
      std::thread::Builder::spawn_unchecked::{{closure}}
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/std/src/thread/mod.rs:480:30
      core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/core/src/ops/function.rs:227:5
  15: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/alloc/src/boxed.rs:1546:9
      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/alloc/src/boxed.rs:1546:9
      std::sys::unix::thread::Thread::new::thread_start
             at /rustc/53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b/library/std/src/sys/unix/thread.rs:71:17
  16: start_thread
  17: clone

Is the issue reproducible? If so, please provide reproduction instructions.

I ran into this as I was developing a testdrive test, but unfortunately is is not easily repeatable. I hope it can be fixed based on code inspection.