replay: Ignore partial replay requests if node was dropped.

We've seen some errors where a partial replay is requested, whose source
 node was dropped at some point.
This commit allows the domains to simply ignore such requests, if the
node was already dropped.

NOTE: This assumes that the issue we've been seeing is due to some
messages being handled out of order by the domain (which is natural, if
it's receiving requests for multiple sources), such that a `Start
Replay` + `Drop node` messages are sent but received in inverse order.

Fixes: ENG-2852
Change-Id: Iaed4a2a9e13325476f056e17fc2ef8410382ccd2
Reviewed-on: https://gerrit.readyset.name/c/readyset/+/4758
Reviewed-by: Griffin Smith <griffin@readyset.io>
Tested-by: Buildkite CI

readyset-dataflow/src/domain/mod.rs

@@ -2557,6 +2557,16 @@ impl Domain {
             _ => internal!(),
         };
 
+        if self
+            .nodes
+            .get(*source)
+            .filter(|n| n.borrow().is_dropped())
+            .is_some()
+        {
+            warn!(?tag, node = ?source, domain = ?self.index, "replay path started with removed node; ignoring...");
+            return Ok(());
+        }
+
         let state = self.state.get(*source).ok_or_else(|| {
             internal_err!(
                 "migration replay path (tag {:?}) started with non-materialized node",