Backport #51908 to 23.5: Fix deadlock on DatabaseCatalog shutdown

src/Interpreters/DatabaseCatalog.cpp

@@ -56,6 +56,7 @@ namespace ErrorCodes
     extern const int DATABASE_ACCESS_DENIED;
     extern const int LOGICAL_ERROR;
     extern const int HAVE_DEPENDENT_OBJECTS;
+    extern const int UNFINISHED;
 }
 
 TemporaryTableHolder::TemporaryTableHolder(ContextPtr context_, const TemporaryTableHolder::Creator & creator, const ASTPtr & query)
@@ -196,6 +197,9 @@ void DatabaseCatalog::startupBackgroundCleanup()
 
 void DatabaseCatalog::shutdownImpl()
 {
+    is_shutting_down = true;
+    wait_table_finally_dropped.notify_all();
+
     if (cleanup_task)
         (*cleanup_task)->deactivate();
 
@@ -1153,8 +1157,13 @@ void DatabaseCatalog::waitTableFinallyDropped(const UUID & uuid)
     std::unique_lock lock{tables_marked_dropped_mutex};
     wait_table_finally_dropped.wait(lock, [&]() TSA_REQUIRES(tables_marked_dropped_mutex) -> bool
     {
-        return !tables_marked_dropped_ids.contains(uuid);
+        return !tables_marked_dropped_ids.contains(uuid) || is_shutting_down;
     });
+
+    /// TSA doesn't support unique_lock
+    if (TSA_SUPPRESS_WARNING_FOR_READ(tables_marked_dropped_ids).contains(uuid))
+        throw Exception(ErrorCodes::UNFINISHED, "Did not finish dropping the table with UUID {} because the server is shutting down, "
+                                                "will finish after restart", uuid);
 }
 
 void DatabaseCatalog::addDependencies(

src/Interpreters/DatabaseCatalog.h

@@ -305,6 +305,8 @@ class DatabaseCatalog : boost::noncopyable, WithMutableContext
 
     Poco::Logger * log;
 
+    std::atomic_bool is_shutting_down = false;
+
     /// Do not allow simultaneous execution of DDL requests on the same table.
     /// database name -> database guard -> (table name mutex, counter),
     /// counter: how many threads are running a query on the table at the same time