MDEV-25954: Trim os_aio_wait_until_no_pending_writes()

It turns out that we had some unnecessary waits for no outstanding
write requests to exist. They were basically working around a
bug that was fixed in MDEV-25953.

On write completion callback, blocks will be marked clean.
So, it is sufficient to consult buf_pool.flush_list to determine
which writes have not been completed yet.

On FLUSH TABLES...FOR EXPORT we must still wait for all pending
asynchronous writes to complete, because buf_flush_file_space()
would merely guarantee that writes will have been initiated.

storage/innobase/buf/buf0dblwr.cc

@@ -689,7 +689,6 @@ void buf_dblwr_t::flush_buffered_writes()
 {
   if (!is_initialised() || !srv_use_doublewrite_buf)
   {
-    os_aio_wait_until_no_pending_writes();
     fil_flush_file_spaces();
     return;
   }

storage/innobase/buf/buf0flu.cc

@@ -1546,11 +1546,6 @@ static std::atomic_flag log_flush_pending;
 /** Advance log_sys.get_flushed_lsn() */
 static void log_flush(void *)
 {
-  /* Between batches, we try to prevent I/O stalls by these calls.
-  This should not be needed for correctness. */
-  os_aio_wait_until_no_pending_writes();
-  fil_flush_file_spaces();
-
   /* Guarantee progress for buf_flush_lists(). */
   log_buffer_flush_to_disk(true);
   log_flush_pending.clear();

storage/innobase/fil/fil0fil.cc

@@ -1743,8 +1743,6 @@ void fil_close_tablespace(ulint id)
 	Thus we can clean the tablespace out of buf_pool
 	completely and permanently. */
 	while (buf_flush_dirty_pages(id));
-	/* Ensure that all asynchronous IO is completed. */
-	os_aio_wait_until_no_pending_writes();
 	ut_ad(space->is_stopping());
 
 	/* If the free is successful, the X lock will be released before

storage/innobase/include/os0file.h

@@ -1111,10 +1111,9 @@ void os_aio_free();
 @retval DB_IO_ERROR on I/O error */
 dberr_t os_aio(const IORequest &type, void *buf, os_offset_t offset, size_t n);
 
-/** Waits until there are no pending writes in os_aio_write_array. There can
-be other, synchronous, pending writes. */
-void
-os_aio_wait_until_no_pending_writes();
+/** Wait until there are no pending asynchronous writes.
+Only used on FLUSH TABLES...FOR EXPORT. */
+void os_aio_wait_until_no_pending_writes();
 
 
 /** Prints info of the aio arrays.

storage/innobase/os/os0file.cc

@@ -3788,8 +3788,8 @@ static void os_aio_wait_until_no_pending_writes_low()
      tpool::tpool_wait_end();
 }
 
-/** Waits until there are no pending writes. There can
-be other, synchronous, pending writes. */
+/** Wait until there are no pending asynchronous writes.
+Only used on FLUSH TABLES...FOR EXPORT. */
 void os_aio_wait_until_no_pending_writes()
 {
   os_aio_wait_until_no_pending_writes_low();

storage/innobase/row/row0import.cc

@@ -4234,6 +4234,16 @@ row_import_for_mysql(
 	of delete marked records that couldn't be purged in Phase I. */
 	while (buf_flush_dirty_pages(prebuilt->table->space_id));
 
+	for (ulint count = 0; prebuilt->table->space->referenced(); count++) {
+		/* Issue a warning every 10.24 seconds, starting after
+		2.56 seconds */
+		if ((count & 511) == 128) {
+			ib::warn() << "Waiting for flush to complete on "
+				   << prebuilt->table->name;
+		}
+		os_thread_sleep(20000);
+	}
+
 	ib::info() << "Phase IV - Flush complete";
 	prebuilt->table->space->set_imported();