MDEV-25998 InnoDB removes the tablespace from default encrypt list early

Problem:
=========
As a part of MDEV-14398 patch, InnoDB added and removed
the tablespace from default encrypt list. But InnoDB removes
the tablespace from the default encrypt list too early due to
i) other encryption thread working on the tablespace
ii) When tablespace is being flushed at the end of
key rotation

InnoDB fails to decrypt/encrypt the tablespace since
the tablespace removed too early and it leads to
test case failure.

Solution:
=========
Avoid the removal of tablespace from default_encrypt_list
only when
1) Another active encryption thread working on tablespace
2) Eligible for tablespace key rotation
3) Tablespace is in flushing phase

Removed the workaround in encryption.innodb_encryption_filekeys test case.

mysql-test/suite/encryption/t/innodb_encryption_filekeys.test

@@ -48,12 +48,6 @@ while ($cnt)
     {
         real_sleep 1;
         dec $cnt;
-        if ($cnt == 200)
-	{
-	  --disable_query_log
-	  set global innodb_encrypt_tables = on;
-	  --enable_query_log
-	}
     }
 }
 if (!$success)
@@ -84,12 +78,6 @@ while ($cnt)
     {
         real_sleep 1;
         dec $cnt;
-        if ($cnt == 200)
-	{
-	  --disable_query_log
-	  set global innodb_encrypt_tables = off;
-	  --enable_query_log
-	}
     }
 }
 if (!$success)
@@ -119,12 +107,6 @@ while ($cnt)
     {
         real_sleep 1;
         dec $cnt;
-        if ($cnt == 200)
-	{
-	  --disable_query_log
-	  set global innodb_encrypt_tables=on;
-	  --enable_query_log
-	}
     }
 }
 if (!$success)

storage/innobase/fil/fil0crypt.cc

@@ -1092,6 +1092,33 @@ struct rotate_thread_t {
 	}
 };
 
+/** Avoid the removal of the tablespace from
+default_encrypt_list only when
+1) Another active encryption thread working on tablespace
+2) Eligible for tablespace key rotation
+3) Tablespace is in flushing phase
+@return true if tablespace should be removed from
+default encrypt */
+static bool fil_crypt_must_remove(const fil_space_t &space)
+{
+  ut_ad(space.purpose == FIL_TYPE_TABLESPACE);
+  fil_space_crypt_t *crypt_data = space.crypt_data;
+  mutex_own(&fil_system->mutex);
+  const ulong encrypt_tables= srv_encrypt_tables;
+  if (!crypt_data)
+    return !encrypt_tables;
+  if (!crypt_data->is_key_found())
+    return true;
+
+  mutex_enter(&crypt_data->mutex);
+  const bool remove= (space.is_stopping() || crypt_data->not_encrypted()) &&
+    (!crypt_data->rotate_state.flushing &&
+     !encrypt_tables == !!crypt_data->min_key_version &&
+     !crypt_data->rotate_state.active_threads);
+  mutex_exit(&crypt_data->mutex);
+  return remove;
+}
+
 /***********************************************************************
 Check if space needs rotation given a key_state
 @param[in,out]		state		Key rotation state
@@ -1429,8 +1456,7 @@ inline fil_space_t *fil_system_t::default_encrypt_next(
       If there is a change in innodb_encrypt_tables variables
       value then don't remove the last processed tablespace
       from the default encrypt list. */
-      if (released && (!recheck || space->crypt_data) &&
-          !encrypt == !srv_encrypt_tables)
+      if (released && !recheck && fil_crypt_must_remove(*space))
       {
         ut_a(!default_encrypt_tables.empty());
         default_encrypt_tables.remove(*space);