MDEV-22556: Incorrect result for window function when using encrypt-t…

…mp-files=ON

The issue here is that end_of_file for encrypted temporary IO_CACHE (used by filesort) is updated
using lseek.
Encryption adds storage overhead and hides it from the caller by recalculating offsets and lengths.
Two different IO_CACHE cannot possibly modify the same file
because the encryption key is randomly generated and stored in the IO_CACHE.
So when the tempfiles are encrypted DO NOT use lseek to change end_of_file.

Further observations about updating end_of_file using lseek
1) The end_of_file update is only used for binlog index files
2) The whole point is to update file length when the file was modified via a different file descriptor.
3) The temporary IO_CACHE files can never be modified via a different file descriptor.
4) For encrypted temporary IO_CACHE, end_of_file should not be updated with lseek

mysql-test/suite/encryption/r/tempfiles_encrypted.result

@@ -0,0 +1,18 @@
+#
+# Tests when the temporary files are encrypted
+#
+select @@encrypt_tmp_files;
+@@encrypt_tmp_files
+1
+#
+# MDEV-22556: Incorrect result for window function when using encrypt-tmp-files=ON
+#
+set @save_sort_buffer_size=@@sort_buffer_size;
+set sort_buffer_size= 2000;
+create table t1( a DECIMAL(12,0) DEFAULT NULL, b VARCHAR(20) DEFAULT NULL, c DECIMAL(12,0) DEFAULT NULL)engine=INNODB;
+insert into t1 select seq, seq, seq from seq_1_to_5000;
+select count(*) from (select a, b, c, ROW_NUMBER() OVER (PARTITION BY a) FROM t1)q;
+count(*)
+5000
+set @@sort_buffer_size=@save_sort_buffer_size;
+drop table t1;

mysql-test/suite/encryption/t/tempfiles_encrypted.opt

@@ -0,0 +1 @@
+--encrypt-tmp_files=ON

mysql-test/suite/encryption/t/tempfiles_encrypted.test

@@ -0,0 +1,23 @@
+--echo #
+--echo # Tests when the temporary files are encrypted
+--echo #
+
+source include/have_file_key_management_plugin.inc;
+source include/have_sequence.inc;
+source include/have_innodb.inc;
+
+select @@encrypt_tmp_files;
+
+--echo #
+--echo # MDEV-22556: Incorrect result for window function when using encrypt-tmp-files=ON
+--echo #
+
+set @save_sort_buffer_size=@@sort_buffer_size;
+set sort_buffer_size= 2000;
+create table t1( a DECIMAL(12,0) DEFAULT NULL, b VARCHAR(20) DEFAULT NULL, c DECIMAL(12,0) DEFAULT NULL)engine=INNODB;
+insert into t1 select seq, seq, seq from seq_1_to_5000;
+select count(*) from (select a, b, c, ROW_NUMBER() OVER (PARTITION BY a) FROM t1)q;
+
+set @@sort_buffer_size=@save_sort_buffer_size;
+
+drop table t1;

mysys/mf_iocache.c

@@ -504,8 +504,11 @@ my_bool reinit_io_cache(IO_CACHE *info, enum cache_type type,
       if (info->type == WRITE_CACHE)
 	info->end_of_file= my_b_tell(info);
       else
-	info->end_of_file= mysql_file_seek(info->file, 0L, MY_SEEK_END,
-					   MYF(0));
+      {
+        if (!(info->myflags & MY_ENCRYPT))
+          info->end_of_file= mysql_file_seek(info->file, 0L,
+                                             MY_SEEK_END, MYF(0));
+      }
     }
     /* flush cache if we want to reuse it */
     if (!clear_cache && my_b_flush_io_cache(info,1))