MDEV-37263 Hang or crash when shrinking innodb_buffer_pool_size

buf_pool_t::shrink(): If we run out of pages to evict from buf_pool.LRU,
abort the operation. Also, do not leak the spare block that we may have
allocated.

Author: dr-m

mysql-test/suite/innodb/r/innodb_buffer_pool_shrink.result

@@ -0,0 +1,11 @@
+call mtr.add_suppression("innodb_buffer_pool_size change aborted");
+CREATE TABLE t (c INT) ENGINE=InnoDB PARTITION BY HASH(c) PARTITIONS 512;
+BEGIN;
+SELECT * FROM t LOCK IN SHARE MODE;
+c
+SET @save_size = @@GLOBAL.innodb_buffer_pool_size;
+SET GLOBAL innodb_buffer_pool_size=6291456;
+COMMIT;
+SET GLOBAL innodb_buffer_pool_size=6291456;
+SET GLOBAL innodb_buffer_pool_size = @save_size;
+DROP TABLE t;

mysql-test/suite/innodb/t/innodb_buffer_pool_shrink.test

@@ -0,0 +1,14 @@
+--source include/have_innodb.inc
+--source include/have_partition.inc
+call mtr.add_suppression("innodb_buffer_pool_size change aborted");
+CREATE TABLE t (c INT) ENGINE=InnoDB PARTITION BY HASH(c) PARTITIONS 512;
+BEGIN;
+SELECT * FROM t LOCK IN SHARE MODE;
+SET @save_size = @@GLOBAL.innodb_buffer_pool_size;
+--error 0,ER_WRONG_USAGE
+SET GLOBAL innodb_buffer_pool_size=6291456;
+COMMIT;
+--error 0,ER_WRONG_USAGE
+SET GLOBAL innodb_buffer_pool_size=6291456;
+SET GLOBAL innodb_buffer_pool_size = @save_size;
+DROP TABLE t;

storage/innobase/buf/buf0buf.cc

@@ -1823,6 +1823,12 @@ ATTRIBUTE_COLD buf_pool_t::shrink_status buf_pool_t::shrink(size_t size)
     goto next;
   }
 
+  if (block)
+    buf_LRU_block_free_non_file_page(block);
+
+  if (!UT_LIST_GET_LEN(LRU) && n_blocks_to_withdraw)
+    return SHRINK_ABORT;
+
   if (UT_LIST_GET_LEN(free) + UT_LIST_GET_LEN(LRU) < usable_size() / 20)
     return SHRINK_ABORT;