InnoDB wastes 62 of every 16,384 pages in XDES/IBUF_BITMAP extent

The problem is that once the segments of a tablespace are bigger than
32 pages, fragment pages are no longer allocated for use, yet they are
still reserved whenever a new fragment extent is allocated. This is a
direct consequence of mainly two facts: whenever a new descriptor page
is needed (every 16384 pages), the extent that contains the descriptor
page cannot be assigned to a segment and is instead used as a fragment
extent; and a segment can only allocate up to 32 fragment pages since
the array used to track fragment pages belonging to a segment is limited
to 32 entries per segment.

The solution is to allow for fragment extents to be leased to segments
whenever there are free fragment extents available. A fragment extent
is considered available if the only used pages in the extent are the
extent descriptor and ibuf bitmap pages. A new extent state is used to
tag leased extents and to ensure that they are returned to the space
free fragment list once no longer being used by a segment.

Additionally, a new system variable named innodb_lease_fragment_extents
is introduced to control whether free fragment extents are allocated to
segments.

This is an incompatible change. Once a fragment extent is allocated to
a segment, the table that contains the segment is no longer compatible
with earlier MySQL versions.

Also, the two reserved pages per leased extent are counted towards the
size of the segment.  This means that index and table size information
provided by statements such as SHOW TABLE STATUS will include the size
of the two additional pages per leased extent.

client/mysqltest.cc

@@ -3052,6 +3052,8 @@ void do_exec(struct st_command *command)
           command->first_argument, ds_res.str);
     }
 
+    var_set_int("$exec_exit_status", status);
+
     DBUG_PRINT("info",
                ("error: %d, status: %d", error, status));
 
@@ -3078,6 +3080,10 @@ void do_exec(struct st_command *command)
     die("command \"%s\" succeeded - should have failed with errno %d...",
         command->first_argument, command->expected_errors.err[0].code.errnum);
   }
+  else
+  {
+    var_set_int("$exec_exit_status", error);
+  }
 
   dynstr_free(&ds_cmd);
   DBUG_VOID_RETURN;

mysql-test/include/have_innodb_ruby.inc

@@ -0,0 +1,11 @@
+disable_query_log;
+disable_result_log;
+disable_abort_on_error;
+exec innodb_space --help;
+enable_abort_on_error;
+if ($exec_exit_status != 0)
+{
+  skip Test requires innodb_ruby;
+}
+enable_result_log;
+enable_query_log;

mysql-test/suite/innodb/r/innodb_fragment_extent.result

@@ -0,0 +1,60 @@
+#
+# Test that fragment extents can be fully used.
+#
+
+# Setup.
+SET @old_innodb_file_per_table = @@GLOBAL.innodb_file_per_table;
+SET @old_innodb_lease_fragment_extents = @@GLOBAL.innodb_lease_fragment_extents;
+SET GLOBAL innodb_file_per_table = ON;
+SET GLOBAL innodb_lease_fragment_extents = ON;
+CREATE VIEW buffer_pool AS
+SELECT CONVERT(VARIABLE_VALUE, UNSIGNED) AS PAGES_DIRTY FROM
+INFORMATION_SCHEMA.GLOBAL_STATUS WHERE
+VARIABLE_NAME = 'INNODB_BUFFER_POOL_PAGES_DIRTY';
+CREATE TABLE t1 (a BIGINT PRIMARY KEY, b VARCHAR(1024), c VARCHAR(1024))
+ENGINE=InnoDB;
+CREATE PROCEDURE p1(k BIGINT, c BIGINT)
+BEGIN
+SET autocommit = OFF;
+WHILE c > 0 DO
+INSERT INTO t1 VALUES (k, REPEAT('b', 1024), REPEAT('c', 1024));
+IF (k MOD 1024 = 0) THEN COMMIT; END IF;
+SET k = k + 1;
+SET c = c - 1;
+END WHILE;
+SET autocommit = ON;
+END|
+# Populate table so that more than 16384 pages are used.
+CALL p1(0, 130000);
+# The free_frag list should now only contain one extent.
+start_page  bitmap                                                          
+0           ##########################################################......
+# Show that the fragment extent was leased and its pages used.
+start       end         count       type                
+0           0           1           FSP_HDR             
+1           1           1           IBUF_BITMAP         
+2           2           1           INODE               
+3           57          55          INDEX               
+58          63          6           ALLOCATED           
+64          16383       16320       INDEX               
+16384       16384       1           XDES                
+16385       16385       1           IBUF_BITMAP         
+16386       18605       2220        INDEX               
+18606       19455       850         ALLOCATED           
+# Delete rows so that the fragment extent becomes unused.
+DELETE FROM t1 WHERE a >= 114400;
+# Show that the fragment extent is returned to the free_frag list.
+start_page  bitmap                                                          
+0           ########################################################........
+16384       ##..............................................................
+# Repopulate table.
+CALL p1(114400, 15600);
+# Show that the fragment extent can be reused.
+start_page  bitmap                                                          
+0           ##########################################################......
+# Cleanup.
+DROP TABLE t1;
+DROP PROCEDURE p1;
+DROP VIEW buffer_pool;
+SET @@GLOBAL.innodb_file_per_table = @old_innodb_file_per_table;
+SET @@GLOBAL.innodb_lease_fragment_extents = @old_innodb_lease_fragment_extents;

mysql-test/suite/innodb/t/innodb_fragment_extent.test

@@ -0,0 +1,81 @@
+--source include/have_innodb.inc
+--source include/have_innodb_ruby.inc
+
+--echo #
+--echo # Test that fragment extents can be fully used.
+--echo #
+--echo
+
+--echo # Setup.
+SET @old_innodb_file_per_table = @@GLOBAL.innodb_file_per_table;
+SET @old_innodb_lease_fragment_extents = @@GLOBAL.innodb_lease_fragment_extents;
+
+let $MYSQLD_DATADIR= `select @@datadir`;
+
+SET GLOBAL innodb_file_per_table = ON;
+SET GLOBAL innodb_lease_fragment_extents = ON;
+
+CREATE VIEW buffer_pool AS
+  SELECT CONVERT(VARIABLE_VALUE, UNSIGNED) AS PAGES_DIRTY FROM
+    INFORMATION_SCHEMA.GLOBAL_STATUS WHERE
+      VARIABLE_NAME = 'INNODB_BUFFER_POOL_PAGES_DIRTY';
+
+CREATE TABLE t1 (a BIGINT PRIMARY KEY, b VARCHAR(1024), c VARCHAR(1024))
+  ENGINE=InnoDB;
+
+delimiter |;
+
+CREATE PROCEDURE p1(k BIGINT, c BIGINT)
+BEGIN
+  SET autocommit = OFF;
+  WHILE c > 0 DO
+    INSERT INTO t1 VALUES (k, REPEAT('b', 1024), REPEAT('c', 1024));
+    IF (k MOD 1024 = 0) THEN COMMIT; END IF;
+    SET k = k + 1;
+    SET c = c - 1;
+  END WHILE;
+  SET autocommit = ON;
+END|
+
+delimiter ;|
+
+--echo # Populate table so that more than 16384 pages are used.
+CALL p1(0, 130000);
+
+# Wait for dirty pages to be flushed to disk.
+let $wait_condition = SELECT PAGES_DIRTY = 0 FROM buffer_pool;
+--source include/wait_condition.inc
+
+--echo # The free_frag list should now only contain one extent.
+--exec innodb_space -f $MYSQLD_DATADIR/test/t1.ibd list-summary -L free_frag
+
+--echo # Show that the fragment extent was leased and its pages used.
+--exec innodb_space -f $MYSQLD_DATADIR/test/t1.ibd space-page-type-regions
+
+--echo # Delete rows so that the fragment extent becomes unused.
+DELETE FROM t1 WHERE a >= 114400;
+
+# Wait for dirty pages to be flushed to disk.
+let $wait_condition = SELECT PAGES_DIRTY = 0 FROM buffer_pool;
+--source include/wait_condition.inc
+
+--echo # Show that the fragment extent is returned to the free_frag list.
+--exec innodb_space -f $MYSQLD_DATADIR/test/t1.ibd list-summary -L free_frag
+
+--echo # Repopulate table.
+CALL p1(114400, 15600);
+
+# Wait for dirty pages to be flushed to disk.
+let $wait_condition = SELECT PAGES_DIRTY = 0 FROM buffer_pool;
+--source include/wait_condition.inc
+
+--echo # Show that the fragment extent can be reused.
+--exec innodb_space -f $MYSQLD_DATADIR/test/t1.ibd list-summary -L free_frag
+
+--echo # Cleanup.
+DROP TABLE t1;
+DROP PROCEDURE p1;
+DROP VIEW buffer_pool;
+
+SET @@GLOBAL.innodb_file_per_table = @old_innodb_file_per_table;
+SET @@GLOBAL.innodb_lease_fragment_extents = @old_innodb_lease_fragment_extents;

mysql-test/suite/sys_vars/r/innodb_lease_fragment_extents_basic.result

@@ -0,0 +1,92 @@
+SET @start_global_value = @@global.innodb_lease_fragment_extents;
+SELECT @start_global_value;
+@start_global_value
+0
+Valid values are 'ON' and 'OFF'
+SELECT @@GLOBAL.innodb_lease_fragment_extents IN (0, 1);
+@@GLOBAL.innodb_lease_fragment_extents IN (0, 1)
+1
+SELECT @@GLOBAL.innodb_lease_fragment_extents;
+@@GLOBAL.innodb_lease_fragment_extents
+0
+SELECT @@SESSION.innodb_lease_fragment_extents;
+ERROR HY000: Variable 'innodb_lease_fragment_extents' is a GLOBAL variable
+SHOW GLOBAL VARIABLES LIKE 'innodb_lease_fragment_extents';
+Variable_name	Value
+innodb_lease_fragment_extents	OFF
+SHOW SESSION VARIABLES LIKE 'innodb_lease_fragment_extents';
+Variable_name	Value
+innodb_lease_fragment_extents	OFF
+SELECT * FROM information_schema.global_variables WHERE variable_name='innodb_lease_fragment_extents';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_LEASE_FRAGMENT_EXTENTS	OFF
+SELECT * FROM information_schema.session_variables WHERE variable_name='innodb_lease_fragment_extents';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_LEASE_FRAGMENT_EXTENTS	OFF
+SET GLOBAL innodb_lease_fragment_extents='OFF';
+SELECT @@GLOBAL.innodb_lease_fragment_extents;
+@@GLOBAL.innodb_lease_fragment_extents
+0
+SELECT * FROM information_schema.global_variables WHERE variable_name='innodb_lease_fragment_extents';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_LEASE_FRAGMENT_EXTENTS	OFF
+SELECT * FROM information_schema.session_variables WHERE variable_name='innodb_lease_fragment_extents';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_LEASE_FRAGMENT_EXTENTS	OFF
+SET @@GLOBAL.innodb_lease_fragment_extents=1;
+SELECT @@GLOBAL.innodb_lease_fragment_extents;
+@@GLOBAL.innodb_lease_fragment_extents
+1
+SELECT * FROM information_schema.global_variables WHERE variable_name='innodb_lease_fragment_extents';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_LEASE_FRAGMENT_EXTENTS	ON
+SELECT * FROM information_schema.session_variables WHERE variable_name='innodb_lease_fragment_extents';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_LEASE_FRAGMENT_EXTENTS	ON
+SET GLOBAL innodb_lease_fragment_extents=0;
+SELECT @@GLOBAL.innodb_lease_fragment_extents;
+@@GLOBAL.innodb_lease_fragment_extents
+0
+SELECT * FROM information_schema.global_variables WHERE variable_name='innodb_lease_fragment_extents';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_LEASE_FRAGMENT_EXTENTS	OFF
+SELECT * FROM information_schema.session_variables WHERE variable_name='innodb_lease_fragment_extents';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_LEASE_FRAGMENT_EXTENTS	OFF
+SET @@GLOBAL.innodb_lease_fragment_extents='ON';
+SELECT @@GLOBAL.innodb_lease_fragment_extents;
+@@GLOBAL.innodb_lease_fragment_extents
+1
+SELECT * FROM information_schema.global_variables WHERE variable_name='innodb_lease_fragment_extents';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_LEASE_FRAGMENT_EXTENTS	ON
+SELECT * FROM information_schema.session_variables WHERE variable_name='innodb_lease_fragment_extents';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_LEASE_FRAGMENT_EXTENTS	ON
+SET SESSION innodb_lease_fragment_extents='OFF';
+ERROR HY000: Variable 'innodb_lease_fragment_extents' is a GLOBAL variable and should be set with SET GLOBAL
+SET @@SESSION.innodb_lease_fragment_extents='ON';
+ERROR HY000: Variable 'innodb_lease_fragment_extents' is a GLOBAL variable and should be set with SET GLOBAL
+SET GLOBAL innodb_lease_fragment_extents=1.1;
+ERROR 42000: Incorrect argument type to variable 'innodb_lease_fragment_extents'
+SET GLOBAL innodb_lease_fragment_extents=1e1;
+ERROR 42000: Incorrect argument type to variable 'innodb_lease_fragment_extents'
+SET GLOBAL innodb_lease_fragment_extents=2;
+ERROR 42000: Variable 'innodb_lease_fragment_extents' can't be set to the value of '2'
+NOTE: The following should fail with ER_WRONG_VALUE_FOR_VAR (BUG#50643)
+SET GLOBAL innodb_lease_fragment_extents=-3;
+SELECT @@GLOBAL.innodb_lease_fragment_extents;
+@@GLOBAL.innodb_lease_fragment_extents
+1
+SELECT * FROM information_schema.global_variables WHERE variable_name='innodb_lease_fragment_extents';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_LEASE_FRAGMENT_EXTENTS	ON
+SELECT * FROM information_schema.session_variables WHERE variable_name='innodb_lease_fragment_extents';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_LEASE_FRAGMENT_EXTENTS	ON
+SET GLOBAL innodb_lease_fragment_extents='AUTO';
+ERROR 42000: Variable 'innodb_lease_fragment_extents' can't be set to the value of 'AUTO'
+SET @@GLOBAL.innodb_lease_fragment_extents = @start_global_value;
+SELECT @@GLOBAL.innodb_lease_fragment_extents;
+@@GLOBAL.innodb_lease_fragment_extents
+0

mysql-test/suite/sys_vars/t/innodb_lease_fragment_extents_basic.test

@@ -0,0 +1,65 @@
+--source include/have_innodb.inc
+
+SET @start_global_value = @@global.innodb_lease_fragment_extents;
+SELECT @start_global_value;
+
+#
+# Global scope only
+#
+--echo Valid values are 'ON' and 'OFF'
+SELECT @@GLOBAL.innodb_lease_fragment_extents IN (0, 1);
+SELECT @@GLOBAL.innodb_lease_fragment_extents;
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT @@SESSION.innodb_lease_fragment_extents;
+SHOW GLOBAL VARIABLES LIKE 'innodb_lease_fragment_extents';
+SHOW SESSION VARIABLES LIKE 'innodb_lease_fragment_extents';
+SELECT * FROM information_schema.global_variables WHERE variable_name='innodb_lease_fragment_extents';
+SELECT * FROM information_schema.session_variables WHERE variable_name='innodb_lease_fragment_extents';
+
+#
+# Read-write variable
+#
+SET GLOBAL innodb_lease_fragment_extents='OFF';
+SELECT @@GLOBAL.innodb_lease_fragment_extents;
+SELECT * FROM information_schema.global_variables WHERE variable_name='innodb_lease_fragment_extents';
+SELECT * FROM information_schema.session_variables WHERE variable_name='innodb_lease_fragment_extents';
+SET @@GLOBAL.innodb_lease_fragment_extents=1;
+SELECT @@GLOBAL.innodb_lease_fragment_extents;
+SELECT * FROM information_schema.global_variables WHERE variable_name='innodb_lease_fragment_extents';
+SELECT * FROM information_schema.session_variables WHERE variable_name='innodb_lease_fragment_extents';
+SET GLOBAL innodb_lease_fragment_extents=0;
+SELECT @@GLOBAL.innodb_lease_fragment_extents;
+SELECT * FROM information_schema.global_variables WHERE variable_name='innodb_lease_fragment_extents';
+SELECT * FROM information_schema.session_variables WHERE variable_name='innodb_lease_fragment_extents';
+SET @@GLOBAL.innodb_lease_fragment_extents='ON';
+SELECT @@GLOBAL.innodb_lease_fragment_extents;
+SELECT * FROM information_schema.global_variables WHERE variable_name='innodb_lease_fragment_extents';
+SELECT * FROM information_schema.session_variables WHERE variable_name='innodb_lease_fragment_extents';
+--error ER_GLOBAL_VARIABLE
+SET SESSION innodb_lease_fragment_extents='OFF';
+--error ER_GLOBAL_VARIABLE
+SET @@SESSION.innodb_lease_fragment_extents='ON';
+
+#
+# Boolean type.
+#
+--error ER_WRONG_TYPE_FOR_VAR
+SET GLOBAL innodb_lease_fragment_extents=1.1;
+--error ER_WRONG_TYPE_FOR_VAR
+SET GLOBAL innodb_lease_fragment_extents=1e1;
+--error ER_WRONG_VALUE_FOR_VAR
+SET GLOBAL innodb_lease_fragment_extents=2;
+--echo NOTE: The following should fail with ER_WRONG_VALUE_FOR_VAR (BUG#50643)
+SET GLOBAL innodb_lease_fragment_extents=-3;
+SELECT @@GLOBAL.innodb_lease_fragment_extents;
+SELECT * FROM information_schema.global_variables WHERE variable_name='innodb_lease_fragment_extents';
+SELECT * FROM information_schema.session_variables WHERE variable_name='innodb_lease_fragment_extents';
+--error ER_WRONG_VALUE_FOR_VAR
+SET GLOBAL innodb_lease_fragment_extents='AUTO';
+
+#
+# Cleanup
+#
+
+SET @@GLOBAL.innodb_lease_fragment_extents = @start_global_value;
+SELECT @@GLOBAL.innodb_lease_fragment_extents;