Reduce the granularity of innodb_log_file_size

In Mariabackup, we would want the backed-up redo log file size to be
a multiple of 512 bytes, or OS_FILE_LOG_BLOCK_SIZE. However, at startup,
InnoDB would be picky, requiring the file size to be a multiple of
innodb_page_size.

Furthermore, InnoDB would require the parameter to be a multiple of
one megabyte, while the minimum granularity is 512 bytes. Because
the data-file-oriented fil_io() API is being used for writing the
InnoDB redo log, writes will for now require innodb_log_file_size to
be a multiple of the maximum innodb_page_size (65536 bytes).

To complicate matters, InnoDB startup divided srv_log_file_size by
UNIV_PAGE_SIZE, so that initially, the unit was bytes, and later it
was innodb_page_size. We will simplify this and keep srv_log_file_size
in bytes at all times.

innobase_log_file_size: Remove. Remove some obsolete checks against
overflow on 32-bit systems. srv_log_file_size is always 64 bits, and
the maximum size 512GiB in multiples of innodb_page_size always fits
in ulint (which is 32 or 64 bits). 512GiB would be 8,388,608*64KiB or
134,217,728*4KiB.

log_init(): Remove the parameter file_size that was always passed as
srv_log_file_size.

log_set_capacity(): Add a parameter for passing the requested file size.

srv_log_file_size_requested: Declare static in srv0start.cc.

create_log_file(), create_log_files(),
innobase_start_or_create_for_mysql(): Invoke fil_node_create()
with srv_log_file_size expressed in multiples of innodb_page_size.

innobase_start_or_create_for_mysql(): Require the redo log file sizes
to be multiples of 512 bytes.

mysql-test/suite/innodb/r/log_file.result

@@ -340,7 +340,7 @@ WHERE engine='innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 1
 1
-FOUND 1 /Resizing redo log from 1\*\d+ to 3\*\d+ pages; LSN=\d+/ in mysqld.1.err
+FOUND 1 /Resizing redo log from 1\*\d+ to 3\*\d+ bytes; LSN=\d+/ in mysqld.1.err
 # Cleanup
 bak_ib_logfile0
 bak_ib_logfile1

mysql-test/suite/innodb/r/log_file_size.result

@@ -33,16 +33,16 @@ ERROR 42000: Unknown storage engine 'InnoDB'
 FOUND 1 /InnoDB: innodb_read_only prevents crash recovery/ in mysqld.1.err
 SELECT * FROM t1;
 ERROR 42000: Unknown storage engine 'InnoDB'
-FOUND 2 /redo log from 3\*[0-9]+ to 2\*[0-9]+ pages/ in mysqld.1.err
+FOUND 2 /redo log from 3\*[0-9]+ to 2\*[0-9]+ bytes/ in mysqld.1.err
 SELECT * FROM t1;
 ERROR 42000: Unknown storage engine 'InnoDB'
-FOUND 3 /redo log from 3\*[0-9]+ to 2\*[0-9]+ pages/ in mysqld.1.err
+FOUND 3 /redo log from 3\*[0-9]+ to 2\*[0-9]+ bytes/ in mysqld.1.err
 SELECT * FROM t1;
 ERROR 42000: Unknown storage engine 'InnoDB'
 FOUND 2 /InnoDB: innodb_read_only prevents crash recovery/ in mysqld.1.err
 SELECT * FROM t1;
 ERROR 42000: Unknown storage engine 'InnoDB'
-FOUND 4 /redo log from 3\*[0-9]+ to 2\*[0-9]+ pages/ in mysqld.1.err
+FOUND 4 /redo log from 3\*[0-9]+ to 2\*[0-9]+ bytes/ in mysqld.1.err
 SELECT * FROM t1;
 ERROR 42000: Unknown storage engine 'InnoDB'
 SELECT * FROM t1;
@@ -56,7 +56,7 @@ ERROR 42000: Unknown storage engine 'InnoDB'
 FOUND 1 /InnoDB: Setting log file .*ib_logfile[0-9]+ size to/ in mysqld.1.err
 SELECT * FROM t1;
 ERROR 42000: Unknown storage engine 'InnoDB'
-FOUND 1 /InnoDB: Log file .*ib_logfile0 size 7 is not a multiple of innodb_page_size/ in mysqld.1.err
+FOUND 1 /InnoDB: Log file .*ib_logfile0 size 7 is not a multiple of 512 bytes/ in mysqld.1.err
 SELECT * FROM t1;
 ERROR 42000: Unknown storage engine 'InnoDB'
 FOUND 1 /InnoDB: Log file .*ib_logfile1 is of different size 1048576 bytes than other log files/ in mysqld.1.err

mysql-test/suite/innodb/t/log_file.test

@@ -218,7 +218,7 @@ eval $check_no_innodb;
 --source include/start_mysqld.inc
 eval $check_yes_innodb;
 --source include/shutdown_mysqld.inc
---let SEARCH_PATTERN=Resizing redo log from 1\*\d+ to 3\*\d+ pages; LSN=\d+
+--let SEARCH_PATTERN=Resizing redo log from 1\*\d+ to 3\*\d+ bytes; LSN=\d+
 --source include/search_pattern_in_file.inc
 
 --let $restart_parameters=

mysql-test/suite/innodb/t/log_file_size.test

@@ -100,14 +100,14 @@ let SEARCH_PATTERN= InnoDB: innodb_read_only prevents crash recovery;
 --source include/restart_mysqld.inc
 --error ER_UNKNOWN_STORAGE_ENGINE
 SELECT * FROM t1;
-let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
+let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ bytes;
 --source include/search_pattern_in_file.inc
 
 --let $restart_parameters= --debug=d,innodb_log_abort_5
 --source include/restart_mysqld.inc
 --error ER_UNKNOWN_STORAGE_ENGINE
 SELECT * FROM t1;
-let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
+let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ bytes;
 --source include/search_pattern_in_file.inc
 
 --let $restart_parameters= --innodb-read-only
@@ -122,7 +122,7 @@ let SEARCH_PATTERN= InnoDB: innodb_read_only prevents crash recovery;
 --error ER_UNKNOWN_STORAGE_ENGINE
 SELECT * FROM t1;
 
-let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
+let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ bytes;
 --source include/search_pattern_in_file.inc
 
 --let $restart_parameters= --debug=d,innodb_log_abort_7
@@ -170,7 +170,7 @@ EOF
 --source include/start_mysqld.inc
 --error ER_UNKNOWN_STORAGE_ENGINE
 SELECT * FROM t1;
-let SEARCH_PATTERN= InnoDB: Log file .*ib_logfile0 size 7 is not a multiple of innodb_page_size;
+let SEARCH_PATTERN= InnoDB: Log file .*ib_logfile0 size 7 is not a multiple of 512 bytes;
 --source include/search_pattern_in_file.inc
 --remove_file $MYSQLD_DATADIR/ib_logfile0
 --move_file $MYSQLD_DATADIR/ib_logfile101 $MYSQLD_DATADIR/ib_logfile0

storage/innobase/handler/ha_innodb.cc

@@ -212,7 +212,7 @@ static ulong innobase_commit_concurrency = 0;
 static ulong innobase_read_io_threads;
 static ulong innobase_write_io_threads;
 
-static long long innobase_buffer_pool_size, innobase_log_file_size;
+static long long innobase_buffer_pool_size;
 
 /** Percentage of the buffer pool to reserve for 'old' blocks.
 Connected to buf_LRU_old_ratio. */
@@ -4240,8 +4240,6 @@ innobase_init(
 
 	srv_file_flush_method_str = innobase_file_flush_method;
 
-	srv_log_file_size = (ib_uint64_t) innobase_log_file_size;
-
 	if (UNIV_PAGE_SIZE_DEF != srv_page_size) {
 		ib::info() << "innodb_page_size=" << srv_page_size;
 
@@ -21184,10 +21182,12 @@ static MYSQL_SYSVAR_LONG(log_buffer_size, innobase_log_buffer_size,
   "The size of the buffer which InnoDB uses to write log to the log files on disk.",
   NULL, NULL, 16*1024*1024L, 256*1024L, LONG_MAX, 1024);
 
-static MYSQL_SYSVAR_LONGLONG(log_file_size, innobase_log_file_size,
+static MYSQL_SYSVAR_ULONGLONG(log_file_size, srv_log_file_size,
   PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
   "Size of each log file in a log group.",
-  NULL, NULL, 48 << 20, 1 << 20, 512ULL << 30, 1 << 20);
+  NULL, NULL, 48 << 20, 1 << 20, 512ULL << 30, UNIV_PAGE_SIZE_MAX);
+/* OS_FILE_LOG_BLOCK_SIZE would be more appropriate than UNIV_PAGE_SIZE_MAX,
+but fil_space_t is being used for the redo log, and it uses data pages. */
 
 static MYSQL_SYSVAR_ULONG(log_files_in_group, srv_n_log_files,
   PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,

storage/innobase/include/log0log.h

@@ -156,17 +156,17 @@ void
 log_sys_init();
 
 /** Initialize the redo log.
-@param[in]	n_files		number of files
-@param[in]	file_size	file size in bytes */
+@param[in]	n_files		number of files */
 void
-log_init(ulint n_files, lsn_t file_size);
+log_init(ulint n_files);
 /** Calculate the recommended highest values for lsn - last_checkpoint_lsn
 and lsn - buf_get_oldest_modification().
+@param[in]	file_size	requested innodb_log_file_size
 @retval true on success
 @retval false if the smallest log group is too small to
 accommodate the number of OS threads in the database server */
 bool
-log_set_capacity()
+log_set_capacity(ulonglong file_size)
 	MY_ATTRIBUTE((warn_unused_result));
 
 /******************************************************//**

storage/innobase/include/srv0srv.h

@@ -360,17 +360,9 @@ extern char*	srv_log_group_home_dir;
 /** Maximum number of srv_n_log_files, or innodb_log_files_in_group */
 #define SRV_N_LOG_FILES_MAX 100
 extern ulong	srv_n_log_files;
-/** At startup, this is the current redo log file size.
-During startup, if this is different from srv_log_file_size_requested
-(innodb_log_file_size), the redo log will be rebuilt and this size
-will be initialized to srv_log_file_size_requested.
-When upgrading from a previous redo log format, this will be set to 0,
-and writing to the redo log is not allowed.
-
-During startup, this is in bytes, and later converted to pages. */
-extern ib_uint64_t	srv_log_file_size;
-/** The value of the startup parameter innodb_log_file_size */
-extern ib_uint64_t	srv_log_file_size_requested;
+/** The InnoDB redo log file size, or 0 when changing the redo log format
+at startup (while disallowing writes to the redo log). */
+extern ulonglong	srv_log_file_size;
 extern ulint	srv_log_buffer_size;
 extern ulong	srv_flush_log_at_trx_commit;
 extern uint	srv_flush_log_at_timeout;

storage/innobase/log/log0log.cc

@@ -669,18 +669,17 @@ log_group_set_fields(
 
 /** Calculate the recommended highest values for lsn - last_checkpoint_lsn
 and lsn - buf_get_oldest_modification().
+@param[in]	file_size	requested innodb_log_file_size
 @retval true on success
 @retval false if the smallest log group is too small to
 accommodate the number of OS threads in the database server */
 bool
-log_set_capacity()
+log_set_capacity(ulonglong file_size)
 {
 	lsn_t		margin;
 	ulint		free;
 
-	lsn_t smallest_capacity = ((srv_log_file_size_requested
-				    << srv_page_size_shift)
-				   - LOG_FILE_HDR_SIZE)
+	lsn_t smallest_capacity = (file_size - LOG_FILE_HDR_SIZE)
 		* srv_n_log_files;
 	/* Add extra safety */
 	smallest_capacity -= smallest_capacity / 10;
@@ -798,10 +797,9 @@ log_sys_init()
 }
 
 /** Initialize the redo log.
-@param[in]	n_files		number of files
-@param[in]	file_size	file size in bytes */
+@param[in]	n_files		number of files */
 void
-log_init(ulint n_files, lsn_t file_size)
+log_init(ulint n_files)
 {
 	ulint	i;
 	log_group_t*	group = &log_sys->log;
@@ -810,7 +808,7 @@ log_init(ulint n_files, lsn_t file_size)
 	group->format = srv_encrypt_log
 		? LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED
 		: LOG_HEADER_FORMAT_CURRENT;
-	group->file_size = file_size;
+	group->file_size = srv_log_file_size;
 	group->state = LOG_GROUP_OK;
 	group->lsn = LOG_START_LSN;
 	group->lsn_offset = LOG_FILE_HDR_SIZE;

storage/innobase/srv/srv0srv.cc

@@ -201,17 +201,9 @@ static os_event_t	srv_master_thread_disabled_event;
 char*	srv_log_group_home_dir;
 
 ulong	srv_n_log_files;
-/** At startup, this is the current redo log file size.
-During startup, if this is different from srv_log_file_size_requested
-(innodb_log_file_size), the redo log will be rebuilt and this size
-will be initialized to srv_log_file_size_requested.
-When upgrading from a previous redo log format, this will be set to 0,
-and writing to the redo log is not allowed.
-
-During startup, this is in bytes, and later converted to pages. */
-ib_uint64_t	srv_log_file_size;
-/** The value of the startup parameter innodb_log_file_size */
-ib_uint64_t	srv_log_file_size_requested;
+/** The InnoDB redo log file size, or 0 when changing the redo log format
+at startup (while disallowing writes to the redo log). */
+ulonglong	srv_log_file_size;
 /** copy of innodb_log_buffer_size, but in database pages */
 ulint		srv_log_buffer_size;
 /** innodb_flush_log_at_trx_commit */
@@ -1161,8 +1153,6 @@ srv_normalize_init_values(void)
 
 	srv_tmp_space.normalize();
 
-	srv_log_file_size /= UNIV_PAGE_SIZE;
-
 	srv_log_buffer_size /= UNIV_PAGE_SIZE;
 
 	srv_lock_table_size = 5 * (srv_buf_pool_size / UNIV_PAGE_SIZE);

storage/innobase/srv/srv0start.cc

@@ -138,6 +138,8 @@ bool	srv_is_being_started;
 bool	srv_sys_tablespaces_open;
 /** TRUE if the server was successfully started */
 bool	srv_was_started;
+/** The original value of srv_log_file_size (innodb_log_file_size) */
+static ulonglong	srv_log_file_size_requested;
 /** TRUE if innobase_start_or_create_for_mysql() has been called */
 static bool		srv_start_has_been_called;
 
@@ -374,17 +376,13 @@ create_log_file(
 	}
 
 	ib::info() << "Setting log file " << name << " size to "
-		<< (srv_log_file_size >> (20 - UNIV_PAGE_SIZE_SHIFT))
-		<< " MB";
+		<< srv_log_file_size << " bytes";
 
-	ret = os_file_set_size(name, *file,
-			       (os_offset_t) srv_log_file_size
-			       << UNIV_PAGE_SIZE_SHIFT,
+	ret = os_file_set_size(name, *file, srv_log_file_size,
 			       srv_read_only_mode);
 	if (!ret) {
-		ib::error() << "Cannot set log file " << name << " to size "
-			<< (srv_log_file_size >> (20 - UNIV_PAGE_SIZE_SHIFT))
-			<< " MB";
+		ib::error() << "Cannot set log file " << name << " size to "
+			<< srv_log_file_size << " bytes";
 		return(DB_ERROR);
 	}
 
@@ -467,17 +465,17 @@ create_log_files(
 	ut_a(fil_validate());
 	ut_a(log_space != NULL);
 
+	const ulint size = ulint(srv_log_file_size >> srv_page_size_shift);
+
 	logfile0 = fil_node_create(
-		logfilename, (ulint) srv_log_file_size,
-		log_space, false, false);
+		logfilename, size, log_space, false, false);
 	ut_a(logfile0);
 
 	for (unsigned i = 1; i < srv_n_log_files; i++) {
 
 		sprintf(logfilename + dirnamelen, "ib_logfile%u", i);
 
-		if (!fil_node_create(logfilename,
-				     (ulint) srv_log_file_size,
+		if (!fil_node_create(logfilename, size,
 				     log_space, false, false)) {
 
 			ib::error()
@@ -488,8 +486,8 @@ create_log_files(
 		}
 	}
 
-	log_init(srv_n_log_files, srv_log_file_size * UNIV_PAGE_SIZE);
-	if (!log_set_capacity()) {
+	log_init(srv_n_log_files);
+	if (!log_set_capacity(srv_log_file_size_requested)) {
 		return(DB_ERROR);
 	}
 
@@ -1410,7 +1408,7 @@ srv_prepare_to_delete_redo_log_files(
 
 			info << srv_n_log_files << "*"
 			     << srv_log_file_size_requested
-			     << " pages; LSN=" << flushed_lsn;
+			     << " bytes; LSN=" << flushed_lsn;
 		}
 
 		/* Flush the old log files. */
@@ -1863,8 +1861,7 @@ innobase_start_or_create_for_mysql()
 		srv_start_state_set(SRV_START_STATE_IO);
 	}
 
-	if (srv_n_log_files * srv_log_file_size * UNIV_PAGE_SIZE
-	    >= 512ULL * 1024ULL * 1024ULL * 1024ULL) {
+	if (srv_n_log_files * srv_log_file_size >= 512ULL << 30) {
 		/* log_block_convert_lsn_to_no() limits the returned block
 		number to 1G and given that OS_FILE_LOG_BLOCK_SIZE is 512
 		bytes, then we have a limit of 512 GB. If that limit is to
@@ -1875,18 +1872,8 @@ innobase_start_or_create_for_mysql()
 		return(srv_init_abort(DB_ERROR));
 	}
 
-	if (srv_n_log_files * srv_log_file_size >= ULINT_MAX) {
-		/* fil_io() takes ulint as an argument and we are passing
-		(next_offset / UNIV_PAGE_SIZE) to it in log_group_write_buf().
-		So (next_offset / UNIV_PAGE_SIZE) must be less than ULINT_MAX.
-		So next_offset must be < ULINT_MAX * UNIV_PAGE_SIZE. This
-		means that we are limited to ULINT_MAX * UNIV_PAGE_SIZE which
-		is 64 TB on 32 bit systems. */
-		ib::error() << "Combined size of log files must be < "
-			<< ULINT_MAX / 1073741824 * UNIV_PAGE_SIZE << " GB";
-
-		return(srv_init_abort(DB_ERROR));
-	}
+	compile_time_assert(ulonglong(ULINT_MAX) * UNIV_PAGE_SIZE_MIN
+			    >= 512ULL << 30);
 
 	os_normalize_path(srv_data_home);
 
@@ -2025,27 +2012,22 @@ innobase_start_or_create_for_mysql()
 
 			ut_a(size != (os_offset_t) -1);
 
-			if (size & ((1 << UNIV_PAGE_SIZE_SHIFT) - 1)) {
+			if (size & (OS_FILE_LOG_BLOCK_SIZE - 1)) {
 
 				ib::error() << "Log file " << logfilename
 					<< " size " << size << " is not a"
-					" multiple of innodb_page_size";
+					" multiple of 512 bytes";
 				return(srv_init_abort(DB_ERROR));
 			}
 
-			size >>= UNIV_PAGE_SIZE_SHIFT;
-
 			if (i == 0) {
 				srv_log_file_size = size;
 			} else if (size != srv_log_file_size) {
 
 				ib::error() << "Log file " << logfilename
-					<< " is of different size "
-					<< (size << UNIV_PAGE_SIZE_SHIFT)
+					<< " is of different size " << size
 					<< " bytes than other log files "
-					<< (srv_log_file_size
-					    << UNIV_PAGE_SIZE_SHIFT)
-					<< " bytes!";
+					<< srv_log_file_size << " bytes!";
 				return(srv_init_abort(DB_ERROR));
 			}
 		}
@@ -2066,23 +2048,23 @@ innobase_start_or_create_for_mysql()
 		ut_a(fil_validate());
 		ut_a(log_space);
 
-		/* srv_log_file_size is measured in pages; if page size is 16KB,
-		then we have a limit of 64TB on 32 bit systems */
-		ut_a(srv_log_file_size <= ULINT_MAX);
+		ut_a(srv_log_file_size <= 512ULL << 30);
+
+		const ulint size = 1 + ulint((srv_log_file_size - 1)
+					     >> srv_page_size_shift);
 
-		for (unsigned j = 0; j < i; j++) {
+		for (unsigned j = 0; j < srv_n_log_files_found; j++) {
 			sprintf(logfilename + dirnamelen, "ib_logfile%u", j);
 
-			if (!fil_node_create(logfilename,
-					     (ulint) srv_log_file_size,
+			if (!fil_node_create(logfilename, size,
 					     log_space, false, false)) {
 				return(srv_init_abort(DB_ERROR));
 			}
 		}
 
-		log_init(i, srv_log_file_size * UNIV_PAGE_SIZE);
+		log_init(srv_n_log_files_found);
 
-		if (!log_set_capacity()) {
+		if (!log_set_capacity(srv_log_file_size_requested)) {
 			return(srv_init_abort(DB_ERROR));
 		}
 	}