Skip to content
Permalink
Browse files

Reduce the granularity of innodb_log_file_size

In Mariabackup, we would want the backed-up redo log file size to be
a multiple of 512 bytes, or OS_FILE_LOG_BLOCK_SIZE. However, at startup,
InnoDB would be picky, requiring the file size to be a multiple of
innodb_page_size.

Furthermore, InnoDB would require the parameter to be a multiple of
one megabyte, while the minimum granularity is 512 bytes. Because
the data-file-oriented fil_io() API is being used for writing the
InnoDB redo log, writes will for now require innodb_log_file_size to
be a multiple of the maximum innodb_page_size (65536 bytes).

To complicate matters, InnoDB startup divided srv_log_file_size by
UNIV_PAGE_SIZE, so that initially, the unit was bytes, and later it
was innodb_page_size. We will simplify this and keep srv_log_file_size
in bytes at all times.

innobase_log_file_size: Remove. Remove some obsolete checks against
overflow on 32-bit systems. srv_log_file_size is always 64 bits, and
the maximum size 512GiB in multiples of innodb_page_size always fits
in ulint (which is 32 or 64 bits). 512GiB would be 8,388,608*64KiB or
134,217,728*4KiB.

log_init(): Remove the parameter file_size that was always passed as
srv_log_file_size.

log_set_capacity(): Add a parameter for passing the requested file size.

srv_log_file_size_requested: Declare static in srv0start.cc.

create_log_file(), create_log_files(),
innobase_start_or_create_for_mysql(): Invoke fil_node_create()
with srv_log_file_size expressed in multiples of innodb_page_size.

innobase_start_or_create_for_mysql(): Require the redo log file sizes
to be multiples of 512 bytes.
  • Loading branch information...
dr-m committed Jun 5, 2017
1 parent e903d45 commit 84e4e4506ffee95d71f1cac916176e9b64ba1bd8
@@ -340,7 +340,7 @@ WHERE engine='innodb'
AND support IN ('YES', 'DEFAULT', 'ENABLED');
1
1
FOUND 1 /Resizing redo log from 1\*\d+ to 3\*\d+ pages; LSN=\d+/ in mysqld.1.err
FOUND 1 /Resizing redo log from 1\*\d+ to 3\*\d+ bytes; LSN=\d+/ in mysqld.1.err
# Cleanup
bak_ib_logfile0
bak_ib_logfile1
@@ -33,16 +33,16 @@ ERROR 42000: Unknown storage engine 'InnoDB'
FOUND 1 /InnoDB: innodb_read_only prevents crash recovery/ in mysqld.1.err
SELECT * FROM t1;
ERROR 42000: Unknown storage engine 'InnoDB'
FOUND 2 /redo log from 3\*[0-9]+ to 2\*[0-9]+ pages/ in mysqld.1.err
FOUND 2 /redo log from 3\*[0-9]+ to 2\*[0-9]+ bytes/ in mysqld.1.err
SELECT * FROM t1;
ERROR 42000: Unknown storage engine 'InnoDB'
FOUND 3 /redo log from 3\*[0-9]+ to 2\*[0-9]+ pages/ in mysqld.1.err
FOUND 3 /redo log from 3\*[0-9]+ to 2\*[0-9]+ bytes/ in mysqld.1.err
SELECT * FROM t1;
ERROR 42000: Unknown storage engine 'InnoDB'
FOUND 2 /InnoDB: innodb_read_only prevents crash recovery/ in mysqld.1.err
SELECT * FROM t1;
ERROR 42000: Unknown storage engine 'InnoDB'
FOUND 4 /redo log from 3\*[0-9]+ to 2\*[0-9]+ pages/ in mysqld.1.err
FOUND 4 /redo log from 3\*[0-9]+ to 2\*[0-9]+ bytes/ in mysqld.1.err
SELECT * FROM t1;
ERROR 42000: Unknown storage engine 'InnoDB'
SELECT * FROM t1;
@@ -56,7 +56,7 @@ ERROR 42000: Unknown storage engine 'InnoDB'
FOUND 1 /InnoDB: Setting log file .*ib_logfile[0-9]+ size to/ in mysqld.1.err
SELECT * FROM t1;
ERROR 42000: Unknown storage engine 'InnoDB'
FOUND 1 /InnoDB: Log file .*ib_logfile0 size 7 is not a multiple of innodb_page_size/ in mysqld.1.err
FOUND 1 /InnoDB: Log file .*ib_logfile0 size 7 is not a multiple of 512 bytes/ in mysqld.1.err
SELECT * FROM t1;
ERROR 42000: Unknown storage engine 'InnoDB'
FOUND 1 /InnoDB: Log file .*ib_logfile1 is of different size 1048576 bytes than other log files/ in mysqld.1.err
@@ -218,7 +218,7 @@ eval $check_no_innodb;
--source include/start_mysqld.inc
eval $check_yes_innodb;
--source include/shutdown_mysqld.inc
--let SEARCH_PATTERN=Resizing redo log from 1\*\d+ to 3\*\d+ pages; LSN=\d+
--let SEARCH_PATTERN=Resizing redo log from 1\*\d+ to 3\*\d+ bytes; LSN=\d+
--source include/search_pattern_in_file.inc

--let $restart_parameters=
@@ -100,14 +100,14 @@ let SEARCH_PATTERN= InnoDB: innodb_read_only prevents crash recovery;
--source include/restart_mysqld.inc
--error ER_UNKNOWN_STORAGE_ENGINE
SELECT * FROM t1;
let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ bytes;
--source include/search_pattern_in_file.inc

--let $restart_parameters= --debug=d,innodb_log_abort_5
--source include/restart_mysqld.inc
--error ER_UNKNOWN_STORAGE_ENGINE
SELECT * FROM t1;
let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ bytes;
--source include/search_pattern_in_file.inc

--let $restart_parameters= --innodb-read-only
@@ -122,7 +122,7 @@ let SEARCH_PATTERN= InnoDB: innodb_read_only prevents crash recovery;
--error ER_UNKNOWN_STORAGE_ENGINE
SELECT * FROM t1;

let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ bytes;
--source include/search_pattern_in_file.inc

--let $restart_parameters= --debug=d,innodb_log_abort_7
@@ -170,7 +170,7 @@ EOF
--source include/start_mysqld.inc
--error ER_UNKNOWN_STORAGE_ENGINE
SELECT * FROM t1;
let SEARCH_PATTERN= InnoDB: Log file .*ib_logfile0 size 7 is not a multiple of innodb_page_size;
let SEARCH_PATTERN= InnoDB: Log file .*ib_logfile0 size 7 is not a multiple of 512 bytes;
--source include/search_pattern_in_file.inc
--remove_file $MYSQLD_DATADIR/ib_logfile0
--move_file $MYSQLD_DATADIR/ib_logfile101 $MYSQLD_DATADIR/ib_logfile0
@@ -212,7 +212,7 @@ static ulong innobase_commit_concurrency = 0;
static ulong innobase_read_io_threads;
static ulong innobase_write_io_threads;

static long long innobase_buffer_pool_size, innobase_log_file_size;
static long long innobase_buffer_pool_size;

/** Percentage of the buffer pool to reserve for 'old' blocks.
Connected to buf_LRU_old_ratio. */
@@ -4240,8 +4240,6 @@ innobase_init(

srv_file_flush_method_str = innobase_file_flush_method;

srv_log_file_size = (ib_uint64_t) innobase_log_file_size;

if (UNIV_PAGE_SIZE_DEF != srv_page_size) {
ib::info() << "innodb_page_size=" << srv_page_size;

"The size of the buffer which InnoDB uses to write log to the log files on disk.",
NULL, NULL, 16*1024*1024L, 256*1024L, LONG_MAX, 1024);

static MYSQL_SYSVAR_LONGLONG(log_file_size, innobase_log_file_size,
static MYSQL_SYSVAR_ULONGLONG(log_file_size, srv_log_file_size,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
"Size of each log file in a log group.",
NULL, NULL, 48 << 20, 1 << 20, 512ULL << 30, 1 << 20);
NULL, NULL, 48 << 20, 1 << 20, 512ULL << 30, UNIV_PAGE_SIZE_MAX);
/* OS_FILE_LOG_BLOCK_SIZE would be more appropriate than UNIV_PAGE_SIZE_MAX,
but fil_space_t is being used for the redo log, and it uses data pages. */

static MYSQL_SYSVAR_ULONG(log_files_in_group, srv_n_log_files,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
@@ -156,17 +156,17 @@ void
log_sys_init();

/** Initialize the redo log.
@param[in] n_files number of files
@param[in] file_size file size in bytes */
@param[in] n_files number of files */
void
log_init(ulint n_files, lsn_t file_size);
log_init(ulint n_files);
/** Calculate the recommended highest values for lsn - last_checkpoint_lsn
and lsn - buf_get_oldest_modification().
@param[in] file_size requested innodb_log_file_size
@retval true on success
@retval false if the smallest log group is too small to
accommodate the number of OS threads in the database server */
bool
log_set_capacity()
log_set_capacity(ulonglong file_size)
MY_ATTRIBUTE((warn_unused_result));

/******************************************************//**
@@ -360,17 +360,9 @@ extern char* srv_log_group_home_dir;
/** Maximum number of srv_n_log_files, or innodb_log_files_in_group */
#define SRV_N_LOG_FILES_MAX 100
extern ulong srv_n_log_files;
/** At startup, this is the current redo log file size.
During startup, if this is different from srv_log_file_size_requested
(innodb_log_file_size), the redo log will be rebuilt and this size
will be initialized to srv_log_file_size_requested.
When upgrading from a previous redo log format, this will be set to 0,
and writing to the redo log is not allowed.
During startup, this is in bytes, and later converted to pages. */
extern ib_uint64_t srv_log_file_size;
/** The value of the startup parameter innodb_log_file_size */
extern ib_uint64_t srv_log_file_size_requested;
/** The InnoDB redo log file size, or 0 when changing the redo log format
at startup (while disallowing writes to the redo log). */
extern ulonglong srv_log_file_size;
extern ulint srv_log_buffer_size;
extern ulong srv_flush_log_at_trx_commit;
extern uint srv_flush_log_at_timeout;
@@ -669,18 +669,17 @@ log_group_set_fields(

/** Calculate the recommended highest values for lsn - last_checkpoint_lsn
and lsn - buf_get_oldest_modification().
@param[in] file_size requested innodb_log_file_size
@retval true on success
@retval false if the smallest log group is too small to
accommodate the number of OS threads in the database server */
bool
log_set_capacity()
log_set_capacity(ulonglong file_size)
{
lsn_t margin;
ulint free;

lsn_t smallest_capacity = ((srv_log_file_size_requested
<< srv_page_size_shift)
- LOG_FILE_HDR_SIZE)
lsn_t smallest_capacity = (file_size - LOG_FILE_HDR_SIZE)
* srv_n_log_files;
/* Add extra safety */
smallest_capacity -= smallest_capacity / 10;
@@ -798,10 +797,9 @@ log_sys_init()
}

/** Initialize the redo log.
@param[in] n_files number of files
@param[in] file_size file size in bytes */
@param[in] n_files number of files */
void
log_init(ulint n_files, lsn_t file_size)
log_init(ulint n_files)
{
ulint i;
log_group_t* group = &log_sys->log;
@@ -810,7 +808,7 @@ log_init(ulint n_files, lsn_t file_size)
group->format = srv_encrypt_log
? LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED
: LOG_HEADER_FORMAT_CURRENT;
group->file_size = file_size;
group->file_size = srv_log_file_size;
group->state = LOG_GROUP_OK;
group->lsn = LOG_START_LSN;
group->lsn_offset = LOG_FILE_HDR_SIZE;
@@ -201,17 +201,9 @@ static os_event_t srv_master_thread_disabled_event;
char* srv_log_group_home_dir;

ulong srv_n_log_files;
/** At startup, this is the current redo log file size.
During startup, if this is different from srv_log_file_size_requested
(innodb_log_file_size), the redo log will be rebuilt and this size
will be initialized to srv_log_file_size_requested.
When upgrading from a previous redo log format, this will be set to 0,
and writing to the redo log is not allowed.
During startup, this is in bytes, and later converted to pages. */
ib_uint64_t srv_log_file_size;
/** The value of the startup parameter innodb_log_file_size */
ib_uint64_t srv_log_file_size_requested;
/** The InnoDB redo log file size, or 0 when changing the redo log format
at startup (while disallowing writes to the redo log). */
ulonglong srv_log_file_size;
/** copy of innodb_log_buffer_size, but in database pages */
ulint srv_log_buffer_size;
/** innodb_flush_log_at_trx_commit */
@@ -1161,8 +1153,6 @@ srv_normalize_init_values(void)

srv_tmp_space.normalize();

srv_log_file_size /= UNIV_PAGE_SIZE;

srv_log_buffer_size /= UNIV_PAGE_SIZE;

srv_lock_table_size = 5 * (srv_buf_pool_size / UNIV_PAGE_SIZE);
@@ -138,6 +138,8 @@ bool srv_is_being_started;
bool srv_sys_tablespaces_open;
/** TRUE if the server was successfully started */
bool srv_was_started;
/** The original value of srv_log_file_size (innodb_log_file_size) */
static ulonglong srv_log_file_size_requested;
/** TRUE if innobase_start_or_create_for_mysql() has been called */
static bool srv_start_has_been_called;

@@ -374,17 +376,13 @@ create_log_file(
}

ib::info() << "Setting log file " << name << " size to "
<< (srv_log_file_size >> (20 - UNIV_PAGE_SIZE_SHIFT))
<< " MB";
<< srv_log_file_size << " bytes";

ret = os_file_set_size(name, *file,
(os_offset_t) srv_log_file_size
<< UNIV_PAGE_SIZE_SHIFT,
ret = os_file_set_size(name, *file, srv_log_file_size,
srv_read_only_mode);
if (!ret) {
ib::error() << "Cannot set log file " << name << " to size "
<< (srv_log_file_size >> (20 - UNIV_PAGE_SIZE_SHIFT))
<< " MB";
ib::error() << "Cannot set log file " << name << " size to "
<< srv_log_file_size << " bytes";
return(DB_ERROR);
}

@@ -467,17 +465,17 @@ create_log_files(
ut_a(fil_validate());
ut_a(log_space != NULL);

const ulint size = ulint(srv_log_file_size >> srv_page_size_shift);

logfile0 = fil_node_create(
logfilename, (ulint) srv_log_file_size,
log_space, false, false);
logfilename, size, log_space, false, false);
ut_a(logfile0);

for (unsigned i = 1; i < srv_n_log_files; i++) {

sprintf(logfilename + dirnamelen, "ib_logfile%u", i);

if (!fil_node_create(logfilename,
(ulint) srv_log_file_size,
if (!fil_node_create(logfilename, size,
log_space, false, false)) {

ib::error()
@@ -488,8 +486,8 @@ create_log_files(
}
}

log_init(srv_n_log_files, srv_log_file_size * UNIV_PAGE_SIZE);
if (!log_set_capacity()) {
log_init(srv_n_log_files);
if (!log_set_capacity(srv_log_file_size_requested)) {
return(DB_ERROR);
}

@@ -1410,7 +1408,7 @@ srv_prepare_to_delete_redo_log_files(

info << srv_n_log_files << "*"
<< srv_log_file_size_requested
<< " pages; LSN=" << flushed_lsn;
<< " bytes; LSN=" << flushed_lsn;
}

/* Flush the old log files. */
@@ -1863,8 +1861,7 @@ innobase_start_or_create_for_mysql()
srv_start_state_set(SRV_START_STATE_IO);
}

if (srv_n_log_files * srv_log_file_size * UNIV_PAGE_SIZE
>= 512ULL * 1024ULL * 1024ULL * 1024ULL) {
if (srv_n_log_files * srv_log_file_size >= 512ULL << 30) {
/* log_block_convert_lsn_to_no() limits the returned block
number to 1G and given that OS_FILE_LOG_BLOCK_SIZE is 512
bytes, then we have a limit of 512 GB. If that limit is to
@@ -1875,18 +1872,8 @@ innobase_start_or_create_for_mysql()
return(srv_init_abort(DB_ERROR));
}

if (srv_n_log_files * srv_log_file_size >= ULINT_MAX) {
/* fil_io() takes ulint as an argument and we are passing
(next_offset / UNIV_PAGE_SIZE) to it in log_group_write_buf().
So (next_offset / UNIV_PAGE_SIZE) must be less than ULINT_MAX.
So next_offset must be < ULINT_MAX * UNIV_PAGE_SIZE. This
means that we are limited to ULINT_MAX * UNIV_PAGE_SIZE which
is 64 TB on 32 bit systems. */
ib::error() << "Combined size of log files must be < "
<< ULINT_MAX / 1073741824 * UNIV_PAGE_SIZE << " GB";

return(srv_init_abort(DB_ERROR));
}
compile_time_assert(ulonglong(ULINT_MAX) * UNIV_PAGE_SIZE_MIN
>= 512ULL << 30);

os_normalize_path(srv_data_home);

@@ -2025,27 +2012,22 @@ innobase_start_or_create_for_mysql()

ut_a(size != (os_offset_t) -1);

if (size & ((1 << UNIV_PAGE_SIZE_SHIFT) - 1)) {
if (size & (OS_FILE_LOG_BLOCK_SIZE - 1)) {

ib::error() << "Log file " << logfilename
<< " size " << size << " is not a"
" multiple of innodb_page_size";
" multiple of 512 bytes";
return(srv_init_abort(DB_ERROR));
}

size >>= UNIV_PAGE_SIZE_SHIFT;

if (i == 0) {
srv_log_file_size = size;
} else if (size != srv_log_file_size) {

ib::error() << "Log file " << logfilename
<< " is of different size "
<< (size << UNIV_PAGE_SIZE_SHIFT)
<< " is of different size " << size
<< " bytes than other log files "
<< (srv_log_file_size
<< UNIV_PAGE_SIZE_SHIFT)
<< " bytes!";
<< srv_log_file_size << " bytes!";
return(srv_init_abort(DB_ERROR));
}
}
@@ -2066,23 +2048,23 @@ innobase_start_or_create_for_mysql()
ut_a(fil_validate());
ut_a(log_space);

/* srv_log_file_size is measured in pages; if page size is 16KB,
then we have a limit of 64TB on 32 bit systems */
ut_a(srv_log_file_size <= ULINT_MAX);
ut_a(srv_log_file_size <= 512ULL << 30);

const ulint size = 1 + ulint((srv_log_file_size - 1)
>> srv_page_size_shift);

for (unsigned j = 0; j < i; j++) {
for (unsigned j = 0; j < srv_n_log_files_found; j++) {
sprintf(logfilename + dirnamelen, "ib_logfile%u", j);

if (!fil_node_create(logfilename,
(ulint) srv_log_file_size,
if (!fil_node_create(logfilename, size,
log_space, false, false)) {
return(srv_init_abort(DB_ERROR));
}
}

log_init(i, srv_log_file_size * UNIV_PAGE_SIZE);
log_init(srv_n_log_files_found);

if (!log_set_capacity()) {
if (!log_set_capacity(srv_log_file_size_requested)) {
return(srv_init_abort(DB_ERROR));
}
}

0 comments on commit 84e4e45

Please sign in to comment.
You can’t perform that action at this time.