Permalink
Browse files

Dump core for mysqld without large memory buffer

Summary:
Core files can be very helpful for debugging production crashes but core dump has been disabled on production due to multiple reasons. One of the reasons is that the core file can be huge due to the huge InnoDB buffer pool within the process. This diff introduces a new command line argument '--innodb-dump-core-without-large-mem-buf', when it is set, all the large memory allocation by function os_mem_alloc_large() in InnoDB won't be dumped in the core file by calling Linux API madvise() with argument MADV_DONTDUMP. It can be disabled by '--skip-innodb-dump-core-without-large-mem-buf'. os_mem_alloc_large() is used to allocate large memory chunk for InnoDB buffer pool, file data merge, row log buffer, etc. Buffer pool is the main one. With this feature, the core size will be less than 3GB when the buffer pool size is 60GB so it won't impact production.
-- This sys var only take effect when core dump is enabled by setting 'core-file' in my.cnf.
-- By default '--innodb-dump-core-without-large-mem-buf' is set in prod to dump smaller cores, but it is not set by default in MTR so that full core file can be dumped.
-- This diff also makes sys var core_file readonly.
-- There are other bugs that prevent mysqld from dumping cores, e.g. when mysqld switching user from root to a non-root user, etc. Those will be addressed in separated diffs.

Test Plan:
-- New test cases:
(1) innodb.mysqld_core_dump_without_large_mem_buf.test
(2) innodb.mysqld_core_dump_without_large_mem_buf_with_resizing.test
(3) sys_vars.innodb_dump_core_without_large_mem_buf_basic.test

-- Manual test on a prod host and confirmed that small core is dumped
(1) Build RPM with this diff and install it on a reserved prod host
(2) Add 'core-file' into the '[mysqld]' section to enable core dump
(3) Set 'innodb_buffer_pool_size' to 53GB
(4) Start mysqld and find out the PID of mysqld
(5) Use 'kill -s SIGSEGV <PID>' to kill mysqld and generate core file
(6) Use 'cat /proc/sys/kernel/core_pattern' to find out the core file pattern and location
(7) Find the core file at the location with the expected name mysqld.<PID>, the size was about 3GB

Reviewers: santoshb, jtolmer

Reviewed By: jtolmer

Subscribers: webscalesql-eng

Differential Revision: https://reviews.facebook.net/D54555
  • Loading branch information...
pengtfb committed Feb 15, 2016
1 parent c69af69 commit d3054d2915ffcde166cd67760ac6a618c6d02b82
@@ -0,0 +1,103 @@
--echo # Get the full path name of the PID file
--let $pid_file= query_get_value(SELECT @@pid_file, @@pid_file, 1)
--let PIDFILE= $pid_file

--echo # Expecting a "crash", but don't restart the server until it is told to
--echo # Expected max core size is $expected_max_core_size MB
--let MAXCORESIZE= $expected_max_core_size

--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect

perl;

my $pid_file = $ENV{'PIDFILE'} or die "PIDFILE not set";
my $expected_max_core_size = $ENV{'MAXCORESIZE'} or die "MAXCORESIZE not set";

# The argument is in MB
$expected_max_core_size = $expected_max_core_size * 1024 * 1024;

# Get PID of mysqld
open(my $fh, '<', $pid_file) || die "Cannot open pid file $pid_file\n";
my $pid = <$fh>;
$pid =~ s/\s//g;
close($fh);

if ($pid eq "") {
die "Couldn't retrieve PID from PID file.\n";
}

# The current time in seconds since epoch
$cur_time = time;

# Kill mysqld to dump a core
system("kill", "-s", "SIGSEGV", "$pid");
print "# Perl: Sent a SIGSEGV to mysqld to dump a core.\n";

# Get the core file pattern, e.g. /var/tmp/cores/%e.%p
$core_pattern = `cat /proc/sys/kernel/core_pattern`;

$last_slash = rindex($core_pattern, '/');

# The core file directory, e.g. /var/tmp/cores
$core_dir = substr($core_pattern, 0, $last_slash);

$found_core = 0;
$core_size = 0;
$core_size_good = 0;

# Check the files in the core file directory
$wait_sec = 60;
while ($wait_sec > 0) {
opendir(my $dir, $core_dir) or die "Failed to open dir $core_dir: $!\n";
while (my $file = readdir($dir)) {
# If the core file name contains the PID
if (index($file, $pid) != -1) {
# The last write time in seconds since epoch
$full_path = $core_dir . '/' . $file;
@stat = stat($full_path);
$core_size = $stat[7];
$write_secs = $stat[9];

# If the file was written within a minute
if ($cur_time <= $write_secs && $write_secs - $cur_time < 60) {
$found_core = 1;
if ($core_size < $expected_max_core_size) {
$core_size_good = 1;
}
# Remove the core file to avoid it get accumulated over time
unlink $full_path;
last;
}
}
}
closedir($dir);

if ($found_core) {
last;
}
# Sleep 1 second and try again
--$wait_sec;
sleep 1;
}

if ($found_core) {
if ($core_size_good) {
print "# Perl: OK! Found the core file and it's small!\n";
} else {
print "# Perl: Failed! Found the core file but it's too big ($core_size)!\n";
}
} else {
print "# Perl: Failed! Didn't find the core file!\n";
}

EOF

--echo # Make server restart
--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect

--enable_reconnect

--echo # Wait for server to be back online
--source include/wait_until_connected_again.inc

--disable_reconnect
@@ -5283,6 +5283,7 @@ ($$$)
# override defaults above.

my $found_skip_core= 0;
my $innodb_dump_core_without_large_mem_buf= 0;
my $found_no_console= 0;
my $found_log_error= 0;
foreach my $arg ( @$extra_opts )
@@ -5301,6 +5302,10 @@ ($$$)
{
$found_skip_core= 1;
}
elsif ($arg eq "--innodb-dump-core-without-large-mem-buf")
{
$innodb_dump_core_without_large_mem_buf= 1;
}
elsif ($arg eq "--no-console")
{
$found_no_console= 1;
@@ -5335,6 +5340,11 @@ ($$$)
mtr_add_arg($args, "%s", "--core-file");
}

# Set the default value to false so that the full core will be dumped
if ( !$innodb_dump_core_without_large_mem_buf)
{
mtr_add_arg($args, "--skip-innodb-dump-core-without-large-mem-buf");
}
return $args;
}

@@ -0,0 +1,9 @@
# Shutdown server
# Restart server with --log-error
# Get the full path name of the PID file
# Expecting a "crash", but don't restart the server until it is told to
# Expected max core size is 3584 MB
# Perl: Sent a SIGSEGV to mysqld to dump a core.
# Perl: OK! Found the core file and it's small!
# Make server restart
# Wait for server to be back online
@@ -0,0 +1,15 @@
# Shutdown server
# Restart server with --log-error
set global innodb_file_format=`Barracuda`;
set global innodb_file_per_table=ON;
set global innodb_adaptive_hash_index=ON;
set global innodb_buffer_pool_size = 21474836480;
set global innodb_adaptive_hash_index=OFF;
set global innodb_buffer_pool_size = 64424509440;
# Get the full path name of the PID file
# Expecting a "crash", but don't restart the server until it is told to
# Expected max core size is 5632 MB
# Perl: Sent a SIGSEGV to mysqld to dump a core.
# Perl: OK! Found the core file and it's small!
# Make server restart
# Wait for server to be back online
@@ -0,0 +1,2 @@
--innodb-dump-core-without-large-mem-buf
--innodb-buffer-pool-size=60G
@@ -0,0 +1,25 @@
################################################################################
# This test is to test if mysqld can dump a core without large memory buffers.
# See opt file for the config:
# (1) --innodb-dump-core-without-large-mem-buf is set
# (2) the buffer pool is set to be large so that without dropping the large
# memory buffers the core size will be much greater than 3.5GB (the actual
# core size is less than 3GB now but set the limit to 3.5GB here in case
# the memory footprint increases in the future)

--source include/not_valgrind.inc
--source include/have_innodb.inc

# Embedded mode doesn't support restart
--source include/not_embedded.inc

--echo # Shutdown server
--source include/shutdown_mysqld.inc

--echo # Restart server with --log-error
--exec echo "restart:--log-error=$MYSQLTEST_VARDIR/log/core_dump.err" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
--enable_reconnect
--source include/wait_until_connected_again.inc

--let $expected_max_core_size = 3584
--source include/mysqld_core_dump.inc
@@ -0,0 +1,3 @@
--innodb-dump-core-without-large-mem-buf
--innodb-buffer-pool-size=60G
--innodb-buffer-pool-resizing-timeout=60
@@ -0,0 +1,46 @@
################################################################################
# This test is to test if mysqld can dump a core without large memory buffers.
# See opt file for the config:
# (1) --innodb-dump-core-without-large-mem-buf is set
# (2) the buffer pool is set to be large initially, shrink it, then expand
# it back to the original large size, without dropping the large memory
# buffers the core size will be much greater than 5.5GB (the actual
# core size is less than 5GB now but set the limit to 5.5GB here in case
# the memory footprint increases in the future)

--source include/not_valgrind.inc
--source include/have_innodb.inc

# Embedded mode doesn't support restart
--source include/not_embedded.inc

--echo # Shutdown server
--source include/shutdown_mysqld.inc

--echo # Restart server with --log-error
--exec echo "restart:--log-error=$MYSQLTEST_VARDIR/log/core_dump_with_resizing.err" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
--enable_reconnect
--source include/wait_until_connected_again.inc

--source suite/innodb/t/innodb-buffer-pool-resize-setup.inc

# Shrink buffer pool to 20GB
set global innodb_buffer_pool_size = 21474836480;
--source include/wait_condition.inc

set global innodb_adaptive_hash_index=OFF;

# Expand buffer pool back to 60GB
set global innodb_buffer_pool_size = 64424509440;
--source include/wait_condition.inc

--disable_query_log
set global innodb_buffer_pool_size = @old_innodb_buffer_pool_size;
set global innodb_file_format = @old_innodb_file_format;
set global innodb_file_per_table = @old_innodb_file_per_table;
set global innodb_adaptive_hash_index = @old_innodb_adaptive_hash_index;
--enable_query_log
--source include/wait_condition.inc

--let $expected_max_core_size = 5632
--source include/mysqld_core_dump.inc
@@ -1,31 +1,23 @@
SELECT * FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE '%core%';
VARIABLE_NAME VARIABLE_VALUE
CORE_FILE ON
SHOW GLOBAL VARIABLES LIKE '%core%';
Variable_name Value
core_file ON
SET @old_val = @@global.core_file;
SELECT @old_val;
@old_val
1
SET GLOBAL core_file = FALSE;
SELECT @@core_file;
@@core_file
0
SHOW GLOBAL VARIABLES LIKE '%core%';
Variable_name Value
core_file OFF
SET GLOBAL core_file = TRUE;
SELECT @@core_file;
@@core_file
select @@global.core_file;
@@global.core_file
1
SHOW GLOBAL VARIABLES LIKE '%core%';
select @@session.core_file;
ERROR HY000: Variable 'core_file' is a GLOBAL variable
show global variables like 'core_file';
Variable_name Value
core_file ON
SET @@global.core_file = @old_val;
SELECT @@global.core_file;
@@global.core_file
1
SHOW GLOBAL VARIABLES LIKE '%core%';
show session variables like 'core_file';
Variable_name Value
core_file ON
select * from information_schema.global_variables where variable_name='core_file';
VARIABLE_NAME VARIABLE_VALUE
CORE_FILE ON
select * from information_schema.session_variables where variable_name='core_file';
VARIABLE_NAME VARIABLE_VALUE
CORE_FILE ON
set global core_file = default;
ERROR HY000: Variable 'core_file' is a read only variable
Expected error 'Read only variable'
set global core_file = true;
ERROR HY000: Variable 'core_file' is a read only variable
Expected error 'Read only variable'
@@ -0,0 +1,23 @@
select @@global.innodb_dump_core_without_large_mem_buf;
@@global.innodb_dump_core_without_large_mem_buf
0
select @@session.innodb_dump_core_without_large_mem_buf;
ERROR HY000: Variable 'innodb_dump_core_without_large_mem_buf' is a GLOBAL variable
show global variables like 'innodb_dump_core_without_large_mem_buf';
Variable_name Value
innodb_dump_core_without_large_mem_buf OFF
show session variables like 'innodb_dump_core_without_large_mem_buf';
Variable_name Value
innodb_dump_core_without_large_mem_buf OFF
select * from information_schema.global_variables where variable_name='innodb_dump_core_without_large_mem_buf';
VARIABLE_NAME VARIABLE_VALUE
INNODB_DUMP_CORE_WITHOUT_LARGE_MEM_BUF OFF
select * from information_schema.session_variables where variable_name='innodb_dump_core_without_large_mem_buf';
VARIABLE_NAME VARIABLE_VALUE
INNODB_DUMP_CORE_WITHOUT_LARGE_MEM_BUF OFF
set global innodb_dump_core_without_large_mem_buf = default;
ERROR HY000: Variable 'innodb_dump_core_without_large_mem_buf' is a read only variable
Expected error 'Read only variable'
set global innodb_dump_core_without_large_mem_buf = true;
ERROR HY000: Variable 'innodb_dump_core_without_large_mem_buf' is a read only variable
Expected error 'Read only variable'
@@ -1,20 +1,20 @@
SELECT * FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE '%core%';
SHOW GLOBAL VARIABLES LIKE '%core%';
--source include/load_sysvars.inc

SET @old_val = @@global.core_file;
SELECT @old_val;
select @@global.core_file;

SET GLOBAL core_file = FALSE;
SELECT @@core_file;
--error ER_INCORRECT_GLOBAL_LOCAL_VAR
select @@session.core_file;

SHOW GLOBAL VARIABLES LIKE '%core%';
show global variables like 'core_file';
show session variables like 'core_file';

SET GLOBAL core_file = TRUE;
SELECT @@core_file;
select * from information_schema.global_variables where variable_name='core_file';
select * from information_schema.session_variables where variable_name='core_file';

SHOW GLOBAL VARIABLES LIKE '%core%';
--error ER_INCORRECT_GLOBAL_LOCAL_VAR
set global core_file = default;
--echo Expected error 'Read only variable'

SET @@global.core_file = @old_val;
SELECT @@global.core_file;

SHOW GLOBAL VARIABLES LIKE '%core%';
--error ER_INCORRECT_GLOBAL_LOCAL_VAR
set global core_file = true;
--echo Expected error 'Read only variable'
@@ -0,0 +1,21 @@
--source include/have_innodb.inc
--source include/load_sysvars.inc

select @@global.innodb_dump_core_without_large_mem_buf;

--error ER_INCORRECT_GLOBAL_LOCAL_VAR
select @@session.innodb_dump_core_without_large_mem_buf;

show global variables like 'innodb_dump_core_without_large_mem_buf';
show session variables like 'innodb_dump_core_without_large_mem_buf';

select * from information_schema.global_variables where variable_name='innodb_dump_core_without_large_mem_buf';
select * from information_schema.session_variables where variable_name='innodb_dump_core_without_large_mem_buf';

--error ER_INCORRECT_GLOBAL_LOCAL_VAR
set global innodb_dump_core_without_large_mem_buf = default;
--echo Expected error 'Read only variable'

--error ER_INCORRECT_GLOBAL_LOCAL_VAR
set global innodb_dump_core_without_large_mem_buf = true;
--echo Expected error 'Read only variable'
@@ -796,7 +796,7 @@ static bool fix_binlog_format_after_update(sys_var *self, THD *thd,
my_bool opt_core_file = FALSE;
static Sys_var_mybool Sys_core_file(
"core_file", "write a core-file on crashes",
GLOBAL_VAR(opt_core_file), NO_CMD_LINE, DEFAULT(FALSE));
READ_ONLY GLOBAL_VAR(opt_core_file), NO_CMD_LINE, DEFAULT(FALSE));

static Sys_var_enum Sys_binlog_format(
"binlog_format", "What form of binary logging the master will "
Oops, something went wrong.

0 comments on commit d3054d2

Please sign in to comment.