FB8-118: Port slow query log rotation (facebook#1015) (facebook#1015)

Summary: Jira issue: https://jira.percona.com/browse/FB8-118 Reference Patch: facebook@3445f81 Porting notes: 1. fixed PS-1484: slowlog rotation and expiration doesn't work with the default slow query log name 2. `max_slowlog_files` was replaced with `slowlog_space_limit` ---------- facebook@3445f81 ---------- This diff is a combination of the following two commits from Percona: percona/percona-server@725a238c percona/percona-server@85c461c8 Adds two new variables `max_slowlog_size` and `max_slowlog_files` as documented here: https://www.percona.com/doc/percona-server/LATEST/flexibility/slowlog_rotation.html Originally Reviewed By: hermanlee Pull Request resolved: facebook#1015 Reference Patch: facebook@b857c4840db Differential Revision: D15158133 (facebook@06ea967) fbshipit-source-id: b14ee4a7a9e
inikep · Jul 16, 2021 · 5e6cd91 · 5e6cd91
1 parent 4e665e9
commit 5e6cd91
Show file tree

Hide file tree

Showing 13 changed files with 988 additions and 4 deletions.
diff --git a/mysql-test/include/assert_number_of_files.inc b/mysql-test/include/assert_number_of_files.inc
@@ -0,0 +1,44 @@
+# ==== Purpose ====
+#
+# Assert that number of files matching given $file_spec is equal to expected.
+#
+# ==== Usage ====
+#
+# --let file_spec = PATH_REGEX
+# --let expected_number = INTEGER
+# --source include/assert_number_of_files.inc
+#
+# Parameters:
+#   $file_spec
+#     The path with wildcards e.g. $MYSQLTEST_VARDIR/abcd*
+#
+#   $expected_number
+#     The expected number of files
+
+if ($VALGRIND_TEST)
+{
+  --sleep 5
+}
+
+--perl
+  use strict;
+  my $dir = $ENV{'MYSQL_TMP_DIR'} or die "MYSQL_TMP_DIR not set";
+  my $file_spec = $ENV{'file_spec'} or die "file_spec not set";
+  my @files = <$ENV{'file_spec'}>;
+  open (OUTPUT, ">$dir/number_of_files.inc") ;
+  print OUTPUT "--let \$number_of_files = ", scalar(@files), "\n";
+  close (OUTPUT);
+EOF
+--source $MYSQL_TMP_DIR/number_of_files.inc
+--remove_file $MYSQL_TMP_DIR/number_of_files.inc
+
+
+if ($expected_number != $number_of_files)
+{
+  --echo ====================== Test assertion failed: ======================
+  --echo The number of files ($number_of_files) is different that expected $expected_number
+  --echo ====================================================================
+  --die Test assertion failed in assert_number_of_files.inc
+}
+
+--echo include/assert_number_of_files.inc [The number of files matches expected $expected_number]
diff --git a/mysql-test/r/mysqld--help-notwin.result b/mysql-test/r/mysqld--help-notwin.result
@@ -717,6 +717,10 @@ The following options may be given as the first argument:
  --max-seeks-for-key=# 
  Limit assumed max number of seeks when looking up rows
  based on a key
+ --max-slowlog-size=# 
+ Slow query log will be rotated automatically when the
+ size exceeds this value. The default is 0, don't limit
+ the size.
  --max-sort-length=# The number of bytes to use when sorting long values with
  PAD SPACE collations (only the first max_sort_length
  bytes of each value are used; the rest are ignored)
@@ -1456,6 +1460,10 @@ The following options may be given as the first argument:
  Log slow queries to given log file. Defaults logging to
  hostname-slow.log. Must be enabled to activate other slow
  log options
+ --slowlog-space-limit=# 
+ Maximum space to use for all slow query logs. Works only
+ with max_slowlog_size enabled. Default is 0, this feature
+ is disabled.
  --socket=name       Socket file to use for connection
  --sort-buffer-size=# 
  Each thread that needs to do a sort allocates a buffer of
@@ -1779,6 +1787,7 @@ max-prepared-stmt-count 16382
 max-relay-log-size 0
 max-running-queries 0
 max-seeks-for-key 18446744073709551615
+max-slowlog-size 0
 max-sort-length 1024
 max-sp-recursion-depth 0
 max-user-connections 0
@@ -1973,6 +1982,7 @@ slave-transaction-retries 10
 slave-type-conversions 
 slow-launch-time 2
 slow-query-log FALSE
+slowlog-space-limit 0
 sort-buffer-size 262144
 sporadic-binlog-dump-fail FALSE
 sql-log-bin-triggers TRUE

diff --git a/mysql-test/r/percona_slowlog_size_limits.result b/mysql-test/r/percona_slowlog_size_limits.result
@@ -0,0 +1,67 @@
+SET @old_slow_query_log = @@global.slow_query_log;
+SET @old_log_output = @@global.log_output;
+SET @old_slow_query_log_file = @@global.slow_query_log_file;
+SET @old_max_slowlog_size = @@global.max_slowlog_size;
+SET @old_slowlog_space_limit = @@global.slowlog_space_limit;
+SET long_query_time = 0;
+SET GLOBAL log_output = FILE;
+SET GLOBAL slow_query_log = 1;
+# Case 1 (PS-1484): Test if slowlog_space_limit is working correctly
+#                   with the default slow query log name
+SET GLOBAL max_slowlog_size = 4096;
+SET GLOBAL slowlog_space_limit = @@global.max_slowlog_size * 3;
+include/assert_number_of_files.inc [The number of files matches expected 3]
+# Case 2 (Bug 1416582): Slow query log is rotated before it should
+#                       when using max_slowlog_size
+SET GLOBAL max_slowlog_size = 10240000;
+SET GLOBAL slow_query_log_file = 'MYSQLTEST_VARDIR/abcd';
+SET GLOBAL slow_query_log = 0;
+SET GLOBAL slow_query_log = 1;
+SET GLOBAL slow_query_log = 0;
+SET GLOBAL slow_query_log = 1;
+FLUSH LOGS;
+FLUSH LOGS;
+FLUSH LOGS;
+include/assert.inc [Slow query log number should not be incremented and log should be abcd.000001]
+# Case 3: Test if each slow_query_log_file call rotates slow log 
+SET GLOBAL slow_query_log_file = 'MYSQLTEST_VARDIR/zxcv';
+SET GLOBAL slow_query_log_file = 'MYSQLTEST_VARDIR/abcd';
+SET GLOBAL slow_query_log_file = 'MYSQLTEST_VARDIR/zxcv';
+include/assert.inc [Slow query log should be zxcv.000002]
+SET GLOBAL slow_query_log_file = 'MYSQLTEST_VARDIR/abcd';
+include/assert.inc [Slow query log should be abcd.000003]
+# Case 4: Test if slowlog_space_limit is working correctly
+SET GLOBAL max_slowlog_size = 4096;
+SET GLOBAL slowlog_space_limit = @@global.max_slowlog_size * 5;
+SET GLOBAL slow_query_log_file = 'MYSQLTEST_VARDIR/abcd';
+include/assert_number_of_files.inc [The number of files matches expected 5]
+SET GLOBAL slow_query_log_file = 'MYSQLTEST_VARDIR/zxcv';
+include/assert_number_of_files.inc [The number of files matches expected 5]
+SET GLOBAL slow_query_log_file = 'MYSQLTEST_VARDIR/abcd';
+include/assert_number_of_files.inc [The number of files matches expected 5]
+# Case 5: Rotating log but should not delete previous logs
+SET GLOBAL slow_query_log_file = 'MYSQLTEST_VARDIR/abcd';
+SET GLOBAL slow_query_log_file = 'MYSQLTEST_VARDIR/abcd';
+include/assert_number_of_files.inc [The number of files matches expected 7]
+# Case 6: Calling "SET slowlog_space_limit" should reduce number of logs
+SET GLOBAL max_slowlog_size = 4096;
+SET GLOBAL slowlog_space_limit = @@global.max_slowlog_size * 2;
+include/assert_number_of_files.inc [The number of files matches expected 4]
+# Case 7: Delete all small logs with max_slowlog_size = 0
+SET GLOBAL max_slowlog_size = 0;
+SET GLOBAL slowlog_space_limit = 4096;
+include/assert_number_of_files.inc [The number of files matches expected 1]
+# Case 8: Rotate log should delete previous log
+SET GLOBAL max_slowlog_size = 10240000;
+SET GLOBAL slow_query_log_file = 'MYSQLTEST_VARDIR/abcd';
+include/assert_number_of_files.inc [The number of files matches expected 1]
+# Case 9: Check if disable max_slowlog_size is working correctly
+SET GLOBAL max_slowlog_size = 0;
+SET GLOBAL slow_query_log_file = 'MYSQLTEST_VARDIR/abcd';
+include/assert_number_of_files.inc [The number of files matches expected 2]
+include/assert.inc [Slow query log should stay as abcd]
+SET @@global.slow_query_log = @old_slow_query_log;
+SET @@global.log_output = @old_log_output;
+SET @@global.slow_query_log_file = @old_slow_query_log_file;
+SET @@global.max_slowlog_size = @old_max_slowlog_size;
+SET @@global.slowlog_space_limit = @old_slowlog_space_limit;
diff --git a/mysql-test/suite/sys_vars/inc/check_global_integer.inc b/mysql-test/suite/sys_vars/inc/check_global_integer.inc
@@ -0,0 +1,206 @@
+#############################################################################
+# ==== Purpose ====                                                         #
+#                                                                           #
+# Check if global integer $VARIABLE_NAME returns errors if it's outside a   #
+# range between $VARIABLE_MIN_VALUE (default: 0) and $VARIABLE_MAX_VALUE.   #
+#                                                                           #
+# Additionally we check the behaviour of this variable for:                 #
+# * Default values                                                          #
+# * Valid and invalid values                                                #
+# * Scope and access method                                                 #
+#                                                                           #
+# References: http://dev.mysql.com/doc/refman/8.0/en/                       #
+#                        server-system-variables.html                       #
+#                                                                           #
+# ==== Usage ====                                                           #
+#                                                                           #
+# --let $VARIABLE_NAME = 'NAME'                                             #
+# --let $VARIABLE_MAX_VALUE = NUMBER                                        #
+# [--let $VARIABLE_MIN_VALUE = NUMBER ]                                     #
+# [--let $VARIABLE_BLOCK_SIZE = NUMBER ]                                    #
+# --source suite/sys_vars/inc/check_global_integer.inc                      #
+#                                                                           #
+# ==== Examples ====                                                        #
+#                                                                           #
+# --let $VARIABLE_NAME = slowlog_space_limit                                #
+# --let $VARIABLE_MAX_VALUE = 18446744073709551615                          #
+# --source suite/sys_vars/inc/check_global_integer.inc                      #
+#                                                                           #
+# --let $VARIABLE_NAME = slave_pending_jobs_size_max                        #
+# --let $VARIABLE_MIN_VALUE = 1024                                          #
+# --let $VARIABLE_BLOCK_SIZE = 1024                                         #
+# --let $VARIABLE_MAX_VALUE = 18446744073709551615                          #
+# --source suite/sys_vars/inc/check_global_integer.inc                      #
+#############################################################################
+
+################################################################
+# Initialize auxiliary varaibles                               #
+################################################################
+
+--let $GLOBAL_VAR_NAME = @@global.$VARIABLE_NAME
+--let $SESSION_VAR_NAME = @@session.$VARIABLE_NAME
+
+--let $CHECK_MIN_VALUE = 0
+if ($VARIABLE_MIN_VALUE)
+{
+--let $CHECK_MIN_VALUE = $VARIABLE_MIN_VALUE
+}
+
+--let $CHECK_BLOCK_SIZE = 1
+if ($VARIABLE_BLOCK_SIZE)
+{
+  --let $CHECK_BLOCK_SIZE = $VARIABLE_BLOCK_SIZE
+}
+--let $CHECK_MIN_VALUE_PLUS_BLOCK_SIZE = `SELECT $CHECK_MIN_VALUE + $CHECK_BLOCK_SIZE`
+
+################################################################
+# Saving initial values of $VARIABLE_NAME variable in          #
+# a temporary variable                                         #
+################################################################
+
+--eval SET @save_original_variable = $GLOBAL_VAR_NAME
+SELECT @save_original_variable;
+
+################################################################
+# Display the DEFAULT value of $VARIABLE_NAME                  #
+################################################################
+
+--eval SET $GLOBAL_VAR_NAME = $CHECK_MIN_VALUE_PLUS_BLOCK_SIZE
+--eval SET $GLOBAL_VAR_NAME = DEFAULT
+--eval SELECT $GLOBAL_VAR_NAME
+
+--error ER_GLOBAL_VARIABLE
+--eval SET $SESSION_VAR_NAME = $CHECK_MIN_VALUE + 2
+--error ER_GLOBAL_VARIABLE
+--eval SET $SESSION_VAR_NAME = DEFAULT
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+--eval SELECT $SESSION_VAR_NAME
+
+################################################################
+# Check if NULL or empty value is accepted                     #
+################################################################
+
+--error ER_WRONG_TYPE_FOR_VAR
+--eval SET $GLOBAL_VAR_NAME = NULL
+
+--error ER_WRONG_TYPE_FOR_VAR
+--eval SET $GLOBAL_VAR_NAME = ''
+
+--error ER_WRONG_TYPE_FOR_VAR
+--eval SET $GLOBAL_VAR_NAME = ' '
+
+--error ER_GLOBAL_VARIABLE
+--eval SET $SESSION_VAR_NAME = NULL
+
+--error ER_GLOBAL_VARIABLE
+--eval SET $SESSION_VAR_NAME = ''
+
+--error ER_GLOBAL_VARIABLE
+--eval SET $SESSION_VAR_NAME = ' '
+
+#############################################################################
+# Change the value of $VARIABLE_NAME to a valid value for GLOBAL scope      #
+#############################################################################
+
+--eval SET $GLOBAL_VAR_NAME = $CHECK_MIN_VALUE
+--eval SELECT $GLOBAL_VAR_NAME
+
+--eval SET $GLOBAL_VAR_NAME = $CHECK_MIN_VALUE_PLUS_BLOCK_SIZE
+--eval SELECT $GLOBAL_VAR_NAME
+
+if ($CHECK_MIN_VALUE == 0)
+{
+ if ($CHECK_BLOCK_SIZE == 1)
+ {
+  --eval SET $GLOBAL_VAR_NAME = TRUE
+  --eval SELECT $GLOBAL_VAR_NAME
+
+  --eval SET $GLOBAL_VAR_NAME = FALSE
+  --eval SELECT $GLOBAL_VAR_NAME
+ }
+}
+
+--eval SET $GLOBAL_VAR_NAME = $VARIABLE_MAX_VALUE
+--eval SELECT $GLOBAL_VAR_NAME
+
+--eval SET $GLOBAL_VAR_NAME = DEFAULT
+--eval SELECT $GLOBAL_VAR_NAME
+
+#############################################################################
+# Change the value of $VARIABLE_NAME to an invalid value for GLOBAL scope   #
+#############################################################################
+
+--eval SET $GLOBAL_VAR_NAME = -$CHECK_MIN_VALUE_PLUS_BLOCK_SIZE
+--eval SELECT $GLOBAL_VAR_NAME
+
+--error ER_WRONG_TYPE_FOR_VAR
+--eval SET $GLOBAL_VAR_NAME = 'ABC'
+--eval SELECT $GLOBAL_VAR_NAME
+
+--error ER_WRONG_TYPE_FOR_VAR
+--eval SET $GLOBAL_VAR_NAME = ON
+--eval SELECT $GLOBAL_VAR_NAME
+
+--error ER_WRONG_TYPE_FOR_VAR
+--eval SET $GLOBAL_VAR_NAME = 'OFF'
+--eval SELECT $GLOBAL_VAR_NAME
+
+--error ER_WRONG_TYPE_FOR_VAR
+--eval SET $GLOBAL_VAR_NAME = $CHECK_MIN_VALUE_PLUS_BLOCK_SIZE.14
+--eval SELECT $GLOBAL_VAR_NAME
+
+--error ER_WRONG_TYPE_FOR_VAR
+--eval SET $GLOBAL_VAR_NAME = NONE
+--eval SELECT $GLOBAL_VAR_NAME
+
+if ($VARIABLE_MAX_VALUE == 18446744073709551615)
+{
+--error ER_WRONG_TYPE_FOR_VAR
+}
+--eval SET $GLOBAL_VAR_NAME = -$VARIABLE_MAX_VALUE
+--eval SELECT $GLOBAL_VAR_NAME
+
+if ($VARIABLE_MAX_VALUE == 18446744073709551615)
+{
+--error ER_DATA_OUT_OF_RANGE
+}
+--eval SET $GLOBAL_VAR_NAME = $VARIABLE_MAX_VALUE + 1
+--eval SELECT $GLOBAL_VAR_NAME
+
+#############################################################################
+# Check if the value in GLOBAL Table matches value in variable              #
+#############################################################################
+
+--let $assert_text= Must return 1
+--let $assert_cond= $GLOBAL_VAR_NAME = VARIABLE_VALUE FROM performance_schema.global_variables WHERE VARIABLE_NAME="$VARIABLE_NAME"
+--source include/assert.inc
+
+#############################################################################
+# Check if $VARIABLE_NAME variable can be accessed with and without @@ sign #
+#############################################################################
+
+--eval SET GLOBAL $VARIABLE_NAME = $CHECK_MIN_VALUE_PLUS_BLOCK_SIZE
+--eval SET $GLOBAL_VAR_NAME = $CHECK_MIN_VALUE_PLUS_BLOCK_SIZE
+
+--error ER_PARSE_ERROR
+--eval SET global.$VARIABLE_NAME = $CHECK_MIN_VALUE_PLUS_BLOCK_SIZE
+
+--error ER_GLOBAL_VARIABLE
+--eval SET @@$VARIABLE_NAME = $CHECK_MIN_VALUE_PLUS_BLOCK_SIZE
+
+--error ER_GLOBAL_VARIABLE
+--eval SET $VARIABLE_NAME = $CHECK_MIN_VALUE_PLUS_BLOCK_SIZE
+
+--error ER_UNKNOWN_TABLE
+--eval SELECT global.$VARIABLE_NAME
+
+--error ER_BAD_FIELD_ERROR
+--eval SELECT GLOBAL $VARIABLE_NAME
+
+--eval SELECT @@$VARIABLE_NAME
+
+#############################################################################
+# Restoring the original value of $VARIABLE_NAME variable                   #
+#############################################################################
+
+--eval SET $GLOBAL_VAR_NAME = @save_original_variable