Merge remote-tracking branch 'origin/11.4' into 11.5

Author: abarkov

.gitlab-ci.yml

@@ -447,41 +447,66 @@ fedora install:
       - installed-database.sql
       - upgraded-database.sql
 
-cppcheck: 
+cppcheck:
+  allow_failure: true
   stage: sast
   needs: []
   variables:
     GIT_STRATEGY: fetch
     GIT_SUBMODULE_STRATEGY: normal
   script:
     - yum install -y cppcheck diffutils
-    # --template: use a single-line template
+    # --template: output format
     # --force: check large directories without warning
     # -i<directory>: ignore this directory when scanning
+    # -I<directory>: include path, reduces false positives 
+    #                related to inability to resolve symbols
     # -j: run multiple cppcheck threads
     # Use newline to escape colon in yaml
     - >
-      cppcheck --template="{file}:{line}: {severity}: {message}" --force
+      cppcheck --template="{file}:{line}\n{code}\n{severity}: {message}" --force --check-level=exhaustive
       client dbug extra include libmariadb libmysqld libservices mysql-test mysys mysys_ssl pcre plugin
       strings tests unittest vio wsrep-lib sql sql-common storage
       -istorage/mroonga -istorage/tokudb -istorage/spider -istorage/rocksdb -iextra/ -ilibmariadb/ -istorage/columnstore
-      --output-file=cppcheck.txt -j $(nproc)
-    # Parallel jobs may output findings in an nondeterministic order. Sort to match ignorelist.
-    - cat cppcheck.txt | sort > cppcheck_sorted.txt
-    # Remove line numbers for diff
-    - sed 's/:[^:]*:/:/' cppcheck_sorted.txt > cppcheck_sorted_no_line_numbers.txt
+      -Iinclude -Istorage/innobase/include
+      --output-file=initial-cppcheck_output.txt -j $(nproc)
+    # when including {code} in the cppcheck template, some more pre-processing needs to be done
+    #
+    # sample cppcheck finding: <file>:<line>
+    #                          foo.bar()
+    #                          ^
+    #                          <severity>: <message>
+    #
+    # 1. remove all lines with "^"
+    # 2. merge every 3 lines into 1 so it can be sorted (example: <file> foo.bar() <severity>: <message>)
+    # 3. sort to match ignorelist since parallel jobs may output findings in an nondeterministic order
+    # 4. remove findings likely to be false positives (i.e, "unknown macros")
+    # 5. remove line numbers for diffing against ignorelist
+    - |
+      cat initial-cppcheck_output.txt | grep -v '\^$' > preprocessed-cppcheck_circumflex_removed.txt
+      cat preprocessed-cppcheck_circumflex_removed.txt | awk 'NR%3==1 {printf "%s", (NR==1) ? "" : "\n"; printf "%s", $0} NR%3!=1 {printf " %s", $0}' > preprocessed-cppcheck_oneline.txt
+      cat preprocessed-cppcheck_oneline.txt | sort > preprocessed-cppcheck_sorted.txt
+      cat preprocessed-cppcheck_sorted.txt | grep -v "There is an unknown macro here somewhere" > results-cppcheck_all_findings.txt
+      sed 's/:[0-9]\+//' results-cppcheck_all_findings.txt > preprocessed_final-cppcheck_no_line_nums.txt
     # Only print new issues not found in ignore list
     - echo "Problems found in ignore list that were not discovered by cppcheck (may have been fixed)."
-    - diff --changed-group-format='%>' --unchanged-group-format='' cppcheck_sorted_no_line_numbers.txt tests/code_quality/cppcheck_ignorelist.txt || true
+    - diff --changed-group-format='%>' --unchanged-group-format='' preprocessed_final-cppcheck_no_line_nums.txt tests/code_quality/cppcheck_ignorelist.txt || true
     - echo "Problems found by cppcheck that were not in ignore list."
-    - diff --changed-group-format='%<' --unchanged-group-format='' cppcheck_sorted_no_line_numbers.txt tests/code_quality/cppcheck_ignorelist.txt > lines_not_ignored.txt || true
-    - cat lines_not_ignored.txt && test ! -s lines_not_ignored.txt
+    - diff --changed-group-format='%<' --unchanged-group-format='' preprocessed_final-cppcheck_no_line_nums.txt tests/code_quality/cppcheck_ignorelist.txt > results-cppcheck_new_findings.txt || true
+    - cat results-cppcheck_new_findings.txt && test ! -s results-cppcheck_new_findings.txt
   artifacts:
     when: always
     paths:
-      - cppcheck_sorted.txt
+      # save all steps of pre-processing in-case it ever breaks
+      - initial-cppcheck_output.txt
+      - preprocessed-cppcheck_circumflex_removed.txt
+      - preprocessed-cppcheck_sorted.txt
+      - preprocessed_final-cppcheck_no_line_nums.txt
+      - results-cppcheck_all_findings.txt
+      - results-cppcheck_new_findings.txt
 
 flawfinder: 
+  allow_failure: true
   stage: sast
   needs: []
   variables:
@@ -503,11 +528,12 @@ flawfinder:
     - echo "Problems found in ignore list that were not discovered by flawfinder (may have been fixed)."
     - diff --changed-group-format='%>' --unchanged-group-format='' flawfinder-min-level5.json tests/code_quality/flawfinder_ignorelist.json || true
     - echo "Problems found by flawfinder that were not in ignore list."
-    - diff --changed-group-format='%<' --unchanged-group-format='' flawfinder-min-level5.json tests/code_quality/flawfinder_ignorelist.json > lines_not_ignored.txt || true
-    - cat lines_not_ignored.txt && test ! -s lines_not_ignored.txt
+    - diff --changed-group-format='%<' --unchanged-group-format='' flawfinder-min-level5.json tests/code_quality/flawfinder_ignorelist.json > flawfinder_new_findings.txt || true
+    - cat flawfinder_new_findings.txt && test ! -s flawfinder_new_findings.txt
   artifacts:
     when: always
     paths:
+      - flawfinder_new_findings.txt
       - flawfinder-all-vulnerabilities.html
       - flawfinder-min-level5.json
 

CMakeLists.txt

@@ -193,13 +193,20 @@ ELSE()
   SET (SKIP_COMPONENTS "N-O-N-E")
 ENDIF()
 
-OPTION(NOT_FOR_DISTRIBUTION "Allow linking with GPLv2-incompatible system libraries. Only set it you never plan to distribute the resulting binaries" OFF)
+
+IF("${MYSQL_NO_DASH_VERSION}" VERSION_LESS 11.2)
+  SET(MEMPROTECT_DEFAULT ON)
+ELSE()
+  SET(MEMPROTECT_DEFAULT OFF)
+ENDIF()
+
+OPTION(WITH_PROTECT_STATEMENT_MEMROOT "Enable protection of statement's memory root after first SP/PS execution. Turned into account only for debug build"
+${MEMPROTECT_DEFAULT})
 
 #
 #  Enable protection of statement's memory root after first SP/PS execution.
 #  Can be switched on only for debug build.
 #
-OPTION(WITH_PROTECT_STATEMENT_MEMROOT "Enable protection of statement's memory root after first SP/PS execution. Turned into account only for debug build" OFF)
 IF (WITH_PROTECT_STATEMENT_MEMROOT)
   SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DPROTECT_STATEMENT_MEMROOT")
   SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DPROTECT_STATEMENT_MEMROOT")

cmake/maintainer.cmake

@@ -27,15 +27,16 @@ SET(MY_WARNING_FLAGS
   -Wenum-conversion
   -Wextra
   -Wformat-security
+  -Winconsistent-missing-override
   -Wmissing-braces
   -Wno-format-truncation
   -Wno-init-self
   -Wno-nonnull-compare
   -Wno-null-conversion
   -Wno-unused-parameter
   -Wno-unused-private-field
-  -Woverloaded-virtual
   -Wnon-virtual-dtor
+  -Woverloaded-virtual
   -Wvla
   -Wwrite-strings
   )
@@ -44,7 +45,7 @@ FOREACH(F ${MY_WARNING_FLAGS})
   MY_CHECK_AND_SET_COMPILER_FLAG(${F} DEBUG RELWITHDEBINFO)
 ENDFOREACH()
 
-SET(MY_ERROR_FLAGS -Werror -fno-operator-names)
+SET(MY_ERROR_FLAGS -Werror -fno-operator-names -Wsuggest-override)
 
 IF(CMAKE_COMPILER_IS_GNUCC AND CMAKE_C_COMPILER_VERSION VERSION_LESS "6.0.0")
   SET(MY_ERROR_FLAGS ${MY_ERROR_FLAGS} -Wno-error=maybe-uninitialized)

cmake/wsrep.cmake

@@ -17,7 +17,9 @@
 #
 # Galera library does not compile with windows
 #
-IF (NOT WITHOUT_SERVER)
+IF(WITHOUT_SERVER)
+  OPTION(WITH_WSREP "Galera server compatibility in build client utilities" ON)
+ELSE()
 IF(UNIX)
   SET(with_wsrep_default ON)
 ELSE()
@@ -67,4 +69,4 @@ ENDIF()
 IF (NOT WIN32)
   ADD_FEATURE_INFO(WSREP WITH_WSREP "WSREP replication API (to use, e.g. Galera Replication library)")
 ENDIF()
-ENDIF(NOT WITHOUT_SERVER)
+ENDIF(WITHOUT_SERVER)

extra/mariabackup/xtrabackup.cc

@@ -1336,7 +1336,7 @@ enum options_xtrabackup
   OPT_INNODB_DATA_FILE_BUFFERING,
   OPT_INNODB_DATA_FILE_WRITE_THROUGH,
   OPT_INNODB_LOG_FILE_SIZE,
-  OPT_INNODB_LOG_FILES_IN_GROUP,
+  OPT_INNODB_LOG_WRITE_AHEAD_SIZE,
   OPT_INNODB_OPEN_FILES,
   OPT_XTRA_DEBUG_SYNC,
   OPT_INNODB_CHECKSUM_ALGORITHM,
@@ -1918,6 +1918,10 @@ struct my_option xb_server_options[] =
   {"innodb_log_group_home_dir", OPT_INNODB_LOG_GROUP_HOME_DIR,
    "Path to InnoDB log files.", &srv_log_group_home_dir,
    &srv_log_group_home_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+  {"innodb_log_write_ahead_size", OPT_INNODB_LOG_WRITE_AHEAD_SIZE,
+   "ib_logfile0 write size",
+   (G_PTR*) &log_sys.write_size, (G_PTR*) &srv_log_file_size, 0,
+   GET_UINT, REQUIRED_ARG, 512, 512, 4096, 0, 1, 0},
   {"innodb_max_dirty_pages_pct", OPT_INNODB_MAX_DIRTY_PAGES_PCT,
    "Percentage of dirty pages allowed in bufferpool.",
    (G_PTR*) &srv_max_buf_pool_modified_pct,
@@ -2366,6 +2370,11 @@ xb_get_one_option(const struct my_option *opt,
 
 static bool innodb_init_param()
 {
+	if (!ut_is_2pow(log_sys.write_size)) {
+		msg("InnoDB: innodb_log_write_ahead_size=%u"
+		    " is not a power of two", log_sys.write_size);
+		return true;
+	}
 	srv_is_being_started = TRUE;
 	/* === some variables from mysqld === */
 	memset((G_PTR) &mysql_tmpdir_list, 0, sizeof(mysql_tmpdir_list));
@@ -3390,7 +3399,7 @@ static bool xtrabackup_copy_logfile()
   ut_a(dst_log_file);
   ut_ad(recv_sys.is_initialised());
   const size_t sequence_offset{log_sys.is_encrypted() ? 8U + 5U : 5U};
-  const size_t block_size_1{log_sys.get_block_size() - 1};
+  const size_t block_size_1{log_sys.write_size - 1};
 
   ut_ad(!log_sys.is_pmem());
 
@@ -3465,7 +3474,7 @@ static bool xtrabackup_copy_logfile()
         if (r == recv_sys_t::GOT_EOF)
           break;
 
-        if (recv_sys.offset < log_sys.get_block_size())
+        if (recv_sys.offset < log_sys.write_size)
           break;
 
         if (xtrabackup_throttle && io_ticket-- < 0)

extra/perror.c

@@ -352,7 +352,8 @@ int main(int argc,char *argv[])
       {
         found= 1;
         if (verbose)
-          printf("MariaDB error code %3d (%s): %s\n", code, name, msg);
+          printf("MariaDB error code %3d (%s): %s\n"
+                 "Learn more: https://mariadb.com/kb/en/e%3d/\n", code, name, msg, code);
         else
           puts(msg);
       }

mysql-test/include/ctype_myanmar.inc

@@ -1293,11 +1293,9 @@ INSERT INTO t1 (s1) VALUES
 (_ucs2 0x101C1000103A10181000103A),
 (_ucs2 0x101C103910181000103A /* tea */);
 
-# enable view-protocol after fix MDEV-27871
---disable_view_protocol
-
-SELECT id, IF(LEFT(s1,1)='-',s1,CONCAT(HEX(WEIGHT_STRING(s1)),'\t', HEX(CONVERT(s1 USING ucs2)))) FROM t1 ORDER BY id;
---enable_view_protocol
+--disable_service_connection
+SELECT id, IF(LEFT(s1,1)='-',s1,CONCAT(HEX(WEIGHT_STRING(s1)),'\t', HEX(CONVERT(s1 USING ucs2)))) as exp FROM t1 ORDER BY id;
+--enable_service_connection
 
 DROP TABLE t1;