Merge 10.6 into 10.11

client/CMakeLists.txt

@@ -16,7 +16,7 @@
 
 INCLUDE_DIRECTORIES(
   ${CMAKE_SOURCE_DIR}/include
-  ${PCRE_INCLUDES}
+  ${PCRE_INCLUDE_DIRS}
   ${CMAKE_SOURCE_DIR}/mysys_ssl
   ${ZLIB_INCLUDE_DIR}
   ${SSL_INCLUDE_DIRS}

cmake/os/AIX.cmake

@@ -34,5 +34,8 @@ ELSE()
   SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES -maix64 -pthread -mcmodel=large")
 ENDIF()
 
+# fcntl(fd, F_SETFL, O_DIRECT) is not supported; O_DIRECT is an open(2) flag
+SET(HAVE_FCNTL_DIRECT 0 CACHE INTERNAL "")
+
 # make it WARN by default, not AUTO (that implies -Werror)
 SET(MYSQL_MAINTAINER_MODE "WARN" CACHE STRING "Enable MariaDB maintainer-specific warnings. One of: NO (warnings are disabled) WARN (warnings are enabled) ERR (warnings are errors) AUTO (warnings are errors in Debug only)")

cmake/os/SunOS.cmake

@@ -17,6 +17,10 @@ INCLUDE(CheckSymbolExists)
 INCLUDE(CheckCSourceRuns)
 INCLUDE(CheckCSourceCompiles) 
 
+# fcntl(fd, F_SETFL, O_DIRECT) is not supported,
+# and directio(3C) would only work on UFS or NFS, not ZFS.
+SET(HAVE_FCNTL_DIRECT 0 CACHE INTERNAL "")
+
 # Enable 64 bit file offsets
 SET(_FILE_OFFSET_BITS 64)
 

cmake/os/WindowsCache.cmake

@@ -44,6 +44,7 @@ SET(HAVE_EXECINFO_H CACHE  INTERNAL "")
 SET(HAVE_FCHMOD CACHE  INTERNAL "")
 SET(HAVE_FCNTL CACHE  INTERNAL "")
 SET(HAVE_FCNTL_H 1 CACHE  INTERNAL "")
+SET(HAVE_FCNTL_DIRECT 0 CACHE  INTERNAL "")
 SET(HAVE_FCNTL_NONBLOCK CACHE  INTERNAL "")
 SET(HAVE_FDATASYNC CACHE  INTERNAL "")
 SET(HAVE_DECL_FDATASYNC CACHE INTERNAL "")

cmake/pcre.cmake

@@ -1,12 +1,12 @@
-INCLUDE (CheckCSourceRuns)
 INCLUDE (ExternalProject)
 
 SET(WITH_PCRE "auto" CACHE STRING
    "Which pcre to use (possible values are 'bundled', 'system', or 'auto')")
 
 MACRO(BUNDLE_PCRE2)
   SET(dir "${CMAKE_BINARY_DIR}/extra/pcre2")
-  SET(PCRE_INCLUDES ${dir}/src/pcre2-build ${dir}/src/pcre2/src)
+  SET(PCRE_INCLUDE_DIRS ${dir}/src/pcre2-build ${dir}/src/pcre2/src)
+  MESSAGE(STATUS "Will download and bundle pcre2")
   SET(byproducts)
   FOREACH(lib pcre2-posix pcre2-8)
     ADD_LIBRARY(${lib} STATIC IMPORTED GLOBAL)
@@ -76,18 +76,26 @@ SET_TARGET_PROPERTIES(pcre2 PROPERTIES EXCLUDE_FROM_ALL TRUE)
 ENDMACRO()
 
 MACRO (CHECK_PCRE)
-  IF(WITH_PCRE STREQUAL "system" OR WITH_PCRE STREQUAL "auto")
-    CHECK_LIBRARY_EXISTS(pcre2-8 pcre2_match_8 "" HAVE_PCRE2)
-  ENDIF()
-  IF(NOT HAVE_PCRE2 OR WITH_PCRE STREQUAL "bundled")
-    IF (WITH_PCRE STREQUAL "system")
-      MESSAGE(FATAL_ERROR "system pcre2-8 library is not found or unusable")
+  IF (NOT TARGET pcre2 AND NOT PCRE_FOUND)
+    IF(WITH_PCRE STREQUAL "system" OR WITH_PCRE STREQUAL "auto")
+      FIND_PACKAGE(PkgConfig QUIET)
+      PKG_CHECK_MODULES(PCRE libpcre2-8)
+      # in case pkg-config or libpcre2-8.pc is not installed:
+      IF(NOT PCRE_FOUND)
+        UNSET(PCRE_FOUND CACHE)
+        CHECK_LIBRARY_EXISTS(pcre2-8 pcre2_match_8 "" PCRE_FOUND)
+      ENDIF()
     ENDIF()
-    BUNDLE_PCRE2()
-  ELSE()
-    CHECK_LIBRARY_EXISTS(pcre2-posix PCRE2regcomp "" NEEDS_PCRE2_DEBIAN_HACK)
-    IF(NEEDS_PCRE2_DEBIAN_HACK)
-      SET(PCRE2_DEBIAN_HACK "-Dregcomp=PCRE2regcomp -Dregexec=PCRE2regexec -Dregerror=PCRE2regerror -Dregfree=PCRE2regfree")
+    IF(NOT PCRE_FOUND OR WITH_PCRE STREQUAL "bundled")
+      IF (WITH_PCRE STREQUAL "system")
+        MESSAGE(FATAL_ERROR "system pcre2-8 library is not found or unusable")
+      ENDIF()
+      BUNDLE_PCRE2()
+    ELSE()
+      CHECK_LIBRARY_EXISTS(pcre2-posix PCRE2regcomp "" NEEDS_PCRE2_DEBIAN_HACK)
+      IF(NEEDS_PCRE2_DEBIAN_HACK)
+        SET(PCRE2_DEBIAN_HACK "-Dregcomp=PCRE2regcomp -Dregexec=PCRE2regexec -Dregerror=PCRE2regerror -Dregfree=PCRE2regfree")
+      ENDIF()
     ENDIF()
   ENDIF()
 ENDMACRO()

cmake/plugin.cmake

@@ -44,7 +44,7 @@ MACRO(MYSQL_ADD_PLUGIN)
   # Add common include directories
   INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include 
                     ${CMAKE_SOURCE_DIR}/sql
-                    ${PCRE_INCLUDES}
+                    ${PCRE_INCLUDE_DIRS}
                     ${SSL_INCLUDE_DIRS}
                     ${ZLIB_INCLUDE_DIR})
 

config.h.cmake

@@ -30,6 +30,7 @@
 #cmakedefine HAVE_DLFCN_H 1
 #cmakedefine HAVE_EXECINFO_H 1
 #cmakedefine HAVE_FCNTL_H 1
+#cmakedefine HAVE_FCNTL_DIRECT 1
 #cmakedefine HAVE_FENV_H 1
 #cmakedefine HAVE_FLOAT_H 1
 #cmakedefine HAVE_FNMATCH_H 1

configure.cmake

@@ -706,6 +706,7 @@ CHECK_SYMBOL_EXISTS(O_NONBLOCK "unistd.h;fcntl.h" HAVE_FCNTL_NONBLOCK)
 IF(NOT HAVE_FCNTL_NONBLOCK)
  SET(NO_FCNTL_NONBLOCK 1)
 ENDIF()
+CHECK_SYMBOL_EXISTS(O_DIRECT "fcntl.h" HAVE_FCNTL_DIRECT)
 
 #
 # Test for how the C compiler does inline, if at all

extra/mariabackup/CMakeLists.txt

@@ -36,7 +36,7 @@ INCLUDE_DIRECTORIES(
   )
 
 IF(NOT HAVE_SYSTEM_REGEX)
-  INCLUDE_DIRECTORIES(${PCRE_INCLUDES})
+  INCLUDE_DIRECTORIES(${PCRE_INCLUDE_DIRS})
   ADD_DEFINITIONS(${PCRE2_DEBIAN_HACK})
 ENDIF()
 

extra/mariabackup/fil_cur.cc

@@ -199,11 +199,13 @@ xb_fil_cur_open(
 		return(XB_FIL_CUR_SKIP);
 	}
 
+#ifdef HAVE_FCNTL_DIRECT
 	if (srv_file_flush_method == SRV_O_DIRECT
 	    || srv_file_flush_method == SRV_O_DIRECT_NO_FSYNC) {
 
 		os_file_set_nocache(cursor->file, node->name, "OPEN");
 	}
+#endif
 
 	posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL);
 

libmysqld/CMakeLists.txt

@@ -23,7 +23,7 @@ ${CMAKE_SOURCE_DIR}/libmysqld
 ${CMAKE_SOURCE_DIR}/sql
 ${CMAKE_SOURCE_DIR}/tpool
 ${CMAKE_BINARY_DIR}/sql 
-${PCRE_INCLUDES}
+${PCRE_INCLUDE_DIRS}
 ${LIBFMT_INCLUDE_DIR}
 ${ZLIB_INCLUDE_DIR}
 ${SSL_INCLUDE_DIRS}

libmysqld/examples/CMakeLists.txt

@@ -15,7 +15,7 @@
 
 INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include
                     ${CMAKE_SOURCE_DIR}/libmysqld/include
-                    ${PCRE_INCLUDES}
+                    ${PCRE_INCLUDE_DIRS}
                     ${CMAKE_SOURCE_DIR}/sql
                     ${MY_READLINE_INCLUDE_DIR}
                     )

mysql-test/mariadb-test-run.pl

@@ -4506,7 +4506,7 @@ ($$)
      qr|InnoDB: io_setup\(\) failed with EAGAIN|,
      qr|io_uring_queue_init\(\) failed with|,
      qr|InnoDB: liburing disabled|,
-     qr/InnoDB: Failed to set (O_DIRECT|DIRECTIO_ON) on file/,
+     qr/InnoDB: Failed to set O_DIRECT on file/,
      qr|setrlimit could not change the size of core files to 'infinity';|,
      qr|feedback plugin: failed to retrieve the MAC address|,
      qr|Plugin 'FEEDBACK' init function returned error|,

mysql-test/suite/innodb/t/doublewrite.test

@@ -39,6 +39,9 @@ commit work;
 SET GLOBAL innodb_fast_shutdown = 0;
 let $shutdown_timeout=;
 --source include/restart_mysqld.inc
+# Ensure that buf_flush_page_cleaner() has woken up from its
+# first my_cond_timedwait() and gone idle.
+sleep 1;
 --source ../include/no_checkpoint_start.inc
 connect (dml,localhost,root,,);
 XA START 'x';

mysql-test/suite/perfschema/t/sxlock_func.test

@@ -5,7 +5,6 @@
 --source include/not_embedded.inc
 --source include/have_perfschema.inc
 --source include/have_innodb.inc
---source include/maybe_debug.inc
 
 UPDATE performance_schema.setup_instruments SET enabled = 'NO', timed = 'YES';
 

plugin/feedback/CMakeLists.txt

@@ -1,5 +1,5 @@
 INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql
-                    ${PCRE_INCLUDES}
+                    ${PCRE_INCLUDE_DIRS}
                     ${SSL_INCLUDE_DIRS})
 
 SET(FEEDBACK_SOURCES feedback.cc sender_thread.cc

plugin/qc_info/CMakeLists.txt

@@ -1,4 +1,4 @@
 INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql
-                    ${PCRE_INCLUDES})
+                    ${PCRE_INCLUDE_DIRS})
 
 MYSQL_ADD_PLUGIN(QUERY_CACHE_INFO qc_info.cc RECOMPILE_FOR_EMBEDDED)

scripts/mysql_install_db.sh

@@ -695,7 +695,7 @@ then
   then
     echo
     echo "You can start the MariaDB daemon with:"
-    echo "cd '$basedir' ; $bindir/mariadb-safe --datadir='$ldata'"
+    echo "cd '$basedir' ; $bindir/mariadbd-safe --datadir='$ldata'"
     echo
     echo "You can test the MariaDB daemon with mysql-test-run.pl"
     echo "cd '$basedir/@INSTALL_MYSQLTESTDIR@' ; perl mariadb-test-run.pl"

sql/CMakeLists.txt

@@ -55,7 +55,7 @@ INCLUDE_DIRECTORIES(
 ${CMAKE_SOURCE_DIR}/include
 ${CMAKE_SOURCE_DIR}/sql
 ${LIBFMT_INCLUDE_DIR}
-${PCRE_INCLUDES}
+${PCRE_INCLUDE_DIRS}
 ${ZLIB_INCLUDE_DIR}
 ${SSL_INCLUDE_DIRS}
 ${CMAKE_BINARY_DIR}/sql

storage/innobase/buf/buf0dump.cc

@@ -180,7 +180,7 @@ static void buf_dump_generate_path(char *path, size_t path_size)
 	char	buf[FN_REFLEN];
 
 	mysql_mutex_lock(&LOCK_global_system_variables);
-	snprintf(buf, sizeof buf, "%s/%s", get_buf_dump_dir(),
+	snprintf(buf, sizeof buf, "%s" FN_ROOTDIR "%s", get_buf_dump_dir(),
 		 srv_buf_dump_filename);
 	mysql_mutex_unlock(&LOCK_global_system_variables);
 
@@ -214,7 +214,7 @@ static void buf_dump_generate_path(char *path, size_t path_size)
 			format = "%s%s";
 			break;
 		default:
-			format = "%s/%s";
+			format = "%s" FN_ROOTDIR "%s";
 		}
 
 		snprintf(path, path_size, format,

storage/innobase/buf/buf0flu.cc

@@ -1759,6 +1759,28 @@ ulint buf_flush_LRU(ulint max_n, bool evict)
     buf_pool.try_LRU_scan= true;
     pthread_cond_broadcast(&buf_pool.done_free);
   }
+  else if (!pages && !buf_pool.try_LRU_scan &&
+           !buf_pool.LRU_warned.test_and_set(std::memory_order_acquire))
+  {
+    /* For example, with the minimum innodb_buffer_pool_size=5M and
+    the default innodb_page_size=16k there are only a little over 316
+    pages in the buffer pool. The buffer pool can easily be exhausted
+    by a workload of some dozen concurrent connections. The system could
+    reach a deadlock like the following:
+
+    (1) Many threads are waiting in buf_LRU_get_free_block()
+    for buf_pool.done_free.
+    (2) Some threads are waiting for a page latch which is held by
+    another thread that is waiting in buf_LRU_get_free_block().
+    (3) This thread is the only one that could make progress, but
+    we fail to do so because all the pages that we scanned are
+    buffer-fixed or latched by some thread. */
+    sql_print_warning("InnoDB: Could not free any blocks in the buffer pool!"
+                      " %zu blocks are in use and %zu free."
+                      " Consider increasing innodb_buffer_pool_size.",
+                      UT_LIST_GET_LEN(buf_pool.LRU),
+                      UT_LIST_GET_LEN(buf_pool.free));
+  }
 
   return pages;
 }
@@ -2391,11 +2413,19 @@ static ulint page_cleaner_flush_pages_recommendation(ulint last_pages_in,
 	goto func_exit;
 }
 
+TPOOL_SUPPRESS_TSAN
+bool buf_pool_t::need_LRU_eviction() const
+{
+  /* try_LRU_scan==false means that buf_LRU_get_free_block() is waiting
+  for buf_flush_page_cleaner() to evict some blocks */
+  return UNIV_UNLIKELY(!try_LRU_scan ||
+                       (UT_LIST_GET_LEN(LRU) > BUF_LRU_MIN_LEN &&
+                        UT_LIST_GET_LEN(free) < srv_LRU_scan_depth / 2));
+}
+
 #if defined __aarch64__&&defined __GNUC__&&__GNUC__==4&&!defined __clang__
-/* Avoid GCC 4.8.5 internal compiler error "could not split insn".
-We would only need this for buf_flush_page_cleaner(),
-but GCC 4.8.5 does not support pop_options. */
-# pragma GCC optimize ("O0")
+/* Avoid GCC 4.8.5 internal compiler error "could not split insn". */
+__attribute__((optimize(0)))
 #endif
 /** page_cleaner thread tasked with flushing dirty pages from the buffer
 pools. As of now we'll have only one coordinator. */
@@ -2429,21 +2459,24 @@ static void buf_flush_page_cleaner()
     }
 
     mysql_mutex_lock(&buf_pool.flush_list_mutex);
-    if (buf_pool.ran_out())
-      goto no_wait;
-    else if (srv_shutdown_state > SRV_SHUTDOWN_INITIATED)
-      break;
+    if (!buf_pool.need_LRU_eviction())
+    {
+      if (srv_shutdown_state > SRV_SHUTDOWN_INITIATED)
+        break;
 
-    if (buf_pool.page_cleaner_idle() &&
-        (!UT_LIST_GET_LEN(buf_pool.flush_list) ||
-         srv_max_dirty_pages_pct_lwm == 0.0))
-      /* We are idle; wait for buf_pool.page_cleaner_wakeup() */
-      my_cond_wait(&buf_pool.do_flush_list,
-                   &buf_pool.flush_list_mutex.m_mutex);
-    else
-      my_cond_timedwait(&buf_pool.do_flush_list,
-                        &buf_pool.flush_list_mutex.m_mutex, &abstime);
-  no_wait:
+      if (buf_pool.page_cleaner_idle() &&
+          (!UT_LIST_GET_LEN(buf_pool.flush_list) ||
+           srv_max_dirty_pages_pct_lwm == 0.0))
+      {
+        buf_pool.LRU_warned.clear(std::memory_order_release);
+        /* We are idle; wait for buf_pool.page_cleaner_wakeup() */
+        my_cond_wait(&buf_pool.do_flush_list,
+                     &buf_pool.flush_list_mutex.m_mutex);
+      }
+      else
+        my_cond_timedwait(&buf_pool.do_flush_list,
+                          &buf_pool.flush_list_mutex.m_mutex, &abstime);
+    }
     set_timespec(abstime, 1);
 
     lsn_limit= buf_flush_sync_lsn;
@@ -2475,7 +2508,7 @@ static void buf_flush_page_cleaner()
       }
       while (false);
 
-      if (!buf_pool.ran_out())
+      if (!buf_pool.need_LRU_eviction())
         continue;
       mysql_mutex_lock(&buf_pool.flush_list_mutex);
       oldest_lsn= buf_pool.get_oldest_modification(0);
@@ -2504,7 +2537,7 @@ static void buf_flush_page_cleaner()
       if (oldest_lsn >= soft_lsn_limit)
         buf_flush_async_lsn= soft_lsn_limit= 0;
     }
-    else if (buf_pool.ran_out())
+    else if (buf_pool.need_LRU_eviction())
     {
       buf_pool.page_cleaner_set_idle(false);
       buf_pool.n_flush_inc();
@@ -2619,9 +2652,13 @@ static void buf_flush_page_cleaner()
                                    MONITOR_FLUSH_ADAPTIVE_PAGES,
                                    n_flushed);
     }
-    else if (buf_flush_async_lsn <= oldest_lsn)
+    else if (buf_flush_async_lsn <= oldest_lsn &&
+             !buf_pool.need_LRU_eviction())
       goto check_oldest_and_set_idle;
+    else
+      mysql_mutex_lock(&buf_pool.mutex);
 
+    n= srv_max_io_capacity;
     n= n >= n_flushed ? n - n_flushed : 0;
     goto LRU_flush;
   }