Merge 10.5 into 10.6

Author: dr-m

client/mysqldump.c

@@ -42,6 +42,11 @@
 /* on merge conflict, bump to a higher version again */
 #define DUMP_VERSION "10.19"
 
+/**
+  First mysql version supporting sequences.
+*/
+#define FIRST_SEQUENCE_VERSION 100300
+
 #include <my_global.h>
 #include <my_sys.h>
 #include <my_user.h>
@@ -93,6 +98,11 @@
 /* Max length GTID position that we will output. */
 #define MAX_GTID_LENGTH 1024
 
+/* Dump sequence/tables control */
+#define DUMP_TABLE_ALL -1
+#define DUMP_TABLE_TABLE 0
+#define DUMP_TABLE_SEQUENCE 1
+
 static my_bool ignore_table_data(const uchar *hash_key, size_t len);
 static void add_load_option(DYNAMIC_STRING *str, const char *option,
                              const char *option_value);
@@ -1079,6 +1089,22 @@ static int get_options(int *argc, char ***argv)
   if ((ho_error= handle_options(argc, argv, my_long_options, get_one_option)))
     return(ho_error);
 
+  /*
+    Dumping under --system=stats with --replace or --inser-ignore is safe and will not
+    retult into race condition. Otherwise dump only structure and ignore data by default
+    while dumping.
+  */
+  if (!(opt_system & OPT_SYSTEM_STATS) && !(opt_ignore || opt_replace_into))
+  {
+    if (my_hash_insert(&ignore_data,
+                       (uchar*) my_strdup(PSI_NOT_INSTRUMENTED,
+                                          "mysql.innodb_index_stats", MYF(MY_WME))) ||
+        my_hash_insert(&ignore_data,
+                       (uchar*) my_strdup(PSI_NOT_INSTRUMENTED,
+                                          "mysql.innodb_table_stats", MYF(MY_WME))))
+      return(EX_EOM);
+  }
+
   if (opt_system & OPT_SYSTEM_ALL)
       opt_system|= ~0;
 
@@ -3931,14 +3957,6 @@ static void dump_table(const char *table, const char *db, const uchar *hash_key,
   MYSQL_ROW     row;
   DBUG_ENTER("dump_table");
 
-  /*
-    Check does table has a sequence structure and if has apply different sql queries
-  */
-  if (check_if_ignore_table(table, table_type) & IGNORE_SEQUENCE_TABLE)
-  {
-    get_sequence_structure(table, db);
-    DBUG_VOID_RETURN;
-  }
   /*
     Make sure you get the create table info before the following check for
     --no-data flag below. Otherwise, the create table info won't be printed.
@@ -4440,18 +4458,36 @@ static void dump_table(const char *table, const char *db, const uchar *hash_key,
 } /* dump_table */
 
 
-static char *getTableName(int reset)
+static char *getTableName(int reset, int want_sequences)
 {
   MYSQL_ROW row;
 
   if (!get_table_name_result)
   {
-    if (!(get_table_name_result= mysql_list_tables(mysql,NullS)))
-      return(NULL);
+    if (mysql_get_server_version(mysql) >= FIRST_SEQUENCE_VERSION)
+    {
+      const char *query= "SHOW FULL TABLES";
+      if (mysql_query_with_error_report(mysql, 0, query))
+        return (NULL);
+
+      if (!(get_table_name_result= mysql_store_result(mysql)))
+        return (NULL);
+    }
+    else
+    {
+      if (!(get_table_name_result= mysql_list_tables(mysql,NullS)))
+        return(NULL);
+    }
   }
   if ((row= mysql_fetch_row(get_table_name_result)))
-    return((char*) row[0]);
+  {
+    if (want_sequences != DUMP_TABLE_ALL)
+      while (row && MY_TEST(strcmp(row[1], "SEQUENCE")) == want_sequences)
+        row= mysql_fetch_row(get_table_name_result);
 
+    if (row)
+      return((char*) row[0]);
+  }
   if (reset)
     mysql_data_seek(get_table_name_result,0);      /* We want to read again */
   else
@@ -4844,14 +4880,16 @@ static int dump_all_servers()
 
 static int dump_all_stats()
 {
-  my_bool prev_no_create_info;
+  my_bool prev_no_create_info, prev_opt_replace_into;
 
   if (mysql_select_db(mysql, "mysql"))
   {
     DB_error(mysql, "when selecting the database");
     return 1;                   /* If --force */
   }
   fprintf(md_result_file,"\nUSE mysql;\n");
+  prev_opt_replace_into= opt_replace_into;
+  opt_replace_into|= !opt_ignore;
   prev_no_create_info= opt_no_create_info;
   opt_no_create_info= 1; /* don't overwrite recreate tables */
   /* EITS added in 10.0.1 */
@@ -4870,6 +4908,7 @@ static int dump_all_stats()
     dump_table("innodb_table_stats", "mysql", NULL, 0);
   }
   opt_no_create_info= prev_no_create_info;
+  opt_replace_into= prev_opt_replace_into;
   return 0;
 }
 
@@ -4880,12 +4919,14 @@ static int dump_all_stats()
 
 static int dump_all_timezones()
 {
-  my_bool opt_prev_no_create_info;
+  my_bool opt_prev_no_create_info, opt_prev_replace_into;
   if (mysql_select_db(mysql, "mysql"))
   {
     DB_error(mysql, "when selecting the database");
     return 1;                   /* If --force */
   }
+  opt_prev_replace_into= opt_replace_into;
+  opt_replace_into|= !opt_ignore;
   opt_prev_no_create_info= opt_no_create_info;
   opt_no_create_info= 1;
   fprintf(md_result_file,"\nUSE mysql;\n");
@@ -4895,6 +4936,7 @@ static int dump_all_timezones()
   dump_table("time_zone_transition", "mysql", NULL, 0);
   dump_table("time_zone_transition_type", "mysql", NULL, 0);
   opt_no_create_info= opt_prev_no_create_info;
+  opt_replace_into= opt_prev_replace_into;
   return 0;
 }
 
@@ -5384,7 +5426,7 @@ static int dump_all_tables_in_db(char *database)
   {
     DYNAMIC_STRING query;
     init_dynamic_string_checked(&query, "LOCK TABLES ", 256, 1024);
-    for (numrows= 0 ; (table= getTableName(1)) ; )
+    for (numrows= 0 ; (table= getTableName(1, DUMP_TABLE_ALL)) ; )
     {
       char *end= strmov(afterdot, table);
       if (include_table((uchar*) hash_key,end - hash_key))
@@ -5418,7 +5460,19 @@ static int dump_all_tables_in_db(char *database)
       DBUG_RETURN(1);
     }
   }
-  while ((table= getTableName(0)))
+
+  if (mysql_get_server_version(mysql) >= FIRST_SEQUENCE_VERSION &&
+      !opt_no_create_info)
+  {
+    // First process sequences
+    while ((table= getTableName(1, DUMP_TABLE_SEQUENCE)))
+    {
+      char *end= strmov(afterdot, table);
+      if (include_table((uchar*) hash_key, end - hash_key))
+        get_sequence_structure(table, database);
+    }
+  }
+  while ((table= getTableName(0, DUMP_TABLE_TABLE)))
   {
     char *end= strmov(afterdot, table);
     if (include_table((uchar*) hash_key, end - hash_key))
@@ -5567,7 +5621,7 @@ static my_bool dump_all_views_in_db(char *database)
   {
     DYNAMIC_STRING query;
     init_dynamic_string_checked(&query, "LOCK TABLES ", 256, 1024);
-    for (numrows= 0 ; (table= getTableName(1)); )
+    for (numrows= 0 ; (table= getTableName(1, DUMP_TABLE_TABLE)); )
     {
       char *end= strmov(afterdot, table);
       if (include_table((uchar*) hash_key,end - hash_key))
@@ -5590,7 +5644,7 @@ static my_bool dump_all_views_in_db(char *database)
     else
       verbose_msg("-- dump_all_views_in_db : logs flushed successfully!\n");
   }
-  while ((table= getTableName(0)))
+  while ((table= getTableName(0, DUMP_TABLE_TABLE)))
   {
     char *end= strmov(afterdot, table);
     if (include_table((uchar*) hash_key, end - hash_key))
@@ -5720,7 +5774,7 @@ static int get_sys_var_lower_case_table_names()
 
 static int dump_selected_tables(char *db, char **table_names, int tables)
 {
-  char table_buff[NAME_LEN*2+3];
+  char table_buff[NAME_LEN*2+3], table_type[NAME_LEN];
   DYNAMIC_STRING lock_tables_query;
   char **dump_tables, **pos, **end;
   int lower_case_table_names;
@@ -5817,9 +5871,22 @@ static int dump_selected_tables(char *db, char **table_names, int tables)
       DBUG_RETURN(1);
     }
   }
+
+  if (mysql_get_server_version(mysql) >= FIRST_SEQUENCE_VERSION)
+  {
+    /* Dump Sequence first */
+    for (pos= dump_tables; pos < end; pos++)
+    {
+      DBUG_PRINT("info",("Dumping sequence(?) %s", *pos));
+      if (check_if_ignore_table(*pos, table_type) & IGNORE_SEQUENCE_TABLE)
+        get_sequence_structure(*pos, db);
+    }
+  }
   /* Dump each selected table */
   for (pos= dump_tables; pos < end; pos++)
   {
+    if (check_if_ignore_table(*pos, table_type) & IGNORE_SEQUENCE_TABLE)
+      continue;
     DBUG_PRINT("info",("Dumping table %s", *pos));
     dump_table(*pos, db, NULL, 0);
     if (opt_dump_triggers &&

cmake/pcre.cmake

@@ -36,8 +36,8 @@ MACRO(BUNDLE_PCRE2)
   ExternalProject_Add(
     pcre2
     PREFIX   "${dir}"
-    URL      "http://ftp.pcre.org/pub/pcre/pcre2-10.34.zip"
-    URL_MD5  fdb10dba7f3be43730966bebdd3755ef
+    URL      "http://ftp.pcre.org/pub/pcre/pcre2-10.36.zip"
+    URL_MD5  ba9e743af42aac5642f7504b12af4116
     INSTALL_COMMAND ""
     CMAKE_ARGS
       "-DPCRE2_BUILD_TESTS=OFF"

debian/autobake-deb.sh

@@ -18,10 +18,20 @@ set -e
 export DEB_BUILD_OPTIONS="nocheck $DEB_BUILD_OPTIONS"
 
 # Take the files and part of control from MCS directory
-if [[ -d storage/columnstore/columnstore/debian ]]; then
+if [[ -d storage/columnstore/columnstore/debian ]]
+then
   cp -v storage/columnstore/columnstore/debian/mariadb-plugin-columnstore.* debian/
   echo >> debian/control
   cat storage/columnstore/columnstore/debian/control >> debian/control
+
+  # From Debian Bullseye/Ubuntu Hirsute onwards libreadline is gone, so build with it
+  # only on older releases where it is still available. This can be removed once
+  # MCOL-4535 lands in MariaDB.
+  if apt-cache madison libreadline-gplv2-dev | grep 'libreadline-gplv2-dev' >/dev/null 2>&1
+  then
+    sed 's/libpcre2-dev,/libpcre2-dev, libreadline-gplv2-dev [amd64],/' -i debian/control
+  fi
+
   # ColumnStore is explcitly disabled in the native build, so allow it now
   # when build it when triggered by autobake-deb.sh
   sed '/-DPLUGIN_COLUMNSTORE=NO/d' -i debian/rules
@@ -60,6 +70,13 @@ then
   sed "/Package: libmariadbd-dev/,/^$/d" -i debian/control
 fi
 
+## Skip TokuDB if arch is not amd64
+if [[ ! $(dpkg-architecture -q DEB_BUILD_ARCH) =~ amd64 ]]
+then
+  sed '/Package: mariadb-plugin-tokudb/,/^$/d' -i debian/control
+fi
+
+
 if [[ $(arch) =~ i[346]86 ]]
 then
   sed "/Package: mariadb-plugin-rocksdb/,/^$/d" -i debian/control
@@ -75,7 +92,7 @@ then
   echo "usr/bin/sst_dump" >> debian/mariadb-plugin-rocksdb.install
 fi
 
-# From Debian Buster/Ubuntu Bionic, libcurl4 replaces libcurl3.
+# From Debian Buster/Ubuntu Bionic, libcurl4 replaces libcurl3
 if ! apt-cache madison libcurl4 | grep 'libcurl4' >/dev/null 2>&1
 then
   sed 's/libcurl4/libcurl3/g' -i debian/control

debian/control

@@ -35,7 +35,6 @@ Build-Depends: bison,
                libnuma-dev [linux-any],
                libpam0g-dev,
                libpcre2-dev,
-               libreadline-gplv2-dev [amd64],
                libsnappy-dev,
                libssl-dev,
                libssl-dev:native,

debian/salsa-ci.yml

@@ -37,6 +37,14 @@ build:
     - apt-get update && eatmydata apt-get install --yes --no-install-recommends aptitude devscripts ccache equivs
     - cd ${WORKING_DIR}/${SOURCE_DIR}
     - eatmydata install-build-deps.sh .
+    - |
+      # From Debian Bullseye/Ubuntu Hirsute onwards libreadline is gone, so build with it
+      # only on older releases where it is still available. This can be removed once
+      # MCOL-4535 lands in MariaDB.
+      if apt-cache madison libreadline-gplv2-dev | grep 'libreadline-gplv2-dev' >/dev/null 2>&1
+      then
+       apt-get install --yes libreadline-gplv2-dev
+      fi
     - update-ccache-symlinks; ccache -z # Zero out ccache counters
     - while true; do sleep 600; echo "10 minutes passed" >&2; done & # Progress keeper since build is long and silent
     - debian/autobake-deb.sh |& tail -n 10000 # Keep Gitlab-CI output under 4 MB
@@ -146,6 +154,12 @@ blhc:
   # Prime the apt cache so later apt commands can run
   apt-get update
 
+# Readline was removed from Debian Sid (and Bullseye) in Feb 2021. To be able to install older
+# versions of MariaDB that depend on it, fetch and install it from Buster.
+.test-install-readline-in-sid-for-backwards-compat: &test-install-readline-in-sid-for-backwards-compat |
+  curl -O http://ftp.de.debian.org/debian/pool/main/r/readline5/libreadline5_5.2+dfsg-3+b13_amd64.deb
+  apt install -y ./libreadline5_5.2+dfsg-3+b13_amd64.deb
+
 .test-verify-initial: &test-verify-initial |
   dpkg -l | grep -iE 'maria|mysql|galera' || true # List installed
   service mysql status || service mariadb status # Early MariaDB 10.5 only had 'mariadb'
@@ -664,6 +678,7 @@ mariadb.org-10.5 to mariadb-10.6 upgrade:
     - curl -sS https://mariadb.org/mariadb_release_signing_key.asc -o /etc/apt/trusted.gpg.d/mariadb.asc
     - echo 'deb http://mirror.one.com/mariadb/repo/10.5/debian sid main' > /etc/apt/sources.list.d/mariadb.list
     - apt-get update
+    - *test-install-readline-in-sid-for-backwards-compat
     # Package libmariadbclient-dev from mariadb.org conficts with libmariadb-dev in Sid, so cannot use wildcard that would include it
     - apt-get install -y 'mariadb*' libmariadb3 'libmariadb-*' 'libmariadbd*'
     - *test-verify-initial
@@ -698,6 +713,7 @@ mariadb.org-10.4 to mariadb-10.6 upgrade:
     - curl -sS https://mariadb.org/mariadb_release_signing_key.asc -o /etc/apt/trusted.gpg.d/mariadb.asc
     - echo 'deb http://mirror.one.com/mariadb/repo/10.4/debian sid main' > /etc/apt/sources.list.d/mariadb.list
     - apt-get update
+    - *test-install-readline-in-sid-for-backwards-compat
     - apt-get install -y mariadb-server-10.4
     # MariaDB.org version of 10.4 and early 10.5 do not install an init file, so
     # it must be installed here manually
@@ -731,6 +747,7 @@ mariadb.org-10.3 to mariadb-10.6 upgrade:
     - curl -sS https://mariadb.org/mariadb_release_signing_key.asc -o /etc/apt/trusted.gpg.d/mariadb.asc
     - echo 'deb http://mirror.one.com/mariadb/repo/10.3/debian sid main' > /etc/apt/sources.list.d/mariadb.list
     - apt-get update
+    - *test-install-readline-in-sid-for-backwards-compat
     - apt-get install -y mariadb-server-10.3
     # Verify initial state before upgrade
     - dpkg -l | grep -iE 'maria|mysql|galera' || true # List installed
@@ -767,6 +784,7 @@ mariadb.org-10.2 to mariadb-10.6 upgrade:
     - curl -sS https://mariadb.org/mariadb_release_signing_key.asc -o /etc/apt/trusted.gpg.d/mariadb.asc
     - echo 'deb http://mirror.one.com/mariadb/repo/10.2/debian sid main' > /etc/apt/sources.list.d/mariadb.list
     - apt-get update
+    - *test-install-readline-in-sid-for-backwards-compat
     - apt-get install -y mariadb-server-10.2
     # Verify initial state before upgrade
     - dpkg -l | grep -iE 'maria|mysql|galera' || true # List installed

include/mysql/service_wsrep.h

@@ -90,6 +90,8 @@ extern struct wsrep_service_st {
   bool                        (*wsrep_thd_set_wsrep_aborter_func)(MYSQL_THD bf_thd, MYSQL_THD thd);
   void                        (*wsrep_report_bf_lock_wait_func)(const MYSQL_THD thd,
                                                                 unsigned long long trx_id);
+  void                        (*wsrep_thd_kill_LOCK_func)(const MYSQL_THD thd);
+  void                        (*wsrep_thd_kill_UNLOCK_func)(const MYSQL_THD thd);
 } *wsrep_service;
 
 #define MYSQL_SERVICE_WSREP_INCLUDED
@@ -107,6 +109,8 @@ extern struct wsrep_service_st {
 #define wsrep_prepare_key_for_innodb(A,B,C,D,E,F,G) wsrep_service->wsrep_prepare_key_for_innodb_func(A,B,C,D,E,F,G)
 #define wsrep_thd_LOCK(T) wsrep_service->wsrep_thd_LOCK_func(T)
 #define wsrep_thd_UNLOCK(T) wsrep_service->wsrep_thd_UNLOCK_func(T)
+#define wsrep_thd_kill_LOCK(T) wsrep_service->wsrep_thd_kill_LOCK_func(T)
+#define wsrep_thd_kill_UNLOCK(T) wsrep_service->wsrep_thd_kill_UNLOCK_func(T)
 #define wsrep_thd_query(T) wsrep_service->wsrep_thd_query_func(T)
 #define wsrep_thd_retry_counter(T) wsrep_service->wsrep_thd_retry_counter_func(T)
 #define wsrep_thd_ignore_table(T) wsrep_service->wsrep_thd_ignore_table_func(T)
@@ -170,6 +174,9 @@ extern "C" void wsrep_thd_LOCK(const MYSQL_THD thd);
 /* Unlock thd wsrep lock */
 extern "C" void wsrep_thd_UNLOCK(const MYSQL_THD thd);
 
+extern "C" void wsrep_thd_kill_LOCK(const MYSQL_THD thd);
+extern "C" void wsrep_thd_kill_UNLOCK(const MYSQL_THD thd);
+
 /* Return thd client state string */
 extern "C" const char* wsrep_thd_client_state_str(const MYSQL_THD thd);
 /* Return thd client mode string */

include/mysql_com.h

@@ -24,6 +24,7 @@
 #include "my_decimal_limits.h"
 
 #define HOSTNAME_LENGTH 60
+#define HOSTNAME_LENGTH_STR STRINGIFY_ARG(HOSTNAME_LENGTH)
 #define SYSTEM_CHARSET_MBMAXLEN 3
 #define NAME_CHAR_LEN	64              /* Field/table name length */
 #define USERNAME_CHAR_LENGTH 128