Merge branch '10.4' into 10.4.30

.gitignore

@@ -7,6 +7,7 @@
 *.rpm
 .*.swp
 *.ninja
+.ccls-cache/
 .ninja_*
 .gdb_history
 .vs/

.gitlab-ci.yml

@@ -27,6 +27,7 @@ stages:
   - build
   - test
   - Salsa-CI
+  - sast
 
 default:
   # Base image for builds and tests unless otherwise defined
@@ -42,7 +43,7 @@ variables:
   CMAKE_FLAGS: "-DPLUGIN_COLUMNSTORE=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_S3=NO -DPLUGIN_MROONGA=NO -DPLUGIN_CONNECT=NO -DPLUGIN_MROONGA=NO -DPLUGIN_TOKUDB=NO -DPLUGIN_PERFSCHEMA=NO -DWITH_WSREP=OFF"
   # Major version dictates which branches share the same ccache. E.g. 10.6-abc
   # and 10.6-xyz will have the same cache.
-  MARIADB_MAJOR_VERSION: "10.6"
+  MARIADB_MAJOR_VERSION: "10.4"
   # NOTE! Currently ccache is only used on the Centos8 build. As each job has
   # sufficiently different environments they are unable to benefit from each
   # other's ccaches. As each build generates about 1 GB of ccache, having
@@ -206,7 +207,7 @@ fedora-sanitizer:
       - builddir/_CPack_Packages/Linux/RPM/SPECS/
   parallel:
     matrix:
-      - SANITIZER: [-DWITH_ASAN=YES, -DWITH_TSAN=YES, -DWITH_UBSAN=YES, -DWITH_MSAN=YES]
+      - SANITIZER: [-DWITH_ASAN=YES, -DWITH_TSAN=YES, -DWITH_UBSAN=YES]
 
 centos8:
   stage: build
@@ -298,6 +299,7 @@ centos7:
       main.mysqldump : Field separator argument is not what is expected; check the manual when executing 'SELECT INTO OUTFILE'
       main.flush_logs_not_windows : query 'flush logs' succeeded - should have failed with error ER_CANT_CREATE_FILE (1004)
       main.mysql_upgrade_noengine : upgrade output order does not match the expected
+      main.func_math              : MDEV-20966 - Wrong error code
       " > skiplist
     - ./mtr --suite=main --force --parallel=auto --xml-report=$CI_PROJECT_DIR/junit.xml --skip-test-list=skiplist $RESTART_POLICY
 
@@ -331,6 +333,7 @@ mysql-test-run-asan:
   needs:
     - "fedora-sanitizer: [-DWITH_ASAN=YES]"
   <<: *mysql-test-run-def
+  allow_failure: true
   artifacts:
     when: always  # Also show results when tests fail
     reports:
@@ -369,22 +372,6 @@ mysql-test-run-ubsan:
       junit:
         - junit.xml
 
-mysql-test-run-msan:
-  stage: test
-  variables:
-    RESTART_POLICY: "--force-restart"
-  dependencies:
-    - "fedora-sanitizer: [-DWITH_MSAN=YES]"
-  needs:
-    - "fedora-sanitizer: [-DWITH_MSAN=YES]"
-  <<: *mysql-test-run-def
-  allow_failure: true
-  artifacts:
-    when: always  # Also show results when tests fail
-    reports:
-      junit:
-        - junit.xml
-
 rpmlint:
   stage: test
   dependencies:
@@ -439,52 +426,70 @@ fedora install:
       - installed-database.sql
       - upgraded-database.sql
 
-fedora upgrade:
-  stage: test
-  dependencies:
-    - fedora
-  needs:
-    - fedora
+cppcheck: 
+  stage: sast
+  needs: []
+  variables:
+    GIT_STRATEGY: fetch
+    GIT_SUBMODULE_STRATEGY: normal
   script:
-    - dnf install -y mariadb-server
-    # Fedora does not support running services in Docker (like Debian packages do) so start it manually
-    - /usr/libexec/mariadb-check-socket
-    - /usr/libexec/mariadb-prepare-db-dir
-    - sudo -u mysql /usr/libexec/mariadbd --basedir=/usr & sleep 10
-    # Dump database contents in installed state
-    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > old-installed-database.sql
-    - /usr/libexec/mariadb-check-upgrade
-    # Dump database contents in upgraded state
-    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > old-upgraded-database.sql
-    - mariadb --skip-column-names -e "SELECT @@version, @@version_comment"  # Show version
-    # @TODO: Upgrade from Fedora 33 MariaDB 10.4 to MariaDB.org latest does not work
-    # so do this manual step to remove conflicts until packaging is fixed
-    - yum remove -y mariadb-server-utils mariadb-gssapi-server mariadb-cracklib-password-check mariadb-backup mariadb-connector-c-config
-    - rm -f rpm/*debuginfo*  # Not relevant in this test
-    - yum install -y rpm/*.rpm
-    # nothing provides galera-4 on Fedora, so this step fails if built with wsrep
-    - mysql -e "SHUTDOWN;"
-    - /usr/bin/mariadb-install-db  # This step should not do anything on upgrades, just exit
-    - sudo -u mysql /usr/sbin/mariadbd & sleep 10
-    # Dump database contents in installed state
-    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > new-installed-database.sql || true
-    # The step above fails on: mariadb-dump: Couldn't execute 'show events': Cannot proceed, because event scheduler is disabled (1577)
-    # @TODO: Since we did a manual start, we also need to run upgrade manually
-    - /usr/bin/mariadb-upgrade
-    # Dump database contents in upgraded state
-    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > new-upgraded-database.sql
-    - |
-      mariadb --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version
-      grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't upgrade properly"
-    - mariadb --table -e "SELECT * FROM mysql.global_priv; SHOW CREATE USER root@localhost; SHOW CREATE USER 'mariadb.sys'@localhost"
-    - mariadb --table -e "SELECT * FROM mysql.plugin; SHOW PLUGINS"
+    - yum install -y cppcheck diffutils
+    # --template: use a single-line template
+    # --force: check large directories without warning
+    # -i<directory>: ignore this directory when scanning
+    # -j: run multiple cppcheck threads
+    # Use newline to escape colon in yaml
+    - >
+      cppcheck --template="{file}:{line}: {severity}: {message}" --force
+      client dbug extra include libmariadb libmysqld libservices mysql-test mysys mysys_ssl pcre plugin
+      strings tests unittest vio wsrep-lib sql sql-common storage
+      -istorage/mroonga -istorage/tokudb -istorage/spider -istorage/rocksdb -iextra/ -ilibmariadb/ -istorage/columnstore
+      --output-file=cppcheck.txt -j $(nproc)
+    # Parallel jobs may output findings in an nondeterministic order. Sort to match ignorelist.
+    - cat cppcheck.txt | sort > cppcheck_sorted.txt
+    # Remove line numbers for diff
+    - sed 's/:[^:]*:/:/' cppcheck_sorted.txt > cppcheck_sorted_no_line_numbers.txt
+    # Only print new issues not found in ignore list
+    - echo "Problems found in ignore list that were not discovered by cppcheck (may have been fixed)."
+    - diff --changed-group-format='%>' --unchanged-group-format='' cppcheck_sorted_no_line_numbers.txt tests/code_quality/cppcheck_ignorelist.txt || true
+    - echo "Problems found by cppcheck that were not in ignore list."
+    - diff --changed-group-format='%<' --unchanged-group-format='' cppcheck_sorted_no_line_numbers.txt tests/code_quality/cppcheck_ignorelist.txt > lines_not_ignored.txt || true
+    - cat lines_not_ignored.txt && test ! -s lines_not_ignored.txt
   artifacts:
+    when: always
     paths:
-      - old-installed-database.sql
-      - old-upgraded-database.sql
-      - new-installed-database.sql
-      - new-upgraded-database.sql
+      - cppcheck_sorted.txt
 
+flawfinder: 
+  stage: sast
+  needs: []
+  variables:
+    GIT_STRATEGY: fetch
+    GIT_SUBMODULE_STRATEGY: normal
+  script:
+    - yum install -y python3 python3-pip jq diffutils git
+    - pip install flawfinder
+    - flawfinder --falsepositive --quiet --html . > flawfinder-all-vulnerabilities.html
+    - cat flawfinder-all-vulnerabilities.html | grep "Hits ="
+    - flawfinder --falsepositive --quiet --minlevel=5 --sarif . > flawfinder-output.json
+    # FlawFinder's --sarif output will display all vulnerabilities despite having --minlevel=5 specified.
+    # Therefore, we postprocess the results with jq and filter out findings where the vulnerability level is less than 5.
+    # Also in the SARIF output format, the vulnerabilities are ranked as 0.2/0.4/0.6/0.8/1.0 which correspond to the --minlevel=1/2/3/4/5 of FlawFinder.
+    # Additionally, we sort the results because individual findings are consistent across different runs, but their ordering may not be.
+    # Vulnerabilities can also be ignored in-line (/* Flawfinder: ignore */), but this option was chosen as to not clutter the codebase.
+    - jq 'del(.runs[] | .tool | .driver | .rules) | del(.runs[] | .results[] | select(.rank < 1)) | del(.runs[] | .results[] | .locations[] | .physicalLocation | .region | .startLine) | .runs[0].results|=sort_by(.fingerprints)' flawfinder-output.json > flawfinder-min-level5.json
+    # Diff against known vulnerabilities, but ignore the line number.
+    - echo "Problems found in ignore list that were not discovered by flawfinder (may have been fixed)."
+    - diff --changed-group-format='%>' --unchanged-group-format='' flawfinder-min-level5.json tests/code_quality/flawfinder_ignorelist.json || true
+    - echo "Problems found by flawfinder that were not in ignore list."
+    - diff --changed-group-format='%<' --unchanged-group-format='' flawfinder-min-level5.json tests/code_quality/flawfinder_ignorelist.json > lines_not_ignored.txt || true
+    - cat lines_not_ignored.txt && test ! -s lines_not_ignored.txt
+  artifacts:
+    when: always
+    paths:
+      - flawfinder-all-vulnerabilities.html
+      - flawfinder-min-level5.json
+
 # Once all RPM builds and tests have passed, also run the DEB builds and tests
 # @NOTE: This is likely to work well only on salsa.debian.org as the Gitlab.com
 # runners are too small for everything this stage does.

CMakeLists.txt

@@ -27,8 +27,6 @@ IF(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
    "None" "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
 ENDIF()
 
-PROJECT(MySQL)
-
 IF(POLICY CMP0022)
   CMAKE_POLICY(SET CMP0022 NEW)
 ENDIF()
@@ -42,6 +40,8 @@ IF(POLICY CMP0075)
   CMAKE_POLICY(SET CMP0075 NEW)
 ENDIF()
 
+PROJECT(MySQL)
+
 MESSAGE(STATUS "Running cmake version ${CMAKE_VERSION}")
 
 SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}

VERSION

@@ -1,4 +1,4 @@
 MYSQL_VERSION_MAJOR=10
 MYSQL_VERSION_MINOR=4
-MYSQL_VERSION_PATCH=30
+MYSQL_VERSION_PATCH=31
 SERVER_MATURITY=stable

client/mysqltest.cc

@@ -8943,6 +8943,8 @@ int util_query(MYSQL* org_mysql, const char* query){
           org_mysql->unix_socket);
 
       cur_con->util_mysql= mysql;
+      if (mysql->charset != org_mysql->charset)
+        mysql_set_character_set(mysql, org_mysql->charset->csname);
     }
   }
   else

cmake/aws_sdk.cmake

@@ -10,7 +10,7 @@ FUNCTION (CHECK_AWS_SDK RETVAL REASON)
     SKIP_AWS_SDK("AWS_SDK_EXTERNAL_PROJECT is not ON")
   ENDIF()
   IF(NOT NOT_FOR_DISTRIBUTION)
-    SKIP_AWS_SDK("AWS SDK has Apache 2.0 License which is not complatible with GPLv2. Set -DNOT_FOR_DISTRIBUTION=ON if you need it")
+    SKIP_AWS_SDK("AWS SDK has Apache 2.0 License which is not compatible with GPLv2. Set -DNOT_FOR_DISTRIBUTION=ON if you need it")
   ENDIF()
   # Check compiler support
   IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU")

include/handler_ername.h

@@ -79,3 +79,4 @@
 { "HA_ERR_ABORTED_BY_USER", HA_ERR_ABORTED_BY_USER, "" },
 { "HA_ERR_DISK_FULL", HA_ERR_DISK_FULL, "" },
 { "HA_ERR_INCOMPATIBLE_DEFINITION", HA_ERR_INCOMPATIBLE_DEFINITION, "" },
+{ "HA_ERR_NO_ENCRYPTION", HA_ERR_NO_ENCRYPTION, "" },

include/json_lib.h

@@ -35,6 +35,7 @@ typedef struct st_json_string_t
   const uchar *c_str;    /* Current position in JSON string */
   const uchar *str_end;  /* The end on the string. */
   my_wc_t c_next;        /* UNICODE of the last read character */
+  int c_next_len;        /* character lenght of the last read character. */
   int error;             /* error code. */
 
   CHARSET_INFO *cs;      /* Character set of the JSON string. */
@@ -48,7 +49,7 @@ void json_string_set_cs(json_string_t *s, CHARSET_INFO *i_cs);
 void json_string_set_str(json_string_t *s,
                          const uchar *str, const uchar *end);
 #define json_next_char(j) \
-  (j)->wc((j)->cs, &(j)->c_next, (j)->c_str, (j)->str_end)
+  ((j)->c_next_len= (j)->wc((j)->cs, &(j)->c_next, (j)->c_str, (j)->str_end))
 #define json_eos(j) ((j)->c_str >= (j)->str_end)
 /*
   read_string_const_chr() reads the next character of the string constant

include/my_base.h

@@ -48,6 +48,7 @@
 #define HA_OPEN_NO_PSI_CALL             1024U   /* Don't call/connect PSI */
 #define HA_OPEN_MERGE_TABLE		2048U
 #define HA_OPEN_FOR_CREATE              4096U
+#define HA_OPEN_FOR_DROP                (1U << 13) /* Open part of drop */
 
 /*
   Allow opening even if table is incompatible as this is for ALTER TABLE which
@@ -516,14 +517,15 @@ enum ha_base_keytype {
 #define HA_ERR_DISK_FULL          189
 #define HA_ERR_INCOMPATIBLE_DEFINITION 190
 #define HA_ERR_FTS_TOO_MANY_WORDS_IN_PHRASE 191 /* Too many words in a phrase */
-#define HA_ERR_DECRYPTION_FAILED  192 /* Table encrypted but decypt failed */
+#define HA_ERR_DECRYPTION_FAILED  192 /* Table encrypted but decrypt failed */
 #define HA_ERR_FK_DEPTH_EXCEEDED  193 /* FK cascade depth exceeded */
 #define HA_ERR_TABLESPACE_MISSING 194  /* Missing Tablespace */
 #define HA_ERR_SEQUENCE_INVALID_DATA 195
 #define HA_ERR_SEQUENCE_RUN_OUT   196
 #define HA_ERR_COMMIT_ERROR       197
 #define HA_ERR_PARTITION_LIST     198
-#define HA_ERR_LAST               198  /* Copy of last error nr * */
+#define HA_ERR_NO_ENCRYPTION      199
+#define HA_ERR_LAST               199  /* Copy of last error nr * */
 
 /* Number of different errors */
 #define HA_ERR_ERRORS            (HA_ERR_LAST - HA_ERR_FIRST + 1)

include/my_handler_errors.h

@@ -109,7 +109,8 @@ static const char *handler_error_messages[]=
   "Sequence has been run out",
   "Sequence values are conflicting",
   "Error during commit",
-  "Cannot select partitions"
+  "Cannot select partitions",
+  "Cannot initialize encryption. Check that all encryption parameters have been set"
 };
 
 #endif /* MYSYS_MY_HANDLER_ERRORS_INCLUDED */

include/mysql/service_wsrep.h

@@ -57,6 +57,7 @@ extern struct wsrep_service_st {
   my_bool                     (*wsrep_on_func)(const MYSQL_THD thd);
   bool                        (*wsrep_prepare_key_for_innodb_func)(MYSQL_THD thd, const unsigned char*, size_t, const unsigned char*, size_t, struct wsrep_buf*, size_t*);
   void                        (*wsrep_thd_LOCK_func)(const MYSQL_THD thd);
+  int                         (*wsrep_thd_TRYLOCK_func)(const MYSQL_THD thd);
   void                        (*wsrep_thd_UNLOCK_func)(const MYSQL_THD thd);
   const char *                (*wsrep_thd_query_func)(const MYSQL_THD thd);
   int                         (*wsrep_thd_retry_counter_func)(const MYSQL_THD thd);
@@ -83,7 +84,6 @@ extern struct wsrep_service_st {
   my_bool                     (*wsrep_get_debug_func)();
   void                        (*wsrep_commit_ordered_func)(MYSQL_THD thd);
   my_bool                     (*wsrep_thd_is_applying_func)(const MYSQL_THD thd);
-  bool                        (*wsrep_thd_set_wsrep_aborter_func)(MYSQL_THD bf_thd, MYSQL_THD thd);
   void                        (*wsrep_report_bf_lock_wait_func)(const MYSQL_THD thd,
                                                                 unsigned long long trx_id);
   void                        (*wsrep_thd_kill_LOCK_func)(const MYSQL_THD thd);
@@ -105,6 +105,7 @@ extern struct wsrep_service_st {
 #define wsrep_on(thd) (thd) && WSREP_ON && wsrep_service->wsrep_on_func(thd)
 #define wsrep_prepare_key_for_innodb(A,B,C,D,E,F,G) wsrep_service->wsrep_prepare_key_for_innodb_func(A,B,C,D,E,F,G)
 #define wsrep_thd_LOCK(T) wsrep_service->wsrep_thd_LOCK_func(T)
+#define wsrep_thd_TRYLOCK(T) wsrep_service->wsrep_thd_TRYLOCK_func(T)
 #define wsrep_thd_UNLOCK(T) wsrep_service->wsrep_thd_UNLOCK_func(T)
 #define wsrep_thd_kill_LOCK(T) wsrep_service->wsrep_thd_kill_LOCK_func(T)
 #define wsrep_thd_kill_UNLOCK(T) wsrep_service->wsrep_thd_kill_UNLOCK_func(T)
@@ -130,7 +131,6 @@ extern struct wsrep_service_st {
 #define wsrep_get_debug() wsrep_service->wsrep_get_debug_func()
 #define wsrep_commit_ordered(T) wsrep_service->wsrep_commit_ordered_func(T)
 #define wsrep_thd_is_applying(T) wsrep_service->wsrep_thd_is_applying_func(T)
-#define wsrep_thd_set_wsrep_aborter(T) wsrep_service->wsrep_thd_set_wsrep_aborter_func(T1, T2)
 #define wsrep_report_bf_lock_wait(T,I) wsrep_service->wsrep_report_bf_lock_wait(T,I)
 #define wsrep_thd_set_PA_unsafe(T) wsrep_service->wsrep_thd_set_PA_unsafe_func(T)
 #else
@@ -164,6 +164,8 @@ void wsrep_set_data_home_dir(const char *data_dir);
 extern "C" my_bool wsrep_on(const MYSQL_THD thd);
 /* Lock thd wsrep lock */
 extern "C" void wsrep_thd_LOCK(const MYSQL_THD thd);
+/* Try thd wsrep lock. Return non-zero if lock could not be taken. */
+extern "C" int wsrep_thd_TRYLOCK(const MYSQL_THD thd);
 /* Unlock thd wsrep lock */
 extern "C" void wsrep_thd_UNLOCK(const MYSQL_THD thd);
 
@@ -186,8 +188,6 @@ extern "C" my_bool wsrep_thd_is_local(const MYSQL_THD thd);
 /* Return true if thd is in high priority mode */
 /* todo: rename to is_high_priority() */
 extern "C" my_bool wsrep_thd_is_applying(const MYSQL_THD thd);
-/* set wsrep_aborter for the target THD */
-extern "C" bool wsrep_thd_set_wsrep_aborter(MYSQL_THD bf_thd, MYSQL_THD victim_thd);
 /* Return true if thd is in TOI mode */
 extern "C" my_bool wsrep_thd_is_toi(const MYSQL_THD thd);
 /* Return true if thd is in replicating TOI mode */
@@ -228,7 +228,6 @@ extern "C" my_bool wsrep_get_debug();
 
 extern "C" void wsrep_commit_ordered(MYSQL_THD thd);
 extern "C" my_bool wsrep_thd_is_applying(const MYSQL_THD thd);
-extern "C" bool wsrep_thd_set_wsrep_aborter(MYSQL_THD bf_thd, MYSQL_THD victim_thd);
 extern "C" void wsrep_report_bf_lock_wait(const THD *thd,
                                           unsigned long long trx_id);
 /* declare parallel applying unsafety for the THD */

mysql-test/include/ctype_E05C.inc

@@ -76,6 +76,15 @@ INSERT INTO t1 VALUES (_BINARY'\\''
 SELECT a, HEX(a) FROM t1;
 DROP TABLE t1;
 
+#
+# test how strings are written into view's frm
+#
+disable_view_protocol;
+create view v1 as select hex('à\'), hex('à\t');
+select * from v1;
+drop view v1;
+enable_view_protocol;
+
 # Checking that with character_set_client=binary 0x5C in 0xE05C
 # is treated as escape rather than the second byte of a multi-byte character,
 # even if character_set_connection is big5/cp932/gbk/sjis.
@@ -109,5 +118,5 @@ SELECT HEX(a) FROM t1;
 DROP TABLE t1;
 --enable_view_protocol
 
---echo # Start of ctype_E05C.inc
+--echo # End of ctype_E05C.inc
 

mysql-test/include/wait_until_connected_again.inc

@@ -11,7 +11,7 @@ let $counter= 5000;
 let $mysql_errno= 9999;
 while ($mysql_errno)
 {
-  --error 0,ER_ACCESS_DENIED_ERROR,ER_SERVER_SHUTDOWN,ER_CONNECTION_KILLED,ER_LOCK_WAIT_TIMEOUT,2002,2006,2013
+  --error 0,ER_ACCESS_DENIED_ERROR,ER_SERVER_SHUTDOWN,ER_CONNECTION_KILLED,ER_LOCK_WAIT_TIMEOUT,2002,2006,2013,HA_ERR_NO_ENCRYPTION
   show status;
 
   dec $counter;
@@ -30,6 +30,10 @@ while ($mysql_errno)
   {
     let $mysql_errno=0;
   }
+  if ($mysql_errno == 199)
+  {
+    let $mysql_errno=0;
+  }
   --sleep 0.1
 }
 --enable_query_log