MDEV-28366 GLOBAL debug_dbug setting affected by collation_connection=utf16...

When the system variables @@debug_dbug was assigned to
some expression, Sys_debug_dbug::do_check() did not properly
convert the value from the expression character set to utf8.
So the value was erroneously re-interpretted as utf8 without
conversion. In case of a tricky expression character set
(e.g. utf16le), this led to unexpected results.

Fix:

Re-using Sys_var_charptr::do_string_check() in Sys_debug_dbug::do_check().

Author: abarkov

mysql-test/suite/sys_vars/r/debug_dbug_utf16le.result

@@ -0,0 +1,31 @@
+#
+# Start of 10.5 tests
+#
+#
+# MDEV-28366 GLOBAL debug_dbug setting affected by collation_connection=utf16...
+#
+SET NAMES utf8;
+SET collation_connection=utf16le_general_ci;
+SET debug_dbug='d,any_random_string';
+SELECT @@debug_dbug;
+@@debug_dbug
+d,any_random_string
+SET debug_dbug=CONCAT('d,', _latin1 0xDF);
+SELECT @@debug_dbug;
+@@debug_dbug
+d,ß
+SELECT HEX(@@debug_dbug);
+HEX(@@debug_dbug)
+642CC39F
+SET @@debug_dbug=NULL;
+SELECT @@debug_dbug;
+@@debug_dbug
+
+SET @@debug_dbug=DEFAULT;
+SELECT @@debug_dbug;
+@@debug_dbug
+
+SET NAMES latin1;
+#
+# End of 10.5 tests
+#

mysql-test/suite/sys_vars/t/debug_dbug_utf16le.test

@@ -0,0 +1,29 @@
+--source include/have_debug.inc
+--source include/have_utf16.inc
+
+--echo #
+--echo # Start of 10.5 tests
+--echo #
+
+--echo #
+--echo # MDEV-28366 GLOBAL debug_dbug setting affected by collation_connection=utf16...
+--echo #
+
+SET NAMES utf8;
+SET collation_connection=utf16le_general_ci;
+SET debug_dbug='d,any_random_string';
+SELECT @@debug_dbug;
+SET debug_dbug=CONCAT('d,', _latin1 0xDF);
+SELECT @@debug_dbug;
+SELECT HEX(@@debug_dbug);
+
+SET @@debug_dbug=NULL;
+SELECT @@debug_dbug;
+
+SET @@debug_dbug=DEFAULT;
+SELECT @@debug_dbug;
+SET NAMES latin1;
+
+--echo #
+--echo # End of 10.5 tests
+--echo #

sql/sys_vars.inl

@@ -947,21 +947,10 @@ public:
   { option.var_type|= GET_STR; }
   bool do_check(THD *thd, set_var *var)
   {
-    char buff[STRING_BUFFER_USUAL_SIZE];
-    String str(buff, sizeof(buff), system_charset_info), *res;
-
-    if (!(res=var->value->val_str(&str)))
-    {
+    bool rc= Sys_var_charptr::do_string_check(thd, var, charset(thd));
+    if (var->save_result.string_value.str == nullptr)
       var->save_result.string_value.str= const_cast<char*>("");
-      var->save_result.string_value.length= 0;
-    }
-    else
-    {
-      size_t len= res->length();
-      var->save_result.string_value.str= thd->strmake(res->ptr(), len);
-      var->save_result.string_value.length= len;
-    }
-    return false;
+    return rc;
   }
   bool session_update(THD *thd, set_var *var)
   {