MDEV-28386 UBSAN: runtime error: negation of -X cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in my_strntoull_8bit on SELECT ... OCT

The code in my_strntoull_8bit() and my_strntoull_mb2_or_mb4()
could hit undefinite behavior by negating of LONGLONG_MIN.
Fixing the code to avoid this.

Author: abarkov

mysql-test/main/ctype_ucs.result

@@ -6540,5 +6540,14 @@ DROP VIEW v1;
 DROP TABLE t1;
 SET NAMES utf8mb3;
 #
+# MDEV-28386 UBSAN: runtime error: negation of -X cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in my_strntoull_8bit on SELECT ... OCT
+#
+CREATE TABLE t1 (c TEXT CHARACTER SET ucs2);
+INSERT INTO t1 VALUES ('-9223372036854775808.5');
+SELECT OCT(c) FROM t1;
+OCT(c)
+1000000000000000000000
+DROP TABLE t1;
+#
 # End of 10.5 tests
 #

mysql-test/main/ctype_ucs.test

@@ -1217,6 +1217,15 @@ DROP VIEW v1;
 DROP TABLE t1;
 SET NAMES utf8mb3;
 
+--echo #
+--echo # MDEV-28386 UBSAN: runtime error: negation of -X cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in my_strntoull_8bit on SELECT ... OCT
+--echo #
+
+CREATE TABLE t1 (c TEXT CHARACTER SET ucs2);
+INSERT INTO t1 VALUES ('-9223372036854775808.5');
+SELECT OCT(c) FROM t1;
+DROP TABLE t1;
+
 
 --echo #
 --echo # End of 10.5 tests

mysql-test/main/func_str.result

@@ -5317,5 +5317,18 @@ SELECT SUBSTR(0,@a) FROM t;
 SUBSTR(0,@a)
 DROP TABLE t;
 #
+# MDEV-28386 UBSAN: runtime error: negation of -X cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in my_strntoull_8bit on SELECT ... OCT
+#
+CREATE TABLE t1 (c BLOB);
+INSERT INTO t1 VALUES ('-9223372036854775808.5');
+SELECT OCT(c) FROM t1;
+OCT(c)
+1000000000000000000000
+SELECT BIN(c) FROM t1;
+BIN(c)
+1000000000000000000000000000000000000000000000000000000000000000
+DROP TABLE t1;
+DO OCT(-9223372036854775808);
+#
 # End of 10.5 tests
 #

mysql-test/main/func_str.test

@@ -2358,6 +2358,19 @@ CREATE TABLE t (c1 INT,c2 CHAR);
 SELECT SUBSTR(0,@a) FROM t;
 DROP TABLE t;
 
+--echo #
+--echo # MDEV-28386 UBSAN: runtime error: negation of -X cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in my_strntoull_8bit on SELECT ... OCT
+--echo #
+
+CREATE TABLE t1 (c BLOB);
+INSERT INTO t1 VALUES ('-9223372036854775808.5');
+SELECT OCT(c) FROM t1;
+SELECT BIN(c) FROM t1;
+DROP TABLE t1;
+
+DO OCT(-9223372036854775808);
+
+
 --echo #
 --echo # End of 10.5 tests
 --echo #

strings/ctype-simple.c

@@ -758,7 +758,10 @@ ulonglong my_strntoull_8bit(CHARSET_INFO *cs,
     return (~(ulonglong) 0);
   }
 
-  return (negative ? -((longlong) i) : (longlong) i);
+  /* Avoid undefinite behavior - negation of LONGLONG_MIN */
+  return negative && (longlong) i != LONGLONG_MIN ?
+        -((longlong) i) :
+         (longlong) i;
 
 noconv:
   err[0]= EDOM;

strings/ctype-ucs2.c

@@ -616,7 +616,10 @@ my_strntoull_mb2_or_mb4(CHARSET_INFO *cs,
     return (~(ulonglong) 0);
   }
 
-  return (negative ? -((longlong) res) : (longlong) res);
+  /* Avoid undefinite behavior - negation of LONGLONG_MIN */
+  return negative && (longlong) res != LONGLONG_MIN ?
+         -((longlong) res) :
+          (longlong) res;
 }