MDEV-28345 ASAN: use-after-poison or unknown-crash in my_strtod_int from charset_info_st::strntod or test_if_number

This patch fixes two problems:

- The code inside my_strtod_int() in strings/dtoa.c could test the byte
  behind the end of the string when processing the mantissa.
  Rewriting the code to avoid this.

- The code in test_if_number() in sql/sql_analyse.cc called my_atof()
  which is unsafe and makes the called my_strtod_int() look behind
  the end of the string if the input string is not 0-terminated.
  Fixing test_if_number() to use my_strtod() instead, passing the correct
  end pointer.

Author: abarkov

mysql-test/main/func_analyse.result

@@ -222,3 +222,22 @@ DROP TABLE t1;
 #
 # End of 10.4 tests
 #
+#
+# Start of 10.5 tests
+#
+#
+# MDEV-28345 ASAN: use-after-poison or unknown-crash in my_strtod_int from charset_info_st::strntod or test_if_number
+#
+SET sql_mode='';
+CREATE TABLE t1 (c CHAR(10) KEY);
+INSERT INTO t1 VALUES (1.755555555);
+Warnings:
+Warning	1265	Data truncated for column 'c' at row 1
+SELECT * FROM t1 PROCEDURE ANALYSE();
+Field_name	Min_value	Max_value	Min_length	Max_length	Empties_or_zeros	Nulls	Avg_value_or_avg_length	Std	Optimal_fieldtype
+test.t1.c	1.75555555	1.75555555	10	10	0	0	10.0000	NULL	ENUM('1.75555555') NOT NULL
+DROP TABLE t1;
+SET sql_mode=DEFAULT;
+#
+# End of 10.5 tests
+#

mysql-test/main/func_analyse.test

@@ -230,3 +230,23 @@ DROP TABLE t1;
 --echo #
 --echo # End of 10.4 tests
 --echo #
+
+
+--echo #
+--echo # Start of 10.5 tests
+--echo #
+
+--echo #
+--echo # MDEV-28345 ASAN: use-after-poison or unknown-crash in my_strtod_int from charset_info_st::strntod or test_if_number
+--echo #
+
+SET sql_mode='';
+CREATE TABLE t1 (c CHAR(10) KEY);
+INSERT INTO t1 VALUES (1.755555555);
+SELECT * FROM t1 PROCEDURE ANALYSE();
+DROP TABLE t1;
+SET sql_mode=DEFAULT;
+
+--echo #
+--echo # End of 10.5 tests
+--echo #

mysql-test/main/type_num_innodb.result

@@ -88,3 +88,27 @@ DROP TABLE t1,t2;
 #
 # End of 10.2 tests
 #
+#
+# Start of 10.5 tests
+#
+#
+# MDEV-28345 ASAN: use-after-poison or unknown-crash in my_strtod_int from charset_info_st::strntod or test_if_number
+#
+CREATE TABLE t1 (c BLOB) ENGINE=InnoDB;
+INSERT INTO t1 VALUES ('0.0e'),('0.0e+0');
+SELECT * FROM t1 WHERE COALESCE(c)=0.0;
+c
+0.0e
+0.0e+0
+Warnings:
+Warning	1292	Truncated incorrect DOUBLE value: '0.0e'
+SELECT * FROM t1 WHERE COALESCE(c)=0.0e0;
+c
+0.0e
+0.0e+0
+Warnings:
+Warning	1292	Truncated incorrect DOUBLE value: '0.0e'
+DROP TABLE t1;
+#
+# End of 10.5 tests
+#

mysql-test/main/type_num_innodb.test

@@ -39,3 +39,22 @@ DROP TABLE t1,t2;
 --echo #
 --echo # End of 10.2 tests
 --echo #
+
+
+--echo #
+--echo # Start of 10.5 tests
+--echo #
+
+--echo #
+--echo # MDEV-28345 ASAN: use-after-poison or unknown-crash in my_strtod_int from charset_info_st::strntod or test_if_number
+--echo #
+
+CREATE TABLE t1 (c BLOB) ENGINE=InnoDB;
+INSERT INTO t1 VALUES ('0.0e'),('0.0e+0');
+SELECT * FROM t1 WHERE COALESCE(c)=0.0;
+SELECT * FROM t1 WHERE COALESCE(c)=0.0e0;
+DROP TABLE t1;
+
+--echo #
+--echo # End of 10.5 tests
+--echo #

sql/sql_analyse.cc

@@ -258,7 +258,9 @@ bool test_if_number(NUM_INFO *info, const char *str, uint str_len)
       info->decimals++;
     if (str == end)
     {
-      info->dval = my_atof(begin);
+      int error;
+      const char *end2= end;
+      info->dval= my_strtod(begin, (char **) &end2, &error);
       DBUG_RETURN(1);
     }
   }

strings/dtoa.c

@@ -1465,22 +1465,22 @@ static double my_strtod_int(const char *s00, char **se, int *error, char *buf, s
     s00= s;
     esign= 0;
     if (++s < end)
-      switch (c= *s) {
+      switch (*s) {
       case '-': esign= 1;
         /* fall through */
-      case '+': c= *++s;
+      case '+': s++;
       }
-    if (s < end && c >= '0' && c <= '9')
+    if (s < end && *s >= '0' && *s <= '9')
     {
-      while (s < end && c == '0')
-        c= *++s;
-      if (s < end && c > '0' && c <= '9') {
-        L= c - '0';
+      while (s < end && *s == '0')
+        s++;
+      if (s < end && *s > '0' && *s <= '9') {
+        L= *s - '0';
         s1= s;
-        while (++s < end && (c= *s) >= '0' && c <= '9')
+        while (++s < end && *s >= '0' && *s <= '9')
         {
           if (L < 19999)
-            L= 10*L + c - '0';
+            L= 10*L + *s - '0';
         }
         if (s - s1 > 8 || L > 19999)
           /* Avoid confusion from exponents