Bug #33030289 REGRESSION: ASSERTION `REC_SZ <= M_ELEMENT_SIZE' FAILED.

When sorting decimal values, the function make_sortkey_from_item() may
return UINT_MAX as length of the generated key in case of evaluation
failures. This could hit an assert in the prioriy queue code.

Relax the assert() in Bounded_queue::push.

Change-Id: Ia2aec6030db279269f2ca2a7b08ad892beb4acf3

Author: Tor Didriksen

mysql-test/r/filesort.result

@@ -182,3 +182,16 @@ SELECT 'a' AS f1 FROM t1 WHERE a='8' GROUP BY f1 ORDER BY CONCAT(f1);
 f1
 a
 DROP TABLE t1;
+#
+# Bug #33030289 REGRESSION: ASSERTION `REC_SZ <= M_ELEMENT_SIZE' FAILED.
+#
+CREATE TABLE t(a DECIMAL(55,19) NOT NULL);
+INSERT INTO t VALUES(0),(1),(2),(3),(4),(5);
+SELECT
+(
+SELECT 1 FROM t
+ORDER BY ST_HAUSDORFFDISTANCE(a, ST_ASTEXT(1,'axis-order=lat-long')), a
+LIMIT 1
+);
+ERROR 22023: Invalid GIS data provided to function st_astext.
+DROP TABLE t;

mysql-test/t/filesort.test

@@ -174,3 +174,18 @@ CREATE TABLE t1 (a LONGTEXT);
 INSERT INTO t1 VALUES ('8');
 SELECT 'a' AS f1 FROM t1 WHERE a='8' GROUP BY f1 ORDER BY CONCAT(f1);
 DROP TABLE t1;
+
+--echo #
+--echo # Bug #33030289 REGRESSION: ASSERTION `REC_SZ <= M_ELEMENT_SIZE' FAILED.
+--echo #
+
+CREATE TABLE t(a DECIMAL(55,19) NOT NULL);
+INSERT INTO t VALUES(0),(1),(2),(3),(4),(5);
+--error ER_GIS_INVALID_DATA
+SELECT
+(
+  SELECT 1 FROM t
+  ORDER BY ST_HAUSDORFFDISTANCE(a, ST_ASTEXT(1,'axis-order=lat-long')), a
+  LIMIT 1
+);
+DROP TABLE t;

sql/bounded_queue.h

@@ -117,12 +117,16 @@ class Bounded_queue {
       const Key_type &pq_top = m_queue.top();
       [[maybe_unused]] const uint rec_sz =
           m_sort_param->make_sortkey(pq_top, element_size, opaque);
-      assert(rec_sz <= m_element_size);
+      // UINT_MAX means error, but we do not want to add a dependency
+      // on class THD here, as in current_thd->is_error().
+      assert(rec_sz <= m_element_size || rec_sz == UINT_MAX);
       m_queue.update_top();
     } else {
       [[maybe_unused]] const uint rec_sz = m_sort_param->make_sortkey(
           m_sort_keys[m_queue.size()], element_size, opaque);
-      assert(rec_sz <= m_element_size);
+      // UINT_MAX means error, but we do not want to add a dependency
+      // on class THD here, as in current_thd->is_error().
+      assert(rec_sz <= m_element_size || rec_sz == UINT_MAX);
       m_queue.push(m_sort_keys[m_queue.size()]);
     }
   }