MDEV-38372 ASAN error in InnoDB on malformed WKB

Geometry::is_binary_valid() method added, and
used to check WKB before storing in the engine.

Author: Alexey Botchkov

mysql-test/suite/funcs_1/r/storedproc.result

@@ -17283,8 +17283,7 @@ set f1 = f1;
 return f1;
 END//
 SELECT fn104('\0\0\0\0
\0\0\0\0\0\0\0\0\0\0\0\0?\0\0\0\0\0\0?\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0@\0\0\0\0\0\0@');
-fn104('\0\0\0\0
\0\0\0\0\0\0\0\0\0\0\0\0?\0\0\0\0\0\0?\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0@\0\0\0\0\0\0@')
-
??@@@@
+ERROR 22003: Cannot get geometry object from data you send to the GEOMETRY field
 DROP FUNCTION IF EXISTS fn105;
 CREATE FUNCTION fn105( f1 point) returns point
 BEGIN
@@ -18418,8 +18417,7 @@ set f1 = f1;
 SELECT f1;
 END//
 CALL sp104('\0\0\0\0
\0\0\0\0\0\0\0\0\0\0\0\0?\0\0\0\0\0\0?\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0@\0\0\0\0\0\0@');
-f1
-
??@@@@
+ERROR 22003: Cannot get geometry object from data you send to the GEOMETRY field
 DROP PROCEDURE IF EXISTS sp105;
 CREATE PROCEDURE sp105( f1 point)
 BEGIN

mysql-test/suite/innodb_gis/r/0.result

@@ -617,3 +617,15 @@ commit;
 disconnect a;
 connection default;
 drop table t1;
+#
+# MDEV-38372 ASAN error in InnoDB on malformed WKB
+#
+create table t1 (
+id int auto_increment primary key,
+geom geometry not null,
+spatial index (geom)
+) engine=innodb;
+insert t1 (geom) values (X'000000000107000000FF000000');
+ERROR 22003: Cannot get geometry object from data you send to the GEOMETRY field
+drop table t1;
+# End of 10.6 tests

mysql-test/suite/innodb_gis/t/0.test

@@ -40,5 +40,18 @@ disconnect a;
 connection default;
 drop table t1;
 
+--echo #
+--echo # MDEV-38372 ASAN error in InnoDB on malformed WKB
+--echo #
+create table t1 (
+    id int auto_increment primary key,
+    geom geometry not null,
+    spatial index (geom)
+) engine=innodb;
+
+--error ER_CANT_CREATE_GEOMETRY_OBJECT
+insert t1 (geom) values (X'000000000107000000FF000000');
+drop table t1;
 
+--echo # End of 10.6 tests
 

sql/spatial.h

@@ -269,6 +269,19 @@ class Geometry
 
   virtual const Class_info *get_class_info() const=0;
   virtual uint32 get_data_size() const=0;
+
+  /*
+    The 'binary_valid' spatial object can be stored in the database
+    and be the correct argument to the spatial functions.
+    It can still be not valid by the OPENGIS standard like
+    having self-intersecting borders.
+  */
+  bool is_binary_valid() const
+  {
+    uint32 data_size= get_data_size();
+
+    return (data_size == GET_SIZE_ERROR) ? false : !no_data(m_data, data_size);
+  }
   virtual bool init_from_wkt(Gis_read_stream *trs, String *wkb)=0;
   /* returns the length of the wkb that was read */
   virtual uint init_from_wkb(const char *wkb, uint len, wkbByteOrder bo,

sql/sql_type_geom.cc

@@ -854,6 +854,12 @@ int Field_geom::store(const char *from, size_t length, CHARSET_INFO *cs)
 	wkb_type > (uint32) Geometry::wkb_last)
       goto err;
 
+    const char *dummy;
+    Geometry_buffer buffer;
+    Geometry *geom= Geometry::construct(&buffer, from, uint32(length));
+    if (!geom || !geom->is_binary_valid())
+      goto err;
+
     if (m_type_handler->geometry_type() != Type_handler_geometry::GEOM_GEOMETRY &&
         m_type_handler->geometry_type() != Type_handler_geometry::GEOM_GEOMETRYCOLLECTION &&
         (uint32) m_type_handler->geometry_type() != wkb_type)
@@ -866,21 +872,14 @@ int Field_geom::store(const char *from, size_t length, CHARSET_INFO *cs)
       if (!tab_name)
         tab_name= "";
 
-      Geometry_buffer buffer;
-      Geometry *geom= NULL;
-      String wkt;
-      const char *dummy;
-      wkt.set_charset(&my_charset_latin1);
-      if (!(geom= Geometry::construct(&buffer, from, uint32(length))) ||
-          geom->as_wkt(&wkt, &dummy))
+      StringBuffer<STRING_BUFFER_USUAL_SIZE> wkt(&my_charset_latin1);
+      if (geom->as_wkt(&wkt, &dummy))
         goto err;
 
       my_error(ER_TRUNCATED_WRONG_VALUE_FOR_FIELD, MYF(0),
                Geometry::ci_collection[m_type_handler->geometry_type()]->m_name.str,
-	       wkt.c_ptr_safe(),
-               db, tab_name, field_name.str,
-               (ulong) table->in_use->get_stmt_da()->
-               current_row_for_warning());
+               wkt.c_ptr_safe(), db, tab_name, field_name.str,
+               (ulong) table->in_use->get_stmt_da()->current_row_for_warning());
       goto err_exit;
     }