MDEV-37055 UBSAN: 32801 is outside the range of representable values of type 'short'

avoid overflows caused by float precision loss

Author: vuvova

mysql-test/main/vector2.result

@@ -461,4 +461,11 @@ delete from t;
 insert into t values (0x31313131);
 delete from t;
 drop table t;
+#
+# MDEV-37055 UBSAN: 32801 is outside the range of representable values of type 'short'
+#
+create table t1 (v vector (1) not null,vector index (v));
+insert into t1 values (0xf0a08080);
+insert into t1 values (0xfa000000);
+drop table t1;
 # End of 11.8 tests

mysql-test/main/vector2.test

@@ -339,5 +339,12 @@ delete from t;
 insert into t values (0x31313131);
 delete from t;
 drop table t;
+--echo #
+--echo # MDEV-37055 UBSAN: 32801 is outside the range of representable values of type 'short'
+--echo #
+create table t1 (v vector (1) not null,vector index (v));
+insert into t1 values (0xf0a08080);
+insert into t1 values (0xfa000000);
+drop table t1;
 
 --echo # End of 11.8 tests

sql/vector_mhnsw.cc

@@ -100,6 +100,8 @@ struct FVector
 
     FVector *vec= align_ptr(mem);
     vec->scale= scale ? scale/32767 : 1;
+    if (std::round(scale/vec->scale) > 32767)
+      vec->scale= std::nextafter(vec->scale, scale > 0 ? FLT_MAX : -FLT_MAX);
     for (size_t i= 0; i < vec_len; i++)
       vec->dims[i] = static_cast<int16_t>(std::round(get_float(v + i) / vec->scale));
     vec->postprocess(vec_len);