MDEV-9372 select 100 between 1 and 9223372036854775808 returns false

Integer comparison of INT expressions with different signess in BETWEEN
is not safe. Switching to DECIMAL comparison in case if INT arguments
have different signess.

Author: Alexander Barkov

mysql-test/r/bigint.result

@@ -502,3 +502,9 @@ a
 SELECT * FROM t1 WHERE a IN (0.8,0.9);
 a
 DROP TABLE t1;
+#
+# MDEV-9372 select 100 between 1 and 9223372036854775808 returns false
+#
+SELECT 100 BETWEEN 1 AND 9223372036854775808;
+100 BETWEEN 1 AND 9223372036854775808
+1

mysql-test/t/bigint.test

@@ -409,3 +409,8 @@ SELECT * FROM t1 WHERE a=0.9;
 SELECT * FROM t1 WHERE a IN (0.8,0.9);
 
 DROP TABLE t1;
+
+--echo #
+--echo # MDEV-9372 select 100 between 1 and 9223372036854775808 returns false
+--echo #
+SELECT 100 BETWEEN 1 AND 9223372036854775808;

sql/item_cmpfunc.cc

@@ -160,10 +160,11 @@ static int cmp_row_type(Item* item1, Item* item2)
 
 static int agg_cmp_type(Item_result *type, Item **items, uint nitems)
 {
-  uint i;
+  uint unsigned_count= items[0]->unsigned_flag;
   type[0]= items[0]->cmp_type();
-  for (i= 1 ; i < nitems ; i++)
+  for (uint i= 1 ; i < nitems ; i++)
   {
+    unsigned_count+= items[i]->unsigned_flag;
     type[0]= item_cmp_type(type[0], items[i]->cmp_type());
     /*
       When aggregating types of two row expressions we have to check
@@ -175,6 +176,12 @@ static int agg_cmp_type(Item_result *type, Item **items, uint nitems)
     if (type[0] == ROW_RESULT && cmp_row_type(items[0], items[i]))
       return 1;     // error found: invalid usage of rows
   }
+  /**
+    If all arguments are of INT type but have different unsigned_flag values,
+    switch to DECIMAL_RESULT.
+  */
+  if (type[0] == INT_RESULT && unsigned_count != nitems && unsigned_count != 0)
+    type[0]= DECIMAL_RESULT;
   return 0;
 }