MDEV-33836 Compute modulus correctly in sequence

When the sequence is unsigned bigint, it needs to be cast to unsigned
for correct computation of the modulus.

Author: mariadb-YuchenPei

mysql-test/suite/sql_sequence/auto_increment.result

@@ -47,5 +47,20 @@ create sequence s as bigint start with -9223372036854775805 minvalue -9223372036
 drop sequence s;
 set global auto_increment_increment= default, auto_increment_offset= default;
 #
+# MDEV-33836 Assertion `(ulonglong) next_free_value % real_increment == (ulonglong) offset' failed in void sequence_definition::adjust_values(longlong)
+#
+CREATE SEQUENCE s AS BIGINT UNSIGNED START WITH 9223372036854775800 INCREMENT 0;
+set @old_AUTO_INCREMENT_INCREMENT=@@global.AUTO_INCREMENT_INCREMENT;
+set global AUTO_INCREMENT_INCREMENT=10;
+SELECT NEXTVAL (s);
+NEXTVAL (s)
+9223372036854775800
+FLUSH TABLES WITH READ LOCK;
+UPDATE s SET a=1;
+Got one of the listed errors
+unlock tables;
+set global AUTO_INCREMENT_INCREMENT=@old_AUTO_INCREMENT_INCREMENT;
+drop sequence s;
+#
 # End of 11.5 tests
 #

mysql-test/suite/sql_sequence/auto_increment.test

@@ -48,6 +48,23 @@ create sequence s as bigint start with -9223372036854775805 minvalue -9223372036
 drop sequence s;
 
 set global auto_increment_increment= default, auto_increment_offset= default;
+
+--echo #
+--echo # MDEV-33836 Assertion `(ulonglong) next_free_value % real_increment == (ulonglong) offset' failed in void sequence_definition::adjust_values(longlong)
+--echo #
+
+CREATE SEQUENCE s AS BIGINT UNSIGNED START WITH 9223372036854775800 INCREMENT 0;
+set @old_AUTO_INCREMENT_INCREMENT=@@global.AUTO_INCREMENT_INCREMENT;
+set global AUTO_INCREMENT_INCREMENT=10;
+SELECT NEXTVAL (s);
+FLUSH TABLES WITH READ LOCK;
+# ER_CANT_UPDATE_WITH_READLOCK when executed normally
+# ER_BAD_FIELD_ERROR when executed as a prepared statement
+--error ER_CANT_UPDATE_WITH_READLOCK,ER_BAD_FIELD_ERROR
+UPDATE s SET a=1;
+unlock tables;
+set global AUTO_INCREMENT_INCREMENT=@old_AUTO_INCREMENT_INCREMENT;
+drop sequence s;
 --enable_ps2_protocol
 
 --echo #

mysql-test/suite/sql_sequence/setval.result

@@ -418,4 +418,15 @@ setval(s, 32767)
 select nextval(s);
 ERROR HY000: Sequence 'test.s' has run out
 drop sequence s;
+#
+# MDEV-33836 Assertion `(ulonglong) next_free_value % real_increment == (ulonglong) offset' failed in void sequence_definition::adjust_values(longlong)
+#
+CREATE SEQUENCE s AS BIGINT UNSIGNED START WITH 9223372036854775800 INCREMENT 0;
+set @old_AUTO_INCREMENT_INCREMENT=@@global.AUTO_INCREMENT_INCREMENT;
+set global AUTO_INCREMENT_INCREMENT=100;
+SELECT SETVAL (s,12345678901234567890);
+SETVAL (s,12345678901234567890)
+12345678901234567890
+drop sequence s;
+set global AUTO_INCREMENT_INCREMENT=@old_AUTO_INCREMENT_INCREMENT;
 # End of 11.5 tests

mysql-test/suite/sql_sequence/setval.test

@@ -253,4 +253,15 @@ select nextval(s);
 drop sequence s;
 --enable_ps2_protocol
 
+--echo #
+--echo # MDEV-33836 Assertion `(ulonglong) next_free_value % real_increment == (ulonglong) offset' failed in void sequence_definition::adjust_values(longlong)
+--echo #
+
+CREATE SEQUENCE s AS BIGINT UNSIGNED START WITH 9223372036854775800 INCREMENT 0;
+set @old_AUTO_INCREMENT_INCREMENT=@@global.AUTO_INCREMENT_INCREMENT;
+set global AUTO_INCREMENT_INCREMENT=100;
+SELECT SETVAL (s,12345678901234567890);
+drop sequence s;
+set global AUTO_INCREMENT_INCREMENT=@old_AUTO_INCREMENT_INCREMENT;
+
 --echo # End of 11.5 tests

sql/sql_sequence.cc

@@ -751,7 +751,10 @@ void sequence_definition::adjust_values(longlong next_value)
 
       next_free_value % real_increment == offset
     */
-    off= next_free_value % real_increment;
+    if (is_unsigned)
+      off= (ulonglong) next_free_value % real_increment;
+    else
+      off= next_free_value % real_increment;
     if (off < 0)
       off+= real_increment;
     to_add= (real_increment + offset - off) % real_increment;