MDEV-17085: CHECKSUM TABLE EXTENDED does not work correctly

The problem was in calculating of the mask to clear unused null bits in case of using full byte.
MariaDB · Jan 16, 2019 · 1ecccb5 · 1ecccb5
1 parent 235374a
commit 1ecccb5
Show file tree

Hide file tree

Showing 4 changed files with 53 additions and 1 deletion.
diff --git a/mysql-test/r/row-checksum-old.result b/mysql-test/r/row-checksum-old.result
@@ -85,3 +85,19 @@ checksum table t1 extended;
 Table	Checksum
 test.t1	4108368782
 drop table t1;
+#
+# MDEV-17085: CHECKSUM TABLE EXTENDED does not work correctly
+#
+CREATE TABLE t1 ( c1 int NOT NULL, c2 int NOT NULL, c4 varchar(20), c5 varchar(20), c6 varchar(20), c7 varchar(20), c8 varchar(20), c9 varchar(20), c10 varchar(20), c11 varchar(20), c12 varchar(20), c13 varchar(20), c14 varchar(20), c15 varchar(20), c16 varchar(20), c19 int NOT NULL, c20 int NOT NULL, c21 varchar(20), c22 VARCHAR(20), c23 varchar(20));
+insert into t1 values (5,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,"dog",NULL,NULL);
+# Important is that checksum is different from following
+CHECKSUM TABLE t1 EXTENDED;
+Table	Checksum
+test.t1	2514025256
+UPDATE t1 SET c21='cat' WHERE  c1=5;
+# Important is that checksum is different from above
+CHECKSUM TABLE t1 EXTENDED;
+Table	Checksum
+test.t1	2326430205
+drop table t1;
+# End of 5.5 tests
diff --git a/mysql-test/r/row-checksum.result b/mysql-test/r/row-checksum.result
@@ -85,3 +85,19 @@ checksum table t1 extended;
 Table	Checksum
 test.t1	3885665021
 drop table t1;
+#
+# MDEV-17085: CHECKSUM TABLE EXTENDED does not work correctly
+#
+CREATE TABLE t1 ( c1 int NOT NULL, c2 int NOT NULL, c4 varchar(20), c5 varchar(20), c6 varchar(20), c7 varchar(20), c8 varchar(20), c9 varchar(20), c10 varchar(20), c11 varchar(20), c12 varchar(20), c13 varchar(20), c14 varchar(20), c15 varchar(20), c16 varchar(20), c19 int NOT NULL, c20 int NOT NULL, c21 varchar(20), c22 VARCHAR(20), c23 varchar(20));
+insert into t1 values (5,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,"dog",NULL,NULL);
+# Important is that checksum is different from following
+CHECKSUM TABLE t1 EXTENDED;
+Table	Checksum
+test.t1	2514025256
+UPDATE t1 SET c21='cat' WHERE  c1=5;
+# Important is that checksum is different from above
+CHECKSUM TABLE t1 EXTENDED;
+Table	Checksum
+test.t1	2326430205
+drop table t1;
+# End of 5.5 tests
diff --git a/mysql-test/t/row-checksum.test b/mysql-test/t/row-checksum.test
@@ -60,3 +60,20 @@ checksum table t1;
 checksum table t1 quick;
 checksum table t1 extended;
 drop table t1;
+
+--echo #
+--echo # MDEV-17085: CHECKSUM TABLE EXTENDED does not work correctly
+--echo #
+
+CREATE TABLE t1 ( c1 int NOT NULL, c2 int NOT NULL, c4 varchar(20), c5 varchar(20), c6 varchar(20), c7 varchar(20), c8 varchar(20), c9 varchar(20), c10 varchar(20), c11 varchar(20), c12 varchar(20), c13 varchar(20), c14 varchar(20), c15 varchar(20), c16 varchar(20), c19 int NOT NULL, c20 int NOT NULL, c21 varchar(20), c22 VARCHAR(20), c23 varchar(20));
+
+insert into t1 values (5,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,"dog",NULL,NULL);
+--echo # Important is that checksum is different from following
+CHECKSUM TABLE t1 EXTENDED;
+UPDATE t1 SET c21='cat' WHERE  c1=5;
+--echo # Important is that checksum is different from above
+CHECKSUM TABLE t1 EXTENDED;
+
+drop table t1;
+
+--echo # End of 5.5 tests
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
@@ -7844,7 +7844,10 @@ bool mysql_checksum_table(THD *thd, TABLE_LIST *tables,
       {
 	/* calculating table's checksum */
 	ha_checksum crc= 0;
-        uchar null_mask=256 -  (1 << t->s->last_null_bit_pos);
+        DBUG_ASSERT(t->s->last_null_bit_pos < 8);
+        uchar null_mask= (t->s->last_null_bit_pos ?
+                          (256 -  (1 << t->s->last_null_bit_pos)):
+                          0);
 
         t->use_all_columns();