Merge branch '10.1' into pull-request-97

Merge downstream Debian packaging (MDEV-6284)

Author: vuvova

VERSION

@@ -1,4 +1,4 @@
 MYSQL_VERSION_MAJOR=10
 MYSQL_VERSION_MINOR=1
-MYSQL_VERSION_PATCH=7
+MYSQL_VERSION_PATCH=8
 MYSQL_VERSION_EXTRA=

config.h.cmake

@@ -87,6 +87,7 @@
 #cmakedefine HAVE_SYS_UTSNAME_H 1
 #cmakedefine HAVE_SYS_STAT_H 1
 #cmakedefine HAVE_SYS_STREAM_H 1
+#cmakedefine HAVE_SYS_SYSCALL_H 1
 #cmakedefine HAVE_SYS_TERMCAP_H 1
 #cmakedefine HAVE_SYS_TIMEB_H 1
 #cmakedefine HAVE_SYS_TIMES_H 1

configure.cmake

@@ -228,6 +228,7 @@ CHECK_INCLUDE_FILES ("sys/types.h;sys/shm.h" HAVE_SYS_SHM_H)
 CHECK_INCLUDE_FILES (sys/socket.h HAVE_SYS_SOCKET_H)
 CHECK_INCLUDE_FILES (sys/stat.h HAVE_SYS_STAT_H)
 CHECK_INCLUDE_FILES (sys/stream.h HAVE_SYS_STREAM_H)
+CHECK_INCLUDE_FILES (sys/syscall.h HAVE_SYS_SYSCALL_H)
 CHECK_INCLUDE_FILES (sys/termcap.h HAVE_SYS_TERMCAP_H)
 CHECK_INCLUDE_FILES ("curses.h;term.h" HAVE_TERM_H)
 CHECK_INCLUDE_FILES (asm/termbits.h HAVE_ASM_TERMBITS_H)

debian/additions/my.cnf

@@ -156,6 +156,7 @@ innodb_flush_method	= O_DIRECT
 #
 [galera]
 # Mandatory settings
+#wsrep_on=ON
 #wsrep_provider=
 #wsrep_cluster_address=
 #binlog_format=row

extra/innochecksum.cc

@@ -62,6 +62,7 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */
 #include "fsp0fsp.h"             /* fsp_flags_get_page_size() &
                                     fsp_flags_get_zip_size() */
 #include "ut0crc32.h"            /* ut_crc32_init() */
+#include "fsp0pagecompress.h"    /* fil_get_compression_alg_name */
 
 #ifdef UNIV_NONINL
 # include "fsp0fsp.ic"
@@ -109,6 +110,8 @@ int n_fil_page_type_xdes;
 int n_fil_page_type_blob;
 int n_fil_page_type_zblob;
 int n_fil_page_type_other;
+int n_fil_page_type_page_compressed;
+int n_fil_page_type_page_compressed_encrypted;
 
 int n_fil_page_max_index_id;
 
@@ -152,6 +155,8 @@ struct per_index_stats {
 
 std::map<unsigned long long, per_index_stats> index_ids;
 
+bool encrypted = false;
+
 /* Get the page size of the filespace from the filespace header. */
 static
 my_bool
@@ -197,6 +202,8 @@ get_page_size(
   {
     compressed= true;
   }
+
+
   return TRUE;
 }
 
@@ -515,6 +522,18 @@ parse_page(
                }
                n_fil_page_type_zblob++;
                break;
+       case FIL_PAGE_PAGE_COMPRESSED:
+	       if (per_page_details) {
+		       printf("FIL_PAGE_PAGE_COMPRESSED\n");
+	       }
+	       n_fil_page_type_page_compressed++;
+	       break;
+       case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED:
+	       if (per_page_details) {
+		       printf("FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED\n");
+	       }
+	       n_fil_page_type_page_compressed_encrypted++;
+	       break;
        default:
                if (per_page_details) {
                        printf("FIL_PAGE_TYPE_OTHER\n");
@@ -604,6 +623,8 @@ print_stats()
        printf("%d\tFIL_PAGE_TYPE_XDES\n", n_fil_page_type_xdes);
        printf("%d\tFIL_PAGE_TYPE_BLOB\n", n_fil_page_type_blob);
        printf("%d\tFIL_PAGE_TYPE_ZBLOB\n", n_fil_page_type_zblob);
+       printf("%d\tFIL_PAGE_PAGE_COMPRESSED\n", n_fil_page_type_page_compressed);
+       printf("%d\tFIL_PAGE_PAGE_COMPRESSED_ENCRYPTED\n", n_fil_page_type_page_compressed_encrypted);
        printf("%d\tother\n", n_fil_page_type_other);
        printf("%d\tmax index_id\n", n_fil_page_max_index_id);
        printf("undo type: %d insert, %d update, %d other\n",
@@ -791,7 +812,9 @@ int main(int argc, char **argv)
   while (!feof(f))
   {
     int page_ok = 1;
+
     bytes= fread(buf, 1, physical_page_size, f);
+
     if (!bytes && feof(f))
     {
       print_stats();
@@ -809,58 +832,126 @@ int main(int argc, char **argv)
       return 1;
     }
 
-    if (compressed) {
-      /* compressed pages */
-      if (!page_zip_verify_checksum(buf, physical_page_size)) {
-        fprintf(stderr, "Fail; page %lu invalid (fails compressed page checksum).\n", ct);
-        if (!skip_corrupt)
-        {
-          free(big_buf);
-	  free(big_xdes);
-          return 1;
-        }
-        page_ok = 0;
-      }
+    ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE);
+    ulint key_version = mach_read_from_4(buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
+
+    if (key_version && page_type != FIL_PAGE_PAGE_COMPRESSED) {
+	    encrypted = true;
     } else {
+	    encrypted = false;
+    }
+
+    ulint comp_method = 0;
 
-      /* check the "stored log sequence numbers" */
-      logseq= mach_read_from_4(buf + FIL_PAGE_LSN + 4);
-      logseqfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM + 4);
+    if (encrypted) {
+	    comp_method = mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE);
+    } else {
+	    comp_method = mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
+    }
+
+    ulint comp_size = mach_read_from_2(buf+FIL_PAGE_DATA);
+    ib_uint32_t encryption_checksum = mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
+
+
+    if (page_type == FIL_PAGE_PAGE_COMPRESSED) {
+      /* Page compressed tables do not have any checksum */
       if (debug)
-        printf("page %lu: log sequence number: first = %lu; second = %lu\n", ct, logseq, logseqfield);
-      if (logseq != logseqfield)
-      {
-        fprintf(stderr, "Fail; page %lu invalid (fails log sequence number check)\n", ct);
-        if (!skip_corrupt)
-        {
-          free(big_buf);
-	  free(big_xdes);
-          return 1;
+	fprintf(stderr, "Page %lu page compressed with method %s real_size %lu\n", ct,
+	        fil_get_compression_alg_name(comp_method), comp_size);
+      page_ok = 1;
+    } else if (compressed) {
+        /* compressed pages */
+	ulint crccsum = page_zip_calc_checksum(buf, physical_page_size, SRV_CHECKSUM_ALGORITHM_CRC32);
+	ulint icsum = page_zip_calc_checksum(buf, physical_page_size,  SRV_CHECKSUM_ALGORITHM_INNODB);
+
+        if (debug) {
+	  if (key_version != 0) {
+	    fprintf(stderr,
+		    "Page %lu encrypted key_version %lu calculated = %lu; crc32 = %lu; recorded = %u\n",
+		    ct, key_version, icsum, crccsum, encryption_checksum);
+	  }
+        }
+
+	if (encrypted) {
+	  if (encryption_checksum != 0 && crccsum != encryption_checksum && icsum != encryption_checksum) {
+	    if (debug)
+	      fprintf(stderr, "page %lu: compressed: calculated = %lu; crc32 = %lu; recorded = %u\n",
+		      ct, icsum, crccsum, encryption_checksum);
+	    fprintf(stderr, "Fail; page %lu invalid (fails compressed page checksum).\n", ct);
+	  }
+	} else {
+          if (!page_zip_verify_checksum(buf, physical_page_size)) {
+            fprintf(stderr, "Fail; page %lu invalid (fails compressed page checksum).\n", ct);
+            if (!skip_corrupt)
+            {
+              free(big_buf);
+	      free(big_xdes);
+              return 1;
+            }
+            page_ok = 0;
+          }
+	}
+    } else {
+      if (key_version != 0) {
+      /* Encrypted page */
+        if (debug) {
+	  if (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
+	    fprintf(stderr,
+		    "Page %lu page compressed with method %s real_size %lu and encrypted key_version %lu checksum %u\n",
+		    ct, fil_get_compression_alg_name(comp_method), comp_size, key_version, encryption_checksum);
+	  } else {
+	    fprintf(stderr,
+		    "Page %lu encrypted key_version %lu checksum %u\n",
+		    ct, key_version, encryption_checksum);
+	  }
         }
-        page_ok = 0;
       }
 
-      /* check old method of checksumming */
-      oldcsum= buf_calc_page_old_checksum(buf);
-      oldcsumfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM);
-      if (debug)
-        printf("page %lu: old style: calculated = %lu; recorded = %lu\n", ct, oldcsum, oldcsumfield);
-      if (oldcsumfield != mach_read_from_4(buf + FIL_PAGE_LSN) && oldcsumfield != oldcsum)
-      {
-        fprintf(stderr, "Fail;  page %lu invalid (fails old style checksum)\n", ct);
-        if (!skip_corrupt)
+      /* Page compressed tables do not contain FIL tailer */
+      if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED && page_type != FIL_PAGE_PAGE_COMPRESSED) {
+        /* check the "stored log sequence numbers" */
+         logseq= mach_read_from_4(buf + FIL_PAGE_LSN + 4);
+        logseqfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM + 4);
+        if (debug)
+          printf("page %lu: log sequence number: first = %lu; second = %lu\n", ct, logseq, logseqfield);
+        if (logseq != logseqfield)
         {
-          free(big_buf);
-	  free(big_xdes);
-          return 1;
+          fprintf(stderr, "Fail; page %lu invalid (fails log sequence number check)\n", ct);
+          if (!skip_corrupt)
+          {
+            free(big_buf);
+	    free(big_xdes);
+            return 1;
+          }
+          page_ok = 0;
+        }
+
+        /* check old method of checksumming */
+        oldcsum= buf_calc_page_old_checksum(buf);
+        oldcsumfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM);
+        if (debug)
+          printf("page %lu: old style: calculated = %lu; recorded = %lu\n", ct, oldcsum, oldcsumfield);
+        if (oldcsumfield != mach_read_from_4(buf + FIL_PAGE_LSN) && oldcsumfield != oldcsum)
+        {
+          fprintf(stderr, "Fail;  page %lu invalid (fails old style checksum)\n", ct);
+          if (!skip_corrupt)
+          {
+            free(big_buf);
+	    free(big_xdes);
+            return 1;
+          }
+          page_ok = 0;
         }
-        page_ok = 0;
       }
 
       /* now check the new method */
       csum= buf_calc_page_new_checksum(buf);
       crc32= buf_calc_page_crc32(buf);
       csumfield= mach_read_from_4(buf + FIL_PAGE_SPACE_OR_CHKSUM);
+
+      if (key_version)
+	      csumfield = encryption_checksum;
+
       if (debug)
         printf("page %lu: new style: calculated = %lu; crc32 = %lu; recorded = %lu\n",
                ct, csum, crc32, csumfield);
@@ -903,7 +994,10 @@ int main(int argc, char **argv)
       continue;
     }
 
-    parse_page(buf, xdes);
+    /* Can't parse compressed or/and encrypted pages */
+    if (page_type != FIL_PAGE_PAGE_COMPRESSED && !encrypted) {
+      parse_page(buf, xdes);
+    }
 
     if (verbose)
     {

include/my_base.h

@@ -501,7 +501,9 @@ enum ha_base_keytype {
 #define HA_ERR_DISK_FULL          189
 #define HA_ERR_INCOMPATIBLE_DEFINITION 190
 #define HA_ERR_FTS_TOO_MANY_WORDS_IN_PHRASE 191 /* Too many words in a phrase */
-#define HA_ERR_LAST               191    /* Copy of last error nr */
+#define HA_ERR_DECRYPTION_FAILED  192 /* Table encrypted but
+						decypt failed */
+#define HA_ERR_LAST               192    /* Copy of last error nr */
 
 /* Number of different errors */
 #define HA_ERR_ERRORS            (HA_ERR_LAST - HA_ERR_FIRST + 1)

include/my_handler_errors.h

@@ -94,7 +94,8 @@ static const char *handler_error_messages[]=
   "Operation was interrupted by end user (probably kill command?)",
   "Disk full",
   "Incompatible key or row definition between the MariaDB .frm file and the information in the storage engine. You have to dump and restore the table to fix this",
-  "Too many words in a FTS phrase or proximity search"
+  "Too many words in a FTS phrase or proximity search",
+  "Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match."
 };
 
 #endif /* MYSYS_MY_HANDLER_ERRORS_INCLUDED */

mysql-test/extra/binlog_tests/mysqlbinlog_row_engine.inc

@@ -77,7 +77,7 @@ eval CREATE TABLE t1 (
   #
   c28 DATE,
   c29 DATETIME,
-  c30 TIMESTAMP,
+  c30 TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
   c31 TIME,
   c32 YEAR,
   #

mysql-test/extra/rpl_tests/rpl_extra_col_slave.test

@@ -398,7 +398,8 @@ sync_slave_with_master;
   STOP SLAVE;
   RESET SLAVE;
   eval CREATE TABLE t9 (a INT KEY, b BLOB, c CHAR(5),
-                        d TIMESTAMP,
+                        d TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP 
+                          ON UPDATE CURRENT_TIMESTAMP,
                         e INT NOT NULL,
                         f text not null,
                         g text,

mysql-test/include/ctype_numconv.inc

@@ -1166,7 +1166,7 @@ create table t2 as select concat(a) from t1;
 show create table t2;
 drop table t1, t2;
 
-create table t1 (a timestamp);
+create table t1 (a timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP);
 insert into t1 values (0);
 insert into t1 values (20010203040506);
 insert into t1 values (19800203040506);
@@ -1478,7 +1478,7 @@ select hex(a) from v1;
 drop table t1;
 drop view v1;
 
-create table t1 (a timestamp);
+create table t1 (a timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP);
 insert into t1 values (0);
 insert into t1 values (20010203040506);
 insert into t1 values (19800203040506);