Port compressed, and fix uncompressed, page support in innochecksum

Summary:
Ports support for compressed pages in innochecksum to 5.6,
and fixes uncompressed pages innochecksum.

1. Makes innochecksum support compressed tables & different page sizes.
2. 5.6 innochecksum didn't work for 5.6 uncompressed pages. It only checked
   for old format checksum rather than both old and new form. Fixed that.

Test Plan: run innochecksum on 5.6 compressed & uncompressed pages.

Reviewers: steaphan, nizamordulu

Reviewed By: steaphan

extra/CMakeLists.txt

@@ -92,6 +92,7 @@ IF(WITH_INNOBASE_STORAGE_ENGINE)
       ../storage/innobase/buf/buf0checksum.cc
       ../storage/innobase/ut/ut0crc32.cc
       ../storage/innobase/ut/ut0ut.cc
+      ../storage/innobase/page/page0zip.cc
      )
   MYSQL_ADD_EXECUTABLE(innochecksum innochecksum.cc ${INNOBASE_SOURCES})
   TARGET_LINK_LIBRARIES(innochecksum mysys mysys_ssl)

extra/innochecksum.cc

@@ -56,6 +56,7 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */
 #include "buf0checksum.h"        /* buf_calc_page_*() */
 #include "fil0fil.h"             /* FIL_* */
 #include "page0page.h"           /* PAGE_* */
+#include "page0zip.h"            /* page_zip_*() */
 #include "trx0undo.h"            /* TRX_* */
 #include "fsp0fsp.h"             /* fsp_flags_get_page_size() &
                                     fsp_flags_get_zip_size() */
@@ -633,11 +634,15 @@ int main(int argc, char **argv)
     return 1;
   }
 
-  /* This tool currently does not support Compressed tables */
   if (logical_page_size != physical_page_size)
   {
-    fprintf(stderr, "Error; This file contains compressed pages\n");
-    return 1;
+    printf("Table is compressed\n");
+    printf("Key block size is %luK\n", physical_page_size);
+  }
+  else
+  {
+    printf("Table is uncompressed\n");
+    printf("Page size is %luK\n", physical_page_size);
   }
 
   pages= (ulint) (size / physical_page_size);
@@ -709,42 +714,52 @@ int main(int argc, char **argv)
       return 1;
     }
 
-    /* check the "stored log sequence numbers" */
-    logseq= mach_read_from_4(buf + FIL_PAGE_LSN + 4);
-    logseqfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM + 4);
-    if (debug)
-      printf("page %lu: log sequence number: first = %lu; second = %lu\n", ct, logseq, logseqfield);
-    if (logseq != logseqfield)
-    {
-      fprintf(stderr, "Fail; page %lu invalid (fails log sequence number check)\n", ct);
-      if (!skip_corrupt) return 1;
-      page_ok = 0;
-    }
-
-    /* check old method of checksumming */
-    oldcsum= buf_calc_page_old_checksum(buf);
-    oldcsumfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM);
-    if (debug)
-      printf("page %lu: old style: calculated = %lu; recorded = %lu\n", ct, oldcsum, oldcsumfield);
-    if (oldcsumfield != mach_read_from_4(buf + FIL_PAGE_LSN) && oldcsumfield != oldcsum)
-    {
-      fprintf(stderr, "Fail;  page %lu invalid (fails old style checksum)\n", ct);
-      if (!skip_corrupt) return 1;
-      page_ok = 0;
-    }
-
-    /* now check the new method */
-    csum= buf_calc_page_new_checksum(buf);
-    crc32s= buf_calc_page_crc32fb(buf);
-    csumfield= mach_read_from_4(buf + FIL_PAGE_SPACE_OR_CHKSUM);
-    if (debug)
-      printf("page %lu: new style: calculated = %lu; crc32c = %u; crc32cfb = %u; recorded = %lu\n",
-          ct, csum, crc32s.crc32c, crc32s.crc32cfb, csumfield);
-    if (csumfield != 0 && crc32s.crc32c != csumfield && crc32s.crc32cfb != csumfield && csum != csumfield)
-    {
-      fprintf(stderr, "Fail; page %lu invalid (fails innodb and crc32 checksum)\n", ct);
-      if (!skip_corrupt) return 1;
-      page_ok = 0;
+    if (logical_page_size != physical_page_size) {
+      /* compressed pages */
+      if (!page_zip_verify_checksum(buf, physical_page_size)) {
+        fprintf(stderr, "Fail; page %lu invalid (fails compressed page checksum).\n", ct);
+        if (!skip_corrupt)
+          return 1;
+        page_ok = 0;
+      }
+    } else {
+      /* check the "stored log sequence numbers" */
+      logseq= mach_read_from_4(buf + FIL_PAGE_LSN + 4);
+      logseqfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM + 4);
+      if (debug)
+        printf("page %lu: log sequence number: first = %lu; second = %lu\n", ct, logseq, logseqfield);
+      if (logseq != logseqfield)
+      {
+        fprintf(stderr, "Fail; page %lu invalid (fails log sequence number check)\n", ct);
+        if (!skip_corrupt) return 1;
+        page_ok = 0;
+      }
+      /* check old method of checksumming */
+      oldcsum= buf_calc_page_old_checksum(buf);
+      oldcsumfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM);
+      crc32s= buf_calc_page_crc32fb(buf);
+      if (debug)
+        printf("page %lu: old style: calculated = %lu; recorded = %lu\n", ct, oldcsum, oldcsumfield);
+      if (oldcsumfield != mach_read_from_4(buf + FIL_PAGE_LSN)
+          && oldcsumfield != oldcsum && crc32s.crc32c != oldcsumfield
+          && crc32s.crc32cfb != oldcsumfield)
+      {
+        fprintf(stderr, "Fail;  page %lu invalid (fails old style checksum)\n", ct);
+        if (!skip_corrupt) return 1;
+        page_ok = 0;
+      }
+      /* now check the new method */
+      csum= buf_calc_page_new_checksum(buf);
+      csumfield= mach_read_from_4(buf + FIL_PAGE_SPACE_OR_CHKSUM);
+      if (debug)
+        printf("page %lu: new style: calculated = %lu; crc32c = %u; crc32cfb = %u; recorded = %lu\n",
+               ct, csum, crc32s.crc32c, crc32s.crc32cfb, csumfield);
+      if (csumfield != 0 && crc32s.crc32c != csumfield && crc32s.crc32cfb != csumfield && csum != csumfield)
+      {
+        fprintf(stderr, "Fail; page %lu invalid (fails innodb and crc32 checksum)\n", ct);
+        if (!skip_corrupt) return 1;
+        page_ok = 0;
+      }
     }
 
     /* end if this was the last page we were supposed to check */

storage/innobase/buf/buf0checksum.cc

@@ -34,14 +34,14 @@ Created Aug 11, 2011 Vasil Dimov
 #include "srv0srv.h" /* SRV_CHECKSUM_* */
 #include "buf0types.h"
 
+#endif /* !UNIV_INNOCHECKSUM */
+
 /** the macro MYSQL_SYSVAR_ENUM() requires "long unsigned int" and if we
 use srv_checksum_algorithm_t here then we get a compiler error:
 ha_innodb.cc:12251: error: cannot convert 'srv_checksum_algorithm_t*' to
   'long unsigned int*' in initialization */
 UNIV_INTERN ulong	srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_FACEBOOK;
 
-#endif /* !UNIV_INNOCHECKSUM */
-
 /********************************************************************//**
 Calculates a page CRC32 which is stored to the page when it is written
 to a file. Note that we must be careful to calculate the same value on

storage/innobase/include/buf0buf.h

@@ -96,9 +96,6 @@ extern buf_block_t*	back_block1;	/*!< first block, for --apply-log */
 extern buf_block_t*	back_block2;	/*!< second block, for page reorganize */
 #endif /* !UNIV_HOTBACKUP */
 
-/** Magic value to use instead of checksums when they are disabled */
-#define BUF_NO_CHECKSUM_MAGIC 0xDEADBEEFUL
-
 /** @brief States of a control block
 @see buf_page_t
 

storage/innobase/include/buf0checksum.h

@@ -28,11 +28,10 @@ Created Aug 11, 2011 Vasil Dimov
 
 #include "univ.i"
 
-#ifndef UNIV_INNOCHECKSUM
-
 #include "buf0types.h"
 
-#endif /* !UNIV_INNOCHECKSUM */
+/** Magic value to use instead of checksums when they are disabled */
+#define BUF_NO_CHECKSUM_MAGIC 0xDEADBEEFUL
 
 /********************************************************************//**
 Calculates a page CRC32 which is stored to the page when it is written
@@ -83,8 +82,6 @@ buf_calc_page_old_checksum(
 /*=======================*/
 	const byte*	page);	/*!< in: buffer page */
 
-#ifndef UNIV_INNOCHECKSUM
-
 /********************************************************************//**
 Return a printable string describing the checksum algorithm.
 @return	algorithm name */
@@ -96,6 +93,4 @@ buf_checksum_algorithm_name(
 
 extern ulong	srv_checksum_algorithm;
 
-#endif /* !UNIV_INNOCHECKSUM */
-
 #endif /* buf0checksum_h */

storage/innobase/include/page0zip.h

@@ -1,3 +1,4 @@
+
 /*****************************************************************************
 
 Copyright (c) 2005, 2013, Oracle and/or its affiliates. All Rights Reserved.
@@ -32,13 +33,15 @@ Created June 2005 by Marko Makela
 # define UNIV_INLINE
 #endif
 
-#include "mtr0types.h"
 #include "page0types.h"
 #include "buf0types.h"
+#ifndef UNIV_INNOCHECKSUM
+#include "mtr0types.h"
 #include "dict0types.h"
 #include "srv0srv.h"
 #include "trx0types.h"
 #include "mem0mem.h"
+#endif /* !UNIV_INNOCHECKSUM */
 #include "zlib.h"
 
 /* Compression level to be used by zlib. Settable by user. */
@@ -53,6 +56,7 @@ extern my_bool	page_zip_log_pages;
 extern my_bool page_zip_zlib_wrap;
 extern uint page_zip_zlib_strategy;
 
+#ifndef UNIV_INNOCHECKSUM
 /**********************************************************************//**
 Determine the size of a compressed page in bytes.
 @return	size in bytes */
@@ -115,6 +119,7 @@ page_zip_set_alloc(
 /*===============*/
 	void*		stream,		/*!< in/out: zlib stream */
 	mem_heap_t*	heap);		/*!< in: memory heap to use */
+#endif /* !UNIV_INNOCHECKSUM */
 
 /**********************************************************************//**
 Compress a page.
@@ -189,6 +194,7 @@ page_zip_validate(
 	__attribute__((nonnull(1,2)));
 #endif /* UNIV_ZIP_DEBUG */
 
+#ifndef UNIV_INNOCHECKSUM
 /**********************************************************************//**
 Determine how big record can be inserted without recompressing the page.
 @return a positive number indicating the maximum size of a record
@@ -422,6 +428,8 @@ page_zip_reorganize(
 	dict_index_t*	index,	/*!< in: index of the B-tree node */
 	mtr_t*		mtr)	/*!< in: mini-transaction */
 	__attribute__((nonnull));
+#endif /* !UNIV_INNOCHECKSUM */
+
 #ifndef UNIV_HOTBACKUP
 /**********************************************************************//**
 Copy the records of a page byte for byte.  Do not copy the page header
@@ -478,6 +486,8 @@ page_zip_verify_checksum(
 /*=====================*/
 	const void*	data,	/*!< in: compressed page */
 	ulint		size);	/*!< in: size of compressed page */
+
+#ifndef UNIV_INNOCHECKSUM
 /**********************************************************************//**
 Write a log record of compressing an index page without the data on the page. */
 UNIV_INLINE
@@ -557,7 +567,7 @@ page_zip_init_d_stream(
 	z_stream* strm,
 	int window_bits,
 	ibool read_zlib_header);
-
+#endif /* !UNIV_INNOCHECKSUM */
 
 #ifndef UNIV_HOTBACKUP
 /** Check if a pointer to an uncompressed page matches a compressed page.
@@ -584,8 +594,10 @@ from outside the buffer pool.
 # define UNIV_INLINE	UNIV_INLINE_ORIGINAL
 #endif
 
+#ifndef UNIV_INNOCHECKSUM
 #ifndef UNIV_NONINL
 # include "page0zip.ic"
 #endif
+#endif /* !UNIV_INNOCHECKSUM */
 
 #endif /* page0zip_h */

storage/innobase/include/srv0srv.h

@@ -377,7 +377,6 @@ extern my_bool			srv_stats_auto_recalc;
 extern ulong	srv_use_doublewrite_buf;
 extern my_bool	srv_doublewrite_reset;
 extern ulong	srv_doublewrite_batch_size;
-extern ulong	srv_checksum_algorithm;
 
 extern double	srv_max_buf_pool_modified_pct;
 extern ulong	srv_max_purge_lag;

storage/innobase/page/page0zip.cc

@@ -37,30 +37,39 @@ using namespace std;
 #endif
 #undef THIS_MODULE
 #include "fil0fil.h"
+#include "buf0checksum.h"
+#ifndef UNIV_INNOCHECKSUM
 #include "page0page.h"
 #include "mtr0log.h"
-#include "ut0sort.h"
 #include "dict0dict.h"
 #include "btr0cur.h"
-#include "page0types.h"
 #include "log0recv.h"
+#endif /* !UNIV_INNOCHECKSUM */
+#include "ut0sort.h"
+#include "page0types.h"
 #ifndef UNIV_HOTBACKUP
+#ifndef UNIV_INNOCHECKSUM
 # include "buf0buf.h"
-# include "buf0lru.h"
 # include "btr0sea.h"
 # include "dict0boot.h"
 # include "lock0lock.h"
-# include "srv0mon.h"
 # include "srv0srv.h"
+#endif /* !UNIV_INNOCHECKSUM */
+# include "buf0lru.h"
+# include "srv0mon.h"
 # include "ut0crc32.h"
 #else /* !UNIV_HOTBACKUP */
-# include "buf0checksum.h"
 # define lock_move_reorganize_page(block, temp_block)	((void) 0)
 # define buf_LRU_stat_inc_unzip()			((void) 0)
 #endif /* !UNIV_HOTBACKUP */
 #include "blind_fwrite.h"
 
+#ifdef UNIV_INNOCHECKSUM
+#include "mach0data.h"
+#endif /* UNIV_INNOCHECKSUM */
+
 #ifndef UNIV_HOTBACKUP
+#ifndef UNIV_INNOCHECKSUM
 /** Statistics on compression, indexed by page_zip_des_t::ssize - 1 */
 UNIV_INTERN page_zip_stat_t		page_zip_stat[PAGE_ZIP_SSIZE_MAX];
 /** Statistics on compression, indexed by index->id */
@@ -70,6 +79,7 @@ UNIV_INTERN ib_mutex_t			page_zip_stat_per_index_mutex;
 #ifdef HAVE_PSI_INTERFACE
 UNIV_INTERN mysql_pfs_key_t		page_zip_stat_per_index_mutex_key;
 #endif /* HAVE_PSI_INTERFACE */
+#endif /* !UNIV_INNOCHECKSUM */
 #endif /* !UNIV_HOTBACKUP */
 
 /* Compression level to be used by zlib. Settable by user. */
@@ -151,6 +161,7 @@ page_zip_fail_func(
 # define page_zip_fail(fmt_args) /* empty */
 #endif /* UNIV_DEBUG || UNIV_ZIP_DEBUG */
 
+#ifndef UNIV_INNOCHECKSUM
 #ifndef UNIV_HOTBACKUP
 /**********************************************************************//**
 Determine the guaranteed free space on an empty page.
@@ -4959,6 +4970,7 @@ page_zip_parse_compress(
 
 	return(ptr + 8 + size + trailer_size);
 }
+#endif /* !UNIV_INNOCHECKSUM */
 
 /**********************************************************************//**
 Calculate the compressed page checksum.
@@ -5039,6 +5051,10 @@ page_zip_verify_checksum(
 #error "FIL_PAGE_LSN must be 64 bit aligned"
 #endif
 
+#ifndef UNIV_INNOCHECKSUM
+	/* innochecksum doesn't compile with ut_d. Since we don't
+	need to check for empty pages when running innochecksum,
+	just don't include this code. */
 	/* Check if page is empty */
 	if (stored == 0
 	    && *reinterpret_cast<const ib_uint64_t*>(static_cast<const char*>(
@@ -5054,6 +5070,7 @@ page_zip_verify_checksum(
 		/* Empty page */
 		return(TRUE);
 	}
+#endif
 
 	calc = static_cast<ib_uint32_t>(page_zip_calc_checksum(
 		data, size, static_cast<srv_checksum_algorithm_t>(