Allow reading of headerless compressed pages

Summary:
WebScaleSQL Feature: 2012 InnoDB Compression

The complete version of 2012 compression writes headerless zlib data
blobs.  This is true in both the 5.1 and 5.6 Facebook branches.

This change allows WebScaleSQL to read those pages, in addition to
those with headers.

But, this still writes only with zlib headers, not without.

A later diff will add support to WebScaleSQL for actually writing
these headerless pages.

Needed to backport inflateReset2() from a future version of zlib in
order to reset a decompression stream without allocating memory for its
internal state and with specifying windowBits. This was necessary to
automatically determine whether a compressed page was compressed using
adler32 checksums or not. There should not be any issues in terms of
backwards compatibility with this. After this, a clean build is required
for stress tests to pass.

Test Plan: Tested reading data compressed by the FB branch.  Worked.

Reviewers: pivanof, inaam-rana

Reviewed By: inaam-rana

Subscribers: WebScaleSQL, jtolmer

Differential Revision: https://reviews.facebook.net/D17691

storage/innobase/btr/btr0cur.cc

@@ -5339,6 +5339,7 @@ btr_copy_zblob_prefix(
 	mem_heap_t*	heap;
 	int		err;
 	z_stream	d_stream;
+	ibool		inflate_inited = FALSE;
 
 	d_stream.next_out = buf;
 	d_stream.avail_out = static_cast<uInt>(len);
@@ -5355,9 +5356,6 @@ btr_copy_zblob_prefix(
 	ut_ad(zip_size <= UNIV_ZIP_SIZE_MAX);
 	ut_ad(space_id);
 
-	err = inflateInit(&d_stream);
-	ut_a(err == Z_OK);
-
 	for (;;) {
 		buf_page_t*	bpage;
 		ulint		next_page_no;
@@ -5405,6 +5403,13 @@ btr_copy_zblob_prefix(
 		d_stream.next_in = bpage->zip.data + offset;
 		d_stream.avail_in = static_cast<uInt>(zip_size - offset);
 
+		if (!inflate_inited) {
+			inflate_inited = page_zip_init_d_stream(&d_stream,
+				15 /* Default windowBits for zlib */,
+				FALSE);
+			ut_a(inflate_inited);
+		}
+
 		err = inflate(&d_stream, Z_NO_FLUSH);
 		switch (err) {
 		case Z_OK:

storage/innobase/include/page0zip.h

@@ -39,6 +39,7 @@ Created June 2005 by Marko Makela
 #include "srv0srv.h"
 #include "trx0types.h"
 #include "mem0mem.h"
+#include "zlib.h"
 
 /* Compression level to be used by zlib. Settable by user. */
 extern uint	page_zip_level;
@@ -506,6 +507,26 @@ void
 page_zip_reset_stat_per_index();
 /*===========================*/
 
+/**********************************************************************//**
+This function determines the sign for window_bits and reads the zlib header
+from the decompress stream. The data may have been compressed with a negative
+(no adler32 headers) or a positive (with adler32 headers) window_bits.
+Regardless of the current value of page_zip_zlib_wrap, we always
+first try the positive window_bits then negative window_bits, because the
+surest way to determine if the stream has adler32 headers is to see if the
+stream begins with the zlib header together with the adler32 value of it.
+This adds a tiny bit of overhead for the pages that were compressed without
+adler32s.
+@return TRUE if stream is initialized and zlib header was read, FALSE
+if data can be decompressed with neither window_bits nor -window_bits */
+UNIV_INTERN
+ibool
+page_zip_init_d_stream(
+	z_stream* strm,
+	int window_bits,
+	ibool read_zlib_header);
+
+
 #ifndef UNIV_HOTBACKUP
 /** Check if a pointer to an uncompressed page matches a compressed page.
 When we IMPORT a tablespace the blocks and accompanying frames are allocted

storage/innobase/page/page0zip.cc

@@ -2941,6 +2941,64 @@ page_zip_decompress_clust(
 	return(TRUE);
 }
 
+/**********************************************************************//**
+This function determines the sign for window_bits and reads the zlib header
+from the decompress stream. The data may have been compressed with a negative
+(no adler32 headers) or a positive (with adler32 headers) window_bits.
+Regardless of the current value of page_zip_zlib_wrap, we always
+first try the positive window_bits then negative window_bits, because the
+surest way to determine if the stream has adler32 headers is to see if the
+stream begins with the zlib header together with the adler32 value of it.
+This adds a tiny bit of overhead for the pages that were compressed without
+adler32s.
+@return TRUE if stream is initialized and zlib header was read, FALSE
+if data can be decompressed with neither window_bits nor -window_bits */
+UNIV_INTERN
+ibool
+page_zip_init_d_stream(
+	z_stream* strm,
+	int window_bits,
+	ibool read_zlib_header)
+{
+	/* Save initial stream position, in case a reset is required. */
+	Bytef* next_in = strm->next_in;
+	Bytef* next_out = strm->next_out;
+	ulint avail_in = strm->avail_in;
+	ulint avail_out = strm->avail_out;
+
+	if (UNIV_UNLIKELY(inflateInit2(strm, window_bits) != Z_OK)) {
+		/* init must always succeed regardless of window_bits */
+		ut_error;
+	}
+	/* Try decoding a zlib header assuming adler32. */
+	if (inflate(strm, Z_BLOCK) == Z_OK)
+		/* A valid header was found, all is well.  So, we return
+		   with the stream positioned just after this header. */
+		return(TRUE);
+
+	/* A valid header was not found, so now we need to re-try this
+	   read assuming there is *no* header (negative window_bits).
+           So, we need to reset the stream to the original position,
+           and change the window_bits to negative, with inflateReset2(). */
+	strm->next_in = next_in;
+	strm->next_out = next_out;
+	strm->avail_in = avail_in;
+	strm->avail_out = avail_out;
+	if (UNIV_UNLIKELY(inflateReset2(strm, -window_bits) != Z_OK)) {
+		/* init must always succeed regardless of window_bits */
+		ut_error;
+	}
+
+	if (read_zlib_header) {
+		/* No valid header was found, and we still want the header
+                   read to have happened, with negative window_bits. */
+		return(inflate(strm, Z_BLOCK) == Z_OK);
+	}
+	/* Did not find a header, but didn't require one, so just return
+           with the stream position reset to where it originally was. */
+	return(TRUE);
+}
+
 /**********************************************************************//**
 Decompress a page.  This function should tolerate errors on the compressed
 page.  Instead of letting assertions fail, it will return FALSE if an
@@ -3052,13 +3110,7 @@ page_zip_decompress(
 	d_stream.next_out = page + PAGE_ZIP_START;
 	d_stream.avail_out = UNIV_PAGE_SIZE - PAGE_ZIP_START;
 
-	if (UNIV_UNLIKELY(inflateInit2(&d_stream, UNIV_PAGE_SIZE_SHIFT)
-			  != Z_OK)) {
-		ut_error;
-	}
-
-	/* Decode the zlib header and the index information. */
-	if (UNIV_UNLIKELY(inflate(&d_stream, Z_BLOCK) != Z_OK)) {
+	if (!page_zip_init_d_stream(&d_stream, UNIV_PAGE_SIZE_SHIFT, TRUE)) {
 
 		page_zip_fail(("page_zip_decompress:"
 			       " 1 inflate(Z_BLOCK)=%s\n", d_stream.msg));

zlib/inflate.c

@@ -125,6 +125,44 @@ z_streamp strm;
     return Z_OK;
 }
 
+int ZEXPORT inflateReset2(strm, windowBits)
+z_streamp strm;
+int windowBits;
+{
+    int wrap;
+    struct inflate_state FAR *state;
+
+    /* get the state */
+    if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR;
+    state = (struct inflate_state FAR *)strm->state;
+
+    /* extract wrap request from windowBits parameter */
+    if (windowBits < 0) {
+        wrap = 0;
+        windowBits = -windowBits;
+    }
+    else {
+        wrap = (windowBits >> 4) + 1;
+#ifdef GUNZIP
+        if (windowBits < 48)
+            windowBits &= 15;
+#endif
+    }
+
+    /* set number of window bits, free window if different */
+    if (windowBits && (windowBits < 8 || windowBits > 15))
+        return Z_STREAM_ERROR;
+    if (state->window != Z_NULL && state->wbits != (unsigned)windowBits) {
+        ZFREE(strm, state->window);
+        state->window = Z_NULL;
+    }
+
+    /* update state and reset the rest of it */
+    state->wrap = wrap;
+    state->wbits = (unsigned)windowBits;
+    return inflateReset(strm);
+}
+
 int ZEXPORT inflatePrime(strm, bits, value)
 z_streamp strm;
 int bits;

zlib/zlib.h

@@ -796,6 +796,18 @@ ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm));
    stream state was inconsistent (such as zalloc or state being NULL).
 */
 
+ZEXTERN int ZEXPORT inflateReset2 OF((z_streamp strm,
+                                      int windowBits));
+/*
+     This function is the same as inflateReset, but it also permits changing
+   the wrap and window size requests.  The windowBits parameter is interpreted
+   the same as it is for inflateInit2.
+
+     inflateReset2 returns Z_OK if success, or Z_STREAM_ERROR if the source
+   stream state was inconsistent (such as zalloc or state being Z_NULL), or if
+   the windowBits parameter is invalid.
+*/
+
 ZEXTERN int ZEXPORT inflatePrime OF((z_streamp strm,
                                      int bits,
                                      int value));