Merge branch 'merge/merge-tokudb-5.6' into 10.1

storage/tokudb/CMakeLists.txt

@@ -1,4 +1,4 @@
-SET(TOKUDB_VERSION 5.6.41-84.1)
+SET(TOKUDB_VERSION 5.6.49-89.0)
 # PerconaFT only supports x86-64 and cmake-2.8.9+
 IF(CMAKE_VERSION VERSION_LESS "2.8.9")
   MESSAGE(STATUS "CMake 2.8.9 or higher is required by TokuDB")
@@ -94,6 +94,7 @@ IF(DEFINED TOKUDB_NOPATCH_CONFIG)
     ADD_DEFINITIONS("-DTOKUDB_NOPATCH_CONFIG=${TOKUDB_NOPATCH_CONFIG}")
 ENDIF()
 
+MY_CHECK_AND_SET_COMPILER_FLAG(-Wno-missing-format-attribute)
 MY_CHECK_AND_SET_COMPILER_FLAG(-Wno-missing-field-initializers)
 
 IF (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/PerconaFT/")

storage/tokudb/PerconaFT/cmake_modules/TokuFeatureDetection.cmake

@@ -134,4 +134,4 @@ static __thread int tlsvar = 0;
 int main(void) { return tlsvar; }" HAVE_GNU_TLS)
 
 ## set TOKUDB_REVISION
-set(CMAKE_TOKUDB_REVISION 0 CACHE INTEGER "Revision of tokudb.")
+set(CMAKE_TOKUDB_REVISION 0 CACHE STRING "Revision of tokudb.")

storage/tokudb/PerconaFT/cmake_modules/TokuSetupCompiler.cmake

@@ -47,20 +47,18 @@ include(CheckCCompilerFlag)
 include(CheckCXXCompilerFlag)
 
 ## adds a compiler flag if the compiler supports it
-macro(set_cflags_if_supported)
+macro(prepend_cflags_if_supported)
   foreach(flag ${ARGN})
     MY_CHECK_AND_SET_COMPILER_FLAG(${flag})
   endforeach(flag)
-endmacro(set_cflags_if_supported)
+endmacro(prepend_cflags_if_supported)
 
 if (NOT DEFINED MYSQL_PROJECT_NAME_DOCSTRING)
   set (OPTIONAL_CFLAGS "${OPTIONAL_CFLAGS} -Wmissing-format-attribute")
 endif()
 
 ## disable some warnings
-## missing-format-attribute causes warnings in some MySQL include files
-## if the library is built as a part of TokuDB MySQL storage engine
-set_cflags_if_supported(
+prepend_cflags_if_supported(
   -Wno-missing-field-initializers
   -Wstrict-null-sentinel
   -Winit-self
@@ -77,7 +75,6 @@ set_cflags_if_supported(
   -fno-exceptions
   -Wno-error=nonnull-compare
   )
-## set_cflags_if_supported_named("-Weffc++" -Weffcpp)
 
 if (CMAKE_CXX_FLAGS MATCHES -fno-implicit-templates)
   # must append this because mysql sets -fno-implicit-templates and we need to override it
@@ -89,24 +86,18 @@ endif()
 
 ## Clang has stricter POD checks.  So, only enable this warning on our other builds (Linux + GCC)
 if (NOT CMAKE_CXX_COMPILER_ID MATCHES Clang)
-  set_cflags_if_supported(
+  prepend_cflags_if_supported(
     -Wpacked
     )
 endif ()
 
 option (PROFILING "Allow profiling and debug" ON)
 if (PROFILING)
-  set_cflags_if_supported(
+  prepend_cflags_if_supported(
     -fno-omit-frame-pointer
   )
 endif ()
 
-## this hits with optimized builds somewhere in ftleaf_split, we don't
-## know why but we don't think it's a big deal
-set_cflags_if_supported(
-  -Wno-error=strict-overflow
-  )
-
 # new flag sets in MySQL 8.0 seem to explicitly disable this
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fexceptions")
 
@@ -144,7 +135,7 @@ else ()
 endif ()
 
 ## set warnings
-set_cflags_if_supported(
+prepend_cflags_if_supported(
   -Wextra
   -Wbad-function-cast
   -Wno-missing-noreturn
@@ -167,7 +158,7 @@ set_cflags_if_supported(
 
 if (NOT CMAKE_CXX_COMPILER_ID STREQUAL Clang)
   # Disabling -Wcast-align with clang.  TODO: fix casting and re-enable it, someday.
-  set_cflags_if_supported(-Wcast-align)
+  prepend_cflags_if_supported(-Wcast-align)
 endif ()
 
 ## never want these

storage/tokudb/PerconaFT/ft/logger/logger.cc

@@ -50,6 +50,9 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 #include "util/status.h"
 
 int writing_rollback = 0;
+extern "C" {
+  uint force_recovery = 0;
+}
 
 static const int log_format_version = TOKU_LOG_VERSION;
 

storage/tokudb/PerconaFT/ft/serialize/rbtree_mhs.h

@@ -193,6 +193,7 @@ namespace MhsRbTree {
             BlockPair(OUUInt64 o, OUUInt64 s) : _offset(o), _size(s) {}
             BlockPair(const BlockPair &o)
                 : _offset(o._offset), _size(o._size) {}
+            BlockPair& operator=(const BlockPair&) = default;
 
             int operator<(const BlockPair &rhs) const {
                 return _offset < rhs._offset;

storage/tokudb/PerconaFT/ft/tests/cachetable-simple-close.cc

@@ -195,13 +195,13 @@ static void test_multiple_cachefiles(bool use_same_hash) {
 
         char fname1[strlen(TOKU_TEST_FILENAME) + sizeof("_1")];    
         strcpy(fname1, TOKU_TEST_FILENAME);
-        strncat(fname1, "_1", sizeof("_1"));
+        strcat(fname1, "_1");
         char fname2[strlen(TOKU_TEST_FILENAME) + sizeof("_2")];    
         strcpy(fname2, TOKU_TEST_FILENAME);
-        strncat(fname2, "_2", sizeof("_2"));
+        strcat(fname2, "_2");
         char fname3[strlen(TOKU_TEST_FILENAME) + sizeof("_3")];    
         strcpy(fname3, TOKU_TEST_FILENAME);
-        strncat(fname3, "_3", sizeof("_3"));
+        strcat(fname3, "_3");
 
         unlink(fname1);
         unlink(fname2);
@@ -280,10 +280,10 @@ static void test_evictor(void) {
 
     char fname1[strlen(TOKU_TEST_FILENAME) + sizeof("_1")];    
     strcpy(fname1, TOKU_TEST_FILENAME);
-    strncat(fname1, "_1", sizeof("_1"));
+    strcat(fname1, "_1");
     char fname2[strlen(TOKU_TEST_FILENAME) + sizeof("_2")];    
     strcpy(fname2, TOKU_TEST_FILENAME);
-    strncat(fname2, "_2", sizeof("_2"));
+    strcat(fname2, "_2");
 
     unlink(fname1);
     unlink(fname2);

storage/tokudb/PerconaFT/ft/tests/ft-bfe-query.cc

@@ -337,7 +337,7 @@ static void test_prefetching(void) {
     sn.layout_version_original = FT_LAYOUT_VERSION;
     sn.height = 1;
     sn.n_children = 3;
-    sn.dirty = 1;
+    sn.set_dirty();
     sn.oldest_referenced_xid_known = TXNID_NONE;
 
     uint64_t key1 = 100;

storage/tokudb/PerconaFT/ft/tests/ft-clock-test.cc

@@ -133,7 +133,7 @@ static void test1(int fd, FT ft_h, FTNODE *dn) {
     for (int i = 0; i < (*dn)->n_children; i++) {
         invariant(BP_STATE(*dn, i) == PT_AVAIL);
     }
-    (*dn)->dirty = 1;
+    (*dn)->set_dirty();
     toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
     toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
     toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
@@ -246,7 +246,7 @@ static void test_serialize_nonleaf(void) {
     sn.layout_version_original = FT_LAYOUT_VERSION;
     sn.height = 1;
     sn.n_children = 2;
-    sn.dirty = 1;
+    sn.set_dirty();
     sn.oldest_referenced_xid_known = TXNID_NONE;
     MALLOC_N(2, sn.bp);
     DBT pivotkey;
@@ -384,7 +384,7 @@ static void test_serialize_leaf(void) {
     sn.layout_version_original = FT_LAYOUT_VERSION;
     sn.height = 0;
     sn.n_children = 2;
-    sn.dirty = 1;
+    sn.set_dirty();
     sn.oldest_referenced_xid_known = TXNID_NONE;
     MALLOC_N(sn.n_children, sn.bp);
     DBT pivotkey;

storage/tokudb/PerconaFT/ft/tests/ft-serialize-benchmark.cc

@@ -95,7 +95,7 @@ static void test_serialize_leaf(int valsize,
     sn->layout_version_original = FT_LAYOUT_VERSION;
     sn->height = 0;
     sn->n_children = 8;
-    sn->dirty = 1;
+    sn->set_dirty();
     sn->oldest_referenced_xid_known = TXNID_NONE;
     MALLOC_N(sn->n_children, sn->bp);
     sn->pivotkeys.create_empty();
@@ -173,7 +173,7 @@ static void test_serialize_leaf(int valsize,
     for (int i = 0; i < ser_runs; i++) {
         gettimeofday(&t[0], NULL);
         ndd = NULL;
-        sn->dirty = 1;
+        sn->set_dirty();
         r = toku_serialize_ftnode_to(
             fd, make_blocknum(20), sn, &ndd, true, ft->ft, false);
         invariant(r == 0);
@@ -265,7 +265,7 @@ static void test_serialize_nonleaf(int valsize,
     sn.layout_version_original = FT_LAYOUT_VERSION;
     sn.height = 1;
     sn.n_children = 8;
-    sn.dirty = 1;
+    sn.set_dirty();
     sn.oldest_referenced_xid_known = TXNID_NONE;
     MALLOC_N(sn.n_children, sn.bp);
     sn.pivotkeys.create_empty();

storage/tokudb/PerconaFT/ft/tests/ft-serialize-test.cc

@@ -238,7 +238,7 @@ static void test_serialize_leaf_check_msn(enum ftnode_verify_type bft,
     sn.layout_version_original = FT_LAYOUT_VERSION;
     sn.height = 0;
     sn.n_children = 2;
-    sn.dirty = 1;
+    sn.set_dirty();
     sn.oldest_referenced_xid_known = TXNID_NONE;
     MALLOC_N(sn.n_children, sn.bp);
     DBT pivotkey;
@@ -381,7 +381,7 @@ static void test_serialize_leaf_with_large_pivots(enum ftnode_verify_type bft,
     sn.layout_version_original = FT_LAYOUT_VERSION;
     sn.height = 0;
     sn.n_children = nrows;
-    sn.dirty = 1;
+    sn.set_dirty();
     sn.oldest_referenced_xid_known = TXNID_NONE;
 
     MALLOC_N(sn.n_children, sn.bp);
@@ -538,7 +538,7 @@ static void test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft,
     sn.layout_version_original = FT_LAYOUT_VERSION;
     sn.height = 0;
     sn.n_children = 1;
-    sn.dirty = 1;
+    sn.set_dirty();
     sn.oldest_referenced_xid_known = TXNID_NONE;
 
     XMALLOC_N(sn.n_children, sn.bp);
@@ -693,7 +693,7 @@ static void test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft,
     sn.layout_version_original = FT_LAYOUT_VERSION;
     sn.height = 0;
     sn.n_children = 1;
-    sn.dirty = 1;
+    sn.set_dirty();
     sn.oldest_referenced_xid_known = TXNID_NONE;
 
     MALLOC_N(sn.n_children, sn.bp);
@@ -845,7 +845,7 @@ static void test_serialize_leaf_with_empty_basement_nodes(
     sn.layout_version_original = FT_LAYOUT_VERSION;
     sn.height = 0;
     sn.n_children = 7;
-    sn.dirty = 1;
+    sn.set_dirty();
     sn.oldest_referenced_xid_known = TXNID_NONE;
     MALLOC_N(sn.n_children, sn.bp);
     DBT pivotkeys[6];
@@ -989,7 +989,7 @@ static void test_serialize_leaf_with_multiple_empty_basement_nodes(
     sn.layout_version_original = FT_LAYOUT_VERSION;
     sn.height = 0;
     sn.n_children = 4;
-    sn.dirty = 1;
+    sn.set_dirty();
     sn.oldest_referenced_xid_known = TXNID_NONE;
     MALLOC_N(sn.n_children, sn.bp);
     DBT pivotkeys[3];
@@ -1100,7 +1100,7 @@ static void test_serialize_nonleaf(enum ftnode_verify_type bft, bool do_clone) {
     sn.layout_version_original = FT_LAYOUT_VERSION;
     sn.height = 1;
     sn.n_children = 2;
-    sn.dirty = 1;
+    sn.set_dirty();
     sn.oldest_referenced_xid_known = TXNID_NONE;
     MALLOC_N(2, sn.bp);
     DBT pivotkey;

storage/tokudb/PerconaFT/ft/tests/ft-test-header.cc

@@ -57,7 +57,7 @@ static void test_header (void) {
     assert(r==0);
     // now insert some info into the header
     FT ft = t->ft;
-    ft->h->dirty = 1;
+    ft->h->set_dirty();
     // cast away const because we actually want to fiddle with the header
     // in this test
     *((int *) &ft->h->layout_version_original) = 13;

storage/tokudb/PerconaFT/ft/tests/make-tree.cc

@@ -88,7 +88,7 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen)
     leafnode->max_msn_applied_to_node_on_disk = msn;
 
     // don't forget to dirty the node
-    leafnode->dirty = 1;
+    leafnode->set_dirty();
 }
 
 static void

storage/tokudb/PerconaFT/ft/tests/mempool-115.cc

@@ -102,7 +102,7 @@ class bndata_bugfix_test {
         sn.layout_version_original = FT_LAYOUT_VERSION;
         sn.height = 0;
         sn.n_children = 2;
-        sn.dirty = 1;
+        sn.set_dirty();
         sn.oldest_referenced_xid_known = TXNID_NONE;
         MALLOC_N(sn.n_children, sn.bp);
         DBT pivotkey;

storage/tokudb/PerconaFT/ft/tests/msnfilter.cc

@@ -161,7 +161,7 @@ append_leaf(FT_HANDLE ft, FTNODE leafnode, void *key, uint32_t keylen, void *val
     }
 
     // don't forget to dirty the node
-    leafnode->dirty = 1;
+    leafnode->set_dirty();
 }
 
 static void 

storage/tokudb/PerconaFT/ft/tests/recovery-test5123.cc

@@ -49,9 +49,9 @@ static void test_5123(void) {
     test_setup(TOKU_TEST_FILENAME, &logger, &ct);
 
     int r;
-    TXNID_PAIR one = {.parent_id64 = (TXNID)1, TXNID_NONE};
-    TXNID_PAIR two = {.parent_id64 = (TXNID)2, TXNID_NONE};
-    TXNID_PAIR three = {.parent_id64 = (TXNID)3, TXNID_NONE};
+    TXNID_PAIR one = { (TXNID)1, TXNID_NONE};
+    TXNID_PAIR two = { (TXNID)2, TXNID_NONE};
+    TXNID_PAIR three = { (TXNID)3, TXNID_NONE};
 
     toku_log_xbegin(logger, NULL, false, one, TXNID_PAIR_NONE);
     toku_log_xbegin(logger, NULL, false, three, TXNID_PAIR_NONE);

storage/tokudb/PerconaFT/ft/tests/test-checkpoint-during-flush.cc

@@ -245,7 +245,7 @@ doit (bool after_child_pin) {
         true
         );
     assert(node->height == 1);
-    assert(!node->dirty);
+    assert(!node->dirty());
     assert(node->n_children == 1);
     if (after_child_pin) {
         assert(toku_bnc_nbytesinbuf(BNC(node, 0)) == 0);
@@ -265,7 +265,7 @@ doit (bool after_child_pin) {
         true
         );
     assert(node->height == 0);
-    assert(!node->dirty);
+    assert(!node->dirty());
     assert(node->n_children == 1);
     if (after_child_pin) {
         assert(BLB_NBYTESINDATA(node,0) > 0);

storage/tokudb/PerconaFT/ft/tests/test-checkpoint-during-merge.cc

@@ -270,7 +270,7 @@ doit (int state) {
         true
         );
     assert(node->height == 1);
-    assert(!node->dirty);
+    assert(!node->dirty());
     BLOCKNUM left_child, right_child;
     // cases where we expect the checkpoint to contain the merge
     if (state == ft_flush_aflter_merge || state == flt_flush_before_unpin_remove) {
@@ -301,7 +301,7 @@ doit (int state) {
             true
             );
         assert(node->height == 0);
-        assert(!node->dirty);
+        assert(!node->dirty());
         assert(node->n_children == 1);
         assert(BLB_DATA(node, 0)->num_klpairs() == 1);
         toku_unpin_ftnode(c_ft->ft, node);
@@ -318,7 +318,7 @@ doit (int state) {
             true
             );
         assert(node->height == 0);
-        assert(!node->dirty);
+        assert(!node->dirty());
         assert(node->n_children == 1);
         assert(BLB_DATA(node, 0)->num_klpairs() == 1);
         toku_unpin_ftnode(c_ft->ft, node);
@@ -336,7 +336,7 @@ doit (int state) {
             true
             );
         assert(node->height == 0);
-        assert(!node->dirty);
+        assert(!node->dirty());
         assert(node->n_children == 1);
         assert(BLB_DATA(node, 0)->num_klpairs() == 2);
         toku_unpin_ftnode(c_ft->ft, node);