Merge branch 'merge-tokudb-5.6' into 10.0

storage/tokudb/CMakeLists.txt

@@ -1,4 +1,4 @@
-SET(TOKUDB_VERSION 5.6.37-82.2)
+SET(TOKUDB_VERSION 5.6.38-83.0)
 # PerconaFT only supports x86-64 and cmake-2.8.9+
 IF(CMAKE_VERSION VERSION_LESS "2.8.9")
   MESSAGE(STATUS "CMake 2.8.9 or higher is required by TokuDB")

storage/tokudb/PerconaFT/CMakeLists.txt

@@ -12,9 +12,14 @@ set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "")
 # detect when we are being built as a subproject
 if (DEFINED MYSQL_PROJECT_NAME_DOCSTRING)
   add_definitions( -DMYSQL_TOKUDB_ENGINE=1)
+  # Extended PFS instrumentation:
+  # -DTOKU_PFS_MUTEX_EXTENDED_CACHETABLEMMUTEX=1
+  if (WITH_PERFSCHEMA_STORAGE_ENGINE)
+    add_definitions(-DTOKU_MYSQL_WITH_PFS)
+  endif ()
+  include_directories(${CMAKE_SOURCE_DIR}/include)
   if ((CMAKE_BUILD_TYPE MATCHES "Debug") AND
       (CMAKE_CXX_FLAGS_DEBUG MATCHES " -DENABLED_DEBUG_SYNC"))
-    include_directories(${CMAKE_SOURCE_DIR}/include)
     include_directories(${CMAKE_SOURCE_DIR}/sql)
   endif ()
 endif ()

storage/tokudb/PerconaFT/buildheader/make_tdb.cc

@@ -444,14 +444,15 @@ static void print_db_key_range_struct (void) {
     sort_and_dump_fields("db_key_range", false, NULL);
 }
 
-static void print_db_lsn_struct (void) {
-    field_counter=0;
-    /* A dummy field to make sizeof(DB_LSN) equal in C and C++ */
-    const char *extra[] = { "char dummy", NULL };
-    sort_and_dump_fields("db_lsn", false, extra);
+static void print_db_lsn_struct(void) {
+    field_counter = 0;
+    // FT-692
+    STRUCT_SETUP(DB_LSN, file, "uint32_t %s");
+    STRUCT_SETUP(DB_LSN, offset, "uint32_t %s");
+    sort_and_dump_fields("db_lsn", false, NULL);
 }
 
-static void print_dbt_struct (void) {
+static void print_dbt_struct(void) {
     field_counter=0;
 #if 0 && DB_VERSION_MAJOR==4 && DB_VERSION_MINOR==1
     STRUCT_SETUP(DBT, app_private, "void*%s");

storage/tokudb/PerconaFT/cmake_modules/TokuSetupCompiler.cmake

@@ -85,15 +85,21 @@ macro(set_ldflags_if_supported)
   endforeach(flag)
 endmacro(set_ldflags_if_supported)
 
+if (NOT DEFINED MYSQL_PROJECT_NAME_DOCSTRING)
+  set (OPTIONAL_CFLAGS "${OPTIONAL_CFLAGS} -Wmissing-format-attribute")
+endif()
+
 ## disable some warnings
+## missing-format-attribute causes warnings in some MySQL include files
+## if the library is built as a part of TokuDB MySQL storage engine
 set_cflags_if_supported(
   -Wno-missing-field-initializers
   -Wstrict-null-sentinel
   -Winit-self
   -Wswitch
   -Wtrampolines
   -Wlogical-op
-  -Wmissing-format-attribute
+  ${OPTIONAL_CFLAGS}
   -Wno-error=missing-format-attribute
   -Wno-error=address-of-array-temporary
   -Wno-error=tautological-constant-out-of-range-compare
@@ -178,8 +184,8 @@ set_cflags_if_supported(
   -Wmissing-prototypes
   -Wmissing-declarations
   -Wpointer-arith
-  -Wmissing-format-attribute
   -Wshadow
+  ${OPTIONAL_CFLAGS}
   ## other flags to try:
   #-Wunsafe-loop-optimizations
   #-Wpointer-arith

storage/tokudb/PerconaFT/ft/cachetable/background_job_manager.cc

@@ -42,17 +42,20 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 
 #include "cachetable/background_job_manager.h"
 
+toku_instr_key *bjm_jobs_lock_mutex_key;
+toku_instr_key *bjm_jobs_wait_key;
+
 struct background_job_manager_struct {
     bool accepting_jobs;
     uint32_t num_jobs;
     toku_cond_t jobs_wait;
     toku_mutex_t jobs_lock;
 };
 
-void bjm_init(BACKGROUND_JOB_MANAGER* pbjm) {
+void bjm_init(BACKGROUND_JOB_MANAGER *pbjm) {
     BACKGROUND_JOB_MANAGER XCALLOC(bjm);
-    toku_mutex_init(&bjm->jobs_lock, 0);    
-    toku_cond_init(&bjm->jobs_wait, NULL);
+    toku_mutex_init(*bjm_jobs_lock_mutex_key, &bjm->jobs_lock, nullptr);
+    toku_cond_init(*bjm_jobs_wait_key, &bjm->jobs_wait, nullptr);
     bjm->accepting_jobs = true;
     bjm->num_jobs = 0;
     *pbjm = bjm;

storage/tokudb/PerconaFT/ft/cachetable/cachetable.cc

@@ -57,6 +57,25 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 #include "util/status.h"
 #include "util/context.h"
 
+toku_instr_key *cachetable_m_mutex_key;
+toku_instr_key *cachetable_ev_thread_lock_mutex_key;
+
+toku_instr_key *cachetable_m_list_lock_key;
+toku_instr_key *cachetable_m_pending_lock_expensive_key;
+toku_instr_key *cachetable_m_pending_lock_cheap_key;
+toku_instr_key *cachetable_m_lock_key;
+
+toku_instr_key *cachetable_value_key;
+toku_instr_key *cachetable_disk_nb_rwlock_key;
+
+toku_instr_key *cachetable_p_refcount_wait_key;
+toku_instr_key *cachetable_m_flow_control_cond_key;
+toku_instr_key *cachetable_m_ev_thread_cond_key;
+
+toku_instr_key *cachetable_disk_nb_mutex_key;
+toku_instr_key *log_internal_lock_mutex_key;
+toku_instr_key *eviction_thread_key;
+
 ///////////////////////////////////////////////////////////////////////////////////
 // Engine status
 //
@@ -779,18 +798,25 @@ void pair_init(PAIR p,
     p->checkpoint_complete_callback = write_callback.checkpoint_complete_callback;
     p->write_extraargs = write_callback.write_extraargs;
 
-    p->count = 0; // <CER> Is zero the correct init value?
+    p->count = 0;  // <CER> Is zero the correct init value?
     p->refcount = 0;
     p->num_waiting_on_refs = 0;
-    toku_cond_init(&p->refcount_wait, NULL);
+    toku_cond_init(*cachetable_p_refcount_wait_key, &p->refcount_wait, nullptr);
     p->checkpoint_pending = false;
 
     p->mutex = list->get_mutex_for_pair(fullhash);
     assert(p->mutex);
-    p->value_rwlock.init(p->mutex);
-    nb_mutex_init(&p->disk_nb_mutex);
+    p->value_rwlock.init(p->mutex
+#ifdef TOKU_MYSQL_WITH_PFS
+                         ,
+                         *cachetable_value_key
+#endif
+                         );
+    nb_mutex_init(*cachetable_disk_nb_mutex_key,
+                  *cachetable_disk_nb_rwlock_key,
+                  &p->disk_nb_mutex);
 
-    p->size_evicting_estimate = 0; // <CER> Is zero the correct init value?
+    p->size_evicting_estimate = 0;  // <CER> Is zero the correct init value?
 
     p->ev = ev;
     p->list = list;
@@ -3229,16 +3255,26 @@ void pair_list::init() {
 #if defined(HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP)
     pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
 #else
-    // TODO: need to figure out how to make writer-preferential rwlocks
-    // happen on osx
+// TODO: need to figure out how to make writer-preferential rwlocks
+// happen on osx
 #endif
-    toku_pthread_rwlock_init(&m_list_lock, &attr);
-    toku_pthread_rwlock_init(&m_pending_lock_expensive, &attr);    
-    toku_pthread_rwlock_init(&m_pending_lock_cheap, &attr);    
+    toku_pthread_rwlock_init(*cachetable_m_list_lock_key, &m_list_lock, &attr);
+    toku_pthread_rwlock_init(*cachetable_m_pending_lock_expensive_key,
+                             &m_pending_lock_expensive,
+                             &attr);
+    toku_pthread_rwlock_init(
+        *cachetable_m_pending_lock_cheap_key, &m_pending_lock_cheap, &attr);
     XCALLOC_N(m_table_size, m_table);
     XCALLOC_N(m_num_locks, m_mutexes);
     for (uint64_t i = 0; i < m_num_locks; i++) {
-        toku_mutex_init(&m_mutexes[i].aligned_mutex, NULL);
+        toku_mutex_init(
+#ifdef TOKU_PFS_MUTEX_EXTENDED_CACHETABLEMMUTEX
+            *cachetable_m_mutex_key,
+#else
+            toku_uninstrumented,
+#endif
+            &m_mutexes[i].aligned_mutex,
+            nullptr);
     }
 }
 
@@ -3578,9 +3614,9 @@ ENSURE_POD(evictor);
 // This is the function that runs eviction on its own thread.
 //
 static void *eviction_thread(void *evictor_v) {
-    evictor* CAST_FROM_VOIDP(evictor, evictor_v);
+    evictor *CAST_FROM_VOIDP(evictor, evictor_v);
     evictor->run_eviction_thread();
-    return evictor_v;
+    return toku_pthread_done(evictor_v);
 }
 
 //
@@ -3629,23 +3665,27 @@ int evictor::init(long _size_limit, pair_list* _pl, cachefile_list* _cf_list, KI
 
     m_pl = _pl;
     m_cf_list = _cf_list;
-    m_kibbutz = _kibbutz;    
-    toku_mutex_init(&m_ev_thread_lock, NULL);
-    toku_cond_init(&m_flow_control_cond, NULL);
-    toku_cond_init(&m_ev_thread_cond, NULL);
-    m_num_sleepers = 0;    
+    m_kibbutz = _kibbutz;
+    toku_mutex_init(
+        *cachetable_ev_thread_lock_mutex_key, &m_ev_thread_lock, nullptr);
+    toku_cond_init(
+        *cachetable_m_flow_control_cond_key, &m_flow_control_cond, nullptr);
+    toku_cond_init(
+        *cachetable_m_ev_thread_cond_key, &m_ev_thread_cond, nullptr);
+    m_num_sleepers = 0;
     m_ev_thread_is_running = false;
     m_period_in_seconds = eviction_period;
 
     unsigned int seed = (unsigned int) time(NULL);
     int r = myinitstate_r(seed, m_random_statebuf, sizeof m_random_statebuf, &m_random_data);
     assert_zero(r);
 
-    // start the background thread    
+    // start the background thread
     m_run_thread = true;
     m_num_eviction_thread_runs = 0;
     m_ev_thread_init = false;
-    r = toku_pthread_create(&m_ev_thread, NULL, eviction_thread, this); 
+    r = toku_pthread_create(
+        *eviction_thread_key, &m_ev_thread, nullptr, eviction_thread, this);
     if (r == 0) {
         m_ev_thread_init = true;
     }
@@ -4702,7 +4742,7 @@ static_assert(std::is_pod<cachefile_list>::value, "cachefile_list isn't POD");
 void cachefile_list::init() {
     m_next_filenum_to_use.fileid = 0;
     m_next_hash_id_to_use = 0;
-    toku_pthread_rwlock_init(&m_lock, NULL);
+    toku_pthread_rwlock_init(*cachetable_m_lock_key, &m_lock, nullptr);
     m_active_filenum.create();
     m_active_fileid.create();
     m_stale_fileid.create();

storage/tokudb/PerconaFT/ft/cachetable/checkpoint.cc

@@ -86,8 +86,16 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 #include "util/frwlock.h"
 #include "util/status.h"
 
-void
-toku_checkpoint_get_status(CACHETABLE ct, CHECKPOINT_STATUS statp) {
+toku_instr_key *checkpoint_safe_mutex_key;
+toku_instr_key *checkpoint_safe_rwlock_key;
+toku_instr_key *multi_operation_lock_key;
+toku_instr_key *low_priority_multi_operation_lock_key;
+
+toku_instr_key *rwlock_cond_key;
+toku_instr_key *rwlock_wait_read_key;
+toku_instr_key *rwlock_wait_write_key;
+
+void toku_checkpoint_get_status(CACHETABLE ct, CHECKPOINT_STATUS statp) {
     cp_status.init();
     CP_STATUS_VAL(CP_PERIOD) = toku_get_checkpoint_period_unlocked(ct);
     *statp = cp_status;
@@ -116,11 +124,14 @@ multi_operation_lock_init(void) {
 #if defined(HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP)
     pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
 #else
-    // TODO: need to figure out how to make writer-preferential rwlocks
-    // happen on osx
+// TODO: need to figure out how to make writer-preferential rwlocks
+// happen on osx
 #endif
-    toku_pthread_rwlock_init(&multi_operation_lock, &attr);
-    toku_pthread_rwlock_init(&low_priority_multi_operation_lock, &attr);
+    toku_pthread_rwlock_init(
+        *multi_operation_lock_key, &multi_operation_lock, &attr);
+    toku_pthread_rwlock_init(*low_priority_multi_operation_lock_key,
+                             &low_priority_multi_operation_lock,
+                             &attr);
     pthread_rwlockattr_destroy(&attr);
     locked_mo = false;
 }
@@ -145,10 +156,15 @@ multi_operation_checkpoint_unlock(void) {
     toku_pthread_rwlock_wrunlock(&low_priority_multi_operation_lock);    
 }
 
-static void
-checkpoint_safe_lock_init(void) {
-    toku_mutex_init(&checkpoint_safe_mutex, NULL);
-    checkpoint_safe_lock.init(&checkpoint_safe_mutex);
+static void checkpoint_safe_lock_init(void) {
+    toku_mutex_init(
+        *checkpoint_safe_mutex_key, &checkpoint_safe_mutex, nullptr);
+    checkpoint_safe_lock.init(&checkpoint_safe_mutex
+#ifdef TOKU_MYSQL_WITH_PFS
+                              ,
+                              *checkpoint_safe_rwlock_key
+#endif
+                              );
     locked_cs = false;
 }