MDEV-6450 - MariaDB crash on Power8 when built with advance tool chain

Part of this work is based on Stewart Smitch's memory barrier and lower priori
patches for power8.

- Added memory syncronization for innodb & xtradb for power8.
- Added HAVE_WINDOWS_MM_FENCE to CMakeList.txt
- Added os_isync to fix a syncronization problem on power
- Added log_get_lsn_nowait which is now used srv_error_monitor_thread to ensur
  if log mutex is locked.

All changes done both for InnoDB and Xtradb

include/my_cpu.h

@@ -0,0 +1,44 @@
+/* Copyright (c) 2013, MariaDB foundation Ab and SkySQL
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA
+*/
+
+/* instructions for specific cpu's */
+
+/*
+  Macros for adjusting thread priority (hardware multi-threading)
+  The defines are the same ones used by the linux kernel
+*/
+
+#if defined(__powerpc__)
+/* Very low priority */
+#define HMT_very_low() asm volatile("or 31,31,31")
+/* Low priority */
+#define HMT_low() asm volatile("or 1,1,1")
+/* Medium low priority */
+#define HMT_medium_low() asm volatile("or 6,6,6")
+/* Medium priority */
+#define HMT_medium() asm volatile("or 2,2,2")
+/* Medium high priority */
+#define HMT_medium_high() asm volatile("or 5,5,5")
+/* High priority */
+#define HMT_high() asm volatile("or 3,3,3")
+#else
+#define HMT_very_low()
+#define HMT_low()
+#define HMT_medium_low()
+#define HMT_medium()
+#define HMT_medium_high()
+#define HMT_high()
+#endif

storage/innobase/CMakeLists.txt

@@ -85,12 +85,39 @@ IF(NOT CMAKE_CROSSCOMPILING)
   }"
   HAVE_IB_GCC_ATOMIC_BUILTINS
   )
+  CHECK_C_SOURCE_RUNS(
+  "#include<stdint.h>
+  int main()
+  {
+    __sync_synchronize();
+    return(0);
+  }"
+  HAVE_IB_GCC_SYNC_SYNCHRONISE
+  )
+  CHECK_C_SOURCE_RUNS(
+  "#include<stdint.h>
+  int main()
+  {
+    __atomic_thread_fence(__ATOMIC_ACQUIRE);
+    __atomic_thread_fence(__ATOMIC_RELEASE);
+    return(0);
+  }"
+  HAVE_IB_GCC_ATOMIC_THREAD_FENCE
+  )
 ENDIF()
 
 IF(HAVE_IB_GCC_ATOMIC_BUILTINS)
  ADD_DEFINITIONS(-DHAVE_IB_GCC_ATOMIC_BUILTINS=1)
 ENDIF()
 
+IF(HAVE_IB_GCC_SYNC_SYNCHRONISE)
+ ADD_DEFINITIONS(-DHAVE_IB_GCC_SYNC_SYNCHRONISE=1)
+ENDIF()
+
+IF(HAVE_IB_GCC_ATOMIC_THREAD_FENCE)
+ ADD_DEFINITIONS(-DHAVE_IB_GCC_ATOMIC_THREAD_FENCE=1)
+ENDIF()
+
  # either define HAVE_IB_ATOMIC_PTHREAD_T_GCC or not
 IF(NOT CMAKE_CROSSCOMPILING)
   CHECK_C_SOURCE_RUNS(
@@ -169,10 +196,21 @@ IF(CMAKE_SYSTEM_NAME STREQUAL "SunOS")
       return(0);
     }
   " HAVE_IB_ATOMIC_PTHREAD_T_SOLARIS)
+  CHECK_C_SOURCE_COMPILES(
+  "#include <mbarrier.h>
+  int main() {
+    __machine_r_barrier();
+    __machine_w_barrier();
+    return(0);
+  }"
+  HAVE_IB_MACHINE_BARRIER_SOLARIS)
   ENDIF()
   IF(HAVE_IB_ATOMIC_PTHREAD_T_SOLARIS)
     ADD_DEFINITIONS(-DHAVE_IB_ATOMIC_PTHREAD_T_SOLARIS=1)
   ENDIF()
+  IF(HAVE_IB_MACHINE_BARRIER_SOLARIS)
+    ADD_DEFINITIONS(-DHAVE_IB_MACHINE_BARRIER_SOLARIS=1)
+  ENDIF()
 ENDIF()
 
 
@@ -190,6 +228,7 @@ ENDIF()
 
 IF(MSVC)
   ADD_DEFINITIONS(-DHAVE_WINDOWS_ATOMICS)
+  ADD_DEFINITIONS(-DHAVE_WINDOWS_MM_FENCE)
 ENDIF()
 
 

storage/innobase/include/log0log.h

@@ -154,6 +154,13 @@ UNIV_INLINE
 ib_uint64_t
 log_get_lsn(void);
 /*=============*/
+/************************************************************//**
+Gets the current lsn.
+@return	current lsn */
+UNIV_INLINE
+lsn_t
+log_get_lsn_nowait(void);
+/*=============*/
 /****************************************************************
 Gets the log group capacity. It is OK to read the value without
 holding log_sys->mutex because it is constant.

storage/innobase/include/log0log.ic

@@ -411,6 +411,26 @@ log_get_lsn(void)
 	return(lsn);
 }
 
+/************************************************************//**
+Gets the current lsn with a trylock
+@return	current lsn or 0 if false*/
+UNIV_INLINE
+lsn_t
+log_get_lsn_nowait(void)
+/*=============*/
+{
+	lsn_t	lsn;
+
+        if (mutex_enter_nowait(&(log_sys->mutex)))
+          return 0;
+
+	lsn = log_sys->lsn;
+
+	mutex_exit(&(log_sys->mutex));
+
+	return(lsn);
+}
+
 /****************************************************************
 Gets the log group capacity. It is OK to read the value without
 holding log_sys->mutex because it is constant.

storage/innobase/include/os0sync.h

@@ -416,6 +416,54 @@ clobbered */
 	"Mutexes and rw_locks use InnoDB's own implementation"
 #endif
 
+/** barrier definitions for memory ordering */
+#ifdef HAVE_IB_GCC_ATOMIC_THREAD_FENCE
+# define HAVE_MEMORY_BARRIER
+# define os_rmb	__atomic_thread_fence(__ATOMIC_ACQUIRE)
+# define os_wmb	__atomic_thread_fence(__ATOMIC_RELEASE)
+#ifdef __powerpc__
+# define os_isync  __asm __volatile ("isync":::"memory")
+#else
+#define os_isync do { } while(0)
+#endif
+
+# define IB_MEMORY_BARRIER_STARTUP_MSG \
+	"GCC builtin __atomic_thread_fence() is used for memory barrier"
+
+#elif defined(HAVE_IB_GCC_SYNC_SYNCHRONISE)
+# define HAVE_MEMORY_BARRIER
+# define os_rmb	__sync_synchronize()
+# define os_wmb	__sync_synchronize()
+# define os_isync __sync_synchronize()
+# define IB_MEMORY_BARRIER_STARTUP_MSG \
+	"GCC builtin __sync_synchronize() is used for memory barrier"
+
+#elif defined(HAVE_IB_MACHINE_BARRIER_SOLARIS)
+# define HAVE_MEMORY_BARRIER
+# include <mbarrier.h>
+# define os_rmb	__machine_r_barrier()
+# define os_wmb	__machine_w_barrier()
+# define os_isync os_rmb; os_wmb
+# define IB_MEMORY_BARRIER_STARTUP_MSG \
+	"Soralis memory ordering functions are used for memory barrier"
+
+#elif defined(HAVE_WINDOWS_MM_FENCE)
+# define HAVE_MEMORY_BARRIER
+# include <mmintrin.h>
+# define os_rmb	_mm_lfence()
+# define os_wmb	_mm_sfence()
+# define os_isync os_rmb; os_wmb
+# define IB_MEMORY_BARRIER_STARTUP_MSG \
+	"_mm_lfence() and _mm_sfence() are used for memory barrier"
+
+#else
+# define os_rmb do { } while(0)
+# define os_wmb do { } while(0)
+# define os_isync do { } while(0)
+# define IB_MEMORY_BARRIER_STARTUP_MSG \
+	"Memory barrier is not used"
+#endif
+
 #ifndef UNIV_NONINL
 #include "os0sync.ic"
 #endif

storage/innobase/include/sync0rw.ic

@@ -200,14 +200,14 @@ rw_lock_lock_word_decr(
 	ulint		amount)		/*!< in: amount to decrement */
 {
 #ifdef INNODB_RW_LOCKS_USE_ATOMICS
-        lint local_lock_word = lock->lock_word;
-	while (local_lock_word > 0) {
+	lint local_lock_word;
+        os_rmb;
+	while ((local_lock_word= lock->lock_word) > 0) {
 		if (os_compare_and_swap_lint(&lock->lock_word,
 					     local_lock_word,
 					     local_lock_word - amount)) {
 			return(TRUE);
 		}
-		local_lock_word = lock->lock_word;
 	}
 	return(FALSE);
 #else /* INNODB_RW_LOCKS_USE_ATOMICS */

storage/innobase/include/sync0sync.ic

@@ -92,6 +92,7 @@ mutex_test_and_set(
 		ut_a(mutex->lock_word == 0);
 
 		mutex->lock_word = 1;
+                os_wmb;
 	}
 
 	return((byte)ret);
@@ -147,6 +148,7 @@ mutex_get_waiters(
 
 	ptr = &(mutex->waiters);
 
+        os_rmb;
 	return(*ptr);		/* Here we assume that the read of a single
 				word from memory is atomic */
 }
@@ -181,6 +183,7 @@ mutex_exit_func(
 	to wake up possible hanging threads if
 	they are missed in mutex_signal_object. */
 
+        os_isync;
 	if (mutex_get_waiters(mutex) != 0) {
 
 		mutex_signal_object(mutex);

storage/innobase/srv/srv0srv.c

@@ -411,7 +411,12 @@ UNIV_INTERN ibool	srv_use_checksums = TRUE;
 UNIV_INTERN ulong	srv_replication_delay		= 0;
 
 /*-------------------------------------------*/
+#ifdef HAVE_MEMORY_BARRIER
+/* No idea to wait long with memory barriers */
+UNIV_INTERN ulong	srv_n_spin_wait_rounds	= 15;
+#else
 UNIV_INTERN ulong	srv_n_spin_wait_rounds	= 30;
+#endif
 UNIV_INTERN ulong	srv_n_free_tickets_to_enter = 500;
 UNIV_INTERN ulong	srv_thread_sleep_delay = 10000;
 UNIV_INTERN ulong	srv_spin_wait_delay	= 6;
@@ -2459,9 +2464,10 @@ srv_error_monitor_thread(
 	/* Try to track a strange bug reported by Harald Fuchs and others,
 	where the lsn seems to decrease at times */
 
-	new_lsn = log_get_lsn();
+        /* We have to use nowait to ensure we don't block */
+	new_lsn= log_get_lsn_nowait();
 
-	if (new_lsn < old_lsn) {
+	if (new_lsn && new_lsn < old_lsn) {
 		ut_print_timestamp(stderr);
 		fprintf(stderr,
 			"  InnoDB: Error: old log sequence number %llu"
@@ -2473,7 +2479,8 @@ srv_error_monitor_thread(
 		ut_ad(0);
 	}
 
-	old_lsn = new_lsn;
+        if (new_lsn)
+		old_lsn = new_lsn;
 
 	if (difftime(time(NULL), srv_last_monitor_time) > 60) {
 		/* We referesh InnoDB Monitor values so that averages are

storage/innobase/sync/sync0arr.c

@@ -791,6 +791,7 @@ sync_arr_cell_can_wake_up(
 
 		lock = cell->wait_object;
 
+                os_rmb;
 		if (lock->lock_word > 0) {
 		/* Either unlocked or only read locked. */
 
@@ -802,6 +803,7 @@ sync_arr_cell_can_wake_up(
 		lock = cell->wait_object;
 
                 /* lock_word == 0 means all readers have left */
+                os_rmb;
 		if (lock->lock_word == 0) {
 
 			return(TRUE);
@@ -810,6 +812,7 @@ sync_arr_cell_can_wake_up(
 		lock = cell->wait_object;
 
                 /* lock_word > 0 means no writer or reserved writer */
+                os_rmb;
 		if (lock->lock_word > 0) {
 
 			return(TRUE);