MDEV-28836: Memory alignment cleanup

Table_cache_instance: Define the structure aligned at
the CPU cache line, and remove a pad[] data member.
Krunal Bauskar reported this to improve performance on ARMv8.

aligned_malloc(): Wrapper for the Microsoft _aligned_malloc()
and the ISO/IEC 9899:2011 <stdlib.h> aligned_alloc().
Note: The parameters are in the Microsoft order (size, alignment),
opposite of aligned_alloc(alignment, size).
Note: The standard defines that size must be an integer multiple
of alignment. It is enforced by AddressSanitizer but not by GNU libc
on Linux.

aligned_free(): Wrapper for the Microsoft _aligned_free() and
the standard free().

HAVE_ALIGNED_ALLOC: A new test. Unfortunately, support for
aligned_alloc() may still be missing on some platforms.
We will fall back to posix_memalign() for those cases.

HAVE_MEMALIGN: Remove, along with any use of the nonstandard memalign().

PFS_ALIGNEMENT (sic): Removed; we will use CPU_LEVEL1_DCACHE_LINESIZE.

PFS_ALIGNED: Defined using the C++11 keyword alignas.

buf_pool_t::page_hash_table::create(),
lock_sys_t::hash_table::create():
lock_sys_t::hash_table::resize(): Pad the allocation size to an
integer multiple of the alignment.

Reviewed by: Vladislav Vaintroub

cmake/os/WindowsCache.cmake

@@ -23,6 +23,7 @@ IF(MSVC)
 SET(BFD_H_EXISTS 0 CACHE INTERNAL "")
 SET(HAVE_ACCESS 1 CACHE  INTERNAL "")
 SET(HAVE_ALARM CACHE  INTERNAL "")
+SET(HAVE_ALIGNED_ALLOC CACHE INTERNAL "")
 SET(HAVE_ALLOCA_H CACHE  INTERNAL "")
 SET(HAVE_ARPA_INET_H CACHE  INTERNAL "")
 SET(HAVE_BACKTRACE CACHE  INTERNAL "")
@@ -93,7 +94,6 @@ SET(HAVE_MALLINFO CACHE  INTERNAL "")
 SET(HAVE_MALLINFO2 CACHE  INTERNAL "")
 SET(HAVE_MALLOC_H 1 CACHE  INTERNAL "")
 SET(HAVE_MALLOC_ZONE CACHE  INTERNAL "")
-SET(HAVE_MEMALIGN CACHE  INTERNAL "")
 SET(HAVE_MEMCPY 1 CACHE  INTERNAL "")
 SET(HAVE_MEMMOVE 1 CACHE  INTERNAL "")
 SET(HAVE_MEMORY_H 1 CACHE  INTERNAL "")

config.h.cmake

@@ -19,6 +19,7 @@
 /* Headers we may want to use. */
 #cmakedefine STDC_HEADERS 1
 #cmakedefine _GNU_SOURCE 1
+#cmakedefine HAVE_ALIGNED_ALLOC 1
 #cmakedefine HAVE_ALLOCA_H 1
 #cmakedefine HAVE_ARPA_INET_H 1
 #cmakedefine HAVE_ASM_TERMBITS_H 1
@@ -163,7 +164,6 @@
 #cmakedefine HAVE_LRAND48 1
 #cmakedefine HAVE_LOCALTIME_R 1
 #cmakedefine HAVE_LSTAT 1
-#cmakedefine HAVE_MEMALIGN 1
 /* #cmakedefine HAVE_MLOCK 1 see Bug#54662 */
 #cmakedefine HAVE_NL_LANGINFO 1
 #cmakedefine HAVE_MADVISE 1

configure.cmake

@@ -324,6 +324,7 @@ ENDIF()
 CHECK_FUNCTION_EXISTS (accept4 HAVE_ACCEPT4)
 CHECK_FUNCTION_EXISTS (access HAVE_ACCESS)
 CHECK_FUNCTION_EXISTS (alarm HAVE_ALARM)
+CHECK_FUNCTION_EXISTS (aligned_alloc HAVE_ALIGNED_ALLOC)
 SET(HAVE_ALLOCA 1)
 CHECK_FUNCTION_EXISTS (backtrace HAVE_BACKTRACE)
 CHECK_FUNCTION_EXISTS (backtrace_symbols HAVE_BACKTRACE_SYMBOLS)
@@ -420,7 +421,6 @@ CHECK_FUNCTION_EXISTS (thr_setconcurrency HAVE_THR_SETCONCURRENCY)
 CHECK_FUNCTION_EXISTS (thr_yield HAVE_THR_YIELD)
 CHECK_FUNCTION_EXISTS (vasprintf HAVE_VASPRINTF)
 CHECK_FUNCTION_EXISTS (vsnprintf HAVE_VSNPRINTF)
-CHECK_FUNCTION_EXISTS (memalign HAVE_MEMALIGN)
 CHECK_FUNCTION_EXISTS (nl_langinfo HAVE_NL_LANGINFO)
 
 IF(HAVE_SYS_EVENT_H)

include/aligned.h

@@ -0,0 +1,34 @@
+/*
+   Copyright (c) 2022, MariaDB Corporation.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335  USA */
+
+inline void *aligned_malloc(size_t size, size_t alignment)
+{
+#ifdef _WIN32
+  return _aligned_malloc(size, alignment);
+#elif defined HAVE_ALIGNED_ALLOC
+  return aligned_alloc(alignment, size);
+#else
+  void *result;
+  if (posix_memalign(&result, alignment, size))
+    result= NULL;
+  return result;
+#endif
+}
+
+inline void aligned_free(void *ptr)
+{
+  IF_WIN(_aligned_free,free)(ptr);
+}

include/my_global.h

@@ -1,6 +1,6 @@
 /*
    Copyright (c) 2001, 2013, Oracle and/or its affiliates.
-   Copyright (c) 2009, 2021, MariaDB Corporation.
+   Copyright (c) 2009, 2022, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by

mysys/my_lockmem.c

@@ -1,4 +1,5 @@
 /* Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2022, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -17,6 +18,7 @@
 
 #include "mysys_priv.h"
 #include "mysys_err.h"
+#include "aligned.h"
 #include <my_list.h>
 
 #ifdef HAVE_MLOCK
@@ -39,7 +41,7 @@ uchar *my_malloc_lock(uint size,myf MyFlags)
   DBUG_ENTER("my_malloc_lock");
 
   size=((size-1) & ~(pagesize-1))+pagesize;
-  if (!(ptr=memalign(pagesize,size)))
+  if (!(ptr=aligned_malloc(size,pagesize)))
   {
     if (MyFlags & (MY_FAE+MY_WME))
       my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_FATAL), size);
@@ -91,7 +93,7 @@ void my_free_lock(uchar *ptr)
   }
   mysql_mutex_unlock(&THR_LOCK_malloc);
   my_free(element);
-  free(ptr);					/* Free even if not locked */
+  aligned_free(ptr);				/* Free even if not locked */
 }
 
 #endif /* HAVE_MLOCK */

sql/table_cache.cc

@@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2012, Oracle and/or its affiliates.
-   Copyright (c) 2010, 2011 Monty Program Ab
+   Copyright (c) 2010, 2022, MariaDB Corporation.
    Copyright (C) 2013 Sergey Vojtovich and MariaDB Foundation
 
    This program is free software; you can redistribute it and/or modify
@@ -50,6 +50,7 @@
 #include "lf.h"
 #include "table.h"
 #include "sql_base.h"
+#include "aligned.h"
 
 
 /** Configuration. */
@@ -122,6 +123,7 @@ struct Table_cache_instance
     records, Share_free_tables::List (TABLE::prev and TABLE::next),
     TABLE::in_use.
   */
+  alignas(CPU_LEVEL1_DCACHE_LINESIZE)
   mysql_mutex_t LOCK_table_cache;
   I_P_List <TABLE, I_P_List_adapter<TABLE, &TABLE::global_free_next,
                                     &TABLE::global_free_prev>,
@@ -130,11 +132,10 @@ struct Table_cache_instance
   ulong records;
   uint mutex_waits;
   uint mutex_nowaits;
-  /** Avoid false sharing between instances */
-  char pad[CPU_LEVEL1_DCACHE_LINESIZE];
 
   Table_cache_instance(): records(0), mutex_waits(0), mutex_nowaits(0)
   {
+    static_assert(!(sizeof(*this) % CPU_LEVEL1_DCACHE_LINESIZE), "alignment");
     mysql_mutex_init(key_LOCK_table_cache, &LOCK_table_cache,
                      MY_MUTEX_INIT_FAST);
   }
@@ -146,6 +147,10 @@ struct Table_cache_instance
     DBUG_ASSERT(records == 0);
   }
 
+  static void *operator new[](size_t size)
+  { return aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE); }
+  static void operator delete[](void *ptr) { aligned_free(ptr); }
+
   /**
     Lock table cache mutex and check contention.
 

storage/innobase/buf/buf0buf.cc

@@ -1085,9 +1085,10 @@ inline const buf_block_t *buf_pool_t::chunk_t::not_freed() const
 void buf_pool_t::page_hash_table::create(ulint n)
 {
   n_cells= ut_find_prime(n);
-  const size_t size= pad(n_cells) * sizeof *array;
-  void* v= aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE);
-  memset(v, 0, size);
+  const size_t size= MY_ALIGN(pad(n_cells) * sizeof *array,
+                              CPU_LEVEL1_DCACHE_LINESIZE);
+  void *v= aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE);
+  memset_aligned<CPU_LEVEL1_DCACHE_LINESIZE>(v, 0, size);
   array= static_cast<hash_chain*>(v);
 }
 

storage/innobase/include/buf0buf.h

@@ -332,27 +332,6 @@ buf_page_is_corrupted(
 	ulint			fsp_flags)
 	MY_ATTRIBUTE((warn_unused_result));
 
-inline void *aligned_malloc(size_t size, size_t align)
-{
-#ifdef _MSC_VER
-  return _aligned_malloc(size, align);
-#else
-  void *result;
-  if (posix_memalign(&result, align, size))
-    result= NULL;
-  return result;
-#endif
-}
-
-inline void aligned_free(void *ptr)
-{
-#ifdef _MSC_VER
-  _aligned_free(ptr);
-#else
-  free(ptr);
-#endif
-}
-
 /** Read the key version from the page. In full crc32 format,
 key version is stored at {0-3th} bytes. In other format, it is
 stored in 26th position.

storage/innobase/include/univ.i

@@ -67,6 +67,7 @@ support cross-platform development and expose comonly used SQL names. */
 
 #include <my_global.h>
 #include "my_counter.h"
+#include "aligned.h"
 #include <m_string.h>
 #include <mysqld_error.h>
 

storage/innobase/include/ut0pool.h

@@ -64,15 +64,8 @@ struct Pool {
 
 		ut_a(m_start == 0);
 
-#ifdef _MSC_VER
 		m_start = static_cast<Element*>(
-			_aligned_malloc(m_size, CPU_LEVEL1_DCACHE_LINESIZE));
-#else
-		void* start;
-		ut_a(!posix_memalign(&start, CPU_LEVEL1_DCACHE_LINESIZE,
-				     m_size));
-		m_start = static_cast<Element*>(start);
-#endif
+			aligned_malloc(m_size, CPU_LEVEL1_DCACHE_LINESIZE));
 		memset_aligned<CPU_LEVEL1_DCACHE_LINESIZE>(
 			m_start, 0, m_size);
 

storage/innobase/lock/lock0lock.cc

@@ -75,9 +75,10 @@ struct TableLockGetNode
 void lock_sys_t::hash_table::create(ulint n)
 {
   n_cells= ut_find_prime(n);
-  const size_t size= pad(n_cells) * sizeof *array;
-  void* v= aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE);
-  memset(v, 0, size);
+  const size_t size= MY_ALIGN(pad(n_cells) * sizeof *array,
+                              CPU_LEVEL1_DCACHE_LINESIZE);
+  void *v= aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE);
+  memset_aligned<CPU_LEVEL1_DCACHE_LINESIZE>(v, 0, size);
   array= static_cast<hash_cell_t*>(v);
 }
 
@@ -87,9 +88,10 @@ void lock_sys_t::hash_table::resize(ulint n)
 {
   ut_ad(lock_sys.is_writer());
   ulint new_n_cells= ut_find_prime(n);
-  const size_t size= pad(new_n_cells) * sizeof *array;
-  void* v= aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE);
-  memset(v, 0, size);
+  const size_t size= MY_ALIGN(pad(new_n_cells) * sizeof *array,
+                              CPU_LEVEL1_DCACHE_LINESIZE);
+  void *v= aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE);
+  memset_aligned<CPU_LEVEL1_DCACHE_LINESIZE>(v, 0, size);
   hash_cell_t *new_array= static_cast<hash_cell_t*>(v);
 
   for (auto i= pad(n_cells); i--; )

storage/perfschema/pfs_digest.cc

@@ -1,4 +1,5 @@
 /* Copyright (c) 2008, 2021, Oracle and/or its affiliates.
+  Copyright (c) 2022, MariaDB Corporation.
 
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License, version 2.0,
@@ -56,7 +57,7 @@ bool flag_statements_digest= true;
   Current index in Stat array where new record is to be inserted.
   index 0 is reserved for "all else" case when entire array is full.
 */
-static PFS_ALIGNED PFS_cacheline_uint32 digest_monotonic_index;
+PFS_ALIGNED static PFS_cacheline_uint32 digest_monotonic_index;
 
 bool digest_full= false;
 

storage/perfschema/pfs_events_stages.h

@@ -1,4 +1,5 @@
 /* Copyright (c) 2010, 2021, Oracle and/or its affiliates.
+  Copyright (c) 2022, MariaDB Corporation.
 
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License, version 2.0,
@@ -49,7 +50,7 @@ extern bool flag_events_stages_history;
 extern bool flag_events_stages_history_long;
 
 extern bool events_stages_history_long_full;
-extern PFS_ALIGNED PFS_cacheline_uint32 events_stages_history_long_index;
+PFS_ALIGNED extern PFS_cacheline_uint32 events_stages_history_long_index;
 extern PFS_events_stages *events_stages_history_long_array;
 extern ulong events_stages_history_long_size;
 

storage/perfschema/pfs_events_statements.h

@@ -1,4 +1,5 @@
 /* Copyright (c) 2010, 2021, Oracle and/or its affiliates.
+  Copyright (c) 2022, MariaDB Corporation.
 
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License, version 2.0,
@@ -132,7 +133,7 @@ extern bool flag_events_statements_history;
 extern bool flag_events_statements_history_long;
 
 extern bool events_statements_history_long_full;
-extern PFS_ALIGNED PFS_cacheline_uint32 events_statements_history_long_index;
+PFS_ALIGNED extern PFS_cacheline_uint32 events_statements_history_long_index;
 extern PFS_events_statements *events_statements_history_long_array;
 extern size_t events_statements_history_long_size;
 

storage/perfschema/pfs_events_waits.h

@@ -1,5 +1,5 @@
 /* Copyright (c) 2008, 2021, Oracle and/or its affiliates..
-   Copyright (c) 2017, 2019, MariaDB Corporation.
+   Copyright (c) 2017, 2012, MariaDB Corporation.
 
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License, version 2.0,
@@ -127,7 +127,7 @@ extern bool flag_global_instrumentation;
 extern bool flag_thread_instrumentation;
 
 extern bool events_waits_history_long_full;
-extern PFS_ALIGNED PFS_cacheline_uint32 events_waits_history_long_index;
+PFS_ALIGNED extern PFS_cacheline_uint32 events_waits_history_long_index;
 extern PFS_events_waits *events_waits_history_long_array;
 extern ulong events_waits_history_long_size;