Permalink
Browse files

Bug #18871046 SET NUMA MEMPOLICY FOR OPTIMUM MYSQLD PERFORMANCE

Problem:

This is a contributed patch (bug#72811). The original contributor
states that, "Historically, the suggested way to run MySQL on NUMA
machines has been to run with "numactl --interleave=all" so that
things like the InnoDB buffer pool are spread across all NUMA
nodes." Currently InnoDB buffer pool is allocated without using
NUMA features.

Solution:

Provide an option (--innodb_numa_interleave) to allocate InnoDB buffer
pool using MPOL_INTERLEAVE memory policy of NUMA. What this patch does
is the following:

1.  If the option innodb_numa_interleave is enabled, then set the numa
    memory policy MPOL_INTERLEAVE for the mysqld process.
2.  Allocate the buffer pool.
3.  If the option innodb_numa_interleave is enabled, then set the numa
    memory policy back to MPOL_DEFAULT for the mysqld process.

So all allocations that happen between 1 and 3 will be using the
MPOL_INTERLEAVE memory policy.  This is a basic patch to make InnoDB
numa aware.  More fine grained control is possible and will be taken
up later.

rb#7309 approved by allen.
  • Loading branch information...
gurusami committed Jul 9, 2015
1 parent 6d836cf commit 242fa2c92de304637e794e531df1f1b86b8d1dee
View
@@ -1,4 +1,4 @@
/* Copyright (c) 2009, 2014, Oracle and/or its affiliates. All rights reserved.
/* Copyright (c) 2009, 2015, Oracle and/or its affiliates. All rights reserved.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -664,5 +664,6 @@
/* CPU information */
#cmakedefine CPU_LEVEL1_DCACHE_LINESIZE @CPU_LEVEL1_DCACHE_LINESIZE@
#cmakedefine HAVE_LIBNUMA 1
#endif
View
@@ -1,4 +1,4 @@
# Copyright (c) 2009, 2014, Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2009, 2015, Oracle and/or its affiliates. All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -1138,3 +1138,17 @@ SET(CMAKE_EXTRA_INCLUDE_FILES)
CHECK_STRUCT_HAS_MEMBER("struct dirent" d_ino "dirent.h" STRUCT_DIRENT_HAS_D_INO)
CHECK_STRUCT_HAS_MEMBER("struct dirent" d_namlen "dirent.h" STRUCT_DIRENT_HAS_D_NAMLEN)
SET(SPRINTF_RETURNS_INT 1)
CHECK_INCLUDE_FILES (numaif.h HAVE_NUMAIF_H)
SET(WITH_NUMA 1 CACHE BOOL "Explicitly set NUMA memory allocation policy")
IF(HAVE_NUMAIF_H AND WITH_NUMA)
SET(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} numa)
CHECK_C_SOURCE_COMPILES(
"
#include <numaif.h>
int main()
{
set_mempolicy(MPOL_DEFAULT, 0, 0);
}"
HAVE_LIBNUMA)
ENDIF()
@@ -0,0 +1,18 @@
let $numa_support = `SELECT COUNT(VARIABLE_VALUE) = 1 FROM
INFORMATION_SCHEMA.GLOBAL_VARIABLES
WHERE VARIABLE_NAME='innodb_numa_interleave'`;
if ( $numa_support == 0 )
{
--skip Test requires: NUMA must be enabled
}
if ( $numa_support == 1 )
{
let $numa = `SELECT @@GLOBAL.innodb_numa_interleave = 'OFF'`;
if ( $numa == 1 )
{
--skip Test requires: NUMA must be enabled
}
}
@@ -0,0 +1,10 @@
SELECT @@GLOBAL.innodb_numa_interleave;
@@GLOBAL.innodb_numa_interleave
1
SET @@GLOBAL.innodb_numa_interleave=off;
ERROR HY000: Variable 'innodb_numa_interleave' is a read only variable
SELECT @@GLOBAL.innodb_use_native_aio;
@@GLOBAL.innodb_use_native_aio
0
SELECT @@SESSION.innodb_use_native_aio;
ERROR HY000: Variable 'innodb_use_native_aio' is a GLOBAL variable
@@ -0,0 +1,13 @@
--source include/have_innodb.inc
--source include/have_numa.inc
SELECT @@GLOBAL.innodb_numa_interleave;
--error ER_INCORRECT_GLOBAL_LOCAL_VAR
SET @@GLOBAL.innodb_numa_interleave=off;
SELECT @@GLOBAL.innodb_use_native_aio;
--error ER_INCORRECT_GLOBAL_LOCAL_VAR
SELECT @@SESSION.innodb_use_native_aio;
@@ -1,4 +1,4 @@
# Copyright (c) 2006, 2011, Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2006, 2015, Oracle and/or its affiliates. All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -29,6 +29,9 @@ IF(UNIX)
ADD_DEFINITIONS(-DLINUX_NATIVE_AIO=1)
LINK_LIBRARIES(aio)
ENDIF()
IF(HAVE_LIBNUMA)
LINK_LIBRARIES(numa)
ENDIF()
ELSEIF(CMAKE_SYSTEM_NAME MATCHES "HP*")
ADD_DEFINITIONS("-DUNIV_HPUX")
ELSEIF(CMAKE_SYSTEM_NAME STREQUAL "AIX")
@@ -53,6 +53,10 @@ Created 11/5/1995 Heikki Tuuri
#include "page0zip.h"
#include "srv0mon.h"
#include "buf0checksum.h"
#ifdef HAVE_LIBNUMA
#include <numa.h>
#include <numaif.h>
#endif // HAVE_LIBNUMA
/*
IMPLEMENTATION OF THE BUFFER POOL
@@ -1442,6 +1446,21 @@ buf_pool_init(
ut_ad(n_instances <= MAX_BUFFER_POOLS);
ut_ad(n_instances == srv_buf_pool_instances);
#ifdef HAVE_LIBNUMA
if (srv_numa_interleave) {
ib_logf(IB_LOG_LEVEL_INFO,
"Setting NUMA memory policy to MPOL_INTERLEAVE");
if (set_mempolicy(MPOL_INTERLEAVE,
numa_all_nodes_ptr->maskp,
numa_all_nodes_ptr->size) != 0) {
ib_logf(IB_LOG_LEVEL_WARN,
"Failed to set NUMA memory policy to"
" MPOL_INTERLEAVE (error: %s).",
strerror(errno));
}
}
#endif // HAVE_LIBNUMA
buf_pool_ptr = (buf_pool_t*) mem_zalloc(
n_instances * sizeof *buf_pool_ptr);
@@ -1462,6 +1481,18 @@ buf_pool_init(
btr_search_sys_create(buf_pool_get_curr_size() / sizeof(void*) / 64);
#ifdef HAVE_LIBNUMA
if (srv_numa_interleave) {
ib_logf(IB_LOG_LEVEL_INFO,
"Setting NUMA memory policy to MPOL_DEFAULT");
if (set_mempolicy(MPOL_DEFAULT, NULL, 0) != 0) {
ib_logf(IB_LOG_LEVEL_WARN,
"Failed to set NUMA memory policy to"
" MPOL_DEFAULT (error: %s).", strerror(errno));
}
}
#endif // HAVE_LIBNUMA
return(DB_SUCCESS);
}
@@ -16301,6 +16301,13 @@ static MYSQL_SYSVAR_BOOL(use_native_aio, srv_use_native_aio,
"Use native AIO if supported on this platform.",
NULL, NULL, TRUE);
#ifdef HAVE_LIBNUMA
static MYSQL_SYSVAR_BOOL(numa_interleave, srv_numa_interleave,
PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY,
"Use NUMA interleave memory policy to allocate InnoDB buffer pool.",
NULL, NULL, FALSE);
#endif // HAVE_LIBNUMA
static MYSQL_SYSVAR_BOOL(api_enable_binlog, ib_binlog_enabled,
PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY,
"Enable binlog for applications direct access InnoDB through InnoDB APIs",
@@ -16579,6 +16586,9 @@ static struct st_mysql_sys_var* innobase_system_variables[]= {
MYSQL_SYSVAR(version),
MYSQL_SYSVAR(use_sys_malloc),
MYSQL_SYSVAR(use_native_aio),
#ifdef HAVE_LIBNUMA
MYSQL_SYSVAR(numa_interleave),
#endif // HAVE_LIBNUMA
MYSQL_SYSVAR(change_buffering),
MYSQL_SYSVAR(change_buffer_max_size),
#if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG
@@ -220,6 +220,7 @@ OS (provided we compiled Innobase with it in), otherwise we will
use simulated aio we build below with threads.
Currently we support native aio on windows and linux */
extern my_bool srv_use_native_aio;
extern my_bool srv_numa_interleave;
#ifdef __WIN__
extern ibool srv_use_native_conditions;
#endif /* __WIN__ */
@@ -878,6 +879,7 @@ struct srv_slot_t{
#else /* !UNIV_HOTBACKUP */
# define srv_use_adaptive_hash_indexes FALSE
# define srv_use_native_aio FALSE
# define srv_numa_interleave FALSE
# define srv_force_recovery 0UL
# define srv_set_io_thread_op_info(t,info) ((void) 0)
# define srv_reset_io_thread_op_info() ((void) 0)
@@ -147,6 +147,7 @@ OS (provided we compiled Innobase with it in), otherwise we will
use simulated aio we build below with threads.
Currently we support native aio on windows and linux */
UNIV_INTERN my_bool srv_use_native_aio = TRUE;
UNIV_INTERN my_bool srv_numa_interleave = FALSE;
#ifdef __WIN__
/* Windows native condition variables. We use runtime loading / function

0 comments on commit 242fa2c

Please sign in to comment.