Permalink
Browse files

Adds memory allocator hooks api to memcached

The code here is for the most part final. With that said I haven't
done any clean up so keep that in mind. For example you will see
that this commit has a .DS_Store file in it.

Change-Id: Ie67c15c8e0dcc9f8f6e5cf026861e9ad54b180a6
  • Loading branch information...
1 parent 3797ae0 commit 52cacf7c3b32b29f6d63b7c698c38664dd50da5f Mike Wiederhold committed with Feb 10, 2012
View
@@ -5,6 +5,7 @@ man_MANS = doc/memcached.1
bin_PROGRAMS = engine_testapp memcached mcstat
noinst_PROGRAMS = sizes testapp timedrun
pkginclude_HEADERS = \
+ include/memcached/allocator_hooks.h \
include/memcached/callback.h \
include/memcached/config_parser.h \
include/memcached/engine.h \
@@ -20,9 +21,11 @@ pkginclude_HEADERS = \
include/memcached/vbucket.h \
include/memcached/visibility.h
+
# libmemcached_utilities.la needs to be listed first because some of the
# other modules depend on it
-pkglib_LTLIBRARIES = libmemcached_utilities.la \
+pkglib_LTLIBRARIES = libmalloc_interposer.la \
+ libmemcached_utilities.la \
ascii_scrub.la \
basic_engine_testsuite.la \
default_engine.la \
@@ -94,12 +97,21 @@ memcached_SOURCES = \
daemon/stats.c \
daemon/stats.h \
daemon/thread.c \
+ daemon/mem_hooks.c \
+ daemon/mem_hooks.h \
trace.h
memcached_CPPFLAGS = $(CPPFLAGS) -I$(top_srcdir)/daemon
memcached_LDFLAGS =-R '$(pkglibdir)' -R '$(libdir)'
memcached_CFLAGS = @PROFILER_FLAGS@
memcached_DEPENDENCIES = libmemcached_utilities.la
-memcached_LDADD = @PROFILER_LDFLAGS@ libmemcached_utilities.la -levent $(APPLICATION_LIBS)
+memcached_LDADD = @PROFILER_LDFLAGS@ $(MALLOC_LIBS) libmemcached_utilities.la -levent $(APPLICATION_LIBS)
+
+libmalloc_interposer_la_CPPFLAGS = $(CPPFLAGS)
+libmalloc_interposer_la_LDFLAGS =-R '$(pkglibdir)' -R '$(libdir)'
+libmalloc_interposer_la_SOURCES = \
+ memory/malloc/malloc_interposer.h \
+ memory/malloc/malloc_interposer.c
+
if EMBEDDED_LIBEVENT
libevent.stamp:
View
@@ -8,6 +8,7 @@ AC_CANONICAL_SYSTEM
AC_CONFIG_SRCDIR(Makefile.am)
AM_INIT_AUTOMAKE(AC_PACKAGE_NAME, AC_PACKAGE_VERSION)
AM_CONFIG_HEADER(config.h)
+AC_CONFIG_MACRO_DIR([m4])
AC_PROG_CC
AC_DISABLE_STATIC
@@ -508,6 +509,8 @@ AC_ARG_ENABLE([breakdancer],
AM_CONDITIONAL(ENABLE_BREAKDANCE,
test "x$ac_cv_enable_breakdancer" = "xyes" -a "x$PYTHON" != "xno")
+PANDORA_HAVE_BETTER_MALLOC
+
dnl Generate output files
CPPFLAGS="-I\${top_srcdir}/include ${CPPFLAGS}"
AC_CONFIG_FILES(Makefile config/Doxyfile config/Doxyfile-api)
View
@@ -0,0 +1,65 @@
+
+#include "mem_hooks.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+#if defined(HAVE_LIBTCMALLOC)
+#include <google/malloc_extension_c.h>
+#include <google/malloc_hook_c.h>
+#else
+#error You need to write the malloc interposer
+#endif
+
+bool mc_add_new_hook(void (*hook)(const void* ptr, size_t size)) {
+ return MallocHook_AddNewHook(hook) == 1 ? true : false;
+}
+
+bool mc_remove_new_hook(void (*hook)(const void* ptr, size_t size)) {
+ return MallocHook_RemoveNewHook(hook);
+}
+
+bool mc_add_delete_hook(void (*hook)(const void* ptr)) {
+ return MallocHook_AddDeleteHook(hook);
+}
+
+bool mc_remove_delete_hook(void (*hook)(const void* ptr)) {
+ return MallocHook_RemoveDeleteHook(hook);
+}
+
+void mc_get_allocator_stats(allocator_stat stats[]) {
+#if defined(HAVE_LIBTCMALLOC)
+ size_t allocated_memory = 0;
+ size_t heap_size = 0;
+ size_t pageheap_free_bytes = 0;
+ size_t pageheap_unmapped_bytes = 0;
+ size_t max_thread_cache_bytes = 0;
+ size_t current_thread_cache_bytes = 0;
+
+ MallocExtension_GetNumericProperty("generic.current_allocated_bytes",
+ &allocated_memory);
+ MallocExtension_GetNumericProperty("generic.heap_size",
+ &heap_size);
+ MallocExtension_GetNumericProperty("tcmalloc.pageheap_free_bytes",
+ &pageheap_free_bytes);
+ MallocExtension_GetNumericProperty("tcmalloc.pageheap_unmapped_bytes",
+ &pageheap_unmapped_bytes);
+ MallocExtension_GetNumericProperty("tcmalloc.max_total_thread_cache_bytes",
+ &max_thread_cache_bytes);
+ MallocExtension_GetNumericProperty("tcmalloc.current_total_thread_cache_bytes",
+ &current_thread_cache_bytes);
+
+ stats[0].key = strdup("tcmalloc_allocated_bytes");
+ stats[0].value = allocated_memory;
+ stats[1].key = strdup("tcmalloc_heap_size");
+ stats[1].value = heap_size;
+ stats[2].key = strdup("tcmalloc_free_bytes");
+ stats[2].value = pageheap_free_bytes;
+ stats[3].key = strdup("tcmalloc_unmapped_bytes");
+ stats[3].value = pageheap_unmapped_bytes;
+ stats[4].key = strdup("tcmalloc_max_thread_cache_bytes");
+ stats[4].value = max_thread_cache_bytes;
+ stats[5].key = strdup("tcmalloc_current_thread_cache_bytes");
+ stats[5].value = current_thread_cache_bytes;
+#endif
+}
View
@@ -0,0 +1,18 @@
+
+#ifndef MEM_HOOKS_H
+#define MEM_HOOKS_H
+
+#include "config.h"
+#include <memcached/allocator_hooks.h>
+
+#ifndef __cplusplus
+#include <stdbool.h>
+#endif
+
+bool mc_add_new_hook(void (*hook)(const void* ptr, size_t size));
+bool mc_remove_new_hook(void (*hook)(const void* ptr, size_t size));
+bool mc_add_delete_hook(void (*hook)(const void* ptr));
+bool mc_remove_delete_hook(void (*hook)(const void* ptr));
+void mc_get_allocator_stats(allocator_stat stats[]);
+
+#endif /* MEM_HOOKS_H */
View
@@ -16,6 +16,7 @@
#include "config.h"
#include "memcached.h"
#include "memcached/extension_loggers.h"
+#include "mem_hooks.h"
#include "utilities/engine_loader.h"
#include <signal.h>
@@ -7065,14 +7066,23 @@ static SERVER_HANDLE_V1 *get_server_api(void)
.perform_callbacks = perform_callbacks,
};
+ static ALLOCATOR_HOOKS_API hooks_api = {
+ .add_new_hook = mc_add_new_hook,
+ .remove_new_hook = mc_remove_new_hook,
+ .add_delete_hook = mc_add_delete_hook,
+ .remove_delete_hook = mc_remove_delete_hook,
+ .get_allocator_stats = mc_get_allocator_stats
+ };
+
static SERVER_HANDLE_V1 rv = {
.interface = 1,
.core = &core_api,
.stat = &server_stat_api,
.extension = &extension_api,
.callback = &callback_api,
.log = &server_log_api,
- .cookie = &server_cookie_api
+ .cookie = &server_cookie_api,
+ .alloc_hooks = &hooks_api
};
if (rv.engine == NULL) {
@@ -0,0 +1,47 @@
+
+/**
+ * Use this file as an abstraction to the underlying hooks api
+ */
+
+#ifndef ALLOCATOR_HOOKS_H
+#define ALLOCATOR_HOOKS_H
+
+#if defined(HAVE_LIBTCMALLOC)
+#define ALLOCATOR_STATS_SIZE 6
+#else
+#define ALLOCATOR_STATS_SIZE 0
+#endif
+
+#ifndef __cplusplus
+#include <stdbool.h>
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef __WIN32__
+#undef interface
+#endif
+
+typedef struct allocator_stat {
+ char* key;
+ size_t value;
+} allocator_stat;
+
+/**
+ * Engine allocator hooks for memory tracking.
+ */
+typedef struct engine_allocator_hooks_v1 {
+ bool (*add_new_hook)(void (*hook)(const void* ptr, size_t size));
+ bool (*remove_new_hook)(void (*hook)(const void* ptr, size_t size));
+ bool (*add_delete_hook)(void (*hook)(const void* ptr));
+ bool (*remove_delete_hook)(void (*hook)(const void* ptr));
+ void (*get_allocator_stats)(allocator_stat stats[]);
+} ALLOCATOR_HOOKS_API;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ALLOCATOR_HOOKS_H */
@@ -15,6 +15,7 @@
#include "memcached/config_parser.h"
#include "memcached/server_api.h"
#include "memcached/callback.h"
+#include "memcached/allocator_hooks.h"
#include "memcached/extension.h"
#include "memcached/vbucket.h"
#include "memcached/engine_common.h"
@@ -74,6 +75,7 @@ extern "C" {
ENGINE_HANDLE *engine;
SERVER_LOG_API *log;
SERVER_COOKIE_API *cookie;
+ ALLOCATOR_HOOKS_API *alloc_hooks;
};
typedef enum { TAP_MUTATION = 1,
@@ -0,0 +1,34 @@
+dnl Copyright (C) 2009 Sun Microsystems
+dnl This file is free software; Sun Microsystems
+dnl gives unlimited permission to copy and/or distribute it,
+dnl with or without modifications, as long as this notice is preserved.
+
+AC_DEFUN([PANDORA_HAVE_BETTER_MALLOC],[
+ AC_REQUIRE([AC_FUNC_MALLOC])
+ AC_REQUIRE([AC_FUNC_REALLOC])
+
+
+ AC_ARG_ENABLE([tcmalloc],
+ [AS_HELP_STRING([--enable-tcmalloc],
+ [Enable linking with tcmalloc @<:@default=off@:>@])],
+ [ac_enable_tcmalloc="$enableval"],
+ [ac_enable_tcmalloc="no"])
+
+ AC_ARG_WITH([tcmalloc-headers],
+ [AS_HELP_STRING([--with-tcmalloc-headers],
+ [Location of the tcmalloc header files])],
+ [ac_cv_with_tcmalloc_headers="$withval"],
+ [ac_cv_with_tcmalloc_headers=""])
+
+ if test $ac_enable_tcmalloc != "no"; then
+dnl AC_CHECK_LIB(tcmalloc, malloc, [MALLOC_LIBS="-ltcmalloc"], [AC_MSG_ERROR([Unable to link with libtmalloc"])])
+ AC_DEFINE([HAVE_LIBTCMALLOC], [1], [Define if using tcmalloc])
+ AS_IF(test "x${ac_cv_with_tcmalloc_headers}" != "x",
+ [CPPFLAGS="-I${ac_cv_with_tcmalloc_headers} $CPPFLAGS" ])
+ MALLOC_LIBS="-ltcmalloc"
+ AC_CHECK_HEADERS(google/malloc_extension_c.h, [], [AC_MSG_ERROR([google/malloc_extension_c.h not found])])
+ AC_CHECK_HEADERS(google/malloc_hook_c.h, [], [AC_MSG_ERROR([google/malloc_hook_c.h no found])])
+ fi
+
+ AC_SUBST([MALLOC_LIBS])
+])
View
@@ -0,0 +1,44 @@
+
+#include "allocator_stats.h"
+
+void get_allocator_stats(alloc_stat *stats, int *length) {
+#if defined(HAVE_LIBTCMALLOC)
+ *length = 6;
+ size_t allocated_memory = 0;
+ size_t heap_size = 0;
+ size_t pageheap_free_bytes = 0;
+ size_t pageheap_unmapped_bytes = 0;
+ size_t max_thread_cache_bytes = 0;
+ size_t current_thread_cache_bytes = 0;
+
+ MallocExtension_GetNumericProperty("generic.current_allocated_bytes",
+ &allocated_memory);
+ MallocExtension_GetNumericProperty("generic.heap_size",
+ &heap_size);
+ MallocExtension_GetNumericProperty("tcmalloc.pageheap_free_bytes",
+ &pageheap_free_bytes);
+ MallocExtension_GetNumericProperty("tcmalloc.pageheap_unmapped_bytes",
+ &pageheap_unmapped_bytes);
+ MallocExtension_GetNumericProperty("tcmalloc.max_total_thread_cache_bytes",
+ &max_thread_cache_bytes);
+ MallocExtension_GetNumericProperty("tcmalloc.current_total_thread_cache_bytes",
+ &current_thread_cache_bytes);
+/*
+ stats = (alloc_stat*)malloc(sizeof(alloc_stat) * 6);
+ (*stats).key = strdup("tcmalloc_allocated_bytes");
+ (*stats).value = allocated_memory;
+ (*(stats + 1)).key = strdup("tcmalloc_heap_size");
+ (*(stats + 1)).value = heap_size;
+ (*(stats + 2)).key = strdup("tcmalloc_free_bytes");
+ (*(stats + 2)).value = pageheap_free_bytes;
+ (*(stats + 3)).key = strdup("tcmalloc_unmapped_bytes");
+ (*(stats + 3)).value = pageheap_unmapped_bytes;
+ (*(stats + 4)).key = strdup("tcmalloc_max_thread_cache_bytes");
+ (*(stats + 4)).value = max_thread_cache_bytes;
+ (*(stats + 5)).key = strdup("tcmalloc_current_thread_cache_bytes");
+ (*(stats + 5)).value = current_thread_cache_bytes;
+ */
+#else
+ *length = 0;
+#endif
+}
View
@@ -0,0 +1,25 @@
+
+
+#ifndef ALLOCATOR_STATS_H
+#define ALLOCATOR_STATS_H
+
+#include "config.h"
+#include <stdlib.h>
+#include <string.h>
+
+#if defined(HAVE_LIBTCMALLOC)
+#include <google/malloc_extension_c.h>
+#else
+#error Make some default malloc stats
+#endif
+
+typedef struct alloc_stat {
+ char* key;
+ size_t value;
+} alloc_stat;
+
+void get_allocator_stats(alloc_stat*, int*);
+
+
+#endif
+
@@ -0,0 +1,15 @@
+
+
+#include <stdlib.h>
+#include "malloc_interposer.h"
+
+static const interpose_t interposers[] \
+ __attribute__ ((section("__DATA, __interpose"))) = {
@trondn

trondn Feb 10, 2012

Owner

This doesn't compile with my compiler... I believe we need a more "portable" way of doing this..

+ { &mc_malloc, &malloc },
+};
+
+void* mc_malloc(size_t size) {
+ void *ret = malloc(size);
+ total_mem += size;
+ return ret;
+}
@@ -0,0 +1,17 @@
+#ifndef MEMCACHED_DEFAULT_ENGINE_H
+#define MEMCACHED_DEFAULT_ENGINE_H
+
+#include <stdio.h>
+#include <stdint.h>
+#include <unistd.h>
+
+uint64_t total_mem = 0;
+
+typedef struct interpose_s {
+ void* (*new_func)();
+ void* (*orig_func)();
+} interpose_t;
+
+void* mc_malloc(size_t);
+
+#endif

0 comments on commit 52cacf7

Please sign in to comment.