Skip to content

Commit

Permalink
mm/slub: use stackdepot to save stack trace in objects
Browse files Browse the repository at this point in the history 
Many stack traces are similar so there are many similar arrays.
Stackdepot saves each unique stack only once.

Replace field addrs in struct track with depot_stack_handle_t handle.  Use
stackdepot to save stack trace.

The benefits are smaller memory overhead and possibility to aggregate
per-cache statistics in the following patch using the stackdepot handle
instead of matching stacks manually.

[ vbabka@suse.cz: rebase to 5.17-rc1 and adjust accordingly ]

This was initially merged as commit 7886914 and reverted by commit
ae14c63 due to several issues, that should now be fixed.
The problem of unconditional memory overhead by stackdepot has been
addressed by commit 2dba5eb ("lib/stackdepot: allow optional init
and stack_table allocation by kvmalloc()"), so the dependency on
stackdepot will result in extra memory usage only when a slab cache
tracking is actually enabled, and not for all CONFIG_SLUB_DEBUG builds.
The build failures on some architectures were also addressed, and the
reported issue with xfs/433 test did not reproduce on 5.17-rc1 with this
patch.

Signed-off-by: Oliver Glitta <glittao@gmail.com>
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Cc: David Rientjes <rientjes@google.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
  • Loading branch information
@tehcaster
Oliver Glitta authored and tehcaster committed Feb 8, 2022
1 parent f633f9a commit ba10d4b
Show file tree
Hide file tree
Showing 2 changed files with 50 additions and 39 deletions.
1 change: 1 addition & 0 deletions init/Kconfig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1871,6 +1871,7 @@ config SLUB_DEBUG
default y
bool "Enable SLUB debugging support" if EXPERT
depends on SLUB && SYSFS
select STACKDEPOT if STACKTRACE_SUPPORT
help
SLUB has extensive debug support features. Disabling these can
result in significant savings in code size. This also disables
Expand Down
88 changes: 49 additions & 39 deletions mm/slub.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#include <linux/cpuset.h>
#include <linux/mempolicy.h>
#include <linux/ctype.h>
#include <linux/stackdepot.h>
#include <linux/debugobjects.h>
#include <linux/kallsyms.h>
#include <linux/kfence.h>
Expand Down Expand Up @@ -264,8 +265,8 @@ static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
#define TRACK_ADDRS_COUNT 16
struct track {
unsigned long addr; /* Called from address */
#ifdef CONFIG_STACKTRACE
unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */
#ifdef CONFIG_STACKDEPOT
depot_stack_handle_t handle;
#endif
int cpu; /* Was running on cpu */
int pid; /* Pid context */
Expand Down Expand Up @@ -724,22 +725,20 @@ static struct track *get_track(struct kmem_cache *s, void *object,
return kasan_reset_tag(p + alloc);
}

static void set_track(struct kmem_cache *s, void *object,
enum track_item alloc, unsigned long addr)
static noinline void
set_track(struct kmem_cache *s, void *object, enum track_item alloc,
unsigned long addr, gfp_t flags)
{
struct track *p = get_track(s, object, alloc);

#ifdef CONFIG_STACKTRACE
#ifdef CONFIG_STACKDEPOT
unsigned long entries[TRACK_ADDRS_COUNT];
unsigned int nr_entries;

metadata_access_enable();
nr_entries = stack_trace_save(kasan_reset_tag(p->addrs),
TRACK_ADDRS_COUNT, 3);
metadata_access_disable();

if (nr_entries < TRACK_ADDRS_COUNT)
p->addrs[nr_entries] = 0;
nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 3);
p->handle = stack_depot_save(entries, nr_entries, flags);
#endif

p->addr = addr;
p->cpu = smp_processor_id();
p->pid = current->pid;
Expand All @@ -759,20 +758,19 @@ static void init_tracking(struct kmem_cache *s, void *object)

static void print_track(const char *s, struct track *t, unsigned long pr_time)
{
depot_stack_handle_t handle __maybe_unused;

if (!t->addr)
return;

pr_err("%s in %pS age=%lu cpu=%u pid=%d\n",
s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid);
#ifdef CONFIG_STACKTRACE
{
int i;
for (i = 0; i < TRACK_ADDRS_COUNT; i++)
if (t->addrs[i])
pr_err("\t%pS\n", (void *)t->addrs[i]);
else
break;
}
#ifdef CONFIG_STACKDEPOT
handle = READ_ONCE(t->handle);
if (handle)
stack_depot_print(handle);
else
pr_err("object allocation/free stack trace missing\n");
#endif
}

Expand Down Expand Up @@ -1304,9 +1302,9 @@ static inline int alloc_consistency_checks(struct kmem_cache *s,
return 1;
}

static noinline int alloc_debug_processing(struct kmem_cache *s,
struct slab *slab,
void *object, unsigned long addr)
static noinline int
alloc_debug_processing(struct kmem_cache *s, struct slab *slab, void *object,
unsigned long addr, gfp_t flags)
{
if (s->flags & SLAB_CONSISTENCY_CHECKS) {
if (!alloc_consistency_checks(s, slab, object))
Expand All @@ -1315,7 +1313,7 @@ static noinline int alloc_debug_processing(struct kmem_cache *s,

/* Success perform special debug activities for allocs */
if (s->flags & SLAB_STORE_USER)
set_track(s, object, TRACK_ALLOC, addr);
set_track(s, object, TRACK_ALLOC, addr, flags);
trace(s, slab, object, 1);
init_object(s, object, SLUB_RED_ACTIVE);
return 1;
Expand Down Expand Up @@ -1395,7 +1393,7 @@ static noinline int free_debug_processing(
}

if (s->flags & SLAB_STORE_USER)
set_track(s, object, TRACK_FREE, addr);
set_track(s, object, TRACK_FREE, addr, GFP_NOWAIT);
trace(s, slab, object, 0);
/* Freepointer not overwritten by init_object(), SLAB_POISON moved it */
init_object(s, object, SLUB_RED_INACTIVE);
Expand Down Expand Up @@ -1632,7 +1630,8 @@ static inline
void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) {}

static inline int alloc_debug_processing(struct kmem_cache *s,
struct slab *slab, void *object, unsigned long addr) { return 0; }
struct slab *slab, void *object, unsigned long addr,
gfp_t flags) { return 0; }

static inline int free_debug_processing(
struct kmem_cache *s, struct slab *slab,
Expand Down Expand Up @@ -3033,7 +3032,7 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
check_new_slab:

if (kmem_cache_debug(s)) {
if (!alloc_debug_processing(s, slab, freelist, addr)) {
if (!alloc_debug_processing(s, slab, freelist, addr, gfpflags)) {
/* Slab failed checks. Next slab needed */
goto new_slab;
} else {
Expand Down Expand Up @@ -4221,6 +4220,9 @@ static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags)
s->remote_node_defrag_ratio = 1000;
#endif

if (s->flags & SLAB_STORE_USER && IS_ENABLED(CONFIG_STACKDEPOT))
stack_depot_init();

/* Initialize the pre-computed randomized freelist if slab is up */
if (slab_state >= UP) {
if (init_cache_random_seq(s))
Expand Down Expand Up @@ -4352,18 +4354,26 @@ void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
objp = fixup_red_left(s, objp);
trackp = get_track(s, objp, TRACK_ALLOC);
kpp->kp_ret = (void *)trackp->addr;
#ifdef CONFIG_STACKTRACE
for (i = 0; i < KS_ADDRS_COUNT && i < TRACK_ADDRS_COUNT; i++) {
kpp->kp_stack[i] = (void *)trackp->addrs[i];
if (!kpp->kp_stack[i])
break;
}
#ifdef CONFIG_STACKDEPOT
{
depot_stack_handle_t handle;
unsigned long *entries;
unsigned int nr_entries;

handle = READ_ONCE(trackp->handle);
if (handle) {
nr_entries = stack_depot_fetch(handle, &entries);
for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++)
kpp->kp_stack[i] = (void *)entries[i];
}

trackp = get_track(s, objp, TRACK_FREE);
for (i = 0; i < KS_ADDRS_COUNT && i < TRACK_ADDRS_COUNT; i++) {
kpp->kp_free_stack[i] = (void *)trackp->addrs[i];
if (!kpp->kp_free_stack[i])
break;
trackp = get_track(s, objp, TRACK_FREE);
handle = READ_ONCE(trackp->handle);
if (handle) {
nr_entries = stack_depot_fetch(handle, &entries);
for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++)
kpp->kp_free_stack[i] = (void *)entries[i];
}
}
#endif
#endif
Expand Down

0 comments on commit ba10d4b

Please sign in to comment.