From 5dd6a1ad4568236b02034e05ef74295c0c40dbe2 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 8 Aug 2016 13:52:43 -0400 Subject: [PATCH] add mem fence-post verification code defining MEMFENCE will tell the gc to validate all object tags (before they get freed) to attempt to detect out-of-bounds writes that corrupted a object tag or the freelist --- src/gc-debug.c | 119 +++++++++++++++++++++++++++++++++++++++++++++++++ src/gc.c | 15 +++++-- src/gc.h | 9 ++++ src/options.h | 5 +++ 4 files changed, 144 insertions(+), 4 deletions(-) diff --git a/src/gc-debug.c b/src/gc-debug.c index 3536eaaf75095..880c3e9235799 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -4,6 +4,12 @@ #include #include +// re-include assert.h without NDEBUG, +// so that we can always use the assert macro in this file +// for use under their respective enable flags +#undef NDEBUG +#include + #ifdef __cplusplus extern "C" { #endif @@ -250,6 +256,119 @@ void gc_verify(jl_ptls_t ptls) } #endif +#ifdef MEMFENCE +static uint8_t freelist_map[GC_PAGE_SZ / sizeof(void*) / 8]; +static int freelist_zerod; +void gc_verify_tags(void) +{ + // verify the freelist chains look valid + for (int t_i = 0; t_i < jl_n_threads; t_i++) { + jl_ptls_t ptls2 = jl_all_tls_states[t_i]; + for (int i = 0; i < JL_GC_N_POOLS; i++) { + // for all pools, iterate its freelist + jl_gc_pool_t *p = &ptls2->heap.norm_pools[i]; + jl_taggedvalue_t *next = p->freelist; + jl_taggedvalue_t *last = NULL; + char *allocating = gc_page_data(next); + while (next) { + // and assert that the freelist values aren't gc-marked + assert(next->bits.gc == 0); + // TODO: verify they are ordered and on the right byte boundaries + if (gc_page_data(next) != gc_page_data(last)) { + // and verify that the chain looks valid + jl_gc_pagemeta_t *pg = page_metadata(next); + assert(pg->osize == p->osize); + if (gc_page_data(next) != allocating) { + // when not currently allocating on this page, fl_begin_offset should be correct + assert(next == page_pfl_beg(pg)); + } + } + last = next; + next = next->next; + } + } + } + + + // verify that all the objects on every page are either valid julia objects + // or are part of the freelist or are on the allocated half of a page + for (int region_i = 0; region_i < REGION_COUNT; region_i++) { + if (!regions[region_i].pages) + break; + region_t *region = ®ions[region_i]; + for (int pg_i = 0; pg_i <= region->ub; pg_i++) { + uint32_t line = region->allocmap[pg_i]; + if (line) { + for (int j = 0; j < 32; j++) { + if ((line >> j) & 1) { + // for all pages in use + jl_gc_pagemeta_t *pg = ®ion->meta[pg_i*32 + j]; + int p_n = pg->pool_n; + int t_n = pg->thread_n; + jl_ptls_t ptls2 = jl_all_tls_states[t_n]; + jl_gc_pool_t *p = &ptls2->heap.norm_pools[p_n]; + int osize = pg->osize; + char *data = pg->data; + char *page_begin = data + GC_PAGE_OFFSET; + jl_taggedvalue_t *v = (jl_taggedvalue_t*)page_begin; + char *lim = data + GC_PAGE_SZ - osize; + // reset the freelist map to zero + if (!freelist_zerod) { + memset(freelist_map, 0, sizeof(freelist_map)); + freelist_zerod = 1; + } + // check for p in new newpages list + jl_taggedvalue_t *halfpages = p->newpages; + while (halfpages) { + char *cur_page = gc_page_data((char*)halfpages - 1); + if (cur_page == data) { + lim = (char*)halfpages - 1; + break; + } + halfpages = *(jl_taggedvalue_t**)cur_page; + } + // compute the freelist_map + if (pg->nfree) { + jl_taggedvalue_t *next = NULL; + if (gc_page_data(p->freelist) == data) { + // currently allocating on this page + next = p->freelist; + assert(page_metadata(next)->osize == osize); + freelist_zerod = 0; + } + else if (pg->fl_begin_offset != (uint16_t)-1) { + // part of free list exists on this page + next = page_pfl_beg(pg); + freelist_zerod = 0; + } + assert(halfpages || next); + while (gc_page_data(next) == data) { + int obj_idx = (((char*)next) - page_begin) / sizeof(void*); + freelist_map[obj_idx / 8] |= 1 << (obj_idx % 7); + next = next->next; + } + } + // validate all of the tags on the page + while ((char*)v <= lim) { + int obj_idx = (((char*)v) - page_begin) / sizeof(void*); + int in_freelist = freelist_map[obj_idx / 8] & (1 << (obj_idx % 7)); + if (!in_freelist) { + jl_value_t *dt = jl_typeof(jl_valueof(v)); + // 0x10 (type) and 0x20 (singleton) are used by the incremental serializer to invalidate objects + if (dt != (jl_value_t*)jl_buff_tag && v->header != 0x10 && v->header != 0x20) { + assert(jl_typeof(dt) == (jl_value_t*)jl_datatype_type); + } + } + v = (jl_taggedvalue_t*)((char*)v + osize); + } + } + } + } + } + } +} +#endif + #ifdef GC_DEBUG_ENV JL_DLLEXPORT jl_gc_debug_env_t jl_gc_debug_env = { 0, 0, diff --git a/src/gc.c b/src/gc.c index b5ac77e6a7c4d..c95a9fbc62d59 100644 --- a/src/gc.c +++ b/src/gc.c @@ -975,8 +975,14 @@ static jl_taggedvalue_t **sweep_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, jl_t assert(!freedall); pg->has_marked = has_marked; pg->has_young = has_young; - pg->fl_begin_offset = pfl_begin ? (char*)pfl_begin - data : (uint16_t)-1; - pg->fl_end_offset = pfl_begin ? (char*)pfl - data : (uint16_t)-1; + if (pfl_begin) { + pg->fl_begin_offset = (char*)pfl_begin - data; + pg->fl_end_offset = (char*)pfl - data; + } + else { + pg->fl_begin_offset = -1; + pg->fl_end_offset = -1; + } pg->nfree = pg_nfree; if (sweep_full) { @@ -1054,7 +1060,7 @@ static void gc_sweep_pool(int sweep_full) // update metadata of pages that were pointed to by freelist or newpages from a pool // i.e. pages being the current allocation target - for (int t_i = 0;t_i < jl_n_threads;t_i++) { + for (int t_i = 0; t_i < jl_n_threads; t_i++) { jl_ptls_t ptls2 = jl_all_tls_states[t_i]; for (int i = 0; i < JL_GC_N_POOLS; i++) { jl_gc_pool_t *p = &ptls2->heap.norm_pools[i]; @@ -1087,7 +1093,7 @@ static void gc_sweep_pool(int sweep_full) // null out terminal pointers of free lists - for (int t_i = 0;t_i < jl_n_threads;t_i++) { + for (int t_i = 0; t_i < jl_n_threads; t_i++) { for (int i = 0; i < JL_GC_N_POOLS; i++) { *pfl[t_i * JL_GC_N_POOLS + i] = NULL; } @@ -1787,6 +1793,7 @@ static void _jl_gc_collect(jl_ptls_t ptls, int full) sweep_weak_refs(); gc_sweep_other(ptls, sweep_full); gc_scrub(); + gc_verify_tags(); gc_sweep_pool(sweep_full); // sweeping is over // 6. if it is a quick sweep, put back the remembered objects in queued state diff --git a/src/gc.h b/src/gc.h index 8051e1bc7424f..6f95dc63c175e 100644 --- a/src/gc.h +++ b/src/gc.h @@ -346,6 +346,15 @@ STATIC_INLINE void gc_time_count_mallocd_array(int bits) estimate_freed, sweep_full) #endif +#ifdef MEMFENCE +void gc_verify_tags(void); +#else +static inline void gc_verify_tags(void) +{ +} +#endif + + #ifdef GC_VERIFY extern jl_value_t *lostval; void gc_verify(jl_ptls_t ptls); diff --git a/src/options.h b/src/options.h index 1f5af35b02edd..c48f729b6865e 100644 --- a/src/options.h +++ b/src/options.h @@ -36,6 +36,11 @@ // catch invalid accesses. // #define MEMDEBUG +// with MEMFENCE, the object pool headers are verified during sweep +// to help detect corruption due to fence-post write errors +// #define MEMFENCE + + // GC_VERIFY force a full verification gc along with every quick gc to ensure no // reachable memory is freed #ifndef GC_VERIFY