Skip to content

Commit

Permalink
More scalable pool allocator (JuliaLang#50137)
Browse files Browse the repository at this point in the history
  • Loading branch information
@d-netto
d-netto authored and RAI CI (GitHub Action Automation) committed Nov 10, 2023
1 parent 9e244f5 commit 87a861d
Show file tree
Hide file tree
Showing 7 changed files with 362 additions and 558 deletions.
148 changes: 39 additions & 109 deletions src/gc-debug.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,19 +27,16 @@ jl_gc_pagemeta_t *jl_gc_page_metadata(void *data)
// the end of the page.
JL_DLLEXPORT jl_taggedvalue_t *jl_gc_find_taggedvalue_pool(char *p, size_t *osize_p)
{
if (!page_metadata(p))
if (!gc_alloc_map_is_set(p))
// Not in the pool
return NULL;
struct jl_gc_metadata_ext info = page_metadata_ext(p);
jl_gc_pagemeta_t *meta = page_metadata(p);
char *page_begin = gc_page_data(p) + GC_PAGE_OFFSET;
// In the page header
if (p < page_begin)
return NULL;
size_t ofs = p - page_begin;
// Check if this is a free page
if (!(info.pagetable0->allocmap[info.pagetable0_i32] & (uint32_t)(1 << info.pagetable0_i)))
return NULL;
int osize = info.meta->osize;
int osize = meta->osize;
// Shouldn't be needed, just in case
if (osize == 0)
return NULL;
Expand Down Expand Up @@ -111,44 +108,14 @@ static void gc_clear_mark_page(jl_gc_pagemeta_t *pg, int bits)
}
}

static void gc_clear_mark_pagetable0(pagetable0_t *pagetable0, int bits)
{
for (int pg_i = 0; pg_i < REGION0_PG_COUNT / 32; pg_i++) {
uint32_t line = pagetable0->allocmap[pg_i];
if (line) {
for (int j = 0; j < 32; j++) {
if ((line >> j) & 1) {
gc_clear_mark_page(pagetable0->meta[pg_i * 32 + j], bits);
}
}
}
}
}

static void gc_clear_mark_pagetable1(pagetable1_t *pagetable1, int bits)
{
for (int pg_i = 0; pg_i < REGION1_PG_COUNT / 32; pg_i++) {
uint32_t line = pagetable1->allocmap0[pg_i];
if (line) {
for (int j = 0; j < 32; j++) {
if ((line >> j) & 1) {
gc_clear_mark_pagetable0(pagetable1->meta0[pg_i * 32 + j], bits);
}
}
}
}
}

static void gc_clear_mark_pagetable(int bits)
static void gc_clear_mark_outer(int bits)
{
for (int pg_i = 0; pg_i < (REGION2_PG_COUNT + 31) / 32; pg_i++) {
uint32_t line = memory_map.allocmap1[pg_i];
if (line) {
for (int j = 0; j < 32; j++) {
if ((line >> j) & 1) {
gc_clear_mark_pagetable1(memory_map.meta1[pg_i * 32 + j], bits);
}
}
for (int i = 0; i < gc_n_threads; i++) {
jl_ptls_t ptls2 = gc_all_tls_states[i];
jl_gc_pagemeta_t *pg = ptls2->page_metadata_allocd;
while (pg != NULL) {
gc_clear_mark_page(pg, bits);
pg = pg->next;
}
}
}
Expand Down Expand Up @@ -184,7 +151,7 @@ static void clear_mark(int bits)
v = v->next;
}

gc_clear_mark_pagetable(bits);
gc_clear_mark_outer(bits);
}

static void restore(void)
Expand Down Expand Up @@ -561,7 +528,6 @@ void gc_scrub_record_task(jl_task_t *t)

JL_NO_ASAN static void gc_scrub_range(char *low, char *high)
{
jl_ptls_t ptls = jl_current_task->ptls;
jl_jmp_buf *old_buf = jl_get_safe_restore();
jl_jmp_buf buf;
if (jl_setjmp(buf, 0)) {
Expand Down Expand Up @@ -786,45 +752,37 @@ void gc_final_pause_end(int64_t t0, int64_t tend)

static void gc_stats_pagetable0(pagetable0_t *pagetable0, unsigned *p0)
{
for (int pg_i = 0; pg_i < REGION0_PG_COUNT / 32; pg_i++) {
uint32_t line = pagetable0->allocmap[pg_i] | pagetable0->freemap[pg_i];
if (line) {
for (int j = 0; j < 32; j++) {
if ((line >> j) & 1) {
(*p0)++;
}
}
for (int pg_i = 0; pg_i < REGION0_PG_COUNT; pg_i++) {
uint8_t meta = pagetable0->meta[pg_i];
assert(meta == GC_PAGE_UNMAPPED || meta == GC_PAGE_ALLOCATED ||
meta == GC_PAGE_LAZILY_FREED || meta == GC_PAGE_FREED);
if (meta != GC_PAGE_UNMAPPED) {
(*p0)++;
}
}
}

static void gc_stats_pagetable1(pagetable1_t *pagetable1, unsigned *p1, unsigned *p0)
{
for (int pg_i = 0; pg_i < REGION1_PG_COUNT / 32; pg_i++) {
uint32_t line = pagetable1->allocmap0[pg_i] | pagetable1->freemap0[pg_i];
if (line) {
for (int j = 0; j < 32; j++) {
if ((line >> j) & 1) {
(*p1)++;
gc_stats_pagetable0(pagetable1->meta0[pg_i * 32 + j], p0);
}
}
for (int pg_i = 0; pg_i < REGION1_PG_COUNT; pg_i++) {
pagetable0_t *pagetable0 = pagetable1->meta0[pg_i];
if (pagetable0 == NULL) {
continue;
}
(*p1)++;
gc_stats_pagetable0(pagetable0, p0);
}
}

static void gc_stats_pagetable(unsigned *p2, unsigned *p1, unsigned *p0)
{
for (int pg_i = 0; pg_i < (REGION2_PG_COUNT + 31) / 32; pg_i++) {
uint32_t line = memory_map.allocmap1[pg_i] | memory_map.freemap1[pg_i];
if (line) {
for (int j = 0; j < 32; j++) {
if ((line >> j) & 1) {
(*p2)++;
gc_stats_pagetable1(memory_map.meta1[pg_i * 32 + j], p1, p0);
}
}
for (int pg_i = 0; pg_i < REGION2_PG_COUNT; pg_i++) {
pagetable1_t *pagetable1 = alloc_map.meta1[pg_i];
if (pagetable1 == NULL) {
continue;
}
(*p2)++;
gc_stats_pagetable1(pagetable1, p1, p0);
}
}

Expand All @@ -833,7 +791,7 @@ void jl_print_gc_stats(JL_STREAM *s)
#ifdef _OS_LINUX_
malloc_stats();
#endif
double ptime = jl_clock_now() - process_t0;
double ptime = jl_hrtime() - process_t0;
jl_safe_printf("exec time\t%.5f sec\n", ptime);
if (gc_num.pause > 0) {
jl_safe_printf("gc time \t%.5f sec (%2.1f%%) in %d (%d full) collections\n",
Expand Down Expand Up @@ -1054,7 +1012,7 @@ void jl_gc_debug_init(void)
#endif

#ifdef GC_FINAL_STATS
process_t0 = jl_clock_now();
process_t0 = jl_hrtime();
#endif
}

Expand Down Expand Up @@ -1176,7 +1134,7 @@ void gc_stats_big_obj(void)
static int64_t poolobj_sizes[4];
static int64_t empty_pages;

static void gc_count_pool_page(jl_gc_pagemeta_t *pg)
static void gc_count_pool_page(jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT
{
int osize = pg->osize;
char *data = pg->data;
Expand All @@ -1195,44 +1153,16 @@ static void gc_count_pool_page(jl_gc_pagemeta_t *pg)
}
}

static void gc_count_pool_pagetable0(pagetable0_t *pagetable0)
{
for (int pg_i = 0; pg_i < REGION0_PG_COUNT / 32; pg_i++) {
uint32_t line = pagetable0->allocmap[pg_i];
if (line) {
for (int j = 0; j < 32; j++) {
if ((line >> j) & 1) {
gc_count_pool_page(pagetable0->meta[pg_i * 32 + j]);
}
}
}
}
}

static void gc_count_pool_pagetable1(pagetable1_t *pagetable1)
{
for (int pg_i = 0; pg_i < REGION1_PG_COUNT / 32; pg_i++) {
uint32_t line = pagetable1->allocmap0[pg_i];
if (line) {
for (int j = 0; j < 32; j++) {
if ((line >> j) & 1) {
gc_count_pool_pagetable0(pagetable1->meta0[pg_i * 32 + j]);
}
}
}
}
}

static void gc_count_pool_pagetable(void)
{
for (int pg_i = 0; pg_i < (REGION2_PG_COUNT + 31) / 32; pg_i++) {
uint32_t line = memory_map.allocmap1[pg_i];
if (line) {
for (int j = 0; j < 32; j++) {
if ((line >> j) & 1) {
gc_count_pool_pagetable1(memory_map.meta1[pg_i * 32 + j]);
}
for (int i = 0; i < gc_n_threads; i++) {
jl_ptls_t ptls2 = gc_all_tls_states[i];
jl_gc_pagemeta_t *pg = ptls2->page_metadata_allocd;
while (pg != NULL) {
if (gc_alloc_map_is_set(pg->data)) {
gc_count_pool_page(pg);
}
pg = pg->next;
}
}
}
Expand Down

0 comments on commit 87a861d

Please sign in to comment.