-
Notifications
You must be signed in to change notification settings - Fork 4.6k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Decommit region tails #66008
Decommit region tails #66008
Changes from all commits
4261aa9
e072850
b1c8ba4
2cbe5b4
2ff0279
242db94
ad6ec7e
6731496
4a3a8da
eccc68c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -11431,6 +11431,7 @@ void gc_heap::init_heap_segment (heap_segment* seg, gc_heap* hp | |
heap_segment_plan_allocated (seg) = heap_segment_mem (seg); | ||
heap_segment_allocated (seg) = heap_segment_mem (seg); | ||
heap_segment_saved_allocated (seg) = heap_segment_mem (seg); | ||
heap_segment_decommit_target (seg) = heap_segment_reserved (seg); | ||
#ifdef BACKGROUND_GC | ||
heap_segment_background_allocated (seg) = 0; | ||
heap_segment_saved_bg_allocated (seg) = 0; | ||
|
@@ -11546,6 +11547,7 @@ void gc_heap::decommit_heap_segment_pages (heap_segment* seg, | |
{ | ||
if (use_large_pages_p) | ||
return; | ||
|
||
uint8_t* page_start = align_on_page (heap_segment_allocated(seg)); | ||
assert (heap_segment_committed (seg) >= page_start); | ||
|
||
|
@@ -11561,12 +11563,6 @@ void gc_heap::decommit_heap_segment_pages (heap_segment* seg, | |
size_t gc_heap::decommit_heap_segment_pages_worker (heap_segment* seg, | ||
uint8_t* new_committed) | ||
{ | ||
#ifdef USE_REGIONS | ||
if (!dt_high_memory_load_p()) | ||
{ | ||
return 0; | ||
} | ||
#endif | ||
assert (!use_large_pages_p); | ||
uint8_t* page_start = align_on_page (new_committed); | ||
ptrdiff_t size = heap_segment_committed (seg) - page_start; | ||
|
@@ -12351,8 +12347,7 @@ void gc_heap::distribute_free_regions() | |
heap_budget_in_region_units[i][large_free_region] = 0; | ||
for (int gen = soh_gen0; gen < total_generation_count; gen++) | ||
{ | ||
ptrdiff_t budget_gen = hp->estimate_gen_growth (gen); | ||
assert (budget_gen >= 0); | ||
ptrdiff_t budget_gen = max (hp->estimate_gen_growth (gen), 0); | ||
int kind = gen >= loh_generation; | ||
size_t budget_gen_in_region_units = (budget_gen + (region_size[kind] - 1)) / region_size[kind]; | ||
dprintf (REGIONS_LOG, ("h%2d gen %d has an estimated growth of %Id bytes (%Id regions)", i, gen, budget_gen, budget_gen_in_region_units)); | ||
|
@@ -12499,7 +12494,6 @@ void gc_heap::distribute_free_regions() | |
} | ||
|
||
#ifdef MULTIPLE_HEAPS | ||
gradual_decommit_in_progress_p = FALSE; | ||
for (int kind = basic_free_region; kind < count_free_region_kinds; kind++) | ||
{ | ||
if (global_regions_to_decommit[kind].get_num_free_regions() != 0) | ||
|
@@ -22142,7 +22136,7 @@ void gc_heap::garbage_collect (int n) | |
} | ||
|
||
descr_generations ("BEGIN"); | ||
#ifdef TRACE_GC | ||
#if defined(TRACE_GC) && defined(USE_REGIONS) | ||
if (heap_number == 0) | ||
{ | ||
#ifdef MULTIPLE_HEAPS | ||
|
@@ -22166,7 +22160,7 @@ void gc_heap::garbage_collect (int n) | |
} | ||
} | ||
} | ||
#endif // TRACE_GC | ||
#endif // TRACE_GC && USE_REGIONS | ||
|
||
#ifdef VERIFY_HEAP | ||
if ((GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC) && | ||
|
@@ -30221,7 +30215,7 @@ heap_segment* gc_heap::find_first_valid_region (heap_segment* region, bool compa | |
set_region_plan_gen_num (current_region, plan_gen_num); | ||
} | ||
|
||
if (gen_num != 0) | ||
if (gen_num >= soh_gen2) | ||
{ | ||
dprintf (REGIONS_LOG, (" gen%d decommit end of region %Ix(%Ix)", | ||
gen_num, current_region, heap_segment_mem (current_region))); | ||
|
@@ -39562,7 +39556,7 @@ ptrdiff_t gc_heap::estimate_gen_growth (int gen_number) | |
gen_number, heap_number, budget_gen, new_allocation_gen, free_list_space_gen)); | ||
#endif //USE_REGIONS | ||
|
||
return max(0, budget_gen); | ||
return budget_gen; | ||
} | ||
|
||
void gc_heap::decommit_ephemeral_segment_pages() | ||
|
@@ -39573,14 +39567,71 @@ void gc_heap::decommit_ephemeral_segment_pages() | |
} | ||
|
||
#if defined(MULTIPLE_HEAPS) && defined(USE_REGIONS) | ||
// for regions, this is done at the regions level | ||
return; | ||
for (int gen_number = soh_gen0; gen_number <= soh_gen1; gen_number++) | ||
{ | ||
generation *gen = generation_of (gen_number); | ||
heap_segment* tail_region = generation_tail_region (gen); | ||
uint8_t* previous_decommit_target = heap_segment_decommit_target (tail_region); | ||
|
||
// reset the decommit targets to make sure we don't decommit inadvertently | ||
for (heap_segment* region = generation_start_segment_rw (gen); region != nullptr; region = heap_segment_next (region)) | ||
{ | ||
heap_segment_decommit_target (region) = heap_segment_reserved (region); | ||
} | ||
|
||
ptrdiff_t budget_gen = estimate_gen_growth (gen_number) + loh_size_threshold; | ||
|
||
if (budget_gen >= 0) | ||
{ | ||
// we need more than the regions we have - nothing to decommit | ||
continue; | ||
} | ||
|
||
// we may have too much committed - let's see if we can decommit in the tail region | ||
ptrdiff_t tail_region_size = heap_segment_reserved (tail_region) - heap_segment_mem (tail_region); | ||
ptrdiff_t unneeded_tail_size = min (-budget_gen, tail_region_size); | ||
uint8_t *decommit_target = heap_segment_reserved (tail_region) - unneeded_tail_size; | ||
decommit_target = max (decommit_target, heap_segment_allocated (tail_region)); | ||
|
||
if (decommit_target < previous_decommit_target) | ||
{ | ||
// we used to have a higher target - do exponential smoothing by computing | ||
// essentially decommit_target = 1/3*decommit_target + 2/3*previous_decommit_target | ||
// computation below is slightly different to avoid overflow | ||
ptrdiff_t target_decrease = previous_decommit_target - decommit_target; | ||
decommit_target += target_decrease * 2 / 3; | ||
} | ||
|
||
//#define STRESS_DECOMMIT 1 | ||
#ifdef STRESS_DECOMMIT | ||
// our decommit logic should work for a random decommit target within tail_region - make sure it does | ||
decommit_target = heap_segment_mem (tail_region) + gc_rand::get_rand (heap_segment_reserved (tail_region) - heap_segment_mem (tail_region)); | ||
#endif //STRESS_DECOMMIT | ||
|
||
heap_segment_decommit_target (tail_region) = decommit_target; | ||
|
||
if (decommit_target < heap_segment_committed (tail_region)) | ||
{ | ||
gradual_decommit_in_progress_p = TRUE; | ||
|
||
dprintf (1, ("h%2d gen %d reduce_commit by %IdkB", | ||
heap_number, | ||
gen_number, | ||
(heap_segment_committed (tail_region) - decommit_target)/1024)); | ||
} | ||
dprintf(3, ("h%2d gen %d allocated: %IdkB committed: %IdkB target: %IdkB", | ||
heap_number, | ||
gen_number, | ||
(heap_segment_allocated (tail_region) - heap_segment_mem (tail_region))/1024, | ||
(heap_segment_committed (tail_region) - heap_segment_mem (tail_region))/1024, | ||
(decommit_target - heap_segment_mem (tail_region))/1024)); | ||
} | ||
#else //MULTIPLE_HEAPS && USE_REGIONS | ||
|
||
dynamic_data* dd0 = dynamic_data_of (0); | ||
|
||
ptrdiff_t desired_allocation = dd_new_allocation (dd0) + | ||
estimate_gen_growth (soh_gen1) + | ||
max (estimate_gen_growth (soh_gen1), 0) + | ||
loh_size_threshold; | ||
|
||
size_t slack_space = | ||
|
@@ -39687,7 +39738,11 @@ bool gc_heap::decommit_step () | |
} | ||
} | ||
} | ||
#else //USE_REGIONS | ||
if (use_large_pages_p) | ||
{ | ||
return (decommit_size != 0); | ||
} | ||
#endif //USE_REGIONS | ||
#ifdef MULTIPLE_HEAPS | ||
// should never get here for large pages because decommit_ephemeral_segment_pages | ||
// will not do anything if use_large_pages_p is true | ||
|
@@ -39699,46 +39754,93 @@ bool gc_heap::decommit_step () | |
decommit_size += hp->decommit_ephemeral_segment_pages_step (); | ||
} | ||
#endif //MULTIPLE_HEAPS | ||
#endif //USE_REGIONS | ||
return (decommit_size != 0); | ||
} | ||
|
||
#ifdef MULTIPLE_HEAPS | ||
// return the decommitted size | ||
#ifndef USE_REGIONS | ||
size_t gc_heap::decommit_ephemeral_segment_pages_step () | ||
{ | ||
// we rely on desired allocation not being changed outside of GC | ||
assert (ephemeral_heap_segment->saved_desired_allocation == dd_desired_allocation (dynamic_data_of (0))); | ||
|
||
uint8_t* decommit_target = heap_segment_decommit_target (ephemeral_heap_segment); | ||
size_t EXTRA_SPACE = 2 * OS_PAGE_SIZE; | ||
decommit_target += EXTRA_SPACE; | ||
uint8_t* committed = heap_segment_committed (ephemeral_heap_segment); | ||
if (decommit_target < committed) | ||
size_t size = 0; | ||
#ifdef USE_REGIONS | ||
for (int gen_number = soh_gen0; gen_number <= soh_gen1; gen_number++) | ||
{ | ||
generation* gen = generation_of (gen_number); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I would not take the msl if There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ok, I skipped taking the msl lock for Regarding tying up GC thread 0, hopefully we won't be tying it up for very long, because we limit the amount of decommit to max_decommit_step_size. |
||
heap_segment* seg = generation_tail_region (gen); | ||
#else // USE_REGIONS | ||
{ | ||
// we rely on other threads not messing with committed if we are about to trim it down | ||
assert (ephemeral_heap_segment->saved_committed == heap_segment_committed (ephemeral_heap_segment)); | ||
heap_segment* seg = ephemeral_heap_segment; | ||
// we rely on desired allocation not being changed outside of GC | ||
assert (seg->saved_desired_allocation == dd_desired_allocation (dynamic_data_of (0))); | ||
#endif // USE_REGIONS | ||
|
||
// how much would we need to decommit to get to decommit_target in one step? | ||
size_t full_decommit_size = (committed - decommit_target); | ||
uint8_t* decommit_target = heap_segment_decommit_target (seg); | ||
size_t EXTRA_SPACE = 2 * OS_PAGE_SIZE; | ||
decommit_target += EXTRA_SPACE; | ||
#ifdef STRESS_DECOMMIT | ||
// our decommit logic should work for a random decommit target within tail_region - make sure it does | ||
// tail region now may be different from what decommit_ephemeral_segment_pages saw | ||
decommit_target = heap_segment_mem (seg) + gc_rand::get_rand (heap_segment_reserved (seg) - heap_segment_mem (seg)); | ||
#endif //STRESS_DECOMMIT | ||
uint8_t* committed = heap_segment_committed (seg); | ||
uint8_t* allocated = (seg == ephemeral_heap_segment) ? alloc_allocated : heap_segment_allocated (seg); | ||
if ((allocated <= decommit_target) && (decommit_target < committed)) | ||
{ | ||
#ifdef USE_REGIONS | ||
if (gen_number == soh_gen0) | ||
{ | ||
// for gen 0, sync with the allocator by taking the more space lock | ||
// and re-read the variables | ||
// | ||
// we call try_enter_spin_lock here instead of enter_spin_lock because | ||
// calling enter_spin_lock from this thread can deadlock at the start | ||
// of a GC - if gc_started is already true, we call wait_for_gc_done(), | ||
// but we are on GC thread 0, so GC cannot make progress | ||
if (!try_enter_spin_lock (&more_space_lock_soh)) | ||
{ | ||
continue; | ||
} | ||
add_saved_spinlock_info (false, me_acquire, mt_decommit_step); | ||
seg = generation_tail_region (gen); | ||
#ifndef STRESS_DECOMMIT | ||
decommit_target = heap_segment_decommit_target (seg); | ||
decommit_target += EXTRA_SPACE; | ||
#endif | ||
committed = heap_segment_committed (seg); | ||
allocated = (seg == ephemeral_heap_segment) ? alloc_allocated : heap_segment_allocated (seg); | ||
} | ||
if ((allocated <= decommit_target) && (decommit_target < committed)) | ||
#else // USE_REGIONS | ||
// we rely on other threads not messing with committed if we are about to trim it down | ||
assert (seg->saved_committed == heap_segment_committed (seg)); | ||
#endif // USE_REGIONS | ||
{ | ||
// how much would we need to decommit to get to decommit_target in one step? | ||
size_t full_decommit_size = (committed - decommit_target); | ||
|
||
// don't do more than max_decommit_step_size per step | ||
size_t decommit_size = min (max_decommit_step_size, full_decommit_size); | ||
// don't do more than max_decommit_step_size per step | ||
size_t decommit_size = min (max_decommit_step_size, full_decommit_size); | ||
|
||
// figure out where the new committed should be | ||
uint8_t* new_committed = (committed - decommit_size); | ||
size_t size = decommit_heap_segment_pages_worker (ephemeral_heap_segment, new_committed); | ||
// figure out where the new committed should be | ||
uint8_t* new_committed = (committed - decommit_size); | ||
size += decommit_heap_segment_pages_worker (seg, new_committed); | ||
|
||
#ifdef _DEBUG | ||
ephemeral_heap_segment->saved_committed = committed - size; | ||
seg->saved_committed = committed - size; | ||
#endif // _DEBUG | ||
|
||
return size; | ||
} | ||
#ifdef USE_REGIONS | ||
if (gen_number == soh_gen0) | ||
{ | ||
// for gen 0, we took the more space lock - leave it again | ||
add_saved_spinlock_info (false, me_release, mt_decommit_step); | ||
leave_spin_lock (&more_space_lock_soh); | ||
} | ||
#endif // USE_REGIONS | ||
} | ||
} | ||
return 0; | ||
return size; | ||
} | ||
#endif //!USE_REGIONS | ||
#endif //MULTIPLE_HEAPS | ||
|
||
//This is meant to be called by decide_on_compacting. | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
if we are not even doing a gen1 GC, would we need to goto
soh_gen1
here?if we are doing a gen1 GC we would have called
decommit_heap_segment_pages
and decommitted (almost) everything afterheap_segment_allocated
for gen1 regions -if
dt_high_memory_load_p()
is true. so you could avoid doing the gen1 region in that case. however, I'm also fine if you want to avoid complicating this code path here.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
now we are not decommitting end of region space for gen2 since it's only doing the decommitting when memory load is high, this could still make a difference for small benchmarks (as in, regions would use more memory)
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
There are two reasons for considering the situation in
soh_gen1
even if we are not doing a gen 1 GC:We need to consider what to do with gen 2 (and UOH as well), agreed. This should be a separate PR, I think.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
actually I think for now we should keep the behavior for
decommit_heap_segment_pages
for now, for gen2/UOH - there's no reason not to because that's only done during a full blocking GC. and it's the same behavior for segments. we can make a separate PR to improve this behavior.