diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp index 93f9b18ad9f5a..51805b205ddbc 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp @@ -88,9 +88,10 @@ void ShenandoahGlobalHeuristics::choose_global_collection_set(ShenandoahCollecti size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0; log_info(gc, ergo)("Adaptive CSet Selection for GLOBAL. Max Young Evacuation: %zu" - "%s, Max Old Evacuation: %zu%s, Actual Free: %zu%s.", + "%s, Max Old Evacuation: %zu%s, Max Either Evacuation: %zu%s, Actual Free: %zu%s.", byte_size_in_proper_unit(max_young_cset), proper_unit_for_byte_size(max_young_cset), byte_size_in_proper_unit(max_old_cset), proper_unit_for_byte_size(max_old_cset), + byte_size_in_proper_unit(unaffiliated_young_memory), proper_unit_for_byte_size(unaffiliated_young_memory), byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free)); for (size_t idx = 0; idx < size; idx++) { @@ -133,9 +134,8 @@ void ShenandoahGlobalHeuristics::choose_global_collection_set(ShenandoahCollecti cset->add_region(r); } } - if (regions_transferred_to_old > 0) { - heap->generation_sizer()->force_transfer_to_old(regions_transferred_to_old); + assert(young_evac_reserve > regions_transferred_to_old * region_size_bytes, "young reserve cannot be negative"); heap->young_generation()->set_evacuation_reserve(young_evac_reserve - regions_transferred_to_old * region_size_bytes); heap->old_generation()->set_evacuation_reserve(old_evac_reserve + regions_transferred_to_old * region_size_bytes); } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index e963bcc35bb47..1e4d40cfa46b8 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -606,12 +606,12 @@ void ShenandoahOldHeuristics::set_trigger_if_old_is_fragmented(size_t first_old_ } void ShenandoahOldHeuristics::set_trigger_if_old_is_overgrown() { - size_t old_used = _old_generation->used() + _old_generation->get_humongous_waste(); + // used() includes humongous waste + size_t old_used = _old_generation->used(); size_t trigger_threshold = _old_generation->usage_trigger_threshold(); // Detects unsigned arithmetic underflow assert(old_used <= _heap->capacity(), - "Old used (%zu, %zu) must not be more than heap capacity (%zu)", - _old_generation->used(), _old_generation->get_humongous_waste(), _heap->capacity()); + "Old used (%zu) must not be more than heap capacity (%zu)", _old_generation->used(), _heap->capacity()); if (old_used > trigger_threshold) { _growth_trigger = true; } @@ -683,7 +683,8 @@ bool ShenandoahOldHeuristics::should_start_gc() { if (_growth_trigger) { // Growth may be falsely triggered during mixed evacuations, before the mixed-evacuation candidates have been // evacuated. Before acting on a false trigger, we check to confirm the trigger condition is still satisfied. - const size_t current_usage = _old_generation->used() + _old_generation->get_humongous_waste(); + // _old_generation->used() includes humongous waste. + const size_t current_usage = _old_generation->used(); const size_t trigger_threshold = _old_generation->usage_trigger_threshold(); const size_t heap_size = heap->capacity(); const size_t ignore_threshold = (ShenandoahIgnoreOldGrowthBelowPercentage * heap_size) / 100; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 69827d72f4453..8dccf5a057cfc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -56,7 +56,8 @@ class ShenandoahLeftRightIterator { ShenandoahRegionPartitions* _partitions; ShenandoahFreeSetPartitionId _partition; public: - explicit ShenandoahLeftRightIterator(ShenandoahRegionPartitions* partitions, ShenandoahFreeSetPartitionId partition, bool use_empty = false) + explicit ShenandoahLeftRightIterator(ShenandoahRegionPartitions* partitions, + ShenandoahFreeSetPartitionId partition, bool use_empty = false) : _idx(0), _end(0), _partitions(partitions), _partition(partition) { _idx = use_empty ? _partitions->leftmost_empty(_partition) : _partitions->leftmost(_partition); _end = use_empty ? _partitions->rightmost_empty(_partition) : _partitions->rightmost(_partition); @@ -87,7 +88,8 @@ class ShenandoahRightLeftIterator { ShenandoahRegionPartitions* _partitions; ShenandoahFreeSetPartitionId _partition; public: - explicit ShenandoahRightLeftIterator(ShenandoahRegionPartitions* partitions, ShenandoahFreeSetPartitionId partition, bool use_empty = false) + explicit ShenandoahRightLeftIterator(ShenandoahRegionPartitions* partitions, + ShenandoahFreeSetPartitionId partition, bool use_empty = false) : _idx(0), _end(0), _partitions(partitions), _partition(partition) { _idx = use_empty ? _partitions->rightmost_empty(_partition) : _partitions->rightmost(_partition); _end = use_empty ? _partitions->leftmost_empty(_partition) : _partitions->leftmost(_partition); @@ -166,9 +168,61 @@ ShenandoahRegionPartitions::ShenandoahRegionPartitions(size_t max_regions, Shena _free_set(free_set), _membership{ ShenandoahSimpleBitMap(max_regions), ShenandoahSimpleBitMap(max_regions) , ShenandoahSimpleBitMap(max_regions) } { + initialize_old_collector(); make_all_regions_unavailable(); } +void ShenandoahFreeSet::account_for_pip_regions(size_t mutator_regions, size_t mutator_bytes, + size_t collector_regions, size_t collector_bytes) { + shenandoah_assert_heaplocked(); + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + + // We have removed all of these regions from their respective partition. Each pip region is "in" the NotFree partition. + // We want to account for all pip pad memory as if it had been consumed from within the Mutator partition. + // + // After we finish promote in place, the pad memory will be deallocated and made available within the OldCollector + // region. At that time, we will transfer the used memory from the Mutator partition to the OldCollector parttion, + // and then we will unallocate the pad memory. + + + _partitions.decrease_region_counts(ShenandoahFreeSetPartitionId::Mutator, mutator_regions); + _partitions.decrease_region_counts(ShenandoahFreeSetPartitionId::Collector, collector_regions); + + // Increase used by remnant fill objects placed in both Mutator and Collector partitions + _partitions.increase_used(ShenandoahFreeSetPartitionId::Mutator, mutator_bytes); + _partitions.increase_used(ShenandoahFreeSetPartitionId::Collector, collector_bytes); + + // Now transfer all of the memory contained within Collector pip regions from the Collector to the Mutator. + // Each of these regions is treated as fully used, even though some of the region's memory may be artifically used, + // to be recycled and put into allocatable OldCollector partition after the region has been promoted in place. + _partitions.transfer_used_capacity_from_to(ShenandoahFreeSetPartitionId::Collector, ShenandoahFreeSetPartitionId::Mutator, + collector_regions); + + // Conservatively, act as if we've promoted from both Mutator and Collector partitions + recompute_total_affiliated(); + recompute_total_young_used(); + recompute_total_global_used(); +} + +ShenandoahFreeSetPartitionId ShenandoahFreeSet::prepare_to_promote_in_place(size_t idx, size_t bytes) { + shenandoah_assert_heaplocked(); + size_t min_remnant_size = PLAB::min_size() * HeapWordSize; + ShenandoahFreeSetPartitionId p = _partitions.membership(idx); + if (bytes >= min_remnant_size) { + assert((p == ShenandoahFreeSetPartitionId::Mutator) || (p == ShenandoahFreeSetPartitionId::Collector), + "PIP region must be associated with young"); + _partitions.raw_clear_membership(idx, p); + } else { + assert(p == ShenandoahFreeSetPartitionId::NotFree, "We did not fill this region and do not need to adjust used"); + } + return p; +} + inline bool ShenandoahFreeSet::can_allocate_from(ShenandoahHeapRegion *r) const { return r->is_empty() || (r->is_trash() && !_heap->is_concurrent_weak_root_in_progress()); } @@ -196,6 +250,54 @@ inline bool ShenandoahFreeSet::has_alloc_capacity(ShenandoahHeapRegion *r) const return alloc_capacity(r) > 0; } +// This is used for unit testing. Do not use in production code. +void ShenandoahFreeSet::resize_old_collector_capacity(size_t regions) { + shenandoah_assert_heaplocked(); + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + size_t original_old_regions = _partitions.get_capacity(ShenandoahFreeSetPartitionId::OldCollector) / region_size_bytes; + size_t unaffiliated_mutator_regions = _partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::Mutator); + size_t unaffiliated_collector_regions = _partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::Collector); + size_t unaffiliated_old_collector_regions = _partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::OldCollector); + if (regions > original_old_regions) { + size_t regions_to_transfer = regions - original_old_regions; + if (regions_to_transfer <= unaffiliated_mutator_regions + unaffiliated_collector_regions) { + size_t regions_from_mutator = + (regions_to_transfer > unaffiliated_mutator_regions)? unaffiliated_mutator_regions: regions_to_transfer; + regions_to_transfer -= regions_from_mutator; + size_t regions_from_collector = regions_to_transfer; + if (regions_from_mutator > 0) { + transfer_empty_regions_from_to(ShenandoahFreeSetPartitionId::Mutator, ShenandoahFreeSetPartitionId::OldCollector, + regions_from_mutator); + } + if (regions_from_collector > 0) { + transfer_empty_regions_from_to(ShenandoahFreeSetPartitionId::Collector, ShenandoahFreeSetPartitionId::OldCollector, + regions_from_mutator); + } + } else { + fatal("Could not resize old for unit test"); + } + } else if (regions < original_old_regions) { + size_t regions_to_transfer = original_old_regions - regions; + if (regions_to_transfer <= unaffiliated_old_collector_regions) { + transfer_empty_regions_from_to(ShenandoahFreeSetPartitionId::OldCollector, ShenandoahFreeSetPartitionId::Mutator, + regions_to_transfer); + } else { + fatal("Could not resize old for unit test"); + } + } + // else, old generation is already appropriately sized +} + +void ShenandoahFreeSet::reset_bytes_allocated_since_gc_start(size_t initial_bytes_allocated) { + shenandoah_assert_heaplocked(); + _mutator_bytes_allocated_since_gc_start = initial_bytes_allocated; +} + +void ShenandoahFreeSet::increase_bytes_allocated(size_t bytes) { + shenandoah_assert_heaplocked(); + _mutator_bytes_allocated_since_gc_start += bytes; +} + inline idx_t ShenandoahRegionPartitions::leftmost(ShenandoahFreeSetPartitionId which_partition) const { assert (which_partition < NumPartitions, "selected free partition must be valid"); idx_t idx = _leftmosts[int(which_partition)]; @@ -218,6 +320,12 @@ inline idx_t ShenandoahRegionPartitions::rightmost(ShenandoahFreeSetPartitionId return idx; } +void ShenandoahRegionPartitions::initialize_old_collector() { + _capacity[int(ShenandoahFreeSetPartitionId::OldCollector)] = 0; + _region_counts[int(ShenandoahFreeSetPartitionId::OldCollector)] = 0; + _empty_region_counts[int(ShenandoahFreeSetPartitionId::OldCollector)] = 0; +} + void ShenandoahRegionPartitions::make_all_regions_unavailable() { shenandoah_assert_heaplocked(); for (size_t partition_id = 0; partition_id < IntNumPartitions; partition_id++) { @@ -227,15 +335,19 @@ void ShenandoahRegionPartitions::make_all_regions_unavailable() { _leftmosts_empty[partition_id] = _max; _rightmosts_empty[partition_id] = -1;; _capacity[partition_id] = 0; + _region_counts[partition_id] = 0; + _empty_region_counts[partition_id] = 0; _used[partition_id] = 0; + _humongous_waste[partition_id] = 0; _available[partition_id] = FreeSetUnderConstruction; } - _region_counts[int(ShenandoahFreeSetPartitionId::Mutator)] = _region_counts[int(ShenandoahFreeSetPartitionId::Collector)] = 0; } void ShenandoahRegionPartitions::establish_mutator_intervals(idx_t mutator_leftmost, idx_t mutator_rightmost, idx_t mutator_leftmost_empty, idx_t mutator_rightmost_empty, - size_t mutator_region_count, size_t mutator_used) { + size_t total_mutator_regions, size_t empty_mutator_regions, + size_t mutator_region_count, size_t mutator_used, + size_t mutator_humongous_waste_bytes) { shenandoah_assert_heaplocked(); _leftmosts[int(ShenandoahFreeSetPartitionId::Mutator)] = mutator_leftmost; @@ -245,10 +357,13 @@ void ShenandoahRegionPartitions::establish_mutator_intervals(idx_t mutator_leftm _region_counts[int(ShenandoahFreeSetPartitionId::Mutator)] = mutator_region_count; _used[int(ShenandoahFreeSetPartitionId::Mutator)] = mutator_used; - _capacity[int(ShenandoahFreeSetPartitionId::Mutator)] = mutator_region_count * _region_size_bytes; + _capacity[int(ShenandoahFreeSetPartitionId::Mutator)] = total_mutator_regions * _region_size_bytes; + _humongous_waste[int(ShenandoahFreeSetPartitionId::Mutator)] = mutator_humongous_waste_bytes; _available[int(ShenandoahFreeSetPartitionId::Mutator)] = _capacity[int(ShenandoahFreeSetPartitionId::Mutator)] - _used[int(ShenandoahFreeSetPartitionId::Mutator)]; + _empty_region_counts[int(ShenandoahFreeSetPartitionId::Mutator)] = empty_mutator_regions; + _leftmosts[int(ShenandoahFreeSetPartitionId::Collector)] = _max; _rightmosts[int(ShenandoahFreeSetPartitionId::Collector)] = -1; _leftmosts_empty[int(ShenandoahFreeSetPartitionId::Collector)] = _max; @@ -257,13 +372,20 @@ void ShenandoahRegionPartitions::establish_mutator_intervals(idx_t mutator_leftm _region_counts[int(ShenandoahFreeSetPartitionId::Collector)] = 0; _used[int(ShenandoahFreeSetPartitionId::Collector)] = 0; _capacity[int(ShenandoahFreeSetPartitionId::Collector)] = 0; + _humongous_waste[int(ShenandoahFreeSetPartitionId::Collector)] = 0; _available[int(ShenandoahFreeSetPartitionId::Collector)] = 0; + + _empty_region_counts[int(ShenandoahFreeSetPartitionId::Collector)] = 0; } -void ShenandoahRegionPartitions::establish_old_collector_intervals(idx_t old_collector_leftmost, idx_t old_collector_rightmost, +void ShenandoahRegionPartitions::establish_old_collector_intervals(idx_t old_collector_leftmost, + idx_t old_collector_rightmost, idx_t old_collector_leftmost_empty, idx_t old_collector_rightmost_empty, - size_t old_collector_region_count, size_t old_collector_used) { + size_t total_old_collector_region_count, + size_t old_collector_empty, size_t old_collector_regions, + size_t old_collector_used, + size_t old_collector_humongous_waste_bytes) { shenandoah_assert_heaplocked(); _leftmosts[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_leftmost; @@ -271,11 +393,14 @@ void ShenandoahRegionPartitions::establish_old_collector_intervals(idx_t old_col _leftmosts_empty[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_leftmost_empty; _rightmosts_empty[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_rightmost_empty; - _region_counts[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_region_count; + _region_counts[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_regions; _used[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_used; - _capacity[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_region_count * _region_size_bytes; + _capacity[int(ShenandoahFreeSetPartitionId::OldCollector)] = total_old_collector_region_count * _region_size_bytes; + _humongous_waste[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_humongous_waste_bytes; _available[int(ShenandoahFreeSetPartitionId::OldCollector)] = _capacity[int(ShenandoahFreeSetPartitionId::OldCollector)] - _used[int(ShenandoahFreeSetPartitionId::OldCollector)]; + + _empty_region_counts[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_empty; } void ShenandoahRegionPartitions::increase_used(ShenandoahFreeSetPartitionId which_partition, size_t bytes) { @@ -289,15 +414,125 @@ void ShenandoahRegionPartitions::increase_used(ShenandoahFreeSetPartitionId whic _used[int(which_partition)], _capacity[int(which_partition)], bytes); } -inline void ShenandoahRegionPartitions::shrink_interval_if_range_modifies_either_boundary( - ShenandoahFreeSetPartitionId partition, idx_t low_idx, idx_t high_idx) { +void ShenandoahRegionPartitions::decrease_used(ShenandoahFreeSetPartitionId which_partition, size_t bytes) { + shenandoah_assert_heaplocked(); + assert (which_partition < NumPartitions, "Partition must be valid"); + assert (_used[int(which_partition)] >= bytes, "Must not use less than zero after decrease"); + _used[int(which_partition)] -= bytes; + _available[int(which_partition)] += bytes; +} + +void ShenandoahRegionPartitions::increase_humongous_waste(ShenandoahFreeSetPartitionId which_partition, size_t bytes) { + shenandoah_assert_heaplocked(); + assert (which_partition < NumPartitions, "Partition must be valid"); + _humongous_waste[int(which_partition)] += bytes; +} + +size_t ShenandoahRegionPartitions::get_humongous_waste(ShenandoahFreeSetPartitionId which_partition) { + assert (which_partition < NumPartitions, "Partition must be valid"); + return _humongous_waste[int(which_partition)];; +} + +void ShenandoahRegionPartitions::set_capacity_of(ShenandoahFreeSetPartitionId which_partition, size_t value) { + shenandoah_assert_heaplocked(); + assert (which_partition < NumPartitions, "selected free set must be valid"); + _capacity[int(which_partition)] = value; + _available[int(which_partition)] = value - _used[int(which_partition)]; +} + + +void ShenandoahRegionPartitions::increase_capacity(ShenandoahFreeSetPartitionId which_partition, size_t bytes) { + shenandoah_assert_heaplocked(); + assert (which_partition < NumPartitions, "Partition must be valid"); + _capacity[int(which_partition)] += bytes; + _available[int(which_partition)] += bytes; +} + +void ShenandoahRegionPartitions::transfer_used_capacity_from_to(ShenandoahFreeSetPartitionId from_partition, + ShenandoahFreeSetPartitionId to_partition, size_t regions) { + shenandoah_assert_heaplocked(); + size_t bytes = regions * ShenandoahHeapRegion::region_size_bytes(); + assert (from_partition < NumPartitions, "Partition must be valid"); + assert (to_partition < NumPartitions, "Partition must be valid"); + assert(_capacity[int(from_partition)] >= bytes, "Cannot remove more capacity bytes than are present"); + assert(_used[int(from_partition)] >= bytes, "Cannot transfer used bytes that are not used"); + + // available is unaffected by transfer + _capacity[int(from_partition)] -= bytes; + _used[int(from_partition)] -= bytes; + _capacity[int(to_partition)] += bytes; + _used[int(to_partition)] += bytes; +} + +void ShenandoahRegionPartitions::decrease_capacity(ShenandoahFreeSetPartitionId which_partition, size_t bytes) { + shenandoah_assert_heaplocked(); + assert (which_partition < NumPartitions, "Partition must be valid"); + assert(_capacity[int(which_partition)] >= bytes, "Cannot remove more capacity bytes than are present"); + assert(_available[int(which_partition)] >= bytes, "Cannot shrink capacity unless capacity is unused"); + _capacity[int(which_partition)] -= bytes; + _available[int(which_partition)] -= bytes; +} + +void ShenandoahRegionPartitions::increase_available(ShenandoahFreeSetPartitionId which_partition, size_t bytes) { + shenandoah_assert_heaplocked(); + assert (which_partition < NumPartitions, "Partition must be valid"); + _available[int(which_partition)] += bytes; +} + +void ShenandoahRegionPartitions::decrease_available(ShenandoahFreeSetPartitionId which_partition, size_t bytes) { + shenandoah_assert_heaplocked(); + assert (which_partition < NumPartitions, "Partition must be valid"); + assert(_available[int(which_partition)] >= bytes, "Cannot remove more available bytes than are present"); + _available[int(which_partition)] -= bytes; +} + +size_t ShenandoahRegionPartitions::get_available(ShenandoahFreeSetPartitionId which_partition) { + assert (which_partition < NumPartitions, "Partition must be valid"); + return _available[int(which_partition)];; +} + +void ShenandoahRegionPartitions::increase_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions) { + _region_counts[int(which_partition)] += regions; +} + +void ShenandoahRegionPartitions::decrease_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions) { + assert(_region_counts[int(which_partition)] >= regions, "Cannot remove more regions than are present"); + _region_counts[int(which_partition)] -= regions; +} + +void ShenandoahRegionPartitions::increase_empty_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions) { + _empty_region_counts[int(which_partition)] += regions; +} + +void ShenandoahRegionPartitions::decrease_empty_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions) { + assert(_empty_region_counts[int(which_partition)] >= regions, "Cannot remove more regions than are present"); + _empty_region_counts[int(which_partition)] -= regions; +} + +void ShenandoahRegionPartitions::one_region_is_no_longer_empty(ShenandoahFreeSetPartitionId partition) { + decrease_empty_region_counts(partition, (size_t) 1); +} + +// All members of partition between low_idx and high_idx inclusive have been removed. +void ShenandoahRegionPartitions::shrink_interval_if_range_modifies_either_boundary( + ShenandoahFreeSetPartitionId partition, idx_t low_idx, idx_t high_idx, size_t num_regions) { assert((low_idx <= high_idx) && (low_idx >= 0) && (high_idx < _max), "Range must span legal index values"); + size_t span = high_idx + 1 - low_idx; + bool regions_are_contiguous = (span == num_regions); if (low_idx == leftmost(partition)) { assert (!_membership[int(partition)].is_set(low_idx), "Do not shrink interval if region not removed"); if (high_idx + 1 == _max) { - _leftmosts[int(partition)] = _max; + if (regions_are_contiguous) { + _leftmosts[int(partition)] = _max; + } else { + _leftmosts[int(partition)] = find_index_of_next_available_region(partition, low_idx + 1); + } } else { - _leftmosts[int(partition)] = find_index_of_next_available_region(partition, high_idx + 1); + if (regions_are_contiguous) { + _leftmosts[int(partition)] = find_index_of_next_available_region(partition, high_idx + 1); + } else { + _leftmosts[int(partition)] = find_index_of_next_available_region(partition, low_idx + 1); + } } if (_leftmosts_empty[int(partition)] < _leftmosts[int(partition)]) { // This gets us closer to where we need to be; we'll scan further when leftmosts_empty is requested. @@ -307,9 +542,17 @@ inline void ShenandoahRegionPartitions::shrink_interval_if_range_modifies_either if (high_idx == _rightmosts[int(partition)]) { assert (!_membership[int(partition)].is_set(high_idx), "Do not shrink interval if region not removed"); if (low_idx == 0) { - _rightmosts[int(partition)] = -1; + if (regions_are_contiguous) { + _rightmosts[int(partition)] = -1; + } else { + _rightmosts[int(partition)] = find_index_of_previous_available_region(partition, high_idx - 1); + } } else { - _rightmosts[int(partition)] = find_index_of_previous_available_region(partition, low_idx - 1); + if (regions_are_contiguous) { + _rightmosts[int(partition)] = find_index_of_previous_available_region(partition, low_idx - 1); + } else { + _rightmosts[int(partition)] = find_index_of_previous_available_region(partition, high_idx - 1); + } } if (_rightmosts_empty[int(partition)] > _rightmosts[int(partition)]) { // This gets us closer to where we need to be; we'll scan further when rightmosts_empty is requested. @@ -324,12 +567,55 @@ inline void ShenandoahRegionPartitions::shrink_interval_if_range_modifies_either } } -inline void ShenandoahRegionPartitions::shrink_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, idx_t idx) { - shrink_interval_if_range_modifies_either_boundary(partition, idx, idx); +void ShenandoahRegionPartitions::establish_interval(ShenandoahFreeSetPartitionId partition, idx_t low_idx, + idx_t high_idx, idx_t low_empty_idx, idx_t high_empty_idx) { +#ifdef ASSERT + assert (partition < NumPartitions, "invalid partition"); + if (low_idx != max()) { + assert((low_idx <= high_idx) && (low_idx >= 0) && (high_idx < _max), "Range must span legal index values"); + assert (in_free_set(partition, low_idx), "Must be in partition of established interval"); + assert (in_free_set(partition, high_idx), "Must be in partition of established interval"); + } + if (low_empty_idx != max()) { + ShenandoahHeapRegion* r = ShenandoahHeap::heap()->get_region(low_empty_idx); + assert (in_free_set(partition, low_empty_idx) && (r->is_trash() || r->free() == _region_size_bytes), + "Must be empty and in partition of established interval"); + r = ShenandoahHeap::heap()->get_region(high_empty_idx); + assert (in_free_set(partition, high_empty_idx), "Must be in partition of established interval"); + } +#endif + + _leftmosts[int(partition)] = low_idx; + _rightmosts[int(partition)] = high_idx; + _leftmosts_empty[int(partition)] = low_empty_idx; + _rightmosts_empty[int(partition)] = high_empty_idx; +} + +inline void ShenandoahRegionPartitions::shrink_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, + idx_t idx) { + shrink_interval_if_range_modifies_either_boundary(partition, idx, idx, 1); +} + +// Some members of partition between low_idx and high_idx inclusive have been added. +void ShenandoahRegionPartitions:: +expand_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId partition, idx_t low_idx, idx_t high_idx, + idx_t low_empty_idx, idx_t high_empty_idx) { + if (_leftmosts[int(partition)] > low_idx) { + _leftmosts[int(partition)] = low_idx; + } + if (_rightmosts[int(partition)] < high_idx) { + _rightmosts[int(partition)] = high_idx; + } + if (_leftmosts_empty[int(partition)] > low_empty_idx) { + _leftmosts_empty[int(partition)] = low_empty_idx; + } + if (_rightmosts_empty[int(partition)] < high_empty_idx) { + _rightmosts_empty[int(partition)] = high_empty_idx; + } } -inline void ShenandoahRegionPartitions::expand_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, - idx_t idx, size_t region_available) { +void ShenandoahRegionPartitions::expand_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, + idx_t idx, size_t region_available) { if (_leftmosts[int(partition)] > idx) { _leftmosts[int(partition)] = idx; } @@ -355,15 +641,23 @@ void ShenandoahRegionPartitions::retire_range_from_partition( assert (partition < NumPartitions, "Cannot remove from free partitions if not already free"); for (idx_t idx = low_idx; idx <= high_idx; idx++) { +#ifdef ASSERT + ShenandoahHeapRegion* r = ShenandoahHeap::heap()->get_region(idx); assert (in_free_set(partition, idx), "Must be in partition to remove from partition"); + assert(r->is_empty() || r->is_trash(), "Region must be empty or trash"); +#endif _membership[int(partition)].clear_bit(idx); } - _region_counts[int(partition)] -= high_idx + 1 - low_idx; - shrink_interval_if_range_modifies_either_boundary(partition, low_idx, high_idx); + size_t num_regions = high_idx + 1 - low_idx; + decrease_region_counts(partition, num_regions); + decrease_empty_region_counts(partition, num_regions); + shrink_interval_if_range_modifies_either_boundary(partition, low_idx, high_idx, num_regions); } -void ShenandoahRegionPartitions::retire_from_partition(ShenandoahFreeSetPartitionId partition, idx_t idx, size_t used_bytes) { +size_t ShenandoahRegionPartitions::retire_from_partition(ShenandoahFreeSetPartitionId partition, + idx_t idx, size_t used_bytes) { + size_t waste_bytes = 0; // Note: we may remove from free partition even if region is not entirely full, such as when available < PLAB::min_size() assert (idx < _max, "index is sane: %zu < %zu", idx, _max); assert (partition < NumPartitions, "Cannot remove from free partitions if not already free"); @@ -371,13 +665,28 @@ void ShenandoahRegionPartitions::retire_from_partition(ShenandoahFreeSetPartitio if (used_bytes < _region_size_bytes) { // Count the alignment pad remnant of memory as used when we retire this region - increase_used(partition, _region_size_bytes - used_bytes); + size_t fill_padding = _region_size_bytes - used_bytes; + waste_bytes = fill_padding; + increase_used(partition, fill_padding); } _membership[int(partition)].clear_bit(idx); + decrease_region_counts(partition, 1); shrink_interval_if_boundary_modified(partition, idx); - _region_counts[int(partition)]--; + + // This region is fully used, whether or not top() equals end(). It + // is retired and no more memory will be allocated from within it. + + return waste_bytes; +} + +void ShenandoahRegionPartitions::unretire_to_partition(ShenandoahHeapRegion* r, ShenandoahFreeSetPartitionId which_partition) { + shenandoah_assert_heaplocked(); + make_free(r->index(), which_partition, r->free()); } + +// The caller is responsible for increasing capacity and available and used in which_partition, and decreasing the +// same quantities for the original partition void ShenandoahRegionPartitions::make_free(idx_t idx, ShenandoahFreeSetPartitionId which_partition, size_t available) { shenandoah_assert_heaplocked(); assert (idx < _max, "index is sane: %zu < %zu", idx, _max); @@ -386,11 +695,7 @@ void ShenandoahRegionPartitions::make_free(idx_t idx, ShenandoahFreeSetPartition assert (available <= _region_size_bytes, "Available cannot exceed region size"); _membership[int(which_partition)].set_bit(idx); - _capacity[int(which_partition)] += _region_size_bytes; - _used[int(which_partition)] += _region_size_bytes - available; - _available[int(which_partition)] += available; expand_interval_if_boundary_modified(which_partition, idx, available); - _region_counts[int(which_partition)]++; } bool ShenandoahRegionPartitions::is_mutator_partition(ShenandoahFreeSetPartitionId p) { @@ -409,9 +714,10 @@ bool ShenandoahRegionPartitions::available_implies_empty(size_t available_in_reg return (available_in_region == _region_size_bytes); } - -void ShenandoahRegionPartitions::move_from_partition_to_partition(idx_t idx, ShenandoahFreeSetPartitionId orig_partition, - ShenandoahFreeSetPartitionId new_partition, size_t available) { +// Do not adjust capacities, available, or used. Return used delta. +size_t ShenandoahRegionPartitions:: +move_from_partition_to_partition_with_deferred_accounting(idx_t idx, ShenandoahFreeSetPartitionId orig_partition, + ShenandoahFreeSetPartitionId new_partition, size_t available) { ShenandoahHeapRegion* r = ShenandoahHeap::heap()->get_region(idx); shenandoah_assert_heaplocked(); assert (idx < _max, "index is sane: %zu < %zu", idx, _max); @@ -449,37 +755,37 @@ void ShenandoahRegionPartitions::move_from_partition_to_partition(idx_t idx, She _membership[int(orig_partition)].clear_bit(idx); _membership[int(new_partition)].set_bit(idx); + return used; +} + +void ShenandoahRegionPartitions::move_from_partition_to_partition(idx_t idx, ShenandoahFreeSetPartitionId orig_partition, + ShenandoahFreeSetPartitionId new_partition, size_t available) { + size_t used = move_from_partition_to_partition_with_deferred_accounting(idx, orig_partition, new_partition, available); + // We decreased used, which increases available, but then we decrease available by full region size below + decrease_used(orig_partition, used); + _region_counts[int(orig_partition)]--; _capacity[int(orig_partition)] -= _region_size_bytes; - _used[int(orig_partition)] -= used; - _available[int(orig_partition)] -= available; + _available[int(orig_partition)] -= _region_size_bytes; shrink_interval_if_boundary_modified(orig_partition, idx); - _capacity[int(new_partition)] += _region_size_bytes;; - _used[int(new_partition)] += used; - _available[int(new_partition)] += available; + _capacity[int(new_partition)] += _region_size_bytes; + _available[int(new_partition)] += _region_size_bytes; + _region_counts[int(new_partition)]++; + // We increased availableby full region size above, but decrease it by used within this region now. + increase_used(new_partition, used); expand_interval_if_boundary_modified(new_partition, idx, available); - _region_counts[int(orig_partition)]--; - _region_counts[int(new_partition)]++; + if (available == _region_size_bytes) { + _empty_region_counts[int(orig_partition)]--; + _empty_region_counts[int(new_partition)]++; + } } const char* ShenandoahRegionPartitions::partition_membership_name(idx_t idx) const { return partition_name(membership(idx)); } -inline ShenandoahFreeSetPartitionId ShenandoahRegionPartitions::membership(idx_t idx) const { - assert (idx < _max, "index is sane: %zu < %zu", idx, _max); - ShenandoahFreeSetPartitionId result = ShenandoahFreeSetPartitionId::NotFree; - for (uint partition_id = 0; partition_id < UIntNumPartitions; partition_id++) { - if (_membership[partition_id].is_set(idx)) { - assert(result == ShenandoahFreeSetPartitionId::NotFree, "Region should reside in only one partition"); - result = (ShenandoahFreeSetPartitionId) partition_id; - } - } - return result; -} - #ifdef ASSERT inline bool ShenandoahRegionPartitions::partition_id_matches(idx_t idx, ShenandoahFreeSetPartitionId test_partition) const { assert (idx < _max, "index is sane: %zu < %zu", idx, _max); @@ -532,7 +838,8 @@ inline idx_t ShenandoahRegionPartitions::find_index_of_next_available_cluster_of idx_t rightmost_idx = rightmost(which_partition); idx_t leftmost_idx = leftmost(which_partition); if ((rightmost_idx < leftmost_idx) || (start_index > rightmost_idx)) return _max; - idx_t result = _membership[int(which_partition)].find_first_consecutive_set_bits(start_index, rightmost_idx + 1, cluster_size); + idx_t result = + _membership[int(which_partition)].find_first_consecutive_set_bits(start_index, rightmost_idx + 1, cluster_size); if (result > rightmost_idx) { result = _max; } @@ -594,7 +901,19 @@ idx_t ShenandoahRegionPartitions::rightmost_empty(ShenandoahFreeSetPartitionId w #ifdef ASSERT -void ShenandoahRegionPartitions::assert_bounds() { +void ShenandoahRegionPartitions::assert_bounds(bool validate_totals) { + + size_t capacities[UIntNumPartitions]; + size_t used[UIntNumPartitions]; + size_t regions[UIntNumPartitions]; + size_t humongous_waste[UIntNumPartitions]; + + // We don't know whether young retired regions belonged to Mutator or Collector before they were retired. + // We just tally the total, and divide it to make matches work if possible. + size_t young_retired_regions = 0; + size_t young_retired_used = 0; + size_t young_retired_capacity = 0; + size_t young_humongous_waste = 0; idx_t leftmosts[UIntNumPartitions]; idx_t rightmosts[UIntNumPartitions]; @@ -606,21 +925,64 @@ void ShenandoahRegionPartitions::assert_bounds() { empty_leftmosts[i] = _max; rightmosts[i] = -1; empty_rightmosts[i] = -1; + capacities[i] = 0; + used[i] = 0; + regions[i] = 0; + humongous_waste[i] = 0; } for (idx_t i = 0; i < _max; i++) { ShenandoahFreeSetPartitionId partition = membership(i); + size_t capacity = _free_set->alloc_capacity(i); switch (partition) { case ShenandoahFreeSetPartitionId::NotFree: - break; + { + assert(!validate_totals || (capacity != _region_size_bytes), "Should not be retired if empty"); + ShenandoahHeapRegion* r = ShenandoahHeap::heap()->get_region(i); + if (r->is_humongous()) { + if (r->is_old()) { + regions[int(ShenandoahFreeSetPartitionId::OldCollector)]++; + used[int(ShenandoahFreeSetPartitionId::OldCollector)] += _region_size_bytes; + capacities[int(ShenandoahFreeSetPartitionId::OldCollector)] += _region_size_bytes; + humongous_waste[int(ShenandoahFreeSetPartitionId::OldCollector)] += capacity; + } else { + assert(r->is_young(), "Must be young if not old"); + young_retired_regions++; + // Count entire region as used even if there is some waste. + young_retired_used += _region_size_bytes; + young_retired_capacity += _region_size_bytes; + young_humongous_waste += capacity; + } + } else { + assert(r->is_cset() || (capacity < PLAB::min_size() * HeapWordSize), + "Expect retired remnant size to be smaller than min plab size"); + // This region has been retired already or it is in the cset. In either case, we set capacity to zero + // so that the entire region will be counted as used. We count young cset regions as "retired". + capacity = 0; + if (r->is_old()) { + regions[int(ShenandoahFreeSetPartitionId::OldCollector)]++; + used[int(ShenandoahFreeSetPartitionId::OldCollector)] += _region_size_bytes - capacity; + capacities[int(ShenandoahFreeSetPartitionId::OldCollector)] += _region_size_bytes; + } else { + assert(r->is_young(), "Must be young if not old"); + young_retired_regions++; + young_retired_used += _region_size_bytes - capacity; + young_retired_capacity += _region_size_bytes; + } + } + } + break; case ShenandoahFreeSetPartitionId::Mutator: case ShenandoahFreeSetPartitionId::Collector: case ShenandoahFreeSetPartitionId::OldCollector: { - size_t capacity = _free_set->alloc_capacity(i); - bool is_empty = (capacity == _region_size_bytes); assert(capacity > 0, "free regions must have allocation capacity"); + bool is_empty = (capacity == _region_size_bytes); + regions[int(partition)]++; + used[int(partition)] += _region_size_bytes - capacity; + capacities[int(partition)] += _region_size_bytes; + if (i < leftmosts[int(partition)]) { leftmosts[int(partition)] = i; } @@ -659,19 +1021,19 @@ void ShenandoahRegionPartitions::assert_bounds() { idx_t beg_off = leftmosts[int(ShenandoahFreeSetPartitionId::Mutator)]; idx_t end_off = rightmosts[int(ShenandoahFreeSetPartitionId::Mutator)]; assert (beg_off >= leftmost(ShenandoahFreeSetPartitionId::Mutator), - "free regions before the leftmost: %zd, bound %zd", + "Mutator free regions before the leftmost: %zd, bound %zd", beg_off, leftmost(ShenandoahFreeSetPartitionId::Mutator)); assert (end_off <= rightmost(ShenandoahFreeSetPartitionId::Mutator), - "free regions past the rightmost: %zd, bound %zd", + "Mutator free regions past the rightmost: %zd, bound %zd", end_off, rightmost(ShenandoahFreeSetPartitionId::Mutator)); beg_off = empty_leftmosts[int(ShenandoahFreeSetPartitionId::Mutator)]; end_off = empty_rightmosts[int(ShenandoahFreeSetPartitionId::Mutator)]; assert (beg_off >= leftmost_empty(ShenandoahFreeSetPartitionId::Mutator), - "free empty regions before the leftmost: %zd, bound %zd", + "Mutator free empty regions before the leftmost: %zd, bound %zd", beg_off, leftmost_empty(ShenandoahFreeSetPartitionId::Mutator)); assert (end_off <= rightmost_empty(ShenandoahFreeSetPartitionId::Mutator), - "free empty regions past the rightmost: %zd, bound %zd", + "Mutator free empty regions past the rightmost: %zd, bound %zd", end_off, rightmost_empty(ShenandoahFreeSetPartitionId::Mutator)); // Performance invariants. Failing these would not break the free partition, but performance would suffer. @@ -682,87 +1044,225 @@ void ShenandoahRegionPartitions::assert_bounds() { assert (leftmost(ShenandoahFreeSetPartitionId::Collector) == _max || partition_id_matches(leftmost(ShenandoahFreeSetPartitionId::Collector), ShenandoahFreeSetPartitionId::Collector), - "leftmost region should be free: %zd", leftmost(ShenandoahFreeSetPartitionId::Collector)); + "Collector leftmost region should be free: %zd", leftmost(ShenandoahFreeSetPartitionId::Collector)); assert (leftmost(ShenandoahFreeSetPartitionId::Collector) == _max || partition_id_matches(rightmost(ShenandoahFreeSetPartitionId::Collector), ShenandoahFreeSetPartitionId::Collector), - "rightmost region should be free: %zd", rightmost(ShenandoahFreeSetPartitionId::Collector)); + "Collector rightmost region should be free: %zd", rightmost(ShenandoahFreeSetPartitionId::Collector)); // If Collector partition is empty, leftmosts will both equal max, rightmosts will both equal zero. // Likewise for empty region partitions. beg_off = leftmosts[int(ShenandoahFreeSetPartitionId::Collector)]; end_off = rightmosts[int(ShenandoahFreeSetPartitionId::Collector)]; assert (beg_off >= leftmost(ShenandoahFreeSetPartitionId::Collector), - "free regions before the leftmost: %zd, bound %zd", + "Collector free regions before the leftmost: %zd, bound %zd", beg_off, leftmost(ShenandoahFreeSetPartitionId::Collector)); assert (end_off <= rightmost(ShenandoahFreeSetPartitionId::Collector), - "free regions past the rightmost: %zd, bound %zd", + "Collector free regions past the rightmost: %zd, bound %zd", end_off, rightmost(ShenandoahFreeSetPartitionId::Collector)); beg_off = empty_leftmosts[int(ShenandoahFreeSetPartitionId::Collector)]; end_off = empty_rightmosts[int(ShenandoahFreeSetPartitionId::Collector)]; assert (beg_off >= _leftmosts_empty[int(ShenandoahFreeSetPartitionId::Collector)], - "free empty regions before the leftmost: %zd, bound %zd", + "Collector free empty regions before the leftmost: %zd, bound %zd", beg_off, leftmost_empty(ShenandoahFreeSetPartitionId::Collector)); assert (end_off <= _rightmosts_empty[int(ShenandoahFreeSetPartitionId::Collector)], - "free empty regions past the rightmost: %zd, bound %zd", + "Collector free empty regions past the rightmost: %zd, bound %zd", end_off, rightmost_empty(ShenandoahFreeSetPartitionId::Collector)); // Performance invariants. Failing these would not break the free partition, but performance would suffer. - assert (leftmost(ShenandoahFreeSetPartitionId::OldCollector) <= _max, "leftmost in bounds: %zd < %zd", + assert (leftmost(ShenandoahFreeSetPartitionId::OldCollector) <= _max, "OldCollector leftmost in bounds: %zd < %zd", leftmost(ShenandoahFreeSetPartitionId::OldCollector), _max); - assert (rightmost(ShenandoahFreeSetPartitionId::OldCollector) < _max, "rightmost in bounds: %zd < %zd", + assert (rightmost(ShenandoahFreeSetPartitionId::OldCollector) < _max, "OldCollector rightmost in bounds: %zd < %zd", rightmost(ShenandoahFreeSetPartitionId::OldCollector), _max); assert (leftmost(ShenandoahFreeSetPartitionId::OldCollector) == _max || partition_id_matches(leftmost(ShenandoahFreeSetPartitionId::OldCollector), ShenandoahFreeSetPartitionId::OldCollector), - "leftmost region should be free: %zd", leftmost(ShenandoahFreeSetPartitionId::OldCollector)); + "OldCollector leftmost region should be free: %zd", leftmost(ShenandoahFreeSetPartitionId::OldCollector)); assert (leftmost(ShenandoahFreeSetPartitionId::OldCollector) == _max || partition_id_matches(rightmost(ShenandoahFreeSetPartitionId::OldCollector), ShenandoahFreeSetPartitionId::OldCollector), - "rightmost region should be free: %zd", rightmost(ShenandoahFreeSetPartitionId::OldCollector)); + "OldCollector rightmost region should be free: %zd", rightmost(ShenandoahFreeSetPartitionId::OldCollector)); // If OldCollector partition is empty, leftmosts will both equal max, rightmosts will both equal zero. // Likewise for empty region partitions. beg_off = leftmosts[int(ShenandoahFreeSetPartitionId::OldCollector)]; end_off = rightmosts[int(ShenandoahFreeSetPartitionId::OldCollector)]; assert (beg_off >= leftmost(ShenandoahFreeSetPartitionId::OldCollector), - "free regions before the leftmost: %zd, bound %zd", + "OldCollector free regions before the leftmost: %zd, bound %zd", beg_off, leftmost(ShenandoahFreeSetPartitionId::OldCollector)); assert (end_off <= rightmost(ShenandoahFreeSetPartitionId::OldCollector), - "free regions past the rightmost: %zd, bound %zd", + "OldCollector free regions past the rightmost: %zd, bound %zd", end_off, rightmost(ShenandoahFreeSetPartitionId::OldCollector)); beg_off = empty_leftmosts[int(ShenandoahFreeSetPartitionId::OldCollector)]; end_off = empty_rightmosts[int(ShenandoahFreeSetPartitionId::OldCollector)]; assert (beg_off >= _leftmosts_empty[int(ShenandoahFreeSetPartitionId::OldCollector)], - "free empty regions before the leftmost: %zd, bound %zd", + "OldCollector free empty regions before the leftmost: %zd, bound %zd", beg_off, leftmost_empty(ShenandoahFreeSetPartitionId::OldCollector)); assert (end_off <= _rightmosts_empty[int(ShenandoahFreeSetPartitionId::OldCollector)], - "free empty regions past the rightmost: %zd, bound %zd", + "OldCollector free empty regions past the rightmost: %zd, bound %zd", end_off, rightmost_empty(ShenandoahFreeSetPartitionId::OldCollector)); + + if (validate_totals) { + // young_retired_regions need to be added to either Mutator or Collector partitions, 100% used. + // Give enough of young_retired_regions, young_retired_capacity, young_retired_user + // to the Mutator partition to top it off so that it matches the running totals. + // + // Give any remnants to the Collector partition. After topping off the Collector partition, its values + // should also match running totals. + + assert(young_retired_regions * _region_size_bytes == young_retired_capacity, "sanity"); + assert(young_retired_capacity == young_retired_used, "sanity"); + + + assert(capacities[int(ShenandoahFreeSetPartitionId::OldCollector)] + == _capacity[int(ShenandoahFreeSetPartitionId::OldCollector)], "Old collector capacities must match"); + assert(used[int(ShenandoahFreeSetPartitionId::OldCollector)] + == _used[int(ShenandoahFreeSetPartitionId::OldCollector)], "Old collector used must match"); + assert(regions[int(ShenandoahFreeSetPartitionId::OldCollector)] + == _capacity[int(ShenandoahFreeSetPartitionId::OldCollector)] / _region_size_bytes, "Old collector regions must match"); + assert(_capacity[int(ShenandoahFreeSetPartitionId::OldCollector)] + >= _used[int(ShenandoahFreeSetPartitionId::OldCollector)], "Old Collector capacity must be >= used"); + assert(_available[int(ShenandoahFreeSetPartitionId::OldCollector)] == + (_capacity[int(ShenandoahFreeSetPartitionId::OldCollector)] - _used[int(ShenandoahFreeSetPartitionId::OldCollector)]), + "Old Collector available must equal capacity minus used"); + assert(_humongous_waste[int(ShenandoahFreeSetPartitionId::OldCollector)] == + humongous_waste[int(ShenandoahFreeSetPartitionId::OldCollector)], "Old Collector humongous waste must match"); + + assert(_capacity[int(ShenandoahFreeSetPartitionId::Mutator)] >= capacities[int(ShenandoahFreeSetPartitionId::Mutator)], + "Capacity total must be >= counted tally"); + size_t mutator_capacity_shortfall = + _capacity[int(ShenandoahFreeSetPartitionId::Mutator)] - capacities[int(ShenandoahFreeSetPartitionId::Mutator)]; + assert(mutator_capacity_shortfall <= young_retired_capacity, "sanity"); + capacities[int(ShenandoahFreeSetPartitionId::Mutator)] += mutator_capacity_shortfall; + young_retired_capacity -= mutator_capacity_shortfall; + capacities[int(ShenandoahFreeSetPartitionId::Collector)] += young_retired_capacity; + + + assert(_used[int(ShenandoahFreeSetPartitionId::Mutator)] >= used[int(ShenandoahFreeSetPartitionId::Mutator)], + "Used total must be >= counted tally"); + size_t mutator_used_shortfall = + _used[int(ShenandoahFreeSetPartitionId::Mutator)] - used[int(ShenandoahFreeSetPartitionId::Mutator)]; + assert(mutator_used_shortfall <= young_retired_used, "sanity"); + used[int(ShenandoahFreeSetPartitionId::Mutator)] += mutator_used_shortfall; + young_retired_used -= mutator_used_shortfall; + used[int(ShenandoahFreeSetPartitionId::Collector)] += young_retired_used; + + assert(_capacity[int(ShenandoahFreeSetPartitionId::Mutator)] / _region_size_bytes + >= regions[int(ShenandoahFreeSetPartitionId::Mutator)], "Region total must be >= counted tally"); + size_t mutator_regions_shortfall = (_capacity[int(ShenandoahFreeSetPartitionId::Mutator)] / _region_size_bytes + - regions[int(ShenandoahFreeSetPartitionId::Mutator)]); + assert(mutator_regions_shortfall <= young_retired_regions, "sanity"); + regions[int(ShenandoahFreeSetPartitionId::Mutator)] += mutator_regions_shortfall; + young_retired_regions -= mutator_regions_shortfall; + regions[int(ShenandoahFreeSetPartitionId::Collector)] += young_retired_regions; + + assert(capacities[int(ShenandoahFreeSetPartitionId::Collector)] == _capacity[int(ShenandoahFreeSetPartitionId::Collector)], + "Collector capacities must match"); + assert(used[int(ShenandoahFreeSetPartitionId::Collector)] == _used[int(ShenandoahFreeSetPartitionId::Collector)], + "Collector used must match"); + assert(regions[int(ShenandoahFreeSetPartitionId::Collector)] + == _capacity[int(ShenandoahFreeSetPartitionId::Collector)] / _region_size_bytes, "Collector regions must match"); + assert(_capacity[int(ShenandoahFreeSetPartitionId::Collector)] >= _used[int(ShenandoahFreeSetPartitionId::Collector)], + "Collector Capacity must be >= used"); + assert(_available[int(ShenandoahFreeSetPartitionId::Collector)] == + (_capacity[int(ShenandoahFreeSetPartitionId::Collector)] - _used[int(ShenandoahFreeSetPartitionId::Collector)]), + "Collector Available must equal capacity minus used"); + + assert(capacities[int(ShenandoahFreeSetPartitionId::Mutator)] == _capacity[int(ShenandoahFreeSetPartitionId::Mutator)], + "Mutator capacities must match"); + assert(used[int(ShenandoahFreeSetPartitionId::Mutator)] == _used[int(ShenandoahFreeSetPartitionId::Mutator)], + "Mutator used must match"); + assert(regions[int(ShenandoahFreeSetPartitionId::Mutator)] + == _capacity[int(ShenandoahFreeSetPartitionId::Mutator)] / _region_size_bytes, "Mutator regions must match"); + assert(_capacity[int(ShenandoahFreeSetPartitionId::Mutator)] >= _used[int(ShenandoahFreeSetPartitionId::Mutator)], + "Mutator capacity must be >= used"); + assert(_available[int(ShenandoahFreeSetPartitionId::Mutator)] == + (_capacity[int(ShenandoahFreeSetPartitionId::Mutator)] - _used[int(ShenandoahFreeSetPartitionId::Mutator)]), + "Mutator available must equal capacity minus used"); + assert(_humongous_waste[int(ShenandoahFreeSetPartitionId::Mutator)] == young_humongous_waste, + "Mutator humongous waste must match"); + } } #endif ShenandoahFreeSet::ShenandoahFreeSet(ShenandoahHeap* heap, size_t max_regions) : _heap(heap), _partitions(max_regions, this), - _alloc_bias_weight(0) + _total_humongous_waste(0), + _alloc_bias_weight(0), + _total_young_used(0), + _total_old_used(0), + _total_global_used(0), + _young_affiliated_regions(0), + _old_affiliated_regions(0), + _global_affiliated_regions(0), + _young_unaffiliated_regions(0), + _global_unaffiliated_regions(0), + _total_young_regions(0), + _total_global_regions(0), + _mutator_bytes_allocated_since_gc_start(0) { clear_internal(); } +// was pip_pad_bytes void ShenandoahFreeSet::add_promoted_in_place_region_to_old_collector(ShenandoahHeapRegion* region) { shenandoah_assert_heaplocked(); size_t plab_min_size_in_bytes = ShenandoahGenerationalHeap::heap()->plab_min_size() * HeapWordSize; - size_t idx = region->index(); - size_t capacity = alloc_capacity(region); - assert(_partitions.membership(idx) == ShenandoahFreeSetPartitionId::NotFree, + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + size_t available_in_region = alloc_capacity(region); + size_t region_index = region->index(); + ShenandoahFreeSetPartitionId p = _partitions.membership(region_index); + assert(_partitions.membership(region_index) == ShenandoahFreeSetPartitionId::NotFree, "Regions promoted in place should have been excluded from Mutator partition"); - if (capacity >= plab_min_size_in_bytes) { - _partitions.make_free(idx, ShenandoahFreeSetPartitionId::OldCollector, capacity); - _heap->old_generation()->augment_promoted_reserve(capacity); - } + + // If region had been retired, its end-of-region alignment pad had been counted as used within the Mutator partition + size_t used_while_awaiting_pip = region_size_bytes; + size_t used_after_pip = region_size_bytes; + if (available_in_region >= plab_min_size_in_bytes) { + used_after_pip -= available_in_region; + } else { + if (available_in_region >= ShenandoahHeap::min_fill_size() * HeapWordSize) { + size_t fill_words = available_in_region / HeapWordSize; + ShenandoahHeap::heap()->old_generation()->card_scan()->register_object(region->top()); + region->allocate_fill(fill_words); + } + available_in_region = 0; + } + + assert(p == ShenandoahFreeSetPartitionId::NotFree, "pip region must be NotFree"); + assert(region->is_young(), "pip region must be young"); + + // Though this region may have been promoted in place from the Collector region, its usage is now accounted within + // the Mutator partition. + _partitions.decrease_used(ShenandoahFreeSetPartitionId::Mutator, used_while_awaiting_pip); + + // decrease capacity adjusts available + _partitions.decrease_capacity(ShenandoahFreeSetPartitionId::Mutator, region_size_bytes); + _partitions.increase_capacity(ShenandoahFreeSetPartitionId::OldCollector, region_size_bytes); + _partitions.increase_used(ShenandoahFreeSetPartitionId::OldCollector, used_after_pip); + region->set_affiliation(ShenandoahAffiliation::OLD_GENERATION); + if (available_in_region > 0) { + assert(available_in_region >= plab_min_size_in_bytes, "enforced above"); + _partitions.increase_region_counts(ShenandoahFreeSetPartitionId::OldCollector, 1); + // make_free() adjusts bounds for OldCollector partition + _partitions.make_free(region_index, ShenandoahFreeSetPartitionId::OldCollector, available_in_region); + _heap->old_generation()->augment_promoted_reserve(available_in_region); + assert(available_in_region != region_size_bytes, "Nothing to promote in place"); + } + // else, leave this region as NotFree + + recompute_total_used(); + // Conservatively, assume that pip regions came from both Mutator and Collector + recompute_total_affiliated(); + _partitions.assert_bounds(true); } HeapWord* ShenandoahFreeSet::allocate_from_partition_with_affiliation(ShenandoahAffiliation affiliation, @@ -1034,7 +1534,6 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah log_debug(gc, free)("Using new region (%zu) for %s (" PTR_FORMAT ").", r->index(), ShenandoahAllocRequest::alloc_type_to_string(req.type()), p2i(&req)); assert(!r->is_affiliated(), "New region %zu should be unaffiliated", r->index()); - r->set_affiliation(req.affiliation()); if (r->is_old()) { // Any OLD region allocated during concurrent coalesce-and-fill does not need to be coalesced and filled because @@ -1046,8 +1545,6 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah r->end_preemptible_coalesce_and_fill(); _heap->old_generation()->clear_cards_for(r); } - _heap->generation_for(r->affiliation())->increment_affiliated_region_count(); - #ifdef ASSERT ShenandoahMarkingContext* const ctx = _heap->marking_context(); assert(ctx->top_at_mark_start(r) == r->bottom(), "Newly established allocation region starts with TAMS equal to bottom"); @@ -1120,6 +1617,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah if (req.is_mutator_alloc()) { assert(req.is_young(), "Mutator allocations always come from young generation."); _partitions.increase_used(ShenandoahFreeSetPartitionId::Mutator, req.actual_size() * HeapWordSize); + increase_bytes_allocated(req.actual_size() * HeapWordSize); } else { assert(req.is_gc_alloc(), "Should be gc_alloc since req wasn't mutator alloc"); @@ -1128,19 +1626,38 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah // PLABs be made parsable at the end of evacuation. This is enabled by retiring all plabs at end of evacuation. r->set_update_watermark(r->top()); if (r->is_old()) { - _partitions.increase_used(ShenandoahFreeSetPartitionId::OldCollector, req.actual_size() * HeapWordSize); + _partitions.increase_used(ShenandoahFreeSetPartitionId::OldCollector, (req.actual_size() + req.waste()) * HeapWordSize); assert(req.type() != ShenandoahAllocRequest::_alloc_gclab, "old-gen allocations use PLAB or shared allocation"); // for plabs, we'll sort the difference between evac and promotion usage when we retire the plab } else { - _partitions.increase_used(ShenandoahFreeSetPartitionId::Collector, req.actual_size() * HeapWordSize); + _partitions.increase_used(ShenandoahFreeSetPartitionId::Collector, (req.actual_size() + req.waste()) * HeapWordSize); } } } - static const size_t min_capacity = (size_t) (ShenandoahHeapRegion::region_size_bytes() * (1.0 - 1.0 / ShenandoahEvacWaste)); size_t ac = alloc_capacity(r); - - if (((result == nullptr) && (ac < min_capacity)) || (alloc_capacity(r) < PLAB::min_size() * HeapWordSize)) { + ShenandoahFreeSetPartitionId orig_partition; + ShenandoahGeneration* request_generation = nullptr; + if (req.is_mutator_alloc()) { + request_generation = _heap->mode()->is_generational()? _heap->young_generation(): _heap->global_generation(); + orig_partition = ShenandoahFreeSetPartitionId::Mutator; + } else if (req.type() == ShenandoahAllocRequest::_alloc_gclab) { + request_generation = _heap->mode()->is_generational()? _heap->young_generation(): _heap->global_generation(); + orig_partition = ShenandoahFreeSetPartitionId::Collector; + } else if (req.type() == ShenandoahAllocRequest::_alloc_plab) { + request_generation = _heap->old_generation(); + orig_partition = ShenandoahFreeSetPartitionId::OldCollector; + } else { + assert(req.type() == ShenandoahAllocRequest::_alloc_shared_gc, "Unexpected allocation type"); + if (req.is_old()) { + request_generation = _heap->old_generation(); + orig_partition = ShenandoahFreeSetPartitionId::OldCollector; + } else { + request_generation = _heap->mode()->is_generational()? _heap->young_generation(): _heap->global_generation(); + orig_partition = ShenandoahFreeSetPartitionId::Collector; + } + } + if (alloc_capacity(r) < PLAB::min_size() * HeapWordSize) { // Regardless of whether this allocation succeeded, if the remaining memory is less than PLAB:min_size(), retire this region. // Note that retire_from_partition() increases used to account for waste. @@ -1148,24 +1665,56 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah // then retire the region so that subsequent searches can find available memory more quickly. size_t idx = r->index(); - ShenandoahFreeSetPartitionId orig_partition; - if (req.is_mutator_alloc()) { - orig_partition = ShenandoahFreeSetPartitionId::Mutator; - } else if (req.type() == ShenandoahAllocRequest::_alloc_gclab) { - orig_partition = ShenandoahFreeSetPartitionId::Collector; - } else if (req.type() == ShenandoahAllocRequest::_alloc_plab) { - orig_partition = ShenandoahFreeSetPartitionId::OldCollector; - } else { - assert(req.type() == ShenandoahAllocRequest::_alloc_shared_gc, "Unexpected allocation type"); - if (req.is_old()) { - orig_partition = ShenandoahFreeSetPartitionId::OldCollector; - } else { - orig_partition = ShenandoahFreeSetPartitionId::Collector; - } + if ((result != nullptr) && in_new_region) { + _partitions.one_region_is_no_longer_empty(orig_partition); + } + size_t waste_bytes = _partitions.retire_from_partition(orig_partition, idx, r->used()); + if (req.is_mutator_alloc() && (waste_bytes > 0)) { + increase_bytes_allocated(waste_bytes); + } + } else if ((result != nullptr) && in_new_region) { + _partitions.one_region_is_no_longer_empty(orig_partition); + } + + switch (orig_partition) { + case ShenandoahFreeSetPartitionId::Mutator: + recompute_total_used(); + if (in_new_region) { + recompute_total_affiliated(); + } + break; + case ShenandoahFreeSetPartitionId::Collector: + recompute_total_used(); + if (in_new_region) { + recompute_total_affiliated(); } - _partitions.retire_from_partition(orig_partition, idx, r->used()); - _partitions.assert_bounds(); + break; + case ShenandoahFreeSetPartitionId::OldCollector: + recompute_total_used(); + if (in_new_region) { + recompute_total_affiliated(); + } + break; + case ShenandoahFreeSetPartitionId::NotFree: + default: + assert(false, "won't happen"); } + _partitions.assert_bounds(true); return result; } @@ -1235,67 +1784,193 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req, bo end++; } - size_t remainder = words_size & ShenandoahHeapRegion::region_size_words_mask(); - // Initialize regions: - for (idx_t i = beg; i <= end; i++) { - ShenandoahHeapRegion* r = _heap->get_region(i); - r->try_recycle_under_lock(); - - assert(i == beg || _heap->get_region(i - 1)->index() + 1 == r->index(), "Should be contiguous"); - assert(r->is_empty(), "Should be empty"); - - r->set_affiliation(req.affiliation()); - - if (is_humongous) { + size_t total_used = 0; + const size_t used_words_in_last_region = words_size & ShenandoahHeapRegion::region_size_words_mask(); + size_t waste_bytes; + // Retire regions from free partition and initialize them. + if (is_humongous) { + // Humongous allocation retires all regions at once: no allocation is possible anymore. + // retire_range_from_partition() will adjust bounds on Mutator free set if appropriate and will recompute affiliated. + _partitions.retire_range_from_partition(ShenandoahFreeSetPartitionId::Mutator, beg, end); + for (idx_t i = beg; i <= end; i++) { + ShenandoahHeapRegion* r = _heap->get_region(i); + assert(i == beg || _heap->get_region(i - 1)->index() + 1 == r->index(), "Should be contiguous"); + r->try_recycle_under_lock(); + assert(r->is_empty(), "Should be empty"); + r->set_affiliation(req.affiliation()); if (i == beg) { r->make_humongous_start(); } else { r->make_humongous_cont(); } - } else { - r->make_regular_allocation(req.affiliation()); - } - - // Trailing region may be non-full, record the remainder there - size_t used_words; - if ((i == end) && (remainder != 0)) { - used_words = remainder; - } else { - used_words = ShenandoahHeapRegion::region_size_words(); + if ((i == end) && (used_words_in_last_region > 0)) { + r->set_top(r->bottom() + used_words_in_last_region); + } else { + // if used_words_in_last_region is zero, then the end region is fully consumed. + r->set_top(r->end()); + } + r->set_update_watermark(r->bottom()); } - r->set_update_watermark(r->bottom()); - r->set_top(r->bottom() + used_words); - } - generation->increase_affiliated_region_count(num); - - size_t total_used = 0; - if (is_humongous) { - // Humongous allocation retires all regions at once: no allocation is possible anymore. - _partitions.retire_range_from_partition(ShenandoahFreeSetPartitionId::Mutator, beg, end); total_used = ShenandoahHeapRegion::region_size_bytes() * num; + waste_bytes = + (used_words_in_last_region == 0)? 0: ShenandoahHeapRegion::region_size_bytes() - used_words_in_last_region * HeapWordSize; } else { // Non-humongous allocation retires only the regions that cannot be used for allocation anymore. + waste_bytes = 0; for (idx_t i = beg; i <= end; i++) { ShenandoahHeapRegion* r = _heap->get_region(i); - if (r->free() < PLAB::min_size() * HeapWordSize) { - _partitions.retire_from_partition(ShenandoahFreeSetPartitionId::Mutator, i, r->used()); + assert(i == beg || _heap->get_region(i - 1)->index() + 1 == r->index(), "Should be contiguous"); + assert(r->is_empty(), "Should be empty"); + r->try_recycle_under_lock(); + r->set_affiliation(req.affiliation()); + r->make_regular_allocation(req.affiliation()); + if ((i == end) && (used_words_in_last_region > 0)) { + r->set_top(r->bottom() + used_words_in_last_region); + } else { + // if used_words_in_last_region is zero, then the end region is fully consumed. + r->set_top(r->end()); } + r->set_update_watermark(r->bottom()); total_used += r->used(); + if (r->free() < PLAB::min_size() * HeapWordSize) { + // retire_from_partition() will adjust bounds on Mutator free set if appropriate and will recompute affiliated. + // It also increases used for the waste bytes, which includes bytes filled at retirement and bytes too small + // to be filled. Only the last iteration may have non-zero waste_bytes. + waste_bytes += _partitions.retire_from_partition(ShenandoahFreeSetPartitionId::Mutator, i, r->used()); + } + } + _partitions.decrease_empty_region_counts(ShenandoahFreeSetPartitionId::Mutator, num); + if (waste_bytes > 0) { + // For humongous allocations, waste_bytes are included in total_used. Since this is not humongous, + // we need to account separately for the waste_bytes. + increase_bytes_allocated(waste_bytes); } } _partitions.increase_used(ShenandoahFreeSetPartitionId::Mutator, total_used); - _partitions.assert_bounds(); - + increase_bytes_allocated(total_used); req.set_actual_size(words_size); - if (remainder != 0 && is_humongous) { - req.set_waste(ShenandoahHeapRegion::region_size_words() - remainder); + // If !is_humongous, the "waste" is made availabe for new allocation + if (waste_bytes > 0) { + req.set_waste(waste_bytes / HeapWordSize); + if (is_humongous) { + _partitions.increase_humongous_waste(ShenandoahFreeSetPartitionId::Mutator, waste_bytes); + _total_humongous_waste += waste_bytes; + } } + + recompute_total_young_used(); + recompute_total_global_used(); + recompute_total_affiliated(); + _partitions.assert_bounds(true); return _heap->get_region(beg)->bottom(); } class ShenandoahRecycleTrashedRegionClosure final : public ShenandoahHeapRegionClosure { +private: + static const ssize_t SentinelUsed = -1; + static const ssize_t SentinelIndex = -1; + static const size_t MaxSavedRegions = 128; + + ShenandoahRegionPartitions* _partitions; + volatile size_t _recycled_region_count; + ssize_t _region_indices[MaxSavedRegions]; + ssize_t _region_used[MaxSavedRegions]; + + void get_lock_and_flush_buffer(size_t region_count, size_t overflow_region_used, size_t overflow_region_index) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahHeapLocker locker(heap->lock()); + size_t recycled_regions = AtomicAccess::load(&_recycled_region_count); + size_t region_tallies[int(ShenandoahRegionPartitions::NumPartitions)]; + size_t used_byte_tallies[int(ShenandoahRegionPartitions::NumPartitions)]; + for (int p = 0; p < int(ShenandoahRegionPartitions::NumPartitions); p++) { + region_tallies[p] = 0; + used_byte_tallies[p] = 0; + } + ShenandoahFreeSetPartitionId p = _partitions->membership(overflow_region_index); + used_byte_tallies[int(p)] += overflow_region_used; + if (region_count <= recycled_regions) { + // _recycled_region_count has not been decremented after I incremented it to obtain region_count, so I will + // try to flush the buffer. + + // Multiple worker threads may attempt to flush this buffer. The first thread to acquire the lock does the work. + // _recycled_region_count is only decreased while holding the heap lock. + if (region_count > recycled_regions) { + region_count = recycled_regions; + } + for (size_t i = 0; i < region_count; i++) { + ssize_t used; + // wait for other threads to finish updating their entries within the region buffer before processing entry + do { + used = _region_used[i]; + } while (used == SentinelUsed); + ssize_t index; + do { + index = _region_indices[i]; + } while (index == SentinelIndex); + + ShenandoahFreeSetPartitionId p = _partitions->membership(index); + assert(p != ShenandoahFreeSetPartitionId::NotFree, "Trashed regions should be in a free partition"); + used_byte_tallies[int(p)] += used; + region_tallies[int(p)]++; + } + if (region_count > 0) { + for (size_t i = 0; i < MaxSavedRegions; i++) { + _region_indices[i] = SentinelIndex; + _region_used[i] = SentinelUsed; + } + } + + // The almost last thing we do before releasing the lock is to set the _recycled_region_count to 0. What happens next? + // + // 1. Any worker thread that attempted to buffer a new region while we were flushing the buffer will have seen + // that _recycled_region_count > MaxSavedRegions. All such worker threads will first wait for the lock, then + // discover that the _recycled_region_count is zero, then, while holding the lock, they will process the + // region so it doesn't have to be placed into the buffer. This handles the large majority of cases. + // + // 2. However, there's a race that can happen, which will result in someewhat different behavior. Suppose + // this thread resets _recycled_region_count to 0. Then some other worker thread increments _recycled_region_count + // in order to stores its region into the buffer and suppose this happens before all of the other worker threads + // which are waiting to acquire the heap lock have finished their efforts to flush the buffer. If this happens, + // then the workers who are waiting to acquire the heap lock and flush the buffer will find that _recycled_region_count + // has decreased from the value it held when they last tried to increment its value. In this case, these worker + // threads will process their overflow region while holding the lock, but they will not attempt to process regions + // newly placed into the buffer. Otherwise, confusion could result. + // + // Assumption: all worker threads who are attempting to acquire lock and flush buffer will finish their efforts before + // the buffer once again overflows. + // How could we avoid depending on this assumption? + // 1. Let MaxSavedRegions be as large as number of regions, or at least as large as the collection set. + // 2. Keep a count of how many times the buffer has been flushed per instantation of the + // ShenandoahRecycleTrashedRegionClosure object, and only consult/update this value while holding the heap lock. + // Need to think about how this helps resolve the race. + _recycled_region_count = 0; + } else { + // Some other thread has already processed the buffer, resetting _recycled_region_count to zero. Its current value + // may be greater than zero because other workers may have accumulated entries into the buffer. But it is "extremely" + // unlikely that it will overflow again before all waiting workers have had a chance to clear their state. While I've + // got the heap lock, I'll go ahead and update the global state for my overflow region. I'll let other heap regions + // accumulate in the buffer to be processed when the buffer is once again full. + region_count = 0; + } + for (size_t p = 0; p < int(ShenandoahRegionPartitions::NumPartitions); p++) { + _partitions->decrease_used(ShenandoahFreeSetPartitionId(p), used_byte_tallies[p]); + } + } + public: - ShenandoahRecycleTrashedRegionClosure(): ShenandoahHeapRegionClosure() {} + ShenandoahRecycleTrashedRegionClosure(ShenandoahRegionPartitions* p): ShenandoahHeapRegionClosure() { + _partitions = p; + _recycled_region_count = 0; + for (size_t i = 0; i < MaxSavedRegions; i++) { + _region_indices[i] = SentinelIndex; + _region_used[i] = SentinelUsed; + } + } void heap_region_do(ShenandoahHeapRegion* r) { r->try_recycle(); @@ -1313,10 +1988,35 @@ void ShenandoahFreeSet::recycle_trash() { ShenandoahHeap* heap = ShenandoahHeap::heap(); heap->assert_gc_workers(heap->workers()->active_workers()); - ShenandoahRecycleTrashedRegionClosure closure; + ShenandoahRecycleTrashedRegionClosure closure(&_partitions); heap->parallel_heap_region_iterate(&closure); } +bool ShenandoahFreeSet::transfer_one_region_from_mutator_to_old_collector(size_t idx, size_t alloc_capacity) { + ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap(); + ShenandoahYoungGeneration* young_gen = gen_heap->young_generation(); + ShenandoahOldGeneration* old_gen = gen_heap->old_generation(); + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + assert(alloc_capacity == region_size_bytes, "Region must be empty"); + if (young_unaffiliated_regions() > 0) { + _partitions.move_from_partition_to_partition(idx, ShenandoahFreeSetPartitionId::Mutator, + ShenandoahFreeSetPartitionId::OldCollector, alloc_capacity); + gen_heap->old_generation()->augment_evacuation_reserve(alloc_capacity); + recompute_total_used(); + // Transferred region is unaffilliated, empty + recompute_total_affiliated(); + _partitions.assert_bounds(true); + return true; + } else { + return false; + } +} + bool ShenandoahFreeSet::flip_to_old_gc(ShenandoahHeapRegion* r) { const size_t idx = r->index(); @@ -1324,14 +2024,9 @@ bool ShenandoahFreeSet::flip_to_old_gc(ShenandoahHeapRegion* r) { assert(can_allocate_from(r), "Should not be allocated"); ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap(); - const size_t region_capacity = alloc_capacity(r); + const size_t region_alloc_capacity = alloc_capacity(r); - bool transferred = gen_heap->generation_sizer()->transfer_to_old(1); - if (transferred) { - _partitions.move_from_partition_to_partition(idx, ShenandoahFreeSetPartitionId::Mutator, - ShenandoahFreeSetPartitionId::OldCollector, region_capacity); - _partitions.assert_bounds(); - _heap->old_generation()->augment_evacuation_reserve(region_capacity); + if (transfer_one_region_from_mutator_to_old_collector(idx, region_alloc_capacity)) { return true; } @@ -1361,15 +2056,20 @@ bool ShenandoahFreeSet::flip_to_old_gc(ShenandoahHeapRegion* r) { // 3. Move this usable region from the mutator partition to the old collector partition _partitions.move_from_partition_to_partition(idx, ShenandoahFreeSetPartitionId::Mutator, - ShenandoahFreeSetPartitionId::OldCollector, region_capacity); - - _partitions.assert_bounds(); - + ShenandoahFreeSetPartitionId::OldCollector, region_alloc_capacity); + // Should have no effect on used, since flipped regions are trashed: zero used */ + // Transferred regions are not affiliated, because they are empty (trash) + recompute_total_affiliated(); + _partitions.assert_bounds(true); // 4. Do not adjust capacities for generations, we just swapped the regions that have already // been accounted for. However, we should adjust the evacuation reserves as those may have changed. shenandoah_assert_heaplocked(); const size_t reserve = _heap->old_generation()->get_evacuation_reserve(); - _heap->old_generation()->set_evacuation_reserve(reserve - unusable_capacity + region_capacity); + _heap->old_generation()->set_evacuation_reserve(reserve - unusable_capacity + region_alloc_capacity); return true; } } @@ -1387,8 +2087,15 @@ void ShenandoahFreeSet::flip_to_gc(ShenandoahHeapRegion* r) { size_t ac = alloc_capacity(r); _partitions.move_from_partition_to_partition(idx, ShenandoahFreeSetPartitionId::Mutator, ShenandoahFreeSetPartitionId::Collector, ac); - _partitions.assert_bounds(); - + recompute_total_used(); + // Transfer only affects unaffiliated regions, which stay in young + recompute_total_affiliated(); + _partitions.assert_bounds(true); // We do not ensure that the region is no longer trash, relying on try_allocate_in(), which always comes next, // to recycle trash before attempting to allocate anything in the region. } @@ -1400,52 +2107,82 @@ void ShenandoahFreeSet::clear() { void ShenandoahFreeSet::clear_internal() { shenandoah_assert_heaplocked(); _partitions.make_all_regions_unavailable(); - + recompute_total_used(); + recompute_total_affiliated(); _alloc_bias_weight = 0; _partitions.set_bias_from_left_to_right(ShenandoahFreeSetPartitionId::Mutator, true); _partitions.set_bias_from_left_to_right(ShenandoahFreeSetPartitionId::Collector, false); _partitions.set_bias_from_left_to_right(ShenandoahFreeSetPartitionId::OldCollector, false); } -void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_cset_regions, size_t &old_cset_regions, +void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_trashed_regions, size_t &old_trashed_regions, size_t &first_old_region, size_t &last_old_region, size_t &old_region_count) { + // This resets all state information, removing all regions from all sets. clear_internal(); first_old_region = _heap->num_regions(); last_old_region = 0; old_region_count = 0; - old_cset_regions = 0; - young_cset_regions = 0; + old_trashed_regions = 0; + young_trashed_regions = 0; + + size_t old_cset_regions = 0; + size_t young_cset_regions = 0; size_t region_size_bytes = _partitions.region_size_bytes(); - size_t max_regions = _partitions.max_regions(); + size_t max_regions = _partitions.max(); size_t mutator_leftmost = max_regions; size_t mutator_rightmost = 0; size_t mutator_leftmost_empty = max_regions; size_t mutator_rightmost_empty = 0; - size_t mutator_regions = 0; - size_t mutator_used = 0; size_t old_collector_leftmost = max_regions; size_t old_collector_rightmost = 0; size_t old_collector_leftmost_empty = max_regions; size_t old_collector_rightmost_empty = 0; - size_t old_collector_regions = 0; + + size_t mutator_empty = 0; + size_t old_collector_empty = 0; + + // These two variables represent the total used within each partition, including humongous waste and retired regions + size_t mutator_used = 0; size_t old_collector_used = 0; + // These two variables represent memory that is wasted within humongous regions due to alignment padding + size_t mutator_humongous_waste = 0; + size_t old_collector_humongous_waste = 0; + + // These two variables track regions that have allocatable memory + size_t mutator_regions = 0; + size_t old_collector_regions = 0; + + // These two variables track regions that are not empty within each partition + size_t affiliated_mutator_regions = 0; + size_t affiliated_old_collector_regions = 0; + + // These two variables represent the total capacity of each partition, including retired regions + size_t total_mutator_regions = 0; + size_t total_old_collector_regions = 0; + + bool is_generational = _heap->mode()->is_generational(); size_t num_regions = _heap->num_regions(); for (size_t idx = 0; idx < num_regions; idx++) { ShenandoahHeapRegion* region = _heap->get_region(idx); if (region->is_trash()) { - // Trashed regions represent regions that had been in the collection partition but have not yet been "cleaned up". - // The cset regions are not "trashed" until we have finished update refs. + // Trashed regions represent immediate garbage identified by final mark and regions that had been in the collection + // partition but have not yet been "cleaned up" following update refs. if (region->is_old()) { - old_cset_regions++; + old_trashed_regions++; } else { assert(region->is_young(), "Trashed region should be old or young"); - young_cset_regions++; + young_trashed_regions++; } } else if (region->is_old()) { // count both humongous and regular regions, but don't count trash (cset) regions. @@ -1460,7 +2197,7 @@ void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_cset_regi // Do not add regions that would almost surely fail allocation size_t ac = alloc_capacity(region); - if (ac > PLAB::min_size() * HeapWordSize) { + if (ac >= PLAB::min_size() * HeapWordSize) { if (region->is_trash() || !region->is_old()) { // Both young and old collected regions (trashed) are placed into the Mutator set _partitions.raw_assign_membership(idx, ShenandoahFreeSetPartitionId::Mutator); @@ -1471,14 +2208,18 @@ void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_cset_regi mutator_rightmost = idx; } if (ac == region_size_bytes) { + mutator_empty++; if (idx < mutator_leftmost_empty) { mutator_leftmost_empty = idx; } if (idx > mutator_rightmost_empty) { mutator_rightmost_empty = idx; } + } else { + affiliated_mutator_regions++; } mutator_regions++; + total_mutator_regions++; mutator_used += (region_size_bytes - ac); } else { // !region->is_trash() && region is_old() @@ -1489,20 +2230,67 @@ void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_cset_regi if (idx > old_collector_rightmost) { old_collector_rightmost = idx; } - if (ac == region_size_bytes) { - if (idx < old_collector_leftmost_empty) { - old_collector_leftmost_empty = idx; - } - if (idx > old_collector_rightmost_empty) { - old_collector_rightmost_empty = idx; - } - } + assert(ac != region_size_bytes, "Empty regions should be in mutator partition"); + affiliated_old_collector_regions++; old_collector_regions++; - old_collector_used += (region_size_bytes - ac); + total_old_collector_regions++; + old_collector_used += region_size_bytes - ac; + } + } else { + // This region does not have enough free to be part of the free set. Count all of its memory as used. + assert(_partitions.membership(idx) == ShenandoahFreeSetPartitionId::NotFree, "Region should have been retired"); + if (region->is_old()) { + old_collector_used += region_size_bytes; + total_old_collector_regions++; + affiliated_old_collector_regions++; + } else { + mutator_used += region_size_bytes; + total_mutator_regions++; + affiliated_mutator_regions++; + } + } + } else { + // This region does not allow allocation (it is retired or is humongous or is in cset). + // Retired and humongous regions generally have no alloc capacity, but cset regions may have large alloc capacity. + if (region->is_cset()) { + if (region->is_old()) { + old_cset_regions++; + } else { + young_cset_regions++; + } + } else { + assert(_partitions.membership(idx) == ShenandoahFreeSetPartitionId::NotFree, "Region should have been retired"); + size_t ac = alloc_capacity(region); + size_t humongous_waste_bytes = 0; + if (region->is_humongous_start()) { + oop obj = cast_to_oop(region->bottom()); + size_t byte_size = obj->size() * HeapWordSize; + size_t region_span = ShenandoahHeapRegion::required_regions(byte_size); + humongous_waste_bytes = region_span * ShenandoahHeapRegion::region_size_bytes() - byte_size; + } + if (region->is_old()) { + old_collector_used += region_size_bytes; + total_old_collector_regions++; + old_collector_humongous_waste += humongous_waste_bytes; + affiliated_old_collector_regions++; + } else { + mutator_used += region_size_bytes; + total_mutator_regions++; + mutator_humongous_waste += humongous_waste_bytes; + affiliated_mutator_regions++; } } } } + // At the start of evacuation, the cset regions are not counted as part of Mutator or OldCollector partitions. + + // At the end of GC, when we rebuild rebuild freeset (which happens before we have recycled the collection set), we treat + // all cset regions as part of capacity, as fully available, as unaffiliated. We place trashed regions into the Mutator + // partition. + + // No need to update generation sizes here. These are the sizes already recognized by the generations. These + // adjustments allow the freeset tallies to match the generation tallies. + log_debug(gc, free)(" At end of prep_to_rebuild, mutator_leftmost: %zu" ", mutator_rightmost: %zu" ", mutator_leftmost_empty: %zu" @@ -1511,7 +2299,6 @@ void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_cset_regi ", mutator_used: %zu", mutator_leftmost, mutator_rightmost, mutator_leftmost_empty, mutator_rightmost_empty, mutator_regions, mutator_used); - log_debug(gc, free)(" old_collector_leftmost: %zu" ", old_collector_rightmost: %zu" ", old_collector_leftmost_empty: %zu" @@ -1520,16 +2307,42 @@ void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_cset_regi ", old_collector_used: %zu", old_collector_leftmost, old_collector_rightmost, old_collector_leftmost_empty, old_collector_rightmost_empty, old_collector_regions, old_collector_used); - + log_debug(gc, free)(" total_mutator_regions: %zu, total_old_collector_regions: %zu" + ", mutator_empty: %zu, old_collector_empty: %zu", + total_mutator_regions, total_old_collector_regions, mutator_empty, old_collector_empty); idx_t rightmost_idx = (mutator_leftmost == max_regions)? -1: (idx_t) mutator_rightmost; idx_t rightmost_empty_idx = (mutator_leftmost_empty == max_regions)? -1: (idx_t) mutator_rightmost_empty; + _partitions.establish_mutator_intervals(mutator_leftmost, rightmost_idx, mutator_leftmost_empty, rightmost_empty_idx, - mutator_regions, mutator_used); + total_mutator_regions + young_cset_regions, mutator_empty, mutator_regions, + mutator_used + young_cset_regions * region_size_bytes, mutator_humongous_waste); rightmost_idx = (old_collector_leftmost == max_regions)? -1: (idx_t) old_collector_rightmost; rightmost_empty_idx = (old_collector_leftmost_empty == max_regions)? -1: (idx_t) old_collector_rightmost_empty; - _partitions.establish_old_collector_intervals(old_collector_leftmost, rightmost_idx, old_collector_leftmost_empty, - rightmost_empty_idx, old_collector_regions, old_collector_used); + _partitions.establish_old_collector_intervals(old_collector_leftmost, rightmost_idx, + old_collector_leftmost_empty, rightmost_empty_idx, + total_old_collector_regions + old_cset_regions, + old_collector_empty, old_collector_regions, + old_collector_used + old_cset_regions * region_size_bytes, + old_collector_humongous_waste); + _total_humongous_waste = mutator_humongous_waste + old_collector_humongous_waste; + _total_young_regions = total_mutator_regions + young_cset_regions; + _total_global_regions = _total_young_regions + total_old_collector_regions + old_cset_regions; + recompute_total_used(); + recompute_total_affiliated(); + _partitions.assert_bounds(true); +#ifdef ASSERT + if (_heap->mode()->is_generational()) { + assert(young_affiliated_regions() == _heap->young_generation()->get_affiliated_region_count(), "sanity"); + } else { + assert(young_affiliated_regions() == _heap->global_generation()->get_affiliated_region_count(), "sanity"); + } +#endif log_debug(gc, free)(" After find_regions_with_alloc_capacity(), Mutator range [%zd, %zd]," " Old Collector range [%zd, %zd]", _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator), @@ -1538,6 +2351,113 @@ void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_cset_regi _partitions.rightmost(ShenandoahFreeSetPartitionId::OldCollector)); } +void ShenandoahFreeSet::transfer_humongous_regions_from_mutator_to_old_collector(size_t xfer_regions, + size_t humongous_waste_bytes) { + shenandoah_assert_heaplocked(); + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + + _partitions.decrease_humongous_waste(ShenandoahFreeSetPartitionId::Mutator, humongous_waste_bytes); + _partitions.decrease_used(ShenandoahFreeSetPartitionId::Mutator, xfer_regions * region_size_bytes); + _partitions.decrease_capacity(ShenandoahFreeSetPartitionId::Mutator, xfer_regions * region_size_bytes); + + _partitions.increase_capacity(ShenandoahFreeSetPartitionId::OldCollector, xfer_regions * region_size_bytes); + _partitions.increase_humongous_waste(ShenandoahFreeSetPartitionId::OldCollector, humongous_waste_bytes); + _partitions.increase_used(ShenandoahFreeSetPartitionId::OldCollector, xfer_regions * region_size_bytes); + + // _total_humongous_waste, _total_global_regions are unaffected by transfer + _total_young_regions -= xfer_regions; + recompute_total_young_used(); + recompute_total_old_used(); + recompute_total_affiliated(); + _partitions.assert_bounds(true); + // global_used is unaffected by this transfer + + // No need to adjust ranges because humongous regions are not allocatable +} + +void ShenandoahFreeSet::transfer_empty_regions_from_to(ShenandoahFreeSetPartitionId source, + ShenandoahFreeSetPartitionId dest, + size_t num_regions) { + assert(dest != source, "precondition"); + shenandoah_assert_heaplocked(); + const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + size_t transferred_regions = 0; + size_t used_transfer = 0; + idx_t source_low_idx = _partitions.max(); + idx_t source_high_idx = -1; + idx_t dest_low_idx = _partitions.max(); + idx_t dest_high_idx = -1; + ShenandoahLeftRightIterator iterator(&_partitions, source, true); + for (idx_t idx = iterator.current(); transferred_regions < num_regions && iterator.has_next(); idx = iterator.next()) { + // Note: can_allocate_from() denotes that region is entirely empty + if (can_allocate_from(idx)) { + if (idx < source_low_idx) { + source_low_idx = idx; + } + if (idx > source_high_idx) { + source_high_idx = idx; + } + if (idx < dest_low_idx) { + dest_low_idx = idx; + } + if (idx > dest_high_idx) { + dest_high_idx = idx; + } + used_transfer += _partitions.move_from_partition_to_partition_with_deferred_accounting(idx, source, dest, region_size_bytes); + transferred_regions++; + } + } + + // All transferred regions are empty. + assert(used_transfer == 0, "empty regions should have no used"); + _partitions.expand_interval_if_range_modifies_either_boundary(dest, dest_low_idx, + dest_high_idx, dest_low_idx, dest_high_idx); + _partitions.shrink_interval_if_range_modifies_either_boundary(source, source_low_idx, source_high_idx, + transferred_regions); + + _partitions.decrease_region_counts(source, transferred_regions); + _partitions.decrease_empty_region_counts(source, transferred_regions); + _partitions.decrease_capacity(source, transferred_regions * region_size_bytes); + + _partitions.increase_capacity(dest, transferred_regions * region_size_bytes); + _partitions.increase_region_counts(dest, transferred_regions); + _partitions.increase_empty_region_counts(dest, transferred_regions); + + // Since only empty regions are transferred, no need to recompute_total_used() + if (source == ShenandoahFreeSetPartitionId::OldCollector) { + assert((dest == ShenandoahFreeSetPartitionId::Collector) || (dest == ShenandoahFreeSetPartitionId::Mutator), "sanity"); + _total_young_regions += transferred_regions; + recompute_total_affiliated(); + } else { + assert((source == ShenandoahFreeSetPartitionId::Collector) || (source == ShenandoahFreeSetPartitionId::Mutator), "sanity"); + if (dest == ShenandoahFreeSetPartitionId::OldCollector) { + _total_young_regions -= transferred_regions; + recompute_total_affiliated(); + } else { + assert((dest == ShenandoahFreeSetPartitionId::Collector) || (dest == ShenandoahFreeSetPartitionId::Mutator), "sanity"); + // No adjustments to total_young_regions if transferring within young + recompute_total_affiliated(); + } + } + _partitions.assert_bounds(true); +} + // Returns number of regions transferred, adds transferred bytes to var argument bytes_transferred size_t ShenandoahFreeSet::transfer_empty_regions_from_collector_set_to_mutator_set(ShenandoahFreeSetPartitionId which_collector, size_t max_xfer_regions, @@ -1545,33 +2465,139 @@ size_t ShenandoahFreeSet::transfer_empty_regions_from_collector_set_to_mutator_s shenandoah_assert_heaplocked(); const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); size_t transferred_regions = 0; + size_t used_transfer = 0; + idx_t collector_low_idx = _partitions.max(); + idx_t collector_high_idx = -1; + idx_t mutator_low_idx = _partitions.max(); + idx_t mutator_high_idx = -1; ShenandoahLeftRightIterator iterator(&_partitions, which_collector, true); for (idx_t idx = iterator.current(); transferred_regions < max_xfer_regions && iterator.has_next(); idx = iterator.next()) { // Note: can_allocate_from() denotes that region is entirely empty if (can_allocate_from(idx)) { - _partitions.move_from_partition_to_partition(idx, which_collector, ShenandoahFreeSetPartitionId::Mutator, region_size_bytes); + if (idx < collector_low_idx) { + collector_low_idx = idx; + } + if (idx > collector_high_idx) { + collector_high_idx = idx; + } + if (idx < mutator_low_idx) { + mutator_low_idx = idx; + } + if (idx > mutator_high_idx) { + mutator_high_idx = idx; + } + used_transfer += _partitions.move_from_partition_to_partition_with_deferred_accounting(idx, which_collector, + ShenandoahFreeSetPartitionId::Mutator, + region_size_bytes); transferred_regions++; bytes_transferred += region_size_bytes; } } + // All transferred regions are empty. + assert(used_transfer == 0, "empty regions should have no used"); + _partitions.expand_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId::Mutator, mutator_low_idx, + mutator_high_idx, mutator_low_idx, mutator_high_idx); + _partitions.shrink_interval_if_range_modifies_either_boundary(which_collector, collector_low_idx, collector_high_idx, + transferred_regions); + + _partitions.decrease_region_counts(which_collector, transferred_regions); + _partitions.decrease_empty_region_counts(which_collector, transferred_regions); + _partitions.decrease_capacity(which_collector, transferred_regions * region_size_bytes); + + _partitions.increase_capacity(ShenandoahFreeSetPartitionId::Mutator, transferred_regions * region_size_bytes); + _partitions.increase_region_counts(ShenandoahFreeSetPartitionId::Mutator, transferred_regions); + _partitions.increase_empty_region_counts(ShenandoahFreeSetPartitionId::Mutator, transferred_regions); + + if (which_collector == ShenandoahFreeSetPartitionId::OldCollector) { + _total_young_regions += transferred_regions; + recompute_total_affiliated(); + } else { + recompute_total_affiliated(); + } + _partitions.assert_bounds(true); return transferred_regions; } // Returns number of regions transferred, adds transferred bytes to var argument bytes_transferred -size_t ShenandoahFreeSet::transfer_non_empty_regions_from_collector_set_to_mutator_set(ShenandoahFreeSetPartitionId which_collector, - size_t max_xfer_regions, - size_t& bytes_transferred) { +size_t ShenandoahFreeSet:: +transfer_non_empty_regions_from_collector_set_to_mutator_set(ShenandoahFreeSetPartitionId which_collector, + size_t max_xfer_regions, size_t& bytes_transferred) { shenandoah_assert_heaplocked(); + size_t region_size_bytes = _partitions.region_size_bytes(); size_t transferred_regions = 0; + size_t used_transfer = 0; + idx_t collector_low_idx = _partitions.max(); + idx_t collector_high_idx = -1; + idx_t mutator_low_idx = _partitions.max(); + idx_t mutator_high_idx = -1; + ShenandoahLeftRightIterator iterator(&_partitions, which_collector, false); for (idx_t idx = iterator.current(); transferred_regions < max_xfer_regions && iterator.has_next(); idx = iterator.next()) { size_t ac = alloc_capacity(idx); if (ac > 0) { - _partitions.move_from_partition_to_partition(idx, which_collector, ShenandoahFreeSetPartitionId::Mutator, ac); + if (idx < collector_low_idx) { + collector_low_idx = idx; + } + if (idx > collector_high_idx) { + collector_high_idx = idx; + } + if (idx < mutator_low_idx) { + mutator_low_idx = idx; + } + if (idx > mutator_high_idx) { + mutator_high_idx = idx; + } + assert (ac < region_size_bytes, "Move empty regions with different function"); + used_transfer += _partitions.move_from_partition_to_partition_with_deferred_accounting(idx, which_collector, + ShenandoahFreeSetPartitionId::Mutator, + ac); transferred_regions++; bytes_transferred += ac; } } + // _empty_region_counts is unaffected, because we transfer only non-empty regions here. + + _partitions.decrease_used(which_collector, used_transfer); + _partitions.expand_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId::Mutator, + mutator_low_idx, mutator_high_idx, _partitions.max(), -1); + _partitions.shrink_interval_if_range_modifies_either_boundary(which_collector, collector_low_idx, collector_high_idx, + transferred_regions); + + _partitions.decrease_region_counts(which_collector, transferred_regions); + _partitions.decrease_capacity(which_collector, transferred_regions * region_size_bytes); + _partitions.increase_capacity(ShenandoahFreeSetPartitionId::Mutator, transferred_regions * region_size_bytes); + _partitions.increase_region_counts(ShenandoahFreeSetPartitionId::Mutator, transferred_regions); + _partitions.increase_used(ShenandoahFreeSetPartitionId::Mutator, used_transfer); + + if (which_collector == ShenandoahFreeSetPartitionId::OldCollector) { + _total_young_regions += transferred_regions; + } + // _total_global_regions unaffected by transfer + recompute_total_used(); + // All transfers are affiliated + if (which_collector == ShenandoahFreeSetPartitionId::OldCollector) { + recompute_total_affiliated(); + } else { + recompute_total_affiliated(); + } + _partitions.assert_bounds(true); return transferred_regions; } @@ -1598,9 +2624,6 @@ void ShenandoahFreeSet::move_regions_from_collector_to_mutator(size_t max_xfer_r transfer_empty_regions_from_collector_set_to_mutator_set(ShenandoahFreeSetPartitionId::OldCollector, max_xfer_regions, old_collector_xfer); max_xfer_regions -= old_collector_regions; - if (old_collector_regions > 0) { - ShenandoahGenerationalHeap::cast(_heap)->generation_sizer()->transfer_to_young(old_collector_regions); - } } // If there are any non-empty regions within Collector partition, we can also move them to the Mutator free partition @@ -1620,77 +2643,62 @@ void ShenandoahFreeSet::move_regions_from_collector_to_mutator(size_t max_xfer_r byte_size_in_proper_unit(old_collector_xfer), proper_unit_for_byte_size(old_collector_xfer)); } - // Overwrite arguments to represent the amount of memory in each generation that is about to be recycled -void ShenandoahFreeSet::prepare_to_rebuild(size_t &young_cset_regions, size_t &old_cset_regions, +void ShenandoahFreeSet::prepare_to_rebuild(size_t &young_trashed_regions, size_t &old_trashed_regions, size_t &first_old_region, size_t &last_old_region, size_t &old_region_count) { shenandoah_assert_heaplocked(); - // This resets all state information, removing all regions from all sets. - clear(); log_debug(gc, free)("Rebuilding FreeSet"); // This places regions that have alloc_capacity into the old_collector set if they identify as is_old() or the // mutator set otherwise. All trashed (cset) regions are affiliated young and placed in mutator set. - find_regions_with_alloc_capacity(young_cset_regions, old_cset_regions, first_old_region, last_old_region, old_region_count); -} - -void ShenandoahFreeSet::establish_generation_sizes(size_t young_region_count, size_t old_region_count) { - assert(young_region_count + old_region_count == ShenandoahHeap::heap()->num_regions(), "Sanity"); - if (ShenandoahHeap::heap()->mode()->is_generational()) { - ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap(); - ShenandoahOldGeneration* old_gen = heap->old_generation(); - ShenandoahYoungGeneration* young_gen = heap->young_generation(); - size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); - - size_t original_old_capacity = old_gen->max_capacity(); - size_t new_old_capacity = old_region_count * region_size_bytes; - size_t new_young_capacity = young_region_count * region_size_bytes; - old_gen->set_capacity(new_old_capacity); - young_gen->set_capacity(new_young_capacity); - - if (new_old_capacity > original_old_capacity) { - size_t region_count = (new_old_capacity - original_old_capacity) / region_size_bytes; - log_info(gc, ergo)("Transfer %zu region(s) from %s to %s, yielding increased size: " PROPERFMT, - region_count, young_gen->name(), old_gen->name(), PROPERFMTARGS(new_old_capacity)); - } else if (new_old_capacity < original_old_capacity) { - size_t region_count = (original_old_capacity - new_old_capacity) / region_size_bytes; - log_info(gc, ergo)("Transfer %zu region(s) from %s to %s, yielding increased size: " PROPERFMT, - region_count, old_gen->name(), young_gen->name(), PROPERFMTARGS(new_young_capacity)); - } - // This balances generations, so clear any pending request to balance. - old_gen->set_region_balance(0); - } + find_regions_with_alloc_capacity(young_trashed_regions, old_trashed_regions, + first_old_region, last_old_region, old_region_count); } -void ShenandoahFreeSet::finish_rebuild(size_t young_cset_regions, size_t old_cset_regions, size_t old_region_count, +void ShenandoahFreeSet::finish_rebuild(size_t young_trashed_regions, size_t old_trashed_regions, size_t old_region_count, bool have_evacuation_reserves) { shenandoah_assert_heaplocked(); size_t young_reserve(0), old_reserve(0); if (_heap->mode()->is_generational()) { - compute_young_and_old_reserves(young_cset_regions, old_cset_regions, have_evacuation_reserves, + compute_young_and_old_reserves(young_trashed_regions, old_trashed_regions, have_evacuation_reserves, young_reserve, old_reserve); } else { young_reserve = (_heap->max_capacity() / 100) * ShenandoahEvacReserve; old_reserve = 0; } - // Move some of the mutator regions in the Collector and OldCollector partitions in order to satisfy + // Move some of the mutator regions into the Collector and OldCollector partitions in order to satisfy // young_reserve and old_reserve. - reserve_regions(young_reserve, old_reserve, old_region_count); - size_t young_region_count = _heap->num_regions() - old_region_count; - establish_generation_sizes(young_region_count, old_region_count); + size_t young_used_regions, old_used_regions, young_used_bytes, old_used_bytes; + reserve_regions(young_reserve, old_reserve, old_region_count, young_used_regions, old_used_regions, + young_used_bytes, old_used_bytes); + _total_young_regions = _heap->num_regions() - old_region_count; + _total_global_regions = _heap->num_regions(); establish_old_collector_alloc_bias(); - _partitions.assert_bounds(); + _partitions.assert_bounds(true); log_status(); } -void ShenandoahFreeSet::compute_young_and_old_reserves(size_t young_cset_regions, size_t old_cset_regions, +/** + * Set young_reserve_result and old_reserve_result to the number of bytes that we desire to set aside to hold the + * results of evacuation to young and old collector spaces respectively during the next evacuation phase. Overwrite + * old_generation region balance in case the original value is incompatible with the current reality. + * + * These values are determined by how much memory is currently available within each generation, which is + * represented by: + * 1. Memory currently available within old and young + * 2. Trashed regions currently residing in young and old, which will become available momentarily + * 3. The value of old_generation->get_region_balance() which represents the number of regions that we plan + * to transfer from old generation to young generation. Prior to each invocation of compute_young_and_old_reserves(), + * this value should computed by ShenandoahGenerationalHeap::compute_old_generation_balance(). + */ +void ShenandoahFreeSet::compute_young_and_old_reserves(size_t young_trashed_regions, size_t old_trashed_regions, bool have_evacuation_reserves, size_t& young_reserve_result, size_t& old_reserve_result) const { shenandoah_assert_generational(); + shenandoah_assert_heaplocked(); const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); - ShenandoahOldGeneration* const old_generation = _heap->old_generation(); size_t old_available = old_generation->available(); size_t old_unaffiliated_regions = old_generation->free_unaffiliated_regions(); @@ -1699,18 +2707,23 @@ void ShenandoahFreeSet::compute_young_and_old_reserves(size_t young_cset_regions size_t young_unaffiliated_regions = young_generation->free_unaffiliated_regions(); // Add in the regions we anticipate to be freed by evacuation of the collection set - old_unaffiliated_regions += old_cset_regions; - young_unaffiliated_regions += young_cset_regions; + old_unaffiliated_regions += old_trashed_regions; + old_available += old_trashed_regions * region_size_bytes; + young_unaffiliated_regions += young_trashed_regions; // Consult old-region balance to make adjustments to current generation capacities and availability. - // The generation region transfers take place after we rebuild. - const ssize_t old_region_balance = old_generation->get_region_balance(); + // The generation region transfers take place after we rebuild. old_region_balance represents number of regions + // to transfer from old to young. + ssize_t old_region_balance = old_generation->get_region_balance(); if (old_region_balance != 0) { #ifdef ASSERT if (old_region_balance > 0) { - assert(old_region_balance <= checked_cast(old_unaffiliated_regions), "Cannot transfer regions that are affiliated"); + assert(old_region_balance <= checked_cast(old_unaffiliated_regions), + "Cannot transfer %zd regions that are affiliated (old_trashed: %zu, old_unaffiliated: %zu)", + old_region_balance, old_trashed_regions, old_unaffiliated_regions); } else { - assert(0 - old_region_balance <= checked_cast(young_unaffiliated_regions), "Cannot transfer regions that are affiliated"); + assert(0 - old_region_balance <= checked_cast(young_unaffiliated_regions), + "Cannot transfer regions that are affiliated"); } #endif @@ -1731,9 +2744,18 @@ void ShenandoahFreeSet::compute_young_and_old_reserves(size_t young_cset_regions const size_t old_evac_reserve = old_generation->get_evacuation_reserve(); young_reserve_result = young_generation->get_evacuation_reserve(); old_reserve_result = promoted_reserve + old_evac_reserve; - assert(old_reserve_result <= old_available, - "Cannot reserve (%zu + %zu) more OLD than is available: %zu", - promoted_reserve, old_evac_reserve, old_available); + if (old_reserve_result > old_available) { + // Try to transfer memory from young to old. + size_t old_deficit = old_reserve_result - old_available; + size_t old_region_deficit = (old_deficit + region_size_bytes - 1) / region_size_bytes; + if (young_unaffiliated_regions < old_region_deficit) { + old_region_deficit = young_unaffiliated_regions; + } + young_unaffiliated_regions -= old_region_deficit; + old_unaffiliated_regions += old_region_deficit; + old_region_balance -= old_region_deficit; + old_generation->set_region_balance(old_region_balance); + } } else { // We are rebuilding at end of GC, so we set aside budgets specified on command line (or defaults) young_reserve_result = (young_capacity * ShenandoahEvacReserve) / 100; @@ -1763,69 +2785,239 @@ void ShenandoahFreeSet::compute_young_and_old_reserves(size_t young_cset_regions // into the collector set or old collector set in order to assure that the memory available for allocations within // the collector set is at least to_reserve and the memory available for allocations within the old collector set // is at least to_reserve_old. -void ShenandoahFreeSet::reserve_regions(size_t to_reserve, size_t to_reserve_old, size_t &old_region_count) { - for (size_t i = _heap->num_regions(); i > 0; i--) { - size_t idx = i - 1; - ShenandoahHeapRegion* r = _heap->get_region(idx); - if (!_partitions.in_free_set(ShenandoahFreeSetPartitionId::Mutator, idx)) { - continue; - } +void ShenandoahFreeSet::reserve_regions(size_t to_reserve, size_t to_reserve_old, size_t &old_region_count, + size_t &young_used_regions, size_t &old_used_regions, + size_t &young_used_bytes, size_t &old_used_bytes) { + const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); - size_t ac = alloc_capacity(r); - assert (ac > 0, "Membership in free set implies has capacity"); - assert (!r->is_old() || r->is_trash(), "Except for trash, mutator_is_free regions should not be affiliated OLD"); + young_used_regions = 0; + old_used_regions = 0; + young_used_bytes = 0; + old_used_bytes = 0; - bool move_to_old_collector = _partitions.available_in(ShenandoahFreeSetPartitionId::OldCollector) < to_reserve_old; - bool move_to_collector = _partitions.available_in(ShenandoahFreeSetPartitionId::Collector) < to_reserve; + idx_t mutator_low_idx = _partitions.max(); + idx_t mutator_high_idx = -1; + idx_t mutator_empty_low_idx = _partitions.max(); + idx_t mutator_empty_high_idx = -1; - if (!move_to_collector && !move_to_old_collector) { - // We've satisfied both to_reserve and to_reserved_old - break; - } + idx_t collector_low_idx = _partitions.max(); + idx_t collector_high_idx = -1; + idx_t collector_empty_low_idx = _partitions.max(); + idx_t collector_empty_high_idx = -1; + + idx_t old_collector_low_idx = _partitions.max(); + idx_t old_collector_high_idx = -1; + idx_t old_collector_empty_low_idx = _partitions.max(); + idx_t old_collector_empty_high_idx = -1; + + size_t used_to_collector = 0; + size_t used_to_old_collector = 0; + size_t regions_to_collector = 0; + size_t regions_to_old_collector = 0; + size_t empty_regions_to_collector = 0; + size_t empty_regions_to_old_collector = 0; - if (move_to_old_collector) { - // We give priority to OldCollector partition because we desire to pack OldCollector regions into higher - // addresses than Collector regions. Presumably, OldCollector regions are more "stable" and less likely to - // be collected in the near future. - if (r->is_trash() || !r->is_affiliated()) { - // OLD regions that have available memory are already in the old_collector free set. - _partitions.move_from_partition_to_partition(idx, ShenandoahFreeSetPartitionId::Mutator, - ShenandoahFreeSetPartitionId::OldCollector, ac); - log_trace(gc, free)(" Shifting region %zu from mutator_free to old_collector_free", idx); + size_t old_collector_available = _partitions.available_in(ShenandoahFreeSetPartitionId::OldCollector);; + size_t collector_available = _partitions.available_in(ShenandoahFreeSetPartitionId::Collector); + + for (size_t i = _heap->num_regions(); i > 0; i--) { + idx_t idx = i - 1; + ShenandoahHeapRegion* r = _heap->get_region(idx); + if (_partitions.in_free_set(ShenandoahFreeSetPartitionId::Mutator, idx)) { + // Note: trashed regions have region_size_bytes alloc capacity. + size_t ac = alloc_capacity(r); + assert (ac > 0, "Membership in free set implies has capacity"); + assert (!r->is_old() || r->is_trash(), "Except for trash, mutator_is_free regions should not be affiliated OLD"); + + bool move_to_old_collector = old_collector_available < to_reserve_old; + bool move_to_collector = collector_available < to_reserve; + + if (move_to_old_collector) { + // We give priority to OldCollector partition because we desire to pack OldCollector regions into higher + // addresses than Collector regions. Presumably, OldCollector regions are more "stable" and less likely to + // be collected in the near future. + if (r->is_trash() || !r->is_affiliated()) { + // OLD regions that have available memory are already in the old_collector free set. + assert(r->is_empty() || r->is_trash(), "Not affiliated implies region %zu is empty", r->index()); + if (idx < old_collector_low_idx) { + old_collector_low_idx = idx; + } + if (idx > old_collector_high_idx) { + old_collector_high_idx = idx; + } + if (idx < old_collector_empty_low_idx) { + old_collector_empty_low_idx = idx; + } + if (idx > old_collector_empty_high_idx) { + old_collector_empty_high_idx = idx; + } + used_to_old_collector += + _partitions.move_from_partition_to_partition_with_deferred_accounting(idx, ShenandoahFreeSetPartitionId::Mutator, + ShenandoahFreeSetPartitionId::OldCollector, ac); + old_collector_available += ac; + regions_to_old_collector++; + empty_regions_to_old_collector++; + + log_trace(gc, free)(" Shifting region %zu from mutator_free to old_collector_free", idx); + log_trace(gc, free)(" Shifted Mutator range [%zd, %zd]," + " Old Collector range [%zd, %zd]", + _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator), + _partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator), + _partitions.leftmost(ShenandoahFreeSetPartitionId::OldCollector), + _partitions.rightmost(ShenandoahFreeSetPartitionId::OldCollector)); + old_region_count++; + continue; + } + } + + if (move_to_collector) { + // Note: In a previous implementation, regions were only placed into the survivor space (collector_is_free) if + // they were entirely empty. This has the effect of causing new Mutator allocation to reside next to objects + // that have already survived at least one GC, mixing ephemeral with longer-lived objects in the same region. + // Any objects that have survived a GC are less likely to immediately become garbage, so a region that contains + // survivor objects is less likely to be selected for the collection set. This alternative implementation allows + // survivor regions to continue accumulating other survivor objects, and makes it more likely that ephemeral objects + // occupy regions comprised entirely of ephemeral objects. These regions are highly likely to be included in the next + // collection set, and they are easily evacuated because they have low density of live objects. + if (idx < collector_low_idx) { + collector_low_idx = idx; + } + if (idx > collector_high_idx) { + collector_high_idx = idx; + } + if (ac == region_size_bytes) { + if (idx < collector_empty_low_idx) { + collector_empty_low_idx = idx; + } + if (idx > collector_empty_high_idx) { + collector_empty_high_idx = idx; + } + empty_regions_to_collector++; + } + used_to_collector += + _partitions.move_from_partition_to_partition_with_deferred_accounting(idx, ShenandoahFreeSetPartitionId::Mutator, + ShenandoahFreeSetPartitionId::Collector, ac); + collector_available += ac; + regions_to_collector++; + if (ac != region_size_bytes) { + young_used_regions++; + young_used_bytes = region_size_bytes - ac; + } + + log_trace(gc, free)(" Shifting region %zu from mutator_free to collector_free", idx); log_trace(gc, free)(" Shifted Mutator range [%zd, %zd]," - " Old Collector range [%zd, %zd]", + " Collector range [%zd, %zd]", _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator), _partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator), - _partitions.leftmost(ShenandoahFreeSetPartitionId::OldCollector), - _partitions.rightmost(ShenandoahFreeSetPartitionId::OldCollector)); - - old_region_count++; + _partitions.leftmost(ShenandoahFreeSetPartitionId::Collector), + _partitions.rightmost(ShenandoahFreeSetPartitionId::Collector)); continue; } - } - - if (move_to_collector) { - // Note: In a previous implementation, regions were only placed into the survivor space (collector_is_free) if - // they were entirely empty. This has the effect of causing new Mutator allocation to reside next to objects - // that have already survived at least one GC, mixing ephemeral with longer-lived objects in the same region. - // Any objects that have survived a GC are less likely to immediately become garbage, so a region that contains - // survivor objects is less likely to be selected for the collection set. This alternative implementation allows - // survivor regions to continue accumulating other survivor objects, and makes it more likely that ephemeral objects - // occupy regions comprised entirely of ephemeral objects. These regions are highly likely to be included in the next - // collection set, and they are easily evacuated because they have low density of live objects. - _partitions.move_from_partition_to_partition(idx, ShenandoahFreeSetPartitionId::Mutator, - ShenandoahFreeSetPartitionId::Collector, ac); - log_trace(gc, free)(" Shifting region %zu from mutator_free to collector_free", idx); - log_trace(gc, free)(" Shifted Mutator range [%zd, %zd]," - " Collector range [%zd, %zd]", - _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator), - _partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator), - _partitions.leftmost(ShenandoahFreeSetPartitionId::Collector), - _partitions.rightmost(ShenandoahFreeSetPartitionId::Collector)); + // Mutator region is not moved to Collector or OldCollector. Still, do the accounting. + if (idx < mutator_low_idx) { + mutator_low_idx = idx; + } + if (idx > mutator_high_idx) { + mutator_high_idx = idx; + } + if ((ac == region_size_bytes) && (idx < mutator_empty_low_idx)) { + mutator_empty_low_idx = idx; + } + if ((ac == region_size_bytes) && (idx > mutator_empty_high_idx)) { + mutator_empty_high_idx = idx; + } + if (ac != region_size_bytes) { + young_used_regions++; + young_used_bytes += region_size_bytes - ac; + } + } else { + // Region is not in Mutator partition. Do the accounting. + ShenandoahFreeSetPartitionId p = _partitions.membership(idx); + size_t ac = alloc_capacity(r); + assert(ac != region_size_bytes, "Empty regions should be in Mutator partion at entry to reserve_regions"); + if (p == ShenandoahFreeSetPartitionId::Collector) { + if (ac != region_size_bytes) { + young_used_regions++; + young_used_bytes = region_size_bytes - ac; + } + // else, unaffiliated region has no used + } else if (p == ShenandoahFreeSetPartitionId::OldCollector) { + if (ac != region_size_bytes) { + old_used_regions++; + old_used_bytes = region_size_bytes - ac; + } + // else, unaffiliated region has no used + } else if (p == ShenandoahFreeSetPartitionId::NotFree) { + // This region has been retired + if (r->is_old()) { + old_used_regions++; + old_used_bytes += region_size_bytes - ac; + } else { + assert(r->is_young(), "Retired region should be old or young"); + young_used_regions++; + young_used_bytes += region_size_bytes - ac; + } + } else { + assert(p == ShenandoahFreeSetPartitionId::OldCollector, "Not mutator and not NotFree, so must be OldCollector"); + assert(!r->is_empty(), "Empty regions should be in Mutator partition at entry to reserve_regions"); + if (idx < old_collector_low_idx) { + old_collector_low_idx = idx; + } + if (idx > old_collector_high_idx) { + old_collector_high_idx = idx; + } + if (idx < old_collector_empty_low_idx) { + old_collector_empty_low_idx = idx; + } + if (idx > old_collector_empty_high_idx) { + old_collector_empty_high_idx = idx; + } + } } } + _partitions.decrease_used(ShenandoahFreeSetPartitionId::Mutator, used_to_old_collector + used_to_collector); + _partitions.decrease_region_counts(ShenandoahFreeSetPartitionId::Mutator, regions_to_old_collector + regions_to_collector); + _partitions.decrease_empty_region_counts(ShenandoahFreeSetPartitionId::Mutator, + empty_regions_to_old_collector + empty_regions_to_collector); + // decrease_capacity() also decreases available + _partitions.decrease_capacity(ShenandoahFreeSetPartitionId::Mutator, + (regions_to_old_collector + regions_to_collector) * region_size_bytes); + // increase_capacity() also increases available + _partitions.increase_capacity(ShenandoahFreeSetPartitionId::Collector, regions_to_collector * region_size_bytes); + _partitions.increase_region_counts(ShenandoahFreeSetPartitionId::Collector, regions_to_collector); + _partitions.increase_empty_region_counts(ShenandoahFreeSetPartitionId::Collector, empty_regions_to_collector); + // increase_capacity() also increases available + _partitions.increase_capacity(ShenandoahFreeSetPartitionId::OldCollector, regions_to_old_collector * region_size_bytes); + _partitions.increase_region_counts(ShenandoahFreeSetPartitionId::OldCollector, regions_to_old_collector); + _partitions.increase_empty_region_counts(ShenandoahFreeSetPartitionId::OldCollector, empty_regions_to_old_collector); + + if (used_to_collector > 0) { + _partitions.increase_used(ShenandoahFreeSetPartitionId::Collector, used_to_collector); + } + + if (used_to_old_collector > 0) { + _partitions.increase_used(ShenandoahFreeSetPartitionId::OldCollector, used_to_old_collector); + } + + _partitions.expand_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId::Collector, + collector_low_idx, collector_high_idx, + collector_empty_low_idx, collector_empty_high_idx); + _partitions.expand_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId::OldCollector, + old_collector_low_idx, old_collector_high_idx, + old_collector_empty_low_idx, old_collector_empty_high_idx); + _partitions.establish_interval(ShenandoahFreeSetPartitionId::Mutator, + mutator_low_idx, mutator_high_idx, mutator_empty_low_idx, mutator_empty_high_idx); + + recompute_total_used(); + recompute_total_affiliated(); + _partitions.assert_bounds(true); if (LogTarget(Info, gc, free)::is_enabled()) { size_t old_reserve = _partitions.available_in(ShenandoahFreeSetPartitionId::OldCollector); if (old_reserve < to_reserve_old) { @@ -1966,6 +3158,7 @@ void ShenandoahFreeSet::log_status() { size_t total_used = 0; size_t total_free = 0; size_t total_free_ext = 0; + size_t total_trashed_free = 0; for (idx_t idx = _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator); idx <= _partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator); idx++) { @@ -1973,7 +3166,9 @@ void ShenandoahFreeSet::log_status() { ShenandoahHeapRegion *r = _heap->get_region(idx); size_t free = alloc_capacity(r); max = MAX2(max, free); - if (r->is_empty()) { + size_t used_in_region = r->used(); + if (r->is_empty() || r->is_trash()) { + used_in_region = 0; total_free_ext += free; if (last_idx + 1 == idx) { empty_contig++; @@ -1983,7 +3178,7 @@ void ShenandoahFreeSet::log_status() { } else { empty_contig = 0; } - total_used += r->used(); + total_used += used_in_region; total_free += free; max_contig = MAX2(max_contig, empty_contig); last_idx = idx; @@ -1991,12 +3186,13 @@ void ShenandoahFreeSet::log_status() { } size_t max_humongous = max_contig * ShenandoahHeapRegion::region_size_bytes(); + // capacity() is capacity of mutator + // used() is used of mutator size_t free = capacity() - used(); - // Since certain regions that belonged to the Mutator free partition at the time of most recent rebuild may have been // retired, the sum of used and capacities within regions that are still in the Mutator free partition may not match // my internally tracked values of used() and free(). - assert(free == total_free, "Free memory should match"); + assert(free == total_free, "Free memory (%zu) should match calculated memory (%zu)", free, total_free); ls.print("Free: %zu%s, Max: %zu%s regular, %zu%s humongous, ", byte_size_in_proper_unit(total_free), proper_unit_for_byte_size(total_free), byte_size_in_proper_unit(max), proper_unit_for_byte_size(max), @@ -2069,6 +3265,27 @@ void ShenandoahFreeSet::log_status() { } } +void ShenandoahFreeSet::decrease_humongous_waste_for_regular_bypass(ShenandoahHeapRegion*r, size_t waste) { + shenandoah_assert_heaplocked(); + assert(_partitions.membership(r->index()) == ShenandoahFreeSetPartitionId::NotFree, "Humongous regions should be NotFree"); + ShenandoahFreeSetPartitionId p = + r->is_old()? ShenandoahFreeSetPartitionId::OldCollector: ShenandoahFreeSetPartitionId::Mutator; + _partitions.decrease_humongous_waste(p, waste); + if (waste >= PLAB::min_size() * HeapWordSize) { + _partitions.decrease_used(p, waste); + _partitions.unretire_to_partition(r, p); + if (r->is_old()) { + recompute_total_used(); + } else { + recompute_total_used(); + } + } + _total_humongous_waste -= waste; +} + + HeapWord* ShenandoahFreeSet::allocate(ShenandoahAllocRequest& req, bool& in_new_region) { shenandoah_assert_heaplocked(); if (ShenandoahHeapRegion::requires_humongous(req.size())) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp index 8ad7055b3d6a4..11d93bf094a01 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp @@ -35,8 +35,12 @@ enum class ShenandoahFreeSetPartitionId : uint8_t { Mutator, // Region is in the Mutator free set: available memory is available to mutators. Collector, // Region is in the Collector free set: available memory is reserved for evacuations. OldCollector, // Region is in the Old Collector free set: - // available memory is reserved for old evacuations and for promotions.. - NotFree // Region is in no free set: it has no available memory + // available memory is reserved for old evacuations and for promotions. + NotFree // Region is in no free set: it has no available memory. Consult region affiliation + // to determine whether this retired region is young or old. If young, the region + // is considered to be part of the Mutator partition. (When we retire from the + // Collector partition, we decrease total_region_count for Collector and increaese + // for Mutator, making similar adjustments to used (net impact on available is neutral). }; // ShenandoahRegionPartitions provides an abstraction to help organize the implementation of ShenandoahFreeSet. This @@ -45,64 +49,92 @@ enum class ShenandoahFreeSetPartitionId : uint8_t { // for which the ShenandoahFreeSetPartitionId is not equal to NotFree. class ShenandoahRegionPartitions { -private: +using idx_t = ShenandoahSimpleBitMap::idx_t; + +public: // We do not maintain counts, capacity, or used for regions that are not free. Informally, if a region is NotFree, it is // in no partition. NumPartitions represents the size of an array that may be indexed by Mutator or Collector. static constexpr ShenandoahFreeSetPartitionId NumPartitions = ShenandoahFreeSetPartitionId::NotFree; static constexpr int IntNumPartitions = int(ShenandoahFreeSetPartitionId::NotFree); static constexpr uint UIntNumPartitions = uint(ShenandoahFreeSetPartitionId::NotFree); - const ssize_t _max; // The maximum number of heap regions +private: + const idx_t _max; // The maximum number of heap regions const size_t _region_size_bytes; const ShenandoahFreeSet* _free_set; // For each partition, we maintain a bitmap of which regions are affiliated with his partition. ShenandoahSimpleBitMap _membership[UIntNumPartitions]; - // For each partition, we track an interval outside of which a region affiliated with that partition is guaranteed // not to be found. This makes searches for free space more efficient. For each partition p, _leftmosts[p] // represents its least index, and its _rightmosts[p] its greatest index. Empty intervals are indicated by the // canonical [_max, -1]. - ssize_t _leftmosts[UIntNumPartitions]; - ssize_t _rightmosts[UIntNumPartitions]; + idx_t _leftmosts[UIntNumPartitions]; + idx_t _rightmosts[UIntNumPartitions]; // Allocation for humongous objects needs to find regions that are entirely empty. For each partion p, _leftmosts_empty[p] // represents the first region belonging to this partition that is completely empty and _rightmosts_empty[p] represents the // last region that is completely empty. If there is no completely empty region in this partition, this is represented // by the canonical [_max, -1]. - ssize_t _leftmosts_empty[UIntNumPartitions]; - ssize_t _rightmosts_empty[UIntNumPartitions]; - - // For each partition p, _capacity[p] represents the total amount of memory within the partition at the time - // of the most recent rebuild, _used[p] represents the total amount of memory that has been allocated within this - // partition (either already allocated as of the rebuild, or allocated since the rebuild). _capacity[p] and _used[p] - // are denoted in bytes. Note that some regions that had been assigned to a particular partition at rebuild time - // may have been retired following the rebuild. The tallies for these regions are still reflected in _capacity[p] - // and _used[p], even though the region may have been removed from the free set. + idx_t _leftmosts_empty[UIntNumPartitions]; + idx_t _rightmosts_empty[UIntNumPartitions]; + + // For each partition p: + // _capacity[p] represents the total amount of memory within the partition, including retired regions, as adjusted + // by transfers of memory between partitions + // _used[p] represents the total amount of memory that has been allocated within this partition (either already + // allocated as of the rebuild, or allocated since the rebuild). + // _available[p] represents the total amount of memory that can be allocated within partition p, calculated from + // _capacity[p] minus _used[p], where the difference is computed and assigned under heap lock + // + // _region_counts[p] represents the number of regions associated with the partition which currently have available memory. + // When a region is retired from partition p, _region_counts[p] is decremented. + // total_region_counts[p] is _capacity[p] / RegionSizeBytes. + // _empty_region_counts[p] is number of regions associated with p which are entirely empty + // + // capacity and used values are expressed in bytes. + // + // When a region is retired, the used[p] is increased to account for alignment waste. capacity is unaffected. + // + // When a region is "flipped", we adjust capacities and region counts for original and destination partitions. We also + // adjust used values when flipping from mutator to collector. Flip to old collector does not need to adjust used because + // only empty regions can be flipped to old collector. + // + // All memory quantities (capacity, available, used) are represented in bytes. + size_t _capacity[UIntNumPartitions]; + size_t _used[UIntNumPartitions]; size_t _available[UIntNumPartitions]; + + // Measured in bytes. + size_t _allocated_since_gc_start[UIntNumPartitions]; + + // Some notes: + // total_region_counts[p] is _capacity[p] / region_size_bytes + // retired_regions[p] is total_region_counts[p] - _region_counts[p] + // _empty_region_counts[p] <= _region_counts[p] <= total_region_counts[p] + // affiliated regions is total_region_counts[p] - empty_region_counts[p] + // used_regions is affilaited_regions * region_size_bytes + // _available[p] is _capacity[p] - _used[p] size_t _region_counts[UIntNumPartitions]; + size_t _empty_region_counts[UIntNumPartitions]; + + // Humongous waste, in bytes, can exist in Mutator partition for recently allocated humongous objects + // and in OldCollector partition for humongous objects that have been promoted in place. + size_t _humongous_waste[UIntNumPartitions]; // For each partition p, _left_to_right_bias is true iff allocations are normally made from lower indexed regions // before higher indexed regions. bool _left_to_right_bias[UIntNumPartitions]; - // Shrink the intervals associated with partition when region idx is removed from this free set - inline void shrink_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, ssize_t idx); - - // Shrink the intervals associated with partition when regions low_idx through high_idx inclusive are removed from this free set - inline void shrink_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId partition, - ssize_t low_idx, ssize_t high_idx); - inline void expand_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, ssize_t idx, size_t capacity); - inline bool is_mutator_partition(ShenandoahFreeSetPartitionId p); inline bool is_young_collector_partition(ShenandoahFreeSetPartitionId p); inline bool is_old_collector_partition(ShenandoahFreeSetPartitionId p); inline bool available_implies_empty(size_t available); #ifndef PRODUCT - void dump_bitmap_row(ssize_t region_idx) const; - void dump_bitmap_range(ssize_t start_region_idx, ssize_t end_region_idx) const; + void dump_bitmap_row(idx_t region_idx) const; + void dump_bitmap_range(idx_t start_region_idx, idx_t end_region_idx) const; void dump_bitmap() const; #endif public: @@ -111,6 +143,11 @@ class ShenandoahRegionPartitions { static const size_t FreeSetUnderConstruction = SIZE_MAX; + inline idx_t max() const { return _max; } + + // At initialization, reset OldCollector tallies + void initialize_old_collector(); + // Remove all regions from all partitions and reset all bounds void make_all_regions_unavailable(); @@ -119,70 +156,116 @@ class ShenandoahRegionPartitions { _membership[int(p)].set_bit(idx); } + // Clear the partition id for a particular region without adjusting interval bounds or usage/capacity tallies + inline void raw_clear_membership(size_t idx, ShenandoahFreeSetPartitionId p) { + _membership[int(p)].clear_bit(idx); + } + + inline void one_region_is_no_longer_empty(ShenandoahFreeSetPartitionId partition); + // Set the Mutator intervals, usage, and capacity according to arguments. Reset the Collector intervals, used, capacity // to represent empty Collector free set. We use this at the end of rebuild_free_set() to avoid the overhead of making // many redundant incremental adjustments to the mutator intervals as the free set is being rebuilt. - void establish_mutator_intervals(ssize_t mutator_leftmost, ssize_t mutator_rightmost, - ssize_t mutator_leftmost_empty, ssize_t mutator_rightmost_empty, - size_t mutator_region_count, size_t mutator_used); + void establish_mutator_intervals(idx_t mutator_leftmost, idx_t mutator_rightmost, + idx_t mutator_leftmost_empty, idx_t mutator_rightmost_empty, + size_t total_mutator_regions, size_t empty_mutator_regions, + size_t mutator_region_count, size_t mutator_used, size_t mutator_humongous_words_waste); // Set the OldCollector intervals, usage, and capacity according to arguments. We use this at the end of rebuild_free_set() // to avoid the overhead of making many redundant incremental adjustments to the mutator intervals as the free set is being // rebuilt. - void establish_old_collector_intervals(ssize_t old_collector_leftmost, ssize_t old_collector_rightmost, - ssize_t old_collector_leftmost_empty, ssize_t old_collector_rightmost_empty, - size_t old_collector_region_count, size_t old_collector_used); + void establish_old_collector_intervals(idx_t old_collector_leftmost, idx_t old_collector_rightmost, + idx_t old_collector_leftmost_empty, idx_t old_collector_rightmost_empty, + size_t total_old_collector_region_count, size_t old_collector_empty, + size_t old_collector_regions, size_t old_collector_used, + size_t old_collector_humongous_words_waste); + + void establish_interval(ShenandoahFreeSetPartitionId partition, idx_t low_idx, idx_t high_idx, + idx_t low_empty_idx, idx_t high_empty_idx); + + // Shrink the intervals associated with partition when region idx is removed from this free set + inline void shrink_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, idx_t idx); + + // Shrink the intervals associated with partition when regions low_idx through high_idx inclusive are removed from this free set + void shrink_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId partition, + idx_t low_idx, idx_t high_idx, size_t num_regions); + + void expand_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, idx_t idx, size_t capacity); + void expand_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId partition, + idx_t low_idx, idx_t high_idx, + idx_t low_empty_idx, idx_t high_empty_idx); // Retire region idx from within partition, , leaving its capacity and used as part of the original free partition's totals. // Requires that region idx is in in the Mutator or Collector partitions. Hereafter, identifies this region as NotFree. // Any remnant of available memory at the time of retirement is added to the original partition's total of used bytes. - void retire_from_partition(ShenandoahFreeSetPartitionId p, ssize_t idx, size_t used_bytes); + // Return the number of waste bytes (if any). + size_t retire_from_partition(ShenandoahFreeSetPartitionId p, idx_t idx, size_t used_bytes); // Retire all regions between low_idx and high_idx inclusive from within partition. Requires that each region idx is // in the same Mutator or Collector partition. Hereafter, identifies each region as NotFree. Assumes that each region // is now considered fully used, since the region is presumably used to represent a humongous object. - void retire_range_from_partition(ShenandoahFreeSetPartitionId partition, ssize_t low_idx, ssize_t high_idx); + void retire_range_from_partition(ShenandoahFreeSetPartitionId partition, idx_t low_idx, idx_t high_idx); + + void unretire_to_partition(ShenandoahHeapRegion* region, ShenandoahFreeSetPartitionId which_partition); // Place region idx into free set which_partition. Requires that idx is currently NotFree. - void make_free(ssize_t idx, ShenandoahFreeSetPartitionId which_partition, size_t region_capacity); + void make_free(idx_t idx, ShenandoahFreeSetPartitionId which_partition, size_t region_capacity); - // Place region idx into free partition new_partition, adjusting used and capacity totals for the original and new partition - // given that available bytes can still be allocated within this region. Requires that idx is currently not NotFree. - void move_from_partition_to_partition(ssize_t idx, ShenandoahFreeSetPartitionId orig_partition, + // Place region idx into free partition new_partition, not adjusting used and capacity totals for the original and new partition. + // available represents bytes that can still be allocated within this region. Requires that idx is currently not NotFree. + size_t move_from_partition_to_partition_with_deferred_accounting(idx_t idx, ShenandoahFreeSetPartitionId orig_partition, + ShenandoahFreeSetPartitionId new_partition, size_t available); + + // Place region idx into free partition new_partition, adjusting used and capacity totals for the original and new partition. + // available represents bytes that can still be allocated within this region. Requires that idx is currently not NotFree. + void move_from_partition_to_partition(idx_t idx, ShenandoahFreeSetPartitionId orig_partition, ShenandoahFreeSetPartitionId new_partition, size_t available); - const char* partition_membership_name(ssize_t idx) const; + void transfer_used_capacity_from_to(ShenandoahFreeSetPartitionId from_partition, ShenandoahFreeSetPartitionId to_partition, + size_t regions); + + const char* partition_membership_name(idx_t idx) const; // Return the index of the next available region >= start_index, or maximum_regions if not found. - inline ssize_t find_index_of_next_available_region(ShenandoahFreeSetPartitionId which_partition, ssize_t start_index) const; + inline idx_t find_index_of_next_available_region(ShenandoahFreeSetPartitionId which_partition, + idx_t start_index) const; // Return the index of the previous available region <= last_index, or -1 if not found. - inline ssize_t find_index_of_previous_available_region(ShenandoahFreeSetPartitionId which_partition, ssize_t last_index) const; + inline idx_t find_index_of_previous_available_region(ShenandoahFreeSetPartitionId which_partition, + idx_t last_index) const; // Return the index of the next available cluster of cluster_size regions >= start_index, or maximum_regions if not found. - inline ssize_t find_index_of_next_available_cluster_of_regions(ShenandoahFreeSetPartitionId which_partition, - ssize_t start_index, size_t cluster_size) const; + inline idx_t find_index_of_next_available_cluster_of_regions(ShenandoahFreeSetPartitionId which_partition, + idx_t start_index, size_t cluster_size) const; // Return the index of the previous available cluster of cluster_size regions <= last_index, or -1 if not found. - inline ssize_t find_index_of_previous_available_cluster_of_regions(ShenandoahFreeSetPartitionId which_partition, - ssize_t last_index, size_t cluster_size) const; + inline idx_t find_index_of_previous_available_cluster_of_regions(ShenandoahFreeSetPartitionId which_partition, + idx_t last_index, size_t cluster_size) const; - inline bool in_free_set(ShenandoahFreeSetPartitionId which_partition, ssize_t idx) const { + inline bool in_free_set(ShenandoahFreeSetPartitionId which_partition, idx_t idx) const { return _membership[int(which_partition)].is_set(idx); } // Returns the ShenandoahFreeSetPartitionId affiliation of region idx, NotFree if this region is not currently in any partition. // This does not enforce that free_set membership implies allocation capacity. - inline ShenandoahFreeSetPartitionId membership(ssize_t idx) const; + inline ShenandoahFreeSetPartitionId membership(idx_t idx) const { + assert (idx < _max, "index is sane: %zu < %zu", idx, _max); + ShenandoahFreeSetPartitionId result = ShenandoahFreeSetPartitionId::NotFree; + for (uint partition_id = 0; partition_id < UIntNumPartitions; partition_id++) { + if (_membership[partition_id].is_set(idx)) { + assert(result == ShenandoahFreeSetPartitionId::NotFree, "Region should reside in only one partition"); + result = (ShenandoahFreeSetPartitionId) partition_id; + } + } + return result; + } #ifdef ASSERT // Returns true iff region idx's membership is which_partition. If which_partition represents a free set, asserts // that the region has allocation capacity. - inline bool partition_id_matches(ssize_t idx, ShenandoahFreeSetPartitionId which_partition) const; + inline bool partition_id_matches(idx_t idx, ShenandoahFreeSetPartitionId which_partition) const; #endif - inline size_t max_regions() const { return _max; } - inline size_t region_size_bytes() const { return _region_size_bytes; }; // The following four methods return the left-most and right-most bounds on ranges of regions representing @@ -192,14 +275,54 @@ class ShenandoahRegionPartitions { // leftmost() and leftmost_empty() return _max, rightmost() and rightmost_empty() return 0 // otherwise, expect the following: // 0 <= leftmost <= leftmost_empty <= rightmost_empty <= rightmost < _max - inline ssize_t leftmost(ShenandoahFreeSetPartitionId which_partition) const; - inline ssize_t rightmost(ShenandoahFreeSetPartitionId which_partition) const; - ssize_t leftmost_empty(ShenandoahFreeSetPartitionId which_partition); - ssize_t rightmost_empty(ShenandoahFreeSetPartitionId which_partition); + inline idx_t leftmost(ShenandoahFreeSetPartitionId which_partition) const; + inline idx_t rightmost(ShenandoahFreeSetPartitionId which_partition) const; + idx_t leftmost_empty(ShenandoahFreeSetPartitionId which_partition); + idx_t rightmost_empty(ShenandoahFreeSetPartitionId which_partition); inline bool is_empty(ShenandoahFreeSetPartitionId which_partition) const; + inline void increase_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions); + inline void decrease_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions); + inline size_t get_region_counts(ShenandoahFreeSetPartitionId which_partition) { + assert (which_partition < NumPartitions, "selected free set must be valid"); + return _region_counts[int(which_partition)]; + } + + inline void increase_empty_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions); + inline void decrease_empty_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions); + inline size_t get_empty_region_counts(ShenandoahFreeSetPartitionId which_partition) { + assert (which_partition < NumPartitions, "selected free set must be valid"); + return _empty_region_counts[int(which_partition)]; + } + + inline void increase_capacity(ShenandoahFreeSetPartitionId which_partition, size_t bytes); + inline void decrease_capacity(ShenandoahFreeSetPartitionId which_partition, size_t bytes); + inline size_t get_capacity(ShenandoahFreeSetPartitionId which_partition) { + assert (which_partition < NumPartitions, "Partition must be valid"); + return _capacity[int(which_partition)]; + } + + inline void increase_available(ShenandoahFreeSetPartitionId which_partition, size_t bytes); + inline void decrease_available(ShenandoahFreeSetPartitionId which_partition, size_t bytes); + inline size_t get_available(ShenandoahFreeSetPartitionId which_partition); + inline void increase_used(ShenandoahFreeSetPartitionId which_partition, size_t bytes); + inline void decrease_used(ShenandoahFreeSetPartitionId which_partition, size_t bytes); + inline size_t get_used(ShenandoahFreeSetPartitionId which_partition) { + assert (which_partition < NumPartitions, "Partition must be valid"); + return _used[int(which_partition)]; + } + + inline void increase_humongous_waste(ShenandoahFreeSetPartitionId which_partition, size_t bytes); + inline void decrease_humongous_waste(ShenandoahFreeSetPartitionId which_partition, size_t bytes) { + shenandoah_assert_heaplocked(); + assert (which_partition < NumPartitions, "Partition must be valid"); + assert(_humongous_waste[int(which_partition)] >= bytes, "Cannot decrease waste beyond what is there"); + _humongous_waste[int(which_partition)] -= bytes; + } + + inline size_t get_humongous_waste(ShenandoahFreeSetPartitionId which_partition); inline void set_bias_from_left_to_right(ShenandoahFreeSetPartitionId which_partition, bool value) { assert (which_partition < NumPartitions, "selected free set must be valid"); @@ -227,10 +350,17 @@ class ShenandoahRegionPartitions { assert(_available[int(which_partition)] == _capacity[int(which_partition)] - _used[int(which_partition)], "Expect available (%zu) equals capacity (%zu) - used (%zu) for partition %s", _available[int(which_partition)], _capacity[int(which_partition)], _used[int(which_partition)], - partition_membership_name(ssize_t(which_partition))); + partition_membership_name(idx_t(which_partition))); return _available[int(which_partition)]; } + // Returns bytes of humongous waste + inline size_t humongous_waste(ShenandoahFreeSetPartitionId which_partition) const { + assert (which_partition < NumPartitions, "selected free set must be valid"); + // This may be called with or without the global heap lock. Changes to _humongous_waste[] are always made with heap lock. + return _humongous_waste[int(which_partition)]; + } + // Return available_in assuming caller does not hold the heap lock. In production builds, available is // returned without acquiring the lock. In debug builds, the global heap lock is acquired in order to // enforce a consistency assert. @@ -243,17 +373,12 @@ class ShenandoahRegionPartitions { (_available[int(which_partition)] == _capacity[int(which_partition)] - _used[int(which_partition)]), "Expect available (%zu) equals capacity (%zu) - used (%zu) for partition %s", _available[int(which_partition)], _capacity[int(which_partition)], _used[int(which_partition)], - partition_membership_name(ssize_t(which_partition))); + partition_membership_name(idx_t(which_partition))); #endif return _available[int(which_partition)]; } - inline void set_capacity_of(ShenandoahFreeSetPartitionId which_partition, size_t value) { - shenandoah_assert_heaplocked(); - assert (which_partition < NumPartitions, "selected free set must be valid"); - _capacity[int(which_partition)] = value; - _available[int(which_partition)] = value - _used[int(which_partition)]; - } + inline void set_capacity_of(ShenandoahFreeSetPartitionId which_partition, size_t value); inline void set_used_by(ShenandoahFreeSetPartitionId which_partition, size_t value) { shenandoah_assert_heaplocked(); @@ -284,7 +409,7 @@ class ShenandoahRegionPartitions { // idx >= leftmost && // idx <= rightmost // } - void assert_bounds() NOT_DEBUG_RETURN; + void assert_bounds(bool validate_totals) NOT_DEBUG_RETURN; }; // Publicly, ShenandoahFreeSet represents memory that is available to mutator threads. The public capacity(), used(), @@ -312,10 +437,13 @@ class ShenandoahRegionPartitions { // during the next GC pass. class ShenandoahFreeSet : public CHeapObj { +using idx_t = ShenandoahSimpleBitMap::idx_t; private: ShenandoahHeap* const _heap; ShenandoahRegionPartitions _partitions; + size_t _total_humongous_waste; + HeapWord* allocate_aligned_plab(size_t size, ShenandoahAllocRequest& req, ShenandoahHeapRegion* r); // Return the address of memory allocated, setting in_new_region to true iff the allocation is taken @@ -330,6 +458,105 @@ class ShenandoahFreeSet : public CHeapObj { const ssize_t INITIAL_ALLOC_BIAS_WEIGHT = 256; + // bytes used by young + size_t _total_young_used; + template + inline void recompute_total_young_used() { + if (UsedByMutatorChanged || UsedByCollectorChanged) { + shenandoah_assert_heaplocked(); + _total_young_used = (_partitions.used_by(ShenandoahFreeSetPartitionId::Mutator) + + _partitions.used_by(ShenandoahFreeSetPartitionId::Collector)); + } + } + + // bytes used by old + size_t _total_old_used; + template + inline void recompute_total_old_used() { + if (UsedByOldCollectorChanged) { + shenandoah_assert_heaplocked(); + _total_old_used =_partitions.used_by(ShenandoahFreeSetPartitionId::OldCollector); + } + } + +public: + // We make this public so that native code can see its value + // bytes used by global + size_t _total_global_used; +private: + // Prerequisite: _total_young_used and _total_old_used are valid + template + inline void recompute_total_global_used() { + if (UsedByMutatorChanged || UsedByCollectorChanged || UsedByOldCollectorChanged) { + shenandoah_assert_heaplocked(); + _total_global_used = _total_young_used + _total_old_used; + } + } + + template + inline void recompute_total_used() { + recompute_total_young_used(); + recompute_total_old_used(); + recompute_total_global_used(); + } + + size_t _young_affiliated_regions; + size_t _old_affiliated_regions; + size_t _global_affiliated_regions; + + size_t _young_unaffiliated_regions; + size_t _global_unaffiliated_regions; + + size_t _total_young_regions; + size_t _total_global_regions; + + size_t _mutator_bytes_allocated_since_gc_start; + + // If only affiliation changes are promote-in-place and generation sizes have not changed, + // we have AffiliatedChangesAreGlobalNeutral + // If only affiliation changes are non-empty regions moved from Mutator to Collector and young size has not changed, + // we have AffiliatedChangesAreYoungNeutral + // If only unaffiliated changes are empty regions from Mutator to/from Collector, we have UnaffiliatedChangesAreYoungNeutral + template + inline void recompute_total_affiliated() { + shenandoah_assert_heaplocked(); + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + if (!UnaffiliatedChangesAreYoungNeutral && (MutatorEmptiesChanged || CollectorEmptiesChanged)) { + _young_unaffiliated_regions = (_partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::Mutator) + + _partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::Collector)); + } + if (!AffiliatedChangesAreYoungNeutral && + (MutatorSizeChanged || CollectorSizeChanged || MutatorEmptiesChanged || CollectorEmptiesChanged)) { + _young_affiliated_regions = ((_partitions.get_capacity(ShenandoahFreeSetPartitionId::Mutator) + + _partitions.get_capacity(ShenandoahFreeSetPartitionId::Collector)) / region_size_bytes - + _young_unaffiliated_regions); + } + if (OldCollectorSizeChanged || OldCollectorEmptiesChanged) { + _old_affiliated_regions = (_partitions.get_capacity(ShenandoahFreeSetPartitionId::OldCollector) / region_size_bytes - + _partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::OldCollector)); + } + if (!AffiliatedChangesAreGlobalNeutral && + (MutatorEmptiesChanged || CollectorEmptiesChanged || OldCollectorEmptiesChanged)) { + _global_unaffiliated_regions = + _young_unaffiliated_regions + _partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::OldCollector); + } + if (!AffiliatedChangesAreGlobalNeutral && + (MutatorSizeChanged || CollectorSizeChanged || MutatorEmptiesChanged || CollectorEmptiesChanged || + OldCollectorSizeChanged || OldCollectorEmptiesChanged)) { + _global_affiliated_regions = _young_affiliated_regions + _old_affiliated_regions; + } +#ifdef ASSERT + if (ShenandoahHeap::heap()->mode()->is_generational()) { + assert(_young_affiliated_regions * ShenandoahHeapRegion::region_size_bytes() >= _total_young_used, "sanity"); + assert(_old_affiliated_regions * ShenandoahHeapRegion::region_size_bytes() >= _total_old_used, "sanity"); + } + assert(_global_affiliated_regions * ShenandoahHeapRegion::region_size_bytes() >= _total_global_used, "sanity"); +#endif + } + // Increases used memory for the partition if the allocation is successful. `in_new_region` will be set // if this is the first allocation in the region. HeapWord* try_allocate_in(ShenandoahHeapRegion* region, ShenandoahAllocRequest& req, bool& in_new_region); @@ -347,6 +574,8 @@ class ShenandoahFreeSet : public CHeapObj { // Precondition: ShenandoahHeapRegion::requires_humongous(req.size()) HeapWord* allocate_contiguous(ShenandoahAllocRequest& req, bool is_humongous); + bool transfer_one_region_from_mutator_to_old_collector(size_t idx, size_t alloc_capacity); + // Change region r from the Mutator partition to the GC's Collector or OldCollector partition. This requires that the // region is entirely empty. // @@ -374,7 +603,8 @@ class ShenandoahFreeSet : public CHeapObj { // Search for allocation in region with same affiliation as request, using given iterator. template - HeapWord* allocate_with_affiliation(Iter& iterator, ShenandoahAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region); + HeapWord* allocate_with_affiliation(Iter& iterator, ShenandoahAffiliation affiliation, + ShenandoahAllocRequest& req, bool& in_new_region); // Return true if the respective generation for this request has free regions. bool can_allocate_in_new_region(const ShenandoahAllocRequest& req); @@ -392,6 +622,10 @@ class ShenandoahFreeSet : public CHeapObj { inline bool has_alloc_capacity(ShenandoahHeapRegion *r) const; + void transfer_empty_regions_from_to(ShenandoahFreeSetPartitionId source_partition, + ShenandoahFreeSetPartitionId dest_partition, + size_t num_regions); + size_t transfer_empty_regions_from_collector_set_to_mutator_set(ShenandoahFreeSetPartitionId which_collector, size_t max_xfer_regions, size_t& bytes_transferred); @@ -399,12 +633,8 @@ class ShenandoahFreeSet : public CHeapObj { size_t max_xfer_regions, size_t& bytes_transferred); - // Determine whether we prefer to allocate from left to right or from right to left within the OldCollector free-set. void establish_old_collector_alloc_bias(); - - // Set max_capacity for young and old generations - void establish_generation_sizes(size_t young_region_count, size_t old_region_count); size_t get_usable_free_words(size_t free_bytes) const; // log status, assuming lock has already been acquired by the caller. @@ -415,10 +645,82 @@ class ShenandoahFreeSet : public CHeapObj { ShenandoahFreeSet(ShenandoahHeap* heap, size_t max_regions); + inline size_t max_regions() const { return _partitions.max(); } + ShenandoahFreeSetPartitionId membership(size_t index) const { return _partitions.membership(index); } + inline void shrink_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId partition, + idx_t low_idx, idx_t high_idx, size_t num_regions) { + return _partitions.shrink_interval_if_range_modifies_either_boundary(partition, low_idx, high_idx, num_regions); + } + + void reset_bytes_allocated_since_gc_start(size_t initial_bytes_allocated); + + void increase_bytes_allocated(size_t bytes); + + inline size_t get_bytes_allocated_since_gc_start() const { + return _mutator_bytes_allocated_since_gc_start; + } + // Public because ShenandoahRegionPartitions assertions require access. inline size_t alloc_capacity(ShenandoahHeapRegion *r) const; inline size_t alloc_capacity(size_t idx) const; + // Return bytes used by old + inline size_t old_used() { + return _total_old_used; + } + + ShenandoahFreeSetPartitionId prepare_to_promote_in_place(size_t idx, size_t bytes); + void account_for_pip_regions(size_t mutator_regions, size_t mutator_bytes, size_t collector_regions, size_t collector_bytes); + + // This is used for unit testing. Not for preoduction. Invokes exit() if old cannot be resized. + void resize_old_collector_capacity(size_t desired_regions); + + // Return bytes used by young + inline size_t young_used() { + return _total_young_used; + } + + // Return bytes used by global + inline size_t global_used() { + return _total_global_used; + } + + size_t global_unaffiliated_regions() { + return _global_unaffiliated_regions; + } + + size_t young_unaffiliated_regions() { + return _young_unaffiliated_regions; + } + + size_t old_unaffiliated_regions() { + return _partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::OldCollector); + } + + size_t young_affiliated_regions() { + return _young_affiliated_regions; + } + + size_t old_affiliated_regions() { + return _old_affiliated_regions; + } + + size_t global_affiliated_regions() { + return _global_affiliated_regions; + } + + size_t total_young_regions() { + return _total_young_regions; + } + + size_t total_old_regions() { + return _partitions.get_capacity(ShenandoahFreeSetPartitionId::OldCollector) / ShenandoahHeapRegion::region_size_bytes(); + } + + size_t total_global_regions() { + return _total_global_regions; + } + void clear(); // Examine the existing free set representation, capturing the current state into var arguments: @@ -464,6 +766,8 @@ class ShenandoahFreeSet : public CHeapObj { // for evacuation, invoke this to make regions available for mutator allocations. void move_regions_from_collector_to_mutator(size_t cset_regions); + void transfer_humongous_regions_from_mutator_to_old_collector(size_t xfer_regions, size_t humongous_waste_words); + void recycle_trash(); // Acquire heap lock and log status, assuming heap lock is not acquired by the caller. @@ -482,6 +786,12 @@ class ShenandoahFreeSet : public CHeapObj { inline size_t used() const { return _partitions.used_by(ShenandoahFreeSetPartitionId::Mutator); } inline size_t available() const { return _partitions.available_in_not_locked(ShenandoahFreeSetPartitionId::Mutator); } + inline size_t total_humongous_waste() const { return _total_humongous_waste; } + inline size_t humongous_waste_in_mutator() const { return _partitions.humongous_waste(ShenandoahFreeSetPartitionId::Mutator); } + inline size_t humongous_waste_in_old() const { return _partitions.humongous_waste(ShenandoahFreeSetPartitionId::OldCollector); } + + void decrease_humongous_waste_for_regular_bypass(ShenandoahHeapRegion* r, size_t waste); + HeapWord* allocate(ShenandoahAllocRequest& req, bool& in_new_region); /* @@ -539,7 +849,8 @@ class ShenandoahFreeSet : public CHeapObj { // Ensure that Collector has at least to_reserve bytes of available memory, and OldCollector has at least old_reserve // bytes of available memory. On input, old_region_count holds the number of regions already present in the // OldCollector partition. Upon return, old_region_count holds the updated number of regions in the OldCollector partition. - void reserve_regions(size_t to_reserve, size_t old_reserve, size_t &old_region_count); + void reserve_regions(size_t to_reserve, size_t old_reserve, size_t &old_region_count, + size_t &young_used_regions, size_t &old_used_regions, size_t &young_used_bytes, size_t &old_used_bytes); // Reserve space for evacuations, with regions reserved for old evacuations placed to the right // of regions reserved of young evacuations. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 78218f5e403f8..027d7e022686e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -237,7 +237,6 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { worker_slices[i] = new ShenandoahHeapRegionSet(); } - ShenandoahGenerationalHeap::TransferResult result; { // The rest of code performs region moves, where region status is undefined // until all phases run together. @@ -251,14 +250,7 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { phase4_compact_objects(worker_slices); - result = phase5_epilog(); - } - if (heap->mode()->is_generational()) { - LogTarget(Info, gc, ergo) lt; - if (lt.is_enabled()) { - LogStream ls(lt); - result.print_on("Full GC", &ls); - } + phase5_epilog(); } // Resize metaspace @@ -984,23 +976,6 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure { r->set_live_data(live); r->reset_alloc_metadata(); } - - void update_generation_usage() { - if (_is_generational) { - _heap->old_generation()->establish_usage(_old_regions, _old_usage, _old_humongous_waste); - _heap->young_generation()->establish_usage(_young_regions, _young_usage, _young_humongous_waste); - } else { - assert(_old_regions == 0, "Old regions only expected in generational mode"); - assert(_old_usage == 0, "Old usage only expected in generational mode"); - assert(_old_humongous_waste == 0, "Old humongous waste only expected in generational mode"); - } - - // In generational mode, global usage should be the sum of young and old. This is also true - // for non-generational modes except that there are no old regions. - _heap->global_generation()->establish_usage(_old_regions + _young_regions, - _old_usage + _young_usage, - _old_humongous_waste + _young_humongous_waste); - } }; void ShenandoahFullGC::compact_humongous_objects() { @@ -1120,10 +1095,9 @@ void ShenandoahFullGC::phase4_compact_objects(ShenandoahHeapRegionSet** worker_s } } -ShenandoahGenerationalHeap::TransferResult ShenandoahFullGC::phase5_epilog() { +void ShenandoahFullGC::phase5_epilog() { GCTraceTime(Info, gc, phases) time("Phase 5: Full GC epilog", _gc_timer); ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahGenerationalHeap::TransferResult result; // Reset complete bitmap. We're about to reset the complete-top-at-mark-start pointer // and must ensure the bitmap is in sync. @@ -1138,12 +1112,6 @@ ShenandoahGenerationalHeap::TransferResult ShenandoahFullGC::phase5_epilog() { ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_rebuild); ShenandoahPostCompactClosure post_compact; heap->heap_region_iterate(&post_compact); - post_compact.update_generation_usage(); - - if (heap->mode()->is_generational()) { - ShenandoahGenerationalFullGC::balance_generations_after_gc(heap); - } - heap->collection_set()->clear(); size_t young_cset_regions, old_cset_regions; size_t first_old, last_old, num_old; @@ -1166,11 +1134,7 @@ ShenandoahGenerationalHeap::TransferResult ShenandoahFullGC::phase5_epilog() { _preserved_marks->restore(heap->workers()); _preserved_marks->reclaim(); - // We defer generation resizing actions until after cset regions have been recycled. We do this even following an - // abbreviated cycle. if (heap->mode()->is_generational()) { - result = ShenandoahGenerationalFullGC::balance_generations_after_rebuilding_free_set(); ShenandoahGenerationalFullGC::rebuild_remembered_set(heap); } - return result; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp index 8b8244f2ce3ab..c69c8cec8fcb2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp @@ -82,8 +82,7 @@ class ShenandoahFullGC : public ShenandoahGC { void phase2_calculate_target_addresses(ShenandoahHeapRegionSet** worker_slices); void phase3_update_references(); void phase4_compact_objects(ShenandoahHeapRegionSet** worker_slices); - ShenandoahGenerationalHeap::TransferResult phase5_epilog(); - + void phase5_epilog(); void distribute_slices(ShenandoahHeapRegionSet** worker_slices); void calculate_target_humongous_objects(); void compact_humongous_objects(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index f74dffe50b049..ea42136561493 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -39,6 +39,8 @@ #include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "utilities/quickSort.hpp" +using idx_t = ShenandoahSimpleBitMap::idx_t; + template class ShenandoahResetBitmapClosure final : public ShenandoahHeapRegionClosure { private: @@ -147,19 +149,8 @@ ShenandoahHeuristics* ShenandoahGeneration::initialize_heuristics(ShenandoahMode return _heuristics; } -size_t ShenandoahGeneration::bytes_allocated_since_gc_start() const { - return AtomicAccess::load(&_bytes_allocated_since_gc_start); -} - -void ShenandoahGeneration::reset_bytes_allocated_since_gc_start(size_t initial_bytes_allocated) { - AtomicAccess::store(&_bytes_allocated_since_gc_start, initial_bytes_allocated); -} - -void ShenandoahGeneration::increase_allocated(size_t bytes) { - AtomicAccess::add(&_bytes_allocated_since_gc_start, bytes, memory_order_relaxed); -} - void ShenandoahGeneration::set_evacuation_reserve(size_t new_val) { + shenandoah_assert_heaplocked(); _evacuation_reserve = new_val; } @@ -271,7 +262,7 @@ void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* const heap // maximum_young_evacuation_reserve is upper bound on memory to be evacuated out of young const size_t maximum_young_evacuation_reserve = (young_generation->max_capacity() * ShenandoahEvacReserve) / 100; - const size_t young_evacuation_reserve = MIN2(maximum_young_evacuation_reserve, young_generation->available_with_reserve()); + size_t young_evacuation_reserve = MIN2(maximum_young_evacuation_reserve, young_generation->available_with_reserve()); // maximum_old_evacuation_reserve is an upper bound on memory evacuated from old and evacuated to old (promoted), // clamped by the old generation space available. @@ -351,6 +342,11 @@ void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* const heap const size_t consumed_by_advance_promotion = select_aged_regions(old_promo_reserve); assert(consumed_by_advance_promotion <= maximum_old_evacuation_reserve, "Cannot promote more than available old-gen memory"); + // If any regions have been selected for promotion in place, this has the effect of decreasing available within mutator + // and collector partitions, due to padding of remnant memory within each promoted in place region. This will affect + // young_evacuation_reserve but not old_evacuation_reserve or consumed_by_advance_promotion. So recompute. + young_evacuation_reserve = MIN2(young_evacuation_reserve, young_generation->available_with_reserve()); + // Note that unused old_promo_reserve might not be entirely consumed_by_advance_promotion. Do not transfer this // to old_evacuation_reserve because this memory is likely very fragmented, and we do not want to increase the likelihood // of old evacuation failure. @@ -435,7 +431,9 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* const heap, size_t excess_old = old_available - old_consumed; size_t unaffiliated_old_regions = old_generation->free_unaffiliated_regions(); size_t unaffiliated_old = unaffiliated_old_regions * region_size_bytes; - assert(old_available >= unaffiliated_old, "Unaffiliated old is a subset of old available"); + assert(old_available >= unaffiliated_old, + "Unaffiliated old (%zu is %zu * %zu) is a subset of old available (%zu)", + unaffiliated_old, unaffiliated_old_regions, region_size_bytes, old_available); // Make sure old_evac_committed is unaffiliated if (old_evacuated_committed > 0) { @@ -465,15 +463,10 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* const heap, size_t excess_regions = excess_old / region_size_bytes; regions_to_xfer = MIN2(excess_regions, unaffiliated_old_regions); } - if (regions_to_xfer > 0) { - bool result = ShenandoahGenerationalHeap::cast(heap)->generation_sizer()->transfer_to_young(regions_to_xfer); - assert(excess_old >= regions_to_xfer * region_size_bytes, - "Cannot transfer (%zu, %zu) more than excess old (%zu)", - regions_to_xfer, region_size_bytes, excess_old); excess_old -= regions_to_xfer * region_size_bytes; - log_debug(gc, ergo)("%s transferred %zu excess regions to young before start of evacuation", - result? "Successfully": "Unsuccessfully", regions_to_xfer); + log_debug(gc, ergo)("Before start of evacuation, total_promotion reserve is young_advance_promoted_reserve: %zu " + "plus excess: old: %zu", young_advance_promoted_reserve_used, excess_old); } // Add in the excess_old memory to hold unanticipated promotions, if any. If there are more unanticipated @@ -531,6 +524,8 @@ size_t ShenandoahGeneration::select_aged_regions(const size_t old_promotion_rese assert_no_in_place_promotions(); auto const heap = ShenandoahGenerationalHeap::heap(); + ShenandoahYoungGeneration* young_gen = heap->young_generation(); + ShenandoahFreeSet* free_set = heap->free_set(); bool* const candidate_regions_for_promotion_by_copy = heap->collection_set()->preselected_regions(); ShenandoahMarkingContext* const ctx = heap->marking_context(); @@ -547,12 +542,28 @@ size_t ShenandoahGeneration::select_aged_regions(const size_t old_promotion_rese // Sort the promotion-eligible regions in order of increasing live-data-bytes so that we can first reclaim regions that require // less evacuation effort. This prioritizes garbage first, expanding the allocation pool early before we reclaim regions that // have more live data. - const size_t num_regions = heap->num_regions(); + const idx_t num_regions = heap->num_regions(); ResourceMark rm; AgedRegionData* sorted_regions = NEW_RESOURCE_ARRAY(AgedRegionData, num_regions); - for (size_t i = 0; i < num_regions; i++) { + ShenandoahFreeSet* freeset = heap->free_set(); + + // Any region that is to be promoted in place needs to be retired from its Collector or Mutator partition. + idx_t pip_low_collector_idx = freeset->max_regions(); + idx_t pip_high_collector_idx = -1; + idx_t pip_low_mutator_idx = freeset->max_regions(); + idx_t pip_high_mutator_idx = -1; + size_t collector_regions_to_pip = 0; + size_t mutator_regions_to_pip = 0; + + size_t pip_mutator_regions = 0; + size_t pip_collector_regions = 0; + size_t pip_mutator_bytes = 0; + size_t pip_collector_bytes = 0; + + size_t min_remnant_size = PLAB::min_size() * HeapWordSize; + for (idx_t i = 0; i < num_regions; i++) { ShenandoahHeapRegion* const r = heap->get_region(i); if (r->is_empty() || !r->has_live() || !r->is_young() || !r->is_regular()) { // skip over regions that aren't regular young with some live data @@ -569,18 +580,54 @@ size_t ShenandoahGeneration::select_aged_regions(const size_t old_promotion_rese // we use this field to indicate that this region should be promoted in place during the evacuation // phase. r->save_top_before_promote(); - - size_t remnant_size = r->free() / HeapWordSize; - if (remnant_size > ShenandoahHeap::min_fill_size()) { - ShenandoahHeap::fill_with_object(original_top, remnant_size); + size_t remnant_bytes = r->free(); + size_t remnant_words = remnant_bytes / HeapWordSize; + assert(ShenandoahHeap::min_fill_size() <= PLAB::min_size(), "Implementation makes invalid assumptions"); + if (remnant_words >= ShenandoahHeap::min_fill_size()) { + ShenandoahHeap::fill_with_object(original_top, remnant_words); // Fill the remnant memory within this region to assure no allocations prior to promote in place. Otherwise, // newly allocated objects will not be parsable when promote in place tries to register them. Furthermore, any // new allocations would not necessarily be eligible for promotion. This addresses both issues. r->set_top(r->end()); - promote_in_place_pad += remnant_size * HeapWordSize; + // The region r is either in the Mutator or Collector partition if remnant_words > heap()->plab_min_size. + // Otherwise, the region is in the NotFree partition. + ShenandoahFreeSetPartitionId p = free_set->membership(i); + if (p == ShenandoahFreeSetPartitionId::Mutator) { + mutator_regions_to_pip++; + if (i < pip_low_mutator_idx) { + pip_low_mutator_idx = i; + } + if (i > pip_high_mutator_idx) { + pip_high_mutator_idx = i; + } + pip_mutator_regions++; + pip_mutator_bytes += remnant_bytes; + } else if (p == ShenandoahFreeSetPartitionId::Collector) { + collector_regions_to_pip++; + if (i < pip_low_collector_idx) { + pip_low_collector_idx = i; + } + if (i > pip_high_collector_idx) { + pip_high_collector_idx = i; + } + pip_collector_regions++; + pip_collector_bytes += remnant_bytes; + } else { + assert((p == ShenandoahFreeSetPartitionId::NotFree) && (remnant_words < heap->plab_min_size()), + "Should be NotFree if not in Collector or Mutator partitions"); + // In this case, the memory is already counted as used and the region has already been retired. There is + // no need for further adjustments to used. Further, the remnant memory for this region will not be + // unallocated or made available to OldCollector after pip. + remnant_bytes = 0; + } + promote_in_place_pad += remnant_bytes; + free_set->prepare_to_promote_in_place(i, remnant_bytes); } else { - // Since the remnant is so small that it cannot be filled, we don't have to worry about any accidental - // allocations occurring within this region before the region is promoted in place. + // Since the remnant is so small that this region has already been retired, we don't have to worry about any + // accidental allocations occurring within this region before the region is promoted in place. + + // This region was already not in the Collector or Mutator set, so no need to remove it. + assert(free_set->membership(i) == ShenandoahFreeSetPartitionId::NotFree, "sanity"); } } // Else, we do not promote this region (either in place or by copy) because it has received new allocations. @@ -621,7 +668,21 @@ size_t ShenandoahGeneration::select_aged_regions(const size_t old_promotion_rese // Subsequent regions may be selected if they have smaller live data. } - log_info(gc, ergo)("Promotion potential of aged regions with sufficient garbage: " PROPERFMT, PROPERFMTARGS(promo_potential)); + if (pip_mutator_regions + pip_collector_regions > 0) { + freeset->account_for_pip_regions(pip_mutator_regions, pip_mutator_bytes, pip_collector_regions, pip_collector_bytes); + } + + // Retire any regions that have been selected for promote in place + if (collector_regions_to_pip > 0) { + freeset->shrink_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId::Collector, + pip_low_collector_idx, pip_high_collector_idx, + collector_regions_to_pip); + } + if (mutator_regions_to_pip > 0) { + freeset->shrink_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId::Mutator, + pip_low_mutator_idx, pip_high_mutator_idx, + mutator_regions_to_pip); + } // Sort in increasing order according to live data bytes. Note that candidates represents the number of regions // that qualify to be promoted by evacuation. @@ -653,6 +714,8 @@ size_t ShenandoahGeneration::select_aged_regions(const size_t old_promotion_rese selected_regions, PROPERFMTARGS(selected_live), PROPERFMTARGS(old_consumed), PROPERFMTARGS(old_promotion_reserve)); } + log_info(gc, ergo)("Promotion potential of aged regions with sufficient garbage: " PROPERFMT, PROPERFMTARGS(promo_potential)); + heap->old_generation()->set_pad_for_promote_in_place(promote_in_place_pad); heap->old_generation()->set_promotion_potential(promo_potential); return old_consumed; @@ -754,9 +817,15 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { // We are preparing for evacuation. At this time, we ignore cset region tallies. size_t first_old, last_old, num_old; - heap->free_set()->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old); + _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old); + + if (heap->mode()->is_generational()) { + ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap(); + gen_heap->compute_old_generation_balance(young_cset_regions, old_cset_regions); + } + // Free set construction uses reserve quantities, because they are known to be valid here - heap->free_set()->finish_rebuild(young_cset_regions, old_cset_regions, num_old, true); + _free_set->finish_rebuild(young_cset_regions, old_cset_regions, num_old, true); } } @@ -800,14 +869,12 @@ void ShenandoahGeneration::cancel_marking() { } ShenandoahGeneration::ShenandoahGeneration(ShenandoahGenerationType type, - uint max_workers, - size_t max_capacity) : + uint max_workers) : _type(type), _task_queues(new ShenandoahObjToScanQueueSet(max_workers)), _ref_processor(new ShenandoahReferenceProcessor(this, MAX2(max_workers, 1U))), - _affiliated_region_count(0), _humongous_waste(0), _evacuation_reserve(0), - _used(0), _bytes_allocated_since_gc_start(0), - _max_capacity(max_capacity), + _evacuation_reserve(0), + _free_set(nullptr), _heuristics(nullptr) { _is_marking_complete.set(); @@ -826,6 +893,11 @@ ShenandoahGeneration::~ShenandoahGeneration() { delete _task_queues; } +void ShenandoahGeneration::post_initialize(ShenandoahHeap* heap) { + _free_set = heap->free_set(); + assert(_free_set != nullptr, "bad initialization order"); +} + void ShenandoahGeneration::reserve_task_queues(uint workers) { _task_queues->reserve(workers); } @@ -853,159 +925,26 @@ void ShenandoahGeneration::scan_remembered_set(bool is_concurrent) { } } -size_t ShenandoahGeneration::increment_affiliated_region_count() { - shenandoah_assert_heaplocked_or_safepoint(); - // During full gc, multiple GC worker threads may change region affiliations without a lock. No lock is enforced - // on read and write of _affiliated_region_count. At the end of full gc, a single thread overwrites the count with - // a coherent value. - return AtomicAccess::add(&_affiliated_region_count, (size_t) 1); -} - -size_t ShenandoahGeneration::decrement_affiliated_region_count() { - shenandoah_assert_heaplocked_or_safepoint(); - // During full gc, multiple GC worker threads may change region affiliations without a lock. No lock is enforced - // on read and write of _affiliated_region_count. At the end of full gc, a single thread overwrites the count with - // a coherent value. - auto affiliated_region_count = AtomicAccess::sub(&_affiliated_region_count, (size_t) 1); - assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || - (used() + _humongous_waste <= affiliated_region_count * ShenandoahHeapRegion::region_size_bytes()), - "used + humongous cannot exceed regions"); - return affiliated_region_count; -} - -size_t ShenandoahGeneration::decrement_affiliated_region_count_without_lock() { - return AtomicAccess::sub(&_affiliated_region_count, (size_t) 1); -} - -size_t ShenandoahGeneration::increase_affiliated_region_count(size_t delta) { - shenandoah_assert_heaplocked_or_safepoint(); - return AtomicAccess::add(&_affiliated_region_count, delta); -} - -size_t ShenandoahGeneration::decrease_affiliated_region_count(size_t delta) { - shenandoah_assert_heaplocked_or_safepoint(); - assert(AtomicAccess::load(&_affiliated_region_count) >= delta, "Affiliated region count cannot be negative"); - - auto const affiliated_region_count = AtomicAccess::sub(&_affiliated_region_count, delta); - assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || - (_used + _humongous_waste <= affiliated_region_count * ShenandoahHeapRegion::region_size_bytes()), - "used + humongous cannot exceed regions"); - return affiliated_region_count; -} - -void ShenandoahGeneration::establish_usage(size_t num_regions, size_t num_bytes, size_t humongous_waste) { - assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint"); - AtomicAccess::store(&_affiliated_region_count, num_regions); - AtomicAccess::store(&_used, num_bytes); - _humongous_waste = humongous_waste; -} - -void ShenandoahGeneration::increase_used(size_t bytes) { - AtomicAccess::add(&_used, bytes); -} - -void ShenandoahGeneration::increase_humongous_waste(size_t bytes) { - if (bytes > 0) { - AtomicAccess::add(&_humongous_waste, bytes); - } -} - -void ShenandoahGeneration::decrease_humongous_waste(size_t bytes) { - if (bytes > 0) { - assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || (_humongous_waste >= bytes), - "Waste (%zu) cannot be negative (after subtracting %zu)", _humongous_waste, bytes); - AtomicAccess::sub(&_humongous_waste, bytes); - } -} - -void ShenandoahGeneration::decrease_used(size_t bytes) { - assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || - (_used >= bytes), "cannot reduce bytes used by generation below zero"); - AtomicAccess::sub(&_used, bytes); -} - -size_t ShenandoahGeneration::used_regions() const { - return AtomicAccess::load(&_affiliated_region_count); -} - -size_t ShenandoahGeneration::free_unaffiliated_regions() const { - size_t result = max_capacity() / ShenandoahHeapRegion::region_size_bytes(); - auto const used_regions = this->used_regions(); - if (used_regions > result) { - result = 0; - } else { - result -= used_regions; - } - return result; -} - -size_t ShenandoahGeneration::used_regions_size() const { - return used_regions() * ShenandoahHeapRegion::region_size_bytes(); -} - size_t ShenandoahGeneration::available() const { - return available(max_capacity()); + size_t result = available(max_capacity()); + return result; } // For ShenandoahYoungGeneration, Include the young available that may have been reserved for the Collector. size_t ShenandoahGeneration::available_with_reserve() const { - return available(max_capacity()); + size_t result = available(max_capacity()); + return result; } size_t ShenandoahGeneration::soft_available() const { - return available(ShenandoahHeap::heap()->soft_max_capacity()); + size_t result = available(ShenandoahHeap::heap()->soft_max_capacity()); + return result; } size_t ShenandoahGeneration::available(size_t capacity) const { - size_t in_use = used() + get_humongous_waste(); - return in_use > capacity ? 0 : capacity - in_use; -} - -size_t ShenandoahGeneration::increase_capacity(size_t increment) { - shenandoah_assert_heaplocked_or_safepoint(); - - // We do not enforce that new capacity >= heap->max_size_for(this). The maximum generation size is treated as a rule of thumb - // which may be violated during certain transitions, such as when we are forcing transfers for the purpose of promoting regions - // in place. - assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || - (_max_capacity + increment <= ShenandoahHeap::heap()->max_capacity()), "Generation cannot be larger than heap size"); - assert(increment % ShenandoahHeapRegion::region_size_bytes() == 0, "Generation capacity must be multiple of region size"); - _max_capacity += increment; - - // This detects arithmetic wraparound on _used - assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || - (used_regions_size() >= used()), - "Affiliated regions must hold more than what is currently used"); - return _max_capacity; -} - -size_t ShenandoahGeneration::set_capacity(size_t byte_size) { - shenandoah_assert_heaplocked_or_safepoint(); - _max_capacity = byte_size; - return _max_capacity; -} - -size_t ShenandoahGeneration::decrease_capacity(size_t decrement) { - shenandoah_assert_heaplocked_or_safepoint(); - - // We do not enforce that new capacity >= heap->min_size_for(this). The minimum generation size is treated as a rule of thumb - // which may be violated during certain transitions, such as when we are forcing transfers for the purpose of promoting regions - // in place. - assert(decrement % ShenandoahHeapRegion::region_size_bytes() == 0, "Generation capacity must be multiple of region size"); - assert(_max_capacity >= decrement, "Generation capacity cannot be negative"); - - _max_capacity -= decrement; - - // This detects arithmetic wraparound on _used - assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || - (used_regions_size() >= used()), - "Affiliated regions must hold more than what is currently used"); - assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || - (_used <= _max_capacity), "Cannot use more than capacity"); - assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || - (used_regions_size() <= _max_capacity), - "Cannot use more than capacity"); - return _max_capacity; + size_t in_use = used(); + size_t result = in_use > capacity ? 0 : capacity - in_use; + return result; } void ShenandoahGeneration::record_success_concurrent(bool abbreviated) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index b926a5a0913c9..424a00f789dd1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -27,6 +27,7 @@ #include "gc/shenandoah/heuristics/shenandoahSpaceInfo.hpp" #include "gc/shenandoah/shenandoahAffiliation.hpp" +#include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahGenerationType.hpp" #include "gc/shenandoah/shenandoahLock.hpp" #include "gc/shenandoah/shenandoahMarkingContext.hpp" @@ -40,7 +41,6 @@ class ShenandoahHeuristics; class ShenandoahMode; class ShenandoahReferenceProcessor; - class ShenandoahGeneration : public CHeapObj, public ShenandoahSpaceInfo { friend class VMStructs; private: @@ -52,26 +52,11 @@ class ShenandoahGeneration : public CHeapObj, public ShenandoahSpaceInfo { ShenandoahReferenceProcessor* const _ref_processor; - volatile size_t _affiliated_region_count; - - // How much free memory is left in the last region of humongous objects. - // This is _not_ included in used, but it _is_ deducted from available, - // which gives the heuristics a more accurate view of how much memory remains - // for allocation. This figure is also included the heap status logging. - // The units are bytes. The value is only changed on a safepoint or under the - // heap lock. - size_t _humongous_waste; - // Bytes reserved within this generation to hold evacuated objects from the collection set size_t _evacuation_reserve; protected: - // Usage - - volatile size_t _used; - volatile size_t _bytes_allocated_since_gc_start; - size_t _max_capacity; - + ShenandoahFreeSet* _free_set; ShenandoahHeuristics* _heuristics; private: @@ -99,41 +84,43 @@ class ShenandoahGeneration : public CHeapObj, public ShenandoahSpaceInfo { // to false. size_t select_aged_regions(size_t old_promotion_reserve); + // Return available assuming that we can allocate no more than capacity bytes within this generation. size_t available(size_t capacity) const; public: ShenandoahGeneration(ShenandoahGenerationType type, - uint max_workers, - size_t max_capacity); + uint max_workers); ~ShenandoahGeneration(); - bool is_young() const { return _type == YOUNG; } - bool is_old() const { return _type == OLD; } - bool is_global() const { return _type == GLOBAL || _type == NON_GEN; } + inline bool is_young() const { return _type == YOUNG; } + inline bool is_old() const { return _type == OLD; } + inline bool is_global() const { return _type == GLOBAL || _type == NON_GEN; } + inline ShenandoahGenerationType type() const { return _type; } // see description in field declaration void set_evacuation_reserve(size_t new_val); size_t get_evacuation_reserve() const; void augment_evacuation_reserve(size_t increment); - inline ShenandoahGenerationType type() const { return _type; } - virtual ShenandoahHeuristics* heuristics() const { return _heuristics; } ShenandoahReferenceProcessor* ref_processor() { return _ref_processor; } virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode); - size_t max_capacity() const override { return _max_capacity; } - virtual size_t used_regions() const; - virtual size_t used_regions_size() const; - virtual size_t free_unaffiliated_regions() const; - size_t used() const override { return AtomicAccess::load(&_used); } + virtual void post_initialize(ShenandoahHeap* heap); + + virtual size_t bytes_allocated_since_gc_start() const override = 0; + virtual size_t used() const override = 0; + virtual size_t used_regions() const = 0; + virtual size_t used_regions_size() const = 0; + virtual size_t get_humongous_waste() const = 0; + virtual size_t free_unaffiliated_regions() const = 0; + virtual size_t get_affiliated_region_count() const = 0; + virtual size_t max_capacity() const override = 0; + size_t available() const override; size_t available_with_reserve() const; - size_t used_including_humongous_waste() const { - return used() + get_humongous_waste(); - } // Returns the memory available based on the _soft_ max heap capacity (soft_max_heap - used). // The soft max heap size may be adjusted lower than the max heap size to cause the trigger @@ -141,23 +128,6 @@ class ShenandoahGeneration : public CHeapObj, public ShenandoahSpaceInfo { // max heap size will cause the adaptive heuristic to run more frequent cycles. size_t soft_available() const override; - size_t bytes_allocated_since_gc_start() const override; - - // Reset the bytes allocated within this generation since the start of GC. The argument initial_bytes_allocated - // is normally zero. In the case that some memory was allocated following the last allocation rate sample that - // precedes the start of GC, the number of bytes allocated is supplied as the initial value of bytes_allocated_since_gc_start. - // We will behave as if these bytes were allocated after the start of GC. - void reset_bytes_allocated_since_gc_start(size_t initial_bytes_allocated); - void increase_allocated(size_t bytes); - - // These methods change the capacity of the generation by adding or subtracting the given number of bytes from the current - // capacity, returning the capacity of the generation following the change. - size_t increase_capacity(size_t increment); - size_t decrease_capacity(size_t decrement); - - // Set the capacity of the generation, returning the value set - size_t set_capacity(size_t byte_size); - void log_status(const char* msg) const; // Used directly by FullGC @@ -217,29 +187,6 @@ class ShenandoahGeneration : public CHeapObj, public ShenandoahSpaceInfo { // Scan remembered set at start of concurrent young-gen marking. void scan_remembered_set(bool is_concurrent); - // Return the updated value of affiliated_region_count - size_t increment_affiliated_region_count(); - - // Return the updated value of affiliated_region_count - size_t decrement_affiliated_region_count(); - // Same as decrement_affiliated_region_count, but w/o the need to hold heap lock before being called. - size_t decrement_affiliated_region_count_without_lock(); - - // Return the updated value of affiliated_region_count - size_t increase_affiliated_region_count(size_t delta); - - // Return the updated value of affiliated_region_count - size_t decrease_affiliated_region_count(size_t delta); - - void establish_usage(size_t num_regions, size_t num_bytes, size_t humongous_waste); - - void increase_used(size_t bytes); - void decrease_used(size_t bytes); - - void increase_humongous_waste(size_t bytes); - void decrease_humongous_waste(size_t bytes); - size_t get_humongous_waste() const { return _humongous_waste; } - virtual bool is_concurrent_mark_in_progress() = 0; void confirm_heuristics_mode(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.cpp deleted file mode 100644 index 17f3d2f199f1a..0000000000000 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.cpp +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. - * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "gc/shared/gc_globals.hpp" -#include "gc/shenandoah/shenandoahGeneration.hpp" -#include "gc/shenandoah/shenandoahGenerationSizer.hpp" -#include "gc/shenandoah/shenandoahHeap.inline.hpp" -#include "gc/shenandoah/shenandoahHeapRegion.hpp" -#include "gc/shenandoah/shenandoahOldGeneration.hpp" -#include "gc/shenandoah/shenandoahYoungGeneration.hpp" -#include "logging/log.hpp" -#include "runtime/globals_extension.hpp" - - -ShenandoahGenerationSizer::ShenandoahGenerationSizer() - : _sizer_kind(SizerDefaults), - _min_desired_young_regions(0), - _max_desired_young_regions(0) { - - if (FLAG_IS_CMDLINE(NewRatio)) { - if (FLAG_IS_CMDLINE(NewSize) || FLAG_IS_CMDLINE(MaxNewSize)) { - log_warning(gc, ergo)("-XX:NewSize and -XX:MaxNewSize override -XX:NewRatio"); - } else { - _sizer_kind = SizerNewRatio; - return; - } - } - - if (NewSize > MaxNewSize) { - if (FLAG_IS_CMDLINE(MaxNewSize)) { - log_warning(gc, ergo)("NewSize (%zuk) is greater than the MaxNewSize (%zuk). " - "A new max generation size of %zuk will be used.", - NewSize/K, MaxNewSize/K, NewSize/K); - } - FLAG_SET_ERGO(MaxNewSize, NewSize); - } - - if (FLAG_IS_CMDLINE(NewSize)) { - _min_desired_young_regions = MAX2(uint(NewSize / ShenandoahHeapRegion::region_size_bytes()), 1U); - if (FLAG_IS_CMDLINE(MaxNewSize)) { - _max_desired_young_regions = MAX2(uint(MaxNewSize / ShenandoahHeapRegion::region_size_bytes()), 1U); - _sizer_kind = SizerMaxAndNewSize; - } else { - _sizer_kind = SizerNewSizeOnly; - } - } else if (FLAG_IS_CMDLINE(MaxNewSize)) { - _max_desired_young_regions = MAX2(uint(MaxNewSize / ShenandoahHeapRegion::region_size_bytes()), 1U); - _sizer_kind = SizerMaxNewSizeOnly; - } -} - -size_t ShenandoahGenerationSizer::calculate_min_young_regions(size_t heap_region_count) { - size_t min_young_regions = (heap_region_count * ShenandoahMinYoungPercentage) / 100; - return MAX2(min_young_regions, (size_t) 1U); -} - -size_t ShenandoahGenerationSizer::calculate_max_young_regions(size_t heap_region_count) { - size_t max_young_regions = (heap_region_count * ShenandoahMaxYoungPercentage) / 100; - return MAX2(max_young_regions, (size_t) 1U); -} - -void ShenandoahGenerationSizer::recalculate_min_max_young_length(size_t heap_region_count) { - assert(heap_region_count > 0, "Heap must be initialized"); - - switch (_sizer_kind) { - case SizerDefaults: - _min_desired_young_regions = calculate_min_young_regions(heap_region_count); - _max_desired_young_regions = calculate_max_young_regions(heap_region_count); - break; - case SizerNewSizeOnly: - _max_desired_young_regions = calculate_max_young_regions(heap_region_count); - _max_desired_young_regions = MAX2(_min_desired_young_regions, _max_desired_young_regions); - break; - case SizerMaxNewSizeOnly: - _min_desired_young_regions = calculate_min_young_regions(heap_region_count); - _min_desired_young_regions = MIN2(_min_desired_young_regions, _max_desired_young_regions); - break; - case SizerMaxAndNewSize: - // Do nothing. Values set on the command line, don't update them at runtime. - break; - case SizerNewRatio: - _min_desired_young_regions = MAX2(uint(heap_region_count / (NewRatio + 1)), 1U); - _max_desired_young_regions = _min_desired_young_regions; - break; - default: - ShouldNotReachHere(); - } - - assert(_min_desired_young_regions <= _max_desired_young_regions, "Invalid min/max young gen size values"); -} - -void ShenandoahGenerationSizer::heap_size_changed(size_t heap_size) { - recalculate_min_max_young_length(heap_size / ShenandoahHeapRegion::region_size_bytes()); -} - -bool ShenandoahGenerationSizer::transfer_regions(ShenandoahGeneration* src, ShenandoahGeneration* dst, size_t regions) const { - const size_t bytes_to_transfer = regions * ShenandoahHeapRegion::region_size_bytes(); - - if (src->free_unaffiliated_regions() < regions) { - // Source does not have enough free regions for this transfer. The caller should have - // already capped the transfer based on available unaffiliated regions. - return false; - } - - if (dst->max_capacity() + bytes_to_transfer > max_size_for(dst)) { - // This transfer would cause the destination generation to grow above its configured maximum size. - return false; - } - - if (src->max_capacity() - bytes_to_transfer < min_size_for(src)) { - // This transfer would cause the source generation to shrink below its configured minimum size. - return false; - } - - src->decrease_capacity(bytes_to_transfer); - dst->increase_capacity(bytes_to_transfer); - const size_t new_size = dst->max_capacity(); - log_info(gc, ergo)("Transfer %zu region(s) from %s to %s, yielding increased size: " PROPERFMT, - regions, src->name(), dst->name(), PROPERFMTARGS(new_size)); - return true; -} - - -size_t ShenandoahGenerationSizer::max_size_for(ShenandoahGeneration* generation) const { - switch (generation->type()) { - case YOUNG: - return max_young_size(); - case OLD: - // On the command line, max size of OLD is specified indirectly, by setting a minimum size of young. - // OLD is what remains within the heap after YOUNG has been sized. - return ShenandoahHeap::heap()->max_capacity() - min_young_size(); - default: - ShouldNotReachHere(); - return 0; - } -} - -size_t ShenandoahGenerationSizer::min_size_for(ShenandoahGeneration* generation) const { - switch (generation->type()) { - case YOUNG: - return min_young_size(); - case OLD: - // On the command line, min size of OLD is specified indirectly, by setting a maximum size of young. - // OLD is what remains within the heap after YOUNG has been sized. - return ShenandoahHeap::heap()->max_capacity() - max_young_size(); - default: - ShouldNotReachHere(); - return 0; - } -} - - -// Returns true iff transfer is successful -bool ShenandoahGenerationSizer::transfer_to_old(size_t regions) const { - ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap(); - return transfer_regions(heap->young_generation(), heap->old_generation(), regions); -} - -// This is used when promoting humongous or highly utilized regular regions in place. It is not required in this situation -// that the transferred regions be unaffiliated. -void ShenandoahGenerationSizer::force_transfer_to_old(size_t regions) const { - ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap(); - ShenandoahGeneration* old_gen = heap->old_generation(); - ShenandoahGeneration* young_gen = heap->young_generation(); - const size_t bytes_to_transfer = regions * ShenandoahHeapRegion::region_size_bytes(); - - young_gen->decrease_capacity(bytes_to_transfer); - old_gen->increase_capacity(bytes_to_transfer); - const size_t new_size = old_gen->max_capacity(); - log_info(gc, ergo)("Forcing transfer of %zu region(s) from %s to %s, yielding increased size: " PROPERFMT, - regions, young_gen->name(), old_gen->name(), PROPERFMTARGS(new_size)); -} - - -bool ShenandoahGenerationSizer::transfer_to_young(size_t regions) const { - ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap(); - return transfer_regions(heap->old_generation(), heap->young_generation(), regions); -} - -size_t ShenandoahGenerationSizer::min_young_size() const { - return min_young_regions() * ShenandoahHeapRegion::region_size_bytes(); -} - -size_t ShenandoahGenerationSizer::max_young_size() const { - return max_young_regions() * ShenandoahHeapRegion::region_size_bytes(); -} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.hpp deleted file mode 100644 index 5752422bb7717..0000000000000 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.hpp +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONSIZER_HPP -#define SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONSIZER_HPP - -#include "utilities/globalDefinitions.hpp" - -class ShenandoahGeneration; -class ShenandoahGenerationalHeap; - -class ShenandoahGenerationSizer { -private: - enum SizerKind { - SizerDefaults, - SizerNewSizeOnly, - SizerMaxNewSizeOnly, - SizerMaxAndNewSize, - SizerNewRatio - }; - SizerKind _sizer_kind; - - size_t _min_desired_young_regions; - size_t _max_desired_young_regions; - - static size_t calculate_min_young_regions(size_t heap_region_count); - static size_t calculate_max_young_regions(size_t heap_region_count); - - // Update the given values for minimum and maximum young gen length in regions - // given the number of heap regions depending on the kind of sizing algorithm. - void recalculate_min_max_young_length(size_t heap_region_count); - - // This will attempt to transfer regions from the `src` generation to `dst` generation. - // If the transfer would violate the configured minimum size for the source or the configured - // maximum size of the destination, it will not perform the transfer and will return false. - // Returns true if the transfer is performed. - bool transfer_regions(ShenandoahGeneration* src, ShenandoahGeneration* dst, size_t regions) const; - - // Return the configured maximum size in bytes for the given generation. - size_t max_size_for(ShenandoahGeneration* generation) const; - - // Return the configured minimum size in bytes for the given generation. - size_t min_size_for(ShenandoahGeneration* generation) const; - -public: - ShenandoahGenerationSizer(); - - // Calculate the maximum length of the young gen given the number of regions - // depending on the sizing algorithm. - void heap_size_changed(size_t heap_size); - - // Minimum size of young generation in bytes as multiple of region size. - size_t min_young_size() const; - size_t min_young_regions() const { - return _min_desired_young_regions; - } - - // Maximum size of young generation in bytes as multiple of region size. - size_t max_young_size() const; - size_t max_young_regions() const { - return _max_desired_young_regions; - } - - // True if transfer succeeds, else false. See transfer_regions. - bool transfer_to_young(size_t regions) const; - bool transfer_to_old(size_t regions) const; - - // force transfer is used when we promote humongous objects. May violate min/max limits on generation sizes - void force_transfer_to_old(size_t regions) const; -}; - -#endif //SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONSIZER_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp index 88b3c086da097..ece4150f577e8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp @@ -244,7 +244,9 @@ void ShenandoahGenerationalControlThread::run_gc_cycle(const ShenandoahGCRequest GCIdMark gc_id_mark; - _heap->reset_bytes_allocated_since_gc_start(); + if (gc_mode() != servicing_old) { + _heap->reset_bytes_allocated_since_gc_start(); + } MetaspaceCombinedStats meta_sizes = MetaspaceUtils::get_combined_statistics(); @@ -288,11 +290,11 @@ void ShenandoahGenerationalControlThread::run_gc_cycle(const ShenandoahGCRequest if (!_heap->cancelled_gc()) { notify_gc_waiters(); notify_alloc_failure_waiters(); + // Report current free set state at the end of cycle if normal completion. + // Do not report if cancelled, since we may not have rebuilt free set and content is unreliable. + _heap->free_set()->log_status_under_lock(); } - // Report current free set state at the end of cycle, whether - // it is a normal completion, or the abort. - _heap->free_set()->log_status_under_lock(); // Notify Universe about new heap usage. This has implications for // global soft refs policy, and we better report it every time heap diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp index ccabdb7b9daa9..4a3faa2c70702 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp @@ -174,11 +174,13 @@ void ShenandoahGenerationalEvacuationTask::promote_in_place(ShenandoahHeapRegion assert(!_generation->is_old(), "Sanity check"); ShenandoahMarkingContext* const marking_context = _heap->young_generation()->complete_marking_context(); HeapWord* const tams = marking_context->top_at_mark_start(region); + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); { - const size_t old_garbage_threshold = (ShenandoahHeapRegion::region_size_bytes() * ShenandoahOldGarbageThreshold) / 100; + const size_t old_garbage_threshold = (region_size_bytes * ShenandoahOldGarbageThreshold) / 100; assert(!_heap->is_concurrent_old_mark_in_progress(), "Cannot promote in place during old marking"); - assert(region->garbage_before_padded_for_promote() < old_garbage_threshold, "Region %zu has too much garbage for promotion", region->index()); + assert(region->garbage_before_padded_for_promote() < old_garbage_threshold, + "Region %zu has too much garbage for promotion", region->index()); assert(region->is_young(), "Only young regions can be promoted"); assert(region->is_regular(), "Use different service to promote humongous regions"); assert(_heap->is_tenurable(region), "Only promote regions that are sufficiently aged"); @@ -225,35 +227,29 @@ void ShenandoahGenerationalEvacuationTask::promote_in_place(ShenandoahHeapRegion ShenandoahHeapLocker locker(_heap->lock()); HeapWord* update_watermark = region->get_update_watermark(); + // pip_unpadded is memory too small to be filled above original top + size_t pip_unpadded = (region->end() - region->top()) * HeapWordSize; + assert((region->top() == region->end()) + || (pip_unpadded == (size_t) ((region->end() - region->top()) * HeapWordSize)), "Invariant"); + assert(pip_unpadded < ShenandoahHeap::min_fill_size() * HeapWordSize, "Sanity"); + size_t pip_pad_bytes = (region->top() - region->get_top_before_promote()) * HeapWordSize; + assert((pip_unpadded == 0) || (pip_pad_bytes == 0), "Only one of pip_unpadded and pip_pad_bytes is non-zero"); // Now that this region is affiliated with old, we can allow it to receive allocations, though it may not be in the - // is_collector_free range. + // is_collector_free range. We'll add it to that range below. region->restore_top_before_promote(); - - size_t region_used = region->used(); +#ifdef ASSERT + size_t region_to_be_used_in_old = region->used(); + assert(region_to_be_used_in_old + pip_pad_bytes + pip_unpadded == region_size_bytes, "invariant"); +#endif // The update_watermark was likely established while we had the artificially high value of top. Make it sane now. assert(update_watermark >= region->top(), "original top cannot exceed preserved update_watermark"); region->set_update_watermark(region->top()); - // Unconditionally transfer one region from young to old. This represents the newly promoted region. - // This expands old and shrinks new by the size of one region. Strictly, we do not "need" to expand old - // if there are already enough unaffiliated regions in old to account for this newly promoted region. - // However, if we do not transfer the capacities, we end up reducing the amount of memory that would have - // otherwise been available to hold old evacuations, because old available is max_capacity - used and now - // we would be trading a fully empty region for a partially used region. - young_gen->decrease_used(region_used); - young_gen->decrement_affiliated_region_count(); - - // transfer_to_old() increases capacity of old and decreases capacity of young - _heap->generation_sizer()->force_transfer_to_old(1); - region->set_affiliation(OLD_GENERATION); - - old_gen->increment_affiliated_region_count(); - old_gen->increase_used(region_used); - - // add_old_collector_free_region() increases promoted_reserve() if available space exceeds plab_min_size() + // Transfer this region from young to old, increasing promoted_reserve if available space exceeds plab_min_size() _heap->free_set()->add_promoted_in_place_region_to_old_collector(region); + region->set_affiliation(OLD_GENERATION); } } @@ -268,7 +264,8 @@ void ShenandoahGenerationalEvacuationTask::promote_humongous(ShenandoahHeapRegio const size_t used_bytes = obj->size() * HeapWordSize; const size_t spanned_regions = ShenandoahHeapRegion::required_regions(used_bytes); - const size_t humongous_waste = spanned_regions * ShenandoahHeapRegion::region_size_bytes() - obj->size() * HeapWordSize; + const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + const size_t humongous_waste = spanned_regions * region_size_bytes - obj->size() * HeapWordSize; const size_t index_limit = region->index() + spanned_regions; ShenandoahOldGeneration* const old_gen = _heap->old_generation(); @@ -282,13 +279,6 @@ void ShenandoahGenerationalEvacuationTask::promote_humongous(ShenandoahHeapRegio // usage totals, including humongous waste, after evacuation is done. log_debug(gc)("promoting humongous region %zu, spanning %zu", region->index(), spanned_regions); - young_gen->decrease_used(used_bytes); - young_gen->decrease_humongous_waste(humongous_waste); - young_gen->decrease_affiliated_region_count(spanned_regions); - - // transfer_to_old() increases capacity of old and decreases capacity of young - _heap->generation_sizer()->force_transfer_to_old(spanned_regions); - // For this region and each humongous continuation region spanned by this humongous object, change // affiliation to OLD_GENERATION and adjust the generation-use tallies. The remnant of memory // in the last humongous region that is not spanned by obj is currently not used. @@ -300,9 +290,8 @@ void ShenandoahGenerationalEvacuationTask::promote_humongous(ShenandoahHeapRegio r->set_affiliation(OLD_GENERATION); } - old_gen->increase_affiliated_region_count(spanned_regions); - old_gen->increase_used(used_bytes); - old_gen->increase_humongous_waste(humongous_waste); + ShenandoahFreeSet* freeset = _heap->free_set(); + freeset->transfer_humongous_regions_from_mutator_to_old_collector(spanned_regions, humongous_waste); } // Since this region may have served previously as OLD, it may hold obsolete object range info. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.cpp index 8d8091472fcbb..ef913362df370 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.cpp @@ -41,7 +41,7 @@ void assert_regions_used_not_more_than_capacity(ShenandoahGeneration* generation } void assert_usage_not_more_than_regions_used(ShenandoahGeneration* generation) { - assert(generation->used_including_humongous_waste() <= generation->used_regions_size(), + assert(generation->used() <= generation->used_regions_size(), "%s consumed can be no larger than span of affiliated regions", generation->name()); } #else @@ -83,7 +83,7 @@ void ShenandoahGenerationalFullGC::handle_completion(ShenandoahHeap* heap) { assert_usage_not_more_than_regions_used(young); // Establish baseline for next old-has-grown trigger. - old->set_live_bytes_after_last_mark(old->used_including_humongous_waste()); + old->set_live_bytes_after_last_mark(old->used()); } void ShenandoahGenerationalFullGC::rebuild_remembered_set(ShenandoahHeap* heap) { @@ -104,33 +104,6 @@ void ShenandoahGenerationalFullGC::rebuild_remembered_set(ShenandoahHeap* heap) heap->old_generation()->set_parsable(true); } -void ShenandoahGenerationalFullGC::balance_generations_after_gc(ShenandoahHeap* heap) { - ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::cast(heap); - ShenandoahOldGeneration* const old_gen = gen_heap->old_generation(); - - size_t old_usage = old_gen->used_regions_size(); - size_t old_capacity = old_gen->max_capacity(); - - assert(old_usage % ShenandoahHeapRegion::region_size_bytes() == 0, "Old usage must align with region size"); - assert(old_capacity % ShenandoahHeapRegion::region_size_bytes() == 0, "Old capacity must align with region size"); - - if (old_capacity > old_usage) { - size_t excess_old_regions = (old_capacity - old_usage) / ShenandoahHeapRegion::region_size_bytes(); - gen_heap->generation_sizer()->transfer_to_young(excess_old_regions); - } else if (old_capacity < old_usage) { - size_t old_regions_deficit = (old_usage - old_capacity) / ShenandoahHeapRegion::region_size_bytes(); - gen_heap->generation_sizer()->force_transfer_to_old(old_regions_deficit); - } - - log_info(gc, ergo)("FullGC done: young usage: " PROPERFMT ", old usage: " PROPERFMT, - PROPERFMTARGS(gen_heap->young_generation()->used()), - PROPERFMTARGS(old_gen->used())); -} - -ShenandoahGenerationalHeap::TransferResult ShenandoahGenerationalFullGC::balance_generations_after_rebuilding_free_set() { - return ShenandoahGenerationalHeap::heap()->balance_generations(); -} - void ShenandoahGenerationalFullGC::log_live_in_old(ShenandoahHeap* heap) { LogTarget(Debug, gc) lt; if (lt.is_enabled()) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.hpp index 06080286f22cd..32abc5455ce84 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.hpp @@ -45,25 +45,11 @@ class ShenandoahGenerationalFullGC { // Records end of cycle for young and old and establishes size of live bytes in old static void handle_completion(ShenandoahHeap* heap); - // Full GC may have promoted regions and may have temporarily violated constraints on the usage and - // capacity of the old generation. This method will balance the accounting of regions between the - // young and old generations. This is somewhat vestigial, but the outcome of this method is used - // when rebuilding the free sets. - static void balance_generations_after_gc(ShenandoahHeap* heap); - // This will compute the target size for the old generation. It will be expressed in terms of // a region surplus and deficit, which will be redistributed accordingly after rebuilding the // free set. static void compute_balances(); - // Rebuilding the free set may have resulted in regions being pulled in to the old generation - // evacuation reserve. For this reason, we must update the usage and capacity of the generations - // again. In the distant past, the free set did not know anything about generations, so we had - // a layer built above it to represent how much young/old memory was available. This layer is - // redundant and adds complexity. We would like to one day remove it. Until then, we must keep it - // synchronized with the free set's view of things. - static ShenandoahGenerationalHeap::TransferResult balance_generations_after_rebuilding_free_set(); - // Logs the number of live bytes marked in the old generation. This is _not_ the same // value used as the baseline for the old generation _after_ the full gc is complete. // The value reported in the logs does not include objects and regions that may be diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp index 5c7b65405cfef..e5f766f9df1ee 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp @@ -27,6 +27,7 @@ #include "gc/shenandoah/shenandoahClosures.inline.hpp" #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahGenerationalControlThread.hpp" #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp" #include "gc/shenandoah/shenandoahGenerationalHeap.hpp" @@ -107,17 +108,25 @@ void ShenandoahGenerationalHeap::initialize_heuristics() { // for old would be total heap - minimum capacity of young. This means the sum of the maximum // allowed for old and young could exceed the total heap size. It remains the case that the // _actual_ capacity of young + old = total. - _generation_sizer.heap_size_changed(max_capacity()); - size_t initial_capacity_young = _generation_sizer.max_young_size(); - size_t max_capacity_young = _generation_sizer.max_young_size(); + size_t region_count = num_regions(); + size_t max_young_regions = MAX2((region_count * ShenandoahMaxYoungPercentage) / 100, (size_t) 1U); + size_t initial_capacity_young = max_young_regions * ShenandoahHeapRegion::region_size_bytes(); + size_t max_capacity_young = initial_capacity_young; + size_t initial_capacity_old = max_capacity() - max_capacity_young; size_t max_capacity_old = max_capacity() - initial_capacity_young; - _young_generation = new ShenandoahYoungGeneration(max_workers(), max_capacity_young); - _old_generation = new ShenandoahOldGeneration(max_workers(), max_capacity_old); + _young_generation = new ShenandoahYoungGeneration(max_workers()); + _old_generation = new ShenandoahOldGeneration(max_workers()); _young_generation->initialize_heuristics(mode()); _old_generation->initialize_heuristics(mode()); } +void ShenandoahGenerationalHeap::post_initialize_heuristics() { + ShenandoahHeap::post_initialize_heuristics(); + _young_generation->post_initialize(this); + _old_generation->post_initialize(this); +} + void ShenandoahGenerationalHeap::initialize_serviceability() { assert(mode()->is_generational(), "Only for the generational mode"); _young_gen_memory_pool = new ShenandoahYoungGenMemoryPool(this); @@ -577,39 +586,13 @@ void ShenandoahGenerationalHeap::retire_plab(PLAB* plab) { retire_plab(plab, thread); } -ShenandoahGenerationalHeap::TransferResult ShenandoahGenerationalHeap::balance_generations() { - shenandoah_assert_heaplocked_or_safepoint(); - - ShenandoahOldGeneration* old_gen = old_generation(); - const ssize_t old_region_balance = old_gen->get_region_balance(); - old_gen->set_region_balance(0); - - if (old_region_balance > 0) { - const auto old_region_surplus = checked_cast(old_region_balance); - const bool success = generation_sizer()->transfer_to_young(old_region_surplus); - return TransferResult { - success, old_region_surplus, "young" - }; - } - - if (old_region_balance < 0) { - const auto old_region_deficit = checked_cast(-old_region_balance); - const bool success = generation_sizer()->transfer_to_old(old_region_deficit); - if (!success) { - old_gen->handle_failed_transfer(); - } - return TransferResult { - success, old_region_deficit, "old" - }; - } - - return TransferResult {true, 0, "none"}; -} - // Make sure old-generation is large enough, but no larger than is necessary, to hold mixed evacuations // and promotions, if we anticipate either. Any deficit is provided by the young generation, subject to // xfer_limit, and any surplus is transferred to the young generation. -// xfer_limit is the maximum we're able to transfer from young to old. +// +// xfer_limit is the maximum we're able to transfer from young to old based on either: +// 1. an assumption that we will be able to replenish memory "borrowed" from young at the end of collection, or +// 2. there is sufficient excess in the allocation runway during GC idle cycles void ShenandoahGenerationalHeap::compute_old_generation_balance(size_t old_xfer_limit, size_t old_cset_regions) { // We can limit the old reserve to the size of anticipated promotions: @@ -637,9 +620,9 @@ void ShenandoahGenerationalHeap::compute_old_generation_balance(size_t old_xfer_ // In the case that ShenandoahOldEvacRatioPercent equals 100, max_old_reserve is limited only by xfer_limit. const double bound_on_old_reserve = old_available + old_xfer_limit + young_reserve; - const double max_old_reserve = (ShenandoahOldEvacRatioPercent == 100)? - bound_on_old_reserve: MIN2(double(young_reserve * ShenandoahOldEvacRatioPercent) / double(100 - ShenandoahOldEvacRatioPercent), - bound_on_old_reserve); + const double max_old_reserve = ((ShenandoahOldEvacRatioPercent == 100)? bound_on_old_reserve: + MIN2(double(young_reserve * ShenandoahOldEvacRatioPercent) + / double(100 - ShenandoahOldEvacRatioPercent), bound_on_old_reserve)); const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); @@ -648,10 +631,12 @@ void ShenandoahGenerationalHeap::compute_old_generation_balance(size_t old_xfer_ if (old_generation()->has_unprocessed_collection_candidates()) { // We want this much memory to be unfragmented in order to reliably evacuate old. This is conservative because we // may not evacuate the entirety of unprocessed candidates in a single mixed evacuation. - const double max_evac_need = (double(old_generation()->unprocessed_collection_candidates_live_memory()) * ShenandoahOldEvacWaste); + const double max_evac_need = + (double(old_generation()->unprocessed_collection_candidates_live_memory()) * ShenandoahOldEvacWaste); assert(old_available >= old_generation()->free_unaffiliated_regions() * region_size_bytes, "Unaffiliated available must be less than total available"); - const double old_fragmented_available = double(old_available - old_generation()->free_unaffiliated_regions() * region_size_bytes); + const double old_fragmented_available = + double(old_available - old_generation()->free_unaffiliated_regions() * region_size_bytes); reserve_for_mixed = max_evac_need + old_fragmented_available; if (reserve_for_mixed > max_old_reserve) { reserve_for_mixed = max_old_reserve; @@ -698,6 +683,7 @@ void ShenandoahGenerationalHeap::compute_old_generation_balance(size_t old_xfer_ } void ShenandoahGenerationalHeap::reset_generation_reserves() { + ShenandoahHeapLocker locker(lock()); young_generation()->set_evacuation_reserve(0); old_generation()->set_evacuation_reserve(0); old_generation()->set_promoted_reserve(0); @@ -1060,15 +1046,6 @@ void ShenandoahGenerationalHeap::complete_degenerated_cycle() { // a more detailed explanation. old_generation()->transfer_pointers_from_satb(); } - - // We defer generation resizing actions until after cset regions have been recycled. - TransferResult result = balance_generations(); - LogTarget(Info, gc, ergo) lt; - if (lt.is_enabled()) { - LogStream ls(lt); - result.print_on("Degenerated GC", &ls); - } - // In case degeneration interrupted concurrent evacuation or update references, we need to clean up // transient state. Otherwise, these actions have no effect. reset_generation_reserves(); @@ -1090,24 +1067,7 @@ void ShenandoahGenerationalHeap::complete_concurrent_cycle() { // throw off the heuristics. entry_global_coalesce_and_fill(); } - - log_info(gc, cset)("Concurrent cycle complete, promotions reserved: %zu, promotions expended: %zu, failed count: %zu, failed bytes: %zu", - old_generation()->get_promoted_reserve(), old_generation()->get_promoted_expended(), - old_generation()->get_promotion_failed_count(), old_generation()->get_promotion_failed_words() * HeapWordSize); - - TransferResult result; - { - ShenandoahHeapLocker locker(lock()); - - result = balance_generations(); - reset_generation_reserves(); - } - - LogTarget(Info, gc, ergo) lt; - if (lt.is_enabled()) { - LogStream ls(lt); - result.print_on("Concurrent GC", &ls); - } + reset_generation_reserves(); } void ShenandoahGenerationalHeap::entry_global_coalesce_and_fill() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp index 4c8c9fe01185c..3eeec26dbbf45 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp @@ -41,6 +41,7 @@ class ShenandoahGenerationalHeap : public ShenandoahHeap { explicit ShenandoahGenerationalHeap(ShenandoahCollectorPolicy* policy); void post_initialize() override; void initialize_heuristics() override; + void post_initialize_heuristics() override; static ShenandoahGenerationalHeap* heap() { assert(ShenandoahCardBarrier, "Should have card barrier to use genenrational heap"); @@ -138,8 +139,6 @@ class ShenandoahGenerationalHeap : public ShenandoahHeap { void print_on(const char* when, outputStream* ss) const; }; - const ShenandoahGenerationSizer* generation_sizer() const { return &_generation_sizer; } - // Zeros out the evacuation and promotion reserves void reset_generation_reserves(); @@ -163,8 +162,6 @@ class ShenandoahGenerationalHeap : public ShenandoahHeap { MemoryPool* _young_gen_memory_pool; MemoryPool* _old_gen_memory_pool; - - ShenandoahGenerationSizer _generation_sizer; }; #endif //SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONALHEAP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp index 6099c41f26258..c9972e2db8111 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp @@ -37,17 +37,38 @@ const char* ShenandoahGlobalGeneration::name() const { } size_t ShenandoahGlobalGeneration::max_capacity() const { - return ShenandoahHeap::heap()->max_capacity(); + size_t total_regions = _free_set->total_global_regions(); + return total_regions * ShenandoahHeapRegion::region_size_bytes(); } +size_t ShenandoahGlobalGeneration::free_unaffiliated_regions() const { + return _free_set->global_unaffiliated_regions(); +} + +size_t ShenandoahGlobalGeneration::used() const { + return _free_set->global_used(); +} + +size_t ShenandoahGlobalGeneration::bytes_allocated_since_gc_start() const { + return _free_set->get_bytes_allocated_since_gc_start(); +} + +size_t ShenandoahGlobalGeneration::get_affiliated_region_count() const { + return _free_set->global_affiliated_regions(); +} + +size_t ShenandoahGlobalGeneration::get_humongous_waste() const { + return _free_set->total_humongous_waste(); +} + + size_t ShenandoahGlobalGeneration::used_regions() const { - ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap(); - assert(heap->mode()->is_generational(), "Region usage accounting is only for generational mode"); - return heap->old_generation()->used_regions() + heap->young_generation()->used_regions(); + return _free_set->global_affiliated_regions(); } size_t ShenandoahGlobalGeneration::used_regions_size() const { - return ShenandoahHeap::heap()->capacity(); + size_t used_regions = _free_set->global_affiliated_regions(); + return used_regions * ShenandoahHeapRegion::region_size_bytes(); } size_t ShenandoahGlobalGeneration::available() const { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp index a823784a4594b..5ceba8ed50ef0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp @@ -32,15 +32,29 @@ // A "generation" that represents the whole heap. class ShenandoahGlobalGeneration : public ShenandoahGeneration { public: - ShenandoahGlobalGeneration(bool generational, uint max_queues, size_t max_capacity) - : ShenandoahGeneration(generational ? GLOBAL : NON_GEN, max_queues, max_capacity) { } + ShenandoahGlobalGeneration(bool generational, uint max_queues) + : ShenandoahGeneration(generational ? GLOBAL : NON_GEN, max_queues) { +#ifdef ASSERT + ShenandoahHeap* heap = ShenandoahHeap::heap(); + bool is_generational = heap->mode()->is_generational(); + assert(is_generational == generational, "sanity"); + assert((is_generational && (type() == ShenandoahGenerationType::GLOBAL)) || + (!is_generational && (type() == ShenandoahGenerationType::NON_GEN)), "OO sanity"); +#endif + } public: const char* name() const override; - size_t max_capacity() const override; + size_t bytes_allocated_since_gc_start() const override; + size_t used() const override; size_t used_regions() const override; size_t used_regions_size() const override; + size_t get_humongous_waste() const override; + size_t free_unaffiliated_regions() const override; + size_t get_affiliated_region_count() const override; + size_t max_capacity() const override; + size_t available() const override; size_t soft_available() const override; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index a058a7bd38d8a..f66d83204a4fc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -201,7 +201,7 @@ jint ShenandoahHeap::initialize() { assert(num_min_regions <= _num_regions, "sanity"); _minimum_size = num_min_regions * reg_size_bytes; - _soft_max_size = SoftMaxHeapSize; + _soft_max_size = clamp(SoftMaxHeapSize, min_capacity(), max_capacity()); _committed = _initial_size; @@ -252,7 +252,7 @@ jint ShenandoahHeap::initialize() { // it means we're under passive mode and we have to initialize old gen // for the purpose of having card table. if (ShenandoahCardBarrier && !(mode()->is_generational())) { - _old_generation = new ShenandoahOldGeneration(max_workers(), max_capacity()); + _old_generation = new ShenandoahOldGeneration(max_workers()); } assert(_heap_region.byte_size() == heap_rs.size(), "Need to know reserved size for card table"); @@ -411,7 +411,6 @@ jint ShenandoahHeap::initialize() { { ShenandoahHeapLocker locker(lock()); - _free_set = new ShenandoahFreeSet(this, _num_regions); for (size_t i = 0; i < _num_regions; i++) { HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i; bool is_committed = i < num_committed_regions; @@ -426,12 +425,21 @@ jint ShenandoahHeap::initialize() { _affiliations[i] = ShenandoahAffiliation::FREE; } + _free_set = new ShenandoahFreeSet(this, _num_regions); - size_t young_cset_regions, old_cset_regions; + post_initialize_heuristics(); // We are initializing free set. We ignore cset region tallies. - size_t first_old, last_old, num_old; + size_t young_cset_regions, old_cset_regions, first_old, last_old, num_old; _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old); + if (mode()->is_generational()) { + ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap(); + // We cannot call + // gen_heap->young_generation()->heuristics()->bytes_of_allocation_runway_before_gc_trigger(young_cset_regions) + // until after the heap is fully initialized. So we make up a safe value here. + size_t allocation_runway = InitialHeapSize / 2; + gen_heap->compute_old_generation_balance(allocation_runway, old_cset_regions); + } _free_set->finish_rebuild(young_cset_regions, old_cset_regions, num_old); } @@ -525,10 +533,14 @@ void ShenandoahHeap::initialize_mode() { } void ShenandoahHeap::initialize_heuristics() { - _global_generation = new ShenandoahGlobalGeneration(mode()->is_generational(), max_workers(), max_capacity()); + _global_generation = new ShenandoahGlobalGeneration(mode()->is_generational(), max_workers()); _global_generation->initialize_heuristics(mode()); } +void ShenandoahHeap::post_initialize_heuristics() { + _global_generation->post_initialize(this); +} + #ifdef _MSC_VER #pragma warning( push ) #pragma warning( disable:4355 ) // 'this' : used in base member initializer list @@ -673,6 +685,8 @@ class ShenandoahInitWorkerGCLABClosure : public ThreadClosure { void ShenandoahHeap::post_initialize() { CollectedHeap::post_initialize(); + check_soft_max_changed(); + // Schedule periodic task to report on gc thread CPU utilization _mmu_tracker.initialize(); @@ -717,75 +731,6 @@ void ShenandoahHeap::decrease_committed(size_t bytes) { _committed -= bytes; } -// For tracking usage based on allocations, it should be the case that: -// * The sum of regions::used == heap::used -// * The sum of a generation's regions::used == generation::used -// * The sum of a generation's humongous regions::free == generation::humongous_waste -// These invariants are checked by the verifier on GC safepoints. -// -// Additional notes: -// * When a mutator's allocation request causes a region to be retired, the -// free memory left in that region is considered waste. It does not contribute -// to the usage, but it _does_ contribute to allocation rate. -// * The bottom of a PLAB must be aligned on card size. In some cases this will -// require padding in front of the PLAB (a filler object). Because this padding -// is included in the region's used memory we include the padding in the usage -// accounting as waste. -// * Mutator allocations are used to compute an allocation rate. -// * There are three sources of waste: -// 1. The padding used to align a PLAB on card size -// 2. Region's free is less than minimum TLAB size and is retired -// 3. The unused portion of memory in the last region of a humongous object -void ShenandoahHeap::increase_used(const ShenandoahAllocRequest& req) { - size_t actual_bytes = req.actual_size() * HeapWordSize; - size_t wasted_bytes = req.waste() * HeapWordSize; - ShenandoahGeneration* generation = generation_for(req.affiliation()); - - if (req.is_gc_alloc()) { - assert(wasted_bytes == 0 || req.type() == ShenandoahAllocRequest::_alloc_plab, "Only PLABs have waste"); - increase_used(generation, actual_bytes + wasted_bytes); - } else { - assert(req.is_mutator_alloc(), "Expected mutator alloc here"); - // padding and actual size both count towards allocation counter - generation->increase_allocated(actual_bytes + wasted_bytes); - - // only actual size counts toward usage for mutator allocations - increase_used(generation, actual_bytes); - - if (wasted_bytes > 0 && ShenandoahHeapRegion::requires_humongous(req.actual_size())) { - increase_humongous_waste(generation,wasted_bytes); - } - } -} - -void ShenandoahHeap::increase_humongous_waste(ShenandoahGeneration* generation, size_t bytes) { - generation->increase_humongous_waste(bytes); - if (!generation->is_global()) { - global_generation()->increase_humongous_waste(bytes); - } -} - -void ShenandoahHeap::decrease_humongous_waste(ShenandoahGeneration* generation, size_t bytes) { - generation->decrease_humongous_waste(bytes); - if (!generation->is_global()) { - global_generation()->decrease_humongous_waste(bytes); - } -} - -void ShenandoahHeap::increase_used(ShenandoahGeneration* generation, size_t bytes) { - generation->increase_used(bytes); - if (!generation->is_global()) { - global_generation()->increase_used(bytes); - } -} - -void ShenandoahHeap::decrease_used(ShenandoahGeneration* generation, size_t bytes) { - generation->decrease_used(bytes); - if (!generation->is_global()) { - global_generation()->decrease_used(bytes); - } -} - size_t ShenandoahHeap::capacity() const { return committed(); } @@ -1034,10 +979,6 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) { req.set_actual_size(0); } - // This is called regardless of the outcome of the allocation to account - // for any waste created by retiring regions with this request. - increase_used(req); - if (result != nullptr) { size_t requested = req.size(); size_t actual = req.actual_size(); @@ -2347,18 +2288,16 @@ void ShenandoahHeap::reset_bytes_allocated_since_gc_start() { // the "forced sample" will not happen, and any recently allocated bytes are "unaccounted for". We pretend these // bytes are allocated after the start of subsequent gc. size_t unaccounted_bytes; + ShenandoahFreeSet* _free_set = free_set(); + size_t bytes_allocated = _free_set->get_bytes_allocated_since_gc_start(); if (mode()->is_generational()) { - size_t bytes_allocated = young_generation()->bytes_allocated_since_gc_start(); unaccounted_bytes = young_generation()->heuristics()->force_alloc_rate_sample(bytes_allocated); - young_generation()->reset_bytes_allocated_since_gc_start(unaccounted_bytes); - unaccounted_bytes = 0; - old_generation()->reset_bytes_allocated_since_gc_start(unaccounted_bytes); } else { - size_t bytes_allocated = global_generation()->bytes_allocated_since_gc_start(); // Single-gen Shenandoah uses global heuristics. unaccounted_bytes = heuristics()->force_alloc_rate_sample(bytes_allocated); } - global_generation()->reset_bytes_allocated_since_gc_start(unaccounted_bytes); + ShenandoahHeapLocker locker(lock()); + _free_set->reset_bytes_allocated_since_gc_start(unaccounted_bytes); } void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) { @@ -2743,6 +2682,9 @@ GrowableArray ShenandoahHeap::memory_pools() { } MemoryUsage ShenandoahHeap::memory_usage() { + assert(_initial_size <= ShenandoahHeap::heap()->max_capacity(), "sanity"); + assert(used() <= ShenandoahHeap::heap()->max_capacity(), "sanity"); + assert(committed() <= ShenandoahHeap::heap()->max_capacity(), "sanity"); return MemoryUsage(_initial_size, used(), committed(), max_capacity()); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index e44bbd55e9203..6cbe44fef09fb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -35,7 +35,6 @@ #include "gc/shenandoah/shenandoahController.hpp" #include "gc/shenandoah/shenandoahEvacOOMHandler.hpp" #include "gc/shenandoah/shenandoahEvacTracker.hpp" -#include "gc/shenandoah/shenandoahGenerationSizer.hpp" #include "gc/shenandoah/shenandoahGenerationType.hpp" #include "gc/shenandoah/shenandoahLock.hpp" #include "gc/shenandoah/shenandoahMmuTracker.hpp" @@ -183,6 +182,7 @@ class ShenandoahHeap : public CollectedHeap { void post_initialize() override; void initialize_mode(); virtual void initialize_heuristics(); + virtual void post_initialize_heuristics(); virtual void print_init_logger() const; void initialize_serviceability() override; @@ -212,14 +212,7 @@ class ShenandoahHeap : public CollectedHeap { volatile size_t _committed; shenandoah_padding(1); - void increase_used(const ShenandoahAllocRequest& req); - public: - void increase_used(ShenandoahGeneration* generation, size_t bytes); - void decrease_used(ShenandoahGeneration* generation, size_t bytes); - void increase_humongous_waste(ShenandoahGeneration* generation, size_t bytes); - void decrease_humongous_waste(ShenandoahGeneration* generation, size_t bytes); - void increase_committed(size_t bytes); void decrease_committed(size_t bytes); @@ -698,8 +691,6 @@ class ShenandoahHeap : public CollectedHeap { size_t size, Metaspace::MetadataType mdtype) override; - void notify_mutator_alloc_words(size_t words, size_t waste); - HeapWord* allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) override; size_t tlab_capacity(Thread *thr) const override; size_t unsafe_max_tlab_alloc(Thread *thread) const override; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index ca0f7460d542e..3cd5cdd2ec32e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -578,16 +578,16 @@ void ShenandoahHeapRegion::recycle_internal() { set_affiliation(FREE); } +// Upon return, this region has been recycled. We try to recycle it. +// We may fail if some other thread recycled it before we do. void ShenandoahHeapRegion::try_recycle_under_lock() { shenandoah_assert_heaplocked(); if (is_trash() && _recycling.try_set()) { if (is_trash()) { - ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahGeneration* generation = heap->generation_for(affiliation()); - - heap->decrease_used(generation, used()); - generation->decrement_affiliated_region_count(); - + // At freeset rebuild time, which precedes recycling of collection set, we treat all cset regions as + // part of capacity, as empty, as fully available, and as unaffiliated. This provides short-lived optimism + // for triggering heuristics. It greatly simplifies and reduces the locking overhead required + // by more time-precise accounting of these details. recycle_internal(); } _recycling.unset(); @@ -608,11 +608,10 @@ void ShenandoahHeapRegion::try_recycle() { if (is_trash() && _recycling.try_set()) { // Double check region state after win the race to set recycling flag if (is_trash()) { - ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahGeneration* generation = heap->generation_for(affiliation()); - heap->decrease_used(generation, used()); - generation->decrement_affiliated_region_count_without_lock(); - + // At freeset rebuild time, which precedes recycling of collection set, we treat all cset regions as + // part of capacity, as empty, as fully available, and as unaffiliated. This provides short-lived optimism + // for triggering and pacing heuristics. It greatly simplifies and reduces the locking overhead required + // by more time-precise accounting of these details. recycle_internal(); } _recycling.unset(); @@ -900,12 +899,11 @@ void ShenandoahHeapRegion::set_affiliation(ShenandoahAffiliation new_affiliation heap->set_affiliation(this, new_affiliation); } -void ShenandoahHeapRegion::decrement_humongous_waste() const { +void ShenandoahHeapRegion::decrement_humongous_waste() { assert(is_humongous(), "Should only use this for humongous regions"); size_t waste_bytes = free(); if (waste_bytes > 0) { ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahGeneration* generation = heap->generation_for(affiliation()); - heap->decrease_humongous_waste(generation, waste_bytes); + heap->free_set()->decrease_humongous_waste_for_regular_bypass(this, waste_bytes); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index b6f65834c0763..32382f5e594b5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -366,6 +366,9 @@ class ShenandoahHeapRegion { // Allocation (return nullptr if full) inline HeapWord* allocate(size_t word_size, const ShenandoahAllocRequest& req); + // Allocate fill after top + inline HeapWord* allocate_fill(size_t word_size); + inline void clear_live_data(); void set_live_data(size_t s); @@ -492,7 +495,7 @@ class ShenandoahHeapRegion { } private: - void decrement_humongous_waste() const; + void decrement_humongous_waste(); void do_commit(); void do_uncommit(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index da1caf24266d0..cad9dc0e9320e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -87,6 +87,23 @@ HeapWord* ShenandoahHeapRegion::allocate_aligned(size_t size, ShenandoahAllocReq } } +HeapWord* ShenandoahHeapRegion::allocate_fill(size_t size) { + shenandoah_assert_heaplocked_or_safepoint(); + assert(is_object_aligned(size), "alloc size breaks alignment: %zu", size); + assert(size >= ShenandoahHeap::min_fill_size(), "Cannot fill unless min fill size"); + + HeapWord* obj = top(); + HeapWord* new_top = obj + size; + ShenandoahHeap::fill_with_object(obj, size); + set_top(new_top); + + assert(is_object_aligned(new_top), "new top breaks alignment: " PTR_FORMAT, p2i(new_top)); + assert(is_object_aligned(obj), "obj is not aligned: " PTR_FORMAT, p2i(obj)); + + return obj; +} + + HeapWord* ShenandoahHeapRegion::allocate(size_t size, const ShenandoahAllocRequest& req) { shenandoah_assert_heaplocked_or_safepoint(); assert(is_object_aligned(size), "alloc size breaks alignment: %zu", size); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.hpp index 472f571264864..c094ec434f5d7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.hpp @@ -33,7 +33,7 @@ class ShenandoahMarkBitMap { public: - typedef size_t idx_t; // Type used for bit and word indices. + typedef size_t idx_t; // Type used for bit and word indices. typedef uintptr_t bm_word_t; // Element type of array that represents the // bitmap, with BitsPerWord bits per element. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.cpp index ebfe5267160fb..d55d1bd814717 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.cpp @@ -62,8 +62,10 @@ MemoryUsage ShenandoahMemoryPool::get_memory_usage() { // the assert below, which would also fail in downstream code. To avoid that, adjust values // to make sense under the race. See JDK-8207200. committed = MAX2(used, committed); - assert(used <= committed, "used: %zu, committed: %zu", used, committed); - + assert(used <= committed, "used: %zu, committed: %zu", used, committed); + assert(initial <= _heap->max_capacity(), "sanity"); + assert(committed <= _heap->max_capacity(), "sanity"); + assert(max <= _heap->max_capacity(), "sanity"); return MemoryUsage(initial, used, committed, max); } @@ -86,6 +88,10 @@ MemoryUsage ShenandoahGenerationalMemoryPool::get_memory_usage() { size_t used = used_in_bytes(); size_t committed = _generation->used_regions_size(); + assert(initial <= _heap->max_capacity(), "sanity"); + assert(used <= _heap->max_capacity(), "sanity"); + assert(committed <= _heap->max_capacity(), "sanity"); + assert(max <= _heap->max_capacity(), "sanity"); return MemoryUsage(initial, used, committed, max); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index d980a9e3e0c21..a44a831ef3df0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -138,21 +138,7 @@ bool ShenandoahOldGC::collect(GCCause::Cause cause) { // collection. heap->concurrent_final_roots(); - // We do not rebuild_free following increments of old marking because memory has not been reclaimed. However, we may - // need to transfer memory to OLD in order to efficiently support the mixed evacuations that might immediately follow. size_t allocation_runway = heap->young_generation()->heuristics()->bytes_of_allocation_runway_before_gc_trigger(0); heap->compute_old_generation_balance(allocation_runway, 0); - - ShenandoahGenerationalHeap::TransferResult result; - { - ShenandoahHeapLocker locker(heap->lock()); - result = heap->balance_generations(); - } - - LogTarget(Info, gc, ergo) lt; - if (lt.is_enabled()) { - LogStream ls(lt); - result.print_on("Old Mark", &ls); - } return true; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index 9c95dad64099e..1f474baef4626 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -1,3 +1,4 @@ + /* * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. @@ -195,8 +196,8 @@ class ShenandoahConcurrentCoalesceAndFillTask : public WorkerTask { } }; -ShenandoahOldGeneration::ShenandoahOldGeneration(uint max_queues, size_t max_capacity) - : ShenandoahGeneration(OLD, max_queues, max_capacity), +ShenandoahOldGeneration::ShenandoahOldGeneration(uint max_queues) + : ShenandoahGeneration(OLD, max_queues), _coalesce_and_fill_region_array(NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, ShenandoahHeap::heap()->num_regions(), mtGC)), _old_heuristics(nullptr), _region_balance(0), @@ -214,6 +215,7 @@ ShenandoahOldGeneration::ShenandoahOldGeneration(uint max_queues, size_t max_cap _growth_before_compaction(INITIAL_GROWTH_BEFORE_COMPACTION), _min_growth_before_compaction ((ShenandoahMinOldGenGrowthPercent * FRACTIONAL_DENOMINATOR) / 100) { + assert(type() == ShenandoahGenerationType::OLD, "OO sanity"); _live_bytes_after_last_mark = ShenandoahHeap::heap()->capacity() * INITIAL_LIVE_FRACTION / FRACTIONAL_DENOMINATOR; // Always clear references for old generation ref_processor()->set_soft_reference_policy(true); @@ -519,12 +521,20 @@ void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent ShenandoahPhaseTimings::final_rebuild_freeset : ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset); ShenandoahHeapLocker locker(heap->lock()); - size_t cset_young_regions, cset_old_regions; + size_t young_trash_regions, old_trash_regions; size_t first_old, last_old, num_old; - heap->free_set()->prepare_to_rebuild(cset_young_regions, cset_old_regions, first_old, last_old, num_old); - // This is just old-gen completion. No future budgeting required here. The only reason to rebuild the freeset here - // is in case there was any immediate old garbage identified. - heap->free_set()->finish_rebuild(cset_young_regions, cset_old_regions, num_old); + heap->free_set()->prepare_to_rebuild(young_trash_regions, old_trash_regions, first_old, last_old, num_old); + // At the end of old-gen, we may find that we have reclaimed immediate garbage, allowing a longer allocation runway. + // We may also find that we have accumulated canddiate regions for mixed evacuation. If so, we will want to expand + // the OldCollector reserve in order to make room for these mixed evacuations. + assert(ShenandoahHeap::heap()->mode()->is_generational(), "sanity"); + assert(young_trash_regions == 0, "sanity"); + ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap(); + size_t allocation_runway = + gen_heap->young_generation()->heuristics()->bytes_of_allocation_runway_before_gc_trigger(young_trash_regions); + gen_heap->compute_old_generation_balance(allocation_runway, old_trash_regions); + + heap->free_set()->finish_rebuild(young_trash_regions, old_trash_regions, num_old); } } @@ -729,11 +739,6 @@ void ShenandoahOldGeneration::handle_evacuation(HeapWord* obj, size_t words, boo // do this in batch, in a background GC thread than to try to carefully dirty only cards // that hold interesting pointers right now. _card_scan->mark_range_as_dirty(obj, words); - - if (promotion) { - // This evacuation was a promotion, track this as allocation against old gen - increase_allocated(words * HeapWordSize); - } } bool ShenandoahOldGeneration::has_unprocessed_collection_candidates() { @@ -839,3 +844,38 @@ void ShenandoahOldGeneration::clear_cards_for(ShenandoahHeapRegion* region) { void ShenandoahOldGeneration::mark_card_as_dirty(void* location) { _card_scan->mark_card_as_dirty((HeapWord*)location); } + +size_t ShenandoahOldGeneration::used() const { + return _free_set->old_used(); +} + +size_t ShenandoahOldGeneration::bytes_allocated_since_gc_start() const { + assert(ShenandoahHeap::heap()->mode()->is_generational(), "NON_GEN implies not generational"); + return 0; +} + +size_t ShenandoahOldGeneration::get_affiliated_region_count() const { + return _free_set->old_affiliated_regions(); +} + +size_t ShenandoahOldGeneration::get_humongous_waste() const { + return _free_set->humongous_waste_in_old(); +} + +size_t ShenandoahOldGeneration::used_regions() const { + return _free_set->old_affiliated_regions(); +} + +size_t ShenandoahOldGeneration::used_regions_size() const { + size_t used_regions = _free_set->old_affiliated_regions(); + return used_regions * ShenandoahHeapRegion::region_size_bytes(); +} + +size_t ShenandoahOldGeneration::max_capacity() const { + size_t total_regions = _free_set->total_old_regions(); + return total_regions * ShenandoahHeapRegion::region_size_bytes(); +} + +size_t ShenandoahOldGeneration::free_unaffiliated_regions() const { + return _free_set->old_unaffiliated_regions(); +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp index 28d2a30493c65..028ee18554c05 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp @@ -94,7 +94,7 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { bool coalesce_and_fill(); public: - ShenandoahOldGeneration(uint max_queues, size_t max_capacity); + ShenandoahOldGeneration(uint max_queues); ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode) override; @@ -145,7 +145,9 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { void configure_plab_for_current_thread(const ShenandoahAllocRequest &req); // See description in field declaration - void set_region_balance(ssize_t balance) { _region_balance = balance; } + void set_region_balance(ssize_t balance) { + _region_balance = balance; + } ssize_t get_region_balance() const { return _region_balance; } // See description in field declaration @@ -331,6 +333,14 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { static const char* state_name(State state); + size_t bytes_allocated_since_gc_start() const override; + size_t used() const override; + size_t used_regions() const override; + size_t used_regions_size() const override; + size_t get_humongous_waste() const override; + size_t free_unaffiliated_regions() const override; + size_t get_affiliated_region_count() const override; + size_t max_capacity() const override; }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.cpp index 3f4bbafb755e3..82a759e34dbdf 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.cpp @@ -25,7 +25,7 @@ #include "gc/shenandoah/shenandoahSimpleBitMap.inline.hpp" -ShenandoahSimpleBitMap::ShenandoahSimpleBitMap(size_t num_bits) : +ShenandoahSimpleBitMap::ShenandoahSimpleBitMap(idx_t num_bits) : _num_bits(num_bits), _num_words(align_up(num_bits, BitsPerWord) / BitsPerWord), _bitmap(NEW_C_HEAP_ARRAY(uintx, _num_words, mtGC)) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.hpp index 3a4cb8cf742fc..a2d664f20161f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.hpp @@ -42,22 +42,23 @@ // represent index, even though index is "inherently" unsigned. There are two reasons for this choice: // 1. We use -1 as a sentinel value to represent empty partitions. This same value may be used to represent // failure to find a previous set bit or previous range of set bits. -// 2. Certain loops are written most naturally if the iterator, which may hold the sentinel -1 value, can be +// 2. Certain loops are written most naturally if the induction variable, which may hold the sentinel -1 value, can be // declared as signed and the terminating condition can be < 0. -typedef ssize_t idx_t; - // ShenandoahSimpleBitMap resembles CHeapBitMap but adds missing support for find_first_consecutive_set_bits() and // find_last_consecutive_set_bits. An alternative refactoring of code would subclass CHeapBitMap, but this might // break abstraction rules, because efficient implementation requires assumptions about superclass internals that // might be violated through future software maintenance. class ShenandoahSimpleBitMap { +public: + typedef ssize_t idx_t; +private: const idx_t _num_bits; const size_t _num_words; uintx* const _bitmap; public: - ShenandoahSimpleBitMap(size_t num_bits); + ShenandoahSimpleBitMap(idx_t num_bits); ~ShenandoahSimpleBitMap(); @@ -116,7 +117,6 @@ class ShenandoahSimpleBitMap { inline void clear_bit(idx_t idx) { assert((idx >= 0) && (idx < _num_bits), "precondition"); - assert(idx >= 0, "precondition"); size_t array_idx = idx >> LogBitsPerWord; uintx bit_number = idx & (BitsPerWord - 1); uintx the_bit = nth_bit(bit_number); @@ -125,7 +125,6 @@ class ShenandoahSimpleBitMap { inline bool is_set(idx_t idx) const { assert((idx >= 0) && (idx < _num_bits), "precondition"); - assert(idx >= 0, "precondition"); size_t array_idx = idx >> LogBitsPerWord; uintx bit_number = idx & (BitsPerWord - 1); uintx the_bit = nth_bit(bit_number); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.inline.hpp index 4582ab9a781dd..84d116cf61896 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.inline.hpp @@ -27,6 +27,8 @@ #include "gc/shenandoah/shenandoahSimpleBitMap.hpp" +using idx_t = ShenandoahSimpleBitMap::idx_t; + inline uintx ShenandoahSimpleBitMap::tail_mask(uintx bit_number) { if (bit_number >= BitsPerWord) { return -1; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index fb5fbbd00a152..543df2422c0c0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -365,30 +365,57 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { // a subset (e.g. the young generation or old generation) of the total heap. class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure { private: - size_t _used, _committed, _garbage, _regions, _humongous_waste, _trashed_regions; + size_t _used, _committed, _garbage, _regions, _humongous_waste, _trashed_regions, _trashed_used; + size_t _region_size_bytes, _min_free_size; public: ShenandoahCalculateRegionStatsClosure() : - _used(0), _committed(0), _garbage(0), _regions(0), _humongous_waste(0), _trashed_regions(0) {}; + _used(0), _committed(0), _garbage(0), _regions(0), _humongous_waste(0), _trashed_regions(0), _trashed_used(0) + { + _region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + // Retired regions are not necessarily filled, thouugh their remnant memory is considered used. + _min_free_size = PLAB::min_size() * HeapWordSize; + }; void heap_region_do(ShenandoahHeapRegion* r) override { - _used += r->used(); - _garbage += r->garbage(); - _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0; - if (r->is_humongous()) { - _humongous_waste += r->free(); - } - if (r->is_trash()) { - _trashed_regions++; + if (r->is_cset() || r->is_trash()) { + // Count the entire cset or trashed (formerly cset) region as used + // Note: Immediate garbage trash regions were never in the cset. + _used += _region_size_bytes; + _garbage += _region_size_bytes - r->get_live_data_bytes(); + if (r->is_trash()) { + _trashed_regions++; + _trashed_used += _region_size_bytes; + } + } else { + if (r->is_humongous()) { + _used += _region_size_bytes; + _garbage += _region_size_bytes - r->get_live_data_bytes(); + _humongous_waste += r->free(); + } else { + size_t alloc_capacity = r->free(); + if (alloc_capacity < _min_free_size) { + // this region has been retired already, count it as entirely consumed + alloc_capacity = 0; + } + size_t bytes_used_in_region = _region_size_bytes - alloc_capacity; + size_t bytes_garbage_in_region = bytes_used_in_region - r->get_live_data_bytes(); + size_t waste_bytes = r->free(); + _used += bytes_used_in_region; + _garbage += bytes_garbage_in_region; + } } + _committed += r->is_committed() ? _region_size_bytes : 0; _regions++; log_debug(gc)("ShenandoahCalculateRegionStatsClosure: adding %zu for %s Region %zu, yielding: %zu", r->used(), (r->is_humongous() ? "humongous" : "regular"), r->index(), _used); } size_t used() const { return _used; } + size_t used_after_recycle() const { return _used - _trashed_used; } size_t committed() const { return _committed; } size_t garbage() const { return _garbage; } size_t regions() const { return _regions; } + size_t trashed_regions() const { return _trashed_regions; } size_t waste() const { return _humongous_waste; } // span is the total memory affiliated with these stats (some of which is in use and other is available) @@ -398,21 +425,21 @@ class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure class ShenandoahGenerationStatsClosure : public ShenandoahHeapRegionClosure { public: - ShenandoahCalculateRegionStatsClosure old; - ShenandoahCalculateRegionStatsClosure young; - ShenandoahCalculateRegionStatsClosure global; + ShenandoahCalculateRegionStatsClosure _old; + ShenandoahCalculateRegionStatsClosure _young; + ShenandoahCalculateRegionStatsClosure _global; void heap_region_do(ShenandoahHeapRegion* r) override { switch (r->affiliation()) { case FREE: return; case YOUNG_GENERATION: - young.heap_region_do(r); - global.heap_region_do(r); + _young.heap_region_do(r); + _global.heap_region_do(r); break; case OLD_GENERATION: - old.heap_region_do(r); - global.heap_region_do(r); + _old.heap_region_do(r); + _global.heap_region_do(r); break; default: ShouldNotReachHere(); @@ -426,23 +453,22 @@ class ShenandoahGenerationStatsClosure : public ShenandoahHeapRegionClosure { byte_size_in_proper_unit(stats.used()), proper_unit_for_byte_size(stats.used())); } - static void validate_usage(const bool adjust_for_padding, + static void validate_usage(const bool adjust_for_padding, const bool adjust_for_trash, const char* label, ShenandoahGeneration* generation, ShenandoahCalculateRegionStatsClosure& stats) { ShenandoahHeap* heap = ShenandoahHeap::heap(); size_t generation_used = generation->used(); size_t generation_used_regions = generation->used_regions(); - if (adjust_for_padding && (generation->is_young() || generation->is_global())) { - size_t pad = heap->old_generation()->get_pad_for_promote_in_place(); - generation_used += pad; - } - guarantee(stats.used() == generation_used, + size_t stats_used = adjust_for_trash? stats.used_after_recycle(): stats.used(); + guarantee(stats_used == generation_used, "%s: generation (%s) used size must be consistent: generation-used: " PROPERFMT ", regions-used: " PROPERFMT, - label, generation->name(), PROPERFMTARGS(generation_used), PROPERFMTARGS(stats.used())); + label, generation->name(), PROPERFMTARGS(generation_used), PROPERFMTARGS(stats_used)); - guarantee(stats.regions() == generation_used_regions, - "%s: generation (%s) used regions (%zu) must equal regions that are in use (%zu)", - label, generation->name(), generation->used_regions(), stats.regions()); + size_t stats_regions = adjust_for_trash? stats.regions() - stats.trashed_regions(): stats.regions(); + guarantee(stats_regions == generation_used_regions, + "%s: generation (%s) used regions (%zu) must equal regions that are in use (%zu)%s", + label, generation->name(), generation->used_regions(), stats_regions, + adjust_for_trash? " (after adjusting for trash)": ""); size_t generation_capacity = generation->max_capacity(); guarantee(stats.non_trashed_span() <= generation_capacity, @@ -463,11 +489,11 @@ class ShenandoahVerifyHeapRegionClosure : public ShenandoahHeapRegionClosure { ShenandoahHeap* _heap; const char* _phase; ShenandoahVerifier::VerifyRegions _regions; -public: + public: ShenandoahVerifyHeapRegionClosure(const char* phase, ShenandoahVerifier::VerifyRegions regions) : - _heap(ShenandoahHeap::heap()), - _phase(phase), - _regions(regions) {}; + _heap(ShenandoahHeap::heap()), + _phase(phase), + _regions(regions) {}; void print_failure(ShenandoahHeapRegion* r, const char* label) { ResourceMark rm; @@ -620,7 +646,7 @@ class ShenandoahVerifierReachableTask : public WorkerTask { }; class ShenandoahVerifyNoIncompleteSatbBuffers : public ThreadClosure { -public: + public: void do_thread(Thread* thread) override { SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread); if (!queue.is_empty()) { @@ -630,7 +656,7 @@ class ShenandoahVerifyNoIncompleteSatbBuffers : public ThreadClosure { }; class ShenandoahVerifierMarkedRegionTask : public WorkerTask { -private: + private: const char* _label; ShenandoahVerifier::VerifyOptions _options; ShenandoahHeap *_heap; @@ -760,7 +786,7 @@ class ShenandoahVerifierMarkedRegionTask : public WorkerTask { class VerifyThreadGCState : public ThreadClosure { private: const char* const _label; - char const _expected; + char const _expected; public: VerifyThreadGCState(const char* label, char expected) : _label(label), _expected(expected) {} @@ -864,16 +890,18 @@ void ShenandoahVerifier::verify_at_safepoint(ShenandoahGeneration* generation, size_t heap_used; if (_heap->mode()->is_generational() && (sizeness == _verify_size_adjusted_for_padding)) { // Prior to evacuation, regular regions that are to be evacuated in place are padded to prevent further allocations - heap_used = _heap->used() + _heap->old_generation()->get_pad_for_promote_in_place(); + // but this padding is already represented in _heap->used() + heap_used = _heap->used(); } else if (sizeness != _verify_size_disable) { heap_used = _heap->used(); } if (sizeness != _verify_size_disable) { - guarantee(cl.used() == heap_used, + size_t cl_size = (sizeness == _verify_size_exact_including_trash)? cl.used(): cl.used_after_recycle(); + guarantee(cl_size == heap_used, "%s: heap used size must be consistent: heap-used = %zu%s, regions-used = %zu%s", label, byte_size_in_proper_unit(heap_used), proper_unit_for_byte_size(heap_used), - byte_size_in_proper_unit(cl.used()), proper_unit_for_byte_size(cl.used())); + byte_size_in_proper_unit(cl_size), proper_unit_for_byte_size(cl_size)); } size_t heap_committed = _heap->committed(); guarantee(cl.committed() == heap_committed, @@ -911,18 +939,19 @@ void ShenandoahVerifier::verify_at_safepoint(ShenandoahGeneration* generation, _heap->heap_region_iterate(&cl); if (LogTarget(Debug, gc)::is_enabled()) { - ShenandoahGenerationStatsClosure::log_usage(_heap->old_generation(), cl.old); - ShenandoahGenerationStatsClosure::log_usage(_heap->young_generation(), cl.young); - ShenandoahGenerationStatsClosure::log_usage(_heap->global_generation(), cl.global); + ShenandoahGenerationStatsClosure::log_usage(_heap->old_generation(), cl._old); + ShenandoahGenerationStatsClosure::log_usage(_heap->young_generation(), cl._young); + ShenandoahGenerationStatsClosure::log_usage(_heap->global_generation(), cl._global); } if (sizeness == _verify_size_adjusted_for_padding) { - ShenandoahGenerationStatsClosure::validate_usage(false, label, _heap->old_generation(), cl.old); - ShenandoahGenerationStatsClosure::validate_usage(true, label, _heap->young_generation(), cl.young); - ShenandoahGenerationStatsClosure::validate_usage(true, label, _heap->global_generation(), cl.global); - } else if (sizeness == _verify_size_exact) { - ShenandoahGenerationStatsClosure::validate_usage(false, label, _heap->old_generation(), cl.old); - ShenandoahGenerationStatsClosure::validate_usage(false, label, _heap->young_generation(), cl.young); - ShenandoahGenerationStatsClosure::validate_usage(false, label, _heap->global_generation(), cl.global); + ShenandoahGenerationStatsClosure::validate_usage(false, true, label, _heap->old_generation(), cl._old); + ShenandoahGenerationStatsClosure::validate_usage(true, true, label, _heap->young_generation(), cl._young); + ShenandoahGenerationStatsClosure::validate_usage(true, true, label, _heap->global_generation(), cl._global); + } else if (sizeness == _verify_size_exact || sizeness == _verify_size_exact_including_trash) { + bool adjust_trash = (sizeness == _verify_size_exact); + ShenandoahGenerationStatsClosure::validate_usage(false, adjust_trash, label, _heap->old_generation(), cl._old); + ShenandoahGenerationStatsClosure::validate_usage(false, adjust_trash, label, _heap->young_generation(), cl._young); + ShenandoahGenerationStatsClosure::validate_usage(false, adjust_trash, label, _heap->global_generation(), cl._global); } // else: sizeness must equal _verify_size_disable } @@ -1139,7 +1168,8 @@ void ShenandoahVerifier::verify_after_update_refs(ShenandoahGeneration* generati _verify_cset_none, // no cset references, all updated _verify_liveness_disable, // no reliable liveness data anymore _verify_regions_nocset, // no cset regions, trash regions have appeared - _verify_size_exact, // expect generation and heap sizes to match exactly + // expect generation and heap sizes to match exactly, including trash + _verify_size_exact_including_trash, _verify_gcstate_stable // update refs had cleaned up forwarded objects ); } @@ -1405,7 +1435,7 @@ void ShenandoahVerifier::verify_before_rebuilding_free_set() { ShenandoahGenerationStatsClosure cl; _heap->heap_region_iterate(&cl); - ShenandoahGenerationStatsClosure::validate_usage(false, "Before free set rebuild", _heap->old_generation(), cl.old); - ShenandoahGenerationStatsClosure::validate_usage(false, "Before free set rebuild", _heap->young_generation(), cl.young); - ShenandoahGenerationStatsClosure::validate_usage(false, "Before free set rebuild", _heap->global_generation(), cl.global); + ShenandoahGenerationStatsClosure::validate_usage(false, true, "Before free set rebuild", _heap->old_generation(), cl._old); + ShenandoahGenerationStatsClosure::validate_usage(false, true, "Before free set rebuild", _heap->young_generation(), cl._young); + ShenandoahGenerationStatsClosure::validate_usage(false, true, "Before free set rebuild", _heap->global_generation(), cl._global); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp index e49990fdc620d..f66d7bbec7752 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp @@ -155,7 +155,10 @@ class ShenandoahVerifier : public CHeapObj { _verify_size_exact, // Expect promote-in-place adjustments: padding inserted to temporarily prevent further allocation in regular regions - _verify_size_adjusted_for_padding + _verify_size_adjusted_for_padding, + + // Expected heap size should not include + _verify_size_exact_including_trash } VerifySize; typedef enum { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp index 849ab691dcf47..86be2fc60be59 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp @@ -30,9 +30,10 @@ #include "gc/shenandoah/shenandoahUtils.hpp" #include "gc/shenandoah/shenandoahYoungGeneration.hpp" -ShenandoahYoungGeneration::ShenandoahYoungGeneration(uint max_queues, size_t max_capacity) : - ShenandoahGeneration(YOUNG, max_queues, max_capacity), +ShenandoahYoungGeneration::ShenandoahYoungGeneration(uint max_queues) : + ShenandoahGeneration(YOUNG, max_queues), _old_gen_task_queues(nullptr) { + assert(type() == ShenandoahGenerationType::YOUNG, "OO sanity"); } void ShenandoahYoungGeneration::set_concurrent_mark_in_progress(bool in_progress) { @@ -95,6 +96,41 @@ ShenandoahHeuristics* ShenandoahYoungGeneration::initialize_heuristics(Shenandoa return _heuristics; } +size_t ShenandoahYoungGeneration::used() const { + return _free_set->young_used(); +} + +size_t ShenandoahYoungGeneration::bytes_allocated_since_gc_start() const { + assert(ShenandoahHeap::heap()->mode()->is_generational(), "Young implies generational"); + return _free_set->get_bytes_allocated_since_gc_start(); +} + +size_t ShenandoahYoungGeneration::get_affiliated_region_count() const { + return _free_set->young_affiliated_regions(); +} + +size_t ShenandoahYoungGeneration::get_humongous_waste() const { + return _free_set->humongous_waste_in_mutator(); +} + +size_t ShenandoahYoungGeneration::used_regions() const { + return _free_set->young_affiliated_regions(); +} + +size_t ShenandoahYoungGeneration::used_regions_size() const { + size_t used_regions = _free_set->young_affiliated_regions(); + return used_regions * ShenandoahHeapRegion::region_size_bytes(); +} + +size_t ShenandoahYoungGeneration::max_capacity() const { + size_t total_regions = _free_set->total_young_regions(); + return total_regions * ShenandoahHeapRegion::region_size_bytes(); +} + +size_t ShenandoahYoungGeneration::free_unaffiliated_regions() const { + return _free_set->young_unaffiliated_regions(); +} + size_t ShenandoahYoungGeneration::available() const { // The collector reserve may eat into what the mutator is allowed to use. Make sure we are looking // at what is available to the mutator when reporting how much memory is available. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp index 14f9a9b3004ea..6b2794b12c337 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp @@ -34,7 +34,7 @@ class ShenandoahYoungGeneration : public ShenandoahGeneration { ShenandoahYoungHeuristics* _young_heuristics; public: - ShenandoahYoungGeneration(uint max_queues, size_t max_capacity); + ShenandoahYoungGeneration(uint max_queues); ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode) override; @@ -73,10 +73,16 @@ class ShenandoahYoungGeneration : public ShenandoahGeneration { return _old_gen_task_queues != nullptr; } - size_t available() const override; - - // Do not override available_with_reserve() because that needs to see memory reserved for Collector + size_t bytes_allocated_since_gc_start() const override; + size_t used() const override; + size_t used_regions() const override; + size_t used_regions_size() const override; + size_t get_humongous_waste() const override; + size_t free_unaffiliated_regions() const override; + size_t get_affiliated_region_count() const override; + size_t max_capacity() const override; + size_t available() const override; size_t soft_available() const override; void prepare_gc() override; diff --git a/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp b/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp index a245f91fa71e9..3968575d089f2 100644 --- a/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp +++ b/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp @@ -34,9 +34,8 @@ nonstatic_field(ShenandoahHeap, _num_regions, size_t) \ nonstatic_field(ShenandoahHeap, _regions, ShenandoahHeapRegion**) \ nonstatic_field(ShenandoahHeap, _log_min_obj_alignment_in_bytes, int) \ - nonstatic_field(ShenandoahHeap, _global_generation, ShenandoahGeneration*) \ + nonstatic_field(ShenandoahHeap, _free_set, ShenandoahFreeSet*) \ volatile_nonstatic_field(ShenandoahHeap, _committed, size_t) \ - volatile_nonstatic_field(ShenandoahGeneration, _used, size_t) \ static_field(ShenandoahHeapRegion, RegionSizeBytes, size_t) \ static_field(ShenandoahHeapRegion, RegionSizeBytesShift, size_t) \ volatile_nonstatic_field(ShenandoahHeapRegion, _state, ShenandoahHeapRegion::RegionState) \ @@ -44,6 +43,7 @@ nonstatic_field(ShenandoahHeapRegion, _bottom, HeapWord* const) \ nonstatic_field(ShenandoahHeapRegion, _top, HeapWord*) \ nonstatic_field(ShenandoahHeapRegion, _end, HeapWord* const) \ + nonstatic_field(ShenandoahFreeSet, _total_global_used, size_t) \ #define VM_INT_CONSTANTS_SHENANDOAH(declare_constant, declare_constant_with_value) \ declare_constant(ShenandoahHeapRegion::_empty_uncommitted) \ @@ -66,7 +66,7 @@ declare_toplevel_type(ShenandoahHeap*) \ declare_toplevel_type(ShenandoahHeapRegion*) \ declare_toplevel_type(ShenandoahHeapRegion::RegionState) \ - declare_toplevel_type(ShenandoahGeneration) \ - declare_toplevel_type(ShenandoahGeneration*) \ + declare_toplevel_type(ShenandoahFreeSet) \ + declare_toplevel_type(ShenandoahFreeSet*) \ #endif // SHARE_GC_SHENANDOAH_VMSTRUCTS_SHENANDOAH_HPP diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahGeneration.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahFreeSet.java similarity index 89% rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahGeneration.java rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahFreeSet.java index bcd59523ae088..9ca8e1deb04e3 100644 --- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahGeneration.java +++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahFreeSet.java @@ -34,7 +34,7 @@ import sun.jvm.hotspot.types.Type; import sun.jvm.hotspot.types.TypeDataBase; -public class ShenandoahGeneration extends VMObject { +public class ShenandoahFreeSet extends VMObject { private static CIntegerField used; static { VM.registerVMInitializedObserver(new Observer() { @@ -45,11 +45,11 @@ public void update(Observable o, Object data) { } private static synchronized void initialize(TypeDataBase db) { - Type type = db.lookupType("ShenandoahGeneration"); - used = type.getCIntegerField("_used"); + Type type = db.lookupType("ShenandoahFreeSet"); + used = type.getCIntegerField("_total_global_used"); } - public ShenandoahGeneration(Address addr) { + public ShenandoahFreeSet(Address addr) { super(addr); } diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahHeap.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahHeap.java index 3109fe2210290..abce4a7d024bb 100644 --- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahHeap.java +++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahHeap.java @@ -43,7 +43,7 @@ public class ShenandoahHeap extends CollectedHeap { private static CIntegerField numRegions; - private static AddressField globalGeneration; + private static AddressField globalFreeSet; private static CIntegerField committed; private static AddressField regions; private static CIntegerField logMinObjAlignmentInBytes; @@ -60,7 +60,7 @@ public void update(Observable o, Object data) { private static synchronized void initialize(TypeDataBase db) { Type type = db.lookupType("ShenandoahHeap"); numRegions = type.getCIntegerField("_num_regions"); - globalGeneration = type.getAddressField("_global_generation"); + globalFreeSet = type.getAddressField("_free_set"); committed = type.getCIntegerField("_committed"); regions = type.getAddressField("_regions"); logMinObjAlignmentInBytes = type.getCIntegerField("_log_min_obj_alignment_in_bytes"); @@ -89,9 +89,9 @@ public long capacity() { @Override public long used() { - Address globalGenerationAddress = globalGeneration.getValue(addr); - ShenandoahGeneration global = VMObjectFactory.newObject(ShenandoahGeneration.class, globalGenerationAddress); - return global.used(); + Address globalFreeSetAddress = globalFreeSet.getValue(addr); + ShenandoahFreeSet freeset = VMObjectFactory.newObject(ShenandoahFreeSet.class, globalFreeSetAddress); + return freeset.used(); } public long committed() { diff --git a/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldGeneration.cpp b/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldGeneration.cpp index 2fd87b48dc2c7..b2491226e5cab 100644 --- a/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldGeneration.cpp +++ b/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldGeneration.cpp @@ -52,7 +52,7 @@ class ShenandoahOldGenerationTest : public ::testing::Test { ShenandoahHeap::heap()->lock()->lock(false); - old = new ShenandoahOldGeneration(8, 1024 * 1024); + old = new ShenandoahOldGeneration(8); old->set_promoted_reserve(512 * HeapWordSize); old->expend_promoted(256 * HeapWordSize); old->set_evacuation_reserve(512 * HeapWordSize); diff --git a/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp b/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp index d08cfd7618bb0..b661bd6bc5717 100644 --- a/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp +++ b/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp @@ -84,7 +84,8 @@ class ShenandoahOldHeuristicTest : public ::testing::Test { _heap->lock()->lock(false); ShenandoahResetRegions reset; _heap->heap_region_iterate(&reset); - _heap->old_generation()->set_capacity(ShenandoahHeapRegion::region_size_bytes() * 10); + // _heap->old_generation()->set_capacity(ShenandoahHeapRegion::region_size_bytes() * 10) + _heap->free_set()->resize_old_collector_capacity(10); _heap->old_generation()->set_evacuation_reserve(ShenandoahHeapRegion::region_size_bytes() * 4); _heuristics->abandon_collection_candidates(); _collection_set->clear(); diff --git a/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java b/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java index c3b09f5ab4c20..79259168bf316 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java @@ -144,12 +144,25 @@ * @requires vm.gc.Shenandoah * @library /test/lib * - * @run main/othervm/timeout=240 -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm/timeout=480 -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions * -XX:+UseShenandoahGC * -XX:-UseTLAB -XX:+ShenandoahVerify * TestSieveObjects */ +/* + * @test id=no-tlab-genshen + * @summary Acceptance tests: collector can deal with retained objects + * @key randomness + * @requires vm.gc.Shenandoah + * @library /test/lib + * + * @run main/othervm/timeout=480 -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:-UseTLAB -XX:+ShenandoahVerify + * TestSieveObjects + */ + import java.util.Random; import jdk.test.lib.Utils; diff --git a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java index bd3c4508c7b84..6473c310ec796 100644 --- a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java +++ b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java @@ -112,8 +112,10 @@ public class TestChurnNotifications { - static final long HEAP_MB = 128; // adjust for test configuration above - static final long TARGET_MB = Long.getLong("target", 2_000); // 2 Gb allocation + static final long HEAP_MB = 128; // adjust for test configuration above + static final long TARGET_MB = Long.getLong("target", 2_000); // 2 Gb allocation + static final long ANTICIPATED_HUMONGOUS_WASTE_PER_ARRAY = 124_272; + // Should we track the churn precisely? // Precise tracking is only reliable when GC is fully stop-the-world. Otherwise, @@ -159,7 +161,7 @@ public void handleNotification(Notification n, Object o) { final int size = 100_000; long count = TARGET_MB * 1024 * 1024 / (16 + 4 * size); - long mem = count * (16 + 4 * size); + long mem = count * (16 + 4 * size + ANTICIPATED_HUMONGOUS_WASTE_PER_ARRAY); for (int c = 0; c < count; c++) { sink = new int[size];