8348400: GenShen: assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || (used_regions_size() <= _max_capacity)) failed: Cannot use more than capacity #

src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp

@@ -124,7 +124,7 @@ void ShenandoahRegionPartitions::dump_bitmap() const {
                _rightmosts[int(ShenandoahFreeSetPartitionId::OldCollector)]);
   log_debug(gc)("Empty Mutator range [%zd, %zd"
                "], Empty Collector range [%zd, %zd"
-               "], Empty Old Collecto range [%zd, %zd]",
+               "], Empty Old Collector range [%zd, %zd]",
                _leftmosts_empty[int(ShenandoahFreeSetPartitionId::Mutator)],
                _rightmosts_empty[int(ShenandoahFreeSetPartitionId::Mutator)],
                _leftmosts_empty[int(ShenandoahFreeSetPartitionId::Collector)],
@@ -1293,13 +1293,22 @@ void ShenandoahFreeSet::flip_to_old_gc(ShenandoahHeapRegion* r) {
                                                ShenandoahFreeSetPartitionId::OldCollector, region_capacity);
   _partitions.assert_bounds();
   _heap->old_generation()->augment_evacuation_reserve(region_capacity);
-  bool transferred = gen_heap->generation_sizer()->transfer_to_old(1);
-  if (!transferred) {
-    log_warning(gc, free)("Forcing transfer of %zu to old reserve.", idx);
-    gen_heap->generation_sizer()->force_transfer_to_old(1);
+
+  if (r->is_young()) {
+    // If this is a trash region and is still affiliated with the young generation
+    // we need to recycle it before decreasing the capacity of the young generation.
+    // Otherwise, we may violate the constraint that the size of regions affiliated
+    // with the young generation is always less than or equal to its maximum capacity
+    if (r->is_trash()) {
+      r->try_recycle_under_lock();
+    }
+
+    const bool transferred = gen_heap->generation_sizer()->transfer_to_old(1);
+    if (!transferred) {
+      log_warning(gc, free)("Forcing transfer of %zu to old reserve.", idx);
+      gen_heap->generation_sizer()->force_transfer_to_old(1);
+    }
   }
-  // We do not ensure that the region is no longer trash, relying on try_allocate_in(), which always comes next,
-  // to recycle trash before attempting to allocate anything in the region.
 }
 
 void ShenandoahFreeSet::flip_to_gc(ShenandoahHeapRegion* r) {

src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp

@@ -1024,15 +1024,19 @@ size_t ShenandoahGeneration::decrease_capacity(size_t decrement) {
 
   _max_capacity -= decrement;
 
-  // This detects arithmetic wraparound on _used
-  assert(ShenandoahHeap::heap()->is_full_gc_in_progress() ||
-         (used_regions_size() >= used()),
-         "Affiliated regions must hold more than what is currently used");
-  assert(ShenandoahHeap::heap()->is_full_gc_in_progress() ||
-         (_used <= _max_capacity), "Cannot use more than capacity");
-  assert(ShenandoahHeap::heap()->is_full_gc_in_progress() ||
-         (used_regions_size() <= _max_capacity),
-         "Cannot use more than capacity");
+#ifdef ASSERT
+  if (!ShenandoahHeap::heap()->is_full_gc_in_progress()) {
+    assert(used_regions_size() >= used(),
+           "Affiliated regions (" EXACTFMT ") must at least hold what is currently used (" EXACTFMT ")",
+           EXACTFMTARGS(used_regions_size()), EXACTFMTARGS(used()));
+    assert(_used <= _max_capacity,
+           "Cannot use (" EXACTFMT ") more than capacity (" EXACTFMT ")",
+           EXACTFMTARGS(_used), EXACTFMTARGS(_max_capacity));
+    assert(used_regions_size() <= _max_capacity,
+           "Affiliated regions (" EXACTFMT ") cannot use more than capacity (" EXACTFMT ")",
+           EXACTFMTARGS(used_regions_size()),EXACTFMTARGS(_max_capacity));
+  }
+#endif
   return _max_capacity;
 }