[GR-32941] Allocate TLABs from chunks instead of as chunks

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveWeightedAverage.java

@@ -24,8 +24,11 @@
  */
 package com.oracle.svm.core.genscavenge;
 
+import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
+
 import org.graalvm.word.UnsignedWord;
 
+import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.util.UnsignedUtils;
 
 /**
@@ -101,7 +104,8 @@ protected double computeAdaptiveAverage(double sample, double avg) {
         return expAvg(avg, sample, adaptiveWeight);
     }
 
-    private static double expAvg(double avg, double sample, double adaptiveWeight) {
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    static double expAvg(double avg, double sample, double adaptiveWeight) {
         assert adaptiveWeight > 0 && adaptiveWeight <= 100 : "weight must be a percentage";
         return (100.0 - adaptiveWeight) * avg / 100.0 + adaptiveWeight * sample / 100.0;
     }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveWeightedAverageStruct.java

@@ -0,0 +1,119 @@
+/*
+ * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.core.genscavenge;
+
+import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
+import static com.oracle.svm.core.genscavenge.AdaptiveWeightedAverage.OLD_THRESHOLD;
+
+import org.graalvm.nativeimage.c.struct.RawField;
+import org.graalvm.nativeimage.c.struct.RawStructure;
+import org.graalvm.word.PointerBase;
+import org.graalvm.word.UnsignedWord;
+
+import com.oracle.svm.core.Uninterruptible;
+import com.oracle.svm.core.jdk.UninterruptibleUtils;
+import com.oracle.svm.core.util.UnsignedUtils;
+
+/**
+ * This class provides a raw structure implementation of {@link AdaptiveWeightedAverage}. For
+ * further information see {@link AdaptiveWeightedAverage}.
+ */
+class AdaptiveWeightedAverageStruct {
+
+    @RawStructure
+    interface Data extends PointerBase {
+
+        @RawField
+        void setWeight(double weight);
+
+        @RawField
+        double getWeight();
+
+        @RawField
+        void setAverage(double average);
+
+        @RawField
+        double getAverage();
+
+        @RawField
+        void setSampleCount(long sampleCount);
+
+        @RawField
+        long getSampleCount();
+
+        @RawField
+        void setIsOld(boolean isOld);
+
+        @RawField
+        boolean getIsOld();
+    }
+
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    static void initialize(Data data, double weight) {
+        initialize(data, weight, 0);
+    }
+
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    static void initialize(Data data, double weight, double avg) {
+        assert weight > 0 && weight <= 100;
+        data.setWeight(weight);
+        data.setAverage(avg);
+    }
+
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    public static double getAverage(Data data) {
+        return data.getAverage();
+    }
+
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    public static void sample(Data data, double value) {
+        data.setSampleCount(data.getSampleCount() + 1);
+        if (!data.getIsOld() && data.getSampleCount() > OLD_THRESHOLD) {
+            data.setIsOld(true);
+        }
+        data.setAverage(computeAdaptiveAverage(data, value, data.getAverage()));
+    }
+
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    public static void sample(Data data, UnsignedWord value) {
+        sample(data, UnsignedUtils.toDouble(value));
+    }
+
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    protected static double computeAdaptiveAverage(Data data, double sample, double avg) {
+        /*
+         * We smoothen the samples by not using weight directly until we've had enough data to make
+         * it meaningful. We'd like the first weight used to be 1, the second to be 1/2, etc until
+         * we have OLD_THRESHOLD/weight samples.
+         */
+        double countWeight = 0;
+        if (!data.getIsOld()) { // avoid division by zero if the counter wraps
+            countWeight = OLD_THRESHOLD / (double) data.getSampleCount();
+        }
+        double adaptiveWeight = UninterruptibleUtils.Math.max(data.getWeight(), countWeight);
+        return AdaptiveWeightedAverage.expAvg(avg, sample, adaptiveWeight);
+    }
+
+}

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AddressRangeCommittedMemoryProvider.java

@@ -351,6 +351,7 @@ protected OutOfMemoryError reportAlignedChunkAllocationFailed(int error) {
     }
 
     @Override
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     public Pointer allocateUnalignedChunk(UnsignedWord nbytes) {
         WordPointer allocOut = UnsafeStackValue.get(WordPointer.class);
         int error = allocateInHeapAddressSpace(nbytes, getAlignmentForUnalignedChunks(), allocOut);

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java

@@ -221,10 +221,7 @@ public static final class OnlyCompletely extends BasicPolicy {
 
         @Override
         public boolean shouldCollectCompletely(boolean followingIncrementalCollection) {
-            if (!followingIncrementalCollection && shouldCollectYoungGenSeparately(false)) {
-                return false;
-            }
-            return true;
+            return followingIncrementalCollection || !shouldCollectYoungGenSeparately(false);
         }
 
         @Override

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/FillerObjectUtil.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.core.genscavenge;
+
+import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
+import static jdk.graal.compiler.replacements.AllocationSnippets.FillContent.WITH_GARBAGE_IF_ASSERTIONS_ENABLED;
+
+import org.graalvm.word.Pointer;
+import org.graalvm.word.UnsignedWord;
+
+import com.oracle.svm.core.Uninterruptible;
+import com.oracle.svm.core.config.ConfigurationValues;
+import com.oracle.svm.core.genscavenge.graal.nodes.FormatArrayNode;
+import com.oracle.svm.core.genscavenge.graal.nodes.FormatObjectNode;
+import com.oracle.svm.core.heap.FillerArray;
+import com.oracle.svm.core.heap.FillerObject;
+import com.oracle.svm.core.hub.LayoutEncoding;
+import com.oracle.svm.core.util.UnsignedUtils;
+
+import jdk.graal.compiler.api.replacements.Fold;
+import jdk.graal.compiler.core.common.NumUtil;
+import jdk.graal.compiler.word.Word;
+import jdk.vm.ci.meta.JavaKind;
+
+public class FillerObjectUtil {
+    private static final Class<?> ARRAY_CLASS = FillerArray.class;
+    private static final JavaKind ARRAY_ELEMENT_KIND = JavaKind.Int;
+    private static final int ARRAY_ELEMENT_SIZE = ARRAY_ELEMENT_KIND.getByteCount();
+
+    @Fold
+    public static UnsignedWord objectMinSize() {
+        return Word.unsigned(ConfigurationValues.getObjectLayout().getMinImageHeapObjectSize());
+    }
+
+    @Fold
+    static int arrayMinSize() {
+        return NumUtil.safeToInt(ConfigurationValues.getObjectLayout().getArraySize(ARRAY_ELEMENT_KIND, 0, false));
+    }
+
+    @Fold
+    static int arrayBaseOffset() {
+        return ConfigurationValues.getObjectLayout().getArrayBaseOffset(ARRAY_ELEMENT_KIND);
+    }
+
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    public static void writeFillerObjectAt(Pointer p, UnsignedWord size) {
+        assert size.aboveThan(0);
+        if (size.aboveOrEqual(arrayMinSize())) {
+            int length = UnsignedUtils.safeToInt(size.subtract(arrayBaseOffset()).unsignedDivide(ARRAY_ELEMENT_SIZE));
+            FormatArrayNode.formatArray(p, ARRAY_CLASS, length, true, false, WITH_GARBAGE_IF_ASSERTIONS_ENABLED, false);
+        } else {
+            FormatObjectNode.formatObject(p, FillerObject.class, true, WITH_GARBAGE_IF_ASSERTIONS_ENABLED, false);
+        }
+        assert LayoutEncoding.getSizeFromObjectInGC(p.toObject()).equal(size);
+    }
+}

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java

@@ -245,7 +245,8 @@ assert getCollectionEpoch().equal(data.getRequestingEpoch()) ||
 
         Timer collectionTimer = timers.collection.start();
         try {
-            ThreadLocalAllocation.disableAndFlushForAllThreads();
+            HeapImpl.getHeapImpl().makeParseable();
+
             GenScavengeMemoryPoolMXBeans.singleton().notifyBeforeCollection();
             HeapImpl.getAccounting().notifyBeforeCollection();
 
@@ -259,6 +260,7 @@ assert getCollectionEpoch().equal(data.getRequestingEpoch()) ||
             collectionTimer.stop();
         }
 
+        resizeAllTlabs();
         accounting.updateCollectionCountAndTime(completeCollection, collectionTimer.totalNanos());
         HeapImpl.getAccounting().notifyAfterCollection();
         GenScavengeMemoryPoolMXBeans.singleton().notifyAfterCollection();
@@ -401,6 +403,14 @@ public static UnsignedWord getChunkBytes() {
         return youngBytes.add(oldBytes);
     }
 
+    private static void resizeAllTlabs() {
+        if (SubstrateGCOptions.TlabOptions.ResizeTLAB.getValue()) {
+            for (IsolateThread thread = VMThreads.firstThread(); thread.isNonNull(); thread = VMThreads.nextThread(thread)) {
+                TlabSupport.resize(thread);
+            }
+        }
+    }
+
     private void printGCBefore(GCCause cause) {
         if (!SubstrateGCOptions.VerboseGC.getValue()) {
             return;
@@ -1316,7 +1326,7 @@ protected PrintGCSummaryOperation() {
 
         @Override
         protected void operate() {
-            ThreadLocalAllocation.disableAndFlushForAllThreads();
+            HeapImpl.getHeapImpl().makeParseable();
 
             Log log = Log.log();
             log.string("GC summary").indent(true);