diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/StackValue.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/StackValue.java index bef78cd3e115..cf5a0e71e1c0 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/StackValue.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/StackValue.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -72,7 +72,7 @@ private StackValue() { * 
      * ComplexValue numberOnStack = StackValue.get(ComplexValue.class);
      * numberOnStack.realPart(3.0);
-     * numberOnStack.imagineryPart(4.0);
+     * numberOnStack.imaginaryPart(4.0);
      * double absoluteValue = absoluteValue(numberOnStack);
      * assert 5.0 == absoluteValue;
      *
diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/c/struct/CStruct.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/c/struct/CStruct.java index 9cdd26aed243..ea7f2af783cc 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/c/struct/CStruct.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/c/struct/CStruct.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -67,10 +67,10 @@ * void realPart(double re); * * @CField("im") - * double imagineryPart(); + * double imaginaryPart(); * * @CField("im") - * void imagineryPart(double im); + * void imaginaryPart(double im); * } * * diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java index a7b2df07733d..1c81aed263b6 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.genscavenge; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; import static com.oracle.svm.core.snippets.KnownIntrinsics.readCallerStackPointer; import org.graalvm.nativeimage.IsolateThread; @@ -52,12 +53,16 @@ import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.heap.ObjectHeader; import com.oracle.svm.core.heap.ObjectVisitor; +import com.oracle.svm.core.hub.DynamicHub; +import com.oracle.svm.core.hub.HubType; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.metaspace.Metaspace; +import com.oracle.svm.core.snippets.KnownIntrinsics; import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.threadlocal.VMThreadLocalSupport; import com.oracle.svm.core.util.Timer; +import jdk.graal.compiler.nodes.java.ArrayLengthNode; import jdk.graal.compiler.word.Word; /** @@ -111,6 +116,7 @@ final class CompactingOldGeneration extends OldGeneration { private final Space space = new Space("Old", "O", false, getAge()); private final MarkStack markStack = new MarkStack(); + private final MarkStack arrayMarkStack = new MarkStack(); private final GreyObjectsWalker toGreyObjectsWalker = new GreyObjectsWalker(); private final PlanningVisitor planningVisitor = new PlanningVisitor(); @@ -158,17 +164,56 @@ boolean scanGreyObjects(boolean incrementalGc) { } toGreyObjectsWalker.walkGreyObjects(); } else { - if (markStack.isEmpty()) { + if (markStack.isEmpty() && arrayMarkStack.isEmpty()) { return false; } GreyToBlackObjectVisitor visitor = GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); do { - visitor.visitObject(markStack.pop()); - } while (!markStack.isEmpty()); + while (!markStack.isEmpty()) { + visitor.visitObject(markStack.popObject()); + } + + // Process array ranges one at a time to avoid bloating the marking stack + if (!arrayMarkStack.isEmpty()) { + scanArrayRange(visitor); + } + } while (!markStack.isEmpty() || !arrayMarkStack.isEmpty()); } return true; } + @AlwaysInline("GC performance") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private void pushOntoMarkStack(Object obj) { + DynamicHub objHub = KnownIntrinsics.readHub(obj); + if (objHub.getHubType() == HubType.OBJECT_ARRAY) { + if (ArrayLengthNode.arrayLength(obj) != 0) { + arrayMarkStack.pushObject(obj); + arrayMarkStack.pushInt(0); + } + } else { + markStack.pushObject(obj); + } + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private void scanArrayRange(GreyToBlackObjectVisitor visitor) { + int index = arrayMarkStack.popInt(); + Object array = arrayMarkStack.popObject(); + + int length = ArrayLengthNode.arrayLength(array); + final int stride = 2048; + int endIndex = index + stride; + if (endIndex < length) { + arrayMarkStack.pushObject(array); + arrayMarkStack.pushInt(endIndex); + } else { + endIndex = length; + } + + visitor.visitObjectArrayRange(array, index, endIndex - index); + } + @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override @@ -196,7 +241,7 @@ public Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHead assert !ObjectHeaderImpl.hasIdentityHashFromAddressInline(oh.readHeaderFromObject(result)); } ObjectHeaderImpl.setMarked(result); - markStack.push(result); + pushOntoMarkStack(result); return result; } @@ -212,7 +257,7 @@ protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.Unal assert originalSpace == space; if (!ObjectHeaderImpl.isMarked(original)) { ObjectHeaderImpl.setMarked(original); - markStack.push(original); + pushOntoMarkStack(original); } return original; } @@ -238,7 +283,7 @@ protected boolean promotePinnedObject(Object obj, HeapChunk.Header originalCh ((AlignedHeapChunk.AlignedHeader) originalChunk).setShouldSweepInsteadOfCompact(true); } ObjectHeaderImpl.setMarked(obj); - markStack.push(obj); + pushOntoMarkStack(obj); return true; } @@ -507,17 +552,20 @@ boolean verifySpaces() { @Override void checkSanityBeforeCollection() { assert markStack.isEmpty(); + assert arrayMarkStack.isEmpty(); } @Override void checkSanityAfterCollection() { assert markStack.isEmpty(); + assert arrayMarkStack.isEmpty(); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void tearDown() { markStack.tearDown(); + arrayMarkStack.tearDown(); space.tearDown(); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index a07f09f67ba8..f7cffa183045 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.genscavenge; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -53,4 +55,9 @@ public void visitObject(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void visitObjectArrayRange(Object o, int firstIndex, int count) { + InteriorObjRefWalker.walkObjectArrayRangeInline(o, firstIndex, count, objRefVisitor); + } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/MarkStack.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/MarkStack.java index e652da9304ff..56522f0c91b8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/MarkStack.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/MarkStack.java @@ -27,13 +27,13 @@ import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; import static jdk.vm.ci.code.CodeUtil.K; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.struct.RawField; import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.nativeimage.c.struct.UniqueLocationIdentity; +import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; @@ -42,21 +42,31 @@ import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.memory.NullableNativeMemory; import com.oracle.svm.core.nmt.NmtCategory; +import com.oracle.svm.core.stack.StackOverflowCheck; import com.oracle.svm.core.util.VMError; import jdk.graal.compiler.api.replacements.Fold; import jdk.graal.compiler.word.ObjectAccess; +import jdk.graal.compiler.word.Word; /** * LIFO stack for objects to visit during the mark phase. Without it, recursive calls could exhaust - * the {@linkplain com.oracle.svm.core.stack.StackOverflowCheck yellow zone stack space} during GC. + * the {@linkplain StackOverflowCheck yellow zone stack space} during GC. Callers can also push + * other kinds of values, for example indexes for object arrays to keep track of the scanned range. */ public final class MarkStack { private static final int SEGMENT_SIZE = 64 * K - /* avoid potential malloc() overallocation */ 64; @Fold static int entriesPerSegment() { - return (SEGMENT_SIZE - SizeOf.get(Segment.class)) / ConfigurationValues.getObjectLayout().getReferenceSize(); + int headerSize = SizeOf.get(Segment.class); + assert headerSize % entrySize() == 0 : "must be aligned"; + return (SEGMENT_SIZE - headerSize) / entrySize(); + } + + @Fold + static int entrySize() { + return ConfigurationValues.getObjectLayout().getReferenceSize(); } private Segment top; @@ -68,42 +78,68 @@ public MarkStack() { @AlwaysInline("GC performance") @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public void push(Object obj) { + public void pushObject(Object obj) { assert obj != null; + Pointer entry = pushSlot(); + ObjectAccess.writeObject(Word.nullPointer(), entry, obj); + } + + @AlwaysInline("GC performance") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void pushInt(int value) { + Pointer entry = pushSlot(); + entry.writeInt(0, value); + } + @AlwaysInline("GC performance") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private Pointer pushSlot() { if (top.isNull() || cursor == entriesPerSegment()) { top = allocateSegment(top); cursor = 0; } - UnsignedWord offset = getOffsetAtIndex(cursor); - ObjectAccess.writeObject(top, offset, obj); + Pointer entry = getEntryAddress(cursor); cursor++; + return entry; } @AlwaysInline("GC performance") @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public Object pop() { + public Object popObject() { assert !isEmpty(); - cursor--; - UnsignedWord offset = getOffsetAtIndex(cursor); - Object obj = ObjectAccess.readObject(top, offset); - + Pointer entry = getEntryAddress(cursor - 1); + Object obj = ObjectAccess.readObject(Word.nullPointer(), entry); assert obj != null; + popSlot(); + return obj; + } + + @AlwaysInline("GC performance") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public int popInt() { + assert !isEmpty(); + + Pointer entry = getEntryAddress(cursor - 1); + int v = entry.readInt(0); + + popSlot(); + return v; + } + + @AlwaysInline("GC performance") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private void popSlot() { + cursor--; if (cursor == 0) { if (top.getNext().isNonNull()) { // free eagerly, use cursor==0 only if completely empty - Segment t = top; - top = top.getNext(); - cursor = entriesPerSegment(); - NullableNativeMemory.free(t); + freeTopSegment(); } else { // keep a single segment } } - - return obj; } @AlwaysInline("GC performance") @@ -134,11 +170,19 @@ private static Segment allocateSegment(Segment next) { return segment; } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private void freeTopSegment() { + Segment t = top; + top = top.getNext(); + cursor = entriesPerSegment(); + NullableNativeMemory.free(t); + } + @AlwaysInline("GC performance") @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - private static UnsignedWord getOffsetAtIndex(int index) { - int refSize = ConfigurationValues.getObjectLayout().getReferenceSize(); - return Word.unsigned(index).multiply(refSize).add(SizeOf.unsigned(Segment.class)); + private Pointer getEntryAddress(int index) { + Pointer firstEntry = ((Pointer) top).add(SizeOf.unsigned(Segment.class)); + return firstEntry.add(Word.unsigned(index).multiply(entrySize())); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java index 9463610bcef9..bb4ae29a37fa 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.hub; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import java.util.function.IntConsumer; import org.graalvm.word.Pointer; @@ -90,7 +92,8 @@ public static void walkObjectInline(Object obj, ObjectReferenceVisitor visitor) walkStoredContinuationInline(obj, visitor); return; case HubType.OBJECT_ARRAY: - walkObjectArrayInline(obj, visitor, objHub); + int length = ArrayLengthNode.arrayLength(obj); + walkObjectArrayRangeInline(obj, objHub, 0, length, visitor); return; case HubType.OTHER: default: @@ -98,6 +101,16 @@ public static void walkObjectInline(Object obj, ObjectReferenceVisitor visitor) } } + @AlwaysInline("De-virtualize calls to ObjectReferenceVisitor") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static void walkObjectArrayRangeInline(Object obj, int firstIndex, int count, ObjectReferenceVisitor visitor) { + DynamicHub objHub = KnownIntrinsics.readHub(obj); + assert objHub.getHubType() == HubType.OBJECT_ARRAY; + assert firstIndex >= 0 && count >= 0 && firstIndex + count >= 0; + assert firstIndex + count <= ArrayLengthNode.arrayLength(obj); + walkObjectArrayRangeInline(obj, objHub, firstIndex, count, visitor); + } + public static void walkInstanceReferenceOffsets(DynamicHub objHub, IntConsumer offsetConsumer) { if (objHub.getHubType() != HubType.INSTANCE && objHub.getHubType() != HubType.REFERENCE_INSTANCE) { throw new IllegalArgumentException("Unsupported hub type: " + objHub.getHubType()); @@ -166,11 +179,10 @@ private static void walkStoredContinuationInline(Object obj, ObjectReferenceVisi @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static void walkObjectArrayInline(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub) { + private static void walkObjectArrayRangeInline(Object obj, DynamicHub objHub, int firstIndex, int count, ObjectReferenceVisitor visitor) { Pointer objPointer = Word.objectToUntrackedPointer(obj); - int length = ArrayLengthNode.arrayLength(obj); - Pointer firstObjRef = objPointer.add(LayoutEncoding.getArrayBaseOffset(objHub.getLayoutEncoding())); - callVisitorInline(obj, visitor, firstObjRef, length); + Pointer firstObjRef = objPointer.add(LayoutEncoding.getArrayElementOffset(objHub.getLayoutEncoding(), firstIndex)); + callVisitorInline(obj, visitor, firstObjRef, count); } @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor")