4 changes: 2 additions & 2 deletions Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6826,7 +6826,7 @@ void SpeculativeJIT::compileNewBoundFunction(Node* node)
JumpList slowPath;

auto butterfly = TrustedImmPtr(nullptr);
emitAllocateJSObjectWithKnownSize<JSBoundFunction>(resultGPR, TrustedImmPtr(structure), butterfly, scratch1GPR, scratch2GPR, slowPath, sizeof(JSBoundFunction));
emitAllocateJSObjectWithKnownSize<JSBoundFunction>(resultGPR, TrustedImmPtr(structure), butterfly, scratch1GPR, scratch2GPR, slowPath, sizeof(JSBoundFunction), SlowAllocationResult::UndefinedBehavior);
storeLinkableConstant(LinkableConstant::globalObject(*this, node), Address(resultGPR, JSBoundFunction::offsetOfScopeChain()));
storeLinkableConstant(LinkableConstant(*this, executable), Address(resultGPR, JSBoundFunction::offsetOfExecutableOrRareData()));
storeValue(JSValueRegs { targetGPR }, Address(resultGPR, JSBoundFunction::offsetOfTargetFunction()));
Expand Down Expand Up @@ -7067,7 +7067,7 @@ void SpeculativeJIT::compileCreateClonedArguments(Node* node)
emitInitializeOutOfLineStorage(storageGPR, outOfLineCapacity, scratchGPR);
}

emitAllocateJSObject<ClonedArguments>(resultGPR, TrustedImmPtr(m_graph.registerStructure(globalObject->clonedArgumentsStructure())), storageGPR, scratchGPR, scratch2GPR, slowCases);
emitAllocateJSObject<ClonedArguments>(resultGPR, TrustedImmPtr(m_graph.registerStructure(globalObject->clonedArgumentsStructure())), storageGPR, scratchGPR, scratch2GPR, slowCases, SlowAllocationResult::UndefinedBehavior);

emitGetCallee(node->origin.semantic, scratchGPR);
storePtr(scratchGPR, Address(resultGPR, ClonedArguments::offsetOfCallee()));
Expand Down
2 changes: 1 addition & 1 deletion Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18430,7 +18430,7 @@ IGNORE_CLANG_WARNINGS_END
// all of the compiler tiers.
jit.emitAllocateWithNonNullAllocator(
params[0].gpr(), actualAllocator, allocatorGPR, params.gpScratch(0),
jumpToSlowPath);
jumpToSlowPath, CCallHelpers::SlowAllocationResult::UndefinedBehavior);

CCallHelpers::Jump jumpToSuccess;
if (!params.fallsThroughToSuccessor(0))
Expand Down
29 changes: 16 additions & 13 deletions Source/JavaScriptCore/jit/AssemblyHelpers.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -796,7 +796,7 @@ void AssemblyHelpers::emitRandomThunk(VM& vm, GPRReg scratch0, GPRReg scratch1,
}
#endif

void AssemblyHelpers::emitAllocateWithNonNullAllocator(GPRReg resultGPR, const JITAllocator& allocator, GPRReg allocatorGPR, GPRReg scratchGPR, JumpList& slowPath)
void AssemblyHelpers::emitAllocateWithNonNullAllocator(GPRReg resultGPR, const JITAllocator& allocator, GPRReg allocatorGPR, GPRReg scratchGPR, JumpList& slowPath, SlowAllocationResult slowAllocationResult)
{
if (Options::forceGCSlowPaths()) {
slowPath.append(jump());
Expand All @@ -820,12 +820,13 @@ void AssemblyHelpers::emitAllocateWithNonNullAllocator(GPRReg resultGPR, const J
#endif

#if CPU(ARM64)
// On ARM64, we can leverage instructions like load-pair and shifted-add to make loading from the free list
// On ARM64, we can leverage instructions like load-pair and shifted-add to make loading from the free list
// and extracting interval information use less instructions.

// Assert that we can use loadPairPtr for the interval bounds and nextInterval/secret.
RELEASE_ASSERT(FreeList::offsetOfIntervalEnd() - FreeList::offsetOfIntervalStart() == sizeof(uintptr_t));
RELEASE_ASSERT(FreeList::offsetOfSecret() - FreeList::offsetOfNextInterval() == sizeof(uintptr_t));
ASSERT(FreeList::offsetOfIntervalEnd() - FreeList::offsetOfIntervalStart() == sizeof(uintptr_t));
ASSERT(FreeList::offsetOfNextInterval() - FreeList::offsetOfIntervalEnd() == sizeof(uintptr_t));
ASSERT(FreeList::offsetOfSecret() - FreeList::offsetOfNextInterval() == sizeof(uintptr_t));

// Bump allocation (fast path)
loadPairPtr(allocatorGPR, TrustedImm32(LocalAllocator::offsetOfFreeList() + FreeList::offsetOfIntervalStart()), resultGPR, scratchGPR);
Expand All @@ -846,9 +847,8 @@ void AssemblyHelpers::emitAllocateWithNonNullAllocator(GPRReg resultGPR, const J
zeroPath = branchTestPtr(ResultCondition::NonZero, resultGPR, TrustedImm32(1));
xor64(Address(resultGPR, FreeCell::offsetOfScrambledBits()), scratchGPR);
addSignExtend64(resultGPR, scratchGPR, dataTempRegister);
storePtr(dataTempRegister, Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfNextInterval()));
addUnsignedRightShift64(resultGPR, scratchGPR, TrustedImm32(32), scratchGPR);
storePtr(scratchGPR, Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfIntervalEnd()));
storePairPtr(scratchGPR, dataTempRegister, allocatorGPR, TrustedImm32(LocalAllocator::offsetOfFreeList() + FreeList::offsetOfIntervalEnd()));
jump(bumpLabel);
#elif CPU(X86_64)
// On x86_64, we can leverage better support for memory operands to directly interact with the free
Expand Down Expand Up @@ -913,14 +913,17 @@ void AssemblyHelpers::emitAllocateWithNonNullAllocator(GPRReg resultGPR, const J
jump(bumpLabel);
#endif

zeroPath.link(this);
xorPtr(resultGPR, resultGPR);
slowPath.append(jump());
if (slowAllocationResult == SlowAllocationResult::ClearToNull) {
zeroPath.link(this);
move(TrustedImm32(0), resultGPR);
slowPath.append(jump());
} else
slowPath.append(zeroPath);

done.link(this);
}

void AssemblyHelpers::emitAllocate(GPRReg resultGPR, const JITAllocator& allocator, GPRReg allocatorGPR, GPRReg scratchGPR, JumpList& slowPath)
void AssemblyHelpers::emitAllocate(GPRReg resultGPR, const JITAllocator& allocator, GPRReg allocatorGPR, GPRReg scratchGPR, JumpList& slowPath, SlowAllocationResult slowAllocationResult)
{
if (allocator.isConstant()) {
if (!allocator.allocator()) {
Expand All @@ -929,10 +932,10 @@ void AssemblyHelpers::emitAllocate(GPRReg resultGPR, const JITAllocator& allocat
}
} else
slowPath.append(branchTestPtr(Zero, allocatorGPR));
emitAllocateWithNonNullAllocator(resultGPR, allocator, allocatorGPR, scratchGPR, slowPath);
emitAllocateWithNonNullAllocator(resultGPR, allocator, allocatorGPR, scratchGPR, slowPath, slowAllocationResult);
}

void AssemblyHelpers::emitAllocateVariableSized(GPRReg resultGPR, CompleteSubspace& subspace, GPRReg allocationSize, GPRReg scratchGPR1, GPRReg scratchGPR2, JumpList& slowPath)
void AssemblyHelpers::emitAllocateVariableSized(GPRReg resultGPR, CompleteSubspace& subspace, GPRReg allocationSize, GPRReg scratchGPR1, GPRReg scratchGPR2, JumpList& slowPath, SlowAllocationResult slowAllocationResult)
{
static_assert(!(MarkedSpace::sizeStep & (MarkedSpace::sizeStep - 1)), "MarkedSpace::sizeStep must be a power of two.");

Expand All @@ -944,7 +947,7 @@ void AssemblyHelpers::emitAllocateVariableSized(GPRReg resultGPR, CompleteSubspa
move(TrustedImmPtr(subspace.allocatorForSizeStep()), scratchGPR2);
loadPtr(BaseIndex(scratchGPR2, scratchGPR1, ScalePtr), scratchGPR1);

emitAllocate(resultGPR, JITAllocator::variable(), scratchGPR1, scratchGPR2, slowPath);
emitAllocate(resultGPR, JITAllocator::variable(), scratchGPR1, scratchGPR2, slowPath, slowAllocationResult);
}

void AssemblyHelpers::restoreCalleeSavesFromEntryFrameCalleeSavesBuffer(EntryFrame*& topEntryFrame)
Expand Down
34 changes: 19 additions & 15 deletions Source/JavaScriptCore/jit/AssemblyHelpers.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1901,56 +1901,60 @@ class AssemblyHelpers : public MacroAssembler {
// that allocator is non-null; allocator can be null as a signal that we don't know what the
// value of allocatorGPR is. Additionally, if the allocator is not null, then there is no need
// to populate allocatorGPR - this code will ignore the contents of allocatorGPR.
void emitAllocateWithNonNullAllocator(GPRReg resultGPR, const JITAllocator& allocator, GPRReg allocatorGPR, GPRReg scratchGPR, JumpList& slowPath);
enum class SlowAllocationResult : uint8_t {
ClearToNull,
UndefinedBehavior,
};
void emitAllocateWithNonNullAllocator(GPRReg resultGPR, const JITAllocator&, GPRReg allocatorGPR, GPRReg scratchGPR, JumpList& slowPath, SlowAllocationResult = SlowAllocationResult::ClearToNull);

void emitAllocate(GPRReg resultGPR, const JITAllocator& allocator, GPRReg allocatorGPR, GPRReg scratchGPR, JumpList& slowPath);
void emitAllocate(GPRReg resultGPR, const JITAllocator&, GPRReg allocatorGPR, GPRReg scratchGPR, JumpList& slowPath, SlowAllocationResult = SlowAllocationResult::ClearToNull);

template<typename StructureType>
void emitAllocateJSCell(GPRReg resultGPR, const JITAllocator& allocator, GPRReg allocatorGPR, StructureType structure, GPRReg scratchGPR, JumpList& slowPath)
void emitAllocateJSCell(GPRReg resultGPR, const JITAllocator& allocator, GPRReg allocatorGPR, StructureType structure, GPRReg scratchGPR, JumpList& slowPath, SlowAllocationResult slowAllocationResult = SlowAllocationResult::ClearToNull)
{
emitAllocate(resultGPR, allocator, allocatorGPR, scratchGPR, slowPath);
emitAllocate(resultGPR, allocator, allocatorGPR, scratchGPR, slowPath, slowAllocationResult);
emitStoreStructureWithTypeInfo(structure, resultGPR, scratchGPR);
}

template<typename StructureType, typename StorageType>
void emitAllocateJSObject(GPRReg resultGPR, const JITAllocator& allocator, GPRReg allocatorGPR, StructureType structure, StorageType storage, GPRReg scratchGPR, JumpList& slowPath)
void emitAllocateJSObject(GPRReg resultGPR, const JITAllocator& allocator, GPRReg allocatorGPR, StructureType structure, StorageType storage, GPRReg scratchGPR, JumpList& slowPath, SlowAllocationResult slowAllocationResult = SlowAllocationResult::ClearToNull)
{
emitAllocateJSCell(resultGPR, allocator, allocatorGPR, structure, scratchGPR, slowPath);
emitAllocateJSCell(resultGPR, allocator, allocatorGPR, structure, scratchGPR, slowPath, slowAllocationResult);
storePtr(storage, Address(resultGPR, JSObject::butterflyOffset()));
}

template<typename ClassType, typename StructureType, typename StorageType>
void emitAllocateJSObjectWithKnownSize(
VM& vm, GPRReg resultGPR, StructureType structure, StorageType storage, GPRReg scratchGPR1,
GPRReg scratchGPR2, JumpList& slowPath, size_t size)
GPRReg scratchGPR2, JumpList& slowPath, size_t size, SlowAllocationResult slowAllocationResult = SlowAllocationResult::ClearToNull)
{
Allocator allocator = allocatorForConcurrently<ClassType>(vm, size, AllocatorForMode::AllocatorIfExists);
emitAllocateJSObject(resultGPR, JITAllocator::constant(allocator), scratchGPR1, structure, storage, scratchGPR2, slowPath);
emitAllocateJSObject(resultGPR, JITAllocator::constant(allocator), scratchGPR1, structure, storage, scratchGPR2, slowPath, slowAllocationResult);
}

template<typename ClassType, typename StructureType, typename StorageType>
void emitAllocateJSObject(VM& vm, GPRReg resultGPR, StructureType structure, StorageType storage, GPRReg scratchGPR1, GPRReg scratchGPR2, JumpList& slowPath)
void emitAllocateJSObject(VM& vm, GPRReg resultGPR, StructureType structure, StorageType storage, GPRReg scratchGPR1, GPRReg scratchGPR2, JumpList& slowPath, SlowAllocationResult slowAllocationResult = SlowAllocationResult::ClearToNull)
{
emitAllocateJSObjectWithKnownSize<ClassType>(vm, resultGPR, structure, storage, scratchGPR1, scratchGPR2, slowPath, ClassType::allocationSize(0));
emitAllocateJSObjectWithKnownSize<ClassType>(vm, resultGPR, structure, storage, scratchGPR1, scratchGPR2, slowPath, ClassType::allocationSize(0), slowAllocationResult);
}

// allocationSize can be aliased with any of the other input GPRs. If it's not aliased then it
// won't be clobbered.
void emitAllocateVariableSized(GPRReg resultGPR, CompleteSubspace& subspace, GPRReg allocationSize, GPRReg scratchGPR1, GPRReg scratchGPR2, JumpList& slowPath);
void emitAllocateVariableSized(GPRReg resultGPR, CompleteSubspace&, GPRReg allocationSize, GPRReg scratchGPR1, GPRReg scratchGPR2, JumpList& slowPath, SlowAllocationResult = SlowAllocationResult::ClearToNull);

template<typename ClassType, typename StructureType>
void emitAllocateVariableSizedCell(VM& vm, GPRReg resultGPR, StructureType structure, GPRReg allocationSize, GPRReg scratchGPR1, GPRReg scratchGPR2, JumpList& slowPath)
void emitAllocateVariableSizedCell(VM& vm, GPRReg resultGPR, StructureType structure, GPRReg allocationSize, GPRReg scratchGPR1, GPRReg scratchGPR2, JumpList& slowPath, SlowAllocationResult slowAllocationResult = SlowAllocationResult::ClearToNull)
{
CompleteSubspace* subspace = subspaceForConcurrently<ClassType>(vm);
RELEASE_ASSERT_WITH_MESSAGE(subspace, "CompleteSubspace is always allocated");
emitAllocateVariableSized(resultGPR, *subspace, allocationSize, scratchGPR1, scratchGPR2, slowPath);
emitAllocateVariableSized(resultGPR, *subspace, allocationSize, scratchGPR1, scratchGPR2, slowPath, slowAllocationResult);
emitStoreStructureWithTypeInfo(structure, resultGPR, scratchGPR2);
}

template<typename ClassType, typename StructureType>
void emitAllocateVariableSizedJSObject(VM& vm, GPRReg resultGPR, StructureType structure, GPRReg allocationSize, GPRReg scratchGPR1, GPRReg scratchGPR2, JumpList& slowPath)
void emitAllocateVariableSizedJSObject(VM& vm, GPRReg resultGPR, StructureType structure, GPRReg allocationSize, GPRReg scratchGPR1, GPRReg scratchGPR2, JumpList& slowPath, SlowAllocationResult slowAllocationResult = SlowAllocationResult::ClearToNull)
{
emitAllocateVariableSizedCell<ClassType>(vm, resultGPR, structure, allocationSize, scratchGPR1, scratchGPR2, slowPath);
emitAllocateVariableSizedCell<ClassType>(vm, resultGPR, structure, allocationSize, scratchGPR1, scratchGPR2, slowPath, slowAllocationResult);
storePtr(TrustedImmPtr(nullptr), Address(resultGPR, JSObject::butterflyOffset()));
}

Expand Down
4 changes: 2 additions & 2 deletions Source/JavaScriptCore/jit/JITOpcodes.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ void JIT::emit_op_new_object(const JSInstruction* currentInstruction)

JumpList slowCases;
auto butterfly = TrustedImmPtr(nullptr);
emitAllocateJSObject(resultReg, JITAllocator::variable(), allocatorReg, structureReg, butterfly, scratchReg, slowCases);
emitAllocateJSObject(resultReg, JITAllocator::variable(), allocatorReg, structureReg, butterfly, scratchReg, slowCases, SlowAllocationResult::UndefinedBehavior);
load8(Address(structureReg, Structure::inlineCapacityOffset()), scratchReg);
emitInitializeInlineStorage(resultReg, scratchReg);
mutatorFence(*m_vm);
Expand Down Expand Up @@ -1409,7 +1409,7 @@ void JIT::emit_op_create_this(const JSInstruction* currentInstruction)

JumpList slowCases;
auto butterfly = TrustedImmPtr(nullptr);
emitAllocateJSObject(resultReg, JITAllocator::variable(), allocatorReg, structureReg, butterfly, scratchReg, slowCases);
emitAllocateJSObject(resultReg, JITAllocator::variable(), allocatorReg, structureReg, butterfly, scratchReg, slowCases, SlowAllocationResult::UndefinedBehavior);
load8(Address(structureReg, Structure::inlineCapacityOffset()), scratchReg);
emitInitializeInlineStorage(resultReg, scratchReg);
mutatorFence(*m_vm);
Expand Down