diff --git a/include/swift/ABI/Coro.h b/include/swift/ABI/Coro.h index 2558360de0f8c..f31f312cfa01a 100644 --- a/include/swift/ABI/Coro.h +++ b/include/swift/ABI/Coro.h @@ -30,6 +30,8 @@ enum class CoroAllocatorKind : uint8_t { Async = 1, // malloc/free Malloc = 2, + // swift_coroFrameAlloc/free + TypedMalloc = 3, }; class CoroAllocatorFlags : public FlagSet { diff --git a/include/swift/Runtime/RuntimeFunctions.def b/include/swift/Runtime/RuntimeFunctions.def index 5ca9743bf2124..8e56ed575c465 100644 --- a/include/swift/Runtime/RuntimeFunctions.def +++ b/include/swift/Runtime/RuntimeFunctions.def @@ -2100,12 +2100,6 @@ FUNCTION(CoroFrameAlloc, Swift, swift_coroFrameAlloc, C_CC, NO_ATTRS, EFFECT(RuntimeEffect::Allocating), UNKNOWN_MEMEFFECTS) -FUNCTION(coroFrameAllocStub, Swift, swift_coroFrameAllocStub, C_CC, - AlwaysAvailable, RETURNS(Int8PtrTy), - ARGS(SizeTy, Int64Ty), - ATTRS(NoUnwind), - EFFECT(RuntimeEffect::Allocating), - UNKNOWN_MEMEFFECTS) // void *_Block_copy(void *block); FUNCTION(BlockCopy, BlocksRuntime, _Block_copy, C_CC, AlwaysAvailable, diff --git a/lib/IRGen/GenCall.cpp b/lib/IRGen/GenCall.cpp index a17adcb19fc08..7060eee1455aa 100644 --- a/lib/IRGen/GenCall.cpp +++ b/lib/IRGen/GenCall.cpp @@ -221,7 +221,11 @@ IRGenFunction::getDefaultCoroutineAllocatorKind() { return CoroAllocatorKind::Async; } if (isCoroutine()) { - return CoroAllocatorKind::Malloc; + if (getOptions().EmitTypeMallocForCoroFrame) { + return CoroAllocatorKind::TypedMalloc; + } else { + return CoroAllocatorKind::Malloc; + } } if (IGM.SwiftCoroCC != llvm::CallingConv::SwiftCoro) { // If the swiftcorocc isn't available, fall back to malloc. @@ -2491,6 +2495,12 @@ SignatureExpansionABIDetails Signature::getUncachedABIDetails( return result; } +Signature Signature::forFunction(llvm::Function *fn) { + auto sig = Signature(fn->getFunctionType(), fn->getAttributes(), + fn->getCallingConv()); + return sig; +} + Signature Signature::forCoroutineContinuation(IRGenModule &IGM, CanSILFunctionType fnType) { assert(fnType->isCoroutine()); @@ -2701,17 +2711,14 @@ irgen::getAsyncFunctionAndSize(IRGenFunction &IGF, return {fn, size}; } -std::pair -irgen::getCoroFunctionAndSize(IRGenFunction &IGF, - FunctionPointer functionPointer, - std::pair values) { - assert(values.first || values.second); +std::tuple +irgen::getCoroFunctionValues(IRGenFunction &IGF, + FunctionPointer functionPointer, + std::tuple values) { + auto [emitFunction, emitSize, emitTypeID] = values; + assert(emitFunction || emitSize || emitTypeID); assert(functionPointer.getKind() != FunctionPointer::Kind::Function); - bool emitFunction = values.first; - bool emitSize = values.second; - assert(emitFunction || emitSize); - // Ensure that the CoroFunctionPointer is not auth'd if it is not used and // that it is not auth'd more than once if it is needed. // @@ -2764,7 +2771,15 @@ irgen::getCoroFunctionAndSize(IRGenFunction &IGF, size = IGF.Builder.CreateLoad(sizePtr, IGF.IGM.Int32Ty, IGF.IGM.getPointerAlignment()); } - return {fn, size}; + + llvm::Value *typeID = nullptr; + if (emitTypeID) { + auto *sizePtr = IGF.Builder.CreateStructGEP(IGF.IGM.CoroFunctionPointerTy, + getCoroPtr(), 2); + typeID = IGF.Builder.CreateLoad(sizePtr, IGF.IGM.Int64Ty, + IGF.IGM.getPointerAlignment()); + } + return {fn, size, typeID}; } namespace { @@ -2804,16 +2819,17 @@ class SyncCallEmission final : public CallEmission { assert(!coroAllocator); if (IsCalleeAllocatedCoroutine) { - llvm::Value *bufferSize32; - std::tie(calleeFunction, bufferSize32) = - getCoroFunctionAndSize(IGF, CurCallee.getFunctionPointer()); + auto kind = IGF.getDefaultCoroutineAllocatorKind(); + llvm::Value *bufferSize32, *mallocTypeId; + std::tie(calleeFunction, bufferSize32, mallocTypeId) = + getCoroFunctionValues(IGF, CurCallee.getFunctionPointer()); auto *bufferSize = IGF.Builder.CreateZExt(bufferSize32, IGF.IGM.SizeTy); - coroStaticFrame = emitAllocYieldOnce2CoroutineFrame(IGF, bufferSize); + coroStaticFrame = + emitAllocYieldOnce2CoroutineFrame(IGF, bufferSize, mallocTypeId); // TODO: CoroutineAccessors: Optimize allocator kind (e.g. async callers // only need to use the TaskAllocator if the // coroutine is suspended across an await). - coroAllocator = emitYieldOnce2CoroutineAllocator( - IGF, IGF.getDefaultCoroutineAllocatorKind()); + coroAllocator = emitYieldOnce2CoroutineAllocator(IGF, kind); } } void end() override { super::end(); } @@ -5179,10 +5195,12 @@ void irgen::emitYieldOnce2CoroutineEntry(IRGenFunction &IGF, auto deallocFn = IGF.IGM.getOpaquePtr(getCoroDeallocFn(IGF.IGM)); auto allocFrameFn = IGF.IGM.getOpaquePtr(getCoroAllocFrameFn(IGF.IGM)); auto deallocFrameFn = IGF.IGM.getOpaquePtr(getCoroDeallocFrameFn(IGF.IGM)); + auto *typeID = IGF.getMallocTypeId(); emitRetconCoroutineEntry( IGF, fnType, buffer, llvm::Intrinsic::coro_id_retcon_once_dynamic, Size(-1) /*dynamic-to-IRGen size*/, IGF.IGM.getCoroStaticFrameAlignment(), - {cfp, allocator}, allocFn, deallocFn, {allocFrameFn, deallocFrameFn}); + {cfp, allocator}, allocFn, deallocFn, + {allocFrameFn, deallocFrameFn, typeID}); } void irgen::emitYieldOnce2CoroutineEntry( IRGenFunction &IGF, LinkEntity coroFunction, CanSILFunctionType fnType, @@ -5213,9 +5231,10 @@ Address irgen::emitAllocYieldManyCoroutineBuffer(IRGenFunction &IGF) { getYieldManyCoroutineBufferAlignment(IGF.IGM)); } -StackAddress irgen::emitAllocYieldOnce2CoroutineFrame(IRGenFunction &IGF, - llvm::Value *size) { - return emitAllocCoroStaticFrame(IGF, size); +StackAddress +irgen::emitAllocYieldOnce2CoroutineFrame(IRGenFunction &IGF, llvm::Value *size, + llvm::Value *mallocTypeId) { + return emitAllocCoroStaticFrame(IGF, size, mallocTypeId); } void irgen::emitDeallocYieldOnceCoroutineBuffer(IRGenFunction &IGF, @@ -5264,14 +5283,15 @@ void irgen::emitStaticDeallocAsyncContext(IRGenFunction &IGF, Address context, } StackAddress irgen::emitAllocCoroStaticFrame(IRGenFunction &IGF, - llvm::Value *size) { + llvm::Value *size, + llvm::Value *mallocTypeId) { + ASSERT(mallocTypeId); // TODO: Avoid swift_task_alloc (async) and malloc (yield_once) if the // suspension doesn't span an apply of an async function or a yield // respectively. - auto retval = IGF.emitDynamicAlloca( - IGF.IGM.Int8Ty, size, Alignment(MaximumAlignment), AllowsTaskAlloc, - IsForCalleeCoroutineFrame_t(IGF.isCalleeAllocatedCoroutine()), - "callee-coro-frame"); + auto retval = + IGF.emitDynamicAlloca(IGF.IGM.Int8Ty, size, Alignment(MaximumAlignment), + AllowsTaskAlloc, mallocTypeId, "callee-coro-frame"); IGF.Builder.CreateLifetimeStart(retval.getAddress(), Size(-1) /*dynamic size*/); return retval; diff --git a/lib/IRGen/GenCall.h b/lib/IRGen/GenCall.h index 690f199b8042b..8cbfc76909167 100644 --- a/lib/IRGen/GenCall.h +++ b/lib/IRGen/GenCall.h @@ -143,9 +143,9 @@ namespace irgen { std::pair getAsyncFunctionAndSize(IRGenFunction &IGF, FunctionPointer functionPointer, std::pair values = {true, true}); - std::pair - getCoroFunctionAndSize(IRGenFunction &IGF, FunctionPointer functionPointer, - std::pair values = {true, true}); + std::tuple getCoroFunctionValues( + IRGenFunction &IGF, FunctionPointer functionPointer, + std::tuple values = {true, true, true}); llvm::CallingConv::ID expandCallingConv(IRGenModule &IGM, SILFunctionTypeRepresentation convention, bool isAsync, bool isCalleeAllocatedCoro); @@ -221,7 +221,8 @@ namespace irgen { NativeCCEntryPointArgumentEmission &emission); StackAddress emitAllocYieldOnce2CoroutineFrame(IRGenFunction &IGF, - llvm::Value *size); + llvm::Value *size, + llvm::Value *mallocTypeId); void emitDeallocYieldOnce2CoroutineFrame(IRGenFunction &IGF, StackAddress allocation); void @@ -252,7 +253,8 @@ namespace irgen { LinkEntity asyncFunction, unsigned asyncContextIndex); - StackAddress emitAllocCoroStaticFrame(IRGenFunction &IGF, llvm::Value *size); + StackAddress emitAllocCoroStaticFrame(IRGenFunction &IGF, llvm::Value *size, + llvm::Value *mallocTypeId); void emitDeallocCoroStaticFrame(IRGenFunction &IGF, StackAddress frame); /// Yield the given values from the current continuation. diff --git a/lib/IRGen/GenCoro.cpp b/lib/IRGen/GenCoro.cpp index ccd1d45a2df3f..9aa212471e7d6 100644 --- a/lib/IRGen/GenCoro.cpp +++ b/lib/IRGen/GenCoro.cpp @@ -21,6 +21,7 @@ #include "ConstantBuilder.h" #include "Explosion.h" #include "GenCoro.h" +#include "GenFunc.h" #include "GenPointerAuth.h" #include "IRGenFunction.h" #include "IRGenModule.h" @@ -664,6 +665,18 @@ struct Allocator { constexpr Field(Kind kind) : kind(kind) {} constexpr operator Kind() { return kind; } + bool isAllocateFunction() const { + switch (kind) { + case Flags: + case Field::Deallocate: + case Field::DeallocateFrame: + return false; + case Field::Allocate: + case Field::AllocateFrame: + return true; + } + } + llvm::Type *getType(IRGenModule &IGM) { switch (kind) { case Flags: @@ -886,7 +899,7 @@ struct Allocator { llvm::Constant *getCoroAllocFn(AllocationKind kind, IRGenModule &IGM) { return IGM.getOrCreateHelperFunction( kind.getAllocationFunctionName(), IGM.CoroAllocationTy, - {IGM.CoroAllocatorPtrTy, IGM.CoroAllocationTy, IGM.SizeTy}, + {IGM.CoroAllocatorPtrTy, IGM.CoroAllocationTy, IGM.SizeTy, IGM.Int64Ty}, [kind](IRGenFunction &IGF) { auto isSwiftCoroCCAvailable = IGF.IGM.SwiftCoroCC == llvm::CallingConv::SwiftCoro; @@ -895,6 +908,7 @@ llvm::Constant *getCoroAllocFn(AllocationKind kind, IRGenModule &IGM) { auto *allocatorValue = parameters.claimNext(); auto allocator = Allocator(allocatorValue, IGF); auto *size = parameters.claimNext(); + auto *typeID = parameters.claimNext(); if (isSwiftCoroCCAvailable) { // swiftcorocc is available, so if there's no allocator pointer, // allocate storage on the stack and return a pointer to it without @@ -912,7 +926,8 @@ llvm::Constant *getCoroAllocFn(AllocationKind kind, IRGenModule &IGM) { }); } auto fnPtr = allocator.getAllocate(kind); - auto *call = IGF.Builder.CreateCall(fnPtr, {frame, allocatorValue, size}); + auto *call = IGF.Builder.CreateCall( + fnPtr, {frame, allocatorValue, size, typeID}); call->setDoesNotThrow(); call->setCallingConv(IGF.IGM.SwiftCC); IGF.Builder.CreateRet(call); @@ -1002,23 +1017,29 @@ llvm::Constant *swift::irgen::getCoroDeallocFrameFn(IRGenModule &IGM) { return getCoroDeallocFn(AllocationKind::Frame, IGM); } -using GetAllocFunctionPointer = FunctionPointer (IRGenModule::*)(); - -static llvm::Constant * -getAddrOfSwiftCoroAllocThunk(StringRef name, - GetAllocFunctionPointer getAlloc, - IRGenModule &IGM) { - auto *ty = IGM.CoroAllocateFnTy; +static llvm::Constant *getAddrOfSwiftCoroAllocatorThunk( + StringRef name, Allocator::Field field, IRGenModule &IGM, + llvm::function_ref + builder) { + auto *ty = field.getFunctionType(IGM); return IGM.getOrCreateHelperFunction( name, ty->getReturnType(), ty->params(), - [getAlloc](IRGenFunction &IGF) { + [builder, field](IRGenFunction &IGF) { auto parameters = IGF.collectParameters(); - parameters.claimNext(); // frame - parameters.claimNext(); // allocator - auto *size = parameters.claimNext(); - auto alloc = (IGF.IGM.*getAlloc)(); - auto *call = IGF.Builder.CreateCall(alloc, {size}); - IGF.Builder.CreateRet(call); + auto *frame = parameters.claimNext(); + auto *allocator = parameters.claimNext(); + // allocate - size; deallocate - address + auto *value = parameters.claimNext(); + auto *typeID = + field.isAllocateFunction() ? parameters.claimNext() : nullptr; + auto *result = builder(IGF, frame, allocator, value, typeID); + if (field.isAllocateFunction()) { + IGF.Builder.CreateRet(result); + } else { + IGF.Builder.CreateRetVoid(); + } }, /*setIsNoInline=*/true, /*forPrologue=*/false, /*isPerformanceConstraint=*/false, @@ -1030,31 +1051,19 @@ getAddrOfSwiftCoroAllocThunk(StringRef name, }); } -using GetDeallocFunctionPointer = FunctionPointer (IRGenModule::*)(); +using GetAllocatorFunctionPointer = FunctionPointer (IRGenModule::*)(); static llvm::Constant * -getAddrOfSwiftCoroDeallocThunk(StringRef name, - GetDeallocFunctionPointer getDealloc, - IRGenModule &IGM) { - auto *ty = IGM.CoroDeallocateFnTy; - return IGM.getOrCreateHelperFunction( - name, ty->getReturnType(), ty->params(), - [getDealloc](IRGenFunction &IGF) { - auto parameters = IGF.collectParameters(); - parameters.claimNext(); // frame - parameters.claimNext(); // allocator - auto *ptr = parameters.claimNext(); - auto dealloc = (IGF.IGM.*getDealloc)(); - IGF.Builder.CreateCall(dealloc, {ptr}); - IGF.Builder.CreateRetVoid(); - }, - /*setIsNoInline=*/true, /*forPrologue=*/false, - /*isPerformanceConstraint=*/false, - /*optionalLinkage=*/nullptr, - /*specialCallingConv=*/std::nullopt, - /*transformAttributes=*/ - [&IGM](llvm::AttributeList &attrs) { - IGM.addSwiftCoroAttributes(attrs, 1); +getAddrOfSwiftCoroAllocatorThunk(StringRef name, Allocator::Field field, + GetAllocatorFunctionPointer getCator, + IRGenModule &IGM) { + return getAddrOfSwiftCoroAllocatorThunk( + name, field, IGM, + [getCator](IRGenFunction &IGF, auto *frame, auto *allocator, auto *value, + auto *typeID) { + auto alloc = (IGF.IGM.*getCator)(); + auto *call = IGF.Builder.CreateCall(alloc, {value}); + return call; }); } @@ -1099,15 +1108,15 @@ static llvm::Constant *getAddrOfGlobalCoroAllocator( } static llvm::Constant *getAddrOfSwiftCoroMalloc(IRGenModule &IGM) { - return getAddrOfSwiftCoroAllocThunk("_swift_coro_malloc", - &IRGenModule::getMallocFunctionPointer, - IGM); + return getAddrOfSwiftCoroAllocatorThunk( + "_swift_coro_malloc", Allocator::Field::Allocate, + &IRGenModule::getMallocFunctionPointer, IGM); } static llvm::Constant *getAddrOfSwiftCoroFree(IRGenModule &IGM) { - return getAddrOfSwiftCoroDeallocThunk("_swift_coro_free", - &IRGenModule::getFreeFunctionPointer, - IGM); + return getAddrOfSwiftCoroAllocatorThunk( + "_swift_coro_free", Allocator::Field::Deallocate, + &IRGenModule::getFreeFunctionPointer, IGM); } llvm::Constant *IRGenModule::getAddrOfGlobalCoroMallocAllocator() { @@ -1117,17 +1126,35 @@ llvm::Constant *IRGenModule::getAddrOfGlobalCoroMallocAllocator() { getAddrOfSwiftCoroFree(*this)); } +static llvm::Constant *getAddrOfSwiftCoroTypedMalloc(IRGenModule &IGM) { + return getAddrOfSwiftCoroAllocatorThunk( + "_swift_coro_typed_malloc", Allocator::Field::Allocate, IGM, + [](IRGenFunction &IGF, auto *frame, auto *allocator, auto *size, + auto *typeID) { + auto callee = getCoroFrameAllocStubFunctionPointer(IGF.IGM); + auto *allocation = IGF.Builder.CreateCall(callee, {size, typeID}); + return allocation; + }); +} + +llvm::Constant *IRGenModule::getAddrOfGlobalCoroTypedMallocAllocator() { + return getAddrOfGlobalCoroAllocator(*this, CoroAllocatorKind::TypedMalloc, + /*shouldDeallocateImmediately=*/true, + getAddrOfSwiftCoroTypedMalloc(*this), + getAddrOfSwiftCoroFree(*this)); +} + static llvm::Constant * getAddrOfSwiftCoroTaskAlloc(IRGenModule &IGM) { - return getAddrOfSwiftCoroAllocThunk("_swift_coro_task_alloc", - &IRGenModule::getTaskAllocFunctionPointer, - IGM); + return getAddrOfSwiftCoroAllocatorThunk( + "_swift_coro_task_alloc", Allocator::Field::Allocate, + &IRGenModule::getTaskAllocFunctionPointer, IGM); } static llvm::Constant * getAddrOfSwiftCoroTaskDealloc(IRGenModule &IGM) { - return getAddrOfSwiftCoroDeallocThunk( - "_swift_coro_task_dealloc", + return getAddrOfSwiftCoroAllocatorThunk( + "_swift_coro_task_dealloc", Allocator::Field::Deallocate, &IRGenModule::getTaskDeallocFunctionPointer, IGM); } @@ -1151,6 +1178,8 @@ irgen::emitYieldOnce2CoroutineAllocator(IRGenFunction &IGF, return IGF.IGM.getAddrOfGlobalCoroAsyncTaskAllocator(); case CoroAllocatorKind::Malloc: return IGF.IGM.getAddrOfGlobalCoroMallocAllocator(); + case CoroAllocatorKind::TypedMalloc: + return IGF.IGM.getAddrOfGlobalCoroTypedMallocAllocator(); } llvm_unreachable("unhandled case"); } diff --git a/lib/IRGen/GenFunc.cpp b/lib/IRGen/GenFunc.cpp index 0660d8df54abc..8213dd9afcf0e 100644 --- a/lib/IRGen/GenFunc.cpp +++ b/lib/IRGen/GenFunc.cpp @@ -872,33 +872,43 @@ llvm::Constant *irgen::getCoroFrameAllocStubFn(IRGenModule &IGM) { // Otherwise, create a stub function to call it when available, or malloc // when it isn't. return IGM.getOrCreateHelperFunction( - "__swift_coroFrameAllocStub", IGM.Int8PtrTy, - {IGM.SizeTy, IGM.Int64Ty}, - [&](IRGenFunction &IGF) { - auto parameters = IGF.collectParameters(); - auto *size = parameters.claimNext(); - auto *coroFrameAllocFn = IGF.IGM.getOpaquePtr(coroAllocPtr); - auto *nullSwiftCoroFrameAlloc = IGF.Builder.CreateCmp( - llvm::CmpInst::Predicate::ICMP_NE, coroFrameAllocFn, - llvm::ConstantPointerNull::get( - cast(coroFrameAllocFn->getType()))); - auto *coroFrameAllocReturn = IGF.createBasicBlock("return-coroFrameAlloc"); - auto *mallocReturn = IGF.createBasicBlock("return-malloc"); - IGF.Builder.CreateCondBr(nullSwiftCoroFrameAlloc, coroFrameAllocReturn, mallocReturn); - - IGF.Builder.emitBlock(coroFrameAllocReturn); - auto *mallocTypeId = parameters.claimNext(); - auto *coroFrameAllocCall = IGF.Builder.CreateCall(IGF.IGM.getCoroFrameAllocFunctionPointer(), {size, mallocTypeId}); - IGF.Builder.CreateRet(coroFrameAllocCall); - - IGF.Builder.emitBlock(mallocReturn); - auto *mallocCall = IGF.Builder.CreateCall(IGF.IGM.getMallocFunctionPointer(), {size}); - IGF.Builder.CreateRet(mallocCall); - }, - /*setIsNoInline=*/false, - /*forPrologue=*/false, - /*isPerformanceConstraint=*/false, - /*optionalLinkageOverride=*/nullptr, llvm::CallingConv::C); + "__swift_coroFrameAllocStub", IGM.Int8PtrTy, {IGM.SizeTy, IGM.Int64Ty}, + [&](IRGenFunction &IGF) { + auto parameters = IGF.collectParameters(); + auto *size = parameters.claimNext(); + auto *coroFrameAllocFn = IGF.IGM.getOpaquePtr(coroAllocPtr); + auto *nullSwiftCoroFrameAlloc = IGF.Builder.CreateCmp( + llvm::CmpInst::Predicate::ICMP_NE, coroFrameAllocFn, + llvm::ConstantPointerNull::get( + cast(coroFrameAllocFn->getType()))); + auto *coroFrameAllocReturn = + IGF.createBasicBlock("return-coroFrameAlloc"); + auto *mallocReturn = IGF.createBasicBlock("return-malloc"); + IGF.Builder.CreateCondBr(nullSwiftCoroFrameAlloc, coroFrameAllocReturn, + mallocReturn); + + IGF.Builder.emitBlock(coroFrameAllocReturn); + auto *mallocTypeId = parameters.claimNext(); + auto *coroFrameAllocCall = IGF.Builder.CreateCall( + IGF.IGM.getCoroFrameAllocFunctionPointer(), {size, mallocTypeId}); + IGF.Builder.CreateRet(coroFrameAllocCall); + + IGF.Builder.emitBlock(mallocReturn); + auto *mallocCall = + IGF.Builder.CreateCall(IGF.IGM.getMallocFunctionPointer(), {size}); + IGF.Builder.CreateRet(mallocCall); + }, + /*setIsNoInline=*/false, + /*forPrologue=*/false, + /*isPerformanceConstraint=*/false, + /*optionalLinkageOverride=*/nullptr, llvm::CallingConv::C); +} + +FunctionPointer irgen::getCoroFrameAllocStubFunctionPointer(IRGenModule &IGM) { + auto *fn = cast(getCoroFrameAllocStubFn(IGM)); + auto sig = Signature::forFunction(fn); + return FunctionPointer::forDirect(FunctionPointerKind::Function, fn, nullptr, + sig); } static Size getOffsetOfOpaqueIsolationField(IRGenModule &IGM, @@ -1631,7 +1641,6 @@ class CoroPartialApplicationForwarderEmission /// Get the continuation function pointer /// - auto sig = Signature::forCoroutineContinuation(subIGF.IGM, origType); auto schemaAndEntity = getCoroutineResumeFunctionPointerAuth(subIGF.IGM, origType); auto pointerAuth = PointerAuthInfo::emit(subIGF, schemaAndEntity.first, diff --git a/lib/IRGen/GenFunc.h b/lib/IRGen/GenFunc.h index 922db3e939b86..f47ecaffce696 100644 --- a/lib/IRGen/GenFunc.h +++ b/lib/IRGen/GenFunc.h @@ -65,6 +65,7 @@ namespace irgen { /// Stub function that weakly links againt the swift_coroFrameAlloc /// function. This is required for back-deployment. llvm::Constant *getCoroFrameAllocStubFn(IRGenModule &IGM); + FunctionPointer getCoroFrameAllocStubFunctionPointer(IRGenModule &IGM); } // end namespace irgen } // end namespace swift diff --git a/lib/IRGen/GenMeta.cpp b/lib/IRGen/GenMeta.cpp index 8adb7bdaf87a7..8d6ecbb8e43a1 100644 --- a/lib/IRGen/GenMeta.cpp +++ b/lib/IRGen/GenMeta.cpp @@ -7677,6 +7677,9 @@ llvm::GlobalValue *irgen::emitCoroFunctionPointer(IRGenModule &IGM, initBuilder.beginStruct(IGM.CoroFunctionPointerTy)); builder.addCompactFunctionReference(function); builder.addInt32(size.getValue()); + auto *typeId = IGM.getMallocTypeId(function); + ASSERT(typeId->getIntegerType() == IGM.Int64Ty); + builder.addInt64(typeId->getLimitedValue()); return cast( IGM.defineCoroFunctionPointer(entity, builder.finishAndCreateFuture())); } diff --git a/lib/IRGen/GenOpaque.cpp b/lib/IRGen/GenOpaque.cpp index 9d9867ffdde83..1ef51b261667c 100644 --- a/lib/IRGen/GenOpaque.cpp +++ b/lib/IRGen/GenOpaque.cpp @@ -560,14 +560,15 @@ StackAddress IRGenFunction::emitDynamicAlloca(SILType T, const llvm::Twine &name) { llvm::Value *size = emitLoadOfSize(*this, T); return emitDynamicAlloca(IGM.Int8Ty, size, Alignment(16), AllowsTaskAlloc, - IsNotForCalleeCoroutineFrame, name); + /*mallocTypeId=*/nullptr, name); } -StackAddress IRGenFunction::emitDynamicAlloca( - llvm::Type *eltTy, llvm::Value *arraySize, Alignment align, - AllowsTaskAlloc_t allowTaskAlloc, - IsForCalleeCoroutineFrame_t forCalleeCoroutineFrame, - const llvm::Twine &name) { +StackAddress IRGenFunction::emitDynamicAlloca(llvm::Type *eltTy, + llvm::Value *arraySize, + Alignment align, + AllowsTaskAlloc_t allowTaskAlloc, + llvm::Value *mallocTypeId, + const llvm::Twine &name) { // Async functions call task alloc. if (allowTaskAlloc && isAsync()) { llvm::Value *byteCount; @@ -601,10 +602,14 @@ StackAddress IRGenFunction::emitDynamicAlloca( auto alignment = llvm::ConstantInt::get(IGM.Int32Ty, align.getValue()); // Allocate memory. This produces an abstract token. + llvm::SmallVector args = {byteCount, alignment}; + if (mallocTypeId) { + args.push_back(mallocTypeId); + } auto *allocToken = Builder.CreateIntrinsicCall( - forCalleeCoroutineFrame ? llvm::Intrinsic::coro_alloca_alloc_frame - : llvm::Intrinsic::coro_alloca_alloc, - {IGM.SizeTy}, {byteCount, alignment}); + mallocTypeId ? llvm::Intrinsic::coro_alloca_alloc_frame + : llvm::Intrinsic::coro_alloca_alloc, + {IGM.SizeTy}, args); // Get the allocation result. auto ptr = Builder.CreateIntrinsicCall(llvm::Intrinsic::coro_alloca_get, diff --git a/lib/IRGen/GenPointerAuth.cpp b/lib/IRGen/GenPointerAuth.cpp index 94b6b81be8b66..66a6b6408036a 100644 --- a/lib/IRGen/GenPointerAuth.cpp +++ b/lib/IRGen/GenPointerAuth.cpp @@ -771,6 +771,16 @@ void ConstantAggregateBuilderBase::addSignedPointer(llvm::Constant *pointer, llvm::ConstantInt::get(IGM().Int64Ty, otherDiscriminator)); } +llvm::ConstantInt *IRGenModule::getMallocTypeId(llvm::Function *fn) { + if (!getOptions().EmitTypeMallocForCoroFrame) { + // Even when typed malloc isn't enabled, a type id may be required for ABI + // reasons (e.g. as an argument to swift_coro_alloc). Use a cheaply + // materialized value. + return llvm::ConstantInt::get(Int64Ty, 0); + } + return getDiscriminatorForString(*this, fn->getName()); +} + llvm::ConstantInt* IRGenFunction::getMallocTypeId() { - return getDiscriminatorForString(IGM, CurFn->getName()); + return IGM.getMallocTypeId(CurFn); } diff --git a/lib/IRGen/IRGenFunction.h b/lib/IRGen/IRGenFunction.h index 203b83af90411..f10cb8cb88feb 100644 --- a/lib/IRGen/IRGenFunction.h +++ b/lib/IRGen/IRGenFunction.h @@ -71,11 +71,6 @@ enum AllowsTaskAlloc_t : bool { AllowsTaskAlloc = true, }; -enum IsForCalleeCoroutineFrame_t : bool { - IsNotForCalleeCoroutineFrame, - IsForCalleeCoroutineFrame, -}; - /// IRGenFunction - Primary class for emitting LLVM instructions for a /// specific function. class IRGenFunction { @@ -315,8 +310,7 @@ class IRGenFunction { StackAddress emitDynamicAlloca(llvm::Type *eltTy, llvm::Value *arraySize, Alignment align, AllowsTaskAlloc_t allowTaskAlloc = AllowsTaskAlloc, - IsForCalleeCoroutineFrame_t forCalleeCoroutineFrame = - IsNotForCalleeCoroutineFrame, + llvm::Value *mallocTypeId = nullptr, const llvm::Twine &name = ""); void emitDeallocateDynamicAlloca(StackAddress address, bool allowTaskDealloc = true, diff --git a/lib/IRGen/IRGenModule.cpp b/lib/IRGen/IRGenModule.cpp index 44069e60f2a4e..9d7512025d90c 100644 --- a/lib/IRGen/IRGenModule.cpp +++ b/lib/IRGen/IRGenModule.cpp @@ -771,10 +771,12 @@ IRGenModule::IRGenModule(IRGenerator &irgen, DifferentiabilityWitnessTy = createStructType( *this, "swift.differentiability_witness", {Int8PtrTy, Int8PtrTy}); - CoroFunctionPointerTy = createStructType(*this, "swift.coro_func_pointer", - {RelativeAddressTy, Int32Ty}, true); + CoroFunctionPointerTy = + createStructType(*this, "swift.coro_func_pointer", + {RelativeAddressTy, Int32Ty, Int64Ty}, true); CoroAllocateFnTy = llvm::FunctionType::get( - CoroAllocationTy, {CoroAllocationTy, CoroAllocatorPtrTy, SizeTy}, /*isVarArg*/ false); + CoroAllocationTy, {CoroAllocationTy, CoroAllocatorPtrTy, SizeTy, Int64Ty}, + /*isVarArg*/ false); CoroDeallocateFnTy = llvm::FunctionType::get( VoidTy, {CoroAllocationTy, CoroAllocatorPtrTy, CoroAllocationTy}, /*isVarArg*/ false); CoroAllocatorFlagsTy = Int32Ty; diff --git a/lib/IRGen/IRGenModule.h b/lib/IRGen/IRGenModule.h index c399e59d81bbf..fab878115ba33 100644 --- a/lib/IRGen/IRGenModule.h +++ b/lib/IRGen/IRGenModule.h @@ -882,7 +882,7 @@ class IRGenModule { llvm::StructType *DifferentiabilityWitnessTy; // { i8*, i8* } // clang-format on - llvm::StructType *CoroFunctionPointerTy; // { i32, i32 } + llvm::StructType *CoroFunctionPointerTy; // { i32, i32, i64 } llvm::FunctionType *CoroAllocateFnTy; llvm::FunctionType *CoroDeallocateFnTy; llvm::IntegerType *CoroAllocatorFlagsTy; @@ -1674,6 +1674,8 @@ private: \ llvm::AttributeList constructInitialAttributes(); StackProtectorMode shouldEmitStackProtector(SILFunction *f); + llvm::ConstantInt *getMallocTypeId(llvm::Function *fn); + void emitProtocolDecl(ProtocolDecl *D); void emitEnumDecl(EnumDecl *D); void emitStructDecl(StructDecl *D); @@ -1734,6 +1736,7 @@ private: \ SILFunction *getSILFunctionForCoroFunctionPointer(llvm::Constant *cfp); llvm::Constant *getAddrOfGlobalCoroMallocAllocator(); + llvm::Constant *getAddrOfGlobalCoroTypedMallocAllocator(); llvm::Constant *getAddrOfGlobalCoroAsyncTaskAllocator(); llvm::Function *getAddrOfDispatchThunk(SILDeclRef declRef, diff --git a/lib/IRGen/IRGenSIL.cpp b/lib/IRGen/IRGenSIL.cpp index c73fd4f705cbd..c5b6341950602 100644 --- a/lib/IRGen/IRGenSIL.cpp +++ b/lib/IRGen/IRGenSIL.cpp @@ -3728,7 +3728,7 @@ static void emitBuiltinStackAlloc(IRGenSILFunction &IGF, auto stackAddress = IGF.emitDynamicAlloca(IGF.IGM.Int8Ty, size, align, DoesNotAllowTaskAlloc, - IsNotForCalleeCoroutineFrame, "temp_alloc"); + /*mallocTypeId=*/nullptr, "temp_alloc"); IGF.setLoweredStackAddress(i, stackAddress); } diff --git a/lib/IRGen/Linking.cpp b/lib/IRGen/Linking.cpp index 6ff01de2bda52..c28a81412c106 100644 --- a/lib/IRGen/Linking.cpp +++ b/lib/IRGen/Linking.cpp @@ -591,6 +591,8 @@ std::string LinkEntity::mangleAsString(ASTContext &Ctx) const { return "_swift_coro_async_allocator"; case CoroAllocatorKind::Malloc: return "_swift_coro_malloc_allocator"; + case CoroAllocatorKind::TypedMalloc: + return "_swift_coro_typed_malloc_allocator"; } } } diff --git a/lib/IRGen/Signature.h b/lib/IRGen/Signature.h index 53c4fa1416a05..76fd9095c4137 100644 --- a/lib/IRGen/Signature.h +++ b/lib/IRGen/Signature.h @@ -212,6 +212,8 @@ class Signature { getUncachedABIDetails(IRGenModule &IGM, CanSILFunctionType formalType, FunctionPointerKind kind); + static Signature forFunction(llvm::Function *fn); + /// Compute the signature of a coroutine's continuation function. static Signature forCoroutineContinuation(IRGenModule &IGM, CanSILFunctionType coroType); diff --git a/test/IRGen/coroutine_accessors.swift b/test/IRGen/coroutine_accessors.swift index 608f6b722dd8e..74a9a80d6948a 100644 --- a/test/IRGen/coroutine_accessors.swift +++ b/test/IRGen/coroutine_accessors.swift @@ -382,15 +382,16 @@ public var force_yield_once_convention : () { // : i32 0 // : i32 1 // : ) +// CHECK: [[TYPE_ID:%[^,]+]] = load i64 // CHECK-64: [[SIZE_64:%[^,]+]] = zext i32 {{%[^,]+}} to i64 -// CHECK-64: [[ALLOCATION:%[^,]+]] = call token @llvm.coro.alloca.alloc.i64(i64 [[SIZE_64]], i32 16) -// CHECK-32: [[ALLOCATION:%[^,]+]] = call token @llvm.coro.alloca.alloc.i32(i32 {{%[^,]+}}, i32 16) +// CHECK-64: [[ALLOCATION:%[^,]+]] = call token{{.*}} @llvm.coro.alloca.alloc.frame.i64(i64 [[SIZE_64]], i32 16, i64 [[TYPE_ID]]) +// CHECK-32: [[ALLOCATION:%[^,]+]] = call token{{.*}} @llvm.coro.alloca.alloc.frame.i32(i32 {{%[^,]+}}, i32 16, i64 [[TYPE_ID]]) // CHECK: [[FRAME:%[^,]+]] = call ptr @llvm.coro.alloca.get(token [[ALLOCATION]]) // CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr [[FRAME]]) // CHECK: [[RAMP:%[^,]+]] = call ptr @llvm.coro.prepare.retcon(ptr @"$s19coroutine_accessors1SV3irmSivx") // CHECK: [[RETVAL:%[^,]+]] = call swiftcc { ptr, ptr } [[RAMP]]( // CHECK-SAME: [[FRAME]], -// CHECK-SAME: _swift_coro_malloc_allocator +// CHECK-apple-SAME: _swift_coro_typed_malloc_allocator // CHECK-SAME: ) // CHECK: [[CONTINUATION:%[^,]+]] = extractvalue { ptr, ptr } [[RETVAL]], 0 // CHECK: [[YIELD:%[^,]+]] = extractvalue { ptr, ptr } [[RETVAL]], 1 @@ -399,7 +400,7 @@ public var force_yield_once_convention : () { // CHECK-SAME: ) // CHECK: call swiftcc void [[CONTINUATION]]( // CHECK-SAME: [[FRAME]], -// CHECK-SAME: _swift_coro_malloc_allocator +// CHECK-apple-SAME: _swift_coro_typed_malloc_allocator // CHECK-SAME: ) // CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr [[FRAME]]) // CHECK: call void @llvm.coro.alloca.free.frame(token [[ALLOCATION]]) @@ -431,9 +432,10 @@ public var force_yield_once_2_convention : () { // : i32 0 // : i32 1 // : ) +// CHECK: [[TYPE_ID:%[^,]+]] = load i64 // CHECK-64: [[SIZE_64:%[^,]+]] = zext i32 {{%[^,]+}} to i64 -// CHECK-64: [[ALLOCATION:%[^,]+]] = call token @llvm.coro.alloca.alloc.frame.i64(i64 [[SIZE_64]], i32 16) -// CHECK-32: [[ALLOCATION:%[^,]+]] = call token @llvm.coro.alloca.alloc.frame.i32(i32 {{%[^,]+}}, i32 16) +// CHECK-64: [[ALLOCATION:%[^,]+]] = call token{{.*}} @llvm.coro.alloca.alloc.frame.i64(i64 [[SIZE_64]], i32 16, i64 [[TYPE_ID]]) +// CHECK-32: [[ALLOCATION:%[^,]+]] = call token{{.*}} @llvm.coro.alloca.alloc.frame.i32(i32 {{%[^,]+}}, i32 16, i64 [[TYPE_ID]]) // CHECK: [[FRAME:%[^,]+]] = call ptr @llvm.coro.alloca.get(token [[ALLOCATION]]) // CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr [[FRAME]]) // CHECK: [[RAMP:%[^,]+]] = call ptr @llvm.coro.prepare.retcon(ptr @"$s19coroutine_accessors1SV3irmSivx") diff --git a/test/IRGen/coroutine_accessors_popless.swift b/test/IRGen/coroutine_accessors_popless.swift index bd6e634856bd1..ff57d1cc97eed 100644 --- a/test/IRGen/coroutine_accessors_popless.swift +++ b/test/IRGen/coroutine_accessors_popless.swift @@ -58,16 +58,17 @@ // CHECK-arm64e-SAME: i64 40879 }, // CHECK-arm64e-SAME: section "llvm.ptrauth", // CHECK-arm64e-SAME: align 8 -// CHECK-LABEL: _swift_coro_malloc_allocator = linkonce_odr hidden constant %swift.coro_allocator { -// CHECK-SAME: i32 258, -// CHECK-SAME: malloc -// CHECK-SAME: free -// CHECK-SAME: } +// CHECK-apple-LABEL: _swift_coro_typed_malloc_allocator = linkonce_odr hidden constant %swift.coro_allocator { +// CHECK-apple-SAME: i32 259, +// CHECK-apple-SAME: _swift_coro_typed_malloc +// CHECK-apple-SAME: _swift_coro_free +// CHECK-apple-SAME: } // CHECK-LABEL: @_swift_coro_alloc( // CHECK-SAME: ptr [[FRAME:%[^,]+]] // CHECK-SAME: ptr swiftcoro [[ALLOCATOR:%[^,]+]], -// CHECK-SAME: i64 [[SIZE:%[^)]+]] +// CHECK-SAME: i64 [[SIZE:%[^,]+]] +// CHECK-SAME: i64 [[TYPE_ID:%[^)]+]] // CHECK-SAME: ) // CHECK-SAME: { // CHECK: entry: @@ -88,7 +89,12 @@ // CHECK-arm64e: [[ALLOCATE_FN_BITS:%[^,]+]] = ptrtoint ptr [[ALLOCATE_FN]] to i64 // CHECK-arm64e: [[ALLOCATE_FN_BITS_AUTHED:%[^,]+]] = call i64 @llvm.ptrauth.auth(i64 [[ALLOCATE_FN_BITS]], i32 0, i64 24469) // CHECK-arm64e: [[ALLOCATE_FN:%[^,]+]] = inttoptr i64 [[ALLOCATE_FN_BITS_AUTHED]] -// CHECK: [[ALLOCATION:%[^,]+]] = call swiftcc ptr [[ALLOCATE_FN]](ptr [[FRAME]], ptr swiftcoro [[ALLOCATOR]], [[INT]] [[SIZE]]) +// CHECK: [[ALLOCATION:%[^,]+]] = call swiftcc ptr [[ALLOCATE_FN]]( +// CHECK-SAME: ptr [[FRAME]], +// CHECK-SAME: ptr swiftcoro [[ALLOCATOR]], +// CHECK-SAME: [[INT]] [[SIZE]], +// CHECK-SAME: [[INT]] [[TYPE_ID]] +// CHECK-SAME: ) // CHECK: ret ptr [[ALLOCATION]] // CHECK: } @@ -100,7 +106,7 @@ // CHECK-SAME: { // CHECK: entry: // CHECK: [[BAIL:%[^,]+]] = icmp eq ptr [[ALLOCATOR]], null -// CHECK: br i1 [[USE_POPLESS]], +// CHECK: br i1 [[BAIL]], // CHECK-SAME: label %null_allocator // CHECK-SAME: label %nonnull_allocator // CHECK: null_allocator: @@ -238,14 +244,15 @@ public var force_yield_once_convention : () { // : i32 0 // : i32 1 // : ) +// CHECK: [[TYPE_ID:%[^,]+]] = load i64 // CHECK: [[SIZE:%[^,]+]] = zext i32 {{%[^,]+}} to i64 -// CHECK: [[ALLOCATION:%[^,]+]] = call token @llvm.coro.alloca.alloc.i64(i64 [[SIZE]], i32 16) +// CHECK: [[ALLOCATION:%[^,]+]] = call token{{.*}} @llvm.coro.alloca.alloc.frame.i64(i64 [[SIZE]], i32 16, i64 [[TYPE_ID]]) // CHECK: [[FRAME:%[^,]+]] = call ptr @llvm.coro.alloca.get(token [[ALLOCATION]]) // CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr [[FRAME]]) // CHECK: [[RAMP:%[^,]+]] = call ptr @llvm.coro.prepare.retcon(ptr @"$s27coroutine_accessors_popless1iSivx") // CHECK: [[RETVAL:%[^,]+]] = call swiftcorocc { ptr, ptr } [[RAMP]]( // CHECK-SAME: [[FRAME]], -// CHECK-SAME: _swift_coro_malloc_allocator +// CHECK-apple-SAME: _swift_coro_typed_malloc_allocator // CHECK-SAME: ) // CHECK: [[CONTINUATION:%[^,]+]] = extractvalue { ptr, ptr } [[RETVAL]], 0 // CHECK: [[YIELD:%[^,]+]] = extractvalue { ptr, ptr } [[RETVAL]], 1 @@ -254,7 +261,7 @@ public var force_yield_once_convention : () { // CHECK-SAME: ) // CHECK: call swiftcorocc void [[CONTINUATION]]( // CHECK-SAME: [[FRAME]], -// CHECK-SAME: _swift_coro_malloc_allocator +// CHECK-apple-SAME: _swift_coro_typed_malloc_allocator // CHECK-SAME: ) // CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr [[FRAME]]) // CHECK: call void @llvm.coro.alloca.free.frame(token [[ALLOCATION]]) @@ -285,8 +292,9 @@ public var force_yield_once_2_convention : () { // : i32 0 // : i32 1 // : ) +// CHECK: [[TYPE_ID:%[^,]+]] = load i64 // CHECK: [[SIZE:%[^,]+]] = zext i32 {{%[^,]+}} to i64 -// CHECK: [[ALLOCATION:%[^,]+]] = call token @llvm.coro.alloca.alloc.frame.i64(i64 [[SIZE]], i32 16) +// CHECK: [[ALLOCATION:%[^,]+]] = call token{{.*}} @llvm.coro.alloca.alloc.frame.i64(i64 [[SIZE]], i32 16, i64 [[TYPE_ID]]) // CHECK: [[FRAME:%[^,]+]] = call ptr @llvm.coro.alloca.get(token [[ALLOCATION]]) // CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr [[FRAME]]) // CHECK: [[RAMP:%[^,]+]] = call ptr @llvm.coro.prepare.retcon(ptr @"$s27coroutine_accessors_popless1iSivx") diff --git a/test/IRGen/modifyaccessor.swift b/test/IRGen/modifyaccessor.swift index 19cdf72db85f8..12dc77d4a1d98 100644 --- a/test/IRGen/modifyaccessor.swift +++ b/test/IRGen/modifyaccessor.swift @@ -12,5 +12,5 @@ extension Dictionary { } // CHECK-LABEL: define {{.*}}sSD14modifyaccessorE9alternateq_Sgx_tciM -// CHECK: [[COROALLOCA:%.*]] = call token @llvm.coro.alloca.alloc +// CHECK: [[COROALLOCA:%.*]] = call token{{.*}} @llvm.coro.alloca.alloc // CHECK: call void @llvm.coro.alloca.free(token [[COROALLOCA]]) diff --git a/test/lit.cfg b/test/lit.cfg index 2cab9478c1a7f..b4800b1e16507 100644 --- a/test/lit.cfg +++ b/test/lit.cfg @@ -3266,7 +3266,7 @@ else: # A FileCheck that automatically supports a large variety of target # conditionalization that's useful in IRGen. -IRGenFileCheck = '%s --check-prefix=CHECK --check-prefix=CHECK-%s --check-prefix=CHECK-%s --check-prefix=CHECK-%s --check-prefix=CHECK-%s --check-prefix=CHECK-%s --check-prefix=CHECK-%s -DINT=i%s -DPTR_SIZE=%s -D#CLASS_METADATA_HEADER=%s' % ( +IRGenFileCheck = '%s --check-prefix=CHECK --check-prefix=CHECK-%s --check-prefix=CHECK-%s --check-prefix=CHECK-%s --check-prefix=CHECK-%s --check-prefix=CHECK-%s --check-prefix=CHECK-%s --check-prefix=CHECK-%s -DINT=i%s -DPTR_SIZE=%s -D#CLASS_METADATA_HEADER=%s' % ( run_filecheck, run_os, run_cpu, @@ -3274,6 +3274,7 @@ IRGenFileCheck = '%s --check-prefix=CHECK --check-prefix=CHECK-%s --check-prefix run_ptrsize, run_ptrauth, run_objc_interop, + run_vendor, run_ptrsize, run_ptrsize, run_class_metadata_header_size)