diff --git a/lib/IRGen/GenCoro.cpp b/lib/IRGen/GenCoro.cpp index efcdde0f69509..ccd1d45a2df3f 100644 --- a/lib/IRGen/GenCoro.cpp +++ b/lib/IRGen/GenCoro.cpp @@ -801,17 +801,24 @@ struct Allocator { Allocator(llvm::Value *address, IRGenFunction &IGF) : address(address), IGF(IGF) {} - llvm::Value *getField(Field field) { + struct FieldLoad { + llvm::Value *address; + llvm::Value *value; + }; + + FieldLoad loadField(Field field) { auto *fieldAddress = IGF.Builder.CreateInBoundsGEP( IGF.IGM.CoroAllocatorTy, address, {llvm::ConstantInt::get(IGF.IGM.Int32Ty, 0), llvm::ConstantInt::get(IGF.IGM.Int32Ty, field.kind)}); - return IGF.Builder.CreateLoad(Address(fieldAddress, field.getType(IGF.IGM), - field.getAlignment(IGF.IGM)), - field.getName()); + auto *value = + IGF.Builder.CreateLoad(Address(fieldAddress, field.getType(IGF.IGM), + field.getAlignment(IGF.IGM)), + field.getName()); + return {fieldAddress, value}; } - llvm::Value *getFlags() { return getField(Field::Flags); } + llvm::Value *getFlags() { return loadField(Field::Flags).value; } FunctionPointer getAllocate(AllocationKind kind) { switch (kind) { @@ -862,10 +869,11 @@ struct Allocator { } FunctionPointer getFunctionPointer(Field field) { - llvm::Value *callee = getField(field); + auto fieldValues = loadField(field); + auto *callee = fieldValues.value; if (auto &schema = field.getSchema(IGF.IGM)) { - auto info = - PointerAuthInfo::emit(IGF, schema, nullptr, PointerAuthEntity()); + auto info = PointerAuthInfo::emit(IGF, schema, fieldValues.address, + field.getEntity(IGF.IGM)); callee = emitPointerAuthAuth(IGF, callee, info); } return FunctionPointer::createUnsigned( @@ -1083,28 +1091,30 @@ static llvm::Constant *getAddrOfGlobalCoroAllocator( return taskAllocator; } -static llvm::Constant *getAddrOfSwiftCoroMalloc( - IRGenModule &IGM) { +static llvm::Constant *getAddrOfGlobalCoroAllocator( + IRGenModule &IGM, CoroAllocatorKind kind, bool shouldDeallocateImmediately, + llvm::Constant *allocFn, llvm::Constant *deallocFn) { + return getAddrOfGlobalCoroAllocator(IGM, kind, shouldDeallocateImmediately, + allocFn, deallocFn, allocFn, deallocFn); +} + +static llvm::Constant *getAddrOfSwiftCoroMalloc(IRGenModule &IGM) { return getAddrOfSwiftCoroAllocThunk("_swift_coro_malloc", &IRGenModule::getMallocFunctionPointer, IGM); } -static llvm::Constant *getAddrOfSwiftCoroFree( - IRGenModule &IGM) { +static llvm::Constant *getAddrOfSwiftCoroFree(IRGenModule &IGM) { return getAddrOfSwiftCoroDeallocThunk("_swift_coro_free", &IRGenModule::getFreeFunctionPointer, IGM); } llvm::Constant *IRGenModule::getAddrOfGlobalCoroMallocAllocator() { - return getAddrOfGlobalCoroAllocator( - *this, CoroAllocatorKind::Malloc, - /*shouldDeallocateImmediately=*/true, - getAddrOfSwiftCoroMalloc(*this), - getAddrOfSwiftCoroFree(*this), - getAddrOfSwiftCoroMalloc(*this), - getAddrOfSwiftCoroFree(*this)); + return getAddrOfGlobalCoroAllocator(*this, CoroAllocatorKind::Malloc, + /*shouldDeallocateImmediately=*/true, + getAddrOfSwiftCoroMalloc(*this), + getAddrOfSwiftCoroFree(*this)); } static llvm::Constant * @@ -1122,13 +1132,10 @@ getAddrOfSwiftCoroTaskDealloc(IRGenModule &IGM) { } llvm::Constant *IRGenModule::getAddrOfGlobalCoroAsyncTaskAllocator() { - return getAddrOfGlobalCoroAllocator( - *this, CoroAllocatorKind::Async, - /*shouldDeallocateImmediately=*/false, - getAddrOfSwiftCoroTaskAlloc(*this), - getAddrOfSwiftCoroTaskDealloc(*this), - getAddrOfSwiftCoroTaskAlloc(*this), - getAddrOfSwiftCoroTaskDealloc(*this)); + return getAddrOfGlobalCoroAllocator(*this, CoroAllocatorKind::Async, + /*shouldDeallocateImmediately=*/false, + getAddrOfSwiftCoroTaskAlloc(*this), + getAddrOfSwiftCoroTaskDealloc(*this)); } llvm::Value * diff --git a/lib/IRGen/IRGen.cpp b/lib/IRGen/IRGen.cpp index e648895d259c5..f1a07b1fdb0df 100644 --- a/lib/IRGen/IRGen.cpp +++ b/lib/IRGen/IRGen.cpp @@ -1087,19 +1087,19 @@ static void setPointerAuthOptions(PointerAuthOptions &opts, PointerAuthSchema(nonABIDataKey, /*address*/ true, Discrimination::Decl); opts.CoroAllocationFunction = PointerAuthSchema( - codeKey, /*address*/ false, Discrimination::Constant, + codeKey, /*address*/ true, Discrimination::Constant, SpecialPointerAuthDiscriminators::CoroAllocationFunction); opts.CoroDeallocationFunction = PointerAuthSchema( - codeKey, /*address*/ false, Discrimination::Constant, + codeKey, /*address*/ true, Discrimination::Constant, SpecialPointerAuthDiscriminators::CoroDeallocationFunction); - opts.CoroAllocationFunction = PointerAuthSchema( - codeKey, /*address*/ false, Discrimination::Constant, + opts.CoroFrameAllocationFunction = PointerAuthSchema( + codeKey, /*address*/ true, Discrimination::Constant, SpecialPointerAuthDiscriminators::CoroFrameAllocationFunction); - opts.CoroDeallocationFunction = PointerAuthSchema( - codeKey, /*address*/ false, Discrimination::Constant, + opts.CoroFrameDeallocationFunction = PointerAuthSchema( + codeKey, /*address*/ true, Discrimination::Constant, SpecialPointerAuthDiscriminators::CoroFrameDeallocationFunction); } diff --git a/test/IRGen/coroutine_accessors.swift b/test/IRGen/coroutine_accessors.swift index d7d6beee8b2ae..663119f1149bb 100644 --- a/test/IRGen/coroutine_accessors.swift +++ b/test/IRGen/coroutine_accessors.swift @@ -26,40 +26,128 @@ // CHECK-arm64e-LABEL: _swift_coro_malloc.ptrauth = private constant { // CHECK-arm64e-SAME: ptr @_swift_coro_malloc, // CHECK-arm64e-SAME: i32 0, -// CHECK-arm64e-SAME: i64 0, +// CHECK-arm64e-SAME: i64 ptrtoint ( +// CHECK-arm64e-SAME: ptr getelementptr inbounds ( +// CHECK-arm64e-SAME: ptr @_swift_coro_malloc_allocator, +// CHECK-arm64e-SAME: i32 0, +// CHECK-arm64e-SAME: i32 1 +// CHECK-arm64e-SAME: ) +// CHECK-arm64e-SAME: ) // CHECK-arm64e-SAME: i64 24469 } // CHECK-arm64e-SAME: section "llvm.ptrauth" // CHECK-arm64e-SAME: align 8 // CHECK-arm64e-LABEL: _swift_coro_free.ptrauth = private constant { // CHECK-arm64e-SAME: ptr @_swift_coro_free, // CHECK-arm64e-SAME: i32 0, -// CHECK-arm64e-SAME: i64 0, +// CHECK-arm64e-SAME: i64 ptrtoint ( +// CHECK-arm64e-SAME: ptr getelementptr inbounds ( +// CHECK-arm64e-SAME: ptr @_swift_coro_malloc_allocator, +// CHECK-arm64e-SAME: i32 0, +// CHECK-arm64e-SAME: i32 2 +// CHECK-arm64e-SAME: ) +// CHECK-arm64e-SAME: ) // CHECK-arm64e-SAME: i64 40879 }, // CHECK-arm64e-SAME: section "llvm.ptrauth", // CHECK-arm64e-SAME: align 8 +// CHECK-arm64e-LABEL: _swift_coro_malloc.ptrauth.1 = private constant { +// CHECK-arm64e-SAME: ptr @_swift_coro_malloc, +// CHECK-arm64e-SAME: i32 0, +// CHECK-arm64e-SAME: i64 ptrtoint ( +// CHECK-arm64e-SAME: ptr getelementptr inbounds ( +// CHECK-arm64e-SAME: ptr @_swift_coro_malloc_allocator, +// CHECK-arm64e-SAME: i32 0, +// CHECK-arm64e-SAME: i32 3 +// CHECK-arm64e-SAME: ) +// CHECK-arm64e-SAME: ) +// CHECK-arm64e-SAME: i64 53841 } +// CHECK-arm64e-SAME: section "llvm.ptrauth" +// CHECK-arm64e-SAME: align 8 +// CHECK-arm64e-LABEL: _swift_coro_free.ptrauth.2 = private constant { +// CHECK-arm64e-SAME: ptr @_swift_coro_free, +// CHECK-arm64e-SAME: i32 0, +// CHECK-arm64e-SAME: i64 ptrtoint ( +// CHECK-arm64e-SAME: ptr getelementptr inbounds ( +// CHECK-arm64e-SAME: ptr @_swift_coro_malloc_allocator, +// CHECK-arm64e-SAME: i32 0, +// CHECK-arm64e-SAME: i32 4 +// CHECK-arm64e-SAME: ) +// CHECK-arm64e-SAME: ) +// CHECK-arm64e-SAME: i64 23464 }, +// CHECK-arm64e-SAME: section "llvm.ptrauth", +// CHECK-arm64e-SAME: align 8 // CHECK-LABEL: _swift_coro_malloc_allocator = linkonce_odr hidden constant %swift.coro_allocator { // CHECK-SAME: i32 258, -// CHECK-SAME: malloc -// CHECK-SAME: free +// CHECK-SAME: _swift_coro_malloc +// CHECK-ar64e-SAME: .ptrauth +// CHECK-SAME: _swift_coro_free +// CHECK-ar64e-SAME: .ptrauth +// CHECK-SAME: _swift_coro_malloc +// CHECK-ar64e-SAME: .ptrauth.1 +// CHECK-SAME: _swift_coro_free +// CHECK-ar64e-SAME: .ptrauth.2 // CHECK-SAME: } // CHECK-arm64e-LABEL: _swift_coro_task_alloc.ptrauth = private constant { // CHECK-arm64e-SAME: ptr @_swift_coro_task_alloc, // CHECK-arm64e-SAME: i32 0, -// CHECK-arm64e-SAME: i64 0, +// CHECK-arm64e-SAME: i64 ptrtoint ( +// CHECK-arm64e-SAME: ptr getelementptr inbounds ( +// CHECK-arm64e-SAME: ptr @_swift_coro_async_allocator, +// CHECK-arm64e-SAME: i32 0, +// CHECK-arm64e-SAME: i32 1 +// CHECK-arm64e-SAME: ) +// CHECK-arm64e-SAME: ) // CHECK-arm64e-SAME: i64 24469 } // CHECK-arm64e-SAME: section "llvm.ptrauth" // CHECK-arm64e-SAME: align 8 // CHECK-arm64e-LABEL: @_swift_coro_task_dealloc.ptrauth = private constant { // CHECK-arm64e-SAME: ptr @_swift_coro_task_dealloc, // CHECK-arm64e-SAME: i32 0, -// CHECK-arm64e-SAME: i64 0, +// CHECK-arm64e-SAME: i64 ptrtoint ( +// CHECK-arm64e-SAME: ptr getelementptr inbounds ( +// CHECK-arm64e-SAME: ptr @_swift_coro_async_allocator, +// CHECK-arm64e-SAME: i32 0, +// CHECK-arm64e-SAME: i32 2 +// CHECK-arm64e-SAME: ) +// CHECK-arm64e-SAME: ) // CHECK-arm64e-SAME: i64 40879 }, // CHECK-arm64e-SAME: section "llvm.ptrauth", // CHECK-arm64e-SAME: align 8 +// CHECK-arm64e-LABEL: _swift_coro_task_alloc.ptrauth.3 = private constant { +// CHECK-arm64e-SAME: ptr @_swift_coro_task_alloc, +// CHECK-arm64e-SAME: i32 0, +// CHECK-arm64e-SAME: i64 ptrtoint ( +// CHECK-arm64e-SAME: ptr getelementptr inbounds ( +// CHECK-arm64e-SAME: ptr @_swift_coro_async_allocator, +// CHECK-arm64e-SAME: i32 0, +// CHECK-arm64e-SAME: i32 3 +// CHECK-arm64e-SAME: ) +// CHECK-arm64e-SAME: ) +// CHECK-arm64e-SAME: i64 53841 } +// CHECK-arm64e-SAME: section "llvm.ptrauth" +// CHECK-arm64e-SAME: align 8 +// CHECK-arm64e-LABEL: @_swift_coro_task_dealloc.ptrauth.4 = private constant { +// CHECK-arm64e-SAME: ptr @_swift_coro_task_dealloc, +// CHECK-arm64e-SAME: i32 0, +// CHECK-arm64e-SAME: i64 ptrtoint ( +// CHECK-arm64e-SAME: ptr getelementptr inbounds ( +// CHECK-arm64e-SAME: ptr @_swift_coro_async_allocator, +// CHECK-arm64e-SAME: i32 0, +// CHECK-arm64e-SAME: i32 4 +// CHECK-arm64e-SAME: ) +// CHECK-arm64e-SAME: ) +// CHECK-arm64e-SAME: i64 23464 }, +// CHECK-arm64e-SAME: section "llvm.ptrauth", +// CHECK-arm64e-SAME: align 8 // CHECK-LABEL: _swift_coro_async_allocator = linkonce_odr hidden constant %swift.coro_allocator { // CHECK-SAME: i32 1, // CHECK-SAME: _swift_coro_task_alloc +// CHECK-ar64e-SAME: .ptrauth +// CHECK-SAME: _swift_coro_task_dealloc +// CHECK-ar64e-SAME: .ptrauth +// CHECK-SAME: _swift_coro_task_alloc +// CHECK-ar64e-SAME: .ptrauth.3 // CHECK-SAME: _swift_coro_task_dealloc +// CHECK-ar64e-SAME: .ptrauth.4 // CHECK-SAME: } // CHECK-LABEL: @_swift_coro_alloc( @@ -74,8 +162,10 @@ // CHECK-SAME: i32 0 // CHECK-SAME: i32 1 // CHECK: [[ALLOCATE_FN:%[^,]+]] = load ptr, ptr [[ALLOCATE_FN_PTR]] +// CHECK-arm64e: [[ALLOCATE_FN_PTR_BITS:%[^,]+]] = ptrtoint ptr [[ALLOCATE_FN_PTR]] to i64 +// CHECK-arm64e: [[ALLOCATE_FN_DISCRIMINATOR:%[^,]+]] = call i64 @llvm.ptrauth.blend(i64 [[ALLOCATE_FN_PTR_BITS]], i64 24469) // CHECK-arm64e: [[ALLOCATE_FN_BITS:%[^,]+]] = ptrtoint ptr [[ALLOCATE_FN]] to i64 -// CHECK-arm64e: [[ALLOCATE_FN_BITS_AUTHED:%[^,]+]] = call i64 @llvm.ptrauth.auth(i64 [[ALLOCATE_FN_BITS]], i32 0, i64 24469) +// CHECK-arm64e: [[ALLOCATE_FN_BITS_AUTHED:%[^,]+]] = call i64 @llvm.ptrauth.auth(i64 [[ALLOCATE_FN_BITS]], i32 0, i64 [[ALLOCATE_FN_DISCRIMINATOR]]) // CHECK-arm64e: [[ALLOCATE_FN:%[^,]+]] = inttoptr i64 [[ALLOCATE_FN_BITS_AUTHED]] // CHECK: [[ALLOCATION:%[^,]+]] = call swiftcc ptr [[ALLOCATE_FN]](ptr [[FRAME]], ptr swiftcoro [[ALLOCATOR]], [[INT]] [[SIZE]]) // CHECK: ret ptr [[ALLOCATION]] @@ -106,8 +196,10 @@ // CHECK-SAME: i32 0 // CHECK-SAME: i32 2 // CHECK: [[DEALLOCATE_FN:%[^,]+]] = load ptr, ptr [[DEALLOCATE_FN_PTR]] +// CHECK-arm64e: [[DEALLOCATE_FN_PTR_BITS:%[^,]+]] = ptrtoint ptr [[DEALLOCATE_FN_PTR]] to i64 +// CHECK-arm64e: [[DEALLOCATE_FN_DISCRIMINATOR:%[^,]+]] = call i64 @llvm.ptrauth.blend(i64 [[DEALLOCATE_FN_PTR_BITS]], i64 40879) // CHECK-arm64e: [[DEALLOCATE_FN_BITS:%[^,]+]] = ptrtoint ptr [[DEALLOCATE_FN]] to i64 -// CHECK-arm64e: [[DEALLOCATE_FN_BITS_AUTHED:%[^,]+]] = call i64 @llvm.ptrauth.auth(i64 [[DEALLOCATE_FN_BITS]], i32 0, i64 40879) +// CHECK-arm64e: [[DEALLOCATE_FN_BITS_AUTHED:%[^,]+]] = call i64 @llvm.ptrauth.auth(i64 [[DEALLOCATE_FN_BITS]], i32 0, i64 [[DEALLOCATE_FN_DISCRIMINATOR]]) // CHECK-arm64e: [[DEALLOCATE_FN:%[^,]+]] = inttoptr i64 [[DEALLOCATE_FN_BITS_AUTHED]] // CHECK: call swiftcc void [[DEALLOCATE_FN]](ptr [[FRAME]], ptr swiftcoro [[ALLOCATOR]], ptr [[ADDRESS]]) // CHECK: ret void