diff --git a/CMakeLists.txt b/CMakeLists.txt index 9eb5c044bf96f..fcba8f264d02d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -271,6 +271,18 @@ option(SWIFT_BUILD_STDLIB_CXX_MODULE "If not building stdlib, controls whether to build the Cxx module" TRUE) +# The swiftClientRetainRelease library is currently only available for Darwin +# platforms. +if(SWIFT_HOST_VARIANT_SDK IN_LIST SWIFT_DARWIN_PLATFORMS) + option(SWIFT_BUILD_CLIENT_RETAIN_RELEASE + "Build the swiftClientRetainRelease library" + TRUE) +else() + option(SWIFT_BUILD_CLIENT_RETAIN_RELEASE + "Build the swiftClientRetainRelease library" + FALSE) +endif() + # In many cases, the CMake build system needs to determine whether to include # a directory, or perform other actions, based on whether the stdlib or SDK is # being built at all -- statically or dynamically. Please note that these diff --git a/include/swift/AST/IRGenOptions.h b/include/swift/AST/IRGenOptions.h index eddbc98292bb3..1758f85534c42 100644 --- a/include/swift/AST/IRGenOptions.h +++ b/include/swift/AST/IRGenOptions.h @@ -542,6 +542,9 @@ class IRGenOptions { // Whether to emit mergeable or non-mergeable traps. unsigned MergeableTraps : 1; + /// Disable the use of swift_retain/releaseClient functions. + unsigned DisableClientRetainRelease : 1; + /// The number of threads for multi-threaded code generation. unsigned NumThreads = 0; @@ -653,6 +656,7 @@ class IRGenOptions { EmitAsyncFramePushPopMetadata(true), EmitTypeMallocForCoroFrame(true), AsyncFramePointerAll(false), UseProfilingMarkerThunks(false), UseCoroCCX8664(false), UseCoroCCArm64(false), MergeableTraps(false), + DisableClientRetainRelease(false), DebugInfoForProfiling(false), CmdArgs(), SanitizeCoverage(llvm::SanitizerCoverageOptions()), TypeInfoFilter(TypeInfoDumpFilter::All), diff --git a/include/swift/Option/FrontendOptions.td b/include/swift/Option/FrontendOptions.td index b72721ee4e3ca..c87dc8e802ce0 100644 --- a/include/swift/Option/FrontendOptions.td +++ b/include/swift/Option/FrontendOptions.td @@ -1457,6 +1457,10 @@ def mergeable_traps : Flag<["-"], "mergeable-traps">, HelpText<"Emit mergeable traps even in optimized builds">; +def disable_client_retain_release : + Flag<["-"], "disable-client-retain-release">, + HelpText<"Disable use of swift_retain/releaseClient functions">; + def enable_new_llvm_pass_manager : Flag<["-"], "enable-new-llvm-pass-manager">, HelpText<"Enable the new llvm pass manager">; diff --git a/include/swift/Runtime/Config.h b/include/swift/Runtime/Config.h index 457cdacc8537d..2930a94824e46 100644 --- a/include/swift/Runtime/Config.h +++ b/include/swift/Runtime/Config.h @@ -248,6 +248,17 @@ extern uintptr_t __COMPATIBILITY_LIBRARIES_CANNOT_CHECK_THE_IS_SWIFT_BIT_DIRECTL // so changing this value is not sufficient. #define SWIFT_DEFAULT_LLVM_CC llvm::CallingConv::C +// Define the calling convention for refcounting functions for targets where it +// differs from the standard calling convention. Currently this is only used for +// swift_retain, swift_release, and some internal helper functions that they +// call. +#if defined(__aarch64__) +#define SWIFT_REFCOUNT_CC SWIFT_CC_PreserveMost +#define SWIFT_REFCOUNT_CC_PRESERVEMOST 1 +#else +#define SWIFT_REFCOUNT_CC +#endif + /// Should we use absolute function pointers instead of relative ones? /// WebAssembly target uses it by default. #ifndef SWIFT_COMPACT_ABSOLUTE_FUNCTION_POINTER diff --git a/include/swift/Runtime/CustomRRABI.h b/include/swift/Runtime/CustomRRABI.h index a3e07d37af24a..d391b225db682 100644 --- a/include/swift/Runtime/CustomRRABI.h +++ b/include/swift/Runtime/CustomRRABI.h @@ -59,6 +59,9 @@ namespace swift { template Param returnTypeHelper(Ret (*)(Param)) {} +template +Param returnTypeHelper(SWIFT_REFCOUNT_CC Ret (*)(Param)) {} + #if defined(__LP64__) || defined(_LP64) #define REGISTER_SUBSTITUTION_PREFIX "" #define REGISTER_PREFIX "x" diff --git a/include/swift/Runtime/HeapObject.h b/include/swift/Runtime/HeapObject.h index 276565dc2b01e..3c89c0e28454c 100644 --- a/include/swift/Runtime/HeapObject.h +++ b/include/swift/Runtime/HeapObject.h @@ -137,6 +137,7 @@ HeapObject* swift_allocEmptyBox(); /// It may also prove worthwhile to have this use a custom CC /// which preserves a larger set of registers. SWIFT_RUNTIME_EXPORT +SWIFT_REFCOUNT_CC HeapObject *swift_retain(HeapObject *object); SWIFT_RUNTIME_EXPORT @@ -173,6 +174,7 @@ bool swift_isDeallocating(HeapObject *object); /// - maybe a variant that can assume a non-null object /// It's unlikely that a custom CC would be beneficial here. SWIFT_RUNTIME_EXPORT +SWIFT_REFCOUNT_CC void swift_release(HeapObject *object); SWIFT_RUNTIME_EXPORT diff --git a/include/swift/Runtime/RuntimeFunctions.def b/include/swift/Runtime/RuntimeFunctions.def index 7709fc11dac54..4cf40fb468b06 100644 --- a/include/swift/Runtime/RuntimeFunctions.def +++ b/include/swift/Runtime/RuntimeFunctions.def @@ -220,6 +220,22 @@ FUNCTION(NativeStrongRelease, Swift, swift_release, C_CC, AlwaysAvailable, EFFECT(RuntimeEffect::RefCounting, RuntimeEffect::Deallocating), UNKNOWN_MEMEFFECTS) +// void *swift_retain(void *ptr); +FUNCTION(NativeStrongRetainClient, Swift, swift_retainClient, SwiftRR_CC, AlwaysAvailable, + RETURNS(RefCountedPtrTy), + ARGS(RefCountedPtrTy), + ATTRS(NoUnwind, FirstParamReturned, WillReturn), + EFFECT(RuntimeEffect::RefCounting), + UNKNOWN_MEMEFFECTS) + +// void swift_release(void *ptr); +FUNCTION(NativeStrongReleaseClient, Swift, swift_releaseClient, SwiftRR_CC, AlwaysAvailable, + RETURNS(VoidTy), + ARGS(RefCountedPtrTy), + ATTRS(NoUnwind), + EFFECT(RuntimeEffect::RefCounting, RuntimeEffect::Deallocating), + UNKNOWN_MEMEFFECTS) + // void *swift_retain_n(void *ptr, int32_t n); FUNCTION(NativeStrongRetainN, Swift, swift_retain_n, C_CC, AlwaysAvailable, RETURNS(RefCountedPtrTy), @@ -420,6 +436,24 @@ FUNCTION(BridgeObjectStrongRelease, Swift, swift_bridgeObjectRelease, EFFECT(RuntimeEffect::RefCounting, RuntimeEffect::Deallocating), UNKNOWN_MEMEFFECTS) +// void *swift_bridgeObjectRetainClient(void *ptr); +FUNCTION(BridgeObjectStrongRetainClient, Swift, swift_bridgeObjectRetainClient, + SwiftRR_CC, AlwaysAvailable, + RETURNS(BridgeObjectPtrTy), + ARGS(BridgeObjectPtrTy), + ATTRS(NoUnwind, FirstParamReturned), + EFFECT(RuntimeEffect::RefCounting), + UNKNOWN_MEMEFFECTS) + +// void *swift_bridgeObjectReleaseClient(void *ptr); +FUNCTION(BridgeObjectStrongReleaseClient, Swift, swift_bridgeObjectReleaseClient, + SwiftRR_CC, AlwaysAvailable, + RETURNS(VoidTy), + ARGS(BridgeObjectPtrTy), + ATTRS(NoUnwind), + EFFECT(RuntimeEffect::RefCounting, RuntimeEffect::Deallocating), + UNKNOWN_MEMEFFECTS) + // void *swift_nonatomic_bridgeObjectRetain(void *ptr); FUNCTION(NonAtomicBridgeObjectStrongRetain, Swift, swift_nonatomic_bridgeObjectRetain, C_CC, AlwaysAvailable, diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index d847b0eab7ca3..a41fdc972b5ba 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -3939,6 +3939,9 @@ static bool ParseIRGenArgs(IRGenOptions &Opts, ArgList &Args, Opts.MergeableTraps = Args.hasArg(OPT_mergeable_traps); + Opts.DisableClientRetainRelease = + Args.hasArg(OPT_disable_client_retain_release); + Opts.EnableObjectiveCProtocolSymbolicReferences = Args.hasFlag(OPT_enable_objective_c_protocol_symbolic_references, OPT_disable_objective_c_protocol_symbolic_references, diff --git a/lib/IRGen/GenHeap.cpp b/lib/IRGen/GenHeap.cpp index 199a50f1fd3d1..c86ec53c61ffb 100644 --- a/lib/IRGen/GenHeap.cpp +++ b/lib/IRGen/GenHeap.cpp @@ -997,11 +997,16 @@ void IRGenFunction::emitNativeStrongRetain(llvm::Value *value, value = Builder.CreateBitCast(value, IGM.RefCountedPtrTy); // Emit the call. - llvm::CallInst *call = Builder.CreateCall( - (atomicity == Atomicity::Atomic) - ? IGM.getNativeStrongRetainFunctionPointer() - : IGM.getNativeNonAtomicStrongRetainFunctionPointer(), - value); + FunctionPointer function; + if (atomicity == Atomicity::Atomic && + IGM.TargetInfo.HasSwiftClientRRLibrary && + !getOptions().DisableClientRetainRelease) + function = IGM.getNativeStrongRetainClientFunctionPointer(); + else if (atomicity == Atomicity::Atomic) + function = IGM.getNativeStrongRetainFunctionPointer(); + else + function = IGM.getNativeNonAtomicStrongRetainFunctionPointer(); + llvm::CallInst *call = Builder.CreateCall(function, value); call->setDoesNotThrow(); call->addParamAttr(0, llvm::Attribute::Returned); } @@ -1257,10 +1262,16 @@ void IRGenFunction::emitNativeStrongRelease(llvm::Value *value, Atomicity atomicity) { if (doesNotRequireRefCounting(value)) return; - emitUnaryRefCountCall(*this, (atomicity == Atomicity::Atomic) - ? IGM.getNativeStrongReleaseFn() - : IGM.getNativeNonAtomicStrongReleaseFn(), - value); + llvm::Constant *function; + if (atomicity == Atomicity::Atomic && + IGM.TargetInfo.HasSwiftClientRRLibrary && + !getOptions().DisableClientRetainRelease) + function = IGM.getNativeStrongReleaseClientFn(); + else if (atomicity == Atomicity::Atomic) + function = IGM.getNativeStrongReleaseFn(); + else + function = IGM.getNativeNonAtomicStrongReleaseFn(); + emitUnaryRefCountCall(*this, function, value); } void IRGenFunction::emitNativeSetDeallocating(llvm::Value *value) { @@ -1353,20 +1364,30 @@ void IRGenFunction::emitUnknownStrongRelease(llvm::Value *value, void IRGenFunction::emitBridgeStrongRetain(llvm::Value *value, Atomicity atomicity) { - emitUnaryRefCountCall(*this, - (atomicity == Atomicity::Atomic) - ? IGM.getBridgeObjectStrongRetainFn() - : IGM.getNonAtomicBridgeObjectStrongRetainFn(), - value); + llvm::Constant *function; + if (atomicity == Atomicity::Atomic && + IGM.TargetInfo.HasSwiftClientRRLibrary && + !getOptions().DisableClientRetainRelease) + function = IGM.getBridgeObjectStrongRetainClientFn(); + else if (atomicity == Atomicity::Atomic) + function = IGM.getBridgeObjectStrongRetainFn(); + else + function = IGM.getNonAtomicBridgeObjectStrongRetainFn(); + emitUnaryRefCountCall(*this, function, value); } void IRGenFunction::emitBridgeStrongRelease(llvm::Value *value, Atomicity atomicity) { - emitUnaryRefCountCall(*this, - (atomicity == Atomicity::Atomic) - ? IGM.getBridgeObjectStrongReleaseFn() - : IGM.getNonAtomicBridgeObjectStrongReleaseFn(), - value); + llvm::Constant *function; + if (atomicity == Atomicity::Atomic && + IGM.TargetInfo.HasSwiftClientRRLibrary && + !getOptions().DisableClientRetainRelease) + function = IGM.getBridgeObjectStrongReleaseClientFn(); + else if (atomicity == Atomicity::Atomic) + function = IGM.getBridgeObjectStrongReleaseFn(); + else + function = IGM.getNonAtomicBridgeObjectStrongReleaseFn(); + emitUnaryRefCountCall(*this, function, value); } void IRGenFunction::emitErrorStrongRetain(llvm::Value *value) { diff --git a/lib/IRGen/IRGenModule.cpp b/lib/IRGen/IRGenModule.cpp index 9ba09c542f9b8..dd2ae5f1da522 100644 --- a/lib/IRGen/IRGenModule.cpp +++ b/lib/IRGen/IRGenModule.cpp @@ -564,8 +564,9 @@ IRGenModule::IRGenModule(IRGenerator &irgen, InvariantMetadataID = getLLVMContext().getMDKindID("invariant.load"); InvariantNode = llvm::MDNode::get(getLLVMContext(), {}); DereferenceableID = getLLVMContext().getMDKindID("dereferenceable"); - + C_CC = getOptions().PlatformCCallingConvention; + SwiftRR_CC = llvm::CallingConv::PreserveMost; // TODO: use "tinycc" on platforms that support it DefaultCC = SWIFT_DEFAULT_LLVM_CC; @@ -1724,6 +1725,11 @@ void IRGenModule::addLinkLibraries() { registerLinkLibrary( LinkLibrary{"objc", LibraryKind::Library, /*static=*/false}); + if (TargetInfo.HasSwiftClientRRLibrary && + !getOptions().DisableClientRetainRelease) + registerLinkLibrary(LinkLibrary{"swiftClientRetainRelease", + LibraryKind::Library, /*static=*/true}); + // If C++ interop is enabled, add -lc++ on Darwin and -lstdc++ on linux. // Also link with C++ bridging utility module (Cxx) and C++ stdlib overlay // (std) if available. diff --git a/lib/IRGen/IRGenModule.h b/lib/IRGen/IRGenModule.h index 6b64264bc93d2..a1e38bd2ae9b2 100644 --- a/lib/IRGen/IRGenModule.h +++ b/lib/IRGen/IRGenModule.h @@ -919,6 +919,7 @@ class IRGenModule { llvm::CallingConv::ID SwiftCC; /// swift calling convention llvm::CallingConv::ID SwiftAsyncCC; /// swift calling convention for async llvm::CallingConv::ID SwiftCoroCC; /// swift calling convention for callee-allocated coroutines + llvm::CallingConv::ID SwiftRR_CC; /// What kind of tail call should be used for async->async calls. llvm::CallInst::TailCallKind AsyncTailCallKind; diff --git a/lib/IRGen/SwiftTargetInfo.cpp b/lib/IRGen/SwiftTargetInfo.cpp index 0b69902420655..0119b11cb04d2 100644 --- a/lib/IRGen/SwiftTargetInfo.cpp +++ b/lib/IRGen/SwiftTargetInfo.cpp @@ -71,6 +71,14 @@ static void configureARM64(IRGenModule &IGM, const llvm::Triple &triple, // half for the kernel. target.SwiftRetainIgnoresNegativeValues = true; + // ARM64 Darwin has swiftClientRetainRelease, but not in Embedded mode. JIT + // mode can't load the static library, so disable it there as well. + if (triple.isOSDarwin() && + !IGM.getSwiftModule()->getASTContext().LangOpts.hasFeature( + Feature::Embedded) && + !IGM.getOptions().UseJIT) + target.HasSwiftClientRRLibrary = true; + target.UsableSwiftAsyncContextAddrIntrinsic = true; } diff --git a/lib/IRGen/SwiftTargetInfo.h b/lib/IRGen/SwiftTargetInfo.h index fc5a5ec85a7f5..2384a81733100 100644 --- a/lib/IRGen/SwiftTargetInfo.h +++ b/lib/IRGen/SwiftTargetInfo.h @@ -114,6 +114,9 @@ class SwiftTargetInfo { /// "negative" pointer values. bool SwiftRetainIgnoresNegativeValues = false; + /// True if the swiftClientRetainRelease static library is available. + bool HasSwiftClientRRLibrary = false; + bool UsableSwiftAsyncContextAddrIntrinsic = false; }; diff --git a/lib/LLVMPasses/LLVMARCOpts.h b/lib/LLVMPasses/LLVMARCOpts.h index 55f48a318d139..a740695d40056 100644 --- a/lib/LLVMPasses/LLVMARCOpts.h +++ b/lib/LLVMPasses/LLVMARCOpts.h @@ -64,6 +64,13 @@ inline RT_Kind classifyInstruction(const llvm::Instruction &I) { .Case("__swift_" #TextualName, RT_ ## Name) #include "LLVMSwift.def" + // Identify "Client" versions of reference counting entry points. +#define SWIFT_FUNC(Name, MemBehavior, TextualName) \ + .Case("swift_" #TextualName "Client", RT_ ## Name) +#define SWIFT_INTERNAL_FUNC_NEVER_NONATOMIC(Name, MemBehavior, TextualName) \ + .Case("__swift_" #TextualName "Client", RT_ ## Name) +#include "LLVMSwift.def" + // Support non-atomic versions of reference counting entry points. #define SWIFT_FUNC(Name, MemBehavior, TextualName) \ .Case("swift_nonatomic_" #TextualName, RT_ ## Name) diff --git a/stdlib/public/CMakeLists.txt b/stdlib/public/CMakeLists.txt index 0c479ae35085a..640b0556664fd 100644 --- a/stdlib/public/CMakeLists.txt +++ b/stdlib/public/CMakeLists.txt @@ -297,6 +297,10 @@ endif() if(SWIFT_BUILD_STDLIB) # These must be kept in dependency order so that any referenced targets # exist at the time we look for them in add_swift_*. + if(SWIFT_BUILD_CLIENT_RETAIN_RELEASE) + add_subdirectory(ClientRetainRelease) + endif() + add_subdirectory(runtime) add_subdirectory(stubs) add_subdirectory(core) @@ -366,4 +370,3 @@ endif() if(SWIFT_BUILD_LIBEXEC) add_subdirectory(libexec) endif() - diff --git a/stdlib/public/ClientRetainRelease/CMakeLists.txt b/stdlib/public/ClientRetainRelease/CMakeLists.txt new file mode 100644 index 0000000000000..5d212a737baf8 --- /dev/null +++ b/stdlib/public/ClientRetainRelease/CMakeLists.txt @@ -0,0 +1,18 @@ +add_swift_target_library(swiftClientRetainRelease + STATIC DONT_EMBED_BITCODE NOSWIFTRT + RetainRelease.s + C_COMPILE_FLAGS ${SWIFT_RUNTIME_CXX_FLAGS} + $<$:-DSWIFT_OBJC_INTEROP=1> + LINK_FLAGS ${SWIFT_RUNTIME_LINK_FLAGS} + SWIFT_COMPILE_FLAGS ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS} + + DEPLOYMENT_VERSION_OSX ${COMPATIBILITY_MINIMUM_DEPLOYMENT_VERSION_OSX} + DEPLOYMENT_VERSION_IOS ${COMPATIBILITY_MINIMUM_DEPLOYMENT_VERSION_IOS} + DEPLOYMENT_VERSION_TVOS ${COMPATIBILITY_MINIMUM_DEPLOYMENT_VERSION_TVOS} + DEPLOYMENT_VERSION_WATCHOS ${COMPATIBILITY_MINIMUM_DEPLOYMENT_VERSION_WATCHOS} + DEPLOYMENT_VERSION_XROS ${COMPATIBILITY_MINIMUM_DEPLOYMENT_VERSION_XROS} + + MACCATALYST_BUILD_FLAVOR "zippered" + + INSTALL_IN_COMPONENT compiler + INSTALL_WITH_SHARED) diff --git a/stdlib/public/ClientRetainRelease/RetainRelease.s b/stdlib/public/ClientRetainRelease/RetainRelease.s new file mode 100644 index 0000000000000..18b28e8d76b7d --- /dev/null +++ b/stdlib/public/ClientRetainRelease/RetainRelease.s @@ -0,0 +1,383 @@ + +// We currently have an implementation for ARM64 Mach-O. +#if __arm64__ && __LP64__ && defined(__APPLE__) && defined(__MACH__) + +// Use the CAS instructions where available. +#if __ARM_FEATURE_ATOMICS +#define USE_CAS 1 +#else +#define USE_CAS 0 +#endif + +// If CAS is not available, we use load/store exclusive. +#define USE_LDX_STX !USE_CAS + +// Use ptrauth instructions where needed. +#if __has_feature(ptrauth_calls) +#define PTRAUTH 1 +#else +#define PTRAUTH 0 +#endif + +// The value of 1 strong refcount in the overall refcount field. +#define STRONG_RC_ONE (1 << 33) + +.subsections_via_symbols + +.data + +// The slowpath mask is in the runtime. Its "address" is the mask, with an +// offset so that it's still correct when the weak reference resolves to zero. +.weak_reference __swift_retainRelease_slowpath_mask_v1 + +// Grab our own copy of the slowpath mask. This mask is a value which indicates +// when we must call into the runtime slowpath. If the object's refcount field +// has any bits set that are in the mask, then we must take the slow path. The +// offset is the value it will have when the variable isn't present at runtime, +// and needs to be the correct mask for older runtimes. +// +// This variable goes into a special section so it can be located and runtime +// to override the value. Instrumentation can set it to all 1s to ensure the +// slow path is always used. +.section __DATA,__swift5_rr_mask +.align 3 +LretainRelease_slowpath_mask: +.quad __swift_retainRelease_slowpath_mask_v1 + 0x8000000000000000 + + +.text +.align 2 + +// Macro for conditionally emitting instructions. When `condition` is true, the +// rest of the line is emitted. When false, nothing is emitted. More readable +// shorthand for #if blocks when there's only one instruction to conditionalize. +.macro CONDITIONAL condition line:vararg +.ifb \line +.err CONDITIONAL used with no instruction +.endif +.if \condition +\line +.endif +.endmacro + + +// Save or load all of the registers that we promise to preserve that aren't +// preserved by the standard calling convention. The macro parameter is either +// step or ldp to save or load. +.macro SAVE_LOAD_REGS inst, pushStack +.if \pushStack + \inst x2, x3, [sp, #-0x70]! +.else + \inst x2, x3, [sp, #0x0] +.endif + \inst x4, x5, [sp, #0x10] + \inst x6, x7, [sp, #0x20] + \inst x8, x9, [sp, #0x30] + \inst x10, x11,[sp, #0x40] + \inst x12, x13,[sp, #0x50] + \inst x14, x15,[sp, #0x60] +.endmacro + +.macro SAVE_REGS + SAVE_LOAD_REGS stp, 1 +.endmacro + +.macro LOAD_REGS + SAVE_LOAD_REGS ldp, 0 +.endmacro + +.macro CALL_SLOWPATH func +// Tail call the slow path. + b \func +.endmacro + + +// NOTE: we're using the preserve_most calling convention, so x9-15 are off +// limits, in addition to the usual x19 and up. Any calls to functions that use +// the standard calling convention need to save/restore x9-x15. + +.globl _swift_retain_preservemost +.weak_definition _swift_retain_preservemost +_swift_retain_preservemost: + stp x0, x9, [sp, #-0x50]! + stp x10, x11, [sp, #0x10] + stp x12, x13, [sp, #0x20] + stp x14, x15, [sp, #0x30] + stp fp, lr, [sp, #0x40]; + add fp, sp, #0x40 + + // Clear the unused bits from the pointer + and x0, x0, #0x0ffffffffffffff8 + bl _swift_retain + + ldp fp, lr, [sp, #0x40] + ldp x14, x15, [sp, #0x30] + ldp x12, x13, [sp, #0x20] + ldp x10, x11, [sp, #0x10] + ldp x0, x9, [sp], #0x50 + CONDITIONAL PTRAUTH, \ + retab + CONDITIONAL !PTRAUTH, \ + ret + +.globl _swift_release_preservemost +.weak_definition _swift_release_preservemost +_swift_release_preservemost: + str x9, [sp, #-0x50]! + stp x10, x11, [sp, #0x10] + stp x12, x13, [sp, #0x20] + stp x14, x15, [sp, #0x30] + stp fp, lr, [sp, #0x40]; + add fp, sp, #0x40 + + // Clear the unused bits from the pointer + and x0, x0, #0x0ffffffffffffff8 + bl _swift_release + + ldp fp, lr, [sp, #0x40] + ldp x14, x15, [sp, #0x30] + ldp x12, x13, [sp, #0x20] + ldp x10, x11, [sp, #0x10] + ldr x9, [sp], #0x50 + CONDITIONAL PTRAUTH, \ + retab + CONDITIONAL !PTRAUTH, \ + ret + +.private_extern _swift_bridgeObjectReleaseClient +#if SWIFT_OBJC_INTEROP +_swift_bridgeObjectReleaseClient: + tbz x0, #63, LbridgeObjectReleaseNotTagged + ret +LbridgeObjectReleaseNotTagged: + tbnz x0, #62, _bridgeObjectReleaseClientObjC + and x0, x0, 0x0ffffffffffffff8 + +#else +_swift_bridgeObjectReleaseClient: + and x0, x0, 0x0ffffffffffffff8 +#endif + +.alt_entry _swift_releaseClient +.private_extern _swift_releaseClient +_swift_releaseClient: +// RR of NULL or values with high bit set is a no-op. + cmp x0, #0 + b.le Lrelease_ret + +// We'll operate on the address of the refcount field, which is 8 bytes into +// the object. + add x1, x0, #8 + +// Load the current value in the refcount field when using CAS. + CONDITIONAL USE_CAS, \ + ldr x16, [x1] + +// The compare-and-swap goes back to here when it needs to retry. +Lrelease_retry: +// Get the slow path mask and see if the refcount field has any of those bits +// set. + adrp x17, LretainRelease_slowpath_mask@PAGE + ldr x17, [x17, LretainRelease_slowpath_mask@PAGEOFF] + +// Load-exclusive of the current value in the refcount field when using LLSC. +// stxr does not update x16 like cas does, so this load must be inside the loop. +// ldxr/stxr are not guaranteed to make forward progress if there are memory +// accesses between them, so we need to do this after getting the mask above. + CONDITIONAL USE_LDX_STX, \ + ldxr x16, [x1] + + tst x16, x17 + +// Also check if we're releazing with a refcount of 0. That will initiate +// dealloc and requires calling the slow path. We don't try to decrement and +// then call dealloc in that case. We'll just immediately go to the slow path +// and let it take care of the entire operation. + mov x17, #STRONG_RC_ONE + ccmp x16, x17, #0x8, eq + +// If the refcount value matches the slow path mask, or the strong refcount is +// zero, then go to the slow path. + b.lt Lslowpath_release + +// We're good to proceed with the fast path. Compute the new value of the +// refcount field. + sub x17, x16, x17 + +#if USE_CAS +// Save a copy of the old value so we can determine if the CAS succeeded. + mov x2, x16 + +// Compare and swap the new value into the refcount field. Perform the operation +// with release memory ordering so that dealloc on another thread will see all +// stores performed on this thread prior to calling release. + casl x16, x17, [x1] + +// The previous value of the refcount field is now in x16. We succeeded if that +// value is the same as the old value we had before. If we failed, retry. + cmp x2, x16 + b.ne Lrelease_retry +#elif USE_LDX_STX +// Try to store the updated value. + stlxr w16, x17, [x1] + +// On failure, retry. + cbnz w16, Lrelease_retry +#else +#error Either USE_CAS or USE_LDX_STX must be set. +#endif + +// On success, return. +Lrelease_ret: + ret + +Lslowpath_release: + CONDITIONAL USE_LDX_STX, \ + clrex + CALL_SLOWPATH _swift_release_preservemost + +.alt_entry _bridgeObjectReleaseClientObjC +_bridgeObjectReleaseClientObjC: + CONDITIONAL PTRAUTH, \ + pacibsp + stp x0, x9, [sp, #-0x50]! + stp x10, x11, [sp, #0x10] + stp x12, x13, [sp, #0x20] + stp x14, x15, [sp, #0x30] + stp fp, lr, [sp, #0x40]; + add fp, sp, #0x40 + + // Clear the unused bits from the pointer + and x0, x0, #0x0ffffffffffffff8 + bl _objc_release + + ldp fp, lr, [sp, #0x40] + ldp x14, x15, [sp, #0x30] + ldp x12, x13, [sp, #0x20] + ldp x10, x11, [sp, #0x10] + ldp x0, x9, [sp], #0x50 +LbridgeObjectReleaseObjCRet: + CONDITIONAL PTRAUTH, \ + retab + CONDITIONAL !PTRAUTH, \ + ret + + +.private_extern _swift_bridgeObjectRetainClient +#if SWIFT_OBJC_INTEROP +_swift_bridgeObjectRetainClient: + tbz x0, #63, LbridgeObjectRetainNotTagged + ret +LbridgeObjectRetainNotTagged: + tbnz x0, #62, _swift_bridgeObjectRetainClientObjC + +.alt_entry _swift_retainClient +#else +.set _swift_bridgeObjectRetainClient, _swift_retainClient +#endif + +.private_extern _swift_retainClient +_swift_retainClient: +// RR of NULL or values with high bit set is a no-op. + cmp x0, #0 + b.le Lretain_ret + +// Mask off spare bits that may have come in from bridgeObjectRetain. Keep the +// original value in x0 since we have to return it. + and x1, x0, 0xffffffffffffff8 + +// We'll operate on the address of the refcount field, which is 8 bytes into +// the object. + add x1, x1, #8 + +// Load the current value of the refcount field when using CAS. + CONDITIONAL USE_CAS, \ + ldr x16, [x1] + +Lretain_retry: +// Get the slow path mask and see if the refcount field has any of those bits +// set. + adrp x17, LretainRelease_slowpath_mask@PAGE + ldr x17, [x17, LretainRelease_slowpath_mask@PAGEOFF] + +// Load-exclusive of the current value in the refcount field when using LLSC. +// stxr does not update x16 like cas does, so this load must be inside the loop. +// ldxr/stxr are not guaranteed to make forward progress if there are memory +// accesses between them, so we need to do this after getting the mask above. + CONDITIONAL USE_LDX_STX, \ + ldxr x16, [x1] + + tst x16, x17 + + b.ne Lslowpath_retain + +// Compute a refcount field with the strong refcount incremented. +// TODO: overflow checking + mov x17, #STRONG_RC_ONE + add x17, x16, x17 + +#if USE_CAS +// Save the old value so we can check if the CAS succeeded. + mov x2, x16 + +// Compare and swap the new value into the refcount field. Retain can use +// relaxed memory ordering. + cas x16, x17, [x1] + +// The previous value of the refcount field is now in x16. We succeeded if that +// value is the same as the old value we had before. If we failed, retry. + cmp x2, x16 + b.ne Lretain_retry + +#elif USE_LDX_STX +// Try to store the updated value. + stxr w16, x17, [x1] + +// On failure, retry. + cbnz w16, Lretain_retry +#else +#error Either USE_CAS or USE_LDX_STX must be set. +#endif + +// If we succeeded, return. Retain returns the object pointer being retained, +// which is still in x0 at this point. + +Lretain_ret: + ret + +Lslowpath_retain: + CONDITIONAL USE_LDX_STX, \ + clrex + CALL_SLOWPATH _swift_retain_preservemost + +.alt_entry _swift_bridgeObjectRetainClientObjC +_swift_bridgeObjectRetainClientObjC: + CONDITIONAL PTRAUTH, \ + pacibsp + stp x0, x9, [sp, #-0x50]! + stp x10, x11, [sp, #0x10] + stp x12, x13, [sp, #0x20] + stp x14, x15, [sp, #0x30] + stp fp, lr, [sp, #0x40]; + add fp, sp, #0x40 + + // Clear the unused bits from the pointer + and x0, x0, #0x0ffffffffffffff8 + bl _objc_retain + + ldp fp, lr, [sp, #0x40] + ldp x14, x15, [sp, #0x30] + ldp x12, x13, [sp, #0x20] + ldp x10, x11, [sp, #0x10] + ldp x0, x9, [sp], #0x50 + CONDITIONAL PTRAUTH, \ + retab + CONDITIONAL !PTRAUTH, \ + ret + +#else + +.private_extern _placeholderSymbol +.set _placeholderSymbol, 0 + +#endif diff --git a/stdlib/public/SwiftShims/swift/shims/RefCount.h b/stdlib/public/SwiftShims/swift/shims/RefCount.h index 736042da2f6a7..69b3b953e0338 100644 --- a/stdlib/public/SwiftShims/swift/shims/RefCount.h +++ b/stdlib/public/SwiftShims/swift/shims/RefCount.h @@ -182,7 +182,8 @@ namespace swift { } // FIXME: HACK: copied from HeapObject.cpp -extern "C" SWIFT_LIBRARY_VISIBILITY SWIFT_NOINLINE SWIFT_USED void +extern "C" SWIFT_LIBRARY_VISIBILITY SWIFT_NOINLINE SWIFT_USED SWIFT_REFCOUNT_CC +void _swift_release_dealloc(swift::HeapObject *object); namespace swift { @@ -714,6 +715,7 @@ class RefCounts { // Out-of-line slow paths. SWIFT_NOINLINE + SWIFT_REFCOUNT_CC HeapObject *incrementSlow(RefCountBits oldbits, uint32_t inc); SWIFT_NOINLINE @@ -815,7 +817,7 @@ class RefCounts { // can be directly returned from swift_retain. This makes the call to // incrementSlow() a tail call. SWIFT_ALWAYS_INLINE - HeapObject *increment(uint32_t inc = 1) { + HeapObject *increment(HeapObject *returning, uint32_t inc = 1) { auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); // Constant propagation will remove this in swift_retain, it should only @@ -835,7 +837,7 @@ class RefCounts { } } while (!refCounts.compare_exchange_weak(oldbits, newbits, std::memory_order_relaxed)); - return getHeapObject(); + return returning; } SWIFT_ALWAYS_INLINE @@ -1010,6 +1012,7 @@ class RefCounts { // First slow path of doDecrement, where the object may need to be deinited. // Side table is handled in the second slow path, doDecrementSideTable(). template + SWIFT_REFCOUNT_CC bool doDecrementSlow(RefCountBits oldbits, uint32_t dec) { RefCountBits newbits; @@ -1379,7 +1382,7 @@ class HeapObjectSideTableEntry { // STRONG void incrementStrong(uint32_t inc) { - refCounts.increment(inc); + refCounts.increment(nullptr, inc); } template diff --git a/stdlib/public/SwiftShims/swift/shims/Visibility.h b/stdlib/public/SwiftShims/swift/shims/Visibility.h index d97f97a750e37..f589c50f0ca1c 100644 --- a/stdlib/public/SwiftShims/swift/shims/Visibility.h +++ b/stdlib/public/SwiftShims/swift/shims/Visibility.h @@ -109,7 +109,9 @@ #define SWIFT_WEAK_IMPORT #endif -#if __has_attribute(musttail) +// WASM says yes to __has_attribute(musttail) but doesn't support using it, so +// exclude WASM from SWIFT_MUSTTAIL. +#if __has_attribute(musttail) && !defined(__wasm__) #define SWIFT_MUSTTAIL [[clang::musttail]] #else #define SWIFT_MUSTTAIL diff --git a/stdlib/public/core/CMakeLists.txt b/stdlib/public/core/CMakeLists.txt index 7146ea7f6c82f..09998fececf94 100644 --- a/stdlib/public/core/CMakeLists.txt +++ b/stdlib/public/core/CMakeLists.txt @@ -445,6 +445,11 @@ if(TARGET libSwiftScan) list(APPEND tooling_stdlib_deps libSwiftScan) endif() +set(stdlib_link_libraries) +if(SWIFT_BUILD_CLIENT_RETAIN_RELEASE) + list(APPEND stdlib_link_libraries swiftClientRetainRelease) +endif() + add_swift_target_library(swiftCore ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} ${swiftCore_common_options} @@ -452,6 +457,7 @@ add_swift_target_library(swiftCore FILE_DEPENDS ${swiftCore_common_dependencies} DEPENDS ${tooling_stdlib_deps} + LINK_LIBRARIES ${stdlib_link_libraries} INSTALL_IN_COMPONENT stdlib MACCATALYST_BUILD_FLAVOR diff --git a/stdlib/public/runtime/HeapObject.cpp b/stdlib/public/runtime/HeapObject.cpp index 38ee10a62e6f5..cc74c85980f3f 100644 --- a/stdlib/public/runtime/HeapObject.cpp +++ b/stdlib/public/runtime/HeapObject.cpp @@ -61,6 +61,44 @@ using namespace swift; #error "The runtime must be built with a compiler that supports swiftcall." #endif +#if SWIFT_REFCOUNT_CC_PRESERVEMOST +// These assembly definitions support the swiftClientRetainRelease library which +// is currently implemented for ARM64 Mach-O. +#if __arm64__ && __LP64__ && defined(__APPLE__) && defined(__MACH__) +asm(R"( +// Define a mask used by ClientRetainRelease to determine when it must call into +// the runtime. The symbol's address is used as the mask, rather than its +// contents, to eliminate one load instruction when using it. This is imported +// weakly, which makes its address zero when running against older runtimes. +// ClientRetainRelease references it using an addend of 0x8000000000000000, +// whicrh produces the appropriate mask in that case. Since the mask is still +// unchanged in this version of the runtime, we export this symbol as zero. If a +// different mask is ever needed, the address of this symbol needs to be set to +// 0x8000000000000000 less than that value so that it comes out right in +// ClientRetainRelease. + .globl __swift_retainRelease_slowpath_mask_v1 + .set __swift_retainRelease_slowpath_mask_v1, 0 + +// Define aliases for swift_retain/release that indicate they use preservemost. +// ClientRetainRelease will reference these so that it can fall back to a +// register-preserving register on older runtimes. + .globl _swift_retain_preservemost + .set _swift_retain_preservemost, _swift_retain + .globl _swift_release_preservemost + .set _swift_release_preservemost, _swift_release + +// A weak definition can only be overridden by a strong definition if the +// library with the strong definition contains at least one weak definition. +// Create a placeholder weak definition here to allow that to work. + .weak_definition _swift_release_preservemost_weak_placeholder + .globl _swift_release_preservemost_weak_placeholder + _swift_release_preservemost_weak_placeholder: + .byte 0 +)"); +#endif +#endif + + /// Returns true if the pointer passed to a native retain or release is valid. /// If false, the operation should immediately return. SWIFT_ALWAYS_INLINE @@ -116,6 +154,20 @@ static HeapObject *_swift_tryRetain_(HeapObject *object) return _ ## name ## _ args; \ } while(0) +// SWIFT_REFCOUNT_CC functions make the call to the "might be swizzled" path +// through an adapter marked noinline and with the refcount CC. This allows the +// fast path to avoid pushing a stack frame. Without this adapter, clang emits +// code that pushes a stack frame right away, then does the fast path or slow +// path. +#define CALL_IMPL_SWIFT_REFCOUNT_CC(name, args) \ + do { \ + if (SWIFT_UNLIKELY( \ + _swift_enableSwizzlingOfAllocationAndRefCountingFunctions_forInstrumentsOnly \ + .load(std::memory_order_relaxed))) \ + SWIFT_MUSTTAIL return _##name##_adapter args; \ + return _##name##_ args; \ + } while (0) + #define CALL_IMPL_CHECK(name, args) do { \ void *fptr; \ memcpy(&fptr, (void *)&_ ## name, sizeof(fptr)); \ @@ -421,26 +473,46 @@ HeapObject *swift::swift_allocEmptyBox() { } // Forward-declare this, but define it after swift_release. -extern "C" SWIFT_LIBRARY_VISIBILITY SWIFT_NOINLINE SWIFT_USED void +extern "C" SWIFT_LIBRARY_VISIBILITY SWIFT_NOINLINE SWIFT_USED SWIFT_REFCOUNT_CC +void _swift_release_dealloc(HeapObject *object); -SWIFT_ALWAYS_INLINE -static HeapObject *_swift_retain_(HeapObject *object) { +SWIFT_ALWAYS_INLINE static HeapObject *_swift_retain_(HeapObject *object) { SWIFT_RT_TRACK_INVOCATION(object, swift_retain); if (isValidPointerForNativeRetain(object)) { + // swift_bridgeObjectRetain might call us with a pointer that has spare bits + // set, and expects us to return that unmasked value. Mask off those bits + // for the actual increment operation. + HeapObject *masked = (HeapObject *)((uintptr_t)object & + ~heap_object_abi::SwiftSpareBitsMask); + // Return the result of increment() to make the eventual call to // incrementSlow a tail call, which avoids pushing a stack frame on the fast // path on ARM64. - return object->refCounts.increment(1); + return masked->refCounts.increment(object, 1); } return object; } +SWIFT_REFCOUNT_CC +static HeapObject *_swift_retain_adapterImpl(HeapObject *object) { + HeapObject *masked = + (HeapObject *)((uintptr_t)object & ~heap_object_abi::SwiftSpareBitsMask); + _swift_retain(masked); + return object; +} + +// This strange construct prevents the compiler from creating an unnecessary +// stack frame in swift_retain. A direct tail call to _swift_retain_adapterImpl +// somehow causes clang to emit a stack frame. +static HeapObject *(*SWIFT_REFCOUNT_CC volatile _swift_retain_adapter)( + HeapObject *object) = _swift_retain_adapterImpl; + HeapObject *swift::swift_retain(HeapObject *object) { #ifdef SWIFT_THREADING_NONE return swift_nonatomic_retain(object); #else - CALL_IMPL(swift_retain, (object)); + CALL_IMPL_SWIFT_REFCOUNT_CC(swift_retain, (object)); #endif } @@ -457,7 +529,7 @@ SWIFT_ALWAYS_INLINE static HeapObject *_swift_retain_n_(HeapObject *object, uint32_t n) { SWIFT_RT_TRACK_INVOCATION(object, swift_retain_n); if (isValidPointerForNativeRetain(object)) - object->refCounts.increment(n); + return object->refCounts.increment(object, n); return object; } @@ -483,11 +555,16 @@ static void _swift_release_(HeapObject *object) { object->refCounts.decrementAndMaybeDeinit(1); } +SWIFT_REFCOUNT_CC SWIFT_NOINLINE +static void _swift_release_adapter(HeapObject *object) { + _swift_release(object); +} + void swift::swift_release(HeapObject *object) { #ifdef SWIFT_THREADING_NONE swift_nonatomic_release(object); #else - CALL_IMPL(swift_release, (object)); + CALL_IMPL_SWIFT_REFCOUNT_CC(swift_release, (object)); #endif } diff --git a/test/IRGen/function_types.sil b/test/IRGen/function_types.sil index dd4eadcc3a2fb..30c9c2eb3367b 100644 --- a/test/IRGen/function_types.sil +++ b/test/IRGen/function_types.sil @@ -26,8 +26,8 @@ entry(%x : $@convention(thin) () -> ()): // CHECK-LABEL: define{{( dllexport)?}}{{( protected)?}} swiftcc { ptr, ptr } @thick_func_value(ptr %0, ptr %1) {{.*}} { // CHECK-NEXT: entry: -// CHECK-NEXT: call ptr @swift_retain(ptr returned %1) {{#[0-9]+}} -// CHECK-NEXT: call void @swift_release(ptr %1) {{#[0-9]+}} +// CHECK-NEXT: call{{( preserve_mostcc)?}} ptr @swift_retain{{(Client)?}}(ptr returned %1) {{#[0-9]+}} +// CHECK-NEXT: call{{( preserve_mostcc)?}} void @swift_release{{(Client)?}}(ptr %1) {{#[0-9]+}} // CHECK-NEXT: %3 = insertvalue { ptr, ptr } undef, ptr %0, 0 // CHECK-NEXT: %4 = insertvalue { ptr, ptr } %3, ptr %1, 1 // CHECK-NEXT: ret { ptr, ptr } %4