[IRGen][Runtime] Add emit-into-client retain/release calls for Darwin ARM64.

CMakeLists.txt

@@ -271,6 +271,19 @@ option(SWIFT_BUILD_STDLIB_CXX_MODULE
   "If not building stdlib, controls whether to build the Cxx module"
   TRUE)
 
+# The swiftClientRetainRelease library is currently only available for Darwin
+# platforms.
+if(SWIFT_HOST_VARIANT_SDK IN_LIST SWIFT_DARWIN_PLATFORMS)
+  # Off by default everywhere for now.
+  option(SWIFT_BUILD_CLIENT_RETAIN_RELEASE
+    "Build the swiftClientRetainRelease library"
+    FALSE)
+else()
+  option(SWIFT_BUILD_CLIENT_RETAIN_RELEASE
+    "Build the swiftClientRetainRelease library"
+    FALSE)
+endif()
+
 # In many cases, the CMake build system needs to determine whether to include
 # a directory, or perform other actions, based on whether the stdlib or SDK is
 # being built at all -- statically or dynamically. Please note that these

include/swift/AST/IRGenOptions.h

@@ -548,6 +548,9 @@ class IRGenOptions {
   // Whether to emit mergeable or non-mergeable traps.
   unsigned MergeableTraps : 1;
 
+  /// Enable the use of swift_retain/releaseClient functions.
+  unsigned EnableClientRetainRelease : 1;
+
   /// The number of threads for multi-threaded code generation.
   unsigned NumThreads = 0;
 
@@ -675,6 +678,7 @@ class IRGenOptions {
         EmitAsyncFramePushPopMetadata(true), EmitTypeMallocForCoroFrame(true),
         AsyncFramePointerAll(false), UseProfilingMarkerThunks(false),
         UseCoroCCX8664(false), UseCoroCCArm64(false), MergeableTraps(false),
+        EnableClientRetainRelease(false),
         DebugInfoForProfiling(false), CmdArgs(),
         SanitizeCoverage(llvm::SanitizerCoverageOptions()),
         TypeInfoFilter(TypeInfoDumpFilter::All),

include/swift/Option/FrontendOptions.td

@@ -1477,6 +1477,13 @@ def mergeable_traps :
   Flag<["-"], "mergeable-traps">,
   HelpText<"Emit mergeable traps even in optimized builds">;
 
+def enable_client_retain_release :
+  Flag<["-"], "enable-client-retain-release">,
+  HelpText<"Enable use of swift_retain/releaseClient functions">;
+def disable_client_retain_release :
+  Flag<["-"], "disable-client-retain-release">,
+  HelpText<"Disable use of swift_retain/releaseClient functions">;
+
 def enable_new_llvm_pass_manager :
   Flag<["-"], "enable-new-llvm-pass-manager">,
   HelpText<"Enable the new llvm pass manager">;

include/swift/Runtime/Config.h

@@ -248,6 +248,17 @@ extern uintptr_t __COMPATIBILITY_LIBRARIES_CANNOT_CHECK_THE_IS_SWIFT_BIT_DIRECTL
 // so changing this value is not sufficient.
 #define SWIFT_DEFAULT_LLVM_CC llvm::CallingConv::C
 
+// Define the calling convention for refcounting functions for targets where it
+// differs from the standard calling convention. Currently this is only used for
+// swift_retain, swift_release, and some internal helper functions that they
+// call.
+#if defined(__aarch64__)
+#define SWIFT_REFCOUNT_CC SWIFT_CC_PreserveMost
+#define SWIFT_REFCOUNT_CC_PRESERVEMOST 1
+#else
+#define SWIFT_REFCOUNT_CC
+#endif
+
 /// Should we use absolute function pointers instead of relative ones?
 /// WebAssembly target uses it by default.
 #ifndef SWIFT_COMPACT_ABSOLUTE_FUNCTION_POINTER

include/swift/Runtime/CustomRRABI.h

@@ -59,6 +59,9 @@ namespace swift {
 template <typename Ret, typename Param>
 Param returnTypeHelper(Ret (*)(Param)) {}
 
+template <typename Ret, typename Param>
+Param returnTypeHelper(SWIFT_REFCOUNT_CC Ret (*)(Param)) {}
+
 #if defined(__LP64__) || defined(_LP64)
 #define REGISTER_SUBSTITUTION_PREFIX ""
 #define REGISTER_PREFIX "x"

include/swift/Runtime/HeapObject.h

@@ -137,6 +137,7 @@ HeapObject* swift_allocEmptyBox();
 /// It may also prove worthwhile to have this use a custom CC
 /// which preserves a larger set of registers.
 SWIFT_RUNTIME_EXPORT
+SWIFT_REFCOUNT_CC
 HeapObject *swift_retain(HeapObject *object);
 
 SWIFT_RUNTIME_EXPORT
@@ -173,6 +174,7 @@ bool swift_isDeallocating(HeapObject *object);
 ///      - maybe a variant that can assume a non-null object
 /// It's unlikely that a custom CC would be beneficial here.
 SWIFT_RUNTIME_EXPORT
+SWIFT_REFCOUNT_CC
 void swift_release(HeapObject *object);
 
 SWIFT_RUNTIME_EXPORT

include/swift/Runtime/RuntimeFunctions.def

@@ -220,6 +220,22 @@ FUNCTION(NativeStrongRelease, Swift, swift_release, C_CC, AlwaysAvailable,
          EFFECT(RuntimeEffect::RefCounting, RuntimeEffect::Deallocating),
          UNKNOWN_MEMEFFECTS)
 
+// void *swift_retain(void *ptr);
+FUNCTION(NativeStrongRetainClient, Swift, swift_retainClient, SwiftClientRR_CC, AlwaysAvailable,
+         RETURNS(RefCountedPtrTy),
+         ARGS(RefCountedPtrTy),
+         ATTRS(NoUnwind, FirstParamReturned, WillReturn),
+         EFFECT(RuntimeEffect::RefCounting),
+         UNKNOWN_MEMEFFECTS)
+
+// void swift_release(void *ptr);
+FUNCTION(NativeStrongReleaseClient, Swift, swift_releaseClient, SwiftClientRR_CC, AlwaysAvailable,
+         RETURNS(VoidTy),
+         ARGS(RefCountedPtrTy),
+         ATTRS(NoUnwind),
+         EFFECT(RuntimeEffect::RefCounting, RuntimeEffect::Deallocating),
+         UNKNOWN_MEMEFFECTS)
+
 // void *swift_retain_n(void *ptr, int32_t n);
 FUNCTION(NativeStrongRetainN, Swift, swift_retain_n, C_CC, AlwaysAvailable,
          RETURNS(RefCountedPtrTy),
@@ -420,6 +436,24 @@ FUNCTION(BridgeObjectStrongRelease, Swift, swift_bridgeObjectRelease,
          EFFECT(RuntimeEffect::RefCounting, RuntimeEffect::Deallocating),
          UNKNOWN_MEMEFFECTS)
 
+// void *swift_bridgeObjectRetainClient(void *ptr);
+FUNCTION(BridgeObjectStrongRetainClient, Swift, swift_bridgeObjectRetainClient,
+         SwiftClientRR_CC, AlwaysAvailable,
+         RETURNS(BridgeObjectPtrTy),
+         ARGS(BridgeObjectPtrTy),
+         ATTRS(NoUnwind, FirstParamReturned),
+         EFFECT(RuntimeEffect::RefCounting),
+         UNKNOWN_MEMEFFECTS)
+
+// void *swift_bridgeObjectReleaseClient(void *ptr);
+FUNCTION(BridgeObjectStrongReleaseClient, Swift, swift_bridgeObjectReleaseClient,
+         SwiftClientRR_CC, AlwaysAvailable,
+         RETURNS(VoidTy),
+         ARGS(BridgeObjectPtrTy),
+         ATTRS(NoUnwind),
+         EFFECT(RuntimeEffect::RefCounting, RuntimeEffect::Deallocating),
+         UNKNOWN_MEMEFFECTS)
+
 // void *swift_nonatomic_bridgeObjectRetain(void *ptr);
 FUNCTION(NonAtomicBridgeObjectStrongRetain, Swift, swift_nonatomic_bridgeObjectRetain,
          C_CC, AlwaysAvailable,

lib/Frontend/CompilerInvocation.cpp

@@ -3983,6 +3983,11 @@ static bool ParseIRGenArgs(IRGenOptions &Opts, ArgList &Args,
 
   Opts.MergeableTraps = Args.hasArg(OPT_mergeable_traps);
 
+  Opts.EnableClientRetainRelease =
+    Args.hasFlag(OPT_enable_client_retain_release,
+                 OPT_disable_client_retain_release,
+                 Opts.EnableClientRetainRelease);
+
   Opts.EnableObjectiveCProtocolSymbolicReferences =
     Args.hasFlag(OPT_enable_objective_c_protocol_symbolic_references,
                  OPT_disable_objective_c_protocol_symbolic_references,

lib/IRGen/GenHeap.cpp

@@ -997,11 +997,16 @@ void IRGenFunction::emitNativeStrongRetain(llvm::Value *value,
     value = Builder.CreateBitCast(value, IGM.RefCountedPtrTy);
 
   // Emit the call.
-  llvm::CallInst *call = Builder.CreateCall(
-      (atomicity == Atomicity::Atomic)
-          ? IGM.getNativeStrongRetainFunctionPointer()
-          : IGM.getNativeNonAtomicStrongRetainFunctionPointer(),
-      value);
+  FunctionPointer function;
+  if (atomicity == Atomicity::Atomic &&
+      IGM.TargetInfo.HasSwiftClientRRLibrary &&
+      getOptions().EnableClientRetainRelease)
+    function = IGM.getNativeStrongRetainClientFunctionPointer();
+  else if (atomicity == Atomicity::Atomic)
+    function = IGM.getNativeStrongRetainFunctionPointer();
+  else
+    function = IGM.getNativeNonAtomicStrongRetainFunctionPointer();
+  llvm::CallInst *call = Builder.CreateCall(function, value);
   call->setDoesNotThrow();
   call->addParamAttr(0, llvm::Attribute::Returned);
 }
@@ -1257,10 +1262,16 @@ void IRGenFunction::emitNativeStrongRelease(llvm::Value *value,
                                             Atomicity atomicity) {
   if (doesNotRequireRefCounting(value))
     return;
-  emitUnaryRefCountCall(*this, (atomicity == Atomicity::Atomic)
-                                   ? IGM.getNativeStrongReleaseFn()
-                                   : IGM.getNativeNonAtomicStrongReleaseFn(),
-                        value);
+  llvm::Constant *function;
+  if (atomicity == Atomicity::Atomic &&
+      IGM.TargetInfo.HasSwiftClientRRLibrary &&
+      getOptions().EnableClientRetainRelease)
+    function = IGM.getNativeStrongReleaseClientFn();
+  else if (atomicity == Atomicity::Atomic)
+    function = IGM.getNativeStrongReleaseFn();
+  else
+    function = IGM.getNativeNonAtomicStrongReleaseFn();
+  emitUnaryRefCountCall(*this, function, value);
 }
 
 void IRGenFunction::emitNativeSetDeallocating(llvm::Value *value) {
@@ -1353,20 +1364,30 @@ void IRGenFunction::emitUnknownStrongRelease(llvm::Value *value,
 
 void IRGenFunction::emitBridgeStrongRetain(llvm::Value *value,
                                            Atomicity atomicity) {
-  emitUnaryRefCountCall(*this,
-                        (atomicity == Atomicity::Atomic)
-                            ? IGM.getBridgeObjectStrongRetainFn()
-                            : IGM.getNonAtomicBridgeObjectStrongRetainFn(),
-                        value);
+  llvm::Constant *function;
+  if (atomicity == Atomicity::Atomic &&
+      IGM.TargetInfo.HasSwiftClientRRLibrary &&
+      getOptions().EnableClientRetainRelease)
+    function = IGM.getBridgeObjectStrongRetainClientFn();
+  else if (atomicity == Atomicity::Atomic)
+    function = IGM.getBridgeObjectStrongRetainFn();
+  else
+    function = IGM.getNonAtomicBridgeObjectStrongRetainFn();
+  emitUnaryRefCountCall(*this, function, value);
 }
 
 void IRGenFunction::emitBridgeStrongRelease(llvm::Value *value,
                                             Atomicity atomicity) {
-  emitUnaryRefCountCall(*this,
-                        (atomicity == Atomicity::Atomic)
-                            ? IGM.getBridgeObjectStrongReleaseFn()
-                            : IGM.getNonAtomicBridgeObjectStrongReleaseFn(),
-                        value);
+  llvm::Constant *function;
+  if (atomicity == Atomicity::Atomic &&
+      IGM.TargetInfo.HasSwiftClientRRLibrary &&
+      getOptions().EnableClientRetainRelease)
+    function = IGM.getBridgeObjectStrongReleaseClientFn();
+  else if (atomicity == Atomicity::Atomic)
+    function = IGM.getBridgeObjectStrongReleaseFn();
+  else
+    function = IGM.getNonAtomicBridgeObjectStrongReleaseFn();
+  emitUnaryRefCountCall(*this, function, value);
 }
 
 void IRGenFunction::emitErrorStrongRetain(llvm::Value *value) {

lib/IRGen/IRGenModule.cpp

@@ -564,8 +564,9 @@ IRGenModule::IRGenModule(IRGenerator &irgen,
   InvariantMetadataID = getLLVMContext().getMDKindID("invariant.load");
   InvariantNode = llvm::MDNode::get(getLLVMContext(), {});
   DereferenceableID = getLLVMContext().getMDKindID("dereferenceable");
-  
+
   C_CC = getOptions().PlatformCCallingConvention;
+  SwiftClientRR_CC = llvm::CallingConv::PreserveMost;
   // TODO: use "tinycc" on platforms that support it
   DefaultCC = SWIFT_DEFAULT_LLVM_CC;
 
@@ -1730,6 +1731,11 @@ void IRGenModule::addLinkLibraries() {
     registerLinkLibrary(
         LinkLibrary{"objc", LibraryKind::Library, /*static=*/false});
 
+  if (TargetInfo.HasSwiftClientRRLibrary &&
+      getOptions().EnableClientRetainRelease)
+    registerLinkLibrary(LinkLibrary{"swiftClientRetainRelease",
+                                    LibraryKind::Library, /*static=*/true});
+
   // If C++ interop is enabled, add -lc++ on Darwin and -lstdc++ on linux.
   // Also link with C++ bridging utility module (Cxx) and C++ stdlib overlay
   // (std) if available.

lib/IRGen/IRGenModule.h

@@ -921,6 +921,7 @@ class IRGenModule {
   llvm::CallingConv::ID SwiftCC;       /// swift calling convention
   llvm::CallingConv::ID SwiftAsyncCC;  /// swift calling convention for async
   llvm::CallingConv::ID SwiftCoroCC;   /// swift calling convention for callee-allocated coroutines
+  llvm::CallingConv::ID SwiftClientRR_CC; /// swift client retain/release calling convention
 
   /// What kind of tail call should be used for async->async calls.
   llvm::CallInst::TailCallKind AsyncTailCallKind;

lib/IRGen/SwiftTargetInfo.cpp

@@ -71,6 +71,14 @@ static void configureARM64(IRGenModule &IGM, const llvm::Triple &triple,
   // half for the kernel.
   target.SwiftRetainIgnoresNegativeValues = true;
 
+  // ARM64 Darwin has swiftClientRetainRelease, but not in Embedded mode. JIT
+  // mode can't load the static library, so disable it there as well.
+  if (triple.isOSDarwin() &&
+      !IGM.getSwiftModule()->getASTContext().LangOpts.hasFeature(
+          Feature::Embedded) &&
+      !IGM.getOptions().UseJIT)
+    target.HasSwiftClientRRLibrary = true;
+
   target.UsableSwiftAsyncContextAddrIntrinsic = true;
 }
 

lib/IRGen/SwiftTargetInfo.h

@@ -114,6 +114,9 @@ class SwiftTargetInfo {
   /// "negative" pointer values.
   bool SwiftRetainIgnoresNegativeValues = false;
 
+  /// True if the swiftClientRetainRelease static library is available.
+  bool HasSwiftClientRRLibrary = false;
+
   bool UsableSwiftAsyncContextAddrIntrinsic = false;
 };
 

lib/LLVMPasses/LLVMARCOpts.h

@@ -64,6 +64,13 @@ inline RT_Kind classifyInstruction(const llvm::Instruction &I) {
     .Case("__swift_" #TextualName, RT_ ## Name)
 #include "LLVMSwift.def"
 
+    // Identify "Client" versions of reference counting entry points.
+#define SWIFT_FUNC(Name, MemBehavior, TextualName) \
+    .Case("swift_" #TextualName "Client", RT_ ## Name)
+#define SWIFT_INTERNAL_FUNC_NEVER_NONATOMIC(Name, MemBehavior, TextualName) \
+    .Case("__swift_" #TextualName "Client", RT_ ## Name)
+#include "LLVMSwift.def"
+
     // Support non-atomic versions of reference counting entry points.
 #define SWIFT_FUNC(Name, MemBehavior, TextualName) \
     .Case("swift_nonatomic_" #TextualName, RT_ ## Name)

stdlib/public/CMakeLists.txt

@@ -320,6 +320,10 @@ endif()
 if(SWIFT_BUILD_STDLIB)
   # These must be kept in dependency order so that any referenced targets
   # exist at the time we look for them in add_swift_*.
+  if(SWIFT_BUILD_CLIENT_RETAIN_RELEASE)
+    add_subdirectory(ClientRetainRelease)
+  endif()
+
   add_subdirectory(runtime)
   add_subdirectory(stubs)
   add_subdirectory(core)
@@ -389,4 +393,3 @@ endif()
 if(SWIFT_BUILD_LIBEXEC)
 add_subdirectory(libexec)
 endif()
-

stdlib/public/ClientRetainRelease/CMakeLists.txt

@@ -0,0 +1,18 @@
+add_swift_target_library(swiftClientRetainRelease
+  STATIC DONT_EMBED_BITCODE NOSWIFTRT
+  RetainRelease.s
+  C_COMPILE_FLAGS ${SWIFT_RUNTIME_CXX_FLAGS}
+  $<$<BOOL:${SWIFT_STDLIB_ENABLE_OBJC_INTEROP}>:-DSWIFT_OBJC_INTEROP=1>
+  LINK_FLAGS ${SWIFT_RUNTIME_LINK_FLAGS}
+  SWIFT_COMPILE_FLAGS ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS}
+
+  DEPLOYMENT_VERSION_OSX ${COMPATIBILITY_MINIMUM_DEPLOYMENT_VERSION_OSX}
+  DEPLOYMENT_VERSION_IOS ${COMPATIBILITY_MINIMUM_DEPLOYMENT_VERSION_IOS}
+  DEPLOYMENT_VERSION_TVOS ${COMPATIBILITY_MINIMUM_DEPLOYMENT_VERSION_TVOS}
+  DEPLOYMENT_VERSION_WATCHOS ${COMPATIBILITY_MINIMUM_DEPLOYMENT_VERSION_WATCHOS}
+  DEPLOYMENT_VERSION_XROS ${COMPATIBILITY_MINIMUM_DEPLOYMENT_VERSION_XROS}
+
+  MACCATALYST_BUILD_FLAVOR "zippered"
+
+  INSTALL_IN_COMPONENT compiler
+  INSTALL_WITH_SHARED)