[ObjC][ARC] Use operand bundle 'clang.arc.rv' instead of explicitly

emitting retainRV or claimRV calls in the IR

Background:

This patch makes changes to the front-end and middle-end that are
needed to fix a longstanding problem where llvm breaks ARC's autorelease
optimization (see the link below) by separating calls from the marker
instructions or retainRV/claimRV calls. The backend changes are in
https://reviews.llvm.org/D92569.

https://clang.llvm.org/docs/AutomaticReferenceCounting.html#arc-runtime-objc-autoreleasereturnvalue

What this patch does to fix the problem:

- The front-end adds operand bundle "clang.arc.rv" to calls, which
  indicates the call is implicitly followed by a marker instruction and
  an implicit retainRV/claimRV call that consumes the call result. In
  addition, it emits a call to @llvm.objc.clang.arc.noop.use, which
  consumes the call result, to prevent the middle-end passes from changing
  the return type of the called function. This is currently done only when
  the target is arm64 and the optimization level is higher than -O0.

- ARC optimizer temporarily emits retainRV/claimRV calls after the calls
  with the operand bundle in the IR and removes the inserted calls after
  processing the function.

- ARC contract pass emits retainRV/claimRV calls after the call with the
  operand bundle. It doesn't remove the operand bundle on the call since
  the backend needs it to emit the marker instruction. The retainRV and
  claimRV calls are emitted late in the pipeline to prevent optimization
  passes from transforming the IR in a way that makes it harder for the
  ARC middle-end passes to figure out the def-use relationship between
  the call and the retainRV/claimRV calls (which is the cause of
  PR31925).

- The function inliner removes an autoreleaseRV call in the callee if
  nothing in the callee prevents it from being paired up with the
  retainRV/claimRV call in the caller. It then inserts a release call if
  the call is annotated with claimRV since autoreleaseRV+claimRV is
  equivalent to a release. If it cannot find an autoreleaseRV call, it
  tries to transfer the operand bundle to a function call in the callee.
  This is important since ARC optimizer can remove the autoreleaseRV
  returning the callee result, which makes it impossible to pair it up
  with the retainRV/claimRV call in the caller. If that fails, it simply
  emits a retain call in the IR if the implicit call is a call to
  retainRV and does nothing if it's a call to claimRV.

Future work:

- Use the operand bundle on x86-64.

- Fix the auto upgrader to convert call+retainRV/claimRV pairs into
  calls annotated with the operand bundles.

rdar://71443534

Differential Revision: https://reviews.llvm.org/D92808

clang/lib/CodeGen/CGObjC.cpp

@@ -23,6 +23,7 @@
 #include "clang/Basic/Diagnostic.h"
 #include "clang/CodeGen/CGFunctionInfo.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/Analysis/ObjCARCUtil.h"
 #include "llvm/BinaryFormat/MachO.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/InlineAsm.h"
@@ -2078,6 +2079,15 @@ void CodeGenFunction::EmitARCIntrinsicUse(ArrayRef<llvm::Value*> values) {
   EmitNounwindRuntimeCall(fn, values);
 }
 
+/// Emit a call to "clang.arc.noop.use", which consumes the result of a call
+/// that has operand bundle "clang.arc.rv".
+void CodeGenFunction::EmitARCNoopIntrinsicUse(ArrayRef<llvm::Value *> values) {
+  llvm::Function *&fn = CGM.getObjCEntrypoints().clang_arc_noop_use;
+  if (!fn)
+    fn = CGM.getIntrinsic(llvm::Intrinsic::objc_clang_arc_noop_use);
+  EmitNounwindRuntimeCall(fn, values);
+}
+
 static void setARCRuntimeFunctionLinkage(CodeGenModule &CGM, llvm::Value *RTF) {
   if (auto *F = dyn_cast<llvm::Function>(RTF)) {
     // If the target runtime doesn't naturally support ARC, emit weak
@@ -2304,10 +2314,11 @@ static void emitAutoreleasedReturnValueMarker(CodeGenFunction &CGF) {
     // with this marker yet, so leave a breadcrumb for the ARC
     // optimizer to pick up.
     } else {
-      const char *markerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
-      if (!CGF.CGM.getModule().getModuleFlag(markerKey)) {
+      const char *retainRVMarkerKey = llvm::objcarc::getRVMarkerModuleFlagStr();
+      if (!CGF.CGM.getModule().getModuleFlag(retainRVMarkerKey)) {
         auto *str = llvm::MDString::get(CGF.getLLVMContext(), assembly);
-        CGF.CGM.getModule().addModuleFlag(llvm::Module::Error, markerKey, str);
+        CGF.CGM.getModule().addModuleFlag(llvm::Module::Error,
+                                          retainRVMarkerKey, str);
       }
     }
   }
@@ -2317,22 +2328,54 @@ static void emitAutoreleasedReturnValueMarker(CodeGenFunction &CGF) {
     CGF.Builder.CreateCall(marker, None, CGF.getBundlesForFunclet(marker));
 }
 
+static llvm::Value *emitOptimizedARCReturnCall(llvm::Value *value,
+                                               bool IsRetainRV,
+                                               CodeGenFunction &CGF) {
+  emitAutoreleasedReturnValueMarker(CGF);
+
+  // Add operand bundle "clang.arc.rv" to the call instead of emitting retainRV
+  // or claimRV calls in the IR. We currently do this only when the optimization
+  // level isn't -O0 since global-isel, which is currently run at -O0, doesn't
+  // know about the operand bundle.
+
+  // FIXME: Do this when the target isn't aarch64.
+  if (CGF.CGM.getCodeGenOpts().OptimizationLevel > 0 &&
+      CGF.CGM.getTarget().getTriple().isAArch64()) {
+    llvm::Value *bundleArgs[] = {llvm::ConstantInt::get(
+        CGF.Int64Ty, llvm::objcarc::getRVOperandBundleEnum(IsRetainRV))};
+    SmallVector<llvm::OperandBundleDef, 1> bundles;
+    bundles.emplace_back("clang.arc.rv", bundleArgs);
+    auto *oldCall = cast<llvm::CallBase>(value);
+    llvm::CallBase *newCall = llvm::CallBase::Create(oldCall, bundles, oldCall);
+    newCall->copyMetadata(*oldCall);
+    oldCall->replaceAllUsesWith(newCall);
+    oldCall->eraseFromParent();
+    CGF.EmitARCNoopIntrinsicUse(newCall);
+    return newCall;
+  }
+
+  bool isNoTail =
+      CGF.CGM.getTargetCodeGenInfo().markARCOptimizedReturnCallsAsNoTail();
+  llvm::CallInst::TailCallKind tailKind =
+      isNoTail ? llvm::CallInst::TCK_NoTail : llvm::CallInst::TCK_None;
+  ObjCEntrypoints &EPs = CGF.CGM.getObjCEntrypoints();
+  llvm::Function *&EP = IsRetainRV
+                            ? EPs.objc_retainAutoreleasedReturnValue
+                            : EPs.objc_unsafeClaimAutoreleasedReturnValue;
+  llvm::Intrinsic::ID IID =
+      IsRetainRV ? llvm::Intrinsic::objc_retainAutoreleasedReturnValue
+                 : llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue;
+  return emitARCValueOperation(CGF, value, nullptr, EP, IID, tailKind);
+}
+
 /// Retain the given object which is the result of a function call.
 ///   call i8* \@objc_retainAutoreleasedReturnValue(i8* %value)
 ///
 /// Yes, this function name is one character away from a different
 /// call with completely different semantics.
 llvm::Value *
 CodeGenFunction::EmitARCRetainAutoreleasedReturnValue(llvm::Value *value) {
-  emitAutoreleasedReturnValueMarker(*this);
-  llvm::CallInst::TailCallKind tailKind =
-      CGM.getTargetCodeGenInfo().markARCOptimizedReturnCallsAsNoTail()
-          ? llvm::CallInst::TCK_NoTail
-          : llvm::CallInst::TCK_None;
-  return emitARCValueOperation(
-      *this, value, nullptr,
-      CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue,
-      llvm::Intrinsic::objc_retainAutoreleasedReturnValue, tailKind);
+  return emitOptimizedARCReturnCall(value, true, *this);
 }
 
 /// Claim a possibly-autoreleased return value at +0.  This is only
@@ -2344,15 +2387,7 @@ CodeGenFunction::EmitARCRetainAutoreleasedReturnValue(llvm::Value *value) {
 ///   call i8* \@objc_unsafeClaimAutoreleasedReturnValue(i8* %value)
 llvm::Value *
 CodeGenFunction::EmitARCUnsafeClaimAutoreleasedReturnValue(llvm::Value *value) {
-  emitAutoreleasedReturnValueMarker(*this);
-  llvm::CallInst::TailCallKind tailKind =
-      CGM.getTargetCodeGenInfo().markARCOptimizedReturnCallsAsNoTail()
-          ? llvm::CallInst::TCK_NoTail
-          : llvm::CallInst::TCK_None;
-  return emitARCValueOperation(
-      *this, value, nullptr,
-      CGM.getObjCEntrypoints().objc_unsafeClaimAutoreleasedReturnValue,
-      llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue, tailKind);
+  return emitOptimizedARCReturnCall(value, false, *this);
 }
 
 /// Release the given object.

clang/lib/CodeGen/CodeGenFunction.h

@@ -4202,6 +4202,8 @@ class CodeGenFunction : public CodeGenTypeCache {
 
   void EmitARCIntrinsicUse(ArrayRef<llvm::Value*> values);
 
+  void EmitARCNoopIntrinsicUse(ArrayRef<llvm::Value *> values);
+
   static Destroyer destroyARCStrongImprecise;
   static Destroyer destroyARCStrongPrecise;
   static Destroyer destroyARCWeak;

clang/lib/CodeGen/CodeGenModule.h

@@ -210,6 +210,9 @@ struct ObjCEntrypoints {
 
   /// void clang.arc.use(...);
   llvm::Function *clang_arc_use;
+
+  /// void clang.arc.noop.use(...);
+  llvm::Function *clang_arc_noop_use;
 };
 
 /// This class records statistics on instrumentation based profiling.

clang/test/CodeGenObjC/arc-rv-attr.m

@@ -0,0 +1,177 @@
+// RUN: %clang_cc1 -triple arm64-apple-ios9 -fobjc-runtime=ios-9.0 -fobjc-arc -O -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK
+
+@class A;
+
+A *makeA(void);
+
+void test_assign() {
+  __unsafe_unretained id x;
+  x = makeA();
+}
+// CHECK-LABEL: define{{.*}} void @test_assign()
+// CHECK:         [[X:%.*]] = alloca i8*
+// CHECK:         [[T0:%.*]] = call [[A:.*]]* @makeA() [ "clang.arc.rv"(i64 1) ]
+// CHECK-NEXT:    call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]])
+// CHECK-NEXT:    [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8*
+// CHECK-NEXT:    store i8* [[T1]], i8** [[X]]
+// CHECK-NEXT:    bitcast
+// CHECK-NEXT:    lifetime.end
+// CHECK-NEXT:    ret void
+
+void test_assign_assign() {
+  __unsafe_unretained id x, y;
+  x = y = makeA();
+}
+// CHECK-LABEL: define{{.*}} void @test_assign_assign()
+// CHECK:         [[X:%.*]] = alloca i8*
+// CHECK:         [[Y:%.*]] = alloca i8*
+// CHECK:         [[T0:%.*]] = call [[A]]* @makeA() [ "clang.arc.rv"(i64 1) ]
+// CHECK-NEXT:    call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]])
+// CHECK-NEXT:    [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8*
+// CHECK-NEXT:    store i8* [[T1]], i8** [[Y]]
+// CHECK-NEXT:    store i8* [[T1]], i8** [[X]]
+// CHECK-NEXT:    bitcast
+// CHECK-NEXT:    lifetime.end
+// CHECK-NEXT:    bitcast
+// CHECK-NEXT:    lifetime.end
+// CHECK-NEXT:    ret void
+
+void test_strong_assign_assign() {
+  __strong id x;
+  __unsafe_unretained id y;
+  x = y = makeA();
+}
+// CHECK-LABEL: define{{.*}} void @test_strong_assign_assign()
+// CHECK:         [[X:%.*]] = alloca i8*
+// CHECK:         [[Y:%.*]] = alloca i8*
+// CHECK:         [[T0:%.*]] = call [[A]]* @makeA() [ "clang.arc.rv"(i64 0) ]
+// CHECK-NEXT:    call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]])
+// CHECK-NEXT:    [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8*
+// CHECK-NEXT:    store i8* [[T1]], i8** [[Y]]
+// CHECK-NEXT:    [[OLD:%.*]] = load i8*, i8** [[X]]
+// CHECK-NEXT:    store i8* [[T1]], i8** [[X]]
+// CHECK-NEXT:    call void @llvm.objc.release(i8* [[OLD]]
+// CHECK-NEXT:    bitcast
+// CHECK-NEXT:    lifetime.end
+// CHECK-NEXT:    [[T0:%.*]] = load i8*, i8** [[X]]
+// CHECK-NEXT:    call void @llvm.objc.release(i8* [[T0]])
+// CHECK-NEXT:    bitcast
+// CHECK-NEXT:    lifetime.end
+// CHECK-NEXT:    ret void
+
+void test_assign_strong_assign() {
+  __unsafe_unretained id x;
+  __strong id y;
+  x = y = makeA();
+}
+// CHECK-LABEL: define{{.*}} void @test_assign_strong_assign()
+// CHECK:         [[X:%.*]] = alloca i8*
+// CHECK:         [[Y:%.*]] = alloca i8*
+// CHECK:         [[T0:%.*]] = call [[A]]* @makeA() [ "clang.arc.rv"(i64 0) ]
+// CHECK-NEXT:    call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]])
+// CHECK-NEXT:    [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8*
+// CHECK-NEXT:    [[OLD:%.*]] = load i8*, i8** [[Y]]
+// CHECK-NEXT:    store i8* [[T1]], i8** [[Y]]
+// CHECK-NEXT:    call void @llvm.objc.release(i8* [[OLD]]
+// CHECK-NEXT:    store i8* [[T1]], i8** [[X]]
+// CHECK-NEXT:    [[T0:%.*]] = load i8*, i8** [[Y]]
+// CHECK-NEXT:    call void @llvm.objc.release(i8* [[T0]])
+// CHECK-NEXT:    bitcast
+// CHECK-NEXT:    lifetime.end
+// CHECK-NEXT:    bitcast
+// CHECK-NEXT:    lifetime.end
+// CHECK-NEXT:    ret void
+
+void test_init() {
+  __unsafe_unretained id x = makeA();
+}
+// CHECK-LABEL: define{{.*}} void @test_init()
+// CHECK:         [[X:%.*]] = alloca i8*
+// CHECK:         [[T0:%.*]] = call [[A]]* @makeA() [ "clang.arc.rv"(i64 1) ]
+// CHECK-NEXT:    call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]])
+// CHECK-NEXT:    [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8*
+// CHECK-NEXT:    store i8* [[T1]], i8** [[X]]
+// CHECK-NEXT:    bitcast
+// CHECK-NEXT:    lifetime.end
+// CHECK-NEXT:    ret void
+
+void test_init_assignment() {
+  __unsafe_unretained id x;
+  __unsafe_unretained id y = x = makeA();
+}
+// CHECK-LABEL: define{{.*}} void @test_init_assignment()
+// CHECK:         [[X:%.*]] = alloca i8*
+// CHECK:         [[Y:%.*]] = alloca i8*
+// CHECK:         [[T0:%.*]] = call [[A]]* @makeA() [ "clang.arc.rv"(i64 1) ]
+// CHECK-NEXT:    call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]])
+// CHECK-NEXT:    [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8*
+// CHECK-NEXT:    store i8* [[T1]], i8** [[X]]
+// CHECK-NEXT:    store i8* [[T1]], i8** [[Y]]
+// CHECK-NEXT:    bitcast
+// CHECK-NEXT:    lifetime.end
+// CHECK-NEXT:    bitcast
+// CHECK-NEXT:    lifetime.end
+// CHECK-NEXT:    ret void
+
+void test_strong_init_assignment() {
+  __unsafe_unretained id x;
+  __strong id y = x = makeA();
+}
+// CHECK-LABEL: define{{.*}} void @test_strong_init_assignment()
+// CHECK:         [[X:%.*]] = alloca i8*
+// CHECK:         [[Y:%.*]] = alloca i8*
+// CHECK:         [[T0:%.*]] = call [[A]]* @makeA() [ "clang.arc.rv"(i64 0) ]
+// CHECK-NEXT:    call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]])
+// CHECK-NEXT:    [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8*
+// CHECK-NEXT:    store i8* [[T1]], i8** [[X]]
+// CHECK-NEXT:    store i8* [[T1]], i8** [[Y]]
+// CHECK-NEXT:    [[T0:%.*]] = load i8*, i8** [[Y]]
+// CHECK-NEXT:    call void @llvm.objc.release(i8* [[T0]])
+// CHECK-NEXT:    bitcast
+// CHECK-NEXT:    lifetime.end
+// CHECK-NEXT:    bitcast
+// CHECK-NEXT:    lifetime.end
+// CHECK-NEXT:    ret void
+
+void test_init_strong_assignment() {
+  __strong id x;
+  __unsafe_unretained id y = x = makeA();
+}
+// CHECK-LABEL: define{{.*}} void @test_init_strong_assignment()
+// CHECK:         [[X:%.*]] = alloca i8*
+// CHECK:         [[Y:%.*]] = alloca i8*
+// CHECK:         [[T0:%.*]] = call [[A]]* @makeA() [ "clang.arc.rv"(i64 0) ]
+// CHECK-NEXT:    call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]])
+// CHECK-NEXT:    [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8*
+// CHECK-NEXT:    [[OLD:%.*]] = load i8*, i8** [[X]]
+// CHECK-NEXT:    store i8* [[T1]], i8** [[X]]
+// CHECK-NEXT:    call void @llvm.objc.release(i8* [[OLD]])
+// CHECK-NEXT:    store i8* [[T1]], i8** [[Y]]
+// CHECK-NEXT:    bitcast
+// CHECK-NEXT:    lifetime.end
+// CHECK-NEXT:    [[T0:%.*]] = load i8*, i8** [[X]]
+// CHECK-NEXT:    call void @llvm.objc.release(i8* [[T0]])
+// CHECK-NEXT:    bitcast
+// CHECK-NEXT:    lifetime.end
+// CHECK-NEXT:    ret void
+
+void test_ignored() {
+  makeA();
+}
+// CHECK-LABEL: define{{.*}} void @test_ignored()
+// CHECK:         [[T0:%.*]] = call [[A]]* @makeA() [ "clang.arc.rv"(i64 1) ]
+// CHECK-NEXT:    call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]])
+// CHECK-NEXT:    ret void
+
+void test_cast_to_void() {
+  (void) makeA();
+}
+// CHECK-LABEL: define{{.*}} void @test_cast_to_void()
+// CHECK:         [[T0:%.*]] = call [[A]]* @makeA() [ "clang.arc.rv"(i64 1) ]
+// CHECK-NEXT:    call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]])
+// CHECK-NEXT:    ret void
+
+// This is always at the end of the module.
+
+// CHECK-OPTIMIZED: !llvm.module.flags = !{!0,
+// CHECK-OPTIMIZED: !0 = !{i32 1, !"clang.arc.retainAutoreleasedReturnValueMarker", !"mov{{.*}}marker for objc_retainAutoreleaseReturnValue"}

clang/test/CodeGenObjC/arc-unsafeclaim.m

@@ -4,11 +4,10 @@
 //   Make sure it works on x86-32.
 // RUN: %clang_cc1 -triple i386-apple-darwin11 -fobjc-runtime=macosx-fragile-10.11 -fobjc-arc -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-UNOPTIMIZED -check-prefix=CHECK-MARKED -check-prefix=CALL
 
-//   Make sure it works on ARM.
+//   Make sure it works on ARM64.
 // RUN: %clang_cc1 -triple arm64-apple-ios9 -fobjc-runtime=ios-9.0 -fobjc-arc -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-UNOPTIMIZED -check-prefix=CHECK-MARKED -check-prefix=CALL
-// RUN: %clang_cc1 -triple arm64-apple-ios9 -fobjc-runtime=ios-9.0 -fobjc-arc -O -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-OPTIMIZED -check-prefix=CALL
 
-//   Make sure it works on ARM64.
+//   Make sure it works on ARM.
 // RUN: %clang_cc1 -triple armv7-apple-ios9 -fobjc-runtime=ios-9.0 -fobjc-arc -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-UNOPTIMIZED -check-prefix=CHECK-MARKED -check-prefix=CALL
 // RUN: %clang_cc1 -triple armv7-apple-ios9 -fobjc-runtime=ios-9.0 -fobjc-arc -O -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-OPTIMIZED -check-prefix=CALL
 

llvm/docs/LangRef.rst

@@ -2325,6 +2325,18 @@ When lowered, any relocated value will be recorded in the corresponding
 :ref:`stackmap entry <statepoint-stackmap-format>`.  See the intrinsic description
 for further details.
 
+ObjC ARC RetainRV/ClaimRV Operand Bundles
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A ``"clang.arc.rv"`` operand bundle on a call indicates the call is implicitly
+followed by a marker instruction and a call to an ObjC runtime function that
+uses the result of the call. If the argument passed to the operand bundle is 0,
+``@objc_retainAutoreleasedReturnValue`` is called. If 1 is passed,
+``@objc_unsafeClaimAutoreleasedReturnValue`` is called.
+
+The operand bundle is needed to ensure the call is immediately followed by the
+marker instruction or the ObjC runtime call in the final output.
+
 .. _moduleasm:
 
 Module-Level Inline Assembly

llvm/include/llvm/Analysis/ObjCARCUtil.h

@@ -0,0 +1,48 @@
+//===- ObjCARCUtil.h - ObjC ARC Utility Functions ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This file defines ARC utility functions which are used by various parts of
+/// the compiler.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_ANALYSIS_OBJCARCUTIL_H
+#define LLVM_LIB_ANALYSIS_OBJCARCUTIL_H
+
+#include "llvm/IR/InstrTypes.h"
+#include "llvm/IR/LLVMContext.h"
+
+namespace llvm {
+namespace objcarc {
+
+static inline const char *getRVMarkerModuleFlagStr() {
+  return "clang.arc.retainAutoreleasedReturnValueMarker";
+}
+
+enum RVOperandBundle : unsigned { RVOB_Retain, RVOB_Claim };
+
+static RVOperandBundle getRVOperandBundleEnum(bool IsRetain) {
+  return IsRetain ? RVOB_Retain : RVOB_Claim;
+}
+
+static inline bool hasRVOpBundle(const CallBase *CB, bool IsRetain) {
+  auto B = CB->getOperandBundle(LLVMContext::OB_clang_arc_rv);
+  if (!B.hasValue())
+    return false;
+  return cast<ConstantInt>(B->Inputs[0])->getZExtValue() ==
+         getRVOperandBundleEnum(IsRetain);
+}
+
+static inline bool hasRVOpBundle(const CallBase *CB) {
+  return CB->getOperandBundle(LLVMContext::OB_clang_arc_rv).hasValue();
+}
+
+} // end namespace objcarc
+} // end namespace llvm
+
+#endif