diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp index fbf942d06ca6ed..a7bbbd20cc31bf 100644 --- a/clang/lib/CodeGen/CGAtomic.cpp +++ b/clang/lib/CodeGen/CGAtomic.cpp @@ -19,6 +19,7 @@ #include "clang/CodeGen/CGFunctionInfo.h" #include "clang/Frontend/FrontendDiagnostic.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/Frontend/Atomic/Atomic.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/Operator.h" @@ -27,275 +28,322 @@ using namespace clang; using namespace CodeGen; namespace { - class AtomicInfo { - CodeGenFunction &CGF; - QualType AtomicTy; +class AtomicInfo : public llvm::AtomicInfo { + CodeGenFunction &CGF; + QualType AtomicTy; + QualType ValueTy; + TypeEvaluationKind EvaluationKind; + LValue LVal; + CGBitFieldInfo BFI; + +public: + llvm::AtomicInfo &getBase() { return *this; } + + static AtomicInfo fromValue(CodeGenFunction &CGF, LValue &lvalue) { + if (lvalue.isSimple()) + return fromSimple(CGF, lvalue); + if (lvalue.isBitField()) + return fromBitfield(CGF, lvalue); + if (lvalue.isVectorElt()) + return fromVectorElt(CGF, lvalue); + if (lvalue.isExtVectorElt()) + return fromExtVectorElt(CGF, lvalue); + llvm_unreachable("Type incompatible with atomics"); + } + + static AtomicInfo fromSimple(CodeGenFunction &CGF, LValue &lvalue) { + assert(lvalue.isSimple()); + ASTContext &C = CGF.getContext(); + + QualType AtomicTy = lvalue.getType(); QualType ValueTy; - uint64_t AtomicSizeInBits; - uint64_t ValueSizeInBits; - CharUnits AtomicAlign; - CharUnits ValueAlign; - TypeEvaluationKind EvaluationKind; - bool UseLibcall; - LValue LVal; - CGBitFieldInfo BFI; - public: - AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) - : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0), - EvaluationKind(TEK_Scalar), UseLibcall(true) { - assert(!lvalue.isGlobalReg()); - ASTContext &C = CGF.getContext(); - if (lvalue.isSimple()) { - AtomicTy = lvalue.getType(); - if (auto *ATy = AtomicTy->getAs()) - ValueTy = ATy->getValueType(); - else - ValueTy = AtomicTy; - EvaluationKind = CGF.getEvaluationKind(ValueTy); - - uint64_t ValueAlignInBits; - uint64_t AtomicAlignInBits; - TypeInfo ValueTI = C.getTypeInfo(ValueTy); - ValueSizeInBits = ValueTI.Width; - ValueAlignInBits = ValueTI.Align; - - TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); - AtomicSizeInBits = AtomicTI.Width; - AtomicAlignInBits = AtomicTI.Align; - - assert(ValueSizeInBits <= AtomicSizeInBits); - assert(ValueAlignInBits <= AtomicAlignInBits); - - AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); - ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); - if (lvalue.getAlignment().isZero()) - lvalue.setAlignment(AtomicAlign); - - LVal = lvalue; - } else if (lvalue.isBitField()) { - ValueTy = lvalue.getType(); - ValueSizeInBits = C.getTypeSize(ValueTy); - auto &OrigBFI = lvalue.getBitFieldInfo(); - auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment()); - AtomicSizeInBits = C.toBits( - C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) - .alignTo(lvalue.getAlignment())); - llvm::Value *BitFieldPtr = lvalue.getRawBitFieldPointer(CGF); - auto OffsetInChars = - (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * - lvalue.getAlignment(); - llvm::Value *StoragePtr = CGF.Builder.CreateConstGEP1_64( - CGF.Int8Ty, BitFieldPtr, OffsetInChars.getQuantity()); - StoragePtr = CGF.Builder.CreateAddrSpaceCast( - StoragePtr, CGF.UnqualPtrTy, "atomic_bitfield_base"); - BFI = OrigBFI; - BFI.Offset = Offset; - BFI.StorageSize = AtomicSizeInBits; - BFI.StorageOffset += OffsetInChars; - llvm::Type *StorageTy = CGF.Builder.getIntNTy(AtomicSizeInBits); - LVal = LValue::MakeBitfield( - Address(StoragePtr, StorageTy, lvalue.getAlignment()), BFI, - lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo()); - AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); - if (AtomicTy.isNull()) { - llvm::APInt Size( - /*numBits=*/32, - C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity()); - AtomicTy = C.getConstantArrayType(C.CharTy, Size, nullptr, - ArraySizeModifier::Normal, - /*IndexTypeQuals=*/0); - } - AtomicAlign = ValueAlign = lvalue.getAlignment(); - } else if (lvalue.isVectorElt()) { - ValueTy = lvalue.getType()->castAs()->getElementType(); - ValueSizeInBits = C.getTypeSize(ValueTy); - AtomicTy = lvalue.getType(); - AtomicSizeInBits = C.getTypeSize(AtomicTy); - AtomicAlign = ValueAlign = lvalue.getAlignment(); - LVal = lvalue; - } else { - assert(lvalue.isExtVectorElt()); - ValueTy = lvalue.getType(); - ValueSizeInBits = C.getTypeSize(ValueTy); - AtomicTy = ValueTy = CGF.getContext().getExtVectorType( - lvalue.getType(), cast( - lvalue.getExtVectorAddress().getElementType()) - ->getNumElements()); - AtomicSizeInBits = C.getTypeSize(AtomicTy); - AtomicAlign = ValueAlign = lvalue.getAlignment(); - LVal = lvalue; - } - UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( - AtomicSizeInBits, C.toBits(lvalue.getAlignment())); - } + if (auto *ATy = AtomicTy->getAs()) + ValueTy = ATy->getValueType(); + else + ValueTy = AtomicTy; + TypeEvaluationKind EvaluationKind = CGF.getEvaluationKind(ValueTy); - QualType getAtomicType() const { return AtomicTy; } - QualType getValueType() const { return ValueTy; } - CharUnits getAtomicAlignment() const { return AtomicAlign; } - uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } - uint64_t getValueSizeInBits() const { return ValueSizeInBits; } - TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } - bool shouldUseLibcall() const { return UseLibcall; } - const LValue &getAtomicLValue() const { return LVal; } - llvm::Value *getAtomicPointer() const { - if (LVal.isSimple()) - return LVal.emitRawPointer(CGF); - else if (LVal.isBitField()) - return LVal.getRawBitFieldPointer(CGF); - else if (LVal.isVectorElt()) - return LVal.getRawVectorPointer(CGF); - assert(LVal.isExtVectorElt()); - return LVal.getRawExtVectorPointer(CGF); - } - Address getAtomicAddress() const { - llvm::Type *ElTy; - if (LVal.isSimple()) - ElTy = LVal.getAddress().getElementType(); - else if (LVal.isBitField()) - ElTy = LVal.getBitFieldAddress().getElementType(); - else if (LVal.isVectorElt()) - ElTy = LVal.getVectorAddress().getElementType(); - else - ElTy = LVal.getExtVectorAddress().getElementType(); - return Address(getAtomicPointer(), ElTy, getAtomicAlignment()); - } + TypeInfo ValueTI = C.getTypeInfo(ValueTy); + auto ValueSizeInBits = ValueTI.Width; + auto ValueAlignInBits = ValueTI.Align; - Address getAtomicAddressAsAtomicIntPointer() const { - return castToAtomicIntPointer(getAtomicAddress()); - } + TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); + auto AtomicSizeInBits = AtomicTI.Width; + auto AtomicAlignInBits = AtomicTI.Align; - /// Is the atomic size larger than the underlying value type? - /// - /// Note that the absence of padding does not mean that atomic - /// objects are completely interchangeable with non-atomic - /// objects: we might have promoted the alignment of a type - /// without making it bigger. - bool hasPadding() const { - return (ValueSizeInBits != AtomicSizeInBits); - } + assert(ValueSizeInBits <= AtomicSizeInBits); + assert(ValueAlignInBits <= AtomicAlignInBits); + + auto AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits).getAsAlign(); + auto ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits).getAsAlign(); + if (lvalue.getAlignment().isZero()) + lvalue.setAlignment(CharUnits::fromQuantity(AtomicAlign)); + + auto LVal = lvalue; + auto ElTy = LVal.getAddress().getElementType(); - bool emitMemSetZeroIfNecessary() const; + bool UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( + AtomicSizeInBits, C.toBits(lvalue.getAlignment())); + return AtomicInfo(CGF, ElTy, LVal, AtomicTy, ValueTy, EvaluationKind, {}, + AtomicSizeInBits, ValueSizeInBits, AtomicAlign, + ValueAlign, UseLibcall); + } - llvm::Value *getAtomicSizeValue() const { - CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); - return CGF.CGM.getSize(size); + static AtomicInfo fromBitfield(CodeGenFunction &CGF, LValue &lvalue) { + assert(lvalue.isBitField()); + ASTContext &C = CGF.getContext(); + + auto ValueTy = lvalue.getType(); + auto ValueSizeInBits = C.getTypeSize(ValueTy); + auto &OrigBFI = lvalue.getBitFieldInfo(); + auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment()); + auto AtomicSizeInBits = C.toBits( + C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) + .alignTo(lvalue.getAlignment())); + llvm::Value *BitFieldPtr = lvalue.getRawBitFieldPointer(CGF); + auto OffsetInChars = + (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * + lvalue.getAlignment(); + llvm::Value *StoragePtr = CGF.Builder.CreateConstGEP1_64( + CGF.Int8Ty, BitFieldPtr, OffsetInChars.getQuantity()); + StoragePtr = CGF.Builder.CreateAddrSpaceCast(StoragePtr, CGF.UnqualPtrTy, + "atomic_bitfield_base"); + auto BFI = OrigBFI; + BFI.Offset = Offset; + BFI.StorageSize = AtomicSizeInBits; + BFI.StorageOffset += OffsetInChars; + llvm::Type *StorageTy = CGF.Builder.getIntNTy(AtomicSizeInBits); + auto LVal = LValue::MakeBitfield( + Address(StoragePtr, StorageTy, lvalue.getAlignment()), BFI, + lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo()); + auto AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); + if (AtomicTy.isNull()) { + llvm::APInt Size( + /*numBits=*/32, + C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity()); + AtomicTy = C.getConstantArrayType(C.CharTy, Size, nullptr, + ArraySizeModifier::Normal, + /*IndexTypeQuals=*/0); } + auto ValueAlign = lvalue.getAlignment().getAsAlign(); + auto AtomicAlign = ValueAlign; - /// Cast the given pointer to an integer pointer suitable for atomic - /// operations if the source. - Address castToAtomicIntPointer(Address Addr) const; + auto ElTy = LVal.getBitFieldAddress().getElementType(); - /// If Addr is compatible with the iN that will be used for an atomic - /// operation, bitcast it. Otherwise, create a temporary that is suitable - /// and copy the value across. - Address convertToAtomicIntPointer(Address Addr) const; + TypeEvaluationKind EvaluationKind = TEK_Scalar; + bool UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( + AtomicSizeInBits, C.toBits(lvalue.getAlignment())); + return AtomicInfo(CGF, ElTy, LVal, AtomicTy, ValueTy, EvaluationKind, {}, + AtomicSizeInBits, ValueSizeInBits, AtomicAlign, + ValueAlign, UseLibcall); + } - /// Turn an atomic-layout object into an r-value. - RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, - SourceLocation loc, bool AsValue) const; + static AtomicInfo fromVectorElt(CodeGenFunction &CGF, LValue &lvalue) { + assert(lvalue.isVectorElt()); + ASTContext &C = CGF.getContext(); + + auto ValueTy = lvalue.getType()->castAs()->getElementType(); + auto ValueSizeInBits = C.getTypeSize(ValueTy); + auto AtomicTy = lvalue.getType(); + auto AtomicSizeInBits = C.getTypeSize(AtomicTy); + auto ValueAlign = lvalue.getAlignment().getAsAlign(); + auto AtomicAlign = ValueAlign; + auto LVal = lvalue; + + auto ElTy = LVal.getVectorAddress().getElementType(); + + TypeEvaluationKind EvaluationKind = TEK_Scalar; + bool UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( + AtomicSizeInBits, + C.toBits(lvalue.getAlignment())); // TODO: Move to llvm::AtomicInfo + return AtomicInfo(CGF, ElTy, LVal, AtomicTy, ValueTy, EvaluationKind, {}, + AtomicSizeInBits, ValueSizeInBits, AtomicAlign, + ValueAlign, UseLibcall); + } - llvm::Value *getScalarRValValueOrNull(RValue RVal) const; + static AtomicInfo fromExtVectorElt(CodeGenFunction &CGF, LValue &lvalue) { + assert(lvalue.isExtVectorElt()); + ASTContext &C = CGF.getContext(); + + auto ValueTy = lvalue.getType(); + auto ValueSizeInBits = C.getTypeSize(ValueTy); + auto AtomicTy = ValueTy = CGF.getContext().getExtVectorType( + lvalue.getType(), cast( + lvalue.getExtVectorAddress().getElementType()) + ->getNumElements()); + auto AtomicSizeInBits = C.getTypeSize(AtomicTy); + auto ValueAlign = lvalue.getAlignment().getAsAlign(); + auto AtomicAlign = ValueAlign; + auto LVal = lvalue; + + auto ElTy = LVal.getExtVectorAddress().getElementType(); + + TypeEvaluationKind EvaluationKind = TEK_Scalar; + bool UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( + AtomicSizeInBits, + C.toBits(lvalue.getAlignment())); // TODO: Move to llvm::AtomicInfo + return AtomicInfo(CGF, ElTy, LVal, AtomicTy, ValueTy, EvaluationKind, {}, + AtomicSizeInBits, ValueSizeInBits, AtomicAlign, + ValueAlign, UseLibcall); + } - /// Converts an rvalue to integer value if needed. - llvm::Value *convertRValueToInt(RValue RVal, bool CmpXchg = false) const; + AtomicInfo(CodeGenFunction &CGF, llvm::Type *Ty, LValue &lvalue, + QualType AtomicTy, QualType ValueTy, + TypeEvaluationKind EvaluationKind, CGBitFieldInfo BFI, + uint64_t AtomicSizeInBits, uint64_t ValueSizeInBits, + llvm::Align AtomicAlign, llvm::Align ValueAlign, bool UseLibcall) + : llvm::AtomicInfo(&CGF.Builder, Ty, AtomicSizeInBits, + ValueSizeInBits, AtomicAlign, + ValueAlign, UseLibcall), + CGF(CGF), AtomicTy(AtomicTy), ValueTy(ValueTy), + EvaluationKind(EvaluationKind), LVal(lvalue), BFI(BFI) {} + + QualType getAtomicType() const { return AtomicTy; } + QualType getValueType() const { return ValueTy; } + CharUnits getAtomicAlignment() const { + return CharUnits::fromQuantity(AtomicAlign); + } + TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } + const LValue &getAtomicLValue() const { return LVal; } + llvm::Value *getAtomicPointer() const override { + if (LVal.isSimple()) + return LVal.emitRawPointer(CGF); + else if (LVal.isBitField()) + return LVal.getRawBitFieldPointer(CGF); + else if (LVal.isVectorElt()) + return LVal.getRawVectorPointer(CGF); + assert(LVal.isExtVectorElt()); + return LVal.getRawExtVectorPointer(CGF); + } + Address getAtomicAddress() const { + llvm::Type *ElTy; + if (LVal.isSimple()) + ElTy = LVal.getAddress().getElementType(); + else if (LVal.isBitField()) + ElTy = LVal.getBitFieldAddress().getElementType(); + else if (LVal.isVectorElt()) + ElTy = LVal.getVectorAddress().getElementType(); + else + ElTy = LVal.getExtVectorAddress().getElementType(); + return Address(getAtomicPointer(), ElTy, getAtomicAlignment()); + } - RValue ConvertToValueOrAtomic(llvm::Value *IntVal, AggValueSlot ResultSlot, - SourceLocation Loc, bool AsValue, - bool CmpXchg = false) const; + void decorateWithTBAA(llvm::Instruction *I) override { + CGF.CGM.DecorateInstructionWithTBAA(I, LVal.getTBAAInfo()); + } - /// Copy an atomic r-value into atomic-layout memory. - void emitCopyIntoMemory(RValue rvalue) const; + llvm::AllocaInst *CreateAlloca(llvm::Type *Ty, + const llvm::Twine &Name) override { + return CGF.CreateTempAlloca(Ty, Name); + } - /// Project an l-value down to the value field. - LValue projectValue() const { - assert(LVal.isSimple()); - Address addr = getAtomicAddress(); - if (hasPadding()) - addr = CGF.Builder.CreateStructGEP(addr, 0); + Address getAtomicAddressAsAtomicIntPointer() const { + return castToAtomicIntPointer(getAtomicAddress()); + } - return LValue::MakeAddr(addr, getValueType(), CGF.getContext(), - LVal.getBaseInfo(), LVal.getTBAAInfo()); - } + bool emitMemSetZeroIfNecessary() const; - /// Emits atomic load. - /// \returns Loaded value. - RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, - bool AsValue, llvm::AtomicOrdering AO, - bool IsVolatile); + llvm::Value *getAtomicSizeValue() const { + CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); + return CGF.CGM.getSize(size); + } + + /// Cast the given pointer to an integer pointer suitable for atomic + /// operations if the source. + Address castToAtomicIntPointer(Address Addr) const; + + /// If Addr is compatible with the iN that will be used for an atomic + /// operation, bitcast it. Otherwise, create a temporary that is suitable + /// and copy the value across. + Address convertToAtomicIntPointer(Address Addr) const; + + /// Turn an atomic-layout object into an r-value. + RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, + SourceLocation loc, bool AsValue) const; + + llvm::Value *getScalarRValValueOrNull(RValue RVal) const; + + /// Converts an rvalue to integer value if needed. + llvm::Value *convertRValueToInt(RValue RVal, bool CmpXchg = false) const; - /// Emits atomic compare-and-exchange sequence. - /// \param Expected Expected value. - /// \param Desired Desired value. - /// \param Success Atomic ordering for success operation. - /// \param Failure Atomic ordering for failed operation. - /// \param IsWeak true if atomic operation is weak, false otherwise. - /// \returns Pair of values: previous value from storage (value type) and - /// boolean flag (i1 type) with true if success and false otherwise. - std::pair - EmitAtomicCompareExchange(RValue Expected, RValue Desired, - llvm::AtomicOrdering Success = - llvm::AtomicOrdering::SequentiallyConsistent, - llvm::AtomicOrdering Failure = - llvm::AtomicOrdering::SequentiallyConsistent, - bool IsWeak = false); - - /// Emits atomic update. - /// \param AO Atomic ordering. - /// \param UpdateOp Update operation for the current lvalue. - void EmitAtomicUpdate(llvm::AtomicOrdering AO, + RValue ConvertToValueOrAtomic(llvm::Value *IntVal, AggValueSlot ResultSlot, + SourceLocation Loc, bool AsValue, + bool CmpXchg = false) const; + + /// Copy an atomic r-value into atomic-layout memory. + void emitCopyIntoMemory(RValue rvalue) const; + + /// Project an l-value down to the value field. + LValue projectValue() const { + assert(LVal.isSimple()); + Address addr = getAtomicAddress(); + if (hasPadding()) + addr = CGF.Builder.CreateStructGEP(addr, 0); + + return LValue::MakeAddr(addr, getValueType(), CGF.getContext(), + LVal.getBaseInfo(), LVal.getTBAAInfo()); + } + + /// Emits atomic load. + /// \returns Loaded value. + RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, + bool AsValue, llvm::AtomicOrdering AO, bool IsVolatile); + + /// Emits atomic compare-and-exchange sequence. + /// \param Expected Expected value. + /// \param Desired Desired value. + /// \param Success Atomic ordering for success operation. + /// \param Failure Atomic ordering for failed operation. + /// \param IsWeak true if atomic operation is weak, false otherwise. + /// \returns Pair of values: previous value from storage (value type) and + /// boolean flag (i1 type) with true if success and false otherwise. + std::pair + EmitAtomicCompareExchange(RValue Expected, RValue Desired, + llvm::AtomicOrdering Success = + llvm::AtomicOrdering::SequentiallyConsistent, + llvm::AtomicOrdering Failure = + llvm::AtomicOrdering::SequentiallyConsistent, + bool IsWeak = false); + + /// Emits atomic update. + /// \param AO Atomic ordering. + /// \param UpdateOp Update operation for the current lvalue. + void EmitAtomicUpdate(llvm::AtomicOrdering AO, + const llvm::function_ref &UpdateOp, + bool IsVolatile); + /// Emits atomic update. + /// \param AO Atomic ordering. + void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, + bool IsVolatile); + + /// Materialize an atomic r-value in atomic-layout memory. + Address materializeRValue(RValue rvalue) const; + + /// Creates temp alloca for intermediate operations on atomic value. + Address CreateTempAlloca() const; + +private: + bool requiresMemSetZero(llvm::Type *type) const; + + /// Emits atomic load as a libcall. + void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, llvm::AtomicOrdering AO, + bool IsVolatile); + /// Emit atomic update as libcalls. + void + EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, + const llvm::function_ref &UpdateOp, + bool IsVolatile); + /// Emit atomic update as LLVM instructions. + void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, const llvm::function_ref &UpdateOp, bool IsVolatile); - /// Emits atomic update. - /// \param AO Atomic ordering. - void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, + /// Emit atomic update as libcalls. + void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, + bool IsVolatile); + /// Emit atomic update as LLVM instructions. + void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal, bool IsVolatile); - - /// Materialize an atomic r-value in atomic-layout memory. - Address materializeRValue(RValue rvalue) const; - - /// Creates temp alloca for intermediate operations on atomic value. - Address CreateTempAlloca() const; - private: - bool requiresMemSetZero(llvm::Type *type) const; - - - /// Emits atomic load as a libcall. - void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, - llvm::AtomicOrdering AO, bool IsVolatile); - /// Emits atomic load as LLVM instruction. - llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile, - bool CmpXchg = false); - /// Emits atomic compare-and-exchange op as a libcall. - llvm::Value *EmitAtomicCompareExchangeLibcall( - llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, - llvm::AtomicOrdering Success = - llvm::AtomicOrdering::SequentiallyConsistent, - llvm::AtomicOrdering Failure = - llvm::AtomicOrdering::SequentiallyConsistent); - /// Emits atomic compare-and-exchange op as LLVM instruction. - std::pair EmitAtomicCompareExchangeOp( - llvm::Value *ExpectedVal, llvm::Value *DesiredVal, - llvm::AtomicOrdering Success = - llvm::AtomicOrdering::SequentiallyConsistent, - llvm::AtomicOrdering Failure = - llvm::AtomicOrdering::SequentiallyConsistent, - bool IsWeak = false); - /// Emit atomic update as libcalls. - void - EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, - const llvm::function_ref &UpdateOp, - bool IsVolatile); - /// Emit atomic update as LLVM instructions. - void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, - const llvm::function_ref &UpdateOp, - bool IsVolatile); - /// Emit atomic update as libcalls. - void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, - bool IsVolatile); - /// Emit atomic update as LLVM instructions. - void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal, - bool IsVolatile); - }; +}; } Address AtomicInfo::CreateTempAlloca() const { @@ -1002,7 +1050,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { // need to make sure (via temporaries if necessary) that all incoming values // are compatible. LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy); - AtomicInfo Atomics(*this, AtomicVal); + AtomicInfo Atomics = AtomicInfo::fromValue(*this, AtomicVal); if (ShouldCastToIntPtrTy) { Ptr = Atomics.castToAtomicIntPointer(Ptr); @@ -1401,17 +1449,6 @@ RValue AtomicInfo::convertAtomicTempToRValue(Address addr, LVal.getBaseInfo(), TBAAAccessInfo())); } -/// Return true if \param ValTy is a type that should be casted to integer -/// around the atomic memory operation. If \param CmpXchg is true, then the -/// cast of a floating point type is made as that instruction can not have -/// floating point operands. TODO: Allow compare-and-exchange and FP - see -/// comment in AtomicExpandPass.cpp. -static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg) { - if (ValTy->isFloatingPointTy()) - return ValTy->isX86_FP80Ty() || CmpXchg; - return !ValTy->isIntegerTy() && !ValTy->isPointerTy(); -} - RValue AtomicInfo::ConvertToValueOrAtomic(llvm::Value *Val, AggValueSlot ResultSlot, SourceLocation Loc, bool AsValue, @@ -1470,28 +1507,12 @@ void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args); } -llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, - bool IsVolatile, bool CmpXchg) { - // Okay, we're doing this natively. - Address Addr = getAtomicAddress(); - if (shouldCastToInt(Addr.getElementType(), CmpXchg)) - Addr = castToAtomicIntPointer(Addr); - llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); - Load->setAtomic(AO); - - // Other decoration. - if (IsVolatile) - Load->setVolatile(true); - CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); - return Load; -} - /// An LValue is a candidate for having its loads and stores be made atomic if /// we are operating under /volatile:ms *and* the LValue itself is volatile and /// performing such an operation can be performed without a libcall. bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) { if (!CGM.getLangOpts().MSVolatile) return false; - AtomicInfo AI(*this, LV); + AtomicInfo AI = AtomicInfo::fromValue(*this, LV); bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType()); // An atomic is inline if we don't need to use a libcall. bool AtomicIsInline = !AI.shouldUseLibcall(); @@ -1551,9 +1572,8 @@ RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, llvm::AtomicOrdering AO, bool IsVolatile, AggValueSlot resultSlot) { - AtomicInfo Atomics(*this, src); - return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, - IsVolatile); + return AtomicInfo::fromValue(*this, src) + .EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, IsVolatile); } /// Copy an r-value into memory as part of storing to an atomic type. @@ -1601,8 +1621,7 @@ Address AtomicInfo::materializeRValue(RValue rvalue) const { // Otherwise, make a temporary and materialize into it. LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType()); - AtomicInfo Atomics(CGF, TempLV); - Atomics.emitCopyIntoMemory(rvalue); + AtomicInfo::fromValue(CGF, TempLV).emitCopyIntoMemory(rvalue); return TempLV.getAddress(); } @@ -1635,70 +1654,18 @@ llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal, bool CmpXchg) const { return CGF.Builder.CreateLoad(Addr); } -std::pair AtomicInfo::EmitAtomicCompareExchangeOp( - llvm::Value *ExpectedVal, llvm::Value *DesiredVal, - llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { - // Do the atomic store. - Address Addr = getAtomicAddressAsAtomicIntPointer(); - auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal, - Success, Failure); - // Other decoration. - Inst->setVolatile(LVal.isVolatileQualified()); - Inst->setWeak(IsWeak); - - // Okay, turn that back into the original value type. - auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0); - auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1); - return std::make_pair(PreviousVal, SuccessFailureVal); -} - -llvm::Value * -AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr, - llvm::Value *DesiredAddr, - llvm::AtomicOrdering Success, - llvm::AtomicOrdering Failure) { - // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, - // void *desired, int success, int failure); - CallArgList Args; - Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); - Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy); - Args.add(RValue::get(ExpectedAddr), CGF.getContext().VoidPtrTy); - Args.add(RValue::get(DesiredAddr), CGF.getContext().VoidPtrTy); - Args.add(RValue::get( - llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))), - CGF.getContext().IntTy); - Args.add(RValue::get( - llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))), - CGF.getContext().IntTy); - auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange", - CGF.getContext().BoolTy, Args); - - return SuccessFailureRVal.getScalarVal(); -} - std::pair AtomicInfo::EmitAtomicCompareExchange( RValue Expected, RValue Desired, llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { - // Check whether we should use a library call. - if (shouldUseLibcall()) { - // Produce a source address. - Address ExpectedAddr = materializeRValue(Expected); - llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); - llvm::Value *DesiredPtr = materializeRValue(Desired).emitRawPointer(CGF); - auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, - Success, Failure); - return std::make_pair( - convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), - SourceLocation(), /*AsValue=*/false), - Res); - } - // If we've got a scalar value of the right size, try to avoid going - // through memory. - auto *ExpectedVal = convertRValueToInt(Expected, /*CmpXchg=*/true); + Address ExpectedAddr = materializeRValue(Expected); + llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); auto *DesiredVal = convertRValueToInt(Desired, /*CmpXchg=*/true); - auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, - Failure, IsWeak); + + std::pair Res = + getBase().EmitAtomicCompareExchange(ExpectedPtr, DesiredVal, Success, + Failure, LVal.isVolatileQualified(), + IsWeak); return std::make_pair( ConvertToValueOrAtomic(Res.first, AggValueSlot::ignored(), SourceLocation(), /*AsValue=*/false, @@ -1784,9 +1751,9 @@ void AtomicInfo::EmitAtomicUpdateLibcall( EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF); - auto *Res = + auto Res = EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure); - CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); + CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); CGF.EmitBlock(ExitBB, /*IsFinished=*/true); } @@ -1872,9 +1839,9 @@ void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF); - auto *Res = + auto Res = EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure); - CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); + CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); CGF.EmitBlock(ExitBB, /*IsFinished=*/true); } @@ -1953,7 +1920,7 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, rvalue.getAggregateAddress().getElementType() == dest.getAddress().getElementType()); - AtomicInfo atomics(*this, dest); + AtomicInfo atomics = AtomicInfo ::fromValue(*this, dest); LValue LVal = atomics.getAtomicLValue(); // If this is an initialization, just put the value there normally. @@ -1988,7 +1955,8 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, // Do the atomic store. Address Addr = atomics.getAtomicAddress(); if (llvm::Value *Value = atomics.getScalarRValValueOrNull(rvalue)) - if (shouldCastToInt(Value->getType(), /*CmpXchg=*/false)) { + if (llvm::AtomicInfo::shouldCastToInt(Value->getType(), + /*CmpXchg=*/false)) { Addr = atomics.castToAtomicIntPointer(Addr); ValToStore = Builder.CreateIntCast(ValToStore, Addr.getElementType(), /*isSigned=*/false); @@ -2028,21 +1996,18 @@ std::pair CodeGenFunction::EmitAtomicCompareExchange( assert(!Desired.isAggregate() || Desired.getAggregateAddress().getElementType() == Obj.getAddress().getElementType()); - AtomicInfo Atomics(*this, Obj); - - return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, - IsWeak); + return AtomicInfo::fromValue(*this, Obj) + .EmitAtomicCompareExchange(Expected, Desired, Success, Failure, IsWeak); } void CodeGenFunction::EmitAtomicUpdate( LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref &UpdateOp, bool IsVolatile) { - AtomicInfo Atomics(*this, LVal); - Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile); + AtomicInfo::fromValue(*this, LVal).EmitAtomicUpdate(AO, UpdateOp, IsVolatile); } void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { - AtomicInfo atomics(*this, dest); + AtomicInfo atomics = AtomicInfo::fromValue(*this, dest); switch (atomics.getEvaluationKind()) { case TEK_Scalar: { diff --git a/flang/test/Integration/OpenMP/atomic-update-complex.f90 b/flang/test/Integration/OpenMP/atomic-update-complex.f90 new file mode 100644 index 00000000000000..6e0b419d11f952 --- /dev/null +++ b/flang/test/Integration/OpenMP/atomic-update-complex.f90 @@ -0,0 +1,42 @@ +!===----------------------------------------------------------------------===! +! This directory can be used to add Integration tests involving multiple +! stages of the compiler (for eg. from Fortran to LLVM IR). It should not +! contain executable tests. We should only add tests here sparingly and only +! if there is no other way to test. Repeat this message in each test that is +! added to this directory and sub-directories. +!===----------------------------------------------------------------------===! + +!RUN: %flang_fc1 -emit-llvm -fopenmp %s -o - | FileCheck %s + +!CHECK: define void @_QQmain() { +!CHECK: %[[X_NEW_VAL:.*]] = alloca { float, float }, align 8 +!CHECK: {{.*}} = alloca { float, float }, i64 1, align 8 +!CHECK: %[[ORIG_VAL:.*]] = alloca { float, float }, i64 1, align 8 +!CHECK: store { float, float } { float 2.000000e+00, float 2.000000e+00 }, ptr %[[ORIG_VAL]], align 4 +!CHECK: br label %entry + +!CHECK: entry: +!CHECK: %[[ATOMIC_TEMP_LOAD:.*]] = alloca { float, float }, align 16 +!CHECK: call void @__atomic_load(i64 8, ptr %[[ORIG_VAL]], ptr %[[ATOMIC_TEMP_LOAD]], i32 0) +!CHECK: %[[PHI_NODE_ENTRY_1:.*]] = load { float, float }, ptr %[[ATOMIC_TEMP_LOAD]], align 16 +!CHECK: br label %.atomic.cont + +!CHECK: .atomic.cont +!CHECK: %[[VAL_4:.*]] = phi { float, float } [ %[[PHI_NODE_ENTRY_1]], %entry ], [ %{{.*}}, %.atomic.cont ] +!CHECK: %[[VAL_5:.*]] = extractvalue { float, float } %[[VAL_4]], 0 +!CHECK: %[[VAL_6:.*]] = extractvalue { float, float } %[[VAL_4]], 1 +!CHECK: %[[VAL_7:.*]] = fadd contract float %[[VAL_5]], 1.000000e+00 +!CHECK: %[[VAL_8:.*]] = fadd contract float %[[VAL_6]], 1.000000e+00 +!CHECK: %[[VAL_9:.*]] = insertvalue { float, float } undef, float %[[VAL_7]], 0 +!CHECK: %[[VAL_10:.*]] = insertvalue { float, float } %[[VAL_9]], float %[[VAL_8]], 1 +!CHECK: store { float, float } %[[VAL_10]], ptr %[[X_NEW_VAL]], align 4 +!CHECK: %[[VAL_11:.*]] = call i1 @__atomic_compare_exchange(i64 8, ptr %[[ORIG_VAL]], ptr %[[ATOMIC_TEMP_LOAD]], ptr %[[X_NEW_VAL]], i32 2, i32 2) +!CHECK: %[[VAL_12:.*]] = load { float, float }, ptr %[[ATOMIC_TEMP_LOAD]], align 4 +!CHECK: br i1 %[[VAL_11]], label %.atomic.exit, label %.atomic.cont +program main + complex*8 ia, ib + ia = (2, 2) + !$omp atomic update + ia = ia + (1, 1) + !$omp end atomic +end program diff --git a/llvm/include/llvm/Frontend/Atomic/Atomic.h b/llvm/include/llvm/Frontend/Atomic/Atomic.h new file mode 100644 index 00000000000000..05296ef44c90f1 --- /dev/null +++ b/llvm/include/llvm/Frontend/Atomic/Atomic.h @@ -0,0 +1,235 @@ +//===--- Atomic.h - Shared codegen of atomic operations -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FRONTEND_ATOMIC_ATOMIC_H +#define LLVM_FRONTEND_ATOMIC_ATOMIC_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/CodeGen/RuntimeLibcalls.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Operator.h" + +namespace llvm { + +template struct AtomicInfo { + IRBuilderTy *Builder; + + Type *Ty; + uint64_t AtomicSizeInBits; + uint64_t ValueSizeInBits; + llvm::Align AtomicAlign; + llvm::Align ValueAlign; + bool UseLibcall; + +public: + AtomicInfo(IRBuilderTy *Builder, Type *Ty, uint64_t AtomicSizeInBits, + uint64_t ValueSizeInBits, llvm::Align AtomicAlign, + llvm::Align ValueAlign, bool UseLibcall) + : Builder(Builder), Ty(Ty), AtomicSizeInBits(AtomicSizeInBits), + ValueSizeInBits(ValueSizeInBits), AtomicAlign(AtomicAlign), + ValueAlign(ValueAlign), UseLibcall(UseLibcall) {} + + virtual ~AtomicInfo() = default; + + llvm::Align getAtomicAlignment() const { return AtomicAlign; } + uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } + uint64_t getValueSizeInBits() const { return ValueSizeInBits; } + bool shouldUseLibcall() const { return UseLibcall; } + llvm::Type *getAtomicTy() const { return Ty; } + + virtual llvm::Value *getAtomicPointer() const = 0; + virtual void decorateWithTBAA(Instruction *I) {} + virtual llvm::AllocaInst *CreateAlloca(llvm::Type *Ty, + const llvm::Twine &Name) = 0; + + /// Is the atomic size larger than the underlying value type? + /// + /// Note that the absence of padding does not mean that atomic + /// objects are completely interchangeable with non-atomic + /// objects: we might have promoted the alignment of a type + /// without making it bigger. + bool hasPadding() const { return (ValueSizeInBits != AtomicSizeInBits); } + + LLVMContext &getLLVMContext() const { return Builder->getContext(); } + + static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg) { + if (ValTy->isFloatingPointTy()) + return ValTy->isX86_FP80Ty() || CmpXchg; + return !ValTy->isIntegerTy() && !ValTy->isPointerTy(); + } + + llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile, + bool CmpXchg = false) { + Value *Ptr = getAtomicPointer(); + Type *AtomicTy = Ty; + if (shouldCastToInt(Ty, CmpXchg)) + AtomicTy = llvm::IntegerType::get(getLLVMContext(), AtomicSizeInBits); + LoadInst *Load = + Builder->CreateAlignedLoad(AtomicTy, Ptr, AtomicAlign, "atomic-load"); + Load->setAtomic(AO); + + // Other decoration. + if (IsVolatile) + Load->setVolatile(true); + decorateWithTBAA(Load); + return Load; + } + + static CallInst *emitAtomicLibcalls2(IRBuilderTy *Builder, StringRef fnName, + Type *ResultType, + ArrayRef Args) { + auto &C = Builder->getContext(); + + SmallVector ArgTys; + for (Value *Arg : Args) + ArgTys.push_back(Arg->getType()); + FunctionType *FnType = FunctionType::get(ResultType, ArgTys, false); + Module *M = Builder->GetInsertBlock()->getModule(); + + // TODO: Use llvm::TargetLowering for Libcall ABI + llvm::AttrBuilder fnAttrB(C); + fnAttrB.addAttribute(llvm::Attribute::NoUnwind); + fnAttrB.addAttribute(llvm::Attribute::WillReturn); + llvm::AttributeList fnAttrs = llvm::AttributeList::get( + C, llvm::AttributeList::FunctionIndex, fnAttrB); + FunctionCallee LibcallFn = M->getOrInsertFunction(fnName, FnType, fnAttrs); + CallInst *Call = Builder->CreateCall(LibcallFn, Args); + + return Call; + } + + llvm::Value *getAtomicSizeValue() const { + auto &C = getLLVMContext(); + + // TODO: Get from llvm::TargetMachine / clang::TargetInfo + uint16_t SizeTBits = 64; + uint16_t BitsPerByte = 8; + return llvm::ConstantInt::get(llvm::IntegerType::get(C, SizeTBits), + AtomicSizeInBits / BitsPerByte); + } + + std::pair EmitAtomicCompareExchangeLibcall( + llvm::Value *ExpectedVal, llvm::Value *DesiredVal, + llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure) { + auto &C = getLLVMContext(); + + // __atomic_compare_exchange's expected and desired are passed by pointers + // FIXME: types + + // TODO: Get from llvm::TargetMachine / clang::TargetInfo + uint64_t IntBits = 32; + + // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, + // void *desired, int success, int failure); + llvm::Value *Args[6] = { + getAtomicSizeValue(), + getAtomicPointer(), + ExpectedVal, + DesiredVal, + llvm::Constant::getIntegerValue( + llvm::IntegerType::get(C, IntBits), + llvm::APInt(IntBits, (uint64_t)Success, /*signed=*/true)), + llvm::Constant::getIntegerValue( + llvm::IntegerType::get(C, IntBits), + llvm::APInt(IntBits, (uint64_t)Failure, /*signed=*/true)), + }; + auto Res = emitAtomicLibcalls2(Builder, "__atomic_compare_exchange", + llvm::IntegerType::getInt1Ty(C), Args); + return std::make_pair(ExpectedVal, Res); + } + + Value *castToAtomicIntPointer(Value *addr) const { + return addr; // opaque pointer + } + + Value *getAtomicAddressAsAtomicIntPointer() const { + return castToAtomicIntPointer(getAtomicPointer()); + } + + std::pair + EmitAtomicCompareExchangeOp(llvm::Value *ExpectedVal, llvm::Value *DesiredVal, + llvm::AtomicOrdering Success, + llvm::AtomicOrdering Failure, + bool IsVolatile = false, bool IsWeak = false) { + // Do the atomic store. + Value *Addr = getAtomicAddressAsAtomicIntPointer(); + auto *Inst = Builder->CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal, + getAtomicAlignment(), Success, + Failure, llvm::SyncScope::System); + // Other decoration. + Inst->setVolatile(IsVolatile); + Inst->setWeak(IsWeak); + + // Okay, turn that back into the original value type. + auto *PreviousVal = Builder->CreateExtractValue(Inst, /*Idxs=*/0); + auto *SuccessFailureVal = Builder->CreateExtractValue(Inst, /*Idxs=*/1); + return std::make_pair(PreviousVal, SuccessFailureVal); + } + + std::pair + EmitAtomicCompareExchange(llvm::Value *ExpectedVal, llvm::Value *DesiredVal, + llvm::AtomicOrdering Success, + llvm::AtomicOrdering Failure, bool IsVolatile, + bool IsWeak) { + + // Check whether we should use a library call. + if (shouldUseLibcall()) { + return EmitAtomicCompareExchangeLibcall(ExpectedVal, DesiredVal, Success, + Failure); + } + + // If we've got a scalar value of the right size, try to avoid going + // through memory. + auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, + Failure, IsVolatile, IsWeak); + return Res; + } + + // void __atomic_load(size_t size, void *mem, void *return, int order); + std::pair + EmitAtomicLoadLibcall(llvm::AtomicOrdering AO) { + LLVMContext &Ctx = getLLVMContext(); + Type *SizedIntTy = Type::getIntNTy(Ctx, getAtomicSizeInBits() * 8); + Type *ResultTy; + SmallVector Args; + AttributeList Attr; + Module *M = Builder->GetInsertBlock()->getModule(); + const DataLayout &DL = M->getDataLayout(); + Args.push_back(ConstantInt::get(DL.getIntPtrType(Ctx), + this->getAtomicSizeInBits() / 8)); + Value *PtrVal = getAtomicPointer(); + PtrVal = Builder->CreateAddrSpaceCast(PtrVal, PointerType::getUnqual(Ctx)); + Args.push_back(PtrVal); + AllocaInst *AllocaResult = + CreateAlloca(Ty, getAtomicPointer()->getName() + "atomic.temp.load"); + const Align AllocaAlignment = DL.getPrefTypeAlign(SizedIntTy); + AllocaResult->setAlignment(AllocaAlignment); + Args.push_back(AllocaResult); + Constant *OrderingVal = + ConstantInt::get(Type::getInt32Ty(Ctx), (int)toCABI(AO)); + Args.push_back(OrderingVal); + ResultTy = Type::getVoidTy(Ctx); + SmallVector ArgTys; + for (Value *Arg : Args) + ArgTys.push_back(Arg->getType()); + FunctionType *FnType = FunctionType::get(ResultTy, ArgTys, false); + FunctionCallee LibcallFn = + M->getOrInsertFunction("__atomic_load", FnType, Attr); + CallInst *Call = Builder->CreateCall(LibcallFn, Args); + Call->setAttributes(Attr); + return std::make_pair( + Builder->CreateAlignedLoad(Ty, AllocaResult, AllocaAlignment), + AllocaResult); + } +}; +} // end namespace llvm + +#endif /* LLVM_FRONTEND_ATOMIC_ATOMIC_H */ diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h index bff49dab4a313d..badd6740acf9a9 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -15,6 +15,7 @@ #define LLVM_FRONTEND_OPENMP_OMPIRBUILDER_H #include "llvm/Analysis/MemorySSAUpdater.h" +#include "llvm/Frontend/Atomic/Atomic.h" #include "llvm/Frontend/OpenMP/OMPConstants.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/IRBuilder.h" @@ -457,6 +458,29 @@ class OpenMPIRBuilder { T(Triple(M.getTargetTriple())) {} ~OpenMPIRBuilder(); + class AtomicInfo : public llvm::AtomicInfo> { + + llvm::Value *AtomicVar; + + public: + AtomicInfo(IRBuilder<> *Builder, llvm::Type *Ty, uint64_t AtomicSizeInBits, + uint64_t ValueSizeInBits, llvm::Align AtomicAlign, + llvm::Align ValueAlign, bool UseLibcall, llvm::Value *AtomicVar) + : llvm::AtomicInfo>(Builder, Ty, AtomicSizeInBits, + ValueSizeInBits, AtomicAlign, + ValueAlign, UseLibcall), + AtomicVar(AtomicVar) {} + + llvm::Value *getAtomicPointer() const override { return AtomicVar; } + void decorateWithTBAA(llvm::Instruction *I) override {} + llvm::AllocaInst *CreateAlloca(llvm::Type *Ty, + const llvm::Twine &Name) override { + llvm::AllocaInst *allocaInst = Builder->CreateAlloca(Ty); + allocaInst->setName(Name); + return allocaInst; + } + }; + /// Initialize the internal state, this will put structures types and /// potentially other helpers into the underlying module. Must be called /// before any other method and only once! This internal state includes types diff --git a/llvm/lib/Frontend/Atomic/Atomic.cpp b/llvm/lib/Frontend/Atomic/Atomic.cpp new file mode 100644 index 00000000000000..1b051b31a92348 --- /dev/null +++ b/llvm/lib/Frontend/Atomic/Atomic.cpp @@ -0,0 +1,18 @@ +//===--- Atomic.cpp - Shared codegen of atomic operations -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Frontend/Atomic/Atomic.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/Frontend/Atomic/Atomic.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Operator.h" + +namespace {} // namespace + +namespace llvm {} // end namespace llvm diff --git a/llvm/lib/Frontend/Atomic/CMakeLists.txt b/llvm/lib/Frontend/Atomic/CMakeLists.txt new file mode 100644 index 00000000000000..06f950e4bee4b9 --- /dev/null +++ b/llvm/lib/Frontend/Atomic/CMakeLists.txt @@ -0,0 +1,15 @@ +add_llvm_component_library(LLVMFrontendAtomic + Atomic.cpp + + ADDITIONAL_HEADER_DIRS + ${LLVM_MAIN_INCLUDE_DIR}/llvm/Frontend/Atomic + + DEPENDS + LLVMAnalysis + LLVMTargetParser + + LINK_COMPONENTS + Core + Support + Analysis + ) diff --git a/llvm/lib/Frontend/CMakeLists.txt b/llvm/lib/Frontend/CMakeLists.txt index 62dd0da1e6c2de..b305ce7d771ce7 100644 --- a/llvm/lib/Frontend/CMakeLists.txt +++ b/llvm/lib/Frontend/CMakeLists.txt @@ -1,3 +1,4 @@ +add_subdirectory(Atomic) add_subdirectory(Driver) add_subdirectory(HLSL) add_subdirectory(OpenACC) diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp index ccec66fcb7bacf..55018b89873c35 100644 --- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -23,6 +23,7 @@ #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/Frontend/Atomic/Atomic.h" #include "llvm/Frontend/Offloading/Utility.h" #include "llvm/Frontend/OpenMP/OMPGridValues.h" #include "llvm/IR/Attributes.h" @@ -6033,6 +6034,52 @@ std::pair OpenMPIRBuilder::emitAtomicUpdate( Res.second = Res.first; else Res.second = emitRMWOpAsInstruction(Res.first, Expr, RMWOp); + } else if (RMWOp == llvm::AtomicRMWInst::BinOp::BAD_BINOP) { + LoadInst *OldVal = + Builder.CreateLoad(XElemTy, X, X->getName() + ".atomic.load"); + OldVal->setAtomic(AO); + const DataLayout &LoadDL = OldVal->getModule()->getDataLayout(); + unsigned LoadSize = + LoadDL.getTypeStoreSize(OldVal->getPointerOperand()->getType()); + OpenMPIRBuilder::AtomicInfo atomicInfo(&Builder, XElemTy, LoadSize * 8, + LoadSize * 8, OldVal->getAlign(), + OldVal->getAlign(), true, X); + auto AtomicLoadRes = atomicInfo.EmitAtomicLoadLibcall(AO); + BasicBlock *CurBB = Builder.GetInsertBlock(); + Instruction *CurBBTI = CurBB->getTerminator(); + CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable(); + BasicBlock *ExitBB = + CurBB->splitBasicBlock(CurBBTI, X->getName() + ".atomic.exit"); + BasicBlock *ContBB = CurBB->splitBasicBlock(CurBB->getTerminator(), + X->getName() + ".atomic.cont"); + ContBB->getTerminator()->eraseFromParent(); + Builder.restoreIP(AllocaIP); + AllocaInst *NewAtomicAddr = Builder.CreateAlloca(XElemTy); + NewAtomicAddr->setName(X->getName() + "x.new.val"); + Builder.SetInsertPoint(ContBB); + llvm::PHINode *PHI = Builder.CreatePHI(OldVal->getType(), 2); + PHI->addIncoming(AtomicLoadRes.first, CurBB); + Value *OldExprVal = PHI; + Value *Upd = UpdateOp(OldExprVal, Builder); + Builder.CreateStore(Upd, NewAtomicAddr); + AtomicOrdering Failure = + llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); + auto Result = atomicInfo.EmitAtomicCompareExchangeLibcall( + AtomicLoadRes.second, NewAtomicAddr, AO, Failure); + LoadInst *PHILoad = Builder.CreateLoad(XElemTy, Result.first); + PHI->addIncoming(PHILoad, Builder.GetInsertBlock()); + Builder.CreateCondBr(Result.second, ExitBB, ContBB); + OldVal->eraseFromParent(); + Res.first = OldExprVal; + Res.second = Upd; + + if (UnreachableInst *ExitTI = + dyn_cast(ExitBB->getTerminator())) { + CurBBTI->eraseFromParent(); + Builder.SetInsertPoint(ExitBB); + } else { + Builder.SetInsertPoint(ExitTI); + } } else { IntegerType *IntCastTy = IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());