diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst index 4b1cc1d500c846..d34e867f5e6151 100644 --- a/clang/docs/LanguageExtensions.rst +++ b/clang/docs/LanguageExtensions.rst @@ -3867,6 +3867,30 @@ builtin function, and are named with a ``__opencl_`` prefix. The macros and ``__OPENCL_MEMORY_SCOPE_SUB_GROUP`` are provided, with values corresponding to the enumerators of OpenCL's ``memory_scope`` enumeration.) +__scoped_atomic builtins +------------------------ + +Clang provides a set of atomics taking a memory scope argument. These atomics +are identical to the standard GNU / GCC atomic builtins but taking an extra +memory scope argument. These are designed to be a generic alternative to the +``__opencl_atomic_*`` builtin functions for targets that support atomic memory +scopes. + +Atomic memory scopes are designed to assist optimizations for systems with +several levels of memory hierarchy like GPUs. The following memory scopes are +currently supported: + +* ``__MEMORY_SCOPE_SYSTEM`` +* ``__MEMORY_SCOPE_DEVICE`` +* ``__MEMORY_SCOPE_WRKGRP`` +* ``__MEMORY_SCOPE_WVFRNT`` +* ``__MEMORY_SCOPE_SINGLE`` + +This controls whether or not the atomic operation is ordered with respect to the +whole system, the current device, an OpenCL workgroup, wavefront, or just a +single thread. If these are used on a target that does not support atomic +scopes, then they will behave exactly as the standard GNU atomic builtins. + Low-level ARM exclusive memory builtins --------------------------------------- diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h index a9c4c67a60e8e8..a41f2d66b37b69 100644 --- a/clang/include/clang/AST/Expr.h +++ b/clang/include/clang/AST/Expr.h @@ -6498,7 +6498,7 @@ class AtomicExpr : public Expr { return cast(SubExprs[ORDER_FAIL]); } Expr *getVal2() const { - if (Op == AO__atomic_exchange) + if (Op == AO__atomic_exchange || Op == AO__scoped_atomic_exchange) return cast(SubExprs[ORDER_FAIL]); assert(NumSubExprs > VAL2); return cast(SubExprs[VAL2]); @@ -6539,7 +6539,9 @@ class AtomicExpr : public Expr { getOp() == AO__opencl_atomic_compare_exchange_weak || getOp() == AO__hip_atomic_compare_exchange_weak || getOp() == AO__atomic_compare_exchange || - getOp() == AO__atomic_compare_exchange_n; + getOp() == AO__atomic_compare_exchange_n || + getOp() == AO__scoped_atomic_compare_exchange || + getOp() == AO__scoped_atomic_compare_exchange_n; } bool isOpenCL() const { @@ -6569,13 +6571,13 @@ class AtomicExpr : public Expr { /// \return empty atomic scope model if the atomic op code does not have /// scope operand. static std::unique_ptr getScopeModel(AtomicOp Op) { - auto Kind = - (Op >= AO__opencl_atomic_load && Op <= AO__opencl_atomic_fetch_max) - ? AtomicScopeModelKind::OpenCL - : (Op >= AO__hip_atomic_load && Op <= AO__hip_atomic_fetch_max) - ? AtomicScopeModelKind::HIP - : AtomicScopeModelKind::None; - return AtomicScopeModel::create(Kind); + if (Op >= AO__opencl_atomic_load && Op <= AO__opencl_atomic_fetch_max) + return AtomicScopeModel::create(AtomicScopeModelKind::OpenCL); + else if (Op >= AO__hip_atomic_load && Op <= AO__hip_atomic_fetch_max) + return AtomicScopeModel::create(AtomicScopeModelKind::HIP); + else if (Op >= AO__scoped_atomic_load && Op <= AO__scoped_atomic_fetch_max) + return AtomicScopeModel::create(AtomicScopeModelKind::Generic); + return AtomicScopeModel::create(AtomicScopeModelKind::None); } /// Get atomic scope model. diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def index ec39e926889b93..4dcbaf8a7beaa6 100644 --- a/clang/include/clang/Basic/Builtins.def +++ b/clang/include/clang/Basic/Builtins.def @@ -904,6 +904,32 @@ BUILTIN(__atomic_signal_fence, "vi", "n") BUILTIN(__atomic_always_lock_free, "bzvCD*", "nE") BUILTIN(__atomic_is_lock_free, "bzvCD*", "nE") +// GNU atomic builtins with atomic scopes. +ATOMIC_BUILTIN(__scoped_atomic_load, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_load_n, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_store, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_store_n, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_exchange, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_exchange_n, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_compare_exchange, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_compare_exchange_n, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_add, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_sub, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_and, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_or, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_xor, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_nand, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_add_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_sub_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_and_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_or_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_xor_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_max_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_min_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_nand_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_min, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_max, "v.", "t") + // OpenCL 2.0 atomic builtins. ATOMIC_BUILTIN(__opencl_atomic_init, "v.", "t") ATOMIC_BUILTIN(__opencl_atomic_load, "v.", "t") diff --git a/clang/include/clang/Basic/SyncScope.h b/clang/include/clang/Basic/SyncScope.h index 7919f64c6daf97..bc7ec7b5cf777e 100644 --- a/clang/include/clang/Basic/SyncScope.h +++ b/clang/include/clang/Basic/SyncScope.h @@ -40,6 +40,11 @@ namespace clang { /// Update getAsString. /// enum class SyncScope { + SystemScope, + DeviceScope, + WorkgroupScope, + WavefrontScope, + SingleScope, HIPSingleThread, HIPWavefront, HIPWorkgroup, @@ -54,6 +59,16 @@ enum class SyncScope { inline llvm::StringRef getAsString(SyncScope S) { switch (S) { + case SyncScope::SystemScope: + return "system_scope"; + case SyncScope::DeviceScope: + return "device_scope"; + case SyncScope::WorkgroupScope: + return "workgroup_scope"; + case SyncScope::WavefrontScope: + return "wavefront_scope"; + case SyncScope::SingleScope: + return "single_scope"; case SyncScope::HIPSingleThread: return "hip_singlethread"; case SyncScope::HIPWavefront: @@ -77,7 +92,7 @@ inline llvm::StringRef getAsString(SyncScope S) { } /// Defines the kind of atomic scope models. -enum class AtomicScopeModelKind { None, OpenCL, HIP }; +enum class AtomicScopeModelKind { None, OpenCL, HIP, Generic }; /// Defines the interface for synch scope model. class AtomicScopeModel { @@ -205,6 +220,56 @@ class AtomicScopeHIPModel : public AtomicScopeModel { } }; +/// Defines the generic atomic scope model. +class AtomicScopeGenericModel : public AtomicScopeModel { +public: + /// The enum values match predefined built-in macros __ATOMIC_SCOPE_*. + enum ID { + System = 0, + Device = 1, + Workgroup = 2, + Wavefront = 3, + Single = 4, + Last = Single + }; + + AtomicScopeGenericModel() = default; + + SyncScope map(unsigned S) const override { + switch (static_cast(S)) { + case Device: + return SyncScope::DeviceScope; + case System: + return SyncScope::SystemScope; + case Workgroup: + return SyncScope::WorkgroupScope; + case Wavefront: + return SyncScope::WavefrontScope; + case Single: + return SyncScope::SingleScope; + } + llvm_unreachable("Invalid language sync scope value"); + } + + bool isValid(unsigned S) const override { + return S >= static_cast(System) && + S <= static_cast(Last); + } + + ArrayRef getRuntimeValues() const override { + static_assert(Last == Single, "Does not include all sync scopes"); + static const unsigned Scopes[] = { + static_cast(Device), static_cast(System), + static_cast(Workgroup), static_cast(Wavefront), + static_cast(Single)}; + return llvm::ArrayRef(Scopes); + } + + unsigned getFallBackValue() const override { + return static_cast(System); + } +}; + inline std::unique_ptr AtomicScopeModel::create(AtomicScopeModelKind K) { switch (K) { @@ -214,6 +279,8 @@ AtomicScopeModel::create(AtomicScopeModelKind K) { return std::make_unique(); case AtomicScopeModelKind::HIP: return std::make_unique(); + case AtomicScopeModelKind::Generic: + return std::make_unique(); } llvm_unreachable("Invalid atomic scope model kind"); } diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp index 55c6b732b7081f..b125fc676da841 100644 --- a/clang/lib/AST/Expr.cpp +++ b/clang/lib/AST/Expr.cpp @@ -4887,6 +4887,7 @@ unsigned AtomicExpr::getNumSubExprs(AtomicOp Op) { case AO__atomic_load_n: return 2; + case AO__scoped_atomic_load_n: case AO__opencl_atomic_load: case AO__hip_atomic_load: case AO__c11_atomic_store: @@ -4921,6 +4922,26 @@ unsigned AtomicExpr::getNumSubExprs(AtomicOp Op) { case AO__atomic_fetch_max: return 3; + case AO__scoped_atomic_load: + case AO__scoped_atomic_store: + case AO__scoped_atomic_store_n: + case AO__scoped_atomic_fetch_add: + case AO__scoped_atomic_fetch_sub: + case AO__scoped_atomic_fetch_and: + case AO__scoped_atomic_fetch_or: + case AO__scoped_atomic_fetch_xor: + case AO__scoped_atomic_fetch_nand: + case AO__scoped_atomic_add_fetch: + case AO__scoped_atomic_sub_fetch: + case AO__scoped_atomic_and_fetch: + case AO__scoped_atomic_or_fetch: + case AO__scoped_atomic_xor_fetch: + case AO__scoped_atomic_nand_fetch: + case AO__scoped_atomic_min_fetch: + case AO__scoped_atomic_max_fetch: + case AO__scoped_atomic_fetch_min: + case AO__scoped_atomic_fetch_max: + case AO__scoped_atomic_exchange_n: case AO__hip_atomic_exchange: case AO__hip_atomic_fetch_add: case AO__hip_atomic_fetch_sub: @@ -4942,6 +4963,7 @@ unsigned AtomicExpr::getNumSubExprs(AtomicOp Op) { case AO__atomic_exchange: return 4; + case AO__scoped_atomic_exchange: case AO__c11_atomic_compare_exchange_strong: case AO__c11_atomic_compare_exchange_weak: return 5; @@ -4952,6 +4974,10 @@ unsigned AtomicExpr::getNumSubExprs(AtomicOp Op) { case AO__atomic_compare_exchange: case AO__atomic_compare_exchange_n: return 6; + + case AO__scoped_atomic_compare_exchange: + case AO__scoped_atomic_compare_exchange_n: + return 7; } llvm_unreachable("unknown atomic op"); } diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp index ab4a013de5f552..c04cb313c3387a 100644 --- a/clang/lib/AST/StmtPrinter.cpp +++ b/clang/lib/AST/StmtPrinter.cpp @@ -1841,6 +1841,7 @@ void StmtPrinter::VisitAtomicExpr(AtomicExpr *Node) { PrintExpr(Node->getPtr()); if (Node->getOp() != AtomicExpr::AO__c11_atomic_load && Node->getOp() != AtomicExpr::AO__atomic_load_n && + Node->getOp() != AtomicExpr::AO__scoped_atomic_load_n && Node->getOp() != AtomicExpr::AO__opencl_atomic_load && Node->getOp() != AtomicExpr::AO__hip_atomic_load) { OS << ", "; diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp index 379c833af32a2d..52e6ddb7d6afb0 100644 --- a/clang/lib/CodeGen/CGAtomic.cpp +++ b/clang/lib/CodeGen/CGAtomic.cpp @@ -507,9 +507,11 @@ static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder, default: llvm_unreachable("Unexpected min/max operation"); case AtomicExpr::AO__atomic_max_fetch: + case AtomicExpr::AO__scoped_atomic_max_fetch: Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT; break; case AtomicExpr::AO__atomic_min_fetch: + case AtomicExpr::AO__scoped_atomic_min_fetch: Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT; break; } @@ -544,7 +546,9 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, FailureOrder, Size, Order, Scope); return; case AtomicExpr::AO__atomic_compare_exchange: - case AtomicExpr::AO__atomic_compare_exchange_n: { + case AtomicExpr::AO__atomic_compare_exchange_n: + case AtomicExpr::AO__scoped_atomic_compare_exchange: + case AtomicExpr::AO__scoped_atomic_compare_exchange_n: { if (llvm::ConstantInt *IsWeakC = dyn_cast(IsWeak)) { emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, Val1, Val2, FailureOrder, Size, Order, Scope); @@ -577,7 +581,9 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__opencl_atomic_load: case AtomicExpr::AO__hip_atomic_load: case AtomicExpr::AO__atomic_load_n: - case AtomicExpr::AO__atomic_load: { + case AtomicExpr::AO__atomic_load: + case AtomicExpr::AO__scoped_atomic_load_n: + case AtomicExpr::AO__scoped_atomic_load: { llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); Load->setAtomic(Order, Scope); Load->setVolatile(E->isVolatile()); @@ -589,7 +595,9 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__opencl_atomic_store: case AtomicExpr::AO__hip_atomic_store: case AtomicExpr::AO__atomic_store: - case AtomicExpr::AO__atomic_store_n: { + case AtomicExpr::AO__atomic_store_n: + case AtomicExpr::AO__scoped_atomic_store: + case AtomicExpr::AO__scoped_atomic_store_n: { llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); Store->setAtomic(Order, Scope); @@ -602,10 +610,13 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__opencl_atomic_exchange: case AtomicExpr::AO__atomic_exchange_n: case AtomicExpr::AO__atomic_exchange: + case AtomicExpr::AO__scoped_atomic_exchange_n: + case AtomicExpr::AO__scoped_atomic_exchange: Op = llvm::AtomicRMWInst::Xchg; break; case AtomicExpr::AO__atomic_add_fetch: + case AtomicExpr::AO__scoped_atomic_add_fetch: PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd : llvm::Instruction::Add; [[fallthrough]]; @@ -613,11 +624,13 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__hip_atomic_fetch_add: case AtomicExpr::AO__opencl_atomic_fetch_add: case AtomicExpr::AO__atomic_fetch_add: + case AtomicExpr::AO__scoped_atomic_fetch_add: Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd : llvm::AtomicRMWInst::Add; break; case AtomicExpr::AO__atomic_sub_fetch: + case AtomicExpr::AO__scoped_atomic_sub_fetch: PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub : llvm::Instruction::Sub; [[fallthrough]]; @@ -625,17 +638,20 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__hip_atomic_fetch_sub: case AtomicExpr::AO__opencl_atomic_fetch_sub: case AtomicExpr::AO__atomic_fetch_sub: + case AtomicExpr::AO__scoped_atomic_fetch_sub: Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub : llvm::AtomicRMWInst::Sub; break; case AtomicExpr::AO__atomic_min_fetch: + case AtomicExpr::AO__scoped_atomic_min_fetch: PostOpMinMax = true; [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_min: case AtomicExpr::AO__hip_atomic_fetch_min: case AtomicExpr::AO__opencl_atomic_fetch_min: case AtomicExpr::AO__atomic_fetch_min: + case AtomicExpr::AO__scoped_atomic_fetch_min: Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FMin : (E->getValueType()->isSignedIntegerType() @@ -644,12 +660,14 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, break; case AtomicExpr::AO__atomic_max_fetch: + case AtomicExpr::AO__scoped_atomic_max_fetch: PostOpMinMax = true; [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_max: case AtomicExpr::AO__hip_atomic_fetch_max: case AtomicExpr::AO__opencl_atomic_fetch_max: case AtomicExpr::AO__atomic_fetch_max: + case AtomicExpr::AO__scoped_atomic_fetch_max: Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FMax : (E->getValueType()->isSignedIntegerType() @@ -658,40 +676,48 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, break; case AtomicExpr::AO__atomic_and_fetch: + case AtomicExpr::AO__scoped_atomic_and_fetch: PostOp = llvm::Instruction::And; [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_and: case AtomicExpr::AO__hip_atomic_fetch_and: case AtomicExpr::AO__opencl_atomic_fetch_and: case AtomicExpr::AO__atomic_fetch_and: + case AtomicExpr::AO__scoped_atomic_fetch_and: Op = llvm::AtomicRMWInst::And; break; case AtomicExpr::AO__atomic_or_fetch: + case AtomicExpr::AO__scoped_atomic_or_fetch: PostOp = llvm::Instruction::Or; [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_or: case AtomicExpr::AO__hip_atomic_fetch_or: case AtomicExpr::AO__opencl_atomic_fetch_or: case AtomicExpr::AO__atomic_fetch_or: + case AtomicExpr::AO__scoped_atomic_fetch_or: Op = llvm::AtomicRMWInst::Or; break; case AtomicExpr::AO__atomic_xor_fetch: + case AtomicExpr::AO__scoped_atomic_xor_fetch: PostOp = llvm::Instruction::Xor; [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_xor: case AtomicExpr::AO__hip_atomic_fetch_xor: case AtomicExpr::AO__opencl_atomic_fetch_xor: case AtomicExpr::AO__atomic_fetch_xor: + case AtomicExpr::AO__scoped_atomic_fetch_xor: Op = llvm::AtomicRMWInst::Xor; break; case AtomicExpr::AO__atomic_nand_fetch: + case AtomicExpr::AO__scoped_atomic_nand_fetch: PostOp = llvm::Instruction::And; // the NOT is special cased below [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_nand: case AtomicExpr::AO__atomic_fetch_nand: + case AtomicExpr::AO__scoped_atomic_fetch_nand: Op = llvm::AtomicRMWInst::Nand; break; } @@ -711,7 +737,8 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, else if (PostOp) Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI, LoadVal1); - if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) + if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch || + E->getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch) Result = CGF.Builder.CreateNot(Result); CGF.Builder.CreateStore(Result, Dest); } @@ -861,20 +888,24 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { llvm_unreachable("Already handled above with EmitAtomicInit!"); case AtomicExpr::AO__atomic_load_n: + case AtomicExpr::AO__scoped_atomic_load_n: case AtomicExpr::AO__c11_atomic_load: case AtomicExpr::AO__opencl_atomic_load: case AtomicExpr::AO__hip_atomic_load: break; case AtomicExpr::AO__atomic_load: + case AtomicExpr::AO__scoped_atomic_load: Dest = EmitPointerWithAlignment(E->getVal1()); break; case AtomicExpr::AO__atomic_store: + case AtomicExpr::AO__scoped_atomic_store: Val1 = EmitPointerWithAlignment(E->getVal1()); break; case AtomicExpr::AO__atomic_exchange: + case AtomicExpr::AO__scoped_atomic_exchange: Val1 = EmitPointerWithAlignment(E->getVal1()); Dest = EmitPointerWithAlignment(E->getVal2()); break; @@ -887,14 +918,19 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__hip_atomic_compare_exchange_strong: case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: + case AtomicExpr::AO__scoped_atomic_compare_exchange: + case AtomicExpr::AO__scoped_atomic_compare_exchange_n: Val1 = EmitPointerWithAlignment(E->getVal1()); - if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) + if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange || + E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange) Val2 = EmitPointerWithAlignment(E->getVal2()); else Val2 = EmitValToTemp(*this, E->getVal2()); OrderFail = EmitScalarExpr(E->getOrderFail()); if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n || - E->getOp() == AtomicExpr::AO__atomic_compare_exchange) + E->getOp() == AtomicExpr::AO__atomic_compare_exchange || + E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange_n || + E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange) IsWeak = EmitScalarExpr(E->getWeak()); break; @@ -934,6 +970,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__opencl_atomic_fetch_min: case AtomicExpr::AO__hip_atomic_fetch_max: case AtomicExpr::AO__hip_atomic_fetch_min: + case AtomicExpr::AO__scoped_atomic_fetch_add: + case AtomicExpr::AO__scoped_atomic_fetch_max: + case AtomicExpr::AO__scoped_atomic_fetch_min: + case AtomicExpr::AO__scoped_atomic_fetch_sub: + case AtomicExpr::AO__scoped_atomic_add_fetch: + case AtomicExpr::AO__scoped_atomic_max_fetch: + case AtomicExpr::AO__scoped_atomic_min_fetch: + case AtomicExpr::AO__scoped_atomic_sub_fetch: ShouldCastToIntPtrTy = !MemTy->isFloatingType(); [[fallthrough]]; @@ -963,6 +1007,16 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__opencl_atomic_fetch_xor: case AtomicExpr::AO__opencl_atomic_store: case AtomicExpr::AO__opencl_atomic_exchange: + case AtomicExpr::AO__scoped_atomic_fetch_and: + case AtomicExpr::AO__scoped_atomic_fetch_nand: + case AtomicExpr::AO__scoped_atomic_fetch_or: + case AtomicExpr::AO__scoped_atomic_fetch_xor: + case AtomicExpr::AO__scoped_atomic_and_fetch: + case AtomicExpr::AO__scoped_atomic_nand_fetch: + case AtomicExpr::AO__scoped_atomic_or_fetch: + case AtomicExpr::AO__scoped_atomic_xor_fetch: + case AtomicExpr::AO__scoped_atomic_store_n: + case AtomicExpr::AO__scoped_atomic_exchange_n: Val1 = EmitValToTemp(*this, E->getVal1()); break; } @@ -1039,6 +1093,22 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__opencl_atomic_fetch_or: case AtomicExpr::AO__opencl_atomic_fetch_sub: case AtomicExpr::AO__opencl_atomic_fetch_xor: + case AtomicExpr::AO__scoped_atomic_fetch_add: + case AtomicExpr::AO__scoped_atomic_fetch_and: + case AtomicExpr::AO__scoped_atomic_fetch_max: + case AtomicExpr::AO__scoped_atomic_fetch_min: + case AtomicExpr::AO__scoped_atomic_fetch_nand: + case AtomicExpr::AO__scoped_atomic_fetch_or: + case AtomicExpr::AO__scoped_atomic_fetch_sub: + case AtomicExpr::AO__scoped_atomic_fetch_xor: + case AtomicExpr::AO__scoped_atomic_add_fetch: + case AtomicExpr::AO__scoped_atomic_and_fetch: + case AtomicExpr::AO__scoped_atomic_max_fetch: + case AtomicExpr::AO__scoped_atomic_min_fetch: + case AtomicExpr::AO__scoped_atomic_nand_fetch: + case AtomicExpr::AO__scoped_atomic_or_fetch: + case AtomicExpr::AO__scoped_atomic_sub_fetch: + case AtomicExpr::AO__scoped_atomic_xor_fetch: // For these, only library calls for certain sizes exist. UseOptimizedLibcall = true; break; @@ -1047,6 +1117,10 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__atomic_store: case AtomicExpr::AO__atomic_exchange: case AtomicExpr::AO__atomic_compare_exchange: + case AtomicExpr::AO__scoped_atomic_load: + case AtomicExpr::AO__scoped_atomic_store: + case AtomicExpr::AO__scoped_atomic_exchange: + case AtomicExpr::AO__scoped_atomic_compare_exchange: // Use the generic version if we don't know that the operand will be // suitably aligned for the optimized version. if (Misaligned) @@ -1071,6 +1145,10 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__opencl_atomic_exchange: case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: + case AtomicExpr::AO__scoped_atomic_load_n: + case AtomicExpr::AO__scoped_atomic_store_n: + case AtomicExpr::AO__scoped_atomic_exchange_n: + case AtomicExpr::AO__scoped_atomic_compare_exchange_n: // Only use optimized library calls for sizes for which they exist. // FIXME: Size == 16 optimized library functions exist too. if (Size == 1 || Size == 2 || Size == 4 || Size == 8) @@ -1131,6 +1209,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__hip_atomic_compare_exchange_strong: case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: + case AtomicExpr::AO__scoped_atomic_compare_exchange: + case AtomicExpr::AO__scoped_atomic_compare_exchange_n: LibCallName = "__atomic_compare_exchange"; RetTy = getContext().BoolTy; HaveRetTy = true; @@ -1150,6 +1230,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__c11_atomic_exchange: case AtomicExpr::AO__hip_atomic_exchange: case AtomicExpr::AO__opencl_atomic_exchange: + case AtomicExpr::AO__scoped_atomic_exchange: + case AtomicExpr::AO__scoped_atomic_exchange_n: LibCallName = "__atomic_exchange"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), MemTy, E->getExprLoc(), TInfo.Width); @@ -1161,6 +1243,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__c11_atomic_store: case AtomicExpr::AO__hip_atomic_store: case AtomicExpr::AO__opencl_atomic_store: + case AtomicExpr::AO__scoped_atomic_store: + case AtomicExpr::AO__scoped_atomic_store_n: LibCallName = "__atomic_store"; RetTy = getContext().VoidTy; HaveRetTy = true; @@ -1174,17 +1258,21 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__c11_atomic_load: case AtomicExpr::AO__hip_atomic_load: case AtomicExpr::AO__opencl_atomic_load: + case AtomicExpr::AO__scoped_atomic_load: + case AtomicExpr::AO__scoped_atomic_load_n: LibCallName = "__atomic_load"; break; // T __atomic_add_fetch_N(T *mem, T val, int order) // T __atomic_fetch_add_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_add_fetch: + case AtomicExpr::AO__scoped_atomic_add_fetch: PostOp = llvm::Instruction::Add; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_add: case AtomicExpr::AO__c11_atomic_fetch_add: case AtomicExpr::AO__hip_atomic_fetch_add: case AtomicExpr::AO__opencl_atomic_fetch_add: + case AtomicExpr::AO__scoped_atomic_fetch_add: LibCallName = "__atomic_fetch_add"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), LoweredMemTy, E->getExprLoc(), TInfo.Width); @@ -1192,12 +1280,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { // T __atomic_and_fetch_N(T *mem, T val, int order) // T __atomic_fetch_and_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_and_fetch: + case AtomicExpr::AO__scoped_atomic_and_fetch: PostOp = llvm::Instruction::And; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_and: case AtomicExpr::AO__c11_atomic_fetch_and: case AtomicExpr::AO__hip_atomic_fetch_and: case AtomicExpr::AO__opencl_atomic_fetch_and: + case AtomicExpr::AO__scoped_atomic_fetch_and: LibCallName = "__atomic_fetch_and"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), MemTy, E->getExprLoc(), TInfo.Width); @@ -1205,12 +1295,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { // T __atomic_or_fetch_N(T *mem, T val, int order) // T __atomic_fetch_or_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_or_fetch: + case AtomicExpr::AO__scoped_atomic_or_fetch: PostOp = llvm::Instruction::Or; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_or: case AtomicExpr::AO__c11_atomic_fetch_or: case AtomicExpr::AO__hip_atomic_fetch_or: case AtomicExpr::AO__opencl_atomic_fetch_or: + case AtomicExpr::AO__scoped_atomic_fetch_or: LibCallName = "__atomic_fetch_or"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), MemTy, E->getExprLoc(), TInfo.Width); @@ -1218,12 +1310,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { // T __atomic_sub_fetch_N(T *mem, T val, int order) // T __atomic_fetch_sub_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_sub_fetch: + case AtomicExpr::AO__scoped_atomic_sub_fetch: PostOp = llvm::Instruction::Sub; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_sub: case AtomicExpr::AO__c11_atomic_fetch_sub: case AtomicExpr::AO__hip_atomic_fetch_sub: case AtomicExpr::AO__opencl_atomic_fetch_sub: + case AtomicExpr::AO__scoped_atomic_fetch_sub: LibCallName = "__atomic_fetch_sub"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), LoweredMemTy, E->getExprLoc(), TInfo.Width); @@ -1231,21 +1325,25 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { // T __atomic_xor_fetch_N(T *mem, T val, int order) // T __atomic_fetch_xor_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_xor_fetch: + case AtomicExpr::AO__scoped_atomic_xor_fetch: PostOp = llvm::Instruction::Xor; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_xor: case AtomicExpr::AO__c11_atomic_fetch_xor: case AtomicExpr::AO__hip_atomic_fetch_xor: case AtomicExpr::AO__opencl_atomic_fetch_xor: + case AtomicExpr::AO__scoped_atomic_fetch_xor: LibCallName = "__atomic_fetch_xor"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), MemTy, E->getExprLoc(), TInfo.Width); break; case AtomicExpr::AO__atomic_min_fetch: + case AtomicExpr::AO__scoped_atomic_min_fetch: PostOpMinMax = true; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_min: case AtomicExpr::AO__c11_atomic_fetch_min: + case AtomicExpr::AO__scoped_atomic_fetch_min: case AtomicExpr::AO__hip_atomic_fetch_min: case AtomicExpr::AO__opencl_atomic_fetch_min: LibCallName = E->getValueType()->isSignedIntegerType() @@ -1255,12 +1353,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { LoweredMemTy, E->getExprLoc(), TInfo.Width); break; case AtomicExpr::AO__atomic_max_fetch: + case AtomicExpr::AO__scoped_atomic_max_fetch: PostOpMinMax = true; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_max: case AtomicExpr::AO__c11_atomic_fetch_max: case AtomicExpr::AO__hip_atomic_fetch_max: case AtomicExpr::AO__opencl_atomic_fetch_max: + case AtomicExpr::AO__scoped_atomic_fetch_max: LibCallName = E->getValueType()->isSignedIntegerType() ? "__atomic_fetch_max" : "__atomic_fetch_umax"; @@ -1270,10 +1370,12 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { // T __atomic_nand_fetch_N(T *mem, T val, int order) // T __atomic_fetch_nand_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_nand_fetch: + case AtomicExpr::AO__scoped_atomic_nand_fetch: PostOp = llvm::Instruction::And; // the NOT is special cased below [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_nand: case AtomicExpr::AO__c11_atomic_fetch_nand: + case AtomicExpr::AO__scoped_atomic_fetch_nand: LibCallName = "__atomic_fetch_nand"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), MemTy, E->getExprLoc(), TInfo.Width); @@ -1330,7 +1432,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1); } - if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) + if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch || + E->getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch) ResVal = Builder.CreateNot(ResVal); Builder.CreateStore(ResVal, Dest.withElementType(ResVal->getType())); @@ -1347,12 +1450,16 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { E->getOp() == AtomicExpr::AO__opencl_atomic_store || E->getOp() == AtomicExpr::AO__hip_atomic_store || E->getOp() == AtomicExpr::AO__atomic_store || - E->getOp() == AtomicExpr::AO__atomic_store_n; + E->getOp() == AtomicExpr::AO__atomic_store_n || + E->getOp() == AtomicExpr::AO__scoped_atomic_store || + E->getOp() == AtomicExpr::AO__scoped_atomic_store_n; bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || E->getOp() == AtomicExpr::AO__opencl_atomic_load || E->getOp() == AtomicExpr::AO__hip_atomic_load || E->getOp() == AtomicExpr::AO__atomic_load || - E->getOp() == AtomicExpr::AO__atomic_load_n; + E->getOp() == AtomicExpr::AO__atomic_load_n || + E->getOp() == AtomicExpr::AO__scoped_atomic_load || + E->getOp() == AtomicExpr::AO__scoped_atomic_load_n; if (isa(Order)) { auto ord = cast(Order)->getZExtValue(); diff --git a/clang/lib/CodeGen/Targets/AMDGPU.cpp b/clang/lib/CodeGen/Targets/AMDGPU.cpp index b654e3f12af8d4..03ac6b78598fc8 100644 --- a/clang/lib/CodeGen/Targets/AMDGPU.cpp +++ b/clang/lib/CodeGen/Targets/AMDGPU.cpp @@ -471,20 +471,25 @@ AMDGPUTargetCodeGenInfo::getLLVMSyncScopeID(const LangOptions &LangOpts, std::string Name; switch (Scope) { case SyncScope::HIPSingleThread: + case SyncScope::SingleScope: Name = "singlethread"; break; case SyncScope::HIPWavefront: case SyncScope::OpenCLSubGroup: + case SyncScope::WavefrontScope: Name = "wavefront"; break; case SyncScope::HIPWorkgroup: case SyncScope::OpenCLWorkGroup: + case SyncScope::WorkgroupScope: Name = "workgroup"; break; case SyncScope::HIPAgent: case SyncScope::OpenCLDevice: + case SyncScope::DeviceScope: Name = "agent"; break; + case SyncScope::SystemScope: case SyncScope::HIPSystem: case SyncScope::OpenCLAllSVMDevices: Name = ""; diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp index 17948dcebd7e55..16a2947e642aa5 100644 --- a/clang/lib/Frontend/InitPreprocessor.cpp +++ b/clang/lib/Frontend/InitPreprocessor.cpp @@ -809,6 +809,13 @@ static void InitializePredefinedMacros(const TargetInfo &TI, Builder.defineMacro("__ATOMIC_ACQ_REL", "4"); Builder.defineMacro("__ATOMIC_SEQ_CST", "5"); + // Define macros for the clang atomic scopes. + Builder.defineMacro("__MEMORY_SCOPE_SYSTEM", "0"); + Builder.defineMacro("__MEMORY_SCOPE_DEVICE", "1"); + Builder.defineMacro("__MEMORY_SCOPE_WRKGRP", "2"); + Builder.defineMacro("__MEMORY_SCOPE_WVFRNT", "3"); + Builder.defineMacro("__MEMORY_SCOPE_SINGLE", "4"); + // Define macros for the OpenCL memory scope. // The values should match AtomicScopeOpenCLModel::ID enum. static_assert( diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index fc6ee6b2c5ab4f..a729cff53fc11b 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -7653,6 +7653,8 @@ static bool isValidOrderingForOp(int64_t Ordering, AtomicExpr::AtomicOp Op) { case AtomicExpr::AO__hip_atomic_load: case AtomicExpr::AO__atomic_load_n: case AtomicExpr::AO__atomic_load: + case AtomicExpr::AO__scoped_atomic_load_n: + case AtomicExpr::AO__scoped_atomic_load: return OrderingCABI != llvm::AtomicOrderingCABI::release && OrderingCABI != llvm::AtomicOrderingCABI::acq_rel; @@ -7661,6 +7663,8 @@ static bool isValidOrderingForOp(int64_t Ordering, AtomicExpr::AtomicOp Op) { case AtomicExpr::AO__hip_atomic_store: case AtomicExpr::AO__atomic_store: case AtomicExpr::AO__atomic_store_n: + case AtomicExpr::AO__scoped_atomic_store: + case AtomicExpr::AO__scoped_atomic_store_n: return OrderingCABI != llvm::AtomicOrderingCABI::consume && OrderingCABI != llvm::AtomicOrderingCABI::acquire && OrderingCABI != llvm::AtomicOrderingCABI::acq_rel; @@ -7737,13 +7741,19 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, Op <= AtomicExpr::AO__opencl_atomic_fetch_max; bool IsHIP = Op >= AtomicExpr::AO__hip_atomic_load && Op <= AtomicExpr::AO__hip_atomic_fetch_max; + bool IsScoped = Op >= AtomicExpr::AO__scoped_atomic_load && + Op <= AtomicExpr::AO__scoped_atomic_fetch_max; bool IsC11 = (Op >= AtomicExpr::AO__c11_atomic_init && Op <= AtomicExpr::AO__c11_atomic_fetch_min) || IsOpenCL; bool IsN = Op == AtomicExpr::AO__atomic_load_n || Op == AtomicExpr::AO__atomic_store_n || Op == AtomicExpr::AO__atomic_exchange_n || - Op == AtomicExpr::AO__atomic_compare_exchange_n; + Op == AtomicExpr::AO__atomic_compare_exchange_n || + Op == AtomicExpr::AO__scoped_atomic_load_n || + Op == AtomicExpr::AO__scoped_atomic_store_n || + Op == AtomicExpr::AO__scoped_atomic_exchange_n || + Op == AtomicExpr::AO__scoped_atomic_compare_exchange_n; // Bit mask for extra allowed value types other than integers for atomic // arithmetic operations. Add/sub allow pointer and floating point. Min/max // allow floating point. @@ -7764,10 +7774,12 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, case AtomicExpr::AO__opencl_atomic_load: case AtomicExpr::AO__hip_atomic_load: case AtomicExpr::AO__atomic_load_n: + case AtomicExpr::AO__scoped_atomic_load_n: Form = Load; break; case AtomicExpr::AO__atomic_load: + case AtomicExpr::AO__scoped_atomic_load: Form = LoadCopy; break; @@ -7776,12 +7788,18 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, case AtomicExpr::AO__hip_atomic_store: case AtomicExpr::AO__atomic_store: case AtomicExpr::AO__atomic_store_n: + case AtomicExpr::AO__scoped_atomic_store: + case AtomicExpr::AO__scoped_atomic_store_n: Form = Copy; break; case AtomicExpr::AO__atomic_fetch_add: case AtomicExpr::AO__atomic_fetch_sub: case AtomicExpr::AO__atomic_add_fetch: case AtomicExpr::AO__atomic_sub_fetch: + case AtomicExpr::AO__scoped_atomic_fetch_add: + case AtomicExpr::AO__scoped_atomic_fetch_sub: + case AtomicExpr::AO__scoped_atomic_add_fetch: + case AtomicExpr::AO__scoped_atomic_sub_fetch: case AtomicExpr::AO__c11_atomic_fetch_add: case AtomicExpr::AO__c11_atomic_fetch_sub: case AtomicExpr::AO__opencl_atomic_fetch_add: @@ -7795,6 +7813,10 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, case AtomicExpr::AO__atomic_fetch_min: case AtomicExpr::AO__atomic_max_fetch: case AtomicExpr::AO__atomic_min_fetch: + case AtomicExpr::AO__scoped_atomic_fetch_max: + case AtomicExpr::AO__scoped_atomic_fetch_min: + case AtomicExpr::AO__scoped_atomic_max_fetch: + case AtomicExpr::AO__scoped_atomic_min_fetch: case AtomicExpr::AO__c11_atomic_fetch_max: case AtomicExpr::AO__c11_atomic_fetch_min: case AtomicExpr::AO__opencl_atomic_fetch_max: @@ -7822,6 +7844,14 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, case AtomicExpr::AO__atomic_or_fetch: case AtomicExpr::AO__atomic_xor_fetch: case AtomicExpr::AO__atomic_nand_fetch: + case AtomicExpr::AO__scoped_atomic_fetch_and: + case AtomicExpr::AO__scoped_atomic_fetch_or: + case AtomicExpr::AO__scoped_atomic_fetch_xor: + case AtomicExpr::AO__scoped_atomic_fetch_nand: + case AtomicExpr::AO__scoped_atomic_and_fetch: + case AtomicExpr::AO__scoped_atomic_or_fetch: + case AtomicExpr::AO__scoped_atomic_xor_fetch: + case AtomicExpr::AO__scoped_atomic_nand_fetch: Form = Arithmetic; break; @@ -7829,10 +7859,12 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, case AtomicExpr::AO__hip_atomic_exchange: case AtomicExpr::AO__opencl_atomic_exchange: case AtomicExpr::AO__atomic_exchange_n: + case AtomicExpr::AO__scoped_atomic_exchange_n: Form = Xchg; break; case AtomicExpr::AO__atomic_exchange: + case AtomicExpr::AO__scoped_atomic_exchange: Form = GNUXchg; break; @@ -7847,12 +7879,15 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, case AtomicExpr::AO__atomic_compare_exchange: case AtomicExpr::AO__atomic_compare_exchange_n: + case AtomicExpr::AO__scoped_atomic_compare_exchange: + case AtomicExpr::AO__scoped_atomic_compare_exchange_n: Form = GNUCmpXchg; break; } unsigned AdjustedNumArgs = NumArgs[Form]; - if ((IsOpenCL || IsHIP) && Op != AtomicExpr::AO__opencl_atomic_init) + if ((IsOpenCL || IsHIP || IsScoped) && + Op != AtomicExpr::AO__opencl_atomic_init) ++AdjustedNumArgs; // Check we have the right number of arguments. if (Args.size() < AdjustedNumArgs) { diff --git a/clang/test/CodeGen/scoped-atomic-ops.c b/clang/test/CodeGen/scoped-atomic-ops.c new file mode 100644 index 00000000000000..b0032046639b89 --- /dev/null +++ b/clang/test/CodeGen/scoped-atomic-ops.c @@ -0,0 +1,331 @@ +// RUN: %clang_cc1 %s -emit-llvm -o - -triple=amdgcn-amd-amdhsa -ffreestanding \ +// RUN: -fvisibility=hidden | FileCheck %s + +// CHECK-LABEL: define hidden i32 @fi1a( +// CHECK: [[TMP0:%.*]] = load atomic i32, ptr [[PTR0:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = load atomic i32, ptr [[PTR1:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = load atomic i32, ptr [[PTR2:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = load atomic i32, ptr [[PTR3:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = load atomic i32, ptr [[PTR4:.+]] syncscope("singlethread-one-as") monotonic, align 4 +int fi1a(int *i) { + int v; + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + return v; +} + +// CHECK-LABEL: define hidden i32 @fi1b( +// CHECK: [[TMP0:%.*]] = load atomic i32, ptr [[PTR0:%.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = load atomic i32, ptr [[PTR1:%.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = load atomic i32, ptr [[PTR2:%.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = load atomic i32, ptr [[PTR3:%.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = load atomic i32, ptr [[PTR4:%.+]] syncscope("singlethread-one-as") monotonic, align 4 +// +int fi1b(int *i) { + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + return *i; +} + +// CHECK-LABEL: define hidden void @fi2a( +// CHECK: store atomic i32 [[TMP0:%.+]], ptr [[PTR0:%.+]] syncscope("one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP1:%.+]], ptr [[PTR1:%.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP2:%.+]], ptr [[PTR2:%.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP3:%.+]], ptr [[PTR3:%.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP4:%.+]], ptr [[PTR4:%.+]] syncscope("singlethread-one-as") monotonic, align 4 +// +void fi2a(int *i) { + int v = 1; + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); +} + +// CHECK-LABEL: define hidden void @fi2b( +// CHECK: store atomic i32 [[TMP0:%.+]], ptr [[PTR0:%.+]] syncscope("one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP1:%.+]], ptr [[PTR1:%.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP2:%.+]], ptr [[PTR2:%.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP3:%.+]], ptr [[PTR3:%.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP4:%.+]], ptr [[PTR4:%.+]] syncscope("singlethread-one-as") monotonic, align 4 +void fi2b(int *i) { + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); +} + +// CHECK-LABEL: define hidden void @fi3a( +// CHECK: [[TMP0:%.*]] = atomicrmw add ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = atomicrmw sub ptr [[PTR1:%.+]], i32 [[VAL1:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = atomicrmw and ptr [[PTR2:%.+]], i32 [[VAL2:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = atomicrmw or ptr [[PTR3:%.+]], i32 [[VAL3:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = atomicrmw xor ptr [[PTR4:%.+]], i32 [[VAL4:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP5:%.*]] = atomicrmw nand ptr [[PTR5:%.+]], i32 [[VAL5:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP6:%.*]] = atomicrmw min ptr [[PTR6:%.+]], i32 [[VAL6:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP7:%.*]] = atomicrmw max ptr [[PTR7:%.+]], i32 [[VAL7:.+]] syncscope("one-as") monotonic, align 4 +void fi3a(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); +} + +// CHECK-LABEL: define hidden void @fi3b( +// CHECK: [[TMP0:%.*]] = atomicrmw add ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = atomicrmw sub ptr [[PTR1:%.+]], i32 [[VAL1:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = atomicrmw and ptr [[PTR2:%.+]], i32 [[VAL2:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = atomicrmw or ptr [[PTR3:%.+]], i32 [[VAL3:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = atomicrmw xor ptr [[PTR4:%.+]], i32 [[VAL4:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP5:%.*]] = atomicrmw nand ptr [[PTR5:%.+]], i32 [[VAL5:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP6:%.*]] = atomicrmw min ptr [[PTR6:%.+]], i32 [[VAL6:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP7:%.*]] = atomicrmw max ptr [[PTR7:%.+]], i32 [[VAL7:.+]] syncscope("agent-one-as") monotonic, align 4 +void fi3b(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); +} + +// CHECK-LABEL: define hidden void @fi3c( +// CHECK: [[TMP0:%.*]] = atomicrmw add ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = atomicrmw sub ptr [[PTR1:%.+]], i32 [[VAL1:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = atomicrmw and ptr [[PTR2:%.+]], i32 [[VAL2:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = atomicrmw or ptr [[PTR3:%.+]], i32 [[VAL3:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = atomicrmw xor ptr [[PTR4:%.+]], i32 [[VAL4:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP5:%.*]] = atomicrmw nand ptr [[PTR5:%.+]], i32 [[VAL5:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP6:%.*]] = atomicrmw min ptr [[PTR6:%.+]], i32 [[VAL6:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP7:%.*]] = atomicrmw max ptr [[PTR7:%.+]], i32 [[VAL7:.+]] syncscope("workgroup-one-as") monotonic, align 4 +void fi3c(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); +} + +// CHECK-LABEL: define hidden void @fi3d( +// CHECK: [[TMP0:%.*]] = atomicrmw add ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = atomicrmw sub ptr [[PTR1:%.+]], i32 [[VAL1:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = atomicrmw and ptr [[PTR2:%.+]], i32 [[VAL2:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = atomicrmw or ptr [[PTR3:%.+]], i32 [[VAL3:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = atomicrmw xor ptr [[PTR4:%.+]], i32 [[VAL4:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP5:%.*]] = atomicrmw nand ptr [[PTR5:%.+]], i32 [[VAL5:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP6:%.*]] = atomicrmw min ptr [[PTR6:%.+]], i32 [[VAL6:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP7:%.*]] = atomicrmw max ptr [[PTR7:%.+]], i32 [[VAL7:.+]] syncscope("wavefront-one-as") monotonic, align 4 +void fi3d(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); +} + +// CHECK-LABEL: define hidden void @fi3e( +// CHECK: [[TMP0:%.*]] = atomicrmw add ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = atomicrmw sub ptr [[PTR1:%.+]], i32 [[VAL1:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = atomicrmw and ptr [[PTR2:%.+]], i32 [[VAL2:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = atomicrmw or ptr [[PTR3:%.+]], i32 [[VAL3:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = atomicrmw xor ptr [[PTR4:%.+]], i32 [[VAL4:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP5:%.*]] = atomicrmw nand ptr [[PTR5:%.+]], i32 [[VAL5:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP6:%.*]] = atomicrmw min ptr [[PTR6:%.+]], i32 [[VAL6:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP7:%.*]] = atomicrmw max ptr [[PTR7:%.+]], i32 [[VAL7:.+]] syncscope("singlethread-one-as") monotonic, align 4 +void fi3e(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi4a( +// CHECK: [[TMP0:%.*]] = cmpxchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("one-as") acquire acquire, align 4 +_Bool fi4a(int *i) { + int cmp = 0; + int desired = 1; + return __scoped_atomic_compare_exchange(i, &cmp, &desired, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_SYSTEM); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi4b( +// CHECK: [[TMP0:%.*]] = cmpxchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("agent-one-as") acquire acquire, align 4 +_Bool fi4b(int *i) { + int cmp = 0; + int desired = 1; + return __scoped_atomic_compare_exchange(i, &cmp, &desired, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_DEVICE); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi4c( +// CHECK: [[TMP0:%.*]] = cmpxchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("workgroup-one-as") acquire acquire, align 4 +_Bool fi4c(int *i) { + int cmp = 0; + int desired = 1; + return __scoped_atomic_compare_exchange(i, &cmp, &desired, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_WRKGRP); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi4d( +// CHECK: [[TMP0:%.*]] = cmpxchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("wavefront-one-as") acquire acquire, align 4 +_Bool fi4d(int *i) { + int cmp = 0; + int desired = 1; + return __scoped_atomic_compare_exchange(i, &cmp, &desired, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_WVFRNT); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi4e( +// CHECK: [[TMP0:%.*]] = cmpxchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("singlethread-one-as") acquire acquire, align 4 +_Bool fi4e(int *i) { + int cmp = 0; + int desired = 1; + return __scoped_atomic_compare_exchange(i, &cmp, &desired, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_SINGLE); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi5a( +// CHECK: [[TMP0:%.*]] = cmpxchg weak ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("one-as") acquire acquire, align 4 +_Bool fi5a(int *i) { + int cmp = 0; + return __scoped_atomic_compare_exchange_n(i, &cmp, 1, 1, __ATOMIC_ACQUIRE, + __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_SYSTEM); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi5b( +// CHECK: [[TMP0:%.*]] = cmpxchg weak ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("agent-one-as") acquire acquire, align 4 +_Bool fi5b(int *i) { + int cmp = 0; + return __scoped_atomic_compare_exchange_n(i, &cmp, 1, 1, __ATOMIC_ACQUIRE, + __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_DEVICE); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi5c( +// CHECK: [[TMP0:%.*]] = cmpxchg weak ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("workgroup-one-as") acquire acquire, align 4 +_Bool fi5c(int *i) { + int cmp = 0; + return __scoped_atomic_compare_exchange_n( + i, &cmp, 1, 1, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, __MEMORY_SCOPE_WRKGRP); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi5d( +// CHECK: [[TMP0:%.*]] = cmpxchg weak ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("wavefront-one-as") acquire acquire, align 4 +_Bool fi5d(int *i) { + int cmp = 0; + return __scoped_atomic_compare_exchange_n( + i, &cmp, 1, 1, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, __MEMORY_SCOPE_WVFRNT); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi5e( +// CHECK: [[TMP0:%.*]] = cmpxchg weak ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("singlethread-one-as") acquire acquire, align 4 +_Bool fi5e(int *i) { + int cmp = 0; + return __scoped_atomic_compare_exchange_n( + i, &cmp, 1, 1, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, __MEMORY_SCOPE_SINGLE); +} + +// CHECK-LABEL: define hidden i32 @fi6a( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("one-as") monotonic, align 4 +int fi6a(int *c, int *d) { + int ret; + __scoped_atomic_exchange(c, d, &ret, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + return ret; +} + +// CHECK-LABEL: define hidden i32 @fi6b( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("agent-one-as") monotonic, align 4 +int fi6b(int *c, int *d) { + int ret; + __scoped_atomic_exchange(c, d, &ret, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + return ret; +} + +// CHECK-LABEL: define hidden i32 @fi6c( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("workgroup-one-as") monotonic, align 4 +int fi6c(int *c, int *d) { + int ret; + __scoped_atomic_exchange(c, d, &ret, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + return ret; +} + +// CHECK-LABEL: define hidden i32 @fi6d( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("wavefront-one-as") monotonic, align 4 +int fi6d(int *c, int *d) { + int ret; + __scoped_atomic_exchange(c, d, &ret, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + return ret; +} + +// CHECK-LABEL: define hidden i32 @fi6e( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("singlethread-one-as") monotonic, align 4 +int fi6e(int *c, int *d) { + int ret; + __scoped_atomic_exchange(c, d, &ret, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + return ret; +} + +// CHECK-LABEL: define hidden zeroext i1 @fi7a( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i8 [[VAL0:.+]] syncscope("one-as") monotonic, align 1 +_Bool fi7a(_Bool *c) { + return __scoped_atomic_exchange_n(c, 1, __ATOMIC_RELAXED, + __MEMORY_SCOPE_SYSTEM); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi7b( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i8 [[VAL0:.+]] syncscope("agent-one-as") monotonic, align 1 +_Bool fi7b(_Bool *c) { + return __scoped_atomic_exchange_n(c, 1, __ATOMIC_RELAXED, + __MEMORY_SCOPE_DEVICE); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi7c( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i8 [[VAL0:.+]] syncscope("workgroup-one-as") monotonic, align 1 +_Bool fi7c(_Bool *c) { + return __scoped_atomic_exchange_n(c, 1, __ATOMIC_RELAXED, + __MEMORY_SCOPE_WRKGRP); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi7d( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i8 [[VAL0:.+]] syncscope("wavefront-one-as") monotonic, align 1 +_Bool fi7d(_Bool *c) { + return __scoped_atomic_exchange_n(c, 1, __ATOMIC_RELAXED, + __MEMORY_SCOPE_WVFRNT); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi7e( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i8 [[VAL0:.+]] syncscope("singlethread-one-as") monotonic, align 1 +_Bool fi7e(_Bool *c) { + return __scoped_atomic_exchange_n(c, 1, __ATOMIC_RELAXED, + __MEMORY_SCOPE_SINGLE); +} diff --git a/clang/test/Preprocessor/init-aarch64.c b/clang/test/Preprocessor/init-aarch64.c index 2b7cc57f230333..b0333b3f023890 100644 --- a/clang/test/Preprocessor/init-aarch64.c +++ b/clang/test/Preprocessor/init-aarch64.c @@ -217,6 +217,11 @@ // AARCH64-NEXT: #define __LONG_MAX__ 9223372036854775807L // AARCH64-NEXT: #define __LONG_WIDTH__ 64 // AARCH64-NEXT: #define __LP64__ 1 +// AARCH64-NEXT: #define __MEMORY_SCOPE_DEVICE 1 +// AARCH64-NEXT: #define __MEMORY_SCOPE_SINGLE 4 +// AARCH64-NEXT: #define __MEMORY_SCOPE_SYSTEM 0 +// AARCH64-NEXT: #define __MEMORY_SCOPE_WRKGRP 2 +// AARCH64-NEXT: #define __MEMORY_SCOPE_WVFRNT 3 // AARCH64-NEXT: #define __NO_INLINE__ 1 // AARCH64-NEXT: #define __NO_MATH_ERRNO__ 1 // AARCH64-NEXT: #define __OBJC_BOOL_IS_BOOL 0 diff --git a/clang/test/Preprocessor/init-loongarch.c b/clang/test/Preprocessor/init-loongarch.c index e235a728302153..10a4cc7e66e830 100644 --- a/clang/test/Preprocessor/init-loongarch.c +++ b/clang/test/Preprocessor/init-loongarch.c @@ -177,6 +177,11 @@ // LA32: #define __LONG_LONG_MAX__ 9223372036854775807LL // LA32: #define __LONG_MAX__ 2147483647L // LA32: #define __LONG_WIDTH__ 32 +// LA32: #define __MEMORY_SCOPE_DEVICE 1 +// LA32: #define __MEMORY_SCOPE_SINGLE 4 +// LA32: #define __MEMORY_SCOPE_SYSTEM 0 +// LA32: #define __MEMORY_SCOPE_WRKGRP 2 +// LA32: #define __MEMORY_SCOPE_WVFRNT 3 // LA32: #define __NO_INLINE__ 1 // LA32: #define __NO_MATH_ERRNO__ 1 // LA32: #define __OBJC_BOOL_IS_BOOL 0 @@ -494,6 +499,11 @@ // LA64: #define __LONG_MAX__ 9223372036854775807L // LA64: #define __LONG_WIDTH__ 64 // LA64: #define __LP64__ 1 +// LA64: #define __MEMORY_SCOPE_DEVICE 1 +// LA64: #define __MEMORY_SCOPE_SINGLE 4 +// LA64: #define __MEMORY_SCOPE_SYSTEM 0 +// LA64: #define __MEMORY_SCOPE_WRKGRP 2 +// LA64: #define __MEMORY_SCOPE_WVFRNT 3 // LA64: #define __NO_INLINE__ 1 // LA64: #define __NO_MATH_ERRNO__ 1 // LA64: #define __OBJC_BOOL_IS_BOOL 0 diff --git a/clang/test/Preprocessor/init.c b/clang/test/Preprocessor/init.c index a0a2879cb58c7f..c3dbd94b2f741b 100644 --- a/clang/test/Preprocessor/init.c +++ b/clang/test/Preprocessor/init.c @@ -1742,6 +1742,11 @@ // WEBASSEMBLY64-NEXT:#define __LONG_MAX__ 9223372036854775807L // WEBASSEMBLY64-NEXT:#define __LONG_WIDTH__ 64 // WEBASSEMBLY64-NEXT:#define __LP64__ 1 +// WEBASSEMBLY-NEXT:#define __MEMORY_SCOPE_DEVICE 1 +// WEBASSEMBLY-NEXT:#define __MEMORY_SCOPE_SINGLE 4 +// WEBASSEMBLY-NEXT:#define __MEMORY_SCOPE_SYSTEM 0 +// WEBASSEMBLY-NEXT:#define __MEMORY_SCOPE_WRKGRP 2 +// WEBASSEMBLY-NEXT:#define __MEMORY_SCOPE_WVFRNT 3 // WEBASSEMBLY-NEXT:#define __NO_INLINE__ 1 // WEBASSEMBLY-NEXT:#define __NO_MATH_ERRNO__ 1 // WEBASSEMBLY-NEXT:#define __OBJC_BOOL_IS_BOOL 0 @@ -2057,6 +2062,11 @@ // AVR:#define __LDBL_MIN__ 1.17549435e-38L // AVR:#define __LONG_LONG_MAX__ 9223372036854775807LL // AVR:#define __LONG_MAX__ 2147483647L +// AVR:#define __MEMORY_SCOPE_DEVICE 1 +// AVR:#define __MEMORY_SCOPE_SINGLE 4 +// AVR:#define __MEMORY_SCOPE_SYSTEM 0 +// AVR:#define __MEMORY_SCOPE_WRKGRP 2 +// AVR:#define __MEMORY_SCOPE_WVFRNT 3 // AVR:#define __NO_INLINE__ 1 // AVR:#define __ORDER_BIG_ENDIAN__ 4321 // AVR:#define __ORDER_LITTLE_ENDIAN__ 1234 @@ -2348,6 +2358,11 @@ // RISCV32: #define __LITTLE_ENDIAN__ 1 // RISCV32: #define __LONG_LONG_MAX__ 9223372036854775807LL // RISCV32: #define __LONG_MAX__ 2147483647L +// RISCV32: #define __MEMORY_SCOPE_DEVICE 1 +// RISCV32: #define __MEMORY_SCOPE_SINGLE 4 +// RISCV32: #define __MEMORY_SCOPE_SYSTEM 0 +// RISCV32: #define __MEMORY_SCOPE_WRKGRP 2 +// RISCV32: #define __MEMORY_SCOPE_WVFRNT 3 // RISCV32: #define __NO_INLINE__ 1 // RISCV32: #define __POINTER_WIDTH__ 32 // RISCV32: #define __PRAGMA_REDEFINE_EXTNAME 1 @@ -2555,6 +2570,11 @@ // RISCV64: #define __LONG_LONG_MAX__ 9223372036854775807LL // RISCV64: #define __LONG_MAX__ 9223372036854775807L // RISCV64: #define __LP64__ 1 +// RISCV64: #define __MEMORY_SCOPE_DEVICE 1 +// RISCV64: #define __MEMORY_SCOPE_SINGLE 4 +// RISCV64: #define __MEMORY_SCOPE_SYSTEM 0 +// RISCV64: #define __MEMORY_SCOPE_WRKGRP 2 +// RISCV64: #define __MEMORY_SCOPE_WVFRNT 3 // RISCV64: #define __NO_INLINE__ 1 // RISCV64: #define __POINTER_WIDTH__ 64 // RISCV64: #define __PRAGMA_REDEFINE_EXTNAME 1 diff --git a/clang/test/Sema/scoped-atomic-ops.c b/clang/test/Sema/scoped-atomic-ops.c new file mode 100644 index 00000000000000..59e638c646664c --- /dev/null +++ b/clang/test/Sema/scoped-atomic-ops.c @@ -0,0 +1,101 @@ +// RUN: %clang_cc1 -x c -triple=amdgcn-amd-amdhsa -verify -fsyntax-only %s +// RUN: %clang_cc1 -x c -triple=x86_64-pc-linux-gnu -verify -fsyntax-only %s + +int fi1a(int *i) { + int v; + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + return v; +} + +int fi1b(int *i) { + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 3, have 2}} + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + return *i; +} + +int fi2a(int *i) { + int v; + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + return v; +} + +void fi2b(int *i) { + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); +} + +void fi3a(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); +} + +void fi3b(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *b = __scoped_atomic_fetch_sub(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *c = __scoped_atomic_fetch_and(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *d = __scoped_atomic_fetch_or(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *e = __scoped_atomic_fetch_xor(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *f = __scoped_atomic_fetch_nand(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *g = __scoped_atomic_fetch_min(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *h = __scoped_atomic_fetch_max(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} +} + +void fi3c(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} +} + +void fi3d(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} +} + +int fi4a(int *i) { + int cmp = 0; + int desired = 1; + return __scoped_atomic_compare_exchange(i, &cmp, &desired, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_SYSTEM); +} + +int fi5a(int *i) { + int cmp = 0; + return __scoped_atomic_compare_exchange_n(i, &cmp, 1, 1, __ATOMIC_ACQUIRE, + __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_SYSTEM); +} + +int fi6a(int *c, int *d) { + int ret; + __scoped_atomic_exchange(c, d, &ret, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + return ret; +} + +int fi7a(_Bool *c) { + return __scoped_atomic_exchange_n(c, 1, __ATOMIC_RELAXED, + __MEMORY_SCOPE_SYSTEM); +}