diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp index f319b176513f8..1bdab1563dda2 100644 --- a/clang/lib/CodeGen/CGExprScalar.cpp +++ b/clang/lib/CodeGen/CGExprScalar.cpp @@ -3847,7 +3847,16 @@ LValue ScalarExprEmitter::EmitCompoundAssignLValue( llvm::PHINode *atomicPHI = nullptr; if (const AtomicType *atomicTy = LHSTy->getAs()) { QualType type = atomicTy->getValueType(); - if (!type->isBooleanType() && type->isIntegerType() && + const bool isFloat = type->isFloatingType(); + const bool isInteger = type->isIntegerType(); + + bool isPowerOfTwo = false; + if (isFloat || isInteger) { + llvm::Type *IRTy = CGF.ConvertType(type); + uint64_t StoreBits = CGF.CGM.getDataLayout().getTypeStoreSizeInBits(IRTy); + isPowerOfTwo = llvm::isPowerOf2_64(StoreBits); + } + if (!type->isBooleanType() && (isInteger || (isPowerOfTwo && isFloat)) && !(type->isUnsignedIntegerType() && CGF.SanOpts.has(SanitizerKind::UnsignedIntegerOverflow)) && CGF.getLangOpts().getSignedOverflowBehavior() != @@ -3862,12 +3871,14 @@ LValue ScalarExprEmitter::EmitCompoundAssignLValue( case BO_ShrAssign: break; case BO_AddAssign: - AtomicOp = llvm::AtomicRMWInst::Add; - Op = llvm::Instruction::Add; + AtomicOp = + isFloat ? llvm::AtomicRMWInst::FAdd : llvm::AtomicRMWInst::Add; + Op = isFloat ? llvm::Instruction::FAdd : llvm::Instruction::Add; break; case BO_SubAssign: - AtomicOp = llvm::AtomicRMWInst::Sub; - Op = llvm::Instruction::Sub; + AtomicOp = + isFloat ? llvm::AtomicRMWInst::FSub : llvm::AtomicRMWInst::Sub; + Op = isFloat ? llvm::Instruction::FSub : llvm::Instruction::Sub; break; case BO_AndAssign: AtomicOp = llvm::AtomicRMWInst::And; diff --git a/clang/test/CodeGen/aarch64-lsfe-atomics.c b/clang/test/CodeGen/aarch64-lsfe-atomics.c new file mode 100644 index 0000000000000..428d038df198b --- /dev/null +++ b/clang/test/CodeGen/aarch64-lsfe-atomics.c @@ -0,0 +1,123 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6 +// RUN: %clang_cc1 %s -emit-llvm -o - -triple=aarch64-linux-gnu | FileCheck %s --check-prefix=CHECK-LLVM + +_Atomic(float) f; +_Atomic(double) d; +_Atomic(__bf16) bf; +_Atomic(_Float16) h; + +// CHECK-LLVM-LABEL: define dso_local void @test_float_add( +// CHECK-LLVM-SAME: float noundef [[VAL:%.*]]) #[[ATTR0:[0-9]+]] { +// CHECK-LLVM-NEXT: [[ENTRY:.*:]] +// CHECK-LLVM-NEXT: [[VAL_ADDR:%.*]] = alloca float, align 4 +// CHECK-LLVM-NEXT: store float [[VAL]], ptr [[VAL_ADDR]], align 4 +// CHECK-LLVM-NEXT: [[TMP0:%.*]] = load float, ptr [[VAL_ADDR]], align 4 +// CHECK-LLVM-NEXT: [[TMP1:%.*]] = atomicrmw fadd ptr @f, float [[TMP0]] seq_cst, align 4 +// CHECK-LLVM-NEXT: [[TMP2:%.*]] = fadd float [[TMP1]], [[TMP0]] +// CHECK-LLVM-NEXT: ret void +// +void test_float_add(float val) { + f += val; +} + +// CHECK-LLVM-LABEL: define dso_local void @test_double_add( +// CHECK-LLVM-SAME: double noundef [[VAL:%.*]]) #[[ATTR0]] { +// CHECK-LLVM-NEXT: [[ENTRY:.*:]] +// CHECK-LLVM-NEXT: [[VAL_ADDR:%.*]] = alloca double, align 8 +// CHECK-LLVM-NEXT: store double [[VAL]], ptr [[VAL_ADDR]], align 8 +// CHECK-LLVM-NEXT: [[TMP0:%.*]] = load double, ptr [[VAL_ADDR]], align 8 +// CHECK-LLVM-NEXT: [[TMP1:%.*]] = atomicrmw fadd ptr @d, double [[TMP0]] seq_cst, align 8 +// CHECK-LLVM-NEXT: [[TMP2:%.*]] = fadd double [[TMP1]], [[TMP0]] +// CHECK-LLVM-NEXT: ret void +// +void test_double_add(double val) { + d += val; +} + +// CHECK-LLVM-LABEL: define dso_local void @test_bf16_add( +// CHECK-LLVM-SAME: bfloat noundef [[VAL:%.*]]) #[[ATTR0]] { +// CHECK-LLVM-NEXT: [[ENTRY:.*:]] +// CHECK-LLVM-NEXT: [[VAL_ADDR:%.*]] = alloca bfloat, align 2 +// CHECK-LLVM-NEXT: store bfloat [[VAL]], ptr [[VAL_ADDR]], align 2 +// CHECK-LLVM-NEXT: [[TMP0:%.*]] = load bfloat, ptr [[VAL_ADDR]], align 2 +// CHECK-LLVM-NEXT: [[EXT:%.*]] = fpext bfloat [[TMP0]] to float +// CHECK-LLVM-NEXT: [[CONV:%.*]] = fptrunc float [[EXT]] to bfloat +// CHECK-LLVM-NEXT: [[TMP1:%.*]] = atomicrmw fadd ptr @bf, bfloat [[CONV]] seq_cst, align 2 +// CHECK-LLVM-NEXT: [[TMP2:%.*]] = fadd bfloat [[TMP1]], [[CONV]] +// CHECK-LLVM-NEXT: ret void +// +void test_bf16_add(__bf16 val){ + bf += val; +} + +// CHECK-LLVM-LABEL: define dso_local void @test_f16_add( +// CHECK-LLVM-SAME: half noundef [[VAL:%.*]]) #[[ATTR0]] { +// CHECK-LLVM-NEXT: [[ENTRY:.*:]] +// CHECK-LLVM-NEXT: [[VAL_ADDR:%.*]] = alloca half, align 2 +// CHECK-LLVM-NEXT: store half [[VAL]], ptr [[VAL_ADDR]], align 2 +// CHECK-LLVM-NEXT: [[TMP0:%.*]] = load half, ptr [[VAL_ADDR]], align 2 +// CHECK-LLVM-NEXT: [[TMP1:%.*]] = atomicrmw fadd ptr @h, half [[TMP0]] seq_cst, align 2 +// CHECK-LLVM-NEXT: [[TMP2:%.*]] = fadd half [[TMP1]], [[TMP0]] +// CHECK-LLVM-NEXT: ret void +// +void test_f16_add(_Float16 val){ + h += val; +} + +// CHECK-LLVM-LABEL: define dso_local void @test_float_sub( +// CHECK-LLVM-SAME: float noundef [[VAL:%.*]]) #[[ATTR0]] { +// CHECK-LLVM-NEXT: [[ENTRY:.*:]] +// CHECK-LLVM-NEXT: [[VAL_ADDR:%.*]] = alloca float, align 4 +// CHECK-LLVM-NEXT: store float [[VAL]], ptr [[VAL_ADDR]], align 4 +// CHECK-LLVM-NEXT: [[TMP0:%.*]] = load float, ptr [[VAL_ADDR]], align 4 +// CHECK-LLVM-NEXT: [[TMP1:%.*]] = atomicrmw fsub ptr @f, float [[TMP0]] seq_cst, align 4 +// CHECK-LLVM-NEXT: [[TMP2:%.*]] = fsub float [[TMP1]], [[TMP0]] +// CHECK-LLVM-NEXT: ret void +// +void test_float_sub(float val) { + f -= val; +} + +// CHECK-LLVM-LABEL: define dso_local void @test_double_sub( +// CHECK-LLVM-SAME: double noundef [[VAL:%.*]]) #[[ATTR0]] { +// CHECK-LLVM-NEXT: [[ENTRY:.*:]] +// CHECK-LLVM-NEXT: [[VAL_ADDR:%.*]] = alloca double, align 8 +// CHECK-LLVM-NEXT: store double [[VAL]], ptr [[VAL_ADDR]], align 8 +// CHECK-LLVM-NEXT: [[TMP0:%.*]] = load double, ptr [[VAL_ADDR]], align 8 +// CHECK-LLVM-NEXT: [[TMP1:%.*]] = atomicrmw fsub ptr @d, double [[TMP0]] seq_cst, align 8 +// CHECK-LLVM-NEXT: [[TMP2:%.*]] = fsub double [[TMP1]], [[TMP0]] +// CHECK-LLVM-NEXT: ret void +// +void test_double_sub(double val){ + d -= val; +} + +// CHECK-LLVM-LABEL: define dso_local void @test_bf16_sub( +// CHECK-LLVM-SAME: bfloat noundef [[VAL:%.*]]) #[[ATTR0]] { +// CHECK-LLVM-NEXT: [[ENTRY:.*:]] +// CHECK-LLVM-NEXT: [[VAL_ADDR:%.*]] = alloca bfloat, align 2 +// CHECK-LLVM-NEXT: store bfloat [[VAL]], ptr [[VAL_ADDR]], align 2 +// CHECK-LLVM-NEXT: [[TMP0:%.*]] = load bfloat, ptr [[VAL_ADDR]], align 2 +// CHECK-LLVM-NEXT: [[EXT:%.*]] = fpext bfloat [[TMP0]] to float +// CHECK-LLVM-NEXT: [[CONV:%.*]] = fptrunc float [[EXT]] to bfloat +// CHECK-LLVM-NEXT: [[TMP1:%.*]] = atomicrmw fsub ptr @bf, bfloat [[CONV]] seq_cst, align 2 +// CHECK-LLVM-NEXT: [[TMP2:%.*]] = fsub bfloat [[TMP1]], [[CONV]] +// CHECK-LLVM-NEXT: ret void +// +void test_bf16_sub(__bf16 val){ + bf -= val; +} + +// CHECK-LLVM-LABEL: define dso_local void @test_f16_sub( +// CHECK-LLVM-SAME: half noundef [[VAL:%.*]]) #[[ATTR0]] { +// CHECK-LLVM-NEXT: [[ENTRY:.*:]] +// CHECK-LLVM-NEXT: [[VAL_ADDR:%.*]] = alloca half, align 2 +// CHECK-LLVM-NEXT: store half [[VAL]], ptr [[VAL_ADDR]], align 2 +// CHECK-LLVM-NEXT: [[TMP0:%.*]] = load half, ptr [[VAL_ADDR]], align 2 +// CHECK-LLVM-NEXT: [[TMP1:%.*]] = atomicrmw fsub ptr @h, half [[TMP0]] seq_cst, align 2 +// CHECK-LLVM-NEXT: [[TMP2:%.*]] = fsub half [[TMP1]], [[TMP0]] +// CHECK-LLVM-NEXT: ret void +// +void test_f16_sub(_Float16 val){ + h -= val; +}