diff --git a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp index 8d660a0a2c721..65812318cc1f9 100644 --- a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp @@ -1948,6 +1948,95 @@ static bool isIntegerVectorBinOp(mlir::Type ty) { return vecTy && mlir::isa(vecTy.getElementType()); } +// Construct a cir.fmuladd op to represent a fused mul-add of `mulOp` and +// `addend`. Use negMul and negAdd to negate the first operand of the mul or +// the addend respectively. This allows fmuladd to represent a*b-c, or c-a*b. +// Patterns in LLVM should catch the negated forms and translate them to +// efficient operations. +static mlir::Value buildFMulAdd(mlir::Location addLoc, cir::FMulOp mulOp, + mlir::Value addend, CIRGenBuilderTy &builder, + bool negMul, bool negAdd) { + mlir::Location loc = builder.getFusedLoc({mulOp.getLoc(), addLoc}); + mlir::Value mulOp0 = mulOp.getLhs(); + mlir::Value mulOp1 = mulOp.getRhs(); + if (negMul) + mulOp0 = builder.createFNeg(loc, mulOp0); + if (negAdd) + addend = builder.createFNeg(loc, addend); + + // Carry the mul's fenv attribute so a constrained fmul yields a constrained + // fmuladd; the builder is under the add's FP options, not the mul's. + mlir::Value fmuladd = + cir::FMulAddOp::create(builder, loc, addend.getType(), mulOp0, mulOp1, + addend, mulOp.getFenvAttr()); + mulOp.erase(); + return fmuladd; +} + +// Check whether it would be legal to emit a cir.fmuladd op to represent op +// and if so, build it. +// +// Checks that (a) the operation is fusable, and (b) -ffp-contract=on. +// Does NOT check the type of the operation - it's assumed that this function +// will be called from contexts where it's known that the type is contractable. +static mlir::Value tryEmitFMulAdd(mlir::Location loc, const BinOpInfo &op, + CIRGenBuilderTy &builder, + bool isSub = false) { + assert((op.opcode == BO_Add || op.opcode == BO_AddAssign || + op.opcode == BO_Sub || op.opcode == BO_SubAssign) && + "Only fadd/fsub can be the root of an fmuladd."); + + // Check whether this op is fusable, i.e. -ffp-contract=on. -ffp-contract=fast + // needs fast-math flags on the fmul/fadd, which CIR does not model yet, so it + // fuses nowhere for now. + assert(!cir::MissingFeatures::fastMathFlags()); + if (!op.fpFeatures.allowFPContractWithinStatement()) + return nullptr; + + mlir::Value lhs = op.lhs; + mlir::Value rhs = op.rhs; + + // Peek through fneg to look for fmul. Make sure the fneg has no other users, + // and that it is the only use of its operand. + bool negLHS = false; + if (auto lhsNeg = lhs.getDefiningOp()) { + if (lhsNeg.getResult().use_empty() && lhsNeg.getInput().hasOneUse()) { + lhs = lhsNeg.getInput(); + negLHS = true; + } + } + + bool negRHS = false; + if (auto rhsNeg = rhs.getDefiningOp()) { + if (rhsNeg.getResult().use_empty() && rhsNeg.getInput().hasOneUse()) { + rhs = rhsNeg.getInput(); + negRHS = true; + } + } + + // We have a potentially fusable op. Look for a mul on one of the operands. + // Also make sure that the mul result isn't used directly. In that case, + // there's no point creating a muladd operation. + if (auto lhsMul = lhs.getDefiningOp()) { + if (lhsMul.getResult().use_empty() || negLHS) { + // If we looked through fneg, erase it. + if (negLHS) + op.lhs.getDefiningOp().erase(); + return buildFMulAdd(loc, lhsMul, op.rhs, builder, negLHS, isSub); + } + } + if (auto rhsMul = rhs.getDefiningOp()) { + if (rhsMul.getResult().use_empty() || negRHS) { + // If we looked through fneg, erase it. + if (negRHS) + op.rhs.getDefiningOp().erase(); + return buildFMulAdd(loc, rhsMul, op.lhs, builder, isSub ^ negRHS, false); + } + } + + return nullptr; +} + mlir::Value ScalarExprEmitter::emitMul(const BinOpInfo &ops) { const mlir::Location loc = cgf.getLoc(ops.loc); if (!isIntegerVectorBinOp(ops.lhs.getType()) && @@ -2047,6 +2136,9 @@ mlir::Value ScalarExprEmitter::emitAdd(const BinOpInfo &ops) { if (cir::isFPOrVectorOfFPType(ops.lhs.getType())) { CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ops.fpFeatures); + // Try to form an fmuladd. + if (mlir::Value fmuladd = tryEmitFMulAdd(loc, ops, builder)) + return fmuladd; return builder.createFAdd(loc, ops.lhs, ops.rhs); } @@ -2095,6 +2187,10 @@ mlir::Value ScalarExprEmitter::emitSub(const BinOpInfo &ops) { if (cir::isFPOrVectorOfFPType(ops.lhs.getType())) { CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ops.fpFeatures); + // Try to form an fmuladd. + if (mlir::Value fmuladd = + tryEmitFMulAdd(loc, ops, builder, /*isSub=*/true)) + return fmuladd; return builder.createFSub(loc, ops.lhs, ops.rhs); } diff --git a/clang/test/CIR/CodeGen/fp-contract-on-pragma.cpp b/clang/test/CIR/CodeGen/fp-contract-on-pragma.cpp new file mode 100644 index 0000000000000..1996896edd26b --- /dev/null +++ b/clang/test/CIR/CodeGen/fp-contract-on-pragma.cpp @@ -0,0 +1,122 @@ +// ClangIR port of clang/test/CodeGen/fp-contract-on-pragma.cpp. +// The CIR-lowered and classic CodeGen LLVM IR match here, so both feed LLVM. + +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++11 -Wno-unused-value -fclangir -emit-cir %s -o %t.cir +// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++11 -Wno-unused-value -fclangir -emit-llvm %s -o %t.ll +// RUN: FileCheck --input-file=%t.ll %s -check-prefix=LLVM +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++11 -Wno-unused-value -emit-llvm %s -o %t-ogcg.ll +// RUN: FileCheck --input-file=%t-ogcg.ll %s -check-prefix=LLVM + +// Is FP_CONTRACT honored in a simple case? +float fp_contract_1(float a, float b, float c) { +#pragma clang fp contract(on) + return a * b + c; +} +// CIR-LABEL: cir.func {{.*}}@_Z13fp_contract_1fff +// CIR: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float +// CIR-NOT: cir.fmul +// LLVM-LABEL: @_Z13fp_contract_1fff +// LLVM: call float @llvm.fmuladd.f32 + +// Is FP_CONTRACT state cleared on exiting compound statements? +float fp_contract_2(float a, float b, float c) { + { +#pragma clang fp contract(on) + } + return a * b + c; +} +// CIR-LABEL: cir.func {{.*}}@_Z13fp_contract_2fff +// CIR: cir.fmul %{{.*}}, %{{.*}} : !cir.float +// CIR: cir.fadd %{{.*}}, %{{.*}} : !cir.float +// CIR-NOT: cir.fmuladd +// LLVM-LABEL: @_Z13fp_contract_2fff +// LLVM: %[[M:.*]] = fmul float +// LLVM: fadd float %[[M]], + +// Does FP_CONTRACT survive template instantiation? +class Foo {}; +Foo operator+(Foo, Foo); + +template +T template_muladd(T a, T b, T c) { +#pragma clang fp contract(on) + return a * b + c; +} +// CIR-LABEL: cir.func {{.*}}@_Z15template_muladdIfET_S0_S0_S0_ +// CIR: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float +// LLVM-LABEL: @_Z15template_muladdIfET_S0_S0_S0_ +// LLVM: call {{.*}}float @llvm.fmuladd.f32 + +// fp_contract_3 is just a caller; the fused op lives in the instantiated +// template_muladd checked above. It is emitted in a different order under the +// classic CodeGen path, so it carries no checks of its own here. +float fp_contract_3(float a, float b, float c) { + return template_muladd(a, b, c); +} + +template +class fp_contract_4 { + float method(float a, float b, float c) { +#pragma clang fp contract(on) + return a * b + c; + } +}; +template class fp_contract_4; +// CIR-LABEL: cir.func {{.*}}@_ZN13fp_contract_4IiE6methodEfff +// CIR: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float +// LLVM-LABEL: @_ZN13fp_contract_4IiE6methodEfff +// LLVM: call float @llvm.fmuladd.f32 + +// Check file-scoped FP_CONTRACT +#pragma clang fp contract(on) +float fp_contract_5(float a, float b, float c) { + return a * b + c; +} +// CIR-LABEL: cir.func {{.*}}@_Z13fp_contract_5fff +// CIR: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float +// LLVM-LABEL: @_Z13fp_contract_5fff +// LLVM: call float @llvm.fmuladd.f32 + +#pragma clang fp contract(off) +float fp_contract_6(float a, float b, float c) { + return a * b + c; +} +// CIR-LABEL: cir.func {{.*}}@_Z13fp_contract_6fff +// CIR: cir.fmul %{{.*}}, %{{.*}} : !cir.float +// CIR: cir.fadd %{{.*}}, %{{.*}} : !cir.float +// CIR-NOT: cir.fmuladd +// LLVM-LABEL: @_Z13fp_contract_6fff +// LLVM: %[[M:.*]] = fmul float +// LLVM: fadd float %[[M]], + +// If the multiply has multiple uses, don't produce fmuladd. +// This used to assert (PR25719): +// https://llvm.org/bugs/show_bug.cgi?id=25719 +float fp_contract_7(float a, float b, float c) { +#pragma clang fp contract(on) + return (a = 2 * b) - c; +} +// CIR-LABEL: cir.func {{.*}}@_Z13fp_contract_7fff +// CIR: cir.fmul %{{.*}}, %{{.*}} : !cir.float +// CIR: cir.fsub %{{.*}}, %{{.*}} : !cir.float +// CIR-NOT: cir.fmuladd +// LLVM-LABEL: @_Z13fp_contract_7fff +// LLVM: %[[M:.*]] = fmul float +// LLVM: fsub float %[[M]], + +// contract(on) only fuses within a statement: a mul and add in separate +// statements are not contracted. +float fp_contract_8(float a, float b, float c) { +#pragma clang fp contract(on) + float t = a * b; + return t + c; +} +// CIR-LABEL: cir.func {{.*}}@_Z13fp_contract_8fff +// CIR: cir.fmul %{{.*}}, %{{.*}} : !cir.float +// CIR: cir.fadd %{{.*}}, %{{.*}} : !cir.float +// CIR-NOT: cir.fmuladd +// LLVM-LABEL: @_Z13fp_contract_8fff +// LLVM: fmul float +// LLVM: fadd float +// LLVM-NOT: call float @llvm.fmuladd.f32 diff --git a/clang/test/CIR/CodeGen/fp-contract-pragma.cpp b/clang/test/CIR/CodeGen/fp-contract-pragma.cpp new file mode 100644 index 0000000000000..f5f0c0884dc52 --- /dev/null +++ b/clang/test/CIR/CodeGen/fp-contract-pragma.cpp @@ -0,0 +1,215 @@ +// ClangIR port of clang/test/CodeGen/fp-contract-pragma.cpp. +// The CIR-lowered and classic CodeGen LLVM IR match here, so both feed LLVM. + +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++11 -Wno-unused-value -fclangir -emit-cir %s -o %t.cir +// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++11 -Wno-unused-value -fclangir -emit-llvm %s -o %t.ll +// RUN: FileCheck --input-file=%t.ll %s -check-prefix=LLVM +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++11 -Wno-unused-value -emit-llvm %s -o %t-ogcg.ll +// RUN: FileCheck --input-file=%t-ogcg.ll %s -check-prefix=LLVM + +// Is FP_CONTRACT honored in a simple case? +float fp_contract_1(float a, float b, float c) { + #pragma STDC FP_CONTRACT ON + return a * b + c; +} +// CIR-LABEL: cir.func {{.*}}@_Z13fp_contract_1fff +// CIR: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float +// CIR-NOT: cir.fmul +// LLVM-LABEL: @_Z13fp_contract_1fff +// LLVM: call float @llvm.fmuladd.f32 + +// Is FP_CONTRACT state cleared on exiting compound statements? +float fp_contract_2(float a, float b, float c) { + { + #pragma STDC FP_CONTRACT ON + } + return a * b + c; +} +// CIR-LABEL: cir.func {{.*}}@_Z13fp_contract_2fff +// CIR: cir.fmul %{{.*}}, %{{.*}} : !cir.float +// CIR: cir.fadd %{{.*}}, %{{.*}} : !cir.float +// CIR-NOT: cir.fmuladd +// LLVM-LABEL: @_Z13fp_contract_2fff +// LLVM: %[[M:.*]] = fmul float +// LLVM: fadd float %[[M]], + +// Does FP_CONTRACT survive template instantiation? +class Foo {}; +Foo operator+(Foo, Foo); + +template +T template_muladd(T a, T b, T c) { + #pragma STDC FP_CONTRACT ON + return a * b + c; +} +// The fmuladd is emitted in the instantiated template body. +// CIR-LABEL: cir.func {{.*}}@_Z15template_muladdIfET_S0_S0_S0_ +// CIR: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float +// LLVM-LABEL: @_Z15template_muladdIfET_S0_S0_S0_ +// LLVM: call {{.*}}float @llvm.fmuladd.f32 + +// fp_contract_3 is just a caller; the fused op lives in the instantiated +// template_muladd checked above. It is emitted in a different order under the +// classic CodeGen path, so it carries no checks of its own here. +float fp_contract_3(float a, float b, float c) { + return template_muladd(a, b, c); +} + +template class fp_contract_4 { + float method(float a, float b, float c) { + #pragma STDC FP_CONTRACT ON + return a * b + c; + } +}; +template class fp_contract_4; +// CIR-LABEL: cir.func {{.*}}@_ZN13fp_contract_4IiE6methodEfff +// CIR: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float +// LLVM-LABEL: @_ZN13fp_contract_4IiE6methodEfff +// LLVM: call float @llvm.fmuladd.f32 + +// Check file-scoped FP_CONTRACT +#pragma STDC FP_CONTRACT ON +float fp_contract_5(float a, float b, float c) { + return a * b + c; +} +// CIR-LABEL: cir.func {{.*}}@_Z13fp_contract_5fff +// CIR: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float +// LLVM-LABEL: @_Z13fp_contract_5fff +// LLVM: call float @llvm.fmuladd.f32 + +#pragma STDC FP_CONTRACT OFF +float fp_contract_6(float a, float b, float c) { + return a * b + c; +} +// CIR-LABEL: cir.func {{.*}}@_Z13fp_contract_6fff +// CIR: cir.fmul %{{.*}}, %{{.*}} : !cir.float +// CIR: cir.fadd %{{.*}}, %{{.*}} : !cir.float +// CIR-NOT: cir.fmuladd +// LLVM-LABEL: @_Z13fp_contract_6fff +// LLVM: %[[M:.*]] = fmul float +// LLVM: fadd float %[[M]], + +// If the multiply has multiple uses, don't produce fmuladd. +// This used to assert (PR25719): +// https://llvm.org/bugs/show_bug.cgi?id=25719 +float fp_contract_7(float a, float b, float c) { + #pragma STDC FP_CONTRACT ON + return (a = 2 * b) - c; +} +// CIR-LABEL: cir.func {{.*}}@_Z13fp_contract_7fff +// CIR: cir.fmul %{{.*}}, %{{.*}} : !cir.float +// CIR: cir.fsub %{{.*}}, %{{.*}} : !cir.float +// CIR-NOT: cir.fmuladd +// LLVM-LABEL: @_Z13fp_contract_7fff +// LLVM: %[[M:.*]] = fmul float +// LLVM: fsub float %[[M]], + +// a * b - c => fmuladd(a, b, -c) +float fp_contract_8(float a, float b, float c) { + #pragma STDC FP_CONTRACT ON + return a * b - c; +} +// CIR-LABEL: cir.func {{.*}}@_Z13fp_contract_8fff +// CIR: cir.fneg %{{.*}} : !cir.float +// CIR: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float +// LLVM-LABEL: @_Z13fp_contract_8fff +// LLVM: fneg float +// LLVM: call float @llvm.fmuladd.f32 + +// c - a * b => fmuladd(-a, b, c) (mul on the RHS of a subtraction) +float fp_contract_9(float a, float b, float c) { + #pragma STDC FP_CONTRACT ON + return c - a * b; +} +// CIR-LABEL: cir.func {{.*}}@_Z13fp_contract_9fff +// CIR: cir.fneg %{{.*}} : !cir.float +// CIR: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float +// LLVM-LABEL: @_Z13fp_contract_9fff +// LLVM: fneg float +// LLVM: call float @llvm.fmuladd.f32 + +// -(a * b) + c => fmuladd(-a, b, c) (peek through fneg on the LHS) +float fp_contract_10(float a, float b, float c) { + #pragma STDC FP_CONTRACT ON + return -(a * b) + c; +} +// CIR-LABEL: cir.func {{.*}}@_Z14fp_contract_10fff +// CIR: cir.fneg %{{.*}} : !cir.float +// CIR: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float +// CIR-NOT: cir.fadd +// LLVM-LABEL: @_Z14fp_contract_10fff +// LLVM: fneg float +// LLVM: call float @llvm.fmuladd.f32 + +// -(a * b) - c => fmuladd(-a, b, -c) (fneg both the mul operand and addend) +float fp_contract_11(float a, float b, float c) { + #pragma STDC FP_CONTRACT ON + return -(a * b) - c; +} +// CIR-LABEL: cir.func {{.*}}@_Z14fp_contract_11fff +// CIR: cir.fneg %{{.*}} : !cir.float +// CIR: cir.fneg %{{.*}} : !cir.float +// CIR: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float +// LLVM-LABEL: @_Z14fp_contract_11fff +// LLVM: fneg float +// LLVM: fneg float +// LLVM: call float @llvm.fmuladd.f32 + +// c + -(a * b) => fmuladd(-a, b, c) (peek through fneg on the RHS) +float fp_contract_12(float a, float b, float c) { + #pragma STDC FP_CONTRACT ON + return c + -(a * b); +} +// CIR-LABEL: cir.func {{.*}}@_Z14fp_contract_12fff +// CIR: cir.fneg %{{.*}} : !cir.float +// CIR: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float +// CIR-NOT: cir.fadd +// LLVM-LABEL: @_Z14fp_contract_12fff +// LLVM: fneg float +// LLVM: call float @llvm.fmuladd.f32 + +// c - -(a * b) => fmuladd(a, b, c) (the two negations cancel; no fneg) +float fp_contract_13(float a, float b, float c) { + #pragma STDC FP_CONTRACT ON + return c - -(a * b); +} +// CIR-LABEL: cir.func {{.*}}@_Z14fp_contract_13fff +// CIR-NOT: cir.fneg +// CIR: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float +// LLVM-LABEL: @_Z14fp_contract_13fff +// LLVM-NOT: fneg float +// LLVM: call float @llvm.fmuladd.f32 + +// Mul reused by the assignment, so no fusion. At -O0 the negation stays an +// fneg+fadd instead of the fsub the -O3 original test expects. +float fp_contract_14(float a, float b, float c) { + #pragma STDC FP_CONTRACT ON + float d; + return (d = -(a * b)) + c; +} +// CIR-LABEL: cir.func {{.*}}@_Z14fp_contract_14fff +// CIR: cir.fmul %{{.*}}, %{{.*}} : !cir.float +// CIR: cir.fneg %{{.*}} : !cir.float +// CIR: cir.fadd %{{.*}}, %{{.*}} : !cir.float +// CIR-NOT: cir.fmuladd +// LLVM-LABEL: @_Z14fp_contract_14fff +// LLVM: fmul float +// LLVM: fneg float +// LLVM: fadd float + +// Same as above, with the negation applied to the assignment result. +float fp_contract_15(float a, float b, float c) { + #pragma STDC FP_CONTRACT ON + float d; + return -(d = (a * b)) + c; +} +// CIR-LABEL: cir.func {{.*}}@_Z14fp_contract_15fff +// CIR: cir.fmul %{{.*}}, %{{.*}} : !cir.float +// CIR: cir.fneg %{{.*}} : !cir.float +// CIR: cir.fadd %{{.*}}, %{{.*}} : !cir.float +// CIR-NOT: cir.fmuladd +// LLVM-LABEL: @_Z14fp_contract_15fff +// LLVM: fmul float +// LLVM: fneg float +// LLVM: fadd float diff --git a/clang/test/CIR/CodeGen/fp-contract.c b/clang/test/CIR/CodeGen/fp-contract.c new file mode 100644 index 0000000000000..6f4a8e3011b31 --- /dev/null +++ b/clang/test/CIR/CodeGen/fp-contract.c @@ -0,0 +1,143 @@ +// Test that -ffp-contract=on fuses a*b+c / a*b-c into cir.fmuladd and that +// -ffp-contract=off does not. The CIR-lowered and classic CodeGen LLVM IR +// match here, so both feed the LLVM-* prefixes. +// +// TODO: drop -fno-clangir-call-conv-lowering once x86_64 calling-convention +// lowering supports vector types (needed by fmuladd_vec). + +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -fno-clangir-call-conv-lowering -ffp-contract=on -emit-cir %s -o %t.cir +// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR-ON +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -fno-clangir-call-conv-lowering -ffp-contract=off -emit-cir %s -o %t-off.cir +// RUN: FileCheck --input-file=%t-off.cir %s -check-prefix=CIR-OFF + +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -fno-clangir-call-conv-lowering -ffp-contract=on -emit-llvm %s -o %t.ll +// RUN: FileCheck --input-file=%t.ll %s -check-prefix=LLVM-ON +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -fno-clangir-call-conv-lowering -ffp-contract=off -emit-llvm %s -o %t-off.ll +// RUN: FileCheck --input-file=%t-off.ll %s -check-prefix=LLVM-OFF + +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -ffp-contract=on -emit-llvm %s -o %t.ll +// RUN: FileCheck --input-file=%t.ll %s -check-prefix=LLVM-ON +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -ffp-contract=off -emit-llvm %s -o %t-off.ll +// RUN: FileCheck --input-file=%t-off.ll %s -check-prefix=LLVM-OFF + +// Under strict FP the fused op carries an fenv attribute and lowers to the +// constrained fmuladd intrinsic. +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -fno-clangir-call-conv-lowering -ffp-contract=on -fexperimental-strict-floating-point -ffp-exception-behavior=strict -emit-cir %s -o %t-strict.cir +// RUN: FileCheck --input-file=%t-strict.cir %s -check-prefix=CIR-STRICT +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -fno-clangir-call-conv-lowering -ffp-contract=on -fexperimental-strict-floating-point -ffp-exception-behavior=strict -emit-llvm %s -o %t-strict.ll +// RUN: FileCheck --input-file=%t-strict.ll %s -check-prefix=LLVM-STRICT +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -ffp-contract=on -fexperimental-strict-floating-point -ffp-exception-behavior=strict -emit-llvm %s -o %t-strict-ogcg.ll +// RUN: FileCheck --input-file=%t-strict-ogcg.ll %s -check-prefix=LLVM-STRICT + +// a * b + c => fmuladd(a, b, c) +float fmuladd_add(float a, float b, float c) { + return a * b + c; +} +// CIR-ON-LABEL: cir.func {{.*}}@fmuladd_add +// CIR-ON: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float +// CIR-ON-NOT: cir.fmul + +// CIR-OFF-LABEL: cir.func {{.*}}@fmuladd_add +// CIR-OFF: cir.fmul %{{.*}}, %{{.*}} : !cir.float +// CIR-OFF: cir.fadd %{{.*}}, %{{.*}} : !cir.float +// CIR-OFF-NOT: cir.fmuladd + +// LLVM-ON-LABEL: @fmuladd_add +// LLVM-ON: call float @llvm.fmuladd.f32 +// LLVM-OFF-LABEL: @fmuladd_add +// LLVM-OFF: fmul float +// LLVM-OFF: fadd float + + +// c + a * b => fmuladd(a, b, c) (mul on the RHS) +float fmuladd_add_rhs(float a, float b, float c) { + return c + a * b; +} +// CIR-ON-LABEL: cir.func {{.*}}@fmuladd_add_rhs +// CIR-ON: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float + +// LLVM-ON-LABEL: @fmuladd_add_rhs +// LLVM-ON: call float @llvm.fmuladd.f32 + +// a * b - c => fmuladd(a, b, -c) +float fmuladd_sub(float a, float b, float c) { + return a * b - c; +} +// CIR-ON-LABEL: cir.func {{.*}}@fmuladd_sub +// CIR-ON: %[[NEG:.*]] = cir.fneg %{{.*}} : !cir.float +// CIR-ON: cir.fmuladd %{{.*}}, %{{.*}}, %[[NEG]] : !cir.float + +// LLVM-ON-LABEL: @fmuladd_sub +// LLVM-ON: %[[NEG:.*]] = fneg float +// LLVM-ON: call float @llvm.fmuladd.f32(float %{{.*}}, float %{{.*}}, float %[[NEG]]) + +// If the mul result is used elsewhere, it must NOT be fused. +float no_fmuladd_reused_mul(float a, float b, float c, float *p) { + float m = a * b; + *p = m; + return m + c; +} +// CIR-ON-LABEL: cir.func {{.*}}@no_fmuladd_reused_mul +// CIR-ON: cir.fmul %{{.*}}, %{{.*}} : !cir.float +// CIR-ON: cir.fadd %{{.*}}, %{{.*}} : !cir.float +// CIR-ON-NOT: cir.fmuladd + +// LLVM-ON-LABEL: @no_fmuladd_reused_mul +// LLVM-ON: fmul float +// LLVM-ON: fadd float +// LLVM-ON-NOT: call float @llvm.fmuladd.f32 + +// Vector: a * b + c => fmuladd on the vector type. +typedef float float4 __attribute__((ext_vector_type(4))); +float4 fmuladd_vec(float4 a, float4 b, float4 c) { + return a * b + c; +} +// CIR-ON-LABEL: cir.func {{.*}}@fmuladd_vec +// CIR-ON: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.vector<4 x !cir.float> + +// LLVM-ON-LABEL: @fmuladd_vec +// LLVM-ON: call <4 x float> @llvm.fmuladd.v4f32 + +// Strict FP: fused op carries an fenv attr, lowering to the constrained +// fmuladd intrinsic. +float fmuladd_strict(float a, float b, float c) { + return a * b + c; +} +// CIR-STRICT-LABEL: cir.func {{.*}}@fmuladd_strict +// CIR-STRICT: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float {fenv = #cir.fenv<{{.*}}strict_except = true>} +// LLVM-STRICT-LABEL: @fmuladd_strict +// LLVM-STRICT: call float @llvm.experimental.constrained.fmuladd.f32 + +// Strict FP with a negated addend: the fmuladd carries the mul's fenv while +// the fneg (which takes none) lowers to a plain fneg. +float fmuladd_sub_strict(float a, float b, float c) { + return a * b - c; +} +// CIR-STRICT-LABEL: cir.func {{.*}}@fmuladd_sub_strict +// CIR-STRICT: cir.fneg %{{.*}} : !cir.float +// CIR-STRICT: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float {fenv = #cir.fenv<{{.*}}strict_except = true>} +// LLVM-STRICT-LABEL: @fmuladd_sub_strict +// LLVM-STRICT: fneg float +// LLVM-STRICT: call float @llvm.experimental.constrained.fmuladd.f32 + +// Compound assignment routes through emitAdd/emitSub, so += and -= fuse too. +float fmuladd_add_assign(float x, float a, float b) { + x += a * b; + return x; +} +// CIR-ON-LABEL: cir.func {{.*}}@fmuladd_add_assign +// CIR-ON: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float +// LLVM-ON-LABEL: @fmuladd_add_assign +// LLVM-ON: call float @llvm.fmuladd.f32 + +// x -= a * b picks negMul off isSub with the mul on the RHS. +float fmuladd_sub_assign(float x, float a, float b) { + x -= a * b; + return x; +} +// CIR-ON-LABEL: cir.func {{.*}}@fmuladd_sub_assign +// CIR-ON: cir.fneg %{{.*}} : !cir.float +// CIR-ON: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float +// LLVM-ON-LABEL: @fmuladd_sub_assign +// LLVM-ON: fneg float +// LLVM-ON: call float @llvm.fmuladd.f32