diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 217b0fddd247d..d31156d5c5efc 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -12324,6 +12324,54 @@ static SDValue emitFloatCompareMask(SDValue LHS, SDValue RHS, SDValue TVal, return Cmp; } +static bool isGTorGE(ISD::CondCode CC) { + return CC == ISD::SETGT || CC == ISD::SETGE; +} + +static bool isLTorLE(ISD::CondCode CC) { + return CC == ISD::SETLT || CC == ISD::SETLE; +} + +// See if a conditional (LHS CC RHS ? TrueVal : FalseVal) is lower-saturating. +// All of these conditions (and their <= and >= counterparts) will do: +// x < k ? k : x +// x > k ? x : k +// k < x ? x : k +// k > x ? k : x +static bool isLowerSaturate(ISD::CondCode CC, SDValue LHS, SDValue RHS, + SDValue TrueVal, SDValue FalseVal, SDValue K) { + return (isGTorGE(CC) && + ((K == LHS && K == TrueVal) || (K == RHS && K == FalseVal))) || + (isLTorLE(CC) && + ((K == RHS && K == TrueVal) || (K == LHS && K == FalseVal))); +} + +// Check if a condition of the type x < k ? k : x can be converted into a +// bit operation instead of conditional moves when k is 0 or -1. +static bool isLowerSaturatingConditional(ISD::CondCode CC, SDValue LHS, + SDValue RHS, SDValue TrueVal, + SDValue FalseVal, SDValue &SatValue, + SDValue &LowerSatConstant) { + SDValue *K = isa(LHS) ? &LHS + : isa(RHS) ? &RHS + : nullptr; + if (!K) + return false; + + SDValue KTmp = isa(TrueVal) ? TrueVal : FalseVal; + SatValue = (KTmp == TrueVal) ? FalseVal : TrueVal; + SDValue VTmp = (*K == LHS) ? RHS : LHS; + + if (*K != KTmp || SatValue != VTmp) + return false; + + if (!isLowerSaturate(CC, LHS, RHS, TrueVal, FalseVal, *K)) + return false; + + LowerSatConstant = *K; + return true; +} + SDValue AArch64TargetLowering::LowerSELECT_CC( ISD::CondCode CC, SDValue LHS, SDValue RHS, SDValue TVal, SDValue FVal, iterator_range Users, SDNodeFlags Flags, @@ -12357,24 +12405,6 @@ SDValue AArch64TargetLowering::LowerSELECT_CC( ConstantSDNode *CTVal = dyn_cast(TVal); ConstantSDNode *RHSC = dyn_cast(RHS); - // Check for SMAX(lhs, 0) and SMIN(lhs, 0) patterns. - // (SELECT_CC setgt, lhs, 0, lhs, 0) -> (BIC lhs, (SRA lhs, typesize-1)) - // (SELECT_CC setlt, lhs, 0, lhs, 0) -> (AND lhs, (SRA lhs, typesize-1)) - // Both require less instructions than compare and conditional select. - if ((CC == ISD::SETGT || CC == ISD::SETLT) && LHS == TVal && - RHSC && RHSC->isZero() && CFVal && CFVal->isZero() && - LHS.getValueType() == RHS.getValueType()) { - EVT VT = LHS.getValueType(); - SDValue Shift = - DAG.getNode(ISD::SRA, DL, VT, LHS, - DAG.getConstant(VT.getSizeInBits() - 1, DL, VT)); - - if (CC == ISD::SETGT) - Shift = DAG.getNOT(DL, Shift, VT); - - return DAG.getNode(ISD::AND, DL, VT, LHS, Shift); - } - // Check for sign bit test patterns that can use TST optimization. // (SELECT_CC setlt, sign_extend_inreg, 0, tval, fval) // -> TST %operand, sign_bit; CSEL @@ -12418,6 +12448,43 @@ SDValue AArch64TargetLowering::LowerSELECT_CC( } } + // Try to convert expressions of the form x < k ? k : x (and similar forms) + // into more efficient bit operations, which is possible when k is 0 or -1. + // Only allow this transformation on full-width operations. + SDValue LowerSatConstant; + SDValue SatValue; + if (isLowerSaturatingConditional(CC, LHS, RHS, TVal, FVal, SatValue, + LowerSatConstant)) { + EVT SatVT = SatValue.getValueType(); + SDValue ShiftV = + DAG.getNode(ISD::SRA, DL, SatVT, SatValue, + DAG.getConstant(SatVT.getSizeInBits() - 1, DL, SatVT)); + if (isNullConstant(LowerSatConstant)) { + ShiftV = DAG.getNOT(DL, ShiftV, SatVT); + return DAG.getNode(ISD::AND, DL, SatVT, SatValue, ShiftV); + } + if (isAllOnesConstant(LowerSatConstant)) + return DAG.getNode(ISD::OR, DL, SatVT, SatValue, ShiftV); + } + + // Check for SMAX(lhs, 0) and SMIN(lhs, 0) patterns. + // (SELECT_CC setgt, lhs, 0, lhs, 0) -> (BIC lhs, (SRA lhs, typesize-1)) + // (SELECT_CC setlt, lhs, 0, lhs, 0) -> (AND lhs, (SRA lhs, typesize-1)) + // Both require less instructions than compare and conditional select. + if ((CC == ISD::SETGT || CC == ISD::SETLT) && LHS == TVal && RHSC && + RHSC->isZero() && CFVal && CFVal->isZero() && + LHS.getValueType() == RHS.getValueType()) { + EVT VT = LHS.getValueType(); + SDValue Shift = + DAG.getNode(ISD::SRA, DL, VT, LHS, + DAG.getConstant(VT.getSizeInBits() - 1, DL, VT)); + + if (CC == ISD::SETGT) + Shift = DAG.getNOT(DL, Shift, VT); + + return DAG.getNode(ISD::AND, DL, VT, LHS, Shift); + } + unsigned Opcode = AArch64ISD::CSEL; // If both the TVal and the FVal are constants, see if we can swap them in diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index e08a4f5ebda08..0964c4d800ba2 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -5121,8 +5121,7 @@ static SDValue LowerSaturatingConditional(SDValue Op, SelectionDAG &DAG) { // It returns true if the transformation can be made, and in such case // returns x in V, and k in SatK. static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V, - SDValue &SatK) -{ + SDValue &SatK) { SDValue LHS = Op.getOperand(0); SDValue RHS = Op.getOperand(1); ISD::CondCode CC = cast(Op.getOperand(4))->get(); @@ -5222,11 +5221,11 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { return SatValue; // Try to convert expressions of the form x < k ? k : x (and similar forms) - // into more efficient bit operations, which is possible when k is 0 or -1 + // into more efficient bit operations, which is possible when k is 0 or -1. // On ARM and Thumb-2 which have flexible operand 2 this will result in // single instructions. On Thumb the shift and the bit operation will be two // instructions. - // Only allow this transformation on full-width (32-bit) operations + // Only allow this transformation on full-width (32-bit) operations. SDValue LowerSatConstant; SDValue SatValue; if (VT == MVT::i32 && @@ -5250,6 +5249,12 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { ConstantSDNode *RHSC = dyn_cast(RHS); if (Op.getValueType().isInteger()) { + // (SELECT_CC setlt, x, 0, 1, 0) -> SRL(x, bw-1) + if (CC == ISD::SETLT && isNullConstant(RHS) && isOneConstant(TrueVal) && + isNullConstant(FalseVal) && LHS.getValueType() == VT) + return DAG.getNode(ISD::SRL, dl, VT, LHS, + DAG.getConstant(VT.getSizeInBits() - 1, dl, VT)); + // Check for SMAX(lhs, 0) and SMIN(lhs, 0) patterns. // (SELECT_CC setgt, lhs, 0, lhs, 0) -> (BIC lhs, (SRA lhs, typesize-1)) // (SELECT_CC setlt, lhs, 0, lhs, 0) -> (AND lhs, (SRA lhs, typesize-1)) @@ -5267,12 +5272,6 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { return DAG.getNode(ISD::AND, dl, VT, LHS, Shift); } - - // (SELECT_CC setlt, x, 0, 1, 0) -> SRL(x, bw-1) - if (CC == ISD::SETLT && isNullConstant(RHS) && isOneConstant(TrueVal) && - isNullConstant(FalseVal) && LHS.getValueType() == VT) - return DAG.getNode(ISD::SRL, dl, VT, LHS, - DAG.getConstant(VT.getSizeInBits() - 1, dl, VT)); } if (LHS.getValueType() == MVT::i32) { diff --git a/llvm/test/CodeGen/AArch64/arm64-fmax-safe.ll b/llvm/test/CodeGen/AArch64/arm64-fmax-safe.ll index 4e3fac3650e98..f96ea0d2ed879 100644 --- a/llvm/test/CodeGen/AArch64/arm64-fmax-safe.ll +++ b/llvm/test/CodeGen/AArch64/arm64-fmax-safe.ll @@ -64,8 +64,7 @@ define float @test_cross_fail(float %lhs, float %rhs) { define i64 @test_integer(i64 %in) { ; CHECK-LABEL: test_integer: ; CHECK: // %bb.0: -; CHECK-NEXT: cmp x0, #0 -; CHECK-NEXT: csel x0, xzr, x0, mi +; CHECK-NEXT: bic x0, x0, x0, asr #63 ; CHECK-NEXT: ret %cmp = icmp slt i64 %in, 0 %val = select i1 %cmp, i64 0, i64 %in diff --git a/llvm/test/CodeGen/AArch64/arm64-fmax.ll b/llvm/test/CodeGen/AArch64/arm64-fmax.ll index f311139e193a5..ab1b978c363f7 100644 --- a/llvm/test/CodeGen/AArch64/arm64-fmax.ll +++ b/llvm/test/CodeGen/AArch64/arm64-fmax.ll @@ -59,8 +59,7 @@ define float @test_cross_fail(float %lhs, float %rhs) { define i64 @test_integer(i64 %in) { ; CHECK-LABEL: test_integer: ; CHECK: // %bb.0: -; CHECK-NEXT: cmp x0, #0 -; CHECK-NEXT: csel x0, xzr, x0, mi +; CHECK-NEXT: bic x0, x0, x0, asr #63 ; CHECK-NEXT: ret %cmp = icmp slt i64 %in, 0 %val = select i1 %cmp, i64 0, i64 %in diff --git a/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll b/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll index f95d338198c4d..67f598727a032 100644 --- a/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll +++ b/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll @@ -23,8 +23,8 @@ define i1 @test_signed_i1_f32(float %f) nounwind { ; CHECK-SD-LABEL: test_signed_i1_f32: ; CHECK-SD: // %bb.0: ; CHECK-SD-NEXT: fcvtzs w8, s0 -; CHECK-SD-NEXT: ands w8, w8, w8, asr #31 -; CHECK-SD-NEXT: csinv w8, w8, wzr, pl +; CHECK-SD-NEXT: and w8, w8, w8, asr #31 +; CHECK-SD-NEXT: orr w8, w8, w8, lsr #31 ; CHECK-SD-NEXT: and w0, w8, #0x1 ; CHECK-SD-NEXT: ret ; @@ -268,8 +268,8 @@ define i1 @test_signed_i1_f64(double %f) nounwind { ; CHECK-SD-LABEL: test_signed_i1_f64: ; CHECK-SD: // %bb.0: ; CHECK-SD-NEXT: fcvtzs w8, d0 -; CHECK-SD-NEXT: ands w8, w8, w8, asr #31 -; CHECK-SD-NEXT: csinv w8, w8, wzr, pl +; CHECK-SD-NEXT: and w8, w8, w8, asr #31 +; CHECK-SD-NEXT: orr w8, w8, w8, lsr #31 ; CHECK-SD-NEXT: and w0, w8, #0x1 ; CHECK-SD-NEXT: ret ; @@ -518,16 +518,16 @@ define i1 @test_signed_i1_f16(half %f) nounwind { ; CHECK-SD-CVT: // %bb.0: ; CHECK-SD-CVT-NEXT: fcvt s0, h0 ; CHECK-SD-CVT-NEXT: fcvtzs w8, s0 -; CHECK-SD-CVT-NEXT: ands w8, w8, w8, asr #31 -; CHECK-SD-CVT-NEXT: csinv w8, w8, wzr, pl +; CHECK-SD-CVT-NEXT: and w8, w8, w8, asr #31 +; CHECK-SD-CVT-NEXT: orr w8, w8, w8, lsr #31 ; CHECK-SD-CVT-NEXT: and w0, w8, #0x1 ; CHECK-SD-CVT-NEXT: ret ; ; CHECK-SD-FP16-LABEL: test_signed_i1_f16: ; CHECK-SD-FP16: // %bb.0: ; CHECK-SD-FP16-NEXT: fcvtzs w8, h0 -; CHECK-SD-FP16-NEXT: ands w8, w8, w8, asr #31 -; CHECK-SD-FP16-NEXT: csinv w8, w8, wzr, pl +; CHECK-SD-FP16-NEXT: and w8, w8, w8, asr #31 +; CHECK-SD-FP16-NEXT: orr w8, w8, w8, lsr #31 ; CHECK-SD-FP16-NEXT: and w0, w8, #0x1 ; CHECK-SD-FP16-NEXT: ret ; diff --git a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll index 145aef9123a4e..892af7852e1bc 100644 --- a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll +++ b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll @@ -2350,14 +2350,14 @@ define <2 x i1> @test_signed_v2f64_v2i1(<2 x double> %f) { ; CHECK-SD-LABEL: test_signed_v2f64_v2i1: ; CHECK-SD: // %bb.0: ; CHECK-SD-NEXT: mov d1, v0.d[1] -; CHECK-SD-NEXT: fcvtzs w9, d0 -; CHECK-SD-NEXT: fcvtzs w8, d1 -; CHECK-SD-NEXT: ands w8, w8, w8, asr #31 -; CHECK-SD-NEXT: csinv w8, w8, wzr, pl -; CHECK-SD-NEXT: ands w9, w9, w9, asr #31 -; CHECK-SD-NEXT: csinv w9, w9, wzr, pl -; CHECK-SD-NEXT: fmov s0, w9 -; CHECK-SD-NEXT: mov v0.s[1], w8 +; CHECK-SD-NEXT: fcvtzs w8, d0 +; CHECK-SD-NEXT: fcvtzs w9, d1 +; CHECK-SD-NEXT: and w8, w8, w8, asr #31 +; CHECK-SD-NEXT: orr w8, w8, w8, asr #31 +; CHECK-SD-NEXT: and w9, w9, w9, asr #31 +; CHECK-SD-NEXT: fmov s0, w8 +; CHECK-SD-NEXT: orr w9, w9, w9, asr #31 +; CHECK-SD-NEXT: mov v0.s[1], w9 ; CHECK-SD-NEXT: // kill: def $d0 killed $d0 killed $q0 ; CHECK-SD-NEXT: ret ; diff --git a/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll b/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll index 96a7a9d039c21..4c760e7f9d32b 100644 --- a/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll +++ b/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll @@ -329,5 +329,40 @@ define <2 x i64> @sel_shift_bool_v2i64(<2 x i1> %t) { %shl = select <2 x i1> %t, <2 x i64> , <2 x i64> zeroinitializer ret <2 x i64> %shl } + +; x < 0 ? 0 : x +define i32 @sat0_inverted(i32 %x) { +; CHECK-SD-LABEL: sat0_inverted: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: bic w0, w0, w0, asr #31 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: sat0_inverted: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: cmp w0, #0 +; CHECK-GI-NEXT: csel w0, w0, wzr, gt +; CHECK-GI-NEXT: ret + %cmp = icmp slt i32 %x, 0 + %sel = select i1 %cmp, i32 0, i32 %x + ret i32 %sel +} + +; x < -1 ? -1 : x +define i32 @sat1_inverted(i32 %x) { +; CHECK-SD-LABEL: sat1_inverted: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: orr w0, w0, w0, asr #31 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: sat1_inverted: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: cmn w0, #1 +; CHECK-GI-NEXT: csinv w0, w0, wzr, gt +; CHECK-GI-NEXT: ret + %cmp = icmp slt i32 %x, -1 + %sel = select i1 %cmp, i32 -1, i32 %x + ret i32 %sel +} + ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: ; CHECK: {{.*}} diff --git a/llvm/test/CodeGen/AArch64/win64_vararg.ll b/llvm/test/CodeGen/AArch64/win64_vararg.ll index 0f4cb44a6f73b..6b845c6163882 100644 --- a/llvm/test/CodeGen/AArch64/win64_vararg.ll +++ b/llvm/test/CodeGen/AArch64/win64_vararg.ll @@ -133,8 +133,7 @@ define i32 @fp(ptr, i64, ptr, ...) local_unnamed_addr #6 { ; CHECK-NEXT: mov x4, xzr ; CHECK-NEXT: orr x0, x8, #0x2 ; CHECK-NEXT: bl __stdio_common_vsprintf -; CHECK-NEXT: cmn w0, #1 -; CHECK-NEXT: csinv w0, w0, wzr, gt +; CHECK-NEXT: orr w0, w0, w0, asr #31 ; CHECK-NEXT: .seh_startepilogue ; CHECK-NEXT: ldp x29, x30, [sp, #24] // 16-byte Folded Reload ; CHECK-NEXT: .seh_save_fplr 24 @@ -268,8 +267,7 @@ define i32 @snprintf(ptr, i64, ptr, ...) local_unnamed_addr #5 { ; CHECK-NEXT: mov x4, xzr ; CHECK-NEXT: orr x0, x8, #0x2 ; CHECK-NEXT: bl __stdio_common_vsprintf -; CHECK-NEXT: cmn w0, #1 -; CHECK-NEXT: csinv w0, w0, wzr, gt +; CHECK-NEXT: orr w0, w0, w0, asr #31 ; CHECK-NEXT: .seh_startepilogue ; CHECK-NEXT: ldp x21, x30, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: .seh_save_lrpair x21, 32