diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index a651f8a682c48..023e91f33e0f5 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -15512,6 +15512,23 @@ static SDValue foldOverflowCheck(SDNode *Op, SelectionDAG &DAG, bool IsAdd) { CsetOp.getOperand(3)); } +// (ADC x 0 cond) => (CINC x HS cond) +static SDValue foldADCToCINC(SDNode *N, SelectionDAG &DAG) { + SDValue LHS = N->getOperand(0); + SDValue RHS = N->getOperand(1); + SDValue Cond = N->getOperand(2); + + if (!isNullConstant(RHS)) + return SDValue(); + + EVT VT = N->getValueType(0); + SDLoc DL(N); + + // (CINC x cc cond) <=> (CSINC x x !cc cond) + SDValue CC = DAG.getConstant(AArch64CC::LO, DL, MVT::i32); + return DAG.getNode(AArch64ISD::CSINC, DL, VT, LHS, LHS, CC, Cond); +} + static SDValue performAddSubCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) { @@ -18721,7 +18738,9 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N, case AArch64ISD::ANDS: return performFlagSettingCombine(N, DCI, ISD::AND); case AArch64ISD::ADC: - return foldOverflowCheck(N, DAG, /* IsAdd */ true); + if (auto R = foldOverflowCheck(N, DAG, /* IsAdd */ true)) + return R; + return foldADCToCINC(N, DAG); case AArch64ISD::SBC: return foldOverflowCheck(N, DAG, /* IsAdd */ false); case AArch64ISD::ADCS: diff --git a/llvm/test/CodeGen/AArch64/adc.ll b/llvm/test/CodeGen/AArch64/adc.ll index faa43d665d25f..4b1393ffc624a 100644 --- a/llvm/test/CodeGen/AArch64/adc.ll +++ b/llvm/test/CodeGen/AArch64/adc.ll @@ -30,13 +30,13 @@ define i128 @test_imm(i128 %a) { ; CHECK-LE-LABEL: test_imm: ; CHECK-LE: ; %bb.0: ; CHECK-LE-NEXT: adds x0, x0, #12 -; CHECK-LE-NEXT: adc x1, x1, xzr +; CHECK-LE-NEXT: cinc x1, x1, hs ; CHECK-LE-NEXT: ret ; ; CHECK-BE-LABEL: test_imm: ; CHECK-BE: // %bb.0: ; CHECK-BE-NEXT: adds x1, x1, #12 -; CHECK-BE-NEXT: adc x0, x0, xzr +; CHECK-BE-NEXT: cinc x0, x0, hs ; CHECK-BE-NEXT: ret %val = add i128 %a, 12 diff --git a/llvm/test/CodeGen/AArch64/addcarry-crash.ll b/llvm/test/CodeGen/AArch64/addcarry-crash.ll index be48fb79e7048..91c7ee7292c63 100644 --- a/llvm/test/CodeGen/AArch64/addcarry-crash.ll +++ b/llvm/test/CodeGen/AArch64/addcarry-crash.ll @@ -9,7 +9,7 @@ define i64 @foo(i64* nocapture readonly %ptr, i64 %a, i64 %b, i64 %c) local_unna ; CHECK-NEXT: lsr x9, x1, #32 ; CHECK-NEXT: cmn x3, x2 ; CHECK-NEXT: mul x8, x8, x9 -; CHECK-NEXT: adc x0, x8, xzr +; CHECK-NEXT: cinc x0, x8, hs ; CHECK-NEXT: ret entry: %0 = lshr i64 %a, 32 diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-O0.ll b/llvm/test/CodeGen/AArch64/atomicrmw-O0.ll index 71a4ed000c7e3..af8a46a598a2e 100644 --- a/llvm/test/CodeGen/AArch64/atomicrmw-O0.ll +++ b/llvm/test/CodeGen/AArch64/atomicrmw-O0.ll @@ -219,8 +219,7 @@ define i128 @test_rmw_add_128(i128* %dst) { ; NOLSE-NEXT: ldr x8, [sp, #32] // 8-byte Folded Reload ; NOLSE-NEXT: ldr x13, [sp, #24] // 8-byte Folded Reload ; NOLSE-NEXT: adds x14, x8, #1 -; NOLSE-NEXT: mov x9, xzr -; NOLSE-NEXT: adc x15, x11, x9 +; NOLSE-NEXT: cinc x15, x11, hs ; NOLSE-NEXT: .LBB4_2: // %atomicrmw.start ; NOLSE-NEXT: // Parent Loop BB4_1 Depth=1 ; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 @@ -271,14 +270,13 @@ define i128 @test_rmw_add_128(i128* %dst) { ; LSE-NEXT: ldr x10, [sp, #72] // 8-byte Folded Reload ; LSE-NEXT: ldr x8, [sp, #64] // 8-byte Folded Reload ; LSE-NEXT: ldr x9, [sp, #56] // 8-byte Folded Reload -; LSE-NEXT: adds x2, x8, #1 -; LSE-NEXT: mov x11, xzr -; LSE-NEXT: adc x11, x10, x11 -; LSE-NEXT: // kill: def $x2 killed $x2 def $x2_x3 -; LSE-NEXT: mov x3, x11 ; LSE-NEXT: mov x0, x8 ; LSE-NEXT: mov x1, x10 ; LSE-NEXT: stp x0, x1, [sp, #8] // 16-byte Folded Spill +; LSE-NEXT: adds x2, x8, #1 +; LSE-NEXT: cinc x11, x10, hs +; LSE-NEXT: // kill: def $x2 killed $x2 def $x2_x3 +; LSE-NEXT: mov x3, x11 ; LSE-NEXT: caspal x0, x1, x2, x3, [x9] ; LSE-NEXT: stp x0, x1, [sp, #24] // 16-byte Folded Spill ; LSE-NEXT: mov x9, x1 diff --git a/llvm/test/CodeGen/AArch64/icmp-shift-opt.ll b/llvm/test/CodeGen/AArch64/icmp-shift-opt.ll index f095ee8a4abe7..74a1c28032d89 100644 --- a/llvm/test/CodeGen/AArch64/icmp-shift-opt.ll +++ b/llvm/test/CodeGen/AArch64/icmp-shift-opt.ll @@ -11,7 +11,7 @@ define i128 @opt_setcc_lt_power_of_2(i128 %a) nounwind { ; CHECK-NEXT: .LBB0_1: // %loop ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: adds x0, x0, #1 -; CHECK-NEXT: adc x1, x1, xzr +; CHECK-NEXT: cinc x1, x1, hs ; CHECK-NEXT: orr x8, x1, x0, lsr #60 ; CHECK-NEXT: cbnz x8, .LBB0_1 ; CHECK-NEXT: // %bb.2: // %exit