diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 418ef38daac29..216a4ebf24aaa 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -4580,12 +4580,15 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { } } - // canonicalize (sub X, (vscale * C)) to (add X, (vscale * -C)) + // canonicalize (sub X, (vscale * C)) to (add X, (vscale * -C)) if this is the + // only use of the vscale value or if (vscale * -C) is a valid add immediate. // avoid if ISD::MUL handling is poor and ISD::SHL isn't an option. - if (N1.getOpcode() == ISD::VSCALE && N1.hasOneUse()) { + if (N1.getOpcode() == ISD::VSCALE) { const APInt &IntVal = N1.getConstantOperandAPInt(0); - if (!IntVal.isPowerOf2() || - hasOperation(ISD::MUL, N1.getOperand(0).getValueType())) + if ((N1.hasOneUse() || + TLI.isLegalAddScalableImmediate(-IntVal.getSExtValue())) && + (!IntVal.isPowerOf2() || + hasOperation(ISD::MUL, N1.getOperand(0).getValueType()))) return DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getVScale(DL, VT, -IntVal)); } diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index fe48a883ecef4..b7c3fa4e50655 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -19959,8 +19959,8 @@ bool AArch64TargetLowering::isLegalAddImmediate(int64_t Immed) const { } bool AArch64TargetLowering::isLegalAddScalableImmediate(int64_t Imm) const { - // We will only emit addvl/inc* instructions for SVE2 - if (!Subtarget->hasSVE2()) + // We will only emit addvl/inc* instructions if the subtarget allows it. + if (!Subtarget->useScalarIncVL()) return false; // addvl's immediates are in terms of the number of bytes in a register. diff --git a/llvm/test/CodeGen/AArch64/sve-vl-arith.ll b/llvm/test/CodeGen/AArch64/sve-vl-arith.ll index 0a9f4948a5f77..f94b9a6c1e653 100644 --- a/llvm/test/CodeGen/AArch64/sve-vl-arith.ll +++ b/llvm/test/CodeGen/AArch64/sve-vl-arith.ll @@ -551,6 +551,298 @@ define i32 @decd_scalar_i32(i32 %a) { ret i32 %sub } +define void @decb_incb_scalar_i64(i64 %a, i64 %b, ptr %p, ptr %q) { +; NO_SCALAR_INC-LABEL: decb_incb_scalar_i64: +; NO_SCALAR_INC: // %bb.0: +; NO_SCALAR_INC-NEXT: rdvl x8, #1 +; NO_SCALAR_INC-NEXT: add x9, x1, x8 +; NO_SCALAR_INC-NEXT: sub x8, x0, x8 +; NO_SCALAR_INC-NEXT: str x9, [x2] +; NO_SCALAR_INC-NEXT: str x8, [x3] +; NO_SCALAR_INC-NEXT: ret +; +; CHECK-LABEL: decb_incb_scalar_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: incb x1 +; CHECK-NEXT: decb x0 +; CHECK-NEXT: str x1, [x2] +; CHECK-NEXT: str x0, [x3] +; CHECK-NEXT: ret +; +; NO_FAST_INC-LABEL: decb_incb_scalar_i64: +; NO_FAST_INC: // %bb.0: +; NO_FAST_INC-NEXT: addvl x8, x1, #1 +; NO_FAST_INC-NEXT: addvl x9, x0, #-1 +; NO_FAST_INC-NEXT: str x8, [x2] +; NO_FAST_INC-NEXT: str x9, [x3] +; NO_FAST_INC-NEXT: ret + %vscale = call i64 @llvm.vscale.i64() + %mul = mul i64 %vscale, 16 + %sub = sub i64 %a, %mul + %add = add i64 %b, %mul + store i64 %add, ptr %p, align 1 + store i64 %sub, ptr %q, align 1 + ret void +} + +define void @dech_inch_scalar_i64(i64 %a, i64 %b, ptr %p, ptr %q) { +; NO_SCALAR_INC-LABEL: dech_inch_scalar_i64: +; NO_SCALAR_INC: // %bb.0: +; NO_SCALAR_INC-NEXT: cnth x8 +; NO_SCALAR_INC-NEXT: add x9, x1, x8 +; NO_SCALAR_INC-NEXT: sub x8, x0, x8 +; NO_SCALAR_INC-NEXT: str x9, [x2] +; NO_SCALAR_INC-NEXT: str x8, [x3] +; NO_SCALAR_INC-NEXT: ret +; +; CHECK-LABEL: dech_inch_scalar_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: inch x1 +; CHECK-NEXT: dech x0 +; CHECK-NEXT: str x1, [x2] +; CHECK-NEXT: str x0, [x3] +; CHECK-NEXT: ret +; +; NO_FAST_INC-LABEL: dech_inch_scalar_i64: +; NO_FAST_INC: // %bb.0: +; NO_FAST_INC-NEXT: inch x1 +; NO_FAST_INC-NEXT: dech x0 +; NO_FAST_INC-NEXT: str x1, [x2] +; NO_FAST_INC-NEXT: str x0, [x3] +; NO_FAST_INC-NEXT: ret + %vscale = call i64 @llvm.vscale.i64() + %mul = mul i64 %vscale, 8 + %sub = sub i64 %a, %mul + %add = add i64 %b, %mul + store i64 %add, ptr %p, align 1 + store i64 %sub, ptr %q, align 1 + ret void +} + +define void @decw_incw_scalar_i64(i64 %a, i64 %b, ptr %p, ptr %q) { +; NO_SCALAR_INC-LABEL: decw_incw_scalar_i64: +; NO_SCALAR_INC: // %bb.0: +; NO_SCALAR_INC-NEXT: cntw x8 +; NO_SCALAR_INC-NEXT: add x9, x1, x8 +; NO_SCALAR_INC-NEXT: sub x8, x0, x8 +; NO_SCALAR_INC-NEXT: str x9, [x2] +; NO_SCALAR_INC-NEXT: str x8, [x3] +; NO_SCALAR_INC-NEXT: ret +; +; CHECK-LABEL: decw_incw_scalar_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: incw x1 +; CHECK-NEXT: decw x0 +; CHECK-NEXT: str x1, [x2] +; CHECK-NEXT: str x0, [x3] +; CHECK-NEXT: ret +; +; NO_FAST_INC-LABEL: decw_incw_scalar_i64: +; NO_FAST_INC: // %bb.0: +; NO_FAST_INC-NEXT: incw x1 +; NO_FAST_INC-NEXT: decw x0 +; NO_FAST_INC-NEXT: str x1, [x2] +; NO_FAST_INC-NEXT: str x0, [x3] +; NO_FAST_INC-NEXT: ret + %vscale = call i64 @llvm.vscale.i64() + %mul = mul i64 %vscale, 4 + %sub = sub i64 %a, %mul + %add = add i64 %b, %mul + store i64 %add, ptr %p, align 1 + store i64 %sub, ptr %q, align 1 + ret void +} + +define void @decd_incd_scalar_i64(i64 %a, i64 %b, ptr %p, ptr %q) { +; NO_SCALAR_INC-LABEL: decd_incd_scalar_i64: +; NO_SCALAR_INC: // %bb.0: +; NO_SCALAR_INC-NEXT: cntd x8 +; NO_SCALAR_INC-NEXT: add x9, x1, x8 +; NO_SCALAR_INC-NEXT: sub x8, x0, x8 +; NO_SCALAR_INC-NEXT: str x9, [x2] +; NO_SCALAR_INC-NEXT: str x8, [x3] +; NO_SCALAR_INC-NEXT: ret +; +; CHECK-LABEL: decd_incd_scalar_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: incd x1 +; CHECK-NEXT: decd x0 +; CHECK-NEXT: str x1, [x2] +; CHECK-NEXT: str x0, [x3] +; CHECK-NEXT: ret +; +; NO_FAST_INC-LABEL: decd_incd_scalar_i64: +; NO_FAST_INC: // %bb.0: +; NO_FAST_INC-NEXT: incd x1 +; NO_FAST_INC-NEXT: decd x0 +; NO_FAST_INC-NEXT: str x1, [x2] +; NO_FAST_INC-NEXT: str x0, [x3] +; NO_FAST_INC-NEXT: ret + %vscale = call i64 @llvm.vscale.i64() + %mul = mul i64 %vscale, 2 + %sub = sub i64 %a, %mul + %add = add i64 %b, %mul + store i64 %add, ptr %p, align 1 + store i64 %sub, ptr %q, align 1 + ret void +} + +define void @decb_incb_scalar_i32(i32 %a, i32 %b, ptr %p, ptr %q) { +; NO_SCALAR_INC-LABEL: decb_incb_scalar_i32: +; NO_SCALAR_INC: // %bb.0: +; NO_SCALAR_INC-NEXT: rdvl x8, #1 +; NO_SCALAR_INC-NEXT: sub w9, w0, w8 +; NO_SCALAR_INC-NEXT: add w8, w1, w8 +; NO_SCALAR_INC-NEXT: str w9, [x2] +; NO_SCALAR_INC-NEXT: str w8, [x3] +; NO_SCALAR_INC-NEXT: ret +; +; CHECK-LABEL: decb_incb_scalar_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1 +; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 +; CHECK-NEXT: decb x0 +; CHECK-NEXT: incb x1 +; CHECK-NEXT: str w0, [x2] +; CHECK-NEXT: str w1, [x3] +; CHECK-NEXT: ret +; +; NO_FAST_INC-LABEL: decb_incb_scalar_i32: +; NO_FAST_INC: // %bb.0: +; NO_FAST_INC-NEXT: // kill: def $w1 killed $w1 def $x1 +; NO_FAST_INC-NEXT: // kill: def $w0 killed $w0 def $x0 +; NO_FAST_INC-NEXT: addvl x8, x1, #1 +; NO_FAST_INC-NEXT: addvl x9, x0, #-1 +; NO_FAST_INC-NEXT: str w9, [x2] +; NO_FAST_INC-NEXT: str w8, [x3] +; NO_FAST_INC-NEXT: ret + %vscale = call i64 @llvm.vscale.i64() + %mul = mul i64 %vscale, 16 + %vl = trunc i64 %mul to i32 + %sub = sub i32 %a, %vl + %add = add i32 %b, %vl + store i32 %sub, ptr %p, align 1 + store i32 %add, ptr %q, align 1 + ret void +} + +define void @dech_inch_scalar_i32(i32 %a, i32 %b, ptr %p, ptr %q) { +; NO_SCALAR_INC-LABEL: dech_inch_scalar_i32: +; NO_SCALAR_INC: // %bb.0: +; NO_SCALAR_INC-NEXT: cnth x8 +; NO_SCALAR_INC-NEXT: sub w9, w0, w8 +; NO_SCALAR_INC-NEXT: add w8, w1, w8 +; NO_SCALAR_INC-NEXT: str w9, [x2] +; NO_SCALAR_INC-NEXT: str w8, [x3] +; NO_SCALAR_INC-NEXT: ret +; +; CHECK-LABEL: dech_inch_scalar_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1 +; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 +; CHECK-NEXT: dech x0 +; CHECK-NEXT: inch x1 +; CHECK-NEXT: str w0, [x2] +; CHECK-NEXT: str w1, [x3] +; CHECK-NEXT: ret +; +; NO_FAST_INC-LABEL: dech_inch_scalar_i32: +; NO_FAST_INC: // %bb.0: +; NO_FAST_INC-NEXT: // kill: def $w1 killed $w1 def $x1 +; NO_FAST_INC-NEXT: // kill: def $w0 killed $w0 def $x0 +; NO_FAST_INC-NEXT: dech x0 +; NO_FAST_INC-NEXT: inch x1 +; NO_FAST_INC-NEXT: str w0, [x2] +; NO_FAST_INC-NEXT: str w1, [x3] +; NO_FAST_INC-NEXT: ret + %vscale = call i64 @llvm.vscale.i64() + %mul = mul i64 %vscale, 8 + %vl = trunc i64 %mul to i32 + %sub = sub i32 %a, %vl + %add = add i32 %b, %vl + store i32 %sub, ptr %p, align 1 + store i32 %add, ptr %q, align 1 + ret void +} + +define void @decw_incw_scalar_i32(i32 %a, i32 %b, ptr %p, ptr %q) { +; NO_SCALAR_INC-LABEL: decw_incw_scalar_i32: +; NO_SCALAR_INC: // %bb.0: +; NO_SCALAR_INC-NEXT: cntw x8 +; NO_SCALAR_INC-NEXT: sub w9, w0, w8 +; NO_SCALAR_INC-NEXT: add w8, w1, w8 +; NO_SCALAR_INC-NEXT: str w9, [x2] +; NO_SCALAR_INC-NEXT: str w8, [x3] +; NO_SCALAR_INC-NEXT: ret +; +; CHECK-LABEL: decw_incw_scalar_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1 +; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 +; CHECK-NEXT: decw x0 +; CHECK-NEXT: incw x1 +; CHECK-NEXT: str w0, [x2] +; CHECK-NEXT: str w1, [x3] +; CHECK-NEXT: ret +; +; NO_FAST_INC-LABEL: decw_incw_scalar_i32: +; NO_FAST_INC: // %bb.0: +; NO_FAST_INC-NEXT: // kill: def $w1 killed $w1 def $x1 +; NO_FAST_INC-NEXT: // kill: def $w0 killed $w0 def $x0 +; NO_FAST_INC-NEXT: decw x0 +; NO_FAST_INC-NEXT: incw x1 +; NO_FAST_INC-NEXT: str w0, [x2] +; NO_FAST_INC-NEXT: str w1, [x3] +; NO_FAST_INC-NEXT: ret + %vscale = call i64 @llvm.vscale.i64() + %mul = mul i64 %vscale, 4 + %vl = trunc i64 %mul to i32 + %sub = sub i32 %a, %vl + %add = add i32 %b, %vl + store i32 %sub, ptr %p, align 1 + store i32 %add, ptr %q, align 1 + ret void +} + +define void @decd_incb_scalar_i32(i32 %a, i32 %b, ptr %p, ptr %q) { +; NO_SCALAR_INC-LABEL: decd_incb_scalar_i32: +; NO_SCALAR_INC: // %bb.0: +; NO_SCALAR_INC-NEXT: cntd x8 +; NO_SCALAR_INC-NEXT: sub w9, w0, w8 +; NO_SCALAR_INC-NEXT: add w8, w1, w8 +; NO_SCALAR_INC-NEXT: str w9, [x2] +; NO_SCALAR_INC-NEXT: str w8, [x3] +; NO_SCALAR_INC-NEXT: ret +; +; CHECK-LABEL: decd_incb_scalar_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1 +; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 +; CHECK-NEXT: decd x0 +; CHECK-NEXT: incd x1 +; CHECK-NEXT: str w0, [x2] +; CHECK-NEXT: str w1, [x3] +; CHECK-NEXT: ret +; +; NO_FAST_INC-LABEL: decd_incb_scalar_i32: +; NO_FAST_INC: // %bb.0: +; NO_FAST_INC-NEXT: // kill: def $w1 killed $w1 def $x1 +; NO_FAST_INC-NEXT: // kill: def $w0 killed $w0 def $x0 +; NO_FAST_INC-NEXT: decd x0 +; NO_FAST_INC-NEXT: incd x1 +; NO_FAST_INC-NEXT: str w0, [x2] +; NO_FAST_INC-NEXT: str w1, [x3] +; NO_FAST_INC-NEXT: ret + %vscale = call i64 @llvm.vscale.i64() + %mul = mul i64 %vscale, 2 + %vl = trunc i64 %mul to i32 + %sub = sub i32 %a, %vl + %add = add i32 %b, %vl + store i32 %sub, ptr %p, align 1 + store i32 %add, ptr %q, align 1 + ret void +} + declare i16 @llvm.vscale.i16() declare i32 @llvm.vscale.i32() declare i64 @llvm.vscale.i64()