diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index ee06217fc5d87..a00fb1ddbd65c 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -2598,6 +2598,31 @@ instCombineSVEVectorMlaU(InstCombiner &IC, IntrinsicInst &II) { return std::nullopt; } +static std::optional +instCombineSVEPairwiseAddLong(InstCombiner &IC, IntrinsicInst &II) { + assert((II.getIntrinsicID() == Intrinsic::aarch64_sve_sadalp || + II.getIntrinsicID() == Intrinsic::aarch64_sve_uadalp) && + "Expected SADALP or UADALP intrinsic"); + + // We are looking for add(adalp(%pred, zeroinitializer, %in), %acc) + if (!II.hasOneUse() || !match(II.getArgOperand(1), m_Zero())) + return std::nullopt; + + auto *User = cast(*II.user_begin()); + Value *Acc; + if (!match(User, m_c_Add(m_Specific(&II), m_Value(Acc)))) + return std::nullopt; + + IC.Builder.SetInsertPoint(User); + Value *PairwiseAddLong = IC.Builder.CreateIntrinsic( + II.getIntrinsicID(), {II.getType()}, + {II.getArgOperand(0), Acc, II.getArgOperand(2)}); + + IC.replaceInstUsesWith(*User, PairwiseAddLong); + IC.eraseInstFromFunction(*User); + return ⅈ // II is now trivially dead and will get erased. +} + static std::optional instCombineSVEVectorAdd(InstCombiner &IC, IntrinsicInst &II) { if (auto MLA = instCombineSVEVectorFuseMulAddSub @sadalp_i8_separate_add( %pg, %a, %b) { +; CHECK-LABEL: define @sadalp_i8_separate_add( +; CHECK-SAME: [[PG:%.*]], [[A:%.*]], [[B:%.*]]) { +; CHECK-NEXT: [[OUT2:%.*]] = call @llvm.aarch64.sve.sadalp.nxv8i16( [[PG]], [[A]], [[B]]) +; CHECK-NEXT: ret [[OUT2]] +; + %out = call @llvm.aarch64.sve.sadalp.nxv8i16( %pg, zeroinitializer, %b) + %out2 = add %out, %a + ret %out2 +} + +define @sadalp_i16_separate_add( %pg, %a, %b) { +; CHECK-LABEL: define @sadalp_i16_separate_add( +; CHECK-SAME: [[PG:%.*]], [[A:%.*]], [[B:%.*]]) { +; CHECK-NEXT: [[OUT2:%.*]] = call @llvm.aarch64.sve.sadalp.nxv4i32( [[PG]], [[A]], [[B]]) +; CHECK-NEXT: ret [[OUT2]] +; + %out = call @llvm.aarch64.sve.sadalp.nxv4i32( %pg, zeroinitializer, %b) + %out2 = add %out, %a + ret %out2 +} + +define @sadalp_i32_separate_add( %pg, %a, %b) { +; CHECK-LABEL: define @sadalp_i32_separate_add( +; CHECK-SAME: [[PG:%.*]], [[A:%.*]], [[B:%.*]]) { +; CHECK-NEXT: [[OUT2:%.*]] = call @llvm.aarch64.sve.sadalp.nxv2i64( [[PG]], [[A]], [[B]]) +; CHECK-NEXT: ret [[OUT2]] +; + %out = call @llvm.aarch64.sve.sadalp.nxv2i64( %pg, zeroinitializer, %b) + %out2 = add %out, %a + ret %out2 +} + +define @sadalp_i32_separate_add_rev( %pg, %a, %b) { +; CHECK-LABEL: define @sadalp_i32_separate_add_rev( +; CHECK-SAME: [[PG:%.*]], [[A:%.*]], [[B:%.*]]) { +; CHECK-NEXT: [[OUT2:%.*]] = call @llvm.aarch64.sve.sadalp.nxv2i64( [[PG]], [[A]], [[B]]) +; CHECK-NEXT: ret [[OUT2]] +; + %out = call @llvm.aarch64.sve.sadalp.nxv2i64( %pg, zeroinitializer, %b) + %out2 = add %a, %out + ret %out2 +} + +define @uadalp_i8_separate_add( %pg, %a, %b) { +; CHECK-LABEL: define @uadalp_i8_separate_add( +; CHECK-SAME: [[PG:%.*]], [[A:%.*]], [[B:%.*]]) { +; CHECK-NEXT: [[OUT2:%.*]] = call @llvm.aarch64.sve.uadalp.nxv8i16( [[PG]], [[A]], [[B]]) +; CHECK-NEXT: ret [[OUT2]] +; + %out = call @llvm.aarch64.sve.uadalp.nxv8i16( %pg, zeroinitializer, %b) + %out2 = add %out, %a + ret %out2 +} + +define @uadalp_i16_separate_add( %pg, %a, %b) { +; CHECK-LABEL: define @uadalp_i16_separate_add( +; CHECK-SAME: [[PG:%.*]], [[A:%.*]], [[B:%.*]]) { +; CHECK-NEXT: [[OUT2:%.*]] = call @llvm.aarch64.sve.uadalp.nxv4i32( [[PG]], [[A]], [[B]]) +; CHECK-NEXT: ret [[OUT2]] +; + %out = call @llvm.aarch64.sve.uadalp.nxv4i32( %pg, zeroinitializer, %b) + %out2 = add %out, %a + ret %out2 +} + +define @uadalp_i32_separate_add( %pg, %a, %b) { +; CHECK-LABEL: define @uadalp_i32_separate_add( +; CHECK-SAME: [[PG:%.*]], [[A:%.*]], [[B:%.*]]) { +; CHECK-NEXT: [[OUT2:%.*]] = call @llvm.aarch64.sve.uadalp.nxv2i64( [[PG]], [[A]], [[B]]) +; CHECK-NEXT: ret [[OUT2]] +; + %out = call @llvm.aarch64.sve.uadalp.nxv2i64( %pg, zeroinitializer, %b) + %out2 = add %out, %a + ret %out2 +} + +define @sadalp_nonzero_accumulator( %pg, %a, %b, %c) { +; CHECK-LABEL: define @sadalp_nonzero_accumulator( +; CHECK-SAME: [[PG:%.*]], [[A:%.*]], [[B:%.*]], [[C:%.*]]) { +; CHECK-NEXT: [[OUT:%.*]] = call @llvm.aarch64.sve.sadalp.nxv2i64( [[PG]], [[C]], [[B]]) +; CHECK-NEXT: [[OUT2:%.*]] = add [[OUT]], [[A]] +; CHECK-NEXT: ret [[OUT2]] +; + %out = call @llvm.aarch64.sve.sadalp.nxv2i64( %pg, %c, %b) + %out2 = add %out, %a + ret %out2 +}