diff --git a/llvm/include/llvm/Analysis/InstSimplifyFolder.h b/llvm/include/llvm/Analysis/InstSimplifyFolder.h index 58793ed977f68..2832beb9e337c 100644 --- a/llvm/include/llvm/Analysis/InstSimplifyFolder.h +++ b/llvm/include/llvm/Analysis/InstSimplifyFolder.h @@ -120,7 +120,7 @@ class LLVM_ABI InstSimplifyFolder final : public IRBuilderFolder { } Value *FoldBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, Type *Ty, - Instruction *FMFSource) const override { + Instruction *FMFSource = nullptr) const override { return simplifyBinaryIntrinsic(ID, Ty, LHS, RHS, SQ, dyn_cast_if_present(FMFSource)); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index acdb37996a443..8032f74f96004 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -942,10 +942,29 @@ static void recursivelyDeleteDeadRecipes(VPValue *V) { } } +namespace { +/// Get any instruction opcode or intrinsic ID data embedded in recipe \p R. +/// Returns an optional pair, where the first element indicates whether it is +/// an intrinsic ID. +static std::optional> +getOpcodeOrIntrinsicID(const VPSingleDefRecipe *R) { + return TypeSwitch>>(R) + .Case( + [](auto *I) { return std::make_pair(false, I->getOpcode()); }) + .Case([](auto *I) { + return std::make_pair(true, I->getVectorIntrinsicID()); + }) + .Default([](auto *) { return std::nullopt; }); +} +} // namespace + /// Try to fold \p R using InstSimplifyFolder. Will succeed and return a /// non-nullptr Value for a handled \p Opcode if corresponding \p Operands are /// foldable live-ins. -static Value *tryToFoldLiveIns(const VPRecipeBase &R, unsigned Opcode, +static Value *tryToFoldLiveIns(VPSingleDefRecipe &R, + std::pair OpcodeOrIID, ArrayRef Operands, const DataLayout &DL, VPTypeAnalysis &TypeInfo) { SmallVector Ops; @@ -956,6 +975,14 @@ static Value *tryToFoldLiveIns(const VPRecipeBase &R, unsigned Opcode, } InstSimplifyFolder Folder(DL); + if (OpcodeOrIID.first) { + if (R.getNumOperands() != 2) + return nullptr; + unsigned ID = OpcodeOrIID.second; + return Folder.FoldBinaryIntrinsic( + ID, Ops[0], Ops[1], TypeInfo.inferScalarType(R.getVPSingleValue())); + } + unsigned Opcode = OpcodeOrIID.second; if (Instruction::isBinaryOp(Opcode)) return Folder.FoldBinOp(static_cast(Opcode), Ops[0], Ops[1]); @@ -1005,19 +1032,13 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { // Simplification of live-in IR values for SingleDef recipes using // InstSimplifyFolder. - if (TypeSwitch(&R) - .Case([&](auto *I) { - const DataLayout &DL = - Plan->getScalarHeader()->getIRBasicBlock()->getDataLayout(); - Value *V = tryToFoldLiveIns(*I, I->getOpcode(), I->operands(), DL, - TypeInfo); - if (V) - I->replaceAllUsesWith(Plan->getOrAddLiveIn(V)); - return V; - }) - .Default([](auto *) { return false; })) - return; + if (auto OpcodeOrIID = getOpcodeOrIntrinsicID(Def)) { + const DataLayout &DL = + Plan->getScalarHeader()->getIRBasicBlock()->getDataLayout(); + if (Value *V = + tryToFoldLiveIns(*Def, *OpcodeOrIID, Def->operands(), DL, TypeInfo)) + return Def->replaceAllUsesWith(Plan->getOrAddLiveIn(V)); + } // Fold PredPHI LiveIn -> LiveIn. if (auto *PredPHI = dyn_cast(&R)) { @@ -1970,22 +1991,6 @@ struct VPCSEDenseMapInfo : public DenseMapInfo { return Def == getEmptyKey() || Def == getTombstoneKey(); } - /// Get any instruction opcode or intrinsic ID data embedded in recipe \p R. - /// Returns an optional pair, where the first element indicates whether it is - /// an intrinsic ID. - static std::optional> - getOpcodeOrIntrinsicID(const VPSingleDefRecipe *R) { - return TypeSwitch>>(R) - .Case( - [](auto *I) { return std::make_pair(false, I->getOpcode()); }) - .Case([](auto *I) { - return std::make_pair(true, I->getVectorIntrinsicID()); - }) - .Default([](auto *) { return std::nullopt; }); - } - /// If recipe \p R will lower to a GEP with a non-i8 source element type, /// return that source element type. static Type *getGEPSourceElementType(const VPSingleDefRecipe *R) { diff --git a/llvm/test/Transforms/LoopVectorize/constantfolder.ll b/llvm/test/Transforms/LoopVectorize/constantfolder.ll index 66592b0ccf677..bbd208e8a0586 100644 --- a/llvm/test/Transforms/LoopVectorize/constantfolder.ll +++ b/llvm/test/Transforms/LoopVectorize/constantfolder.ll @@ -288,3 +288,84 @@ loop.latch: exit: ret void } + +define void @const_fold_binaryintrinsic(ptr %dst, i64 %d) { +; CHECK-LABEL: define void @const_fold_binaryintrinsic( +; CHECK-SAME: ptr [[DST:%.*]], i64 [[D:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: store i64 3, ptr [[DST]], align 2 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100 +; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop.header + +loop.header: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] + br i1 true, label %loop.latch, label %else + +else: + br label %loop.latch + +loop.latch: + %const.0 = phi i64 [ %d, %else ], [ 0, %loop.header ] + %trunc = call i64 @llvm.umax.i64(i64 %const.0, i64 3) + store i64 %trunc, ptr %dst, align 2 + %iv.next = add i64 %iv, 1 + %cmp = icmp ult i64 %iv.next, 100 + br i1 %cmp, label %loop.header, label %exit + +exit: + ret void +} + +define void @const_fold_widegep(ptr noalias %A, ptr noalias %B) { +; CHECK-LABEL: define void @const_fold_widegep( +; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: store ptr [[A]], ptr [[B]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100 +; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop.header + +loop.header: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] + br i1 true, label %loop.latch, label %else + +else: + br label %loop.latch + +loop.latch: + %const.0 = phi i64 [ 0, %loop.header ], [ %iv, %else ] + %gep.A = getelementptr i64, ptr %A, i64 %const.0 + %gep.B = getelementptr i64, ptr %B, i64 %const.0 + store ptr %gep.A, ptr %gep.B + %iv.next = add nuw nsw i64 %iv, 1 + %exit.cond = icmp ult i64 %iv.next, 100 + br i1 %exit.cond, label %loop.header, label %exit + +exit: + ret void +}