diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index 4cd607c0d0c8d..c7876241c0f0e 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -770,6 +770,45 @@ class TargetTransformInfoImplBase { virtual InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I = nullptr) const { + if (Opcode == Instruction::Switch && CostKind == TTI::TCK_CodeSize && I) { + const SwitchInst *SI = cast(I); + unsigned JumpTableSize, NumSuccs = I->getNumSuccessors(); + auto BrCost = getCFInstrCost(Instruction::Br, CostKind); + if (SI->defaultDestUnreachable()) + NumSuccs--; + + // An unreachable switch + if (NumSuccs == 0) + return TTI::TCC_Free; + + // A trivial unconditional branch. + if (NumSuccs == 1) + return BrCost; + + getEstimatedNumberOfCaseClusters(*SI, JumpTableSize, nullptr, nullptr); + + Type *BoolTy = IntegerType::get(SI->getContext(), 1); + Type *CondTy = SI->getCondition()->getType(); + auto CmpCost = getCmpSelInstrCost( + BinaryOperator::ICmp, BoolTy, CondTy, CmpInst::ICMP_UGT, CostKind, + {TTI::OK_AnyValue, TTI::OP_None}, + {TTI::OK_UniformConstantValue, TTI::OP_None}, nullptr); + + // Assume that lowering the switch block is implemented by binary search + // if no jump table is generated. + if (JumpTableSize == 0) + return llvm::Log2_32_Ceil(NumSuccs) * (CmpCost + BrCost); + + // Cost for jump table: load + jump + default compare + default jump + Type *EntryTy = PointerType::get(SI->getContext(), 0); + Align Alignment = DL.getABITypeAlign(EntryTy); + auto LoadCost = + getMemoryOpCost(Instruction::Load, EntryTy, Alignment, 0, CostKind, + {TTI::OK_AnyValue, TTI::OP_None}, nullptr); + return LoadCost + BrCost + + (SI->defaultDestUnreachable() ? 0 : (CmpCost + BrCost)); + } + // A phi would be free, unless we're costing the throughput because it // will require a register. if (Opcode == Instruction::PHI && CostKind != TTI::TCK_RecipThroughput) diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index 3d8d0a236a3c1..1cdb4397a4683 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -6155,6 +6155,9 @@ X86TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, InstructionCost X86TTIImpl::getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I) const { + if (Opcode == Instruction::Switch) + return BaseT::getCFInstrCost(Opcode, CostKind, I); + if (CostKind != TTI::TCK_RecipThroughput) return Opcode == Instruction::PHI ? TTI::TCC_Free : TTI::TCC_Basic; // Branches are assumed to be predicted. diff --git a/llvm/test/Analysis/CostModel/X86/switch.ll b/llvm/test/Analysis/CostModel/X86/switch.ll new file mode 100644 index 0000000000000..ae0f6fe78d8d0 --- /dev/null +++ b/llvm/test/Analysis/CostModel/X86/switch.ll @@ -0,0 +1,547 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6 +; RUN: opt < %s -passes="print" -cost-kind=throughput 2>&1 -disable-output -mtriple=x86_64-unknown-linux-gnu | FileCheck %s -check-prefixes=CHECK,THROUGHPUT +; RUN: opt < %s -passes="print" -cost-kind=latency 2>&1 -disable-output -mtriple=x86_64-unknown-linux-gnu | FileCheck %s -check-prefixes=CHECK,LATENCY +; RUN: opt < %s -passes="print" -cost-kind=code-size 2>&1 -disable-output -mtriple=x86_64-unknown-linux-gnu | FileCheck %s -check-prefixes=CHECK,CODESIZE + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" + +define i32 @single_succ_switch(i32 %x) { +; THROUGHPUT-LABEL: 'single_succ_switch' +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [ +; THROUGHPUT-NEXT: ] +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 1 +; +; LATENCY-LABEL: 'single_succ_switch' +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [ +; LATENCY-NEXT: ] +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1 +; +; CODESIZE-LABEL: 'single_succ_switch' +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [ +; CODESIZE-NEXT: ] +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1 +; +entry: + switch i32 %x, label %default [ + ] +default: + ret i32 1 +} + +define i32 @dense_switch(i32 %x) { +; THROUGHPUT-LABEL: 'dense_switch' +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [ +; THROUGHPUT-NEXT: i32 0, label %bb0 +; THROUGHPUT-NEXT: i32 1, label %bb1 +; THROUGHPUT-NEXT: i32 2, label %bb2 +; THROUGHPUT-NEXT: i32 3, label %bb3 +; THROUGHPUT-NEXT: i32 4, label %bb4 +; THROUGHPUT-NEXT: ] +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 0 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 1 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 2 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 3 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 4 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable +; +; LATENCY-LABEL: 'dense_switch' +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [ +; LATENCY-NEXT: i32 0, label %bb0 +; LATENCY-NEXT: i32 1, label %bb1 +; LATENCY-NEXT: i32 2, label %bb2 +; LATENCY-NEXT: i32 3, label %bb3 +; LATENCY-NEXT: i32 4, label %bb4 +; LATENCY-NEXT: ] +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable +; +; CODESIZE-LABEL: 'dense_switch' +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: switch i32 %x, label %default [ +; CODESIZE-NEXT: i32 0, label %bb0 +; CODESIZE-NEXT: i32 1, label %bb1 +; CODESIZE-NEXT: i32 2, label %bb2 +; CODESIZE-NEXT: i32 3, label %bb3 +; CODESIZE-NEXT: i32 4, label %bb4 +; CODESIZE-NEXT: ] +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable +; +entry: + switch i32 %x, label %default [ + i32 0, label %bb0 + i32 1, label %bb1 + i32 2, label %bb2 + i32 3, label %bb3 + i32 4, label %bb4 + ] +bb0: + ret i32 0 +bb1: + ret i32 1 +bb2: + ret i32 2 +bb3: + ret i32 3 +bb4: + ret i32 4 +default: + unreachable +} + +define i32 @dense_switch_reachable_default(i32 %x) { +; THROUGHPUT-LABEL: 'dense_switch_reachable_default' +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [ +; THROUGHPUT-NEXT: i32 0, label %bb0 +; THROUGHPUT-NEXT: i32 1, label %bb1 +; THROUGHPUT-NEXT: i32 2, label %bb2 +; THROUGHPUT-NEXT: i32 3, label %bb3 +; THROUGHPUT-NEXT: i32 4, label %bb4 +; THROUGHPUT-NEXT: ] +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 0 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 1 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 2 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 3 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 4 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 5 +; +; LATENCY-LABEL: 'dense_switch_reachable_default' +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [ +; LATENCY-NEXT: i32 0, label %bb0 +; LATENCY-NEXT: i32 1, label %bb1 +; LATENCY-NEXT: i32 2, label %bb2 +; LATENCY-NEXT: i32 3, label %bb3 +; LATENCY-NEXT: i32 4, label %bb4 +; LATENCY-NEXT: ] +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 5 +; +; CODESIZE-LABEL: 'dense_switch_reachable_default' +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: switch i32 %x, label %default [ +; CODESIZE-NEXT: i32 0, label %bb0 +; CODESIZE-NEXT: i32 1, label %bb1 +; CODESIZE-NEXT: i32 2, label %bb2 +; CODESIZE-NEXT: i32 3, label %bb3 +; CODESIZE-NEXT: i32 4, label %bb4 +; CODESIZE-NEXT: ] +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 5 +; +entry: + switch i32 %x, label %default [ + i32 0, label %bb0 + i32 1, label %bb1 + i32 2, label %bb2 + i32 3, label %bb3 + i32 4, label %bb4 + ] +bb0: + ret i32 0 +bb1: + ret i32 1 +bb2: + ret i32 2 +bb3: + ret i32 3 +bb4: + ret i32 4 +default: + ret i32 5 +} + +define i32 @sparse_switch(i32 %x) { +; THROUGHPUT-LABEL: 'sparse_switch' +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [ +; THROUGHPUT-NEXT: i32 0, label %bb0 +; THROUGHPUT-NEXT: i32 100, label %bb1 +; THROUGHPUT-NEXT: i32 200, label %bb2 +; THROUGHPUT-NEXT: i32 300, label %bb3 +; THROUGHPUT-NEXT: i32 400, label %bb4 +; THROUGHPUT-NEXT: ] +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 0 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 1 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 2 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 3 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 4 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable +; +; LATENCY-LABEL: 'sparse_switch' +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [ +; LATENCY-NEXT: i32 0, label %bb0 +; LATENCY-NEXT: i32 100, label %bb1 +; LATENCY-NEXT: i32 200, label %bb2 +; LATENCY-NEXT: i32 300, label %bb3 +; LATENCY-NEXT: i32 400, label %bb4 +; LATENCY-NEXT: ] +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable +; +; CODESIZE-LABEL: 'sparse_switch' +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: switch i32 %x, label %default [ +; CODESIZE-NEXT: i32 0, label %bb0 +; CODESIZE-NEXT: i32 100, label %bb1 +; CODESIZE-NEXT: i32 200, label %bb2 +; CODESIZE-NEXT: i32 300, label %bb3 +; CODESIZE-NEXT: i32 400, label %bb4 +; CODESIZE-NEXT: ] +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable +; +entry: + switch i32 %x, label %default [ + i32 0, label %bb0 + i32 100, label %bb1 + i32 200, label %bb2 + i32 300, label %bb3 + i32 400, label %bb4 + ] +bb0: + ret i32 0 +bb1: + ret i32 1 +bb2: + ret i32 2 +bb3: + ret i32 3 +bb4: + ret i32 4 +default: + unreachable +} + +define i32 @sparse_switch_reachable_default(i32 %x) { +; THROUGHPUT-LABEL: 'sparse_switch_reachable_default' +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [ +; THROUGHPUT-NEXT: i32 0, label %bb0 +; THROUGHPUT-NEXT: i32 100, label %bb1 +; THROUGHPUT-NEXT: i32 200, label %bb2 +; THROUGHPUT-NEXT: i32 300, label %bb3 +; THROUGHPUT-NEXT: i32 400, label %bb4 +; THROUGHPUT-NEXT: ] +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 0 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 1 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 2 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 3 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 4 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 5 +; +; LATENCY-LABEL: 'sparse_switch_reachable_default' +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [ +; LATENCY-NEXT: i32 0, label %bb0 +; LATENCY-NEXT: i32 100, label %bb1 +; LATENCY-NEXT: i32 200, label %bb2 +; LATENCY-NEXT: i32 300, label %bb3 +; LATENCY-NEXT: i32 400, label %bb4 +; LATENCY-NEXT: ] +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 5 +; +; CODESIZE-LABEL: 'sparse_switch_reachable_default' +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: switch i32 %x, label %default [ +; CODESIZE-NEXT: i32 0, label %bb0 +; CODESIZE-NEXT: i32 100, label %bb1 +; CODESIZE-NEXT: i32 200, label %bb2 +; CODESIZE-NEXT: i32 300, label %bb3 +; CODESIZE-NEXT: i32 400, label %bb4 +; CODESIZE-NEXT: ] +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 5 +; +entry: + switch i32 %x, label %default [ + i32 0, label %bb0 + i32 100, label %bb1 + i32 200, label %bb2 + i32 300, label %bb3 + i32 400, label %bb4 + ] +bb0: + ret i32 0 +bb1: + ret i32 1 +bb2: + ret i32 2 +bb3: + ret i32 3 +bb4: + ret i32 4 +default: + ret i32 5 +} + +define i32 @dense_big_switch(i32 %x) { +; THROUGHPUT-LABEL: 'dense_big_switch' +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [ +; THROUGHPUT-NEXT: i32 0, label %bb0 +; THROUGHPUT-NEXT: i32 1, label %bb1 +; THROUGHPUT-NEXT: i32 2, label %bb2 +; THROUGHPUT-NEXT: i32 3, label %bb3 +; THROUGHPUT-NEXT: i32 4, label %bb4 +; THROUGHPUT-NEXT: i32 5, label %bb5 +; THROUGHPUT-NEXT: i32 6, label %bb6 +; THROUGHPUT-NEXT: i32 7, label %bb7 +; THROUGHPUT-NEXT: i32 8, label %bb8 +; THROUGHPUT-NEXT: i32 9, label %bb9 +; THROUGHPUT-NEXT: i32 10, label %bb10 +; THROUGHPUT-NEXT: ] +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 0 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 1 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 2 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 3 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 4 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 5 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 6 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 7 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 8 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 9 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 10 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable +; +; LATENCY-LABEL: 'dense_big_switch' +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [ +; LATENCY-NEXT: i32 0, label %bb0 +; LATENCY-NEXT: i32 1, label %bb1 +; LATENCY-NEXT: i32 2, label %bb2 +; LATENCY-NEXT: i32 3, label %bb3 +; LATENCY-NEXT: i32 4, label %bb4 +; LATENCY-NEXT: i32 5, label %bb5 +; LATENCY-NEXT: i32 6, label %bb6 +; LATENCY-NEXT: i32 7, label %bb7 +; LATENCY-NEXT: i32 8, label %bb8 +; LATENCY-NEXT: i32 9, label %bb9 +; LATENCY-NEXT: i32 10, label %bb10 +; LATENCY-NEXT: ] +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 5 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 6 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 7 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 8 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 9 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 10 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable +; +; CODESIZE-LABEL: 'dense_big_switch' +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: switch i32 %x, label %default [ +; CODESIZE-NEXT: i32 0, label %bb0 +; CODESIZE-NEXT: i32 1, label %bb1 +; CODESIZE-NEXT: i32 2, label %bb2 +; CODESIZE-NEXT: i32 3, label %bb3 +; CODESIZE-NEXT: i32 4, label %bb4 +; CODESIZE-NEXT: i32 5, label %bb5 +; CODESIZE-NEXT: i32 6, label %bb6 +; CODESIZE-NEXT: i32 7, label %bb7 +; CODESIZE-NEXT: i32 8, label %bb8 +; CODESIZE-NEXT: i32 9, label %bb9 +; CODESIZE-NEXT: i32 10, label %bb10 +; CODESIZE-NEXT: ] +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 5 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 6 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 7 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 8 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 9 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 10 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable +; +entry: + switch i32 %x, label %default [ + i32 0, label %bb0 + i32 1, label %bb1 + i32 2, label %bb2 + i32 3, label %bb3 + i32 4, label %bb4 + i32 5, label %bb5 + i32 6, label %bb6 + i32 7, label %bb7 + i32 8, label %bb8 + i32 9, label %bb9 + i32 10, label %bb10 + ] +bb0: + ret i32 0 +bb1: + ret i32 1 +bb2: + ret i32 2 +bb3: + ret i32 3 +bb4: + ret i32 4 +bb5: + ret i32 5 +bb6: + ret i32 6 +bb7: + ret i32 7 +bb8: + ret i32 8 +bb9: + ret i32 9 +bb10: + ret i32 10 +default: + unreachable +} + +define i32 @sparse_big_switch(i32 %x) { +; THROUGHPUT-LABEL: 'sparse_big_switch' +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [ +; THROUGHPUT-NEXT: i32 0, label %bb0 +; THROUGHPUT-NEXT: i32 100, label %bb1 +; THROUGHPUT-NEXT: i32 200, label %bb2 +; THROUGHPUT-NEXT: i32 300, label %bb3 +; THROUGHPUT-NEXT: i32 400, label %bb4 +; THROUGHPUT-NEXT: i32 500, label %bb5 +; THROUGHPUT-NEXT: i32 600, label %bb6 +; THROUGHPUT-NEXT: i32 700, label %bb7 +; THROUGHPUT-NEXT: i32 800, label %bb8 +; THROUGHPUT-NEXT: i32 900, label %bb9 +; THROUGHPUT-NEXT: i32 1000, label %bb10 +; THROUGHPUT-NEXT: ] +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 0 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 1 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 2 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 3 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 4 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 5 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 6 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 7 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 8 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 9 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 10 +; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable +; +; LATENCY-LABEL: 'sparse_big_switch' +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [ +; LATENCY-NEXT: i32 0, label %bb0 +; LATENCY-NEXT: i32 100, label %bb1 +; LATENCY-NEXT: i32 200, label %bb2 +; LATENCY-NEXT: i32 300, label %bb3 +; LATENCY-NEXT: i32 400, label %bb4 +; LATENCY-NEXT: i32 500, label %bb5 +; LATENCY-NEXT: i32 600, label %bb6 +; LATENCY-NEXT: i32 700, label %bb7 +; LATENCY-NEXT: i32 800, label %bb8 +; LATENCY-NEXT: i32 900, label %bb9 +; LATENCY-NEXT: i32 1000, label %bb10 +; LATENCY-NEXT: ] +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 5 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 6 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 7 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 8 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 9 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 10 +; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable +; +; CODESIZE-LABEL: 'sparse_big_switch' +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: switch i32 %x, label %default [ +; CODESIZE-NEXT: i32 0, label %bb0 +; CODESIZE-NEXT: i32 100, label %bb1 +; CODESIZE-NEXT: i32 200, label %bb2 +; CODESIZE-NEXT: i32 300, label %bb3 +; CODESIZE-NEXT: i32 400, label %bb4 +; CODESIZE-NEXT: i32 500, label %bb5 +; CODESIZE-NEXT: i32 600, label %bb6 +; CODESIZE-NEXT: i32 700, label %bb7 +; CODESIZE-NEXT: i32 800, label %bb8 +; CODESIZE-NEXT: i32 900, label %bb9 +; CODESIZE-NEXT: i32 1000, label %bb10 +; CODESIZE-NEXT: ] +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 5 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 6 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 7 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 8 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 9 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 10 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable +; +entry: + switch i32 %x, label %default [ + i32 0, label %bb0 + i32 100, label %bb1 + i32 200, label %bb2 + i32 300, label %bb3 + i32 400, label %bb4 + i32 500, label %bb5 + i32 600, label %bb6 + i32 700, label %bb7 + i32 800, label %bb8 + i32 900, label %bb9 + i32 1000, label %bb10 + ] +bb0: + ret i32 0 +bb1: + ret i32 1 +bb2: + ret i32 2 +bb3: + ret i32 3 +bb4: + ret i32 4 +bb5: + ret i32 5 +bb6: + ret i32 6 +bb7: + ret i32 7 +bb8: + ret i32 8 +bb9: + ret i32 9 +bb10: + ret i32 10 +default: + unreachable +} +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; CHECK: {{.*}} diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll index 2aceb279d47db..b1ae363fdfaaa 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll @@ -288,9 +288,46 @@ define void @switch_all_dests_distinct(ptr %start, ptr %end) { ; COST-LABEL: define void @switch_all_dests_distinct( ; COST-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] { ; COST-NEXT: [[ENTRY:.*]]: +; COST-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64 +; COST-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64 +; COST-NEXT: [[TMP0:%.*]] = add i64 [[END1]], -8 +; COST-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]] +; COST-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 3 +; COST-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 +; COST-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4 +; COST-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; COST: [[VECTOR_PH]]: +; COST-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4 +; COST-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]] +; COST-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 8 +; COST-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]] +; COST-NEXT: br label %[[VECTOR_BODY:.*]] +; COST: [[VECTOR_BODY]]: +; COST-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; COST-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8 +; COST-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]] +; COST-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[NEXT_GEP]], align 1 +; COST-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 -12) +; COST-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 13) +; COST-NEXT: [[TMP8:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], zeroinitializer +; COST-NEXT: [[TMP9:%.*]] = or <4 x i1> [[TMP6]], [[TMP7]] +; COST-NEXT: [[TMP10:%.*]] = or <4 x i1> [[TMP9]], [[TMP8]] +; COST-NEXT: [[TMP11:%.*]] = xor <4 x i1> [[TMP10]], splat (i1 true) +; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 1), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP8]]) +; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP7]]) +; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP6]]) +; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP11]]) +; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; COST-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; COST-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; COST: [[MIDDLE_BLOCK]]: +; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] +; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] +; COST: [[SCALAR_PH]]: +; COST-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ] ; COST-NEXT: br label %[[LOOP_HEADER:.*]] ; COST: [[LOOP_HEADER]]: -; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] ; COST-NEXT: [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1 ; COST-NEXT: switch i64 [[L]], label %[[DEFAULT:.*]] [ ; COST-NEXT: i64 -12, label %[[IF_THEN_1:.*]] @@ -312,7 +349,7 @@ define void @switch_all_dests_distinct(ptr %start, ptr %end) { ; COST: [[LOOP_LATCH]]: ; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1 ; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]] -; COST-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]] +; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP7:![0-9]+]] ; COST: [[EXIT]]: ; COST-NEXT: ret void ; @@ -469,7 +506,7 @@ define void @switch_all_dests_distinct_variant_using_branches(ptr %start, ptr %e ; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP7]]) ; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; COST-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; COST-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; COST-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; COST: [[MIDDLE_BLOCK]]: ; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] ; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] @@ -502,7 +539,7 @@ define void @switch_all_dests_distinct_variant_using_branches(ptr %start, ptr %e ; COST: [[LOOP_LATCH]]: ; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1 ; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]] -; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP7:![0-9]+]] +; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP9:![0-9]+]] ; COST: [[EXIT]]: ; COST-NEXT: ret void ; @@ -639,9 +676,49 @@ define void @switch_multiple_common_dests(ptr %start, ptr %end) { ; COST-LABEL: define void @switch_multiple_common_dests( ; COST-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] { ; COST-NEXT: [[ENTRY:.*]]: +; COST-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64 +; COST-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64 +; COST-NEXT: [[TMP0:%.*]] = add i64 [[END1]], -8 +; COST-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]] +; COST-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 3 +; COST-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 +; COST-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4 +; COST-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; COST: [[VECTOR_PH]]: +; COST-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4 +; COST-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]] +; COST-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 8 +; COST-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]] +; COST-NEXT: br label %[[VECTOR_BODY:.*]] +; COST: [[VECTOR_BODY]]: +; COST-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; COST-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8 +; COST-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]] +; COST-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[NEXT_GEP]], align 1 +; COST-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 -12) +; COST-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], zeroinitializer +; COST-NEXT: [[TMP8:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 13) +; COST-NEXT: [[TMP9:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 14) +; COST-NEXT: [[TMP10:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 15) +; COST-NEXT: [[TMP11:%.*]] = or <4 x i1> [[TMP6]], [[TMP7]] +; COST-NEXT: [[TMP12:%.*]] = or <4 x i1> [[TMP8]], [[TMP9]] +; COST-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP12]], [[TMP10]] +; COST-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP11]], [[TMP13]] +; COST-NEXT: [[TMP15:%.*]] = xor <4 x i1> [[TMP14]], splat (i1 true) +; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP13]]) +; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP11]]) +; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP15]]) +; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; COST-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; COST-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] +; COST: [[MIDDLE_BLOCK]]: +; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] +; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] +; COST: [[SCALAR_PH]]: +; COST-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ] ; COST-NEXT: br label %[[LOOP_HEADER:.*]] ; COST: [[LOOP_HEADER]]: -; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] ; COST-NEXT: [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1 ; COST-NEXT: switch i64 [[L]], label %[[DEFAULT:.*]] [ ; COST-NEXT: i64 -12, label %[[IF_THEN_1:.*]] @@ -662,7 +739,7 @@ define void @switch_multiple_common_dests(ptr %start, ptr %end) { ; COST: [[LOOP_LATCH]]: ; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1 ; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]] -; COST-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]] +; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP11:![0-9]+]] ; COST: [[EXIT]]: ; COST-NEXT: ret void ; @@ -790,9 +867,43 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) { ; COST-LABEL: define void @switch4_default_common_dest_with_case( ; COST-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] { ; COST-NEXT: [[ENTRY:.*]]: +; COST-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64 +; COST-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64 +; COST-NEXT: [[TMP0:%.*]] = add i64 [[END1]], -8 +; COST-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]] +; COST-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 3 +; COST-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 +; COST-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4 +; COST-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; COST: [[VECTOR_PH]]: +; COST-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4 +; COST-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]] +; COST-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 8 +; COST-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]] +; COST-NEXT: br label %[[VECTOR_BODY:.*]] +; COST: [[VECTOR_BODY]]: +; COST-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; COST-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8 +; COST-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]] +; COST-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[NEXT_GEP]], align 1 +; COST-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 -12) +; COST-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 13) +; COST-NEXT: [[TMP8:%.*]] = or <4 x i1> [[TMP6]], [[TMP7]] +; COST-NEXT: [[TMP9:%.*]] = xor <4 x i1> [[TMP8]], splat (i1 true) +; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP7]]) +; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP6]]) +; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP9]]) +; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; COST-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; COST-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] +; COST: [[MIDDLE_BLOCK]]: +; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] +; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] +; COST: [[SCALAR_PH]]: +; COST-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ] ; COST-NEXT: br label %[[LOOP_HEADER:.*]] ; COST: [[LOOP_HEADER]]: -; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] ; COST-NEXT: [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1 ; COST-NEXT: switch i64 [[L]], label %[[DEFAULT:.*]] [ ; COST-NEXT: i64 -12, label %[[IF_THEN_1:.*]] @@ -811,7 +922,7 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) { ; COST: [[LOOP_LATCH]]: ; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1 ; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]] -; COST-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]] +; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP13:![0-9]+]] ; COST: [[EXIT]]: ; COST-NEXT: ret void ; @@ -957,7 +1068,7 @@ define void @switch_under_br_default_common_dest_with_case(ptr %start, ptr %end, ; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP14]]) ; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; COST-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; COST-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] +; COST-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] ; COST: [[MIDDLE_BLOCK]]: ; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] ; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] @@ -987,7 +1098,7 @@ define void @switch_under_br_default_common_dest_with_case(ptr %start, ptr %end, ; COST: [[LOOP_LATCH]]: ; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1 ; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]] -; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP9:![0-9]+]] +; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP15:![0-9]+]] ; COST: [[EXIT]]: ; COST-NEXT: ret void ; @@ -1116,9 +1227,51 @@ define void @br_under_switch_default_common_dest_with_case(ptr %start, ptr %end, ; COST-LABEL: define void @br_under_switch_default_common_dest_with_case( ; COST-SAME: ptr [[START:%.*]], ptr [[END:%.*]], i64 [[X:%.*]]) #[[ATTR0]] { ; COST-NEXT: [[ENTRY:.*]]: +; COST-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64 +; COST-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64 +; COST-NEXT: [[TMP0:%.*]] = add i64 [[END1]], -8 +; COST-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]] +; COST-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 3 +; COST-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 +; COST-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4 +; COST-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; COST: [[VECTOR_PH]]: +; COST-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4 +; COST-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]] +; COST-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 8 +; COST-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]] +; COST-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0 +; COST-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer +; COST-NEXT: br label %[[VECTOR_BODY:.*]] +; COST: [[VECTOR_BODY]]: +; COST-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; COST-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8 +; COST-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]] +; COST-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[NEXT_GEP]], align 1 +; COST-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 -12) +; COST-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 13) +; COST-NEXT: [[TMP8:%.*]] = or <4 x i1> [[TMP6]], [[TMP7]] +; COST-NEXT: [[TMP9:%.*]] = xor <4 x i1> [[TMP8]], splat (i1 true) +; COST-NEXT: [[TMP10:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; COST-NEXT: [[TMP11:%.*]] = xor <4 x i1> [[TMP10]], splat (i1 true) +; COST-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP11]], <4 x i1> zeroinitializer +; COST-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP12]], [[TMP7]] +; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP13]]) +; COST-NEXT: [[TMP14:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP10]], <4 x i1> zeroinitializer +; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP14]]) +; COST-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP14]], [[TMP9]] +; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP15]]) +; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; COST-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; COST-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] +; COST: [[MIDDLE_BLOCK]]: +; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] +; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] +; COST: [[SCALAR_PH]]: +; COST-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ] ; COST-NEXT: br label %[[LOOP_HEADER:.*]] ; COST: [[LOOP_HEADER]]: -; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] ; COST-NEXT: [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1 ; COST-NEXT: switch i64 [[L]], label %[[DEFAULT:.*]] [ ; COST-NEXT: i64 -12, label %[[IF_THEN_1:.*]] @@ -1140,7 +1293,7 @@ define void @br_under_switch_default_common_dest_with_case(ptr %start, ptr %end, ; COST: [[LOOP_LATCH]]: ; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1 ; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]] -; COST-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]] +; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP17:![0-9]+]] ; COST: [[EXIT]]: ; COST-NEXT: ret void ; @@ -1433,6 +1586,14 @@ exit: ; COST: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]} ; COST: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]} ; COST: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]} +; COST: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]} +; COST: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]} +; COST: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]} +; COST: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]} +; COST: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]} +; COST: [[LOOP15]] = distinct !{[[LOOP15]], [[META2]], [[META1]]} +; COST: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]], [[META2]]} +; COST: [[LOOP17]] = distinct !{[[LOOP17]], [[META2]], [[META1]]} ;. ; FORCED: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} ; FORCED: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll index dcfebe32302be..d8ce1d9be901e 100644 --- a/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll +++ b/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll @@ -12,8 +12,86 @@ define dso_local void @test(ptr %start, ptr %end) #0 { ; AVX-NEXT: entry: ; AVX-NEXT: [[I11_NOT1:%.*]] = icmp eq ptr [[START:%.*]], [[END:%.*]] ; AVX-NEXT: br i1 [[I11_NOT1]], label [[EXIT:%.*]], label [[BB12:%.*]] +; AVX: iter.check: +; AVX-NEXT: [[END3:%.*]] = ptrtoint ptr [[END]] to i64 +; AVX-NEXT: [[START4:%.*]] = ptrtoint ptr [[START]] to i64 +; AVX-NEXT: [[TMP0:%.*]] = add i64 [[END3]], -4 +; AVX-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START4]] +; AVX-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 2 +; AVX-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 +; AVX-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 28 +; AVX-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[BB12_PREHEADER:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]] +; AVX: vector.main.loop.iter.check: +; AVX-NEXT: [[MIN_ITERS_CHECK5:%.*]] = icmp ult i64 [[TMP1]], 124 +; AVX-NEXT: br i1 [[MIN_ITERS_CHECK5]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]] +; AVX: vector.ph: +; AVX-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP3]], 24 +; AVX-NEXT: [[N_VEC:%.*]] = and i64 [[TMP3]], 9223372036854775776 +; AVX-NEXT: br label [[VECTOR_BODY:%.*]] +; AVX: vector.body: +; AVX-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; AVX-NEXT: [[TMP4:%.*]] = shl i64 [[INDEX]], 2 +; AVX-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]] +; AVX-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 32 +; AVX-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 64 +; AVX-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 96 +; AVX-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[NEXT_GEP]], align 4 +; AVX-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i32>, ptr [[TMP5]], align 4 +; AVX-NEXT: [[WIDE_LOAD7:%.*]] = load <8 x i32>, ptr [[TMP6]], align 4 +; AVX-NEXT: [[WIDE_LOAD8:%.*]] = load <8 x i32>, ptr [[TMP7]], align 4 +; AVX-NEXT: [[TMP8:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD]], splat (i32 -12) +; AVX-NEXT: [[TMP9:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD6]], splat (i32 -12) +; AVX-NEXT: [[TMP10:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD7]], splat (i32 -12) +; AVX-NEXT: [[TMP11:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD8]], splat (i32 -12) +; AVX-NEXT: [[TMP12:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD]], splat (i32 13) +; AVX-NEXT: [[TMP13:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD6]], splat (i32 13) +; AVX-NEXT: [[TMP14:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD7]], splat (i32 13) +; AVX-NEXT: [[TMP15:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD8]], splat (i32 13) +; AVX-NEXT: [[TMP16:%.*]] = or <8 x i1> [[TMP8]], [[TMP12]] +; AVX-NEXT: [[TMP17:%.*]] = or <8 x i1> [[TMP9]], [[TMP13]] +; AVX-NEXT: [[TMP18:%.*]] = or <8 x i1> [[TMP10]], [[TMP14]] +; AVX-NEXT: [[TMP19:%.*]] = or <8 x i1> [[TMP11]], [[TMP15]] +; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr align 4 [[NEXT_GEP]], <8 x i1> [[TMP16]]) +; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr align 4 [[TMP5]], <8 x i1> [[TMP17]]) +; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr align 4 [[TMP6]], <8 x i1> [[TMP18]]) +; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr align 4 [[TMP7]], <8 x i1> [[TMP19]]) +; AVX-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32 +; AVX-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; AVX-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; AVX: middle.block: +; AVX-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] +; AVX-NEXT: br i1 [[CMP_N]], label [[EXIT]], label [[VEC_EPILOG_ITER_CHECK:%.*]] +; AVX: vec.epilog.iter.check: +; AVX-NEXT: [[TMP21:%.*]] = shl i64 [[N_VEC]], 2 +; AVX-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP21]] +; AVX-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 +; AVX-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[BB12_PREHEADER]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]] +; AVX: vec.epilog.ph: +; AVX-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ] +; AVX-NEXT: [[N_VEC10:%.*]] = and i64 [[TMP3]], 9223372036854775800 +; AVX-NEXT: [[TMP22:%.*]] = shl i64 [[N_VEC10]], 2 +; AVX-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP22]] +; AVX-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]] +; AVX: vec.epilog.vector.body: +; AVX-NEXT: [[INDEX11:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT14:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ] +; AVX-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX11]], 2 +; AVX-NEXT: [[NEXT_GEP12:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]] +; AVX-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x i32>, ptr [[NEXT_GEP12]], align 4 +; AVX-NEXT: [[TMP24:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD13]], splat (i32 -12) +; AVX-NEXT: [[TMP25:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD13]], splat (i32 13) +; AVX-NEXT: [[TMP26:%.*]] = or <8 x i1> [[TMP24]], [[TMP25]] +; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr align 4 [[NEXT_GEP12]], <8 x i1> [[TMP26]]) +; AVX-NEXT: [[INDEX_NEXT14]] = add nuw i64 [[INDEX11]], 8 +; AVX-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT14]], [[N_VEC10]] +; AVX-NEXT: br i1 [[TMP27]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; AVX: vec.epilog.middle.block: +; AVX-NEXT: [[CMP_N15:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC10]] +; AVX-NEXT: br i1 [[CMP_N15]], label [[EXIT]], label [[BB12_PREHEADER]] +; AVX: bb12.preheader: +; AVX-NEXT: [[PTR2_PH:%.*]] = phi ptr [ [[START]], [[BB12]] ], [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[TMP23]], [[VEC_EPILOG_MIDDLE_BLOCK]] ] +; AVX-NEXT: br label [[BB13:%.*]] ; AVX: bb12: -; AVX-NEXT: [[PTR2:%.*]] = phi ptr [ [[PTR_NEXT:%.*]], [[LATCH:%.*]] ], [ [[START]], [[ENTRY:%.*]] ] +; AVX-NEXT: [[PTR2:%.*]] = phi ptr [ [[PTR_NEXT:%.*]], [[LATCH:%.*]] ], [ [[PTR2_PH]], [[BB12_PREHEADER]] ] ; AVX-NEXT: [[VAL:%.*]] = load i32, ptr [[PTR2]], align 4 ; AVX-NEXT: switch i32 [[VAL]], label [[LATCH]] [ ; AVX-NEXT: i32 -12, label [[STORE:%.*]] @@ -25,7 +103,7 @@ define dso_local void @test(ptr %start, ptr %end) #0 { ; AVX: latch: ; AVX-NEXT: [[PTR_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR2]], i64 4 ; AVX-NEXT: [[I11_NOT:%.*]] = icmp eq ptr [[PTR_NEXT]], [[END]] -; AVX-NEXT: br i1 [[I11_NOT]], label [[EXIT]], label [[BB12]] +; AVX-NEXT: br i1 [[I11_NOT]], label [[EXIT]], label [[BB13]], !llvm.loop [[LOOP5:![0-9]+]] ; AVX: exit: ; AVX-NEXT: ret void ;