469 changes: 469 additions & 0 deletions llvm/lib/CodeGen/ExpandVectorPredication.cpp

Large diffs are not rendered by default.

5 changes: 5 additions & 0 deletions llvm/lib/CodeGen/TargetPassConfig.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -864,6 +864,11 @@ void TargetPassConfig::addIRPasses() {
if (getOptLevel() != CodeGenOpt::None && !DisablePartialLibcallInlining)
addPass(createPartiallyInlineLibCallsPass());

// Expand vector predication intrinsics into standard IR instructions.
// This pass has to run before ScalarizeMaskedMemIntrin and ExpandReduction
// passes since it emits those kinds of intrinsics.
addPass(createExpandVectorPredicationPass());

// Add scalarization of target's unsupported masked memory intrinsics pass.
// the unsupported intrinsic will be replaced with a chain of basic blocks,
// that stores/loads element one-by-one if the appropriate mask bit is set.
Expand Down
10 changes: 10 additions & 0 deletions llvm/lib/IR/IntrinsicInst.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -279,13 +279,23 @@ Value *VPIntrinsic::getMaskParam() const {
return nullptr;
}

void VPIntrinsic::setMaskParam(Value *NewMask) {
auto MaskPos = GetMaskParamPos(getIntrinsicID());
setArgOperand(*MaskPos, NewMask);
}

Value *VPIntrinsic::getVectorLengthParam() const {
auto vlenPos = GetVectorLengthParamPos(getIntrinsicID());
if (vlenPos)
return getArgOperand(vlenPos.getValue());
return nullptr;
}

void VPIntrinsic::setVectorLengthParam(Value *NewEVL) {
auto EVLPos = GetVectorLengthParamPos(getIntrinsicID());
setArgOperand(*EVLPos, NewEVL);
}

Optional<int> VPIntrinsic::GetMaskParamPos(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
default:
Expand Down
8 changes: 8 additions & 0 deletions llvm/lib/Target/VE/VETargetTransformInfo.h
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,14 @@ class VETTIImpl : public BasicTTIImplBase<VETTIImpl> {
llvm_unreachable("Unsupported register kind");
}

/// \returns How the target needs this vector-predicated operation to be
/// transformed.
TargetTransformInfo::VPLegalization
getVPLegalizationStrategy(const VPIntrinsic &PI) const {
using VPLegalization = TargetTransformInfo::VPLegalization;
return VPLegalization(VPLegalization::Legal, VPLegalization::Legal);
}

unsigned getMinVectorRegisterBitWidth() const {
// TODO report vregs once vector isel is stable.
return 0;
Expand Down
1 change: 1 addition & 0 deletions llvm/test/CodeGen/AArch64/O0-pipeline.ll
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
; CHECK-NEXT: Shadow Stack GC Lowering
; CHECK-NEXT: Lower constant intrinsics
; CHECK-NEXT: Remove unreachable blocks from the CFG
; CHECK-NEXT: Expand vector predication intrinsics
; CHECK-NEXT: Scalarize Masked Memory Intrinsics
; CHECK-NEXT: Expand reduction intrinsics
; CHECK-NEXT: AArch64 Stack Tagging
Expand Down
1 change: 1 addition & 0 deletions llvm/test/CodeGen/AArch64/O3-pipeline.ll
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
; CHECK-NEXT: Constant Hoisting
; CHECK-NEXT: Replace intrinsics with calls to vector library
; CHECK-NEXT: Partially inline calls to library functions
; CHECK-NEXT: Expand vector predication intrinsics
; CHECK-NEXT: Scalarize Masked Memory Intrinsics
; CHECK-NEXT: Expand reduction intrinsics
; CHECK-NEXT: Stack Safety Analysis
Expand Down
1 change: 1 addition & 0 deletions llvm/test/CodeGen/ARM/O3-pipeline.ll
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
; CHECK-NEXT: Constant Hoisting
; CHECK-NEXT: Replace intrinsics with calls to vector library
; CHECK-NEXT: Partially inline calls to library functions
; CHECK-NEXT: Expand vector predication intrinsics
; CHECK-NEXT: Scalarize Masked Memory Intrinsics
; CHECK-NEXT: Expand reduction intrinsics
; CHECK-NEXT: Natural Loop Information
Expand Down
245 changes: 245 additions & 0 deletions llvm/test/CodeGen/Generic/expand-vp.ll

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions llvm/test/CodeGen/X86/O0-pipeline.ll
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
; CHECK-NEXT: Shadow Stack GC Lowering
; CHECK-NEXT: Lower constant intrinsics
; CHECK-NEXT: Remove unreachable blocks from the CFG
; CHECK-NEXT: Expand vector predication intrinsics
; CHECK-NEXT: Scalarize Masked Memory Intrinsics
; CHECK-NEXT: Expand reduction intrinsics
; CHECK-NEXT: Expand indirectbr instructions
Expand Down
1 change: 1 addition & 0 deletions llvm/test/CodeGen/X86/opt-pipeline.ll
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
; CHECK-NEXT: Constant Hoisting
; CHECK-NEXT: Replace intrinsics with calls to vector library
; CHECK-NEXT: Partially inline calls to library functions
; CHECK-NEXT: Expand vector predication intrinsics
; CHECK-NEXT: Scalarize Masked Memory Intrinsics
; CHECK-NEXT: Expand reduction intrinsics
; CHECK-NEXT: Interleaved Access Pass
Expand Down
1 change: 1 addition & 0 deletions llvm/tools/llc/llc.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,7 @@ int main(int argc, char **argv) {
initializeVectorization(*Registry);
initializeScalarizeMaskedMemIntrinLegacyPassPass(*Registry);
initializeExpandReductionsPass(*Registry);
initializeExpandVectorPredicationPass(*Registry);
initializeHardwareLoopsPass(*Registry);
initializeTransformUtils(*Registry);
initializeReplaceWithVeclibLegacyPass(*Registry);
Expand Down
3 changes: 2 additions & 1 deletion llvm/tools/opt/opt.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -513,7 +513,7 @@ static bool shouldPinPassToLegacyPM(StringRef Pass) {
"safe-stack", "cost-model",
"codegenprepare", "interleaved-load-combine",
"unreachableblockelim", "verify-safepoint-ir",
"atomic-expand",
"atomic-expand", "expandvp",
"hardware-loops", "type-promotion",
"mve-tail-predication", "interleaved-access",
"global-merge", "pre-isel-intrinsic-lowering",
Expand Down Expand Up @@ -591,6 +591,7 @@ int main(int argc, char **argv) {
initializePostInlineEntryExitInstrumenterPass(Registry);
initializeUnreachableBlockElimLegacyPassPass(Registry);
initializeExpandReductionsPass(Registry);
initializeExpandVectorPredicationPass(Registry);
initializeWasmEHPreparePass(Registry);
initializeWriteBitcodePassPass(Registry);
initializeHardwareLoopsPass(Registry);
Expand Down