Skip to content

Commit

Permalink
[LV] Move code from widenGEP to VPWidenGEPRecipe (NFC).
Browse files Browse the repository at this point in the history 
The code in widenGEP has already been transitioned to only rely on
information provided by VPWidenGEPRecipe directly.

Moving the code directly to VPWidenGEPRecipe::execute completes
the transition for the recipe.

It provides the following advantages:

1. Less indirection, easier to see what's going on.
2. Removes accesses to fields of ILV.

2) in particular ensures that no dependencies on
fields in ILV for GEP code generation are re-introduced.

Reviewed By: Ayal

Differential Revision: https://reviews.llvm.org/D114321
  • Loading branch information
@fhahn
fhahn committed Nov 28, 2021
1 parent c73fc74 commit 3495090
Showing 1 changed file with 84 additions and 101 deletions.
185 changes: 84 additions & 101 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View file
Expand Up @@ -496,13 +496,6 @@ class InnerLoopVectorizer {
/// new unrolled loop, where UF is the unroll factor.
using VectorParts = SmallVector<Value *, 2>;

/// Vectorize a single GetElementPtrInst based on information gathered and
/// decisions taken during planning.
void widenGEP(GetElementPtrInst *GEP, VPWidenGEPRecipe *WidenGEPRec,
VPUser &Indices, unsigned UF, ElementCount VF,
bool IsPtrLoopInvariant, SmallBitVector &IsIndexLoopInvariant,
VPTransformState &State);

/// Vectorize a single first-order recurrence or pointer induction PHINode in
/// a block. This method handles the induction variable canonicalization. It
/// supports both VF = 1 for unrolled loops and arbitrary length vectors.
Expand Down Expand Up @@ -567,6 +560,17 @@ class InnerLoopVectorizer {
/// element.
virtual Value *getBroadcastInstrs(Value *V);

/// Add metadata from one instruction to another.
///
/// This includes both the original MDs from \p From and additional ones (\see
/// addNewMetadata). Use this for *newly created* instructions in the vector
/// loop.
void addMetadata(Instruction *To, Instruction *From);

/// Similar to the previous function but it adds the metadata to a
/// vector of instructions.
void addMetadata(ArrayRef<Value *> To, Instruction *From);

protected:
friend class LoopVectorizationPlanner;

Expand Down Expand Up @@ -742,17 +746,6 @@ class InnerLoopVectorizer {
/// vector loop.
void addNewMetadata(Instruction *To, const Instruction *Orig);

/// Add metadata from one instruction to another.
///
/// This includes both the original MDs from \p From and additional ones (\see
/// addNewMetadata). Use this for *newly created* instructions in the vector
/// loop.
void addMetadata(Instruction *To, Instruction *From);

/// Similar to the previous function but it adds the metadata to a
/// vector of instructions.
void addMetadata(ArrayRef<Value *> To, Instruction *From);

/// Collect poison-generating recipes that may generate a poison value that is
/// used after vectorization, even when their operands are not poison. Those
/// recipes meet the following conditions:
Expand Down Expand Up @@ -4725,86 +4718,6 @@ bool InnerLoopVectorizer::useOrderedReductions(RecurrenceDescriptor &RdxDesc) {
return Cost->useOrderedReductions(RdxDesc);
}

void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP,
VPWidenGEPRecipe *WidenGEPRec,
VPUser &Operands, unsigned UF,
ElementCount VF, bool IsPtrLoopInvariant,
SmallBitVector &IsIndexLoopInvariant,
VPTransformState &State) {
// Construct a vector GEP by widening the operands of the scalar GEP as
// necessary. We mark the vector GEP 'inbounds' if appropriate. A GEP
// results in a vector of pointers when at least one operand of the GEP
// is vector-typed. Thus, to keep the representation compact, we only use
// vector-typed operands for loop-varying values.

if (VF.isVector() && IsPtrLoopInvariant && IsIndexLoopInvariant.all()) {
// If we are vectorizing, but the GEP has only loop-invariant operands,
// the GEP we build (by only using vector-typed operands for
// loop-varying values) would be a scalar pointer. Thus, to ensure we
// produce a vector of pointers, we need to either arbitrarily pick an
// operand to broadcast, or broadcast a clone of the original GEP.
// Here, we broadcast a clone of the original.
//
// TODO: If at some point we decide to scalarize instructions having
// loop-invariant operands, this special case will no longer be
// required. We would add the scalarization decision to
// collectLoopScalars() and teach getVectorValue() to broadcast
// the lane-zero scalar value.
auto *Clone = Builder.Insert(GEP->clone());
for (unsigned Part = 0; Part < UF; ++Part) {
Value *EntryPart = Builder.CreateVectorSplat(VF, Clone);
State.set(WidenGEPRec, EntryPart, Part);
addMetadata(EntryPart, GEP);
}
} else {
// If the GEP has at least one loop-varying operand, we are sure to
// produce a vector of pointers. But if we are only unrolling, we want
// to produce a scalar GEP for each unroll part. Thus, the GEP we
// produce with the code below will be scalar (if VF == 1) or vector
// (otherwise). Note that for the unroll-only case, we still maintain
// values in the vector mapping with initVector, as we do for other
// instructions.
for (unsigned Part = 0; Part < UF; ++Part) {
// The pointer operand of the new GEP. If it's loop-invariant, we
// won't broadcast it.
auto *Ptr = IsPtrLoopInvariant
? State.get(Operands.getOperand(0), VPIteration(0, 0))
: State.get(Operands.getOperand(0), Part);

// Collect all the indices for the new GEP. If any index is
// loop-invariant, we won't broadcast it.
SmallVector<Value *, 4> Indices;
for (unsigned I = 1, E = Operands.getNumOperands(); I < E; I++) {
VPValue *Operand = Operands.getOperand(I);
if (IsIndexLoopInvariant[I - 1])
Indices.push_back(State.get(Operand, VPIteration(0, 0)));
else
Indices.push_back(State.get(Operand, Part));
}

// If the GEP instruction is vectorized and was in a basic block that
// needed predication, we can't propagate the poison-generating 'inbounds'
// flag. The control flow has been linearized and the GEP is no longer
// guarded by the predicate, which could make the 'inbounds' properties to
// no longer hold.
bool IsInBounds = GEP->isInBounds() &&
State.MayGeneratePoisonRecipes.count(WidenGEPRec) == 0;

// Create the new GEP. Note that this GEP may be a scalar if VF == 1,
// but it should be a vector, otherwise.
auto *NewGEP =
IsInBounds
? Builder.CreateInBoundsGEP(GEP->getSourceElementType(), Ptr,
Indices)
: Builder.CreateGEP(GEP->getSourceElementType(), Ptr, Indices);
assert((VF.isScalar() || NewGEP->getType()->isVectorTy()) &&
"NewGEP is not a pointer vector");
State.set(WidenGEPRec, NewGEP, Part);
addMetadata(NewGEP, GEP);
}
}
}

void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
VPWidenPHIRecipe *PhiR,
VPTransformState &State) {
Expand Down Expand Up @@ -9875,9 +9788,79 @@ void VPWidenRecipe::execute(VPTransformState &State) {
}

void VPWidenGEPRecipe::execute(VPTransformState &State) {
State.ILV->widenGEP(cast<GetElementPtrInst>(getUnderlyingInstr()), this,
*this, State.UF, State.VF, IsPtrLoopInvariant,
IsIndexLoopInvariant, State);
auto *GEP = cast<GetElementPtrInst>(getUnderlyingInstr());
// Construct a vector GEP by widening the operands of the scalar GEP as
// necessary. We mark the vector GEP 'inbounds' if appropriate. A GEP
// results in a vector of pointers when at least one operand of the GEP
// is vector-typed. Thus, to keep the representation compact, we only use
// vector-typed operands for loop-varying values.

if (State.VF.isVector() && IsPtrLoopInvariant && IsIndexLoopInvariant.all()) {
// If we are vectorizing, but the GEP has only loop-invariant operands,
// the GEP we build (by only using vector-typed operands for
// loop-varying values) would be a scalar pointer. Thus, to ensure we
// produce a vector of pointers, we need to either arbitrarily pick an
// operand to broadcast, or broadcast a clone of the original GEP.
// Here, we broadcast a clone of the original.
//
// TODO: If at some point we decide to scalarize instructions having
// loop-invariant operands, this special case will no longer be
// required. We would add the scalarization decision to
// collectLoopScalars() and teach getVectorValue() to broadcast
// the lane-zero scalar value.
auto *Clone = State.Builder.Insert(GEP->clone());
for (unsigned Part = 0; Part < State.UF; ++Part) {
Value *EntryPart = State.Builder.CreateVectorSplat(State.VF, Clone);
State.set(this, EntryPart, Part);
State.ILV->addMetadata(EntryPart, GEP);
}
} else {
// If the GEP has at least one loop-varying operand, we are sure to
// produce a vector of pointers. But if we are only unrolling, we want
// to produce a scalar GEP for each unroll part. Thus, the GEP we
// produce with the code below will be scalar (if VF == 1) or vector
// (otherwise). Note that for the unroll-only case, we still maintain
// values in the vector mapping with initVector, as we do for other
// instructions.
for (unsigned Part = 0; Part < State.UF; ++Part) {
// The pointer operand of the new GEP. If it's loop-invariant, we
// won't broadcast it.
auto *Ptr = IsPtrLoopInvariant
? State.get(getOperand(0), VPIteration(0, 0))
: State.get(getOperand(0), Part);

// Collect all the indices for the new GEP. If any index is
// loop-invariant, we won't broadcast it.
SmallVector<Value *, 4> Indices;
for (unsigned I = 1, E = getNumOperands(); I < E; I++) {
VPValue *Operand = getOperand(I);
if (IsIndexLoopInvariant[I - 1])
Indices.push_back(State.get(Operand, VPIteration(0, 0)));
else
Indices.push_back(State.get(Operand, Part));
}

// If the GEP instruction is vectorized and was in a basic block that
// needed predication, we can't propagate the poison-generating 'inbounds'
// flag. The control flow has been linearized and the GEP is no longer
// guarded by the predicate, which could make the 'inbounds' properties to
// no longer hold.
bool IsInBounds =
GEP->isInBounds() && State.MayGeneratePoisonRecipes.count(this) == 0;

// Create the new GEP. Note that this GEP may be a scalar if VF == 1,
// but it should be a vector, otherwise.
auto *NewGEP = IsInBounds
? State.Builder.CreateInBoundsGEP(
GEP->getSourceElementType(), Ptr, Indices)
: State.Builder.CreateGEP(GEP->getSourceElementType(),
Ptr, Indices);
assert((State.VF.isScalar() || NewGEP->getType()->isVectorTy()) &&
"NewGEP is not a pointer vector");
State.set(this, NewGEP, Part);
State.ILV->addMetadata(NewGEP, GEP);
}
}
}

void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
Expand Down

0 comments on commit 3495090

Please sign in to comment.