Skip to content

Commit

Permalink
[LV] Vectorize GEPs
Browse files Browse the repository at this point in the history 
This patch adds support for vectorizing GEPs. Previously, we only generated
vector GEPs on-demand when creating gather or scatter operations. All GEPs from
the original loop were scalarized by default, and if a pointer was to be stored
to memory, we would have to build up the pointer vector with insertelement
instructions.

With this patch, we will vectorize all GEPs that haven't already been marked
for scalarization.

The patch refines collectLoopScalars to more exactly identify the scalar GEPs.
The function now more closely resembles collectLoopUniforms. And the patch
moves vector GEP creation out of vectorizeMemoryInstruction and into the main
vectorization loop. The vector GEPs needed for gather and scatter operations
will have already been generated before vectoring the memory accesses.

Differential Revision: https://reviews.llvm.org/D30710

llvm-svn: 298620
  • Loading branch information
@mssimpso
mssimpso committed Mar 23, 2017
1 parent eaada5c commit 4e7b71b
Show file tree
Hide file tree
Showing 4 changed files with 221 additions and 174 deletions.
184 changes: 117 additions & 67 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View file
Expand Up @@ -277,32 +277,6 @@ static Type *ToVectorTy(Type *Scalar, unsigned VF) {
return VectorType::get(Scalar, VF);
}

/// A helper function that returns GEP instruction and knows to skip a
/// 'bitcast'. The 'bitcast' may be skipped if the source and the destination
/// pointee types of the 'bitcast' have the same size.
/// For example:
/// bitcast double** %var to i64* - can be skipped
/// bitcast double** %var to i8* - can not
static GetElementPtrInst *getGEPInstruction(Value *Ptr) {

if (isa<GetElementPtrInst>(Ptr))
return cast<GetElementPtrInst>(Ptr);

if (isa<BitCastInst>(Ptr) &&
isa<GetElementPtrInst>(cast<BitCastInst>(Ptr)->getOperand(0))) {
Type *BitcastTy = Ptr->getType();
Type *GEPTy = cast<BitCastInst>(Ptr)->getSrcTy();
if (!isa<PointerType>(BitcastTy) || !isa<PointerType>(GEPTy))
return nullptr;
Type *Pointee1Ty = cast<PointerType>(BitcastTy)->getPointerElementType();
Type *Pointee2Ty = cast<PointerType>(GEPTy)->getPointerElementType();
const DataLayout &DL = cast<BitCastInst>(Ptr)->getModule()->getDataLayout();
if (DL.getTypeSizeInBits(Pointee1Ty) == DL.getTypeSizeInBits(Pointee2Ty))
return cast<GetElementPtrInst>(cast<BitCastInst>(Ptr)->getOperand(0));
}
return nullptr;
}

// FIXME: The following helper functions have multiple implementations
// in the project. They can be effectively organized in a common Load/Store
// utilities unit.
Expand Down Expand Up @@ -2997,40 +2971,12 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr) {
VectorParts VectorGep;

// Handle consecutive loads/stores.
GetElementPtrInst *Gep = getGEPInstruction(Ptr);
if (ConsecutiveStride) {
Ptr = getScalarValue(Ptr, 0, 0);
} else {
// At this point we should vector version of GEP for Gather or Scatter
assert(CreateGatherScatter && "The instruction should be scalarized");
if (Gep) {
// Vectorizing GEP, across UF parts. We want to get a vector value for base
// and each index that's defined inside the loop, even if it is
// loop-invariant but wasn't hoisted out. Otherwise we want to keep them
// scalar.
SmallVector<VectorParts, 4> OpsV;
for (Value *Op : Gep->operands()) {
Instruction *SrcInst = dyn_cast<Instruction>(Op);
if (SrcInst && OrigLoop->contains(SrcInst))
OpsV.push_back(getVectorValue(Op));
else
OpsV.push_back(VectorParts(UF, Op));
}
for (unsigned Part = 0; Part < UF; ++Part) {
SmallVector<Value *, 4> Ops;
Value *GEPBasePtr = OpsV[0][Part];
for (unsigned i = 1; i < Gep->getNumOperands(); i++)
Ops.push_back(OpsV[i][Part]);
Value *NewGep = Builder.CreateGEP(GEPBasePtr, Ops, "VectorGep");
cast<GetElementPtrInst>(NewGep)->setIsInBounds(Gep->isInBounds());
assert(NewGep->getType()->isVectorTy() && "Expected vector GEP");

NewGep =
Builder.CreateBitCast(NewGep, VectorType::get(Ptr->getType(), VF));
VectorGep.push_back(NewGep);
}
} else
VectorGep = getVectorValue(Ptr);
VectorGep = getVectorValue(Ptr);
}

VectorParts Mask = createBlockInMask(Instr->getParent());
Expand Down Expand Up @@ -4790,7 +4736,72 @@ void InnerLoopVectorizer::vectorizeBlockInLoop(BasicBlock *BB) {
widenPHIInstruction(&I, UF, VF);
continue;
} // End of PHI.
case Instruction::GetElementPtr: {
// Construct a vector GEP by widening the operands of the scalar GEP as
// necessary. We mark the vector GEP 'inbounds' if appropriate. A GEP
// results in a vector of pointers when at least one operand of the GEP
// is vector-typed. Thus, to keep the representation compact, we only use
// vector-typed operands for loop-varying values.
auto *GEP = cast<GetElementPtrInst>(&I);
VectorParts Entry(UF);

if (VF > 1 && OrigLoop->hasLoopInvariantOperands(GEP)) {
// If we are vectorizing, but the GEP has only loop-invariant operands,
// the GEP we build (by only using vector-typed operands for
// loop-varying values) would be a scalar pointer. Thus, to ensure we
// produce a vector of pointers, we need to either arbitrarily pick an
// operand to broadcast, or broadcast a clone of the original GEP.
// Here, we broadcast a clone of the original.
//
// TODO: If at some point we decide to scalarize instructions having
// loop-invariant operands, this special case will no longer be
// required. We would add the scalarization decision to
// collectLoopScalars() and teach getVectorValue() to broadcast
// the lane-zero scalar value.
auto *Clone = Builder.Insert(GEP->clone());
for (unsigned Part = 0; Part < UF; ++Part)
Entry[Part] = Builder.CreateVectorSplat(VF, Clone);
} else {
// If the GEP has at least one loop-varying operand, we are sure to
// produce a vector of pointers. But if we are only unrolling, we want
// to produce a scalar GEP for each unroll part. Thus, the GEP we
// produce with the code below will be scalar (if VF == 1) or vector
// (otherwise). Note that for the unroll-only case, we still maintain
// values in the vector mapping with initVector, as we do for other
// instructions.
for (unsigned Part = 0; Part < UF; ++Part) {

// The pointer operand of the new GEP. If it's loop-invariant, we
// won't broadcast it.
auto *Ptr = OrigLoop->isLoopInvariant(GEP->getPointerOperand())
? GEP->getPointerOperand()
: getVectorValue(GEP->getPointerOperand())[Part];

// Collect all the indices for the new GEP. If any index is
// loop-invariant, we won't broadcast it.
SmallVector<Value *, 4> Indices;
for (auto &U : make_range(GEP->idx_begin(), GEP->idx_end())) {
if (OrigLoop->isLoopInvariant(U.get()))
Indices.push_back(U.get());
else
Indices.push_back(getVectorValue(U.get())[Part]);
}

// Create the new GEP. Note that this GEP may be a scalar if VF == 1,
// but it should be a vector, otherwise.
auto *NewGEP = GEP->isInBounds()
? Builder.CreateInBoundsGEP(Ptr, Indices)
: Builder.CreateGEP(Ptr, Indices);
assert((VF == 1 || NewGEP->getType()->isVectorTy()) &&
"NewGEP is not a pointer vector");
Entry[Part] = NewGEP;
}
}

VectorLoopValueMap.initVector(&I, Entry);
addMetadata(Entry, GEP);
break;
}
case Instruction::UDiv:
case Instruction::SDiv:
case Instruction::SRem:
Expand Down Expand Up @@ -5481,21 +5492,58 @@ void LoopVectorizationCostModel::collectLoopScalars(unsigned VF) {
// If an instruction is uniform after vectorization, it will remain scalar.
Scalars[VF].insert(Uniforms[VF].begin(), Uniforms[VF].end());

// Collect the getelementptr instructions that will not be vectorized. A
// getelementptr instruction is only vectorized if it is used for a legal
// gather or scatter operation.
for (auto *BB : TheLoop->blocks())
// These sets are used to seed the analysis of loop scalars with memory
// access pointer operands that will remain scalar.
SmallSetVector<Instruction *, 8> ScalarPtrs;
SmallPtrSet<Instruction *, 8> PossibleNonScalarPtrs;

// Returns true if the given instruction will not be a gather or scatter
// operation with vectorization factor VF.
auto isScalarDecision = [&](Instruction *I, unsigned VF) {
InstWidening WideningDecision = getWideningDecision(I, VF);
assert(WideningDecision != CM_Unknown &&
"Widening decision should be ready at this moment");
return WideningDecision != CM_GatherScatter;
};

// Collect the initial values that we know will not be vectorized. A value
// will remain scalar if it is only used as the pointer operand of memory
// accesses that are not gather or scatter operations.
for (auto *BB : TheLoop->blocks()) {
for (auto &I : *BB) {
if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
Scalars[VF].insert(GEP);
continue;
}
auto *Ptr = getPointerOperand(&I);

// If there's no pointer operand or the pointer operand is not an
// instruction, there's nothing to do.
auto *Ptr = dyn_cast_or_null<Instruction>(getPointerOperand(&I));
if (!Ptr)
continue;
auto *GEP = getGEPInstruction(Ptr);
if (GEP && getWideningDecision(&I, VF) == CM_GatherScatter)
Scalars[VF].erase(GEP);

// If the pointer has already been identified as scalar (e.g., if it was
// also inditifed as uniform), there's nothing to do.
if (Scalars[VF].count(Ptr))
continue;

// True if all users of Ptr are memory accesses that have Ptr as their
// pointer operand.
auto UsersAreMemAccesses = all_of(Ptr->users(), [&](User *U) -> bool {
return getPointerOperand(U) == Ptr;
});

// If the pointer is used by an instruction other than a memory access,
// it may not remain scalar. If the memory access is a gather or scatter
// operation, the pointer will not remain scalar.
if (!UsersAreMemAccesses || !isScalarDecision(&I, VF))
PossibleNonScalarPtrs.insert(Ptr);
else
ScalarPtrs.insert(Ptr);
}
}

// Add to the set of scalars all the pointers we know will not be vectorized.
for (auto *I : ScalarPtrs)
if (!PossibleNonScalarPtrs.count(I)) {
DEBUG(dbgs() << "LV: Found scalar instruction: " << *I << "\n");
Scalars[VF].insert(I);
}

// An induction variable will remain scalar if all users of the induction
Expand Down Expand Up @@ -5526,6 +5574,8 @@ void LoopVectorizationCostModel::collectLoopScalars(unsigned VF) {
// The induction variable and its update instruction will remain scalar.
Scalars[VF].insert(Ind);
Scalars[VF].insert(IndUpdate);
DEBUG(dbgs() << "LV: Found scalar instruction: " << *Ind << "\n");
DEBUG(dbgs() << "LV: Found scalar instruction: " << *IndUpdate << "\n");
}
}

Expand Down
44 changes: 27 additions & 17 deletions llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
View file
Expand Up @@ -13,23 +13,33 @@ target triple = "x86_64-unknown-linux-gnu"
; scatter operation. %tmp3 (and the induction variable) should not be marked
; uniform-after-vectorization.
;
; CHECK: LV: Found uniform instruction: %tmp0 = getelementptr inbounds %data, %data* %d, i64 0, i32 3, i64 %i
; CHECK-NOT: LV: Found uniform instruction: %tmp3 = getelementptr inbounds %data, %data* %d, i64 0, i32 0, i64 %i
; CHECK-NOT: LV: Found uniform instruction: %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
; CHECK-NOT: LV: Found uniform instruction: %i.next = add nuw nsw i64 %i, 5
; CHECK: vector.body:
; CHECK: %index = phi i64
; CHECK: %vec.ind = phi <16 x i64>
; CHECK: %[[T0:.+]] = mul i64 %index, 5
; CHECK: %[[T1:.+]] = getelementptr inbounds %data, %data* %d, i64 0, i32 3, i64 %[[T0]]
; CHECK: %[[T2:.+]] = bitcast float* %[[T1]] to <80 x float>*
; CHECK: load <80 x float>, <80 x float>* %[[T2]], align 4
; CHECK: %[[T3:.+]] = getelementptr inbounds %data, %data* %d, i64 0, i32 0, i64 %[[T0]]
; CHECK: %[[T4:.+]] = bitcast float* %[[T3]] to <80 x float>*
; CHECK: load <80 x float>, <80 x float>* %[[T4]], align 4
; CHECK: %VectorGep = getelementptr inbounds %data, %data* %d, i64 0, i32 0, <16 x i64> %vec.ind
; CHECK: call void @llvm.masked.scatter.v16f32({{.*}}, <16 x float*> %VectorGep, {{.*}})
; CHECK: br i1 {{.*}}, label %middle.block, label %vector.body
; CHECK: LV: Found uniform instruction: %tmp0 = getelementptr inbounds %data, %data* %d, i64 0, i32 3, i64 %i
; CHECK-NOT: LV: Found uniform instruction: %tmp3 = getelementptr inbounds %data, %data* %d, i64 0, i32 0, i64 %i
; CHECK-NOT: LV: Found uniform instruction: %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
; CHECK-NOT: LV: Found uniform instruction: %i.next = add nuw nsw i64 %i, 5
; CHECK: vector.ph:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x float> undef, float %x, i32 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x float> [[BROADCAST_SPLATINSERT]], <16 x float> undef, <16 x i32> zeroinitializer
; CHECK-NEXT: br label %vector.body
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 5, i64 10, i64 15, i64 20, i64 25, i64 30, i64 35, i64 40, i64 45, i64 50, i64 55, i64 60, i64 65, i64 70, i64 75>, %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 5
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds %data, %data* %d, i64 0, i32 3, i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[TMP1:%.*]] = bitcast float* [[TMP0]] to <80 x float>*
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <80 x float>, <80 x float>* [[TMP1]], align 4
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <80 x float> [[WIDE_VEC]], <80 x float> undef, <16 x i32> <i32 0, i32 5, i32 10, i32 15, i32 20, i32 25, i32 30, i32 35, i32 40, i32 45, i32 50, i32 55, i32 60, i32 65, i32 70, i32 75>
; CHECK-NEXT: [[TMP2:%.*]] = fmul <16 x float> [[BROADCAST_SPLAT]], [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds %data, %data* %d, i64 0, i32 0, <16 x i64> [[VEC_IND]]
; CHECK-NEXT: [[BC:%.*]] = bitcast <16 x float*> [[TMP3]] to <16 x <80 x float>*>
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <16 x <80 x float>*> [[BC]], i32 0
; CHECK-NEXT: [[WIDE_VEC1:%.*]] = load <80 x float>, <80 x float>* [[TMP4]], align 4
; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <80 x float> [[WIDE_VEC1]], <80 x float> undef, <16 x i32> <i32 0, i32 5, i32 10, i32 15, i32 20, i32 25, i32 30, i32 35, i32 40, i32 45, i32 50, i32 55, i32 60, i32 65, i32 70, i32 75>
; CHECK-NEXT: [[TMP5:%.*]] = fadd <16 x float> [[STRIDED_VEC2]], [[TMP2]]
; CHECK-NEXT: call void @llvm.masked.scatter.v16f32(<16 x float> [[TMP5]], <16 x float*> [[TMP3]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 16
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <16 x i64> [[VEC_IND]], <i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80>
; CHECK: br i1 {{.*}}, label %middle.block, label %vector.body

%data = type { [32000 x float], [3 x i32], [4 x i8], [32000 x float] }

Expand Down

0 comments on commit 4e7b71b

Please sign in to comment.