Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 45 additions & 28 deletions llvm/lib/Analysis/Delinearization.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ void llvm::collectParametricTerms(ScalarEvolution &SE, const SCEV *Expr,
LLVM_DEBUG({
dbgs() << "Strides:\n";
for (const SCEV *S : Strides)
dbgs() << *S << "\n";
dbgs() << " " << *S << "\n";
});

for (const SCEV *S : Strides) {
Expand All @@ -193,7 +193,7 @@ void llvm::collectParametricTerms(ScalarEvolution &SE, const SCEV *Expr,
LLVM_DEBUG({
dbgs() << "Terms:\n";
for (const SCEV *T : Terms)
dbgs() << *T << "\n";
dbgs() << " " << *T << "\n";
});

SCEVCollectAddRecMultiplies MulCollector(Terms, SE);
Expand Down Expand Up @@ -294,7 +294,7 @@ void llvm::findArrayDimensions(ScalarEvolution &SE,
LLVM_DEBUG({
dbgs() << "Terms:\n";
for (const SCEV *T : Terms)
dbgs() << *T << "\n";
dbgs() << " " << *T << "\n";
});

// Remove duplicates.
Expand Down Expand Up @@ -325,7 +325,7 @@ void llvm::findArrayDimensions(ScalarEvolution &SE,
LLVM_DEBUG({
dbgs() << "Terms after sorting:\n";
for (const SCEV *T : NewTerms)
dbgs() << *T << "\n";
dbgs() << " " << *T << "\n";
});

if (NewTerms.empty() || !findArrayDimensionsRec(SE, NewTerms, Sizes)) {
Expand All @@ -339,7 +339,7 @@ void llvm::findArrayDimensions(ScalarEvolution &SE,
LLVM_DEBUG({
dbgs() << "Sizes:\n";
for (const SCEV *S : Sizes)
dbgs() << *S << "\n";
dbgs() << " " << *S << "\n";
});
}

Expand All @@ -354,18 +354,27 @@ void llvm::computeAccessFunctions(ScalarEvolution &SE, const SCEV *Expr,
if (!AR->isAffine())
return;

// Clear output vector.
Subscripts.clear();

LLVM_DEBUG(dbgs() << "\ncomputeAccessFunctions\n"
<< "Memory Access Function: " << *Expr << "\n");

const SCEV *Res = Expr;
int Last = Sizes.size() - 1;

for (int i = Last; i >= 0; i--) {
const SCEV *Size = Sizes[i];
const SCEV *Q, *R;
SCEVDivision::divide(SE, Res, Sizes[i], &Q, &R);

SCEVDivision::divide(SE, Res, Size, &Q, &R);

LLVM_DEBUG({
dbgs() << "Res: " << *Res << "\n";
dbgs() << "Sizes[i]: " << *Sizes[i] << "\n";
dbgs() << "Res divided by Sizes[i]:\n";
dbgs() << "Quotient: " << *Q << "\n";
dbgs() << "Remainder: " << *R << "\n";
dbgs() << "Computing 'MemAccFn / Sizes[" << i << "]':\n";
dbgs() << " MemAccFn: " << *Res << "\n";
dbgs() << " Sizes[" << i << "]: " << *Size << "\n";
dbgs() << " Quotient (Leftover): " << *Q << "\n";
dbgs() << " Remainder (Subscript Access Function): " << *R << "\n";
});

Res = Q;
Expand Down Expand Up @@ -397,7 +406,8 @@ void llvm::computeAccessFunctions(ScalarEvolution &SE, const SCEV *Expr,
LLVM_DEBUG({
dbgs() << "Subscripts:\n";
for (const SCEV *S : Subscripts)
dbgs() << *S << "\n";
dbgs() << " " << *S << "\n";
dbgs() << "\n";
});
}

Expand Down Expand Up @@ -454,6 +464,10 @@ void llvm::delinearize(ScalarEvolution &SE, const SCEV *Expr,
SmallVectorImpl<const SCEV *> &Subscripts,
SmallVectorImpl<const SCEV *> &Sizes,
const SCEV *ElementSize) {
// Clear output vectors.
Subscripts.clear();
Sizes.clear();

// First step: collect parametric terms.
SmallVector<const SCEV *, 4> Terms;
collectParametricTerms(SE, Expr, Terms);
Expand All @@ -469,21 +483,6 @@ void llvm::delinearize(ScalarEvolution &SE, const SCEV *Expr,

// Third step: compute the access functions for each subscript.
computeAccessFunctions(SE, Expr, Subscripts, Sizes);

if (Subscripts.empty())
return;

LLVM_DEBUG({
dbgs() << "succeeded to delinearize " << *Expr << "\n";
dbgs() << "ArrayDecl[UnknownSize]";
for (const SCEV *S : Sizes)
dbgs() << "[" << *S << "]";

dbgs() << "\nArrayRef";
for (const SCEV *S : Subscripts)
dbgs() << "[" << *S << "]";
dbgs() << "\n";
});
}

static std::optional<APInt> tryIntoAPInt(const SCEV *S) {
Expand Down Expand Up @@ -646,6 +645,9 @@ bool llvm::delinearizeFixedSizeArray(ScalarEvolution &SE, const SCEV *Expr,
SmallVectorImpl<const SCEV *> &Subscripts,
SmallVectorImpl<const SCEV *> &Sizes,
const SCEV *ElementSize) {
// Clear output vectors.
Subscripts.clear();
Sizes.clear();

// First step: find the fixed array size.
SmallVector<uint64_t, 4> ConstSizes;
Expand All @@ -671,6 +673,7 @@ bool llvm::getIndexExpressionsFromGEP(ScalarEvolution &SE,
assert(Subscripts.empty() && Sizes.empty() &&
"Expected output lists to be empty on entry to this function.");
assert(GEP && "getIndexExpressionsFromGEP called with a null GEP");
LLVM_DEBUG(dbgs() << "\nGEP to delinearize: " << *GEP << "\n");
Type *Ty = nullptr;
bool DroppedFirstDim = false;
for (unsigned i = 1; i < GEP->getNumOperands(); i++) {
Expand All @@ -683,28 +686,43 @@ bool llvm::getIndexExpressionsFromGEP(ScalarEvolution &SE,
continue;
}
Subscripts.push_back(Expr);
LLVM_DEBUG(dbgs() << "Subscripts push_back: " << *Expr << "\n");
continue;
}

auto *ArrayTy = dyn_cast<ArrayType>(Ty);
if (!ArrayTy) {
LLVM_DEBUG(dbgs() << "GEP delinearize failed: " << *Ty
<< " is not an array type.\n");
Subscripts.clear();
Sizes.clear();
return false;
}

Subscripts.push_back(Expr);
LLVM_DEBUG(dbgs() << "Subscripts push_back: " << *Expr << "\n");
if (!(DroppedFirstDim && i == 2))
Sizes.push_back(ArrayTy->getNumElements());

Ty = ArrayTy->getElementType();
}
LLVM_DEBUG({
dbgs() << "Subscripts:\n";
for (const SCEV *S : Subscripts)
dbgs() << *S << "\n";
dbgs() << "\n";
});

return !Subscripts.empty();
}

bool llvm::tryDelinearizeFixedSizeImpl(
ScalarEvolution *SE, Instruction *Inst, const SCEV *AccessFn,
SmallVectorImpl<const SCEV *> &Subscripts, SmallVectorImpl<int> &Sizes) {
// Clear output vectors.
Subscripts.clear();
Sizes.clear();

Value *SrcPtr = getLoadStorePointerOperand(Inst);

// Check the simple case where the array dimensions are fixed size.
Expand Down Expand Up @@ -769,7 +787,6 @@ void printDelinearization(raw_ostream &O, Function *F, LoopInfo *LI,

O << "\n";
O << "Inst:" << Inst << "\n";
O << "In Loop with Header: " << L->getHeader()->getName() << "\n";
O << "AccessFunction: " << *AccessFn << "\n";

SmallVector<const SCEV *, 3> Subscripts, Sizes;
Expand Down
1 change: 0 additions & 1 deletion llvm/test/Analysis/Delinearization/a.ll
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
define void @foo(i64 %n, i64 %m, i64 %o, ptr nocapture %A) #0 {
; CHECK-LABEL: 'foo'
; CHECK-NEXT: Inst: store i32 1, ptr %arrayidx11.us.us, align 4
; CHECK-NEXT: In Loop with Header: for.k
; CHECK-NEXT: AccessFunction: {{\{\{\{}}(28 + (4 * (-4 + (3 * %m)) * %o)),+,(8 * %m * %o)}<%for.i>,+,(12 * %o)}<%for.j>,+,20}<%for.k>
; CHECK-NEXT: Base offset: %A
; CHECK-NEXT: ArrayDecl[UnknownSize][%m][%o] with elements of 4 bytes.
Expand Down
1 change: 0 additions & 1 deletion llvm/test/Analysis/Delinearization/byte_offset.ll
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
define void @foo(ptr %A, i64 %i2, i64 %arg, i1 %c) {
; CHECK-LABEL: 'foo'
; CHECK-NEXT: Inst: store float 0.000000e+00, ptr %arrayidx, align 4
; CHECK-NEXT: In Loop with Header: inner.loop
; CHECK-NEXT: AccessFunction: ({0,+,%i2}<%outer.loop> + %unknown)
; CHECK-NEXT: failed to delinearize
;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,12 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
define void @mat_mul(ptr %C, ptr %A, ptr %B, i64 %N) #0 !kernel_arg_addr_space !2 !kernel_arg_access_qual !3 !kernel_arg_type !4 !kernel_arg_base_type !4 !kernel_arg_type_qual !5 {
; CHECK-LABEL: 'mat_mul'
; CHECK-NEXT: Inst: %tmp = load float, ptr %arrayidx, align 4
; CHECK-NEXT: In Loop with Header: for.inc
; CHECK-NEXT: AccessFunction: {(4 * %N * %call),+,4}<%for.inc>
; CHECK-NEXT: Base offset: %A
; CHECK-NEXT: ArrayDecl[UnknownSize][%N] with elements of 4 bytes.
; CHECK-NEXT: ArrayRef[%call][{0,+,1}<nuw><nsw><%for.inc>]
; CHECK-EMPTY:
; CHECK-NEXT: Inst: %tmp5 = load float, ptr %arrayidx4, align 4
; CHECK-NEXT: In Loop with Header: for.inc
; CHECK-NEXT: AccessFunction: {(4 * %call1),+,(4 * %N)}<%for.inc>
; CHECK-NEXT: Base offset: %B
; CHECK-NEXT: ArrayDecl[UnknownSize][%N] with elements of 4 bytes.
Expand Down
2 changes: 0 additions & 2 deletions llvm/test/Analysis/Delinearization/divide_by_one.ll
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,12 @@ target datalayout = "e-m:e-p:32:32-i1:32-i64:64-a:0-n32"
define void @test(ptr nocapture %dst, i32 %stride, i32 %bs) {
; CHECK-LABEL: 'test'
; CHECK-NEXT: Inst: %0 = load i8, ptr %arrayidx, align 1
; CHECK-NEXT: In Loop with Header: for.body3
; CHECK-NEXT: AccessFunction: {{\{\{}}(-1 + ((1 + %bs) * %stride)),+,(-1 * %stride)}<%for.cond1.preheader>,+,1}<nw><%for.body3>
; CHECK-NEXT: Base offset: %dst
; CHECK-NEXT: ArrayDecl[UnknownSize][%stride] with elements of 1 bytes.
; CHECK-NEXT: ArrayRef[{(1 + %bs),+,-1}<nw><%for.cond1.preheader>][{-1,+,1}<nw><%for.body3>]
; CHECK-EMPTY:
; CHECK-NEXT: Inst: store i8 %0, ptr %arrayidx7, align 1
; CHECK-NEXT: In Loop with Header: for.body3
; CHECK-NEXT: AccessFunction: {{\{\{}}(%stride * %bs),+,(-1 * %stride)}<%for.cond1.preheader>,+,1}<nsw><%for.body3>
; CHECK-NEXT: Base offset: %dst
; CHECK-NEXT: ArrayDecl[UnknownSize][%stride] with elements of 1 bytes.
Expand Down
11 changes: 0 additions & 11 deletions llvm/test/Analysis/Delinearization/fixed_size_array.ll
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
define void @a_i_j_k(ptr %a) {
; CHECK-LABEL: 'a_i_j_k'
; CHECK-NEXT: Inst: store i32 1, ptr %idx, align 4
; CHECK-NEXT: In Loop with Header: for.k
; CHECK-NEXT: AccessFunction: {{\{\{\{}}0,+,1024}<nuw><nsw><%for.i.header>,+,128}<nw><%for.j.header>,+,4}<nw><%for.k>
; CHECK-NEXT: Base offset: %a
; CHECK-NEXT: ArrayDecl[UnknownSize][8][32] with elements of 4 bytes.
Expand Down Expand Up @@ -60,7 +59,6 @@ exit:
define void @a_i_nj_k(ptr %a) {
; CHECK-LABEL: 'a_i_nj_k'
; CHECK-NEXT: Inst: store i32 1, ptr %idx, align 4
; CHECK-NEXT: In Loop with Header: for.k
; CHECK-NEXT: AccessFunction: {{\{\{\{}}896,+,1024}<nuw><nsw><%for.i.header>,+,-128}<nw><%for.j.header>,+,4}<nw><%for.k>
; CHECK-NEXT: Base offset: %a
; CHECK-NEXT: ArrayDecl[UnknownSize][8][32] with elements of 4 bytes.
Expand Down Expand Up @@ -116,14 +114,12 @@ exit:
define void @a_ijk_b_i2jk(ptr %a, ptr %b) {
; CHECK-LABEL: 'a_ijk_b_i2jk'
; CHECK-NEXT: Inst: store i32 1, ptr %a.idx, align 4
; CHECK-NEXT: In Loop with Header: for.k
; CHECK-NEXT: AccessFunction: {{\{\{\{}}0,+,1024}<nuw><nsw><%for.i.header>,+,256}<nw><%for.j.header>,+,4}<nw><%for.k>
; CHECK-NEXT: Base offset: %a
; CHECK-NEXT: ArrayDecl[UnknownSize][4][64] with elements of 4 bytes.
; CHECK-NEXT: ArrayRef[{0,+,1}<nuw><nsw><%for.i.header>][{0,+,1}<nuw><nsw><%for.j.header>][{0,+,1}<nuw><nsw><%for.k>]
; CHECK-EMPTY:
; CHECK-NEXT: Inst: store i32 1, ptr %b.idx, align 4
; CHECK-NEXT: In Loop with Header: for.k
; CHECK-NEXT: AccessFunction: {{\{\{\{}}0,+,1024}<nuw><nsw><%for.i.header>,+,256}<nw><%for.j.header>,+,4}<nw><%for.k>
; CHECK-NEXT: Base offset: %b
; CHECK-NEXT: ArrayDecl[UnknownSize][4][64] with elements of 4 bytes.
Expand Down Expand Up @@ -180,7 +176,6 @@ exit:
define void @a_i_2j1_k(ptr %a) {
; CHECK-LABEL: 'a_i_2j1_k'
; CHECK-NEXT: Inst: store i32 1, ptr %idx, align 4
; CHECK-NEXT: In Loop with Header: for.k
; CHECK-NEXT: AccessFunction: {{\{\{\{}}128,+,1024}<nuw><nsw><%for.i.header>,+,256}<nw><%for.j.header>,+,4}<nw><%for.k>
; CHECK-NEXT: Base offset: %a
; CHECK-NEXT: ArrayDecl[UnknownSize][4][64] with elements of 4 bytes.
Expand Down Expand Up @@ -234,7 +229,6 @@ exit:
define void @a_i_3j_k(ptr %a) {
; CHECK-LABEL: 'a_i_3j_k'
; CHECK-NEXT: Inst: store i32 1, ptr %idx, align 4
; CHECK-NEXT: In Loop with Header: for.k
; CHECK-NEXT: AccessFunction: {{\{\{\{}}0,+,1024}<nuw><nsw><%for.i.header>,+,384}<nw><%for.j.header>,+,4}<nw><%for.k>
; CHECK-NEXT: failed to delinearize
;
Expand Down Expand Up @@ -286,7 +280,6 @@ exit:
define void @a_i_j_3k(ptr %a) {
; CHECK-LABEL: 'a_i_j_3k'
; CHECK-NEXT: Inst: store i32 1, ptr %idx, align 4
; CHECK-NEXT: In Loop with Header: for.k
; CHECK-NEXT: AccessFunction: {{\{\{\{}}0,+,1024}<nuw><nsw><%for.i.header>,+,128}<nw><%for.j.header>,+,12}<nw><%for.k>
; CHECK-NEXT: Base offset: %a
; CHECK-NEXT: ArrayDecl[UnknownSize][8][32] with elements of 4 bytes.
Expand Down Expand Up @@ -338,7 +331,6 @@ exit:
define void @a_i_j2k_i(ptr %a) {
; CHECK-LABEL: 'a_i_j2k_i'
; CHECK-NEXT: Inst: store i32 1, ptr %idx, align 4
; CHECK-NEXT: In Loop with Header: for.k
; CHECK-NEXT: AccessFunction: {{\{\{\{}}0,+,1028}<%for.i.header>,+,256}<nw><%for.j.header>,+,128}<nw><%for.k>
; CHECK-NEXT: failed to delinearize
;
Expand Down Expand Up @@ -390,7 +382,6 @@ exit:
define void @a_i_i_jk(ptr %a) {
; CHECK-LABEL: 'a_i_i_jk'
; CHECK-NEXT: Inst: store i32 1, ptr %idx, align 4
; CHECK-NEXT: In Loop with Header: for.k
; CHECK-NEXT: AccessFunction: {{\{\{\{}}0,+,1152}<%for.i.header>,+,4}<nw><%for.j.header>,+,4}<nw><%for.k>
; CHECK-NEXT: Base offset: %a
; CHECK-NEXT: ArrayDecl[UnknownSize][288] with elements of 4 bytes.
Expand Down Expand Up @@ -441,7 +432,6 @@ exit:
define void @a_i_jk_l(ptr %a) {
; CHECK-LABEL: 'a_i_jk_l'
; CHECK-NEXT: Inst: store i32 1, ptr %idx, align 4
; CHECK-NEXT: In Loop with Header: for.l
; CHECK-NEXT: AccessFunction: {{\{\{\{\{}}0,+,1024}<nuw><nsw><%for.i.header>,+,128}<nw><%for.j.header>,+,128}<nw><%for.k.header>,+,4}<nw><%for.l>
; CHECK-NEXT: Base offset: %a
; CHECK-NEXT: ArrayDecl[UnknownSize][8][32] with elements of 4 bytes.
Expand Down Expand Up @@ -502,7 +492,6 @@ exit:
define void @non_divisible_by_element_size(ptr %a) {
; CHECK-LABEL: 'non_divisible_by_element_size'
; CHECK-NEXT: Inst: store i32 1, ptr %idx, align 4
; CHECK-NEXT: In Loop with Header: for.k
; CHECK-NEXT: AccessFunction: {{\{\{\{}}0,+,256}<nuw><nsw><%for.i.header>,+,32}<nw><%for.j.header>,+,1}<nw><%for.k>
; CHECK-NEXT: failed to delinearize
;
Expand Down
Loading
Loading