Skip to content

Commit

Permalink
[ConstantFold] Refactor load folding
Browse files Browse the repository at this point in the history
This refactors load folding to happen in two cleanly separated
steps: ConstantFoldLoadFromConstPtr() takes a pointer to load from
and decomposes it into a constant initializer base and an offset.
Then ConstantFoldLoadFromConst() loads from that initializer at
the given offset. This makes the core logic independent of having
actual GEP expressions (and those GEP expressions having certain
structure) and will allow exposing ConstantFoldLoadFromConst() as
an independent API in the future.

This is mostly only a refactoring, but it does make the folding
logic slightly more powerful.

Differential Revision: https://reviews.llvm.org/D111023
  • Loading branch information
nikic committed Oct 5, 2021
1 parent d67935e commit c117d77
Show file tree
Hide file tree
Showing 3 changed files with 64 additions and 142 deletions.
190 changes: 56 additions & 134 deletions llvm/lib/Analysis/ConstantFolding.cpp
Expand Up @@ -63,11 +63,6 @@
using namespace llvm;

namespace {
Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP,
ArrayRef<Constant *> Ops,
const DataLayout &DL,
const TargetLibraryInfo *TLI,
bool ForLoadOperand);

//===----------------------------------------------------------------------===//
// Constant Folding internal helper functions
Expand Down Expand Up @@ -550,19 +545,16 @@ bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset, unsigned char *CurPtr,
return false;
}

Constant *FoldReinterpretLoadFromConstPtr(Constant *C, Type *LoadTy,
const DataLayout &DL) {
Constant *FoldReinterpretLoadFromConst(Constant *C, Type *LoadTy,
int64_t Offset, const DataLayout &DL) {
// Bail out early. Not expect to load from scalable global variable.
if (isa<ScalableVectorType>(LoadTy))
return nullptr;

auto *PTy = cast<PointerType>(C->getType());
auto *IntType = dyn_cast<IntegerType>(LoadTy);

// If this isn't an integer load we can't fold it directly.
if (!IntType) {
unsigned AS = PTy->getAddressSpace();

// If this is a float/double load, we can try folding it as an int32/64 load
// and then bitcast the result. This can be useful for union cases. Note
// that address spaces don't matter here since we're not going to result in
Expand All @@ -580,8 +572,7 @@ Constant *FoldReinterpretLoadFromConstPtr(Constant *C, Type *LoadTy,
} else
return nullptr;

C = FoldBitCast(C, MapTy->getPointerTo(AS), DL);
if (Constant *Res = FoldReinterpretLoadFromConstPtr(C, MapTy, DL)) {
if (Constant *Res = FoldReinterpretLoadFromConst(C, MapTy, Offset, DL)) {
if (Res->isNullValue() && !LoadTy->isX86_MMXTy() &&
!LoadTy->isX86_AMXTy())
// Materializing a zero can be done trivially without a bitcast
Expand All @@ -607,19 +598,7 @@ Constant *FoldReinterpretLoadFromConstPtr(Constant *C, Type *LoadTy,
if (BytesLoaded > 32 || BytesLoaded == 0)
return nullptr;

GlobalValue *GVal;
APInt OffsetAI;
if (!IsConstantOffsetFromGlobal(C, GVal, OffsetAI, DL))
return nullptr;

auto *GV = dyn_cast<GlobalVariable>(GVal);
if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer() ||
!GV->getInitializer()->getType()->isSized())
return nullptr;

int64_t Offset = OffsetAI.getSExtValue();
int64_t InitializerSize =
DL.getTypeAllocSize(GV->getInitializer()->getType()).getFixedSize();
int64_t InitializerSize = DL.getTypeAllocSize(C->getType()).getFixedSize();

// If we're not accessing anything in this constant, the result is undefined.
if (Offset <= -1 * static_cast<int64_t>(BytesLoaded))
Expand All @@ -640,7 +619,7 @@ Constant *FoldReinterpretLoadFromConstPtr(Constant *C, Type *LoadTy,
Offset = 0;
}

if (!ReadDataFromGlobal(GV->getInitializer(), Offset, CurPtr, BytesLeft, DL))
if (!ReadDataFromGlobal(C, Offset, CurPtr, BytesLeft, DL))
return nullptr;

APInt ResultVal = APInt(IntType->getBitWidth(), 0);
Expand All @@ -661,111 +640,64 @@ Constant *FoldReinterpretLoadFromConstPtr(Constant *C, Type *LoadTy,
return ConstantInt::get(IntType->getContext(), ResultVal);
}

Constant *ConstantFoldLoadThroughBitcastExpr(ConstantExpr *CE, Type *DestTy,
const DataLayout &DL) {
auto *SrcPtr = CE->getOperand(0);
if (!SrcPtr->getType()->isPointerTy())
/// If this Offset points exactly to the start of an aggregate element, return
/// that element, otherwise return nullptr.
Constant *getConstantAtOffset(Constant *Base, APInt Offset,
const DataLayout &DL) {
if (Offset.isZero())
return Base;

if (!isa<ConstantAggregate>(Base) && !isa<ConstantDataSequential>(Base))
return nullptr;

return ConstantFoldLoadFromConstPtr(SrcPtr, DestTy, DL);
Type *ElemTy = Base->getType();
SmallVector<APInt> Indices = DL.getGEPIndicesForOffset(ElemTy, Offset);
if (!Offset.isZero() || !Indices[0].isZero())
return nullptr;

Constant *C = Base;
for (const APInt &Index : drop_begin(Indices)) {
if (Index.isNegative() || Index.getActiveBits() >= 32)
return nullptr;

C = C->getAggregateElement(Index.getZExtValue());
if (!C)
return nullptr;
}

return C;
}

Constant *ConstantFoldLoadFromConst(Constant *C, Type *Ty, const APInt &Offset,
const DataLayout &DL) {
if (Constant *AtOffset = getConstantAtOffset(C, Offset, DL))
if (Constant *Result = ConstantFoldLoadThroughBitcast(AtOffset, Ty, DL))
return Result;

// Try hard to fold loads from bitcasted strange and non-type-safe things.
if (Offset.getMinSignedBits() <= 64)
return FoldReinterpretLoadFromConst(C, Ty, Offset.getSExtValue(), DL);

return nullptr;
}

} // end anonymous namespace

Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty,
const DataLayout &DL) {
// First, try the easy cases:
APInt Offset(DL.getIndexTypeSizeInBits(C->getType()), 0);
C = cast<Constant>(C->stripAndAccumulateConstantOffsets(
DL, Offset, /* AllowNonInbounds */ true));

if (auto *GV = dyn_cast<GlobalVariable>(C))
if (GV->isConstant() && GV->hasDefinitiveInitializer())
return ConstantFoldLoadThroughBitcast(GV->getInitializer(), Ty, DL);

if (auto *GA = dyn_cast<GlobalAlias>(C))
if (GA->getAliasee() && !GA->isInterposable())
return ConstantFoldLoadFromConstPtr(GA->getAliasee(), Ty, DL);

// If the loaded value isn't a constant expr, we can't handle it.
auto *CE = dyn_cast<ConstantExpr>(C);
if (!CE)
return nullptr;

if (CE->getOpcode() == Instruction::GetElementPtr) {
if (auto *GV = dyn_cast<GlobalVariable>(CE->getOperand(0))) {
if (GV->isConstant() && GV->hasDefinitiveInitializer()) {
if (Constant *V = ConstantFoldLoadThroughGEPConstantExpr(
GV->getInitializer(), CE, Ty, DL))
return V;
}
} else {
// Try to simplify GEP if the pointer operand wasn't a GlobalVariable.
// SymbolicallyEvaluateGEP() with `ForLoadOperand = true` can potentially
// simplify the GEP more than it normally would have been, but should only
// be used for const folding loads.
SmallVector<Constant *> Ops;
for (unsigned I = 0, E = CE->getNumOperands(); I != E; ++I)
Ops.push_back(cast<Constant>(CE->getOperand(I)));
if (auto *Simplified = dyn_cast_or_null<ConstantExpr>(
SymbolicallyEvaluateGEP(cast<GEPOperator>(CE), Ops, DL, nullptr,
/*ForLoadOperand*/ true))) {
// If the symbolically evaluated GEP is another GEP, we can only const
// fold it if the resulting pointer operand is a GlobalValue. Otherwise
// there is nothing else to simplify since the GEP is already in the
// most simplified form.
if (isa<GEPOperator>(Simplified)) {
if (auto *GV = dyn_cast<GlobalVariable>(Simplified->getOperand(0))) {
if (GV->isConstant() && GV->hasDefinitiveInitializer()) {
if (Constant *V = ConstantFoldLoadThroughGEPConstantExpr(
GV->getInitializer(), Simplified, Ty, DL))
return V;
}
}
} else {
return ConstantFoldLoadFromConstPtr(Simplified, Ty, DL);
}
}
}
}

if (CE->getOpcode() == Instruction::BitCast)
if (Constant *LoadedC = ConstantFoldLoadThroughBitcastExpr(CE, Ty, DL))
return LoadedC;

// Instead of loading constant c string, use corresponding integer value
// directly if string length is small enough.
StringRef Str;
if (getConstantStringInfo(CE, Str) && !Str.empty()) {
size_t StrLen = Str.size();
unsigned NumBits = Ty->getPrimitiveSizeInBits();
// Replace load with immediate integer if the result is an integer or fp
// value.
if ((NumBits >> 3) == StrLen + 1 && (NumBits & 7) == 0 &&
(isa<IntegerType>(Ty) || Ty->isFloatingPointTy())) {
APInt StrVal(NumBits, 0);
APInt SingleChar(NumBits, 0);
if (DL.isLittleEndian()) {
for (unsigned char C : reverse(Str.bytes())) {
SingleChar = static_cast<uint64_t>(C);
StrVal = (StrVal << 8) | SingleChar;
}
} else {
for (unsigned char C : Str.bytes()) {
SingleChar = static_cast<uint64_t>(C);
StrVal = (StrVal << 8) | SingleChar;
}
// Append NULL at the end.
SingleChar = 0;
StrVal = (StrVal << 8) | SingleChar;
}

Constant *Res = ConstantInt::get(CE->getContext(), StrVal);
if (Ty->isFloatingPointTy())
Res = ConstantExpr::getBitCast(Res, Ty);
return Res;
}
}
if (Constant *Result = ConstantFoldLoadFromConst(GV->getInitializer(), Ty,
Offset, DL))
return Result;

// If this load comes from anywhere in a constant global, and if the global
// is all undef or zero, we know what it loads.
if (auto *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(CE))) {
if (auto *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(C))) {
if (GV->isConstant() && GV->hasDefinitiveInitializer()) {
if (GV->getInitializer()->isNullValue())
return Constant::getNullValue(Ty);
Expand All @@ -774,8 +706,7 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty,
}
}

// Try hard to fold loads from bitcasted strange and non-type-safe things.
return FoldReinterpretLoadFromConstPtr(CE, Ty, DL);
return nullptr;
}

namespace {
Expand Down Expand Up @@ -867,17 +798,10 @@ Constant *CastGEPIndices(Type *SrcElemTy, ArrayRef<Constant *> Ops,
}

/// Strip the pointer casts, but preserve the address space information.
Constant *StripPtrCastKeepAS(Constant *Ptr, bool ForLoadOperand) {
Constant *StripPtrCastKeepAS(Constant *Ptr) {
assert(Ptr->getType()->isPointerTy() && "Not a pointer type");
auto *OldPtrTy = cast<PointerType>(Ptr->getType());
Ptr = cast<Constant>(Ptr->stripPointerCasts());
if (ForLoadOperand) {
while (isa<GlobalAlias>(Ptr) && !cast<GlobalAlias>(Ptr)->isInterposable() &&
!cast<GlobalAlias>(Ptr)->getBaseObject()->isInterposable()) {
Ptr = cast<GlobalAlias>(Ptr)->getAliasee();
}
}

auto *NewPtrTy = cast<PointerType>(Ptr->getType());

// Preserve the address space number of the pointer.
Expand All @@ -893,8 +817,7 @@ Constant *StripPtrCastKeepAS(Constant *Ptr, bool ForLoadOperand) {
Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP,
ArrayRef<Constant *> Ops,
const DataLayout &DL,
const TargetLibraryInfo *TLI,
bool ForLoadOperand) {
const TargetLibraryInfo *TLI) {
const GEPOperator *InnermostGEP = GEP;
bool InBounds = GEP->isInBounds();

Expand Down Expand Up @@ -939,7 +862,7 @@ Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP,
DL.getIndexedOffsetInType(
SrcElemTy,
makeArrayRef((Value * const *)Ops.data() + 1, Ops.size() - 1)));
Ptr = StripPtrCastKeepAS(Ptr, ForLoadOperand);
Ptr = StripPtrCastKeepAS(Ptr);

// If this is a GEP of a GEP, fold it all into a single GEP.
while (auto *GEP = dyn_cast<GEPOperator>(Ptr)) {
Expand All @@ -961,7 +884,7 @@ Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP,
Ptr = cast<Constant>(GEP->getOperand(0));
SrcElemTy = GEP->getSourceElementType();
Offset += APInt(BitWidth, DL.getIndexedOffsetInType(SrcElemTy, NestedOps));
Ptr = StripPtrCastKeepAS(Ptr, ForLoadOperand);
Ptr = StripPtrCastKeepAS(Ptr);
}

// If the base value for this address is a literal integer value, fold the
Expand Down Expand Up @@ -1069,8 +992,7 @@ Constant *ConstantFoldInstOperandsImpl(const Value *InstOrCE, unsigned Opcode,
return ConstantFoldCastOperand(Opcode, Ops[0], DestTy, DL);

if (auto *GEP = dyn_cast<GEPOperator>(InstOrCE)) {
if (Constant *C = SymbolicallyEvaluateGEP(GEP, Ops, DL, TLI,
/*ForLoadOperand*/ false))
if (Constant *C = SymbolicallyEvaluateGEP(GEP, Ops, DL, TLI))
return C;

return ConstantExpr::getGetElementPtr(GEP->getSourceElementType(), Ops[0],
Expand Down
14 changes: 7 additions & 7 deletions llvm/test/Transforms/InstSimplify/ConstProp/loads.ll
Expand Up @@ -30,11 +30,12 @@ define i16 @test2() {
ret i16 %r
}

; FIXME: Should be able to load through a constant addrspacecast.
define i16 @test2_addrspacecast() {
; CHECK-LABEL: @test2_addrspacecast(
; CHECK-NEXT: [[R:%.*]] = load i16, i16 addrspace(1)* addrspacecast (i16* bitcast ({ { i32, i8 }, i32 }* @g1 to i16*) to i16 addrspace(1)*), align 8
; CHECK-NEXT: ret i16 [[R]]
; LE-LABEL: @test2_addrspacecast(
; LE-NEXT: ret i16 -16657
;
; BE-LABEL: @test2_addrspacecast(
; BE-NEXT: ret i16 -8531
;
%r = load i16, i16 addrspace(1)* addrspacecast(i32* getelementptr ({{i32,i8},i32}, {{i32,i8},i32}* @g1, i32 0, i32 0, i32 0) to i16 addrspace(1)*)
ret i16 %r
Expand Down Expand Up @@ -246,7 +247,7 @@ define i64 @test_leading_zero_size_elems_big() {

define i64 @test_array_of_zero_size_array() {
; CHECK-LABEL: @test_array_of_zero_size_array(
; CHECK-NEXT: ret i64 0
; CHECK-NEXT: ret i64 undef
;
%v = load i64, i64* bitcast ([4294967295 x [0 x i32]]* @g9 to i64*)
ret i64 %v
Expand All @@ -266,8 +267,7 @@ define i32* @test_undef_aggregate() {

define {}* @test_trailing_zero_gep_index() {
; CHECK-LABEL: @test_trailing_zero_gep_index(
; CHECK-NEXT: [[V:%.*]] = load {}*, {}** bitcast (i8* getelementptr inbounds (<{ [8 x i8], [8 x i8] }>, <{ [8 x i8], [8 x i8] }>* @g11, i64 0, i32 1, i64 0) to {}**), align 4
; CHECK-NEXT: ret {}* [[V]]
; CHECK-NEXT: ret {}* null
;
%v = load {}*, {}** bitcast (i8* getelementptr inbounds (<{ [8 x i8], [8 x i8] }>, <{ [8 x i8], [8 x i8] }>* @g11, i32 0, i32 1, i32 0) to {}**), align 4
ret {}* %v
Expand Down
2 changes: 1 addition & 1 deletion llvm/test/Transforms/InstSimplify/load.ll
Expand Up @@ -6,7 +6,7 @@

define i32 @crash_on_zeroinit() {
; CHECK-LABEL: @crash_on_zeroinit(
; CHECK-NEXT: ret i32 0
; CHECK-NEXT: ret i32 undef
;
%load = load i32, i32* bitcast ({}* @zeroinit to i32*)
ret i32 %load
Expand Down

0 comments on commit c117d77

Please sign in to comment.