diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index d2d21c7c4b5e36..614dd98b013b35 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -11184,6 +11184,8 @@ Syntax: = getelementptr , ptr {, }* = getelementptr inbounds , ptr {, }* + = getelementptr nusw , ptr {, }* + = getelementptr nuw , ptr {, }* = getelementptr inrange(S,E) , ptr {, }* = getelementptr , , @@ -11299,27 +11301,47 @@ memory though, even if it happens to point into allocated storage. See the :ref:`Pointer Aliasing Rules ` section for more information. -If the ``inbounds`` keyword is present, the result value of a -``getelementptr`` with any non-zero indices is a -:ref:`poison value ` if one of the following rules is violated: - -* The base pointer has an *in bounds* address of an allocated object, which +The ``getelementptr`` instruction may have a number of attributes that impose +additional rules. If any of the rules are violated, the result value is a +:ref:`poison value `. In cases where the base is a vector of +pointers, the attributes apply to each computation element-wise. + +For ``nusw`` (no unsigned signed wrap): + + * If the type of an index is larger than the pointer index type, the + truncation to the pointer index type preserves the signed value + (``trunc nsw``). + * The multiplication of an index by the type size does not wrap the pointer + index type in a signed sense (``mul nsw``). + * The successive addition of each offset (without adding the base address) + does not wrap the pointer index type in a signed sense (``add nsw``). + * The successive addition of the current address, truncated to the pointer + index type and interpreted as an unsigned number, and each offset, + interpreted as a signed number, does not wrap the pointer index type. + +For ``nuw`` (no unsigned wrap): + + * If the type of an index is larger than the pointer index type, the + truncation to the pointer index type preserves the unsigned value + (``trunc nuw``). + * The multiplication of an index by the type size does not wrap the pointer + index type in an unsigned sense (``mul nuw``). + * The successive addition of each offset (without adding the base address) + does not wrap the pointer index type in an unsigned sense (``add nuw``). + * The successive addition of the current address, truncated to the pointer + index type and interpreted as an unsigned number, and each offset, also + interpreted as an unsigned number, does not wrap the pointer index type + (``add nuw``). + +For ``inbounds`` all rules of the ``nusw`` attribute apply. Additionally, +if the ``getelementptr`` has any non-zero indices, the following rules apply: + + * The base pointer has an *in bounds* address of an allocated object, which means that it points into an allocated object, or to its end. Note that the object does not have to be live anymore; being in-bounds of a deallocated object is sufficient. -* If the type of an index is larger than the pointer index type, the - truncation to the pointer index type preserves the signed value. -* The multiplication of an index by the type size does not wrap the pointer - index type in a signed sense (``nsw``). -* The successive addition of each offset (without adding the base address) does - not wrap the pointer index type in a signed sense (``nsw``). -* The successive addition of the current address, interpreted as an unsigned - number, and each offset, interpreted as a signed number, does not wrap the - unsigned address space and remains *in bounds* of the allocated object. - As a corollary, if the added offset is non-negative, the addition does not - wrap in an unsigned sense (``nuw``). -* In cases where the base is a vector of pointers, the ``inbounds`` keyword - applies to each of the computations element-wise. + * During the successive addition of offsets to the address, the resulting + pointer must remain *in bounds* of the allocated object at each step. Note that ``getelementptr`` with all-zero indices is always considered to be ``inbounds``, even if the base pointer does not point to an allocated object. @@ -11330,6 +11352,10 @@ These rules are based on the assumption that no allocated object may cross the unsigned address space boundary, and no allocated object may be larger than half the pointer index type space. +If ``inbounds`` is present on a ``getelementptr`` instruction, the ``nusw`` +attribute will be automatically set as well. For this reason, the ``nusw`` +will also not be printed in textual IR if ``inbounds`` is already present. + If the ``inrange(Start, End)`` attribute is present, loading from or storing to any pointer derived from the ``getelementptr`` has undefined behavior if the load or store would access memory outside the half-open range diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst index a495e6cb170675..c7c2c2825f58b9 100644 --- a/llvm/docs/ReleaseNotes.rst +++ b/llvm/docs/ReleaseNotes.rst @@ -51,6 +51,7 @@ Changes to the LLVM IR ---------------------- * Added Memory Model Relaxation Annotations (MMRAs). +* Added ``nusw`` and ``nuw`` flags to ``getelementptr`` instruction. * Renamed ``llvm.experimental.vector.reverse`` intrinsic to ``llvm.vector.reverse``. * Renamed ``llvm.experimental.vector.splice`` intrinsic to ``llvm.vector.splice``. * Renamed ``llvm.experimental.vector.interleave2`` intrinsic to ``llvm.vector.interleave2``. diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h index 0cbcdcd9ffac77..df61ec6ed30e0b 100644 --- a/llvm/include/llvm/AsmParser/LLToken.h +++ b/llvm/include/llvm/AsmParser/LLToken.h @@ -109,6 +109,7 @@ enum Kind { kw_fast, kw_nuw, kw_nsw, + kw_nusw, kw_exact, kw_disjoint, kw_inbounds, diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h index 909eb833c601a9..d3b9e96520f88a 100644 --- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -385,7 +385,7 @@ enum ConstantsCodes { CST_CODE_CSTRING = 9, // CSTRING: [values] CST_CODE_CE_BINOP = 10, // CE_BINOP: [opcode, opval, opval] CST_CODE_CE_CAST = 11, // CE_CAST: [opcode, opty, opval] - CST_CODE_CE_GEP = 12, // CE_GEP: [n x operands] + CST_CODE_CE_GEP_OLD = 12, // CE_GEP: [n x operands] CST_CODE_CE_SELECT = 13, // CE_SELECT: [opval, opval, opval] CST_CODE_CE_EXTRACTELT = 14, // CE_EXTRACTELT: [opty, opval, opval] CST_CODE_CE_INSERTELT = 15, // CE_INSERTELT: [opval, opval, opval] @@ -412,6 +412,7 @@ enum ConstantsCodes { // asmdialect|unwind, // asmstr,conststr] CST_CODE_CE_GEP_WITH_INRANGE = 31, // [opty, flags, range, n x operands] + CST_CODE_CE_GEP = 32, // [opty, flags, n x operands] }; /// CastOpcodes - These are values used in the bitcode files to encode which @@ -524,6 +525,14 @@ enum PossiblyExactOperatorOptionalFlags { PEO_EXACT = 0 }; /// PossiblyDisjointInst's SubclassOptionalData contents. enum PossiblyDisjointInstOptionalFlags { PDI_DISJOINT = 0 }; +/// GetElementPtrOptionalFlags - Flags for serializing +/// GEPOperator's SubclassOptionalData contents. +enum GetElementPtrOptionalFlags { + GEP_INBOUNDS = 0, + GEP_NUSW = 1, + GEP_NUW = 2, +}; + /// Encoded AtomicOrdering values. enum AtomicOrderingCodes { ORDERING_NOTATOMIC = 0, diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h index 9ec81903f09c96..a1e5005a9d1da5 100644 --- a/llvm/include/llvm/IR/Constants.h +++ b/llvm/include/llvm/IR/Constants.h @@ -28,6 +28,7 @@ #include "llvm/IR/Constant.h" #include "llvm/IR/ConstantRange.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/GEPNoWrapFlags.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/OperandTraits.h" #include "llvm/IR/User.h" @@ -1198,26 +1199,27 @@ class ConstantExpr : public Constant { /// \param OnlyIfReducedTy see \a getWithOperands() docs. static Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef IdxList, - bool InBounds = false, + GEPNoWrapFlags NW = GEPNoWrapFlags::none(), std::optional InRange = std::nullopt, Type *OnlyIfReducedTy = nullptr) { return getGetElementPtr( - Ty, C, ArrayRef((Value *const *)IdxList.data(), IdxList.size()), - InBounds, InRange, OnlyIfReducedTy); + Ty, C, ArrayRef((Value *const *)IdxList.data(), IdxList.size()), NW, + InRange, OnlyIfReducedTy); } static Constant * - getGetElementPtr(Type *Ty, Constant *C, Constant *Idx, bool InBounds = false, + getGetElementPtr(Type *Ty, Constant *C, Constant *Idx, + GEPNoWrapFlags NW = GEPNoWrapFlags::none(), std::optional InRange = std::nullopt, Type *OnlyIfReducedTy = nullptr) { // This form of the function only exists to avoid ambiguous overload // warnings about whether to convert Idx to ArrayRef or // ArrayRef. - return getGetElementPtr(Ty, C, cast(Idx), InBounds, InRange, + return getGetElementPtr(Ty, C, cast(Idx), NW, InRange, OnlyIfReducedTy); } static Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef IdxList, - bool InBounds = false, + GEPNoWrapFlags NW = GEPNoWrapFlags::none(), std::optional InRange = std::nullopt, Type *OnlyIfReducedTy = nullptr); @@ -1225,18 +1227,18 @@ class ConstantExpr : public Constant { /// "inbounds" flag in LangRef.html for details. static Constant *getInBoundsGetElementPtr(Type *Ty, Constant *C, ArrayRef IdxList) { - return getGetElementPtr(Ty, C, IdxList, true); + return getGetElementPtr(Ty, C, IdxList, GEPNoWrapFlags::inBounds()); } static Constant *getInBoundsGetElementPtr(Type *Ty, Constant *C, Constant *Idx) { // This form of the function only exists to avoid ambiguous overload // warnings about whether to convert Idx to ArrayRef or // ArrayRef. - return getGetElementPtr(Ty, C, Idx, true); + return getGetElementPtr(Ty, C, Idx, GEPNoWrapFlags::inBounds()); } static Constant *getInBoundsGetElementPtr(Type *Ty, Constant *C, ArrayRef IdxList) { - return getGetElementPtr(Ty, C, IdxList, true); + return getGetElementPtr(Ty, C, IdxList, GEPNoWrapFlags::inBounds()); } static Constant *getExtractElement(Constant *Vec, Constant *Idx, diff --git a/llvm/include/llvm/IR/GEPNoWrapFlags.h b/llvm/include/llvm/IR/GEPNoWrapFlags.h new file mode 100644 index 00000000000000..feaccc878de0c5 --- /dev/null +++ b/llvm/include/llvm/IR/GEPNoWrapFlags.h @@ -0,0 +1,93 @@ +//===-- llvm/GEPNoWrapFlags.h - NoWrap flags for GEPs -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the nowrap flags for getelementptr operators. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_IR_GEPNOWRAPFLAGS_H +#define LLVM_IR_GEPNOWRAPFLAGS_H + +namespace llvm { + +/// Represents flags for the getelementptr instruction/expression. +/// The following flags are supported: +/// * inbounds (implies nusw) +/// * nusw (no unsigned signed wrap) +/// * nuw (no unsigned wrap) +/// See LangRef for a description of their semantics. +class GEPNoWrapFlags { + enum : unsigned { + InBoundsFlag = (1 << 0), + NUSWFlag = (1 << 1), + NUWFlag = (1 << 2), + }; + + unsigned Flags; + GEPNoWrapFlags(unsigned Flags) : Flags(Flags) { + assert((!isInBounds() || hasNoUnsignedSignedWrap()) && + "inbounds implies nusw"); + } + +public: + GEPNoWrapFlags() : Flags(0) {} + // For historical reasons, interpret plain boolean as InBounds. + // TODO: Migrate users to pass explicit GEPNoWrapFlags and remove this ctor. + GEPNoWrapFlags(bool IsInBounds) + : Flags(IsInBounds ? (InBoundsFlag | NUSWFlag) : 0) {} + + static GEPNoWrapFlags none() { return GEPNoWrapFlags(); } + static GEPNoWrapFlags inBounds() { + return GEPNoWrapFlags(InBoundsFlag | NUSWFlag); + } + static GEPNoWrapFlags noUnsignedSignedWrap() { + return GEPNoWrapFlags(NUSWFlag); + } + static GEPNoWrapFlags noUnsignedWrap() { return GEPNoWrapFlags(NUWFlag); } + + static GEPNoWrapFlags fromRaw(unsigned Flags) { + return GEPNoWrapFlags(Flags); + } + unsigned getRaw() const { return Flags; } + + bool isInBounds() const { return Flags & InBoundsFlag; } + bool hasNoUnsignedSignedWrap() const { return Flags & NUSWFlag; } + bool hasNoUnsignedWrap() const { return Flags & NUWFlag; } + + GEPNoWrapFlags withoutInBounds() const { + return GEPNoWrapFlags(Flags & ~InBoundsFlag); + } + GEPNoWrapFlags withoutNoUnsignedSignedWrap() const { + return GEPNoWrapFlags(Flags & ~(InBoundsFlag | NUSWFlag)); + } + GEPNoWrapFlags withoutNoUnsignedWrap() const { + return GEPNoWrapFlags(Flags & ~NUWFlag); + } + + bool operator==(GEPNoWrapFlags Other) const { return Flags == Other.Flags; } + bool operator!=(GEPNoWrapFlags Other) const { return !(*this == Other); } + + GEPNoWrapFlags operator&(GEPNoWrapFlags Other) const { + return GEPNoWrapFlags(Flags & Other.Flags); + } + GEPNoWrapFlags operator|(GEPNoWrapFlags Other) const { + return GEPNoWrapFlags(Flags | Other.Flags); + } + GEPNoWrapFlags &operator&=(GEPNoWrapFlags Other) { + Flags &= Other.Flags; + return *this; + } + GEPNoWrapFlags &operator|=(GEPNoWrapFlags Other) { + Flags |= Other.Flags; + return *this; + } +}; + +} // end namespace llvm + +#endif // LLVM_IR_GEPNOWRAPFLAGS_H diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h index 0f7b215b80fd78..8d7c2b0c957dd6 100644 --- a/llvm/include/llvm/IR/Instructions.h +++ b/llvm/include/llvm/IR/Instructions.h @@ -26,6 +26,7 @@ #include "llvm/IR/CFG.h" #include "llvm/IR/Constant.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/GEPNoWrapFlags.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/OperandTraits.h" @@ -1167,13 +1168,26 @@ class GetElementPtrInst : public Instruction { /// a constant offset between them. bool hasAllConstantIndices() const; + /// Set nowrap flags for GEP instruction. + void setNoWrapFlags(GEPNoWrapFlags NW); + /// Set or clear the inbounds flag on this GEP instruction. /// See LangRef.html for the meaning of inbounds on a getelementptr. + /// TODO: Remove this method in favor of setNoWrapFlags(). void setIsInBounds(bool b = true); + /// Get the nowrap flags for the GEP instruction. + GEPNoWrapFlags getNoWrapFlags() const; + /// Determine whether the GEP has the inbounds flag. bool isInBounds() const; + /// Determine whether the GEP has the nusw flag. + bool hasNoUnsignedSignedWrap() const; + + /// Determine whether the GEP has the nuw flag. + bool hasNoUnsignedWrap() const; + /// Accumulate the constant address offset of this GEP if possible. /// /// This routine accepts an APInt into which it will accumulate the constant diff --git a/llvm/include/llvm/IR/Operator.h b/llvm/include/llvm/IR/Operator.h index b2307948bbbc4f..fda26891acfa73 100644 --- a/llvm/include/llvm/IR/Operator.h +++ b/llvm/include/llvm/IR/Operator.h @@ -17,6 +17,7 @@ #include "llvm/ADT/MapVector.h" #include "llvm/IR/Constants.h" #include "llvm/IR/FMF.h" +#include "llvm/IR/GEPNoWrapFlags.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" @@ -399,26 +400,24 @@ class LShrOperator }; class GEPOperator - : public ConcreteOperator { - friend class GetElementPtrInst; - friend class ConstantExpr; - - enum { - IsInBounds = (1 << 0), - }; - - void setIsInBounds(bool B) { - SubclassOptionalData = - (SubclassOptionalData & ~IsInBounds) | (B * IsInBounds); - } - + : public ConcreteOperator { public: /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); + GEPNoWrapFlags getNoWrapFlags() const { + return GEPNoWrapFlags::fromRaw(SubclassOptionalData); + } + /// Test whether this is an inbounds GEP, as defined by LangRef.html. - bool isInBounds() const { - return SubclassOptionalData & IsInBounds; + bool isInBounds() const { return getNoWrapFlags().isInBounds(); } + + bool hasNoUnsignedSignedWrap() const { + return getNoWrapFlags().hasNoUnsignedSignedWrap(); + } + + bool hasNoUnsignedWrap() const { + return getNoWrapFlags().hasNoUnsignedWrap(); } /// Returns the offset of the index with an inrange attachment, or diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp index 31667ff3951f12..705377b97ed903 100644 --- a/llvm/lib/Analysis/ConstantFolding.cpp +++ b/llvm/lib/Analysis/ConstantFolding.cpp @@ -1005,7 +1005,8 @@ Constant *ConstantFoldInstOperandsImpl(const Value *InstOrCE, unsigned Opcode, return C; return ConstantExpr::getGetElementPtr(SrcElemTy, Ops[0], Ops.slice(1), - GEP->isInBounds(), GEP->getInRange()); + GEP->getNoWrapFlags(), + GEP->getInRange()); } if (auto *CE = dyn_cast(InstOrCE)) { diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp index 8ded07ffd8bd25..20a1bd29577124 100644 --- a/llvm/lib/AsmParser/LLLexer.cpp +++ b/llvm/lib/AsmParser/LLLexer.cpp @@ -566,6 +566,7 @@ lltok::Kind LLLexer::LexIdentifier() { KEYWORD(fast); KEYWORD(nuw); KEYWORD(nsw); + KEYWORD(nusw); KEYWORD(exact); KEYWORD(disjoint); KEYWORD(inbounds); diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp index 2902bd9fe17c48..5d2056d2085672 100644 --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -4216,7 +4216,7 @@ bool LLParser::parseValID(ValID &ID, PerFunctionState *PFS, Type *ExpectedTy) { case lltok::kw_extractelement: { unsigned Opc = Lex.getUIntVal(); SmallVector Elts; - bool InBounds = false; + GEPNoWrapFlags NW; bool HasInRange = false; APSInt InRangeStart; APSInt InRangeEnd; @@ -4224,7 +4224,17 @@ bool LLParser::parseValID(ValID &ID, PerFunctionState *PFS, Type *ExpectedTy) { Lex.Lex(); if (Opc == Instruction::GetElementPtr) { - InBounds = EatIfPresent(lltok::kw_inbounds); + while (true) { + if (EatIfPresent(lltok::kw_inbounds)) + NW |= GEPNoWrapFlags::inBounds(); + else if (EatIfPresent(lltok::kw_nusw)) + NW |= GEPNoWrapFlags::noUnsignedSignedWrap(); + else if (EatIfPresent(lltok::kw_nuw)) + NW |= GEPNoWrapFlags::noUnsignedWrap(); + else + break; + } + if (EatIfPresent(lltok::kw_inrange)) { if (parseToken(lltok::lparen, "expected '('")) return true; @@ -4303,8 +4313,8 @@ bool LLParser::parseValID(ValID &ID, PerFunctionState *PFS, Type *ExpectedTy) { if (!GetElementPtrInst::getIndexedType(Ty, Indices)) return error(ID.Loc, "invalid getelementptr indices"); - ID.ConstantVal = ConstantExpr::getGetElementPtr(Ty, Elts[0], Indices, - InBounds, InRange); + ID.ConstantVal = + ConstantExpr::getGetElementPtr(Ty, Elts[0], Indices, NW, InRange); } else if (Opc == Instruction::ShuffleVector) { if (Elts.size() != 3) return error(ID.Loc, "expected three operands to shufflevector"); @@ -8339,8 +8349,18 @@ int LLParser::parseGetElementPtr(Instruction *&Inst, PerFunctionState &PFS) { Value *Ptr = nullptr; Value *Val = nullptr; LocTy Loc, EltLoc; + GEPNoWrapFlags NW; - bool InBounds = EatIfPresent(lltok::kw_inbounds); + while (true) { + if (EatIfPresent(lltok::kw_inbounds)) + NW |= GEPNoWrapFlags::inBounds(); + else if (EatIfPresent(lltok::kw_nusw)) + NW |= GEPNoWrapFlags::noUnsignedSignedWrap(); + else if (EatIfPresent(lltok::kw_nuw)) + NW |= GEPNoWrapFlags::noUnsignedWrap(); + else + break; + } Type *Ty = nullptr; if (parseType(Ty) || @@ -8393,9 +8413,9 @@ int LLParser::parseGetElementPtr(Instruction *&Inst, PerFunctionState &PFS) { if (!GetElementPtrInst::getIndexedType(Ty, Indices)) return error(Loc, "invalid getelementptr indices"); - Inst = GetElementPtrInst::Create(Ty, Ptr, Indices); - if (InBounds) - cast(Inst)->setIsInBounds(true); + GetElementPtrInst *GEP = GetElementPtrInst::Create(Ty, Ptr, Indices); + Inst = GEP; + GEP->setNoWrapFlags(NW); return AteExtraComma ? InstExtraComma : InstNormal; } diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index c9295344f8080f..32b9a033173e93 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -1459,6 +1459,17 @@ unsigned BitcodeReader::getVirtualTypeID(Type *Ty, return TypeID; } +static GEPNoWrapFlags toGEPNoWrapFlags(uint64_t Flags) { + GEPNoWrapFlags NW; + if (Flags & (1 << bitc::GEP_INBOUNDS)) + NW |= GEPNoWrapFlags::inBounds(); + if (Flags & (1 << bitc::GEP_NUSW)) + NW |= GEPNoWrapFlags::noUnsignedSignedWrap(); + if (Flags & (1 << bitc::GEP_NUW)) + NW |= GEPNoWrapFlags::noUnsignedWrap(); + return NW; +} + static bool isConstExprSupported(const BitcodeConstant *BC) { uint8_t Opcode = BC->Opcode; @@ -1614,9 +1625,9 @@ Expected BitcodeReader::materializeValue(unsigned StartValID, C = ConstantExpr::getCompare(BC->Flags, ConstOps[0], ConstOps[1]); break; case Instruction::GetElementPtr: - C = ConstantExpr::getGetElementPtr(BC->SrcElemTy, ConstOps[0], - ArrayRef(ConstOps).drop_front(), - BC->Flags, BC->getInRange()); + C = ConstantExpr::getGetElementPtr( + BC->SrcElemTy, ConstOps[0], ArrayRef(ConstOps).drop_front(), + toGEPNoWrapFlags(BC->Flags), BC->getInRange()); break; case Instruction::ExtractElement: C = ConstantExpr::getExtractElement(ConstOps[0], ConstOps[1]); @@ -1700,8 +1711,7 @@ Expected BitcodeReader::materializeValue(unsigned StartValID, I = GetElementPtrInst::Create(BC->SrcElemTy, Ops[0], ArrayRef(Ops).drop_front(), "constexpr", InsertBB); - if (BC->Flags) - cast(I)->setIsInBounds(); + cast(I)->setNoWrapFlags(toGEPNoWrapFlags(BC->Flags)); break; case Instruction::Select: I = SelectInst::Create(Ops[0], Ops[1], Ops[2], "constexpr", InsertBB); @@ -3321,9 +3331,10 @@ Error BitcodeReader::parseConstants() { break; } case bitc::CST_CODE_CE_INBOUNDS_GEP: // [ty, n x operands] - case bitc::CST_CODE_CE_GEP: // [ty, n x operands] + case bitc::CST_CODE_CE_GEP_OLD: // [ty, n x operands] case bitc::CST_CODE_CE_GEP_WITH_INRANGE_INDEX_OLD: // [ty, flags, n x // operands] + case bitc::CST_CODE_CE_GEP: // [ty, flags, n x operands] case bitc::CST_CODE_CE_GEP_WITH_INRANGE: { // [ty, flags, start, end, n x // operands] if (Record.size() < 2) @@ -3331,27 +3342,29 @@ Error BitcodeReader::parseConstants() { unsigned OpNum = 0; Type *PointeeType = nullptr; if (BitCode == bitc::CST_CODE_CE_GEP_WITH_INRANGE_INDEX_OLD || - BitCode == bitc::CST_CODE_CE_GEP_WITH_INRANGE || Record.size() % 2) + BitCode == bitc::CST_CODE_CE_GEP_WITH_INRANGE || + BitCode == bitc::CST_CODE_CE_GEP || Record.size() % 2) PointeeType = getTypeByID(Record[OpNum++]); - bool InBounds = false; + uint64_t Flags = 0; std::optional InRange; if (BitCode == bitc::CST_CODE_CE_GEP_WITH_INRANGE_INDEX_OLD) { uint64_t Op = Record[OpNum++]; - InBounds = Op & 1; + Flags = Op & 1; // inbounds unsigned InRangeIndex = Op >> 1; // "Upgrade" inrange by dropping it. The feature is too niche to // bother. (void)InRangeIndex; } else if (BitCode == bitc::CST_CODE_CE_GEP_WITH_INRANGE) { - uint64_t Op = Record[OpNum++]; - InBounds = Op & 1; + Flags = Record[OpNum++]; Expected MaybeInRange = readConstantRange(Record, OpNum); if (!MaybeInRange) return MaybeInRange.takeError(); InRange = MaybeInRange.get(); + } else if (BitCode == bitc::CST_CODE_CE_GEP) { + Flags = Record[OpNum++]; } else if (BitCode == bitc::CST_CODE_CE_INBOUNDS_GEP) - InBounds = true; + Flags = (1 << bitc::GEP_INBOUNDS); SmallVector Elts; unsigned BaseTypeID = Record[OpNum]; @@ -3384,7 +3397,8 @@ Error BitcodeReader::parseConstants() { V = BitcodeConstant::create( Alloc, CurTy, - {Instruction::GetElementPtr, InBounds, PointeeType, InRange}, Elts); + {Instruction::GetElementPtr, uint8_t(Flags), PointeeType, InRange}, + Elts); break; } case bitc::CST_CODE_CE_SELECT: { // CE_SELECT: [opval#, opval#, opval#] @@ -5062,14 +5076,15 @@ Error BitcodeReader::parseFunctionBody(Function *F) { unsigned TyID; Type *Ty; - bool InBounds; + GEPNoWrapFlags NW; if (BitCode == bitc::FUNC_CODE_INST_GEP) { - InBounds = Record[OpNum++]; + NW = toGEPNoWrapFlags(Record[OpNum++]); TyID = Record[OpNum++]; Ty = getTypeByID(TyID); } else { - InBounds = BitCode == bitc::FUNC_CODE_INST_INBOUNDS_GEP_OLD; + if (BitCode == bitc::FUNC_CODE_INST_INBOUNDS_GEP_OLD) + NW = GEPNoWrapFlags::inBounds(); TyID = InvalidTypeID; Ty = nullptr; } @@ -5096,7 +5111,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { GEPIdx.push_back(Op); } - I = GetElementPtrInst::Create(Ty, BasePtr, GEPIdx); + auto *GEP = GetElementPtrInst::Create(Ty, BasePtr, GEPIdx); + I = GEP; ResTypeID = TyID; if (cast(I)->getNumIndices() != 0) { @@ -5122,8 +5138,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { ResTypeID = getVirtualTypeID(I->getType(), ResTypeID); InstructionList.push_back(I); - if (InBounds) - cast(I)->setIsInBounds(true); + GEP->setNoWrapFlags(NW); break; } diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index c5fdd1116c9fb2..3d653fe4458f4b 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -1668,6 +1668,13 @@ static uint64_t getOptimizationFlags(const Value *V) { Flags |= 1 << bitc::TIO_NO_SIGNED_WRAP; if (TI->hasNoUnsignedWrap()) Flags |= 1 << bitc::TIO_NO_UNSIGNED_WRAP; + } else if (const auto *GEP = dyn_cast(V)) { + if (GEP->isInBounds()) + Flags |= 1 << bitc::GEP_INBOUNDS; + if (GEP->hasNoUnsignedSignedWrap()) + Flags |= 1 << bitc::GEP_NUSW; + if (GEP->hasNoUnsignedWrap()) + Flags |= 1 << bitc::GEP_NUW; } return Flags; @@ -2779,12 +2786,11 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal, Code = bitc::CST_CODE_CE_GEP; const auto *GO = cast(C); Record.push_back(VE.getTypeID(GO->getSourceElementType())); + Record.push_back(getOptimizationFlags(GO)); if (std::optional Range = GO->getInRange()) { Code = bitc::CST_CODE_CE_GEP_WITH_INRANGE; - Record.push_back(GO->isInBounds()); emitConstantRange(Record, *Range); - } else if (GO->isInBounds()) - Code = bitc::CST_CODE_CE_INBOUNDS_GEP; + } for (unsigned i = 0, e = CE->getNumOperands(); i != e; ++i) { Record.push_back(VE.getTypeID(C->getOperand(i)->getType())); Record.push_back(VE.getValueID(C->getOperand(i))); @@ -2973,7 +2979,7 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Code = bitc::FUNC_CODE_INST_GEP; AbbrevToUse = FUNCTION_INST_GEP_ABBREV; auto &GEPInst = cast(I); - Vals.push_back(GEPInst.isInBounds()); + Vals.push_back(getOptimizationFlags(&I)); Vals.push_back(VE.getTypeID(GEPInst.getSourceElementType())); for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) pushValueAndType(I.getOperand(i), InstID, Vals); @@ -3871,7 +3877,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { { auto Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_GEP)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty Log2_32_Ceil(VE.getTypes().size() + 1))); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index 941f6a7a7d8232..ced5d78f994ab5 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -1417,6 +1417,10 @@ static void WriteOptimizationInfo(raw_ostream &Out, const User *U) { } else if (const GEPOperator *GEP = dyn_cast(U)) { if (GEP->isInBounds()) Out << " inbounds"; + else if (GEP->hasNoUnsignedSignedWrap()) + Out << " nusw"; + if (GEP->hasNoUnsignedWrap()) + Out << " nuw"; if (auto InRange = GEP->getInRange()) { Out << " inrange(" << InRange->getLower() << ", " << InRange->getUpper() << ")"; diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp index 4622ad7e9a0e7d..4c5e1ce3df7c70 100644 --- a/llvm/lib/IR/ConstantFold.cpp +++ b/llvm/lib/IR/ConstantFold.cpp @@ -1721,8 +1721,9 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C, if (auto *GV = dyn_cast(C)) if (!GV->hasExternalWeakLinkage() && GV->getValueType() == PointeeTy && isInBoundsIndices(Idxs)) - return ConstantExpr::getGetElementPtr(PointeeTy, C, Idxs, - /*InBounds=*/true, InRange); + // TODO(gep_nowrap): Can also set NUW here. + return ConstantExpr::getGetElementPtr( + PointeeTy, C, Idxs, GEPNoWrapFlags::inBounds(), InRange); return nullptr; } diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp index db442c54125a7c..cfb89d557db479 100644 --- a/llvm/lib/IR/Constants.cpp +++ b/llvm/lib/IR/Constants.cpp @@ -1568,7 +1568,7 @@ Constant *ConstantExpr::getWithOperands(ArrayRef Ops, Type *Ty, assert(SrcTy || (Ops[0]->getType() == getOperand(0)->getType())); return ConstantExpr::getGetElementPtr( SrcTy ? SrcTy : GEPO->getSourceElementType(), Ops[0], Ops.slice(1), - GEPO->isInBounds(), GEPO->getInRange(), OnlyIfReducedTy); + GEPO->getNoWrapFlags(), GEPO->getInRange(), OnlyIfReducedTy); } case Instruction::ICmp: case Instruction::FCmp: @@ -2348,13 +2348,15 @@ Constant *ConstantExpr::getCompare(unsigned short Predicate, Constant *C1, } Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C, - ArrayRef Idxs, bool InBounds, + ArrayRef Idxs, + GEPNoWrapFlags NW, std::optional InRange, Type *OnlyIfReducedTy) { assert(Ty && "Must specify element type"); assert(isSupportedGetElementPtr(Ty) && "Element type is unsupported!"); - if (Constant *FC = ConstantFoldGetElementPtr(Ty, C, InBounds, InRange, Idxs)) + if (Constant *FC = + ConstantFoldGetElementPtr(Ty, C, NW.isInBounds(), InRange, Idxs)) return FC; // Fold a few common cases. assert(GetElementPtrInst::getIndexedType(Ty, Idxs) && "GEP indices invalid!"); @@ -2390,10 +2392,8 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C, ArgVec.push_back(Idx); } - unsigned SubClassOptionalData = InBounds ? GEPOperator::IsInBounds : 0; const ConstantExprKeyType Key(Instruction::GetElementPtr, ArgVec, 0, - SubClassOptionalData, std::nullopt, Ty, - InRange); + NW.getRaw(), std::nullopt, Ty, InRange); LLVMContextImpl *pImpl = C->getContext().pImpl; return pImpl->ExprConstants.getOrCreate(ReqTy, Key); diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp index 678edc58ad848d..29272e627a1d1d 100644 --- a/llvm/lib/IR/Instruction.cpp +++ b/llvm/lib/IR/Instruction.cpp @@ -441,7 +441,7 @@ void Instruction::dropPoisonGeneratingFlags() { break; case Instruction::GetElementPtr: - cast(this)->setIsInBounds(false); + cast(this)->setNoWrapFlags(GEPNoWrapFlags::none()); break; case Instruction::UIToFP: @@ -660,7 +660,8 @@ void Instruction::copyIRFlags(const Value *V, bool IncludeWrapFlags) { if (auto *SrcGEP = dyn_cast(V)) if (auto *DestGEP = dyn_cast(this)) - DestGEP->setIsInBounds(SrcGEP->isInBounds() || DestGEP->isInBounds()); + DestGEP->setNoWrapFlags(SrcGEP->getNoWrapFlags() | + DestGEP->getNoWrapFlags()); if (auto *NNI = dyn_cast(V)) if (isa(this)) @@ -700,7 +701,8 @@ void Instruction::andIRFlags(const Value *V) { if (auto *SrcGEP = dyn_cast(V)) if (auto *DestGEP = dyn_cast(this)) - DestGEP->setIsInBounds(SrcGEP->isInBounds() && DestGEP->isInBounds()); + DestGEP->setNoWrapFlags(SrcGEP->getNoWrapFlags() & + DestGEP->getNoWrapFlags()); if (auto *NNI = dyn_cast(V)) if (isa(this)) diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index c31d399b01d12f..1213f078d05eca 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -2043,14 +2043,35 @@ bool GetElementPtrInst::hasAllConstantIndices() const { return true; } +void GetElementPtrInst::setNoWrapFlags(GEPNoWrapFlags NW) { + SubclassOptionalData = NW.getRaw(); +} + void GetElementPtrInst::setIsInBounds(bool B) { - cast(this)->setIsInBounds(B); + GEPNoWrapFlags NW = cast(this)->getNoWrapFlags(); + if (B) + NW |= GEPNoWrapFlags::inBounds(); + else + NW = NW.withoutInBounds(); + setNoWrapFlags(NW); +} + +GEPNoWrapFlags GetElementPtrInst::getNoWrapFlags() const { + return cast(this)->getNoWrapFlags(); } bool GetElementPtrInst::isInBounds() const { return cast(this)->isInBounds(); } +bool GetElementPtrInst::hasNoUnsignedSignedWrap() const { + return cast(this)->hasNoUnsignedSignedWrap(); +} + +bool GetElementPtrInst::hasNoUnsignedWrap() const { + return cast(this)->hasNoUnsignedWrap(); +} + bool GetElementPtrInst::accumulateConstantOffset(const DataLayout &DL, APInt &Offset) const { // Delegate to the generic GEPOperator implementation. diff --git a/llvm/lib/IR/Operator.cpp b/llvm/lib/IR/Operator.cpp index 29620ef716f25f..6c9862556f5504 100644 --- a/llvm/lib/IR/Operator.cpp +++ b/llvm/lib/IR/Operator.cpp @@ -42,7 +42,8 @@ bool Operator::hasPoisonGeneratingFlags() const { case Instruction::GetElementPtr: { auto *GEP = cast(this); // Note: inrange exists on constexpr only - return GEP->isInBounds() || GEP->getInRange() != std::nullopt; + return GEP->getNoWrapFlags() != GEPNoWrapFlags::none() || + GEP->getInRange() != std::nullopt; } case Instruction::UIToFP: case Instruction::ZExt: diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp index 1114a8c40114e4..f878bd9465d38b 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp @@ -801,7 +801,7 @@ Value *FatPtrConstMaterializer::materialize(Value *V) { Ops.push_back(cast(U.get())); auto *NewGEP = ConstantExpr::getGetElementPtr( NewSrcTy, Ops[0], ArrayRef(Ops).slice(1), - GEPO->isInBounds(), GEPO->getInRange()); + GEPO->getNoWrapFlags(), GEPO->getInRange()); LLVM_DEBUG(dbgs() << "p7-getting GEP: " << *GEPO << " becomes " << *NewGEP << "\n"); Value *FurtherMap = materialize(NewGEP); diff --git a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp index 08ba08daa9d9a4..471c7ca4d73561 100644 --- a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp +++ b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp @@ -1102,8 +1102,9 @@ bool SeparateConstOffsetFromGEP::splitGEP(GetElementPtrInst *GEP) { // // TODO(jingyue): do some range analysis to keep as many inbounds as // possible. GEPs with inbounds are more friendly to alias analysis. + // TODO(gep_nowrap): Preserve nuw at least. bool GEPWasInBounds = GEP->isInBounds(); - GEP->setIsInBounds(false); + GEP->setNoWrapFlags(GEPNoWrapFlags::none()); // Lowers a GEP to either GEPs with a single index or arithmetic operations. if (LowerGEP) { @@ -1377,8 +1378,9 @@ void SeparateConstOffsetFromGEP::swapGEPOperand(GetElementPtrInst *First, uint64_t ObjectSize; if (!getObjectSize(NewBase, ObjectSize, DAL, TLI) || Offset.ugt(ObjectSize)) { - First->setIsInBounds(false); - Second->setIsInBounds(false); + // TODO(gep_nowrap): Make flag preservation more precise. + First->setNoWrapFlags(GEPNoWrapFlags::none()); + Second->setNoWrapFlags(GEPNoWrapFlags::none()); } else First->setIsInBounds(true); } diff --git a/llvm/lib/Transforms/Utils/FunctionComparator.cpp b/llvm/lib/Transforms/Utils/FunctionComparator.cpp index d95248c84b8602..4c80bfa1bf02db 100644 --- a/llvm/lib/Transforms/Utils/FunctionComparator.cpp +++ b/llvm/lib/Transforms/Utils/FunctionComparator.cpp @@ -436,7 +436,8 @@ int FunctionComparator::cmpConstants(const Constant *L, if (int Res = cmpTypes(GEPL->getSourceElementType(), GEPR->getSourceElementType())) return Res; - if (int Res = cmpNumbers(GEPL->isInBounds(), GEPR->isInBounds())) + if (int Res = cmpNumbers(GEPL->getNoWrapFlags().getRaw(), + GEPR->getNoWrapFlags().getRaw())) return Res; std::optional InRangeL = GEPL->getInRange(); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 3aee17921086d8..e75a1de548f7d4 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1095,7 +1095,10 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe { I->setIsExact(ExactFlags.IsExact); break; case OperationType::GEPOp: - cast(I)->setIsInBounds(GEPFlags.IsInBounds); + // TODO(gep_nowrap): Track the full GEPNoWrapFlags in VPlan. + cast(I)->setNoWrapFlags( + GEPFlags.IsInBounds ? GEPNoWrapFlags::inBounds() + : GEPNoWrapFlags::none()); break; case OperationType::FPMathOp: I->setHasAllowReassoc(FMFs.AllowReassoc); diff --git a/llvm/test/Assembler/flags.ll b/llvm/test/Assembler/flags.ll index e0ad8bf000be15..7d2aafa005e7f4 100644 --- a/llvm/test/Assembler/flags.ll +++ b/llvm/test/Assembler/flags.ll @@ -2,6 +2,7 @@ ; RUN: verify-uselistorder %s @addr = external global i64 +@addr_as1 = external addrspace(1) global i64 define i64 @add_unsigned(i64 %x, i64 %y) { ; CHECK: %z = add nuw i64 %x, %y @@ -316,3 +317,104 @@ define <2 x i32> @test_trunc_both_reversed_vector(<2 x i64> %a) { %res = trunc nsw nuw <2 x i64> %a to <2 x i32> ret <2 x i32> %res } + +define ptr @gep_nuw(ptr %p, i64 %idx) { +; CHECK: %gep = getelementptr nuw i8, ptr %p, i64 %idx + %gep = getelementptr nuw i8, ptr %p, i64 %idx + ret ptr %gep +} + +define ptr @gep_inbounds_nuw(ptr %p, i64 %idx) { +; CHECK: %gep = getelementptr inbounds nuw i8, ptr %p, i64 %idx + %gep = getelementptr inbounds nuw i8, ptr %p, i64 %idx + ret ptr %gep +} + +define ptr @gep_nusw(ptr %p, i64 %idx) { +; CHECK: %gep = getelementptr nusw i8, ptr %p, i64 %idx + %gep = getelementptr nusw i8, ptr %p, i64 %idx + ret ptr %gep +} + +; inbounds implies nusw, so the flag is not printed back. +define ptr @gep_inbounds_nusw(ptr %p, i64 %idx) { +; CHECK: %gep = getelementptr inbounds i8, ptr %p, i64 %idx + %gep = getelementptr inbounds nusw i8, ptr %p, i64 %idx + ret ptr %gep +} + +define ptr @gep_nusw_nuw(ptr %p, i64 %idx) { +; CHECK: %gep = getelementptr nusw nuw i8, ptr %p, i64 %idx + %gep = getelementptr nusw nuw i8, ptr %p, i64 %idx + ret ptr %gep +} + +define ptr @gep_inbounds_nusw_nuw(ptr %p, i64 %idx) { +; CHECK: %gep = getelementptr inbounds nuw i8, ptr %p, i64 %idx + %gep = getelementptr inbounds nusw nuw i8, ptr %p, i64 %idx + ret ptr %gep +} + +define ptr @gep_nuw_nusw_inbounds(ptr %p, i64 %idx) { +; CHECK: %gep = getelementptr inbounds nuw i8, ptr %p, i64 %idx + %gep = getelementptr nuw nusw inbounds i8, ptr %p, i64 %idx + ret ptr %gep +} + +define ptr addrspace(1) @gep_nusw_nuw_as1(ptr addrspace(1) %p, i64 %idx) { +; CHECK: %gep = getelementptr nusw nuw i8, ptr addrspace(1) %p, i64 %idx + %gep = getelementptr nusw nuw i8, ptr addrspace(1) %p, i64 %idx + ret ptr addrspace(1) %gep +} + +define <2 x ptr> @gep_nusw_nuw_vec(<2 x ptr> %p, i64 %idx) { +; CHECK: %gep = getelementptr nusw nuw i8, <2 x ptr> %p, i64 %idx + %gep = getelementptr nusw nuw i8, <2 x ptr> %p, i64 %idx + ret <2 x ptr> %gep +} + +define ptr @const_gep_nuw() { +; CHECK: ret ptr getelementptr nuw (i8, ptr @addr, i64 100) + ret ptr getelementptr nuw (i8, ptr @addr, i64 100) +} + +define ptr @const_gep_inbounds_nuw() { +; CHECK: ret ptr getelementptr inbounds nuw (i8, ptr @addr, i64 100) + ret ptr getelementptr inbounds nuw (i8, ptr @addr, i64 100) +} + +define ptr @const_gep_nusw() { +; CHECK: ret ptr getelementptr nusw (i8, ptr @addr, i64 100) + ret ptr getelementptr nusw (i8, ptr @addr, i64 100) +} + +; inbounds implies nusw, so the flag is not printed back. +define ptr @const_gep_inbounds_nusw() { +; CHECK: ret ptr getelementptr inbounds (i8, ptr @addr, i64 100) + ret ptr getelementptr inbounds nusw (i8, ptr @addr, i64 100) +} + +define ptr @const_gep_nusw_nuw() { +; CHECK: ret ptr getelementptr nusw nuw (i8, ptr @addr, i64 100) + ret ptr getelementptr nusw nuw (i8, ptr @addr, i64 100) +} + +define ptr @const_gep_inbounds_nusw_nuw() { +; CHECK: ret ptr getelementptr inbounds nuw (i8, ptr @addr, i64 100) + ret ptr getelementptr inbounds nusw nuw (i8, ptr @addr, i64 100) +} + +define ptr @const_gep_nuw_nusw_inbounds() { +; CHECK: ret ptr getelementptr inbounds nuw (i8, ptr @addr, i64 100) + ret ptr getelementptr nuw nusw inbounds (i8, ptr @addr, i64 100) +} + +define ptr @const_gep_nuw_inrange() { +; CHECK: ret ptr getelementptr nuw inrange(-8, 16) (i8, ptr @addr, i64 100) + ret ptr getelementptr nuw inrange(-8, 16) (i8, ptr @addr, i64 100) +} + +define ptr addrspace(1) @const_gep_nusw_nuw_as1() { +; CHECK: ret ptr addrspace(1) getelementptr nusw nuw (i8, ptr addrspace(1) @addr_as1, i64 100) + ret ptr addrspace(1) getelementptr nusw nuw (i8, ptr addrspace(1) @addr_as1, i64 100) +} diff --git a/llvm/test/Transforms/InstCombine/freeze.ll b/llvm/test/Transforms/InstCombine/freeze.ll index 391d626a795c7d..5fedb1f8575035 100644 --- a/llvm/test/Transforms/InstCombine/freeze.ll +++ b/llvm/test/Transforms/InstCombine/freeze.ll @@ -1160,6 +1160,28 @@ define i32 @propagate_drop_flags_trunc(i64 %arg) { ret i32 %v1.fr } +define ptr @propagate_drop_flags_gep_nusw(ptr %p) { +; CHECK-LABEL: @propagate_drop_flags_gep_nusw( +; CHECK-NEXT: [[P_FR:%.*]] = freeze ptr [[P:%.*]] +; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[P_FR]], i64 1 +; CHECK-NEXT: ret ptr [[GEP]] +; + %gep = getelementptr nusw i8, ptr %p, i64 1 + %gep.fr = freeze ptr %gep + ret ptr %gep.fr +} + +define ptr @propagate_drop_flags_gep_nuw(ptr %p) { +; CHECK-LABEL: @propagate_drop_flags_gep_nuw( +; CHECK-NEXT: [[P_FR:%.*]] = freeze ptr [[P:%.*]] +; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[P_FR]], i64 1 +; CHECK-NEXT: ret ptr [[GEP]] +; + %gep = getelementptr nuw i8, ptr %p, i64 1 + %gep.fr = freeze ptr %gep + ret ptr %gep.fr +} + declare i32 @llvm.umax.i32(i32 %a, i32 %b) define i32 @freeze_call_with_range_attr(i32 %a) { diff --git a/llvm/test/Transforms/SimplifyCFG/HoistCode.ll b/llvm/test/Transforms/SimplifyCFG/HoistCode.ll index 887d1820168181..e6a255a4b8f086 100644 --- a/llvm/test/Transforms/SimplifyCFG/HoistCode.ll +++ b/llvm/test/Transforms/SimplifyCFG/HoistCode.ll @@ -215,3 +215,63 @@ F: %z2 = trunc nsw nuw i32 %x to i16 ret i16 %z2 } + +define ptr @hoist_gep_flags_both_nuw(i1 %C, ptr %p) { +; CHECK-LABEL: @hoist_gep_flags_both_nuw( +; CHECK-NEXT: common.ret: +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr nuw i8, ptr [[P:%.*]], i64 1 +; CHECK-NEXT: ret ptr [[GEP1]] +; + br i1 %C, label %T, label %F +T: + %gep1 = getelementptr nuw i8, ptr %p, i64 1 + ret ptr %gep1 +F: + %gep2 = getelementptr nuw i8, ptr %p, i64 1 + ret ptr %gep2 +} + +define ptr @hoist_gep_flags_both_nusw(i1 %C, ptr %p) { +; CHECK-LABEL: @hoist_gep_flags_both_nusw( +; CHECK-NEXT: common.ret: +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr nusw i8, ptr [[P:%.*]], i64 1 +; CHECK-NEXT: ret ptr [[GEP1]] +; + br i1 %C, label %T, label %F +T: + %gep1 = getelementptr nusw i8, ptr %p, i64 1 + ret ptr %gep1 +F: + %gep2 = getelementptr nusw i8, ptr %p, i64 1 + ret ptr %gep2 +} + +define ptr @hoist_gep_flags_intersect1(i1 %C, ptr %p) { +; CHECK-LABEL: @hoist_gep_flags_intersect1( +; CHECK-NEXT: common.ret: +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr nusw i8, ptr [[P:%.*]], i64 1 +; CHECK-NEXT: ret ptr [[GEP1]] +; + br i1 %C, label %T, label %F +T: + %gep1 = getelementptr inbounds nuw i8, ptr %p, i64 1 + ret ptr %gep1 +F: + %gep2 = getelementptr nusw i8, ptr %p, i64 1 + ret ptr %gep2 +} + +define ptr @hoist_gep_flags_intersect2(i1 %C, ptr %p) { +; CHECK-LABEL: @hoist_gep_flags_intersect2( +; CHECK-NEXT: common.ret: +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 1 +; CHECK-NEXT: ret ptr [[GEP1]] +; + br i1 %C, label %T, label %F +T: + %gep1 = getelementptr inbounds i8, ptr %p, i64 1 + ret ptr %gep1 +F: + %gep2 = getelementptr nuw i8, ptr %p, i64 1 + ret ptr %gep2 +} diff --git a/llvm/test/tools/llvm-reduce/reduce-flags.ll b/llvm/test/tools/llvm-reduce/reduce-flags.ll index 5d6d1260ac50e0..293504e32f9108 100644 --- a/llvm/test/tools/llvm-reduce/reduce-flags.ll +++ b/llvm/test/tools/llvm-reduce/reduce-flags.ll @@ -57,18 +57,26 @@ define i32 @ashr_exact_keep(i32 %a, i32 %b) { ret i32 %op } -; CHECK-LABEL: @getelementptr_inbounds_drop( +; CHECK-LABEL: @getelementptr_inbounds_nuw_drop_both( ; INTERESTING: getelementptr ; RESULT: getelementptr i32, ptr %a, i64 %b -define ptr @getelementptr_inbounds_drop(ptr %a, i64 %b) { - %op = getelementptr inbounds i32, ptr %a, i64 %b +define ptr @getelementptr_inbounds_nuw_drop_both(ptr %a, i64 %b) { + %op = getelementptr inbounds nuw i32, ptr %a, i64 %b ret ptr %op } -; CHECK-LABEL: @getelementptr_inbounds_keep( +; CHECK-LABEL: @getelementptr_inbounds_keep_only_inbounds( ; INTERESTING: inbounds ; RESULT: getelementptr inbounds i32, ptr %a, i64 %b -define ptr @getelementptr_inbounds_keep(ptr %a, i64 %b) { +define ptr @getelementptr_inbounds_keep_only_inbounds(ptr %a, i64 %b) { + %op = getelementptr inbounds nuw i32, ptr %a, i64 %b + ret ptr %op +} + +; CHECK-LABEL: @getelementptr_inbounds_relax_to_nusw( +; INTERESTING: getelementptr {{inbounds|nusw}} +; RESULT: getelementptr nusw i32, ptr %a, i64 %b +define ptr @getelementptr_inbounds_relax_to_nusw(ptr %a, i64 %b) { %op = getelementptr inbounds i32, ptr %a, i64 %b ret ptr %op } diff --git a/llvm/tools/llvm-reduce/deltas/ReduceInstructionFlags.cpp b/llvm/tools/llvm-reduce/deltas/ReduceInstructionFlags.cpp index ad619a6c02a4d2..ba345d3659b22f 100644 --- a/llvm/tools/llvm-reduce/deltas/ReduceInstructionFlags.cpp +++ b/llvm/tools/llvm-reduce/deltas/ReduceInstructionFlags.cpp @@ -42,8 +42,14 @@ static void reduceFlagsInModule(Oracle &O, ReducerWorkItem &WorkItem) { if (PDI->isDisjoint() && !O.shouldKeep()) PDI->setIsDisjoint(false); } else if (auto *GEP = dyn_cast(&I)) { - if (GEP->isInBounds() && !O.shouldKeep()) - GEP->setIsInBounds(false); + GEPNoWrapFlags NW = GEP->getNoWrapFlags(); + if (NW.isInBounds() && !O.shouldKeep()) + NW = NW.withoutInBounds(); + if (NW.hasNoUnsignedSignedWrap() && !O.shouldKeep()) + NW = NW.withoutNoUnsignedSignedWrap(); + if (NW.hasNoUnsignedWrap() && !O.shouldKeep()) + NW = NW.withoutNoUnsignedWrap(); + GEP->setNoWrapFlags(NW); } else if (auto *FPOp = dyn_cast(&I)) { FastMathFlags Flags = FPOp->getFastMathFlags();