From 6227eb90da2a417125477b27799ae2e8f43b9e49 Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Wed, 26 Nov 2025 12:37:09 -0800 Subject: [PATCH 01/22] Add IR and codegen support for deactivation symbols. Deactivation symbols are a mechanism for allowing object files to disable specific instructions in other object files at link time. The initial use case is for pointer field protection. For more information, see the RFC: https://discourse.llvm.org/t/rfc-deactivation-symbols/85556 Reviewers: ojhunt, nikic, fmayer, arsenm, ahmedbougacha Reviewed By: fmayer Pull Request: https://github.com/llvm/llvm-project/pull/133536 --- llvm/docs/LangRef.rst | 16 ++++ .../llvm/CodeGen/GlobalISel/CallLowering.h | 2 + .../CodeGen/GlobalISel/MachineIRBuilder.h | 5 ++ llvm/include/llvm/CodeGen/ISDOpcodes.h | 4 + llvm/include/llvm/CodeGen/MachineFunction.h | 2 +- llvm/include/llvm/CodeGen/MachineInstr.h | 40 +++++++--- .../llvm/CodeGen/MachineInstrBuilder.h | 27 +++++-- llvm/include/llvm/CodeGen/SelectionDAG.h | 1 + llvm/include/llvm/CodeGen/SelectionDAGISel.h | 1 + llvm/include/llvm/CodeGen/SelectionDAGNodes.h | 16 ++++ llvm/include/llvm/CodeGen/TargetLowering.h | 6 ++ llvm/include/llvm/IR/LLVMContext.h | 3 +- llvm/include/llvm/Target/Target.td | 1 + llvm/lib/CodeGen/GlobalISel/CallLowering.cpp | 4 + llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 3 + .../CodeGen/GlobalISel/MachineIRBuilder.cpp | 6 +- llvm/lib/CodeGen/MIRParser/MILexer.cpp | 1 + llvm/lib/CodeGen/MIRParser/MILexer.h | 1 + llvm/lib/CodeGen/MIRParser/MIParser.cpp | 11 +++ llvm/lib/CodeGen/MIRPrinter.cpp | 5 ++ llvm/lib/CodeGen/MachineFunction.cpp | 4 +- llvm/lib/CodeGen/MachineInstr.cpp | 43 ++++++++--- .../lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 33 ++++++--- .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 15 ++++ .../SelectionDAG/SelectionDAGBuilder.cpp | 19 ++++- .../CodeGen/SelectionDAG/SelectionDAGISel.cpp | 14 +++- llvm/lib/IR/Instructions.cpp | 6 +- llvm/lib/IR/LLVMContext.cpp | 2 + llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 69 +++++++++++++----- .../Target/AArch64/AArch64ISelDAGToDAG.cpp | 5 +- .../Target/AArch64/AArch64ISelLowering.cpp | 3 + .../lib/Target/AArch64/AArch64InstrFormats.td | 3 + llvm/lib/Target/AArch64/AArch64InstrInfo.td | 1 + .../AArch64/GISel/AArch64CallLowering.cpp | 1 + .../InstCombine/InstCombineCalls.cpp | 10 +++ .../Bitcode/operand-bundles-bc-analyzer.ll | 1 + .../CodeGen/AArch64/deactivation-symbols.ll | 73 +++++++++++++++++++ .../MIR/AArch64/deactivation-symbols.mir | 12 +++ .../InstCombine/ptrauth-intrinsics.ll | 28 +++++++ llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 11 ++- 40 files changed, 441 insertions(+), 67 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/deactivation-symbols.ll create mode 100644 llvm/test/CodeGen/MIR/AArch64/deactivation-symbols.mir diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index 33c23f2949765..7cd7e815fd8d7 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -3234,6 +3234,22 @@ A "convergencectrl" operand bundle is only valid on a ``convergent`` operation. When present, the operand bundle must contain exactly one value of token type. See the :doc:`ConvergentOperations` document for details. +Deactivation Symbol Operand Bundles +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A ``"deactivation-symbol"`` operand bundle is valid on the following +instructions (AArch64 only): + +- Call to a normal function with ``notail`` attribute and a first argument and + return value of type ``ptr``. +- Call to ``llvm.ptrauth.sign`` or ``llvm.ptrauth.auth`` intrinsics. + +This operand bundle specifies that if the deactivation symbol is defined +to a valid value for the target, the marked instruction will return the +value of its first argument instead of calling the specified function +or intrinsic. This is achieved with ``PATCHINST`` relocations on the +target instructions (see the AArch64 psABI for details). + .. _moduleasm: Module-Level Inline Assembly diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h index a8bde824527a5..fea900f37ec74 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h @@ -159,6 +159,8 @@ class LLVM_ABI CallLowering { /// True if this call results in convergent operations. bool IsConvergent = true; + + GlobalValue *DeactivationSymbol = nullptr; }; /// Argument handling is mostly uniform between the four places that diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index 40c7792f7e8a2..5f3f1d386569c 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -56,6 +56,7 @@ struct MachineIRBuilderState { MDNode *PCSections = nullptr; /// MMRA Metadata to be set on any instruction we create. MDNode *MMRA = nullptr; + Value *DS = nullptr; /// \name Fields describing the insertion point. /// @{ @@ -369,6 +370,7 @@ class LLVM_ABI MachineIRBuilder { State.II = MI.getIterator(); setPCSections(MI.getPCSections()); setMMRAMetadata(MI.getMMRAMetadata()); + setDeactivationSymbol(MI.getDeactivationSymbol()); } /// @} @@ -405,6 +407,9 @@ class LLVM_ABI MachineIRBuilder { /// Set the PC sections metadata to \p MD for all the next build instructions. void setMMRAMetadata(MDNode *MMRA) { State.MMRA = MMRA; } + Value *getDeactivationSymbol() { return State.DS; } + void setDeactivationSymbol(Value *DS) { State.DS = DS; } + /// Get the current instruction's MMRA metadata. MDNode *getMMRAMetadata() { return State.MMRA; } diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h index cdaa916548c25..b32f3dacbb3a4 100644 --- a/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -1579,6 +1579,10 @@ enum NodeType { // Outputs: Output Chain CLEAR_CACHE, + // Untyped node storing deactivation symbol reference + // (DeactivationSymbolSDNode). + DEACTIVATION_SYMBOL, + /// BUILTIN_OP_END - This must be the last enum value in this list. /// The target-specific pre-isel opcode values start here. BUILTIN_OP_END diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h index ef783f276b7d4..08ffdb2cb469d 100644 --- a/llvm/include/llvm/CodeGen/MachineFunction.h +++ b/llvm/include/llvm/CodeGen/MachineFunction.h @@ -1207,7 +1207,7 @@ class LLVM_ABI MachineFunction { ArrayRef MMOs, MCSymbol *PreInstrSymbol = nullptr, MCSymbol *PostInstrSymbol = nullptr, MDNode *HeapAllocMarker = nullptr, MDNode *PCSections = nullptr, uint32_t CFIType = 0, - MDNode *MMRAs = nullptr); + MDNode *MMRAs = nullptr, Value *DS = nullptr); /// Allocate a string and populate it with the given external symbol name. const char *createExternalSymbolName(StringRef Name); diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h index ca984459c365a..077e39b49df6f 100644 --- a/llvm/include/llvm/CodeGen/MachineInstr.h +++ b/llvm/include/llvm/CodeGen/MachineInstr.h @@ -160,8 +160,9 @@ class MachineInstr /// /// This has to be defined eagerly due to the implementation constraints of /// `PointerSumType` where it is used. - class ExtraInfo final : TrailingObjects { + class ExtraInfo final + : TrailingObjects { public: static ExtraInfo *create(BumpPtrAllocator &Allocator, ArrayRef MMOs, @@ -169,20 +170,23 @@ class MachineInstr MCSymbol *PostInstrSymbol = nullptr, MDNode *HeapAllocMarker = nullptr, MDNode *PCSections = nullptr, uint32_t CFIType = 0, - MDNode *MMRAs = nullptr) { + MDNode *MMRAs = nullptr, Value *DS = nullptr) { bool HasPreInstrSymbol = PreInstrSymbol != nullptr; bool HasPostInstrSymbol = PostInstrSymbol != nullptr; bool HasHeapAllocMarker = HeapAllocMarker != nullptr; bool HasMMRAs = MMRAs != nullptr; bool HasCFIType = CFIType != 0; bool HasPCSections = PCSections != nullptr; + bool HasDS = DS != nullptr; auto *Result = new (Allocator.Allocate( - totalSizeToAlloc( + totalSizeToAlloc( MMOs.size(), HasPreInstrSymbol + HasPostInstrSymbol, - HasHeapAllocMarker + HasPCSections + HasMMRAs, HasCFIType), + HasHeapAllocMarker + HasPCSections + HasMMRAs, HasCFIType, HasDS), alignof(ExtraInfo))) ExtraInfo(MMOs.size(), HasPreInstrSymbol, HasPostInstrSymbol, - HasHeapAllocMarker, HasPCSections, HasCFIType, HasMMRAs); + HasHeapAllocMarker, HasPCSections, HasCFIType, HasMMRAs, + HasDS); // Copy the actual data into the trailing objects. llvm::copy(MMOs, Result->getTrailingObjects()); @@ -202,6 +206,8 @@ class MachineInstr Result->getTrailingObjects()[0] = CFIType; if (HasMMRAs) Result->getTrailingObjects()[MDNodeIdx++] = MMRAs; + if (HasDS) + Result->getTrailingObjects()[0] = DS; return Result; } @@ -240,6 +246,10 @@ class MachineInstr : nullptr; } + Value *getDeactivationSymbol() const { + return HasDS ? getTrailingObjects()[0] : 0; + } + private: friend TrailingObjects; @@ -255,6 +265,7 @@ class MachineInstr const bool HasPCSections; const bool HasCFIType; const bool HasMMRAs; + const bool HasDS; // Implement the `TrailingObjects` internal API. size_t numTrailingObjects(OverloadToken) const { @@ -269,16 +280,17 @@ class MachineInstr size_t numTrailingObjects(OverloadToken) const { return HasCFIType; } + size_t numTrailingObjects(OverloadToken) const { return HasDS; } // Just a boring constructor to allow us to initialize the sizes. Always use // the `create` routine above. ExtraInfo(int NumMMOs, bool HasPreInstrSymbol, bool HasPostInstrSymbol, bool HasHeapAllocMarker, bool HasPCSections, bool HasCFIType, - bool HasMMRAs) + bool HasMMRAs, bool HasDS) : NumMMOs(NumMMOs), HasPreInstrSymbol(HasPreInstrSymbol), HasPostInstrSymbol(HasPostInstrSymbol), HasHeapAllocMarker(HasHeapAllocMarker), HasPCSections(HasPCSections), - HasCFIType(HasCFIType), HasMMRAs(HasMMRAs) {} + HasCFIType(HasCFIType), HasMMRAs(HasMMRAs), HasDS(HasDS) {} }; /// Enumeration of the kinds of inline extra info available. It is important @@ -867,6 +879,14 @@ class MachineInstr return nullptr; } + Value *getDeactivationSymbol() const { + if (!Info) + return nullptr; + if (ExtraInfo *EI = Info.get()) + return EI->getDeactivationSymbol(); + return nullptr; + } + /// Helper to extract a CFI type hash if one has been added. uint32_t getCFIType() const { if (!Info) @@ -1969,6 +1989,8 @@ class MachineInstr /// Set the CFI type for the instruction. LLVM_ABI void setCFIType(MachineFunction &MF, uint32_t Type); + LLVM_ABI void setDeactivationSymbol(MachineFunction &MF, Value *DS); + /// Return the MIFlags which represent both MachineInstrs. This /// should be used when merging two MachineInstrs into one. This routine does /// not modify the MIFlags of this MachineInstr. @@ -2088,7 +2110,7 @@ class MachineInstr void setExtraInfo(MachineFunction &MF, ArrayRef MMOs, MCSymbol *PreInstrSymbol, MCSymbol *PostInstrSymbol, MDNode *HeapAllocMarker, MDNode *PCSections, - uint32_t CFIType, MDNode *MMRAs); + uint32_t CFIType, MDNode *MMRAs, Value *DS); }; /// Special DenseMapInfo traits to compare MachineInstr* by *value* of the diff --git a/llvm/include/llvm/CodeGen/MachineInstrBuilder.h b/llvm/include/llvm/CodeGen/MachineInstrBuilder.h index e705d7d99544c..caeb430d6fd1c 100644 --- a/llvm/include/llvm/CodeGen/MachineInstrBuilder.h +++ b/llvm/include/llvm/CodeGen/MachineInstrBuilder.h @@ -70,29 +70,44 @@ enum { } // end namespace RegState /// Set of metadata that should be preserved when using BuildMI(). This provides -/// a more convenient way of preserving DebugLoc, PCSections and MMRA. +/// a more convenient way of preserving certain data from the original +/// instruction. class MIMetadata { public: MIMetadata() = default; - MIMetadata(DebugLoc DL, MDNode *PCSections = nullptr, MDNode *MMRA = nullptr) - : DL(std::move(DL)), PCSections(PCSections), MMRA(MMRA) {} + MIMetadata(DebugLoc DL, MDNode *PCSections = nullptr, MDNode *MMRA = nullptr, + Value *DeactivationSymbol = nullptr) + : DL(std::move(DL)), PCSections(PCSections), MMRA(MMRA), + DeactivationSymbol(DeactivationSymbol) {} MIMetadata(const DILocation *DI, MDNode *PCSections = nullptr, MDNode *MMRA = nullptr) : DL(DI), PCSections(PCSections), MMRA(MMRA) {} explicit MIMetadata(const Instruction &From) : DL(From.getDebugLoc()), - PCSections(From.getMetadata(LLVMContext::MD_pcsections)) {} + PCSections(From.getMetadata(LLVMContext::MD_pcsections)), + DeactivationSymbol(getDeactivationSymbol(&From)) {} explicit MIMetadata(const MachineInstr &From) - : DL(From.getDebugLoc()), PCSections(From.getPCSections()) {} + : DL(From.getDebugLoc()), PCSections(From.getPCSections()), + DeactivationSymbol(From.getDeactivationSymbol()) {} const DebugLoc &getDL() const { return DL; } MDNode *getPCSections() const { return PCSections; } MDNode *getMMRAMetadata() const { return MMRA; } + Value *getDeactivationSymbol() const { return DeactivationSymbol; } private: DebugLoc DL; MDNode *PCSections = nullptr; MDNode *MMRA = nullptr; + Value *DeactivationSymbol = nullptr; + + static inline Value *getDeactivationSymbol(const Instruction *I) { + if (auto *CB = dyn_cast(I)) + if (auto Bundle = + CB->getOperandBundle(llvm::LLVMContext::OB_deactivation_symbol)) + return Bundle->Inputs[0].get(); + return nullptr; + } }; class MachineInstrBuilder { @@ -348,6 +363,8 @@ class MachineInstrBuilder { MI->setPCSections(*MF, MIMD.getPCSections()); if (MIMD.getMMRAMetadata()) MI->setMMRAMetadata(*MF, MIMD.getMMRAMetadata()); + if (MIMD.getDeactivationSymbol()) + MI->setDeactivationSymbol(*MF, MIMD.getDeactivationSymbol()); return *this; } diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index b024e8a68bd6e..501cbc947132e 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -759,6 +759,7 @@ class SelectionDAG { int64_t offset = 0, unsigned TargetFlags = 0) { return getGlobalAddress(GV, DL, VT, offset, true, TargetFlags); } + LLVM_ABI SDValue getDeactivationSymbol(const GlobalValue *GV); LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget = false); SDValue getTargetFrameIndex(int FI, EVT VT) { return getFrameIndex(FI, VT, true); diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h index c5cdf76f4777e..7add717227963 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h @@ -151,6 +151,7 @@ class SelectionDAGISel { OPC_RecordChild7, OPC_RecordMemRef, OPC_CaptureGlueInput, + OPC_CaptureDeactivationSymbol, OPC_MoveChild, OPC_MoveChild0, OPC_MoveChild1, diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index cfc8a4243e894..aa72e81b2ab54 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -2005,6 +2005,22 @@ class GlobalAddressSDNode : public SDNode { } }; +class DeactivationSymbolSDNode : public SDNode { + friend class SelectionDAG; + + const GlobalValue *TheGlobal; + + DeactivationSymbolSDNode(const GlobalValue *GV, SDVTList VTs) + : SDNode(ISD::DEACTIVATION_SYMBOL, 0, DebugLoc(), VTs), TheGlobal(GV) {} + +public: + const GlobalValue *getGlobal() const { return TheGlobal; } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::DEACTIVATION_SYMBOL; + } +}; + class FrameIndexSDNode : public SDNode { friend class SelectionDAG; diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 7df5d8a09f0f6..b2697c81fd825 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -4765,6 +4765,7 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase { SmallVector InVals; const ConstantInt *CFIType = nullptr; SDValue ConvergenceControlToken; + GlobalValue *DeactivationSymbol = nullptr; std::optional PAI; @@ -4918,6 +4919,11 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase { return *this; } + CallLoweringInfo &setDeactivationSymbol(GlobalValue *Sym) { + DeactivationSymbol = Sym; + return *this; + } + ArgListTy &getArgs() { return Args; } diff --git a/llvm/include/llvm/IR/LLVMContext.h b/llvm/include/llvm/IR/LLVMContext.h index 5972dcb637dfa..d938f4609742b 100644 --- a/llvm/include/llvm/IR/LLVMContext.h +++ b/llvm/include/llvm/IR/LLVMContext.h @@ -98,7 +98,8 @@ class LLVMContext { OB_kcfi = 8, // "kcfi" OB_convergencectrl = 9, // "convergencectrl" OB_align = 10, // "align" - OB_LastBundleID = OB_align // Marker for last bundle ID + OB_deactivation_symbol = 11, // "deactivation-symbol" + OB_LastBundleID = OB_deactivation_symbol }; /// getMDKindID - Return a unique non-zero ID for the specified metadata kind. diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td index 96a7d7c2091d2..54162dc6bb30f 100644 --- a/llvm/include/llvm/Target/Target.td +++ b/llvm/include/llvm/Target/Target.td @@ -694,6 +694,7 @@ class Instruction : InstructionEncoding { // If so, make sure to override // TargetInstrInfo::getInsertSubregLikeInputs. bit variadicOpsAreDefs = false; // Are variadic operands definitions? + bit supportsDeactivationSymbol = false; // Does the instruction have side effects that are not captured by any // operands of the instruction or other flags? diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp index 7be7468300569..e2ed45eec0ecd 100644 --- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp @@ -196,6 +196,10 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB, assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type"); } + if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_deactivation_symbol)) { + Info.DeactivationSymbol = cast(Bundle->Inputs[0]); + } + Info.CB = &CB; Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees); Info.CallConv = CallConv; diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 2ec138b6e186d..e0665d99a891d 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -2917,6 +2917,9 @@ bool IRTranslator::translateIntrinsic( } } + if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_deactivation_symbol)) + MIB->setDeactivationSymbol(*MF, Bundle->Inputs[0].get()); + return true; } diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index 637acd61c8a5f..3906b311addf0 100644 --- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -38,8 +38,10 @@ void MachineIRBuilder::setMF(MachineFunction &MF) { //------------------------------------------------------------------------------ MachineInstrBuilder MachineIRBuilder::buildInstrNoInsert(unsigned Opcode) { - return BuildMI(getMF(), {getDL(), getPCSections(), getMMRAMetadata()}, - getTII().get(Opcode)); + return BuildMI( + getMF(), + {getDL(), getPCSections(), getMMRAMetadata(), getDeactivationSymbol()}, + getTII().get(Opcode)); } MachineInstrBuilder MachineIRBuilder::insertInstr(MachineInstrBuilder MIB) { diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.cpp b/llvm/lib/CodeGen/MIRParser/MILexer.cpp index 8b72c295416a2..dbd56c7414f38 100644 --- a/llvm/lib/CodeGen/MIRParser/MILexer.cpp +++ b/llvm/lib/CodeGen/MIRParser/MILexer.cpp @@ -281,6 +281,7 @@ static MIToken::TokenKind getIdentifierKind(StringRef Identifier) { .Case("heap-alloc-marker", MIToken::kw_heap_alloc_marker) .Case("pcsections", MIToken::kw_pcsections) .Case("cfi-type", MIToken::kw_cfi_type) + .Case("deactivation-symbol", MIToken::kw_deactivation_symbol) .Case("bbsections", MIToken::kw_bbsections) .Case("bb_id", MIToken::kw_bb_id) .Case("unknown-size", MIToken::kw_unknown_size) diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.h b/llvm/lib/CodeGen/MIRParser/MILexer.h index 0627f176b9e00..0407a0e7540d7 100644 --- a/llvm/lib/CodeGen/MIRParser/MILexer.h +++ b/llvm/lib/CodeGen/MIRParser/MILexer.h @@ -136,6 +136,7 @@ struct MIToken { kw_heap_alloc_marker, kw_pcsections, kw_cfi_type, + kw_deactivation_symbol, kw_bbsections, kw_bb_id, kw_unknown_size, diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp index 434a579c3be3f..f35274d4e2edf 100644 --- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp +++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp @@ -1072,6 +1072,7 @@ bool MIParser::parse(MachineInstr *&MI) { Token.isNot(MIToken::kw_heap_alloc_marker) && Token.isNot(MIToken::kw_pcsections) && Token.isNot(MIToken::kw_cfi_type) && + Token.isNot(MIToken::kw_deactivation_symbol) && Token.isNot(MIToken::kw_debug_location) && Token.isNot(MIToken::kw_debug_instr_number) && Token.isNot(MIToken::coloncolon) && Token.isNot(MIToken::lbrace)) { @@ -1120,6 +1121,14 @@ bool MIParser::parse(MachineInstr *&MI) { lex(); } + GlobalValue *DS = nullptr; + if (Token.is(MIToken::kw_deactivation_symbol)) { + lex(); + if (parseGlobalValue(DS)) + return true; + lex(); + } + unsigned InstrNum = 0; if (Token.is(MIToken::kw_debug_instr_number)) { lex(); @@ -1196,6 +1205,8 @@ bool MIParser::parse(MachineInstr *&MI) { MI->setPCSections(MF, PCSections); if (CFIType) MI->setCFIType(MF, CFIType); + if (DS) + MI->setDeactivationSymbol(MF, DS); if (!MemOperands.empty()) MI->setMemRefs(MF, MemOperands); if (InstrNum) diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp index 1d54d72336860..c0554497653f8 100644 --- a/llvm/lib/CodeGen/MIRPrinter.cpp +++ b/llvm/lib/CodeGen/MIRPrinter.cpp @@ -19,6 +19,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" +#include "llvm/CodeGen/MIRFormatter.h" #include "llvm/CodeGen/MIRYamlMapping.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineConstantPool.h" @@ -895,6 +896,10 @@ static void printMI(raw_ostream &OS, MFPrintState &State, } if (uint32_t CFIType = MI.getCFIType()) OS << LS << "cfi-type " << CFIType; + if (Value *DS = MI.getDeactivationSymbol()) { + OS << LS << "deactivation-symbol "; + MIRFormatter::printIRValue(OS, *DS, State.MST); + } if (auto Num = MI.peekDebugInstrNum()) OS << LS << "debug-instr-number " << Num; diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp index bfa5ab274c686..634547ded992f 100644 --- a/llvm/lib/CodeGen/MachineFunction.cpp +++ b/llvm/lib/CodeGen/MachineFunction.cpp @@ -609,10 +609,10 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO, MachineInstr::ExtraInfo *MachineFunction::createMIExtraInfo( ArrayRef MMOs, MCSymbol *PreInstrSymbol, MCSymbol *PostInstrSymbol, MDNode *HeapAllocMarker, MDNode *PCSections, - uint32_t CFIType, MDNode *MMRAs) { + uint32_t CFIType, MDNode *MMRAs, Value *DS) { return MachineInstr::ExtraInfo::create(Allocator, MMOs, PreInstrSymbol, PostInstrSymbol, HeapAllocMarker, - PCSections, CFIType, MMRAs); + PCSections, CFIType, MMRAs, DS); } const char *MachineFunction::createExternalSymbolName(StringRef Name) { diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp index eb46124d9eb5f..18111156efa4f 100644 --- a/llvm/lib/CodeGen/MachineInstr.cpp +++ b/llvm/lib/CodeGen/MachineInstr.cpp @@ -322,15 +322,17 @@ void MachineInstr::setExtraInfo(MachineFunction &MF, MCSymbol *PreInstrSymbol, MCSymbol *PostInstrSymbol, MDNode *HeapAllocMarker, MDNode *PCSections, - uint32_t CFIType, MDNode *MMRAs) { + uint32_t CFIType, MDNode *MMRAs, Value *DS) { bool HasPreInstrSymbol = PreInstrSymbol != nullptr; bool HasPostInstrSymbol = PostInstrSymbol != nullptr; bool HasHeapAllocMarker = HeapAllocMarker != nullptr; bool HasPCSections = PCSections != nullptr; bool HasCFIType = CFIType != 0; bool HasMMRAs = MMRAs != nullptr; + bool HasDS = DS != nullptr; int NumPointers = MMOs.size() + HasPreInstrSymbol + HasPostInstrSymbol + - HasHeapAllocMarker + HasPCSections + HasCFIType + HasMMRAs; + HasHeapAllocMarker + HasPCSections + HasCFIType + HasMMRAs + + HasDS; // Drop all extra info if there is none. if (NumPointers <= 0) { @@ -343,10 +345,10 @@ void MachineInstr::setExtraInfo(MachineFunction &MF, // 32-bit pointers. // FIXME: Maybe we should make the symbols in the extra info mutable? else if (NumPointers > 1 || HasMMRAs || HasHeapAllocMarker || HasPCSections || - HasCFIType) { + HasCFIType || HasDS) { Info.set( MF.createMIExtraInfo(MMOs, PreInstrSymbol, PostInstrSymbol, - HeapAllocMarker, PCSections, CFIType, MMRAs)); + HeapAllocMarker, PCSections, CFIType, MMRAs, DS)); return; } @@ -365,7 +367,7 @@ void MachineInstr::dropMemRefs(MachineFunction &MF) { setExtraInfo(MF, {}, getPreInstrSymbol(), getPostInstrSymbol(), getHeapAllocMarker(), getPCSections(), getCFIType(), - getMMRAMetadata()); + getMMRAMetadata(), getDeactivationSymbol()); } void MachineInstr::setMemRefs(MachineFunction &MF, @@ -377,7 +379,7 @@ void MachineInstr::setMemRefs(MachineFunction &MF, setExtraInfo(MF, MMOs, getPreInstrSymbol(), getPostInstrSymbol(), getHeapAllocMarker(), getPCSections(), getCFIType(), - getMMRAMetadata()); + getMMRAMetadata(), getDeactivationSymbol()); } void MachineInstr::addMemOperand(MachineFunction &MF, @@ -488,7 +490,7 @@ void MachineInstr::setPreInstrSymbol(MachineFunction &MF, MCSymbol *Symbol) { setExtraInfo(MF, memoperands(), Symbol, getPostInstrSymbol(), getHeapAllocMarker(), getPCSections(), getCFIType(), - getMMRAMetadata()); + getMMRAMetadata(), getDeactivationSymbol()); } void MachineInstr::setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol) { @@ -504,7 +506,7 @@ void MachineInstr::setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol) { setExtraInfo(MF, memoperands(), getPreInstrSymbol(), Symbol, getHeapAllocMarker(), getPCSections(), getCFIType(), - getMMRAMetadata()); + getMMRAMetadata(), getDeactivationSymbol()); } void MachineInstr::setHeapAllocMarker(MachineFunction &MF, MDNode *Marker) { @@ -513,7 +515,8 @@ void MachineInstr::setHeapAllocMarker(MachineFunction &MF, MDNode *Marker) { return; setExtraInfo(MF, memoperands(), getPreInstrSymbol(), getPostInstrSymbol(), - Marker, getPCSections(), getCFIType(), getMMRAMetadata()); + Marker, getPCSections(), getCFIType(), getMMRAMetadata(), + getDeactivationSymbol()); } void MachineInstr::setPCSections(MachineFunction &MF, MDNode *PCSections) { @@ -523,7 +526,7 @@ void MachineInstr::setPCSections(MachineFunction &MF, MDNode *PCSections) { setExtraInfo(MF, memoperands(), getPreInstrSymbol(), getPostInstrSymbol(), getHeapAllocMarker(), PCSections, getCFIType(), - getMMRAMetadata()); + getMMRAMetadata(), getDeactivationSymbol()); } void MachineInstr::setCFIType(MachineFunction &MF, uint32_t Type) { @@ -532,7 +535,8 @@ void MachineInstr::setCFIType(MachineFunction &MF, uint32_t Type) { return; setExtraInfo(MF, memoperands(), getPreInstrSymbol(), getPostInstrSymbol(), - getHeapAllocMarker(), getPCSections(), Type, getMMRAMetadata()); + getHeapAllocMarker(), getPCSections(), Type, getMMRAMetadata(), + getDeactivationSymbol()); } void MachineInstr::setMMRAMetadata(MachineFunction &MF, MDNode *MMRAs) { @@ -541,7 +545,18 @@ void MachineInstr::setMMRAMetadata(MachineFunction &MF, MDNode *MMRAs) { return; setExtraInfo(MF, memoperands(), getPreInstrSymbol(), getPostInstrSymbol(), - getHeapAllocMarker(), getPCSections(), getCFIType(), MMRAs); + getHeapAllocMarker(), getPCSections(), getCFIType(), MMRAs, + getDeactivationSymbol()); +} + +void MachineInstr::setDeactivationSymbol(MachineFunction &MF, Value *DS) { + // Do nothing if old and new symbols are the same. + if (DS == getDeactivationSymbol()) + return; + + setExtraInfo(MF, memoperands(), getPreInstrSymbol(), getPostInstrSymbol(), + getHeapAllocMarker(), getPCSections(), getCFIType(), + getMMRAMetadata(), DS); } void MachineInstr::cloneInstrSymbols(MachineFunction &MF, @@ -730,6 +745,8 @@ bool MachineInstr::isIdenticalTo(const MachineInstr &Other, // Call instructions with different CFI types are not identical. if (isCall() && getCFIType() != Other.getCFIType()) return false; + if (getDeactivationSymbol() != Other.getDeactivationSymbol()) + return false; return true; } @@ -2037,6 +2054,8 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST, OS << ','; OS << " cfi-type " << CFIType; } + if (getDeactivationSymbol()) + OS << ", deactivation-symbol " << getDeactivationSymbol()->getName(); if (DebugInstrNum) { if (!FirstOp) diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index 52e8449fe510c..4ad721bf21959 100644 --- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -15,10 +15,12 @@ #include "InstrEmitter.h" #include "SDNodeDbgValue.h" #include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/StackMaps.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetLowering.h" @@ -61,6 +63,8 @@ static unsigned countOperands(SDNode *Node, unsigned NumExpUses, unsigned N = Node->getNumOperands(); while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue) --N; + if (N && Node->getOperand(N - 1).getOpcode() == ISD::DEACTIVATION_SYMBOL) + --N; // Ignore deactivation symbol if it exists. if (N && Node->getOperand(N - 1).getValueType() == MVT::Other) --N; // Ignore chain if it exists. @@ -1222,15 +1226,23 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, } } - if (SDNode *GluedNode = Node->getGluedNode()) { - // FIXME: Possibly iterate over multiple glue nodes? - if (GluedNode->getOpcode() == - ~(unsigned)TargetOpcode::CONVERGENCECTRL_GLUE) { - Register VReg = getVR(GluedNode->getOperand(0), VRBaseMap); - MachineOperand MO = MachineOperand::CreateReg(VReg, /*isDef=*/false, - /*isImp=*/true); - MIB->addOperand(MO); - } + unsigned Op = Node->getNumOperands(); + if (Op != 0 && Node->getOperand(Op - 1)->getOpcode() == + ~(unsigned)TargetOpcode::CONVERGENCECTRL_GLUE) { + Register VReg = getVR(Node->getOperand(Op - 1)->getOperand(0), VRBaseMap); + MachineOperand MO = MachineOperand::CreateReg(VReg, /*isDef=*/false, + /*isImp=*/true); + MIB->addOperand(MO); + Op--; + } + + if (Op != 0 && + Node->getOperand(Op - 1)->getOpcode() == ISD::DEACTIVATION_SYMBOL) { + MI->setDeactivationSymbol( + *MF, const_cast( + cast(Node->getOperand(Op - 1)) + ->getGlobal())); + Op--; } // Run post-isel target hook to adjust this instruction if needed. @@ -1251,7 +1263,8 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, llvm_unreachable("This target-independent node should have been selected!"); case ISD::EntryToken: case ISD::MERGE_VALUES: - case ISD::TokenFactor: // fall thru + case ISD::TokenFactor: + case ISD::DEACTIVATION_SYMBOL: break; case ISD::CopyToReg: { Register DestReg = cast(Node->getOperand(1))->getReg(); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 1b15a207a2d37..06735708d5369 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -1916,6 +1916,21 @@ SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, return SDValue(N, 0); } +SDValue SelectionDAG::getDeactivationSymbol(const GlobalValue *GV) { + SDVTList VTs = getVTList(MVT::Untyped); + FoldingSetNodeID ID; + AddNodeIDNode(ID, ISD::DEACTIVATION_SYMBOL, VTs, {}); + ID.AddPointer(GV); + void *IP = nullptr; + if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) + return SDValue(E, 0); + + auto *N = newSDNode(GV, VTs); + CSEMap.InsertNode(N, IP); + InsertNode(N); + return SDValue(N, 0); +} + SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; SDVTList VTs = getVTList(VT); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 88b35582a9f7d..53d73ad618bd1 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -45,6 +45,7 @@ #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/SelectionDAG.h" +#include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/SelectionDAGTargetInfo.h" #include "llvm/CodeGen/StackMaps.h" #include "llvm/CodeGen/SwiftErrorValueTracking.h" @@ -5376,6 +5377,14 @@ SmallVector SelectionDAGBuilder::getTargetIntrinsicOperands( } } + if (std::optional Bundle = + I.getOperandBundle(LLVMContext::OB_deactivation_symbol)) { + auto *Sym = Bundle->Inputs[0].get(); + SDValue SDSym = getValue(Sym); + SDSym = DAG.getDeactivationSymbol(cast(Sym)); + Ops.push_back(SDSym); + } + if (std::optional Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) { Value *Token = Bundle->Inputs[0].get(); @@ -9116,6 +9125,11 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee, ConvControlToken = getValue(Token); } + GlobalValue *DeactivationSymbol = nullptr; + if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_deactivation_symbol)) { + DeactivationSymbol = cast(Bundle->Inputs[0].get()); + } + TargetLowering::CallLoweringInfo CLI(DAG); CLI.setDebugLoc(getCurSDLoc()) .setChain(getRoot()) @@ -9125,7 +9139,8 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee, .setIsPreallocated( CB.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0) .setCFIType(CFIType) - .setConvergenceControlToken(ConvControlToken); + .setConvergenceControlToken(ConvControlToken) + .setDeactivationSymbol(DeactivationSymbol); // Set the pointer authentication info if we have it. if (PAI) { @@ -9745,7 +9760,7 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) { {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi, - LLVMContext::OB_convergencectrl}); + LLVMContext::OB_convergencectrl, LLVMContext::OB_deactivation_symbol}); SDValue Callee = getValue(I.getCalledOperand()); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 0fad4722b1871..dd8f18d3b8a6a 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -3308,6 +3308,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, case ISD::LIFETIME_START: case ISD::LIFETIME_END: case ISD::PSEUDO_PROBE: + case ISD::DEACTIVATION_SYMBOL: NodeToMatch->setNodeId(-1); // Mark selected. return; case ISD::AssertSext: @@ -3389,7 +3390,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, // These are the current input chain and glue for use when generating nodes. // Various Emit operations change these. For example, emitting a copytoreg // uses and updates these. - SDValue InputChain, InputGlue; + SDValue InputChain, InputGlue, DeactivationSymbol; // ChainNodesMatched - If a pattern matches nodes that have input/output // chains, the OPC_EmitMergeInputChains operation is emitted which indicates @@ -3542,6 +3543,15 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, InputGlue = N->getOperand(N->getNumOperands()-1); continue; + case OPC_CaptureDeactivationSymbol: + // If the current node has a deactivation symbol, capture it in + // DeactivationSymbol. + if (N->getNumOperands() != 0 && + N->getOperand(N->getNumOperands() - 1).getOpcode() == + ISD::DEACTIVATION_SYMBOL) + DeactivationSymbol = N->getOperand(N->getNumOperands() - 1); + continue; + case OPC_MoveChild: { unsigned ChildNo = MatcherTable[MatcherIndex++]; if (ChildNo >= N.getNumOperands()) @@ -4223,6 +4233,8 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, // If this has chain/glue inputs, add them. if (EmitNodeInfo & OPFL_Chain) Ops.push_back(InputChain); + if (DeactivationSymbol.getNode() != nullptr) + Ops.push_back(DeactivationSymbol); if ((EmitNodeInfo & OPFL_GlueInput) && InputGlue.getNode() != nullptr) Ops.push_back(InputGlue); diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index cd39970f5111f..85d3690dd8306 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -620,7 +620,8 @@ bool CallBase::hasReadingOperandBundles() const { // ptrauth) forces a callsite to be at least readonly. return hasOperandBundlesOtherThan({LLVMContext::OB_ptrauth, LLVMContext::OB_kcfi, - LLVMContext::OB_convergencectrl}) && + LLVMContext::OB_convergencectrl, + LLVMContext::OB_deactivation_symbol}) && getIntrinsicID() != Intrinsic::assume; } @@ -628,7 +629,8 @@ bool CallBase::hasClobberingOperandBundles() const { return hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_ptrauth, LLVMContext::OB_kcfi, - LLVMContext::OB_convergencectrl}) && + LLVMContext::OB_convergencectrl, + LLVMContext::OB_deactivation_symbol}) && getIntrinsicID() != Intrinsic::assume; } diff --git a/llvm/lib/IR/LLVMContext.cpp b/llvm/lib/IR/LLVMContext.cpp index 335c210c10e1a..10aba759185a7 100644 --- a/llvm/lib/IR/LLVMContext.cpp +++ b/llvm/lib/IR/LLVMContext.cpp @@ -55,6 +55,8 @@ static StringRef knownBundleName(unsigned BundleTagID) { return "convergencectrl"; case LLVMContext::OB_align: return "align"; + case LLVMContext::OB_deactivation_symbol: + return "deactivation-symbol"; default: llvm_unreachable("unknown bundle id"); } diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index 3aa77bd47930f..0543cdc2e63d4 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -176,7 +176,12 @@ class AArch64AsmPrinter : public AsmPrinter { const MachineOperand *AUTAddrDisc, Register Scratch, std::optional PACKey, - uint64_t PACDisc, Register PACAddrDisc); + uint64_t PACDisc, Register PACAddrDisc, Value *DS); + + // Emit R_AARCH64_PATCHINST, the deactivation symbol relocation. Returns true + // if no instruction should be emitted because the deactivation symbol is + // defined in the current module so this function emitted a NOP instead. + bool emitDeactivationSymbolRelocation(Value *DS); // Emit the sequence for PAC. void emitPtrauthSign(const MachineInstr *MI); @@ -2113,11 +2118,31 @@ void AArch64AsmPrinter::emitPtrauthTailCallHardening(const MachineInstr *TC) { LRCheckMethod); } +bool AArch64AsmPrinter::emitDeactivationSymbolRelocation(Value *DS) { + if (!DS) + return false; + + if (isa(DS)) { + // Just emit the nop directly. + EmitToStreamer(MCInstBuilder(AArch64::HINT).addImm(0)); + return true; + } + MCSymbol *Dot = OutContext.createTempSymbol(); + OutStreamer->emitLabel(Dot); + const MCExpr *DeactDotExpr = MCSymbolRefExpr::create(Dot, OutContext); + + const MCExpr *DSExpr = MCSymbolRefExpr::create( + OutContext.getOrCreateSymbol(DS->getName()), OutContext); + OutStreamer->emitRelocDirective(*DeactDotExpr, "R_AARCH64_PATCHINST", DSExpr, + SMLoc()); + return false; +} + void AArch64AsmPrinter::emitPtrauthAuthResign( Register AUTVal, AArch64PACKey::ID AUTKey, uint64_t AUTDisc, const MachineOperand *AUTAddrDisc, Register Scratch, std::optional PACKey, uint64_t PACDisc, - Register PACAddrDisc) { + Register PACAddrDisc, Value *DS) { const bool IsAUTPAC = PACKey.has_value(); // We expand AUT/AUTPAC into a sequence of the form @@ -2164,15 +2189,17 @@ void AArch64AsmPrinter::emitPtrauthAuthResign( bool AUTZero = AUTDiscReg == AArch64::XZR; unsigned AUTOpc = getAUTOpcodeForKey(AUTKey, AUTZero); - // autiza x16 ; if AUTZero - // autia x16, x17 ; if !AUTZero - MCInst AUTInst; - AUTInst.setOpcode(AUTOpc); - AUTInst.addOperand(MCOperand::createReg(AUTVal)); - AUTInst.addOperand(MCOperand::createReg(AUTVal)); - if (!AUTZero) - AUTInst.addOperand(MCOperand::createReg(AUTDiscReg)); - EmitToStreamer(*OutStreamer, AUTInst); + if (!emitDeactivationSymbolRelocation(DS)) { + // autiza x16 ; if AUTZero + // autia x16, x17 ; if !AUTZero + MCInst AUTInst; + AUTInst.setOpcode(AUTOpc); + AUTInst.addOperand(MCOperand::createReg(AUTVal)); + AUTInst.addOperand(MCOperand::createReg(AUTVal)); + if (!AUTZero) + AUTInst.addOperand(MCOperand::createReg(AUTDiscReg)); + EmitToStreamer(*OutStreamer, AUTInst); + } // Unchecked or checked-but-non-trapping AUT is just an "AUT": we're done. if (!IsAUTPAC && (!ShouldCheck || !ShouldTrap)) @@ -2236,6 +2263,9 @@ void AArch64AsmPrinter::emitPtrauthSign(const MachineInstr *MI) { bool IsZeroDisc = DiscReg == AArch64::XZR; unsigned Opc = getPACOpcodeForKey(Key, IsZeroDisc); + if (emitDeactivationSymbolRelocation(MI->getDeactivationSymbol())) + return; + // paciza x16 ; if IsZeroDisc // pacia x16, x17 ; if !IsZeroDisc MCInst PACInst; @@ -3136,17 +3166,18 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { } case AArch64::AUTx16x17: - emitPtrauthAuthResign(AArch64::X16, - (AArch64PACKey::ID)MI->getOperand(0).getImm(), - MI->getOperand(1).getImm(), &MI->getOperand(2), - AArch64::X17, std::nullopt, 0, 0); + emitPtrauthAuthResign( + AArch64::X16, (AArch64PACKey::ID)MI->getOperand(0).getImm(), + MI->getOperand(1).getImm(), &MI->getOperand(2), AArch64::X17, + std::nullopt, 0, 0, MI->getDeactivationSymbol()); return; case AArch64::AUTxMxN: emitPtrauthAuthResign(MI->getOperand(0).getReg(), (AArch64PACKey::ID)MI->getOperand(3).getImm(), MI->getOperand(4).getImm(), &MI->getOperand(5), - MI->getOperand(1).getReg(), std::nullopt, 0, 0); + MI->getOperand(1).getReg(), std::nullopt, 0, 0, + MI->getDeactivationSymbol()); return; case AArch64::AUTPAC: @@ -3154,7 +3185,8 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { AArch64::X16, (AArch64PACKey::ID)MI->getOperand(0).getImm(), MI->getOperand(1).getImm(), &MI->getOperand(2), AArch64::X17, (AArch64PACKey::ID)MI->getOperand(3).getImm(), - MI->getOperand(4).getImm(), MI->getOperand(5).getReg()); + MI->getOperand(4).getImm(), MI->getOperand(5).getReg(), + MI->getDeactivationSymbol()); return; case AArch64::PAC: @@ -3635,6 +3667,9 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { return; } + if (emitDeactivationSymbolRelocation(MI->getDeactivationSymbol())) + return; + // Finally, do the automated lowerings for everything else. MCInst TmpInst; MCInstLowering.Lower(MI, TmpInst); diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp index 08466667c0fa5..b721c1f533726 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -1557,7 +1557,10 @@ void AArch64DAGToDAGISel::SelectPtrauthAuth(SDNode *N) { extractPtrauthBlendDiscriminators(AUTDisc, CurDAG); if (!Subtarget->isX16X17Safer()) { - SDValue Ops[] = {Val, AUTKey, AUTConstDisc, AUTAddrDisc}; + std::vector Ops = {Val, AUTKey, AUTConstDisc, AUTAddrDisc}; + // Copy deactivation symbol if present. + if (N->getNumOperands() > 4) + Ops.push_back(N->getOperand(4)); SDNode *AUT = CurDAG->getMachineNode(AArch64::AUTxMxN, DL, MVT::i64, MVT::i64, Ops); diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index d4099b56b6d6e..dd70d729ffc91 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -10203,6 +10203,9 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, if (InGlue.getNode()) Ops.push_back(InGlue); + if (CLI.DeactivationSymbol) + Ops.push_back(DAG.getDeactivationSymbol(CLI.DeactivationSymbol)); + // If we're doing a tall call, use a TC_RETURN here rather than an // actual call instruction. if (IsTailCall) { diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td index 6871c2d504cf6..61a8f764e39ed 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td +++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td @@ -2347,6 +2347,7 @@ class BImm pattern> let Inst{25-0} = addr; let DecoderMethod = "DecodeUnconditionalBranch"; + let supportsDeactivationSymbol = true; } class BranchImm pattern> @@ -2404,6 +2405,7 @@ class SignAuthOneData opcode_prefix, bits<2> opcode, string asm, let Inst{11-10} = opcode; let Inst{9-5} = Rn; let Inst{4-0} = Rd; + let supportsDeactivationSymbol = true; } class SignAuthZero opcode_prefix, bits<2> opcode, string asm, @@ -2417,6 +2419,7 @@ class SignAuthZero opcode_prefix, bits<2> opcode, string asm, let Inst{11-10} = opcode; let Inst{9-5} = 0b11111; let Inst{4-0} = Rd; + let supportsDeactivationSymbol = true; } class SignAuthTwoOperand opc, string asm, diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index 03bad8ff8ac8a..b4d8649b31d6d 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -2215,6 +2215,7 @@ let Predicates = [HasPAuth] in { let Size = 12; let Defs = [X16, X17]; let usesCustomInserter = 1; + let supportsDeactivationSymbol = true; } // A standalone pattern is used, so that literal 0 can be passed as $Disc. diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp index 55694efafeed1..7907a3c283624 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp @@ -1421,6 +1421,7 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, } else if (Info.CFIType) { MIB->setCFIType(MF, Info.CFIType->getZExtValue()); } + MIB->setDeactivationSymbol(MF, Info.DeactivationSymbol); MIB.add(Info.Callee); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index 8e4edefec42fd..d903787f00c7f 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -3077,6 +3077,11 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { } case Intrinsic::ptrauth_auth: case Intrinsic::ptrauth_resign: { + // We don't support this optimization on intrinsic calls with deactivation + // symbols, which are represented using operand bundles. + if (II->hasOperandBundles()) + break; + // (sign|resign) + (auth|resign) can be folded by omitting the middle // sign+auth component if the key and discriminator match. bool NeedSign = II->getIntrinsicID() == Intrinsic::ptrauth_resign; @@ -3088,6 +3093,11 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { // whatever we replace this sequence with. Value *AuthKey = nullptr, *AuthDisc = nullptr, *BasePtr; if (const auto *CI = dyn_cast(Ptr)) { + // We don't support this optimization on intrinsic calls with deactivation + // symbols, which are represented using operand bundles. + if (CI->hasOperandBundles()) + break; + BasePtr = CI->getArgOperand(0); if (CI->getIntrinsicID() == Intrinsic::ptrauth_sign) { if (CI->getArgOperand(1) != Key || CI->getArgOperand(2) != Disc) diff --git a/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll b/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll index 5628e17b4936e..01e5b3f6673ae 100644 --- a/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll +++ b/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll @@ -14,6 +14,7 @@ ; CHECK-NEXT: (N); + bool SupportsDeactivationSymbol = + EN->getInstruction().TheDef->getValueAsBit( + "supportsDeactivationSymbol"); + if (SupportsDeactivationSymbol) { + OS << "OPC_CaptureDeactivationSymbol,\n"; + OS.indent(FullIndexWidth + Indent); + } bool IsEmitNode = isa(EN); OS << (IsEmitNode ? "OPC_EmitNode" : "OPC_MorphNodeTo"); bool CompressVTs = EN->getNumVTs() < 3; @@ -1052,8 +1059,8 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N, OS << '\n'; } - return 4 + !CompressVTs + !CompressNodeInfo + NumTypeBytes + - NumOperandBytes + NumCoveredBytes; + return 4 + SupportsDeactivationSymbol + !CompressVTs + !CompressNodeInfo + + NumTypeBytes + NumOperandBytes + NumCoveredBytes; } case Matcher::CompleteMatch: { const CompleteMatchMatcher *CM = cast(N); From d2379effe9db15765e4fd1f7a0589af5f9269f96 Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Wed, 26 Nov 2025 12:39:40 -0800 Subject: [PATCH 02/22] Add deactivation symbol operand to ConstantPtrAuth. Deactivation symbol operands are supported in the code generator by building on the previously added support for IRELATIVE relocations. Reviewers: ojhunt, fmayer, ahmedbougacha, nikic, efriedma-quic Reviewed By: fmayer Pull Request: https://github.com/llvm/llvm-project/pull/133537 --- clang/lib/CodeGen/CGPointerAuth.cpp | 7 ++- llvm/docs/LangRef.rst | 9 ++- llvm/include/llvm/Bitcode/LLVMBitCodes.h | 2 + llvm/include/llvm/IR/Constants.h | 13 ++-- llvm/include/llvm/SandboxIR/Constant.h | 5 +- llvm/lib/AsmParser/LLParser.cpp | 29 ++++++--- llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 21 ++++++- llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 3 +- llvm/lib/IR/AsmWriter.cpp | 4 +- llvm/lib/IR/Constants.cpp | 18 ++++-- llvm/lib/IR/ConstantsContext.h | 3 +- llvm/lib/IR/Core.cpp | 4 +- llvm/lib/IR/Verifier.cpp | 8 +++ llvm/lib/SandboxIR/Constant.cpp | 11 +++- llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 62 +++++++++++++++---- .../InstCombine/InstCombineCalls.cpp | 5 +- llvm/lib/Transforms/Utils/ValueMapper.cpp | 5 +- llvm/test/Assembler/invalid-ptrauth-const6.ll | 6 ++ llvm/test/Bitcode/compatibility.ll | 4 ++ .../test/CodeGen/AArch64/ptrauth-irelative.ll | 17 +++++ .../InstCombine/ptrauth-intrinsics.ll | 9 +++ llvm/test/Verifier/ptrauth-constant.ll | 6 ++ llvm/unittests/SandboxIR/SandboxIRTest.cpp | 2 +- .../Transforms/Utils/ValueMapperTest.cpp | 13 ++-- 24 files changed, 218 insertions(+), 48 deletions(-) create mode 100644 llvm/test/Assembler/invalid-ptrauth-const6.ll create mode 100644 llvm/test/Verifier/ptrauth-constant.ll diff --git a/clang/lib/CodeGen/CGPointerAuth.cpp b/clang/lib/CodeGen/CGPointerAuth.cpp index dbb7bc99ac638..a49a0c91681fe 100644 --- a/clang/lib/CodeGen/CGPointerAuth.cpp +++ b/clang/lib/CodeGen/CGPointerAuth.cpp @@ -440,9 +440,10 @@ CodeGenModule::getConstantSignedPointer(llvm::Constant *Pointer, unsigned Key, IntegerDiscriminator = llvm::ConstantInt::get(Int64Ty, 0); } - return llvm::ConstantPtrAuth::get(Pointer, - llvm::ConstantInt::get(Int32Ty, Key), - IntegerDiscriminator, AddressDiscriminator); + return llvm::ConstantPtrAuth::get( + Pointer, llvm::ConstantInt::get(Int32Ty, Key), IntegerDiscriminator, + AddressDiscriminator, + /*DeactivationSymbol=*/llvm::Constant::getNullValue(DefaultPtrTy)); } /// Does a given PointerAuthScheme require us to sign a value diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index 7cd7e815fd8d7..02865f8a29c67 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -3234,6 +3234,8 @@ A "convergencectrl" operand bundle is only valid on a ``convergent`` operation. When present, the operand bundle must contain exactly one value of token type. See the :doc:`ConvergentOperations` document for details. +.. _deactivationsymbol: + Deactivation Symbol Operand Bundles ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -5300,7 +5302,7 @@ need to refer to the actual function body. Pointer Authentication Constants -------------------------------- -``ptrauth (ptr CST, i32 KEY[, i64 DISC[, ptr ADDRDISC]?]?)`` +``ptrauth (ptr CST, i32 KEY[, i64 DISC[, ptr ADDRDISC[, ptr DS]?]?]?)`` A '``ptrauth``' constant represents a pointer with a cryptographic authentication signature embedded into some bits, as described in the @@ -5329,6 +5331,11 @@ Otherwise, the expression is equivalent to: %tmp2 = call i64 @llvm.ptrauth.sign(i64 ptrtoint (ptr CST to i64), i32 KEY, i64 %tmp1) %val = inttoptr i64 %tmp2 to ptr +If the deactivation symbol operand ``DS`` has a non-null value, +the semantics are as if a :ref:`deactivation-symbol operand bundle +` were added to the ``llvm.ptrauth.sign`` intrinsic +calls above, with ``DS`` as the only operand. + .. _constantexprs: Constant Expressions diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h index 991aa49d787f9..2451d588bdbf7 100644 --- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -437,6 +437,8 @@ enum ConstantsCodes { CST_CODE_CE_GEP_WITH_INRANGE = 31, // [opty, flags, range, n x operands] CST_CODE_CE_GEP = 32, // [opty, flags, n x operands] CST_CODE_PTRAUTH = 33, // [ptr, key, disc, addrdisc] + CST_CODE_PTRAUTH2 = 34, // [ptr, key, disc, addrdisc, + // deactivation_symbol] }; /// CastOpcodes - These are values used in the bitcode files to encode which diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h index e06e6adbc3130..e3f2eb9fa44b8 100644 --- a/llvm/include/llvm/IR/Constants.h +++ b/llvm/include/llvm/IR/Constants.h @@ -1033,10 +1033,10 @@ class ConstantPtrAuth final : public Constant { friend struct ConstantPtrAuthKeyType; friend class Constant; - constexpr static IntrusiveOperandsAllocMarker AllocMarker{4}; + constexpr static IntrusiveOperandsAllocMarker AllocMarker{5}; ConstantPtrAuth(Constant *Ptr, ConstantInt *Key, ConstantInt *Disc, - Constant *AddrDisc); + Constant *AddrDisc, Constant *DeactivationSymbol); void *operator new(size_t s) { return User::operator new(s, AllocMarker); } @@ -1046,7 +1046,8 @@ class ConstantPtrAuth final : public Constant { public: /// Return a pointer signed with the specified parameters. LLVM_ABI static ConstantPtrAuth *get(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc); + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol); /// Produce a new ptrauth expression signing the given value using /// the same schema as is stored in one. @@ -1078,6 +1079,10 @@ class ConstantPtrAuth final : public Constant { return !getAddrDiscriminator()->isNullValue(); } + Constant *getDeactivationSymbol() const { + return cast(Op<4>().get()); + } + /// A constant value for the address discriminator which has special /// significance to ctors/dtors lowering. Regular address discrimination can't /// be applied for them since uses of llvm.global_{c|d}tors are disallowed @@ -1106,7 +1111,7 @@ class ConstantPtrAuth final : public Constant { template <> struct OperandTraits - : public FixedNumOperandTraits {}; + : public FixedNumOperandTraits {}; DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantPtrAuth, Constant) diff --git a/llvm/include/llvm/SandboxIR/Constant.h b/llvm/include/llvm/SandboxIR/Constant.h index 6f682a7059d10..2fe923f6c3866 100644 --- a/llvm/include/llvm/SandboxIR/Constant.h +++ b/llvm/include/llvm/SandboxIR/Constant.h @@ -1363,7 +1363,8 @@ class ConstantPtrAuth final : public Constant { public: /// Return a pointer signed with the specified parameters. LLVM_ABI static ConstantPtrAuth *get(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc); + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol); /// The pointer that is signed in this ptrauth signed pointer. LLVM_ABI Constant *getPointer() const; @@ -1378,6 +1379,8 @@ class ConstantPtrAuth final : public Constant { /// the only global-initializer user of the ptrauth signed pointer. LLVM_ABI Constant *getAddrDiscriminator() const; + Constant *getDeactivationSymbol() const; + /// Whether there is any non-null address discriminator. bool hasAddressDiscriminator() const { return cast(Val)->hasAddressDiscriminator(); diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp index 61d5c2c81df2e..c3678d37607d5 100644 --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -4250,11 +4250,13 @@ bool LLParser::parseValID(ValID &ID, PerFunctionState *PFS, Type *ExpectedTy) { } case lltok::kw_ptrauth: { // ValID ::= 'ptrauth' '(' ptr @foo ',' i32 - // (',' i64 (',' ptr addrdisc)? )? ')' + // (',' i64 (',' ptr addrdisc (',' ptr ds)? + // )? )? ')' Lex.Lex(); Constant *Ptr, *Key; - Constant *Disc = nullptr, *AddrDisc = nullptr; + Constant *Disc = nullptr, *AddrDisc = nullptr, + *DeactivationSymbol = nullptr; if (parseToken(lltok::lparen, "expected '(' in constant ptrauth expression") || @@ -4263,11 +4265,14 @@ bool LLParser::parseValID(ValID &ID, PerFunctionState *PFS, Type *ExpectedTy) { "expected comma in constant ptrauth expression") || parseGlobalTypeAndValue(Key)) return true; - // If present, parse the optional disc/addrdisc. - if (EatIfPresent(lltok::comma)) - if (parseGlobalTypeAndValue(Disc) || - (EatIfPresent(lltok::comma) && parseGlobalTypeAndValue(AddrDisc))) - return true; + // If present, parse the optional disc/addrdisc/ds. + if (EatIfPresent(lltok::comma) && parseGlobalTypeAndValue(Disc)) + return true; + if (EatIfPresent(lltok::comma) && parseGlobalTypeAndValue(AddrDisc)) + return true; + if (EatIfPresent(lltok::comma) && + parseGlobalTypeAndValue(DeactivationSymbol)) + return true; if (parseToken(lltok::rparen, "expected ')' in constant ptrauth expression")) return true; @@ -4298,7 +4303,15 @@ bool LLParser::parseValID(ValID &ID, PerFunctionState *PFS, Type *ExpectedTy) { AddrDisc = ConstantPointerNull::get(PointerType::get(Context, 0)); } - ID.ConstantVal = ConstantPtrAuth::get(Ptr, KeyC, DiscC, AddrDisc); + if (!DeactivationSymbol) + DeactivationSymbol = + ConstantPointerNull::get(PointerType::get(Context, 0)); + if (!DeactivationSymbol->getType()->isPointerTy()) + return error(ID.Loc, + "constant ptrauth deactivation symbol must be a pointer"); + + ID.ConstantVal = + ConstantPtrAuth::get(Ptr, KeyC, DiscC, AddrDisc, DeactivationSymbol); ID.Kind = ValID::t_Constant; return false; } diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 9f3bb230440fb..04cb0a699ebbf 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -1609,7 +1609,16 @@ Expected BitcodeReader::materializeValue(unsigned StartValID, if (!Disc) return error("ptrauth disc operand must be ConstantInt"); - C = ConstantPtrAuth::get(ConstOps[0], Key, Disc, ConstOps[3]); + Constant *DeactivationSymbol = + ConstOps.size() > 4 ? ConstOps[4] + : ConstantPointerNull::get(cast( + ConstOps[3]->getType())); + if (!DeactivationSymbol->getType()->isPointerTy()) + return error( + "ptrauth deactivation symbol operand must be a pointer"); + + C = ConstantPtrAuth::get(ConstOps[0], Key, Disc, ConstOps[3], + DeactivationSymbol); break; } case BitcodeConstant::NoCFIOpcode: { @@ -3813,6 +3822,16 @@ Error BitcodeReader::parseConstants() { (unsigned)Record[2], (unsigned)Record[3]}); break; } + case bitc::CST_CODE_PTRAUTH2: { + if (Record.size() < 5) + return error("Invalid ptrauth record"); + // Ptr, Key, Disc, AddrDisc, DeactivationSymbol + V = BitcodeConstant::create( + Alloc, CurTy, BitcodeConstant::ConstantPtrAuthOpcode, + {(unsigned)Record[0], (unsigned)Record[1], (unsigned)Record[2], + (unsigned)Record[3], (unsigned)Record[4]}); + break; + } } assert(V->getType() == getTypeByID(CurTyID) && "Incorrect result type ID"); diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index 1d0461478b90c..0dd3fa3361fee 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -3030,11 +3030,12 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal, Record.push_back(VE.getTypeID(NC->getGlobalValue()->getType())); Record.push_back(VE.getValueID(NC->getGlobalValue())); } else if (const auto *CPA = dyn_cast(C)) { - Code = bitc::CST_CODE_PTRAUTH; + Code = bitc::CST_CODE_PTRAUTH2; Record.push_back(VE.getValueID(CPA->getPointer())); Record.push_back(VE.getValueID(CPA->getKey())); Record.push_back(VE.getValueID(CPA->getDiscriminator())); Record.push_back(VE.getValueID(CPA->getAddrDiscriminator())); + Record.push_back(VE.getValueID(CPA->getDeactivationSymbol())); } else { #ifndef NDEBUG C->dump(); diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index eebabfd772982..7932765db8359 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -1674,12 +1674,14 @@ static void writeConstantInternal(raw_ostream &Out, const Constant *CV, if (const auto *CPA = dyn_cast(CV)) { Out << "ptrauth ("; - // ptrauth (ptr CST, i32 KEY[, i64 DISC[, ptr ADDRDISC]?]?) + // ptrauth (ptr CST, i32 KEY[, i64 DISC[, ptr ADDRDISC[, ptr DS]?]?]?) unsigned NumOpsToWrite = 2; if (!CPA->getOperand(2)->isNullValue()) NumOpsToWrite = 3; if (!CPA->getOperand(3)->isNullValue()) NumOpsToWrite = 4; + if (!CPA->getOperand(4)->isNullValue()) + NumOpsToWrite = 5; ListSeparator LS; for (unsigned i = 0, e = NumOpsToWrite; i != e; ++i) { diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp index a3aa5e9571657..6b82da140256f 100644 --- a/llvm/lib/IR/Constants.cpp +++ b/llvm/lib/IR/Constants.cpp @@ -2081,28 +2081,33 @@ Value *NoCFIValue::handleOperandChangeImpl(Value *From, Value *To) { // ConstantPtrAuth *ConstantPtrAuth::get(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc) { - Constant *ArgVec[] = {Ptr, Key, Disc, AddrDisc}; + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol) { + Constant *ArgVec[] = {Ptr, Key, Disc, AddrDisc, DeactivationSymbol}; ConstantPtrAuthKeyType MapKey(ArgVec); LLVMContextImpl *pImpl = Ptr->getContext().pImpl; return pImpl->ConstantPtrAuths.getOrCreate(Ptr->getType(), MapKey); } ConstantPtrAuth *ConstantPtrAuth::getWithSameSchema(Constant *Pointer) const { - return get(Pointer, getKey(), getDiscriminator(), getAddrDiscriminator()); + return get(Pointer, getKey(), getDiscriminator(), getAddrDiscriminator(), + getDeactivationSymbol()); } ConstantPtrAuth::ConstantPtrAuth(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc) + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol) : Constant(Ptr->getType(), Value::ConstantPtrAuthVal, AllocMarker) { assert(Ptr->getType()->isPointerTy()); assert(Key->getBitWidth() == 32); assert(Disc->getBitWidth() == 64); assert(AddrDisc->getType()->isPointerTy()); + assert(DeactivationSymbol->getType()->isPointerTy()); setOperand(0, Ptr); setOperand(1, Key); setOperand(2, Disc); setOperand(3, AddrDisc); + setOperand(4, DeactivationSymbol); } /// Remove the constant from the constant table. @@ -2150,6 +2155,11 @@ bool ConstantPtrAuth::hasSpecialAddressDiscriminator(uint64_t Value) const { bool ConstantPtrAuth::isKnownCompatibleWith(const Value *Key, const Value *Discriminator, const DataLayout &DL) const { + // This function may only be validly called to analyze a ptrauth operation + // with no deactivation symbol, so if we have one it isn't compatible. + if (!getDeactivationSymbol()->isNullValue()) + return false; + // If the keys are different, there's no chance for this to be compatible. if (getKey() != Key) return false; diff --git a/llvm/lib/IR/ConstantsContext.h b/llvm/lib/IR/ConstantsContext.h index e3e8d895a63f4..2073e0d42d8e3 100644 --- a/llvm/lib/IR/ConstantsContext.h +++ b/llvm/lib/IR/ConstantsContext.h @@ -539,7 +539,8 @@ struct ConstantPtrAuthKeyType { ConstantPtrAuth *create(TypeClass *Ty) const { return new ConstantPtrAuth(Operands[0], cast(Operands[1]), - cast(Operands[2]), Operands[3]); + cast(Operands[2]), Operands[3], + Operands[4]); } }; diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp index 604730e0d3004..26c4f4ec784cd 100644 --- a/llvm/lib/IR/Core.cpp +++ b/llvm/lib/IR/Core.cpp @@ -1699,7 +1699,9 @@ LLVMValueRef LLVMConstantPtrAuth(LLVMValueRef Ptr, LLVMValueRef Key, LLVMValueRef Disc, LLVMValueRef AddrDisc) { return wrap(ConstantPtrAuth::get( unwrap(Ptr), unwrap(Key), - unwrap(Disc), unwrap(AddrDisc))); + unwrap(Disc), unwrap(AddrDisc), + ConstantPointerNull::get( + cast(unwrap(AddrDisc)->getType())))); } /*-- Opcode mapping */ diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 7cc1980d24c33..a1e14d8f25bf7 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -2732,6 +2732,14 @@ void Verifier::visitConstantPtrAuth(const ConstantPtrAuth *CPA) { Check(CPA->getDiscriminator()->getBitWidth() == 64, "signed ptrauth constant discriminator must be i64 constant integer"); + + Check(CPA->getDeactivationSymbol()->getType()->isPointerTy(), + "signed ptrauth constant deactivation symbol must be a pointer"); + + Check(isa(CPA->getDeactivationSymbol()) || + CPA->getDeactivationSymbol()->isNullValue(), + "signed ptrauth constant deactivation symbol must be a global value " + "or null"); } bool Verifier::verifyAttributeCount(AttributeList Attrs, unsigned Params) { diff --git a/llvm/lib/SandboxIR/Constant.cpp b/llvm/lib/SandboxIR/Constant.cpp index 9de88ef2cf0a0..eb14797af081c 100644 --- a/llvm/lib/SandboxIR/Constant.cpp +++ b/llvm/lib/SandboxIR/Constant.cpp @@ -412,10 +412,12 @@ PointerType *NoCFIValue::getType() const { } ConstantPtrAuth *ConstantPtrAuth::get(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc) { + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol) { auto *LLVMC = llvm::ConstantPtrAuth::get( cast(Ptr->Val), cast(Key->Val), - cast(Disc->Val), cast(AddrDisc->Val)); + cast(Disc->Val), cast(AddrDisc->Val), + cast(DeactivationSymbol->Val)); return cast(Ptr->getContext().getOrCreateConstant(LLVMC)); } @@ -439,6 +441,11 @@ Constant *ConstantPtrAuth::getAddrDiscriminator() const { cast(Val)->getAddrDiscriminator()); } +Constant *ConstantPtrAuth::getDeactivationSymbol() const { + return Ctx.getOrCreateConstant( + cast(Val)->getDeactivationSymbol()); +} + ConstantPtrAuth *ConstantPtrAuth::getWithSameSchema(Constant *Pointer) const { auto *LLVMC = cast(Val)->getWithSameSchema( cast(Pointer->Val)); diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index 0543cdc2e63d4..721ea22c9eae4 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -219,11 +219,9 @@ class AArch64AsmPrinter : public AsmPrinter { // authenticating) void LowerLOADgotAUTH(const MachineInstr &MI); - const MCExpr *emitPAuthRelocationAsIRelative(const MCExpr *Target, - uint16_t Disc, - AArch64PACKey::ID KeyID, - bool HasAddressDiversity, - bool IsDSOLocal); + const MCExpr *emitPAuthRelocationAsIRelative( + const MCExpr *Target, uint16_t Disc, AArch64PACKey::ID KeyID, + bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr); /// tblgen'erated driver function for lowering simple MI->MC /// pseudo instructions. @@ -2386,15 +2384,17 @@ static void emitAddress(MCStreamer &Streamer, MCRegister Reg, } static bool targetSupportsPAuthRelocation(const Triple &TT, - const MCExpr *Target) { + const MCExpr *Target, + const MCExpr *DSExpr) { // No released version of glibc supports PAuth relocations. if (TT.isOSGlibc()) return false; // We emit PAuth constants as IRELATIVE relocations in cases where the // constant cannot be represented as a PAuth relocation: - // 1) The signed value is not a symbol. - return !isa(Target); + // 1) There is a deactivation symbol. + // 2) The signed value is not a symbol. + return !DSExpr && !isa(Target); } static bool targetSupportsIRelativeRelocation(const Triple &TT) { @@ -2447,14 +2447,27 @@ static bool targetSupportsIRelativeRelocation(const Triple &TT) { // mov x1, #12345 // b __emupac_pacda // .popsection +// +// Example (signed null pointer, not address discriminated, with deactivation +// symbol ds): +// +// .8byte .Lpauth_ifunc0 +// .pushsection .text.startup,"ax",@progbits +// .Lpauth_ifunc0: +// mov x0, #0 +// mov x1, #12345 +// .reloc ., R_AARCH64_PATCHINST, ds +// b __emupac_pacda +// ret +// .popsection const MCExpr *AArch64AsmPrinter::emitPAuthRelocationAsIRelative( const MCExpr *Target, uint16_t Disc, AArch64PACKey::ID KeyID, - bool HasAddressDiversity, bool IsDSOLocal) { + bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr) { const Triple &TT = TM.getTargetTriple(); // We only emit an IRELATIVE relocation if the target supports IRELATIVE and // does not support the kind of PAuth relocation that we are trying to emit. - if (targetSupportsPAuthRelocation(TT, Target) || + if (targetSupportsPAuthRelocation(TT, Target, DSExpr) || !targetSupportsIRelativeRelocation(TT)) return nullptr; @@ -2498,6 +2511,16 @@ const MCExpr *AArch64AsmPrinter::emitPAuthRelocationAsIRelative( emitMOVZ(AArch64::X1, Disc, 0); } + if (DSExpr) { + MCSymbol *PrePACInst = OutStreamer->getContext().createTempSymbol(); + OutStreamer->emitLabel(PrePACInst); + + auto *PrePACInstExpr = + MCSymbolRefExpr::create(PrePACInst, OutStreamer->getContext()); + OutStreamer->emitRelocDirective(*PrePACInstExpr, "R_AARCH64_PATCHINST", + DSExpr, SMLoc()); + } + // We don't know the subtarget because this is being emitted for a global // initializer. Because the performance of IFUNC resolvers is unimportant, we // always call the EmuPAC runtime, which will end up using the PAC instruction @@ -2508,6 +2531,12 @@ const MCExpr *AArch64AsmPrinter::emitPAuthRelocationAsIRelative( MCSymbolRefExpr::create(EmuPAC, OutStreamer->getContext()); OutStreamer->emitInstruction(MCInstBuilder(AArch64::B).addExpr(EmuPACRef), *STI); + + // We need a RET despite the above tail call because the deactivation symbol + // may replace the tail call with a NOP. + if (DSExpr) + OutStreamer->emitInstruction( + MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI); OutStreamer->popSection(); return MCSymbolRefExpr::create(IRelativeSym, AArch64::S_FUNCINIT, @@ -2539,6 +2568,13 @@ AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) { Sym = MCConstantExpr::create(Offset.getSExtValue(), Ctx); } + const MCExpr *DSExpr = nullptr; + if (auto *DS = dyn_cast(CPA.getDeactivationSymbol())) { + if (isa(DS)) + return Sym; + DSExpr = MCSymbolRefExpr::create(getSymbol(DS), Ctx); + } + uint64_t KeyID = CPA.getKey()->getZExtValue(); // We later rely on valid KeyID value in AArch64PACKeyIDToString call from // AArch64AuthMCExpr::printImpl, so fail fast. @@ -2559,9 +2595,13 @@ AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) { // Check if we need to represent this with an IRELATIVE and emit it if so. if (auto *IFuncSym = emitPAuthRelocationAsIRelative( Sym, Disc, AArch64PACKey::ID(KeyID), CPA.hasAddressDiscriminator(), - BaseGVB && BaseGVB->isDSOLocal())) + BaseGVB && BaseGVB->isDSOLocal(), DSExpr)) return IFuncSym; + if (DSExpr) + report_fatal_error("deactivation symbols unsupported in constant " + "expressions on this target"); + // Finally build the complete @AUTH expr. return AArch64AuthMCExpr::create(Sym, Disc, AArch64PACKey::ID(KeyID), CPA.hasAddressDiscriminator(), Ctx); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index d903787f00c7f..9543d97616ae3 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -3120,9 +3120,10 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { if (NeedSign && isa(II->getArgOperand(4))) { auto *SignKey = cast(II->getArgOperand(3)); auto *SignDisc = cast(II->getArgOperand(4)); - auto *SignAddrDisc = ConstantPointerNull::get(Builder.getPtrTy()); + auto *Null = ConstantPointerNull::get(Builder.getPtrTy()); auto *NewCPA = ConstantPtrAuth::get(CPA->getPointer(), SignKey, - SignDisc, SignAddrDisc); + SignDisc, /*AddrDisc=*/Null, + /*DeactivationSymbol=*/Null); replaceInstUsesWith( *II, ConstantExpr::getPointerCast(NewCPA, II->getType())); return eraseInstFromFunction(*II); diff --git a/llvm/lib/Transforms/Utils/ValueMapper.cpp b/llvm/lib/Transforms/Utils/ValueMapper.cpp index 9021d8b289baf..6e36006890df4 100644 --- a/llvm/lib/Transforms/Utils/ValueMapper.cpp +++ b/llvm/lib/Transforms/Utils/ValueMapper.cpp @@ -526,8 +526,9 @@ Value *Mapper::mapValue(const Value *V) { if (isa(C)) return getVM()[V] = ConstantVector::get(Ops); if (isa(C)) - return getVM()[V] = ConstantPtrAuth::get(Ops[0], cast(Ops[1]), - cast(Ops[2]), Ops[3]); + return getVM()[V] = + ConstantPtrAuth::get(Ops[0], cast(Ops[1]), + cast(Ops[2]), Ops[3], Ops[4]); // If this is a no-operand constant, it must be because the type was remapped. if (isa(C)) return getVM()[V] = PoisonValue::get(NewTy); diff --git a/llvm/test/Assembler/invalid-ptrauth-const6.ll b/llvm/test/Assembler/invalid-ptrauth-const6.ll new file mode 100644 index 0000000000000..6e8e1d386acc8 --- /dev/null +++ b/llvm/test/Assembler/invalid-ptrauth-const6.ll @@ -0,0 +1,6 @@ +; RUN: not llvm-as < %s 2>&1 | FileCheck %s + +@var = global i32 0 + +; CHECK: error: constant ptrauth deactivation symbol must be a pointer +@ptr = global ptr ptrauth (ptr @var, i32 0, i64 65535, ptr null, i64 0) diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll index e21786e5ee330..53cbe2d6ffd37 100644 --- a/llvm/test/Bitcode/compatibility.ll +++ b/llvm/test/Bitcode/compatibility.ll @@ -217,9 +217,13 @@ declare void @g.f1() ; CHECK: @g.sanitize_address_dyninit = global i32 0, sanitize_address_dyninit ; CHECK: @g.sanitize_multiple = global i32 0, sanitize_memtag, sanitize_address_dyninit +@ds = external global i32 + ; ptrauth constant @auth_var = global ptr ptrauth (ptr @g1, i32 0, i64 65535, ptr null) ; CHECK: @auth_var = global ptr ptrauth (ptr @g1, i32 0, i64 65535) +@auth_var.ds = global ptr ptrauth (ptr @g1, i32 0, i64 65535, ptr null, ptr @ds) +; CHECK: @auth_var.ds = global ptr ptrauth (ptr @g1, i32 0, i64 65535, ptr null, ptr @ds) ;; Aliases ; Format: @ = [Linkage] [Visibility] [DLLStorageClass] [ThreadLocal] diff --git a/llvm/test/CodeGen/AArch64/ptrauth-irelative.ll b/llvm/test/CodeGen/AArch64/ptrauth-irelative.ll index 7857051668dfb..4ee1c19a86490 100644 --- a/llvm/test/CodeGen/AArch64/ptrauth-irelative.ll +++ b/llvm/test/CodeGen/AArch64/ptrauth-irelative.ll @@ -25,6 +25,23 @@ ; CHECK-NEXT: .xword [[FUNC]]@FUNCINIT @dsolocalref = constant ptr ptrauth (ptr @dsolocal, i32 2, i64 2, ptr null), align 8 +@ds = external global i8 + +; CHECK: dsolocalrefds: +; CHECK-NEXT: [[PLACE:.*]]: +; CHECK-NEXT: .section .text.startup +; CHECK-NEXT: [[FUNC:.*]]: +; CHECK-NEXT: adrp x0, dsolocal +; CHECK-NEXT: add x0, x0, :lo12:dsolocal +; CHECK-NEXT: mov x1, #2 +; CHECK-NEXT: [[LABEL:.L.*]]: +; CHECK-NEXT: .reloc [[LABEL]], R_AARCH64_PATCHINST, ds +; CHECK-NEXT: b __emupac_pacda +; CHECK-NEXT: ret +; CHECK-NEXT: .section .rodata +; CHECK-NEXT: .xword [[FUNC]]@FUNCINIT +@dsolocalrefds = constant ptr ptrauth (ptr @dsolocal, i32 2, i64 2, ptr null, ptr @ds), align 8 + ; CHECK: dsolocalref8: ; CHECK-NEXT: [[PLACE:.*]]: ; CHECK-NEXT: .section .text.startup diff --git a/llvm/test/Transforms/InstCombine/ptrauth-intrinsics.ll b/llvm/test/Transforms/InstCombine/ptrauth-intrinsics.ll index 09d9649b09cc1..22c330fe7ae61 100644 --- a/llvm/test/Transforms/InstCombine/ptrauth-intrinsics.ll +++ b/llvm/test/Transforms/InstCombine/ptrauth-intrinsics.ll @@ -188,6 +188,15 @@ define i64 @test_ptrauth_nop_ds2(ptr %p) { ret i64 %authed } +define i64 @test_ptrauth_nop_ds_constant() { +; CHECK-LABEL: @test_ptrauth_nop_ds_constant( +; CHECK-NEXT: [[AUTHED:%.*]] = call i64 @llvm.ptrauth.auth(i64 ptrtoint (ptr ptrauth (ptr @foo, i32 1, i64 1234, ptr null, ptr @ds) to i64), i32 1, i64 1234) +; CHECK-NEXT: ret i64 [[AUTHED]] +; + %authed = call i64 @llvm.ptrauth.auth(i64 ptrtoint(ptr ptrauth(ptr @foo, i32 1, i64 1234, ptr null, ptr @ds) to i64), i32 1, i64 1234) + ret i64 %authed +} + declare i64 @llvm.ptrauth.auth(i64, i32, i64) declare i64 @llvm.ptrauth.sign(i64, i32, i64) declare i64 @llvm.ptrauth.resign(i64, i32, i64, i32, i64) diff --git a/llvm/test/Verifier/ptrauth-constant.ll b/llvm/test/Verifier/ptrauth-constant.ll new file mode 100644 index 0000000000000..7a6d9d2634bc8 --- /dev/null +++ b/llvm/test/Verifier/ptrauth-constant.ll @@ -0,0 +1,6 @@ +; RUN: not opt -passes=verify < %s 2>&1 | FileCheck %s + +@g = external global i8 + +; CHECK: signed ptrauth constant deactivation symbol must be a global value or null +@ptr = global ptr ptrauth (ptr @g, i32 0, i64 65535, ptr null, ptr inttoptr (i64 16 to ptr)) diff --git a/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/llvm/unittests/SandboxIR/SandboxIRTest.cpp index 33928ac118e0c..168502f89cbf8 100644 --- a/llvm/unittests/SandboxIR/SandboxIRTest.cpp +++ b/llvm/unittests/SandboxIR/SandboxIRTest.cpp @@ -1385,7 +1385,7 @@ define ptr @foo() { // Check get(), getKey(), getDiscriminator(), getAddrDiscriminator(). auto *NewPtrAuth = sandboxir::ConstantPtrAuth::get( &F, PtrAuth->getKey(), PtrAuth->getDiscriminator(), - PtrAuth->getAddrDiscriminator()); + PtrAuth->getAddrDiscriminator(), PtrAuth->getDeactivationSymbol()); EXPECT_EQ(NewPtrAuth, PtrAuth); // Check hasAddressDiscriminator(). EXPECT_EQ(PtrAuth->hasAddressDiscriminator(), diff --git a/llvm/unittests/Transforms/Utils/ValueMapperTest.cpp b/llvm/unittests/Transforms/Utils/ValueMapperTest.cpp index 7f12deae2ad1b..60e9c5688c795 100644 --- a/llvm/unittests/Transforms/Utils/ValueMapperTest.cpp +++ b/llvm/unittests/Transforms/Utils/ValueMapperTest.cpp @@ -451,6 +451,10 @@ TEST(ValueMapperTest, mapValuePtrAuth) { PtrTy, false, GlobalValue::ExternalLinkage, nullptr, "Storage0"); std::unique_ptr Storage1 = std::make_unique( PtrTy, false, GlobalValue::ExternalLinkage, nullptr, "Storage1"); + std::unique_ptr DS0 = std::make_unique( + PtrTy, false, GlobalValue::ExternalLinkage, nullptr, "DS0"); + std::unique_ptr DS1 = std::make_unique( + PtrTy, false, GlobalValue::ExternalLinkage, nullptr, "DS1"); ConstantInt *ConstKey = ConstantInt::get(Int32Ty, 1); ConstantInt *ConstDisc = ConstantInt::get(Int64Ty, 1234); @@ -458,11 +462,12 @@ TEST(ValueMapperTest, mapValuePtrAuth) { ValueToValueMapTy VM; VM[Var0.get()] = Var1.get(); VM[Storage0.get()] = Storage1.get(); + VM[DS0.get()] = DS1.get(); - ConstantPtrAuth *Value = - ConstantPtrAuth::get(Var0.get(), ConstKey, ConstDisc, Storage0.get()); - ConstantPtrAuth *MappedValue = - ConstantPtrAuth::get(Var1.get(), ConstKey, ConstDisc, Storage1.get()); + ConstantPtrAuth *Value = ConstantPtrAuth::get(Var0.get(), ConstKey, ConstDisc, + Storage0.get(), DS0.get()); + ConstantPtrAuth *MappedValue = ConstantPtrAuth::get( + Var1.get(), ConstKey, ConstDisc, Storage1.get(), DS1.get()); EXPECT_EQ(ValueMapper(VM).mapValue(*Value), MappedValue); } From a33fd4437216fff3d092e5056a78c4f430b2f9da Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Wed, 26 Nov 2025 21:40:15 +0100 Subject: [PATCH 03/22] Revert "[tysan] Type Sanitizer support for SystemZ" (#169726) Reverts llvm/llvm-project#162396 --- clang/lib/Driver/ToolChains/Linux.cpp | 2 +- compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake | 2 +- compiler-rt/lib/tysan/tysan_platform.h | 6 ------ 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/clang/lib/Driver/ToolChains/Linux.cpp b/clang/lib/Driver/ToolChains/Linux.cpp index cdbf21fb90263..2c741a38fce1a 100644 --- a/clang/lib/Driver/ToolChains/Linux.cpp +++ b/clang/lib/Driver/ToolChains/Linux.cpp @@ -922,7 +922,7 @@ SanitizerMask Linux::getSupportedSanitizers() const { if (IsX86_64 || IsMIPS64 || IsAArch64 || IsPowerPC64 || IsSystemZ || IsLoongArch64 || IsRISCV64) Res |= SanitizerKind::Thread; - if (IsX86_64 || IsAArch64 || IsSystemZ) + if (IsX86_64 || IsAArch64) Res |= SanitizerKind::Type; if (IsX86_64 || IsSystemZ || IsPowerPC64) Res |= SanitizerKind::KernelMemory; diff --git a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake index f2317de8916e9..c10367715396e 100644 --- a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake +++ b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake @@ -89,7 +89,7 @@ else() set(ALL_TSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64} ${PPC64} ${S390X} ${LOONGARCH64} ${RISCV64}) endif() -set(ALL_TYSAN_SUPPORTED_ARCH ${X86_64} ${ARM64} ${S390X}) +set(ALL_TYSAN_SUPPORTED_ARCH ${X86_64} ${ARM64}) set(ALL_UBSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${RISCV64} ${MIPS32} ${MIPS64} ${PPC64} ${S390X} ${SPARC} ${SPARCV9} ${HEXAGON} ${LOONGARCH64}) diff --git a/compiler-rt/lib/tysan/tysan_platform.h b/compiler-rt/lib/tysan/tysan_platform.h index 7d84ef228216d..19f77f0cace6b 100644 --- a/compiler-rt/lib/tysan/tysan_platform.h +++ b/compiler-rt/lib/tysan/tysan_platform.h @@ -45,12 +45,6 @@ struct Mapping48 { static const uptr kPtrShift = 3; }; #define TYSAN_RUNTIME_VMA 1 -#elif defined(__s390x__) -struct Mapping { - static const uptr kShadowAddr = 0x080000000000ULL; - static const uptr kAppAddr = 0x460000000000ULL; - static const uptr kAppMemMsk = ~0xC00000000000ULL; -}; #else #error "TySan not supported for this platform!" #endif From 75ca83563de13ebbf381a0e9e9d97dfbf98ea0f5 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Wed, 26 Nov 2025 12:41:37 -0800 Subject: [PATCH 04/22] [bazel] Fix build after #169086 (#169725) Just required wiring up some additional AMDGPU table generated files. --- utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 6d2eedbfe2415..81451509de276 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -1694,6 +1694,12 @@ gentbl_cc_library( "-gen-dialect-defs", "-dialect=amdgpu", ], + "include/mlir/Dialect/AMDGPU/IR/AMDGPUTypes.h.inc": [ + "-gen-typedef-decls", + ], + "include/mlir/Dialect/AMDGPU/IR/AMDGPUTypes.cpp.inc": [ + "-gen-typedef-defs", + ], "include/mlir/Dialect/AMDGPU/IR/AMDGPUEnums.h.inc": ["-gen-enum-decls"], "include/mlir/Dialect/AMDGPU/IR/AMDGPUEnums.cpp.inc": ["-gen-enum-defs"], "include/mlir/Dialect/AMDGPU/IR/AMDGPU.h.inc": ["-gen-op-decls"], From 8e4208f83a9bb7a4cd550e90e70f3b77499c623e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?= =?UTF-8?q?=E3=83=B3=29?= Date: Wed, 26 Nov 2025 13:12:48 -0800 Subject: [PATCH 05/22] [flang][cuda][rt] Add entry point to get the allocation stream (#169608) --- flang-rt/lib/cuda/allocator.cpp | 11 +++- .../unittests/Runtime/CUDA/Allocatable.cpp | 51 +++++++++++++++++++ flang/include/flang/Runtime/CUDA/allocator.h | 3 ++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/flang-rt/lib/cuda/allocator.cpp b/flang-rt/lib/cuda/allocator.cpp index 5436051002265..d2aa832883e65 100644 --- a/flang-rt/lib/cuda/allocator.cpp +++ b/flang-rt/lib/cuda/allocator.cpp @@ -19,8 +19,6 @@ #include "flang/Runtime/CUDA/common.h" #include "flang/Support/Fortran.h" -#include "cuda_runtime.h" - namespace Fortran::runtime::cuda { struct DeviceAllocation { @@ -133,6 +131,15 @@ void RTDEF(CUFRegisterAllocator)() { allocatorRegistry.Register( kUnifiedAllocatorPos, {&CUFAllocUnified, CUFFreeUnified}); } + +cudaStream_t RTDECL(CUFAssociatedGetStream)(void *p) { + int pos = findAllocation(p); + if (pos >= 0) { + cudaStream_t stream = deviceAllocations[pos].stream; + return stream; + } + return nullptr; +} } void *CUFAllocPinned( diff --git a/flang-rt/unittests/Runtime/CUDA/Allocatable.cpp b/flang-rt/unittests/Runtime/CUDA/Allocatable.cpp index 9935ae0eaac2f..4e65326b31a62 100644 --- a/flang-rt/unittests/Runtime/CUDA/Allocatable.cpp +++ b/flang-rt/unittests/Runtime/CUDA/Allocatable.cpp @@ -121,3 +121,54 @@ TEST(AllocatableCUFTest, StreamDeviceAllocatable) { cudaDeviceSynchronize(); EXPECT_EQ(cudaSuccess, cudaGetLastError()); } + +TEST(AllocatableAsyncTest, StreamDeviceAllocatable) { + using Fortran::common::TypeCategory; + RTNAME(CUFRegisterAllocator)(); + // REAL(4), DEVICE, ALLOCATABLE :: a(:) + auto a{createAllocatable(TypeCategory::Real, 4)}; + a->SetAllocIdx(kDeviceAllocatorPos); + EXPECT_EQ((int)kDeviceAllocatorPos, a->GetAllocIdx()); + EXPECT_FALSE(a->HasAddendum()); + RTNAME(AllocatableSetBounds)(*a, 0, 1, 10); + + cudaStream_t stream; + cudaStreamCreate(&stream); + EXPECT_EQ(cudaSuccess, cudaGetLastError()); + + RTNAME(AllocatableAllocate) + (*a, /*asyncObject=*/(int64_t *)&stream, /*hasStat=*/false, + /*errMsg=*/nullptr, __FILE__, __LINE__); + EXPECT_TRUE(a->IsAllocated()); + cudaDeviceSynchronize(); + EXPECT_EQ(cudaSuccess, cudaGetLastError()); + cudaStream_t s = RTDECL(CUFAssociatedGetStream)(a->raw().base_addr); + EXPECT_EQ(s, stream); + RTNAME(AllocatableDeallocate) + (*a, /*hasStat=*/false, /*errMsg=*/nullptr, __FILE__, __LINE__); + EXPECT_FALSE(a->IsAllocated()); + cudaDeviceSynchronize(); + + cudaStream_t defaultStream = 0; + RTNAME(AllocatableAllocate) + (*a, /*asyncObject=*/(int64_t *)&defaultStream, /*hasStat=*/false, + /*errMsg=*/nullptr, __FILE__, __LINE__); + EXPECT_TRUE(a->IsAllocated()); + cudaDeviceSynchronize(); + EXPECT_EQ(cudaSuccess, cudaGetLastError()); + cudaStream_t d = RTDECL(CUFAssociatedGetStream)(a->raw().base_addr); + EXPECT_EQ(d, defaultStream); + RTNAME(AllocatableDeallocate) + (*a, /*hasStat=*/false, /*errMsg=*/nullptr, __FILE__, __LINE__); + EXPECT_FALSE(a->IsAllocated()); + cudaDeviceSynchronize(); + + RTNAME(AllocatableAllocate) + (*a, /*asyncObject=*/nullptr, /*hasStat=*/false, /*errMsg=*/nullptr, __FILE__, + __LINE__); + EXPECT_TRUE(a->IsAllocated()); + cudaDeviceSynchronize(); + EXPECT_EQ(cudaSuccess, cudaGetLastError()); + cudaStream_t empty = RTDECL(CUFAssociatedGetStream)(a->raw().base_addr); + EXPECT_EQ(empty, nullptr); +} diff --git a/flang/include/flang/Runtime/CUDA/allocator.h b/flang/include/flang/Runtime/CUDA/allocator.h index 59fdb22b6e663..4e38482a7de30 100644 --- a/flang/include/flang/Runtime/CUDA/allocator.h +++ b/flang/include/flang/Runtime/CUDA/allocator.h @@ -13,11 +13,14 @@ #include "flang/Runtime/descriptor-consts.h" #include "flang/Runtime/entry-names.h" +#include "cuda_runtime.h" + namespace Fortran::runtime::cuda { extern "C" { void RTDECL(CUFRegisterAllocator)(); +cudaStream_t RTDECL(CUFAssociatedGetStream)(void *); } void *CUFAllocPinned(std::size_t, std::int64_t *); From 47efff777d907fcabda59d925dfed3040c7308be Mon Sep 17 00:00:00 2001 From: Kai Nacke Date: Wed, 26 Nov 2025 16:16:13 -0500 Subject: [PATCH 06/22] [SystemZ] Emit optional argument area length field (#169679) The Language Environment (LE) reserves 128 byte for the argument area when the optional field is not present. If the argument area is larger, then the field must be present to guarantee that the space is reserved on stack extension. Creating this field when alloca() is used may reduce the needed stack space in case alloca() causes a stack extension. --- llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp | 32 ++++++++- llvm/test/CodeGen/SystemZ/zos-ppa1-argarea.ll | 66 +++++++++++++++++++ 2 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 llvm/test/CodeGen/SystemZ/zos-ppa1-argarea.ll diff --git a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp index e31d7c6a86476..f061272d3fad4 100644 --- a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp +++ b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp @@ -1270,7 +1270,7 @@ void SystemZAsmPrinter::emitFunctionBodyEnd() { static void emitPPA1Flags(std::unique_ptr &OutStreamer, bool VarArg, bool StackProtector, bool FPRMask, bool VRMask, - bool EHBlock, bool HasName) { + bool EHBlock, bool HasArgAreaLength, bool HasName) { enum class PPA1Flag1 : uint8_t { DSA64Bit = (0x80 >> 0), VarArg = (0x80 >> 7), @@ -1282,8 +1282,9 @@ static void emitPPA1Flags(std::unique_ptr &OutStreamer, bool VarArg, LLVM_MARK_AS_BITMASK_ENUM(ExternalProcedure) }; enum class PPA1Flag3 : uint8_t { + HasArgAreaLength = (0x80 >> 1), FPRMask = (0x80 >> 2), - LLVM_MARK_AS_BITMASK_ENUM(FPRMask) + LLVM_MARK_AS_BITMASK_ENUM(HasArgAreaLength) }; enum class PPA1Flag4 : uint8_t { EPMOffsetPresent = (0x80 >> 0), @@ -1307,6 +1308,9 @@ static void emitPPA1Flags(std::unique_ptr &OutStreamer, bool VarArg, if (StackProtector) Flags2 |= PPA1Flag2::STACKPROTECTOR; + if (HasArgAreaLength) + Flags3 |= PPA1Flag3::HasArgAreaLength; // Add emit ArgAreaLength flag. + // SavedGPRMask, SavedFPRMask, and SavedVRMask are precomputed in. if (FPRMask) Flags3 |= PPA1Flag3::FPRMask; // Add emit FPR mask flag. @@ -1339,6 +1343,9 @@ static void emitPPA1Flags(std::unique_ptr &OutStreamer, bool VarArg, OutStreamer->emitInt8(static_cast(Flags2)); // Flags 2. OutStreamer->AddComment("PPA1 Flags 3"); + if ((Flags3 & PPA1Flag3::HasArgAreaLength) == PPA1Flag3::HasArgAreaLength) + OutStreamer->AddComment( + " Bit 1: 1 = Argument Area Length is in optional area"); if ((Flags3 & PPA1Flag3::FPRMask) == PPA1Flag3::FPRMask) OutStreamer->AddComment(" Bit 2: 1 = FP Reg Mask is in optional area"); OutStreamer->emitInt8( @@ -1477,12 +1484,26 @@ void SystemZAsmPrinter::emitPPA1(MCSymbol *FnEndSym) { bool NeedEmitEHBlock = !MF->getLandingPads().empty(); + // Optional Argument Area Length. + // Note: This represents the length of the argument area that we reserve + // in our stack for setting up arguments for calls to other + // routines. If this optional field is not set, LE will reserve + // 128 bytes for the argument area. This optional field is + // created if greater than 128 bytes is required - to guarantee + // the required space is reserved on stack extension in the new + // extension. This optional field is also created if the + // routine has alloca(). This may reduce stack space + // if alloca() call causes a stack extension. + bool HasArgAreaLength = + (AllocaReg != 0) || (MFFrame.getMaxCallFrameSize() > 128); + bool HasName = MF->getFunction().hasName() && MF->getFunction().getName().size() > 0; emitPPA1Flags(OutStreamer, MF->getFunction().isVarArg(), MFFrame.hasStackProtectorIndex(), SavedFPRMask != 0, - TargetHasVector && SavedVRMask != 0, NeedEmitEHBlock, HasName); + TargetHasVector && SavedVRMask != 0, NeedEmitEHBlock, + HasArgAreaLength, HasName); OutStreamer->AddComment("Length/4 of Parms"); OutStreamer->emitInt16( @@ -1490,6 +1511,11 @@ void SystemZAsmPrinter::emitPPA1(MCSymbol *FnEndSym) { OutStreamer->AddComment("Length of Code"); OutStreamer->emitAbsoluteSymbolDiff(FnEndSym, CurrentFnEPMarkerSym, 4); + if (HasArgAreaLength) { + OutStreamer->AddComment("Argument Area Length"); + OutStreamer->emitInt32(MFFrame.getMaxCallFrameSize()); + } + // Emit saved FPR mask and offset to FPR save area (0x20 of flags 3). if (SavedFPRMask) { OutStreamer->AddComment("FPR mask"); diff --git a/llvm/test/CodeGen/SystemZ/zos-ppa1-argarea.ll b/llvm/test/CodeGen/SystemZ/zos-ppa1-argarea.ll new file mode 100644 index 0000000000000..511bc46567607 --- /dev/null +++ b/llvm/test/CodeGen/SystemZ/zos-ppa1-argarea.ll @@ -0,0 +1,66 @@ +; RUN: llc < %s -mtriple=s390x-ibm-zos -emit-gnuas-syntax-on-zos=0 | FileCheck %s +%struct.LargeStruct_t = type { [33 x i32] } + +@GlobLargeS = hidden global %struct.LargeStruct_t zeroinitializer, align 4 +@GlobInt = hidden global i32 0, align 4 + +; === Check that function with small frame does not emit PPA1 Argument Area Length. +define void @fSmallOutArgArea() { +; CHECK-LABEL: L#EPM_fSmallOutArgArea_0 DS 0H +; CHECK: * Bit 1: 1 = Leaf function +; CHECK: * Bit 2: 0 = Does not use alloca +; CHECK: DC XL4'00000008' +; CHECK: fSmallOutArgArea DS 0H +; CHECK: L#PPA1_fSmallOutArgArea_0 DS 0H +; CHECK: * PPA1 Flags 3 +; CHECK: DC XL1'00' + ret void +} + +; === Check that function with large frame does emit PPA1 Argument Area Length. +define void @fLargeOutArgArea() { +; CHECK-LABEL: L#EPM_fLargeOutArgArea_0 DS 0H +; CHECK: * Bit 1: 0 = Non-leaf function +; CHECK: * Bit 2: 0 = Does not use alloca +; CHECK: DC XL4'00000220' +; CHECK: fLargeOutArgArea DS 0H +; CHECK: L#PPA1_fLargeOutArgArea_0 DS 0H +; CHECK: * PPA1 Flags 3 +; CHECK: * Bit 1: 1 = Argument Area Length is in optional area +; CHECK: DC XL1'40' +; CHECK: * Argument Area Length +; CHECK: DC XL4'00000140' + %1 = load [33 x i32], ptr @GlobLargeS, align 4 + call void @fLargeParm([33 x i32] inreg %1) + ret void +} + +; === Check that function with parameter does emit PPA1 Length/4 of parms +define void @fLargeParm([33 x i64] inreg %arr) { +; CHECK-LABEL: L#EPM_fLargeParm_0 DS 0H +; CHECK: * Length/4 of Parms +; CHECK: DC XL2'0042' + %1 = extractvalue [33 x i64] %arr, 1 + call void @foo(i64 %1) + ret void +} + +; === Check that function with alloca call does emit PPA1 Argument Area Length. +define hidden void @fHasAlloca() { +; CHECK-LABEL: L#EPM_fHasAlloca_0 DS 0H +; CHECK: * Bit 2: 1 = Uses alloca +; CHECK: fHasAlloca DS 0H +; CHECK: L#PPA1_fHasAlloca_0 DS 0H +; CHECK: * PPA1 Flags 3 +; CHECK: * Bit 1: 1 = Argument Area Length is in optional area +; CHECK: DC XL1'40' +; CHECK: * Argument Area Length +; CHECK: DC XL4'00000040' + %p = alloca ptr, align 4 + %1 = load i32, ptr @GlobInt, align 4 + %2 = alloca i8, i32 %1, align 8 + store ptr %2, ptr %p, align 4 + ret void +} + +declare void @foo(i64) From 48454241cde713c450e2369983b8c98b7ab16f19 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Wed, 26 Nov 2025 13:21:07 -0800 Subject: [PATCH 07/22] [SPIRV] Fix a warning This patch fixes: llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp:245:25: error: unused variable 'TII' [-Werror,-Wunused-variable] --- llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index 0b89e5f4cf316..8b1a09caf907d 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -242,7 +242,7 @@ void SPIRVGlobalRegistry::invalidateMachineInstr(MachineInstr *MI) { // remove. const SPIRVSubtarget &ST = MI->getMF()->getSubtarget(); - const SPIRVInstrInfo *TII = ST.getInstrInfo(); + [[maybe_unused]] const SPIRVInstrInfo *TII = ST.getInstrInfo(); assert(!TII->isAliasingInstr(*MI) && "Cannot invalidate aliasing instructions."); assert(MI->getOpcode() != SPIRV::OpFunction && From 9871d7089890f357308804987ceae1e98c5c42a3 Mon Sep 17 00:00:00 2001 From: Razvan Lupusoru Date: Wed, 26 Nov 2025 13:26:04 -0800 Subject: [PATCH 08/22] [mlir][acc] Introduce ACCImplicitDeclare pass for globals handling (#169720) This commit introduces the ACCImplicitDeclare pass to the OpenACC dialect, complementing ACCImplicitData by handling global variables referenced in OpenACC compute regions and routines. Overview: --------- The pass applies implicit `acc declare` actions to global variables referenced in OpenACC regions. While the OpenACC spec focuses on implicit data mapping (handled by ACCImplicitData), implicit declare is advantageous and required for specific cases: 1. Globals referenced in implicit `acc routine` - Since data mapping only applies to compute regions, globals in routines must use `acc declare`. 2. Compiler-generated globals - Type descriptors, runtime names, and error reporting strings introduced during compilation that wouldn't be visible for user-provided `acc declare` directives. 3. Constant globals - Constants like filename strings or initialization values benefit from being marked with `acc declare` rather than being mapped repeatedly (e.g., 1000 kernel launches shouldn't map the same constant 1000 times). Implementation: --------------- The pass performs this in two phases: 1. Hoisting: Non-constant globals in compute regions have their address-of operations hoisted out of the region when possible, allowing implicit data mapping instead of declare marking. 2. Declaration: Remaining that must be device available (constants, globals in routines, globals in recipe operations) are marked with the acc.declare attribute. The pass processes: - OpenACC compute constructs (parallel, kernels, serial) - Functions marked with acc routine - Private, firstprivate, and reduction recipes (when used) - Initialization regions of existing declared globals Requirements: ------------- The pass requires operations to implement: - acc::AddressOfGlobalOpInterface (for address-of ops) - acc::GlobalVariableOpInterface (for global definitions) - acc::IndirectGlobalAccessOpInterface (for indirect access) --- mlir/include/mlir/Dialect/OpenACC/OpenACC.h | 5 + .../mlir/Dialect/OpenACC/Transforms/Passes.td | 28 ++ .../OpenACC/Transforms/ACCImplicitDeclare.cpp | 429 ++++++++++++++++++ .../Dialect/OpenACC/Transforms/CMakeLists.txt | 1 + .../Dialect/OpenACC/acc-implicit-declare.mlir | 175 +++++++ 5 files changed, 638 insertions(+) create mode 100644 mlir/lib/Dialect/OpenACC/Transforms/ACCImplicitDeclare.cpp create mode 100644 mlir/test/Dialect/OpenACC/acc-implicit-declare.mlir diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h index 8571a2d9d0939..252a78648dd74 100644 --- a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h +++ b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h @@ -180,6 +180,11 @@ static constexpr StringLiteral getRoutineInfoAttrName() { return StringLiteral("acc.routine_info"); } +/// Used to check whether the current operation is an `acc routine` +inline bool isAccRoutineOp(mlir::Operation *op) { + return op->hasAttr(mlir::acc::getRoutineInfoAttrName()); +} + static constexpr StringLiteral getFromDefaultClauseAttrName() { return StringLiteral("acc.from_default"); } diff --git a/mlir/include/mlir/Dialect/OpenACC/Transforms/Passes.td b/mlir/include/mlir/Dialect/OpenACC/Transforms/Passes.td index cad78df2fbb0b..713aaabee65f0 100644 --- a/mlir/include/mlir/Dialect/OpenACC/Transforms/Passes.td +++ b/mlir/include/mlir/Dialect/OpenACC/Transforms/Passes.td @@ -63,6 +63,34 @@ def ACCImplicitData : Pass<"acc-implicit-data", "mlir::ModuleOp"> { ]; } +def ACCImplicitDeclare : Pass<"acc-implicit-declare", "mlir::ModuleOp"> { + let summary = "Applies implicit acc declare to globals referenced in compute and routine acc regions"; + let description = [{ + This pass applies implicit `acc declare` actions to global variables + referenced in OpenACC compute regions and routine functions. + + The pass performs the following actions: + + 1. Hoists address-of operations for non-constant globals out of OpenACC + regions when they can be implicitly mapped rather than declared. + + 2. Collects global symbols referenced in: + - OpenACC compute constructs (parallel, kernels, serial) + - Functions marked with acc routine + - Initialization regions of existing acc declare globals + - Private/firstprivate/reduction recipe operations + + 3. Marks collected globals with the acc.declare attribute using the + copyin data clause. + + The pass avoids unnecessary declare marking by: + - Skipping function symbols (which use acc routine instead) + - Hoisting non-constant global references that can use implicit mapping + - Only processing symbols that are not already valid in device regions + }]; + let dependentDialects = ["mlir::acc::OpenACCDialect"]; +} + def ACCImplicitRoutine : Pass<"acc-implicit-routine", "mlir::ModuleOp"> { let summary = "Generate implicit acc routine for functions in acc regions"; let description = [{ diff --git a/mlir/lib/Dialect/OpenACC/Transforms/ACCImplicitDeclare.cpp b/mlir/lib/Dialect/OpenACC/Transforms/ACCImplicitDeclare.cpp new file mode 100644 index 0000000000000..766f690e21459 --- /dev/null +++ b/mlir/lib/Dialect/OpenACC/Transforms/ACCImplicitDeclare.cpp @@ -0,0 +1,429 @@ +//===- ACCImplicitDeclare.cpp ---------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass applies implicit `acc declare` actions to global variables +// referenced in OpenACC compute regions and routine functions. +// +// Overview: +// --------- +// Global references in an acc regions (for globals not marked with `acc +// declare` by the user) can be handled in one of two ways: +// - Mapped through data clauses +// - Implicitly marked as `acc declare` (this pass) +// +// Thus, the OpenACC specification focuses solely on implicit data mapping rules +// whose implementation is captured in `ACCImplicitData` pass. +// +// However, it is both advantageous and required for certain cases to +// use implicit `acc declare` instead: +// - Any functions that are implicitly marked as `acc routine` through +// `ACCImplicitRoutine` may reference globals. Since data mapping +// is only possible for compute regions, such globals can only be +// made available on device through `acc declare`. +// - Compiler can generate and use globals for cases needed in IR +// representation such as type descriptors or various names needed for +// runtime calls and error reporting - such cases often are introduced +// after a frontend semantic checking is done since it is related to +// implementation detail. Thus, such compiler generated globals would +// not have been visible for a user to mark with `acc declare`. +// - Constant globals such as filename strings or data initialization values +// are values that do not get mutated but are still needed for appropriate +// runtime execution. If a kernel is launched 1000 times, it is not a +// good idea to map such a global 1000 times. Therefore, such globals +// benefit from being marked with `acc declare`. +// +// This pass automatically +// marks global variables with the `acc.declare` attribute when they are +// referenced in OpenACC compute constructs or routine functions and meet +// the criteria noted above, ensuring +// they are properly handled for device execution. +// +// The pass performs two main optimizations: +// +// 1. Hoisting: For non-constant globals referenced in compute regions, the +// pass hoists the address-of operation out of the region when possible, +// allowing them to be implicitly mapped through normal data clause +// mechanisms rather than requiring declare marking. +// +// 2. Declaration: For globals that must be available on the device (constants, +// globals in routines, globals in recipe operations), the pass adds the +// `acc.declare` attribute with the copyin data clause. +// +// Requirements: +// ------------- +// To use this pass in a pipeline, the following requirements must be met: +// +// 1. Operation Interface Implementation: Operations that compute addresses +// of global variables must implement the `acc::AddressOfGlobalOpInterface` +// and those that represent globals must implement the +// `acc::GlobalOpInterface`. Additionally, any operations that indirectly +// access globals must implement the `acc::IndirectGlobalAccessOpInterface`. +// +// 2. Analysis Registration (Optional): If custom behavior is needed for +// determining if a symbol use is valid within GPU regions, the dialect +// should pre-register the `acc::OpenACCSupport` analysis. +// +// Examples: +// --------- +// +// Example 1: Non-constant global in compute region (hoisted) +// +// Before: +// memref.global @g_scalar : memref = dense<0.0> +// func.func @test() { +// acc.serial { +// %addr = memref.get_global @g_scalar : memref +// %val = memref.load %addr[] : memref +// acc.yield +// } +// } +// +// After: +// memref.global @g_scalar : memref = dense<0.0> +// func.func @test() { +// %addr = memref.get_global @g_scalar : memref +// acc.serial { +// %val = memref.load %addr[] : memref +// acc.yield +// } +// } +// +// Example 2: Constant global in compute region (declared) +// +// Before: +// memref.global constant @g_const : memref = dense<1.0> +// func.func @test() { +// acc.serial { +// %addr = memref.get_global @g_const : memref +// %val = memref.load %addr[] : memref +// acc.yield +// } +// } +// +// After: +// memref.global constant @g_const : memref = dense<1.0> +// {acc.declare = #acc.declare} +// func.func @test() { +// acc.serial { +// %addr = memref.get_global @g_const : memref +// %val = memref.load %addr[] : memref +// acc.yield +// } +// } +// +// Example 3: Global in acc routine (declared) +// +// Before: +// memref.global @g_data : memref = dense<0.0> +// acc.routine @routine_0 func(@device_func) +// func.func @device_func() attributes {acc.routine_info = ...} { +// %addr = memref.get_global @g_data : memref +// %val = memref.load %addr[] : memref +// } +// +// After: +// memref.global @g_data : memref = dense<0.0> +// {acc.declare = #acc.declare} +// acc.routine @routine_0 func(@device_func) +// func.func @device_func() attributes {acc.routine_info = ...} { +// %addr = memref.get_global @g_data : memref +// %val = memref.load %addr[] : memref +// } +// +// Example 4: Global in private recipe (declared if recipe is used) +// +// Before: +// memref.global @g_init : memref = dense<0.0> +// acc.private.recipe @priv_recipe : memref init { +// ^bb0(%arg0: memref): +// %alloc = memref.alloc() : memref +// %global = memref.get_global @g_init : memref +// %val = memref.load %global[] : memref +// memref.store %val, %alloc[] : memref +// acc.yield %alloc : memref +// } destroy { ... } +// func.func @test() { +// %var = memref.alloc() : memref +// %priv = acc.private varPtr(%var : memref) +// recipe(@priv_recipe) -> memref +// acc.parallel private(%priv : memref) { ... } +// } +// +// After: +// memref.global @g_init : memref = dense<0.0> +// {acc.declare = #acc.declare} +// acc.private.recipe @priv_recipe : memref init { +// ^bb0(%arg0: memref): +// %alloc = memref.alloc() : memref +// %global = memref.get_global @g_init : memref +// %val = memref.load %global[] : memref +// memref.store %val, %alloc[] : memref +// acc.yield %alloc : memref +// } destroy { ... } +// func.func @test() { +// %var = memref.alloc() : memref +// %priv = acc.private varPtr(%var : memref) +// recipe(@priv_recipe) -> memref +// acc.parallel private(%priv : memref) { ... } +// } +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/OpenACC/Transforms/Passes.h" + +#include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h" +#include "mlir/Dialect/OpenACC/OpenACC.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/IR/BuiltinOps.h" +#include "mlir/IR/Operation.h" +#include "mlir/IR/Value.h" +#include "mlir/Interfaces/FunctionInterfaces.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/TypeSwitch.h" + +namespace mlir { +namespace acc { +#define GEN_PASS_DEF_ACCIMPLICITDECLARE +#include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc" +} // namespace acc +} // namespace mlir + +#define DEBUG_TYPE "acc-implicit-declare" + +using namespace mlir; + +namespace { + +using GlobalOpSetT = llvm::SmallSetVector; + +/// Checks whether a use of the requested `globalOp` should be considered +/// for hoisting out of acc region due to avoid `acc declare`ing something +/// that instead should be implicitly mapped. +static bool isGlobalUseCandidateForHoisting(Operation *globalOp, + Operation *user, + SymbolRefAttr symbol, + acc::OpenACCSupport &accSupport) { + // This symbol is valid in GPU region. This means semantics + // would change if moved to host - therefore it is not a candidate. + if (accSupport.isValidSymbolUse(user, symbol)) + return false; + + bool isConstant = false; + bool isFunction = false; + + if (auto globalVarOp = dyn_cast(globalOp)) + isConstant = globalVarOp.isConstant(); + + if (isa(globalOp)) + isFunction = true; + + // Constants should be kept in device code to ensure they are duplicated. + // Function references should be kept in device code to ensure their device + // addresses are computed. Everything else should be hoisted since we already + // proved they are not valid symbols in GPU region. + return !isConstant && !isFunction; +} + +/// Checks whether it is valid to use acc.declare marking on the global. +bool isValidForAccDeclare(Operation *globalOp) { + // For functions - we use acc.routine marking instead. + return !isa(globalOp); +} + +/// Checks whether a recipe operation has meaningful use of its symbol that +/// justifies processing its regions for global references. Returns false if: +/// 1. The recipe has no symbol uses at all, or +/// 2. The only symbol use is the recipe's own symbol definition +template +static bool hasRelevantRecipeUse(RecipeOpT &recipeOp, ModuleOp &mod) { + std::optional symbolUses = recipeOp.getSymbolUses(mod); + + // No recipe symbol uses. + if (!symbolUses.has_value() || symbolUses->empty()) + return false; + + // If more than one use, assume it's used. + auto begin = symbolUses->begin(); + auto end = symbolUses->end(); + if (begin != end && std::next(begin) != end) + return true; + + // If single use, check if the use is the recipe itself. + const SymbolTable::SymbolUse &use = *symbolUses->begin(); + return use.getUser() != recipeOp.getOperation(); +} + +// Hoists addr_of operations for non-constant globals out of OpenACC regions. +// This way - they are implicitly mapped instead of being considered for +// implicit declare. +template +static void hoistNonConstantDirectUses(AccConstructT accOp, + acc::OpenACCSupport &accSupport) { + accOp.walk([&](acc::AddressOfGlobalOpInterface addrOfOp) { + SymbolRefAttr symRef = addrOfOp.getSymbol(); + if (symRef) { + Operation *globalOp = + SymbolTable::lookupNearestSymbolFrom(addrOfOp, symRef); + if (isGlobalUseCandidateForHoisting(globalOp, addrOfOp, symRef, + accSupport)) { + addrOfOp->moveBefore(accOp); + LLVM_DEBUG( + llvm::dbgs() << "Hoisted:\n\t" << addrOfOp << "\n\tfrom:\n\t"; + accOp->print(llvm::dbgs(), + OpPrintingFlags{}.skipRegions().enableDebugInfo()); + llvm::dbgs() << "\n"); + } + } + }); +} + +// Collects the globals referenced in a device region +static void collectGlobalsFromDeviceRegion(Region ®ion, + GlobalOpSetT &globals, + acc::OpenACCSupport &accSupport, + SymbolTable &symTab) { + region.walk([&](Operation *op) { + // 1) Only consider relevant operations which use symbols + auto addrOfOp = dyn_cast(op); + if (addrOfOp) { + SymbolRefAttr symRef = addrOfOp.getSymbol(); + // 2) Found an operation which uses the symbol. Next determine if it + // is a candidate for `acc declare`. Some of the criteria considered + // is whether this symbol is not already a device one (either because + // acc declare is already used or this is a CUF global). + Operation *globalOp = nullptr; + bool isCandidate = !accSupport.isValidSymbolUse(op, symRef, &globalOp); + // 3) Add the candidate to the set of globals to be `acc declare`d. + if (isCandidate && globalOp && isValidForAccDeclare(globalOp)) + globals.insert(globalOp); + } else if (auto indirectAccessOp = + dyn_cast(op)) { + // Process operations that indirectly access globals + llvm::SmallVector symbols; + indirectAccessOp.getReferencedSymbols(symbols, &symTab); + for (SymbolRefAttr symRef : symbols) + if (Operation *globalOp = symTab.lookup(symRef.getLeafReference())) + if (isValidForAccDeclare(globalOp)) + globals.insert(globalOp); + } + }); +} + +// Adds the declare attribute to the operation `op`. +static void addDeclareAttr(MLIRContext *context, Operation *op, + acc::DataClause clause) { + op->setAttr(acc::getDeclareAttrName(), + acc::DeclareAttr::get(context, + acc::DataClauseAttr::get(context, clause))); +} + +// This pass applies implicit declare actions for globals referenced in +// OpenACC compute and routine regions. +class ACCImplicitDeclare + : public acc::impl::ACCImplicitDeclareBase { +public: + using ACCImplicitDeclareBase::ACCImplicitDeclareBase; + + void runOnOperation() override { + ModuleOp mod = getOperation(); + MLIRContext *context = &getContext(); + acc::OpenACCSupport &accSupport = getAnalysis(); + + // 1) Start off by hoisting any AddressOf operations out of acc region + // for any cases we do not want to `acc declare`. This is because we can + // rely on implicit data mapping in majority of cases without uselessly + // polluting the device globals. + mod.walk([&](Operation *op) { + TypeSwitch(op) + .Case( + [&](auto accOp) { + hoistNonConstantDirectUses(accOp, accSupport); + }); + }); + + // 2) Collect global symbols which need to be `acc declare`d. Do it for + // compute regions, acc routine, and existing globals with the declare + // attribute. + SymbolTable symTab(mod); + GlobalOpSetT globalsToAccDeclare; + mod.walk([&](Operation *op) { + TypeSwitch(op) + .Case( + [&](auto accOp) { + collectGlobalsFromDeviceRegion( + accOp.getRegion(), globalsToAccDeclare, accSupport, symTab); + }) + .Case([&](auto func) { + if (acc::isAccRoutineOp(func) && !func.isExternal()) + collectGlobalsFromDeviceRegion(func.getFunctionBody(), + globalsToAccDeclare, accSupport, + symTab); + }) + .Case([&](auto globalVarOp) { + if (globalVarOp->getAttr(acc::getDeclareAttrName())) + if (Region *initRegion = globalVarOp.getInitRegion()) + collectGlobalsFromDeviceRegion(*initRegion, globalsToAccDeclare, + accSupport, symTab); + }) + .Case([&](auto privateRecipe) { + if (hasRelevantRecipeUse(privateRecipe, mod)) { + collectGlobalsFromDeviceRegion(privateRecipe.getInitRegion(), + globalsToAccDeclare, accSupport, + symTab); + collectGlobalsFromDeviceRegion(privateRecipe.getDestroyRegion(), + globalsToAccDeclare, accSupport, + symTab); + } + }) + .Case([&](auto firstprivateRecipe) { + if (hasRelevantRecipeUse(firstprivateRecipe, mod)) { + collectGlobalsFromDeviceRegion(firstprivateRecipe.getInitRegion(), + globalsToAccDeclare, accSupport, + symTab); + collectGlobalsFromDeviceRegion( + firstprivateRecipe.getDestroyRegion(), globalsToAccDeclare, + accSupport, symTab); + collectGlobalsFromDeviceRegion(firstprivateRecipe.getCopyRegion(), + globalsToAccDeclare, accSupport, + symTab); + } + }) + .Case([&](auto reductionRecipe) { + if (hasRelevantRecipeUse(reductionRecipe, mod)) { + collectGlobalsFromDeviceRegion(reductionRecipe.getInitRegion(), + globalsToAccDeclare, accSupport, + symTab); + collectGlobalsFromDeviceRegion( + reductionRecipe.getCombinerRegion(), globalsToAccDeclare, + accSupport, symTab); + } + }); + }); + + // 3) Finally, generate the appropriate declare actions needed to ensure + // this is considered for device global. + for (Operation *globalOp : globalsToAccDeclare) { + LLVM_DEBUG( + llvm::dbgs() << "Global is being `acc declare copyin`d: "; + globalOp->print(llvm::dbgs(), + OpPrintingFlags{}.skipRegions().enableDebugInfo()); + llvm::dbgs() << "\n"); + + // Mark it as declare copyin. + addDeclareAttr(context, globalOp, acc::DataClause::acc_copyin); + + // TODO: May need to create the global constructor which does the mapping + // action. It is not yet clear if this is needed yet (since the globals + // might just end up in the GPU image without requiring mapping via + // runtime). + } + } +}; + +} // namespace diff --git a/mlir/lib/Dialect/OpenACC/Transforms/CMakeLists.txt b/mlir/lib/Dialect/OpenACC/Transforms/CMakeLists.txt index 028af0362f26e..2c6da87c66a11 100644 --- a/mlir/lib/Dialect/OpenACC/Transforms/CMakeLists.txt +++ b/mlir/lib/Dialect/OpenACC/Transforms/CMakeLists.txt @@ -1,5 +1,6 @@ add_mlir_dialect_library(MLIROpenACCTransforms ACCImplicitData.cpp + ACCImplicitDeclare.cpp ACCImplicitRoutine.cpp LegalizeDataValues.cpp diff --git a/mlir/test/Dialect/OpenACC/acc-implicit-declare.mlir b/mlir/test/Dialect/OpenACC/acc-implicit-declare.mlir new file mode 100644 index 0000000000000..74ff3384c093c --- /dev/null +++ b/mlir/test/Dialect/OpenACC/acc-implicit-declare.mlir @@ -0,0 +1,175 @@ +// RUN: mlir-opt %s --pass-pipeline="builtin.module(acc-implicit-declare)" -split-input-file 2>&1 | FileCheck %s + +// ----- + +// Test that non-constant scalar globals in compute regions are hoisted +// instead of being marked with acc declare + +memref.global @gscalar : memref = dense<0.0> + +func.func @test_scalar_in_serial() { + acc.serial { + %addr = memref.get_global @gscalar : memref + %load = memref.load %addr[] : memref + acc.yield + } + return +} + +// Expected to hoist this global access out of acc region instead of marking +// with `acc declare`. +// CHECK-LABEL: func.func @test_scalar_in_serial +// CHECK: memref.get_global @gscalar +// CHECK: acc.serial +// CHECK-NOT: acc.declare + +// ----- + +// Test that constant globals are marked with acc declare + +memref.global constant @gscalarconst : memref = dense<1.0> + +func.func @test_constant_in_serial() { + acc.serial { + %addr = memref.get_global @gscalarconst : memref + %load = memref.load %addr[] : memref + acc.yield + } + return +} + +// This is expected to be `acc declare`'d since it is a constant. +// CHECK: memref.global constant @gscalarconst {{.*}} {acc.declare = #acc.declare} + +// ----- + +// Test globals referenced in acc routine functions + +memref.global @gscalar_routine : memref = dense<0.0> + +acc.routine @acc_routine_0 func(@test_scalar_in_accroutine) +func.func @test_scalar_in_accroutine() attributes {acc.routine_info = #acc.routine_info<[@acc_routine_0]>} { + %addr = memref.get_global @gscalar_routine : memref + %load = memref.load %addr[] : memref + return +} + +// Global should be acc declare'd because it's in an acc routine +// CHECK: memref.global @gscalar_routine {{.*}} {acc.declare = #acc.declare} + +// ----- + +// Test constant globals in acc routine + +memref.global constant @gscalarconst_routine : memref = dense<1.0> + +acc.routine @acc_routine_0 func(@test_constant_in_accroutine) +func.func @test_constant_in_accroutine() attributes {acc.routine_info = #acc.routine_info<[@acc_routine_0]>} { + %addr = memref.get_global @gscalarconst_routine : memref + %load = memref.load %addr[] : memref + return +} + +// CHECK: memref.global constant @gscalarconst_routine {{.*}} {acc.declare = #acc.declare} + +// ----- + +// Test acc.private.recipe with global reference - referenced variant + +memref.global @global_for_private : memref = dense<0.0> + +acc.private.recipe @private_recipe_with_global : memref init { +^bb0(%arg0: memref): + %0 = memref.alloc() : memref + %global_addr = memref.get_global @global_for_private : memref + %global_val = memref.load %global_addr[] : memref + memref.store %global_val, %0[] : memref + acc.yield %0 : memref +} destroy { +^bb0(%arg0: memref): + memref.dealloc %arg0 : memref + acc.terminator +} + +func.func @test_private_recipe_referenced() { + %var = memref.alloc() : memref + %priv = acc.private varPtr(%var : memref) recipe(@private_recipe_with_global) -> memref + acc.parallel private(%priv : memref) { + %load = memref.load %var[] : memref + acc.yield + } + memref.dealloc %var : memref + return +} + +// Global should be acc declare'd because the recipe is referenced +// CHECK: memref.global @global_for_private {{.*}} {acc.declare = #acc.declare} + +// ----- + +// Test acc.private.recipe with global reference - unreferenced variant + +memref.global @global_for_private_unused : memref = dense<0.0> + +acc.private.recipe @private_recipe_unused : memref init { +^bb0(%arg0: memref): + %0 = memref.alloc() : memref + %global_addr = memref.get_global @global_for_private_unused : memref + %global_val = memref.load %global_addr[] : memref + memref.store %global_val, %0[] : memref + acc.yield %0 : memref +} destroy { +^bb0(%arg0: memref): + memref.dealloc %arg0 : memref + acc.terminator +} + +func.func @test_private_recipe_not_referenced() { + %var = memref.alloc() : memref + acc.parallel { + %load = memref.load %var[] : memref + acc.yield + } + memref.dealloc %var : memref + return +} + +// Global should NOT be acc declare'd because the recipe is not referenced +// CHECK-NOT: memref.global @global_for_private_unused {{.*}} {acc.declare + +// ----- + +// Test globals in different compute constructs (parallel, kernels, serial) + +memref.global @global_parallel : memref = dense<0.0> +memref.global @global_kernels : memref = dense<0.0> +memref.global constant @global_serial_const : memref = dense<1.0> + +func.func @test_multiple_constructs() { + acc.parallel { + %addr = memref.get_global @global_parallel : memref + %load = memref.load %addr[] : memref + acc.yield + } + acc.kernels { + %addr = memref.get_global @global_kernels : memref + %load = memref.load %addr[] : memref + acc.terminator + } + acc.serial { + %addr = memref.get_global @global_serial_const : memref + %load = memref.load %addr[] : memref + acc.yield + } + return +} + +// Non-constant globals ARE hoisted before their compute regions +// Constant global should be marked with acc.declare +// CHECK: memref.global constant @global_serial_const {{.*}} {acc.declare = #acc.declare} +// CHECK-LABEL: func.func @test_multiple_constructs +// CHECK: memref.get_global @global_parallel +// CHECK-NEXT: acc.parallel +// CHECK: memref.get_global @global_kernels +// CHECK-NEXT: acc.kernels + From 3d596ad09234c92067012b33a81be8dc48aa2e47 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Wed, 26 Nov 2025 13:29:50 -0800 Subject: [PATCH 09/22] [clang][Driver] Use -no-canonical-prefixes in hip-spirv-backend-opt test (#169717) Otherwise the test can fail in weirder setups (like ours downstream where the actual binary path only contains the hash of the object). This makes the test more resilient, more consistent with other driver tests, and allows us to assert that the binary is named clang rather than clang-. --- clang/test/Driver/hip-spirv-backend-opt.c | 28 +++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/clang/test/Driver/hip-spirv-backend-opt.c b/clang/test/Driver/hip-spirv-backend-opt.c index 88c4a848d5760..10d9a0b01caf3 100644 --- a/clang/test/Driver/hip-spirv-backend-opt.c +++ b/clang/test/Driver/hip-spirv-backend-opt.c @@ -4,58 +4,58 @@ // RUN: %clang --offload-new-driver --target=x86_64-unknown-linux-gnu --offload-arch=amdgcnspirv \ // RUN: -nogpuinc -nogpulib -### -x hip %s -save-temps \ -// RUN: -use-spirv-backend --offload-device-only -S \ +// RUN: -use-spirv-backend --offload-device-only -S -no-canonical-prefixes \ // RUN: 2>&1 | FileCheck %s --check-prefixes=CHECK-SPIRV-TRANSLATOR,CHECK-SPIRV-BACKEND-TEXTUAL // RUN: %clang --offload-new-driver --target=x86_64-unknown-linux-gnu --offload-arch=amdgcnspirv \ // RUN: -nogpuinc -nogpulib -### -x hip %s -save-temps \ -// RUN: -use-spirv-backend --offload-device-only \ +// RUN: -use-spirv-backend --offload-device-only -no-canonical-prefixes \ // RUN: 2>&1 | FileCheck %s --check-prefixes=CHECK-SPIRV-TRANSLATOR,CHECK-SPIRV-BACKEND-BINARY // The new driver's behavior is to emit LLVM IR for --offload-device-only and -fgpu-rdc (independently of SPIR-V). // RUN: %clang --offload-new-driver --target=x86_64-unknown-linux-gnu --offload-arch=amdgcnspirv \ // RUN: -### -nogpuinc -nogpulib -x hip %s -save-temps \ -// RUN: -use-spirv-backend --offload-device-only -S -fgpu-rdc \ +// RUN: -use-spirv-backend --offload-device-only -S -fgpu-rdc -no-canonical-prefixes \ // RUN: 2>&1 | FileCheck %s --check-prefixes=CHECK-SPIRV-TRANSLATOR,CHECK-SPIRV-BACKEND-LL,CHECK-FGPU-RDC // The new driver's behavior is to emit LLVM IR for --offload-device-only and -fgpu-rdc (independently of SPIR-V). // RUN: %clang --offload-new-driver --target=x86_64-unknown-linux-gnu --offload-arch=amdgcnspirv \ // RUN: -nogpuinc -nogpulib -### -x hip %s -save-temps \ -// RUN: -use-spirv-backend --offload-device-only -fgpu-rdc \ +// RUN: -use-spirv-backend --offload-device-only -fgpu-rdc -no-canonical-prefixes \ // RUN: 2>&1 | FileCheck %s --check-prefixes=CHECK-SPIRV-TRANSLATOR,CHECK-SPIRV-BACKEND-BC,CHECK-FGPU-RDC // --offload-device-only is always unset --- testing interactions with -S and -fgpu-rdc // RUN: %clang --offload-new-driver --target=x86_64-unknown-linux-gnu --offload-arch=amdgcnspirv \ // RUN: -nogpuinc -nogpulib -### -x hip %s -save-temps \ -// RUN: -use-spirv-backend -S -fgpu-rdc \ +// RUN: -use-spirv-backend -S -fgpu-rdc -no-canonical-prefixes \ // RUN: 2>&1 | FileCheck %s --check-prefixes=CHECK-SPIRV-TRANSLATOR,CHECK-SPIRV-BACKEND-BC,CHECK-FGPU-RDC // RUN: %clang --offload-new-driver --target=x86_64-unknown-linux-gnu --offload-arch=amdgcnspirv \ // RUN: -nogpuinc -nogpulib -### -x hip %s -save-temps \ -// RUN: -use-spirv-backend -S \ +// RUN: -use-spirv-backend -S -no-canonical-prefixes \ // RUN: 2>&1 | FileCheck %s --check-prefixes=CHECK-SPIRV-TRANSLATOR,CHECK-SPIRV-BACKEND-BC // RUN: %clang --offload-new-driver --target=x86_64-unknown-linux-gnu --offload-arch=amdgcnspirv \ // RUN: -nogpuinc -nogpulib -### -x hip %s -save-temps \ -// RUN: -use-spirv-backend -fgpu-rdc \ +// RUN: -use-spirv-backend -fgpu-rdc -no-canonical-prefixes \ // RUN: 2>&1 | FileCheck %s --check-prefixes=CHECK-SPIRV-TRANSLATOR,CHECK-SPIRV-BACKEND-BC,CHECK-CLANG-LINKER-WRAPPER // RUN: %clang --offload-new-driver --target=x86_64-unknown-linux-gnu --offload-arch=amdgcnspirv \ // RUN: -nogpuinc -nogpulib -### -x hip %s -save-temps \ -// RUN: -use-spirv-backend \ +// RUN: -use-spirv-backend -no-canonical-prefixes \ // RUN: 2>&1 | FileCheck %s --check-prefixes=CHECK-SPIRV-TRANSLATOR,CHECK-SPIRV-BACKEND-BC,CHECK-CLANG-LINKER-WRAPPER // RUN: %clang --no-offload-new-driver --target=x86_64-unknown-linux-gnu --offload-arch=amdgcnspirv \ // RUN: -nogpuinc -nogpulib -### -x hip %s -save-temps \ -// RUN: -use-spirv-backend \ +// RUN: -use-spirv-backend -no-canonical-prefixes \ // RUN: 2>&1 | FileCheck %s --check-prefixes=CHECK-SPIRV-TRANSLATOR,CHECK-SPIRV-BACKEND-BC,CHECK-SPIRV-BACKEND-BINARY-EQ-TRIPLE // CHECK-SPIRV-TRANSLATOR-NOT: "{{.*llvm-spirv.*}}" -// CHECK-SPIRV-BACKEND-TEXTUAL: "{{.*}}clang{{.*}}" "-cc1" "-triple" "spirv64-amd-amdhsa" {{.*}} "-S" -// CHECK-SPIRV-BACKEND-BINARY: "{{.*}}clang{{.*}}" "-cc1" "-triple" "spirv64-amd-amdhsa" {{.*}} "-emit-obj" -// CHECK-SPIRV-BACKEND-BC: "{{.*}}clang{{.*}}" "-cc1" "-triple" "spirv64-amd-amdhsa" {{.*}} "-emit-llvm-bc" -// CHECK-SPIRV-BACKEND-LL: "{{.*}}clang{{.*}}" "-cc1" "-triple" "spirv64-amd-amdhsa" {{.*}} "-emit-llvm" -// CHECK-SPIRV-BACKEND-BINARY-EQ-TRIPLE: "{{.*}}clang{{.*}}" "-cc1" {{.*}}"-triple=spirv64-amd-amdhsa" {{.*}}"-emit-obj" +// CHECK-SPIRV-BACKEND-TEXTUAL: "{{.*clang(\.exe)?}}" "-cc1" "-triple" "spirv64-amd-amdhsa" {{.*}} "-S" +// CHECK-SPIRV-BACKEND-BINARY: "{{.*clang(\.exe)?}}" "-cc1" "-triple" "spirv64-amd-amdhsa" {{.*}} "-emit-obj" +// CHECK-SPIRV-BACKEND-BC: "{{.*clang(\.exe)?}}" "-cc1" "-triple" "spirv64-amd-amdhsa" {{.*}} "-emit-llvm-bc" +// CHECK-SPIRV-BACKEND-LL: "{{.*clang(\.exe)?}}" "-cc1" "-triple" "spirv64-amd-amdhsa" {{.*}} "-emit-llvm" +// CHECK-SPIRV-BACKEND-BINARY-EQ-TRIPLE: "{{.*clang(\.exe)?}}" "-cc1" {{.*}}"-triple=spirv64-amd-amdhsa" {{.*}}"-emit-obj" // CHECK-FGPU-RDC-SAME: {{.*}} "-fgpu-rdc" // CHECK-CLANG-LINKER-WRAPPER: "{{.*}}clang-linker-wrapper" "--should-extract=amdgcnspirv" {{.*}} "--device-compiler=spirv64-amd-amdhsa=-use-spirv-backend" From d5778a7ff55688de0d6c87204fdd8d32ac1bdc99 Mon Sep 17 00:00:00 2001 From: Hristo Hristov Date: Thu, 27 Nov 2025 00:04:27 +0200 Subject: [PATCH 10/22] [libc++] Applied `[[nodiscard]]` to concurrency (partially) (#169463) `[[nodiscard]]` should be applied to functions where discarding the return value is most likely a correctness issue. - https://libcxx.llvm.org/CodingGuidelines.html#apply-nodiscard-where-relevant The following utilities have been annotated in this patch: - [x] `` - [x] `` - [x] `` - [x] `` - [x] `` - [x] `` N.B. Some classes don't provide all specified methods, which were not annotated. --- .../__condition_variable/condition_variable.h | 2 +- libcxx/include/__mutex/mutex.h | 4 +- libcxx/include/__thread/thread.h | 8 +- libcxx/include/barrier | 4 +- libcxx/include/latch | 6 +- libcxx/include/mutex | 21 +-- libcxx/include/semaphore | 8 +- .../test/libcxx/thread/nodiscard.verify.cpp | 144 ++++++++++++++++++ .../thread.jthread/nodiscard.verify.cpp | 29 ---- 9 files changed, 173 insertions(+), 53 deletions(-) create mode 100644 libcxx/test/libcxx/thread/nodiscard.verify.cpp delete mode 100644 libcxx/test/std/thread/thread.jthread/nodiscard.verify.cpp diff --git a/libcxx/include/__condition_variable/condition_variable.h b/libcxx/include/__condition_variable/condition_variable.h index 1e8edd5dcb009..b7151930e9226 100644 --- a/libcxx/include/__condition_variable/condition_variable.h +++ b/libcxx/include/__condition_variable/condition_variable.h @@ -170,7 +170,7 @@ class _LIBCPP_EXPORTED_FROM_ABI condition_variable { wait_for(unique_lock& __lk, const chrono::duration<_Rep, _Period>& __d, _Predicate __pred); typedef __libcpp_condvar_t* native_handle_type; - _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__cv_; } + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__cv_; } private: void diff --git a/libcxx/include/__mutex/mutex.h b/libcxx/include/__mutex/mutex.h index 68c8842b35eda..e9cedf8db1cca 100644 --- a/libcxx/include/__mutex/mutex.h +++ b/libcxx/include/__mutex/mutex.h @@ -37,11 +37,11 @@ class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_CAPABILITY("mutex") mutex { # endif _LIBCPP_ACQUIRE_CAPABILITY() void lock(); - _LIBCPP_TRY_ACQUIRE_CAPABILITY(true) bool try_lock() _NOEXCEPT; + [[__nodiscard__]] _LIBCPP_TRY_ACQUIRE_CAPABILITY(true) bool try_lock() _NOEXCEPT; _LIBCPP_RELEASE_CAPABILITY void unlock() _NOEXCEPT; typedef __libcpp_mutex_t* native_handle_type; - _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__m_; } + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__m_; } }; static_assert(is_nothrow_default_constructible::value, "the default constructor for std::mutex must be nothrow"); diff --git a/libcxx/include/__thread/thread.h b/libcxx/include/__thread/thread.h index a3b672bc0f0e7..561f092ddb7c0 100644 --- a/libcxx/include/__thread/thread.h +++ b/libcxx/include/__thread/thread.h @@ -242,13 +242,13 @@ class _LIBCPP_EXPORTED_FROM_ABI thread { _LIBCPP_HIDE_FROM_ABI void swap(thread& __t) _NOEXCEPT { std::swap(__t_, __t.__t_); } - _LIBCPP_HIDE_FROM_ABI bool joinable() const _NOEXCEPT { return !__libcpp_thread_isnull(&__t_); } + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool joinable() const _NOEXCEPT { return !__libcpp_thread_isnull(&__t_); } void join(); void detach(); - _LIBCPP_HIDE_FROM_ABI id get_id() const _NOEXCEPT { return __libcpp_thread_get_id(&__t_); } - _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() _NOEXCEPT { return __t_; } + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI id get_id() const _NOEXCEPT { return __libcpp_thread_get_id(&__t_); } + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() _NOEXCEPT { return __t_; } - static unsigned hardware_concurrency() _NOEXCEPT; + [[__nodiscard__]] static unsigned hardware_concurrency() _NOEXCEPT; }; inline _LIBCPP_HIDE_FROM_ABI void swap(thread& __x, thread& __y) _NOEXCEPT { __x.swap(__y); } diff --git a/libcxx/include/barrier b/libcxx/include/barrier index 41fbfb3e8fb7b..5f9b471f01741 100644 --- a/libcxx/include/barrier +++ b/libcxx/include/barrier @@ -158,7 +158,9 @@ class barrier { public: using arrival_token = typename __barrier_base<_CompletionF>::arrival_token; - static _LIBCPP_HIDE_FROM_ABI constexpr ptrdiff_t max() noexcept { return __barrier_base<_CompletionF>::max(); } + [[nodiscard]] static _LIBCPP_HIDE_FROM_ABI constexpr ptrdiff_t max() noexcept { + return __barrier_base<_CompletionF>::max(); + } _LIBCPP_HIDE_FROM_ABI explicit barrier(ptrdiff_t __count, _CompletionF __completion = _CompletionF()) : __b_(__count, std::move(__completion)) { diff --git a/libcxx/include/latch b/libcxx/include/latch index c3b8f62e9b50e..33268d9655f25 100644 --- a/libcxx/include/latch +++ b/libcxx/include/latch @@ -70,7 +70,9 @@ class latch { atomic __a_; public: - static _LIBCPP_HIDE_FROM_ABI constexpr ptrdiff_t max() noexcept { return numeric_limits::max(); } + [[nodiscard]] static _LIBCPP_HIDE_FROM_ABI constexpr ptrdiff_t max() noexcept { + return numeric_limits::max(); + } inline _LIBCPP_HIDE_FROM_ABI constexpr explicit latch(ptrdiff_t __expected) : __a_(__expected) { _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( @@ -97,7 +99,7 @@ public: if (__old == __update) __a_.notify_all(); } - inline _LIBCPP_HIDE_FROM_ABI bool try_wait() const noexcept { + [[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool try_wait() const noexcept { auto __value = __a_.load(memory_order_acquire); return try_wait_impl(__value); } diff --git a/libcxx/include/mutex b/libcxx/include/mutex index 0b81f1bb1c8a6..bec0185ede21a 100644 --- a/libcxx/include/mutex +++ b/libcxx/include/mutex @@ -229,12 +229,12 @@ public: recursive_mutex& operator=(const recursive_mutex&) = delete; void lock(); - bool try_lock() _NOEXCEPT; + [[__nodiscard__]] bool try_lock() _NOEXCEPT; void unlock() _NOEXCEPT; typedef __libcpp_recursive_mutex_t* native_handle_type; - _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__m_; } + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__m_; } }; class _LIBCPP_EXPORTED_FROM_ABI timed_mutex { @@ -251,14 +251,14 @@ public: public: void lock(); - bool try_lock() _NOEXCEPT; + [[__nodiscard__]] bool try_lock() _NOEXCEPT; template - _LIBCPP_HIDE_FROM_ABI bool try_lock_for(const chrono::duration<_Rep, _Period>& __d) { + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool try_lock_for(const chrono::duration<_Rep, _Period>& __d) { return try_lock_until(chrono::steady_clock::now() + __d); } template - _LIBCPP_HIDE_FROM_ABI bool try_lock_until(const chrono::time_point<_Clock, _Duration>& __t) { + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool try_lock_until(const chrono::time_point<_Clock, _Duration>& __t) { using namespace chrono; unique_lock __lk(__m_); bool __no_timeout = _Clock::now() < __t; @@ -288,14 +288,14 @@ public: recursive_timed_mutex& operator=(const recursive_timed_mutex&) = delete; void lock(); - bool try_lock() _NOEXCEPT; + [[__nodiscard__]] bool try_lock() _NOEXCEPT; template - _LIBCPP_HIDE_FROM_ABI bool try_lock_for(const chrono::duration<_Rep, _Period>& __d) { + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool try_lock_for(const chrono::duration<_Rep, _Period>& __d) { return try_lock_until(chrono::steady_clock::now() + __d); } template - _LIBCPP_HIDE_FROM_ABI bool try_lock_until(const chrono::time_point<_Clock, _Duration>& __t) { + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool try_lock_until(const chrono::time_point<_Clock, _Duration>& __t) { using namespace chrono; __thread_id __id = this_thread::get_id(); unique_lock __lk(__m_); @@ -320,7 +320,7 @@ public: }; template -_LIBCPP_NO_THREAD_SAFETY_ANALYSIS _LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, _L1& __l1) { +[[__nodiscard__]] _LIBCPP_NO_THREAD_SAFETY_ANALYSIS _LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, _L1& __l1) { unique_lock<_L0> __u0(__l0, try_to_lock_t()); if (__u0.owns_lock()) { if (__l1.try_lock()) { @@ -335,7 +335,8 @@ _LIBCPP_NO_THREAD_SAFETY_ANALYSIS _LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, # ifndef _LIBCPP_CXX03_LANG template -_LIBCPP_NO_THREAD_SAFETY_ANALYSIS _LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, _L1& __l1, _L2& __l2, _L3&... __l3) { +[[__nodiscard__]] _LIBCPP_NO_THREAD_SAFETY_ANALYSIS + _LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, _L1& __l1, _L2& __l2, _L3&... __l3) { int __r = 0; unique_lock<_L0> __u0(__l0, try_to_lock); if (__u0.owns_lock()) { diff --git a/libcxx/include/semaphore b/libcxx/include/semaphore index 99c4ad24b35ec..1f19d50e32af7 100644 --- a/libcxx/include/semaphore +++ b/libcxx/include/semaphore @@ -133,7 +133,7 @@ class counting_semaphore { public: static_assert(__least_max_value >= 0, "The least maximum value must be a positive number"); - static constexpr ptrdiff_t max() noexcept { return __least_max_value; } + [[nodiscard]] static constexpr ptrdiff_t max() noexcept { return __least_max_value; } _LIBCPP_HIDE_FROM_ABI constexpr explicit counting_semaphore(ptrdiff_t __count) : __semaphore_(__count) { _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( @@ -156,12 +156,12 @@ public: } _LIBCPP_HIDE_FROM_ABI void acquire() { __semaphore_.acquire(); } template - _LIBCPP_HIDE_FROM_ABI bool try_acquire_for(chrono::duration<_Rep, _Period> const& __rel_time) { + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool try_acquire_for(chrono::duration<_Rep, _Period> const& __rel_time) { return __semaphore_.try_acquire_for(chrono::duration_cast(__rel_time)); } - _LIBCPP_HIDE_FROM_ABI bool try_acquire() { return __semaphore_.try_acquire(); } + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool try_acquire() { return __semaphore_.try_acquire(); } template - _LIBCPP_HIDE_FROM_ABI bool try_acquire_until(chrono::time_point<_Clock, _Duration> const& __abs_time) { + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool try_acquire_until(chrono::time_point<_Clock, _Duration> const& __abs_time) { auto const __current = _Clock::now(); if (__current >= __abs_time) return try_acquire(); diff --git a/libcxx/test/libcxx/thread/nodiscard.verify.cpp b/libcxx/test/libcxx/thread/nodiscard.verify.cpp new file mode 100644 index 0000000000000..19e43f88db700 --- /dev/null +++ b/libcxx/test/libcxx/thread/nodiscard.verify.cpp @@ -0,0 +1,144 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03 +// UNSUPPORTED: no-threads + +// Check that functions are marked [[nodiscard]] + +#include +#include +#include +#include +#include +#include + +#include "test_macros.h" + +const auto timePoint = std::chrono::steady_clock::now(); + +void test() { + // Threads + { + std::thread th; + + th.joinable(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + th.get_id(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + th.native_handle(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + th.hardware_concurrency(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + } +#if TEST_STD_VER >= 20 + { + std::jthread jt; + + jt.joinable(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + jt.get_id(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + jt.native_handle(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + jt.get_stop_source(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + jt.get_stop_token(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + jt.hardware_concurrency(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + } +#endif + + // Mutual exclusion + + { // + std::mutex m; + + m.try_lock(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + m.native_handle(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + } + { + std::recursive_mutex m; + + m.try_lock(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + m.native_handle(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + } + { + std::timed_mutex m; + + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + m.try_lock(); + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + m.try_lock_for(std::chrono::nanoseconds{82}); + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + m.try_lock_until(timePoint); + } + { + std::recursive_timed_mutex m; + + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + m.try_lock(); + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + m.try_lock_for(std::chrono::nanoseconds{82}); + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + m.try_lock_until(timePoint); + } + { + std::mutex m1; + std::mutex m2; + std::mutex m3; + + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + std::try_lock(m1, m2); + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + std::try_lock(m1, m2, m3); + } + + // Condition variables + + { // + std::condition_variable cv; + + cv.native_handle(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + } + +#if TEST_STD_VER >= 20 + + // Semaphores + + { // + std::counting_semaphore<> cs{0}; + + cs.max(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + cs.try_acquire_for(std::chrono::nanoseconds{82}); + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + cs.try_acquire(); + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + cs.try_acquire_until(timePoint); + + std::binary_semaphore bs{0}; + + bs.max(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + bs.try_acquire_for(std::chrono::nanoseconds{82}); + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + bs.try_acquire(); + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + bs.try_acquire_until(timePoint); + } + + // Latches and barriers + + { // + std::barrier<> b{94}; + + b.max(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + } + { // + std::latch l{94}; + + l.max(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + l.try_wait(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + } + +#endif +} diff --git a/libcxx/test/std/thread/thread.jthread/nodiscard.verify.cpp b/libcxx/test/std/thread/thread.jthread/nodiscard.verify.cpp deleted file mode 100644 index 2ef5cf874da90..0000000000000 --- a/libcxx/test/std/thread/thread.jthread/nodiscard.verify.cpp +++ /dev/null @@ -1,29 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// UNSUPPORTED: no-threads -// UNSUPPORTED: c++03, c++11, c++14, c++17 - -// [[nodiscard]] bool joinable() const noexcept; -// [[nodiscard]] id get_id() const noexcept; -// [[nodiscard]] native_handle_type native_handle(); -// [[nodiscard]] stop_source get_stop_source() noexcept; -// [[nodiscard]] stop_token get_stop_token() const noexcept; -// [[nodiscard]] static unsigned int hardware_concurrency() noexcept; - -#include - -void test() { - std::jthread jt; - jt.joinable(); // expected-warning {{ignoring return value of function}} - jt.get_id(); // expected-warning {{ignoring return value of function}} - jt.native_handle(); // expected-warning {{ignoring return value of function}} - jt.get_stop_source(); // expected-warning {{ignoring return value of function}} - jt.get_stop_token(); // expected-warning {{ignoring return value of function}} - jt.hardware_concurrency(); // expected-warning {{ignoring return value of function}} -} From 216b9fa2275eb11fdb0133870ac81c8da7ff8fcf Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Wed, 26 Nov 2025 13:43:43 -0800 Subject: [PATCH 11/22] [SLP][NFC]Add another test with the user with multiple copyable operands, NFC --- .../X86/multi-node-user-with-copyable-ops.ll | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/multi-node-user-with-copyable-ops.ll diff --git a/llvm/test/Transforms/SLPVectorizer/X86/multi-node-user-with-copyable-ops.ll b/llvm/test/Transforms/SLPVectorizer/X86/multi-node-user-with-copyable-ops.ll new file mode 100644 index 0000000000000..eb3b183fd49eb --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/X86/multi-node-user-with-copyable-ops.ll @@ -0,0 +1,52 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s + +define i1 @test(double %circ_radius, ptr %x, double %0) { +; CHECK-LABEL: define i1 @test( +; CHECK-SAME: double [[CIRC_RADIUS:%.*]], ptr [[X:%.*]], double [[TMP0:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[TMP1:%.*]] = load double, ptr [[X]], align 8 +; CHECK-NEXT: [[ADD20:%.*]] = fadd double [[TMP1]], 0.000000e+00 +; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x double> poison, double [[TMP0]], i32 3 +; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[TMP1]], i32 0 +; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[ADD20]], i32 2 +; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[TMP4]], <4 x double> poison, <4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x double> [[TMP4]], <4 x double> , <4 x i32> +; CHECK-NEXT: [[TMP7:%.*]] = fmul <4 x double> [[TMP5]], [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> poison, <4 x i32> +; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x double> poison, double [[CIRC_RADIUS]], i32 0 +; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> poison, <4 x i32> +; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x double> [[TMP8]], <4 x double> [[TMP10]], <4 x i32> +; CHECK-NEXT: [[TMP12:%.*]] = fadd <4 x double> [[TMP7]], [[TMP11]] +; CHECK-NEXT: [[TMP13:%.*]] = call <4 x double> @llvm.sqrt.v4f64(<4 x double> [[TMP12]]) +; CHECK-NEXT: [[TMP14:%.*]] = fcmp olt <4 x double> [[TMP13]], splat (double 1.000000e+00) +; CHECK-NEXT: [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP14]]) +; CHECK-NEXT: ret i1 [[TMP15]] +; +entry: + %1 = load double, ptr %x, align 8 + %square = fmul double %1, 0.000000e+00 + %square105 = fmul double %1, %1 + %add = fadd double %square, %square105 + %sqrt116 = call double @llvm.sqrt.f64(double %add) + %add20 = fadd double %1, 0.000000e+00 + %square106 = fmul double %add20, 0.000000e+00 + %add25 = fadd double %square105, %square106 + %sqrt115 = call double @llvm.sqrt.f64(double %add25) + %square109 = fmul double %0, 0.000000e+00 + %add39 = fadd double %square106, %circ_radius + %sqrt114 = call double @llvm.sqrt.f64(double %add39) + %add50 = fadd double %square, %square109 + %sqrt = call double @llvm.sqrt.f64(double %add50) + %cmp52 = fcmp olt double %sqrt116, 1.000000e+00 + %cmp54 = fcmp olt double %sqrt115, 1.000000e+00 + %or.cond = or i1 %cmp52, %cmp54 + %cmp57 = fcmp olt double %sqrt114, 1.000000e+00 + %or.cond112 = or i1 %or.cond, %cmp57 + %cmp60 = fcmp olt double %sqrt, 1.000000e+00 + %or.cond113 = or i1 %or.cond112, %cmp60 + ret i1 %or.cond113 +} + +declare double @llvm.sqrt.f64(double) #0 + From 20d95c807092755114fdc8cc3dba49e3f6820eb2 Mon Sep 17 00:00:00 2001 From: Andy Kaylor Date: Wed, 26 Nov 2025 14:49:17 -0800 Subject: [PATCH 12/22] [CIR] Add undef handling to enable global lambdas (#169721) This change adds undef handling that was needed to enable global lambdas. There was no lambda-specific code needed, but the global lambda handling needed to initialize a global with an undef value. [CIR] Handle undef init of struct This adds handling for a case where Clang initializes a struct to undef with a constant copy. This required adding support for undef constants and lowering undef attributes to LLVM IR. --- clang/lib/CIR/Dialect/IR/CIRDialect.cpp | 6 ++++ .../CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp | 22 +++++++++---- clang/test/CIR/CodeGen/lambda.cpp | 33 +++++++++++++++++++ 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp index 6bf543cf794b7..f1bacff7fc691 100644 --- a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp +++ b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp @@ -330,6 +330,12 @@ static LogicalResult checkConstantTypes(mlir::Operation *op, mlir::Type opType, "zero expects struct, array, vector, or complex type"); } + if (mlir::isa(attrType)) { + if (!mlir::isa(opType)) + return success(); + return op->emitOpError("undef expects non-void type"); + } + if (mlir::isa(attrType)) { if (!mlir::isa(opType)) return op->emitOpError("result type (") diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp index cd923a15af132..0c34d87734c3e 100644 --- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp +++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp @@ -240,7 +240,7 @@ class CIRAttrToValue { .Case( + cir::UndefAttr, cir::VTableAttr, cir::ZeroAttr>( [&](auto attrT) { return visitCirAttr(attrT); }) .Default([&](auto attrT) { return mlir::Value(); }); } @@ -254,6 +254,7 @@ class CIRAttrToValue { mlir::Value visitCirAttr(cir::ConstVectorAttr attr); mlir::Value visitCirAttr(cir::GlobalViewAttr attr); mlir::Value visitCirAttr(cir::TypeInfoAttr attr); + mlir::Value visitCirAttr(cir::UndefAttr attr); mlir::Value visitCirAttr(cir::VTableAttr attr); mlir::Value visitCirAttr(cir::ZeroAttr attr); @@ -591,6 +592,13 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::TypeInfoAttr typeInfoAttr) { return result; } +/// UndefAttr visitor. +mlir::Value CIRAttrToValue::visitCirAttr(cir::UndefAttr undefAttr) { + mlir::Location loc = parentOp->getLoc(); + return mlir::LLVM::UndefOp::create( + rewriter, loc, converter->convertType(undefAttr.getType())); +} + // VTableAttr visitor. mlir::Value CIRAttrToValue::visitCirAttr(cir::VTableAttr vtableArr) { mlir::Type llvmTy = converter->convertType(vtableArr.getType()); @@ -2046,9 +2054,11 @@ CIRToLLVMGlobalOpLowering::matchAndRewriteRegionInitializedGlobal( cir::GlobalOp op, mlir::Attribute init, mlir::ConversionPatternRewriter &rewriter) const { // TODO: Generalize this handling when more types are needed here. - assert((isa(init))); + assert( + (isa( + init))); // TODO(cir): once LLVM's dialect has proper equivalent attributes this // should be updated. For now, we use a custom op to initialize globals @@ -2106,8 +2116,8 @@ mlir::LogicalResult CIRToLLVMGlobalOpLowering::matchAndRewrite( } else if (mlir::isa( - init.value())) { + cir::TypeInfoAttr, cir::UndefAttr, cir::VTableAttr, + cir::ZeroAttr>(init.value())) { // TODO(cir): once LLVM's dialect has proper equivalent attributes this // should be updated. For now, we use a custom op to initialize globals // to the appropriate value. diff --git a/clang/test/CIR/CodeGen/lambda.cpp b/clang/test/CIR/CodeGen/lambda.cpp index 91380b9bea296..1d06496a85530 100644 --- a/clang/test/CIR/CodeGen/lambda.cpp +++ b/clang/test/CIR/CodeGen/lambda.cpp @@ -8,6 +8,39 @@ // We declare anonymous record types to represent lambdas. Rather than trying to // to match the declarations, we establish variables for these when they are used. +auto global_lambda = [](){}; +void use_global_lambda() { + global_lambda(); +} + +// CIR: cir.global "private" internal dso_local @global_lambda = #cir.undef : ![[REC_LAM_GLOBAL_LAMBDA:.*]] {alignment = 1 : i64} +// CIR: cir.func lambda internal private dso_local @_ZNK3$_0clEv(%[[THIS_ARG:.*]]: !cir.ptr {{.*}}) +// CIR: %[[THIS:.*]] = cir.alloca !cir.ptr, !cir.ptr>, ["this", init] +// CIR: cir.store %[[THIS_ARG]], %[[THIS]] +// CIR: cir.load %[[THIS]] +// +// CIR: cir.func {{.*}} @_Z17use_global_lambdav() +// CIR: %[[LAMBDA:.*]] = cir.get_global @global_lambda : !cir.ptr +// CIR: cir.call @_ZNK3$_0clEv(%[[LAMBDA]]) : (!cir.ptr) -> () + +// LLVM: @global_lambda = internal global %[[REC_LAM_GLOBAL_LAMBDA:.*]] undef, align 1 +// LLVM: define internal void @"_ZNK3$_0clEv"(ptr %[[THIS_ARG:.*]]) +// LLVM: %[[THIS_ADDR:.*]] = alloca ptr +// LLVM: store ptr %[[THIS_ARG]], ptr %[[THIS_ADDR]] +// LLVM: %[[THIS:.*]] = load ptr, ptr %[[THIS_ADDR]] +// +// LLVM: define dso_local void @_Z17use_global_lambdav() +// LLVM: call void @"_ZNK3$_0clEv"(ptr @global_lambda) + +// OGCG: @global_lambda = internal global %[[REC_LAM_GLOBAL_LAMBDA:.*]] undef, align 1 +// OGCG: define dso_local void @_Z17use_global_lambdav() +// OGCG: call void @"_ZNK3$_0clEv"(ptr noundef nonnull align 1 dereferenceable(1) @global_lambda) +// +// OGCG: define internal void @"_ZNK3$_0clEv"(ptr {{.*}} %[[THIS_ARG:.*]]) +// OGCG: %[[THIS_ADDR:.*]] = alloca ptr +// OGCG: store ptr %[[THIS_ARG]], ptr %[[THIS_ADDR]] +// OGCG: %[[THIS:.*]] = load ptr, ptr %[[THIS_ADDR]] + void fn() { auto a = [](){}; a(); From dab44135df10d9e29a38f25e112a847020ee2831 Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Wed, 26 Nov 2025 15:00:23 -0800 Subject: [PATCH 13/22] Fix sanitizer failure introduced by #133537 --- llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index 721ea22c9eae4..8267414e78955 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -2478,6 +2478,7 @@ const MCExpr *AArch64AsmPrinter::emitPAuthRelocationAsIRelative( std::unique_ptr STI( TM.getTarget().createMCSubtargetInfo(TT, "", "")); assert(STI && "Unable to create subtarget info"); + this->STI = static_cast(&*STI); MCSymbol *Place = OutStreamer->getContext().createTempSymbol(); OutStreamer->emitLabel(Place); From 2bef14c1a9dc939598ef12999f05527a662e42fa Mon Sep 17 00:00:00 2001 From: Andy Kaylor Date: Wed, 26 Nov 2025 15:05:44 -0800 Subject: [PATCH 14/22] [CIR][NFC] Move builtin tests to their own directory (#169737) This moves all builtin-related CodeGen tests to a new directory, separate from the main clang/test/CIR/CodeGen directory. This will make it easier to run the basic CodeGen tests without running the builtin tests. This is specifically intended to move those tests which include `immintrin.h` or any of its variants, which take a very long time to compile with a debug build. --- clang/test/CIR/{CodeGen => CodeGenBuiltins}/X86/avx-builtins.c | 0 .../{CodeGen => CodeGenBuiltins}/X86/avx10_2_512bf16-builtins.c | 0 .../CIR/{CodeGen => CodeGenBuiltins}/X86/avx10_2bf16-builtins.c | 0 .../test/CIR/{CodeGen => CodeGenBuiltins}/X86/avx512bw-builtins.c | 0 .../test/CIR/{CodeGen => CodeGenBuiltins}/X86/avx512f-builtins.c | 0 .../CIR/{CodeGen => CodeGenBuiltins}/X86/avx512fp16-builtins.c | 0 clang/test/CIR/{CodeGen => CodeGenBuiltins}/X86/bmi-builtins.c | 0 clang/test/CIR/{CodeGen => CodeGenBuiltins}/X86/lzcnt-builtins.c | 0 clang/test/CIR/{CodeGen => CodeGenBuiltins}/X86/sse-builtins.c | 0 clang/test/CIR/{CodeGen => CodeGenBuiltins}/X86/sse2-builtins.c | 0 clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtin-fcmp-sse.c | 0 clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtin-isfpclass.c | 0 clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtin_bit.cpp | 0 clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtin_call.cpp | 0 clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtin_inline.c | 0 .../test/CIR/{CodeGen => CodeGenBuiltins}/builtin_new_delete.cpp | 0 clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtin_prefetch.c | 0 clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtin_printf.cpp | 0 .../test/CIR/{CodeGen => CodeGenBuiltins}/builtins-elementwise.c | 0 .../CIR/{CodeGen => CodeGenBuiltins}/builtins-floating-point.c | 0 clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtins-overflow.cpp | 0 clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtins.cpp | 0 22 files changed, 0 insertions(+), 0 deletions(-) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/X86/avx-builtins.c (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/X86/avx10_2_512bf16-builtins.c (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/X86/avx10_2bf16-builtins.c (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/X86/avx512bw-builtins.c (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/X86/avx512f-builtins.c (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/X86/avx512fp16-builtins.c (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/X86/bmi-builtins.c (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/X86/lzcnt-builtins.c (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/X86/sse-builtins.c (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/X86/sse2-builtins.c (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtin-fcmp-sse.c (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtin-isfpclass.c (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtin_bit.cpp (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtin_call.cpp (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtin_inline.c (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtin_new_delete.cpp (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtin_prefetch.c (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtin_printf.cpp (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtins-elementwise.c (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtins-floating-point.c (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtins-overflow.cpp (100%) rename clang/test/CIR/{CodeGen => CodeGenBuiltins}/builtins.cpp (100%) diff --git a/clang/test/CIR/CodeGen/X86/avx-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/avx-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/avx-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/avx-builtins.c diff --git a/clang/test/CIR/CodeGen/X86/avx10_2_512bf16-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/avx10_2_512bf16-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/avx10_2_512bf16-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/avx10_2_512bf16-builtins.c diff --git a/clang/test/CIR/CodeGen/X86/avx10_2bf16-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/avx10_2bf16-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/avx10_2bf16-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/avx10_2bf16-builtins.c diff --git a/clang/test/CIR/CodeGen/X86/avx512bw-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/avx512bw-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/avx512bw-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/avx512bw-builtins.c diff --git a/clang/test/CIR/CodeGen/X86/avx512f-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/avx512f-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/avx512f-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/avx512f-builtins.c diff --git a/clang/test/CIR/CodeGen/X86/avx512fp16-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/avx512fp16-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/avx512fp16-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/avx512fp16-builtins.c diff --git a/clang/test/CIR/CodeGen/X86/bmi-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/bmi-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/bmi-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/bmi-builtins.c diff --git a/clang/test/CIR/CodeGen/X86/lzcnt-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/lzcnt-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/lzcnt-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/lzcnt-builtins.c diff --git a/clang/test/CIR/CodeGen/X86/sse-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/sse-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/sse-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/sse-builtins.c diff --git a/clang/test/CIR/CodeGen/X86/sse2-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/sse2-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/sse2-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/sse2-builtins.c diff --git a/clang/test/CIR/CodeGen/builtin-fcmp-sse.c b/clang/test/CIR/CodeGenBuiltins/builtin-fcmp-sse.c similarity index 100% rename from clang/test/CIR/CodeGen/builtin-fcmp-sse.c rename to clang/test/CIR/CodeGenBuiltins/builtin-fcmp-sse.c diff --git a/clang/test/CIR/CodeGen/builtin-isfpclass.c b/clang/test/CIR/CodeGenBuiltins/builtin-isfpclass.c similarity index 100% rename from clang/test/CIR/CodeGen/builtin-isfpclass.c rename to clang/test/CIR/CodeGenBuiltins/builtin-isfpclass.c diff --git a/clang/test/CIR/CodeGen/builtin_bit.cpp b/clang/test/CIR/CodeGenBuiltins/builtin_bit.cpp similarity index 100% rename from clang/test/CIR/CodeGen/builtin_bit.cpp rename to clang/test/CIR/CodeGenBuiltins/builtin_bit.cpp diff --git a/clang/test/CIR/CodeGen/builtin_call.cpp b/clang/test/CIR/CodeGenBuiltins/builtin_call.cpp similarity index 100% rename from clang/test/CIR/CodeGen/builtin_call.cpp rename to clang/test/CIR/CodeGenBuiltins/builtin_call.cpp diff --git a/clang/test/CIR/CodeGen/builtin_inline.c b/clang/test/CIR/CodeGenBuiltins/builtin_inline.c similarity index 100% rename from clang/test/CIR/CodeGen/builtin_inline.c rename to clang/test/CIR/CodeGenBuiltins/builtin_inline.c diff --git a/clang/test/CIR/CodeGen/builtin_new_delete.cpp b/clang/test/CIR/CodeGenBuiltins/builtin_new_delete.cpp similarity index 100% rename from clang/test/CIR/CodeGen/builtin_new_delete.cpp rename to clang/test/CIR/CodeGenBuiltins/builtin_new_delete.cpp diff --git a/clang/test/CIR/CodeGen/builtin_prefetch.c b/clang/test/CIR/CodeGenBuiltins/builtin_prefetch.c similarity index 100% rename from clang/test/CIR/CodeGen/builtin_prefetch.c rename to clang/test/CIR/CodeGenBuiltins/builtin_prefetch.c diff --git a/clang/test/CIR/CodeGen/builtin_printf.cpp b/clang/test/CIR/CodeGenBuiltins/builtin_printf.cpp similarity index 100% rename from clang/test/CIR/CodeGen/builtin_printf.cpp rename to clang/test/CIR/CodeGenBuiltins/builtin_printf.cpp diff --git a/clang/test/CIR/CodeGen/builtins-elementwise.c b/clang/test/CIR/CodeGenBuiltins/builtins-elementwise.c similarity index 100% rename from clang/test/CIR/CodeGen/builtins-elementwise.c rename to clang/test/CIR/CodeGenBuiltins/builtins-elementwise.c diff --git a/clang/test/CIR/CodeGen/builtins-floating-point.c b/clang/test/CIR/CodeGenBuiltins/builtins-floating-point.c similarity index 100% rename from clang/test/CIR/CodeGen/builtins-floating-point.c rename to clang/test/CIR/CodeGenBuiltins/builtins-floating-point.c diff --git a/clang/test/CIR/CodeGen/builtins-overflow.cpp b/clang/test/CIR/CodeGenBuiltins/builtins-overflow.cpp similarity index 100% rename from clang/test/CIR/CodeGen/builtins-overflow.cpp rename to clang/test/CIR/CodeGenBuiltins/builtins-overflow.cpp diff --git a/clang/test/CIR/CodeGen/builtins.cpp b/clang/test/CIR/CodeGenBuiltins/builtins.cpp similarity index 100% rename from clang/test/CIR/CodeGen/builtins.cpp rename to clang/test/CIR/CodeGenBuiltins/builtins.cpp From 36bed4d0cd3e0e3871ad2360a7524da245e289d0 Mon Sep 17 00:00:00 2001 From: John Holdsworth Date: Thu, 27 Nov 2025 00:14:15 +0100 Subject: [PATCH 15/22] [lld][MachO] Follow-up to use madvise() for threaded file page-in. (#157917) Further to https://github.com/llvm/llvm-project/pull/147134#discussion_r2337246489, switch to use the madvise() api to page in mmap'd files and 1) All new code compiled in #if LLVM_ENABLE_THREADS is set so it can be seen where the changes were from this PR. 2) The new PR moves to use madvise() instead of the ad-hoc page referencing code I wrote which should avoid SIGSEGVs if the buffer is deallocated. 3) A new property SerialBackgroundQueue().stopAllWork to be used to stop background workers when there is no further call for them. Usually the background "page-in" threads have completed first but it seems with this troublesome test this is not always the case and buffers stored in the static input file cache are being deallocated while being referenced. --------- Co-authored-by: James Henderson --- lld/MachO/Driver.cpp | 46 +++++++++++++++++++++++++---------- lld/MachO/InputFiles.cpp | 3 ++- lld/test/MachO/read-workers.s | 3 --- llvm/lib/Object/Archive.cpp | 3 ++- 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp index 32b20993af67c..28c817c54c85d 100644 --- a/lld/MachO/Driver.cpp +++ b/lld/MachO/Driver.cpp @@ -41,6 +41,7 @@ #include "llvm/Object/Archive.h" #include "llvm/Option/ArgList.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Parallel.h" #include "llvm/Support/Path.h" @@ -53,6 +54,10 @@ #include "llvm/TextAPI/Architecture.h" #include "llvm/TextAPI/PackedVersion.h" +#if !_WIN32 +#include +#endif + using namespace llvm; using namespace llvm::MachO; using namespace llvm::object; @@ -292,12 +297,13 @@ struct DeferredFile { using DeferredFiles = std::vector; #if LLVM_ENABLE_THREADS -class SerialBackgroundQueue { +class SerialBackgroundWorkQueue { std::deque> queue; std::thread *running; std::mutex mutex; public: + std::atomic_bool stopAllWork = false; void queueWork(std::function work) { mutex.lock(); if (running && queue.empty()) { @@ -312,7 +318,7 @@ class SerialBackgroundQueue { queue.emplace_back(std::move(work)); if (!running) running = new std::thread([&]() { - while (true) { + while (!stopAllWork) { mutex.lock(); if (queue.empty()) { mutex.unlock(); @@ -331,6 +337,8 @@ class SerialBackgroundQueue { } }; +static SerialBackgroundWorkQueue pageInQueue; + // Most input files have been mapped but not yet paged in. // This code forces the page-ins on multiple threads so // the process is not stalled waiting on disk buffer i/o. @@ -339,8 +347,8 @@ void multiThreadedPageInBackground(DeferredFiles &deferred) { static const size_t largeArchive = 10 * 1024 * 1024; #ifndef NDEBUG using namespace std::chrono; - std::atomic_int numDeferedFilesTouched = 0; static std::atomic_uint64_t totalBytes = 0; + std::atomic_int numDeferedFilesAdvised = 0; auto t0 = high_resolution_clock::now(); #endif @@ -348,24 +356,34 @@ void multiThreadedPageInBackground(DeferredFiles &deferred) { const StringRef &buff = deferredFile.buffer.getBuffer(); if (buff.size() > largeArchive) return; + #ifndef NDEBUG totalBytes += buff.size(); - numDeferedFilesTouched += 1; + numDeferedFilesAdvised += 1; #endif - +#if _WIN32 // Reference all file's mmap'd pages to load them into memory. - for (const char *page = buff.data(), *end = page + buff.size(); page < end; - page += pageSize) { + for (const char *page = buff.data(), *end = page + buff.size(); + page < end && !pageInQueue.stopAllWork; page += pageSize) { [[maybe_unused]] volatile char t = *page; (void)t; } +#else +#define DEBUG_TYPE "lld-madvise" + auto aligned = + llvm::alignDown(reinterpret_cast(buff.data()), pageSize); + if (madvise((void *)aligned, buff.size(), MADV_WILLNEED) < 0) + LLVM_DEBUG(llvm::dbgs() << "madvise error: " << strerror(errno) << "\n"); +#undef DEBUG_TYPE +#endif }; + { // Create scope for waiting for the taskGroup std::atomic_size_t index = 0; llvm::parallel::TaskGroup taskGroup; for (int w = 0; w < config->readWorkers; w++) taskGroup.spawn([&index, &preloadDeferredFile, &deferred]() { - while (true) { + while (!pageInQueue.stopAllWork) { size_t localIndex = index.fetch_add(1); if (localIndex >= deferred.size()) break; @@ -373,17 +391,17 @@ void multiThreadedPageInBackground(DeferredFiles &deferred) { } }); } + #ifndef NDEBUG auto dt = high_resolution_clock::now() - t0; if (Process::GetEnv("LLD_MULTI_THREAD_PAGE")) llvm::dbgs() << "multiThreadedPageIn " << totalBytes << "/" - << numDeferedFilesTouched << "/" << deferred.size() << "/" + << numDeferedFilesAdvised << "/" << deferred.size() << "/" << duration_cast(dt).count() / 1000. << "\n"; #endif } static void multiThreadedPageIn(const DeferredFiles &deferred) { - static SerialBackgroundQueue pageInQueue; pageInQueue.queueWork([=]() { DeferredFiles files = deferred; multiThreadedPageInBackground(files); @@ -489,7 +507,7 @@ static InputFile *processFile(std::optional buffer, continue; } - if (archiveContents) + if (config->readWorkers && archiveContents) archiveContents->push_back({path, isLazy, *mb}); if (!hasObjCSection(*mb)) continue; @@ -1447,6 +1465,8 @@ static void createFiles(const InputArgList &args) { multiThreadedPageIn(archiveContents); for (auto *archive : archives) archive->addLazySymbols(); + + pageInQueue.stopAllWork = true; } #endif } @@ -1845,8 +1865,8 @@ bool link(ArrayRef argsArr, llvm::raw_ostream &stdoutOS, "'"); config->readWorkers = workers; #else - error(arg->getSpelling() + - ": option unavailable because lld was not built with thread support"); + warn(arg->getSpelling() + + ": option unavailable because lld was not built with thread support"); #endif } if (auto *arg = args.getLastArg(OPT_threads_eq)) { diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp index efcffc9c53383..81caef5f15ae1 100644 --- a/lld/MachO/InputFiles.cpp +++ b/lld/MachO/InputFiles.cpp @@ -217,7 +217,8 @@ std::optional macho::readFile(StringRef path) { if (entry != cachedReads.end()) return entry->second; - ErrorOr> mbOrErr = MemoryBuffer::getFile(path); + ErrorOr> mbOrErr = + MemoryBuffer::getFile(path, false, /*RequiresNullTerminator=*/false); if (std::error_code ec = mbOrErr.getError()) { error("cannot open " + path + ": " + ec.message()); return std::nullopt; diff --git a/lld/test/MachO/read-workers.s b/lld/test/MachO/read-workers.s index 294106ba0b084..4d2f88c2a757c 100644 --- a/lld/test/MachO/read-workers.s +++ b/lld/test/MachO/read-workers.s @@ -1,7 +1,4 @@ # REQUIRES: x86 && thread_support -## Sometimes fails, particularly in an ASAN build, do not run until -## https://github.com/llvm/llvm-project/pull/157917 addresses the cause. -# UNSUPPORTED: target={{.*}} # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o ## A non-negative integer is allowed. diff --git a/llvm/lib/Object/Archive.cpp b/llvm/lib/Object/Archive.cpp index 861c284253f7a..8e4a5ea5fc612 100644 --- a/llvm/lib/Object/Archive.cpp +++ b/llvm/lib/Object/Archive.cpp @@ -582,7 +582,8 @@ Expected Archive::Child::getBuffer() const { if (!FullNameOrErr) return FullNameOrErr.takeError(); const std::string &FullName = *FullNameOrErr; - ErrorOr> Buf = MemoryBuffer::getFile(FullName); + ErrorOr> Buf = + MemoryBuffer::getFile(FullName, false, /*RequiresNullTerminator=*/false); if (std::error_code EC = Buf.getError()) return errorCodeToError(EC); Parent->ThinBuffers.push_back(std::move(*Buf)); From 2f71e606c96c6b1a534b829fc10ff2d3ef497688 Mon Sep 17 00:00:00 2001 From: Ryan Mast <3969255+nightlark@users.noreply.github.com> Date: Wed, 26 Nov 2025 15:14:33 -0800 Subject: [PATCH 16/22] github-upload-release.py: add requirements and lock files for installing dependencies (#169461) Adds requirements.txt and lock files for installing dependencies for github-upload-release.py script. Signed-off-by: Ryan Mast --- .../utils/git/requirements_upload_release.txt | 326 ++++++++++++++++++ .../git/requirements_upload_release.txt.in | 10 + llvm/utils/release/github-upload-release.py | 3 + 3 files changed, 339 insertions(+) create mode 100644 llvm/utils/git/requirements_upload_release.txt create mode 100644 llvm/utils/git/requirements_upload_release.txt.in diff --git a/llvm/utils/git/requirements_upload_release.txt b/llvm/utils/git/requirements_upload_release.txt new file mode 100644 index 0000000000000..bc0ca450f7444 --- /dev/null +++ b/llvm/utils/git/requirements_upload_release.txt @@ -0,0 +1,326 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --generate-hashes --python-platform linux --python-version 3.10 -o requirements_upload_release.txt requirements_upload_release.txt.in +certifi==2025.11.12 \ + --hash=sha256:97de8790030bbd5c2d96b7ec782fc2f7820ef8dba6db909ccf95449f2d062d4b \ + --hash=sha256:d8ab5478f2ecd78af242878415affce761ca6bc54a22a27e026d7c25357c3316 + # via requests +cffi==2.0.0 \ + --hash=sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb \ + --hash=sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b \ + --hash=sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f \ + --hash=sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9 \ + --hash=sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44 \ + --hash=sha256:0f6084a0ea23d05d20c3edcda20c3d006f9b6f3fefeac38f59262e10cef47ee2 \ + --hash=sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c \ + --hash=sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75 \ + --hash=sha256:1cd13c99ce269b3ed80b417dcd591415d3372bcac067009b6e0f59c7d4015e65 \ + --hash=sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e \ + --hash=sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a \ + --hash=sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e \ + --hash=sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25 \ + --hash=sha256:2081580ebb843f759b9f617314a24ed5738c51d2aee65d31e02f6f7a2b97707a \ + --hash=sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe \ + --hash=sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b \ + --hash=sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91 \ + --hash=sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592 \ + --hash=sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187 \ + --hash=sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c \ + --hash=sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1 \ + --hash=sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94 \ + --hash=sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba \ + --hash=sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb \ + --hash=sha256:3f4d46d8b35698056ec29bca21546e1551a205058ae1a181d871e278b0b28165 \ + --hash=sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529 \ + --hash=sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca \ + --hash=sha256:4647afc2f90d1ddd33441e5b0e85b16b12ddec4fca55f0d9671fef036ecca27c \ + --hash=sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6 \ + --hash=sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c \ + --hash=sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0 \ + --hash=sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743 \ + --hash=sha256:61d028e90346df14fedc3d1e5441df818d095f3b87d286825dfcbd6459b7ef63 \ + --hash=sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5 \ + --hash=sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5 \ + --hash=sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4 \ + --hash=sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d \ + --hash=sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b \ + --hash=sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93 \ + --hash=sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205 \ + --hash=sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27 \ + --hash=sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512 \ + --hash=sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d \ + --hash=sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c \ + --hash=sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037 \ + --hash=sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26 \ + --hash=sha256:89472c9762729b5ae1ad974b777416bfda4ac5642423fa93bd57a09204712322 \ + --hash=sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb \ + --hash=sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c \ + --hash=sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8 \ + --hash=sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4 \ + --hash=sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414 \ + --hash=sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9 \ + --hash=sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664 \ + --hash=sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9 \ + --hash=sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775 \ + --hash=sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739 \ + --hash=sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc \ + --hash=sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062 \ + --hash=sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe \ + --hash=sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9 \ + --hash=sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92 \ + --hash=sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5 \ + --hash=sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13 \ + --hash=sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d \ + --hash=sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26 \ + --hash=sha256:cb527a79772e5ef98fb1d700678fe031e353e765d1ca2d409c92263c6d43e09f \ + --hash=sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495 \ + --hash=sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b \ + --hash=sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6 \ + --hash=sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c \ + --hash=sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef \ + --hash=sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5 \ + --hash=sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18 \ + --hash=sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad \ + --hash=sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3 \ + --hash=sha256:de8dad4425a6ca6e4e5e297b27b5c824ecc7581910bf9aee86cb6835e6812aa7 \ + --hash=sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5 \ + --hash=sha256:e6e73b9e02893c764e7e8d5bb5ce277f1a009cd5243f8228f75f842bf937c534 \ + --hash=sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49 \ + --hash=sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2 \ + --hash=sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5 \ + --hash=sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453 \ + --hash=sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf + # via + # cryptography + # pynacl +charset-normalizer==3.4.4 \ + --hash=sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad \ + --hash=sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93 \ + --hash=sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394 \ + --hash=sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89 \ + --hash=sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc \ + --hash=sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86 \ + --hash=sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63 \ + --hash=sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d \ + --hash=sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f \ + --hash=sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8 \ + --hash=sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0 \ + --hash=sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505 \ + --hash=sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161 \ + --hash=sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af \ + --hash=sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152 \ + --hash=sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318 \ + --hash=sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72 \ + --hash=sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4 \ + --hash=sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e \ + --hash=sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3 \ + --hash=sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576 \ + --hash=sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c \ + --hash=sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1 \ + --hash=sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8 \ + --hash=sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1 \ + --hash=sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2 \ + --hash=sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44 \ + --hash=sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26 \ + --hash=sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88 \ + --hash=sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016 \ + --hash=sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede \ + --hash=sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf \ + --hash=sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a \ + --hash=sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc \ + --hash=sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0 \ + --hash=sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84 \ + --hash=sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db \ + --hash=sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1 \ + --hash=sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7 \ + --hash=sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed \ + --hash=sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8 \ + --hash=sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133 \ + --hash=sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e \ + --hash=sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef \ + --hash=sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14 \ + --hash=sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2 \ + --hash=sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0 \ + --hash=sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d \ + --hash=sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828 \ + --hash=sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f \ + --hash=sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf \ + --hash=sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6 \ + --hash=sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328 \ + --hash=sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090 \ + --hash=sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa \ + --hash=sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381 \ + --hash=sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c \ + --hash=sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb \ + --hash=sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc \ + --hash=sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a \ + --hash=sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec \ + --hash=sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc \ + --hash=sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac \ + --hash=sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e \ + --hash=sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313 \ + --hash=sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569 \ + --hash=sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3 \ + --hash=sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d \ + --hash=sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525 \ + --hash=sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894 \ + --hash=sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3 \ + --hash=sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9 \ + --hash=sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a \ + --hash=sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9 \ + --hash=sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14 \ + --hash=sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25 \ + --hash=sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50 \ + --hash=sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf \ + --hash=sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1 \ + --hash=sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3 \ + --hash=sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac \ + --hash=sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e \ + --hash=sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815 \ + --hash=sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c \ + --hash=sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6 \ + --hash=sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6 \ + --hash=sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e \ + --hash=sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4 \ + --hash=sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84 \ + --hash=sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69 \ + --hash=sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15 \ + --hash=sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191 \ + --hash=sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0 \ + --hash=sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897 \ + --hash=sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd \ + --hash=sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2 \ + --hash=sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794 \ + --hash=sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d \ + --hash=sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074 \ + --hash=sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3 \ + --hash=sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224 \ + --hash=sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838 \ + --hash=sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a \ + --hash=sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d \ + --hash=sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d \ + --hash=sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f \ + --hash=sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8 \ + --hash=sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490 \ + --hash=sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966 \ + --hash=sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9 \ + --hash=sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3 \ + --hash=sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e \ + --hash=sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608 + # via requests +cryptography==46.0.3 \ + --hash=sha256:00a5e7e87938e5ff9ff5447ab086a5706a957137e6e433841e9d24f38a065217 \ + --hash=sha256:01ca9ff2885f3acc98c29f1860552e37f6d7c7d013d7334ff2a9de43a449315d \ + --hash=sha256:09859af8466b69bc3c27bdf4f5d84a665e0f7ab5088412e9e2ec49758eca5cbc \ + --hash=sha256:0abf1ffd6e57c67e92af68330d05760b7b7efb243aab8377e583284dbab72c71 \ + --hash=sha256:1000713389b75c449a6e979ffc7dcc8ac90b437048766cef052d4d30b8220971 \ + --hash=sha256:109d4ddfadf17e8e7779c39f9b18111a09efb969a301a31e987416a0191ed93a \ + --hash=sha256:10b01676fc208c3e6feeb25a8b83d81767e8059e1fe86e1dc62d10a3018fa926 \ + --hash=sha256:10ca84c4668d066a9878890047f03546f3ae0a6b8b39b697457b7757aaf18dbc \ + --hash=sha256:15ab9b093e8f09daab0f2159bb7e47532596075139dd74365da52ecc9cb46c5d \ + --hash=sha256:191bb60a7be5e6f54e30ba16fdfae78ad3a342a0599eb4193ba88e3f3d6e185b \ + --hash=sha256:22d7e97932f511d6b0b04f2bfd818d73dcd5928db509460aaf48384778eb6d20 \ + --hash=sha256:23b1a8f26e43f47ceb6d6a43115f33a5a37d57df4ea0ca295b780ae8546e8044 \ + --hash=sha256:36e627112085bb3b81b19fed209c05ce2a52ee8b15d161b7c643a7d5a88491f3 \ + --hash=sha256:39b6755623145ad5eff1dab323f4eae2a32a77a7abef2c5089a04a3d04366715 \ + --hash=sha256:3b51b8ca4f1c6453d8829e1eb7299499ca7f313900dd4d89a24b8b87c0a780d4 \ + --hash=sha256:402b58fc32614f00980b66d6e56a5b4118e6cb362ae8f3fda141ba4689bd4506 \ + --hash=sha256:416260257577718c05135c55958b674000baef9a1c7d9e8f306ec60d71db850f \ + --hash=sha256:46acf53b40ea38f9c6c229599a4a13f0d46a6c3fa9ef19fc1a124d62e338dfa0 \ + --hash=sha256:4b7387121ac7d15e550f5cb4a43aef2559ed759c35df7336c402bb8275ac9683 \ + --hash=sha256:50fc3343ac490c6b08c0cf0d704e881d0d660be923fd3076db3e932007e726e3 \ + --hash=sha256:516ea134e703e9fe26bcd1277a4b59ad30586ea90c365a87781d7887a646fe21 \ + --hash=sha256:549e234ff32571b1f4076ac269fcce7a808d3bf98b76c8dd560e42dbc66d7d91 \ + --hash=sha256:5d7f93296ee28f68447397bf5198428c9aeeab45705a55d53a6343455dcb2c3c \ + --hash=sha256:5ecfccd2329e37e9b7112a888e76d9feca2347f12f37918facbb893d7bb88ee8 \ + --hash=sha256:6276eb85ef938dc035d59b87c8a7dc559a232f954962520137529d77b18ff1df \ + --hash=sha256:6b5063083824e5509fdba180721d55909ffacccc8adbec85268b48439423d78c \ + --hash=sha256:6eae65d4c3d33da080cff9c4ab1f711b15c1d9760809dad6ea763f3812d254cb \ + --hash=sha256:6f61efb26e76c45c4a227835ddeae96d83624fb0d29eb5df5b96e14ed1a0afb7 \ + --hash=sha256:71e842ec9bc7abf543b47cf86b9a743baa95f4677d22baa4c7d5c69e49e9bc04 \ + --hash=sha256:760f83faa07f8b64e9c33fc963d790a2edb24efb479e3520c14a45741cd9b2db \ + --hash=sha256:78a97cf6a8839a48c49271cdcbd5cf37ca2c1d6b7fdd86cc864f302b5e9bf459 \ + --hash=sha256:7ce938a99998ed3c8aa7e7272dca1a610401ede816d36d0693907d863b10d9ea \ + --hash=sha256:8a6e050cb6164d3f830453754094c086ff2d0b2f3a897a1d9820f6139a1f0914 \ + --hash=sha256:9394673a9f4de09e28b5356e7fff97d778f8abad85c9d5ac4a4b7e25a0de7717 \ + --hash=sha256:94cd0549accc38d1494e1f8de71eca837d0509d0d44bf11d158524b0e12cebf9 \ + --hash=sha256:a04bee9ab6a4da801eb9b51f1b708a1b5b5c9eb48c03f74198464c66f0d344ac \ + --hash=sha256:a23582810fedb8c0bc47524558fb6c56aac3fc252cb306072fd2815da2a47c32 \ + --hash=sha256:a2c0cd47381a3229c403062f764160d57d4d175e022c1df84e168c6251a22eec \ + --hash=sha256:a8b17438104fed022ce745b362294d9ce35b4c2e45c1d958ad4a4b019285f4a1 \ + --hash=sha256:a9a3008438615669153eb86b26b61e09993921ebdd75385ddd748702c5adfddb \ + --hash=sha256:b02cf04496f6576afffef5ddd04a0cb7d49cf6be16a9059d793a30b035f6b6ac \ + --hash=sha256:b419ae593c86b87014b9be7396b385491ad7f320bde96826d0dd174459e54665 \ + --hash=sha256:c0a7bb1a68a5d3471880e264621346c48665b3bf1c3759d682fc0864c540bd9e \ + --hash=sha256:c70cc23f12726be8f8bc72e41d5065d77e4515efae3690326764ea1b07845cfb \ + --hash=sha256:c8daeb2d2174beb4575b77482320303f3d39b8e81153da4f0fb08eb5fe86a6c5 \ + --hash=sha256:cb3d760a6117f621261d662bccc8ef5bc32ca673e037c83fbe565324f5c46936 \ + --hash=sha256:d55f3dffadd674514ad19451161118fd010988540cee43d8bc20675e775925de \ + --hash=sha256:d89c3468de4cdc4f08a57e214384d0471911a3830fcdaf7a8cc587e42a866372 \ + --hash=sha256:db391fa7c66df6762ee3f00c95a89e6d428f4d60e7abc8328f4fe155b5ac6e54 \ + --hash=sha256:dfb781ff7eaa91a6f7fd41776ec37c5853c795d3b358d4896fdbb5df168af422 \ + --hash=sha256:e5bf0ed4490068a2e72ac03d786693adeb909981cc596425d09032d372bcc849 \ + --hash=sha256:e7aec276d68421f9574040c26e2a7c3771060bc0cff408bae1dcb19d3ab1e63c \ + --hash=sha256:ef639cb3372f69ec44915fafcd6698b6cc78fbe0c2ea41be867f6ed612811963 \ + --hash=sha256:f260d0d41e9b4da1ed1e0f1ce571f97fe370b152ab18778e9e8f67d6af432018 + # via pyjwt +idna==3.11 \ + --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ + --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 + # via requests +pycparser==2.23 \ + --hash=sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2 \ + --hash=sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934 + # via cffi +pygithub==2.8.1 \ + --hash=sha256:23a0a5bca93baef082e03411bf0ce27204c32be8bfa7abc92fe4a3e132936df0 \ + --hash=sha256:341b7c78521cb07324ff670afd1baa2bf5c286f8d9fd302c1798ba594a5400c9 + # via -r requirements_upload_release.txt.in +pyjwt==2.9.0 \ + --hash=sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850 \ + --hash=sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c + # via pygithub +pynacl==1.6.1 \ + --hash=sha256:262a8de6bba4aee8a66f5edf62c214b06647461c9b6b641f8cd0cb1e3b3196fe \ + --hash=sha256:2b12f1b97346f177affcdfdc78875ff42637cb40dcf79484a97dae3448083a78 \ + --hash=sha256:319de653ef84c4f04e045eb250e6101d23132372b0a61a7acf91bac0fda8e58c \ + --hash=sha256:3206fa98737fdc66d59b8782cecc3d37d30aeec4593d1c8c145825a345bba0f0 \ + --hash=sha256:3384a454adf5d716a9fadcb5eb2e3e72cd49302d1374a60edc531c9957a9b014 \ + --hash=sha256:3cd787ec1f5c155dc8ecf39b1333cfef41415dc96d392f1ce288b4fe970df489 \ + --hash=sha256:4ce50d19f1566c391fedc8dc2f2f5be265ae214112ebe55315e41d1f36a7f0a9 \ + --hash=sha256:53543b4f3d8acb344f75fd4d49f75e6572fce139f4bfb4815a9282296ff9f4c0 \ + --hash=sha256:543f869140f67d42b9b8d47f922552d7a967e6c116aad028c9bfc5f3f3b3a7b7 \ + --hash=sha256:5953e8b8cfadb10889a6e7bd0f53041a745d1b3d30111386a1bb37af171e6daf \ + --hash=sha256:5a3becafc1ee2e5ea7f9abc642f56b82dcf5be69b961e782a96ea52b55d8a9fc \ + --hash=sha256:5f5b35c1a266f8a9ad22525049280a600b19edd1f785bccd01ae838437dcf935 \ + --hash=sha256:6b35d93ab2df03ecb3aa506be0d3c73609a51449ae0855c2e89c7ed44abde40b \ + --hash=sha256:7713f8977b5d25f54a811ec9efa2738ac592e846dd6e8a4d3f7578346a841078 \ + --hash=sha256:7d7c09749450c385301a3c20dca967a525152ae4608c0a096fe8464bfc3df93d \ + --hash=sha256:8d361dac0309f2b6ad33b349a56cd163c98430d409fa503b10b70b3ad66eaa1d \ + --hash=sha256:9fd1a4eb03caf8a2fe27b515a998d26923adb9ddb68db78e35ca2875a3830dde \ + --hash=sha256:a2bb472458c7ca959aeeff8401b8efef329b0fc44a89d3775cffe8fad3398ad8 \ + --hash=sha256:a569a4069a7855f963940040f35e87d8bc084cb2d6347428d5ad20550a0a1a21 \ + --hash=sha256:a6f9fd6d6639b1e81115c7f8ff16b8dedba1e8098d2756275d63d208b0e32021 \ + --hash=sha256:c2228054f04bf32d558fb89bb99f163a8197d5a9bf4efa13069a7fa8d4b93fc3 \ + --hash=sha256:d8615ee34d01c8e0ab3f302dcdd7b32e2bcf698ba5f4809e7cc407c8cdea7717 \ + --hash=sha256:d984c91fe3494793b2a1fb1e91429539c6c28e9ec8209d26d25041ec599ccf63 \ + --hash=sha256:dece79aecbb8f4640a1adbb81e4aa3bfb0e98e99834884a80eb3f33c7c30e708 \ + --hash=sha256:e49a3f3d0da9f79c1bec2aa013261ab9fa651c7da045d376bd306cf7c1792993 \ + --hash=sha256:e735c3a1bdfde3834503baf1a6d74d4a143920281cb724ba29fb84c9f49b9c48 \ + --hash=sha256:fc734c1696ffd49b40f7c1779c89ba908157c57345cf626be2e0719488a076d3 + # via pygithub +requests==2.32.4 \ + --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c \ + --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422 + # via pygithub +typing-extensions==4.13.2 \ + --hash=sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c \ + --hash=sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef + # via + # cryptography + # pygithub +urllib3==2.2.3 \ + --hash=sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac \ + --hash=sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9 + # via + # pygithub + # requests diff --git a/llvm/utils/git/requirements_upload_release.txt.in b/llvm/utils/git/requirements_upload_release.txt.in new file mode 100644 index 0000000000000..d795ecd4f680e --- /dev/null +++ b/llvm/utils/git/requirements_upload_release.txt.in @@ -0,0 +1,10 @@ +# Convert this file into a requirements.txt file by running: +# +# pip install pip-tools +# pip-compile --generate-hashes -o requirements_upload_release.txt requirements_upload_release.txt.in +# +# or with uv by running: +# +# uv pip compile --generate-hashes --python-platform linux --python-version 3.10 -o requirements_upload_release.txt requirements_upload_release.txt.in + +PyGithub==2.8.1 diff --git a/llvm/utils/release/github-upload-release.py b/llvm/utils/release/github-upload-release.py index d58bb544e17dd..893cda0484ac8 100755 --- a/llvm/utils/release/github-upload-release.py +++ b/llvm/utils/release/github-upload-release.py @@ -10,6 +10,9 @@ # Create and manage releases in the llvm github project. # # This script requires python3 and the PyGithub module. + +# Requirements and lockfiles can be found in llvm/utils/git/requirements.upload_release.txt.in +# and llvm/utils/git/requirements.upload_release.txt # # Example Usage: # From 49516ba0e3a64bafc523c5f03594f607a0cf24ca Mon Sep 17 00:00:00 2001 From: gulfemsavrun Date: Wed, 26 Nov 2025 16:38:47 -0800 Subject: [PATCH 17/22] [llvm-objdump] Optimize live element tracking (#158763) This patch significantly optimizes the LiveElementPrinter by replacing a slow linear search with efficient hash map lookups. It refactors the code to use a map-based system for tracking live element addresses and managing column assignments, leading to a major performance improvement for large binaries. --- llvm/tools/llvm-objdump/SourcePrinter.cpp | 259 +++++++++++++++++----- llvm/tools/llvm-objdump/SourcePrinter.h | 53 ++++- llvm/tools/llvm-objdump/llvm-objdump.cpp | 10 +- 3 files changed, 251 insertions(+), 71 deletions(-) diff --git a/llvm/tools/llvm-objdump/SourcePrinter.cpp b/llvm/tools/llvm-objdump/SourcePrinter.cpp index b0ff89da97123..a4891a3620ee9 100644 --- a/llvm/tools/llvm-objdump/SourcePrinter.cpp +++ b/llvm/tools/llvm-objdump/SourcePrinter.cpp @@ -50,11 +50,6 @@ void InlinedFunction::dump(raw_ostream &OS) const { void InlinedFunction::printElementLine(raw_ostream &OS, object::SectionedAddress Addr, bool IsEnd) const { - bool LiveIn = !IsEnd && Range.LowPC == Addr.Address; - bool LiveOut = IsEnd && Range.HighPC == Addr.Address; - if (!(LiveIn || LiveOut)) - return; - uint32_t CallFile, CallLine, CallColumn, CallDiscriminator; InlinedFuncDie.getCallerFrame(CallFile, CallLine, CallColumn, CallDiscriminator); @@ -126,8 +121,41 @@ void LiveElementPrinter::addInlinedFunction(DWARFDie FuncDie, DWARFUnit *U = InlinedFuncDie.getDwarfUnit(); const char *InlinedFuncName = InlinedFuncDie.getName(DINameKind::LinkageName); DWARFAddressRange Range{FuncLowPC, FuncHighPC, SectionIndex}; + // Add the new element to the main vector. LiveElements.emplace_back(std::make_unique( InlinedFuncName, U, FuncDie, InlinedFuncDie, Range)); + + LiveElement *LE = LiveElements.back().get(); + // Map the element's low address (LowPC) to its pointer for fast range start + // lookup. + LiveElementsByAddress[FuncLowPC].push_back(LE); + // Map the element's high address (HighPC) to its pointer for fast range end + // lookup. + LiveElementsByEndAddress[FuncHighPC].push_back(LE); + // Map the pointer to its DWARF discovery index for deterministic + // ordering. + ElementPtrToIndex[LE] = LiveElements.size() - 1; +} + +/// Registers the most recently added LiveVariable into all data structures. +void LiveElementPrinter::registerNewVariable() { + assert( + !LiveElements.empty() && + "registerNewVariable called before element was added to LiveElements."); + LiveVariable *CurrentVar = + static_cast(LiveElements.back().get()); + assert(ElementPtrToIndex.count(CurrentVar) == 0 && + "Element already registered!"); + + // Map from a LiveElement pointer to its index in the LiveElements. + ElementPtrToIndex[CurrentVar] = LiveElements.size() - 1; + + if (const std::optional &Range = + CurrentVar->getLocExpr().Range) { + // Add the variable to address-based maps. + LiveElementsByAddress[Range->LowPC].push_back(CurrentVar); + LiveElementsByEndAddress[Range->HighPC].push_back(CurrentVar); + } } void LiveElementPrinter::addVariable(DWARFDie FuncDie, DWARFDie VarDie) { @@ -160,6 +188,9 @@ void LiveElementPrinter::addVariable(DWARFDie FuncDie, DWARFDie VarDie) { LiveElements.emplace_back( std::make_unique(WholeFuncExpr, VarName, U, FuncDie)); } + + // Register the new variable with all data structures. + registerNewVariable(); } } @@ -205,14 +236,52 @@ unsigned LiveElementPrinter::moveToFirstVarColumn(formatted_raw_ostream &OS) { return FirstUnprintedLogicalColumn; } -unsigned LiveElementPrinter::findFreeColumn() { - for (unsigned ColIdx = 0; ColIdx < ActiveCols.size(); ++ColIdx) - if (!ActiveCols[ColIdx].isActive()) - return ColIdx; +unsigned LiveElementPrinter::getOrCreateColumn(unsigned ElementIdx) { + // Check if the element already has an assigned column. + auto it = ElementToColumn.find(ElementIdx); + if (it != ElementToColumn.end()) + return it->second; + + unsigned ColIdx; + if (!FreeCols.empty()) { + // Get the smallest available index from the set. + ColIdx = *FreeCols.begin(); + // Remove the index from the set. + FreeCols.erase(FreeCols.begin()); + } else { + // No free columns, so create a new one. + ColIdx = ActiveCols.size(); + ActiveCols.emplace_back(); + } - size_t OldSize = ActiveCols.size(); - ActiveCols.grow(std::max(OldSize * 2, 1)); - return OldSize; + // Assign the element to the column and update the map. + ElementToColumn[ElementIdx] = ColIdx; + ActiveCols[ColIdx].ElementIdx = ElementIdx; + return ColIdx; +} + +void LiveElementPrinter::freeColumn(unsigned ColIdx) { + unsigned ElementIdx = ActiveCols[ColIdx].ElementIdx; + + // Clear the column's data. + ActiveCols[ColIdx].clear(); + + // Remove the element's entry from the map and add the column to the free + // list. + ElementToColumn.erase(ElementIdx); + FreeCols.insert(ColIdx); +} + +std::vector +LiveElementPrinter::getSortedActiveElementIndices() const { + // Get all element indices that currently have an assigned column. + std::vector Indices; + for (const auto &Pair : ElementToColumn) + Indices.push_back(Pair.first); + + // Sort by the DWARF discovery order. + llvm::stable_sort(Indices); + return Indices; } void LiveElementPrinter::dump() const { @@ -239,57 +308,112 @@ void LiveElementPrinter::addCompileUnit(DWARFDie D) { void LiveElementPrinter::update(object::SectionedAddress ThisAddr, object::SectionedAddress NextAddr, bool IncludeDefinedVars) { - // Do not create live ranges when debug-inlined-funcs option is provided with - // line format option. + // Exit early if only printing function limits. if (DbgInlinedFunctions == DFLimitsOnly) return; - // First, check variables which have already been assigned a column, so - // that we don't change their order. - SmallSet CheckedElementIdxs; + // Free columns identified in the previous cycle. + for (unsigned ColIdx : ColumnsToFreeNextCycle) + freeColumn(ColIdx); + ColumnsToFreeNextCycle.clear(); + + // Update status of active columns and collect those to free next cycle. for (unsigned ColIdx = 0, End = ActiveCols.size(); ColIdx < End; ++ColIdx) { if (!ActiveCols[ColIdx].isActive()) continue; - CheckedElementIdxs.insert(ActiveCols[ColIdx].ElementIdx); const std::unique_ptr &LE = LiveElements[ActiveCols[ColIdx].ElementIdx]; ActiveCols[ColIdx].LiveIn = LE->liveAtAddress(ThisAddr); ActiveCols[ColIdx].LiveOut = LE->liveAtAddress(NextAddr); - std::string Name = Demangle ? demangle(LE->getName()) : LE->getName(); - LLVM_DEBUG(dbgs() << "pass 1, " << ThisAddr.Address << "-" - << NextAddr.Address << ", " << Name << ", Col " << ColIdx - << ": LiveIn=" << ActiveCols[ColIdx].LiveIn - << ", LiveOut=" << ActiveCols[ColIdx].LiveOut << "\n"); - if (!ActiveCols[ColIdx].LiveIn && !ActiveCols[ColIdx].LiveOut) + LLVM_DEBUG({ + std::string Name = Demangle ? demangle(LE->getName()) : LE->getName(); + dbgs() << "pass 1, " << ThisAddr.Address << "-" << NextAddr.Address + << ", " << Name << ", Col " << ColIdx + << ": LiveIn=" << ActiveCols[ColIdx].LiveIn + << ", LiveOut=" << ActiveCols[ColIdx].LiveOut << "\n"; + }); + + // If element is fully dead, deactivate column immediately. + if (!ActiveCols[ColIdx].LiveIn && !ActiveCols[ColIdx].LiveOut) { ActiveCols[ColIdx].ElementIdx = Column::NullElementIdx; + continue; + } + + // Mark for cleanup in the next cycle if range ends here. + if (ActiveCols[ColIdx].LiveIn && !ActiveCols[ColIdx].LiveOut) + ColumnsToFreeNextCycle.push_back(ColIdx); } // Next, look for variables which don't already have a column, but which - // are now live. + // are now live (those starting at ThisAddr or NextAddr). if (IncludeDefinedVars) { - for (unsigned ElementIdx = 0, End = LiveElements.size(); ElementIdx < End; - ++ElementIdx) { - if (CheckedElementIdxs.count(ElementIdx)) + // Collect all elements starting at ThisAddr and NextAddr. + std::vector> NewLiveElements; + auto CollectNewElements = [&](const auto &It) { + if (It == LiveElementsByAddress.end()) + return; + + const std::vector &ElementList = It->second; + for (LiveElement *LE : ElementList) { + auto IndexIt = ElementPtrToIndex.find(LE); + assert(IndexIt != ElementPtrToIndex.end() && + "LiveElement in address map but missing from index map!"); + + // Get the element index for sorting and column management. + unsigned ElementIdx = IndexIt->second; + // Skip elements that already have a column. + if (ElementToColumn.count(ElementIdx)) + continue; + + bool LiveIn = LE->liveAtAddress(ThisAddr); + bool LiveOut = LE->liveAtAddress(NextAddr); + if (!LiveIn && !LiveOut) + continue; + + NewLiveElements.emplace_back(ElementIdx, LE); + } + }; + + // Collect elements starting at ThisAddr. + CollectNewElements(LiveElementsByAddress.find(ThisAddr.Address)); + // Collect elements starting at NextAddr (the address immediately + // following the instruction). + CollectNewElements(LiveElementsByAddress.find(NextAddr.Address)); + // Sort elements by DWARF discovery order for deterministic column + // assignment. + llvm::stable_sort(NewLiveElements, [](const auto &A, const auto &B) { + return A.first < B.first; + }); + + // Assign columns in deterministic order. + for (const auto &ElementPair : NewLiveElements) { + unsigned ElementIdx = ElementPair.first; + // Skip if element was already added from the first range. + if (ElementToColumn.count(ElementIdx)) continue; - const std::unique_ptr &LE = LiveElements[ElementIdx]; + LiveElement *LE = ElementPair.second; bool LiveIn = LE->liveAtAddress(ThisAddr); bool LiveOut = LE->liveAtAddress(NextAddr); - if (!LiveIn && !LiveOut) - continue; - unsigned ColIdx = findFreeColumn(); - std::string Name = Demangle ? demangle(LE->getName()) : LE->getName(); - LLVM_DEBUG(dbgs() << "pass 2, " << ThisAddr.Address << "-" - << NextAddr.Address << ", " << Name << ", Col " - << ColIdx << ": LiveIn=" << LiveIn - << ", LiveOut=" << LiveOut << "\n"); - ActiveCols[ColIdx].ElementIdx = ElementIdx; + // Assign or create a column. + unsigned ColIdx = getOrCreateColumn(ElementIdx); + LLVM_DEBUG({ + std::string Name = Demangle ? demangle(LE->getName()) : LE->getName(); + dbgs() << "pass 2, " << ThisAddr.Address << "-" << NextAddr.Address + << ", " << Name << ", Col " << ColIdx << ": LiveIn=" << LiveIn + << ", LiveOut=" << LiveOut << "\n"; + }); + ActiveCols[ColIdx].LiveIn = LiveIn; ActiveCols[ColIdx].LiveOut = LiveOut; ActiveCols[ColIdx].MustDrawLabel = true; + + // Mark for cleanup next cycle if range ends here. + if (ActiveCols[ColIdx].LiveIn && !ActiveCols[ColIdx].LiveOut) + ColumnsToFreeNextCycle.push_back(ColIdx); } } } @@ -360,7 +484,13 @@ void LiveElementPrinter::printAfterOtherLine(formatted_raw_ostream &OS, void LiveElementPrinter::printBetweenInsts(formatted_raw_ostream &OS, bool MustPrint) { bool PrintedSomething = false; - for (unsigned ColIdx = 0, End = ActiveCols.size(); ColIdx < End; ++ColIdx) { + // Get all active elements, sorted by discovery order. + std::vector SortedElementIndices = getSortedActiveElementIndices(); + // The outer loop iterates over the deterministic DWARF discovery order. + for (unsigned ElementIdx : SortedElementIndices) { + // Look up the physical column index (ColIdx) assigned to this + // element. We use .at() because we are certain the element is active. + unsigned ColIdx = ElementToColumn.at(ElementIdx); if (ActiveCols[ColIdx].isActive() && ActiveCols[ColIdx].MustDrawLabel) { // First we need to print the live range markers for any active // columns to the left of this one. @@ -375,8 +505,7 @@ void LiveElementPrinter::printBetweenInsts(formatted_raw_ostream &OS, OS << " "; } - const std::unique_ptr &LE = - LiveElements[ActiveCols[ColIdx].ElementIdx]; + const std::unique_ptr &LE = LiveElements[ElementIdx]; // Then print the variable name and location of the new live range, // with box drawing characters joining it to the live range line. OS << getLineChar(ActiveCols[ColIdx].LiveIn ? LineChar::LabelCornerActive @@ -438,22 +567,40 @@ void LiveElementPrinter::printAfterInst(formatted_raw_ostream &OS) { } } -void LiveElementPrinter::printStartLine(formatted_raw_ostream &OS, - object::SectionedAddress Addr) { - // Print a line to idenfity the start of an inlined function if line format - // is specified. - if (DbgInlinedFunctions == DFLimitsOnly) - for (const std::unique_ptr &LE : LiveElements) - LE->printElementLine(OS, Addr, false); -} +void LiveElementPrinter::printBoundaryLine(formatted_raw_ostream &OS, + object::SectionedAddress Addr, + bool IsEnd) { + // Only print the start/end line for inlined functions if DFLimitsOnly is + // enabled. + if (DbgInlinedFunctions != DFLimitsOnly) + return; -void LiveElementPrinter::printEndLine(formatted_raw_ostream &OS, - object::SectionedAddress Addr) { - // Print a line to idenfity the end of an inlined function if line format is - // specified. - if (DbgInlinedFunctions == DFLimitsOnly) - for (const std::unique_ptr &LE : LiveElements) - LE->printElementLine(OS, Addr, true); + // Select the appropriate map based on whether we are checking the start + // (LowPC) or end (HighPC) address. + const auto &AddressMap = + IsEnd ? LiveElementsByEndAddress : LiveElementsByAddress; + + // Use the map to find all elements that start/end at the given address. + std::vector ElementIndices; + auto It = AddressMap.find(Addr.Address); + if (It != AddressMap.end()) { + for (LiveElement *LE : It->second) { + // Look up the element index from the pointer. + auto IndexIt = ElementPtrToIndex.find(LE); + assert(IndexIt != ElementPtrToIndex.end() && + "LiveElement found in address map but missing index!"); + ElementIndices.push_back(IndexIt->second); + } + } + + // Sort the indices to ensure deterministic output order (by DWARF discovery + // order). + llvm::stable_sort(ElementIndices); + + for (unsigned ElementIdx : ElementIndices) { + LiveElement *LE = LiveElements[ElementIdx].get(); + LE->printElementLine(OS, Addr, IsEnd); + } } bool SourcePrinter::cacheSource(const DILineInfo &LineInfo) { diff --git a/llvm/tools/llvm-objdump/SourcePrinter.h b/llvm/tools/llvm-objdump/SourcePrinter.h index 19acc871707aa..ad3ea122b4346 100644 --- a/llvm/tools/llvm-objdump/SourcePrinter.h +++ b/llvm/tools/llvm-objdump/SourcePrinter.h @@ -9,13 +9,16 @@ #ifndef LLVM_TOOLS_LLVM_OBJDUMP_SOURCEPRINTER_H #define LLVM_TOOLS_LLVM_OBJDUMP_SOURCEPRINTER_H +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/IndexedMap.h" +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/StringSet.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" #include "llvm/DebugInfo/Symbolize/Symbolize.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSubtargetInfo.h" #include "llvm/Support/FormattedStream.h" +#include #include #include @@ -78,6 +81,7 @@ class LiveVariable : public LiveElement { bool liveAtAddress(object::SectionedAddress Addr) const override; void print(raw_ostream &OS, const MCRegisterInfo &MRI) const override; void dump(raw_ostream &OS) const override; + const DWARFLocationExpression &getLocExpr() const { return LocExpr; } }; /// Helper class for printing source locations for variables and inlined @@ -95,17 +99,39 @@ class LiveElementPrinter { static constexpr unsigned NullElementIdx = std::numeric_limits::max(); + + // Clear the column's data. + void clear() { + ElementIdx = NullElementIdx; + LiveIn = false; + LiveOut = false; + MustDrawLabel = false; + } }; - // All live elements we know about in the object/image file. + // Vector that owns all LiveElement objects for memory management. std::vector> LiveElements; - - // The columns we are currently drawing. - IndexedMap ActiveCols; + // Map for fast lookup of live elements by their starting address (LowPC). + llvm::MapVector> LiveElementsByAddress; + // Map for fast lookup of live elements by their ending address (HighPC). + llvm::MapVector> + LiveElementsByEndAddress; + // Map from a LiveElement pointer to its index in the LiveElements vector. + llvm::DenseMap ElementPtrToIndex; + // Map from a live element index to column index for efficient lookup. + llvm::DenseMap ElementToColumn; + // Vector of columns currently used for printing live ranges. + std::vector ActiveCols; + // Set of available column indices kept sorted for efficient reuse. + std::set FreeCols; + // Vector of available column indices that can be reused. + std::vector ColumnsToFreeNextCycle; const MCRegisterInfo &MRI; const MCSubtargetInfo &STI; + void registerNewVariable(); + void addInlinedFunction(DWARFDie FuncDie, DWARFDie InlinedFuncDie); void addVariable(DWARFDie FuncDie, DWARFDie VarDie); @@ -122,11 +148,19 @@ class LiveElementPrinter { // put live element lines. Pick a less overloaded word. unsigned moveToFirstVarColumn(formatted_raw_ostream &OS); - unsigned findFreeColumn(); + // Get an existing column for a live element, or find a free one. + unsigned getOrCreateColumn(unsigned ElementIdx); + + // Free a column when its element is no longer live. + void freeColumn(unsigned ColIdx); + + // Returns the indices of all currently active elements, sorted by their DWARF + // discovery order. + std::vector getSortedActiveElementIndices() const; public: LiveElementPrinter(const MCRegisterInfo &MRI, const MCSubtargetInfo &STI) - : ActiveCols(Column()), MRI(MRI), STI(STI) {} + : MRI(MRI), STI(STI) {} void dump() const; @@ -170,10 +204,9 @@ class LiveElementPrinter { /// Print the live element ranges to the right of a disassembled instruction. void printAfterInst(formatted_raw_ostream &OS); - /// Print a line to idenfity the start of a live element. - void printStartLine(formatted_raw_ostream &OS, object::SectionedAddress Addr); - /// Print a line to idenfity the end of a live element. - void printEndLine(formatted_raw_ostream &OS, object::SectionedAddress Addr); + /// Print a line to idenfity the start/end of a live element. + void printBoundaryLine(formatted_raw_ostream &OS, + object::SectionedAddress Addr, bool IsEnd); }; class SourcePrinter { diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp index 3ec644a472bfc..38c3f31441c06 100644 --- a/llvm/tools/llvm-objdump/llvm-objdump.cpp +++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp @@ -688,7 +688,7 @@ class PrettyPrinter { LiveElementPrinter &LEP) { if (SP && (PrintSource || PrintLines)) SP->printSourceLine(OS, Address, ObjectFilename, LEP); - LEP.printStartLine(OS, Address); + LEP.printBoundaryLine(OS, Address, false); LEP.printBetweenInsts(OS, false); printRawData(Bytes, Address.Address, OS, STI); @@ -941,7 +941,7 @@ class ARMPrettyPrinter : public PrettyPrinter { LiveElementPrinter &LEP) override { if (SP && (PrintSource || PrintLines)) SP->printSourceLine(OS, Address, ObjectFilename, LEP); - LEP.printStartLine(OS, Address); + LEP.printBoundaryLine(OS, Address, false); LEP.printBetweenInsts(OS, false); size_t Start = OS.tell(); @@ -996,7 +996,7 @@ class AArch64PrettyPrinter : public PrettyPrinter { LiveElementPrinter &LEP) override { if (SP && (PrintSource || PrintLines)) SP->printSourceLine(OS, Address, ObjectFilename, LEP); - LEP.printStartLine(OS, Address); + LEP.printBoundaryLine(OS, Address, false); LEP.printBetweenInsts(OS, false); size_t Start = OS.tell(); @@ -1035,7 +1035,7 @@ class RISCVPrettyPrinter : public PrettyPrinter { LiveElementPrinter &LEP) override { if (SP && (PrintSource || PrintLines)) SP->printSourceLine(OS, Address, ObjectFilename, LEP); - LEP.printStartLine(OS, Address); + LEP.printBoundaryLine(OS, Address, false); LEP.printBetweenInsts(OS, false); size_t Start = OS.tell(); @@ -2601,7 +2601,7 @@ disassembleObject(ObjectFile &Obj, const ObjectFile &DbgObj, object::SectionedAddress NextAddr = { SectionAddr + Index + VMAAdjustment + Size, Section.getIndex()}; - LEP.printEndLine(FOS, NextAddr); + LEP.printBoundaryLine(FOS, NextAddr, true); Index += Size; } From 8cc02597f119674b7592e4db5dabf476c97ac8fa Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Thu, 27 Nov 2025 12:03:03 +1100 Subject: [PATCH 18/22] [ORC] Clear stale ElemToPendingSN entries in WaitingOnGraph. (#169747) WaitingOnGraph::processReadyOrFailed was not clearing stale entries from the ElemToPendingSN map. If symbols were removed from the ExecutionSession and then re-added this could lead to dependencies on the stale entries, triggering a use-after-free bug. https://github.com/llvm/llvm-project/issues/169135 --- .../llvm/ExecutionEngine/Orc/WaitingOnGraph.h | 22 +++++++-- .../Orc/WaitingOnGraphTest.cpp | 46 +++++++++++++++++++ 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h b/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h index 0b46c7fb1f445..93412d9d22f8c 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h @@ -338,9 +338,9 @@ template class WaitingOnGraph { // incorporate NewSNs. std::vector> ReadyNodes, FailedNodes; processReadyOrFailed(ModifiedPendingSNs, ReadyNodes, FailedNodes, - SuperNodeDeps, ElemToPendingSN, FailedSNs); + SuperNodeDeps, FailedSNs, &ElemToPendingSN); processReadyOrFailed(NewSNs, ReadyNodes, FailedNodes, SuperNodeDeps, - ElemToNewSN, FailedSNs); + FailedSNs, nullptr); CoalesceToPendingSNs.coalesce(ModifiedPendingSNs, ElemToPendingSN); CoalesceToPendingSNs.coalesce(NewSNs, ElemToPendingSN); @@ -591,8 +591,11 @@ template class WaitingOnGraph { std::vector> &Ready, std::vector> &Failed, SuperNodeDepsMap &SuperNodeDeps, - ElemToSuperNodeMap &ElemToSNs, - const std::vector &FailedSNs) { + const std::vector &FailedSNs, + ElemToSuperNodeMap *ElemToSNs) { + + SmallVector ToRemoveFromElemToSNs; + for (size_t I = 0; I != SNs.size();) { auto &SN = SNs[I]; @@ -609,6 +612,8 @@ template class WaitingOnGraph { bool SNReady = SN->Deps.empty(); if (SNReady || SNFailed) { + if (ElemToSNs) + ToRemoveFromElemToSNs.push_back(SN.get()); auto &NodeList = SNFailed ? Failed : Ready; NodeList.push_back(std::move(SN)); std::swap(SN, SNs.back()); @@ -616,6 +621,15 @@ template class WaitingOnGraph { } else ++I; } + + // Update ElemToSNs (if passed) to remove elements pointing at SN. + for (auto *SN : ToRemoveFromElemToSNs) { + for (auto &[Container, Elems] : SN->defs()) { + auto &Row = (*ElemToSNs)[Container]; + for (auto &Elem : Elems) + Row.erase(Elem); + } + } } std::vector> PendingSNs; diff --git a/llvm/unittests/ExecutionEngine/Orc/WaitingOnGraphTest.cpp b/llvm/unittests/ExecutionEngine/Orc/WaitingOnGraphTest.cpp index 0d4a5212c1f0c..1d550b1cfbc19 100644 --- a/llvm/unittests/ExecutionEngine/Orc/WaitingOnGraphTest.cpp +++ b/llvm/unittests/ExecutionEngine/Orc/WaitingOnGraphTest.cpp @@ -532,6 +532,52 @@ TEST_F(WaitingOnGraphTest, Emit_ZigZag) { EXPECT_TRUE(PendingSNs.empty()); } +TEST_F(WaitingOnGraphTest, Emit_ReEmit) { + // Test for the bug in https://github.com/llvm/llvm-project/issues/169135, + // which was caused by stale entries in the ElemsToPendingSNs map. + // + // To trigger the bug we need to: + // 1. Create a SuperNode with an unmet dependence, causing it to be added to + // ElemsToPendingSNs. + // 2. Cause that SuperNode to become ready (bug left stale entries in map) + // 3. Remove the node from the Ready map (this is equivalent to removal of a + // symbol in an ORC session, and allows new SuperNodes to depend on the + // stale entry). + // 4. Add a new node that references the previously emitted/removed SuperNode + // This triggers access of the stale entry, and should error out in + // sanitizer builds. + + SuperNodeBuilder B; + + // 1. Create SuperNode with unmet dependence. + ContainerElementsMap Defs0({{0, {0}}}); + ContainerElementsMap Deps0({{0, {1}}}); + B.add(Defs0, Deps0); + emit(TestGraph::simplify(B.takeSuperNodes())); + + EXPECT_TRUE(Ready.empty()); + + // 2. Cause previous SuperNode to become ready. + ContainerElementsMap Defs1({{0, {1}}}); + B.add(Defs1, ContainerElementsMap()); + emit(TestGraph::simplify(B.takeSuperNodes())); + + // Check that both nodes have become ready. + EXPECT_EQ(Ready, merge(Defs0, Defs1)); + + // 3. Erase Ready nodes to simulate removal from the graph. + Ready.clear(); + + // 4. Emit a new dependence on the original def. + ContainerElementsMap Defs2({{0, {2}}}); + ContainerElementsMap Deps2({{0, {0}}}); + B.add(Defs2, Deps2); + auto ER = emit(TestGraph::simplify(B.takeSuperNodes())); + + // We expect the new dependence to remain pending. + EXPECT_TRUE(ER.Ready.empty()); +} + TEST_F(WaitingOnGraphTest, Fail_Empty) { // Check that failing an empty set is a no-op. auto FR = G.fail(ContainerElementsMap()); From b7eb9883dc9014a392f6435ba9b4058c8f8efd3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ko=C5=A1=C3=ADk?= Date: Thu, 27 Nov 2025 02:09:18 +0100 Subject: [PATCH 19/22] [lldb] Use InlHostByteOrder in RegisterValue::SetValueFromData (#169624) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An existing code can be further simplified. --------- Co-authored-by: Matej Košík --- lldb/source/Utility/RegisterValue.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lldb/source/Utility/RegisterValue.cpp b/lldb/source/Utility/RegisterValue.cpp index c28c9e2d4d106..4d762dc80e7f5 100644 --- a/lldb/source/Utility/RegisterValue.cpp +++ b/lldb/source/Utility/RegisterValue.cpp @@ -196,9 +196,7 @@ Status RegisterValue::SetValueFromData(const RegisterInfo ®_info, SetUInt64(src.GetMaxU64(&src_offset, src_len)); else { std::vector native_endian_src(src_len, 0); - src.ExtractBytes(src_offset, src_len, - llvm::sys::IsLittleEndianHost ? eByteOrderLittle - : eByteOrderBig, + src.ExtractBytes(src_offset, src_len, endian::InlHostByteOrder(), native_endian_src.data()); llvm::APInt uint = llvm::APInt::getZero(src_len * 8); llvm::LoadIntFromMemory(uint, native_endian_src.data(), src_len); From e2a29eca56bf92c8f3c5b5c88259211579b66182 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Wed, 19 Nov 2025 17:12:42 -0800 Subject: [PATCH 20/22] [UBSan] Use -fsanitize-handler-preserve-all-regs in codegen Pull Request: https://github.com/llvm/llvm-project/pull/168645 --- clang/lib/CodeGen/BackendUtil.cpp | 2 ++ clang/lib/CodeGen/CGExpr.cpp | 8 +++++++ clang/lib/Driver/SanitizerArgs.cpp | 5 +++-- .../CodeGen/cfi-icall-trap-recover-runtime.c | 2 +- .../cfi-vcall-trap-recover-runtime.cpp | 2 +- clang/test/Driver/fsanitize.c | 18 ++++++++++++---- .../TestCases/override-callback.c | 21 ++++++++++++++++--- .../Instrumentation/BoundsChecking.h | 7 +++++-- llvm/lib/Passes/PassBuilder.cpp | 7 +++++++ .../Instrumentation/BoundsChecking.cpp | 7 ++++++- .../BoundsChecking/runtimes.ll | 19 +++++++++++++++++ 11 files changed, 84 insertions(+), 14 deletions(-) diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index 5590d217e96ff..82ca831f35da2 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -1134,6 +1134,8 @@ void EmitAssemblyHelper::RunOptimizationPipeline( CodeGenOpts.SanitizeMinimalRuntime), /*MayReturn=*/ CodeGenOpts.SanitizeRecover.has(SanitizerKind::LocalBounds), + /*HandlerPreserveAllRegs=*/ + static_cast(CodeGenOpts.SanitizeHandlerPreserveAllRegs), }; } FPM.addPass(BoundsCheckingPass(Options)); diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp index 221d0d6016a9c..c8f669b69d991 100644 --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -3819,6 +3819,8 @@ static void emitCheckHandlerCall(CodeGenFunction &CGF, bool NeedsAbortSuffix = IsFatal && RecoverKind != CheckRecoverableKind::Unrecoverable; bool MinimalRuntime = CGF.CGM.getCodeGenOpts().SanitizeMinimalRuntime; + bool HandlerPreserveAllRegs = + CGF.CGM.getCodeGenOpts().SanitizeHandlerPreserveAllRegs; const SanitizerHandlerInfo &CheckInfo = SanitizerHandlers[CheckHandler]; const StringRef CheckName = CheckInfo.Name; std::string FnName = "__ubsan_handle_" + CheckName.str(); @@ -3828,6 +3830,8 @@ static void emitCheckHandlerCall(CodeGenFunction &CGF, FnName += "_minimal"; if (NeedsAbortSuffix) FnName += "_abort"; + if (HandlerPreserveAllRegs && !NeedsAbortSuffix) + FnName += "_preserve"; bool MayReturn = !IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable; @@ -3848,6 +3852,10 @@ static void emitCheckHandlerCall(CodeGenFunction &CGF, (CGF.CurCodeDecl && CGF.CurCodeDecl->hasAttr()); if (NoMerge) HandlerCall->addFnAttr(llvm::Attribute::NoMerge); + if (HandlerPreserveAllRegs && !NeedsAbortSuffix) { + // N.B. there is also a clang::CallingConv which is not what we want here. + HandlerCall->setCallingConv(llvm::CallingConv::PreserveAll); + } if (!MayReturn) { HandlerCall->setDoesNotReturn(); CGF.Builder.CreateUnreachable(); diff --git a/clang/lib/Driver/SanitizerArgs.cpp b/clang/lib/Driver/SanitizerArgs.cpp index 9902cbbf99436..d5a0b65ab758f 100644 --- a/clang/lib/Driver/SanitizerArgs.cpp +++ b/clang/lib/Driver/SanitizerArgs.cpp @@ -419,6 +419,7 @@ SanitizerArgs::SanitizerArgs(const ToolChain &TC, const Driver &D = TC.getDriver(); SanitizerMask TrappingKinds = parseSanitizeTrapArgs(D, Args, DiagnoseErrors); SanitizerMask InvalidTrappingKinds = TrappingKinds & NotAllowedWithTrap; + const llvm::Triple &Triple = TC.getTriple(); MinimalRuntime = Args.hasFlag(options::OPT_fsanitize_minimal_runtime, @@ -426,7 +427,8 @@ SanitizerArgs::SanitizerArgs(const ToolChain &TC, HandlerPreserveAllRegs = Args.hasFlag(options::OPT_fsanitize_handler_preserve_all_regs, options::OPT_fno_sanitize_handler_preserve_all_regs, - HandlerPreserveAllRegs); + HandlerPreserveAllRegs) && + MinimalRuntime && (Triple.isAArch64() || Triple.isX86_64()); // The object size sanitizer should not be enabled at -O0. Arg *OptLevel = Args.getLastArg(options::OPT_O_Group); @@ -494,7 +496,6 @@ SanitizerArgs::SanitizerArgs(const ToolChain &TC, // -fsanitize=function and -fsanitize=kcfi instrument indirect function // calls to load a type hash before the function label. Therefore, an // execute-only target doesn't support the function and kcfi sanitizers. - const llvm::Triple &Triple = TC.getTriple(); if (isExecuteOnlyTarget(Triple, Args)) { if (SanitizerMask KindsToDiagnose = Add & NotAllowedWithExecuteOnly & ~DiagnosedKinds) { diff --git a/clang/test/CodeGen/cfi-icall-trap-recover-runtime.c b/clang/test/CodeGen/cfi-icall-trap-recover-runtime.c index 9fbb1221ab39a..2c44842f9d28e 100644 --- a/clang/test/CodeGen/cfi-icall-trap-recover-runtime.c +++ b/clang/test/CodeGen/cfi-icall-trap-recover-runtime.c @@ -171,7 +171,7 @@ void xf(); // PRESERVE_MIN-NEXT: [[TMP3:%.*]] = call i1 @llvm.type.test(ptr [[TMP2]], metadata !"_ZTSFvE"), !nosanitize [[META10:![0-9]+]] // PRESERVE_MIN-NEXT: br i1 [[TMP3]], label %[[CONT:.*]], label %[[HANDLER_CFI_CHECK_FAIL:.*]], !prof [[PROF11:![0-9]+]], !nosanitize [[META10]] // PRESERVE_MIN: [[HANDLER_CFI_CHECK_FAIL]]: -// PRESERVE_MIN-NEXT: call void @__ubsan_handle_cfi_check_fail_minimal() #[[ATTR4:[0-9]+]], !nosanitize [[META10]] +// PRESERVE_MIN-NEXT: call preserve_allcc void @__ubsan_handle_cfi_check_fail_minimal_preserve() #[[ATTR4:[0-9]+]], !nosanitize [[META10]] // PRESERVE_MIN-NEXT: br label %[[CONT]], !nosanitize [[META10]] // PRESERVE_MIN: [[CONT]]: // PRESERVE_MIN-NEXT: call void (...) [[TMP2]]() diff --git a/clang/test/CodeGenCXX/cfi-vcall-trap-recover-runtime.cpp b/clang/test/CodeGenCXX/cfi-vcall-trap-recover-runtime.cpp index 0130d9e33cd9d..2451d31e9a489 100644 --- a/clang/test/CodeGenCXX/cfi-vcall-trap-recover-runtime.cpp +++ b/clang/test/CodeGenCXX/cfi-vcall-trap-recover-runtime.cpp @@ -127,7 +127,7 @@ struct S1 { // PRESERVE_MIN-NEXT: [[TMP2:%.*]] = call i1 @llvm.type.test(ptr [[VTABLE]], metadata !"all-vtables"), !nosanitize [[META5]] // PRESERVE_MIN-NEXT: br i1 [[TMP1]], label %[[CONT:.*]], label %[[HANDLER_CFI_CHECK_FAIL:.*]], !prof [[PROF6:![0-9]+]], !nosanitize [[META5]] // PRESERVE_MIN: [[HANDLER_CFI_CHECK_FAIL]]: -// PRESERVE_MIN-NEXT: call void @__ubsan_handle_cfi_check_fail_minimal() #[[ATTR3:[0-9]+]], !nosanitize [[META5]] +// PRESERVE_MIN-NEXT: call preserve_allcc void @__ubsan_handle_cfi_check_fail_minimal_preserve() #[[ATTR3:[0-9]+]], !nosanitize [[META5]] // PRESERVE_MIN-NEXT: br label %[[CONT]], !nosanitize [[META5]] // PRESERVE_MIN: [[CONT]]: // PRESERVE_MIN-NEXT: [[VFN:%.*]] = getelementptr inbounds ptr, ptr [[VTABLE]], i64 0 diff --git a/clang/test/Driver/fsanitize.c b/clang/test/Driver/fsanitize.c index f2a4d8c50ec23..c02b8828062f2 100644 --- a/clang/test/Driver/fsanitize.c +++ b/clang/test/Driver/fsanitize.c @@ -984,10 +984,20 @@ // CHECK-UBSAN-MINIMAL: "-fsanitize={{((signed-integer-overflow|integer-divide-by-zero|shift-base|shift-exponent|unreachable|return|vla-bound|alignment|null|pointer-overflow|float-cast-overflow|array-bounds|enum|bool|builtin|returns-nonnull-attribute|nonnull-attribute|function),?){18}"}} // CHECK-UBSAN-MINIMAL: "-fsanitize-minimal-runtime" -// RUN: %clang --target=x86_64-linux-gnu -fsanitize=undefined -fsanitize-minimal-runtime -fsanitize-handler-preserve-all-regs %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-MINIMAL-PRESERVE -// CHECK-UBSAN-MINIMAL-PRESERVE: "-fsanitize={{((signed-integer-overflow|integer-divide-by-zero|shift-base|shift-exponent|unreachable|return|vla-bound|alignment|null|pointer-overflow|float-cast-overflow|array-bounds|enum|bool|builtin|returns-nonnull-attribute|nonnull-attribute|function),?){18}"}} -// CHECK-UBSAN-MINIMAL-PRESERVE: "-fsanitize-minimal-runtime" -// CHECK-UBSAN-MINIMAL-PRESERVE: "-fsanitize-handler-preserve-all-regs +// RUN: %clang --target=x86_64-linux-gnu -fsanitize=undefined -fsanitize-minimal-runtime -fsanitize-handler-preserve-all-regs %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-MINIMAL-PRESERVE-X86-64 +// CHECK-UBSAN-MINIMAL-PRESERVE-X86-64: "-fsanitize={{((signed-integer-overflow|integer-divide-by-zero|shift-base|shift-exponent|unreachable|return|vla-bound|alignment|null|pointer-overflow|float-cast-overflow|array-bounds|enum|bool|builtin|returns-nonnull-attribute|nonnull-attribute|function),?){18}"}} +// CHECK-UBSAN-MINIMAL-PRESERVE-X86-64: "-fsanitize-minimal-runtime" +// CHECK-UBSAN-MINIMAL-PRESERVE-X86-64: "-fsanitize-handler-preserve-all-regs + +// RUN: %clang --target=aarch64-linux-gnu -fsanitize=undefined -fsanitize-minimal-runtime -fsanitize-handler-preserve-all-regs %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-MINIMAL-PRESERVE-AARCH64 +// CHECK-UBSAN-MINIMAL-PRESERVE-AARCH64: "-fsanitize={{((signed-integer-overflow|integer-divide-by-zero|shift-base|shift-exponent|unreachable|return|vla-bound|alignment|null|pointer-overflow|float-cast-overflow|array-bounds|enum|bool|builtin|returns-nonnull-attribute|nonnull-attribute|function),?){18}"}} +// CHECK-UBSAN-MINIMAL-PRESERVE-AARCH64: "-fsanitize-minimal-runtime" +// CHECK-UBSAN-MINIMAL-PRESERVE-AARCH64: "-fsanitize-handler-preserve-all-regs + +// RUN: %clang --target=i386-linux-gnu -fsanitize=undefined -fsanitize-minimal-runtime -fsanitize-handler-preserve-all-regs %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-MINIMAL-PRESERVE-I386 +// CHECK-UBSAN-MINIMAL-PRESERVE-I386: "-fsanitize={{((signed-integer-overflow|integer-divide-by-zero|shift-base|shift-exponent|unreachable|return|vla-bound|alignment|null|pointer-overflow|float-cast-overflow|array-bounds|enum|bool|builtin|returns-nonnull-attribute|nonnull-attribute|function),?){18}"}} +// CHECK-UBSAN-MINIMAL-PRESERVE-I386: "-fsanitize-minimal-runtime" +// CHECK-UBSAN-MINIMAL-PRESERVE-I386-NOT: "-fsanitize-handler-preserve-all-regs // RUN: %clang --target=x86_64-linux-gnu -fsanitize=integer -fsanitize-trap=integer %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-INTSAN-TRAP // CHECK-INTSAN-TRAP: "-fsanitize-trap=integer-divide-by-zero,shift-base,shift-exponent,signed-integer-overflow,unsigned-integer-overflow,unsigned-shift-base,implicit-unsigned-integer-truncation,implicit-signed-integer-truncation,implicit-integer-sign-change" diff --git a/compiler-rt/test/ubsan_minimal/TestCases/override-callback.c b/compiler-rt/test/ubsan_minimal/TestCases/override-callback.c index aaed134b3ae81..8c04a0091cb11 100644 --- a/compiler-rt/test/ubsan_minimal/TestCases/override-callback.c +++ b/compiler-rt/test/ubsan_minimal/TestCases/override-callback.c @@ -1,6 +1,7 @@ -// RUN: %clang_min_runtime -fsanitize=implicit-integer-sign-change %s -o %t && %run %t 2>&1 | FileCheck %s -// RUN: %clang_min_runtime -fsanitize=implicit-integer-sign-change -fno-sanitize-recover=all %s -o %t && not --crash %run %t 2>&1 | FileCheck %s -// RUN: %clang_min_runtime -fsanitize=implicit-integer-sign-change -fno-sanitize-recover=all -DOVERRIDE=1 %s -o %t && not --crash %run %t 2>&1 | FileCheck %s --check-prefixes=FATAL +// RUN: %clang_min_runtime -fsanitize=implicit-integer-sign-change %s -o %t && %run %t 2>&1 | FileCheck %s +// RUN: %clang_min_runtime -fsanitize=implicit-integer-sign-change -fsanitize-handler-preserve-all-regs -DPRESERVE %s -o %t && %run %t 2>&1 | FileCheck %s --check-prefixes=PRESERVE +// RUN: %clang_min_runtime -fsanitize=implicit-integer-sign-change -fno-sanitize-recover=all %s -o %t && not --crash %run %t 2>&1 | FileCheck %s +// RUN: %clang_min_runtime -fsanitize=implicit-integer-sign-change -fno-sanitize-recover=all -DOVERRIDE=1 %s -o %t && not --crash %run %t 2>&1 | FileCheck %s --check-prefixes=FATAL #include #include @@ -9,8 +10,21 @@ static int Result; void __ubsan_report_error(const char *kind, uintptr_t caller) { +// -fsanitize-handler-preserve-all-regs is ignored on other architectures. +// Prented we called to other handler on those. +#if defined(PRESERVE) && !defined(__aarch64__) && !defined(__x86_64__) + fprintf(stderr, "CUSTOM_CALLBACK_PRESERVE: %s\n", kind); +#else fprintf(stderr, "CUSTOM_CALLBACK: %s\n", kind); +#endif +} + +#if defined(__aarch64__) || defined(__x86_64__) +[[clang::preserve_all]] void __ubsan_report_error_preserve(const char *kind, + uintptr_t caller) { + fprintf(stderr, "CUSTOM_CALLBACK_PRESERVE: %s\n", kind); } +#endif #if OVERRIDE void __ubsan_report_error_fatal(const char *kind, uintptr_t caller) { @@ -21,5 +35,6 @@ void __ubsan_report_error_fatal(const char *kind, uintptr_t caller) { int main(int argc, const char **argv) { int32_t t0 = (~((uint32_t)0)); // CHECK: CUSTOM_CALLBACK: implicit-conversion + // PRESERVE: CUSTOM_CALLBACK_PRESERVE: implicit-conversion // FATAL: FATAL_CALLBACK: implicit-conversion } diff --git a/llvm/include/llvm/Transforms/Instrumentation/BoundsChecking.h b/llvm/include/llvm/Transforms/Instrumentation/BoundsChecking.h index 8e7df5e6b10f0..e4bfcd395c2d6 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/BoundsChecking.h +++ b/llvm/include/llvm/Transforms/Instrumentation/BoundsChecking.h @@ -11,6 +11,7 @@ #include "llvm/IR/PassManager.h" #include "llvm/Support/Compiler.h" +#include "llvm/TargetParser/Triple.h" #include namespace llvm { @@ -23,10 +24,12 @@ class BoundsCheckingPass : public PassInfoMixin { public: struct Options { struct Runtime { - Runtime(bool MinRuntime, bool MayReturn) - : MinRuntime(MinRuntime), MayReturn(MayReturn) {} + Runtime(bool MinRuntime, bool MayReturn, bool HandlerPreserveAllRegs) + : MinRuntime(MinRuntime), MayReturn(MayReturn), + HandlerPreserveAllRegs(HandlerPreserveAllRegs) {} bool MinRuntime; bool MayReturn; + bool HandlerPreserveAllRegs; }; std::optional Rt; // Trap if empty. bool Merge = false; diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 0d190ea448931..f5281ea69b512 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -1590,24 +1590,31 @@ parseBoundsCheckingOptions(StringRef Params) { Options.Rt = { /*MinRuntime=*/false, /*MayReturn=*/true, + /*HandlerPreserveAllRegs=*/false, }; } else if (ParamName == "rt-abort") { Options.Rt = { /*MinRuntime=*/false, /*MayReturn=*/false, + /*HandlerPreserveAllRegs=*/false, }; } else if (ParamName == "min-rt") { Options.Rt = { /*MinRuntime=*/true, /*MayReturn=*/true, + /*HandlerPreserveAllRegs=*/false, }; } else if (ParamName == "min-rt-abort") { Options.Rt = { /*MinRuntime=*/true, /*MayReturn=*/false, + /*HandlerPreserveAllRegs=*/false, }; } else if (ParamName == "merge") { Options.Merge = true; + } else if (ParamName == "handler-preserve-all-regs") { + if (Options.Rt) + Options.Rt->HandlerPreserveAllRegs = true; } else { StringRef ParamEQ; StringRef Val; diff --git a/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp b/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp index 9239ae8741afb..b5a8f79e26436 100644 --- a/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp +++ b/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp @@ -178,6 +178,8 @@ getRuntimeCallName(const BoundsCheckingPass::Options::Runtime &Opts) { Name += "_minimal"; if (!Opts.MayReturn) Name += "_abort"; + else if (Opts.HandlerPreserveAllRegs) + Name += "_preserve"; return Name; } @@ -267,7 +269,10 @@ static bool addBoundsChecking(Function &F, TargetLibraryInfo &TLI, TrapCall->setDoesNotReturn(); IRB.CreateUnreachable(); } - + // The preserve-all logic is somewhat duplicated in CGExpr.cpp for + // local-bounds. Make sure to change that too. + if (Opts.Rt && Opts.Rt->HandlerPreserveAllRegs && MayReturn) + TrapCall->setCallingConv(CallingConv::PreserveAll); if (!MayReturn && SingleTrapBB && !DebugTrapBB) ReuseTrapBB = TrapBB; diff --git a/llvm/test/Instrumentation/BoundsChecking/runtimes.ll b/llvm/test/Instrumentation/BoundsChecking/runtimes.ll index 84dd51cd3fa28..74e1eef7ebe35 100644 --- a/llvm/test/Instrumentation/BoundsChecking/runtimes.ll +++ b/llvm/test/Instrumentation/BoundsChecking/runtimes.ll @@ -8,6 +8,9 @@ ; RUN: opt < %s -passes='bounds-checking' -S | FileCheck %s --check-prefixes=RTABORT-NOMERGE ; RUN: opt < %s -passes='bounds-checking' -S | FileCheck %s --check-prefixes=MINRT-NOMERGE ; RUN: opt < %s -passes='bounds-checking' -S | FileCheck %s --check-prefixes=MINRTABORT-NOMERGE + +; RUN: opt < %s -passes='bounds-checking' -S | FileCheck %s --check-prefixes=MINRT-PRESERVE-NOMERGE +; RUN: opt < %s -passes='bounds-checking' -S | FileCheck %s --check-prefixes=MINRTABORT-NOMERGE ; ; RUN: opt < %s -passes='bounds-checking' -S | FileCheck %s --check-prefixes=TR-GUARD-COMMON,TR-GUARD-THREE ; RUN: opt < %s -passes='bounds-checking' -S | FileCheck %s --check-prefixes=TR-GUARD-COMMON,TR-GUARD-THIRTEEN @@ -95,6 +98,22 @@ define void @f1(i64 %x) nounwind { ; RTABORT-NOMERGE-NEXT: call void @__ubsan_handle_local_out_of_bounds_abort() #[[ATTR2:[0-9]+]], !nosanitize [[META0]] ; RTABORT-NOMERGE-NEXT: unreachable, !nosanitize [[META0]] ; +; MINRT-PRESERVE-NOMERGE-LABEL: define void @f1( +; MINRT-PRESERVE-NOMERGE-SAME: i64 [[X:%.*]]) #[[ATTR0:[0-9]+]] { +; MINRT-PRESERVE-NOMERGE-NEXT: [[TMP1:%.*]] = mul i64 16, [[X]] +; MINRT-PRESERVE-NOMERGE-NEXT: [[TMP2:%.*]] = alloca i128, i64 [[X]], align 8 +; MINRT-PRESERVE-NOMERGE-NEXT: [[TMP3:%.*]] = sub i64 [[TMP1]], 0, !nosanitize [[META0:![0-9]+]] +; MINRT-PRESERVE-NOMERGE-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP3]], 16, !nosanitize [[META0]] +; MINRT-PRESERVE-NOMERGE-NEXT: [[TMP5:%.*]] = or i1 false, [[TMP4]], !nosanitize [[META0]] +; MINRT-PRESERVE-NOMERGE-NEXT: [[TMP6:%.*]] = or i1 false, [[TMP5]], !nosanitize [[META0]] +; MINRT-PRESERVE-NOMERGE-NEXT: br i1 [[TMP6]], label %[[TRAP:.*]], label %[[BB7:.*]] +; MINRT-PRESERVE-NOMERGE: [[BB7]]: +; MINRT-PRESERVE-NOMERGE-NEXT: [[TMP8:%.*]] = load i128, ptr [[TMP2]], align 4 +; MINRT-PRESERVE-NOMERGE-NEXT: ret void +; MINRT-PRESERVE-NOMERGE: [[TRAP]]: +; MINRT-PRESERVE-NOMERGE-NEXT: call preserve_allcc void @__ubsan_handle_local_out_of_bounds_minimal_preserve() #[[ATTR1:[0-9]+]], !nosanitize [[META0]] +; MINRT-PRESERVE-NOMERGE-NEXT: br label %[[BB7]], !nosanitize [[META0]] +; ; MINRT-NOMERGE-LABEL: define void @f1( ; MINRT-NOMERGE-SAME: i64 [[X:%.*]]) #[[ATTR0:[0-9]+]] { ; MINRT-NOMERGE-NEXT: [[TMP1:%.*]] = mul i64 16, [[X]] From 48a9b07264e0b7d515806295272771af69186801 Mon Sep 17 00:00:00 2001 From: Jim Lin Date: Thu, 27 Nov 2025 09:32:26 +0800 Subject: [PATCH 21/22] [AMDGPU] Remove unused functions isSigned. NFC (#169750) These have been unused since https://github.com/llvm/llvm-project/pull/145483. --- .../lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp index 8e35ba77d69aa..71ea9ef6fc050 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp @@ -143,14 +143,6 @@ class AMDGPUCodeGenPrepareImpl bool canBreakPHINode(const PHINode &I); - /// \returns True if binary operation \p I is a signed binary operation, false - /// otherwise. - bool isSigned(const BinaryOperator &I) const; - - /// \returns True if the condition of 'select' operation \p I comes from a - /// signed 'icmp' operation, false otherwise. - bool isSigned(const SelectInst &I) const; - /// Return true if \p T is a legal scalar floating point type. bool isLegalFloatingTy(const Type *T) const; @@ -304,16 +296,6 @@ bool AMDGPUCodeGenPrepareImpl::run() { return MadeChange; } -bool AMDGPUCodeGenPrepareImpl::isSigned(const BinaryOperator &I) const { - return I.getOpcode() == Instruction::AShr || - I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::SRem; -} - -bool AMDGPUCodeGenPrepareImpl::isSigned(const SelectInst &I) const { - return isa(I.getOperand(0)) && - cast(I.getOperand(0))->isSigned(); -} - bool AMDGPUCodeGenPrepareImpl::isLegalFloatingTy(const Type *Ty) const { return Ty->isFloatTy() || Ty->isDoubleTy() || (Ty->isHalfTy() && ST.has16BitInsts()); From 1ff5c89176f9171bd50b1f005cbf019a5d72e0b6 Mon Sep 17 00:00:00 2001 From: lonely eagle <2020382038@qq.com> Date: Thu, 27 Nov 2025 09:33:16 +0800 Subject: [PATCH 22/22] [mlir][dataflow] Add arguemnt print for test-liveness-analysis (#169625) Add arguemnt print for test-liveness-analysis to better debug remove-dead-values pass. --------- Co-authored-by: Mehdi Amini --- .../Analysis/DataFlow/test-liveness-analysis.mlir | 14 +++++++++++++- .../lib/Analysis/DataFlow/TestLivenessAnalysis.cpp | 11 +++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/mlir/test/Analysis/DataFlow/test-liveness-analysis.mlir b/mlir/test/Analysis/DataFlow/test-liveness-analysis.mlir index 3748be74eb0f3..768f1cfcb3d02 100644 --- a/mlir/test/Analysis/DataFlow/test-liveness-analysis.mlir +++ b/mlir/test/Analysis/DataFlow/test-liveness-analysis.mlir @@ -184,6 +184,18 @@ func.func private @private0(%0 : i32) -> i32 { // CHECK-NEXT: result #0: live // CHECK-LABEL: test_tag: y: // CHECK-NEXT: result #0: not live +// CHECK-LABEL: test_tag: for: +// CHECK-NEXT: operand #0: live +// CHECK-NEXT: operand #1: live +// CHECK-NEXT: operand #2: live +// CHECK-NEXT: operand #3: live +// CHECK-NEXT: operand #4: not live +// CHECK-NEXT: result #0: live +// CHECK-NEXT: result #1: not live +// CHECK-NEXT: region: #0: +// CHECK-NEXT: argument: #0: live +// CHECK-NEXT: argument: #1: not live +// CHECK-NEXT: argument: #2: not live func.func @test_7_type_3(%arg0: memref) { %c0 = arith.constant {tag = "zero"} 0 : index %c10 = arith.constant {tag = "ten"} 10 : index @@ -194,7 +206,7 @@ func.func @test_7_type_3(%arg0: memref) { %1 = arith.addi %x, %x : i32 %2 = func.call @private0(%1) : (i32) -> i32 scf.yield %2, %arg3 : i32, i32 - } + } {tag = "for"} memref.store %0#0, %arg0[] : memref return } diff --git a/mlir/test/lib/Analysis/DataFlow/TestLivenessAnalysis.cpp b/mlir/test/lib/Analysis/DataFlow/TestLivenessAnalysis.cpp index 8e2f03b644e49..99f72c6c86f20 100644 --- a/mlir/test/lib/Analysis/DataFlow/TestLivenessAnalysis.cpp +++ b/mlir/test/lib/Analysis/DataFlow/TestLivenessAnalysis.cpp @@ -56,6 +56,17 @@ struct TestLivenessAnalysisPass liveness->print(os); os << "\n"; } + for (auto [regionIndex, region] : llvm::enumerate(op->getRegions())) { + os << " region: #" << regionIndex << ":\n"; + for (auto [argumntIndex, argument] : + llvm::enumerate(region.getArguments())) { + const Liveness *liveness = livenessAnalysis.getLiveness(argument); + assert(liveness && "expected a sparse lattice"); + os << " argument: #" << argumntIndex << ": "; + liveness->print(os); + os << "\n"; + } + } }); } };