diff --git a/llvm/include/llvm/IR/PseudoProbe.h b/llvm/include/llvm/IR/PseudoProbe.h index 79726c0eee8be..60b1be169f87d 100644 --- a/llvm/include/llvm/IR/PseudoProbe.h +++ b/llvm/include/llvm/IR/PseudoProbe.h @@ -30,7 +30,8 @@ enum class PseudoProbeType { Block = 0, IndirectCall, DirectCall }; enum class PseudoProbeAttributes { Reserved = 0x1, - Sentinel = 0x2, // A place holder for split function entry address. + Sentinel = 0x2, // A place holder for split function entry address. + HasDiscriminator = 0x4, // for probes with a discriminator }; // The saturated distrution factor representing 100% for block probes. @@ -91,6 +92,10 @@ static inline bool isSentinelProbe(uint32_t Flags) { return Flags & (uint32_t)PseudoProbeAttributes::Sentinel; } +static inline bool hasDiscriminator(uint32_t Flags) { + return Flags & (uint32_t)PseudoProbeAttributes::HasDiscriminator; +} + std::optional extractProbe(const Instruction &Inst); void setProbeDistributionFactor(Instruction &Inst, float Factor); diff --git a/llvm/include/llvm/MC/MCPseudoProbe.h b/llvm/include/llvm/MC/MCPseudoProbe.h index cd1bf805b50a0..4904cb4ca5456 100644 --- a/llvm/include/llvm/MC/MCPseudoProbe.h +++ b/llvm/include/llvm/MC/MCPseudoProbe.h @@ -30,12 +30,15 @@ // 0 - block probe, 1 - indirect call, 2 - direct call // ATTRIBUTE (uint3) // 1 - reserved +// 2 - Sentinel +// 4 - HasDiscriminator // ADDRESS_TYPE (uint1) // 0 - code address for regular probes (for downwards compatibility) // - GUID of linkage name for sentinel probes // 1 - address delta // CODE_ADDRESS (uint64 or ULEB128) // code address or address delta, depending on ADDRESS_TYPE +// DISCRIMINATOR (ULEB128) if HasDiscriminator // INLINED FUNCTION RECORDS // A list of NUM_INLINED_FUNCTIONS entries describing each of the inlined // callees. Each record contains: @@ -108,6 +111,7 @@ class MCPseudoProbeBase { protected: uint64_t Guid; uint64_t Index; + uint32_t Discriminator; uint8_t Attributes; uint8_t Type; // The value should be equal to PseudoProbeReservedId::Last + 1 which is @@ -116,8 +120,8 @@ class MCPseudoProbeBase { const static uint32_t PseudoProbeFirstId = 1; public: - MCPseudoProbeBase(uint64_t G, uint64_t I, uint64_t At, uint8_t T) - : Guid(G), Index(I), Attributes(At), Type(T) {} + MCPseudoProbeBase(uint64_t G, uint64_t I, uint64_t At, uint8_t T, uint32_t D) + : Guid(G), Index(I), Discriminator(D), Attributes(At), Type(T) {} bool isEntry() const { return Index == PseudoProbeFirstId; } @@ -125,6 +129,8 @@ class MCPseudoProbeBase { uint64_t getIndex() const { return Index; } + uint32_t getDiscriminator() const { return Discriminator; } + uint8_t getAttributes() const { return Attributes; } uint8_t getType() const { return Type; } @@ -155,8 +161,9 @@ class MCPseudoProbe : public MCPseudoProbeBase { public: MCPseudoProbe(MCSymbol *Label, uint64_t Guid, uint64_t Index, uint64_t Type, - uint64_t Attributes) - : MCPseudoProbeBase(Guid, Index, Attributes, Type), Label(Label) { + uint64_t Attributes, uint32_t Discriminator) + : MCPseudoProbeBase(Guid, Index, Attributes, Type, Discriminator), + Label(Label) { assert(Type <= 0xFF && "Probe type too big to encode, exceeding 2^8"); assert(Attributes <= 0xFF && "Probe attributes too big to encode, exceeding 2^16"); @@ -175,8 +182,9 @@ class MCDecodedPseudoProbe : public MCPseudoProbeBase { public: MCDecodedPseudoProbe(uint64_t Ad, uint64_t G, uint32_t I, PseudoProbeType K, - uint8_t At, MCDecodedPseudoProbeInlineTree *Tree) - : MCPseudoProbeBase(G, I, At, static_cast(K)), Address(Ad), + uint8_t At, uint32_t D, + MCDecodedPseudoProbeInlineTree *Tree) + : MCPseudoProbeBase(G, I, At, static_cast(K), D), Address(Ad), InlineTree(Tree){}; uint64_t getAddress() const { return Address; } diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h index f5891b24ae4b4..8b081db4dedd1 100644 --- a/llvm/include/llvm/MC/MCStreamer.h +++ b/llvm/include/llvm/MC/MCStreamer.h @@ -636,7 +636,7 @@ class MCStreamer { /// \param Symbol - The function containing the trap. /// \param Lang - The language code for the exception entry. /// \param Reason - The reason code for the exception entry. - virtual void emitXCOFFExceptDirective(const MCSymbol *Symbol, + virtual void emitXCOFFExceptDirective(const MCSymbol *Symbol, const MCSymbol *Trap, unsigned Lang, unsigned Reason, unsigned FunctionSize, bool hasDebug); @@ -1100,7 +1100,7 @@ class MCStreamer { /// Emit the a pseudo probe into the current section. virtual void emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type, - uint64_t Attr, + uint64_t Attr, uint64_t Discriminator, const MCPseudoProbeInlineStack &InlineStack, MCSymbol *FnSym); diff --git a/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp index 0540a4c2ae02a..7f7eef9f10e2d 100644 --- a/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp @@ -42,8 +42,13 @@ void PseudoProbeHandler::emitPseudoProbe(uint64_t Guid, uint64_t Index, ReversedInlineStack.emplace_back(CallerGuid, CallerProbeId); InlinedAt = InlinedAt->getInlinedAt(); } - + uint64_t Discriminator = 0; + // For now only block probes have FS discriminators. See + // MIRFSDiscriminator.cpp for more details. + if (DebugLoc && + !DILocation::isPseudoProbeDiscriminator(DebugLoc->getDiscriminator())) + Discriminator = DebugLoc->getDiscriminator(); SmallVector InlineStack(llvm::reverse(ReversedInlineStack)); - Asm->OutStreamer->emitPseudoProbe(Guid, Index, Type, Attr, InlineStack, - Asm->CurrentFnSym); + Asm->OutStreamer->emitPseudoProbe(Guid, Index, Type, Attr, Discriminator, + InlineStack, Asm->CurrentFnSym); } diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp index c11d28b746f60..e3d11a2a33744 100644 --- a/llvm/lib/MC/MCAsmStreamer.cpp +++ b/llvm/lib/MC/MCAsmStreamer.cpp @@ -196,7 +196,7 @@ class MCAsmStreamer final : public MCStreamer { void emitXCOFFRefDirective(const MCSymbol *Symbol) override; - void emitXCOFFExceptDirective(const MCSymbol *Symbol, + void emitXCOFFExceptDirective(const MCSymbol *Symbol, const MCSymbol *Trap, unsigned Lang, unsigned Reason, unsigned FunctionSize, bool hasDebug) override; @@ -377,8 +377,9 @@ class MCAsmStreamer final : public MCStreamer { void emitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI) override; void emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type, - uint64_t Attr, - const MCPseudoProbeInlineStack &InlineStack, MCSymbol *FnSym) override; + uint64_t Attr, uint64_t Discriminator, + const MCPseudoProbeInlineStack &InlineStack, + MCSymbol *FnSym) override; void emitBundleAlignMode(Align Alignment) override; void emitBundleLock(bool AlignToEnd) override; @@ -950,7 +951,7 @@ void MCAsmStreamer::emitXCOFFRefDirective(const MCSymbol *Symbol) { } void MCAsmStreamer::emitXCOFFExceptDirective(const MCSymbol *Symbol, - const MCSymbol *Trap, + const MCSymbol *Trap, unsigned Lang, unsigned Reason, unsigned FunctionSize, @@ -2336,11 +2337,14 @@ void MCAsmStreamer::emitInstruction(const MCInst &Inst, EmitEOL(); } -void MCAsmStreamer::emitPseudoProbe( - uint64_t Guid, uint64_t Index, uint64_t Type, uint64_t Attr, - const MCPseudoProbeInlineStack &InlineStack, MCSymbol *FnSym) { - OS << "\t.pseudoprobe\t" << Guid << " " << Index << " " << Type << " " - << Attr; +void MCAsmStreamer::emitPseudoProbe(uint64_t Guid, uint64_t Index, + uint64_t Type, uint64_t Attr, + uint64_t Discriminator, + const MCPseudoProbeInlineStack &InlineStack, + MCSymbol *FnSym) { + OS << "\t.pseudoprobe\t" << Guid << " " << Index << " " << Type << " " << Attr; + if (Discriminator) + OS << " " << Discriminator; // Emit inline stack like // @ GUIDmain:3 @ GUIDCaller:1 @ GUIDDirectCaller:11 for (const auto &Site : InlineStack) diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp index c49e51332b126..472f55af84014 100644 --- a/llvm/lib/MC/MCParser/AsmParser.cpp +++ b/llvm/lib/MC/MCParser/AsmParser.cpp @@ -5853,24 +5853,23 @@ bool AsmParser::parseDirectivePseudoProbe() { int64_t Index; int64_t Type; int64_t Attr; + int64_t Discriminator = 0; - if (getLexer().is(AsmToken::Integer)) { - if (parseIntToken(Guid, "unexpected token in '.pseudoprobe' directive")) - return true; - } + if (parseIntToken(Guid, "unexpected token in '.pseudoprobe' directive")) + return true; - if (getLexer().is(AsmToken::Integer)) { - if (parseIntToken(Index, "unexpected token in '.pseudoprobe' directive")) - return true; - } + if (parseIntToken(Index, "unexpected token in '.pseudoprobe' directive")) + return true; - if (getLexer().is(AsmToken::Integer)) { - if (parseIntToken(Type, "unexpected token in '.pseudoprobe' directive")) - return true; - } + if (parseIntToken(Type, "unexpected token in '.pseudoprobe' directive")) + return true; - if (getLexer().is(AsmToken::Integer)) { - if (parseIntToken(Attr, "unexpected token in '.pseudoprobe' directive")) + if (parseIntToken(Attr, "unexpected token in '.pseudoprobe' directive")) + return true; + + if (hasDiscriminator(Attr)) { + if (parseIntToken(Discriminator, + "unexpected token in '.pseudoprobe' directive")) return true; } @@ -5912,7 +5911,8 @@ bool AsmParser::parseDirectivePseudoProbe() { if (parseEOL()) return true; - getStreamer().emitPseudoProbe(Guid, Index, Type, Attr, InlineStack, FnSym); + getStreamer().emitPseudoProbe(Guid, Index, Type, Attr, Discriminator, + InlineStack, FnSym); return false; } diff --git a/llvm/lib/MC/MCPseudoProbe.cpp b/llvm/lib/MC/MCPseudoProbe.cpp index a9460b86d22a2..35eafec721695 100644 --- a/llvm/lib/MC/MCPseudoProbe.cpp +++ b/llvm/lib/MC/MCPseudoProbe.cpp @@ -8,6 +8,7 @@ #include "llvm/MC/MCPseudoProbe.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/IR/PseudoProbe.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" @@ -58,10 +59,14 @@ void MCPseudoProbe::emit(MCObjectStreamer *MCOS, // Type (bit 0 to 3), with bit 4 to 6 for attributes. // Flag (bit 7, 0 - code address, 1 - address delta). This indicates whether // the following field is a symbolic code address or an address delta. + // Emit FS discriminator assert(Type <= 0xF && "Probe type too big to encode, exceeding 15"); - assert(Attributes <= 0x7 && + auto NewAttributes = Attributes; + if (Discriminator) + NewAttributes |= (uint32_t)PseudoProbeAttributes::HasDiscriminator; + assert(NewAttributes <= 0x7 && "Probe attributes too big to encode, exceeding 7"); - uint8_t PackedType = Type | (Attributes << 4); + uint8_t PackedType = Type | (NewAttributes << 4); uint8_t Flag = !IsSentinel ? ((int8_t)MCPseudoProbeFlag::AddressDelta << 7) : 0; MCOS->emitInt8(Flag | PackedType); @@ -81,6 +86,9 @@ void MCPseudoProbe::emit(MCObjectStreamer *MCOS, MCOS->emitInt64(Guid); } + if (Discriminator) + MCOS->emitULEB128IntValue(Discriminator); + LLVM_DEBUG({ dbgs().indent(MCPseudoProbeTable::DdgPrintIndent); dbgs() << "Probe: " << Index << "\n"; @@ -222,11 +230,11 @@ void MCPseudoProbeSections::emit(MCObjectStreamer *MCOS) { for (const auto &Inlinee : Inlinees) { // Emit the group guarded by a sentinel probe. - MCPseudoProbe SentinelProbe(const_cast(FuncSym), - MD5Hash(FuncSym->getName()), - (uint32_t)PseudoProbeReservedId::Invalid, - (uint32_t)PseudoProbeType::Block, - (uint32_t)PseudoProbeAttributes::Sentinel); + MCPseudoProbe SentinelProbe( + const_cast(FuncSym), MD5Hash(FuncSym->getName()), + (uint32_t)PseudoProbeReservedId::Invalid, + (uint32_t)PseudoProbeType::Block, + (uint32_t)PseudoProbeAttributes::Sentinel, 0); const MCPseudoProbe *Probe = &SentinelProbe; Inlinee.second->emit(MCOS, Probe); } @@ -310,6 +318,8 @@ void MCDecodedPseudoProbe::print(raw_ostream &OS, OS << Guid << " "; } OS << "Index: " << Index << " "; + if (Discriminator) + OS << "Discriminator: " << Discriminator << " "; OS << "Type: " << PseudoProbeTypeStr[static_cast(Type)] << " "; std::string InlineContextStr = getInlineContextStr(GUID2FuncMAP); if (InlineContextStr.size()) { @@ -491,11 +501,19 @@ bool MCPseudoProbeDecoder::buildAddress2ProbeMap( } } + uint32_t Discriminator = 0; + if (hasDiscriminator(Attr)) { + auto ErrorOrDiscriminator = readUnsignedNumber(); + if (!ErrorOrDiscriminator) + return false; + Discriminator = std::move(*ErrorOrDiscriminator); + } + if (Cur && !isSentinelProbe(Attr)) { // Populate Address2ProbesMap auto &Probes = Address2ProbesMap[Addr]; Probes.emplace_back(Addr, Cur->Guid, Index, PseudoProbeType(Kind), Attr, - Cur); + Discriminator, Cur); Cur->addProbes(&Probes.back()); } LastAddr = Addr; diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp index 8f29153783066..0cb04594522b2 100644 --- a/llvm/lib/MC/MCStreamer.cpp +++ b/llvm/lib/MC/MCStreamer.cpp @@ -1105,7 +1105,7 @@ void MCStreamer::emitInstruction(const MCInst &Inst, const MCSubtargetInfo &) { } void MCStreamer::emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type, - uint64_t Attr, + uint64_t Attr, uint64_t Discriminator, const MCPseudoProbeInlineStack &InlineStack, MCSymbol *FnSym) { auto &Context = getContext(); @@ -1117,7 +1117,7 @@ void MCStreamer::emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type, emitLabel(ProbeSym); // Create a (local) probe entry with the symbol. - MCPseudoProbe Probe(ProbeSym, Guid, Index, Type, Attr); + MCPseudoProbe Probe(ProbeSym, Guid, Index, Type, Attr, Discriminator); // Add the probe entry to this section's entries. Context.getMCPseudoProbeTable().getProbeSections().addPseudoProbe( @@ -1196,7 +1196,7 @@ void MCStreamer::emitXCOFFRefDirective(const MCSymbol *Symbol) { } void MCStreamer::emitXCOFFExceptDirective(const MCSymbol *Symbol, - const MCSymbol *Trap, + const MCSymbol *Trap, unsigned Lang, unsigned Reason, unsigned FunctionSize, bool hasDebug) { diff --git a/llvm/test/tools/llvm-profgen/Inputs/fs-discriminator-probe.perfbin b/llvm/test/tools/llvm-profgen/Inputs/fs-discriminator-probe.perfbin new file mode 100755 index 0000000000000..c356e84ef1ae8 Binary files /dev/null and b/llvm/test/tools/llvm-profgen/Inputs/fs-discriminator-probe.perfbin differ diff --git a/llvm/test/tools/llvm-profgen/pseudoprobe-decoding-discriminator.test b/llvm/test/tools/llvm-profgen/pseudoprobe-decoding-discriminator.test new file mode 100644 index 0000000000000..efdc6399beb49 --- /dev/null +++ b/llvm/test/tools/llvm-profgen/pseudoprobe-decoding-discriminator.test @@ -0,0 +1,62 @@ +; RUN: llvm-profgen --format=text --perfscript=%s --binary=%S/Inputs/fs-discriminator-probe.perfbin --output=%t --show-pseudo-probe --show-disassembly-only | FileCheck %s + +; CHECK: : +; CHECK: [Probe]: FUNC: quick_sort Index: 1 Type: Block +; CHECK: [Probe]: FUNC: quick_sort Index: 1 Discriminator: 15360 Type: Block +; CHECK: [Probe]: FUNC: quick_sort Index: 4 Type: IndirectCall +; CHECK: [Probe]: FUNC: quick_sort Index: 5 Type: DirectCall + + +; original code: +; clang -O3 -g -mllvm --enable-fs-discriminator -fdebug-info-for-profiling -fpseudo-probe-for-profiling qsort.c -o a.out +#include +#include + +void swap(int *a, int *b) { + int t = *a; + *a = *b; + *b = t; +} + +int partition_pivot_last(int* array, int low, int high) { + int pivot = array[high]; + int i = low - 1; + for (int j = low; j < high; j++) + if (array[j] < pivot) + swap(&array[++i], &array[j]); + swap(&array[i + 1], &array[high]); + return (i + 1); +} + +int partition_pivot_first(int* array, int low, int high) { + int pivot = array[low]; + int i = low + 1; + for (int j = low + 1; j <= high; j++) + if (array[j] < pivot) { if (j != i) swap(&array[i], &array[j]); i++;} + swap(&array[i - 1], &array[low]); + return i - 1; +} + +void quick_sort(int* array, int low, int high, int (*partition_func)(int *, int, int)) { + if (low < high) { + int pi = (*partition_func)(array, low, high); + quick_sort(array, low, pi - 1, partition_func); + quick_sort(array, pi + 1, high, partition_func); + } +} + +int main() { + const int size = 200; + int sum = 0; + int *array = malloc(size * sizeof(int)); + for(int i = 0; i < 100 * 1000; i++) { + for(int j = 0; j < size; j++) + array[j] = j % 10 ? rand() % size: j; + int (*fptr)(int *, int, int) = i % 3 ? partition_pivot_last : partition_pivot_first; + quick_sort(array, 0, size - 1, fptr); + sum += array[i % size]; + } + printf("sum=%d\n", sum); + + return 0; +}