Skip to content

Commit

Permalink
[PseudoProbe] Encode/Decode FS discriminator
Browse files Browse the repository at this point in the history
Encoding FS discriminators for block probes. Decoding them correspondingly.

The encoding/decoding of FS discriminators are conditional, only for probes with a non-zero discriminator. This saves encoding size, also ensures downwards-compatiblity.

Reviewed By: wenlei

Differential Revision: https://reviews.llvm.org/D147651
  • Loading branch information
htyu committed May 10, 2023
1 parent 9272d0f commit 9849291
Show file tree
Hide file tree
Showing 10 changed files with 149 additions and 47 deletions.
7 changes: 6 additions & 1 deletion llvm/include/llvm/IR/PseudoProbe.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ enum class PseudoProbeType { Block = 0, IndirectCall, DirectCall };

enum class PseudoProbeAttributes {
Reserved = 0x1,
Sentinel = 0x2, // A place holder for split function entry address.
Sentinel = 0x2, // A place holder for split function entry address.
HasDiscriminator = 0x4, // for probes with a discriminator
};

// The saturated distrution factor representing 100% for block probes.
Expand Down Expand Up @@ -91,6 +92,10 @@ static inline bool isSentinelProbe(uint32_t Flags) {
return Flags & (uint32_t)PseudoProbeAttributes::Sentinel;
}

static inline bool hasDiscriminator(uint32_t Flags) {
return Flags & (uint32_t)PseudoProbeAttributes::HasDiscriminator;
}

std::optional<PseudoProbe> extractProbe(const Instruction &Inst);

void setProbeDistributionFactor(Instruction &Inst, float Factor);
Expand Down
20 changes: 14 additions & 6 deletions llvm/include/llvm/MC/MCPseudoProbe.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,15 @@
// 0 - block probe, 1 - indirect call, 2 - direct call
// ATTRIBUTE (uint3)
// 1 - reserved
// 2 - Sentinel
// 4 - HasDiscriminator
// ADDRESS_TYPE (uint1)
// 0 - code address for regular probes (for downwards compatibility)
// - GUID of linkage name for sentinel probes
// 1 - address delta
// CODE_ADDRESS (uint64 or ULEB128)
// code address or address delta, depending on ADDRESS_TYPE
// DISCRIMINATOR (ULEB128) if HasDiscriminator
// INLINED FUNCTION RECORDS
// A list of NUM_INLINED_FUNCTIONS entries describing each of the inlined
// callees. Each record contains:
Expand Down Expand Up @@ -108,6 +111,7 @@ class MCPseudoProbeBase {
protected:
uint64_t Guid;
uint64_t Index;
uint32_t Discriminator;
uint8_t Attributes;
uint8_t Type;
// The value should be equal to PseudoProbeReservedId::Last + 1 which is
Expand All @@ -116,15 +120,17 @@ class MCPseudoProbeBase {
const static uint32_t PseudoProbeFirstId = 1;

public:
MCPseudoProbeBase(uint64_t G, uint64_t I, uint64_t At, uint8_t T)
: Guid(G), Index(I), Attributes(At), Type(T) {}
MCPseudoProbeBase(uint64_t G, uint64_t I, uint64_t At, uint8_t T, uint32_t D)
: Guid(G), Index(I), Discriminator(D), Attributes(At), Type(T) {}

bool isEntry() const { return Index == PseudoProbeFirstId; }

uint64_t getGuid() const { return Guid; }

uint64_t getIndex() const { return Index; }

uint32_t getDiscriminator() const { return Discriminator; }

uint8_t getAttributes() const { return Attributes; }

uint8_t getType() const { return Type; }
Expand Down Expand Up @@ -155,8 +161,9 @@ class MCPseudoProbe : public MCPseudoProbeBase {

public:
MCPseudoProbe(MCSymbol *Label, uint64_t Guid, uint64_t Index, uint64_t Type,
uint64_t Attributes)
: MCPseudoProbeBase(Guid, Index, Attributes, Type), Label(Label) {
uint64_t Attributes, uint32_t Discriminator)
: MCPseudoProbeBase(Guid, Index, Attributes, Type, Discriminator),
Label(Label) {
assert(Type <= 0xFF && "Probe type too big to encode, exceeding 2^8");
assert(Attributes <= 0xFF &&
"Probe attributes too big to encode, exceeding 2^16");
Expand All @@ -175,8 +182,9 @@ class MCDecodedPseudoProbe : public MCPseudoProbeBase {

public:
MCDecodedPseudoProbe(uint64_t Ad, uint64_t G, uint32_t I, PseudoProbeType K,
uint8_t At, MCDecodedPseudoProbeInlineTree *Tree)
: MCPseudoProbeBase(G, I, At, static_cast<uint8_t>(K)), Address(Ad),
uint8_t At, uint32_t D,
MCDecodedPseudoProbeInlineTree *Tree)
: MCPseudoProbeBase(G, I, At, static_cast<uint8_t>(K), D), Address(Ad),
InlineTree(Tree){};

uint64_t getAddress() const { return Address; }
Expand Down
4 changes: 2 additions & 2 deletions llvm/include/llvm/MC/MCStreamer.h
Original file line number Diff line number Diff line change
Expand Up @@ -636,7 +636,7 @@ class MCStreamer {
/// \param Symbol - The function containing the trap.
/// \param Lang - The language code for the exception entry.
/// \param Reason - The reason code for the exception entry.
virtual void emitXCOFFExceptDirective(const MCSymbol *Symbol,
virtual void emitXCOFFExceptDirective(const MCSymbol *Symbol,
const MCSymbol *Trap,
unsigned Lang, unsigned Reason,
unsigned FunctionSize, bool hasDebug);
Expand Down Expand Up @@ -1100,7 +1100,7 @@ class MCStreamer {

/// Emit the a pseudo probe into the current section.
virtual void emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type,
uint64_t Attr,
uint64_t Attr, uint64_t Discriminator,
const MCPseudoProbeInlineStack &InlineStack,
MCSymbol *FnSym);

Expand Down
11 changes: 8 additions & 3 deletions llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,13 @@ void PseudoProbeHandler::emitPseudoProbe(uint64_t Guid, uint64_t Index,
ReversedInlineStack.emplace_back(CallerGuid, CallerProbeId);
InlinedAt = InlinedAt->getInlinedAt();
}

uint64_t Discriminator = 0;
// For now only block probes have FS discriminators. See
// MIRFSDiscriminator.cpp for more details.
if (DebugLoc &&
!DILocation::isPseudoProbeDiscriminator(DebugLoc->getDiscriminator()))
Discriminator = DebugLoc->getDiscriminator();
SmallVector<InlineSite, 8> InlineStack(llvm::reverse(ReversedInlineStack));
Asm->OutStreamer->emitPseudoProbe(Guid, Index, Type, Attr, InlineStack,
Asm->CurrentFnSym);
Asm->OutStreamer->emitPseudoProbe(Guid, Index, Type, Attr, Discriminator,
InlineStack, Asm->CurrentFnSym);
}
22 changes: 13 additions & 9 deletions llvm/lib/MC/MCAsmStreamer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ class MCAsmStreamer final : public MCStreamer {

void emitXCOFFRefDirective(const MCSymbol *Symbol) override;

void emitXCOFFExceptDirective(const MCSymbol *Symbol,
void emitXCOFFExceptDirective(const MCSymbol *Symbol,
const MCSymbol *Trap,
unsigned Lang, unsigned Reason,
unsigned FunctionSize, bool hasDebug) override;
Expand Down Expand Up @@ -377,8 +377,9 @@ class MCAsmStreamer final : public MCStreamer {
void emitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI) override;

void emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type,
uint64_t Attr,
const MCPseudoProbeInlineStack &InlineStack, MCSymbol *FnSym) override;
uint64_t Attr, uint64_t Discriminator,
const MCPseudoProbeInlineStack &InlineStack,
MCSymbol *FnSym) override;

void emitBundleAlignMode(Align Alignment) override;
void emitBundleLock(bool AlignToEnd) override;
Expand Down Expand Up @@ -950,7 +951,7 @@ void MCAsmStreamer::emitXCOFFRefDirective(const MCSymbol *Symbol) {
}

void MCAsmStreamer::emitXCOFFExceptDirective(const MCSymbol *Symbol,
const MCSymbol *Trap,
const MCSymbol *Trap,
unsigned Lang,
unsigned Reason,
unsigned FunctionSize,
Expand Down Expand Up @@ -2336,11 +2337,14 @@ void MCAsmStreamer::emitInstruction(const MCInst &Inst,
EmitEOL();
}

void MCAsmStreamer::emitPseudoProbe(
uint64_t Guid, uint64_t Index, uint64_t Type, uint64_t Attr,
const MCPseudoProbeInlineStack &InlineStack, MCSymbol *FnSym) {
OS << "\t.pseudoprobe\t" << Guid << " " << Index << " " << Type << " "
<< Attr;
void MCAsmStreamer::emitPseudoProbe(uint64_t Guid, uint64_t Index,
uint64_t Type, uint64_t Attr,
uint64_t Discriminator,
const MCPseudoProbeInlineStack &InlineStack,
MCSymbol *FnSym) {
OS << "\t.pseudoprobe\t" << Guid << " " << Index << " " << Type << " " << Attr;
if (Discriminator)
OS << " " << Discriminator;
// Emit inline stack like
// @ GUIDmain:3 @ GUIDCaller:1 @ GUIDDirectCaller:11
for (const auto &Site : InlineStack)
Expand Down
30 changes: 15 additions & 15 deletions llvm/lib/MC/MCParser/AsmParser.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5853,24 +5853,23 @@ bool AsmParser::parseDirectivePseudoProbe() {
int64_t Index;
int64_t Type;
int64_t Attr;
int64_t Discriminator = 0;

if (getLexer().is(AsmToken::Integer)) {
if (parseIntToken(Guid, "unexpected token in '.pseudoprobe' directive"))
return true;
}
if (parseIntToken(Guid, "unexpected token in '.pseudoprobe' directive"))
return true;

if (getLexer().is(AsmToken::Integer)) {
if (parseIntToken(Index, "unexpected token in '.pseudoprobe' directive"))
return true;
}
if (parseIntToken(Index, "unexpected token in '.pseudoprobe' directive"))
return true;

if (getLexer().is(AsmToken::Integer)) {
if (parseIntToken(Type, "unexpected token in '.pseudoprobe' directive"))
return true;
}
if (parseIntToken(Type, "unexpected token in '.pseudoprobe' directive"))
return true;

if (getLexer().is(AsmToken::Integer)) {
if (parseIntToken(Attr, "unexpected token in '.pseudoprobe' directive"))
if (parseIntToken(Attr, "unexpected token in '.pseudoprobe' directive"))
return true;

if (hasDiscriminator(Attr)) {
if (parseIntToken(Discriminator,
"unexpected token in '.pseudoprobe' directive"))
return true;
}

Expand Down Expand Up @@ -5912,7 +5911,8 @@ bool AsmParser::parseDirectivePseudoProbe() {
if (parseEOL())
return true;

getStreamer().emitPseudoProbe(Guid, Index, Type, Attr, InlineStack, FnSym);
getStreamer().emitPseudoProbe(Guid, Index, Type, Attr, Discriminator,
InlineStack, FnSym);
return false;
}

Expand Down
34 changes: 26 additions & 8 deletions llvm/lib/MC/MCPseudoProbe.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

#include "llvm/MC/MCPseudoProbe.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/IR/PseudoProbe.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
Expand Down Expand Up @@ -58,10 +59,14 @@ void MCPseudoProbe::emit(MCObjectStreamer *MCOS,
// Type (bit 0 to 3), with bit 4 to 6 for attributes.
// Flag (bit 7, 0 - code address, 1 - address delta). This indicates whether
// the following field is a symbolic code address or an address delta.
// Emit FS discriminator
assert(Type <= 0xF && "Probe type too big to encode, exceeding 15");
assert(Attributes <= 0x7 &&
auto NewAttributes = Attributes;
if (Discriminator)
NewAttributes |= (uint32_t)PseudoProbeAttributes::HasDiscriminator;
assert(NewAttributes <= 0x7 &&
"Probe attributes too big to encode, exceeding 7");
uint8_t PackedType = Type | (Attributes << 4);
uint8_t PackedType = Type | (NewAttributes << 4);
uint8_t Flag =
!IsSentinel ? ((int8_t)MCPseudoProbeFlag::AddressDelta << 7) : 0;
MCOS->emitInt8(Flag | PackedType);
Expand All @@ -81,6 +86,9 @@ void MCPseudoProbe::emit(MCObjectStreamer *MCOS,
MCOS->emitInt64(Guid);
}

if (Discriminator)
MCOS->emitULEB128IntValue(Discriminator);

LLVM_DEBUG({
dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
dbgs() << "Probe: " << Index << "\n";
Expand Down Expand Up @@ -222,11 +230,11 @@ void MCPseudoProbeSections::emit(MCObjectStreamer *MCOS) {

for (const auto &Inlinee : Inlinees) {
// Emit the group guarded by a sentinel probe.
MCPseudoProbe SentinelProbe(const_cast<MCSymbol *>(FuncSym),
MD5Hash(FuncSym->getName()),
(uint32_t)PseudoProbeReservedId::Invalid,
(uint32_t)PseudoProbeType::Block,
(uint32_t)PseudoProbeAttributes::Sentinel);
MCPseudoProbe SentinelProbe(
const_cast<MCSymbol *>(FuncSym), MD5Hash(FuncSym->getName()),
(uint32_t)PseudoProbeReservedId::Invalid,
(uint32_t)PseudoProbeType::Block,
(uint32_t)PseudoProbeAttributes::Sentinel, 0);
const MCPseudoProbe *Probe = &SentinelProbe;
Inlinee.second->emit(MCOS, Probe);
}
Expand Down Expand Up @@ -310,6 +318,8 @@ void MCDecodedPseudoProbe::print(raw_ostream &OS,
OS << Guid << " ";
}
OS << "Index: " << Index << " ";
if (Discriminator)
OS << "Discriminator: " << Discriminator << " ";
OS << "Type: " << PseudoProbeTypeStr[static_cast<uint8_t>(Type)] << " ";
std::string InlineContextStr = getInlineContextStr(GUID2FuncMAP);
if (InlineContextStr.size()) {
Expand Down Expand Up @@ -491,11 +501,19 @@ bool MCPseudoProbeDecoder::buildAddress2ProbeMap(
}
}

uint32_t Discriminator = 0;
if (hasDiscriminator(Attr)) {
auto ErrorOrDiscriminator = readUnsignedNumber<uint32_t>();
if (!ErrorOrDiscriminator)
return false;
Discriminator = std::move(*ErrorOrDiscriminator);
}

if (Cur && !isSentinelProbe(Attr)) {
// Populate Address2ProbesMap
auto &Probes = Address2ProbesMap[Addr];
Probes.emplace_back(Addr, Cur->Guid, Index, PseudoProbeType(Kind), Attr,
Cur);
Discriminator, Cur);
Cur->addProbes(&Probes.back());
}
LastAddr = Addr;
Expand Down
6 changes: 3 additions & 3 deletions llvm/lib/MC/MCStreamer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1105,7 +1105,7 @@ void MCStreamer::emitInstruction(const MCInst &Inst, const MCSubtargetInfo &) {
}

void MCStreamer::emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type,
uint64_t Attr,
uint64_t Attr, uint64_t Discriminator,
const MCPseudoProbeInlineStack &InlineStack,
MCSymbol *FnSym) {
auto &Context = getContext();
Expand All @@ -1117,7 +1117,7 @@ void MCStreamer::emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type,
emitLabel(ProbeSym);

// Create a (local) probe entry with the symbol.
MCPseudoProbe Probe(ProbeSym, Guid, Index, Type, Attr);
MCPseudoProbe Probe(ProbeSym, Guid, Index, Type, Attr, Discriminator);

// Add the probe entry to this section's entries.
Context.getMCPseudoProbeTable().getProbeSections().addPseudoProbe(
Expand Down Expand Up @@ -1196,7 +1196,7 @@ void MCStreamer::emitXCOFFRefDirective(const MCSymbol *Symbol) {
}

void MCStreamer::emitXCOFFExceptDirective(const MCSymbol *Symbol,
const MCSymbol *Trap,
const MCSymbol *Trap,
unsigned Lang, unsigned Reason,
unsigned FunctionSize,
bool hasDebug) {
Expand Down
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
; RUN: llvm-profgen --format=text --perfscript=%s --binary=%S/Inputs/fs-discriminator-probe.perfbin --output=%t --show-pseudo-probe --show-disassembly-only | FileCheck %s

; CHECK: <quick_sort>:
; CHECK: [Probe]: FUNC: quick_sort Index: 1 Type: Block
; CHECK: [Probe]: FUNC: quick_sort Index: 1 Discriminator: 15360 Type: Block
; CHECK: [Probe]: FUNC: quick_sort Index: 4 Type: IndirectCall
; CHECK: [Probe]: FUNC: quick_sort Index: 5 Type: DirectCall


; original code:
; clang -O3 -g -mllvm --enable-fs-discriminator -fdebug-info-for-profiling -fpseudo-probe-for-profiling qsort.c -o a.out
#include <stdio.h>
#include <stdlib.h>

void swap(int *a, int *b) {
int t = *a;
*a = *b;
*b = t;
}

int partition_pivot_last(int* array, int low, int high) {
int pivot = array[high];
int i = low - 1;
for (int j = low; j < high; j++)
if (array[j] < pivot)
swap(&array[++i], &array[j]);
swap(&array[i + 1], &array[high]);
return (i + 1);
}

int partition_pivot_first(int* array, int low, int high) {
int pivot = array[low];
int i = low + 1;
for (int j = low + 1; j <= high; j++)
if (array[j] < pivot) { if (j != i) swap(&array[i], &array[j]); i++;}
swap(&array[i - 1], &array[low]);
return i - 1;
}

void quick_sort(int* array, int low, int high, int (*partition_func)(int *, int, int)) {
if (low < high) {
int pi = (*partition_func)(array, low, high);
quick_sort(array, low, pi - 1, partition_func);
quick_sort(array, pi + 1, high, partition_func);
}
}

int main() {
const int size = 200;
int sum = 0;
int *array = malloc(size * sizeof(int));
for(int i = 0; i < 100 * 1000; i++) {
for(int j = 0; j < size; j++)
array[j] = j % 10 ? rand() % size: j;
int (*fptr)(int *, int, int) = i % 3 ? partition_pivot_last : partition_pivot_first;
quick_sort(array, 0, size - 1, fptr);
sum += array[i % size];
}
printf("sum=%d\n", sum);

return 0;
}

0 comments on commit 9849291

Please sign in to comment.