Skip to content

[BOLT] Compress YAML pseudo probe information #166680

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: users/aaupov/spr/main.bolt-compress-yaml-pseudo-probe-information
Choose a base branch
from
Draft

[BOLT] Compress YAML pseudo probe information #166680

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions bolt/include/bolt/Profile/ProfileYAMLMapping.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ struct BinaryBasicBlockProfile {
std::vector<CallSiteInfo> CallSites;
std::vector<SuccessorInfo> Successors;
std::vector<PseudoProbeInfo> PseudoProbes;
std::string PseudoProbesStr;

bool operator==(const BinaryBasicBlockProfile &Other) const {
return Index == Other.Index;
Expand All @@ -163,6 +164,7 @@ template <> struct MappingTraits<bolt::BinaryBasicBlockProfile> {
std::vector<bolt::SuccessorInfo>());
YamlIO.mapOptional("probes", BBP.PseudoProbes,
std::vector<bolt::PseudoProbeInfo>());
YamlIO.mapOptional("probe", BBP.PseudoProbesStr, std::string());
}
};

Expand Down Expand Up @@ -207,6 +209,7 @@ struct BinaryFunctionProfile {
uint64_t ExternEntryCount{0};
std::vector<BinaryBasicBlockProfile> Blocks;
std::vector<InlineTreeNode> InlineTree;
std::string InlineTreeStr;
bool Used{false};
};
} // end namespace bolt
Expand All @@ -223,6 +226,7 @@ template <> struct MappingTraits<bolt::BinaryFunctionProfile> {
std::vector<bolt::BinaryBasicBlockProfile>());
YamlIO.mapOptional("inline_tree", BFP.InlineTree,
std::vector<bolt::InlineTreeNode>());
YamlIO.mapOptional("ppit", BFP.InlineTreeStr, std::string());
}
};

Expand Down
7 changes: 3 additions & 4 deletions bolt/include/bolt/Profile/YAMLProfileWriter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,9 @@ class YAMLProfileWriter {
GUIDNumMap HashIdxMap;
};

static std::tuple<std::vector<yaml::bolt::InlineTreeNode>, InlineTreeMapTy>
convertBFInlineTree(const MCPseudoProbeDecoder &Decoder,
const InlineTreeDesc &InlineTree,
const BinaryFunction &BF);
static InlineTreeMapTy convertBFInlineTree(
const MCPseudoProbeDecoder &Decoder, const InlineTreeDesc &InlineTree,
const BinaryFunction &BF, yaml::bolt::BinaryFunctionProfile &YamlBF);

static std::tuple<yaml::bolt::ProfilePseudoProbeDesc, InlineTreeDesc>
convertPseudoProbeDesc(const MCPseudoProbeDecoder &PseudoProbeDecoder);
Expand Down
1 change: 1 addition & 0 deletions bolt/include/bolt/Utils/CommandLineOpts.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ enum GadgetScannerKind { GS_PACRET, GS_PAUTH, GS_ALL };

extern llvm::cl::bits<GadgetScannerKind> GadgetScannersToRun;

enum ProbesWriteMode : char { PWM_None = 0, PWM_Default, PWM_Compact };
} // namespace opts

namespace llvm {
Expand Down
7 changes: 3 additions & 4 deletions bolt/lib/Profile/DataAggregator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ MaxSamples("max-samples",
cl::cat(AggregatorCategory));

extern cl::opt<opts::ProfileFormatKind> ProfileFormat;
extern cl::opt<bool> ProfileWritePseudoProbes;
extern cl::opt<ProbesWriteMode> ProfileWritePseudoProbes;
extern cl::opt<std::string> SaveProfile;

cl::opt<bool> ReadPreAggregated(
Expand Down Expand Up @@ -2390,9 +2390,8 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
DenseMap<const MCDecodedPseudoProbeInlineTree *, uint32_t>
InlineTreeNodeId;
if (BF->getGUID()) {
std::tie(YamlBF.InlineTree, InlineTreeNodeId) =
YAMLProfileWriter::convertBFInlineTree(*PseudoProbeDecoder,
InlineTree, *BF);
InlineTreeNodeId = YAMLProfileWriter::convertBFInlineTree(
*PseudoProbeDecoder, InlineTree, *BF, YamlBF);
}
// Fetch probes belonging to all fragments
const AddressProbesMap &ProbeMap =
Expand Down
6 changes: 6 additions & 0 deletions bolt/lib/Profile/YAMLProfileReader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,7 @@ bool YAMLProfileReader::parseFunctionProfile(
static void
decodeYamlInlineTree(const yaml::bolt::ProfilePseudoProbeDesc &YamlPD,
yaml::bolt::BinaryFunctionProfile &YamlBF) {
// Decompress inline tree
uint32_t ParentId = 0;
uint32_t PrevGUIDIdx = 0;
for (yaml::bolt::InlineTreeNode &InlineTreeNode : YamlBF.InlineTree) {
Expand All @@ -384,6 +385,11 @@ decodeYamlInlineTree(const yaml::bolt::ProfilePseudoProbeDesc &YamlPD,
InlineTreeNode.Hash = YamlPD.Hash[HashIdx];
InlineTreeNode.ParentIndexDelta = ParentId;
}
// Decompress probe descriptors
for (yaml::bolt::BinaryBasicBlockProfile &BB : YamlBF.Blocks) {
if (BB.PseudoProbesStr.empty())
continue;
}
}

Error YAMLProfileReader::preprocessProfile(BinaryContext &BC) {
Expand Down
97 changes: 82 additions & 15 deletions bolt/lib/Profile/YAMLProfileWriter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include "bolt/Profile/ProfileReaderBase.h"
#include "bolt/Rewrite/RewriteInstance.h"
#include "bolt/Utils/CommandLineOpts.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/MC/MCPseudoProbe.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FileSystem.h"
Expand All @@ -25,10 +26,14 @@
namespace opts {
using namespace llvm;
extern cl::opt<bool> ProfileUseDFS;
cl::opt<bool> ProfileWritePseudoProbes(
cl::opt<ProbesWriteMode> ProfileWritePseudoProbes(
"profile-write-pseudo-probes",
cl::desc("Use pseudo probes in profile generation"), cl::Hidden,
cl::cat(BoltOptCategory));
cl::desc("Write pseudo probes into YAML profile"),
cl::init(ProbesWriteMode::PWM_None),
cl::values(clEnumValN(ProbesWriteMode::PWM_Default, "", "default"),
clEnumValN(ProbesWriteMode::PWM_Compact, "compact",
"compressed encoding")),
cl::ValueOptional, cl::cat(BoltOutputCategory));
} // namespace opts

namespace llvm {
Expand Down Expand Up @@ -149,17 +154,57 @@ void YAMLProfileWriter::BlockProbeCtx::finalize(
}
};

auto compressVec = [](auto Vec, std::string &Out, bool Probes) {
if (Vec.size() == 1) {
if (Probes ? Vec.front() == 1 : Vec.front() == 0)
return;
}
// Invariant: current run is [StartGroup, EndGroup]
auto StartGroup = Vec.begin();
auto EndGroup = Vec.begin();
ListSeparator LS(",");
while (StartGroup != Vec.end()) {
++EndGroup;
const size_t Distance = EndGroup - StartGroup;
const uint64_t Delta = *EndGroup - *StartGroup;
const bool AtEnd = EndGroup == Vec.end();
// Happy case: advance the group
if (!AtEnd && Delta == Distance)
continue;
// Otherwise print the current group
Out += LS;
APInt Int(64, *StartGroup);
Out += toString(Int, 36, false);

Int = APInt(64, Distance - 1, false);
if (Distance > 1)
Out += '^' + toString(Int, 36, false);
if (AtEnd)
break;
StartGroup = EndGroup;
}
};

// Check identical block probes and merge them
std::unordered_map<std::vector<uint64_t>, std::vector<uint32_t>, ProbeHasher>
ProbesToNodes;
for (auto &[NodeId, Probes] : NodeToProbes) {
llvm::sort(Probes);
ProbesToNodes[Probes].emplace_back(NodeId);
}
ListSeparator LS(" ");
std::string &Probe = YamlBB.PseudoProbesStr;
for (auto &[Probes, Nodes] : ProbesToNodes) {
llvm::sort(Nodes);
YamlBB.PseudoProbes.emplace_back(
yaml::bolt::PseudoProbeInfo{Probes, Nodes});
if (opts::ProfileWritePseudoProbes == opts::ProbesWriteMode::PWM_Default) {
YamlBB.PseudoProbes.emplace_back(
yaml::bolt::PseudoProbeInfo{Probes, Nodes});
} else {
Probe += LS;
compressVec(Probes, Probe, true);
Probe += "_";
compressVec(Nodes, Probe, false);
}
}
for (yaml::bolt::CallSiteInfo &CSI : YamlBB.CallSites) {
auto It = CallProbes.find(CSI.Offset);
Expand All @@ -183,18 +228,17 @@ void YAMLProfileWriter::BlockProbeCtx::finalize(
}
}

std::tuple<std::vector<yaml::bolt::InlineTreeNode>,
YAMLProfileWriter::InlineTreeMapTy>
YAMLProfileWriter::convertBFInlineTree(const MCPseudoProbeDecoder &Decoder,
const InlineTreeDesc &InlineTree,
const BinaryFunction &BF) {
YAMLProfileWriter::InlineTreeMapTy YAMLProfileWriter::convertBFInlineTree(
const MCPseudoProbeDecoder &Decoder, const InlineTreeDesc &InlineTree,
const BinaryFunction &BF, yaml::bolt::BinaryFunctionProfile &YamlBF) {
DenseMap<const MCDecodedPseudoProbeInlineTree *, uint32_t> InlineTreeNodeId;
std::vector<yaml::bolt::InlineTreeNode> YamlInlineTree;
std::string InlineTreeStr;
uint64_t Addr = BF.getAddress();
uint64_t Size = BF.getSize();
auto Probes = Decoder.getAddress2ProbesMap().find(Addr, Addr + Size);
if (Probes.empty())
return {YamlInlineTree, InlineTreeNodeId};
return InlineTreeNodeId;
const MCDecodedPseudoProbe &Probe = *Probes.begin();
const MCDecodedPseudoProbeInlineTree *Root = Probe.getInlineTreeNode();
while (Root->hasInlineSite())
Expand All @@ -215,7 +259,30 @@ YAMLProfileWriter::convertBFInlineTree(const MCPseudoProbeDecoder &Decoder,
Node.ParentId - PrevParent, Node.InlineSite, GUIDIdx, 0, 0});
PrevParent = Node.ParentId;
}
return {YamlInlineTree, InlineTreeNodeId};
if (opts::ProfileWritePseudoProbes == opts::ProbesWriteMode::PWM_Default) {
YamlBF.InlineTree = YamlInlineTree;
return InlineTreeNodeId;
}
assert(opts::ProfileWritePseudoProbes == opts::ProbesWriteMode::PWM_Compact);
// Write compact form
std::string &Out = YamlBF.InlineTreeStr;
raw_string_ostream OS(Out);
ListSeparator LS(" ");
for (const yaml::bolt::InlineTreeNode &Node : YamlInlineTree) {
Out += LS;
APInt Int(32, Node.GUIDIndex);
if (Node.GUIDIndex != UINT32_MAX)
Out += toString(Int, 36, false);
Out += '_';
Int = APInt(32, Node.ParentIndexDelta);
if (Node.ParentIndexDelta)
Out += toString(Int, 36, false);
Out += '_';
Int = APInt(32, Node.CallSiteProbe);
if (Node.CallSiteProbe)
Out += toString(Int, 36, false);
}
return InlineTreeNodeId;
}

yaml::bolt::BinaryFunctionProfile
Expand All @@ -241,8 +308,8 @@ YAMLProfileWriter::convert(const BinaryFunction &BF, bool UseDFS,
YamlBF.ExternEntryCount = BF.getExternEntryCount();
DenseMap<const MCDecodedPseudoProbeInlineTree *, uint32_t> InlineTreeNodeId;
if (PseudoProbeDecoder && BF.getGUID()) {
std::tie(YamlBF.InlineTree, InlineTreeNodeId) =
convertBFInlineTree(*PseudoProbeDecoder, InlineTree, BF);
InlineTreeNodeId =
convertBFInlineTree(*PseudoProbeDecoder, InlineTree, BF, YamlBF);
}

BinaryFunction::BasicBlockOrderType Order;
Expand Down Expand Up @@ -419,7 +486,7 @@ std::error_code YAMLProfileWriter::writeProfile(const RewriteInstance &RI) {
// Add probe inline tree nodes.
InlineTreeDesc InlineTree;
if (const MCPseudoProbeDecoder *Decoder =
opts::ProfileWritePseudoProbes ? BC.getPseudoProbeDecoder() : nullptr)
opts::ProfileWritePseudoProbes ? BC.getPseudoProbeDecoder() : nullptr)
std::tie(BP.PseudoProbeDesc, InlineTree) = convertPseudoProbeDesc(*Decoder);

// Add all function objects.
Expand Down
2 changes: 1 addition & 1 deletion bolt/lib/Rewrite/PseudoProbeRewriter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ static cl::opt<PrintPseudoProbesOptions> PrintPseudoProbes(
clEnumValN(PPP_All, "all", "enable all debugging printout")),
cl::Hidden, cl::cat(BoltCategory));

extern cl::opt<bool> ProfileWritePseudoProbes;
extern cl::opt<ProbesWriteMode> ProfileWritePseudoProbes;
extern cl::opt<bool> StaleMatchingWithPseudoProbes;
} // namespace opts

Expand Down
Loading
You are viewing a condensed version of this merge commit. You can view the full changes here.