diff --git a/llvm/include/llvm/Analysis/CtxProfAnalysis.h b/llvm/include/llvm/Analysis/CtxProfAnalysis.h index ea292250c63a9..c3d9b429c1515 100644 --- a/llvm/include/llvm/Analysis/CtxProfAnalysis.h +++ b/llvm/include/llvm/Analysis/CtxProfAnalysis.h @@ -21,12 +21,6 @@ namespace llvm { class CtxProfAnalysis; -// Setting initial capacity to 1 because all contexts must have at least 1 -// counter, and then, because all contexts belonging to a function have the same -// size, there'll be at most one other heap allocation. -using CtxProfFlatProfile = - std::map>; - /// The instrumented contextual profile, produced by the CtxProfAnalysis. class PGOContextualProfile { friend class CtxProfAnalysis; @@ -38,7 +32,7 @@ class PGOContextualProfile { PGOCtxProfContext Index; FunctionInfo(StringRef Name) : Name(Name) {} }; - std::optional Profiles; + std::optional Profile; // For the GUIDs in this module, associate metadata about each function which // we'll need when we maintain the profiles during IPO transformations. std::map FuncInfo; @@ -56,12 +50,12 @@ class PGOContextualProfile { PGOContextualProfile(const PGOContextualProfile &) = delete; PGOContextualProfile(PGOContextualProfile &&) = default; - operator bool() const { return Profiles.has_value(); } - - const PGOCtxProfContext::CallTargetMapTy &profiles() const { - return *Profiles; + const PGOCtxProfContext::CallTargetMapTy &contexts() const { + return Profile->Contexts; } + const PGOCtxProfile &profiles() const { return *Profile; } + bool isFunctionKnown(const Function &F) const { return getDefinedFunctionGUID(F) != 0; } diff --git a/llvm/include/llvm/ProfileData/PGOCtxProfReader.h b/llvm/include/llvm/ProfileData/PGOCtxProfReader.h index ffffae1a872a5..52a1b42f87ac0 100644 --- a/llvm/include/llvm/ProfileData/PGOCtxProfReader.h +++ b/llvm/include/llvm/ProfileData/PGOCtxProfReader.h @@ -164,6 +164,24 @@ class PGOCtxProfContext final : public internal::IndexNode { } }; +// Setting initial capacity to 1 because all contexts must have at least 1 +// counter, and then, because all contexts belonging to a function have the same +// size, there'll be at most one other heap allocation. +using CtxProfFlatProfile = + std::map>; + +using CtxProfContextualProfiles = + std::map; +struct PGOCtxProfile { + CtxProfContextualProfiles Contexts; + CtxProfFlatProfile FlatProfiles; + + PGOCtxProfile() = default; + PGOCtxProfile(const PGOCtxProfile &) = delete; + PGOCtxProfile(PGOCtxProfile &&) = default; + PGOCtxProfile &operator=(PGOCtxProfile &&) = default; +}; + class PGOCtxProfileReader final { StringRef Magic; BitstreamCursor Cursor; @@ -173,18 +191,21 @@ class PGOCtxProfileReader final { Error unsupported(const Twine &); Expected, PGOCtxProfContext>> - readContext(bool ExpectIndex); - bool canReadContext(); + readProfile(PGOCtxProfileBlockIDs Kind); + + bool canEnterBlockWithID(PGOCtxProfileBlockIDs ID); + Error enterBlockWithID(PGOCtxProfileBlockIDs ID); + + Error loadContexts(CtxProfContextualProfiles &); public: PGOCtxProfileReader(StringRef Buffer) : Magic(Buffer.substr(0, PGOCtxProfileWriter::ContainerMagic.size())), Cursor(Buffer.substr(PGOCtxProfileWriter::ContainerMagic.size())) {} - Expected> loadContexts(); + Expected loadProfiles(); }; -void convertCtxProfToYaml(raw_ostream &OS, - const PGOCtxProfContext::CallTargetMapTy &); +void convertCtxProfToYaml(raw_ostream &OS, const PGOCtxProfile &); } // namespace llvm #endif diff --git a/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h b/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h index 43a190ae0aa05..81b59f72185b1 100644 --- a/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h +++ b/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h @@ -23,7 +23,9 @@ enum PGOCtxProfileRecords { Invalid = 0, Version, Guid, CalleeIndex, Counters }; enum PGOCtxProfileBlockIDs { ProfileMetadataBlockID = bitc::FIRST_APPLICATION_BLOCKID, - ContextNodeBlockID = ProfileMetadataBlockID + 1 + ContextsSectionBlockID = ProfileMetadataBlockID + 1, + ContextRootBlockID = ContextsSectionBlockID + 1, + ContextNodeBlockID = ContextRootBlockID + 1, }; /// Write one or more ContextNodes to the provided raw_fd_stream. @@ -60,10 +62,13 @@ enum PGOCtxProfileBlockIDs { /// like value profiling - which would appear as additional records. For /// example, value profiling would produce a new record with a new record ID, /// containing the profiled values (much like the counters) -class PGOCtxProfileWriter final { +class PGOCtxProfileWriter : public ctx_profile::ProfileWriter { + enum class EmptyContextCriteria { None, EntryIsZero, AllAreZero }; + BitstreamWriter Writer; - void writeCounters(const ctx_profile::ContextNode &Node); + void writeGuid(ctx_profile::GUID Guid); + void writeCounters(ArrayRef Counters); void writeImpl(std::optional CallerIndex, const ctx_profile::ContextNode &Node); @@ -72,11 +77,13 @@ class PGOCtxProfileWriter final { std::optional VersionOverride = std::nullopt); ~PGOCtxProfileWriter() { Writer.ExitBlock(); } - void write(const ctx_profile::ContextNode &); + void startContextSection() override; + void writeContextual(const ctx_profile::ContextNode &RootNode) override; + void endContextSection() override; // constants used in writing which a reader may find useful. static constexpr unsigned CodeLen = 2; - static constexpr uint32_t CurrentVersion = 1; + static constexpr uint32_t CurrentVersion = 2; static constexpr unsigned VBREncodingBits = 6; static constexpr StringRef ContainerMagic = "CTXP"; }; diff --git a/llvm/lib/Analysis/CtxProfAnalysis.cpp b/llvm/lib/Analysis/CtxProfAnalysis.cpp index bbf29e0d370e7..3f87e408bfefe 100644 --- a/llvm/lib/Analysis/CtxProfAnalysis.cpp +++ b/llvm/lib/Analysis/CtxProfAnalysis.cpp @@ -88,7 +88,7 @@ PGOContextualProfile CtxProfAnalysis::run(Module &M, return {}; } PGOCtxProfileReader Reader(MB.get()->getBuffer()); - auto MaybeCtx = Reader.loadContexts(); + auto MaybeCtx = Reader.loadProfiles(); if (!MaybeCtx) { M.getContext().emitError("contextual profile file is invalid: " + toString(MaybeCtx.takeError())); @@ -99,16 +99,16 @@ PGOContextualProfile CtxProfAnalysis::run(Module &M, for (const auto &F : M) if (!F.isDeclaration()) if (auto GUID = AssignGUIDPass::getGUID(F); - MaybeCtx->find(GUID) != MaybeCtx->end()) + MaybeCtx->Contexts.find(GUID) != MaybeCtx->Contexts.end()) ProfileRootsInModule.insert(GUID); // Trim first the roots that aren't in this module. - for (auto &[RootGuid, _] : llvm::make_early_inc_range(*MaybeCtx)) + for (auto &[RootGuid, _] : llvm::make_early_inc_range(MaybeCtx->Contexts)) if (!ProfileRootsInModule.contains(RootGuid)) - MaybeCtx->erase(RootGuid); + MaybeCtx->Contexts.erase(RootGuid); // If none of the roots are in the module, we have no profile (for this // module) - if (MaybeCtx->empty()) + if (MaybeCtx->Contexts.empty()) return {}; // OK, so we have a valid profile and it's applicable to roots in this module. @@ -146,7 +146,7 @@ PGOContextualProfile CtxProfAnalysis::run(Module &M, } // If we made it this far, the Result is valid - which we mark by setting // .Profiles. - Result.Profiles = std::move(*MaybeCtx); + Result.Profile = std::move(*MaybeCtx); Result.initIndex(); return Result; } @@ -164,7 +164,7 @@ CtxProfAnalysisPrinterPass::CtxProfAnalysisPrinterPass(raw_ostream &OS) PreservedAnalyses CtxProfAnalysisPrinterPass::run(Module &M, ModuleAnalysisManager &MAM) { CtxProfAnalysis::Result &C = MAM.getResult(M); - if (!C) { + if (C.contexts().empty()) { OS << "No contextual profile was provided.\n"; return PreservedAnalyses::all(); } @@ -245,7 +245,7 @@ void PGOContextualProfile::initIndex() { for (auto &[Guid, FI] : FuncInfo) InsertionPoints[Guid] = &FI.Index; preorderVisit( - *Profiles, [&](PGOCtxProfContext &Ctx) { + Profile->Contexts, [&](PGOCtxProfContext &Ctx) { auto InsertIt = InsertionPoints.find(Ctx.guid()); if (InsertIt == InsertionPoints.end()) return; @@ -270,7 +270,7 @@ void PGOContextualProfile::update(Visitor V, const Function &F) { void PGOContextualProfile::visit(ConstVisitor V, const Function *F) const { if (!F) return preorderVisit(*Profiles, V); + const PGOCtxProfContext>(Profile->Contexts, V); assert(isFunctionKnown(*F)); GlobalValue::GUID G = getDefinedFunctionGUID(*F); for (const auto *Node = FuncInfo.find(G)->second.Index.Next; Node; @@ -279,11 +279,11 @@ void PGOContextualProfile::visit(ConstVisitor V, const Function *F) const { } const CtxProfFlatProfile PGOContextualProfile::flatten() const { - assert(Profiles.has_value()); + assert(Profile.has_value()); CtxProfFlatProfile Flat; preorderVisit( - *Profiles, [&](const PGOCtxProfContext &Ctx) { + Profile->Contexts, [&](const PGOCtxProfContext &Ctx) { auto [It, Ins] = Flat.insert({Ctx.guid(), {}}); if (Ins) { llvm::append_range(It->second, Ctx.counters()); diff --git a/llvm/lib/ProfileData/PGOCtxProfReader.cpp b/llvm/lib/ProfileData/PGOCtxProfReader.cpp index e1363cfafdfd4..d44d17514ae66 100644 --- a/llvm/lib/ProfileData/PGOCtxProfReader.cpp +++ b/llvm/lib/ProfileData/PGOCtxProfReader.cpp @@ -58,19 +58,25 @@ Error PGOCtxProfileReader::unsupported(const Twine &Msg) { return make_error(instrprof_error::unsupported_version, Msg); } -bool PGOCtxProfileReader::canReadContext() { +bool PGOCtxProfileReader::canEnterBlockWithID(PGOCtxProfileBlockIDs ID) { auto Blk = advance(); if (!Blk) { consumeError(Blk.takeError()); return false; } - return Blk->Kind == BitstreamEntry::SubBlock && - Blk->ID == PGOCtxProfileBlockIDs::ContextNodeBlockID; + return Blk->Kind == BitstreamEntry::SubBlock && Blk->ID == ID; +} +Error PGOCtxProfileReader::enterBlockWithID(PGOCtxProfileBlockIDs ID) { + RET_ON_ERR(Cursor.EnterSubBlock(ID)); + return Error::success(); } Expected, PGOCtxProfContext>> -PGOCtxProfileReader::readContext(bool ExpectIndex) { - RET_ON_ERR(Cursor.EnterSubBlock(PGOCtxProfileBlockIDs::ContextNodeBlockID)); +PGOCtxProfileReader::readProfile(PGOCtxProfileBlockIDs Kind) { + assert((Kind == PGOCtxProfileBlockIDs::ContextRootBlockID || + Kind == PGOCtxProfileBlockIDs::ContextNodeBlockID) && + "Unexpected profile kind"); + RET_ON_ERR(enterBlockWithID(Kind)); std::optional Guid; std::optional> Counters; @@ -78,6 +84,7 @@ PGOCtxProfileReader::readContext(bool ExpectIndex) { SmallVector RecordValues; + const bool ExpectIndex = Kind == PGOCtxProfileBlockIDs::ContextNodeBlockID; // We don't prescribe the order in which the records come in, and we are ok // if other unsupported records appear. We seek in the current subblock until // we get all we know. @@ -121,8 +128,8 @@ PGOCtxProfileReader::readContext(bool ExpectIndex) { PGOCtxProfContext Ret(*Guid, std::move(*Counters)); - while (canReadContext()) { - EXPECT_OR_RET(SC, readContext(true)); + while (canEnterBlockWithID(PGOCtxProfileBlockIDs::ContextNodeBlockID)) { + EXPECT_OR_RET(SC, readProfile(PGOCtxProfileBlockIDs::ContextNodeBlockID)); auto &Targets = Ret.callsites()[*SC->first]; auto [_, Inserted] = Targets.insert({SC->second.guid(), std::move(SC->second)}); @@ -168,16 +175,23 @@ Error PGOCtxProfileReader::readMetadata() { return Error::success(); } -Expected> -PGOCtxProfileReader::loadContexts() { - std::map Ret; - RET_ON_ERR(readMetadata()); - while (canReadContext()) { - EXPECT_OR_RET(E, readContext(false)); - auto Key = E->second.guid(); - if (!Ret.insert({Key, std::move(E->second)}).second) - return wrongValue("Duplicate roots"); +Error PGOCtxProfileReader::loadContexts(CtxProfContextualProfiles &P) { + if (canEnterBlockWithID(PGOCtxProfileBlockIDs::ContextsSectionBlockID)) { + RET_ON_ERR(enterBlockWithID(PGOCtxProfileBlockIDs::ContextsSectionBlockID)); + while (canEnterBlockWithID(PGOCtxProfileBlockIDs::ContextRootBlockID)) { + EXPECT_OR_RET(E, readProfile(PGOCtxProfileBlockIDs::ContextRootBlockID)); + auto Key = E->second.guid(); + if (!P.insert({Key, std::move(E->second)}).second) + return wrongValue("Duplicate roots"); + } } + return Error::success(); +} + +Expected PGOCtxProfileReader::loadProfiles() { + RET_ON_ERR(readMetadata()); + PGOCtxProfile Ret; + RET_ON_ERR(loadContexts(Ret.Contexts)); return std::move(Ret); } @@ -225,7 +239,9 @@ void toYaml(yaml::Output &Out, Out.endSequence(); } -void toYaml(yaml::Output &Out, const PGOCtxProfContext &Ctx) { +void toYaml(yaml::Output &Out, GlobalValue::GUID Guid, + const SmallVectorImpl &Counters, + const PGOCtxProfContext::CallsiteMapTy &Callsites) { yaml::EmptyContext Empty; Out.beginMapping(); void *SaveInfo = nullptr; @@ -233,33 +249,44 @@ void toYaml(yaml::Output &Out, const PGOCtxProfContext &Ctx) { { Out.preflightKey("Guid", /*Required=*/true, /*SameAsDefault=*/false, UseDefault, SaveInfo); - auto Guid = Ctx.guid(); yaml::yamlize(Out, Guid, true, Empty); Out.postflightKey(nullptr); } { Out.preflightKey("Counters", true, false, UseDefault, SaveInfo); Out.beginFlowSequence(); - for (size_t I = 0U, E = Ctx.counters().size(); I < E; ++I) { + for (size_t I = 0U, E = Counters.size(); I < E; ++I) { Out.preflightFlowElement(I, SaveInfo); - uint64_t V = Ctx.counters()[I]; + uint64_t V = Counters[I]; yaml::yamlize(Out, V, true, Empty); Out.postflightFlowElement(SaveInfo); } Out.endFlowSequence(); Out.postflightKey(nullptr); } - if (!Ctx.callsites().empty()) { + if (!Callsites.empty()) { Out.preflightKey("Callsites", true, false, UseDefault, SaveInfo); - toYaml(Out, Ctx.callsites()); + toYaml(Out, Callsites); Out.postflightKey(nullptr); } Out.endMapping(); } +void toYaml(yaml::Output &Out, const PGOCtxProfContext &Ctx) { + toYaml(Out, Ctx.guid(), Ctx.counters(), Ctx.callsites()); +} + } // namespace -void llvm::convertCtxProfToYaml( - raw_ostream &OS, const PGOCtxProfContext::CallTargetMapTy &Profiles) { +void llvm::convertCtxProfToYaml(raw_ostream &OS, + const PGOCtxProfile &Profiles) { yaml::Output Out(OS); - toYaml(Out, Profiles); + void *SaveInfo = nullptr; + bool UseDefault = false; + Out.beginMapping(); + if (!Profiles.Contexts.empty()) { + Out.preflightKey("Contexts", false, false, UseDefault, SaveInfo); + toYaml(Out, Profiles.Contexts); + Out.postflightKey(nullptr); + } + Out.endMapping(); } \ No newline at end of file diff --git a/llvm/lib/ProfileData/PGOCtxProfWriter.cpp b/llvm/lib/ProfileData/PGOCtxProfWriter.cpp index 3d3da84817489..c8ea779602ae5 100644 --- a/llvm/lib/ProfileData/PGOCtxProfWriter.cpp +++ b/llvm/lib/ProfileData/PGOCtxProfWriter.cpp @@ -13,14 +13,19 @@ #include "llvm/ProfileData/PGOCtxProfWriter.h" #include "llvm/Bitstream/BitCodeEnums.h" #include "llvm/ProfileData/CtxInstrContextNode.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Error.h" -#include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/YAMLTraits.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; using namespace llvm::ctx_profile; +static cl::opt IncludeEmpty( + "ctx-prof-include-empty", cl::init(false), + cl::desc( + "Also write profiles with all-zero counters. Mostly for debugging.")); + PGOCtxProfileWriter::PGOCtxProfileWriter( raw_ostream &Out, std::optional VersionOverride) : Writer(Out, 0) { @@ -44,9 +49,11 @@ PGOCtxProfileWriter::PGOCtxProfileWriter( DescribeBlock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID, "Metadata"); DescribeRecord(PGOCtxProfileRecords::Version, "Version"); DescribeBlock(PGOCtxProfileBlockIDs::ContextNodeBlockID, "Context"); + DescribeBlock(PGOCtxProfileBlockIDs::ContextRootBlockID, "Root"); DescribeRecord(PGOCtxProfileRecords::Guid, "GUID"); DescribeRecord(PGOCtxProfileRecords::CalleeIndex, "CalleeIndex"); DescribeRecord(PGOCtxProfileRecords::Counters, "Counters"); + DescribeBlock(PGOCtxProfileBlockIDs::ContextsSectionBlockID, "Contexts"); } Writer.ExitBlock(); Writer.EnterSubblock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID, CodeLen); @@ -55,12 +62,17 @@ PGOCtxProfileWriter::PGOCtxProfileWriter( SmallVector({Version})); } -void PGOCtxProfileWriter::writeCounters(const ContextNode &Node) { +void PGOCtxProfileWriter::writeCounters(ArrayRef Counters) { Writer.EmitCode(bitc::UNABBREV_RECORD); Writer.EmitVBR(PGOCtxProfileRecords::Counters, VBREncodingBits); - Writer.EmitVBR(Node.counters_size(), VBREncodingBits); - for (uint32_t I = 0U; I < Node.counters_size(); ++I) - Writer.EmitVBR64(Node.counters()[I], VBREncodingBits); + Writer.EmitVBR(Counters.size(), VBREncodingBits); + for (uint32_t I = 0U; I < Counters.size(); ++I) + Writer.EmitVBR64(Counters[I], VBREncodingBits); +} + +void PGOCtxProfileWriter::writeGuid(ctx_profile::GUID Guid) { + Writer.EmitRecord(PGOCtxProfileRecords::Guid, + SmallVector{Guid}); } // recursively write all the subcontexts. We do need to traverse depth first to @@ -69,13 +81,16 @@ void PGOCtxProfileWriter::writeCounters(const ContextNode &Node) { // keep the implementation simple. void PGOCtxProfileWriter::writeImpl(std::optional CallerIndex, const ContextNode &Node) { - Writer.EnterSubblock(PGOCtxProfileBlockIDs::ContextNodeBlockID, CodeLen); - Writer.EmitRecord(PGOCtxProfileRecords::Guid, - SmallVector{Node.guid()}); + if (!IncludeEmpty && Node.entrycount() == 0) + return; + Writer.EnterSubblock(CallerIndex ? PGOCtxProfileBlockIDs::ContextNodeBlockID + : PGOCtxProfileBlockIDs::ContextRootBlockID, + CodeLen); + writeGuid(Node.guid()); if (CallerIndex) Writer.EmitRecord(PGOCtxProfileRecords::CalleeIndex, SmallVector{*CallerIndex}); - writeCounters(Node); + writeCounters({Node.counters(), Node.counters_size()}); for (uint32_t I = 0U; I < Node.callsites_size(); ++I) for (const auto *Subcontext = Node.subContexts()[I]; Subcontext; Subcontext = Subcontext->next()) @@ -83,7 +98,15 @@ void PGOCtxProfileWriter::writeImpl(std::optional CallerIndex, Writer.ExitBlock(); } -void PGOCtxProfileWriter::write(const ContextNode &RootNode) { +void PGOCtxProfileWriter::startContextSection() { + Writer.EnterSubblock(PGOCtxProfileBlockIDs::ContextsSectionBlockID, CodeLen); +} + +void PGOCtxProfileWriter::endContextSection() { + Writer.ExitBlock(); +} + +void PGOCtxProfileWriter::writeContextual(const ContextNode &RootNode) { writeImpl(std::nullopt, RootNode); } @@ -96,6 +119,9 @@ struct SerializableCtxRepresentation { std::vector Counters; std::vector> Callsites; }; +struct SerializableProfileRepresentation { + std::vector Contexts; +}; ctx_profile::ContextNode * createNode(std::vector> &Nodes, @@ -142,10 +168,16 @@ template <> struct yaml::MappingTraits { } }; +template <> struct yaml::MappingTraits { + static void mapping(yaml::IO &IO, SerializableProfileRepresentation &SPR) { + IO.mapOptional("Contexts", SPR.Contexts); + } +}; + Error llvm::createCtxProfFromYAML(StringRef Profile, raw_ostream &Out) { yaml::Input In(Profile); - std::vector DCList; - In >> DCList; + SerializableProfileRepresentation SPR; + In >> SPR; if (In.error()) return createStringError(In.error(), "incorrect yaml content"); std::vector> Nodes; @@ -153,12 +185,17 @@ Error llvm::createCtxProfFromYAML(StringRef Profile, raw_ostream &Out) { if (EC) return createStringError(EC, "failed to open output"); PGOCtxProfileWriter Writer(Out); - for (const auto &DC : DCList) { - auto *TopList = createNode(Nodes, DC); - if (!TopList) - return createStringError( - "Unexpected error converting internal structure to ctx profile"); - Writer.write(*TopList); + + if (!SPR.Contexts.empty()) { + Writer.startContextSection(); + for (const auto &DC : SPR.Contexts) { + auto *TopList = createNode(Nodes, DC); + if (!TopList) + return createStringError( + "Unexpected error converting internal structure to ctx profile"); + Writer.writeContextual(*TopList); + } + Writer.endContextSection(); } if (EC) return createStringError(EC, "failed to write output"); diff --git a/llvm/lib/Transforms/IPO/ElimAvailExtern.cpp b/llvm/lib/Transforms/IPO/ElimAvailExtern.cpp index afd5e3ac15ac1..bfcb7f62f2670 100644 --- a/llvm/lib/Transforms/IPO/ElimAvailExtern.cpp +++ b/llvm/lib/Transforms/IPO/ElimAvailExtern.cpp @@ -133,7 +133,8 @@ EliminateAvailableExternallyPass::run(Module &M, ModuleAnalysisManager &MAM) { // that's imported, its optimizations will, thus, differ, and be specialized // for this contextual information. Eliding it in favor of the original would // undo these optimizations. - if (!eliminateAvailableExternally(M, /*Convert=*/(CtxProf && !!(*CtxProf)))) + if (!eliminateAvailableExternally( + M, /*Convert=*/(CtxProf && !CtxProf->contexts().empty()))) return PreservedAnalyses::all(); return PreservedAnalyses::none(); } diff --git a/llvm/lib/Transforms/IPO/FunctionImport.cpp b/llvm/lib/Transforms/IPO/FunctionImport.cpp index c3d0a1a3a046e..29a1283d9ab21 100644 --- a/llvm/lib/Transforms/IPO/FunctionImport.cpp +++ b/llvm/lib/Transforms/IPO/FunctionImport.cpp @@ -724,12 +724,12 @@ class WorkloadImportsManager : public ModuleImportsManager { auto Buffer = std::move(BufferOrErr.get()); PGOCtxProfileReader Reader(Buffer->getBuffer()); - auto Ctx = Reader.loadContexts(); + auto Ctx = Reader.loadProfiles(); if (!Ctx) { report_fatal_error("Failed to parse contextual profiles"); return; } - const auto &CtxMap = *Ctx; + const auto &CtxMap = Ctx->Contexts; SetVector ContainedGUIDs; for (const auto &[RootGuid, Root] : CtxMap) { // Avoid ContainedGUIDs to get in/out of scope. Reuse its memory for diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp index dbc733826944b..f4be64f3e9cfa 100644 --- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp +++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp @@ -171,7 +171,7 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M, << setIsVerbose(); }); } - } else if (CtxProfPromoteAlwaysInline && CtxProf && + } else if (CtxProfPromoteAlwaysInline && !CtxProf.contexts().empty() && CB->isIndirectCall()) { CtxProfAnalysis::collectIndirectCallPromotionList(*CB, CtxProf, ICPCandidates); @@ -260,7 +260,7 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M, // iteration because the next iteration may not happen and we may // miss inlining it. // FIXME: enable for ctxprof. - if (!CtxProf) + if (CtxProf.contexts().empty()) if (tryPromoteCall(*ICB)) NewCallee = ICB->getCalledFunction(); } diff --git a/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp b/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp index ca29d8b7519cb..e6aa374a221da 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp @@ -435,7 +435,7 @@ PreservedAnalyses PGOCtxProfFlatteningPass::run(Module &M, removeInstrumentation(F); }); auto &CtxProf = MAM.getResult(M); - if (!CtxProf) + if (CtxProf.contexts().empty()) return PreservedAnalyses::none(); const auto FlattenedProfile = CtxProf.flatten(); diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp index c78cd0b92ab94..66fda5b1ec7c9 100644 --- a/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -2357,7 +2357,7 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, AAResults *CalleeAAR, bool InsertLifetime, Function *ForwardVarArgsTo) { - if (!CtxProf) + if (CtxProf.contexts().empty()) return InlineFunction(CB, IFI, MergeAttributes, CalleeAAR, InsertLifetime, ForwardVarArgsTo); diff --git a/llvm/test/tools/llvm-ctxprof-util/Inputs/invalid-bad-subctx.yaml b/llvm/test/tools/llvm-ctxprof-util/Inputs/invalid-bad-subctx.yaml index 2c2527d75ad2a..fe3321fc6962a 100644 --- a/llvm/test/tools/llvm-ctxprof-util/Inputs/invalid-bad-subctx.yaml +++ b/llvm/test/tools/llvm-ctxprof-util/Inputs/invalid-bad-subctx.yaml @@ -1,4 +1,5 @@ -- Guid: 123 - Counters: [1, 2] - Callsites: - Guid: 1 +Contexts: + - Guid: 123 + Counters: [1, 2] + Callsites: - Guid: 1 diff --git a/llvm/test/tools/llvm-ctxprof-util/Inputs/invalid-no-counters.yaml b/llvm/test/tools/llvm-ctxprof-util/Inputs/invalid-no-counters.yaml index 7944d92e62ab7..f9704381a5106 100644 --- a/llvm/test/tools/llvm-ctxprof-util/Inputs/invalid-no-counters.yaml +++ b/llvm/test/tools/llvm-ctxprof-util/Inputs/invalid-no-counters.yaml @@ -1 +1,2 @@ -- Guid: 1231 +Contexts: + - Guid: 1231 diff --git a/llvm/test/tools/llvm-ctxprof-util/Inputs/invalid-no-section.yaml b/llvm/test/tools/llvm-ctxprof-util/Inputs/invalid-no-section.yaml new file mode 100644 index 0000000000000..4224687246d1c --- /dev/null +++ b/llvm/test/tools/llvm-ctxprof-util/Inputs/invalid-no-section.yaml @@ -0,0 +1 @@ +- Guid: 1 diff --git a/llvm/test/tools/llvm-ctxprof-util/Inputs/invalid-no-vector.yaml b/llvm/test/tools/llvm-ctxprof-util/Inputs/invalid-no-vector.yaml deleted file mode 100644 index 362277183dec9..0000000000000 --- a/llvm/test/tools/llvm-ctxprof-util/Inputs/invalid-no-vector.yaml +++ /dev/null @@ -1 +0,0 @@ -Guid: 1 diff --git a/llvm/test/tools/llvm-ctxprof-util/Inputs/valid.yaml b/llvm/test/tools/llvm-ctxprof-util/Inputs/valid.yaml index 9bbf82d59c913..0de489dd0b1eb 100644 --- a/llvm/test/tools/llvm-ctxprof-util/Inputs/valid.yaml +++ b/llvm/test/tools/llvm-ctxprof-util/Inputs/valid.yaml @@ -1,13 +1,14 @@ -- Guid: 1000 - Counters: [ 1, 2, 3 ] - Callsites: - - [ ] - - - Guid: 2000 - Counters: [ 4, 5 ] - - Guid: 18446744073709551613 - Counters: [ 6, 7, 8 ] - - - Guid: 3000 - Counters: [ 40, 50 ] -- Guid: 18446744073709551612 - Counters: [ 5, 9, 10 ] +Contexts: + - Guid: 1000 + Counters: [ 1, 2, 3 ] + Callsites: + - [ ] + - - Guid: 2000 + Counters: [ 4, 5 ] + - Guid: 18446744073709551613 + Counters: [ 6, 7, 8 ] + - - Guid: 3000 + Counters: [ 40, 50 ] + - Guid: 18446744073709551612 + Counters: [ 5, 9, 10 ] diff --git a/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util-negative.test b/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util-negative.test index d1f20ffdbc1c4..487d5ae1d17be 100644 --- a/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util-negative.test +++ b/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util-negative.test @@ -5,7 +5,7 @@ ; RUN: not llvm-ctxprof-util fromYAML nofile.yaml 2>&1 | FileCheck %s --check-prefix=NO_FLAG ; RUN: not llvm-ctxprof-util fromYAML --input nofile.yaml 2>&1 | FileCheck -DMSG=%errc_ENOENT %s --check-prefix=NO_FILE ; RUN: not llvm-ctxprof-util fromYAML --input %S/Inputs/bad.yaml 2>&1 | FileCheck %s --check-prefix=BAD_FORMAT -; RUN: not llvm-ctxprof-util fromYAML --input %S/Inputs/invalid-no-vector.yaml 2>&1 | FileCheck %s --check-prefix=NO_VECTOR +; RUN: not llvm-ctxprof-util fromYAML --input %S/Inputs/invalid-no-section.yaml 2>&1 | FileCheck %s --check-prefix=NO_SECTION ; RUN: not llvm-ctxprof-util fromYAML --input %S/Inputs/invalid-no-ctx.yaml 2>&1 | FileCheck %s --check-prefix=NO_CTX ; RUN: not llvm-ctxprof-util fromYAML --input %S/Inputs/invalid-no-counters.yaml 2>&1 | FileCheck %s --check-prefix=NO_COUNTERS ; RUN: not llvm-ctxprof-util fromYAML --input %S/Inputs/invalid-bad-subctx.yaml 2>&1 | FileCheck %s --check-prefix=BAD_SUBCTX @@ -16,9 +16,9 @@ ; INVALID_CMD: Unknown subcommand 'invalidCmd' ; NO_FLAG: Unknown command line argument 'nofile.yaml'. ; NO_FILE: 'nofile.yaml': [[MSG]] -; BAD_FORMAT: YAML:1:3: error: not a mapping -; NO_VECTOR: YAML:1:1: error: not a sequence -; NO_CTX: YAML:1:2: error: not a mapping -; NO_COUNTERS: YAML:1:3: error: missing required key 'Counters' -; BAD_SUBCTX: YAML:3:16: error: not a sequence +; BAD_FORMAT: YAML:1:1: error: not a mapping +; NO_SECTION: YAML:1:1: error: not a mapping +; NO_CTX: YAML:1:1: error: not a mapping +; NO_COUNTERS: YAML:2:5: error: missing required key 'Counters' +; BAD_SUBCTX: YAML:4:18: error: not a sequence ; NO_DIR: failed to open output diff --git a/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util.test b/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util.test index 30bc8bce05410..f68912f73aaa0 100644 --- a/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util.test +++ b/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util.test @@ -18,33 +18,35 @@ ; EMPTY: ; EMPTY-NEXT: -; EMPTY-NEXT: +; EMPTY-NEXT: ; EMPTY-NEXT: ; VALID: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: -; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: +; VALID-NEXT: + \ No newline at end of file diff --git a/llvm/tools/llvm-ctxprof-util/llvm-ctxprof-util.cpp b/llvm/tools/llvm-ctxprof-util/llvm-ctxprof-util.cpp index 314144ac6624c..5903919f96db5 100644 --- a/llvm/tools/llvm-ctxprof-util/llvm-ctxprof-util.cpp +++ b/llvm/tools/llvm-ctxprof-util/llvm-ctxprof-util.cpp @@ -74,7 +74,7 @@ Error convertToYaml() { if (EC) return createStringError(EC, "failed to open output"); PGOCtxProfileReader Reader(BufOrError.get()->getBuffer()); - auto Prof = Reader.loadContexts(); + auto Prof = Reader.loadProfiles(); if (!Prof) return Prof.takeError(); llvm::convertCtxProfToYaml(Out, *Prof); diff --git a/llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp b/llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp index a7950e1249084..478988035d5e1 100644 --- a/llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp +++ b/llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp @@ -99,7 +99,7 @@ TEST_F(PGOCtxProfRWTest, RoundTrip) { { PGOCtxProfileWriter Writer(Out); for (auto &[_, R] : roots()) - Writer.write(*R); + Writer.writeContextual(*R); } } { @@ -121,7 +121,7 @@ TEST_F(PGOCtxProfRWTest, RoundTrip) { EXPECT_TRUE(AnalyzerDump.find("getBuffer()); - auto Expected = Reader.loadContexts(); + auto Expected = Reader.loadProfiles(); ASSERT_TRUE(!!Expected); auto &Ctxes = *Expected; EXPECT_EQ(Ctxes.size(), roots().size()); @@ -149,7 +149,7 @@ TEST_F(PGOCtxProfRWTest, InvalidCounters) { ASSERT_FALSE(EC); { PGOCtxProfileWriter Writer(Out); - Writer.write(*R); + Writer.writeContextual(*R); } } { @@ -157,7 +157,7 @@ TEST_F(PGOCtxProfRWTest, InvalidCounters) { ASSERT_TRUE(!!MB); ASSERT_NE(*MB, nullptr); PGOCtxProfileReader Reader((*MB)->getBuffer()); - auto Expected = Reader.loadContexts(); + auto Expected = Reader.loadProfiles(); EXPECT_FALSE(Expected); consumeError(Expected.takeError()); } @@ -165,14 +165,14 @@ TEST_F(PGOCtxProfRWTest, InvalidCounters) { TEST_F(PGOCtxProfRWTest, Empty) { PGOCtxProfileReader Reader(""); - auto Expected = Reader.loadContexts(); + auto Expected = Reader.loadProfiles(); EXPECT_FALSE(Expected); consumeError(Expected.takeError()); } TEST_F(PGOCtxProfRWTest, Invalid) { PGOCtxProfileReader Reader("Surely this is not valid"); - auto Expected = Reader.loadContexts(); + auto Expected = Reader.loadProfiles(); EXPECT_FALSE(Expected); consumeError(Expected.takeError()); } @@ -194,7 +194,7 @@ TEST_F(PGOCtxProfRWTest, ValidButEmpty) { ASSERT_NE(*MB, nullptr); PGOCtxProfileReader Reader((*MB)->getBuffer()); - auto Expected = Reader.loadContexts(); + auto Expected = Reader.loadProfiles(); EXPECT_TRUE(!!Expected); EXPECT_TRUE(Expected->empty()); } @@ -216,7 +216,7 @@ TEST_F(PGOCtxProfRWTest, WrongVersion) { ASSERT_NE(*MB, nullptr); PGOCtxProfileReader Reader((*MB)->getBuffer()); - auto Expected = Reader.loadContexts(); + auto Expected = Reader.loadProfiles(); EXPECT_FALSE(Expected); consumeError(Expected.takeError()); } @@ -230,8 +230,8 @@ TEST_F(PGOCtxProfRWTest, DuplicateRoots) { ASSERT_FALSE(EC); { PGOCtxProfileWriter Writer(Out); - Writer.write(*createNode(1, 1, 1)); - Writer.write(*createNode(1, 1, 1)); + Writer.writeContextual(*createNode(1, 1, 1)); + Writer.writeContextual(*createNode(1, 1, 1)); } } { @@ -239,7 +239,7 @@ TEST_F(PGOCtxProfRWTest, DuplicateRoots) { ASSERT_TRUE(!!MB); ASSERT_NE(*MB, nullptr); PGOCtxProfileReader Reader((*MB)->getBuffer()); - auto Expected = Reader.loadContexts(); + auto Expected = Reader.loadProfiles(); EXPECT_FALSE(Expected); consumeError(Expected.takeError()); } @@ -257,7 +257,7 @@ TEST_F(PGOCtxProfRWTest, DuplicateTargets) { auto *L2 = createNode(2, 1, 0, L1); R->subContexts()[0] = L2; PGOCtxProfileWriter Writer(Out); - Writer.write(*R); + Writer.writeContextual(*R); } } { @@ -265,7 +265,7 @@ TEST_F(PGOCtxProfRWTest, DuplicateTargets) { ASSERT_TRUE(!!MB); ASSERT_NE(*MB, nullptr); PGOCtxProfileReader Reader((*MB)->getBuffer()); - auto Expected = Reader.loadContexts(); + auto Expected = Reader.loadProfiles(); EXPECT_FALSE(Expected); consumeError(Expected.takeError()); }