diff --git a/llvm/include/llvm/Object/ELFTypes.h b/llvm/include/llvm/Object/ELFTypes.h index e9a417d3d4fb3..467ab6fd3c1e9 100644 --- a/llvm/include/llvm/Object/ELFTypes.h +++ b/llvm/include/llvm/Object/ELFTypes.h @@ -834,30 +834,32 @@ struct BBAddrMap { bool OmitBBEntries : 1; bool CallsiteEndOffsets : 1; bool BBHash : 1; + bool PostLinkCfg : 1; bool hasPGOAnalysis() const { return FuncEntryCount || BBFreq || BrProb; } bool hasPGOAnalysisBBData() const { return BBFreq || BrProb; } // Encodes to minimum bit width representation. - uint8_t encode() const { - return (static_cast(FuncEntryCount) << 0) | - (static_cast(BBFreq) << 1) | - (static_cast(BrProb) << 2) | - (static_cast(MultiBBRange) << 3) | - (static_cast(OmitBBEntries) << 4) | - (static_cast(CallsiteEndOffsets) << 5) | - (static_cast(BBHash) << 6); + uint16_t encode() const { + return (static_cast(FuncEntryCount) << 0) | + (static_cast(BBFreq) << 1) | + (static_cast(BrProb) << 2) | + (static_cast(MultiBBRange) << 3) | + (static_cast(OmitBBEntries) << 4) | + (static_cast(CallsiteEndOffsets) << 5) | + (static_cast(BBHash) << 6) | + (static_cast(PostLinkCfg) << 7); } // Decodes from minimum bit width representation and validates no // unnecessary bits are used. - static Expected decode(uint8_t Val) { + static Expected decode(uint16_t Val) { Features Feat{ static_cast(Val & (1 << 0)), static_cast(Val & (1 << 1)), static_cast(Val & (1 << 2)), static_cast(Val & (1 << 3)), static_cast(Val & (1 << 4)), static_cast(Val & (1 << 5)), - static_cast(Val & (1 << 6))}; + static_cast(Val & (1 << 6)), static_cast(Val & (1 << 7))}; if (Feat.encode() != Val) return createStringError( std::error_code(), "invalid encoding for BBAddrMap::Features: 0x%x", @@ -867,10 +869,11 @@ struct BBAddrMap { bool operator==(const Features &Other) const { return std::tie(FuncEntryCount, BBFreq, BrProb, MultiBBRange, - OmitBBEntries, CallsiteEndOffsets, BBHash) == + OmitBBEntries, CallsiteEndOffsets, BBHash, PostLinkCfg) == std::tie(Other.FuncEntryCount, Other.BBFreq, Other.BrProb, Other.MultiBBRange, Other.OmitBBEntries, - Other.CallsiteEndOffsets, Other.BBHash); + Other.CallsiteEndOffsets, Other.BBHash, + Other.PostLinkCfg); } }; @@ -1010,23 +1013,30 @@ struct PGOAnalysisMap { /// probability associated with it. struct SuccessorEntry { /// Unique ID of this successor basic block. - uint32_t ID; + uint32_t ID = 0; /// Branch Probability of the edge to this successor taken from MBPI. BranchProbability Prob; + /// Raw edge count from the post link profile (e.g., from bolt or + /// propeller). + uint64_t PostLinkFreq = 0; bool operator==(const SuccessorEntry &Other) const { - return std::tie(ID, Prob) == std::tie(Other.ID, Other.Prob); + return std::tie(ID, Prob, PostLinkFreq) == + std::tie(Other.ID, Other.Prob, Other.PostLinkFreq); } }; /// Block frequency taken from MBFI BlockFrequency BlockFreq; + /// Raw block count taken from the post link profile (e.g., from bolt or + /// propeller). + uint64_t PostLinkBlockFreq = 0; /// List of successors of the current block llvm::SmallVector Successors; bool operator==(const PGOBBEntry &Other) const { - return std::tie(BlockFreq, Successors) == - std::tie(Other.BlockFreq, Other.Successors); + return std::tie(BlockFreq, PostLinkBlockFreq, Successors) == + std::tie(Other.BlockFreq, PostLinkBlockFreq, Other.Successors); } }; diff --git a/llvm/include/llvm/ObjectYAML/ELFYAML.h b/llvm/include/llvm/ObjectYAML/ELFYAML.h index a7c7c7c436dc2..a8236ca37b5ed 100644 --- a/llvm/include/llvm/ObjectYAML/ELFYAML.h +++ b/llvm/include/llvm/ObjectYAML/ELFYAML.h @@ -166,7 +166,7 @@ struct BBAddrMapEntry { std::optional Hash; }; uint8_t Version; - llvm::yaml::Hex8 Feature; + llvm::yaml::Hex16 Feature; struct BBRangeEntry { llvm::yaml::Hex64 BaseAddress; @@ -203,8 +203,10 @@ struct PGOAnalysisMapEntry { struct SuccessorEntry { uint32_t ID; llvm::yaml::Hex32 BrProb; + std::optional PostLinkBrFreq; }; std::optional BBFreq; + std::optional PostLinkBBFreq; std::optional> Successors; }; std::optional FuncEntryCount; diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 8aa488f0efd8f..f65d88a669f13 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -1443,7 +1443,7 @@ getBBAddrMapFeature(const MachineFunction &MF, int NumMBBSectionRanges, MF.hasBBSections() && NumMBBSectionRanges > 1, // Use static_cast to avoid breakage of tests on windows. static_cast(BBAddrMapSkipEmitBBEntries), HasCalls, - static_cast(EmitBBHash)}; + static_cast(EmitBBHash), false}; } void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) { diff --git a/llvm/lib/Object/ELF.cpp b/llvm/lib/Object/ELF.cpp index 6da97f9b3755d..354c51d66419c 100644 --- a/llvm/lib/Object/ELF.cpp +++ b/llvm/lib/Object/ELF.cpp @@ -831,17 +831,17 @@ decodeBBAddrMapImpl(const ELFFile &EF, }; uint8_t Version = 0; - uint8_t Feature = 0; + uint16_t Feature = 0; BBAddrMap::Features FeatEnable{}; while (!ULEBSizeErr && !MetadataDecodeErr && Cur && Cur.tell() < Content.size()) { Version = Data.getU8(Cur); if (!Cur) break; - if (Version < 2 || Version > 4) + if (Version < 2 || Version > 5) return createError("unsupported SHT_LLVM_BB_ADDR_MAP version: " + Twine(static_cast(Version))); - Feature = Data.getU8(Cur); // Feature byte + Feature = Version < 5 ? Data.getU8(Cur) : Data.getU16(Cur); if (!Cur) break; auto FeatEnableOrErr = BBAddrMap::Features::decode(Feature); @@ -858,6 +858,11 @@ decodeBBAddrMapImpl(const ELFFile &EF, "basic block hash feature is enabled: version = " + Twine(static_cast(Version)) + " feature = " + Twine(static_cast(Feature))); + if (FeatEnable.PostLinkCfg && Version < 5) + return createError("version should be >= 5 for SHT_LLVM_BB_ADDR_MAP when " + "post link cfg feature is enabled: version = " + + Twine(static_cast(Version)) + + " feature = " + Twine(static_cast(Feature))); uint32_t NumBlocksInBBRange = 0; uint32_t NumBBRanges = 1; typename ELFFile::uintX_t RangeBaseAddress = 0; @@ -946,6 +951,10 @@ decodeBBAddrMapImpl(const ELFFile &EF, uint64_t BBF = FeatEnable.BBFreq ? readULEB128As(Data, Cur, ULEBSizeErr) : 0; + uint32_t PostLinkBBFreq = + FeatEnable.PostLinkCfg + ? readULEB128As(Data, Cur, ULEBSizeErr) + : 0; // Branch probability llvm::SmallVector @@ -955,13 +964,20 @@ decodeBBAddrMapImpl(const ELFFile &EF, for (uint64_t I = 0; I < SuccCount; ++I) { uint32_t BBID = readULEB128As(Data, Cur, ULEBSizeErr); uint32_t BrProb = readULEB128As(Data, Cur, ULEBSizeErr); + uint32_t PostLinkFreq = + FeatEnable.PostLinkCfg + ? readULEB128As(Data, Cur, ULEBSizeErr) + : 0; + if (PGOAnalyses) - Successors.push_back({BBID, BranchProbability::getRaw(BrProb)}); + Successors.push_back( + {BBID, BranchProbability::getRaw(BrProb), PostLinkFreq}); } } if (PGOAnalyses) - PGOBBEntries.push_back({BlockFrequency(BBF), std::move(Successors)}); + PGOBBEntries.push_back( + {BlockFrequency(BBF), PostLinkBBFreq, std::move(Successors)}); } if (PGOAnalyses) diff --git a/llvm/lib/ObjectYAML/ELFEmitter.cpp b/llvm/lib/ObjectYAML/ELFEmitter.cpp index 8b75fbe8291f0..8530785d07c93 100644 --- a/llvm/lib/ObjectYAML/ELFEmitter.cpp +++ b/llvm/lib/ObjectYAML/ELFEmitter.cpp @@ -1465,13 +1465,19 @@ void ELFState::writeSectionContent( for (const auto &[Idx, E] : llvm::enumerate(*Section.Entries)) { // Write version and feature values. if (Section.Type == llvm::ELF::SHT_LLVM_BB_ADDR_MAP) { - if (E.Version > 4) + if (E.Version > 5) WithColor::warning() << "unsupported SHT_LLVM_BB_ADDR_MAP version: " << static_cast(E.Version) << "; encoding using the most recent version"; CBA.write(E.Version); - CBA.write(E.Feature); - SHeader.sh_size += 2; + SHeader.sh_size += 1; + if (E.Version < 5) { + CBA.write(static_cast(E.Feature)); + SHeader.sh_size += 1; + } else { + CBA.write(E.Feature, ELFT::Endianness); + SHeader.sh_size += 2; + } } auto FeatureOrErr = llvm::object::BBAddrMap::Features::decode(E.Feature); bool MultiBBRangeFeatureEnabled = false; @@ -1556,11 +1562,15 @@ void ELFState::writeSectionContent( for (const auto &PGOBBE : PGOBBEntries) { if (PGOBBE.BBFreq) SHeader.sh_size += CBA.writeULEB128(*PGOBBE.BBFreq); + if (FeatureOrErr->PostLinkCfg || PGOBBE.PostLinkBBFreq.has_value()) + SHeader.sh_size += CBA.writeULEB128(PGOBBE.PostLinkBBFreq.value_or(0)); if (PGOBBE.Successors) { SHeader.sh_size += CBA.writeULEB128(PGOBBE.Successors->size()); - for (const auto &[ID, BrProb] : *PGOBBE.Successors) { + for (const auto &[ID, BrProb, PostLinkBrFreq] : *PGOBBE.Successors) { SHeader.sh_size += CBA.writeULEB128(ID); SHeader.sh_size += CBA.writeULEB128(BrProb); + if (FeatureOrErr->PostLinkCfg || PostLinkBrFreq.has_value()) + SHeader.sh_size += CBA.writeULEB128(PostLinkBrFreq.value_or(0)); } } } diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp index f8a84b075b779..e5e5fc20728e8 100644 --- a/llvm/lib/ObjectYAML/ELFYAML.cpp +++ b/llvm/lib/ObjectYAML/ELFYAML.cpp @@ -1886,7 +1886,7 @@ void MappingTraits::mapping( IO &IO, ELFYAML::BBAddrMapEntry &E) { assert(IO.getContext() && "The IO context is not initialized"); IO.mapRequired("Version", E.Version); - IO.mapOptional("Feature", E.Feature, Hex8(0)); + IO.mapOptional("Feature", E.Feature, Hex16(0)); IO.mapOptional("NumBBRanges", E.NumBBRanges); IO.mapOptional("BBRanges", E.BBRanges); } @@ -1920,6 +1920,7 @@ void MappingTraits::mapping( IO &IO, ELFYAML::PGOAnalysisMapEntry::PGOBBEntry &E) { assert(IO.getContext() && "The IO context is not initialized"); IO.mapOptional("BBFreq", E.BBFreq); + IO.mapOptional("PostLinkBBFreq", E.PostLinkBBFreq); IO.mapOptional("Successors", E.Successors); } @@ -1929,6 +1930,7 @@ void MappingTraits:: assert(IO.getContext() && "The IO context is not initialized"); IO.mapRequired("ID", E.ID); IO.mapRequired("BrProb", E.BrProb); + IO.mapOptional("PostLinkBrFreq", E.PostLinkBrFreq); } void MappingTraits::mapping(IO &IO, diff --git a/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-feature-warning.test b/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-feature-warning.test new file mode 100644 index 0000000000000..24726c34d3509 --- /dev/null +++ b/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-feature-warning.test @@ -0,0 +1,37 @@ +## This test checks that we output a warning when the specified version is too old to support the given features. + +# RUN: yaml2obj %s -o %t +# RUN: llvm-readobj --bb-addr-map %t 2>&1 | FileCheck -DFILE=%t %s + +--- !ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_EXEC + +# CHECK: BBAddrMap [ +# CHECK-NEXT: warning: '[[FILE]]': unable to dump SHT_LLVM_BB_ADDR_MAP section with index 1: version should be >= 3 for SHT_LLVM_BB_ADDR_MAP when callsite offsets feature is enabled: version = 2 feature = 32 +Sections: + - Name: '.llvm_bb_addr_map (1)' + Type: SHT_LLVM_BB_ADDR_MAP + Entries: + - Version: 2 + Feature: 0x20 + +# CHECK: BBAddrMap [ +# CHECK-NEXT: warning: '[[FILE]]': unable to dump SHT_LLVM_BB_ADDR_MAP section with index 2: version should be >= 4 for SHT_LLVM_BB_ADDR_MAP when basic block hash feature is enabled: version = 3 feature = 64 + + - Name: '.llvm_bb_addr_map (2)' + Type: SHT_LLVM_BB_ADDR_MAP + Entries: + - Version: 3 + Feature: 0x40 + +# CHECK: BBAddrMap [ +# CHECK-NEXT: warning: '[[FILE]]': unable to dump SHT_LLVM_BB_ADDR_MAP section with index 3: version should be >= 5 for SHT_LLVM_BB_ADDR_MAP when post link cfg feature is enabled: version = 4 feature = 128 + + - Name: '.llvm_bb_addr_map (3)' + Type: SHT_LLVM_BB_ADDR_MAP + Entries: + - Version: 4 + Feature: 0x80 diff --git a/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-pgo-analysis-map.test b/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-pgo-analysis-map.test index 5faafd4d83b2f..8e9d2271b8721 100644 --- a/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-pgo-analysis-map.test +++ b/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-pgo-analysis-map.test @@ -15,7 +15,7 @@ ## Check that a malformed section can be handled. # RUN: yaml2obj %s -DBITS=32 -DSIZE=24 -o %t2.o -# RUN: llvm-readobj %t2.o --bb-addr-map 2>&1 | FileCheck --match-full-lines %s -DOFFSET=0x00000018 -DFILE=%t2.o --check-prefix=TRUNCATED +# RUN: llvm-readobj %t2.o --bb-addr-map 2>&1 | FileCheck --match-full-lines %s -DOFFSET=0x00000015 -DFILE=%t2.o --check-prefix=TRUNCATED ## Check that missing features can be handled. # RUN: yaml2obj %s -DBITS=32 -DFEATURE=0x2 -o %t3.o @@ -59,17 +59,20 @@ # CHECK-NEXT: { # RAW-NEXT: Frequency: 100 # PRETTY-NEXT: Frequency: 1.0 +# CHECK-NEXT: PostLink Frequency: 10 # CHECK-NEXT: Successors [ # CHECK-NEXT: { # CHECK-NEXT: ID: 2 # RAW-NEXT: Probability: 0x80000000 # PRETTY-NEXT: Probability: 0x80000000 / 0x80000000 = 100.00% +# CHECK-NEXT: PostLink Probability: 7 # CHECK-NEXT: } # CHECK-NEXT: ] # CHECK-NEXT: } # CHECK-NEXT: { # RAW-NEXT: Frequency: 100 # PRETTY-NEXT: Frequency: 1.0 +# CHECK-NEXT: PostLink Frequency: 0 # CHECK-NEXT: Successors [ # CHECK-NEXT: ] # CHECK-NEXT: } @@ -172,8 +175,8 @@ Sections: ShSize: [[SIZE=]] Link: .text Entries: - - Version: 2 - Feature: 0x7 + - Version: 5 + Feature: 0x87 BBRanges: - BaseAddress: [[ADDR=0x11111]] BBEntries: @@ -197,10 +200,12 @@ Sections: PGOAnalyses: - FuncEntryCount: 100 PGOBBEntries: - - BBFreq: 100 + - BBFreq: 100 + PostLinkBBFreq: 10 Successors: - - ID: 2 - BrProb: 0x80000000 + - ID: 2 + BrProb: 0x80000000 + PostLinkBrFreq: 7 - BBFreq: 100 Successors: [] - FuncEntryCount: 8888 diff --git a/llvm/test/tools/obj2yaml/ELF/bb-addr-map-pgo-analysis-map.yaml b/llvm/test/tools/obj2yaml/ELF/bb-addr-map-pgo-analysis-map.yaml index 299bf463cf4bc..645507af080cb 100644 --- a/llvm/test/tools/obj2yaml/ELF/bb-addr-map-pgo-analysis-map.yaml +++ b/llvm/test/tools/obj2yaml/ELF/bb-addr-map-pgo-analysis-map.yaml @@ -15,7 +15,7 @@ # VALID-NEXT: Type: SHT_LLVM_BB_ADDR_MAP # VALID-NEXT: Entries: # VALID-NEXT: - Version: 2 -# VALID-NEXT: Feature: 0x7 +# VALID-NEXT: Feature: 0x87 ## The 'BaseAddress' field is omitted when it's zero. # VALID-NEXT: BBRanges: # VALID-NEXT: - BBEntries: @@ -43,17 +43,23 @@ # VALID-NEXT: PGOAnalyses: # VALID-NEXT: - FuncEntryCount: 100 # VALID-NEXT: PGOBBEntries: -# VALID-NEXT: - BBFreq: 100 +# VALID-NEXT: - BBFreq: 100 +# VALID-NEXT: PostLinkBBFreq: 10 # VALID-NEXT: Successors: -# VALID-NEXT: - ID: 2 -# VALID-NEXT: BrProb: 0x80000000 -# VALID-NEXT: - ID: 4 -# VALID-NEXT: BrProb: 0x80000000 -# VALID-NEXT: - BBFreq: 50 +# VALID-NEXT: - ID: 2 +# VALID-NEXT: BrProb: 0x80000000 +# VALID-NEXT: PostLinkBrFreq: 7 +# VALID-NEXT: - ID: 4 +# VALID-NEXT: BrProb: 0x80000000 +# VALID-NEXT: PostLinkBrFreq: 0 +# VALID-NEXT: - BBFreq: 50 +# VALID-NEXT: PostLinkBBFreq: 0 # VALID-NEXT: Successors: -# VALID-NEXT: - ID: 4 -# VALID-NEXT: BrProb: 0xFFFFFFFF -# VALID-NEXT: - BBFreq: 100 +# VALID-NEXT: - ID: 4 +# VALID-NEXT: BrProb: 0xFFFFFFFF +# VALID-NEXT: PostLinkBrFreq: 0 +# VALID-NEXT: - BBFreq: 100 +# VALID-NEXT: PostLinkBBFreq: 3 # VALID-NEXT: Successors: [] # VALID-NEXT: PGOBBEntries: # VALID-NEXT: - BBFreq: 20 @@ -69,7 +75,7 @@ Sections: ShSize: [[SIZE=]] Entries: - Version: 2 - Feature: 0x7 + Feature: 0x87 BBRanges: - BaseAddress: 0x0 BBEntries: @@ -97,17 +103,20 @@ Sections: PGOAnalyses: - FuncEntryCount: 100 PGOBBEntries: - - BBFreq: 100 + - BBFreq: 100 + PostLinkBBFreq: 10 Successors: - - ID: 2 - BrProb: 0x80000000 - - ID: 4 - BrProb: 0x80000000 - - BBFreq: 50 + - ID: 2 + BrProb: 0x80000000 + PostLinkBrFreq: 7 + - ID: 4 + BrProb: 0x80000000 + - BBFreq: 50 Successors: - - ID: 4 - BrProb: 0xFFFFFFFF - - BBFreq: 100 + - ID: 4 + BrProb: 0xFFFFFFFF + - BBFreq: 100 + PostLinkBBFreq: 3 Successors: [] - PGOBBEntries: - BBFreq: 20 diff --git a/llvm/test/tools/yaml2obj/ELF/bb-addr-map-pgo-analysis-map.yaml b/llvm/test/tools/yaml2obj/ELF/bb-addr-map-pgo-analysis-map.yaml index a4cb572e6d993..ac9c8d402b0a6 100644 --- a/llvm/test/tools/yaml2obj/ELF/bb-addr-map-pgo-analysis-map.yaml +++ b/llvm/test/tools/yaml2obj/ELF/bb-addr-map-pgo-analysis-map.yaml @@ -6,8 +6,9 @@ # Case 4: Specify Entries. # CHECK: Name: .llvm_bb_addr_map (1) # CHECK: SectionData ( -# CHECK-NEXT: 0000: 02072000 00000000 0000010B 010203E8 -# CHECK-NEXT: 0010: 07E80702 0CEEDDBB F70E0D91 A2C48801 +# CHECK-NEXT: 0000: 02872000 00000000 0000010B 010203E8 +# CHECK-NEXT: 0010: 07E80764 020CEEDD BBF70E28 0D91A2C4 +# CHECK-NEXT: 0020: 880100 # CHECK-NEXT: ) # Case 7: Not including a field which is enabled in feature doesn't emit value @@ -26,12 +27,12 @@ Sections: ## Test the following cases: ## 1) We can produce an .llvm_bb_addr_map section from a description with -## Entries and PGO Analysis data. +## Entries and PGO Analysis and Post Link data. - Name: '.llvm_bb_addr_map (1)' Type: SHT_LLVM_BB_ADDR_MAP Entries: - Version: 2 - Feature: 0x7 + Feature: 0x87 BBRanges: - BaseAddress: 0x0000000000000020 BBEntries: @@ -42,12 +43,14 @@ Sections: PGOAnalyses: - FuncEntryCount: 1000 PGOBBEntries: - - BBFreq: 1000 + - BBFreq: 1000 + PostLinkBBFreq: 100 Successors: - - ID: 12 - BrProb: 0xeeeeeeee - - ID: 13 - BrProb: 0x11111111 + - ID: 12 + BrProb: 0xeeeeeeee + PostLinkBrFreq: 40 + - ID: 13 + BrProb: 0x11111111 ## 2) According to feature we have FuncEntryCount but none is provided in yaml - Name: '.llvm_bb_addr_map (2)' @@ -66,7 +69,7 @@ Sections: ## Check that yaml2obj generates a warning when we use unsupported feature. # RUN: yaml2obj --docnum=2 %s 2>&1 | FileCheck %s --check-prefix=INVALID-FEATURE -# INVALID-FEATURE: warning: invalid encoding for BBAddrMap::Features: 0xf0 +# INVALID-FEATURE: warning: invalid encoding for BBAddrMap::Features: 0x100 --- !ELF FileHeader: @@ -79,4 +82,4 @@ Sections: Entries: - Version: 2 ## Specify unsupported feature - Feature: 0xF0 + Feature: 0x100 diff --git a/llvm/test/tools/yaml2obj/ELF/bb-addr-map.yaml b/llvm/test/tools/yaml2obj/ELF/bb-addr-map.yaml index 339e419b39458..05d77d67e4468 100644 --- a/llvm/test/tools/yaml2obj/ELF/bb-addr-map.yaml +++ b/llvm/test/tools/yaml2obj/ELF/bb-addr-map.yaml @@ -220,7 +220,7 @@ Sections: ## Check that yaml2obj generates a warning when we use unsupported versions. # RUN: yaml2obj --docnum=3 %s 2>&1 | FileCheck %s --check-prefix=INVALID-VERSION -# INVALID-VERSION: warning: unsupported SHT_LLVM_BB_ADDR_MAP version: 5; encoding using the most recent version +# INVALID-VERSION: warning: unsupported SHT_LLVM_BB_ADDR_MAP version: 6; encoding using the most recent version --- !ELF FileHeader: @@ -232,4 +232,4 @@ Sections: Type: SHT_LLVM_BB_ADDR_MAP Entries: ## Specify unsupported version - - Version: 5 + - Version: 6 diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp index 423a11fd5b72a..6f09da5a4099f 100644 --- a/llvm/tools/llvm-readobj/ELFDumper.cpp +++ b/llvm/tools/llvm-readobj/ELFDumper.cpp @@ -8188,6 +8188,8 @@ void LLVMELFDumper::printBBAddrMaps(bool PrettyPGOAnalysis) { } else { W.printNumber("Frequency", PBBE.BlockFreq.getFrequency()); } + if (PAM.FeatEnable.PostLinkCfg) + W.printNumber("PostLink Frequency", PBBE.PostLinkBlockFreq); } if (PAM.FeatEnable.BrProb) { @@ -8200,6 +8202,8 @@ void LLVMELFDumper::printBBAddrMaps(bool PrettyPGOAnalysis) { } else { W.printHex("Probability", Succ.Prob.getNumerator()); } + if (PAM.FeatEnable.PostLinkCfg) + W.printNumber("PostLink Probability", Succ.PostLinkFreq); } } } diff --git a/llvm/tools/obj2yaml/elf2yaml.cpp b/llvm/tools/obj2yaml/elf2yaml.cpp index 68e18f6c79202..4364d15a8b455 100644 --- a/llvm/tools/obj2yaml/elf2yaml.cpp +++ b/llvm/tools/obj2yaml/elf2yaml.cpp @@ -895,7 +895,7 @@ ELFDumper::dumpBBAddrMapSection(const Elf_Shdr *Shdr) { std::vector PGOAnalyses; DataExtractor::Cursor Cur(0); uint8_t Version = 0; - uint8_t Feature = 0; + uint16_t Feature = 0; uint64_t Address = 0; while (Cur && Cur.tell() < Content.size()) { if (Shdr->sh_type == ELF::SHT_LLVM_BB_ADDR_MAP) { @@ -905,7 +905,7 @@ ELFDumper::dumpBBAddrMapSection(const Elf_Shdr *Shdr) { errc::invalid_argument, "invalid SHT_LLVM_BB_ADDR_MAP section version: " + Twine(static_cast(Version))); - Feature = Data.getU8(Cur); + Feature = Version < 5 ? Data.getU8(Cur) : Data.getU16(Cur); } uint64_t NumBBRanges = 1; uint64_t NumBlocks = 0; @@ -972,6 +972,8 @@ ELFDumper::dumpBBAddrMapSection(const Elf_Shdr *Shdr) { auto &PGOBBEntry = PGOBBEntries.emplace_back(); if (FeatureOrErr->BBFreq) { PGOBBEntry.BBFreq = Data.getULEB128(Cur); + if (FeatureOrErr->PostLinkCfg) + PGOBBEntry.PostLinkBBFreq = Data.getULEB128(Cur); if (!Cur) break; } @@ -982,7 +984,10 @@ ELFDumper::dumpBBAddrMapSection(const Elf_Shdr *Shdr) { for (uint64_t SuccIdx = 0; Cur && SuccIdx < SuccCount; ++SuccIdx) { uint32_t ID = Data.getULEB128(Cur); uint32_t BrProb = Data.getULEB128(Cur); - SuccEntries.push_back({ID, BrProb}); + std::optional PostLinkBrFreq; + if (FeatureOrErr->PostLinkCfg) + PostLinkBrFreq = Data.getULEB128(Cur); + SuccEntries.push_back({ID, BrProb, PostLinkBrFreq}); } } } diff --git a/llvm/unittests/Object/ELFObjectFileTest.cpp b/llvm/unittests/Object/ELFObjectFileTest.cpp index d6a3ca53b2154..1e2955ae40a66 100644 --- a/llvm/unittests/Object/ELFObjectFileTest.cpp +++ b/llvm/unittests/Object/ELFObjectFileTest.cpp @@ -531,7 +531,7 @@ TEST(ELFObjectFileTest, InvalidDecodeBBAddrMap) { // Check that we can detect unsupported versions. SmallString<128> UnsupportedVersionYamlString(CommonYamlString); UnsupportedVersionYamlString += R"( - - Version: 5 + - Version: 6 BBRanges: - BaseAddress: 0x11111 BBEntries: @@ -543,7 +543,7 @@ TEST(ELFObjectFileTest, InvalidDecodeBBAddrMap) { { SCOPED_TRACE("unsupported version"); DoCheck(UnsupportedVersionYamlString, - "unsupported SHT_LLVM_BB_ADDR_MAP version: 5"); + "unsupported SHT_LLVM_BB_ADDR_MAP version: 6"); } SmallString<128> ZeroBBRangesYamlString(CommonYamlString); @@ -1181,8 +1181,8 @@ TEST(ELFObjectFileTest, ReadPGOAnalysisMap) { Type: SHT_LLVM_BB_ADDR_MAP # Link: 0 (by default, can be overriden) Entries: - - Version: 2 - Feature: 0x7 + - Version: 5 + Feature: 0x87 BBRanges: - BaseAddress: 0x44444 BBEntries: @@ -1205,7 +1205,8 @@ TEST(ELFObjectFileTest, ReadPGOAnalysisMap) { PGOAnalyses: - FuncEntryCount: 1000 PGOBBEntries: - - BBFreq: 1000 + - BBFreq: 1000 + PostLinkBBFreq: 50 Successors: - ID: 1 BrProb: 0x22222222 @@ -1243,8 +1244,8 @@ TEST(ELFObjectFileTest, ReadPGOAnalysisMap) { Type: SHT_LLVM_BB_ADDR_MAP # Link: 0 (by default, can be overriden) Entries: - - Version: 2 - Feature: 0xc + - Version: 5 + Feature: 0x8c BBRanges: - BaseAddress: 0x66666 BBEntries: @@ -1265,8 +1266,9 @@ TEST(ELFObjectFileTest, ReadPGOAnalysisMap) { PGOAnalyses: - PGOBBEntries: - Successors: - - ID: 1 - BrProb: 0x22222222 + - ID: 1 + BrProb: 0x22222222 + PostLinkBrFreq: 7 - ID: 2 BrProb: 0xcccccccc - Successors: @@ -1278,59 +1280,66 @@ TEST(ELFObjectFileTest, ReadPGOAnalysisMap) { BBAddrMap E1 = { {{0x11111, {{1, 0x0, 0x1, {false, true, false, false, false}, {}, 0}}}}}; PGOAnalysisMap P1 = { - 892, {}, {true, false, false, false, false, false, false}}; + 892, {}, {true, false, false, false, false, false, false, false}}; BBAddrMap E2 = { {{0x22222, {{2, 0x0, 0x2, {false, false, true, false, false}, {}, 0}}}}}; PGOAnalysisMap P2 = {{}, - {{BlockFrequency(343), {}}}, - {false, true, false, false, false, false, false}}; + {{BlockFrequency(343), 0, {}}}, + {false, true, false, false, false, false, false, false}}; BBAddrMap E3 = { {{0x33333, {{0, 0x0, 0x3, {false, true, true, false, false}, {}, 0}, {1, 0x3, 0x3, {false, false, true, false, false}, {}, 0}, {2, 0x6, 0x3, {false, false, false, false, false}, {}, 0}}}}}; - PGOAnalysisMap P3 = {{}, - {{{}, - {{1, BranchProbability::getRaw(0x1111'1111)}, - {2, BranchProbability::getRaw(0xeeee'eeee)}}}, - {{}, {{2, BranchProbability::getRaw(0xffff'ffff)}}}, - {{}, {}}}, - {false, false, true, false, false, false, false}}; + PGOAnalysisMap P3 = { + {}, + {{{}, + 0, + {{1, BranchProbability::getRaw(0x1111'1111), 0}, + {2, BranchProbability::getRaw(0xeeee'eeee), 0}}}, + {{}, 0, {{2, BranchProbability::getRaw(0xffff'ffff), 0}}}, + {{}, 0, {}}}, + {false, false, true, false, false, false, false, false}}; BBAddrMap E4 = { {{0x44444, {{0, 0x0, 0x4, {false, false, false, true, true}, {}, 0}, {1, 0x4, 0x4, {false, false, false, false, false}, {}, 0}, {2, 0x8, 0x4, {false, false, false, false, false}, {}, 0}, {3, 0xc, 0x4, {false, false, false, false, false}, {}, 0}}}}}; - PGOAnalysisMap P4 = { - 1000, - {{BlockFrequency(1000), - {{1, BranchProbability::getRaw(0x2222'2222)}, - {2, BranchProbability::getRaw(0x3333'3333)}, - {3, BranchProbability::getRaw(0xaaaa'aaaa)}}}, - {BlockFrequency(133), - {{2, BranchProbability::getRaw(0x1111'1111)}, - {3, BranchProbability::getRaw(0xeeee'eeee)}}}, - {BlockFrequency(18), {{3, BranchProbability::getRaw(0xffff'ffff)}}}, - {BlockFrequency(1000), {}}}, - {true, true, true, false, false, false, false}}; + PGOAnalysisMap P4 = {1000, + {{BlockFrequency(1000), + 50, + {{1, BranchProbability::getRaw(0x2222'2222), 0}, + {2, BranchProbability::getRaw(0x3333'3333), 0}, + {3, BranchProbability::getRaw(0xaaaa'aaaa), 0}}}, + {BlockFrequency(133), + 0, + {{2, BranchProbability::getRaw(0x1111'1111), 0}, + {3, BranchProbability::getRaw(0xeeee'eeee), 0}}}, + {BlockFrequency(18), + 0, + {{3, BranchProbability::getRaw(0xffff'ffff), 0}}}, + {BlockFrequency(1000), 0, {}}}, + {true, true, true, false, false, false, false, true}}; BBAddrMap E5 = { {{0x55555, {{2, 0x0, 0x2, {false, false, true, false, false}, {}, 0}}}}}; PGOAnalysisMap P5 = { - {}, {}, {false, false, false, false, false, false, false}}; + {}, {}, {false, false, false, false, false, false, false, false}}; BBAddrMap E6 = { {{0x66666, {{0, 0x0, 0x6, {false, true, true, false, false}, {}, 0}, {1, 0x6, 0x6, {false, false, true, false, false}, {}, 0}}}, {0x666661, {{2, 0x0, 0x6, {false, false, false, false, false}, {}, 0}}}}}; - PGOAnalysisMap P6 = {{}, - {{{}, - {{1, BranchProbability::getRaw(0x2222'2222)}, - {2, BranchProbability::getRaw(0xcccc'cccc)}}}, - {{}, {{2, BranchProbability::getRaw(0x8888'8888)}}}, - {{}, {}}}, - {false, false, true, true, false, false, false}}; + PGOAnalysisMap P6 = { + {}, + {{{}, + 0, + {{1, BranchProbability::getRaw(0x2222'2222), 7}, + {2, BranchProbability::getRaw(0xcccc'cccc), 0}}}, + {{}, 0, {{2, BranchProbability::getRaw(0x8888'8888), 0}}}, + {{}, 0, {}}}, + {false, false, true, true, false, false, false, true}}; std::vector Section0BBAddrMaps = {E4, E5, E6}; std::vector Section1BBAddrMaps = {E3}; @@ -1465,7 +1474,7 @@ TEST(ELFObjectFileTest, ReadPGOAnalysisMap) { DoCheckFails( TruncatedYamlString, /*TextSectionIndex=*/std::nullopt, "unable to read SHT_LLVM_BB_ADDR_MAP section with index 6: " - "unexpected end of data at offset 0xa while reading [0x3, 0xb)"); + "unexpected end of data at offset 0xa while reading [0x4, 0xc)"); // Check that we can read the other section's bb-address-maps which are // valid. DoCheckSucceeds(TruncatedYamlString, /*TextSectionIndex=*/2, diff --git a/llvm/unittests/Object/ELFTypesTest.cpp b/llvm/unittests/Object/ELFTypesTest.cpp index 1765e15003963..9e99b4a6d7bf3 100644 --- a/llvm/unittests/Object/ELFTypesTest.cpp +++ b/llvm/unittests/Object/ELFTypesTest.cpp @@ -101,22 +101,24 @@ static_assert( "PGOAnalysisMap should use the same type for basic block ID as BBAddrMap"); TEST(ELFTypesTest, BBAddrMapFeaturesEncodingTest) { - const std::array Decoded = { - {{false, false, false, false, false, false, false}, - {true, false, false, false, false, false, false}, - {false, true, false, false, false, false, false}, - {false, false, true, false, false, false, false}, - {false, false, false, true, false, false, false}, - {true, true, false, false, false, false, false}, - {false, true, true, false, false, false, false}, - {false, true, true, true, false, false, false}, - {true, true, true, true, false, false, false}, - {false, false, false, false, true, false, false}, - {false, false, false, false, false, true, false}, - {false, false, false, false, false, false, true}}}; - const std::array Encoded = { + const std::array Decoded = { + {{false, false, false, false, false, false, false, false}, + {true, false, false, false, false, false, false, false}, + {false, true, false, false, false, false, false, false}, + {false, false, true, false, false, false, false, false}, + {false, false, false, true, false, false, false, false}, + {true, true, false, false, false, false, false, false}, + {false, true, true, false, false, false, false, false}, + {false, true, true, true, false, false, false, false}, + {true, true, true, true, false, false, false, false}, + {false, false, false, false, true, false, false, false}, + {false, false, false, false, false, true, false, false}, + {false, false, false, false, false, false, true, false}, + {false, false, false, false, false, false, false, true}, + {false, false, false, false, false, false, true, true}}}; + const std::array Encoded = { {0b0000, 0b0001, 0b0010, 0b0100, 0b1000, 0b0011, 0b0110, 0b1110, 0b1111, - 0b1'0000, 0b10'0000, 0b100'0000}}; + 0b1'0000, 0b10'0000, 0b100'0000, 0b1000'0000, 0b1100'0000}}; for (const auto &[Feat, EncodedVal] : llvm::zip(Decoded, Encoded)) EXPECT_EQ(Feat.encode(), EncodedVal); for (const auto &[Feat, EncodedVal] : llvm::zip(Decoded, Encoded)) { @@ -129,9 +131,9 @@ TEST(ELFTypesTest, BBAddrMapFeaturesEncodingTest) { TEST(ELFTypesTest, BBAddrMapFeaturesInvalidEncodingTest) { const std::array Errors = { - "invalid encoding for BBAddrMap::Features: 0x80", - "invalid encoding for BBAddrMap::Features: 0xf0"}; - const std::array Values = {{0b1000'0000, 0b1111'0000}}; + "invalid encoding for BBAddrMap::Features: 0x100", + "invalid encoding for BBAddrMap::Features: 0x1000"}; + const std::array Values = {{0b1'0000'0000, 0b1'0000'0000'0000}}; for (const auto &[Val, Error] : llvm::zip(Values, Errors)) { EXPECT_THAT_ERROR(BBAddrMap::Features::decode(Val).takeError(), FailedWithMessage(Error));