Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 12 additions & 9 deletions llvm/include/llvm/Analysis/StaticDataProfileInfo.h
Original file line number Diff line number Diff line change
Expand Up @@ -60,11 +60,18 @@ class StaticDataProfileInfo {
LLVM_ABI StaticDataHotness getConstantHotnessUsingProfileCount(
const Constant *C, const ProfileSummaryInfo *PSI, uint64_t Count) const;

/// Return the hotness based on section prefix \p SectionPrefix.
LLVM_ABI StaticDataHotness getSectionHotnessUsingDataAccessProfile(
std::optional<StringRef> SectionPrefix) const;

/// Return the string representation of the hotness enum \p Hotness.
LLVM_ABI StringRef hotnessToStr(StaticDataHotness Hotness) const;

bool EnableDataAccessProf = false;

public:
StaticDataProfileInfo() = default;
StaticDataProfileInfo(bool EnableDataAccessProf)
: EnableDataAccessProf(EnableDataAccessProf) {}

/// If \p Count is not nullopt, add it to the profile count of the constant \p
/// C in a saturating way, and clamp the count to \p getInstrMaxCountValue if
Expand All @@ -73,14 +80,10 @@ class StaticDataProfileInfo {
LLVM_ABI void addConstantProfileCount(const Constant *C,
std::optional<uint64_t> Count);

/// Return a section prefix for the constant \p C based on its profile count.
/// - If a constant doesn't have a counter, return an empty string.
/// - Otherwise,
/// - If it has a hot count, return "hot".
/// - If it is seen by unprofiled function, return an empty string.
/// - If it has a cold count, return "unlikely".
/// - Otherwise (e.g. it's used by lukewarm functions), return an empty
/// string.
/// Given a constant \p C, returns a section prefix.
/// If \p C is a global variable, the section prefix is the bigger one
/// between its existing section prefix and its use profile count. Otherwise,
/// the section prefix is based on its use profile count.
LLVM_ABI StringRef getConstantSectionPrefix(
const Constant *C, const ProfileSummaryInfo *PSI) const;
};
Expand Down
70 changes: 69 additions & 1 deletion llvm/lib/Analysis/StaticDataProfileInfo.cpp
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
#include "llvm/Analysis/StaticDataProfileInfo.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/Module.h"
#include "llvm/InitializePasses.h"
#include "llvm/ProfileData/InstrProf.h"

#define DEBUG_TYPE "static-data-profile-info"

using namespace llvm;

namespace llvm {
Expand Down Expand Up @@ -79,6 +83,17 @@ StaticDataProfileInfo::getConstantHotnessUsingProfileCount(
return StaticDataHotness::LukewarmOrUnknown;
}

StaticDataProfileInfo::StaticDataHotness
StaticDataProfileInfo::getSectionHotnessUsingDataAccessProfile(
std::optional<StringRef> MaybeSectionPrefix) const {
if (!MaybeSectionPrefix)
return StaticDataHotness::LukewarmOrUnknown;
StringRef Prefix = *MaybeSectionPrefix;
assert((Prefix == "hot" || Prefix == "unlikely") &&
"Expect section_prefix to be one of hot or unlikely");
return Prefix == "hot" ? StaticDataHotness::Hot : StaticDataHotness::Cold;
}

StringRef StaticDataProfileInfo::hotnessToStr(StaticDataHotness Hotness) const {
switch (Hotness) {
case StaticDataHotness::Cold:
Expand All @@ -101,13 +116,66 @@ StaticDataProfileInfo::getConstantProfileCount(const Constant *C) const {
StringRef StaticDataProfileInfo::getConstantSectionPrefix(
const Constant *C, const ProfileSummaryInfo *PSI) const {
std::optional<uint64_t> Count = getConstantProfileCount(C);

#ifndef NDEBUG
auto DbgPrintPrefix = [](StringRef Prefix) {
return Prefix.empty() ? "<empty>" : Prefix;
};
#endif

if (EnableDataAccessProf) {
// Module flag `HasDataAccessProf` is 1 -> empty section prefix means
// unknown hotness except for string literals.
if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(C);
GV && llvm::memprof::IsAnnotationOK(*GV) &&
!GV->getName().starts_with(".str")) {
auto HotnessFromDataAccessProf =
getSectionHotnessUsingDataAccessProfile(GV->getSectionPrefix());

if (!Count) {
StringRef Prefix = hotnessToStr(HotnessFromDataAccessProf);
LLVM_DEBUG(dbgs() << GV->getName() << " has section prefix "
<< DbgPrintPrefix(Prefix)
<< ", solely from data access profiles\n");
return Prefix;
}

// Both data access profiles and PGO counters are available. Use the
// hotter one.
auto HotnessFromPGO = getConstantHotnessUsingProfileCount(C, PSI, *Count);
StaticDataHotness GlobalVarHotness = StaticDataHotness::LukewarmOrUnknown;
if (HotnessFromDataAccessProf == StaticDataHotness::Hot ||
HotnessFromPGO == StaticDataHotness::Hot) {
GlobalVarHotness = StaticDataHotness::Hot;
} else if (HotnessFromDataAccessProf ==
StaticDataHotness::LukewarmOrUnknown ||
HotnessFromPGO == StaticDataHotness::LukewarmOrUnknown) {
GlobalVarHotness = StaticDataHotness::LukewarmOrUnknown;
} else {
GlobalVarHotness = StaticDataHotness::Cold;
}
StringRef Prefix = hotnessToStr(GlobalVarHotness);
LLVM_DEBUG(
dbgs() << GV->getName() << " has section prefix "
<< DbgPrintPrefix(Prefix)
<< ", the max from data access profiles as "
<< DbgPrintPrefix(hotnessToStr(HotnessFromDataAccessProf))
<< " and PGO counters as "
<< DbgPrintPrefix(hotnessToStr(HotnessFromPGO)) << "\n");
return Prefix;
}
}
if (!Count)
return "";
return hotnessToStr(getConstantHotnessUsingProfileCount(C, PSI, *Count));
}

bool StaticDataProfileInfoWrapperPass::doInitialization(Module &M) {
Info.reset(new StaticDataProfileInfo());
bool EnableDataAccessProf = false;
if (auto *MD = mdconst::extract_or_null<ConstantInt>(
M.getModuleFlag("EnableDataAccessProf")))
EnableDataAccessProf = MD->getZExtValue();
Info.reset(new StaticDataProfileInfo(EnableDataAccessProf));
return false;
}

Expand Down
98 changes: 92 additions & 6 deletions llvm/test/CodeGen/X86/global-variable-partition-with-dap.ll
Original file line number Diff line number Diff line change
@@ -1,17 +1,101 @@
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

;; A minimal test case. Subsequent PRs will expand on this test case
;; (e.g., with more functions, variables and profiles) and test the hotness
;; reconcillation implementation.
;; Requires asserts for -debug-only.
; REQUIRES: asserts

; RUN: rm -rf %t && split-file %s %t && cd %t

; RUN: llc -mtriple=x86_64-unknown-linux-gnu -relocation-model=pic \
; RUN: -partition-static-data-sections=true \
; RUN: -debug-only=static-data-profile-info \
; RUN: -data-sections=true -unique-section-names=false \
; RUN: input-with-data-access-prof-on.ll -o - 2>&1 | FileCheck %s --check-prefixes=LOG,IR

; RUN: llc -mtriple=x86_64-unknown-linux-gnu -relocation-model=pic \
; RUN: -partition-static-data-sections=true \
; RUN: -debug-only=static-data-profile-info \
; RUN: -data-sections=true -unique-section-names=false \
; RUN: %s -o - 2>&1 | FileCheck %s --check-prefix=IR
; RUN: input-with-data-access-prof-off.ll -o - 2>&1 | FileCheck %s --check-prefixes=OFF

; LOG: hot_bss has section prefix hot, the max from data access profiles as hot and PGO counters as hot
; LOG: data_unknown_hotness has section prefix <empty>, the max from data access profiles as <empty> and PGO counters as unlikely
; LOG: external_relro_array has section prefix unlikely, solely from data access profiles

; IR: .type hot_bss,@object
; IR-NEXT: .section .bss.hot.,"aw"
; IR: .type data_unknown_hotness,@object
; IR-NEXT: .section .data,"aw"
; IR: .type external_relro_array,@object
; IR-NEXT: .section .data.rel.ro.unlikely.,"aw"


; OFF: .type hot_bss,@object
; OFF-NEXT: .section .bss.hot.,"aw"
; OFF: .type data_unknown_hotness,@object
; OFF-NEXT: .section .data.unlikely.,"aw"
;; Global variable section prefix metadata is not used when
;; module flag `EnableDataAccessProf` is 0, and @external_relro_array has
;; external linkage, so analysis based on PGO counters doesn't apply.
; OFF: .type external_relro_array,@object # @external_relro_array
; OFF-NEXT: .section .data.rel.ro,"aw"

;--- input-with-data-access-prof-on.ll
; Internal vars
@hot_bss = internal global i32 0, !section_prefix !17
@data_unknown_hotness = internal global i32 1
; External vars
@external_relro_array = constant [2 x ptr] [ptr @hot_bss, ptr @data_unknown_hotness], !section_prefix !18

define void @cold_func() !prof !15 {
%9 = load i32, ptr @data_unknown_hotness
%11 = call i32 (...) @func_taking_arbitrary_param(i32 %9)
ret void
}

define void @hot_func() !prof !14 {
%9 = load i32, ptr @hot_bss
%11 = call i32 (...) @func_taking_arbitrary_param(i32 %9)
ret void
}

declare i32 @func_taking_arbitrary_param(...)

; IR: .section .bss.hot.,"aw"
!llvm.module.flags = !{!0, !1}

!0 = !{i32 2, !"EnableDataAccessProf", i32 1}
!1 = !{i32 1, !"ProfileSummary", !2}
!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
!3 = !{!"ProfileFormat", !"InstrProf"}
!4 = !{!"TotalCount", i64 1460183}
!5 = !{!"MaxCount", i64 849024}
!6 = !{!"MaxInternalCount", i64 32769}
!7 = !{!"MaxFunctionCount", i64 849024}
!8 = !{!"NumCounts", i64 23627}
!9 = !{!"NumFunctions", i64 3271}
!10 = !{!"DetailedSummary", !11}
!11 = !{!12, !13}
!12 = !{i32 990000, i64 166, i32 73}
!13 = !{i32 999999, i64 3, i32 1443}
!14 = !{!"function_entry_count", i64 100000}
!15 = !{!"function_entry_count", i64 1}
!16 = !{!"branch_weights", i32 1, i32 99999}
!17 = !{!"section_prefix", !"hot"}
!18 = !{!"section_prefix", !"unlikely"}

;--- input-with-data-access-prof-off.ll
; Same as file above except that module flag `EnableDataAccessProf` has value 0.
; Internal vars
@hot_bss = internal global i32 0, !section_prefix !17
@data_unknown_hotness = internal global i32 1
; External vars
@external_relro_array = constant [2 x ptr] [ptr @hot_bss, ptr @data_unknown_hotness], !section_prefix !18

define void @cold_func() !prof !15 {
%9 = load i32, ptr @data_unknown_hotness
%11 = call i32 (...) @func_taking_arbitrary_param(i32 %9)
ret void
}

define void @hot_func() !prof !14 {
%9 = load i32, ptr @hot_bss
Expand All @@ -21,8 +105,9 @@ define void @hot_func() !prof !14 {

declare i32 @func_taking_arbitrary_param(...)

!llvm.module.flags = !{!1}
!llvm.module.flags = !{!0, !1}

!0 = !{i32 2, !"EnableDataAccessProf", i32 0}
!1 = !{i32 1, !"ProfileSummary", !2}
!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
!3 = !{!"ProfileFormat", !"InstrProf"}
Expand All @@ -40,3 +125,4 @@ declare i32 @func_taking_arbitrary_param(...)
!15 = !{!"function_entry_count", i64 1}
!16 = !{!"branch_weights", i32 1, i32 99999}
!17 = !{!"section_prefix", !"hot"}
!18 = !{!"section_prefix", !"unlikely"}