Skip to content

Commit

Permalink
Reland "[Propeller] Promote functions with propeller profiles to .tex…
Browse files Browse the repository at this point in the history
…t.hot."

This relands commit 4d8d258.

The major change here is using 'addUsedIfAvailable<BasicBlockSectionsProfileReader>()` to make sure we don't change the pipeline tests.

Differential Revision: https://reviews.llvm.org/D126518
  • Loading branch information
rlavaee committed May 27, 2022
1 parent b709f07 commit 08cc058
Show file tree
Hide file tree
Showing 12 changed files with 318 additions and 176 deletions.
109 changes: 109 additions & 0 deletions llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
@@ -0,0 +1,109 @@
//===-- BasicBlockSectionsProfileReader.h - BB sections profile reader pass ==//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This pass creates the basic block cluster info by reading the basic block
// sections profile. The cluster info will be used by the basic-block-sections
// pass to arrange basic blocks in their sections.
//
//===----------------------------------------------------------------------===//

#ifndef LLVM_ANALYSIS_BASICBLOCKSECTIONSINFO_H
#define LLVM_ANALYSIS_BASICBLOCKSECTIONSINFO_H

#include "llvm/ADT/Optional.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/LineIterator.h"
#include "llvm/Support/MemoryBuffer.h"

using namespace llvm;

namespace llvm {

// The cluster information for a machine basic block.
struct BBClusterInfo {
// MachineBasicBlock ID.
unsigned MBBNumber;
// Cluster ID this basic block belongs to.
unsigned ClusterID;
// Position of basic block within the cluster.
unsigned PositionInCluster;
};

using ProgramBBClusterInfoMapTy = StringMap<SmallVector<BBClusterInfo>>;

class BasicBlockSectionsProfileReader : public ImmutablePass {
public:
static char ID;

BasicBlockSectionsProfileReader(const MemoryBuffer *Buf)
: ImmutablePass(ID), MBuf(Buf) {
initializeBasicBlockSectionsProfileReaderPass(
*PassRegistry::getPassRegistry());
};

BasicBlockSectionsProfileReader() : ImmutablePass(ID) {
initializeBasicBlockSectionsProfileReaderPass(
*PassRegistry::getPassRegistry());
}

StringRef getPassName() const override {
return "Basic Block Sections Profile Reader";
}

// Returns true if basic block sections profile exist for function \p
// FuncName.
bool isFunctionHot(StringRef FuncName) const;

// Returns a pair with first element representing whether basic block sections
// profile exist for the function \p FuncName, and the second element
// representing the basic block sections profile (cluster info) for this
// function. If the first element is true and the second element is empty, it
// means unique basic block sections are desired for all basic blocks of the
// function.
std::pair<bool, SmallVector<BBClusterInfo>>
getBBClusterInfoForFunction(StringRef FuncName) const;

/// Read profiles of basic blocks if available here.
void initializePass() override;

private:
StringRef getAliasName(StringRef FuncName) const {
auto R = FuncAliasMap.find(FuncName);
return R == FuncAliasMap.end() ? FuncName : R->second;
}

// This contains the basic-block-sections profile.
const MemoryBuffer *MBuf = nullptr;

// This encapsulates the BB cluster information for the whole program.
//
// For every function name, it contains the cluster information for (all or
// some of) its basic blocks. The cluster information for every basic block
// includes its cluster ID along with the position of the basic block in that
// cluster.
ProgramBBClusterInfoMapTy ProgramBBClusterInfo;

// Some functions have alias names. We use this map to find the main alias
// name for which we have mapping in ProgramBBClusterInfo.
StringMap<StringRef> FuncAliasMap;
};

// Creates a BasicBlockSectionsProfileReader pass to parse the basic block
// sections profile. \p Buf is a memory buffer that contains the list of
// functions and basic block ids to selectively enable basic block sections.
ImmutablePass *
createBasicBlockSectionsProfileReaderPass(const MemoryBuffer *Buf);

} // namespace llvm
#endif // LLVM_ANALYSIS_BASICBLOCKSECTIONSINFO_H
6 changes: 2 additions & 4 deletions llvm/include/llvm/CodeGen/Passes.h
Expand Up @@ -51,10 +51,8 @@ namespace llvm {
FunctionPass *createUnreachableBlockEliminationPass();

/// createBasicBlockSections Pass - This pass assigns sections to machine
/// basic blocks and is enabled with -fbasic-block-sections. Buf is a memory
/// buffer that contains the list of functions and basic block ids to
/// selectively enable basic block sections.
MachineFunctionPass *createBasicBlockSectionsPass(const MemoryBuffer *Buf);
/// basic blocks and is enabled with -fbasic-block-sections.
MachineFunctionPass *createBasicBlockSectionsPass();

/// createMachineFunctionSplitterPass - This pass splits machine functions
/// using profile information.
Expand Down
1 change: 1 addition & 0 deletions llvm/include/llvm/InitializePasses.h
Expand Up @@ -76,6 +76,7 @@ void initializeAssumptionCacheTrackerPass(PassRegistry&);
void initializeAtomicExpandPass(PassRegistry&);
void initializeAttributorLegacyPassPass(PassRegistry&);
void initializeAttributorCGSCCLegacyPassPass(PassRegistry &);
void initializeBasicBlockSectionsProfileReaderPass(PassRegistry &);
void initializeBasicBlockSectionsPass(PassRegistry &);
void initializeBDCELegacyPassPass(PassRegistry&);
void initializeBarrierNoopPass(PassRegistry&);
Expand Down
169 changes: 16 additions & 153 deletions llvm/lib/CodeGen/BasicBlockSections.cpp
Expand Up @@ -69,25 +69,17 @@
//===----------------------------------------------------------------------===//

#include "llvm/ADT/Optional.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
#include "llvm/CodeGen/BasicBlockSectionUtils.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/Passes.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/InitializePasses.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/LineIterator.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Target/TargetMachine.h"

using llvm::SmallSet;
using llvm::SmallVector;
using llvm::StringMap;
using llvm::StringRef;
using namespace llvm;

// Placing the cold clusters in a separate section mitigates against poor
Expand All @@ -107,41 +99,11 @@ cl::opt<bool> BBSectionsDetectSourceDrift(

namespace {

// This struct represents the cluster information for a machine basic block.
struct BBClusterInfo {
// MachineBasicBlock ID.
unsigned MBBNumber;
// Cluster ID this basic block belongs to.
unsigned ClusterID;
// Position of basic block within the cluster.
unsigned PositionInCluster;
};

using ProgramBBClusterInfoMapTy = StringMap<SmallVector<BBClusterInfo, 4>>;

class BasicBlockSections : public MachineFunctionPass {
public:
static char ID;

// This contains the basic-block-sections profile.
const MemoryBuffer *MBuf = nullptr;

// This encapsulates the BB cluster information for the whole program.
//
// For every function name, it contains the cluster information for (all or
// some of) its basic blocks. The cluster information for every basic block
// includes its cluster ID along with the position of the basic block in that
// cluster.
ProgramBBClusterInfoMapTy ProgramBBClusterInfo;

// Some functions have alias names. We use this map to find the main alias
// name for which we have mapping in ProgramBBClusterInfo.
StringMap<StringRef> FuncAliasMap;

BasicBlockSections(const MemoryBuffer *Buf)
: MachineFunctionPass(ID), MBuf(Buf) {
initializeBasicBlockSectionsPass(*PassRegistry::getPassRegistry());
};
BasicBlockSectionsProfileReader *BBSectionsProfileReader = nullptr;

BasicBlockSections() : MachineFunctionPass(ID) {
initializeBasicBlockSectionsPass(*PassRegistry::getPassRegistry());
Expand All @@ -153,9 +115,6 @@ class BasicBlockSections : public MachineFunctionPass {

void getAnalysisUsage(AnalysisUsage &AU) const override;

/// Read profiles of basic blocks if available here.
bool doInitialization(Module &M) override;

/// Identify basic blocks that need separate sections and prepare to emit them
/// accordingly.
bool runOnMachineFunction(MachineFunction &MF) override;
Expand Down Expand Up @@ -205,29 +164,26 @@ static void updateBranches(

// This function provides the BBCluster information associated with a function.
// Returns true if a valid association exists and false otherwise.
static bool getBBClusterInfoForFunction(
const MachineFunction &MF, const StringMap<StringRef> FuncAliasMap,
const ProgramBBClusterInfoMapTy &ProgramBBClusterInfo,
bool getBBClusterInfoForFunction(
const MachineFunction &MF,
BasicBlockSectionsProfileReader *BBSectionsProfileReader,
std::vector<Optional<BBClusterInfo>> &V) {
// Get the main alias name for the function.
auto FuncName = MF.getName();
auto R = FuncAliasMap.find(FuncName);
StringRef AliasName = R == FuncAliasMap.end() ? FuncName : R->second;

// Find the assoicated cluster information.
auto P = ProgramBBClusterInfo.find(AliasName);
if (P == ProgramBBClusterInfo.end())
std::pair<bool, SmallVector<BBClusterInfo, 4>> P =
BBSectionsProfileReader->getBBClusterInfoForFunction(MF.getName());
if (!P.first)
return false;

if (P->second.empty()) {
if (P.second.empty()) {
// This indicates that sections are desired for all basic blocks of this
// function. We clear the BBClusterInfo vector to denote this.
V.clear();
return true;
}

V.resize(MF.getNumBlockIDs());
for (auto bbClusterInfo : P->second) {
for (auto bbClusterInfo : P.second) {
// Bail out if the cluster information contains invalid MBB numbers.
if (bbClusterInfo.MBBNumber >= MF.getNumBlockIDs())
return false;
Expand Down Expand Up @@ -376,9 +332,11 @@ bool BasicBlockSections::runOnMachineFunction(MachineFunction &MF) {
return true;
}

BBSectionsProfileReader = &getAnalysis<BasicBlockSectionsProfileReader>();

std::vector<Optional<BBClusterInfo>> FuncBBClusterInfo;
if (BBSectionsType == BasicBlockSection::List &&
!getBBClusterInfoForFunction(MF, FuncAliasMap, ProgramBBClusterInfo,
!getBBClusterInfoForFunction(MF, BBSectionsProfileReader,
FuncBBClusterInfo))
return true;
MF.setBBSectionsType(BBSectionsType);
Expand Down Expand Up @@ -426,107 +384,12 @@ bool BasicBlockSections::runOnMachineFunction(MachineFunction &MF) {
return true;
}

// Basic Block Sections can be enabled for a subset of machine basic blocks.
// This is done by passing a file containing names of functions for which basic
// block sections are desired. Additionally, machine basic block ids of the
// functions can also be specified for a finer granularity. Moreover, a cluster
// of basic blocks could be assigned to the same section.
// A file with basic block sections for all of function main and three blocks
// for function foo (of which 1 and 2 are placed in a cluster) looks like this:
// ----------------------------
// list.txt:
// !main
// !foo
// !!1 2
// !!4
static Error getBBClusterInfo(const MemoryBuffer *MBuf,
ProgramBBClusterInfoMapTy &ProgramBBClusterInfo,
StringMap<StringRef> &FuncAliasMap) {
assert(MBuf);
line_iterator LineIt(*MBuf, /*SkipBlanks=*/true, /*CommentMarker=*/'#');

auto invalidProfileError = [&](auto Message) {
return make_error<StringError>(
Twine("Invalid profile " + MBuf->getBufferIdentifier() + " at line " +
Twine(LineIt.line_number()) + ": " + Message),
inconvertibleErrorCode());
};

auto FI = ProgramBBClusterInfo.end();

// Current cluster ID corresponding to this function.
unsigned CurrentCluster = 0;
// Current position in the current cluster.
unsigned CurrentPosition = 0;

// Temporary set to ensure every basic block ID appears once in the clusters
// of a function.
SmallSet<unsigned, 4> FuncBBIDs;

for (; !LineIt.is_at_eof(); ++LineIt) {
StringRef S(*LineIt);
if (S[0] == '@')
continue;
// Check for the leading "!"
if (!S.consume_front("!") || S.empty())
break;
// Check for second "!" which indicates a cluster of basic blocks.
if (S.consume_front("!")) {
if (FI == ProgramBBClusterInfo.end())
return invalidProfileError(
"Cluster list does not follow a function name specifier.");
SmallVector<StringRef, 4> BBIndexes;
S.split(BBIndexes, ' ');
// Reset current cluster position.
CurrentPosition = 0;
for (auto BBIndexStr : BBIndexes) {
unsigned long long BBIndex;
if (getAsUnsignedInteger(BBIndexStr, 10, BBIndex))
return invalidProfileError(Twine("Unsigned integer expected: '") +
BBIndexStr + "'.");
if (!FuncBBIDs.insert(BBIndex).second)
return invalidProfileError(Twine("Duplicate basic block id found '") +
BBIndexStr + "'.");
if (!BBIndex && CurrentPosition)
return invalidProfileError("Entry BB (0) does not begin a cluster.");

FI->second.emplace_back(BBClusterInfo{
((unsigned)BBIndex), CurrentCluster, CurrentPosition++});
}
CurrentCluster++;
} else { // This is a function name specifier.
// Function aliases are separated using '/'. We use the first function
// name for the cluster info mapping and delegate all other aliases to
// this one.
SmallVector<StringRef, 4> Aliases;
S.split(Aliases, '/');
for (size_t i = 1; i < Aliases.size(); ++i)
FuncAliasMap.try_emplace(Aliases[i], Aliases.front());

// Prepare for parsing clusters of this function name.
// Start a new cluster map for this function name.
FI = ProgramBBClusterInfo.try_emplace(Aliases.front()).first;
CurrentCluster = 0;
FuncBBIDs.clear();
}
}
return Error::success();
}

bool BasicBlockSections::doInitialization(Module &M) {
if (!MBuf)
return false;
if (auto Err = getBBClusterInfo(MBuf, ProgramBBClusterInfo, FuncAliasMap))
report_fatal_error(std::move(Err));
return false;
}

void BasicBlockSections::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesAll();
AU.addRequired<BasicBlockSectionsProfileReader>();
MachineFunctionPass::getAnalysisUsage(AU);
}

MachineFunctionPass *
llvm::createBasicBlockSectionsPass(const MemoryBuffer *Buf) {
return new BasicBlockSections(Buf);
MachineFunctionPass *llvm::createBasicBlockSectionsPass() {
return new BasicBlockSections();
}

0 comments on commit 08cc058

Please sign in to comment.