From d208cd0072c15a84a2305f8d953482e6a85ddb83 Mon Sep 17 00:00:00 2001 From: Rahman Lavaee Date: Tue, 23 Sep 2025 19:44:33 +0000 Subject: [PATCH 1/7] [Propeller] Ingest the CFG profile. --- .../CodeGen/BasicBlockSectionsProfileReader.h | 11 ++++ .../BasicBlockSectionsProfileReader.cpp | 54 +++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h index f0cfa7663c5fa..f14d4104e332e 100644 --- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h +++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h @@ -50,6 +50,10 @@ struct FunctionPathAndClusterInfo { // the edge a -> b (a is not cloned). The index of the path in this vector // determines the `UniqueBBID::CloneID` of the cloned blocks in that path. SmallVector> ClonePaths; + // Node counts for each basic block. + DenseMap NodeCounts; + // Edge counts for each edge, stored as a nested map. + DenseMap> EdgeCounts; }; class BasicBlockSectionsProfileReader { @@ -77,6 +81,11 @@ class BasicBlockSectionsProfileReader { SmallVector> getClonePathsForFunction(StringRef FuncName) const; + // Returns the profile count for the edge from `SrcBBID` to `SinkBBID` in + // function `FuncName` or zero if it does not exist. + uint64_t getEdgeCount(StringRef FuncName, const UniqueBBID &SrcBBID, + const UniqueBBID &SinkBBID); + private: StringRef getAliasName(StringRef FuncName) const { auto R = FuncAliasMap.find(FuncName); @@ -183,6 +192,8 @@ class BasicBlockSectionsProfileReaderWrapperPass : public ImmutablePass { SmallVector> getClonePathsForFunction(StringRef FuncName) const; + uint64_t getEdgeCount(StringRef FuncName, const UniqueBBID &SrcBBID, const UniqueBBID &DestBBID) const; + // Initializes the FunctionNameToDIFilename map for the current module and // then reads the profile for the matching functions. bool doInitialization(Module &M) override; diff --git a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp index 7baeb3fd7bcee..ef04d7bb670b5 100644 --- a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp +++ b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp @@ -76,6 +76,22 @@ BasicBlockSectionsProfileReader::getClonePathsForFunction( return ProgramPathAndClusterInfo.lookup(getAliasName(FuncName)).ClonePaths; } + uint64_t BasicBlockSectionsProfileReader::getEdgeCount(StringRef FuncName, const UniqueBBID &SrcBBID, + const UniqueBBID &SinkBBID) { + auto It = ProgramPathAndClusterInfo.find(getAliasName(FuncName)); + if (It == ProgramPathAndClusterInfo.end()) + return 0; + auto NodeIt = It->second.EdgeCounts.find(SrcBBID); + if (NodeIt == It->second.EdgeCounts.end()) + return 0; + auto EdgeIt = NodeIt->second.find(DestBBID); + if (EdgeIt == NodeIt->second.end()) + return 0; + return EdgeIt->second; + } + + + // Reads the version 1 basic block sections profile. Profile for each function // is encoded as follows: // m @@ -240,6 +256,38 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() { } continue; } + case 'g': { // CFG profile specifier. + // Skip the profile when we the profile iterator (FI) refers to the + // past-the-end element. + if (FI == ProgramPathAndClusterInfo.end()) + continue; + // For each node, its CFG profile is encoded as + // :,:,:,... + for (auto BasicBlockEdgeProfile : Values) { + if (BasicBlockEdgeProfile.empty()) + continue; + SmallVector NodeEdgeCounts; + BasicBlockEdgeProfile.split(NodeEdgeCounts, ','); + UniqueBBID SrcBBID; + for (int i = 0; i < NodeEdgeCounts.size(); ++i) { + auto [BBIDStr, CountStr] = NodeEdgeCounts[i].split(':'); + auto BBID = parseUniqueBBID(BBIDStr); + if (!BBID) + return BBID.takeError(); + unsigned long long Count = 0; + if (getAsUnsignedInteger(CountStr, 10, Count)) + return createProfileParseError( + Twine("unsigned integer expected: '") + CountStr + "'"); + if (i == 0) { + // The first element represents the source and its total count. + FI->second.NodeCounts[SrcBBID = *BBID] = Count; + continue; + } + FI->second.EdgeCounts[SrcBBID][*BBID] = Count; + } + } + continue; + } default: return createProfileParseError(Twine("invalid specifier: '") + Twine(Specifier) + "'"); @@ -440,11 +488,17 @@ BasicBlockSectionsProfileReaderWrapperPass::getClonePathsForFunction( return BBSPR.getClonePathsForFunction(FuncName); } +uint64_t BasicBlockSectionsProfileReaderWrapperPass::getEdgeCount(StringRef FuncName, const UniqueBBID &SrcBBID, const UniqueBBID &SinkBBID) const { + return BBSPR.getEdgeCount(FuncName, SrcBBID, SinkBBID); +} + BasicBlockSectionsProfileReader & BasicBlockSectionsProfileReaderWrapperPass::getBBSPR() { return BBSPR; } + + ImmutablePass *llvm::createBasicBlockSectionsProfileReaderWrapperPass( const MemoryBuffer *Buf) { return new BasicBlockSectionsProfileReaderWrapperPass(Buf); From 7cabfa569c52bec88b5151fdf767be9ecda494e7 Mon Sep 17 00:00:00 2001 From: Rahman Lavaee Date: Tue, 23 Sep 2025 19:44:51 +0000 Subject: [PATCH 2/7] clang-format. --- .../CodeGen/BasicBlockSectionsProfileReader.h | 3 +- .../BasicBlockSectionsProfileReader.cpp | 34 +++++++++---------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h index f14d4104e332e..c683ec6111513 100644 --- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h +++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h @@ -192,7 +192,8 @@ class BasicBlockSectionsProfileReaderWrapperPass : public ImmutablePass { SmallVector> getClonePathsForFunction(StringRef FuncName) const; - uint64_t getEdgeCount(StringRef FuncName, const UniqueBBID &SrcBBID, const UniqueBBID &DestBBID) const; + uint64_t getEdgeCount(StringRef FuncName, const UniqueBBID &SrcBBID, + const UniqueBBID &DestBBID) const; // Initializes the FunctionNameToDIFilename map for the current module and // then reads the profile for the matching functions. diff --git a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp index ef04d7bb670b5..97d56ce8942e8 100644 --- a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp +++ b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp @@ -76,21 +76,19 @@ BasicBlockSectionsProfileReader::getClonePathsForFunction( return ProgramPathAndClusterInfo.lookup(getAliasName(FuncName)).ClonePaths; } - uint64_t BasicBlockSectionsProfileReader::getEdgeCount(StringRef FuncName, const UniqueBBID &SrcBBID, - const UniqueBBID &SinkBBID) { - auto It = ProgramPathAndClusterInfo.find(getAliasName(FuncName)); - if (It == ProgramPathAndClusterInfo.end()) - return 0; - auto NodeIt = It->second.EdgeCounts.find(SrcBBID); - if (NodeIt == It->second.EdgeCounts.end()) - return 0; - auto EdgeIt = NodeIt->second.find(DestBBID); - if (EdgeIt == NodeIt->second.end()) - return 0; - return EdgeIt->second; - } - - +uint64_t BasicBlockSectionsProfileReader::getEdgeCount( + StringRef FuncName, const UniqueBBID &SrcBBID, const UniqueBBID &SinkBBID) { + auto It = ProgramPathAndClusterInfo.find(getAliasName(FuncName)); + if (It == ProgramPathAndClusterInfo.end()) + return 0; + auto NodeIt = It->second.EdgeCounts.find(SrcBBID); + if (NodeIt == It->second.EdgeCounts.end()) + return 0; + auto EdgeIt = NodeIt->second.find(DestBBID); + if (EdgeIt == NodeIt->second.end()) + return 0; + return EdgeIt->second; +} // Reads the version 1 basic block sections profile. Profile for each function // is encoded as follows: @@ -488,7 +486,9 @@ BasicBlockSectionsProfileReaderWrapperPass::getClonePathsForFunction( return BBSPR.getClonePathsForFunction(FuncName); } -uint64_t BasicBlockSectionsProfileReaderWrapperPass::getEdgeCount(StringRef FuncName, const UniqueBBID &SrcBBID, const UniqueBBID &SinkBBID) const { +uint64_t BasicBlockSectionsProfileReaderWrapperPass::getEdgeCount( + StringRef FuncName, const UniqueBBID &SrcBBID, + const UniqueBBID &SinkBBID) const { return BBSPR.getEdgeCount(FuncName, SrcBBID, SinkBBID); } @@ -497,8 +497,6 @@ BasicBlockSectionsProfileReaderWrapperPass::getBBSPR() { return BBSPR; } - - ImmutablePass *llvm::createBasicBlockSectionsProfileReaderWrapperPass( const MemoryBuffer *Buf) { return new BasicBlockSectionsProfileReaderWrapperPass(Buf); From 15bd1b161786a445cf8480757f0b7deefebe61ed Mon Sep 17 00:00:00 2001 From: Rahman Lavaee Date: Tue, 23 Sep 2025 20:32:10 +0000 Subject: [PATCH 3/7] Fix const. --- llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h | 2 +- llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h index c683ec6111513..82dd5feb31dba 100644 --- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h +++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h @@ -84,7 +84,7 @@ class BasicBlockSectionsProfileReader { // Returns the profile count for the edge from `SrcBBID` to `SinkBBID` in // function `FuncName` or zero if it does not exist. uint64_t getEdgeCount(StringRef FuncName, const UniqueBBID &SrcBBID, - const UniqueBBID &SinkBBID); + const UniqueBBID &SinkBBID) const; private: StringRef getAliasName(StringRef FuncName) const { diff --git a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp index 97d56ce8942e8..912654b45aa7e 100644 --- a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp +++ b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp @@ -77,14 +77,14 @@ BasicBlockSectionsProfileReader::getClonePathsForFunction( } uint64_t BasicBlockSectionsProfileReader::getEdgeCount( - StringRef FuncName, const UniqueBBID &SrcBBID, const UniqueBBID &SinkBBID) { + StringRef FuncName, const UniqueBBID &SrcBBID, const UniqueBBID &SinkBBID) const { auto It = ProgramPathAndClusterInfo.find(getAliasName(FuncName)); if (It == ProgramPathAndClusterInfo.end()) return 0; auto NodeIt = It->second.EdgeCounts.find(SrcBBID); if (NodeIt == It->second.EdgeCounts.end()) return 0; - auto EdgeIt = NodeIt->second.find(DestBBID); + auto EdgeIt = NodeIt->second.find(SinkBBID); if (EdgeIt == NodeIt->second.end()) return 0; return EdgeIt->second; From 0ed45908c27e7167392985548a36f690d6ff5875 Mon Sep 17 00:00:00 2001 From: Rahman Lavaee Date: Wed, 24 Sep 2025 00:27:53 +0000 Subject: [PATCH 4/7] Add and update tests. --- .../X86/basic-block-sections-clusters-error.ll | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/llvm/test/CodeGen/X86/basic-block-sections-clusters-error.ll b/llvm/test/CodeGen/X86/basic-block-sections-clusters-error.ll index d6f3d5010b556..751ab76722c07 100644 --- a/llvm/test/CodeGen/X86/basic-block-sections-clusters-error.ll +++ b/llvm/test/CodeGen/X86/basic-block-sections-clusters-error.ll @@ -57,6 +57,19 @@ ; RUN: echo 'p 1 2 3 2' >> %t13 ; RUN: not --crash llc < %s -O0 -mtriple=x86_64 -function-sections -basic-block-sections=%t13 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR13 ; CHECK-ERROR13: LLVM ERROR: invalid profile {{.*}} at line 4: duplicate cloned block in path: '2' +; RUN: echo 'v1' > %t14 +; RUN: echo 'f dummy1' >> %t14 +; RUN: echo 'c 0 1' >> %t14 +; RUN: echo 'g 0,1:2' >> %t14 +; RUN: not --crash llc < %s -O0 -mtriple=x86_64 -function-sections -basic-block-sections=%t14 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR14 +; CHECK-ERROR14: LLVM ERROR: invalid profile {{.*}} at line 4: unsigned integer expected: '' +; RUN: echo 'v1' > %t15 +; RUN: echo 'f dummy1' >> %t15 +; RUN: echo 'c 0 1' >> %t15 +; RUN: echo 'g 0:4,1:2:3' >> %t15 +; RUN: not --crash llc < %s -O0 -mtriple=x86_64 -function-sections -basic-block-sections=%t15 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR15 +; CHECK-ERROR15: LLVM ERROR: invalid profile {{.*}} at line 4: unsigned integer expected: '2:3' + define i32 @dummy1(i32 %x, i32 %y, i32 %z) { entry: From 95552ccf9846693c34738ebc33e6995e1f320030 Mon Sep 17 00:00:00 2001 From: Rahman Lavaee Date: Wed, 24 Sep 2025 00:54:11 +0000 Subject: [PATCH 5/7] Add the cfg test. --- .../CodeGen/X86/basic-block-sections-cfg.ll | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 llvm/test/CodeGen/X86/basic-block-sections-cfg.ll diff --git a/llvm/test/CodeGen/X86/basic-block-sections-cfg.ll b/llvm/test/CodeGen/X86/basic-block-sections-cfg.ll new file mode 100644 index 0000000000000..b8eadc3cac36e --- /dev/null +++ b/llvm/test/CodeGen/X86/basic-block-sections-cfg.ll @@ -0,0 +1,40 @@ +; BB section test with CFG. +; +;; Profile for version 1: +; RUN: echo 'v1' > %t +; RUN: echo 'f foo' >> %t +; RUN: echo 'g 0:10,1:9,2:1 1:8,3:8 2:2,3:2 3:11' >> %t +; RUN: echo 'c 0 2 3' >> %t +; +; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t | FileCheck %s +; +define void @foo(i1 zeroext) nounwind { + %2 = alloca i8, align 1 + %3 = zext i1 %0 to i8 + store i8 %3, ptr %2, align 1 + %4 = load i8, ptr %2, align 1 + %5 = trunc i8 %4 to i1 + br i1 %5, label %6, label %8 + +6: ; preds = %1 + %7 = call i32 @bar() + br label %10 + +8: ; preds = %1 + %9 = call i32 @baz() + br label %10 + +10: ; preds = %8, %6 + ret void +} + +declare i32 @bar() #1 + +declare i32 @baz() #1 + +; CHECK: .section .text.foo,"ax",@progbits +; CHECK: callq baz +; CHECK: retq +; CHECK: .section .text.split.foo,"ax",@progbits +; CHECK: callq bar + From 6a1b49e8d1ae54992b3a29e2eaef86d7f9b0d19f Mon Sep 17 00:00:00 2001 From: Rahman Lavaee Date: Wed, 24 Sep 2025 02:47:26 +0000 Subject: [PATCH 6/7] clang-format. --- llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp index 912654b45aa7e..d552bc346b262 100644 --- a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp +++ b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp @@ -77,7 +77,8 @@ BasicBlockSectionsProfileReader::getClonePathsForFunction( } uint64_t BasicBlockSectionsProfileReader::getEdgeCount( - StringRef FuncName, const UniqueBBID &SrcBBID, const UniqueBBID &SinkBBID) const { + StringRef FuncName, const UniqueBBID &SrcBBID, + const UniqueBBID &SinkBBID) const { auto It = ProgramPathAndClusterInfo.find(getAliasName(FuncName)); if (It == ProgramPathAndClusterInfo.end()) return 0; From 6d88773553a7d6e5f0b74592cbaa12b171f4e2ab Mon Sep 17 00:00:00 2001 From: Rahman Lavaee Date: Wed, 24 Sep 2025 16:33:50 +0000 Subject: [PATCH 7/7] Fix type comparision warning. --- llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp index d552bc346b262..fbcd614b85d18 100644 --- a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp +++ b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp @@ -268,7 +268,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() { SmallVector NodeEdgeCounts; BasicBlockEdgeProfile.split(NodeEdgeCounts, ','); UniqueBBID SrcBBID; - for (int i = 0; i < NodeEdgeCounts.size(); ++i) { + for (size_t i = 0; i < NodeEdgeCounts.size(); ++i) { auto [BBIDStr, CountStr] = NodeEdgeCounts[i].split(':'); auto BBID = parseUniqueBBID(BBIDStr); if (!BBID)