Port the remaining middle-end passes to the new pass manager

lgc/include/lgc/patch/Patch.h

@@ -46,19 +46,19 @@ class PassManager;
 
 void initializeLegacyLowerFragColorExportPass(PassRegistry &);
 void initializeLegacyLowerVertexFetchPass(PassRegistry &);
-void initializePatchBufferOpPass(PassRegistry &);
+void initializeLegacyPatchBufferOpPass(PassRegistry &);
 void initializeLegacyPatchCheckShaderCachePass(PassRegistry &);
 void initializeLegacyPatchCopyShaderPass(PassRegistry &);
 void initializeLegacyPatchEntryPointMutatePass(PassRegistry &);
 void initializeLegacyPatchInOutImportExportPass(PassRegistry &);
-void initializePatchLlvmIrInclusionPass(PassRegistry &);
+void initializeLegacyPatchLlvmIrInclusionPass(PassRegistry &);
 void initializeLegacyPatchLoadScalarizerPass(PassRegistry &);
 void initializeLegacyPatchLoopMetadataPass(PassRegistry &);
 void initializeLegacyPatchNullFragShaderPass(PassRegistry &);
 void initializeLegacyPatchPeepholeOptPass(PassRegistry &);
 void initializeLegacyPatchPreparePipelineAbiPass(PassRegistry &);
 void initializeLegacyPatchResourceCollectPass(PassRegistry &);
-void initializePatchSetupTargetFeaturesPass(PassRegistry &);
+void initializeLegacyPatchSetupTargetFeaturesPass(PassRegistry &);
 void initializeLegacyPatchWorkaroundsPass(PassRegistry &);
 void initializeLegacyPatchReadFirstLanePass(PassRegistry &);
 void initializeLegacyPatchWaveSizeAdjustPass(PassRegistry &);
@@ -76,19 +76,19 @@ class LegacyPatchCheckShaderCache;
 inline void initializePatchPasses(llvm::PassRegistry &passRegistry) {
   initializeLegacyLowerFragColorExportPass(passRegistry);
   initializeLegacyLowerVertexFetchPass(passRegistry);
-  initializePatchBufferOpPass(passRegistry);
+  initializeLegacyPatchBufferOpPass(passRegistry);
   initializeLegacyPatchCheckShaderCachePass(passRegistry);
   initializeLegacyPatchCopyShaderPass(passRegistry);
   initializeLegacyPatchEntryPointMutatePass(passRegistry);
   initializeLegacyPatchInOutImportExportPass(passRegistry);
-  initializePatchLlvmIrInclusionPass(passRegistry);
+  initializeLegacyPatchLlvmIrInclusionPass(passRegistry);
   initializeLegacyPatchLoadScalarizerPass(passRegistry);
   initializeLegacyPatchLoopMetadataPass(passRegistry);
   initializeLegacyPatchNullFragShaderPass(passRegistry);
   initializeLegacyPatchPeepholeOptPass(passRegistry);
   initializeLegacyPatchPreparePipelineAbiPass(passRegistry);
   initializeLegacyPatchResourceCollectPass(passRegistry);
-  initializePatchSetupTargetFeaturesPass(passRegistry);
+  initializeLegacyPatchSetupTargetFeaturesPass(passRegistry);
   initializeLegacyPatchWorkaroundsPass(passRegistry);
   initializeLegacyPatchReadFirstLanePass(passRegistry);
   initializeLegacyPatchWaveSizeAdjustPass(passRegistry);
@@ -97,19 +97,19 @@ inline void initializePatchPasses(llvm::PassRegistry &passRegistry) {
 
 llvm::ModulePass *createLegacyLowerFragColorExport();
 llvm::ModulePass *createLegacyLowerVertexFetch();
-llvm::FunctionPass *createPatchBufferOp();
+llvm::FunctionPass *createLegacyPatchBufferOp();
 LegacyPatchCheckShaderCache *createLegacyPatchCheckShaderCache();
 llvm::ModulePass *createLegacyPatchCopyShader();
 llvm::ModulePass *createLegacyPatchEntryPointMutate();
 llvm::ModulePass *createLegacyPatchInOutImportExport();
-llvm::ModulePass *createPatchLlvmIrInclusion();
+llvm::ModulePass *createLegacyPatchLlvmIrInclusion();
 llvm::FunctionPass *createLegacyPatchLoadScalarizer();
 llvm::LoopPass *createLegacyPatchLoopMetadata();
 llvm::ModulePass *createLegacyPatchNullFragShader();
 llvm::FunctionPass *createLegacyPatchPeepholeOpt();
 llvm::ModulePass *createLegacyPatchPreparePipelineAbi(bool onlySetCallingConvs);
 llvm::ModulePass *createLegacyPatchResourceCollect();
-llvm::ModulePass *createPatchSetupTargetFeatures();
+llvm::ModulePass *createLegacyPatchSetupTargetFeatures();
 llvm::ModulePass *createLegacyPatchWorkarounds();
 llvm::FunctionPass *createLegacyPatchReadFirstLane();
 llvm::ModulePass *createLegacyPatchWaveSizeAdjust();

lgc/patch/PatchBufferOp.h → lgc/include/lgc/patch/PatchBufferOp.h

@@ -35,22 +35,22 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstVisitor.h"
 
-namespace llvm {
-class LegacyDivergenceAnalysis;
-} // namespace llvm
-
 namespace lgc {
 
 class PipelineState;
 
 // =====================================================================================================================
 // Represents the pass of LLVM patching operations for buffer operations
-class PatchBufferOp final : public llvm::FunctionPass, public llvm::InstVisitor<PatchBufferOp> {
+class PatchBufferOp : public llvm::InstVisitor<PatchBufferOp>, public llvm::PassInfoMixin<PatchBufferOp> {
 public:
-  PatchBufferOp();
+  llvm::PreservedAnalyses run(llvm::Function &function, llvm::FunctionAnalysisManager &analysisManager);
 
-  void getAnalysisUsage(llvm::AnalysisUsage &analysisUsage) const override;
-  bool runOnFunction(llvm::Function &function) override;
+  // NOTE: Once the switch to the new pass manager is completed, the isDivergent argument can be remove and the
+  // divergent analysis can be put back as class attribute.
+  bool runImpl(llvm::Function &function, PipelineState *pipelineState,
+               std::function<bool(const llvm::Value &)> isDivergent);
+
+  static llvm::StringRef name() { return "Patch LLVM for buffer operations"; }
 
   // Visitors
   void visitAtomicCmpXchgInst(llvm::AtomicCmpXchgInst &atomicCmpXchgInst);
@@ -70,12 +70,7 @@ class PatchBufferOp final : public llvm::FunctionPass, public llvm::InstVisitor<
   void visitICmpInst(llvm::ICmpInst &icmpInst);
   void visitPtrToIntInst(llvm::PtrToIntInst &ptrToIntInst);
 
-  static char ID; // ID of this pass
-
 private:
-  PatchBufferOp(const PatchBufferOp &) = delete;
-  PatchBufferOp &operator=(const PatchBufferOp &) = delete;
-
   llvm::Value *getPointerOperandAsInst(llvm::Value *const value);
   llvm::Value *getBaseAddressFromBufferDesc(llvm::Value *const bufferDesc) const;
   void copyMetadata(llvm::Value *const dest, const llvm::Value *const src) const;
@@ -95,13 +90,33 @@ class PatchBufferOp final : public llvm::FunctionPass, public llvm::InstVisitor<
   llvm::DenseMap<PhiIncoming, llvm::Value *> m_incompletePhis; // The incomplete phi map.
   llvm::DenseSet<llvm::Value *> m_invariantSet;                // The invariant set.
   llvm::DenseSet<llvm::Value *> m_divergenceSet;               // The divergence set.
-  llvm::LegacyDivergenceAnalysis *m_divergenceAnalysis;        // The divergence analysis.
   llvm::SmallVector<llvm::Instruction *, 16> m_postVisitInsts; // The post process instruction set.
   std::unique_ptr<llvm::IRBuilder<>> m_builder;                // The IRBuilder.
   llvm::LLVMContext *m_context;                                // The LLVM context.
   PipelineState *m_pipelineState;                              // The pipeline state
 
+  std::function<bool(const llvm::Value &)> m_isDivergent;
+
   static constexpr unsigned MinMemOpLoopBytes = 256;
 };
 
+// =====================================================================================================================
+// Represents the pass of LLVM patching operations for buffer operations
+class LegacyPatchBufferOp final : public llvm::FunctionPass, public llvm::InstVisitor<LegacyPatchBufferOp> {
+public:
+  LegacyPatchBufferOp();
+
+  bool runOnFunction(llvm::Function &function) override;
+
+  void getAnalysisUsage(llvm::AnalysisUsage &analysisUsage) const override;
+
+  static char ID; // NOLINT
+
+private:
+  LegacyPatchBufferOp(const LegacyPatchBufferOp &) = delete;
+  LegacyPatchBufferOp &operator=(const LegacyPatchBufferOp &) = delete;
+
+  PatchBufferOp m_impl;
+};
+
 } // namespace lgc

lgc/patch/PatchLlvmIrInclusion.h → lgc/include/lgc/patch/PatchLlvmIrInclusion.h

@@ -36,9 +36,20 @@ namespace lgc {
 
 // =====================================================================================================================
 // Represents the pass of LLVM patch operations of including LLVM IR as a separate section in the ELF binary.
-class PatchLlvmIrInclusion : public LegacyPatch {
+class PatchLlvmIrInclusion : public Patch, public llvm::PassInfoMixin<PatchLlvmIrInclusion> {
 public:
-  PatchLlvmIrInclusion();
+  llvm::PreservedAnalyses run(llvm::Module &module, llvm::ModuleAnalysisManager &analysisManager);
+
+  bool runImpl(llvm::Module &module);
+
+  static llvm::StringRef name() { return "Include LLVM IR as a separate section in the ELF binary"; }
+};
+
+// =====================================================================================================================
+// Represents the pass of LLVM patch operations of including LLVM IR as a separate section in the ELF binary.
+class LegacyPatchLlvmIrInclusion : public LegacyPatch {
+public:
+  LegacyPatchLlvmIrInclusion();
 
   bool runOnModule(llvm::Module &module) override;
 
@@ -47,8 +58,10 @@ class PatchLlvmIrInclusion : public LegacyPatch {
   static char ID; // ID of this pass
 
 private:
-  PatchLlvmIrInclusion(const PatchLlvmIrInclusion &) = delete;
-  PatchLlvmIrInclusion &operator=(const PatchLlvmIrInclusion &) = delete;
+  LegacyPatchLlvmIrInclusion(const LegacyPatchLlvmIrInclusion &) = delete;
+  LegacyPatchLlvmIrInclusion &operator=(const LegacyPatchLlvmIrInclusion &) = delete;
+
+  PatchLlvmIrInclusion m_impl;
 };
 
 } // namespace lgc

lgc/include/lgc/patch/PatchSetupTargetFeatures.h

@@ -0,0 +1,56 @@
+/*
+ ***********************************************************************************************************************
+ *
+ *  Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in all
+ *  copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *  SOFTWARE.
+ *
+ **********************************************************************************************************************/
+/**
+ ***********************************************************************************************************************
+ * @file  PatchSetupTargetFeatures.h
+ * @brief LLPC header file: contains declaration of class lgc::PatchSetupTargetFeatures.
+ ***********************************************************************************************************************
+ */
+#pragma once
+
+#include "lgc/patch/Patch.h"
+#include "lgc/state/PipelineShaders.h"
+#include "lgc/state/PipelineState.h"
+#include "lgc/util/BuilderBase.h"
+
+namespace lgc {
+
+// =====================================================================================================================
+// Pass to set up target features on shader entry-points
+class PatchSetupTargetFeatures : public Patch, public llvm::PassInfoMixin<PatchSetupTargetFeatures> {
+public:
+  llvm::PreservedAnalyses run(llvm::Module &module, llvm::ModuleAnalysisManager &analysisManager);
+
+  bool runImpl(llvm::Module &module, PipelineState *pipelineState);
+
+  static llvm::StringRef name() { return "Patch LLVM to set up target features"; }
+
+private:
+  void setupTargetFeatures(llvm::Module *module);
+
+  PipelineState *m_pipelineState;
+};
+
+} // namespace lgc

lgc/patch/Patch.cpp

@@ -34,17 +34,20 @@
 #include "lgc/PassManager.h"
 #include "lgc/builder/BuilderReplayer.h"
 #include "lgc/patch/FragColorExport.h"
+#include "lgc/patch/PatchBufferOp.h"
 #include "lgc/patch/PatchCheckShaderCache.h"
 #include "lgc/patch/PatchCopyShader.h"
 #include "lgc/patch/PatchEntryPointMutate.h"
 #include "lgc/patch/PatchInOutImportExport.h"
 #include "lgc/patch/PatchInitializeWorkgroupMemory.h"
+#include "lgc/patch/PatchLlvmIrInclusion.h"
 #include "lgc/patch/PatchLoadScalarizer.h"
 #include "lgc/patch/PatchLoopMetadata.h"
 #include "lgc/patch/PatchPeepholeOpt.h"
 #include "lgc/patch/PatchPreparePipelineAbi.h"
 #include "lgc/patch/PatchReadFirstLane.h"
 #include "lgc/patch/PatchResourceCollect.h"
+#include "lgc/patch/PatchSetupTargetFeatures.h"
 #include "lgc/patch/PatchWaveSizeAdjust.h"
 #include "lgc/patch/PatchWorkarounds.h"
 #include "lgc/patch/VertexFetch.h"
@@ -85,6 +88,7 @@
 #include "llvm/Transforms/Scalar/SimplifyCFG.h"
 #include "llvm/Transforms/Scalar/SpeculativeExecution.h"
 #include "llvm/Transforms/Utils.h"
+#include "llvm/Transforms/Utils/Mem2Reg.h"
 
 #define DEBUG_TYPE "lgc-patch"
 
@@ -180,8 +184,57 @@ void Patch::addPasses(PipelineState *pipelineState, lgc::PassManager &passMgr, b
     LgcContext::createAndAddStartStopTimer(passMgr, patchTimer, true);
   }
 
-  // NOTE: The new pass manager is not fully implemented yet. We should add all
-  // the other patching passes here.
+  // Patch buffer operations (must be after optimizations)
+  passMgr.addPass(createModuleToFunctionPassAdaptor(PatchBufferOp()));
+  passMgr.addPass(createModuleToFunctionPassAdaptor(InstCombinePass(2)));
+
+  // Fully prepare the pipeline ABI (must be after optimizations)
+  passMgr.addPass(PatchPreparePipelineAbi(/* onlySetCallingConvs = */ false));
+
+  const bool canUseNgg = pipelineState->isGraphics() && pipelineState->getTargetInfo().getGfxIpVersion().major == 10 &&
+                         (pipelineState->getOptions().nggFlags & NggFlagDisable) == 0;
+  if (canUseNgg) {
+    // Stop timer for patching passes and restart timer for optimization passes.
+    if (patchTimer) {
+      LgcContext::createAndAddStartStopTimer(passMgr, patchTimer, false);
+      LgcContext::createAndAddStartStopTimer(passMgr, optTimer, true);
+    }
+
+    // Extra optimizations after NGG primitive shader creation
+    passMgr.addPass(AlwaysInlinerPass());
+    passMgr.addPass(GlobalDCEPass());
+    passMgr.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
+    passMgr.addPass(createModuleToFunctionPassAdaptor(ADCEPass()));
+    passMgr.addPass(createModuleToFunctionPassAdaptor(InstCombinePass()));
+    passMgr.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass()));
+
+    // Stop timer for optimization passes and restart timer for patching passes.
+    if (patchTimer) {
+      LgcContext::createAndAddStartStopTimer(passMgr, optTimer, false);
+      LgcContext::createAndAddStartStopTimer(passMgr, patchTimer, true);
+    }
+  }
+
+  // Set up target features in shader entry-points.
+  // NOTE: Needs to be done after post-NGG function inlining, because LLVM refuses to inline something
+  // with conflicting attributes. Attributes could conflict on GFX10 because PatchSetupTargetFeatures
+  // adds a target feature to determine wave32 or wave64.
+  passMgr.addPass(PatchSetupTargetFeatures());
+
+  // Include LLVM IR as a separate section in the ELF binary
+  if (pipelineState->getOptions().includeIr)
+    passMgr.addPass(PatchLlvmIrInclusion());
+
+  // Stop timer for patching passes.
+  if (patchTimer)
+    LgcContext::createAndAddStartStopTimer(passMgr, patchTimer, false);
+
+  // Dump the result
+  if (raw_ostream *outs = getLgcOuts()) {
+    passMgr.addPass(PrintModulePass(*outs,
+                                    "===============================================================================\n"
+                                    "// LLPC pipeline patching results\n"));
+  }
 }
 
 // =====================================================================================================================
@@ -278,7 +331,7 @@ void LegacyPatch::addPasses(PipelineState *pipelineState, legacy::PassManager &p
   }
 
   // Patch buffer operations (must be after optimizations)
-  passMgr.add(createPatchBufferOp());
+  passMgr.add(createLegacyPatchBufferOp());
   passMgr.add(createInstructionCombiningPass(2));
 
   // Fully prepare the pipeline ABI (must be after optimizations)
@@ -312,11 +365,11 @@ void LegacyPatch::addPasses(PipelineState *pipelineState, legacy::PassManager &p
   // NOTE: Needs to be done after post-NGG function inlining, because LLVM refuses to inline something
   // with conflicting attributes. Attributes could conflict on GFX10 because PatchSetupTargetFeatures
   // adds a target feature to determine wave32 or wave64.
-  passMgr.add(createPatchSetupTargetFeatures());
+  passMgr.add(createLegacyPatchSetupTargetFeatures());
 
   // Include LLVM IR as a separate section in the ELF binary
   if (pipelineState->getOptions().includeIr)
-    passMgr.add(createPatchLlvmIrInclusion());
+    passMgr.add(createLegacyPatchLlvmIrInclusion());
 
   // Stop timer for patching passes.
   if (patchTimer)
@@ -353,7 +406,7 @@ void Patch::addOptimizationPasses(lgc::PassManager &passMgr, CodeGenOpt::Level o
   passMgr.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass()));
   passMgr.addPass(createModuleToFunctionPassAdaptor(ReassociatePass()));
   passMgr.addPass(createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor(LoopRotatePass())));
-  passMgr.addPass(createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor(LICMPass())));
+  passMgr.addPass(createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor(LICMPass(), true)));
   passMgr.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass()));
   passMgr.addPass(createModuleToFunctionPassAdaptor(InstCombinePass(1)));
   passMgr.addPass(createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor(IndVarSimplifyPass())));