[RISCV] Add new pass to transform undef to pseudo for vector values.

RISC-V vector instruction has register overlapping constraint for certain
instructions, and will cause illegal instruction trap if violated, we use
early clobber to model this constraint, but it can't prevent register allocator
allocated same or overlapped if the input register is undef value, so convert
IMPLICIT_DEF to temporary pseudo could prevent that happen, it's not best way
to resolve this. Ideally we should model the constraint right, but before we
model the constraint right, it's the approach to prevent that happen.

See also: #50157

Reviewed By: craig.topper

Differential Revision: https://reviews.llvm.org/D129735

llvm/lib/Target/RISCV/CMakeLists.txt

@@ -36,6 +36,7 @@ add_llvm_target(RISCVCodeGen
   RISCVMergeBaseOffset.cpp
   RISCVRedundantCopyElimination.cpp
   RISCVRegisterInfo.cpp
+  RISCVRVVInitUndef.cpp
   RISCVSExtWRemoval.cpp
   RISCVStripWSuffix.cpp
   RISCVSubtarget.cpp

llvm/lib/Target/RISCV/RISCV.h

@@ -71,6 +71,10 @@ void initializeRISCVInsertVSETVLIPass(PassRegistry &);
 FunctionPass *createRISCVRedundantCopyEliminationPass();
 void initializeRISCVRedundantCopyEliminationPass(PassRegistry &);
 
+FunctionPass *createRISCVInitUndefPass();
+void initializeRISCVInitUndefPass(PassRegistry &);
+extern char &RISCVInitUndefID;
+
 InstructionSelector *createRISCVInstructionSelector(const RISCVTargetMachine &,
                                                     RISCVSubtarget &,
                                                     RISCVRegisterBankInfo &);

llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp

@@ -113,6 +113,11 @@ void RISCVAsmPrinter::emitInstruction(const MachineInstr *MI) {
   case RISCV::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
     LowerHWASAN_CHECK_MEMACCESS(*MI);
     return;
+  case RISCV::PseudoRVVInitUndefM1:
+  case RISCV::PseudoRVVInitUndefM2:
+  case RISCV::PseudoRVVInitUndefM4:
+  case RISCV::PseudoRVVInitUndefM8:
+    return;
   }
 
   MCInst TmpInst;

llvm/lib/Target/RISCV/RISCVInstrInfo.td

@@ -1870,6 +1870,14 @@ def : Pat<(binop_allwusers<add> GPR:$rs1, (AddiPair:$rs2)),
                  (AddiPairImmSmall AddiPair:$rs2))>;
 }
 
+/// Empty pseudo for RISCVInitUndefPass
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0, Size = 0, isCodeGenOnly = 1 in {
+  def PseudoRVVInitUndefM1     :  Pseudo<(outs VR:$vd), (ins), [], "">;
+  def PseudoRVVInitUndefM2     :  Pseudo<(outs VRM2:$vd), (ins), [], "">;
+  def PseudoRVVInitUndefM4     :  Pseudo<(outs VRM4:$vd), (ins), [], "">;
+  def PseudoRVVInitUndefM8     :  Pseudo<(outs VRM8:$vd), (ins), [], "">;
+}
+
 //===----------------------------------------------------------------------===//
 // Standard extensions
 //===----------------------------------------------------------------------===//

llvm/lib/Target/RISCV/RISCVRVVInitUndef.cpp

@@ -0,0 +1,275 @@
+//===- RISCVInitUndef.cpp - Initialize undef vector value to pseudo -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a function pass that initializes undef vector value to
+// temporary pseudo instruction and remove it in expandpseudo pass to prevent
+// register allocation resulting in a constraint violated result for vector
+// instruction.
+//
+// RISC-V vector instruction has register overlapping constraint for certain
+// instructions, and will cause illegal instruction trap if violated, we use
+// early clobber to model this constraint, but it can't prevent register
+// allocator allocated same or overlapped if the input register is undef value,
+// so convert IMPLICIT_DEF to temporary pseudo instruction and remove it later
+// could prevent that happen, it's not best way to resolve this, and it might
+// change the order of program or increase the register pressure, so ideally we
+// should model the constraint right, but before we model the constraint right,
+// it's the only way to prevent that happen.
+//
+// When we enable the subregister liveness option, it will also trigger same
+// issue due to the partial of register is undef. If we pseudoinit the whole
+// register, then it will generate redundant COPY instruction. Currently, it
+// will generate INSERT_SUBREG to make sure the whole register is occupied
+// when program encounter operation that has early-clobber constraint.
+//
+//
+// See also: https://github.com/llvm/llvm-project/issues/50157
+//
+//===----------------------------------------------------------------------===//
+
+#include "RISCV.h"
+#include "RISCVSubtarget.h"
+#include "llvm/CodeGen/DetectDeadLanes.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "riscv-init-undef"
+#define RISCV_INIT_UNDEF_NAME "RISCV init undef pass"
+
+namespace {
+
+class RISCVInitUndef : public MachineFunctionPass {
+  const TargetInstrInfo *TII;
+  MachineRegisterInfo *MRI;
+  const RISCVSubtarget *ST;
+  const TargetRegisterInfo *TRI;
+
+public:
+  static char ID;
+
+  RISCVInitUndef() : MachineFunctionPass(ID) {
+    initializeRISCVInitUndefPass(*PassRegistry::getPassRegistry());
+  }
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  StringRef getPassName() const override { return RISCV_INIT_UNDEF_NAME; }
+
+private:
+  bool processBasicBlock(MachineFunction &MF, MachineBasicBlock &MBB,
+                         const DeadLaneDetector &DLD);
+  bool handleImplicitDef(MachineBasicBlock &MBB,
+                         MachineBasicBlock::iterator &Inst);
+  bool isVectorRegClass(const Register R);
+  const TargetRegisterClass *
+  getVRLargestSuperClass(const TargetRegisterClass *RC) const;
+  bool handleSubReg(MachineFunction &MF, MachineInstr &MI,
+                    const DeadLaneDetector &DLD);
+};
+
+} // end anonymous namespace
+
+char RISCVInitUndef::ID = 0;
+INITIALIZE_PASS(RISCVInitUndef, DEBUG_TYPE, RISCV_INIT_UNDEF_NAME, false, false)
+char &llvm::RISCVInitUndefID = RISCVInitUndef::ID;
+
+const TargetRegisterClass *
+RISCVInitUndef::getVRLargestSuperClass(const TargetRegisterClass *RC) const {
+  if (RISCV::VRM8RegClass.hasSubClassEq(RC))
+    return &RISCV::VRM8RegClass;
+  if (RISCV::VRM4RegClass.hasSubClassEq(RC))
+    return &RISCV::VRM4RegClass;
+  if (RISCV::VRM2RegClass.hasSubClassEq(RC))
+    return &RISCV::VRM2RegClass;
+  if (RISCV::VRRegClass.hasSubClassEq(RC))
+    return &RISCV::VRRegClass;
+  return RC;
+}
+
+bool RISCVInitUndef::isVectorRegClass(const Register R) {
+  const TargetRegisterClass *RC = MRI->getRegClass(R);
+  return RISCV::VRRegClass.hasSubClassEq(RC) ||
+         RISCV::VRM2RegClass.hasSubClassEq(RC) ||
+         RISCV::VRM4RegClass.hasSubClassEq(RC) ||
+         RISCV::VRM8RegClass.hasSubClassEq(RC);
+}
+
+static unsigned getUndefInitOpcode(unsigned RegClassID) {
+  switch (RegClassID) {
+  case RISCV::VRRegClassID:
+    return RISCV::PseudoRVVInitUndefM1;
+  case RISCV::VRM2RegClassID:
+    return RISCV::PseudoRVVInitUndefM2;
+  case RISCV::VRM4RegClassID:
+    return RISCV::PseudoRVVInitUndefM4;
+  case RISCV::VRM8RegClassID:
+    return RISCV::PseudoRVVInitUndefM8;
+  default:
+    llvm_unreachable("Unexpected register class.");
+  }
+}
+
+bool RISCVInitUndef::handleImplicitDef(MachineBasicBlock &MBB,
+                                       MachineBasicBlock::iterator &Inst) {
+  const TargetRegisterInfo &TRI =
+      *MBB.getParent()->getSubtarget().getRegisterInfo();
+
+  assert(Inst->getOpcode() == TargetOpcode::IMPLICIT_DEF);
+
+  Register Reg = Inst->getOperand(0).getReg();
+  if (!Reg.isVirtual())
+    return false;
+
+  bool NeedPseudoInit = false;
+  SmallVector<MachineOperand *, 1> UserMOs;
+  for (MachineOperand &MO : MRI->use_nodbg_operands(Reg)) {
+    MachineInstr *UserMI = MO.getParent();
+
+    bool HasEarlyClobber = false;
+    bool TiedToDef = false;
+    for (MachineOperand &UseMO : UserMI->operands()) {
+      if (!UseMO.isReg())
+        continue;
+      if (UseMO.isEarlyClobber())
+        HasEarlyClobber = true;
+      if (UseMO.isUse() && UseMO.isTied() &&
+          TRI.regsOverlap(UseMO.getReg(), Reg))
+        TiedToDef = true;
+    }
+    if (HasEarlyClobber && !TiedToDef) {
+      NeedPseudoInit = true;
+      UserMOs.push_back(&MO);
+    }
+  }
+
+  if (!NeedPseudoInit)
+    return false;
+
+  LLVM_DEBUG(
+      dbgs() << "Emitting PseudoRVVInitUndef for implicit vector register "
+             << Reg << '\n');
+
+  unsigned RegClassID = getVRLargestSuperClass(MRI->getRegClass(Reg))->getID();
+  unsigned Opcode = getUndefInitOpcode(RegClassID);
+
+  BuildMI(MBB, Inst, Inst->getDebugLoc(), TII->get(Opcode), Reg);
+
+  Inst = MBB.erase(Inst);
+
+  for (auto MO : UserMOs)
+    MO->setIsUndef(false);
+
+  return true;
+}
+
+static bool isEarlyClobberMI(MachineInstr &MI) {
+  return llvm::any_of(MI.defs(), [&](const MachineOperand &DefMO) {
+    return DefMO.isReg() && DefMO.isEarlyClobber();
+  });
+}
+
+bool RISCVInitUndef::handleSubReg(MachineFunction &MF, MachineInstr &MI,
+                                  const DeadLaneDetector &DLD) {
+  bool Changed = false;
+
+  for (MachineOperand &UseMO : MI.uses()) {
+    if (!UseMO.isReg())
+      continue;
+    if (!UseMO.getReg().isVirtual())
+      continue;
+
+    Register Reg = UseMO.getReg();
+    DeadLaneDetector::VRegInfo Info =
+        DLD.getVRegInfo(Register::virtReg2Index(Reg));
+
+    if (Info.UsedLanes == Info.DefinedLanes)
+      continue;
+
+    const TargetRegisterClass *TargetRegClass =
+        getVRLargestSuperClass(MRI->getRegClass(Reg));
+
+    LaneBitmask NeedDef = Info.UsedLanes & ~Info.DefinedLanes;
+
+    LLVM_DEBUG({
+      dbgs() << "Instruction has undef subregister.\n";
+      dbgs() << printReg(Reg, nullptr)
+             << " Used: " << PrintLaneMask(Info.UsedLanes)
+             << " Def: " << PrintLaneMask(Info.DefinedLanes)
+             << " Need Def: " << PrintLaneMask(NeedDef) << "\n";
+    });
+
+    SmallVector<unsigned> SubRegIndexNeedInsert;
+    TRI->getCoveringSubRegIndexes(*MRI, TargetRegClass, NeedDef,
+                                  SubRegIndexNeedInsert);
+
+    Register LatestReg = Reg;
+
+    for (auto ind : SubRegIndexNeedInsert) {
+      Changed = true;
+      const TargetRegisterClass *SubRegClass =
+          getVRLargestSuperClass(TRI->getSubRegisterClass(TargetRegClass, ind));
+      Register TmpInitSubReg = MRI->createVirtualRegister(SubRegClass);
+      BuildMI(*MI.getParent(), &MI, MI.getDebugLoc(),
+              TII->get(getUndefInitOpcode(SubRegClass->getID())),
+              TmpInitSubReg);
+      Register NewReg = MRI->createVirtualRegister(TargetRegClass);
+      BuildMI(*MI.getParent(), &MI, MI.getDebugLoc(),
+              TII->get(TargetOpcode::INSERT_SUBREG), NewReg)
+          .addReg(LatestReg)
+          .addReg(TmpInitSubReg)
+          .addImm(ind);
+      LatestReg = NewReg;
+    }
+
+    UseMO.setReg(LatestReg);
+  }
+
+  return Changed;
+}
+
+bool RISCVInitUndef::processBasicBlock(MachineFunction &MF,
+                                       MachineBasicBlock &MBB,
+                                       const DeadLaneDetector &DLD) {
+  bool Changed = false;
+  for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); ++I) {
+    MachineInstr &MI = *I;
+    if (MI.isImplicitDef()) {
+      auto DstReg = MI.getOperand(0).getReg();
+      if (isVectorRegClass(DstReg))
+        Changed |= handleImplicitDef(MBB, I);
+    }
+    if (ST->enableSubRegLiveness() && isEarlyClobberMI(MI))
+      Changed |= handleSubReg(MF, MI, DLD);
+  }
+  return Changed;
+}
+
+bool RISCVInitUndef::runOnMachineFunction(MachineFunction &MF) {
+  ST = &MF.getSubtarget<RISCVSubtarget>();
+  if (!ST->hasVInstructions())
+    return false;
+
+  MRI = &MF.getRegInfo();
+  TII = ST->getInstrInfo();
+  TRI = MRI->getTargetRegisterInfo();
+
+  bool Changed = false;
+  DeadLaneDetector DLD(MRI, TRI);
+  DLD.computeSubRegisterLaneBitInfo();
+
+  for (MachineBasicBlock &BB : MF)
+    Changed |= processBasicBlock(MF, BB, DLD);
+
+  return Changed;
+}
+
+FunctionPass *llvm::createRISCVInitUndefPass() { return new RISCVInitUndef(); }

llvm/lib/Target/RISCV/RISCVTargetMachine.cpp

@@ -81,6 +81,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
   initializeRISCVExpandPseudoPass(*PR);
   initializeRISCVInsertVSETVLIPass(*PR);
   initializeRISCVDAGToDAGISelPass(*PR);
+  initializeRISCVInitUndefPass(*PR);
 }
 
 static StringRef computeDataLayout(const Triple &TT) {
@@ -260,6 +261,7 @@ class RISCVPassConfig : public TargetPassConfig {
   void addMachineSSAOptimization() override;
   void addPreRegAlloc() override;
   void addPostRegAlloc() override;
+  void addOptimizedRegAlloc() override;
 };
 } // namespace
 
@@ -355,6 +357,13 @@ void RISCVPassConfig::addPreRegAlloc() {
   addPass(createRISCVInsertVSETVLIPass());
 }
 
+void RISCVPassConfig::addOptimizedRegAlloc() {
+  if (getOptimizeRegAlloc())
+    insertPass(&DetectDeadLanesID, &RISCVInitUndefID);
+
+  TargetPassConfig::addOptimizedRegAlloc();
+}
+
 void RISCVPassConfig::addPostRegAlloc() {
   if (TM->getOptLevel() != CodeGenOpt::None && EnableRedundantCopyElimination)
     addPass(createRISCVRedundantCopyEliminationPass());

llvm/test/CodeGen/RISCV/O3-pipeline.ll

@@ -108,6 +108,7 @@
 ; CHECK-NEXT:       RISCV Merge Base Offset
 ; CHECK-NEXT:       RISCV Insert VSETVLI pass
 ; CHECK-NEXT:       Detect Dead Lanes
+; CHECK-NEXT:       RISCV init undef pass
 ; CHECK-NEXT:       Process Implicit Definitions
 ; CHECK-NEXT:       Remove unreachable machine basic blocks
 ; CHECK-NEXT:       Live Variable Analysis