[DebugInfo][InstrRef] Track values fused into stack spills

During register allocation, some instructions can have stack spills fused
into them. It means that when vregs are allocated on the stack we can
convert:

    SETCCr %0
    DBG_VALUE %0

to

    SETCCm %stack.0
    DBG_VALUE %stack.0

Unfortunately instruction referencing finds this harder: a store to the
stack doesn't have a specific operand number, therefore we don't substitute
the old operand for a new operand, and the location is dropped. This patch
implements a solution: just recognise the memory operand attached to an
instruction with a Special Number (TM), and record a substitution between
the old value and the new one.

This patch adds substitution code to InlineSpiller to record such fused
spills, and tracking in InstrRefBasedLDV to recognise such values, and
produce the value numbers for them. Everything to do with the movement of
stack-defined values is already handled in InstrRefBasedLDV.

Differential Revision: https://reviews.llvm.org/D111317

llvm/include/llvm/CodeGen/MachineFunction.h

@@ -544,6 +544,10 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
   /// instruction referencing.
   bool useDebugInstrRef() const;
 
+  /// A reserved operand number representing the instructions memory operand,
+  /// for instructions that have a stack spill fused into them.
+  const static unsigned int DebugOperandMemNumber;
+
   MachineFunction(Function &F, const LLVMTargetMachine &Target,
                   const TargetSubtargetInfo &STI, unsigned FunctionNum,
                   MachineModuleInfo &MMI);

llvm/lib/CodeGen/InlineSpiller.cpp

@@ -928,6 +928,39 @@ foldMemoryOperand(ArrayRef<std::pair<MachineInstr *, unsigned>> Ops,
   // Update the call site info.
   if (MI->isCandidateForCallSiteEntry())
     MI->getMF()->moveCallSiteInfo(MI, FoldMI);
+
+  // If we've folded a store into an instruction labelled with debug-info,
+  // record a substitution from the old operand to the memory operand. Handle
+  // the simple common case where operand 0 is the one being folded, plus when
+  // the destination operand is also a tied def. More values could be
+  // substituted / preserved with more analysis.
+  if (MI->peekDebugInstrNum() && Ops[0].second == 0) {
+    // Helper lambda.
+    auto MakeSubstitution = [this,FoldMI,MI,&Ops]() {
+      // Substitute old operand zero to the new instructions memory operand.
+      unsigned OldOperandNum = Ops[0].second;
+      unsigned NewNum = FoldMI->getDebugInstrNum();
+      unsigned OldNum = MI->getDebugInstrNum();
+      MF.makeDebugValueSubstitution({OldNum, OldOperandNum},
+                         {NewNum, MachineFunction::DebugOperandMemNumber});
+    };
+
+    const MachineOperand &Op0 = MI->getOperand(Ops[0].second);
+    if (Ops.size() == 1 && Op0.isDef()) {
+      MakeSubstitution();
+    } else if (Ops.size() == 2 && Op0.isDef() && MI->getOperand(1).isTied() &&
+               Op0.getReg() == MI->getOperand(1).getReg()) {
+      MakeSubstitution();
+    }
+  } else if (MI->peekDebugInstrNum()) {
+    // This is a debug-labelled instruction, but the operand being folded isn't
+    // at operand zero. Most likely this means it's a load being folded in.
+    // Substitute any register defs from operand zero up to the one being
+    // folded -- past that point, we don't know what the new operand indexes
+    // will be.
+    MF.substituteDebugValuesForInst(*MI, *FoldMI, Ops[0].second);
+  }
+
   MI->eraseFromParent();
 
   // Insert any new instructions other than FoldMI into the LIS maps.

llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp

@@ -884,6 +884,27 @@ InstrRefBasedLDV::extractSpillBaseRegAndOffset(const MachineInstr &MI) {
   return MTracker->getOrTrackSpillLoc({Reg, Offset});
 }
 
+Optional<LocIdx> InstrRefBasedLDV::findLocationForMemOperand(const MachineInstr &MI) {
+  SpillLocationNo SpillLoc =  extractSpillBaseRegAndOffset(MI);
+
+  // Where in the stack slot is this value defined -- i.e., what size of value
+  // is this? An important question, because it could be loaded into a register
+  // from the stack at some point. Happily the memory operand will tell us
+  // the size written to the stack.
+  auto *MemOperand = *MI.memoperands_begin();
+  unsigned SizeInBits = MemOperand->getSizeInBits();
+
+  // Find that position in the stack indexes we're tracking.
+  auto IdxIt = MTracker->StackSlotIdxes.find({SizeInBits, 0});
+  if (IdxIt == MTracker->StackSlotIdxes.end())
+    // That index is not tracked. This is suprising, and unlikely to ever
+    // occur, but the safe action is to indicate the variable is optimised out.
+    return None;
+
+  unsigned SpillID = MTracker->getSpillIDWithIdx(SpillLoc, IdxIt->second);
+  return MTracker->getSpillMLoc(SpillID);
+}
+
 /// End all previous ranges related to @MI and start a new range from @MI
 /// if it is a DBG_VALUE instr.
 bool InstrRefBasedLDV::transferDebugValue(const MachineInstr &MI) {
@@ -1010,16 +1031,25 @@ bool InstrRefBasedLDV::transferDebugInstrRef(MachineInstr &MI,
     const MachineInstr &TargetInstr = *InstrIt->second.first;
     uint64_t BlockNo = TargetInstr.getParent()->getNumber();
 
-    // Pick out the designated operand.
-    assert(OpNo < TargetInstr.getNumOperands());
-    const MachineOperand &MO = TargetInstr.getOperand(OpNo);
-
-    // Today, this can only be a register.
-    assert(MO.isReg() && MO.isDef());
-
-    unsigned LocID = MTracker->getLocID(MO.getReg());
-    LocIdx L = MTracker->LocIDToLocIdx[LocID];
-    NewID = ValueIDNum(BlockNo, InstrIt->second.second, L);
+    // Pick out the designated operand. It might be a memory reference, if
+    // a register def was folded into a stack store.
+    if (OpNo == MachineFunction::DebugOperandMemNumber &&
+        TargetInstr.hasOneMemOperand()) {
+      Optional<LocIdx> L = findLocationForMemOperand(TargetInstr);
+      if (L)
+        NewID = ValueIDNum(BlockNo, InstrIt->second.second, *L);
+    } else if (OpNo != MachineFunction::DebugOperandMemNumber) {
+      assert(OpNo < TargetInstr.getNumOperands());
+      const MachineOperand &MO = TargetInstr.getOperand(OpNo);
+
+      // Today, this can only be a register.
+      assert(MO.isReg() && MO.isDef());
+
+      unsigned LocID = MTracker->getLocID(MO.getReg());
+      LocIdx L = MTracker->LocIDToLocIdx[LocID];
+      NewID = ValueIDNum(BlockNo, InstrIt->second.second, L);
+    }
+    // else: NewID is left as None.
   } else if (PHIIt != DebugPHINumToValue.end() && PHIIt->InstrNum == InstNo) {
     // It's actually a PHI value. Which value it is might not be obvious, use
     // the resolver helper to find out.
@@ -1278,6 +1308,16 @@ void InstrRefBasedLDV::transferRegisterDef(MachineInstr &MI) {
   for (auto *MO : RegMaskPtrs)
     MTracker->writeRegMask(MO, CurBB, CurInst);
 
+  // If this instruction writes to a spill slot, def that slot.
+  if (hasFoldedStackStore(MI)) {
+    SpillLocationNo SpillNo = extractSpillBaseRegAndOffset(MI);
+    for (unsigned int I = 0; I < MTracker->NumSlotIdxes; ++I) {
+      unsigned SpillID = MTracker->getSpillIDWithIdx(SpillNo, I);
+      LocIdx L = MTracker->getSpillMLoc(SpillID);
+      MTracker->setMLoc(L, ValueIDNum(CurBB, CurInst, L));
+    }
+  }
+
   if (!TTracker)
     return;
 
@@ -1303,6 +1343,16 @@ void InstrRefBasedLDV::transferRegisterDef(MachineInstr &MI) {
       if (MO->clobbersPhysReg(Reg))
         TTracker->clobberMloc(L.Idx, MI.getIterator(), false);
   }
+
+  // Tell TTracker about any folded stack store.
+  if (hasFoldedStackStore(MI)) {
+    SpillLocationNo SpillNo = extractSpillBaseRegAndOffset(MI);
+    for (unsigned int I = 0; I < MTracker->NumSlotIdxes; ++I) {
+      unsigned SpillID = MTracker->getSpillIDWithIdx(SpillNo, I);
+      LocIdx L = MTracker->getSpillMLoc(SpillID);
+      TTracker->clobberMloc(L, MI.getIterator(), true);
+    }
+  }
 }
 
 void InstrRefBasedLDV::performCopy(Register SrcRegNum, Register DstRegNum) {
@@ -1342,6 +1392,12 @@ bool InstrRefBasedLDV::isSpillInstruction(const MachineInstr &MI,
   if (!MI.hasOneMemOperand())
     return false;
 
+  // Reject any memory operand that's aliased -- we can't guarantee its value.
+  auto MMOI = MI.memoperands_begin();
+  const PseudoSourceValue *PVal = (*MMOI)->getPseudoValue();
+  if (PVal->isAliased(MFI))
+    return false;
+
   if (!MI.getSpillSize(TII) && !MI.getFoldedSpillSize(TII))
     return false; // This is not a spill instruction, since no valid size was
                   // returned from either function.
@@ -1393,6 +1449,13 @@ bool InstrRefBasedLDV::transferSpillOrRestoreInst(MachineInstr &MI) {
 
   LLVM_DEBUG(dbgs() << "Examining instruction: "; MI.dump(););
 
+  // Strictly limit ourselves to plain loads and stores, not all instructions
+  // that can access the stack.
+  int FIDummy;
+  if (!TII->isStoreToStackSlotPostFE(MI, FIDummy) &&
+      !TII->isLoadFromStackSlotPostFE(MI, FIDummy))
+    return false;
+
   // First, if there are any DBG_VALUEs pointing at a spill slot that is
   // written to, terminate that variable location. The value in memory
   // will have changed. DbgEntityHistoryCalculator doesn't try to detect this.

llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h

@@ -984,6 +984,21 @@ class InstrRefBasedLDV : public LDVImpl {
   void dump_mloc_transfer(const MLocTransferMap &mloc_transfer) const;
 
   bool isCalleeSaved(LocIdx L) const;
+
+  bool hasFoldedStackStore(const MachineInstr &MI) {
+    // Instruction must have a memory operand that's a stack slot, and isn't
+    // aliased, meaning it's a spill from regalloc instead of a variable.
+    // If it's aliased, we can't guarantee its value.
+    if (!MI.hasOneMemOperand())
+      return false;
+    auto *MemOperand = *MI.memoperands_begin();
+    return MemOperand->isStore() &&
+           MemOperand->getPseudoValue() &&
+           MemOperand->getPseudoValue()->kind() == PseudoSourceValue::FixedStack
+           && !MemOperand->getPseudoValue()->isAliased(MFI);
+  }
+
+  Optional<LocIdx> findLocationForMemOperand(const MachineInstr &MI);
 };
 
 } // namespace LiveDebugValues

llvm/lib/CodeGen/MachineFunction.cpp

@@ -974,6 +974,9 @@ void MachineFunction::makeDebugValueSubstitution(DebugInstrOperandPair A,
                                                  unsigned Subreg) {
   // Catch any accidental self-loops.
   assert(A.first != B.first);
+  // Don't allow any substitutions _from_ the memory operand number.
+  assert(A.second != DebugOperandMemNumber);
+
   DebugValueSubstitutions.push_back({A, B, Subreg});
 }
 
@@ -1239,6 +1242,9 @@ bool MachineFunction::useDebugInstrRef() const {
   return false;
 }
 
+// Use one million as a high / reserved number.
+const unsigned MachineFunction::DebugOperandMemNumber = 1000000;
+
 /// \}
 
 //===----------------------------------------------------------------------===//

llvm/test/DebugInfo/MIR/InstrRef/memory-operand-folding-tieddef.mir

@@ -0,0 +1,175 @@
+# RUN: llc %s -o - -experimental-debug-variable-locations \
+# RUN:    -start-before=x86-flags-copy-lowering -stop-after=virtregrewriter \
+# RUN: | FileCheck %s
+#
+# This test is for stack spill folding -- the INC32r near the end of the MIR
+# below show be morphed into an INC32m by the register allocator, making it
+# load-operate-store to %stack.0. We should track this fact in the substitution
+# table, by adding a substitution to the memory-operand operand number.
+#
+# The INC32r is a tied-def instruction.
+#
+# NB: This test would more ideally start at phi-node-elimination, where
+# register allocation begins, however for some reason the INC32r gets
+# transformed into a different instruction if we do that. Start at the last
+# optimisation pass before regalloc instead.
+#
+# CHECK:      debugValueSubstitutions:
+# CHECK-NEXT:  - { srcinst: 1, srcop: 0, dstinst: 2, dstop: 1000000, subreg: 0 }
+# CHECK-LABEL: bb.5:
+# CHECK: INC32m %stack.0, {{.*}} debug-instr-number 2,
+--- |
+  ; ModuleID = 'reduced.ll'
+  source_filename = "reduced.ll"
+  target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+  %"class.llvm::APInt" = type { i32, %union.anon }
+  %union.anon = type { i64 }
+
+  define void @_ZNK4llvm5APInt5magicEv() local_unnamed_addr align 2 !dbg !7 {
+    ret void
+  }
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!3, !4, !5}
+  !llvm.ident = !{!6}
+
+  !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
+  !1 = !DIFile(filename: "test.c", directory: ".")
+  !2 = !{}
+  !3 = !{i32 7, !"Dwarf Version", i32 4}
+  !4 = !{i32 2, !"Debug Info Version", i32 3}
+  !5 = !{i32 1, !"wchar_size", i32 4}
+  !6 = !{!"clang"}
+  !7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 3, type: !8, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+  !8 = !DISubroutineType(types: !9)
+  !9 = !{!10, !11, !11}
+  !10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+  !11 = !DIBasicType(name: "short", size: 16, encoding: DW_ATE_signed)
+  !12 = !DILocalVariable(name: "bar", arg: 1, scope: !7, file: !1, line: 3, type: !11)
+  !13 = !DILocation(line: 0, scope: !7)
+  !14 = !DILocalVariable(name: "baz", arg: 2, scope: !7, file: !1, line: 3, type: !11)
+  !15 = distinct !DILexicalBlock(scope: !7, file: !1, line: 8, column: 7)
+
+...
+---
+name:            _ZNK4llvm5APInt5magicEv
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr64 }
+  - { id: 1, class: gr32 }
+  - { id: 2, class: gr64 }
+  - { id: 3, class: gr32 }
+  - { id: 4, class: gr64 }
+  - { id: 5, class: gr32 }
+  - { id: 6, class: gr32 }
+  - { id: 7, class: gr32 }
+  - { id: 8, class: gr64 }
+  - { id: 9, class: gr32 }
+  - { id: 10, class: gr64 }
+  - { id: 11, class: gr32 }
+  - { id: 12, class: gr64 }
+  - { id: 13, class: gr32 }
+  - { id: 14, class: gr64 }
+  - { id: 15, class: gr32 }
+  - { id: 16, class: gr64 }
+  - { id: 17, class: gr64 }
+  - { id: 18, class: gr64 }
+  - { id: 19, class: gr32 }
+  - { id: 20, class: gr8 }
+  - { id: 21, class: gr32 }
+  - { id: 22, class: gr64 }
+  - { id: 23, class: gr64 }
+  - { id: 24, class: gr64 }
+  - { id: 25, class: gr64 }
+  - { id: 26, class: gr64 }
+  - { id: 27, class: gr32 }
+  - { id: 28, class: gr8 }
+  - { id: 29, class: gr64 }
+  - { id: 30, class: gr64 }
+  - { id: 31, class: gr64 }
+  - { id: 32, class: gr64 }
+  - { id: 33, class: gr64 }
+  - { id: 34, class: gr64 }
+  - { id: 35, class: gr64 }
+  - { id: 36, class: gr64 }
+  - { id: 37, class: gr64 }
+  - { id: 38, class: gr8 }
+frameInfo:
+  maxAlignment:    1
+  hasCalls:        true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    %15:gr32 = IMPLICIT_DEF
+    %14:gr64 = IMPLICIT_DEF
+    %17:gr64 = IMPLICIT_DEF
+    %18:gr64 = IMPLICIT_DEF
+    %19:gr32 = MOV32r0 implicit-def dead $eflags
+    %22:gr64 = IMPLICIT_DEF
+    %23:gr64 = IMPLICIT_DEF
+    %24:gr64 = IMPLICIT_DEF
+    %25:gr64 = IMPLICIT_DEF
+    %26:gr64 = IMPLICIT_DEF
+    %30:gr64 = IMPLICIT_DEF
+    %31:gr64 = IMPLICIT_DEF
+    %32:gr64 = IMPLICIT_DEF
+    %33:gr64 = IMPLICIT_DEF
+    %34:gr64 = IMPLICIT_DEF
+    %35:gr64 = IMPLICIT_DEF
+    %36:gr64 = IMPLICIT_DEF
+    %37:gr64 = IMPLICIT_DEF
+    %21:gr32 = IMPLICIT_DEF
+  
+  bb.1:
+    %0:gr64 = PHI %14, %bb.0, %12, %bb.5
+    %1:gr32 = PHI %15, %bb.0, %11, %bb.5
+    %2:gr64 = PHI %14, %bb.0, %10, %bb.5
+    %3:gr32 = PHI %15, %bb.0, %9, %bb.5
+    %4:gr64 = PHI %14, %bb.0, %8, %bb.5
+    %5:gr32 = PHI %15, %bb.0, %7, %bb.5, debug-location !13
+    %6:gr32 = PHI %15, %bb.0, %13, %bb.5, debug-location !13
+    %16:gr64 = ADD64rr %4, %4, implicit-def dead $eflags, debug-location !13
+    MOV32mr %17, 1, $noreg, 0, $noreg, %5, debug-location !13 :: (store (s32) into `i32* undef`, align 8)
+    MOV64mr %18, 1, $noreg, 0, $noreg, killed %16, debug-location !13 :: (store (s64) into `i64* undef`)
+    %20:gr8 = COPY %19.sub_8bit
+    TEST8rr %20, %20, implicit-def $eflags, debug-location !13
+    JCC_1 %bb.3, 5, implicit $eflags, debug-location !13
+    JMP_1 %bb.2, debug-location !13
+  
+  bb.2:
+  
+  bb.3:
+    successors: %bb.4, %bb.5
+  
+    %7:gr32 = PHI %5, %bb.1, %21, %bb.2, debug-location !13
+    MOV32mr %22, 1, $noreg, 0, $noreg, %7, debug-location !13 :: (store (s32) into `i32* undef`, align 8)
+    %8:gr64 = MOV64rm %23, 1, $noreg, 0, $noreg, debug-location !13 :: (load (s64) from `i64* undef`)
+    MOV32mr %24, 1, $noreg, 0, $noreg, %3, debug-location !13 :: (store (s32) into `i32* undef`, align 8)
+    MOV64mi32 %25, 1, $noreg, 0, $noreg, 0, debug-location !13 :: (store (s64) into `i64* undef`)
+    %28:gr8 = COPY %19.sub_8bit
+    TEST8rr %28, %28, implicit-def $eflags, debug-location !13
+    JCC_1 %bb.5, 5, implicit $eflags, debug-location !13
+    JMP_1 %bb.4, debug-location !13
+  
+  bb.4:
+    %29:gr64 = ADD64rr %2, %2, implicit-def dead $eflags, debug-location !13
+    MOV64mr %30, 1, $noreg, 0, $noreg, killed %29, debug-location !13 :: (store (s64) into `i64* undef`)
+  
+  bb.5:
+    %9:gr32 = MOV32rm %26, 1, $noreg, 0, $noreg, debug-location !13 :: (load (s32) from `i32* undef`, align 8)
+    %10:gr64 = MOV64rm %31, 1, $noreg, 0, $noreg, debug-location !13 :: (load (s64) from `i64* undef`)
+    %12:gr64 = ADD64rr %0, %0, implicit-def dead $eflags, debug-location !13
+    MOV32mr %32, 1, $noreg, 0, $noreg, %1, debug-location !13 :: (store (s32) into `i32* undef`, align 8)
+    MOV64mr %33, 1, $noreg, 0, $noreg, %12, debug-location !13 :: (store (s64) into `i64* undef`)
+    %11:gr32 = MOV32rm %34, 1, $noreg, 0, $noreg, debug-location !13 :: (load (s32) from `i32* undef`, align 8)
+    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp, debug-location !13
+    $rdi = COPY %35, debug-location !13
+    $rsi = COPY %36, debug-location !13
+    CALL64r %37, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit-def $rsp, implicit-def $ssp, implicit-def $al, debug-location !13
+    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp, debug-location !13
+    %13:gr32 = INC32r %6, implicit-def dead $eflags, debug-instr-number 1, debug-location !13
+    DBG_INSTR_REF 1, 0, !12, !DIExpression(), debug-location !13
+    JMP_1 %bb.1, debug-location !13
+...