[AArch64] Move machine bundle unpacking to PreEmit2 phase.

This patch adjusts the placement of the bundle unpacking to just before
code emission. In particular, this means bundle unpacking happens AFTER
the machine outliner. With the previous position, the machine outliner
may outline parts of a bundle, which breaks them up.

This is an issue for BLR_RVMARKER handling, as illustrated by the
rvmarker-pseudo-expansion-and-outlining.mir test case. The machine
outliner should not break up the bundles created during pseudo
expansion.

This should fix PR49082.

Reviewed By: SjoerdMeijer

Differential Revision: https://reviews.llvm.org/D96294

llvm/lib/Target/AArch64/AArch64TargetMachine.cpp

@@ -431,6 +431,7 @@ class AArch64PassConfig : public TargetPassConfig {
   void addPostRegAlloc() override;
   void addPreSched2() override;
   void addPreEmitPass() override;
+  void addPreEmitPass2() override;
 
   std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
 };
@@ -695,8 +696,11 @@ void AArch64PassConfig::addPreEmitPass() {
   if (TM->getOptLevel() != CodeGenOpt::None && EnableCollectLOH &&
       TM->getTargetTriple().isOSBinFormatMachO())
     addPass(createAArch64CollectLOHPass());
+}
 
-  // SVE bundles move prefixes with destructive operations.
+void AArch64PassConfig::addPreEmitPass2() {
+  // SVE bundles move prefixes with destructive operations. BLR_RVMARKER pseudo
+  // instructions are lowered to bundles as well.
   addPass(createUnpackMachineBundles(nullptr));
 }
 

llvm/test/CodeGen/AArch64/O0-pipeline.ll

@@ -64,10 +64,10 @@
 ; CHECK-NEXT:       Implement the 'patchable-function' attribute
 ; CHECK-NEXT:       AArch64 Branch Targets
 ; CHECK-NEXT:       Branch relaxation pass
-; CHECK-NEXT:       Unpack machine instruction bundles
 ; CHECK-NEXT:       Contiguously Lay Out Funclets
 ; CHECK-NEXT:       StackMap Liveness Analysis
 ; CHECK-NEXT:       Live DEBUG_VALUE analysis
+; CHECK-NEXT:       Unpack machine instruction bundles
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
 ; CHECK-NEXT:       AArch64 Assembly Printer

llvm/test/CodeGen/AArch64/O3-pipeline.ll

@@ -196,12 +196,12 @@
 ; CHECK-NEXT:       AArch64 Branch Targets
 ; CHECK-NEXT:       Branch relaxation pass
 ; CHECK-NEXT:       AArch64 Compress Jump Tables
-; CHECK-NEXT:       Unpack machine instruction bundles
 ; CHECK-NEXT:       Contiguously Lay Out Funclets
 ; CHECK-NEXT:       StackMap Liveness Analysis
 ; CHECK-NEXT:       Live DEBUG_VALUE analysis
 ; CHECK-NEXT:     Machine Outliner
 ; CHECK-NEXT:     FunctionPass Manager
+; CHECK-NEXT:       Unpack machine instruction bundles
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
 ; CHECK-NEXT:       AArch64 Assembly Printer

llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll

@@ -34,6 +34,10 @@
 ; HOTNESS-NEXT:  Executing Pass 'Function Pass Manager'
 ; HOTNESS-NEXT: Executing Pass 'Verify generated machine code'
 ; HOTNESS-NEXT: Freeing Pass 'Verify generated machine code'
+; HOTNESS-NEXT: Executing Pass 'Unpack machine instruction bundles'
+; HOTNESS-NEXT: Freeing Pass 'Unpack machine instruction bundles'
+; HOTNESS-NEXT: Executing Pass 'Verify generated machine code'
+; HOTNESS-NEXT: Freeing Pass 'Verify generated machine code'
 ; HOTNESS-NEXT: Executing Pass 'Lazy Machine Block Frequency Analysis'
 ; HOTNESS-NEXT: Executing Pass 'Machine Optimization Remark Emitter'
 ; HOTNESS-NEXT: Building MachineBlockFrequencyInfo on the fly
@@ -51,6 +55,10 @@
 ; NO_HOTNESS-NEXT:  Executing Pass 'Function Pass Manager'
 ; NO_HOTNESS-NEXT: Executing Pass 'Verify generated machine code'
 ; NO_HOTNESS-NEXT: Freeing Pass 'Verify generated machine code'
+; NO_HOTNESS-NEXT: Executing Pass 'Unpack machine instruction bundles'
+; NO_HOTNESS-NEXT: Freeing Pass 'Unpack machine instruction bundles'
+; NO_HOTNESS-NEXT: Executing Pass 'Verify generated machine code'
+; NO_HOTNESS-NEXT: Freeing Pass 'Verify generated machine code'
 ; NO_HOTNESS-NEXT: Executing Pass 'Lazy Machine Block Frequency Analysis'
 ; NO_HOTNESS-NEXT: Executing Pass 'Machine Optimization Remark Emitter'
 ; NO_HOTNESS-NEXT: Executing Pass 'AArch64 Assembly Printer'

llvm/test/CodeGen/AArch64/rvmarker-pseudo-expansion-and-outlining.mir

@@ -1,21 +1,22 @@
 # RUN: llc -enable-machine-outliner -start-before=aarch64-expand-pseudo -mtriple=arm64-apple-ios -o - %s | FileCheck %s
 #
-# FIXME: The calls to _cb1 & _cb2 should be followed by the marker instruction
-#        'mov x29, x29'. The marker should not get outlined.
+#  The calls to _cb1 & _cb2 should be followed by the marker instruction
+#  'mov x29, x29'. The marker should not get outlined.
 #
 # CHECK-LABEL: _fn1:
 # CHECK:       bb.0:
 # CHECK:        bl _cb1
+# CHECK-NEXT:   mov x29, x29
 # CHECK-NEXT:   bl _OUTLINED_FUNCTION_0
 #
 # CHECK-LABEL: _fn2:
 # CHECK:       bb.0:
 # CHECK:        bl _cb2
+# CHECK-NEXT:   mov x29, x29
 # CHECK-NEXT:   bl _OUTLINED_FUNCTION_0
 #
 # CHECK-LABEL: _OUTLINED_FUNCTION_0:
 # CHECK:       bb.0:
-# CHECK-NEXT:    mov x29, x29
 # CHECK-NEXT:    orr w12, wzr, #0x3
 # CHECK-NEXT:    orr w12, wzr, #0x3
 # CHECK-NEXT:    orr w12, wzr, #0x3