Revert "[X86] Respect code models more when determining if a global reference can fit in 32 bits"

[aeubanks]

Reverts #75386

Breaks build.

[llvmbot]

@llvm/pr-subscribers-backend-x86

Author: Arthur Eubanks (aeubanks)

Changes

Reverts llvm/llvm-project#75386

Breaks build.

---

Patch is 33.59 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/75500.diff

7 Files Affected:

• (modified) llvm/include/llvm/Target/TargetMachine.h (+1-1)
• (modified) llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp (+2-2)
• (modified) llvm/lib/Target/TargetMachine.cpp (+1-7)
• (modified) llvm/lib/Target/X86/X86FastISel.cpp (+4-2)
• (modified) llvm/lib/Target/X86/X86ISelDAGToDAG.cpp (+7-14)
• (modified) llvm/lib/Target/X86/X86Subtarget.cpp (+8-8)
• (modified) llvm/test/CodeGen/X86/code-model-elf.ll (+119-174)

diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h
index 4c29f25bedf419..4830ecbe1cd634 100644
--- a/llvm/include/llvm/Target/TargetMachine.h
+++ b/llvm/include/llvm/Target/TargetMachine.h
@@ -239,7 +239,7 @@ class TargetMachine {
   void setCodeModel(CodeModel::Model CM) { CMModel = CM; }
 
   void setLargeDataThreshold(uint64_t LDT) { LargeDataThreshold = LDT; }
-  bool isLargeGlobalValue(const GlobalValue *GV) const;
+  bool isLargeGlobalObject(const GlobalObject *GO) const;
 
   bool isPositionIndependent() const;
 
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index 18f4d546aa82d7..27d47aab4b6838 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -645,7 +645,7 @@ getELFSectionNameForGlobal(const GlobalObject *GO, SectionKind Kind,
     Name = ".rodata.cst";
     Name += utostr(EntrySize);
   } else {
-    Name = getSectionPrefixForGlobal(Kind, TM.isLargeGlobalValue(GO));
+    Name = getSectionPrefixForGlobal(Kind, TM.isLargeGlobalObject(GO));
   }
 
   bool HasPrefix = false;
@@ -765,7 +765,7 @@ getGlobalObjectInfo(const GlobalObject *GO, const TargetMachine &TM) {
     Group = C->getName();
     IsComdat = C->getSelectionKind() == Comdat::Any;
   }
-  if (TM.isLargeGlobalValue(GO))
+  if (TM.isLargeGlobalObject(GO))
     Flags |= ELF::SHF_X86_64_LARGE;
   return {Group, IsComdat, Flags};
 }
diff --git a/llvm/lib/Target/TargetMachine.cpp b/llvm/lib/Target/TargetMachine.cpp
index 3f96bd37755e58..5428e14eca5f8d 100644
--- a/llvm/lib/Target/TargetMachine.cpp
+++ b/llvm/lib/Target/TargetMachine.cpp
@@ -39,16 +39,10 @@ TargetMachine::TargetMachine(const Target &T, StringRef DataLayoutString,
 
 TargetMachine::~TargetMachine() = default;
 
-bool TargetMachine::isLargeGlobalValue(const GlobalValue *GVal) const {
+bool TargetMachine::isLargeGlobalObject(const GlobalObject *GO) const {
   if (getTargetTriple().getArch() != Triple::x86_64)
     return false;
 
-  auto *GO = GVal->getAliaseeObject();
-
-  // Be conservative if we can't find an underlying GlobalObject.
-  if (!GO)
-    return true;
-
   auto *GV = dyn_cast<GlobalVariable>(GO);
 
   // Functions/GlobalIFuncs are only large under the large code model.
diff --git a/llvm/lib/Target/X86/X86FastISel.cpp b/llvm/lib/Target/X86/X86FastISel.cpp
index fd14a8a7dc2a9e..425c52dbe7b104 100644
--- a/llvm/lib/Target/X86/X86FastISel.cpp
+++ b/llvm/lib/Target/X86/X86FastISel.cpp
@@ -716,8 +716,10 @@ bool X86FastISel::handleConstantAddresses(const Value *V, X86AddressMode &AM) {
       return false;
 
     // Can't handle large objects yet.
-    if (TM.isLargeGlobalValue(GV))
-      return false;
+    if (auto *GO = dyn_cast<GlobalObject>(GV)) {
+      if (TM.isLargeGlobalObject(GO))
+        return false;
+    }
 
     // Can't handle TLS yet.
     if (GV->isThreadLocal())
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index ce4fbfe88739df..545039b79f1616 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -2927,10 +2927,6 @@ bool X86DAGToDAGISel::selectAddr(SDNode *Parent, SDValue N, SDValue &Base,
 }
 
 bool X86DAGToDAGISel::selectMOV64Imm32(SDValue N, SDValue &Imm) {
-  // Cannot use 32 bit constants to reference objects in kernel code model.
-  if (TM.getCodeModel() == CodeModel::Kernel)
-    return false;
-
   // In static codegen with small code model, we can get the address of a label
   // into a register with 'movl'
   if (N->getOpcode() != X86ISD::Wrapper)
@@ -2944,18 +2940,15 @@ bool X86DAGToDAGISel::selectMOV64Imm32(SDValue N, SDValue &Imm) {
     return false;
 
   Imm = N;
-  // Small/medium code model can reference non-TargetGlobalAddress objects with
-  // 32 bit constants.
-  if (N->getOpcode() != ISD::TargetGlobalAddress) {
-    return TM.getCodeModel() == CodeModel::Small ||
-           TM.getCodeModel() == CodeModel::Medium;
-  }
+  if (N->getOpcode() != ISD::TargetGlobalAddress)
+    return TM.getCodeModel() == CodeModel::Small;
 
-  const GlobalValue *GV = cast<GlobalAddressSDNode>(N)->getGlobal();
-  if (std::optional<ConstantRange> CR = GV->getAbsoluteSymbolRange())
-    return CR->getUnsignedMax().ult(1ull << 32);
+  std::optional<ConstantRange> CR =
+      cast<GlobalAddressSDNode>(N)->getGlobal()->getAbsoluteSymbolRange();
+  if (!CR)
+    return TM.getCodeModel() == CodeModel::Small;
 
-  return !TM.isLargeGlobalValue(GV);
+  return CR->getUnsignedMax().ult(1ull << 32);
 }
 
 bool X86DAGToDAGISel::selectLEA64_32Addr(SDValue N, SDValue &Base,
diff --git a/llvm/lib/Target/X86/X86Subtarget.cpp b/llvm/lib/Target/X86/X86Subtarget.cpp
index d63f1ca1695b20..0f76daffd46339 100644
--- a/llvm/lib/Target/X86/X86Subtarget.cpp
+++ b/llvm/lib/Target/X86/X86Subtarget.cpp
@@ -86,16 +86,16 @@ X86Subtarget::classifyLocalReference(const GlobalValue *GV) const {
       CodeModel::Model CM = TM.getCodeModel();
       assert(CM != CodeModel::Tiny &&
              "Tiny codesize model not supported on X86");
-      // In the large code model, all text is far from any global data, so we
-      // use GOTOFF.
+      // In the large code model, even referencing a global under the large data
+      // threshold which is considered "small", we need to use GOTOFF.
       if (CM == CodeModel::Large)
         return X86II::MO_GOTOFF;
-      // Large GlobalValues use GOTOFF, otherwise use RIP-rel access.
-      if (GV)
-        return TM.isLargeGlobalValue(GV) ? X86II::MO_GOTOFF : X86II::MO_NO_FLAG;
-      // GV == nullptr is for all other non-GlobalValue global data like the
-      // constant pool, jump tables, labels, etc. The small and medium code
-      // models treat these as accessible with a RIP-rel access.
+      // Large objects use GOTOFF, otherwise use RIP-rel access.
+      if (auto *GO = dyn_cast_or_null<GlobalObject>(GV))
+        return TM.isLargeGlobalObject(GO) ? X86II::MO_GOTOFF
+                                          : X86II::MO_NO_FLAG;
+      // For non-GlobalObjects, the small and medium code models treat them as
+      // accessible with a RIP-rel access.
       return X86II::MO_NO_FLAG;
     }
 
diff --git a/llvm/test/CodeGen/X86/code-model-elf.ll b/llvm/test/CodeGen/X86/code-model-elf.ll
index a43736e2a5884a..0f58ccaf298dab 100644
--- a/llvm/test/CodeGen/X86/code-model-elf.ll
+++ b/llvm/test/CodeGen/X86/code-model-elf.ll
@@ -40,7 +40,6 @@ target triple = "x86_64--linux"
 
 @global_data = dso_local global [10 x i32] [i32 1, i32 2, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0], align 16
 @static_data = internal global [10 x i32] zeroinitializer, align 16
-@static_data_alias = internal constant ptr getelementptr inbounds ([10 x i32], ptr @static_data, i64 0, i64 2), align 8
 @extern_data = external global [10 x i32], align 16
 @thread_data = external thread_local global i32, align 4
 @unknown_size_data = dso_local global [0 x i32] zeroinitializer, align 16
@@ -97,67 +96,12 @@ define dso_local ptr @lea_static_data() #0 {
 ; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L0$pb(%rip), %rax
 ; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L0$pb, %rcx
 ; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
-; LARGE-SMALL-DATA-PIC-NEXT:    movl $static_data@GOTOFF, %eax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $static_data@GOTOFF, %rax
 ; LARGE-SMALL-DATA-PIC-NEXT:    addq %rcx, %rax
 ; LARGE-SMALL-DATA-PIC-NEXT:    retq
   ret ptr @static_data
 }
 
-define dso_local ptr @lea_static_data_alias() #0 {
-; SMALL-STATIC-LABEL: lea_static_data_alias:
-; SMALL-STATIC:       # %bb.0:
-; SMALL-STATIC-NEXT:    movl $static_data_alias, %eax
-; SMALL-STATIC-NEXT:    retq
-;
-; MEDIUM-STATIC-LABEL: lea_static_data_alias:
-; MEDIUM-STATIC:       # %bb.0:
-; MEDIUM-STATIC-NEXT:    movabsq $static_data_alias, %rax
-; MEDIUM-STATIC-NEXT:    retq
-;
-; LARGE-STATIC-LABEL: lea_static_data_alias:
-; LARGE-STATIC:       # %bb.0:
-; LARGE-STATIC-NEXT:    movabsq $static_data_alias, %rax
-; LARGE-STATIC-NEXT:    retq
-;
-; SMALL-PIC-LABEL: lea_static_data_alias:
-; SMALL-PIC:       # %bb.0:
-; SMALL-PIC-NEXT:    leaq static_data_alias(%rip), %rax
-; SMALL-PIC-NEXT:    retq
-;
-; MEDIUM-SMALL-DATA-PIC-LABEL: lea_static_data_alias:
-; MEDIUM-SMALL-DATA-PIC:       # %bb.0:
-; MEDIUM-SMALL-DATA-PIC-NEXT:    leaq static_data_alias(%rip), %rax
-; MEDIUM-SMALL-DATA-PIC-NEXT:    retq
-;
-; MEDIUM-PIC-LABEL: lea_static_data_alias:
-; MEDIUM-PIC:       # %bb.0:
-; MEDIUM-PIC-NEXT:    leaq _GLOBAL_OFFSET_TABLE_(%rip), %rcx
-; MEDIUM-PIC-NEXT:    movabsq $static_data_alias@GOTOFF, %rax
-; MEDIUM-PIC-NEXT:    addq %rcx, %rax
-; MEDIUM-PIC-NEXT:    retq
-;
-; LARGE-PIC-LABEL: lea_static_data_alias:
-; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L1$pb:
-; LARGE-PIC-NEXT:    leaq .L1$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L1$pb, %rcx
-; LARGE-PIC-NEXT:    addq %rax, %rcx
-; LARGE-PIC-NEXT:    movabsq $static_data_alias@GOTOFF, %rax
-; LARGE-PIC-NEXT:    addq %rcx, %rax
-; LARGE-PIC-NEXT:    retq
-;
-; LARGE-SMALL-DATA-PIC-LABEL: lea_static_data_alias:
-; LARGE-SMALL-DATA-PIC:       # %bb.0:
-; LARGE-SMALL-DATA-PIC-NEXT:  .L1$pb:
-; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L1$pb(%rip), %rax
-; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L1$pb, %rcx
-; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
-; LARGE-SMALL-DATA-PIC-NEXT:    movl $static_data_alias@GOTOFF, %eax
-; LARGE-SMALL-DATA-PIC-NEXT:    addq %rcx, %rax
-; LARGE-SMALL-DATA-PIC-NEXT:    retq
-  ret ptr @static_data_alias
-}
-
 define dso_local ptr @lea_global_data() #0 {
 ; SMALL-STATIC-LABEL: lea_global_data:
 ; SMALL-STATIC:       # %bb.0:
@@ -193,9 +137,9 @@ define dso_local ptr @lea_global_data() #0 {
 ;
 ; LARGE-PIC-LABEL: lea_global_data:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L2$pb:
-; LARGE-PIC-NEXT:    leaq .L2$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L2$pb, %rcx
+; LARGE-PIC-NEXT:  .L1$pb:
+; LARGE-PIC-NEXT:    leaq .L1$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L1$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $global_data@GOTOFF, %rax
 ; LARGE-PIC-NEXT:    addq %rcx, %rax
@@ -203,11 +147,11 @@ define dso_local ptr @lea_global_data() #0 {
 ;
 ; LARGE-SMALL-DATA-PIC-LABEL: lea_global_data:
 ; LARGE-SMALL-DATA-PIC:       # %bb.0:
-; LARGE-SMALL-DATA-PIC-NEXT:  .L2$pb:
-; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L2$pb(%rip), %rax
-; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L2$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:  .L1$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L1$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L1$pb, %rcx
 ; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
-; LARGE-SMALL-DATA-PIC-NEXT:    movl $global_data@GOTOFF, %eax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $global_data@GOTOFF, %rax
 ; LARGE-SMALL-DATA-PIC-NEXT:    addq %rcx, %rax
 ; LARGE-SMALL-DATA-PIC-NEXT:    retq
   ret ptr @global_data
@@ -246,9 +190,9 @@ define dso_local ptr @lea_extern_data() #0 {
 ;
 ; LARGE-PIC-LABEL: lea_extern_data:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L3$pb:
-; LARGE-PIC-NEXT:    leaq .L3$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L3$pb, %rcx
+; LARGE-PIC-NEXT:  .L2$pb:
+; LARGE-PIC-NEXT:    leaq .L2$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L2$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $extern_data@GOT, %rax
 ; LARGE-PIC-NEXT:    movq (%rcx,%rax), %rax
@@ -256,11 +200,11 @@ define dso_local ptr @lea_extern_data() #0 {
 ;
 ; LARGE-SMALL-DATA-PIC-LABEL: lea_extern_data:
 ; LARGE-SMALL-DATA-PIC:       # %bb.0:
-; LARGE-SMALL-DATA-PIC-NEXT:  .L3$pb:
-; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L3$pb(%rip), %rax
-; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L3$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:  .L2$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L2$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L2$pb, %rcx
 ; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
-; LARGE-SMALL-DATA-PIC-NEXT:    movl $extern_data@GOT, %eax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $extern_data@GOT, %rax
 ; LARGE-SMALL-DATA-PIC-NEXT:    movq (%rcx,%rax), %rax
 ; LARGE-SMALL-DATA-PIC-NEXT:    retq
   ret ptr @extern_data
@@ -303,9 +247,9 @@ define dso_local ptr @lea_unknown_size_data() #0 {
 ;
 ; LARGE-PIC-LABEL: lea_unknown_size_data:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L4$pb:
-; LARGE-PIC-NEXT:    leaq .L4$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L4$pb, %rcx
+; LARGE-PIC-NEXT:  .L3$pb:
+; LARGE-PIC-NEXT:    leaq .L3$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L3$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $unknown_size_data@GOTOFF, %rax
 ; LARGE-PIC-NEXT:    addq %rcx, %rax
@@ -313,9 +257,9 @@ define dso_local ptr @lea_unknown_size_data() #0 {
 ;
 ; LARGE-SMALL-DATA-PIC-LABEL: lea_unknown_size_data:
 ; LARGE-SMALL-DATA-PIC:       # %bb.0:
-; LARGE-SMALL-DATA-PIC-NEXT:  .L4$pb:
-; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L4$pb(%rip), %rax
-; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L4$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:  .L3$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L3$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L3$pb, %rcx
 ; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $unknown_size_data@GOTOFF, %rax
 ; LARGE-SMALL-DATA-PIC-NEXT:    addq %rcx, %rax
@@ -323,6 +267,7 @@ define dso_local ptr @lea_unknown_size_data() #0 {
   ret ptr @unknown_size_data
 }
 
+; TODO: make nopic small and medium instruction sequence the same
 define dso_local ptr @lea_forced_small_data() #0 {
 ; SMALL-STATIC-LABEL: lea_forced_small_data:
 ; SMALL-STATIC:       # %bb.0:
@@ -331,12 +276,12 @@ define dso_local ptr @lea_forced_small_data() #0 {
 ;
 ; MEDIUM-STATIC-LABEL: lea_forced_small_data:
 ; MEDIUM-STATIC:       # %bb.0:
-; MEDIUM-STATIC-NEXT:    movl $forced_small_data, %eax
+; MEDIUM-STATIC-NEXT:    movabsq $forced_small_data, %rax
 ; MEDIUM-STATIC-NEXT:    retq
 ;
 ; LARGE-STATIC-LABEL: lea_forced_small_data:
 ; LARGE-STATIC:       # %bb.0:
-; LARGE-STATIC-NEXT:    movl $forced_small_data, %eax
+; LARGE-STATIC-NEXT:    movabsq $forced_small_data, %rax
 ; LARGE-STATIC-NEXT:    retq
 ;
 ; SMALL-PIC-LABEL: lea_forced_small_data:
@@ -356,21 +301,21 @@ define dso_local ptr @lea_forced_small_data() #0 {
 ;
 ; LARGE-PIC-LABEL: lea_forced_small_data:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L5$pb:
-; LARGE-PIC-NEXT:    leaq .L5$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L5$pb, %rcx
+; LARGE-PIC-NEXT:  .L4$pb:
+; LARGE-PIC-NEXT:    leaq .L4$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L4$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
-; LARGE-PIC-NEXT:    movl $forced_small_data@GOTOFF, %eax
+; LARGE-PIC-NEXT:    movabsq $forced_small_data@GOTOFF, %rax
 ; LARGE-PIC-NEXT:    addq %rcx, %rax
 ; LARGE-PIC-NEXT:    retq
 ;
 ; LARGE-SMALL-DATA-PIC-LABEL: lea_forced_small_data:
 ; LARGE-SMALL-DATA-PIC:       # %bb.0:
-; LARGE-SMALL-DATA-PIC-NEXT:  .L5$pb:
-; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L5$pb(%rip), %rax
-; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L5$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:  .L4$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L4$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L4$pb, %rcx
 ; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
-; LARGE-SMALL-DATA-PIC-NEXT:    movl $forced_small_data@GOTOFF, %eax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $forced_small_data@GOTOFF, %rax
 ; LARGE-SMALL-DATA-PIC-NEXT:    addq %rcx, %rax
 ; LARGE-SMALL-DATA-PIC-NEXT:    retq
   ret ptr @forced_small_data
@@ -385,13 +330,13 @@ define dso_local i32 @load_forced_small_data() #0 {
 ;
 ; MEDIUM-STATIC-LABEL: load_forced_small_data:
 ; MEDIUM-STATIC:       # %bb.0:
-; MEDIUM-STATIC-NEXT:    movl $forced_small_data, %eax
+; MEDIUM-STATIC-NEXT:    movabsq $forced_small_data, %rax
 ; MEDIUM-STATIC-NEXT:    movl 8(%rax), %eax
 ; MEDIUM-STATIC-NEXT:    retq
 ;
 ; LARGE-STATIC-LABEL: load_forced_small_data:
 ; LARGE-STATIC:       # %bb.0:
-; LARGE-STATIC-NEXT:    movl $forced_small_data, %eax
+; LARGE-STATIC-NEXT:    movabsq $forced_small_data, %rax
 ; LARGE-STATIC-NEXT:    movl 8(%rax), %eax
 ; LARGE-STATIC-NEXT:    retq
 ;
@@ -414,21 +359,21 @@ define dso_local i32 @load_forced_small_data() #0 {
 ;
 ; LARGE-PIC-LABEL: load_forced_small_data:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L6$pb:
-; LARGE-PIC-NEXT:    leaq .L6$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L6$pb, %rcx
+; LARGE-PIC-NEXT:  .L5$pb:
+; LARGE-PIC-NEXT:    leaq .L5$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L5$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
-; LARGE-PIC-NEXT:    movl $forced_small_data@GOTOFF, %eax
+; LARGE-PIC-NEXT:    movabsq $forced_small_data@GOTOFF, %rax
 ; LARGE-PIC-NEXT:    movl 8(%rcx,%rax), %eax
 ; LARGE-PIC-NEXT:    retq
 ;
 ; LARGE-SMALL-DATA-PIC-LABEL: load_forced_small_data:
 ; LARGE-SMALL-DATA-PIC:       # %bb.0:
-; LARGE-SMALL-DATA-PIC-NEXT:  .L6$pb:
-; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L6$pb(%rip), %rax
-; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L6$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:  .L5$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L5$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L5$pb, %rcx
 ; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
-; LARGE-SMALL-DATA-PIC-NEXT:    movl $forced_small_data@GOTOFF, %eax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $forced_small_data@GOTOFF, %rax
 ; LARGE-SMALL-DATA-PIC-NEXT:    movl 8(%rcx,%rax), %eax
 ; LARGE-SMALL-DATA-PIC-NEXT:    retq
   %rv = load i32, ptr getelementptr inbounds (i32, ptr @forced_small_data, i64 2)
@@ -439,7 +384,7 @@ define dso_local i32 @load_forced_small_data() #0 {
 define dso_local ptr @lea_forced_large_data() #0 {
 ; SMALL-STATIC-LABEL: lea_forced_large_data:
 ; SMALL-STATIC:       # %bb.0:
-; SMALL-STATIC-NEXT:    movabsq $forced_large_data, %rax
+; SMALL-STATIC-NEXT:    movl $forced_large_data, %eax
 ; SMALL-STATIC-NEXT:    retq
 ;
 ; MEDIUM-STATIC-LABEL: lea_forced_large_data:
@@ -474,9 +419,9 @@ define dso_local ptr @lea_forced_large_data() #0 {
 ;
 ; LARGE-PIC-LABEL: lea_forced_large_data:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L7$pb:
-; LARGE-PIC-NEXT:    leaq .L7$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L7$pb, %rcx
+; LARGE-PIC-NEXT:  .L6$pb:
+; LARGE-PIC-NEXT:    leaq .L6$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L6$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $forced_large_data@GOTOFF, %rax
 ; LARGE-PIC-NEXT:    addq %rcx, %rax
@@ -484,9 +429,9 @@ define dso_local ptr @lea_forced_large_data() #0 {
 ;
 ; LARGE-SMALL-DATA-PIC-LABEL: lea_forced_large_data:
 ; LARGE-SMALL-DATA-PIC:       # %bb.0:
-; LARGE-SMALL-DATA-PIC-NEXT:  .L7$pb:
-; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L7$pb(%rip), %rax
-; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L7$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:  .L6$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L6$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L6$pb, %rcx
 ; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $forced_large_data@GOTOFF, %rax
 ; LARGE-SMALL-DATA-PIC-NEXT:    addq %rcx, %rax
@@ -534,9 +479,9 @@ define dso_local i32 @load_forced_large_data() #0 {
 ;
 ; LARGE-PIC-LABEL: load_forced_large_data:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L8$pb:
-; LARGE-PIC-NEXT:    leaq .L8$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L8$pb, %rcx
+; LARGE-PIC-NEXT:  .L7$pb:
+; LARGE-PIC-NEXT:    leaq .L7$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L7$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $forced_large_data@GOTOFF, %rax
 ; LARGE-PIC-NEXT:    movl 8(%rcx,%rax), %eax
@@ -544,9 +489,9 @@ define dso_local i32 @load_forced_large_data() #0 {
 ;
 ; LARGE-SMALL-DATA-PIC-LABEL: load_forced_large_data:
 ; LARGE-SMALL-DATA-PIC:       # %bb.0:
-; LARGE-SMALL-DATA-PIC-NEXT:  .L8$pb:
-; ...
[truncated]