diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index a3513ca439ac2..f19dcb5be1722 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -71,27 +71,6 @@ using namespace llvm; namespace { -enum class IFuncLowering { - SymbolResolverIfSupported, - SymbolResolverAlways, - SymbolResolverNever -}; - -static cl::opt PreferredIFuncLowering( - "arm64-darwin-ifunc-symbol_resolver", - cl::init(IFuncLowering::SymbolResolverNever), - cl::desc("Pick the lowering for ifuncs on darwin platforms"), cl::Hidden, - cl::values( - clEnumValN( - IFuncLowering::SymbolResolverIfSupported, "if_supported", - "Use .symbol_resolver's when known to be supported by the linker."), - clEnumValN(IFuncLowering::SymbolResolverAlways, "always", - "Always use .symbol_resolvers. NOTE: this might not be " - "supported by the linker in all cases."), - clEnumValN(IFuncLowering::SymbolResolverNever, "never", - "Use a manual lowering, doing what the linker would have " - "done, but in the compiler."))); - class AArch64AsmPrinter : public AsmPrinter { AArch64MCInstLower MCInstLowering; FaultMaps FM; @@ -224,9 +203,6 @@ class AArch64AsmPrinter : public AsmPrinter { } void emitGlobalIFunc(Module &M, const GlobalIFunc &GI) override; - - void emitLinkerSymbolResolver(Module &M, const GlobalIFunc &GI); - void emitManualSymbolResolver(Module &M, const GlobalIFunc &GI); }; } // end anonymous namespace @@ -1838,51 +1814,26 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { EmitToStreamer(*OutStreamer, TmpInst); } -void AArch64AsmPrinter::emitLinkerSymbolResolver(Module &M, - const GlobalIFunc &GI) { - OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection()); - - MCSymbol *Name = getSymbol(&GI); - - // NOTE: non-global .symbol_resolvers are not yet supported by Darwin linkers - - if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective()) - OutStreamer->emitSymbolAttribute(Name, MCSA_Global); - else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage()) - OutStreamer->emitSymbolAttribute(Name, MCSA_WeakReference); - else - assert(GI.hasLocalLinkage() && "Invalid ifunc linkage"); - - OutStreamer->emitCodeAlignment(Align(4), STI); - OutStreamer->emitLabel(Name); - OutStreamer->emitSymbolAttribute(Name, MCSA_SymbolResolver); - emitVisibility(Name, GI.getVisibility()); +void AArch64AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) { + if (!TM.getTargetTriple().isOSBinFormatMachO()) + return AsmPrinter::emitGlobalIFunc(M, GI); - // ld-prime does not seem to support aliases of symbol resolvers, so we have - // to tail call the resolver manually. - OutStreamer->emitInstruction( - MCInstBuilder(AArch64::B) - .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))), - *STI); -} + // On Darwin platforms, emit a manually-constructed .symbol_resolver that + // implements the symbol resolution duties of the IFunc. + // + // Normally, this would be handled by linker magic, but unfortunately there are + // a few limitations in ld64 and ld-prime's implementation of .symbol_resolver + // that mean we can't always use them: + // + // * resolvers cannot be the target of an alias + // * resolvers cannot have private linkage + // * resolvers cannot have linkonce linkage + // * resolvers cannot appear in executables + // * resolvers cannot appear in bundles + // + // This works around that by emitting a close approximation of what the linker + // would have done. -/// \brief Emit a manually-constructed .symbol_resolver that implements the -/// symbol resolution duties of the IFunc. -/// -/// Normally, this would be handled by linker magic, but unfortunately there are -/// a few limitations in ld64 and ld-prime's implementation of .symbol_resolver -/// that mean we can't always use them: -/// -/// * resolvers cannot be the target of an alias -/// * resolvers cannot have private linkage -/// * resolvers cannot have linkonce linkage -/// * resolvers cannot appear in executables -/// * resolvers cannot appear in bundles -/// -/// This works around that by emitting a close approximation of what the linker -/// would have done. -void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, - const GlobalIFunc &GI) { auto EmitLinkage = [&](MCSymbol *Sym) { if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective()) OutStreamer->emitSymbolAttribute(Sym, MCSA_Global); @@ -1964,33 +1915,35 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, .addReg(AArch64::X16), *STI); + // These stub helpers are only ever called once, so here we're optimizing for + // minimum size by using the pre-indexed store variants, which saves a few + // bytes of instructions to bump & restore sp. + // _ifunc.stub_helper: - // stp fp, lr, [sp, #-16] - // sub fp, sp, 16 - // stp x1, x0, [sp, #-32] - // stp x3, x2, [sp, #-48] - // stp x5, x4, [sp, #-64] - // stp x7, x6, [sp, #-80] - // stp d1, d0, [sp, #-96] - // stp d3, d2, [sp, #-112] - // stp d5, d4, [sp, #-128] - // stp d7, d6, [sp, #-144] - // sub sp, sp, 144 + // stp fp, lr, [sp, #-16]! + // mov fp, sp + // stp x1, x0, [sp, #-16]! + // stp x3, x2, [sp, #-16]! + // stp x5, x4, [sp, #-16]! + // stp x7, x6, [sp, #-16]! + // stp d1, d0, [sp, #-16]! + // stp d3, d2, [sp, #-16]! + // stp d5, d4, [sp, #-16]! + // stp d7, d6, [sp, #-16]! // bl _resolver // adrp x16, lazy_pointer@GOTPAGE // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] // str x0, [x16] // mov x16, x0 - // add sp, sp, 144 - // ldp d7, d6, [sp, #-144] - // ldp d5, d4, [sp, #-128] - // ldp d3, d2, [sp, #-112] - // ldp d1, d0, [sp, #-96] - // ldp x7, x6, [sp, #-80] - // ldp x5, x4, [sp, #-64] - // ldp x3, x2, [sp, #-48] - // ldp x1, x0, [sp, #-32] - // ldp fp, lr, [sp, #-16] + // ldp d7, d6, [sp], #16 + // ldp d5, d4, [sp], #16 + // ldp d3, d2, [sp], #16 + // ldp d1, d0, [sp], #16 + // ldp x7, x6, [sp], #16 + // ldp x5, x4, [sp], #16 + // ldp x3, x2, [sp], #16 + // ldp x1, x0, [sp], #16 + // ldp fp, lr, [sp], #16 // br x16 EmitLinkage(StubHelper); @@ -1998,43 +1951,39 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, OutStreamer->emitLabel(StubHelper); emitVisibility(StubHelper, GI.getVisibility()); - OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) + .addReg(AArch64::SP) .addReg(AArch64::FP) .addReg(AArch64::LR) .addReg(AArch64::SP) .addImm(-2), *STI); - OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBXri) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) .addReg(AArch64::FP) .addReg(AArch64::SP) - .addImm(16) + .addImm(0) .addImm(0), *STI); - for (int I = 0; I != 8; I += 2) - OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi) - .addReg(AArch64::X1 + I) - .addReg(AArch64::X0 + I) + for (int I = 0; I != 4; ++I) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) .addReg(AArch64::SP) - .addImm(-4 - I), + .addReg(AArch64::X1 + 2 * I) + .addReg(AArch64::X0 + 2 * I) + .addReg(AArch64::SP) + .addImm(-2), *STI); - for (int I = 0; I != 8; I += 2) - OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPDi) - .addReg(AArch64::D1 + I) - .addReg(AArch64::D0 + I) + for (int I = 0; I != 4; ++I) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPDpre) + .addReg(AArch64::SP) + .addReg(AArch64::D1 + 2 * I) + .addReg(AArch64::D0 + 2 * I) .addReg(AArch64::SP) - .addImm(-12 - I), + .addImm(-2), *STI); - OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBXri) - .addReg(AArch64::SP) - .addReg(AArch64::SP) - .addImm(144) - .addImm(0), - *STI); - OutStreamer->emitInstruction( MCInstBuilder(AArch64::BL) .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))), @@ -2081,34 +2030,30 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, .addImm(0), *STI); - OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) - .addReg(AArch64::SP) - .addReg(AArch64::SP) - .addImm(144) - .addImm(0), - *STI); - - for (int I = 6; I != -2; I -= 2) - OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPDi) - .addReg(AArch64::D1 + I) - .addReg(AArch64::D0 + I) + for (int I = 3; I != -1; --I) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPDpost) + .addReg(AArch64::SP) + .addReg(AArch64::D1 + 2 * I) + .addReg(AArch64::D0 + 2 * I) .addReg(AArch64::SP) - .addImm(-12 - I), + .addImm(2), *STI); - for (int I = 6; I != -2; I -= 2) - OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXi) - .addReg(AArch64::X1 + I) - .addReg(AArch64::X0 + I) + for (int I = 3; I != -1; --I) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost) .addReg(AArch64::SP) - .addImm(-4 - I), + .addReg(AArch64::X1 + 2 * I) + .addReg(AArch64::X0 + 2 * I) + .addReg(AArch64::SP) + .addImm(2), *STI); - OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXi) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost) + .addReg(AArch64::SP) .addReg(AArch64::FP) .addReg(AArch64::LR) .addReg(AArch64::SP) - .addImm(-2), + .addImm(2), *STI); OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e() @@ -2118,27 +2063,6 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, *STI); } -void AArch64AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) { - if (!TM.getTargetTriple().isOSBinFormatMachO()) - return AsmPrinter::emitGlobalIFunc(M, GI); - - switch (PreferredIFuncLowering) { - case IFuncLowering::SymbolResolverAlways: - return emitLinkerSymbolResolver(M, GI); - case IFuncLowering::SymbolResolverNever: - return emitManualSymbolResolver(M, GI); - case IFuncLowering::SymbolResolverIfSupported: - if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective()) - return emitLinkerSymbolResolver(M, GI); - else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage()) - // NOTE: non-global .symbol_resolvers are not yet supported by Darwin - // linkers - return emitManualSymbolResolver(M, GI); - else - assert(GI.hasLocalLinkage() && "Invalid ifunc linkage"); - } -} - // Force static initialization. extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() { RegisterAsmPrinter X(getTheAArch64leTarget()); diff --git a/llvm/test/CodeGen/AArch64/ifunc-asm.ll b/llvm/test/CodeGen/AArch64/ifunc-asm.ll index 7eff692da83ff..18c57b577333e 100644 --- a/llvm/test/CodeGen/AArch64/ifunc-asm.ll +++ b/llvm/test/CodeGen/AArch64/ifunc-asm.ll @@ -1,86 +1,72 @@ ; RUN: llc -mtriple=arm64-unknown-linux-gnu %s -filetype=asm -o - | FileCheck %s --check-prefixes=ELF -; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -arm64-darwin-ifunc-symbol_resolver=always | FileCheck %s --check-prefixes=MACHO,MACHO-LINKER -; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -arm64-darwin-ifunc-symbol_resolver=if_supported | FileCheck %s --check-prefixes=MACHO,MACHO-DEFAULT -; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -arm64-darwin-ifunc-symbol_resolver=never | FileCheck %s --check-prefixes=MACHO,MACHO-MANUAL -; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - | FileCheck %s --check-prefixes=MACHO,MACHO-MANUAL -; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -global-isel | FileCheck %s --check-prefixes=MACHO,MACHO-MANUAL +; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - | FileCheck %s --check-prefix=MACHO +; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -global-isel | FileCheck %s --check-prefix=MACHO define internal ptr @the_resolver() { entry: ret ptr null } -; ELF: .type the_resolver,@function -; ELF-NEXT: the_resolver: +; ELF: .type the_resolver,@function +; ELF-NEXT: the_resolver: -; MACHO: .p2align 2 -; MACHO-NEXT: _the_resolver +; MACHO: .p2align 2 +; MACHO-NEXT: _the_resolver: @global_ifunc = ifunc i32 (i32), ptr @the_resolver -; ELF: .globl global_ifunc -; ELF-NEXT: .type global_ifunc,@gnu_indirect_function -; ELF-NEXT: .set global_ifunc, the_resolver +; ELF: .globl global_ifunc +; ELF-NEXT: .type global_ifunc,@gnu_indirect_function +; ELF-NEXT: .set global_ifunc, the_resolver -; MACHO-LINKER: .globl _global_ifunc -; MACHO-LINKER-NEXT: .p2align 2 -; MACHO-LINKER-NEXT: _global_ifunc: -; MACHO-LINKER-NEXT: .symbol_resolver _global_ifunc -; MACHO-LINKER-NEXT: b _the_resolver +; MACHO: .section __DATA,__data +; MACHO-NEXT: .globl _global_ifunc.lazy_pointer +; MACHO-NEXT: .p2align 3, 0x0 +; MACHO-NEXT: _global_ifunc.lazy_pointer: +; MACHO-NEXT: .quad _global_ifunc.stub_helper -; MACHO-DEFAULT: .globl _global_ifunc -; MACHO-DEFAULT-NEXT: .p2align 2 -; MACHO-DEFAULT-NEXT: _global_ifunc: -; MACHO-DEFAULT-NEXT: .symbol_resolver _global_ifunc -; MACHO-DEFAULT-NEXT: b _the_resolver - -; MACHO-MANUAL: .section __DATA,__data -; MACHO-MANUAL-NEXT: .globl _global_ifunc.lazy_pointer -; MACHO-MANUAL-NEXT: .p2align 3, 0x0 -; MACHO-MANUAL-NEXT: _global_ifunc.lazy_pointer: -; MACHO-MANUAL-NEXT: .quad _global_ifunc.stub_helper - -; MACHO-MANUAL: .section __TEXT,__text,regular,pure_instructions -; MACHO-MANUAL-NEXT: .globl _global_ifunc -; MACHO-MANUAL-NEXT: .p2align 2 -; MACHO-MANUAL-NEXT: _global_ifunc: -; MACHO-MANUAL-NEXT: adrp x16, _global_ifunc.lazy_pointer@GOTPAGE -; MACHO-MANUAL-NEXT: ldr x16, [x16, _global_ifunc.lazy_pointer@GOTPAGEOFF] -; MACHO-MANUAL-NEXT: ldr x16, [x16] -; MACHO-MANUAL-NEXT: br x16 -; MACHO-MANUAL-NEXT: .globl _global_ifunc.stub_helper -; MACHO-MANUAL-NEXT: .p2align 2 -; MACHO-MANUAL-NEXT: _global_ifunc.stub_helper: -; MACHO-MANUAL-NEXT: stp x29, x30, [sp, #-16] -; MACHO-MANUAL-NEXT: sub x29, sp, #16 -; MACHO-MANUAL-NEXT: stp x1, x0, [sp, #-32] -; MACHO-MANUAL-NEXT: stp x3, x2, [sp, #-48] -; MACHO-MANUAL-NEXT: stp x5, x4, [sp, #-64] -; MACHO-MANUAL-NEXT: stp x7, x6, [sp, #-80] -; MACHO-MANUAL-NEXT: stp d1, d0, [sp, #-96] -; MACHO-MANUAL-NEXT: stp d3, d2, [sp, #-112] -; MACHO-MANUAL-NEXT: stp d5, d4, [sp, #-128] -; MACHO-MANUAL-NEXT: stp d7, d6, [sp, #-144] -; MACHO-MANUAL-NEXT: sub sp, sp, #144 -; MACHO-MANUAL-NEXT: bl _the_resolver -; MACHO-MANUAL-NEXT: adrp x16, _global_ifunc.lazy_pointer@GOTPAGE -; MACHO-MANUAL-NEXT: ldr x16, [x16, _global_ifunc.lazy_pointer@GOTPAGEOFF] -; MACHO-MANUAL-NEXT: str x0, [x16] -; MACHO-MANUAL-NEXT: add x16, x0, #0 -; MACHO-MANUAL-NEXT: add sp, sp, #144 -; MACHO-MANUAL-NEXT: ldp d7, d6, [sp, #-144] -; MACHO-MANUAL-NEXT: ldp d5, d4, [sp, #-128] -; MACHO-MANUAL-NEXT: ldp d3, d2, [sp, #-112] -; MACHO-MANUAL-NEXT: ldp d1, d0, [sp, #-96] -; MACHO-MANUAL-NEXT: ldp x7, x6, [sp, #-80] -; MACHO-MANUAL-NEXT: ldp x5, x4, [sp, #-64] -; MACHO-MANUAL-NEXT: ldp x3, x2, [sp, #-48] -; MACHO-MANUAL-NEXT: ldp x1, x0, [sp, #-32] -; MACHO-MANUAL-NEXT: ldp x29, x30, [sp, #-16] -; MACHO-MANUAL-NEXT: br x16 +; MACHO: .section __TEXT,__text,regular,pure_instructions +; MACHO-NEXT: .globl _global_ifunc +; MACHO-NEXT: .p2align 2 +; MACHO-NEXT: _global_ifunc: +; MACHO-NEXT: adrp x16, _global_ifunc.lazy_pointer@GOTPAGE +; MACHO-NEXT: ldr x16, [x16, _global_ifunc.lazy_pointer@GOTPAGEOFF] +; MACHO-NEXT: ldr x16, [x16] +; MACHO-NEXT: br x16 +; MACHO-NEXT: .globl _global_ifunc.stub_helper +; MACHO-NEXT: .p2align 2 +; MACHO-NEXT: _global_ifunc.stub_helper: +; MACHO-NEXT: stp x29, x30, [sp, #-16]! +; MACHO-NEXT: mov x29, sp +; MACHO-NEXT: stp x1, x0, [sp, #-16]! +; MACHO-NEXT: stp x3, x2, [sp, #-16]! +; MACHO-NEXT: stp x5, x4, [sp, #-16]! +; MACHO-NEXT: stp x7, x6, [sp, #-16]! +; MACHO-NEXT: stp d1, d0, [sp, #-16]! +; MACHO-NEXT: stp d3, d2, [sp, #-16]! +; MACHO-NEXT: stp d5, d4, [sp, #-16]! +; MACHO-NEXT: stp d7, d6, [sp, #-16]! +; MACHO-NEXT: bl _the_resolver +; MACHO-NEXT: adrp x16, _global_ifunc.lazy_pointer@GOTPAGE +; MACHO-NEXT: ldr x16, [x16, _global_ifunc.lazy_pointer@GOTPAGEOFF] +; MACHO-NEXT: str x0, [x16] +; MACHO-NEXT: add x16, x0, #0 +; MACHO-NEXT: ldp d7, d6, [sp], #16 +; MACHO-NEXT: ldp d5, d4, [sp], #16 +; MACHO-NEXT: ldp d3, d2, [sp], #16 +; MACHO-NEXT: ldp d1, d0, [sp], #16 +; MACHO-NEXT: ldp x7, x6, [sp], #16 +; MACHO-NEXT: ldp x5, x4, [sp], #16 +; MACHO-NEXT: ldp x3, x2, [sp], #16 +; MACHO-NEXT: ldp x1, x0, [sp], #16 +; MACHO-NEXT: ldp x29, x30, [sp], #16 +; MACHO-NEXT: br x16 @weak_ifunc = weak ifunc i32 (i32), ptr @the_resolver ; ELF: .type weak_ifunc,@gnu_indirect_function -; MACHO-LINKER: .symbol_resolver _weak_ifunc -; MACHO-MANUAL: _weak_ifunc.stub_helper: -; MACHO-DEFEAULT: _weak_ifunc.stub_helper: \ No newline at end of file +; MACHO: .weak_reference _weak_ifunc.lazy_pointer +; MACHO: _weak_ifunc.lazy_pointer: +; MACHO: .weak_reference _weak_ifunc +; MACHO: _weak_ifunc: +; MACHO: .weak_reference _weak_ifunc.stub_helper +; MACHO: _weak_ifunc.stub_helper: \ No newline at end of file