diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index 26b3a14e22b2a..1b5b7c556c79f 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -1901,6 +1901,9 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, OutStreamer->switchSection(OutContext.getObjectFileInfo()->getDataSection()); + // _ifunc.lazy_pointer: + // .quad _ifunc.stub_helper + EmitLinkage(LazyPointer); OutStreamer->emitLabel(LazyPointer); emitVisibility(LazyPointer, GI.getVisibility()); @@ -1908,18 +1911,18 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection()); - MCSymbol *Stub = getSymbol(&GI); + // _ifunc: + // adrp x16, lazy_pointer@GOTPAGE + // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] + // ldr x16, [x16] + // br x16 + MCSymbol *Stub = getSymbol(&GI); EmitLinkage(Stub); OutStreamer->emitCodeAlignment(Align(4), STI); OutStreamer->emitLabel(Stub); emitVisibility(Stub, GI.getVisibility()); - // adrp x16, lazy_pointer@GOTPAGE - // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] - // ldr x16, [x16] - // br x16 - { MCInst Adrp; Adrp.setOpcode(AArch64::ADRP); @@ -1960,39 +1963,40 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, .addReg(AArch64::X16), *STI); + // _ifunc.stub_helper: + // stp fp, lr, [sp, #-16] + // sub fp, sp, 16 + // stp x1, x0, [sp, #-32] + // stp x3, x2, [sp, #-48] + // stp x5, x4, [sp, #-64] + // stp x7, x6, [sp, #-80] + // stp d1, d0, [sp, #-96] + // stp d3, d2, [sp, #-112] + // stp d5, d4, [sp, #-128] + // stp d7, d6, [sp, #-144] + // sub sp, sp, 144 + // bl _resolver + // adrp x16, lazy_pointer@GOTPAGE + // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] + // str x0, [x16] + // mov x16, x0 + // add sp, sp, 144 + // ldp d7, d6, [sp, #-144] + // ldp d5, d4, [sp, #-128] + // ldp d3, d2, [sp, #-112] + // ldp d1, d0, [sp, #-96] + // ldp x7, x6, [sp, #-80] + // ldp x5, x4, [sp, #-64] + // ldp x3, x2, [sp, #-48] + // ldp x1, x0, [sp, #-32] + // ldp fp, lr, [sp, #-16] + // br x16 + EmitLinkage(StubHelper); OutStreamer->emitCodeAlignment(Align(4), STI); OutStreamer->emitLabel(StubHelper); emitVisibility(StubHelper, GI.getVisibility()); - // stp fp, lr, [sp, #-16] - // sub fp, sp, 16 - // stp x1, x0, [sp, #-32] - // stp x3, x2, [sp, #-48] - // stp x5, x4, [sp, #-64] - // stp x7, x6, [sp, #-80] - // stp d1, d0, [sp, #-96] - // stp d3, d2, [sp, #-112] - // stp d5, d4, [sp, #-128] - // stp d7, d6, [sp, #-144] - // sub sp, sp, 144 - // bl _resolver - // adrp x16, lazy_pointer@GOTPAGE - // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] - // str x0, [x16] - // mov x16, x0 - // add sp, sp, 144 - // ldp d7, d6, [sp, #-144] - // ldp d5, d4, [sp, #-128] - // ldp d3, d2, [sp, #-112] - // ldp d1, d0, [sp, #-96] - // ldp x7, x6, [sp, #-80] - // ldp x5, x4, [sp, #-64] - // ldp x3, x2, [sp, #-48] - // ldp x1, x0, [sp, #-32] - // ldp fp, lr, [sp, #-16] - // br x16 - OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi) .addReg(AArch64::FP) .addReg(AArch64::LR) diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp index 37158900d2404..b0f4b9d984372 100644 --- a/llvm/lib/Target/X86/X86AsmPrinter.cpp +++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp @@ -14,6 +14,7 @@ #include "X86AsmPrinter.h" #include "MCTargetDesc/X86ATTInstPrinter.h" #include "MCTargetDesc/X86BaseInfo.h" +#include "MCTargetDesc/X86MCTargetDesc.h" #include "MCTargetDesc/X86TargetStreamer.h" #include "TargetInfo/X86TargetInfo.h" #include "X86InstrInfo.h" @@ -34,6 +35,7 @@ #include "llvm/MC/MCCodeEmitter.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" +#include "llvm/MC/MCInst.h" #include "llvm/MC/MCInstBuilder.h" #include "llvm/MC/MCSectionCOFF.h" #include "llvm/MC/MCSectionELF.h" @@ -534,30 +536,112 @@ void X86AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) { if (!TM.getTargetTriple().isOSBinFormatMachO()) return AsmPrinter::emitGlobalIFunc(M, GI); - OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection()); + auto EmitLinkage = [&](MCSymbol *Sym) { + if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective()) + OutStreamer->emitSymbolAttribute(Sym, MCSA_Global); + else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage()) + OutStreamer->emitSymbolAttribute(Sym, MCSA_WeakReference); + else + assert(GI.hasLocalLinkage() && "Invalid ifunc linkage"); + }; - MCSymbol *Name = getSymbol(&GI); + MCSymbol *LazyPointer = + TM.getObjFileLowering()->getContext().getOrCreateSymbol( + "_" + GI.getName() + ".lazy_pointer"); + MCSymbol *StubHelper = + TM.getObjFileLowering()->getContext().getOrCreateSymbol( + "_" + GI.getName() + ".stub_helper"); - if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective()) - OutStreamer->emitSymbolAttribute(Name, MCSA_Global); - else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage()) - OutStreamer->emitSymbolAttribute(Name, MCSA_WeakReference); - else - assert(GI.hasLocalLinkage() && "Invalid ifunc linkage"); + OutStreamer->switchSection(OutContext.getObjectFileInfo()->getDataSection()); + + // _ifunc.lazy_pointer: + // .quad _ifunc.stub_helper + EmitLinkage(LazyPointer); + OutStreamer->emitLabel(LazyPointer); + emitVisibility(LazyPointer, GI.getVisibility()); + OutStreamer->emitValue(MCSymbolRefExpr::create(StubHelper, OutContext), 8); + + OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection()); + + // _ifunc: + // jmpq *lazy_pointer(%rip) + + MCSymbol *Stub = getSymbol(&GI); + EmitLinkage(Stub); + OutStreamer->emitCodeAlignment(Align(16), Subtarget); + OutStreamer->emitLabel(Stub); + emitVisibility(Stub, GI.getVisibility()); + + OutStreamer->emitInstruction( + MCInstBuilder(X86::JMP32m) + .addReg(X86::RIP) + .addImm(1) + .addReg(0) + .addOperand(MCOperand::createExpr( + MCSymbolRefExpr::create(LazyPointer, OutContext))) + .addReg(0), + *Subtarget); + + // _ifunc.stub_helper: + // push %rax + // push %rdi + // push %rsi + // push %rdx + // push %rcx + // push %r8 + // push %r9 + // callq foo + // movq %rax,lazy_pointer(%rip) + // pop %r9 + // pop %r8 + // pop %rcx + // pop %rdx + // pop %rsi + // pop %rdi + // pop %rax + // jmpq *lazy_pointer(%rip) + + EmitLinkage(StubHelper); OutStreamer->emitCodeAlignment(Align(16), Subtarget); - OutStreamer->emitLabel(Name); - OutStreamer->emitSymbolAttribute(Name, MCSA_SymbolResolver); - emitVisibility(Name, GI.getVisibility()); - - // ld64 does not seem to support aliases of symbol resolvers, so we have to - // tail call the resolver manually. - MCInst JMP; - JMP.setOpcode(X86::JMP_4); - JMP.addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))); - OutStreamer->emitInstruction(JMP, *Subtarget); - - // FIXME: do the manual .symbol_resolver lowering that we did in AArch64AsmPrinter. + OutStreamer->emitLabel(StubHelper); + emitVisibility(StubHelper, GI.getVisibility()); + + for (int Reg : + {X86::RAX, X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9}) + OutStreamer->emitInstruction(MCInstBuilder(X86::PUSH64r).addReg(Reg), + *Subtarget); + + OutStreamer->emitInstruction( + MCInstBuilder(X86::CALL64pcrel32) + .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))), + *Subtarget); + + OutStreamer->emitInstruction( + MCInstBuilder(X86::MOV64mr) + .addReg(X86::RIP) + .addImm(1) + .addReg(0) + .addOperand(MCOperand::createExpr( + MCSymbolRefExpr::create(LazyPointer, OutContext))) + .addReg(0) + .addReg(X86::RAX), + *Subtarget); + + for (int Reg : + {X86::R9, X86::R8, X86::RCX, X86::RDX, X86::RSI, X86::RDI, X86::RAX}) + OutStreamer->emitInstruction(MCInstBuilder(X86::POP64r).addReg(Reg), + *Subtarget); + + OutStreamer->emitInstruction( + MCInstBuilder(X86::JMP32m) + .addReg(X86::RIP) + .addImm(1) + .addReg(0) + .addOperand(MCOperand::createExpr( + MCSymbolRefExpr::create(LazyPointer, OutContext))) + .addReg(0), + *Subtarget); } static bool printAsmMRegister(const X86AsmPrinter &P, const MachineOperand &MO, diff --git a/llvm/test/CodeGen/X86/ifunc-asm.ll b/llvm/test/CodeGen/X86/ifunc-asm.ll index 76efda7115320..0f66febbe95b2 100644 --- a/llvm/test/CodeGen/X86/ifunc-asm.ll +++ b/llvm/test/CodeGen/X86/ifunc-asm.ll @@ -5,20 +5,44 @@ define internal ptr @foo_resolver() { entry: ret ptr null } -; ELF: .type foo_resolver,@function -; ELF-NEXT: foo_resolver: +; ELF: .type foo_resolver,@function +; ELF-NEXT: foo_resolver: -; MACHO: .p2align 4, 0x90 -; MACHO-NEXT: _foo_resolver +; MACHO: .p2align 4, 0x90 +; MACHO-NEXT: _foo_resolver @foo_ifunc = ifunc i32 (i32), ptr @foo_resolver -; ELF: .globl foo_ifunc -; ELF-NEXT: .type foo_ifunc,@gnu_indirect_function -; ELF-NEXT: .set foo_ifunc, foo_resolver +; ELF: .globl foo_ifunc +; ELF-NEXT: .type foo_ifunc,@gnu_indirect_function +; ELF-NEXT: .set foo_ifunc, foo_resolver -; MACHO: .globl _foo_ifunc -; MACHO-NEXT: .p2align 4, 0x90 -; MACHO-NEXT: _foo_ifunc: -; MACHO-NEXT: .symbol_resolver _foo_ifunc -; MACHO-NEXT: jmp _foo_resolver +; MACHO: .section __DATA,__data +; MACHO-NEXT: .globl _foo_ifunc.lazy_pointer +; MACHO-NEXT: _foo_ifunc.lazy_pointer: +; MACHO-NEXT: .quad _foo_ifunc.stub_helper +; MACHO-NEXT: .section __TEXT,__text,regular,pure_instructions +; MACHO-NEXT: .globl _foo_ifunc +; MACHO-NEXT: .p2align 4, 0x90 +; MACHO-NEXT: _foo_ifunc: +; MACHO-NEXT: jmpl *_foo_ifunc.lazy_pointer(%rip) +; MACHO-NEXT: .globl _foo_ifunc.stub_helper +; MACHO-NEXT: .p2align 4, 0x90 +; MACHO-NEXT: _foo_ifunc.stub_helper: +; MACHO-NEXT: pushq %rax +; MACHO-NEXT: pushq %rdi +; MACHO-NEXT: pushq %rsi +; MACHO-NEXT: pushq %rdx +; MACHO-NEXT: pushq %rcx +; MACHO-NEXT: pushq %r8 +; MACHO-NEXT: pushq %r9 +; MACHO-NEXT: callq _foo_resolver +; MACHO-NEXT: movq %rax, _foo_ifunc.lazy_pointer(%rip) +; MACHO-NEXT: popq %r9 +; MACHO-NEXT: popq %r8 +; MACHO-NEXT: popq %rcx +; MACHO-NEXT: popq %rdx +; MACHO-NEXT: popq %rsi +; MACHO-NEXT: popq %rdi +; MACHO-NEXT: popq %rax +; MACHO-NEXT: jmpl *_foo_ifunc.lazy_pointer(%rip) \ No newline at end of file