diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td index 09e31988e5bd8..0ccf23683f0b3 100644 --- a/llvm/lib/Target/X86/X86InstrCompiler.td +++ b/llvm/lib/Target/X86/X86InstrCompiler.td @@ -260,7 +260,7 @@ let isPseudo = 1, SchedRW = [WriteSystem] in { // Pseudo instructions used by KCFI. //===----------------------------------------------------------------------===// let - Defs = [R10, EFLAGS] in { + Defs = [R10, R11, EFLAGS] in { def KCFI_CHECK : PseudoI< (outs), (ins GR64:$ptr, i32imm:$type), []>, Sched<[]>; } diff --git a/llvm/lib/Target/X86/X86KCFI.cpp b/llvm/lib/Target/X86/X86KCFI.cpp index b4bced17048a1..4086f28804fca 100644 --- a/llvm/lib/Target/X86/X86KCFI.cpp +++ b/llvm/lib/Target/X86/X86KCFI.cpp @@ -63,6 +63,33 @@ bool X86KCFI::emitCheck(MachineBasicBlock &MBB, if (MBBI->isBundled() && !std::prev(MBBI)->isBundle()) report_fatal_error("Cannot emit a KCFI check for a bundled call"); + MachineFunction &MF = *MBB.getParent(); + // If the call target is a memory operand, unfold it and use R11 for the + // call, so KCFI_CHECK won't have to recompute the address. + switch (MBBI->getOpcode()) { + case X86::CALL64m: + case X86::CALL64m_NT: + case X86::TAILJMPm64: + case X86::TAILJMPm64_REX: { + MachineBasicBlock::instr_iterator OrigCall = MBBI; + SmallVector NewMIs; + if (!TII->unfoldMemoryOperand(MF, *OrigCall, X86::R11, /*UnfoldLoad=*/true, + /*UnfoldStore=*/false, NewMIs)) + report_fatal_error("Failed to unfold memory operand for a KCFI check"); + for (auto *NewMI : NewMIs) + MBBI = MBB.insert(OrigCall, NewMI); + assert(MBBI->isCall() && + "Unexpected instruction after memory operand unfolding"); + if (OrigCall->shouldUpdateCallSiteInfo()) + MF.moveCallSiteInfo(&*OrigCall, &*MBBI); + MBBI->setCFIType(MF, OrigCall->getCFIType()); + OrigCall->eraseFromParent(); + break; + } + default: + break; + } + MachineInstr *Check = BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(X86::KCFI_CHECK)) .getInstr(); @@ -73,9 +100,6 @@ bool X86KCFI::emitCheck(MachineBasicBlock &MBB, case X86::TAILJMPr64: case X86::TAILJMPr64_REX: assert(Target.isReg() && "Unexpected target operand for an indirect call"); - // KCFI_CHECK uses r10 as a temporary register. - assert(Target.getReg() != X86::R10 && - "Unsupported target register for a KCFI call"); Check->addOperand(MachineOperand::CreateReg(Target.getReg(), false)); Target.setIsRenamable(false); break; @@ -93,7 +117,7 @@ bool X86KCFI::emitCheck(MachineBasicBlock &MBB, } Check->addOperand(MachineOperand::CreateImm(MBBI->getCFIType())); - MBBI->setCFIType(*MBB.getParent(), 0); + MBBI->setCFIType(MF, 0); // If not already bundled, bundle the check and the call to prevent // further changes. diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp index 157ec3631ed08..9b15bad41ff5c 100644 --- a/llvm/lib/Target/X86/X86MCInstLower.cpp +++ b/llvm/lib/Target/X86/X86MCInstLower.cpp @@ -1364,14 +1364,17 @@ void X86AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) { // and thus emitting potential call target gadgets at each indirect call // site, load a negated constant to a register and compare that to the // expected value at the call target. + const Register AddrReg = MI.getOperand(0).getReg(); const uint32_t Type = MI.getOperand(1).getImm(); - EmitAndCountInstruction(MCInstBuilder(X86::MOV32ri) - .addReg(X86::R10D) - .addImm(-MaskKCFIType(Type))); + // The check is immediately before the call. If the call target is in R10, + // we can clobber R11 for the check instead. + unsigned TempReg = AddrReg == X86::R10 ? X86::R11D : X86::R10D; + EmitAndCountInstruction( + MCInstBuilder(X86::MOV32ri).addReg(TempReg).addImm(-MaskKCFIType(Type))); EmitAndCountInstruction(MCInstBuilder(X86::ADD32rm) .addReg(X86::NoRegister) - .addReg(X86::R10D) - .addReg(MI.getOperand(0).getReg()) + .addReg(TempReg) + .addReg(AddrReg) .addImm(1) .addReg(X86::NoRegister) .addImm(-(PrefixNops + 4)) diff --git a/llvm/test/CodeGen/X86/kcfi.ll b/llvm/test/CodeGen/X86/kcfi.ll index 362df82a5b20b..4b93fdddb82d7 100644 --- a/llvm/test/CodeGen/X86/kcfi.ll +++ b/llvm/test/CodeGen/X86/kcfi.ll @@ -37,8 +37,8 @@ define void @f1(ptr noundef %x) !kcfi_type !1 { ; MIR-LABEL: name: f1 ; MIR: body: ; ISEL: CALL64r %0, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, cfi-type 12345678 -; KCFI: BUNDLE implicit-def $r10, implicit-def $r10d, implicit-def $r10w, implicit-def $r10b, implicit-def $r10bh, implicit-def $r10wh, implicit-def $eflags, implicit-def $rsp, implicit-def $esp, implicit-def $sp, implicit-def $spl, implicit-def $sph, implicit-def $hsp, implicit-def $ssp, implicit killed $rdi, implicit $rsp, implicit $ssp { -; KCFI-NEXT: KCFI_CHECK $rdi, 12345678, implicit-def $r10, implicit-def $eflags +; KCFI: BUNDLE{{.*}} { +; KCFI-NEXT: KCFI_CHECK $rdi, 12345678, implicit-def $r10, implicit-def $r11, implicit-def $eflags ; KCFI-NEXT: CALL64r killed $rdi, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp ; KCFI-NEXT: } call void %x() [ "kcfi"(i32 12345678) ] @@ -52,8 +52,8 @@ define void @f2(ptr noundef %x) { ; MIR-LABEL: name: f2 ; MIR: body: ; ISEL: TCRETURNri64 %0, 0, csr_64, implicit $rsp, implicit $ssp, cfi-type 12345678 -; KCFI: BUNDLE implicit-def $r10, implicit-def $r10d, implicit-def $r10w, implicit-def $r10b, implicit-def $r10bh, implicit-def $r10wh, implicit-def $eflags, implicit killed $rdi, implicit $rsp, implicit $ssp { -; KCFI-NEXT: KCFI_CHECK $rdi, 12345678, implicit-def $r10, implicit-def $eflags +; KCFI: BUNDLE{{.*}} { +; KCFI-NEXT: KCFI_CHECK $rdi, 12345678, implicit-def $r10, implicit-def $r11, implicit-def $eflags ; KCFI-NEXT: TAILJMPr64 killed $rdi, csr_64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp ; KCFI-NEXT: } tail call void %x() [ "kcfi"(i32 12345678) ] @@ -66,9 +66,9 @@ define void @f3(ptr noundef %x) #0 { ; MIR-LABEL: name: f3 ; MIR: body: ; ISEL: CALL64pcrel32 &__llvm_retpoline_r11, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit killed $r11, cfi-type 12345678 -; KCFI: BUNDLE implicit-def $r10, implicit-def $r10d, implicit-def $r10w, implicit-def $r10b, implicit-def $r10bh, implicit-def $r10wh, implicit-def $eflags, implicit-def $rsp, implicit-def $esp, implicit-def $sp, implicit-def $spl, implicit-def $sph, implicit-def $hsp, implicit-def $ssp, implicit killed $r11, implicit $rsp, implicit $ssp { -; KCFI-NEXT: KCFI_CHECK $r11, 12345678, implicit-def $r10, implicit-def $eflags -; KCFI-NEXT: CALL64pcrel32 &__llvm_retpoline_r11, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit killed $r11 +; KCFI: BUNDLE{{.*}} { +; KCFI-NEXT: KCFI_CHECK $r11, 12345678, implicit-def $r10, implicit-def $r11, implicit-def $eflags +; KCFI-NEXT: CALL64pcrel32 &__llvm_retpoline_r11, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit internal killed $r11 ; KCFI-NEXT: } call void %x() [ "kcfi"(i32 12345678) ] ret void @@ -80,9 +80,9 @@ define void @f4(ptr noundef %x) #0 { ; MIR-LABEL: name: f4 ; MIR: body: ; ISEL: TCRETURNdi64 &__llvm_retpoline_r11, 0, csr_64, implicit $rsp, implicit $ssp, implicit killed $r11, cfi-type 12345678 -; KCFI: BUNDLE implicit-def $r10, implicit-def $r10d, implicit-def $r10w, implicit-def $r10b, implicit-def $r10bh, implicit-def $r10wh, implicit-def $eflags, implicit killed $r11, implicit $rsp, implicit $ssp { -; KCFI-NEXT: KCFI_CHECK $r11, 12345678, implicit-def $r10, implicit-def $eflags -; KCFI-NEXT: TAILJMPd64 &__llvm_retpoline_r11, csr_64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp, implicit killed $r11 +; KCFI: BUNDLE{{.*}} { +; KCFI-NEXT: KCFI_CHECK $r11, 12345678, implicit-def $r10, implicit-def $r11, implicit-def $eflags +; KCFI-NEXT: TAILJMPd64 &__llvm_retpoline_r11, csr_64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp, implicit internal killed $r11 ; KCFI-NEXT: } tail call void %x() [ "kcfi"(i32 12345678) ] ret void @@ -108,6 +108,36 @@ define void @f6(ptr noundef %x) !kcfi_type !3 { ret void } +@g = external local_unnamed_addr global ptr, align 8 + +define void @f7() { +; MIR-LABEL: name: f7 +; MIR: body: +; ISEL: TCRETURNmi64 killed %0, 1, $noreg, 0, $noreg, 0, csr_64, implicit $rsp, implicit $ssp, cfi-type 12345678 +; KCFI: $r11 = MOV64rm killed renamable $rax, 1, $noreg, 0, $noreg +; KCFI-NEXT: BUNDLE{{.*}} { +; KCFI-NEXT: KCFI_CHECK $r11, 12345678, implicit-def $r10, implicit-def $r11, implicit-def $eflags +; KCFI-NEXT: TAILJMPr64 internal $r11, csr_64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp +; KCFI-NEXT: } + %1 = load ptr, ptr @g, align 8 + tail call void %1() [ "kcfi"(i32 12345678) ] + ret void +} + +define void @f8() { +; MIR-LABEL: name: f8 +; MIR: body: +; ISEL: CALL64m killed %0, 1, $noreg, 0, $noreg, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, cfi-type 12345678 +; KCFI: $r11 = MOV64rm killed renamable $rax, 1, $noreg, 0, $noreg +; KCFI-NEXT: BUNDLE{{.*}} { +; KCFI-NEXT: KCFI_CHECK $r11, 12345678, implicit-def $r10, implicit-def $r11, implicit-def $eflags +; KCFI-NEXT: CALL64r internal $r11, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp +; KCFI-NEXT: } + %1 = load ptr, ptr @g, align 8 + call void %1() [ "kcfi"(i32 12345678) ] + ret void +} + attributes #0 = { "target-features"="+retpoline-indirect-branches,+retpoline-indirect-calls" } !llvm.module.flags = !{!0}