105 changes: 105 additions & 0 deletions llvm/lib/Target/X86/X86MCInstLower.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,15 @@
#include "llvm/MC/MCFixup.h"
#include "llvm/MC/MCInst.h"
#include "llvm/MC/MCInstBuilder.h"
#include "llvm/MC/MCSection.h"
#include "llvm/MC/MCStreamer.h"
#include "llvm/MC/MCSymbol.h"
#include "llvm/MC/MCSymbolELF.h"
#include "llvm/MC/MCSectionELF.h"
#include "llvm/Support/TargetRegistry.h"
#include "llvm/Support/ELF.h"
#include "llvm/Target/TargetLoweringObjectFile.h"

using namespace llvm;

namespace {
Expand Down Expand Up @@ -1018,6 +1024,99 @@ void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI,
getSubtargetInfo());
}

void X86AsmPrinter::recordSled(MCSymbol *Sled, const MachineInstr &MI,
SledKind Kind) {
auto Fn = MI.getParent()->getParent()->getFunction();
auto Attr = Fn->getFnAttribute("function-instrument");
bool AlwaysInstrument =
Attr.isStringAttribute() && Attr.getValueAsString() == "xray-always";
Sleds.emplace_back(
XRayFunctionEntry{Sled, CurrentFnSym, Kind, AlwaysInstrument, Fn});
}

void X86AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI,
X86MCInstLower &MCIL) {
// We want to emit the following pattern:
//
// .Lxray_sled_N:
// .palign 2, ...
// jmp .tmpN
// # 9 bytes worth of noops
// .tmpN
//
// We need the 9 bytes because at runtime, we'd be patching over the full 11
// bytes with the following pattern:
//
// mov %r10, <function id, 32-bit> // 6 bytes
// call <relative offset, 32-bits> // 5 bytes
//
auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
OutStreamer->EmitLabel(CurSled);
OutStreamer->EmitCodeAlignment(4);
auto Target = OutContext.createTempSymbol();

// Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
// an operand (computed as an offset from the jmp instruction).
// FIXME: Find another less hacky way do force the relative jump.
OutStreamer->EmitBytes("\xeb\x09");
EmitNops(*OutStreamer, 9, Subtarget->is64Bit(), getSubtargetInfo());
OutStreamer->EmitLabel(Target);
recordSled(CurSled, MI, SledKind::FUNCTION_ENTER);
}

void X86AsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI,
X86MCInstLower &MCIL) {
// Since PATCHABLE_RET takes the opcode of the return statement as an
// argument, we use that to emit the correct form of the RET that we want.
// i.e. when we see this:
//
// PATCHABLE_RET X86::RET ...
//
// We should emit the RET followed by sleds.
//
// .Lxray_sled_N:
// ret # or equivalent instruction
// # 10 bytes worth of noops
//
// This just makes sure that the alignment for the next instruction is 2.
auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
OutStreamer->EmitLabel(CurSled);
unsigned OpCode = MI.getOperand(0).getImm();
MCInst Ret;
Ret.setOpcode(OpCode);
for (auto &MO : make_range(MI.operands_begin() + 1, MI.operands_end()))
if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO))
Ret.addOperand(MaybeOperand.getValue());
OutStreamer->EmitInstruction(Ret, getSubtargetInfo());
EmitNops(*OutStreamer, 10, Subtarget->is64Bit(), getSubtargetInfo());
recordSled(CurSled, MI, SledKind::FUNCTION_EXIT);
}

void X86AsmPrinter::EmitXRayTable() {
if (Sleds.empty())
return;
if (Subtarget->isTargetELF()) {
auto *Section = OutContext.getELFSection(
"xray_instr_map", ELF::SHT_PROGBITS,
ELF::SHF_ALLOC | ELF::SHF_GROUP | ELF::SHF_MERGE, 0,
CurrentFnSym->getName());
auto PrevSection = OutStreamer->getCurrentSectionOnly();
OutStreamer->SwitchSection(Section);
for (const auto &Sled : Sleds) {
OutStreamer->EmitSymbolValue(Sled.Sled, 8);
OutStreamer->EmitSymbolValue(CurrentFnSym, 8);
auto Kind = static_cast<uint8_t>(Sled.Kind);
OutStreamer->EmitBytes(
StringRef(reinterpret_cast<const char *>(&Kind), 1));
OutStreamer->EmitBytes(
StringRef(reinterpret_cast<const char *>(&Sled.AlwaysInstrument), 1));
OutStreamer->EmitZeros(14);
}
OutStreamer->SwitchSection(PrevSection);
}
Sleds.clear();
}

// Returns instruction preceding MBBI in MachineFunction.
// If MBBI is the first instruction of the first basic block, returns null.
static MachineBasicBlock::const_iterator
Expand Down Expand Up @@ -1259,6 +1358,12 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
case TargetOpcode::PATCHPOINT:
return LowerPATCHPOINT(*MI, MCInstLowering);

case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
return LowerPATCHABLE_FUNCTION_ENTER(*MI, MCInstLowering);

case TargetOpcode::PATCHABLE_RET:
return LowerPATCHABLE_RET(*MI, MCInstLowering);

case X86::MORESTACK_RET:
EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget)));
return;
Expand Down
13 changes: 13 additions & 0 deletions llvm/test/CodeGen/X86/xray-attribute-instrumentation.ll
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
; RUN: llc -filetype=asm -o - -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s

define i32 @foo() nounwind noinline uwtable "function-instrument"="xray-always" {
; CHECK-LABEL: Lxray_sled_0:
; CHECK-NEXT: .p2align 2, 0x90
; CHECK-NEXT: .ascii "\353\t"
; CHECK-NEXT: nopw 512(%rax,%rax)
; CHECK-LABEL: Ltmp0:
ret i32 0
; CHECK-LABEL: Lxray_sled_1:
; CHECK-NEXT: retq
; CHECK-NEXT: nopw %cs:512(%rax,%rax)
}
9 changes: 9 additions & 0 deletions llvm/test/CodeGen/X86/xray-selective-instrumentation-miss.ll
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
; RUN: llc -mcpu=nehalem < %s | not grep xray_sled_

target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
target triple = "x86_64-apple-darwin8"

define i32 @foo() nounwind uwtable "xray-instruction-threshold"="3" {
entry:
ret i32 0
}
9 changes: 9 additions & 0 deletions llvm/test/CodeGen/X86/xray-selective-instrumentation.ll
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
; RUN: llc -mcpu=nehalem < %s | grep xray_sled_

target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
target triple = "x86_64-apple-darwin8"

define i32 @foo() nounwind uwtable "xray-instruction-threshold"="1" {
entry:
ret i32 0
}
6 changes: 3 additions & 3 deletions llvm/utils/TableGen/InstrInfoEmitter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
OS << "namespace llvm {\n";
OS << "struct " << ClassName << " : public TargetInstrInfo {\n"
<< " explicit " << ClassName
<< "(int CFSetupOpcode = -1, int CFDestroyOpcode = -1, int CatchRetOpcode = -1);\n"
<< "(int CFSetupOpcode = -1, int CFDestroyOpcode = -1, int CatchRetOpcode = -1, int ReturnOpcode = -1);\n"
<< " ~" << ClassName << "() override {}\n"
<< "};\n";
OS << "} // end llvm namespace\n";
Expand All @@ -443,8 +443,8 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
OS << "extern const unsigned " << TargetName << "InstrNameIndices[];\n";
OS << "extern const char " << TargetName << "InstrNameData[];\n";
OS << ClassName << "::" << ClassName
<< "(int CFSetupOpcode, int CFDestroyOpcode, int CatchRetOpcode)\n"
<< " : TargetInstrInfo(CFSetupOpcode, CFDestroyOpcode, CatchRetOpcode) {\n"
<< "(int CFSetupOpcode, int CFDestroyOpcode, int CatchRetOpcode, int ReturnOpcode)\n"
<< " : TargetInstrInfo(CFSetupOpcode, CFDestroyOpcode, CatchRetOpcode, ReturnOpcode) {\n"
<< " InitMCInstrInfo(" << TargetName << "Insts, " << TargetName
<< "InstrNameIndices, " << TargetName << "InstrNameData, "
<< NumberedInstructions.size() << ");\n}\n";
Expand Down