perf_jit_trampoline.c

#include "Python.h"
#include "pycore_ceval.h"         // _PyPerf_Callbacks
#include "pycore_frame.h"
#include "pycore_interp.h"
#include "pycore_runtime.h"       // _PyRuntime


#ifdef PY_HAVE_PERF_TRAMPOLINE

#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>             // mmap()
#include <sys/types.h>
#include <unistd.h>               // sysconf()
#include <sys/time.h>           // gettimeofday()
#include <sys/syscall.h>

// ----------------------------------
//         Perf jitdump API
// ----------------------------------

typedef struct {
    FILE* perf_map;
    PyThread_type_lock map_lock;
    void* mapped_buffer;
    size_t mapped_size;
    int code_id;
} PerfMapJitState;

static PerfMapJitState perf_jit_map_state;

/*
Usually the binary and libraries are mapped in separate region like below:

  address ->
   --+---------------------+--//--+---------------------+--
     | .text | .data | ... |      | .text | .data | ... |
   --+---------------------+--//--+---------------------+--
         myprog                      libc.so

So it'd be easy and straight-forward to find a mapped binary or library from an
address.

But for JIT code, the code arena only cares about the code section. But the
resulting DSOs (which is generated by perf inject -j) contain ELF headers and
unwind info too. Then it'd generate following address space with synthesized
MMAP events. Let's say it has a sample between address B and C.

                                               sample
                                                 |
  address ->                         A       B   v   C
  ---------------------------------------------------------------------------------------------------
  /tmp/jitted-PID-0.so   | (headers) | .text | unwind info |
  /tmp/jitted-PID-1.so           | (headers) | .text | unwind info |
  /tmp/jitted-PID-2.so                   | (headers) | .text | unwind info |
    ...
  ---------------------------------------------------------------------------------------------------

If it only maps the .text section, it'd find the jitted-PID-1.so but cannot see
the unwind info. If it maps both .text section and unwind sections, the sample
could be mapped to either jitted-PID-0.so or jitted-PID-1.so and it's confusing
which one is right. So to make perf happy we have non-overlapping ranges for each
DSO:

  address ->
  -------------------------------------------------------------------------------------------------------
  /tmp/jitted-PID-0.so   | (headers) | .text | unwind info |
  /tmp/jitted-PID-1.so                         | (headers) | .text | unwind info |
  /tmp/jitted-PID-2.so                                               | (headers) | .text | unwind info |
    ...
  -------------------------------------------------------------------------------------------------------

As the trampolines are constant, we add a constant padding but in general the padding needs to have the
size of the unwind info rounded to 16 bytes. In general, for our trampolines this is 0x50
 */

#define PERF_JIT_CODE_PADDING 0x100
#define trampoline_api _PyRuntime.ceval.perf.trampoline_api

typedef uint64_t uword;
typedef const char* CodeComments;

#define Pd "d"
#define MB (1024 * 1024)

#define EM_386      3
#define EM_X86_64   62
#define EM_ARM      40
#define EM_AARCH64  183
#define EM_RISCV    243

#define TARGET_ARCH_IA32   0
#define TARGET_ARCH_X64    0
#define TARGET_ARCH_ARM    0
#define TARGET_ARCH_ARM64  0
#define TARGET_ARCH_RISCV32 0
#define TARGET_ARCH_RISCV64 0

#define FLAG_generate_perf_jitdump 0
#define FLAG_write_protect_code 0
#define FLAG_write_protect_vm_isolate 0
#define FLAG_code_comments 0

#define UNREACHABLE()

static uword GetElfMachineArchitecture(void) {
#if TARGET_ARCH_IA32
    return EM_386;
#elif TARGET_ARCH_X64
    return EM_X86_64;
#elif TARGET_ARCH_ARM
    return EM_ARM;
#elif TARGET_ARCH_ARM64
    return EM_AARCH64;
#elif TARGET_ARCH_RISCV32 || TARGET_ARCH_RISCV64
    return EM_RISCV;
#else
    UNREACHABLE();
    return 0;
#endif
}

typedef struct {
    uint32_t magic;
    uint32_t version;
    uint32_t size;
    uint32_t elf_mach_target;
    uint32_t reserved;
    uint32_t process_id;
    uint64_t time_stamp;
    uint64_t flags;
} Header;

 enum PerfEvent {
    PerfLoad = 0,
    PerfMove = 1,
    PerfDebugInfo = 2,
    PerfClose = 3,
    PerfUnwindingInfo = 4
};

struct BaseEvent {
    uint32_t event;
    uint32_t size;
    uint64_t time_stamp;
  };

typedef struct {
    struct BaseEvent base;
    uint32_t process_id;
    uint32_t thread_id;
    uint64_t vma;
    uint64_t code_address;
    uint64_t code_size;
    uint64_t code_id;
} CodeLoadEvent;

typedef struct {
    struct BaseEvent base;
    uint64_t unwind_data_size;
    uint64_t eh_frame_hdr_size;
    uint64_t mapped_size;
} CodeUnwindingInfoEvent;

static const intptr_t nanoseconds_per_second = 1000000000;

// Dwarf encoding constants

static const uint8_t DwarfUData4 = 0x03;
static const uint8_t DwarfSData4 = 0x0b;
static const uint8_t DwarfPcRel = 0x10;
static const uint8_t DwarfDataRel = 0x30;
// static uint8_t DwarfOmit = 0xff;
typedef struct {
    unsigned char version;
    unsigned char eh_frame_ptr_enc;
    unsigned char fde_count_enc;
    unsigned char table_enc;
    int32_t eh_frame_ptr;
    int32_t eh_fde_count;
    int32_t from;
    int32_t to;
} EhFrameHeader;

static int64_t get_current_monotonic_ticks(void) {
    struct timespec ts;
    if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) {
        UNREACHABLE();
        return 0;
    }
    // Convert to nanoseconds.
    int64_t result = ts.tv_sec;
    result *= nanoseconds_per_second;
    result += ts.tv_nsec;
    return result;
}

static int64_t get_current_time_microseconds(void) {
  // gettimeofday has microsecond resolution.
  struct timeval tv;
  if (gettimeofday(&tv, NULL) < 0) {
    UNREACHABLE();
    return 0;
  }
  return ((int64_t)(tv.tv_sec) * 1000000) + tv.tv_usec;
}


static size_t round_up(int64_t value, int64_t multiple) {
    if (multiple == 0) {
        // Avoid division by zero
        return value;
    }

    int64_t remainder = value % multiple;
    if (remainder == 0) {
        // Value is already a multiple of 'multiple'
        return value;
    }

    // Calculate the difference to the next multiple
    int64_t difference = multiple - remainder;

    // Add the difference to the value
    int64_t rounded_up_value = value + difference;

    return rounded_up_value;
}


static void perf_map_jit_write_fully(const void* buffer, size_t size) {
    FILE* out_file = perf_jit_map_state.perf_map;
    const char* ptr = (const char*)(buffer);
    while (size > 0) {
        const size_t written = fwrite(ptr, 1, size, out_file);
        if (written == 0) {
            UNREACHABLE();
            break;
        }
        size -= written;
        ptr += written;
    }
}

static void perf_map_jit_write_header(int pid, FILE* out_file) {
    Header header;
    header.magic = 0x4A695444;
    header.version = 1;
    header.size = sizeof(Header);
    header.elf_mach_target = GetElfMachineArchitecture();
    header.process_id = pid;
    header.time_stamp = get_current_time_microseconds();
    header.flags = 0;
    perf_map_jit_write_fully(&header, sizeof(header));
}

static void* perf_map_jit_init(void) {
    char filename[100];
    int pid = getpid();
    snprintf(filename, sizeof(filename) - 1, "/tmp/jit-%d.dump", pid);
    const int fd = open(filename, O_CREAT | O_TRUNC | O_RDWR, 0666);
    if (fd == -1) {
        return NULL;
    }

    const long page_size = sysconf(_SC_PAGESIZE);  // NOLINT(runtime/int)
    if (page_size == -1) {
        close(fd);
        return NULL;
    }

    // The perf jit interface forces us to map the first page of the file
    // to signal that we are using the interface.
    perf_jit_map_state.mapped_buffer = mmap(NULL, page_size, PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, 0);
    if (perf_jit_map_state.mapped_buffer == NULL) {
        close(fd);
        return NULL;
    }
    perf_jit_map_state.mapped_size = page_size;
    perf_jit_map_state.perf_map = fdopen(fd, "w+");
    if (perf_jit_map_state.perf_map == NULL) {
        close(fd);
        return NULL;
    }
    setvbuf(perf_jit_map_state.perf_map, NULL, _IOFBF, 2 * MB);
    perf_map_jit_write_header(pid, perf_jit_map_state.perf_map);

    perf_jit_map_state.map_lock = PyThread_allocate_lock();
    if (perf_jit_map_state.map_lock == NULL) {
        fclose(perf_jit_map_state.perf_map);
        return NULL;
    }
    perf_jit_map_state.code_id = 0;

    trampoline_api.code_padding = PERF_JIT_CODE_PADDING;
    return &perf_jit_map_state;
}

/* DWARF definitions. */

#define DWRF_CIE_VERSION 1

enum {
    DWRF_CFA_nop = 0x0,
    DWRF_CFA_offset_extended = 0x5,
    DWRF_CFA_def_cfa = 0xc,
    DWRF_CFA_def_cfa_offset = 0xe,
    DWRF_CFA_offset_extended_sf = 0x11,
    DWRF_CFA_advance_loc = 0x40,
    DWRF_CFA_offset = 0x80
};

enum
  {
    DWRF_EH_PE_absptr = 0x00,
    DWRF_EH_PE_omit = 0xff,

    /* FDE data encoding.  */
    DWRF_EH_PE_uleb128 = 0x01,
    DWRF_EH_PE_udata2 = 0x02,
    DWRF_EH_PE_udata4 = 0x03,
    DWRF_EH_PE_udata8 = 0x04,
    DWRF_EH_PE_sleb128 = 0x09,
    DWRF_EH_PE_sdata2 = 0x0a,
    DWRF_EH_PE_sdata4 = 0x0b,
    DWRF_EH_PE_sdata8 = 0x0c,
    DWRF_EH_PE_signed = 0x08,

    /* FDE flags.  */
    DWRF_EH_PE_pcrel = 0x10,
    DWRF_EH_PE_textrel = 0x20,
    DWRF_EH_PE_datarel = 0x30,
    DWRF_EH_PE_funcrel = 0x40,
    DWRF_EH_PE_aligned = 0x50,

    DWRF_EH_PE_indirect = 0x80
  };

enum { DWRF_TAG_compile_unit = 0x11 };

enum { DWRF_children_no = 0, DWRF_children_yes = 1 };

enum { DWRF_AT_name = 0x03, DWRF_AT_stmt_list = 0x10, DWRF_AT_low_pc = 0x11, DWRF_AT_high_pc = 0x12 };

enum { DWRF_FORM_addr = 0x01, DWRF_FORM_data4 = 0x06, DWRF_FORM_string = 0x08 };

enum { DWRF_LNS_extended_op = 0, DWRF_LNS_copy = 1, DWRF_LNS_advance_pc = 2, DWRF_LNS_advance_line = 3 };

enum { DWRF_LNE_end_sequence = 1, DWRF_LNE_set_address = 2 };

enum {
#ifdef __x86_64__
    /* Yes, the order is strange, but correct. */
    DWRF_REG_AX,
    DWRF_REG_DX,
    DWRF_REG_CX,
    DWRF_REG_BX,
    DWRF_REG_SI,
    DWRF_REG_DI,
    DWRF_REG_BP,
    DWRF_REG_SP,
    DWRF_REG_8,
    DWRF_REG_9,
    DWRF_REG_10,
    DWRF_REG_11,
    DWRF_REG_12,
    DWRF_REG_13,
    DWRF_REG_14,
    DWRF_REG_15,
    DWRF_REG_RA,
#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__)
    DWRF_REG_SP = 31,
    DWRF_REG_RA = 30,
#else
#    error "Unsupported target architecture"
#endif
};

typedef struct ELFObjectContext
{
    uint8_t* p; /* Pointer to next address in obj.space. */
    uint8_t* startp; /* Pointer to start address in obj.space. */
    uint8_t* eh_frame_p; /* Pointer to start address in obj.space. */
    uint32_t code_size; /* Size of machine code. */
} ELFObjectContext;

/* Append a null-terminated string. */
static uint32_t
elfctx_append_string(ELFObjectContext* ctx, const char* str)
{
    uint8_t* p = ctx->p;
    uint32_t ofs = (uint32_t)(p - ctx->startp);
    do {
        *p++ = (uint8_t)*str;
    } while (*str++);
    ctx->p = p;
    return ofs;
}

/* Append a SLEB128 value. */
static void
elfctx_append_sleb128(ELFObjectContext* ctx, int32_t v)
{
    uint8_t* p = ctx->p;
    for (; (uint32_t)(v + 0x40) >= 0x80; v >>= 7) {
        *p++ = (uint8_t)((v & 0x7f) | 0x80);
    }
    *p++ = (uint8_t)(v & 0x7f);
    ctx->p = p;
}

/* Append a ULEB128 to buffer. */
static void
elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v)
{
    uint8_t* p = ctx->p;
    for (; v >= 0x80; v >>= 7) {
        *p++ = (char)((v & 0x7f) | 0x80);
    }
    *p++ = (char)v;
    ctx->p = p;
}

/* Shortcuts to generate DWARF structures. */
#define DWRF_U8(x) (*p++ = (x))
#define DWRF_I8(x) (*(int8_t*)p = (x), p++)
#define DWRF_U16(x) (*(uint16_t*)p = (x), p += 2)
#define DWRF_U32(x) (*(uint32_t*)p = (x), p += 4)
#define DWRF_ADDR(x) (*(uintptr_t*)p = (x), p += sizeof(uintptr_t))
#define DWRF_UV(x) (ctx->p = p, elfctx_append_uleb128(ctx, (x)), p = ctx->p)
#define DWRF_SV(x) (ctx->p = p, elfctx_append_sleb128(ctx, (x)), p = ctx->p)
#define DWRF_STR(str) (ctx->p = p, elfctx_append_string(ctx, (str)), p = ctx->p)
#define DWRF_ALIGNNOP(s)                                                                                \
    while ((uintptr_t)p & ((s)-1)) {                                                                    \
        *p++ = DWRF_CFA_nop;                                                                            \
    }
#define DWRF_SECTION(name, stmt)                                                                        \
    {                                                                                                   \
        uint32_t* szp_##name = (uint32_t*)p;                                                            \
        p += 4;                                                                                         \
        stmt;                                                                                           \
        *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4);                                       \
    }

/* Initialize .eh_frame section. */
static void
elf_init_ehframe(ELFObjectContext* ctx)
{
    uint8_t* p = ctx->p;
    uint8_t* framep = p;

    /* Emit DWARF EH CIE. */
    DWRF_SECTION(CIE, DWRF_U32(0); /* Offset to CIE itself. */
                 DWRF_U8(DWRF_CIE_VERSION);
                 DWRF_STR("zR"); /* Augmentation. */
                 DWRF_UV(1); /* Code alignment factor. */
                 DWRF_SV(-(int64_t)sizeof(uintptr_t)); /* Data alignment factor. */
                 DWRF_U8(DWRF_REG_RA); /* Return address register. */
                 DWRF_UV(1);
                 DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); /* Augmentation data. */
                 DWRF_U8(DWRF_CFA_def_cfa); DWRF_UV(DWRF_REG_SP); DWRF_UV(sizeof(uintptr_t));
                 DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); DWRF_UV(1);
                 DWRF_ALIGNNOP(sizeof(uintptr_t));
    )

    ctx->eh_frame_p = p;

    /* Emit DWARF EH FDE. */
    DWRF_SECTION(FDE, DWRF_U32((uint32_t)(p - framep)); /* Offset to CIE. */
                 DWRF_U32(-0x30); /* Machine code offset relative to .text. */
                 DWRF_U32(ctx->code_size); /* Machine code length. */
                 DWRF_U8(0); /* Augmentation data. */
    /* Registers saved in CFRAME. */
#ifdef __x86_64__
                 DWRF_U8(DWRF_CFA_advance_loc | 4);
                 DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16);
                 DWRF_U8(DWRF_CFA_advance_loc | 6);
                 DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(8);
    /* Extra registers saved for JIT-compiled code. */
#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__)
                 DWRF_U8(DWRF_CFA_advance_loc | 1);
                 DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16);
                 DWRF_U8(DWRF_CFA_offset | 29); DWRF_UV(2);
                 DWRF_U8(DWRF_CFA_offset | 30); DWRF_UV(1);
                 DWRF_U8(DWRF_CFA_advance_loc | 3);
                 DWRF_U8(DWRF_CFA_offset | -(64 - 29));
                 DWRF_U8(DWRF_CFA_offset | -(64 - 30));
                 DWRF_U8(DWRF_CFA_def_cfa_offset);
                 DWRF_UV(0);
#else
#    error "Unsupported target architecture"
#endif
                 DWRF_ALIGNNOP(sizeof(uintptr_t));)

    ctx->p = p;
}

static void perf_map_jit_write_entry(void *state, const void *code_addr,
                         unsigned int code_size, PyCodeObject *co)
{

    if (perf_jit_map_state.perf_map == NULL) {
        void* ret = perf_map_jit_init();
        if(ret == NULL){
            return;
        }
    }

    const char *entry = "";
    if (co->co_qualname != NULL) {
        entry = PyUnicode_AsUTF8(co->co_qualname);
    }
    const char *filename = "";
    if (co->co_filename != NULL) {
        filename = PyUnicode_AsUTF8(co->co_filename);
    }


    size_t perf_map_entry_size = snprintf(NULL, 0, "py::%s:%s", entry, filename) + 1;
    char* perf_map_entry = (char*) PyMem_RawMalloc(perf_map_entry_size);
    if (perf_map_entry == NULL) {
        return;
    }
    snprintf(perf_map_entry, perf_map_entry_size, "py::%s:%s", entry, filename);

    const size_t name_length = strlen(perf_map_entry);
    uword base = (uword)code_addr;
    uword size = code_size;

    // Write the code unwinding info event.

    // Create unwinding information (eh frame)
    ELFObjectContext ctx;
    char buffer[1024];
    ctx.code_size = code_size;
    ctx.startp = ctx.p = (uint8_t*)buffer;
    elf_init_ehframe(&ctx);
    int eh_frame_size = ctx.p - ctx.startp;

    // Populate the unwind info event for perf
    CodeUnwindingInfoEvent ev2;
    ev2.base.event = PerfUnwindingInfo;
    ev2.base.time_stamp = get_current_monotonic_ticks();
    ev2.unwind_data_size = sizeof(EhFrameHeader) + eh_frame_size;
    // Ensure we have enough space between DSOs when perf maps them
    assert(ev2.unwind_data_size <= PERF_JIT_CODE_PADDING);
    ev2.eh_frame_hdr_size = sizeof(EhFrameHeader);
    ev2.mapped_size = round_up(ev2.unwind_data_size, 16);
    int content_size = sizeof(ev2) + sizeof(EhFrameHeader) + eh_frame_size;
    int padding_size = round_up(content_size, 8) - content_size;
    ev2.base.size = content_size + padding_size;
    perf_map_jit_write_fully(&ev2, sizeof(ev2));


    // Populate the eh Frame header
    EhFrameHeader f;
    f.version = 1;
    f.eh_frame_ptr_enc = DwarfSData4 | DwarfPcRel;
    f.fde_count_enc = DwarfUData4;
    f.table_enc = DwarfSData4 | DwarfDataRel;
    f.eh_frame_ptr = -(eh_frame_size + 4 * sizeof(unsigned char));
    f.eh_fde_count = 1;
    f.from = -(round_up(code_size, 8) + eh_frame_size);
    int cie_size = ctx.eh_frame_p - ctx.startp;
    f.to = -(eh_frame_size - cie_size);

    perf_map_jit_write_fully(ctx.startp, eh_frame_size);
    perf_map_jit_write_fully(&f, sizeof(f));

    char padding_bytes[] = "\0\0\0\0\0\0\0\0";
    perf_map_jit_write_fully(&padding_bytes, padding_size);

    // Write the code load event.
    CodeLoadEvent ev;
    ev.base.event = PerfLoad;
    ev.base.size = sizeof(ev) + (name_length+1) + size;
    ev.base.time_stamp = get_current_monotonic_ticks();
    ev.process_id = getpid();
    ev.thread_id = syscall(SYS_gettid);
    ev.vma = base;
    ev.code_address = base;
    ev.code_size = size;
    perf_jit_map_state.code_id += 1;
    ev.code_id = perf_jit_map_state.code_id;

    perf_map_jit_write_fully(&ev, sizeof(ev));
    perf_map_jit_write_fully(perf_map_entry, name_length+1);
    perf_map_jit_write_fully((void*)(base), size);
    return;
}

static int perf_map_jit_fini(void* state) {
    if (perf_jit_map_state.perf_map != NULL) {
        // close the file
        PyThread_acquire_lock(perf_jit_map_state.map_lock, 1);
        fclose(perf_jit_map_state.perf_map);
        PyThread_release_lock(perf_jit_map_state.map_lock);

        // clean up the lock and state
        PyThread_free_lock(perf_jit_map_state.map_lock);
        perf_jit_map_state.perf_map = NULL;
    }
    if (perf_jit_map_state.mapped_buffer != NULL) {
        munmap(perf_jit_map_state.mapped_buffer, perf_jit_map_state.mapped_size);
    }
    trampoline_api.state = NULL;
    return 0;
}

_PyPerf_Callbacks _Py_perfmap_jit_callbacks = {
    &perf_map_jit_init,
    &perf_map_jit_write_entry,
    &perf_map_jit_fini,
};

#endif