Skip to content

Commit

Permalink
bpf: Add BPF_TRAMPOLINE_BATCH_ATTACH support
Browse files Browse the repository at this point in the history 
Adding BPF_TRAMPOLINE_BATCH_ATTACH support, that allows to attach
tracing multiple fentry/fexit pograms to trampolines within one
syscall.

Currently each tracing program is attached in seprate bpf syscall
and more importantly by separate register_ftrace_direct call, which
registers trampoline in ftrace subsystem. We can save some cycles
by simple using its batch variant register_ftrace_direct_ips.

Before:

 Performance counter stats for './src/bpftrace -ve kfunc:__x64_sys_s*
    { printf("test\n"); } i:ms:10 { printf("exit\n"); exit();}' (5 runs):

     2,199,433,771      cycles:k               ( +-  0.55% )
       936,105,469      cycles:u               ( +-  0.37% )

             26.48 +- 3.57 seconds time elapsed  ( +- 13.49% )

After:

 Performance counter stats for './src/bpftrace -ve kfunc:__x64_sys_s*
    { printf("test\n"); } i:ms:10 { printf("exit\n"); exit();}' (5 runs):

     1,456,854,867      cycles:k               ( +-  0.57% )
       937,737,431      cycles:u               ( +-  0.13% )

             12.44 +- 2.98 seconds time elapsed  ( +- 23.95% )

The new BPF_TRAMPOLINE_BATCH_ATTACH syscall command expects
following data in union bpf_attr:

  struct {
          __aligned_u64   in;
          __aligned_u64   out;
          __u32           count;
  } trampoline_batch;

  in    - pointer to user space array with file descrptors of loaded bpf
          programs to attach
  out   - pointer to user space array for resulting link descriptor
  count - number of 'in/out' file descriptors

Basically the new code gets programs from 'in' file descriptors and
attaches them the same way the current code does, apart from the last
step that registers probe ip with trampoline. This is done at the end
with new register_ftrace_direct_ips function.

The resulting link descriptors are written in 'out' array and match
'in' array file descriptors order.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
  • Loading branch information
@olsajiri @intel-lab-lkp
olsajiri authored and intel-lab-lkp committed Oct 22, 2020
1 parent 922b1d2 commit 5ef0150
Show file tree
Hide file tree
Showing 4 changed files with 164 additions and 15 deletions.
15 changes: 13 additions & 2 deletions include/linux/bpf.h
View file
Expand Up @@ -583,6 +583,13 @@ enum bpf_tramp_prog_type {
BPF_TRAMP_REPLACE, /* more than MAX */
};

struct bpf_trampoline_batch {
int count;
int idx;
unsigned long *ips;
unsigned long *addrs;
};

struct bpf_trampoline {
/* hlist for trampoline_table */
struct hlist_node hlist;
Expand Down Expand Up @@ -644,11 +651,14 @@ static __always_inline unsigned int bpf_dispatcher_nop_func(
return bpf_func(ctx, insnsi);
}
#ifdef CONFIG_BPF_JIT
int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr,
struct bpf_trampoline_batch *batch);
int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
struct bpf_trampoline *bpf_trampoline_get(u64 key,
struct bpf_attach_target_info *tgt_info);
void bpf_trampoline_put(struct bpf_trampoline *tr);
struct bpf_trampoline_batch *bpf_trampoline_batch_alloc(int count);
void bpf_trampoline_batch_free(struct bpf_trampoline_batch *batch);
#define BPF_DISPATCHER_INIT(_name) { \
.mutex = __MUTEX_INITIALIZER(_name.mutex), \
.func = &_name##_func, \
Expand Down Expand Up @@ -693,7 +703,8 @@ void bpf_ksym_add(struct bpf_ksym *ksym);
void bpf_ksym_del(struct bpf_ksym *ksym);
#else
static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,
struct bpf_trampoline *tr)
struct bpf_trampoline *tr,
struct bpf_trampoline_batch *batch)
{
return -ENOTSUPP;
}
Expand Down
7 changes: 7 additions & 0 deletions include/uapi/linux/bpf.h
View file
Expand Up @@ -125,6 +125,7 @@ enum bpf_cmd {
BPF_ITER_CREATE,
BPF_LINK_DETACH,
BPF_PROG_BIND_MAP,
BPF_TRAMPOLINE_BATCH_ATTACH,
};

enum bpf_map_type {
Expand Down Expand Up @@ -631,6 +632,12 @@ union bpf_attr {
__u32 prog_fd;
} raw_tracepoint;

struct { /* anonymous struct used by BPF_TRAMPOLINE_BATCH_ATTACH */
__aligned_u64 in;
__aligned_u64 out;
__u32 count;
} trampoline_batch;

struct { /* anonymous struct for BPF_BTF_LOAD */
__aligned_u64 btf;
__aligned_u64 btf_log_buf;
Expand Down
88 changes: 84 additions & 4 deletions kernel/bpf/syscall.c
View file
Expand Up @@ -2553,7 +2553,8 @@ static const struct bpf_link_ops bpf_tracing_link_lops = {

static int bpf_tracing_prog_attach(struct bpf_prog *prog,
int tgt_prog_fd,
u32 btf_id)
u32 btf_id,
struct bpf_trampoline_batch *batch)
{
struct bpf_link_primer link_primer;
struct bpf_prog *tgt_prog = NULL;
Expand Down Expand Up @@ -2678,7 +2679,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
if (err)
goto out_unlock;

err = bpf_trampoline_link_prog(prog, tr);
err = bpf_trampoline_link_prog(prog, tr, batch);
if (err) {
bpf_link_cleanup(&link_primer);
link = NULL;
Expand Down Expand Up @@ -2826,7 +2827,7 @@ static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
tp_name = prog->aux->attach_func_name;
break;
}
return bpf_tracing_prog_attach(prog, 0, 0);
return bpf_tracing_prog_attach(prog, 0, 0, NULL);
case BPF_PROG_TYPE_RAW_TRACEPOINT:
case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
if (strncpy_from_user(buf,
Expand Down Expand Up @@ -2879,6 +2880,81 @@ static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
return err;
}

#define BPF_RAW_TRACEPOINT_OPEN_BATCH_LAST_FIELD trampoline_batch.count

static int bpf_trampoline_batch(const union bpf_attr *attr, int cmd)
{
void __user *uout = u64_to_user_ptr(attr->trampoline_batch.out);
void __user *uin = u64_to_user_ptr(attr->trampoline_batch.in);
struct bpf_trampoline_batch *batch = NULL;
struct bpf_prog *prog;
int count, ret, i, fd;
u32 *in, *out;

if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN_BATCH))
return -EINVAL;

if (!uin || !uout)
return -EINVAL;

count = attr->trampoline_batch.count;

in = kcalloc(count, sizeof(u32), GFP_KERNEL);
out = kcalloc(count, sizeof(u32), GFP_KERNEL);
if (!in || !out) {
kfree(in);
kfree(out);
return -ENOMEM;
}

ret = copy_from_user(in, uin, count * sizeof(u32));
if (ret)
goto out_clean;

/* test read out array */
ret = copy_to_user(uout, out, count * sizeof(u32));
if (ret)
goto out_clean;

batch = bpf_trampoline_batch_alloc(count);
if (!batch)
goto out_clean;

for (i = 0; i < count; i++) {
if (cmd == BPF_TRAMPOLINE_BATCH_ATTACH) {
prog = bpf_prog_get(in[i]);
if (IS_ERR(prog)) {
ret = PTR_ERR(prog);
goto out_clean;
}

ret = -EINVAL;
if (prog->type != BPF_PROG_TYPE_TRACING)
goto out_clean;
if (prog->type == BPF_PROG_TYPE_TRACING &&
prog->expected_attach_type == BPF_TRACE_RAW_TP)
goto out_clean;

fd = bpf_tracing_prog_attach(prog, 0, 0, batch);
if (fd < 0)
goto out_clean;

out[i] = fd;
}
}

ret = register_ftrace_direct_ips(batch->ips, batch->addrs, batch->idx);
if (!ret)
WARN_ON_ONCE(copy_to_user(uout, out, count * sizeof(u32)));

out_clean:
/* XXX cleanup partialy attached array */
bpf_trampoline_batch_free(batch);
kfree(in);
kfree(out);
return ret;
}

static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,
enum bpf_attach_type attach_type)
{
Expand Down Expand Up @@ -4018,7 +4094,8 @@ static int tracing_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *
else if (prog->type == BPF_PROG_TYPE_EXT)
return bpf_tracing_prog_attach(prog,
attr->link_create.target_fd,
attr->link_create.target_btf_id);
attr->link_create.target_btf_id,
NULL);
return -EINVAL;
}

Expand Down Expand Up @@ -4437,6 +4514,9 @@ SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, siz
case BPF_RAW_TRACEPOINT_OPEN:
err = bpf_raw_tracepoint_open(&attr);
break;
case BPF_TRAMPOLINE_BATCH_ATTACH:
err = bpf_trampoline_batch(&attr, cmd);
break;
case BPF_BTF_LOAD:
err = bpf_btf_load(&attr);
break;
Expand Down
69 changes: 60 additions & 9 deletions kernel/bpf/trampoline.c
View file
Expand Up @@ -107,6 +107,51 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key)
return tr;
}

static int bpf_trampoline_batch_add(struct bpf_trampoline_batch *batch,
unsigned long ip, unsigned long addr)
{
int idx = batch->idx;

if (idx >= batch->count)
return -EINVAL;

batch->ips[idx] = ip;
batch->addrs[idx] = addr;
batch->idx++;
return 0;
}

struct bpf_trampoline_batch *bpf_trampoline_batch_alloc(int count)
{
struct bpf_trampoline_batch *batch;

batch = kmalloc(sizeof(*batch), GFP_KERNEL);
if (!batch)
return NULL;

batch->ips = kcalloc(count, sizeof(batch->ips[0]), GFP_KERNEL);
batch->addrs = kcalloc(count, sizeof(batch->addrs[0]), GFP_KERNEL);
if (!batch->ips || !batch->addrs) {
kfree(batch->ips);
kfree(batch->addrs);
kfree(batch);
return NULL;
}

batch->count = count;
batch->idx = 0;
return batch;
}

void bpf_trampoline_batch_free(struct bpf_trampoline_batch *batch)
{
if (!batch)
return;
kfree(batch->ips);
kfree(batch->addrs);
kfree(batch);
}

static int is_ftrace_location(void *ip)
{
long addr;
Expand Down Expand Up @@ -144,7 +189,8 @@ static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_ad
}

/* first time registering */
static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
static int register_fentry(struct bpf_trampoline *tr, void *new_addr,
struct bpf_trampoline_batch *batch)
{
void *ip = tr->func.addr;
int ret;
Expand All @@ -154,9 +200,12 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
return ret;
tr->func.ftrace_managed = ret;

if (tr->func.ftrace_managed)
ret = register_ftrace_direct((long)ip, (long)new_addr);
else
if (tr->func.ftrace_managed) {
if (batch)
ret = bpf_trampoline_batch_add(batch, (long)ip, (long)new_addr);
else
ret = register_ftrace_direct((long)ip, (long)new_addr);
} else
ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, NULL, new_addr);
return ret;
}
Expand Down Expand Up @@ -185,7 +234,8 @@ bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total)
return tprogs;
}

static int bpf_trampoline_update(struct bpf_trampoline *tr)
static int bpf_trampoline_update(struct bpf_trampoline *tr,
struct bpf_trampoline_batch *batch)
{
void *old_image = tr->image + ((tr->selector + 1) & 1) * PAGE_SIZE/2;
void *new_image = tr->image + (tr->selector & 1) * PAGE_SIZE/2;
Expand Down Expand Up @@ -230,7 +280,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr)
err = modify_fentry(tr, old_image, new_image);
else
/* first time registering */
err = register_fentry(tr, new_image);
err = register_fentry(tr, new_image, batch);
if (err)
goto out;
tr->selector++;
Expand Down Expand Up @@ -261,7 +311,8 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)
}
}

int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr,
struct bpf_trampoline_batch *batch)
{
enum bpf_tramp_prog_type kind;
int err = 0;
Expand Down Expand Up @@ -299,7 +350,7 @@ int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
}
hlist_add_head(&prog->aux->tramp_hlist, &tr->progs_hlist[kind]);
tr->progs_cnt[kind]++;
err = bpf_trampoline_update(tr);
err = bpf_trampoline_update(tr, batch);
if (err) {
hlist_del(&prog->aux->tramp_hlist);
tr->progs_cnt[kind]--;
Expand All @@ -326,7 +377,7 @@ int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
}
hlist_del(&prog->aux->tramp_hlist);
tr->progs_cnt[kind]--;
err = bpf_trampoline_update(tr);
err = bpf_trampoline_update(tr, NULL);
out:
mutex_unlock(&tr->mutex);
return err;
Expand Down

0 comments on commit 5ef0150

Please sign in to comment.