Skip to content
Please note that GitHub no longer supports Internet Explorer.

We recommend upgrading to the latest Microsoft Edge, Google Chrome, or Firefox.

Learn more
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add btf loading support for programs #219

Open
wants to merge 2 commits into
base: master
from
Open

Conversation

@bobrik
Copy link
Contributor

bobrik commented Nov 26, 2019

This applies changes from the following commit in BCC 0.9.0:

It requires a newer LLVM to produce BTF (I use 9.0), but also supports older
versions where it works without producing BTF just like before.

Consider the following code:

struct key {
  u32 prog_id;
  u32 act;
};

// Max. number of XDP programs we expect concurrently
const u8 max_progs = 10;

// Number of valid actions
const int num_actions = XDP_REDIRECT+1;

// Use small LRU hash to avoid losing observations
BPF_F_TABLE("lru_hash", struct key, u64, exceptions, max_progs * num_actions, 0);

// Generates function tracepoint__xdp__xdp_exception
TRACEPOINT_PROBE(xdp, xdp_exception) {
  struct key key = {
    .prog_id = args->prog_id,
    .act = args->act,
  };

  u64 *val = exceptions.lookup(&key);
  if (!val) {
    u64 one = 1;
    // This may fail, but there is no useful action
    // we can take in that case.
    exceptions.insert(&key, &one);
  } else {
    // Not atomic, but that's not a deal breaker.
    *val += 1;
  }
  return 0;
}

Without this change we cannot see any annotations in xlated code:

$ sudo bpftool prog dump xlated id 1849
   0: (61) r2 = *(u32 *)(r1 +8)
   1: (63) *(u32 *)(r10 -8) = r2
   2: (61) r1 = *(u32 *)(r1 +12)
   3: (63) *(u32 *)(r10 -4) = r1
   4: (18) r1 = map[id:4555]
   6: (bf) r2 = r10
   7: (07) r2 += -8
   8: (85) call __htab_map_lookup_elem#100208
   9: (15) if r0 == 0x0 goto pc+4
  10: (71) r1 = *(u8 *)(r0 +35)
  11: (55) if r1 != 0x0 goto pc+1
  12: (72) *(u8 *)(r0 +35) = 1
  13: (07) r0 += 56
  14: (55) if r0 != 0x0 goto pc+11
  15: (b7) r1 = 1
  16: (7b) *(u64 *)(r10 -16) = r1
  17: (18) r1 = map[id:4555]
  19: (bf) r2 = r10
  20: (07) r2 += -8
  21: (bf) r3 = r10
  22: (07) r3 += -16
  23: (b7) r4 = 1
  24: (85) call htab_lru_map_update_elem#104272
  25: (05) goto pc+3
  26: (79) r1 = *(u64 *)(r0 +0)
  27: (07) r1 += 1
  28: (7b) *(u64 *)(r0 +0) = r1
  29: (b7) r0 = 0
  30: (95) exit

With this change we can:

$ sudo bpftool prog dump xlated id 2384
int tracepoint__xdp__xdp_exception(struct tracepoint__xdp__xdp_exception * args):
; .prog_id = args->prog_id,
   0: (61) r2 = *(u32 *)(r1 +8)
; struct key key = {
   1: (63) *(u32 *)(r10 -8) = r2
; .act = args->act,
   2: (61) r1 = *(u32 *)(r1 +12)
; struct key key = {
   3: (63) *(u32 *)(r10 -4) = r1
; u64 *val = bpf_map_lookup_elem((void *)bpf_pseudo_fd(1, -1), &key);
   4: (18) r1 = map[id:7585]
   6: (bf) r2 = r10
;
   7: (07) r2 += -8
; u64 *val = bpf_map_lookup_elem((void *)bpf_pseudo_fd(1, -1), &key);
   8: (85) call __htab_map_lookup_elem#100208
   9: (15) if r0 == 0x0 goto pc+4
  10: (71) r1 = *(u8 *)(r0 +35)
  11: (55) if r1 != 0x0 goto pc+1
  12: (72) *(u8 *)(r0 +35) = 1
  13: (07) r0 += 56
; if (!val) {
  14: (55) if r0 != 0x0 goto pc+11
  15: (b7) r1 = 1
; u64 one = 1;
  16: (7b) *(u64 *)(r10 -16) = r1
; bpf_map_update_elem((void *)bpf_pseudo_fd(1, -1), &key, &one, BPF_NOEXIST);
  17: (18) r1 = map[id:7585]
  19: (bf) r2 = r10
;
  20: (07) r2 += -8
  21: (bf) r3 = r10
  22: (07) r3 += -16
; bpf_map_update_elem((void *)bpf_pseudo_fd(1, -1), &key, &one, BPF_NOEXIST);
  23: (b7) r4 = 1
  24: (85) call htab_lru_map_update_elem#104272
  25: (05) goto pc+3
; *val += 1;
  26: (79) r1 = *(u64 *)(r0 +0)
  27: (07) r1 += 1
  28: (7b) *(u64 *)(r0 +0) = r1
; return 0;
  29: (b7) r0 = 0
  30: (95) exit

Same goes for jited:

$ sudo bpftool prog dump jited id 2384
int tracepoint__xdp__xdp_exception(struct tracepoint__xdp__xdp_exception * args):
0xffffffffc01e84c4:
; .prog_id = args->prog_id,
   0:	push   %rbp
   1:	mov    %rsp,%rbp
   4:	sub    $0x10,%rsp
   b:	push   %rbx
   c:	push   %r13
   e:	push   %r14
  10:	push   %r15
  12:	pushq  $0x0
  14:	mov    0x8(%rdi),%esi
; struct key key = {
  17:	mov    %esi,-0x8(%rbp)
; .act = args->act,
  1a:	mov    0xc(%rdi),%edi
; struct key key = {
  1d:	mov    %edi,-0x4(%rbp)
; u64 *val = bpf_map_lookup_elem((void *)bpf_pseudo_fd(1, -1), &key);
  20:	movabs $0xffff8c4361f44800,%rdi
  2a:	mov    %rbp,%rsi
;
  2d:	add    $0xfffffffffffffff8,%rsi
; u64 *val = bpf_map_lookup_elem((void *)bpf_pseudo_fd(1, -1), &key);
  31:	callq  0xffffffffcb98f8dc
  36:	cmp    $0x0,%rax
  3a:	je     0x000000000000004f
  3c:	movzbq 0x23(%rax),%rdi
  41:	cmp    $0x0,%rdi
  45:	jne    0x000000000000004b
  47:	movb   $0x1,0x23(%rax)
  4b:	add    $0x38,%rax
; if (!val) {
  4f:	cmp    $0x0,%rax
  53:	jne    0x0000000000000082
  55:	mov    $0x1,%edi
; u64 one = 1;
  5a:	mov    %rdi,-0x10(%rbp)
; bpf_map_update_elem((void *)bpf_pseudo_fd(1, -1), &key, &one, BPF_NOEXIST);
  5e:	movabs $0xffff8c4361f44800,%rdi
  68:	mov    %rbp,%rsi
;
  6b:	add    $0xfffffffffffffff8,%rsi
  6f:	mov    %rbp,%rdx
  72:	add    $0xfffffffffffffff0,%rdx
; bpf_map_update_elem((void *)bpf_pseudo_fd(1, -1), &key, &one, BPF_NOEXIST);
  76:	mov    $0x1,%ecx
  7b:	callq  0xffffffffcb9908bc
  80:	jmp    0x000000000000008e
; *val += 1;
  82:	mov    0x0(%rax),%rdi
  86:	add    $0x1,%rdi
  8a:	mov    %rdi,0x0(%rax)
; return 0;
  8e:	xor    %eax,%eax
  90:	pop    %rbx
  91:	pop    %r15
  93:	pop    %r14
  95:	pop    %r13
  97:	pop    %rbx
  98:	leaveq
  99:	retq
This applies changes from the following commit in BCC 0.9.0:

* iovisor/bcc@48ca781

It requires a newer LLVM to produce BTF (I use 9.0), but also supports older
versions where it works without producing BTF just like before.

Consider the following code:

```c
struct key {
  u32 prog_id;
  u32 act;
};

// Max. number of XDP programs we expect concurrently
const u8 max_progs = 10;

// Number of valid actions
const int num_actions = XDP_REDIRECT+1;

// Use small LRU hash to avoid losing observations
BPF_F_TABLE("lru_hash", struct key, u64, exceptions, max_progs * num_actions, 0);

// Generates function tracepoint__xdp__xdp_exception
TRACEPOINT_PROBE(xdp, xdp_exception) {
  struct key key = {
    .prog_id = args->prog_id,
    .act = args->act,
  };

  u64 *val = exceptions.lookup(&key);
  if (!val) {
    u64 one = 1;
    // This may fail, but there is no useful action
    // we can take in that case.
    exceptions.insert(&key, &one);
  } else {
    // Not atomic, but that's not a deal breaker.
    *val += 1;
  }
  return 0;
}
```

Without this change we cannot see any annotations in xlated code:

```
$ sudo bpftool prog dump xlated id 1849
   0: (61) r2 = *(u32 *)(r1 +8)
   1: (63) *(u32 *)(r10 -8) = r2
   2: (61) r1 = *(u32 *)(r1 +12)
   3: (63) *(u32 *)(r10 -4) = r1
   4: (18) r1 = map[id:4555]
   6: (bf) r2 = r10
   7: (07) r2 += -8
   8: (85) call __htab_map_lookup_elem#100208
   9: (15) if r0 == 0x0 goto pc+4
  10: (71) r1 = *(u8 *)(r0 +35)
  11: (55) if r1 != 0x0 goto pc+1
  12: (72) *(u8 *)(r0 +35) = 1
  13: (07) r0 += 56
  14: (55) if r0 != 0x0 goto pc+11
  15: (b7) r1 = 1
  16: (7b) *(u64 *)(r10 -16) = r1
  17: (18) r1 = map[id:4555]
  19: (bf) r2 = r10
  20: (07) r2 += -8
  21: (bf) r3 = r10
  22: (07) r3 += -16
  23: (b7) r4 = 1
  24: (85) call htab_lru_map_update_elem#104272
  25: (05) goto pc+3
  26: (79) r1 = *(u64 *)(r0 +0)
  27: (07) r1 += 1
  28: (7b) *(u64 *)(r0 +0) = r1
  29: (b7) r0 = 0
  30: (95) exit
```

With this change we can:

```
$ sudo bpftool prog dump xlated id 2384
int tracepoint__xdp__xdp_exception(struct tracepoint__xdp__xdp_exception * args):
; .prog_id = args->prog_id,
   0: (61) r2 = *(u32 *)(r1 +8)
; struct key key = {
   1: (63) *(u32 *)(r10 -8) = r2
; .act = args->act,
   2: (61) r1 = *(u32 *)(r1 +12)
; struct key key = {
   3: (63) *(u32 *)(r10 -4) = r1
; u64 *val = bpf_map_lookup_elem((void *)bpf_pseudo_fd(1, -1), &key);
   4: (18) r1 = map[id:7585]
   6: (bf) r2 = r10
;
   7: (07) r2 += -8
; u64 *val = bpf_map_lookup_elem((void *)bpf_pseudo_fd(1, -1), &key);
   8: (85) call __htab_map_lookup_elem#100208
   9: (15) if r0 == 0x0 goto pc+4
  10: (71) r1 = *(u8 *)(r0 +35)
  11: (55) if r1 != 0x0 goto pc+1
  12: (72) *(u8 *)(r0 +35) = 1
  13: (07) r0 += 56
; if (!val) {
  14: (55) if r0 != 0x0 goto pc+11
  15: (b7) r1 = 1
; u64 one = 1;
  16: (7b) *(u64 *)(r10 -16) = r1
; bpf_map_update_elem((void *)bpf_pseudo_fd(1, -1), &key, &one, BPF_NOEXIST);
  17: (18) r1 = map[id:7585]
  19: (bf) r2 = r10
;
  20: (07) r2 += -8
  21: (bf) r3 = r10
  22: (07) r3 += -16
; bpf_map_update_elem((void *)bpf_pseudo_fd(1, -1), &key, &one, BPF_NOEXIST);
  23: (b7) r4 = 1
  24: (85) call htab_lru_map_update_elem#104272
  25: (05) goto pc+3
; *val += 1;
  26: (79) r1 = *(u64 *)(r0 +0)
  27: (07) r1 += 1
  28: (7b) *(u64 *)(r0 +0) = r1
; return 0;
  29: (b7) r0 = 0
  30: (95) exit
```

Same goes for jited:

```
$ sudo bpftool prog dump jited id 2384
int tracepoint__xdp__xdp_exception(struct tracepoint__xdp__xdp_exception * args):
0xffffffffc01e84c4:
; .prog_id = args->prog_id,
   0:	push   %rbp
   1:	mov    %rsp,%rbp
   4:	sub    $0x10,%rsp
   b:	push   %rbx
   c:	push   %r13
   e:	push   %r14
  10:	push   %r15
  12:	pushq  $0x0
  14:	mov    0x8(%rdi),%esi
; struct key key = {
  17:	mov    %esi,-0x8(%rbp)
; .act = args->act,
  1a:	mov    0xc(%rdi),%edi
; struct key key = {
  1d:	mov    %edi,-0x4(%rbp)
; u64 *val = bpf_map_lookup_elem((void *)bpf_pseudo_fd(1, -1), &key);
  20:	movabs $0xffff8c4361f44800,%rdi
  2a:	mov    %rbp,%rsi
;
  2d:	add    $0xfffffffffffffff8,%rsi
; u64 *val = bpf_map_lookup_elem((void *)bpf_pseudo_fd(1, -1), &key);
  31:	callq  0xffffffffcb98f8dc
  36:	cmp    $0x0,%rax
  3a:	je     0x000000000000004f
  3c:	movzbq 0x23(%rax),%rdi
  41:	cmp    $0x0,%rdi
  45:	jne    0x000000000000004b
  47:	movb   $0x1,0x23(%rax)
  4b:	add    $0x38,%rax
; if (!val) {
  4f:	cmp    $0x0,%rax
  53:	jne    0x0000000000000082
  55:	mov    $0x1,%edi
; u64 one = 1;
  5a:	mov    %rdi,-0x10(%rbp)
; bpf_map_update_elem((void *)bpf_pseudo_fd(1, -1), &key, &one, BPF_NOEXIST);
  5e:	movabs $0xffff8c4361f44800,%rdi
  68:	mov    %rbp,%rsi
;
  6b:	add    $0xfffffffffffffff8,%rsi
  6f:	mov    %rbp,%rdx
  72:	add    $0xfffffffffffffff0,%rdx
; bpf_map_update_elem((void *)bpf_pseudo_fd(1, -1), &key, &one, BPF_NOEXIST);
  76:	mov    $0x1,%ecx
  7b:	callq  0xffffffffcb9908bc
  80:	jmp    0x000000000000008e
; *val += 1;
  82:	mov    0x0(%rax),%rdi
  86:	add    $0x1,%rdi
  8a:	mov    %rdi,0x0(%rax)
; return 0;
  8e:	xor    %eax,%eax
  90:	pop    %rbx
  91:	pop    %r15
  93:	pop    %r14
  95:	pop    %r13
  97:	pop    %rbx
  98:	leaveq
  99:	retq
```
@bobrik

This comment has been minimized.

Copy link
Contributor Author

bobrik commented Nov 27, 2019

@iaguis, I'd love to take advantage of this in ebpf_exporter.

@bobrik

This comment has been minimized.

Copy link
Contributor Author

bobrik commented Nov 30, 2019

@schu, do you mind having a look? Thanks!

@bobrik

This comment has been minimized.

Copy link
Contributor Author

bobrik commented Dec 16, 2019

Pushed a commit to make this work on libbcc 0.11.0.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Projects
None yet
1 participant
You can’t perform that action at this time.