Merge pull request #313 from bobrik/ivan/kstack

Add kstack decoder

.vscode/config-schema.yaml

@@ -94,7 +94,22 @@ definitions:
               - name
             properties:
               name:
-                type: string
+                enum:
+                  - cgroup
+                  - dname
+                  - inet_ip
+                  - kstack
+                  - ksym
+                  - majorminor
+                  - pci_class
+                  - pci_device
+                  - pci_subclass
+                  - pci_vendor
+                  - regexp
+                  - static_map
+                  - string
+                  - syscall
+                  - uint
               static_map:
                 type: object
               allow_unknown:

decoder/decoder.go

@@ -6,6 +6,7 @@ import (
 	"sync"
 
 	"github.com/cloudflare/ebpf_exporter/v2/config"
+	"github.com/cloudflare/ebpf_exporter/v2/kallsyms"
 )
 
 // ErrSkipLabelSet instructs exporter to skip label set
@@ -31,22 +32,28 @@ func NewSet() (*Set, error) {
 		return nil, fmt.Errorf("error creating cgroup decoder: %v", err)
 	}
 
+	ksym, err := kallsyms.NewDecoder("/proc/kallsyms")
+	if err != nil {
+		return nil, fmt.Errorf("error creating ksym decoder: %v", err)
+	}
+
 	return &Set{
 		decoders: map[string]Decoder{
 			"cgroup":       cgroup,
-			"ksym":         &KSym{},
-			"majorminor":   &MajorMinor{},
-			"regexp":       &Regexp{},
-			"static_map":   &StaticMap{},
-			"string":       &String{},
 			"dname":        &Dname{},
-			"uint":         &UInt{},
 			"inet_ip":      &InetIP{},
-			"pci_vendor":   &PCIVendor{},
-			"pci_device":   &PCIDevice{},
+			"kstack":       &KStack{ksym},
+			"ksym":         &KSym{ksym},
+			"majorminor":   &MajorMinor{},
 			"pci_class":    &PCIClass{},
+			"pci_device":   &PCIDevice{},
 			"pci_subclass": &PCISubClass{},
+			"pci_vendor":   &PCIVendor{},
+			"regexp":       &Regexp{},
+			"static_map":   &StaticMap{},
+			"string":       &String{},
 			"syscall":      &Syscall{},
+			"uint":         &UInt{},
 		},
 	}, nil
 }

decoder/kstack.go

@@ -0,0 +1,38 @@
+package decoder
+
+import (
+	"strings"
+
+	"github.com/cloudflare/ebpf_exporter/v2/config"
+	"github.com/cloudflare/ebpf_exporter/v2/kallsyms"
+	"github.com/cloudflare/ebpf_exporter/v2/util"
+)
+
+// KStack is a decoder that transforms an array of kernel frame addresses to a newline separated stack of symbols
+type KStack struct {
+	decoder *kallsyms.Decoder
+}
+
+// Decode transforms an array of kernel frame addresses to a newline separated stack of symbols
+func (k *KStack) Decode(in []byte, _ config.Decoder) ([]byte, error) {
+	addrs := []uintptr{}
+	for off := 0; off < len(in); off += 8 {
+		ptr := util.GetHostByteOrder().Uint64(in[off : off+8])
+		if ptr == 0 {
+			break
+		}
+
+		addrs = append(addrs, uintptr(ptr))
+	}
+
+	stack := make([]string, len(addrs))
+	for i, frame := range k.decoder.Stack(addrs) {
+		if frame.Sym == "" {
+			stack[i] = "??"
+		} else {
+			stack[i] = frame.Sym
+		}
+	}
+
+	return []byte(strings.Join(stack, "\n")), nil
+}

decoder/kstack_test.go

@@ -0,0 +1,95 @@
+package decoder
+
+import (
+	"bytes"
+	"os"
+	"testing"
+
+	"github.com/cloudflare/ebpf_exporter/v2/config"
+	"github.com/cloudflare/ebpf_exporter/v2/kallsyms"
+)
+
+func TestKStackDecoder(t *testing.T) {
+	cases := []struct {
+		in  []byte
+		out []byte
+	}{
+		{
+			in: []byte{
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			},
+			out: []byte(""),
+		},
+		{
+			in: []byte{
+				0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			},
+			out: []byte("one\none\ntwo"),
+		},
+		{
+			in: []byte{
+				0xab, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			},
+			out: []byte("three\ntwo\ntwo"),
+		},
+		{
+			in: []byte{
+				0xab, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			},
+			out: []byte("three\nzero"),
+		},
+		{
+			in: []byte{
+				0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0xac, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			},
+			out: []byte("??\nthree\n??"),
+		},
+	}
+
+	fd, err := os.CreateTemp("", "kallsyms")
+	if err != nil {
+		t.Fatalf("Error creating temporary file for kallsyms: %v", err)
+	}
+
+	defer os.Remove(fd.Name())
+
+	_, err = fd.WriteString("0000000000000004 T one\n0000000000000006 T two\n00000000000000aa T three\n")
+	if err != nil {
+		t.Fatalf("Error writing fake kallsyms data to %q: %v", fd.Name(), err)
+	}
+
+	decoder, err := kallsyms.NewDecoder(fd.Name())
+	if err != nil {
+		t.Fatalf("Error creating ksym decoder for %q: %v", fd.Name(), err)
+	}
+
+	_, err = fd.WriteString("0000000000000002 T zero\n")
+	if err != nil {
+		t.Fatalf("Error writing additional fake kallsyms data to %q: %v", fd.Name(), err)
+	}
+
+	d := KStack{decoder}
+
+	for _, c := range cases {
+		out, err := d.Decode(c.in, config.Decoder{})
+		if err != nil {
+			t.Errorf("Error decoding %#v: %s", c.in, err)
+		}
+
+		if !bytes.Equal(out, c.out) {
+			t.Errorf("Expected %q, got %q", c.out, out)
+		}
+	}
+}

decoder/ksym.go

@@ -4,31 +4,23 @@ import (
 	"fmt"
 
 	"github.com/cloudflare/ebpf_exporter/v2/config"
+	"github.com/cloudflare/ebpf_exporter/v2/kallsyms"
 	"github.com/cloudflare/ebpf_exporter/v2/util"
-	"github.com/iovisor/gobpf/pkg/ksym"
 )
 
 // KSym is a decoder that transforms kernel address to a function name
 type KSym struct {
-	cache map[string][]byte
+	decoder *kallsyms.Decoder
 }
 
 // Decode transforms kernel address to a function name
 func (k *KSym) Decode(in []byte, _ config.Decoder) ([]byte, error) {
-	if k.cache == nil {
-		k.cache = map[string][]byte{}
-	}
-
-	addr := fmt.Sprintf("%x", util.GetHostByteOrder().Uint64(in))
-
-	if _, ok := k.cache[addr]; !ok {
-		name, err := ksym.Ksym(addr)
-		if err != nil {
-			return []byte(fmt.Sprintf("unknown_addr:0x%s", addr)), nil
-		}
+	ptr := util.GetHostByteOrder().Uint64(in)
 
-		k.cache[addr] = []byte(name)
+	sym := k.decoder.Sym(uintptr(ptr))
+	if sym == "" {
+		sym = fmt.Sprintf("unknown_addr:0x%x", ptr)
 	}
 
-	return k.cache[addr], nil
+	return []byte(sym), nil
 }

decoder/ksym_test.go

@@ -2,39 +2,63 @@ package decoder
 
 import (
 	"bytes"
+	"os"
 	"testing"
 
 	"github.com/cloudflare/ebpf_exporter/v2/config"
+	"github.com/cloudflare/ebpf_exporter/v2/kallsyms"
 )
 
 func TestKsymDecoder(t *testing.T) {
 	cases := []struct {
-		in    []byte
-		cache map[string][]byte
-		out   []byte
+		in  []byte
+		out []byte
 	}{
 		{
-			in:    []byte{0x6, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
-			cache: map[string][]byte{"6": []byte("call_six")},
-			out:   []byte("call_six"),
+			in:  []byte{0x28, 0x08, 0xd9, 0x19, 0xeb, 0xff, 0xff, 0xff},
+			out: []byte("unknown_addr:0xffffffeb19d90828"),
 		},
 		{
-			in:    []byte{0x6, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
-			cache: map[string][]byte{"7": []byte("call_seven")},
-			out:   []byte("unknown_addr:0x6"),
+			in:  []byte{0x30, 0x08, 0xd9, 0x19, 0xeb, 0xff, 0xff, 0xff},
+			out: []byte("pipe_lock"),
+		},
+		{
+			in:  []byte{0x70, 0x08, 0xd9, 0x19, 0xeb, 0xff, 0xff, 0xff},
+			out: []byte("pipe_unlock"),
+		},
+		{
+			in:  []byte{0x78, 0x08, 0xd9, 0x19, 0xeb, 0xff, 0xff, 0xff},
+			out: []byte("unknown_addr:0xffffffeb19d90878"),
 		},
 	}
 
-	for _, c := range cases {
-		d := &KSym{cache: c.cache}
+	fd, err := os.CreateTemp("", "kallsyms")
+	if err != nil {
+		t.Fatalf("Error creating temporary file for kallsyms: %v", err)
+	}
+
+	defer os.Remove(fd.Name())
 
+	_, err = fd.WriteString("ffffffeb19d90830 T pipe_lock\nffffffeb19d90870 T pipe_unlock\n")
+	if err != nil {
+		t.Fatalf("Error writing fake kallsyms data to %q: %v", fd.Name(), err)
+	}
+
+	decoder, err := kallsyms.NewDecoder(fd.Name())
+	if err != nil {
+		t.Fatalf("Error creating ksym decoder for %q: %v", fd.Name(), err)
+	}
+
+	d := KSym{decoder}
+
+	for _, c := range cases {
 		out, err := d.Decode(c.in, config.Decoder{})
 		if err != nil {
-			t.Errorf("Error decoding %#v with cache set to %#v: %s", c.in, c.cache, err)
+			t.Errorf("Error decoding %#v: %s", c.in, err)
 		}
 
 		if !bytes.Equal(out, c.out) {
-			t.Errorf("Expected %s, got %s", c.out, out)
+			t.Errorf("Expected %q, got %q", c.out, out)
 		}
 	}
 }

examples/kstack.bpf.c

@@ -0,0 +1,35 @@
+#include <vmlinux.h>
+#include <bpf/bpf_tracing.h>
+#include "maps.bpf.h"
+
+#define MAX_STACK_DEPTH 20
+
+// Skipping 3 frames off the top as they are just bpf trampoline
+#define SKIP_FRAMES (3 & BPF_F_SKIP_FIELD_MASK)
+
+struct key_t {
+    u64 kstack[MAX_STACK_DEPTH];
+};
+
+struct {
+    __uint(type, BPF_MAP_TYPE_LRU_HASH);
+    __uint(max_entries, 128);
+    __type(key, struct key_t);
+    __type(value, u64);
+} mark_page_accessed_total SEC(".maps");
+
+SEC("fentry/mark_page_accessed")
+int mark_page_accessed(struct pt_regs *ctx)
+{
+    struct key_t key = {};
+
+    if (bpf_get_stack(ctx, &key.kstack, sizeof(key.kstack), SKIP_FRAMES) < 0) {
+        return 0;
+    }
+
+    increment_map(&mark_page_accessed_total, &key, 1);
+
+    return 0;
+}
+
+char LICENSE[] SEC("license") = "GPL";

examples/kstack.yaml

@@ -0,0 +1,9 @@
+metrics:
+  counters:
+    - name: mark_page_accessed_total
+      help: Total number of calls to mark_page_accessed by kstack
+      labels:
+        - name: kstack
+          size: 160
+          decoders:
+            - name: kstack