eBPF-based TLS connections tracing for Golang applications

Dockerfile

@@ -1,5 +1,5 @@
 FROM golang:1.19-bullseye AS builder
-RUN apt-get update && apt-get install -y libsystemd-dev
+RUN apt update && apt install -y libsystemd-dev
 COPY go.mod /tmp/src/
 COPY go.sum /tmp/src/
 WORKDIR /tmp/src/
@@ -10,6 +10,6 @@ ARG VERSION=unknown
 RUN CGO_ENABLED=1 go install -mod=readonly -ldflags "-X main.version=$VERSION" /tmp/src
 
 FROM debian:bullseye
-RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/*
+RUN apt update && apt install -y ca-certificates && apt clean
 COPY --from=builder /go/bin/coroot-node-agent /usr/bin/coroot-node-agent
 ENTRYPOINT ["coroot-node-agent"]

containers/container.go

@@ -1,6 +1,7 @@
 package containers
 
 import (
+	"github.com/cilium/ebpf/link"
 	"github.com/coroot/coroot-node-agent/cgroup"
 	"github.com/coroot/coroot-node-agent/common"
 	"github.com/coroot/coroot-node-agent/ebpftracer"
@@ -89,6 +90,7 @@ type Process struct {
 	Pid       uint32
 	StartedAt time.Time
 	NetNsId   string
+	uprobes   []link.Link
 }
 
 func (p *Process) isHostNs() bool {
@@ -356,7 +358,7 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) {
 	}
 }
 
-func (c *Container) onProcessStart(pid uint32) {
+func (c *Container) onProcessStart(pid uint32, uprobes []link.Link) {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 	stats, err := TaskstatsPID(pid)
@@ -369,7 +371,7 @@ func (c *Container) onProcessStart(pid uint32) {
 	}
 	defer ns.Close()
 	c.zombieAt = time.Time{}
-	c.processes[pid] = &Process{Pid: pid, StartedAt: stats.BeginTime, NetNsId: ns.UniqueId()}
+	c.processes[pid] = &Process{Pid: pid, StartedAt: stats.BeginTime, NetNsId: ns.UniqueId(), uprobes: uprobes}
 
 	if c.startedAt.IsZero() {
 		c.startedAt = stats.BeginTime
@@ -390,6 +392,11 @@ func (c *Container) onProcessStart(pid uint32) {
 func (c *Container) onProcessExit(pid uint32, oomKill bool) {
 	c.lock.Lock()
 	defer c.lock.Unlock()
+	if p := c.processes[pid]; p != nil {
+		for _, u := range p.uprobes {
+			_ = u.Close()
+		}
+	}
 	delete(c.processes, pid)
 	if len(c.processes) == 0 {
 		c.zombieAt = time.Now()

containers/registry.go

@@ -79,7 +79,7 @@ func NewRegistry(reg prometheus.Registerer, kernelVersion string) (*Registry, er
 		return nil, err
 	}
 
-	cs := &Registry{
+	r := &Registry{
 		reg:    reg,
 		events: make(chan ebpftracer.Event, 10000),
 
@@ -88,17 +88,17 @@ func NewRegistry(reg prometheus.Registerer, kernelVersion string) (*Registry, er
 		containersById:       map[ContainerID]*Container{},
 		containersByCgroupId: map[string]*Container{},
 		containersByPid:      map[uint32]*Container{},
+
+		tracer: ebpftracer.NewTracer(kernelVersion, *flags.DisableL7Tracing),
 	}
 
-	go cs.handleEvents(cs.events)
-	t, err := ebpftracer.NewTracer(cs.events, kernelVersion, *flags.DisableL7Tracing)
-	if err != nil {
-		close(cs.events)
+	go r.handleEvents(r.events)
+	if err = r.tracer.Run(r.events); err != nil {
+		close(r.events)
 		return nil, err
 	}
-	cs.tracer = t
 
-	return cs, nil
+	return r, nil
 }
 
 func (r *Registry) Close() {
@@ -167,7 +167,8 @@ func (r *Registry) handleEvents(ch <-chan ebpftracer.Event) {
 					}
 				}
 				if c := r.getOrCreateContainer(e.Pid); c != nil {
-					c.onProcessStart(e.Pid)
+					uprobes := r.tracer.AttachGoTlsUprobes(e.Pid)
+					c.onProcessStart(e.Pid, uprobes)
 				}
 			case ebpftracer.EventTypeProcessExit:
 				if c := r.containersByPid[e.Pid]; c != nil {

ebpftracer/Dockerfile

@@ -5,14 +5,26 @@ RUN apk add llvm clang libbpf-dev linux-headers
 COPY ebpf /tmp/ebpf
 WORKDIR /tmp/ebpf
 
-RUN clang -g -O2 -target bpf -D__KERNEL=416 -c ebpf.c -o ebpf416.o && llvm-strip --strip-debug ebpf416.o
-RUN clang -g -O2 -target bpf -D__KERNEL=420 -c ebpf.c -o ebpf420.o && llvm-strip --strip-debug ebpf420.o
-RUN clang -g -O2 -target bpf -D__KERNEL=506 -c ebpf.c -o ebpf506.o && llvm-strip --strip-debug ebpf506.o
-RUN clang -g -O2 -target bpf -D__KERNEL=512 -c ebpf.c -o ebpf512.o && llvm-strip --strip-debug ebpf512.o
+RUN clang -g -O2 -target bpf -D__KERNEL_FROM=416 -D__TARGET_ARCH_x86 -c ebpf.c -o ebpf416x86.o && llvm-strip --strip-debug ebpf416x86.o
+RUN clang -g -O2 -target bpf -D__KERNEL_FROM=420 -D__TARGET_ARCH_x86 -c ebpf.c -o ebpf420x86.o && llvm-strip --strip-debug ebpf420x86.o
+RUN clang -g -O2 -target bpf -D__KERNEL_FROM=506 -D__TARGET_ARCH_x86 -c ebpf.c -o ebpf506x86.o && llvm-strip --strip-debug ebpf506x86.o
+RUN clang -g -O2 -target bpf -D__KERNEL_FROM=512 -D__TARGET_ARCH_x86 -c ebpf.c -o ebpf512x86.o && llvm-strip --strip-debug ebpf512x86.o
+RUN clang -g -O2 -target bpf -D__KERNEL_FROM=416 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf416arm64.o && llvm-strip --strip-debug ebpf416arm64.o
+RUN clang -g -O2 -target bpf -D__KERNEL_FROM=420 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf420arm64.o && llvm-strip --strip-debug ebpf420arm64.o
+RUN clang -g -O2 -target bpf -D__KERNEL_FROM=506 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf506arm64.o && llvm-strip --strip-debug ebpf506arm64.o
+RUN clang -g -O2 -target bpf -D__KERNEL_FROM=512 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf512arm64.o && llvm-strip --strip-debug ebpf512arm64.o
 
-RUN echo -en '// generated - do not edit\npackage ebpftracer\nvar ebpfProg = []struct{v string; p []byte}{\n' > ebpf.go \
-	&& echo -en '\t{"v5.12", []byte("' >> ebpf.go && hexdump -v -e '"\x" 1/1 "%02x"' ebpf512.o >> ebpf.go && echo '")},' >> ebpf.go \
-	&& echo -en '\t{"v5.6", []byte("' >> ebpf.go && hexdump -v -e '"\x" 1/1 "%02x"' ebpf506.o >> ebpf.go && echo '")},' >> ebpf.go \
-	&& echo -en '\t{"v4.20", []byte("' >> ebpf.go && hexdump -v -e '"\x" 1/1 "%02x"' ebpf420.o >> ebpf.go && echo '")},' >> ebpf.go \
-	&& echo -en '\t{"v4.16", []byte("' >> ebpf.go && hexdump -v -e '"\x" 1/1 "%02x"' ebpf416.o >> ebpf.go && echo '")},' >> ebpf.go \
+RUN echo -en '// generated - do not edit\npackage ebpftracer\nvar ebpfProg = map[string][]struct{v string; p []byte}{\n' > ebpf.go \
+	&& echo -en '\t"amd64": {\n' >> ebpf.go \
+	&& echo -en '\t\t{"v5.12", []byte("' >> ebpf.go && hexdump -v -e '"\x" 1/1 "%02x"' ebpf512x86.o >> ebpf.go && echo '")},' >> ebpf.go \
+	&& echo -en '\t\t{"v5.6", []byte("' >> ebpf.go && hexdump -v -e '"\x" 1/1 "%02x"' ebpf506x86.o >> ebpf.go && echo '")},' >> ebpf.go \
+	&& echo -en '\t\t{"v4.20", []byte("' >> ebpf.go && hexdump -v -e '"\x" 1/1 "%02x"' ebpf420x86.o >> ebpf.go && echo '")},' >> ebpf.go \
+	&& echo -en '\t\t{"v4.16", []byte("' >> ebpf.go && hexdump -v -e '"\x" 1/1 "%02x"' ebpf416x86.o >> ebpf.go && echo '")},' >> ebpf.go \
+	&& echo -en '\t},\n'>> ebpf.go \
+	&& echo -en '\t"arm64": {\n' >> ebpf.go \
+	&& echo -en '\t\t{"v5.12", []byte("' >> ebpf.go && hexdump -v -e '"\x" 1/1 "%02x"' ebpf512arm64.o >> ebpf.go && echo '")},' >> ebpf.go \
+	&& echo -en '\t\t{"v5.6", []byte("' >> ebpf.go && hexdump -v -e '"\x" 1/1 "%02x"' ebpf506arm64.o >> ebpf.go && echo '")},' >> ebpf.go \
+	&& echo -en '\t\t{"v4.20", []byte("' >> ebpf.go && hexdump -v -e '"\x" 1/1 "%02x"' ebpf420arm64.o >> ebpf.go && echo '")},' >> ebpf.go \
+	&& echo -en '\t\t{"v4.16", []byte("' >> ebpf.go && hexdump -v -e '"\x" 1/1 "%02x"' ebpf416arm64.o >> ebpf.go && echo '")},' >> ebpf.go \
+	&& echo -en '\t},\n'>> ebpf.go \
 	&& echo -en '}\n'>> ebpf.go

ebpftracer/ebpf/ebpf.c

@@ -1,4 +1,5 @@
 #include <uapi/linux/bpf.h>
+#include "vmlinux.h"
 #include <bpf/bpf_helpers.h>
 #include <bpf/bpf_core_read.h>
 #include <bpf/bpf_tracing.h>
@@ -29,11 +30,11 @@
     bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \
 })
 
-
 #include "proc.c"
 #include "file.c"
 #include "tcp/state.c"
 #include "tcp/retransmit.c"
 #include "l7/l7.c"
+#include "l7/tls.c"
 
 char _license[] SEC("license") = "GPL";

ebpftracer/ebpf/file.c

@@ -75,6 +75,7 @@ int trace_exit(struct trace_event_raw_sys_exit__stub* ctx)
 	return 0;
 }
 
+#if defined(__TARGET_ARCH_x86)
 SEC("tracepoint/syscalls/sys_enter_open")
 int sys_enter_open(struct trace_event_raw_sys_enter__stub* ctx)
 {
@@ -86,6 +87,7 @@ int sys_exit_open(struct trace_event_raw_sys_exit__stub* ctx)
 {
 	return trace_exit(ctx);
 }
+#endif
 
 SEC("tracepoint/syscalls/sys_enter_openat")
 int sys_enter_openat(struct trace_event_raw_sys_enter__stub* ctx)