Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: Initial attempt for eh_frame based stack unwinding in eBPF #150

Closed
wants to merge 6 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,6 @@ Dockerfile
/images
/.github
/tmp
/local_workspace
/.git
/pkg/agent/*.bpf.o
9 changes: 8 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,12 @@
/tmp
/out
/bin
pkg/agent/parca-agent.bpf.o
pkg/agent/*.bpf.o
TODO.md
local_workspace
NOTES.md
minikube.iso
minikube-*.iso
/*.txt
.kernel_config
/*.service
1 change: 1 addition & 0 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ run:
skip-dirs:
- internal/pprof
- internal/go
- pkg/stack/frame
linters-settings:
errcheck:
exclude: ./.errcheck_excludes.txt
Expand Down
3 changes: 2 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ WORKDIR /parca-agent
COPY go.mod go.sum /parca-agent/
RUN go mod download -modcacherw

COPY parca-agent.bpf.c vmlinux.h Makefile /parca-agent/
COPY Makefile /parca-agent/
COPY ./bpf /parca-agent/bpf
COPY ./3rdparty /parca-agent/3rdparty
RUN make bpf

Expand Down
12 changes: 8 additions & 4 deletions Dockerfile.dev
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,19 @@ WORKDIR /parca-agent
COPY go.mod go.sum /parca-agent/
RUN go mod download -modcacherw

COPY parca-agent.bpf.c vmlinux.h Makefile /parca-agent/
COPY Makefile /parca-agent/
COPY ./bpf /parca-agent/bpf
COPY ./3rdparty /parca-agent/3rdparty
RUN make bpf
RUN make clean bpf

COPY . /parca-agent
RUN make build

# TODO(kakkoyun): Add Delve
RUN go install github.com/go-delve/delve/cmd/dlv@v1.7.2

RUN cp /go/bin/dlv /bin/dlv
RUN cp /parca-agent/dist/parca-agent /bin/parca-agent

CMD ["/bin/parca-agent"]
EXPOSE 7071

ENTRYPOINT ["/bin/dlv", "--listen=:40000", "--headless=true", "--api-version=2", "--accept-multiclient", "exec", "--continue", "--"]
21 changes: 11 additions & 10 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
.PHONY: all
all: bpf build
all: clean bpf build

# tools:
CMD_LLC ?= llc
Expand Down Expand Up @@ -29,9 +29,10 @@ OUT_DIR ?= dist
GO_SRC := $(shell find . -type f -name '*.go')
OUT_BIN := $(OUT_DIR)/parca-agent
OUT_BIN_DEBUG_INFO := $(OUT_DIR)/debug-info
BPF_SRC := parca-agent.bpf.c
VMLINUX := vmlinux.h
OUT_BPF := pkg/agent/parca-agent.bpf.o
BPF_ROOT := bpf
BPF_SRC := $(BPF_ROOT)/*.bpf.c
VMLINUX := $(BPF_ROOT)/vmlinux.h
BPF_OBJECTS := cpu_profiler cpu_profiler_with_unwinding
BPF_HEADERS := 3rdparty/include
BPF_BUNDLE := $(OUT_DIR)/parca-agent.bpf.tar.gz
LIBBPF_SRC := 3rdparty/libbpf/src
Expand Down Expand Up @@ -98,11 +99,11 @@ $(BPF_BUNDLE): $(BPF_SRC) $(LIBBPF_HEADERS)/bpf $(BPF_HEADERS)
cp $$(find $^ -type f) $(bpf_bundle_dir)

.PHONY: bpf
bpf: $(OUT_BPF)
bpf: $(BPF_OBJECTS)

linux_arch := $(ARCH:x86_64=x86)
ifndef DOCKER
$(OUT_BPF): $(BPF_SRC) $(LIBBPF_HEADERS) $(LIBBPF_OBJ) | $(OUT_DIR) $(bpf_compile_tools)
$(BPF_OBJECTS): %: bpf/%.bpf.c $(LIBBPF_HEADERS) $(LIBBPF_OBJ) | $(OUT_DIR) $(bpf_compile_tools)
mkdir -p pkg/agent
@v=$$($(CMD_CLANG) --version); test $$(echo $${v#*version} | head -n1 | cut -d '.' -f1) -ge '9' || (echo 'required minimum clang version: 9' ; false)
$(CMD_CLANG) -S \
Expand Down Expand Up @@ -130,7 +131,7 @@ $(OUT_BPF): $(BPF_SRC) $(LIBBPF_HEADERS) $(LIBBPF_OBJ) | $(OUT_DIR) $(bpf_compil
-nostdinc \
-target bpf \
-O2 -emit-llvm -c -g $< -o $(@:.o=.ll)
$(CMD_LLC) -march=bpf -filetype=obj -o $@ $(@:.o=.ll)
$(CMD_LLC) -march=bpf -filetype=obj -o pkg/agent/$@.bpf.o $(@:.o=.ll)
rm $(@:.o=.ll)
else
$(OUT_BPF): $(DOCKER_BUILDER) | $(OUT_DIR)
Expand Down Expand Up @@ -180,7 +181,7 @@ mostlyclean:

.PHONY: clean
clean:
rm pkg/agent/parca-agent.bpf.o
-rm pkg/agent/*.bpf.o
-FILE="$(docker_builder_file)" ; \
if [ -r "$$FILE" ] ; then \
$(CMD_DOCKER) rmi "$$(< $$FILE)" ; \
Expand Down Expand Up @@ -227,15 +228,15 @@ README.md: $(CMD_EMBEDMD) $(OUT_DIR)/help.txt deploy/manifests
$(CMD_EMBEDMD) -w README.md

.PHONY: format
format: go-fmt check-license
format: go-fmt c-fmt check-license

.PHONY: c-fmt
c-fmt:
clang-format -i --style=GNU $(BPF_SRC)

.PHONY: go-fmt
go-fmt:
go fmt $(shell go list ./... | grep -E -v "pkg/internal/pprof|pkg/internal/go")
go fmt $(shell go list ./... | grep -E -v "pkg/internal/pprof|pkg/internal/go|pkg/stack/frame")

.PHONY: check-license
check-license:
Expand Down
9 changes: 6 additions & 3 deletions Tiltfile
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
docker_prune_settings(num_builds=5)
docker_prune_settings(num_builds=2)

# allow_k8s_contexts('admin@k8s-festive-perlman')
# default_registry('ttl.sh/tilt-parca-dev')

## Parca Agent

docker_build(
'parca.io/parca/parca-agent:dev', '',
dockerfile='Dockerfile.dev',
only=['./cmd', './pkg', './3rdparty', './go.mod', './go.sum', './parca-agent.bpf.c', './vmlinux.h', './Makefile'],
only=['./cmd', './pkg', './3rdparty', './go.mod', './go.sum', './bpf', './Makefile'],
)
k8s_yaml('deploy/tilt/parca-agent-daemonSet.yaml')
k8s_resource('parca-agent', port_forwards=[7071])
k8s_resource('parca-agent', port_forwards=[7071, 40000])
41 changes: 32 additions & 9 deletions parca-agent.bpf.c → bpf/cpu_profiler.bpf.c
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
#define KBUILD_MODNAME "parca-agent"

#undef container_of
//#include "bpf_core_read.h"
#include <bpf_core_read.h>
#include <bpf_endian.h>
#include <bpf_helpers.h>
Expand All @@ -33,6 +32,7 @@
#define MAX_STACK_ADDRESSES 1024
// Max depth of each stack trace to track
#define MAX_STACK_DEPTH 127
#define MAX_ENTRIES 10240

#define BPF_MAP(_name, _type, _key_type, _value_type, _max_entries) \
struct bpf_map_def SEC ("maps") _name = { \
Expand All @@ -53,8 +53,26 @@
.max_entries = _max_entries, \
};

#define BPF_HASH(_name, _key_type, _value_type) \
BPF_MAP (_name, BPF_MAP_TYPE_HASH, _key_type, _value_type, 10240);
#define BPF_HASH(_name, _key_type, _value_type, _max_entries) \
BPF_MAP (_name, BPF_MAP_TYPE_HASH, _key_type, _value_type, _max_entries);

#define LOG(_pid, _msg) \
{ \
log_event_t e = { .pid = _pid, .message = _msg }; \
bpf_ringbuf_output (&events, &e, sizeof (e), 0); \
}

typedef struct log_event
{
u32 pid;
char message[32];
// u64 addr;
} log_event_t;

struct bpf_map_def SEC ("maps") events = {
.type = BPF_MAP_TYPE_RINGBUF,
.max_entries = 1 << 24,
};

/*============================= INTERNAL STRUCTS ============================*/

Expand All @@ -67,7 +85,7 @@ typedef struct stack_count_key

/*================================ MAPS =====================================*/

BPF_HASH (counts, stack_count_key_t, u64);
BPF_HASH (counts, stack_count_key_t, u64, MAX_ENTRIES);
BPF_STACK_TRACE (stack_traces, MAX_STACK_ADDRESSES);

/*=========================== HELPER FUNCTIONS ==============================*/
Expand All @@ -90,7 +108,8 @@ bpf_map_lookup_or_try_init (void *map, const void *key, const void *init)
return bpf_map_lookup_elem (map, key);
}

// This code gets a bit complex. Probably not suitable for casual hacking.
/*=========================== BPF FUNCTIONS ==============================*/

SEC ("perf_event")
int
do_sample (struct bpf_perf_event_data *ctx)
Expand All @@ -105,18 +124,22 @@ do_sample (struct bpf_perf_event_data *ctx)
// create map key
stack_count_key_t key = { .pid = tgid };

// get stacks
key.user_stack_id = bpf_get_stackid (ctx, &stack_traces, BPF_F_USER_STACK);
// get user stack
u32 zero = 0; // First element is the PID to lookup.
key.user_stack_id = 0;
int stack_id = bpf_get_stackid (ctx, &stack_traces, BPF_F_USER_STACK);
if (stack_id >= 0)
key.user_stack_id = stack_id;

// get kernel stack
key.kernel_stack_id = bpf_get_stackid (ctx, &stack_traces, 0);

u64 zero = 0;
u64 *count;
count = bpf_map_lookup_or_try_init (&counts, &key, &zero);
if (!count)
return 0;

__sync_fetch_and_add (count, 1);

return 0;
}

Expand Down
Loading