Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

wip: python debug logs #3184

Closed
wants to merge 15 commits into from
Closed

wip: python debug logs #3184

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
4acf2b8
python debug logs
korniltsev Apr 12, 2024
08dfc90
typecheck init
korniltsev Apr 24, 2024
a7ebbc3
fix uninitialized var
korniltsev Apr 24, 2024
8b1f185
typecheck frame and code
korniltsev Apr 24, 2024
bcc7008
validate pystr
korniltsev Apr 24, 2024
ac93b8d
disable typecheck for non 3.8
korniltsev Apr 25, 2024
c2ee1a2
cleanup
korniltsev Apr 25, 2024
10409fc
maybe fix pthread_self reading
korniltsev Apr 25, 2024
740f3fc
log moar
korniltsev Apr 25, 2024
9ce473d
fix
korniltsev Apr 25, 2024
d4785c9
bump
korniltsev Apr 25, 2024
2bff6dd
fix
korniltsev Apr 25, 2024
c29120c
fix
korniltsev Apr 25, 2024
47fd5d8
typecheck
korniltsev Apr 30, 2024
d3b7d61
page warmup
korniltsev Apr 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
37 changes: 20 additions & 17 deletions ebpf/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
GO ?= go
RIDESHARE_REPO ?= korniltsev
RIDESHARE="testdata/rideshare-flask-no-pip"
RIDESHARE_REPO ?= pyroscope
RIDESHARE=testdata/rideshare-flask-no-pip

ifeq ($(shell uname -s),Linux)
ifeq ($(shell uname -m),x86_64)
Expand Down Expand Up @@ -29,12 +29,12 @@ python/dwarfdump:
git submodule update --init --recursive

echo "//go:build amd64 && linux" > python/python_offsets_gen_amd64.go
go run cmd/python_dwarfdump/main.go $(shell find testdata/python-x64 -name libpy\*.so\*) \
go run cmd/python_dwarfdump/main.go $(shell find testdata/python-x64 -name libpy\*.so\* | grep -v pyston) \
$(shell find testdata/python-x64 | grep -E "/python3\\.[0-9]+") >> python/python_offsets_gen_amd64.go
go fmt python/python_offsets_gen_amd64.go

echo "//go:build arm64 && linux" > python/python_offsets_gen_arm64.go
go run cmd/python_dwarfdump/main.go $(shell find testdata/python-arm64 -name libpy\*.so\*) \
go run cmd/python_dwarfdump/main.go $(shell find testdata/python-arm64 -name libpy\*.so\* | grep -v pyston) \
$(shell find testdata/python-arm64 | grep -E "/python3\\.[0-9]+") >> python/python_offsets_gen_arm64.go
go fmt python/python_offsets_gen_arm64.go

Expand Down Expand Up @@ -92,16 +92,19 @@ go/test/arm64: ebpf.arm64.test

.phony: rideshare/gen
rideshare/gen:
git submodule update --init --recursive
docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.8-slim --build-arg="PYTHON_VERSION=3.8-slim" $(RIDESHARE)
docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.9-slim --build-arg="PYTHON_VERSION=3.9-slim" $(RIDESHARE)
docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.10-slim --build-arg="PYTHON_VERSION=3.10-slim" $(RIDESHARE)
docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.11-slim --build-arg="PYTHON_VERSION=3.11-slim" $(RIDESHARE)
docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.12-slim --build-arg="PYTHON_VERSION=3.12-slim" $(RIDESHARE)
docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.13-rc-slim --build-arg="PYTHON_VERSION=3.13-rc-slim" $(RIDESHARE)
docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.8-alpine --build-arg="PYTHON_VERSION=3.8-alpine" $(RIDESHARE)
docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.9-alpine --build-arg="PYTHON_VERSION=3.9-alpine" $(RIDESHARE)
docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.10-alpine --build-arg="PYTHON_VERSION=3.10-alpine" $(RIDESHARE)
docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.11-alpine --build-arg="PYTHON_VERSION=3.11-alpine" $(RIDESHARE)
docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.12-alpine --build-arg="PYTHON_VERSION=3.12-alpine" $(RIDESHARE)
docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.13-rc-alpine --build-arg="PYTHON_VERSION=3.13-rc-alpine" $(RIDESHARE)
# git submodule update --init --recursive
# docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.8-slim --build-arg="PYTHON_VERSION=3.8-slim" $(RIDESHARE)
# docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.9-slim --build-arg="PYTHON_VERSION=3.9-slim" $(RIDESHARE)
# docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.10-slim --build-arg="PYTHON_VERSION=3.10-slim" $(RIDESHARE)
# docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.11-slim --build-arg="PYTHON_VERSION=3.11-slim" $(RIDESHARE)
# docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.12-slim --build-arg="PYTHON_VERSION=3.12-slim" $(RIDESHARE)
# docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.13-rc-slim --build-arg="PYTHON_VERSION=3.13-rc-slim" $(RIDESHARE)
# docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.8-alpine --build-arg="PYTHON_VERSION=3.8-alpine" $(RIDESHARE)
# docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.9-alpine --build-arg="PYTHON_VERSION=3.9-alpine" $(RIDESHARE)
# docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.10-alpine --build-arg="PYTHON_VERSION=3.10-alpine" $(RIDESHARE)
# docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.11-alpine --build-arg="PYTHON_VERSION=3.11-alpine" $(RIDESHARE)
# docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.12-alpine --build-arg="PYTHON_VERSION=3.12-alpine" $(RIDESHARE)
# docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:3.13-rc-alpine --build-arg="PYTHON_VERSION=3.13-rc-alpine" $(RIDESHARE)
docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:ubuntu-18.04 --build-arg="BASE=ubuntu:18.04" --build-arg="FLASK_VERSION=2.0.3" -f $(RIDESHARE)/ubuntu.Dockerfile $(RIDESHARE)
docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:ubuntu-20.04 --build-arg="BASE=ubuntu:20.04" --build-arg="FLASK_VERSION=3.0.3" -f $(RIDESHARE)/ubuntu.Dockerfile $(RIDESHARE)
docker buildx build --platform=linux/amd64,linux/arm64 --push -t $(RIDESHARE_REPO)/ebpf-testdata-rideshare:ubuntu-22.04 --build-arg="BASE=ubuntu:22.04" --build-arg="FLASK_VERSION=3.0.3" -f $(RIDESHARE)/ubuntu.Dockerfile $(RIDESHARE)
24 changes: 19 additions & 5 deletions ebpf/bpf/profile.bpf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
#include "bpf_tracing.h"
#include "profile.bpf.h"
#include "pid.h"
#include "ume.h"

#define PF_KTHREAD 0x00200000

Expand All @@ -28,12 +27,10 @@ int do_perf_event(struct bpf_perf_event_data *ctx) {
return 0;
}
int flags = 0;
if (pyro_bpf_core_read(&flags, sizeof(flags), &task->flags)) {
bpf_dbg_printk("failed to read task->flags\n");
if (bpf_core_read(&flags, sizeof(flags), &task->flags)) {
return 0;
}
if (flags & PF_KTHREAD) {
bpf_dbg_printk("skipping kthread %d\n", tgid);
return 0;
}

Expand All @@ -46,7 +43,6 @@ int do_perf_event(struct bpf_perf_event_data *ctx) {
.padding_ = 0
};
if (bpf_map_update_elem(&pids, &tgid, &unknown, BPF_NOEXIST)) {
bpf_dbg_printk("failed to update pids map. probably concurrent update\n");
return 0;
}
struct pid_event event = {
Expand Down Expand Up @@ -123,4 +119,22 @@ int BPF_KPROBE(exec, void *_) {
return 0;
}

//// sched_process_exit
//SEC("kprobe/sched_process_exit")
//int BPF_KPROBE(sched_process_exit, struct task_struct *task) {
// u32 pid = 0;
// bpf_core_read(&pid, sizeof(pid), &task->pid);
// bpf_printk("sched_process_exit pid=%d\n", pid);
// return 0;
//}
//
//// kernel_clone_args
//SEC("kprobe/kernel_clone")
//int BPF_KPROBE(kernel_clone, struct kernel_clone_args *args) {
// u32 pid = 0;
// bpf_core_read(&pid, sizeof(pid), &args->child_tid);
// bpf_printk("kernel_clone pid=%d\n", pid);
// return 0;
//}

char _license[] SEC("license") = "GPL";
17 changes: 0 additions & 17 deletions ebpf/bpf/profile.bpf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,6 @@ struct pid_config {
uint8_t padding_;
};

#define OP_REQUEST_UNKNOWN_PROCESS_INFO 1
#define OP_PID_DEAD 2
#define OP_REQUEST_EXEC_PROCESS_INFO 3

struct pid_event {
uint32_t op;
uint32_t pid;
};
struct pid_event e__;






Expand All @@ -40,11 +28,6 @@ struct {
} pids SEC(".maps");


struct {
__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
__uint(key_size, sizeof(u32));
__uint(value_size, sizeof(u32));
} events SEC(".maps");


struct {
Expand Down
60 changes: 31 additions & 29 deletions ebpf/bpf/pthread_amd64.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,18 @@

#include "vmlinux.h"
#include "bpf_helpers.h"
#include "ume.h"
#include "bpf_core_read.h"
#include "pyoffsets.h"


#if !defined(__TARGET_ARCH_x86)
#error "Wrong architecture"
#endif

static int pthread_getspecific_musl(const struct libc *libc, int32_t key, void **out, const void *tls_base);
static int pthread_getspecific_glibc(const struct libc *libc, int32_t key, void **out, const void *tls_base);
static int pthread_getspecific_musl(void *ctx, const struct libc *libc, int32_t key, void **out, const void *fsbase);
static int pthread_getspecific_glibc(void *ctx, const struct libc *libc, int32_t key, void **out, const void *fsbase);

static __always_inline int pyro_pthread_getspecific(struct libc *libc, int32_t key, void **out) {
static __always_inline int pyro_pthread_getspecific(void *ctx, struct libc *libc, int32_t key, void **out) {
if (key == -1) {
return -1;
}
Expand All @@ -27,40 +27,46 @@ static __always_inline int pyro_pthread_getspecific(struct libc *libc, int32_t k
return -1;
}
void *tls_base = NULL;
short unsigned int fsindex = 0;


if (pyro_bpf_core_read(&tls_base, sizeof(tls_base), &task->thread.fsbase)) {
log_debug("pyro_pthread_getspecific(amd64) key=%d pthread_size=%llx o_pthread_specific1stblock=%llx", key, libc->pthread_size, libc->pthread_specific1stblock);
if (bpf_core_read(&tls_base, sizeof(tls_base), &task->thread.fsbase)) {
log_error("pyro_pthread_getspecific(amd64) failed to read fsbase");
return -1;
}
if (bpf_core_read(&fsindex, sizeof(fsindex), &task->thread.fsindex)) {
log_error("pyro_pthread_getspecific(amd64) failed to read fsindex");
return -1;
}
log_debug("pyro_pthread_getspecific(amd64) fsbase = 0x%llx fsindex = 0x%x musl=%d", tls_base, fsindex, libc->musl);


if (libc->musl) {
return pthread_getspecific_musl(libc, key, out, tls_base);
return pthread_getspecific_musl(ctx, libc, key, out, tls_base);

}
return pthread_getspecific_glibc(libc, key, out, tls_base);
return pthread_getspecific_glibc(ctx, libc, key, out, tls_base);

}

static __always_inline int pthread_getspecific_glibc(const struct libc *libc, int32_t key, void **out, const void *tls_base) {
void *tmp = NULL;
static __always_inline int pthread_getspecific_glibc(void *ctx, const struct libc *libc, int32_t key, void **out, const void *fsbase) {
void *tmp[2] = {NULL, NULL};
if (key >= 32) {
return -1; // it is possible to implement this branch, but it's not needed as autoTLSkey is almost always 0
}
void *thread_self = NULL;
try_read(thread_self, fsbase + 0x10);
log_debug("pthread_getspecific_glibc(amd64) thread_self=%llx", thread_self);
// This assumes autoTLSkey < 32, which means that the TLS is stored in
// pthread->specific_1stblock[autoTLSkey]
if (bpf_probe_read_user(
&tmp,
sizeof(tmp),
tls_base + libc->pthread_specific1stblock + key * 0x10 + 0x08)) {
return -1;
}
*out = tmp;
try_read(tmp, thread_self + libc->pthread_specific1stblock + key * 0x10)
log_debug("pthread_getspecific_glibc(amd64) res=%llx %llx", tmp[0], tmp[1]);
*out = tmp[1];
return 0;
}

static __always_inline int pthread_getspecific_musl(const struct libc *libc, int32_t key, void **out,
const void *tls_base) {
static __always_inline int pthread_getspecific_musl(void *ctx, const struct libc *libc, int32_t key, void **out,
const void *fsbase) {
// example from musl 1.2.4 from alpine 3.18
// static void *__pthread_getspecific(pthread_key_t k)
// {
Expand All @@ -85,16 +91,12 @@ static __always_inline int pthread_getspecific_musl(const struct libc *libc, int
// 56409: 48 8b 04 f8 mov rax,QWORD PTR [rax+rdi*8]
// 5640d: c3 ret
void *tmp = NULL;

if (bpf_probe_read_user(&tmp,sizeof(tmp), tls_base)) {
return -1;
}
if (bpf_probe_read_user(&tmp, sizeof(tmp), tmp + libc->pthread_specific1stblock)) {
return -1;
}
if (bpf_probe_read_user(&tmp, sizeof(tmp), tmp + key * 0x8)) {
return -1;
}
try_read(tmp, fsbase)
log_debug("pthread_getspecific_musl(amd64) tmp=%llx", tmp);
try_read(tmp, tmp + libc->pthread_specific1stblock)
log_debug("pthread_getspecific_musl(amd64) tmp2=%llx", tmp);
try_read(tmp, tmp + key * 0x8)
log_debug("pthread_getspecific_musl(amd64) res=%llx", tmp);
*out = tmp;
return 0;
}
Expand Down
41 changes: 19 additions & 22 deletions ebpf/bpf/pthread_arm64.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,17 @@

#include "vmlinux.h"
#include "bpf_helpers.h"
#include "ume.h"
#include "bpf_core_read.h"
#include "pyoffsets.h"

#if !defined(__TARGET_ARCH_arm64)
#error "Wrong architecture"
#endif

static int pthread_getspecific_musl(const struct libc *libc, int32_t key, void **out, const void *tls_base);
static int pthread_getspecific_glibc(const struct libc *libc, int32_t key, void **out, const void *tls_base);
static int pthread_getspecific_musl(void *ctx, const struct libc *libc, int32_t key, void **out, const void *tls_base);
static int pthread_getspecific_glibc(void *ctx, const struct libc *libc, int32_t key, void **out, const void *tls_base);

static __always_inline int pyro_pthread_getspecific(struct libc *libc, int32_t key, void **out) {
static __always_inline int pyro_pthread_getspecific(void *ctx, struct libc *libc, int32_t key, void **out) {
if (key == -1) {
return -1;
}
Expand All @@ -26,22 +26,24 @@ static __always_inline int pyro_pthread_getspecific(struct libc *libc, int32_t k
return -1;
}
void *tls_base = NULL;

if (pyro_bpf_core_read(&tls_base, sizeof(tls_base), &task->thread.uw.tp_value)) {
log_debug("pyro_pthread_getspecific(arm64) key=%d pthread_size=%llx o_pthread_specific1stblock=%llx", key, libc->pthread_size, libc->pthread_specific1stblock);
if (bpf_core_read(&tls_base, sizeof(tls_base), &task->thread.uw.tp_value)) {
log_error("pyro_pthread_getspecific(arm64) failed to read task->thread.uw.tp_value");
return -1;
}
log_debug("pyro_pthread_getspecific(arm64) tls_base=%llx musl=%d", tls_base, libc->musl);


if (libc->musl) {
return pthread_getspecific_musl(libc, key, out, tls_base);
return pthread_getspecific_musl(ctx, libc, key, out, tls_base);
} else {
return pthread_getspecific_glibc(libc, key, out, tls_base);
return pthread_getspecific_glibc(ctx, libc, key, out, tls_base);
}

return 0;
}

int __always_inline pthread_getspecific_glibc(const struct libc *libc, int32_t key, void **out, const void *tls_base) {
int __always_inline pthread_getspecific_glibc(void *ctx, const struct libc *libc, int32_t key, void **out, const void *tls_base) {
void *res = NULL;
if (key >= 32) {
return -1; // it is possible to implement this branch, but it's not needed as autoTLSkey is almost always 0
Expand All @@ -53,17 +55,15 @@ int __always_inline pthread_getspecific_glibc(const struct libc *libc, int32_t k

tls_base -= libc->pthread_size;

if (bpf_probe_read_user(
&res,
sizeof(res),
tls_base + libc->pthread_specific1stblock + key * 0x10 + 0x08)) {
return -1;
}
try_read(
res,
tls_base + libc->pthread_specific1stblock + key * 0x10 + 0x08)
log_debug("pthread_getspecific_glibc(arm64) res=%llx", res);
*out = res;
return 0;
}

int __always_inline pthread_getspecific_musl(const struct libc *libc, int32_t key, void **out, const void *tls_base) {
int __always_inline pthread_getspecific_musl(void *ctx, const struct libc *libc, int32_t key, void **out, const void *tls_base) {

// example from musl 1.2.4 from alpine 3.18
// static void *__pthread_getspecific(pthread_key_t k)
Expand All @@ -80,12 +80,9 @@ int __always_inline pthread_getspecific_musl(const struct libc *libc, int32_t ke
// 5fc5c: f8605820 ldr x0, [x1, w0, uxtw #3]
// 5fc60: d65f03c0 ret
void *tmp;
if (bpf_probe_read_user(&tmp,sizeof(tmp), tls_base - libc->pthread_size + libc->pthread_specific1stblock)) {
return -1;
}
if (bpf_probe_read_user(&tmp, sizeof(tmp), tmp + key * 0x8)) {
return -1;
}
try_read (tmp, tls_base - libc->pthread_size + libc->pthread_specific1stblock)
try_read(tmp, tmp + key * 0x8)
log_debug("pthread_getspecific_musl(arm64) res=%llx", tmp);
*out = tmp;
return 0;
}
Expand Down