Merge pull request #43 from kubeshark/feature/handle_go_custom_sockets

add tls capture support for custom golang sockets

bpf/common.c

@@ -114,14 +114,6 @@ static __always_inline struct ssl_info new_ssl_info() {
     return info;
 }
 
-static __always_inline struct go_info new_go_info() {
-    struct go_info info = {
-        .ssl_info = {.fd = invalid_fd, .created_at_nano = bpf_ktime_get_ns()},
-        .called_interface_type = 0,
-    };
-    return info;
-}
-
 static __always_inline struct ssl_info lookup_ssl_info(struct pt_regs* ctx, struct bpf_map_def* map_fd, __u64 pid_tgid) {
     struct ssl_info* infoPtr = bpf_map_lookup_elem(map_fd, &pid_tgid);
     struct ssl_info info = new_ssl_info();

bpf/fd_to_address_tracepoints.c

@@ -31,7 +31,7 @@ SEC("tracepoint/syscalls/sys_enter_accept4")
 void sys_enter_accept4(struct sys_enter_accept4_ctx* ctx) {
 	__u64 id = bpf_get_current_pid_tgid();
 
-	if (!should_target(id >> 32, NULL)) {
+	if (!should_target(id >> 32)) {
 		return;
 	}
 
@@ -57,7 +57,7 @@ SEC("tracepoint/syscalls/sys_exit_accept4")
 void sys_exit_accept4(struct sys_exit_accept4_ctx* ctx) {
 	__u64 id = bpf_get_current_pid_tgid();
 
-	if (!should_target(id >> 32, NULL)) {
+	if (!should_target(id >> 32)) {
 		return;
 	}
 
@@ -124,7 +124,7 @@ SEC("tracepoint/syscalls/sys_enter_connect")
 void sys_enter_connect(struct sys_enter_connect_ctx* ctx) {
 	__u64 id = bpf_get_current_pid_tgid();
 
-	if (!should_target(id >> 32, NULL)) {
+	if (!should_target(id >> 32)) {
 		return;
 	}
 
@@ -151,7 +151,7 @@ SEC("tracepoint/syscalls/sys_exit_connect")
 void sys_exit_connect(struct sys_exit_connect_ctx* ctx) {
 	__u64 id = bpf_get_current_pid_tgid();
 
-	if (!should_target(id >> 32, NULL)) {
+	if (!should_target(id >> 32)) {
 		return;
 	}
 

bpf/fd_tracepoints.c

@@ -61,7 +61,7 @@ SEC("tracepoint/syscalls/sys_enter_read")
 void sys_enter_read(struct sys_enter_read_write_ctx* ctx) {
 	__u64 id = bpf_get_current_pid_tgid();
 
-	if (!should_target(id >> 32, NULL)) {
+	if (!should_target(id >> 32)) {
 		return;
 	}
 
@@ -78,7 +78,7 @@ SEC("tracepoint/syscalls/sys_enter_write")
 void sys_enter_write(struct sys_enter_read_write_ctx* ctx) {
 	__u64 id = bpf_get_current_pid_tgid();
 
-	if (!should_target(id >> 32, NULL)) {
+	if (!should_target(id >> 32)) {
 		return;
 	}
 

bpf/go_uprobes.c

@@ -106,7 +106,7 @@ static __always_inline __u32 get_goid_from_thread_local_storage(__u64* goroutine
 }
 #endif
 
-static __always_inline int go_crypto_tls_get_fd_from_tcp_conn(struct pt_regs* ctx, enum ABI abi, __u32* fd, __u64* go_interface_type) {
+static __always_inline int go_crypto_tls_get_fd_from_tcp_conn(struct pt_regs* ctx, enum ABI abi, __u32 pid, __u32* fd) {
     struct go_interface conn;
     long err;
     __u64 addr;
@@ -130,15 +130,43 @@ static __always_inline int go_crypto_tls_get_fd_from_tcp_conn(struct pt_regs* ct
     if (err != 0) {
         return err;
     }
-    *go_interface_type = conn.type;
+    struct pid_offset o = {
+        .pid = pid,
+        .symbol_offset = conn.type,
+    };
+
+    void* net_conn_struct_ptr;
+
+    struct pid_info* pi = bpf_map_lookup_elem(&pids_info, &o);
+    if (pi != NULL && pi->sys_fd_offset != -1) {
+        if (pi->is_interface) {
+            // connection socket is interface, read this interface:
+            err = bpf_probe_read(&conn, sizeof(conn), conn.ptr);
+            if (err != 0) {
+                return err;
+            }
+        }
+
+        err = bpf_probe_read(&net_conn_struct_ptr, sizeof(net_conn_struct_ptr), conn.ptr);
+        if (err != 0) {
+            return err;
+        }
 
-    void* net_fd_ptr;
-    err = bpf_probe_read(&net_fd_ptr, sizeof(net_fd_ptr), conn.ptr);
+        err = bpf_probe_read(fd, sizeof(*fd), net_conn_struct_ptr + pi->sys_fd_offset);
+        if (err != 0) {
+            return err;
+        }
+
+        return 0;
+    }
+
+    err = bpf_probe_read(&net_conn_struct_ptr, sizeof(net_conn_struct_ptr), conn.ptr);
     if (err != 0) {
         return err;
     }
 
-    err = bpf_probe_read(fd, sizeof(*fd), net_fd_ptr + 0x10);
+    // if pids information is not descovered, supposing to find system file descriptor at the offset 0x10
+    err = bpf_probe_read(fd, sizeof(*fd), net_conn_struct_ptr + 0x10);
     if (err != 0) {
         return err;
     }
@@ -149,51 +177,52 @@ static __always_inline int go_crypto_tls_get_fd_from_tcp_conn(struct pt_regs* ct
 static __always_inline void go_crypto_tls_uprobe(struct pt_regs* ctx, struct bpf_map_def* go_context, enum ABI abi) {
     __u64 pid_tgid = bpf_get_current_pid_tgid();
     __u64 pid = pid_tgid >> 32;
-    if (!should_target(pid, NULL)) {
+    if (!should_target(pid)) {
         return;
     }
 
-    struct go_info go_info = new_go_info();
-    struct ssl_info* info = &go_info.ssl_info;
+    struct ssl_info info = new_ssl_info();
     long err;
 
 #if defined(bpf_target_arm64)
-    err = bpf_probe_read(&info->buffer_len, sizeof(__u32), (void*)GO_ABI_INTERNAL_PT_REGS_SP(ctx) + 0x18);
+    err = bpf_probe_read(&info.buffer_len, sizeof(__u32), (void*)GO_ABI_INTERNAL_PT_REGS_SP(ctx) + 0x18);
     if (err != 0) {
         log_error(ctx, LOG_ERROR_READING_BYTES_COUNT, pid_tgid, err, ORIGIN_SSL_UPROBE_CODE);
         return;
     }
 #elif defined(bpf_target_x86)
     if (abi == ABI0) {
-        err = bpf_probe_read(&info->buffer_len, sizeof(__u32), (void*)GO_ABI_0_PT_REGS_SP(ctx) + 0x18);
+        err = bpf_probe_read(&info.buffer_len, sizeof(__u32), (void*)GO_ABI_0_PT_REGS_SP(ctx) + 0x18);
         if (err != 0) {
             log_error(ctx, LOG_ERROR_READING_BYTES_COUNT, pid_tgid, err, ORIGIN_SSL_UPROBE_CODE);
             return;
         }
     } else {
-        info->buffer_len = GO_ABI_INTERNAL_PT_REGS_R2(ctx);
+        info.buffer_len = GO_ABI_INTERNAL_PT_REGS_R2(ctx);
     }
 #endif
 
 #if defined(bpf_target_x86)
     if (abi == ABI0) {
-        err = bpf_probe_read(&info->buffer, sizeof(__u32), (void*)GO_ABI_0_PT_REGS_SP(ctx) + 0x11);
+        err = bpf_probe_read(&info.buffer, sizeof(__u32), (void*)GO_ABI_0_PT_REGS_SP(ctx) + 0x11);
         if (err != 0) {
             log_error(ctx, LOG_ERROR_READING_FROM_SSL_BUFFER, pid_tgid, err, ORIGIN_SSL_UPROBE_CODE);
             return;
         }
         // We basically add 00 suffix to the hex address.
-        info->buffer = (void*)((long)info->buffer << 8);
+        info.buffer = (void*)((long)info.buffer << 8);
     } else {
 #endif
-        info->buffer = (void*)GO_ABI_INTERNAL_PT_REGS_R4(ctx);
+        info.buffer = (void*)GO_ABI_INTERNAL_PT_REGS_R4(ctx);
 #if defined(bpf_target_x86)
     }
 #endif
-    if (go_crypto_tls_get_fd_from_tcp_conn(ctx, abi, &info->fd, &go_info.called_interface_type) != 0) {
+    __u32 fd = invalid_fd;
+    if (go_crypto_tls_get_fd_from_tcp_conn(ctx, abi, pid, &fd) != 0) {
         log_error(ctx, LOG_ERROR_GETTING_GO_TCP_CONN_FD, pid_tgid, 0l, 0l);
         return;
     }
+    info.fd = fd;
 
     __u64 goroutine_id;
     if (abi == ABI0) {
@@ -212,7 +241,7 @@ static __always_inline void go_crypto_tls_uprobe(struct pt_regs* ctx, struct bpf
         goroutine_id = GO_ABI_INTERNAL_PT_REGS_GP(ctx);
     }
     __u64 pid_fp = pid << 32 | goroutine_id;
-    err = bpf_map_update_elem(go_context, &pid_fp, info, BPF_ANY);
+    err = bpf_map_update_elem(go_context, &pid_fp, &info, BPF_ANY);
 
     if (err != 0) {
         log_error(ctx, LOG_ERROR_PUTTING_SSL_CONTEXT, pid_tgid, err, 0l);
@@ -224,8 +253,7 @@ static __always_inline void go_crypto_tls_uprobe(struct pt_regs* ctx, struct bpf
 static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs* ctx, struct bpf_map_def* go_context, struct bpf_map_def* go_user_kernel_context, __u32 flags, enum ABI abi) {
     __u64 pid_tgid = bpf_get_current_pid_tgid();
     __u64 pid = pid_tgid >> 32;
-    struct pid_info* p_info;
-    if (!should_target(pid, &p_info)) {
+    if (!should_target(pid)) {
         return;
     }
 
@@ -246,12 +274,11 @@ static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs* ctx, struct
         goroutine_id = GO_ABI_INTERNAL_PT_REGS_GP(ctx);
     }
     __u64 pid_fp = pid << 32 | goroutine_id;
-    struct go_info* go_info_ptr = bpf_map_lookup_elem(go_context, &pid_fp);
+    struct ssl_info* info_ptr = bpf_map_lookup_elem(go_context, &pid_fp);
 
-    if (go_info_ptr == NULL) {
+    if (info_ptr == NULL) {
         return;
     }
-    struct ssl_info* info_ptr = &go_info_ptr->ssl_info;
     bpf_map_delete_elem(go_context, &pid_fp);
 
     struct ssl_info info;
@@ -294,11 +321,7 @@ static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs* ctx, struct
     // but sometimes the uprobe is called twice in a row without the tcp kprobes in between to fill in
     // the entry again. Keeping it in the map and rely on LRU logic.
     if (address_info == NULL) {
-        // Report error only if tls crypto object is not TCP connection (go:itab.*net.TCPConn,net.Conn)
-        if (go_info_ptr->called_interface_type == 0 || go_info_ptr->called_interface_type == p_info->go_tcp_conn_offset)
-        {
-            log_error(ctx, LOG_ERROR_GETTING_GO_USER_KERNEL_CONTEXT, pid_tgid, info_ptr->fd, err);
-        }
+        log_error(ctx, LOG_ERROR_GETTING_GO_USER_KERNEL_CONTEXT, pid_tgid, info_ptr->fd, err);
         return;
     }
 

bpf/include/common.h

@@ -15,7 +15,6 @@ static void send_chunk_part(struct pt_regs* ctx, __u8* buffer, __u64 id, struct
 static void send_chunk(struct pt_regs* ctx, __u8* buffer, __u64 id, struct tls_chunk* chunk);
 static void output_ssl_chunk(struct pt_regs* ctx, struct ssl_info* info, int count_bytes, __u64 id, __u32 flags);
 static struct ssl_info new_ssl_info();
-static struct go_info new_go_info();
 static struct ssl_info lookup_ssl_info(struct pt_regs* ctx, struct bpf_map_def* map_fd, __u64 pid_tgid);
 
 #endif /* __COMMON__ */

bpf/include/log.h

@@ -26,15 +26,15 @@ struct log_message {
 
 static __always_inline void log_error(void* ctx, __u16 message_code, __u64 arg1, __u64 arg2, __u64 arg3) {
 	struct log_message entry = {};
-	
+
 	entry.level = LOG_LEVEL_ERROR;
 	entry.message_code = message_code;
 	entry.arg1 = arg1;
 	entry.arg2 = arg2;
 	entry.arg3 = arg3;
-	
+
 	long err = bpf_perf_event_output(ctx, &log_buffer, BPF_F_CURRENT_CPU, &entry, sizeof(struct log_message));
-	
+
 	if (err != 0) {
 		char msg[] = "Error writing log error to perf buffer - %ld";
 		bpf_trace_printk(msg, sizeof(msg), err);
@@ -43,15 +43,15 @@ static __always_inline void log_error(void* ctx, __u16 message_code, __u64 arg1,
 
 static __always_inline void log_info(void* ctx, __u16 message_code, __u64 arg1, __u64 arg2, __u64 arg3) {
 	struct log_message entry = {};
-	
+
 	entry.level = LOG_LEVEL_INFO;
 	entry.message_code = message_code;
 	entry.arg1 = arg1;
 	entry.arg2 = arg2;
 	entry.arg3 = arg3;
-	
+
 	long err = bpf_perf_event_output(ctx, &log_buffer, BPF_F_CURRENT_CPU, &entry, sizeof(struct log_message));
-	
+
 	if (err != 0) {
 		char msg[] = "Error writing log info to perf buffer - %ld";
 		bpf_trace_printk(msg, sizeof(msg), arg1, err);
@@ -60,15 +60,15 @@ static __always_inline void log_info(void* ctx, __u16 message_code, __u64 arg1,
 
 static __always_inline void log_debug(void* ctx, __u16 message_code, __u64 arg1, __u64 arg2, __u64 arg3) {
 	struct log_message entry = {};
-	
+
 	entry.level = LOG_LEVEL_DEBUG;
 	entry.message_code = message_code;
 	entry.arg1 = arg1;
 	entry.arg2 = arg2;
 	entry.arg3 = arg3;
-	
+
 	long err = bpf_perf_event_output(ctx, &log_buffer, BPF_F_CURRENT_CPU, &entry, sizeof(struct log_message));
-	
+
 	if (err != 0) {
 		char msg[] = "Error writing log debug to perf buffer - %ld";
 		bpf_trace_printk(msg, sizeof(msg), arg1, err);

bpf/include/maps.h

@@ -66,11 +66,13 @@ struct goid_offsets {
 
 struct pid_info {
     __u64 go_tcp_conn_offset;
+    __s64 sys_fd_offset;
+    __u64 is_interface;
 };
 
-struct go_info {
-    struct ssl_info ssl_info;
-    __u64 called_interface_type;
+struct pid_offset {
+    __u64 pid;
+    __u64 symbol_offset;
 };
 
 const struct goid_offsets* unused __attribute__((unused));
@@ -103,7 +105,8 @@ struct {
     BPF_MAP(_name, BPF_MAP_TYPE_LRU_HASH, _key_type, _value_type, MAX_ENTRIES_LRU_HASH)
 
 // Generic
-BPF_HASH(pids_map, __u32, struct pid_info);
+BPF_HASH(pids_map, __u32, __u32);
+BPF_HASH(pids_info, struct pid_offset, struct pid_info);
 BPF_LRU_HASH(connection_context, __u64, conn_flags);
 BPF_PERF_OUTPUT(chunks_buffer);
 BPF_PERF_OUTPUT(log_buffer);
@@ -114,8 +117,8 @@ BPF_LRU_HASH(openssl_read_context, __u64, struct ssl_info);
 
 // Go specific
 BPF_HASH(goid_offsets_map, __u32, struct goid_offsets);
-BPF_LRU_HASH(go_write_context, __u64, struct go_info);
-BPF_LRU_HASH(go_read_context, __u64, struct go_info);
+BPF_LRU_HASH(go_write_context, __u64, struct ssl_info);
+BPF_LRU_HASH(go_read_context, __u64, struct ssl_info);
 BPF_LRU_HASH(go_kernel_write_context, __u64, __u32);
 BPF_LRU_HASH(go_kernel_read_context, __u64, __u32);
 BPF_LRU_HASH(go_user_kernel_write_context, __u64, struct address_info);

bpf/include/pids.h

@@ -6,22 +6,17 @@ Copyright (C) Kubeshark
 #ifndef __PIDS__
 #define __PIDS__
 
-int should_target(__u32 pid, struct pid_info** p_info) {
-	struct pid_info* p = bpf_map_lookup_elem(&pids_map, &pid);
+int should_target(__u32 pid) {
+	__u32* shouldTarget = bpf_map_lookup_elem(&pids_map, &pid);
 
-	if (p != NULL) {
-		if (p_info)
-			*p_info = p;
+	if (shouldTarget != NULL && *shouldTarget == 1) {
 		return 1;
 	}
 
 	__u32 globalPid = 0;
-	p = bpf_map_lookup_elem(&pids_map, &globalPid);
+	__u32* shouldTargetGlobally = bpf_map_lookup_elem(&pids_map, &globalPid);
 
-	if (p && p_info)
-		*p_info = p;
-
-	return p != NULL;
+	return shouldTargetGlobally != NULL && *shouldTargetGlobally == 1;
 }
 
 #endif /* __PIDS__ */

bpf/openssl_uprobes.c

@@ -44,7 +44,7 @@ static __always_inline void ssl_uprobe(struct pt_regs* ctx, void* ssl, void* buf
 
 	__u64 id = bpf_get_current_pid_tgid();
 
-	if (!should_target(id >> 32, NULL)) {
+	if (!should_target(id >> 32)) {
 		return;
 	}
 
@@ -63,7 +63,7 @@ static __always_inline void ssl_uprobe(struct pt_regs* ctx, void* ssl, void* buf
 static __always_inline void ssl_uretprobe(struct pt_regs* ctx, struct bpf_map_def* map_fd, __u32 flags) {
 	__u64 id = bpf_get_current_pid_tgid();
 
-	if (!should_target(id >> 32, NULL)) {
+	if (!should_target(id >> 32)) {
 		return;
 	}
 

bpf/tcp_kprobes.c

@@ -81,7 +81,7 @@ static __always_inline void tcp_kprobe(struct pt_regs* ctx, struct bpf_map_def*
 
 	__u64 id = bpf_get_current_pid_tgid();
 
-	if (!should_target(id >> 32, NULL)) {
+	if (!should_target(id >> 32)) {
 		return;
 	}