eBPF tracing backend return value parsing

cmd/dlv/cmds/commands.go

@@ -673,6 +673,7 @@ func traceCmd(cmd *cobra.Command, args []string) {
 			done := make(chan struct{})
 			defer close(done)
 			go func() {
+				gFnEntrySeen := map[int]struct{}{}
 				for {
 					select {
 					case <-done:
@@ -694,7 +695,16 @@ func traceCmd(cmd *cobra.Command, args []string) {
 									params.WriteString(p.Value)
 								}
 							}
-							fmt.Fprintf(os.Stderr, "> (%d) %s(%s)\n", t.GoroutineID, t.FunctionName, params.String())
+							_, seen := gFnEntrySeen[t.GoroutineID]
+							if seen {
+								for _, p := range t.ReturnParams {
+									fmt.Fprintf(os.Stderr, "=> %#v\n", p.Value)
+								}
+								delete(gFnEntrySeen, t.GoroutineID)
+							} else {
+								gFnEntrySeen[t.GoroutineID] = struct{}{}
+								fmt.Fprintf(os.Stderr, "> (%d) %s(%s)\n", t.GoroutineID, t.FunctionName, params.String())
+							}
 						}
 					}
 				}

cmd/dlv/dlv_test.go

@@ -871,7 +871,7 @@ func TestTraceEBPF(t *testing.T) {
 	dlvbin, tmpdir := getDlvBinEBPF(t)
 	defer os.RemoveAll(tmpdir)
 
-	expected := []byte("> (1) main.foo(99, 9801)\n")
+	expected := []byte("> (1) main.foo(99, 9801)\n=> \"9900\"")
 
 	fixtures := protest.FindFixturesDir()
 	cmd := exec.Command(dlvbin, "trace", "--ebpf", "--output", filepath.Join(tmpdir, "__debug"), filepath.Join(fixtures, "issue573.go"), "foo")

pkg/proc/breakpoints.go

@@ -494,7 +494,6 @@ func (t *Target) SetEBPFTracepoint(fnName string) error {
 	}
 	// Start putting together the argument map. This will tell the eBPF program
 	// all of the arguments we want to trace and how to find them.
-	var args []ebpf.UProbeArgMap
 	fn, ok := t.BinInfo().LookupFunc[fnName]
 	if !ok {
 		return fmt.Errorf("could not find function %s", fnName)
@@ -533,16 +532,14 @@ func (t *Target) SetEBPFTracepoint(fnName string) error {
 	}
 	_, l, _ := t.BinInfo().PCToLine(fn.Entry)
 
+	var args []ebpf.UProbeArgMap
 	varEntries := reader.Variables(dwarfTree, fn.Entry, l, variablesFlags)
 	for _, entry := range varEntries {
-		isret, _ := entry.Val(dwarf.AttrVarParam).(bool)
-		if isret {
-			continue
-		}
 		_, dt, err := readVarEntry(entry.Tree, fn.cu.image)
 		if err != nil {
 			return err
 		}
+
 		offset, pieces, _, err := t.BinInfo().Location(entry, dwarf.AttrLocation, fn.Entry, op.DwarfRegisters{}, nil)
 		if err != nil {
 			return err
@@ -553,8 +550,16 @@ func (t *Target) SetEBPFTracepoint(fnName string) error {
 				paramPieces = append(paramPieces, int(piece.Val))
 			}
 		}
+		isret, _ := entry.Val(dwarf.AttrVarParam).(bool)
 		offset += int64(t.BinInfo().Arch.PtrSize())
-		args = append(args, ebpf.UProbeArgMap{Offset: offset, Size: dt.Size(), Kind: dt.Common().ReflectKind, Pieces: paramPieces, InReg: len(pieces) > 0})
+		args = append(args, ebpf.UProbeArgMap{
+			Offset: offset,
+			Size:   dt.Size(),
+			Kind:   dt.Common().ReflectKind,
+			Pieces: paramPieces,
+			InReg:  len(pieces) > 0,
+			Ret:    isret,
+		})
 	}
 
 	// Finally, set the uprobe on the function.

pkg/proc/core/core.go

@@ -281,6 +281,14 @@ func (dbp *process) SetUProbe(fnName string, goidOffset int64, args []ebpf.UProb
 // StartCallInjection notifies the backend that we are about to inject a function call.
 func (p *process) StartCallInjection() (func(), error) { return func() {}, nil }
 
+func (dbp *process) EnableURetProbes() error {
+	panic("not implemented")
+}
+
+func (dbp *process) DisableURetProbes() error {
+	panic("not implemented")
+}
+
 // ReadMemory will return memory from the core file at the specified location and put the
 // read memory into `data`, returning the length read, and returning an error if
 // the length read is shorter than the length of the `data` buffer.

pkg/proc/internal/ebpf/context.go

@@ -13,6 +13,7 @@ type UProbeArgMap struct {
 	Kind   reflect.Kind // Kind of variable.
 	Pieces []int        // Pieces of the variables as stored in registers.
 	InReg  bool         // True if this param is contained in a register.
+	Ret    bool         // True if this param is a return value.
 }
 
 type RawUProbeParam struct {
@@ -26,7 +27,8 @@ type RawUProbeParam struct {
 }
 
 type RawUProbeParams struct {
-	FnAddr      int
-	GoroutineID int
-	InputParams []*RawUProbeParam
+	FnAddr       int
+	GoroutineID  int
+	InputParams  []*RawUProbeParam
+	ReturnParams []*RawUProbeParam
 }

pkg/proc/internal/ebpf/helpers.go

@@ -6,6 +6,7 @@ package ebpf
 // #include "./trace_probe/function_vals.bpf.h"
 import "C"
 import (
+	"debug/elf"
 	_ "embed"
 	"encoding/binary"
 	"errors"
@@ -51,11 +52,11 @@ func (ctx *EBPFContext) AttachUprobe(pid int, name string, offset uint32) error
 	return err
 }
 
-func (ctx *EBPFContext) UpdateArgMap(key uint64, goidOffset int64, args []UProbeArgMap, gAddrOffset uint64) error {
+func (ctx *EBPFContext) UpdateArgMap(key uint64, goidOffset int64, args []UProbeArgMap, gAddrOffset uint64, isret bool) error {
 	if ctx.bpfArgMap == nil {
 		return errors.New("eBPF map not loaded")
 	}
-	params := createFunctionParameterList(key, goidOffset, args)
+	params := createFunctionParameterList(key, goidOffset, args, isret)
 	params.g_addr_offset = C.longlong(gAddrOffset)
 	return ctx.bpfArgMap.Update(unsafe.Pointer(&key), unsafe.Pointer(&params))
 }
@@ -82,7 +83,7 @@ func LoadEBPFTracingProgram() (*EBPFContext, error) {
 	var ctx EBPFContext
 	var err error
 
-	ctx.bpfModule, err = bpf.NewModuleFromBuffer(TraceProbeBytes, "trace.o")
+	ctx.bpfModule, err = bpf.NewModuleFromBuffer(TraceProbeBytes, "trace_probe/trace.o")
 	if err != nil {
 		return nil, err
 	}
@@ -114,7 +115,7 @@ func LoadEBPFTracingProgram() (*EBPFContext, error) {
 				return
 			}
 
-			parsed := ParseFunctionParameterList(b)
+			parsed := parseFunctionParameterList(b)
 
 			ctx.m.Lock()
 			ctx.parsedBpfEvents = append(ctx.parsedBpfEvents, parsed)
@@ -125,7 +126,7 @@ func LoadEBPFTracingProgram() (*EBPFContext, error) {
 	return &ctx, nil
 }
 
-func ParseFunctionParameterList(rawParamBytes []byte) RawUProbeParams {
+func parseFunctionParameterList(rawParamBytes []byte) RawUProbeParams {
 	params := (*C.function_parameter_list_t)(unsafe.Pointer(&rawParamBytes[0]))
 
 	defer runtime.KeepAlive(params) // Ensure the param is not garbage collected.
@@ -134,10 +135,10 @@ func ParseFunctionParameterList(rawParamBytes []byte) RawUProbeParams {
 	rawParams.FnAddr = int(params.fn_addr)
 	rawParams.GoroutineID = int(params.goroutine_id)
 
-	for i := 0; i < int(params.n_parameters); i++ {
+	parseParam := func(param C.function_parameter_t) *RawUProbeParam {
 		iparam := &RawUProbeParam{}
 		data := make([]byte, 0x60)
-		ret := params.params[i]
+		ret := param
 		iparam.Kind = reflect.Kind(ret.kind)
 
 		val := C.GoBytes(unsafe.Pointer(&ret.val), C.int(ret.size))
@@ -161,22 +162,30 @@ func ParseFunctionParameterList(rawParamBytes []byte) RawUProbeParams {
 			iparam.Base = FakeAddressBase + 0x30
 			iparam.Len = int64(strLen)
 		}
+		return iparam
+	}
 
-		rawParams.InputParams = append(rawParams.InputParams, iparam)
+	for i := 0; i < int(params.n_parameters); i++ {
+		rawParams.InputParams = append(rawParams.InputParams, parseParam(params.params[i]))
+	}
+	for i := 0; i < int(params.n_ret_parameters); i++ {
+		rawParams.ReturnParams = append(rawParams.ReturnParams, parseParam(params.ret_params[i]))
 	}
 
 	return rawParams
 }
 
-func createFunctionParameterList(entry uint64, goidOffset int64, args []UProbeArgMap) C.function_parameter_list_t {
+func createFunctionParameterList(entry uint64, goidOffset int64, args []UProbeArgMap, isret bool) C.function_parameter_list_t {
 	var params C.function_parameter_list_t
 	params.goid_offset = C.uint(goidOffset)
-	params.n_parameters = C.uint(len(args))
 	params.fn_addr = C.uint(entry)
-	for i, arg := range args {
+	params.is_ret = C.bool(isret)
+	params.n_parameters = C.uint(0)
+	params.n_ret_parameters = C.uint(0)
+	for _, arg := range args {
 		var param C.function_parameter_t
 		param.size = C.uint(arg.Size)
-		param.offset = C.uint(arg.Offset)
+		param.offset = C.int(arg.Offset)
 		param.kind = C.uint(arg.Kind)
 		if arg.InReg {
 			param.in_reg = true
@@ -188,7 +197,40 @@ func createFunctionParameterList(entry uint64, goidOffset int64, args []UProbeAr
 				param.reg_nums[i] = C.int(arg.Pieces[i])
 			}
 		}
-		params.params[i] = param
+		if !arg.Ret {
+			params.params[params.n_parameters] = param
+			params.n_parameters++
+		} else {
+			params.ret_params[params.n_ret_parameters] = param
+			params.n_ret_parameters++
+		}
 	}
 	return params
 }
+
+func AddressToOffset(f *elf.File, addr uint64) (uint32, error) {
+	sectionsToSearchForSymbol := []*elf.Section{}
+
+	for i := range f.Sections {
+		if f.Sections[i].Flags == elf.SHF_ALLOC+elf.SHF_EXECINSTR {
+			sectionsToSearchForSymbol = append(sectionsToSearchForSymbol, f.Sections[i])
+		}
+	}
+
+	var executableSection *elf.Section
+
+	// Find what section the symbol is in by checking the executable section's
+	// addr space.
+	for m := range sectionsToSearchForSymbol {
+		if addr > sectionsToSearchForSymbol[m].Addr &&
+			addr < sectionsToSearchForSymbol[m].Addr+sectionsToSearchForSymbol[m].Size {
+			executableSection = sectionsToSearchForSymbol[m]
+		}
+	}
+
+	if executableSection == nil {
+		return 0, errors.New("could not find symbol in executable sections of binary")
+	}
+
+	return uint32(addr - executableSection.Addr + executableSection.Offset), nil
+}

pkg/proc/internal/ebpf/helpers_disabled.go

@@ -4,6 +4,7 @@
 package ebpf
 
 import (
+	"debug/elf"
 	"errors"
 )
 
@@ -18,7 +19,11 @@ func (ctx *EBPFContext) AttachUprobe(pid int, name string, offset uint32) error
 	return errors.New("eBPF is disabled")
 }
 
-func (ctx *EBPFContext) UpdateArgMap(key uint64, goidOffset int64, args []UProbeArgMap, gAddrOffset uint64) error {
+func (ctx *EBPFContext) AttachURetprobe(pid int, name string, offset uint32) error {
+	return errors.New("eBPF is disabled")
+}
+
+func (ctx *EBPFContext) UpdateArgMap(key uint64, goidOffset int64, args []UProbeArgMap, gAddrOffset uint64, isret bool) error {
 	return errors.New("eBPF is disabled")
 }
 
@@ -34,6 +39,6 @@ func LoadEBPFTracingProgram() (*EBPFContext, error) {
 	return nil, errors.New("eBPF disabled")
 }
 
-func ParseFunctionParameterList(rawParamBytes []byte) RawUProbeParams {
-	return RawUProbeParams{}
+func AddressToOffset(f *elf.File, addr uint64) (uint32, error) {
+	return 0, errors.New("eBPF disabled")
 }

pkg/proc/internal/ebpf/trace_probe/function_vals.bpf.h

@@ -8,7 +8,7 @@ typedef struct function_parameter {
       unsigned int size; 
 
       // Offset from stack pointer. This should only be set from the Go side.
-      unsigned int offset; 
+      int offset;
 
       // If true, the parameter is passed in a register.
       bool in_reg;      
@@ -20,7 +20,7 @@ typedef struct function_parameter {
       int reg_nums[6]; 
 
       // The following are filled in by the eBPF program.
-      unsigned int daddr;   // Data address.
+      size_t daddr;   // Data address.
       char val[0x30];       // Value of the parameter.
       char deref_val[0x30]; // Dereference value of the parameter.
 } function_parameter_t;
@@ -33,6 +33,11 @@ typedef struct function_parameter_list {
       int goroutine_id;
 
       unsigned int fn_addr;
+      bool is_ret;
+
       unsigned int n_parameters;          // number of parameters.
       function_parameter_t params[6];     // list of parameters.
+
+      unsigned int n_ret_parameters;      // number of return parameters.
+      function_parameter_t ret_params[6]; // list of return parameters.
 } function_parameter_list_t;