Skip to content

Commit

Permalink
bpf,metrics: add line/file information to cilium-dbg bpf metrics list
Browse files Browse the repository at this point in the history
Fixes #27748

Example output:
```
~ cilium bpf metrics list
REASON                    DIRECTION   PACKETS   BYTES    LINE   FILE
Interface                 INGRESS     151       9368     1092   bpf_host.c
Interface                 INGRESS     731       139248   659    bpf_overlay.c
Success                   EGRESS      137       8212     1531   bpf_host.c
Success                   EGRESS      9         744      1286   bpf_lxc.c
Success                   INGRESS     11        926      86     l3.h
Success                   INGRESS     1403      299078   231    trace.h
Unsupported L3 protocol   EGRESS      6         460      1466   bpf_lxc.c
```

```
~ cilium bpf metrics list -o json
  {
    "reason": "Success",
    "direction": "egress",
    "packets": 406587,
    "bytes": 78023315,
    "line": 40,
    "file": "encap.h"
  },
  {
    "reason": "Unsupported L3 protocol",
    "direction": "egress",
    "packets": 167,
    "bytes": 12050,
    "line": 1466,
    "file": "bpf_lxc.c"
  }
```

Signed-off-by: Timo Beckers <timo@isovalent.com>
  • Loading branch information
ti-mo committed Mar 21, 2024
1 parent dd3349f commit 794f4a6
Show file tree
Hide file tree
Showing 11 changed files with 100 additions and 33 deletions.
4 changes: 3 additions & 1 deletion bpf/lib/common.h
Original file line number Diff line number Diff line change
Expand Up @@ -435,7 +435,9 @@ struct metrics_key {
__u8 reason; /* 0: forwarded, >0 dropped */
__u8 dir:2, /* 1: ingress 2: egress */
pad:6;
__u16 reserved[3]; /* reserved for future extension */
__u16 line; /* __MAGIC_LINE__ */
__u8 file; /* __MAGIC_FILE__, needs to fit __source_file_name_to_id */
__u8 reserved[3]; /* reserved for future extension */
};


Expand Down
12 changes: 6 additions & 6 deletions bpf/lib/drop.h
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ _send_drop_notify(__u8 file, __u16 line, struct __ctx_buff *ctx,
ctx_store_meta(ctx, 3, dst_id);
ctx_store_meta(ctx, 4, exitcode | file << 8 | line << 16);

update_metrics(ctx_full_len(ctx), direction, (__u8)reason);
_update_metrics(ctx_full_len(ctx), direction, (__u8)reason, line, file);
ret = tail_call_internal(ctx, CILIUM_CALL_DROP_NOTIFY, NULL);
/* ignore the returned error, use caller-provided exitcode */

Expand All @@ -130,7 +130,7 @@ int _send_drop_notify(__u8 file __maybe_unused, __u16 line __maybe_unused,
__u32 dst __maybe_unused, __u32 dst_id __maybe_unused,
__u32 reason, __u32 exitcode, enum metric_dir direction)
{
update_metrics(ctx_full_len(ctx), direction, (__u8)reason);
_update_metrics(ctx_full_len(ctx), direction, (__u8)reason, line, file);
return exitcode;
}
#endif /* DROP_NOTIFY */
Expand Down Expand Up @@ -164,19 +164,19 @@ int _send_drop_notify(__u8 file __maybe_unused, __u16 line __maybe_unused,
})

#define send_drop_notify(ctx, src, dst, dst_id, reason, exitcode, direction) \
_send_drop_notify(__MAGIC_FILE__, __LINE__, ctx, src, dst, dst_id, \
_send_drop_notify(__MAGIC_FILE__, __MAGIC_LINE__, ctx, src, dst, dst_id, \
__DROP_REASON(reason), exitcode, direction)

#define send_drop_notify_error(ctx, src, reason, exitcode, direction) \
_send_drop_notify(__MAGIC_FILE__, __LINE__, ctx, src, 0, 0, \
_send_drop_notify(__MAGIC_FILE__, __MAGIC_LINE__, ctx, src, 0, 0, \
__DROP_REASON(reason), exitcode, direction)

#define send_drop_notify_ext(ctx, src, dst, dst_id, reason, ext_err, exitcode, direction) \
_send_drop_notify(__MAGIC_FILE__, __LINE__, ctx, src, dst, dst_id, \
_send_drop_notify(__MAGIC_FILE__, __MAGIC_LINE__, ctx, src, dst, dst_id, \
__DROP_REASON_EXT(reason, ext_err), exitcode, direction)

#define send_drop_notify_error_ext(ctx, src, reason, ext_err, exitcode, direction) \
_send_drop_notify(__MAGIC_FILE__, __LINE__, ctx, src, 0, 0, \
_send_drop_notify(__MAGIC_FILE__, __MAGIC_LINE__, ctx, src, 0, 0, \
__DROP_REASON_EXT(reason, ext_err), exitcode, direction)

#endif /* __LIB_DROP__ */
9 changes: 6 additions & 3 deletions bpf/lib/metrics.h
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,18 @@
* is the drop error code.
* Update the metrics map.
*/
static __always_inline void update_metrics(__u64 bytes, __u8 direction,
__u8 reason)
#define update_metrics(bytes, direction, reason) \
_update_metrics(bytes, direction, reason, __MAGIC_LINE__, __MAGIC_FILE__)
static __always_inline void _update_metrics(__u64 bytes, __u8 direction,
__u8 reason, __u16 line, __u8 file)
{
struct metrics_value *entry, new_entry = {};
struct metrics_key key = {};

key.reason = reason;
key.dir = direction;

key.line = line;
key.file = file;

entry = map_lookup_elem(&METRICS_MAP, &key);
if (entry) {
Expand Down
7 changes: 7 additions & 0 deletions bpf/lib/source_info.h
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ __id_for_file(const char *const header_name)
_strcase_(2, "bpf_lxc.c");
_strcase_(3, "bpf_overlay.c");
_strcase_(4, "bpf_xdp.c");
_strcase_(5, "bpf_sock.c");
_strcase_(6, "bpf_network.c");

/* header files from bpf/lib/ */
_strcase_(101, "arp.h");
Expand All @@ -46,6 +48,11 @@ __id_for_file(const char *const header_name)
_strcase_(105, "nodeport.h");
_strcase_(106, "lb.h");
_strcase_(107, "mcast.h");
_strcase_(108, "ipv4.h");
_strcase_(109, "conntrack.h");
_strcase_(110, "l3.h");
_strcase_(111, "trace.h");
_strcase_(112, "encap.h");

/* @@ source files list end */

Expand Down
33 changes: 19 additions & 14 deletions bpf/lib/trace.h
Original file line number Diff line number Diff line change
Expand Up @@ -79,35 +79,37 @@ enum {
*
* Update metrics based on a trace event
*/
#define update_trace_metrics(ctx, obs_point, reason) \
_update_trace_metrics(ctx, obs_point, reason, __MAGIC_LINE__, __MAGIC_FILE__)
static __always_inline void
update_trace_metrics(struct __ctx_buff *ctx, enum trace_point obs_point,
enum trace_reason reason)
_update_trace_metrics(struct __ctx_buff *ctx, enum trace_point obs_point,
enum trace_reason reason, __u16 line, __u8 file)
{
__u8 encrypted;

switch (obs_point) {
case TRACE_TO_LXC:
update_metrics(ctx_full_len(ctx), METRIC_INGRESS,
REASON_FORWARDED);
_update_metrics(ctx_full_len(ctx), METRIC_INGRESS,
REASON_FORWARDED, line, file);
break;
case TRACE_TO_HOST:
case TRACE_TO_STACK:
case TRACE_TO_OVERLAY:
case TRACE_TO_NETWORK:
update_metrics(ctx_full_len(ctx), METRIC_EGRESS,
REASON_FORWARDED);
_update_metrics(ctx_full_len(ctx), METRIC_EGRESS,
REASON_FORWARDED, line, file);
break;
case TRACE_FROM_HOST:
case TRACE_FROM_STACK:
case TRACE_FROM_OVERLAY:
case TRACE_FROM_NETWORK:
encrypted = reason & TRACE_REASON_ENCRYPTED;
if (!encrypted)
update_metrics(ctx_full_len(ctx), METRIC_INGRESS,
REASON_PLAINTEXT);
_update_metrics(ctx_full_len(ctx), METRIC_INGRESS,
REASON_PLAINTEXT, line, file);
else
update_metrics(ctx_full_len(ctx), METRIC_INGRESS,
REASON_DECRYPT);
_update_metrics(ctx_full_len(ctx), METRIC_INGRESS,
REASON_DECRYPT, line, file);
break;
/* TRACE_FROM_LXC, i.e endpoint-to-endpoint delivery is handled
* separately in ipv*_local_delivery() where we can bump an egress
Expand Down Expand Up @@ -183,17 +185,20 @@ emit_trace_notify(enum trace_point obs_point, __u32 monitor)
return true;
}

#define send_trace_notify(ctx, obs_point, src, dst, dst_id, ifindex, reason, monitor) \
_send_trace_notify(ctx, obs_point, src, dst, dst_id, ifindex, reason, monitor, \
__MAGIC_LINE__, __MAGIC_FILE__)
static __always_inline void
send_trace_notify(struct __ctx_buff *ctx, enum trace_point obs_point,
__u32 src, __u32 dst, __u16 dst_id, __u32 ifindex,
enum trace_reason reason, __u32 monitor)
_send_trace_notify(struct __ctx_buff *ctx, enum trace_point obs_point,
__u32 src, __u32 dst, __u16 dst_id, __u32 ifindex,
enum trace_reason reason, __u32 monitor, __u16 line, __u8 file)
{
__u64 ctx_len = ctx_full_len(ctx);
__u64 cap_len = min_t(__u64, monitor ? : TRACE_PAYLOAD_LEN,
ctx_len);
struct trace_notify msg __align_stack_8;

update_trace_metrics(ctx, obs_point, reason);
_update_trace_metrics(ctx, obs_point, reason, line, file);

if (!emit_trace_notify(obs_point, monitor))
return;
Expand Down
22 changes: 18 additions & 4 deletions cilium-dbg/cmd/bpf_metrics_list.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ const (
directionTitle = "DIRECTION"
packetsTitle = "PACKETS"
bytesTitle = "BYTES"
lineTitle = "LINE"
fileTitle = "FILE"
)

type metricsRow struct {
Expand All @@ -31,13 +33,17 @@ type metricsRow struct {
direction string
packets uint64
bytes uint64
line uint16
file string
}

type jsonMetric struct {
Reason string `json:"reason"`
Direction string `json:"direction"`
Packets uint64 `json:"packets"`
Bytes uint64 `json:"bytes"`
Line uint16 `json:"line"`
File string `json:"file"`
}

type jsonMetrics []*jsonMetric
Expand Down Expand Up @@ -85,6 +91,8 @@ func listJSONMetrics(bpfMetricsList []*metricsRow) {
type key struct {
reason string
direction string
line uint16
file string
}

metrics := make(map[key]*jsonMetric)
Expand All @@ -93,12 +101,16 @@ func listJSONMetrics(bpfMetricsList []*metricsRow) {
k := key{
reason: monitorAPI.DropReason(row.reasonCode),
direction: strings.ToLower(row.direction),
line: row.line,
file: row.file,
}

if _, ok := metrics[k]; !ok {
metrics[k] = &jsonMetric{
Reason: monitorAPI.DropReason(row.reasonCode),
Direction: strings.ToLower(row.direction),
Line: row.line,
File: row.file,
}
}

Expand All @@ -124,13 +136,13 @@ func listHumanReadableMetrics(bpfMetricsList []*metricsRow) {
}

w := tabwriter.NewWriter(os.Stdout, 5, 0, 3, ' ', 0)
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", reasonTitle, directionTitle, packetsTitle, bytesTitle)
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", reasonTitle, directionTitle, packetsTitle, bytesTitle, lineTitle, fileTitle)

const numColumns = 4
const numColumns = 6
rows := [][numColumns]string{}

for _, row := range bpfMetricsList {
rows = append(rows, [numColumns]string{row.reasonDesc, row.direction, fmt.Sprintf("%d", row.packets), fmt.Sprintf("%d", row.bytes)})
rows = append(rows, [numColumns]string{row.reasonDesc, row.direction, fmt.Sprintf("%d", row.packets), fmt.Sprintf("%d", row.bytes), fmt.Sprintf("%d", row.line), row.file})
}

sort.Slice(rows, func(i, j int) bool {
Expand All @@ -146,7 +158,7 @@ func listHumanReadableMetrics(bpfMetricsList []*metricsRow) {
})

for _, r := range rows {
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", r[0], r[1], r[2], r[3])
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", r[0], r[1], r[2], r[3], r[4], r[5])
}

w.Flush()
Expand All @@ -159,6 +171,8 @@ func extractRow(key *metricsmap.Key, values *metricsmap.Values) *metricsRow {
key.Direction(),
values.Count(),
values.Bytes(),
key.Line,
key.FileName(),
}
}

Expand Down
19 changes: 19 additions & 0 deletions cilium-dbg/cmd/bpf_metrics_list_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ func (s *BPFMetricsMapSuite) TestDumpMetrics(c *C) {
Key: metricsmap.Key{Reason: 132, Dir: 2},
Values: metricsmap.Values{{Count: 1, Bytes: 1}},
},
{
Key: metricsmap.Key{Reason: 140, Dir: 2, Line: 1337, File: 1},
Values: metricsmap.Values{{Count: 400, Bytes: 4000}},
},
},
),
}
Expand All @@ -58,24 +62,39 @@ func (s *BPFMetricsMapSuite) TestDumpMetrics(c *C) {
return strings.ToLower(metricsmap.MetricDirection(uint8(d)))
}

file := func(f uint8) string {
return monitorAPI.BPFFileName(f)
}

want := jsonMetrics{
{
Reason: reason(0),
Direction: dir(1),
Packets: 100,
Bytes: 1000,
File: file(0),
},
{
Reason: reason(0),
Direction: dir(2),
Packets: 200,
Bytes: 2000,
File: file(0),
},
{
Reason: reason(132),
Direction: dir(2),
Packets: 301,
Bytes: 3001,
File: file(0),
},
{
Reason: reason(140),
Direction: dir(2),
Line: 1337,
File: file(1),
Packets: 400,
Bytes: 4000,
},
}

Expand Down
5 changes: 3 additions & 2 deletions contrib/scripts/check-source-info.sh
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,10 @@ fi
# functions.
#
required_files=$(
grep -E 'send_drop_notify(|_error|_ext|_error_ext)?' bpf/*.c bpf/lib/*.h |
grep -E 'send_drop_notify(|_error|_ext|_error_ext)?|update_(trace_)?metrics|send_trace_notify' bpf/*.c bpf/lib/*.h |
cut -f1 -d: |
sort -u |
grep -v "metrics.h" |
xargs -n1 basename
)

Expand All @@ -66,7 +67,7 @@ done
#
for f in $defined_files; do
if ! grep --silent -w "$f" <<<"$required_files"; then
echo "$0: $f is not using send_drop_notify*, please remove it from ${source_info_h}" >&2
echo "$0: $f is not using send_drop_notify*, update_(trace_)metrics or send_trace_notify, please remove it from ${source_info_h}" >&2
retval=1
fi
done
Expand Down
Binary file modified pkg/bpf/testdata/unreachable-tailcall.o
Binary file not shown.
15 changes: 12 additions & 3 deletions pkg/maps/metricsmap/metricsmap.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,13 @@ var direction = map[uint8]string{

// Key must be in sync with struct metrics_key in <bpf/lib/common.h>
type Key struct {
Reason uint8 `align:"reason"`
Dir uint8 `align:"dir"`
Reserved [3]uint16 `align:"reserved"`
Reason uint8 `align:"reason"`
Dir uint8 `align:"dir"`
// Line contains the line number of the metrics statement.
Line uint16 `align:"line"`
// File is the number of the source file containing the metrics statement.
File uint8 `align:"file"`
Reserved [3]uint8 `align:"reserved"`
}

// Value must be in sync with struct metrics_value in <bpf/lib/common.h>
Expand Down Expand Up @@ -125,6 +129,11 @@ func (k *Key) DropForwardReason() string {
return monitorAPI.DropReason(k.Reason)
}

// FileName returns the filename where the event occurred, in string format.
func (k *Key) FileName() string {
return monitorAPI.BPFFileName(k.File)
}

// IsDrop checks if the reason is drop or not.
func (k *Key) IsDrop() bool {
return k.Reason == monitorAPI.DropInvalid || k.Reason >= monitorAPI.DropMin
Expand Down
7 changes: 7 additions & 0 deletions pkg/monitor/api/files.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ var files = map[uint8]string{
2: "bpf_lxc.c",
3: "bpf_overlay.c",
4: "bpf_xdp.c",
5: "bpf_sock.c",
6: "bpf_network.c",

// header files from bpf/lib/
101: "arp.h",
Expand All @@ -23,6 +25,11 @@ var files = map[uint8]string{
105: "nodeport.h",
106: "lb.h",
107: "mcast.h",
108: "ipv4.h",
109: "conntrack.h",
110: "l3.h",
111: "trace.h",
112: "encap.h",

// @@ source files list end
}
Expand Down

0 comments on commit 794f4a6

Please sign in to comment.