Skip to content

Commit

Permalink
feat: Add more Kubernetes metadata (#46)
Browse files Browse the repository at this point in the history
## Which problem is this PR solving?
Adds more Kubernetes metadata to events. Followed the conventions listed
in OTel:
https://opentelemetry.io/docs/specs/otel/resource/semantic_conventions/k8s/

- Closes #49 

## Short description of the changes
Added:
- source / dest UIDs
- namespace name
- service name
- node name / UID
- container name

TODO:
- ~[ ] Figure out how to get cluster name~
- [x] Check that service name is working correctly
- [x] Use semconv package for field names
- [x] Normalize source / dest IP names
- ~[ ] Write unit tests for k8s metadata transformation functions~



## How to verify that this has the expected result
- Run in a k8s cluster and send data to Honeycomb
- Make sure to run greeting service to see more interesting data
![Screenshot 2023-08-01 at 5 56 30
PM](https://github.com/honeycombio/honeycomb-ebpf-agent/assets/8810222/3c373f01-d1e7-47c6-ab2e-35f9b25218ae)

Data being emitted:


![Screenshot 2023-08-03 at 3 11 01
PM](https://github.com/honeycombio/honeycomb-ebpf-agent/assets/8810222/76081824-cf91-4303-a165-b9d168b10f18)
  • Loading branch information
pkanal committed Aug 3, 2023
1 parent b0bb6c8 commit 0384f29
Show file tree
Hide file tree
Showing 4 changed files with 101 additions and 24 deletions.
92 changes: 70 additions & 22 deletions bpf/probes/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,21 +9,23 @@ import (
"log"
"net"
"os"
"strings"

"github.com/cilium/ebpf/link"
"github.com/cilium/ebpf/perf"
"github.com/honeycombio/libhoney-go"
semconv "go.opentelemetry.io/otel/semconv/v1.20.0"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)

//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -target amd64,arm64 -cc clang -cflags $CFLAGS bpf tcp_probe.c

const mapKey uint32 = 0

func Setup() {
func Setup(client *kubernetes.Clientset) {
// Load pre-compiled programs and maps into the kernel.
objs := bpfObjects{}
if err := loadBpfObjects(&objs, nil); err != nil {
Expand Down Expand Up @@ -74,22 +76,11 @@ func Setup() {

// log.Printf("event: %+v\n", event)

sendEvent(event)
sendEvent(event, client)
}
}

func getPodByIPAddr(ipAddr string) v1.Pod {
// creates the in-cluster config
config, err := rest.InClusterConfig()
if err != nil {
panic(err.Error())
}
// creates the clientset
client, err := kubernetes.NewForConfig(config)
if err != nil {
panic(err.Error())
}

func getPodByIPAddr(client *kubernetes.Clientset, ipAddr string) v1.Pod {
pods, _ := client.CoreV1().Pods(v1.NamespaceAll).List(context.TODO(), metav1.ListOptions{})

var matchedPod v1.Pod
Expand All @@ -103,22 +94,79 @@ func getPodByIPAddr(ipAddr string) v1.Pod {
return matchedPod
}

func getServiceForPod(client *kubernetes.Clientset, inputPod v1.Pod) v1.Service {
// get list of services
services, _ := client.CoreV1().Services(v1.NamespaceAll).List(context.TODO(), metav1.ListOptions{})
var matchedService v1.Service
// loop over services
for _, service := range services.Items {
set := labels.Set(service.Spec.Selector)
listOptions := metav1.ListOptions{LabelSelector: set.AsSelector().String()}
pods, err := client.CoreV1().Pods(v1.NamespaceAll).List(context.TODO(), listOptions)
if err != nil {
log.Println(err)
}
for _, pod := range pods.Items {
if pod.Name == inputPod.Name {
matchedService = service
}
}
}

return matchedService
}

func getNodeByPod(client *kubernetes.Clientset, pod v1.Pod) *v1.Node {
node, _ := client.CoreV1().Nodes().Get(context.TODO(), pod.Spec.NodeName, metav1.GetOptions{})
return node
}

// Send event to Honeycomb
func sendEvent(event bpfTcpEvent) {
func sendEvent(event bpfTcpEvent, client *kubernetes.Clientset) {

sourceIpAddr := intToIP(event.Saddr).String()
destIpAddr := intToIP(event.Daddr).String()

destPod := getPodByIPAddr(destIpAddr)
sourcePod := getPodByIPAddr(sourceIpAddr)
destPod := getPodByIPAddr(client, destIpAddr)
sourcePod := getPodByIPAddr(client, sourceIpAddr)
sourceNode := getNodeByPod(client, sourcePod)

ev := libhoney.NewEvent()
ev.AddField("name", "tcp_event")
ev.AddField("duration_ms", (event.EndTime-event.StartTime)/1_000_000) // convert ns to ms
ev.AddField("source", fmt.Sprintf("%s:%d", sourceIpAddr, event.Sport))
ev.AddField("dest", fmt.Sprintf("%s:%d", destIpAddr, event.Dport))
ev.AddField("k8s.pod.dest.name", destPod.Name)
ev.AddField("k8s.pod.source.name", sourcePod.Name)
// IP Address / port
ev.AddField(string(semconv.NetSockHostAddrKey), sourceIpAddr)
ev.AddField("destination.address", destIpAddr)
ev.AddField(string(semconv.NetHostPortKey), event.Sport)
ev.AddField("destination.port", event.Dport)

// dest pod
ev.AddField(fmt.Sprintf("destination.%s", semconv.K8SPodNameKey), destPod.Name)
ev.AddField(fmt.Sprintf("destination.%s", semconv.K8SPodUIDKey), destPod.UID)

// source pod
ev.AddField(string(semconv.K8SPodNameKey), sourcePod.Name)
ev.AddField(string(semconv.K8SPodUIDKey), sourcePod.UID)

// namespace
ev.AddField(string(semconv.K8SNamespaceNameKey), sourcePod.Namespace)

// service
// no semconv for service yet
ev.AddField("k8s.service.name", getServiceForPod(client, sourcePod).Name)

// node
ev.AddField(string(semconv.K8SNodeNameKey), sourceNode.Name)
ev.AddField(string(semconv.K8SNodeUIDKey), sourceNode.UID)

// container names
if len(sourcePod.Spec.Containers) > 0 {
var containerNames []string
for _, container := range sourcePod.Spec.Containers {
containerNames = append(containerNames, container.Name)
}
ev.AddField(string(semconv.K8SContainerNameKey), strings.Join(containerNames, ","))
}

err := ev.Send()
if err != nil {
Expand Down
6 changes: 5 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ require (
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01 // indirect
github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.1 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
Expand All @@ -35,6 +36,9 @@ require (
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
go.opentelemetry.io/otel v1.16.0 // indirect
go.opentelemetry.io/otel/metric v1.16.0 // indirect
go.opentelemetry.io/otel/trace v1.16.0 // indirect
golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
Expand Down
11 changes: 11 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,13 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8=
Expand Down Expand Up @@ -208,6 +213,12 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s=
go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4=
go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo=
go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4=
go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs=
go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
Expand Down
16 changes: 15 additions & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import (
"github.com/honeycombio/ebpf-agent/bpf/probes"
"github.com/honeycombio/ebpf-agent/utils"
"github.com/honeycombio/libhoney-go"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)

const Version string = "0.0.2"
Expand Down Expand Up @@ -54,8 +56,20 @@ func main() {

defer libhoney.Close()

// creates the in-cluster config
config, err := rest.InClusterConfig()
if err != nil {
panic(err.Error())
}
// creates the clientset
client, err := kubernetes.NewForConfig(config)

if err != nil {
panic(err.Error())
}

// setup probes
probes.Setup()
probes.Setup(client)
}

func getEnvOrDefault(key string, defaultValue string) string {
Expand Down

0 comments on commit 0384f29

Please sign in to comment.