maint: fix debug address (#185)

## Which problem is this PR solving?

- connection refused when trying to access port 6060

## Short description of the changes

- adjust hard-coded address from `localhost:6060` to `0.0.0.0:6060`
- add local testing setup for using pyroscope

## How to verify that this has the expected result

Uncomment the pyroscope pieces in `deployment.yaml` and `DEBUG=true`.
```sh
make docker-build
make apply-pyroscope-server
export POD_NAME=$(kubectl get pods --namespace default -l "app.kubernetes.io/name=pyroscope,app.kubernetes.io/instance=pyroscope" -o jsonpath="{.items[0].metadata.name}")
export CONTAINER_PORT=$(kubectl get pod --namespace default $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
kubectl --namespace default port-forward $POD_NAME 4040:$CONTAINER_PORT
make apply-agent
```
navigate to `http://localhost:4040/` and see scrape targets with health
up and details of last scrape. Go to Continuous Profiling / Tag Explorer
to see flamegraph.

Makefile

@@ -83,3 +83,22 @@ swarm: apply-agent apply-echoserver
 .PHONY: unswarm
 #: teardown load test agent and echoserver
 unswarm: unapply-echoserver unapply-agent
+
+.PHONY: apply-pyroscope-server
+#: spin up a pyroscope server in k8s cluster
+apply-pyroscope-server:
+	helm repo add pyroscope-io https://pyroscope-io.github.io/helm-chart
+	helm repo update
+	helm install pyroscope pyroscope-io/pyroscope -f smoke-tests/pyroscope_values.yaml
+
+.PHONY: port-forward-pyroscope
+#: port forward pyroscope server to localhost:4040. doesnt work, run manually.
+port-forward-pyroscope:
+	export POD_NAME=$(kubectl get pods --namespace default -l "app.kubernetes.io/name=pyroscope,app.kubernetes.io/instance=pyroscope" -o jsonpath="{.items[0].metadata.name}")
+	export CONTAINER_PORT=$(kubectl get pod --namespace default $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
+	kubectl --namespace default port-forward $POD_NAME 4040:$CONTAINER_PORT
+
+.PHONY: unapply-pyroscope-server
+#: tear down pyroscope server
+unapply-pyroscope-server:
+	helm uninstall pyroscope

config/config.go

@@ -10,7 +10,7 @@ import (
 )
 
 // TODO hard-coded for now, make configurable
-const DebugAddr = "localhost:6060"
+const DebugAddr = "0.0.0.0:6060"
 
 var maxcount = flag.Int("c", -1, "Only grab this many packets, then exit")
 var statsevery = flag.Int("stats", 1000, "Output statistics every N packets")

debug/debug_service.go

@@ -20,7 +20,7 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-const addr = "localhost:6060"
+const addr = "0.0.0.0:6060"
 
 // injectable debug service
 type DebugService struct {

smoke-tests/deployment.yaml

@@ -72,6 +72,13 @@ spec:
     metadata:
       labels:
         name: hny-network-agent
+        # add these annotations to enable profiling
+        # annotations:
+        #   pyroscope.io/scrape: 'true'
+        #   pyroscope.io/application-name: 'snoopy'
+        #   pyroscope.io/profile-cpu-enabled: 'true'
+        #   pyroscope.io/profile-mem-enabled: 'true'
+        #   pyroscope.io/port: '6060'
     spec:
       serviceAccountName: honeycomb-sa
       hostNetwork: true

smoke-tests/pyroscope_values.yaml

@@ -0,0 +1,67 @@
+---
+image:
+  repository: pyroscope/pyroscope
+  tag: 0.33.0
+  pullPolicy: IfNotPresent
+# Create Cluster Role and bind it to Pyroscope to enable it watching Kubernetes resources.
+rbac:
+  create: true
+# Pyroscope configuration.
+pyroscopeConfigs:
+  log-level: debug
+  scrape-configs:
+    # Example scrape config for pods
+    #
+    # The relabeling allows the actual pod scrape endpoint to be configured via the
+    # following annotations:
+    #
+    # * `pyroscope.io/scrape`: Only scrape pods that have a value of `true`.
+    # * `pyroscope.io/application-name`: Name of the application being profiled.
+    # * `pyroscope.io/scheme`: If the metrics endpoint is secured then you will need
+    # to set this to `https` & most likely set the `tls_config` of the scrape config.
+    # * `pyroscope.io/port`: Scrape the pod on the indicated port.
+    # * `pyroscope.io/profile-{profile_name}-path`: Specifies URL path exposing pprof profile.
+    # * `pyroscope.io/profile-{profile_name}-param-{param_key}`: Overrides scrape URL parameters.
+    #
+    # Kubernetes labels will be added as Pyroscope labels on metrics via the
+    # `labelmap` relabeling action.
+    - job-name: 'kubernetes-pods'
+      enabled-profiles: [cpu, mem]
+      kubernetes-sd-configs:
+        - role: pod
+      relabel-configs:
+        - source-labels: [__meta_kubernetes_pod_annotation_pyroscope_io_scrape]
+          action: keep
+          regex: true
+        - source-labels:
+            [__meta_kubernetes_pod_annotation_pyroscope_io_application_name]
+          action: replace
+          target-label: __name__
+        - source-labels:
+            [__meta_kubernetes_pod_annotation_pyroscope_io_spy_name]
+          action: replace
+          target-label: __spy_name__
+        - source-labels: [__meta_kubernetes_pod_annotation_pyroscope_io_scheme]
+          action: replace
+          regex: (https?)
+          target-label: __scheme__
+        - source-labels:
+            [__address__, __meta_kubernetes_pod_annotation_pyroscope_io_port]
+          action: replace
+          regex: ([^:]+)(?::\d+)?;(\d+)
+          replacement: $1:$2
+          target-label: __address__
+        - action: labelmap
+          regex: __meta_kubernetes_pod_label_(.+)
+        - source-labels: [__meta_kubernetes_namespace]
+          action: replace
+          target-label: kubernetes_namespace
+        - source-labels: [__meta_kubernetes_pod_name]
+          action: replace
+          target-label: kubernetes_pod_name
+        - source-labels: [__meta_kubernetes_pod_phase]
+          regex: Pending|Succeeded|Failed|Completed
+          action: drop
+        - action: labelmap
+          regex: __meta_kubernetes_pod_annotation_pyroscope_io_profile_(.+)
+          replacement: __profile_$1