Makefile: new target kind-debug to debug cilium operator & agent in kind cluster

.devcontainer/devcontainer.json

@@ -1,6 +1,6 @@
 {
   "name": "Cilium",
-  "image": "quay.io/cilium/cilium-builder:5141c18382e52cd104eb01f690b65b090072dac7@sha256:f5bb06c651e8d0b0af984dc1255f3275dd8df3086034f9d42a6e36c55c2d56da",
+  "image": "quay.io/cilium/cilium-builder:e090f844673ec4978f6d4215d13766de3c49b09c@sha256:628e3fb96527c0a61ce670e738d971f57b6ce3d06b9d1c1f02c74128eacf28e4",
   "workspaceFolder": "/go/src/github.com/cilium/cilium",
   "workspaceMount": "source=${localWorkspaceFolder},target=/go/src/github.com/cilium/cilium,type=bind",
   "features": {

.nvim/nvim-dap.lua

@@ -15,10 +15,10 @@ dap.adapters.cilium_kind_worker_2 = {
     host = "127.0.0.1",
     port = 23412
 }
-dap.adapters.cilium_kind_worker_1 = {
+dap.adapters.cilium_operator_kind_worker_1 = {
     type = "server",
     host = "127.0.0.1",
-    port = 23411
+    port = 23511
 }
 dap.configurations.go = {
     {
@@ -56,5 +56,17 @@ dap.configurations.go = {
                 to = "/go/src/github.com/cilium/cilium"
             }
         }
+    },
+    {
+        type = "cilium_operator_kind_worker_1",
+        request = "attach",
+        name = "Attach to Cilium Operator",
+        mode = "remote",
+        substitutePath = {
+            {
+                from = "${workspaceFolder}",
+                to = "/go/src/github.com/cilium/cilium"
+            }
+        }
     }
 }

.vscode/launch.json

@@ -47,5 +47,17 @@
 			"logOutput": "rpc",
 			"substitutePath": [{"from": "${workspaceFolder}", "to": "/go/src/github.com/cilium/cilium"}],
 		},
-    ]
+		{
+			"name": "Attach to Cilium Operator",
+			"type": "go",
+			"request": "attach",
+			"mode": "remote",
+			"port":23511,
+			"host":"127.0.0.1",
+			"showLog": true,
+			"trace": "log",
+			"logOutput": "rpc",
+			"substitutePath": [{"from": "${workspaceFolder}", "to": "/go/src/github.com/cilium/cilium"}],
+		}
+	]
 }

Makefile

@@ -489,7 +489,7 @@ kind-image-agent: kind-ready ## Build cilium-dev docker image and import it into
 
 $(eval $(call KIND_ENV,kind-image-operator))
 kind-image-operator: kind-ready ## Build cilium-operator-dev docker image and import it into kind.
-	$(QUIET)$(MAKE) dev-docker-operator-generic-image DOCKER_IMAGE_TAG=$(LOCAL_IMAGE_TAG)
+	$(QUIET)$(MAKE) dev-docker-operator-generic-image$(DEBUGGER_SUFFIX) DOCKER_IMAGE_TAG=$(LOCAL_IMAGE_TAG)
 	@echo "  DEPLOY image to kind ($(LOCAL_OPERATOR_IMAGE))"
 	$(QUIET)$(CONTAINER_ENGINE) push $(LOCAL_OPERATOR_IMAGE)
 	$(QUIET)kind load docker-image $(LOCAL_OPERATOR_IMAGE)
@@ -533,6 +533,9 @@ endef
 # kind-image-agent-debug
 $(eval $(call DEBUG_KIND_TEMPLATE,-agent))
 
+# kind-image-operator-debug
+$(eval $(call DEBUG_KIND_TEMPLATE,-operator))
+
 $(eval $(call KIND_ENV,kind-debug-agent))
 kind-debug-agent: ## Create a local kind development environment with cilium-agent attached to a debugger.
 	$(QUIET)$(MAKE) kind-ready 2>/dev/null \
@@ -547,6 +550,19 @@ kind-debug-agent: ## Create a local kind development environment with cilium-age
 	@echo " - 23401: cilium-agent (kind-control-plane)"
 	@echo " - 23411: cilium-agent (kind-worker)"
 
+$(eval $(call KIND_ENV,kind-debug))
+kind-debug: ## Create a local kind development environment with cilium-agent & cilium-operator attached to a debugger.
+	$(QUIET)$(MAKE) kind-ready 2>/dev/null \
+		|| $(MAKE) kind
+	$(MAKE) kind-image-agent-debug
+	$(MAKE) kind-image-operator-debug
+	$(MAKE) kind-check-cilium 2>/dev/null \
+		|| $(MAKE) kind-install-cilium
+	@echo "Attach delve to localhost on these ports to continue:"
+	@echo " - 23401: cilium-agent    (kind-control-plane)"
+	@echo " - 23411: cilium-agent    (kind-worker)"
+	@echo " - 23511: cilium-operator (kind-worker)"
+
 precheck: check-go-version logging-subsys-field ## Peform build precheck for the source code.
 ifeq ($(SKIP_K8S_CODE_GEN_CHECK),"false")
 	@$(ECHO_CHECK) contrib/scripts/check-k8s-code-gen.sh

Makefile.docker

@@ -49,7 +49,7 @@ GIT_IGNORE_FILES := $(shell find . -not -path "./vendor*" -name .gitignore -prin
 	@echo "/hack" > $@
 	@echo ".git" >> $@
 	@echo "/Makefile.docker" >> $@
-	echo $(dir $(GIT_IGNORE_FILES)) | tr ' ' '\n' | xargs -P1 -n1 -I {DIR} sed \
+	@echo $(dir $(GIT_IGNORE_FILES)) | tr ' ' '\n' | xargs -P1 -n1 -I {DIR} sed \
 		-e '# Remove lines with white space, comments and files that must be passed to docker, "$$" due to make. #' \
 			-e '/^[[:space:]]*$$/d' -e '/^#/d' -e '/GIT_VERSION/d' \
 		-e '# Apply pattern in all directories if it contains no "/", keep "!" up front. #' \
@@ -97,6 +97,7 @@ endif
 		--build-arg LIBNETWORK_PLUGIN=${LIBNETWORK_PLUGIN} \
 		--build-arg CILIUM_SHA=$(firstword $(GIT_VERSION)) \
 		--build-arg OPERATOR_VARIANT=$(IMAGE_NAME) \
+		--build-arg DEBUG_HOLD=$(DEBUG_HOLD) \
 		--target $(5) \
 		-t $(IMAGE_REPOSITORY)/$(IMAGE_NAME)$${UNSTRIPPED}$(DOCKER_IMAGE_SUFFIX):$(4) .
 ifneq ($(KIND_LOAD),)
@@ -140,6 +141,7 @@ $(eval $(call DOCKER_IMAGE_TEMPLATE,docker-clustermesh-apiserver-image,images/cl
 # to build also 'docker-operator-image', where the stem would be empty otherwise.
 $(eval $(call DOCKER_IMAGE_TEMPLATE,docker-opera%-image,images/operator/Dockerfile,opera%,$(DOCKER_IMAGE_TAG),release))
 $(eval $(call DOCKER_IMAGE_TEMPLATE,dev-docker-opera%-image,images/operator/Dockerfile,opera%,$(DOCKER_IMAGE_TAG),release))
+$(eval $(call DOCKER_IMAGE_TEMPLATE,dev-docker-opera%-image-debug,images/operator/Dockerfile,opera%,$(DOCKER_IMAGE_TAG),debug))
 
 #
 # docker-*-all targets are mainly used from the CI

api/v1/Makefile

@@ -3,7 +3,7 @@
 include ../../Makefile.defs
 
 # Update this via images/scripts/update-cilium-builder-image.sh
-CILIUM_BUILDER_IMAGE=quay.io/cilium/cilium-builder:5141c18382e52cd104eb01f690b65b090072dac7@sha256:f5bb06c651e8d0b0af984dc1255f3275dd8df3086034f9d42a6e36c55c2d56da
+CILIUM_BUILDER_IMAGE=quay.io/cilium/cilium-builder:e090f844673ec4978f6d4215d13766de3c49b09c@sha256:628e3fb96527c0a61ce670e738d971f57b6ce3d06b9d1c1f02c74128eacf28e4
 
 .PHONY: proto
 proto:

contrib/scripts/kind.sh

@@ -75,7 +75,8 @@ if [[ -n "${image}" ]]; then
 fi
 
 node_config() {
-    local port="234$1$2"
+    local agentDebugPort="234$1$2"
+    local operatorDebugPort="235$1$2"
     local max="$3"
 
     echo "  extraMounts:"
@@ -84,7 +85,11 @@ node_config() {
     if [[ "${max}" -lt 10 ]]; then
         echo "  extraPortMappings:"
         echo "  - containerPort: 2345"
-        echo "    hostPort: $port"
+        echo "    hostPort: $agentDebugPort"
+        echo "    listenAddress: \"127.0.0.1\""
+        echo "    protocol: TCP"
+        echo "  - containerPort: 2346"
+        echo "    hostPort: $operatorDebugPort"
         echo "    listenAddress: \"127.0.0.1\""
         echo "    protocol: TCP"
     fi

contrib/testing/kind-values.yaml

@@ -6,6 +6,9 @@ operator:
   image:
     override: "localhost:5000/cilium/operator-generic:local"
     suffix: ""
+  nodeSelector:
+    kubernetes.io/os: linux
+    kubernetes.io/hostname: kind-worker
 ipam:
   mode: kubernetes
 ipv6:

images/builder/Dockerfile

@@ -56,7 +56,13 @@ ENV GOROOT /usr/local/go
 ENV GOPATH /go
 ENV PATH "${GOROOT}/bin:${GOPATH}/bin:${PATH}"
 
-RUN go install github.com/go-delve/delve/cmd/dlv@latest
+RUN CGO_ENABLED=0 go install github.com/go-delve/delve/cmd/dlv@latest
+
+WORKDIR /go/src/github.com/cilium/cilium/images/builder
+RUN --mount=type=bind,readwrite,target=/go/src/github.com/cilium/cilium/images/builder \
+    --mount=type=cache,target=/root/.cache \
+    --mount=type=cache,target=/go/pkg \
+    ./build-debug-wrapper.sh
 
 WORKDIR /go/src/github.com/cilium/cilium/images/builder
 RUN --mount=type=bind,readwrite,target=/go/src/github.com/cilium/cilium/images/builder \

images/builder/build-debug-wrapper.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Copyright Authors of Cilium
+# SPDX-License-Identifier: Apache-2.0
+
+set -o xtrace
+set -o errexit
+set -o pipefail
+set -o nounset
+
+SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+for arch in amd64 arm64 ; do
+  mkdir -p "/out/linux/${arch}/bin"
+  GOARCH="${arch}" CGO_ENABLED=0 go build -ldflags "-s -w" -o "/out/linux/${arch}/bin/debug-wrapper" "${SCRIPT_DIR}/debug-wrapper.go"
+done

images/builder/debug-wrapper.go

@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Cilium
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+)
+
+func main() {
+	bin, err := os.Executable()
+	if err != nil {
+		log.Println(err)
+		os.Exit(1)
+	}
+
+	debugPort, ok := os.LookupEnv("DEBUG_PORT")
+	if !ok || debugPort == "" {
+		debugPort = "2345"
+	}
+
+	debugContinue := true
+	if debugHold, ok := os.LookupEnv("DEBUG_HOLD"); ok && debugHold == "true" {
+		debugContinue = false
+	}
+
+	args := []string{
+		fmt.Sprintf("--listen=:%s", debugPort),
+		"--headless=true",
+		fmt.Sprintf("--continue=%t", debugContinue),
+		"--log=true",
+		"--log-output=debugger,debuglineerr,gdbwire,lldbout,rpc",
+		"--accept-multiclient",
+		"--api-version=2",
+		"exec",
+		fmt.Sprintf("%s-bin", bin),
+		"--",
+	}
+
+	args = append(args, os.Args[1:]...)
+
+	cmd := exec.Command("/usr/bin/dlv", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	if err = cmd.Run(); err != nil {
+		if exitErr := (&exec.ExitError{}); errors.As(err, &exitErr) {
+			os.Exit(exitErr.ExitCode())
+		} else {
+			log.Printf("failed to execute dlv: %v", err)
+			os.Exit(1)
+		}
+	}
+}

images/cilium/Dockerfile

@@ -1,10 +1,9 @@
 # Copyright Authors of Cilium
 # SPDX-License-Identifier: Apache-2.0
 
-ARG CILIUM_BUILDER_IMAGE=quay.io/cilium/cilium-builder:5141c18382e52cd104eb01f690b65b090072dac7@sha256:f5bb06c651e8d0b0af984dc1255f3275dd8df3086034f9d42a6e36c55c2d56da
+ARG CILIUM_BUILDER_IMAGE=quay.io/cilium/cilium-builder:e090f844673ec4978f6d4215d13766de3c49b09c@sha256:628e3fb96527c0a61ce670e738d971f57b6ce3d06b9d1c1f02c74128eacf28e4
 ARG CILIUM_RUNTIME_IMAGE=quay.io/cilium/cilium-runtime:f0c4a2c2ce4af34136027dbe184053f429b8a556@sha256:479d828e08389c4e27403ec88115a2dddbb9f934b2c393b891813d6e3ba81253
 
-
 # cilium-envoy from github.com/cilium/proxy
 #
 FROM quay.io/cilium/cilium-envoy:505865fff55aa3e7e3e490044c428651e0fe339c@sha256:10a5e0fe552d3312753796e98a07366826ae4863bb3f5e704e597b2a21ceceec as cilium-envoy
@@ -106,6 +105,8 @@ FROM release as debug
 ARG TARGETOS
 # TARGETARCH is an automatic platform ARG enabled by Docker BuildKit.
 ARG TARGETARCH
+ARG DEBUG_HOLD
+ENV DEBUG_HOLD=${DEBUG_HOLD}
+COPY --from=builder /tmp/install/${TARGETOS}/${TARGETARCH}/usr/bin/cilium-agent /usr/bin/cilium-agent-bin
 COPY --from=builder /go/bin/dlv /usr/bin/dlv
-RUN mv /usr/bin/cilium-agent /usr/bin/cilium-agent-bin
-COPY images/scripts/debug-wrapper.sh /usr/bin/cilium-agent
+COPY --from=builder /out/${TARGETOS}/${TARGETARCH}/bin/debug-wrapper /usr/bin/cilium-agent

images/operator/Dockerfile

@@ -4,6 +4,7 @@
 ARG BASE_IMAGE=scratch
 ARG GOLANG_IMAGE=docker.io/library/golang:1.20.1@sha256:2edf6aab2d57644f3fe7407132a0d1770846867465a39c2083770cf62734b05d
 ARG ALPINE_IMAGE=docker.io/library/alpine:3.17.2@sha256:69665d02cb32192e52e07644d76bc6f25abeb5410edc1c7a81a10ba3f0efb90a
+ARG CILIUM_BUILDER_IMAGE=quay.io/cilium/cilium-builder:e090f844673ec4978f6d4215d13766de3c49b09c@sha256:628e3fb96527c0a61ce670e738d971f57b6ce3d06b9d1c1f02c74128eacf28e4
 
 # BUILDPLATFORM is an automatic platform ARG enabled by Docker BuildKit.
 # Represents the plataform where the build is happening, do not mix with
@@ -69,3 +70,26 @@ COPY --from=builder /out/${TARGETOS}/${TARGETARCH}/LICENSE.all /LICENSE.all
 WORKDIR /
 ENV GOPS_CONFIG_DIR=/
 CMD ["/usr/bin/cilium-${OPERATOR_VARIANT}"]
+
+FROM --platform=${BUILDPLATFORM} ${CILIUM_BUILDER_IMAGE} as debug-tools
+
+#
+# Cilium Operator debug image.
+#
+# Typical image bulids will stop above at the 'release' target, but
+# developers follow this Dockerfile to the end. Starting from a release
+# image, install delve debugger and wrap the cilium-operator binary calls
+# with a script that automatically provisions the debugger on a
+# dedicated port.
+FROM release as debug
+# TARGETOS is an automatic platform ARG enabled by Docker BuildKit.
+ARG TARGETOS
+# TARGETARCH is an automatic platform ARG enabled by Docker BuildKit.
+ARG TARGETARCH
+ARG OPERATOR_VARIANT
+ARG DEBUG_HOLD
+ENV DEBUG_PORT=2346
+ENV DEBUG_HOLD=${DEBUG_HOLD}
+COPY --from=builder /out/${TARGETOS}/${TARGETARCH}/usr/bin/cilium-${OPERATOR_VARIANT} /usr/bin/cilium-${OPERATOR_VARIANT}-bin
+COPY --from=debug-tools /go/bin/dlv /usr/bin/dlv
+COPY --from=debug-tools /out/${TARGETOS}/${TARGETARCH}/bin/debug-wrapper /usr/bin/cilium-${OPERATOR_VARIANT}

test/k8s/manifests/demo-customcalls.yaml

@@ -99,7 +99,7 @@ metadata:
 spec:
   containers:
   - name: cilium-builder
-    image: quay.io/cilium/cilium-builder:5141c18382e52cd104eb01f690b65b090072dac7@sha256:f5bb06c651e8d0b0af984dc1255f3275dd8df3086034f9d42a6e36c55c2d56da
+    image: quay.io/cilium/cilium-builder:e090f844673ec4978f6d4215d13766de3c49b09c@sha256:628e3fb96527c0a61ce670e738d971f57b6ce3d06b9d1c1f02c74128eacf28e4
     workingDir: /cilium
     command: ["sleep"]
     args: