Refactor generate-k8s-api in Makefile

.github/workflows/lint-go.yaml

@@ -111,8 +111,6 @@ jobs:
           # Set GOBIN to ensure 'go install' binaries end up in the same directory
           # as the one actions/setup-go adds to PATH, regardless of GOPATH.
           export GOBIN="$HOME/go/bin"
-          # check-k8s-code-gen.sh needs a GOPATH set that contains the Cilium repo at src/github.com/cilium/cilium.
-          export GOPATH="$GITHUB_WORKSPACE"
 
           cd src/github.com/cilium/cilium
           contrib/scripts/check-k8s-code-gen.sh

Makefile

@@ -55,63 +55,6 @@ TEST_LDFLAGS=-ldflags "-X github.com/cilium/cilium/pkg/kvstore.consulDummyAddres
 
 TEST_UNITTEST_LDFLAGS=-ldflags "-X github.com/cilium/cilium/pkg/datapath.DatapathSHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
 
-define generate_k8s_api
-	cd "./vendor/k8s.io/code-generator" && \
-	GO111MODULE=off bash ./generate-groups.sh $(1) \
-	    $(2) \
-	    $(3) \
-	    $(4) \
-	    --go-header-file "$(PWD)/hack/custom-boilerplate.go.txt"
-endef
-
-define generate_deepequal
-	go run github.com/cilium/deepequal-gen \
-	--input-dirs $(1) \
-	-O zz_generated.deepequal \
-	--go-header-file "$(PWD)/hack/custom-boilerplate.go.txt"
-endef
-
-define generate_k8s_api_all
-	$(call generate_k8s_api,all,github.com/cilium/cilium/pkg/k8s/client,$(1),$(2))
-	$(call generate_deepequal,"$(call join-with-comma,$(foreach pkg,$(2),$(1)/$(subst ",,$(subst :,/,$(pkg)))))")
-endef
-
-define generate_k8s_api_deepcopy_deepequal
-	$(call generate_k8s_api,deepcopy,github.com/cilium/cilium/pkg/k8s/client,$(1),$(2))
-	@# Explanation for the 'subst' below:
-	@#   $(subst ",,$(subst :,/,$(pkg))) - replace all ':' with '/' and replace
-	@#    all '"' with '' from $pkg
-	@#   $(foreach pkg,$(2),$(1)/$(subst ",,$(subst :,/,$(pkg)))) - for each
-	@#    "$pkg", with the characters replaced, create a new string with the
-	@#    prefix $(1)
-	@#   Finally replace all spaces with commas from the generated strings.
-	$(call generate_deepequal,"$(call join-with-comma,$(foreach pkg,$(2),$(1)/$(subst ",,$(subst :,/,$(pkg)))))")
-endef
-
-define generate_k8s_api_deepcopy_deepequal_client
-	$(call generate_k8s_api,deepcopy$(comma)client,github.com/cilium/cilium/pkg/k8s/slim/k8s/$(1),$(2),$(3))
-	$(call generate_deepequal,"$(call join-with-comma,$(foreach pkg,$(3),$(2)/$(subst ",,$(subst :,/,$(pkg)))))")
-endef
-
-define generate_k8s_protobuf
-	go install k8s.io/code-generator/cmd/go-to-protobuf/protoc-gen-gogo
-	go install golang.org/x/tools/cmd/goimports
-
-	go run k8s.io/code-generator/cmd/go-to-protobuf \
-		--apimachinery-packages='-k8s.io/apimachinery/pkg/util/intstr,$\
-                                -k8s.io/apimachinery/pkg/api/resource,$\
-                                -k8s.io/apimachinery/pkg/runtime/schema,$\
-                                -k8s.io/apimachinery/pkg/runtime,$\
-                                -k8s.io/apimachinery/pkg/apis/meta/v1,$\
-                                -k8s.io/apimachinery/pkg/apis/meta/v1beta1'\
-		--drop-embedded-fields="github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1.TypeMeta" \
-		--proto-import="$(PWD)" \
-		--proto-import="$(PWD)/vendor" \
-		--proto-import="$(PWD)/tools/protobuf" \
-		--packages=$(1) \
-		--go-header-file "$(PWD)/hack/custom-boilerplate.go.txt"
-endef
-
 build: check-sources $(SUBDIRS) ## Builds all the components for Cilium by executing make in the respective sub directories.
 
 build-container: check-sources ## Builds components required for cilium-agent container.
@@ -317,72 +260,81 @@ generate-operator-api: api/v1/operator/openapi.yaml ## Generate cilium-operator
 generate-hubble-api: api/v1/flow/flow.proto api/v1/peer/peer.proto api/v1/observer/observer.proto api/v1/relay/relay.proto ## Generate hubble proto Go sources.
 	$(QUIET) $(MAKE) $(SUBMAKEOPTS) -C api/v1
 
+define generate_k8s_api
+	$(QUIET) cd "./vendor/k8s.io/code-generator" && \
+	bash ./generate-groups.sh $(1) \
+	    $(2) \
+	    $(3) \
+	    $(4) \
+	    --go-header-file "$(PWD)/hack/custom-boilerplate.go.txt" \
+	    --output-base $(5)
+endef
+
+define generate_deepequal
+	$(GO) run github.com/cilium/deepequal-gen \
+	--input-dirs $(subst $(space),$(comma),$(1)) \
+	--go-header-file "$(PWD)/hack/custom-boilerplate.go.txt" \
+	--output-file-base zz_generated.deepequal \
+	--output-base $(2)
+endef
+
+define generate_deepcopy
+	$(GO) run k8s.io/code-generator/cmd/deepcopy-gen \
+	--input-dirs $(subst $(space),$(comma),$(1)) \
+	--go-header-file "$(PWD)/hack/custom-boilerplate.go.txt" \
+	--output-file-base zz_generated.deepcopy \
+	--output-base $(2)
+endef
+
+define generate_k8s_protobuf
+	$(GO) install k8s.io/code-generator/cmd/go-to-protobuf/protoc-gen-gogo && \
+	$(GO) install golang.org/x/tools/cmd/goimports && \
+	$(GO) run k8s.io/code-generator/cmd/go-to-protobuf \
+		--apimachinery-packages='-k8s.io/apimachinery/pkg/util/intstr,$\
+                                -k8s.io/apimachinery/pkg/api/resource,$\
+                                -k8s.io/apimachinery/pkg/runtime/schema,$\
+                                -k8s.io/apimachinery/pkg/runtime,$\
+                                -k8s.io/apimachinery/pkg/apis/meta/v1,$\
+                                -k8s.io/apimachinery/pkg/apis/meta/v1beta1'\
+		--drop-embedded-fields="github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1.TypeMeta" \
+		--proto-import="$(PWD)" \
+		--proto-import="$(PWD)/vendor" \
+		--proto-import="$(PWD)/tools/protobuf" \
+		--packages=$(subst $(newline),$(comma),$(1)) \
+		--go-header-file "$(PWD)/hack/custom-boilerplate.go.txt" \
+		--output-base=$(2)
+endef
+
+define K8S_PROTO_PACKAGES
+github.com/cilium/cilium/pkg/k8s/slim/k8s/api/core/v1
+github.com/cilium/cilium/pkg/k8s/slim/k8s/api/discovery/v1
+github.com/cilium/cilium/pkg/k8s/slim/k8s/api/discovery/v1beta1
+github.com/cilium/cilium/pkg/k8s/slim/k8s/api/networking/v1
+github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/apiextensions/v1
+github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1
+github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1beta1
+github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/util/intstr
+endef
+
 generate-k8s-api: ## Generate Cilium k8s API client, deepcopy and deepequal Go sources.
 	$(ASSERT_CILIUM_MODULE)
 
-	$(call generate_k8s_protobuf,$\
-	github.com/cilium/cilium/pkg/k8s/slim/k8s/api/core/v1$(comma)$\
-	github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1$(comma)$\
-	github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1beta1$(comma)$\
-	github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/util/intstr$(comma)$\
-	github.com/cilium/cilium/pkg/k8s/slim/k8s/api/discovery/v1$(comma)$\
-	github.com/cilium/cilium/pkg/k8s/slim/k8s/api/discovery/v1beta1$(comma)$\
-	github.com/cilium/cilium/pkg/k8s/slim/k8s/api/networking/v1$(comma)$\
-	github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/apiextensions/v1)
-	$(call generate_k8s_api_deepcopy_deepequal_client,client,github.com/cilium/cilium/pkg/k8s/slim/k8s/api,"$\
-	discovery:v1beta1\
-	discovery:v1\
-	networking:v1\
-	core:v1")
-	$(call generate_k8s_api_deepcopy_deepequal_client,apiextensions-client,github.com/cilium/cilium/pkg/k8s/slim/k8s/apis,"$\
-	apiextensions:v1")
-	$(call generate_k8s_api_deepcopy_deepequal,github.com/cilium/cilium/pkg/k8s/slim/k8s/apis,"$\
-	util:intstr\
-	meta:v1\
-	meta:v1beta1")
-	$(call generate_k8s_api_deepcopy_deepequal,github.com/cilium/cilium/pkg/k8s/slim/k8s,"$\
-	apis:labels")
-	$(call generate_k8s_api_deepcopy_deepequal,github.com/cilium/cilium/pkg,"$\
-	aws:types\
-	azure:types\
-	ipam:types\
-	alibabacloud:types\
-	k8s:types\
-	k8s:utils\
-	maps:auth\
-	maps:ctmap\
-	maps:encrypt\
-	maps:eppolicymap\
-	maps:eventsmap\
-	maps:fragmap\
-	maps:ipcache\
-	maps:ipmasq\
-	maps:lbmap\
-	maps:lxcmap\
-	maps:metricsmap\
-	maps:nat\
-	maps:neighborsmap\
-	maps:policymap\
-	maps:signalmap\
-	maps:sockmap\
-	maps:srv6map\
-	maps:tunnel\
-	maps:vtep\
-	node:types\
-	policy:api\
-	service:store")
-	$(call generate_k8s_api_deepcopy_deepequal,github.com/cilium/cilium/pkg/policy,"api:kafka")
-	$(call generate_k8s_api_all,github.com/cilium/cilium/pkg/k8s/apis,"cilium.io:v2 cilium.io:v2alpha1")
-	$(call generate_k8s_api_deepcopy_deepequal,github.com/cilium/cilium/pkg/aws,"eni:types")
-	$(call generate_k8s_api_deepcopy_deepequal,github.com/cilium/cilium/pkg/alibabacloud,"eni:types")
-	$(call generate_k8s_api_deepcopy_deepequal,github.com/cilium/cilium/api,"v1:models")
-	$(call generate_k8s_api_deepcopy_deepequal,github.com/cilium/cilium,"$\
-	pkg:bpf\
-	pkg:k8s\
-	pkg:labels\
-	pkg:loadbalancer\
-	pkg:tuple\
-	pkg:recorder")
+	$(eval TMPDIR := $(shell mktemp -d))
+
+	$(QUIET) $(call generate_k8s_protobuf,${K8S_PROTO_PACKAGES},"$(TMPDIR)")
+
+	$(eval DEEPEQUAL_PACKAGES := $(shell grep "\+deepequal-gen" -l -r --include \*.go --exclude-dir 'vendor' . | xargs dirname {} | sort | uniq | grep -x -v '.' | sed 's|\.\/|github.com/cilium/cilium\/|g'))
+	$(QUIET) $(call generate_deepequal,${DEEPEQUAL_PACKAGES},"$(TMPDIR)")
+
+	$(eval DEEPCOPY_PACKAGES := $(shell grep "\+k8s:deepcopy-gen" -l -r --include \*.go --exclude-dir 'vendor' . | xargs dirname {} | sort | uniq | grep -x -v '.' | sed 's|\.\/|github.com/cilium/cilium\/|g'))
+	$(QUIET) $(call generate_deepcopy,${DEEPCOPY_PACKAGES},"$(TMPDIR)")
+
+	$(QUIET) $(call generate_k8s_api,client,github.com/cilium/cilium/pkg/k8s/slim/k8s/client,github.com/cilium/cilium/pkg/k8s/slim/k8s/api,"discovery:v1beta1 discovery:v1 networking:v1 core:v1","$(TMPDIR)")
+	$(QUIET) $(call generate_k8s_api,client,github.com/cilium/cilium/pkg/k8s/slim/k8s/apiextensions-client,github.com/cilium/cilium/pkg/k8s/slim/k8s/apis,"apiextensions:v1","$(TMPDIR)")
+	$(QUIET) $(call generate_k8s_api,client$(comma)lister$(comma)informer,github.com/cilium/cilium/pkg/k8s/client,github.com/cilium/cilium/pkg/k8s/apis,"cilium.io:v2 cilium.io:v2alpha1","$(TMPDIR)")
+
+	$(QUIET) cp -r "$(TMPDIR)/github.com/cilium/cilium/." ./
+	$(QUIET) rm -rf "$(TMPDIR)"
 
 check-k8s-clusterrole: ## Ensures there is no diff between preflight's clusterrole and runtime's clusterrole.
 	./contrib/scripts/check-preflight-clusterrole.sh

Makefile.defs

@@ -10,6 +10,11 @@ space := $(empty) $(empty)
 comma := ,
 join-with-comma = $(subst $(space),$(comma),$(strip $1))
 
+define newline
+
+
+endef
+
 ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 RELATIVE_DIR := $(shell echo $(realpath .) | sed "s;$(ROOT_DIR)[/]*;;")
 include $(ROOT_DIR)/Makefile.quiet

contrib/scripts/check-k8s-code-gen.sh

@@ -4,9 +4,16 @@ set -e
 set -o pipefail
 
 # Delete all zz_generated.deepcopy.go files
-find . -not -path "./vendor/*" -not -path "./_build/*" -iname "*zz_generated.deepcopy.go" -exec rm  {} \;
+find . -not -path "./vendor/*" -not -path "./_build/*" -name "zz_generated.deepcopy.go" -exec rm  {} \;
 # Delete all zz_generated.deepequal.go files
-find . -not -path "./vendor/*" -not -path "./_build/*" -iname "*zz_generated.deepequal.go" -exec rm  {} \;
+find . -not -path "./vendor/*" -not -path "./_build/*" -name "zz_generated.deepequal.go" -exec rm  {} \;
+# Delete all generated proto and proto go files
+find . -not -path "./vendor/*" -not -path "./_build/*" -name "generated.pb.go" -exec rm  {} \;
+find . -not -path "./vendor/*" -not -path "./_build/*" -name "generated.proto" -exec rm  {} \;
+# Delete cilium clientsets, informers & listers
+rm -rf ./pkg/k8s/client/{clientset,informers,listers}
+# Delete k8s slim clients
+rm -rf ./pkg/k8s/slim/k8s/{client,apiextensions-client}
 
 # Generate all files
 make generate-k8s-api manifests
@@ -15,9 +22,9 @@ make generate-k8s-api manifests
 diff="$(git diff)"
 
 if [ -n "$diff" ]; then
-	echo "Ungenerated deepcopy source code:"
+	echo "Ungenerated source code:"
 	echo "$diff"
-	echo "Please run make generate-k8s-api and submit your changes"
+	echo "Please run make generate-k8s-api & make manifests and submit your changes"
 	exit 1
 fi
 

go.mod

@@ -19,7 +19,7 @@ require (
 	github.com/blang/semver/v4 v4.0.0
 	github.com/cilium/coverbee v0.3.2
 	github.com/cilium/customvet v0.0.0-20221207181232-aa8731fa2d27
-	github.com/cilium/deepequal-gen v0.0.0-20200406125435-ad6a9003139e
+	github.com/cilium/deepequal-gen v0.0.0-20230330134849-754271daeec2
 	github.com/cilium/ebpf v0.10.0
 	github.com/cilium/ipam v0.0.0-20220824141044-46ef3d556735
 	github.com/cilium/kafka v0.0.0-20180809090225-01ce283b732b
@@ -242,7 +242,7 @@ require (
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	k8s.io/component-base v0.26.0 // indirect
-	k8s.io/gengo v0.0.0-20220902162205-c0856e24416d // indirect
+	k8s.io/gengo v0.0.0-20230306165830-ab3349d207d4 // indirect
 	k8s.io/klog v1.0.0 // indirect
 	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect