From 2b15c29126985f03e5736fc3e35a54cc14b75f0a Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Wed, 26 Jul 2023 18:05:47 -0700 Subject: [PATCH 1/4] Add k8s Network Conformance tests command to makefile --- hack/aks/Makefile | 10 ++++++ hack/scripts/k8se2e-tests.sh | 61 ++++++++++++++++++++++++++++++++++++ hack/scripts/k8se2e.sh | 8 +++++ 3 files changed, 79 insertions(+) create mode 100644 hack/scripts/k8se2e-tests.sh create mode 100644 hack/scripts/k8se2e.sh diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 14e2f1e0f7..135c0664bb 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -61,6 +61,16 @@ vars: ## Show the input vars configured for the cluster commands @echo NODE_COUNT=$(NODE_COUNT) @echo VMSS_NAME=$(VMSS_NAME) +##@ k8s Network conformance tests + +k8se2e: k8se2e-build k8se2e-test ## Alias to run build and test + +k8se2e-build: ## Build k8s e2e test suite + cd ../scripts && bash ./k8se2e.sh $(GROUP) $(CLUSTER) + +k8se2e-test: ## Run k8s network conformance test + cd ../scripts && bash ./k8se2e-tests.sh $(OS) $(TYPE) + ##@ SWIFT Infra diff --git a/hack/scripts/k8se2e-tests.sh b/hack/scripts/k8se2e-tests.sh new file mode 100644 index 0000000000..5fa60b09cc --- /dev/null +++ b/hack/scripts/k8se2e-tests.sh @@ -0,0 +1,61 @@ +# Taint Linux nodes so that windows tests do not run on them and ensure no LinuxOnly tests run on windows nodes +if [[ 'windows' == $OS ]] +then +SKIP="|LinuxOnly" +kubectl taint nodes -l kubernetes.azure.com/mode=system node-role.kubernetes.io/control-plane:NoSchedule +fi + + + +if [[ 'basic' == $TYPE ]] +then +echo "Testing Datapath" +echo "./ginkgo --nodes=4 \ +./e2e.test -- \ +--num-nodes=2 \ +--provider=skeleton \ +--ginkgo.focus='(.*).Networking.should|(.*).Networking.Granular|(.*)kubernetes.api' \ +--ginkgo.skip='SCTP|Disruptive|Slow|hostNetwork|kube-proxy|IPv6' \ +--ginkgo.flakeAttempts=3 \ +--ginkgo.v \ +--node-os-distro=$OS \ +--kubeconfig=$HOME/.kube/config" +./ginkgo --nodes=4 \ +./e2e.test -- \ +--num-nodes=2 \ +--provider=skeleton \ +--ginkgo.focus='(.*).Networking.should|(.*).Networking.Granular|(.*)kubernetes.api' \ +--ginkgo.skip='SCTP|Disruptive|Slow|hostNetwork|kube-proxy|IPv6' \ +--ginkgo.flakeAttempts=3 \ +--ginkgo.v \ +--node-os-distro=$OS \ +--kubeconfig=$HOME/.kube/config +else +echo "Testing Datapath, DNS, PortForward, Service, and Hostport" +echo "./ginkgo --nodes=4 \ +./e2e.test -- \ +--num-nodes=2 \ +--provider=skeleton \ +--ginkgo.focus='(.*).Networking.should|(.*).Networking.Granular|(.*)kubernetes.api|\[sig-network\].DNS.should|\[sig-cli\].Kubectl.Port|Services.*\[Conformance\].*|\[sig-network\](.*)HostPort|\[sig-scheduling\](.*)hostPort' \ +--ginkgo.skip='SCTP|Disruptive|Slow|hostNetwork|kube-proxy|IPv6|resolv|exists conflict$SKIP' \ +--ginkgo.flakeAttempts=3 \ +--ginkgo.v \ +--node-os-distro=$OS \ +--kubeconfig=$HOME/.kube/config" +./ginkgo --nodes=4 \ +./e2e.test -- \ +--num-nodes=2 \ +--provider=skeleton \ +--ginkgo.focus='(.*).Networking.should|(.*).Networking.Granular|(.*)kubernetes.api|\[sig-network\].DNS.should|\[sig-cli\].Kubectl.Port|Services.*\[Conformance\].*|\[sig-network\](.*)HostPort|\[sig-scheduling\](.*)hostPort' \ +--ginkgo.skip="SCTP|Disruptive|Slow|hostNetwork|kube-proxy|IPv6|resolv|exists conflict$SKIP" \ +--ginkgo.flakeAttempts=3 \ +--ginkgo.v \ +--node-os-distro=$OS \ +--kubeconfig=$HOME/.kube/config +fi + +# Untaint Linux nodes once testing is complete +if [[ 'windows' == $OS ]] +then +kubectl taint nodes -l kubernetes.azure.com/mode=system node-role.kubernetes.io/control-plane:NoSchedule- +fi diff --git a/hack/scripts/k8se2e.sh b/hack/scripts/k8se2e.sh new file mode 100644 index 0000000000..b59f7070fe --- /dev/null +++ b/hack/scripts/k8se2e.sh @@ -0,0 +1,8 @@ +echo "-g $GROUP -n $CLUSTER" + +eval k8sVersion="v"$( az aks show -g $GROUP -n $CLUSTER --query "currentKubernetesVersion") +echo "$k8sversion e2e Test Suite" + +curl -L https://dl.k8s.io/$k8sVersion/kubernetes-test-linux-amd64.tar.gz -o ./kubernetes-test-linux-amd64.tar.gz + +tar -xvzf kubernetes-test-linux-amd64.tar.gz --strip-components=3 kubernetes/test/bin/ginkgo kubernetes/test/bin/e2e.test From 3c7a53a72d47be076ef009ab8723dca38ab44a42 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Wed, 26 Jul 2023 18:37:02 -0700 Subject: [PATCH 2/4] Add lines to README --- hack/aks/Makefile | 18 ++++++++---------- hack/aks/README.md | 3 +++ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 135c0664bb..cafa57ea6b 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -50,6 +50,14 @@ unset-kubeconf: ## Deletes the kubeconf for $CLUSTER shell: ## print $AZCLI so it can be used outside of make @echo $(AZCLI) +k8se2e: k8se2e-build k8se2e-test ## Alias to run build and test + +k8se2e-build: ## Build k8s e2e test suite + cd ../scripts && bash ./k8se2e.sh $(GROUP) $(CLUSTER) + +k8se2e-test: ## Run k8s network conformance test, use TYPE=basic for only datapath tests + cd ../scripts && bash ./k8se2e-tests.sh $(OS) $(TYPE) + vars: ## Show the input vars configured for the cluster commands @echo CLUSTER=$(CLUSTER) @echo GROUP=$(GROUP) @@ -61,16 +69,6 @@ vars: ## Show the input vars configured for the cluster commands @echo NODE_COUNT=$(NODE_COUNT) @echo VMSS_NAME=$(VMSS_NAME) -##@ k8s Network conformance tests - -k8se2e: k8se2e-build k8se2e-test ## Alias to run build and test - -k8se2e-build: ## Build k8s e2e test suite - cd ../scripts && bash ./k8se2e.sh $(GROUP) $(CLUSTER) - -k8se2e-test: ## Run k8s network conformance test - cd ../scripts && bash ./k8se2e-tests.sh $(OS) $(TYPE) - ##@ SWIFT Infra diff --git a/hack/aks/README.md b/hack/aks/README.md index 26a38bec0a..a6c7387198 100644 --- a/hack/aks/README.md +++ b/hack/aks/README.md @@ -13,6 +13,9 @@ Utilities set-kubeconf Adds the kubeconf for $CLUSTER unset-kubeconf Deletes the kubeconf for $CLUSTER shell print $AZCLI so it can be used outside of make + k8se2e Alias for single command to build and test k8se2e + k8se2e-build Build k8se2e tests based off $CLUSTER + k8se2e-test Run k8se2e tests based off $CLUSTER and $OS SWIFT Infra vars Show the env vars configured for the swift command From ab62213f00d059a5327d2d1fbe3879c693543d08 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Thu, 27 Jul 2023 09:58:22 -0700 Subject: [PATCH 3/4] Allow numbers to show in make help call --- hack/aks/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index cafa57ea6b..8be1514799 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -25,16 +25,16 @@ VNET ?= $(CLUSTER) ##@ Help help: ## Display this help - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[0-9a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) ##@ Utilities -azlogin: +azlogin: ## Login and set account to $SUB @$(AZCLI) login @$(AZCLI) account set -s $(SUB) -azcfg: +azcfg: ## Set the $AZCLI to use aks-preview @$(AZCLI) extension add --name aks-preview --yes @$(AZCLI) extension update --name aks-preview @@ -222,7 +222,7 @@ windows-cniv1-up: rg-up overlay-net-up ## Bring up a Windows CNIv1 cluster @$(MAKE) set-kubeconf -linux-cniv1-up: rg-up overlay-net-up +linux-cniv1-up: rg-up overlay-net-up ## Bring up a Linux CNIv1 cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ From 22d2f85c3d1b35d7ff1f8d8d7591054384d814bb Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Thu, 27 Jul 2023 11:29:32 -0700 Subject: [PATCH 4/4] Addressing Comments --- Makefile | 37 +++++++++++++++++++++++-------------- hack/aks/Makefile | 8 -------- hack/aks/README.md | 3 --- hack/scripts/k8se2e.sh | 4 ++-- 4 files changed, 25 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index f8b155324d..316523d6c0 100644 --- a/Makefile +++ b/Makefile @@ -142,7 +142,7 @@ azure-ipam: azure-ipam-binary azure-ipam-archive revision: ## print the current git revision @echo $(REVISION) - + version: ## prints the root version @echo $(ACN_VERSION) @@ -161,15 +161,15 @@ cni-dropgz-test-version: ## prints the cni-dropgz version @echo $(CNI_DROPGZ_TEST_VERSION) cns-version: - @echo $(CNS_VERSION) + @echo $(CNS_VERSION) npm-version: - @echo $(NPM_VERSION) + @echo $(NPM_VERSION) zapai-version: ## prints the zapai version @echo $(ZAPAI_VERSION) -##@ Binaries +##@ Binaries # Build the delegated IPAM plugin binary. azure-ipam-binary: @@ -509,8 +509,8 @@ manifest-build: # util target to compose multiarch container manifests from plat $(MAKE) manifest-add PLATFORM=$(PLATFORM);\ )\ )\ - - + + manifest-push: # util target to push multiarch container manifest. $(CONTAINER_BUILDER) manifest push --all $(IMAGE_REGISTRY)/$(IMAGE):$(TAG) docker://$(IMAGE_REGISTRY)/$(IMAGE):$(TAG) @@ -534,7 +534,7 @@ acncli-manifest-push: ## push acncli multiplat container manifest acncli-skopeo-archive: ## export tar archive of acncli multiplat container manifest. $(MAKE) manifest-skopeo-archive \ IMAGE=$(ACNCLI_IMAGE) \ - TAG=$(ACN_VERSION) + TAG=$(ACN_VERSION) cni-dropgz-manifest-build: ## build cni-dropgz multiplat container manifest. $(MAKE) manifest-build \ @@ -692,7 +692,7 @@ ifeq ($(GOOS),linux) endif -##@ Utils +##@ Utils clean: ## Clean build artifacts. $(RMDIR) $(OUTPUT_DIR) @@ -723,7 +723,7 @@ workspace: ## Set up the Go workspace. go work use ./dropgz go work use ./zapai -##@ Test +##@ Test COVER_PKG ?= . #Restart case is used for cni load test pipeline for restarting the nodes cluster. @@ -762,6 +762,15 @@ test-azure-ipam: ## run the unit test for azure-ipam kind: kind create cluster --config ./test/kind/kind.yaml +test-k8se2e: test-k8se2e-build test-k8se2e-only ## Alias to run build and test + +test-k8se2e-build: ## Build k8s e2e test suite + cd hack/scripts && bash ./k8se2e.sh $(GROUP) $(CLUSTER) + cd ../.. + +test-k8se2e-only: ## Run k8s network conformance test, use TYPE=basic for only datapath tests + cd hack/scripts && bash ./k8se2e-tests.sh $(OS) $(TYPE) + cd ../.. ##@ Utilities @@ -781,7 +790,7 @@ gitconfig: ## configure the local git repository setup: tools install-hooks gitconfig ## performs common required repo setup -##@ Tools +##@ Tools $(TOOLS_DIR)/go.mod: cd $(TOOLS_DIR); go mod init && go mod tidy @@ -791,7 +800,7 @@ $(CONTROLLER_GEN): $(TOOLS_DIR)/go.mod controller-gen: $(CONTROLLER_GEN) ## Build controller-gen -protoc: +protoc: source ${REPO_ROOT}/scripts/install-protoc.sh $(GOCOV): $(TOOLS_DIR)/go.mod @@ -824,13 +833,13 @@ $(MOCKGEN): $(TOOLS_DIR)/go.mod mockgen: $(MOCKGEN) ## Build mockgen -clean-tools: +clean-tools: rm -r build/tools/bin tools: acncli gocov gocov-xml go-junit-report golangci-lint gofumpt protoc ## Build bins for build tools -##@ Help +##@ Help help: ## Display this help - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[0-9a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 8be1514799..6091dc88a1 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -50,14 +50,6 @@ unset-kubeconf: ## Deletes the kubeconf for $CLUSTER shell: ## print $AZCLI so it can be used outside of make @echo $(AZCLI) -k8se2e: k8se2e-build k8se2e-test ## Alias to run build and test - -k8se2e-build: ## Build k8s e2e test suite - cd ../scripts && bash ./k8se2e.sh $(GROUP) $(CLUSTER) - -k8se2e-test: ## Run k8s network conformance test, use TYPE=basic for only datapath tests - cd ../scripts && bash ./k8se2e-tests.sh $(OS) $(TYPE) - vars: ## Show the input vars configured for the cluster commands @echo CLUSTER=$(CLUSTER) @echo GROUP=$(GROUP) diff --git a/hack/aks/README.md b/hack/aks/README.md index a6c7387198..26a38bec0a 100644 --- a/hack/aks/README.md +++ b/hack/aks/README.md @@ -13,9 +13,6 @@ Utilities set-kubeconf Adds the kubeconf for $CLUSTER unset-kubeconf Deletes the kubeconf for $CLUSTER shell print $AZCLI so it can be used outside of make - k8se2e Alias for single command to build and test k8se2e - k8se2e-build Build k8se2e tests based off $CLUSTER - k8se2e-test Run k8se2e tests based off $CLUSTER and $OS SWIFT Infra vars Show the env vars configured for the swift command diff --git a/hack/scripts/k8se2e.sh b/hack/scripts/k8se2e.sh index b59f7070fe..aac14c1fc8 100644 --- a/hack/scripts/k8se2e.sh +++ b/hack/scripts/k8se2e.sh @@ -1,7 +1,7 @@ -echo "-g $GROUP -n $CLUSTER" +echo "Cluster -g $GROUP -n $CLUSTER" eval k8sVersion="v"$( az aks show -g $GROUP -n $CLUSTER --query "currentKubernetesVersion") -echo "$k8sversion e2e Test Suite" +echo $k8sVersion "e2e Test Suite" curl -L https://dl.k8s.io/$k8sVersion/kubernetes-test-linux-amd64.tar.gz -o ./kubernetes-test-linux-amd64.tar.gz