From 5b84ea61e6db2bf8d503f7a7723fa8308e15875d Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Wed, 12 Jul 2023 13:14:54 -0700 Subject: [PATCH 01/10] use team pool --- .../cilium-overlay-load-test-template.yaml | 17 ++++++++++++++++- .pipelines/cni/pipeline.yaml | 4 ++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml index 044535bc6d..540c0afc06 100644 --- a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml +++ b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml @@ -8,8 +8,9 @@ parameters: stages: - - stage: createAKScluster + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" dependsOn: ${{ parameters.dependsOn }} displayName: "AKS Cluster with Cilium" jobs: @@ -22,6 +23,8 @@ stages: nodeCount: ${{ parameters.nodeCount }} vmSize: ${{ parameters.vmSize }} - stage: install_cilium + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" dependsOn: createAKScluster displayName: "Install Cilium on AKS Overlay" jobs: @@ -62,6 +65,8 @@ stages: cd ../../../.. kubectl get po -owide -A - stage: pod_deployment + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" dependsOn: install_cilium displayName: "Pod Deployment" jobs: @@ -73,6 +78,8 @@ stages: scaleup: 2400 os: linux - stage: validate_state + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" dependsOn: pod_deployment displayName: "Validate State" jobs: @@ -82,6 +89,8 @@ stages: parameters: clusterName: ${{ parameters.clusterName }} - stage: restart_nodes + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" dependsOn: validate_state displayName: "Restart Node" jobs: @@ -91,6 +100,8 @@ stages: parameters: clusterName: ${{ parameters.clusterName }} - stage: validate_restart_state + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" dependsOn: restart_nodes displayName: "Validate Restart State" jobs: @@ -101,6 +112,8 @@ stages: clusterName: ${{ parameters.clusterName }} restartCase: "true" - stage: connectivity_tests + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" dependsOn: validate_restart_state displayName: "Connectivity Tests" jobs: @@ -134,6 +147,8 @@ stages: name: "CiliumConnectivityTests" displayName: "Run Cilium Connectivity Tests" - stage: delete + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" displayName: "Delete Resources" dependsOn: - connectivity_tests diff --git a/.pipelines/cni/pipeline.yaml b/.pipelines/cni/pipeline.yaml index 3fda84f807..1a66ece452 100644 --- a/.pipelines/cni/pipeline.yaml +++ b/.pipelines/cni/pipeline.yaml @@ -3,12 +3,12 @@ trigger: none stages: - stage: setup + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" displayName: Setup jobs: - job: env displayName: Setup - pool: - name: "$(BUILD_POOL_NAME_DEFAULT)" steps: - script: | echo "Setting up environment" From 273208a4c69487b68dd8297cf3b69ae6173074bf Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Wed, 12 Jul 2023 17:09:51 -0700 Subject: [PATCH 02/10] standard lb --- hack/aks/Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index b16019b1dc..6869e3543e 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -87,7 +87,7 @@ overlay-byocni-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ - --load-balancer-sku basic \ + --load-balancer-sku standard \ --network-plugin none \ --network-plugin-mode overlay \ --pod-cidr 192.168.0.0/16 \ @@ -100,7 +100,7 @@ overlay-no-kube-proxy-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI c $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ - --load-balancer-sku basic \ + --load-balancer-sku standard \ --network-plugin none \ --network-plugin-mode overlay \ --pod-cidr 192.168.0.0/16 \ @@ -114,7 +114,7 @@ overlay-cilium-up: rg-up overlay-net-up ## Brings up an Overlay AzCNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ - --load-balancer-sku basic \ + --load-balancer-sku standard \ --network-plugin azure \ --network-dataplane cilium \ --network-plugin-mode overlay \ @@ -128,7 +128,7 @@ overlay-up: rg-up overlay-net-up ## Brings up an Overlay AzCNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ - --load-balancer-sku basic \ + --load-balancer-sku standard \ --network-plugin azure \ --network-plugin-mode overlay \ --pod-cidr 192.168.0.0/16 \ @@ -141,7 +141,7 @@ swift-byocni-up: rg-up swift-net-up ## Bring up a SWIFT BYO CNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ - --load-balancer-sku basic \ + --load-balancer-sku standard \ --network-plugin none \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --pod-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/podnet \ From 558c2e09dd4655c48d942b0b0839e5fe94655869 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Thu, 13 Jul 2023 13:30:25 -0700 Subject: [PATCH 03/10] change sku --- hack/aks/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 6869e3543e..443397f55b 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -10,7 +10,7 @@ AZCLI ?= docker run --rm -v $(AZCFG):/root/.azure -v $(KUBECFG):/root/.kube -v # overrideable defaults REGION ?= westus2 OS_SKU ?= Ubuntu -VM_SIZE ?= Standard_B2s +VM_SIZE ?= Standard_DS3_v2 NODE_COUNT ?= 2 # overrideable variables From f7017d948f09337a989b7775d03e6c18765735c3 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Thu, 13 Jul 2023 16:51:58 -0700 Subject: [PATCH 04/10] use slash 8 --- hack/aks/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 443397f55b..141fd122c7 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -90,7 +90,7 @@ overlay-byocni-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster --load-balancer-sku standard \ --network-plugin none \ --network-plugin-mode overlay \ - --pod-cidr 192.168.0.0/16 \ + --pod-cidr 10.0.0.0/8 \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --no-ssh-key \ --yes @@ -103,7 +103,7 @@ overlay-no-kube-proxy-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI c --load-balancer-sku standard \ --network-plugin none \ --network-plugin-mode overlay \ - --pod-cidr 192.168.0.0/16 \ + --pod-cidr 10.0.0.0/8 \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --no-ssh-key \ --kube-proxy-config ./kube-proxy.json \ @@ -118,7 +118,7 @@ overlay-cilium-up: rg-up overlay-net-up ## Brings up an Overlay AzCNI cluster --network-plugin azure \ --network-dataplane cilium \ --network-plugin-mode overlay \ - --pod-cidr 192.168.0.0/16 \ + --pod-cidr 10.0.0.0/8 \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --no-ssh-key \ --yes @@ -131,7 +131,7 @@ overlay-up: rg-up overlay-net-up ## Brings up an Overlay AzCNI cluster --load-balancer-sku standard \ --network-plugin azure \ --network-plugin-mode overlay \ - --pod-cidr 192.168.0.0/16 \ + --pod-cidr 10.0.0.0/8 \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --no-ssh-key \ --yes From 6dd6d3324649503dceb4c34019ca0690888850ea Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Fri, 14 Jul 2023 10:01:05 -0700 Subject: [PATCH 05/10] use different service cidr --- hack/aks/Makefile | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 74419d960d..75d0b2a94e 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -76,8 +76,8 @@ swift-net-up: ## Create vnet, nodenet and podnet subnets $(AZCLI) network vnet subnet create -g $(GROUP) --vnet-name $(VNET) --name podnet --address-prefixes 10.241.0.0/16 -o none overlay-net-up: ## Create vnet, nodenet subnets - $(AZCLI) network vnet create -g $(GROUP) -l $(REGION) --name $(VNET) --address-prefixes 10.0.0.0/8 -o none - $(AZCLI) network vnet subnet create -g $(GROUP) --vnet-name $(VNET) --name nodenet --address-prefix 10.0.0.0/9 -o none + $(AZCLI) network vnet create -g $(GROUP) -l $(REGION) --name $(VNET) --address-prefixes 10.0.0.0/9 -o none + $(AZCLI) network vnet subnet create -g $(GROUP) --vnet-name $(VNET) --name nodenet --address-prefix 10.0.0.0/12 -o none ##@ AKS Clusters @@ -94,6 +94,8 @@ overlay-byocni-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster --network-plugin none \ --network-plugin-mode overlay \ --pod-cidr 10.128.0.0/9 \ + --dns-service-ip 192.168.0.10 \ + --service-cidr 192.168.0.0/24 \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --no-ssh-key \ --yes @@ -107,6 +109,8 @@ overlay-no-kube-proxy-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI c --network-plugin none \ --network-plugin-mode overlay \ --pod-cidr 10.128.0.0/9 \ + --dns-service-ip 192.168.0.10 \ + --service-cidr 192.168.0.0/24 \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --no-ssh-key \ --kube-proxy-config ./kube-proxy.json \ @@ -122,6 +126,8 @@ overlay-cilium-up: rg-up overlay-net-up ## Brings up an Overlay AzCNI cluster --network-dataplane cilium \ --network-plugin-mode overlay \ --pod-cidr 10.128.0.0/9 \ + --dns-service-ip 192.168.0.10 \ + --service-cidr 192.168.0.0/24 \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --no-ssh-key \ --yes @@ -135,6 +141,8 @@ overlay-up: rg-up overlay-net-up ## Brings up an Overlay AzCNI cluster --network-plugin azure \ --network-plugin-mode overlay \ --pod-cidr 10.128.0.0/9 \ + --dns-service-ip 192.168.0.10 \ + --service-cidr 192.168.0.0/24 \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --no-ssh-key \ --yes From f859f6f8fdebfefb26a57200cedd8387b8ad7f92 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Mon, 17 Jul 2023 10:17:17 -0700 Subject: [PATCH 06/10] service cidr --- hack/aks/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 75d0b2a94e..4e99f1f0a7 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -95,7 +95,7 @@ overlay-byocni-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster --network-plugin-mode overlay \ --pod-cidr 10.128.0.0/9 \ --dns-service-ip 192.168.0.10 \ - --service-cidr 192.168.0.0/24 \ + --service-cidr 192.168.0.0/16 \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --no-ssh-key \ --yes @@ -110,7 +110,7 @@ overlay-no-kube-proxy-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI c --network-plugin-mode overlay \ --pod-cidr 10.128.0.0/9 \ --dns-service-ip 192.168.0.10 \ - --service-cidr 192.168.0.0/24 \ + --service-cidr 192.168.0.0/16 \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --no-ssh-key \ --kube-proxy-config ./kube-proxy.json \ @@ -127,7 +127,7 @@ overlay-cilium-up: rg-up overlay-net-up ## Brings up an Overlay AzCNI cluster --network-plugin-mode overlay \ --pod-cidr 10.128.0.0/9 \ --dns-service-ip 192.168.0.10 \ - --service-cidr 192.168.0.0/24 \ + --service-cidr 192.168.0.0/16 \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --no-ssh-key \ --yes @@ -142,7 +142,7 @@ overlay-up: rg-up overlay-net-up ## Brings up an Overlay AzCNI cluster --network-plugin-mode overlay \ --pod-cidr 10.128.0.0/9 \ --dns-service-ip 192.168.0.10 \ - --service-cidr 192.168.0.0/24 \ + --service-cidr 192.168.0.0/16 \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --no-ssh-key \ --yes From d895c089b4354cf7f1f4fe55c41bdfc8ed6c0719 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Wed, 16 Aug 2023 12:09:50 -0700 Subject: [PATCH 07/10] service cidr --- hack/aks/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 5b215c91b5..8c62e2cada 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -180,7 +180,9 @@ cilium-overlay-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster --load-balancer-sku basic \ --network-plugin none \ --network-plugin-mode overlay \ - --pod-cidr 192.168.0.0/16 \ + --pod-cidr 10.128.0.0/9 \ + --dns-service-ip 192.168.0.10 \ + --service-cidr 192.168.0.0/16 \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --no-ssh-key \ --kube-proxy-config ./kube-proxy.json \ From f12dd36e5d13c2c21b7a56102bfa686f76fbbbb3 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Wed, 16 Aug 2023 14:21:11 -0700 Subject: [PATCH 08/10] standard lb for other nodepools --- hack/aks/Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 8c62e2cada..bd11c26b8f 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -11,10 +11,10 @@ AZCLI ?= docker run --rm -v $(AZCFG):/root/.azure -v $(KUBECFG):/root/.kube -v REGION ?= westus2 OS_SKU ?= Ubuntu WINDOWS_OS_SKU ?= Windows2022 -VM_SIZE = Standard_DS3_v2 +VM_SIZE = Standard_DS4_v3 NODE_COUNT ?= 2 -K8S_VER ?= 1.25 # Used only for ubuntu 18 as K8S 1.24.9, as K8S > 1.25 have Ubuntu 22 -WINDOWS_VM_SKU = Standard_DS3_v2 +K8S_VER ?= 1.26 # Used only for ubuntu 18 as K8S 1.24.9, as K8S > 1.25 have Ubuntu 22 +WINDOWS_VM_SKU = Standard_DS4_v3 # overrideable variables SUB ?= $(AZURE_SUBSCRIPTION) @@ -165,7 +165,7 @@ swift-up: rg-up swift-net-up ## Bring up a SWIFT AzCNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ - --load-balancer-sku basic \ + --load-balancer-sku standard \ --network-plugin azure \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --pod-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/podnet \ @@ -177,7 +177,7 @@ cilium-overlay-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ - --load-balancer-sku basic \ + --load-balancer-sku standard \ --network-plugin none \ --network-plugin-mode overlay \ --pod-cidr 10.128.0.0/9 \ @@ -193,7 +193,7 @@ cilium-podsubnet-up: rg-up swift-net-up ## Bring up a SWIFT BYO CNI cluster with $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ - --load-balancer-sku basic \ + --load-balancer-sku standard \ --network-plugin none \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --pod-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/podnet \ From 6cf43b1225f6512a104d61de81fdee56c6d933f8 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Tue, 22 Aug 2023 13:28:01 -0700 Subject: [PATCH 09/10] region override --- hack/aks/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index bd11c26b8f..dbda9ef005 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -8,7 +8,7 @@ AZIMG = mcr.microsoft.com/azure-cli AZCLI ?= docker run --rm -v $(AZCFG):/root/.azure -v $(KUBECFG):/root/.kube -v $(SSH):/root/.ssh -v $(PWD):/root/tmpsrc $(AZIMG) az # overrideable defaults -REGION ?= westus2 +REGION = westus2 OS_SKU ?= Ubuntu WINDOWS_OS_SKU ?= Windows2022 VM_SIZE = Standard_DS4_v3 From 2d19b2c4108a2f247572a4b2294e879514ccf72a Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Tue, 22 Aug 2023 13:29:35 -0700 Subject: [PATCH 10/10] region --- hack/aks/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index dbda9ef005..bd11c26b8f 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -8,7 +8,7 @@ AZIMG = mcr.microsoft.com/azure-cli AZCLI ?= docker run --rm -v $(AZCFG):/root/.azure -v $(KUBECFG):/root/.kube -v $(SSH):/root/.ssh -v $(PWD):/root/tmpsrc $(AZIMG) az # overrideable defaults -REGION = westus2 +REGION ?= westus2 OS_SKU ?= Ubuntu WINDOWS_OS_SKU ?= Windows2022 VM_SIZE = Standard_DS4_v3