Skip to content

Commit

Permalink
ci:[CNI] Windows cniv1 load test pipeline (#2024)
Browse files Browse the repository at this point in the history
CI:[CNI] Windows cniv1 load test pipeline
  • Loading branch information
vipul-21 committed Jun 28, 2023
1 parent c10900e commit e45ad21
Show file tree
Hide file tree
Showing 24 changed files with 647 additions and 34 deletions.
12 changes: 7 additions & 5 deletions .pipelines/cni/cilium/cilium-overlay-load-test-template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ stages:
inlineScript: |
set -ex
az extension add --name aks-preview
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
ls -lah
pwd
kubectl cluster-info
Expand Down Expand Up @@ -70,6 +70,8 @@ stages:
- template: ../load-test-templates/pod-deployment-template.yaml
parameters:
clusterName: ${{ parameters.clusterName }}
scaleup: 2400
os: linux
- stage: validate_state
dependsOn: pod_deployment
displayName: "Validate State"
Expand Down Expand Up @@ -122,7 +124,7 @@ stages:
addSpnToEnvironment: true
inlineScript: |
set -ex
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
name: "GetCluster"
displayName: "Get AKS Cluster"
- script: |
Expand All @@ -149,9 +151,9 @@ stages:
if [ "$(DELETE_RESOURCES)" ]
then
echo "Deleting Cluster and resource group"
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION)
make -C ./hack/swift down AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${{ parameters.clusterName }}-$(make revision)
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION)
make -C ./hack/aks down AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${{ parameters.clusterName }}-$(make revision)
echo "Cluster and resources down"
else
echo "Deletion of resources is False"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ parameters:
clusterName: ""
nodeCount: ""
vmSize: ""
windowsVMSize: ""

steps:
- task: AzureCLI@1
Expand All @@ -13,7 +14,7 @@ steps:
addSpnToEnvironment: true
inlineScript: |
set -ex
make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION)
make -C ./hack/swift ${{ parameters.clusterType }} AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${{ parameters.clusterName }}-$(make revision) NODE_COUNT=${{ parameters.nodeCount }} VM_SIZE=${{ parameters.vmSize }}
make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION)
make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${{ parameters.clusterName }}-$(make revision) NODE_COUNT=${{ parameters.nodeCount }} VM_SIZE=${{ parameters.vmSize }} WINDOWS_VM_SKU=${{ parameters.windowsVMSize }} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD}
name: "CreateAksCluster"
displayName: "Create AKS Cluster"
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
parameters:
clusterName: ""

scaleup: 1000
os: ""
iterations: 4

steps:
- task: AzureCLI@1
displayName: "Pod Deployment"
Expand All @@ -12,6 +15,6 @@ steps:
inlineScript: |
set -ex
az extension add --name aks-preview
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
cd test/integration/load
go test -timeout 30m -tags load -run ^TestLoad$ -tags=load -iterations=4 -scaleup=2400
go test -timeout 30m -tags load -run ^TestLoad$ -tags=load -iterations=${{ parameters.iterations }} -scaleup=${{ parameters.scaleup }} -os=${{ parameters.os }}
6 changes: 3 additions & 3 deletions .pipelines/cni/load-test-templates/restart-node-template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,15 @@ steps:
inlineScript: |
echo "Scale up the pods and immediated restart the nodes"
clusterName=${{ parameters.clusterName }}-$(make revision)
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${clusterName}
make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION)
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}
make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION)
cd test/integration/load
echo "Scaling the pods down to 100 per node"
go test -count 1 -timeout 30m -tags load -run ^TestScaleDeployment$ -tags=load -scaleup=1000 -skip-wait=true
cd ../../../
echo "Restarting the nodes"
vmss_name=$(az vmss list -g MC_${clusterName}_${clusterName}_$(LOCATION) --query "[].name" -o tsv)
make -C ./hack/swift restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(LOCATION) VMSS_NAME=$vmss_name
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(LOCATION) VMSS_NAME=$vmss_name
cd test/integration/load
go test -count 1 -timeout 30m -tags load -run ^TestScaleDeployment$ -tags=load -replicas=1000
name: "RestartNodes"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
parameters:
clusterName: ""
os: "linux"
restartCase: "false"

steps:
Expand All @@ -10,10 +11,9 @@ steps:
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
export RESTART_CASE=${{ parameters.restartCase }}
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
kubectl get pods -A
make test-validate-state
make test-validate-state OS=${{ parameters.os }} RESTART_CASE=${{ parameters.restartCase }}
name: "ValidateState"
displayName: "Validate State"
retryCountOnTaskFailure: 3
11 changes: 11 additions & 0 deletions .pipelines/cni/pipeline.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,14 @@ stages:
nodeCount: 10
vmSize: "Standard_DS4_v2"
dependsOn: setup
- template: singletenancy/windows-cni-load-test-template.yaml
parameters:
name: win_cniv1
clusterType: "windows-cniv1-up"
clusterName: "win-cniv1"
nodeCount: 2
vmSize: "Standard_B2s"
dependsOn: setup
windowsVMSize: ${WINDOWS_VM_SKU}
os: windows
cni: cniv1
134 changes: 134 additions & 0 deletions .pipelines/cni/singletenancy/windows-cni-load-test-template.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
parameters:
dependsOn: ""
name: ""
clusterType: ""
clusterName: ""
nodeCount: ""
vmSize: ""
windowsVMSize: ""
os: ""
cni: ""

stages:
- stage: createAKSclusterWindows
dependsOn: ${{ parameters.dependsOn }}
displayName: "Windows AKS Cluster ${{ parameters.cni }}"
jobs:
- job: create_aks_cluster_with_${{ parameters.name }}
steps:
- template: ../load-test-templates/create-cluster-template.yaml
parameters:
clusterType: ${{ parameters.clusterType }}
clusterName: ${{ parameters.clusterName }}
nodeCount: ${{ parameters.nodeCount }}
vmSize: ${{ parameters.vmSize }}
windowsVMSize: ${{ parameters.windowsVMSize }}
- stage: build_images
dependsOn: ${{ parameters.dependsOn }}
displayName: "Build CNI Images"
jobs:
- job: build_cni_images
pool:
name: "$(BUILD_POOL_NAME_LINUX_AMD64)"
strategy:
matrix:
windows_cniv1_amd64:
os: windows
name: cni-plugin
arch: amd64
os_version: ltsc2022
steps:
- template: ../../containers/container-template.yaml
parameters:
arch: $(arch)
name: $(name)
os: $(os)
os_version: $(os_version)
- stage: update_cni
dependsOn:
- createAKSclusterWindows
- build_images
displayName: "Update CNI on Cluster"
jobs:
- job: deploy_pods
strategy:
matrix:
windows_cniv1_amd64:
os: windows
arch: amd64
os_version: ltsc2022
steps:
- task: AzureCLI@1
inputs:
azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -ex
export CNI_IMAGE=$(make acncli-image-name-and-tag OS=$(os) ARCH=$(arch) OS_VERSION=$(os_version))
az extension add --name aks-preview
clusterName=${{ parameters.clusterName }}-$(make revision)
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}
make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION)
envsubst < ./hack/manifests/windows-update.yaml | kubectl apply -f -
name: "UploadCNI"
displayName: "Upload CNI"
- script: |
set -ex
kubectl rollout status daemonset/azure-cni-windows -n kube-system
kubectl get pods -A
name: "WaitForCNI"
displayName: "Wait For CNI"
- stage: pod_deployment_windows
dependsOn: update_cni
displayName: "Pod Deployment"
jobs:
- job: deploy_pods
steps:
- template: ../load-test-templates/pod-deployment-template.yaml
parameters:
clusterName: ${{ parameters.clusterName }}
scaleup: ${WINDOWS_SCALEUP}
os: ${{ parameters.os }}
cni: ${{ parameters.cni }}
iterations: ${WINDOWS_ITERATIONS}
- stage: validate_state_windows
dependsOn: pod_deployment_windows
displayName: "Validate State"
jobs:
- job: validate_state
steps:
- template: ../load-test-templates/validate-state-template.yaml
parameters:
clusterName: ${{ parameters.clusterName }}
os: ${{ parameters.os }}
cni: ${{ parameters.cni }}
- stage: delete_resources
displayName: "Delete Resources"
dependsOn:
- validate_state_windows
jobs:
- job: delete_resources
steps:
- task: AzureCLI@1
inputs:
azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -ex
if [ "$(DELETE_RESOURCES)" ]
then
echo "Deleting Cluster and resource group"
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION)
make -C ./hack/aks down AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${{ parameters.clusterName }}-$(make revision)
echo "Cluster and resources down"
else
echo "Deletion of resources is False"
fi
name: "CleanUpCluster"
displayName: "Cleanup cluster"
condition: always()
8 changes: 4 additions & 4 deletions .pipelines/singletenancy/aks-swift/e2e-step-template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ steps:
inlineScript: |
mkdir -p ~/.kube/
echo "Create AKS cluster"
make -C ./hack/swift azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
make -C ./hack/swift byocni-up AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-${{ parameters.osSku }}-$(make revision) OSSKU=${{ parameters.osSku }}
make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
make -C ./hack/aks byocni-up AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-${{ parameters.osSku }}-$(make revision) OSSKU=${{ parameters.osSku }}
echo "Cluster successfully created"
displayName: Create test cluster
condition: succeeded()
Expand Down Expand Up @@ -124,8 +124,8 @@ steps:
addSpnToEnvironment: true
inlineScript: |
echo "Deleting cluster"
make -C ./hack/swift azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
make -C ./hack/swift down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-${{ parameters.osSku }}-$(make revision)
make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
make -C ./hack/aks down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-${{ parameters.osSku }}-$(make revision)
echo "Cluster and resources down"
name: "Cleanupcluster"
displayName: "Cleanup cluster"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ steps:
inlineScript: |
mkdir -p ~/.kube/
echo "Create AKS Overlay cluster"
make -C ./hack/swift azcfg AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST)
make -C ./hack/swift overlay-no-kube-proxy-up AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) VM_SIZE=Standard_B2ms
make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST)
make -C ./hack/aks overlay-no-kube-proxy-up AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) VM_SIZE=Standard_B2ms
echo "Cluster successfully created"
displayName: Create Overlay cluster
condition: succeeded()
Expand Down Expand Up @@ -189,8 +189,8 @@ steps:
addSpnToEnvironment: true
inlineScript: |
echo "Deleting cluster"
make -C ./hack/swift azcfg AZCLI=az
make -C ./hack/swift down SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
make -C ./hack/aks azcfg AZCLI=az
make -C ./hack/aks down SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
echo "Cluster and resources down"
name: "Cleanupcluster"
displayName: "Cleanup cluster"
Expand Down
8 changes: 4 additions & 4 deletions .pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ steps:
inlineScript: |
mkdir -p ~/.kube/
echo "Create AKS cluster"
make -C ./hack/swift azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
make -C ./hack/swift swift-no-kube-proxy-up AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) VM_SIZE=Standard_B2ms
make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
make -C ./hack/aks swift-no-kube-proxy-up AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) VM_SIZE=Standard_B2ms
echo "Cluster successfully created"
displayName: Create test cluster
condition: succeeded()
Expand Down Expand Up @@ -173,8 +173,8 @@ steps:
addSpnToEnvironment: true
inlineScript: |
echo "Deleting cluster"
make -C ./hack/swift azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
make -C ./hack/swift down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision)
make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
make -C ./hack/aks down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision)
echo "Cluster and resources down"
name: "Cleanupcluster"
displayName: "Cleanup cluster"
Expand Down
15 changes: 14 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -438,6 +438,19 @@ npm-image-pull: ## pull cns container image.
IMAGE=$(NPM_IMAGE) \
TAG=$(NPM_PLATFORM_TAG)

# cni-plugin

cni-plugin-image: ## build cni plugin container image.
$(MAKE) container \
DOCKERFILE=cni/build/$(OS).Dockerfile \
IMAGE=$(ACNCLI_IMAGE) \
EXTRA_BUILD_ARGS='--build-arg CNI_AI_PATH=$(CNI_AI_PATH) --build-arg CNI_AI_ID=$(CNI_AI_ID) --build-arg OS_VERSION=$(OS_VERSION)' \
PLATFORM=$(PLATFORM) \
TAG=$(ACNCLI_PLATFORM_TAG) \
OS=$(OS) \
ARCH=$(ARCH) \
OS_VERSION=$(OS_VERSION)


## Legacy

Expand Down Expand Up @@ -724,7 +737,7 @@ test-integration: ## run all integration tests.
go test -mod=readonly -buildvcs=false -timeout 1h -coverpkg=./... -race -covermode atomic -coverprofile=coverage.out -tags=integration ./test/integration...

test-validate-state:
cd test/integration/load && go test -count 1 -timeout 30m -tags load -run ^TestValidateState -tags=load -restart-case=$(RESTART_CASE)
cd test/integration/load && go test -count 1 -timeout 30m -tags load -run ^TestValidateState -tags=load -restart-case=$(RESTART_CASE) -os=$(OS)
cd ../../..

test-cyclonus: ## run the cyclonus test for npm.
Expand Down
20 changes: 20 additions & 0 deletions cni/build/windows.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
ARG OS_VERSION
FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.20 AS builder
ARG VERSION
ARG CNI_AI_PATH
ARG CNI_AI_ID
WORKDIR /azure-container-networking
COPY . .
RUN GOOS=windows CGO_ENABLED=0 go build -a -o azure-vnet.exe -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/network/plugin/main.go
RUN GOOS=windows CGO_ENABLED=0 go build -a -o azure-vnet-telemetry.exe -trimpath -ldflags "-X main.version="$VERSION" -X "$CNI_AI_PATH"="$CNI_AI_ID"" -gcflags="-dwarflocationlists=true" cni/telemetry/service/telemetrymain.go
RUN GOOS=windows CGO_ENABLED=0 go build -a -o azure-vnet-ipam.exe -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/ipam/plugin/main.go

FROM mcr.microsoft.com/windows/servercore:${OS_VERSION}
SHELL ["powershell", "-command"]
COPY --from=builder /azure-container-networking/azure-vnet.exe azure-vnet.exe
COPY --from=builder /azure-container-networking/azure-vnet-telemetry.exe azure-vnet-telemetry.exe
COPY --from=builder /azure-container-networking/azure-vnet-ipam.exe azure-vnet-ipam.exe

# This would be replaced with dropgz version of windows.
COPY --from=builder /azure-container-networking/hack/scripts/updatecni.ps1 updatecni.ps1
ENTRYPOINT ["powershell.exe", ".\\updatecni.ps1"]
Loading

0 comments on commit e45ad21

Please sign in to comment.