From 8f00357bcd4605bb4284ecb3d59a4246d23b7014 Mon Sep 17 00:00:00 2001 From: Vipul Singh Date: Thu, 13 Jul 2023 20:40:26 -0700 Subject: [PATCH] ci: [CNI] Adding aks cluster creation steps for k8s e2e test (#2052) * ci: [CNI] Adding aks cluster creation steps for k8s e2e test * Add validate step to the pipeline * Adding the telemetry config to the cluster (cherry picked from commit 846e508b17d0ff292693744c1db5ba932692a6f7) --- .../windows-cni-load-test-template.yaml | 134 ++++++++++++++++++ .pipelines/pipeline.yaml | 57 ++++++++ .../singletenancy/aks/e2e-job-template.yaml | 44 ++++++ .../singletenancy/aks/e2e-step-template.yaml | 95 +++++++++++++ Makefile | 43 ++++-- cni/build/windows.Dockerfile | 21 +++ dropgz/build/cniTest.Dockerfile | 3 + hack/scripts/updatecni.ps1 | 57 ++++++++ hack/swift/Makefile | 49 ++++++- .../manifests/cni/cni-installer-v1.yaml | 79 +++++++++++ 10 files changed, 563 insertions(+), 19 deletions(-) create mode 100644 .pipelines/cni/singletenancy/windows-cni-load-test-template.yaml create mode 100644 .pipelines/singletenancy/aks/e2e-job-template.yaml create mode 100644 .pipelines/singletenancy/aks/e2e-step-template.yaml create mode 100644 cni/build/windows.Dockerfile create mode 100644 hack/scripts/updatecni.ps1 create mode 100644 test/integration/manifests/cni/cni-installer-v1.yaml diff --git a/.pipelines/cni/singletenancy/windows-cni-load-test-template.yaml b/.pipelines/cni/singletenancy/windows-cni-load-test-template.yaml new file mode 100644 index 00000000000..d4063758662 --- /dev/null +++ b/.pipelines/cni/singletenancy/windows-cni-load-test-template.yaml @@ -0,0 +1,134 @@ +parameters: + dependsOn: "" + name: "" + clusterType: "" + clusterName: "" + nodeCount: "" + vmSize: "" + windowsVMSize: "" + os: "" + cni: "" + +stages: + - stage: createAKSclusterWindows + dependsOn: ${{ parameters.dependsOn }} + displayName: "Windows AKS Cluster ${{ parameters.cni }}" + jobs: + - job: create_aks_cluster_with_${{ parameters.name }} + steps: + - template: ../load-test-templates/create-cluster-template.yaml + parameters: + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }} + nodeCount: ${{ parameters.nodeCount }} + vmSize: ${{ parameters.vmSize }} + windowsVMSize: ${{ parameters.windowsVMSize }} + - stage: build_images + dependsOn: ${{ parameters.dependsOn }} + displayName: "Build CNI Images" + jobs: + - job: build_cni_images + pool: + name: "$(BUILD_POOL_NAME_LINUX_AMD64)" + strategy: + matrix: + windows_cniv1_amd64: + os: windows + name: cni-plugin + arch: amd64 + os_version: ltsc2022 + steps: + - template: ../../containers/container-template.yaml + parameters: + arch: $(arch) + name: $(name) + os: $(os) + os_version: $(os_version) + - stage: update_cni + dependsOn: + - createAKSclusterWindows + - build_images + displayName: "Update CNI on Cluster" + jobs: + - job: deploy_pods + strategy: + matrix: + windows_cniv1_amd64: + os: windows + arch: amd64 + os_version: ltsc2022 + steps: + - task: AzureCLI@1 + inputs: + azureSubscription: $(TEST_SUB_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -ex + export CNI_IMAGE=$(make cni-plugin-image-name-and-tag OS=$(os) ARCH=$(arch) OS_VERSION=$(os_version)) + az extension add --name aks-preview + clusterName=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName} + make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION) + envsubst < ./hack/manifests/windows-update.yaml | kubectl apply -f - + name: "UploadCNI" + displayName: "Upload CNI" + - script: | + set -ex + kubectl rollout status daemonset/azure-cni-windows -n kube-system + kubectl get pods -A + name: "WaitForCNI" + displayName: "Wait For CNI" + - stage: pod_deployment_windows + dependsOn: update_cni + displayName: "Pod Deployment" + jobs: + - job: deploy_pods + steps: + - template: ../load-test-templates/pod-deployment-template.yaml + parameters: + clusterName: ${{ parameters.clusterName }} + scaleup: ${WINDOWS_SCALEUP} + os: ${{ parameters.os }} + cni: ${{ parameters.cni }} + iterations: ${WINDOWS_ITERATIONS} + - stage: validate_state_windows + dependsOn: pod_deployment_windows + displayName: "Validate State" + jobs: + - job: validate_state + steps: + - template: ../load-test-templates/validate-state-template.yaml + parameters: + clusterName: ${{ parameters.clusterName }} + os: ${{ parameters.os }} + cni: ${{ parameters.cni }} + - stage: delete_resources + displayName: "Delete Resources" + dependsOn: + - validate_state_windows + jobs: + - job: delete_resources + steps: + - task: AzureCLI@1 + inputs: + azureSubscription: $(TEST_SUB_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -ex + if [ "$(DELETE_RESOURCES)" ] + then + echo "Deleting Cluster and resource group" + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION) + make -C ./hack/aks down AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${{ parameters.clusterName }}-$(make revision) + echo "Cluster and resources down" + else + echo "Deletion of resources is False" + fi + name: "CleanUpCluster" + displayName: "Cleanup cluster" + condition: always() diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index f136e418469..5ff1317b8bf 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -37,6 +37,7 @@ stages: echo "##vso[task.setvariable variable=Tag;isOutput=true]$(make version)" echo "##vso[task.setvariable variable=cniVersion;isOutput=true]$(make cni-version)" echo "##vso[task.setvariable variable=npmVersion;isOutput=true]$(make npm-version)" + echo "##vso[task.setvariable variable=dropgzVersion;isOutput=true]$(make cni-dropgz-version)" cat /etc/os-release uname -a sudo chown -R $(whoami):$(whoami) . @@ -167,6 +168,16 @@ stages: arch: amd64 name: cni-dropgz-test os: linux + cni_plugin_windows2022_amd64: + os: windows + name: cni-plugin + arch: amd64 + os_version: ltsc2022 + cni_plugin_windows2019_amd64: + os: windows + name: cni-plugin + arch: amd64 + os_version: ltsc2019 cns_linux_amd64: arch: amd64 name: cns @@ -344,6 +355,48 @@ stages: clusterName: "swifte2e" osSku: "Ubuntu" + - template: singletenancy/aks/e2e-job-template.yaml + parameters: + name: "aks_ubuntu_18_04_linux_e2e" + displayName: AKS Ubuntu 18.04 + arch: 'amd64' + os: 'linux' + clusterType: linux-cniv1-up + clusterName: 'ubuntu18e2e' + k8sVersion: 1.24.9 + + - template: singletenancy/aks/e2e-job-template.yaml + parameters: + name: "aks_ubuntu_22_linux_e2e" + displayName: AKS Ubuntu 22 + arch: 'amd64' + os: 'linux' + clusterType: linux-cniv1-up + clusterName: 'ubuntu22e2e' + k8sVersion: 1.25 + + - template: singletenancy/aks/e2e-job-template.yaml + parameters: + name: "aks_windows_19_03_e2e" + displayName: AKS Windows 1903 + arch: amd64 + os: windows + clusterType: windows-cniv1-up + clusterName: 'win19e2e' + windowsOsSku: 'Windows2019' + os_version: 'ltsc2019' + + - template: singletenancy/aks/e2e-job-template.yaml + parameters: + name: "aks_windows_22_e2e" + displayName: AKS Windows 2022 + arch: amd64 + os: windows + clusterType: windows-cniv1-up + clusterName: 'win22e2e' + windowsOsSku: 'Windows2022' + os_version: 'ltsc2022' + - template: singletenancy/aks-engine/e2e-job-template.yaml parameters: name: "ubuntu_18_04_linux_e2e" @@ -389,6 +442,10 @@ stages: dependsOn: - "aks_swift_e2e" - "cilium_e2e" + - "aks_ubuntu_18_04_linux_e2e" + - "aks_windows_19_03_e2e" + - "aks_ubuntu_22_linux_e2e" + - "aks_windows_22_e2e" - "ubuntu_18_04_linux_e2e" - "windows_19_03_e2e" jobs: diff --git a/.pipelines/singletenancy/aks/e2e-job-template.yaml b/.pipelines/singletenancy/aks/e2e-job-template.yaml new file mode 100644 index 00000000000..464d4f75524 --- /dev/null +++ b/.pipelines/singletenancy/aks/e2e-job-template.yaml @@ -0,0 +1,44 @@ +parameters: + name: "" + displayName: "" + arch: "" + os: "" + clusterType: "" + clusterName: "" + k8sVersion: "" + windowsOsSku: "" + os_version: "" + +stages: + - stage: ${{ parameters.name }} + displayName: E2E - ${{ parameters.displayName }} + dependsOn: + - setup + - publish + jobs: + - job: ${{ parameters.name }} + displayName: Singletenancy AKS - (${{ parameters.name }}) + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + demands: + - agent.os -equals Linux + - Role -equals $(CUSTOM_E2E_ROLE) + variables: + GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path + GOBIN: "$(GOPATH)/bin" # Go binaries path + modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" + dropgzVersion: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.dropgzVersion'] ] + cniVersion: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.cniVersion'] ] + steps: + - template: e2e-step-template.yaml + parameters: + name: ${{ parameters.name }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }} + arch: ${{ parameters.arch }} + os: ${{ parameters.os }} + k8sVersion: ${{ parameters.k8sVersion }} + windowsOsSku: ${{ parameters.windowsOsSku }} + os_version: ${{ parameters.os_version }} + version: $(dropgzVersion) + cniVersion: $(cniVersion) \ No newline at end of file diff --git a/.pipelines/singletenancy/aks/e2e-step-template.yaml b/.pipelines/singletenancy/aks/e2e-step-template.yaml new file mode 100644 index 00000000000..0fd05f3e6dd --- /dev/null +++ b/.pipelines/singletenancy/aks/e2e-step-template.yaml @@ -0,0 +1,95 @@ +parameters: + name: "" + clusterType: "" + clusterName: "" + nodeCount: "" + vmSize: "" + windowsVMSize: "" + k8sVersion: "" + version: "" + os: "" + windowsOsSku: "" + cniVersion: "" + os_version: "" + +steps: + - bash: | + go version + go env + mkdir -p '$(GOBIN)' + mkdir -p '$(GOPATH)/pkg' + mkdir -p '$(modulePath)' + echo '##vso[task.prependpath]$(GOBIN)' + echo '##vso[task.prependpath]$(GOROOT)/bin' + name: "GoEnv" + displayName: "Set up the Go environment" + - task: AzureCLI@1 + inputs: + azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + mkdir -p ~/.kube/ + echo "Create AKS cluster" + echo "parameters ${{ parameters.windowsOsSku }}" + make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) + make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) K8S_VER=${{ parameters.k8sVersion }} WINDOWS_OS_SKU=${{ parameters.windowsOsSku }} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} + echo "Cluster successfully created" + displayName: Create test cluster + - script: | + echo "Upload CNI" + if [ "${{parameters.os}}" == "windows" ]; then + export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS='linux' ARCH=${{ parameters.arch }} CNI_DROPGZ_VERSION=${{ parameters.version }}) + envsubst < ./test/integration/manifests/cni/cni-installer-v1.yaml | kubectl apply -f - + kubectl rollout status daemonset/azure-cni -n kube-system + export CNI_IMAGE=$(make cni-plugin-image-name-and-tag OS=${{ parameters.os }} ARCH=${{ parameters.arch }} OS_VERSION=${{ parameters.os_version }} CNI_VERSION=${{ parameters.cniVersion }}) + envsubst < ./hack/manifests/windows-update.yaml | kubectl apply -f - + kubectl rollout status daemonset/azure-cni-windows -n kube-system + else + export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS=${{ parameters.os }} ARCH=${{ parameters.arch }} CNI_DROPGZ_VERSION=${{ parameters.version }}) + envsubst < ./test/integration/manifests/cni/cni-installer-v1.yaml | kubectl apply -f - + kubectl rollout status daemonset/azure-cni -n kube-system + fi + name: "UploadCni" + displayName: "Upload CNI" + - task: AzureCLI@1 + inputs: + azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + clusterName=${{ parameters.clusterName }}-$(make revision) + echo "Restarting nodes" + for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} + done + displayName: "Restart Nodes" + - script: | + kubectl get pods -A -o wide + echo "Deploying test pods" + cd test/integration/load + go test -count 1 -timeout 30m -tags load -run ^TestLoad$ -tags=load -iterations=2 -scaleup=40 -os=${{ parameters.os }} + cd ../../.. + # Remove this once we have cniv1 support for validating the test cluster + echo "Validate State skipped for linux cniv1 for now" + if [ "${{parameters.os}}" == "windows" ]; then + make test-validate-state OS=${{ parameters.os }} + fi + displayName: "Validate State" + retryCountOnTaskFailure: 3 + - task: AzureCLI@1 + inputs: + azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + echo "Skipping Deleting cluster" + make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) + make -C ./hack/aks down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) + echo "Cluster and resources down" + displayName: "Delete test cluster" + condition: always() + diff --git a/Makefile b/Makefile index e65776018ce..5328cc52282 100644 --- a/Makefile +++ b/Makefile @@ -140,7 +140,7 @@ azure-ipam: azure-ipam-binary azure-ipam-archive revision: ## print the current git revision @echo $(REVISION) - + version: ## prints the root version @echo $(ACN_VERSION) @@ -159,15 +159,15 @@ cni-dropgz-test-version: ## prints the cni-dropgz version @echo $(CNI_DROPGZ_TEST_VERSION) cns-version: - @echo $(CNS_VERSION) + @echo $(CNS_VERSION) npm-version: - @echo $(NPM_VERSION) + @echo $(NPM_VERSION) zapai-version: ## prints the zapai version @echo $(ZAPAI_VERSION) -##@ Binaries +##@ Binaries # Build the delegated IPAM plugin binary. azure-ipam-binary: @@ -245,6 +245,7 @@ endif ## Image name definitions. ACNCLI_IMAGE = acncli +CNI_PLUGIN_IMAGE = azure-cni-plugin CNI_DROPGZ_IMAGE = cni-dropgz CNI_DROPGZ_TEST_IMAGE = cni-dropgz-test CNS_IMAGE = azure-cns @@ -252,6 +253,7 @@ NPM_IMAGE = azure-npm ## Image platform tags. ACNCLI_PLATFORM_TAG ?= $(subst /,-,$(PLATFORM))$(if $(OS_VERSION),-$(OS_VERSION),)-$(ACN_VERSION) +CNI_PLUGIN_PLATFORM_TAG ?= $(subst /,-,$(PLATFORM))$(if $(OS_VERSION),-$(OS_VERSION),)-$(CNI_VERSION) CNI_DROPGZ_PLATFORM_TAG ?= $(subst /,-,$(PLATFORM))$(if $(OS_VERSION),-$(OS_VERSION),)-$(CNI_DROPGZ_VERSION) CNI_DROPGZ_TEST_PLATFORM_TAG ?= $(subst /,-,$(PLATFORM))$(if $(OS_VERSION),-$(OS_VERSION),)-$(CNI_DROPGZ_TEST_VERSION) CNS_PLATFORM_TAG ?= $(subst /,-,$(PLATFORM))$(if $(OS_VERSION),-$(OS_VERSION),)-$(CNS_VERSION) @@ -436,6 +438,21 @@ npm-image-pull: ## pull cns container image. IMAGE=$(NPM_IMAGE) \ TAG=$(NPM_PLATFORM_TAG) +# cni-plugin - Specifically used for windows clusters, will be removed once we have Dropgz for windows +cni-plugin-image-name-and-tag: # util target to print the CNI plugin image name and tag. + @echo $(IMAGE_REGISTRY)/$(CNI_PLUGIN_IMAGE):$(CNI_PLUGIN_PLATFORM_TAG) + +cni-plugin-image: ## build cni plugin container image. + $(MAKE) container \ + DOCKERFILE=cni/build/$(OS).Dockerfile \ + IMAGE=$(CNI_PLUGIN_IMAGE) \ + EXTRA_BUILD_ARGS='--build-arg CNI_AI_PATH=$(CNI_AI_PATH) --build-arg CNI_AI_ID=$(CNI_AI_ID) --build-arg OS_VERSION=$(OS_VERSION)' \ + PLATFORM=$(PLATFORM) \ + TAG=$(CNI_PLUGIN_PLATFORM_TAG) \ + OS=$(OS) \ + ARCH=$(ARCH) \ + OS_VERSION=$(OS_VERSION) + ## Legacy @@ -490,8 +507,8 @@ manifest-build: # util target to compose multiarch container manifests from plat $(MAKE) manifest-add PLATFORM=$(PLATFORM);\ )\ )\ - - + + manifest-push: # util target to push multiarch container manifest. $(CONTAINER_BUILDER) manifest push --all $(IMAGE_REGISTRY)/$(IMAGE):$(TAG) docker://$(IMAGE_REGISTRY)/$(IMAGE):$(TAG) @@ -515,7 +532,7 @@ acncli-manifest-push: ## push acncli multiplat container manifest acncli-skopeo-archive: ## export tar archive of acncli multiplat container manifest. $(MAKE) manifest-skopeo-archive \ IMAGE=$(ACNCLI_IMAGE) \ - TAG=$(ACN_VERSION) + TAG=$(ACN_VERSION) cni-dropgz-manifest-build: ## build cni-dropgz multiplat container manifest. $(MAKE) manifest-build \ @@ -667,7 +684,7 @@ ifeq ($(GOOS),linux) endif -##@ Utils +##@ Utils clean: ## Clean build artifacts. $(RMDIR) $(OUTPUT_DIR) @@ -698,7 +715,7 @@ workspace: ## Set up the Go workspace. go work use ./dropgz go work use ./zapai -##@ Test +##@ Test COVER_PKG ?= . #Restart case is used for cni load test pipeline for restarting the nodes cluster. @@ -756,7 +773,7 @@ gitconfig: ## configure the local git repository setup: tools install-hooks gitconfig ## performs common required repo setup -##@ Tools +##@ Tools $(TOOLS_DIR)/go.mod: cd $(TOOLS_DIR); go mod init && go mod tidy @@ -766,7 +783,7 @@ $(CONTROLLER_GEN): $(TOOLS_DIR)/go.mod controller-gen: $(CONTROLLER_GEN) ## Build controller-gen -protoc: +protoc: source ${REPO_ROOT}/scripts/install-protoc.sh $(GOCOV): $(TOOLS_DIR)/go.mod @@ -799,13 +816,13 @@ $(MOCKGEN): $(TOOLS_DIR)/go.mod mockgen: $(MOCKGEN) ## Build mockgen -clean-tools: +clean-tools: rm -r build/tools/bin tools: acncli gocov gocov-xml go-junit-report golangci-lint gofumpt protoc ## Build bins for build tools -##@ Help +##@ Help help: ## Display this help @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/cni/build/windows.Dockerfile b/cni/build/windows.Dockerfile new file mode 100644 index 00000000000..285382dd379 --- /dev/null +++ b/cni/build/windows.Dockerfile @@ -0,0 +1,21 @@ +ARG OS_VERSION +FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.20 AS builder +ARG VERSION +ARG CNI_AI_PATH +ARG CNI_AI_ID +WORKDIR /azure-container-networking +COPY . . +RUN GOOS=windows CGO_ENABLED=0 go build -a -o azure-vnet.exe -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/network/plugin/main.go +RUN GOOS=windows CGO_ENABLED=0 go build -a -o azure-vnet-telemetry.exe -trimpath -ldflags "-X main.version="$VERSION" -X "$CNI_AI_PATH"="$CNI_AI_ID"" -gcflags="-dwarflocationlists=true" cni/telemetry/service/telemetrymain.go +RUN GOOS=windows CGO_ENABLED=0 go build -a -o azure-vnet-ipam.exe -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/ipam/plugin/main.go + +FROM mcr.microsoft.com/windows/servercore:${OS_VERSION} +SHELL ["powershell", "-command"] +COPY --from=builder /azure-container-networking/azure-vnet.exe azure-vnet.exe +COPY --from=builder /azure-container-networking/azure-vnet-telemetry.exe azure-vnet-telemetry.exe +COPY --from=builder /azure-container-networking/telemetry/azure-vnet-telemetry.config azure-vnet-telemetry.config +COPY --from=builder /azure-container-networking/azure-vnet-ipam.exe azure-vnet-ipam.exe + +# This would be replaced with dropgz version of windows. +COPY --from=builder /azure-container-networking/hack/scripts/updatecni.ps1 updatecni.ps1 +ENTRYPOINT ["powershell.exe", ".\\updatecni.ps1"] diff --git a/dropgz/build/cniTest.Dockerfile b/dropgz/build/cniTest.Dockerfile index 21722963ce0..083901003a6 100644 --- a/dropgz/build/cniTest.Dockerfile +++ b/dropgz/build/cniTest.Dockerfile @@ -16,9 +16,12 @@ WORKDIR /dropgz COPY dropgz . COPY --from=azure-ipam /azure-ipam/*.conflist pkg/embed/fs COPY --from=azure-ipam /azure-ipam/bin/* pkg/embed/fs +COPY --from=azure-vnet /azure-container-networking/cni/azure-$OS.conflist pkg/embed/fs/azure.conflist COPY --from=azure-vnet /azure-container-networking/cni/azure-$OS-swift.conflist pkg/embed/fs/azure-swift.conflist COPY --from=azure-vnet /azure-container-networking/cni/azure-$OS-swift-overlay.conflist pkg/embed/fs/azure-swift-overlay.conflist COPY --from=azure-vnet /azure-container-networking/bin/* pkg/embed/fs +COPY --from=azure-vnet /azure-container-networking/telemetry/azure-vnet-telemetry.config pkg/embed/fs + RUN cd pkg/embed/fs/ && sha256sum * > sum.txt RUN gzip --verbose --best --recursive pkg/embed/fs && for f in pkg/embed/fs/*.gz; do mv -- "$f" "${f%%.gz}"; done diff --git a/hack/scripts/updatecni.ps1 b/hack/scripts/updatecni.ps1 new file mode 100644 index 00000000000..630c5fbbc72 --- /dev/null +++ b/hack/scripts/updatecni.ps1 @@ -0,0 +1,57 @@ +Write-Host $env:CONTAINER_SANDBOX_MOUNT_POINT +$sourceCNI = $env:CONTAINER_SANDBOX_MOUNT_POINT + "azure-vnet.exe" +$sourceIpam = $env:CONTAINER_SANDBOX_MOUNT_POINT + "azure-vnet-ipam.exe" +$sourceTelemetry = $env:CONTAINER_SANDBOX_MOUNT_POINT + "azure-vnet-telemetry.exe" +$sourceTelemetryConfig = $env:CONTAINER_SANDBOX_MOUNT_POINT + "azure-vnet-telemetry.config" + +$sourceCNIVersion = & "$sourceCNI" -v +$currentVersion = "" +$sourceTelemetryVersion = & "$sourceTelemetry" -v +$currentTelemetryVersion = "" + +$cniExists = Test-Path "C:\k\azurecni\bin\azure-vnet.exe" +$telemetryExists = Test-Path "C:\k\azurecni\bin\azure-vnet-telemetry.exe" + +Write-Host "Source $sourceCNIVersion" +Write-Host "Source Telemetry $sourceTelemetryVersion" + +if ($cniExists) { + $currentVersion = & "C:\k\azurecni\bin\azure-vnet.exe" -v +} + +if($telemetryExists){ + $currentTelemetryVersion = & "C:\k\azurecni\bin\azure-vnet-telemetry.exe" -v +} + + +Write-Host "Current Host $currentVersion" +Write-Host "Current Telemetry $currentTelemetryVersion" + +## check telemetry was already installed so not to get stuck in a infinite loop of rebooting and killing the process +if ($currentTelemetryVersion -ne $sourceTelemetryVersion){ + $processes = Get-Process -Name azure-vnet-telemetry -ErrorAction SilentlyContinue + for ($i = 0; $i -lt $processes.Count; $i++) { + Write-Host "Killing azure-vnet-telemetry process..." + $processes[$i].Kill() + } + Write-Host "copying azure-vnet-telemetry to windows node..." + Remove-Item "C:\k\azurecni\bin\azure-vnet-telemetry.exe" + Copy-Item $sourceTelemetry -Destination "C:\k\azurecni\bin" + + Write-Host "copying azure-vnet-telemetry.config to windows node..." + Remove-Item "C:\k\azurecni\bin\azure-vnet-telemetry.config" + Copy-Item $sourceTelemetryConfig -Destination "C:\k\azurecni\bin" +} + +## check CNI was already installed so not to get stuck in a infinite loop of rebooting +if ($currentVersion -ne $sourceCNIVersion){ + Write-Host "copying azure-vnet to windows node..." + Remove-Item "C:\k\azurecni\bin\azure-vnet.exe" + Copy-Item $sourceCNI -Destination "C:\k\azurecni\bin" + + Write-Host "copying azure-vnet-ipam to windows node..." + Remove-Item "C:\k\azurecni\bin\azure-vnet-ipam.exe" + Copy-Item $sourceIpam -Destination "C:\k\azurecni\bin" +} + +Start-Sleep -s 1000 diff --git a/hack/swift/Makefile b/hack/swift/Makefile index 58d185d2506..126c38c9660 100644 --- a/hack/swift/Makefile +++ b/hack/swift/Makefile @@ -8,10 +8,13 @@ AZIMG = mcr.microsoft.com/azure-cli AZCLI ?= docker run --rm -v $(AZCFG):/root/.azure -v $(KUBECFG):/root/.kube -v $(SSH):/root/.ssh $(AZIMG) az # overrideable defaults -REGION ?= westus2 -OS_SKU ?= Ubuntu -VM_SIZE ?= Standard_B2s -NODE_COUNT ?= 2 +REGION ?= westus2 +OS_SKU ?= Ubuntu +WINDOWS_OS_SKU ?= Windows2022 +VM_SIZE ?= Standard_B2s +NODE_COUNT ?= 2 +K8S_VER ?= 1.25 # Used only for ubuntu 18 as K8S 1.24.9, as K8S > 1.25 have Ubuntu 22 +WINDOWS_VM_SKU ?= Standard_B2s # overrideable variables SUB ?= $(AZURE_SUBSCRIPTION) @@ -33,7 +36,7 @@ azlogin: azcfg: @$(AZCLI) extension add --name aks-preview --yes - @$(AZCLI) extension update --name aks-preview + @$(AZCLI) extension update --name aks-preview set-kubeconf: ## Adds the kubeconf for $CLUSTER $(AZCLI) aks get-credentials -n $(CLUSTER) -g $(GROUP) @@ -61,7 +64,7 @@ vars: ## Show the input vars configured for the cluster commands ##@ SWIFT Infra -rg-up: ## Create resource group +rg-up: ## Create resource group @$(AZCLI) group create --location $(REGION) --name $(GROUP) rg-down: ## Delete resource group @@ -162,6 +165,40 @@ swift-up: rg-up swift-net-up ## Bring up a SWIFT AzCNI cluster --yes @$(MAKE) set-kubeconf +windows-cniv1-up: rg-up overlay-net-up ## Bring up a Windows CNIv1 cluster + $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --node-count $(NODE_COUNT) \ + --node-vm-size $(VM_SIZE) \ + --network-plugin azure \ + --windows-admin-password $(WINDOWS_PASSWORD) \ + --windows-admin-username $(WINDOWS_USERNAME) \ + --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ + --no-ssh-key \ + --yes + + $(AZCLI) aks nodepool add --resource-group $(GROUP) --cluster-name $(CLUSTER) \ + --os-type Windows \ + --os-sku $(WINDOWS_OS_SKU) \ + --max-pods 250 \ + --name npwin \ + --node-count $(NODE_COUNT) \ + -s $(WINDOWS_VM_SKU) + + @$(MAKE) set-kubeconf + +linux-cniv1-up: rg-up overlay-net-up + $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --node-count $(NODE_COUNT) \ + --node-vm-size $(VM_SIZE) \ + --network-plugin azure \ + --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ + --kubernetes-version $(K8S_VER) \ + --os-sku $(OS_SKU) \ + --no-ssh-key \ + --yes + + @$(MAKE) set-kubeconf + down: ## Delete the cluster $(AZCLI) aks delete -g $(GROUP) -n $(CLUSTER) --yes @$(MAKE) unset-kubeconf diff --git a/test/integration/manifests/cni/cni-installer-v1.yaml b/test/integration/manifests/cni/cni-installer-v1.yaml new file mode 100644 index 00000000000..74565cca130 --- /dev/null +++ b/test/integration/manifests/cni/cni-installer-v1.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: azure-cni + namespace: kube-system + labels: + app: azure-cni +spec: + selector: + matchLabels: + k8s-app: azure-cni + template: + metadata: + labels: + k8s-app: azure-cni + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: kubernetes.io/os + operator: In + values: + - linux + priorityClassName: system-node-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + initContainers: + - name: azure-cni + image: ${DROP_GZ_URL} + imagePullPolicy: Always + command: ["/dropgz"] + args: + - deploy + - azure-vnet + - -o + - /opt/cni/bin/azure-vnet + - azure-vnet-ipam + - -o + - /opt/cni/bin/azure-vnet-ipam + - azure-vnet-telemetry + - -o + - /opt/cni/bin/azure-vnet-telemetry + - azure.conflist + - -o + - /etc/cni/net.d/10-azure.conflist + - azure-vnet-telemetry.config + - -o + - /opt/cni/bin/azure-vnet-telemetry.config + volumeMounts: + - name: cni-bin + mountPath: /opt/cni/bin + - name: cni-conflist + mountPath: /etc/cni/net.d + containers: + - name: pause + image: mcr.microsoft.com/oss/kubernetes/pause:3.6 + hostNetwork: true + volumes: + - name: cni-conflist + hostPath: + path: /etc/cni/net.d + type: Directory + - name: cni-bin + hostPath: + path: /opt/cni/bin + type: Directory