diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index e55af8c9e88..e3588181a5a 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -36,6 +36,7 @@ stages: echo "##vso[task.setvariable variable=Tag;isOutput=true]$(make version)" echo "##vso[task.setvariable variable=cniVersion;isOutput=true]$(make cni-version)" echo "##vso[task.setvariable variable=npmVersion;isOutput=true]$(make npm-version)" + echo "##vso[task.setvariable variable=dropgzVersion;isOutput=true]$(make cni-dropgz-version)" cat /etc/os-release uname -a sudo chown -R $(whoami):$(whoami) . @@ -166,6 +167,11 @@ stages: arch: amd64 name: cni-dropgz-test os: linux + cni_plugin_windows2019_amd64: + os: windows + name: cni-plugin + arch: amd64 + os_version: ltsc2019 cns_linux_amd64: arch: amd64 name: cns @@ -330,25 +336,45 @@ stages: clusterName: "swifte2e" osSku: "Ubuntu" - - template: singletenancy/aks-engine/e2e-job-template.yaml + - template: singletenancy/aks/e2e-job-template.yaml parameters: - name: "ubuntu_18_04_linux_e2e" - displayName: Ubuntu 18.04 - pipelineBuildImage: "$(BUILD_IMAGE)" - clusterDefinition: "cniLinux1804.json" - clusterDefinitionCniTypeKey: "azureCNIURLLinux" - clusterDefinitionCniBuildOS: "linux" - clusterDefinitionCniBuildExt: ".tgz" + name: "aks_ubuntu_18_04_linux_e2e" + displayName: AKS Ubuntu 18.04 + arch: amd64 + os: linux + clusterType: linux-cniv1-up + clusterName: "ubuntu18e2e" + k8sVersion: 1.24.9 - - template: singletenancy/aks-engine/e2e-job-template.yaml + - template: singletenancy/aks/e2e-job-template.yaml parameters: - name: "windows_19_03_e2e" - displayName: "Windows 1903" - pipelineBuildImage: "$(BUILD_IMAGE)" - clusterDefinition: "cniWindows1903.json" - clusterDefinitionCniTypeKey: "azureCNIURLWindows" - clusterDefinitionCniBuildOS: "windows" - clusterDefinitionCniBuildExt: ".zip" + name: "aks_windows_19_03_e2e" + displayName: AKS Windows 1903 + arch: amd64 + os: linux + clusterType: windows-cniv1-up + clusterName: "win19e2e" + windowsOsSku: Windows2019 + + # - template: singletenancy/aks-engine/e2e-job-template.yaml + # parameters: + # name: "ubuntu_18_04_linux_e2e" + # displayName: Ubuntu 18.04 + # pipelineBuildImage: "$(BUILD_IMAGE)" + # clusterDefinition: "cniLinux1804.json" + # clusterDefinitionCniTypeKey: "azureCNIURLLinux" + # clusterDefinitionCniBuildOS: "linux" + # clusterDefinitionCniBuildExt: ".tgz" + + # - template: singletenancy/aks-engine/e2e-job-template.yaml + # parameters: + # name: "windows_19_03_e2e" + # displayName: "Windows 1903" + # pipelineBuildImage: "$(BUILD_IMAGE)" + # clusterDefinition: "cniWindows1903.json" + # clusterDefinitionCniTypeKey: "azureCNIURLWindows" + # clusterDefinitionCniBuildOS: "windows" + # clusterDefinitionCniBuildExt: ".zip" - stage: validate2 displayName: Validate Tags diff --git a/.pipelines/singletenancy/aks/e2e-job-template.yaml b/.pipelines/singletenancy/aks/e2e-job-template.yaml new file mode 100644 index 00000000000..31655ed92dd --- /dev/null +++ b/.pipelines/singletenancy/aks/e2e-job-template.yaml @@ -0,0 +1,40 @@ +parameters: + name: "" + displayName: "" + arch: "" + os: "" + clusterType: "" + clusterName: "" + k8sVersion: "" + windowsOsSku: "" + +stages: + - stage: ${{ parameters.name }} + displayName: E2E - ${{ parameters.displayName }} + dependsOn: + - setup + - publish + jobs: + - job: ${{ parameters.name }} + displayName: Singletenancy AKS - (${{ parameters.name }}) + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + demands: + - agent.os -equals Linux + - Role -equals $(CUSTOM_E2E_ROLE) + variables: + GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path + GOBIN: "$(GOPATH)/bin" # Go binaries path + modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" + dropgzVersion: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.dropgzVersion'] ] + steps: + - template: e2e-step-template.yaml + parameters: + name: ${{ parameters.name }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }} + arch: ${{ parameters.arch }} + os: ${{ parameters.os }} + k8sVersion: ${{ parameters.k8sVersion }} + version: $(dropgzVersion) + windowsOsSku: ${{ parameters.windowsOsSku}} \ No newline at end of file diff --git a/.pipelines/singletenancy/aks/e2e-step-template.yaml b/.pipelines/singletenancy/aks/e2e-step-template.yaml new file mode 100644 index 00000000000..b84b7f3449c --- /dev/null +++ b/.pipelines/singletenancy/aks/e2e-step-template.yaml @@ -0,0 +1,61 @@ +parameters: + name: "" + clusterType: "" + clusterName: "" + nodeCount: "" + vmSize: "" + windowsVMSize: "" + k8sVersion: "" + version: "" + os: "" + windowsOsSku: "" + +steps: + - bash: | + go version + go env + mkdir -p '$(GOBIN)' + mkdir -p '$(GOPATH)/pkg' + mkdir -p '$(modulePath)' + echo '##vso[task.prependpath]$(GOBIN)' + echo '##vso[task.prependpath]$(GOROOT)/bin' + name: "GoEnv" + displayName: "Set up the Go environment" + - task: AzureCLI@1 + inputs: + azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + mkdir -p ~/.kube/ + echo "Create AKS cluster" + make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) + make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) K8S_VER=${{ parameters.k8sVersion }} \ + WINDOWS_OS_SKU=${{parameters.windowsOsSku}} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} + echo "Cluster successfully created" + displayName: Create test cluster + - script: | + echo "Upload CNI" + export DROP_GZ_URL=$( make cni-dropgz-image-name-and-tag OS=${{parameters.os}} ARCH=${{parameters.arch}} CNI_DROPGZ_VERSION=${{parameters.version}}) + envsubst < ./test/integration/manifests/cni/cni-installer-v1.yaml | kubectl apply -f - + if [ "${{parameters.os}}" == "windows" ]; then + export CNI_IMAGE=$(make acncli-image-name-and-tag OS=${{parameters.os}} ARCH=${{parameters.arch}} ARCH=${{parameters.arch}}) + envsubst < ./hack/manifests/windows-update.yaml | kubectl apply -f - + fi + name: "UploadCni" + displayName: "Upload CNI" + - task: AzureCLI@2 + inputs: + azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + echo "Deleting cluster" + make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) + make -C ./hack/aks down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) + echo "Cluster and resources down" + name: "Cleanupcluster" + displayName: "Cleanup cluster" + condition: always() diff --git a/hack/aks/Makefile b/hack/aks/Makefile index b16019b1dc5..f783d78a454 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -8,10 +8,13 @@ AZIMG = mcr.microsoft.com/azure-cli AZCLI ?= docker run --rm -v $(AZCFG):/root/.azure -v $(KUBECFG):/root/.kube -v $(SSH):/root/.ssh -v $(PWD):/root/tmpsrc $(AZIMG) az # overrideable defaults -REGION ?= westus2 -OS_SKU ?= Ubuntu -VM_SIZE ?= Standard_B2s -NODE_COUNT ?= 2 +REGION ?= westus2 +OS_SKU ?= Ubuntu +WINDOWS_OS_SKU ?= Windows2022 +VM_SIZE ?= Standard_B2s +NODE_COUNT ?= 2 +K8S_VER ?= 1.24.9 # Used only for ubuntu 18, as K8S > 1.25 have Ubuntu 22 +WINDOWS_VM_SKU ?= Standard_B2s # overrideable variables SUB ?= $(AZURE_SUBSCRIPTION) @@ -203,7 +206,7 @@ windows-cniv1-up: rg-up overlay-net-up ## Bring up a Windows CNIv1 cluster $(AZCLI) aks nodepool add --resource-group $(GROUP) --cluster-name $(CLUSTER) \ --os-type Windows \ - --os-sku Windows2022 \ + --os-sku $(WINDOWS_OS_SKU) \ --max-pods 250 \ --name npwin \ --node-count $(NODE_COUNT) \ @@ -211,6 +214,19 @@ windows-cniv1-up: rg-up overlay-net-up ## Bring up a Windows CNIv1 cluster @$(MAKE) set-kubeconf +linux-cniv1-up: rg-up overlay-net-up + $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --node-count $(NODE_COUNT) \ + --node-vm-size $(VM_SIZE) \ + --network-plugin azure \ + --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ + --kubernetes-version $(K8S_VER) \ + --os-sku $(OS_SKU) \ + --no-ssh-key \ + --yes + + @$(MAKE) set-kubeconf + down: ## Delete the cluster $(AZCLI) aks delete -g $(GROUP) -n $(CLUSTER) --yes @$(MAKE) unset-kubeconf diff --git a/test/integration/manifests/cni/cni-installer-v1.yaml b/test/integration/manifests/cni/cni-installer-v1.yaml new file mode 100644 index 00000000000..7cc135b1e08 --- /dev/null +++ b/test/integration/manifests/cni/cni-installer-v1.yaml @@ -0,0 +1,76 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: azure-cni + namespace: kube-system + labels: + app: azure-cni +spec: + selector: + matchLabels: + k8s-app: azure-cni + template: + metadata: + labels: + k8s-app: azure-cni + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: kubernetes.io/os + operator: In + values: + - linux + priorityClassName: system-node-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + initContainers: + - name: azure-cni + image: ${DROP_GZ_URL} + imagePullPolicy: Always + command: ["/dropgz"] + args: + - deploy + - azure-vnet + - -o + - /opt/cni/bin/azure-vnet + - azure-vnet-ipam + - -o + - /opt/cni/bin/azure-vnet-ipam + - azure-vnet-telemetry + - -o + - /opt/cni/bin/azure-vnet-telemetry + - azure.conflist + - -o + - /etc/cni/net.d/10-azure.conflist + volumeMounts: + - name: cni-bin + mountPath: /opt/cni/bin + - name: cni-conflist + mountPath: /etc/cni/net.d + containers: + - name: pause + image: mcr.microsoft.com/oss/kubernetes/pause:3.6 + hostNetwork: true + volumes: + - name: cni-conflist + hostPath: + path: /etc/cni/net.d + type: Directory + - name: cni-bin + hostPath: + path: /opt/cni/bin + type: Directory