Skip to content

Commit

Permalink
ci: [CNI] replace aks engine cluster with aks cluster creation
Browse files Browse the repository at this point in the history
  • Loading branch information
vipul-21 committed Jul 11, 2023
1 parent 672cce9 commit 66a665d
Show file tree
Hide file tree
Showing 5 changed files with 238 additions and 19 deletions.
54 changes: 40 additions & 14 deletions .pipelines/pipeline.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ stages:
echo "##vso[task.setvariable variable=Tag;isOutput=true]$(make version)"
echo "##vso[task.setvariable variable=cniVersion;isOutput=true]$(make cni-version)"
echo "##vso[task.setvariable variable=npmVersion;isOutput=true]$(make npm-version)"
echo "##vso[task.setvariable variable=dropgzVersion;isOutput=true]$(make cni-dropgz-version)"
cat /etc/os-release
uname -a
sudo chown -R $(whoami):$(whoami) .
Expand Down Expand Up @@ -166,6 +167,11 @@ stages:
arch: amd64
name: cni-dropgz-test
os: linux
cni_plugin_windows2019_amd64:
os: windows
name: cni-plugin
arch: amd64
os_version: ltsc2019
cns_linux_amd64:
arch: amd64
name: cns
Expand Down Expand Up @@ -330,25 +336,45 @@ stages:
clusterName: "swifte2e"
osSku: "Ubuntu"

- template: singletenancy/aks-engine/e2e-job-template.yaml
- template: singletenancy/aks/e2e-job-template.yaml
parameters:
name: "ubuntu_18_04_linux_e2e"
displayName: Ubuntu 18.04
pipelineBuildImage: "$(BUILD_IMAGE)"
clusterDefinition: "cniLinux1804.json"
clusterDefinitionCniTypeKey: "azureCNIURLLinux"
clusterDefinitionCniBuildOS: "linux"
clusterDefinitionCniBuildExt: ".tgz"
displayName: AKS Ubuntu 18.04
arch: amd64
os: linux
clusterType: linux-cniv1-up
clusterName: "ubuntu18e2e"
k8sVersion: 1.24.9

- template: singletenancy/aks-engine/e2e-job-template.yaml
- template: singletenancy/aks/e2e-job-template.yaml
parameters:
name: "windows_19_03_e2e"
displayName: "Windows 1903"
pipelineBuildImage: "$(BUILD_IMAGE)"
clusterDefinition: "cniWindows1903.json"
clusterDefinitionCniTypeKey: "azureCNIURLWindows"
clusterDefinitionCniBuildOS: "windows"
clusterDefinitionCniBuildExt: ".zip"
displayName: AKS Windows 1903
arch: amd64
os: linux
clusterType: windows-cniv1-up
clusterName: "win19e2e"
windowsOsSku: Windows2019

# - template: singletenancy/aks-engine/e2e-job-template.yaml
# parameters:
# name: "ubuntu_18_04_linux_e2e"
# displayName: Ubuntu 18.04
# pipelineBuildImage: "$(BUILD_IMAGE)"
# clusterDefinition: "cniLinux1804.json"
# clusterDefinitionCniTypeKey: "azureCNIURLLinux"
# clusterDefinitionCniBuildOS: "linux"
# clusterDefinitionCniBuildExt: ".tgz"

# - template: singletenancy/aks-engine/e2e-job-template.yaml
# parameters:
# name: "windows_19_03_e2e"
# displayName: "Windows 1903"
# pipelineBuildImage: "$(BUILD_IMAGE)"
# clusterDefinition: "cniWindows1903.json"
# clusterDefinitionCniTypeKey: "azureCNIURLWindows"
# clusterDefinitionCniBuildOS: "windows"
# clusterDefinitionCniBuildExt: ".zip"

- stage: validate2
displayName: Validate Tags
Expand Down
40 changes: 40 additions & 0 deletions .pipelines/singletenancy/aks/e2e-job-template.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
parameters:
name: ""
displayName: ""
arch: ""
os: ""
clusterType: ""
clusterName: ""
k8sVersion: ""
windowsOsSku: ""

stages:
- stage: ${{ parameters.name }}
displayName: E2E - ${{ parameters.displayName }}
dependsOn:
- setup
- publish
jobs:
- job: ${{ parameters.name }}
displayName: Singletenancy AKS - (${{ parameters.name }})
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
- agent.os -equals Linux
- Role -equals $(CUSTOM_E2E_ROLE)
variables:
GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
GOBIN: "$(GOPATH)/bin" # Go binaries path
modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
dropgzVersion: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.dropgzVersion'] ]
steps:
- template: e2e-step-template.yaml
parameters:
name: ${{ parameters.name }}
clusterType: ${{ parameters.clusterType }}
clusterName: ${{ parameters.clusterName }}
arch: ${{ parameters.arch }}
os: ${{ parameters.os }}
k8sVersion: ${{ parameters.k8sVersion }}
version: $(dropgzVersion)
windowsOsSku: ${{ parameters.windowsOsSku}}
61 changes: 61 additions & 0 deletions .pipelines/singletenancy/aks/e2e-step-template.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
parameters:
name: ""
clusterType: ""
clusterName: ""
nodeCount: ""
vmSize: ""
windowsVMSize: ""
k8sVersion: ""
version: ""
os: ""
windowsOsSku: ""

steps:
- bash: |
go version
go env
mkdir -p '$(GOBIN)'
mkdir -p '$(GOPATH)/pkg'
mkdir -p '$(modulePath)'
echo '##vso[task.prependpath]$(GOBIN)'
echo '##vso[task.prependpath]$(GOROOT)/bin'
name: "GoEnv"
displayName: "Set up the Go environment"
- task: AzureCLI@1
inputs:
azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
mkdir -p ~/.kube/
echo "Create AKS cluster"
make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) K8S_VER=${{ parameters.k8sVersion }} \
WINDOWS_OS_SKU=${{parameters.windowsOsSku}} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD}
echo "Cluster successfully created"
displayName: Create test cluster
- script: |
echo "Upload CNI"
export DROP_GZ_URL=$( make cni-dropgz-image-name-and-tag OS=${{parameters.os}} ARCH=${{parameters.arch}} CNI_DROPGZ_VERSION=${{parameters.version}})
envsubst < ./test/integration/manifests/cni/cni-installer-v1.yaml | kubectl apply -f -
if [ "${{parameters.os}}" == "windows" ]; then
export CNI_IMAGE=$(make acncli-image-name-and-tag OS=${{parameters.os}} ARCH=${{parameters.arch}} ARCH=${{parameters.arch}})
envsubst < ./hack/manifests/windows-update.yaml | kubectl apply -f -
fi
name: "UploadCni"
displayName: "Upload CNI"
- task: AzureCLI@2
inputs:
azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
echo "Deleting cluster"
make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
make -C ./hack/aks down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision)
echo "Cluster and resources down"
name: "Cleanupcluster"
displayName: "Cleanup cluster"
condition: always()
26 changes: 21 additions & 5 deletions hack/aks/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,13 @@ AZIMG = mcr.microsoft.com/azure-cli
AZCLI ?= docker run --rm -v $(AZCFG):/root/.azure -v $(KUBECFG):/root/.kube -v $(SSH):/root/.ssh -v $(PWD):/root/tmpsrc $(AZIMG) az

# overrideable defaults
REGION ?= westus2
OS_SKU ?= Ubuntu
VM_SIZE ?= Standard_B2s
NODE_COUNT ?= 2
REGION ?= westus2
OS_SKU ?= Ubuntu
WINDOWS_OS_SKU ?= Windows2022
VM_SIZE ?= Standard_B2s
NODE_COUNT ?= 2
K8S_VER ?= 1.24.9 # Used only for ubuntu 18, as K8S > 1.25 have Ubuntu 22
WINDOWS_VM_SKU ?= Standard_B2s

# overrideable variables
SUB ?= $(AZURE_SUBSCRIPTION)
Expand Down Expand Up @@ -203,14 +206,27 @@ windows-cniv1-up: rg-up overlay-net-up ## Bring up a Windows CNIv1 cluster

$(AZCLI) aks nodepool add --resource-group $(GROUP) --cluster-name $(CLUSTER) \
--os-type Windows \
--os-sku Windows2022 \
--os-sku $(WINDOWS_OS_SKU) \
--max-pods 250 \
--name npwin \
--node-count $(NODE_COUNT) \
-s $(WINDOWS_VM_SKU)

@$(MAKE) set-kubeconf

linux-cniv1-up: rg-up overlay-net-up
$(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \
--node-count $(NODE_COUNT) \
--node-vm-size $(VM_SIZE) \
--network-plugin azure \
--vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \
--kubernetes-version $(K8S_VER) \
--os-sku $(OS_SKU) \
--no-ssh-key \
--yes

@$(MAKE) set-kubeconf

down: ## Delete the cluster
$(AZCLI) aks delete -g $(GROUP) -n $(CLUSTER) --yes
@$(MAKE) unset-kubeconf
Expand Down
76 changes: 76 additions & 0 deletions test/integration/manifests/cni/cni-installer-v1.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: azure-cni
namespace: kube-system
labels:
app: azure-cni
spec:
selector:
matchLabels:
k8s-app: azure-cni
template:
metadata:
labels:
k8s-app: azure-cni
annotations:
cluster-autoscaler.kubernetes.io/daemonset-pod: "true"
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: type
operator: NotIn
values:
- virtual-kubelet
- key: kubernetes.io/os
operator: In
values:
- linux
priorityClassName: system-node-critical
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- operator: "Exists"
effect: NoExecute
- operator: "Exists"
effect: NoSchedule
initContainers:
- name: azure-cni
image: ${DROP_GZ_URL}
imagePullPolicy: Always
command: ["/dropgz"]
args:
- deploy
- azure-vnet
- -o
- /opt/cni/bin/azure-vnet
- azure-vnet-ipam
- -o
- /opt/cni/bin/azure-vnet-ipam
- azure-vnet-telemetry
- -o
- /opt/cni/bin/azure-vnet-telemetry
- azure.conflist
- -o
- /etc/cni/net.d/10-azure.conflist
volumeMounts:
- name: cni-bin
mountPath: /opt/cni/bin
- name: cni-conflist
mountPath: /etc/cni/net.d
containers:
- name: pause
image: mcr.microsoft.com/oss/kubernetes/pause:3.6
hostNetwork: true
volumes:
- name: cni-conflist
hostPath:
path: /etc/cni/net.d
type: Directory
- name: cni-bin
hostPath:
path: /opt/cni/bin
type: Directory

0 comments on commit 66a665d

Please sign in to comment.