diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index 46f320b19b..f24a75e853 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -274,18 +274,22 @@ stages: parameters: name: $(name) platforms: $(platforms) - + - template: singletenancy/cilium/cilium-e2e-job-template.yaml parameters: name: "cilium_e2e" displayName: Cilium pipelineBuildImage: "$(BUILD_IMAGE)" + testDropgz: "" + clusterName: "ciliume2e" - template: singletenancy/aks-swift/e2e-job-template.yaml parameters: name: "aks_swift_e2e" displayName: AKS Swift pipelineBuildImage: "$(BUILD_IMAGE)" + testDropgz: "" + clusterName: "swifte2e" - template: singletenancy/aks-engine/e2e-job-template.yaml parameters: @@ -331,6 +335,7 @@ stages: displayName: Cleanup dependsOn: - "aks_swift_e2e" + - "cilium_e2e" - "ubuntu_18_04_linux_e2e" - "windows_19_03_e2e" - "windows_20_04_e2e" diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml index a68af2f475..f79c23f867 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml @@ -2,6 +2,8 @@ parameters: name: "" displayName: "" pipelineBuildImage: "$(BUILD_IMAGE)" + testDropgz: "" + clusterName: "" stages: - stage: ${{ parameters.name }} @@ -26,3 +28,5 @@ stages: - template: e2e-step-template.yaml parameters: name: ${{ parameters.name }} + testDropgz: ${{ parameters.testDropgz }} + clusterName: ${{ parameters.clusterName }} diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml index 1e156e9318..8917289301 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml @@ -1,5 +1,7 @@ parameters: name: "" + testDropgz: "" + clusterName: "" steps: - bash: | @@ -30,7 +32,7 @@ steps: mkdir -p ~/.kube/ echo "Create AKS cluster" make -C ./hack/swift azcfg AZCLI=az - make -C ./hack/swift byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=swifte2e-$(make revision) + make -C ./hack/swift byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) echo "Cluster successfully created" displayName: Create test cluster condition: succeeded() @@ -57,7 +59,7 @@ steps: sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl kubectl cluster-info kubectl get po -owide -A - sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_DROPGZ_VERSION=$(make cni-dropgz-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true + sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_DROPGZ_VERSION=$(make cni-dropgz-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true TEST_DROPGZ=${{ parameters.testDropgz }} retryCountOnTaskFailure: 3 name: "aksswifte2e" displayName: "Run AKS Swift E2E" diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml index e743518e9b..3eced7361f 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml @@ -2,6 +2,8 @@ parameters: name: "" displayName: "" pipelineBuildImage: "$(BUILD_IMAGE)" + testDropgz: "" + clusterName: "" stages: - stage: ${{ parameters.name }} @@ -26,3 +28,5 @@ stages: - template: cilium-e2e-step-template.yaml parameters: name: ${{ parameters.name }} + testDropgz: ${{ parameters.testDropgz }} + clusterName: ${{ parameters.clusterName }} diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml index 6125cedf87..aeb976e5d6 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml @@ -1,5 +1,7 @@ parameters: name: "" + testDropgz: "" + clusterName: "" steps: - bash: | @@ -30,7 +32,7 @@ steps: mkdir -p ~/.kube/ echo "Create AKS cluster" make -C ./hack/swift azcfg AZCLI=az - make -C ./hack/swift byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=ciliume2e-$(make revision) + make -C ./hack/swift byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) echo "Cluster successfully created" displayName: Create test cluster condition: succeeded() @@ -84,7 +86,7 @@ steps: - script: | echo "Start Azilium E2E Tests" kubectl get po -owide -A - sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_DROPGZ_VERSION=$(make cni-dropgz-version) INSTALL_CNS=true INSTALL_AZILIUM=true + sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_DROPGZ_VERSION=$(make cni-dropgz-version) INSTALL_CNS=true INSTALL_AZILIUM=true TEST_DROPGZ=${{ parameters.testDropgz }} retryCountOnTaskFailure: 3 name: "aziliumTest" displayName: "Run Azilium E2E" diff --git a/.pipelines/submodules-pipeline.yaml b/.pipelines/submodules-pipeline.yaml new file mode 100644 index 0000000000..eedc9b551a --- /dev/null +++ b/.pipelines/submodules-pipeline.yaml @@ -0,0 +1,279 @@ +pr: + branches: + include: + - master + paths: + include: + - "zapai/*" + - "azure-ipam/*" + - "dropgz/*" + exclude: + - "*" + +trigger: + paths: + include: + - "zapai/*" + - "azure-ipam/*" + - "dropgz/*" + exclude: + - "*" + tags: + include: + - "zapai/*" + - "azure-ipam/*" + - "dropgz/*" + exclude: + - "*" + +stages: + - stage: setup + displayName: ACN + jobs: + - job: env + displayName: Setup + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" + steps: + - script: | + BUILD_NUMBER=$(Build.BuildNumber) + echo "##vso[task.setvariable variable=StorageID;isOutput=true]$(echo ${BUILD_NUMBER//./-})" + echo "##vso[task.setvariable variable=Tag;isOutput=true]$(make version)" + echo "##vso[task.setvariable variable=cniVersion;isOutput=true]$(make cni-version)" + echo "##vso[task.setvariable variable=npmVersion;isOutput=true]$(make npm-version)" + cat /etc/os-release + uname -a + sudo chown -R $(whoami):$(whoami) . + go version + go env + which go + echo $PATH + name: "EnvironmentalVariables" + displayName: "Set environmental variables" + condition: always() + + - stage: test + displayName: Test ACN + dependsOn: + - setup + jobs: + - job: test + displayName: Run Tests + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" + steps: + - script: | + make tools + sudo -E env "PATH=$PATH" make test-azure-ipam; + retryCountOnTaskFailure: 3 + name: "Test" + displayName: "Run Tests" + + - stage: binaries + displayName: Build Binaries + dependsOn: + - setup + - test + jobs: + - job: build + displayName: Build Binaries + variables: + STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ] + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" + steps: + - script: | + make all-binaries-platforms + name: "BuildAllPlatformBinaries" + displayName: "Build all platform binaries" + + - script: | + mkdir -p ./output/bins + cd ./output + find . -name '*.tgz' -print -exec mv -t ./bins/ {} + + find . -name '*.zip' -print -exec mv -t ./bins/ {} + + shopt -s extglob + rm -rf !("bins") + name: "PrepareArtifacts" + displayName: "Prepare Artifacts" + + - task: CopyFiles@2 + inputs: + sourceFolder: "output" + targetFolder: $(Build.ArtifactStagingDirectory) + condition: succeeded() + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: "output" + pathtoPublish: "$(Build.ArtifactStagingDirectory)" + condition: succeeded() + + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + inlineScript: | + echo Creating storage container with name acn-$(STORAGE_ID) and account name $(STORAGE_ACCOUNT_NAME) + az storage container create -n acn-$(STORAGE_ID) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container + az storage blob upload-batch -d acn-$(STORAGE_ID) -s ./output/bins/ --account-name $(STORAGE_ACCOUNT_NAME) + displayName: Create artifact storage container + condition: succeeded() + + - publish: ./test/apimodels/ + artifact: clusterdefinitions + + - stage: containerize + displayName: Build Images + dependsOn: + - setup + - test + jobs: + - job: containerize_linux_amd64 + displayName: Build Images + pool: + name: "$(BUILD_POOL_NAME_LINUX_AMD64)" + strategy: + matrix: + acncli_linux_amd64: + arch: amd64 + os: linux + name: acncli + cni_dropgz_test_linux_amd64: + arch: amd64 + os: linux + name: cni-dropgz-test + cns_linux_amd64: + arch: amd64 + os: linux + name: cns + npm_linux_amd64: + arch: amd64 + os: linux + name: npm + steps: + - template: containers/container-template.yaml + parameters: + name: $(name) + os: $(os) + arch: $(arch) + - job: containerize_linux_arm64 + displayName: Build Images + variables: + TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ] + pool: + name: "$(BUILD_POOL_NAME_LINUX_ARM64)" + strategy: + matrix: + acncli_linux_arm64: + arch: arm64 + os: linux + name: acncli + cni_dropgz_test_linux_arm64: + arch: arm64 + os: linux + name: cni-dropgz-test + cns_linux_arm64: + arch: arm64 + os: linux + name: cns + npm_linux_arm64: + arch: arm64 + os: linux + name: npm + steps: + - template: containers/container-template.yaml + parameters: + name: $(name) + os: $(os) + arch: $(arch) + - job: containerize_windows_amd64 + timeoutInMinutes: 120 + displayName: Build Images + variables: + TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ] + pool: + name: "$(BUILD_POOL_NAME_WINDOWS_AMD64)" + strategy: + matrix: + cns_windows_amd64: + arch: amd64 + name: cns + npm_windows_amd64: + arch: amd64 + name: npm + steps: + - template: containers/container-template-windows.yaml + parameters: + name: $(name) + arch: $(arch) + tag: $(TAG) + + - stage: publish + displayName: Publish Multiarch Manifests + dependsOn: + - containerize + jobs: + - job: manifest + displayName: Compile Manifests + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" + strategy: + matrix: + acncli: + name: acncli + platforms: linux/amd64 linux/arm64 + cni_dropgz_test: + name: cni-dropgz-test + platforms: linux/amd64 linux/arm64 + cns: + name: cns + platforms: linux/amd64 linux/arm64 windows/amd64 + npm: + name: npm + platforms: linux/amd64 linux/arm64 windows/amd64 + steps: + - template: containers/manifest-template.yaml + parameters: + name: $(name) + platforms: $(platforms) + + - template: singletenancy/cilium/cilium-e2e-job-template.yaml + parameters: + name: "cilium_e2e" + displayName: Cilium + pipelineBuildImage: "$(BUILD_IMAGE)" + testDropgz: true + clusterName: "submodules-ciliume2e" + + - template: singletenancy/aks-swift/e2e-job-template.yaml + parameters: + name: "aks_swift_e2e" + displayName: AKS Swift + pipelineBuildImage: "$(BUILD_IMAGE)" + testDropgz: true + clusterName: "submodules-swifte2e" + + - stage: cleanup + displayName: Cleanup + dependsOn: + - cilium_e2e + - aks_swift_e2e + jobs: + - job: delete_remote_artifacts + displayName: Delete remote artifacts + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + demands: agent.os -equals Linux + steps: + - checkout: none + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + inlineScript: | + BUILD_NUMBER=$(Build.BuildNumber) + BUILD_NUMBER=${BUILD_NUMBER//./-} + echo Deleting storage container with name acn-$BUILD_NUMBER and account name $(STORAGE_ACCOUNT_NAME) + az storage container delete -n acn-$BUILD_NUMBER --account-name $(STORAGE_ACCOUNT_NAME) + displayName: Cleanup remote Azure storage container diff --git a/Makefile b/Makefile index 7be3d2724f..32d5c97566 100644 --- a/Makefile +++ b/Makefile @@ -716,6 +716,9 @@ test-extended-cyclonus: ## run the cyclonus test for npm. cd test/cyclonus && bash ./test-cyclonus.sh extended cd .. +test-azure-ipam: ## run the unit test for azure-ipam + cd $(AZURE_IPAM_DIR) && go test + kind: kind create cluster --config ./test/kind/kind.yaml diff --git a/azure-ipam/ipam_test.go b/azure-ipam/ipam_test.go index 485ee4f928..ce84c8ebe7 100644 --- a/azure-ipam/ipam_test.go +++ b/azure-ipam/ipam_test.go @@ -162,16 +162,6 @@ func TestCmdAdd(t *testing.T) { IP: net.IPv4(10, 0, 1, 10), Mask: net.CIDRMask(24, 32), }, - Gateway: net.IPv4(10, 0, 0, 1), - }, - }, - Routes: []*cniTypes.Route{ - { - Dst: net.IPNet{ - IP: net.IPv4(0, 0, 0, 0), - Mask: net.CIDRMask(0, 32), - }, - GW: net.IPv4(10, 0, 0, 1), }, }, DNS: cniTypes.DNS{}, diff --git a/azure-ipam/main.go b/azure-ipam/main.go index f5e13e5aab..d040da9d91 100644 --- a/azure-ipam/main.go +++ b/azure-ipam/main.go @@ -22,10 +22,10 @@ func main() { func executePlugin() error { // logger config loggerCfg := &logger.Config{ - Level: "debug", - Filepath: "/var/log/azure-ipam.log", - MaxSizeInMB: 5, // MegaBytes - MaxBackups: 8, + Level: "debug", + Filepath: "/var/log/azure-ipam.log", + MaxSizeInMB: 5, // MegaBytes + MaxBackups: 8, } // Create logger pluginLogger, cleanup, err := logger.New(loggerCfg) diff --git a/test/integration/manifests/cns/daemonset.yaml b/test/integration/manifests/cns/daemonset.yaml index 33e2d112ea..d3990ac85c 100644 --- a/test/integration/manifests/cns/daemonset.yaml +++ b/test/integration/manifests/cns/daemonset.yaml @@ -83,7 +83,7 @@ spec: fieldPath: spec.nodeName initContainers: - name: init-cni-dropgz - image: acnpublic.azurecr.io/cni-dropgz-test:latest + image: acnpublic.azurecr.io/cni-dropgz:latest imagePullPolicy: Always command: ["/dropgz"] volumeMounts: diff --git a/test/integration/setup_test.go b/test/integration/setup_test.go index 976ed02f38..ffd3eac586 100644 --- a/test/integration/setup_test.go +++ b/test/integration/setup_test.go @@ -16,6 +16,7 @@ import ( const ( exitFail = 1 + envTestDropgz = "TEST_DROPGZ" envCNIDropgzVersion = "CNI_DROPGZ_VERSION" envCNSVersion = "CNS_VERSION" envInstallCNS = "INSTALL_CNS" @@ -104,11 +105,21 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, l // check environment scenario log.Printf("Checking environment scenario") + if installBoolDropgz := os.Getenv(envTestDropgz); installBoolDropgz != "" { + if testDropgzScenario, err := strconv.ParseBool(installBoolDropgz); err == nil && testDropgzScenario == true { + log.Printf("Env %v set to true, deploy cniTest.Dockerfile", envTestDropgz) + initImage, _ := parseImageString("acnpublic.azurecr.io/cni-dropgz-test:latest") + cns.Spec.Template.Spec.InitContainers[0].Image = getImageString(initImage, cniDropgzVersion) + } + } else { + log.Printf("Env %v not set to true, deploying cni.Dockerfile", envTestDropgz) + initImage, _ := parseImageString(cns.Spec.Template.Spec.InitContainers[0].Image) + cns.Spec.Template.Spec.InitContainers[0].Image = getImageString(initImage, cniDropgzVersion) + } + if installBool1 := os.Getenv(envInstallAzureVnet); installBool1 != "" { if azureVnetScenario, err := strconv.ParseBool(installBool1); err == nil && azureVnetScenario == true { log.Printf("Env %v set to true, deploy azure-vnet", envInstallAzureVnet) - initImage, _ := parseImageString(cns.Spec.Template.Spec.InitContainers[0].Image) - cns.Spec.Template.Spec.InitContainers[0].Image = getImageString(initImage, cniDropgzVersion) cns.Spec.Template.Spec.InitContainers[0].Args = []string{"deploy", "azure-vnet", "-o", "/opt/cni/bin/azure-vnet", "azure-swift.conflist", "-o", "/etc/cni/net.d/10-azure.conflist"} } // setup the CNS swiftconfigmap @@ -122,8 +133,6 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, l if installBool2 := os.Getenv(envInstallAzilium); installBool2 != "" { if aziliumScenario, err := strconv.ParseBool(installBool2); err == nil && aziliumScenario == true { log.Printf("Env %v set to true, deploy azure-ipam and cilium-cni", envInstallAzilium) - initImage, _ := parseImageString(cns.Spec.Template.Spec.InitContainers[0].Image) - cns.Spec.Template.Spec.InitContainers[0].Image = getImageString(initImage, cniDropgzVersion) cns.Spec.Template.Spec.InitContainers[0].Args = []string{"deploy", "azure-ipam", "-o", "/opt/cni/bin/azure-ipam", "azilium.conflist", "-o", "/etc/cni/net.d/05-cilium.conflist"} } // setup the CNS ciliumconfigmap