diff --git a/.pipelines/cni/singletenancy/windows-cni-load-test-template.yaml b/.pipelines/cni/singletenancy/windows-cni-load-test-template.yaml index d4063758662..c945d4cb4ab 100644 --- a/.pipelines/cni/singletenancy/windows-cni-load-test-template.yaml +++ b/.pipelines/cni/singletenancy/windows-cni-load-test-template.yaml @@ -15,6 +15,8 @@ stages: displayName: "Windows AKS Cluster ${{ parameters.cni }}" jobs: - job: create_aks_cluster_with_${{ parameters.name }} + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" steps: - template: ../load-test-templates/create-cluster-template.yaml parameters: @@ -32,10 +34,10 @@ stages: name: "$(BUILD_POOL_NAME_LINUX_AMD64)" strategy: matrix: - windows_cniv1_amd64: - os: windows - name: cni-plugin + cni_dropgz_windows2022_amd64: arch: amd64 + name: cni-dropgz-test + os: windows os_version: ltsc2022 steps: - template: ../../containers/container-template.yaml @@ -45,15 +47,17 @@ stages: os: $(os) os_version: $(os_version) - stage: update_cni - dependsOn: + dependsOn: - createAKSclusterWindows - build_images displayName: "Update CNI on Cluster" jobs: - job: deploy_pods + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" strategy: matrix: - windows_cniv1_amd64: + cni_dropgz_windows2022_amd64: os: windows arch: amd64 os_version: ltsc2022 @@ -66,12 +70,12 @@ stages: addSpnToEnvironment: true inlineScript: | set -ex - export CNI_IMAGE=$(make cni-plugin-image-name-and-tag OS=$(os) ARCH=$(arch) OS_VERSION=$(os_version)) - az extension add --name aks-preview clusterName=${{ parameters.clusterName }}-$(make revision) make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName} make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION) - envsubst < ./hack/manifests/windows-update.yaml | kubectl apply -f - + dropgz_version=$(make cni-dropgz-version) + export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS=$(os) ARCH=$(arch) OS_VERSION=$(os_version) CNI_DROPGZ_VERSION=${dropgz_version}) + envsubst < ./test/integration/manifests/cni/cni-installer-v1-windows.yaml | kubectl apply -f - name: "UploadCNI" displayName: "Upload CNI" - script: | @@ -85,6 +89,8 @@ stages: displayName: "Pod Deployment" jobs: - job: deploy_pods + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" steps: - template: ../load-test-templates/pod-deployment-template.yaml parameters: @@ -98,6 +104,8 @@ stages: displayName: "Validate State" jobs: - job: validate_state + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" steps: - template: ../load-test-templates/validate-state-template.yaml parameters: @@ -110,6 +118,8 @@ stages: - validate_state_windows jobs: - job: delete_resources + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" steps: - task: AzureCLI@1 inputs: diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index 01c90507cea..0fbd29a6eb7 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -165,16 +165,11 @@ stages: arch: amd64 name: cni-dropgz-test os: linux - cni_plugin_windows2022_amd64: - os: windows - name: cni-plugin + cni_dropgz_test_windows2022_amd64: arch: amd64 - os_version: ltsc2022 - cni_plugin_windows2019_amd64: + name: cni-dropgz-test os: windows - name: cni-plugin - arch: amd64 - os_version: ltsc2019 + os_version: ltsc2022 cns_linux_amd64: arch: amd64 name: cns @@ -292,7 +287,7 @@ stages: platforms: linux/amd64 linux/arm64 cni_dropgz_test: name: cni-dropgz-test - platforms: linux/amd64 linux/arm64 + platforms: linux/amd64 linux/arm64 windows/amd64 cns: name: cns os_versions: ltsc2019 ltsc2022 diff --git a/.pipelines/singletenancy/aks/e2e-step-template.yaml b/.pipelines/singletenancy/aks/e2e-step-template.yaml index 9dfb729b474..3bdab1d0f96 100644 --- a/.pipelines/singletenancy/aks/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks/e2e-step-template.yaml @@ -44,8 +44,9 @@ steps: export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS='linux' ARCH=${{ parameters.arch }} CNI_DROPGZ_VERSION=${{ parameters.version }}) envsubst < ./test/integration/manifests/cni/cni-installer-v1.yaml | kubectl apply -f - kubectl rollout status daemonset/azure-cni -n kube-system - export CNI_IMAGE=$(make cni-plugin-image-name-and-tag OS=${{ parameters.os }} ARCH=${{ parameters.arch }} OS_VERSION=${{ parameters.os_version }} CNI_VERSION=${{ parameters.cniVersion }}) - envsubst < ./hack/manifests/windows-update.yaml | kubectl apply -f - + echo "Deploying on windows nodes" + export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS='windows' ARCH=${{ parameters.arch }} OS_VERSION=${{ parameters.os_version }} CNI_DROPGZ_VERSION=${{ parameters.version }}) + envsubst < ./test/integration/manifests/cni/cni-installer-v1-windows.yaml | kubectl apply -f - kubectl rollout status daemonset/azure-cni-windows -n kube-system else export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS=${{ parameters.os }} ARCH=${{ parameters.arch }} CNI_DROPGZ_VERSION=${{ parameters.version }}) diff --git a/Makefile b/Makefile index 112f06009ec..d2e9cb8b3c2 100644 --- a/Makefile +++ b/Makefile @@ -341,7 +341,7 @@ cni-dropgz-image-name-and-tag: # util target to print the CNI dropgz image name cni-dropgz-image: ## build cni-dropgz container image. $(MAKE) container \ DOCKERFILE=dropgz/build/$(OS).Dockerfile \ - EXTRA_BUILD_ARGS='--build-arg OS=$(OS) --build-arg ARCH=$(ARCH)' \ + EXTRA_BUILD_ARGS='--build-arg OS=$(OS) --build-arg ARCH=$(ARCH) --build-arg OS_VERSION=$(OS_VERSION)' \ IMAGE=$(CNI_DROPGZ_IMAGE) \ TAG=$(CNI_DROPGZ_PLATFORM_TAG) @@ -365,8 +365,8 @@ cni-dropgz-test-image-name-and-tag: # util target to print the CNI dropgz test i cni-dropgz-test-image: ## build cni-dropgz-test container image. $(MAKE) container \ - DOCKERFILE=dropgz/build/cniTest.Dockerfile \ - EXTRA_BUILD_ARGS='--build-arg OS=$(OS)' \ + DOCKERFILE=dropgz/build/cniTest_$(OS).Dockerfile \ + EXTRA_BUILD_ARGS='--build-arg OS=$(OS) --build-arg ARCH=$(ARCH) --build-arg OS_VERSION=$(OS_VERSION)' \ IMAGE=$(CNI_DROPGZ_TEST_IMAGE) \ TAG=$(CNI_DROPGZ_TEST_PLATFORM_TAG) diff --git a/dropgz/build/cniTest.Dockerfile b/dropgz/build/cniTest_linux.Dockerfile similarity index 100% rename from dropgz/build/cniTest.Dockerfile rename to dropgz/build/cniTest_linux.Dockerfile diff --git a/dropgz/build/cniTest_windows.Dockerfile b/dropgz/build/cniTest_windows.Dockerfile new file mode 100644 index 00000000000..874d46b95d2 --- /dev/null +++ b/dropgz/build/cniTest_windows.Dockerfile @@ -0,0 +1,30 @@ +ARG ARCH +ARG OS_VERSION +FROM --platform=linux/${ARCH} mcr.microsoft.com/oss/go/microsoft/golang:1.20 AS azure-vnet +ARG VERSION +WORKDIR /azure-container-networking +COPY . . +RUN GOOS=windows CGO_ENABLED=0 go build -a -o azure-vnet.exe -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/network/plugin/main.go +RUN GOOS=windows CGO_ENABLED=0 go build -a -o azure-vnet-telemetry.exe -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/telemetry/service/telemetrymain.go +RUN GOOS=windows CGO_ENABLED=0 go build -a -o azure-vnet-ipam.exe -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/ipam/plugin/main.go + +FROM --platform=linux/${ARCH} mcr.microsoft.com/cbl-mariner/base/core:2.0 AS compressor +ARG OS +WORKDIR /dropgz +COPY dropgz . +COPY --from=azure-vnet /azure-container-networking/azure-vnet.exe pkg/embed/fs +COPY --from=azure-vnet /azure-container-networking/azure-vnet-telemetry.exe pkg/embed/fs +COPY --from=azure-vnet /azure-container-networking/azure-vnet-ipam.exe pkg/embed/fs +COPY --from=azure-vnet /azure-container-networking/telemetry/azure-vnet-telemetry.config pkg/embed/fs +RUN cd pkg/embed/fs/ && sha256sum * > sum.txt +RUN gzip --verbose --best --recursive pkg/embed/fs && for f in pkg/embed/fs/*.gz; do mv -- "$f" "${f%%.gz}"; done + +FROM --platform=linux/${ARCH} mcr.microsoft.com/oss/go/microsoft/golang:1.20 AS dropgz +ARG VERSION +WORKDIR /dropgz +COPY --from=compressor /dropgz . +RUN GOOS=windows CGO_ENABLED=0 go build -a -o bin/dropgz.exe -trimpath -ldflags "-X github.com/Azure/azure-container-networking/dropgz/internal/buildinfo.Version="$VERSION"" -gcflags="-dwarflocationlists=true" main.go + +FROM mcr.microsoft.com/windows/nanoserver:${OS_VERSION} +COPY --from=dropgz /dropgz/bin/dropgz.exe dropgz.exe +ENTRYPOINT [ "dropgz.exe" ] diff --git a/dropgz/build/windows.Dockerfile b/dropgz/build/windows.Dockerfile new file mode 100644 index 00000000000..d9cfdab6cb1 --- /dev/null +++ b/dropgz/build/windows.Dockerfile @@ -0,0 +1,36 @@ +ARG ARCH +ARG OS_VERSION +FROM --platform=linux/${ARCH} mcr.microsoft.com/cbl-mariner/base/core:2.0 AS tar +RUN tdnf install -y tar +RUN tdnf install -y unzip +RUN tdnf upgrade -y && tdnf install -y ca-certificates + +FROM tar AS azure-vnet +ARG AZCNI_VERSION=v1.5.4 +ARG VERSION +ARG OS +ARG ARCH +WORKDIR /azure-container-networking +COPY . . +RUN curl -LO --cacert /etc/ssl/certs/ca-certificates.crt https://github.com/Azure/azure-container-networking/releases/download/$AZCNI_VERSION/azure-vnet-cni-$OS-$ARCH-$AZCNI_VERSION.zip && unzip -o azure-vnet-cni-$OS-$ARCH-$AZCNI_VERSION.zip + +FROM --platform=linux/${ARCH} mcr.microsoft.com/cbl-mariner/base/core:2.0 AS compressor +ARG OS +WORKDIR /dropgz +COPY dropgz . +COPY --from=azure-vnet /azure-container-networking/azure-vnet.exe pkg/embed/fs +COPY --from=azure-vnet /azure-container-networking/azure-vnet-telemetry.exe pkg/embed/fs +COPY --from=azure-vnet /azure-container-networking/azure-vnet-ipam.exe pkg/embed/fs +COPY --from=azure-vnet /azure-container-networking/azure-vnet-telemetry.config pkg/embed/fs +RUN cd pkg/embed/fs/ && sha256sum * > sum.txt +RUN gzip --verbose --best --recursive pkg/embed/fs && for f in pkg/embed/fs/*.gz; do mv -- "$f" "${f%%.gz}"; done + +FROM --platform=linux/${ARCH} mcr.microsoft.com/oss/go/microsoft/golang:1.20 AS dropgz +ARG VERSION +WORKDIR /dropgz +COPY --from=compressor /dropgz . +RUN GOOS=windows CGO_ENABLED=0 go build -a -o bin/dropgz.exe -trimpath -ldflags "-X github.com/Azure/azure-container-networking/dropgz/internal/buildinfo.Version="$VERSION"" -gcflags="-dwarflocationlists=true" main.go + +FROM mcr.microsoft.com/windows/nanoserver:${OS_VERSION} +COPY --from=dropgz /dropgz/bin/dropgz.exe dropgz.exe +ENTRYPOINT [ "dropgz.exe" ] diff --git a/dropgz/pkg/embed/payload.go b/dropgz/pkg/embed/payload.go index b36f8cb4792..fd69d56eb6d 100644 --- a/dropgz/pkg/embed/payload.go +++ b/dropgz/pkg/embed/payload.go @@ -7,8 +7,8 @@ import ( "io" "io/fs" "os" + "path" "path/filepath" - "strings" "github.com/pkg/errors" "go.uber.org/zap" @@ -16,13 +16,13 @@ import ( const ( cwd = "fs" - pathPrefix = cwd + string(filepath.Separator) oldFileSuffix = ".old" ) var ErrArgsMismatched = errors.New("mismatched argument count") // embedfs contains the embedded files for deployment, as a read-only FileSystem containing only "embedfs/". +// //nolint:typecheck // dir is populated at build. //go:embed fs var embedfs embed.FS @@ -36,7 +36,8 @@ func Contents() ([]string, error) { if d.IsDir() { return nil } - contents = append(contents, strings.TrimPrefix(path, pathPrefix)) + _, filename := filepath.Split(path) + contents = append(contents, filename) return nil }) if err != nil { @@ -69,10 +70,10 @@ func (c *compoundReadCloser) Close() error { return nil } -func Extract(path string) (*compoundReadCloser, error) { - f, err := embedfs.Open(filepath.Join(cwd, path)) +func Extract(p string) (*compoundReadCloser, error) { + f, err := embedfs.Open(path.Join(cwd, p)) if err != nil { - return nil, errors.Wrapf(err, "failed to open file %s", path) + return nil, errors.Wrapf(err, "failed to open file %s", p) } r, err := gzip.NewReader(bufio.NewReader(f)) if err != nil { diff --git a/hack/manifests/windows-update.yaml b/test/integration/manifests/cni/cni-installer-v1-windows.yaml similarity index 51% rename from hack/manifests/windows-update.yaml rename to test/integration/manifests/cni/cni-installer-v1-windows.yaml index 2f21f31516c..ca303efff19 100644 --- a/hack/manifests/windows-update.yaml +++ b/test/integration/manifests/cni/cni-installer-v1-windows.yaml @@ -36,19 +36,42 @@ spec: hostNetwork: true serviceAccountName: azure-cni tolerations: - - operator: Exists - effect: NoSchedule - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - containers: + - key: CriticalAddonsOnly + operator: Exists + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + initContainers: + - name: delete-azure-vnet-telemetry + image: mcr.microsoft.com/powershell:lts-nanoserver-ltsc2022 + command: ["powershell.exe", "-command"] + args: ["if (Get-Process -Name 'azure-vnet-telemetry' -ErrorAction SilentlyContinue) { Stop-Process -Name 'azure-vnet-telemetry' -Force }"] - name: cni-drop - image: ${CNI_IMAGE} + image: ${DROP_GZ_URL} imagePullPolicy: Always + command: ["%CONTAINER_SANDBOX_MOUNT_POINT%/dropgz.exe"] + args: + - deploy + - azure-vnet.exe + - -o + - /k/azurecni/bin/azure-vnet.exe + - azure-vnet-ipam.exe + - -o + - /k/azurecni/bin/azure-vnet-ipam.exe + - azure-vnet-telemetry.exe + - -o + - /k/azurecni/bin/azure-vnet-telemetry.exe + - azure-vnet-telemetry.config + - -o + - /k/azurecni/bin/azure-vnet-telemetry.config volumeMounts: - name: cni-bin mountPath: /k/azurecni/bin/ + containers: + - name: pause + image: mcr.microsoft.com/oss/kubernetes/pause:3.6 + command: ["%CONTAINER_SANDBOX_MOUNT_POINT%/pause.exe"] volumes: - name: cni-bin hostPath: