Merge pull request #334 from chewong/build-windows-container

test: run CSI driver e2e tests on Windows clusters

Makefile

@@ -15,11 +15,12 @@
 PKG = sigs.k8s.io/azuredisk-csi-driver
 GIT_COMMIT ?= $(shell git rev-parse HEAD)
 REGISTRY ?= andyzhangx
+REGISTRY_NAME = $(shell echo $(REGISTRY) | sed "s/.azurecr.io//g")
 DRIVER_NAME = disk.csi.azure.com
 IMAGE_NAME = azuredisk-csi
 IMAGE_VERSION ?= v0.7.0
-# Use a custom version for E2E tests if we are in Prow
-ifdef AZURE_CREDENTIALS
+# Use a custom version for E2E tests if we are testing in CI
+ifdef CI
 override IMAGE_VERSION := e2e-$(GIT_COMMIT)
 endif
 IMAGE_TAG = $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION)
@@ -38,7 +39,8 @@ endif
 GOPATH ?= $(shell go env GOPATH)
 GOBIN ?= $(GOPATH)/bin
 GO111MODULE = off
-export GOPATH GOBIN GO111MODULE
+DOCKER_CLI_EXPERIMENTAL = enabled
+export GOPATH GOBIN GO111MODULE DOCKER_CLI_EXPERIMENTAL
 
 .PHONY: all
 all: azuredisk
@@ -66,13 +68,18 @@ e2e-test:
 
 .PHONY: e2e-bootstrap
 e2e-bootstrap: install-helm
-	# Only build and push the image if it does not exist in the registry
 	docker pull $(IMAGE_TAG) || make azuredisk-container push
+ifdef TEST_WINDOWS
+	helm install azuredisk-csi-driver charts/latest/azuredisk-csi-driver --namespace kube-system --wait --timeout=15m -v=5 --debug \
+		--set image.azuredisk.repository=$(REGISTRY)/$(IMAGE_NAME) \
+		--set image.azuredisk.tag=$(IMAGE_VERSION) \
+		--set windows.enabled=true
+else
 	helm install azuredisk-csi-driver charts/latest/azuredisk-csi-driver --namespace kube-system --wait --timeout=15m -v=5 --debug \
-		--set image.azuredisk.pullPolicy=IfNotPresent \
 		--set image.azuredisk.repository=$(REGISTRY)/$(IMAGE_NAME) \
 		--set image.azuredisk.tag=$(IMAGE_VERSION) \
 		--set snapshot.enabled=true
+endif
 
 .PHONY: install-helm
 install-helm:
@@ -92,21 +99,32 @@ azuredisk-windows:
 	if [ ! -d ./vendor ]; then dep ensure -vendor-only; fi
 	CGO_ENABLED=0 GOOS=windows go build -a -ldflags ${LDFLAGS} -o _output/azurediskplugin.exe ./pkg/azurediskplugin
 
-.PHONY: container
-container: azuredisk
-	docker build --no-cache -t $(IMAGE_TAG) -f ./pkg/azurediskplugin/Dockerfile .
-
 .PHONY: azuredisk-container
-azuredisk-container: azuredisk
+azuredisk-container:
+ifdef CI
+	az acr login --name $(REGISTRY_NAME)
+	make azuredisk azuredisk-windows
+	az acr build --registry $(REGISTRY_NAME) -t $(IMAGE_TAG)-linux-amd64 -f ./pkg/azurediskplugin/Dockerfile --platform linux .
+	az acr build --registry $(REGISTRY_NAME) -t $(IMAGE_TAG)-windows-1809-amd64 -f ./pkg/azurediskplugin/Windows.Dockerfile --platform windows .
+	docker manifest create $(IMAGE_TAG) $(IMAGE_TAG)-linux-amd64 $(IMAGE_TAG)-windows-1809-amd64
+	docker manifest inspect $(IMAGE_TAG)
+else
+ifdef TEST_WINDOWS
+	make azuredisk-windows
+	docker build --no-cache -t $(IMAGE_TAG) -f ./pkg/azurediskplugin/Windows.Dockerfile .
+else
+	make azurediskfi
 	docker build --no-cache -t $(IMAGE_TAG) -f ./pkg/azurediskplugin/Dockerfile .
-
-.PHONY: azuredisk-container-windows
-azuredisk-container-windows: azuredisk-windows
-	docker build --no-cache --platform windows/amd64 -t $(IMAGE_TAG) -f ./pkg/azurediskplugin/Windows.Dockerfile .
+endif
+endif
 
 .PHONY: push
 push:
+ifdef CI
+	docker manifest push --purge $(IMAGE_TAG)
+else
 	docker push $(IMAGE_TAG)
+endif
 
 .PHONY: push-latest
 push-latest:
@@ -121,4 +139,4 @@ build-push: azuredisk-container
 .PHONY: clean
 clean:
 	go clean -r -x
-	-rm -rf _output
+	-rm -rf _output

charts/latest/azuredisk-csi-driver/templates/csi-azuredisk-node-windows.yaml

@@ -26,10 +26,10 @@ spec:
               name: plugin-dir
           image: "{{ .Values.windows.image.livenessProbe.repository }}:{{ .Values.windows.image.livenessProbe.tag }}"
           args:
-            - --csi-address=$(CSI_ENDPOINT)
-            - --probe-timeout=3s
-            - --health-port=29603
-            - --v=5
+            - "--csi-address=$(CSI_ENDPOINT)"
+            - "--probe-timeout=3s"
+            - "--health-port=29603"
+            - "--v=5"
           env:
             - name: CSI_ENDPOINT
               value: unix://C:\\csi\\csi.sock
@@ -44,9 +44,9 @@ spec:
         - name: node-driver-registrar
           image: "{{ .Values.windows.image.nodeDriverRegistrar.repository }}:{{ .Values.windows.image.nodeDriverRegistrar.tag }}"
           args:
-            - --v=5
-            - --csi-address=$(CSI_ENDPOINT)
-            - --kubelet-registration-path=C:\\var\\lib\\kubelet\\plugins\\disk.csi.azure.com\\csi.sock
+            - "--v=5"
+            - "--csi-address=$(CSI_ENDPOINT)"
+            - "--kubelet-registration-path=C:\\var\\lib\\kubelet\\plugins\\disk.csi.azure.com\\csi.sock"
           env:
             - name: CSI_ENDPOINT
               value: unix://C:\\csi\\csi.sock
@@ -72,9 +72,9 @@ spec:
         - name: azuredisk
           image: "{{ .Values.image.azuredisk.repository }}:{{ .Values.image.azuredisk.tag }}"
           args:
-            - --v=5
-            - --endpoint=$(CSI_ENDPOINT)
-            - --nodeid=$(KUBE_NODE_NAME)
+            - "--v=5"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--nodeid=$(KUBE_NODE_NAME)"
             - "--metrics-address=0.0.0.0:{{ .Values.node.metricsPort }}"
           ports:
             - containerPort: 29603

deploy/csi-azuredisk-node-windows.yaml

@@ -23,10 +23,10 @@ spec:
               name: plugin-dir
           image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:master
           args:
-            - --csi-address=$(CSI_ENDPOINT)
-            - --probe-timeout=3s
-            - --health-port=29603
-            - --v=5
+            - "--csi-address=$(CSI_ENDPOINT)"
+            - "--probe-timeout=3s"
+            - "--health-port=29603"
+            - "--v=5"
           env:
             - name: CSI_ENDPOINT
               value: unix://C:\\csi\\csi.sock
@@ -40,9 +40,9 @@ spec:
         - name: node-driver-registrar
           image: mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:master
           args:
-            - --v=5
-            - --csi-address=$(CSI_ENDPOINT)
-            - --kubelet-registration-path=C:\\var\\lib\\kubelet\\plugins\\disk.csi.azure.com\\csi.sock
+            - "--v=5"
+            - "--csi-address=$(CSI_ENDPOINT)"
+            - "--kubelet-registration-path=C:\\var\\lib\\kubelet\\plugins\\disk.csi.azure.com\\csi.sock"
           env:
             - name: CSI_ENDPOINT
               value: unix://C:\\csi\\csi.sock
@@ -67,9 +67,9 @@ spec:
         - name: azuredisk
           image: mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:latest
           args:
-            - --v=5
-            - --endpoint=$(CSI_ENDPOINT)
-            - --nodeid=$(KUBE_NODE_NAME)
+            - "--v=5"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--nodeid=$(KUBE_NODE_NAME)"
           ports:
             - containerPort: 29603
               name: healthz

test/e2e/pre_provisioning_test.go

@@ -82,7 +82,7 @@ var _ = ginkgo.Describe("Pre-Provisioned", func() {
 			diskSize := fmt.Sprintf("%dGi", defaultDiskSize)
 			pods := []testsuites.PodDetails{
 				{
-					Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data",
+					Cmd: convertToPowershellCommandIfNecessary("echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data"),
 					Volumes: []testsuites.VolumeDetails{
 						{
 							VolumeID:  volumeID,
@@ -95,6 +95,7 @@ var _ = ginkgo.Describe("Pre-Provisioned", func() {
 							},
 						},
 					},
+					IsWindows: isWindowsCluster,
 				},
 			}
 			test := testsuites.PreProvisionedReadOnlyVolumeTest{

test/e2e/testsuites/dynamically_provisioned_read_only_volume_tester.go

@@ -28,8 +28,6 @@ import (
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
-var expectedReadOnlyLog = "Read-only file system"
-
 // DynamicallyProvisionedReadOnlyVolumeTest will provision required StorageClass(es), PVC(s) and Pod(s)
 // Waiting for the PV provisioner to create a new PV
 // Testing that the Pod(s) cannot write to the volume when mounted
@@ -40,6 +38,7 @@ type DynamicallyProvisionedReadOnlyVolumeTest struct {
 
 func (t *DynamicallyProvisionedReadOnlyVolumeTest) Run(client clientset.Interface, namespace *v1.Namespace) {
 	for _, pod := range t.Pods {
+		expectedReadOnlyLog := "Read-only file system"
 		if pod.IsWindows {
 			expectedReadOnlyLog = "FileOpenFailure"
 		}

test/e2e/testsuites/pre_provisioned_read_only_volume_tester.go

@@ -37,6 +37,11 @@ type PreProvisionedReadOnlyVolumeTest struct {
 
 func (t *PreProvisionedReadOnlyVolumeTest) Run(client clientset.Interface, namespace *v1.Namespace) {
 	for _, pod := range t.Pods {
+		expectedReadOnlyLog := "Read-only file system"
+		if pod.IsWindows {
+			expectedReadOnlyLog = "FileOpenFailure"
+		}
+
 		tpod, cleanup := pod.SetupWithPreProvisionedVolumes(client, namespace, t.CSIDriver)
 		// defer must be called here for resources not get removed before using them
 		for i := range cleanup {