Squashed 'release-tools/' changes from 7b96bea..f325590

kubernetes-csi/csi-release-tools@f325590 Merge kubernetes-csi/csi-release-tools#149 from pohly/cluster-logs
kubernetes-csi/csi-release-tools@4b03b30 Merge kubernetes-csi/csi-release-tools#155 from pohly/owners
kubernetes-csi/csi-release-tools@a6453c8 owners: introduce aliases
kubernetes-csi/csi-release-tools@ad83def Merge kubernetes-csi/csi-release-tools#153 from pohly/fix-image-builds
kubernetes-csi/csi-release-tools@5561780 build.make: fix image publishng
kubernetes-csi/csi-release-tools@29bd39b Merge kubernetes-csi/csi-release-tools#152 from pohly/bump-csi-test
kubernetes-csi/csi-release-tools@bc42793 prow.sh: use csi-test v4.2.0
kubernetes-csi/csi-release-tools@b546baa Merge kubernetes-csi/csi-release-tools#150 from mauriciopoppe/windows-multiarch-args
kubernetes-csi/csi-release-tools@bfbb6f3 add parameter base_image and addon_image to BUILD_PARAMETERS
kubernetes-csi/csi-release-tools@2d61d3b Merge kubernetes-csi/csi-release-tools#151 from humblec/cm
kubernetes-csi/csi-release-tools@48e71f0 Replace `which` command ( non standard)  with `command -v` builtin
kubernetes-csi/csi-release-tools@feb20e2 prow.sh: collect cluster logs

git-subtree-dir: release-tools
git-subtree-split: f3255906abf2e4134cb3db4ff79ddfb5ce1e91aa

KUBERNETES_CSI_OWNERS_ALIASES

@@ -0,0 +1,44 @@
+# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md
+
+aliases:
+
+  # SIG-Storage chairs and leads should always have approval rights in all repos.
+  # Others may be added as needed here or in each repo.
+  kubernetes-csi-approvers:
+  - jsafrane
+  - msau42
+  - saad-ali
+  - xing-yang
+
+  # Reviewers are automatically assigned to new PRs. The following
+  # reviewers will be active in all repos. Other reviewers can be
+  # added in each repo.
+  #
+  # Reviewers are encouraged to set the "Busy" flag in their GitHub status
+  # when they are temporarily unable to review PRs.
+  kubernetes-csi-reviewers:
+  - andyzhangx
+  - chrishenzie
+  - ggriffiths
+  - gnufied
+  - j-griffith
+  - Jiawei0227
+  - jingxu97
+  - jsafrane
+  - pohly
+  - xing-yang
+
+# This documents who previously contributed to Kubernetes-CSI
+# as approver.
+emeritus_approver:
+- lpabon
+- sbezverk
+- vladimirvivien
+
+# This documents who previously contributed to Kubernetes-CSI
+# as reviewer.
+emeritus_reviewer:
+- lpabon
+- saad-ali
+- sbezverk
+- vladimirvivien

OWNERS

@@ -1,11 +1,8 @@
 # See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md
 
 approvers:
-- saad-ali
-- msau42
+- kubernetes-csi-approvers
 - pohly
 
 reviewers:
-- saad-ali
-- msau42
-- pohly
+- kubernetes-csi-reviewers

OWNERS_ALIASES

@@ -0,0 +1 @@
+KUBERNETES_CSI_OWNERS_ALIASES

build.make

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# force the usage of /bin/bash instead of /bin/sh
+SHELL := /bin/bash
+
 .PHONY: build-% build container-% container push-% push clean test
 
 # A space-separated list of all commands in the repository, must be
@@ -63,26 +66,35 @@ endif
 # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables
 # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below.
 
-# BUILD_PLATFORMS contains a set of <os> <arch> <suffix> triplets,
+# BUILD_PLATFORMS contains a set of tuples [os arch suffix base_image addon_image]
 # separated by semicolon. An empty variable or empty entry (= just a
 # semicolon) builds for the default platform of the current Go
 # toolchain.
 BUILD_PLATFORMS =
 
 # Add go ldflags using LDFLAGS at the time of compilation.
-IMPORTPATH_LDFLAGS = -X main.version=$(REV) 
+IMPORTPATH_LDFLAGS = -X main.version=$(REV)
 EXT_LDFLAGS = -extldflags "-static"
-LDFLAGS = 
+LDFLAGS =
 FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS)
 # This builds each command (= the sub-directories of ./cmd) for the target platform(s)
 # defined by BUILD_PLATFORMS.
 $(CMDS:%=build-%): build-%: check-go-version-go
 	mkdir -p bin
-	echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \
+	# os_arch_seen captures all of the $$os-$$arch seen for the current binary
+	# that we want to build, if we've seen an $$os-$$arch before it means that
+	# we don't need to build it again, this is done to avoid building
+	# the windows binary multiple times (see the default value of $$BUILD_PLATFORMS)
+	export os_arch_seen="" && echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
+		os_arch_seen_pre=$${os_arch_seen%%$$os-$$arch*}; \
+		if ! [ $${#os_arch_seen_pre} = $${#os_arch_seen} ]; then \
+			continue; \
+		fi; \
 		if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \
 			echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \
 			exit 1; \
 		fi; \
+		os_arch_seen+=";$$os-$$arch"; \
 	done
 
 $(CMDS:%=container-%): container-%: build-%
@@ -131,30 +143,46 @@ DOCKER_BUILDX_CREATE_ARGS ?=
 # the tag for the resulting multiarch image.
 $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-%
 	set -ex; \
-	DOCKER_CLI_EXPERIMENTAL=enabled; \
-	export DOCKER_CLI_EXPERIMENTAL; \
+	export DOCKER_CLI_EXPERIMENTAL=enabled; \
 	docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \
 	trap "docker buildx rm multiarchimage-buildertest" EXIT; \
 	dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \
 	dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \
 	if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \
 	if ! [ -f "$$dockerfile_windows" ]; then \
-		build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe//g' -e 's/; *;/;/g')"; \
+		build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe *[^ ]* *[^ ]*//g' -e 's/; *;/;/g' -e 's/;[ ]*$$//')"; \
 	fi; \
 	pushMultiArch () { \
 		tag=$$1; \
-		echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \
+		echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
+			escaped_base_image=$${base_image/:/-}; \
+			if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \
 			docker buildx build --push \
-				--tag $(IMAGE_NAME):$$arch-$$os-$$tag \
+				--tag $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag \
 				--platform=$$os/$$arch \
 				--file $$(eval echo \$${dockerfile_$$os}) \
 				--build-arg binary=./bin/$*$$suffix \
 				--build-arg ARCH=$$arch \
+				--build-arg BASE_IMAGE=$$base_image \
+				--build-arg ADDON_IMAGE=$$addon_image \
 				--label revision=$(REV) \
 				.; \
 		done; \
-		images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \
+		images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
+			escaped_base_image=$${base_image/:/-}; \
+			if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \
+			echo $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \
+		done); \
 		docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \
+		echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
+			if [ $$os = "windows" ]; then \
+				escaped_base_image=$${base_image/:/-}; \
+				if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \
+				image=$(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \
+				os_version=$$(docker manifest inspect mcr.microsoft.com/windows/$${base_image} | grep "os.version" | head -n 1 | awk '{print $$2}' | sed -e 's/"//g') || true; \
+				docker manifest annotate --os-version $$os_version $(IMAGE_NAME):$$tag $$image; \
+			fi; \
+		done; \
 		docker manifest push -p $(IMAGE_NAME):$$tag; \
 	}; \
 	if [ $(PULL_BASE_REF) = "master" ]; then \
@@ -288,4 +316,3 @@ test-spelling:
 test-boilerplate:
 	@ echo; echo "### $@:"
 	@ ./release-tools/verify-boilerplate.sh "$(pwd)"
-

cloudbuild.yaml

@@ -27,7 +27,7 @@ steps:
   # The image must contain bash and curl. Ideally it should also contain
   # the desired version of Go (currently defined in release-tools/prow.sh),
   # but that just speeds up the build and is not required.
-  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8'
+  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20210331-c732583'
     entrypoint: ./.cloudbuild.sh
     env:
     - GIT_TAG=${_GIT_TAG}

prow.sh

@@ -77,7 +77,10 @@ version_to_git () {
     esac
 }
 
-configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries"
+# the list of windows versions was matched from:
+# - https://hub.docker.com/_/microsoft-windows-nanoserver
+# - https://hub.docker.com/_/microsoft-windows-servercore
+configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries"
 
 # If we have a vendor directory, then use it. We must be careful to only
 # use this for "make" invocations inside the project's repo itself because
@@ -236,7 +239,7 @@ configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package"
 # of the cluster. The alternative would have been to (cross-)compile csi-sanity
 # and install it inside the cluster, which is not necessarily easier.
 configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo"
-configvar CSI_PROW_SANITY_VERSION v4.0.2 "csi-test version" # v4.0.2
+configvar CSI_PROW_SANITY_VERSION v4.2.0 "csi-test version"
 configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package"
 configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock"
 configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver"
@@ -632,11 +635,16 @@ EOF
 
 # Deletes kind cluster inside a prow job
 delete_cluster_inside_prow_job() {
+    local name="$1"
+
     # Inside a real Prow job it is better to clean up at runtime
     # instead of leaving that to the Prow job cleanup code
     # because the later sometimes times out (https://github.com/kubernetes-csi/csi-release-tools/issues/24#issuecomment-554765872).
+    #
+    # This is also a good time to collect logs.
     if [ "$JOB_NAME" ]; then
         if kind get clusters | grep -q csi-prow; then
+            run kind export logs --name=csi-prow "${ARTIFACTS}/cluster-logs/$name"
             run kind delete cluster --name=csi-prow || die "kind delete failed"
         fi
         unset KUBECONFIG
@@ -723,7 +731,7 @@ install_csi_driver () {
     fi
 }
 
-# Installs all nessesary snapshotter CRDs  
+# Installs all nessesary snapshotter CRDs
 install_snapshot_crds() {
   # Wait until volumesnapshot CRDs are in place.
   CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd"
@@ -770,7 +778,7 @@ install_snapshot_controller() {
     fi
     echo "$(date +%H:%M:%S)" "waiting for snapshot RBAC setup complete, attempt #$cnt"
 	cnt=$((cnt + 1))
-    sleep 10   
+    sleep 10
   done
 
   SNAPSHOT_CONTROLLER_YAML="${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml"
@@ -839,7 +847,7 @@ install_snapshot_controller() {
     fi
     echo "$(date +%H:%M:%S)" "waiting for snapshot controller deployment to complete, attempt #$cnt"
 	cnt=$((cnt + 1))
-    sleep 10   
+    sleep 10
   done
 }
 
@@ -979,7 +987,7 @@ run_sanity () (
 kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- mkdir "\$@" && echo "\$@"
 EOF
     # Using "rm -rf" as fallback for "rmdir" is a workaround for:
-    # Node Service 
+    # Node Service
     #     should work
     # /nvme/gopath.tmp/src/github.com/kubernetes-csi/csi-test/pkg/sanity/node.go:624
     # STEP: reusing connection to CSI driver at dns:///172.17.0.2:30896
@@ -1143,7 +1151,7 @@ make_test_to_junit () {
 # version_gt 1.3.1 v1.2.0  (returns true)
 # version_gt 1.1.1 release-1.2.0  (returns false)
 # version_gt 1.2.0 1.2.2  (returns false)
-function version_gt() { 
+function version_gt() {
     versions=$(for ver in "$@"; do ver=${ver#release-}; ver=${ver#kubernetes-}; echo "${ver#v}"; done)
     greaterVersion=${1#"release-"};
     greaterVersion=${greaterVersion#"kubernetes-"};
@@ -1206,7 +1214,7 @@ main () {
                 if [ "$rbac_file_path" == "" ]; then
                     rbac_file_path="$(pwd)/deploy/kubernetes/rbac.yaml"
                 fi
-                
+
                 if [ -e "$rbac_file_path" ]; then
                     # This is one of those components which has its own RBAC rules (like external-provisioner).
                     # We are testing a locally built image and also want to test with the the current,
@@ -1271,7 +1279,7 @@ main () {
                     fi
                 fi
             fi
-            delete_cluster_inside_prow_job
+            delete_cluster_inside_prow_job non-alpha
         fi
 
         if tests_need_alpha_cluster && [ "${CSI_PROW_E2E_ALPHA_GATES}" ]; then
@@ -1306,7 +1314,7 @@ main () {
                     fi
                 fi
             fi
-            delete_cluster_inside_prow_job
+            delete_cluster_inside_prow_job alpha
         fi
     fi
 

verify-shellcheck.sh

@@ -84,7 +84,7 @@ done < <(find . -name "*.sh" \
 # detect if the host machine has the required shellcheck version installed
 # if so, we will use that instead.
 HAVE_SHELLCHECK=false
-if which shellcheck &>/dev/null; then
+if command -v shellcheck &>/dev/null; then
   detected_version="$(shellcheck --version | grep 'version: .*')"
   if [[ "${detected_version}" = "version: ${SHELLCHECK_VERSION}" ]]; then
     HAVE_SHELLCHECK=true