From 40e572a08e6999d1eddfd372b16949f1cf46d979 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 16 Feb 2022 17:32:19 +0100 Subject: [PATCH] Squashed 'release-tools/' changes from a6a1a797..335339f0 https://github.com/kubernetes-csi/csi-release-tools/commit/335339f0 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/187 from mauriciopoppe/remove-eol-windows-versions https://github.com/kubernetes-csi/csi-release-tools/commit/890b87a2 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/188 from pwschuurman/update-release-notes-docs https://github.com/kubernetes-csi/csi-release-tools/commit/274bc9ba Update Sidecar Release Process documentation to reference latest syntax for release-notes tool https://github.com/kubernetes-csi/csi-release-tools/commit/87b6c372 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/185 from Garima-Negi/fix-OWNERS-files https://github.com/kubernetes-csi/csi-release-tools/commit/f1de2c66 Fix OWNERS file - squashed commits https://github.com/kubernetes-csi/csi-release-tools/commit/59ae38b7 Remove EOL windows versions from BUILD_PLATFORMS https://github.com/kubernetes-csi/csi-release-tools/commit/5d664712 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/186 from humblec/sp https://github.com/kubernetes-csi/csi-release-tools/commit/d066f1ba Correct prow.sh typo and make codespell linter pass https://github.com/kubernetes-csi/csi-release-tools/commit/762e22d0 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/184 from pohly/image-publishing-troubleshooting https://github.com/kubernetes-csi/csi-release-tools/commit/81e26c3f SIDECAR_RELEASE_PROCESS.md: add troubleshooting for image publishing https://github.com/kubernetes-csi/csi-release-tools/commit/31aa44d1 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/182 from chrishenzie/csi-sanity-version https://github.com/kubernetes-csi/csi-release-tools/commit/f49e141c Update csi-sanity test suite to v4.3.0 https://github.com/kubernetes-csi/csi-release-tools/commit/d9815c28 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/181 from mauriciopoppe/armv7-support https://github.com/kubernetes-csi/csi-release-tools/commit/05c18012 Add support to build arm/v7 images https://github.com/kubernetes-csi/csi-release-tools/commit/4aedf357 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/178 from xing-yang/timeout https://github.com/kubernetes-csi/csi-release-tools/commit/2b9897eb Increase build timeout https://github.com/kubernetes-csi/csi-release-tools/commit/51d37029 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/177 from mauriciopoppe/kind-image-1.23 https://github.com/kubernetes-csi/csi-release-tools/commit/a30efeac Add kind image for 1.23 git-subtree-dir: release-tools git-subtree-split: 335339f059da0b8b1947794a8c75d9e5b973cb79 --- KUBERNETES_CSI_OWNERS_ALIASES | 10 +------ SIDECAR_RELEASE_PROCESS.md | 53 +++++++++++++++++++++++++++++------ build.make | 28 +++++++++--------- cloudbuild.yaml | 5 ++-- prow.sh | 9 +++--- 5 files changed, 67 insertions(+), 38 deletions(-) diff --git a/KUBERNETES_CSI_OWNERS_ALIASES b/KUBERNETES_CSI_OWNERS_ALIASES index 8ea922edb..2f3e2dfc9 100644 --- a/KUBERNETES_CSI_OWNERS_ALIASES +++ b/KUBERNETES_CSI_OWNERS_ALIASES @@ -31,15 +31,7 @@ aliases: # This documents who previously contributed to Kubernetes-CSI # as approver. -emeritus_approver: +emeritus_approvers: - lpabon - sbezverk - vladimirvivien - -# This documents who previously contributed to Kubernetes-CSI -# as reviewer. -emeritus_reviewer: -- lpabon -- saad-ali -- sbezverk -- vladimirvivien diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index e4b30e898..8977fbe6d 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -46,10 +46,13 @@ naming convention `-on-`. ## Release Process 1. Identify all issues and ongoing PRs that should go into the release, and drive them to resolution. -1. Download v2.8+ [K8s release notes - generator](https://github.com/kubernetes/release/tree/HEAD/cmd/release-notes) +1. Download the latest version of the + [K8s release notes generator](https://github.com/kubernetes/release/tree/HEAD/cmd/release-notes) +1. Create a + [Github personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) + with `repo:public_repo` access 1. Generate release notes for the release. Replace arguments with the relevant - information. + information. * Clean up old cached information (also needed if you are generating release notes for multiple repos) ```bash @@ -57,15 +60,24 @@ naming convention `-on-`. ``` * For new minor releases on master: ```bash - GITHUB_TOKEN= release-notes --discover=mergebase-to-latest - --github-org=kubernetes-csi --github-repo=external-provisioner - --required-author="" --output out.md + GITHUB_TOKEN= release-notes \ + --discover=mergebase-to-latest \ + --org=kubernetes-csi \ + --repo=external-provisioner \ + --required-author="" \ + --markdown-links \ + --output out.md ``` * For new patch releases on a release branch: ```bash - GITHUB_TOKEN= release-notes --discover=patch-to-latest --branch=release-1.1 - --github-org=kubernetes-csi --github-repo=external-provisioner - --required-author="" --output out.md + GITHUB_TOKEN= release-notes \ + --discover=patch-to-latest \ + --branch=release-1.1 \ + --org=kubernetes-csi \ + --repo=external-provisioner \ + --required-author="" \ + --markdown-links \ + --output out.md ``` 1. Compare the generated output to the new commits for the release to check if any notable change missed a release note. @@ -100,6 +112,29 @@ naming convention `-on-`. and [k/k in-tree](https://github.com/kubernetes/kubernetes/tree/HEAD/test/e2e/testing-manifests/storage-csi/hostpath/hostpath) +### Troubleshooting + +#### Image build jobs + +The following jobs are triggered after tagging to produce the corresponding +image(s): +https://k8s-testgrid.appspot.com/sig-storage-image-build + +Clicking on a failed build job opens that job in https://prow.k8s.io. Next to +the job title is a rerun icon (circle with arrow). Clicking it opens a popup +with a "rerun" button that maintainers with enough permissions can use. If in +doubt, ask someone on #sig-release to rerun the job. + +Another way to rerun a job is to search for it in https://prow.k8s.io and click +the rerun icon in the resulting job list: +https://prow.k8s.io/?job=canary-csi-test-push-images + +#### Verify images + +Canary and staged images can be viewed at https://console.cloud.google.com/gcr/images/k8s-staging-sig-storage + +Promoted images can be viewed at https://console.cloud.google.com/gcr/images/k8s-artifacts-prod/us/sig-storage + ## Adding support for a new Kubernetes release 1. Add the new release to `k8s_versions` in diff --git a/build.make b/build.make index 6caf64fd5..ff428890d 100644 --- a/build.make +++ b/build.make @@ -70,7 +70,7 @@ endif # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below. -# BUILD_PLATFORMS contains a set of tuples [os arch suffix base_image addon_image] +# BUILD_PLATFORMS contains a set of tuples [os arch buildx_platform suffix base_image addon_image] # separated by semicolon. An empty variable or empty entry (= just a # semicolon) builds for the default platform of the current Go # toolchain. @@ -85,12 +85,12 @@ FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS) # defined by BUILD_PLATFORMS. $(CMDS:%=build-%): build-%: check-go-version-go mkdir -p bin - # os_arch_seen captures all of the $$os-$$arch seen for the current binary - # that we want to build, if we've seen an $$os-$$arch before it means that + # os_arch_seen captures all of the $$os-$$arch-$$buildx_platform seen for the current binary + # that we want to build, if we've seen an $$os-$$arch-$$buildx_platform before it means that # we don't need to build it again, this is done to avoid building # the windows binary multiple times (see the default value of $$BUILD_PLATFORMS) - export os_arch_seen="" && echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ - os_arch_seen_pre=$${os_arch_seen%%$$os-$$arch*}; \ + export os_arch_seen="" && echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch buildx_platform suffix base_image addon_image; do \ + os_arch_seen_pre=$${os_arch_seen%%$$os-$$arch-$$buildx_platform*}; \ if ! [ $${#os_arch_seen_pre} = $${#os_arch_seen} ]; then \ continue; \ fi; \ @@ -98,7 +98,7 @@ $(CMDS:%=build-%): build-%: check-go-version-go echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \ exit 1; \ fi; \ - os_arch_seen+=";$$os-$$arch"; \ + os_arch_seen+=";$$os-$$arch-$$buildx_platform"; \ done $(CMDS:%=container-%): container-%: build-% @@ -154,16 +154,17 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% dockerfile_windows=$$(if [ -e ./$(CMDS_DIR)/$*/Dockerfile.Windows ]; then echo ./$(CMDS_DIR)/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \ if ! [ -f "$$dockerfile_windows" ]; then \ - build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe *[^ ]* *[^ ]*//g' -e 's/; *;/;/g' -e 's/;[ ]*$$//')"; \ + build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *[^ ]* *.exe *[^ ]* *[^ ]*//g' -e 's/; *;/;/g' -e 's/;[ ]*$$//')"; \ fi; \ pushMultiArch () { \ tag=$$1; \ - echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch buildx_platform suffix base_image addon_image; do \ escaped_base_image=$${base_image/:/-}; \ + escaped_buildx_platform=$${buildx_platform//\//-}; \ if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ docker buildx build --push \ - --tag $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag \ - --platform=$$os/$$arch \ + --tag $(IMAGE_NAME):$$escaped_buildx_platform-$$os-$$escaped_base_image$$tag \ + --platform=$$os/$$buildx_platform \ --file $$(eval echo \$${dockerfile_$$os}) \ --build-arg binary=./bin/$*$$suffix \ --build-arg ARCH=$$arch \ @@ -172,13 +173,14 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% --label revision=$(REV) \ .; \ done; \ - images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch buildx_platform suffix base_image addon_image; do \ escaped_base_image=$${base_image/:/-}; \ + escaped_buildx_platform=$${buildx_platform//\//-}; \ if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ - echo $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \ + echo $(IMAGE_NAME):$$escaped_buildx_platform-$$os-$$escaped_base_image$$tag; \ done); \ docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \ - echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch buildx_platform suffix base_image addon_image; do \ if [ $$os = "windows" ]; then \ escaped_base_image=$${base_image/:/-}; \ if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ diff --git a/cloudbuild.yaml b/cloudbuild.yaml index f36f265fc..4bcffad3a 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -16,9 +16,8 @@ # To promote release images, see https://github.com/kubernetes/k8s.io/tree/HEAD/k8s.gcr.io/images/k8s-staging-sig-storage. # This must be specified in seconds. If omitted, defaults to 600s (10 mins). -# Building three images in external-snapshotter takes roughly half an hour, -# sometimes more. -timeout: 3600s +# Building three images in external-snapshotter takes more than an hour. +timeout: 7200s # This prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF, # or any new substitutions added in the future. options: diff --git a/prow.sh b/prow.sh index ac8d8b0ca..8053b205a 100755 --- a/prow.sh +++ b/prow.sh @@ -78,7 +78,7 @@ version_to_git () { # the list of windows versions was matched from: # - https://hub.docker.com/_/microsoft-windows-nanoserver # - https://hub.docker.com/_/microsoft-windows-servercore -configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm -arm; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2; windows amd64 .exe nanoserver:ltsc2022 servercore:ltsc2022" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" +configvar CSI_PROW_BUILD_PLATFORMS "linux amd64 amd64; linux ppc64le ppc64le -ppc64le; linux s390x s390x -s390x; linux arm arm -arm; linux arm64 arm64 -arm64; linux arm arm/v7 -armv7; windows amd64 amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 amd64 .exe nanoserver:20H2 servercore:20H2; windows amd64 amd64 .exe nanoserver:ltsc2022 servercore:ltsc2022" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" # If we have a vendor directory, then use it. We must be careful to only # use this for "make" invocations inside the project's repo itself because @@ -149,7 +149,8 @@ configvar CSI_PROW_KIND_VERSION "$(kind_version_default)" "kind" # kind images to use. Must match the kind version. # The release notes of each kind release list the supported images. -configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.22.0@sha256:b8bda84bb3a190e6e028b1760d277454a72267a5454b57db34437c34a588d047 +configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.23.0@sha256:49824ab1727c04e56a21a5d8372a402fcd32ea51ac96a2706a12af38934f81ac +kindest/node:v1.22.0@sha256:b8bda84bb3a190e6e028b1760d277454a72267a5454b57db34437c34a588d047 kindest/node:v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6 kindest/node:v1.20.7@sha256:cbeaf907fc78ac97ce7b625e4bf0de16e3ea725daf6b04f930bd14c67c671ff9 kindest/node:v1.19.11@sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729 @@ -233,7 +234,7 @@ configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" # of the cluster. The alternative would have been to (cross-)compile csi-sanity # and install it inside the cluster, which is not necessarily easier. configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo" -configvar CSI_PROW_SANITY_VERSION v4.2.0 "csi-test version" +configvar CSI_PROW_SANITY_VERSION v4.3.0 "csi-test version" configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package" configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock" configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver" @@ -736,7 +737,7 @@ install_csi_driver () { fi } -# Installs all nessesary snapshotter CRDs +# Installs all necessary snapshotter CRDs install_snapshot_crds() { # Wait until volumesnapshot CRDs are in place. CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd"