From 3a89671ea4c3f4e44c4085a3dd8b990ed452167e Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Wed, 5 Jun 2024 13:26:22 +0000 Subject: [PATCH 1/2] Squashed 'release-tools/' changes from b54c1ba49..f40f0ccd4 f40f0ccd4 Merge pull request #256 from solumath/master cfa92106c Instruction update 379a1bb9b Merge pull request #255 from humblec/sidecar-md a5667bbbb fix typo in sidecar release process 49676850e Merge pull request #254 from bells17/add-github-actions d9bd160c2 Update skip list in codespell GitHub Action adb3af9df Merge pull request #252 from bells17/update-go-version f5aebfc9f Add GitHub Actions workflows b82ee3888 Merge pull request #253 from bells17/fix-typo c31745621 Fix typo 0a7850561 Bump to Go 1.22.3 edd89ad58 Merge pull request #251 from jsafrane/add-logcheck 043fd0991 Add test-logcheck target d7535ae0c Merge pull request #250 from jsafrane/go-1.22 b52e7ad35 Update go to 1.22.2 14fdb6f66 Merge pull request #247 from msau42/prow dc4d0ae20 Merge pull request #249 from jsafrane/use-go-version e681b170e Use .go-version to get Kubernetes go version 9b4352e9f Update release playbook c7bb972cc Fix release notes script to use fixed tags 463a0e9f5 Add script to update specific go modules git-subtree-dir: release-tools git-subtree-split: f40f0ccd458f2d4555e3ca98d69b5a984bae0f14 --- .github/dependabot.yaml | 12 +++ .github/workflows/codespell.yml | 15 +++ .github/workflows/trivy.yaml | 29 ++++++ SIDECAR_RELEASE_PROCESS.md | 11 ++- build.make | 7 ++ ...otes.sh => generate-patch-release-notes.sh | 30 +++--- go-modules-targeted-update.sh | 96 +++++++++++++++++++ prow.sh | 12 ++- verify-logcheck.sh | 37 +++++++ 9 files changed, 232 insertions(+), 17 deletions(-) create mode 100644 .github/dependabot.yaml create mode 100644 .github/workflows/codespell.yml create mode 100644 .github/workflows/trivy.yaml rename generate_patch_release_notes.sh => generate-patch-release-notes.sh (74%) create mode 100755 go-modules-targeted-update.sh create mode 100755 verify-logcheck.sh diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 00000000000..814a34493f2 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,12 @@ +version: 2 +enable-beta-ecosystems: true +updates: +- package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + labels: + - "area/dependency" + - "release-note-none" + - "ok-to-test" + open-pull-requests-limit: 10 diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml new file mode 100644 index 00000000000..e74edcef5e9 --- /dev/null +++ b/.github/workflows/codespell.yml @@ -0,0 +1,15 @@ +# GitHub Action to automate the identification of common misspellings in text files. +# https://github.com/codespell-project/actions-codespell +# https://github.com/codespell-project/codespell +name: codespell +on: [push, pull_request] +jobs: + codespell: + name: Check for spelling errors + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: codespell-project/actions-codespell@master + with: + check_filenames: true + skip: "*.png,*.jpg,*.svg,*.sum,./.git,./.github/workflows/codespell.yml,./prow.sh" diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml new file mode 100644 index 00000000000..472984780a9 --- /dev/null +++ b/.github/workflows/trivy.yaml @@ -0,0 +1,29 @@ +name: Run Trivy scanner for Go version vulnerabilities +on: + push: + branches: + - master + pull_request: +jobs: + trivy: + name: Build + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Get Go version + id: go-version + run: | + GO_VERSION=$(cat prow.sh | grep "configvar CSI_PROW_GO_VERSION_BUILD" | awk '{print $3}' | sed 's/"//g') + echo "version=$GO_VERSION" >> $GITHUB_OUTPUT + + - name: Run Trivy scanner for Go version vulnerabilities + uses: aquasecurity/trivy-action@master + with: + image-ref: 'golang:${{ steps.go-version.outputs.version }}' + format: 'table' + exit-code: '1' + ignore-unfixed: true + vuln-type: 'library' + severity: 'CRITICAL,HIGH,MEDIUM,LOW,UNKNOWN' diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 4f5d033c235..aab8d6e215a 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -46,9 +46,12 @@ naming convention `-on-`. ## Release Process 1. Identify all issues and ongoing PRs that should go into the release, and drive them to resolution. -1. Update dependencies for sidecars via - [go-modules-update.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/HEAD/release-tools/go-modules-update.sh), - and get PRs approved and merged. +1. Update dependencies for sidecars + 1. For new minor versions, use + [go-modules-update.sh](https://github.com/kubernetes-csi/csi-release-tools/blob/HEAD/go-modules-update.sh), + 1. For CVE fixes on patch versions, use + [go-modules-targeted-update.sh](https://github.com/kubernetes-csi/csi-release-tools/blob/HEAD/go-modules-targeted-update.sh), + Read the instructions at the top of the script. 1. Check that all [canary CI jobs](https://testgrid.k8s.io/sig-storage-csi-ci) are passing, and that test coverage is adequate for the changes that are going into the release. @@ -81,7 +84,7 @@ naming convention `-on-`. 1. Compare the generated output to the new commits for the release to check if any notable change missed a release note. 1. Reword release notes as needed, ideally in the original PRs so that the - release notes can be regnerated. Make sure to check notes for breaking + release notes can be regenerated. Make sure to check notes for breaking changes and deprecations. 1. If release is a new major/minor version, create a new `CHANGELOG-..md` file. diff --git a/build.make b/build.make index bceab34d267..fe120c52b91 100644 --- a/build.make +++ b/build.make @@ -322,3 +322,10 @@ test-spelling: test-boilerplate: @ echo; echo "### $@:" @ ./release-tools/verify-boilerplate.sh "$(pwd)" + +# Test klog usage. This test is optional and must be explicitly added to `test` target in the main Makefile: +# test: test-logcheck +.PHONY: test-logcheck +test-logcheck: + @ echo; echo "### $@:" + @ ./release-tools/verify-logcheck.sh diff --git a/generate_patch_release_notes.sh b/generate-patch-release-notes.sh similarity index 74% rename from generate_patch_release_notes.sh rename to generate-patch-release-notes.sh index 606c24d22de..536a1490234 100755 --- a/generate_patch_release_notes.sh +++ b/generate-patch-release-notes.sh @@ -23,13 +23,17 @@ # CSI_RELEASE_TOKEN: Github token needed for generating release notes # GITHUB_USER: Github username to create PRs with # +# Required tools: +# - gh +# - release-notes (https://github.com/kubernetes/release/blob/master/cmd/release-notes/README.md) +# # Instructions: -# 1. Login with "gh auth login" -# 2. Copy this script to the kubernetes-csi directory (one directory above the -# repos) -# 3. Update the repos and versions in the $releases array -# 4. Set environment variables -# 5. Run script from the kubernetes-csi directory +# 1. Install the required tools +# 2. Login with "gh auth login" +# 3. Copy this script to the kubernetes-csi directory (one directory above the repos) +# 4. Update the repos and versions in the $releases array +# 5. Set environment variables +# 6. Run script from the kubernetes-csi directory # # Caveats: # - This script doesn't handle regenerating and updating existing PRs yet. @@ -48,7 +52,7 @@ function gen_patch_relnotes() { rm out.md || true rm -rf /tmp/k8s-repo || true GITHUB_TOKEN="$CSI_RELEASE_TOKEN" \ - release-notes --discover=patch-to-latest --branch="$2" \ + release-notes --start-rev="$3" --end-rev="$2" --branch="$2" \ --org=kubernetes-csi --repo="$1" \ --required-author="" --markdown-links --output out.md } @@ -57,11 +61,14 @@ for rel in "${releases[@]}"; do read -r repo version <<< "$rel" # Parse minor version - minorPattern="(^[[:digit:]]+\.[[:digit:]]+)\." - [[ "$version" =~ $minorPattern ]] + minorPatchPattern="(^[[:digit:]]+\.[[:digit:]]+)\.([[:digit:]]+)" + [[ "$version" =~ $minorPatchPattern ]] minor="${BASH_REMATCH[1]}" + patch="${BASH_REMATCH[2]}" - echo "$repo" "$version" "$minor" + echo "$repo $version $minor $patch" + prevPatch="$((patch-1))" + prevVer="v$minor.$prevPatch" pushd "$repo/CHANGELOG" @@ -74,7 +81,7 @@ for rel in "${releases[@]}"; do git checkout --track "upstream/release-$minor" -b "$branch" # Generate release notes - gen_patch_relnotes "$repo" "release-$minor" + gen_patch_relnotes "$repo" "release-$minor" "$prevVer" cat > tmp.md <> tmp.md echo >> tmp.md + rm out.md file="CHANGELOG-$minor.md" cat "$file" >> tmp.md diff --git a/go-modules-targeted-update.sh b/go-modules-targeted-update.sh new file mode 100755 index 00000000000..52146f31d61 --- /dev/null +++ b/go-modules-targeted-update.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Usage: go-modules-targeted-update.sh +# +# Batch update specific dependencies for sidecars. +# +# Required environment variables +# CSI_RELEASE_TOKEN: Github token needed for generating release notes +# GITHUB_USER: Github username to create PRs with +# +# Instructions: +# 1. Login with "gh auth login" +# 2. Copy this script to the Github org directory (one directory above the +# repos) +# 3. Change $modules, $releases and $org if needed. +# 4. Set environment variables +# 5. Run script from the Github org directory +# +# Caveats: +# - This script doesn't handle interface incompatibility of updates. +# You need to resolve interface incompatibility case by case. The +# most frequent case is to update the interface(new parameters, +# name change of the method, etc.)in the sidecar repo and make sure +# the build and test pass. + + +set -e +set -x + +org="kubernetes-csi" + +modules=( +"github.com/kubernetes-csi/csi-lib-utils@v0.15.1" +) + +releases=( +#"external-attacher release-4.4" +#"external-provisioner release-3.6" +#"external-resizer release-1.9" +#"external-snapshotter release-6.3" +#"node-driver-registrar release-2.9" +) + +for rel in "${releases[@]}"; do + + read -r repo branch <<< "$rel" + if [ "$repo" != "#" ]; then + ( + cd "$repo" + git fetch upstream + + if [ "$(git rev-parse --verify "module-update-$branch" 2>/dev/null)" ]; then + git checkout master && git branch -D "module-update-$branch" + fi + git checkout -B "module-update-$branch" "upstream/$branch" + + for mod in "${modules[@]}"; do + go get "$mod" + done + go mod tidy + go mod vendor + + git add --all + git commit -m "Update go modules" + git push origin "module-update-$branch" --force + + # Create PR +prbody=$(cat </dev/null || mktemp -d -t csi-lib-utils.XXXXXX) +trap 'rm -rf "${CSI_LIB_UTIL_TEMP}"' EXIT + +echo "Installing logcheck to temp dir: sigs.k8s.io/logtools/logcheck@v${LOGCHECK_VERSION}" +GOBIN="${CSI_LIB_UTIL_TEMP}" go install "sigs.k8s.io/logtools/logcheck@v${LOGCHECK_VERSION}" +echo "Verifying logcheck: ${CSI_LIB_UTIL_TEMP}/logcheck -check-contextual ${CSI_LIB_UTIL_ROOT}/..." +"${CSI_LIB_UTIL_TEMP}/logcheck" -check-contextual -check-with-helpers "${CSI_LIB_UTIL_ROOT}/..." From 229efea1af4c61ee54a8bcdadc4b3005ce3e8902 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Wed, 5 Jun 2024 13:31:03 +0000 Subject: [PATCH 2/2] test: fix stdlib error in trivy gh action --- .github/workflows/trivy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index 528596cb1fc..b6d4a607338 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -12,7 +12,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v5 with: - go-version: 1.22.3 + go-version: 1.22.4 id: go - name: Checkout code