From 7c7862dc89c39aac52c8ad18caa470b9f4b70977 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Mon, 27 May 2024 19:31:52 +0000 Subject: [PATCH 01/22] feat: enable hermetic library generation --- .../{.OwlBot.yaml => .OwlBot-hermetic.yaml} | 0 .../hermetic_library_generation.yaml | 39 ++++++ .../update_googleapis_committish.yaml | 42 ++++++ generation/hermetic_library_generation.sh | 125 ++++++++++++++++++ generation/update_googleapis_committish.sh | 92 +++++++++++++ generation_config.yaml | 58 ++++++++ 6 files changed, 356 insertions(+) rename .github/{.OwlBot.yaml => .OwlBot-hermetic.yaml} (100%) create mode 100644 .github/workflows/hermetic_library_generation.yaml create mode 100644 .github/workflows/update_googleapis_committish.yaml create mode 100644 generation/hermetic_library_generation.sh create mode 100644 generation/update_googleapis_committish.sh create mode 100644 generation_config.yaml diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot-hermetic.yaml similarity index 100% rename from .github/.OwlBot.yaml rename to .github/.OwlBot-hermetic.yaml diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml new file mode 100644 index 00000000000..a7e05557b4d --- /dev/null +++ b/.github/workflows/hermetic_library_generation.yaml @@ -0,0 +1,39 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# GitHub action job to test core java library features on +# downstream client libraries before they are released. +name: Hermetic library generation upon generation config change through pull requests +on: + pull_request: + +jobs: + library_generation: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} + - name: Generate changed libraries + shell: bash + run: | + set -x + [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" + [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" + bash generation/hermetic_library_generation.sh \ + --target_branch ${{ github.base_ref }} \ + --current_branch ${{ github.head_ref }} \ + --image_tag $(cat generation_config.yaml | yq .gapic_generator_version) + env: + GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} diff --git a/.github/workflows/update_googleapis_committish.yaml b/.github/workflows/update_googleapis_committish.yaml new file mode 100644 index 00000000000..aac08d039e4 --- /dev/null +++ b/.github/workflows/update_googleapis_committish.yaml @@ -0,0 +1,42 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# GitHub action job to test core java library features on +# downstream client libraries before they are released. +name: Update googleapis commit +on: + schedule: + - cron: '0 2 * * *' + workflow_dispatch: + +jobs: + update-googleapis-committish: + runs-on: ubuntu-22.04 + env: + # the branch into which the pull request is merged + base_branch: main + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} + - name: Update googleapis committish to latest + shell: bash + run: | + set -x + [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" + [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" + bash generation/update_googleapis_committish.sh \ + --base_branch "${base_branch}"\ + --repo ${{ github.repository }} + env: + GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} diff --git a/generation/hermetic_library_generation.sh b/generation/hermetic_library_generation.sh new file mode 100644 index 00000000000..f425d9c12e8 --- /dev/null +++ b/generation/hermetic_library_generation.sh @@ -0,0 +1,125 @@ +#!/bin/bash +set -e +# This script should be run at the root of the repository. +# This script is used to, when a pull request changes the generation +# configuration (generation_config.yaml by default): +# 1. Find whether the last commit in this pull request contains changes to +# the generation configuration and exit early if it doesn't have such a change +# since the generation result would be the same. +# 2. Compare generation configurations in the current branch (with which the +# pull request associated) and target branch (into which the pull request is +# merged); +# 3. Generate changed libraries using library_generation image; +# 4. Commit the changes to the pull request, if any. +# 5. Edit the PR body with generated pull request description, if applicable. + +# The following commands need to be installed before running the script: +# 1. git +# 2. gh +# 3. docker + +# The parameters of this script is: +# 1. target_branch, the branch into which the pull request is merged. +# 2. current_branch, the branch with which the pull request is associated. +# 3. image_tag, the tag of gcr.io/cloud-devrel-public-resources/java-library-generation. +# 3. [optional] generation_config, the path to the generation configuration, +# the default value is generation_config.yaml in the repository root. +while [[ $# -gt 0 ]]; do +key="$1" +case "${key}" in + --target_branch) + target_branch="$2" + shift + ;; + --current_branch) + current_branch="$2" + shift + ;; + --image_tag) + image_tag="$2" + shift + ;; + --generation_config) + generation_config="$2" + shift + ;; + *) + echo "Invalid option: [$1]" + exit 1 + ;; +esac +shift +done + +if [ -z "${target_branch}" ]; then + echo "missing required argument --target_branch" + exit 1 +fi + +if [ -z "${current_branch}" ]; then + echo "missing required argument --current_branch" + exit 1 +fi + +if [ -z "${image_tag}" ]; then + echo "missing required argument --image_tag" + exit 1 +fi + +if [ -z "${generation_config}" ]; then + generation_config=generation_config.yaml + echo "Use default generation config: ${generation_config}" +fi + +workspace_name="/workspace" +baseline_generation_config="baseline_generation_config.yaml" +message="chore: generate libraries at $(date)" + +git checkout "${target_branch}" +git checkout "${current_branch}" +# if the last commit doesn't contain changes to generation configuration, +# do not generate again as the result will be the same. +change_of_last_commit="$(git diff-tree --no-commit-id --name-only HEAD~1..HEAD -r)" +if [[ ! ("${change_of_last_commit}" == *"${generation_config}"*) ]]; then + echo "The last commit doesn't contain any changes to the generation_config.yaml, skipping the whole generation process." || true + exit 0 +fi +# copy generation configuration from target branch to current branch. +git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}" +config_diff=$(diff "${generation_config}" "${baseline_generation_config}" || true) + +# run hermetic code generation docker image. +docker run \ + --rm \ + -u "$(id -u):$(id -g)" \ + -v "$(pwd):${workspace_name}" \ + -v "${HOME}/.m2:/home/.m2" \ + gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ + --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ + --current-generation-config-path="${workspace_name}/${generation_config}" + + +# commit the change to the pull request. +if [[ $(basename $(pwd)) == "google-cloud-java" ]]; then + git add java-* pom.xml gapic-libraries-bom/pom.xml versions.txt +else + # The image leaves intermediate folders and files it works with. Here we remove them + rm -rdf output googleapis baseline_generation_config.yaml pr_description.txt + git add . +fi +changed_files=$(git diff --cached --name-only) +if [[ "${changed_files}" == "" ]]; then + echo "There is no generated code change with the generation config change ${config_diff}." + echo "Skip committing to the pull request." + exit 0 +fi + +echo "Configuration diff:" +echo "${config_diff}" +git commit -m "${message}" +git push +# set pr body if pr_description.txt is generated. +if [[ -f "pr_description.txt" ]]; then + pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") + gh pr edit "${pr_num}" --body "$(cat pr_description.txt)" +fi diff --git a/generation/update_googleapis_committish.sh b/generation/update_googleapis_committish.sh new file mode 100644 index 00000000000..5dfcaba567d --- /dev/null +++ b/generation/update_googleapis_committish.sh @@ -0,0 +1,92 @@ +#!/bin/bash +set -e +# This script should be run at the root of the repository. +# This script is used to update googleapis committish to latest in generation +# configuration at the time of running and create a pull request. + +# The following commands need to be installed before running the script: +# 1. git +# 2. gh + +# The parameters of this script is: +# 1. base_branch, the base branch of the result pull request. +# 2. repo, organization/repo-name, e.g., googleapis/google-cloud-java +# 3. [optional] generation_config, the path to the generation configuration, +# the default value is generation_config.yaml in the repository root. +while [[ $# -gt 0 ]]; do +key="$1" +case "${key}" in + --base_branch) + base_branch="$2" + shift + ;; + --repo) + repo="$2" + shift + ;; + --generation_config) + generation_config="$2" + shift + ;; + *) + echo "Invalid option: [$1]" + exit 1 + ;; +esac +shift +done + +if [ -z "${base_branch}" ]; then + echo "missing required argument --base_branch" + exit 1 +fi + +if [ -z "${repo}" ]; then + echo "missing required argument --repo" + exit 1 +fi + +if [ -z "${generation_config}" ]; then + generation_config="generation_config.yaml" + echo "Use default generation config: ${generation_config}" +fi + +current_branch="generate-libraries-${base_branch}" +title="chore: update googleapis committish at $(date)" + +# try to find a open pull request associated with the branch +pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") +# create a branch if there's no open pull request associated with the +# branch; otherwise checkout the pull request. +if [ -z "${pr_num}" ]; then + git checkout -b "${current_branch}" +else + gh pr checkout "${pr_num}" +fi + +mkdir tmp-googleapis +# use partial clone because only commit history is needed. +git clone --filter=blob:none https://github.com/googleapis/googleapis.git tmp-googleapis +pushd tmp-googleapis +git pull +latest_commit=$(git rev-parse HEAD) +popd +rm -rf tmp-googleapis +sed -i -e "s/^googleapis_commitish.*$/googleapis_commitish: ${latest_commit}/" "${generation_config}" + +git add "${generation_config}" +changed_files=$(git diff --cached --name-only) +if [[ "${changed_files}" == "" ]]; then + echo "The latest googleapis commit is not changed." + echo "Skip committing to the pull request." + exit 0 +fi +git commit -m "${title}" +if [ -z "${pr_num}" ]; then + git remote add remote_repo https://cloud-java-bot:"${GH_TOKEN}@github.com/${repo}.git" + git fetch -q --unshallow remote_repo + git push -f remote_repo "${current_branch}" + gh pr create --title "${title}" --head "${current_branch}" --body "${title}" --base "${base_branch}" +else + git push +fi diff --git a/generation_config.yaml b/generation_config.yaml new file mode 100644 index 00000000000..3e38a9d5460 --- /dev/null +++ b/generation_config.yaml @@ -0,0 +1,58 @@ +gapic_generator_version: 2.40.1 +protoc_version: '25.2' +googleapis_commitish: 6f289d775912966eb0cf04bda91e5e355c998d30 +libraries_bom_version: 26.38.0 +template_excludes: + - ".kokoro/continuous/common.cfg" + - ".kokoro/nightly/common.cfg" + - ".kokoro/nightly/integration.cfg" + - ".kokoro/nightly/java8-samples.cfg" + - ".kokoro/nightly/java11-samples.cfg" + - ".kokoro/nightly/samples.cfg" + - ".kokoro/build.bat" + - ".kokoro/presubmit/common.cfg" + - ".kokoro/presubmit/java8-samples.cfg" + - ".kokoro/presubmit/java11-samples.cfg" + - ".kokoro/presubmit/samples.cfg" + - ".kokoro/presubmit/graalvm-native.cfg" + - ".kokoro/presubmit/graalvm-native-17.cfg" + - ".kokoro/release/common.cfg" + - "samples/install-without-bom/pom.xml" + - "samples/snapshot/pom.xml" + - "samples/snippets/pom.xml" + - ".github/CODEOWNERS" + - ".github/sync-repo-settings.yaml" + - ".github/release-please.yml" + - ".github/blunderbuss.yml" + - ".github/workflows/samples.yaml" + - ".github/workflows/ci.yaml" + - ".kokoro/common.sh" + - ".kokoro/build.sh" + - ".kokoro/dependencies.sh" + - ".kokoro/requirements.in" + - ".kokoro/requirements.txt" +libraries: + - api_shortname: "spanner" + name_pretty: "Cloud Spanner" + product_documentation: "https://cloud.google.com/spanner/docs/" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-spanner/latest/history" + api_description: "is a fully managed, mission-critical, \nrelational database service that offers transactional consistency at global scale, \nschemas, SQL (ANSI 2011 with extensions), and automatic, synchronous replication \nfor high availability.\n\nBe sure to activate the Cloud Spanner API on the Developer's Console to\nuse Cloud Spanner from your project." + issue_tracker: "https://issuetracker.google.com/issues?q=componentid:190851%2B%20status:open" + release_level: "stable" + language: "java" + min_java_version: 8 + repo: "googleapis/java-spanner" + repo_short: "java-spanner" + distribution_name: "com.google.cloud:google-cloud-spanner" + api_id: "spanner.googleapis.com" + transport: "grpc" + requires_billing: true + codeowner_team: "@googleapis/api-spanner-java" + library_type: "GAPIC_COMBO" + excluded_poms: "google-cloud-spanner-bom" + recommended_package: "com.google.cloud.spanner" + GAPICs: + - proto_path: google/spanner/admin/database/v1 + - proto_path: google/spanner/admin/instance/v1 + - proto_path: google/spanner/executor/v1 + - proto_path: google/spanner/v1 From c28ff6520f2889542b1ac7d48d2c2bc660b3724e Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Mon, 27 May 2024 20:47:59 +0000 Subject: [PATCH 02/22] fix config yaml syntax --- generation_config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/generation_config.yaml b/generation_config.yaml index 3e38a9d5460..3e3f2cc1789 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -52,7 +52,7 @@ libraries: excluded_poms: "google-cloud-spanner-bom" recommended_package: "com.google.cloud.spanner" GAPICs: - - proto_path: google/spanner/admin/database/v1 - - proto_path: google/spanner/admin/instance/v1 - - proto_path: google/spanner/executor/v1 - - proto_path: google/spanner/v1 + - proto_path: google/spanner/admin/database/v1 + - proto_path: google/spanner/admin/instance/v1 + - proto_path: google/spanner/executor/v1 + - proto_path: google/spanner/v1 From d97769f6cacd5f3ccee178643b794417868316e2 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 28 May 2024 00:08:49 +0000 Subject: [PATCH 03/22] do not map runners home folder --- generation/hermetic_library_generation.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/generation/hermetic_library_generation.sh b/generation/hermetic_library_generation.sh index f425d9c12e8..9464642d9fa 100644 --- a/generation/hermetic_library_generation.sh +++ b/generation/hermetic_library_generation.sh @@ -93,7 +93,6 @@ docker run \ --rm \ -u "$(id -u):$(id -g)" \ -v "$(pwd):${workspace_name}" \ - -v "${HOME}/.m2:/home/.m2" \ gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ --current-generation-config-path="${workspace_name}/${generation_config}" From ef4c1aa4626220d5f7caa20f9944d23b06bbf5ad Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 28 May 2024 00:39:50 +0000 Subject: [PATCH 04/22] use copyright update comittish --- generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation_config.yaml b/generation_config.yaml index 3e3f2cc1789..771e7a27f9e 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -1,6 +1,6 @@ gapic_generator_version: 2.40.1 protoc_version: '25.2' -googleapis_commitish: 6f289d775912966eb0cf04bda91e5e355c998d30 +googleapis_committish: 3597f7db2191c00b100400991ef96e52d62f5841 libraries_bom_version: 26.38.0 template_excludes: - ".kokoro/continuous/common.cfg" From a8d46308d42b9c674c3246f04fb1c60514e4efbf Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 28 May 2024 00:41:59 +0000 Subject: [PATCH 05/22] correct proto_path --- generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation_config.yaml b/generation_config.yaml index 771e7a27f9e..3e3f2cc1789 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -1,6 +1,6 @@ gapic_generator_version: 2.40.1 protoc_version: '25.2' -googleapis_committish: 3597f7db2191c00b100400991ef96e52d62f5841 +googleapis_commitish: 6f289d775912966eb0cf04bda91e5e355c998d30 libraries_bom_version: 26.38.0 template_excludes: - ".kokoro/continuous/common.cfg" From c76147b47ac125a4a495c900ed37d6637514c3bb Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 28 May 2024 14:29:06 +0000 Subject: [PATCH 06/22] update protoc --- generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation_config.yaml b/generation_config.yaml index 3e3f2cc1789..2720e97de7c 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -1,5 +1,5 @@ gapic_generator_version: 2.40.1 -protoc_version: '25.2' +protoc_version: '25.3' googleapis_commitish: 6f289d775912966eb0cf04bda91e5e355c998d30 libraries_bom_version: 26.38.0 template_excludes: From b5b4ea2ec6a142c67826d0aa3c99bcd878dde0a5 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 29 May 2024 21:32:35 +0000 Subject: [PATCH 07/22] preserve pr_description --- generation/hermetic_library_generation.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generation/hermetic_library_generation.sh b/generation/hermetic_library_generation.sh index 9464642d9fa..d7e55e3c67d 100644 --- a/generation/hermetic_library_generation.sh +++ b/generation/hermetic_library_generation.sh @@ -103,8 +103,8 @@ if [[ $(basename $(pwd)) == "google-cloud-java" ]]; then git add java-* pom.xml gapic-libraries-bom/pom.xml versions.txt else # The image leaves intermediate folders and files it works with. Here we remove them - rm -rdf output googleapis baseline_generation_config.yaml pr_description.txt - git add . + rm -rdf output googleapis "${baseline_generation_config}" + git add --all -- ':!pr_description.txt' fi changed_files=$(git diff --cached --name-only) if [[ "${changed_files}" == "" ]]; then From 4889b085e77f9fef3c421d008d771ff6cb0ea730 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 4 Jun 2024 20:50:27 +0000 Subject: [PATCH 08/22] update gapic_generator_version to 2.41.0 --- generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation_config.yaml b/generation_config.yaml index 2720e97de7c..0dedec8c0d8 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -1,4 +1,4 @@ -gapic_generator_version: 2.40.1 +gapic_generator_version: 2.41.0 protoc_version: '25.3' googleapis_commitish: 6f289d775912966eb0cf04bda91e5e355c998d30 libraries_bom_version: 26.38.0 From de9954870e2efffb3403edd50ba4e0a8830eb790 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 12 Jun 2024 17:11:48 +0000 Subject: [PATCH 09/22] infer image tag from config yaml --- .../workflows/hermetic_library_generation.yaml | 3 +-- generation/hermetic_library_generation.sh | 15 ++++----------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml index a7e05557b4d..23385a1bed1 100644 --- a/.github/workflows/hermetic_library_generation.yaml +++ b/.github/workflows/hermetic_library_generation.yaml @@ -33,7 +33,6 @@ jobs: [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" bash generation/hermetic_library_generation.sh \ --target_branch ${{ github.base_ref }} \ - --current_branch ${{ github.head_ref }} \ - --image_tag $(cat generation_config.yaml | yq .gapic_generator_version) + --current_branch ${{ github.head_ref }} env: GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} diff --git a/generation/hermetic_library_generation.sh b/generation/hermetic_library_generation.sh index d7e55e3c67d..6c3f22d8f9e 100644 --- a/generation/hermetic_library_generation.sh +++ b/generation/hermetic_library_generation.sh @@ -21,7 +21,6 @@ set -e # The parameters of this script is: # 1. target_branch, the branch into which the pull request is merged. # 2. current_branch, the branch with which the pull request is associated. -# 3. image_tag, the tag of gcr.io/cloud-devrel-public-resources/java-library-generation. # 3. [optional] generation_config, the path to the generation configuration, # the default value is generation_config.yaml in the repository root. while [[ $# -gt 0 ]]; do @@ -35,10 +34,6 @@ case "${key}" in current_branch="$2" shift ;; - --image_tag) - image_tag="$2" - shift - ;; --generation_config) generation_config="$2" shift @@ -61,14 +56,9 @@ if [ -z "${current_branch}" ]; then exit 1 fi -if [ -z "${image_tag}" ]; then - echo "missing required argument --image_tag" - exit 1 -fi - if [ -z "${generation_config}" ]; then generation_config=generation_config.yaml - echo "Use default generation config: ${generation_config}" + echo "Using default generation config: ${generation_config}" fi workspace_name="/workspace" @@ -88,6 +78,9 @@ fi git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}" config_diff=$(diff "${generation_config}" "${baseline_generation_config}" || true) +# parse image tag from the generation configuration. +image_tag=$(grep "gapic_generator_version" "${generation_config}" | cut -d ':' -f 2 | xargs) + # run hermetic code generation docker image. docker run \ --rm \ From 14b187f0b81174a9bfffab11c15d492ca3e43df7 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 12 Jun 2024 17:18:31 +0000 Subject: [PATCH 10/22] correct workflow name --- .github/workflows/update_googleapis_committish.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_googleapis_committish.yaml b/.github/workflows/update_googleapis_committish.yaml index aac08d039e4..7cc9356f093 100644 --- a/.github/workflows/update_googleapis_committish.yaml +++ b/.github/workflows/update_googleapis_committish.yaml @@ -13,7 +13,7 @@ # limitations under the License. # GitHub action job to test core java library features on # downstream client libraries before they are released. -name: Update googleapis commit +name: Update googleapis committish on: schedule: - cron: '0 2 * * *' From 25deaf1b297c77bda18aa08ce8f2b0a3966af3f5 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 12 Jun 2024 17:45:21 +0000 Subject: [PATCH 11/22] update config scripts and yamls --- .../scripts}/hermetic_library_generation.sh | 0 .../scripts/update_generation_config.sh | 39 ++++++++++++++--- .../workflows/update_generation_config.yaml | 42 +++++++++++++++++++ 3 files changed, 76 insertions(+), 5 deletions(-) rename {generation => .github/scripts}/hermetic_library_generation.sh (100%) rename generation/update_googleapis_committish.sh => .github/scripts/update_generation_config.sh (59%) create mode 100644 .github/workflows/update_generation_config.yaml diff --git a/generation/hermetic_library_generation.sh b/.github/scripts/hermetic_library_generation.sh similarity index 100% rename from generation/hermetic_library_generation.sh rename to .github/scripts/hermetic_library_generation.sh diff --git a/generation/update_googleapis_committish.sh b/.github/scripts/update_generation_config.sh similarity index 59% rename from generation/update_googleapis_committish.sh rename to .github/scripts/update_generation_config.sh index 5dfcaba567d..561a313040f 100644 --- a/generation/update_googleapis_committish.sh +++ b/.github/scripts/update_generation_config.sh @@ -1,12 +1,32 @@ #!/bin/bash set -e # This script should be run at the root of the repository. -# This script is used to update googleapis committish to latest in generation -# configuration at the time of running and create a pull request. +# This script is used to update googleapis_commitish, gapic_generator_version, +# and libraries_bom_version in generation configuration at the time of running +# and create a pull request. # The following commands need to be installed before running the script: # 1. git # 2. gh +# 3. jq + +# Utility functions +# Get the latest released version of a Maven artifact. +function get_latest_released_version() { + local group_id=$1 + local artifact_id=$2 + latest=$(curl -s "https://search.maven.org/solrsearch/select?q=g:${group_id}+AND+a:${artifact_id}&core=gav&rows=500&wt=json" | jq -r '.response.docs[] | select(.v | test("^[0-9]+(\\.[0-9]+)*$")) | .v' | sort -V | tail -n 1) + echo "${latest}" +} + +# Update a key to a new value in the generation config. +function update_config() { + local key_word=$1 + local new_value=$2 + local file=$3 + echo "Update ${key_word} to ${new_value} in ${file}" + sed -i -e "s/^${key_word}.*$/${key_word}: ${new_value}/" "${file}" +} # The parameters of this script is: # 1. base_branch, the base branch of the result pull request. @@ -52,7 +72,7 @@ if [ -z "${generation_config}" ]; then fi current_branch="generate-libraries-${base_branch}" -title="chore: update googleapis committish at $(date)" +title="chore: Update generation configuration at $(date)" # try to find a open pull request associated with the branch pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") @@ -72,12 +92,20 @@ git pull latest_commit=$(git rev-parse HEAD) popd rm -rf tmp-googleapis -sed -i -e "s/^googleapis_commitish.*$/googleapis_commitish: ${latest_commit}/" "${generation_config}" +update_config "googleapis_commitish" "${latest_commit}" "${generation_config}" + +# update gapic-generator-java version to the latest +latest_version=$(get_latest_released_version "com.google.api" "gapic-generator-java") +update_config "gapic_generator_version" "${latest_version}" "${generation_config}" + +# update libraries-bom version to the latest +latest_version=$(get_latest_released_version "com.google.cloud" "libraries-bom") +update_config "libraries_bom_version" "${latest_version}" "${generation_config}" git add "${generation_config}" changed_files=$(git diff --cached --name-only) if [[ "${changed_files}" == "" ]]; then - echo "The latest googleapis commit is not changed." + echo "The latest generation config is not changed." echo "Skip committing to the pull request." exit 0 fi @@ -89,4 +117,5 @@ if [ -z "${pr_num}" ]; then gh pr create --title "${title}" --head "${current_branch}" --body "${title}" --base "${base_branch}" else git push + gh pr edit "${pr_num}" --title "${title}" --body "${title}" fi diff --git a/.github/workflows/update_generation_config.yaml b/.github/workflows/update_generation_config.yaml new file mode 100644 index 00000000000..70b513312b7 --- /dev/null +++ b/.github/workflows/update_generation_config.yaml @@ -0,0 +1,42 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# GitHub action job to test core java library features on +# downstream client libraries before they are released. +name: Update generation configuration +on: + schedule: + - cron: '0 2 * * *' + workflow_dispatch: + +jobs: + update-generation-config: + runs-on: ubuntu-22.04 + env: + # the branch into which the pull request is merged + base_branch: main + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} + - name: Update params in generation config to latest + shell: bash + run: | + set -x + [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" + [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" + bash .github/scripts/update_generation_config.sh.sh \ + --base_branch "${base_branch}"\ + --repo ${{ github.repository }} + env: + GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} From 280a8cd21f2f38bde9ca5b0312db7bb5094114e8 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 20 Jun 2024 20:41:53 +0000 Subject: [PATCH 12/22] remove old update_googleapis_committish workflow --- .../update_googleapis_committish.yaml | 42 ------------------- 1 file changed, 42 deletions(-) delete mode 100644 .github/workflows/update_googleapis_committish.yaml diff --git a/.github/workflows/update_googleapis_committish.yaml b/.github/workflows/update_googleapis_committish.yaml deleted file mode 100644 index 7cc9356f093..00000000000 --- a/.github/workflows/update_googleapis_committish.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# GitHub action job to test core java library features on -# downstream client libraries before they are released. -name: Update googleapis committish -on: - schedule: - - cron: '0 2 * * *' - workflow_dispatch: - -jobs: - update-googleapis-committish: - runs-on: ubuntu-22.04 - env: - # the branch into which the pull request is merged - base_branch: main - steps: - - uses: actions/checkout@v4 - with: - token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} - - name: Update googleapis committish to latest - shell: bash - run: | - set -x - [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" - [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" - bash generation/update_googleapis_committish.sh \ - --base_branch "${base_branch}"\ - --repo ${{ github.repository }} - env: - GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} From 53d15479f4c68d047e0ce02c61b0631b9c67f15c Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 20 Jun 2024 21:33:34 +0000 Subject: [PATCH 13/22] sync config structure with that of google-cloud-java --- generation_config.yaml | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/generation_config.yaml b/generation_config.yaml index 0dedec8c0d8..41b18987fe3 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -1,38 +1,8 @@ gapic_generator_version: 2.41.0 -protoc_version: '25.3' googleapis_commitish: 6f289d775912966eb0cf04bda91e5e355c998d30 libraries_bom_version: 26.38.0 -template_excludes: - - ".kokoro/continuous/common.cfg" - - ".kokoro/nightly/common.cfg" - - ".kokoro/nightly/integration.cfg" - - ".kokoro/nightly/java8-samples.cfg" - - ".kokoro/nightly/java11-samples.cfg" - - ".kokoro/nightly/samples.cfg" - - ".kokoro/build.bat" - - ".kokoro/presubmit/common.cfg" - - ".kokoro/presubmit/java8-samples.cfg" - - ".kokoro/presubmit/java11-samples.cfg" - - ".kokoro/presubmit/samples.cfg" - - ".kokoro/presubmit/graalvm-native.cfg" - - ".kokoro/presubmit/graalvm-native-17.cfg" - - ".kokoro/release/common.cfg" - - "samples/install-without-bom/pom.xml" - - "samples/snapshot/pom.xml" - - "samples/snippets/pom.xml" - - ".github/CODEOWNERS" - - ".github/sync-repo-settings.yaml" - - ".github/release-please.yml" - - ".github/blunderbuss.yml" - - ".github/workflows/samples.yaml" - - ".github/workflows/ci.yaml" - - ".kokoro/common.sh" - - ".kokoro/build.sh" - - ".kokoro/dependencies.sh" - - ".kokoro/requirements.in" - - ".kokoro/requirements.txt" libraries: - - api_shortname: "spanner" + - api_shortname: spanner name_pretty: "Cloud Spanner" product_documentation: "https://cloud.google.com/spanner/docs/" client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-spanner/latest/history" From 8a11c5b4860da994710391e6e8a78d4107c86574 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 25 Jun 2024 18:54:27 +0000 Subject: [PATCH 14/22] remove quotes from config yamls --- generation_config.yaml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/generation_config.yaml b/generation_config.yaml index 41b18987fe3..30f888bd8ec 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -3,24 +3,24 @@ googleapis_commitish: 6f289d775912966eb0cf04bda91e5e355c998d30 libraries_bom_version: 26.38.0 libraries: - api_shortname: spanner - name_pretty: "Cloud Spanner" - product_documentation: "https://cloud.google.com/spanner/docs/" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-spanner/latest/history" - api_description: "is a fully managed, mission-critical, \nrelational database service that offers transactional consistency at global scale, \nschemas, SQL (ANSI 2011 with extensions), and automatic, synchronous replication \nfor high availability.\n\nBe sure to activate the Cloud Spanner API on the Developer's Console to\nuse Cloud Spanner from your project." - issue_tracker: "https://issuetracker.google.com/issues?q=componentid:190851%2B%20status:open" - release_level: "stable" - language: "java" + name_pretty: Cloud Spanner + product_documentation: https://cloud.google.com/spanner/docs/ + client_documentation: https://cloud.google.com/java/docs/reference/google-cloud-spanner/latest/history + api_description: is a fully managed, mission-critical, \nrelational database service that offers transactional consistency at global scale, \nschemas, SQL (ANSI 2011 with extensions), and automatic, synchronous replication \nfor high availability.\n\nBe sure to activate the Cloud Spanner API on the Developer's Console to\nuse Cloud Spanner from your project. + issue_tracker: https://issuetracker.google.com/issues?q=componentid:190851%2B%20status:open + release_level: stable + language: java min_java_version: 8 - repo: "googleapis/java-spanner" - repo_short: "java-spanner" - distribution_name: "com.google.cloud:google-cloud-spanner" - api_id: "spanner.googleapis.com" - transport: "grpc" + repo: googleapis/java-spanner + repo_short: java-spanner + distribution_name: com.google.cloud:google-cloud-spanner + api_id: spanner.googleapis.com + transport: grpc requires_billing: true - codeowner_team: "@googleapis/api-spanner-java" - library_type: "GAPIC_COMBO" - excluded_poms: "google-cloud-spanner-bom" - recommended_package: "com.google.cloud.spanner" + codeowner_team: @googleapis/api-spanner-java + library_type: GAPIC_COMBO + excluded_poms: google-cloud-spanner-bom + recommended_package: com.google.cloud.spanner GAPICs: - proto_path: google/spanner/admin/database/v1 - proto_path: google/spanner/admin/instance/v1 From cbc138c2ae84acf6b3a589c09201952c803c813a Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 25 Jun 2024 20:20:13 +0000 Subject: [PATCH 15/22] fix typo in update_generation_config.yaml --- .github/workflows/update_generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_generation_config.yaml b/.github/workflows/update_generation_config.yaml index 70b513312b7..3cf77399264 100644 --- a/.github/workflows/update_generation_config.yaml +++ b/.github/workflows/update_generation_config.yaml @@ -35,7 +35,7 @@ jobs: set -x [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" - bash .github/scripts/update_generation_config.sh.sh \ + bash .github/scripts/update_generation_config.sh \ --base_branch "${base_branch}"\ --repo ${{ github.repository }} env: From 8973d2a07f8b9092a955b299bf47ca430c8f11a7 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 25 Jun 2024 20:26:09 +0000 Subject: [PATCH 16/22] correct --- .github/workflows/hermetic_library_generation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml index 23385a1bed1..0c7a82a3ca5 100644 --- a/.github/workflows/hermetic_library_generation.yaml +++ b/.github/workflows/hermetic_library_generation.yaml @@ -31,7 +31,7 @@ jobs: set -x [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" - bash generation/hermetic_library_generation.sh \ + bash .github/scriptes/hermetic_library_generation.sh \ --target_branch ${{ github.base_ref }} \ --current_branch ${{ github.head_ref }} env: From a0b45ae1935ac18a3d063629dba7691487a6045c Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 26 Jun 2024 20:47:18 +0000 Subject: [PATCH 17/22] quote codeowners_team in generation config --- generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation_config.yaml b/generation_config.yaml index 30f888bd8ec..9d5277763fa 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -17,7 +17,7 @@ libraries: api_id: spanner.googleapis.com transport: grpc requires_billing: true - codeowner_team: @googleapis/api-spanner-java + codeowner_team: '@googleapis/api-spanner-java' library_type: GAPIC_COMBO excluded_poms: google-cloud-spanner-bom recommended_package: com.google.cloud.spanner From ba64a29206111c6785fb2cef2146def117e8ca4e Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 26 Jun 2024 20:50:02 +0000 Subject: [PATCH 18/22] update generator version --- generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation_config.yaml b/generation_config.yaml index 9d5277763fa..76413a350ca 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -1,4 +1,4 @@ -gapic_generator_version: 2.41.0 +gapic_generator_version: 2.42.0 googleapis_commitish: 6f289d775912966eb0cf04bda91e5e355c998d30 libraries_bom_version: 26.38.0 libraries: From aa0c430370737199331908271cb5d44a24cb7c50 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 2 Jul 2024 15:29:20 +0000 Subject: [PATCH 19/22] fix path to hermetic_library_generation --- .github/workflows/hermetic_library_generation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml index 0c7a82a3ca5..3ea9ae43e4a 100644 --- a/.github/workflows/hermetic_library_generation.yaml +++ b/.github/workflows/hermetic_library_generation.yaml @@ -31,7 +31,7 @@ jobs: set -x [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" - bash .github/scriptes/hermetic_library_generation.sh \ + bash .github/scripts/hermetic_library_generation.sh \ --target_branch ${{ github.base_ref }} \ --current_branch ${{ github.head_ref }} env: From aca01579d07085c85419dd24aa9df7b6e0f75c3e Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 2 Jul 2024 17:30:17 +0000 Subject: [PATCH 20/22] remove escaped newlines --- generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation_config.yaml b/generation_config.yaml index 76413a350ca..8f89fb578b3 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -6,7 +6,7 @@ libraries: name_pretty: Cloud Spanner product_documentation: https://cloud.google.com/spanner/docs/ client_documentation: https://cloud.google.com/java/docs/reference/google-cloud-spanner/latest/history - api_description: is a fully managed, mission-critical, \nrelational database service that offers transactional consistency at global scale, \nschemas, SQL (ANSI 2011 with extensions), and automatic, synchronous replication \nfor high availability.\n\nBe sure to activate the Cloud Spanner API on the Developer's Console to\nuse Cloud Spanner from your project. + api_description: is a fully managed, mission-critical, relational database service that offers transactional consistency at global scale, \nschemas, SQL (ANSI 2011 with extensions), and automatic, synchronous replication \nfor high availability.\n\nBe sure to activate the Cloud Spanner API on the Developer's Console to\nuse Cloud Spanner from your project. issue_tracker: https://issuetracker.google.com/issues?q=componentid:190851%2B%20status:open release_level: stable language: java From 56459f16075357837eab65b9bace048d172b783c Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 2 Jul 2024 23:39:15 +0000 Subject: [PATCH 21/22] fixes to hermetic_library_generation --- .github/workflows/hermetic_library_generation.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml index 3ea9ae43e4a..7146cc3dc1c 100644 --- a/.github/workflows/hermetic_library_generation.yaml +++ b/.github/workflows/hermetic_library_generation.yaml @@ -19,6 +19,8 @@ on: jobs: library_generation: + # skip pull requests coming from a forked repository + if: github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From dfee1649acf1d190fb1913e6f8a204cf88b0cf1a Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 4 Jul 2024 04:57:36 +0000 Subject: [PATCH 22/22] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ba3bc5d59c7..64531b8e903 100644 --- a/README.md +++ b/README.md @@ -50,20 +50,20 @@ If you are using Maven without the BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.39.0') +implementation platform('com.google.cloud:libraries-bom:26.42.0') implementation 'com.google.cloud:google-cloud-spanner' ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-spanner:6.67.0' +implementation 'com.google.cloud:google-cloud-spanner:6.71.0' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-spanner" % "6.67.0" +libraryDependencies += "com.google.cloud" % "google-cloud-spanner" % "6.71.0" ``` @@ -671,7 +671,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-spanner/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-spanner.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-spanner/6.67.0 +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-spanner/6.71.0 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles