From 7f9461f267e96b3e5eedc43e41a869a3774f8db5 Mon Sep 17 00:00:00 2001 From: yoshi-automation Date: Tue, 16 Jun 2020 12:31:10 -0700 Subject: [PATCH 1/4] ci(java): switch to secret manager from keystore * ci(java): switch to secret manager from keystore, use java-docs-samples user for samples tests * fix: we don't need to supply a service account Source-Author: Jeff Ching Source-Date: Mon Jun 15 11:45:21 2020 -0700 Source-Repo: googleapis/synthtool Source-Sha: d1addcdf80aa9ddef8c932c89c919024bbad7af3 Source-Link: https://github.com/googleapis/synthtool/commit/d1addcdf80aa9ddef8c932c89c919024bbad7af3 --- .kokoro/build.sh | 2 +- .kokoro/nightly/samples.cfg | 16 +++++-------- .kokoro/populate-secrets.sh | 43 +++++++++++++++++++++++++++++++++++ .kokoro/presubmit/samples.cfg | 14 ++++-------- .kokoro/trampoline.sh | 2 ++ synth.metadata | 4 ++-- 6 files changed, 59 insertions(+), 22 deletions(-) create mode 100755 .kokoro/populate-secrets.sh diff --git a/.kokoro/build.sh b/.kokoro/build.sh index feda37d3d..7535c0aac 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -39,7 +39,7 @@ retry_with_backoff 3 10 \ # if GOOGLE_APPLICATION_CREDIENTIALS is specified as a relative path prepend Kokoro root directory onto it if [[ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" && "${GOOGLE_APPLICATION_CREDENTIALS}" != /* ]]; then - export GOOGLE_APPLICATION_CREDENTIALS=$(realpath ${KOKORO_ROOT}/src/${GOOGLE_APPLICATION_CREDENTIALS}) + export GOOGLE_APPLICATION_CREDENTIALS=$(realpath ${KOKORO_GFILE_DIR}/${GOOGLE_APPLICATION_CREDENTIALS}) fi RETURN_CODE=0 diff --git a/.kokoro/nightly/samples.cfg b/.kokoro/nightly/samples.cfg index 20aabd55d..03434c9e0 100644 --- a/.kokoro/nightly/samples.cfg +++ b/.kokoro/nightly/samples.cfg @@ -24,19 +24,15 @@ env_vars: { env_vars: { key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "keystore/73713_java_it_service_account" + value: "secret_manager/java-docs-samples-service-account" } +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-docs-samples-service-account" +} + env_vars: { key: "ENABLE_BUILD_COP" value: "true" } - -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "java_it_service_account" - } - } -} diff --git a/.kokoro/populate-secrets.sh b/.kokoro/populate-secrets.sh new file mode 100755 index 000000000..f52514257 --- /dev/null +++ b/.kokoro/populate-secrets.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright 2020 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;} +function msg { println "$*" >&2 ;} +function println { printf '%s\n' "$(now) $*" ;} + + +# Populates requested secrets set in SECRET_MANAGER_KEYS from service account: +# kokoro-trampoline@cloud-devrel-kokoro-resources.iam.gserviceaccount.com +SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager" +msg "Creating folder on disk for secrets: ${SECRET_LOCATION}" +mkdir -p ${SECRET_LOCATION} +for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g") +do + msg "Retrieving secret ${key}" + docker run --entrypoint=gcloud \ + --volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR} \ + gcr.io/google.com/cloudsdktool/cloud-sdk \ + secrets versions access latest \ + --project cloud-devrel-kokoro-resources \ + --secret ${key} > \ + "${SECRET_LOCATION}/${key}" + if [[ $? == 0 ]]; then + msg "Secret written to ${SECRET_LOCATION}/${key}" + else + msg "Error retrieving secret ${key}" + fi +done diff --git a/.kokoro/presubmit/samples.cfg b/.kokoro/presubmit/samples.cfg index 1171aead0..f502ddf1d 100644 --- a/.kokoro/presubmit/samples.cfg +++ b/.kokoro/presubmit/samples.cfg @@ -24,14 +24,10 @@ env_vars: { env_vars: { key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "keystore/73713_java_it_service_account" + value: "secret_manager/java-docs-samples-service-account" } -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "java_it_service_account" - } - } -} +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-docs-samples-service-account" +} \ No newline at end of file diff --git a/.kokoro/trampoline.sh b/.kokoro/trampoline.sh index ba17ce014..9da0f8398 100644 --- a/.kokoro/trampoline.sh +++ b/.kokoro/trampoline.sh @@ -21,4 +21,6 @@ function cleanup() { echo "cleanup"; } trap cleanup EXIT + +$(dirname $0)/populate-secrets.sh # Secret Manager secrets. python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" diff --git a/synth.metadata b/synth.metadata index e5335f663..3700604a4 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-datastore.git", - "sha": "39c8d723b318d08ca494b71167eaa80b1df6423d" + "sha": "f1874f1b2745724ccc2345f35b65fb577f53683e" } }, { @@ -19,7 +19,7 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "987270824bd26f6a8c716d5e2022057b8ae7b26e" + "sha": "d1addcdf80aa9ddef8c932c89c919024bbad7af3" } } ], From 46c85f6140c88b17df252df3ab512908c6f1874f Mon Sep 17 00:00:00 2001 From: yoshi-automation Date: Tue, 16 Jun 2020 12:31:25 -0700 Subject: [PATCH 2/4] samples(java): fix bad character in samples.cfg from copy/paste Somehow an invisible character was copy/pasted into the file and Kokoro cannot parse the file. Source-Author: Jeff Ching Source-Date: Mon Jun 15 15:42:10 2020 -0700 Source-Repo: googleapis/synthtool Source-Sha: 6d3eed67a45fd58f9c7bfa173c32e4fd4fed058f Source-Link: https://github.com/googleapis/synthtool/commit/6d3eed67a45fd58f9c7bfa173c32e4fd4fed058f --- .kokoro/nightly/samples.cfg | 6 +++--- .kokoro/presubmit/samples.cfg | 6 +++--- synth.metadata | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.kokoro/nightly/samples.cfg b/.kokoro/nightly/samples.cfg index 03434c9e0..f25429314 100644 --- a/.kokoro/nightly/samples.cfg +++ b/.kokoro/nightly/samples.cfg @@ -28,9 +28,9 @@ env_vars: { } env_vars: { - key: "SECRET_MANAGER_KEYS" - value: "java-docs-samples-service-account" -} + key: "SECRET_MANAGER_KEYS" + value: "java-docs-samples-service-account" +} env_vars: { key: "ENABLE_BUILD_COP" diff --git a/.kokoro/presubmit/samples.cfg b/.kokoro/presubmit/samples.cfg index f502ddf1d..01e096004 100644 --- a/.kokoro/presubmit/samples.cfg +++ b/.kokoro/presubmit/samples.cfg @@ -28,6 +28,6 @@ env_vars: { } env_vars: { - key: "SECRET_MANAGER_KEYS" - value: "java-docs-samples-service-account" -} \ No newline at end of file + key: "SECRET_MANAGER_KEYS" + value: "java-docs-samples-service-account" +} \ No newline at end of file diff --git a/synth.metadata b/synth.metadata index 3700604a4..cdba7c958 100644 --- a/synth.metadata +++ b/synth.metadata @@ -19,7 +19,7 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "d1addcdf80aa9ddef8c932c89c919024bbad7af3" + "sha": "6d3eed67a45fd58f9c7bfa173c32e4fd4fed058f" } } ], From aacdac9c372def16629a7d5f8bf51b234ff6aca0 Mon Sep 17 00:00:00 2001 From: yoshi-automation Date: Tue, 16 Jun 2020 12:31:41 -0700 Subject: [PATCH 3/4] ci(java): switch integration test secrets to secret manager Source-Author: Jeff Ching Source-Date: Tue Jun 16 09:26:06 2020 -0700 Source-Repo: googleapis/synthtool Source-Sha: 6eb80fa9f96433af8dbeb1c8323aa80cda49b374 Source-Link: https://github.com/googleapis/synthtool/commit/6eb80fa9f96433af8dbeb1c8323aa80cda49b374 --- .kokoro/nightly/integration.cfg | 12 ++++-------- .kokoro/presubmit/integration.cfg | 12 ++++-------- synth.metadata | 2 +- 3 files changed, 9 insertions(+), 17 deletions(-) diff --git a/.kokoro/nightly/integration.cfg b/.kokoro/nightly/integration.cfg index 40c4abb7b..d3299e0c0 100644 --- a/.kokoro/nightly/integration.cfg +++ b/.kokoro/nightly/integration.cfg @@ -28,14 +28,10 @@ env_vars: { env_vars: { key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "keystore/73713_java_it_service_account" + value: "secret_manager/java-it-service-accout" } -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "java_it_service_account" - } - } +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-it-service-accout" } diff --git a/.kokoro/presubmit/integration.cfg b/.kokoro/presubmit/integration.cfg index 522e5b101..7647ce93e 100644 --- a/.kokoro/presubmit/integration.cfg +++ b/.kokoro/presubmit/integration.cfg @@ -24,14 +24,10 @@ env_vars: { env_vars: { key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "keystore/73713_java_it_service_account" + value: "secret_manager/java-it-service-accout" } -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "java_it_service_account" - } - } +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-it-service-accout" } diff --git a/synth.metadata b/synth.metadata index cdba7c958..6b33d123f 100644 --- a/synth.metadata +++ b/synth.metadata @@ -19,7 +19,7 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "6d3eed67a45fd58f9c7bfa173c32e4fd4fed058f" + "sha": "6eb80fa9f96433af8dbeb1c8323aa80cda49b374" } } ], From 07e8fea44348920edca7f7b1346a45d0e14f1255 Mon Sep 17 00:00:00 2001 From: yoshi-automation Date: Tue, 16 Jun 2020 12:31:41 -0700 Subject: [PATCH 4/4] ci(java): fix typo in java integration test secret name Source-Author: BenWhitehead Source-Date: Tue Jun 16 13:32:19 2020 -0400 Source-Repo: googleapis/synthtool Source-Sha: c4f3059c27591eb24d6942a0e357ec94c80459f2 Source-Link: https://github.com/googleapis/synthtool/commit/c4f3059c27591eb24d6942a0e357ec94c80459f2 --- .kokoro/nightly/integration.cfg | 4 ++-- .kokoro/presubmit/integration.cfg | 4 ++-- synth.metadata | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.kokoro/nightly/integration.cfg b/.kokoro/nightly/integration.cfg index d3299e0c0..0048c8ece 100644 --- a/.kokoro/nightly/integration.cfg +++ b/.kokoro/nightly/integration.cfg @@ -28,10 +28,10 @@ env_vars: { env_vars: { key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "secret_manager/java-it-service-accout" + value: "secret_manager/java-it-service-account" } env_vars: { key: "SECRET_MANAGER_KEYS" - value: "java-it-service-accout" + value: "java-it-service-account" } diff --git a/.kokoro/presubmit/integration.cfg b/.kokoro/presubmit/integration.cfg index 7647ce93e..dded67a9d 100644 --- a/.kokoro/presubmit/integration.cfg +++ b/.kokoro/presubmit/integration.cfg @@ -24,10 +24,10 @@ env_vars: { env_vars: { key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "secret_manager/java-it-service-accout" + value: "secret_manager/java-it-service-account" } env_vars: { key: "SECRET_MANAGER_KEYS" - value: "java-it-service-accout" + value: "java-it-service-account" } diff --git a/synth.metadata b/synth.metadata index 6b33d123f..c6089f985 100644 --- a/synth.metadata +++ b/synth.metadata @@ -19,7 +19,7 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "6eb80fa9f96433af8dbeb1c8323aa80cda49b374" + "sha": "c4f3059c27591eb24d6942a0e357ec94c80459f2" } } ],