From a7ed11ec0863c422ba2e73aafa75eab22c32b33d Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Tue, 5 Oct 2021 10:27:14 -0600 Subject: [PATCH] build: use trampoline_v2 for python samples and allow custom dockerfile (#1232) * Use trampoline_v2 ([Docker CI Helper](https://github.com/GoogleCloudPlatform/docker-ci-helper)) for all python samples. * Optionally allow samples to run with a Dockerfile specified in the library repository at `.kokoro/docker/samples`. On a successful test run, this Dockerfile is uploaded to `gcr.io/cloud-devrel-kokoro-resources/{repo-name}-samples-docker`. The option is enabled by passing `custom_samples_dockerfile=True` to `common.py_library` in `owlbot.py`. ```py templated_files = common.py_library(microgenerator=True, custom_samples_dockerfile=True) ``` See https://github.com/googleapis/python-asset/pull/308 for an example of moving samples to to trampoline_v2. See https://github.com/googleapis/python-recaptcha-enterprise/pull/128 for an example of moving to trampoline_v2 + using a custom Dockerfile. --- synthtool/gcp/common.py | 4 ++++ .../.kokoro/samples/lint/common.cfg | 15 +++++++++++++-- .../.kokoro/samples/python3.6/common.cfg | 15 +++++++++++++-- .../.kokoro/samples/python3.6/periodic-head.cfg | 7 +++++++ .../.kokoro/samples/python3.6/periodic.cfg | 9 ++++++++- .../.kokoro/samples/python3.7/common.cfg | 15 +++++++++++++-- .../.kokoro/samples/python3.7/periodic.cfg | 9 ++++++++- .../.kokoro/samples/python3.8/common.cfg | 15 +++++++++++++-- .../.kokoro/samples/python3.8/periodic-head.cfg | 7 +++++++ .../.kokoro/samples/python3.8/periodic.cfg | 9 ++++++++- .../.kokoro/samples/python3.9/common.cfg | 15 +++++++++++++-- .../.kokoro/samples/python3.9/periodic-head.cfg | 7 +++++++ .../.kokoro/samples/python3.9/periodic.cfg | 9 ++++++++- .../.kokoro/test-samples-against-head.sh | 2 -- .../python_library/.kokoro/test-samples.sh | 2 -- .../gcp/templates/python_library/.trampolinerc | 17 ++++++++++++++--- 16 files changed, 136 insertions(+), 21 deletions(-) diff --git a/synthtool/gcp/common.py b/synthtool/gcp/common.py index 1f54ad3f4..7d8b98b4a 100644 --- a/synthtool/gcp/common.py +++ b/synthtool/gcp/common.py @@ -237,6 +237,10 @@ def py_library(self, **kwargs) -> Path: if "samples" not in kwargs: self.excludes += ["samples/AUTHORING_GUIDE.md", "samples/CONTRIBUTING.md"] + # Assume the python-docs-samples Dockerfile is used for samples by default + if "custom_samples_dockerfile" not in kwargs: + kwargs["custom_samples_dockerfile"] = False + ret = self._generic_library("python_library", **kwargs) # If split_system_tests is set to True, we disable the system diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/lint/common.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/lint/common.cfg index 5d9c326e8..4fc0eb9ec 100644 --- a/synthtool/gcp/templates/python_library/.kokoro/samples/lint/common.cfg +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/lint/common.cfg @@ -17,13 +17,24 @@ env_vars: { key: "TRAMPOLINE_BUILD_FILE" value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples.sh" } +{% if custom_samples_dockerfile %} +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/{{ metadata['repo']['repo'].split('/')[1] }}-samples-docker" +} +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_DOCKERFILE" + value: ".kokoro/docker/samples/Dockerfile" +} +{% else %} # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" } - +{% endif %} # Download secrets for samples gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" @@ -31,4 +42,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/common.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/common.cfg index 528154d36..555b92cf8 100644 --- a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/common.cfg +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/common.cfg @@ -23,13 +23,24 @@ env_vars: { key: "TRAMPOLINE_BUILD_FILE" value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples.sh" } +{% if custom_samples_dockerfile %} +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/{{ metadata['repo']['repo'].split('/')[1] }}-samples-docker" +} +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_DOCKERFILE" + value: ".kokoro/docker/samples/Dockerfile" +} +{% else %} # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" } - +{% endif %} # Download secrets for samples gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" @@ -37,4 +48,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/periodic-head.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/periodic-head.cfg index 8e181c510..da31f1cdf 100644 --- a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/periodic-head.cfg +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/periodic-head.cfg @@ -9,3 +9,10 @@ env_vars: { key: "TRAMPOLINE_BUILD_FILE" value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples-against-head.sh" } +{% if custom_samples_dockerfile %} +# Upload the docker image after successful builds. +env_vars: { + key: "TRAMPOLINE_IMAGE_UPLOAD" + value: "true" +} +{% endif %} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/periodic.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/periodic.cfg index 50fec9649..f7ea9d9b9 100644 --- a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/periodic.cfg +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.6/periodic.cfg @@ -3,4 +3,11 @@ env_vars: { key: "INSTALL_LIBRARY_FROM_SOURCE" value: "False" -} \ No newline at end of file +} +{% if custom_samples_dockerfile %} +# Upload the docker image after successful builds. +env_vars: { + key: "TRAMPOLINE_IMAGE_UPLOAD" + value: "true" +} +{% endif %} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/common.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/common.cfg index 6508144d1..c6690823a 100644 --- a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/common.cfg +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/common.cfg @@ -23,13 +23,24 @@ env_vars: { key: "TRAMPOLINE_BUILD_FILE" value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples.sh" } +{% if custom_samples_dockerfile %} +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/{{ metadata['repo']['repo'].split('/')[1] }}-samples-docker" +} +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_DOCKERFILE" + value: ".kokoro/docker/samples/Dockerfile" +} +{% else %} # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" } - +{% endif %} # Download secrets for samples gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" @@ -37,4 +48,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/periodic.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/periodic.cfg index 50fec9649..f7ea9d9b9 100644 --- a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/periodic.cfg +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.7/periodic.cfg @@ -3,4 +3,11 @@ env_vars: { key: "INSTALL_LIBRARY_FROM_SOURCE" value: "False" -} \ No newline at end of file +} +{% if custom_samples_dockerfile %} +# Upload the docker image after successful builds. +env_vars: { + key: "TRAMPOLINE_IMAGE_UPLOAD" + value: "true" +} +{% endif %} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/common.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/common.cfg index 7b1a70079..246db2fb9 100644 --- a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/common.cfg +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/common.cfg @@ -23,13 +23,24 @@ env_vars: { key: "TRAMPOLINE_BUILD_FILE" value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples.sh" } +{% if custom_samples_dockerfile %} +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/{{ metadata['repo']['repo'].split('/')[1] }}-samples-docker" +} +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_DOCKERFILE" + value: ".kokoro/docker/samples/Dockerfile" +} +{% else %} # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" } - +{% endif %} # Download secrets for samples gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" @@ -37,4 +48,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/periodic-head.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/periodic-head.cfg index 8e181c510..da31f1cdf 100644 --- a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/periodic-head.cfg +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/periodic-head.cfg @@ -9,3 +9,10 @@ env_vars: { key: "TRAMPOLINE_BUILD_FILE" value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples-against-head.sh" } +{% if custom_samples_dockerfile %} +# Upload the docker image after successful builds. +env_vars: { + key: "TRAMPOLINE_IMAGE_UPLOAD" + value: "true" +} +{% endif %} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/periodic.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/periodic.cfg index 50fec9649..f7ea9d9b9 100644 --- a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/periodic.cfg +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.8/periodic.cfg @@ -3,4 +3,11 @@ env_vars: { key: "INSTALL_LIBRARY_FROM_SOURCE" value: "False" -} \ No newline at end of file +} +{% if custom_samples_dockerfile %} +# Upload the docker image after successful builds. +env_vars: { + key: "TRAMPOLINE_IMAGE_UPLOAD" + value: "true" +} +{% endif %} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.9/common.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.9/common.cfg index c0ea19f29..aff2d9412 100644 --- a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.9/common.cfg +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.9/common.cfg @@ -23,13 +23,24 @@ env_vars: { key: "TRAMPOLINE_BUILD_FILE" value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples.sh" } +{% if custom_samples_dockerfile %} +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/{{ metadata['repo']['repo'].split('/')[1] }}-samples-docker" +} +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_DOCKERFILE" + value: ".kokoro/docker/samples/Dockerfile" +} +{% else %} # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" } - +{% endif %} # Download secrets for samples gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" @@ -37,4 +48,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.9/periodic-head.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.9/periodic-head.cfg index 8e181c510..da31f1cdf 100644 --- a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.9/periodic-head.cfg +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.9/periodic-head.cfg @@ -9,3 +9,10 @@ env_vars: { key: "TRAMPOLINE_BUILD_FILE" value: "github/{{ metadata['repo']['repo'].split('/')[1] }}/.kokoro/test-samples-against-head.sh" } +{% if custom_samples_dockerfile %} +# Upload the docker image after successful builds. +env_vars: { + key: "TRAMPOLINE_IMAGE_UPLOAD" + value: "true" +} +{% endif %} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.9/periodic.cfg b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.9/periodic.cfg index 50fec9649..f7ea9d9b9 100644 --- a/synthtool/gcp/templates/python_library/.kokoro/samples/python3.9/periodic.cfg +++ b/synthtool/gcp/templates/python_library/.kokoro/samples/python3.9/periodic.cfg @@ -3,4 +3,11 @@ env_vars: { key: "INSTALL_LIBRARY_FROM_SOURCE" value: "False" -} \ No newline at end of file +} +{% if custom_samples_dockerfile %} +# Upload the docker image after successful builds. +env_vars: { + key: "TRAMPOLINE_IMAGE_UPLOAD" + value: "true" +} +{% endif %} \ No newline at end of file diff --git a/synthtool/gcp/templates/python_library/.kokoro/test-samples-against-head.sh b/synthtool/gcp/templates/python_library/.kokoro/test-samples-against-head.sh index 3257bec69..ba3a707b0 100755 --- a/synthtool/gcp/templates/python_library/.kokoro/test-samples-against-head.sh +++ b/synthtool/gcp/templates/python_library/.kokoro/test-samples-against-head.sh @@ -23,6 +23,4 @@ set -eo pipefail # Enables `**` to include files nested inside sub-folders shopt -s globstar -cd github/{{ metadata['repo']['repo'].split('/')[1] }} - exec .kokoro/test-samples-impl.sh diff --git a/synthtool/gcp/templates/python_library/.kokoro/test-samples.sh b/synthtool/gcp/templates/python_library/.kokoro/test-samples.sh index 1cfacce49..11c042d34 100755 --- a/synthtool/gcp/templates/python_library/.kokoro/test-samples.sh +++ b/synthtool/gcp/templates/python_library/.kokoro/test-samples.sh @@ -24,8 +24,6 @@ set -eo pipefail # Enables `**` to include files nested inside sub-folders shopt -s globstar -cd github/{{ metadata['repo']['repo'].split('/')[1] }} - # Run periodic samples tests at latest release if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then # preserving the test runner implementation. diff --git a/synthtool/gcp/templates/python_library/.trampolinerc b/synthtool/gcp/templates/python_library/.trampolinerc index 383b6ec89..0eee72ab6 100644 --- a/synthtool/gcp/templates/python_library/.trampolinerc +++ b/synthtool/gcp/templates/python_library/.trampolinerc @@ -16,15 +16,26 @@ # Add required env vars here. required_envvars+=( - "STAGING_BUCKET" - "V2_STAGING_BUCKET" ) # Add env vars which are passed down into the container here. pass_down_envvars+=( + "NOX_SESSION" + ############### + # Docs builds + ############### "STAGING_BUCKET" "V2_STAGING_BUCKET" - "NOX_SESSION" + ################## + # Samples builds + ################## + "INSTALL_LIBRARY_FROM_SOURCE" + "RUN_TESTS_SESSION" + "BUILD_SPECIFIC_GCLOUD_PROJECT" + # Target directories. + "RUN_TESTS_DIRS" + # The nox session to run. + "RUN_TESTS_SESSION" ) # Prevent unintentional override on the default image.