From 47701bc9d5ff6c5157e0a99b365e774776eab504 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Fri, 22 May 2020 03:10:03 +0000
Subject: [PATCH 01/38] testing: prototype trampoline_v2

---
 .kokoro/tests/run_tests.sh |  21 ++-
 .kokoro/trampoline_v2.sh   | 284 +++++++++++++++++++++++++++++++++++++
 .trampolinerc              |  40 ++++++
 3 files changed, 339 insertions(+), 6 deletions(-)
 create mode 100755 .kokoro/trampoline_v2.sh
 create mode 100644 .trampolinerc

diff --git a/.kokoro/tests/run_tests.sh b/.kokoro/tests/run_tests.sh
index a4200e43f24..7a194ea59d6 100755
--- a/.kokoro/tests/run_tests.sh
+++ b/.kokoro/tests/run_tests.sh
@@ -1,5 +1,4 @@
 #!/bin/bash
-
 # Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -32,10 +31,17 @@ if [[ $* == *--only-diff-head* ]]; then
     DIFF_FROM="HEAD~.."
 fi
 
-cd github/python-docs-samples
+if [[ -z "${PROJECT_ROOT:-}" ]]; then
+    PROJECT_ROOT="github/python-docs-samples"
+fi
+
+cd "${PROJECT_ROOT}"
+
+# add user's pip binary path to PATH
+export PATH="${HOME}/.local/bin:${PATH}"
 
 # install nox for testing
-pip install -q nox
+pip install --user -q nox
 
 # Use secrets acessor service account to get secrets
 if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then
@@ -63,9 +69,12 @@ export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json
 export DATALABELING_ENDPOINT="test-datalabeling.sandbox.googleapis.com:443"
 
 # Run Cloud SQL proxy (background process exit when script does)
-wget --quiet https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O cloud_sql_proxy && chmod +x cloud_sql_proxy
-./cloud_sql_proxy -instances="${MYSQL_INSTANCE}"=tcp:3306 &>> cloud_sql_proxy.log &
-./cloud_sql_proxy -instances="${POSTGRES_INSTANCE}"=tcp:5432 &>> cloud_sql_proxy.log &
+wget --quiet https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 \
+     -O ${HOME}/cloud_sql_proxy && chmod +x ${HOME}/cloud_sql_proxy
+${HOME}/cloud_sql_proxy -instances="${MYSQL_INSTANCE}"=tcp:3306 &>> \
+       ${HOME}/cloud_sql_proxy.log &
+${HOME}/cloud_sql_proxy -instances="${POSTGRES_INSTANCE}"=tcp:5432 &>> \
+       ${HOME}/cloud_sql_proxy.log &
 echo -e "\nCloud SQL proxy started."
 
 echo -e "\n******************** TESTING PROJECTS ********************"
diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
new file mode 100755
index 00000000000..72d7cfc81cf
--- /dev/null
+++ b/.kokoro/trampoline_v2.sh
@@ -0,0 +1,284 @@
+#!/usr/bin/env bash
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# trampoline_v2.sh
+#
+# This script does 3 things.
+#
+# 1. Prepare the Docker image for the test
+# 2. Run the Docker with appropriate flags to run the test
+# 3. Upload the newly built Docker image
+#
+# in a way that is somewhat compatible with trampoline_v1.
+#
+# To run this script, first download few files from gcs to /dev/shm.
+# (/dev/shm is passed into the container as KOKORO_GFILE_DIR).
+#
+# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/secrets_viewer_service_account.json /dev/shm
+# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/automl_secrets.txt /dev/shm
+#
+# Then run the script.
+# .kokoro/trampoline_v2.sh
+#
+# You can optionally change these environment variables:
+#
+# TRAMPOLINE_IMAGE: The docker image to use.
+# TRAMPOLINE_IMAGE_SOURCE: The location of the Dockerfile.
+# RUN_TESTS_SESSION: The nox session to run.
+# TRAMPOLINE_BUILD_FILE: The script to run in the docker container.
+
+set -euo pipefail
+
+if command -v tput >/dev/null && [[ -n "${TERM:-}" ]]; then
+  readonly IO_COLOR_RED="$(tput setaf 1)"
+  readonly IO_COLOR_GREEN="$(tput setaf 2)"
+  readonly IO_COLOR_YELLOW="$(tput setaf 3)"
+  readonly IO_COLOR_RESET="$(tput sgr0)"
+else
+  readonly IO_COLOR_RED=""
+  readonly IO_COLOR_GREEN=""
+  readonly IO_COLOR_YELLOW=""
+  readonly IO_COLOR_RESET=""
+fi
+
+# Logs a message using the given color. The first argument must be one of the
+# IO_COLOR_* variables defined above, such as "${IO_COLOR_YELLOW}". The
+# remaining arguments will be logged in the given color. The log message will
+# also have an RFC-3339 timestamp prepended (in UTC).
+function log_impl() {
+    local color="$1"
+    shift
+    local timestamp="$(date -u "+%Y-%m-%dT%H:%M:%SZ")"
+    echo "================================================================"
+    echo "${color}${timestamp}:" "$@" "${IO_COLOR_RESET}"
+    echo "================================================================"
+}
+
+# Logs the given message with normal coloring and a timestamp.
+function log() {
+  log_impl "${IO_COLOR_RESET}" "$@"
+}
+
+# Logs the given message in green with a timestamp.
+function log_green() {
+  log_impl "${IO_COLOR_GREEN}" "$@"
+}
+
+# Logs the given message in yellow with a timestamp.
+function log_yellow() {
+  log_impl "${IO_COLOR_YELLOW}" "$@"
+}
+
+# Logs the given message in red with a timestamp.
+function log_red() {
+  log_impl "${IO_COLOR_RED}" "$@"
+}
+
+function repo_root() {
+    local dir="$1"
+    while [[ ! -d "${dir}/.git" ]]; do
+	dir="$(dirname "$dir")"
+    done
+    echo "${dir}"
+}
+
+readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX)
+readonly tmphome="${tmpdir}/h"
+mkdir -p "${tmphome}"
+
+PROGRAM_PATH="$(realpath "$0")"
+PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")"
+PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")"
+
+RUNNING_IN_CI="false"
+
+if [[ -n "${KOKORO_GFILE_DIR:-}" ]]; then
+    # descriptive env var for indicating it's on CI.
+    RUNNING_IN_CI="true"
+
+    # Now we're re-using the trampoline service account.
+    # Potentially we can pass down this key into Docker for
+    # bootstrapping secret.
+    SERVICE_ACCOUNT_KEY_FILE="${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json"
+
+    mkdir -p "${tmpdir}/gcloud"
+    gcloud_config_dir="${tmpdir}/gcloud"
+
+    log "Using isolated gcloud config: ${gcloud_config_dir}."
+    export CLOUDSDK_CONFIG="${gcloud_config_dir}"
+
+    log "Using ${SERVICE_ACCOUNT_KEY_FILE} for authentication."
+    gcloud auth activate-service-account \
+	   --key-file "${SERVICE_ACCOUNT_KEY_FILE}"
+    gcloud auth configure-docker --quiet
+fi
+
+log_yellow "Changing to the project root: ${PROJECT_ROOT}."
+cd "${PROJECT_ROOT}"
+
+required_envvars=(
+    # The basic trampoline configurations.
+    "TRAMPOLINE_IMAGE"
+    "TRAMPOLINE_BUILD_FILE"
+)
+
+pass_down_envvars=(
+    # Default empty list.
+)
+
+if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then
+    source "${PROJECT_ROOT}/.trampolinerc"
+fi
+
+log_yellow "Checking environment variables."
+for e in "${required_envvars[@]}"
+do
+    if [[ -z "${!e:-}" ]]; then
+	log "Missing ${e} env var. Aborting."
+	exit 1
+    fi
+done
+
+log_yellow "Preparing Docker image."
+# Download the docker image specified by `TRAMPOLINE_IMAGE`
+
+set +e  # ignore error on docker operations
+# We may want to add --max-concurrent-downloads flag.
+
+log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}."
+if docker pull "${TRAMPOLINE_IMAGE}"; then
+    log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}."
+    has_cache="true"
+else
+    log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}."
+    has_cache="false"
+fi
+
+
+update_cache="false"
+if [[ -n "${TRAMPOLINE_IMAGE_SOURCE:-}" ]]; then
+    # Build the Docker image from the source.
+    context_dir=$(dirname "${TRAMPOLINE_IMAGE_SOURCE}")
+    docker_build_flags=(
+	"-f" "${TRAMPOLINE_IMAGE_SOURCE}"
+	"-t" "${TRAMPOLINE_IMAGE}"
+    )
+    if [[ "${has_cache}" == "true" ]]; then
+	docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}")
+    fi
+
+    log_yellow "Start building the docker image."
+    if docker build "${docker_build_flags[@]}" "${context_dir}"; then
+	log_green "Finished building the docker image."
+	update_cache="true"
+    else
+	log_red "Failed to build the Docker image. Aborting."
+	exit 1
+    fi
+else
+    if [[ "${has_cache}" != "true" ]]; then
+	log_red "failed to download the image ${TRAMPOLINE_IMAGE}, aborting."
+	exit 1
+    fi
+fi
+
+# The default user for a Docker container has uid 0 (root). To avoid
+# creating root-owned files in the build directory we tell docker to
+# use the current user ID.
+docker_uid="$(id -u)"
+docker_gid="$(id -g)"
+docker_user="$(id -un)"
+
+# We use an array for the flags so they are easier to document.
+docker_flags=(
+    # Remove the container after it exists.
+    "--rm"
+
+    # Use the host network.
+    "--network=host"
+
+    # Run in priviledged mode. We are not using docker for sandboxing or
+    # isolation, just for packaging our dev tools.
+    "--privileged"
+
+    # Pass down the KOKORO_GFILE_DIR
+    "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/gfile"
+    "--env" "KOKORO_GFILE_DIR=/gfile"
+
+    # Tells scripts whether they are running as part of CI or not.
+    "--env" "RUNNING_IN_CI=${RUNNING_IN_CI:-no}"
+
+    # Run the docker script and this user id. Because the docker image gets to
+    # write in ${PWD} you typically want this to be your user id.
+    "--user" "${docker_uid}:${docker_gid}"
+
+    # Pass down the USER.
+    "--env" "USER=${docker_user}"
+
+    # Mount the project directory inside the Docker container.
+    "--volume" "${PWD}:/v"
+    "--workdir" "/v"
+    "--env" "PROJECT_ROOT=/v"
+
+    # Mount the temporary home directory.
+    "--volume" "${tmphome}:/h"
+    "--env" "HOME=/h"
+)
+
+# Add an option for nicer output if the build gets a tty.
+if [[ -t 0 ]]; then
+    docker_flags+=("-it")
+fi
+
+# Passing down env vars
+for e in "${pass_down_envvars[@]}"
+do
+    docker_flags+=("--env" "${e}=${!e}")
+done
+
+# If arguments are given, all arguments will become the commands run
+# in the container, otherwise run TRAMPOLINE_BUILD_FILE.
+
+if [[ $# -ge 1 ]]; then
+    log_yellow "Running the given commands '" "${@:1}" "' in the container."
+    readonly commands=("${@:1}")
+else
+    log_yellow "Running the tests in a Docker container."
+    readonly commands=("/v/${TRAMPOLINE_BUILD_FILE}")
+fi
+
+echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}"
+docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}"
+
+test_retval=$?
+
+if [[ ${test_retval} -eq 0 ]]; then
+    log_green "Build finished with ${test_retval}"
+else
+    log_red "Build finished with ${test_retval}"
+fi
+
+if [[ "${RUNNING_IN_CI}" == "true" ]] && \
+       [[ "${update_cache}" == "true" ]] && \
+       [[ -z "${KOKORO_GITHUB_PULL_REQUEST_NUMBER:-}" ]] && \
+       [[ $test_retval == 0 ]]; then
+    # Only upload it when the test passes.
+    log_yellow "Uploading the Docker image."
+    if docker push "${TRAMPOLINE_IMAGE}"; then
+	log_green "Finished uploading the Docker image."
+    else
+	log_red "Failed uploading the Docker image."
+    fi
+fi
diff --git a/.trampolinerc b/.trampolinerc
new file mode 100644
index 00000000000..ece1b3ee26f
--- /dev/null
+++ b/.trampolinerc
@@ -0,0 +1,40 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+required_envvars+=(
+    # The nox session to run.
+    "RUN_TESTS_SESSION"
+)
+
+pass_down_envvars+=(
+    "RUN_TESTS_SESSION"
+)
+
+# Set default values. You can override them with env vars.
+if [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then
+    # Set default image.
+    TRAMPOLINE_IMAGE="gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker"
+fi
+
+if [[ -z "${TRAMPOLINE_IMAGE_SOURCE:-}" ]]; then
+    TRAMPOLINE_IMAGE_SOURCE=".kokoro/docker/Dockerfile"
+fi
+
+if [[ -z "${RUN_TESTS_SESSION:-}" ]]; then
+    RUN_TESTS_SESSION="py-3.7"
+fi
+
+if [[ -z "${TRAMPOLINE_BUILD_FILE:-}" ]]; then
+   TRAMPOLINE_BUILD_FILE=".kokoro/tests/run_tests_diff_head.sh"
+fi

From 5b858a31f5a38ddaca77aefa415d8595cba3a57b Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Fri, 22 May 2020 06:47:24 +0000
Subject: [PATCH 02/38] add RUN_TESTS_DIRS envvar

---
 .kokoro/tests/run_tests.sh | 24 +++++++++++++++++++++++-
 .kokoro/trampoline_v2.sh   |  7 +++++--
 .trampolinerc              |  4 ++++
 3 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/.kokoro/tests/run_tests.sh b/.kokoro/tests/run_tests.sh
index 7a194ea59d6..8d032a22186 100755
--- a/.kokoro/tests/run_tests.sh
+++ b/.kokoro/tests/run_tests.sh
@@ -101,8 +101,27 @@ for file in **/requirements.txt; do
     file=$(dirname "$file")
     cd "$file"
 
+    # First we look up the environment variable `RUN_TESTS_DIRS`. If
+    # the value is set, we'll iterate through the colon separated
+    # directory list.
+    if [[ -n "${RUN_TESTS_DIRS:-}" ]]; then
+	IFS=":"
+	read -ra run_tests_dirs <<< "${RUN_TESTS_DIRS}"
+	match=0
+	for d in "${run_tests_dirs[@]}"; do
+	    # If the current dir starts with one of the
+	    # RUN_TESTS_DIRS, we should run the tests.
+	    if [[ "${file}" = "${d}"* ]]; then
+		match=1
+		break
+	    fi
+	done
+	IFS=" "
+	if [[ $match -eq 0 ]]; then
+	    continue
+	fi
     # If $DIFF_FROM is set, use it to check for changes in this directory.
-    if [[ -n "${DIFF_FROM:-}" ]] && [[ "${test_all}" == "false" ]]; then
+    elif [[ -n "${DIFF_FROM:-}" ]] && [[ "${test_all}" == "false" ]]; then
         git diff --quiet "$DIFF_FROM" .
         CHANGED=$?
         if [[ "$CHANGED" -eq 0 ]]; then
@@ -146,6 +165,9 @@ for file in **/requirements.txt; do
       echo -e "\n Testing completed.\n"
     fi
 
+    # Remove noxfile.py if it's not tracked by git.
+    git ls-files --error-unmatch noxfile.py > /dev/null 2>&1 | rm noxfile.py
+
 done
 cd "$ROOT"
 
diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index 72d7cfc81cf..50a347070e3 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -38,6 +38,8 @@
 # TRAMPOLINE_IMAGE_SOURCE: The location of the Dockerfile.
 # RUN_TESTS_SESSION: The nox session to run.
 # TRAMPOLINE_BUILD_FILE: The script to run in the docker container.
+# RUN_TESTS_DIR: A colon separated directory list relative to the
+#                project root.
 
 set -euo pipefail
 
@@ -245,12 +247,13 @@ fi
 # Passing down env vars
 for e in "${pass_down_envvars[@]}"
 do
-    docker_flags+=("--env" "${e}=${!e}")
+    if [[ -n "${!e:-}" ]]; then
+	docker_flags+=("--env" "${e}=${!e}")
+    fi
 done
 
 # If arguments are given, all arguments will become the commands run
 # in the container, otherwise run TRAMPOLINE_BUILD_FILE.
-
 if [[ $# -ge 1 ]]; then
     log_yellow "Running the given commands '" "${@:1}" "' in the container."
     readonly commands=("${@:1}")
diff --git a/.trampolinerc b/.trampolinerc
index ece1b3ee26f..f230691fd52 100644
--- a/.trampolinerc
+++ b/.trampolinerc
@@ -18,6 +18,10 @@ required_envvars+=(
 )
 
 pass_down_envvars+=(
+    "BUILD_SPECIFIC_GCLOUD_PROJECT"
+    "REPORT_TO_BUILD_COP_BOT"
+    # The nox session to run.
+    "RUN_TESTS_DIRS"
     "RUN_TESTS_SESSION"
 )
 

From 1481b8e169c1690df5008e6114c475b804b32273 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Fri, 22 May 2020 07:17:53 +0000
Subject: [PATCH 03/38] add a script for running tests locally

---
 scripts/run_tests_local.sh | 64 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
 create mode 100755 scripts/run_tests_local.sh

diff --git a/scripts/run_tests_local.sh b/scripts/run_tests_local.sh
new file mode 100755
index 00000000000..9ffa3afcd62
--- /dev/null
+++ b/scripts/run_tests_local.sh
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# run_tests_local.sh
+#
+# This script is a helper script for running tests with
+# .kokoro/trampoline_v2.sh.
+# run_tests_local.sh directory (sessions..)
+
+set -euo pipefail
+
+sessions=(
+    "lint"
+    "py-3.6"
+    "py-3.7"
+)
+
+# The only required argument is a directory for running the tests.
+if [[ $# -lt 1 ]]; then
+    echo "Please provide at least one argument."
+    echo "Usage: run_tests_local.sh directory (sessions..)"
+    exit 1
+fi
+
+function repo_root() {
+    local dir="$1"
+    while [[ ! -d "${dir}/.git" ]]; do
+	dir="$(dirname "$dir")"
+    done
+    echo "${dir}"
+}
+
+PROGRAM_PATH="$(realpath "$0")"
+PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")"
+PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")"
+
+directory="$(realpath "$1")"
+relative_dir=${directory#"${PROJECT_ROOT}/"}
+export RUN_TESTS_DIRS="${relative_dir}"
+
+if [[ $# -ge 2 ]]; then
+    sessions=("${@:2}")
+fi
+
+echo "Running tests for directory: ${directory}"
+echo "Sessions: ${sessions[@]}"
+
+for session in "${sessions[@]}"
+do
+    export RUN_TESTS_SESSION="${session}"
+    "${PROJECT_ROOT}/.kokoro/trampoline_v2.sh"
+done

From 24465876cd71814c14d666b25013f32a01f2a088 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Fri, 22 May 2020 07:36:20 +0000
Subject: [PATCH 04/38] disable docker build for run_tests_local.sh

---
 .kokoro/trampoline_v2.sh   | 2 +-
 scripts/run_tests_local.sh | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index 50a347070e3..10083524f2d 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -170,7 +170,7 @@ fi
 
 
 update_cache="false"
-if [[ -n "${TRAMPOLINE_IMAGE_SOURCE:-}" ]]; then
+if [[ "${TRAMPOLINE_IMAGE_SOURCE:-none}" != "none" ]]; then
     # Build the Docker image from the source.
     context_dir=$(dirname "${TRAMPOLINE_IMAGE_SOURCE}")
     docker_build_flags=(
diff --git a/scripts/run_tests_local.sh b/scripts/run_tests_local.sh
index 9ffa3afcd62..ad096173516 100755
--- a/scripts/run_tests_local.sh
+++ b/scripts/run_tests_local.sh
@@ -50,6 +50,10 @@ directory="$(realpath "$1")"
 relative_dir=${directory#"${PROJECT_ROOT}/"}
 export RUN_TESTS_DIRS="${relative_dir}"
 
+if [[ -z "${TRAMPOLINE_IMAGE_SOURCE:-}" ]]; then
+    export TRAMPOLINE_IMAGE_SOURCE="none"
+fi
+
 if [[ $# -ge 2 ]]; then
     sessions=("${@:2}")
 fi

From 6dad1901d07947f92e3e8c1fdb3338304be5db73 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Fri, 22 May 2020 07:49:50 +0000
Subject: [PATCH 05/38] don't rely on git, remove noxfile.py only if we copied
 it

---
 .kokoro/tests/run_tests.sh | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/.kokoro/tests/run_tests.sh b/.kokoro/tests/run_tests.sh
index 8d032a22186..ad347ca0ff0 100755
--- a/.kokoro/tests/run_tests.sh
+++ b/.kokoro/tests/run_tests.sh
@@ -136,13 +136,16 @@ for file in **/requirements.txt; do
 
     # If no local noxfile exists, copy the one from root
     if [[ ! -f "noxfile.py" ]]; then
-      PARENT_DIR=$(cd ../ && pwd)
-      while [[ "$PARENT_DIR" != "$ROOT" && ! -f "$PARENT_DIR/noxfile-template.py" ]];
-      do
-        PARENT_DIR=$(dirname "$PARENT_DIR")
-      done
-      cp "$PARENT_DIR/noxfile-template.py" "./noxfile.py"
-      echo -e "\n Using noxfile-template from parent folder ($PARENT_DIR). \n"
+	PARENT_DIR=$(cd ../ && pwd)
+	while [[ "$PARENT_DIR" != "$ROOT" && ! -f "$PARENT_DIR/noxfile-template.py" ]];
+	do
+            PARENT_DIR=$(dirname "$PARENT_DIR")
+	done
+	cp "$PARENT_DIR/noxfile-template.py" "./noxfile.py"
+	echo -e "\n Using noxfile-template from parent folder ($PARENT_DIR). \n"
+	cleanup_noxfile=1
+    else
+	cleanup_noxfile=0
     fi
 
     # Use nox to execute the tests for the project.
@@ -165,8 +168,10 @@ for file in **/requirements.txt; do
       echo -e "\n Testing completed.\n"
     fi
 
-    # Remove noxfile.py if it's not tracked by git.
-    git ls-files --error-unmatch noxfile.py > /dev/null 2>&1 | rm noxfile.py
+    # Remove noxfile.py if we copied.
+    if [[ $cleanup_noxfile -eq 1 ]]; then
+	rm noxfile.py
+    fi
 
 done
 cd "$ROOT"

From c601780bee07d7e4a0e0f5c2052c0f2cf1c0457b Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Fri, 22 May 2020 07:53:13 +0000
Subject: [PATCH 06/38] fix tab

---
 .kokoro/tests/run_tests.sh | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/.kokoro/tests/run_tests.sh b/.kokoro/tests/run_tests.sh
index ad347ca0ff0..2bf43ad5d80 100755
--- a/.kokoro/tests/run_tests.sh
+++ b/.kokoro/tests/run_tests.sh
@@ -136,16 +136,16 @@ for file in **/requirements.txt; do
 
     # If no local noxfile exists, copy the one from root
     if [[ ! -f "noxfile.py" ]]; then
-	PARENT_DIR=$(cd ../ && pwd)
-	while [[ "$PARENT_DIR" != "$ROOT" && ! -f "$PARENT_DIR/noxfile-template.py" ]];
-	do
+        PARENT_DIR=$(cd ../ && pwd)
+        while [[ "$PARENT_DIR" != "$ROOT" && ! -f "$PARENT_DIR/noxfile-template.py" ]];
+        do
             PARENT_DIR=$(dirname "$PARENT_DIR")
-	done
-	cp "$PARENT_DIR/noxfile-template.py" "./noxfile.py"
-	echo -e "\n Using noxfile-template from parent folder ($PARENT_DIR). \n"
-	cleanup_noxfile=1
+        done
+        cp "$PARENT_DIR/noxfile-template.py" "./noxfile.py"
+        echo -e "\n Using noxfile-template from parent folder ($PARENT_DIR). \n"
+        cleanup_noxfile=1
     else
-	cleanup_noxfile=0
+        cleanup_noxfile=0
     fi
 
     # Use nox to execute the tests for the project.
@@ -170,7 +170,7 @@ for file in **/requirements.txt; do
 
     # Remove noxfile.py if we copied.
     if [[ $cleanup_noxfile -eq 1 ]]; then
-	rm noxfile.py
+        rm noxfile.py
     fi
 
 done

From 343328253ce6aedf4aa7b7599bf3537a68f234da Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Fri, 22 May 2020 08:05:53 +0000
Subject: [PATCH 07/38] remove the tmpdir at the end

---
 .kokoro/trampoline_v2.sh | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index 10083524f2d..65b2e104338 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -88,6 +88,15 @@ function log_red() {
   log_impl "${IO_COLOR_RED}" "$@"
 }
 
+readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX)
+readonly tmphome="${tmpdir}/h"
+mkdir -p "${tmphome}"
+
+function cleanup() {
+    rm -rf "${tmpdir}"
+}
+trap cleanup EXIT
+
 function repo_root() {
     local dir="$1"
     while [[ ! -d "${dir}/.git" ]]; do
@@ -96,10 +105,6 @@ function repo_root() {
     echo "${dir}"
 }
 
-readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX)
-readonly tmphome="${tmpdir}/h"
-mkdir -p "${tmphome}"
-
 PROGRAM_PATH="$(realpath "$0")"
 PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")"
 PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")"

From 65b49addb196fbd8a0ea2e2bada5f696a3156ed2 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Fri, 22 May 2020 18:49:38 +0000
Subject: [PATCH 08/38] add TRAMPOLINE_IMAGE_UPLOAD envvar

also cleanup the docs
---
 .kokoro/trampoline_v2.sh | 29 +++++++++++++++++------------
 .trampolinerc            | 17 +++++++++++++++++
 2 files changed, 34 insertions(+), 12 deletions(-)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index 65b2e104338..b1b84932035 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -36,10 +36,14 @@
 #
 # TRAMPOLINE_IMAGE: The docker image to use.
 # TRAMPOLINE_IMAGE_SOURCE: The location of the Dockerfile.
-# RUN_TESTS_SESSION: The nox session to run.
+# TRAMPOLINE_IMAGE_UPLOAD:
+#     (true|false): Whether to upload the Docker image after the
+#                   successful builds.
 # TRAMPOLINE_BUILD_FILE: The script to run in the docker container.
-# RUN_TESTS_DIR: A colon separated directory list relative to the
-#                project root.
+#
+# Potentially there are some repo specific envvars in .trampolinerc in
+# the project root.
+
 
 set -euo pipefail
 
@@ -55,10 +59,12 @@ else
   readonly IO_COLOR_RESET=""
 fi
 
-# Logs a message using the given color. The first argument must be one of the
-# IO_COLOR_* variables defined above, such as "${IO_COLOR_YELLOW}". The
-# remaining arguments will be logged in the given color. The log message will
-# also have an RFC-3339 timestamp prepended (in UTC).
+# Logs a message using the given color. The first argument must be one
+# of the IO_COLOR_* variables defined above, such as
+# "${IO_COLOR_YELLOW}". The remaining arguments will be logged in the
+# given color. The log message will also have an RFC-3339 timestamp
+# prepended (in UTC). You can disable the color output by setting
+# TERM=vt100.
 function log_impl() {
     local color="$1"
     shift
@@ -278,11 +284,10 @@ else
     log_red "Build finished with ${test_retval}"
 fi
 
-if [[ "${RUNNING_IN_CI}" == "true" ]] && \
-       [[ "${update_cache}" == "true" ]] && \
-       [[ -z "${KOKORO_GITHUB_PULL_REQUEST_NUMBER:-}" ]] && \
-       [[ $test_retval == 0 ]]; then
-    # Only upload it when the test passes.
+# Only upload it when the test passes.
+if [[ "${update_cache}" == "true" ]] && \
+       [[ $test_retval == 0 ]] && \
+       [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]]; then
     log_yellow "Uploading the Docker image."
     if docker push "${TRAMPOLINE_IMAGE}"; then
 	log_green "Finished uploading the Docker image."
diff --git a/.trampolinerc b/.trampolinerc
index f230691fd52..396fe4fcbb3 100644
--- a/.trampolinerc
+++ b/.trampolinerc
@@ -12,6 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Environment variables for python-docs-samples:
+# RUN_TESTS_SESSION: The nox session to run.
+# RUN_TESTS_DIR: A colon separated directory list relative to the
+# project root.
+
 required_envvars+=(
     # The nox session to run.
     "RUN_TESTS_SESSION"
@@ -25,6 +30,14 @@ pass_down_envvars+=(
     "RUN_TESTS_SESSION"
 )
 
+# Prevent unintentional override on the default image.
+
+if [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]] && \
+   [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then
+   echo "Please set TRAMPOLINE_IMAGE if you want to upload the Docker image."
+   exit 1
+fi
+
 # Set default values. You can override them with env vars.
 if [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then
     # Set default image.
@@ -42,3 +55,7 @@ fi
 if [[ -z "${TRAMPOLINE_BUILD_FILE:-}" ]]; then
    TRAMPOLINE_BUILD_FILE=".kokoro/tests/run_tests_diff_head.sh"
 fi
+
+if [[ -z "${TRAMPOLINE_IMAGE_UPLOAD:-}" ]]; then
+    TRAMPOLINE_IMAGE_UPLOAD="false"
+fi

From a93c319ba9c1d6a0c46075be79c6a471c8ef8329 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Fri, 22 May 2020 18:55:40 +0000
Subject: [PATCH 09/38] add example

code cleanup
---
 scripts/run_tests_local.sh | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/scripts/run_tests_local.sh b/scripts/run_tests_local.sh
index ad096173516..86dc7a06007 100755
--- a/scripts/run_tests_local.sh
+++ b/scripts/run_tests_local.sh
@@ -18,10 +18,14 @@
 # This script is a helper script for running tests with
 # .kokoro/trampoline_v2.sh.
 # run_tests_local.sh directory (sessions..)
+#
+# Example for running lint, py-3.6 and py-3.7 for cdn directory:
+# $ cd cdn
+# $ ../scripts/run_tests_local.sh .
 
 set -euo pipefail
 
-sessions=(
+default_sessions=(
     "lint"
     "py-3.6"
     "py-3.7"
@@ -56,6 +60,8 @@ fi
 
 if [[ $# -ge 2 ]]; then
     sessions=("${@:2}")
+else
+    sessions=("${default_sessions[@]}")
 fi
 
 echo "Running tests for directory: ${directory}"

From bc6390a7ff6aaafd00e5a3f3d09fde25688aee64 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Fri, 22 May 2020 19:04:38 +0000
Subject: [PATCH 10/38] add .trampolinerc_tmpl

---
 .trampolinerc      |  3 ++-
 .trampolinerc_tmpl | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 1 deletion(-)
 create mode 100644 .trampolinerc_tmpl

diff --git a/.trampolinerc b/.trampolinerc
index 396fe4fcbb3..60088c23cf2 100644
--- a/.trampolinerc
+++ b/.trampolinerc
@@ -25,8 +25,9 @@ required_envvars+=(
 pass_down_envvars+=(
     "BUILD_SPECIFIC_GCLOUD_PROJECT"
     "REPORT_TO_BUILD_COP_BOT"
-    # The nox session to run.
+    # Target directories.
     "RUN_TESTS_DIRS"
+    # The nox session to run.
     "RUN_TESTS_SESSION"
 )
 
diff --git a/.trampolinerc_tmpl b/.trampolinerc_tmpl
new file mode 100644
index 00000000000..664f3138c07
--- /dev/null
+++ b/.trampolinerc_tmpl
@@ -0,0 +1,46 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Template for .trampolinerc
+
+# Add required env vars here.
+required_envvars+=(
+)
+
+# Add env vars which are passed down into the container here.
+pass_down_envvars+=(
+)
+
+# Prevent unintentional override on the default image.
+if [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]] && \
+   [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then
+   echo "Please set TRAMPOLINE_IMAGE if you want to upload the Docker image."
+   exit 1
+fi
+
+if [[ -z "${TRAMPOLINE_IMAGE_UPLOAD:-}" ]]; then
+    TRAMPOLINE_IMAGE_UPLOAD="false"
+fi
+
+if [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then
+    # TRAMPOLINE_IMAGE="gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker"
+fi
+
+if [[ -z "${TRAMPOLINE_IMAGE_SOURCE:-}" ]]; then
+    # TRAMPOLINE_IMAGE_SOURCE=".kokoro/docker/Dockerfile"
+fi
+
+if [[ -z "${TRAMPOLINE_BUILD_FILE:-}" ]]; then
+    # TRAMPOLINE_BUILD_FILE=".kokoro/build.sh"
+fi

From 13c203153f8cb7c957cf006904116cf1f4970764 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Sat, 23 May 2020 04:57:57 +0000
Subject: [PATCH 11/38] allow docker in docker

---
 .kokoro/docker/Dockerfile | 13 ++++++++++++
 .kokoro/trampoline_v2.sh  | 44 ++++++++++++++++++++++++++-------------
 2 files changed, 42 insertions(+), 15 deletions(-)

diff --git a/.kokoro/docker/Dockerfile b/.kokoro/docker/Dockerfile
index bf8fd7433a3..4087c5002e5 100644
--- a/.kokoro/docker/Dockerfile
+++ b/.kokoro/docker/Dockerfile
@@ -161,4 +161,17 @@ ENV PATH /google-cloud-sdk/bin:$PATH
 # Enable redis-server on boot.
 RUN sudo systemctl enable redis-server.service
 
+# Create a user and allow sudo
+ARG UID=0
+ARG GID=0
+ARG USERNAME=kbuilder
+ARG DOCKER_GID=999
+
+# Allow access docker socker in the host.
+RUN groupadd -g ${DOCKER_GID} "host-docker"
+RUN groupadd -g ${GID} "${USERNAME}"
+RUN useradd -d /h -u ${UID} -g ${GID} ${USERNAME}
+RUN adduser "${USERNAME}" "host-docker"
+RUN echo "${USERNAME} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+
 CMD ["python3.6"]
diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index b1b84932035..b651b288786 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -180,6 +180,17 @@ else
 fi
 
 
+# The default user for a Docker container has uid 0 (root). To avoid
+# creating root-owned files in the build directory we tell docker to
+# use the current user ID.
+user_uid="$(id -u)"
+user_gid="$(id -g)"
+user_name="$(id -un)"
+
+# To allow docker in docker, we add the user to the docker group in
+# the host os.
+docker_gid=$(cut -d: -f3 < <(getent group docker))
+
 update_cache="false"
 if [[ "${TRAMPOLINE_IMAGE_SOURCE:-none}" != "none" ]]; then
     # Build the Docker image from the source.
@@ -187,6 +198,10 @@ if [[ "${TRAMPOLINE_IMAGE_SOURCE:-none}" != "none" ]]; then
     docker_build_flags=(
 	"-f" "${TRAMPOLINE_IMAGE_SOURCE}"
 	"-t" "${TRAMPOLINE_IMAGE}"
+	"--build-arg" "UID=${user_uid}"
+	"--build-arg" "GID=${user_gid}"
+	"--build-arg" "USERNAME=${user_name}"
+	"--build-arg" "DOCKER_GID=${docker_gid}"
     )
     if [[ "${has_cache}" == "true" ]]; then
 	docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}")
@@ -207,13 +222,6 @@ else
     fi
 fi
 
-# The default user for a Docker container has uid 0 (root). To avoid
-# creating root-owned files in the build directory we tell docker to
-# use the current user ID.
-docker_uid="$(id -u)"
-docker_gid="$(id -g)"
-docker_user="$(id -un)"
-
 # We use an array for the flags so they are easier to document.
 docker_flags=(
     # Remove the container after it exists.
@@ -233,21 +241,27 @@ docker_flags=(
     # Tells scripts whether they are running as part of CI or not.
     "--env" "RUNNING_IN_CI=${RUNNING_IN_CI:-no}"
 
-    # Run the docker script and this user id. Because the docker image gets to
+    # Run the docker script with the user id. Because the docker image gets to
     # write in ${PWD} you typically want this to be your user id.
-    "--user" "${docker_uid}:${docker_gid}"
+    # Also to allow docker in docker, we use docker gid on the host.
+    "--user" "${user_uid}:${docker_gid}"
 
     # Pass down the USER.
-    "--env" "USER=${docker_user}"
+    "--env" "USER=${user_name}"
 
-    # Mount the project directory inside the Docker container.
-    "--volume" "${PWD}:/v"
-    "--workdir" "/v"
-    "--env" "PROJECT_ROOT=/v"
+    # Mount the project directory inside the Docker container.  To
+    # allow docker in docker correctly mount the volume, we use the
+    # same path for the volume.
+    "--volume" "${PWD}:${PWD}"
+    "--workdir" "${PWD}"
+    "--env" "PROJECT_ROOT=${PWD}"
 
     # Mount the temporary home directory.
     "--volume" "${tmphome}:/h"
     "--env" "HOME=/h"
+
+    # Allow docker in docker.
+    "--volume" "/var/run/docker.sock:/var/run/docker.sock"
 )
 
 # Add an option for nicer output if the build gets a tty.
@@ -270,7 +284,7 @@ if [[ $# -ge 1 ]]; then
     readonly commands=("${@:1}")
 else
     log_yellow "Running the tests in a Docker container."
-    readonly commands=("/v/${TRAMPOLINE_BUILD_FILE}")
+    readonly commands=("${PWD}/${TRAMPOLINE_BUILD_FILE}")
 fi
 
 echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}"

From c4057c674e6b271e4132c44b5ff88f00e8bd838b Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Sat, 23 May 2020 17:24:25 +0000
Subject: [PATCH 12/38] ignore failure when creating a group

---
 .kokoro/docker/Dockerfile | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/.kokoro/docker/Dockerfile b/.kokoro/docker/Dockerfile
index 4087c5002e5..eafbdae19b2 100644
--- a/.kokoro/docker/Dockerfile
+++ b/.kokoro/docker/Dockerfile
@@ -167,11 +167,19 @@ ARG GID=0
 ARG USERNAME=kbuilder
 ARG DOCKER_GID=999
 
-# Allow access docker socker in the host.
-RUN groupadd -g ${DOCKER_GID} "host-docker"
-RUN groupadd -g ${GID} "${USERNAME}"
-RUN useradd -d /h -u ${UID} -g ${GID} ${USERNAME}
-RUN adduser "${USERNAME}" "host-docker"
+# Add a new user to the container image.
+# To allow access to the docker socket on the host, we use the docker
+# group on the host.
+
+# First make sure the group exists. Ignore a failure in case
+# there is already a group with that id.
+RUN groupadd -g ${DOCKER_GID} "host-docker" | true
+
+# Add a new user with the caller's uid and the docker goup id on the
+# host.
+RUN useradd -d /h -u ${UID} -g ${DOCKER_GID} ${USERNAME}
+
+# Allow nopasswd sudo
 RUN echo "${USERNAME} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
 
 CMD ["python3.6"]

From a7f2b112790b5459c3059f9b8ace1e72cf765351 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Sat, 23 May 2020 17:39:03 +0000
Subject: [PATCH 13/38] use /v for project root

---
 .kokoro/trampoline_v2.sh | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index b651b288786..1c5b648694e 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -252,9 +252,9 @@ docker_flags=(
     # Mount the project directory inside the Docker container.  To
     # allow docker in docker correctly mount the volume, we use the
     # same path for the volume.
-    "--volume" "${PWD}:${PWD}"
-    "--workdir" "${PWD}"
-    "--env" "PROJECT_ROOT=${PWD}"
+    "--volume" "${PWD}:/v"
+    "--workdir" "/v"
+    "--env" "PROJECT_ROOT=/v"
 
     # Mount the temporary home directory.
     "--volume" "${tmphome}:/h"
@@ -262,6 +262,10 @@ docker_flags=(
 
     # Allow docker in docker.
     "--volume" "/var/run/docker.sock:/var/run/docker.sock"
+
+    # Mount the /tmp so that docker in docker can mount the files
+    # there correctly.
+    "--volume" "/tmp:/tmp"
 )
 
 # Add an option for nicer output if the build gets a tty.
@@ -284,7 +288,7 @@ if [[ $# -ge 1 ]]; then
     readonly commands=("${@:1}")
 else
     log_yellow "Running the tests in a Docker container."
-    readonly commands=("${PWD}/${TRAMPOLINE_BUILD_FILE}")
+    readonly commands=("/v/${TRAMPOLINE_BUILD_FILE}")
 fi
 
 echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}"

From 11b92ba814717a6d7c865d65fe58e33b9839ab51 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Sat, 23 May 2020 17:46:36 +0000
Subject: [PATCH 14/38] use $PROJECT_ROOT instead of $PWD

---
 .kokoro/trampoline_v2.sh | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index 1c5b648694e..0b171c76c68 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -249,10 +249,8 @@ docker_flags=(
     # Pass down the USER.
     "--env" "USER=${user_name}"
 
-    # Mount the project directory inside the Docker container.  To
-    # allow docker in docker correctly mount the volume, we use the
-    # same path for the volume.
-    "--volume" "${PWD}:/v"
+    # Mount the project directory inside the Docker container.
+    "--volume" "${PROJECT_ROOT}:/v"
     "--workdir" "/v"
     "--env" "PROJECT_ROOT=/v"
 

From 3bc8822d6c5c56c09fe11d8edecf997cbabd36b3 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Sat, 23 May 2020 18:03:05 +0000
Subject: [PATCH 15/38] temporarily remove the github/python-docs-samples
 prefix

---
 .kokoro/trampoline_v2.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index 0b171c76c68..bd609c5613c 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -286,7 +286,9 @@ if [[ $# -ge 1 ]]; then
     readonly commands=("${@:1}")
 else
     log_yellow "Running the tests in a Docker container."
-    readonly commands=("/v/${TRAMPOLINE_BUILD_FILE}")
+    # Temporary workaround to remove unnecessary prefix.
+    real_build_file=${TRAMPOLINE_BUILD_FILE#"github/python-docs-samples/"}
+    readonly commands=("/v/${real_build_file}")
 fi
 
 echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}"

From fd88e4b5b694b3b981bea71a8a8b4a0be359815d Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Sat, 23 May 2020 21:46:26 +0000
Subject: [PATCH 16/38] also pass down KOKORO_KEYSTORE_DIR

---
 .kokoro/trampoline_v2.sh | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index bd609c5613c..0db518cb6d0 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -234,10 +234,6 @@ docker_flags=(
     # isolation, just for packaging our dev tools.
     "--privileged"
 
-    # Pass down the KOKORO_GFILE_DIR
-    "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/gfile"
-    "--env" "KOKORO_GFILE_DIR=/gfile"
-
     # Tells scripts whether they are running as part of CI or not.
     "--env" "RUNNING_IN_CI=${RUNNING_IN_CI:-no}"
 
@@ -264,8 +260,17 @@ docker_flags=(
     # Mount the /tmp so that docker in docker can mount the files
     # there correctly.
     "--volume" "/tmp:/tmp"
+
+    # Pass down the KOKORO_GFILE_DIR and KOKORO_KEYSTORE_DIR
+    # TODO(tmatsuo): This part is not portable.
+    "--env" "TRAMPOLINE_SECRET_DIR=/secrets"
+    "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/secrets/gfile"
+    "--env" "KOKORO_GFILE_DIR=/secrets/gfile"
+    "--volume" "${KOKORO_KEYSTORE_DIR:-/dev/shm}:/secrets/keystore"
+    "--env" "KOKORO_KEYSTORE_DIR=/secrets/keystore"
 )
 
+
 # Add an option for nicer output if the build gets a tty.
 if [[ -t 0 ]]; then
     docker_flags+=("-it")

From dfb7fcaac007451b51c17980b7d11f30034302e6 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Sat, 23 May 2020 21:51:37 +0000
Subject: [PATCH 17/38] pass down Kokoro dynamic variables

---
 .kokoro/trampoline_v2.sh | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index 0db518cb6d0..2f1d6c8ef20 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -148,7 +148,14 @@ required_envvars=(
 )
 
 pass_down_envvars=(
-    # Default empty list.
+    # KOKORO dynamic variables.
+    "KOKORO_BUILD_NUMBER"
+    "KOKORO_BUILD_ID"
+    "KOKORO_JOB_NAME"
+    "KOKORO_GIT_COMMIT"
+    "KOKORO_GITHUB_COMMIT"
+    "KOKORO_GITHUB_PULL_REQUEST_NUMBER"
+    "KOKORO_GITHUB_PULL_REQUEST_COMMIT"
 )
 
 if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then

From 12c0e583e3e0c5b935848ed909ef45618b922fab Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Sun, 24 May 2020 18:21:32 +0000
Subject: [PATCH 18/38] introduce TRAMPOLINE_CI and TRAMPOLINE_V2 envvar

---
 .kokoro/trampoline_v2.sh | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index 2f1d6c8ef20..515bfc2d5a4 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -116,11 +116,19 @@ PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")"
 PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")"
 
 RUNNING_IN_CI="false"
+TRAMPOLINE_V2="true"
 
-if [[ -n "${KOKORO_GFILE_DIR:-}" ]]; then
+# If it's running on Kokoro, RUNNING_IN_CI will be true and
+# TRAMPOLINE_CI is set to 'kokoro'. Both envvars will be passing down
+# to the container for telling which CI system we're in.
+if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then
     # descriptive env var for indicating it's on CI.
     RUNNING_IN_CI="true"
+    TRAMPOLINE_CI="kokoro"
+fi
 
+# Configure the service account for pulling the docker image.
+if [[ -n "${KOKORO_GFILE_DIR:-}" ]]; then
     # Now we're re-using the trampoline service account.
     # Potentially we can pass down this key into Docker for
     # bootstrapping secret.
@@ -148,6 +156,13 @@ required_envvars=(
 )
 
 pass_down_envvars=(
+    # TRAMPOLINE_V2 variables.
+    # Tells scripts whether they are running as part of CI or not.
+    "RUNNING_IN_CI"
+    # Indicates which CI system we're in.
+    "TRAMPOLINE_CI"
+    # Indicates we're running trampoline_v2.
+    "TRAMPOLINE_V2"
     # KOKORO dynamic variables.
     "KOKORO_BUILD_NUMBER"
     "KOKORO_BUILD_ID"
@@ -241,9 +256,6 @@ docker_flags=(
     # isolation, just for packaging our dev tools.
     "--privileged"
 
-    # Tells scripts whether they are running as part of CI or not.
-    "--env" "RUNNING_IN_CI=${RUNNING_IN_CI:-no}"
-
     # Run the docker script with the user id. Because the docker image gets to
     # write in ${PWD} you typically want this to be your user id.
     # Also to allow docker in docker, we use docker gid on the host.
@@ -267,7 +279,6 @@ docker_flags=(
     # Mount the /tmp so that docker in docker can mount the files
     # there correctly.
     "--volume" "/tmp:/tmp"
-
     # Pass down the KOKORO_GFILE_DIR and KOKORO_KEYSTORE_DIR
     # TODO(tmatsuo): This part is not portable.
     "--env" "TRAMPOLINE_SECRET_DIR=/secrets"
@@ -277,7 +288,6 @@ docker_flags=(
     "--env" "KOKORO_KEYSTORE_DIR=/secrets/keystore"
 )
 
-
 # Add an option for nicer output if the build gets a tty.
 if [[ -t 0 ]]; then
     docker_flags+=("-it")

From a8bf07611895058a3b2eb317ffba92ed226e8c71 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Mon, 25 May 2020 03:21:28 +0000
Subject: [PATCH 19/38] use --entrypoint instead of giving the command

---
 .kokoro/trampoline_v2.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index 515bfc2d5a4..3653686a9dc 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -306,15 +306,17 @@ done
 if [[ $# -ge 1 ]]; then
     log_yellow "Running the given commands '" "${@:1}" "' in the container."
     readonly commands=("${@:1}")
+    echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}"
+    docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}"
 else
     log_yellow "Running the tests in a Docker container."
     # Temporary workaround to remove unnecessary prefix.
     real_build_file=${TRAMPOLINE_BUILD_FILE#"github/python-docs-samples/"}
-    readonly commands=("/v/${real_build_file}")
+    docker_flags+=("--entrypoint=${real_build_file}")
+    echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}"
+    docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}"
 fi
 
-echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}"
-docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}"
 
 test_retval=$?
 

From 5c40d90dc13fded4827b7a37185fce68ffcbab35 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Mon, 25 May 2020 22:06:46 +0000
Subject: [PATCH 20/38] add kbiulder's uid as the default arg

---
 .kokoro/docker/Dockerfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.kokoro/docker/Dockerfile b/.kokoro/docker/Dockerfile
index eafbdae19b2..fcc0b30638c 100644
--- a/.kokoro/docker/Dockerfile
+++ b/.kokoro/docker/Dockerfile
@@ -162,8 +162,9 @@ ENV PATH /google-cloud-sdk/bin:$PATH
 RUN sudo systemctl enable redis-server.service
 
 # Create a user and allow sudo
-ARG UID=0
-ARG GID=0
+
+# kbuilder uid on the default Kokoro image
+ARG UID=1000
 ARG USERNAME=kbuilder
 ARG DOCKER_GID=999
 

From fcdf8afa12a52250f41aa5ff1e2434e579eea98b Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Tue, 26 May 2020 02:06:42 +0000
Subject: [PATCH 21/38] do not show the docker commands by default

---
 .kokoro/trampoline_v2.sh | 11 +++++++++--
 .trampolinerc            |  6 ++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index 3653686a9dc..27dac5dc2b8 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -230,6 +230,9 @@ if [[ "${TRAMPOLINE_IMAGE_SOURCE:-none}" != "none" ]]; then
     fi
 
     log_yellow "Start building the docker image."
+    if [[ "${TRAMPOLINE_SHOW_COMMAND:-false}" == "true" ]]; then
+	echo "docker build" "${docker_build_flags[@]}" "${context_dir}"
+    fi
     if docker build "${docker_build_flags[@]}" "${context_dir}"; then
 	log_green "Finished building the docker image."
 	update_cache="true"
@@ -306,14 +309,18 @@ done
 if [[ $# -ge 1 ]]; then
     log_yellow "Running the given commands '" "${@:1}" "' in the container."
     readonly commands=("${@:1}")
-    echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}"
+    if [[ "${TRAMPOLINE_SHOW_COMMAND:-false}" == "true" ]]; then
+	echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}"
+    fi
     docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}"
 else
     log_yellow "Running the tests in a Docker container."
     # Temporary workaround to remove unnecessary prefix.
     real_build_file=${TRAMPOLINE_BUILD_FILE#"github/python-docs-samples/"}
     docker_flags+=("--entrypoint=${real_build_file}")
-    echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}"
+    if [[ "${TRAMPOLINE_SHOW_COMMAND:-false}" == "true" ]]; then
+	echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}"
+    fi
     docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}"
 fi
 
diff --git a/.trampolinerc b/.trampolinerc
index 60088c23cf2..a790970e55d 100644
--- a/.trampolinerc
+++ b/.trampolinerc
@@ -60,3 +60,9 @@ fi
 if [[ -z "${TRAMPOLINE_IMAGE_UPLOAD:-}" ]]; then
     TRAMPOLINE_IMAGE_UPLOAD="false"
 fi
+
+if [[ -z "${TRAMPOLINE_SHOW_COMMAND:-}" ]]; then
+    # We are sure there's no secrets in the command line in docker
+    # bulid and docker run commands.
+    TRAMPOLINE_SHOW_COMMAND="true"
+fi

From dc426b836dcedf0ab70997c42e09db94a1e2f2ff Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Tue, 26 May 2020 19:06:43 +0000
Subject: [PATCH 22/38] make the mount point configurable

---
 .kokoro/trampoline_v2.sh | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index 27dac5dc2b8..b37c0d466c5 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -40,6 +40,8 @@
 #     (true|false): Whether to upload the Docker image after the
 #                   successful builds.
 # TRAMPOLINE_BUILD_FILE: The script to run in the docker container.
+# TRAMPOLINE_WORKSPACE: The workspace path in the docker container.
+#                       Defaults to /workspace.
 #
 # Potentially there are some repo specific envvars in .trampolinerc in
 # the project root.
@@ -118,6 +120,9 @@ PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")"
 RUNNING_IN_CI="false"
 TRAMPOLINE_V2="true"
 
+# The workspace in the container, defaults to /workspace.
+TRAMPOLINE_WORKSPACE="${TRAMPOLINE_WORKSPACE:-/workspace}"
+
 # If it's running on Kokoro, RUNNING_IN_CI will be true and
 # TRAMPOLINE_CI is set to 'kokoro'. Both envvars will be passing down
 # to the container for telling which CI system we're in.
@@ -137,10 +142,10 @@ if [[ -n "${KOKORO_GFILE_DIR:-}" ]]; then
     mkdir -p "${tmpdir}/gcloud"
     gcloud_config_dir="${tmpdir}/gcloud"
 
-    log "Using isolated gcloud config: ${gcloud_config_dir}."
+    log_yellow "Using isolated gcloud config: ${gcloud_config_dir}."
     export CLOUDSDK_CONFIG="${gcloud_config_dir}"
 
-    log "Using ${SERVICE_ACCOUNT_KEY_FILE} for authentication."
+    log_yellow "Using ${SERVICE_ACCOUNT_KEY_FILE} for authentication."
     gcloud auth activate-service-account \
 	   --key-file "${SERVICE_ACCOUNT_KEY_FILE}"
     gcloud auth configure-docker --quiet
@@ -268,9 +273,9 @@ docker_flags=(
     "--env" "USER=${user_name}"
 
     # Mount the project directory inside the Docker container.
-    "--volume" "${PROJECT_ROOT}:/v"
-    "--workdir" "/v"
-    "--env" "PROJECT_ROOT=/v"
+    "--volume" "${PROJECT_ROOT}:${TRAMPOLINE_WORKSPACE}"
+    "--workdir" "${TRAMPOLINE_WORKSPACE}"
+    "--env" "PROJECT_ROOT=${TRAMPOLINE_WORKSPACE}"
 
     # Mount the temporary home directory.
     "--volume" "${tmphome}:/h"

From 41d5ff860d1ab0e72c647a1877c4c222f1fe77b1 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Tue, 26 May 2020 20:50:01 +0000
Subject: [PATCH 23/38] change TRAMPOLINE_IMAGE_SOURCE to TRAMPOLINE_DOCKERFILE

---
 .kokoro/trampoline_v2.sh   | 8 ++++----
 .trampolinerc              | 4 ++--
 .trampolinerc_tmpl         | 4 ++--
 scripts/run_tests_local.sh | 4 ++--
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index b37c0d466c5..01ee96298e6 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -35,7 +35,7 @@
 # You can optionally change these environment variables:
 #
 # TRAMPOLINE_IMAGE: The docker image to use.
-# TRAMPOLINE_IMAGE_SOURCE: The location of the Dockerfile.
+# TRAMPOLINE_DOCKERFILE: The location of the Dockerfile.
 # TRAMPOLINE_IMAGE_UPLOAD:
 #     (true|false): Whether to upload the Docker image after the
 #                   successful builds.
@@ -219,11 +219,11 @@ user_name="$(id -un)"
 docker_gid=$(cut -d: -f3 < <(getent group docker))
 
 update_cache="false"
-if [[ "${TRAMPOLINE_IMAGE_SOURCE:-none}" != "none" ]]; then
+if [[ "${TRAMPOLINE_DOCKERFILE:-none}" != "none" ]]; then
     # Build the Docker image from the source.
-    context_dir=$(dirname "${TRAMPOLINE_IMAGE_SOURCE}")
+    context_dir=$(dirname "${TRAMPOLINE_DOCKERFILE}")
     docker_build_flags=(
-	"-f" "${TRAMPOLINE_IMAGE_SOURCE}"
+	"-f" "${TRAMPOLINE_DOCKERFILE}"
 	"-t" "${TRAMPOLINE_IMAGE}"
 	"--build-arg" "UID=${user_uid}"
 	"--build-arg" "GID=${user_gid}"
diff --git a/.trampolinerc b/.trampolinerc
index a790970e55d..b40411719b2 100644
--- a/.trampolinerc
+++ b/.trampolinerc
@@ -45,8 +45,8 @@ if [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then
     TRAMPOLINE_IMAGE="gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker"
 fi
 
-if [[ -z "${TRAMPOLINE_IMAGE_SOURCE:-}" ]]; then
-    TRAMPOLINE_IMAGE_SOURCE=".kokoro/docker/Dockerfile"
+if [[ -z "${TRAMPOLINE_DOCKERFILE:-}" ]]; then
+    TRAMPOLINE_DOCKERFILE=".kokoro/docker/Dockerfile"
 fi
 
 if [[ -z "${RUN_TESTS_SESSION:-}" ]]; then
diff --git a/.trampolinerc_tmpl b/.trampolinerc_tmpl
index 664f3138c07..a36cc0575cf 100644
--- a/.trampolinerc_tmpl
+++ b/.trampolinerc_tmpl
@@ -37,8 +37,8 @@ if [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then
     # TRAMPOLINE_IMAGE="gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker"
 fi
 
-if [[ -z "${TRAMPOLINE_IMAGE_SOURCE:-}" ]]; then
-    # TRAMPOLINE_IMAGE_SOURCE=".kokoro/docker/Dockerfile"
+if [[ -z "${TRAMPOLINE_DOCKERFILE:-}" ]]; then
+    # TRAMPOLINE_DOCKERFILE=".kokoro/docker/Dockerfile"
 fi
 
 if [[ -z "${TRAMPOLINE_BUILD_FILE:-}" ]]; then
diff --git a/scripts/run_tests_local.sh b/scripts/run_tests_local.sh
index 86dc7a06007..6189305fbe9 100755
--- a/scripts/run_tests_local.sh
+++ b/scripts/run_tests_local.sh
@@ -54,8 +54,8 @@ directory="$(realpath "$1")"
 relative_dir=${directory#"${PROJECT_ROOT}/"}
 export RUN_TESTS_DIRS="${relative_dir}"
 
-if [[ -z "${TRAMPOLINE_IMAGE_SOURCE:-}" ]]; then
-    export TRAMPOLINE_IMAGE_SOURCE="none"
+if [[ -z "${TRAMPOLINE_DOCKERFILE:-}" ]]; then
+    export TRAMPOLINE_DOCKERFILE="none"
 fi
 
 if [[ $# -ge 2 ]]; then

From 0ec909aee2520c26908739d92da55f81cbe353ef Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Wed, 27 May 2020 00:43:55 +0000
Subject: [PATCH 24/38] remove unnecessary workaround

---
 .kokoro/trampoline_v2.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index 01ee96298e6..71a7aafd8b6 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -320,9 +320,7 @@ if [[ $# -ge 1 ]]; then
     docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}"
 else
     log_yellow "Running the tests in a Docker container."
-    # Temporary workaround to remove unnecessary prefix.
-    real_build_file=${TRAMPOLINE_BUILD_FILE#"github/python-docs-samples/"}
-    docker_flags+=("--entrypoint=${real_build_file}")
+    docker_flags+=("--entrypoint=${TRAMPOLINE_BUILD_FILE}")
     if [[ "${TRAMPOLINE_SHOW_COMMAND:-false}" == "true" ]]; then
 	echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}"
     fi

From 110817a6647187475c9f5f40e3dc524cfe2de0f1 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Sat, 23 May 2020 17:09:43 +0000
Subject: [PATCH 25/38] Trial to run the trampoline_v2 in Kokoro.

---
 .kokoro/lint/common.cfg          |  7 ++++++-
 .kokoro/lint/continuous.cfg      |  2 +-
 .kokoro/lint/periodic.cfg        |  2 +-
 .kokoro/lint/presubmit.cfg       |  2 +-
 .kokoro/python2.7/common.cfg     | 10 +++++++---
 .kokoro/python2.7/continuous.cfg |  2 +-
 .kokoro/python2.7/periodic.cfg   |  4 ++--
 .kokoro/python2.7/presubmit.cfg  |  2 +-
 .kokoro/python3.6/common.cfg     |  7 ++++++-
 .kokoro/python3.6/continuous.cfg |  2 +-
 .kokoro/python3.6/periodic.cfg   |  4 ++--
 .kokoro/python3.6/presubmit.cfg  |  2 +-
 .kokoro/python3.7/common.cfg     |  7 ++++++-
 .kokoro/python3.7/continuous.cfg |  2 +-
 .kokoro/python3.7/periodic.cfg   |  4 ++--
 .kokoro/python3.7/presubmit.cfg  |  3 +--
 .kokoro/python3.8/common.cfg     |  7 ++++++-
 .kokoro/python3.8/continuous.cfg |  2 +-
 .kokoro/python3.8/periodic.cfg   |  4 ++--
 .kokoro/python3.8/presubmit.cfg  |  2 +-
 20 files changed, 50 insertions(+), 27 deletions(-)

diff --git a/.kokoro/lint/common.cfg b/.kokoro/lint/common.cfg
index ba353086364..be8e83b0fea 100644
--- a/.kokoro/lint/common.cfg
+++ b/.kokoro/lint/common.cfg
@@ -24,7 +24,7 @@ env_vars: {
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
 
 # Use the trampoline script to run in docker.
-build_file: "python-docs-samples/.kokoro/trampoline.sh"
+build_file: "python-docs-samples/.kokoro/trampoline_v2.sh"
 
 # Download secrets from Cloud Storage.
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
@@ -41,3 +41,8 @@ env_vars: {
     key: "RUN_TESTS_SESSION"
     value: "lint"
 }
+
+env_vars: {
+    key: "TRAMPOLINE_DOCKERFILE"
+    value: ".kokoro/docker/Dockerfile"
+}
diff --git a/.kokoro/lint/continuous.cfg b/.kokoro/lint/continuous.cfg
index 6068e84c5ca..cfbe29058c8 100644
--- a/.kokoro/lint/continuous.cfg
+++ b/.kokoro/lint/continuous.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_head.sh"
+    value: ".kokoro/tests/run_tests_diff_head.sh"
 }
diff --git a/.kokoro/lint/periodic.cfg b/.kokoro/lint/periodic.cfg
index 90879b0bec7..76a0afa1ee4 100644
--- a/.kokoro/lint/periodic.cfg
+++ b/.kokoro/lint/periodic.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests.sh"
+    value: ".kokoro/tests/run_tests.sh"
 }
diff --git a/.kokoro/lint/presubmit.cfg b/.kokoro/lint/presubmit.cfg
index f876bd804e6..1cf6a4b7382 100644
--- a/.kokoro/lint/presubmit.cfg
+++ b/.kokoro/lint/presubmit.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_master.sh"
+    value: ".kokoro/tests/run_tests_diff_master.sh"
 }
diff --git a/.kokoro/python2.7/common.cfg b/.kokoro/python2.7/common.cfg
index 75c60ed35c9..2841b53acac 100644
--- a/.kokoro/python2.7/common.cfg
+++ b/.kokoro/python2.7/common.cfg
@@ -26,7 +26,7 @@ env_vars: {
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
 
 # Use the trampoline script to run in docker.
-build_file: "python-docs-samples/.kokoro/trampoline.sh"
+build_file: "python-docs-samples/.kokoro/trampoline_v2.sh"
 
 # Download secrets from Cloud Storage.
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
@@ -44,9 +44,13 @@ env_vars: {
     value: "py-2.7"
 }
 
-# Declare build specific Cloud project. It still uses the common one,
-# but we'll update the value once we have more Cloud projects.
+# Declare build specific Cloud project.
 env_vars: {
     key: "BUILD_SPECIFIC_GCLOUD_PROJECT"
     value: "python-docs-samples-tests"
 }
+
+env_vars: {
+    key: "TRAMPOLINE_DOCKERFILE"
+    value: ".kokoro/docker/Dockerfile"
+}
diff --git a/.kokoro/python2.7/continuous.cfg b/.kokoro/python2.7/continuous.cfg
index 6068e84c5ca..cfbe29058c8 100644
--- a/.kokoro/python2.7/continuous.cfg
+++ b/.kokoro/python2.7/continuous.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_head.sh"
+    value: ".kokoro/tests/run_tests_diff_head.sh"
 }
diff --git a/.kokoro/python2.7/periodic.cfg b/.kokoro/python2.7/periodic.cfg
index d94e43dc81a..f962097d226 100644
--- a/.kokoro/python2.7/periodic.cfg
+++ b/.kokoro/python2.7/periodic.cfg
@@ -17,10 +17,10 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests.sh"
+    value: ".kokoro/tests/run_tests.sh"
 }
 
 env_vars: {
     key: "REPORT_TO_BUILD_COP_BOT"
-    value: "True"
+    value: "true"
 }
diff --git a/.kokoro/python2.7/presubmit.cfg b/.kokoro/python2.7/presubmit.cfg
index f876bd804e6..1cf6a4b7382 100644
--- a/.kokoro/python2.7/presubmit.cfg
+++ b/.kokoro/python2.7/presubmit.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_master.sh"
+    value: ".kokoro/tests/run_tests_diff_master.sh"
 }
diff --git a/.kokoro/python3.6/common.cfg b/.kokoro/python3.6/common.cfg
index 217802d4747..64461dc1587 100644
--- a/.kokoro/python3.6/common.cfg
+++ b/.kokoro/python3.6/common.cfg
@@ -26,7 +26,7 @@ env_vars: {
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
 
 # Use the trampoline script to run in docker.
-build_file: "python-docs-samples/.kokoro/trampoline.sh"
+build_file: "python-docs-samples/.kokoro/trampoline_v2.sh"
 
 # Download secrets from Cloud Storage.
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
@@ -50,3 +50,8 @@ env_vars: {
     key: "BUILD_SPECIFIC_GCLOUD_PROJECT"
     value: "python-docs-samples-tests-py36"
 }
+
+env_vars: {
+    key: "TRAMPOLINE_DOCKERFILE"
+    value: ".kokoro/docker/Dockerfile"
+}
diff --git a/.kokoro/python3.6/continuous.cfg b/.kokoro/python3.6/continuous.cfg
index 6068e84c5ca..cfbe29058c8 100644
--- a/.kokoro/python3.6/continuous.cfg
+++ b/.kokoro/python3.6/continuous.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_head.sh"
+    value: ".kokoro/tests/run_tests_diff_head.sh"
 }
diff --git a/.kokoro/python3.6/periodic.cfg b/.kokoro/python3.6/periodic.cfg
index d94e43dc81a..f962097d226 100644
--- a/.kokoro/python3.6/periodic.cfg
+++ b/.kokoro/python3.6/periodic.cfg
@@ -17,10 +17,10 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests.sh"
+    value: ".kokoro/tests/run_tests.sh"
 }
 
 env_vars: {
     key: "REPORT_TO_BUILD_COP_BOT"
-    value: "True"
+    value: "true"
 }
diff --git a/.kokoro/python3.6/presubmit.cfg b/.kokoro/python3.6/presubmit.cfg
index f876bd804e6..1cf6a4b7382 100644
--- a/.kokoro/python3.6/presubmit.cfg
+++ b/.kokoro/python3.6/presubmit.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_master.sh"
+    value: ".kokoro/tests/run_tests_diff_master.sh"
 }
diff --git a/.kokoro/python3.7/common.cfg b/.kokoro/python3.7/common.cfg
index d8c8ae56eed..372695b38ed 100644
--- a/.kokoro/python3.7/common.cfg
+++ b/.kokoro/python3.7/common.cfg
@@ -26,7 +26,7 @@ env_vars: {
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
 
 # Use the trampoline script to run in docker.
-build_file: "python-docs-samples/.kokoro/trampoline.sh"
+build_file: "python-docs-samples/.kokoro/trampoline_v2.sh"
 
 # Download secrets from Cloud Storage.
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
@@ -50,3 +50,8 @@ env_vars: {
     key: "BUILD_SPECIFIC_GCLOUD_PROJECT"
     value: "python-docs-samples-tests-py37"
 }
+
+env_vars: {
+    key: "TRAMPOLINE_DOCKERFILE"
+    value: ".kokoro/docker/Dockerfile"
+}
diff --git a/.kokoro/python3.7/continuous.cfg b/.kokoro/python3.7/continuous.cfg
index 6068e84c5ca..cfbe29058c8 100644
--- a/.kokoro/python3.7/continuous.cfg
+++ b/.kokoro/python3.7/continuous.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_head.sh"
+    value: ".kokoro/tests/run_tests_diff_head.sh"
 }
diff --git a/.kokoro/python3.7/periodic.cfg b/.kokoro/python3.7/periodic.cfg
index d94e43dc81a..f962097d226 100644
--- a/.kokoro/python3.7/periodic.cfg
+++ b/.kokoro/python3.7/periodic.cfg
@@ -17,10 +17,10 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests.sh"
+    value: ".kokoro/tests/run_tests.sh"
 }
 
 env_vars: {
     key: "REPORT_TO_BUILD_COP_BOT"
-    value: "True"
+    value: "true"
 }
diff --git a/.kokoro/python3.7/presubmit.cfg b/.kokoro/python3.7/presubmit.cfg
index f876bd804e6..2196a2e4085 100644
--- a/.kokoro/python3.7/presubmit.cfg
+++ b/.kokoro/python3.7/presubmit.cfg
@@ -14,8 +14,7 @@
 
 # Format: //devtools/kokoro/config/proto/build.proto
 
-# Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_master.sh"
+    value: ".kokoro/tests/run_tests_diff_master.sh"
 }
diff --git a/.kokoro/python3.8/common.cfg b/.kokoro/python3.8/common.cfg
index c6ef0fe0f38..90340e9dfb6 100644
--- a/.kokoro/python3.8/common.cfg
+++ b/.kokoro/python3.8/common.cfg
@@ -26,7 +26,7 @@ env_vars: {
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
 
 # Use the trampoline script to run in docker.
-build_file: "python-docs-samples/.kokoro/trampoline.sh"
+build_file: "python-docs-samples/.kokoro/trampoline_v2.sh"
 
 # Download secrets from Cloud Storage.
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
@@ -50,3 +50,8 @@ env_vars: {
     key: "BUILD_SPECIFIC_GCLOUD_PROJECT"
     value: "python-docs-samples-tests-py38"
 }
+
+env_vars: {
+    key: "TRAMPOLINE_DOCKERFILE"
+    value: ".kokoro/docker/Dockerfile"
+}
diff --git a/.kokoro/python3.8/continuous.cfg b/.kokoro/python3.8/continuous.cfg
index 6068e84c5ca..cfbe29058c8 100644
--- a/.kokoro/python3.8/continuous.cfg
+++ b/.kokoro/python3.8/continuous.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_head.sh"
+    value: ".kokoro/tests/run_tests_diff_head.sh"
 }
diff --git a/.kokoro/python3.8/periodic.cfg b/.kokoro/python3.8/periodic.cfg
index d94e43dc81a..f962097d226 100644
--- a/.kokoro/python3.8/periodic.cfg
+++ b/.kokoro/python3.8/periodic.cfg
@@ -17,10 +17,10 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests.sh"
+    value: ".kokoro/tests/run_tests.sh"
 }
 
 env_vars: {
     key: "REPORT_TO_BUILD_COP_BOT"
-    value: "True"
+    value: "true"
 }
diff --git a/.kokoro/python3.8/presubmit.cfg b/.kokoro/python3.8/presubmit.cfg
index f876bd804e6..1cf6a4b7382 100644
--- a/.kokoro/python3.8/presubmit.cfg
+++ b/.kokoro/python3.8/presubmit.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_master.sh"
+    value: ".kokoro/tests/run_tests_diff_master.sh"
 }

From e9e23a1666d7903f0cc1ac527f59219e9b7dbcce Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Wed, 27 May 2020 17:57:00 +0000
Subject: [PATCH 26/38] summon BCB again

---
 .kokoro/tests/run_tests.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.kokoro/tests/run_tests.sh b/.kokoro/tests/run_tests.sh
index 2bf43ad5d80..11359880d17 100755
--- a/.kokoro/tests/run_tests.sh
+++ b/.kokoro/tests/run_tests.sh
@@ -152,11 +152,11 @@ for file in **/requirements.txt; do
     nox -s "$RUN_TESTS_SESSION"
     EXIT=$?
 
-    # If REPORT_TO_BUILD_COP_BOT is set to "True", send the test log
+    # If REPORT_TO_BUILD_COP_BOT is set to "true", send the test log
     # to the Build Cop Bot.
     # See:
     # https://github.com/googleapis/repo-automation-bots/tree/master/packages/buildcop.
-    if [[ "${REPORT_TO_BUILD_COP_BOT:-}" == "True" ]]; then
+    if [[ "${REPORT_TO_BUILD_COP_BOT:-}" == "true" ]]; then
       chmod +x $KOKORO_GFILE_DIR/linux_amd64/buildcop
       $KOKORO_GFILE_DIR/linux_amd64/buildcop
     fi

From 9cec5d322c8393409d420293b81d1b994b481192 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Wed, 27 May 2020 19:25:13 +0000
Subject: [PATCH 27/38] only decrypt the secret when the key exists

also prevents decrypt-secrets.sh to override existing files.
also stop unnecessary (and harmful) workaround.
---
 .kokoro/tests/run_tests.sh | 16 ++++++----------
 scripts/decrypt-secrets.sh |  8 ++++++++
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/.kokoro/tests/run_tests.sh b/.kokoro/tests/run_tests.sh
index 11359880d17..825403c3483 100755
--- a/.kokoro/tests/run_tests.sh
+++ b/.kokoro/tests/run_tests.sh
@@ -43,19 +43,18 @@ export PATH="${HOME}/.local/bin:${PATH}"
 # install nox for testing
 pip install --user -q nox
 
-# Use secrets acessor service account to get secrets
+# Use secrets acessor service account to get secrets.
 if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then
     gcloud auth activate-service-account \
 	   --key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \
 	   --project="cloud-devrel-kokoro-resources"
+    # This script will create 3 files:
+    # - testing/test-env.sh
+    # - testing/service-account.json
+    # - testing/client-secrets.json
+    ./scripts/decrypt-secrets.sh
 fi
 
-# This script will create 3 files:
-# - testing/test-env.sh
-# - testing/service-account.json
-# - testing/client-secrets.json
-./scripts/decrypt-secrets.sh
-
 source ./testing/test-env.sh
 export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json
 
@@ -176,7 +175,4 @@ for file in **/requirements.txt; do
 done
 cd "$ROOT"
 
-# Workaround for Kokoro permissions issue: delete secrets
-rm testing/{test-env.sh,client-secrets.json,service-account.json}
-
 exit "$RTN"
diff --git a/scripts/decrypt-secrets.sh b/scripts/decrypt-secrets.sh
index c14d23793a0..dc004c091c5 100755
--- a/scripts/decrypt-secrets.sh
+++ b/scripts/decrypt-secrets.sh
@@ -20,6 +20,14 @@ ROOT=$( dirname "$DIR" )
 # Work from the project root.
 cd $ROOT
 
+# Prevent it from overriding files.
+if [[ -f "testing/test-env.sh" ]] || \
+       [[ -f "testing/service-account.json" ]] || \
+       [[ -f "testing/client-secrets.json" ]]; then
+    echo "One or more target files exist, aborting."
+    exit 1
+fi
+
 # Use SECRET_MANAGER_PROJECT if set, fallback to cloud-devrel-kokoro-resources.
 PROJECT_ID="${SECRET_MANAGER_PROJECT:-cloud-devrel-kokoro-resources}"
 

From 8b53a48f9462c017a2acc595f704a28eb29d6a0e Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Wed, 27 May 2020 21:07:24 +0000
Subject: [PATCH 28/38] remove secret if we used decrypt-secrets.sh before the
 test

---
 .kokoro/tests/run_tests.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.kokoro/tests/run_tests.sh b/.kokoro/tests/run_tests.sh
index 825403c3483..dc8ef572985 100755
--- a/.kokoro/tests/run_tests.sh
+++ b/.kokoro/tests/run_tests.sh
@@ -175,4 +175,9 @@ for file in **/requirements.txt; do
 done
 cd "$ROOT"
 
+# Remove secrets if we used decrypt-secrets.sh.
+if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then
+    rm testing/{test-env.sh,client-secrets.json,service-account.json}
+fi
+
 exit "$RTN"

From ecb4b2e31078a9260f17ceab48042f1a89a3dd59 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Wed, 27 May 2020 21:30:55 +0000
Subject: [PATCH 29/38] add 2 more envvars for build cop bot

---
 .kokoro/trampoline_v2.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index 71a7aafd8b6..efeec6f38b9 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -176,6 +176,9 @@ pass_down_envvars=(
     "KOKORO_GITHUB_COMMIT"
     "KOKORO_GITHUB_PULL_REQUEST_NUMBER"
     "KOKORO_GITHUB_PULL_REQUEST_COMMIT"
+    # For Build Cop Bot
+    "KOKORO_GITHUB_COMMIT_URL"
+    "KOKORO_GITHUB_PULL_REQUEST_URL"
 )
 
 if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then

From c755d533f0ab5b08694e26e5941eeb224a82ce4c Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Thu, 28 May 2020 00:02:12 +0000
Subject: [PATCH 30/38] use user's gid

we can add the user to the docker group for accessing docker socket.
---
 .kokoro/trampoline_v2.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index efeec6f38b9..476bf3af92e 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -269,8 +269,7 @@ docker_flags=(
 
     # Run the docker script with the user id. Because the docker image gets to
     # write in ${PWD} you typically want this to be your user id.
-    # Also to allow docker in docker, we use docker gid on the host.
-    "--user" "${user_uid}:${docker_gid}"
+    "--user" "${user_uid}:${user_gid}"
 
     # Pass down the USER.
     "--env" "USER=${user_name}"

From c42e4cc092ea99fc2166e7b41ef15ce960763de1 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Thu, 28 May 2020 00:49:56 +0000
Subject: [PATCH 31/38] setup kokoro service account only on kokoro

---
 .kokoro/trampoline_v2.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index 476bf3af92e..b32eba2fb5c 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -133,7 +133,7 @@ if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then
 fi
 
 # Configure the service account for pulling the docker image.
-if [[ -n "${KOKORO_GFILE_DIR:-}" ]]; then
+if [[ "${TRAMPOLINE_CI:-}" == "kokoro" ]]; then
     # Now we're re-using the trampoline service account.
     # Potentially we can pass down this key into Docker for
     # bootstrapping secret.

From 498514a785651f36a8d679d2baeebb945d3e77ca Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Thu, 28 May 2020 02:33:01 +0000
Subject: [PATCH 32/38] address review comments

---
 .kokoro/tests/run_tests.sh | 33 ++++++++++++++++++++++-----------
 .kokoro/trampoline_v2.sh   | 34 +++++++++++++++++++++-------------
 scripts/run_tests_local.sh |  8 +++++---
 3 files changed, 48 insertions(+), 27 deletions(-)

diff --git a/.kokoro/tests/run_tests.sh b/.kokoro/tests/run_tests.sh
index dc8ef572985..0160a0590f6 100755
--- a/.kokoro/tests/run_tests.sh
+++ b/.kokoro/tests/run_tests.sh
@@ -68,12 +68,12 @@ export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json
 export DATALABELING_ENDPOINT="test-datalabeling.sandbox.googleapis.com:443"
 
 # Run Cloud SQL proxy (background process exit when script does)
-wget --quiet https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 \
-     -O ${HOME}/cloud_sql_proxy && chmod +x ${HOME}/cloud_sql_proxy
-${HOME}/cloud_sql_proxy -instances="${MYSQL_INSTANCE}"=tcp:3306 &>> \
-       ${HOME}/cloud_sql_proxy.log &
-${HOME}/cloud_sql_proxy -instances="${POSTGRES_INSTANCE}"=tcp:5432 &>> \
+sudo wget --quiet https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 \
+     -O /bin/cloud_sql_proxy && sudo chmod +x /bin/cloud_sql_proxy
+cloud_sql_proxy -instances="${MYSQL_INSTANCE}"=tcp:3306 &>> \
        ${HOME}/cloud_sql_proxy.log &
+cloud_sql_proxy -instances="${POSTGRES_INSTANCE}"=tcp:5432 &>> \
+       ${HOME}/cloud_sql_proxy-postgres.log &
 echo -e "\nCloud SQL proxy started."
 
 echo -e "\n******************** TESTING PROJECTS ********************"
@@ -102,12 +102,23 @@ for file in **/requirements.txt; do
 
     # First we look up the environment variable `RUN_TESTS_DIRS`. If
     # the value is set, we'll iterate through the colon separated
-    # directory list.
+    # directory list. If the target directory is not under any
+    # directory in the list, we skip this directory.
+    # This environment variables are primarily for
+    # `scripts/run_tests_local.sh`.
+    #
+    # The value must be a colon separated list of relative paths from
+    # the project root.
+    #
+    # Example:
+    #   cdn:appengine/flexible
+    #     run tests for `cdn` and `appengine/flexible` directories.
+    #   logging/cloud-client
+    #     only run tests for `logging/cloud-client` directory.
+    #
     if [[ -n "${RUN_TESTS_DIRS:-}" ]]; then
-	IFS=":"
-	read -ra run_tests_dirs <<< "${RUN_TESTS_DIRS}"
 	match=0
-	for d in "${run_tests_dirs[@]}"; do
+	for d in $(echo "${RUN_TESTS_DIRS}" | tr ":" "\n"); do
 	    # If the current dir starts with one of the
 	    # RUN_TESTS_DIRS, we should run the tests.
 	    if [[ "${file}" = "${d}"* ]]; then
@@ -115,12 +126,12 @@ for file in **/requirements.txt; do
 		break
 	    fi
 	done
-	IFS=" "
 	if [[ $match -eq 0 ]]; then
 	    continue
 	fi
+    fi
     # If $DIFF_FROM is set, use it to check for changes in this directory.
-    elif [[ -n "${DIFF_FROM:-}" ]] && [[ "${test_all}" == "false" ]]; then
+    if [[ -n "${DIFF_FROM:-}" ]] && [[ "${test_all}" == "false" ]]; then
         git diff --quiet "$DIFF_FROM" .
         CHANGED=$?
         if [[ "$CHANGED" -eq 0 ]]; then
diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index b32eba2fb5c..f4e95d55881 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -32,16 +32,19 @@
 # Then run the script.
 # .kokoro/trampoline_v2.sh
 #
-# You can optionally change these environment variables:
-#
+# These environment variables are required:
 # TRAMPOLINE_IMAGE: The docker image to use.
 # TRAMPOLINE_DOCKERFILE: The location of the Dockerfile.
+#
+# You can optionally change these environment variables:
 # TRAMPOLINE_IMAGE_UPLOAD:
 #     (true|false): Whether to upload the Docker image after the
 #                   successful builds.
 # TRAMPOLINE_BUILD_FILE: The script to run in the docker container.
 # TRAMPOLINE_WORKSPACE: The workspace path in the docker container.
 #                       Defaults to /workspace.
+# TRAMPOLINE_SKIP_DOWNLOAD_IMAGE: Skip downloading the image when you
+#                                 know you have the image locally.
 #
 # Potentially there are some repo specific envvars in .trampolinerc in
 # the project root.
@@ -194,19 +197,24 @@ do
     fi
 done
 
-log_yellow "Preparing Docker image."
-# Download the docker image specified by `TRAMPOLINE_IMAGE`
-
-set +e  # ignore error on docker operations
-# We may want to add --max-concurrent-downloads flag.
-
-log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}."
-if docker pull "${TRAMPOLINE_IMAGE}"; then
-    log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}."
+if [[ "${TRAMPOLINE_SKIP_DOWNLOAD_IMAGE:-false}" == "true" ]]; then
+    log_yellow "Re-using the local Docker image."
     has_cache="true"
 else
-    log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}."
-    has_cache="false"
+    log_yellow "Preparing Docker image."
+    # Download the docker image specified by `TRAMPOLINE_IMAGE`
+
+    set +e  # ignore error on docker operations
+    # We may want to add --max-concurrent-downloads flag.
+
+    log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}."
+    if docker pull "${TRAMPOLINE_IMAGE}"; then
+	log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}."
+	has_cache="true"
+    else
+	log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}."
+	has_cache="false"
+    fi
 fi
 
 
diff --git a/scripts/run_tests_local.sh b/scripts/run_tests_local.sh
index 6189305fbe9..b4e8b79ae39 100755
--- a/scripts/run_tests_local.sh
+++ b/scripts/run_tests_local.sh
@@ -54,9 +54,8 @@ directory="$(realpath "$1")"
 relative_dir=${directory#"${PROJECT_ROOT}/"}
 export RUN_TESTS_DIRS="${relative_dir}"
 
-if [[ -z "${TRAMPOLINE_DOCKERFILE:-}" ]]; then
-    export TRAMPOLINE_DOCKERFILE="none"
-fi
+# We want to test this directory regardless of whether there's a change.
+export TRAMPOLINE_BUILD_FILE=".kokoro/tests/run_tests.sh"
 
 if [[ $# -ge 2 ]]; then
     sessions=("${@:2}")
@@ -71,4 +70,7 @@ for session in "${sessions[@]}"
 do
     export RUN_TESTS_SESSION="${session}"
     "${PROJECT_ROOT}/.kokoro/trampoline_v2.sh"
+    # We can re-use the image after the first iteration.
+    export TRAMPOLINE_SKIP_DOWNLOAD_IMAGE="true"
+    export TRAMPOLINE_DOCKERFILE="none"
 done

From dc7c0129a369bcd380080f98892419b41569d4cf Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Thu, 28 May 2020 22:21:21 +0000
Subject: [PATCH 33/38] correctly report the exit value

---
 .kokoro/trampoline_v2.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index f4e95d55881..69fc04e8801 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -357,3 +357,5 @@ if [[ "${update_cache}" == "true" ]] && \
 	log_red "Failed uploading the Docker image."
     fi
 fi
+
+exit "${test_retval}"

From 84da9149d063fc7cb5d0353123db2dbbfd940f01 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Thu, 28 May 2020 22:21:42 +0000
Subject: [PATCH 34/38] temporarily limit the test to vision/automl

---
 .kokoro/trampoline_v2.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index 69fc04e8801..1df381260a3 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -116,6 +116,9 @@ function repo_root() {
     echo "${dir}"
 }
 
+# Temporarily limit the test to vision/automl
+RUN_TESTS_DIRS="vision/automl"
+
 PROGRAM_PATH="$(realpath "$0")"
 PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")"
 PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")"

From 0b158ddf16f9c934b511502ad1ef716189b0810b Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Thu, 28 May 2020 22:34:20 +0000
Subject: [PATCH 35/38] remove unnecessary build args

---
 .kokoro/docker/Dockerfile | 13 +++----------
 .kokoro/trampoline_v2.sh  |  4 ++--
 2 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/.kokoro/docker/Dockerfile b/.kokoro/docker/Dockerfile
index fcc0b30638c..924f427a530 100644
--- a/.kokoro/docker/Dockerfile
+++ b/.kokoro/docker/Dockerfile
@@ -166,19 +166,12 @@ RUN sudo systemctl enable redis-server.service
 # kbuilder uid on the default Kokoro image
 ARG UID=1000
 ARG USERNAME=kbuilder
-ARG DOCKER_GID=999
 
 # Add a new user to the container image.
-# To allow access to the docker socket on the host, we use the docker
-# group on the host.
+# This is needed for ssh and sudo access.
 
-# First make sure the group exists. Ignore a failure in case
-# there is already a group with that id.
-RUN groupadd -g ${DOCKER_GID} "host-docker" | true
-
-# Add a new user with the caller's uid and the docker goup id on the
-# host.
-RUN useradd -d /h -u ${UID} -g ${DOCKER_GID} ${USERNAME}
+# Add a new user with the caller's uid and the username.
+RUN useradd -d /h -u ${UID} ${USERNAME}
 
 # Allow nopasswd sudo
 RUN echo "${USERNAME} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index 1df381260a3..0a520e75f38 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -240,9 +240,7 @@ if [[ "${TRAMPOLINE_DOCKERFILE:-none}" != "none" ]]; then
 	"-f" "${TRAMPOLINE_DOCKERFILE}"
 	"-t" "${TRAMPOLINE_IMAGE}"
 	"--build-arg" "UID=${user_uid}"
-	"--build-arg" "GID=${user_gid}"
 	"--build-arg" "USERNAME=${user_name}"
-	"--build-arg" "DOCKER_GID=${docker_gid}"
     )
     if [[ "${has_cache}" == "true" ]]; then
 	docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}")
@@ -280,6 +278,8 @@ docker_flags=(
 
     # Run the docker script with the user id. Because the docker image gets to
     # write in ${PWD} you typically want this to be your user id.
+    # To allow docker in docker, we need to use docker gid on the host.
+    # Now we passing the wrong one in order to see the build report failure.
     "--user" "${user_uid}:${user_gid}"
 
     # Pass down the USER.

From 2479339c88caf963e3c22b61d3b760e19ef5f56b Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Thu, 28 May 2020 22:43:16 +0000
Subject: [PATCH 36/38] use docker gid

---
 .kokoro/trampoline_v2.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index 0a520e75f38..e92904fba30 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -279,8 +279,7 @@ docker_flags=(
     # Run the docker script with the user id. Because the docker image gets to
     # write in ${PWD} you typically want this to be your user id.
     # To allow docker in docker, we need to use docker gid on the host.
-    # Now we passing the wrong one in order to see the build report failure.
-    "--user" "${user_uid}:${user_gid}"
+    "--user" "${user_uid}:${docker_gid}"
 
     # Pass down the USER.
     "--env" "USER=${user_name}"

From b13de77d89afb819d1714629642861e93cf095ff Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Thu, 28 May 2020 22:49:20 +0000
Subject: [PATCH 37/38] test everything

---
 .kokoro/trampoline_v2.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index e92904fba30..68916d19307 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -116,9 +116,6 @@ function repo_root() {
     echo "${dir}"
 }
 
-# Temporarily limit the test to vision/automl
-RUN_TESTS_DIRS="vision/automl"
-
 PROGRAM_PATH="$(realpath "$0")"
 PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")"
 PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")"

From 74fd55f050e64746e469db492934768e4ef02e89 Mon Sep 17 00:00:00 2001
From: Takashi Matsuo <tmatsuo@google.com>
Date: Fri, 29 May 2020 21:47:06 +0000
Subject: [PATCH 38/38] download cloud_sql_proxy to ${HOME} again

---
 .kokoro/tests/run_tests.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.kokoro/tests/run_tests.sh b/.kokoro/tests/run_tests.sh
index 0160a0590f6..a868f5de7c9 100755
--- a/.kokoro/tests/run_tests.sh
+++ b/.kokoro/tests/run_tests.sh
@@ -68,11 +68,11 @@ export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json
 export DATALABELING_ENDPOINT="test-datalabeling.sandbox.googleapis.com:443"
 
 # Run Cloud SQL proxy (background process exit when script does)
-sudo wget --quiet https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 \
-     -O /bin/cloud_sql_proxy && sudo chmod +x /bin/cloud_sql_proxy
-cloud_sql_proxy -instances="${MYSQL_INSTANCE}"=tcp:3306 &>> \
+wget --quiet https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 \
+     -O ${HOME}/cloud_sql_proxy && chmod +x ${HOME}/cloud_sql_proxy
+${HOME}/cloud_sql_proxy -instances="${MYSQL_INSTANCE}"=tcp:3306 &>> \
        ${HOME}/cloud_sql_proxy.log &
-cloud_sql_proxy -instances="${POSTGRES_INSTANCE}"=tcp:5432 &>> \
+${HOME}/cloud_sql_proxy -instances="${POSTGRES_INSTANCE}"=tcp:5432 &>> \
        ${HOME}/cloud_sql_proxy-postgres.log &
 echo -e "\nCloud SQL proxy started."