From ffc058a08c523b3811b021ec1bc30201452a36c9 Mon Sep 17 00:00:00 2001
From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com>
Date: Mon, 28 Aug 2023 15:15:11 +0000
Subject: [PATCH] feat: add `generate_library.sh` without post processing
 (#1916)

* feat: add

* generate gapic and proto folder

* refactor utilities

* add an action to verify

* checkout googleapis-gen

* setup repo name

* add commit hash of googleapis-gen

* change secret

* change token

* change to git clone

* change user name

* add input list

* include resources folder in main

* remove grpc version in `*ServiceGrpc.java`

* change destination path

* compare generation result with googleapis-gen

* fix type in diff command

* checkout repo using checkout action

* checkout repos as nested repo

* sparse checkout googleapis

* Revert "sparse checkout googleapis"

This reverts commit 3d612f8e4949251d7d97070f507c31ff4170d85a.

* change library

* change step name

* add a readme

* make grpc version optional

* make protobuf version optional

* checkout master branch, rather than a commit hash

* allow snapshot version of generator

* download snapshot of generator parent pom

* update README

* download generator and grpc using mvn

* change error message

* add maven central mirror

* add comments in utilities

* add comments

* add an integration test

* fail fast if no file is found

* do not delete google/

* get protobuf version from WORKSPACE

* add instructions on download `google/` from googleapis

* add comments

* update description of `destination_path`

* update comments

* download dependencies using `curl`

* increase download time

* remove comment

* add samples directory in readme

* remove prerequisite about `proto_path`

* add explanation in prerequisite

* add example to generate showcase

* add a comment
---
 .../workflows/verify_library_generation.yaml  |  30 +++
 library_generation/README.md                  | 135 +++++++++++
 .../gapic-generator-java-wrapper              |   7 +
 library_generation/generate_library.sh        | 160 +++++++++++++
 .../generate_library_integration_test.sh      |  98 ++++++++
 library_generation/utilities.sh               | 211 ++++++++++++++++++
 6 files changed, 641 insertions(+)
 create mode 100644 .github/workflows/verify_library_generation.yaml
 create mode 100644 library_generation/README.md
 create mode 100755 library_generation/gapic-generator-java-wrapper
 create mode 100755 library_generation/generate_library.sh
 create mode 100755 library_generation/generate_library_integration_test.sh
 create mode 100755 library_generation/utilities.sh

diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml
new file mode 100644
index 0000000000..a90f03208d
--- /dev/null
+++ b/.github/workflows/verify_library_generation.yaml
@@ -0,0 +1,30 @@
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+    paths:
+    - library_generation/**
+
+  workflow_dispatch:
+name: verify_library_generation_against_googleapis-gen
+jobs:
+  verify_library_generation:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        java: [ 8 ]
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-java@v3
+      with:
+        java-version: ${{ matrix.java }}
+        distribution: temurin
+        cache: maven
+    - name: Run integration tests
+      run: |
+        set -x
+        library_generation/generate_library_integration_test.sh \
+        -p google/bigtable/v2 \
+        -d google-cloud-bigtable-v2-java \
+        --googleapis_gen_url https://cloud-java-bot:${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }}@github.com/googleapis/googleapis-gen.git
diff --git a/library_generation/README.md b/library_generation/README.md
new file mode 100644
index 0000000000..e9b4b868bc
--- /dev/null
+++ b/library_generation/README.md
@@ -0,0 +1,135 @@
+# Generate GAPIC Client Library without post-processing
+
+The script, `generate_library.sh`, allows you to generate a GAPIC client library from proto files.
+
+## Environment
+
+Use Linux environment and install java runtime environment (8 or above).
+
+## Prerequisite
+Protos referenced by protos in `proto_path` (see `proto_path` below) should be copied to the current
+working directory (refers as `$cwd`, a directory contains `generate_library.sh`).
+The directory structure should be the same as import statements in protos.
+
+For example, we want to generate from `folder1/folder2/protoA`, so `proto_path` 
+should be set to `folder1/folder2` (a relative path from `$cwd`). 
+protoA imports protoB as `folder3/folder4/protoB`, then there should 
+be `folder3/folder4` (containing protoB) in `$cwd`.
+
+In order to generate a GAPIC library, you need to pull `google/` from [googleapis](https://github.com/googleapis/googleapis)
+and put it into `$cwd` since protos in `google/` are likely referenced by 
+protos from which the library are generated.
+
+## Parameters to run `generate_library.sh`
+
+You need to run the script with the following parameters.
+
+### proto_path
+A directory in `$cwd` and copy proto files into it. 
+The absolute path of `proto_path` is `$cwd/$proto_path`. 
+
+Use `-p` or `--proto_path` to specify the value.
+
+### destination_path 
+A directory within `$cwd`. 
+This is the path in which the generated library will reside. 
+The absolute path of `destination_path` is `$cwd/$destination_path`. 
+
+Use `-d` or `--destination_path` to specify the value.
+   
+Note that you do not need to create `$detination_path` beforehand.
+
+The directory structure of the generated library is
+```
+$destination_path
+  |_gapic-*
+  |    |_src
+  |       |_main
+  |          |_java
+  |          |_resources
+  |       |_test
+  |_grpc-*
+  |    |_src
+  |       |_main
+  |          |_java
+  |    
+  |_proto-*
+  |    |_src
+  |       |_main
+  |          |_java
+  |          |_proto
+  |_samples
+      |_snippets
+          |_generated
+```
+You can't build the library as-is since it does not have `pom.xml` or `build.gradle`.
+To use the library, copy the generated files to the corresponding directory
+of a library repository, e.g., `google-cloud-java`.
+
+### gapic_generator_version
+You can find the released version of gapic-generator-java in [maven central](https://repo1.maven.org/maven2/com/google/api/gapic-generator-java/).
+
+Use `--gapic_generator_version` to specify the value.
+
+Note that you can specify a `SNAPSHOT` version as long as you have build the SNAPSHOT of gapic-generator-java in your maven
+local repository.
+
+### protobuf_version (optional)
+You can find the released version of protobuf in [GitHub](https://github.com/protocolbuffers/protobuf/releases/).
+The default value is defined in `gapic-generator-java-pom-parent/pom.xml`.
+
+Use `--protobuf_version` to specify the value.
+
+Note that if specified, the version should be compatible with gapic-generator-java.
+
+### grpc_version (optional)
+You can find the released version of grpc in [maven central](https://repo1.maven.org/maven2/io/grpc/protoc-gen-grpc-java/).
+The default value is defined in `gapic-generator-java-pom-parent/pom.xml`.
+
+Use `--grpc_version` to specify the value.
+
+Note that if specified, the version should be compatible with gapic-generator-java.
+
+### transport (optional)
+One of GAPIC options passed to the generator. The value is either `grpc` or `grpc+rest`.
+The default value is `grpc`.
+
+Use `--transport` to specify the value.
+
+### rest_numeric_enums (optional)
+One of GAPIC options passed to the generator. The value is either `true` or `false`.
+The default value is `true`.
+
+Use `--rest_numeric_enums` to specify the value.
+
+### include_samples (optional)
+Whether generates code samples. The value is either `true` or `false`. 
+The default value is `true`.
+
+Use `--include_samples` to specify the value.
+
+## An example to generate a client library
+```
+library_generation/generate_library.sh \
+-p google/cloud/confidentialcomputing/v1 \
+-d google-cloud-confidentialcomputing-v1-java \
+--gapic_generator_version 2.24.0 \
+--protobuf_version 23.2 \
+--grpc_version 1.55.1 \
+--transport grpc+rest \
+--rest_numeric_enums true \
+--include_samples true
+```
+
+## An example to generate showcase client
+```
+library_generation/generate_library.sh \
+-p schema/google/showcase/v1beta1 \ # google/ should be in library_generation/.
+-d output \
+--gapic_generator_version 2.24.0 \
+--protobuf_version 23.2 \
+--grpc_version 1.55.1 \
+--transport grpc+rest \
+--rest_numeric_enums false \
+--include_samples false
+```
diff --git a/library_generation/gapic-generator-java-wrapper b/library_generation/gapic-generator-java-wrapper
new file mode 100755
index 0000000000..e23d7a7df4
--- /dev/null
+++ b/library_generation/gapic-generator-java-wrapper
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Wrap gapic-generator-java.jar because protoc requires the plugin to be executable.
+working_directory=$(dirname "$(readlink -f "$0")")
+exec java -classpath "$working_directory/gapic-generator-java-$gapic_generator_version.jar" com.google.api.generator.Main
diff --git a/library_generation/generate_library.sh b/library_generation/generate_library.sh
new file mode 100755
index 0000000000..134c3abc8c
--- /dev/null
+++ b/library_generation/generate_library.sh
@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+# parse input parameters
+while [[ $# -gt 0 ]]
+do
+key="$1"
+case $key in
+    -p|--proto_path)
+    proto_path="$2"
+    shift
+    ;;
+    -d|--destination_path)
+    destination_path="$2"
+    shift
+    ;;
+    --gapic_generator_version)
+    gapic_generator_version="$2"
+    # export this variable so that it can be used in gapic-generator-java-wrapper.sh
+    export gapic_generator_version
+    shift
+    ;;
+    --protobuf_version)
+    protobuf_version="$2"
+    shift
+    ;;
+    --grpc_version)
+    grpc_version="$2"
+    shift
+    ;;
+    --transport)
+    transport="$2"
+    shift
+    ;;
+    --rest_numeric_enums)
+    rest_numeric_enums="$2"
+    shift
+    ;;
+    --include_samples)
+    include_samples="$2"
+    shift
+    ;;
+    *)
+    echo "Invalid option: [$1]"
+    exit 1
+    ;;
+esac
+shift # past argument or value
+done
+
+working_directory=$(dirname "$(readlink -f "$0")")
+# source utility functions
+cd "$working_directory"
+source ./utilities.sh
+
+if [ -z "$protobuf_version" ]; then
+  protobuf_version=$(get_protobuf_version "$gapic_generator_version")
+fi
+
+if [ -z "$grpc_version" ]; then
+  grpc_version=$(get_grpc_version "$gapic_generator_version")
+fi
+
+if [ -z "$transport" ]; then
+  transport="grpc"
+fi
+
+if [ -z "$rest_numeric_enums" ]; then
+  rest_numeric_enums="true"
+fi
+
+if [ -z "$include_samples" ]; then
+  include_samples="true"
+fi
+
+cd "$working_directory"
+mkdir -p "$destination_path"
+destination_path="$working_directory/$destination_path"
+##################### Section 0 #####################
+# prepare tooling
+#####################################################
+cd "$working_directory"
+# the order of services entries in gapic_metadata.json is relevant to the
+# order of proto file, sort the proto files with respect to their name to
+# get a fixed order.
+proto_files=$(find "$proto_path" -type f  -name "*.proto" | sort)
+folder_name=$(extract_folder_name "$destination_path")
+# download gapic-generator-java, protobuf and grpc plugin.
+download_tools "$gapic_generator_version" "$protobuf_version" "$grpc_version"
+##################### Section 1 #####################
+# generate grpc-*/
+#####################################################
+cd "$working_directory"
+"$protoc_path"/protoc "--plugin=protoc-gen-rpc-plugin=$working_directory/protoc-gen-grpc-java-$grpc_version-linux-x86_64.exe" \
+"--rpc-plugin_out=:$destination_path/java_grpc.jar" \
+$proto_files
+# unzip java_grpc.jar to grpc-*/src/main/java
+unzip_src_files "grpc"
+# remove empty files in grpc-*/src/main/java
+remove_empty_files "grpc"
+# remove grpc version in *ServiceGrpc.java file so the content is identical with bazel build.
+remove_grpc_version
+###################### Section 2 #####################
+## generate gapic-*/, part of proto-*/, samples/
+######################################################
+cd "$working_directory"
+"$protoc_path"/protoc --experimental_allow_proto3_optional \
+"--plugin=protoc-gen-java_gapic=$working_directory/gapic-generator-java-wrapper" \
+"--java_gapic_out=metadata:$destination_path/java_gapic_srcjar_raw.srcjar.zip" \
+"--java_gapic_opt=$(get_gapic_opts)" \
+${proto_files} $(search_additional_protos)
+
+unzip -o -q "$destination_path/java_gapic_srcjar_raw.srcjar.zip" -d "$destination_path"
+# Sync'\''d to the output file name in Writer.java.
+unzip -o -q "$destination_path/temp-codegen.srcjar" -d "$destination_path/java_gapic_srcjar"
+# Resource name source files.
+proto_dir=$destination_path/java_gapic_srcjar/proto/src/main/java
+if [ ! -d "$proto_dir" ]; then
+  # Some APIs don'\''t have resource name helpers, like BigQuery v2.
+  # Create an empty file so we can finish building. Gating the resource name rule definition
+  # on file existences go against Bazel'\''s design patterns, so we'\''ll simply delete all empty
+  # files during the final packaging process (see java_gapic_pkg.bzl)
+  mkdir -p "$proto_dir"
+  touch "$proto_dir"/PlaceholderFile.java
+fi
+
+cd "$working_directory"
+# move java_gapic_srcjar/src/main to gapic-*/src.
+mv_src_files "gapic" "main"
+# remove empty files in gapic-*/src/main/java
+remove_empty_files "gapic"
+# move java_gapic_srcjar/src/test to gapic-*/src
+mv_src_files "gapic" "test"
+if [ "$include_samples" == "true" ]; then
+  # move java_gapic_srcjar/samples/snippets to samples/snippets
+  mv_src_files "samples" "main"
+fi
+##################### Section 3 #####################
+# generate proto-*/
+#####################################################
+cd "$working_directory"
+"$protoc_path"/protoc "--java_out=$destination_path/java_proto.jar" $proto_files
+# move java_gapic_srcjar/proto/src/main/java (generated resource name helper class)
+# to proto-*/src/main
+mv_src_files "proto" "main"
+# unzip java_proto.jar to proto-*/src/main/java
+unzip_src_files "proto"
+# remove empty files in proto-*/src/main/java
+remove_empty_files "proto"
+# copy proto files to proto-*/src/main/proto
+for proto_src in $proto_files; do
+    mkdir -p "$destination_path/proto-$folder_name/src/main/proto"
+    cp -f --parents "$proto_src" "$destination_path/proto-$folder_name/src/main/proto"
+done
+##################### Section 4 #####################
+# rm tar files
+#####################################################
+cd "$destination_path"
+rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_proto.jar temp-codegen.srcjar
\ No newline at end of file
diff --git a/library_generation/generate_library_integration_test.sh b/library_generation/generate_library_integration_test.sh
new file mode 100755
index 0000000000..7203b2918e
--- /dev/null
+++ b/library_generation/generate_library_integration_test.sh
@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+set -xeo pipefail
+
+# This script is used to test the result of `generate_library.sh` against generated
+# source code in googleapis-gen repository.
+# Specifically, this script will do
+# 1. checkout the master branch of googleapis/google and WORKSPACE
+# 2. parse version of gapic-generator-java, protobuf and grpc from WORKSPACE
+# 3. generate a library with given proto_path and destination_path by invoking
+#    `generate_library.sh`. GAPIC options to generate a library will be parsed
+#    from proto_path/BUILD.bazel.
+# 4. checkout the master branch googleapis-gen repository and compare the result.
+while [[ $# -gt 0 ]]
+do
+key="$1"
+case $key in
+    -p|--proto_path)
+    proto_path="$2"
+    shift
+    ;;
+    -d|--destination_path)
+    destination_path="$2"
+    shift
+    ;;
+    --googleapis_gen_url)
+    googleapis_gen_url="$2"
+    shift
+    ;;
+    *)
+    echo "Invalid option: [$1]"
+    exit 1
+    ;;
+esac
+shift # past argument or value
+done
+
+get_version_from_WORKSPACE() {
+  version_key_word=$1
+  workspace=$2
+  delimiter=$3
+  version="$(grep -m 1 "$version_key_word"  "$workspace" | sed 's/\"\(.*\)\".*/\1/' | cut -d "$delimiter" -f2 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
+  echo "$version"
+}
+
+working_directory=$(dirname "$(readlink -f "$0")")
+cd "$working_directory"
+# checkout the master branch of googleapis/google (proto files) and WORKSPACE
+echo "Checking out googlapis repository..."
+git clone --branch=master --depth 1 -q https://github.com/googleapis/googleapis.git
+cp -r googleapis/google .
+cp googleapis/WORKSPACE .
+rm -rf googleapis
+# parse version of gapic-generator-java, protobuf and grpc from WORKSPACE
+gapic_generator_version=$(get_version_from_WORKSPACE "_gapic_generator_java_version" WORKSPACE "=")
+echo "The version of gapic-generator-java is $gapic_generator_version."
+protobuf_version=$(get_version_from_WORKSPACE "protobuf-" WORKSPACE "-")
+echo "The version of protobuf is $protobuf_version"
+grpc_version=$(get_version_from_WORKSPACE "_grpc_version" WORKSPACE "=")
+echo "The version of protoc-gen-grpc-java plugin is $gapic_generator_version."
+# parse GAPIC options from proto_path/BUILD.bazel
+cd $"$working_directory"
+transport="grpc"
+if grep -A 15 "java_gapic_library(" "$proto_path/BUILD.bazel" | grep -q "grpc+rest"; then
+  transport="grpc+rest"
+fi
+rest_numeric_enums="true"
+if grep -A 15 "java_gapic_library(" "$proto_path/BUILD.bazel" | grep -q "rest_numeric_enums = False"; then
+  rest_numeric_enums="false"
+fi
+include_samples="false"
+if grep -A 15 "java_gapic_assembly_gradle_pkg(" "$proto_path/BUILD.bazel" | grep -q "include_samples = True"; then
+  include_samples="true"
+fi
+echo "GAPIC options are transport=$transport, rest_numeric_enums=$rest_numeric_enums, include_samples=$include_samples."
+# generate GAPIC client library
+echo "Generating library from $proto_path, to $destination_path..."
+"$working_directory"/generate_library.sh \
+-p "$proto_path" \
+-d "$destination_path" \
+--gapic_generator_version "$gapic_generator_version" \
+--protobuf_version "$protobuf_version" \
+--grpc_version "$grpc_version" \
+--transport "$transport" \
+--rest_numeric_enums "$rest_numeric_enums" \
+--include_samples "$include_samples"
+
+echo "Generate library finished."
+echo "Checking out googleapis-gen repository..."
+git clone --branch=master --depth 1 -q "$googleapis_gen_url"
+
+echo "Compare generation result..."
+cd "$working_directory"
+diff -r "googleapis-gen/$proto_path/$destination_path" "$destination_path" -x "*gradle*"
+echo "Comparison finished, no difference is found."
+# clean up
+cd "$working_directory"
+rm -rf WORKSPACE googleapis-gen "$destination_path"
diff --git a/library_generation/utilities.sh b/library_generation/utilities.sh
new file mode 100755
index 0000000000..e1ef5b3e25
--- /dev/null
+++ b/library_generation/utilities.sh
@@ -0,0 +1,211 @@
+#!/usr/bin/env bash
+
+set -xeo pipefail
+
+# define utility functions
+
+extract_folder_name() {
+  destination_path=$1
+  folder_name=${destination_path##*/}
+  echo "$folder_name"
+}
+
+remove_empty_files() {
+  category=$1
+  find "$destination_path/$category-$folder_name/src/main/java" -type f -size 0 | while read -r f; do rm -f "${f}"; done
+  if [ -d "$destination_path/$category-$folder_name/src/main/java/samples" ]; then
+      mv "$destination_path/$category-$folder_name/src/main/java/samples" "$destination_path/$category-$folder_name"
+  fi
+}
+
+# Move generated files to folders in destination_path.
+mv_src_files() {
+  category=$1 # one of gapic, proto, samples
+  type=$2 # one of main, test
+  if [ "$category" == "samples" ]; then
+    src_suffix="samples/snippets/generated/src/main/java/com"
+    folder_suffix="samples/snippets/generated"
+  elif [ "$category" == "proto" ]; then
+    src_suffix="$category/src/$type/java"
+    folder_suffix="$category-$folder_name/src/$type"
+  else
+    src_suffix="src/$type"
+    folder_suffix="$category-$folder_name/src"
+  fi
+  mkdir -p "$destination_path/$folder_suffix"
+  cp -r "$destination_path/java_gapic_srcjar/$src_suffix" "$destination_path/$folder_suffix"
+  if [ "$category" != "samples" ]; then
+    rm -r -f "$destination_path/$folder_suffix/java/META-INF"
+  fi
+}
+
+# unzip jar file
+unzip_src_files() {
+  category=$1
+  jar_file=java_$category.jar
+  mkdir -p "$destination_path/$category-$folder_name/src/main/java"
+  unzip -q -o "$destination_path/$jar_file" -d "$destination_path/$category-$folder_name/src/main/java"
+  rm -r -f "$destination_path/$category-$folder_name/src/main/java/META-INF"
+}
+
+find_additional_protos_in_yaml() {
+  pattern=$1
+  find_result=$(grep --include=\*.yaml -rw "$proto_path" -e "$pattern")
+  if [ -n "$find_result" ]; then
+    echo "$find_result"
+  fi
+}
+
+# Apart from proto files in proto_path, additional protos are needed in order
+# to generate GAPIC client libraries.
+# In most cases, these protos should be within google/ directory, which is
+# pulled from googleapis as a prerequisite.
+# Search additional protos in .yaml files.
+search_additional_protos() {
+  additional_protos="google/cloud/common_resources.proto" # used by every library
+  iam_policy=$(find_additional_protos_in_yaml "name: google.iam.v1.IAMPolicy")
+  if [ -n "$iam_policy" ]; then
+    additional_protos="$additional_protos google/iam/v1/iam_policy.proto"
+  fi
+  locations=$(find_additional_protos_in_yaml "name: google.cloud.location.Locations")
+  if [ -n "$locations" ]; then
+    additional_protos="$additional_protos google/cloud/location/locations.proto"
+  fi
+  echo "$additional_protos"
+}
+
+# get gapic options from .yaml and .json files from proto_path.
+get_gapic_opts() {
+  gapic_config=$(find "${proto_path}" -type f -name "*gapic.yaml")
+  if [ -z "${gapic_config}" ]; then
+    gapic_config=""
+  else
+    gapic_config="gapic-config=$gapic_config,"
+  fi
+  grpc_service_config=$(find "${proto_path}" -type f -name "*service_config.json")
+  api_service_config=$(find "${proto_path}" -maxdepth 1 -type f \( -name "*.yaml" ! -name "*gapic.yaml" \))
+  if [ "${rest_numeric_enums}" == "true" ]; then
+    rest_numeric_enums="rest-numeric-enums,"
+  else
+    rest_numeric_enums=""
+  fi
+  echo "transport=$transport,${rest_numeric_enums}grpc-service-config=$grpc_service_config,${gapic_config}api-service-config=$api_service_config"
+}
+
+remove_grpc_version() {
+  find "$destination_path" -type f -name "*Grpc.java" -exec \
+  sed -i 's/value = \"by gRPC proto compiler.*/value = \"by gRPC proto compiler\",/g' {} \;
+}
+
+download_gapic_generator_pom_parent() {
+  gapic_generator_version=$1
+  cd "$working_directory"
+  if [ ! -f "gapic-generator-java-pom-parent-$gapic_generator_version.pom" ]; then
+    if [[ "$gapic_generator_version" == *"-SNAPSHOT" ]]; then
+      # copy a SNAPSHOT version from maven local repository.
+      copy_from "$HOME/.m2/repository/com/google/api/gapic-generator-java-pom-parent/$gapic_generator_version/gapic-generator-java-pom-parent-$gapic_generator_version.pom" \
+      "gapic-generator-java-pom-parent-$gapic_generator_version.pom"
+      return
+    fi
+    # download gapic-generator-java-pom-parent from Google maven central mirror.
+    download_from \
+    "https://maven-central.storage-download.googleapis.com/maven2/com/google/api/gapic-generator-java-pom-parent/$gapic_generator_version/gapic-generator-java-pom-parent-$gapic_generator_version.pom" \
+    "gapic-generator-java-pom-parent-$gapic_generator_version.pom"
+  fi
+  # file exists, do not need to download again.
+}
+
+get_grpc_version() {
+  gapic_generator_version=$1
+  # get grpc version from gapic-generator-java-pom-parent/pom.xml
+  download_gapic_generator_pom_parent "$gapic_generator_version"
+  grpc_version=$(grep grpc.version "gapic-generator-java-pom-parent-$gapic_generator_version.pom" | sed 's/<grpc\.version>\(.*\)<\/grpc\.version>/\1/' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+  echo "$grpc_version"
+}
+
+get_protobuf_version() {
+  gapic_generator_version=$1
+  # get protobuf version from gapic-generator-java-pom-parent/pom.xml
+  download_gapic_generator_pom_parent "$gapic_generator_version"
+  protobuf_version=$(grep protobuf.version "gapic-generator-java-pom-parent-$gapic_generator_version.pom" | sed 's/<protobuf\.version>\(.*\)<\/protobuf\.version>/\1/' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | cut -d "." -f2-)
+  echo "$protobuf_version"
+}
+
+download_tools() {
+  gapic_generator_version=$1
+  protobuf_version=$2
+  grpc_version=$3
+  download_generator "$gapic_generator_version"
+  download_protobuf "$protobuf_version"
+  download_grpc_plugin "$grpc_version"
+}
+
+download_generator() {
+  gapic_generator_version=$1
+  cd "$working_directory"
+  if [ ! -f "gapic-generator-java-$gapic_generator_version.jar" ]; then
+    if [[ "$gapic_generator_version" == *"-SNAPSHOT" ]]; then
+      # copy a SNAPSHOT version from maven local repository.
+      copy_from "$HOME/.m2/repository/com/google/api/gapic-generator-java/$gapic_generator_version/gapic-generator-java-$gapic_generator_version.jar" \
+      "gapic-generator-java-$gapic_generator_version.jar"
+      return
+    fi
+    # download gapic-generator-java from Google maven central mirror.
+    download_from \
+    "https://maven-central.storage-download.googleapis.com/maven2/com/google/api/gapic-generator-java/$gapic_generator_version/gapic-generator-java-$gapic_generator_version.jar" \
+    "gapic-generator-java-$gapic_generator_version.jar"
+  fi
+}
+
+download_protobuf() {
+  protobuf_version=$1
+  cd "$working_directory"
+  if [ ! -d "protobuf-$protobuf_version.zip" ]; then
+    # pull proto files and protoc from protobuf repository as maven central
+    # doesn't have proto files
+    download_from \
+    "https://github.com/protocolbuffers/protobuf/releases/download/v$protobuf_version/protoc-$protobuf_version-linux-x86_64.zip" \
+    "protobuf-$protobuf_version.zip" \
+    "GitHub"
+    unzip -o -q "protobuf-$protobuf_version.zip" -d "protobuf-$protobuf_version"
+    cp -r "protobuf-$protobuf_version/include/google" "$working_directory"
+    rm "protobuf-$protobuf_version.zip"
+  fi
+
+  protoc_path=$working_directory/protobuf-$protobuf_version/bin
+  echo "protoc version: $("$protoc_path"/protoc --version)"
+}
+
+download_grpc_plugin() {
+  grpc_version=$1
+  cd "$working_directory"
+  if [ ! -f "protoc-gen-grpc-java-$grpc_version-linux-x86_64.exe" ]; then
+    # download protoc-gen-grpc-java plugin from Google maven central mirror.
+    download_from \
+    "https://maven-central.storage-download.googleapis.com/maven2/io/grpc/protoc-gen-grpc-java/$grpc_version/protoc-gen-grpc-java-$grpc_version-linux-x86_64.exe" \
+    "protoc-gen-grpc-java-$grpc_version-linux-x86_64.exe"
+    chmod +x "protoc-gen-grpc-java-$grpc_version-linux-x86_64.exe"
+  fi
+}
+
+download_from() {
+  url=$1
+  save_as=$2
+  repo=$3
+  # fail-fast, 30 seconds at most, retry 2 times
+  curl -LJ -o "$save_as" --fail -m 30 --retry 2 "$url" || download_fail "$save_as" "$repo"
+}
+
+copy_from() {
+  local_repo=$1
+  save_as=$2
+  cp "$local_repo" "$save_as" || \
+    download_fail "$save_as" "maven local"
+}
+
+download_fail() {
+  artifact=$1
+  repo=${2:-"maven central mirror"}
+  >&2 echo "Fail to download $artifact from $repo repository. Please install $artifact first if you want to download a SNAPSHOT."
+  exit 1
+}