fix(java): run mvn install with retry (#453)

* fix(java): run mvn install with retry

* fix invocation of command

synthtool/gcp/templates/java_library/.kokoro/build.sh

@@ -20,17 +20,22 @@ scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}"))
 ## cd to the parent directory, i.e. the root of the git repo
 cd ${scriptDir}/..
 
+# include common functions
+source ${scriptDir}/common.sh
+
 # Print out Java version
 java -version
 echo ${JOB_TYPE}
 
-mvn install -B -V \
-  -DskipTests=true \
-  -Dclirr.skip=true \
-  -Denforcer.skip=true \
-  -Dmaven.javadoc.skip=true \
-  -Dgcloud.download.skip=true \
-  -T 1C
+# attempt to install 3 times with exponential backoff (starting with 10 seconds)
+retry_with_backoff 3 10 \
+  mvn install -B -V \
+    -DskipTests=true \
+    -Dclirr.skip=true \
+    -Denforcer.skip=true \
+    -Dmaven.javadoc.skip=true \
+    -Dgcloud.download.skip=true \
+    -T 1C
 
 # if GOOGLE_APPLICATION_CREDIENTIALS is specified as a relative path prepend Kokoro root directory onto it
 if [[ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" && "${GOOGLE_APPLICATION_CREDENTIALS}" != /* ]]; then

synthtool/gcp/templates/java_library/.kokoro/common.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# set -eo pipefail
+
+function retry_with_backoff {
+    attempts_left=$1
+    sleep_seconds=$2
+    shift 2
+    command=$@
+
+    echo "${command}"
+    ${command}
+    exit_code=$?
+
+    if [[ $exit_code == 0 ]]
+    then
+        return 0
+    fi
+
+    # failure
+    if [[ ${attempts_left} > 0 ]]
+    then
+        echo "failure (${exit_code}), sleeping ${sleep_seconds}..."
+        sleep ${sleep_seconds}
+        new_attempts=$((${attempts_left} - 1))
+        new_sleep=$((${sleep_seconds} * 2))
+        retry_with_backoff ${new_attempts} ${new_sleep} ${command}
+    fi
+
+    return $exit_code
+}

synthtool/gcp/templates/java_library/.kokoro/dependencies.sh

@@ -15,7 +15,13 @@
 
 set -eo pipefail
 
-cd github/{{ metadata['repo']['repo_short'] }}/
+## Get the directory of the build script
+scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}"))
+## cd to the parent directory, i.e. the root of the git repo
+cd ${scriptDir}/..
+
+# include common functions
+source ${scriptDir}/common.sh
 
 # Print out Java
 java -version
@@ -24,8 +30,9 @@ echo $JOB_TYPE
 export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=128m"
 
 # this should run maven enforcer
-mvn install -B -V \
-  -DskipTests=true \
-  -Dclirr.skip=true
+retry_with_backoff 3 10 \
+  mvn install -B -V \
+    -DskipTests=true \
+    -Dclirr.skip=true
 
 mvn -B dependency:analyze -DfailOnWarning=true

synthtool/gcp/templates/java_library/.kokoro/linkage-monitor.sh

@@ -17,18 +17,26 @@ set -eo pipefail
 # Display commands being run.
 set -x
 
-cd github/{{ metadata['repo']['repo_short'] }}/
+## Get the directory of the build script
+scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}"))
+## cd to the parent directory, i.e. the root of the git repo
+cd ${scriptDir}/..
+
+# include common functions
+source ${scriptDir}/common.sh
 
 # Print out Java version
 java -version
 echo ${JOB_TYPE}
 
-mvn install -B -V \
-  -DskipTests=true \
-  -Dclirr.skip=true \
-  -Denforcer.skip=true \
-  -Dmaven.javadoc.skip=true \
-  -Dgcloud.download.skip=true
+# attempt to install 3 times with exponential backoff (starting with 10 seconds)
+retry_with_backoff 3 10 \
+  mvn install -B -V \
+    -DskipTests=true \
+    -Dclirr.skip=true \
+    -Denforcer.skip=true \
+    -Dmaven.javadoc.skip=true \
+    -Dgcloud.download.skip=true
 
 # Kokoro job cloud-opensource-java/ubuntu/linkage-monitor-gcs creates this JAR
 JAR=linkage-monitor-latest-all-deps.jar