From 6328849420751b3418421925c83a6ff19c98ed74 Mon Sep 17 00:00:00 2001
From: Chris Banes <chrisbanes@google.com>
Date: Mon, 5 Nov 2018 16:06:24 +1100
Subject: [PATCH] Try out Google Cloud Build [ci skip]

Many thanks to https://github.com/pixiteapps/android-cloud-build for most of the
implementation.
---
 cloudbuild.yaml                    | 165 +++++++++++++++++++++++++++++
 cloudbuild/android/Dockerfile      |  48 +++++++++
 cloudbuild/android/README.md       |  17 +++
 cloudbuild/android/cloudbuild.yaml |   5 +
 cloudbuild/cloudbuild.yaml         |  10 ++
 cloudbuild/setup_env.sh            |  23 ++++
 cloudbuild/tar/Dockerfile          |   6 ++
 cloudbuild/tar/README.md           |  28 +++++
 cloudbuild/tar/cloudbuild.yaml     |   5 +
 9 files changed, 307 insertions(+)
 create mode 100644 cloudbuild.yaml
 create mode 100644 cloudbuild/android/Dockerfile
 create mode 100644 cloudbuild/android/README.md
 create mode 100644 cloudbuild/android/cloudbuild.yaml
 create mode 100644 cloudbuild/cloudbuild.yaml
 create mode 100755 cloudbuild/setup_env.sh
 create mode 100644 cloudbuild/tar/Dockerfile
 create mode 100644 cloudbuild/tar/README.md
 create mode 100644 cloudbuild/tar/cloudbuild.yaml

diff --git a/cloudbuild.yaml b/cloudbuild.yaml
new file mode 100644
index 0000000000..661c7f8de1
--- /dev/null
+++ b/cloudbuild.yaml
@@ -0,0 +1,165 @@
+steps:
+
+#
+# Get a build number
+#
+# Download the config bucket, which stores the build number.
+- name: 'gcr.io/cloud-builders/gsutil'
+  id: copy_config
+  waitFor: ['-']  # The '-' indicates that this step begins immediately.
+  # we use rsync and not cp so that this step doesn't fail the first time it's run
+  args: ['rsync', 'gs://${PROJECT_ID}-config/', '/config']
+  volumes:
+  - name: 'config'
+    path: '/config'
+
+# Runs the setup_env.sh script, which writes build environment
+# variables to build_environment.sh
+- name: 'gcr.io/$PROJECT_ID/tar'
+  id: setup_env
+  entrypoint: 'bash'
+  args: ['-c', 'cloudbuild/setup_env.sh']
+  waitFor: ['copy_config']
+  volumes:
+  - name: 'config'
+    path: '/config'
+
+# Save the updated build number to cloud storage
+- name: 'gcr.io/cloud-builders/gsutil'
+  id: save_env
+  args: ['cp', '/config/version.txt', 'gs://${PROJECT_ID}-config/version.txt']
+  waitFor: ['setup_env']
+  volumes:
+  - name: 'config'
+    path: '/config'
+
+#
+# Extract the cache
+#
+# The gradle build cache is stored as a tarball in Google Cloud Storage to make builds faster.
+#
+- name: 'gcr.io/cloud-builders/gsutil'
+  id: copy_build_cache
+  waitFor: ['-']  # The '-' indicates that this step begins immediately.
+  # we use rsync and not cp so that this step doesn't fail the first time it's run
+  args: ['rsync', 'gs://${PROJECT_ID}-cache/', '/build_cache']
+  volumes:
+  - name: 'build_cache'
+    path: '/build_cache'
+
+- name: 'gcr.io/$PROJECT_ID/tar'
+  id: extract_build_cache
+  waitFor: ['copy_build_cache']
+  # This might fail the first time, but that's okay
+  entrypoint: 'bash'
+  args:
+  - '-c'
+  - |
+    tar xpzf /build_cache/cache.tgz -C /build_cache || echo "No cache found."
+  volumes:
+  - name: 'build_cache'
+    path: '/build_cache'
+
+#
+# Build the project
+#
+# In order to get the build number that we calculated earlier, we need to source the
+# build_environment.sh file *in each step it's needed* before running our build.
+#
+- name: 'gcr.io/$PROJECT_ID/android'
+  id: build
+  entrypoint: 'bash'
+  args:
+    - '-c'
+    - |
+      source build_environment.sh
+      ./gradlew --stacktrace bundleDebug assembleDebug app:assembleDebugAndroidTest
+#-Ptivi.versioncode=${BUILD_NUM}
+  <<: &env
+    env:
+    - 'TERM=dumb'
+    - 'JAVA_TOOL_OPTIONS="-Xmx1024m"'
+    - 'GRADLE_USER_HOME=/build_cache/.gradle'
+    - 'GRADLE_OPTS="-Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.workers.max=2 -Dkotlin.incremental=false"'
+  waitFor:
+#    - decrypt_secrets
+    - extract_build_cache
+  volumes:
+  - name: 'build_cache'
+    path: '/build_cache'
+
+#
+# Save the APKs
+#
+- name: 'gcr.io/cloud-builders/gsutil'
+  args: ['-q', 'cp', '-r', 'app/build/outputs/apk', 'gs://${PROJECT_ID}-artifacts/$BRANCH_NAME-$BUILD_ID/']
+  waitFor: ['build']
+
+#
+# Unit Tests
+#
+# Run the unit tests using the same type of step as the build.
+#
+- name: 'gcr.io/$PROJECT_ID/android'
+  id: unit_tests
+  entrypoint: 'bash'
+  args:
+  - '-c'
+  - |
+    source build_environment.sh
+    ./gradlew --stacktrace check
+  <<: *env
+  waitFor: ['build']
+  volumes:
+  - name: 'build_cache'
+    path: '/build_cache'
+
+#
+# Store the unit test reports
+#
+- name: 'gcr.io/cloud-builders/gsutil'
+  id: store_unit_test_reports
+  args: ['-q', 'cp', '-r', 'app/build/reports/', 'gs://${PROJECT_ID}-artifacts/$BRANCH_NAME-$BUILD_ID/']
+  waitFor: ['unit_tests']
+
+#
+# Store the unit test results
+#
+- name: 'gcr.io/cloud-builders/gsutil'
+  id: store_unit_test_results
+  args: ['-q', 'cp', '-r', 'app/build/test-results/', 'gs://${PROJECT_ID}-artifacts/$BRANCH_NAME-$BUILD_ID/']
+  waitFor: ['unit_tests']
+
+#
+# Cleanup
+#
+
+# Compress the gradle build cache
+- name: 'gcr.io/$PROJECT_ID/tar'
+  id: compress_cache
+  args: ['cpvzf', '/build_cache/cache.tgz', '-C', '/build_cache', '/build_cache/.gradle']
+  waitFor: ['unit_tests']
+  volumes:
+  - name: 'build_cache'
+    path: '/build_cache'
+
+# Store the build cache
+- name: gcr.io/cloud-builders/gsutil
+  args: ['cp', '/build_cache/cache.tgz', 'gs://${PROJECT_ID}-cache/cache.tgz']
+  waitFor: ['compress_cache']
+  volumes:
+  - name: 'build_cache'
+    path: '/build_cache'
+
+timeout: 1800s
+
+# This build requires more than 3.75 GB of memory, so I have to use a HIGHCPU machine
+# which has 30 GB of memory.  This means I can give Gradle lots of processes to run
+# highly parallelized.
+#
+# A standard machine is $0.003 per build minute, but these high cpu machines are
+# $0.016 per build minute. That means that a 15 minute build will cost $0.045 on
+# a standard machine, but $0.24 on the larger machine.  A machine half that size
+# from CircleCI would cost $0.024 per build minute, so that saves 1/3 the cost.
+#options:
+#  machineType: 'N1_HIGHCPU_8'
diff --git a/cloudbuild/android/Dockerfile b/cloudbuild/android/Dockerfile
new file mode 100644
index 0000000000..b3645b63a3
--- /dev/null
+++ b/cloudbuild/android/Dockerfile
@@ -0,0 +1,48 @@
+# we use a gcr.io image and not openjdk:8-jdk-slim because it loads faster in the google Google Cloud Build environment
+FROM gcr.io/cloud-builders/javac
+
+LABEL maintainer="ryan@pixiteapps.com"
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# make Apt non-interactive
+RUN echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90builder \
+  && echo 'DPkg::Options "--force-confnew";' >> /etc/apt/apt.conf.d/90builder
+
+# Install Dependencies
+RUN apt-get update && \
+    apt-get install -y \
+        git locales sudo openssh-client ca-certificates tar gzip parallel \
+        zip unzip bzip2 gnupg curl wget
+
+# Set timezone to UTC by default
+RUN ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime
+
+# Use unicode
+RUN locale-gen C.UTF-8 || true
+ENV LANG=C.UTF-8
+
+ARG sdk_version=sdk-tools-linux-4333796.zip
+ARG android_home=/opt/android/sdk
+
+# Install Android SDK
+RUN sudo mkdir -p ${android_home} && \
+    curl --silent --show-error --location --fail --retry 3 --output /tmp/${sdk_version} https://dl.google.com/android/repository/${sdk_version} && \
+    unzip -q /tmp/${sdk_version} -d ${android_home} && \
+    rm /tmp/${sdk_version}
+
+# Set environment variables
+ENV ANDROID_HOME ${android_home}
+ENV ADB_INSTALL_TIMEOUT 120
+ENV PATH=${ANDROID_HOME}/emulator:${ANDROID_HOME}/tools:${ANDROID_HOME}/tools/bin:${ANDROID_HOME}/platform-tools:${PATH}
+
+RUN mkdir ~/.android && echo '### User Sources for Android SDK Manager' > ~/.android/repositories.cfg
+
+RUN yes | sdkmanager --licenses && sdkmanager --update
+
+# Update SDK manager and install system image, platform and build tools
+RUN sdkmanager \
+    "tools" \
+    "platform-tools" \
+    "build-tools;28.0.3" \
+    "platforms;android-28"
diff --git a/cloudbuild/android/README.md b/cloudbuild/android/README.md
new file mode 100644
index 0000000000..87de5334a2
--- /dev/null
+++ b/cloudbuild/android/README.md
@@ -0,0 +1,17 @@
+# android
+
+This is an image that contains the latest Android SDK and NDK, used to build Android projects.
+
+## Examples
+
+The following examples demonstrate build requests that use this builder.
+
+### Fetch the contents of a remote URL
+
+This `cloudbuild.yaml` extracts a zip archive, overwriting any existing data.
+
+```
+steps:
+- name: gcr.io/$PROJECT_ID/unzip
+  args: ['-o', '-q', 'cache.zip']
+```
diff --git a/cloudbuild/android/cloudbuild.yaml b/cloudbuild/android/cloudbuild.yaml
new file mode 100644
index 0000000000..b3f5b7c23d
--- /dev/null
+++ b/cloudbuild/android/cloudbuild.yaml
@@ -0,0 +1,5 @@
+steps:
+- name: gcr.io/cloud-builders/docker
+  args: ['build', '-t', 'gcr.io/$PROJECT_ID/android', '.']
+
+images: ['gcr.io/$PROJECT_ID/android']
\ No newline at end of file
diff --git a/cloudbuild/cloudbuild.yaml b/cloudbuild/cloudbuild.yaml
new file mode 100644
index 0000000000..1f0f9fed32
--- /dev/null
+++ b/cloudbuild/cloudbuild.yaml
@@ -0,0 +1,10 @@
+steps:
+- name: gcr.io/cloud-builders/docker
+  args: ['build', '-t', 'gcr.io/$PROJECT_ID/tar', './tar']
+
+- name: gcr.io/cloud-builders/docker
+  args: ['build', '-t', 'gcr.io/$PROJECT_ID/android', './android']
+
+images: 
+- 'gcr.io/$PROJECT_ID/tar'
+- 'gcr.io/$PROJECT_ID/android'
diff --git a/cloudbuild/setup_env.sh b/cloudbuild/setup_env.sh
new file mode 100755
index 0000000000..68ed164ff6
--- /dev/null
+++ b/cloudbuild/setup_env.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+version_file="/config/version.txt"
+
+__LAST_BUILD_NUM=1900
+
+if [ -f "${version_file}" ];then
+  echo "Reading existing version from ${version_file}"
+  source "${version_file}"
+fi
+
+BUILD_NUM=$((__LAST_BUILD_NUM+1))
+
+echo
+echo "       Last build number: ${__LAST_BUILD_NUM}"
+echo "Updating to build number: ${BUILD_NUM}"
+
+echo "__LAST_BUILD_NUM=${BUILD_NUM}" > "${version_file}"
+
+OUTPUT_FILE=build_environment.sh
+echo "export BUILD_NUM=${BUILD_NUM}" > ${OUTPUT_FILE}
+
+echo "Wrote output file: ${OUTPUT_FILE}"
diff --git a/cloudbuild/tar/Dockerfile b/cloudbuild/tar/Dockerfile
new file mode 100644
index 0000000000..d00f2b995c
--- /dev/null
+++ b/cloudbuild/tar/Dockerfile
@@ -0,0 +1,6 @@
+FROM launcher.gcr.io/google/ubuntu16_04
+
+RUN apt-get update && \
+    apt-get -y install tar bzip2 gzip
+
+ENTRYPOINT ["tar"]
\ No newline at end of file
diff --git a/cloudbuild/tar/README.md b/cloudbuild/tar/README.md
new file mode 100644
index 0000000000..e4f979e53b
--- /dev/null
+++ b/cloudbuild/tar/README.md
@@ -0,0 +1,28 @@
+# tar
+
+This is a tool build to simply invoke the
+[`tar`](https://linux.die.net/man/1/tar) command.
+
+Arguments passed to this builder will be passed to `tar` directly.
+
+## Examples
+
+The following examples demonstrate build requests that use this builder.
+
+### Archive and compress caches
+
+This `cloudbuild.yaml` archives and compresses the build cache directories.
+
+```
+steps:
+- name: gcr.io/$PROJECT_ID/tar
+  args: ['cpzf', 'cache.tar.gz', '.gradle', '/root/.gradle', '/root/.m2']
+```
+
+### Extract an existing archive
+
+```
+steps:
+- name: gcr.io/$PROJECT_ID/tar
+  args: ['cpzf', 'cache.tar.gz', '.gradle', '/root/.gradle', '/root/.m2']
+```
\ No newline at end of file
diff --git a/cloudbuild/tar/cloudbuild.yaml b/cloudbuild/tar/cloudbuild.yaml
new file mode 100644
index 0000000000..c67d9f1093
--- /dev/null
+++ b/cloudbuild/tar/cloudbuild.yaml
@@ -0,0 +1,5 @@
+steps:
+- name: gcr.io/cloud-builders/docker
+  args: ['build', '-t', 'gcr.io/$PROJECT_ID/tar', '.']
+
+images: ['gcr.io/$PROJECT_ID/tar']
\ No newline at end of file