CI Build caching

.github/workflows/ci.yml

@@ -15,9 +15,24 @@ jobs:
         ubuntu: ["20.04", "19.10", "18.04"]
     steps:
     - uses: actions/checkout@v2
+    - uses: actions/cache@v1
+      name: Cache build artifact store
+      with:
+        path: ./cache
+        # https://github.com/actions/cache/issues/109 "Enable always writing cache to support hermetic build systems"
+        # https://github.com/actions/cache/issues/239#issuecomment-606950711 Investigate this workaround if cache starts filling up
+        key: store-${{ runner.os }}-${{ matrix.ubuntu }}-${{ matrix.llvm }}-${{ github.sha }}
+        restore-keys: |
+          store-${{ runner.os }}-${{ matrix.ubuntu }}-${{ matrix.llvm }}-
+          store-${{ runner.os }}-${{ matrix.ubuntu }}-
+          store-${{ runner.os }}-
     - name: Build LLVM ${{ matrix.llvm }} on ${{ matrix.ubuntu }}
       run: |
         docker build . -t docker.pkg.github.com/trailofbits/cxx-common/llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest -f Dockerfile --build-arg UBUNTU_BASE=ubuntu:${{ matrix.ubuntu }} --build-arg LLVM_VERSION=${{ matrix.llvm }}
+    - name: Grab Cache
+      run: |
+        docker build . -t docker.pkg.github.com/trailofbits/cxx-common/llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64-build:latest -f Dockerfile --build-arg UBUNTU_BASE=ubuntu:${{ matrix.ubuntu }} --build-arg LLVM_VERSION=${{ matrix.llvm }} --target cxx-common-build
+        docker run --rm --entrypoint /bin/bash -v $(pwd)/cache:/tmp docker.pkg.github.com/trailofbits/cxx-common/llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64-build:latest -c "cp -r ./cache/* /tmp"
     - name: Push Image for LLVM ${{ matrix.llvm }} on ${{ matrix.ubuntu }}
       if: github.event_name == 'push' && github.ref == 'refs/heads/master'
       run: |
@@ -46,6 +61,15 @@ jobs:
     runs-on: macos-latest
     steps:
     - uses: actions/checkout@v2
+    - uses: actions/cache@v1
+      name: Cache build artifact store
+      with:
+        path: ./cache
+        # https://github.com/actions/cache/issues/109 "Enable always writing cache to support hermetic build systems"
+        # https://github.com/actions/cache/issues/239#issuecomment-606950711 Investigate this workaround if cache starts filling up
+        key: store-${{ runner.os }}-${{ github.sha }}
+        restore-keys: |
+          store-${{ runner.os }}-
     - name: Set up Python 3.8
       uses: actions/setup-python@v1
       with:

Dockerfile

@@ -16,7 +16,7 @@ RUN apt-get update && \
 
 # bootstrap image should be what's needed to get a more reproducible build
 # environment for cxx-common
-FROM base as bootstrap
+FROM base as cxx-common-build
 ARG BOOTSTRAP
 ARG LIBRARIES
 ARG LLVM_VERSION
@@ -31,19 +31,24 @@ RUN pip3 install requests
 
 RUN mkdir -p /cxx-common
 WORKDIR /cxx-common
+
+# Will try to use cache at './cache'
+# Get cache using
+#   docker build -t cxx-common-build --target cxx-common-build .
+#   docker run --rm --entrypoint /bin/bash -v $(pwd)/cache:/tmp cxx-common-build -c "cp -r ./cache /tmp"
 COPY . ./
 
 RUN mkdir -p "${BOOTSTRAP}" && mkdir -p "${LIBRARIES}"
 
 RUN ./pkgman.py \
   --c_compiler=/usr/bin/clang \
   --cxx_compiler=/usr/bin/clang++ \
+  --verbose \
+  --use_ccache \
   --repository_path="${BOOTSTRAP}" \
-  --packages=cmake && \
-  rm -rf build && mkdir build && \
-  rm -rf sources && mkdir sources
+  --packages=cmake
 
-RUN mkdir -p /cache && ./pkgman.py \
+RUN ./pkgman.py \
   --c_compiler=/usr/bin/clang \
   --cxx_compiler=/usr/bin/clang++ \
   --llvm_version=${LLVM_VERSION} \
@@ -52,28 +57,16 @@ RUN mkdir -p /cache && ./pkgman.py \
   --exclude_libcxx \
   "--additional_paths=${BOOTSTRAP}/cmake/bin" \
   "--repository_path=${LIBRARIES}" \
-  "--packages=z3,llvm" && \
-  rm -rf build && mkdir build && \
-  rm -rf sources && mkdir sources && rm -rf /cache
-
-# cxx-common-build should be image that contains all dependencies necessary to
-# build cxx-common
-FROM bootstrap as cxx-common-build
-
-WORKDIR /cxx-common
-ARG BOOTSTRAP
-ARG LIBRARIES
+  "--packages=z3,llvm"
 
 RUN mkdir -p /cache && ./pkgman.py \
   --cxx_compiler="${LIBRARIES}/llvm/bin/clang++" \
   --c_compiler="${LIBRARIES}/llvm/bin/clang" \
-  --use_ccache \
   --verbose \
+  --use_ccache \
   "--additional_paths=${BOOTSTRAP}/cmake/bin:${LIBRARIES}/llvm/bin" \
   "--repository_path=${LIBRARIES}" \
-  "--packages=cmake,google,xed" && \
-  rm -rf build && mkdir build && \
-  rm -rf sources && mkdir sources && rm -rf /cache
+  "--packages=cmake,google,xed"
 
 # dist image should be minimal artifact image
 FROM base as dist

pkgman/installers/common.py

@@ -100,6 +100,9 @@ def google_installer_glog(properties):
       print(" x Failed to create the build folder")
       return False
 
+  if properties["ccache"]:
+      set_ccache_compiler()
+
   cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator()
   cmake_command += ["-DCMAKE_CXX_STANDARD=11",
                     "-DBUILD_TESTING=OFF",
@@ -141,6 +144,9 @@ def common_installer_capstone(properties):
       print(" x Failed to create the build folder")
       return False
 
+  if properties["ccache"]:
+      set_ccache_compiler()
+
   cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator()
   cmake_command += ["-DCMAKE_EXE_LINKER_FLAGS=-g",
                     "-DCMAKE_C_FLAGS=-g",
@@ -247,6 +253,8 @@ def google_installer_gflags(properties):
       print(" x Failed to create the build folder")
       return False
 
+  if properties["ccache"]:
+      set_ccache_compiler()
 
   cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator()
   cmake_command += ["-DCMAKE_INSTALL_PREFIX=" + os.path.join(repository_path, "gflags"),
@@ -289,6 +297,9 @@ def google_installer_googletest(properties):
       print(" x Failed to create the build folder")
       return False
 
+  if properties["ccache"]:
+      set_ccache_compiler()
+
   cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator(False)
   cmake_command += ["-DCMAKE_CXX_STANDARD=11",
                     "-DCMAKE_POSITION_INDEPENDENT_CODE=ON",
@@ -363,6 +374,9 @@ def google_installer_protobuf(properties):
       print(" x Failed to create the build folder")
       return False
 
+  if properties["ccache"]:
+      set_ccache_compiler()
+
   cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator(False)
   cmake_command += ["-DPROTOBUF_ROOT=" + source_folder,
                     "-DBUILD_SHARED_LIBS=OFF",
@@ -433,6 +447,9 @@ def common_installer_capnproto(properties):
       print(" x Failed to create the build folder")
       return False
 
+  if properties["ccache"]:
+      set_ccache_compiler()
+
   cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator()
   cmake_command += ["-DCMAKE_CXX_STANDARD=11",
                     "-DCMAKE_CXX_EXTENSIONS=ON",
@@ -590,6 +607,13 @@ def common_installer_llvm(properties):
     arch_list += ";AArch64;Sparc;NVPTX;ARM"
   arch_list += "'"
 
+  if properties["ccache"]:
+    # Remove this so we don't clash with LLVM's built-in ccache config
+    if "CMAKE_CXX_COMPILER_LAUNCHER" in os.environ:
+      del(os.environ["CMAKE_CXX_COMPILER_LAUNCHER"])
+    if "CMAKE_C_COMPILER_LAUNCHER" in os.environ:
+      del(os.environ["CMAKE_C_COMPILER_LAUNCHER"])
+
   cppstd = "11"
   if int(properties["llvm_version"]) > 900:
     cppstd = "14"
@@ -604,13 +628,14 @@ def common_installer_llvm(properties):
     cmake_command += ["-DLLVM_ENABLE_Z3_SOLVER=OFF", "-DCLANG_ANALYZER_ENABLE_Z3_SOLVER=OFF"]
 
   if properties["ccache"]:
-    print(" i Enabling ccache on /cache ... ")
+    ccache_dir = f"{os.getcwd()}/cache/ccache"
+    print(f" i Enabling ccache on {ccache_dir} ... ")
     # some versions of LLVM use CCACHE_MAX_SIZE, others use CCACHE_SIZE
     cmake_command.extend(
         ["-DLLVM_CCACHE_BUILD=ON",
-         "-DLLVM_CCACHE_SIZE=10G",
-         "-DLLVM_CCACHE_DIR=/cache",
-         "-DLLVM_CCACHE_MAXSIZE=10G"])
+         "-DLLVM_CCACHE_SIZE=5G",
+         f'-DLLVM_CCACHE_DIR="{ccache_dir}"',
+         "-DLLVM_CCACHE_MAXSIZE=5G"])
 
   if use_libcxx:
     if int(properties["llvm_version"]) < 371:
@@ -662,6 +687,9 @@ def common_installer_z3(properties):
       print(" x Failed to create the build folder")
       return False
 
+  if properties["ccache"]:
+      set_ccache_compiler()
+
   cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator()
   cmake_command += ["-DZ3_BUILD_LIBZ3_SHARED=False",
                     "-DZ3_ENABLE_EXAMPLE_TARGETS=False",

pkgman/installers/unix.py

@@ -53,6 +53,11 @@ def unix_installer_boost(properties, default_toolset):
   if "CMAKE_CXX_COMPILER" in os.environ:
     os.environ["CXX"] = os.environ["CMAKE_CXX_COMPILER"]
 
+  if properties["ccache"]:
+    set_ccache_compiler()
+    os.environ["CXX"] = f"ccache {os.environ['CXX']}"
+    os.environ["CC"] = f"ccache {os.environ['CC']}"
+
   configure_command = [source_folder + "/bootstrap.sh", "--prefix=" + os.path.join(repository_path, "boost"), "--with-toolset=" + toolset_name]
   if not run_program("Running the bootstrap script...", configure_command, source_folder, verbose=verbose_output):
     return False
@@ -118,7 +123,12 @@ def unix_installer_cmake(properties):
   if os.environ.get("CMAKE_CXX_COMPILER") is not None:
     os.environ["CXX"] = os.environ["CMAKE_CXX_COMPILER"]
 
-  if not run_program("Running the bootstrap script...", ["./bootstrap", "--verbose", "--parallel=" + str(multiprocessing.cpu_count()), "--prefix=" + destination_path], source_folder, verbose=verbose_output):
+  enable_ccache = None
+  if properties["ccache"]:
+    set_ccache_compiler()
+    enable_ccache = "--enable-ccache"
+
+  if not run_program("Running the bootstrap script...", ["./bootstrap", "--verbose", "--parallel=" + str(multiprocessing.cpu_count()), "--prefix=" + destination_path] + ([enable_ccache] if enable_ccache else []), source_folder, verbose=verbose_output):
     return False
 
   if not run_program("Building the source code...", ["make", "-j" + str(multiprocessing.cpu_count())], source_folder, verbose=verbose_output):

pkgman/installers/utils.py

@@ -32,8 +32,27 @@ def get_env_compiler_settings():
   if os.environ.get("CMAKE_C_COMPILER") is not None:
     cmake_compiler_settings.append("-DCMAKE_C_COMPILER=" + os.environ["CMAKE_C_COMPILER"])
 
+  if os.environ.get("CMAKE_CXX_COMPILER_LAUNCHER") is not None:
+    cmake_compiler_settings.append("-DCMAKE_CXX_COMPILER_LAUNCHER=" + os.environ["CMAKE_CXX_COMPILER_LAUNCHER"])
+
+  if os.environ.get("CMAKE_C_COMPILER_LAUNCHER") is not None:
+    cmake_compiler_settings.append("-DCMAKE_C_COMPILER_LAUNCHER=" + os.environ["CMAKE_C_COMPILER_LAUNCHER"])
+
   return cmake_compiler_settings
 
+
+def set_ccache_compiler():
+    """
+    Set the compiler environment variables to use ccache.
+
+    NOTE: LLVM uses its own ccache settings. Make sure these are synchronized
+    """
+    os.environ["CMAKE_CXX_COMPILER_LAUNCHER"] = "ccache"
+    os.environ["CMAKE_C_COMPILER_LAUNCHER"] = "ccache"
+    # Only works if we never change directories in the script(s)
+    os.environ["CCACHE_DIR"] = f'"{os.getcwd()}/cache/ccache"'
+
+
 def get_parallel_build_options():
   processor_count = str(multiprocessing.cpu_count())
 

travis.sh

@@ -89,7 +89,7 @@ osx_initialize() {
   xcode-select --install 2>&1 > /dev/null
 
   printf " > Installing the required packages...\n"
-  brew install coreutils
+  brew install coreutils ccache
   if [ $? -ne 0 ] ; then
     printf " x Could not install the required dependencies\n"
     return 1
@@ -126,7 +126,7 @@ linux_build() {
   printf " > Launching the build script for CMake...\n"
 
   printf "\n===\n"
-  python3 pkgman.py --c_compiler=$(which clang) --cxx_compiler=$(which clang++) --verbose "--repository_path=${bootstrap_repository}" "--packages=cmake"
+  python3 pkgman.py --use_ccache --c_compiler=$(which clang) --cxx_compiler=$(which clang++) --verbose "--repository_path=${bootstrap_repository}" "--packages=cmake"
   local pkgman_error=$?
   printf "===\n\n"
 
@@ -138,7 +138,7 @@ linux_build() {
   printf " > Launching the build script for LLVM...\n"
 
   printf "\n===\n"
-  python3 pkgman.py --c_compiler=$(which clang) --cxx_compiler=$(which clang++) --exclude_libcxx --verbose "--additional_paths=${bootstrap_repository}/cmake/bin" "--repository_path=${bootstrap_repository}" "--packages=llvm"
+  python3 pkgman.py --use_ccache --c_compiler=$(which clang) --cxx_compiler=$(which clang++) --exclude_libcxx --verbose "--additional_paths=${bootstrap_repository}/cmake/bin" "--repository_path=${bootstrap_repository}" "--packages=llvm"
   local pkgman_error=$?
   printf "===\n\n"
 
@@ -181,7 +181,7 @@ linux_build() {
   printf " > Re-launching the build script using the newly built clang...\n"
 
   printf "\n===\n"
-  python3 pkgman.py "--cxx_compiler=${bootstrap_repository}/llvm/bin/clang++" "--c_compiler=${bootstrap_repository}/llvm/bin/clang" --verbose "--additional_paths=${bootstrap_repository}/cmake/bin:${bootstrap_repository}/llvm/bin:${custom_bin_path}" "--repository_path=${library_repository}" "--packages=cmake,capstone,google,xed,capnproto"
+  python3 pkgman.py --use_ccache "--cxx_compiler=${bootstrap_repository}/llvm/bin/clang++" "--c_compiler=${bootstrap_repository}/llvm/bin/clang" --verbose "--additional_paths=${bootstrap_repository}/cmake/bin:${bootstrap_repository}/llvm/bin:${custom_bin_path}" "--repository_path=${library_repository}" "--packages=cmake,capstone,google,xed,capnproto"
   local pkgman_error=$?
   printf "===\n\n"
 
@@ -220,7 +220,7 @@ osx_build() {
   printf " > Launching the build script for CMake...\n"
 
   printf "\n===\n"
-  python3 pkgman.py --verbose "--repository_path=${bootstrap_repository}" "--packages=cmake"
+  python3 pkgman.py --verbose --use_ccache "--repository_path=${bootstrap_repository}" "--packages=cmake"
   local pkgman_error=$?
   printf "===\n\n"
 
@@ -232,7 +232,7 @@ osx_build() {
   printf " > Launching the build script for LLVM...\n"
 
   printf "\n===\n"
-  python3 pkgman.py --verbose "--additional_paths=${bootstrap_repository}/cmake/bin" "--repository_path=${library_repository}" "--packages=llvm"
+  python3 pkgman.py --verbose --use_ccache "--additional_paths=${bootstrap_repository}/cmake/bin" "--use_ccache" "--repository_path=${library_repository}" "--packages=llvm"
   local pkgman_error=$?
   printf "===\n\n"
 
@@ -275,7 +275,7 @@ osx_build() {
   printf " > Re-launching the build script using the newly built clang...\n"
 
   printf "\n===\n"
-  python3 pkgman.py "--cxx_compiler=${library_repository}/llvm/bin/clang++" "--c_compiler=${library_repository}/llvm/bin/clang" --verbose "--additional_paths=${bootstrap_repository}/cmake/bin:${library_repository}/llvm/bin:${custom_bin_path}" "--repository_path=${library_repository}" "--packages=cmake,capstone,google,xed,capnproto"
+  python3 pkgman.py --use_ccache "--cxx_compiler=${library_repository}/llvm/bin/clang++" "--c_compiler=${library_repository}/llvm/bin/clang" --verbose "--additional_paths=${bootstrap_repository}/cmake/bin:${library_repository}/llvm/bin:${custom_bin_path}" "--repository_path=${library_repository}" "--packages=cmake,capstone,google,xed,capnproto"
   local pkgman_error=$?
   printf "===\n\n"