diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 982cef76..362ef885 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,9 +15,24 @@ jobs: ubuntu: ["20.04", "19.10", "18.04"] steps: - uses: actions/checkout@v2 + - uses: actions/cache@v1 + name: Cache build artifact store + with: + path: ./cache + # https://github.com/actions/cache/issues/109 "Enable always writing cache to support hermetic build systems" + # https://github.com/actions/cache/issues/239#issuecomment-606950711 Investigate this workaround if cache starts filling up + key: store-${{ runner.os }}-${{ matrix.ubuntu }}-${{ matrix.llvm }}-${{ github.sha }} + restore-keys: | + store-${{ runner.os }}-${{ matrix.ubuntu }}-${{ matrix.llvm }}- + store-${{ runner.os }}-${{ matrix.ubuntu }}- + store-${{ runner.os }}- - name: Build LLVM ${{ matrix.llvm }} on ${{ matrix.ubuntu }} run: | docker build . -t docker.pkg.github.com/trailofbits/cxx-common/llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest -f Dockerfile --build-arg UBUNTU_BASE=ubuntu:${{ matrix.ubuntu }} --build-arg LLVM_VERSION=${{ matrix.llvm }} + - name: Grab Cache + run: | + docker build . -t docker.pkg.github.com/trailofbits/cxx-common/llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64-build:latest -f Dockerfile --build-arg UBUNTU_BASE=ubuntu:${{ matrix.ubuntu }} --build-arg LLVM_VERSION=${{ matrix.llvm }} --target cxx-common-build + docker run --rm --entrypoint /bin/bash -v $(pwd)/cache:/tmp docker.pkg.github.com/trailofbits/cxx-common/llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64-build:latest -c "cp -r ./cache/* /tmp" - name: Push Image for LLVM ${{ matrix.llvm }} on ${{ matrix.ubuntu }} if: github.event_name == 'push' && github.ref == 'refs/heads/master' run: | @@ -46,6 +61,15 @@ jobs: runs-on: macos-latest steps: - uses: actions/checkout@v2 + - uses: actions/cache@v1 + name: Cache build artifact store + with: + path: ./cache + # https://github.com/actions/cache/issues/109 "Enable always writing cache to support hermetic build systems" + # https://github.com/actions/cache/issues/239#issuecomment-606950711 Investigate this workaround if cache starts filling up + key: store-${{ runner.os }}-${{ github.sha }} + restore-keys: | + store-${{ runner.os }}- - name: Set up Python 3.8 uses: actions/setup-python@v1 with: diff --git a/Dockerfile b/Dockerfile index 4773b5bb..c6119a86 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN apt-get update && \ # bootstrap image should be what's needed to get a more reproducible build # environment for cxx-common -FROM base as bootstrap +FROM base as cxx-common-build ARG BOOTSTRAP ARG LIBRARIES ARG LLVM_VERSION @@ -31,6 +31,11 @@ RUN pip3 install requests RUN mkdir -p /cxx-common WORKDIR /cxx-common + +# Will try to use cache at './cache' +# Get cache using +# docker build -t cxx-common-build --target cxx-common-build . +# docker run --rm --entrypoint /bin/bash -v $(pwd)/cache:/tmp cxx-common-build -c "cp -r ./cache /tmp" COPY . ./ RUN mkdir -p "${BOOTSTRAP}" && mkdir -p "${LIBRARIES}" @@ -38,12 +43,12 @@ RUN mkdir -p "${BOOTSTRAP}" && mkdir -p "${LIBRARIES}" RUN ./pkgman.py \ --c_compiler=/usr/bin/clang \ --cxx_compiler=/usr/bin/clang++ \ + --verbose \ + --use_ccache \ --repository_path="${BOOTSTRAP}" \ - --packages=cmake && \ - rm -rf build && mkdir build && \ - rm -rf sources && mkdir sources + --packages=cmake -RUN mkdir -p /cache && ./pkgman.py \ +RUN ./pkgman.py \ --c_compiler=/usr/bin/clang \ --cxx_compiler=/usr/bin/clang++ \ --llvm_version=${LLVM_VERSION} \ @@ -52,28 +57,16 @@ RUN mkdir -p /cache && ./pkgman.py \ --exclude_libcxx \ "--additional_paths=${BOOTSTRAP}/cmake/bin" \ "--repository_path=${LIBRARIES}" \ - "--packages=z3,llvm" && \ - rm -rf build && mkdir build && \ - rm -rf sources && mkdir sources && rm -rf /cache - -# cxx-common-build should be image that contains all dependencies necessary to -# build cxx-common -FROM bootstrap as cxx-common-build - -WORKDIR /cxx-common -ARG BOOTSTRAP -ARG LIBRARIES + "--packages=z3,llvm" RUN mkdir -p /cache && ./pkgman.py \ --cxx_compiler="${LIBRARIES}/llvm/bin/clang++" \ --c_compiler="${LIBRARIES}/llvm/bin/clang" \ - --use_ccache \ --verbose \ + --use_ccache \ "--additional_paths=${BOOTSTRAP}/cmake/bin:${LIBRARIES}/llvm/bin" \ "--repository_path=${LIBRARIES}" \ - "--packages=cmake,google,xed" && \ - rm -rf build && mkdir build && \ - rm -rf sources && mkdir sources && rm -rf /cache + "--packages=cmake,google,xed" # dist image should be minimal artifact image FROM base as dist diff --git a/pkgman/installers/common.py b/pkgman/installers/common.py index a6638ed9..bea2edc6 100644 --- a/pkgman/installers/common.py +++ b/pkgman/installers/common.py @@ -100,6 +100,9 @@ def google_installer_glog(properties): print(" x Failed to create the build folder") return False + if properties["ccache"]: + set_ccache_compiler() + cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator() cmake_command += ["-DCMAKE_CXX_STANDARD=11", "-DBUILD_TESTING=OFF", @@ -141,6 +144,9 @@ def common_installer_capstone(properties): print(" x Failed to create the build folder") return False + if properties["ccache"]: + set_ccache_compiler() + cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator() cmake_command += ["-DCMAKE_EXE_LINKER_FLAGS=-g", "-DCMAKE_C_FLAGS=-g", @@ -247,6 +253,8 @@ def google_installer_gflags(properties): print(" x Failed to create the build folder") return False + if properties["ccache"]: + set_ccache_compiler() cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator() cmake_command += ["-DCMAKE_INSTALL_PREFIX=" + os.path.join(repository_path, "gflags"), @@ -289,6 +297,9 @@ def google_installer_googletest(properties): print(" x Failed to create the build folder") return False + if properties["ccache"]: + set_ccache_compiler() + cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator(False) cmake_command += ["-DCMAKE_CXX_STANDARD=11", "-DCMAKE_POSITION_INDEPENDENT_CODE=ON", @@ -363,6 +374,9 @@ def google_installer_protobuf(properties): print(" x Failed to create the build folder") return False + if properties["ccache"]: + set_ccache_compiler() + cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator(False) cmake_command += ["-DPROTOBUF_ROOT=" + source_folder, "-DBUILD_SHARED_LIBS=OFF", @@ -433,6 +447,9 @@ def common_installer_capnproto(properties): print(" x Failed to create the build folder") return False + if properties["ccache"]: + set_ccache_compiler() + cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator() cmake_command += ["-DCMAKE_CXX_STANDARD=11", "-DCMAKE_CXX_EXTENSIONS=ON", @@ -590,6 +607,13 @@ def common_installer_llvm(properties): arch_list += ";AArch64;Sparc;NVPTX;ARM" arch_list += "'" + if properties["ccache"]: + # Remove this so we don't clash with LLVM's built-in ccache config + if "CMAKE_CXX_COMPILER_LAUNCHER" in os.environ: + del(os.environ["CMAKE_CXX_COMPILER_LAUNCHER"]) + if "CMAKE_C_COMPILER_LAUNCHER" in os.environ: + del(os.environ["CMAKE_C_COMPILER_LAUNCHER"]) + cppstd = "11" if int(properties["llvm_version"]) > 900: cppstd = "14" @@ -604,13 +628,14 @@ def common_installer_llvm(properties): cmake_command += ["-DLLVM_ENABLE_Z3_SOLVER=OFF", "-DCLANG_ANALYZER_ENABLE_Z3_SOLVER=OFF"] if properties["ccache"]: - print(" i Enabling ccache on /cache ... ") + ccache_dir = f"{os.getcwd()}/cache/ccache" + print(f" i Enabling ccache on {ccache_dir} ... ") # some versions of LLVM use CCACHE_MAX_SIZE, others use CCACHE_SIZE cmake_command.extend( ["-DLLVM_CCACHE_BUILD=ON", - "-DLLVM_CCACHE_SIZE=10G", - "-DLLVM_CCACHE_DIR=/cache", - "-DLLVM_CCACHE_MAXSIZE=10G"]) + "-DLLVM_CCACHE_SIZE=5G", + f'-DLLVM_CCACHE_DIR="{ccache_dir}"', + "-DLLVM_CCACHE_MAXSIZE=5G"]) if use_libcxx: if int(properties["llvm_version"]) < 371: @@ -662,6 +687,9 @@ def common_installer_z3(properties): print(" x Failed to create the build folder") return False + if properties["ccache"]: + set_ccache_compiler() + cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator() cmake_command += ["-DZ3_BUILD_LIBZ3_SHARED=False", "-DZ3_ENABLE_EXAMPLE_TARGETS=False", diff --git a/pkgman/installers/unix.py b/pkgman/installers/unix.py index 94e771e8..fb52aaff 100644 --- a/pkgman/installers/unix.py +++ b/pkgman/installers/unix.py @@ -53,6 +53,11 @@ def unix_installer_boost(properties, default_toolset): if "CMAKE_CXX_COMPILER" in os.environ: os.environ["CXX"] = os.environ["CMAKE_CXX_COMPILER"] + if properties["ccache"]: + set_ccache_compiler() + os.environ["CXX"] = f"ccache {os.environ['CXX']}" + os.environ["CC"] = f"ccache {os.environ['CC']}" + configure_command = [source_folder + "/bootstrap.sh", "--prefix=" + os.path.join(repository_path, "boost"), "--with-toolset=" + toolset_name] if not run_program("Running the bootstrap script...", configure_command, source_folder, verbose=verbose_output): return False @@ -118,7 +123,12 @@ def unix_installer_cmake(properties): if os.environ.get("CMAKE_CXX_COMPILER") is not None: os.environ["CXX"] = os.environ["CMAKE_CXX_COMPILER"] - if not run_program("Running the bootstrap script...", ["./bootstrap", "--verbose", "--parallel=" + str(multiprocessing.cpu_count()), "--prefix=" + destination_path], source_folder, verbose=verbose_output): + enable_ccache = None + if properties["ccache"]: + set_ccache_compiler() + enable_ccache = "--enable-ccache" + + if not run_program("Running the bootstrap script...", ["./bootstrap", "--verbose", "--parallel=" + str(multiprocessing.cpu_count()), "--prefix=" + destination_path] + ([enable_ccache] if enable_ccache else []), source_folder, verbose=verbose_output): return False if not run_program("Building the source code...", ["make", "-j" + str(multiprocessing.cpu_count())], source_folder, verbose=verbose_output): diff --git a/pkgman/installers/utils.py b/pkgman/installers/utils.py index 372197e4..f37f4ed9 100644 --- a/pkgman/installers/utils.py +++ b/pkgman/installers/utils.py @@ -32,8 +32,27 @@ def get_env_compiler_settings(): if os.environ.get("CMAKE_C_COMPILER") is not None: cmake_compiler_settings.append("-DCMAKE_C_COMPILER=" + os.environ["CMAKE_C_COMPILER"]) + if os.environ.get("CMAKE_CXX_COMPILER_LAUNCHER") is not None: + cmake_compiler_settings.append("-DCMAKE_CXX_COMPILER_LAUNCHER=" + os.environ["CMAKE_CXX_COMPILER_LAUNCHER"]) + + if os.environ.get("CMAKE_C_COMPILER_LAUNCHER") is not None: + cmake_compiler_settings.append("-DCMAKE_C_COMPILER_LAUNCHER=" + os.environ["CMAKE_C_COMPILER_LAUNCHER"]) + return cmake_compiler_settings + +def set_ccache_compiler(): + """ + Set the compiler environment variables to use ccache. + + NOTE: LLVM uses its own ccache settings. Make sure these are synchronized + """ + os.environ["CMAKE_CXX_COMPILER_LAUNCHER"] = "ccache" + os.environ["CMAKE_C_COMPILER_LAUNCHER"] = "ccache" + # Only works if we never change directories in the script(s) + os.environ["CCACHE_DIR"] = f'"{os.getcwd()}/cache/ccache"' + + def get_parallel_build_options(): processor_count = str(multiprocessing.cpu_count()) diff --git a/travis.sh b/travis.sh index ced4fa68..5b9c1c64 100755 --- a/travis.sh +++ b/travis.sh @@ -89,7 +89,7 @@ osx_initialize() { xcode-select --install 2>&1 > /dev/null printf " > Installing the required packages...\n" - brew install coreutils + brew install coreutils ccache if [ $? -ne 0 ] ; then printf " x Could not install the required dependencies\n" return 1 @@ -126,7 +126,7 @@ linux_build() { printf " > Launching the build script for CMake...\n" printf "\n===\n" - python3 pkgman.py --c_compiler=$(which clang) --cxx_compiler=$(which clang++) --verbose "--repository_path=${bootstrap_repository}" "--packages=cmake" + python3 pkgman.py --use_ccache --c_compiler=$(which clang) --cxx_compiler=$(which clang++) --verbose "--repository_path=${bootstrap_repository}" "--packages=cmake" local pkgman_error=$? printf "===\n\n" @@ -138,7 +138,7 @@ linux_build() { printf " > Launching the build script for LLVM...\n" printf "\n===\n" - python3 pkgman.py --c_compiler=$(which clang) --cxx_compiler=$(which clang++) --exclude_libcxx --verbose "--additional_paths=${bootstrap_repository}/cmake/bin" "--repository_path=${bootstrap_repository}" "--packages=llvm" + python3 pkgman.py --use_ccache --c_compiler=$(which clang) --cxx_compiler=$(which clang++) --exclude_libcxx --verbose "--additional_paths=${bootstrap_repository}/cmake/bin" "--repository_path=${bootstrap_repository}" "--packages=llvm" local pkgman_error=$? printf "===\n\n" @@ -181,7 +181,7 @@ linux_build() { printf " > Re-launching the build script using the newly built clang...\n" printf "\n===\n" - python3 pkgman.py "--cxx_compiler=${bootstrap_repository}/llvm/bin/clang++" "--c_compiler=${bootstrap_repository}/llvm/bin/clang" --verbose "--additional_paths=${bootstrap_repository}/cmake/bin:${bootstrap_repository}/llvm/bin:${custom_bin_path}" "--repository_path=${library_repository}" "--packages=cmake,capstone,google,xed,capnproto" + python3 pkgman.py --use_ccache "--cxx_compiler=${bootstrap_repository}/llvm/bin/clang++" "--c_compiler=${bootstrap_repository}/llvm/bin/clang" --verbose "--additional_paths=${bootstrap_repository}/cmake/bin:${bootstrap_repository}/llvm/bin:${custom_bin_path}" "--repository_path=${library_repository}" "--packages=cmake,capstone,google,xed,capnproto" local pkgman_error=$? printf "===\n\n" @@ -220,7 +220,7 @@ osx_build() { printf " > Launching the build script for CMake...\n" printf "\n===\n" - python3 pkgman.py --verbose "--repository_path=${bootstrap_repository}" "--packages=cmake" + python3 pkgman.py --verbose --use_ccache "--repository_path=${bootstrap_repository}" "--packages=cmake" local pkgman_error=$? printf "===\n\n" @@ -232,7 +232,7 @@ osx_build() { printf " > Launching the build script for LLVM...\n" printf "\n===\n" - python3 pkgman.py --verbose "--additional_paths=${bootstrap_repository}/cmake/bin" "--repository_path=${library_repository}" "--packages=llvm" + python3 pkgman.py --verbose --use_ccache "--additional_paths=${bootstrap_repository}/cmake/bin" "--use_ccache" "--repository_path=${library_repository}" "--packages=llvm" local pkgman_error=$? printf "===\n\n" @@ -275,7 +275,7 @@ osx_build() { printf " > Re-launching the build script using the newly built clang...\n" printf "\n===\n" - python3 pkgman.py "--cxx_compiler=${library_repository}/llvm/bin/clang++" "--c_compiler=${library_repository}/llvm/bin/clang" --verbose "--additional_paths=${bootstrap_repository}/cmake/bin:${library_repository}/llvm/bin:${custom_bin_path}" "--repository_path=${library_repository}" "--packages=cmake,capstone,google,xed,capnproto" + python3 pkgman.py --use_ccache "--cxx_compiler=${library_repository}/llvm/bin/clang++" "--c_compiler=${library_repository}/llvm/bin/clang" --verbose "--additional_paths=${bootstrap_repository}/cmake/bin:${library_repository}/llvm/bin:${custom_bin_path}" "--repository_path=${library_repository}" "--packages=cmake,capstone,google,xed,capnproto" local pkgman_error=$? printf "===\n\n"