Backport #46240 to 23.1: Use sccache and clang-tidy-cache with S3 cache backend

CMakeLists.txt

@@ -581,7 +581,7 @@ if (NATIVE_BUILD_TARGETS
         COMMAND ${CMAKE_COMMAND}
             "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
             "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
-            "-DENABLE_CCACHE=${ENABLE_CCACHE}"
+            "-DCOMPILER_CACHE=${COMPILER_CACHE}"
             # Avoid overriding .cargo/config.toml with native toolchain.
             "-DENABLE_RUST=OFF"
             "-DENABLE_CLICKHOUSE_SELF_EXTRACTING=${ENABLE_CLICKHOUSE_SELF_EXTRACTING}"

cmake/ccache.cmake

@@ -1,5 +1,6 @@
 # Setup integration with ccache to speed up builds, see https://ccache.dev/
 
+# Matches both ccache and sccache
 if (CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" OR CMAKE_C_COMPILER_LAUNCHER MATCHES "ccache")
     # custom compiler launcher already defined, most likely because cmake was invoked with like "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache" or
     # via environment variable --> respect setting and trust that the launcher was specified correctly
@@ -8,45 +9,65 @@ if (CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" OR CMAKE_C_COMPILER_LAUNCHER MA
     return()
 endif()
 
-option(ENABLE_CCACHE "Speedup re-compilations using ccache (external tool)" ON)
+set(ENABLE_CCACHE "default" CACHE STRING "Deprecated, use COMPILER_CACHE=(auto|ccache|sccache|disabled)")
+if (NOT ENABLE_CCACHE STREQUAL "default")
+    message(WARNING "The -DENABLE_CCACHE is deprecated in favor of -DCOMPILER_CACHE")
+endif()
+
+set(COMPILER_CACHE "auto" CACHE STRING "Speedup re-compilations using the caching tools; valid options are 'auto' (ccache, then sccache), 'ccache', 'sccache', or 'disabled'")
 
-if (NOT ENABLE_CCACHE)
-    message(STATUS "Using ccache: no (disabled via configuration)")
+# It has pretty complex logic, because the ENABLE_CCACHE is deprecated, but still should
+# control the COMPILER_CACHE
+# After it will be completely removed, the following block will be much simpler
+if (COMPILER_CACHE STREQUAL "ccache" OR (ENABLE_CCACHE AND NOT ENABLE_CCACHE STREQUAL "default"))
+    find_program (CCACHE_EXECUTABLE ccache)
+elseif(COMPILER_CACHE STREQUAL "disabled" OR NOT ENABLE_CCACHE STREQUAL "default")
+    message(STATUS "Using *ccache: no (disabled via configuration)")
     return()
+elseif(COMPILER_CACHE STREQUAL "auto")
+    find_program (CCACHE_EXECUTABLE ccache sccache)
+elseif(COMPILER_CACHE STREQUAL "sccache")
+    find_program (CCACHE_EXECUTABLE sccache)
+else()
+    message(${RECONFIGURE_MESSAGE_LEVEL} "The COMPILER_CACHE must be one of (auto|ccache|sccache|disabled), given '${COMPILER_CACHE}'")
 endif()
 
-find_program (CCACHE_EXECUTABLE ccache)
 
 if (NOT CCACHE_EXECUTABLE)
-    message(${RECONFIGURE_MESSAGE_LEVEL} "Using ccache: no (Could not find find ccache. To significantly reduce compile times for the 2nd, 3rd, etc. build, it is highly recommended to install ccache. To suppress this message, run cmake with -DENABLE_CCACHE=0)")
+    message(${RECONFIGURE_MESSAGE_LEVEL} "Using *ccache: no (Could not find find ccache or sccache. To significantly reduce compile times for the 2nd, 3rd, etc. build, it is highly recommended to install one of them. To suppress this message, run cmake with -DCOMPILER_CACHE=disabled)")
     return()
 endif()
 
-execute_process(COMMAND ${CCACHE_EXECUTABLE} "-V" OUTPUT_VARIABLE CCACHE_VERSION)
-string(REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION})
+if (CCACHE_EXECUTABLE MATCHES "/ccache$")
+    execute_process(COMMAND ${CCACHE_EXECUTABLE} "-V" OUTPUT_VARIABLE CCACHE_VERSION)
+    string(REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION})
 
-set (CCACHE_MINIMUM_VERSION 3.3)
+    set (CCACHE_MINIMUM_VERSION 3.3)
 
-if (CCACHE_VERSION VERSION_LESS_EQUAL ${CCACHE_MINIMUM_VERSION})
-    message(${RECONFIGURE_MESSAGE_LEVEL} "Using ccache: no (found ${CCACHE_EXECUTABLE} (version ${CCACHE_VERSION}), the minimum required version is ${CCACHE_MINIMUM_VERSION}")
-    return()
-endif()
+    if (CCACHE_VERSION VERSION_LESS_EQUAL ${CCACHE_MINIMUM_VERSION})
+        message(${RECONFIGURE_MESSAGE_LEVEL} "Using ccache: no (found ${CCACHE_EXECUTABLE} (version ${CCACHE_VERSION}), the minimum required version is ${CCACHE_MINIMUM_VERSION}")
+        return()
+    endif()
+
+    message(STATUS "Using ccache: ${CCACHE_EXECUTABLE} (version ${CCACHE_VERSION})")
+    set(LAUNCHER ${CCACHE_EXECUTABLE})
 
-message(STATUS "Using ccache: ${CCACHE_EXECUTABLE} (version ${CCACHE_VERSION})")
-set(LAUNCHER ${CCACHE_EXECUTABLE})
-
-# Work around a well-intended but unfortunate behavior of ccache 4.0 & 4.1 with
-# environment variable SOURCE_DATE_EPOCH. This variable provides an alternative
-# to source-code embedded timestamps (__DATE__/__TIME__) and therefore helps with
-# reproducible builds (*). SOURCE_DATE_EPOCH is set automatically by the
-# distribution, e.g. Debian. Ccache 4.0 & 4.1 incorporate SOURCE_DATE_EPOCH into
-# the hash calculation regardless they contain timestamps or not. This invalidates
-# the cache whenever SOURCE_DATE_EPOCH changes. As a fix, ignore SOURCE_DATE_EPOCH.
-#
-# (*) https://reproducible-builds.org/specs/source-date-epoch/
-if (CCACHE_VERSION VERSION_GREATER_EQUAL "4.0" AND CCACHE_VERSION VERSION_LESS "4.2")
-    message(STATUS "Ignore SOURCE_DATE_EPOCH for ccache 4.0 / 4.1")
-    set(LAUNCHER env -u SOURCE_DATE_EPOCH ${CCACHE_EXECUTABLE})
+    # Work around a well-intended but unfortunate behavior of ccache 4.0 & 4.1 with
+    # environment variable SOURCE_DATE_EPOCH. This variable provides an alternative
+    # to source-code embedded timestamps (__DATE__/__TIME__) and therefore helps with
+    # reproducible builds (*). SOURCE_DATE_EPOCH is set automatically by the
+    # distribution, e.g. Debian. Ccache 4.0 & 4.1 incorporate SOURCE_DATE_EPOCH into
+    # the hash calculation regardless they contain timestamps or not. This invalidates
+    # the cache whenever SOURCE_DATE_EPOCH changes. As a fix, ignore SOURCE_DATE_EPOCH.
+    #
+    # (*) https://reproducible-builds.org/specs/source-date-epoch/
+    if (CCACHE_VERSION VERSION_GREATER_EQUAL "4.0" AND CCACHE_VERSION VERSION_LESS "4.2")
+        message(STATUS "Ignore SOURCE_DATE_EPOCH for ccache 4.0 / 4.1")
+        set(LAUNCHER env -u SOURCE_DATE_EPOCH ${CCACHE_EXECUTABLE})
+    endif()
+elseif(CCACHE_EXECUTABLE MATCHES "/sccache$")
+    message(STATUS "Using sccache: ${CCACHE_EXECUTABLE}")
+    set(LAUNCHER ${CCACHE_EXECUTABLE})
 endif()
 
 set (CMAKE_CXX_COMPILER_LAUNCHER ${LAUNCHER} ${CMAKE_CXX_COMPILER_LAUNCHER})

docker/packager/binary/Dockerfile

@@ -23,8 +23,10 @@ RUN apt-get update && \
         libc6 \
         libc6-dev \
         libc6-dev-arm64-cross \
-        zstd && \
-    apt-get clean
+        python3-boto3 \
+        zstd \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists
 
 ENV CC=clang-${LLVM_VERSION}
 ENV CXX=clang++-${LLVM_VERSION}
@@ -56,7 +58,7 @@ RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \
     && rm -rf cctools-port
 
 # Download toolchain and SDK for Darwin
-RUN wget -nv https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.0.sdk.tar.xz
+RUN curl -sL -O https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.0.sdk.tar.xz
 
 # NOTE: Seems like gcc-11 is too new for ubuntu20 repository
 RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
@@ -87,8 +89,10 @@ ENV PATH="$PATH:/usr/local/go/bin"
 ENV GOPATH=/workdir/go
 ENV GOCACHE=/workdir/
 
-RUN curl https://raw.githubusercontent.com/matus-chochlik/ctcache/7fd516e91c17779cbc6fc18bd119313d9532dd90/clang-tidy-cache -Lo /usr/bin/clang-tidy-cache \
-  && chmod +x /usr/bin/clang-tidy-cache
+ARG CLANG_TIDY_SHA1=c191254ea00d47ade11d7170ef82fe038c213774
+RUN curl -Lo /usr/bin/clang-tidy-cache \
+        "https://raw.githubusercontent.com/matus-chochlik/ctcache/$CLANG_TIDY_SHA1/clang-tidy-cache" \
+    && chmod +x /usr/bin/clang-tidy-cache
 
 RUN mkdir /workdir && chmod 777 /workdir
 WORKDIR /workdir

docker/packager/binary/build.sh

@@ -6,6 +6,7 @@ exec &> >(ts)
 ccache_status () {
     ccache --show-config ||:
     ccache --show-stats ||:
+    SCCACHE_NO_DAEMON=1 sccache --show-stats ||:
 }
 
 [ -O /build ] || git config --global --add safe.directory /build