Merge pull request #30011 from ClickHouse/hermetic-builds

Hermetic builds

PreLoad.cmake

@@ -18,3 +18,27 @@ if (NOT DEFINED ENV{CLION_IDE} AND NOT DEFINED ENV{XCODE_IDE})
         set(CMAKE_GENERATOR "Ninja" CACHE INTERNAL "" FORCE)
     endif ()
 endif()
+
+
+# Default toolchain - this is needed to avoid dependency on OS files.
+execute_process(COMMAND uname -s OUTPUT_VARIABLE OS)
+execute_process(COMMAND uname -m OUTPUT_VARIABLE ARCH)
+
+if (OS MATCHES "Linux"
+    AND NOT DEFINED CMAKE_TOOLCHAIN_FILE
+    AND NOT UNBUNDLED
+    AND NOT DISABLE_HERMETIC_BUILD
+    AND ($ENV{CC} MATCHES ".*clang.*" OR CMAKE_C_COMPILER MATCHES ".*clang.*")
+    AND (USE_STATIC_LIBRARIES OR NOT DEFINED USE_STATIC_LIBRARIES))
+
+    if (ARCH MATCHES "amd64|x86_64")
+        set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-x86_64.cmake" CACHE INTERNAL "" FORCE)
+    elseif (ARCH MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)")
+        set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-aarch64.cmake" CACHE INTERNAL "" FORCE)
+    elseif (ARCH MATCHES "^(ppc64le.*|PPC64LE.*)")
+        set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-ppc64le.cmake" CACHE INTERNAL "" FORCE)
+    else ()
+        message (FATAL_ERROR "Unsupported architecture: ${ARCH}")
+    endif ()
+
+endif()

cmake/add_warning.cmake

@@ -12,13 +12,13 @@ macro (add_warning flag)
     if (SUPPORTS_CXXFLAG_${underscored_flag})
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W${flag}")
     else ()
-        message (WARNING "Flag -W${flag} is unsupported")
+        message (STATUS "Flag -W${flag} is unsupported")
     endif ()
 
     if (SUPPORTS_CFLAG_${underscored_flag})
         set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -W${flag}")
     else ()
-        message (WARNING "Flag -W${flag} is unsupported")
+        message (STATUS "Flag -W${flag} is unsupported")
     endif ()
 
 endmacro ()
@@ -39,7 +39,7 @@ macro (target_add_warning target flag)
     if (SUPPORTS_CXXFLAG_${underscored_flag})
         target_compile_options (${target} PRIVATE "-W${flag}")
     else ()
-        message (WARNING "Flag -W${flag} is unsupported")
+        message (STATUS "Flag -W${flag} is unsupported")
     endif ()
 endmacro ()
 

cmake/linux/default_libs.cmake

@@ -5,8 +5,12 @@ set (DEFAULT_LIBS "-nodefaultlibs")
 
 # We need builtins from Clang's RT even without libcxx - for ubsan+int128.
 # See https://bugs.llvm.org/show_bug.cgi?id=16404
-if (COMPILER_CLANG AND NOT CMAKE_CROSSCOMPILING)
-    execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-libgcc-file-name --rtlib=compiler-rt OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
+if (COMPILER_CLANG)
+    execute_process (COMMAND ${CMAKE_CXX_COMPILER} --target=${CMAKE_CXX_COMPILER_TARGET} --print-libgcc-file-name --rtlib=compiler-rt OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+    if (NOT EXISTS "${BUILTINS_LIBRARY}")
+        set (BUILTINS_LIBRARY "-lgcc")
+    endif ()
 else ()
     set (BUILTINS_LIBRARY "-lgcc")
 endif ()

contrib/krb5-cmake/CMakeLists.txt

@@ -500,7 +500,6 @@ function(preprocess_et out_var)
       COMMAND perl "${KRB5_SOURCE_DIR}/util/et/compile_et" -d "${KRB5_SOURCE_DIR}/util/et" ${in_f}
       DEPENDS ${in_f} "${KRB5_SOURCE_DIR}/util/et/compile_et"
       WORKING_DIRECTORY ${ET_PATH}
-      COMMENT "Creating preprocessed file ${F_C}"
       VERBATIM
       )
     list(APPEND result ${F_C})
@@ -526,7 +525,6 @@ add_custom_command(
 add_custom_target(
     ERROR_MAP_H
     DEPENDS "${KRB5_SOURCE_DIR}/lib/gssapi/krb5/error_map.h"
-    COMMENT "generating error_map.h"
     VERBATIM
 )
 
@@ -539,14 +537,12 @@ add_custom_command(
 add_custom_target(
     ERRMAP_H
     DEPENDS "${KRB5_SOURCE_DIR}/lib/gssapi/generic/errmap.h"
-    COMMENT "generating errmap.h"
     VERBATIM
 )
 
 add_custom_target(
     KRB_5_H
     DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/include/krb5/krb5.h"
-    COMMENT "generating krb5.h"
     VERBATIM
 )
 

contrib/protobuf-cmake/CMakeLists.txt

@@ -211,7 +211,9 @@ add_library(protobuf::libprotoc ALIAS libprotoc)
 
 set(protoc_files ${protobuf_source_dir}/src/google/protobuf/compiler/main.cc)
 
-if (NOT CMAKE_CROSSCOMPILING)
+if (CMAKE_HOST_SYSTEM_NAME STREQUAL CMAKE_SYSTEM_NAME
+    AND CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL CMAKE_SYSTEM_PROCESSOR)
+
     add_executable(protoc ${protoc_files})
     target_link_libraries(protoc libprotoc libprotobuf pthread)
     add_executable(protobuf::protoc ALIAS protoc)

docker/test/fasttest/run.sh

@@ -159,6 +159,7 @@ function clone_submodules
         cd "$FASTTEST_SOURCE"
 
         SUBMODULES_TO_UPDATE=(
+            contrib/sysroot
             contrib/magic_enum
             contrib/abseil-cpp
             contrib/boost

docker/test/pvs/Dockerfile

@@ -40,7 +40,7 @@ RUN set -x \
 ENV CCACHE_DIR=/test_output/ccache
 
 CMD echo "Running PVS version $PKG_VERSION" && mkdir -p $CCACHE_DIR && cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic  \
-    && cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"USE_INTERNAL_PROTOBUF_LIBRARY"=OFF -D"USE_INTERNAL_GRPC_LIBRARY"=OFF -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang\+\+-13 \
+    && cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"DISABLE_HERMETIC_BUILD"=ON -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang\+\+-13 \
     && ninja re2_st clickhouse_grpc_protos \
     && pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j 4 -l ./licence.lic; \
     cp /repo_folder/pvs-studio.log /test_output; \

src/Common/renameat2.cpp

@@ -99,11 +99,16 @@ bool supportsRenameat2()
     return supports;
 }
 
+}
+
 #else
 
 #define RENAME_NOREPLACE -1
 #define RENAME_EXCHANGE -1
 
+namespace DB
+{
+
 static bool renameat2(const std::string &, const std::string &, int)
 {
     return false;
@@ -114,8 +119,13 @@ bool supportsRenameat2()
     return false;
 }
 
+}
+
 #endif
 
+namespace DB
+{
+
 static void renameNoReplaceFallback(const std::string & old_path, const std::string & new_path)
 {
     /// NOTE it's unsafe