ClickHouse · abyss7 · Oct 30, 2019 · Oct 15, 2019 · Oct 23, 2019 · Oct 25, 2019
diff --git a/.gitignore b/.gitignore
@@ -245,3 +245,6 @@ website/package-lock.json
 /.ccls-cache
 
 /compile_commands.json
+
+# Toolchains
+/cmake/toolchain/*
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -13,7 +13,10 @@ foreach(policy
 endforeach()
 
 project(ClickHouse)
+
+include (cmake/arch.cmake)
 include (cmake/target.cmake)
+include (cmake/tools.cmake)
 
 # Ignore export() since we don't use it,
 # but it gets broken with a global targets via link_libraries()
@@ -26,8 +29,6 @@ set(CMAKE_LINK_DEPENDS_NO_SHARED 1) # Do not relink all depended targets on .so
 set(CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Debug;Release;MinSizeRel" CACHE STRING "" FORCE)
 set(CMAKE_DEBUG_POSTFIX "d" CACHE STRING "Generate debug library name with a postfix.")    # To be consistent with CMakeLists from contrib libs.
 
-include (cmake/arch.cmake)
-
 option(ENABLE_IPO "Enable inter-procedural optimization (aka LTO)" OFF) # need cmake 3.9+
 if(ENABLE_IPO)
     cmake_policy(SET CMP0069 NEW)
@@ -230,7 +231,6 @@ include(cmake/dbms_glob_sources.cmake)
 if (OS_LINUX)
     include(cmake/linux/default_libs.cmake)
 elseif (OS_DARWIN)
-    include(cmake/darwin/sdk.cmake)
     include(cmake/darwin/default_libs.cmake)
 endif ()
 

diff --git a/cmake/arch.cmake b/cmake/arch.cmake
@@ -17,6 +17,7 @@ endif ()
 
 if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64le.*|PPC64LE.*)")
     set (ARCH_PPC64LE 1)
+    # FIXME: move this check into tools.cmake
     if (COMPILER_CLANG OR (COMPILER_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8))
         message(FATAL_ERROR "Only gcc-8 is supported for powerpc architecture")
     endif ()

diff --git a/cmake/darwin/default_libs.cmake b/cmake/darwin/default_libs.cmake
@@ -11,6 +11,14 @@ message(STATUS "Default libraries: ${DEFAULT_LIBS}")
 set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS})
 set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
 
+# Minimal supported SDK version
+
+set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mmacosx-version-min=10.14")
+set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mmacosx-version-min=10.14")
+
+set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -mmacosx-version-min=10.14")
+set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -mmacosx-version-min=10.14")
+
 # Global libraries
 
 add_library(global-libs INTERFACE)

diff --git a/cmake/darwin/sdk.cmake b/cmake/darwin/sdk.cmake
diff --git a/cmake/darwin/toolchain-x86_64.cmake b/cmake/darwin/toolchain-x86_64.cmake
@@ -0,0 +1,13 @@
+set (CMAKE_SYSTEM_NAME "Darwin")
+set (CMAKE_SYSTEM_PROCESSOR "x86_64")
+set (CMAKE_C_COMPILER_TARGET "x86_64-apple-darwin")
+set (CMAKE_CXX_COMPILER_TARGET "x86_64-apple-darwin")
+set (CMAKE_OSX_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/darwin-x86_64")
+
+set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)  # disable linkage check - it doesn't work in CMake
+
+set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
+
+set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
diff --git a/cmake/linux/default_libs.cmake b/cmake/linux/default_libs.cmake
@@ -5,7 +5,7 @@ set (DEFAULT_LIBS "-nodefaultlibs")
 
 # We need builtins from Clang's RT even without libcxx - for ubsan+int128.
 # See https://bugs.llvm.org/show_bug.cgi?id=16404
-if (COMPILER_CLANG)
+if (COMPILER_CLANG AND NOT (CMAKE_CROSSCOMPILING AND ARCH_AARCH64))
     execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
 else ()
     set (BUILTINS_LIBRARY "-lgcc")

diff --git a/cmake/linux/toolchain-aarch64.cmake b/cmake/linux/toolchain-aarch64.cmake
@@ -0,0 +1,20 @@
+set (CMAKE_SYSTEM_NAME "Linux")
+set (CMAKE_SYSTEM_PROCESSOR "aarch64")
+set (CMAKE_C_COMPILER_TARGET "aarch64-linux-gnu")
+set (CMAKE_CXX_COMPILER_TARGET "aarch64-linux-gnu")
+set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64/aarch64-linux-gnu/libc")
+
+# We don't use compiler from toolchain because it's gcc-8, and we provide support only for gcc-9.
+set (CMAKE_AR "${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64/bin/aarch64-linux-gnu-ar")
+
+set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64")
+set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64")
+
+set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
+set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
+
+set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
+
+set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
diff --git a/cmake/target.cmake b/cmake/target.cmake
@@ -9,62 +9,8 @@ elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin")
     add_definitions(-D OS_DARWIN)
 endif ()
 
-if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-    set (COMPILER_GCC 1)
-elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-    set (COMPILER_CLANG 1)
-endif ()
-
-if (COMPILER_GCC)
-    # Require minimum version of gcc
-    set (GCC_MINIMUM_VERSION 8)
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${GCC_MINIMUM_VERSION} AND NOT CMAKE_VERSION VERSION_LESS 2.8.9)
-        message (FATAL_ERROR "GCC version must be at least ${GCC_MINIMUM_VERSION}. For example, if GCC ${GCC_MINIMUM_VERSION} is available under gcc-${GCC_MINIMUM_VERSION}, g++-${GCC_MINIMUM_VERSION} names, do the following: export CC=gcc-${GCC_MINIMUM_VERSION} CXX=g++-${GCC_MINIMUM_VERSION}; rm -rf CMakeCache.txt CMakeFiles; and re run cmake or ./release.")
-    endif ()
-elseif (COMPILER_CLANG)
-    # Require minimum version of clang
-    set (CLANG_MINIMUM_VERSION 7)
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
-        message (FATAL_ERROR "Clang version must be at least ${CLANG_MINIMUM_VERSION}.")
-    endif ()
-else ()
-    message (WARNING "You are using an unsupported compiler. Compilation has only been tested with Clang 6+ and GCC 7+.")
-endif ()
-
-string(REGEX MATCH "-?[0-9]+(.[0-9]+)?$" COMPILER_POSTFIX ${CMAKE_CXX_COMPILER})
-
-if (OS_LINUX)
-    find_program (LLD_PATH NAMES "lld${COMPILER_POSTFIX}" "lld")
-    find_program (GOLD_PATH NAMES "ld.gold" "gold")
-endif()
-
-option (LINKER_NAME "Linker name or full path")
-if (NOT LINKER_NAME)
-    if (COMPILER_CLANG AND LLD_PATH)
-        set (LINKER_NAME "lld")
-    elseif (GOLD_PATH)
-        set (LINKER_NAME "gold")
-    endif ()
-endif ()
-
-if (LINKER_NAME)
-    message(STATUS "Using linker: ${LINKER_NAME} (selected from: LLD_PATH=${LLD_PATH}; GOLD_PATH=${GOLD_PATH}; COMPILER_POSTFIX=${COMPILER_POSTFIX})")
-    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
-    set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
-endif ()
-
 if (CMAKE_CROSSCOMPILING)
     if (OS_DARWIN)
-        set (CMAKE_SYSTEM_PROCESSOR x86_64)
-        set (CMAKE_C_COMPILER_TARGET x86_64-apple-darwin)
-        set (CMAKE_CXX_COMPILER_TARGET x86_64-apple-darwin)
-
-        set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
-        set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
-
-        set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
-        set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
-
         # FIXME: broken dependencies
         set (USE_SNAPPY OFF CACHE INTERNAL "")
         set (ENABLE_SSL OFF CACHE INTERNAL "")
@@ -73,12 +19,19 @@ if (CMAKE_CROSSCOMPILING)
         set (ENABLE_READLINE OFF CACHE INTERNAL "")
         set (ENABLE_ICU OFF CACHE INTERNAL "")
         set (ENABLE_FASTOPS OFF CACHE INTERNAL "")
-
-        message (STATUS "Cross-compiling for Darwin")
+    elseif (OS_LINUX)
+        if (ARCH_AARCH64)
+            # FIXME: broken dependencies
+            set (ENABLE_PROTOBUF OFF CACHE INTERNAL "")
+            set (ENABLE_PARQUET OFF CACHE INTERNAL "")
+            set (ENABLE_MYSQL OFF CACHE INTERNAL "")
+        endif ()
     else ()
-        message (FATAL_ERROR "Trying to cross-compile to unsupported target: ${CMAKE_SYSTEM_NAME}!")
+        message (FATAL_ERROR "Trying to cross-compile to unsupported system: ${CMAKE_SYSTEM_NAME}!")
     endif ()
 
     # Don't know why but CXX_STANDARD doesn't work for cross-compilation
     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
+
+    message (STATUS "Cross-compiling for target: ${CMAKE_CXX_COMPILE_TARGET}")
 endif ()
diff --git a/cmake/toolchain/darwin-x86_64/README.txt b/cmake/toolchain/darwin-x86_64/README.txt
@@ -0,0 +1,2 @@
+wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz
+tar --strip-components=1 xJf MacOSX10.14.sdk.tar.xz
diff --git a/cmake/toolchain/linux-aarch64/README.txt b/cmake/toolchain/linux-aarch64/README.txt
@@ -0,0 +1,2 @@
+wget https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en -O gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz
+tar --strip-components=1 xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz
diff --git a/cmake/tools.cmake b/cmake/tools.cmake
@@ -0,0 +1,41 @@
+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    set (COMPILER_GCC 1)
+elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+    set (COMPILER_CLANG 1)
+endif ()
+
+if (COMPILER_GCC)
+    # Require minimum version of gcc
+    set (GCC_MINIMUM_VERSION 8)
+    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${GCC_MINIMUM_VERSION} AND NOT CMAKE_VERSION VERSION_LESS 2.8.9)
+        message (FATAL_ERROR "GCC version must be at least ${GCC_MINIMUM_VERSION}. For example, if GCC ${GCC_MINIMUM_VERSION} is available under gcc-${GCC_MINIMUM_VERSION}, g++-${GCC_MINIMUM_VERSION} names, do the following: export CC=gcc-${GCC_MINIMUM_VERSION} CXX=g++-${GCC_MINIMUM_VERSION}; rm -rf CMakeCache.txt CMakeFiles; and re run cmake or ./release.")
+    endif ()
+elseif (COMPILER_CLANG)
+    # Require minimum version of clang
+    set (CLANG_MINIMUM_VERSION 7)
+    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
+        message (FATAL_ERROR "Clang version must be at least ${CLANG_MINIMUM_VERSION}.")
+    endif ()
+else ()
+    message (WARNING "You are using an unsupported compiler. Compilation has only been tested with Clang 6+ and GCC 7+.")
+endif ()
+
+option (LINKER_NAME "Linker name or full path")
+
+find_program (LLD_PATH NAMES "ld.lld" "lld")
+find_program (GOLD_PATH NAMES "ld.gold" "gold")
+
+if (NOT LINKER_NAME)
+    if (LLD_PATH)
+        set (LINKER_NAME "lld")
+    elseif (GOLD_PATH)
+        set (LINKER_NAME "gold")
+    endif ()
+endif ()
+
+if (LINKER_NAME)
+    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
+    set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
+
+    message(STATUS "Using custom linker by name: ${LINKER_NAME}")
+endif ()
diff --git a/dbms/src/Common/StackTrace.cpp b/dbms/src/Common/StackTrace.cpp
@@ -30,7 +30,7 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext
             else
                 error << "Address: " << info.si_addr;
 
-#if defined(__x86_64__) && !defined(__FreeBSD__) && !defined(__APPLE__)
+#if defined(__x86_64__) && !defined(__FreeBSD__) && !defined(__APPLE__) && !defined(__arm__)
             auto err_mask = context.uc_mcontext.gregs[REG_ERR];
             if ((err_mask & 0x02))
                 error << " Access: write.";

diff --git a/dbms/src/Common/tests/int_hashes_perf.cpp b/dbms/src/Common/tests/int_hashes_perf.cpp
@@ -31,7 +31,7 @@ void setAffinity()
 
 static inline ALWAYS_INLINE UInt64 rdtsc()
 {
-#if __x86_64__
+#if defined(__x86_64__)
     UInt32 a, d;
     __asm__ volatile ("rdtsc" : "=a" (a), "=d" (d));
     return static_cast<UInt64>(a) | (static_cast<UInt64>(d) << 32);
@@ -109,7 +109,7 @@ static inline size_t murmurMix(UInt64 x)
 }
 
 
-#if __x86_64__
+#if defined(__x86_64__)
 static inline size_t crc32Hash(UInt64 x)
 {
     UInt64 crc = -1ULL;
@@ -309,7 +309,7 @@ int main(int argc, char ** argv)
     if (!method || method == 8) test<mulShift>  (n, data.data(), "7: mulShift");
     if (!method || method == 9) test<tabulation>(n, data.data(), "8: tabulation");
 
-#if __x86_64__
+#if defined(__x86_64__)
     if (!method || method == 10) test<crc32Hash> (n, data.data(), "9: crc32");
 #endif
 

diff --git a/dbms/src/IO/tests/parse_int_perf.cpp b/dbms/src/IO/tests/parse_int_perf.cpp
@@ -17,7 +17,7 @@
 
 UInt64 rdtsc()
 {
-#if __x86_64__
+#if defined(__x86_64__)
     UInt64 val;
     __asm__ __volatile__("rdtsc" : "=A" (val) :);
     return val;

diff --git a/dbms/src/Interpreters/tests/hash_map_string.cpp b/dbms/src/Interpreters/tests/hash_map_string.cpp
@@ -131,7 +131,7 @@ struct FastHash64
 };
 
 
-#if __x86_64__
+#if defined(__x86_64__)
 struct CrapWow
 {
     size_t operator() (CompactStringRef x) const
@@ -381,7 +381,7 @@ int main(int argc, char ** argv)
             << std::endl;
     }
 
-#if __x86_64__
+#if defined(__x86_64__)
     if (!m || m == 3)
     {
         Stopwatch watch;

diff --git a/docker/packager/packager b/docker/packager/packager
@@ -107,6 +107,14 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, cache, di
     result = []
     cmake_flags = ['$CMAKE_FLAGS', '-DADD_GDB_INDEX_FOR_GOLD=1']
 
+    is_clang = "clang" in compiler
+    is_cross_compile = ("darwin" in compiler) or ("aarch64" in compiler)
+
+    # Explicitly use LLD with Clang by default.
+    # Don't force linker for cross-compilation.
+    if is_clang and not is_cross_compile:
+        cmake_flags = ['$CMAKE_FLAGS', '-DLINKER_NAME=lld']
+
     if compiler.endswith("-darwin"):
         cc = compiler[:-len("-darwin")]
     else:
@@ -124,9 +132,11 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, cache, di
     if "darwin" in compiler:
         cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/x86_64-apple-darwin-ar")
         cmake_flags.append("-DCMAKE_RANLIB:FILEPATH=/cctools/bin/x86_64-apple-darwin-ranlib")
-        cmake_flags.append("-DCMAKE_SYSTEM_NAME=Darwin")
-        cmake_flags.append("-DSDK_PATH=/cctools/MacOSX10.14.sdk")
         cmake_flags.append("-DLINKER_NAME=/cctools/bin/x86_64-apple-darwin-ld")
+        cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-x86_64.cmake")
+
+    if "aarch64" in compiler:
+        cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake")
 
     if sanitizer:
         result.append("SANITIZER={}".format(sanitizer))
@@ -181,7 +191,7 @@ if __name__ == "__main__":
     parser.add_argument("--clickhouse-repo-path", default="../../")
     parser.add_argument("--output-dir", required=True)
     parser.add_argument("--build-type", choices=("debug", ""), default="")
-    parser.add_argument("--compiler", choices=("clang-6.0", "clang-7", "gcc-7", "clang-8", "clang-8-darwin", "gcc-8", "gcc-9"), default="gcc-7")
+    parser.add_argument("--compiler", choices=("clang-6.0", "clang-7", "gcc-7", "clang-8", "clang-8-darwin", "clang-8-aarch64", "gcc-8", "gcc-9"), default="gcc-7")
     parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="")
     parser.add_argument("--unbundled", action="store_true")
     parser.add_argument("--split-binary", action="store_true")